88 		".pushsection __rseq_cs, \"aw\"\n\t"			\
89 		".balign 32\n\t"					\
90 		__rseq_str(label) ":\n\t"					\
91 		".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
92 …str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
93 		".popsection\n\t"					\
94 		".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"		\
95 		".word " __rseq_str(label) "b, 0x0\n\t"			\
96 		".popsection\n\t"
111 		".pushsection __rseq_exit_point_array, \"aw\"\n\t"	\
112 		".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
113 		".popsection\n\t"
117 		"adr r0, " __rseq_str(cs_label) "\n\t"			\
118 		"str r0, %[" __rseq_str(rseq_cs) "]\n\t"		\
119 		__rseq_str(label) ":\n\t"
123 		"ldr r0, %[" __rseq_str(current_cpu_id) "]\n\t"	\
124 		"cmp %[" __rseq_str(cpu_id) "], r0\n\t"		\
125 		"bne " __rseq_str(label) "\n\t"
130 		".balign 32\n\t"					\
131 		__rseq_str(table_label) ":\n\t"				\
132 		".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
133 …str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
134 		".word " __rseq_str(RSEQ_SIG) "\n\t"			\
135 		__rseq_str(label) ":\n\t"				\
137 		"b %l[" __rseq_str(abort_label) "]\n\t"
146 		__rseq_str(label) ":\n\t"				\
148 		"b %l[" __rseq_str(cmpfail_label) "]\n\t"
169 		"ldr r0, %[v]\n\t"  in rseq_cmpeqv_storev()
170 		"cmp %[expect], r0\n\t"  in rseq_cmpeqv_storev()
171 		"bne %l[cmpfail]\n\t"  in rseq_cmpeqv_storev()
175 		"ldr r0, %[v]\n\t"  in rseq_cmpeqv_storev()
176 		"cmp %[expect], r0\n\t"  in rseq_cmpeqv_storev()
177 		"bne %l[error2]\n\t"  in rseq_cmpeqv_storev()
180 		"str %[newv], %[v]\n\t"  in rseq_cmpeqv_storev()
181 		"2:\n\t"  in rseq_cmpeqv_storev()
183 		"b 5f\n\t"  in rseq_cmpeqv_storev()
185 		"5:\n\t"  in rseq_cmpeqv_storev()
236 		"ldr r0, %[v]\n\t"  in rseq_cmpnev_storeoffp_load()
237 		"cmp %[expectnot], r0\n\t"  in rseq_cmpnev_storeoffp_load()
238 		"beq %l[cmpfail]\n\t"  in rseq_cmpnev_storeoffp_load()
242 		"ldr r0, %[v]\n\t"  in rseq_cmpnev_storeoffp_load()
243 		"cmp %[expectnot], r0\n\t"  in rseq_cmpnev_storeoffp_load()
244 		"beq %l[error2]\n\t"  in rseq_cmpnev_storeoffp_load()
246 		"str r0, %[load]\n\t"  in rseq_cmpnev_storeoffp_load()
247 		"add r0, %[voffp]\n\t"  in rseq_cmpnev_storeoffp_load()
248 		"ldr r0, [r0]\n\t"  in rseq_cmpnev_storeoffp_load()
250 		"str r0, %[v]\n\t"  in rseq_cmpnev_storeoffp_load()
251 		"2:\n\t"  in rseq_cmpnev_storeoffp_load()
253 		"b 5f\n\t"  in rseq_cmpnev_storeoffp_load()
255 		"5:\n\t"  in rseq_cmpnev_storeoffp_load()
308 		"ldr r0, %[v]\n\t"  in rseq_addv()
309 		"add r0, %[count]\n\t"  in rseq_addv()
311 		"str r0, %[v]\n\t"  in rseq_addv()
312 		"2:\n\t"  in rseq_addv()
314 		"b 5f\n\t"  in rseq_addv()
316 		"5:\n\t"  in rseq_addv()
362 		"ldr r0, %[v]\n\t"  in rseq_cmpeqv_trystorev_storev()
363 		"cmp %[expect], r0\n\t"  in rseq_cmpeqv_trystorev_storev()
364 		"bne %l[cmpfail]\n\t"  in rseq_cmpeqv_trystorev_storev()
368 		"ldr r0, %[v]\n\t"  in rseq_cmpeqv_trystorev_storev()
369 		"cmp %[expect], r0\n\t"  in rseq_cmpeqv_trystorev_storev()
370 		"bne %l[error2]\n\t"  in rseq_cmpeqv_trystorev_storev()
373 		"str %[newv2], %[v2]\n\t"  in rseq_cmpeqv_trystorev_storev()
376 		"str %[newv], %[v]\n\t"  in rseq_cmpeqv_trystorev_storev()
377 		"2:\n\t"  in rseq_cmpeqv_trystorev_storev()
379 		"b 5f\n\t"  in rseq_cmpeqv_trystorev_storev()
381 		"5:\n\t"  in rseq_cmpeqv_trystorev_storev()
437 		"ldr r0, %[v]\n\t"  in rseq_cmpeqv_trystorev_storev_release()
438 		"cmp %[expect], r0\n\t"  in rseq_cmpeqv_trystorev_storev_release()
439 		"bne %l[cmpfail]\n\t"  in rseq_cmpeqv_trystorev_storev_release()
443 		"ldr r0, %[v]\n\t"  in rseq_cmpeqv_trystorev_storev_release()
444 		"cmp %[expect], r0\n\t"  in rseq_cmpeqv_trystorev_storev_release()
445 		"bne %l[error2]\n\t"  in rseq_cmpeqv_trystorev_storev_release()
448 		"str %[newv2], %[v2]\n\t"  in rseq_cmpeqv_trystorev_storev_release()
450 		"dmb\n\t"	/* full mb provides store-release */  in rseq_cmpeqv_trystorev_storev_release()
452 		"str %[newv], %[v]\n\t"  in rseq_cmpeqv_trystorev_storev_release()
453 		"2:\n\t"  in rseq_cmpeqv_trystorev_storev_release()
455 		"b 5f\n\t"  in rseq_cmpeqv_trystorev_storev_release()
457 		"5:\n\t"  in rseq_cmpeqv_trystorev_storev_release()
514 		"ldr r0, %[v]\n\t"  in rseq_cmpeqv_cmpeqv_storev()
515 		"cmp %[expect], r0\n\t"  in rseq_cmpeqv_cmpeqv_storev()
516 		"bne %l[cmpfail]\n\t"  in rseq_cmpeqv_cmpeqv_storev()
518 		"ldr r0, %[v2]\n\t"  in rseq_cmpeqv_cmpeqv_storev()
519 		"cmp %[expect2], r0\n\t"  in rseq_cmpeqv_cmpeqv_storev()
520 		"bne %l[cmpfail]\n\t"  in rseq_cmpeqv_cmpeqv_storev()
524 		"ldr r0, %[v]\n\t"  in rseq_cmpeqv_cmpeqv_storev()
525 		"cmp %[expect], r0\n\t"  in rseq_cmpeqv_cmpeqv_storev()
526 		"bne %l[error2]\n\t"  in rseq_cmpeqv_cmpeqv_storev()
527 		"ldr r0, %[v2]\n\t"  in rseq_cmpeqv_cmpeqv_storev()
528 		"cmp %[expect2], r0\n\t"  in rseq_cmpeqv_cmpeqv_storev()
529 		"bne %l[error3]\n\t"  in rseq_cmpeqv_cmpeqv_storev()
532 		"str %[newv], %[v]\n\t"  in rseq_cmpeqv_cmpeqv_storev()
533 		"2:\n\t"  in rseq_cmpeqv_cmpeqv_storev()
535 		"b 5f\n\t"  in rseq_cmpeqv_cmpeqv_storev()
537 		"5:\n\t"  in rseq_cmpeqv_cmpeqv_storev()
593 		"str %[src], %[rseq_scratch0]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
594 		"str %[dst], %[rseq_scratch1]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
595 		"str %[len], %[rseq_scratch2]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
600 		"ldr r0, %[v]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
601 		"cmp %[expect], r0\n\t"  in rseq_cmpeqv_trymemcpy_storev()
602 		"bne 5f\n\t"  in rseq_cmpeqv_trymemcpy_storev()
606 		"ldr r0, %[v]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
607 		"cmp %[expect], r0\n\t"  in rseq_cmpeqv_trymemcpy_storev()
608 		"bne 7f\n\t"  in rseq_cmpeqv_trymemcpy_storev()
611 		"cmp %[len], #0\n\t" \  in rseq_cmpeqv_trymemcpy_storev()
612 		"beq 333f\n\t" \  in rseq_cmpeqv_trymemcpy_storev()
613 		"222:\n\t" \  in rseq_cmpeqv_trymemcpy_storev()
614 		"ldrb %%r0, [%[src]]\n\t" \  in rseq_cmpeqv_trymemcpy_storev()
615 		"strb %%r0, [%[dst]]\n\t" \  in rseq_cmpeqv_trymemcpy_storev()
616 		"adds %[src], #1\n\t" \  in rseq_cmpeqv_trymemcpy_storev()
617 		"adds %[dst], #1\n\t" \  in rseq_cmpeqv_trymemcpy_storev()
618 		"subs %[len], #1\n\t" \  in rseq_cmpeqv_trymemcpy_storev()
619 		"bne 222b\n\t" \  in rseq_cmpeqv_trymemcpy_storev()
620 		"333:\n\t" \  in rseq_cmpeqv_trymemcpy_storev()
623 		"str %[newv], %[v]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
624 		"2:\n\t"  in rseq_cmpeqv_trymemcpy_storev()
627 		"ldr %[len], %[rseq_scratch2]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
628 		"ldr %[dst], %[rseq_scratch1]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
629 		"ldr %[src], %[rseq_scratch0]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
630 		"b 8f\n\t"  in rseq_cmpeqv_trymemcpy_storev()
633 				      "ldr %[len], %[rseq_scratch2]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
634 				      "ldr %[dst], %[rseq_scratch1]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
635 				      "ldr %[src], %[rseq_scratch0]\n\t",  in rseq_cmpeqv_trymemcpy_storev()
639 					"ldr %[len], %[rseq_scratch2]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
640 					"ldr %[dst], %[rseq_scratch1]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
641 					"ldr %[src], %[rseq_scratch0]\n\t",  in rseq_cmpeqv_trymemcpy_storev()
646 					"ldr %[len], %[rseq_scratch2]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
647 					"ldr %[dst], %[rseq_scratch1]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
648 					"ldr %[src], %[rseq_scratch0]\n\t",  in rseq_cmpeqv_trymemcpy_storev()
652 					"ldr %[len], %[rseq_scratch2]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
653 					"ldr %[dst], %[rseq_scratch1]\n\t"  in rseq_cmpeqv_trymemcpy_storev()
654 					"ldr %[src], %[rseq_scratch0]\n\t",  in rseq_cmpeqv_trymemcpy_storev()
657 		"8:\n\t"  in rseq_cmpeqv_trymemcpy_storev()
717 		"str %[src], %[rseq_scratch0]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
718 		"str %[dst], %[rseq_scratch1]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
719 		"str %[len], %[rseq_scratch2]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
724 		"ldr r0, %[v]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
725 		"cmp %[expect], r0\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
726 		"bne 5f\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
730 		"ldr r0, %[v]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
731 		"cmp %[expect], r0\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
732 		"bne 7f\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
735 		"cmp %[len], #0\n\t" \  in rseq_cmpeqv_trymemcpy_storev_release()
736 		"beq 333f\n\t" \  in rseq_cmpeqv_trymemcpy_storev_release()
737 		"222:\n\t" \  in rseq_cmpeqv_trymemcpy_storev_release()
738 		"ldrb %%r0, [%[src]]\n\t" \  in rseq_cmpeqv_trymemcpy_storev_release()
739 		"strb %%r0, [%[dst]]\n\t" \  in rseq_cmpeqv_trymemcpy_storev_release()
740 		"adds %[src], #1\n\t" \  in rseq_cmpeqv_trymemcpy_storev_release()
741 		"adds %[dst], #1\n\t" \  in rseq_cmpeqv_trymemcpy_storev_release()
742 		"subs %[len], #1\n\t" \  in rseq_cmpeqv_trymemcpy_storev_release()
743 		"bne 222b\n\t" \  in rseq_cmpeqv_trymemcpy_storev_release()
744 		"333:\n\t" \  in rseq_cmpeqv_trymemcpy_storev_release()
746 		"dmb\n\t"	/* full mb provides store-release */  in rseq_cmpeqv_trymemcpy_storev_release()
748 		"str %[newv], %[v]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
749 		"2:\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
752 		"ldr %[len], %[rseq_scratch2]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
753 		"ldr %[dst], %[rseq_scratch1]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
754 		"ldr %[src], %[rseq_scratch0]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
755 		"b 8f\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
758 				      "ldr %[len], %[rseq_scratch2]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
759 				      "ldr %[dst], %[rseq_scratch1]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
760 				      "ldr %[src], %[rseq_scratch0]\n\t",  in rseq_cmpeqv_trymemcpy_storev_release()
764 					"ldr %[len], %[rseq_scratch2]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
765 					"ldr %[dst], %[rseq_scratch1]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
766 					"ldr %[src], %[rseq_scratch0]\n\t",  in rseq_cmpeqv_trymemcpy_storev_release()
771 					"ldr %[len], %[rseq_scratch2]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
772 					"ldr %[dst], %[rseq_scratch1]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
773 					"ldr %[src], %[rseq_scratch0]\n\t",  in rseq_cmpeqv_trymemcpy_storev_release()
777 					"ldr %[len], %[rseq_scratch2]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
778 					"ldr %[dst], %[rseq_scratch1]\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()
779 					"ldr %[src], %[rseq_scratch0]\n\t",  in rseq_cmpeqv_trymemcpy_storev_release()
782 		"8:\n\t"  in rseq_cmpeqv_trymemcpy_storev_release()