10  * 2. Redistributions in binary form must reproduce the above copyright
84 #define	_ATOMIC_OP_IMPL(t, w, s, op, llsc_asm_op, lse_asm_op, pre, bar, a, l) \  argument
92 	    "1: ld"#a"xr"#s"	%"#w"0, [%2]\n"				\
93 	    "   "#llsc_asm_op"	%"#w"0, %"#w"0, %"#w"3\n"		\
94 	    "   st"#l"xr"#s"	%w1, %"#w"0, [%2]\n"			\
109 	    "ld"#lse_asm_op#a#l#s"	%"#w"2, %"#w"0, [%1]\n"		\
126 	_ATOMIC_OP_IMPL(8,  w, b, op, llsc_asm_op, lse_asm_op, pre,	\
128 	_ATOMIC_OP_IMPL(16, w, h, op, llsc_asm_op, lse_asm_op, pre,	\
130 	_ATOMIC_OP_IMPL(32, w,  , op, llsc_asm_op, lse_asm_op, pre,	\
155 #define	_ATOMIC_CMPSET_IMPL(t, w, s, bar, a, l)				\  argument
163 	    "   ld"#a"xr"#s"	%"#w"0, [%2]\n"				\
164 	    "   cmp		%"#w"0, %"#w"3\n"			\
165 	    "   b.ne		2f\n"					\
166 	    "   st"#l"xr"#s"	%w1, %"#w"4, [%2]\n"			\
168 	    "2:"							\
185 	    "cas"#a#l#s"	%"#w"1, %"#w"4, [%3]\n"			\
186 	    "cmp		%"#w"1, %"#w"2\n"			\
215 	    "   ld"#a"xr"#s"	%"#w"0, [%2]\n"				\
216 	    "   cmp		%"#w"0, %"#w"3\n"			\
218 	    "   st"#l"xr"#s"	%w1, %"#w"4, [%2]\n"			\
237 	    "cas"#a#l#s"	%"#w"1, %"#w"4, [%3]\n"			\
238 	    "cmp		%"#w"1, %"#w"2\n"			\
261 	_ATOMIC_CMPSET_IMPL(8,  w, b, bar, a, l)			\
262 	_ATOMIC_CMPSET_IMPL(16, w, h, bar, a, l)			\
263 	_ATOMIC_CMPSET_IMPL(32, w,  , bar, a, l)			\
279 #define	_ATOMIC_FETCHADD_IMPL(t, w)					\  argument
286 	    "1: ldxr	%"#w"2, [%3]\n"					\
287 	    "   add	%"#w"0, %"#w"2, %"#w"4\n"			\
288 	    "   stxr	%w1, %"#w"0, [%3]\n"				\
304 	    "ldadd	%"#w"2, %"#w"0, [%1]\n"				\
322 _ATOMIC_FETCHADD_IMPL(32, w)
333 #define	_ATOMIC_SWAP_IMPL(t, w, zreg)					\  argument
340 	    "1: ldxr	%"#w"1, [%2]\n"					\
341 	    "   stxr	%w0, %"#w"3, [%2]\n"				\
357 	    "swp	%"#w"2, %"#w"0, [%1]\n"				\
381 	    "1: ldxr	%"#w"1, [%2]\n"					\
382 	    "   stxr	%w0, "#zreg", [%2]\n"				\
405 _ATOMIC_SWAP_IMPL(32, w, wzr)
412 #define	_ATOMIC_TEST_OP_IMPL(t, w, op, llsc_asm_op, lse_asm_op, bar, a)	\  argument
420 	    "1: ld"#a"xr	%"#w"2, [%3]\n"				\
421 	    "  "#llsc_asm_op"	%"#w"0, %"#w"2, %"#w"4\n"		\
422 	    "   stxr		%w1, %"#w"0, [%3]\n"			\
439 	    "ld"#lse_asm_op#a"	%"#w"2, %"#w"0, [%1]\n"			\
458 	_ATOMIC_TEST_OP_IMPL(32, w, op, llsc_asm_op, lse_asm_op,     ,  ) \
459 	_ATOMIC_TEST_OP_IMPL(32, w, op, llsc_asm_op, lse_asm_op, acq_, a) \
466 #define	_ATOMIC_LOAD_ACQ_IMPL(t, w, s)					\  in _ATOMIC_TEST_OP()  argument
473 	    "ldar"#s"	%"#w"0, [%1]\n"					\  in _ATOMIC_TEST_OP()
483 _ATOMIC_LOAD_ACQ_IMPL(8,  w, b)
484 _ATOMIC_LOAD_ACQ_IMPL(16, w, h)
485 _ATOMIC_LOAD_ACQ_IMPL(32, w,  )
488 #define	_ATOMIC_STORE_REL_IMPL(t, w, s)					\  argument
493 	    "stlr"#s"	%"#w"0, [%1]\n"					\
499 _ATOMIC_STORE_REL_IMPL(8,  w, b)
500 _ATOMIC_STORE_REL_IMPL(16, w, h)
501 _ATOMIC_STORE_REL_IMPL(32, w,  )