include/asm-generic/percpu.h

78 #define raw_cpu_generic_read(pcp)					\
80 	*raw_cpu_ptr(&(pcp));						\
83 #define raw_cpu_generic_to_op(pcp, val, op)				\
85 	*raw_cpu_ptr(&(pcp)) op val;					\
88 #define raw_cpu_generic_add_return(pcp, val)				\
90 	TYPEOF_UNQUAL(pcp) *__p = raw_cpu_ptr(&(pcp));			\
96 #define raw_cpu_generic_xchg(pcp, nval)					\
98 	TYPEOF_UNQUAL(pcp) *__p = raw_cpu_ptr(&(pcp));			\
99 	TYPEOF_UNQUAL(pcp) __ret;					\
105 #define __cpu_fallback_try_cmpxchg(pcp, ovalp, nval, _cmpxchg)		\
107 	TYPEOF_UNQUAL(pcp) __val, __old = *(ovalp);			\
108 	__val = _cmpxchg(pcp, __old, nval);				\
114 #define raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)			\
116 	TYPEOF_UNQUAL(pcp) *__p = raw_cpu_ptr(&(pcp));			\
117 	TYPEOF_UNQUAL(pcp) __val = *__p, ___old = *(ovalp);		\
129 #define raw_cpu_generic_cmpxchg(pcp, oval, nval)			\
131 	TYPEOF_UNQUAL(pcp) __old = (oval);				\
132 	raw_cpu_generic_try_cmpxchg(pcp, &__old, nval);			\
136 #define __this_cpu_generic_read_nopreempt(pcp)				\
138 	TYPEOF_UNQUAL(pcp) ___ret;					\
140 	___ret = READ_ONCE(*raw_cpu_ptr(&(pcp)));			\
145 #define __this_cpu_generic_read_noirq(pcp)				\
147 	TYPEOF_UNQUAL(pcp) ___ret;					\
150 	___ret = raw_cpu_generic_read(pcp);				\
155 #define this_cpu_generic_read(pcp)					\
157 	TYPEOF_UNQUAL(pcp) __ret;					\
158 	if (__native_word(pcp))						\
159 		__ret = __this_cpu_generic_read_nopreempt(pcp);		\
161 		__ret = __this_cpu_generic_read_noirq(pcp);		\
165 #define this_cpu_generic_to_op(pcp, val, op)				\
169 	raw_cpu_generic_to_op(pcp, val, op);				\
174 #define this_cpu_generic_add_return(pcp, val)				\
176 	TYPEOF_UNQUAL(pcp) __ret;					\
179 	__ret = raw_cpu_generic_add_return(pcp, val);			\
184 #define this_cpu_generic_xchg(pcp, nval)				\
186 	TYPEOF_UNQUAL(pcp) __ret;					\
189 	__ret = raw_cpu_generic_xchg(pcp, nval);			\
194 #define this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)			\
199 	__ret = raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval);		\
204 #define this_cpu_generic_cmpxchg(pcp, oval, nval)			\
206 	TYPEOF_UNQUAL(pcp) __ret;					\
209 	__ret = raw_cpu_generic_cmpxchg(pcp, oval, nval);		\
215 #define raw_cpu_read_1(pcp)		raw_cpu_generic_read(pcp)
218 #define raw_cpu_read_2(pcp)		raw_cpu_generic_read(pcp)
221 #define raw_cpu_read_4(pcp)		raw_cpu_generic_read(pcp)
224 #define raw_cpu_read_8(pcp)		raw_cpu_generic_read(pcp)
228 #define raw_cpu_write_1(pcp, val)	raw_cpu_generic_to_op(pcp, val, =)
231 #define raw_cpu_write_2(pcp, val)	raw_cpu_generic_to_op(pcp, val, =)
234 #define raw_cpu_write_4(pcp, val)	raw_cpu_generic_to_op(pcp, val, =)
237 #define raw_cpu_write_8(pcp, val)	raw_cpu_generic_to_op(pcp, val, =)
241 #define raw_cpu_add_1(pcp, val)		raw_cpu_generic_to_op(pcp, val, +=)
244 #define raw_cpu_add_2(pcp, val)		raw_cpu_generic_to_op(pcp, val, +=)
247 #define raw_cpu_add_4(pcp, val)		raw_cpu_generic_to_op(pcp, val, +=)
250 #define raw_cpu_add_8(pcp, val)		raw_cpu_generic_to_op(pcp, val, +=)
254 #define raw_cpu_and_1(pcp, val)		raw_cpu_generic_to_op(pcp, val, &=)
257 #define raw_cpu_and_2(pcp, val)		raw_cpu_generic_to_op(pcp, val, &=)
260 #define raw_cpu_and_4(pcp, val)		raw_cpu_generic_to_op(pcp, val, &=)
263 #define raw_cpu_and_8(pcp, val)		raw_cpu_generic_to_op(pcp, val, &=)
267 #define raw_cpu_or_1(pcp, val)		raw_cpu_generic_to_op(pcp, val, |=)
270 #define raw_cpu_or_2(pcp, val)		raw_cpu_generic_to_op(pcp, val, |=)
273 #define raw_cpu_or_4(pcp, val)		raw_cpu_generic_to_op(pcp, val, |=)
276 #define raw_cpu_or_8(pcp, val)		raw_cpu_generic_to_op(pcp, val, |=)
280 #define raw_cpu_add_return_1(pcp, val)	raw_cpu_generic_add_return(pcp, val)
283 #define raw_cpu_add_return_2(pcp, val)	raw_cpu_generic_add_return(pcp, val)
286 #define raw_cpu_add_return_4(pcp, val)	raw_cpu_generic_add_return(pcp, val)
289 #define raw_cpu_add_return_8(pcp, val)	raw_cpu_generic_add_return(pcp, val)
293 #define raw_cpu_xchg_1(pcp, nval)	raw_cpu_generic_xchg(pcp, nval)
296 #define raw_cpu_xchg_2(pcp, nval)	raw_cpu_generic_xchg(pcp, nval)
299 #define raw_cpu_xchg_4(pcp, nval)	raw_cpu_generic_xchg(pcp, nval)
302 #define raw_cpu_xchg_8(pcp, nval)	raw_cpu_generic_xchg(pcp, nval)
307 #define raw_cpu_try_cmpxchg_1(pcp, ovalp, nval) \
308 	__cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_1)
310 #define raw_cpu_try_cmpxchg_1(pcp, ovalp, nval) \
311 	raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
316 #define raw_cpu_try_cmpxchg_2(pcp, ovalp, nval) \
317 	__cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_2)
319 #define raw_cpu_try_cmpxchg_2(pcp, ovalp, nval) \
320 	raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
325 #define raw_cpu_try_cmpxchg_4(pcp, ovalp, nval) \
326 	__cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_4)
328 #define raw_cpu_try_cmpxchg_4(pcp, ovalp, nval) \
329 	raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
334 #define raw_cpu_try_cmpxchg_8(pcp, ovalp, nval) \
335 	__cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg_8)
337 #define raw_cpu_try_cmpxchg_8(pcp, ovalp, nval) \
338 	raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
344 #define raw_cpu_try_cmpxchg64(pcp, ovalp, nval) \
345 	__cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg64)
347 #define raw_cpu_try_cmpxchg64(pcp, ovalp, nval) \
348 	raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
353 #define raw_cpu_try_cmpxchg128(pcp, ovalp, nval) \
354 	__cpu_fallback_try_cmpxchg(pcp, ovalp, nval, raw_cpu_cmpxchg128)
356 #define raw_cpu_try_cmpxchg128(pcp, ovalp, nval) \
357 	raw_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
362 #define raw_cpu_cmpxchg_1(pcp, oval, nval) \
363 	raw_cpu_generic_cmpxchg(pcp, oval, nval)
366 #define raw_cpu_cmpxchg_2(pcp, oval, nval) \
367 	raw_cpu_generic_cmpxchg(pcp, oval, nval)
370 #define raw_cpu_cmpxchg_4(pcp, oval, nval) \
371 	raw_cpu_generic_cmpxchg(pcp, oval, nval)
374 #define raw_cpu_cmpxchg_8(pcp, oval, nval) \
375 	raw_cpu_generic_cmpxchg(pcp, oval, nval)
379 #define raw_cpu_cmpxchg64(pcp, oval, nval) \
380 	raw_cpu_generic_cmpxchg(pcp, oval, nval)
383 #define raw_cpu_cmpxchg128(pcp, oval, nval) \
384 	raw_cpu_generic_cmpxchg(pcp, oval, nval)
388 #define this_cpu_read_1(pcp)		this_cpu_generic_read(pcp)
391 #define this_cpu_read_2(pcp)		this_cpu_generic_read(pcp)
394 #define this_cpu_read_4(pcp)		this_cpu_generic_read(pcp)
397 #define this_cpu_read_8(pcp)		this_cpu_generic_read(pcp)
401 #define this_cpu_write_1(pcp, val)	this_cpu_generic_to_op(pcp, val, =)
404 #define this_cpu_write_2(pcp, val)	this_cpu_generic_to_op(pcp, val, =)
407 #define this_cpu_write_4(pcp, val)	this_cpu_generic_to_op(pcp, val, =)
410 #define this_cpu_write_8(pcp, val)	this_cpu_generic_to_op(pcp, val, =)
414 #define this_cpu_add_1(pcp, val)	this_cpu_generic_to_op(pcp, val, +=)
417 #define this_cpu_add_2(pcp, val)	this_cpu_generic_to_op(pcp, val, +=)
420 #define this_cpu_add_4(pcp, val)	this_cpu_generic_to_op(pcp, val, +=)
423 #define this_cpu_add_8(pcp, val)	this_cpu_generic_to_op(pcp, val, +=)
427 #define this_cpu_and_1(pcp, val)	this_cpu_generic_to_op(pcp, val, &=)
430 #define this_cpu_and_2(pcp, val)	this_cpu_generic_to_op(pcp, val, &=)
433 #define this_cpu_and_4(pcp, val)	this_cpu_generic_to_op(pcp, val, &=)
436 #define this_cpu_and_8(pcp, val)	this_cpu_generic_to_op(pcp, val, &=)
440 #define this_cpu_or_1(pcp, val)		this_cpu_generic_to_op(pcp, val, |=)
443 #define this_cpu_or_2(pcp, val)		this_cpu_generic_to_op(pcp, val, |=)
446 #define this_cpu_or_4(pcp, val)		this_cpu_generic_to_op(pcp, val, |=)
449 #define this_cpu_or_8(pcp, val)		this_cpu_generic_to_op(pcp, val, |=)
453 #define this_cpu_add_return_1(pcp, val)	this_cpu_generic_add_return(pcp, val)
456 #define this_cpu_add_return_2(pcp, val)	this_cpu_generic_add_return(pcp, val)
459 #define this_cpu_add_return_4(pcp, val)	this_cpu_generic_add_return(pcp, val)
462 #define this_cpu_add_return_8(pcp, val)	this_cpu_generic_add_return(pcp, val)
466 #define this_cpu_xchg_1(pcp, nval)	this_cpu_generic_xchg(pcp, nval)
469 #define this_cpu_xchg_2(pcp, nval)	this_cpu_generic_xchg(pcp, nval)
472 #define this_cpu_xchg_4(pcp, nval)	this_cpu_generic_xchg(pcp, nval)
475 #define this_cpu_xchg_8(pcp, nval)	this_cpu_generic_xchg(pcp, nval)
480 #define this_cpu_try_cmpxchg_1(pcp, ovalp, nval) \
481 	__cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_1)
483 #define this_cpu_try_cmpxchg_1(pcp, ovalp, nval) \
484 	this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
489 #define this_cpu_try_cmpxchg_2(pcp, ovalp, nval) \
490 	__cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_2)
492 #define this_cpu_try_cmpxchg_2(pcp, ovalp, nval) \
493 	this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
498 #define this_cpu_try_cmpxchg_4(pcp, ovalp, nval) \
499 	__cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_4)
501 #define this_cpu_try_cmpxchg_4(pcp, ovalp, nval) \
502 	this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
507 #define this_cpu_try_cmpxchg_8(pcp, ovalp, nval) \
508 	__cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg_8)
510 #define this_cpu_try_cmpxchg_8(pcp, ovalp, nval) \
511 	this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
517 #define this_cpu_try_cmpxchg64(pcp, ovalp, nval) \
518 	__cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg64)
520 #define this_cpu_try_cmpxchg64(pcp, ovalp, nval) \
521 	this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
526 #define this_cpu_try_cmpxchg128(pcp, ovalp, nval) \
527 	__cpu_fallback_try_cmpxchg(pcp, ovalp, nval, this_cpu_cmpxchg128)
529 #define this_cpu_try_cmpxchg128(pcp, ovalp, nval) \
530 	this_cpu_generic_try_cmpxchg(pcp, ovalp, nval)
535 #define this_cpu_cmpxchg_1(pcp, oval, nval) \
536 	this_cpu_generic_cmpxchg(pcp, oval, nval)
539 #define this_cpu_cmpxchg_2(pcp, oval, nval) \
540 	this_cpu_generic_cmpxchg(pcp, oval, nval)
543 #define this_cpu_cmpxchg_4(pcp, oval, nval) \
544 	this_cpu_generic_cmpxchg(pcp, oval, nval)
547 #define this_cpu_cmpxchg_8(pcp, oval, nval) \
548 	this_cpu_generic_cmpxchg(pcp, oval, nval)
552 #define this_cpu_cmpxchg64(pcp, oval, nval) \
553 	this_cpu_generic_cmpxchg(pcp, oval, nval)
556 #define this_cpu_cmpxchg128(pcp, oval, nval) \
557 	this_cpu_generic_cmpxchg(pcp, oval, nval)