1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_X86_PERCPU_H 3 #define _ASM_X86_PERCPU_H 4 5 #ifdef CONFIG_X86_64 6 # define __percpu_seg gs 7 # define __percpu_rel (%rip) 8 #else 9 # define __percpu_seg fs 10 # define __percpu_rel 11 #endif 12 13 #ifdef __ASSEMBLER__ 14 15 #ifdef CONFIG_SMP 16 # define __percpu %__percpu_seg: 17 #else 18 # define __percpu 19 #endif 20 21 #define PER_CPU_VAR(var) __percpu(var)__percpu_rel 22 23 #else /* !__ASSEMBLY__: */ 24 25 #include <linux/args.h> 26 #include <linux/build_bug.h> 27 #include <linux/stringify.h> 28 #include <asm/asm.h> 29 30 #ifdef CONFIG_SMP 31 32 #define __force_percpu_prefix "%%"__stringify(__percpu_seg)":" 33 34 #ifdef CONFIG_CC_HAS_NAMED_AS 35 36 #ifdef __CHECKER__ 37 # define __seg_gs __attribute__((address_space(__seg_gs))) 38 # define __seg_fs __attribute__((address_space(__seg_fs))) 39 #endif 40 41 #define __percpu_prefix 42 #define __percpu_seg_override CONCATENATE(__seg_, __percpu_seg) 43 44 #else /* !CONFIG_CC_HAS_NAMED_AS: */ 45 46 #define __percpu_prefix __force_percpu_prefix 47 #define __percpu_seg_override 48 49 #endif /* CONFIG_CC_HAS_NAMED_AS */ 50 51 /* 52 * Compared to the generic __my_cpu_offset version, the following 53 * saves one instruction and avoids clobbering a temp register. 54 */ 55 #define __my_cpu_offset this_cpu_read(this_cpu_off) 56 57 /* 58 * arch_raw_cpu_ptr should not be used in 32-bit VDSO for a 64-bit 59 * kernel, because games are played with CONFIG_X86_64 there and 60 * sizeof(this_cpu_off) becames 4. 61 */ 62 #ifndef BUILD_VDSO32_64 63 #define arch_raw_cpu_ptr(_ptr) \ 64 ({ \ 65 unsigned long tcp_ptr__ = raw_cpu_read_long(this_cpu_off); \ 66 \ 67 tcp_ptr__ += (__force unsigned long)(_ptr); \ 68 (TYPEOF_UNQUAL(*(_ptr)) __force __kernel *)tcp_ptr__; \ 69 }) 70 #else 71 #define arch_raw_cpu_ptr(_ptr) \ 72 ({ \ 73 BUILD_BUG(); \ 74 (TYPEOF_UNQUAL(*(_ptr)) __force __kernel *)0; \ 75 }) 76 #endif 77 78 #define PER_CPU_VAR(var) %__percpu_seg:(var)__percpu_rel 79 80 #else /* !CONFIG_SMP: */ 81 82 #define __force_percpu_prefix 83 #define __percpu_prefix 84 #define __percpu_seg_override 85 86 #define PER_CPU_VAR(var) (var)__percpu_rel 87 88 #endif /* CONFIG_SMP */ 89 90 #if defined(CONFIG_USE_X86_SEG_SUPPORT) && defined(USE_TYPEOF_UNQUAL) 91 # define __my_cpu_type(var) typeof(var) 92 # define __my_cpu_ptr(ptr) (ptr) 93 # define __my_cpu_var(var) (var) 94 95 # define __percpu_qual __percpu_seg_override 96 #else 97 # define __my_cpu_type(var) typeof(var) __percpu_seg_override 98 # define __my_cpu_ptr(ptr) (__my_cpu_type(*(ptr))*)(__force uintptr_t)(ptr) 99 # define __my_cpu_var(var) (*__my_cpu_ptr(&(var))) 100 #endif 101 102 #define __force_percpu_arg(x) __force_percpu_prefix "%" #x 103 #define __percpu_arg(x) __percpu_prefix "%" #x 104 105 /* 106 * For arch-specific code, we can use direct single-insn ops (they 107 * don't give an lvalue though). 108 */ 109 110 #define __pcpu_type_1 u8 111 #define __pcpu_type_2 u16 112 #define __pcpu_type_4 u32 113 #define __pcpu_type_8 u64 114 115 #define __pcpu_cast_1(val) ((u8)(((unsigned long) val) & 0xff)) 116 #define __pcpu_cast_2(val) ((u16)(((unsigned long) val) & 0xffff)) 117 #define __pcpu_cast_4(val) ((u32)(((unsigned long) val) & 0xffffffff)) 118 #define __pcpu_cast_8(val) ((u64)(val)) 119 120 #define __pcpu_op_1(op) op "b " 121 #define __pcpu_op_2(op) op "w " 122 #define __pcpu_op_4(op) op "l " 123 #define __pcpu_op_8(op) op "q " 124 125 #define __pcpu_reg_1(mod, x) mod "q" (x) 126 #define __pcpu_reg_2(mod, x) mod "r" (x) 127 #define __pcpu_reg_4(mod, x) mod "r" (x) 128 #define __pcpu_reg_8(mod, x) mod "r" (x) 129 130 #define __pcpu_reg_imm_1(x) "qi" (x) 131 #define __pcpu_reg_imm_2(x) "ri" (x) 132 #define __pcpu_reg_imm_4(x) "ri" (x) 133 #define __pcpu_reg_imm_8(x) "re" (x) 134 135 #ifdef CONFIG_USE_X86_SEG_SUPPORT 136 137 #define __raw_cpu_read(size, qual, pcp) \ 138 ({ \ 139 *(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)); \ 140 }) 141 142 #define __raw_cpu_write(size, qual, pcp, val) \ 143 do { \ 144 *(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)) = (val); \ 145 } while (0) 146 147 #define __raw_cpu_read_const(pcp) __raw_cpu_read(, , pcp) 148 149 #else /* !CONFIG_USE_X86_SEG_SUPPORT: */ 150 151 #define __raw_cpu_read(size, qual, _var) \ 152 ({ \ 153 __pcpu_type_##size pfo_val__; \ 154 \ 155 asm qual (__pcpu_op_##size("mov") \ 156 __percpu_arg([var]) ", %[val]" \ 157 : [val] __pcpu_reg_##size("=", pfo_val__) \ 158 : [var] "m" (__my_cpu_var(_var))); \ 159 \ 160 (typeof(_var))(unsigned long) pfo_val__; \ 161 }) 162 163 #define __raw_cpu_write(size, qual, _var, _val) \ 164 do { \ 165 __pcpu_type_##size pto_val__ = __pcpu_cast_##size(_val); \ 166 \ 167 if (0) { \ 168 TYPEOF_UNQUAL(_var) pto_tmp__; \ 169 pto_tmp__ = (_val); \ 170 (void)pto_tmp__; \ 171 } \ 172 asm qual (__pcpu_op_##size("mov") "%[val], " \ 173 __percpu_arg([var]) \ 174 : [var] "=m" (__my_cpu_var(_var)) \ 175 : [val] __pcpu_reg_imm_##size(pto_val__)); \ 176 } while (0) 177 178 /* 179 * The generic per-CPU infrastrucutre is not suitable for 180 * reading const-qualified variables. 181 */ 182 #define __raw_cpu_read_const(pcp) ({ BUILD_BUG(); (typeof(pcp))0; }) 183 184 #endif /* CONFIG_USE_X86_SEG_SUPPORT */ 185 186 #define __raw_cpu_read_stable(size, _var) \ 187 ({ \ 188 __pcpu_type_##size pfo_val__; \ 189 \ 190 asm(__pcpu_op_##size("mov") \ 191 __force_percpu_arg(a[var]) ", %[val]" \ 192 : [val] __pcpu_reg_##size("=", pfo_val__) \ 193 : [var] "i" (&(_var))); \ 194 \ 195 (typeof(_var))(unsigned long) pfo_val__; \ 196 }) 197 198 #define percpu_unary_op(size, qual, op, _var) \ 199 ({ \ 200 asm qual (__pcpu_op_##size(op) __percpu_arg([var]) \ 201 : [var] "+m" (__my_cpu_var(_var))); \ 202 }) 203 204 #define percpu_binary_op(size, qual, op, _var, _val) \ 205 do { \ 206 __pcpu_type_##size pto_val__ = __pcpu_cast_##size(_val); \ 207 \ 208 if (0) { \ 209 TYPEOF_UNQUAL(_var) pto_tmp__; \ 210 pto_tmp__ = (_val); \ 211 (void)pto_tmp__; \ 212 } \ 213 asm qual (__pcpu_op_##size(op) "%[val], " __percpu_arg([var]) \ 214 : [var] "+m" (__my_cpu_var(_var)) \ 215 : [val] __pcpu_reg_imm_##size(pto_val__)); \ 216 } while (0) 217 218 /* 219 * Generate a per-CPU add to memory instruction and optimize code 220 * if one is added or subtracted. 221 */ 222 #define percpu_add_op(size, qual, var, val) \ 223 do { \ 224 const int pao_ID__ = \ 225 (__builtin_constant_p(val) && \ 226 ((val) == 1 || \ 227 (val) == (typeof(val))-1)) ? (int)(val) : 0; \ 228 \ 229 if (0) { \ 230 TYPEOF_UNQUAL(var) pao_tmp__; \ 231 pao_tmp__ = (val); \ 232 (void)pao_tmp__; \ 233 } \ 234 if (pao_ID__ == 1) \ 235 percpu_unary_op(size, qual, "inc", var); \ 236 else if (pao_ID__ == -1) \ 237 percpu_unary_op(size, qual, "dec", var); \ 238 else \ 239 percpu_binary_op(size, qual, "add", var, val); \ 240 } while (0) 241 242 /* 243 * Add return operation 244 */ 245 #define percpu_add_return_op(size, qual, _var, _val) \ 246 ({ \ 247 __pcpu_type_##size paro_tmp__ = __pcpu_cast_##size(_val); \ 248 \ 249 asm qual (__pcpu_op_##size("xadd") "%[tmp], " \ 250 __percpu_arg([var]) \ 251 : [tmp] __pcpu_reg_##size("+", paro_tmp__), \ 252 [var] "+m" (__my_cpu_var(_var)) \ 253 : : "memory"); \ 254 (typeof(_var))(unsigned long) (paro_tmp__ + _val); \ 255 }) 256 257 /* 258 * raw_cpu_xchg() can use a load-store since 259 * it is not required to be IRQ-safe. 260 */ 261 #define raw_percpu_xchg_op(_var, _nval) \ 262 ({ \ 263 TYPEOF_UNQUAL(_var) pxo_old__ = raw_cpu_read(_var); \ 264 \ 265 raw_cpu_write(_var, _nval); \ 266 \ 267 pxo_old__; \ 268 }) 269 270 /* 271 * this_cpu_xchg() is implemented using CMPXCHG without a LOCK prefix. 272 * XCHG is expensive due to the implied LOCK prefix. The processor 273 * cannot prefetch cachelines if XCHG is used. 274 */ 275 #define this_percpu_xchg_op(_var, _nval) \ 276 ({ \ 277 TYPEOF_UNQUAL(_var) pxo_old__ = this_cpu_read(_var); \ 278 \ 279 do { } while (!this_cpu_try_cmpxchg(_var, &pxo_old__, _nval)); \ 280 \ 281 pxo_old__; \ 282 }) 283 284 /* 285 * CMPXCHG has no such implied lock semantics as a result it is much 286 * more efficient for CPU-local operations. 287 */ 288 #define percpu_cmpxchg_op(size, qual, _var, _oval, _nval) \ 289 ({ \ 290 __pcpu_type_##size pco_old__ = __pcpu_cast_##size(_oval); \ 291 __pcpu_type_##size pco_new__ = __pcpu_cast_##size(_nval); \ 292 \ 293 asm qual (__pcpu_op_##size("cmpxchg") "%[nval], " \ 294 __percpu_arg([var]) \ 295 : [oval] "+a" (pco_old__), \ 296 [var] "+m" (__my_cpu_var(_var)) \ 297 : [nval] __pcpu_reg_##size(, pco_new__) \ 298 : "memory"); \ 299 \ 300 (typeof(_var))(unsigned long) pco_old__; \ 301 }) 302 303 #define percpu_try_cmpxchg_op(size, qual, _var, _ovalp, _nval) \ 304 ({ \ 305 bool success; \ 306 __pcpu_type_##size *pco_oval__ = (__pcpu_type_##size *)(_ovalp); \ 307 __pcpu_type_##size pco_old__ = *pco_oval__; \ 308 __pcpu_type_##size pco_new__ = __pcpu_cast_##size(_nval); \ 309 \ 310 asm qual (__pcpu_op_##size("cmpxchg") "%[nval], " \ 311 __percpu_arg([var]) \ 312 CC_SET(z) \ 313 : CC_OUT(z) (success), \ 314 [oval] "+a" (pco_old__), \ 315 [var] "+m" (__my_cpu_var(_var)) \ 316 : [nval] __pcpu_reg_##size(, pco_new__) \ 317 : "memory"); \ 318 if (unlikely(!success)) \ 319 *pco_oval__ = pco_old__; \ 320 \ 321 likely(success); \ 322 }) 323 324 #if defined(CONFIG_X86_32) && !defined(CONFIG_UML) 325 326 #define percpu_cmpxchg64_op(size, qual, _var, _oval, _nval) \ 327 ({ \ 328 union { \ 329 u64 var; \ 330 struct { \ 331 u32 low, high; \ 332 }; \ 333 } old__, new__; \ 334 \ 335 old__.var = _oval; \ 336 new__.var = _nval; \ 337 \ 338 asm_inline qual ( \ 339 ALTERNATIVE("call this_cpu_cmpxchg8b_emu", \ 340 "cmpxchg8b " __percpu_arg([var]), X86_FEATURE_CX8) \ 341 : ALT_OUTPUT_SP([var] "+m" (__my_cpu_var(_var)), \ 342 "+a" (old__.low), "+d" (old__.high)) \ 343 : "b" (new__.low), "c" (new__.high), \ 344 "S" (&(_var)) \ 345 : "memory"); \ 346 \ 347 old__.var; \ 348 }) 349 350 #define raw_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg64_op(8, , pcp, oval, nval) 351 #define this_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg64_op(8, volatile, pcp, oval, nval) 352 353 #define percpu_try_cmpxchg64_op(size, qual, _var, _ovalp, _nval) \ 354 ({ \ 355 bool success; \ 356 u64 *_oval = (u64 *)(_ovalp); \ 357 union { \ 358 u64 var; \ 359 struct { \ 360 u32 low, high; \ 361 }; \ 362 } old__, new__; \ 363 \ 364 old__.var = *_oval; \ 365 new__.var = _nval; \ 366 \ 367 asm_inline qual ( \ 368 ALTERNATIVE("call this_cpu_cmpxchg8b_emu", \ 369 "cmpxchg8b " __percpu_arg([var]), X86_FEATURE_CX8) \ 370 CC_SET(z) \ 371 : ALT_OUTPUT_SP(CC_OUT(z) (success), \ 372 [var] "+m" (__my_cpu_var(_var)), \ 373 "+a" (old__.low), "+d" (old__.high)) \ 374 : "b" (new__.low), "c" (new__.high), \ 375 "S" (&(_var)) \ 376 : "memory"); \ 377 if (unlikely(!success)) \ 378 *_oval = old__.var; \ 379 \ 380 likely(success); \ 381 }) 382 383 #define raw_cpu_try_cmpxchg64(pcp, ovalp, nval) percpu_try_cmpxchg64_op(8, , pcp, ovalp, nval) 384 #define this_cpu_try_cmpxchg64(pcp, ovalp, nval) percpu_try_cmpxchg64_op(8, volatile, pcp, ovalp, nval) 385 386 #endif /* defined(CONFIG_X86_32) && !defined(CONFIG_UML) */ 387 388 #ifdef CONFIG_X86_64 389 #define raw_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg_op(8, , pcp, oval, nval); 390 #define this_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg_op(8, volatile, pcp, oval, nval); 391 392 #define raw_cpu_try_cmpxchg64(pcp, ovalp, nval) percpu_try_cmpxchg_op(8, , pcp, ovalp, nval); 393 #define this_cpu_try_cmpxchg64(pcp, ovalp, nval) percpu_try_cmpxchg_op(8, volatile, pcp, ovalp, nval); 394 395 #define percpu_cmpxchg128_op(size, qual, _var, _oval, _nval) \ 396 ({ \ 397 union { \ 398 u128 var; \ 399 struct { \ 400 u64 low, high; \ 401 }; \ 402 } old__, new__; \ 403 \ 404 old__.var = _oval; \ 405 new__.var = _nval; \ 406 \ 407 asm_inline qual ( \ 408 ALTERNATIVE("call this_cpu_cmpxchg16b_emu", \ 409 "cmpxchg16b " __percpu_arg([var]), X86_FEATURE_CX16) \ 410 : ALT_OUTPUT_SP([var] "+m" (__my_cpu_var(_var)), \ 411 "+a" (old__.low), "+d" (old__.high)) \ 412 : "b" (new__.low), "c" (new__.high), \ 413 "S" (&(_var)) \ 414 : "memory"); \ 415 \ 416 old__.var; \ 417 }) 418 419 #define raw_cpu_cmpxchg128(pcp, oval, nval) percpu_cmpxchg128_op(16, , pcp, oval, nval) 420 #define this_cpu_cmpxchg128(pcp, oval, nval) percpu_cmpxchg128_op(16, volatile, pcp, oval, nval) 421 422 #define percpu_try_cmpxchg128_op(size, qual, _var, _ovalp, _nval) \ 423 ({ \ 424 bool success; \ 425 u128 *_oval = (u128 *)(_ovalp); \ 426 union { \ 427 u128 var; \ 428 struct { \ 429 u64 low, high; \ 430 }; \ 431 } old__, new__; \ 432 \ 433 old__.var = *_oval; \ 434 new__.var = _nval; \ 435 \ 436 asm_inline qual ( \ 437 ALTERNATIVE("call this_cpu_cmpxchg16b_emu", \ 438 "cmpxchg16b " __percpu_arg([var]), X86_FEATURE_CX16) \ 439 CC_SET(z) \ 440 : ALT_OUTPUT_SP(CC_OUT(z) (success), \ 441 [var] "+m" (__my_cpu_var(_var)), \ 442 "+a" (old__.low), "+d" (old__.high)) \ 443 : "b" (new__.low), "c" (new__.high), \ 444 "S" (&(_var)) \ 445 : "memory"); \ 446 if (unlikely(!success)) \ 447 *_oval = old__.var; \ 448 \ 449 likely(success); \ 450 }) 451 452 #define raw_cpu_try_cmpxchg128(pcp, ovalp, nval) percpu_try_cmpxchg128_op(16, , pcp, ovalp, nval) 453 #define this_cpu_try_cmpxchg128(pcp, ovalp, nval) percpu_try_cmpxchg128_op(16, volatile, pcp, ovalp, nval) 454 455 #endif /* CONFIG_X86_64 */ 456 457 #define raw_cpu_read_1(pcp) __raw_cpu_read(1, , pcp) 458 #define raw_cpu_read_2(pcp) __raw_cpu_read(2, , pcp) 459 #define raw_cpu_read_4(pcp) __raw_cpu_read(4, , pcp) 460 #define raw_cpu_write_1(pcp, val) __raw_cpu_write(1, , pcp, val) 461 #define raw_cpu_write_2(pcp, val) __raw_cpu_write(2, , pcp, val) 462 #define raw_cpu_write_4(pcp, val) __raw_cpu_write(4, , pcp, val) 463 464 #define this_cpu_read_1(pcp) __raw_cpu_read(1, volatile, pcp) 465 #define this_cpu_read_2(pcp) __raw_cpu_read(2, volatile, pcp) 466 #define this_cpu_read_4(pcp) __raw_cpu_read(4, volatile, pcp) 467 #define this_cpu_write_1(pcp, val) __raw_cpu_write(1, volatile, pcp, val) 468 #define this_cpu_write_2(pcp, val) __raw_cpu_write(2, volatile, pcp, val) 469 #define this_cpu_write_4(pcp, val) __raw_cpu_write(4, volatile, pcp, val) 470 471 #define this_cpu_read_stable_1(pcp) __raw_cpu_read_stable(1, pcp) 472 #define this_cpu_read_stable_2(pcp) __raw_cpu_read_stable(2, pcp) 473 #define this_cpu_read_stable_4(pcp) __raw_cpu_read_stable(4, pcp) 474 475 #define raw_cpu_add_1(pcp, val) percpu_add_op(1, , (pcp), val) 476 #define raw_cpu_add_2(pcp, val) percpu_add_op(2, , (pcp), val) 477 #define raw_cpu_add_4(pcp, val) percpu_add_op(4, , (pcp), val) 478 #define raw_cpu_and_1(pcp, val) percpu_binary_op(1, , "and", (pcp), val) 479 #define raw_cpu_and_2(pcp, val) percpu_binary_op(2, , "and", (pcp), val) 480 #define raw_cpu_and_4(pcp, val) percpu_binary_op(4, , "and", (pcp), val) 481 #define raw_cpu_or_1(pcp, val) percpu_binary_op(1, , "or", (pcp), val) 482 #define raw_cpu_or_2(pcp, val) percpu_binary_op(2, , "or", (pcp), val) 483 #define raw_cpu_or_4(pcp, val) percpu_binary_op(4, , "or", (pcp), val) 484 #define raw_cpu_xchg_1(pcp, val) raw_percpu_xchg_op(pcp, val) 485 #define raw_cpu_xchg_2(pcp, val) raw_percpu_xchg_op(pcp, val) 486 #define raw_cpu_xchg_4(pcp, val) raw_percpu_xchg_op(pcp, val) 487 488 #define this_cpu_add_1(pcp, val) percpu_add_op(1, volatile, (pcp), val) 489 #define this_cpu_add_2(pcp, val) percpu_add_op(2, volatile, (pcp), val) 490 #define this_cpu_add_4(pcp, val) percpu_add_op(4, volatile, (pcp), val) 491 #define this_cpu_and_1(pcp, val) percpu_binary_op(1, volatile, "and", (pcp), val) 492 #define this_cpu_and_2(pcp, val) percpu_binary_op(2, volatile, "and", (pcp), val) 493 #define this_cpu_and_4(pcp, val) percpu_binary_op(4, volatile, "and", (pcp), val) 494 #define this_cpu_or_1(pcp, val) percpu_binary_op(1, volatile, "or", (pcp), val) 495 #define this_cpu_or_2(pcp, val) percpu_binary_op(2, volatile, "or", (pcp), val) 496 #define this_cpu_or_4(pcp, val) percpu_binary_op(4, volatile, "or", (pcp), val) 497 #define this_cpu_xchg_1(pcp, nval) this_percpu_xchg_op(pcp, nval) 498 #define this_cpu_xchg_2(pcp, nval) this_percpu_xchg_op(pcp, nval) 499 #define this_cpu_xchg_4(pcp, nval) this_percpu_xchg_op(pcp, nval) 500 501 #define raw_cpu_add_return_1(pcp, val) percpu_add_return_op(1, , pcp, val) 502 #define raw_cpu_add_return_2(pcp, val) percpu_add_return_op(2, , pcp, val) 503 #define raw_cpu_add_return_4(pcp, val) percpu_add_return_op(4, , pcp, val) 504 #define raw_cpu_cmpxchg_1(pcp, oval, nval) percpu_cmpxchg_op(1, , pcp, oval, nval) 505 #define raw_cpu_cmpxchg_2(pcp, oval, nval) percpu_cmpxchg_op(2, , pcp, oval, nval) 506 #define raw_cpu_cmpxchg_4(pcp, oval, nval) percpu_cmpxchg_op(4, , pcp, oval, nval) 507 #define raw_cpu_try_cmpxchg_1(pcp, ovalp, nval) percpu_try_cmpxchg_op(1, , pcp, ovalp, nval) 508 #define raw_cpu_try_cmpxchg_2(pcp, ovalp, nval) percpu_try_cmpxchg_op(2, , pcp, ovalp, nval) 509 #define raw_cpu_try_cmpxchg_4(pcp, ovalp, nval) percpu_try_cmpxchg_op(4, , pcp, ovalp, nval) 510 511 #define this_cpu_add_return_1(pcp, val) percpu_add_return_op(1, volatile, pcp, val) 512 #define this_cpu_add_return_2(pcp, val) percpu_add_return_op(2, volatile, pcp, val) 513 #define this_cpu_add_return_4(pcp, val) percpu_add_return_op(4, volatile, pcp, val) 514 #define this_cpu_cmpxchg_1(pcp, oval, nval) percpu_cmpxchg_op(1, volatile, pcp, oval, nval) 515 #define this_cpu_cmpxchg_2(pcp, oval, nval) percpu_cmpxchg_op(2, volatile, pcp, oval, nval) 516 #define this_cpu_cmpxchg_4(pcp, oval, nval) percpu_cmpxchg_op(4, volatile, pcp, oval, nval) 517 #define this_cpu_try_cmpxchg_1(pcp, ovalp, nval) percpu_try_cmpxchg_op(1, volatile, pcp, ovalp, nval) 518 #define this_cpu_try_cmpxchg_2(pcp, ovalp, nval) percpu_try_cmpxchg_op(2, volatile, pcp, ovalp, nval) 519 #define this_cpu_try_cmpxchg_4(pcp, ovalp, nval) percpu_try_cmpxchg_op(4, volatile, pcp, ovalp, nval) 520 521 /* 522 * Per-CPU atomic 64-bit operations are only available under 64-bit kernels. 523 * 32-bit kernels must fall back to generic operations. 524 */ 525 #ifdef CONFIG_X86_64 526 527 #define raw_cpu_read_8(pcp) __raw_cpu_read(8, , pcp) 528 #define raw_cpu_write_8(pcp, val) __raw_cpu_write(8, , pcp, val) 529 530 #define this_cpu_read_8(pcp) __raw_cpu_read(8, volatile, pcp) 531 #define this_cpu_write_8(pcp, val) __raw_cpu_write(8, volatile, pcp, val) 532 533 #define this_cpu_read_stable_8(pcp) __raw_cpu_read_stable(8, pcp) 534 535 #define raw_cpu_add_8(pcp, val) percpu_add_op(8, , (pcp), val) 536 #define raw_cpu_and_8(pcp, val) percpu_binary_op(8, , "and", (pcp), val) 537 #define raw_cpu_or_8(pcp, val) percpu_binary_op(8, , "or", (pcp), val) 538 #define raw_cpu_add_return_8(pcp, val) percpu_add_return_op(8, , pcp, val) 539 #define raw_cpu_xchg_8(pcp, nval) raw_percpu_xchg_op(pcp, nval) 540 #define raw_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(8, , pcp, oval, nval) 541 #define raw_cpu_try_cmpxchg_8(pcp, ovalp, nval) percpu_try_cmpxchg_op(8, , pcp, ovalp, nval) 542 543 #define this_cpu_add_8(pcp, val) percpu_add_op(8, volatile, (pcp), val) 544 #define this_cpu_and_8(pcp, val) percpu_binary_op(8, volatile, "and", (pcp), val) 545 #define this_cpu_or_8(pcp, val) percpu_binary_op(8, volatile, "or", (pcp), val) 546 #define this_cpu_add_return_8(pcp, val) percpu_add_return_op(8, volatile, pcp, val) 547 #define this_cpu_xchg_8(pcp, nval) this_percpu_xchg_op(pcp, nval) 548 #define this_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(8, volatile, pcp, oval, nval) 549 #define this_cpu_try_cmpxchg_8(pcp, ovalp, nval) percpu_try_cmpxchg_op(8, volatile, pcp, ovalp, nval) 550 551 #define raw_cpu_read_long(pcp) raw_cpu_read_8(pcp) 552 553 #else /* !CONFIG_X86_64: */ 554 555 /* There is no generic 64-bit read stable operation for 32-bit targets. */ 556 #define this_cpu_read_stable_8(pcp) ({ BUILD_BUG(); (typeof(pcp))0; }) 557 558 #define raw_cpu_read_long(pcp) raw_cpu_read_4(pcp) 559 560 #endif /* CONFIG_X86_64 */ 561 562 #define this_cpu_read_const(pcp) __raw_cpu_read_const(pcp) 563 564 /* 565 * this_cpu_read() makes the compiler load the per-CPU variable every time 566 * it is accessed while this_cpu_read_stable() allows the value to be cached. 567 * this_cpu_read_stable() is more efficient and can be used if its value 568 * is guaranteed to be valid across CPUs. The current users include 569 * current_task and cpu_current_top_of_stack, both of which are 570 * actually per-thread variables implemented as per-CPU variables and 571 * thus stable for the duration of the respective task. 572 */ 573 #define this_cpu_read_stable(pcp) __pcpu_size_call_return(this_cpu_read_stable_, pcp) 574 575 #define x86_this_cpu_constant_test_bit(_nr, _var) \ 576 ({ \ 577 unsigned long __percpu *addr__ = \ 578 (unsigned long __percpu *)&(_var) + ((_nr) / BITS_PER_LONG); \ 579 \ 580 !!((1UL << ((_nr) % BITS_PER_LONG)) & raw_cpu_read(*addr__)); \ 581 }) 582 583 #define x86_this_cpu_variable_test_bit(_nr, _var) \ 584 ({ \ 585 bool oldbit; \ 586 \ 587 asm volatile("btl %[nr], " __percpu_arg([var]) \ 588 CC_SET(c) \ 589 : CC_OUT(c) (oldbit) \ 590 : [var] "m" (__my_cpu_var(_var)), \ 591 [nr] "rI" (_nr)); \ 592 oldbit; \ 593 }) 594 595 #define x86_this_cpu_test_bit(_nr, _var) \ 596 (__builtin_constant_p(_nr) \ 597 ? x86_this_cpu_constant_test_bit(_nr, _var) \ 598 : x86_this_cpu_variable_test_bit(_nr, _var)) 599 600 601 #include <asm-generic/percpu.h> 602 603 /* We can use this directly for local CPU (faster). */ 604 DECLARE_PER_CPU_CACHE_HOT(unsigned long, this_cpu_off); 605 606 #endif /* !__ASSEMBLER__ */ 607 608 #ifdef CONFIG_SMP 609 610 /* 611 * Define the "EARLY_PER_CPU" macros. These are used for some per_cpu 612 * variables that are initialized and accessed before there are per_cpu 613 * areas allocated. 614 */ 615 616 #define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue) \ 617 DEFINE_PER_CPU(_type, _name) = _initvalue; \ 618 __typeof__(_type) _name##_early_map[NR_CPUS] __initdata = \ 619 { [0 ... NR_CPUS-1] = _initvalue }; \ 620 __typeof__(_type) *_name##_early_ptr __refdata = _name##_early_map 621 622 #define DEFINE_EARLY_PER_CPU_READ_MOSTLY(_type, _name, _initvalue) \ 623 DEFINE_PER_CPU_READ_MOSTLY(_type, _name) = _initvalue; \ 624 __typeof__(_type) _name##_early_map[NR_CPUS] __initdata = \ 625 { [0 ... NR_CPUS-1] = _initvalue }; \ 626 __typeof__(_type) *_name##_early_ptr __refdata = _name##_early_map 627 628 #define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \ 629 EXPORT_PER_CPU_SYMBOL(_name) 630 631 #define DECLARE_EARLY_PER_CPU(_type, _name) \ 632 DECLARE_PER_CPU(_type, _name); \ 633 extern __typeof__(_type) *_name##_early_ptr; \ 634 extern __typeof__(_type) _name##_early_map[] 635 636 #define DECLARE_EARLY_PER_CPU_READ_MOSTLY(_type, _name) \ 637 DECLARE_PER_CPU_READ_MOSTLY(_type, _name); \ 638 extern __typeof__(_type) *_name##_early_ptr; \ 639 extern __typeof__(_type) _name##_early_map[] 640 641 #define early_per_cpu_ptr(_name) (_name##_early_ptr) 642 #define early_per_cpu_map(_name, _idx) (_name##_early_map[_idx]) 643 644 #define early_per_cpu(_name, _cpu) \ 645 *(early_per_cpu_ptr(_name) ? \ 646 &early_per_cpu_ptr(_name)[_cpu] : \ 647 &per_cpu(_name, _cpu)) 648 649 #else /* !CONFIG_SMP: */ 650 #define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue) \ 651 DEFINE_PER_CPU(_type, _name) = _initvalue 652 653 #define DEFINE_EARLY_PER_CPU_READ_MOSTLY(_type, _name, _initvalue) \ 654 DEFINE_PER_CPU_READ_MOSTLY(_type, _name) = _initvalue 655 656 #define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \ 657 EXPORT_PER_CPU_SYMBOL(_name) 658 659 #define DECLARE_EARLY_PER_CPU(_type, _name) \ 660 DECLARE_PER_CPU(_type, _name) 661 662 #define DECLARE_EARLY_PER_CPU_READ_MOSTLY(_type, _name) \ 663 DECLARE_PER_CPU_READ_MOSTLY(_type, _name) 664 665 #define early_per_cpu(_name, _cpu) per_cpu(_name, _cpu) 666 #define early_per_cpu_ptr(_name) NULL 667 /* no early_per_cpu_map() */ 668 669 #endif /* !CONFIG_SMP */ 670 671 #endif /* _ASM_X86_PERCPU_H */ 672