1 #ifndef _X86_PROCESSOR_H_ 2 #define _X86_PROCESSOR_H_ 3 4 #include "libcflat.h" 5 #include "desc.h" 6 #include "msr.h" 7 #include <bitops.h> 8 #include <stdint.h> 9 10 #define NONCANONICAL 0xaaaaaaaaaaaaaaaaull 11 12 #ifdef __x86_64__ 13 # define R "r" 14 # define W "q" 15 # define S "8" 16 #else 17 # define R "e" 18 # define W "l" 19 # define S "4" 20 #endif 21 22 #define DB_VECTOR 1 23 #define BP_VECTOR 3 24 #define UD_VECTOR 6 25 #define DF_VECTOR 8 26 #define TS_VECTOR 10 27 #define NP_VECTOR 11 28 #define SS_VECTOR 12 29 #define GP_VECTOR 13 30 #define PF_VECTOR 14 31 #define AC_VECTOR 17 32 #define CP_VECTOR 21 33 34 #define X86_CR0_PE BIT(0) 35 #define X86_CR0_MP BIT(1) 36 #define X86_CR0_EM BIT(2) 37 #define X86_CR0_TS BIT(3) 38 #define X86_CR0_ET BIT(4) 39 #define X86_CR0_NE BIT(5) 40 #define X86_CR0_WP BIT(16) 41 #define X86_CR0_AM BIT(18) 42 #define X86_CR0_NW BIT(29) 43 #define X86_CR0_CD BIT(30) 44 #define X86_CR0_PG BIT(31) 45 46 #define X86_CR3_PCID_MASK GENMASK(11, 0) 47 48 #define X86_CR4_VME BIT(0) 49 #define X86_CR4_PVI BIT(1) 50 #define X86_CR4_TSD BIT(2) 51 #define X86_CR4_DE BIT(3) 52 #define X86_CR4_PSE BIT(4) 53 #define X86_CR4_PAE BIT(5) 54 #define X86_CR4_MCE BIT(6) 55 #define X86_CR4_PGE BIT(7) 56 #define X86_CR4_PCE BIT(8) 57 #define X86_CR4_OSFXSR BIT(9) 58 #define X86_CR4_OSXMMEXCPT BIT(10) 59 #define X86_CR4_UMIP BIT(11) 60 #define X86_CR4_LA57 BIT(12) 61 #define X86_CR4_VMXE BIT(13) 62 #define X86_CR4_SMXE BIT(14) 63 /* UNUSED BIT(15) */ 64 #define X86_CR4_FSGSBASE BIT(16) 65 #define X86_CR4_PCIDE BIT(17) 66 #define X86_CR4_OSXSAVE BIT(18) 67 #define X86_CR4_KL BIT(19) 68 #define X86_CR4_SMEP BIT(20) 69 #define X86_CR4_SMAP BIT(21) 70 #define X86_CR4_PKE BIT(22) 71 #define X86_CR4_CET BIT(23) 72 #define X86_CR4_PKS BIT(24) 73 74 #define X86_EFLAGS_CF BIT(0) 75 #define X86_EFLAGS_FIXED BIT(1) 76 #define X86_EFLAGS_PF BIT(2) 77 /* RESERVED 0 BIT(3) */ 78 #define X86_EFLAGS_AF BIT(4) 79 /* RESERVED 0 BIT(5) */ 80 #define X86_EFLAGS_ZF BIT(6) 81 #define X86_EFLAGS_SF BIT(7) 82 #define X86_EFLAGS_TF BIT(8) 83 #define X86_EFLAGS_IF BIT(9) 84 #define X86_EFLAGS_DF BIT(10) 85 #define X86_EFLAGS_OF BIT(11) 86 #define X86_EFLAGS_IOPL GENMASK(13, 12) 87 #define X86_EFLAGS_NT BIT(14) 88 /* RESERVED 0 BIT(15) */ 89 #define X86_EFLAGS_RF BIT(16) 90 #define X86_EFLAGS_VM BIT(17) 91 #define X86_EFLAGS_AC BIT(18) 92 #define X86_EFLAGS_VIF BIT(19) 93 #define X86_EFLAGS_VIP BIT(20) 94 #define X86_EFLAGS_ID BIT(21) 95 96 #define X86_EFLAGS_ALU (X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | \ 97 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF) 98 99 100 /* 101 * CPU features 102 */ 103 104 enum cpuid_output_regs { 105 EAX, 106 EBX, 107 ECX, 108 EDX 109 }; 110 111 struct cpuid { u32 a, b, c, d; }; 112 113 static inline struct cpuid raw_cpuid(u32 function, u32 index) 114 { 115 struct cpuid r; 116 asm volatile ("cpuid" 117 : "=a"(r.a), "=b"(r.b), "=c"(r.c), "=d"(r.d) 118 : "0"(function), "2"(index)); 119 return r; 120 } 121 122 static inline struct cpuid cpuid_indexed(u32 function, u32 index) 123 { 124 u32 level = raw_cpuid(function & 0xf0000000, 0).a; 125 if (level < function) 126 return (struct cpuid) { 0, 0, 0, 0 }; 127 return raw_cpuid(function, index); 128 } 129 130 static inline struct cpuid cpuid(u32 function) 131 { 132 return cpuid_indexed(function, 0); 133 } 134 135 static inline u8 cpuid_maxphyaddr(void) 136 { 137 if (raw_cpuid(0x80000000, 0).a < 0x80000008) 138 return 36; 139 return raw_cpuid(0x80000008, 0).a & 0xff; 140 } 141 142 static inline bool is_intel(void) 143 { 144 struct cpuid c = cpuid(0); 145 u32 name[4] = {c.b, c.d, c.c }; 146 147 return strcmp((char *)name, "GenuineIntel") == 0; 148 } 149 150 #define CPUID(a, b, c, d) ((((unsigned long long) a) << 32) | (b << 16) | \ 151 (c << 8) | d) 152 153 /* 154 * Each X86_FEATURE_XXX definition is 64-bit and contains the following 155 * CPUID meta-data: 156 * 157 * [63:32] : input value for EAX 158 * [31:16] : input value for ECX 159 * [15:8] : output register 160 * [7:0] : bit position in output register 161 */ 162 163 /* 164 * Basic Leafs, a.k.a. Intel defined 165 */ 166 #define X86_FEATURE_MWAIT (CPUID(0x1, 0, ECX, 3)) 167 #define X86_FEATURE_VMX (CPUID(0x1, 0, ECX, 5)) 168 #define X86_FEATURE_PCID (CPUID(0x1, 0, ECX, 17)) 169 #define X86_FEATURE_MOVBE (CPUID(0x1, 0, ECX, 22)) 170 #define X86_FEATURE_TSC_DEADLINE_TIMER (CPUID(0x1, 0, ECX, 24)) 171 #define X86_FEATURE_XSAVE (CPUID(0x1, 0, ECX, 26)) 172 #define X86_FEATURE_OSXSAVE (CPUID(0x1, 0, ECX, 27)) 173 #define X86_FEATURE_RDRAND (CPUID(0x1, 0, ECX, 30)) 174 #define X86_FEATURE_MCE (CPUID(0x1, 0, EDX, 7)) 175 #define X86_FEATURE_APIC (CPUID(0x1, 0, EDX, 9)) 176 #define X86_FEATURE_CLFLUSH (CPUID(0x1, 0, EDX, 19)) 177 #define X86_FEATURE_XMM (CPUID(0x1, 0, EDX, 25)) 178 #define X86_FEATURE_XMM2 (CPUID(0x1, 0, EDX, 26)) 179 #define X86_FEATURE_TSC_ADJUST (CPUID(0x7, 0, EBX, 1)) 180 #define X86_FEATURE_HLE (CPUID(0x7, 0, EBX, 4)) 181 #define X86_FEATURE_SMEP (CPUID(0x7, 0, EBX, 7)) 182 #define X86_FEATURE_INVPCID (CPUID(0x7, 0, EBX, 10)) 183 #define X86_FEATURE_RTM (CPUID(0x7, 0, EBX, 11)) 184 #define X86_FEATURE_SMAP (CPUID(0x7, 0, EBX, 20)) 185 #define X86_FEATURE_PCOMMIT (CPUID(0x7, 0, EBX, 22)) 186 #define X86_FEATURE_CLFLUSHOPT (CPUID(0x7, 0, EBX, 23)) 187 #define X86_FEATURE_CLWB (CPUID(0x7, 0, EBX, 24)) 188 #define X86_FEATURE_UMIP (CPUID(0x7, 0, ECX, 2)) 189 #define X86_FEATURE_PKU (CPUID(0x7, 0, ECX, 3)) 190 #define X86_FEATURE_LA57 (CPUID(0x7, 0, ECX, 16)) 191 #define X86_FEATURE_RDPID (CPUID(0x7, 0, ECX, 22)) 192 #define X86_FEATURE_SHSTK (CPUID(0x7, 0, ECX, 7)) 193 #define X86_FEATURE_IBT (CPUID(0x7, 0, EDX, 20)) 194 #define X86_FEATURE_SPEC_CTRL (CPUID(0x7, 0, EDX, 26)) 195 #define X86_FEATURE_ARCH_CAPABILITIES (CPUID(0x7, 0, EDX, 29)) 196 #define X86_FEATURE_PKS (CPUID(0x7, 0, ECX, 31)) 197 198 /* 199 * Extended Leafs, a.k.a. AMD defined 200 */ 201 #define X86_FEATURE_SVM (CPUID(0x80000001, 0, ECX, 2)) 202 #define X86_FEATURE_NX (CPUID(0x80000001, 0, EDX, 20)) 203 #define X86_FEATURE_GBPAGES (CPUID(0x80000001, 0, EDX, 26)) 204 #define X86_FEATURE_RDTSCP (CPUID(0x80000001, 0, EDX, 27)) 205 #define X86_FEATURE_LM (CPUID(0x80000001, 0, EDX, 29)) 206 #define X86_FEATURE_RDPRU (CPUID(0x80000008, 0, EBX, 4)) 207 #define X86_FEATURE_AMD_IBPB (CPUID(0x80000008, 0, EBX, 12)) 208 #define X86_FEATURE_NPT (CPUID(0x8000000A, 0, EDX, 0)) 209 #define X86_FEATURE_LBRV (CPUID(0x8000000A, 0, EDX, 1)) 210 #define X86_FEATURE_NRIPS (CPUID(0x8000000A, 0, EDX, 3)) 211 #define X86_FEATURE_TSCRATEMSR (CPUID(0x8000000A, 0, EDX, 4)) 212 #define X86_FEATURE_PAUSEFILTER (CPUID(0x8000000A, 0, EDX, 10)) 213 #define X86_FEATURE_PFTHRESHOLD (CPUID(0x8000000A, 0, EDX, 12)) 214 #define X86_FEATURE_VGIF (CPUID(0x8000000A, 0, EDX, 16)) 215 216 217 static inline bool this_cpu_has(u64 feature) 218 { 219 u32 input_eax = feature >> 32; 220 u32 input_ecx = (feature >> 16) & 0xffff; 221 u32 output_reg = (feature >> 8) & 0xff; 222 u8 bit = feature & 0xff; 223 struct cpuid c; 224 u32 *tmp; 225 226 c = cpuid_indexed(input_eax, input_ecx); 227 tmp = (u32 *)&c; 228 229 return ((*(tmp + (output_reg % 32))) & (1 << bit)); 230 } 231 232 struct far_pointer32 { 233 u32 offset; 234 u16 selector; 235 } __attribute__((packed)); 236 237 struct descriptor_table_ptr { 238 u16 limit; 239 ulong base; 240 } __attribute__((packed)); 241 242 static inline void clac(void) 243 { 244 asm volatile (".byte 0x0f, 0x01, 0xca" : : : "memory"); 245 } 246 247 static inline void stac(void) 248 { 249 asm volatile (".byte 0x0f, 0x01, 0xcb" : : : "memory"); 250 } 251 252 static inline u16 read_cs(void) 253 { 254 unsigned val; 255 256 asm volatile ("mov %%cs, %0" : "=mr"(val)); 257 return val; 258 } 259 260 static inline u16 read_ds(void) 261 { 262 unsigned val; 263 264 asm volatile ("mov %%ds, %0" : "=mr"(val)); 265 return val; 266 } 267 268 static inline u16 read_es(void) 269 { 270 unsigned val; 271 272 asm volatile ("mov %%es, %0" : "=mr"(val)); 273 return val; 274 } 275 276 static inline u16 read_ss(void) 277 { 278 unsigned val; 279 280 asm volatile ("mov %%ss, %0" : "=mr"(val)); 281 return val; 282 } 283 284 static inline u16 read_fs(void) 285 { 286 unsigned val; 287 288 asm volatile ("mov %%fs, %0" : "=mr"(val)); 289 return val; 290 } 291 292 static inline u16 read_gs(void) 293 { 294 unsigned val; 295 296 asm volatile ("mov %%gs, %0" : "=mr"(val)); 297 return val; 298 } 299 300 static inline unsigned long read_rflags(void) 301 { 302 unsigned long f; 303 asm volatile ("pushf; pop %0\n\t" : "=rm"(f)); 304 return f; 305 } 306 307 static inline void write_ds(unsigned val) 308 { 309 asm volatile ("mov %0, %%ds" : : "rm"(val) : "memory"); 310 } 311 312 static inline void write_es(unsigned val) 313 { 314 asm volatile ("mov %0, %%es" : : "rm"(val) : "memory"); 315 } 316 317 static inline void write_ss(unsigned val) 318 { 319 asm volatile ("mov %0, %%ss" : : "rm"(val) : "memory"); 320 } 321 322 static inline void write_fs(unsigned val) 323 { 324 asm volatile ("mov %0, %%fs" : : "rm"(val) : "memory"); 325 } 326 327 static inline void write_gs(unsigned val) 328 { 329 asm volatile ("mov %0, %%gs" : : "rm"(val) : "memory"); 330 } 331 332 static inline void write_rflags(unsigned long f) 333 { 334 asm volatile ("push %0; popf\n\t" : : "rm"(f)); 335 } 336 337 static inline void set_iopl(int iopl) 338 { 339 unsigned long flags = read_rflags() & ~X86_EFLAGS_IOPL; 340 flags |= iopl * (X86_EFLAGS_IOPL / 3); 341 write_rflags(flags); 342 } 343 344 /* 345 * Don't use the safe variants for rdmsr() or wrmsr(). The exception fixup 346 * infrastructure uses per-CPU data and thus consumes GS.base. Various tests 347 * temporarily modify MSR_GS_BASE and will explode when trying to determine 348 * whether or not RDMSR/WRMSR faulted. 349 */ 350 static inline u64 rdmsr(u32 index) 351 { 352 u32 a, d; 353 asm volatile ("rdmsr" : "=a"(a), "=d"(d) : "c"(index) : "memory"); 354 return a | ((u64)d << 32); 355 } 356 357 static inline void wrmsr(u32 index, u64 val) 358 { 359 u32 a = val, d = val >> 32; 360 asm volatile ("wrmsr" : : "a"(a), "d"(d), "c"(index) : "memory"); 361 } 362 363 static inline int rdmsr_safe(u32 index, uint64_t *val) 364 { 365 uint32_t a, d; 366 367 asm volatile (ASM_TRY("1f") 368 "rdmsr\n\t" 369 "1:" 370 : "=a"(a), "=d"(d) 371 : "c"(index) : "memory"); 372 373 *val = (uint64_t)a | ((uint64_t)d << 32); 374 return exception_vector(); 375 } 376 377 static inline int wrmsr_safe(u32 index, u64 val) 378 { 379 u32 a = val, d = val >> 32; 380 381 asm volatile (ASM_TRY("1f") 382 "wrmsr\n\t" 383 "1:" 384 : : "a"(a), "d"(d), "c"(index) : "memory"); 385 return exception_vector(); 386 } 387 388 static inline uint64_t rdpmc(uint32_t index) 389 { 390 uint32_t a, d; 391 asm volatile ("rdpmc" : "=a"(a), "=d"(d) : "c"(index)); 392 return a | ((uint64_t)d << 32); 393 } 394 395 static inline int write_cr0_safe(ulong val) 396 { 397 asm volatile(ASM_TRY("1f") 398 "mov %0,%%cr0\n\t" 399 "1:": : "r" (val)); 400 return exception_vector(); 401 } 402 403 static inline void write_cr0(ulong val) 404 { 405 int vector = write_cr0_safe(val); 406 407 assert_msg(!vector, "Unexpected fault '%d' writing CR0 = %lx", 408 vector, val); 409 } 410 411 static inline ulong read_cr0(void) 412 { 413 ulong val; 414 asm volatile ("mov %%cr0, %0" : "=r"(val) : : "memory"); 415 return val; 416 } 417 418 static inline void write_cr2(ulong val) 419 { 420 asm volatile ("mov %0, %%cr2" : : "r"(val) : "memory"); 421 } 422 423 static inline ulong read_cr2(void) 424 { 425 ulong val; 426 asm volatile ("mov %%cr2, %0" : "=r"(val) : : "memory"); 427 return val; 428 } 429 430 static inline int write_cr3_safe(ulong val) 431 { 432 asm volatile(ASM_TRY("1f") 433 "mov %0,%%cr3\n\t" 434 "1:": : "r" (val)); 435 return exception_vector(); 436 } 437 438 static inline void write_cr3(ulong val) 439 { 440 int vector = write_cr3_safe(val); 441 442 assert_msg(!vector, "Unexpected fault '%d' writing CR3 = %lx", 443 vector, val); 444 } 445 446 static inline ulong read_cr3(void) 447 { 448 ulong val; 449 asm volatile ("mov %%cr3, %0" : "=r"(val) : : "memory"); 450 return val; 451 } 452 453 static inline void update_cr3(void *cr3) 454 { 455 write_cr3((ulong)cr3); 456 } 457 458 static inline int write_cr4_safe(ulong val) 459 { 460 asm volatile(ASM_TRY("1f") 461 "mov %0,%%cr4\n\t" 462 "1:": : "r" (val)); 463 return exception_vector(); 464 } 465 466 static inline void write_cr4(ulong val) 467 { 468 int vector = write_cr4_safe(val); 469 470 assert_msg(!vector, "Unexpected fault '%d' writing CR4 = %lx", 471 vector, val); 472 } 473 474 static inline ulong read_cr4(void) 475 { 476 ulong val; 477 asm volatile ("mov %%cr4, %0" : "=r"(val) : : "memory"); 478 return val; 479 } 480 481 static inline void write_cr8(ulong val) 482 { 483 asm volatile ("mov %0, %%cr8" : : "r"(val) : "memory"); 484 } 485 486 static inline ulong read_cr8(void) 487 { 488 ulong val; 489 asm volatile ("mov %%cr8, %0" : "=r"(val) : : "memory"); 490 return val; 491 } 492 493 static inline void lgdt(const struct descriptor_table_ptr *ptr) 494 { 495 asm volatile ("lgdt %0" : : "m"(*ptr)); 496 } 497 498 static inline void sgdt(struct descriptor_table_ptr *ptr) 499 { 500 asm volatile ("sgdt %0" : "=m"(*ptr)); 501 } 502 503 static inline void lidt(const struct descriptor_table_ptr *ptr) 504 { 505 asm volatile ("lidt %0" : : "m"(*ptr)); 506 } 507 508 static inline void sidt(struct descriptor_table_ptr *ptr) 509 { 510 asm volatile ("sidt %0" : "=m"(*ptr)); 511 } 512 513 static inline void lldt(u16 val) 514 { 515 asm volatile ("lldt %0" : : "rm"(val)); 516 } 517 518 static inline u16 sldt(void) 519 { 520 u16 val; 521 asm volatile ("sldt %0" : "=rm"(val)); 522 return val; 523 } 524 525 static inline void ltr(u16 val) 526 { 527 asm volatile ("ltr %0" : : "rm"(val)); 528 } 529 530 static inline u16 str(void) 531 { 532 u16 val; 533 asm volatile ("str %0" : "=rm"(val)); 534 return val; 535 } 536 537 static inline void write_dr0(void *val) 538 { 539 asm volatile ("mov %0, %%dr0" : : "r"(val) : "memory"); 540 } 541 542 static inline void write_dr1(void *val) 543 { 544 asm volatile ("mov %0, %%dr1" : : "r"(val) : "memory"); 545 } 546 547 static inline void write_dr2(void *val) 548 { 549 asm volatile ("mov %0, %%dr2" : : "r"(val) : "memory"); 550 } 551 552 static inline void write_dr3(void *val) 553 { 554 asm volatile ("mov %0, %%dr3" : : "r"(val) : "memory"); 555 } 556 557 static inline void write_dr6(ulong val) 558 { 559 asm volatile ("mov %0, %%dr6" : : "r"(val) : "memory"); 560 } 561 562 static inline ulong read_dr6(void) 563 { 564 ulong val; 565 asm volatile ("mov %%dr6, %0" : "=r"(val)); 566 return val; 567 } 568 569 static inline void write_dr7(ulong val) 570 { 571 asm volatile ("mov %0, %%dr7" : : "r"(val) : "memory"); 572 } 573 574 static inline ulong read_dr7(void) 575 { 576 ulong val; 577 asm volatile ("mov %%dr7, %0" : "=r"(val)); 578 return val; 579 } 580 581 static inline void pause(void) 582 { 583 asm volatile ("pause"); 584 } 585 586 static inline void cli(void) 587 { 588 asm volatile ("cli"); 589 } 590 591 static inline void sti(void) 592 { 593 asm volatile ("sti"); 594 } 595 596 static inline unsigned long long rdrand(void) 597 { 598 long long r; 599 600 asm volatile("rdrand %0\n\t" 601 "jc 1f\n\t" 602 "mov $0, %0\n\t" 603 "1:\n\t" : "=r" (r)); 604 return r; 605 } 606 607 static inline unsigned long long rdtsc(void) 608 { 609 long long r; 610 611 #ifdef __x86_64__ 612 unsigned a, d; 613 614 asm volatile ("rdtsc" : "=a"(a), "=d"(d)); 615 r = a | ((long long)d << 32); 616 #else 617 asm volatile ("rdtsc" : "=A"(r)); 618 #endif 619 return r; 620 } 621 622 /* 623 * Per the advice in the SDM, volume 2, the sequence "mfence; lfence" 624 * executed immediately before rdtsc ensures that rdtsc will be 625 * executed only after all previous instructions have executed and all 626 * previous loads and stores are globally visible. In addition, the 627 * lfence immediately after rdtsc ensures that rdtsc will be executed 628 * prior to the execution of any subsequent instruction. 629 */ 630 static inline unsigned long long fenced_rdtsc(void) 631 { 632 unsigned long long tsc; 633 634 #ifdef __x86_64__ 635 unsigned int eax, edx; 636 637 asm volatile ("mfence; lfence; rdtsc; lfence" : "=a"(eax), "=d"(edx)); 638 tsc = eax | ((unsigned long long)edx << 32); 639 #else 640 asm volatile ("mfence; lfence; rdtsc; lfence" : "=A"(tsc)); 641 #endif 642 return tsc; 643 } 644 645 static inline unsigned long long rdtscp(u32 *aux) 646 { 647 long long r; 648 649 #ifdef __x86_64__ 650 unsigned a, d; 651 652 asm volatile ("rdtscp" : "=a"(a), "=d"(d), "=c"(*aux)); 653 r = a | ((long long)d << 32); 654 #else 655 asm volatile ("rdtscp" : "=A"(r), "=c"(*aux)); 656 #endif 657 return r; 658 } 659 660 static inline void wrtsc(u64 tsc) 661 { 662 wrmsr(MSR_IA32_TSC, tsc); 663 } 664 665 static inline void irq_disable(void) 666 { 667 asm volatile("cli"); 668 } 669 670 /* Note that irq_enable() does not ensure an interrupt shadow due 671 * to the vagaries of compiler optimizations. If you need the 672 * shadow, use a single asm with "sti" and the instruction after it. 673 */ 674 static inline void irq_enable(void) 675 { 676 asm volatile("sti"); 677 } 678 679 static inline void invlpg(volatile void *va) 680 { 681 asm volatile("invlpg (%0)" ::"r" (va) : "memory"); 682 } 683 684 static inline void safe_halt(void) 685 { 686 asm volatile("sti; hlt"); 687 } 688 689 static inline u32 read_pkru(void) 690 { 691 unsigned int eax, edx; 692 unsigned int ecx = 0; 693 unsigned int pkru; 694 695 asm volatile(".byte 0x0f,0x01,0xee\n\t" 696 : "=a" (eax), "=d" (edx) 697 : "c" (ecx)); 698 pkru = eax; 699 return pkru; 700 } 701 702 static inline void write_pkru(u32 pkru) 703 { 704 unsigned int eax = pkru; 705 unsigned int ecx = 0; 706 unsigned int edx = 0; 707 708 asm volatile(".byte 0x0f,0x01,0xef\n\t" 709 : : "a" (eax), "c" (ecx), "d" (edx)); 710 } 711 712 static inline bool is_canonical(u64 addr) 713 { 714 int va_width = (raw_cpuid(0x80000008, 0).a & 0xff00) >> 8; 715 int shift_amt = 64 - va_width; 716 717 return (s64)(addr << shift_amt) >> shift_amt == addr; 718 } 719 720 static inline void clear_bit(int bit, u8 *addr) 721 { 722 __asm__ __volatile__("btr %1, %0" 723 : "+m" (*addr) : "Ir" (bit) : "cc", "memory"); 724 } 725 726 static inline void set_bit(int bit, u8 *addr) 727 { 728 __asm__ __volatile__("bts %1, %0" 729 : "+m" (*addr) : "Ir" (bit) : "cc", "memory"); 730 } 731 732 static inline void flush_tlb(void) 733 { 734 ulong cr4; 735 736 cr4 = read_cr4(); 737 write_cr4(cr4 ^ X86_CR4_PGE); 738 write_cr4(cr4); 739 } 740 741 static inline int has_spec_ctrl(void) 742 { 743 return !!(this_cpu_has(X86_FEATURE_SPEC_CTRL)); 744 } 745 746 static inline int cpu_has_efer_nx(void) 747 { 748 return !!(this_cpu_has(X86_FEATURE_NX)); 749 } 750 751 #endif 752