1 #ifndef _X86_PROCESSOR_H_ 2 #define _X86_PROCESSOR_H_ 3 4 #include "libcflat.h" 5 #include "desc.h" 6 #include "msr.h" 7 #include <stdint.h> 8 9 #define NONCANONICAL 0xaaaaaaaaaaaaaaaaull 10 11 #ifdef __x86_64__ 12 # define R "r" 13 # define W "q" 14 # define S "8" 15 #else 16 # define R "e" 17 # define W "l" 18 # define S "4" 19 #endif 20 21 #define DB_VECTOR 1 22 #define BP_VECTOR 3 23 #define UD_VECTOR 6 24 #define DF_VECTOR 8 25 #define TS_VECTOR 10 26 #define NP_VECTOR 11 27 #define SS_VECTOR 12 28 #define GP_VECTOR 13 29 #define PF_VECTOR 14 30 #define AC_VECTOR 17 31 #define CP_VECTOR 21 32 33 #define X86_CR0_PE 0x00000001 34 #define X86_CR0_MP 0x00000002 35 #define X86_CR0_EM 0x00000004 36 #define X86_CR0_TS 0x00000008 37 #define X86_CR0_WP 0x00010000 38 #define X86_CR0_AM 0x00040000 39 #define X86_CR0_NW 0x20000000 40 #define X86_CR0_CD 0x40000000 41 #define X86_CR0_PG 0x80000000 42 #define X86_CR3_PCID_MASK 0x00000fff 43 #define X86_CR4_TSD 0x00000004 44 #define X86_CR4_DE 0x00000008 45 #define X86_CR4_PSE 0x00000010 46 #define X86_CR4_PAE 0x00000020 47 #define X86_CR4_MCE 0x00000040 48 #define X86_CR4_PGE 0x00000080 49 #define X86_CR4_PCE 0x00000100 50 #define X86_CR4_UMIP 0x00000800 51 #define X86_CR4_LA57 0x00001000 52 #define X86_CR4_VMXE 0x00002000 53 #define X86_CR4_PCIDE 0x00020000 54 #define X86_CR4_OSXSAVE 0x00040000 55 #define X86_CR4_SMEP 0x00100000 56 #define X86_CR4_SMAP 0x00200000 57 #define X86_CR4_PKE 0x00400000 58 #define X86_CR4_CET 0x00800000 59 #define X86_CR4_PKS 0x01000000 60 61 #define X86_EFLAGS_CF 0x00000001 62 #define X86_EFLAGS_FIXED 0x00000002 63 #define X86_EFLAGS_PF 0x00000004 64 #define X86_EFLAGS_AF 0x00000010 65 #define X86_EFLAGS_ZF 0x00000040 66 #define X86_EFLAGS_SF 0x00000080 67 #define X86_EFLAGS_TF 0x00000100 68 #define X86_EFLAGS_IF 0x00000200 69 #define X86_EFLAGS_DF 0x00000400 70 #define X86_EFLAGS_OF 0x00000800 71 #define X86_EFLAGS_IOPL 0x00003000 72 #define X86_EFLAGS_NT 0x00004000 73 #define X86_EFLAGS_RF 0x00010000 74 #define X86_EFLAGS_VM 0x00020000 75 #define X86_EFLAGS_AC 0x00040000 76 77 #define X86_EFLAGS_ALU (X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | \ 78 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF) 79 80 81 /* 82 * CPU features 83 */ 84 85 enum cpuid_output_regs { 86 EAX, 87 EBX, 88 ECX, 89 EDX 90 }; 91 92 struct cpuid { u32 a, b, c, d; }; 93 94 static inline struct cpuid raw_cpuid(u32 function, u32 index) 95 { 96 struct cpuid r; 97 asm volatile ("cpuid" 98 : "=a"(r.a), "=b"(r.b), "=c"(r.c), "=d"(r.d) 99 : "0"(function), "2"(index)); 100 return r; 101 } 102 103 static inline struct cpuid cpuid_indexed(u32 function, u32 index) 104 { 105 u32 level = raw_cpuid(function & 0xf0000000, 0).a; 106 if (level < function) 107 return (struct cpuid) { 0, 0, 0, 0 }; 108 return raw_cpuid(function, index); 109 } 110 111 static inline struct cpuid cpuid(u32 function) 112 { 113 return cpuid_indexed(function, 0); 114 } 115 116 static inline u8 cpuid_maxphyaddr(void) 117 { 118 if (raw_cpuid(0x80000000, 0).a < 0x80000008) 119 return 36; 120 return raw_cpuid(0x80000008, 0).a & 0xff; 121 } 122 123 static inline bool is_intel(void) 124 { 125 struct cpuid c = cpuid(0); 126 u32 name[4] = {c.b, c.d, c.c }; 127 128 return strcmp((char *)name, "GenuineIntel") == 0; 129 } 130 131 #define CPUID(a, b, c, d) ((((unsigned long long) a) << 32) | (b << 16) | \ 132 (c << 8) | d) 133 134 /* 135 * Each X86_FEATURE_XXX definition is 64-bit and contains the following 136 * CPUID meta-data: 137 * 138 * [63:32] : input value for EAX 139 * [31:16] : input value for ECX 140 * [15:8] : output register 141 * [7:0] : bit position in output register 142 */ 143 144 /* 145 * Basic Leafs, a.k.a. Intel defined 146 */ 147 #define X86_FEATURE_MWAIT (CPUID(0x1, 0, ECX, 3)) 148 #define X86_FEATURE_VMX (CPUID(0x1, 0, ECX, 5)) 149 #define X86_FEATURE_PCID (CPUID(0x1, 0, ECX, 17)) 150 #define X86_FEATURE_MOVBE (CPUID(0x1, 0, ECX, 22)) 151 #define X86_FEATURE_TSC_DEADLINE_TIMER (CPUID(0x1, 0, ECX, 24)) 152 #define X86_FEATURE_XSAVE (CPUID(0x1, 0, ECX, 26)) 153 #define X86_FEATURE_OSXSAVE (CPUID(0x1, 0, ECX, 27)) 154 #define X86_FEATURE_RDRAND (CPUID(0x1, 0, ECX, 30)) 155 #define X86_FEATURE_MCE (CPUID(0x1, 0, EDX, 7)) 156 #define X86_FEATURE_APIC (CPUID(0x1, 0, EDX, 9)) 157 #define X86_FEATURE_CLFLUSH (CPUID(0x1, 0, EDX, 19)) 158 #define X86_FEATURE_XMM (CPUID(0x1, 0, EDX, 25)) 159 #define X86_FEATURE_XMM2 (CPUID(0x1, 0, EDX, 26)) 160 #define X86_FEATURE_TSC_ADJUST (CPUID(0x7, 0, EBX, 1)) 161 #define X86_FEATURE_HLE (CPUID(0x7, 0, EBX, 4)) 162 #define X86_FEATURE_SMEP (CPUID(0x7, 0, EBX, 7)) 163 #define X86_FEATURE_INVPCID (CPUID(0x7, 0, EBX, 10)) 164 #define X86_FEATURE_RTM (CPUID(0x7, 0, EBX, 11)) 165 #define X86_FEATURE_SMAP (CPUID(0x7, 0, EBX, 20)) 166 #define X86_FEATURE_PCOMMIT (CPUID(0x7, 0, EBX, 22)) 167 #define X86_FEATURE_CLFLUSHOPT (CPUID(0x7, 0, EBX, 23)) 168 #define X86_FEATURE_CLWB (CPUID(0x7, 0, EBX, 24)) 169 #define X86_FEATURE_UMIP (CPUID(0x7, 0, ECX, 2)) 170 #define X86_FEATURE_PKU (CPUID(0x7, 0, ECX, 3)) 171 #define X86_FEATURE_LA57 (CPUID(0x7, 0, ECX, 16)) 172 #define X86_FEATURE_RDPID (CPUID(0x7, 0, ECX, 22)) 173 #define X86_FEATURE_SHSTK (CPUID(0x7, 0, ECX, 7)) 174 #define X86_FEATURE_IBT (CPUID(0x7, 0, EDX, 20)) 175 #define X86_FEATURE_SPEC_CTRL (CPUID(0x7, 0, EDX, 26)) 176 #define X86_FEATURE_ARCH_CAPABILITIES (CPUID(0x7, 0, EDX, 29)) 177 #define X86_FEATURE_PKS (CPUID(0x7, 0, ECX, 31)) 178 179 /* 180 * Extended Leafs, a.k.a. AMD defined 181 */ 182 #define X86_FEATURE_SVM (CPUID(0x80000001, 0, ECX, 2)) 183 #define X86_FEATURE_NX (CPUID(0x80000001, 0, EDX, 20)) 184 #define X86_FEATURE_GBPAGES (CPUID(0x80000001, 0, EDX, 26)) 185 #define X86_FEATURE_RDTSCP (CPUID(0x80000001, 0, EDX, 27)) 186 #define X86_FEATURE_LM (CPUID(0x80000001, 0, EDX, 29)) 187 #define X86_FEATURE_RDPRU (CPUID(0x80000008, 0, EBX, 4)) 188 #define X86_FEATURE_AMD_IBPB (CPUID(0x80000008, 0, EBX, 12)) 189 #define X86_FEATURE_NPT (CPUID(0x8000000A, 0, EDX, 0)) 190 #define X86_FEATURE_LBRV (CPUID(0x8000000A, 0, EDX, 1)) 191 #define X86_FEATURE_NRIPS (CPUID(0x8000000A, 0, EDX, 3)) 192 #define X86_FEATURE_TSCRATEMSR (CPUID(0x8000000A, 0, EDX, 4)) 193 #define X86_FEATURE_PAUSEFILTER (CPUID(0x8000000A, 0, EDX, 10)) 194 #define X86_FEATURE_PFTHRESHOLD (CPUID(0x8000000A, 0, EDX, 12)) 195 #define X86_FEATURE_VGIF (CPUID(0x8000000A, 0, EDX, 16)) 196 197 198 static inline bool this_cpu_has(u64 feature) 199 { 200 u32 input_eax = feature >> 32; 201 u32 input_ecx = (feature >> 16) & 0xffff; 202 u32 output_reg = (feature >> 8) & 0xff; 203 u8 bit = feature & 0xff; 204 struct cpuid c; 205 u32 *tmp; 206 207 c = cpuid_indexed(input_eax, input_ecx); 208 tmp = (u32 *)&c; 209 210 return ((*(tmp + (output_reg % 32))) & (1 << bit)); 211 } 212 213 struct far_pointer32 { 214 u32 offset; 215 u16 selector; 216 } __attribute__((packed)); 217 218 struct descriptor_table_ptr { 219 u16 limit; 220 ulong base; 221 } __attribute__((packed)); 222 223 static inline void clac(void) 224 { 225 asm volatile (".byte 0x0f, 0x01, 0xca" : : : "memory"); 226 } 227 228 static inline void stac(void) 229 { 230 asm volatile (".byte 0x0f, 0x01, 0xcb" : : : "memory"); 231 } 232 233 static inline u16 read_cs(void) 234 { 235 unsigned val; 236 237 asm volatile ("mov %%cs, %0" : "=mr"(val)); 238 return val; 239 } 240 241 static inline u16 read_ds(void) 242 { 243 unsigned val; 244 245 asm volatile ("mov %%ds, %0" : "=mr"(val)); 246 return val; 247 } 248 249 static inline u16 read_es(void) 250 { 251 unsigned val; 252 253 asm volatile ("mov %%es, %0" : "=mr"(val)); 254 return val; 255 } 256 257 static inline u16 read_ss(void) 258 { 259 unsigned val; 260 261 asm volatile ("mov %%ss, %0" : "=mr"(val)); 262 return val; 263 } 264 265 static inline u16 read_fs(void) 266 { 267 unsigned val; 268 269 asm volatile ("mov %%fs, %0" : "=mr"(val)); 270 return val; 271 } 272 273 static inline u16 read_gs(void) 274 { 275 unsigned val; 276 277 asm volatile ("mov %%gs, %0" : "=mr"(val)); 278 return val; 279 } 280 281 static inline unsigned long read_rflags(void) 282 { 283 unsigned long f; 284 asm volatile ("pushf; pop %0\n\t" : "=rm"(f)); 285 return f; 286 } 287 288 static inline void write_ds(unsigned val) 289 { 290 asm volatile ("mov %0, %%ds" : : "rm"(val) : "memory"); 291 } 292 293 static inline void write_es(unsigned val) 294 { 295 asm volatile ("mov %0, %%es" : : "rm"(val) : "memory"); 296 } 297 298 static inline void write_ss(unsigned val) 299 { 300 asm volatile ("mov %0, %%ss" : : "rm"(val) : "memory"); 301 } 302 303 static inline void write_fs(unsigned val) 304 { 305 asm volatile ("mov %0, %%fs" : : "rm"(val) : "memory"); 306 } 307 308 static inline void write_gs(unsigned val) 309 { 310 asm volatile ("mov %0, %%gs" : : "rm"(val) : "memory"); 311 } 312 313 static inline void write_rflags(unsigned long f) 314 { 315 asm volatile ("push %0; popf\n\t" : : "rm"(f)); 316 } 317 318 static inline void set_iopl(int iopl) 319 { 320 unsigned long flags = read_rflags() & ~X86_EFLAGS_IOPL; 321 flags |= iopl * (X86_EFLAGS_IOPL / 3); 322 write_rflags(flags); 323 } 324 325 static inline u64 rdmsr(u32 index) 326 { 327 u32 a, d; 328 asm volatile ("rdmsr" : "=a"(a), "=d"(d) : "c"(index) : "memory"); 329 return a | ((u64)d << 32); 330 } 331 332 static inline void wrmsr(u32 index, u64 val) 333 { 334 u32 a = val, d = val >> 32; 335 asm volatile ("wrmsr" : : "a"(a), "d"(d), "c"(index) : "memory"); 336 } 337 338 static inline int rdmsr_checking(u32 index) 339 { 340 asm volatile (ASM_TRY("1f") 341 "rdmsr\n\t" 342 "1:" 343 : : "c"(index) : "memory", "eax", "edx"); 344 return exception_vector(); 345 } 346 347 static inline int wrmsr_checking(u32 index, u64 val) 348 { 349 u32 a = val, d = val >> 32; 350 351 asm volatile (ASM_TRY("1f") 352 "wrmsr\n\t" 353 "1:" 354 : : "a"(a), "d"(d), "c"(index) : "memory"); 355 return exception_vector(); 356 } 357 358 static inline uint64_t rdpmc(uint32_t index) 359 { 360 uint32_t a, d; 361 asm volatile ("rdpmc" : "=a"(a), "=d"(d) : "c"(index)); 362 return a | ((uint64_t)d << 32); 363 } 364 365 static inline void write_cr0(ulong val) 366 { 367 asm volatile ("mov %0, %%cr0" : : "r"(val) : "memory"); 368 } 369 370 static inline ulong read_cr0(void) 371 { 372 ulong val; 373 asm volatile ("mov %%cr0, %0" : "=r"(val) : : "memory"); 374 return val; 375 } 376 377 static inline void write_cr2(ulong val) 378 { 379 asm volatile ("mov %0, %%cr2" : : "r"(val) : "memory"); 380 } 381 382 static inline ulong read_cr2(void) 383 { 384 ulong val; 385 asm volatile ("mov %%cr2, %0" : "=r"(val) : : "memory"); 386 return val; 387 } 388 389 static inline void write_cr3(ulong val) 390 { 391 asm volatile ("mov %0, %%cr3" : : "r"(val) : "memory"); 392 } 393 394 static inline ulong read_cr3(void) 395 { 396 ulong val; 397 asm volatile ("mov %%cr3, %0" : "=r"(val) : : "memory"); 398 return val; 399 } 400 401 static inline void update_cr3(void *cr3) 402 { 403 write_cr3((ulong)cr3); 404 } 405 406 static inline void write_cr4(ulong val) 407 { 408 asm volatile ("mov %0, %%cr4" : : "r"(val) : "memory"); 409 } 410 411 static inline ulong read_cr4(void) 412 { 413 ulong val; 414 asm volatile ("mov %%cr4, %0" : "=r"(val) : : "memory"); 415 return val; 416 } 417 418 static inline void write_cr8(ulong val) 419 { 420 asm volatile ("mov %0, %%cr8" : : "r"(val) : "memory"); 421 } 422 423 static inline ulong read_cr8(void) 424 { 425 ulong val; 426 asm volatile ("mov %%cr8, %0" : "=r"(val) : : "memory"); 427 return val; 428 } 429 430 static inline void lgdt(const struct descriptor_table_ptr *ptr) 431 { 432 asm volatile ("lgdt %0" : : "m"(*ptr)); 433 } 434 435 static inline void sgdt(struct descriptor_table_ptr *ptr) 436 { 437 asm volatile ("sgdt %0" : "=m"(*ptr)); 438 } 439 440 static inline void lidt(const struct descriptor_table_ptr *ptr) 441 { 442 asm volatile ("lidt %0" : : "m"(*ptr)); 443 } 444 445 static inline void sidt(struct descriptor_table_ptr *ptr) 446 { 447 asm volatile ("sidt %0" : "=m"(*ptr)); 448 } 449 450 static inline void lldt(u16 val) 451 { 452 asm volatile ("lldt %0" : : "rm"(val)); 453 } 454 455 static inline u16 sldt(void) 456 { 457 u16 val; 458 asm volatile ("sldt %0" : "=rm"(val)); 459 return val; 460 } 461 462 static inline void ltr(u16 val) 463 { 464 asm volatile ("ltr %0" : : "rm"(val)); 465 } 466 467 static inline u16 str(void) 468 { 469 u16 val; 470 asm volatile ("str %0" : "=rm"(val)); 471 return val; 472 } 473 474 static inline void write_dr0(void *val) 475 { 476 asm volatile ("mov %0, %%dr0" : : "r"(val) : "memory"); 477 } 478 479 static inline void write_dr1(void *val) 480 { 481 asm volatile ("mov %0, %%dr1" : : "r"(val) : "memory"); 482 } 483 484 static inline void write_dr2(void *val) 485 { 486 asm volatile ("mov %0, %%dr2" : : "r"(val) : "memory"); 487 } 488 489 static inline void write_dr3(void *val) 490 { 491 asm volatile ("mov %0, %%dr3" : : "r"(val) : "memory"); 492 } 493 494 static inline void write_dr6(ulong val) 495 { 496 asm volatile ("mov %0, %%dr6" : : "r"(val) : "memory"); 497 } 498 499 static inline ulong read_dr6(void) 500 { 501 ulong val; 502 asm volatile ("mov %%dr6, %0" : "=r"(val)); 503 return val; 504 } 505 506 static inline void write_dr7(ulong val) 507 { 508 asm volatile ("mov %0, %%dr7" : : "r"(val) : "memory"); 509 } 510 511 static inline ulong read_dr7(void) 512 { 513 ulong val; 514 asm volatile ("mov %%dr7, %0" : "=r"(val)); 515 return val; 516 } 517 518 static inline void pause(void) 519 { 520 asm volatile ("pause"); 521 } 522 523 static inline void cli(void) 524 { 525 asm volatile ("cli"); 526 } 527 528 static inline void sti(void) 529 { 530 asm volatile ("sti"); 531 } 532 533 static inline unsigned long long rdrand(void) 534 { 535 long long r; 536 537 asm volatile("rdrand %0\n\t" 538 "jc 1f\n\t" 539 "mov $0, %0\n\t" 540 "1:\n\t" : "=r" (r)); 541 return r; 542 } 543 544 static inline unsigned long long rdtsc(void) 545 { 546 long long r; 547 548 #ifdef __x86_64__ 549 unsigned a, d; 550 551 asm volatile ("rdtsc" : "=a"(a), "=d"(d)); 552 r = a | ((long long)d << 32); 553 #else 554 asm volatile ("rdtsc" : "=A"(r)); 555 #endif 556 return r; 557 } 558 559 /* 560 * Per the advice in the SDM, volume 2, the sequence "mfence; lfence" 561 * executed immediately before rdtsc ensures that rdtsc will be 562 * executed only after all previous instructions have executed and all 563 * previous loads and stores are globally visible. In addition, the 564 * lfence immediately after rdtsc ensures that rdtsc will be executed 565 * prior to the execution of any subsequent instruction. 566 */ 567 static inline unsigned long long fenced_rdtsc(void) 568 { 569 unsigned long long tsc; 570 571 #ifdef __x86_64__ 572 unsigned int eax, edx; 573 574 asm volatile ("mfence; lfence; rdtsc; lfence" : "=a"(eax), "=d"(edx)); 575 tsc = eax | ((unsigned long long)edx << 32); 576 #else 577 asm volatile ("mfence; lfence; rdtsc; lfence" : "=A"(tsc)); 578 #endif 579 return tsc; 580 } 581 582 static inline unsigned long long rdtscp(u32 *aux) 583 { 584 long long r; 585 586 #ifdef __x86_64__ 587 unsigned a, d; 588 589 asm volatile ("rdtscp" : "=a"(a), "=d"(d), "=c"(*aux)); 590 r = a | ((long long)d << 32); 591 #else 592 asm volatile ("rdtscp" : "=A"(r), "=c"(*aux)); 593 #endif 594 return r; 595 } 596 597 static inline void wrtsc(u64 tsc) 598 { 599 wrmsr(MSR_IA32_TSC, tsc); 600 } 601 602 static inline void irq_disable(void) 603 { 604 asm volatile("cli"); 605 } 606 607 /* Note that irq_enable() does not ensure an interrupt shadow due 608 * to the vagaries of compiler optimizations. If you need the 609 * shadow, use a single asm with "sti" and the instruction after it. 610 */ 611 static inline void irq_enable(void) 612 { 613 asm volatile("sti"); 614 } 615 616 static inline void invlpg(volatile void *va) 617 { 618 asm volatile("invlpg (%0)" ::"r" (va) : "memory"); 619 } 620 621 static inline void safe_halt(void) 622 { 623 asm volatile("sti; hlt"); 624 } 625 626 static inline u32 read_pkru(void) 627 { 628 unsigned int eax, edx; 629 unsigned int ecx = 0; 630 unsigned int pkru; 631 632 asm volatile(".byte 0x0f,0x01,0xee\n\t" 633 : "=a" (eax), "=d" (edx) 634 : "c" (ecx)); 635 pkru = eax; 636 return pkru; 637 } 638 639 static inline void write_pkru(u32 pkru) 640 { 641 unsigned int eax = pkru; 642 unsigned int ecx = 0; 643 unsigned int edx = 0; 644 645 asm volatile(".byte 0x0f,0x01,0xef\n\t" 646 : : "a" (eax), "c" (ecx), "d" (edx)); 647 } 648 649 static inline bool is_canonical(u64 addr) 650 { 651 int va_width = (raw_cpuid(0x80000008, 0).a & 0xff00) >> 8; 652 int shift_amt = 64 - va_width; 653 654 return (s64)(addr << shift_amt) >> shift_amt == addr; 655 } 656 657 static inline void clear_bit(int bit, u8 *addr) 658 { 659 __asm__ __volatile__("btr %1, %0" 660 : "+m" (*addr) : "Ir" (bit) : "cc", "memory"); 661 } 662 663 static inline void set_bit(int bit, u8 *addr) 664 { 665 __asm__ __volatile__("bts %1, %0" 666 : "+m" (*addr) : "Ir" (bit) : "cc", "memory"); 667 } 668 669 static inline void flush_tlb(void) 670 { 671 ulong cr4; 672 673 cr4 = read_cr4(); 674 write_cr4(cr4 ^ X86_CR4_PGE); 675 write_cr4(cr4); 676 } 677 678 static inline int has_spec_ctrl(void) 679 { 680 return !!(this_cpu_has(X86_FEATURE_SPEC_CTRL)); 681 } 682 683 static inline int cpu_has_efer_nx(void) 684 { 685 return !!(this_cpu_has(X86_FEATURE_NX)); 686 } 687 688 static inline bool cpuid_osxsave(void) 689 { 690 return cpuid(1).c & (1 << (X86_FEATURE_OSXSAVE % 32)); 691 } 692 693 #endif 694