1 #ifndef _X86_PROCESSOR_H_ 2 #define _X86_PROCESSOR_H_ 3 4 #include "libcflat.h" 5 #include "desc.h" 6 #include "msr.h" 7 #include <stdint.h> 8 9 #define NONCANONICAL 0xaaaaaaaaaaaaaaaaull 10 11 #ifdef __x86_64__ 12 # define R "r" 13 # define W "q" 14 # define S "8" 15 #else 16 # define R "e" 17 # define W "l" 18 # define S "4" 19 #endif 20 21 #define DB_VECTOR 1 22 #define BP_VECTOR 3 23 #define UD_VECTOR 6 24 #define DF_VECTOR 8 25 #define TS_VECTOR 10 26 #define NP_VECTOR 11 27 #define SS_VECTOR 12 28 #define GP_VECTOR 13 29 #define PF_VECTOR 14 30 #define AC_VECTOR 17 31 #define CP_VECTOR 21 32 33 #define X86_CR0_PE 0x00000001 34 #define X86_CR0_MP 0x00000002 35 #define X86_CR0_EM 0x00000004 36 #define X86_CR0_TS 0x00000008 37 #define X86_CR0_WP 0x00010000 38 #define X86_CR0_AM 0x00040000 39 #define X86_CR0_NW 0x20000000 40 #define X86_CR0_CD 0x40000000 41 #define X86_CR0_PG 0x80000000 42 #define X86_CR3_PCID_MASK 0x00000fff 43 #define X86_CR4_TSD 0x00000004 44 #define X86_CR4_DE 0x00000008 45 #define X86_CR4_PSE 0x00000010 46 #define X86_CR4_PAE 0x00000020 47 #define X86_CR4_MCE 0x00000040 48 #define X86_CR4_PGE 0x00000080 49 #define X86_CR4_PCE 0x00000100 50 #define X86_CR4_UMIP 0x00000800 51 #define X86_CR4_LA57 0x00001000 52 #define X86_CR4_VMXE 0x00002000 53 #define X86_CR4_PCIDE 0x00020000 54 #define X86_CR4_OSXSAVE 0x00040000 55 #define X86_CR4_SMEP 0x00100000 56 #define X86_CR4_SMAP 0x00200000 57 #define X86_CR4_PKE 0x00400000 58 #define X86_CR4_CET 0x00800000 59 #define X86_CR4_PKS 0x01000000 60 61 #define X86_EFLAGS_CF 0x00000001 62 #define X86_EFLAGS_FIXED 0x00000002 63 #define X86_EFLAGS_PF 0x00000004 64 #define X86_EFLAGS_AF 0x00000010 65 #define X86_EFLAGS_ZF 0x00000040 66 #define X86_EFLAGS_SF 0x00000080 67 #define X86_EFLAGS_TF 0x00000100 68 #define X86_EFLAGS_IF 0x00000200 69 #define X86_EFLAGS_DF 0x00000400 70 #define X86_EFLAGS_OF 0x00000800 71 #define X86_EFLAGS_IOPL 0x00003000 72 #define X86_EFLAGS_NT 0x00004000 73 #define X86_EFLAGS_VM 0x00020000 74 #define X86_EFLAGS_AC 0x00040000 75 76 #define X86_EFLAGS_ALU (X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | \ 77 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF) 78 79 80 /* 81 * CPU features 82 */ 83 84 enum cpuid_output_regs { 85 EAX, 86 EBX, 87 ECX, 88 EDX 89 }; 90 91 struct cpuid { u32 a, b, c, d; }; 92 93 static inline struct cpuid raw_cpuid(u32 function, u32 index) 94 { 95 struct cpuid r; 96 asm volatile ("cpuid" 97 : "=a"(r.a), "=b"(r.b), "=c"(r.c), "=d"(r.d) 98 : "0"(function), "2"(index)); 99 return r; 100 } 101 102 static inline struct cpuid cpuid_indexed(u32 function, u32 index) 103 { 104 u32 level = raw_cpuid(function & 0xf0000000, 0).a; 105 if (level < function) 106 return (struct cpuid) { 0, 0, 0, 0 }; 107 return raw_cpuid(function, index); 108 } 109 110 static inline struct cpuid cpuid(u32 function) 111 { 112 return cpuid_indexed(function, 0); 113 } 114 115 static inline u8 cpuid_maxphyaddr(void) 116 { 117 if (raw_cpuid(0x80000000, 0).a < 0x80000008) 118 return 36; 119 return raw_cpuid(0x80000008, 0).a & 0xff; 120 } 121 122 static inline bool is_intel(void) 123 { 124 struct cpuid c = cpuid(0); 125 u32 name[4] = {c.b, c.d, c.c }; 126 127 return strcmp((char *)name, "GenuineIntel") == 0; 128 } 129 130 #define CPUID(a, b, c, d) ((((unsigned long long) a) << 32) | (b << 16) | \ 131 (c << 8) | d) 132 133 /* 134 * Each X86_FEATURE_XXX definition is 64-bit and contains the following 135 * CPUID meta-data: 136 * 137 * [63:32] : input value for EAX 138 * [31:16] : input value for ECX 139 * [15:8] : output register 140 * [7:0] : bit position in output register 141 */ 142 143 /* 144 * Intel CPUID features 145 */ 146 #define X86_FEATURE_MWAIT (CPUID(0x1, 0, ECX, 3)) 147 #define X86_FEATURE_VMX (CPUID(0x1, 0, ECX, 5)) 148 #define X86_FEATURE_PCID (CPUID(0x1, 0, ECX, 17)) 149 #define X86_FEATURE_MOVBE (CPUID(0x1, 0, ECX, 22)) 150 #define X86_FEATURE_TSC_DEADLINE_TIMER (CPUID(0x1, 0, ECX, 24)) 151 #define X86_FEATURE_XSAVE (CPUID(0x1, 0, ECX, 26)) 152 #define X86_FEATURE_OSXSAVE (CPUID(0x1, 0, ECX, 27)) 153 #define X86_FEATURE_RDRAND (CPUID(0x1, 0, ECX, 30)) 154 #define X86_FEATURE_MCE (CPUID(0x1, 0, EDX, 7)) 155 #define X86_FEATURE_APIC (CPUID(0x1, 0, EDX, 9)) 156 #define X86_FEATURE_CLFLUSH (CPUID(0x1, 0, EDX, 19)) 157 #define X86_FEATURE_XMM (CPUID(0x1, 0, EDX, 25)) 158 #define X86_FEATURE_XMM2 (CPUID(0x1, 0, EDX, 26)) 159 #define X86_FEATURE_TSC_ADJUST (CPUID(0x7, 0, EBX, 1)) 160 #define X86_FEATURE_HLE (CPUID(0x7, 0, EBX, 4)) 161 #define X86_FEATURE_SMEP (CPUID(0x7, 0, EBX, 7)) 162 #define X86_FEATURE_INVPCID (CPUID(0x7, 0, EBX, 10)) 163 #define X86_FEATURE_RTM (CPUID(0x7, 0, EBX, 11)) 164 #define X86_FEATURE_SMAP (CPUID(0x7, 0, EBX, 20)) 165 #define X86_FEATURE_PCOMMIT (CPUID(0x7, 0, EBX, 22)) 166 #define X86_FEATURE_CLFLUSHOPT (CPUID(0x7, 0, EBX, 23)) 167 #define X86_FEATURE_CLWB (CPUID(0x7, 0, EBX, 24)) 168 #define X86_FEATURE_UMIP (CPUID(0x7, 0, ECX, 2)) 169 #define X86_FEATURE_PKU (CPUID(0x7, 0, ECX, 3)) 170 #define X86_FEATURE_LA57 (CPUID(0x7, 0, ECX, 16)) 171 #define X86_FEATURE_RDPID (CPUID(0x7, 0, ECX, 22)) 172 #define X86_FEATURE_SHSTK (CPUID(0x7, 0, ECX, 7)) 173 #define X86_FEATURE_IBT (CPUID(0x7, 0, EDX, 20)) 174 #define X86_FEATURE_SPEC_CTRL (CPUID(0x7, 0, EDX, 26)) 175 #define X86_FEATURE_ARCH_CAPABILITIES (CPUID(0x7, 0, EDX, 29)) 176 #define X86_FEATURE_PKS (CPUID(0x7, 0, ECX, 31)) 177 #define X86_FEATURE_NX (CPUID(0x80000001, 0, EDX, 20)) 178 #define X86_FEATURE_LM (CPUID(0x80000001, 0, EDX, 29)) 179 #define X86_FEATURE_RDPRU (CPUID(0x80000008, 0, EBX, 4)) 180 181 /* 182 * AMD CPUID features 183 */ 184 #define X86_FEATURE_SVM (CPUID(0x80000001, 0, ECX, 2)) 185 #define X86_FEATURE_RDTSCP (CPUID(0x80000001, 0, EDX, 27)) 186 #define X86_FEATURE_AMD_IBPB (CPUID(0x80000008, 0, EBX, 12)) 187 #define X86_FEATURE_NPT (CPUID(0x8000000A, 0, EDX, 0)) 188 #define X86_FEATURE_NRIPS (CPUID(0x8000000A, 0, EDX, 3)) 189 190 191 static inline bool this_cpu_has(u64 feature) 192 { 193 u32 input_eax = feature >> 32; 194 u32 input_ecx = (feature >> 16) & 0xffff; 195 u32 output_reg = (feature >> 8) & 0xff; 196 u8 bit = feature & 0xff; 197 struct cpuid c; 198 u32 *tmp; 199 200 c = cpuid_indexed(input_eax, input_ecx); 201 tmp = (u32 *)&c; 202 203 return ((*(tmp + (output_reg % 32))) & (1 << bit)); 204 } 205 206 struct far_pointer32 { 207 u32 offset; 208 u16 selector; 209 } __attribute__((packed)); 210 211 struct descriptor_table_ptr { 212 u16 limit; 213 ulong base; 214 } __attribute__((packed)); 215 216 static inline void barrier(void) 217 { 218 asm volatile ("" : : : "memory"); 219 } 220 221 static inline void clac(void) 222 { 223 asm volatile (".byte 0x0f, 0x01, 0xca" : : : "memory"); 224 } 225 226 static inline void stac(void) 227 { 228 asm volatile (".byte 0x0f, 0x01, 0xcb" : : : "memory"); 229 } 230 231 static inline u16 read_cs(void) 232 { 233 unsigned val; 234 235 asm volatile ("mov %%cs, %0" : "=mr"(val)); 236 return val; 237 } 238 239 static inline u16 read_ds(void) 240 { 241 unsigned val; 242 243 asm volatile ("mov %%ds, %0" : "=mr"(val)); 244 return val; 245 } 246 247 static inline u16 read_es(void) 248 { 249 unsigned val; 250 251 asm volatile ("mov %%es, %0" : "=mr"(val)); 252 return val; 253 } 254 255 static inline u16 read_ss(void) 256 { 257 unsigned val; 258 259 asm volatile ("mov %%ss, %0" : "=mr"(val)); 260 return val; 261 } 262 263 static inline u16 read_fs(void) 264 { 265 unsigned val; 266 267 asm volatile ("mov %%fs, %0" : "=mr"(val)); 268 return val; 269 } 270 271 static inline u16 read_gs(void) 272 { 273 unsigned val; 274 275 asm volatile ("mov %%gs, %0" : "=mr"(val)); 276 return val; 277 } 278 279 static inline unsigned long read_rflags(void) 280 { 281 unsigned long f; 282 asm volatile ("pushf; pop %0\n\t" : "=rm"(f)); 283 return f; 284 } 285 286 static inline void write_ds(unsigned val) 287 { 288 asm volatile ("mov %0, %%ds" : : "rm"(val) : "memory"); 289 } 290 291 static inline void write_es(unsigned val) 292 { 293 asm volatile ("mov %0, %%es" : : "rm"(val) : "memory"); 294 } 295 296 static inline void write_ss(unsigned val) 297 { 298 asm volatile ("mov %0, %%ss" : : "rm"(val) : "memory"); 299 } 300 301 static inline void write_fs(unsigned val) 302 { 303 asm volatile ("mov %0, %%fs" : : "rm"(val) : "memory"); 304 } 305 306 static inline void write_gs(unsigned val) 307 { 308 asm volatile ("mov %0, %%gs" : : "rm"(val) : "memory"); 309 } 310 311 static inline void write_rflags(unsigned long f) 312 { 313 asm volatile ("push %0; popf\n\t" : : "rm"(f)); 314 } 315 316 static inline void set_iopl(int iopl) 317 { 318 unsigned long flags = read_rflags() & ~X86_EFLAGS_IOPL; 319 flags |= iopl * (X86_EFLAGS_IOPL / 3); 320 write_rflags(flags); 321 } 322 323 static inline u64 rdmsr(u32 index) 324 { 325 u32 a, d; 326 asm volatile ("rdmsr" : "=a"(a), "=d"(d) : "c"(index) : "memory"); 327 return a | ((u64)d << 32); 328 } 329 330 static inline void wrmsr(u32 index, u64 val) 331 { 332 u32 a = val, d = val >> 32; 333 asm volatile ("wrmsr" : : "a"(a), "d"(d), "c"(index) : "memory"); 334 } 335 336 static inline int rdmsr_checking(u32 index) 337 { 338 asm volatile (ASM_TRY("1f") 339 "rdmsr\n\t" 340 "1:" 341 : : "c"(index) : "memory", "eax", "edx"); 342 return exception_vector(); 343 } 344 345 static inline int wrmsr_checking(u32 index, u64 val) 346 { 347 u32 a = val, d = val >> 32; 348 349 asm volatile (ASM_TRY("1f") 350 "wrmsr\n\t" 351 "1:" 352 : : "a"(a), "d"(d), "c"(index) : "memory"); 353 return exception_vector(); 354 } 355 356 static inline uint64_t rdpmc(uint32_t index) 357 { 358 uint32_t a, d; 359 asm volatile ("rdpmc" : "=a"(a), "=d"(d) : "c"(index)); 360 return a | ((uint64_t)d << 32); 361 } 362 363 static inline void write_cr0(ulong val) 364 { 365 asm volatile ("mov %0, %%cr0" : : "r"(val) : "memory"); 366 } 367 368 static inline ulong read_cr0(void) 369 { 370 ulong val; 371 asm volatile ("mov %%cr0, %0" : "=r"(val) : : "memory"); 372 return val; 373 } 374 375 static inline void write_cr2(ulong val) 376 { 377 asm volatile ("mov %0, %%cr2" : : "r"(val) : "memory"); 378 } 379 380 static inline ulong read_cr2(void) 381 { 382 ulong val; 383 asm volatile ("mov %%cr2, %0" : "=r"(val) : : "memory"); 384 return val; 385 } 386 387 static inline void write_cr3(ulong val) 388 { 389 asm volatile ("mov %0, %%cr3" : : "r"(val) : "memory"); 390 } 391 392 static inline ulong read_cr3(void) 393 { 394 ulong val; 395 asm volatile ("mov %%cr3, %0" : "=r"(val) : : "memory"); 396 return val; 397 } 398 399 static inline void update_cr3(void *cr3) 400 { 401 write_cr3((ulong)cr3); 402 } 403 404 static inline void write_cr4(ulong val) 405 { 406 asm volatile ("mov %0, %%cr4" : : "r"(val) : "memory"); 407 } 408 409 static inline ulong read_cr4(void) 410 { 411 ulong val; 412 asm volatile ("mov %%cr4, %0" : "=r"(val) : : "memory"); 413 return val; 414 } 415 416 static inline void write_cr8(ulong val) 417 { 418 asm volatile ("mov %0, %%cr8" : : "r"(val) : "memory"); 419 } 420 421 static inline ulong read_cr8(void) 422 { 423 ulong val; 424 asm volatile ("mov %%cr8, %0" : "=r"(val) : : "memory"); 425 return val; 426 } 427 428 static inline void lgdt(const struct descriptor_table_ptr *ptr) 429 { 430 asm volatile ("lgdt %0" : : "m"(*ptr)); 431 } 432 433 static inline void sgdt(struct descriptor_table_ptr *ptr) 434 { 435 asm volatile ("sgdt %0" : "=m"(*ptr)); 436 } 437 438 static inline void lidt(const struct descriptor_table_ptr *ptr) 439 { 440 asm volatile ("lidt %0" : : "m"(*ptr)); 441 } 442 443 static inline void sidt(struct descriptor_table_ptr *ptr) 444 { 445 asm volatile ("sidt %0" : "=m"(*ptr)); 446 } 447 448 static inline void lldt(unsigned val) 449 { 450 asm volatile ("lldt %0" : : "rm"(val)); 451 } 452 453 static inline u16 sldt(void) 454 { 455 u16 val; 456 asm volatile ("sldt %0" : "=rm"(val)); 457 return val; 458 } 459 460 static inline void ltr(u16 val) 461 { 462 asm volatile ("ltr %0" : : "rm"(val)); 463 } 464 465 static inline u16 str(void) 466 { 467 u16 val; 468 asm volatile ("str %0" : "=rm"(val)); 469 return val; 470 } 471 472 static inline void write_dr6(ulong val) 473 { 474 asm volatile ("mov %0, %%dr6" : : "r"(val) : "memory"); 475 } 476 477 static inline ulong read_dr6(void) 478 { 479 ulong val; 480 asm volatile ("mov %%dr6, %0" : "=r"(val)); 481 return val; 482 } 483 484 static inline void write_dr7(ulong val) 485 { 486 asm volatile ("mov %0, %%dr7" : : "r"(val) : "memory"); 487 } 488 489 static inline ulong read_dr7(void) 490 { 491 ulong val; 492 asm volatile ("mov %%dr7, %0" : "=r"(val)); 493 return val; 494 } 495 496 static inline void pause(void) 497 { 498 asm volatile ("pause"); 499 } 500 501 static inline void cli(void) 502 { 503 asm volatile ("cli"); 504 } 505 506 static inline void sti(void) 507 { 508 asm volatile ("sti"); 509 } 510 511 static inline unsigned long long rdtsc(void) 512 { 513 long long r; 514 515 #ifdef __x86_64__ 516 unsigned a, d; 517 518 asm volatile ("rdtsc" : "=a"(a), "=d"(d)); 519 r = a | ((long long)d << 32); 520 #else 521 asm volatile ("rdtsc" : "=A"(r)); 522 #endif 523 return r; 524 } 525 526 /* 527 * Per the advice in the SDM, volume 2, the sequence "mfence; lfence" 528 * executed immediately before rdtsc ensures that rdtsc will be 529 * executed only after all previous instructions have executed and all 530 * previous loads and stores are globally visible. In addition, the 531 * lfence immediately after rdtsc ensures that rdtsc will be executed 532 * prior to the execution of any subsequent instruction. 533 */ 534 static inline unsigned long long fenced_rdtsc(void) 535 { 536 unsigned long long tsc; 537 538 #ifdef __x86_64__ 539 unsigned int eax, edx; 540 541 asm volatile ("mfence; lfence; rdtsc; lfence" : "=a"(eax), "=d"(edx)); 542 tsc = eax | ((unsigned long long)edx << 32); 543 #else 544 asm volatile ("mfence; lfence; rdtsc; lfence" : "=A"(tsc)); 545 #endif 546 return tsc; 547 } 548 549 static inline unsigned long long rdtscp(u32 *aux) 550 { 551 long long r; 552 553 #ifdef __x86_64__ 554 unsigned a, d; 555 556 asm volatile ("rdtscp" : "=a"(a), "=d"(d), "=c"(*aux)); 557 r = a | ((long long)d << 32); 558 #else 559 asm volatile ("rdtscp" : "=A"(r), "=c"(*aux)); 560 #endif 561 return r; 562 } 563 564 static inline void wrtsc(u64 tsc) 565 { 566 unsigned a = tsc, d = tsc >> 32; 567 568 asm volatile("wrmsr" : : "a"(a), "d"(d), "c"(0x10)); 569 } 570 571 static inline void irq_disable(void) 572 { 573 asm volatile("cli"); 574 } 575 576 /* Note that irq_enable() does not ensure an interrupt shadow due 577 * to the vagaries of compiler optimizations. If you need the 578 * shadow, use a single asm with "sti" and the instruction after it. 579 */ 580 static inline void irq_enable(void) 581 { 582 asm volatile("sti"); 583 } 584 585 static inline void invlpg(volatile void *va) 586 { 587 asm volatile("invlpg (%0)" ::"r" (va) : "memory"); 588 } 589 590 static inline void safe_halt(void) 591 { 592 asm volatile("sti; hlt"); 593 } 594 595 static inline u32 read_pkru(void) 596 { 597 unsigned int eax, edx; 598 unsigned int ecx = 0; 599 unsigned int pkru; 600 601 asm volatile(".byte 0x0f,0x01,0xee\n\t" 602 : "=a" (eax), "=d" (edx) 603 : "c" (ecx)); 604 pkru = eax; 605 return pkru; 606 } 607 608 static inline void write_pkru(u32 pkru) 609 { 610 unsigned int eax = pkru; 611 unsigned int ecx = 0; 612 unsigned int edx = 0; 613 614 asm volatile(".byte 0x0f,0x01,0xef\n\t" 615 : : "a" (eax), "c" (ecx), "d" (edx)); 616 } 617 618 static inline bool is_canonical(u64 addr) 619 { 620 return (s64)(addr << 16) >> 16 == addr; 621 } 622 623 static inline void clear_bit(int bit, u8 *addr) 624 { 625 __asm__ __volatile__("btr %1, %0" 626 : "+m" (*addr) : "Ir" (bit) : "cc", "memory"); 627 } 628 629 static inline void set_bit(int bit, u8 *addr) 630 { 631 __asm__ __volatile__("bts %1, %0" 632 : "+m" (*addr) : "Ir" (bit) : "cc", "memory"); 633 } 634 635 static inline void flush_tlb(void) 636 { 637 ulong cr4; 638 639 cr4 = read_cr4(); 640 write_cr4(cr4 ^ X86_CR4_PGE); 641 write_cr4(cr4); 642 } 643 644 static inline int has_spec_ctrl(void) 645 { 646 return !!(this_cpu_has(X86_FEATURE_SPEC_CTRL)); 647 } 648 649 static inline int cpu_has_efer_nx(void) 650 { 651 return !!(this_cpu_has(X86_FEATURE_NX)); 652 } 653 654 static inline bool cpuid_osxsave(void) 655 { 656 return cpuid(1).c & (1 << (X86_FEATURE_OSXSAVE % 32)); 657 } 658 659 #endif 660