1 #ifndef LIBCFLAT_PROCESSOR_H 2 #define LIBCFLAT_PROCESSOR_H 3 4 #include "libcflat.h" 5 #include "msr.h" 6 #include <stdint.h> 7 8 #ifdef __x86_64__ 9 # define R "r" 10 # define W "q" 11 # define S "8" 12 #else 13 # define R "e" 14 # define W "l" 15 # define S "4" 16 #endif 17 18 #define DB_VECTOR 1 19 #define BP_VECTOR 3 20 #define UD_VECTOR 6 21 #define DF_VECTOR 8 22 #define TS_VECTOR 10 23 #define NP_VECTOR 11 24 #define SS_VECTOR 12 25 #define GP_VECTOR 13 26 #define PF_VECTOR 14 27 #define AC_VECTOR 17 28 29 #define X86_CR0_PE 0x00000001 30 #define X86_CR0_MP 0x00000002 31 #define X86_CR0_EM 0x00000004 32 #define X86_CR0_TS 0x00000008 33 #define X86_CR0_WP 0x00010000 34 #define X86_CR0_AM 0x00040000 35 #define X86_CR0_NW 0x20000000 36 #define X86_CR0_CD 0x40000000 37 #define X86_CR0_PG 0x80000000 38 #define X86_CR3_PCID_MASK 0x00000fff 39 #define X86_CR4_TSD 0x00000004 40 #define X86_CR4_DE 0x00000008 41 #define X86_CR4_PSE 0x00000010 42 #define X86_CR4_PAE 0x00000020 43 #define X86_CR4_MCE 0x00000040 44 #define X86_CR4_PGE 0x00000080 45 #define X86_CR4_PCE 0x00000100 46 #define X86_CR4_UMIP 0x00000800 47 #define X86_CR4_LA57 0x00001000 48 #define X86_CR4_VMXE 0x00002000 49 #define X86_CR4_PCIDE 0x00020000 50 #define X86_CR4_OSXSAVE 0x00040000 51 #define X86_CR4_SMEP 0x00100000 52 #define X86_CR4_SMAP 0x00200000 53 #define X86_CR4_PKE 0x00400000 54 #define X86_CR4_PKS 0x01000000 55 56 #define X86_EFLAGS_CF 0x00000001 57 #define X86_EFLAGS_FIXED 0x00000002 58 #define X86_EFLAGS_PF 0x00000004 59 #define X86_EFLAGS_AF 0x00000010 60 #define X86_EFLAGS_ZF 0x00000040 61 #define X86_EFLAGS_SF 0x00000080 62 #define X86_EFLAGS_TF 0x00000100 63 #define X86_EFLAGS_IF 0x00000200 64 #define X86_EFLAGS_DF 0x00000400 65 #define X86_EFLAGS_OF 0x00000800 66 #define X86_EFLAGS_IOPL 0x00003000 67 #define X86_EFLAGS_NT 0x00004000 68 #define X86_EFLAGS_VM 0x00020000 69 #define X86_EFLAGS_AC 0x00040000 70 71 #define X86_EFLAGS_ALU (X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | \ 72 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF) 73 74 #define X86_IA32_EFER 0xc0000080 75 #define X86_EFER_LMA (1UL << 8) 76 77 /* 78 * CPU features 79 */ 80 81 enum cpuid_output_regs { 82 EAX, 83 EBX, 84 ECX, 85 EDX 86 }; 87 88 struct cpuid { u32 a, b, c, d; }; 89 90 static inline struct cpuid raw_cpuid(u32 function, u32 index) 91 { 92 struct cpuid r; 93 asm volatile ("cpuid" 94 : "=a"(r.a), "=b"(r.b), "=c"(r.c), "=d"(r.d) 95 : "0"(function), "2"(index)); 96 return r; 97 } 98 99 static inline struct cpuid cpuid_indexed(u32 function, u32 index) 100 { 101 u32 level = raw_cpuid(function & 0xf0000000, 0).a; 102 if (level < function) 103 return (struct cpuid) { 0, 0, 0, 0 }; 104 return raw_cpuid(function, index); 105 } 106 107 static inline struct cpuid cpuid(u32 function) 108 { 109 return cpuid_indexed(function, 0); 110 } 111 112 static inline u8 cpuid_maxphyaddr(void) 113 { 114 if (raw_cpuid(0x80000000, 0).a < 0x80000008) 115 return 36; 116 return raw_cpuid(0x80000008, 0).a & 0xff; 117 } 118 119 #define CPUID(a, b, c, d) ((((unsigned long long) a) << 32) | (b << 16) | \ 120 (c << 8) | d) 121 122 /* 123 * Each X86_FEATURE_XXX definition is 64-bit and contains the following 124 * CPUID meta-data: 125 * 126 * [63:32] : input value for EAX 127 * [31:16] : input value for ECX 128 * [15:8] : output register 129 * [7:0] : bit position in output register 130 */ 131 132 /* 133 * Intel CPUID features 134 */ 135 #define X86_FEATURE_MWAIT (CPUID(0x1, 0, ECX, 3)) 136 #define X86_FEATURE_VMX (CPUID(0x1, 0, ECX, 5)) 137 #define X86_FEATURE_PCID (CPUID(0x1, 0, ECX, 17)) 138 #define X86_FEATURE_MOVBE (CPUID(0x1, 0, ECX, 22)) 139 #define X86_FEATURE_TSC_DEADLINE_TIMER (CPUID(0x1, 0, ECX, 24)) 140 #define X86_FEATURE_XSAVE (CPUID(0x1, 0, ECX, 26)) 141 #define X86_FEATURE_OSXSAVE (CPUID(0x1, 0, ECX, 27)) 142 #define X86_FEATURE_RDRAND (CPUID(0x1, 0, ECX, 30)) 143 #define X86_FEATURE_MCE (CPUID(0x1, 0, EDX, 7)) 144 #define X86_FEATURE_APIC (CPUID(0x1, 0, EDX, 9)) 145 #define X86_FEATURE_CLFLUSH (CPUID(0x1, 0, EDX, 19)) 146 #define X86_FEATURE_XMM (CPUID(0x1, 0, EDX, 25)) 147 #define X86_FEATURE_XMM2 (CPUID(0x1, 0, EDX, 26)) 148 #define X86_FEATURE_TSC_ADJUST (CPUID(0x7, 0, EBX, 1)) 149 #define X86_FEATURE_HLE (CPUID(0x7, 0, EBX, 4)) 150 #define X86_FEATURE_SMEP (CPUID(0x7, 0, EBX, 7)) 151 #define X86_FEATURE_INVPCID (CPUID(0x7, 0, EBX, 10)) 152 #define X86_FEATURE_RTM (CPUID(0x7, 0, EBX, 11)) 153 #define X86_FEATURE_SMAP (CPUID(0x7, 0, EBX, 20)) 154 #define X86_FEATURE_PCOMMIT (CPUID(0x7, 0, EBX, 22)) 155 #define X86_FEATURE_CLFLUSHOPT (CPUID(0x7, 0, EBX, 23)) 156 #define X86_FEATURE_CLWB (CPUID(0x7, 0, EBX, 24)) 157 #define X86_FEATURE_UMIP (CPUID(0x7, 0, ECX, 2)) 158 #define X86_FEATURE_PKU (CPUID(0x7, 0, ECX, 3)) 159 #define X86_FEATURE_LA57 (CPUID(0x7, 0, ECX, 16)) 160 #define X86_FEATURE_RDPID (CPUID(0x7, 0, ECX, 22)) 161 #define X86_FEATURE_SPEC_CTRL (CPUID(0x7, 0, EDX, 26)) 162 #define X86_FEATURE_ARCH_CAPABILITIES (CPUID(0x7, 0, EDX, 29)) 163 #define X86_FEATURE_PKS (CPUID(0x7, 0, ECX, 31)) 164 #define X86_FEATURE_NX (CPUID(0x80000001, 0, EDX, 20)) 165 #define X86_FEATURE_RDPRU (CPUID(0x80000008, 0, EBX, 4)) 166 167 /* 168 * AMD CPUID features 169 */ 170 #define X86_FEATURE_SVM (CPUID(0x80000001, 0, ECX, 2)) 171 #define X86_FEATURE_RDTSCP (CPUID(0x80000001, 0, EDX, 27)) 172 #define X86_FEATURE_AMD_IBPB (CPUID(0x80000008, 0, EBX, 12)) 173 #define X86_FEATURE_NPT (CPUID(0x8000000A, 0, EDX, 0)) 174 #define X86_FEATURE_NRIPS (CPUID(0x8000000A, 0, EDX, 3)) 175 176 177 static inline bool this_cpu_has(u64 feature) 178 { 179 u32 input_eax = feature >> 32; 180 u32 input_ecx = (feature >> 16) & 0xffff; 181 u32 output_reg = (feature >> 8) & 0xff; 182 u8 bit = feature & 0xff; 183 struct cpuid c; 184 u32 *tmp; 185 186 c = cpuid_indexed(input_eax, input_ecx); 187 tmp = (u32 *)&c; 188 189 return ((*(tmp + (output_reg % 32))) & (1 << bit)); 190 } 191 192 struct far_pointer32 { 193 u32 offset; 194 u16 selector; 195 } __attribute__((packed)); 196 197 struct descriptor_table_ptr { 198 u16 limit; 199 ulong base; 200 } __attribute__((packed)); 201 202 static inline void barrier(void) 203 { 204 asm volatile ("" : : : "memory"); 205 } 206 207 static inline void clac(void) 208 { 209 asm volatile (".byte 0x0f, 0x01, 0xca" : : : "memory"); 210 } 211 212 static inline void stac(void) 213 { 214 asm volatile (".byte 0x0f, 0x01, 0xcb" : : : "memory"); 215 } 216 217 static inline u16 read_cs(void) 218 { 219 unsigned val; 220 221 asm volatile ("mov %%cs, %0" : "=mr"(val)); 222 return val; 223 } 224 225 static inline u16 read_ds(void) 226 { 227 unsigned val; 228 229 asm volatile ("mov %%ds, %0" : "=mr"(val)); 230 return val; 231 } 232 233 static inline u16 read_es(void) 234 { 235 unsigned val; 236 237 asm volatile ("mov %%es, %0" : "=mr"(val)); 238 return val; 239 } 240 241 static inline u16 read_ss(void) 242 { 243 unsigned val; 244 245 asm volatile ("mov %%ss, %0" : "=mr"(val)); 246 return val; 247 } 248 249 static inline u16 read_fs(void) 250 { 251 unsigned val; 252 253 asm volatile ("mov %%fs, %0" : "=mr"(val)); 254 return val; 255 } 256 257 static inline u16 read_gs(void) 258 { 259 unsigned val; 260 261 asm volatile ("mov %%gs, %0" : "=mr"(val)); 262 return val; 263 } 264 265 static inline unsigned long read_rflags(void) 266 { 267 unsigned long f; 268 asm volatile ("pushf; pop %0\n\t" : "=rm"(f)); 269 return f; 270 } 271 272 static inline void write_ds(unsigned val) 273 { 274 asm volatile ("mov %0, %%ds" : : "rm"(val) : "memory"); 275 } 276 277 static inline void write_es(unsigned val) 278 { 279 asm volatile ("mov %0, %%es" : : "rm"(val) : "memory"); 280 } 281 282 static inline void write_ss(unsigned val) 283 { 284 asm volatile ("mov %0, %%ss" : : "rm"(val) : "memory"); 285 } 286 287 static inline void write_fs(unsigned val) 288 { 289 asm volatile ("mov %0, %%fs" : : "rm"(val) : "memory"); 290 } 291 292 static inline void write_gs(unsigned val) 293 { 294 asm volatile ("mov %0, %%gs" : : "rm"(val) : "memory"); 295 } 296 297 static inline void write_rflags(unsigned long f) 298 { 299 asm volatile ("push %0; popf\n\t" : : "rm"(f)); 300 } 301 302 static inline void set_iopl(int iopl) 303 { 304 unsigned long flags = read_rflags() & ~X86_EFLAGS_IOPL; 305 flags |= iopl * (X86_EFLAGS_IOPL / 3); 306 write_rflags(flags); 307 } 308 309 static inline u64 rdmsr(u32 index) 310 { 311 u32 a, d; 312 asm volatile ("rdmsr" : "=a"(a), "=d"(d) : "c"(index) : "memory"); 313 return a | ((u64)d << 32); 314 } 315 316 static inline void wrmsr(u32 index, u64 val) 317 { 318 u32 a = val, d = val >> 32; 319 asm volatile ("wrmsr" : : "a"(a), "d"(d), "c"(index) : "memory"); 320 } 321 322 static inline uint64_t rdpmc(uint32_t index) 323 { 324 uint32_t a, d; 325 asm volatile ("rdpmc" : "=a"(a), "=d"(d) : "c"(index)); 326 return a | ((uint64_t)d << 32); 327 } 328 329 static inline void write_cr0(ulong val) 330 { 331 asm volatile ("mov %0, %%cr0" : : "r"(val) : "memory"); 332 } 333 334 static inline ulong read_cr0(void) 335 { 336 ulong val; 337 asm volatile ("mov %%cr0, %0" : "=r"(val) : : "memory"); 338 return val; 339 } 340 341 static inline void write_cr2(ulong val) 342 { 343 asm volatile ("mov %0, %%cr2" : : "r"(val) : "memory"); 344 } 345 346 static inline ulong read_cr2(void) 347 { 348 ulong val; 349 asm volatile ("mov %%cr2, %0" : "=r"(val) : : "memory"); 350 return val; 351 } 352 353 static inline void write_cr3(ulong val) 354 { 355 asm volatile ("mov %0, %%cr3" : : "r"(val) : "memory"); 356 } 357 358 static inline ulong read_cr3(void) 359 { 360 ulong val; 361 asm volatile ("mov %%cr3, %0" : "=r"(val) : : "memory"); 362 return val; 363 } 364 365 static inline void update_cr3(void *cr3) 366 { 367 write_cr3((ulong)cr3); 368 } 369 370 static inline void write_cr4(ulong val) 371 { 372 asm volatile ("mov %0, %%cr4" : : "r"(val) : "memory"); 373 } 374 375 static inline ulong read_cr4(void) 376 { 377 ulong val; 378 asm volatile ("mov %%cr4, %0" : "=r"(val) : : "memory"); 379 return val; 380 } 381 382 static inline void write_cr8(ulong val) 383 { 384 asm volatile ("mov %0, %%cr8" : : "r"(val) : "memory"); 385 } 386 387 static inline ulong read_cr8(void) 388 { 389 ulong val; 390 asm volatile ("mov %%cr8, %0" : "=r"(val) : : "memory"); 391 return val; 392 } 393 394 static inline void lgdt(const struct descriptor_table_ptr *ptr) 395 { 396 asm volatile ("lgdt %0" : : "m"(*ptr)); 397 } 398 399 static inline void sgdt(struct descriptor_table_ptr *ptr) 400 { 401 asm volatile ("sgdt %0" : "=m"(*ptr)); 402 } 403 404 static inline void lidt(const struct descriptor_table_ptr *ptr) 405 { 406 asm volatile ("lidt %0" : : "m"(*ptr)); 407 } 408 409 static inline void sidt(struct descriptor_table_ptr *ptr) 410 { 411 asm volatile ("sidt %0" : "=m"(*ptr)); 412 } 413 414 static inline void lldt(unsigned val) 415 { 416 asm volatile ("lldt %0" : : "rm"(val)); 417 } 418 419 static inline u16 sldt(void) 420 { 421 u16 val; 422 asm volatile ("sldt %0" : "=rm"(val)); 423 return val; 424 } 425 426 static inline void ltr(u16 val) 427 { 428 asm volatile ("ltr %0" : : "rm"(val)); 429 } 430 431 static inline u16 str(void) 432 { 433 u16 val; 434 asm volatile ("str %0" : "=rm"(val)); 435 return val; 436 } 437 438 static inline void write_dr6(ulong val) 439 { 440 asm volatile ("mov %0, %%dr6" : : "r"(val) : "memory"); 441 } 442 443 static inline ulong read_dr6(void) 444 { 445 ulong val; 446 asm volatile ("mov %%dr6, %0" : "=r"(val)); 447 return val; 448 } 449 450 static inline void write_dr7(ulong val) 451 { 452 asm volatile ("mov %0, %%dr7" : : "r"(val) : "memory"); 453 } 454 455 static inline ulong read_dr7(void) 456 { 457 ulong val; 458 asm volatile ("mov %%dr7, %0" : "=r"(val)); 459 return val; 460 } 461 462 static inline void pause(void) 463 { 464 asm volatile ("pause"); 465 } 466 467 static inline void cli(void) 468 { 469 asm volatile ("cli"); 470 } 471 472 static inline void sti(void) 473 { 474 asm volatile ("sti"); 475 } 476 477 static inline unsigned long long rdtsc(void) 478 { 479 long long r; 480 481 #ifdef __x86_64__ 482 unsigned a, d; 483 484 asm volatile ("rdtsc" : "=a"(a), "=d"(d)); 485 r = a | ((long long)d << 32); 486 #else 487 asm volatile ("rdtsc" : "=A"(r)); 488 #endif 489 return r; 490 } 491 492 /* 493 * Per the advice in the SDM, volume 2, the sequence "mfence; lfence" 494 * executed immediately before rdtsc ensures that rdtsc will be 495 * executed only after all previous instructions have executed and all 496 * previous loads and stores are globally visible. In addition, the 497 * lfence immediately after rdtsc ensures that rdtsc will be executed 498 * prior to the execution of any subsequent instruction. 499 */ 500 static inline unsigned long long fenced_rdtsc(void) 501 { 502 unsigned long long tsc; 503 504 #ifdef __x86_64__ 505 unsigned int eax, edx; 506 507 asm volatile ("mfence; lfence; rdtsc; lfence" : "=a"(eax), "=d"(edx)); 508 tsc = eax | ((unsigned long long)edx << 32); 509 #else 510 asm volatile ("mfence; lfence; rdtsc; lfence" : "=A"(tsc)); 511 #endif 512 return tsc; 513 } 514 515 static inline unsigned long long rdtscp(u32 *aux) 516 { 517 long long r; 518 519 #ifdef __x86_64__ 520 unsigned a, d; 521 522 asm volatile ("rdtscp" : "=a"(a), "=d"(d), "=c"(*aux)); 523 r = a | ((long long)d << 32); 524 #else 525 asm volatile ("rdtscp" : "=A"(r), "=c"(*aux)); 526 #endif 527 return r; 528 } 529 530 static inline void wrtsc(u64 tsc) 531 { 532 unsigned a = tsc, d = tsc >> 32; 533 534 asm volatile("wrmsr" : : "a"(a), "d"(d), "c"(0x10)); 535 } 536 537 static inline void irq_disable(void) 538 { 539 asm volatile("cli"); 540 } 541 542 /* Note that irq_enable() does not ensure an interrupt shadow due 543 * to the vagaries of compiler optimizations. If you need the 544 * shadow, use a single asm with "sti" and the instruction after it. 545 */ 546 static inline void irq_enable(void) 547 { 548 asm volatile("sti"); 549 } 550 551 static inline void invlpg(volatile void *va) 552 { 553 asm volatile("invlpg (%0)" ::"r" (va) : "memory"); 554 } 555 556 static inline void safe_halt(void) 557 { 558 asm volatile("sti; hlt"); 559 } 560 561 static inline u32 read_pkru(void) 562 { 563 unsigned int eax, edx; 564 unsigned int ecx = 0; 565 unsigned int pkru; 566 567 asm volatile(".byte 0x0f,0x01,0xee\n\t" 568 : "=a" (eax), "=d" (edx) 569 : "c" (ecx)); 570 pkru = eax; 571 return pkru; 572 } 573 574 static inline void write_pkru(u32 pkru) 575 { 576 unsigned int eax = pkru; 577 unsigned int ecx = 0; 578 unsigned int edx = 0; 579 580 asm volatile(".byte 0x0f,0x01,0xef\n\t" 581 : : "a" (eax), "c" (ecx), "d" (edx)); 582 } 583 584 static inline bool is_canonical(u64 addr) 585 { 586 return (s64)(addr << 16) >> 16 == addr; 587 } 588 589 static inline void clear_bit(int bit, u8 *addr) 590 { 591 __asm__ __volatile__("btr %1, %0" 592 : "+m" (*addr) : "Ir" (bit) : "cc", "memory"); 593 } 594 595 static inline void set_bit(int bit, u8 *addr) 596 { 597 __asm__ __volatile__("bts %1, %0" 598 : "+m" (*addr) : "Ir" (bit) : "cc", "memory"); 599 } 600 601 static inline void flush_tlb(void) 602 { 603 ulong cr4; 604 605 cr4 = read_cr4(); 606 write_cr4(cr4 ^ X86_CR4_PGE); 607 write_cr4(cr4); 608 } 609 610 static inline int has_spec_ctrl(void) 611 { 612 return !!(this_cpu_has(X86_FEATURE_SPEC_CTRL)); 613 } 614 615 static inline int cpu_has_efer_nx(void) 616 { 617 return !!(this_cpu_has(X86_FEATURE_NX)); 618 } 619 620 static inline bool cpuid_osxsave(void) 621 { 622 return cpuid(1).c & (1 << (X86_FEATURE_OSXSAVE % 32)); 623 } 624 625 #endif 626