1 #ifndef LIBCFLAT_PROCESSOR_H 2 #define LIBCFLAT_PROCESSOR_H 3 4 #include "libcflat.h" 5 #include "msr.h" 6 #include <stdint.h> 7 8 #ifdef __x86_64__ 9 # define R "r" 10 # define W "q" 11 # define S "8" 12 #else 13 # define R "e" 14 # define W "l" 15 # define S "4" 16 #endif 17 18 #define DB_VECTOR 1 19 #define BP_VECTOR 3 20 #define UD_VECTOR 6 21 #define DF_VECTOR 8 22 #define TS_VECTOR 10 23 #define NP_VECTOR 11 24 #define SS_VECTOR 12 25 #define GP_VECTOR 13 26 #define PF_VECTOR 14 27 #define AC_VECTOR 17 28 29 #define X86_CR0_PE 0x00000001 30 #define X86_CR0_MP 0x00000002 31 #define X86_CR0_EM 0x00000004 32 #define X86_CR0_TS 0x00000008 33 #define X86_CR0_WP 0x00010000 34 #define X86_CR0_AM 0x00040000 35 #define X86_CR0_NW 0x20000000 36 #define X86_CR0_CD 0x40000000 37 #define X86_CR0_PG 0x80000000 38 #define X86_CR3_PCID_MASK 0x00000fff 39 #define X86_CR4_TSD 0x00000004 40 #define X86_CR4_DE 0x00000008 41 #define X86_CR4_PSE 0x00000010 42 #define X86_CR4_PAE 0x00000020 43 #define X86_CR4_MCE 0x00000040 44 #define X86_CR4_PGE 0x00000080 45 #define X86_CR4_PCE 0x00000100 46 #define X86_CR4_UMIP 0x00000800 47 #define X86_CR4_LA57 0x00001000 48 #define X86_CR4_VMXE 0x00002000 49 #define X86_CR4_PCIDE 0x00020000 50 #define X86_CR4_OSXSAVE 0x00040000 51 #define X86_CR4_SMEP 0x00100000 52 #define X86_CR4_SMAP 0x00200000 53 #define X86_CR4_PKE 0x00400000 54 55 #define X86_EFLAGS_CF 0x00000001 56 #define X86_EFLAGS_FIXED 0x00000002 57 #define X86_EFLAGS_PF 0x00000004 58 #define X86_EFLAGS_AF 0x00000010 59 #define X86_EFLAGS_ZF 0x00000040 60 #define X86_EFLAGS_SF 0x00000080 61 #define X86_EFLAGS_TF 0x00000100 62 #define X86_EFLAGS_IF 0x00000200 63 #define X86_EFLAGS_DF 0x00000400 64 #define X86_EFLAGS_OF 0x00000800 65 #define X86_EFLAGS_IOPL 0x00003000 66 #define X86_EFLAGS_NT 0x00004000 67 #define X86_EFLAGS_VM 0x00020000 68 #define X86_EFLAGS_AC 0x00040000 69 70 #define X86_EFLAGS_ALU (X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | \ 71 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF) 72 73 #define X86_IA32_EFER 0xc0000080 74 #define X86_EFER_LMA (1UL << 8) 75 76 /* 77 * CPU features 78 */ 79 80 enum cpuid_output_regs { 81 EAX, 82 EBX, 83 ECX, 84 EDX 85 }; 86 87 struct cpuid { u32 a, b, c, d; }; 88 89 static inline struct cpuid raw_cpuid(u32 function, u32 index) 90 { 91 struct cpuid r; 92 asm volatile ("cpuid" 93 : "=a"(r.a), "=b"(r.b), "=c"(r.c), "=d"(r.d) 94 : "0"(function), "2"(index)); 95 return r; 96 } 97 98 static inline struct cpuid cpuid_indexed(u32 function, u32 index) 99 { 100 u32 level = raw_cpuid(function & 0xf0000000, 0).a; 101 if (level < function) 102 return (struct cpuid) { 0, 0, 0, 0 }; 103 return raw_cpuid(function, index); 104 } 105 106 static inline struct cpuid cpuid(u32 function) 107 { 108 return cpuid_indexed(function, 0); 109 } 110 111 static inline u8 cpuid_maxphyaddr(void) 112 { 113 if (raw_cpuid(0x80000000, 0).a < 0x80000008) 114 return 36; 115 return raw_cpuid(0x80000008, 0).a & 0xff; 116 } 117 118 #define CPUID(a, b, c, d) ((((unsigned long long) a) << 32) | (b << 16) | \ 119 (c << 8) | d) 120 121 /* 122 * Each X86_FEATURE_XXX definition is 64-bit and contains the following 123 * CPUID meta-data: 124 * 125 * [63:32] : input value for EAX 126 * [31:16] : input value for ECX 127 * [15:8] : output register 128 * [7:0] : bit position in output register 129 */ 130 131 /* 132 * Intel CPUID features 133 */ 134 #define X86_FEATURE_MWAIT (CPUID(0x1, 0, ECX, 3)) 135 #define X86_FEATURE_VMX (CPUID(0x1, 0, ECX, 5)) 136 #define X86_FEATURE_PCID (CPUID(0x1, 0, ECX, 17)) 137 #define X86_FEATURE_MOVBE (CPUID(0x1, 0, ECX, 22)) 138 #define X86_FEATURE_TSC_DEADLINE_TIMER (CPUID(0x1, 0, ECX, 24)) 139 #define X86_FEATURE_XSAVE (CPUID(0x1, 0, ECX, 26)) 140 #define X86_FEATURE_OSXSAVE (CPUID(0x1, 0, ECX, 27)) 141 #define X86_FEATURE_RDRAND (CPUID(0x1, 0, ECX, 30)) 142 #define X86_FEATURE_MCE (CPUID(0x1, 0, EDX, 7)) 143 #define X86_FEATURE_APIC (CPUID(0x1, 0, EDX, 9)) 144 #define X86_FEATURE_CLFLUSH (CPUID(0x1, 0, EDX, 19)) 145 #define X86_FEATURE_XMM (CPUID(0x1, 0, EDX, 25)) 146 #define X86_FEATURE_XMM2 (CPUID(0x1, 0, EDX, 26)) 147 #define X86_FEATURE_TSC_ADJUST (CPUID(0x7, 0, EBX, 1)) 148 #define X86_FEATURE_HLE (CPUID(0x7, 0, EBX, 4)) 149 #define X86_FEATURE_SMEP (CPUID(0x7, 0, EBX, 7)) 150 #define X86_FEATURE_INVPCID (CPUID(0x7, 0, EBX, 10)) 151 #define X86_FEATURE_RTM (CPUID(0x7, 0, EBX, 11)) 152 #define X86_FEATURE_SMAP (CPUID(0x7, 0, EBX, 20)) 153 #define X86_FEATURE_PCOMMIT (CPUID(0x7, 0, EBX, 22)) 154 #define X86_FEATURE_CLFLUSHOPT (CPUID(0x7, 0, EBX, 23)) 155 #define X86_FEATURE_CLWB (CPUID(0x7, 0, EBX, 24)) 156 #define X86_FEATURE_UMIP (CPUID(0x7, 0, ECX, 2)) 157 #define X86_FEATURE_PKU (CPUID(0x7, 0, ECX, 3)) 158 #define X86_FEATURE_LA57 (CPUID(0x7, 0, ECX, 16)) 159 #define X86_FEATURE_RDPID (CPUID(0x7, 0, ECX, 22)) 160 #define X86_FEATURE_SPEC_CTRL (CPUID(0x7, 0, EDX, 26)) 161 #define X86_FEATURE_ARCH_CAPABILITIES (CPUID(0x7, 0, EDX, 29)) 162 #define X86_FEATURE_NX (CPUID(0x80000001, 0, EDX, 20)) 163 #define X86_FEATURE_RDPRU (CPUID(0x80000008, 0, EBX, 4)) 164 165 /* 166 * AMD CPUID features 167 */ 168 #define X86_FEATURE_SVM (CPUID(0x80000001, 0, ECX, 2)) 169 #define X86_FEATURE_RDTSCP (CPUID(0x80000001, 0, EDX, 27)) 170 #define X86_FEATURE_AMD_IBPB (CPUID(0x80000008, 0, EBX, 12)) 171 #define X86_FEATURE_NPT (CPUID(0x8000000A, 0, EDX, 0)) 172 #define X86_FEATURE_NRIPS (CPUID(0x8000000A, 0, EDX, 3)) 173 174 175 static inline bool this_cpu_has(u64 feature) 176 { 177 u32 input_eax = feature >> 32; 178 u32 input_ecx = (feature >> 16) & 0xffff; 179 u32 output_reg = (feature >> 8) & 0xff; 180 u8 bit = feature & 0xff; 181 struct cpuid c; 182 u32 *tmp; 183 184 c = cpuid_indexed(input_eax, input_ecx); 185 tmp = (u32 *)&c; 186 187 return ((*(tmp + (output_reg % 32))) & (1 << bit)); 188 } 189 190 struct far_pointer32 { 191 u32 offset; 192 u16 selector; 193 } __attribute__((packed)); 194 195 struct descriptor_table_ptr { 196 u16 limit; 197 ulong base; 198 } __attribute__((packed)); 199 200 static inline void barrier(void) 201 { 202 asm volatile ("" : : : "memory"); 203 } 204 205 static inline void clac(void) 206 { 207 asm volatile (".byte 0x0f, 0x01, 0xca" : : : "memory"); 208 } 209 210 static inline void stac(void) 211 { 212 asm volatile (".byte 0x0f, 0x01, 0xcb" : : : "memory"); 213 } 214 215 static inline u16 read_cs(void) 216 { 217 unsigned val; 218 219 asm volatile ("mov %%cs, %0" : "=mr"(val)); 220 return val; 221 } 222 223 static inline u16 read_ds(void) 224 { 225 unsigned val; 226 227 asm volatile ("mov %%ds, %0" : "=mr"(val)); 228 return val; 229 } 230 231 static inline u16 read_es(void) 232 { 233 unsigned val; 234 235 asm volatile ("mov %%es, %0" : "=mr"(val)); 236 return val; 237 } 238 239 static inline u16 read_ss(void) 240 { 241 unsigned val; 242 243 asm volatile ("mov %%ss, %0" : "=mr"(val)); 244 return val; 245 } 246 247 static inline u16 read_fs(void) 248 { 249 unsigned val; 250 251 asm volatile ("mov %%fs, %0" : "=mr"(val)); 252 return val; 253 } 254 255 static inline u16 read_gs(void) 256 { 257 unsigned val; 258 259 asm volatile ("mov %%gs, %0" : "=mr"(val)); 260 return val; 261 } 262 263 static inline unsigned long read_rflags(void) 264 { 265 unsigned long f; 266 asm volatile ("pushf; pop %0\n\t" : "=rm"(f)); 267 return f; 268 } 269 270 static inline void write_ds(unsigned val) 271 { 272 asm volatile ("mov %0, %%ds" : : "rm"(val) : "memory"); 273 } 274 275 static inline void write_es(unsigned val) 276 { 277 asm volatile ("mov %0, %%es" : : "rm"(val) : "memory"); 278 } 279 280 static inline void write_ss(unsigned val) 281 { 282 asm volatile ("mov %0, %%ss" : : "rm"(val) : "memory"); 283 } 284 285 static inline void write_fs(unsigned val) 286 { 287 asm volatile ("mov %0, %%fs" : : "rm"(val) : "memory"); 288 } 289 290 static inline void write_gs(unsigned val) 291 { 292 asm volatile ("mov %0, %%gs" : : "rm"(val) : "memory"); 293 } 294 295 static inline void write_rflags(unsigned long f) 296 { 297 asm volatile ("push %0; popf\n\t" : : "rm"(f)); 298 } 299 300 static inline void set_iopl(int iopl) 301 { 302 unsigned long flags = read_rflags() & ~X86_EFLAGS_IOPL; 303 flags |= iopl * (X86_EFLAGS_IOPL / 3); 304 write_rflags(flags); 305 } 306 307 static inline u64 rdmsr(u32 index) 308 { 309 u32 a, d; 310 asm volatile ("rdmsr" : "=a"(a), "=d"(d) : "c"(index) : "memory"); 311 return a | ((u64)d << 32); 312 } 313 314 static inline void wrmsr(u32 index, u64 val) 315 { 316 u32 a = val, d = val >> 32; 317 asm volatile ("wrmsr" : : "a"(a), "d"(d), "c"(index) : "memory"); 318 } 319 320 static inline uint64_t rdpmc(uint32_t index) 321 { 322 uint32_t a, d; 323 asm volatile ("rdpmc" : "=a"(a), "=d"(d) : "c"(index)); 324 return a | ((uint64_t)d << 32); 325 } 326 327 static inline void write_cr0(ulong val) 328 { 329 asm volatile ("mov %0, %%cr0" : : "r"(val) : "memory"); 330 } 331 332 static inline ulong read_cr0(void) 333 { 334 ulong val; 335 asm volatile ("mov %%cr0, %0" : "=r"(val) : : "memory"); 336 return val; 337 } 338 339 static inline void write_cr2(ulong val) 340 { 341 asm volatile ("mov %0, %%cr2" : : "r"(val) : "memory"); 342 } 343 344 static inline ulong read_cr2(void) 345 { 346 ulong val; 347 asm volatile ("mov %%cr2, %0" : "=r"(val) : : "memory"); 348 return val; 349 } 350 351 static inline void write_cr3(ulong val) 352 { 353 asm volatile ("mov %0, %%cr3" : : "r"(val) : "memory"); 354 } 355 356 static inline ulong read_cr3(void) 357 { 358 ulong val; 359 asm volatile ("mov %%cr3, %0" : "=r"(val) : : "memory"); 360 return val; 361 } 362 363 static inline void update_cr3(void *cr3) 364 { 365 write_cr3((ulong)cr3); 366 } 367 368 static inline void write_cr4(ulong val) 369 { 370 asm volatile ("mov %0, %%cr4" : : "r"(val) : "memory"); 371 } 372 373 static inline ulong read_cr4(void) 374 { 375 ulong val; 376 asm volatile ("mov %%cr4, %0" : "=r"(val) : : "memory"); 377 return val; 378 } 379 380 static inline void write_cr8(ulong val) 381 { 382 asm volatile ("mov %0, %%cr8" : : "r"(val) : "memory"); 383 } 384 385 static inline ulong read_cr8(void) 386 { 387 ulong val; 388 asm volatile ("mov %%cr8, %0" : "=r"(val) : : "memory"); 389 return val; 390 } 391 392 static inline void lgdt(const struct descriptor_table_ptr *ptr) 393 { 394 asm volatile ("lgdt %0" : : "m"(*ptr)); 395 } 396 397 static inline void sgdt(struct descriptor_table_ptr *ptr) 398 { 399 asm volatile ("sgdt %0" : "=m"(*ptr)); 400 } 401 402 static inline void lidt(const struct descriptor_table_ptr *ptr) 403 { 404 asm volatile ("lidt %0" : : "m"(*ptr)); 405 } 406 407 static inline void sidt(struct descriptor_table_ptr *ptr) 408 { 409 asm volatile ("sidt %0" : "=m"(*ptr)); 410 } 411 412 static inline void lldt(unsigned val) 413 { 414 asm volatile ("lldt %0" : : "rm"(val)); 415 } 416 417 static inline u16 sldt(void) 418 { 419 u16 val; 420 asm volatile ("sldt %0" : "=rm"(val)); 421 return val; 422 } 423 424 static inline void ltr(u16 val) 425 { 426 asm volatile ("ltr %0" : : "rm"(val)); 427 } 428 429 static inline u16 str(void) 430 { 431 u16 val; 432 asm volatile ("str %0" : "=rm"(val)); 433 return val; 434 } 435 436 static inline void write_dr6(ulong val) 437 { 438 asm volatile ("mov %0, %%dr6" : : "r"(val) : "memory"); 439 } 440 441 static inline ulong read_dr6(void) 442 { 443 ulong val; 444 asm volatile ("mov %%dr6, %0" : "=r"(val)); 445 return val; 446 } 447 448 static inline void write_dr7(ulong val) 449 { 450 asm volatile ("mov %0, %%dr7" : : "r"(val) : "memory"); 451 } 452 453 static inline ulong read_dr7(void) 454 { 455 ulong val; 456 asm volatile ("mov %%dr7, %0" : "=r"(val)); 457 return val; 458 } 459 460 static inline void pause(void) 461 { 462 asm volatile ("pause"); 463 } 464 465 static inline void cli(void) 466 { 467 asm volatile ("cli"); 468 } 469 470 static inline void sti(void) 471 { 472 asm volatile ("sti"); 473 } 474 475 static inline unsigned long long rdtsc(void) 476 { 477 long long r; 478 479 #ifdef __x86_64__ 480 unsigned a, d; 481 482 asm volatile ("rdtsc" : "=a"(a), "=d"(d)); 483 r = a | ((long long)d << 32); 484 #else 485 asm volatile ("rdtsc" : "=A"(r)); 486 #endif 487 return r; 488 } 489 490 /* 491 * Per the advice in the SDM, volume 2, the sequence "mfence; lfence" 492 * executed immediately before rdtsc ensures that rdtsc will be 493 * executed only after all previous instructions have executed and all 494 * previous loads and stores are globally visible. In addition, the 495 * lfence immediately after rdtsc ensures that rdtsc will be executed 496 * prior to the execution of any subsequent instruction. 497 */ 498 static inline unsigned long long fenced_rdtsc(void) 499 { 500 unsigned long long tsc; 501 502 #ifdef __x86_64__ 503 unsigned int eax, edx; 504 505 asm volatile ("mfence; lfence; rdtsc; lfence" : "=a"(eax), "=d"(edx)); 506 tsc = eax | ((unsigned long long)edx << 32); 507 #else 508 asm volatile ("mfence; lfence; rdtsc; lfence" : "=A"(tsc)); 509 #endif 510 return tsc; 511 } 512 513 static inline unsigned long long rdtscp(u32 *aux) 514 { 515 long long r; 516 517 #ifdef __x86_64__ 518 unsigned a, d; 519 520 asm volatile ("rdtscp" : "=a"(a), "=d"(d), "=c"(*aux)); 521 r = a | ((long long)d << 32); 522 #else 523 asm volatile ("rdtscp" : "=A"(r), "=c"(*aux)); 524 #endif 525 return r; 526 } 527 528 static inline void wrtsc(u64 tsc) 529 { 530 unsigned a = tsc, d = tsc >> 32; 531 532 asm volatile("wrmsr" : : "a"(a), "d"(d), "c"(0x10)); 533 } 534 535 static inline void irq_disable(void) 536 { 537 asm volatile("cli"); 538 } 539 540 /* Note that irq_enable() does not ensure an interrupt shadow due 541 * to the vagaries of compiler optimizations. If you need the 542 * shadow, use a single asm with "sti" and the instruction after it. 543 */ 544 static inline void irq_enable(void) 545 { 546 asm volatile("sti"); 547 } 548 549 static inline void invlpg(volatile void *va) 550 { 551 asm volatile("invlpg (%0)" ::"r" (va) : "memory"); 552 } 553 554 static inline void safe_halt(void) 555 { 556 asm volatile("sti; hlt"); 557 } 558 559 static inline u32 read_pkru(void) 560 { 561 unsigned int eax, edx; 562 unsigned int ecx = 0; 563 unsigned int pkru; 564 565 asm volatile(".byte 0x0f,0x01,0xee\n\t" 566 : "=a" (eax), "=d" (edx) 567 : "c" (ecx)); 568 pkru = eax; 569 return pkru; 570 } 571 572 static inline void write_pkru(u32 pkru) 573 { 574 unsigned int eax = pkru; 575 unsigned int ecx = 0; 576 unsigned int edx = 0; 577 578 asm volatile(".byte 0x0f,0x01,0xef\n\t" 579 : : "a" (eax), "c" (ecx), "d" (edx)); 580 } 581 582 static inline bool is_canonical(u64 addr) 583 { 584 return (s64)(addr << 16) >> 16 == addr; 585 } 586 587 static inline void clear_bit(int bit, u8 *addr) 588 { 589 __asm__ __volatile__("btr %1, %0" 590 : "+m" (*addr) : "Ir" (bit) : "cc", "memory"); 591 } 592 593 static inline void set_bit(int bit, u8 *addr) 594 { 595 __asm__ __volatile__("bts %1, %0" 596 : "+m" (*addr) : "Ir" (bit) : "cc", "memory"); 597 } 598 599 static inline void flush_tlb(void) 600 { 601 ulong cr4; 602 603 cr4 = read_cr4(); 604 write_cr4(cr4 ^ X86_CR4_PGE); 605 write_cr4(cr4); 606 } 607 608 static inline int has_spec_ctrl(void) 609 { 610 return !!(this_cpu_has(X86_FEATURE_SPEC_CTRL)); 611 } 612 613 static inline int cpu_has_efer_nx(void) 614 { 615 return !!(this_cpu_has(X86_FEATURE_NX)); 616 } 617 618 static inline bool cpuid_osxsave(void) 619 { 620 return cpuid(1).c & (1 << (X86_FEATURE_OSXSAVE % 32)); 621 } 622 623 #endif 624