1 #ifndef LIBCFLAT_PROCESSOR_H 2 #define LIBCFLAT_PROCESSOR_H 3 4 #include "libcflat.h" 5 #include "msr.h" 6 #include <stdint.h> 7 8 #ifdef __x86_64__ 9 # define R "r" 10 # define W "q" 11 # define S "8" 12 #else 13 # define R "e" 14 # define W "l" 15 # define S "4" 16 #endif 17 18 #define DB_VECTOR 1 19 #define BP_VECTOR 3 20 #define UD_VECTOR 6 21 #define DF_VECTOR 8 22 #define TS_VECTOR 10 23 #define NP_VECTOR 11 24 #define SS_VECTOR 12 25 #define GP_VECTOR 13 26 #define PF_VECTOR 14 27 #define AC_VECTOR 17 28 29 #define X86_CR0_PE 0x00000001 30 #define X86_CR0_MP 0x00000002 31 #define X86_CR0_EM 0x00000004 32 #define X86_CR0_TS 0x00000008 33 #define X86_CR0_WP 0x00010000 34 #define X86_CR0_AM 0x00040000 35 #define X86_CR0_NW 0x20000000 36 #define X86_CR0_CD 0x40000000 37 #define X86_CR0_PG 0x80000000 38 #define X86_CR3_PCID_MASK 0x00000fff 39 #define X86_CR4_TSD 0x00000004 40 #define X86_CR4_DE 0x00000008 41 #define X86_CR4_PSE 0x00000010 42 #define X86_CR4_PAE 0x00000020 43 #define X86_CR4_MCE 0x00000040 44 #define X86_CR4_PGE 0x00000080 45 #define X86_CR4_PCE 0x00000100 46 #define X86_CR4_UMIP 0x00000800 47 #define X86_CR4_LA57 0x00001000 48 #define X86_CR4_VMXE 0x00002000 49 #define X86_CR4_PCIDE 0x00020000 50 #define X86_CR4_SMEP 0x00100000 51 #define X86_CR4_SMAP 0x00200000 52 #define X86_CR4_PKE 0x00400000 53 54 #define X86_EFLAGS_CF 0x00000001 55 #define X86_EFLAGS_FIXED 0x00000002 56 #define X86_EFLAGS_PF 0x00000004 57 #define X86_EFLAGS_AF 0x00000010 58 #define X86_EFLAGS_ZF 0x00000040 59 #define X86_EFLAGS_SF 0x00000080 60 #define X86_EFLAGS_TF 0x00000100 61 #define X86_EFLAGS_IF 0x00000200 62 #define X86_EFLAGS_DF 0x00000400 63 #define X86_EFLAGS_OF 0x00000800 64 #define X86_EFLAGS_IOPL 0x00003000 65 #define X86_EFLAGS_NT 0x00004000 66 #define X86_EFLAGS_VM 0x00020000 67 #define X86_EFLAGS_AC 0x00040000 68 69 #define X86_EFLAGS_ALU (X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | \ 70 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF) 71 72 #define X86_IA32_EFER 0xc0000080 73 #define X86_EFER_LMA (1UL << 8) 74 75 /* 76 * CPU features 77 */ 78 79 enum cpuid_output_regs { 80 EAX, 81 EBX, 82 ECX, 83 EDX 84 }; 85 86 struct cpuid { u32 a, b, c, d; }; 87 88 static inline struct cpuid raw_cpuid(u32 function, u32 index) 89 { 90 struct cpuid r; 91 asm volatile ("cpuid" 92 : "=a"(r.a), "=b"(r.b), "=c"(r.c), "=d"(r.d) 93 : "0"(function), "2"(index)); 94 return r; 95 } 96 97 static inline struct cpuid cpuid_indexed(u32 function, u32 index) 98 { 99 u32 level = raw_cpuid(function & 0xf0000000, 0).a; 100 if (level < function) 101 return (struct cpuid) { 0, 0, 0, 0 }; 102 return raw_cpuid(function, index); 103 } 104 105 static inline struct cpuid cpuid(u32 function) 106 { 107 return cpuid_indexed(function, 0); 108 } 109 110 static inline u8 cpuid_maxphyaddr(void) 111 { 112 if (raw_cpuid(0x80000000, 0).a < 0x80000008) 113 return 36; 114 return raw_cpuid(0x80000008, 0).a & 0xff; 115 } 116 117 #define CPUID(a, b, c, d) ((((unsigned long long) a) << 32) | (b << 16) | \ 118 (c << 8) | d) 119 120 /* 121 * Each X86_FEATURE_XXX definition is 64-bit and contains the following 122 * CPUID meta-data: 123 * 124 * [63:32] : input value for EAX 125 * [31:16] : input value for ECX 126 * [15:8] : output register 127 * [7:0] : bit position in output register 128 */ 129 130 /* 131 * Intel CPUID features 132 */ 133 #define X86_FEATURE_MWAIT (CPUID(0x1, 0, ECX, 3)) 134 #define X86_FEATURE_VMX (CPUID(0x1, 0, ECX, 5)) 135 #define X86_FEATURE_PCID (CPUID(0x1, 0, ECX, 17)) 136 #define X86_FEATURE_MOVBE (CPUID(0x1, 0, ECX, 22)) 137 #define X86_FEATURE_TSC_DEADLINE_TIMER (CPUID(0x1, 0, ECX, 24)) 138 #define X86_FEATURE_XSAVE (CPUID(0x1, 0, ECX, 26)) 139 #define X86_FEATURE_OSXSAVE (CPUID(0x1, 0, ECX, 27)) 140 #define X86_FEATURE_RDRAND (CPUID(0x1, 0, ECX, 30)) 141 #define X86_FEATURE_MCE (CPUID(0x1, 0, EDX, 7)) 142 #define X86_FEATURE_APIC (CPUID(0x1, 0, EDX, 9)) 143 #define X86_FEATURE_CLFLUSH (CPUID(0x1, 0, EDX, 19)) 144 #define X86_FEATURE_XMM (CPUID(0x1, 0, EDX, 25)) 145 #define X86_FEATURE_XMM2 (CPUID(0x1, 0, EDX, 26)) 146 #define X86_FEATURE_TSC_ADJUST (CPUID(0x7, 0, EBX, 1)) 147 #define X86_FEATURE_HLE (CPUID(0x7, 0, EBX, 4)) 148 #define X86_FEATURE_SMEP (CPUID(0x7, 0, EBX, 7)) 149 #define X86_FEATURE_INVPCID (CPUID(0x7, 0, EBX, 10)) 150 #define X86_FEATURE_RTM (CPUID(0x7, 0, EBX, 11)) 151 #define X86_FEATURE_SMAP (CPUID(0x7, 0, EBX, 20)) 152 #define X86_FEATURE_PCOMMIT (CPUID(0x7, 0, EBX, 22)) 153 #define X86_FEATURE_CLFLUSHOPT (CPUID(0x7, 0, EBX, 23)) 154 #define X86_FEATURE_CLWB (CPUID(0x7, 0, EBX, 24)) 155 #define X86_FEATURE_UMIP (CPUID(0x7, 0, ECX, 2)) 156 #define X86_FEATURE_PKU (CPUID(0x7, 0, ECX, 3)) 157 #define X86_FEATURE_LA57 (CPUID(0x7, 0, ECX, 16)) 158 #define X86_FEATURE_RDPID (CPUID(0x7, 0, ECX, 22)) 159 #define X86_FEATURE_SPEC_CTRL (CPUID(0x7, 0, EDX, 26)) 160 #define X86_FEATURE_ARCH_CAPABILITIES (CPUID(0x7, 0, EDX, 29)) 161 #define X86_FEATURE_NX (CPUID(0x80000001, 0, EDX, 20)) 162 #define X86_FEATURE_RDPRU (CPUID(0x80000008, 0, EBX, 4)) 163 164 /* 165 * AMD CPUID features 166 */ 167 #define X86_FEATURE_SVM (CPUID(0x80000001, 0, ECX, 2)) 168 #define X86_FEATURE_RDTSCP (CPUID(0x80000001, 0, EDX, 27)) 169 #define X86_FEATURE_AMD_IBPB (CPUID(0x80000008, 0, EBX, 12)) 170 #define X86_FEATURE_NPT (CPUID(0x8000000A, 0, EDX, 0)) 171 #define X86_FEATURE_NRIPS (CPUID(0x8000000A, 0, EDX, 3)) 172 173 174 static inline bool this_cpu_has(u64 feature) 175 { 176 u32 input_eax = feature >> 32; 177 u32 input_ecx = (feature >> 16) & 0xffff; 178 u32 output_reg = (feature >> 8) & 0xff; 179 u8 bit = feature & 0xff; 180 struct cpuid c; 181 u32 *tmp; 182 183 c = cpuid_indexed(input_eax, input_ecx); 184 tmp = (u32 *)&c; 185 186 return ((*(tmp + (output_reg % 32))) & (1 << bit)); 187 } 188 189 struct far_pointer32 { 190 u32 offset; 191 u16 selector; 192 } __attribute__((packed)); 193 194 struct descriptor_table_ptr { 195 u16 limit; 196 ulong base; 197 } __attribute__((packed)); 198 199 static inline void barrier(void) 200 { 201 asm volatile ("" : : : "memory"); 202 } 203 204 static inline void clac(void) 205 { 206 asm volatile (".byte 0x0f, 0x01, 0xca" : : : "memory"); 207 } 208 209 static inline void stac(void) 210 { 211 asm volatile (".byte 0x0f, 0x01, 0xcb" : : : "memory"); 212 } 213 214 static inline u16 read_cs(void) 215 { 216 unsigned val; 217 218 asm volatile ("mov %%cs, %0" : "=mr"(val)); 219 return val; 220 } 221 222 static inline u16 read_ds(void) 223 { 224 unsigned val; 225 226 asm volatile ("mov %%ds, %0" : "=mr"(val)); 227 return val; 228 } 229 230 static inline u16 read_es(void) 231 { 232 unsigned val; 233 234 asm volatile ("mov %%es, %0" : "=mr"(val)); 235 return val; 236 } 237 238 static inline u16 read_ss(void) 239 { 240 unsigned val; 241 242 asm volatile ("mov %%ss, %0" : "=mr"(val)); 243 return val; 244 } 245 246 static inline u16 read_fs(void) 247 { 248 unsigned val; 249 250 asm volatile ("mov %%fs, %0" : "=mr"(val)); 251 return val; 252 } 253 254 static inline u16 read_gs(void) 255 { 256 unsigned val; 257 258 asm volatile ("mov %%gs, %0" : "=mr"(val)); 259 return val; 260 } 261 262 static inline unsigned long read_rflags(void) 263 { 264 unsigned long f; 265 asm volatile ("pushf; pop %0\n\t" : "=rm"(f)); 266 return f; 267 } 268 269 static inline void write_ds(unsigned val) 270 { 271 asm volatile ("mov %0, %%ds" : : "rm"(val) : "memory"); 272 } 273 274 static inline void write_es(unsigned val) 275 { 276 asm volatile ("mov %0, %%es" : : "rm"(val) : "memory"); 277 } 278 279 static inline void write_ss(unsigned val) 280 { 281 asm volatile ("mov %0, %%ss" : : "rm"(val) : "memory"); 282 } 283 284 static inline void write_fs(unsigned val) 285 { 286 asm volatile ("mov %0, %%fs" : : "rm"(val) : "memory"); 287 } 288 289 static inline void write_gs(unsigned val) 290 { 291 asm volatile ("mov %0, %%gs" : : "rm"(val) : "memory"); 292 } 293 294 static inline void write_rflags(unsigned long f) 295 { 296 asm volatile ("push %0; popf\n\t" : : "rm"(f)); 297 } 298 299 static inline void set_iopl(int iopl) 300 { 301 unsigned long flags = read_rflags() & ~X86_EFLAGS_IOPL; 302 flags |= iopl * (X86_EFLAGS_IOPL / 3); 303 write_rflags(flags); 304 } 305 306 static inline u64 rdmsr(u32 index) 307 { 308 u32 a, d; 309 asm volatile ("rdmsr" : "=a"(a), "=d"(d) : "c"(index) : "memory"); 310 return a | ((u64)d << 32); 311 } 312 313 static inline void wrmsr(u32 index, u64 val) 314 { 315 u32 a = val, d = val >> 32; 316 asm volatile ("wrmsr" : : "a"(a), "d"(d), "c"(index) : "memory"); 317 } 318 319 static inline uint64_t rdpmc(uint32_t index) 320 { 321 uint32_t a, d; 322 asm volatile ("rdpmc" : "=a"(a), "=d"(d) : "c"(index)); 323 return a | ((uint64_t)d << 32); 324 } 325 326 static inline void write_cr0(ulong val) 327 { 328 asm volatile ("mov %0, %%cr0" : : "r"(val) : "memory"); 329 } 330 331 static inline ulong read_cr0(void) 332 { 333 ulong val; 334 asm volatile ("mov %%cr0, %0" : "=r"(val) : : "memory"); 335 return val; 336 } 337 338 static inline void write_cr2(ulong val) 339 { 340 asm volatile ("mov %0, %%cr2" : : "r"(val) : "memory"); 341 } 342 343 static inline ulong read_cr2(void) 344 { 345 ulong val; 346 asm volatile ("mov %%cr2, %0" : "=r"(val) : : "memory"); 347 return val; 348 } 349 350 static inline void write_cr3(ulong val) 351 { 352 asm volatile ("mov %0, %%cr3" : : "r"(val) : "memory"); 353 } 354 355 static inline ulong read_cr3(void) 356 { 357 ulong val; 358 asm volatile ("mov %%cr3, %0" : "=r"(val) : : "memory"); 359 return val; 360 } 361 362 static inline void write_cr4(ulong val) 363 { 364 asm volatile ("mov %0, %%cr4" : : "r"(val) : "memory"); 365 } 366 367 static inline ulong read_cr4(void) 368 { 369 ulong val; 370 asm volatile ("mov %%cr4, %0" : "=r"(val) : : "memory"); 371 return val; 372 } 373 374 static inline void write_cr8(ulong val) 375 { 376 asm volatile ("mov %0, %%cr8" : : "r"(val) : "memory"); 377 } 378 379 static inline ulong read_cr8(void) 380 { 381 ulong val; 382 asm volatile ("mov %%cr8, %0" : "=r"(val) : : "memory"); 383 return val; 384 } 385 386 static inline void lgdt(const struct descriptor_table_ptr *ptr) 387 { 388 asm volatile ("lgdt %0" : : "m"(*ptr)); 389 } 390 391 static inline void sgdt(struct descriptor_table_ptr *ptr) 392 { 393 asm volatile ("sgdt %0" : "=m"(*ptr)); 394 } 395 396 static inline void lidt(const struct descriptor_table_ptr *ptr) 397 { 398 asm volatile ("lidt %0" : : "m"(*ptr)); 399 } 400 401 static inline void sidt(struct descriptor_table_ptr *ptr) 402 { 403 asm volatile ("sidt %0" : "=m"(*ptr)); 404 } 405 406 static inline void lldt(unsigned val) 407 { 408 asm volatile ("lldt %0" : : "rm"(val)); 409 } 410 411 static inline u16 sldt(void) 412 { 413 u16 val; 414 asm volatile ("sldt %0" : "=rm"(val)); 415 return val; 416 } 417 418 static inline void ltr(u16 val) 419 { 420 asm volatile ("ltr %0" : : "rm"(val)); 421 } 422 423 static inline u16 str(void) 424 { 425 u16 val; 426 asm volatile ("str %0" : "=rm"(val)); 427 return val; 428 } 429 430 static inline void write_dr6(ulong val) 431 { 432 asm volatile ("mov %0, %%dr6" : : "r"(val) : "memory"); 433 } 434 435 static inline ulong read_dr6(void) 436 { 437 ulong val; 438 asm volatile ("mov %%dr6, %0" : "=r"(val)); 439 return val; 440 } 441 442 static inline void write_dr7(ulong val) 443 { 444 asm volatile ("mov %0, %%dr7" : : "r"(val) : "memory"); 445 } 446 447 static inline ulong read_dr7(void) 448 { 449 ulong val; 450 asm volatile ("mov %%dr7, %0" : "=r"(val)); 451 return val; 452 } 453 454 static inline void pause(void) 455 { 456 asm volatile ("pause"); 457 } 458 459 static inline void cli(void) 460 { 461 asm volatile ("cli"); 462 } 463 464 static inline void sti(void) 465 { 466 asm volatile ("sti"); 467 } 468 469 static inline unsigned long long rdtsc(void) 470 { 471 long long r; 472 473 #ifdef __x86_64__ 474 unsigned a, d; 475 476 asm volatile ("rdtsc" : "=a"(a), "=d"(d)); 477 r = a | ((long long)d << 32); 478 #else 479 asm volatile ("rdtsc" : "=A"(r)); 480 #endif 481 return r; 482 } 483 484 /* 485 * Per the advice in the SDM, volume 2, the sequence "mfence; lfence" 486 * executed immediately before rdtsc ensures that rdtsc will be 487 * executed only after all previous instructions have executed and all 488 * previous loads and stores are globally visible. In addition, the 489 * lfence immediately after rdtsc ensures that rdtsc will be executed 490 * prior to the execution of any subsequent instruction. 491 */ 492 static inline unsigned long long fenced_rdtsc(void) 493 { 494 unsigned long long tsc; 495 496 #ifdef __x86_64__ 497 unsigned int eax, edx; 498 499 asm volatile ("mfence; lfence; rdtsc; lfence" : "=a"(eax), "=d"(edx)); 500 tsc = eax | ((unsigned long long)edx << 32); 501 #else 502 asm volatile ("mfence; lfence; rdtsc; lfence" : "=A"(tsc)); 503 #endif 504 return tsc; 505 } 506 507 static inline unsigned long long rdtscp(u32 *aux) 508 { 509 long long r; 510 511 #ifdef __x86_64__ 512 unsigned a, d; 513 514 asm volatile ("rdtscp" : "=a"(a), "=d"(d), "=c"(*aux)); 515 r = a | ((long long)d << 32); 516 #else 517 asm volatile ("rdtscp" : "=A"(r), "=c"(*aux)); 518 #endif 519 return r; 520 } 521 522 static inline void wrtsc(u64 tsc) 523 { 524 unsigned a = tsc, d = tsc >> 32; 525 526 asm volatile("wrmsr" : : "a"(a), "d"(d), "c"(0x10)); 527 } 528 529 static inline void irq_disable(void) 530 { 531 asm volatile("cli"); 532 } 533 534 /* Note that irq_enable() does not ensure an interrupt shadow due 535 * to the vagaries of compiler optimizations. If you need the 536 * shadow, use a single asm with "sti" and the instruction after it. 537 */ 538 static inline void irq_enable(void) 539 { 540 asm volatile("sti"); 541 } 542 543 static inline void invlpg(volatile void *va) 544 { 545 asm volatile("invlpg (%0)" ::"r" (va) : "memory"); 546 } 547 548 static inline void safe_halt(void) 549 { 550 asm volatile("sti; hlt"); 551 } 552 553 static inline u32 read_pkru(void) 554 { 555 unsigned int eax, edx; 556 unsigned int ecx = 0; 557 unsigned int pkru; 558 559 asm volatile(".byte 0x0f,0x01,0xee\n\t" 560 : "=a" (eax), "=d" (edx) 561 : "c" (ecx)); 562 pkru = eax; 563 return pkru; 564 } 565 566 static inline void write_pkru(u32 pkru) 567 { 568 unsigned int eax = pkru; 569 unsigned int ecx = 0; 570 unsigned int edx = 0; 571 572 asm volatile(".byte 0x0f,0x01,0xef\n\t" 573 : : "a" (eax), "c" (ecx), "d" (edx)); 574 } 575 576 static inline bool is_canonical(u64 addr) 577 { 578 return (s64)(addr << 16) >> 16 == addr; 579 } 580 581 static inline void clear_bit(int bit, u8 *addr) 582 { 583 __asm__ __volatile__("btr %1, %0" 584 : "+m" (*addr) : "Ir" (bit) : "cc", "memory"); 585 } 586 587 static inline void set_bit(int bit, u8 *addr) 588 { 589 __asm__ __volatile__("bts %1, %0" 590 : "+m" (*addr) : "Ir" (bit) : "cc", "memory"); 591 } 592 593 static inline void flush_tlb(void) 594 { 595 ulong cr4; 596 597 cr4 = read_cr4(); 598 write_cr4(cr4 ^ X86_CR4_PGE); 599 write_cr4(cr4); 600 } 601 602 static inline int has_spec_ctrl(void) 603 { 604 return !!(this_cpu_has(X86_FEATURE_SPEC_CTRL)); 605 } 606 607 static inline int cpu_has_efer_nx(void) 608 { 609 return !!(this_cpu_has(X86_FEATURE_NX)); 610 } 611 612 #endif 613