1 #ifndef LIBCFLAT_PROCESSOR_H 2 #define LIBCFLAT_PROCESSOR_H 3 4 #include "libcflat.h" 5 #include "msr.h" 6 #include <stdint.h> 7 8 #ifdef __x86_64__ 9 # define R "r" 10 # define W "q" 11 # define S "8" 12 #else 13 # define R "e" 14 # define W "l" 15 # define S "4" 16 #endif 17 18 #define DB_VECTOR 1 19 #define BP_VECTOR 3 20 #define UD_VECTOR 6 21 #define DF_VECTOR 8 22 #define TS_VECTOR 10 23 #define NP_VECTOR 11 24 #define SS_VECTOR 12 25 #define GP_VECTOR 13 26 #define PF_VECTOR 14 27 #define AC_VECTOR 17 28 29 #define X86_CR0_PE 0x00000001 30 #define X86_CR0_MP 0x00000002 31 #define X86_CR0_EM 0x00000004 32 #define X86_CR0_TS 0x00000008 33 #define X86_CR0_WP 0x00010000 34 #define X86_CR0_AM 0x00040000 35 #define X86_CR0_NW 0x20000000 36 #define X86_CR0_CD 0x40000000 37 #define X86_CR0_PG 0x80000000 38 #define X86_CR3_PCID_MASK 0x00000fff 39 #define X86_CR4_TSD 0x00000004 40 #define X86_CR4_DE 0x00000008 41 #define X86_CR4_PSE 0x00000010 42 #define X86_CR4_PAE 0x00000020 43 #define X86_CR4_MCE 0x00000040 44 #define X86_CR4_PGE 0x00000080 45 #define X86_CR4_PCE 0x00000100 46 #define X86_CR4_UMIP 0x00000800 47 #define X86_CR4_LA57 0x00001000 48 #define X86_CR4_VMXE 0x00002000 49 #define X86_CR4_PCIDE 0x00020000 50 #define X86_CR4_SMEP 0x00100000 51 #define X86_CR4_SMAP 0x00200000 52 #define X86_CR4_PKE 0x00400000 53 54 #define X86_EFLAGS_CF 0x00000001 55 #define X86_EFLAGS_FIXED 0x00000002 56 #define X86_EFLAGS_PF 0x00000004 57 #define X86_EFLAGS_AF 0x00000010 58 #define X86_EFLAGS_ZF 0x00000040 59 #define X86_EFLAGS_SF 0x00000080 60 #define X86_EFLAGS_TF 0x00000100 61 #define X86_EFLAGS_IF 0x00000200 62 #define X86_EFLAGS_DF 0x00000400 63 #define X86_EFLAGS_OF 0x00000800 64 #define X86_EFLAGS_IOPL 0x00003000 65 #define X86_EFLAGS_NT 0x00004000 66 #define X86_EFLAGS_AC 0x00040000 67 68 #define X86_EFLAGS_ALU (X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | \ 69 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF) 70 71 #define X86_IA32_EFER 0xc0000080 72 #define X86_EFER_LMA (1UL << 8) 73 74 /* 75 * CPU features 76 */ 77 78 enum cpuid_output_regs { 79 EAX, 80 EBX, 81 ECX, 82 EDX 83 }; 84 85 struct cpuid { u32 a, b, c, d; }; 86 87 static inline struct cpuid raw_cpuid(u32 function, u32 index) 88 { 89 struct cpuid r; 90 asm volatile ("cpuid" 91 : "=a"(r.a), "=b"(r.b), "=c"(r.c), "=d"(r.d) 92 : "0"(function), "2"(index)); 93 return r; 94 } 95 96 static inline struct cpuid cpuid_indexed(u32 function, u32 index) 97 { 98 u32 level = raw_cpuid(function & 0xf0000000, 0).a; 99 if (level < function) 100 return (struct cpuid) { 0, 0, 0, 0 }; 101 return raw_cpuid(function, index); 102 } 103 104 static inline struct cpuid cpuid(u32 function) 105 { 106 return cpuid_indexed(function, 0); 107 } 108 109 static inline u8 cpuid_maxphyaddr(void) 110 { 111 if (raw_cpuid(0x80000000, 0).a < 0x80000008) 112 return 36; 113 return raw_cpuid(0x80000008, 0).a & 0xff; 114 } 115 116 #define CPUID(a, b, c, d) ((((unsigned long long) a) << 32) | (b << 16) | \ 117 (c << 8) | d) 118 119 /* 120 * Each X86_FEATURE_XXX definition is 64-bit and contains the following 121 * CPUID meta-data: 122 * 123 * [63:32] : input value for EAX 124 * [31:16] : input value for ECX 125 * [15:8] : output register 126 * [7:0] : bit position in output register 127 */ 128 129 /* 130 * Intel CPUID features 131 */ 132 #define X86_FEATURE_MWAIT (CPUID(0x1, 0, ECX, 3)) 133 #define X86_FEATURE_VMX (CPUID(0x1, 0, ECX, 5)) 134 #define X86_FEATURE_PCID (CPUID(0x1, 0, ECX, 17)) 135 #define X86_FEATURE_MOVBE (CPUID(0x1, 0, ECX, 22)) 136 #define X86_FEATURE_TSC_DEADLINE_TIMER (CPUID(0x1, 0, ECX, 24)) 137 #define X86_FEATURE_XSAVE (CPUID(0x1, 0, ECX, 26)) 138 #define X86_FEATURE_OSXSAVE (CPUID(0x1, 0, ECX, 27)) 139 #define X86_FEATURE_RDRAND (CPUID(0x1, 0, ECX, 30)) 140 #define X86_FEATURE_MCE (CPUID(0x1, 0, EDX, 7)) 141 #define X86_FEATURE_APIC (CPUID(0x1, 0, EDX, 9)) 142 #define X86_FEATURE_CLFLUSH (CPUID(0x1, 0, EDX, 19)) 143 #define X86_FEATURE_XMM (CPUID(0x1, 0, EDX, 25)) 144 #define X86_FEATURE_XMM2 (CPUID(0x1, 0, EDX, 26)) 145 #define X86_FEATURE_TSC_ADJUST (CPUID(0x7, 0, EBX, 1)) 146 #define X86_FEATURE_HLE (CPUID(0x7, 0, EBX, 4)) 147 #define X86_FEATURE_SMEP (CPUID(0x7, 0, EBX, 7)) 148 #define X86_FEATURE_INVPCID (CPUID(0x7, 0, EBX, 10)) 149 #define X86_FEATURE_RTM (CPUID(0x7, 0, EBX, 11)) 150 #define X86_FEATURE_SMAP (CPUID(0x7, 0, EBX, 20)) 151 #define X86_FEATURE_PCOMMIT (CPUID(0x7, 0, EBX, 22)) 152 #define X86_FEATURE_CLFLUSHOPT (CPUID(0x7, 0, EBX, 23)) 153 #define X86_FEATURE_CLWB (CPUID(0x7, 0, EBX, 24)) 154 #define X86_FEATURE_UMIP (CPUID(0x7, 0, ECX, 2)) 155 #define X86_FEATURE_PKU (CPUID(0x7, 0, ECX, 3)) 156 #define X86_FEATURE_LA57 (CPUID(0x7, 0, ECX, 16)) 157 #define X86_FEATURE_RDPID (CPUID(0x7, 0, ECX, 22)) 158 #define X86_FEATURE_SPEC_CTRL (CPUID(0x7, 0, EDX, 26)) 159 #define X86_FEATURE_ARCH_CAPABILITIES (CPUID(0x7, 0, EDX, 29)) 160 #define X86_FEATURE_NX (CPUID(0x80000001, 0, EDX, 20)) 161 #define X86_FEATURE_RDPRU (CPUID(0x80000008, 0, EBX, 4)) 162 163 /* 164 * AMD CPUID features 165 */ 166 #define X86_FEATURE_SVM (CPUID(0x80000001, 0, ECX, 2)) 167 #define X86_FEATURE_RDTSCP (CPUID(0x80000001, 0, EDX, 27)) 168 #define X86_FEATURE_AMD_IBPB (CPUID(0x80000008, 0, EBX, 12)) 169 #define X86_FEATURE_NPT (CPUID(0x8000000A, 0, EDX, 0)) 170 #define X86_FEATURE_NRIPS (CPUID(0x8000000A, 0, EDX, 3)) 171 172 173 static inline bool this_cpu_has(u64 feature) 174 { 175 u32 input_eax = feature >> 32; 176 u32 input_ecx = (feature >> 16) & 0xffff; 177 u32 output_reg = (feature >> 8) & 0xff; 178 u8 bit = feature & 0xff; 179 struct cpuid c; 180 u32 *tmp; 181 182 c = cpuid_indexed(input_eax, input_ecx); 183 tmp = (u32 *)&c; 184 185 return ((*(tmp + (output_reg % 32))) & (1 << bit)); 186 } 187 188 struct far_pointer32 { 189 u32 offset; 190 u16 selector; 191 } __attribute__((packed)); 192 193 struct descriptor_table_ptr { 194 u16 limit; 195 ulong base; 196 } __attribute__((packed)); 197 198 static inline void barrier(void) 199 { 200 asm volatile ("" : : : "memory"); 201 } 202 203 static inline void clac(void) 204 { 205 asm volatile (".byte 0x0f, 0x01, 0xca" : : : "memory"); 206 } 207 208 static inline void stac(void) 209 { 210 asm volatile (".byte 0x0f, 0x01, 0xcb" : : : "memory"); 211 } 212 213 static inline u16 read_cs(void) 214 { 215 unsigned val; 216 217 asm volatile ("mov %%cs, %0" : "=mr"(val)); 218 return val; 219 } 220 221 static inline u16 read_ds(void) 222 { 223 unsigned val; 224 225 asm volatile ("mov %%ds, %0" : "=mr"(val)); 226 return val; 227 } 228 229 static inline u16 read_es(void) 230 { 231 unsigned val; 232 233 asm volatile ("mov %%es, %0" : "=mr"(val)); 234 return val; 235 } 236 237 static inline u16 read_ss(void) 238 { 239 unsigned val; 240 241 asm volatile ("mov %%ss, %0" : "=mr"(val)); 242 return val; 243 } 244 245 static inline u16 read_fs(void) 246 { 247 unsigned val; 248 249 asm volatile ("mov %%fs, %0" : "=mr"(val)); 250 return val; 251 } 252 253 static inline u16 read_gs(void) 254 { 255 unsigned val; 256 257 asm volatile ("mov %%gs, %0" : "=mr"(val)); 258 return val; 259 } 260 261 static inline unsigned long read_rflags(void) 262 { 263 unsigned long f; 264 asm volatile ("pushf; pop %0\n\t" : "=rm"(f)); 265 return f; 266 } 267 268 static inline void write_ds(unsigned val) 269 { 270 asm volatile ("mov %0, %%ds" : : "rm"(val) : "memory"); 271 } 272 273 static inline void write_es(unsigned val) 274 { 275 asm volatile ("mov %0, %%es" : : "rm"(val) : "memory"); 276 } 277 278 static inline void write_ss(unsigned val) 279 { 280 asm volatile ("mov %0, %%ss" : : "rm"(val) : "memory"); 281 } 282 283 static inline void write_fs(unsigned val) 284 { 285 asm volatile ("mov %0, %%fs" : : "rm"(val) : "memory"); 286 } 287 288 static inline void write_gs(unsigned val) 289 { 290 asm volatile ("mov %0, %%gs" : : "rm"(val) : "memory"); 291 } 292 293 static inline void write_rflags(unsigned long f) 294 { 295 asm volatile ("push %0; popf\n\t" : : "rm"(f)); 296 } 297 298 static inline void set_iopl(int iopl) 299 { 300 unsigned long flags = read_rflags() & ~X86_EFLAGS_IOPL; 301 flags |= iopl * (X86_EFLAGS_IOPL / 3); 302 write_rflags(flags); 303 } 304 305 static inline u64 rdmsr(u32 index) 306 { 307 u32 a, d; 308 asm volatile ("rdmsr" : "=a"(a), "=d"(d) : "c"(index) : "memory"); 309 return a | ((u64)d << 32); 310 } 311 312 static inline void wrmsr(u32 index, u64 val) 313 { 314 u32 a = val, d = val >> 32; 315 asm volatile ("wrmsr" : : "a"(a), "d"(d), "c"(index) : "memory"); 316 } 317 318 static inline uint64_t rdpmc(uint32_t index) 319 { 320 uint32_t a, d; 321 asm volatile ("rdpmc" : "=a"(a), "=d"(d) : "c"(index)); 322 return a | ((uint64_t)d << 32); 323 } 324 325 static inline void write_cr0(ulong val) 326 { 327 asm volatile ("mov %0, %%cr0" : : "r"(val) : "memory"); 328 } 329 330 static inline ulong read_cr0(void) 331 { 332 ulong val; 333 asm volatile ("mov %%cr0, %0" : "=r"(val) : : "memory"); 334 return val; 335 } 336 337 static inline void write_cr2(ulong val) 338 { 339 asm volatile ("mov %0, %%cr2" : : "r"(val) : "memory"); 340 } 341 342 static inline ulong read_cr2(void) 343 { 344 ulong val; 345 asm volatile ("mov %%cr2, %0" : "=r"(val) : : "memory"); 346 return val; 347 } 348 349 static inline void write_cr3(ulong val) 350 { 351 asm volatile ("mov %0, %%cr3" : : "r"(val) : "memory"); 352 } 353 354 static inline ulong read_cr3(void) 355 { 356 ulong val; 357 asm volatile ("mov %%cr3, %0" : "=r"(val) : : "memory"); 358 return val; 359 } 360 361 static inline void write_cr4(ulong val) 362 { 363 asm volatile ("mov %0, %%cr4" : : "r"(val) : "memory"); 364 } 365 366 static inline ulong read_cr4(void) 367 { 368 ulong val; 369 asm volatile ("mov %%cr4, %0" : "=r"(val) : : "memory"); 370 return val; 371 } 372 373 static inline void write_cr8(ulong val) 374 { 375 asm volatile ("mov %0, %%cr8" : : "r"(val) : "memory"); 376 } 377 378 static inline ulong read_cr8(void) 379 { 380 ulong val; 381 asm volatile ("mov %%cr8, %0" : "=r"(val) : : "memory"); 382 return val; 383 } 384 385 static inline void lgdt(const struct descriptor_table_ptr *ptr) 386 { 387 asm volatile ("lgdt %0" : : "m"(*ptr)); 388 } 389 390 static inline void sgdt(struct descriptor_table_ptr *ptr) 391 { 392 asm volatile ("sgdt %0" : "=m"(*ptr)); 393 } 394 395 static inline void lidt(const struct descriptor_table_ptr *ptr) 396 { 397 asm volatile ("lidt %0" : : "m"(*ptr)); 398 } 399 400 static inline void sidt(struct descriptor_table_ptr *ptr) 401 { 402 asm volatile ("sidt %0" : "=m"(*ptr)); 403 } 404 405 static inline void lldt(unsigned val) 406 { 407 asm volatile ("lldt %0" : : "rm"(val)); 408 } 409 410 static inline u16 sldt(void) 411 { 412 u16 val; 413 asm volatile ("sldt %0" : "=rm"(val)); 414 return val; 415 } 416 417 static inline void ltr(u16 val) 418 { 419 asm volatile ("ltr %0" : : "rm"(val)); 420 } 421 422 static inline u16 str(void) 423 { 424 u16 val; 425 asm volatile ("str %0" : "=rm"(val)); 426 return val; 427 } 428 429 static inline void write_dr6(ulong val) 430 { 431 asm volatile ("mov %0, %%dr6" : : "r"(val) : "memory"); 432 } 433 434 static inline ulong read_dr6(void) 435 { 436 ulong val; 437 asm volatile ("mov %%dr6, %0" : "=r"(val)); 438 return val; 439 } 440 441 static inline void write_dr7(ulong val) 442 { 443 asm volatile ("mov %0, %%dr7" : : "r"(val) : "memory"); 444 } 445 446 static inline ulong read_dr7(void) 447 { 448 ulong val; 449 asm volatile ("mov %%dr7, %0" : "=r"(val)); 450 return val; 451 } 452 453 static inline void pause(void) 454 { 455 asm volatile ("pause"); 456 } 457 458 static inline void cli(void) 459 { 460 asm volatile ("cli"); 461 } 462 463 static inline void sti(void) 464 { 465 asm volatile ("sti"); 466 } 467 468 static inline unsigned long long rdtsc(void) 469 { 470 long long r; 471 472 #ifdef __x86_64__ 473 unsigned a, d; 474 475 asm volatile ("rdtsc" : "=a"(a), "=d"(d)); 476 r = a | ((long long)d << 32); 477 #else 478 asm volatile ("rdtsc" : "=A"(r)); 479 #endif 480 return r; 481 } 482 483 /* 484 * Per the advice in the SDM, volume 2, the sequence "mfence; lfence" 485 * executed immediately before rdtsc ensures that rdtsc will be 486 * executed only after all previous instructions have executed and all 487 * previous loads and stores are globally visible. In addition, the 488 * lfence immediately after rdtsc ensures that rdtsc will be executed 489 * prior to the execution of any subsequent instruction. 490 */ 491 static inline unsigned long long fenced_rdtsc(void) 492 { 493 unsigned long long tsc; 494 495 #ifdef __x86_64__ 496 unsigned int eax, edx; 497 498 asm volatile ("mfence; lfence; rdtsc; lfence" : "=a"(eax), "=d"(edx)); 499 tsc = eax | ((unsigned long long)edx << 32); 500 #else 501 asm volatile ("mfence; lfence; rdtsc; lfence" : "=A"(tsc)); 502 #endif 503 return tsc; 504 } 505 506 static inline unsigned long long rdtscp(u32 *aux) 507 { 508 long long r; 509 510 #ifdef __x86_64__ 511 unsigned a, d; 512 513 asm volatile ("rdtscp" : "=a"(a), "=d"(d), "=c"(*aux)); 514 r = a | ((long long)d << 32); 515 #else 516 asm volatile ("rdtscp" : "=A"(r), "=c"(*aux)); 517 #endif 518 return r; 519 } 520 521 static inline void wrtsc(u64 tsc) 522 { 523 unsigned a = tsc, d = tsc >> 32; 524 525 asm volatile("wrmsr" : : "a"(a), "d"(d), "c"(0x10)); 526 } 527 528 static inline void irq_disable(void) 529 { 530 asm volatile("cli"); 531 } 532 533 /* Note that irq_enable() does not ensure an interrupt shadow due 534 * to the vagaries of compiler optimizations. If you need the 535 * shadow, use a single asm with "sti" and the instruction after it. 536 */ 537 static inline void irq_enable(void) 538 { 539 asm volatile("sti"); 540 } 541 542 static inline void invlpg(volatile void *va) 543 { 544 asm volatile("invlpg (%0)" ::"r" (va) : "memory"); 545 } 546 547 static inline void safe_halt(void) 548 { 549 asm volatile("sti; hlt"); 550 } 551 552 static inline u32 read_pkru(void) 553 { 554 unsigned int eax, edx; 555 unsigned int ecx = 0; 556 unsigned int pkru; 557 558 asm volatile(".byte 0x0f,0x01,0xee\n\t" 559 : "=a" (eax), "=d" (edx) 560 : "c" (ecx)); 561 pkru = eax; 562 return pkru; 563 } 564 565 static inline void write_pkru(u32 pkru) 566 { 567 unsigned int eax = pkru; 568 unsigned int ecx = 0; 569 unsigned int edx = 0; 570 571 asm volatile(".byte 0x0f,0x01,0xef\n\t" 572 : : "a" (eax), "c" (ecx), "d" (edx)); 573 } 574 575 static inline bool is_canonical(u64 addr) 576 { 577 return (s64)(addr << 16) >> 16 == addr; 578 } 579 580 static inline void clear_bit(int bit, u8 *addr) 581 { 582 __asm__ __volatile__("btr %1, %0" 583 : "+m" (*addr) : "Ir" (bit) : "cc", "memory"); 584 } 585 586 static inline void set_bit(int bit, u8 *addr) 587 { 588 __asm__ __volatile__("bts %1, %0" 589 : "+m" (*addr) : "Ir" (bit) : "cc", "memory"); 590 } 591 592 static inline void flush_tlb(void) 593 { 594 ulong cr4; 595 596 cr4 = read_cr4(); 597 write_cr4(cr4 ^ X86_CR4_PGE); 598 write_cr4(cr4); 599 } 600 601 static inline int has_spec_ctrl(void) 602 { 603 return !!(this_cpu_has(X86_FEATURE_SPEC_CTRL)); 604 } 605 606 static inline int cpu_has_efer_nx(void) 607 { 608 return !!(this_cpu_has(X86_FEATURE_NX)); 609 } 610 611 #endif 612