1 #ifndef LIBCFLAT_PROCESSOR_H 2 #define LIBCFLAT_PROCESSOR_H 3 4 #include "libcflat.h" 5 #include "msr.h" 6 #include <stdint.h> 7 8 #ifdef __x86_64__ 9 # define R "r" 10 # define W "q" 11 # define S "8" 12 #else 13 # define R "e" 14 # define W "l" 15 # define S "4" 16 #endif 17 18 #define DF_VECTOR 8 19 #define TS_VECTOR 10 20 #define NP_VECTOR 11 21 #define SS_VECTOR 12 22 #define GP_VECTOR 13 23 #define PF_VECTOR 14 24 #define AC_VECTOR 17 25 26 #define X86_CR0_PE 0x00000001 27 #define X86_CR0_MP 0x00000002 28 #define X86_CR0_EM 0x00000004 29 #define X86_CR0_TS 0x00000008 30 #define X86_CR0_WP 0x00010000 31 #define X86_CR0_AM 0x00040000 32 #define X86_CR0_PG 0x80000000 33 #define X86_CR3_PCID_MASK 0x00000fff 34 #define X86_CR4_TSD 0x00000004 35 #define X86_CR4_DE 0x00000008 36 #define X86_CR4_PSE 0x00000010 37 #define X86_CR4_PAE 0x00000020 38 #define X86_CR4_MCE 0x00000040 39 #define X86_CR4_PGE 0x00000080 40 #define X86_CR4_PCE 0x00000100 41 #define X86_CR4_UMIP 0x00000800 42 #define X86_CR4_VMXE 0x00002000 43 #define X86_CR4_PCIDE 0x00020000 44 #define X86_CR4_SMEP 0x00100000 45 #define X86_CR4_SMAP 0x00200000 46 #define X86_CR4_PKE 0x00400000 47 48 #define X86_EFLAGS_CF 0x00000001 49 #define X86_EFLAGS_FIXED 0x00000002 50 #define X86_EFLAGS_PF 0x00000004 51 #define X86_EFLAGS_AF 0x00000010 52 #define X86_EFLAGS_ZF 0x00000040 53 #define X86_EFLAGS_SF 0x00000080 54 #define X86_EFLAGS_TF 0x00000100 55 #define X86_EFLAGS_IF 0x00000200 56 #define X86_EFLAGS_DF 0x00000400 57 #define X86_EFLAGS_OF 0x00000800 58 #define X86_EFLAGS_IOPL 0x00003000 59 #define X86_EFLAGS_NT 0x00004000 60 #define X86_EFLAGS_AC 0x00040000 61 62 #define X86_EFLAGS_ALU (X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | \ 63 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF) 64 65 #define X86_IA32_EFER 0xc0000080 66 #define X86_EFER_LMA (1UL << 8) 67 68 /* 69 * CPU features 70 */ 71 72 enum cpuid_output_regs { 73 EAX, 74 EBX, 75 ECX, 76 EDX 77 }; 78 79 struct cpuid { u32 a, b, c, d; }; 80 81 static inline struct cpuid raw_cpuid(u32 function, u32 index) 82 { 83 struct cpuid r; 84 asm volatile ("cpuid" 85 : "=a"(r.a), "=b"(r.b), "=c"(r.c), "=d"(r.d) 86 : "0"(function), "2"(index)); 87 return r; 88 } 89 90 static inline struct cpuid cpuid_indexed(u32 function, u32 index) 91 { 92 u32 level = raw_cpuid(function & 0xf0000000, 0).a; 93 if (level < function) 94 return (struct cpuid) { 0, 0, 0, 0 }; 95 return raw_cpuid(function, index); 96 } 97 98 static inline struct cpuid cpuid(u32 function) 99 { 100 return cpuid_indexed(function, 0); 101 } 102 103 static inline u8 cpuid_maxphyaddr(void) 104 { 105 if (raw_cpuid(0x80000000, 0).a < 0x80000008) 106 return 36; 107 return raw_cpuid(0x80000008, 0).a & 0xff; 108 } 109 110 #define CPUID(a, b, c, d) ((((unsigned long long) a) << 32) | (b << 16) | \ 111 (c << 8) | d) 112 113 /* 114 * Each X86_FEATURE_XXX definition is 64-bit and contains the following 115 * CPUID meta-data: 116 * 117 * [63:32] : input value for EAX 118 * [31:16] : input value for ECX 119 * [15:8] : output register 120 * [7:0] : bit position in output register 121 */ 122 123 /* 124 * Intel CPUID features 125 */ 126 #define X86_FEATURE_MWAIT (CPUID(0x1, 0, ECX, 3)) 127 #define X86_FEATURE_VMX (CPUID(0x1, 0, ECX, 5)) 128 #define X86_FEATURE_PCID (CPUID(0x1, 0, ECX, 17)) 129 #define X86_FEATURE_MOVBE (CPUID(0x1, 0, ECX, 22)) 130 #define X86_FEATURE_TSC_DEADLINE_TIMER (CPUID(0x1, 0, ECX, 24)) 131 #define X86_FEATURE_XSAVE (CPUID(0x1, 0, ECX, 26)) 132 #define X86_FEATURE_OSXSAVE (CPUID(0x1, 0, ECX, 27)) 133 #define X86_FEATURE_RDRAND (CPUID(0x1, 0, ECX, 30)) 134 #define X86_FEATURE_MCE (CPUID(0x1, 0, EDX, 7)) 135 #define X86_FEATURE_APIC (CPUID(0x1, 0, EDX, 9)) 136 #define X86_FEATURE_CLFLUSH (CPUID(0x1, 0, EDX, 19)) 137 #define X86_FEATURE_XMM (CPUID(0x1, 0, EDX, 25)) 138 #define X86_FEATURE_XMM2 (CPUID(0x1, 0, EDX, 26)) 139 #define X86_FEATURE_TSC_ADJUST (CPUID(0x7, 0, EBX, 1)) 140 #define X86_FEATURE_HLE (CPUID(0x7, 0, EBX, 4)) 141 #define X86_FEATURE_INVPCID_SINGLE (CPUID(0x7, 0, EBX, 7)) 142 #define X86_FEATURE_INVPCID (CPUID(0x7, 0, EBX, 10)) 143 #define X86_FEATURE_RTM (CPUID(0x7, 0, EBX, 11)) 144 #define X86_FEATURE_SMAP (CPUID(0x7, 0, EBX, 20)) 145 #define X86_FEATURE_PCOMMIT (CPUID(0x7, 0, EBX, 22)) 146 #define X86_FEATURE_CLFLUSHOPT (CPUID(0x7, 0, EBX, 23)) 147 #define X86_FEATURE_CLWB (CPUID(0x7, 0, EBX, 24)) 148 #define X86_FEATURE_UMIP (CPUID(0x7, 0, ECX, 2)) 149 #define X86_FEATURE_PKU (CPUID(0x7, 0, ECX, 3)) 150 #define X86_FEATURE_LA57 (CPUID(0x7, 0, ECX, 16)) 151 #define X86_FEATURE_RDPID (CPUID(0x7, 0, ECX, 22)) 152 #define X86_FEATURE_SPEC_CTRL (CPUID(0x7, 0, EDX, 26)) 153 #define X86_FEATURE_ARCH_CAPABILITIES (CPUID(0x7, 0, EDX, 29)) 154 #define X86_FEATURE_NX (CPUID(0x80000001, 0, EDX, 20)) 155 #define X86_FEATURE_RDPRU (CPUID(0x80000008, 0, EBX, 4)) 156 157 /* 158 * AMD CPUID features 159 */ 160 #define X86_FEATURE_SVM (CPUID(0x80000001, 0, ECX, 2)) 161 #define X86_FEATURE_RDTSCP (CPUID(0x80000001, 0, EDX, 27)) 162 #define X86_FEATURE_AMD_IBPB (CPUID(0x80000008, 0, EBX, 12)) 163 #define X86_FEATURE_NPT (CPUID(0x8000000A, 0, EDX, 0)) 164 #define X86_FEATURE_NRIPS (CPUID(0x8000000A, 0, EDX, 3)) 165 166 167 static inline bool this_cpu_has(u64 feature) 168 { 169 u32 input_eax = feature >> 32; 170 u32 input_ecx = (feature >> 16) & 0xffff; 171 u32 output_reg = (feature >> 8) & 0xff; 172 u8 bit = feature & 0xff; 173 struct cpuid c; 174 u32 *tmp; 175 176 c = cpuid_indexed(input_eax, input_ecx); 177 tmp = (u32 *)&c; 178 179 return ((*(tmp + (output_reg % 32))) & (1 << bit)); 180 } 181 182 struct far_pointer32 { 183 u32 offset; 184 u16 selector; 185 } __attribute__((packed)); 186 187 struct descriptor_table_ptr { 188 u16 limit; 189 ulong base; 190 } __attribute__((packed)); 191 192 static inline void barrier(void) 193 { 194 asm volatile ("" : : : "memory"); 195 } 196 197 static inline void clac(void) 198 { 199 asm volatile (".byte 0x0f, 0x01, 0xca" : : : "memory"); 200 } 201 202 static inline void stac(void) 203 { 204 asm volatile (".byte 0x0f, 0x01, 0xcb" : : : "memory"); 205 } 206 207 static inline u16 read_cs(void) 208 { 209 unsigned val; 210 211 asm volatile ("mov %%cs, %0" : "=mr"(val)); 212 return val; 213 } 214 215 static inline u16 read_ds(void) 216 { 217 unsigned val; 218 219 asm volatile ("mov %%ds, %0" : "=mr"(val)); 220 return val; 221 } 222 223 static inline u16 read_es(void) 224 { 225 unsigned val; 226 227 asm volatile ("mov %%es, %0" : "=mr"(val)); 228 return val; 229 } 230 231 static inline u16 read_ss(void) 232 { 233 unsigned val; 234 235 asm volatile ("mov %%ss, %0" : "=mr"(val)); 236 return val; 237 } 238 239 static inline u16 read_fs(void) 240 { 241 unsigned val; 242 243 asm volatile ("mov %%fs, %0" : "=mr"(val)); 244 return val; 245 } 246 247 static inline u16 read_gs(void) 248 { 249 unsigned val; 250 251 asm volatile ("mov %%gs, %0" : "=mr"(val)); 252 return val; 253 } 254 255 static inline unsigned long read_rflags(void) 256 { 257 unsigned long f; 258 asm volatile ("pushf; pop %0\n\t" : "=rm"(f)); 259 return f; 260 } 261 262 static inline void write_ds(unsigned val) 263 { 264 asm volatile ("mov %0, %%ds" : : "rm"(val) : "memory"); 265 } 266 267 static inline void write_es(unsigned val) 268 { 269 asm volatile ("mov %0, %%es" : : "rm"(val) : "memory"); 270 } 271 272 static inline void write_ss(unsigned val) 273 { 274 asm volatile ("mov %0, %%ss" : : "rm"(val) : "memory"); 275 } 276 277 static inline void write_fs(unsigned val) 278 { 279 asm volatile ("mov %0, %%fs" : : "rm"(val) : "memory"); 280 } 281 282 static inline void write_gs(unsigned val) 283 { 284 asm volatile ("mov %0, %%gs" : : "rm"(val) : "memory"); 285 } 286 287 static inline void write_rflags(unsigned long f) 288 { 289 asm volatile ("push %0; popf\n\t" : : "rm"(f)); 290 } 291 292 static inline void set_iopl(int iopl) 293 { 294 unsigned long flags = read_rflags() & ~X86_EFLAGS_IOPL; 295 flags |= iopl * (X86_EFLAGS_IOPL / 3); 296 write_rflags(flags); 297 } 298 299 static inline u64 rdmsr(u32 index) 300 { 301 u32 a, d; 302 asm volatile ("rdmsr" : "=a"(a), "=d"(d) : "c"(index) : "memory"); 303 return a | ((u64)d << 32); 304 } 305 306 static inline void wrmsr(u32 index, u64 val) 307 { 308 u32 a = val, d = val >> 32; 309 asm volatile ("wrmsr" : : "a"(a), "d"(d), "c"(index) : "memory"); 310 } 311 312 static inline uint64_t rdpmc(uint32_t index) 313 { 314 uint32_t a, d; 315 asm volatile ("rdpmc" : "=a"(a), "=d"(d) : "c"(index)); 316 return a | ((uint64_t)d << 32); 317 } 318 319 static inline void write_cr0(ulong val) 320 { 321 asm volatile ("mov %0, %%cr0" : : "r"(val) : "memory"); 322 } 323 324 static inline ulong read_cr0(void) 325 { 326 ulong val; 327 asm volatile ("mov %%cr0, %0" : "=r"(val) : : "memory"); 328 return val; 329 } 330 331 static inline void write_cr2(ulong val) 332 { 333 asm volatile ("mov %0, %%cr2" : : "r"(val) : "memory"); 334 } 335 336 static inline ulong read_cr2(void) 337 { 338 ulong val; 339 asm volatile ("mov %%cr2, %0" : "=r"(val) : : "memory"); 340 return val; 341 } 342 343 static inline void write_cr3(ulong val) 344 { 345 asm volatile ("mov %0, %%cr3" : : "r"(val) : "memory"); 346 } 347 348 static inline ulong read_cr3(void) 349 { 350 ulong val; 351 asm volatile ("mov %%cr3, %0" : "=r"(val) : : "memory"); 352 return val; 353 } 354 355 static inline void write_cr4(ulong val) 356 { 357 asm volatile ("mov %0, %%cr4" : : "r"(val) : "memory"); 358 } 359 360 static inline ulong read_cr4(void) 361 { 362 ulong val; 363 asm volatile ("mov %%cr4, %0" : "=r"(val) : : "memory"); 364 return val; 365 } 366 367 static inline void write_cr8(ulong val) 368 { 369 asm volatile ("mov %0, %%cr8" : : "r"(val) : "memory"); 370 } 371 372 static inline ulong read_cr8(void) 373 { 374 ulong val; 375 asm volatile ("mov %%cr8, %0" : "=r"(val) : : "memory"); 376 return val; 377 } 378 379 static inline void lgdt(const struct descriptor_table_ptr *ptr) 380 { 381 asm volatile ("lgdt %0" : : "m"(*ptr)); 382 } 383 384 static inline void sgdt(struct descriptor_table_ptr *ptr) 385 { 386 asm volatile ("sgdt %0" : "=m"(*ptr)); 387 } 388 389 static inline void lidt(const struct descriptor_table_ptr *ptr) 390 { 391 asm volatile ("lidt %0" : : "m"(*ptr)); 392 } 393 394 static inline void sidt(struct descriptor_table_ptr *ptr) 395 { 396 asm volatile ("sidt %0" : "=m"(*ptr)); 397 } 398 399 static inline void lldt(unsigned val) 400 { 401 asm volatile ("lldt %0" : : "rm"(val)); 402 } 403 404 static inline u16 sldt(void) 405 { 406 u16 val; 407 asm volatile ("sldt %0" : "=rm"(val)); 408 return val; 409 } 410 411 static inline void ltr(u16 val) 412 { 413 asm volatile ("ltr %0" : : "rm"(val)); 414 } 415 416 static inline u16 str(void) 417 { 418 u16 val; 419 asm volatile ("str %0" : "=rm"(val)); 420 return val; 421 } 422 423 static inline void write_dr6(ulong val) 424 { 425 asm volatile ("mov %0, %%dr6" : : "r"(val) : "memory"); 426 } 427 428 static inline ulong read_dr6(void) 429 { 430 ulong val; 431 asm volatile ("mov %%dr6, %0" : "=r"(val)); 432 return val; 433 } 434 435 static inline void write_dr7(ulong val) 436 { 437 asm volatile ("mov %0, %%dr7" : : "r"(val) : "memory"); 438 } 439 440 static inline ulong read_dr7(void) 441 { 442 ulong val; 443 asm volatile ("mov %%dr7, %0" : "=r"(val)); 444 return val; 445 } 446 447 static inline void pause(void) 448 { 449 asm volatile ("pause"); 450 } 451 452 static inline void cli(void) 453 { 454 asm volatile ("cli"); 455 } 456 457 static inline void sti(void) 458 { 459 asm volatile ("sti"); 460 } 461 462 static inline unsigned long long rdtsc(void) 463 { 464 long long r; 465 466 #ifdef __x86_64__ 467 unsigned a, d; 468 469 asm volatile ("rdtsc" : "=a"(a), "=d"(d)); 470 r = a | ((long long)d << 32); 471 #else 472 asm volatile ("rdtsc" : "=A"(r)); 473 #endif 474 return r; 475 } 476 477 static inline unsigned long long rdtscp(u32 *aux) 478 { 479 long long r; 480 481 #ifdef __x86_64__ 482 unsigned a, d; 483 484 asm volatile ("rdtscp" : "=a"(a), "=d"(d), "=c"(*aux)); 485 r = a | ((long long)d << 32); 486 #else 487 asm volatile ("rdtscp" : "=A"(r), "=c"(*aux)); 488 #endif 489 return r; 490 } 491 492 static inline void wrtsc(u64 tsc) 493 { 494 unsigned a = tsc, d = tsc >> 32; 495 496 asm volatile("wrmsr" : : "a"(a), "d"(d), "c"(0x10)); 497 } 498 499 static inline void irq_disable(void) 500 { 501 asm volatile("cli"); 502 } 503 504 /* Note that irq_enable() does not ensure an interrupt shadow due 505 * to the vagaries of compiler optimizations. If you need the 506 * shadow, use a single asm with "sti" and the instruction after it. 507 */ 508 static inline void irq_enable(void) 509 { 510 asm volatile("sti"); 511 } 512 513 static inline void invlpg(volatile void *va) 514 { 515 asm volatile("invlpg (%0)" ::"r" (va) : "memory"); 516 } 517 518 static inline void safe_halt(void) 519 { 520 asm volatile("sti; hlt"); 521 } 522 523 static inline u32 read_pkru(void) 524 { 525 unsigned int eax, edx; 526 unsigned int ecx = 0; 527 unsigned int pkru; 528 529 asm volatile(".byte 0x0f,0x01,0xee\n\t" 530 : "=a" (eax), "=d" (edx) 531 : "c" (ecx)); 532 pkru = eax; 533 return pkru; 534 } 535 536 static inline void write_pkru(u32 pkru) 537 { 538 unsigned int eax = pkru; 539 unsigned int ecx = 0; 540 unsigned int edx = 0; 541 542 asm volatile(".byte 0x0f,0x01,0xef\n\t" 543 : : "a" (eax), "c" (ecx), "d" (edx)); 544 } 545 546 static inline bool is_canonical(u64 addr) 547 { 548 return (s64)(addr << 16) >> 16 == addr; 549 } 550 551 static inline void clear_bit(int bit, u8 *addr) 552 { 553 __asm__ __volatile__("btr %1, %0" 554 : "+m" (*addr) : "Ir" (bit) : "cc", "memory"); 555 } 556 557 static inline void set_bit(int bit, u8 *addr) 558 { 559 __asm__ __volatile__("bts %1, %0" 560 : "+m" (*addr) : "Ir" (bit) : "cc", "memory"); 561 } 562 563 static inline void flush_tlb(void) 564 { 565 ulong cr4; 566 567 cr4 = read_cr4(); 568 write_cr4(cr4 ^ X86_CR4_PGE); 569 write_cr4(cr4); 570 } 571 572 static inline int has_spec_ctrl(void) 573 { 574 return !!(this_cpu_has(X86_FEATURE_SPEC_CTRL)); 575 } 576 577 static inline int cpu_has_efer_nx(void) 578 { 579 return !!(this_cpu_has(X86_FEATURE_NX)); 580 } 581 582 #endif 583