1 #ifndef _X86_PROCESSOR_H_ 2 #define _X86_PROCESSOR_H_ 3 4 #include "libcflat.h" 5 #include "desc.h" 6 #include "msr.h" 7 #include <bitops.h> 8 #include <stdint.h> 9 10 #define NONCANONICAL 0xaaaaaaaaaaaaaaaaull 11 12 #ifdef __x86_64__ 13 # define R "r" 14 # define W "q" 15 # define S "8" 16 #else 17 # define R "e" 18 # define W "l" 19 # define S "4" 20 #endif 21 22 #define DB_VECTOR 1 23 #define BP_VECTOR 3 24 #define UD_VECTOR 6 25 #define DF_VECTOR 8 26 #define TS_VECTOR 10 27 #define NP_VECTOR 11 28 #define SS_VECTOR 12 29 #define GP_VECTOR 13 30 #define PF_VECTOR 14 31 #define AC_VECTOR 17 32 #define CP_VECTOR 21 33 34 #define X86_CR0_PE_BIT (0) 35 #define X86_CR0_PE BIT(X86_CR0_PE_BIT) 36 #define X86_CR0_MP_BIT (1) 37 #define X86_CR0_MP BIT(X86_CR0_MP_BIT) 38 #define X86_CR0_EM_BIT (2) 39 #define X86_CR0_EM BIT(X86_CR0_EM_BIT) 40 #define X86_CR0_TS_BIT (3) 41 #define X86_CR0_TS BIT(X86_CR0_TS_BIT) 42 #define X86_CR0_ET_BIT (4) 43 #define X86_CR0_ET BIT(X86_CR0_ET_BIT) 44 #define X86_CR0_NE_BIT (5) 45 #define X86_CR0_NE BIT(X86_CR0_NE_BIT) 46 #define X86_CR0_WP_BIT (16) 47 #define X86_CR0_WP BIT(X86_CR0_WP_BIT) 48 #define X86_CR0_AM_BIT (18) 49 #define X86_CR0_AM BIT(X86_CR0_AM_BIT) 50 #define X86_CR0_NW_BIT (29) 51 #define X86_CR0_NW BIT(X86_CR0_NW_BIT) 52 #define X86_CR0_CD_BIT (30) 53 #define X86_CR0_CD BIT(X86_CR0_CD_BIT) 54 #define X86_CR0_PG_BIT (31) 55 #define X86_CR0_PG BIT(X86_CR0_PG_BIT) 56 57 #define X86_CR3_PCID_MASK GENMASK(11, 0) 58 59 #define X86_CR4_VME_BIT (0) 60 #define X86_CR4_VME BIT(X86_CR4_VME_BIT) 61 #define X86_CR4_PVI_BIT (1) 62 #define X86_CR4_PVI BIT(X86_CR4_PVI_BIT) 63 #define X86_CR4_TSD_BIT (2) 64 #define X86_CR4_TSD BIT(X86_CR4_TSD_BIT) 65 #define X86_CR4_DE_BIT (3) 66 #define X86_CR4_DE BIT(X86_CR4_DE_BIT) 67 #define X86_CR4_PSE_BIT (4) 68 #define X86_CR4_PSE BIT(X86_CR4_PSE_BIT) 69 #define X86_CR4_PAE_BIT (5) 70 #define X86_CR4_PAE BIT(X86_CR4_PAE_BIT) 71 #define X86_CR4_MCE_BIT (6) 72 #define X86_CR4_MCE BIT(X86_CR4_MCE_BIT) 73 #define X86_CR4_PGE_BIT (7) 74 #define X86_CR4_PGE BIT(X86_CR4_PGE_BIT) 75 #define X86_CR4_PCE_BIT (8) 76 #define X86_CR4_PCE BIT(X86_CR4_PCE_BIT) 77 #define X86_CR4_OSFXSR_BIT (9) 78 #define X86_CR4_OSFXSR BIT(X86_CR4_OSFXSR_BIT) 79 #define X86_CR4_OSXMMEXCPT_BIT (10) 80 #define X86_CR4_OSXMMEXCPT BIT(X86_CR4_OSXMMEXCPT_BIT) 81 #define X86_CR4_UMIP_BIT (11) 82 #define X86_CR4_UMIP BIT(X86_CR4_UMIP_BIT) 83 #define X86_CR4_LA57_BIT (12) 84 #define X86_CR4_LA57 BIT(X86_CR4_LA57_BIT) 85 #define X86_CR4_VMXE_BIT (13) 86 #define X86_CR4_VMXE BIT(X86_CR4_VMXE_BIT) 87 #define X86_CR4_SMXE_BIT (14) 88 #define X86_CR4_SMXE BIT(X86_CR4_SMXE_BIT) 89 /* UNUSED (15) */ 90 #define X86_CR4_FSGSBASE_BIT (16) 91 #define X86_CR4_FSGSBASE BIT(X86_CR4_FSGSBASE_BIT) 92 #define X86_CR4_PCIDE_BIT (17) 93 #define X86_CR4_PCIDE BIT(X86_CR4_PCIDE_BIT) 94 #define X86_CR4_OSXSAVE_BIT (18) 95 #define X86_CR4_OSXSAVE BIT(X86_CR4_OSXSAVE_BIT) 96 #define X86_CR4_KL_BIT (19) 97 #define X86_CR4_KL BIT(X86_CR4_KL_BIT) 98 #define X86_CR4_SMEP_BIT (20) 99 #define X86_CR4_SMEP BIT(X86_CR4_SMEP_BIT) 100 #define X86_CR4_SMAP_BIT (21) 101 #define X86_CR4_SMAP BIT(X86_CR4_SMAP_BIT) 102 #define X86_CR4_PKE_BIT (22) 103 #define X86_CR4_PKE BIT(X86_CR4_PKE_BIT) 104 #define X86_CR4_CET_BIT (23) 105 #define X86_CR4_CET BIT(X86_CR4_CET_BIT) 106 #define X86_CR4_PKS_BIT (24) 107 #define X86_CR4_PKS BIT(X86_CR4_PKS_BIT) 108 109 #define X86_EFLAGS_CF_BIT (0) 110 #define X86_EFLAGS_CF BIT(X86_EFLAGS_CF_BIT) 111 #define X86_EFLAGS_FIXED_BIT (1) 112 #define X86_EFLAGS_FIXED BIT(X86_EFLAGS_FIXED_BIT) 113 #define X86_EFLAGS_PF_BIT (2) 114 #define X86_EFLAGS_PF BIT(X86_EFLAGS_PF_BIT) 115 /* RESERVED 0 (3) */ 116 #define X86_EFLAGS_AF_BIT (4) 117 #define X86_EFLAGS_AF BIT(X86_EFLAGS_AF_BIT) 118 /* RESERVED 0 (5) */ 119 #define X86_EFLAGS_ZF_BIT (6) 120 #define X86_EFLAGS_ZF BIT(X86_EFLAGS_ZF_BIT) 121 #define X86_EFLAGS_SF_BIT (7) 122 #define X86_EFLAGS_SF BIT(X86_EFLAGS_SF_BIT) 123 #define X86_EFLAGS_TF_BIT (8) 124 #define X86_EFLAGS_TF BIT(X86_EFLAGS_TF_BIT) 125 #define X86_EFLAGS_IF_BIT (9) 126 #define X86_EFLAGS_IF BIT(X86_EFLAGS_IF_BIT) 127 #define X86_EFLAGS_DF_BIT (10) 128 #define X86_EFLAGS_DF BIT(X86_EFLAGS_DF_BIT) 129 #define X86_EFLAGS_OF_BIT (11) 130 #define X86_EFLAGS_OF BIT(X86_EFLAGS_OF_BIT) 131 #define X86_EFLAGS_IOPL GENMASK(13, 12) 132 #define X86_EFLAGS_NT_BIT (14) 133 #define X86_EFLAGS_NT BIT(X86_EFLAGS_NT_BIT) 134 /* RESERVED 0 (15) */ 135 #define X86_EFLAGS_RF_BIT (16) 136 #define X86_EFLAGS_RF BIT(X86_EFLAGS_RF_BIT) 137 #define X86_EFLAGS_VM_BIT (17) 138 #define X86_EFLAGS_VM BIT(X86_EFLAGS_VM_BIT) 139 #define X86_EFLAGS_AC_BIT (18) 140 #define X86_EFLAGS_AC BIT(X86_EFLAGS_AC_BIT) 141 #define X86_EFLAGS_VIF_BIT (19) 142 #define X86_EFLAGS_VIF BIT(X86_EFLAGS_VIF_BIT) 143 #define X86_EFLAGS_VIP_BIT (20) 144 #define X86_EFLAGS_VIP BIT(X86_EFLAGS_VIP_BIT) 145 #define X86_EFLAGS_ID_BIT (21) 146 #define X86_EFLAGS_ID BIT(X86_EFLAGS_ID_BIT) 147 148 #define X86_EFLAGS_ALU (X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | \ 149 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF) 150 151 152 /* 153 * CPU features 154 */ 155 156 enum cpuid_output_regs { 157 EAX, 158 EBX, 159 ECX, 160 EDX 161 }; 162 163 struct cpuid { u32 a, b, c, d; }; 164 165 static inline struct cpuid raw_cpuid(u32 function, u32 index) 166 { 167 struct cpuid r; 168 asm volatile ("cpuid" 169 : "=a"(r.a), "=b"(r.b), "=c"(r.c), "=d"(r.d) 170 : "0"(function), "2"(index)); 171 return r; 172 } 173 174 static inline struct cpuid cpuid_indexed(u32 function, u32 index) 175 { 176 u32 level = raw_cpuid(function & 0xf0000000, 0).a; 177 if (level < function) 178 return (struct cpuid) { 0, 0, 0, 0 }; 179 return raw_cpuid(function, index); 180 } 181 182 static inline struct cpuid cpuid(u32 function) 183 { 184 return cpuid_indexed(function, 0); 185 } 186 187 static inline u8 cpuid_maxphyaddr(void) 188 { 189 if (raw_cpuid(0x80000000, 0).a < 0x80000008) 190 return 36; 191 return raw_cpuid(0x80000008, 0).a & 0xff; 192 } 193 194 static inline bool is_intel(void) 195 { 196 struct cpuid c = cpuid(0); 197 u32 name[4] = {c.b, c.d, c.c }; 198 199 return strcmp((char *)name, "GenuineIntel") == 0; 200 } 201 202 #define CPUID(a, b, c, d) ((((unsigned long long) a) << 32) | (b << 16) | \ 203 (c << 8) | d) 204 205 /* 206 * Each X86_FEATURE_XXX definition is 64-bit and contains the following 207 * CPUID meta-data: 208 * 209 * [63:32] : input value for EAX 210 * [31:16] : input value for ECX 211 * [15:8] : output register 212 * [7:0] : bit position in output register 213 */ 214 215 /* 216 * Basic Leafs, a.k.a. Intel defined 217 */ 218 #define X86_FEATURE_MWAIT (CPUID(0x1, 0, ECX, 3)) 219 #define X86_FEATURE_VMX (CPUID(0x1, 0, ECX, 5)) 220 #define X86_FEATURE_PDCM (CPUID(0x1, 0, ECX, 15)) 221 #define X86_FEATURE_PCID (CPUID(0x1, 0, ECX, 17)) 222 #define X86_FEATURE_X2APIC (CPUID(0x1, 0, ECX, 21)) 223 #define X86_FEATURE_MOVBE (CPUID(0x1, 0, ECX, 22)) 224 #define X86_FEATURE_TSC_DEADLINE_TIMER (CPUID(0x1, 0, ECX, 24)) 225 #define X86_FEATURE_XSAVE (CPUID(0x1, 0, ECX, 26)) 226 #define X86_FEATURE_OSXSAVE (CPUID(0x1, 0, ECX, 27)) 227 #define X86_FEATURE_RDRAND (CPUID(0x1, 0, ECX, 30)) 228 #define X86_FEATURE_MCE (CPUID(0x1, 0, EDX, 7)) 229 #define X86_FEATURE_APIC (CPUID(0x1, 0, EDX, 9)) 230 #define X86_FEATURE_CLFLUSH (CPUID(0x1, 0, EDX, 19)) 231 #define X86_FEATURE_XMM (CPUID(0x1, 0, EDX, 25)) 232 #define X86_FEATURE_XMM2 (CPUID(0x1, 0, EDX, 26)) 233 #define X86_FEATURE_TSC_ADJUST (CPUID(0x7, 0, EBX, 1)) 234 #define X86_FEATURE_HLE (CPUID(0x7, 0, EBX, 4)) 235 #define X86_FEATURE_SMEP (CPUID(0x7, 0, EBX, 7)) 236 #define X86_FEATURE_INVPCID (CPUID(0x7, 0, EBX, 10)) 237 #define X86_FEATURE_RTM (CPUID(0x7, 0, EBX, 11)) 238 #define X86_FEATURE_SMAP (CPUID(0x7, 0, EBX, 20)) 239 #define X86_FEATURE_PCOMMIT (CPUID(0x7, 0, EBX, 22)) 240 #define X86_FEATURE_CLFLUSHOPT (CPUID(0x7, 0, EBX, 23)) 241 #define X86_FEATURE_CLWB (CPUID(0x7, 0, EBX, 24)) 242 #define X86_FEATURE_UMIP (CPUID(0x7, 0, ECX, 2)) 243 #define X86_FEATURE_PKU (CPUID(0x7, 0, ECX, 3)) 244 #define X86_FEATURE_LA57 (CPUID(0x7, 0, ECX, 16)) 245 #define X86_FEATURE_RDPID (CPUID(0x7, 0, ECX, 22)) 246 #define X86_FEATURE_SHSTK (CPUID(0x7, 0, ECX, 7)) 247 #define X86_FEATURE_IBT (CPUID(0x7, 0, EDX, 20)) 248 #define X86_FEATURE_SPEC_CTRL (CPUID(0x7, 0, EDX, 26)) 249 #define X86_FEATURE_FLUSH_L1D (CPUID(0x7, 0, EDX, 28)) 250 #define X86_FEATURE_ARCH_CAPABILITIES (CPUID(0x7, 0, EDX, 29)) 251 #define X86_FEATURE_PKS (CPUID(0x7, 0, ECX, 31)) 252 253 /* 254 * Extended Leafs, a.k.a. AMD defined 255 */ 256 #define X86_FEATURE_SVM (CPUID(0x80000001, 0, ECX, 2)) 257 #define X86_FEATURE_PERFCTR_CORE (CPUID(0x80000001, 0, ECX, 23)) 258 #define X86_FEATURE_NX (CPUID(0x80000001, 0, EDX, 20)) 259 #define X86_FEATURE_GBPAGES (CPUID(0x80000001, 0, EDX, 26)) 260 #define X86_FEATURE_RDTSCP (CPUID(0x80000001, 0, EDX, 27)) 261 #define X86_FEATURE_LM (CPUID(0x80000001, 0, EDX, 29)) 262 #define X86_FEATURE_RDPRU (CPUID(0x80000008, 0, EBX, 4)) 263 #define X86_FEATURE_AMD_IBPB (CPUID(0x80000008, 0, EBX, 12)) 264 #define X86_FEATURE_NPT (CPUID(0x8000000A, 0, EDX, 0)) 265 #define X86_FEATURE_LBRV (CPUID(0x8000000A, 0, EDX, 1)) 266 #define X86_FEATURE_NRIPS (CPUID(0x8000000A, 0, EDX, 3)) 267 #define X86_FEATURE_TSCRATEMSR (CPUID(0x8000000A, 0, EDX, 4)) 268 #define X86_FEATURE_PAUSEFILTER (CPUID(0x8000000A, 0, EDX, 10)) 269 #define X86_FEATURE_PFTHRESHOLD (CPUID(0x8000000A, 0, EDX, 12)) 270 #define X86_FEATURE_VGIF (CPUID(0x8000000A, 0, EDX, 16)) 271 #define X86_FEATURE_VNMI (CPUID(0x8000000A, 0, EDX, 25)) 272 #define X86_FEATURE_AMD_PMU_V2 (CPUID(0x80000022, 0, EAX, 0)) 273 274 static inline bool this_cpu_has(u64 feature) 275 { 276 u32 input_eax = feature >> 32; 277 u32 input_ecx = (feature >> 16) & 0xffff; 278 u32 output_reg = (feature >> 8) & 0xff; 279 u8 bit = feature & 0xff; 280 struct cpuid c; 281 u32 *tmp; 282 283 c = cpuid_indexed(input_eax, input_ecx); 284 tmp = (u32 *)&c; 285 286 return ((*(tmp + (output_reg % 32))) & (1 << bit)); 287 } 288 289 struct far_pointer32 { 290 u32 offset; 291 u16 selector; 292 } __attribute__((packed)); 293 294 struct descriptor_table_ptr { 295 u16 limit; 296 ulong base; 297 } __attribute__((packed)); 298 299 static inline void clac(void) 300 { 301 asm volatile (".byte 0x0f, 0x01, 0xca" : : : "memory"); 302 } 303 304 static inline void stac(void) 305 { 306 asm volatile (".byte 0x0f, 0x01, 0xcb" : : : "memory"); 307 } 308 309 static inline u16 read_cs(void) 310 { 311 unsigned val; 312 313 asm volatile ("mov %%cs, %0" : "=mr"(val)); 314 return val; 315 } 316 317 static inline u16 read_ds(void) 318 { 319 unsigned val; 320 321 asm volatile ("mov %%ds, %0" : "=mr"(val)); 322 return val; 323 } 324 325 static inline u16 read_es(void) 326 { 327 unsigned val; 328 329 asm volatile ("mov %%es, %0" : "=mr"(val)); 330 return val; 331 } 332 333 static inline u16 read_ss(void) 334 { 335 unsigned val; 336 337 asm volatile ("mov %%ss, %0" : "=mr"(val)); 338 return val; 339 } 340 341 static inline u16 read_fs(void) 342 { 343 unsigned val; 344 345 asm volatile ("mov %%fs, %0" : "=mr"(val)); 346 return val; 347 } 348 349 static inline u16 read_gs(void) 350 { 351 unsigned val; 352 353 asm volatile ("mov %%gs, %0" : "=mr"(val)); 354 return val; 355 } 356 357 static inline unsigned long read_rflags(void) 358 { 359 unsigned long f; 360 asm volatile ("pushf; pop %0\n\t" : "=rm"(f)); 361 return f; 362 } 363 364 static inline void write_ds(unsigned val) 365 { 366 asm volatile ("mov %0, %%ds" : : "rm"(val) : "memory"); 367 } 368 369 static inline void write_es(unsigned val) 370 { 371 asm volatile ("mov %0, %%es" : : "rm"(val) : "memory"); 372 } 373 374 static inline void write_ss(unsigned val) 375 { 376 asm volatile ("mov %0, %%ss" : : "rm"(val) : "memory"); 377 } 378 379 static inline void write_fs(unsigned val) 380 { 381 asm volatile ("mov %0, %%fs" : : "rm"(val) : "memory"); 382 } 383 384 static inline void write_gs(unsigned val) 385 { 386 asm volatile ("mov %0, %%gs" : : "rm"(val) : "memory"); 387 } 388 389 static inline void write_rflags(unsigned long f) 390 { 391 asm volatile ("push %0; popf\n\t" : : "rm"(f)); 392 } 393 394 static inline void set_iopl(int iopl) 395 { 396 unsigned long flags = read_rflags() & ~X86_EFLAGS_IOPL; 397 flags |= iopl * (X86_EFLAGS_IOPL / 3); 398 write_rflags(flags); 399 } 400 401 /* 402 * Don't use the safe variants for rdmsr() or wrmsr(). The exception fixup 403 * infrastructure uses per-CPU data and thus consumes GS.base. Various tests 404 * temporarily modify MSR_GS_BASE and will explode when trying to determine 405 * whether or not RDMSR/WRMSR faulted. 406 */ 407 static inline u64 rdmsr(u32 index) 408 { 409 u32 a, d; 410 asm volatile ("rdmsr" : "=a"(a), "=d"(d) : "c"(index) : "memory"); 411 return a | ((u64)d << 32); 412 } 413 414 static inline void wrmsr(u32 index, u64 val) 415 { 416 u32 a = val, d = val >> 32; 417 asm volatile ("wrmsr" : : "a"(a), "d"(d), "c"(index) : "memory"); 418 } 419 420 #define rdreg64_safe(insn, index, val) \ 421 ({ \ 422 uint32_t a, d; \ 423 int vector; \ 424 \ 425 vector = asm_safe_out2(insn, "=a"(a), "=d"(d), "c"(index)); \ 426 \ 427 if (vector) \ 428 *(val) = 0; \ 429 else \ 430 *(val) = (uint64_t)a | ((uint64_t)d << 32); \ 431 vector; \ 432 }) 433 434 #define wrreg64_safe(insn, index, val) \ 435 ({ \ 436 uint32_t eax = (val), edx = (val) >> 32; \ 437 \ 438 asm_safe(insn, "a" (eax), "d" (edx), "c" (index)); \ 439 }) 440 441 442 static inline int rdmsr_safe(u32 index, uint64_t *val) 443 { 444 return rdreg64_safe("rdmsr", index, val); 445 } 446 447 static inline int wrmsr_safe(u32 index, u64 val) 448 { 449 return wrreg64_safe("wrmsr", index, val); 450 } 451 452 static inline int rdpmc_safe(u32 index, uint64_t *val) 453 { 454 return rdreg64_safe("rdpmc", index, val); 455 } 456 457 static inline uint64_t rdpmc(uint32_t index) 458 { 459 uint64_t val; 460 int vector = rdpmc_safe(index, &val); 461 462 assert_msg(!vector, "Unexpected %s on RDPMC(%" PRId32 ")", 463 exception_mnemonic(vector), index); 464 return val; 465 } 466 467 static inline int xgetbv_safe(u32 index, u64 *result) 468 { 469 return rdreg64_safe(".byte 0x0f,0x01,0xd0", index, result); 470 } 471 472 static inline int xsetbv_safe(u32 index, u64 value) 473 { 474 return wrreg64_safe(".byte 0x0f,0x01,0xd1", index, value); 475 } 476 477 static inline int write_cr0_safe(ulong val) 478 { 479 return asm_safe("mov %0,%%cr0", "r" (val)); 480 } 481 482 static inline void write_cr0(ulong val) 483 { 484 int vector = write_cr0_safe(val); 485 486 assert_msg(!vector, "Unexpected fault '%d' writing CR0 = %lx", 487 vector, val); 488 } 489 490 static inline ulong read_cr0(void) 491 { 492 ulong val; 493 asm volatile ("mov %%cr0, %0" : "=r"(val) : : "memory"); 494 return val; 495 } 496 497 static inline void write_cr2(ulong val) 498 { 499 asm volatile ("mov %0, %%cr2" : : "r"(val) : "memory"); 500 } 501 502 static inline ulong read_cr2(void) 503 { 504 ulong val; 505 asm volatile ("mov %%cr2, %0" : "=r"(val) : : "memory"); 506 return val; 507 } 508 509 static inline int write_cr3_safe(ulong val) 510 { 511 return asm_safe("mov %0,%%cr3", "r" (val)); 512 } 513 514 static inline void write_cr3(ulong val) 515 { 516 int vector = write_cr3_safe(val); 517 518 assert_msg(!vector, "Unexpected fault '%d' writing CR3 = %lx", 519 vector, val); 520 } 521 522 static inline ulong read_cr3(void) 523 { 524 ulong val; 525 asm volatile ("mov %%cr3, %0" : "=r"(val) : : "memory"); 526 return val; 527 } 528 529 static inline void update_cr3(void *cr3) 530 { 531 write_cr3((ulong)cr3); 532 } 533 534 static inline int write_cr4_safe(ulong val) 535 { 536 return asm_safe("mov %0,%%cr4", "r" (val)); 537 } 538 539 static inline void write_cr4(ulong val) 540 { 541 int vector = write_cr4_safe(val); 542 543 assert_msg(!vector, "Unexpected fault '%d' writing CR4 = %lx", 544 vector, val); 545 } 546 547 static inline ulong read_cr4(void) 548 { 549 ulong val; 550 asm volatile ("mov %%cr4, %0" : "=r"(val) : : "memory"); 551 return val; 552 } 553 554 static inline void write_cr8(ulong val) 555 { 556 asm volatile ("mov %0, %%cr8" : : "r"(val) : "memory"); 557 } 558 559 static inline ulong read_cr8(void) 560 { 561 ulong val; 562 asm volatile ("mov %%cr8, %0" : "=r"(val) : : "memory"); 563 return val; 564 } 565 566 static inline void lgdt(const struct descriptor_table_ptr *ptr) 567 { 568 asm volatile ("lgdt %0" : : "m"(*ptr)); 569 } 570 571 static inline void sgdt(struct descriptor_table_ptr *ptr) 572 { 573 asm volatile ("sgdt %0" : "=m"(*ptr)); 574 } 575 576 static inline void lidt(const struct descriptor_table_ptr *ptr) 577 { 578 asm volatile ("lidt %0" : : "m"(*ptr)); 579 } 580 581 static inline void sidt(struct descriptor_table_ptr *ptr) 582 { 583 asm volatile ("sidt %0" : "=m"(*ptr)); 584 } 585 586 static inline void lldt(u16 val) 587 { 588 asm volatile ("lldt %0" : : "rm"(val)); 589 } 590 591 static inline u16 sldt(void) 592 { 593 u16 val; 594 asm volatile ("sldt %0" : "=rm"(val)); 595 return val; 596 } 597 598 static inline void ltr(u16 val) 599 { 600 asm volatile ("ltr %0" : : "rm"(val)); 601 } 602 603 static inline u16 str(void) 604 { 605 u16 val; 606 asm volatile ("str %0" : "=rm"(val)); 607 return val; 608 } 609 610 static inline void write_dr0(void *val) 611 { 612 asm volatile ("mov %0, %%dr0" : : "r"(val) : "memory"); 613 } 614 615 static inline void write_dr1(void *val) 616 { 617 asm volatile ("mov %0, %%dr1" : : "r"(val) : "memory"); 618 } 619 620 static inline void write_dr2(void *val) 621 { 622 asm volatile ("mov %0, %%dr2" : : "r"(val) : "memory"); 623 } 624 625 static inline void write_dr3(void *val) 626 { 627 asm volatile ("mov %0, %%dr3" : : "r"(val) : "memory"); 628 } 629 630 static inline void write_dr6(ulong val) 631 { 632 asm volatile ("mov %0, %%dr6" : : "r"(val) : "memory"); 633 } 634 635 static inline ulong read_dr6(void) 636 { 637 ulong val; 638 asm volatile ("mov %%dr6, %0" : "=r"(val)); 639 return val; 640 } 641 642 static inline void write_dr7(ulong val) 643 { 644 asm volatile ("mov %0, %%dr7" : : "r"(val) : "memory"); 645 } 646 647 static inline ulong read_dr7(void) 648 { 649 ulong val; 650 asm volatile ("mov %%dr7, %0" : "=r"(val)); 651 return val; 652 } 653 654 static inline void pause(void) 655 { 656 asm volatile ("pause"); 657 } 658 659 static inline void cli(void) 660 { 661 asm volatile ("cli"); 662 } 663 664 static inline void sti(void) 665 { 666 asm volatile ("sti"); 667 } 668 669 static inline unsigned long long rdrand(void) 670 { 671 long long r; 672 673 asm volatile("rdrand %0\n\t" 674 "jc 1f\n\t" 675 "mov $0, %0\n\t" 676 "1:\n\t" : "=r" (r)); 677 return r; 678 } 679 680 static inline unsigned long long rdtsc(void) 681 { 682 long long r; 683 684 #ifdef __x86_64__ 685 unsigned a, d; 686 687 asm volatile ("rdtsc" : "=a"(a), "=d"(d)); 688 r = a | ((long long)d << 32); 689 #else 690 asm volatile ("rdtsc" : "=A"(r)); 691 #endif 692 return r; 693 } 694 695 /* 696 * Per the advice in the SDM, volume 2, the sequence "mfence; lfence" 697 * executed immediately before rdtsc ensures that rdtsc will be 698 * executed only after all previous instructions have executed and all 699 * previous loads and stores are globally visible. In addition, the 700 * lfence immediately after rdtsc ensures that rdtsc will be executed 701 * prior to the execution of any subsequent instruction. 702 */ 703 static inline unsigned long long fenced_rdtsc(void) 704 { 705 unsigned long long tsc; 706 707 #ifdef __x86_64__ 708 unsigned int eax, edx; 709 710 asm volatile ("mfence; lfence; rdtsc; lfence" : "=a"(eax), "=d"(edx)); 711 tsc = eax | ((unsigned long long)edx << 32); 712 #else 713 asm volatile ("mfence; lfence; rdtsc; lfence" : "=A"(tsc)); 714 #endif 715 return tsc; 716 } 717 718 static inline unsigned long long rdtscp(u32 *aux) 719 { 720 long long r; 721 722 #ifdef __x86_64__ 723 unsigned a, d; 724 725 asm volatile ("rdtscp" : "=a"(a), "=d"(d), "=c"(*aux)); 726 r = a | ((long long)d << 32); 727 #else 728 asm volatile ("rdtscp" : "=A"(r), "=c"(*aux)); 729 #endif 730 return r; 731 } 732 733 static inline void wrtsc(u64 tsc) 734 { 735 wrmsr(MSR_IA32_TSC, tsc); 736 } 737 738 static inline void irq_disable(void) 739 { 740 asm volatile("cli"); 741 } 742 743 /* Note that irq_enable() does not ensure an interrupt shadow due 744 * to the vagaries of compiler optimizations. If you need the 745 * shadow, use a single asm with "sti" and the instruction after it. 746 */ 747 static inline void irq_enable(void) 748 { 749 asm volatile("sti"); 750 } 751 752 static inline void invlpg(volatile void *va) 753 { 754 asm volatile("invlpg (%0)" ::"r" (va) : "memory"); 755 } 756 757 758 static inline int invpcid_safe(unsigned long type, void *desc) 759 { 760 /* invpcid (%rax), %rbx */ 761 return asm_safe(".byte 0x66,0x0f,0x38,0x82,0x18", "a" (desc), "b" (type)); 762 } 763 764 static inline void safe_halt(void) 765 { 766 asm volatile("sti; hlt"); 767 } 768 769 static inline u32 read_pkru(void) 770 { 771 unsigned int eax, edx; 772 unsigned int ecx = 0; 773 unsigned int pkru; 774 775 asm volatile(".byte 0x0f,0x01,0xee\n\t" 776 : "=a" (eax), "=d" (edx) 777 : "c" (ecx)); 778 pkru = eax; 779 return pkru; 780 } 781 782 static inline void write_pkru(u32 pkru) 783 { 784 unsigned int eax = pkru; 785 unsigned int ecx = 0; 786 unsigned int edx = 0; 787 788 asm volatile(".byte 0x0f,0x01,0xef\n\t" 789 : : "a" (eax), "c" (ecx), "d" (edx)); 790 } 791 792 static inline bool is_canonical(u64 addr) 793 { 794 int va_width = (raw_cpuid(0x80000008, 0).a & 0xff00) >> 8; 795 int shift_amt = 64 - va_width; 796 797 return (s64)(addr << shift_amt) >> shift_amt == addr; 798 } 799 800 static inline void clear_bit(int bit, u8 *addr) 801 { 802 __asm__ __volatile__("btr %1, %0" 803 : "+m" (*addr) : "Ir" (bit) : "cc", "memory"); 804 } 805 806 static inline void set_bit(int bit, u8 *addr) 807 { 808 __asm__ __volatile__("bts %1, %0" 809 : "+m" (*addr) : "Ir" (bit) : "cc", "memory"); 810 } 811 812 static inline void flush_tlb(void) 813 { 814 ulong cr4; 815 816 cr4 = read_cr4(); 817 write_cr4(cr4 ^ X86_CR4_PGE); 818 write_cr4(cr4); 819 } 820 821 static inline void generate_non_canonical_gp(void) 822 { 823 *(volatile u64 *)NONCANONICAL = 0; 824 } 825 826 static inline void generate_ud(void) 827 { 828 asm volatile ("ud2"); 829 } 830 831 static inline void generate_de(void) 832 { 833 asm volatile ( 834 "xor %%eax, %%eax\n\t" 835 "xor %%ebx, %%ebx\n\t" 836 "xor %%edx, %%edx\n\t" 837 "idiv %%ebx\n\t" 838 ::: "eax", "ebx", "edx"); 839 } 840 841 static inline void generate_bp(void) 842 { 843 asm volatile ("int3"); 844 } 845 846 static inline void generate_single_step_db(void) 847 { 848 write_rflags(read_rflags() | X86_EFLAGS_TF); 849 asm volatile("nop"); 850 } 851 852 static inline uint64_t generate_usermode_ac(void) 853 { 854 /* 855 * Trigger an #AC by writing 8 bytes to a 4-byte aligned address. 856 * Disclaimer: It is assumed that the stack pointer is aligned 857 * on a 16-byte boundary as x86_64 stacks should be. 858 */ 859 asm volatile("movq $0, -0x4(%rsp)"); 860 861 return 0; 862 } 863 864 /* 865 * Switch from 64-bit to 32-bit mode and generate #OF via INTO. Note, if RIP 866 * or RSP holds a 64-bit value, this helper will NOT generate #OF. 867 */ 868 static inline void generate_of(void) 869 { 870 struct far_pointer32 fp = { 871 .offset = (uintptr_t)&&into, 872 .selector = KERNEL_CS32, 873 }; 874 uintptr_t rsp; 875 876 asm volatile ("mov %%rsp, %0" : "=r"(rsp)); 877 878 if (fp.offset != (uintptr_t)&&into) { 879 printf("Code address too high.\n"); 880 return; 881 } 882 if ((u32)rsp != rsp) { 883 printf("Stack address too high.\n"); 884 return; 885 } 886 887 asm goto ("lcall *%0" : : "m" (fp) : "rax" : into); 888 return; 889 into: 890 asm volatile (".code32;" 891 "movl $0x7fffffff, %eax;" 892 "addl %eax, %eax;" 893 "into;" 894 "lret;" 895 ".code64"); 896 __builtin_unreachable(); 897 } 898 899 static inline void fnop(void) 900 { 901 asm volatile("fnop"); 902 } 903 904 /* If CR0.TS is set in L2, #NM is generated. */ 905 static inline void generate_cr0_ts_nm(void) 906 { 907 write_cr0((read_cr0() & ~X86_CR0_EM) | X86_CR0_TS); 908 fnop(); 909 } 910 911 /* If CR0.TS is cleared and CR0.EM is set, #NM is generated. */ 912 static inline void generate_cr0_em_nm(void) 913 { 914 write_cr0((read_cr0() & ~X86_CR0_TS) | X86_CR0_EM); 915 fnop(); 916 } 917 918 #endif 919