1 #ifndef _X86_PROCESSOR_H_ 2 #define _X86_PROCESSOR_H_ 3 4 #include "libcflat.h" 5 #include "desc.h" 6 #include "msr.h" 7 #include <bitops.h> 8 #include <stdint.h> 9 10 #define NONCANONICAL 0xaaaaaaaaaaaaaaaaull 11 12 #ifdef __x86_64__ 13 # define R "r" 14 # define W "q" 15 # define S "8" 16 #else 17 # define R "e" 18 # define W "l" 19 # define S "4" 20 #endif 21 22 #define DE_VECTOR 0 23 #define DB_VECTOR 1 24 #define NMI_VECTOR 2 25 #define BP_VECTOR 3 26 #define OF_VECTOR 4 27 #define BR_VECTOR 5 28 #define UD_VECTOR 6 29 #define NM_VECTOR 7 30 #define DF_VECTOR 8 31 #define TS_VECTOR 10 32 #define NP_VECTOR 11 33 #define SS_VECTOR 12 34 #define GP_VECTOR 13 35 #define PF_VECTOR 14 36 #define MF_VECTOR 16 37 #define AC_VECTOR 17 38 #define MC_VECTOR 18 39 #define CP_VECTOR 21 40 41 #define X86_CR0_PE_BIT (0) 42 #define X86_CR0_PE BIT(X86_CR0_PE_BIT) 43 #define X86_CR0_MP_BIT (1) 44 #define X86_CR0_MP BIT(X86_CR0_MP_BIT) 45 #define X86_CR0_EM_BIT (2) 46 #define X86_CR0_EM BIT(X86_CR0_EM_BIT) 47 #define X86_CR0_TS_BIT (3) 48 #define X86_CR0_TS BIT(X86_CR0_TS_BIT) 49 #define X86_CR0_ET_BIT (4) 50 #define X86_CR0_ET BIT(X86_CR0_ET_BIT) 51 #define X86_CR0_NE_BIT (5) 52 #define X86_CR0_NE BIT(X86_CR0_NE_BIT) 53 #define X86_CR0_WP_BIT (16) 54 #define X86_CR0_WP BIT(X86_CR0_WP_BIT) 55 #define X86_CR0_AM_BIT (18) 56 #define X86_CR0_AM BIT(X86_CR0_AM_BIT) 57 #define X86_CR0_NW_BIT (29) 58 #define X86_CR0_NW BIT(X86_CR0_NW_BIT) 59 #define X86_CR0_CD_BIT (30) 60 #define X86_CR0_CD BIT(X86_CR0_CD_BIT) 61 #define X86_CR0_PG_BIT (31) 62 #define X86_CR0_PG BIT(X86_CR0_PG_BIT) 63 64 #define X86_CR3_PCID_MASK GENMASK(11, 0) 65 66 #define X86_CR4_VME_BIT (0) 67 #define X86_CR4_VME BIT(X86_CR4_VME_BIT) 68 #define X86_CR4_PVI_BIT (1) 69 #define X86_CR4_PVI BIT(X86_CR4_PVI_BIT) 70 #define X86_CR4_TSD_BIT (2) 71 #define X86_CR4_TSD BIT(X86_CR4_TSD_BIT) 72 #define X86_CR4_DE_BIT (3) 73 #define X86_CR4_DE BIT(X86_CR4_DE_BIT) 74 #define X86_CR4_PSE_BIT (4) 75 #define X86_CR4_PSE BIT(X86_CR4_PSE_BIT) 76 #define X86_CR4_PAE_BIT (5) 77 #define X86_CR4_PAE BIT(X86_CR4_PAE_BIT) 78 #define X86_CR4_MCE_BIT (6) 79 #define X86_CR4_MCE BIT(X86_CR4_MCE_BIT) 80 #define X86_CR4_PGE_BIT (7) 81 #define X86_CR4_PGE BIT(X86_CR4_PGE_BIT) 82 #define X86_CR4_PCE_BIT (8) 83 #define X86_CR4_PCE BIT(X86_CR4_PCE_BIT) 84 #define X86_CR4_OSFXSR_BIT (9) 85 #define X86_CR4_OSFXSR BIT(X86_CR4_OSFXSR_BIT) 86 #define X86_CR4_OSXMMEXCPT_BIT (10) 87 #define X86_CR4_OSXMMEXCPT BIT(X86_CR4_OSXMMEXCPT_BIT) 88 #define X86_CR4_UMIP_BIT (11) 89 #define X86_CR4_UMIP BIT(X86_CR4_UMIP_BIT) 90 #define X86_CR4_LA57_BIT (12) 91 #define X86_CR4_LA57 BIT(X86_CR4_LA57_BIT) 92 #define X86_CR4_VMXE_BIT (13) 93 #define X86_CR4_VMXE BIT(X86_CR4_VMXE_BIT) 94 #define X86_CR4_SMXE_BIT (14) 95 #define X86_CR4_SMXE BIT(X86_CR4_SMXE_BIT) 96 /* UNUSED (15) */ 97 #define X86_CR4_FSGSBASE_BIT (16) 98 #define X86_CR4_FSGSBASE BIT(X86_CR4_FSGSBASE_BIT) 99 #define X86_CR4_PCIDE_BIT (17) 100 #define X86_CR4_PCIDE BIT(X86_CR4_PCIDE_BIT) 101 #define X86_CR4_OSXSAVE_BIT (18) 102 #define X86_CR4_OSXSAVE BIT(X86_CR4_OSXSAVE_BIT) 103 #define X86_CR4_KL_BIT (19) 104 #define X86_CR4_KL BIT(X86_CR4_KL_BIT) 105 #define X86_CR4_SMEP_BIT (20) 106 #define X86_CR4_SMEP BIT(X86_CR4_SMEP_BIT) 107 #define X86_CR4_SMAP_BIT (21) 108 #define X86_CR4_SMAP BIT(X86_CR4_SMAP_BIT) 109 #define X86_CR4_PKE_BIT (22) 110 #define X86_CR4_PKE BIT(X86_CR4_PKE_BIT) 111 #define X86_CR4_CET_BIT (23) 112 #define X86_CR4_CET BIT(X86_CR4_CET_BIT) 113 #define X86_CR4_PKS_BIT (24) 114 #define X86_CR4_PKS BIT(X86_CR4_PKS_BIT) 115 116 #define X86_EFLAGS_CF_BIT (0) 117 #define X86_EFLAGS_CF BIT(X86_EFLAGS_CF_BIT) 118 #define X86_EFLAGS_FIXED_BIT (1) 119 #define X86_EFLAGS_FIXED BIT(X86_EFLAGS_FIXED_BIT) 120 #define X86_EFLAGS_PF_BIT (2) 121 #define X86_EFLAGS_PF BIT(X86_EFLAGS_PF_BIT) 122 /* RESERVED 0 (3) */ 123 #define X86_EFLAGS_AF_BIT (4) 124 #define X86_EFLAGS_AF BIT(X86_EFLAGS_AF_BIT) 125 /* RESERVED 0 (5) */ 126 #define X86_EFLAGS_ZF_BIT (6) 127 #define X86_EFLAGS_ZF BIT(X86_EFLAGS_ZF_BIT) 128 #define X86_EFLAGS_SF_BIT (7) 129 #define X86_EFLAGS_SF BIT(X86_EFLAGS_SF_BIT) 130 #define X86_EFLAGS_TF_BIT (8) 131 #define X86_EFLAGS_TF BIT(X86_EFLAGS_TF_BIT) 132 #define X86_EFLAGS_IF_BIT (9) 133 #define X86_EFLAGS_IF BIT(X86_EFLAGS_IF_BIT) 134 #define X86_EFLAGS_DF_BIT (10) 135 #define X86_EFLAGS_DF BIT(X86_EFLAGS_DF_BIT) 136 #define X86_EFLAGS_OF_BIT (11) 137 #define X86_EFLAGS_OF BIT(X86_EFLAGS_OF_BIT) 138 #define X86_EFLAGS_IOPL GENMASK(13, 12) 139 #define X86_EFLAGS_NT_BIT (14) 140 #define X86_EFLAGS_NT BIT(X86_EFLAGS_NT_BIT) 141 /* RESERVED 0 (15) */ 142 #define X86_EFLAGS_RF_BIT (16) 143 #define X86_EFLAGS_RF BIT(X86_EFLAGS_RF_BIT) 144 #define X86_EFLAGS_VM_BIT (17) 145 #define X86_EFLAGS_VM BIT(X86_EFLAGS_VM_BIT) 146 #define X86_EFLAGS_AC_BIT (18) 147 #define X86_EFLAGS_AC BIT(X86_EFLAGS_AC_BIT) 148 #define X86_EFLAGS_VIF_BIT (19) 149 #define X86_EFLAGS_VIF BIT(X86_EFLAGS_VIF_BIT) 150 #define X86_EFLAGS_VIP_BIT (20) 151 #define X86_EFLAGS_VIP BIT(X86_EFLAGS_VIP_BIT) 152 #define X86_EFLAGS_ID_BIT (21) 153 #define X86_EFLAGS_ID BIT(X86_EFLAGS_ID_BIT) 154 155 #define X86_EFLAGS_ALU (X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | \ 156 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF) 157 158 159 /* 160 * CPU features 161 */ 162 163 enum cpuid_output_regs { 164 EAX, 165 EBX, 166 ECX, 167 EDX 168 }; 169 170 struct cpuid { u32 a, b, c, d; }; 171 172 static inline struct cpuid raw_cpuid(u32 function, u32 index) 173 { 174 struct cpuid r; 175 asm volatile ("cpuid" 176 : "=a"(r.a), "=b"(r.b), "=c"(r.c), "=d"(r.d) 177 : "0"(function), "2"(index)); 178 return r; 179 } 180 181 static inline struct cpuid cpuid_indexed(u32 function, u32 index) 182 { 183 u32 level = raw_cpuid(function & 0xf0000000, 0).a; 184 if (level < function) 185 return (struct cpuid) { 0, 0, 0, 0 }; 186 return raw_cpuid(function, index); 187 } 188 189 static inline struct cpuid cpuid(u32 function) 190 { 191 return cpuid_indexed(function, 0); 192 } 193 194 static inline u8 cpuid_maxphyaddr(void) 195 { 196 if (raw_cpuid(0x80000000, 0).a < 0x80000008) 197 return 36; 198 return raw_cpuid(0x80000008, 0).a & 0xff; 199 } 200 201 static inline bool is_intel(void) 202 { 203 struct cpuid c = cpuid(0); 204 u32 name[4] = {c.b, c.d, c.c }; 205 206 return strcmp((char *)name, "GenuineIntel") == 0; 207 } 208 209 #define CPUID(a, b, c, d) ((((unsigned long long) a) << 32) | (b << 16) | \ 210 (c << 8) | d) 211 212 /* 213 * Each X86_FEATURE_XXX definition is 64-bit and contains the following 214 * CPUID meta-data: 215 * 216 * [63:32] : input value for EAX 217 * [31:16] : input value for ECX 218 * [15:8] : output register 219 * [7:0] : bit position in output register 220 */ 221 222 /* 223 * Basic Leafs, a.k.a. Intel defined 224 */ 225 #define X86_FEATURE_MWAIT (CPUID(0x1, 0, ECX, 3)) 226 #define X86_FEATURE_VMX (CPUID(0x1, 0, ECX, 5)) 227 #define X86_FEATURE_PDCM (CPUID(0x1, 0, ECX, 15)) 228 #define X86_FEATURE_PCID (CPUID(0x1, 0, ECX, 17)) 229 #define X86_FEATURE_X2APIC (CPUID(0x1, 0, ECX, 21)) 230 #define X86_FEATURE_MOVBE (CPUID(0x1, 0, ECX, 22)) 231 #define X86_FEATURE_TSC_DEADLINE_TIMER (CPUID(0x1, 0, ECX, 24)) 232 #define X86_FEATURE_XSAVE (CPUID(0x1, 0, ECX, 26)) 233 #define X86_FEATURE_OSXSAVE (CPUID(0x1, 0, ECX, 27)) 234 #define X86_FEATURE_RDRAND (CPUID(0x1, 0, ECX, 30)) 235 #define X86_FEATURE_MCE (CPUID(0x1, 0, EDX, 7)) 236 #define X86_FEATURE_APIC (CPUID(0x1, 0, EDX, 9)) 237 #define X86_FEATURE_CLFLUSH (CPUID(0x1, 0, EDX, 19)) 238 #define X86_FEATURE_XMM (CPUID(0x1, 0, EDX, 25)) 239 #define X86_FEATURE_XMM2 (CPUID(0x1, 0, EDX, 26)) 240 #define X86_FEATURE_TSC_ADJUST (CPUID(0x7, 0, EBX, 1)) 241 #define X86_FEATURE_HLE (CPUID(0x7, 0, EBX, 4)) 242 #define X86_FEATURE_SMEP (CPUID(0x7, 0, EBX, 7)) 243 #define X86_FEATURE_INVPCID (CPUID(0x7, 0, EBX, 10)) 244 #define X86_FEATURE_RTM (CPUID(0x7, 0, EBX, 11)) 245 #define X86_FEATURE_SMAP (CPUID(0x7, 0, EBX, 20)) 246 #define X86_FEATURE_PCOMMIT (CPUID(0x7, 0, EBX, 22)) 247 #define X86_FEATURE_CLFLUSHOPT (CPUID(0x7, 0, EBX, 23)) 248 #define X86_FEATURE_CLWB (CPUID(0x7, 0, EBX, 24)) 249 #define X86_FEATURE_UMIP (CPUID(0x7, 0, ECX, 2)) 250 #define X86_FEATURE_PKU (CPUID(0x7, 0, ECX, 3)) 251 #define X86_FEATURE_LA57 (CPUID(0x7, 0, ECX, 16)) 252 #define X86_FEATURE_RDPID (CPUID(0x7, 0, ECX, 22)) 253 #define X86_FEATURE_SHSTK (CPUID(0x7, 0, ECX, 7)) 254 #define X86_FEATURE_IBT (CPUID(0x7, 0, EDX, 20)) 255 #define X86_FEATURE_SPEC_CTRL (CPUID(0x7, 0, EDX, 26)) 256 #define X86_FEATURE_FLUSH_L1D (CPUID(0x7, 0, EDX, 28)) 257 #define X86_FEATURE_ARCH_CAPABILITIES (CPUID(0x7, 0, EDX, 29)) 258 #define X86_FEATURE_PKS (CPUID(0x7, 0, ECX, 31)) 259 260 /* 261 * Extended Leafs, a.k.a. AMD defined 262 */ 263 #define X86_FEATURE_SVM (CPUID(0x80000001, 0, ECX, 2)) 264 #define X86_FEATURE_PERFCTR_CORE (CPUID(0x80000001, 0, ECX, 23)) 265 #define X86_FEATURE_NX (CPUID(0x80000001, 0, EDX, 20)) 266 #define X86_FEATURE_GBPAGES (CPUID(0x80000001, 0, EDX, 26)) 267 #define X86_FEATURE_RDTSCP (CPUID(0x80000001, 0, EDX, 27)) 268 #define X86_FEATURE_LM (CPUID(0x80000001, 0, EDX, 29)) 269 #define X86_FEATURE_RDPRU (CPUID(0x80000008, 0, EBX, 4)) 270 #define X86_FEATURE_AMD_IBPB (CPUID(0x80000008, 0, EBX, 12)) 271 #define X86_FEATURE_NPT (CPUID(0x8000000A, 0, EDX, 0)) 272 #define X86_FEATURE_LBRV (CPUID(0x8000000A, 0, EDX, 1)) 273 #define X86_FEATURE_NRIPS (CPUID(0x8000000A, 0, EDX, 3)) 274 #define X86_FEATURE_TSCRATEMSR (CPUID(0x8000000A, 0, EDX, 4)) 275 #define X86_FEATURE_PAUSEFILTER (CPUID(0x8000000A, 0, EDX, 10)) 276 #define X86_FEATURE_PFTHRESHOLD (CPUID(0x8000000A, 0, EDX, 12)) 277 #define X86_FEATURE_VGIF (CPUID(0x8000000A, 0, EDX, 16)) 278 #define X86_FEATURE_VNMI (CPUID(0x8000000A, 0, EDX, 25)) 279 #define X86_FEATURE_AMD_PMU_V2 (CPUID(0x80000022, 0, EAX, 0)) 280 281 static inline bool this_cpu_has(u64 feature) 282 { 283 u32 input_eax = feature >> 32; 284 u32 input_ecx = (feature >> 16) & 0xffff; 285 u32 output_reg = (feature >> 8) & 0xff; 286 u8 bit = feature & 0xff; 287 struct cpuid c; 288 u32 *tmp; 289 290 c = cpuid_indexed(input_eax, input_ecx); 291 tmp = (u32 *)&c; 292 293 return ((*(tmp + (output_reg % 32))) & (1 << bit)); 294 } 295 296 struct far_pointer32 { 297 u32 offset; 298 u16 selector; 299 } __attribute__((packed)); 300 301 struct descriptor_table_ptr { 302 u16 limit; 303 ulong base; 304 } __attribute__((packed)); 305 306 static inline void clac(void) 307 { 308 asm volatile (".byte 0x0f, 0x01, 0xca" : : : "memory"); 309 } 310 311 static inline void stac(void) 312 { 313 asm volatile (".byte 0x0f, 0x01, 0xcb" : : : "memory"); 314 } 315 316 static inline u16 read_cs(void) 317 { 318 unsigned val; 319 320 asm volatile ("mov %%cs, %0" : "=mr"(val)); 321 return val; 322 } 323 324 static inline u16 read_ds(void) 325 { 326 unsigned val; 327 328 asm volatile ("mov %%ds, %0" : "=mr"(val)); 329 return val; 330 } 331 332 static inline u16 read_es(void) 333 { 334 unsigned val; 335 336 asm volatile ("mov %%es, %0" : "=mr"(val)); 337 return val; 338 } 339 340 static inline u16 read_ss(void) 341 { 342 unsigned val; 343 344 asm volatile ("mov %%ss, %0" : "=mr"(val)); 345 return val; 346 } 347 348 static inline u16 read_fs(void) 349 { 350 unsigned val; 351 352 asm volatile ("mov %%fs, %0" : "=mr"(val)); 353 return val; 354 } 355 356 static inline u16 read_gs(void) 357 { 358 unsigned val; 359 360 asm volatile ("mov %%gs, %0" : "=mr"(val)); 361 return val; 362 } 363 364 static inline unsigned long read_rflags(void) 365 { 366 unsigned long f; 367 asm volatile ("pushf; pop %0\n\t" : "=rm"(f)); 368 return f; 369 } 370 371 static inline void write_ds(unsigned val) 372 { 373 asm volatile ("mov %0, %%ds" : : "rm"(val) : "memory"); 374 } 375 376 static inline void write_es(unsigned val) 377 { 378 asm volatile ("mov %0, %%es" : : "rm"(val) : "memory"); 379 } 380 381 static inline void write_ss(unsigned val) 382 { 383 asm volatile ("mov %0, %%ss" : : "rm"(val) : "memory"); 384 } 385 386 static inline void write_fs(unsigned val) 387 { 388 asm volatile ("mov %0, %%fs" : : "rm"(val) : "memory"); 389 } 390 391 static inline void write_gs(unsigned val) 392 { 393 asm volatile ("mov %0, %%gs" : : "rm"(val) : "memory"); 394 } 395 396 static inline void write_rflags(unsigned long f) 397 { 398 asm volatile ("push %0; popf\n\t" : : "rm"(f)); 399 } 400 401 static inline void set_iopl(int iopl) 402 { 403 unsigned long flags = read_rflags() & ~X86_EFLAGS_IOPL; 404 flags |= iopl * (X86_EFLAGS_IOPL / 3); 405 write_rflags(flags); 406 } 407 408 /* 409 * Don't use the safe variants for rdmsr() or wrmsr(). The exception fixup 410 * infrastructure uses per-CPU data and thus consumes GS.base. Various tests 411 * temporarily modify MSR_GS_BASE and will explode when trying to determine 412 * whether or not RDMSR/WRMSR faulted. 413 */ 414 static inline u64 rdmsr(u32 index) 415 { 416 u32 a, d; 417 asm volatile ("rdmsr" : "=a"(a), "=d"(d) : "c"(index) : "memory"); 418 return a | ((u64)d << 32); 419 } 420 421 static inline void wrmsr(u32 index, u64 val) 422 { 423 u32 a = val, d = val >> 32; 424 asm volatile ("wrmsr" : : "a"(a), "d"(d), "c"(index) : "memory"); 425 } 426 427 #define rdreg64_safe(insn, index, val) \ 428 ({ \ 429 uint32_t a, d; \ 430 int vector; \ 431 \ 432 vector = asm_safe_out2(insn, "=a"(a), "=d"(d), "c"(index)); \ 433 \ 434 if (vector) \ 435 *(val) = 0; \ 436 else \ 437 *(val) = (uint64_t)a | ((uint64_t)d << 32); \ 438 vector; \ 439 }) 440 441 #define wrreg64_safe(insn, index, val) \ 442 ({ \ 443 uint32_t eax = (val), edx = (val) >> 32; \ 444 \ 445 asm_safe(insn, "a" (eax), "d" (edx), "c" (index)); \ 446 }) 447 448 449 static inline int rdmsr_safe(u32 index, uint64_t *val) 450 { 451 return rdreg64_safe("rdmsr", index, val); 452 } 453 454 static inline int wrmsr_safe(u32 index, u64 val) 455 { 456 return wrreg64_safe("wrmsr", index, val); 457 } 458 459 static inline int rdpmc_safe(u32 index, uint64_t *val) 460 { 461 return rdreg64_safe("rdpmc", index, val); 462 } 463 464 static inline uint64_t rdpmc(uint32_t index) 465 { 466 uint64_t val; 467 int vector = rdpmc_safe(index, &val); 468 469 assert_msg(!vector, "Unexpected %s on RDPMC(%" PRId32 ")", 470 exception_mnemonic(vector), index); 471 return val; 472 } 473 474 static inline int xgetbv_safe(u32 index, u64 *result) 475 { 476 return rdreg64_safe(".byte 0x0f,0x01,0xd0", index, result); 477 } 478 479 static inline int xsetbv_safe(u32 index, u64 value) 480 { 481 return wrreg64_safe(".byte 0x0f,0x01,0xd1", index, value); 482 } 483 484 static inline int write_cr0_safe(ulong val) 485 { 486 return asm_safe("mov %0,%%cr0", "r" (val)); 487 } 488 489 static inline void write_cr0(ulong val) 490 { 491 int vector = write_cr0_safe(val); 492 493 assert_msg(!vector, "Unexpected fault '%d' writing CR0 = %lx", 494 vector, val); 495 } 496 497 static inline ulong read_cr0(void) 498 { 499 ulong val; 500 asm volatile ("mov %%cr0, %0" : "=r"(val) : : "memory"); 501 return val; 502 } 503 504 static inline void write_cr2(ulong val) 505 { 506 asm volatile ("mov %0, %%cr2" : : "r"(val) : "memory"); 507 } 508 509 static inline ulong read_cr2(void) 510 { 511 ulong val; 512 asm volatile ("mov %%cr2, %0" : "=r"(val) : : "memory"); 513 return val; 514 } 515 516 static inline int write_cr3_safe(ulong val) 517 { 518 return asm_safe("mov %0,%%cr3", "r" (val)); 519 } 520 521 static inline void write_cr3(ulong val) 522 { 523 int vector = write_cr3_safe(val); 524 525 assert_msg(!vector, "Unexpected fault '%d' writing CR3 = %lx", 526 vector, val); 527 } 528 529 static inline ulong read_cr3(void) 530 { 531 ulong val; 532 asm volatile ("mov %%cr3, %0" : "=r"(val) : : "memory"); 533 return val; 534 } 535 536 static inline void update_cr3(void *cr3) 537 { 538 write_cr3((ulong)cr3); 539 } 540 541 static inline int write_cr4_safe(ulong val) 542 { 543 return asm_safe("mov %0,%%cr4", "r" (val)); 544 } 545 546 static inline void write_cr4(ulong val) 547 { 548 int vector = write_cr4_safe(val); 549 550 assert_msg(!vector, "Unexpected fault '%d' writing CR4 = %lx", 551 vector, val); 552 } 553 554 static inline ulong read_cr4(void) 555 { 556 ulong val; 557 asm volatile ("mov %%cr4, %0" : "=r"(val) : : "memory"); 558 return val; 559 } 560 561 static inline void write_cr8(ulong val) 562 { 563 asm volatile ("mov %0, %%cr8" : : "r"(val) : "memory"); 564 } 565 566 static inline ulong read_cr8(void) 567 { 568 ulong val; 569 asm volatile ("mov %%cr8, %0" : "=r"(val) : : "memory"); 570 return val; 571 } 572 573 static inline void lgdt(const struct descriptor_table_ptr *ptr) 574 { 575 asm volatile ("lgdt %0" : : "m"(*ptr)); 576 } 577 578 static inline void sgdt(struct descriptor_table_ptr *ptr) 579 { 580 asm volatile ("sgdt %0" : "=m"(*ptr)); 581 } 582 583 static inline void lidt(const struct descriptor_table_ptr *ptr) 584 { 585 asm volatile ("lidt %0" : : "m"(*ptr)); 586 } 587 588 static inline void sidt(struct descriptor_table_ptr *ptr) 589 { 590 asm volatile ("sidt %0" : "=m"(*ptr)); 591 } 592 593 static inline void lldt(u16 val) 594 { 595 asm volatile ("lldt %0" : : "rm"(val)); 596 } 597 598 static inline u16 sldt(void) 599 { 600 u16 val; 601 asm volatile ("sldt %0" : "=rm"(val)); 602 return val; 603 } 604 605 static inline void ltr(u16 val) 606 { 607 asm volatile ("ltr %0" : : "rm"(val)); 608 } 609 610 static inline u16 str(void) 611 { 612 u16 val; 613 asm volatile ("str %0" : "=rm"(val)); 614 return val; 615 } 616 617 static inline void write_dr0(void *val) 618 { 619 asm volatile ("mov %0, %%dr0" : : "r"(val) : "memory"); 620 } 621 622 static inline void write_dr1(void *val) 623 { 624 asm volatile ("mov %0, %%dr1" : : "r"(val) : "memory"); 625 } 626 627 static inline void write_dr2(void *val) 628 { 629 asm volatile ("mov %0, %%dr2" : : "r"(val) : "memory"); 630 } 631 632 static inline void write_dr3(void *val) 633 { 634 asm volatile ("mov %0, %%dr3" : : "r"(val) : "memory"); 635 } 636 637 static inline void write_dr6(ulong val) 638 { 639 asm volatile ("mov %0, %%dr6" : : "r"(val) : "memory"); 640 } 641 642 static inline ulong read_dr6(void) 643 { 644 ulong val; 645 asm volatile ("mov %%dr6, %0" : "=r"(val)); 646 return val; 647 } 648 649 static inline void write_dr7(ulong val) 650 { 651 asm volatile ("mov %0, %%dr7" : : "r"(val) : "memory"); 652 } 653 654 static inline ulong read_dr7(void) 655 { 656 ulong val; 657 asm volatile ("mov %%dr7, %0" : "=r"(val)); 658 return val; 659 } 660 661 static inline void pause(void) 662 { 663 asm volatile ("pause"); 664 } 665 666 static inline void cli(void) 667 { 668 asm volatile ("cli"); 669 } 670 671 /* 672 * See also safe_halt(). 673 */ 674 static inline void sti(void) 675 { 676 asm volatile ("sti"); 677 } 678 679 /* 680 * Enable interrupts and ensure that interrupts are evaluated upon return from 681 * this function, i.e. execute a nop to consume the STi interrupt shadow. 682 */ 683 static inline void sti_nop(void) 684 { 685 asm volatile ("sti; nop"); 686 } 687 688 /* 689 * Enable interrupts for one instruction (nop), to allow the CPU to process all 690 * interrupts that are already pending. 691 */ 692 static inline void sti_nop_cli(void) 693 { 694 asm volatile ("sti; nop; cli"); 695 } 696 697 static inline unsigned long long rdrand(void) 698 { 699 long long r; 700 701 asm volatile("rdrand %0\n\t" 702 "jc 1f\n\t" 703 "mov $0, %0\n\t" 704 "1:\n\t" : "=r" (r)); 705 return r; 706 } 707 708 static inline unsigned long long rdtsc(void) 709 { 710 long long r; 711 712 #ifdef __x86_64__ 713 unsigned a, d; 714 715 asm volatile ("rdtsc" : "=a"(a), "=d"(d)); 716 r = a | ((long long)d << 32); 717 #else 718 asm volatile ("rdtsc" : "=A"(r)); 719 #endif 720 return r; 721 } 722 723 /* 724 * Per the advice in the SDM, volume 2, the sequence "mfence; lfence" 725 * executed immediately before rdtsc ensures that rdtsc will be 726 * executed only after all previous instructions have executed and all 727 * previous loads and stores are globally visible. In addition, the 728 * lfence immediately after rdtsc ensures that rdtsc will be executed 729 * prior to the execution of any subsequent instruction. 730 */ 731 static inline unsigned long long fenced_rdtsc(void) 732 { 733 unsigned long long tsc; 734 735 #ifdef __x86_64__ 736 unsigned int eax, edx; 737 738 asm volatile ("mfence; lfence; rdtsc; lfence" : "=a"(eax), "=d"(edx)); 739 tsc = eax | ((unsigned long long)edx << 32); 740 #else 741 asm volatile ("mfence; lfence; rdtsc; lfence" : "=A"(tsc)); 742 #endif 743 return tsc; 744 } 745 746 static inline unsigned long long rdtscp(u32 *aux) 747 { 748 long long r; 749 750 #ifdef __x86_64__ 751 unsigned a, d; 752 753 asm volatile ("rdtscp" : "=a"(a), "=d"(d), "=c"(*aux)); 754 r = a | ((long long)d << 32); 755 #else 756 asm volatile ("rdtscp" : "=A"(r), "=c"(*aux)); 757 #endif 758 return r; 759 } 760 761 static inline void wrtsc(u64 tsc) 762 { 763 wrmsr(MSR_IA32_TSC, tsc); 764 } 765 766 767 static inline void invlpg(volatile void *va) 768 { 769 asm volatile("invlpg (%0)" ::"r" (va) : "memory"); 770 } 771 772 773 static inline int invpcid_safe(unsigned long type, void *desc) 774 { 775 /* invpcid (%rax), %rbx */ 776 return asm_safe(".byte 0x66,0x0f,0x38,0x82,0x18", "a" (desc), "b" (type)); 777 } 778 779 /* 780 * Execute HLT in an STI interrupt shadow to ensure that a pending IRQ that's 781 * intended to be a wake event arrives *after* HLT is executed. Modern CPUs, 782 * except for a few oddballs that KVM is unlikely to run on, block IRQs for one 783 * instruction after STI, *if* RFLAGS.IF=0 before STI. Note, Intel CPUs may 784 * block other events beyond regular IRQs, e.g. may block NMIs and SMIs too. 785 */ 786 static inline void safe_halt(void) 787 { 788 asm volatile("sti; hlt"); 789 } 790 791 static inline u32 read_pkru(void) 792 { 793 unsigned int eax, edx; 794 unsigned int ecx = 0; 795 unsigned int pkru; 796 797 asm volatile(".byte 0x0f,0x01,0xee\n\t" 798 : "=a" (eax), "=d" (edx) 799 : "c" (ecx)); 800 pkru = eax; 801 return pkru; 802 } 803 804 static inline void write_pkru(u32 pkru) 805 { 806 unsigned int eax = pkru; 807 unsigned int ecx = 0; 808 unsigned int edx = 0; 809 810 asm volatile(".byte 0x0f,0x01,0xef\n\t" 811 : : "a" (eax), "c" (ecx), "d" (edx)); 812 } 813 814 static inline bool is_canonical(u64 addr) 815 { 816 int va_width = (raw_cpuid(0x80000008, 0).a & 0xff00) >> 8; 817 int shift_amt = 64 - va_width; 818 819 return (s64)(addr << shift_amt) >> shift_amt == addr; 820 } 821 822 static inline void clear_bit(int bit, u8 *addr) 823 { 824 __asm__ __volatile__("btr %1, %0" 825 : "+m" (*addr) : "Ir" (bit) : "cc", "memory"); 826 } 827 828 static inline void set_bit(int bit, u8 *addr) 829 { 830 __asm__ __volatile__("bts %1, %0" 831 : "+m" (*addr) : "Ir" (bit) : "cc", "memory"); 832 } 833 834 static inline void flush_tlb(void) 835 { 836 ulong cr4; 837 838 cr4 = read_cr4(); 839 write_cr4(cr4 ^ X86_CR4_PGE); 840 write_cr4(cr4); 841 } 842 843 static inline void generate_non_canonical_gp(void) 844 { 845 *(volatile u64 *)NONCANONICAL = 0; 846 } 847 848 static inline void generate_ud(void) 849 { 850 asm volatile ("ud2"); 851 } 852 853 static inline void generate_de(void) 854 { 855 asm volatile ( 856 "xor %%eax, %%eax\n\t" 857 "xor %%ebx, %%ebx\n\t" 858 "xor %%edx, %%edx\n\t" 859 "idiv %%ebx\n\t" 860 ::: "eax", "ebx", "edx"); 861 } 862 863 static inline void generate_bp(void) 864 { 865 asm volatile ("int3"); 866 } 867 868 static inline void generate_single_step_db(void) 869 { 870 write_rflags(read_rflags() | X86_EFLAGS_TF); 871 asm volatile("nop"); 872 } 873 874 static inline uint64_t generate_usermode_ac(void) 875 { 876 /* 877 * Trigger an #AC by writing 8 bytes to a 4-byte aligned address. 878 * Disclaimer: It is assumed that the stack pointer is aligned 879 * on a 16-byte boundary as x86_64 stacks should be. 880 */ 881 asm volatile("movq $0, -0x4(%rsp)"); 882 883 return 0; 884 } 885 886 /* 887 * Switch from 64-bit to 32-bit mode and generate #OF via INTO. Note, if RIP 888 * or RSP holds a 64-bit value, this helper will NOT generate #OF. 889 */ 890 static inline void generate_of(void) 891 { 892 struct far_pointer32 fp = { 893 .offset = (uintptr_t)&&into, 894 .selector = KERNEL_CS32, 895 }; 896 uintptr_t rsp; 897 898 asm volatile ("mov %%rsp, %0" : "=r"(rsp)); 899 900 if (fp.offset != (uintptr_t)&&into) { 901 printf("Code address too high.\n"); 902 return; 903 } 904 if ((u32)rsp != rsp) { 905 printf("Stack address too high.\n"); 906 return; 907 } 908 909 asm goto ("lcall *%0" : : "m" (fp) : "rax" : into); 910 return; 911 into: 912 asm volatile (".code32;" 913 "movl $0x7fffffff, %eax;" 914 "addl %eax, %eax;" 915 "into;" 916 "lret;" 917 ".code64"); 918 __builtin_unreachable(); 919 } 920 921 static inline void fnop(void) 922 { 923 asm volatile("fnop"); 924 } 925 926 /* If CR0.TS is set in L2, #NM is generated. */ 927 static inline void generate_cr0_ts_nm(void) 928 { 929 write_cr0((read_cr0() & ~X86_CR0_EM) | X86_CR0_TS); 930 fnop(); 931 } 932 933 /* If CR0.TS is cleared and CR0.EM is set, #NM is generated. */ 934 static inline void generate_cr0_em_nm(void) 935 { 936 write_cr0((read_cr0() & ~X86_CR0_TS) | X86_CR0_EM); 937 fnop(); 938 } 939 940 #endif 941