1 #ifndef _X86_PROCESSOR_H_ 2 #define _X86_PROCESSOR_H_ 3 4 #include "libcflat.h" 5 #include "desc.h" 6 #include "msr.h" 7 #include <bitops.h> 8 #include <stdint.h> 9 10 #define NONCANONICAL 0xaaaaaaaaaaaaaaaaull 11 12 #ifdef __x86_64__ 13 # define R "r" 14 # define W "q" 15 # define S "8" 16 #else 17 # define R "e" 18 # define W "l" 19 # define S "4" 20 #endif 21 22 #define DE_VECTOR 0 23 #define DB_VECTOR 1 24 #define NMI_VECTOR 2 25 #define BP_VECTOR 3 26 #define OF_VECTOR 4 27 #define BR_VECTOR 5 28 #define UD_VECTOR 6 29 #define NM_VECTOR 7 30 #define DF_VECTOR 8 31 #define TS_VECTOR 10 32 #define NP_VECTOR 11 33 #define SS_VECTOR 12 34 #define GP_VECTOR 13 35 #define PF_VECTOR 14 36 #define MF_VECTOR 16 37 #define AC_VECTOR 17 38 #define MC_VECTOR 18 39 #define XM_VECTOR 19 40 #define XF_VECTOR XM_VECTOR /* AMD */ 41 #define VE_VECTOR 20 /* Intel only */ 42 #define CP_VECTOR 21 43 #define HV_VECTOR 28 /* AMD only */ 44 #define VC_VECTOR 29 /* AMD only */ 45 #define SX_VECTOR 30 /* AMD only */ 46 47 #define X86_CR0_PE_BIT (0) 48 #define X86_CR0_PE BIT(X86_CR0_PE_BIT) 49 #define X86_CR0_MP_BIT (1) 50 #define X86_CR0_MP BIT(X86_CR0_MP_BIT) 51 #define X86_CR0_EM_BIT (2) 52 #define X86_CR0_EM BIT(X86_CR0_EM_BIT) 53 #define X86_CR0_TS_BIT (3) 54 #define X86_CR0_TS BIT(X86_CR0_TS_BIT) 55 #define X86_CR0_ET_BIT (4) 56 #define X86_CR0_ET BIT(X86_CR0_ET_BIT) 57 #define X86_CR0_NE_BIT (5) 58 #define X86_CR0_NE BIT(X86_CR0_NE_BIT) 59 #define X86_CR0_WP_BIT (16) 60 #define X86_CR0_WP BIT(X86_CR0_WP_BIT) 61 #define X86_CR0_AM_BIT (18) 62 #define X86_CR0_AM BIT(X86_CR0_AM_BIT) 63 #define X86_CR0_NW_BIT (29) 64 #define X86_CR0_NW BIT(X86_CR0_NW_BIT) 65 #define X86_CR0_CD_BIT (30) 66 #define X86_CR0_CD BIT(X86_CR0_CD_BIT) 67 #define X86_CR0_PG_BIT (31) 68 #define X86_CR0_PG BIT(X86_CR0_PG_BIT) 69 70 #define X86_CR3_PCID_MASK GENMASK(11, 0) 71 72 #define X86_CR4_VME_BIT (0) 73 #define X86_CR4_VME BIT(X86_CR4_VME_BIT) 74 #define X86_CR4_PVI_BIT (1) 75 #define X86_CR4_PVI BIT(X86_CR4_PVI_BIT) 76 #define X86_CR4_TSD_BIT (2) 77 #define X86_CR4_TSD BIT(X86_CR4_TSD_BIT) 78 #define X86_CR4_DE_BIT (3) 79 #define X86_CR4_DE BIT(X86_CR4_DE_BIT) 80 #define X86_CR4_PSE_BIT (4) 81 #define X86_CR4_PSE BIT(X86_CR4_PSE_BIT) 82 #define X86_CR4_PAE_BIT (5) 83 #define X86_CR4_PAE BIT(X86_CR4_PAE_BIT) 84 #define X86_CR4_MCE_BIT (6) 85 #define X86_CR4_MCE BIT(X86_CR4_MCE_BIT) 86 #define X86_CR4_PGE_BIT (7) 87 #define X86_CR4_PGE BIT(X86_CR4_PGE_BIT) 88 #define X86_CR4_PCE_BIT (8) 89 #define X86_CR4_PCE BIT(X86_CR4_PCE_BIT) 90 #define X86_CR4_OSFXSR_BIT (9) 91 #define X86_CR4_OSFXSR BIT(X86_CR4_OSFXSR_BIT) 92 #define X86_CR4_OSXMMEXCPT_BIT (10) 93 #define X86_CR4_OSXMMEXCPT BIT(X86_CR4_OSXMMEXCPT_BIT) 94 #define X86_CR4_UMIP_BIT (11) 95 #define X86_CR4_UMIP BIT(X86_CR4_UMIP_BIT) 96 #define X86_CR4_LA57_BIT (12) 97 #define X86_CR4_LA57 BIT(X86_CR4_LA57_BIT) 98 #define X86_CR4_VMXE_BIT (13) 99 #define X86_CR4_VMXE BIT(X86_CR4_VMXE_BIT) 100 #define X86_CR4_SMXE_BIT (14) 101 #define X86_CR4_SMXE BIT(X86_CR4_SMXE_BIT) 102 /* UNUSED (15) */ 103 #define X86_CR4_FSGSBASE_BIT (16) 104 #define X86_CR4_FSGSBASE BIT(X86_CR4_FSGSBASE_BIT) 105 #define X86_CR4_PCIDE_BIT (17) 106 #define X86_CR4_PCIDE BIT(X86_CR4_PCIDE_BIT) 107 #define X86_CR4_OSXSAVE_BIT (18) 108 #define X86_CR4_OSXSAVE BIT(X86_CR4_OSXSAVE_BIT) 109 #define X86_CR4_KL_BIT (19) 110 #define X86_CR4_KL BIT(X86_CR4_KL_BIT) 111 #define X86_CR4_SMEP_BIT (20) 112 #define X86_CR4_SMEP BIT(X86_CR4_SMEP_BIT) 113 #define X86_CR4_SMAP_BIT (21) 114 #define X86_CR4_SMAP BIT(X86_CR4_SMAP_BIT) 115 #define X86_CR4_PKE_BIT (22) 116 #define X86_CR4_PKE BIT(X86_CR4_PKE_BIT) 117 #define X86_CR4_CET_BIT (23) 118 #define X86_CR4_CET BIT(X86_CR4_CET_BIT) 119 #define X86_CR4_PKS_BIT (24) 120 #define X86_CR4_PKS BIT(X86_CR4_PKS_BIT) 121 122 #define X86_EFLAGS_CF_BIT (0) 123 #define X86_EFLAGS_CF BIT(X86_EFLAGS_CF_BIT) 124 #define X86_EFLAGS_FIXED_BIT (1) 125 #define X86_EFLAGS_FIXED BIT(X86_EFLAGS_FIXED_BIT) 126 #define X86_EFLAGS_PF_BIT (2) 127 #define X86_EFLAGS_PF BIT(X86_EFLAGS_PF_BIT) 128 /* RESERVED 0 (3) */ 129 #define X86_EFLAGS_AF_BIT (4) 130 #define X86_EFLAGS_AF BIT(X86_EFLAGS_AF_BIT) 131 /* RESERVED 0 (5) */ 132 #define X86_EFLAGS_ZF_BIT (6) 133 #define X86_EFLAGS_ZF BIT(X86_EFLAGS_ZF_BIT) 134 #define X86_EFLAGS_SF_BIT (7) 135 #define X86_EFLAGS_SF BIT(X86_EFLAGS_SF_BIT) 136 #define X86_EFLAGS_TF_BIT (8) 137 #define X86_EFLAGS_TF BIT(X86_EFLAGS_TF_BIT) 138 #define X86_EFLAGS_IF_BIT (9) 139 #define X86_EFLAGS_IF BIT(X86_EFLAGS_IF_BIT) 140 #define X86_EFLAGS_DF_BIT (10) 141 #define X86_EFLAGS_DF BIT(X86_EFLAGS_DF_BIT) 142 #define X86_EFLAGS_OF_BIT (11) 143 #define X86_EFLAGS_OF BIT(X86_EFLAGS_OF_BIT) 144 #define X86_EFLAGS_IOPL GENMASK(13, 12) 145 #define X86_EFLAGS_NT_BIT (14) 146 #define X86_EFLAGS_NT BIT(X86_EFLAGS_NT_BIT) 147 /* RESERVED 0 (15) */ 148 #define X86_EFLAGS_RF_BIT (16) 149 #define X86_EFLAGS_RF BIT(X86_EFLAGS_RF_BIT) 150 #define X86_EFLAGS_VM_BIT (17) 151 #define X86_EFLAGS_VM BIT(X86_EFLAGS_VM_BIT) 152 #define X86_EFLAGS_AC_BIT (18) 153 #define X86_EFLAGS_AC BIT(X86_EFLAGS_AC_BIT) 154 #define X86_EFLAGS_VIF_BIT (19) 155 #define X86_EFLAGS_VIF BIT(X86_EFLAGS_VIF_BIT) 156 #define X86_EFLAGS_VIP_BIT (20) 157 #define X86_EFLAGS_VIP BIT(X86_EFLAGS_VIP_BIT) 158 #define X86_EFLAGS_ID_BIT (21) 159 #define X86_EFLAGS_ID BIT(X86_EFLAGS_ID_BIT) 160 161 #define X86_EFLAGS_ALU (X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | \ 162 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF) 163 164 165 /* 166 * CPU features 167 */ 168 169 enum cpuid_output_regs { 170 EAX, 171 EBX, 172 ECX, 173 EDX 174 }; 175 176 struct cpuid { u32 a, b, c, d; }; 177 178 static inline struct cpuid raw_cpuid(u32 function, u32 index) 179 { 180 struct cpuid r; 181 asm volatile ("cpuid" 182 : "=a"(r.a), "=b"(r.b), "=c"(r.c), "=d"(r.d) 183 : "0"(function), "2"(index)); 184 return r; 185 } 186 187 static inline struct cpuid cpuid_indexed(u32 function, u32 index) 188 { 189 u32 level = raw_cpuid(function & 0xf0000000, 0).a; 190 if (level < function) 191 return (struct cpuid) { 0, 0, 0, 0 }; 192 return raw_cpuid(function, index); 193 } 194 195 static inline struct cpuid cpuid(u32 function) 196 { 197 return cpuid_indexed(function, 0); 198 } 199 200 static inline u8 cpuid_maxphyaddr(void) 201 { 202 if (raw_cpuid(0x80000000, 0).a < 0x80000008) 203 return 36; 204 return raw_cpuid(0x80000008, 0).a & 0xff; 205 } 206 207 static inline bool is_intel(void) 208 { 209 struct cpuid c = cpuid(0); 210 u32 name[4] = {c.b, c.d, c.c }; 211 212 return strcmp((char *)name, "GenuineIntel") == 0; 213 } 214 215 #define CPUID(a, b, c, d) ((((unsigned long long) a) << 32) | (b << 16) | \ 216 (c << 8) | d) 217 218 /* 219 * Each X86_FEATURE_XXX definition is 64-bit and contains the following 220 * CPUID meta-data: 221 * 222 * [63:32] : input value for EAX 223 * [31:16] : input value for ECX 224 * [15:8] : output register 225 * [7:0] : bit position in output register 226 */ 227 228 /* 229 * Basic Leafs, a.k.a. Intel defined 230 */ 231 #define X86_FEATURE_MWAIT (CPUID(0x1, 0, ECX, 3)) 232 #define X86_FEATURE_VMX (CPUID(0x1, 0, ECX, 5)) 233 #define X86_FEATURE_PDCM (CPUID(0x1, 0, ECX, 15)) 234 #define X86_FEATURE_PCID (CPUID(0x1, 0, ECX, 17)) 235 #define X86_FEATURE_X2APIC (CPUID(0x1, 0, ECX, 21)) 236 #define X86_FEATURE_MOVBE (CPUID(0x1, 0, ECX, 22)) 237 #define X86_FEATURE_TSC_DEADLINE_TIMER (CPUID(0x1, 0, ECX, 24)) 238 #define X86_FEATURE_XSAVE (CPUID(0x1, 0, ECX, 26)) 239 #define X86_FEATURE_OSXSAVE (CPUID(0x1, 0, ECX, 27)) 240 #define X86_FEATURE_RDRAND (CPUID(0x1, 0, ECX, 30)) 241 #define X86_FEATURE_MCE (CPUID(0x1, 0, EDX, 7)) 242 #define X86_FEATURE_APIC (CPUID(0x1, 0, EDX, 9)) 243 #define X86_FEATURE_CLFLUSH (CPUID(0x1, 0, EDX, 19)) 244 #define X86_FEATURE_XMM (CPUID(0x1, 0, EDX, 25)) 245 #define X86_FEATURE_XMM2 (CPUID(0x1, 0, EDX, 26)) 246 #define X86_FEATURE_TSC_ADJUST (CPUID(0x7, 0, EBX, 1)) 247 #define X86_FEATURE_HLE (CPUID(0x7, 0, EBX, 4)) 248 #define X86_FEATURE_SMEP (CPUID(0x7, 0, EBX, 7)) 249 #define X86_FEATURE_INVPCID (CPUID(0x7, 0, EBX, 10)) 250 #define X86_FEATURE_RTM (CPUID(0x7, 0, EBX, 11)) 251 #define X86_FEATURE_SMAP (CPUID(0x7, 0, EBX, 20)) 252 #define X86_FEATURE_PCOMMIT (CPUID(0x7, 0, EBX, 22)) 253 #define X86_FEATURE_CLFLUSHOPT (CPUID(0x7, 0, EBX, 23)) 254 #define X86_FEATURE_CLWB (CPUID(0x7, 0, EBX, 24)) 255 #define X86_FEATURE_UMIP (CPUID(0x7, 0, ECX, 2)) 256 #define X86_FEATURE_PKU (CPUID(0x7, 0, ECX, 3)) 257 #define X86_FEATURE_LA57 (CPUID(0x7, 0, ECX, 16)) 258 #define X86_FEATURE_RDPID (CPUID(0x7, 0, ECX, 22)) 259 #define X86_FEATURE_SHSTK (CPUID(0x7, 0, ECX, 7)) 260 #define X86_FEATURE_IBT (CPUID(0x7, 0, EDX, 20)) 261 #define X86_FEATURE_SPEC_CTRL (CPUID(0x7, 0, EDX, 26)) 262 #define X86_FEATURE_FLUSH_L1D (CPUID(0x7, 0, EDX, 28)) 263 #define X86_FEATURE_ARCH_CAPABILITIES (CPUID(0x7, 0, EDX, 29)) 264 #define X86_FEATURE_PKS (CPUID(0x7, 0, ECX, 31)) 265 266 /* 267 * KVM defined leafs 268 */ 269 #define KVM_FEATURE_ASYNC_PF (CPUID(0x40000001, 0, EAX, 4)) 270 #define KVM_FEATURE_ASYNC_PF_INT (CPUID(0x40000001, 0, EAX, 14)) 271 272 /* 273 * Extended Leafs, a.k.a. AMD defined 274 */ 275 #define X86_FEATURE_SVM (CPUID(0x80000001, 0, ECX, 2)) 276 #define X86_FEATURE_PERFCTR_CORE (CPUID(0x80000001, 0, ECX, 23)) 277 #define X86_FEATURE_NX (CPUID(0x80000001, 0, EDX, 20)) 278 #define X86_FEATURE_GBPAGES (CPUID(0x80000001, 0, EDX, 26)) 279 #define X86_FEATURE_RDTSCP (CPUID(0x80000001, 0, EDX, 27)) 280 #define X86_FEATURE_LM (CPUID(0x80000001, 0, EDX, 29)) 281 #define X86_FEATURE_RDPRU (CPUID(0x80000008, 0, EBX, 4)) 282 #define X86_FEATURE_AMD_IBPB (CPUID(0x80000008, 0, EBX, 12)) 283 #define X86_FEATURE_NPT (CPUID(0x8000000A, 0, EDX, 0)) 284 #define X86_FEATURE_LBRV (CPUID(0x8000000A, 0, EDX, 1)) 285 #define X86_FEATURE_NRIPS (CPUID(0x8000000A, 0, EDX, 3)) 286 #define X86_FEATURE_TSCRATEMSR (CPUID(0x8000000A, 0, EDX, 4)) 287 #define X86_FEATURE_PAUSEFILTER (CPUID(0x8000000A, 0, EDX, 10)) 288 #define X86_FEATURE_PFTHRESHOLD (CPUID(0x8000000A, 0, EDX, 12)) 289 #define X86_FEATURE_VGIF (CPUID(0x8000000A, 0, EDX, 16)) 290 #define X86_FEATURE_VNMI (CPUID(0x8000000A, 0, EDX, 25)) 291 #define X86_FEATURE_AMD_PMU_V2 (CPUID(0x80000022, 0, EAX, 0)) 292 293 static inline bool this_cpu_has(u64 feature) 294 { 295 u32 input_eax = feature >> 32; 296 u32 input_ecx = (feature >> 16) & 0xffff; 297 u32 output_reg = (feature >> 8) & 0xff; 298 u8 bit = feature & 0xff; 299 struct cpuid c; 300 u32 *tmp; 301 302 c = cpuid_indexed(input_eax, input_ecx); 303 tmp = (u32 *)&c; 304 305 return ((*(tmp + (output_reg % 32))) & (1 << bit)); 306 } 307 308 struct far_pointer32 { 309 u32 offset; 310 u16 selector; 311 } __attribute__((packed)); 312 313 struct descriptor_table_ptr { 314 u16 limit; 315 ulong base; 316 } __attribute__((packed)); 317 318 static inline void clac(void) 319 { 320 asm volatile (".byte 0x0f, 0x01, 0xca" : : : "memory"); 321 } 322 323 static inline void stac(void) 324 { 325 asm volatile (".byte 0x0f, 0x01, 0xcb" : : : "memory"); 326 } 327 328 static inline u16 read_cs(void) 329 { 330 unsigned val; 331 332 asm volatile ("mov %%cs, %0" : "=mr"(val)); 333 return val; 334 } 335 336 static inline u16 read_ds(void) 337 { 338 unsigned val; 339 340 asm volatile ("mov %%ds, %0" : "=mr"(val)); 341 return val; 342 } 343 344 static inline u16 read_es(void) 345 { 346 unsigned val; 347 348 asm volatile ("mov %%es, %0" : "=mr"(val)); 349 return val; 350 } 351 352 static inline u16 read_ss(void) 353 { 354 unsigned val; 355 356 asm volatile ("mov %%ss, %0" : "=mr"(val)); 357 return val; 358 } 359 360 static inline u16 read_fs(void) 361 { 362 unsigned val; 363 364 asm volatile ("mov %%fs, %0" : "=mr"(val)); 365 return val; 366 } 367 368 static inline u16 read_gs(void) 369 { 370 unsigned val; 371 372 asm volatile ("mov %%gs, %0" : "=mr"(val)); 373 return val; 374 } 375 376 static inline unsigned long read_rflags(void) 377 { 378 unsigned long f; 379 asm volatile ("pushf; pop %0\n\t" : "=rm"(f)); 380 return f; 381 } 382 383 static inline void write_ds(unsigned val) 384 { 385 asm volatile ("mov %0, %%ds" : : "rm"(val) : "memory"); 386 } 387 388 static inline void write_es(unsigned val) 389 { 390 asm volatile ("mov %0, %%es" : : "rm"(val) : "memory"); 391 } 392 393 static inline void write_ss(unsigned val) 394 { 395 asm volatile ("mov %0, %%ss" : : "rm"(val) : "memory"); 396 } 397 398 static inline void write_fs(unsigned val) 399 { 400 asm volatile ("mov %0, %%fs" : : "rm"(val) : "memory"); 401 } 402 403 static inline void write_gs(unsigned val) 404 { 405 asm volatile ("mov %0, %%gs" : : "rm"(val) : "memory"); 406 } 407 408 static inline void write_rflags(unsigned long f) 409 { 410 asm volatile ("push %0; popf\n\t" : : "rm"(f)); 411 } 412 413 static inline void set_iopl(int iopl) 414 { 415 unsigned long flags = read_rflags() & ~X86_EFLAGS_IOPL; 416 flags |= iopl * (X86_EFLAGS_IOPL / 3); 417 write_rflags(flags); 418 } 419 420 /* 421 * Don't use the safe variants for rdmsr() or wrmsr(). The exception fixup 422 * infrastructure uses per-CPU data and thus consumes GS.base. Various tests 423 * temporarily modify MSR_GS_BASE and will explode when trying to determine 424 * whether or not RDMSR/WRMSR faulted. 425 */ 426 static inline u64 rdmsr(u32 index) 427 { 428 u32 a, d; 429 asm volatile ("rdmsr" : "=a"(a), "=d"(d) : "c"(index) : "memory"); 430 return a | ((u64)d << 32); 431 } 432 433 static inline void wrmsr(u32 index, u64 val) 434 { 435 u32 a = val, d = val >> 32; 436 asm volatile ("wrmsr" : : "a"(a), "d"(d), "c"(index) : "memory"); 437 } 438 439 #define __rdreg64_safe(fep, insn, index, val) \ 440 ({ \ 441 uint32_t a, d; \ 442 int vector; \ 443 \ 444 vector = __asm_safe_out2(fep, insn, "=a"(a), "=d"(d), "c"(index));\ 445 \ 446 if (vector) \ 447 *(val) = 0; \ 448 else \ 449 *(val) = (uint64_t)a | ((uint64_t)d << 32); \ 450 vector; \ 451 }) 452 453 #define rdreg64_safe(insn, index, val) \ 454 __rdreg64_safe("", insn, index, val) 455 456 #define __wrreg64_safe(fep, insn, index, val) \ 457 ({ \ 458 uint32_t eax = (val), edx = (val) >> 32; \ 459 \ 460 __asm_safe(fep, insn, "a" (eax), "d" (edx), "c" (index)); \ 461 }) 462 463 #define wrreg64_safe(insn, index, val) \ 464 __wrreg64_safe("", insn, index, val) 465 466 static inline int rdmsr_safe(u32 index, uint64_t *val) 467 { 468 return rdreg64_safe("rdmsr", index, val); 469 } 470 471 static inline int rdmsr_fep_safe(u32 index, uint64_t *val) 472 { 473 return __rdreg64_safe(KVM_FEP, "rdmsr", index, val); 474 } 475 476 static inline int wrmsr_safe(u32 index, u64 val) 477 { 478 return wrreg64_safe("wrmsr", index, val); 479 } 480 481 static inline int wrmsr_fep_safe(u32 index, u64 val) 482 { 483 return __wrreg64_safe(KVM_FEP, "wrmsr", index, val); 484 } 485 486 static inline int rdpmc_safe(u32 index, uint64_t *val) 487 { 488 return rdreg64_safe("rdpmc", index, val); 489 } 490 491 static inline uint64_t rdpmc(uint32_t index) 492 { 493 uint64_t val; 494 int vector = rdpmc_safe(index, &val); 495 496 assert_msg(!vector, "Unexpected %s on RDPMC(%" PRId32 ")", 497 exception_mnemonic(vector), index); 498 return val; 499 } 500 501 static inline int xgetbv_safe(u32 index, u64 *result) 502 { 503 return rdreg64_safe(".byte 0x0f,0x01,0xd0", index, result); 504 } 505 506 static inline int xsetbv_safe(u32 index, u64 value) 507 { 508 return wrreg64_safe(".byte 0x0f,0x01,0xd1", index, value); 509 } 510 511 static inline int write_cr0_safe(ulong val) 512 { 513 return asm_safe("mov %0,%%cr0", "r" (val)); 514 } 515 516 static inline void write_cr0(ulong val) 517 { 518 int vector = write_cr0_safe(val); 519 520 assert_msg(!vector, "Unexpected fault '%d' writing CR0 = %lx", 521 vector, val); 522 } 523 524 static inline ulong read_cr0(void) 525 { 526 ulong val; 527 asm volatile ("mov %%cr0, %0" : "=r"(val) : : "memory"); 528 return val; 529 } 530 531 static inline void write_cr2(ulong val) 532 { 533 asm volatile ("mov %0, %%cr2" : : "r"(val) : "memory"); 534 } 535 536 static inline ulong read_cr2(void) 537 { 538 ulong val; 539 asm volatile ("mov %%cr2, %0" : "=r"(val) : : "memory"); 540 return val; 541 } 542 543 static inline int write_cr3_safe(ulong val) 544 { 545 return asm_safe("mov %0,%%cr3", "r" (val)); 546 } 547 548 static inline void write_cr3(ulong val) 549 { 550 int vector = write_cr3_safe(val); 551 552 assert_msg(!vector, "Unexpected fault '%d' writing CR3 = %lx", 553 vector, val); 554 } 555 556 static inline ulong read_cr3(void) 557 { 558 ulong val; 559 asm volatile ("mov %%cr3, %0" : "=r"(val) : : "memory"); 560 return val; 561 } 562 563 static inline void update_cr3(void *cr3) 564 { 565 write_cr3((ulong)cr3); 566 } 567 568 static inline int write_cr4_safe(ulong val) 569 { 570 return asm_safe("mov %0,%%cr4", "r" (val)); 571 } 572 573 static inline void write_cr4(ulong val) 574 { 575 int vector = write_cr4_safe(val); 576 577 assert_msg(!vector, "Unexpected fault '%d' writing CR4 = %lx", 578 vector, val); 579 } 580 581 static inline ulong read_cr4(void) 582 { 583 ulong val; 584 asm volatile ("mov %%cr4, %0" : "=r"(val) : : "memory"); 585 return val; 586 } 587 588 static inline void write_cr8(ulong val) 589 { 590 asm volatile ("mov %0, %%cr8" : : "r"(val) : "memory"); 591 } 592 593 static inline ulong read_cr8(void) 594 { 595 ulong val; 596 asm volatile ("mov %%cr8, %0" : "=r"(val) : : "memory"); 597 return val; 598 } 599 600 static inline void lgdt(const struct descriptor_table_ptr *ptr) 601 { 602 asm volatile ("lgdt %0" : : "m"(*ptr)); 603 } 604 605 static inline int lgdt_safe(const struct descriptor_table_ptr *ptr) 606 { 607 return asm_safe("lgdt %0", "m"(*ptr)); 608 } 609 610 static inline int lgdt_fep_safe(const struct descriptor_table_ptr *ptr) 611 { 612 return asm_fep_safe("lgdt %0", "m"(*ptr)); 613 } 614 615 static inline void sgdt(struct descriptor_table_ptr *ptr) 616 { 617 asm volatile ("sgdt %0" : "=m"(*ptr)); 618 } 619 620 static inline void lidt(const struct descriptor_table_ptr *ptr) 621 { 622 asm volatile ("lidt %0" : : "m"(*ptr)); 623 } 624 625 static inline int lidt_safe(const struct descriptor_table_ptr *ptr) 626 { 627 return asm_safe("lidt %0", "m"(*ptr)); 628 } 629 630 static inline int lidt_fep_safe(const struct descriptor_table_ptr *ptr) 631 { 632 return asm_fep_safe("lidt %0", "m"(*ptr)); 633 } 634 635 static inline void sidt(struct descriptor_table_ptr *ptr) 636 { 637 asm volatile ("sidt %0" : "=m"(*ptr)); 638 } 639 640 static inline void lldt(u16 val) 641 { 642 asm volatile ("lldt %0" : : "rm"(val)); 643 } 644 645 static inline int lldt_safe(u16 val) 646 { 647 return asm_safe("lldt %0", "rm"(val)); 648 } 649 650 static inline int lldt_fep_safe(u16 val) 651 { 652 return asm_safe("lldt %0", "rm"(val)); 653 } 654 655 static inline u16 sldt(void) 656 { 657 u16 val; 658 asm volatile ("sldt %0" : "=rm"(val)); 659 return val; 660 } 661 662 static inline void ltr(u16 val) 663 { 664 asm volatile ("ltr %0" : : "rm"(val)); 665 } 666 667 static inline int ltr_safe(u16 val) 668 { 669 return asm_safe("ltr %0", "rm"(val)); 670 } 671 672 static inline int ltr_fep_safe(u16 val) 673 { 674 return asm_safe("ltr %0", "rm"(val)); 675 } 676 677 static inline u16 str(void) 678 { 679 u16 val; 680 asm volatile ("str %0" : "=rm"(val)); 681 return val; 682 } 683 684 static inline void write_dr0(void *val) 685 { 686 asm volatile ("mov %0, %%dr0" : : "r"(val) : "memory"); 687 } 688 689 static inline void write_dr1(void *val) 690 { 691 asm volatile ("mov %0, %%dr1" : : "r"(val) : "memory"); 692 } 693 694 static inline void write_dr2(void *val) 695 { 696 asm volatile ("mov %0, %%dr2" : : "r"(val) : "memory"); 697 } 698 699 static inline void write_dr3(void *val) 700 { 701 asm volatile ("mov %0, %%dr3" : : "r"(val) : "memory"); 702 } 703 704 static inline void write_dr6(ulong val) 705 { 706 asm volatile ("mov %0, %%dr6" : : "r"(val) : "memory"); 707 } 708 709 static inline ulong read_dr6(void) 710 { 711 ulong val; 712 asm volatile ("mov %%dr6, %0" : "=r"(val)); 713 return val; 714 } 715 716 static inline void write_dr7(ulong val) 717 { 718 asm volatile ("mov %0, %%dr7" : : "r"(val) : "memory"); 719 } 720 721 static inline ulong read_dr7(void) 722 { 723 ulong val; 724 asm volatile ("mov %%dr7, %0" : "=r"(val)); 725 return val; 726 } 727 728 static inline void pause(void) 729 { 730 asm volatile ("pause"); 731 } 732 733 static inline void cli(void) 734 { 735 asm volatile ("cli"); 736 } 737 738 /* 739 * See also safe_halt(). 740 */ 741 static inline void sti(void) 742 { 743 asm volatile ("sti"); 744 } 745 746 /* 747 * Enable interrupts and ensure that interrupts are evaluated upon return from 748 * this function, i.e. execute a nop to consume the STi interrupt shadow. 749 */ 750 static inline void sti_nop(void) 751 { 752 asm volatile ("sti; nop"); 753 } 754 755 /* 756 * Enable interrupts for one instruction (nop), to allow the CPU to process all 757 * interrupts that are already pending. 758 */ 759 static inline void sti_nop_cli(void) 760 { 761 asm volatile ("sti; nop; cli"); 762 } 763 764 static inline unsigned long long rdrand(void) 765 { 766 long long r; 767 768 asm volatile("rdrand %0\n\t" 769 "jc 1f\n\t" 770 "mov $0, %0\n\t" 771 "1:\n\t" : "=r" (r)); 772 return r; 773 } 774 775 static inline unsigned long long rdtsc(void) 776 { 777 long long r; 778 779 #ifdef __x86_64__ 780 unsigned a, d; 781 782 asm volatile ("rdtsc" : "=a"(a), "=d"(d)); 783 r = a | ((long long)d << 32); 784 #else 785 asm volatile ("rdtsc" : "=A"(r)); 786 #endif 787 return r; 788 } 789 790 /* 791 * Per the advice in the SDM, volume 2, the sequence "mfence; lfence" 792 * executed immediately before rdtsc ensures that rdtsc will be 793 * executed only after all previous instructions have executed and all 794 * previous loads and stores are globally visible. In addition, the 795 * lfence immediately after rdtsc ensures that rdtsc will be executed 796 * prior to the execution of any subsequent instruction. 797 */ 798 static inline unsigned long long fenced_rdtsc(void) 799 { 800 unsigned long long tsc; 801 802 #ifdef __x86_64__ 803 unsigned int eax, edx; 804 805 asm volatile ("mfence; lfence; rdtsc; lfence" : "=a"(eax), "=d"(edx)); 806 tsc = eax | ((unsigned long long)edx << 32); 807 #else 808 asm volatile ("mfence; lfence; rdtsc; lfence" : "=A"(tsc)); 809 #endif 810 return tsc; 811 } 812 813 static inline unsigned long long rdtscp(u32 *aux) 814 { 815 long long r; 816 817 #ifdef __x86_64__ 818 unsigned a, d; 819 820 asm volatile ("rdtscp" : "=a"(a), "=d"(d), "=c"(*aux)); 821 r = a | ((long long)d << 32); 822 #else 823 asm volatile ("rdtscp" : "=A"(r), "=c"(*aux)); 824 #endif 825 return r; 826 } 827 828 static inline void wrtsc(u64 tsc) 829 { 830 wrmsr(MSR_IA32_TSC, tsc); 831 } 832 833 834 static inline void invlpg(volatile void *va) 835 { 836 asm volatile("invlpg (%0)" ::"r" (va) : "memory"); 837 } 838 839 struct invpcid_desc { 840 u64 pcid : 12; 841 u64 rsv : 52; 842 u64 addr : 64; 843 }; 844 845 static inline int invpcid_safe(unsigned long type, struct invpcid_desc *desc) 846 { 847 /* invpcid (%rax), %rbx */ 848 return asm_safe(".byte 0x66,0x0f,0x38,0x82,0x18", "a" (desc), "b" (type)); 849 } 850 851 /* 852 * Execute HLT in an STI interrupt shadow to ensure that a pending IRQ that's 853 * intended to be a wake event arrives *after* HLT is executed. Modern CPUs, 854 * except for a few oddballs that KVM is unlikely to run on, block IRQs for one 855 * instruction after STI, *if* RFLAGS.IF=0 before STI. Note, Intel CPUs may 856 * block other events beyond regular IRQs, e.g. may block NMIs and SMIs too. 857 */ 858 static inline void safe_halt(void) 859 { 860 asm volatile("sti; hlt"); 861 } 862 863 static inline u32 read_pkru(void) 864 { 865 unsigned int eax, edx; 866 unsigned int ecx = 0; 867 unsigned int pkru; 868 869 asm volatile(".byte 0x0f,0x01,0xee\n\t" 870 : "=a" (eax), "=d" (edx) 871 : "c" (ecx)); 872 pkru = eax; 873 return pkru; 874 } 875 876 static inline void write_pkru(u32 pkru) 877 { 878 unsigned int eax = pkru; 879 unsigned int ecx = 0; 880 unsigned int edx = 0; 881 882 asm volatile(".byte 0x0f,0x01,0xef\n\t" 883 : : "a" (eax), "c" (ecx), "d" (edx)); 884 } 885 886 static inline bool is_canonical(u64 addr) 887 { 888 int va_width = (raw_cpuid(0x80000008, 0).a & 0xff00) >> 8; 889 int shift_amt = 64 - va_width; 890 891 return (s64)(addr << shift_amt) >> shift_amt == addr; 892 } 893 894 static inline void clear_bit(int bit, u8 *addr) 895 { 896 __asm__ __volatile__("lock; btr %1, %0" 897 : "+m" (*addr) : "Ir" (bit) : "cc", "memory"); 898 } 899 900 static inline void set_bit(int bit, u8 *addr) 901 { 902 __asm__ __volatile__("lock; bts %1, %0" 903 : "+m" (*addr) : "Ir" (bit) : "cc", "memory"); 904 } 905 906 static inline void flush_tlb(void) 907 { 908 ulong cr4; 909 910 cr4 = read_cr4(); 911 write_cr4(cr4 ^ X86_CR4_PGE); 912 write_cr4(cr4); 913 } 914 915 static inline void generate_non_canonical_gp(void) 916 { 917 *(volatile u64 *)NONCANONICAL = 0; 918 } 919 920 static inline void generate_ud(void) 921 { 922 asm volatile ("ud2"); 923 } 924 925 static inline void generate_de(void) 926 { 927 asm volatile ( 928 "xor %%eax, %%eax\n\t" 929 "xor %%ebx, %%ebx\n\t" 930 "xor %%edx, %%edx\n\t" 931 "idiv %%ebx\n\t" 932 ::: "eax", "ebx", "edx"); 933 } 934 935 static inline void generate_bp(void) 936 { 937 asm volatile ("int3"); 938 } 939 940 static inline void generate_single_step_db(void) 941 { 942 write_rflags(read_rflags() | X86_EFLAGS_TF); 943 asm volatile("nop"); 944 } 945 946 static inline uint64_t generate_usermode_ac(void) 947 { 948 /* 949 * Trigger an #AC by writing 8 bytes to a 4-byte aligned address. 950 * Disclaimer: It is assumed that the stack pointer is aligned 951 * on a 16-byte boundary as x86_64 stacks should be. 952 */ 953 asm volatile("movq $0, -0x4(%rsp)"); 954 955 return 0; 956 } 957 958 /* 959 * Switch from 64-bit to 32-bit mode and generate #OF via INTO. Note, if RIP 960 * or RSP holds a 64-bit value, this helper will NOT generate #OF. 961 */ 962 static inline void generate_of(void) 963 { 964 struct far_pointer32 fp = { 965 .offset = (uintptr_t)&&into, 966 .selector = KERNEL_CS32, 967 }; 968 uintptr_t rsp; 969 970 asm volatile ("mov %%rsp, %0" : "=r"(rsp)); 971 972 if (fp.offset != (uintptr_t)&&into) { 973 printf("Code address too high.\n"); 974 return; 975 } 976 if ((u32)rsp != rsp) { 977 printf("Stack address too high.\n"); 978 return; 979 } 980 981 asm goto ("lcall *%0" : : "m" (fp) : "rax" : into); 982 return; 983 into: 984 asm volatile (".code32;" 985 "movl $0x7fffffff, %eax;" 986 "addl %eax, %eax;" 987 "into;" 988 "lret;" 989 ".code64"); 990 __builtin_unreachable(); 991 } 992 993 static inline void fnop(void) 994 { 995 asm volatile("fnop"); 996 } 997 998 /* If CR0.TS is set in L2, #NM is generated. */ 999 static inline void generate_cr0_ts_nm(void) 1000 { 1001 write_cr0((read_cr0() & ~X86_CR0_EM) | X86_CR0_TS); 1002 fnop(); 1003 } 1004 1005 /* If CR0.TS is cleared and CR0.EM is set, #NM is generated. */ 1006 static inline void generate_cr0_em_nm(void) 1007 { 1008 write_cr0((read_cr0() & ~X86_CR0_TS) | X86_CR0_EM); 1009 fnop(); 1010 } 1011 1012 #endif 1013