1 #ifndef _X86_PROCESSOR_H_ 2 #define _X86_PROCESSOR_H_ 3 4 #include "libcflat.h" 5 #include "desc.h" 6 #include "msr.h" 7 #include <bitops.h> 8 #include <stdint.h> 9 10 #define CANONICAL_48_VAL 0xffffaaaaaaaaaaaaull 11 #define CANONICAL_57_VAL 0xffaaaaaaaaaaaaaaull 12 #define NONCANONICAL 0xaaaaaaaaaaaaaaaaull 13 14 #ifdef __x86_64__ 15 # define R "r" 16 # define W "q" 17 # define S "8" 18 #else 19 # define R "e" 20 # define W "l" 21 # define S "4" 22 #endif 23 24 #define DE_VECTOR 0 25 #define DB_VECTOR 1 26 #define NMI_VECTOR 2 27 #define BP_VECTOR 3 28 #define OF_VECTOR 4 29 #define BR_VECTOR 5 30 #define UD_VECTOR 6 31 #define NM_VECTOR 7 32 #define DF_VECTOR 8 33 #define TS_VECTOR 10 34 #define NP_VECTOR 11 35 #define SS_VECTOR 12 36 #define GP_VECTOR 13 37 #define PF_VECTOR 14 38 #define MF_VECTOR 16 39 #define AC_VECTOR 17 40 #define MC_VECTOR 18 41 #define XM_VECTOR 19 42 #define XF_VECTOR XM_VECTOR /* AMD */ 43 #define VE_VECTOR 20 /* Intel only */ 44 #define CP_VECTOR 21 45 #define HV_VECTOR 28 /* AMD only */ 46 #define VC_VECTOR 29 /* AMD only */ 47 #define SX_VECTOR 30 /* AMD only */ 48 49 #define X86_CR0_PE_BIT (0) 50 #define X86_CR0_PE BIT(X86_CR0_PE_BIT) 51 #define X86_CR0_MP_BIT (1) 52 #define X86_CR0_MP BIT(X86_CR0_MP_BIT) 53 #define X86_CR0_EM_BIT (2) 54 #define X86_CR0_EM BIT(X86_CR0_EM_BIT) 55 #define X86_CR0_TS_BIT (3) 56 #define X86_CR0_TS BIT(X86_CR0_TS_BIT) 57 #define X86_CR0_ET_BIT (4) 58 #define X86_CR0_ET BIT(X86_CR0_ET_BIT) 59 #define X86_CR0_NE_BIT (5) 60 #define X86_CR0_NE BIT(X86_CR0_NE_BIT) 61 #define X86_CR0_WP_BIT (16) 62 #define X86_CR0_WP BIT(X86_CR0_WP_BIT) 63 #define X86_CR0_AM_BIT (18) 64 #define X86_CR0_AM BIT(X86_CR0_AM_BIT) 65 #define X86_CR0_NW_BIT (29) 66 #define X86_CR0_NW BIT(X86_CR0_NW_BIT) 67 #define X86_CR0_CD_BIT (30) 68 #define X86_CR0_CD BIT(X86_CR0_CD_BIT) 69 #define X86_CR0_PG_BIT (31) 70 #define X86_CR0_PG BIT(X86_CR0_PG_BIT) 71 72 #define X86_CR3_PCID_MASK GENMASK(11, 0) 73 74 #define X86_CR4_VME_BIT (0) 75 #define X86_CR4_VME BIT(X86_CR4_VME_BIT) 76 #define X86_CR4_PVI_BIT (1) 77 #define X86_CR4_PVI BIT(X86_CR4_PVI_BIT) 78 #define X86_CR4_TSD_BIT (2) 79 #define X86_CR4_TSD BIT(X86_CR4_TSD_BIT) 80 #define X86_CR4_DE_BIT (3) 81 #define X86_CR4_DE BIT(X86_CR4_DE_BIT) 82 #define X86_CR4_PSE_BIT (4) 83 #define X86_CR4_PSE BIT(X86_CR4_PSE_BIT) 84 #define X86_CR4_PAE_BIT (5) 85 #define X86_CR4_PAE BIT(X86_CR4_PAE_BIT) 86 #define X86_CR4_MCE_BIT (6) 87 #define X86_CR4_MCE BIT(X86_CR4_MCE_BIT) 88 #define X86_CR4_PGE_BIT (7) 89 #define X86_CR4_PGE BIT(X86_CR4_PGE_BIT) 90 #define X86_CR4_PCE_BIT (8) 91 #define X86_CR4_PCE BIT(X86_CR4_PCE_BIT) 92 #define X86_CR4_OSFXSR_BIT (9) 93 #define X86_CR4_OSFXSR BIT(X86_CR4_OSFXSR_BIT) 94 #define X86_CR4_OSXMMEXCPT_BIT (10) 95 #define X86_CR4_OSXMMEXCPT BIT(X86_CR4_OSXMMEXCPT_BIT) 96 #define X86_CR4_UMIP_BIT (11) 97 #define X86_CR4_UMIP BIT(X86_CR4_UMIP_BIT) 98 #define X86_CR4_LA57_BIT (12) 99 #define X86_CR4_LA57 BIT(X86_CR4_LA57_BIT) 100 #define X86_CR4_VMXE_BIT (13) 101 #define X86_CR4_VMXE BIT(X86_CR4_VMXE_BIT) 102 #define X86_CR4_SMXE_BIT (14) 103 #define X86_CR4_SMXE BIT(X86_CR4_SMXE_BIT) 104 /* UNUSED (15) */ 105 #define X86_CR4_FSGSBASE_BIT (16) 106 #define X86_CR4_FSGSBASE BIT(X86_CR4_FSGSBASE_BIT) 107 #define X86_CR4_PCIDE_BIT (17) 108 #define X86_CR4_PCIDE BIT(X86_CR4_PCIDE_BIT) 109 #define X86_CR4_OSXSAVE_BIT (18) 110 #define X86_CR4_OSXSAVE BIT(X86_CR4_OSXSAVE_BIT) 111 #define X86_CR4_KL_BIT (19) 112 #define X86_CR4_KL BIT(X86_CR4_KL_BIT) 113 #define X86_CR4_SMEP_BIT (20) 114 #define X86_CR4_SMEP BIT(X86_CR4_SMEP_BIT) 115 #define X86_CR4_SMAP_BIT (21) 116 #define X86_CR4_SMAP BIT(X86_CR4_SMAP_BIT) 117 #define X86_CR4_PKE_BIT (22) 118 #define X86_CR4_PKE BIT(X86_CR4_PKE_BIT) 119 #define X86_CR4_CET_BIT (23) 120 #define X86_CR4_CET BIT(X86_CR4_CET_BIT) 121 #define X86_CR4_PKS_BIT (24) 122 #define X86_CR4_PKS BIT(X86_CR4_PKS_BIT) 123 124 #define X86_EFLAGS_CF_BIT (0) 125 #define X86_EFLAGS_CF BIT(X86_EFLAGS_CF_BIT) 126 #define X86_EFLAGS_FIXED_BIT (1) 127 #define X86_EFLAGS_FIXED BIT(X86_EFLAGS_FIXED_BIT) 128 #define X86_EFLAGS_PF_BIT (2) 129 #define X86_EFLAGS_PF BIT(X86_EFLAGS_PF_BIT) 130 /* RESERVED 0 (3) */ 131 #define X86_EFLAGS_AF_BIT (4) 132 #define X86_EFLAGS_AF BIT(X86_EFLAGS_AF_BIT) 133 /* RESERVED 0 (5) */ 134 #define X86_EFLAGS_ZF_BIT (6) 135 #define X86_EFLAGS_ZF BIT(X86_EFLAGS_ZF_BIT) 136 #define X86_EFLAGS_SF_BIT (7) 137 #define X86_EFLAGS_SF BIT(X86_EFLAGS_SF_BIT) 138 #define X86_EFLAGS_TF_BIT (8) 139 #define X86_EFLAGS_TF BIT(X86_EFLAGS_TF_BIT) 140 #define X86_EFLAGS_IF_BIT (9) 141 #define X86_EFLAGS_IF BIT(X86_EFLAGS_IF_BIT) 142 #define X86_EFLAGS_DF_BIT (10) 143 #define X86_EFLAGS_DF BIT(X86_EFLAGS_DF_BIT) 144 #define X86_EFLAGS_OF_BIT (11) 145 #define X86_EFLAGS_OF BIT(X86_EFLAGS_OF_BIT) 146 #define X86_EFLAGS_IOPL GENMASK(13, 12) 147 #define X86_EFLAGS_NT_BIT (14) 148 #define X86_EFLAGS_NT BIT(X86_EFLAGS_NT_BIT) 149 /* RESERVED 0 (15) */ 150 #define X86_EFLAGS_RF_BIT (16) 151 #define X86_EFLAGS_RF BIT(X86_EFLAGS_RF_BIT) 152 #define X86_EFLAGS_VM_BIT (17) 153 #define X86_EFLAGS_VM BIT(X86_EFLAGS_VM_BIT) 154 #define X86_EFLAGS_AC_BIT (18) 155 #define X86_EFLAGS_AC BIT(X86_EFLAGS_AC_BIT) 156 #define X86_EFLAGS_VIF_BIT (19) 157 #define X86_EFLAGS_VIF BIT(X86_EFLAGS_VIF_BIT) 158 #define X86_EFLAGS_VIP_BIT (20) 159 #define X86_EFLAGS_VIP BIT(X86_EFLAGS_VIP_BIT) 160 #define X86_EFLAGS_ID_BIT (21) 161 #define X86_EFLAGS_ID BIT(X86_EFLAGS_ID_BIT) 162 163 #define X86_EFLAGS_ALU (X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | \ 164 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF) 165 166 167 /* 168 * CPU features 169 */ 170 171 enum cpuid_output_regs { 172 EAX, 173 EBX, 174 ECX, 175 EDX 176 }; 177 178 struct cpuid { u32 a, b, c, d; }; 179 180 static inline struct cpuid raw_cpuid(u32 function, u32 index) 181 { 182 struct cpuid r; 183 asm volatile ("cpuid" 184 : "=a"(r.a), "=b"(r.b), "=c"(r.c), "=d"(r.d) 185 : "0"(function), "2"(index)); 186 return r; 187 } 188 189 static inline struct cpuid cpuid_indexed(u32 function, u32 index) 190 { 191 u32 level = raw_cpuid(function & 0xf0000000, 0).a; 192 if (level < function) 193 return (struct cpuid) { 0, 0, 0, 0 }; 194 return raw_cpuid(function, index); 195 } 196 197 static inline struct cpuid cpuid(u32 function) 198 { 199 return cpuid_indexed(function, 0); 200 } 201 202 static inline u8 cpuid_maxphyaddr(void) 203 { 204 if (raw_cpuid(0x80000000, 0).a < 0x80000008) 205 return 36; 206 return raw_cpuid(0x80000008, 0).a & 0xff; 207 } 208 209 static inline bool is_intel(void) 210 { 211 struct cpuid c = cpuid(0); 212 u32 name[4] = {c.b, c.d, c.c }; 213 214 return strcmp((char *)name, "GenuineIntel") == 0; 215 } 216 217 #define CPUID(a, b, c, d) ((((unsigned long long) a) << 32) | (b << 16) | \ 218 (c << 8) | d) 219 220 /* 221 * Each X86_FEATURE_XXX definition is 64-bit and contains the following 222 * CPUID meta-data: 223 * 224 * [63:32] : input value for EAX 225 * [31:16] : input value for ECX 226 * [15:8] : output register 227 * [7:0] : bit position in output register 228 */ 229 230 /* 231 * Basic Leafs, a.k.a. Intel defined 232 */ 233 #define X86_FEATURE_MWAIT (CPUID(0x1, 0, ECX, 3)) 234 #define X86_FEATURE_VMX (CPUID(0x1, 0, ECX, 5)) 235 #define X86_FEATURE_PDCM (CPUID(0x1, 0, ECX, 15)) 236 #define X86_FEATURE_PCID (CPUID(0x1, 0, ECX, 17)) 237 #define X86_FEATURE_X2APIC (CPUID(0x1, 0, ECX, 21)) 238 #define X86_FEATURE_MOVBE (CPUID(0x1, 0, ECX, 22)) 239 #define X86_FEATURE_TSC_DEADLINE_TIMER (CPUID(0x1, 0, ECX, 24)) 240 #define X86_FEATURE_XSAVE (CPUID(0x1, 0, ECX, 26)) 241 #define X86_FEATURE_OSXSAVE (CPUID(0x1, 0, ECX, 27)) 242 #define X86_FEATURE_RDRAND (CPUID(0x1, 0, ECX, 30)) 243 #define X86_FEATURE_MCE (CPUID(0x1, 0, EDX, 7)) 244 #define X86_FEATURE_APIC (CPUID(0x1, 0, EDX, 9)) 245 #define X86_FEATURE_CLFLUSH (CPUID(0x1, 0, EDX, 19)) 246 #define X86_FEATURE_DS (CPUID(0x1, 0, EDX, 21)) 247 #define X86_FEATURE_XMM (CPUID(0x1, 0, EDX, 25)) 248 #define X86_FEATURE_XMM2 (CPUID(0x1, 0, EDX, 26)) 249 #define X86_FEATURE_TSC_ADJUST (CPUID(0x7, 0, EBX, 1)) 250 #define X86_FEATURE_HLE (CPUID(0x7, 0, EBX, 4)) 251 #define X86_FEATURE_SMEP (CPUID(0x7, 0, EBX, 7)) 252 #define X86_FEATURE_INVPCID (CPUID(0x7, 0, EBX, 10)) 253 #define X86_FEATURE_RTM (CPUID(0x7, 0, EBX, 11)) 254 #define X86_FEATURE_SMAP (CPUID(0x7, 0, EBX, 20)) 255 #define X86_FEATURE_PCOMMIT (CPUID(0x7, 0, EBX, 22)) 256 #define X86_FEATURE_CLFLUSHOPT (CPUID(0x7, 0, EBX, 23)) 257 #define X86_FEATURE_CLWB (CPUID(0x7, 0, EBX, 24)) 258 #define X86_FEATURE_INTEL_PT (CPUID(0x7, 0, EBX, 25)) 259 #define X86_FEATURE_UMIP (CPUID(0x7, 0, ECX, 2)) 260 #define X86_FEATURE_PKU (CPUID(0x7, 0, ECX, 3)) 261 #define X86_FEATURE_LA57 (CPUID(0x7, 0, ECX, 16)) 262 #define X86_FEATURE_RDPID (CPUID(0x7, 0, ECX, 22)) 263 #define X86_FEATURE_SHSTK (CPUID(0x7, 0, ECX, 7)) 264 #define X86_FEATURE_IBT (CPUID(0x7, 0, EDX, 20)) 265 #define X86_FEATURE_SPEC_CTRL (CPUID(0x7, 0, EDX, 26)) 266 #define X86_FEATURE_FLUSH_L1D (CPUID(0x7, 0, EDX, 28)) 267 #define X86_FEATURE_ARCH_CAPABILITIES (CPUID(0x7, 0, EDX, 29)) 268 #define X86_FEATURE_PKS (CPUID(0x7, 0, ECX, 31)) 269 270 /* 271 * KVM defined leafs 272 */ 273 #define KVM_FEATURE_ASYNC_PF (CPUID(0x40000001, 0, EAX, 4)) 274 #define KVM_FEATURE_ASYNC_PF_INT (CPUID(0x40000001, 0, EAX, 14)) 275 276 /* 277 * Extended Leafs, a.k.a. AMD defined 278 */ 279 #define X86_FEATURE_SVM (CPUID(0x80000001, 0, ECX, 2)) 280 #define X86_FEATURE_PERFCTR_CORE (CPUID(0x80000001, 0, ECX, 23)) 281 #define X86_FEATURE_NX (CPUID(0x80000001, 0, EDX, 20)) 282 #define X86_FEATURE_GBPAGES (CPUID(0x80000001, 0, EDX, 26)) 283 #define X86_FEATURE_RDTSCP (CPUID(0x80000001, 0, EDX, 27)) 284 #define X86_FEATURE_LM (CPUID(0x80000001, 0, EDX, 29)) 285 #define X86_FEATURE_RDPRU (CPUID(0x80000008, 0, EBX, 4)) 286 #define X86_FEATURE_AMD_IBPB (CPUID(0x80000008, 0, EBX, 12)) 287 #define X86_FEATURE_NPT (CPUID(0x8000000A, 0, EDX, 0)) 288 #define X86_FEATURE_LBRV (CPUID(0x8000000A, 0, EDX, 1)) 289 #define X86_FEATURE_NRIPS (CPUID(0x8000000A, 0, EDX, 3)) 290 #define X86_FEATURE_TSCRATEMSR (CPUID(0x8000000A, 0, EDX, 4)) 291 #define X86_FEATURE_PAUSEFILTER (CPUID(0x8000000A, 0, EDX, 10)) 292 #define X86_FEATURE_PFTHRESHOLD (CPUID(0x8000000A, 0, EDX, 12)) 293 #define X86_FEATURE_VGIF (CPUID(0x8000000A, 0, EDX, 16)) 294 #define X86_FEATURE_VNMI (CPUID(0x8000000A, 0, EDX, 25)) 295 #define X86_FEATURE_AMD_PMU_V2 (CPUID(0x80000022, 0, EAX, 0)) 296 297 static inline bool this_cpu_has(u64 feature) 298 { 299 u32 input_eax = feature >> 32; 300 u32 input_ecx = (feature >> 16) & 0xffff; 301 u32 output_reg = (feature >> 8) & 0xff; 302 u8 bit = feature & 0xff; 303 struct cpuid c; 304 u32 *tmp; 305 306 c = cpuid_indexed(input_eax, input_ecx); 307 tmp = (u32 *)&c; 308 309 return ((*(tmp + (output_reg % 32))) & (1 << bit)); 310 } 311 312 struct far_pointer32 { 313 u32 offset; 314 u16 selector; 315 } __attribute__((packed)); 316 317 struct descriptor_table_ptr { 318 u16 limit; 319 ulong base; 320 } __attribute__((packed)); 321 322 static inline void clac(void) 323 { 324 asm volatile (".byte 0x0f, 0x01, 0xca" : : : "memory"); 325 } 326 327 static inline void stac(void) 328 { 329 asm volatile (".byte 0x0f, 0x01, 0xcb" : : : "memory"); 330 } 331 332 static inline u16 read_cs(void) 333 { 334 unsigned val; 335 336 asm volatile ("mov %%cs, %0" : "=mr"(val)); 337 return val; 338 } 339 340 static inline u16 read_ds(void) 341 { 342 unsigned val; 343 344 asm volatile ("mov %%ds, %0" : "=mr"(val)); 345 return val; 346 } 347 348 static inline u16 read_es(void) 349 { 350 unsigned val; 351 352 asm volatile ("mov %%es, %0" : "=mr"(val)); 353 return val; 354 } 355 356 static inline u16 read_ss(void) 357 { 358 unsigned val; 359 360 asm volatile ("mov %%ss, %0" : "=mr"(val)); 361 return val; 362 } 363 364 static inline u16 read_fs(void) 365 { 366 unsigned val; 367 368 asm volatile ("mov %%fs, %0" : "=mr"(val)); 369 return val; 370 } 371 372 static inline u16 read_gs(void) 373 { 374 unsigned val; 375 376 asm volatile ("mov %%gs, %0" : "=mr"(val)); 377 return val; 378 } 379 380 static inline unsigned long read_rflags(void) 381 { 382 unsigned long f; 383 asm volatile ("pushf; pop %0\n\t" : "=rm"(f)); 384 return f; 385 } 386 387 static inline void write_ds(unsigned val) 388 { 389 asm volatile ("mov %0, %%ds" : : "rm"(val) : "memory"); 390 } 391 392 static inline void write_es(unsigned val) 393 { 394 asm volatile ("mov %0, %%es" : : "rm"(val) : "memory"); 395 } 396 397 static inline void write_ss(unsigned val) 398 { 399 asm volatile ("mov %0, %%ss" : : "rm"(val) : "memory"); 400 } 401 402 static inline void write_fs(unsigned val) 403 { 404 asm volatile ("mov %0, %%fs" : : "rm"(val) : "memory"); 405 } 406 407 static inline void write_gs(unsigned val) 408 { 409 asm volatile ("mov %0, %%gs" : : "rm"(val) : "memory"); 410 } 411 412 static inline void write_rflags(unsigned long f) 413 { 414 asm volatile ("push %0; popf\n\t" : : "rm"(f)); 415 } 416 417 static inline void set_iopl(int iopl) 418 { 419 unsigned long flags = read_rflags() & ~X86_EFLAGS_IOPL; 420 flags |= iopl * (X86_EFLAGS_IOPL / 3); 421 write_rflags(flags); 422 } 423 424 /* 425 * Don't use the safe variants for rdmsr() or wrmsr(). The exception fixup 426 * infrastructure uses per-CPU data and thus consumes GS.base. Various tests 427 * temporarily modify MSR_GS_BASE and will explode when trying to determine 428 * whether or not RDMSR/WRMSR faulted. 429 */ 430 static inline u64 rdmsr(u32 index) 431 { 432 u32 a, d; 433 asm volatile ("rdmsr" : "=a"(a), "=d"(d) : "c"(index) : "memory"); 434 return a | ((u64)d << 32); 435 } 436 437 static inline void wrmsr(u32 index, u64 val) 438 { 439 u32 a = val, d = val >> 32; 440 asm volatile ("wrmsr" : : "a"(a), "d"(d), "c"(index) : "memory"); 441 } 442 443 #define __rdreg64_safe(fep, insn, index, val) \ 444 ({ \ 445 uint32_t a, d; \ 446 int vector; \ 447 \ 448 vector = __asm_safe_out2(fep, insn, "=a"(a), "=d"(d), "c"(index));\ 449 \ 450 if (vector) \ 451 *(val) = 0; \ 452 else \ 453 *(val) = (uint64_t)a | ((uint64_t)d << 32); \ 454 vector; \ 455 }) 456 457 #define rdreg64_safe(insn, index, val) \ 458 __rdreg64_safe("", insn, index, val) 459 460 #define __wrreg64_safe(fep, insn, index, val) \ 461 ({ \ 462 uint32_t eax = (val), edx = (val) >> 32; \ 463 \ 464 __asm_safe(fep, insn, "a" (eax), "d" (edx), "c" (index)); \ 465 }) 466 467 #define wrreg64_safe(insn, index, val) \ 468 __wrreg64_safe("", insn, index, val) 469 470 static inline int rdmsr_safe(u32 index, uint64_t *val) 471 { 472 return rdreg64_safe("rdmsr", index, val); 473 } 474 475 static inline int rdmsr_fep_safe(u32 index, uint64_t *val) 476 { 477 return __rdreg64_safe(KVM_FEP, "rdmsr", index, val); 478 } 479 480 static inline int wrmsr_safe(u32 index, u64 val) 481 { 482 return wrreg64_safe("wrmsr", index, val); 483 } 484 485 static inline int wrmsr_fep_safe(u32 index, u64 val) 486 { 487 return __wrreg64_safe(KVM_FEP, "wrmsr", index, val); 488 } 489 490 static inline int rdpmc_safe(u32 index, uint64_t *val) 491 { 492 return rdreg64_safe("rdpmc", index, val); 493 } 494 495 static inline uint64_t rdpmc(uint32_t index) 496 { 497 uint64_t val; 498 int vector = rdpmc_safe(index, &val); 499 500 assert_msg(!vector, "Unexpected %s on RDPMC(%" PRId32 ")", 501 exception_mnemonic(vector), index); 502 return val; 503 } 504 505 static inline int xgetbv_safe(u32 index, u64 *result) 506 { 507 return rdreg64_safe(".byte 0x0f,0x01,0xd0", index, result); 508 } 509 510 static inline int xsetbv_safe(u32 index, u64 value) 511 { 512 return wrreg64_safe(".byte 0x0f,0x01,0xd1", index, value); 513 } 514 515 static inline int write_cr0_safe(ulong val) 516 { 517 return asm_safe("mov %0,%%cr0", "r" (val)); 518 } 519 520 static inline void write_cr0(ulong val) 521 { 522 int vector = write_cr0_safe(val); 523 524 assert_msg(!vector, "Unexpected fault '%d' writing CR0 = %lx", 525 vector, val); 526 } 527 528 static inline ulong read_cr0(void) 529 { 530 ulong val; 531 asm volatile ("mov %%cr0, %0" : "=r"(val) : : "memory"); 532 return val; 533 } 534 535 static inline void write_cr2(ulong val) 536 { 537 asm volatile ("mov %0, %%cr2" : : "r"(val) : "memory"); 538 } 539 540 static inline ulong read_cr2(void) 541 { 542 ulong val; 543 asm volatile ("mov %%cr2, %0" : "=r"(val) : : "memory"); 544 return val; 545 } 546 547 static inline int write_cr3_safe(ulong val) 548 { 549 return asm_safe("mov %0,%%cr3", "r" (val)); 550 } 551 552 static inline void write_cr3(ulong val) 553 { 554 int vector = write_cr3_safe(val); 555 556 assert_msg(!vector, "Unexpected fault '%d' writing CR3 = %lx", 557 vector, val); 558 } 559 560 static inline ulong read_cr3(void) 561 { 562 ulong val; 563 asm volatile ("mov %%cr3, %0" : "=r"(val) : : "memory"); 564 return val; 565 } 566 567 static inline void update_cr3(void *cr3) 568 { 569 write_cr3((ulong)cr3); 570 } 571 572 static inline int write_cr4_safe(ulong val) 573 { 574 return asm_safe("mov %0,%%cr4", "r" (val)); 575 } 576 577 static inline void write_cr4(ulong val) 578 { 579 int vector = write_cr4_safe(val); 580 581 assert_msg(!vector, "Unexpected fault '%d' writing CR4 = %lx", 582 vector, val); 583 } 584 585 static inline ulong read_cr4(void) 586 { 587 ulong val; 588 asm volatile ("mov %%cr4, %0" : "=r"(val) : : "memory"); 589 return val; 590 } 591 592 static inline void write_cr8(ulong val) 593 { 594 asm volatile ("mov %0, %%cr8" : : "r"(val) : "memory"); 595 } 596 597 static inline ulong read_cr8(void) 598 { 599 ulong val; 600 asm volatile ("mov %%cr8, %0" : "=r"(val) : : "memory"); 601 return val; 602 } 603 604 static inline void lgdt(const struct descriptor_table_ptr *ptr) 605 { 606 asm volatile ("lgdt %0" : : "m"(*ptr)); 607 } 608 609 static inline int lgdt_safe(const struct descriptor_table_ptr *ptr) 610 { 611 return asm_safe("lgdt %0", "m"(*ptr)); 612 } 613 614 static inline int lgdt_fep_safe(const struct descriptor_table_ptr *ptr) 615 { 616 return asm_fep_safe("lgdt %0", "m"(*ptr)); 617 } 618 619 static inline void sgdt(struct descriptor_table_ptr *ptr) 620 { 621 asm volatile ("sgdt %0" : "=m"(*ptr)); 622 } 623 624 static inline void lidt(const struct descriptor_table_ptr *ptr) 625 { 626 asm volatile ("lidt %0" : : "m"(*ptr)); 627 } 628 629 static inline int lidt_safe(const struct descriptor_table_ptr *ptr) 630 { 631 return asm_safe("lidt %0", "m"(*ptr)); 632 } 633 634 static inline int lidt_fep_safe(const struct descriptor_table_ptr *ptr) 635 { 636 return asm_fep_safe("lidt %0", "m"(*ptr)); 637 } 638 639 static inline void sidt(struct descriptor_table_ptr *ptr) 640 { 641 asm volatile ("sidt %0" : "=m"(*ptr)); 642 } 643 644 static inline void lldt(u16 val) 645 { 646 asm volatile ("lldt %0" : : "rm"(val)); 647 } 648 649 static inline int lldt_safe(u16 val) 650 { 651 return asm_safe("lldt %0", "rm"(val)); 652 } 653 654 static inline int lldt_fep_safe(u16 val) 655 { 656 return asm_safe("lldt %0", "rm"(val)); 657 } 658 659 static inline u16 sldt(void) 660 { 661 u16 val; 662 asm volatile ("sldt %0" : "=rm"(val)); 663 return val; 664 } 665 666 static inline void ltr(u16 val) 667 { 668 asm volatile ("ltr %0" : : "rm"(val)); 669 } 670 671 static inline int ltr_safe(u16 val) 672 { 673 return asm_safe("ltr %0", "rm"(val)); 674 } 675 676 static inline int ltr_fep_safe(u16 val) 677 { 678 return asm_safe("ltr %0", "rm"(val)); 679 } 680 681 static inline u16 str(void) 682 { 683 u16 val; 684 asm volatile ("str %0" : "=rm"(val)); 685 return val; 686 } 687 688 static inline void write_dr0(void *val) 689 { 690 asm volatile ("mov %0, %%dr0" : : "r"(val) : "memory"); 691 } 692 693 static inline void write_dr1(void *val) 694 { 695 asm volatile ("mov %0, %%dr1" : : "r"(val) : "memory"); 696 } 697 698 static inline void write_dr2(void *val) 699 { 700 asm volatile ("mov %0, %%dr2" : : "r"(val) : "memory"); 701 } 702 703 static inline void write_dr3(void *val) 704 { 705 asm volatile ("mov %0, %%dr3" : : "r"(val) : "memory"); 706 } 707 708 static inline void write_dr6(ulong val) 709 { 710 asm volatile ("mov %0, %%dr6" : : "r"(val) : "memory"); 711 } 712 713 static inline ulong read_dr6(void) 714 { 715 ulong val; 716 asm volatile ("mov %%dr6, %0" : "=r"(val)); 717 return val; 718 } 719 720 static inline void write_dr7(ulong val) 721 { 722 asm volatile ("mov %0, %%dr7" : : "r"(val) : "memory"); 723 } 724 725 static inline ulong read_dr7(void) 726 { 727 ulong val; 728 asm volatile ("mov %%dr7, %0" : "=r"(val)); 729 return val; 730 } 731 732 static inline void pause(void) 733 { 734 asm volatile ("pause"); 735 } 736 737 static inline void cli(void) 738 { 739 asm volatile ("cli"); 740 } 741 742 /* 743 * See also safe_halt(). 744 */ 745 static inline void sti(void) 746 { 747 asm volatile ("sti"); 748 } 749 750 /* 751 * Enable interrupts and ensure that interrupts are evaluated upon return from 752 * this function, i.e. execute a nop to consume the STi interrupt shadow. 753 */ 754 static inline void sti_nop(void) 755 { 756 asm volatile ("sti; nop"); 757 } 758 759 /* 760 * Enable interrupts for one instruction (nop), to allow the CPU to process all 761 * interrupts that are already pending. 762 */ 763 static inline void sti_nop_cli(void) 764 { 765 asm volatile ("sti; nop; cli"); 766 } 767 768 static inline unsigned long long rdrand(void) 769 { 770 long long r; 771 772 asm volatile("rdrand %0\n\t" 773 "jc 1f\n\t" 774 "mov $0, %0\n\t" 775 "1:\n\t" : "=r" (r)); 776 return r; 777 } 778 779 static inline unsigned long long rdtsc(void) 780 { 781 long long r; 782 783 #ifdef __x86_64__ 784 unsigned a, d; 785 786 asm volatile ("rdtsc" : "=a"(a), "=d"(d)); 787 r = a | ((long long)d << 32); 788 #else 789 asm volatile ("rdtsc" : "=A"(r)); 790 #endif 791 return r; 792 } 793 794 /* 795 * Per the advice in the SDM, volume 2, the sequence "mfence; lfence" 796 * executed immediately before rdtsc ensures that rdtsc will be 797 * executed only after all previous instructions have executed and all 798 * previous loads and stores are globally visible. In addition, the 799 * lfence immediately after rdtsc ensures that rdtsc will be executed 800 * prior to the execution of any subsequent instruction. 801 */ 802 static inline unsigned long long fenced_rdtsc(void) 803 { 804 unsigned long long tsc; 805 806 #ifdef __x86_64__ 807 unsigned int eax, edx; 808 809 asm volatile ("mfence; lfence; rdtsc; lfence" : "=a"(eax), "=d"(edx)); 810 tsc = eax | ((unsigned long long)edx << 32); 811 #else 812 asm volatile ("mfence; lfence; rdtsc; lfence" : "=A"(tsc)); 813 #endif 814 return tsc; 815 } 816 817 static inline unsigned long long rdtscp(u32 *aux) 818 { 819 long long r; 820 821 #ifdef __x86_64__ 822 unsigned a, d; 823 824 asm volatile ("rdtscp" : "=a"(a), "=d"(d), "=c"(*aux)); 825 r = a | ((long long)d << 32); 826 #else 827 asm volatile ("rdtscp" : "=A"(r), "=c"(*aux)); 828 #endif 829 return r; 830 } 831 832 static inline void wrtsc(u64 tsc) 833 { 834 wrmsr(MSR_IA32_TSC, tsc); 835 } 836 837 838 static inline void invlpg(volatile void *va) 839 { 840 asm volatile("invlpg (%0)" ::"r" (va) : "memory"); 841 } 842 843 struct invpcid_desc { 844 u64 pcid : 12; 845 u64 rsv : 52; 846 u64 addr : 64; 847 }; 848 849 static inline int invpcid_safe(unsigned long type, struct invpcid_desc *desc) 850 { 851 /* invpcid (%rax), %rbx */ 852 return asm_safe(".byte 0x66,0x0f,0x38,0x82,0x18", "a" (desc), "b" (type)); 853 } 854 855 /* 856 * Execute HLT in an STI interrupt shadow to ensure that a pending IRQ that's 857 * intended to be a wake event arrives *after* HLT is executed. Modern CPUs, 858 * except for a few oddballs that KVM is unlikely to run on, block IRQs for one 859 * instruction after STI, *if* RFLAGS.IF=0 before STI. Note, Intel CPUs may 860 * block other events beyond regular IRQs, e.g. may block NMIs and SMIs too. 861 */ 862 static inline void safe_halt(void) 863 { 864 asm volatile("sti; hlt"); 865 } 866 867 static inline u32 read_pkru(void) 868 { 869 unsigned int eax, edx; 870 unsigned int ecx = 0; 871 unsigned int pkru; 872 873 asm volatile(".byte 0x0f,0x01,0xee\n\t" 874 : "=a" (eax), "=d" (edx) 875 : "c" (ecx)); 876 pkru = eax; 877 return pkru; 878 } 879 880 static inline void write_pkru(u32 pkru) 881 { 882 unsigned int eax = pkru; 883 unsigned int ecx = 0; 884 unsigned int edx = 0; 885 886 asm volatile(".byte 0x0f,0x01,0xef\n\t" 887 : : "a" (eax), "c" (ecx), "d" (edx)); 888 } 889 890 static inline bool is_canonical(u64 addr) 891 { 892 int va_width = (raw_cpuid(0x80000008, 0).a & 0xff00) >> 8; 893 int shift_amt = 64 - va_width; 894 895 return (s64)(addr << shift_amt) >> shift_amt == addr; 896 } 897 898 static inline void clear_bit(int bit, u8 *addr) 899 { 900 __asm__ __volatile__("lock; btr %1, %0" 901 : "+m" (*addr) : "Ir" (bit) : "cc", "memory"); 902 } 903 904 static inline void set_bit(int bit, u8 *addr) 905 { 906 __asm__ __volatile__("lock; bts %1, %0" 907 : "+m" (*addr) : "Ir" (bit) : "cc", "memory"); 908 } 909 910 static inline void flush_tlb(void) 911 { 912 ulong cr4; 913 914 cr4 = read_cr4(); 915 write_cr4(cr4 ^ X86_CR4_PGE); 916 write_cr4(cr4); 917 } 918 919 static inline void generate_non_canonical_gp(void) 920 { 921 *(volatile u64 *)NONCANONICAL = 0; 922 } 923 924 static inline void generate_ud(void) 925 { 926 asm volatile ("ud2"); 927 } 928 929 static inline void generate_de(void) 930 { 931 asm volatile ( 932 "xor %%eax, %%eax\n\t" 933 "xor %%ebx, %%ebx\n\t" 934 "xor %%edx, %%edx\n\t" 935 "idiv %%ebx\n\t" 936 ::: "eax", "ebx", "edx"); 937 } 938 939 static inline void generate_bp(void) 940 { 941 asm volatile ("int3"); 942 } 943 944 static inline void generate_single_step_db(void) 945 { 946 write_rflags(read_rflags() | X86_EFLAGS_TF); 947 asm volatile("nop"); 948 } 949 950 static inline uint64_t generate_usermode_ac(void) 951 { 952 /* 953 * Trigger an #AC by writing 8 bytes to a 4-byte aligned address. 954 * Disclaimer: It is assumed that the stack pointer is aligned 955 * on a 16-byte boundary as x86_64 stacks should be. 956 */ 957 asm volatile("movq $0, -0x4(%rsp)"); 958 959 return 0; 960 } 961 962 /* 963 * Switch from 64-bit to 32-bit mode and generate #OF via INTO. Note, if RIP 964 * or RSP holds a 64-bit value, this helper will NOT generate #OF. 965 */ 966 static inline void generate_of(void) 967 { 968 struct far_pointer32 fp = { 969 .offset = (uintptr_t)&&into, 970 .selector = KERNEL_CS32, 971 }; 972 uintptr_t rsp; 973 974 asm volatile ("mov %%rsp, %0" : "=r"(rsp)); 975 976 if (fp.offset != (uintptr_t)&&into) { 977 printf("Code address too high.\n"); 978 return; 979 } 980 if ((u32)rsp != rsp) { 981 printf("Stack address too high.\n"); 982 return; 983 } 984 985 asm goto ("lcall *%0" : : "m" (fp) : "rax" : into); 986 return; 987 into: 988 asm volatile (".code32;" 989 "movl $0x7fffffff, %eax;" 990 "addl %eax, %eax;" 991 "into;" 992 "lret;" 993 ".code64"); 994 __builtin_unreachable(); 995 } 996 997 static inline void fnop(void) 998 { 999 asm volatile("fnop"); 1000 } 1001 1002 /* If CR0.TS is set in L2, #NM is generated. */ 1003 static inline void generate_cr0_ts_nm(void) 1004 { 1005 write_cr0((read_cr0() & ~X86_CR0_EM) | X86_CR0_TS); 1006 fnop(); 1007 } 1008 1009 /* If CR0.TS is cleared and CR0.EM is set, #NM is generated. */ 1010 static inline void generate_cr0_em_nm(void) 1011 { 1012 write_cr0((read_cr0() & ~X86_CR0_TS) | X86_CR0_EM); 1013 fnop(); 1014 } 1015 1016 #endif 1017