1 #ifndef _X86_PROCESSOR_H_ 2 #define _X86_PROCESSOR_H_ 3 4 #include "libcflat.h" 5 #include "desc.h" 6 #include "msr.h" 7 #include <bitops.h> 8 #include <stdint.h> 9 10 #define CANONICAL_48_VAL 0xffffaaaaaaaaaaaaull 11 #define CANONICAL_57_VAL 0xffaaaaaaaaaaaaaaull 12 #define NONCANONICAL 0xaaaaaaaaaaaaaaaaull 13 14 #define LAM57_MASK GENMASK_ULL(62, 57) 15 #define LAM48_MASK GENMASK_ULL(62, 48) 16 17 /* 18 * Get a linear address by combining @addr with a non-canonical pattern in the 19 * @mask bits. 20 */ 21 static inline u64 get_non_canonical(u64 addr, u64 mask) 22 { 23 return (addr & ~mask) | (NONCANONICAL & mask); 24 } 25 26 #ifdef __x86_64__ 27 # define R "r" 28 # define W "q" 29 # define S "8" 30 #else 31 # define R "e" 32 # define W "l" 33 # define S "4" 34 #endif 35 36 #define DE_VECTOR 0 37 #define DB_VECTOR 1 38 #define NMI_VECTOR 2 39 #define BP_VECTOR 3 40 #define OF_VECTOR 4 41 #define BR_VECTOR 5 42 #define UD_VECTOR 6 43 #define NM_VECTOR 7 44 #define DF_VECTOR 8 45 #define TS_VECTOR 10 46 #define NP_VECTOR 11 47 #define SS_VECTOR 12 48 #define GP_VECTOR 13 49 #define PF_VECTOR 14 50 #define MF_VECTOR 16 51 #define AC_VECTOR 17 52 #define MC_VECTOR 18 53 #define XM_VECTOR 19 54 #define XF_VECTOR XM_VECTOR /* AMD */ 55 #define VE_VECTOR 20 /* Intel only */ 56 #define CP_VECTOR 21 57 #define HV_VECTOR 28 /* AMD only */ 58 #define VC_VECTOR 29 /* AMD only */ 59 #define SX_VECTOR 30 /* AMD only */ 60 61 #define X86_CR0_PE_BIT (0) 62 #define X86_CR0_PE BIT(X86_CR0_PE_BIT) 63 #define X86_CR0_MP_BIT (1) 64 #define X86_CR0_MP BIT(X86_CR0_MP_BIT) 65 #define X86_CR0_EM_BIT (2) 66 #define X86_CR0_EM BIT(X86_CR0_EM_BIT) 67 #define X86_CR0_TS_BIT (3) 68 #define X86_CR0_TS BIT(X86_CR0_TS_BIT) 69 #define X86_CR0_ET_BIT (4) 70 #define X86_CR0_ET BIT(X86_CR0_ET_BIT) 71 #define X86_CR0_NE_BIT (5) 72 #define X86_CR0_NE BIT(X86_CR0_NE_BIT) 73 #define X86_CR0_WP_BIT (16) 74 #define X86_CR0_WP BIT(X86_CR0_WP_BIT) 75 #define X86_CR0_AM_BIT (18) 76 #define X86_CR0_AM BIT(X86_CR0_AM_BIT) 77 #define X86_CR0_NW_BIT (29) 78 #define X86_CR0_NW BIT(X86_CR0_NW_BIT) 79 #define X86_CR0_CD_BIT (30) 80 #define X86_CR0_CD BIT(X86_CR0_CD_BIT) 81 #define X86_CR0_PG_BIT (31) 82 #define X86_CR0_PG BIT(X86_CR0_PG_BIT) 83 84 #define X86_CR3_PCID_MASK GENMASK(11, 0) 85 #define X86_CR3_LAM_U57_BIT (61) 86 #define X86_CR3_LAM_U57 BIT_ULL(X86_CR3_LAM_U57_BIT) 87 #define X86_CR3_LAM_U48_BIT (62) 88 #define X86_CR3_LAM_U48 BIT_ULL(X86_CR3_LAM_U48_BIT) 89 90 #define X86_CR4_VME_BIT (0) 91 #define X86_CR4_VME BIT(X86_CR4_VME_BIT) 92 #define X86_CR4_PVI_BIT (1) 93 #define X86_CR4_PVI BIT(X86_CR4_PVI_BIT) 94 #define X86_CR4_TSD_BIT (2) 95 #define X86_CR4_TSD BIT(X86_CR4_TSD_BIT) 96 #define X86_CR4_DE_BIT (3) 97 #define X86_CR4_DE BIT(X86_CR4_DE_BIT) 98 #define X86_CR4_PSE_BIT (4) 99 #define X86_CR4_PSE BIT(X86_CR4_PSE_BIT) 100 #define X86_CR4_PAE_BIT (5) 101 #define X86_CR4_PAE BIT(X86_CR4_PAE_BIT) 102 #define X86_CR4_MCE_BIT (6) 103 #define X86_CR4_MCE BIT(X86_CR4_MCE_BIT) 104 #define X86_CR4_PGE_BIT (7) 105 #define X86_CR4_PGE BIT(X86_CR4_PGE_BIT) 106 #define X86_CR4_PCE_BIT (8) 107 #define X86_CR4_PCE BIT(X86_CR4_PCE_BIT) 108 #define X86_CR4_OSFXSR_BIT (9) 109 #define X86_CR4_OSFXSR BIT(X86_CR4_OSFXSR_BIT) 110 #define X86_CR4_OSXMMEXCPT_BIT (10) 111 #define X86_CR4_OSXMMEXCPT BIT(X86_CR4_OSXMMEXCPT_BIT) 112 #define X86_CR4_UMIP_BIT (11) 113 #define X86_CR4_UMIP BIT(X86_CR4_UMIP_BIT) 114 #define X86_CR4_LA57_BIT (12) 115 #define X86_CR4_LA57 BIT(X86_CR4_LA57_BIT) 116 #define X86_CR4_VMXE_BIT (13) 117 #define X86_CR4_VMXE BIT(X86_CR4_VMXE_BIT) 118 #define X86_CR4_SMXE_BIT (14) 119 #define X86_CR4_SMXE BIT(X86_CR4_SMXE_BIT) 120 /* UNUSED (15) */ 121 #define X86_CR4_FSGSBASE_BIT (16) 122 #define X86_CR4_FSGSBASE BIT(X86_CR4_FSGSBASE_BIT) 123 #define X86_CR4_PCIDE_BIT (17) 124 #define X86_CR4_PCIDE BIT(X86_CR4_PCIDE_BIT) 125 #define X86_CR4_OSXSAVE_BIT (18) 126 #define X86_CR4_OSXSAVE BIT(X86_CR4_OSXSAVE_BIT) 127 #define X86_CR4_KL_BIT (19) 128 #define X86_CR4_KL BIT(X86_CR4_KL_BIT) 129 #define X86_CR4_SMEP_BIT (20) 130 #define X86_CR4_SMEP BIT(X86_CR4_SMEP_BIT) 131 #define X86_CR4_SMAP_BIT (21) 132 #define X86_CR4_SMAP BIT(X86_CR4_SMAP_BIT) 133 #define X86_CR4_PKE_BIT (22) 134 #define X86_CR4_PKE BIT(X86_CR4_PKE_BIT) 135 #define X86_CR4_CET_BIT (23) 136 #define X86_CR4_CET BIT(X86_CR4_CET_BIT) 137 #define X86_CR4_PKS_BIT (24) 138 #define X86_CR4_PKS BIT(X86_CR4_PKS_BIT) 139 #define X86_CR4_LAM_SUP_BIT (28) 140 #define X86_CR4_LAM_SUP BIT(X86_CR4_LAM_SUP_BIT) 141 142 #define X86_EFLAGS_CF_BIT (0) 143 #define X86_EFLAGS_CF BIT(X86_EFLAGS_CF_BIT) 144 #define X86_EFLAGS_FIXED_BIT (1) 145 #define X86_EFLAGS_FIXED BIT(X86_EFLAGS_FIXED_BIT) 146 #define X86_EFLAGS_PF_BIT (2) 147 #define X86_EFLAGS_PF BIT(X86_EFLAGS_PF_BIT) 148 /* RESERVED 0 (3) */ 149 #define X86_EFLAGS_AF_BIT (4) 150 #define X86_EFLAGS_AF BIT(X86_EFLAGS_AF_BIT) 151 /* RESERVED 0 (5) */ 152 #define X86_EFLAGS_ZF_BIT (6) 153 #define X86_EFLAGS_ZF BIT(X86_EFLAGS_ZF_BIT) 154 #define X86_EFLAGS_SF_BIT (7) 155 #define X86_EFLAGS_SF BIT(X86_EFLAGS_SF_BIT) 156 #define X86_EFLAGS_TF_BIT (8) 157 #define X86_EFLAGS_TF BIT(X86_EFLAGS_TF_BIT) 158 #define X86_EFLAGS_IF_BIT (9) 159 #define X86_EFLAGS_IF BIT(X86_EFLAGS_IF_BIT) 160 #define X86_EFLAGS_DF_BIT (10) 161 #define X86_EFLAGS_DF BIT(X86_EFLAGS_DF_BIT) 162 #define X86_EFLAGS_OF_BIT (11) 163 #define X86_EFLAGS_OF BIT(X86_EFLAGS_OF_BIT) 164 #define X86_EFLAGS_IOPL GENMASK(13, 12) 165 #define X86_EFLAGS_NT_BIT (14) 166 #define X86_EFLAGS_NT BIT(X86_EFLAGS_NT_BIT) 167 /* RESERVED 0 (15) */ 168 #define X86_EFLAGS_RF_BIT (16) 169 #define X86_EFLAGS_RF BIT(X86_EFLAGS_RF_BIT) 170 #define X86_EFLAGS_VM_BIT (17) 171 #define X86_EFLAGS_VM BIT(X86_EFLAGS_VM_BIT) 172 #define X86_EFLAGS_AC_BIT (18) 173 #define X86_EFLAGS_AC BIT(X86_EFLAGS_AC_BIT) 174 #define X86_EFLAGS_VIF_BIT (19) 175 #define X86_EFLAGS_VIF BIT(X86_EFLAGS_VIF_BIT) 176 #define X86_EFLAGS_VIP_BIT (20) 177 #define X86_EFLAGS_VIP BIT(X86_EFLAGS_VIP_BIT) 178 #define X86_EFLAGS_ID_BIT (21) 179 #define X86_EFLAGS_ID BIT(X86_EFLAGS_ID_BIT) 180 181 #define X86_EFLAGS_ALU (X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | \ 182 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF) 183 184 185 /* 186 * CPU features 187 */ 188 189 enum cpuid_output_regs { 190 EAX, 191 EBX, 192 ECX, 193 EDX 194 }; 195 196 struct cpuid { u32 a, b, c, d; }; 197 198 static inline struct cpuid raw_cpuid(u32 function, u32 index) 199 { 200 struct cpuid r; 201 asm volatile ("cpuid" 202 : "=a"(r.a), "=b"(r.b), "=c"(r.c), "=d"(r.d) 203 : "0"(function), "2"(index)); 204 return r; 205 } 206 207 static inline struct cpuid cpuid_indexed(u32 function, u32 index) 208 { 209 u32 level = raw_cpuid(function & 0xf0000000, 0).a; 210 if (level < function) 211 return (struct cpuid) { 0, 0, 0, 0 }; 212 return raw_cpuid(function, index); 213 } 214 215 static inline struct cpuid cpuid(u32 function) 216 { 217 return cpuid_indexed(function, 0); 218 } 219 220 static inline u8 cpuid_maxphyaddr(void) 221 { 222 if (raw_cpuid(0x80000000, 0).a < 0x80000008) 223 return 36; 224 return raw_cpuid(0x80000008, 0).a & 0xff; 225 } 226 227 static inline bool is_intel(void) 228 { 229 struct cpuid c = cpuid(0); 230 u32 name[4] = {c.b, c.d, c.c }; 231 232 return strcmp((char *)name, "GenuineIntel") == 0; 233 } 234 235 #define CPUID(a, b, c, d) ((((unsigned long long) a) << 32) | (b << 16) | \ 236 (c << 8) | d) 237 238 /* 239 * Each X86_FEATURE_XXX definition is 64-bit and contains the following 240 * CPUID meta-data: 241 * 242 * [63:32] : input value for EAX 243 * [31:16] : input value for ECX 244 * [15:8] : output register 245 * [7:0] : bit position in output register 246 */ 247 248 /* 249 * Basic Leafs, a.k.a. Intel defined 250 */ 251 #define X86_FEATURE_MWAIT (CPUID(0x1, 0, ECX, 3)) 252 #define X86_FEATURE_VMX (CPUID(0x1, 0, ECX, 5)) 253 #define X86_FEATURE_PDCM (CPUID(0x1, 0, ECX, 15)) 254 #define X86_FEATURE_PCID (CPUID(0x1, 0, ECX, 17)) 255 #define X86_FEATURE_X2APIC (CPUID(0x1, 0, ECX, 21)) 256 #define X86_FEATURE_MOVBE (CPUID(0x1, 0, ECX, 22)) 257 #define X86_FEATURE_TSC_DEADLINE_TIMER (CPUID(0x1, 0, ECX, 24)) 258 #define X86_FEATURE_XSAVE (CPUID(0x1, 0, ECX, 26)) 259 #define X86_FEATURE_OSXSAVE (CPUID(0x1, 0, ECX, 27)) 260 #define X86_FEATURE_RDRAND (CPUID(0x1, 0, ECX, 30)) 261 #define X86_FEATURE_MCE (CPUID(0x1, 0, EDX, 7)) 262 #define X86_FEATURE_APIC (CPUID(0x1, 0, EDX, 9)) 263 #define X86_FEATURE_CLFLUSH (CPUID(0x1, 0, EDX, 19)) 264 #define X86_FEATURE_DS (CPUID(0x1, 0, EDX, 21)) 265 #define X86_FEATURE_XMM (CPUID(0x1, 0, EDX, 25)) 266 #define X86_FEATURE_XMM2 (CPUID(0x1, 0, EDX, 26)) 267 #define X86_FEATURE_TSC_ADJUST (CPUID(0x7, 0, EBX, 1)) 268 #define X86_FEATURE_HLE (CPUID(0x7, 0, EBX, 4)) 269 #define X86_FEATURE_SMEP (CPUID(0x7, 0, EBX, 7)) 270 #define X86_FEATURE_INVPCID (CPUID(0x7, 0, EBX, 10)) 271 #define X86_FEATURE_RTM (CPUID(0x7, 0, EBX, 11)) 272 #define X86_FEATURE_SMAP (CPUID(0x7, 0, EBX, 20)) 273 #define X86_FEATURE_PCOMMIT (CPUID(0x7, 0, EBX, 22)) 274 #define X86_FEATURE_CLFLUSHOPT (CPUID(0x7, 0, EBX, 23)) 275 #define X86_FEATURE_CLWB (CPUID(0x7, 0, EBX, 24)) 276 #define X86_FEATURE_INTEL_PT (CPUID(0x7, 0, EBX, 25)) 277 #define X86_FEATURE_UMIP (CPUID(0x7, 0, ECX, 2)) 278 #define X86_FEATURE_PKU (CPUID(0x7, 0, ECX, 3)) 279 #define X86_FEATURE_LA57 (CPUID(0x7, 0, ECX, 16)) 280 #define X86_FEATURE_RDPID (CPUID(0x7, 0, ECX, 22)) 281 #define X86_FEATURE_SHSTK (CPUID(0x7, 0, ECX, 7)) 282 #define X86_FEATURE_IBT (CPUID(0x7, 0, EDX, 20)) 283 #define X86_FEATURE_SPEC_CTRL (CPUID(0x7, 0, EDX, 26)) 284 #define X86_FEATURE_FLUSH_L1D (CPUID(0x7, 0, EDX, 28)) 285 #define X86_FEATURE_ARCH_CAPABILITIES (CPUID(0x7, 0, EDX, 29)) 286 #define X86_FEATURE_PKS (CPUID(0x7, 0, ECX, 31)) 287 #define X86_FEATURE_LAM (CPUID(0x7, 1, EAX, 26)) 288 289 /* 290 * KVM defined leafs 291 */ 292 #define KVM_FEATURE_ASYNC_PF (CPUID(0x40000001, 0, EAX, 4)) 293 #define KVM_FEATURE_ASYNC_PF_INT (CPUID(0x40000001, 0, EAX, 14)) 294 295 /* 296 * Extended Leafs, a.k.a. AMD defined 297 */ 298 #define X86_FEATURE_SVM (CPUID(0x80000001, 0, ECX, 2)) 299 #define X86_FEATURE_PERFCTR_CORE (CPUID(0x80000001, 0, ECX, 23)) 300 #define X86_FEATURE_NX (CPUID(0x80000001, 0, EDX, 20)) 301 #define X86_FEATURE_GBPAGES (CPUID(0x80000001, 0, EDX, 26)) 302 #define X86_FEATURE_RDTSCP (CPUID(0x80000001, 0, EDX, 27)) 303 #define X86_FEATURE_LM (CPUID(0x80000001, 0, EDX, 29)) 304 #define X86_FEATURE_RDPRU (CPUID(0x80000008, 0, EBX, 4)) 305 #define X86_FEATURE_AMD_IBPB (CPUID(0x80000008, 0, EBX, 12)) 306 #define X86_FEATURE_NPT (CPUID(0x8000000A, 0, EDX, 0)) 307 #define X86_FEATURE_LBRV (CPUID(0x8000000A, 0, EDX, 1)) 308 #define X86_FEATURE_NRIPS (CPUID(0x8000000A, 0, EDX, 3)) 309 #define X86_FEATURE_TSCRATEMSR (CPUID(0x8000000A, 0, EDX, 4)) 310 #define X86_FEATURE_PAUSEFILTER (CPUID(0x8000000A, 0, EDX, 10)) 311 #define X86_FEATURE_PFTHRESHOLD (CPUID(0x8000000A, 0, EDX, 12)) 312 #define X86_FEATURE_VGIF (CPUID(0x8000000A, 0, EDX, 16)) 313 #define X86_FEATURE_VNMI (CPUID(0x8000000A, 0, EDX, 25)) 314 #define X86_FEATURE_AMD_PMU_V2 (CPUID(0x80000022, 0, EAX, 0)) 315 316 static inline bool this_cpu_has(u64 feature) 317 { 318 u32 input_eax = feature >> 32; 319 u32 input_ecx = (feature >> 16) & 0xffff; 320 u32 output_reg = (feature >> 8) & 0xff; 321 u8 bit = feature & 0xff; 322 struct cpuid c; 323 u32 *tmp; 324 325 c = cpuid_indexed(input_eax, input_ecx); 326 tmp = (u32 *)&c; 327 328 return ((*(tmp + (output_reg % 32))) & (1 << bit)); 329 } 330 331 struct far_pointer32 { 332 u32 offset; 333 u16 selector; 334 } __attribute__((packed)); 335 336 struct descriptor_table_ptr { 337 u16 limit; 338 ulong base; 339 } __attribute__((packed)); 340 341 static inline void clac(void) 342 { 343 asm volatile (".byte 0x0f, 0x01, 0xca" : : : "memory"); 344 } 345 346 static inline void stac(void) 347 { 348 asm volatile (".byte 0x0f, 0x01, 0xcb" : : : "memory"); 349 } 350 351 static inline u16 read_cs(void) 352 { 353 unsigned val; 354 355 asm volatile ("mov %%cs, %0" : "=mr"(val)); 356 return val; 357 } 358 359 static inline u16 read_ds(void) 360 { 361 unsigned val; 362 363 asm volatile ("mov %%ds, %0" : "=mr"(val)); 364 return val; 365 } 366 367 static inline u16 read_es(void) 368 { 369 unsigned val; 370 371 asm volatile ("mov %%es, %0" : "=mr"(val)); 372 return val; 373 } 374 375 static inline u16 read_ss(void) 376 { 377 unsigned val; 378 379 asm volatile ("mov %%ss, %0" : "=mr"(val)); 380 return val; 381 } 382 383 static inline u16 read_fs(void) 384 { 385 unsigned val; 386 387 asm volatile ("mov %%fs, %0" : "=mr"(val)); 388 return val; 389 } 390 391 static inline u16 read_gs(void) 392 { 393 unsigned val; 394 395 asm volatile ("mov %%gs, %0" : "=mr"(val)); 396 return val; 397 } 398 399 static inline unsigned long read_rflags(void) 400 { 401 unsigned long f; 402 asm volatile ("pushf; pop %0\n\t" : "=rm"(f)); 403 return f; 404 } 405 406 static inline void write_ds(unsigned val) 407 { 408 asm volatile ("mov %0, %%ds" : : "rm"(val) : "memory"); 409 } 410 411 static inline void write_es(unsigned val) 412 { 413 asm volatile ("mov %0, %%es" : : "rm"(val) : "memory"); 414 } 415 416 static inline void write_ss(unsigned val) 417 { 418 asm volatile ("mov %0, %%ss" : : "rm"(val) : "memory"); 419 } 420 421 static inline void write_fs(unsigned val) 422 { 423 asm volatile ("mov %0, %%fs" : : "rm"(val) : "memory"); 424 } 425 426 static inline void write_gs(unsigned val) 427 { 428 asm volatile ("mov %0, %%gs" : : "rm"(val) : "memory"); 429 } 430 431 static inline void write_rflags(unsigned long f) 432 { 433 asm volatile ("push %0; popf\n\t" : : "rm"(f)); 434 } 435 436 static inline void set_iopl(int iopl) 437 { 438 unsigned long flags = read_rflags() & ~X86_EFLAGS_IOPL; 439 flags |= iopl * (X86_EFLAGS_IOPL / 3); 440 write_rflags(flags); 441 } 442 443 /* 444 * Don't use the safe variants for rdmsr() or wrmsr(). The exception fixup 445 * infrastructure uses per-CPU data and thus consumes GS.base. Various tests 446 * temporarily modify MSR_GS_BASE and will explode when trying to determine 447 * whether or not RDMSR/WRMSR faulted. 448 */ 449 static inline u64 rdmsr(u32 index) 450 { 451 u32 a, d; 452 asm volatile ("rdmsr" : "=a"(a), "=d"(d) : "c"(index) : "memory"); 453 return a | ((u64)d << 32); 454 } 455 456 static inline void wrmsr(u32 index, u64 val) 457 { 458 u32 a = val, d = val >> 32; 459 asm volatile ("wrmsr" : : "a"(a), "d"(d), "c"(index) : "memory"); 460 } 461 462 #define __rdreg64_safe(fep, insn, index, val) \ 463 ({ \ 464 uint32_t a, d; \ 465 int vector; \ 466 \ 467 vector = __asm_safe_out2(fep, insn, "=a"(a), "=d"(d), "c"(index));\ 468 \ 469 if (vector) \ 470 *(val) = 0; \ 471 else \ 472 *(val) = (uint64_t)a | ((uint64_t)d << 32); \ 473 vector; \ 474 }) 475 476 #define rdreg64_safe(insn, index, val) \ 477 __rdreg64_safe("", insn, index, val) 478 479 #define __wrreg64_safe(fep, insn, index, val) \ 480 ({ \ 481 uint32_t eax = (val), edx = (val) >> 32; \ 482 \ 483 __asm_safe(fep, insn, "a" (eax), "d" (edx), "c" (index)); \ 484 }) 485 486 #define wrreg64_safe(insn, index, val) \ 487 __wrreg64_safe("", insn, index, val) 488 489 static inline int rdmsr_safe(u32 index, uint64_t *val) 490 { 491 return rdreg64_safe("rdmsr", index, val); 492 } 493 494 static inline int rdmsr_fep_safe(u32 index, uint64_t *val) 495 { 496 return __rdreg64_safe(KVM_FEP, "rdmsr", index, val); 497 } 498 499 static inline int wrmsr_safe(u32 index, u64 val) 500 { 501 return wrreg64_safe("wrmsr", index, val); 502 } 503 504 static inline int wrmsr_fep_safe(u32 index, u64 val) 505 { 506 return __wrreg64_safe(KVM_FEP, "wrmsr", index, val); 507 } 508 509 static inline int rdpmc_safe(u32 index, uint64_t *val) 510 { 511 return rdreg64_safe("rdpmc", index, val); 512 } 513 514 static inline uint64_t rdpmc(uint32_t index) 515 { 516 uint64_t val; 517 int vector = rdpmc_safe(index, &val); 518 519 assert_msg(!vector, "Unexpected %s on RDPMC(%" PRId32 ")", 520 exception_mnemonic(vector), index); 521 return val; 522 } 523 524 static inline int xgetbv_safe(u32 index, u64 *result) 525 { 526 return rdreg64_safe(".byte 0x0f,0x01,0xd0", index, result); 527 } 528 529 static inline int xsetbv_safe(u32 index, u64 value) 530 { 531 return wrreg64_safe(".byte 0x0f,0x01,0xd1", index, value); 532 } 533 534 static inline int write_cr0_safe(ulong val) 535 { 536 return asm_safe("mov %0,%%cr0", "r" (val)); 537 } 538 539 static inline void write_cr0(ulong val) 540 { 541 int vector = write_cr0_safe(val); 542 543 assert_msg(!vector, "Unexpected fault '%d' writing CR0 = %lx", 544 vector, val); 545 } 546 547 static inline ulong read_cr0(void) 548 { 549 ulong val; 550 asm volatile ("mov %%cr0, %0" : "=r"(val) : : "memory"); 551 return val; 552 } 553 554 static inline void write_cr2(ulong val) 555 { 556 asm volatile ("mov %0, %%cr2" : : "r"(val) : "memory"); 557 } 558 559 static inline ulong read_cr2(void) 560 { 561 ulong val; 562 asm volatile ("mov %%cr2, %0" : "=r"(val) : : "memory"); 563 return val; 564 } 565 566 static inline int write_cr3_safe(ulong val) 567 { 568 return asm_safe("mov %0,%%cr3", "r" (val)); 569 } 570 571 static inline void write_cr3(ulong val) 572 { 573 int vector = write_cr3_safe(val); 574 575 assert_msg(!vector, "Unexpected fault '%d' writing CR3 = %lx", 576 vector, val); 577 } 578 579 static inline ulong read_cr3(void) 580 { 581 ulong val; 582 asm volatile ("mov %%cr3, %0" : "=r"(val) : : "memory"); 583 return val; 584 } 585 586 static inline void update_cr3(void *cr3) 587 { 588 write_cr3((ulong)cr3); 589 } 590 591 static inline int write_cr4_safe(ulong val) 592 { 593 return asm_safe("mov %0,%%cr4", "r" (val)); 594 } 595 596 static inline void write_cr4(ulong val) 597 { 598 int vector = write_cr4_safe(val); 599 600 assert_msg(!vector, "Unexpected fault '%d' writing CR4 = %lx", 601 vector, val); 602 } 603 604 static inline ulong read_cr4(void) 605 { 606 ulong val; 607 asm volatile ("mov %%cr4, %0" : "=r"(val) : : "memory"); 608 return val; 609 } 610 611 static inline void write_cr8(ulong val) 612 { 613 asm volatile ("mov %0, %%cr8" : : "r"(val) : "memory"); 614 } 615 616 static inline ulong read_cr8(void) 617 { 618 ulong val; 619 asm volatile ("mov %%cr8, %0" : "=r"(val) : : "memory"); 620 return val; 621 } 622 623 static inline void lgdt(const struct descriptor_table_ptr *ptr) 624 { 625 asm volatile ("lgdt %0" : : "m"(*ptr)); 626 } 627 628 static inline int lgdt_safe(const struct descriptor_table_ptr *ptr) 629 { 630 return asm_safe("lgdt %0", "m"(*ptr)); 631 } 632 633 static inline int lgdt_fep_safe(const struct descriptor_table_ptr *ptr) 634 { 635 return asm_fep_safe("lgdt %0", "m"(*ptr)); 636 } 637 638 static inline void sgdt(struct descriptor_table_ptr *ptr) 639 { 640 asm volatile ("sgdt %0" : "=m"(*ptr)); 641 } 642 643 static inline void lidt(const struct descriptor_table_ptr *ptr) 644 { 645 asm volatile ("lidt %0" : : "m"(*ptr)); 646 } 647 648 static inline int lidt_safe(const struct descriptor_table_ptr *ptr) 649 { 650 return asm_safe("lidt %0", "m"(*ptr)); 651 } 652 653 static inline int lidt_fep_safe(const struct descriptor_table_ptr *ptr) 654 { 655 return asm_fep_safe("lidt %0", "m"(*ptr)); 656 } 657 658 static inline void sidt(struct descriptor_table_ptr *ptr) 659 { 660 asm volatile ("sidt %0" : "=m"(*ptr)); 661 } 662 663 static inline void lldt(u16 val) 664 { 665 asm volatile ("lldt %0" : : "rm"(val)); 666 } 667 668 static inline int lldt_safe(u16 val) 669 { 670 return asm_safe("lldt %0", "rm"(val)); 671 } 672 673 static inline int lldt_fep_safe(u16 val) 674 { 675 return asm_safe("lldt %0", "rm"(val)); 676 } 677 678 static inline u16 sldt(void) 679 { 680 u16 val; 681 asm volatile ("sldt %0" : "=rm"(val)); 682 return val; 683 } 684 685 static inline void ltr(u16 val) 686 { 687 asm volatile ("ltr %0" : : "rm"(val)); 688 } 689 690 static inline int ltr_safe(u16 val) 691 { 692 return asm_safe("ltr %0", "rm"(val)); 693 } 694 695 static inline int ltr_fep_safe(u16 val) 696 { 697 return asm_safe("ltr %0", "rm"(val)); 698 } 699 700 static inline u16 str(void) 701 { 702 u16 val; 703 asm volatile ("str %0" : "=rm"(val)); 704 return val; 705 } 706 707 static inline void write_dr0(void *val) 708 { 709 asm volatile ("mov %0, %%dr0" : : "r"(val) : "memory"); 710 } 711 712 static inline void write_dr1(void *val) 713 { 714 asm volatile ("mov %0, %%dr1" : : "r"(val) : "memory"); 715 } 716 717 static inline void write_dr2(void *val) 718 { 719 asm volatile ("mov %0, %%dr2" : : "r"(val) : "memory"); 720 } 721 722 static inline void write_dr3(void *val) 723 { 724 asm volatile ("mov %0, %%dr3" : : "r"(val) : "memory"); 725 } 726 727 static inline void write_dr6(ulong val) 728 { 729 asm volatile ("mov %0, %%dr6" : : "r"(val) : "memory"); 730 } 731 732 static inline ulong read_dr6(void) 733 { 734 ulong val; 735 asm volatile ("mov %%dr6, %0" : "=r"(val)); 736 return val; 737 } 738 739 static inline void write_dr7(ulong val) 740 { 741 asm volatile ("mov %0, %%dr7" : : "r"(val) : "memory"); 742 } 743 744 static inline ulong read_dr7(void) 745 { 746 ulong val; 747 asm volatile ("mov %%dr7, %0" : "=r"(val)); 748 return val; 749 } 750 751 static inline void pause(void) 752 { 753 asm volatile ("pause"); 754 } 755 756 static inline void cli(void) 757 { 758 asm volatile ("cli"); 759 } 760 761 /* 762 * See also safe_halt(). 763 */ 764 static inline void sti(void) 765 { 766 asm volatile ("sti"); 767 } 768 769 /* 770 * Enable interrupts and ensure that interrupts are evaluated upon return from 771 * this function, i.e. execute a nop to consume the STi interrupt shadow. 772 */ 773 static inline void sti_nop(void) 774 { 775 asm volatile ("sti; nop"); 776 } 777 778 /* 779 * Enable interrupts for one instruction (nop), to allow the CPU to process all 780 * interrupts that are already pending. 781 */ 782 static inline void sti_nop_cli(void) 783 { 784 asm volatile ("sti; nop; cli"); 785 } 786 787 static inline unsigned long long rdrand(void) 788 { 789 long long r; 790 791 asm volatile("rdrand %0\n\t" 792 "jc 1f\n\t" 793 "mov $0, %0\n\t" 794 "1:\n\t" : "=r" (r)); 795 return r; 796 } 797 798 static inline unsigned long long rdtsc(void) 799 { 800 long long r; 801 802 #ifdef __x86_64__ 803 unsigned a, d; 804 805 asm volatile ("rdtsc" : "=a"(a), "=d"(d)); 806 r = a | ((long long)d << 32); 807 #else 808 asm volatile ("rdtsc" : "=A"(r)); 809 #endif 810 return r; 811 } 812 813 /* 814 * Per the advice in the SDM, volume 2, the sequence "mfence; lfence" 815 * executed immediately before rdtsc ensures that rdtsc will be 816 * executed only after all previous instructions have executed and all 817 * previous loads and stores are globally visible. In addition, the 818 * lfence immediately after rdtsc ensures that rdtsc will be executed 819 * prior to the execution of any subsequent instruction. 820 */ 821 static inline unsigned long long fenced_rdtsc(void) 822 { 823 unsigned long long tsc; 824 825 #ifdef __x86_64__ 826 unsigned int eax, edx; 827 828 asm volatile ("mfence; lfence; rdtsc; lfence" : "=a"(eax), "=d"(edx)); 829 tsc = eax | ((unsigned long long)edx << 32); 830 #else 831 asm volatile ("mfence; lfence; rdtsc; lfence" : "=A"(tsc)); 832 #endif 833 return tsc; 834 } 835 836 static inline unsigned long long rdtscp(u32 *aux) 837 { 838 long long r; 839 840 #ifdef __x86_64__ 841 unsigned a, d; 842 843 asm volatile ("rdtscp" : "=a"(a), "=d"(d), "=c"(*aux)); 844 r = a | ((long long)d << 32); 845 #else 846 asm volatile ("rdtscp" : "=A"(r), "=c"(*aux)); 847 #endif 848 return r; 849 } 850 851 static inline void wrtsc(u64 tsc) 852 { 853 wrmsr(MSR_IA32_TSC, tsc); 854 } 855 856 857 static inline void invlpg(volatile void *va) 858 { 859 asm volatile("invlpg (%0)" ::"r" (va) : "memory"); 860 } 861 862 struct invpcid_desc { 863 u64 pcid : 12; 864 u64 rsv : 52; 865 u64 addr : 64; 866 }; 867 868 static inline int invpcid_safe(unsigned long type, struct invpcid_desc *desc) 869 { 870 /* invpcid (%rax), %rbx */ 871 return asm_safe(".byte 0x66,0x0f,0x38,0x82,0x18", "a" (desc), "b" (type)); 872 } 873 874 /* 875 * Execute HLT in an STI interrupt shadow to ensure that a pending IRQ that's 876 * intended to be a wake event arrives *after* HLT is executed. Modern CPUs, 877 * except for a few oddballs that KVM is unlikely to run on, block IRQs for one 878 * instruction after STI, *if* RFLAGS.IF=0 before STI. Note, Intel CPUs may 879 * block other events beyond regular IRQs, e.g. may block NMIs and SMIs too. 880 */ 881 static inline void safe_halt(void) 882 { 883 asm volatile("sti; hlt"); 884 } 885 886 static inline u32 read_pkru(void) 887 { 888 unsigned int eax, edx; 889 unsigned int ecx = 0; 890 unsigned int pkru; 891 892 asm volatile(".byte 0x0f,0x01,0xee\n\t" 893 : "=a" (eax), "=d" (edx) 894 : "c" (ecx)); 895 pkru = eax; 896 return pkru; 897 } 898 899 static inline void write_pkru(u32 pkru) 900 { 901 unsigned int eax = pkru; 902 unsigned int ecx = 0; 903 unsigned int edx = 0; 904 905 asm volatile(".byte 0x0f,0x01,0xef\n\t" 906 : : "a" (eax), "c" (ecx), "d" (edx)); 907 } 908 909 static inline bool is_canonical(u64 addr) 910 { 911 int va_width = (raw_cpuid(0x80000008, 0).a & 0xff00) >> 8; 912 int shift_amt = 64 - va_width; 913 914 return (s64)(addr << shift_amt) >> shift_amt == addr; 915 } 916 917 static inline void clear_bit(int bit, u8 *addr) 918 { 919 __asm__ __volatile__("lock; btr %1, %0" 920 : "+m" (*addr) : "Ir" (bit) : "cc", "memory"); 921 } 922 923 static inline void set_bit(int bit, u8 *addr) 924 { 925 __asm__ __volatile__("lock; bts %1, %0" 926 : "+m" (*addr) : "Ir" (bit) : "cc", "memory"); 927 } 928 929 static inline void flush_tlb(void) 930 { 931 ulong cr4; 932 933 cr4 = read_cr4(); 934 write_cr4(cr4 ^ X86_CR4_PGE); 935 write_cr4(cr4); 936 } 937 938 static inline void generate_non_canonical_gp(void) 939 { 940 *(volatile u64 *)NONCANONICAL = 0; 941 } 942 943 static inline void generate_ud(void) 944 { 945 asm volatile ("ud2"); 946 } 947 948 static inline void generate_de(void) 949 { 950 asm volatile ( 951 "xor %%eax, %%eax\n\t" 952 "xor %%ebx, %%ebx\n\t" 953 "xor %%edx, %%edx\n\t" 954 "idiv %%ebx\n\t" 955 ::: "eax", "ebx", "edx"); 956 } 957 958 static inline void generate_bp(void) 959 { 960 asm volatile ("int3"); 961 } 962 963 static inline void generate_single_step_db(void) 964 { 965 write_rflags(read_rflags() | X86_EFLAGS_TF); 966 asm volatile("nop"); 967 } 968 969 static inline uint64_t generate_usermode_ac(void) 970 { 971 /* 972 * Trigger an #AC by writing 8 bytes to a 4-byte aligned address. 973 * Disclaimer: It is assumed that the stack pointer is aligned 974 * on a 16-byte boundary as x86_64 stacks should be. 975 */ 976 asm volatile("movq $0, -0x4(%rsp)"); 977 978 return 0; 979 } 980 981 /* 982 * Switch from 64-bit to 32-bit mode and generate #OF via INTO. Note, if RIP 983 * or RSP holds a 64-bit value, this helper will NOT generate #OF. 984 */ 985 static inline void generate_of(void) 986 { 987 struct far_pointer32 fp = { 988 .offset = (uintptr_t)&&into, 989 .selector = KERNEL_CS32, 990 }; 991 uintptr_t rsp; 992 993 asm volatile ("mov %%rsp, %0" : "=r"(rsp)); 994 995 if (fp.offset != (uintptr_t)&&into) { 996 printf("Code address too high.\n"); 997 return; 998 } 999 if ((u32)rsp != rsp) { 1000 printf("Stack address too high.\n"); 1001 return; 1002 } 1003 1004 asm goto ("lcall *%0" : : "m" (fp) : "rax" : into); 1005 return; 1006 into: 1007 asm volatile (".code32;" 1008 "movl $0x7fffffff, %eax;" 1009 "addl %eax, %eax;" 1010 "into;" 1011 "lret;" 1012 ".code64"); 1013 __builtin_unreachable(); 1014 } 1015 1016 static inline void fnop(void) 1017 { 1018 asm volatile("fnop"); 1019 } 1020 1021 /* If CR0.TS is set in L2, #NM is generated. */ 1022 static inline void generate_cr0_ts_nm(void) 1023 { 1024 write_cr0((read_cr0() & ~X86_CR0_EM) | X86_CR0_TS); 1025 fnop(); 1026 } 1027 1028 /* If CR0.TS is cleared and CR0.EM is set, #NM is generated. */ 1029 static inline void generate_cr0_em_nm(void) 1030 { 1031 write_cr0((read_cr0() & ~X86_CR0_TS) | X86_CR0_EM); 1032 fnop(); 1033 } 1034 1035 static inline bool is_la57_enabled(void) 1036 { 1037 return !!(read_cr4() & X86_CR4_LA57); 1038 } 1039 1040 static inline bool is_lam_sup_enabled(void) 1041 { 1042 return !!(read_cr4() & X86_CR4_LAM_SUP); 1043 } 1044 1045 static inline bool is_lam_u48_enabled(void) 1046 { 1047 return (read_cr3() & (X86_CR3_LAM_U48 | X86_CR3_LAM_U57)) == X86_CR3_LAM_U48; 1048 } 1049 1050 static inline bool is_lam_u57_enabled(void) 1051 { 1052 return !!(read_cr3() & X86_CR3_LAM_U57); 1053 } 1054 1055 #endif 1056