1 #ifndef _X86_PROCESSOR_H_
2 #define _X86_PROCESSOR_H_
3
4 #include "libcflat.h"
5 #include "desc.h"
6 #include "msr.h"
7 #include <bitops.h>
8 #include <stdint.h>
9
10 #define CANONICAL_48_VAL 0xffffaaaaaaaaaaaaull
11 #define CANONICAL_57_VAL 0xffaaaaaaaaaaaaaaull
12 #define NONCANONICAL 0xaaaaaaaaaaaaaaaaull
13
14 #define LAM57_MASK GENMASK_ULL(62, 57)
15 #define LAM48_MASK GENMASK_ULL(62, 48)
16
17 /*
18 * Get a linear address by combining @addr with a non-canonical pattern in the
19 * @mask bits.
20 */
get_non_canonical(u64 addr,u64 mask)21 static inline u64 get_non_canonical(u64 addr, u64 mask)
22 {
23 return (addr & ~mask) | (NONCANONICAL & mask);
24 }
25
26 #ifdef __x86_64__
27 # define R "r"
28 # define W "q"
29 # define S "8"
30 #else
31 # define R "e"
32 # define W "l"
33 # define S "4"
34 #endif
35
36 #define DE_VECTOR 0
37 #define DB_VECTOR 1
38 #define NMI_VECTOR 2
39 #define BP_VECTOR 3
40 #define OF_VECTOR 4
41 #define BR_VECTOR 5
42 #define UD_VECTOR 6
43 #define NM_VECTOR 7
44 #define DF_VECTOR 8
45 #define TS_VECTOR 10
46 #define NP_VECTOR 11
47 #define SS_VECTOR 12
48 #define GP_VECTOR 13
49 #define PF_VECTOR 14
50 #define MF_VECTOR 16
51 #define AC_VECTOR 17
52 #define MC_VECTOR 18
53 #define XM_VECTOR 19
54 #define XF_VECTOR XM_VECTOR /* AMD */
55 #define VE_VECTOR 20 /* Intel only */
56 #define CP_VECTOR 21
57 #define HV_VECTOR 28 /* AMD only */
58 #define VC_VECTOR 29 /* AMD only */
59 #define SX_VECTOR 30 /* AMD only */
60
61 #define X86_CR0_PE_BIT (0)
62 #define X86_CR0_PE BIT(X86_CR0_PE_BIT)
63 #define X86_CR0_MP_BIT (1)
64 #define X86_CR0_MP BIT(X86_CR0_MP_BIT)
65 #define X86_CR0_EM_BIT (2)
66 #define X86_CR0_EM BIT(X86_CR0_EM_BIT)
67 #define X86_CR0_TS_BIT (3)
68 #define X86_CR0_TS BIT(X86_CR0_TS_BIT)
69 #define X86_CR0_ET_BIT (4)
70 #define X86_CR0_ET BIT(X86_CR0_ET_BIT)
71 #define X86_CR0_NE_BIT (5)
72 #define X86_CR0_NE BIT(X86_CR0_NE_BIT)
73 #define X86_CR0_WP_BIT (16)
74 #define X86_CR0_WP BIT(X86_CR0_WP_BIT)
75 #define X86_CR0_AM_BIT (18)
76 #define X86_CR0_AM BIT(X86_CR0_AM_BIT)
77 #define X86_CR0_NW_BIT (29)
78 #define X86_CR0_NW BIT(X86_CR0_NW_BIT)
79 #define X86_CR0_CD_BIT (30)
80 #define X86_CR0_CD BIT(X86_CR0_CD_BIT)
81 #define X86_CR0_PG_BIT (31)
82 #define X86_CR0_PG BIT(X86_CR0_PG_BIT)
83
84 #define X86_CR3_PCID_MASK GENMASK(11, 0)
85 #define X86_CR3_LAM_U57_BIT (61)
86 #define X86_CR3_LAM_U57 BIT_ULL(X86_CR3_LAM_U57_BIT)
87 #define X86_CR3_LAM_U48_BIT (62)
88 #define X86_CR3_LAM_U48 BIT_ULL(X86_CR3_LAM_U48_BIT)
89
90 #define X86_CR4_VME_BIT (0)
91 #define X86_CR4_VME BIT(X86_CR4_VME_BIT)
92 #define X86_CR4_PVI_BIT (1)
93 #define X86_CR4_PVI BIT(X86_CR4_PVI_BIT)
94 #define X86_CR4_TSD_BIT (2)
95 #define X86_CR4_TSD BIT(X86_CR4_TSD_BIT)
96 #define X86_CR4_DE_BIT (3)
97 #define X86_CR4_DE BIT(X86_CR4_DE_BIT)
98 #define X86_CR4_PSE_BIT (4)
99 #define X86_CR4_PSE BIT(X86_CR4_PSE_BIT)
100 #define X86_CR4_PAE_BIT (5)
101 #define X86_CR4_PAE BIT(X86_CR4_PAE_BIT)
102 #define X86_CR4_MCE_BIT (6)
103 #define X86_CR4_MCE BIT(X86_CR4_MCE_BIT)
104 #define X86_CR4_PGE_BIT (7)
105 #define X86_CR4_PGE BIT(X86_CR4_PGE_BIT)
106 #define X86_CR4_PCE_BIT (8)
107 #define X86_CR4_PCE BIT(X86_CR4_PCE_BIT)
108 #define X86_CR4_OSFXSR_BIT (9)
109 #define X86_CR4_OSFXSR BIT(X86_CR4_OSFXSR_BIT)
110 #define X86_CR4_OSXMMEXCPT_BIT (10)
111 #define X86_CR4_OSXMMEXCPT BIT(X86_CR4_OSXMMEXCPT_BIT)
112 #define X86_CR4_UMIP_BIT (11)
113 #define X86_CR4_UMIP BIT(X86_CR4_UMIP_BIT)
114 #define X86_CR4_LA57_BIT (12)
115 #define X86_CR4_LA57 BIT(X86_CR4_LA57_BIT)
116 #define X86_CR4_VMXE_BIT (13)
117 #define X86_CR4_VMXE BIT(X86_CR4_VMXE_BIT)
118 #define X86_CR4_SMXE_BIT (14)
119 #define X86_CR4_SMXE BIT(X86_CR4_SMXE_BIT)
120 /* UNUSED (15) */
121 #define X86_CR4_FSGSBASE_BIT (16)
122 #define X86_CR4_FSGSBASE BIT(X86_CR4_FSGSBASE_BIT)
123 #define X86_CR4_PCIDE_BIT (17)
124 #define X86_CR4_PCIDE BIT(X86_CR4_PCIDE_BIT)
125 #define X86_CR4_OSXSAVE_BIT (18)
126 #define X86_CR4_OSXSAVE BIT(X86_CR4_OSXSAVE_BIT)
127 #define X86_CR4_KL_BIT (19)
128 #define X86_CR4_KL BIT(X86_CR4_KL_BIT)
129 #define X86_CR4_SMEP_BIT (20)
130 #define X86_CR4_SMEP BIT(X86_CR4_SMEP_BIT)
131 #define X86_CR4_SMAP_BIT (21)
132 #define X86_CR4_SMAP BIT(X86_CR4_SMAP_BIT)
133 #define X86_CR4_PKE_BIT (22)
134 #define X86_CR4_PKE BIT(X86_CR4_PKE_BIT)
135 #define X86_CR4_CET_BIT (23)
136 #define X86_CR4_CET BIT(X86_CR4_CET_BIT)
137 #define X86_CR4_PKS_BIT (24)
138 #define X86_CR4_PKS BIT(X86_CR4_PKS_BIT)
139 #define X86_CR4_LAM_SUP_BIT (28)
140 #define X86_CR4_LAM_SUP BIT(X86_CR4_LAM_SUP_BIT)
141
142 #define X86_EFLAGS_CF_BIT (0)
143 #define X86_EFLAGS_CF BIT(X86_EFLAGS_CF_BIT)
144 #define X86_EFLAGS_FIXED_BIT (1)
145 #define X86_EFLAGS_FIXED BIT(X86_EFLAGS_FIXED_BIT)
146 #define X86_EFLAGS_PF_BIT (2)
147 #define X86_EFLAGS_PF BIT(X86_EFLAGS_PF_BIT)
148 /* RESERVED 0 (3) */
149 #define X86_EFLAGS_AF_BIT (4)
150 #define X86_EFLAGS_AF BIT(X86_EFLAGS_AF_BIT)
151 /* RESERVED 0 (5) */
152 #define X86_EFLAGS_ZF_BIT (6)
153 #define X86_EFLAGS_ZF BIT(X86_EFLAGS_ZF_BIT)
154 #define X86_EFLAGS_SF_BIT (7)
155 #define X86_EFLAGS_SF BIT(X86_EFLAGS_SF_BIT)
156 #define X86_EFLAGS_TF_BIT (8)
157 #define X86_EFLAGS_TF BIT(X86_EFLAGS_TF_BIT)
158 #define X86_EFLAGS_IF_BIT (9)
159 #define X86_EFLAGS_IF BIT(X86_EFLAGS_IF_BIT)
160 #define X86_EFLAGS_DF_BIT (10)
161 #define X86_EFLAGS_DF BIT(X86_EFLAGS_DF_BIT)
162 #define X86_EFLAGS_OF_BIT (11)
163 #define X86_EFLAGS_OF BIT(X86_EFLAGS_OF_BIT)
164 #define X86_EFLAGS_IOPL GENMASK(13, 12)
165 #define X86_EFLAGS_NT_BIT (14)
166 #define X86_EFLAGS_NT BIT(X86_EFLAGS_NT_BIT)
167 /* RESERVED 0 (15) */
168 #define X86_EFLAGS_RF_BIT (16)
169 #define X86_EFLAGS_RF BIT(X86_EFLAGS_RF_BIT)
170 #define X86_EFLAGS_VM_BIT (17)
171 #define X86_EFLAGS_VM BIT(X86_EFLAGS_VM_BIT)
172 #define X86_EFLAGS_AC_BIT (18)
173 #define X86_EFLAGS_AC BIT(X86_EFLAGS_AC_BIT)
174 #define X86_EFLAGS_VIF_BIT (19)
175 #define X86_EFLAGS_VIF BIT(X86_EFLAGS_VIF_BIT)
176 #define X86_EFLAGS_VIP_BIT (20)
177 #define X86_EFLAGS_VIP BIT(X86_EFLAGS_VIP_BIT)
178 #define X86_EFLAGS_ID_BIT (21)
179 #define X86_EFLAGS_ID BIT(X86_EFLAGS_ID_BIT)
180
181 #define X86_EFLAGS_ALU (X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | \
182 X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF)
183
184
185 /*
186 * CPU features
187 */
188
189 enum cpuid_output_regs {
190 EAX,
191 EBX,
192 ECX,
193 EDX
194 };
195
196 struct cpuid { u32 a, b, c, d; };
197
raw_cpuid(u32 function,u32 index)198 static inline struct cpuid raw_cpuid(u32 function, u32 index)
199 {
200 struct cpuid r;
201 asm volatile ("cpuid"
202 : "=a"(r.a), "=b"(r.b), "=c"(r.c), "=d"(r.d)
203 : "0"(function), "2"(index));
204 return r;
205 }
206
cpuid_indexed(u32 function,u32 index)207 static inline struct cpuid cpuid_indexed(u32 function, u32 index)
208 {
209 u32 level = raw_cpuid(function & 0xf0000000, 0).a;
210 if (level < function)
211 return (struct cpuid) { 0, 0, 0, 0 };
212 return raw_cpuid(function, index);
213 }
214
cpuid(u32 function)215 static inline struct cpuid cpuid(u32 function)
216 {
217 return cpuid_indexed(function, 0);
218 }
219
cpuid_maxphyaddr(void)220 static inline u8 cpuid_maxphyaddr(void)
221 {
222 if (raw_cpuid(0x80000000, 0).a < 0x80000008)
223 return 36;
224 return raw_cpuid(0x80000008, 0).a & 0xff;
225 }
226
is_intel(void)227 static inline bool is_intel(void)
228 {
229 struct cpuid c = cpuid(0);
230 u32 name[4] = {c.b, c.d, c.c };
231
232 return strcmp((char *)name, "GenuineIntel") == 0;
233 }
234
235 #define CPUID(a, b, c, d) ((((unsigned long long) a) << 32) | (b << 16) | \
236 (c << 8) | d)
237
238 /*
239 * Each X86_FEATURE_XXX definition is 64-bit and contains the following
240 * CPUID meta-data:
241 *
242 * [63:32] : input value for EAX
243 * [31:16] : input value for ECX
244 * [15:8] : output register
245 * [7:0] : bit position in output register
246 */
247
248 /*
249 * Basic Leafs, a.k.a. Intel defined
250 */
251 #define X86_FEATURE_MWAIT (CPUID(0x1, 0, ECX, 3))
252 #define X86_FEATURE_VMX (CPUID(0x1, 0, ECX, 5))
253 #define X86_FEATURE_PDCM (CPUID(0x1, 0, ECX, 15))
254 #define X86_FEATURE_PCID (CPUID(0x1, 0, ECX, 17))
255 #define X86_FEATURE_X2APIC (CPUID(0x1, 0, ECX, 21))
256 #define X86_FEATURE_MOVBE (CPUID(0x1, 0, ECX, 22))
257 #define X86_FEATURE_TSC_DEADLINE_TIMER (CPUID(0x1, 0, ECX, 24))
258 #define X86_FEATURE_XSAVE (CPUID(0x1, 0, ECX, 26))
259 #define X86_FEATURE_OSXSAVE (CPUID(0x1, 0, ECX, 27))
260 #define X86_FEATURE_RDRAND (CPUID(0x1, 0, ECX, 30))
261 #define X86_FEATURE_MCE (CPUID(0x1, 0, EDX, 7))
262 #define X86_FEATURE_APIC (CPUID(0x1, 0, EDX, 9))
263 #define X86_FEATURE_CLFLUSH (CPUID(0x1, 0, EDX, 19))
264 #define X86_FEATURE_DS (CPUID(0x1, 0, EDX, 21))
265 #define X86_FEATURE_XMM (CPUID(0x1, 0, EDX, 25))
266 #define X86_FEATURE_XMM2 (CPUID(0x1, 0, EDX, 26))
267 #define X86_FEATURE_TSC_ADJUST (CPUID(0x7, 0, EBX, 1))
268 #define X86_FEATURE_HLE (CPUID(0x7, 0, EBX, 4))
269 #define X86_FEATURE_SMEP (CPUID(0x7, 0, EBX, 7))
270 #define X86_FEATURE_INVPCID (CPUID(0x7, 0, EBX, 10))
271 #define X86_FEATURE_RTM (CPUID(0x7, 0, EBX, 11))
272 #define X86_FEATURE_SMAP (CPUID(0x7, 0, EBX, 20))
273 #define X86_FEATURE_PCOMMIT (CPUID(0x7, 0, EBX, 22))
274 #define X86_FEATURE_CLFLUSHOPT (CPUID(0x7, 0, EBX, 23))
275 #define X86_FEATURE_CLWB (CPUID(0x7, 0, EBX, 24))
276 #define X86_FEATURE_INTEL_PT (CPUID(0x7, 0, EBX, 25))
277 #define X86_FEATURE_UMIP (CPUID(0x7, 0, ECX, 2))
278 #define X86_FEATURE_PKU (CPUID(0x7, 0, ECX, 3))
279 #define X86_FEATURE_LA57 (CPUID(0x7, 0, ECX, 16))
280 #define X86_FEATURE_RDPID (CPUID(0x7, 0, ECX, 22))
281 #define X86_FEATURE_SHSTK (CPUID(0x7, 0, ECX, 7))
282 #define X86_FEATURE_IBT (CPUID(0x7, 0, EDX, 20))
283 #define X86_FEATURE_SPEC_CTRL (CPUID(0x7, 0, EDX, 26))
284 #define X86_FEATURE_FLUSH_L1D (CPUID(0x7, 0, EDX, 28))
285 #define X86_FEATURE_ARCH_CAPABILITIES (CPUID(0x7, 0, EDX, 29))
286 #define X86_FEATURE_PKS (CPUID(0x7, 0, ECX, 31))
287 #define X86_FEATURE_LAM (CPUID(0x7, 1, EAX, 26))
288
289 /*
290 * KVM defined leafs
291 */
292 #define KVM_FEATURE_ASYNC_PF (CPUID(0x40000001, 0, EAX, 4))
293 #define KVM_FEATURE_ASYNC_PF_INT (CPUID(0x40000001, 0, EAX, 14))
294
295 /*
296 * Extended Leafs, a.k.a. AMD defined
297 */
298 #define X86_FEATURE_SVM (CPUID(0x80000001, 0, ECX, 2))
299 #define X86_FEATURE_PERFCTR_CORE (CPUID(0x80000001, 0, ECX, 23))
300 #define X86_FEATURE_NX (CPUID(0x80000001, 0, EDX, 20))
301 #define X86_FEATURE_GBPAGES (CPUID(0x80000001, 0, EDX, 26))
302 #define X86_FEATURE_RDTSCP (CPUID(0x80000001, 0, EDX, 27))
303 #define X86_FEATURE_LM (CPUID(0x80000001, 0, EDX, 29))
304 #define X86_FEATURE_RDPRU (CPUID(0x80000008, 0, EBX, 4))
305 #define X86_FEATURE_AMD_IBPB (CPUID(0x80000008, 0, EBX, 12))
306 #define X86_FEATURE_NPT (CPUID(0x8000000A, 0, EDX, 0))
307 #define X86_FEATURE_LBRV (CPUID(0x8000000A, 0, EDX, 1))
308 #define X86_FEATURE_NRIPS (CPUID(0x8000000A, 0, EDX, 3))
309 #define X86_FEATURE_TSCRATEMSR (CPUID(0x8000000A, 0, EDX, 4))
310 #define X86_FEATURE_PAUSEFILTER (CPUID(0x8000000A, 0, EDX, 10))
311 #define X86_FEATURE_PFTHRESHOLD (CPUID(0x8000000A, 0, EDX, 12))
312 #define X86_FEATURE_VGIF (CPUID(0x8000000A, 0, EDX, 16))
313 #define X86_FEATURE_VNMI (CPUID(0x8000000A, 0, EDX, 25))
314 #define X86_FEATURE_AMD_PMU_V2 (CPUID(0x80000022, 0, EAX, 0))
315
this_cpu_has(u64 feature)316 static inline bool this_cpu_has(u64 feature)
317 {
318 u32 input_eax = feature >> 32;
319 u32 input_ecx = (feature >> 16) & 0xffff;
320 u32 output_reg = (feature >> 8) & 0xff;
321 u8 bit = feature & 0xff;
322 struct cpuid c;
323 u32 *tmp;
324
325 c = cpuid_indexed(input_eax, input_ecx);
326 tmp = (u32 *)&c;
327
328 return ((*(tmp + (output_reg % 32))) & (1 << bit));
329 }
330
331 struct far_pointer32 {
332 u32 offset;
333 u16 selector;
334 } __attribute__((packed));
335
336 struct descriptor_table_ptr {
337 u16 limit;
338 ulong base;
339 } __attribute__((packed));
340
clac(void)341 static inline void clac(void)
342 {
343 asm volatile (".byte 0x0f, 0x01, 0xca" : : : "memory");
344 }
345
stac(void)346 static inline void stac(void)
347 {
348 asm volatile (".byte 0x0f, 0x01, 0xcb" : : : "memory");
349 }
350
read_cs(void)351 static inline u16 read_cs(void)
352 {
353 unsigned val;
354
355 asm volatile ("mov %%cs, %0" : "=mr"(val));
356 return val;
357 }
358
read_ds(void)359 static inline u16 read_ds(void)
360 {
361 unsigned val;
362
363 asm volatile ("mov %%ds, %0" : "=mr"(val));
364 return val;
365 }
366
read_es(void)367 static inline u16 read_es(void)
368 {
369 unsigned val;
370
371 asm volatile ("mov %%es, %0" : "=mr"(val));
372 return val;
373 }
374
read_ss(void)375 static inline u16 read_ss(void)
376 {
377 unsigned val;
378
379 asm volatile ("mov %%ss, %0" : "=mr"(val));
380 return val;
381 }
382
read_fs(void)383 static inline u16 read_fs(void)
384 {
385 unsigned val;
386
387 asm volatile ("mov %%fs, %0" : "=mr"(val));
388 return val;
389 }
390
read_gs(void)391 static inline u16 read_gs(void)
392 {
393 unsigned val;
394
395 asm volatile ("mov %%gs, %0" : "=mr"(val));
396 return val;
397 }
398
read_rflags(void)399 static inline unsigned long read_rflags(void)
400 {
401 unsigned long f;
402 asm volatile ("pushf; pop %0\n\t" : "=rm"(f));
403 return f;
404 }
405
write_ds(unsigned val)406 static inline void write_ds(unsigned val)
407 {
408 asm volatile ("mov %0, %%ds" : : "rm"(val) : "memory");
409 }
410
write_es(unsigned val)411 static inline void write_es(unsigned val)
412 {
413 asm volatile ("mov %0, %%es" : : "rm"(val) : "memory");
414 }
415
write_ss(unsigned val)416 static inline void write_ss(unsigned val)
417 {
418 asm volatile ("mov %0, %%ss" : : "rm"(val) : "memory");
419 }
420
write_fs(unsigned val)421 static inline void write_fs(unsigned val)
422 {
423 asm volatile ("mov %0, %%fs" : : "rm"(val) : "memory");
424 }
425
write_gs(unsigned val)426 static inline void write_gs(unsigned val)
427 {
428 asm volatile ("mov %0, %%gs" : : "rm"(val) : "memory");
429 }
430
write_rflags(unsigned long f)431 static inline void write_rflags(unsigned long f)
432 {
433 asm volatile ("push %0; popf\n\t" : : "rm"(f));
434 }
435
set_iopl(int iopl)436 static inline void set_iopl(int iopl)
437 {
438 unsigned long flags = read_rflags() & ~X86_EFLAGS_IOPL;
439 flags |= iopl * (X86_EFLAGS_IOPL / 3);
440 write_rflags(flags);
441 }
442
443 /*
444 * Don't use the safe variants for rdmsr() or wrmsr(). The exception fixup
445 * infrastructure uses per-CPU data and thus consumes GS.base. Various tests
446 * temporarily modify MSR_GS_BASE and will explode when trying to determine
447 * whether or not RDMSR/WRMSR faulted.
448 */
rdmsr(u32 index)449 static inline u64 rdmsr(u32 index)
450 {
451 u32 a, d;
452 asm volatile ("rdmsr" : "=a"(a), "=d"(d) : "c"(index) : "memory");
453 return a | ((u64)d << 32);
454 }
455
wrmsr(u32 index,u64 val)456 static inline void wrmsr(u32 index, u64 val)
457 {
458 u32 a = val, d = val >> 32;
459 asm volatile ("wrmsr" : : "a"(a), "d"(d), "c"(index) : "memory");
460 }
461
462 #define __rdreg64_safe(fep, insn, index, val) \
463 ({ \
464 uint32_t a, d; \
465 int vector; \
466 \
467 vector = __asm_safe_out2(fep, insn, "=a"(a), "=d"(d), "c"(index));\
468 \
469 if (vector) \
470 *(val) = 0; \
471 else \
472 *(val) = (uint64_t)a | ((uint64_t)d << 32); \
473 vector; \
474 })
475
476 #define rdreg64_safe(insn, index, val) \
477 __rdreg64_safe("", insn, index, val)
478
479 #define __wrreg64_safe(fep, insn, index, val) \
480 ({ \
481 uint32_t eax = (val), edx = (val) >> 32; \
482 \
483 __asm_safe(fep, insn, "a" (eax), "d" (edx), "c" (index)); \
484 })
485
486 #define wrreg64_safe(insn, index, val) \
487 __wrreg64_safe("", insn, index, val)
488
rdmsr_safe(u32 index,uint64_t * val)489 static inline int rdmsr_safe(u32 index, uint64_t *val)
490 {
491 return rdreg64_safe("rdmsr", index, val);
492 }
493
rdmsr_fep_safe(u32 index,uint64_t * val)494 static inline int rdmsr_fep_safe(u32 index, uint64_t *val)
495 {
496 return __rdreg64_safe(KVM_FEP, "rdmsr", index, val);
497 }
498
wrmsr_safe(u32 index,u64 val)499 static inline int wrmsr_safe(u32 index, u64 val)
500 {
501 return wrreg64_safe("wrmsr", index, val);
502 }
503
wrmsr_fep_safe(u32 index,u64 val)504 static inline int wrmsr_fep_safe(u32 index, u64 val)
505 {
506 return __wrreg64_safe(KVM_FEP, "wrmsr", index, val);
507 }
508
rdpmc_safe(u32 index,uint64_t * val)509 static inline int rdpmc_safe(u32 index, uint64_t *val)
510 {
511 return rdreg64_safe("rdpmc", index, val);
512 }
513
rdpmc(uint32_t index)514 static inline uint64_t rdpmc(uint32_t index)
515 {
516 uint64_t val;
517 int vector = rdpmc_safe(index, &val);
518
519 assert_msg(!vector, "Unexpected %s on RDPMC(%" PRId32 ")",
520 exception_mnemonic(vector), index);
521 return val;
522 }
523
xgetbv_safe(u32 index,u64 * result)524 static inline int xgetbv_safe(u32 index, u64 *result)
525 {
526 return rdreg64_safe(".byte 0x0f,0x01,0xd0", index, result);
527 }
528
xsetbv_safe(u32 index,u64 value)529 static inline int xsetbv_safe(u32 index, u64 value)
530 {
531 return wrreg64_safe(".byte 0x0f,0x01,0xd1", index, value);
532 }
533
write_cr0_safe(ulong val)534 static inline int write_cr0_safe(ulong val)
535 {
536 return asm_safe("mov %0,%%cr0", "r" (val));
537 }
538
write_cr0(ulong val)539 static inline void write_cr0(ulong val)
540 {
541 int vector = write_cr0_safe(val);
542
543 assert_msg(!vector, "Unexpected fault '%d' writing CR0 = %lx",
544 vector, val);
545 }
546
read_cr0(void)547 static inline ulong read_cr0(void)
548 {
549 ulong val;
550 asm volatile ("mov %%cr0, %0" : "=r"(val) : : "memory");
551 return val;
552 }
553
write_cr2(ulong val)554 static inline void write_cr2(ulong val)
555 {
556 asm volatile ("mov %0, %%cr2" : : "r"(val) : "memory");
557 }
558
read_cr2(void)559 static inline ulong read_cr2(void)
560 {
561 ulong val;
562 asm volatile ("mov %%cr2, %0" : "=r"(val) : : "memory");
563 return val;
564 }
565
write_cr3_safe(ulong val)566 static inline int write_cr3_safe(ulong val)
567 {
568 return asm_safe("mov %0,%%cr3", "r" (val));
569 }
570
write_cr3(ulong val)571 static inline void write_cr3(ulong val)
572 {
573 int vector = write_cr3_safe(val);
574
575 assert_msg(!vector, "Unexpected fault '%d' writing CR3 = %lx",
576 vector, val);
577 }
578
read_cr3(void)579 static inline ulong read_cr3(void)
580 {
581 ulong val;
582 asm volatile ("mov %%cr3, %0" : "=r"(val) : : "memory");
583 return val;
584 }
585
update_cr3(void * cr3)586 static inline void update_cr3(void *cr3)
587 {
588 write_cr3((ulong)cr3);
589 }
590
write_cr4_safe(ulong val)591 static inline int write_cr4_safe(ulong val)
592 {
593 return asm_safe("mov %0,%%cr4", "r" (val));
594 }
595
write_cr4(ulong val)596 static inline void write_cr4(ulong val)
597 {
598 int vector = write_cr4_safe(val);
599
600 assert_msg(!vector, "Unexpected fault '%d' writing CR4 = %lx",
601 vector, val);
602 }
603
read_cr4(void)604 static inline ulong read_cr4(void)
605 {
606 ulong val;
607 asm volatile ("mov %%cr4, %0" : "=r"(val) : : "memory");
608 return val;
609 }
610
write_cr8(ulong val)611 static inline void write_cr8(ulong val)
612 {
613 asm volatile ("mov %0, %%cr8" : : "r"(val) : "memory");
614 }
615
read_cr8(void)616 static inline ulong read_cr8(void)
617 {
618 ulong val;
619 asm volatile ("mov %%cr8, %0" : "=r"(val) : : "memory");
620 return val;
621 }
622
lgdt(const struct descriptor_table_ptr * ptr)623 static inline void lgdt(const struct descriptor_table_ptr *ptr)
624 {
625 asm volatile ("lgdt %0" : : "m"(*ptr));
626 }
627
lgdt_safe(const struct descriptor_table_ptr * ptr)628 static inline int lgdt_safe(const struct descriptor_table_ptr *ptr)
629 {
630 return asm_safe("lgdt %0", "m"(*ptr));
631 }
632
lgdt_fep_safe(const struct descriptor_table_ptr * ptr)633 static inline int lgdt_fep_safe(const struct descriptor_table_ptr *ptr)
634 {
635 return asm_fep_safe("lgdt %0", "m"(*ptr));
636 }
637
sgdt(struct descriptor_table_ptr * ptr)638 static inline void sgdt(struct descriptor_table_ptr *ptr)
639 {
640 asm volatile ("sgdt %0" : "=m"(*ptr));
641 }
642
lidt(const struct descriptor_table_ptr * ptr)643 static inline void lidt(const struct descriptor_table_ptr *ptr)
644 {
645 asm volatile ("lidt %0" : : "m"(*ptr));
646 }
647
lidt_safe(const struct descriptor_table_ptr * ptr)648 static inline int lidt_safe(const struct descriptor_table_ptr *ptr)
649 {
650 return asm_safe("lidt %0", "m"(*ptr));
651 }
652
lidt_fep_safe(const struct descriptor_table_ptr * ptr)653 static inline int lidt_fep_safe(const struct descriptor_table_ptr *ptr)
654 {
655 return asm_fep_safe("lidt %0", "m"(*ptr));
656 }
657
sidt(struct descriptor_table_ptr * ptr)658 static inline void sidt(struct descriptor_table_ptr *ptr)
659 {
660 asm volatile ("sidt %0" : "=m"(*ptr));
661 }
662
lldt(u16 val)663 static inline void lldt(u16 val)
664 {
665 asm volatile ("lldt %0" : : "rm"(val));
666 }
667
lldt_safe(u16 val)668 static inline int lldt_safe(u16 val)
669 {
670 return asm_safe("lldt %0", "rm"(val));
671 }
672
lldt_fep_safe(u16 val)673 static inline int lldt_fep_safe(u16 val)
674 {
675 return asm_safe("lldt %0", "rm"(val));
676 }
677
sldt(void)678 static inline u16 sldt(void)
679 {
680 u16 val;
681 asm volatile ("sldt %0" : "=rm"(val));
682 return val;
683 }
684
ltr(u16 val)685 static inline void ltr(u16 val)
686 {
687 asm volatile ("ltr %0" : : "rm"(val));
688 }
689
ltr_safe(u16 val)690 static inline int ltr_safe(u16 val)
691 {
692 return asm_safe("ltr %0", "rm"(val));
693 }
694
ltr_fep_safe(u16 val)695 static inline int ltr_fep_safe(u16 val)
696 {
697 return asm_safe("ltr %0", "rm"(val));
698 }
699
str(void)700 static inline u16 str(void)
701 {
702 u16 val;
703 asm volatile ("str %0" : "=rm"(val));
704 return val;
705 }
706
write_dr0(void * val)707 static inline void write_dr0(void *val)
708 {
709 asm volatile ("mov %0, %%dr0" : : "r"(val) : "memory");
710 }
711
write_dr1(void * val)712 static inline void write_dr1(void *val)
713 {
714 asm volatile ("mov %0, %%dr1" : : "r"(val) : "memory");
715 }
716
write_dr2(void * val)717 static inline void write_dr2(void *val)
718 {
719 asm volatile ("mov %0, %%dr2" : : "r"(val) : "memory");
720 }
721
write_dr3(void * val)722 static inline void write_dr3(void *val)
723 {
724 asm volatile ("mov %0, %%dr3" : : "r"(val) : "memory");
725 }
726
write_dr6(ulong val)727 static inline void write_dr6(ulong val)
728 {
729 asm volatile ("mov %0, %%dr6" : : "r"(val) : "memory");
730 }
731
read_dr6(void)732 static inline ulong read_dr6(void)
733 {
734 ulong val;
735 asm volatile ("mov %%dr6, %0" : "=r"(val));
736 return val;
737 }
738
write_dr7(ulong val)739 static inline void write_dr7(ulong val)
740 {
741 asm volatile ("mov %0, %%dr7" : : "r"(val) : "memory");
742 }
743
read_dr7(void)744 static inline ulong read_dr7(void)
745 {
746 ulong val;
747 asm volatile ("mov %%dr7, %0" : "=r"(val));
748 return val;
749 }
750
pause(void)751 static inline void pause(void)
752 {
753 asm volatile ("pause");
754 }
755
cli(void)756 static inline void cli(void)
757 {
758 asm volatile ("cli");
759 }
760
761 /*
762 * See also safe_halt().
763 */
sti(void)764 static inline void sti(void)
765 {
766 asm volatile ("sti");
767 }
768
769 /*
770 * Enable interrupts and ensure that interrupts are evaluated upon return from
771 * this function, i.e. execute a nop to consume the STi interrupt shadow.
772 */
sti_nop(void)773 static inline void sti_nop(void)
774 {
775 asm volatile ("sti; nop");
776 }
777
778 /*
779 * Enable interrupts for one instruction (nop), to allow the CPU to process all
780 * interrupts that are already pending.
781 */
sti_nop_cli(void)782 static inline void sti_nop_cli(void)
783 {
784 asm volatile ("sti; nop; cli");
785 }
786
rdrand(void)787 static inline unsigned long long rdrand(void)
788 {
789 long long r;
790
791 asm volatile("rdrand %0\n\t"
792 "jc 1f\n\t"
793 "mov $0, %0\n\t"
794 "1:\n\t" : "=r" (r));
795 return r;
796 }
797
rdtsc(void)798 static inline unsigned long long rdtsc(void)
799 {
800 long long r;
801
802 #ifdef __x86_64__
803 unsigned a, d;
804
805 asm volatile ("rdtsc" : "=a"(a), "=d"(d));
806 r = a | ((long long)d << 32);
807 #else
808 asm volatile ("rdtsc" : "=A"(r));
809 #endif
810 return r;
811 }
812
813 /*
814 * Per the advice in the SDM, volume 2, the sequence "mfence; lfence"
815 * executed immediately before rdtsc ensures that rdtsc will be
816 * executed only after all previous instructions have executed and all
817 * previous loads and stores are globally visible. In addition, the
818 * lfence immediately after rdtsc ensures that rdtsc will be executed
819 * prior to the execution of any subsequent instruction.
820 */
fenced_rdtsc(void)821 static inline unsigned long long fenced_rdtsc(void)
822 {
823 unsigned long long tsc;
824
825 #ifdef __x86_64__
826 unsigned int eax, edx;
827
828 asm volatile ("mfence; lfence; rdtsc; lfence" : "=a"(eax), "=d"(edx));
829 tsc = eax | ((unsigned long long)edx << 32);
830 #else
831 asm volatile ("mfence; lfence; rdtsc; lfence" : "=A"(tsc));
832 #endif
833 return tsc;
834 }
835
rdtscp(u32 * aux)836 static inline unsigned long long rdtscp(u32 *aux)
837 {
838 long long r;
839
840 #ifdef __x86_64__
841 unsigned a, d;
842
843 asm volatile ("rdtscp" : "=a"(a), "=d"(d), "=c"(*aux));
844 r = a | ((long long)d << 32);
845 #else
846 asm volatile ("rdtscp" : "=A"(r), "=c"(*aux));
847 #endif
848 return r;
849 }
850
wrtsc(u64 tsc)851 static inline void wrtsc(u64 tsc)
852 {
853 wrmsr(MSR_IA32_TSC, tsc);
854 }
855
856
invlpg(volatile void * va)857 static inline void invlpg(volatile void *va)
858 {
859 asm volatile("invlpg (%0)" ::"r" (va) : "memory");
860 }
861
862 struct invpcid_desc {
863 u64 pcid : 12;
864 u64 rsv : 52;
865 u64 addr : 64;
866 };
867
invpcid_safe(unsigned long type,struct invpcid_desc * desc)868 static inline int invpcid_safe(unsigned long type, struct invpcid_desc *desc)
869 {
870 /* invpcid (%rax), %rbx */
871 return asm_safe(".byte 0x66,0x0f,0x38,0x82,0x18", "a" (desc), "b" (type));
872 }
873
874 /*
875 * Execute HLT in an STI interrupt shadow to ensure that a pending IRQ that's
876 * intended to be a wake event arrives *after* HLT is executed. Modern CPUs,
877 * except for a few oddballs that KVM is unlikely to run on, block IRQs for one
878 * instruction after STI, *if* RFLAGS.IF=0 before STI. Note, Intel CPUs may
879 * block other events beyond regular IRQs, e.g. may block NMIs and SMIs too.
880 */
safe_halt(void)881 static inline void safe_halt(void)
882 {
883 asm volatile("sti; hlt");
884 }
885
read_pkru(void)886 static inline u32 read_pkru(void)
887 {
888 unsigned int eax, edx;
889 unsigned int ecx = 0;
890 unsigned int pkru;
891
892 asm volatile(".byte 0x0f,0x01,0xee\n\t"
893 : "=a" (eax), "=d" (edx)
894 : "c" (ecx));
895 pkru = eax;
896 return pkru;
897 }
898
write_pkru(u32 pkru)899 static inline void write_pkru(u32 pkru)
900 {
901 unsigned int eax = pkru;
902 unsigned int ecx = 0;
903 unsigned int edx = 0;
904
905 asm volatile(".byte 0x0f,0x01,0xef\n\t"
906 : : "a" (eax), "c" (ecx), "d" (edx));
907 }
908
is_canonical(u64 addr)909 static inline bool is_canonical(u64 addr)
910 {
911 int va_width = (raw_cpuid(0x80000008, 0).a & 0xff00) >> 8;
912 int shift_amt = 64 - va_width;
913
914 return (s64)(addr << shift_amt) >> shift_amt == addr;
915 }
916
clear_bit(int bit,u8 * addr)917 static inline void clear_bit(int bit, u8 *addr)
918 {
919 __asm__ __volatile__("lock; btr %1, %0"
920 : "+m" (*addr) : "Ir" (bit) : "cc", "memory");
921 }
922
set_bit(int bit,u8 * addr)923 static inline void set_bit(int bit, u8 *addr)
924 {
925 __asm__ __volatile__("lock; bts %1, %0"
926 : "+m" (*addr) : "Ir" (bit) : "cc", "memory");
927 }
928
flush_tlb(void)929 static inline void flush_tlb(void)
930 {
931 ulong cr4;
932
933 cr4 = read_cr4();
934 write_cr4(cr4 ^ X86_CR4_PGE);
935 write_cr4(cr4);
936 }
937
generate_non_canonical_gp(void)938 static inline void generate_non_canonical_gp(void)
939 {
940 *(volatile u64 *)NONCANONICAL = 0;
941 }
942
generate_ud(void)943 static inline void generate_ud(void)
944 {
945 asm volatile ("ud2");
946 }
947
generate_de(void)948 static inline void generate_de(void)
949 {
950 asm volatile (
951 "xor %%eax, %%eax\n\t"
952 "xor %%ebx, %%ebx\n\t"
953 "xor %%edx, %%edx\n\t"
954 "idiv %%ebx\n\t"
955 ::: "eax", "ebx", "edx");
956 }
957
generate_bp(void)958 static inline void generate_bp(void)
959 {
960 asm volatile ("int3");
961 }
962
generate_single_step_db(void)963 static inline void generate_single_step_db(void)
964 {
965 write_rflags(read_rflags() | X86_EFLAGS_TF);
966 asm volatile("nop");
967 }
968
generate_usermode_ac(void)969 static inline uint64_t generate_usermode_ac(void)
970 {
971 /*
972 * Trigger an #AC by writing 8 bytes to a 4-byte aligned address.
973 * Disclaimer: It is assumed that the stack pointer is aligned
974 * on a 16-byte boundary as x86_64 stacks should be.
975 */
976 asm volatile("movq $0, -0x4(%rsp)");
977
978 return 0;
979 }
980
981 /*
982 * Switch from 64-bit to 32-bit mode and generate #OF via INTO. Note, if RIP
983 * or RSP holds a 64-bit value, this helper will NOT generate #OF.
984 */
generate_of(void)985 static inline void generate_of(void)
986 {
987 struct far_pointer32 fp = {
988 .offset = (uintptr_t)&&into,
989 .selector = KERNEL_CS32,
990 };
991 uintptr_t rsp;
992
993 asm volatile ("mov %%rsp, %0" : "=r"(rsp));
994
995 if (fp.offset != (uintptr_t)&&into) {
996 printf("Code address too high.\n");
997 return;
998 }
999 if ((u32)rsp != rsp) {
1000 printf("Stack address too high.\n");
1001 return;
1002 }
1003
1004 asm goto ("lcall *%0" : : "m" (fp) : "rax" : into);
1005 return;
1006 into:
1007 asm volatile (".code32;"
1008 "movl $0x7fffffff, %eax;"
1009 "addl %eax, %eax;"
1010 "into;"
1011 "lret;"
1012 ".code64");
1013 __builtin_unreachable();
1014 }
1015
fnop(void)1016 static inline void fnop(void)
1017 {
1018 asm volatile("fnop");
1019 }
1020
1021 /* If CR0.TS is set in L2, #NM is generated. */
generate_cr0_ts_nm(void)1022 static inline void generate_cr0_ts_nm(void)
1023 {
1024 write_cr0((read_cr0() & ~X86_CR0_EM) | X86_CR0_TS);
1025 fnop();
1026 }
1027
1028 /* If CR0.TS is cleared and CR0.EM is set, #NM is generated. */
generate_cr0_em_nm(void)1029 static inline void generate_cr0_em_nm(void)
1030 {
1031 write_cr0((read_cr0() & ~X86_CR0_TS) | X86_CR0_EM);
1032 fnop();
1033 }
1034
is_la57_enabled(void)1035 static inline bool is_la57_enabled(void)
1036 {
1037 return !!(read_cr4() & X86_CR4_LA57);
1038 }
1039
is_lam_sup_enabled(void)1040 static inline bool is_lam_sup_enabled(void)
1041 {
1042 return !!(read_cr4() & X86_CR4_LAM_SUP);
1043 }
1044
is_lam_u48_enabled(void)1045 static inline bool is_lam_u48_enabled(void)
1046 {
1047 return (read_cr3() & (X86_CR3_LAM_U48 | X86_CR3_LAM_U57)) == X86_CR3_LAM_U48;
1048 }
1049
is_lam_u57_enabled(void)1050 static inline bool is_lam_u57_enabled(void)
1051 {
1052 return !!(read_cr3() & X86_CR3_LAM_U57);
1053 }
1054
1055 #endif
1056