xref: /kvm-unit-tests/lib/x86/processor.h (revision 85c21181f45ca45e3dc7a7ba6551c44f73c70cd2) 
1 #ifndef _X86_PROCESSOR_H_
2 #define _X86_PROCESSOR_H_
3 
4 #include "libcflat.h"
5 #include "desc.h"
6 #include "msr.h"
7 #include <bitops.h>
8 #include <stdint.h>
9 
10 #define NONCANONICAL	0xaaaaaaaaaaaaaaaaull
11 
12 #ifdef __x86_64__
13 #  define R "r"
14 #  define W "q"
15 #  define S "8"
16 #else
17 #  define R "e"
18 #  define W "l"
19 #  define S "4"
20 #endif
21 
22 #define DB_VECTOR 1
23 #define BP_VECTOR 3
24 #define UD_VECTOR 6
25 #define DF_VECTOR 8
26 #define TS_VECTOR 10
27 #define NP_VECTOR 11
28 #define SS_VECTOR 12
29 #define GP_VECTOR 13
30 #define PF_VECTOR 14
31 #define AC_VECTOR 17
32 #define CP_VECTOR 21
33 
34 #define X86_CR0_PE_BIT		(0)
35 #define X86_CR0_PE		BIT(X86_CR0_PE_BIT)
36 #define X86_CR0_MP_BIT		(1)
37 #define X86_CR0_MP		BIT(X86_CR0_MP_BIT)
38 #define X86_CR0_EM_BIT		(2)
39 #define X86_CR0_EM		BIT(X86_CR0_EM_BIT)
40 #define X86_CR0_TS_BIT		(3)
41 #define X86_CR0_TS		BIT(X86_CR0_TS_BIT)
42 #define X86_CR0_ET_BIT		(4)
43 #define X86_CR0_ET		BIT(X86_CR0_ET_BIT)
44 #define X86_CR0_NE_BIT		(5)
45 #define X86_CR0_NE		BIT(X86_CR0_NE_BIT)
46 #define X86_CR0_WP_BIT		(16)
47 #define X86_CR0_WP		BIT(X86_CR0_WP_BIT)
48 #define X86_CR0_AM_BIT		(18)
49 #define X86_CR0_AM		BIT(X86_CR0_AM_BIT)
50 #define X86_CR0_NW_BIT		(29)
51 #define X86_CR0_NW		BIT(X86_CR0_NW_BIT)
52 #define X86_CR0_CD_BIT		(30)
53 #define X86_CR0_CD		BIT(X86_CR0_CD_BIT)
54 #define X86_CR0_PG_BIT		(31)
55 #define X86_CR0_PG		BIT(X86_CR0_PG_BIT)
56 
57 #define X86_CR3_PCID_MASK	GENMASK(11, 0)
58 
59 #define X86_CR4_VME_BIT		(0)
60 #define X86_CR4_VME		BIT(X86_CR4_VME_BIT)
61 #define X86_CR4_PVI_BIT		(1)
62 #define X86_CR4_PVI		BIT(X86_CR4_PVI_BIT)
63 #define X86_CR4_TSD_BIT		(2)
64 #define X86_CR4_TSD		BIT(X86_CR4_TSD_BIT)
65 #define X86_CR4_DE_BIT		(3)
66 #define X86_CR4_DE		BIT(X86_CR4_DE_BIT)
67 #define X86_CR4_PSE_BIT		(4)
68 #define X86_CR4_PSE		BIT(X86_CR4_PSE_BIT)
69 #define X86_CR4_PAE_BIT		(5)
70 #define X86_CR4_PAE		BIT(X86_CR4_PAE_BIT)
71 #define X86_CR4_MCE_BIT		(6)
72 #define X86_CR4_MCE		BIT(X86_CR4_MCE_BIT)
73 #define X86_CR4_PGE_BIT		(7)
74 #define X86_CR4_PGE		BIT(X86_CR4_PGE_BIT)
75 #define X86_CR4_PCE_BIT		(8)
76 #define X86_CR4_PCE		BIT(X86_CR4_PCE_BIT)
77 #define X86_CR4_OSFXSR_BIT	(9)
78 #define X86_CR4_OSFXSR		BIT(X86_CR4_OSFXSR_BIT)
79 #define X86_CR4_OSXMMEXCPT_BIT	(10)
80 #define X86_CR4_OSXMMEXCPT	BIT(X86_CR4_OSXMMEXCPT_BIT)
81 #define X86_CR4_UMIP_BIT	(11)
82 #define X86_CR4_UMIP		BIT(X86_CR4_UMIP_BIT)
83 #define X86_CR4_LA57_BIT	(12)
84 #define X86_CR4_LA57		BIT(X86_CR4_LA57_BIT)
85 #define X86_CR4_VMXE_BIT	(13)
86 #define X86_CR4_VMXE		BIT(X86_CR4_VMXE_BIT)
87 #define X86_CR4_SMXE_BIT	(14)
88 #define X86_CR4_SMXE		BIT(X86_CR4_SMXE_BIT)
89 /* UNUSED			(15) */
90 #define X86_CR4_FSGSBASE_BIT	(16)
91 #define X86_CR4_FSGSBASE	BIT(X86_CR4_FSGSBASE_BIT)
92 #define X86_CR4_PCIDE_BIT	(17)
93 #define X86_CR4_PCIDE		BIT(X86_CR4_PCIDE_BIT)
94 #define X86_CR4_OSXSAVE_BIT	(18)
95 #define X86_CR4_OSXSAVE		BIT(X86_CR4_OSXSAVE_BIT)
96 #define X86_CR4_KL_BIT		(19)
97 #define X86_CR4_KL		BIT(X86_CR4_KL_BIT)
98 #define X86_CR4_SMEP_BIT	(20)
99 #define X86_CR4_SMEP		BIT(X86_CR4_SMEP_BIT)
100 #define X86_CR4_SMAP_BIT	(21)
101 #define X86_CR4_SMAP		BIT(X86_CR4_SMAP_BIT)
102 #define X86_CR4_PKE_BIT		(22)
103 #define X86_CR4_PKE		BIT(X86_CR4_PKE_BIT)
104 #define X86_CR4_CET_BIT		(23)
105 #define X86_CR4_CET		BIT(X86_CR4_CET_BIT)
106 #define X86_CR4_PKS_BIT		(24)
107 #define X86_CR4_PKS		BIT(X86_CR4_PKS_BIT)
108 
109 #define X86_EFLAGS_CF_BIT	(0)
110 #define X86_EFLAGS_CF		BIT(X86_EFLAGS_CF_BIT)
111 #define X86_EFLAGS_FIXED_BIT	(1)
112 #define X86_EFLAGS_FIXED	BIT(X86_EFLAGS_FIXED_BIT)
113 #define X86_EFLAGS_PF_BIT	(2)
114 #define X86_EFLAGS_PF		BIT(X86_EFLAGS_PF_BIT)
115 /* RESERVED 0			(3) */
116 #define X86_EFLAGS_AF_BIT	(4)
117 #define X86_EFLAGS_AF		BIT(X86_EFLAGS_AF_BIT)
118 /* RESERVED 0			(5) */
119 #define X86_EFLAGS_ZF_BIT	(6)
120 #define X86_EFLAGS_ZF		BIT(X86_EFLAGS_ZF_BIT)
121 #define X86_EFLAGS_SF_BIT	(7)
122 #define X86_EFLAGS_SF		BIT(X86_EFLAGS_SF_BIT)
123 #define X86_EFLAGS_TF_BIT	(8)
124 #define X86_EFLAGS_TF		BIT(X86_EFLAGS_TF_BIT)
125 #define X86_EFLAGS_IF_BIT	(9)
126 #define X86_EFLAGS_IF		BIT(X86_EFLAGS_IF_BIT)
127 #define X86_EFLAGS_DF_BIT	(10)
128 #define X86_EFLAGS_DF		BIT(X86_EFLAGS_DF_BIT)
129 #define X86_EFLAGS_OF_BIT	(11)
130 #define X86_EFLAGS_OF		BIT(X86_EFLAGS_OF_BIT)
131 #define X86_EFLAGS_IOPL		GENMASK(13, 12)
132 #define X86_EFLAGS_NT_BIT	(14)
133 #define X86_EFLAGS_NT		BIT(X86_EFLAGS_NT_BIT)
134 /* RESERVED 0			(15) */
135 #define X86_EFLAGS_RF_BIT	(16)
136 #define X86_EFLAGS_RF		BIT(X86_EFLAGS_RF_BIT)
137 #define X86_EFLAGS_VM_BIT	(17)
138 #define X86_EFLAGS_VM		BIT(X86_EFLAGS_VM_BIT)
139 #define X86_EFLAGS_AC_BIT	(18)
140 #define X86_EFLAGS_AC		BIT(X86_EFLAGS_AC_BIT)
141 #define X86_EFLAGS_VIF_BIT	(19)
142 #define X86_EFLAGS_VIF		BIT(X86_EFLAGS_VIF_BIT)
143 #define X86_EFLAGS_VIP_BIT	(20)
144 #define X86_EFLAGS_VIP		BIT(X86_EFLAGS_VIP_BIT)
145 #define X86_EFLAGS_ID_BIT	(21)
146 #define X86_EFLAGS_ID		BIT(X86_EFLAGS_ID_BIT)
147 
148 #define X86_EFLAGS_ALU (X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF | \
149 			X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF)
150 
151 
152 /*
153  * CPU features
154  */
155 
156 enum cpuid_output_regs {
157 	EAX,
158 	EBX,
159 	ECX,
160 	EDX
161 };
162 
163 struct cpuid { u32 a, b, c, d; };
164 
165 static inline struct cpuid raw_cpuid(u32 function, u32 index)
166 {
167 	struct cpuid r;
168 	asm volatile ("cpuid"
169 		      : "=a"(r.a), "=b"(r.b), "=c"(r.c), "=d"(r.d)
170 		      : "0"(function), "2"(index));
171 	return r;
172 }
173 
174 static inline struct cpuid cpuid_indexed(u32 function, u32 index)
175 {
176 	u32 level = raw_cpuid(function & 0xf0000000, 0).a;
177 	if (level < function)
178 	return (struct cpuid) { 0, 0, 0, 0 };
179 	return raw_cpuid(function, index);
180 }
181 
182 static inline struct cpuid cpuid(u32 function)
183 {
184 	return cpuid_indexed(function, 0);
185 }
186 
187 static inline u8 cpuid_maxphyaddr(void)
188 {
189 	if (raw_cpuid(0x80000000, 0).a < 0x80000008)
190 	return 36;
191 	return raw_cpuid(0x80000008, 0).a & 0xff;
192 }
193 
194 static inline bool is_intel(void)
195 {
196 	struct cpuid c = cpuid(0);
197 	u32 name[4] = {c.b, c.d, c.c };
198 
199 	return strcmp((char *)name, "GenuineIntel") == 0;
200 }
201 
202 #define	CPUID(a, b, c, d) ((((unsigned long long) a) << 32) | (b << 16) | \
203 			  (c << 8) | d)
204 
205 /*
206  * Each X86_FEATURE_XXX definition is 64-bit and contains the following
207  * CPUID meta-data:
208  *
209  * 	[63:32] :  input value for EAX
210  * 	[31:16] :  input value for ECX
211  * 	[15:8]  :  output register
212  * 	[7:0]   :  bit position in output register
213  */
214 
215 /*
216  * Basic Leafs, a.k.a. Intel defined
217  */
218 #define	X86_FEATURE_MWAIT		(CPUID(0x1, 0, ECX, 3))
219 #define	X86_FEATURE_VMX			(CPUID(0x1, 0, ECX, 5))
220 #define	X86_FEATURE_PDCM		(CPUID(0x1, 0, ECX, 15))
221 #define	X86_FEATURE_PCID		(CPUID(0x1, 0, ECX, 17))
222 #define X86_FEATURE_X2APIC		(CPUID(0x1, 0, ECX, 21))
223 #define	X86_FEATURE_MOVBE		(CPUID(0x1, 0, ECX, 22))
224 #define	X86_FEATURE_TSC_DEADLINE_TIMER	(CPUID(0x1, 0, ECX, 24))
225 #define	X86_FEATURE_XSAVE		(CPUID(0x1, 0, ECX, 26))
226 #define	X86_FEATURE_OSXSAVE		(CPUID(0x1, 0, ECX, 27))
227 #define	X86_FEATURE_RDRAND		(CPUID(0x1, 0, ECX, 30))
228 #define	X86_FEATURE_MCE			(CPUID(0x1, 0, EDX, 7))
229 #define	X86_FEATURE_APIC		(CPUID(0x1, 0, EDX, 9))
230 #define	X86_FEATURE_CLFLUSH		(CPUID(0x1, 0, EDX, 19))
231 #define	X86_FEATURE_XMM			(CPUID(0x1, 0, EDX, 25))
232 #define	X86_FEATURE_XMM2		(CPUID(0x1, 0, EDX, 26))
233 #define	X86_FEATURE_TSC_ADJUST		(CPUID(0x7, 0, EBX, 1))
234 #define	X86_FEATURE_HLE			(CPUID(0x7, 0, EBX, 4))
235 #define	X86_FEATURE_SMEP		(CPUID(0x7, 0, EBX, 7))
236 #define	X86_FEATURE_INVPCID		(CPUID(0x7, 0, EBX, 10))
237 #define	X86_FEATURE_RTM			(CPUID(0x7, 0, EBX, 11))
238 #define	X86_FEATURE_SMAP		(CPUID(0x7, 0, EBX, 20))
239 #define	X86_FEATURE_PCOMMIT		(CPUID(0x7, 0, EBX, 22))
240 #define	X86_FEATURE_CLFLUSHOPT		(CPUID(0x7, 0, EBX, 23))
241 #define	X86_FEATURE_CLWB		(CPUID(0x7, 0, EBX, 24))
242 #define	X86_FEATURE_UMIP		(CPUID(0x7, 0, ECX, 2))
243 #define	X86_FEATURE_PKU			(CPUID(0x7, 0, ECX, 3))
244 #define	X86_FEATURE_LA57		(CPUID(0x7, 0, ECX, 16))
245 #define	X86_FEATURE_RDPID		(CPUID(0x7, 0, ECX, 22))
246 #define	X86_FEATURE_SHSTK		(CPUID(0x7, 0, ECX, 7))
247 #define	X86_FEATURE_IBT			(CPUID(0x7, 0, EDX, 20))
248 #define	X86_FEATURE_SPEC_CTRL		(CPUID(0x7, 0, EDX, 26))
249 #define	X86_FEATURE_ARCH_CAPABILITIES	(CPUID(0x7, 0, EDX, 29))
250 #define	X86_FEATURE_PKS			(CPUID(0x7, 0, ECX, 31))
251 
252 /*
253  * Extended Leafs, a.k.a. AMD defined
254  */
255 #define	X86_FEATURE_SVM			(CPUID(0x80000001, 0, ECX, 2))
256 #define	X86_FEATURE_NX			(CPUID(0x80000001, 0, EDX, 20))
257 #define	X86_FEATURE_GBPAGES		(CPUID(0x80000001, 0, EDX, 26))
258 #define	X86_FEATURE_RDTSCP		(CPUID(0x80000001, 0, EDX, 27))
259 #define	X86_FEATURE_LM			(CPUID(0x80000001, 0, EDX, 29))
260 #define	X86_FEATURE_RDPRU		(CPUID(0x80000008, 0, EBX, 4))
261 #define	X86_FEATURE_AMD_IBPB		(CPUID(0x80000008, 0, EBX, 12))
262 #define	X86_FEATURE_NPT			(CPUID(0x8000000A, 0, EDX, 0))
263 #define	X86_FEATURE_LBRV		(CPUID(0x8000000A, 0, EDX, 1))
264 #define	X86_FEATURE_NRIPS		(CPUID(0x8000000A, 0, EDX, 3))
265 #define X86_FEATURE_TSCRATEMSR		(CPUID(0x8000000A, 0, EDX, 4))
266 #define X86_FEATURE_PAUSEFILTER		(CPUID(0x8000000A, 0, EDX, 10))
267 #define X86_FEATURE_PFTHRESHOLD		(CPUID(0x8000000A, 0, EDX, 12))
268 #define	X86_FEATURE_VGIF		(CPUID(0x8000000A, 0, EDX, 16))
269 
270 
271 static inline bool this_cpu_has(u64 feature)
272 {
273 	u32 input_eax = feature >> 32;
274 	u32 input_ecx = (feature >> 16) & 0xffff;
275 	u32 output_reg = (feature >> 8) & 0xff;
276 	u8 bit = feature & 0xff;
277 	struct cpuid c;
278 	u32 *tmp;
279 
280 	c = cpuid_indexed(input_eax, input_ecx);
281 	tmp = (u32 *)&c;
282 
283 	return ((*(tmp + (output_reg % 32))) & (1 << bit));
284 }
285 
286 struct far_pointer32 {
287 	u32 offset;
288 	u16 selector;
289 } __attribute__((packed));
290 
291 struct descriptor_table_ptr {
292 	u16 limit;
293 	ulong base;
294 } __attribute__((packed));
295 
296 static inline void clac(void)
297 {
298 	asm volatile (".byte 0x0f, 0x01, 0xca" : : : "memory");
299 }
300 
301 static inline void stac(void)
302 {
303 	asm volatile (".byte 0x0f, 0x01, 0xcb" : : : "memory");
304 }
305 
306 static inline u16 read_cs(void)
307 {
308 	unsigned val;
309 
310 	asm volatile ("mov %%cs, %0" : "=mr"(val));
311 	return val;
312 }
313 
314 static inline u16 read_ds(void)
315 {
316 	unsigned val;
317 
318 	asm volatile ("mov %%ds, %0" : "=mr"(val));
319 	return val;
320 }
321 
322 static inline u16 read_es(void)
323 {
324 	unsigned val;
325 
326 	asm volatile ("mov %%es, %0" : "=mr"(val));
327 	return val;
328 }
329 
330 static inline u16 read_ss(void)
331 {
332 	unsigned val;
333 
334 	asm volatile ("mov %%ss, %0" : "=mr"(val));
335 	return val;
336 }
337 
338 static inline u16 read_fs(void)
339 {
340 	unsigned val;
341 
342 	asm volatile ("mov %%fs, %0" : "=mr"(val));
343 	return val;
344 }
345 
346 static inline u16 read_gs(void)
347 {
348 	unsigned val;
349 
350 	asm volatile ("mov %%gs, %0" : "=mr"(val));
351 	return val;
352 }
353 
354 static inline unsigned long read_rflags(void)
355 {
356 	unsigned long f;
357 	asm volatile ("pushf; pop %0\n\t" : "=rm"(f));
358 	return f;
359 }
360 
361 static inline void write_ds(unsigned val)
362 {
363 	asm volatile ("mov %0, %%ds" : : "rm"(val) : "memory");
364 }
365 
366 static inline void write_es(unsigned val)
367 {
368 	asm volatile ("mov %0, %%es" : : "rm"(val) : "memory");
369 }
370 
371 static inline void write_ss(unsigned val)
372 {
373 	asm volatile ("mov %0, %%ss" : : "rm"(val) : "memory");
374 }
375 
376 static inline void write_fs(unsigned val)
377 {
378 	asm volatile ("mov %0, %%fs" : : "rm"(val) : "memory");
379 }
380 
381 static inline void write_gs(unsigned val)
382 {
383 	asm volatile ("mov %0, %%gs" : : "rm"(val) : "memory");
384 }
385 
386 static inline void write_rflags(unsigned long f)
387 {
388 	asm volatile ("push %0; popf\n\t" : : "rm"(f));
389 }
390 
391 static inline void set_iopl(int iopl)
392 {
393 	unsigned long flags = read_rflags() & ~X86_EFLAGS_IOPL;
394 	flags |= iopl * (X86_EFLAGS_IOPL / 3);
395 	write_rflags(flags);
396 }
397 
398 /*
399  * Don't use the safe variants for rdmsr() or wrmsr().  The exception fixup
400  * infrastructure uses per-CPU data and thus consumes GS.base.  Various tests
401  * temporarily modify MSR_GS_BASE and will explode when trying to determine
402  * whether or not RDMSR/WRMSR faulted.
403  */
404 static inline u64 rdmsr(u32 index)
405 {
406 	u32 a, d;
407 	asm volatile ("rdmsr" : "=a"(a), "=d"(d) : "c"(index) : "memory");
408 	return a | ((u64)d << 32);
409 }
410 
411 static inline void wrmsr(u32 index, u64 val)
412 {
413 	u32 a = val, d = val >> 32;
414 	asm volatile ("wrmsr" : : "a"(a), "d"(d), "c"(index) : "memory");
415 }
416 
417 static inline int rdmsr_safe(u32 index, uint64_t *val)
418 {
419 	uint32_t a, d;
420 
421 	asm volatile (ASM_TRY("1f")
422 		      "rdmsr\n\t"
423 		      "1:"
424 		      : "=a"(a), "=d"(d)
425 		      : "c"(index) : "memory");
426 
427 	*val = (uint64_t)a | ((uint64_t)d << 32);
428 	return exception_vector();
429 }
430 
431 static inline int wrmsr_safe(u32 index, u64 val)
432 {
433 	u32 a = val, d = val >> 32;
434 
435 	asm volatile (ASM_TRY("1f")
436 		      "wrmsr\n\t"
437 		      "1:"
438 		      : : "a"(a), "d"(d), "c"(index) : "memory");
439 	return exception_vector();
440 }
441 
442 static inline int rdpmc_safe(u32 index, uint64_t *val)
443 {
444 	uint32_t a, d;
445 
446 	asm volatile (ASM_TRY("1f")
447 		      "rdpmc\n\t"
448 		      "1:"
449 		      : "=a"(a), "=d"(d) : "c"(index) : "memory");
450 	*val = (uint64_t)a | ((uint64_t)d << 32);
451 	return exception_vector();
452 }
453 
454 static inline uint64_t rdpmc(uint32_t index)
455 {
456 	uint64_t val;
457 	int vector = rdpmc_safe(index, &val);
458 
459 	assert_msg(!vector, "Unexpected %s on RDPMC(%d)",
460 		   exception_mnemonic(vector), index);
461 	return val;
462 }
463 
464 static inline int write_cr0_safe(ulong val)
465 {
466 	asm volatile(ASM_TRY("1f")
467 		     "mov %0,%%cr0\n\t"
468 		     "1:": : "r" (val));
469 	return exception_vector();
470 }
471 
472 static inline void write_cr0(ulong val)
473 {
474 	int vector = write_cr0_safe(val);
475 
476 	assert_msg(!vector, "Unexpected fault '%d' writing CR0 = %lx",
477 		   vector, val);
478 }
479 
480 static inline ulong read_cr0(void)
481 {
482 	ulong val;
483 	asm volatile ("mov %%cr0, %0" : "=r"(val) : : "memory");
484 	return val;
485 }
486 
487 static inline void write_cr2(ulong val)
488 {
489 	asm volatile ("mov %0, %%cr2" : : "r"(val) : "memory");
490 }
491 
492 static inline ulong read_cr2(void)
493 {
494 	ulong val;
495 	asm volatile ("mov %%cr2, %0" : "=r"(val) : : "memory");
496 	return val;
497 }
498 
499 static inline int write_cr3_safe(ulong val)
500 {
501 	asm volatile(ASM_TRY("1f")
502 		     "mov %0,%%cr3\n\t"
503 		     "1:": : "r" (val));
504 	return exception_vector();
505 }
506 
507 static inline void write_cr3(ulong val)
508 {
509 	int vector = write_cr3_safe(val);
510 
511 	assert_msg(!vector, "Unexpected fault '%d' writing CR3 = %lx",
512 		   vector, val);
513 }
514 
515 static inline ulong read_cr3(void)
516 {
517 	ulong val;
518 	asm volatile ("mov %%cr3, %0" : "=r"(val) : : "memory");
519 	return val;
520 }
521 
522 static inline void update_cr3(void *cr3)
523 {
524 	write_cr3((ulong)cr3);
525 }
526 
527 static inline int write_cr4_safe(ulong val)
528 {
529 	asm volatile(ASM_TRY("1f")
530 		     "mov %0,%%cr4\n\t"
531 		     "1:": : "r" (val));
532 	return exception_vector();
533 }
534 
535 static inline void write_cr4(ulong val)
536 {
537 	int vector = write_cr4_safe(val);
538 
539 	assert_msg(!vector, "Unexpected fault '%d' writing CR4 = %lx",
540 		   vector, val);
541 }
542 
543 static inline ulong read_cr4(void)
544 {
545 	ulong val;
546 	asm volatile ("mov %%cr4, %0" : "=r"(val) : : "memory");
547 	return val;
548 }
549 
550 static inline void write_cr8(ulong val)
551 {
552 	asm volatile ("mov %0, %%cr8" : : "r"(val) : "memory");
553 }
554 
555 static inline ulong read_cr8(void)
556 {
557 	ulong val;
558 	asm volatile ("mov %%cr8, %0" : "=r"(val) : : "memory");
559 	return val;
560 }
561 
562 static inline void lgdt(const struct descriptor_table_ptr *ptr)
563 {
564 	asm volatile ("lgdt %0" : : "m"(*ptr));
565 }
566 
567 static inline void sgdt(struct descriptor_table_ptr *ptr)
568 {
569 	asm volatile ("sgdt %0" : "=m"(*ptr));
570 }
571 
572 static inline void lidt(const struct descriptor_table_ptr *ptr)
573 {
574 	asm volatile ("lidt %0" : : "m"(*ptr));
575 }
576 
577 static inline void sidt(struct descriptor_table_ptr *ptr)
578 {
579 	asm volatile ("sidt %0" : "=m"(*ptr));
580 }
581 
582 static inline void lldt(u16 val)
583 {
584 	asm volatile ("lldt %0" : : "rm"(val));
585 }
586 
587 static inline u16 sldt(void)
588 {
589 	u16 val;
590 	asm volatile ("sldt %0" : "=rm"(val));
591 	return val;
592 }
593 
594 static inline void ltr(u16 val)
595 {
596 	asm volatile ("ltr %0" : : "rm"(val));
597 }
598 
599 static inline u16 str(void)
600 {
601 	u16 val;
602 	asm volatile ("str %0" : "=rm"(val));
603 	return val;
604 }
605 
606 static inline void write_dr0(void *val)
607 {
608 	asm volatile ("mov %0, %%dr0" : : "r"(val) : "memory");
609 }
610 
611 static inline void write_dr1(void *val)
612 {
613 	asm volatile ("mov %0, %%dr1" : : "r"(val) : "memory");
614 }
615 
616 static inline void write_dr2(void *val)
617 {
618 	asm volatile ("mov %0, %%dr2" : : "r"(val) : "memory");
619 }
620 
621 static inline void write_dr3(void *val)
622 {
623 	asm volatile ("mov %0, %%dr3" : : "r"(val) : "memory");
624 }
625 
626 static inline void write_dr6(ulong val)
627 {
628 	asm volatile ("mov %0, %%dr6" : : "r"(val) : "memory");
629 }
630 
631 static inline ulong read_dr6(void)
632 {
633 	ulong val;
634 	asm volatile ("mov %%dr6, %0" : "=r"(val));
635 	return val;
636 }
637 
638 static inline void write_dr7(ulong val)
639 {
640 	asm volatile ("mov %0, %%dr7" : : "r"(val) : "memory");
641 }
642 
643 static inline ulong read_dr7(void)
644 {
645 	ulong val;
646 	asm volatile ("mov %%dr7, %0" : "=r"(val));
647 	return val;
648 }
649 
650 static inline void pause(void)
651 {
652 	asm volatile ("pause");
653 }
654 
655 static inline void cli(void)
656 {
657 	asm volatile ("cli");
658 }
659 
660 static inline void sti(void)
661 {
662 	asm volatile ("sti");
663 }
664 
665 static inline unsigned long long rdrand(void)
666 {
667 	long long r;
668 
669 	asm volatile("rdrand %0\n\t"
670 		     "jc 1f\n\t"
671 		     "mov $0, %0\n\t"
672 		     "1:\n\t" : "=r" (r));
673 	return r;
674 }
675 
676 static inline unsigned long long rdtsc(void)
677 {
678 	long long r;
679 
680 #ifdef __x86_64__
681 	unsigned a, d;
682 
683 	asm volatile ("rdtsc" : "=a"(a), "=d"(d));
684 	r = a | ((long long)d << 32);
685 #else
686 	asm volatile ("rdtsc" : "=A"(r));
687 #endif
688 	return r;
689 }
690 
691 /*
692  * Per the advice in the SDM, volume 2, the sequence "mfence; lfence"
693  * executed immediately before rdtsc ensures that rdtsc will be
694  * executed only after all previous instructions have executed and all
695  * previous loads and stores are globally visible. In addition, the
696  * lfence immediately after rdtsc ensures that rdtsc will be executed
697  * prior to the execution of any subsequent instruction.
698  */
699 static inline unsigned long long fenced_rdtsc(void)
700 {
701 	unsigned long long tsc;
702 
703 #ifdef __x86_64__
704 	unsigned int eax, edx;
705 
706 	asm volatile ("mfence; lfence; rdtsc; lfence" : "=a"(eax), "=d"(edx));
707 	tsc = eax | ((unsigned long long)edx << 32);
708 #else
709 	asm volatile ("mfence; lfence; rdtsc; lfence" : "=A"(tsc));
710 #endif
711 	return tsc;
712 }
713 
714 static inline unsigned long long rdtscp(u32 *aux)
715 {
716 	long long r;
717 
718 #ifdef __x86_64__
719 	unsigned a, d;
720 
721 	asm volatile ("rdtscp" : "=a"(a), "=d"(d), "=c"(*aux));
722 	r = a | ((long long)d << 32);
723 #else
724 	asm volatile ("rdtscp" : "=A"(r), "=c"(*aux));
725 #endif
726 	return r;
727 }
728 
729 static inline void wrtsc(u64 tsc)
730 {
731 	wrmsr(MSR_IA32_TSC, tsc);
732 }
733 
734 static inline void irq_disable(void)
735 {
736 	asm volatile("cli");
737 }
738 
739 /* Note that irq_enable() does not ensure an interrupt shadow due
740  * to the vagaries of compiler optimizations.  If you need the
741  * shadow, use a single asm with "sti" and the instruction after it.
742  */
743 static inline void irq_enable(void)
744 {
745 	asm volatile("sti");
746 }
747 
748 static inline void invlpg(volatile void *va)
749 {
750 	asm volatile("invlpg (%0)" ::"r" (va) : "memory");
751 }
752 
753 static inline void safe_halt(void)
754 {
755 	asm volatile("sti; hlt");
756 }
757 
758 static inline u32 read_pkru(void)
759 {
760 	unsigned int eax, edx;
761 	unsigned int ecx = 0;
762 	unsigned int pkru;
763 
764 	asm volatile(".byte 0x0f,0x01,0xee\n\t"
765 		     : "=a" (eax), "=d" (edx)
766 		     : "c" (ecx));
767 	pkru = eax;
768 	return pkru;
769 }
770 
771 static inline void write_pkru(u32 pkru)
772 {
773 	unsigned int eax = pkru;
774 	unsigned int ecx = 0;
775 	unsigned int edx = 0;
776 
777 	asm volatile(".byte 0x0f,0x01,0xef\n\t"
778 		     : : "a" (eax), "c" (ecx), "d" (edx));
779 }
780 
781 static inline bool is_canonical(u64 addr)
782 {
783 	int va_width = (raw_cpuid(0x80000008, 0).a & 0xff00) >> 8;
784 	int shift_amt = 64 - va_width;
785 
786 	return (s64)(addr << shift_amt) >> shift_amt == addr;
787 }
788 
789 static inline void clear_bit(int bit, u8 *addr)
790 {
791 	__asm__ __volatile__("btr %1, %0"
792 			     : "+m" (*addr) : "Ir" (bit) : "cc", "memory");
793 }
794 
795 static inline void set_bit(int bit, u8 *addr)
796 {
797 	__asm__ __volatile__("bts %1, %0"
798 			     : "+m" (*addr) : "Ir" (bit) : "cc", "memory");
799 }
800 
801 static inline void flush_tlb(void)
802 {
803 	ulong cr4;
804 
805 	cr4 = read_cr4();
806 	write_cr4(cr4 ^ X86_CR4_PGE);
807 	write_cr4(cr4);
808 }
809 
810 static inline void generate_non_canonical_gp(void)
811 {
812 	*(volatile u64 *)NONCANONICAL = 0;
813 }
814 
815 static inline void generate_ud(void)
816 {
817 	asm volatile ("ud2");
818 }
819 
820 static inline void generate_de(void)
821 {
822 	asm volatile (
823 		"xor %%eax, %%eax\n\t"
824 		"xor %%ebx, %%ebx\n\t"
825 		"xor %%edx, %%edx\n\t"
826 		"idiv %%ebx\n\t"
827 		::: "eax", "ebx", "edx");
828 }
829 
830 static inline void generate_bp(void)
831 {
832 	asm volatile ("int3");
833 }
834 
835 static inline void generate_single_step_db(void)
836 {
837 	write_rflags(read_rflags() | X86_EFLAGS_TF);
838 	asm volatile("nop");
839 }
840 
841 static inline uint64_t generate_usermode_ac(void)
842 {
843 	/*
844 	 * Trigger an #AC by writing 8 bytes to a 4-byte aligned address.
845 	 * Disclaimer: It is assumed that the stack pointer is aligned
846 	 * on a 16-byte boundary as x86_64 stacks should be.
847 	 */
848 	asm volatile("movq $0, -0x4(%rsp)");
849 
850 	return 0;
851 }
852 
853 /*
854  * Switch from 64-bit to 32-bit mode and generate #OF via INTO.  Note, if RIP
855  * or RSP holds a 64-bit value, this helper will NOT generate #OF.
856  */
857 static inline void generate_of(void)
858 {
859 	struct far_pointer32 fp = {
860 		.offset = (uintptr_t)&&into,
861 		.selector = KERNEL_CS32,
862 	};
863 	uintptr_t rsp;
864 
865 	asm volatile ("mov %%rsp, %0" : "=r"(rsp));
866 
867 	if (fp.offset != (uintptr_t)&&into) {
868 		printf("Code address too high.\n");
869 		return;
870 	}
871 	if ((u32)rsp != rsp) {
872 		printf("Stack address too high.\n");
873 		return;
874 	}
875 
876 	asm goto ("lcall *%0" : : "m" (fp) : "rax" : into);
877 	return;
878 into:
879 	asm volatile (".code32;"
880 		      "movl $0x7fffffff, %eax;"
881 		      "addl %eax, %eax;"
882 		      "into;"
883 		      "lret;"
884 		      ".code64");
885 	__builtin_unreachable();
886 }
887 
888 static inline void fnop(void)
889 {
890 	asm volatile("fnop");
891 }
892 
893 /* If CR0.TS is set in L2, #NM is generated. */
894 static inline void generate_cr0_ts_nm(void)
895 {
896 	write_cr0((read_cr0() & ~X86_CR0_EM) | X86_CR0_TS);
897 	fnop();
898 }
899 
900 /* If CR0.TS is cleared and CR0.EM is set, #NM is generated. */
901 static inline void generate_cr0_em_nm(void)
902 {
903 	write_cr0((read_cr0() & ~X86_CR0_TS) | X86_CR0_EM);
904 	fnop();
905 }
906 
907 static inline u8 pmu_version(void)
908 {
909 	return cpuid(10).a & 0xff;
910 }
911 
912 static inline bool this_cpu_has_pmu(void)
913 {
914 	return !!pmu_version();
915 }
916 
917 static inline bool this_cpu_has_perf_global_ctrl(void)
918 {
919 	return pmu_version() > 1;
920 }
921 
922 static inline u8 pmu_nr_gp_counters(void)
923 {
924 	return (cpuid(10).a >> 8) & 0xff;
925 }
926 
927 static inline u8 pmu_gp_counter_width(void)
928 {
929 	return (cpuid(10).a >> 16) & 0xff;
930 }
931 
932 static inline u8 pmu_gp_counter_mask_length(void)
933 {
934 	return (cpuid(10).a >> 24) & 0xff;
935 }
936 
937 static inline u8 pmu_nr_fixed_counters(void)
938 {
939 	struct cpuid id = cpuid(10);
940 
941 	if ((id.a & 0xff) > 1)
942 		return id.d & 0x1f;
943 	else
944 		return 0;
945 }
946 
947 static inline u8 pmu_fixed_counter_width(void)
948 {
949 	struct cpuid id = cpuid(10);
950 
951 	if ((id.a & 0xff) > 1)
952 		return (id.d >> 5) & 0xff;
953 	else
954 		return 0;
955 }
956 
957 static inline bool pmu_gp_counter_is_available(int i)
958 {
959 	/* CPUID.0xA.EBX bit is '1 if they counter is NOT available. */
960 	return !(cpuid(10).b & BIT(i));
961 }
962 
963 static inline u64 this_cpu_perf_capabilities(void)
964 {
965 	if (!this_cpu_has(X86_FEATURE_PDCM))
966 		return 0;
967 
968 	return rdmsr(MSR_IA32_PERF_CAPABILITIES);
969 }
970 
971 #endif
972