1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * KVM dirty page logging test
4 *
5 * Copyright (C) 2018, Red Hat, Inc.
6 */
7 #include <stdio.h>
8 #include <stdlib.h>
9 #include <linux/bitmap.h>
10 #include <linux/bitops.h>
11
12 #include "test_util.h"
13 #include "kvm_util.h"
14 #include "processor.h"
15 #include "svm_util.h"
16 #include "vmx.h"
17
18 /* The memory slot index to track dirty pages */
19 #define TEST_MEM_SLOT_INDEX 1
20
21 /*
22 * Allocate four pages total. Two pages are used to verify that the KVM marks
23 * the accessed page/GFN as marked dirty, but not the "other" page. Times two
24 * so that each "normal" page can be accessed from L2 via an aliased L2 GVA+GPA
25 * (when TDP is enabled), to verify KVM marks _L1's_ page/GFN as dirty (to
26 * detect failures, L2 => L1 GPAs can't be identity mapped in the TDP page
27 * tables, as marking L2's GPA dirty would get a false pass if L1 == L2).
28 */
29 #define TEST_MEM_PAGES 4
30
31 #define TEST_MEM_BASE 0xc0000000
32 #define TEST_MEM_ALIAS_BASE 0xc0002000
33
34 #define TEST_GUEST_ADDR(base, idx) ((base) + (idx) * PAGE_SIZE)
35
36 #define TEST_GVA(idx) TEST_GUEST_ADDR(TEST_MEM_BASE, idx)
37 #define TEST_GPA(idx) TEST_GUEST_ADDR(TEST_MEM_BASE, idx)
38
39 #define TEST_ALIAS_GPA(idx) TEST_GUEST_ADDR(TEST_MEM_ALIAS_BASE, idx)
40
41 #define TEST_HVA(vm, idx) addr_gpa2hva(vm, TEST_GPA(idx))
42
43 #define L2_GUEST_STACK_SIZE 64
44
45 /* Use the page offset bits to communicate the access+fault type. */
46 #define TEST_SYNC_READ_FAULT BIT(0)
47 #define TEST_SYNC_WRITE_FAULT BIT(1)
48 #define TEST_SYNC_NO_FAULT BIT(2)
49
l2_guest_code(vm_vaddr_t base)50 static void l2_guest_code(vm_vaddr_t base)
51 {
52 vm_vaddr_t page0 = TEST_GUEST_ADDR(base, 0);
53 vm_vaddr_t page1 = TEST_GUEST_ADDR(base, 1);
54
55 READ_ONCE(*(u64 *)page0);
56 GUEST_SYNC(page0 | TEST_SYNC_READ_FAULT);
57 WRITE_ONCE(*(u64 *)page0, 1);
58 GUEST_SYNC(page0 | TEST_SYNC_WRITE_FAULT);
59 READ_ONCE(*(u64 *)page0);
60 GUEST_SYNC(page0 | TEST_SYNC_NO_FAULT);
61
62 WRITE_ONCE(*(u64 *)page1, 1);
63 GUEST_SYNC(page1 | TEST_SYNC_WRITE_FAULT);
64 WRITE_ONCE(*(u64 *)page1, 1);
65 GUEST_SYNC(page1 | TEST_SYNC_WRITE_FAULT);
66 READ_ONCE(*(u64 *)page1);
67 GUEST_SYNC(page1 | TEST_SYNC_NO_FAULT);
68
69 /* Exit to L1 and never come back. */
70 vmcall();
71 }
72
l2_guest_code_tdp_enabled(void)73 static void l2_guest_code_tdp_enabled(void)
74 {
75 /*
76 * Use the aliased virtual addresses when running with TDP to verify
77 * that KVM correctly handles the case where a page is dirtied via a
78 * different GPA than would be used by L1.
79 */
80 l2_guest_code(TEST_MEM_ALIAS_BASE);
81 }
82
l2_guest_code_tdp_disabled(void)83 static void l2_guest_code_tdp_disabled(void)
84 {
85 /*
86 * Use the "normal" virtual addresses when running without TDP enabled,
87 * in which case L2 will use the same page tables as L1, and thus needs
88 * to use the same virtual addresses that are mapped into L1.
89 */
90 l2_guest_code(TEST_MEM_BASE);
91 }
92
l1_vmx_code(struct vmx_pages * vmx)93 void l1_vmx_code(struct vmx_pages *vmx)
94 {
95 unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
96 void *l2_rip;
97
98 GUEST_ASSERT(vmx->vmcs_gpa);
99 GUEST_ASSERT(prepare_for_vmx_operation(vmx));
100 GUEST_ASSERT(load_vmcs(vmx));
101
102 if (vmx->eptp_gpa)
103 l2_rip = l2_guest_code_tdp_enabled;
104 else
105 l2_rip = l2_guest_code_tdp_disabled;
106
107 prepare_vmcs(vmx, l2_rip, &l2_guest_stack[L2_GUEST_STACK_SIZE]);
108
109 GUEST_SYNC(TEST_SYNC_NO_FAULT);
110 GUEST_ASSERT(!vmlaunch());
111 GUEST_SYNC(TEST_SYNC_NO_FAULT);
112 GUEST_ASSERT_EQ(vmreadz(VM_EXIT_REASON), EXIT_REASON_VMCALL);
113 GUEST_DONE();
114 }
115
l1_svm_code(struct svm_test_data * svm)116 static void l1_svm_code(struct svm_test_data *svm)
117 {
118 unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE];
119 void *l2_rip;
120
121 if (svm->ncr3_gpa)
122 l2_rip = l2_guest_code_tdp_enabled;
123 else
124 l2_rip = l2_guest_code_tdp_disabled;
125
126 generic_svm_setup(svm, l2_rip, &l2_guest_stack[L2_GUEST_STACK_SIZE]);
127
128 GUEST_SYNC(TEST_SYNC_NO_FAULT);
129 run_guest(svm->vmcb, svm->vmcb_gpa);
130 GUEST_SYNC(TEST_SYNC_NO_FAULT);
131 GUEST_ASSERT_EQ(svm->vmcb->control.exit_code, SVM_EXIT_VMMCALL);
132 GUEST_DONE();
133 }
134
l1_guest_code(void * data)135 static void l1_guest_code(void *data)
136 {
137 if (this_cpu_has(X86_FEATURE_VMX))
138 l1_vmx_code(data);
139 else
140 l1_svm_code(data);
141 }
142
test_handle_ucall_sync(struct kvm_vm * vm,u64 arg,unsigned long * bmap)143 static void test_handle_ucall_sync(struct kvm_vm *vm, u64 arg,
144 unsigned long *bmap)
145 {
146 vm_vaddr_t gva = arg & ~(PAGE_SIZE - 1);
147 int page_nr, i;
148
149 /*
150 * Extract the page number of underlying physical page, which is also
151 * the _L1_ page number. The dirty bitmap _must_ be updated based on
152 * the L1 GPA, not L2 GPA, i.e. whether or not L2 used an aliased GPA
153 * (i.e. if TDP enabled for L2) is irrelevant with respect to the dirty
154 * bitmap and which underlying physical page is accessed.
155 *
156 * Note, gva will be '0' if there was no access, i.e. if the purpose of
157 * the sync is to verify all pages are clean.
158 */
159 if (!gva)
160 page_nr = 0;
161 else if (gva >= TEST_MEM_ALIAS_BASE)
162 page_nr = (gva - TEST_MEM_ALIAS_BASE) >> PAGE_SHIFT;
163 else
164 page_nr = (gva - TEST_MEM_BASE) >> PAGE_SHIFT;
165 TEST_ASSERT(page_nr == 0 || page_nr == 1,
166 "Test bug, unexpected frame number '%u' for arg = %lx", page_nr, arg);
167 TEST_ASSERT(gva || (arg & TEST_SYNC_NO_FAULT),
168 "Test bug, gva must be valid if a fault is expected");
169
170 kvm_vm_get_dirty_log(vm, TEST_MEM_SLOT_INDEX, bmap);
171
172 /*
173 * Check all pages to verify the correct physical page was modified (or
174 * not), and that all pages are clean/dirty as expected.
175 *
176 * If a fault of any kind is expected, the target page should be dirty
177 * as the Dirty bit is set in the gPTE. KVM should create a writable
178 * SPTE even on a read fault, *and* KVM must mark the GFN as dirty
179 * when doing so.
180 */
181 for (i = 0; i < TEST_MEM_PAGES; i++) {
182 if (i == page_nr && (arg & TEST_SYNC_WRITE_FAULT))
183 TEST_ASSERT(*(u64 *)TEST_HVA(vm, i) == 1,
184 "Page %u incorrectly not written by guest", i);
185 else
186 TEST_ASSERT(*(u64 *)TEST_HVA(vm, i) == 0xaaaaaaaaaaaaaaaaULL,
187 "Page %u incorrectly written by guest", i);
188
189 if (i == page_nr && !(arg & TEST_SYNC_NO_FAULT))
190 TEST_ASSERT(test_bit(i, bmap),
191 "Page %u incorrectly reported clean on %s fault",
192 i, arg & TEST_SYNC_READ_FAULT ? "read" : "write");
193 else
194 TEST_ASSERT(!test_bit(i, bmap),
195 "Page %u incorrectly reported dirty", i);
196 }
197 }
198
test_dirty_log(bool nested_tdp)199 static void test_dirty_log(bool nested_tdp)
200 {
201 vm_vaddr_t nested_gva = 0;
202 unsigned long *bmap;
203 struct kvm_vcpu *vcpu;
204 struct kvm_vm *vm;
205 struct ucall uc;
206 bool done = false;
207
208 pr_info("Nested TDP: %s\n", nested_tdp ? "enabled" : "disabled");
209
210 /* Create VM */
211 vm = vm_create_with_one_vcpu(&vcpu, l1_guest_code);
212 if (nested_tdp)
213 vm_enable_tdp(vm);
214
215 if (kvm_cpu_has(X86_FEATURE_VMX))
216 vcpu_alloc_vmx(vm, &nested_gva);
217 else
218 vcpu_alloc_svm(vm, &nested_gva);
219
220 vcpu_args_set(vcpu, 1, nested_gva);
221
222 /* Add an extra memory slot for testing dirty logging */
223 vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS,
224 TEST_MEM_BASE,
225 TEST_MEM_SLOT_INDEX,
226 TEST_MEM_PAGES,
227 KVM_MEM_LOG_DIRTY_PAGES);
228
229 /*
230 * Add an identity map for GVA range [0xc0000000, 0xc0004000). This
231 * affects both L1 and L2. However...
232 */
233 virt_map(vm, TEST_MEM_BASE, TEST_MEM_BASE, TEST_MEM_PAGES);
234
235 /*
236 * ... pages in the L2 GPA address range [0xc0002000, 0xc0004000) will
237 * map to [0xc0000000, 0xc0002000) when TDP is enabled (for L2).
238 *
239 * When TDP is disabled, the L2 guest code will still access the same L1
240 * GPAs as the TDP enabled case.
241 *
242 * Set the Dirty bit in the PTEs used by L2 so that KVM will create
243 * writable SPTEs when handling read faults (if the Dirty bit isn't
244 * set, KVM must intercept the next write to emulate the Dirty bit
245 * update).
246 */
247 if (nested_tdp) {
248 tdp_identity_map_default_memslots(vm);
249 tdp_map(vm, TEST_ALIAS_GPA(0), TEST_GPA(0), PAGE_SIZE);
250 tdp_map(vm, TEST_ALIAS_GPA(1), TEST_GPA(1), PAGE_SIZE);
251
252 *tdp_get_pte(vm, TEST_ALIAS_GPA(0)) |= PTE_DIRTY_MASK(&vm->stage2_mmu);
253 *tdp_get_pte(vm, TEST_ALIAS_GPA(1)) |= PTE_DIRTY_MASK(&vm->stage2_mmu);
254 } else {
255 *vm_get_pte(vm, TEST_GVA(0)) |= PTE_DIRTY_MASK(&vm->mmu);
256 *vm_get_pte(vm, TEST_GVA(1)) |= PTE_DIRTY_MASK(&vm->mmu);
257 }
258
259 bmap = bitmap_zalloc(TEST_MEM_PAGES);
260
261 while (!done) {
262 memset(TEST_HVA(vm, 0), 0xaa, TEST_MEM_PAGES * PAGE_SIZE);
263
264 vcpu_run(vcpu);
265 TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_IO);
266
267 switch (get_ucall(vcpu, &uc)) {
268 case UCALL_ABORT:
269 REPORT_GUEST_ASSERT(uc);
270 /* NOT REACHED */
271 case UCALL_SYNC:
272 test_handle_ucall_sync(vm, uc.args[1], bmap);
273 break;
274 case UCALL_DONE:
275 done = true;
276 break;
277 default:
278 TEST_FAIL("Unknown ucall %lu", uc.cmd);
279 }
280 }
281 }
282
main(int argc,char * argv[])283 int main(int argc, char *argv[])
284 {
285 TEST_REQUIRE(kvm_cpu_has(X86_FEATURE_VMX) || kvm_cpu_has(X86_FEATURE_SVM));
286
287 test_dirty_log(/*nested_tdp=*/false);
288
289 if (kvm_cpu_has_tdp())
290 test_dirty_log(/*nested_tdp=*/true);
291
292 return 0;
293 }
294