1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * KVM dirty logging page splitting test
4 *
5 * Based on dirty_log_perf.c
6 *
7 * Copyright (C) 2018, Red Hat, Inc.
8 * Copyright (C) 2023, Google, Inc.
9 */
10
11 #include <stdio.h>
12 #include <stdlib.h>
13 #include <pthread.h>
14 #include <linux/bitmap.h>
15
16 #include "kvm_util.h"
17 #include "test_util.h"
18 #include "memstress.h"
19 #include "guest_modes.h"
20
21 #define VCPUS 2
22 #define SLOTS 2
23 #define ITERATIONS 2
24
25 static uint64_t guest_percpu_mem_size = DEFAULT_PER_VCPU_MEM_SIZE;
26
27 static enum vm_mem_backing_src_type backing_src = VM_MEM_SRC_ANONYMOUS_HUGETLB;
28
29 static u64 dirty_log_manual_caps;
30 static bool host_quit;
31 static int iteration;
32 static int vcpu_last_completed_iteration[KVM_MAX_VCPUS];
33
34 struct kvm_page_stats {
35 uint64_t pages_4k;
36 uint64_t pages_2m;
37 uint64_t pages_1g;
38 uint64_t hugepages;
39 };
40
get_page_stats(struct kvm_vm * vm,struct kvm_page_stats * stats,const char * stage)41 static void get_page_stats(struct kvm_vm *vm, struct kvm_page_stats *stats, const char *stage)
42 {
43 stats->pages_4k = vm_get_stat(vm, "pages_4k");
44 stats->pages_2m = vm_get_stat(vm, "pages_2m");
45 stats->pages_1g = vm_get_stat(vm, "pages_1g");
46 stats->hugepages = stats->pages_2m + stats->pages_1g;
47
48 pr_debug("\nPage stats after %s: 4K: %ld 2M: %ld 1G: %ld huge: %ld\n",
49 stage, stats->pages_4k, stats->pages_2m, stats->pages_1g,
50 stats->hugepages);
51 }
52
run_vcpu_iteration(struct kvm_vm * vm)53 static void run_vcpu_iteration(struct kvm_vm *vm)
54 {
55 int i;
56
57 iteration++;
58 for (i = 0; i < VCPUS; i++) {
59 while (READ_ONCE(vcpu_last_completed_iteration[i]) !=
60 iteration)
61 ;
62 }
63 }
64
vcpu_worker(struct memstress_vcpu_args * vcpu_args)65 static void vcpu_worker(struct memstress_vcpu_args *vcpu_args)
66 {
67 struct kvm_vcpu *vcpu = vcpu_args->vcpu;
68 int vcpu_idx = vcpu_args->vcpu_idx;
69
70 while (!READ_ONCE(host_quit)) {
71 int current_iteration = READ_ONCE(iteration);
72
73 vcpu_run(vcpu);
74
75 TEST_ASSERT_EQ(get_ucall(vcpu, NULL), UCALL_SYNC);
76
77 vcpu_last_completed_iteration[vcpu_idx] = current_iteration;
78
79 /* Wait for the start of the next iteration to be signaled. */
80 while (current_iteration == READ_ONCE(iteration) &&
81 READ_ONCE(iteration) >= 0 &&
82 !READ_ONCE(host_quit))
83 ;
84 }
85 }
86
run_test(enum vm_guest_mode mode,void * unused)87 static void run_test(enum vm_guest_mode mode, void *unused)
88 {
89 struct kvm_vm *vm;
90 unsigned long **bitmaps;
91 uint64_t guest_num_pages;
92 uint64_t host_num_pages;
93 uint64_t pages_per_slot;
94 int i;
95 struct kvm_page_stats stats_populated;
96 struct kvm_page_stats stats_dirty_logging_enabled;
97 struct kvm_page_stats stats_dirty_pass[ITERATIONS];
98 struct kvm_page_stats stats_clear_pass[ITERATIONS];
99 struct kvm_page_stats stats_dirty_logging_disabled;
100 struct kvm_page_stats stats_repopulated;
101
102 vm = memstress_create_vm(mode, VCPUS, guest_percpu_mem_size,
103 SLOTS, backing_src, false);
104
105 guest_num_pages = (VCPUS * guest_percpu_mem_size) >> vm->page_shift;
106 guest_num_pages = vm_adjust_num_guest_pages(mode, guest_num_pages);
107 host_num_pages = vm_num_host_pages(mode, guest_num_pages);
108 pages_per_slot = host_num_pages / SLOTS;
109 TEST_ASSERT_EQ(host_num_pages, pages_per_slot * SLOTS);
110 TEST_ASSERT(!(host_num_pages % 512),
111 "Number of pages, '%lu' not a multiple of 2MiB", host_num_pages);
112
113 bitmaps = memstress_alloc_bitmaps(SLOTS, pages_per_slot);
114
115 if (dirty_log_manual_caps)
116 vm_enable_cap(vm, KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2,
117 dirty_log_manual_caps);
118
119 /* Start the iterations */
120 iteration = -1;
121 host_quit = false;
122
123 for (i = 0; i < VCPUS; i++)
124 vcpu_last_completed_iteration[i] = -1;
125
126 memstress_start_vcpu_threads(VCPUS, vcpu_worker);
127
128 run_vcpu_iteration(vm);
129 get_page_stats(vm, &stats_populated, "populating memory");
130
131 /* Enable dirty logging */
132 memstress_enable_dirty_logging(vm, SLOTS);
133
134 get_page_stats(vm, &stats_dirty_logging_enabled, "enabling dirty logging");
135
136 while (iteration < ITERATIONS) {
137 run_vcpu_iteration(vm);
138 get_page_stats(vm, &stats_dirty_pass[iteration - 1],
139 "dirtying memory");
140
141 memstress_get_dirty_log(vm, bitmaps, SLOTS);
142
143 if (dirty_log_manual_caps) {
144 memstress_clear_dirty_log(vm, bitmaps, SLOTS, pages_per_slot);
145
146 get_page_stats(vm, &stats_clear_pass[iteration - 1], "clearing dirty log");
147 }
148 }
149
150 /* Disable dirty logging */
151 memstress_disable_dirty_logging(vm, SLOTS);
152
153 get_page_stats(vm, &stats_dirty_logging_disabled, "disabling dirty logging");
154
155 /* Run vCPUs again to fault pages back in. */
156 run_vcpu_iteration(vm);
157 get_page_stats(vm, &stats_repopulated, "repopulating memory");
158
159 /*
160 * Tell the vCPU threads to quit. No need to manually check that vCPUs
161 * have stopped running after disabling dirty logging, the join will
162 * wait for them to exit.
163 */
164 host_quit = true;
165 memstress_join_vcpu_threads(VCPUS);
166
167 memstress_free_bitmaps(bitmaps, SLOTS);
168 memstress_destroy_vm(vm);
169
170 TEST_ASSERT_EQ((stats_populated.pages_2m * 512 +
171 stats_populated.pages_1g * 512 * 512), host_num_pages);
172
173 /*
174 * Check that all huge pages were split. Since large pages can only
175 * exist in the data slot, and the vCPUs should have dirtied all pages
176 * in the data slot, there should be no huge pages left after splitting.
177 * Splitting happens at dirty log enable time without
178 * KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2 and after the first clear pass
179 * with that capability.
180 */
181 if (dirty_log_manual_caps) {
182 TEST_ASSERT_EQ(stats_clear_pass[0].hugepages, 0);
183 TEST_ASSERT(stats_clear_pass[0].pages_4k >= host_num_pages,
184 "Expected at least '%lu' 4KiB pages, found only '%lu'",
185 host_num_pages, stats_clear_pass[0].pages_4k);
186 TEST_ASSERT_EQ(stats_dirty_logging_enabled.hugepages, stats_populated.hugepages);
187 } else {
188 TEST_ASSERT_EQ(stats_dirty_logging_enabled.hugepages, 0);
189 TEST_ASSERT(stats_dirty_logging_enabled.pages_4k >= host_num_pages,
190 "Expected at least '%lu' 4KiB pages, found only '%lu'",
191 host_num_pages, stats_dirty_logging_enabled.pages_4k);
192 }
193
194 /*
195 * Once dirty logging is disabled and the vCPUs have touched all their
196 * memory again, the hugepage counts should be the same as they were
197 * right after initial population of memory.
198 */
199 TEST_ASSERT_EQ(stats_populated.pages_2m, stats_repopulated.pages_2m);
200 TEST_ASSERT_EQ(stats_populated.pages_1g, stats_repopulated.pages_1g);
201 }
202
help(char * name)203 static void help(char *name)
204 {
205 puts("");
206 printf("usage: %s [-h] [-b vcpu bytes] [-s mem type]\n",
207 name);
208 puts("");
209 printf(" -b: specify the size of the memory region which should be\n"
210 " dirtied by each vCPU. e.g. 10M or 3G.\n"
211 " (default: 1G)\n");
212 backing_src_help("-s");
213 puts("");
214 }
215
main(int argc,char * argv[])216 int main(int argc, char *argv[])
217 {
218 int opt;
219
220 TEST_REQUIRE(get_kvm_param_bool("eager_page_split"));
221 TEST_REQUIRE(get_kvm_param_bool("tdp_mmu"));
222
223 while ((opt = getopt(argc, argv, "b:hs:")) != -1) {
224 switch (opt) {
225 case 'b':
226 guest_percpu_mem_size = parse_size(optarg);
227 break;
228 case 'h':
229 help(argv[0]);
230 exit(0);
231 case 's':
232 backing_src = parse_backing_src_type(optarg);
233 break;
234 default:
235 help(argv[0]);
236 exit(1);
237 }
238 }
239
240 if (!is_backing_src_hugetlb(backing_src)) {
241 pr_info("This test will only work reliably with HugeTLB memory. "
242 "It can work with THP, but that is best effort.\n");
243 }
244
245 guest_modes_append_default();
246
247 dirty_log_manual_caps = 0;
248 for_each_guest_mode(run_test, NULL);
249
250 dirty_log_manual_caps =
251 kvm_check_cap(KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2);
252
253 if (dirty_log_manual_caps) {
254 dirty_log_manual_caps &= (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE |
255 KVM_DIRTY_LOG_INITIALLY_SET);
256 for_each_guest_mode(run_test, NULL);
257 } else {
258 pr_info("Skipping testing with MANUAL_PROTECT as it is not supported");
259 }
260
261 return 0;
262 }
263