1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <sys/mman.h> 4 #include <sys/prctl.h> 5 #include <sys/wait.h> 6 #include <stdbool.h> 7 #include <time.h> 8 #include <string.h> 9 #include <numa.h> 10 #include <unistd.h> 11 #include <fcntl.h> 12 #include <stdint.h> 13 #include <err.h> 14 15 #include "../kselftest.h" 16 #include <include/vdso/time64.h> 17 #include "vm_util.h" 18 19 #define KSM_SYSFS_PATH "/sys/kernel/mm/ksm/" 20 #define KSM_FP(s) (KSM_SYSFS_PATH s) 21 #define KSM_SCAN_LIMIT_SEC_DEFAULT 120 22 #define KSM_PAGE_COUNT_DEFAULT 10l 23 #define KSM_PROT_STR_DEFAULT "rw" 24 #define KSM_USE_ZERO_PAGES_DEFAULT false 25 #define KSM_MERGE_ACROSS_NODES_DEFAULT true 26 #define KSM_MERGE_TYPE_DEFAULT 0 27 #define MB (1ul << 20) 28 29 struct ksm_sysfs { 30 unsigned long max_page_sharing; 31 unsigned long merge_across_nodes; 32 unsigned long pages_to_scan; 33 unsigned long run; 34 unsigned long sleep_millisecs; 35 unsigned long stable_node_chains_prune_millisecs; 36 unsigned long use_zero_pages; 37 }; 38 39 enum ksm_merge_type { 40 KSM_MERGE_MADVISE, 41 KSM_MERGE_PRCTL, 42 KSM_MERGE_LAST = KSM_MERGE_PRCTL 43 }; 44 45 enum ksm_test_name { 46 CHECK_KSM_MERGE, 47 CHECK_KSM_UNMERGE, 48 CHECK_KSM_GET_MERGE_TYPE, 49 CHECK_KSM_ZERO_PAGE_MERGE, 50 CHECK_KSM_NUMA_MERGE, 51 KSM_MERGE_TIME, 52 KSM_MERGE_TIME_HUGE_PAGES, 53 KSM_UNMERGE_TIME, 54 KSM_COW_TIME 55 }; 56 57 int debug; 58 59 static int ksm_write_sysfs(const char *file_path, unsigned long val) 60 { 61 return write_sysfs(file_path, val); 62 } 63 64 static int ksm_read_sysfs(const char *file_path, unsigned long *val) 65 { 66 return read_sysfs(file_path, val); 67 } 68 69 static void ksm_print_sysfs(void) 70 { 71 unsigned long max_page_sharing, pages_sharing, pages_shared; 72 unsigned long full_scans, pages_unshared, pages_volatile; 73 unsigned long stable_node_chains, stable_node_dups; 74 long general_profit; 75 76 if (ksm_read_sysfs(KSM_FP("pages_shared"), &pages_shared) || 77 ksm_read_sysfs(KSM_FP("pages_sharing"), &pages_sharing) || 78 ksm_read_sysfs(KSM_FP("max_page_sharing"), &max_page_sharing) || 79 ksm_read_sysfs(KSM_FP("full_scans"), &full_scans) || 80 ksm_read_sysfs(KSM_FP("pages_unshared"), &pages_unshared) || 81 ksm_read_sysfs(KSM_FP("pages_volatile"), &pages_volatile) || 82 ksm_read_sysfs(KSM_FP("stable_node_chains"), &stable_node_chains) || 83 ksm_read_sysfs(KSM_FP("stable_node_dups"), &stable_node_dups) || 84 ksm_read_sysfs(KSM_FP("general_profit"), (unsigned long *)&general_profit)) 85 return; 86 87 printf("pages_shared : %lu\n", pages_shared); 88 printf("pages_sharing : %lu\n", pages_sharing); 89 printf("max_page_sharing : %lu\n", max_page_sharing); 90 printf("full_scans : %lu\n", full_scans); 91 printf("pages_unshared : %lu\n", pages_unshared); 92 printf("pages_volatile : %lu\n", pages_volatile); 93 printf("stable_node_chains: %lu\n", stable_node_chains); 94 printf("stable_node_dups : %lu\n", stable_node_dups); 95 printf("general_profit : %ld\n", general_profit); 96 } 97 98 static void ksm_print_procfs(void) 99 { 100 const char *file_name = "/proc/self/ksm_stat"; 101 char buffer[512]; 102 FILE *f = fopen(file_name, "r"); 103 104 if (!f) { 105 fprintf(stderr, "f %s\n", file_name); 106 perror("fopen"); 107 return; 108 } 109 110 while (fgets(buffer, sizeof(buffer), f)) 111 printf("%s", buffer); 112 113 fclose(f); 114 } 115 116 static int str_to_prot(char *prot_str) 117 { 118 int prot = 0; 119 120 if ((strchr(prot_str, 'r')) != NULL) 121 prot |= PROT_READ; 122 if ((strchr(prot_str, 'w')) != NULL) 123 prot |= PROT_WRITE; 124 if ((strchr(prot_str, 'x')) != NULL) 125 prot |= PROT_EXEC; 126 127 return prot; 128 } 129 130 static void print_help(void) 131 { 132 printf("usage: ksm_tests [-h] <test type> [-a prot] [-p page_count] [-l timeout]\n" 133 "[-z use_zero_pages] [-m merge_across_nodes] [-s size]\n"); 134 135 printf("Supported <test type>:\n" 136 " -M (page merging)\n" 137 " -Z (zero pages merging)\n" 138 " -N (merging of pages in different NUMA nodes)\n" 139 " -U (page unmerging)\n" 140 " -P evaluate merging time and speed.\n" 141 " For this test, the size of duplicated memory area (in MiB)\n" 142 " must be provided using -s option\n" 143 " -H evaluate merging time and speed of area allocated mostly with huge pages\n" 144 " For this test, the size of duplicated memory area (in MiB)\n" 145 " must be provided using -s option\n" 146 " -D evaluate unmerging time and speed when disabling KSM.\n" 147 " For this test, the size of duplicated memory area (in MiB)\n" 148 " must be provided using -s option\n" 149 " -C evaluate the time required to break COW of merged pages.\n\n"); 150 151 printf(" -a: specify the access protections of pages.\n" 152 " <prot> must be of the form [rwx].\n" 153 " Default: %s\n", KSM_PROT_STR_DEFAULT); 154 printf(" -p: specify the number of pages to test.\n" 155 " Default: %ld\n", KSM_PAGE_COUNT_DEFAULT); 156 printf(" -l: limit the maximum running time (in seconds) for a test.\n" 157 " Default: %d seconds\n", KSM_SCAN_LIMIT_SEC_DEFAULT); 158 printf(" -z: change use_zero_pages tunable\n" 159 " Default: %d\n", KSM_USE_ZERO_PAGES_DEFAULT); 160 printf(" -m: change merge_across_nodes tunable\n" 161 " Default: %d\n", KSM_MERGE_ACROSS_NODES_DEFAULT); 162 printf(" -d: turn debugging output on\n"); 163 printf(" -s: the size of duplicated memory area (in MiB)\n"); 164 printf(" -t: KSM merge type\n" 165 " Default: 0\n" 166 " 0: madvise merging\n" 167 " 1: prctl merging\n"); 168 169 exit(0); 170 } 171 172 static void *allocate_memory(void *ptr, int prot, int mapping, char data, size_t map_size) 173 { 174 void *map_ptr = mmap(ptr, map_size, PROT_WRITE, mapping, -1, 0); 175 176 if (!map_ptr) { 177 perror("mmap"); 178 return NULL; 179 } 180 memset(map_ptr, data, map_size); 181 if (mprotect(map_ptr, map_size, prot)) { 182 perror("mprotect"); 183 munmap(map_ptr, map_size); 184 return NULL; 185 } 186 187 return map_ptr; 188 } 189 190 static int ksm_do_scan(int scan_count, struct timespec start_time, int timeout) 191 { 192 struct timespec cur_time; 193 unsigned long cur_scan, init_scan; 194 195 if (ksm_read_sysfs(KSM_FP("full_scans"), &init_scan)) 196 return 1; 197 cur_scan = init_scan; 198 199 while (cur_scan < init_scan + scan_count) { 200 if (ksm_read_sysfs(KSM_FP("full_scans"), &cur_scan)) 201 return 1; 202 if (clock_gettime(CLOCK_MONOTONIC_RAW, &cur_time)) { 203 perror("clock_gettime"); 204 return 1; 205 } 206 if ((cur_time.tv_sec - start_time.tv_sec) > timeout) { 207 printf("Scan time limit exceeded\n"); 208 return 1; 209 } 210 } 211 212 return 0; 213 } 214 215 static int ksm_merge_pages(int merge_type, void *addr, size_t size, 216 struct timespec start_time, int timeout) 217 { 218 if (merge_type == KSM_MERGE_MADVISE) { 219 if (madvise(addr, size, MADV_MERGEABLE)) { 220 perror("madvise"); 221 return 1; 222 } 223 } else if (merge_type == KSM_MERGE_PRCTL) { 224 if (prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0)) { 225 perror("prctl"); 226 return 1; 227 } 228 } 229 230 if (ksm_write_sysfs(KSM_FP("run"), 1)) 231 return 1; 232 233 /* Since merging occurs only after 2 scans, make sure to get at least 2 full scans */ 234 if (ksm_do_scan(2, start_time, timeout)) 235 return 1; 236 237 return 0; 238 } 239 240 static int ksm_unmerge_pages(void *addr, size_t size, 241 struct timespec start_time, int timeout) 242 { 243 if (madvise(addr, size, MADV_UNMERGEABLE)) { 244 perror("madvise"); 245 return 1; 246 } 247 return 0; 248 } 249 250 static bool assert_ksm_pages_count(long dupl_page_count) 251 { 252 unsigned long max_page_sharing, pages_sharing, pages_shared; 253 254 if (ksm_read_sysfs(KSM_FP("pages_shared"), &pages_shared) || 255 ksm_read_sysfs(KSM_FP("pages_sharing"), &pages_sharing) || 256 ksm_read_sysfs(KSM_FP("max_page_sharing"), &max_page_sharing)) 257 return false; 258 259 if (debug) { 260 ksm_print_sysfs(); 261 ksm_print_procfs(); 262 } 263 264 /* 265 * Since there must be at least 2 pages for merging and 1 page can be 266 * shared with the limited number of pages (max_page_sharing), sometimes 267 * there are 'leftover' pages that cannot be merged. For example, if there 268 * are 11 pages and max_page_sharing = 10, then only 10 pages will be 269 * merged and the 11th page won't be affected. As a result, when the number 270 * of duplicate pages is divided by max_page_sharing and the remainder is 1, 271 * pages_shared and pages_sharing values will be equal between dupl_page_count 272 * and dupl_page_count - 1. 273 */ 274 if (dupl_page_count % max_page_sharing == 1 || dupl_page_count % max_page_sharing == 0) { 275 if (pages_shared == dupl_page_count / max_page_sharing && 276 pages_sharing == pages_shared * (max_page_sharing - 1)) 277 return true; 278 } else { 279 if (pages_shared == (dupl_page_count / max_page_sharing + 1) && 280 pages_sharing == dupl_page_count - pages_shared) 281 return true; 282 } 283 284 return false; 285 } 286 287 static int ksm_save_def(struct ksm_sysfs *ksm_sysfs) 288 { 289 if (ksm_read_sysfs(KSM_FP("max_page_sharing"), &ksm_sysfs->max_page_sharing) || 290 numa_available() ? 0 : 291 ksm_read_sysfs(KSM_FP("merge_across_nodes"), &ksm_sysfs->merge_across_nodes) || 292 ksm_read_sysfs(KSM_FP("sleep_millisecs"), &ksm_sysfs->sleep_millisecs) || 293 ksm_read_sysfs(KSM_FP("pages_to_scan"), &ksm_sysfs->pages_to_scan) || 294 ksm_read_sysfs(KSM_FP("run"), &ksm_sysfs->run) || 295 ksm_read_sysfs(KSM_FP("stable_node_chains_prune_millisecs"), 296 &ksm_sysfs->stable_node_chains_prune_millisecs) || 297 ksm_read_sysfs(KSM_FP("use_zero_pages"), &ksm_sysfs->use_zero_pages)) 298 return 1; 299 300 return 0; 301 } 302 303 static int ksm_restore(struct ksm_sysfs *ksm_sysfs) 304 { 305 if (ksm_write_sysfs(KSM_FP("max_page_sharing"), ksm_sysfs->max_page_sharing) || 306 numa_available() ? 0 : 307 ksm_write_sysfs(KSM_FP("merge_across_nodes"), ksm_sysfs->merge_across_nodes) || 308 ksm_write_sysfs(KSM_FP("pages_to_scan"), ksm_sysfs->pages_to_scan) || 309 ksm_write_sysfs(KSM_FP("run"), ksm_sysfs->run) || 310 ksm_write_sysfs(KSM_FP("sleep_millisecs"), ksm_sysfs->sleep_millisecs) || 311 ksm_write_sysfs(KSM_FP("stable_node_chains_prune_millisecs"), 312 ksm_sysfs->stable_node_chains_prune_millisecs) || 313 ksm_write_sysfs(KSM_FP("use_zero_pages"), ksm_sysfs->use_zero_pages)) 314 return 1; 315 316 return 0; 317 } 318 319 static int check_ksm_merge(int merge_type, int mapping, int prot, 320 long page_count, int timeout, size_t page_size) 321 { 322 void *map_ptr; 323 struct timespec start_time; 324 325 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 326 perror("clock_gettime"); 327 return KSFT_FAIL; 328 } 329 330 /* fill pages with the same data and merge them */ 331 map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count); 332 if (!map_ptr) 333 return KSFT_FAIL; 334 335 if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout)) 336 goto err_out; 337 338 /* verify that the right number of pages are merged */ 339 if (assert_ksm_pages_count(page_count)) { 340 printf("OK\n"); 341 munmap(map_ptr, page_size * page_count); 342 if (merge_type == KSM_MERGE_PRCTL) 343 prctl(PR_SET_MEMORY_MERGE, 0, 0, 0, 0); 344 return KSFT_PASS; 345 } 346 347 err_out: 348 printf("Not OK\n"); 349 munmap(map_ptr, page_size * page_count); 350 return KSFT_FAIL; 351 } 352 353 static int check_ksm_unmerge(int merge_type, int mapping, int prot, int timeout, size_t page_size) 354 { 355 void *map_ptr; 356 struct timespec start_time; 357 int page_count = 2; 358 359 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 360 perror("clock_gettime"); 361 return KSFT_FAIL; 362 } 363 364 /* fill pages with the same data and merge them */ 365 map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count); 366 if (!map_ptr) 367 return KSFT_FAIL; 368 369 if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout)) 370 goto err_out; 371 372 /* change 1 byte in each of the 2 pages -- KSM must automatically unmerge them */ 373 memset(map_ptr, '-', 1); 374 memset(map_ptr + page_size, '+', 1); 375 376 /* get at least 1 scan, so KSM can detect that the pages were modified */ 377 if (ksm_do_scan(1, start_time, timeout)) 378 goto err_out; 379 380 /* check that unmerging was successful and 0 pages are currently merged */ 381 if (assert_ksm_pages_count(0)) { 382 printf("OK\n"); 383 munmap(map_ptr, page_size * page_count); 384 return KSFT_PASS; 385 } 386 387 err_out: 388 printf("Not OK\n"); 389 munmap(map_ptr, page_size * page_count); 390 return KSFT_FAIL; 391 } 392 393 static int check_ksm_zero_page_merge(int merge_type, int mapping, int prot, long page_count, 394 int timeout, bool use_zero_pages, size_t page_size) 395 { 396 void *map_ptr; 397 struct timespec start_time; 398 399 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 400 perror("clock_gettime"); 401 return KSFT_FAIL; 402 } 403 404 if (ksm_write_sysfs(KSM_FP("use_zero_pages"), use_zero_pages)) 405 return KSFT_FAIL; 406 407 /* fill pages with zero and try to merge them */ 408 map_ptr = allocate_memory(NULL, prot, mapping, 0, page_size * page_count); 409 if (!map_ptr) 410 return KSFT_FAIL; 411 412 if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout)) 413 goto err_out; 414 415 /* 416 * verify that the right number of pages are merged: 417 * 1) if use_zero_pages is set to 1, empty pages are merged 418 * with the kernel zero page instead of with each other; 419 * 2) if use_zero_pages is set to 0, empty pages are not treated specially 420 * and merged as usual. 421 */ 422 if (use_zero_pages && !assert_ksm_pages_count(0)) 423 goto err_out; 424 else if (!use_zero_pages && !assert_ksm_pages_count(page_count)) 425 goto err_out; 426 427 printf("OK\n"); 428 munmap(map_ptr, page_size * page_count); 429 return KSFT_PASS; 430 431 err_out: 432 printf("Not OK\n"); 433 munmap(map_ptr, page_size * page_count); 434 return KSFT_FAIL; 435 } 436 437 static int get_next_mem_node(int node) 438 { 439 440 long node_size; 441 int mem_node = 0; 442 int i, max_node = numa_max_node(); 443 444 for (i = node + 1; i <= max_node + node; i++) { 445 mem_node = i % (max_node + 1); 446 node_size = numa_node_size(mem_node, NULL); 447 if (node_size > 0) 448 break; 449 } 450 return mem_node; 451 } 452 453 static int get_first_mem_node(void) 454 { 455 return get_next_mem_node(numa_max_node()); 456 } 457 458 static int check_ksm_numa_merge(int merge_type, int mapping, int prot, int timeout, 459 bool merge_across_nodes, size_t page_size) 460 { 461 void *numa1_map_ptr, *numa2_map_ptr; 462 struct timespec start_time; 463 int page_count = 2; 464 int first_node; 465 466 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 467 perror("clock_gettime"); 468 return KSFT_FAIL; 469 } 470 471 if (numa_available() < 0) { 472 perror("NUMA support not enabled"); 473 return KSFT_SKIP; 474 } 475 if (numa_num_configured_nodes() <= 1) { 476 printf("At least 2 NUMA nodes must be available\n"); 477 return KSFT_SKIP; 478 } 479 if (ksm_write_sysfs(KSM_FP("merge_across_nodes"), merge_across_nodes)) 480 return KSFT_FAIL; 481 482 /* allocate 2 pages in 2 different NUMA nodes and fill them with the same data */ 483 first_node = get_first_mem_node(); 484 numa1_map_ptr = numa_alloc_onnode(page_size, first_node); 485 numa2_map_ptr = numa_alloc_onnode(page_size, get_next_mem_node(first_node)); 486 if (!numa1_map_ptr || !numa2_map_ptr) { 487 perror("numa_alloc_onnode"); 488 return KSFT_FAIL; 489 } 490 491 memset(numa1_map_ptr, '*', page_size); 492 memset(numa2_map_ptr, '*', page_size); 493 494 /* try to merge the pages */ 495 if (ksm_merge_pages(merge_type, numa1_map_ptr, page_size, start_time, timeout) || 496 ksm_merge_pages(merge_type, numa2_map_ptr, page_size, start_time, timeout)) 497 goto err_out; 498 499 /* 500 * verify that the right number of pages are merged: 501 * 1) if merge_across_nodes was enabled, 2 duplicate pages will be merged; 502 * 2) if merge_across_nodes = 0, there must be 0 merged pages, since there is 503 * only 1 unique page in each node and they can't be shared. 504 */ 505 if (merge_across_nodes && !assert_ksm_pages_count(page_count)) 506 goto err_out; 507 else if (!merge_across_nodes && !assert_ksm_pages_count(0)) 508 goto err_out; 509 510 numa_free(numa1_map_ptr, page_size); 511 numa_free(numa2_map_ptr, page_size); 512 printf("OK\n"); 513 return KSFT_PASS; 514 515 err_out: 516 numa_free(numa1_map_ptr, page_size); 517 numa_free(numa2_map_ptr, page_size); 518 printf("Not OK\n"); 519 return KSFT_FAIL; 520 } 521 522 static int ksm_merge_hugepages_time(int merge_type, int mapping, int prot, 523 int timeout, size_t map_size) 524 { 525 void *map_ptr, *map_ptr_orig; 526 struct timespec start_time, end_time; 527 unsigned long scan_time_ns; 528 int pagemap_fd, n_normal_pages, n_huge_pages; 529 530 map_size *= MB; 531 size_t len = map_size; 532 533 len -= len % HPAGE_SIZE; 534 map_ptr_orig = mmap(NULL, len + HPAGE_SIZE, PROT_READ | PROT_WRITE, 535 MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE, -1, 0); 536 map_ptr = map_ptr_orig + HPAGE_SIZE - (uintptr_t)map_ptr_orig % HPAGE_SIZE; 537 538 if (map_ptr_orig == MAP_FAILED) 539 err(2, "initial mmap"); 540 541 if (madvise(map_ptr, len, MADV_HUGEPAGE)) 542 err(2, "MADV_HUGEPAGE"); 543 544 pagemap_fd = open("/proc/self/pagemap", O_RDONLY); 545 if (pagemap_fd < 0) 546 err(2, "open pagemap"); 547 548 n_normal_pages = 0; 549 n_huge_pages = 0; 550 for (void *p = map_ptr; p < map_ptr + len; p += HPAGE_SIZE) { 551 if (allocate_transhuge(p, pagemap_fd) < 0) 552 n_normal_pages++; 553 else 554 n_huge_pages++; 555 } 556 printf("Number of normal pages: %d\n", n_normal_pages); 557 printf("Number of huge pages: %d\n", n_huge_pages); 558 559 memset(map_ptr, '*', len); 560 561 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 562 perror("clock_gettime"); 563 goto err_out; 564 } 565 if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout)) 566 goto err_out; 567 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { 568 perror("clock_gettime"); 569 goto err_out; 570 } 571 572 scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + 573 (end_time.tv_nsec - start_time.tv_nsec); 574 575 printf("Total size: %lu MiB\n", map_size / MB); 576 printf("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC, 577 scan_time_ns % NSEC_PER_SEC); 578 printf("Average speed: %.3f MiB/s\n", (map_size / MB) / 579 ((double)scan_time_ns / NSEC_PER_SEC)); 580 581 munmap(map_ptr_orig, len + HPAGE_SIZE); 582 return KSFT_PASS; 583 584 err_out: 585 printf("Not OK\n"); 586 munmap(map_ptr_orig, len + HPAGE_SIZE); 587 return KSFT_FAIL; 588 } 589 590 static int ksm_merge_time(int merge_type, int mapping, int prot, int timeout, size_t map_size) 591 { 592 void *map_ptr; 593 struct timespec start_time, end_time; 594 unsigned long scan_time_ns; 595 596 map_size *= MB; 597 598 map_ptr = allocate_memory(NULL, prot, mapping, '*', map_size); 599 if (!map_ptr) 600 return KSFT_FAIL; 601 602 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 603 perror("clock_gettime"); 604 goto err_out; 605 } 606 if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout)) 607 goto err_out; 608 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { 609 perror("clock_gettime"); 610 goto err_out; 611 } 612 613 scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + 614 (end_time.tv_nsec - start_time.tv_nsec); 615 616 printf("Total size: %lu MiB\n", map_size / MB); 617 printf("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC, 618 scan_time_ns % NSEC_PER_SEC); 619 printf("Average speed: %.3f MiB/s\n", (map_size / MB) / 620 ((double)scan_time_ns / NSEC_PER_SEC)); 621 622 munmap(map_ptr, map_size); 623 return KSFT_PASS; 624 625 err_out: 626 printf("Not OK\n"); 627 munmap(map_ptr, map_size); 628 return KSFT_FAIL; 629 } 630 631 static int ksm_unmerge_time(int merge_type, int mapping, int prot, int timeout, size_t map_size) 632 { 633 void *map_ptr; 634 struct timespec start_time, end_time; 635 unsigned long scan_time_ns; 636 637 map_size *= MB; 638 639 map_ptr = allocate_memory(NULL, prot, mapping, '*', map_size); 640 if (!map_ptr) 641 return KSFT_FAIL; 642 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 643 perror("clock_gettime"); 644 goto err_out; 645 } 646 if (ksm_merge_pages(merge_type, map_ptr, map_size, start_time, timeout)) 647 goto err_out; 648 649 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 650 perror("clock_gettime"); 651 goto err_out; 652 } 653 if (ksm_unmerge_pages(map_ptr, map_size, start_time, timeout)) 654 goto err_out; 655 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { 656 perror("clock_gettime"); 657 goto err_out; 658 } 659 660 scan_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + 661 (end_time.tv_nsec - start_time.tv_nsec); 662 663 printf("Total size: %lu MiB\n", map_size / MB); 664 printf("Total time: %ld.%09ld s\n", scan_time_ns / NSEC_PER_SEC, 665 scan_time_ns % NSEC_PER_SEC); 666 printf("Average speed: %.3f MiB/s\n", (map_size / MB) / 667 ((double)scan_time_ns / NSEC_PER_SEC)); 668 669 munmap(map_ptr, map_size); 670 return KSFT_PASS; 671 672 err_out: 673 printf("Not OK\n"); 674 munmap(map_ptr, map_size); 675 return KSFT_FAIL; 676 } 677 678 static int ksm_cow_time(int merge_type, int mapping, int prot, int timeout, size_t page_size) 679 { 680 void *map_ptr; 681 struct timespec start_time, end_time; 682 unsigned long cow_time_ns; 683 684 /* page_count must be less than 2*page_size */ 685 size_t page_count = 4000; 686 687 map_ptr = allocate_memory(NULL, prot, mapping, '*', page_size * page_count); 688 if (!map_ptr) 689 return KSFT_FAIL; 690 691 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 692 perror("clock_gettime"); 693 return KSFT_FAIL; 694 } 695 for (size_t i = 0; i < page_count - 1; i = i + 2) 696 memset(map_ptr + page_size * i, '-', 1); 697 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { 698 perror("clock_gettime"); 699 return KSFT_FAIL; 700 } 701 702 cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + 703 (end_time.tv_nsec - start_time.tv_nsec); 704 705 printf("Total size: %lu MiB\n\n", (page_size * page_count) / MB); 706 printf("Not merged pages:\n"); 707 printf("Total time: %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC, 708 cow_time_ns % NSEC_PER_SEC); 709 printf("Average speed: %.3f MiB/s\n\n", ((page_size * (page_count / 2)) / MB) / 710 ((double)cow_time_ns / NSEC_PER_SEC)); 711 712 /* Create 2000 pairs of duplicate pages */ 713 for (size_t i = 0; i < page_count - 1; i = i + 2) { 714 memset(map_ptr + page_size * i, '+', i / 2 + 1); 715 memset(map_ptr + page_size * (i + 1), '+', i / 2 + 1); 716 } 717 if (ksm_merge_pages(merge_type, map_ptr, page_size * page_count, start_time, timeout)) 718 goto err_out; 719 720 if (clock_gettime(CLOCK_MONOTONIC_RAW, &start_time)) { 721 perror("clock_gettime"); 722 goto err_out; 723 } 724 for (size_t i = 0; i < page_count - 1; i = i + 2) 725 memset(map_ptr + page_size * i, '-', 1); 726 if (clock_gettime(CLOCK_MONOTONIC_RAW, &end_time)) { 727 perror("clock_gettime"); 728 goto err_out; 729 } 730 731 cow_time_ns = (end_time.tv_sec - start_time.tv_sec) * NSEC_PER_SEC + 732 (end_time.tv_nsec - start_time.tv_nsec); 733 734 printf("Merged pages:\n"); 735 printf("Total time: %ld.%09ld s\n", cow_time_ns / NSEC_PER_SEC, 736 cow_time_ns % NSEC_PER_SEC); 737 printf("Average speed: %.3f MiB/s\n", ((page_size * (page_count / 2)) / MB) / 738 ((double)cow_time_ns / NSEC_PER_SEC)); 739 740 munmap(map_ptr, page_size * page_count); 741 return KSFT_PASS; 742 743 err_out: 744 printf("Not OK\n"); 745 munmap(map_ptr, page_size * page_count); 746 return KSFT_FAIL; 747 } 748 749 int main(int argc, char *argv[]) 750 { 751 int ret = 0, opt; 752 int prot = 0; 753 int ksm_scan_limit_sec = KSM_SCAN_LIMIT_SEC_DEFAULT; 754 int merge_type = KSM_MERGE_TYPE_DEFAULT; 755 long page_count = KSM_PAGE_COUNT_DEFAULT; 756 size_t page_size = sysconf(_SC_PAGESIZE); 757 struct ksm_sysfs ksm_sysfs_old; 758 int test_name = CHECK_KSM_MERGE; 759 bool use_zero_pages = KSM_USE_ZERO_PAGES_DEFAULT; 760 bool merge_across_nodes = KSM_MERGE_ACROSS_NODES_DEFAULT; 761 long size_MB = 0; 762 763 while ((opt = getopt(argc, argv, "dha:p:l:z:m:s:t:MUZNPCHD")) != -1) { 764 switch (opt) { 765 case 'a': 766 prot = str_to_prot(optarg); 767 break; 768 case 'p': 769 page_count = atol(optarg); 770 if (page_count <= 0) { 771 printf("The number of pages must be greater than 0\n"); 772 return KSFT_FAIL; 773 } 774 break; 775 case 'l': 776 ksm_scan_limit_sec = atoi(optarg); 777 if (ksm_scan_limit_sec <= 0) { 778 printf("Timeout value must be greater than 0\n"); 779 return KSFT_FAIL; 780 } 781 break; 782 case 'h': 783 print_help(); 784 break; 785 case 'z': 786 if (strcmp(optarg, "0") == 0) 787 use_zero_pages = 0; 788 else 789 use_zero_pages = 1; 790 break; 791 case 'm': 792 if (strcmp(optarg, "0") == 0) 793 merge_across_nodes = 0; 794 else 795 merge_across_nodes = 1; 796 break; 797 case 'd': 798 debug = 1; 799 break; 800 case 's': 801 size_MB = atoi(optarg); 802 if (size_MB <= 0) { 803 printf("Size must be greater than 0\n"); 804 return KSFT_FAIL; 805 } 806 break; 807 case 't': 808 { 809 int tmp = atoi(optarg); 810 811 if (tmp < 0 || tmp > KSM_MERGE_LAST) { 812 printf("Invalid merge type\n"); 813 return KSFT_FAIL; 814 } 815 merge_type = tmp; 816 } 817 break; 818 case 'M': 819 break; 820 case 'U': 821 test_name = CHECK_KSM_UNMERGE; 822 break; 823 case 'Z': 824 test_name = CHECK_KSM_ZERO_PAGE_MERGE; 825 break; 826 case 'N': 827 test_name = CHECK_KSM_NUMA_MERGE; 828 break; 829 case 'P': 830 test_name = KSM_MERGE_TIME; 831 break; 832 case 'H': 833 test_name = KSM_MERGE_TIME_HUGE_PAGES; 834 break; 835 case 'D': 836 test_name = KSM_UNMERGE_TIME; 837 break; 838 case 'C': 839 test_name = KSM_COW_TIME; 840 break; 841 default: 842 return KSFT_FAIL; 843 } 844 } 845 846 if (prot == 0) 847 prot = str_to_prot(KSM_PROT_STR_DEFAULT); 848 849 if (access(KSM_SYSFS_PATH, F_OK)) { 850 printf("Config KSM not enabled\n"); 851 return KSFT_SKIP; 852 } 853 854 if (ksm_save_def(&ksm_sysfs_old)) { 855 printf("Cannot save default tunables\n"); 856 return KSFT_FAIL; 857 } 858 859 if (ksm_write_sysfs(KSM_FP("run"), 2) || 860 ksm_write_sysfs(KSM_FP("sleep_millisecs"), 0) || 861 numa_available() ? 0 : 862 ksm_write_sysfs(KSM_FP("merge_across_nodes"), 1) || 863 ksm_write_sysfs(KSM_FP("pages_to_scan"), page_count)) 864 return KSFT_FAIL; 865 866 switch (test_name) { 867 case CHECK_KSM_MERGE: 868 ret = check_ksm_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, page_count, 869 ksm_scan_limit_sec, page_size); 870 break; 871 case CHECK_KSM_UNMERGE: 872 ret = check_ksm_unmerge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 873 ksm_scan_limit_sec, page_size); 874 break; 875 case CHECK_KSM_ZERO_PAGE_MERGE: 876 ret = check_ksm_zero_page_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 877 page_count, ksm_scan_limit_sec, use_zero_pages, 878 page_size); 879 break; 880 case CHECK_KSM_NUMA_MERGE: 881 ret = check_ksm_numa_merge(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 882 ksm_scan_limit_sec, merge_across_nodes, page_size); 883 break; 884 case KSM_MERGE_TIME: 885 if (size_MB == 0) { 886 printf("Option '-s' is required.\n"); 887 return KSFT_FAIL; 888 } 889 ret = ksm_merge_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 890 ksm_scan_limit_sec, size_MB); 891 break; 892 case KSM_MERGE_TIME_HUGE_PAGES: 893 if (size_MB == 0) { 894 printf("Option '-s' is required.\n"); 895 return KSFT_FAIL; 896 } 897 ret = ksm_merge_hugepages_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 898 ksm_scan_limit_sec, size_MB); 899 break; 900 case KSM_UNMERGE_TIME: 901 if (size_MB == 0) { 902 printf("Option '-s' is required.\n"); 903 return KSFT_FAIL; 904 } 905 ret = ksm_unmerge_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 906 ksm_scan_limit_sec, size_MB); 907 break; 908 case KSM_COW_TIME: 909 ret = ksm_cow_time(merge_type, MAP_PRIVATE | MAP_ANONYMOUS, prot, 910 ksm_scan_limit_sec, page_size); 911 break; 912 } 913 914 if (ksm_restore(&ksm_sysfs_old)) { 915 printf("Cannot restore default tunables\n"); 916 return KSFT_FAIL; 917 } 918 919 return ret; 920 } 921