Lines Matching +full:data +full:- +full:mirror
1 // SPDX-License-Identifier: GPL-2.0
4 * the linux kernel to help device drivers mirror a process address space in
6 * makes communication and data exchange a lot easier.
38 void *mirror; member
50 #define ALIGN(x, a) (((x) + (a - 1)) & (~((a) - 1)))
82 self->page_size = sysconf(_SC_PAGE_SIZE); in FIXTURE_SETUP()
83 self->page_shift = ffs(self->page_size) - 1; in FIXTURE_SETUP()
85 self->fd = hmm_open(0); in FIXTURE_SETUP()
86 ASSERT_GE(self->fd, 0); in FIXTURE_SETUP()
91 self->page_size = sysconf(_SC_PAGE_SIZE); in FIXTURE_SETUP()
92 self->page_shift = ffs(self->page_size) - 1; in FIXTURE_SETUP()
94 self->fd0 = hmm_open(0); in FIXTURE_SETUP()
95 ASSERT_GE(self->fd0, 0); in FIXTURE_SETUP()
96 self->fd1 = hmm_open(1); in FIXTURE_SETUP()
97 ASSERT_GE(self->fd1, 0); in FIXTURE_SETUP()
102 int ret = close(self->fd); in FIXTURE_TEARDOWN()
105 self->fd = -1; in FIXTURE_TEARDOWN()
110 int ret = close(self->fd0); in FIXTURE_TEARDOWN()
113 self->fd0 = -1; in FIXTURE_TEARDOWN()
115 ret = close(self->fd1); in FIXTURE_TEARDOWN()
117 self->fd1 = -1; in FIXTURE_TEARDOWN()
129 cmd.addr = (__u64)buffer->ptr; in hmm_dmirror_cmd()
130 cmd.ptr = (__u64)buffer->mirror; in hmm_dmirror_cmd()
139 return -errno; in hmm_dmirror_cmd()
141 buffer->cpages = cmd.cpages; in hmm_dmirror_cmd()
142 buffer->faults = cmd.faults; in hmm_dmirror_cmd()
152 if (buffer->ptr) in hmm_buffer_free()
153 munmap(buffer->ptr, buffer->size); in hmm_buffer_free()
154 free(buffer->mirror); in hmm_buffer_free()
173 } while (r == -1 && errno == EINTR); in hmm_create_file()
178 return -1; in hmm_create_file()
186 static int fd = -1; in hmm_random()
230 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
232 size = npages << self->page_shift; in TEST_F()
237 buffer->fd = -1; in TEST_F()
238 buffer->size = size; in TEST_F()
239 buffer->mirror = malloc(size); in TEST_F()
240 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
242 buffer->ptr = mmap(NULL, size, in TEST_F()
245 buffer->fd, 0); in TEST_F()
246 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
252 i = 2 * self->page_size / sizeof(*ptr); in TEST_F()
253 for (ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
256 /* Set buffer permission to read-only. */ in TEST_F()
257 ret = mprotect(buffer->ptr, size, PROT_READ); in TEST_F()
261 val = *(int *)(buffer->ptr + self->page_size); in TEST_F()
265 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages); in TEST_F()
267 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
268 ASSERT_EQ(buffer->faults, 1); in TEST_F()
271 ptr = buffer->mirror; in TEST_F()
272 for (i = 0; i < 2 * self->page_size / sizeof(*ptr); ++i) in TEST_F()
293 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
295 size = npages << self->page_shift; in TEST_F()
300 buffer->fd = -1; in TEST_F()
301 buffer->size = size; in TEST_F()
302 buffer->mirror = malloc(size); in TEST_F()
303 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
305 buffer->ptr = mmap(NULL, size, in TEST_F()
308 buffer->fd, 0); in TEST_F()
309 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
312 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
315 /* Initialize mirror buffer so we can verify it isn't written. */ in TEST_F()
316 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
317 ptr[i] = -i; in TEST_F()
320 ret = mprotect(buffer->ptr, size, PROT_NONE); in TEST_F()
324 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages); in TEST_F()
325 ASSERT_EQ(ret, -EFAULT); in TEST_F()
328 ret = mprotect(buffer->ptr, size, PROT_READ); in TEST_F()
330 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
334 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
335 ASSERT_EQ(ptr[i], -i); in TEST_F()
352 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
354 size = npages << self->page_shift; in TEST_F()
359 buffer->fd = -1; in TEST_F()
360 buffer->size = size; in TEST_F()
361 buffer->mirror = malloc(size); in TEST_F()
362 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
364 buffer->ptr = mmap(NULL, size, in TEST_F()
367 buffer->fd, 0); in TEST_F()
368 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
370 /* Initialize data that the device will write to buffer->ptr. */ in TEST_F()
371 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
375 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); in TEST_F()
377 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
378 ASSERT_EQ(buffer->faults, 1); in TEST_F()
381 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
400 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
402 size = npages << self->page_shift; in TEST_F()
407 buffer->fd = -1; in TEST_F()
408 buffer->size = size; in TEST_F()
409 buffer->mirror = malloc(size); in TEST_F()
410 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
412 buffer->ptr = mmap(NULL, size, in TEST_F()
415 buffer->fd, 0); in TEST_F()
416 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
419 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, 1); in TEST_F()
421 ASSERT_EQ(buffer->cpages, 1); in TEST_F()
422 ASSERT_EQ(buffer->faults, 1); in TEST_F()
424 /* Initialize data that the device will write to buffer->ptr. */ in TEST_F()
425 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
429 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); in TEST_F()
430 ASSERT_EQ(ret, -EPERM); in TEST_F()
433 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
437 ret = mprotect(buffer->ptr, size, PROT_WRITE | PROT_READ); in TEST_F()
441 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); in TEST_F()
443 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
444 ASSERT_EQ(buffer->faults, 1); in TEST_F()
447 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
455 * will copy-on-write if a child process inherits the mapping.
468 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
470 size = npages << self->page_shift; in TEST_F()
475 buffer->fd = -1; in TEST_F()
476 buffer->size = size; in TEST_F()
477 buffer->mirror = malloc(size); in TEST_F()
478 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
480 buffer->ptr = mmap(NULL, size, in TEST_F()
483 buffer->fd, 0); in TEST_F()
484 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
486 /* Initialize buffer->ptr so we can tell if it is written. */ in TEST_F()
487 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
490 /* Initialize data that the device will write to buffer->ptr. */ in TEST_F()
491 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
492 ptr[i] = -i; in TEST_F()
495 if (pid == -1) in TEST_F()
502 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
508 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
510 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
511 ASSERT_EQ(ptr[i], -i); in TEST_F()
513 /* The child process needs its own mirror to its own mm. */ in TEST_F()
520 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
521 ASSERT_EQ(buffer->faults, 1); in TEST_F()
524 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
525 ASSERT_EQ(ptr[i], -i); in TEST_F()
533 * will not copy-on-write if a child process inherits the mapping.
546 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
548 size = npages << self->page_shift; in TEST_F()
553 buffer->fd = -1; in TEST_F()
554 buffer->size = size; in TEST_F()
555 buffer->mirror = malloc(size); in TEST_F()
556 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
558 buffer->ptr = mmap(NULL, size, in TEST_F()
561 buffer->fd, 0); in TEST_F()
562 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
564 /* Initialize buffer->ptr so we can tell if it is written. */ in TEST_F()
565 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
568 /* Initialize data that the device will write to buffer->ptr. */ in TEST_F()
569 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
570 ptr[i] = -i; in TEST_F()
573 if (pid == -1) in TEST_F()
580 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
581 ASSERT_EQ(ptr[i], -i); in TEST_F()
586 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
588 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
589 ASSERT_EQ(ptr[i], -i); in TEST_F()
591 /* The child process needs its own mirror to its own mm. */ in TEST_F()
598 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
599 ASSERT_EQ(buffer->faults, 1); in TEST_F()
602 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
603 ASSERT_EQ(ptr[i], -i); in TEST_F()
628 buffer->fd = -1; in TEST_F()
629 buffer->size = size; in TEST_F()
630 buffer->mirror = malloc(size); in TEST_F()
631 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
633 buffer->ptr = mmap(NULL, size, in TEST_F()
636 buffer->fd, 0); in TEST_F()
637 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
640 npages = size >> self->page_shift; in TEST_F()
641 map = (void *)ALIGN((uintptr_t)buffer->ptr, size); in TEST_F()
644 old_ptr = buffer->ptr; in TEST_F()
645 buffer->ptr = map; in TEST_F()
647 /* Initialize data that the device will write to buffer->ptr. */ in TEST_F()
648 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
652 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); in TEST_F()
654 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
655 ASSERT_EQ(buffer->faults, 1); in TEST_F()
658 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
661 buffer->ptr = old_ptr; in TEST_F()
684 for (idx = 0; --n > 0; ) { in TEST_F()
689 npages = size >> self->page_shift; in TEST_F()
694 buffer->ptr = get_hugepage_region(size, GHR_STRICT); in TEST_F()
695 if (buffer->ptr == NULL) { in TEST_F()
700 buffer->fd = -1; in TEST_F()
701 buffer->size = size; in TEST_F()
702 buffer->mirror = malloc(size); in TEST_F()
703 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
705 /* Initialize data that the device will write to buffer->ptr. */ in TEST_F()
706 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
710 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); in TEST_F()
712 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
713 ASSERT_EQ(buffer->faults, 1); in TEST_F()
716 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
719 free_hugepage_region(buffer->ptr); in TEST_F()
720 buffer->ptr = NULL; in TEST_F()
738 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
740 size = npages << self->page_shift; in TEST_F()
748 buffer->fd = fd; in TEST_F()
749 buffer->size = size; in TEST_F()
750 buffer->mirror = malloc(size); in TEST_F()
751 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
754 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
756 len = pwrite(fd, buffer->mirror, size, 0); in TEST_F()
758 memset(buffer->mirror, 0, size); in TEST_F()
760 buffer->ptr = mmap(NULL, size, in TEST_F()
763 buffer->fd, 0); in TEST_F()
764 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
767 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, npages); in TEST_F()
769 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
770 ASSERT_EQ(buffer->faults, 1); in TEST_F()
773 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
793 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
795 size = npages << self->page_shift; in TEST_F()
803 buffer->fd = fd; in TEST_F()
804 buffer->size = size; in TEST_F()
805 buffer->mirror = malloc(size); in TEST_F()
806 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
808 buffer->ptr = mmap(NULL, size, in TEST_F()
811 buffer->fd, 0); in TEST_F()
812 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
814 /* Initialize data that the device will write to buffer->ptr. */ in TEST_F()
815 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
819 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_WRITE, buffer, npages); in TEST_F()
821 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
822 ASSERT_EQ(buffer->faults, 1); in TEST_F()
825 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
829 len = pread(fd, buffer->mirror, size, 0); in TEST_F()
831 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
849 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
851 size = npages << self->page_shift; in TEST_F()
856 buffer->fd = -1; in TEST_F()
857 buffer->size = size; in TEST_F()
858 buffer->mirror = malloc(size); in TEST_F()
859 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
861 buffer->ptr = mmap(NULL, size, in TEST_F()
864 buffer->fd, 0); in TEST_F()
865 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
868 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
872 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_MIGRATE, buffer, npages); in TEST_F()
874 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
877 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
897 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
899 size = npages << self->page_shift; in TEST_F()
904 buffer->fd = -1; in TEST_F()
905 buffer->size = size; in TEST_F()
906 buffer->mirror = malloc(size); in TEST_F()
907 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
909 buffer->ptr = mmap(NULL, size, in TEST_F()
912 buffer->fd, 0); in TEST_F()
913 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
916 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
920 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_MIGRATE, buffer, npages); in TEST_F()
922 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
925 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
929 for (i = 0, ptr = buffer->ptr; i < size / (2 * sizeof(*ptr)); ++i) in TEST_F()
933 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_MIGRATE, buffer, npages); in TEST_F()
935 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
938 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
954 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
956 size = npages << self->page_shift; in TEST_F()
961 buffer->fd = -1; in TEST_F()
962 buffer->size = size; in TEST_F()
963 buffer->mirror = malloc(size); in TEST_F()
964 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
966 buffer->ptr = mmap(NULL, size, in TEST_F()
969 buffer->fd, 0); in TEST_F()
970 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
973 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_MIGRATE, buffer, npages); in TEST_F()
974 ASSERT_EQ(ret, -ENOENT); in TEST_F()
993 size = npages << self->page_shift; in TEST_F()
998 buffer->fd = -1; in TEST_F()
999 buffer->size = size; in TEST_F()
1000 buffer->mirror = malloc(size); in TEST_F()
1001 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
1004 buffer->ptr = mmap(NULL, size, in TEST_F()
1007 buffer->fd, 0); in TEST_F()
1008 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
1009 p = buffer->ptr; in TEST_F()
1012 ret = hmm_dmirror_cmd(self->fd1, HMM_DMIRROR_MIGRATE, buffer, npages); in TEST_F()
1013 ASSERT_EQ(ret, -EINVAL); in TEST_F()
1016 ret = munmap(buffer->ptr + self->page_size, self->page_size); in TEST_F()
1020 ret = hmm_dmirror_cmd(self->fd1, HMM_DMIRROR_MIGRATE, buffer, 3); in TEST_F()
1021 ASSERT_EQ(ret, -EINVAL); in TEST_F()
1023 /* Page 2 will be a read-only zero page. */ in TEST_F()
1024 ret = mprotect(buffer->ptr + 2 * self->page_size, self->page_size, in TEST_F()
1027 ptr = (int *)(buffer->ptr + 2 * self->page_size); in TEST_F()
1031 /* Page 3 will be read-only. */ in TEST_F()
1032 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size, in TEST_F()
1035 ptr = (int *)(buffer->ptr + 3 * self->page_size); in TEST_F()
1037 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size, in TEST_F()
1041 /* Page 4-5 will be read-write. */ in TEST_F()
1042 ret = mprotect(buffer->ptr + 4 * self->page_size, 2 * self->page_size, in TEST_F()
1045 ptr = (int *)(buffer->ptr + 4 * self->page_size); in TEST_F()
1047 ptr = (int *)(buffer->ptr + 5 * self->page_size); in TEST_F()
1050 /* Now try to migrate pages 2-5 to device 1. */ in TEST_F()
1051 buffer->ptr = p + 2 * self->page_size; in TEST_F()
1052 ret = hmm_dmirror_cmd(self->fd1, HMM_DMIRROR_MIGRATE, buffer, 4); in TEST_F()
1054 ASSERT_EQ(buffer->cpages, 4); in TEST_F()
1057 buffer->ptr = p + 5 * self->page_size; in TEST_F()
1058 ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_MIGRATE, buffer, 1); in TEST_F()
1059 ASSERT_EQ(ret, -ENOENT); in TEST_F()
1060 buffer->ptr = p; in TEST_F()
1062 buffer->ptr = p; in TEST_F()
1080 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
1082 size = npages << self->page_shift; in TEST_F()
1088 buffer->fd = -1; in TEST_F()
1089 buffer->size = size; in TEST_F()
1090 buffer->mirror = malloc(size); in TEST_F()
1091 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
1093 buffer->ptr = mmap(NULL, size, in TEST_F()
1096 buffer->fd, 0); in TEST_F()
1097 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
1100 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
1104 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_MIGRATE, buffer, in TEST_F()
1107 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1110 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
1114 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
1134 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
1136 size = npages << self->page_shift; in TEST_F()
1142 buffer->fd = -1; in TEST_F()
1143 buffer->size = size; in TEST_F()
1144 buffer->mirror = malloc(size); in TEST_F()
1145 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
1147 buffer->ptr = mmap(NULL, size, in TEST_F()
1150 buffer->fd, 0); in TEST_F()
1151 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
1154 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
1158 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, in TEST_F()
1161 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1162 ASSERT_EQ(buffer->faults, 1); in TEST_F()
1165 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
1178 munmap(buffer->ptr + buffer->size / 2, buffer->size / 2); in unmap_buffer()
1179 buffer->ptr = NULL; in unmap_buffer()
1194 npages = ALIGN(HMM_BUFFER_SIZE, self->page_size) >> self->page_shift; in TEST_F()
1196 size = npages << self->page_shift; in TEST_F()
1208 buffer->fd = -1; in TEST_F()
1209 buffer->size = size; in TEST_F()
1210 buffer->mirror = malloc(size); in TEST_F()
1211 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
1213 buffer->ptr = mmap(NULL, size, in TEST_F()
1216 buffer->fd, 0); in TEST_F()
1217 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
1220 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
1227 rc = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_READ, buffer, in TEST_F()
1230 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1231 ASSERT_EQ(buffer->faults, 1); in TEST_F()
1234 for (i = 0, ptr = buffer->mirror; in TEST_F()
1260 size = npages << self->page_shift; in TEST_F()
1265 buffer->fd = -1; in TEST_F()
1266 buffer->size = size; in TEST_F()
1267 buffer->mirror = malloc(npages); in TEST_F()
1268 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
1271 buffer->ptr = mmap(NULL, size, in TEST_F()
1274 buffer->fd, 0); in TEST_F()
1275 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
1276 p = buffer->ptr; in TEST_F()
1279 ret = munmap(buffer->ptr + self->page_size, self->page_size); in TEST_F()
1282 /* Page 2 will be read-only zero page. */ in TEST_F()
1283 ret = mprotect(buffer->ptr + 2 * self->page_size, self->page_size, in TEST_F()
1286 ptr = (int *)(buffer->ptr + 2 * self->page_size); in TEST_F()
1290 /* Page 3 will be read-only. */ in TEST_F()
1291 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size, in TEST_F()
1294 ptr = (int *)(buffer->ptr + 3 * self->page_size); in TEST_F()
1296 ret = mprotect(buffer->ptr + 3 * self->page_size, self->page_size, in TEST_F()
1300 /* Page 4-6 will be read-write. */ in TEST_F()
1301 ret = mprotect(buffer->ptr + 4 * self->page_size, 3 * self->page_size, in TEST_F()
1304 ptr = (int *)(buffer->ptr + 4 * self->page_size); in TEST_F()
1308 buffer->ptr = p + 5 * self->page_size; in TEST_F()
1309 ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_MIGRATE, buffer, 1); in TEST_F()
1311 ASSERT_EQ(buffer->cpages, 1); in TEST_F()
1314 buffer->ptr = p + 6 * self->page_size; in TEST_F()
1315 ret = hmm_dmirror_cmd(self->fd1, HMM_DMIRROR_MIGRATE, buffer, 1); in TEST_F()
1317 ASSERT_EQ(buffer->cpages, 1); in TEST_F()
1320 buffer->ptr = p; in TEST_F()
1321 ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_SNAPSHOT, buffer, npages); in TEST_F()
1323 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1326 m = buffer->mirror; in TEST_F()
1360 for (idx = 0; --n > 0; ) { in TEST_F()
1365 npages = size >> self->page_shift; in TEST_F()
1370 buffer->ptr = get_hugepage_region(size, GHR_STRICT); in TEST_F()
1371 if (buffer->ptr == NULL) { in TEST_F()
1376 buffer->size = size; in TEST_F()
1377 buffer->mirror = malloc(npages); in TEST_F()
1378 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
1380 /* Initialize the pages the device will snapshot in buffer->ptr. */ in TEST_F()
1381 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
1385 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); in TEST_F()
1387 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1390 m = buffer->mirror; in TEST_F()
1395 /* Make the region read-only. */ in TEST_F()
1396 ret = mprotect(buffer->ptr, size, PROT_READ); in TEST_F()
1400 ret = hmm_dmirror_cmd(self->fd, HMM_DMIRROR_SNAPSHOT, buffer, npages); in TEST_F()
1402 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1405 m = buffer->mirror; in TEST_F()
1410 free_hugepage_region(buffer->ptr); in TEST_F()
1411 buffer->ptr = NULL; in TEST_F()
1428 size = npages << self->page_shift; in TEST_F()
1433 buffer->fd = -1; in TEST_F()
1434 buffer->size = size; in TEST_F()
1435 buffer->mirror = malloc(npages); in TEST_F()
1436 ASSERT_NE(buffer->mirror, NULL); in TEST_F()
1439 buffer->ptr = mmap(NULL, size, in TEST_F()
1442 buffer->fd, 0); in TEST_F()
1443 ASSERT_NE(buffer->ptr, MAP_FAILED); in TEST_F()
1446 for (i = 0, ptr = buffer->ptr; i < size / sizeof(*ptr); ++i) in TEST_F()
1449 /* Make region read-only. */ in TEST_F()
1450 ret = mprotect(buffer->ptr, size, PROT_READ); in TEST_F()
1454 ret = hmm_dmirror_cmd(self->fd0, HMM_DMIRROR_READ, buffer, npages); in TEST_F()
1456 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1457 ASSERT_EQ(buffer->faults, 1); in TEST_F()
1460 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
1464 ret = hmm_dmirror_cmd(self->fd1, HMM_DMIRROR_READ, buffer, npages); in TEST_F()
1466 ASSERT_EQ(buffer->cpages, npages); in TEST_F()
1467 ASSERT_EQ(buffer->faults, 1); in TEST_F()
1470 for (i = 0, ptr = buffer->mirror; i < size / sizeof(*ptr); ++i) in TEST_F()
1474 ret = munmap(buffer->ptr + self->page_size, self->page_size); in TEST_F()