iommu/iommufd/pages.c

765 static void pfn_reader_user_init(struct pfn_reader_user *user,
768 	user->upages = NULL;
769 	user->upages_len = 0;
770 	user->upages_start = 0;
771 	user->upages_end = 0;
772 	user->locked = -1;
773 	user->gup_flags = FOLL_LONGTERM;
775 		user->gup_flags |= FOLL_WRITE;
777 	user->file = (pages->type == IOPT_ADDRESS_FILE) ? pages->file : NULL;
778 	user->ufolios = NULL;
779 	user->ufolios_len = 0;
780 	user->ufolios_next = NULL;
781 	user->ufolios_offset = 0;
784 static void pfn_reader_user_destroy(struct pfn_reader_user *user,
787 	if (user->locked != -1) {
788 		if (user->locked)
790 		if (!user->file && pages->source_mm != current->mm)
792 		user->locked = -1;
795 	kfree(user->upages);
796 	user->upages = NULL;
797 	kfree(user->ufolios);
798 	user->ufolios = NULL;
801 static long pin_memfd_pages(struct pfn_reader_user *user, unsigned long start,
807 	struct page **upages = user->upages;
809 	long nfolios = user->ufolios_len / sizeof(*user->ufolios);
816 	nfolios = memfd_pin_folios(user->file, start, end, user->ufolios,
822 	user->ufolios_next = user->ufolios;
823 	user->ufolios_offset = offset;
826 		struct folio *folio = user->ufolios[i];
853 static int pfn_reader_user_pin(struct pfn_reader_user *user,
869 	if (!user->file && !user->upages) {
871 		user->upages_len = npages * sizeof(*user->upages);
872 		user->upages = temp_kmalloc(&user->upages_len, NULL, 0);
873 		if (!user->upages)
877 	if (user->file && !user->ufolios) {
878 		user->ufolios_len = npages * sizeof(*user->ufolios);
879 		user->ufolios = temp_kmalloc(&user->ufolios_len, NULL, 0);
880 		if (!user->ufolios)
884 	if (user->locked == -1) {
890 		if (!user->file && remote_mm) {
894 		user->locked = 0;
897 	unum = user->file ? user->ufolios_len / sizeof(*user->ufolios) :
898 			    user->upages_len / sizeof(*user->upages);
904 	if (user->file) {
906 		rc = pin_memfd_pages(user, start, npages);
909 		rc = pin_user_pages_fast(uptr, npages, user->gup_flags,
910 					 user->upages);
913 		if (!user->locked) {
915 			user->locked = 1;
918 					   user->gup_flags, user->upages,
919 					   &user->locked);
927 	user->upages_start = start_index;
928 	user->upages_end = start_index + rc;
961 			       bool inc, struct pfn_reader_user *user)
966 	if (user && user->locked) {
968 		user->locked = 0;
971 	} else if ((!user || (!user->upages && !user->ufolios)) &&
989 			     bool inc, struct pfn_reader_user *user)
1003 		rc = update_mm_locked_vm(pages, npages, inc, user);
1029  * and processed. If the user lacked the limit then the error unwind will unpin
1034 static int pfn_reader_user_update_pinned(struct pfn_reader_user *user,
1054 	return iopt_pages_update_pinned(pages, npages, inc, user);
1075 	struct pfn_reader_user user;
1080 	return pfn_reader_user_update_pinned(&pfns->user, pfns->pages);
1111 	struct pfn_reader_user *user = &pfns->user;
1143 	if (start_index >= pfns->user.upages_end) {
1144 		rc = pfn_reader_user_pin(&pfns->user, pfns->pages, start_index,
1150 	npages = user->upages_end - start_index;
1151 	start_index -= user->upages_start;
1154 	if (!user->file)
1155 		batch_from_pages(&pfns->batch, user->upages + start_index,
1158 		rc = batch_from_folios(&pfns->batch, &user->ufolios_next,
1159 				       &user->ufolios_offset, npages);
1212 	pfn_reader_user_init(&pfns->user, pages);
1233 	struct pfn_reader_user *user = &pfns->user;
1235 	if (user->upages_end > pfns->batch_end_index) {
1238 		unsigned long npages = user->upages_end - pfns->batch_end_index;
1240 					    user->upages_start;
1242 		if (!user->file) {
1243 			unpin_user_pages(user->upages + start_index, npages);
1245 			long n = user->ufolios_len / sizeof(*user->ufolios);
1247 			unpin_folios(user->ufolios_next,
1248 				     user->ufolios + n - user->ufolios_next);
1251 		user->upages_end = pfns->batch_end_index;
1264 	pfn_reader_user_destroy(&pfns->user, pfns->pages);
1817 			   struct pfn_reader_user *user,
1826 		user->upages = out_pages + (cur_index - start_index);
1827 		rc = pfn_reader_user_pin(user, pages, cur_index, last_index);
1830 		cur_index = user->upages_end;
1860 	struct pfn_reader_user user;
1865 	pfn_reader_user_init(&user, pages);
1866 	user.upages_len = (last_index - start_index + 1) * sizeof(*out_pages);
1894 		rc = iopt_pages_fill(pages, &user, span.start_hole,
1907 	rc = pfn_reader_user_update_pinned(&user, pages);
1910 	user.upages = NULL;
1911 	pfn_reader_user_destroy(&user, pages);
1917 	user.upages = NULL;
1918 	pfn_reader_user_destroy(&user, pages);