1 // SPDX-License-Identifier: GPL-2.0-only
4 #include <linux/fault-inject-usercopy.h>
78  * fault_in_iov_iter_readable - fault in iov iterator for reading
80  * @size: maximum length
83  * @size.  For each iovec, fault in each page that constitutes the iovec.
88  * Always returns 0 for non-userspace iterators.
90 size_t fault_in_iov_iter_readable(const struct iov_iter *i, size_t size)  in fault_in_iov_iter_readable()  argument
93 		size_t n = min(size, iov_iter_count(i));  in fault_in_iov_iter_readable()
94 		n -= fault_in_readable(i->ubuf + i->iov_offset, n);  in fault_in_iov_iter_readable()
95 		return size - n;  in fault_in_iov_iter_readable()
97 		size_t count = min(size, iov_iter_count(i));  in fault_in_iov_iter_readable()
101 		size -= count;  in fault_in_iov_iter_readable()
102 		for (p = iter_iov(i), skip = i->iov_offset; count; p++, skip = 0) {  in fault_in_iov_iter_readable()
103 			size_t len = min(count, p->iov_len - skip);  in fault_in_iov_iter_readable()
108 			ret = fault_in_readable(p->iov_base + skip, len);  in fault_in_iov_iter_readable()
109 			count -= len - ret;  in fault_in_iov_iter_readable()
113 		return count + size;  in fault_in_iov_iter_readable()
120  * fault_in_iov_iter_writeable - fault in iov iterator for writing
122  * @size: maximum length
125  * hardware page faults.  This is primarily useful when we already know that
131  * Always returns 0 for non-user-space iterators.
133 size_t fault_in_iov_iter_writeable(const struct iov_iter *i, size_t size)  in fault_in_iov_iter_writeable()  argument
136 		size_t n = min(size, iov_iter_count(i));  in fault_in_iov_iter_writeable()
137 		n -= fault_in_safe_writeable(i->ubuf + i->iov_offset, n);  in fault_in_iov_iter_writeable()
138 		return size - n;  in fault_in_iov_iter_writeable()
140 		size_t count = min(size, iov_iter_count(i));  in fault_in_iov_iter_writeable()
144 		size -= count;  in fault_in_iov_iter_writeable()
145 		for (p = iter_iov(i), skip = i->iov_offset; count; p++, skip = 0) {  in fault_in_iov_iter_writeable()
146 			size_t len = min(count, p->iov_len - skip);  in fault_in_iov_iter_writeable()
151 			ret = fault_in_safe_writeable(p->iov_base + skip, len);  in fault_in_iov_iter_writeable()
152 			count -= len - ret;  in fault_in_iov_iter_writeable()
156 		return count + size;  in fault_in_iov_iter_writeable()
181 	if (WARN_ON_ONCE(i->data_source))  in _copy_to_iter()
211  * _copy_mc_to_iter - copy to iter with source memory error exception handling
217  * (dax_copy_to_iter()) for dax reads (bypass page-cache and the
218  * block-layer). Upon #MC read(2) aborts and returns EIO or the bytes
224  *   byte-by-byte until the fault happens again. Re-triggering machine
226  *   alignment and poison alignment assumptions to avoid re-triggering
236 	if (WARN_ON_ONCE(i->data_source))  in _copy_mc_to_iter()
255 	if (WARN_ON_ONCE(!i->data_source))  in _copy_from_iter()
273 	if (WARN_ON_ONCE(!i->data_source))  in _copy_from_iter_nocache()
299  * _copy_from_iter_flushcache - write destination through cpu cache
304  * The pmem driver arranges for filesystem-dax to use this facility via
310  * instructions that strand dirty-data in the cache.
316 	if (WARN_ON_ONCE(!i->data_source))  in _copy_from_iter_flushcache()
326 static inline bool page_copy_sane(struct page *page, size_t offset, size_t n)  in page_copy_sane()  argument
328 	struct page *head;  in page_copy_sane()
332 	 * The general case needs to access the page order in order  in page_copy_sane()
333 	 * to compute the page size.  in page_copy_sane()
334 	 * However, we mostly deal with order-0 pages and thus can  in page_copy_sane()
336 	 * page orders.  in page_copy_sane()
341 	head = compound_head(page);  in page_copy_sane()
342 	v += (page - head) << PAGE_SHIFT;  in page_copy_sane()
349 size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,  in copy_page_to_iter()  argument
353 	if (!page_copy_sane(page, offset, bytes))  in copy_page_to_iter()
355 	if (WARN_ON_ONCE(i->data_source))  in copy_page_to_iter()
357 	page += offset / PAGE_SIZE; // first subpage  in copy_page_to_iter()
360 		void *kaddr = kmap_local_page(page);  in copy_page_to_iter()
361 		size_t n = min(bytes, (size_t)PAGE_SIZE - offset);  in copy_page_to_iter()
365 		bytes -= n;  in copy_page_to_iter()
370 			page++;  in copy_page_to_iter()
378 size_t copy_page_to_iter_nofault(struct page *page, unsigned offset, size_t bytes,  in copy_page_to_iter_nofault()  argument
383 	if (!page_copy_sane(page, offset, bytes))  in copy_page_to_iter_nofault()
385 	if (WARN_ON_ONCE(i->data_source))  in copy_page_to_iter_nofault()
387 	page += offset / PAGE_SIZE; // first subpage  in copy_page_to_iter_nofault()
390 		void *kaddr = kmap_local_page(page);  in copy_page_to_iter_nofault()
391 		size_t n = min(bytes, (size_t)PAGE_SIZE - offset);  in copy_page_to_iter_nofault()
398 		bytes -= n;  in copy_page_to_iter_nofault()
403 			page++;  in copy_page_to_iter_nofault()
411 size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,  in copy_page_from_iter()  argument
415 	if (!page_copy_sane(page, offset, bytes))  in copy_page_from_iter()
417 	page += offset / PAGE_SIZE; // first subpage  in copy_page_from_iter()
420 		void *kaddr = kmap_local_page(page);  in copy_page_from_iter()
421 		size_t n = min(bytes, (size_t)PAGE_SIZE - offset);  in copy_page_from_iter()
425 		bytes -= n;  in copy_page_from_iter()
430 			page++;  in copy_page_from_iter()
460 size_t copy_page_from_iter_atomic(struct page *page, size_t offset,  in copy_page_from_iter_atomic()  argument
465 	if (!page_copy_sane(page, offset, bytes))  in copy_page_from_iter_atomic()
467 	if (WARN_ON_ONCE(!i->data_source))  in copy_page_from_iter_atomic()
473 		n = bytes - copied;  in copy_page_from_iter_atomic()
474 		if (PageHighMem(page)) {  in copy_page_from_iter_atomic()
475 			page += offset / PAGE_SIZE;  in copy_page_from_iter_atomic()
477 			n = min_t(size_t, n, PAGE_SIZE - offset);  in copy_page_from_iter_atomic()
480 		p = kmap_atomic(page) + offset;  in copy_page_from_iter_atomic()
485 	} while (PageHighMem(page) && copied != bytes && n > 0);  in copy_page_from_iter_atomic()
491 static void iov_iter_bvec_advance(struct iov_iter *i, size_t size)  in iov_iter_bvec_advance()  argument
495 	if (!i->count)  in iov_iter_bvec_advance()
497 	i->count -= size;  in iov_iter_bvec_advance()
499 	size += i->iov_offset;  in iov_iter_bvec_advance()
501 	for (bvec = i->bvec, end = bvec + i->nr_segs; bvec < end; bvec++) {  in iov_iter_bvec_advance()
502 		if (likely(size < bvec->bv_len))  in iov_iter_bvec_advance()
504 		size -= bvec->bv_len;  in iov_iter_bvec_advance()
506 	i->iov_offset = size;  in iov_iter_bvec_advance()
507 	i->nr_segs -= bvec - i->bvec;  in iov_iter_bvec_advance()
508 	i->bvec = bvec;  in iov_iter_bvec_advance()
511 static void iov_iter_iovec_advance(struct iov_iter *i, size_t size)  in iov_iter_iovec_advance()  argument
515 	if (!i->count)  in iov_iter_iovec_advance()
517 	i->count -= size;  in iov_iter_iovec_advance()
519 	size += i->iov_offset; // from beginning of current segment  in iov_iter_iovec_advance()
520 	for (iov = iter_iov(i), end = iov + i->nr_segs; iov < end; iov++) {  in iov_iter_iovec_advance()
521 		if (likely(size < iov->iov_len))  in iov_iter_iovec_advance()
523 		size -= iov->iov_len;  in iov_iter_iovec_advance()
525 	i->iov_offset = size;  in iov_iter_iovec_advance()
526 	i->nr_segs -= iov - iter_iov(i);  in iov_iter_iovec_advance()
527 	i->__iov = iov;  in iov_iter_iovec_advance()
530 void iov_iter_advance(struct iov_iter *i, size_t size)  in iov_iter_advance()  argument
532 	if (unlikely(i->count < size))  in iov_iter_advance()
533 		size = i->count;  in iov_iter_advance()
535 		i->iov_offset += size;  in iov_iter_advance()
536 		i->count -= size;  in iov_iter_advance()
539 		iov_iter_iovec_advance(i, size);  in iov_iter_advance()
541 		iov_iter_bvec_advance(i, size);  in iov_iter_advance()
543 		i->count -= size;  in iov_iter_advance()
554 	i->count += unroll;  in iov_iter_revert()
557 	if (unroll <= i->iov_offset) {  in iov_iter_revert()
558 		i->iov_offset -= unroll;  in iov_iter_revert()
561 	unroll -= i->iov_offset;  in iov_iter_revert()
568 		const struct bio_vec *bvec = i->bvec;  in iov_iter_revert()
570 			size_t n = (--bvec)->bv_len;  in iov_iter_revert()
571 			i->nr_segs++;  in iov_iter_revert()
573 				i->bvec = bvec;  in iov_iter_revert()
574 				i->iov_offset = n - unroll;  in iov_iter_revert()
577 			unroll -= n;  in iov_iter_revert()
582 			size_t n = (--iov)->iov_len;  in iov_iter_revert()
583 			i->nr_segs++;  in iov_iter_revert()
585 				i->__iov = iov;  in iov_iter_revert()
586 				i->iov_offset = n - unroll;  in iov_iter_revert()
589 			unroll -= n;  in iov_iter_revert()
600 	if (i->nr_segs > 1) {  in iov_iter_single_seg_count()
602 			return min(i->count, iter_iov(i)->iov_len - i->iov_offset);  in iov_iter_single_seg_count()
604 			return min(i->count, i->bvec->bv_len - i->iov_offset);  in iov_iter_single_seg_count()
606 	return i->count;  in iov_iter_single_seg_count()
643  * iov_iter_xarray - Initialise an I/O iterator to use the pages in an xarray
648  * @count: The size of the I/O buffer in bytes.
671  * iov_iter_discard - Initialise an I/O iterator that discards data
674  * @count: The size of the I/O buffer in bytes.
694 	size_t size = i->count;  in iov_iter_aligned_iovec()  local
695 	size_t skip = i->iov_offset;  in iov_iter_aligned_iovec()
698 	for (k = 0; k < i->nr_segs; k++, skip = 0) {  in iov_iter_aligned_iovec()
700 		size_t len = iov->iov_len - skip;  in iov_iter_aligned_iovec()
702 		if (len > size)  in iov_iter_aligned_iovec()
703 			len = size;  in iov_iter_aligned_iovec()
706 		if ((unsigned long)(iov->iov_base + skip) & addr_mask)  in iov_iter_aligned_iovec()
709 		size -= len;  in iov_iter_aligned_iovec()
710 		if (!size)  in iov_iter_aligned_iovec()
719 	size_t size = i->count;  in iov_iter_aligned_bvec()  local
720 	unsigned skip = i->iov_offset;  in iov_iter_aligned_bvec()
723 	for (k = 0; k < i->nr_segs; k++, skip = 0) {  in iov_iter_aligned_bvec()
724 		size_t len = i->bvec[k].bv_len - skip;  in iov_iter_aligned_bvec()
726 		if (len > size)  in iov_iter_aligned_bvec()
727 			len = size;  in iov_iter_aligned_bvec()
730 		if ((unsigned long)(i->bvec[k].bv_offset + skip) & addr_mask)  in iov_iter_aligned_bvec()
733 		size -= len;  in iov_iter_aligned_bvec()
734 		if (!size)  in iov_iter_aligned_bvec()
741  * iov_iter_is_aligned() - Check if the addresses and lengths of each segments
754 		if (i->count & len_mask)  in iov_iter_is_aligned()
756 		if ((unsigned long)(i->ubuf + i->iov_offset) & addr_mask)  in iov_iter_is_aligned()
768 		if (i->count & len_mask)  in iov_iter_is_aligned()
770 		if ((i->xarray_start + i->iov_offset) & addr_mask)  in iov_iter_is_aligned()
781 	size_t size = i->count;  in iov_iter_alignment_iovec()  local
782 	size_t skip = i->iov_offset;  in iov_iter_alignment_iovec()
785 	for (k = 0; k < i->nr_segs; k++, skip = 0) {  in iov_iter_alignment_iovec()
787 		size_t len = iov->iov_len - skip;  in iov_iter_alignment_iovec()
789 			res |= (unsigned long)iov->iov_base + skip;  in iov_iter_alignment_iovec()
790 			if (len > size)  in iov_iter_alignment_iovec()
791 				len = size;  in iov_iter_alignment_iovec()
793 			size -= len;  in iov_iter_alignment_iovec()
794 			if (!size)  in iov_iter_alignment_iovec()
804 	size_t size = i->count;  in iov_iter_alignment_bvec()  local
805 	unsigned skip = i->iov_offset;  in iov_iter_alignment_bvec()
808 	for (k = 0; k < i->nr_segs; k++, skip = 0) {  in iov_iter_alignment_bvec()
809 		size_t len = i->bvec[k].bv_len - skip;  in iov_iter_alignment_bvec()
810 		res |= (unsigned long)i->bvec[k].bv_offset + skip;  in iov_iter_alignment_bvec()
811 		if (len > size)  in iov_iter_alignment_bvec()
812 			len = size;  in iov_iter_alignment_bvec()
814 		size -= len;  in iov_iter_alignment_bvec()
815 		if (!size)  in iov_iter_alignment_bvec()
824 		size_t size = i->count;  in iov_iter_alignment()  local
825 		if (size)  in iov_iter_alignment()
826 			return ((unsigned long)i->ubuf + i->iov_offset) | size;  in iov_iter_alignment()
838 		return (i->xarray_start + i->iov_offset) | i->count;  in iov_iter_alignment()
848 	size_t size = i->count;  in iov_iter_gap_alignment()  local
857 	for (k = 0; k < i->nr_segs; k++) {  in iov_iter_gap_alignment()
859 		if (iov->iov_len) {  in iov_iter_gap_alignment()
860 			unsigned long base = (unsigned long)iov->iov_base;  in iov_iter_gap_alignment()
863 			v = base + iov->iov_len;  in iov_iter_gap_alignment()
864 			if (size <= iov->iov_len)  in iov_iter_gap_alignment()
866 			size -= iov->iov_len;  in iov_iter_gap_alignment()
873 static int want_pages_array(struct page ***res, size_t size,  in want_pages_array()  argument
876 	unsigned int count = DIV_ROUND_UP(size + start, PAGE_SIZE);  in want_pages_array()
882 		*res = kvmalloc_array(count, sizeof(struct page *), GFP_KERNEL);  in want_pages_array()
889 static ssize_t iter_xarray_populate_pages(struct page **pages, struct xarray *xa,  in iter_xarray_populate_pages()
893 	struct page *page;  in iter_xarray_populate_pages()  local
897 	for (page = xas_load(&xas); page; page = xas_next(&xas)) {  in iter_xarray_populate_pages()
898 		if (xas_retry(&xas, page))  in iter_xarray_populate_pages()
901 		/* Has the page moved or been split? */  in iter_xarray_populate_pages()
902 		if (unlikely(page != xas_reload(&xas))) {  in iter_xarray_populate_pages()
907 		pages[ret] = find_subpage(page, xas.xa_index);  in iter_xarray_populate_pages()
917 				     struct page ***pages, size_t maxsize,  in iter_xarray_get_pages()
924 	pos = i->xarray_start + i->iov_offset;  in iter_xarray_get_pages()
931 		return -ENOMEM;  in iter_xarray_get_pages()
932 	nr = iter_xarray_populate_pages(*pages, i->xarray, index, count);  in iter_xarray_get_pages()
936 	maxsize = min_t(size_t, nr * PAGE_SIZE - offset, maxsize);  in iter_xarray_get_pages()
937 	i->iov_offset += maxsize;  in iter_xarray_get_pages()
938 	i->count -= maxsize;  in iter_xarray_get_pages()
942 /* must be done on non-empty ITER_UBUF or ITER_IOVEC one */
943 static unsigned long first_iovec_segment(const struct iov_iter *i, size_t *size)  in first_iovec_segment()  argument
949 		return (unsigned long)i->ubuf + i->iov_offset;  in first_iovec_segment()
951 	for (k = 0, skip = i->iov_offset; k < i->nr_segs; k++, skip = 0) {  in first_iovec_segment()
953 		size_t len = iov->iov_len - skip;  in first_iovec_segment()
957 		if (*size > len)  in first_iovec_segment()
958 			*size = len;  in first_iovec_segment()
959 		return (unsigned long)iov->iov_base + skip;  in first_iovec_segment()
964 /* must be done on non-empty ITER_BVEC one */
965 static struct page *first_bvec_segment(const struct iov_iter *i,  in first_bvec_segment()
966 				       size_t *size, size_t *start)  in first_bvec_segment()  argument
968 	struct page *page;  in first_bvec_segment()  local
969 	size_t skip = i->iov_offset, len;  in first_bvec_segment()
971 	len = i->bvec->bv_len - skip;  in first_bvec_segment()
972 	if (*size > len)  in first_bvec_segment()
973 		*size = len;  in first_bvec_segment()
974 	skip += i->bvec->bv_offset;  in first_bvec_segment()
975 	page = i->bvec->bv_page + skip / PAGE_SIZE;  in first_bvec_segment()
977 	return page;  in first_bvec_segment()
981 		   struct page ***pages, size_t maxsize,  in __iov_iter_get_pages_alloc()
986 	if (maxsize > i->count)  in __iov_iter_get_pages_alloc()
987 		maxsize = i->count;  in __iov_iter_get_pages_alloc()
999 		if (i->nofault)  in __iov_iter_get_pages_alloc()
1007 			return -ENOMEM;  in __iov_iter_get_pages_alloc()
1011 		maxsize = min_t(size_t, maxsize, res * PAGE_SIZE - *start);  in __iov_iter_get_pages_alloc()
1016 		struct page **p;  in __iov_iter_get_pages_alloc()
1017 		struct page *page;  in __iov_iter_get_pages_alloc()  local
1019 		page = first_bvec_segment(i, &maxsize, start);  in __iov_iter_get_pages_alloc()
1022 			return -ENOMEM;  in __iov_iter_get_pages_alloc()
1025 			get_page(p[k] = page + k);  in __iov_iter_get_pages_alloc()
1026 		maxsize = min_t(size_t, maxsize, n * PAGE_SIZE - *start);  in __iov_iter_get_pages_alloc()
1027 		i->count -= maxsize;  in __iov_iter_get_pages_alloc()
1028 		i->iov_offset += maxsize;  in __iov_iter_get_pages_alloc()
1029 		if (i->iov_offset == i->bvec->bv_len) {  in __iov_iter_get_pages_alloc()
1030 			i->iov_offset = 0;  in __iov_iter_get_pages_alloc()
1031 			i->bvec++;  in __iov_iter_get_pages_alloc()
1032 			i->nr_segs--;  in __iov_iter_get_pages_alloc()
1038 	return -EFAULT;  in __iov_iter_get_pages_alloc()
1041 ssize_t iov_iter_get_pages2(struct iov_iter *i, struct page **pages,  in iov_iter_get_pages2()
1053 		struct page ***pages, size_t maxsize, size_t *start)  in iov_iter_get_pages_alloc2()
1070 	size_t skip = i->iov_offset, size = i->count;  in iov_npages()  local
1074 	for (p = iter_iov(i); size; skip = 0, p++) {  in iov_npages()
1075 		unsigned offs = offset_in_page(p->iov_base + skip);  in iov_npages()
1076 		size_t len = min(p->iov_len - skip, size);  in iov_npages()
1079 			size -= len;  in iov_npages()
1090 	size_t skip = i->iov_offset, size = i->count;  in bvec_npages()  local
1094 	for (p = i->bvec; size; skip = 0, p++) {  in bvec_npages()
1095 		unsigned offs = (p->bv_offset + skip) % PAGE_SIZE;  in bvec_npages()
1096 		size_t len = min(p->bv_len - skip, size);  in bvec_npages()
1098 		size -= len;  in bvec_npages()
1108 	if (unlikely(!i->count))  in iov_iter_npages()
1111 		unsigned offs = offset_in_page(i->ubuf + i->iov_offset);  in iov_iter_npages()
1112 		int npages = DIV_ROUND_UP(offs + i->count, PAGE_SIZE);  in iov_iter_npages()
1121 		unsigned offset = (i->xarray_start + i->iov_offset) % PAGE_SIZE;  in iov_iter_npages()
1122 		int npages = DIV_ROUND_UP(offset + i->count, PAGE_SIZE);  in iov_iter_npages()
1133 		return new->bvec = kmemdup(new->bvec,  in dup_iter()
1134 				    new->nr_segs * sizeof(struct bio_vec),  in dup_iter()
1138 		return new->__iov = kmemdup(new->__iov,  in dup_iter()
1139 				   new->nr_segs * sizeof(struct iovec),  in dup_iter()
1150 	int ret = -EFAULT, i;  in copy_compat_iovec_from_user()
1153 		return -EFAULT;  in copy_compat_iovec_from_user()
1164 			ret = -EINVAL;  in copy_compat_iovec_from_user()
1180 	int ret = -EFAULT;  in copy_iovec_from_user()
1183 		return -EFAULT;  in copy_iovec_from_user()
1189 		unsafe_get_user(len, &uiov->iov_len, uaccess_end);  in copy_iovec_from_user()
1190 		unsafe_get_user(buf, &uiov->iov_base, uaccess_end);  in copy_iovec_from_user()
1194 			ret = -EINVAL;  in copy_iovec_from_user()
1197 		iov->iov_base = buf;  in copy_iovec_from_user()
1198 		iov->iov_len = len;  in copy_iovec_from_user()
1201 	} while (--nr_segs);  in copy_iovec_from_user()
1224 		return ERR_PTR(-EINVAL);  in iovec_from_user()
1228 			return ERR_PTR(-ENOMEM);  in iovec_from_user()
1261 	ret = import_ubuf(type, iov->iov_base, iov->iov_len, i);  in __import_iovec_ubuf()
1265 	return i->count;  in __import_iovec_ubuf()
1300 			return -EFAULT;  in __import_iovec()
1303 		if (len > MAX_RW_COUNT - total_len) {  in __import_iovec()
1304 			len = MAX_RW_COUNT - total_len;  in __import_iovec()
1319  * import_iovec() - Copy an array of &struct iovec from userspace
1328  *     on-stack) kernel array.
1335  * on-stack array was used or not (and regardless of whether this function
1354 		return -EFAULT;  in import_ubuf()
1362  * iov_iter_restore() - Restore a &struct iov_iter to the same state as when
1378 	i->iov_offset = state->iov_offset;  in iov_iter_restore()
1379 	i->count = state->count;  in iov_iter_restore()
1383 	 * For the *vec iters, nr_segs + iov is constant - if we increment  in iov_iter_restore()
1387 	 * size, so we can just increment the iov pointer as they are unionzed.  in iov_iter_restore()
1388 	 * ITER_BVEC _may_ be the same size on some archs, but on others it is  in iov_iter_restore()
1393 		i->bvec -= state->nr_segs - i->nr_segs;  in iov_iter_restore()
1395 		i->__iov -= state->nr_segs - i->nr_segs;  in iov_iter_restore()
1396 	i->nr_segs = state->nr_segs;  in iov_iter_restore()
1404 					     struct page ***pages, size_t maxsize,  in iov_iter_extract_xarray_pages()
1409 	struct page *page, **p;  in iov_iter_extract_xarray_pages()  local
1411 	loff_t pos = i->xarray_start + i->iov_offset;  in iov_iter_extract_xarray_pages()
1413 	XA_STATE(xas, i->xarray, index);  in iov_iter_extract_xarray_pages()
1420 		return -ENOMEM;  in iov_iter_extract_xarray_pages()
1424 	for (page = xas_load(&xas); page; page = xas_next(&xas)) {  in iov_iter_extract_xarray_pages()
1425 		if (xas_retry(&xas, page))  in iov_iter_extract_xarray_pages()
1428 		/* Has the page moved or been split? */  in iov_iter_extract_xarray_pages()
1429 		if (unlikely(page != xas_reload(&xas))) {  in iov_iter_extract_xarray_pages()
1434 		p[nr++] = find_subpage(page, xas.xa_index);  in iov_iter_extract_xarray_pages()
1440 	maxsize = min_t(size_t, nr * PAGE_SIZE - offset, maxsize);  in iov_iter_extract_xarray_pages()
1450 					   struct page ***pages, size_t maxsize,  in iov_iter_extract_bvec_pages()
1455 	struct page **p, *page;  in iov_iter_extract_bvec_pages()  local
1456 	size_t skip = i->iov_offset, offset, size;  in iov_iter_extract_bvec_pages()  local
1460 		if (i->nr_segs == 0)  in iov_iter_extract_bvec_pages()
1462 		size = min(maxsize, i->bvec->bv_len - skip);  in iov_iter_extract_bvec_pages()
1463 		if (size)  in iov_iter_extract_bvec_pages()
1465 		i->iov_offset = 0;  in iov_iter_extract_bvec_pages()
1466 		i->nr_segs--;  in iov_iter_extract_bvec_pages()
1467 		i->bvec++;  in iov_iter_extract_bvec_pages()
1471 	skip += i->bvec->bv_offset;  in iov_iter_extract_bvec_pages()
1472 	page = i->bvec->bv_page + skip / PAGE_SIZE;  in iov_iter_extract_bvec_pages()
1476 	maxpages = want_pages_array(pages, size, offset, maxpages);  in iov_iter_extract_bvec_pages()
1478 		return -ENOMEM;  in iov_iter_extract_bvec_pages()
1481 		p[k] = page + k;  in iov_iter_extract_bvec_pages()
1483 	size = min_t(size_t, size, maxpages * PAGE_SIZE - offset);  in iov_iter_extract_bvec_pages()
1484 	iov_iter_advance(i, size);  in iov_iter_extract_bvec_pages()
1485 	return size;  in iov_iter_extract_bvec_pages()
1493 					   struct page ***pages, size_t maxsize,  in iov_iter_extract_kvec_pages()
1498 	struct page **p, *page;  in iov_iter_extract_kvec_pages()  local
1500 	size_t skip = i->iov_offset, offset, len, size;  in iov_iter_extract_kvec_pages()  local
1504 		if (i->nr_segs == 0)  in iov_iter_extract_kvec_pages()
1506 		size = min(maxsize, i->kvec->iov_len - skip);  in iov_iter_extract_kvec_pages()
1507 		if (size)  in iov_iter_extract_kvec_pages()
1509 		i->iov_offset = 0;  in iov_iter_extract_kvec_pages()
1510 		i->nr_segs--;  in iov_iter_extract_kvec_pages()
1511 		i->kvec++;  in iov_iter_extract_kvec_pages()
1515 	kaddr = i->kvec->iov_base + skip;  in iov_iter_extract_kvec_pages()
1519 	maxpages = want_pages_array(pages, size, offset, maxpages);  in iov_iter_extract_kvec_pages()
1521 		return -ENOMEM;  in iov_iter_extract_kvec_pages()
1524 	kaddr -= offset;  in iov_iter_extract_kvec_pages()
1525 	len = offset + size;  in iov_iter_extract_kvec_pages()
1530 			page = vmalloc_to_page(kaddr);  in iov_iter_extract_kvec_pages()
1532 			page = virt_to_page(kaddr);  in iov_iter_extract_kvec_pages()
1534 		p[k] = page;  in iov_iter_extract_kvec_pages()
1535 		len -= seg;  in iov_iter_extract_kvec_pages()
1539 	size = min_t(size_t, size, maxpages * PAGE_SIZE - offset);  in iov_iter_extract_kvec_pages()
1540 	iov_iter_advance(i, size);  in iov_iter_extract_kvec_pages()
1541 	return size;  in iov_iter_extract_kvec_pages()
1546  * each of them.  This should only be used if the iterator is user-backed
1554  * child a copy of the page.
1557 					   struct page ***pages,  in iov_iter_extract_user_pages()
1568 	if (i->data_source == ITER_DEST)  in iov_iter_extract_user_pages()
1572 	if (i->nofault)  in iov_iter_extract_user_pages()
1580 		return -ENOMEM;  in iov_iter_extract_user_pages()
1584 	maxsize = min_t(size_t, maxsize, res * PAGE_SIZE - offset);  in iov_iter_extract_user_pages()
1590  * iov_iter_extract_pages - Extract a list of contiguous pages from an iterator
1594  * @maxpages: The maximum size of the list of pages
1600  * of page contents can be set.
1602  * If *@pages is NULL, a page list will be allocated to the required size and
1604  * that the caller allocated a page list at least @maxpages in size and this
1607  * @extraction_flags can have ITER_ALLOW_P2PDMA set to request peer-to-peer DMA
1615  *  (*) If the iterator is user-backed (ITER_IOVEC/ITER_UBUF), pins will be
1628  * sets *offset0 to the offset into the first page.
1630  * It may also return -ENOMEM and -EFAULT.
1633 			       struct page ***pages,  in iov_iter_extract_pages()
1639 	maxsize = min_t(size_t, min_t(size_t, maxsize, i->count), MAX_RW_COUNT);  in iov_iter_extract_pages()
1659 	return -EFAULT;  in iov_iter_extract_pages()