164 static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes,  in copy_page_to_iter_iovec()  argument
172 	if (unlikely(bytes > i->count))  in copy_page_to_iter_iovec()
173 		bytes = i->count;  in copy_page_to_iter_iovec()
175 	if (unlikely(!bytes))  in copy_page_to_iter_iovec()
179 	wanted = bytes;  in copy_page_to_iter_iovec()
183 	copy = min(bytes, iov->iov_len - skip);  in copy_page_to_iter_iovec()
194 		bytes -= copy;  in copy_page_to_iter_iovec()
196 		while (unlikely(!left && bytes)) {  in copy_page_to_iter_iovec()
199 			copy = min(bytes, iov->iov_len);  in copy_page_to_iter_iovec()
204 			bytes -= copy;  in copy_page_to_iter_iovec()
206 		if (likely(!bytes)) {  in copy_page_to_iter_iovec()
213 		copy = min(bytes, iov->iov_len - skip);  in copy_page_to_iter_iovec()
223 	bytes -= copy;  in copy_page_to_iter_iovec()
224 	while (unlikely(!left && bytes)) {  in copy_page_to_iter_iovec()
227 		copy = min(bytes, iov->iov_len);  in copy_page_to_iter_iovec()
232 		bytes -= copy;  in copy_page_to_iter_iovec()
241 	i->count -= wanted - bytes;  in copy_page_to_iter_iovec()
245 	return wanted - bytes;  in copy_page_to_iter_iovec()
248 static size_t copy_page_from_iter_iovec(struct page *page, size_t offset, size_t bytes,  in copy_page_from_iter_iovec()  argument
256 	if (unlikely(bytes > i->count))  in copy_page_from_iter_iovec()
257 		bytes = i->count;  in copy_page_from_iter_iovec()
259 	if (unlikely(!bytes))  in copy_page_from_iter_iovec()
263 	wanted = bytes;  in copy_page_from_iter_iovec()
267 	copy = min(bytes, iov->iov_len - skip);  in copy_page_from_iter_iovec()
278 		bytes -= copy;  in copy_page_from_iter_iovec()
280 		while (unlikely(!left && bytes)) {  in copy_page_from_iter_iovec()
283 			copy = min(bytes, iov->iov_len);  in copy_page_from_iter_iovec()
288 			bytes -= copy;  in copy_page_from_iter_iovec()
290 		if (likely(!bytes)) {  in copy_page_from_iter_iovec()
297 		copy = min(bytes, iov->iov_len - skip);  in copy_page_from_iter_iovec()
307 	bytes -= copy;  in copy_page_from_iter_iovec()
308 	while (unlikely(!left && bytes)) {  in copy_page_from_iter_iovec()
311 		copy = min(bytes, iov->iov_len);  in copy_page_from_iter_iovec()
316 		bytes -= copy;  in copy_page_from_iter_iovec()
325 	i->count -= wanted - bytes;  in copy_page_from_iter_iovec()
329 	return wanted - bytes;  in copy_page_from_iter_iovec()
375 static size_t copy_page_to_iter_pipe(struct page *page, size_t offset, size_t bytes,  in copy_page_to_iter_pipe()  argument
385 	if (unlikely(bytes > i->count))  in copy_page_to_iter_pipe()
386 		bytes = i->count;  in copy_page_to_iter_pipe()
388 	if (unlikely(!bytes))  in copy_page_to_iter_pipe()
399 			buf->len += bytes;  in copy_page_to_iter_pipe()
400 			i->iov_offset += bytes;  in copy_page_to_iter_pipe()
413 	buf->len = bytes;  in copy_page_to_iter_pipe()
416 	i->iov_offset = offset + bytes;  in copy_page_to_iter_pipe()
419 	i->count -= bytes;  in copy_page_to_iter_pipe()
420 	return bytes;  in copy_page_to_iter_pipe()
425  * bytes.  For each iovec, fault in each page that constitutes the iovec.
430 int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes)  in iov_iter_fault_in_readable()  argument
438 		iterate_iovec(i, bytes, v, iov, skip, ({  in iov_iter_fault_in_readable()
559 static size_t copy_pipe_to_iter(const void *addr, size_t bytes,  in copy_pipe_to_iter()  argument
570 	bytes = n = push_pipe(i, bytes, &i_head, &off);  in copy_pipe_to_iter()
583 	i->count -= bytes;  in copy_pipe_to_iter()
584 	return bytes;  in copy_pipe_to_iter()
594 static size_t csum_and_copy_to_pipe_iter(const void *addr, size_t bytes,  in csum_and_copy_to_pipe_iter()  argument
607 	bytes = n = push_pipe(i, bytes, &i_head, &r);  in csum_and_copy_to_pipe_iter()
623 	i->count -= bytes;  in csum_and_copy_to_pipe_iter()
625 	return bytes;  in csum_and_copy_to_pipe_iter()
628 size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)  in _copy_to_iter()  argument
632 		return copy_pipe_to_iter(addr, bytes, i);  in _copy_to_iter()
635 	iterate_and_advance(i, bytes, v,  in _copy_to_iter()
642 	return bytes;  in _copy_to_iter()
669 static size_t copy_mc_pipe_to_iter(const void *addr, size_t bytes,  in copy_mc_pipe_to_iter()  argument
680 	bytes = n = push_pipe(i, bytes, &i_head, &off);  in copy_mc_pipe_to_iter()
706  * @bytes: total transfer length
711  * block-layer). Upon #MC read(2) aborts and returns EIO or the bytes
726 size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i)  in _copy_mc_to_iter()  argument
732 		return copy_mc_pipe_to_iter(addr, bytes, i);  in _copy_mc_to_iter()
735 	iterate_and_advance(i, bytes, v,  in _copy_mc_to_iter()
743 			bytes = curr_addr - s_addr - rem;  in _copy_mc_to_iter()
744 			return bytes;  in _copy_mc_to_iter()
752 			bytes = curr_addr - s_addr - rem;  in _copy_mc_to_iter()
753 			return bytes;  in _copy_mc_to_iter()
758 	return bytes;  in _copy_mc_to_iter()
763 size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter()  argument
772 	iterate_and_advance(i, bytes, v,  in _copy_from_iter()
779 	return bytes;  in _copy_from_iter()
783 bool _copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter_full()  argument
790 	if (unlikely(i->count < bytes))  in _copy_from_iter_full()
795 	iterate_all_kinds(i, bytes, v, ({  in _copy_from_iter_full()
805 	iov_iter_advance(i, bytes);  in _copy_from_iter_full()
810 size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter_nocache()  argument
817 	iterate_and_advance(i, bytes, v,  in _copy_from_iter_nocache()
825 	return bytes;  in _copy_from_iter_nocache()
833  * @bytes: total transfer length
844 size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter_flushcache()  argument
851 	iterate_and_advance(i, bytes, v,  in _copy_from_iter_flushcache()
860 	return bytes;  in _copy_from_iter_flushcache()
865 bool _copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i)  in _copy_from_iter_full_nocache()  argument
872 	if (unlikely(i->count < bytes))  in _copy_from_iter_full_nocache()
874 	iterate_all_kinds(i, bytes, v, ({  in _copy_from_iter_full_nocache()
884 	iov_iter_advance(i, bytes);  in _copy_from_iter_full_nocache()
913 size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,  in copy_page_to_iter()  argument
916 	if (unlikely(!page_copy_sane(page, offset, bytes)))  in copy_page_to_iter()
920 		size_t wanted = copy_to_iter(kaddr + offset, bytes, i);  in copy_page_to_iter()
924 		return bytes;  in copy_page_to_iter()
926 		return copy_page_to_iter_iovec(page, offset, bytes, i);  in copy_page_to_iter()
928 		return copy_page_to_iter_pipe(page, offset, bytes, i);  in copy_page_to_iter()
932 size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,  in copy_page_from_iter()  argument
935 	if (unlikely(!page_copy_sane(page, offset, bytes)))  in copy_page_from_iter()
943 		size_t wanted = _copy_from_iter(kaddr + offset, bytes, i);  in copy_page_from_iter()
947 		return copy_page_from_iter_iovec(page, offset, bytes, i);  in copy_page_from_iter()
951 static size_t pipe_zero(size_t bytes, struct iov_iter *i)  in pipe_zero()  argument
961 	bytes = n = push_pipe(i, bytes, &i_head, &off);  in pipe_zero()
974 	i->count -= bytes;  in pipe_zero()
975 	return bytes;  in pipe_zero()
978 size_t iov_iter_zero(size_t bytes, struct iov_iter *i)  in iov_iter_zero()  argument
981 		return pipe_zero(bytes, i);  in iov_iter_zero()
982 	iterate_and_advance(i, bytes, v,  in iov_iter_zero()
988 	return bytes;  in iov_iter_zero()
993 		struct iov_iter *i, unsigned long offset, size_t bytes)  in iov_iter_copy_from_user_atomic()  argument
996 	if (unlikely(!page_copy_sane(page, offset, bytes))) {  in iov_iter_copy_from_user_atomic()
1005 	iterate_all_kinds(i, bytes, v,  in iov_iter_copy_from_user_atomic()
1012 	return bytes;  in iov_iter_copy_from_user_atomic()
1215  * @count: The size of the I/O buffer in bytes.
1444 size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum,  in csum_and_copy_from_iter()  argument
1455 	iterate_and_advance(i, bytes, v, ({  in csum_and_copy_from_iter()
1479 	return bytes;  in csum_and_copy_from_iter()
1483 bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum,  in csum_and_copy_from_iter_full()  argument
1494 	if (unlikely(i->count < bytes))  in csum_and_copy_from_iter_full()
1496 	iterate_all_kinds(i, bytes, v, ({  in csum_and_copy_from_iter_full()
1520 	iov_iter_advance(i, bytes);  in csum_and_copy_from_iter_full()
1525 size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *csump,  in csum_and_copy_to_iter()  argument
1534 		return csum_and_copy_to_pipe_iter(addr, bytes, csum, i);  in csum_and_copy_to_iter()
1541 	iterate_and_advance(i, bytes, v, ({  in csum_and_copy_to_iter()
1565 	return bytes;  in csum_and_copy_to_iter()
1569 size_t hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp,  in hash_and_copy_to_iter()  argument
1577 	copied = copy_to_iter(addr, bytes, i);  in hash_and_copy_to_iter()
1803  * Return: Negative error code on error, bytes imported on success
1829 int iov_iter_for_each_range(struct iov_iter *i, size_t bytes,  in iov_iter_for_each_range()  argument
1835 	if (!bytes)  in iov_iter_for_each_range()
1838 	iterate_all_kinds(i, bytes, v, -EINVAL, ({  in iov_iter_for_each_range()