161 static inline unsigned dio_pages_present(struct dio_submit *sdio)  in dio_pages_present()  argument
163 	return sdio->tail - sdio->head;  in dio_pages_present()
169 static inline int dio_refill_pages(struct dio *dio, struct dio_submit *sdio)  in dio_refill_pages()  argument
173 	ret = iov_iter_get_pages(sdio->iter, dio->pages, LONG_MAX, DIO_PAGES,  in dio_refill_pages()
174 				&sdio->from);  in dio_refill_pages()
176 	if (ret < 0 && sdio->blocks_available && (dio->op == REQ_OP_WRITE)) {  in dio_refill_pages()
187 		sdio->head = 0;  in dio_refill_pages()
188 		sdio->tail = 1;  in dio_refill_pages()
189 		sdio->from = 0;  in dio_refill_pages()
190 		sdio->to = PAGE_SIZE;  in dio_refill_pages()
195 		iov_iter_advance(sdio->iter, ret);  in dio_refill_pages()
196 		ret += sdio->from;  in dio_refill_pages()
197 		sdio->head = 0;  in dio_refill_pages()
198 		sdio->tail = (ret + PAGE_SIZE - 1) / PAGE_SIZE;  in dio_refill_pages()
199 		sdio->to = ((ret - 1) & (PAGE_SIZE - 1)) + 1;  in dio_refill_pages()
212 					struct dio_submit *sdio)  in dio_get_page()  argument
214 	if (dio_pages_present(sdio) == 0) {  in dio_get_page()
217 		ret = dio_refill_pages(dio, sdio);  in dio_get_page()
220 		BUG_ON(dio_pages_present(sdio) == 0);  in dio_get_page()
222 	return dio->pages[sdio->head];  in dio_get_page()
390 dio_bio_alloc(struct dio *dio, struct dio_submit *sdio,  in dio_bio_alloc()  argument
412 	sdio->bio = bio;  in dio_bio_alloc()
413 	sdio->logical_offset_in_bio = sdio->cur_page_fs_offset;  in dio_bio_alloc()
423 static inline void dio_bio_submit(struct dio *dio, struct dio_submit *sdio)  in dio_bio_submit()  argument
425 	struct bio *bio = sdio->bio;  in dio_bio_submit()
439 	if (sdio->submit_io) {  in dio_bio_submit()
440 		sdio->submit_io(bio, dio->inode, sdio->logical_offset_in_bio);  in dio_bio_submit()
445 	sdio->bio = NULL;  in dio_bio_submit()
446 	sdio->boundary = 0;  in dio_bio_submit()
447 	sdio->logical_offset_in_bio = 0;  in dio_bio_submit()
453 static inline void dio_cleanup(struct dio *dio, struct dio_submit *sdio)  in dio_cleanup()  argument
455 	while (sdio->head < sdio->tail)  in dio_cleanup()
456 		put_page(dio->pages[sdio->head++]);  in dio_cleanup()
545 static inline int dio_bio_reap(struct dio *dio, struct dio_submit *sdio)  in dio_bio_reap()  argument
549 	if (sdio->reap_counter++ >= 64) {  in dio_bio_reap()
563 		sdio->reap_counter = 0;  in dio_bio_reap()
606  * of available blocks at sdio->blocks_available.  These are in units of the
627 static int get_more_blocks(struct dio *dio, struct dio_submit *sdio,  in get_more_blocks()  argument
635 	unsigned int i_blkbits = sdio->blkbits + sdio->blkfactor;  in get_more_blocks()
644 		BUG_ON(sdio->block_in_file >= sdio->final_block_in_request);  in get_more_blocks()
645 		fs_startblk = sdio->block_in_file >> sdio->blkfactor;  in get_more_blocks()
646 		fs_endblk = (sdio->final_block_in_request - 1) >>  in get_more_blocks()
647 					sdio->blkfactor;  in get_more_blocks()
671 		ret = (*sdio->get_block)(dio->inode, fs_startblk,  in get_more_blocks()
686 static inline int dio_new_bio(struct dio *dio, struct dio_submit *sdio,  in dio_new_bio()  argument
692 	ret = dio_bio_reap(dio, sdio);  in dio_new_bio()
695 	sector = start_sector << (sdio->blkbits - 9);  in dio_new_bio()
696 	nr_pages = min(sdio->pages_in_io, BIO_MAX_PAGES);  in dio_new_bio()
698 	dio_bio_alloc(dio, sdio, map_bh->b_bdev, sector, nr_pages);  in dio_new_bio()
699 	sdio->boundary = 0;  in dio_new_bio()
711 static inline int dio_bio_add_page(struct dio_submit *sdio)  in dio_bio_add_page()  argument
715 	ret = bio_add_page(sdio->bio, sdio->cur_page,  in dio_bio_add_page()
716 			sdio->cur_page_len, sdio->cur_page_offset);  in dio_bio_add_page()
717 	if (ret == sdio->cur_page_len) {  in dio_bio_add_page()
721 		if ((sdio->cur_page_len + sdio->cur_page_offset) == PAGE_SIZE)  in dio_bio_add_page()
722 			sdio->pages_in_io--;  in dio_bio_add_page()
723 		get_page(sdio->cur_page);  in dio_bio_add_page()
724 		sdio->final_block_in_bio = sdio->cur_page_block +  in dio_bio_add_page()
725 			(sdio->cur_page_len >> sdio->blkbits);  in dio_bio_add_page()
743 static inline int dio_send_cur_page(struct dio *dio, struct dio_submit *sdio,  in dio_send_cur_page()  argument
748 	if (sdio->bio) {  in dio_send_cur_page()
749 		loff_t cur_offset = sdio->cur_page_fs_offset;  in dio_send_cur_page()
750 		loff_t bio_next_offset = sdio->logical_offset_in_bio +  in dio_send_cur_page()
751 			sdio->bio->bi_iter.bi_size;  in dio_send_cur_page()
767 		if (sdio->final_block_in_bio != sdio->cur_page_block ||  in dio_send_cur_page()
769 			dio_bio_submit(dio, sdio);  in dio_send_cur_page()
772 	if (sdio->bio == NULL) {  in dio_send_cur_page()
773 		ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh);  in dio_send_cur_page()
778 	if (dio_bio_add_page(sdio) != 0) {  in dio_send_cur_page()
779 		dio_bio_submit(dio, sdio);  in dio_send_cur_page()
780 		ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh);  in dio_send_cur_page()
782 			ret = dio_bio_add_page(sdio);  in dio_send_cur_page()
808 submit_page_section(struct dio *dio, struct dio_submit *sdio, struct page *page,  in submit_page_section()  argument
824 	if (sdio->cur_page == page &&  in submit_page_section()
825 	    sdio->cur_page_offset + sdio->cur_page_len == offset &&  in submit_page_section()
826 	    sdio->cur_page_block +  in submit_page_section()
827 	    (sdio->cur_page_len >> sdio->blkbits) == blocknr) {  in submit_page_section()
828 		sdio->cur_page_len += len;  in submit_page_section()
835 	if (sdio->cur_page) {  in submit_page_section()
836 		ret = dio_send_cur_page(dio, sdio, map_bh);  in submit_page_section()
837 		put_page(sdio->cur_page);  in submit_page_section()
838 		sdio->cur_page = NULL;  in submit_page_section()
844 	sdio->cur_page = page;  in submit_page_section()
845 	sdio->cur_page_offset = offset;  in submit_page_section()
846 	sdio->cur_page_len = len;  in submit_page_section()
847 	sdio->cur_page_block = blocknr;  in submit_page_section()
848 	sdio->cur_page_fs_offset = sdio->block_in_file << sdio->blkbits;  in submit_page_section()
851 	 * If sdio->boundary then we want to schedule the IO now to  in submit_page_section()
854 	if (sdio->boundary) {  in submit_page_section()
855 		ret = dio_send_cur_page(dio, sdio, map_bh);  in submit_page_section()
856 		if (sdio->bio)  in submit_page_section()
857 			dio_bio_submit(dio, sdio);  in submit_page_section()
858 		put_page(sdio->cur_page);  in submit_page_section()
859 		sdio->cur_page = NULL;  in submit_page_section()
873 static inline void dio_zero_block(struct dio *dio, struct dio_submit *sdio,  in dio_zero_block()  argument
881 	sdio->start_zero_done = 1;  in dio_zero_block()
882 	if (!sdio->blkfactor || !buffer_new(map_bh))  in dio_zero_block()
885 	dio_blocks_per_fs_block = 1 << sdio->blkfactor;  in dio_zero_block()
886 	this_chunk_blocks = sdio->block_in_file & (dio_blocks_per_fs_block - 1);  in dio_zero_block()
898 	this_chunk_bytes = this_chunk_blocks << sdio->blkbits;  in dio_zero_block()
901 	if (submit_page_section(dio, sdio, page, 0, this_chunk_bytes,  in dio_zero_block()
902 				sdio->next_block_for_io, map_bh))  in dio_zero_block()
905 	sdio->next_block_for_io += this_chunk_blocks;  in dio_zero_block()
924 static int do_direct_IO(struct dio *dio, struct dio_submit *sdio,  in do_direct_IO()  argument
927 	const unsigned blkbits = sdio->blkbits;  in do_direct_IO()
928 	const unsigned i_blkbits = blkbits + sdio->blkfactor;  in do_direct_IO()
931 	while (sdio->block_in_file < sdio->final_block_in_request) {  in do_direct_IO()
935 		page = dio_get_page(dio, sdio);  in do_direct_IO()
940 		from = sdio->head ? 0 : sdio->from;  in do_direct_IO()
941 		to = (sdio->head == sdio->tail - 1) ? sdio->to : PAGE_SIZE;  in do_direct_IO()
942 		sdio->head++;  in do_direct_IO()
949 			if (sdio->blocks_available == 0) {  in do_direct_IO()
956 				ret = get_more_blocks(dio, sdio, map_bh);  in do_direct_IO()
964 				sdio->blocks_available =  in do_direct_IO()
966 				sdio->next_block_for_io =  in do_direct_IO()
967 					map_bh->b_blocknr << sdio->blkfactor;  in do_direct_IO()
975 				if (!sdio->blkfactor)  in do_direct_IO()
978 				blkmask = (1 << sdio->blkfactor) - 1;  in do_direct_IO()
979 				dio_remainder = (sdio->block_in_file & blkmask);  in do_direct_IO()
993 					sdio->next_block_for_io += dio_remainder;  in do_direct_IO()
994 				sdio->blocks_available -= dio_remainder;  in do_direct_IO()
1013 				if (sdio->block_in_file >=  in do_direct_IO()
1020 				sdio->block_in_file++;  in do_direct_IO()
1031 			if (unlikely(sdio->blkfactor && !sdio->start_zero_done))  in do_direct_IO()
1032 				dio_zero_block(dio, sdio, 0, map_bh);  in do_direct_IO()
1038 			this_chunk_blocks = sdio->blocks_available;  in do_direct_IO()
1042 			u = sdio->final_block_in_request - sdio->block_in_file;  in do_direct_IO()
1048 			if (this_chunk_blocks == sdio->blocks_available)  in do_direct_IO()
1049 				sdio->boundary = buffer_boundary(map_bh);  in do_direct_IO()
1050 			ret = submit_page_section(dio, sdio, page,  in do_direct_IO()
1053 						  sdio->next_block_for_io,  in do_direct_IO()
1059 			sdio->next_block_for_io += this_chunk_blocks;  in do_direct_IO()
1061 			sdio->block_in_file += this_chunk_blocks;  in do_direct_IO()
1064 			sdio->blocks_available -= this_chunk_blocks;  in do_direct_IO()
1066 			BUG_ON(sdio->block_in_file > sdio->final_block_in_request);  in do_direct_IO()
1067 			if (sdio->block_in_file == sdio->final_block_in_request)  in do_direct_IO()
1120  * NOTE: if you pass "sdio" to anything by pointer make sure that function
1139 	struct dio_submit sdio = { 0, };  in do_blockdev_direct_IO()  local
1243 	sdio.blkbits = blkbits;  in do_blockdev_direct_IO()
1244 	sdio.blkfactor = i_blkbits - blkbits;  in do_blockdev_direct_IO()
1245 	sdio.block_in_file = offset >> blkbits;  in do_blockdev_direct_IO()
1247 	sdio.get_block = get_block;  in do_blockdev_direct_IO()
1249 	sdio.submit_io = submit_io;  in do_blockdev_direct_IO()
1250 	sdio.final_block_in_bio = -1;  in do_blockdev_direct_IO()
1251 	sdio.next_block_for_io = -1;  in do_blockdev_direct_IO()
1259 	sdio.iter = iter;  in do_blockdev_direct_IO()
1260 	sdio.final_block_in_request = end >> blkbits;  in do_blockdev_direct_IO()
1266 	if (unlikely(sdio.blkfactor))  in do_blockdev_direct_IO()
1267 		sdio.pages_in_io = 2;  in do_blockdev_direct_IO()
1269 	sdio.pages_in_io += iov_iter_npages(iter, INT_MAX);  in do_blockdev_direct_IO()
1273 	retval = do_direct_IO(dio, &sdio, &map_bh);  in do_blockdev_direct_IO()
1275 		dio_cleanup(dio, &sdio);  in do_blockdev_direct_IO()
1288 	dio_zero_block(dio, &sdio, 1, &map_bh);  in do_blockdev_direct_IO()
1290 	if (sdio.cur_page) {  in do_blockdev_direct_IO()
1293 		ret2 = dio_send_cur_page(dio, &sdio, &map_bh);  in do_blockdev_direct_IO()
1296 		put_page(sdio.cur_page);  in do_blockdev_direct_IO()
1297 		sdio.cur_page = NULL;  in do_blockdev_direct_IO()
1299 	if (sdio.bio)  in do_blockdev_direct_IO()
1300 		dio_bio_submit(dio, &sdio);  in do_blockdev_direct_IO()
1308 	dio_cleanup(dio, &sdio);  in do_blockdev_direct_IO()