Lines Matching +full:no +full:- +full:sdio
1 // SPDX-License-Identifier: GPL-2.0-only
3 * fs/direct-io.c
16 * added support for non-aligned IO.
59 * is determined on a per-invocation basis. When talking to the filesystem
61 * down by dio->blkfactor. Similarly, fs-blocksize quantities are converted
76 finer. blkfactor=2 means 1/4-block
78 unsigned start_zero_done; /* flag: sub-blocksize zeroing has
161 static inline unsigned dio_pages_present(struct dio_submit *sdio) in dio_pages_present() argument
163 return sdio->tail - sdio->head; in dio_pages_present()
169 static inline int dio_refill_pages(struct dio *dio, struct dio_submit *sdio) in dio_refill_pages() argument
173 ret = iov_iter_get_pages(sdio->iter, dio->pages, LONG_MAX, DIO_PAGES, in dio_refill_pages()
174 &sdio->from); in dio_refill_pages()
176 if (ret < 0 && sdio->blocks_available && (dio->op == REQ_OP_WRITE)) { in dio_refill_pages()
183 if (dio->page_errors == 0) in dio_refill_pages()
184 dio->page_errors = ret; in dio_refill_pages()
186 dio->pages[0] = page; in dio_refill_pages()
187 sdio->head = 0; in dio_refill_pages()
188 sdio->tail = 1; in dio_refill_pages()
189 sdio->from = 0; in dio_refill_pages()
190 sdio->to = PAGE_SIZE; in dio_refill_pages()
195 iov_iter_advance(sdio->iter, ret); in dio_refill_pages()
196 ret += sdio->from; in dio_refill_pages()
197 sdio->head = 0; in dio_refill_pages()
198 sdio->tail = (ret + PAGE_SIZE - 1) / PAGE_SIZE; in dio_refill_pages()
199 sdio->to = ((ret - 1) & (PAGE_SIZE - 1)) + 1; in dio_refill_pages()
212 struct dio_submit *sdio) in dio_get_page() argument
214 if (dio_pages_present(sdio) == 0) { in dio_get_page()
217 ret = dio_refill_pages(dio, sdio); in dio_get_page()
220 BUG_ON(dio_pages_present(sdio) == 0); in dio_get_page()
222 return dio->pages[sdio->head]; in dio_get_page()
226 * dio_complete() - called when all DIO BIO I/O has been completed
238 loff_t offset = dio->iocb->ki_pos; in dio_complete()
245 * In that case -EIOCBQUEUED is in fact not an error we want in dio_complete()
248 if (ret == -EIOCBQUEUED) in dio_complete()
251 if (dio->result) { in dio_complete()
252 transferred = dio->result; in dio_complete()
255 if ((dio->op == REQ_OP_READ) && in dio_complete()
256 ((offset + transferred) > dio->i_size)) in dio_complete()
257 transferred = dio->i_size - offset; in dio_complete()
259 if (unlikely(ret == -EFAULT) && transferred) in dio_complete()
264 ret = dio->page_errors; in dio_complete()
266 ret = dio->io_error; in dio_complete()
270 if (dio->end_io) { in dio_complete()
272 err = dio->end_io(dio->iocb, offset, ret, dio->private); in dio_complete()
279 * non-direct readahead, or faulted in by get_user_pages() if the source in dio_complete()
284 * And this page cache invalidation has to be after dio->end_io(), as in dio_complete()
290 ret > 0 && dio->op == REQ_OP_WRITE && in dio_complete()
291 dio->inode->i_mapping->nrpages) { in dio_complete()
292 err = invalidate_inode_pages2_range(dio->inode->i_mapping, in dio_complete()
294 (offset + ret - 1) >> PAGE_SHIFT); in dio_complete()
296 dio_warn_stale_pagecache(dio->iocb->ki_filp); in dio_complete()
299 inode_dio_end(dio->inode); in dio_complete()
307 dio->iocb->ki_pos += transferred; in dio_complete()
309 if (ret > 0 && dio->op == REQ_OP_WRITE) in dio_complete()
310 ret = generic_write_sync(dio->iocb, ret); in dio_complete()
311 dio->iocb->ki_complete(dio->iocb, ret, 0); in dio_complete()
332 struct dio *dio = bio->bi_private; in dio_bio_end_aio()
340 spin_lock_irqsave(&dio->bio_lock, flags); in dio_bio_end_aio()
341 remaining = --dio->refcount; in dio_bio_end_aio()
342 if (remaining == 1 && dio->waiter) in dio_bio_end_aio()
343 wake_up_process(dio->waiter); in dio_bio_end_aio()
344 spin_unlock_irqrestore(&dio->bio_lock, flags); in dio_bio_end_aio()
355 if (dio->result) in dio_bio_end_aio()
356 defer_completion = dio->defer_completion || in dio_bio_end_aio()
357 (dio->op == REQ_OP_WRITE && in dio_bio_end_aio()
358 dio->inode->i_mapping->nrpages); in dio_bio_end_aio()
360 INIT_WORK(&dio->complete_work, dio_aio_complete_work); in dio_bio_end_aio()
361 queue_work(dio->inode->i_sb->s_dio_done_wq, in dio_bio_end_aio()
362 &dio->complete_work); in dio_bio_end_aio()
370 * The BIO completion handler simply queues the BIO up for the process-context
374 * implement a singly-linked list of completed BIOs, at dio->bio_list.
378 struct dio *dio = bio->bi_private; in dio_bio_end_io()
381 spin_lock_irqsave(&dio->bio_lock, flags); in dio_bio_end_io()
382 bio->bi_private = dio->bio_list; in dio_bio_end_io()
383 dio->bio_list = bio; in dio_bio_end_io()
384 if (--dio->refcount == 1 && dio->waiter) in dio_bio_end_io()
385 wake_up_process(dio->waiter); in dio_bio_end_io()
386 spin_unlock_irqrestore(&dio->bio_lock, flags); in dio_bio_end_io()
390 dio_bio_alloc(struct dio *dio, struct dio_submit *sdio, in dio_bio_alloc() argument
403 bio->bi_iter.bi_sector = first_sector; in dio_bio_alloc()
404 bio_set_op_attrs(bio, dio->op, dio->op_flags); in dio_bio_alloc()
405 if (dio->is_async) in dio_bio_alloc()
406 bio->bi_end_io = dio_bio_end_aio; in dio_bio_alloc()
408 bio->bi_end_io = dio_bio_end_io; in dio_bio_alloc()
410 bio->bi_write_hint = dio->iocb->ki_hint; in dio_bio_alloc()
412 sdio->bio = bio; in dio_bio_alloc()
413 sdio->logical_offset_in_bio = sdio->cur_page_fs_offset; in dio_bio_alloc()
421 * bios hold a dio reference between submit_bio and ->end_io.
423 static inline void dio_bio_submit(struct dio *dio, struct dio_submit *sdio) in dio_bio_submit() argument
425 struct bio *bio = sdio->bio; in dio_bio_submit()
428 bio->bi_private = dio; in dio_bio_submit()
430 spin_lock_irqsave(&dio->bio_lock, flags); in dio_bio_submit()
431 dio->refcount++; in dio_bio_submit()
432 spin_unlock_irqrestore(&dio->bio_lock, flags); in dio_bio_submit()
434 if (dio->is_async && dio->op == REQ_OP_READ && dio->should_dirty) in dio_bio_submit()
437 dio->bio_disk = bio->bi_disk; in dio_bio_submit()
439 if (sdio->submit_io) { in dio_bio_submit()
440 sdio->submit_io(bio, dio->inode, sdio->logical_offset_in_bio); in dio_bio_submit()
441 dio->bio_cookie = BLK_QC_T_NONE; in dio_bio_submit()
443 dio->bio_cookie = submit_bio(bio); in dio_bio_submit()
445 sdio->bio = NULL; in dio_bio_submit()
446 sdio->boundary = 0; in dio_bio_submit()
447 sdio->logical_offset_in_bio = 0; in dio_bio_submit()
453 static inline void dio_cleanup(struct dio *dio, struct dio_submit *sdio) in dio_cleanup() argument
455 while (sdio->head < sdio->tail) in dio_cleanup()
456 put_page(dio->pages[sdio->head++]); in dio_cleanup()
462 * all bios have been issued so that dio->refcount can only decrease. This
470 spin_lock_irqsave(&dio->bio_lock, flags); in dio_await_one()
478 while (dio->refcount > 1 && dio->bio_list == NULL) { in dio_await_one()
480 dio->waiter = current; in dio_await_one()
481 spin_unlock_irqrestore(&dio->bio_lock, flags); in dio_await_one()
482 if (!(dio->iocb->ki_flags & IOCB_HIPRI) || in dio_await_one()
483 !blk_poll(dio->bio_disk->queue, dio->bio_cookie, true)) in dio_await_one()
486 spin_lock_irqsave(&dio->bio_lock, flags); in dio_await_one()
487 dio->waiter = NULL; in dio_await_one()
489 if (dio->bio_list) { in dio_await_one()
490 bio = dio->bio_list; in dio_await_one()
491 dio->bio_list = bio->bi_private; in dio_await_one()
493 spin_unlock_irqrestore(&dio->bio_lock, flags); in dio_await_one()
498 * Process one completed BIO. No locks are held.
502 blk_status_t err = bio->bi_status; in dio_bio_complete()
503 bool should_dirty = dio->op == REQ_OP_READ && dio->should_dirty; in dio_bio_complete()
506 if (err == BLK_STS_AGAIN && (bio->bi_opf & REQ_NOWAIT)) in dio_bio_complete()
507 dio->io_error = -EAGAIN; in dio_bio_complete()
509 dio->io_error = -EIO; in dio_bio_complete()
512 if (dio->is_async && should_dirty) { in dio_bio_complete()
522 * Wait on and process all in-flight BIOs. This must only be called once
525 * errors are propagated through dio->io_error and should be propagated via
545 static inline int dio_bio_reap(struct dio *dio, struct dio_submit *sdio) in dio_bio_reap() argument
549 if (sdio->reap_counter++ >= 64) { in dio_bio_reap()
550 while (dio->bio_list) { in dio_bio_reap()
555 spin_lock_irqsave(&dio->bio_lock, flags); in dio_bio_reap()
556 bio = dio->bio_list; in dio_bio_reap()
557 dio->bio_list = bio->bi_private; in dio_bio_reap()
558 spin_unlock_irqrestore(&dio->bio_lock, flags); in dio_bio_reap()
563 sdio->reap_counter = 0; in dio_bio_reap()
579 sb->s_id); in sb_init_dio_done_wq()
581 return -ENOMEM; in sb_init_dio_done_wq()
585 old = cmpxchg(&sb->s_dio_done_wq, NULL, wq); in sb_init_dio_done_wq()
594 struct super_block *sb = dio->inode->i_sb; in dio_set_defer_completion()
596 if (dio->defer_completion) in dio_set_defer_completion()
598 dio->defer_completion = true; in dio_set_defer_completion()
599 if (!sb->s_dio_done_wq) in dio_set_defer_completion()
606 * of available blocks at sdio->blocks_available. These are in units of the
610 * it uses the passed inode-relative block number as the file offset, as usual.
612 * get_block() is passed the number of i_blkbits-sized blocks which direct_io
615 * If the fs has mapped a lot of blocks, it should populate bh->b_size to
617 * bh->b_blocknr.
622 * In the case of filesystem holes: the fs may return an arbitrarily-large
624 * buffer_mapped(). However the direct-io code will only process holes one
625 * block at a time - it will repeatedly call get_block() as it walks the hole.
627 static int get_more_blocks(struct dio *dio, struct dio_submit *sdio, in get_more_blocks() argument
631 sector_t fs_startblk; /* Into file, in filesystem-sized blocks */ in get_more_blocks()
632 sector_t fs_endblk; /* Into file, in filesystem-sized blocks */ in get_more_blocks()
633 unsigned long fs_count; /* Number of filesystem-sized blocks */ in get_more_blocks()
635 unsigned int i_blkbits = sdio->blkbits + sdio->blkfactor; in get_more_blocks()
642 ret = dio->page_errors; in get_more_blocks()
644 BUG_ON(sdio->block_in_file >= sdio->final_block_in_request); in get_more_blocks()
645 fs_startblk = sdio->block_in_file >> sdio->blkfactor; in get_more_blocks()
646 fs_endblk = (sdio->final_block_in_request - 1) >> in get_more_blocks()
647 sdio->blkfactor; in get_more_blocks()
648 fs_count = fs_endblk - fs_startblk + 1; in get_more_blocks()
650 map_bh->b_state = 0; in get_more_blocks()
651 map_bh->b_size = fs_count << i_blkbits; in get_more_blocks()
664 create = dio->op == REQ_OP_WRITE; in get_more_blocks()
665 if (dio->flags & DIO_SKIP_HOLES) { in get_more_blocks()
666 i_size = i_size_read(dio->inode); in get_more_blocks()
667 if (i_size && fs_startblk <= (i_size - 1) >> i_blkbits) in get_more_blocks()
671 ret = (*sdio->get_block)(dio->inode, fs_startblk, in get_more_blocks()
675 dio->private = map_bh->b_private; in get_more_blocks()
684 * There is no bio. Make one now.
686 static inline int dio_new_bio(struct dio *dio, struct dio_submit *sdio, in dio_new_bio() argument
692 ret = dio_bio_reap(dio, sdio); in dio_new_bio()
695 sector = start_sector << (sdio->blkbits - 9); in dio_new_bio()
696 nr_pages = min(sdio->pages_in_io, BIO_MAX_PAGES); in dio_new_bio()
698 dio_bio_alloc(dio, sdio, map_bh->b_bdev, sector, nr_pages); in dio_new_bio()
699 sdio->boundary = 0; in dio_new_bio()
707 * the just-added page.
709 * Return zero on success. Non-zero means the caller needs to start a new BIO.
711 static inline int dio_bio_add_page(struct dio_submit *sdio) in dio_bio_add_page() argument
715 ret = bio_add_page(sdio->bio, sdio->cur_page, in dio_bio_add_page()
716 sdio->cur_page_len, sdio->cur_page_offset); in dio_bio_add_page()
717 if (ret == sdio->cur_page_len) { in dio_bio_add_page()
721 if ((sdio->cur_page_len + sdio->cur_page_offset) == PAGE_SIZE) in dio_bio_add_page()
722 sdio->pages_in_io--; in dio_bio_add_page()
723 get_page(sdio->cur_page); in dio_bio_add_page()
724 sdio->final_block_in_bio = sdio->cur_page_block + in dio_bio_add_page()
725 (sdio->cur_page_len >> sdio->blkbits); in dio_bio_add_page()
736 * starts on-disk at cur_page_block.
743 static inline int dio_send_cur_page(struct dio *dio, struct dio_submit *sdio, in dio_send_cur_page() argument
748 if (sdio->bio) { in dio_send_cur_page()
749 loff_t cur_offset = sdio->cur_page_fs_offset; in dio_send_cur_page()
750 loff_t bio_next_offset = sdio->logical_offset_in_bio + in dio_send_cur_page()
751 sdio->bio->bi_iter.bi_size; in dio_send_cur_page()
756 * Btrfs cannot handle having logically non-contiguous requests in dio_send_cur_page()
759 * Logical: [0-4095][HOLE][8192-12287] in dio_send_cur_page()
760 * Physical: [0-4095] [4096-8191] in dio_send_cur_page()
767 if (sdio->final_block_in_bio != sdio->cur_page_block || in dio_send_cur_page()
769 dio_bio_submit(dio, sdio); in dio_send_cur_page()
772 if (sdio->bio == NULL) { in dio_send_cur_page()
773 ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh); in dio_send_cur_page()
778 if (dio_bio_add_page(sdio) != 0) { in dio_send_cur_page()
779 dio_bio_submit(dio, sdio); in dio_send_cur_page()
780 ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh); in dio_send_cur_page()
782 ret = dio_bio_add_page(sdio); in dio_send_cur_page()
798 * The chunk of page starts on-disk at blocknr.
800 * We perform deferred IO, by recording the last-submitted page inside our
808 submit_page_section(struct dio *dio, struct dio_submit *sdio, struct page *page, in submit_page_section() argument
814 if (dio->op == REQ_OP_WRITE) { in submit_page_section()
824 if (sdio->cur_page == page && in submit_page_section()
825 sdio->cur_page_offset + sdio->cur_page_len == offset && in submit_page_section()
826 sdio->cur_page_block + in submit_page_section()
827 (sdio->cur_page_len >> sdio->blkbits) == blocknr) { in submit_page_section()
828 sdio->cur_page_len += len; in submit_page_section()
835 if (sdio->cur_page) { in submit_page_section()
836 ret = dio_send_cur_page(dio, sdio, map_bh); in submit_page_section()
837 put_page(sdio->cur_page); in submit_page_section()
838 sdio->cur_page = NULL; in submit_page_section()
844 sdio->cur_page = page; in submit_page_section()
845 sdio->cur_page_offset = offset; in submit_page_section()
846 sdio->cur_page_len = len; in submit_page_section()
847 sdio->cur_page_block = blocknr; in submit_page_section()
848 sdio->cur_page_fs_offset = sdio->block_in_file << sdio->blkbits; in submit_page_section()
851 * If sdio->boundary then we want to schedule the IO now to in submit_page_section()
854 if (sdio->boundary) { in submit_page_section()
855 ret = dio_send_cur_page(dio, sdio, map_bh); in submit_page_section()
856 if (sdio->bio) in submit_page_section()
857 dio_bio_submit(dio, sdio); in submit_page_section()
858 put_page(sdio->cur_page); in submit_page_section()
859 sdio->cur_page = NULL; in submit_page_section()
866 * the block for us, we need to fill-in the unused portion of the
867 * block with zeros. This happens only if user-buffer, fileoffset or
868 * io length is not filesystem block-size multiple.
873 static inline void dio_zero_block(struct dio *dio, struct dio_submit *sdio, in dio_zero_block() argument
881 sdio->start_zero_done = 1; in dio_zero_block()
882 if (!sdio->blkfactor || !buffer_new(map_bh)) in dio_zero_block()
885 dio_blocks_per_fs_block = 1 << sdio->blkfactor; in dio_zero_block()
886 this_chunk_blocks = sdio->block_in_file & (dio_blocks_per_fs_block - 1); in dio_zero_block()
896 this_chunk_blocks = dio_blocks_per_fs_block - this_chunk_blocks; in dio_zero_block()
898 this_chunk_bytes = this_chunk_blocks << sdio->blkbits; in dio_zero_block()
901 if (submit_page_section(dio, sdio, page, 0, this_chunk_bytes, in dio_zero_block()
902 sdio->next_block_for_io, map_bh)) in dio_zero_block()
905 sdio->next_block_for_io += this_chunk_blocks; in dio_zero_block()
914 * happily perform page-sized but 512-byte aligned IOs. It is important that
917 * So what we do is to permit the ->get_block function to populate bh.b_size
920 * For best results, the blockdev should be set up with 512-byte i_blkbits and
924 static int do_direct_IO(struct dio *dio, struct dio_submit *sdio, in do_direct_IO() argument
927 const unsigned blkbits = sdio->blkbits; in do_direct_IO()
928 const unsigned i_blkbits = blkbits + sdio->blkfactor; in do_direct_IO()
931 while (sdio->block_in_file < sdio->final_block_in_request) { in do_direct_IO()
935 page = dio_get_page(dio, sdio); in do_direct_IO()
940 from = sdio->head ? 0 : sdio->from; in do_direct_IO()
941 to = (sdio->head == sdio->tail - 1) ? sdio->to : PAGE_SIZE; in do_direct_IO()
942 sdio->head++; in do_direct_IO()
949 if (sdio->blocks_available == 0) { in do_direct_IO()
956 ret = get_more_blocks(dio, sdio, map_bh); in do_direct_IO()
964 sdio->blocks_available = in do_direct_IO()
965 map_bh->b_size >> blkbits; in do_direct_IO()
966 sdio->next_block_for_io = in do_direct_IO()
967 map_bh->b_blocknr << sdio->blkfactor; in do_direct_IO()
970 map_bh->b_bdev, in do_direct_IO()
971 map_bh->b_blocknr, in do_direct_IO()
972 map_bh->b_size >> i_blkbits); in do_direct_IO()
975 if (!sdio->blkfactor) in do_direct_IO()
978 blkmask = (1 << sdio->blkfactor) - 1; in do_direct_IO()
979 dio_remainder = (sdio->block_in_file & blkmask); in do_direct_IO()
983 * starts partway into a fs-block, in do_direct_IO()
984 * dio_remainder will be non-zero. If the IO in do_direct_IO()
990 * on-disk in do_direct_IO()
993 sdio->next_block_for_io += dio_remainder; in do_direct_IO()
994 sdio->blocks_available -= dio_remainder; in do_direct_IO()
1001 /* AKPM: eargh, -ENOTBLK is a hack */ in do_direct_IO()
1002 if (dio->op == REQ_OP_WRITE) { in do_direct_IO()
1004 return -ENOTBLK; in do_direct_IO()
1011 i_size_aligned = ALIGN(i_size_read(dio->inode), in do_direct_IO()
1013 if (sdio->block_in_file >= in do_direct_IO()
1020 sdio->block_in_file++; in do_direct_IO()
1022 dio->result += 1 << blkbits; in do_direct_IO()
1031 if (unlikely(sdio->blkfactor && !sdio->start_zero_done)) in do_direct_IO()
1032 dio_zero_block(dio, sdio, 0, map_bh); in do_direct_IO()
1038 this_chunk_blocks = sdio->blocks_available; in do_direct_IO()
1039 u = (to - from) >> blkbits; in do_direct_IO()
1042 u = sdio->final_block_in_request - sdio->block_in_file; in do_direct_IO()
1048 if (this_chunk_blocks == sdio->blocks_available) in do_direct_IO()
1049 sdio->boundary = buffer_boundary(map_bh); in do_direct_IO()
1050 ret = submit_page_section(dio, sdio, page, in do_direct_IO()
1053 sdio->next_block_for_io, in do_direct_IO()
1059 sdio->next_block_for_io += this_chunk_blocks; in do_direct_IO()
1061 sdio->block_in_file += this_chunk_blocks; in do_direct_IO()
1063 dio->result += this_chunk_bytes; in do_direct_IO()
1064 sdio->blocks_available -= this_chunk_blocks; in do_direct_IO()
1066 BUG_ON(sdio->block_in_file > sdio->final_block_in_request); in do_direct_IO()
1067 if (sdio->block_in_file == sdio->final_block_in_request) in do_direct_IO()
1088 * return code that the caller will hand to ->complete(). in drop_refcount()
1094 spin_lock_irqsave(&dio->bio_lock, flags); in drop_refcount()
1095 ret2 = --dio->refcount; in drop_refcount()
1096 spin_unlock_irqrestore(&dio->bio_lock, flags); in drop_refcount()
1104 * - if the flags value contains DIO_LOCKING we use a fancy locking
1109 * - if the flags value does NOT contain DIO_LOCKING we don't use any
1120 * NOTE: if you pass "sdio" to anything by pointer make sure that function
1131 unsigned i_blkbits = READ_ONCE(inode->i_blkbits); in do_blockdev_direct_IO()
1133 unsigned blocksize_mask = (1 << blkbits) - 1; in do_blockdev_direct_IO()
1134 ssize_t retval = -EINVAL; in do_blockdev_direct_IO()
1136 loff_t offset = iocb->ki_pos; in do_blockdev_direct_IO()
1139 struct dio_submit sdio = { 0, }; in do_blockdev_direct_IO() local
1155 return -ENOMEM; in do_blockdev_direct_IO()
1163 dio->flags = flags; in do_blockdev_direct_IO()
1164 if (dio->flags & DIO_LOCKING && iov_iter_rw(iter) == READ) { in do_blockdev_direct_IO()
1170 dio->i_size = i_size_read(inode); in do_blockdev_direct_IO()
1171 if (iov_iter_rw(iter) == READ && offset >= dio->i_size) { in do_blockdev_direct_IO()
1179 blocksize_mask = (1 << blkbits) - 1; in do_blockdev_direct_IO()
1184 if (dio->flags & DIO_LOCKING && iov_iter_rw(iter) == READ) { in do_blockdev_direct_IO()
1185 struct address_space *mapping = iocb->ki_filp->f_mapping; in do_blockdev_direct_IO()
1187 retval = filemap_write_and_wait_range(mapping, offset, end - 1); in do_blockdev_direct_IO()
1199 dio->is_async = false; in do_blockdev_direct_IO()
1201 dio->is_async = false; in do_blockdev_direct_IO()
1203 dio->is_async = true; in do_blockdev_direct_IO()
1205 dio->inode = inode; in do_blockdev_direct_IO()
1207 dio->op = REQ_OP_WRITE; in do_blockdev_direct_IO()
1208 dio->op_flags = REQ_SYNC | REQ_IDLE; in do_blockdev_direct_IO()
1209 if (iocb->ki_flags & IOCB_NOWAIT) in do_blockdev_direct_IO()
1210 dio->op_flags |= REQ_NOWAIT; in do_blockdev_direct_IO()
1212 dio->op = REQ_OP_READ; in do_blockdev_direct_IO()
1214 if (iocb->ki_flags & IOCB_HIPRI) in do_blockdev_direct_IO()
1215 dio->op_flags |= REQ_HIPRI; in do_blockdev_direct_IO()
1219 * so that we can call ->fsync. in do_blockdev_direct_IO()
1221 if (dio->is_async && iov_iter_rw(iter) == WRITE) { in do_blockdev_direct_IO()
1223 if (iocb->ki_flags & IOCB_DSYNC) in do_blockdev_direct_IO()
1225 else if (!dio->inode->i_sb->s_dio_done_wq) { in do_blockdev_direct_IO()
1231 retval = sb_init_dio_done_wq(dio->inode->i_sb); in do_blockdev_direct_IO()
1243 sdio.blkbits = blkbits; in do_blockdev_direct_IO()
1244 sdio.blkfactor = i_blkbits - blkbits; in do_blockdev_direct_IO()
1245 sdio.block_in_file = offset >> blkbits; in do_blockdev_direct_IO()
1247 sdio.get_block = get_block; in do_blockdev_direct_IO()
1248 dio->end_io = end_io; in do_blockdev_direct_IO()
1249 sdio.submit_io = submit_io; in do_blockdev_direct_IO()
1250 sdio.final_block_in_bio = -1; in do_blockdev_direct_IO()
1251 sdio.next_block_for_io = -1; in do_blockdev_direct_IO()
1253 dio->iocb = iocb; in do_blockdev_direct_IO()
1255 spin_lock_init(&dio->bio_lock); in do_blockdev_direct_IO()
1256 dio->refcount = 1; in do_blockdev_direct_IO()
1258 dio->should_dirty = iter_is_iovec(iter) && iov_iter_rw(iter) == READ; in do_blockdev_direct_IO()
1259 sdio.iter = iter; in do_blockdev_direct_IO()
1260 sdio.final_block_in_request = end >> blkbits; in do_blockdev_direct_IO()
1263 * In case of non-aligned buffers, we may need 2 more in do_blockdev_direct_IO()
1266 if (unlikely(sdio.blkfactor)) in do_blockdev_direct_IO()
1267 sdio.pages_in_io = 2; in do_blockdev_direct_IO()
1269 sdio.pages_in_io += iov_iter_npages(iter, INT_MAX); in do_blockdev_direct_IO()
1273 retval = do_direct_IO(dio, &sdio, &map_bh); in do_blockdev_direct_IO()
1275 dio_cleanup(dio, &sdio); in do_blockdev_direct_IO()
1277 if (retval == -ENOTBLK) { in do_blockdev_direct_IO()
1285 * There may be some unwritten disk at the end of a part-written in do_blockdev_direct_IO()
1286 * fs-block-sized block. Go zero that now. in do_blockdev_direct_IO()
1288 dio_zero_block(dio, &sdio, 1, &map_bh); in do_blockdev_direct_IO()
1290 if (sdio.cur_page) { in do_blockdev_direct_IO()
1293 ret2 = dio_send_cur_page(dio, &sdio, &map_bh); in do_blockdev_direct_IO()
1296 put_page(sdio.cur_page); in do_blockdev_direct_IO()
1297 sdio.cur_page = NULL; in do_blockdev_direct_IO()
1299 if (sdio.bio) in do_blockdev_direct_IO()
1300 dio_bio_submit(dio, &sdio); in do_blockdev_direct_IO()
1308 dio_cleanup(dio, &sdio); in do_blockdev_direct_IO()
1315 if (iov_iter_rw(iter) == READ && (dio->flags & DIO_LOCKING)) in do_blockdev_direct_IO()
1316 inode_unlock(dio->inode); in do_blockdev_direct_IO()
1322 * call aio_complete is when we return -EIOCBQUEUED, so we key on that. in do_blockdev_direct_IO()
1323 * This had *better* be the only place that raises -EIOCBQUEUED. in do_blockdev_direct_IO()
1325 BUG_ON(retval == -EIOCBQUEUED); in do_blockdev_direct_IO()
1326 if (dio->is_async && retval == 0 && dio->result && in do_blockdev_direct_IO()
1327 (iov_iter_rw(iter) == READ || dio->result == count)) in do_blockdev_direct_IO()
1328 retval = -EIOCBQUEUED; in do_blockdev_direct_IO()
1335 BUG_ON(retval != -EIOCBQUEUED); in do_blockdev_direct_IO()
1340 if (dio->flags & DIO_LOCKING && iov_iter_rw(iter) == READ) in do_blockdev_direct_IO()
1361 prefetch(&bdev->bd_disk->part_tbl); in __blockdev_direct_IO()
1362 prefetch(bdev->bd_disk->queue); in __blockdev_direct_IO()
1363 prefetch((char *)bdev->bd_disk->queue + SMP_CACHE_BYTES); in __blockdev_direct_IO()