Lines Matching +full:supports +full:- +full:cqe
1 // SPDX-License-Identifier: GPL-2.0
6 #include <linux/blk-mq.h>
34 return req->flags & REQ_F_SUPPORT_NOWAIT; in io_file_supports_nowait()
43 uiov = u64_to_user_ptr(rw->addr); in io_iov_compat_buffer_select_prep()
45 return -EFAULT; in io_iov_compat_buffer_select_prep()
46 if (__get_user(clen, &uiov->iov_len)) in io_iov_compat_buffer_select_prep()
47 return -EFAULT; in io_iov_compat_buffer_select_prep()
49 return -EINVAL; in io_iov_compat_buffer_select_prep()
51 rw->len = clen; in io_iov_compat_buffer_select_prep()
62 if (rw->len != 1) in io_iov_buffer_select_prep()
63 return -EINVAL; in io_iov_buffer_select_prep()
66 if (req->ctx->compat) in io_iov_buffer_select_prep()
70 uiov = u64_to_user_ptr(rw->addr); in io_iov_buffer_select_prep()
72 return -EFAULT; in io_iov_buffer_select_prep()
73 rw->len = iov.iov_len; in io_iov_buffer_select_prep()
83 rw->kiocb.ki_pos = READ_ONCE(sqe->off); in io_prep_rw()
84 /* used for fixed read/write too - just read unconditionally */ in io_prep_rw()
85 req->buf_index = READ_ONCE(sqe->buf_index); in io_prep_rw()
87 ioprio = READ_ONCE(sqe->ioprio); in io_prep_rw()
93 rw->kiocb.ki_ioprio = ioprio; in io_prep_rw()
95 rw->kiocb.ki_ioprio = get_current_ioprio(); in io_prep_rw()
97 rw->kiocb.dio_complete = NULL; in io_prep_rw()
99 rw->addr = READ_ONCE(sqe->addr); in io_prep_rw()
100 rw->len = READ_ONCE(sqe->len); in io_prep_rw()
101 rw->flags = READ_ONCE(sqe->rw_flags); in io_prep_rw()
114 * Have to do this validation here, as this is in io_read() rw->len in io_prep_rwv()
117 if (req->flags & REQ_F_BUFFER_SELECT) in io_prep_rwv()
125 struct io_ring_ctx *ctx = req->ctx; in io_prep_rw_fixed()
133 if (unlikely(req->buf_index >= ctx->nr_user_bufs)) in io_prep_rw_fixed()
134 return -EFAULT; in io_prep_rw_fixed()
135 index = array_index_nospec(req->buf_index, ctx->nr_user_bufs); in io_prep_rw_fixed()
136 req->imu = ctx->user_bufs[index]; in io_prep_rw_fixed()
151 if (!(req->flags & REQ_F_BUFFER_SELECT)) in io_read_mshot_prep()
152 return -EINVAL; in io_read_mshot_prep()
158 if (rw->addr || rw->len) in io_read_mshot_prep()
159 return -EINVAL; in io_read_mshot_prep()
161 req->flags |= REQ_F_APOLL_MULTISHOT; in io_read_mshot_prep()
167 struct io_async_rw *io = req->async_data; in io_readv_writev_cleanup()
169 kfree(io->free_iovec); in io_readv_writev_cleanup()
176 if (rw->kiocb.ki_pos != -1) in io_kiocb_update_pos()
177 return &rw->kiocb.ki_pos; in io_kiocb_update_pos()
179 if (!(req->file->f_mode & FMODE_STREAM)) { in io_kiocb_update_pos()
180 req->flags |= REQ_F_CUR_POS; in io_kiocb_update_pos()
181 rw->kiocb.ki_pos = req->file->f_pos; in io_kiocb_update_pos()
182 return &rw->kiocb.ki_pos; in io_kiocb_update_pos()
185 rw->kiocb.ki_pos = 0; in io_kiocb_update_pos()
191 req->io_task_work.func = io_queue_iowq; in io_req_task_queue_reissue()
198 struct io_async_rw *io = req->async_data; in io_resubmit_prep()
202 iov_iter_restore(&io->s.iter, &io->s.iter_state); in io_resubmit_prep()
208 umode_t mode = file_inode(req->file)->i_mode; in io_rw_should_reissue()
209 struct io_ring_ctx *ctx = req->ctx; in io_rw_should_reissue()
213 if ((req->flags & REQ_F_NOWAIT) || (io_wq_current_is_worker() && in io_rw_should_reissue()
214 !(ctx->flags & IORING_SETUP_IOPOLL))) in io_rw_should_reissue()
219 * -EAGAIN. in io_rw_should_reissue()
221 if (percpu_ref_is_dying(&ctx->refs)) in io_rw_should_reissue()
224 * Play it safe and assume not safe to re-import and reissue if we're in io_rw_should_reissue()
227 if (!same_thread_group(req->task, current) || !in_task()) in io_rw_should_reissue()
244 if (req->flags & REQ_F_ISREG) { in io_req_end_write()
247 kiocb_end_write(&rw->kiocb); in io_req_end_write()
259 if (rw->kiocb.ki_flags & IOCB_WRITE) { in io_req_io_end()
261 fsnotify_modify(req->file); in io_req_io_end()
263 fsnotify_access(req->file); in io_req_io_end()
269 if (unlikely(res != req->cqe.res)) { in __io_complete_rw_common()
270 if ((res == -EAGAIN || res == -EOPNOTSUPP) && in __io_complete_rw_common()
277 req->flags |= REQ_F_REISSUE | REQ_F_PARTIAL_IO; in __io_complete_rw_common()
281 req->cqe.res = res; in __io_complete_rw_common()
288 struct io_async_rw *io = req->async_data; in io_fixup_rw_res()
291 if (req_has_async_data(req) && io->bytes_done > 0) { in io_fixup_rw_res()
293 res = io->bytes_done; in io_fixup_rw_res()
295 res += io->bytes_done; in io_fixup_rw_res()
303 struct kiocb *kiocb = &rw->kiocb; in io_req_rw_complete()
305 if ((kiocb->ki_flags & IOCB_DIO_CALLER_COMP) && kiocb->dio_complete) { in io_req_rw_complete()
306 long res = kiocb->dio_complete(rw->kiocb.private); in io_req_rw_complete()
313 if (req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)) { in io_req_rw_complete()
314 unsigned issue_flags = ts->locked ? 0 : IO_URING_F_UNLOCKED; in io_req_rw_complete()
316 req->cqe.flags |= io_put_kbuf(req, issue_flags); in io_req_rw_complete()
326 if (!kiocb->dio_complete || !(kiocb->ki_flags & IOCB_DIO_CALLER_COMP)) { in io_complete_rw()
331 req->io_task_work.func = io_req_rw_complete; in io_complete_rw()
340 if (kiocb->ki_flags & IOCB_WRITE) in io_complete_rw_iopoll()
342 if (unlikely(res != req->cqe.res)) { in io_complete_rw_iopoll()
343 if (res == -EAGAIN && io_rw_should_reissue(req)) { in io_complete_rw_iopoll()
344 req->flags |= REQ_F_REISSUE | REQ_F_PARTIAL_IO; in io_complete_rw_iopoll()
347 req->cqe.res = res; in io_complete_rw_iopoll()
350 /* order with io_iopoll_complete() checking ->iopoll_completed */ in io_complete_rw_iopoll()
351 smp_store_release(&req->iopoll_completed, 1); in io_complete_rw_iopoll()
357 if (ret == -EIOCBQUEUED) in io_rw_done()
363 case -ERESTARTSYS: in io_rw_done()
364 case -ERESTARTNOINTR: in io_rw_done()
365 case -ERESTARTNOHAND: in io_rw_done()
366 case -ERESTART_RESTARTBLOCK: in io_rw_done()
372 ret = -EINTR; in io_rw_done()
377 INDIRECT_CALL_2(kiocb->ki_complete, io_complete_rw_iopoll, in io_rw_done()
387 if (ret >= 0 && req->flags & REQ_F_CUR_POS) in kiocb_done()
388 req->file->f_pos = rw->kiocb.ki_pos; in kiocb_done()
389 if (ret >= 0 && (rw->kiocb.ki_complete == io_complete_rw)) { in kiocb_done()
401 io_rw_done(&rw->kiocb, ret); in kiocb_done()
404 if (req->flags & REQ_F_REISSUE) { in kiocb_done()
405 req->flags &= ~REQ_F_REISSUE; in kiocb_done()
419 struct iov_iter *iter = &s->iter; in __io_import_iovec()
420 u8 opcode = req->opcode; in __io_import_iovec()
427 ret = io_import_fixed(ddir, iter, req->imu, rw->addr, rw->len); in __io_import_iovec()
433 buf = u64_to_user_ptr(rw->addr); in __io_import_iovec()
434 sqe_len = rw->len; in __io_import_iovec()
436 if (!io_issue_defs[opcode].vectored || req->flags & REQ_F_BUFFER_SELECT) { in __io_import_iovec()
440 return ERR_PTR(-ENOBUFS); in __io_import_iovec()
441 rw->addr = (unsigned long) buf; in __io_import_iovec()
442 rw->len = sqe_len; in __io_import_iovec()
451 iovec = s->fast_iov; in __io_import_iovec()
453 req->ctx->compat); in __io_import_iovec()
467 iov_iter_save_state(&s->iter, &s->iter_state); in io_import_iovec()
473 return (kiocb->ki_filp->f_mode & FMODE_STREAM) ? NULL : &kiocb->ki_pos; in io_kiocb_ppos()
477 * For files that don't have ->read_iter() and ->write_iter(), handle them
478 * by looping over ->read() or ->write() manually.
482 struct kiocb *kiocb = &rw->kiocb; in loop_rw_iter()
483 struct file *file = kiocb->ki_filp; in loop_rw_iter()
489 * support non-blocking either. For the latter, this just causes in loop_rw_iter()
492 if (kiocb->ki_flags & IOCB_HIPRI) in loop_rw_iter()
493 return -EOPNOTSUPP; in loop_rw_iter()
494 if ((kiocb->ki_flags & IOCB_NOWAIT) && in loop_rw_iter()
495 !(kiocb->ki_filp->f_flags & O_NONBLOCK)) in loop_rw_iter()
496 return -EAGAIN; in loop_rw_iter()
506 addr = iter->ubuf + iter->iov_offset; in loop_rw_iter()
512 addr = u64_to_user_ptr(rw->addr); in loop_rw_iter()
513 len = rw->len; in loop_rw_iter()
517 nr = file->f_op->read(file, addr, len, ppos); in loop_rw_iter()
519 nr = file->f_op->write(file, addr, len, ppos); in loop_rw_iter()
530 rw->addr += nr; in loop_rw_iter()
531 rw->len -= nr; in loop_rw_iter()
532 if (!rw->len) in loop_rw_iter()
545 struct io_async_rw *io = req->async_data; in io_req_map_rw()
547 memcpy(&io->s.iter, iter, sizeof(*iter)); in io_req_map_rw()
548 io->free_iovec = iovec; in io_req_map_rw()
549 io->bytes_done = 0; in io_req_map_rw()
556 io->s.iter.__iov = io->s.fast_iov; in io_req_map_rw()
557 if (iter->__iov != fast_iov) { in io_req_map_rw()
558 iov_off = iter_iov(iter) - fast_iov; in io_req_map_rw()
559 io->s.iter.__iov += iov_off; in io_req_map_rw()
561 if (io->s.fast_iov != fast_iov) in io_req_map_rw()
562 memcpy(io->s.fast_iov + iov_off, fast_iov + iov_off, in io_req_map_rw()
563 sizeof(struct iovec) * iter->nr_segs); in io_req_map_rw()
565 req->flags |= REQ_F_NEED_CLEANUP; in io_req_map_rw()
572 if (!force && !io_cold_defs[req->opcode].prep_async) in io_setup_async_rw()
575 if (!io_cold_defs[req->opcode].async_size) in io_setup_async_rw()
582 return -ENOMEM; in io_setup_async_rw()
585 io_req_map_rw(req, iovec, s->fast_iov, &s->iter); in io_setup_async_rw()
586 iorw = req->async_data; in io_setup_async_rw()
588 iov_iter_save_state(&iorw->s.iter, &iorw->s.iter_state); in io_setup_async_rw()
595 struct io_async_rw *iorw = req->async_data; in io_rw_prep_async()
599 iorw->bytes_done = 0; in io_rw_prep_async()
600 iorw->free_iovec = NULL; in io_rw_prep_async()
602 /* submission path, ->uring_lock should already be taken */ in io_rw_prep_async()
603 ret = io_import_iovec(rw, req, &iov, &iorw->s, 0); in io_rw_prep_async()
608 iorw->free_iovec = iov; in io_rw_prep_async()
609 req->flags |= REQ_F_NEED_CLEANUP; in io_rw_prep_async()
639 struct io_kiocb *req = wait->private; in io_async_buf_func()
648 rw->kiocb.ki_flags &= ~IOCB_WAITQ; in io_async_buf_func()
649 list_del_init(&wait->entry); in io_async_buf_func()
668 struct io_async_rw *io = req->async_data; in io_rw_should_retry()
669 struct wait_page_queue *wait = &io->wpq; in io_rw_should_retry()
671 struct kiocb *kiocb = &rw->kiocb; in io_rw_should_retry()
673 /* never retry for NOWAIT, we just complete with -EAGAIN */ in io_rw_should_retry()
674 if (req->flags & REQ_F_NOWAIT) in io_rw_should_retry()
678 if (kiocb->ki_flags & (IOCB_DIRECT | IOCB_HIPRI)) in io_rw_should_retry()
685 if (file_can_poll(req->file) || !(req->file->f_mode & FMODE_BUF_RASYNC)) in io_rw_should_retry()
688 wait->wait.func = io_async_buf_func; in io_rw_should_retry()
689 wait->wait.private = req; in io_rw_should_retry()
690 wait->wait.flags = 0; in io_rw_should_retry()
691 INIT_LIST_HEAD(&wait->wait.entry); in io_rw_should_retry()
692 kiocb->ki_flags |= IOCB_WAITQ; in io_rw_should_retry()
693 kiocb->ki_flags &= ~IOCB_NOWAIT; in io_rw_should_retry()
694 kiocb->ki_waitq = wait; in io_rw_should_retry()
700 struct file *file = rw->kiocb.ki_filp; in io_iter_do_read()
702 if (likely(file->f_op->read_iter)) in io_iter_do_read()
703 return call_read_iter(file, &rw->kiocb, iter); in io_iter_do_read()
704 else if (file->f_op->read) in io_iter_do_read()
707 return -EINVAL; in io_iter_do_read()
712 return req->flags & REQ_F_ISREG || in need_complete_io()
713 S_ISBLK(file_inode(req->file)->i_mode); in need_complete_io()
719 struct kiocb *kiocb = &rw->kiocb; in io_rw_init_file()
720 struct io_ring_ctx *ctx = req->ctx; in io_rw_init_file()
721 struct file *file = req->file; in io_rw_init_file()
724 if (unlikely(!file || !(file->f_mode & mode))) in io_rw_init_file()
725 return -EBADF; in io_rw_init_file()
727 if (!(req->flags & REQ_F_FIXED_FILE)) in io_rw_init_file()
728 req->flags |= io_file_get_flags(file); in io_rw_init_file()
730 kiocb->ki_flags = file->f_iocb_flags; in io_rw_init_file()
731 ret = kiocb_set_rw_flags(kiocb, rw->flags); in io_rw_init_file()
734 kiocb->ki_flags |= IOCB_ALLOC_CACHE; in io_rw_init_file()
738 * supports async. Otherwise it's impossible to use O_NONBLOCK files in io_rw_init_file()
741 if ((kiocb->ki_flags & IOCB_NOWAIT) || in io_rw_init_file()
742 ((file->f_flags & O_NONBLOCK) && !io_file_supports_nowait(req))) in io_rw_init_file()
743 req->flags |= REQ_F_NOWAIT; in io_rw_init_file()
745 if (ctx->flags & IORING_SETUP_IOPOLL) { in io_rw_init_file()
746 if (!(kiocb->ki_flags & IOCB_DIRECT) || !file->f_op->iopoll) in io_rw_init_file()
747 return -EOPNOTSUPP; in io_rw_init_file()
749 kiocb->private = NULL; in io_rw_init_file()
750 kiocb->ki_flags |= IOCB_HIPRI; in io_rw_init_file()
751 kiocb->ki_complete = io_complete_rw_iopoll; in io_rw_init_file()
752 req->iopoll_completed = 0; in io_rw_init_file()
754 if (kiocb->ki_flags & IOCB_HIPRI) in io_rw_init_file()
755 return -EINVAL; in io_rw_init_file()
756 kiocb->ki_complete = io_complete_rw; in io_rw_init_file()
767 struct kiocb *kiocb = &rw->kiocb; in __io_read()
778 io = req->async_data; in __io_read()
779 s = &io->s; in __io_read()
782 * Safe and required to re-import if we're using provided in __io_read()
796 iov_iter_restore(&s->iter, &s->iter_state); in __io_read()
804 req->cqe.res = iov_iter_count(&s->iter); in __io_read()
810 return ret ?: -EAGAIN; in __io_read()
812 kiocb->ki_flags |= IOCB_NOWAIT; in __io_read()
814 /* Ensure we clear previously set non-block flag */ in __io_read()
815 kiocb->ki_flags &= ~IOCB_NOWAIT; in __io_read()
820 ret = rw_verify_area(READ, req->file, ppos, req->cqe.res); in __io_read()
826 ret = io_iter_do_read(rw, &s->iter); in __io_read()
828 if (ret == -EAGAIN || (req->flags & REQ_F_REISSUE)) { in __io_read()
829 req->flags &= ~REQ_F_REISSUE; in __io_read()
834 if (file_can_poll(req->file) && !io_issue_defs[req->opcode].vectored) in __io_read()
835 return -EAGAIN; in __io_read()
836 /* IOPOLL retry should happen for io-wq threads */ in __io_read()
837 if (!force_nonblock && !(req->ctx->flags & IORING_SETUP_IOPOLL)) in __io_read()
840 if (req->flags & REQ_F_NOWAIT) in __io_read()
843 } else if (ret == -EIOCBQUEUED) { in __io_read()
847 } else if (ret == req->cqe.res || ret <= 0 || !force_nonblock || in __io_read()
848 (req->flags & REQ_F_NOWAIT) || !need_complete_io(req)) { in __io_read()
858 iov_iter_restore(&s->iter, &s->iter_state); in __io_read()
867 io = req->async_data; in __io_read()
868 s = &io->s; in __io_read()
880 iov_iter_advance(&s->iter, ret); in __io_read()
881 if (!iov_iter_count(&s->iter)) in __io_read()
883 io->bytes_done += ret; in __io_read()
884 iov_iter_save_state(&s->iter, &s->iter_state); in __io_read()
888 kiocb->ki_flags &= ~IOCB_WAITQ; in __io_read()
889 return -EAGAIN; in __io_read()
892 req->cqe.res = iov_iter_count(&s->iter); in __io_read()
895 * we get -EIOCBQUEUED, then we'll get a notification when the in __io_read()
899 ret = io_iter_do_read(rw, &s->iter); in __io_read()
900 if (ret == -EIOCBQUEUED) in __io_read()
903 kiocb->ki_flags &= ~IOCB_WAITQ; in __io_read()
904 iov_iter_restore(&s->iter, &s->iter_state); in __io_read()
933 if (!file_can_poll(req->file)) in io_read_mshot()
934 return -EBADFD; in io_read_mshot()
939 * If we get -EAGAIN, recycle our buffer and just let normal poll in io_read_mshot()
942 if (ret == -EAGAIN) { in io_read_mshot()
944 * Reset rw->len to 0 again to avoid clamping future mshot in io_read_mshot()
948 rw->len = 0; in io_read_mshot()
949 return -EAGAIN; in io_read_mshot()
957 * Put our buffer and post a CQE. If we fail to post a CQE, then in io_read_mshot()
961 rw->len = 0; /* similarly to above, reset len to 0 */ in io_read_mshot()
975 return -EAGAIN; in io_read_mshot()
980 * Either an error, or we've hit overflow posting the CQE. For any in io_read_mshot()
994 struct kiocb *kiocb = &rw->kiocb; in io_write()
1004 struct io_async_rw *io = req->async_data; in io_write()
1006 s = &io->s; in io_write()
1007 iov_iter_restore(&s->iter, &s->iter_state); in io_write()
1015 req->cqe.res = iov_iter_count(&s->iter); in io_write()
1022 /* File path supports NOWAIT for non-direct_IO only for block devices. */ in io_write()
1023 if (!(kiocb->ki_flags & IOCB_DIRECT) && in io_write()
1024 !(kiocb->ki_filp->f_mode & FMODE_BUF_WASYNC) && in io_write()
1025 (req->flags & REQ_F_ISREG)) in io_write()
1028 kiocb->ki_flags |= IOCB_NOWAIT; in io_write()
1030 /* Ensure we clear previously set non-block flag */ in io_write()
1031 kiocb->ki_flags &= ~IOCB_NOWAIT; in io_write()
1036 ret = rw_verify_area(WRITE, req->file, ppos, req->cqe.res); in io_write()
1042 if (req->flags & REQ_F_ISREG) in io_write()
1044 kiocb->ki_flags |= IOCB_WRITE; in io_write()
1046 if (likely(req->file->f_op->write_iter)) in io_write()
1047 ret2 = call_write_iter(req->file, kiocb, &s->iter); in io_write()
1048 else if (req->file->f_op->write) in io_write()
1049 ret2 = loop_rw_iter(WRITE, rw, &s->iter); in io_write()
1051 ret2 = -EINVAL; in io_write()
1053 if (req->flags & REQ_F_REISSUE) { in io_write()
1054 req->flags &= ~REQ_F_REISSUE; in io_write()
1055 ret2 = -EAGAIN; in io_write()
1059 * Raw bdev writes will return -EOPNOTSUPP for IOCB_NOWAIT. Just in io_write()
1062 if (ret2 == -EOPNOTSUPP && (kiocb->ki_flags & IOCB_NOWAIT)) in io_write()
1063 ret2 = -EAGAIN; in io_write()
1065 if (ret2 == -EAGAIN && (req->flags & REQ_F_NOWAIT)) in io_write()
1067 if (!force_nonblock || ret2 != -EAGAIN) { in io_write()
1068 /* IOPOLL retry should happen for io-wq threads */ in io_write()
1069 if (ret2 == -EAGAIN && (req->ctx->flags & IORING_SETUP_IOPOLL)) in io_write()
1072 if (ret2 != req->cqe.res && ret2 >= 0 && need_complete_io(req)) { in io_write()
1075 trace_io_uring_short_write(req->ctx, kiocb->ki_pos - ret2, in io_write()
1076 req->cqe.res, ret2); in io_write()
1083 iov_iter_save_state(&s->iter, &s->iter_state); in io_write()
1086 io = req->async_data; in io_write()
1088 io->bytes_done += ret2; in io_write()
1090 if (kiocb->ki_flags & IOCB_WRITE) in io_write()
1092 return ret ? ret : -EAGAIN; in io_write()
1098 iov_iter_restore(&s->iter, &s->iter_state); in io_write()
1101 if (kiocb->ki_flags & IOCB_WRITE) in io_write()
1103 return -EAGAIN; in io_write()
1117 res = io_fixup_rw_res(req, req->cqe.res); in io_rw_fail()
1118 io_req_set_res(req, res, req->cqe.flags); in io_rw_fail()
1132 if (ctx->poll_multi_queue || force_nonspin) in io_do_iopoll()
1135 wq_list_for_each(pos, start, &ctx->iopoll_list) { in io_do_iopoll()
1137 struct file *file = req->file; in io_do_iopoll()
1145 if (READ_ONCE(req->iopoll_completed)) in io_do_iopoll()
1148 if (req->opcode == IORING_OP_URING_CMD) { in io_do_iopoll()
1152 ret = file->f_op->uring_cmd_iopoll(ioucmd, &iob, in io_do_iopoll()
1157 ret = file->f_op->iopoll(&rw->kiocb, &iob, poll_flags); in io_do_iopoll()
1166 READ_ONCE(req->iopoll_completed)) in io_do_iopoll()
1179 /* order with io_complete_rw_iopoll(), e.g. ->result updates */ in io_do_iopoll()
1180 if (!smp_load_acquire(&req->iopoll_completed)) in io_do_iopoll()
1183 req->cqe.flags = io_put_kbuf(req, 0); in io_do_iopoll()
1188 pos = start ? start->next : ctx->iopoll_list.first; in io_do_iopoll()
1189 wq_list_cut(&ctx->iopoll_list, prev, start); in io_do_iopoll()
1191 if (WARN_ON_ONCE(!wq_list_empty(&ctx->submit_state.compl_reqs))) in io_do_iopoll()
1193 ctx->submit_state.compl_reqs.first = pos; in io_do_iopoll()