1 /* SPDX-License-Identifier: GPL-2.0 */
2 #include "io_uring.h"
3 #include "wait.h"
4 #include "loop.h"
5
io_loop_nr_cqes(const struct io_ring_ctx * ctx,const struct iou_loop_params * lp)6 static inline int io_loop_nr_cqes(const struct io_ring_ctx *ctx,
7 const struct iou_loop_params *lp)
8 {
9 return lp->cq_wait_idx - READ_ONCE(ctx->rings->cq.tail);
10 }
11
io_loop_wait_start(struct io_ring_ctx * ctx,unsigned nr_wait)12 static inline void io_loop_wait_start(struct io_ring_ctx *ctx, unsigned nr_wait)
13 {
14 atomic_set(&ctx->cq_wait_nr, nr_wait);
15 set_current_state(TASK_INTERRUPTIBLE);
16 }
17
io_loop_wait_finish(struct io_ring_ctx * ctx)18 static inline void io_loop_wait_finish(struct io_ring_ctx *ctx)
19 {
20 __set_current_state(TASK_RUNNING);
21 atomic_set(&ctx->cq_wait_nr, IO_CQ_WAKE_INIT);
22 }
23
io_loop_wait(struct io_ring_ctx * ctx,struct iou_loop_params * lp,unsigned nr_wait)24 static void io_loop_wait(struct io_ring_ctx *ctx, struct iou_loop_params *lp,
25 unsigned nr_wait)
26 {
27 io_loop_wait_start(ctx, nr_wait);
28
29 if (unlikely(io_local_work_pending(ctx) ||
30 io_loop_nr_cqes(ctx, lp) <= 0) ||
31 READ_ONCE(ctx->check_cq)) {
32 io_loop_wait_finish(ctx);
33 return;
34 }
35
36 mutex_unlock(&ctx->uring_lock);
37 schedule();
38 io_loop_wait_finish(ctx);
39 mutex_lock(&ctx->uring_lock);
40 }
41
__io_run_loop(struct io_ring_ctx * ctx)42 static int __io_run_loop(struct io_ring_ctx *ctx)
43 {
44 struct iou_loop_params lp = {};
45
46 while (true) {
47 int nr_wait, step_res;
48
49 if (unlikely(!ctx->loop_step))
50 return -EFAULT;
51
52 step_res = ctx->loop_step(ctx, &lp);
53 if (step_res == IOU_LOOP_STOP)
54 break;
55 if (step_res != IOU_LOOP_CONTINUE)
56 return -EINVAL;
57
58 nr_wait = io_loop_nr_cqes(ctx, &lp);
59 if (nr_wait > 0)
60 io_loop_wait(ctx, &lp, nr_wait);
61 else
62 nr_wait = 0;
63
64 if (task_work_pending(current)) {
65 mutex_unlock(&ctx->uring_lock);
66 io_run_task_work();
67 mutex_lock(&ctx->uring_lock);
68 }
69 if (unlikely(task_sigpending(current)))
70 return -EINTR;
71 io_run_local_work_locked(ctx, nr_wait);
72
73 if (READ_ONCE(ctx->check_cq) & BIT(IO_CHECK_CQ_OVERFLOW_BIT))
74 io_cqring_overflow_flush_locked(ctx);
75 }
76
77 return 0;
78 }
79
io_run_loop(struct io_ring_ctx * ctx)80 int io_run_loop(struct io_ring_ctx *ctx)
81 {
82 int ret;
83
84 if (!io_allowed_run_tw(ctx))
85 return -EEXIST;
86
87 mutex_lock(&ctx->uring_lock);
88 ret = __io_run_loop(ctx);
89 mutex_unlock(&ctx->uring_lock);
90 return ret;
91 }
92