1*033af2b3SPavel Begunkov /* SPDX-License-Identifier: GPL-2.0 */
2*033af2b3SPavel Begunkov #include "io_uring.h"
3*033af2b3SPavel Begunkov #include "wait.h"
4*033af2b3SPavel Begunkov #include "loop.h"
5*033af2b3SPavel Begunkov
io_loop_nr_cqes(const struct io_ring_ctx * ctx,const struct iou_loop_params * lp)6*033af2b3SPavel Begunkov static inline int io_loop_nr_cqes(const struct io_ring_ctx *ctx,
7*033af2b3SPavel Begunkov const struct iou_loop_params *lp)
8*033af2b3SPavel Begunkov {
9*033af2b3SPavel Begunkov return lp->cq_wait_idx - READ_ONCE(ctx->rings->cq.tail);
10*033af2b3SPavel Begunkov }
11*033af2b3SPavel Begunkov
io_loop_wait_start(struct io_ring_ctx * ctx,unsigned nr_wait)12*033af2b3SPavel Begunkov static inline void io_loop_wait_start(struct io_ring_ctx *ctx, unsigned nr_wait)
13*033af2b3SPavel Begunkov {
14*033af2b3SPavel Begunkov atomic_set(&ctx->cq_wait_nr, nr_wait);
15*033af2b3SPavel Begunkov set_current_state(TASK_INTERRUPTIBLE);
16*033af2b3SPavel Begunkov }
17*033af2b3SPavel Begunkov
io_loop_wait_finish(struct io_ring_ctx * ctx)18*033af2b3SPavel Begunkov static inline void io_loop_wait_finish(struct io_ring_ctx *ctx)
19*033af2b3SPavel Begunkov {
20*033af2b3SPavel Begunkov __set_current_state(TASK_RUNNING);
21*033af2b3SPavel Begunkov atomic_set(&ctx->cq_wait_nr, IO_CQ_WAKE_INIT);
22*033af2b3SPavel Begunkov }
23*033af2b3SPavel Begunkov
io_loop_wait(struct io_ring_ctx * ctx,struct iou_loop_params * lp,unsigned nr_wait)24*033af2b3SPavel Begunkov static void io_loop_wait(struct io_ring_ctx *ctx, struct iou_loop_params *lp,
25*033af2b3SPavel Begunkov unsigned nr_wait)
26*033af2b3SPavel Begunkov {
27*033af2b3SPavel Begunkov io_loop_wait_start(ctx, nr_wait);
28*033af2b3SPavel Begunkov
29*033af2b3SPavel Begunkov if (unlikely(io_local_work_pending(ctx) ||
30*033af2b3SPavel Begunkov io_loop_nr_cqes(ctx, lp) <= 0) ||
31*033af2b3SPavel Begunkov READ_ONCE(ctx->check_cq)) {
32*033af2b3SPavel Begunkov io_loop_wait_finish(ctx);
33*033af2b3SPavel Begunkov return;
34*033af2b3SPavel Begunkov }
35*033af2b3SPavel Begunkov
36*033af2b3SPavel Begunkov mutex_unlock(&ctx->uring_lock);
37*033af2b3SPavel Begunkov schedule();
38*033af2b3SPavel Begunkov io_loop_wait_finish(ctx);
39*033af2b3SPavel Begunkov mutex_lock(&ctx->uring_lock);
40*033af2b3SPavel Begunkov }
41*033af2b3SPavel Begunkov
__io_run_loop(struct io_ring_ctx * ctx)42*033af2b3SPavel Begunkov static int __io_run_loop(struct io_ring_ctx *ctx)
43*033af2b3SPavel Begunkov {
44*033af2b3SPavel Begunkov struct iou_loop_params lp = {};
45*033af2b3SPavel Begunkov
46*033af2b3SPavel Begunkov while (true) {
47*033af2b3SPavel Begunkov int nr_wait, step_res;
48*033af2b3SPavel Begunkov
49*033af2b3SPavel Begunkov if (unlikely(!ctx->loop_step))
50*033af2b3SPavel Begunkov return -EFAULT;
51*033af2b3SPavel Begunkov
52*033af2b3SPavel Begunkov step_res = ctx->loop_step(ctx, &lp);
53*033af2b3SPavel Begunkov if (step_res == IOU_LOOP_STOP)
54*033af2b3SPavel Begunkov break;
55*033af2b3SPavel Begunkov if (step_res != IOU_LOOP_CONTINUE)
56*033af2b3SPavel Begunkov return -EINVAL;
57*033af2b3SPavel Begunkov
58*033af2b3SPavel Begunkov nr_wait = io_loop_nr_cqes(ctx, &lp);
59*033af2b3SPavel Begunkov if (nr_wait > 0)
60*033af2b3SPavel Begunkov io_loop_wait(ctx, &lp, nr_wait);
61*033af2b3SPavel Begunkov else
62*033af2b3SPavel Begunkov nr_wait = 0;
63*033af2b3SPavel Begunkov
64*033af2b3SPavel Begunkov if (task_work_pending(current)) {
65*033af2b3SPavel Begunkov mutex_unlock(&ctx->uring_lock);
66*033af2b3SPavel Begunkov io_run_task_work();
67*033af2b3SPavel Begunkov mutex_lock(&ctx->uring_lock);
68*033af2b3SPavel Begunkov }
69*033af2b3SPavel Begunkov if (unlikely(task_sigpending(current)))
70*033af2b3SPavel Begunkov return -EINTR;
71*033af2b3SPavel Begunkov io_run_local_work_locked(ctx, nr_wait);
72*033af2b3SPavel Begunkov
73*033af2b3SPavel Begunkov if (READ_ONCE(ctx->check_cq) & BIT(IO_CHECK_CQ_OVERFLOW_BIT))
74*033af2b3SPavel Begunkov io_cqring_overflow_flush_locked(ctx);
75*033af2b3SPavel Begunkov }
76*033af2b3SPavel Begunkov
77*033af2b3SPavel Begunkov return 0;
78*033af2b3SPavel Begunkov }
79*033af2b3SPavel Begunkov
io_run_loop(struct io_ring_ctx * ctx)80*033af2b3SPavel Begunkov int io_run_loop(struct io_ring_ctx *ctx)
81*033af2b3SPavel Begunkov {
82*033af2b3SPavel Begunkov int ret;
83*033af2b3SPavel Begunkov
84*033af2b3SPavel Begunkov if (!io_allowed_run_tw(ctx))
85*033af2b3SPavel Begunkov return -EEXIST;
86*033af2b3SPavel Begunkov
87*033af2b3SPavel Begunkov mutex_lock(&ctx->uring_lock);
88*033af2b3SPavel Begunkov ret = __io_run_loop(ctx);
89*033af2b3SPavel Begunkov mutex_unlock(&ctx->uring_lock);
90*033af2b3SPavel Begunkov return ret;
91*033af2b3SPavel Begunkov }
92