1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/kernel.h>
3 #include <linux/errno.h>
4 #include <linux/file.h>
5 #include <linux/io_uring.h>
6
7 #include <trace/events/io_uring.h>
8
9 #include <uapi/linux/io_uring.h>
10
11 #include "io_uring.h"
12 #include "refs.h"
13 #include "cancel.h"
14 #include "timeout.h"
15
16 struct io_timeout {
17 struct file *file;
18 u32 off;
19 u32 target_seq;
20 u32 repeats;
21 struct list_head list;
22 /* head of the link, used by linked timeouts only */
23 struct io_kiocb *head;
24 /* for linked completions */
25 struct io_kiocb *prev;
26 };
27
28 struct io_timeout_rem {
29 struct file *file;
30 u64 addr;
31
32 /* timeout update */
33 ktime_t time;
34 u32 flags;
35 bool ltimeout;
36 };
37
io_parse_user_time(ktime_t * time,u64 arg,unsigned flags)38 static int io_parse_user_time(ktime_t *time, u64 arg, unsigned flags)
39 {
40 struct timespec64 ts;
41
42 if (flags & IORING_TIMEOUT_IMMEDIATE_ARG) {
43 *time = ns_to_ktime(arg);
44 if (*time < 0)
45 return -EINVAL;
46 return 0;
47 }
48
49 if (get_timespec64(&ts, u64_to_user_ptr(arg)))
50 return -EFAULT;
51 if (ts.tv_sec < 0 || ts.tv_nsec < 0)
52 return -EINVAL;
53 *time = timespec64_to_ktime(ts);
54 return 0;
55 }
56
57 static struct io_kiocb *__io_disarm_linked_timeout(struct io_kiocb *req,
58 struct io_kiocb *link);
59
io_is_timeout_noseq(struct io_kiocb * req)60 static inline bool io_is_timeout_noseq(struct io_kiocb *req)
61 {
62 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
63 struct io_timeout_data *data = req->async_data;
64
65 return !timeout->off || data->flags & IORING_TIMEOUT_MULTISHOT;
66 }
67
io_put_req(struct io_kiocb * req)68 static inline void io_put_req(struct io_kiocb *req)
69 {
70 if (req_ref_put_and_test(req)) {
71 io_queue_next(req);
72 io_free_req(req);
73 }
74 }
75
io_timeout_finish(struct io_timeout * timeout,struct io_timeout_data * data)76 static inline bool io_timeout_finish(struct io_timeout *timeout,
77 struct io_timeout_data *data)
78 {
79 if (!(data->flags & IORING_TIMEOUT_MULTISHOT))
80 return true;
81
82 if (!timeout->off || (timeout->repeats && --timeout->repeats))
83 return false;
84
85 return true;
86 }
87
88 static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer);
89
io_timeout_complete(struct io_tw_req tw_req,io_tw_token_t tw)90 static void io_timeout_complete(struct io_tw_req tw_req, io_tw_token_t tw)
91 {
92 struct io_kiocb *req = tw_req.req;
93 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
94 struct io_timeout_data *data = req->async_data;
95 struct io_ring_ctx *ctx = req->ctx;
96
97 if (!io_timeout_finish(timeout, data)) {
98 if (io_req_post_cqe(req, -ETIME, IORING_CQE_F_MORE)) {
99 /* re-arm timer */
100 raw_spin_lock_irq(&ctx->timeout_lock);
101 list_add(&timeout->list, ctx->timeout_list.prev);
102 hrtimer_start(&data->timer, data->time, data->mode);
103 raw_spin_unlock_irq(&ctx->timeout_lock);
104 return;
105 }
106 }
107
108 io_req_task_complete(tw_req, tw);
109 }
110
io_flush_killed_timeouts(struct list_head * list,int err)111 static __cold bool io_flush_killed_timeouts(struct list_head *list, int err)
112 {
113 if (list_empty(list))
114 return false;
115
116 while (!list_empty(list)) {
117 struct io_timeout *timeout;
118 struct io_kiocb *req;
119
120 timeout = list_first_entry(list, struct io_timeout, list);
121 list_del_init(&timeout->list);
122 req = cmd_to_io_kiocb(timeout);
123 if (err)
124 req_set_fail(req);
125 io_req_queue_tw_complete(req, err);
126 }
127
128 return true;
129 }
130
io_kill_timeout(struct io_kiocb * req,struct list_head * list)131 static void io_kill_timeout(struct io_kiocb *req, struct list_head *list)
132 __must_hold(&req->ctx->timeout_lock)
133 {
134 struct io_timeout_data *io = req->async_data;
135
136 if (hrtimer_try_to_cancel(&io->timer) != -1) {
137 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
138
139 atomic_set(&req->ctx->cq_timeouts,
140 atomic_read(&req->ctx->cq_timeouts) + 1);
141 list_move_tail(&timeout->list, list);
142 }
143 }
144
io_flush_timeouts(struct io_ring_ctx * ctx)145 __cold void io_flush_timeouts(struct io_ring_ctx *ctx)
146 {
147 struct io_timeout *timeout, *tmp;
148 LIST_HEAD(list);
149 u32 seq;
150
151 raw_spin_lock_irq(&ctx->timeout_lock);
152 seq = READ_ONCE(ctx->cached_cq_tail) - atomic_read(&ctx->cq_timeouts);
153
154 list_for_each_entry_safe(timeout, tmp, &ctx->timeout_list, list) {
155 struct io_kiocb *req = cmd_to_io_kiocb(timeout);
156 u32 events_needed, events_got;
157
158 if (io_is_timeout_noseq(req))
159 break;
160
161 /*
162 * Since seq can easily wrap around over time, subtract
163 * the last seq at which timeouts were flushed before comparing.
164 * Assuming not more than 2^31-1 events have happened since,
165 * these subtractions won't have wrapped, so we can check if
166 * target is in [last_seq, current_seq] by comparing the two.
167 */
168 events_needed = timeout->target_seq - ctx->cq_last_tm_flush;
169 events_got = seq - ctx->cq_last_tm_flush;
170 if (events_got < events_needed)
171 break;
172
173 io_kill_timeout(req, &list);
174 }
175 ctx->cq_last_tm_flush = seq;
176 raw_spin_unlock_irq(&ctx->timeout_lock);
177 io_flush_killed_timeouts(&list, 0);
178 }
179
io_req_tw_fail_links(struct io_tw_req tw_req,io_tw_token_t tw)180 static void io_req_tw_fail_links(struct io_tw_req tw_req, io_tw_token_t tw)
181 {
182 struct io_kiocb *link = tw_req.req;
183
184 io_tw_lock(link->ctx, tw);
185 while (link) {
186 struct io_kiocb *nxt = link->link;
187 long res = -ECANCELED;
188
189 if (link->flags & REQ_F_FAIL)
190 res = link->cqe.res;
191 link->link = NULL;
192 io_req_set_res(link, res, 0);
193 io_req_task_complete((struct io_tw_req){link}, tw);
194 link = nxt;
195 }
196 }
197
io_fail_links(struct io_kiocb * req)198 static void io_fail_links(struct io_kiocb *req)
199 __must_hold(&req->ctx->completion_lock)
200 {
201 struct io_kiocb *link = req->link;
202 bool ignore_cqes = req->flags & REQ_F_SKIP_LINK_CQES;
203
204 if (!link)
205 return;
206
207 while (link) {
208 if (ignore_cqes)
209 link->flags |= REQ_F_CQE_SKIP;
210 else
211 link->flags &= ~REQ_F_CQE_SKIP;
212 trace_io_uring_fail_link(req, link);
213 link = link->link;
214 }
215
216 link = req->link;
217 link->io_task_work.func = io_req_tw_fail_links;
218 io_req_task_work_add(link);
219 req->link = NULL;
220 }
221
io_remove_next_linked(struct io_kiocb * req)222 static inline void io_remove_next_linked(struct io_kiocb *req)
223 {
224 struct io_kiocb *nxt = req->link;
225
226 req->link = nxt->link;
227 nxt->link = NULL;
228 }
229
io_disarm_next(struct io_kiocb * req)230 void io_disarm_next(struct io_kiocb *req)
231 __must_hold(&req->ctx->completion_lock)
232 {
233 struct io_kiocb *link = NULL;
234
235 if (req->flags & REQ_F_ARM_LTIMEOUT) {
236 link = req->link;
237 req->flags &= ~REQ_F_ARM_LTIMEOUT;
238 if (link && link->opcode == IORING_OP_LINK_TIMEOUT) {
239 io_remove_next_linked(req);
240 io_req_queue_tw_complete(link, -ECANCELED);
241 }
242 } else if (req->flags & REQ_F_LINK_TIMEOUT) {
243 struct io_ring_ctx *ctx = req->ctx;
244
245 raw_spin_lock_irq(&ctx->timeout_lock);
246 if (req->link && req->link->opcode == IORING_OP_LINK_TIMEOUT)
247 link = __io_disarm_linked_timeout(req, req->link);
248
249 raw_spin_unlock_irq(&ctx->timeout_lock);
250 if (link)
251 io_req_queue_tw_complete(link, -ECANCELED);
252 }
253 if (unlikely((req->flags & REQ_F_FAIL) &&
254 !(req->flags & REQ_F_HARDLINK)))
255 io_fail_links(req);
256 }
257
__io_disarm_linked_timeout(struct io_kiocb * req,struct io_kiocb * link)258 static struct io_kiocb *__io_disarm_linked_timeout(struct io_kiocb *req,
259 struct io_kiocb *link)
260 __must_hold(&req->ctx->completion_lock)
261 __must_hold(&req->ctx->timeout_lock)
262 {
263 struct io_timeout_data *io = link->async_data;
264 struct io_timeout *timeout = io_kiocb_to_cmd(link, struct io_timeout);
265
266 io_remove_next_linked(req);
267 timeout->head = NULL;
268 if (hrtimer_try_to_cancel(&io->timer) != -1) {
269 list_del(&timeout->list);
270 return link;
271 }
272
273 return NULL;
274 }
275
io_timeout_fn(struct hrtimer * timer)276 static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer)
277 {
278 struct io_timeout_data *data = container_of(timer,
279 struct io_timeout_data, timer);
280 struct io_kiocb *req = data->req;
281 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
282 struct io_ring_ctx *ctx = req->ctx;
283 unsigned long flags;
284
285 raw_spin_lock_irqsave(&ctx->timeout_lock, flags);
286 list_del_init(&timeout->list);
287 atomic_set(&ctx->cq_timeouts,
288 atomic_read(&ctx->cq_timeouts) + 1);
289 raw_spin_unlock_irqrestore(&ctx->timeout_lock, flags);
290
291 if (!(data->flags & IORING_TIMEOUT_ETIME_SUCCESS))
292 req_set_fail(req);
293
294 io_req_set_res(req, -ETIME, 0);
295 req->io_task_work.func = io_timeout_complete;
296 io_req_task_work_add(req);
297 return HRTIMER_NORESTART;
298 }
299
io_timeout_extract(struct io_ring_ctx * ctx,struct io_cancel_data * cd)300 static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx,
301 struct io_cancel_data *cd)
302 __must_hold(&ctx->timeout_lock)
303 {
304 struct io_timeout *timeout;
305 struct io_timeout_data *io;
306 struct io_kiocb *req = NULL;
307
308 list_for_each_entry(timeout, &ctx->timeout_list, list) {
309 struct io_kiocb *tmp = cmd_to_io_kiocb(timeout);
310
311 if (io_cancel_req_match(tmp, cd)) {
312 req = tmp;
313 break;
314 }
315 }
316 if (!req)
317 return ERR_PTR(-ENOENT);
318
319 io = req->async_data;
320 if (hrtimer_try_to_cancel(&io->timer) == -1)
321 return ERR_PTR(-EALREADY);
322 timeout = io_kiocb_to_cmd(req, struct io_timeout);
323 list_del_init(&timeout->list);
324 return req;
325 }
326
io_timeout_cancel(struct io_ring_ctx * ctx,struct io_cancel_data * cd)327 int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd)
328 __must_hold(&ctx->completion_lock)
329 {
330 struct io_kiocb *req;
331
332 raw_spin_lock_irq(&ctx->timeout_lock);
333 req = io_timeout_extract(ctx, cd);
334 raw_spin_unlock_irq(&ctx->timeout_lock);
335
336 if (IS_ERR(req))
337 return PTR_ERR(req);
338 io_req_task_queue_fail(req, -ECANCELED);
339 return 0;
340 }
341
io_req_task_link_timeout(struct io_tw_req tw_req,io_tw_token_t tw)342 static void io_req_task_link_timeout(struct io_tw_req tw_req, io_tw_token_t tw)
343 {
344 struct io_kiocb *req = tw_req.req;
345 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
346 struct io_kiocb *prev = timeout->prev;
347 int ret;
348
349 if (prev) {
350 if (!tw.cancel) {
351 struct io_cancel_data cd = {
352 .ctx = req->ctx,
353 .data = prev->cqe.user_data,
354 };
355
356 ret = io_try_cancel(req->tctx, &cd, 0);
357 } else {
358 ret = -ECANCELED;
359 }
360 io_req_set_res(req, ret ?: -ETIME, 0);
361 io_req_task_complete(tw_req, tw);
362 io_put_req(prev);
363 } else {
364 io_req_set_res(req, -ETIME, 0);
365 io_req_task_complete(tw_req, tw);
366 }
367 }
368
io_link_timeout_fn(struct hrtimer * timer)369 static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer)
370 {
371 struct io_timeout_data *data = container_of(timer,
372 struct io_timeout_data, timer);
373 struct io_kiocb *prev, *req = data->req;
374 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
375 struct io_ring_ctx *ctx = req->ctx;
376 unsigned long flags;
377
378 raw_spin_lock_irqsave(&ctx->timeout_lock, flags);
379 prev = timeout->head;
380 timeout->head = NULL;
381
382 /*
383 * We don't expect the list to be empty, that will only happen if we
384 * race with the completion of the linked work.
385 */
386 if (prev) {
387 io_remove_next_linked(prev);
388 if (!req_ref_inc_not_zero(prev))
389 prev = NULL;
390 }
391 list_del(&timeout->list);
392 timeout->prev = prev;
393 raw_spin_unlock_irqrestore(&ctx->timeout_lock, flags);
394
395 req->io_task_work.func = io_req_task_link_timeout;
396 io_req_task_work_add(req);
397 return HRTIMER_NORESTART;
398 }
399
io_timeout_get_clock(struct io_timeout_data * data)400 static clockid_t io_timeout_get_clock(struct io_timeout_data *data)
401 {
402 switch (data->flags & IORING_TIMEOUT_CLOCK_MASK) {
403 case IORING_TIMEOUT_BOOTTIME:
404 return CLOCK_BOOTTIME;
405 case IORING_TIMEOUT_REALTIME:
406 return CLOCK_REALTIME;
407 default:
408 /* can't happen, vetted at prep time */
409 WARN_ON_ONCE(1);
410 fallthrough;
411 case 0:
412 return CLOCK_MONOTONIC;
413 }
414 }
415
io_linked_timeout_update(struct io_ring_ctx * ctx,__u64 user_data,ktime_t ts,enum hrtimer_mode mode)416 static int io_linked_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
417 ktime_t ts, enum hrtimer_mode mode)
418 __must_hold(&ctx->timeout_lock)
419 {
420 struct io_timeout_data *io;
421 struct io_timeout *timeout;
422 struct io_kiocb *req = NULL;
423
424 list_for_each_entry(timeout, &ctx->ltimeout_list, list) {
425 struct io_kiocb *tmp = cmd_to_io_kiocb(timeout);
426
427 if (user_data == tmp->cqe.user_data) {
428 req = tmp;
429 break;
430 }
431 }
432 if (!req)
433 return -ENOENT;
434
435 io = req->async_data;
436 if (hrtimer_try_to_cancel(&io->timer) == -1)
437 return -EALREADY;
438 hrtimer_setup(&io->timer, io_link_timeout_fn, io_timeout_get_clock(io), mode);
439 hrtimer_start(&io->timer, ts, mode);
440 return 0;
441 }
442
io_timeout_update(struct io_ring_ctx * ctx,__u64 user_data,ktime_t time,enum hrtimer_mode mode)443 static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
444 ktime_t time, enum hrtimer_mode mode)
445 __must_hold(&ctx->timeout_lock)
446 {
447 struct io_cancel_data cd = { .ctx = ctx, .data = user_data, };
448 struct io_kiocb *req = io_timeout_extract(ctx, &cd);
449 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
450 struct io_timeout_data *data;
451
452 if (IS_ERR(req))
453 return PTR_ERR(req);
454
455 timeout->off = 0; /* noseq */
456 data = req->async_data;
457 data->time = time;
458
459 list_add_tail(&timeout->list, &ctx->timeout_list);
460 hrtimer_setup(&data->timer, io_timeout_fn, io_timeout_get_clock(data), mode);
461 hrtimer_start(&data->timer, data->time, mode);
462 return 0;
463 }
464
io_timeout_remove_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)465 int io_timeout_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
466 {
467 struct io_timeout_rem *tr = io_kiocb_to_cmd(req, struct io_timeout_rem);
468 int ret;
469
470 if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT)))
471 return -EINVAL;
472 if (sqe->addr3 || sqe->__pad2[0])
473 return -EINVAL;
474 if (sqe->buf_index || sqe->len || sqe->splice_fd_in)
475 return -EINVAL;
476
477 tr->ltimeout = false;
478 tr->addr = READ_ONCE(sqe->addr);
479 tr->flags = READ_ONCE(sqe->timeout_flags);
480 if (tr->flags & IORING_TIMEOUT_UPDATE_MASK) {
481 if (hweight32(tr->flags & IORING_TIMEOUT_CLOCK_MASK) > 1)
482 return -EINVAL;
483 if (tr->flags & IORING_LINK_TIMEOUT_UPDATE)
484 tr->ltimeout = true;
485 if (tr->flags & ~(IORING_TIMEOUT_UPDATE_MASK |
486 IORING_TIMEOUT_ABS |
487 IORING_TIMEOUT_IMMEDIATE_ARG))
488 return -EINVAL;
489 ret = io_parse_user_time(&tr->time, READ_ONCE(sqe->addr2), tr->flags);
490 if (ret)
491 return ret;
492 } else if (tr->flags) {
493 /* timeout removal doesn't support flags */
494 return -EINVAL;
495 }
496
497 return 0;
498 }
499
io_translate_timeout_mode(unsigned int flags)500 static inline enum hrtimer_mode io_translate_timeout_mode(unsigned int flags)
501 {
502 return (flags & IORING_TIMEOUT_ABS) ? HRTIMER_MODE_ABS
503 : HRTIMER_MODE_REL;
504 }
505
506 /*
507 * Remove or update an existing timeout command
508 */
io_timeout_remove(struct io_kiocb * req,unsigned int issue_flags)509 int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags)
510 {
511 struct io_timeout_rem *tr = io_kiocb_to_cmd(req, struct io_timeout_rem);
512 struct io_ring_ctx *ctx = req->ctx;
513 int ret;
514
515 if (!(tr->flags & IORING_TIMEOUT_UPDATE)) {
516 struct io_cancel_data cd = { .ctx = ctx, .data = tr->addr, };
517
518 spin_lock(&ctx->completion_lock);
519 ret = io_timeout_cancel(ctx, &cd);
520 spin_unlock(&ctx->completion_lock);
521 } else {
522 enum hrtimer_mode mode = io_translate_timeout_mode(tr->flags);
523
524 raw_spin_lock_irq(&ctx->timeout_lock);
525 if (tr->ltimeout)
526 ret = io_linked_timeout_update(ctx, tr->addr, tr->time, mode);
527 else
528 ret = io_timeout_update(ctx, tr->addr, tr->time, mode);
529 raw_spin_unlock_irq(&ctx->timeout_lock);
530 }
531
532 if (ret < 0)
533 req_set_fail(req);
534 io_req_set_res(req, ret, 0);
535 return IOU_COMPLETE;
536 }
537
__io_timeout_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe,bool is_timeout_link)538 static int __io_timeout_prep(struct io_kiocb *req,
539 const struct io_uring_sqe *sqe,
540 bool is_timeout_link)
541 {
542 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
543 struct io_timeout_data *data;
544 unsigned flags;
545 u32 off = READ_ONCE(sqe->off);
546 int ret;
547
548 if (sqe->addr3 || sqe->__pad2[0])
549 return -EINVAL;
550 if (sqe->buf_index || sqe->len != 1 || sqe->splice_fd_in)
551 return -EINVAL;
552 if (off && is_timeout_link)
553 return -EINVAL;
554 flags = READ_ONCE(sqe->timeout_flags);
555 if (flags & ~(IORING_TIMEOUT_ABS | IORING_TIMEOUT_CLOCK_MASK |
556 IORING_TIMEOUT_ETIME_SUCCESS |
557 IORING_TIMEOUT_MULTISHOT |
558 IORING_TIMEOUT_IMMEDIATE_ARG))
559 return -EINVAL;
560 /* more than one clock specified is invalid, obviously */
561 if (hweight32(flags & IORING_TIMEOUT_CLOCK_MASK) > 1)
562 return -EINVAL;
563 /* multishot requests only make sense with rel values */
564 if (!(~flags & (IORING_TIMEOUT_MULTISHOT | IORING_TIMEOUT_ABS)))
565 return -EINVAL;
566
567 INIT_LIST_HEAD(&timeout->list);
568 timeout->off = off;
569 if (unlikely(off && !(req->ctx->int_flags & IO_RING_F_OFF_TIMEOUT_USED)))
570 req->ctx->int_flags |= IO_RING_F_OFF_TIMEOUT_USED;
571 /*
572 * for multishot reqs w/ fixed nr of repeats, repeats tracks the
573 * remaining nr
574 */
575 timeout->repeats = 0;
576 if ((flags & IORING_TIMEOUT_MULTISHOT) && off > 0)
577 timeout->repeats = off;
578
579 if (WARN_ON_ONCE(req_has_async_data(req)))
580 return -EFAULT;
581 data = io_uring_alloc_async_data(NULL, req);
582 if (!data)
583 return -ENOMEM;
584 data->req = req;
585 data->flags = flags;
586
587 ret = io_parse_user_time(&data->time, READ_ONCE(sqe->addr), flags);
588 if (ret)
589 return ret;
590
591 data->mode = io_translate_timeout_mode(flags);
592
593 if (is_timeout_link) {
594 struct io_submit_link *link = &req->ctx->submit_state.link;
595
596 if (!link->head)
597 return -EINVAL;
598 if (link->last->opcode == IORING_OP_LINK_TIMEOUT)
599 return -EINVAL;
600 timeout->head = link->last;
601 link->last->flags |= REQ_F_ARM_LTIMEOUT;
602 hrtimer_setup(&data->timer, io_link_timeout_fn, io_timeout_get_clock(data),
603 data->mode);
604 } else {
605 hrtimer_setup(&data->timer, io_timeout_fn, io_timeout_get_clock(data), data->mode);
606 }
607 return 0;
608 }
609
io_timeout_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)610 int io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
611 {
612 return __io_timeout_prep(req, sqe, false);
613 }
614
io_link_timeout_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)615 int io_link_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
616 {
617 return __io_timeout_prep(req, sqe, true);
618 }
619
io_timeout(struct io_kiocb * req,unsigned int issue_flags)620 int io_timeout(struct io_kiocb *req, unsigned int issue_flags)
621 {
622 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
623 struct io_ring_ctx *ctx = req->ctx;
624 struct io_timeout_data *data = req->async_data;
625 struct list_head *entry;
626 u32 tail, off = timeout->off;
627
628 raw_spin_lock_irq(&ctx->timeout_lock);
629
630 /*
631 * sqe->off holds how many events that need to occur for this
632 * timeout event to be satisfied. If it isn't set, then this is
633 * a pure timeout request, sequence isn't used.
634 */
635 if (io_is_timeout_noseq(req)) {
636 entry = ctx->timeout_list.prev;
637 goto add;
638 }
639
640 tail = data_race(ctx->cached_cq_tail) - atomic_read(&ctx->cq_timeouts);
641 timeout->target_seq = tail + off;
642
643 /* Update the last seq here in case io_flush_timeouts() hasn't.
644 * This is safe because ->completion_lock is held, and submissions
645 * and completions are never mixed in the same ->completion_lock section.
646 */
647 ctx->cq_last_tm_flush = tail;
648
649 /*
650 * Insertion sort, ensuring the first entry in the list is always
651 * the one we need first.
652 */
653 list_for_each_prev(entry, &ctx->timeout_list) {
654 struct io_timeout *nextt = list_entry(entry, struct io_timeout, list);
655 struct io_kiocb *nxt = cmd_to_io_kiocb(nextt);
656
657 if (io_is_timeout_noseq(nxt))
658 continue;
659 /* nxt.seq is behind @tail, otherwise would've been completed */
660 if (off >= nextt->target_seq - tail)
661 break;
662 }
663 add:
664 list_add(&timeout->list, entry);
665 hrtimer_start(&data->timer, data->time, data->mode);
666 raw_spin_unlock_irq(&ctx->timeout_lock);
667 return IOU_ISSUE_SKIP_COMPLETE;
668 }
669
io_queue_linked_timeout(struct io_kiocb * req)670 void io_queue_linked_timeout(struct io_kiocb *req)
671 {
672 struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout);
673 struct io_ring_ctx *ctx = req->ctx;
674
675 raw_spin_lock_irq(&ctx->timeout_lock);
676 /*
677 * If the back reference is NULL, then our linked request finished
678 * before we got a chance to setup the timer
679 */
680 if (timeout->head) {
681 struct io_timeout_data *data = req->async_data;
682
683 hrtimer_start(&data->timer, data->time, data->mode);
684 list_add_tail(&timeout->list, &ctx->ltimeout_list);
685 }
686 raw_spin_unlock_irq(&ctx->timeout_lock);
687 /* drop submission reference */
688 io_put_req(req);
689 }
690
io_match_task(struct io_kiocb * head,struct io_uring_task * tctx,bool cancel_all)691 static bool io_match_task(struct io_kiocb *head, struct io_uring_task *tctx,
692 bool cancel_all)
693 __must_hold(&head->ctx->timeout_lock)
694 {
695 struct io_kiocb *req;
696
697 if (tctx && head->tctx != tctx)
698 return false;
699 if (cancel_all)
700 return true;
701
702 io_for_each_link(req, head) {
703 if (req->flags & REQ_F_INFLIGHT)
704 return true;
705 }
706 return false;
707 }
708
709 /* Returns true if we found and killed one or more timeouts */
io_kill_timeouts(struct io_ring_ctx * ctx,struct io_uring_task * tctx,bool cancel_all)710 __cold bool io_kill_timeouts(struct io_ring_ctx *ctx, struct io_uring_task *tctx,
711 bool cancel_all)
712 {
713 struct io_timeout *timeout, *tmp;
714 LIST_HEAD(list);
715
716 /*
717 * completion_lock is needed for io_match_task(). Take it before
718 * timeout_lockfirst to keep locking ordering.
719 */
720 spin_lock(&ctx->completion_lock);
721 raw_spin_lock_irq(&ctx->timeout_lock);
722 list_for_each_entry_safe(timeout, tmp, &ctx->timeout_list, list) {
723 struct io_kiocb *req = cmd_to_io_kiocb(timeout);
724
725 if (io_match_task(req, tctx, cancel_all))
726 io_kill_timeout(req, &list);
727 }
728 raw_spin_unlock_irq(&ctx->timeout_lock);
729 spin_unlock(&ctx->completion_lock);
730
731 return io_flush_killed_timeouts(&list, -ECANCELED);
732 }
733