1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/kernel.h>
3 #include <linux/errno.h>
4 #include <linux/file.h>
5 #include <linux/io_uring/cmd.h>
6 #include <linux/security.h>
7 #include <linux/nospec.h>
8
9 #include <uapi/linux/io_uring.h>
10
11 #include "io_uring.h"
12 #include "alloc_cache.h"
13 #include "rsrc.h"
14 #include "kbuf.h"
15 #include "uring_cmd.h"
16 #include "poll.h"
17
io_cmd_cache_free(const void * entry)18 void io_cmd_cache_free(const void *entry)
19 {
20 struct io_async_cmd *ac = (struct io_async_cmd *)entry;
21
22 io_vec_free(&ac->vec);
23 kfree(ac);
24 }
25
io_req_uring_cleanup(struct io_kiocb * req,unsigned int issue_flags)26 static void io_req_uring_cleanup(struct io_kiocb *req, unsigned int issue_flags)
27 {
28 struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
29 struct io_async_cmd *ac = req->async_data;
30
31 if (issue_flags & IO_URING_F_UNLOCKED)
32 return;
33
34 io_alloc_cache_vec_kasan(&ac->vec);
35 if (ac->vec.nr > IO_VEC_CACHE_SOFT_CAP)
36 io_vec_free(&ac->vec);
37
38 if (io_alloc_cache_put(&req->ctx->cmd_cache, ac)) {
39 ioucmd->sqe = NULL;
40 io_req_async_data_clear(req, REQ_F_NEED_CLEANUP);
41 }
42 }
43
io_uring_cmd_cleanup(struct io_kiocb * req)44 void io_uring_cmd_cleanup(struct io_kiocb *req)
45 {
46 io_req_uring_cleanup(req, 0);
47 }
48
io_uring_try_cancel_uring_cmd(struct io_ring_ctx * ctx,struct io_uring_task * tctx,bool cancel_all)49 bool io_uring_try_cancel_uring_cmd(struct io_ring_ctx *ctx,
50 struct io_uring_task *tctx, bool cancel_all)
51 {
52 struct hlist_node *tmp;
53 struct io_kiocb *req;
54 bool ret = false;
55
56 lockdep_assert_held(&ctx->uring_lock);
57
58 hlist_for_each_entry_safe(req, tmp, &ctx->cancelable_uring_cmd,
59 hash_node) {
60 struct io_uring_cmd *cmd = io_kiocb_to_cmd(req,
61 struct io_uring_cmd);
62 struct file *file = req->file;
63
64 if (!cancel_all && req->tctx != tctx)
65 continue;
66
67 if (cmd->flags & IORING_URING_CMD_CANCELABLE) {
68 file->f_op->uring_cmd(cmd, IO_URING_F_CANCEL |
69 IO_URING_F_COMPLETE_DEFER);
70 ret = true;
71 }
72 }
73 io_submit_flush_completions(ctx);
74 return ret;
75 }
76
io_uring_cmd_del_cancelable(struct io_uring_cmd * cmd,unsigned int issue_flags)77 static void io_uring_cmd_del_cancelable(struct io_uring_cmd *cmd,
78 unsigned int issue_flags)
79 {
80 struct io_kiocb *req = cmd_to_io_kiocb(cmd);
81 struct io_ring_ctx *ctx = req->ctx;
82
83 if (!(cmd->flags & IORING_URING_CMD_CANCELABLE))
84 return;
85
86 cmd->flags &= ~IORING_URING_CMD_CANCELABLE;
87 io_ring_submit_lock(ctx, issue_flags);
88 hlist_del(&req->hash_node);
89 io_ring_submit_unlock(ctx, issue_flags);
90 }
91
92 /*
93 * Mark this command as concelable, then io_uring_try_cancel_uring_cmd()
94 * will try to cancel this issued command by sending ->uring_cmd() with
95 * issue_flags of IO_URING_F_CANCEL.
96 *
97 * The command is guaranteed to not be done when calling ->uring_cmd()
98 * with IO_URING_F_CANCEL, but it is driver's responsibility to deal
99 * with race between io_uring canceling and normal completion.
100 */
io_uring_cmd_mark_cancelable(struct io_uring_cmd * cmd,unsigned int issue_flags)101 void io_uring_cmd_mark_cancelable(struct io_uring_cmd *cmd,
102 unsigned int issue_flags)
103 {
104 struct io_kiocb *req = cmd_to_io_kiocb(cmd);
105 struct io_ring_ctx *ctx = req->ctx;
106
107 /*
108 * Doing cancelations on IOPOLL requests are not supported. Both
109 * because they can't get canceled in the block stack, but also
110 * because iopoll completion data overlaps with the hash_node used
111 * for tracking.
112 */
113 if (ctx->flags & IORING_SETUP_IOPOLL)
114 return;
115
116 if (!(cmd->flags & IORING_URING_CMD_CANCELABLE)) {
117 cmd->flags |= IORING_URING_CMD_CANCELABLE;
118 io_ring_submit_lock(ctx, issue_flags);
119 hlist_add_head(&req->hash_node, &ctx->cancelable_uring_cmd);
120 io_ring_submit_unlock(ctx, issue_flags);
121 }
122 }
123 EXPORT_SYMBOL_GPL(io_uring_cmd_mark_cancelable);
124
__io_uring_cmd_do_in_task(struct io_uring_cmd * ioucmd,io_req_tw_func_t task_work_cb,unsigned flags)125 void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,
126 io_req_tw_func_t task_work_cb,
127 unsigned flags)
128 {
129 struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
130
131 if (WARN_ON_ONCE(req->flags & REQ_F_APOLL_MULTISHOT))
132 return;
133
134 req->io_task_work.func = task_work_cb;
135 __io_req_task_work_add(req, flags);
136 }
137 EXPORT_SYMBOL_GPL(__io_uring_cmd_do_in_task);
138
io_req_set_cqe32_extra(struct io_kiocb * req,u64 extra1,u64 extra2)139 static inline void io_req_set_cqe32_extra(struct io_kiocb *req,
140 u64 extra1, u64 extra2)
141 {
142 req->big_cqe.extra1 = extra1;
143 req->big_cqe.extra2 = extra2;
144 }
145
146 /*
147 * Called by consumers of io_uring_cmd, if they originally returned
148 * -EIOCBQUEUED upon receiving the command.
149 */
__io_uring_cmd_done(struct io_uring_cmd * ioucmd,s32 ret,u64 res2,unsigned issue_flags,bool is_cqe32)150 void __io_uring_cmd_done(struct io_uring_cmd *ioucmd, s32 ret, u64 res2,
151 unsigned issue_flags, bool is_cqe32)
152 {
153 struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
154
155 if (WARN_ON_ONCE(req->flags & REQ_F_APOLL_MULTISHOT))
156 return;
157
158 io_uring_cmd_del_cancelable(ioucmd, issue_flags);
159
160 if (ret < 0)
161 req_set_fail(req);
162
163 io_req_set_res(req, ret, 0);
164 if (is_cqe32) {
165 if (req->ctx->flags & IORING_SETUP_CQE_MIXED)
166 req->cqe.flags |= IORING_CQE_F_32;
167 io_req_set_cqe32_extra(req, res2, 0);
168 }
169 io_req_uring_cleanup(req, issue_flags);
170 if (req->ctx->flags & IORING_SETUP_IOPOLL) {
171 /* order with io_iopoll_req_issued() checking ->iopoll_complete */
172 smp_store_release(&req->iopoll_completed, 1);
173 } else if (issue_flags & IO_URING_F_COMPLETE_DEFER) {
174 if (WARN_ON_ONCE(issue_flags & IO_URING_F_UNLOCKED))
175 return;
176 io_req_complete_defer(req);
177 } else {
178 req->io_task_work.func = io_req_task_complete;
179 io_req_task_work_add(req);
180 }
181 }
182 EXPORT_SYMBOL_GPL(__io_uring_cmd_done);
183
io_uring_cmd_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)184 int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
185 {
186 struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
187 struct io_async_cmd *ac;
188
189 if (sqe->__pad1)
190 return -EINVAL;
191
192 ioucmd->flags = READ_ONCE(sqe->uring_cmd_flags);
193 if (ioucmd->flags & ~IORING_URING_CMD_MASK)
194 return -EINVAL;
195
196 if (ioucmd->flags & IORING_URING_CMD_FIXED) {
197 if (ioucmd->flags & IORING_URING_CMD_MULTISHOT)
198 return -EINVAL;
199 req->buf_index = READ_ONCE(sqe->buf_index);
200 }
201
202 if (!!(ioucmd->flags & IORING_URING_CMD_MULTISHOT) !=
203 !!(req->flags & REQ_F_BUFFER_SELECT))
204 return -EINVAL;
205
206 ioucmd->cmd_op = READ_ONCE(sqe->cmd_op);
207
208 ac = io_uring_alloc_async_data(&req->ctx->cmd_cache, req);
209 if (!ac)
210 return -ENOMEM;
211 ioucmd->sqe = sqe;
212 return 0;
213 }
214
215 /*
216 * IORING_SETUP_SQE128 contexts allocate twice the normal SQE size for each
217 * slot.
218 */
uring_sqe_size(struct io_kiocb * req)219 static inline size_t uring_sqe_size(struct io_kiocb *req)
220 {
221 if (req->ctx->flags & IORING_SETUP_SQE128 ||
222 req->opcode == IORING_OP_URING_CMD128)
223 return 2 * sizeof(struct io_uring_sqe);
224 return sizeof(struct io_uring_sqe);
225 }
226
io_uring_cmd_sqe_copy(struct io_kiocb * req)227 void io_uring_cmd_sqe_copy(struct io_kiocb *req)
228 {
229 struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
230 struct io_async_cmd *ac = req->async_data;
231
232 /* Should not happen, as REQ_F_SQE_COPIED covers this */
233 if (WARN_ON_ONCE(ioucmd->sqe == ac->sqes))
234 return;
235 memcpy(ac->sqes, ioucmd->sqe, uring_sqe_size(req));
236 ioucmd->sqe = ac->sqes;
237 }
238
io_uring_cmd(struct io_kiocb * req,unsigned int issue_flags)239 int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags)
240 {
241 struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
242 struct io_ring_ctx *ctx = req->ctx;
243 struct file *file = req->file;
244 int ret;
245
246 if (!file->f_op->uring_cmd)
247 return -EOPNOTSUPP;
248
249 ret = security_uring_cmd(ioucmd);
250 if (ret)
251 return ret;
252
253 if (ctx->flags & IORING_SETUP_SQE128 ||
254 req->opcode == IORING_OP_URING_CMD128)
255 issue_flags |= IO_URING_F_SQE128;
256 if (ctx->flags & (IORING_SETUP_CQE32 | IORING_SETUP_CQE_MIXED))
257 issue_flags |= IO_URING_F_CQE32;
258 if (io_is_compat(ctx))
259 issue_flags |= IO_URING_F_COMPAT;
260 if (ctx->flags & IORING_SETUP_IOPOLL) {
261 if (!file->f_op->uring_cmd_iopoll)
262 return -EOPNOTSUPP;
263 issue_flags |= IO_URING_F_IOPOLL;
264 req->iopoll_completed = 0;
265 if (ctx->flags & IORING_SETUP_HYBRID_IOPOLL) {
266 /* make sure every req only blocks once */
267 req->flags &= ~REQ_F_IOPOLL_STATE;
268 req->iopoll_start = ktime_get_ns();
269 }
270 }
271
272 ret = file->f_op->uring_cmd(ioucmd, issue_flags);
273 if (ioucmd->flags & IORING_URING_CMD_MULTISHOT) {
274 if (ret >= 0)
275 return IOU_ISSUE_SKIP_COMPLETE;
276 }
277 if (ret == -EAGAIN) {
278 ioucmd->flags |= IORING_URING_CMD_REISSUE;
279 return ret;
280 }
281 if (ret == -EIOCBQUEUED)
282 return ret;
283 if (ret < 0)
284 req_set_fail(req);
285 io_req_uring_cleanup(req, issue_flags);
286 io_req_set_res(req, ret, 0);
287 return IOU_COMPLETE;
288 }
289
io_uring_cmd_import_fixed(u64 ubuf,unsigned long len,int rw,struct iov_iter * iter,struct io_uring_cmd * ioucmd,unsigned int issue_flags)290 int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw,
291 struct iov_iter *iter,
292 struct io_uring_cmd *ioucmd,
293 unsigned int issue_flags)
294 {
295 struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
296
297 if (WARN_ON_ONCE(!(ioucmd->flags & IORING_URING_CMD_FIXED)))
298 return -EINVAL;
299
300 return io_import_reg_buf(req, iter, ubuf, len, rw, issue_flags);
301 }
302 EXPORT_SYMBOL_GPL(io_uring_cmd_import_fixed);
303
io_uring_cmd_import_fixed_vec(struct io_uring_cmd * ioucmd,const struct iovec __user * uvec,size_t uvec_segs,int ddir,struct iov_iter * iter,unsigned issue_flags)304 int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd,
305 const struct iovec __user *uvec,
306 size_t uvec_segs,
307 int ddir, struct iov_iter *iter,
308 unsigned issue_flags)
309 {
310 struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
311 struct io_async_cmd *ac = req->async_data;
312 int ret;
313
314 if (WARN_ON_ONCE(!(ioucmd->flags & IORING_URING_CMD_FIXED)))
315 return -EINVAL;
316
317 ret = io_prep_reg_iovec(req, &ac->vec, uvec, uvec_segs);
318 if (ret)
319 return ret;
320
321 return io_import_reg_vec(ddir, iter, req, &ac->vec, uvec_segs,
322 issue_flags);
323 }
324 EXPORT_SYMBOL_GPL(io_uring_cmd_import_fixed_vec);
325
io_uring_cmd_issue_blocking(struct io_uring_cmd * ioucmd)326 void io_uring_cmd_issue_blocking(struct io_uring_cmd *ioucmd)
327 {
328 struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
329
330 io_req_queue_iowq(req);
331 }
332
io_cmd_poll_multishot(struct io_uring_cmd * cmd,unsigned int issue_flags,__poll_t mask)333 int io_cmd_poll_multishot(struct io_uring_cmd *cmd,
334 unsigned int issue_flags, __poll_t mask)
335 {
336 struct io_kiocb *req = cmd_to_io_kiocb(cmd);
337 int ret;
338
339 if (likely(req->flags & REQ_F_APOLL_MULTISHOT))
340 return 0;
341
342 req->flags |= REQ_F_APOLL_MULTISHOT;
343 mask &= ~EPOLLONESHOT;
344
345 ret = io_arm_apoll(req, issue_flags, mask);
346 return ret == IO_APOLL_OK ? -EIOCBQUEUED : -ECANCELED;
347 }
348
io_uring_cmd_post_mshot_cqe32(struct io_uring_cmd * cmd,unsigned int issue_flags,struct io_uring_cqe cqe[2])349 bool io_uring_cmd_post_mshot_cqe32(struct io_uring_cmd *cmd,
350 unsigned int issue_flags,
351 struct io_uring_cqe cqe[2])
352 {
353 struct io_kiocb *req = cmd_to_io_kiocb(cmd);
354
355 if (WARN_ON_ONCE(!(issue_flags & IO_URING_F_MULTISHOT)))
356 return false;
357 return io_req_post_cqe32(req, cqe);
358 }
359
360 /*
361 * Work with io_uring_mshot_cmd_post_cqe() together for committing the
362 * provided buffer upfront
363 */
io_uring_cmd_buffer_select(struct io_uring_cmd * ioucmd,unsigned buf_group,size_t * len,unsigned int issue_flags)364 struct io_br_sel io_uring_cmd_buffer_select(struct io_uring_cmd *ioucmd,
365 unsigned buf_group, size_t *len,
366 unsigned int issue_flags)
367 {
368 struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
369
370 if (!(ioucmd->flags & IORING_URING_CMD_MULTISHOT))
371 return (struct io_br_sel) { .val = -EINVAL };
372
373 if (WARN_ON_ONCE(!io_do_buffer_select(req)))
374 return (struct io_br_sel) { .val = -EINVAL };
375
376 return io_buffer_select(req, len, buf_group, issue_flags);
377 }
378 EXPORT_SYMBOL_GPL(io_uring_cmd_buffer_select);
379
380 /*
381 * Return true if this multishot uring_cmd needs to be completed, otherwise
382 * the event CQE is posted successfully.
383 *
384 * This function must use `struct io_br_sel` returned from
385 * io_uring_cmd_buffer_select() for committing the buffer in the same
386 * uring_cmd submission context.
387 */
io_uring_mshot_cmd_post_cqe(struct io_uring_cmd * ioucmd,struct io_br_sel * sel,unsigned int issue_flags)388 bool io_uring_mshot_cmd_post_cqe(struct io_uring_cmd *ioucmd,
389 struct io_br_sel *sel, unsigned int issue_flags)
390 {
391 struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
392 unsigned int cflags = 0;
393
394 if (!(ioucmd->flags & IORING_URING_CMD_MULTISHOT))
395 return true;
396
397 if (sel->val > 0) {
398 cflags = io_put_kbuf(req, sel->val, sel->buf_list);
399 if (io_req_post_cqe(req, sel->val, cflags | IORING_CQE_F_MORE))
400 return false;
401 }
402
403 io_kbuf_recycle(req, sel->buf_list, issue_flags);
404 if (sel->val < 0)
405 req_set_fail(req);
406 io_req_set_res(req, sel->val, cflags);
407 return true;
408 }
409 EXPORT_SYMBOL_GPL(io_uring_mshot_cmd_post_cqe);
410