1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * io_uring opcode handling table
4 */
5 #include <linux/kernel.h>
6 #include <linux/errno.h>
7 #include <linux/fs.h>
8 #include <linux/file.h>
9 #include <linux/io_uring.h>
10 #include <linux/io_uring/cmd.h>
11
12 #include "io_uring.h"
13 #include "opdef.h"
14 #include "refs.h"
15 #include "tctx.h"
16 #include "sqpoll.h"
17 #include "fdinfo.h"
18 #include "kbuf.h"
19 #include "rsrc.h"
20
21 #include "xattr.h"
22 #include "nop.h"
23 #include "fs.h"
24 #include "splice.h"
25 #include "sync.h"
26 #include "advise.h"
27 #include "openclose.h"
28 #include "uring_cmd.h"
29 #include "epoll.h"
30 #include "statx.h"
31 #include "net.h"
32 #include "msg_ring.h"
33 #include "timeout.h"
34 #include "poll.h"
35 #include "cancel.h"
36 #include "rw.h"
37 #include "waitid.h"
38 #include "futex.h"
39 #include "truncate.h"
40 #include "zcrx.h"
41
io_no_issue(struct io_kiocb * req,unsigned int issue_flags)42 static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
43 {
44 WARN_ON_ONCE(1);
45 return -ECANCELED;
46 }
47
io_eopnotsupp_prep(struct io_kiocb * kiocb,const struct io_uring_sqe * sqe)48 static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,
49 const struct io_uring_sqe *sqe)
50 {
51 return -EOPNOTSUPP;
52 }
53
54 const struct io_issue_def io_issue_defs[] = {
55 [IORING_OP_NOP] = {
56 .audit_skip = 1,
57 .iopoll = 1,
58 .prep = io_nop_prep,
59 .issue = io_nop,
60 },
61 [IORING_OP_READV] = {
62 .needs_file = 1,
63 .unbound_nonreg_file = 1,
64 .pollin = 1,
65 .buffer_select = 1,
66 .plug = 1,
67 .audit_skip = 1,
68 .ioprio = 1,
69 .iopoll = 1,
70 .iopoll_queue = 1,
71 .vectored = 1,
72 .async_size = sizeof(struct io_async_rw),
73 .prep = io_prep_readv,
74 .issue = io_read,
75 },
76 [IORING_OP_WRITEV] = {
77 .needs_file = 1,
78 .hash_reg_file = 1,
79 .unbound_nonreg_file = 1,
80 .pollout = 1,
81 .plug = 1,
82 .audit_skip = 1,
83 .ioprio = 1,
84 .iopoll = 1,
85 .iopoll_queue = 1,
86 .vectored = 1,
87 .async_size = sizeof(struct io_async_rw),
88 .prep = io_prep_writev,
89 .issue = io_write,
90 },
91 [IORING_OP_FSYNC] = {
92 .needs_file = 1,
93 .audit_skip = 1,
94 .prep = io_fsync_prep,
95 .issue = io_fsync,
96 },
97 [IORING_OP_READ_FIXED] = {
98 .needs_file = 1,
99 .unbound_nonreg_file = 1,
100 .pollin = 1,
101 .plug = 1,
102 .audit_skip = 1,
103 .ioprio = 1,
104 .iopoll = 1,
105 .iopoll_queue = 1,
106 .async_size = sizeof(struct io_async_rw),
107 .prep = io_prep_read_fixed,
108 .issue = io_read_fixed,
109 },
110 [IORING_OP_WRITE_FIXED] = {
111 .needs_file = 1,
112 .hash_reg_file = 1,
113 .unbound_nonreg_file = 1,
114 .pollout = 1,
115 .plug = 1,
116 .audit_skip = 1,
117 .ioprio = 1,
118 .iopoll = 1,
119 .iopoll_queue = 1,
120 .async_size = sizeof(struct io_async_rw),
121 .prep = io_prep_write_fixed,
122 .issue = io_write_fixed,
123 },
124 [IORING_OP_POLL_ADD] = {
125 .needs_file = 1,
126 .unbound_nonreg_file = 1,
127 .audit_skip = 1,
128 .prep = io_poll_add_prep,
129 .issue = io_poll_add,
130 },
131 [IORING_OP_POLL_REMOVE] = {
132 .audit_skip = 1,
133 .prep = io_poll_remove_prep,
134 .issue = io_poll_remove,
135 },
136 [IORING_OP_SYNC_FILE_RANGE] = {
137 .needs_file = 1,
138 .audit_skip = 1,
139 .prep = io_sfr_prep,
140 .issue = io_sync_file_range,
141 },
142 [IORING_OP_SENDMSG] = {
143 .needs_file = 1,
144 .unbound_nonreg_file = 1,
145 .pollout = 1,
146 .ioprio = 1,
147 #if defined(CONFIG_NET)
148 .async_size = sizeof(struct io_async_msghdr),
149 .prep = io_sendmsg_prep,
150 .issue = io_sendmsg,
151 #else
152 .prep = io_eopnotsupp_prep,
153 #endif
154 },
155 [IORING_OP_RECVMSG] = {
156 .needs_file = 1,
157 .unbound_nonreg_file = 1,
158 .pollin = 1,
159 .buffer_select = 1,
160 .ioprio = 1,
161 #if defined(CONFIG_NET)
162 .async_size = sizeof(struct io_async_msghdr),
163 .prep = io_recvmsg_prep,
164 .issue = io_recvmsg,
165 #else
166 .prep = io_eopnotsupp_prep,
167 #endif
168 },
169 [IORING_OP_TIMEOUT] = {
170 .audit_skip = 1,
171 .async_size = sizeof(struct io_timeout_data),
172 .prep = io_timeout_prep,
173 .issue = io_timeout,
174 },
175 [IORING_OP_TIMEOUT_REMOVE] = {
176 /* used by timeout updates' prep() */
177 .audit_skip = 1,
178 .prep = io_timeout_remove_prep,
179 .issue = io_timeout_remove,
180 },
181 [IORING_OP_ACCEPT] = {
182 .needs_file = 1,
183 .unbound_nonreg_file = 1,
184 .pollin = 1,
185 .poll_exclusive = 1,
186 .ioprio = 1, /* used for flags */
187 #if defined(CONFIG_NET)
188 .prep = io_accept_prep,
189 .issue = io_accept,
190 #else
191 .prep = io_eopnotsupp_prep,
192 #endif
193 },
194 [IORING_OP_ASYNC_CANCEL] = {
195 .audit_skip = 1,
196 .prep = io_async_cancel_prep,
197 .issue = io_async_cancel,
198 },
199 [IORING_OP_LINK_TIMEOUT] = {
200 .audit_skip = 1,
201 .async_size = sizeof(struct io_timeout_data),
202 .prep = io_link_timeout_prep,
203 .issue = io_no_issue,
204 },
205 [IORING_OP_CONNECT] = {
206 .needs_file = 1,
207 .unbound_nonreg_file = 1,
208 .pollout = 1,
209 #if defined(CONFIG_NET)
210 .async_size = sizeof(struct io_async_msghdr),
211 .prep = io_connect_prep,
212 .issue = io_connect,
213 #else
214 .prep = io_eopnotsupp_prep,
215 #endif
216 },
217 [IORING_OP_FALLOCATE] = {
218 .needs_file = 1,
219 .hash_reg_file = 1,
220 .prep = io_fallocate_prep,
221 .issue = io_fallocate,
222 },
223 [IORING_OP_OPENAT] = {
224 .filter_pdu_size = sizeof_field(struct io_uring_bpf_ctx, open),
225 .prep = io_openat_prep,
226 .issue = io_openat,
227 .filter_populate = io_openat_bpf_populate,
228 },
229 [IORING_OP_CLOSE] = {
230 .prep = io_close_prep,
231 .issue = io_close,
232 },
233 [IORING_OP_FILES_UPDATE] = {
234 .audit_skip = 1,
235 .iopoll = 1,
236 .prep = io_files_update_prep,
237 .issue = io_files_update,
238 },
239 [IORING_OP_STATX] = {
240 .audit_skip = 1,
241 .prep = io_statx_prep,
242 .issue = io_statx,
243 },
244 [IORING_OP_READ] = {
245 .needs_file = 1,
246 .unbound_nonreg_file = 1,
247 .pollin = 1,
248 .buffer_select = 1,
249 .plug = 1,
250 .audit_skip = 1,
251 .ioprio = 1,
252 .iopoll = 1,
253 .iopoll_queue = 1,
254 .async_size = sizeof(struct io_async_rw),
255 .prep = io_prep_read,
256 .issue = io_read,
257 },
258 [IORING_OP_WRITE] = {
259 .needs_file = 1,
260 .hash_reg_file = 1,
261 .unbound_nonreg_file = 1,
262 .pollout = 1,
263 .plug = 1,
264 .audit_skip = 1,
265 .ioprio = 1,
266 .iopoll = 1,
267 .iopoll_queue = 1,
268 .async_size = sizeof(struct io_async_rw),
269 .prep = io_prep_write,
270 .issue = io_write,
271 },
272 [IORING_OP_FADVISE] = {
273 .needs_file = 1,
274 .audit_skip = 1,
275 .prep = io_fadvise_prep,
276 .issue = io_fadvise,
277 },
278 [IORING_OP_MADVISE] = {
279 .audit_skip = 1,
280 .prep = io_madvise_prep,
281 .issue = io_madvise,
282 },
283 [IORING_OP_SEND] = {
284 .needs_file = 1,
285 .unbound_nonreg_file = 1,
286 .pollout = 1,
287 .audit_skip = 1,
288 .ioprio = 1,
289 .buffer_select = 1,
290 #if defined(CONFIG_NET)
291 .async_size = sizeof(struct io_async_msghdr),
292 .prep = io_sendmsg_prep,
293 .issue = io_send,
294 #else
295 .prep = io_eopnotsupp_prep,
296 #endif
297 },
298 [IORING_OP_RECV] = {
299 .needs_file = 1,
300 .unbound_nonreg_file = 1,
301 .pollin = 1,
302 .buffer_select = 1,
303 .audit_skip = 1,
304 .ioprio = 1,
305 #if defined(CONFIG_NET)
306 .async_size = sizeof(struct io_async_msghdr),
307 .prep = io_recvmsg_prep,
308 .issue = io_recv,
309 #else
310 .prep = io_eopnotsupp_prep,
311 #endif
312 },
313 [IORING_OP_OPENAT2] = {
314 .filter_pdu_size = sizeof_field(struct io_uring_bpf_ctx, open),
315 .prep = io_openat2_prep,
316 .issue = io_openat2,
317 .filter_populate = io_openat_bpf_populate,
318 },
319 [IORING_OP_EPOLL_CTL] = {
320 .unbound_nonreg_file = 1,
321 .audit_skip = 1,
322 #if defined(CONFIG_EPOLL)
323 .prep = io_epoll_ctl_prep,
324 .issue = io_epoll_ctl,
325 #else
326 .prep = io_eopnotsupp_prep,
327 #endif
328 },
329 [IORING_OP_SPLICE] = {
330 .needs_file = 1,
331 .hash_reg_file = 1,
332 .unbound_nonreg_file = 1,
333 .audit_skip = 1,
334 .prep = io_splice_prep,
335 .issue = io_splice,
336 },
337 [IORING_OP_PROVIDE_BUFFERS] = {
338 .audit_skip = 1,
339 .iopoll = 1,
340 .prep = io_provide_buffers_prep,
341 .issue = io_manage_buffers_legacy,
342 },
343 [IORING_OP_REMOVE_BUFFERS] = {
344 .audit_skip = 1,
345 .iopoll = 1,
346 .prep = io_remove_buffers_prep,
347 .issue = io_manage_buffers_legacy,
348 },
349 [IORING_OP_TEE] = {
350 .needs_file = 1,
351 .hash_reg_file = 1,
352 .unbound_nonreg_file = 1,
353 .audit_skip = 1,
354 .prep = io_tee_prep,
355 .issue = io_tee,
356 },
357 [IORING_OP_SHUTDOWN] = {
358 .needs_file = 1,
359 #if defined(CONFIG_NET)
360 .prep = io_shutdown_prep,
361 .issue = io_shutdown,
362 #else
363 .prep = io_eopnotsupp_prep,
364 #endif
365 },
366 [IORING_OP_RENAMEAT] = {
367 .prep = io_renameat_prep,
368 .issue = io_renameat,
369 },
370 [IORING_OP_UNLINKAT] = {
371 .prep = io_unlinkat_prep,
372 .issue = io_unlinkat,
373 },
374 [IORING_OP_MKDIRAT] = {
375 .prep = io_mkdirat_prep,
376 .issue = io_mkdirat,
377 },
378 [IORING_OP_SYMLINKAT] = {
379 .prep = io_symlinkat_prep,
380 .issue = io_symlinkat,
381 },
382 [IORING_OP_LINKAT] = {
383 .prep = io_linkat_prep,
384 .issue = io_linkat,
385 },
386 [IORING_OP_MSG_RING] = {
387 .needs_file = 1,
388 .iopoll = 1,
389 .prep = io_msg_ring_prep,
390 .issue = io_msg_ring,
391 },
392 [IORING_OP_FSETXATTR] = {
393 .needs_file = 1,
394 .prep = io_fsetxattr_prep,
395 .issue = io_fsetxattr,
396 },
397 [IORING_OP_SETXATTR] = {
398 .prep = io_setxattr_prep,
399 .issue = io_setxattr,
400 },
401 [IORING_OP_FGETXATTR] = {
402 .needs_file = 1,
403 .prep = io_fgetxattr_prep,
404 .issue = io_fgetxattr,
405 },
406 [IORING_OP_GETXATTR] = {
407 .prep = io_getxattr_prep,
408 .issue = io_getxattr,
409 },
410 [IORING_OP_SOCKET] = {
411 .audit_skip = 1,
412 #if defined(CONFIG_NET)
413 .filter_pdu_size = sizeof_field(struct io_uring_bpf_ctx, socket),
414 .prep = io_socket_prep,
415 .issue = io_socket,
416 .filter_populate = io_socket_bpf_populate,
417 #else
418 .prep = io_eopnotsupp_prep,
419 #endif
420 },
421 [IORING_OP_URING_CMD] = {
422 .buffer_select = 1,
423 .needs_file = 1,
424 .plug = 1,
425 .iopoll = 1,
426 .iopoll_queue = 1,
427 .async_size = sizeof(struct io_async_cmd),
428 .prep = io_uring_cmd_prep,
429 .issue = io_uring_cmd,
430 },
431 [IORING_OP_SEND_ZC] = {
432 .needs_file = 1,
433 .unbound_nonreg_file = 1,
434 .pollout = 1,
435 .audit_skip = 1,
436 .ioprio = 1,
437 #if defined(CONFIG_NET)
438 .async_size = sizeof(struct io_async_msghdr),
439 .prep = io_send_zc_prep,
440 .issue = io_send_zc,
441 #else
442 .prep = io_eopnotsupp_prep,
443 #endif
444 },
445 [IORING_OP_SENDMSG_ZC] = {
446 .needs_file = 1,
447 .unbound_nonreg_file = 1,
448 .pollout = 1,
449 .ioprio = 1,
450 #if defined(CONFIG_NET)
451 .async_size = sizeof(struct io_async_msghdr),
452 .prep = io_send_zc_prep,
453 .issue = io_sendmsg_zc,
454 #else
455 .prep = io_eopnotsupp_prep,
456 #endif
457 },
458 [IORING_OP_READ_MULTISHOT] = {
459 .needs_file = 1,
460 .unbound_nonreg_file = 1,
461 .pollin = 1,
462 .buffer_select = 1,
463 .audit_skip = 1,
464 .async_size = sizeof(struct io_async_rw),
465 .prep = io_read_mshot_prep,
466 .issue = io_read_mshot,
467 },
468 [IORING_OP_WAITID] = {
469 .async_size = sizeof(struct io_waitid_async),
470 .prep = io_waitid_prep,
471 .issue = io_waitid,
472 },
473 [IORING_OP_FUTEX_WAIT] = {
474 #if defined(CONFIG_FUTEX)
475 .prep = io_futex_prep,
476 .issue = io_futex_wait,
477 #else
478 .prep = io_eopnotsupp_prep,
479 #endif
480 },
481 [IORING_OP_FUTEX_WAKE] = {
482 #if defined(CONFIG_FUTEX)
483 .prep = io_futex_prep,
484 .issue = io_futex_wake,
485 #else
486 .prep = io_eopnotsupp_prep,
487 #endif
488 },
489 [IORING_OP_FUTEX_WAITV] = {
490 #if defined(CONFIG_FUTEX)
491 .prep = io_futexv_prep,
492 .issue = io_futexv_wait,
493 #else
494 .prep = io_eopnotsupp_prep,
495 #endif
496 },
497 [IORING_OP_FIXED_FD_INSTALL] = {
498 .needs_file = 1,
499 .prep = io_install_fixed_fd_prep,
500 .issue = io_install_fixed_fd,
501 },
502 [IORING_OP_FTRUNCATE] = {
503 .needs_file = 1,
504 .hash_reg_file = 1,
505 .prep = io_ftruncate_prep,
506 .issue = io_ftruncate,
507 },
508 [IORING_OP_BIND] = {
509 #if defined(CONFIG_NET)
510 .needs_file = 1,
511 .prep = io_bind_prep,
512 .issue = io_bind,
513 .async_size = sizeof(struct io_async_msghdr),
514 #else
515 .prep = io_eopnotsupp_prep,
516 #endif
517 },
518 [IORING_OP_LISTEN] = {
519 #if defined(CONFIG_NET)
520 .needs_file = 1,
521 .prep = io_listen_prep,
522 .issue = io_listen,
523 .async_size = sizeof(struct io_async_msghdr),
524 #else
525 .prep = io_eopnotsupp_prep,
526 #endif
527 },
528 [IORING_OP_RECV_ZC] = {
529 .needs_file = 1,
530 .unbound_nonreg_file = 1,
531 .pollin = 1,
532 .ioprio = 1,
533 #if defined(CONFIG_NET)
534 .prep = io_recvzc_prep,
535 .issue = io_recvzc,
536 #else
537 .prep = io_eopnotsupp_prep,
538 #endif
539 },
540 [IORING_OP_EPOLL_WAIT] = {
541 .needs_file = 1,
542 .audit_skip = 1,
543 .pollin = 1,
544 #if defined(CONFIG_EPOLL)
545 .prep = io_epoll_wait_prep,
546 .issue = io_epoll_wait,
547 #else
548 .prep = io_eopnotsupp_prep,
549 #endif
550 },
551 [IORING_OP_READV_FIXED] = {
552 .needs_file = 1,
553 .unbound_nonreg_file = 1,
554 .pollin = 1,
555 .plug = 1,
556 .audit_skip = 1,
557 .ioprio = 1,
558 .iopoll = 1,
559 .iopoll_queue = 1,
560 .vectored = 1,
561 .async_size = sizeof(struct io_async_rw),
562 .prep = io_prep_readv_fixed,
563 .issue = io_read,
564 },
565 [IORING_OP_WRITEV_FIXED] = {
566 .needs_file = 1,
567 .hash_reg_file = 1,
568 .unbound_nonreg_file = 1,
569 .pollout = 1,
570 .plug = 1,
571 .audit_skip = 1,
572 .ioprio = 1,
573 .iopoll = 1,
574 .iopoll_queue = 1,
575 .vectored = 1,
576 .async_size = sizeof(struct io_async_rw),
577 .prep = io_prep_writev_fixed,
578 .issue = io_write,
579 },
580 [IORING_OP_PIPE] = {
581 .prep = io_pipe_prep,
582 .issue = io_pipe,
583 },
584 [IORING_OP_NOP128] = {
585 .audit_skip = 1,
586 .iopoll = 1,
587 .is_128 = 1,
588 .prep = io_nop_prep,
589 .issue = io_nop,
590 },
591 [IORING_OP_URING_CMD128] = {
592 .buffer_select = 1,
593 .needs_file = 1,
594 .plug = 1,
595 .iopoll = 1,
596 .iopoll_queue = 1,
597 .is_128 = 1,
598 .async_size = sizeof(struct io_async_cmd),
599 .prep = io_uring_cmd_prep,
600 .issue = io_uring_cmd,
601 },
602 };
603
604 const struct io_cold_def io_cold_defs[] = {
605 [IORING_OP_NOP] = {
606 .name = "NOP",
607 },
608 [IORING_OP_READV] = {
609 .name = "READV",
610 .cleanup = io_readv_writev_cleanup,
611 .fail = io_rw_fail,
612 },
613 [IORING_OP_WRITEV] = {
614 .name = "WRITEV",
615 .cleanup = io_readv_writev_cleanup,
616 .fail = io_rw_fail,
617 },
618 [IORING_OP_FSYNC] = {
619 .name = "FSYNC",
620 },
621 [IORING_OP_READ_FIXED] = {
622 .name = "READ_FIXED",
623 .cleanup = io_readv_writev_cleanup,
624 .fail = io_rw_fail,
625 },
626 [IORING_OP_WRITE_FIXED] = {
627 .name = "WRITE_FIXED",
628 .cleanup = io_readv_writev_cleanup,
629 .fail = io_rw_fail,
630 },
631 [IORING_OP_POLL_ADD] = {
632 .name = "POLL_ADD",
633 },
634 [IORING_OP_POLL_REMOVE] = {
635 .name = "POLL_REMOVE",
636 },
637 [IORING_OP_SYNC_FILE_RANGE] = {
638 .name = "SYNC_FILE_RANGE",
639 },
640 [IORING_OP_SENDMSG] = {
641 .name = "SENDMSG",
642 #if defined(CONFIG_NET)
643 .cleanup = io_sendmsg_recvmsg_cleanup,
644 .fail = io_sendrecv_fail,
645 #endif
646 },
647 [IORING_OP_RECVMSG] = {
648 .name = "RECVMSG",
649 #if defined(CONFIG_NET)
650 .cleanup = io_sendmsg_recvmsg_cleanup,
651 .fail = io_sendrecv_fail,
652 #endif
653 },
654 [IORING_OP_TIMEOUT] = {
655 .name = "TIMEOUT",
656 },
657 [IORING_OP_TIMEOUT_REMOVE] = {
658 .name = "TIMEOUT_REMOVE",
659 },
660 [IORING_OP_ACCEPT] = {
661 .name = "ACCEPT",
662 },
663 [IORING_OP_ASYNC_CANCEL] = {
664 .name = "ASYNC_CANCEL",
665 },
666 [IORING_OP_LINK_TIMEOUT] = {
667 .name = "LINK_TIMEOUT",
668 },
669 [IORING_OP_CONNECT] = {
670 .name = "CONNECT",
671 },
672 [IORING_OP_FALLOCATE] = {
673 .name = "FALLOCATE",
674 },
675 [IORING_OP_OPENAT] = {
676 .name = "OPENAT",
677 .cleanup = io_open_cleanup,
678 },
679 [IORING_OP_CLOSE] = {
680 .name = "CLOSE",
681 },
682 [IORING_OP_FILES_UPDATE] = {
683 .name = "FILES_UPDATE",
684 },
685 [IORING_OP_STATX] = {
686 .name = "STATX",
687 .cleanup = io_statx_cleanup,
688 },
689 [IORING_OP_READ] = {
690 .name = "READ",
691 .cleanup = io_readv_writev_cleanup,
692 .fail = io_rw_fail,
693 },
694 [IORING_OP_WRITE] = {
695 .name = "WRITE",
696 .cleanup = io_readv_writev_cleanup,
697 .fail = io_rw_fail,
698 },
699 [IORING_OP_FADVISE] = {
700 .name = "FADVISE",
701 },
702 [IORING_OP_MADVISE] = {
703 .name = "MADVISE",
704 },
705 [IORING_OP_SEND] = {
706 .name = "SEND",
707 #if defined(CONFIG_NET)
708 .cleanup = io_sendmsg_recvmsg_cleanup,
709 .fail = io_sendrecv_fail,
710 #endif
711 },
712 [IORING_OP_RECV] = {
713 .name = "RECV",
714 #if defined(CONFIG_NET)
715 .cleanup = io_sendmsg_recvmsg_cleanup,
716 .fail = io_sendrecv_fail,
717 #endif
718 },
719 [IORING_OP_OPENAT2] = {
720 .name = "OPENAT2",
721 .cleanup = io_open_cleanup,
722 },
723 [IORING_OP_EPOLL_CTL] = {
724 .name = "EPOLL",
725 },
726 [IORING_OP_SPLICE] = {
727 .name = "SPLICE",
728 .cleanup = io_splice_cleanup,
729 },
730 [IORING_OP_PROVIDE_BUFFERS] = {
731 .name = "PROVIDE_BUFFERS",
732 },
733 [IORING_OP_REMOVE_BUFFERS] = {
734 .name = "REMOVE_BUFFERS",
735 },
736 [IORING_OP_TEE] = {
737 .name = "TEE",
738 .cleanup = io_splice_cleanup,
739 },
740 [IORING_OP_SHUTDOWN] = {
741 .name = "SHUTDOWN",
742 },
743 [IORING_OP_RENAMEAT] = {
744 .name = "RENAMEAT",
745 .cleanup = io_renameat_cleanup,
746 },
747 [IORING_OP_UNLINKAT] = {
748 .name = "UNLINKAT",
749 .cleanup = io_unlinkat_cleanup,
750 },
751 [IORING_OP_MKDIRAT] = {
752 .name = "MKDIRAT",
753 .cleanup = io_mkdirat_cleanup,
754 },
755 [IORING_OP_SYMLINKAT] = {
756 .name = "SYMLINKAT",
757 .cleanup = io_link_cleanup,
758 },
759 [IORING_OP_LINKAT] = {
760 .name = "LINKAT",
761 .cleanup = io_link_cleanup,
762 },
763 [IORING_OP_MSG_RING] = {
764 .name = "MSG_RING",
765 .cleanup = io_msg_ring_cleanup,
766 },
767 [IORING_OP_FSETXATTR] = {
768 .name = "FSETXATTR",
769 .cleanup = io_xattr_cleanup,
770 },
771 [IORING_OP_SETXATTR] = {
772 .name = "SETXATTR",
773 .cleanup = io_xattr_cleanup,
774 },
775 [IORING_OP_FGETXATTR] = {
776 .name = "FGETXATTR",
777 .cleanup = io_xattr_cleanup,
778 },
779 [IORING_OP_GETXATTR] = {
780 .name = "GETXATTR",
781 .cleanup = io_xattr_cleanup,
782 },
783 [IORING_OP_SOCKET] = {
784 .name = "SOCKET",
785 },
786 [IORING_OP_URING_CMD] = {
787 .name = "URING_CMD",
788 .sqe_copy = io_uring_cmd_sqe_copy,
789 .cleanup = io_uring_cmd_cleanup,
790 },
791 [IORING_OP_SEND_ZC] = {
792 .name = "SEND_ZC",
793 #if defined(CONFIG_NET)
794 .cleanup = io_send_zc_cleanup,
795 .fail = io_sendrecv_fail,
796 #endif
797 },
798 [IORING_OP_SENDMSG_ZC] = {
799 .name = "SENDMSG_ZC",
800 #if defined(CONFIG_NET)
801 .cleanup = io_send_zc_cleanup,
802 .fail = io_sendrecv_fail,
803 #endif
804 },
805 [IORING_OP_READ_MULTISHOT] = {
806 .name = "READ_MULTISHOT",
807 .cleanup = io_readv_writev_cleanup,
808 },
809 [IORING_OP_WAITID] = {
810 .name = "WAITID",
811 },
812 [IORING_OP_FUTEX_WAIT] = {
813 .name = "FUTEX_WAIT",
814 },
815 [IORING_OP_FUTEX_WAKE] = {
816 .name = "FUTEX_WAKE",
817 },
818 [IORING_OP_FUTEX_WAITV] = {
819 .name = "FUTEX_WAITV",
820 },
821 [IORING_OP_FIXED_FD_INSTALL] = {
822 .name = "FIXED_FD_INSTALL",
823 },
824 [IORING_OP_FTRUNCATE] = {
825 .name = "FTRUNCATE",
826 },
827 [IORING_OP_BIND] = {
828 .name = "BIND",
829 },
830 [IORING_OP_LISTEN] = {
831 .name = "LISTEN",
832 },
833 [IORING_OP_RECV_ZC] = {
834 .name = "RECV_ZC",
835 },
836 [IORING_OP_EPOLL_WAIT] = {
837 .name = "EPOLL_WAIT",
838 },
839 [IORING_OP_READV_FIXED] = {
840 .name = "READV_FIXED",
841 .cleanup = io_readv_writev_cleanup,
842 .fail = io_rw_fail,
843 },
844 [IORING_OP_WRITEV_FIXED] = {
845 .name = "WRITEV_FIXED",
846 .cleanup = io_readv_writev_cleanup,
847 .fail = io_rw_fail,
848 },
849 [IORING_OP_PIPE] = {
850 .name = "PIPE",
851 },
852 [IORING_OP_NOP128] = {
853 .name = "NOP128",
854 },
855 [IORING_OP_URING_CMD128] = {
856 .name = "URING_CMD128",
857 .sqe_copy = io_uring_cmd_sqe_copy,
858 .cleanup = io_uring_cmd_cleanup,
859 },
860 };
861
io_uring_get_opcode(u8 opcode)862 const char *io_uring_get_opcode(u8 opcode)
863 {
864 if (opcode < IORING_OP_LAST)
865 return io_cold_defs[opcode].name;
866 return "INVALID";
867 }
868
io_uring_op_supported(u8 opcode)869 bool io_uring_op_supported(u8 opcode)
870 {
871 if (opcode < IORING_OP_LAST &&
872 io_issue_defs[opcode].prep != io_eopnotsupp_prep)
873 return true;
874 return false;
875 }
876
io_uring_optable_init(void)877 void __init io_uring_optable_init(void)
878 {
879 int i;
880
881 BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs) != IORING_OP_LAST);
882 BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST);
883
884 for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) {
885 BUG_ON(!io_issue_defs[i].prep);
886 if (io_issue_defs[i].prep != io_eopnotsupp_prep)
887 BUG_ON(!io_issue_defs[i].issue);
888 WARN_ON_ONCE(!io_cold_defs[i].name);
889 }
890 }
891