1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * io_uring opcode handling table
4 */
5 #include <linux/kernel.h>
6 #include <linux/errno.h>
7 #include <linux/fs.h>
8 #include <linux/file.h>
9 #include <linux/io_uring.h>
10 #include <linux/io_uring/cmd.h>
11
12 #include "io_uring.h"
13 #include "opdef.h"
14 #include "refs.h"
15 #include "tctx.h"
16 #include "sqpoll.h"
17 #include "fdinfo.h"
18 #include "kbuf.h"
19 #include "rsrc.h"
20
21 #include "xattr.h"
22 #include "nop.h"
23 #include "fs.h"
24 #include "splice.h"
25 #include "sync.h"
26 #include "advise.h"
27 #include "openclose.h"
28 #include "uring_cmd.h"
29 #include "epoll.h"
30 #include "statx.h"
31 #include "net.h"
32 #include "msg_ring.h"
33 #include "timeout.h"
34 #include "poll.h"
35 #include "cancel.h"
36 #include "rw.h"
37 #include "waitid.h"
38 #include "futex.h"
39 #include "truncate.h"
40 #include "zcrx.h"
41
io_no_issue(struct io_kiocb * req,unsigned int issue_flags)42 static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
43 {
44 WARN_ON_ONCE(1);
45 return -ECANCELED;
46 }
47
io_eopnotsupp_prep(struct io_kiocb * kiocb,const struct io_uring_sqe * sqe)48 static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,
49 const struct io_uring_sqe *sqe)
50 {
51 return -EOPNOTSUPP;
52 }
53
54 const struct io_issue_def io_issue_defs[] = {
55 [IORING_OP_NOP] = {
56 .audit_skip = 1,
57 .iopoll = 1,
58 .prep = io_nop_prep,
59 .issue = io_nop,
60 },
61 [IORING_OP_READV] = {
62 .needs_file = 1,
63 .unbound_nonreg_file = 1,
64 .pollin = 1,
65 .buffer_select = 1,
66 .plug = 1,
67 .audit_skip = 1,
68 .ioprio = 1,
69 .iopoll = 1,
70 .vectored = 1,
71 .async_size = sizeof(struct io_async_rw),
72 .prep = io_prep_readv,
73 .issue = io_read,
74 },
75 [IORING_OP_WRITEV] = {
76 .needs_file = 1,
77 .hash_reg_file = 1,
78 .unbound_nonreg_file = 1,
79 .pollout = 1,
80 .plug = 1,
81 .audit_skip = 1,
82 .ioprio = 1,
83 .iopoll = 1,
84 .vectored = 1,
85 .async_size = sizeof(struct io_async_rw),
86 .prep = io_prep_writev,
87 .issue = io_write,
88 },
89 [IORING_OP_FSYNC] = {
90 .needs_file = 1,
91 .audit_skip = 1,
92 .prep = io_fsync_prep,
93 .issue = io_fsync,
94 },
95 [IORING_OP_READ_FIXED] = {
96 .needs_file = 1,
97 .unbound_nonreg_file = 1,
98 .pollin = 1,
99 .plug = 1,
100 .audit_skip = 1,
101 .ioprio = 1,
102 .iopoll = 1,
103 .async_size = sizeof(struct io_async_rw),
104 .prep = io_prep_read_fixed,
105 .issue = io_read_fixed,
106 },
107 [IORING_OP_WRITE_FIXED] = {
108 .needs_file = 1,
109 .hash_reg_file = 1,
110 .unbound_nonreg_file = 1,
111 .pollout = 1,
112 .plug = 1,
113 .audit_skip = 1,
114 .ioprio = 1,
115 .iopoll = 1,
116 .async_size = sizeof(struct io_async_rw),
117 .prep = io_prep_write_fixed,
118 .issue = io_write_fixed,
119 },
120 [IORING_OP_POLL_ADD] = {
121 .needs_file = 1,
122 .unbound_nonreg_file = 1,
123 .audit_skip = 1,
124 .prep = io_poll_add_prep,
125 .issue = io_poll_add,
126 },
127 [IORING_OP_POLL_REMOVE] = {
128 .audit_skip = 1,
129 .prep = io_poll_remove_prep,
130 .issue = io_poll_remove,
131 },
132 [IORING_OP_SYNC_FILE_RANGE] = {
133 .needs_file = 1,
134 .audit_skip = 1,
135 .prep = io_sfr_prep,
136 .issue = io_sync_file_range,
137 },
138 [IORING_OP_SENDMSG] = {
139 .needs_file = 1,
140 .unbound_nonreg_file = 1,
141 .pollout = 1,
142 .ioprio = 1,
143 #if defined(CONFIG_NET)
144 .async_size = sizeof(struct io_async_msghdr),
145 .prep = io_sendmsg_prep,
146 .issue = io_sendmsg,
147 #else
148 .prep = io_eopnotsupp_prep,
149 #endif
150 },
151 [IORING_OP_RECVMSG] = {
152 .needs_file = 1,
153 .unbound_nonreg_file = 1,
154 .pollin = 1,
155 .buffer_select = 1,
156 .ioprio = 1,
157 #if defined(CONFIG_NET)
158 .async_size = sizeof(struct io_async_msghdr),
159 .prep = io_recvmsg_prep,
160 .issue = io_recvmsg,
161 #else
162 .prep = io_eopnotsupp_prep,
163 #endif
164 },
165 [IORING_OP_TIMEOUT] = {
166 .audit_skip = 1,
167 .async_size = sizeof(struct io_timeout_data),
168 .prep = io_timeout_prep,
169 .issue = io_timeout,
170 },
171 [IORING_OP_TIMEOUT_REMOVE] = {
172 /* used by timeout updates' prep() */
173 .audit_skip = 1,
174 .prep = io_timeout_remove_prep,
175 .issue = io_timeout_remove,
176 },
177 [IORING_OP_ACCEPT] = {
178 .needs_file = 1,
179 .unbound_nonreg_file = 1,
180 .pollin = 1,
181 .poll_exclusive = 1,
182 .ioprio = 1, /* used for flags */
183 #if defined(CONFIG_NET)
184 .prep = io_accept_prep,
185 .issue = io_accept,
186 #else
187 .prep = io_eopnotsupp_prep,
188 #endif
189 },
190 [IORING_OP_ASYNC_CANCEL] = {
191 .audit_skip = 1,
192 .prep = io_async_cancel_prep,
193 .issue = io_async_cancel,
194 },
195 [IORING_OP_LINK_TIMEOUT] = {
196 .audit_skip = 1,
197 .async_size = sizeof(struct io_timeout_data),
198 .prep = io_link_timeout_prep,
199 .issue = io_no_issue,
200 },
201 [IORING_OP_CONNECT] = {
202 .needs_file = 1,
203 .unbound_nonreg_file = 1,
204 .pollout = 1,
205 #if defined(CONFIG_NET)
206 .async_size = sizeof(struct io_async_msghdr),
207 .prep = io_connect_prep,
208 .issue = io_connect,
209 #else
210 .prep = io_eopnotsupp_prep,
211 #endif
212 },
213 [IORING_OP_FALLOCATE] = {
214 .needs_file = 1,
215 .hash_reg_file = 1,
216 .prep = io_fallocate_prep,
217 .issue = io_fallocate,
218 },
219 [IORING_OP_OPENAT] = {
220 .filter_pdu_size = sizeof_field(struct io_uring_bpf_ctx, open),
221 .prep = io_openat_prep,
222 .issue = io_openat,
223 .filter_populate = io_openat_bpf_populate,
224 },
225 [IORING_OP_CLOSE] = {
226 .prep = io_close_prep,
227 .issue = io_close,
228 },
229 [IORING_OP_FILES_UPDATE] = {
230 .audit_skip = 1,
231 .iopoll = 1,
232 .prep = io_files_update_prep,
233 .issue = io_files_update,
234 },
235 [IORING_OP_STATX] = {
236 .audit_skip = 1,
237 .prep = io_statx_prep,
238 .issue = io_statx,
239 },
240 [IORING_OP_READ] = {
241 .needs_file = 1,
242 .unbound_nonreg_file = 1,
243 .pollin = 1,
244 .buffer_select = 1,
245 .plug = 1,
246 .audit_skip = 1,
247 .ioprio = 1,
248 .iopoll = 1,
249 .async_size = sizeof(struct io_async_rw),
250 .prep = io_prep_read,
251 .issue = io_read,
252 },
253 [IORING_OP_WRITE] = {
254 .needs_file = 1,
255 .hash_reg_file = 1,
256 .unbound_nonreg_file = 1,
257 .pollout = 1,
258 .plug = 1,
259 .audit_skip = 1,
260 .ioprio = 1,
261 .iopoll = 1,
262 .async_size = sizeof(struct io_async_rw),
263 .prep = io_prep_write,
264 .issue = io_write,
265 },
266 [IORING_OP_FADVISE] = {
267 .needs_file = 1,
268 .audit_skip = 1,
269 .prep = io_fadvise_prep,
270 .issue = io_fadvise,
271 },
272 [IORING_OP_MADVISE] = {
273 .audit_skip = 1,
274 .prep = io_madvise_prep,
275 .issue = io_madvise,
276 },
277 [IORING_OP_SEND] = {
278 .needs_file = 1,
279 .unbound_nonreg_file = 1,
280 .pollout = 1,
281 .audit_skip = 1,
282 .ioprio = 1,
283 .buffer_select = 1,
284 #if defined(CONFIG_NET)
285 .async_size = sizeof(struct io_async_msghdr),
286 .prep = io_sendmsg_prep,
287 .issue = io_send,
288 #else
289 .prep = io_eopnotsupp_prep,
290 #endif
291 },
292 [IORING_OP_RECV] = {
293 .needs_file = 1,
294 .unbound_nonreg_file = 1,
295 .pollin = 1,
296 .buffer_select = 1,
297 .audit_skip = 1,
298 .ioprio = 1,
299 #if defined(CONFIG_NET)
300 .async_size = sizeof(struct io_async_msghdr),
301 .prep = io_recvmsg_prep,
302 .issue = io_recv,
303 #else
304 .prep = io_eopnotsupp_prep,
305 #endif
306 },
307 [IORING_OP_OPENAT2] = {
308 .filter_pdu_size = sizeof_field(struct io_uring_bpf_ctx, open),
309 .prep = io_openat2_prep,
310 .issue = io_openat2,
311 .filter_populate = io_openat_bpf_populate,
312 },
313 [IORING_OP_EPOLL_CTL] = {
314 .unbound_nonreg_file = 1,
315 .audit_skip = 1,
316 #if defined(CONFIG_EPOLL)
317 .prep = io_epoll_ctl_prep,
318 .issue = io_epoll_ctl,
319 #else
320 .prep = io_eopnotsupp_prep,
321 #endif
322 },
323 [IORING_OP_SPLICE] = {
324 .needs_file = 1,
325 .hash_reg_file = 1,
326 .unbound_nonreg_file = 1,
327 .audit_skip = 1,
328 .prep = io_splice_prep,
329 .issue = io_splice,
330 },
331 [IORING_OP_PROVIDE_BUFFERS] = {
332 .audit_skip = 1,
333 .iopoll = 1,
334 .prep = io_provide_buffers_prep,
335 .issue = io_manage_buffers_legacy,
336 },
337 [IORING_OP_REMOVE_BUFFERS] = {
338 .audit_skip = 1,
339 .iopoll = 1,
340 .prep = io_remove_buffers_prep,
341 .issue = io_manage_buffers_legacy,
342 },
343 [IORING_OP_TEE] = {
344 .needs_file = 1,
345 .hash_reg_file = 1,
346 .unbound_nonreg_file = 1,
347 .audit_skip = 1,
348 .prep = io_tee_prep,
349 .issue = io_tee,
350 },
351 [IORING_OP_SHUTDOWN] = {
352 .needs_file = 1,
353 #if defined(CONFIG_NET)
354 .prep = io_shutdown_prep,
355 .issue = io_shutdown,
356 #else
357 .prep = io_eopnotsupp_prep,
358 #endif
359 },
360 [IORING_OP_RENAMEAT] = {
361 .prep = io_renameat_prep,
362 .issue = io_renameat,
363 },
364 [IORING_OP_UNLINKAT] = {
365 .prep = io_unlinkat_prep,
366 .issue = io_unlinkat,
367 },
368 [IORING_OP_MKDIRAT] = {
369 .prep = io_mkdirat_prep,
370 .issue = io_mkdirat,
371 },
372 [IORING_OP_SYMLINKAT] = {
373 .prep = io_symlinkat_prep,
374 .issue = io_symlinkat,
375 },
376 [IORING_OP_LINKAT] = {
377 .prep = io_linkat_prep,
378 .issue = io_linkat,
379 },
380 [IORING_OP_MSG_RING] = {
381 .needs_file = 1,
382 .iopoll = 1,
383 .prep = io_msg_ring_prep,
384 .issue = io_msg_ring,
385 },
386 [IORING_OP_FSETXATTR] = {
387 .needs_file = 1,
388 .prep = io_fsetxattr_prep,
389 .issue = io_fsetxattr,
390 },
391 [IORING_OP_SETXATTR] = {
392 .prep = io_setxattr_prep,
393 .issue = io_setxattr,
394 },
395 [IORING_OP_FGETXATTR] = {
396 .needs_file = 1,
397 .prep = io_fgetxattr_prep,
398 .issue = io_fgetxattr,
399 },
400 [IORING_OP_GETXATTR] = {
401 .prep = io_getxattr_prep,
402 .issue = io_getxattr,
403 },
404 [IORING_OP_SOCKET] = {
405 .audit_skip = 1,
406 #if defined(CONFIG_NET)
407 .filter_pdu_size = sizeof_field(struct io_uring_bpf_ctx, socket),
408 .prep = io_socket_prep,
409 .issue = io_socket,
410 .filter_populate = io_socket_bpf_populate,
411 #else
412 .prep = io_eopnotsupp_prep,
413 #endif
414 },
415 [IORING_OP_URING_CMD] = {
416 .buffer_select = 1,
417 .needs_file = 1,
418 .plug = 1,
419 .iopoll = 1,
420 .async_size = sizeof(struct io_async_cmd),
421 .prep = io_uring_cmd_prep,
422 .issue = io_uring_cmd,
423 },
424 [IORING_OP_SEND_ZC] = {
425 .needs_file = 1,
426 .unbound_nonreg_file = 1,
427 .pollout = 1,
428 .audit_skip = 1,
429 .ioprio = 1,
430 #if defined(CONFIG_NET)
431 .async_size = sizeof(struct io_async_msghdr),
432 .prep = io_send_zc_prep,
433 .issue = io_sendmsg_zc,
434 #else
435 .prep = io_eopnotsupp_prep,
436 #endif
437 },
438 [IORING_OP_SENDMSG_ZC] = {
439 .needs_file = 1,
440 .unbound_nonreg_file = 1,
441 .pollout = 1,
442 .ioprio = 1,
443 #if defined(CONFIG_NET)
444 .async_size = sizeof(struct io_async_msghdr),
445 .prep = io_send_zc_prep,
446 .issue = io_sendmsg_zc,
447 #else
448 .prep = io_eopnotsupp_prep,
449 #endif
450 },
451 [IORING_OP_READ_MULTISHOT] = {
452 .needs_file = 1,
453 .unbound_nonreg_file = 1,
454 .pollin = 1,
455 .buffer_select = 1,
456 .audit_skip = 1,
457 .async_size = sizeof(struct io_async_rw),
458 .prep = io_read_mshot_prep,
459 .issue = io_read_mshot,
460 },
461 [IORING_OP_WAITID] = {
462 .async_size = sizeof(struct io_waitid_async),
463 .prep = io_waitid_prep,
464 .issue = io_waitid,
465 },
466 [IORING_OP_FUTEX_WAIT] = {
467 #if defined(CONFIG_FUTEX)
468 .prep = io_futex_prep,
469 .issue = io_futex_wait,
470 #else
471 .prep = io_eopnotsupp_prep,
472 #endif
473 },
474 [IORING_OP_FUTEX_WAKE] = {
475 #if defined(CONFIG_FUTEX)
476 .prep = io_futex_prep,
477 .issue = io_futex_wake,
478 #else
479 .prep = io_eopnotsupp_prep,
480 #endif
481 },
482 [IORING_OP_FUTEX_WAITV] = {
483 #if defined(CONFIG_FUTEX)
484 .prep = io_futexv_prep,
485 .issue = io_futexv_wait,
486 #else
487 .prep = io_eopnotsupp_prep,
488 #endif
489 },
490 [IORING_OP_FIXED_FD_INSTALL] = {
491 .needs_file = 1,
492 .prep = io_install_fixed_fd_prep,
493 .issue = io_install_fixed_fd,
494 },
495 [IORING_OP_FTRUNCATE] = {
496 .needs_file = 1,
497 .hash_reg_file = 1,
498 .prep = io_ftruncate_prep,
499 .issue = io_ftruncate,
500 },
501 [IORING_OP_BIND] = {
502 #if defined(CONFIG_NET)
503 .needs_file = 1,
504 .prep = io_bind_prep,
505 .issue = io_bind,
506 .async_size = sizeof(struct io_async_msghdr),
507 #else
508 .prep = io_eopnotsupp_prep,
509 #endif
510 },
511 [IORING_OP_LISTEN] = {
512 #if defined(CONFIG_NET)
513 .needs_file = 1,
514 .prep = io_listen_prep,
515 .issue = io_listen,
516 .async_size = sizeof(struct io_async_msghdr),
517 #else
518 .prep = io_eopnotsupp_prep,
519 #endif
520 },
521 [IORING_OP_RECV_ZC] = {
522 .needs_file = 1,
523 .unbound_nonreg_file = 1,
524 .pollin = 1,
525 .ioprio = 1,
526 #if defined(CONFIG_NET)
527 .prep = io_recvzc_prep,
528 .issue = io_recvzc,
529 #else
530 .prep = io_eopnotsupp_prep,
531 #endif
532 },
533 [IORING_OP_EPOLL_WAIT] = {
534 .needs_file = 1,
535 .audit_skip = 1,
536 .pollin = 1,
537 #if defined(CONFIG_EPOLL)
538 .prep = io_epoll_wait_prep,
539 .issue = io_epoll_wait,
540 #else
541 .prep = io_eopnotsupp_prep,
542 #endif
543 },
544 [IORING_OP_READV_FIXED] = {
545 .needs_file = 1,
546 .unbound_nonreg_file = 1,
547 .pollin = 1,
548 .plug = 1,
549 .audit_skip = 1,
550 .ioprio = 1,
551 .iopoll = 1,
552 .vectored = 1,
553 .async_size = sizeof(struct io_async_rw),
554 .prep = io_prep_readv_fixed,
555 .issue = io_read,
556 },
557 [IORING_OP_WRITEV_FIXED] = {
558 .needs_file = 1,
559 .hash_reg_file = 1,
560 .unbound_nonreg_file = 1,
561 .pollout = 1,
562 .plug = 1,
563 .audit_skip = 1,
564 .ioprio = 1,
565 .iopoll = 1,
566 .vectored = 1,
567 .async_size = sizeof(struct io_async_rw),
568 .prep = io_prep_writev_fixed,
569 .issue = io_write,
570 },
571 [IORING_OP_PIPE] = {
572 .prep = io_pipe_prep,
573 .issue = io_pipe,
574 },
575 [IORING_OP_NOP128] = {
576 .audit_skip = 1,
577 .iopoll = 1,
578 .is_128 = 1,
579 .prep = io_nop_prep,
580 .issue = io_nop,
581 },
582 [IORING_OP_URING_CMD128] = {
583 .buffer_select = 1,
584 .needs_file = 1,
585 .plug = 1,
586 .iopoll = 1,
587 .is_128 = 1,
588 .async_size = sizeof(struct io_async_cmd),
589 .prep = io_uring_cmd_prep,
590 .issue = io_uring_cmd,
591 },
592 };
593
594 const struct io_cold_def io_cold_defs[] = {
595 [IORING_OP_NOP] = {
596 .name = "NOP",
597 },
598 [IORING_OP_READV] = {
599 .name = "READV",
600 .cleanup = io_readv_writev_cleanup,
601 .fail = io_rw_fail,
602 },
603 [IORING_OP_WRITEV] = {
604 .name = "WRITEV",
605 .cleanup = io_readv_writev_cleanup,
606 .fail = io_rw_fail,
607 },
608 [IORING_OP_FSYNC] = {
609 .name = "FSYNC",
610 },
611 [IORING_OP_READ_FIXED] = {
612 .name = "READ_FIXED",
613 .cleanup = io_readv_writev_cleanup,
614 .fail = io_rw_fail,
615 },
616 [IORING_OP_WRITE_FIXED] = {
617 .name = "WRITE_FIXED",
618 .cleanup = io_readv_writev_cleanup,
619 .fail = io_rw_fail,
620 },
621 [IORING_OP_POLL_ADD] = {
622 .name = "POLL_ADD",
623 },
624 [IORING_OP_POLL_REMOVE] = {
625 .name = "POLL_REMOVE",
626 },
627 [IORING_OP_SYNC_FILE_RANGE] = {
628 .name = "SYNC_FILE_RANGE",
629 },
630 [IORING_OP_SENDMSG] = {
631 .name = "SENDMSG",
632 #if defined(CONFIG_NET)
633 .cleanup = io_sendmsg_recvmsg_cleanup,
634 .fail = io_sendrecv_fail,
635 #endif
636 },
637 [IORING_OP_RECVMSG] = {
638 .name = "RECVMSG",
639 #if defined(CONFIG_NET)
640 .cleanup = io_sendmsg_recvmsg_cleanup,
641 .fail = io_sendrecv_fail,
642 #endif
643 },
644 [IORING_OP_TIMEOUT] = {
645 .name = "TIMEOUT",
646 },
647 [IORING_OP_TIMEOUT_REMOVE] = {
648 .name = "TIMEOUT_REMOVE",
649 },
650 [IORING_OP_ACCEPT] = {
651 .name = "ACCEPT",
652 },
653 [IORING_OP_ASYNC_CANCEL] = {
654 .name = "ASYNC_CANCEL",
655 },
656 [IORING_OP_LINK_TIMEOUT] = {
657 .name = "LINK_TIMEOUT",
658 },
659 [IORING_OP_CONNECT] = {
660 .name = "CONNECT",
661 },
662 [IORING_OP_FALLOCATE] = {
663 .name = "FALLOCATE",
664 },
665 [IORING_OP_OPENAT] = {
666 .name = "OPENAT",
667 .cleanup = io_open_cleanup,
668 },
669 [IORING_OP_CLOSE] = {
670 .name = "CLOSE",
671 },
672 [IORING_OP_FILES_UPDATE] = {
673 .name = "FILES_UPDATE",
674 },
675 [IORING_OP_STATX] = {
676 .name = "STATX",
677 .cleanup = io_statx_cleanup,
678 },
679 [IORING_OP_READ] = {
680 .name = "READ",
681 .cleanup = io_readv_writev_cleanup,
682 .fail = io_rw_fail,
683 },
684 [IORING_OP_WRITE] = {
685 .name = "WRITE",
686 .cleanup = io_readv_writev_cleanup,
687 .fail = io_rw_fail,
688 },
689 [IORING_OP_FADVISE] = {
690 .name = "FADVISE",
691 },
692 [IORING_OP_MADVISE] = {
693 .name = "MADVISE",
694 },
695 [IORING_OP_SEND] = {
696 .name = "SEND",
697 #if defined(CONFIG_NET)
698 .cleanup = io_sendmsg_recvmsg_cleanup,
699 .fail = io_sendrecv_fail,
700 #endif
701 },
702 [IORING_OP_RECV] = {
703 .name = "RECV",
704 #if defined(CONFIG_NET)
705 .cleanup = io_sendmsg_recvmsg_cleanup,
706 .fail = io_sendrecv_fail,
707 #endif
708 },
709 [IORING_OP_OPENAT2] = {
710 .name = "OPENAT2",
711 .cleanup = io_open_cleanup,
712 },
713 [IORING_OP_EPOLL_CTL] = {
714 .name = "EPOLL",
715 },
716 [IORING_OP_SPLICE] = {
717 .name = "SPLICE",
718 .cleanup = io_splice_cleanup,
719 },
720 [IORING_OP_PROVIDE_BUFFERS] = {
721 .name = "PROVIDE_BUFFERS",
722 },
723 [IORING_OP_REMOVE_BUFFERS] = {
724 .name = "REMOVE_BUFFERS",
725 },
726 [IORING_OP_TEE] = {
727 .name = "TEE",
728 .cleanup = io_splice_cleanup,
729 },
730 [IORING_OP_SHUTDOWN] = {
731 .name = "SHUTDOWN",
732 },
733 [IORING_OP_RENAMEAT] = {
734 .name = "RENAMEAT",
735 .cleanup = io_renameat_cleanup,
736 },
737 [IORING_OP_UNLINKAT] = {
738 .name = "UNLINKAT",
739 .cleanup = io_unlinkat_cleanup,
740 },
741 [IORING_OP_MKDIRAT] = {
742 .name = "MKDIRAT",
743 .cleanup = io_mkdirat_cleanup,
744 },
745 [IORING_OP_SYMLINKAT] = {
746 .name = "SYMLINKAT",
747 .cleanup = io_link_cleanup,
748 },
749 [IORING_OP_LINKAT] = {
750 .name = "LINKAT",
751 .cleanup = io_link_cleanup,
752 },
753 [IORING_OP_MSG_RING] = {
754 .name = "MSG_RING",
755 .cleanup = io_msg_ring_cleanup,
756 },
757 [IORING_OP_FSETXATTR] = {
758 .name = "FSETXATTR",
759 .cleanup = io_xattr_cleanup,
760 },
761 [IORING_OP_SETXATTR] = {
762 .name = "SETXATTR",
763 .cleanup = io_xattr_cleanup,
764 },
765 [IORING_OP_FGETXATTR] = {
766 .name = "FGETXATTR",
767 .cleanup = io_xattr_cleanup,
768 },
769 [IORING_OP_GETXATTR] = {
770 .name = "GETXATTR",
771 .cleanup = io_xattr_cleanup,
772 },
773 [IORING_OP_SOCKET] = {
774 .name = "SOCKET",
775 },
776 [IORING_OP_URING_CMD] = {
777 .name = "URING_CMD",
778 .sqe_copy = io_uring_cmd_sqe_copy,
779 .cleanup = io_uring_cmd_cleanup,
780 },
781 [IORING_OP_SEND_ZC] = {
782 .name = "SEND_ZC",
783 #if defined(CONFIG_NET)
784 .cleanup = io_send_zc_cleanup,
785 .fail = io_sendrecv_fail,
786 #endif
787 },
788 [IORING_OP_SENDMSG_ZC] = {
789 .name = "SENDMSG_ZC",
790 #if defined(CONFIG_NET)
791 .cleanup = io_send_zc_cleanup,
792 .fail = io_sendrecv_fail,
793 #endif
794 },
795 [IORING_OP_READ_MULTISHOT] = {
796 .name = "READ_MULTISHOT",
797 .cleanup = io_readv_writev_cleanup,
798 },
799 [IORING_OP_WAITID] = {
800 .name = "WAITID",
801 },
802 [IORING_OP_FUTEX_WAIT] = {
803 .name = "FUTEX_WAIT",
804 },
805 [IORING_OP_FUTEX_WAKE] = {
806 .name = "FUTEX_WAKE",
807 },
808 [IORING_OP_FUTEX_WAITV] = {
809 .name = "FUTEX_WAITV",
810 },
811 [IORING_OP_FIXED_FD_INSTALL] = {
812 .name = "FIXED_FD_INSTALL",
813 },
814 [IORING_OP_FTRUNCATE] = {
815 .name = "FTRUNCATE",
816 },
817 [IORING_OP_BIND] = {
818 .name = "BIND",
819 },
820 [IORING_OP_LISTEN] = {
821 .name = "LISTEN",
822 },
823 [IORING_OP_RECV_ZC] = {
824 .name = "RECV_ZC",
825 },
826 [IORING_OP_EPOLL_WAIT] = {
827 .name = "EPOLL_WAIT",
828 },
829 [IORING_OP_READV_FIXED] = {
830 .name = "READV_FIXED",
831 .cleanup = io_readv_writev_cleanup,
832 .fail = io_rw_fail,
833 },
834 [IORING_OP_WRITEV_FIXED] = {
835 .name = "WRITEV_FIXED",
836 .cleanup = io_readv_writev_cleanup,
837 .fail = io_rw_fail,
838 },
839 [IORING_OP_PIPE] = {
840 .name = "PIPE",
841 },
842 [IORING_OP_NOP128] = {
843 .name = "NOP128",
844 },
845 [IORING_OP_URING_CMD128] = {
846 .name = "URING_CMD128",
847 .sqe_copy = io_uring_cmd_sqe_copy,
848 .cleanup = io_uring_cmd_cleanup,
849 },
850 };
851
io_uring_get_opcode(u8 opcode)852 const char *io_uring_get_opcode(u8 opcode)
853 {
854 if (opcode < IORING_OP_LAST)
855 return io_cold_defs[opcode].name;
856 return "INVALID";
857 }
858
io_uring_op_supported(u8 opcode)859 bool io_uring_op_supported(u8 opcode)
860 {
861 if (opcode < IORING_OP_LAST &&
862 io_issue_defs[opcode].prep != io_eopnotsupp_prep)
863 return true;
864 return false;
865 }
866
io_uring_optable_init(void)867 void __init io_uring_optable_init(void)
868 {
869 int i;
870
871 BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs) != IORING_OP_LAST);
872 BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST);
873
874 for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) {
875 BUG_ON(!io_issue_defs[i].prep);
876 if (io_issue_defs[i].prep != io_eopnotsupp_prep)
877 BUG_ON(!io_issue_defs[i].issue);
878 WARN_ON_ONCE(!io_cold_defs[i].name);
879 }
880 }
881