1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * io_uring opcode handling table
4 */
5 #include <linux/kernel.h>
6 #include <linux/errno.h>
7 #include <linux/fs.h>
8 #include <linux/file.h>
9 #include <linux/io_uring.h>
10 #include <linux/io_uring/cmd.h>
11
12 #include "io_uring.h"
13 #include "opdef.h"
14 #include "refs.h"
15 #include "tctx.h"
16 #include "sqpoll.h"
17 #include "fdinfo.h"
18 #include "kbuf.h"
19 #include "rsrc.h"
20
21 #include "xattr.h"
22 #include "nop.h"
23 #include "fs.h"
24 #include "splice.h"
25 #include "sync.h"
26 #include "advise.h"
27 #include "openclose.h"
28 #include "uring_cmd.h"
29 #include "epoll.h"
30 #include "statx.h"
31 #include "net.h"
32 #include "msg_ring.h"
33 #include "timeout.h"
34 #include "poll.h"
35 #include "cancel.h"
36 #include "rw.h"
37 #include "waitid.h"
38 #include "futex.h"
39 #include "truncate.h"
40 #include "zcrx.h"
41
io_no_issue(struct io_kiocb * req,unsigned int issue_flags)42 static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
43 {
44 WARN_ON_ONCE(1);
45 return -ECANCELED;
46 }
47
io_eopnotsupp_prep(struct io_kiocb * kiocb,const struct io_uring_sqe * sqe)48 static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,
49 const struct io_uring_sqe *sqe)
50 {
51 return -EOPNOTSUPP;
52 }
53
54 const struct io_issue_def io_issue_defs[] = {
55 [IORING_OP_NOP] = {
56 .audit_skip = 1,
57 .iopoll = 1,
58 .prep = io_nop_prep,
59 .issue = io_nop,
60 },
61 [IORING_OP_READV] = {
62 .needs_file = 1,
63 .unbound_nonreg_file = 1,
64 .pollin = 1,
65 .buffer_select = 1,
66 .plug = 1,
67 .audit_skip = 1,
68 .ioprio = 1,
69 .iopoll = 1,
70 .iopoll_queue = 1,
71 .vectored = 1,
72 .async_size = sizeof(struct io_async_rw),
73 .prep = io_prep_readv,
74 .issue = io_read,
75 },
76 [IORING_OP_WRITEV] = {
77 .needs_file = 1,
78 .hash_reg_file = 1,
79 .unbound_nonreg_file = 1,
80 .pollout = 1,
81 .plug = 1,
82 .audit_skip = 1,
83 .ioprio = 1,
84 .iopoll = 1,
85 .iopoll_queue = 1,
86 .vectored = 1,
87 .async_size = sizeof(struct io_async_rw),
88 .prep = io_prep_writev,
89 .issue = io_write,
90 },
91 [IORING_OP_FSYNC] = {
92 .needs_file = 1,
93 .audit_skip = 1,
94 .prep = io_fsync_prep,
95 .issue = io_fsync,
96 },
97 [IORING_OP_READ_FIXED] = {
98 .needs_file = 1,
99 .unbound_nonreg_file = 1,
100 .pollin = 1,
101 .plug = 1,
102 .audit_skip = 1,
103 .ioprio = 1,
104 .iopoll = 1,
105 .iopoll_queue = 1,
106 .async_size = sizeof(struct io_async_rw),
107 .prep = io_prep_read_fixed,
108 .issue = io_read_fixed,
109 },
110 [IORING_OP_WRITE_FIXED] = {
111 .needs_file = 1,
112 .hash_reg_file = 1,
113 .unbound_nonreg_file = 1,
114 .pollout = 1,
115 .plug = 1,
116 .audit_skip = 1,
117 .ioprio = 1,
118 .iopoll = 1,
119 .iopoll_queue = 1,
120 .async_size = sizeof(struct io_async_rw),
121 .prep = io_prep_write_fixed,
122 .issue = io_write_fixed,
123 },
124 [IORING_OP_POLL_ADD] = {
125 .needs_file = 1,
126 .unbound_nonreg_file = 1,
127 .audit_skip = 1,
128 .prep = io_poll_add_prep,
129 .issue = io_poll_add,
130 },
131 [IORING_OP_POLL_REMOVE] = {
132 .audit_skip = 1,
133 .prep = io_poll_remove_prep,
134 .issue = io_poll_remove,
135 },
136 [IORING_OP_SYNC_FILE_RANGE] = {
137 .needs_file = 1,
138 .audit_skip = 1,
139 .prep = io_sfr_prep,
140 .issue = io_sync_file_range,
141 },
142 [IORING_OP_SENDMSG] = {
143 .needs_file = 1,
144 .unbound_nonreg_file = 1,
145 .pollout = 1,
146 .ioprio = 1,
147 #if defined(CONFIG_NET)
148 .async_size = sizeof(struct io_async_msghdr),
149 .prep = io_sendmsg_prep,
150 .issue = io_sendmsg,
151 #else
152 .prep = io_eopnotsupp_prep,
153 #endif
154 },
155 [IORING_OP_RECVMSG] = {
156 .needs_file = 1,
157 .unbound_nonreg_file = 1,
158 .pollin = 1,
159 .buffer_select = 1,
160 .ioprio = 1,
161 #if defined(CONFIG_NET)
162 .async_size = sizeof(struct io_async_msghdr),
163 .prep = io_recvmsg_prep,
164 .issue = io_recvmsg,
165 #else
166 .prep = io_eopnotsupp_prep,
167 #endif
168 },
169 [IORING_OP_TIMEOUT] = {
170 .audit_skip = 1,
171 .async_size = sizeof(struct io_timeout_data),
172 .prep = io_timeout_prep,
173 .issue = io_timeout,
174 },
175 [IORING_OP_TIMEOUT_REMOVE] = {
176 /* used by timeout updates' prep() */
177 .audit_skip = 1,
178 .prep = io_timeout_remove_prep,
179 .issue = io_timeout_remove,
180 },
181 [IORING_OP_ACCEPT] = {
182 .needs_file = 1,
183 .unbound_nonreg_file = 1,
184 .pollin = 1,
185 .poll_exclusive = 1,
186 .ioprio = 1, /* used for flags */
187 #if defined(CONFIG_NET)
188 .prep = io_accept_prep,
189 .issue = io_accept,
190 #else
191 .prep = io_eopnotsupp_prep,
192 #endif
193 },
194 [IORING_OP_ASYNC_CANCEL] = {
195 .audit_skip = 1,
196 .prep = io_async_cancel_prep,
197 .issue = io_async_cancel,
198 },
199 [IORING_OP_LINK_TIMEOUT] = {
200 .audit_skip = 1,
201 .async_size = sizeof(struct io_timeout_data),
202 .prep = io_link_timeout_prep,
203 .issue = io_no_issue,
204 },
205 [IORING_OP_CONNECT] = {
206 .needs_file = 1,
207 .unbound_nonreg_file = 1,
208 .pollout = 1,
209 #if defined(CONFIG_NET)
210 .async_size = sizeof(struct io_async_msghdr),
211 .prep = io_connect_prep,
212 .issue = io_connect,
213 #else
214 .prep = io_eopnotsupp_prep,
215 #endif
216 },
217 [IORING_OP_FALLOCATE] = {
218 .needs_file = 1,
219 .prep = io_fallocate_prep,
220 .issue = io_fallocate,
221 },
222 [IORING_OP_OPENAT] = {
223 .prep = io_openat_prep,
224 .issue = io_openat,
225 },
226 [IORING_OP_CLOSE] = {
227 .prep = io_close_prep,
228 .issue = io_close,
229 },
230 [IORING_OP_FILES_UPDATE] = {
231 .audit_skip = 1,
232 .iopoll = 1,
233 .prep = io_files_update_prep,
234 .issue = io_files_update,
235 },
236 [IORING_OP_STATX] = {
237 .audit_skip = 1,
238 .prep = io_statx_prep,
239 .issue = io_statx,
240 },
241 [IORING_OP_READ] = {
242 .needs_file = 1,
243 .unbound_nonreg_file = 1,
244 .pollin = 1,
245 .buffer_select = 1,
246 .plug = 1,
247 .audit_skip = 1,
248 .ioprio = 1,
249 .iopoll = 1,
250 .iopoll_queue = 1,
251 .async_size = sizeof(struct io_async_rw),
252 .prep = io_prep_read,
253 .issue = io_read,
254 },
255 [IORING_OP_WRITE] = {
256 .needs_file = 1,
257 .hash_reg_file = 1,
258 .unbound_nonreg_file = 1,
259 .pollout = 1,
260 .plug = 1,
261 .audit_skip = 1,
262 .ioprio = 1,
263 .iopoll = 1,
264 .iopoll_queue = 1,
265 .async_size = sizeof(struct io_async_rw),
266 .prep = io_prep_write,
267 .issue = io_write,
268 },
269 [IORING_OP_FADVISE] = {
270 .needs_file = 1,
271 .audit_skip = 1,
272 .prep = io_fadvise_prep,
273 .issue = io_fadvise,
274 },
275 [IORING_OP_MADVISE] = {
276 .audit_skip = 1,
277 .prep = io_madvise_prep,
278 .issue = io_madvise,
279 },
280 [IORING_OP_SEND] = {
281 .needs_file = 1,
282 .unbound_nonreg_file = 1,
283 .pollout = 1,
284 .audit_skip = 1,
285 .ioprio = 1,
286 .buffer_select = 1,
287 #if defined(CONFIG_NET)
288 .async_size = sizeof(struct io_async_msghdr),
289 .prep = io_sendmsg_prep,
290 .issue = io_send,
291 #else
292 .prep = io_eopnotsupp_prep,
293 #endif
294 },
295 [IORING_OP_RECV] = {
296 .needs_file = 1,
297 .unbound_nonreg_file = 1,
298 .pollin = 1,
299 .buffer_select = 1,
300 .audit_skip = 1,
301 .ioprio = 1,
302 #if defined(CONFIG_NET)
303 .async_size = sizeof(struct io_async_msghdr),
304 .prep = io_recvmsg_prep,
305 .issue = io_recv,
306 #else
307 .prep = io_eopnotsupp_prep,
308 #endif
309 },
310 [IORING_OP_OPENAT2] = {
311 .prep = io_openat2_prep,
312 .issue = io_openat2,
313 },
314 [IORING_OP_EPOLL_CTL] = {
315 .unbound_nonreg_file = 1,
316 .audit_skip = 1,
317 #if defined(CONFIG_EPOLL)
318 .prep = io_epoll_ctl_prep,
319 .issue = io_epoll_ctl,
320 #else
321 .prep = io_eopnotsupp_prep,
322 #endif
323 },
324 [IORING_OP_SPLICE] = {
325 .needs_file = 1,
326 .hash_reg_file = 1,
327 .unbound_nonreg_file = 1,
328 .audit_skip = 1,
329 .prep = io_splice_prep,
330 .issue = io_splice,
331 },
332 [IORING_OP_PROVIDE_BUFFERS] = {
333 .audit_skip = 1,
334 .iopoll = 1,
335 .prep = io_provide_buffers_prep,
336 .issue = io_provide_buffers,
337 },
338 [IORING_OP_REMOVE_BUFFERS] = {
339 .audit_skip = 1,
340 .iopoll = 1,
341 .prep = io_remove_buffers_prep,
342 .issue = io_remove_buffers,
343 },
344 [IORING_OP_TEE] = {
345 .needs_file = 1,
346 .hash_reg_file = 1,
347 .unbound_nonreg_file = 1,
348 .audit_skip = 1,
349 .prep = io_tee_prep,
350 .issue = io_tee,
351 },
352 [IORING_OP_SHUTDOWN] = {
353 .needs_file = 1,
354 #if defined(CONFIG_NET)
355 .prep = io_shutdown_prep,
356 .issue = io_shutdown,
357 #else
358 .prep = io_eopnotsupp_prep,
359 #endif
360 },
361 [IORING_OP_RENAMEAT] = {
362 .prep = io_renameat_prep,
363 .issue = io_renameat,
364 },
365 [IORING_OP_UNLINKAT] = {
366 .prep = io_unlinkat_prep,
367 .issue = io_unlinkat,
368 },
369 [IORING_OP_MKDIRAT] = {
370 .prep = io_mkdirat_prep,
371 .issue = io_mkdirat,
372 },
373 [IORING_OP_SYMLINKAT] = {
374 .prep = io_symlinkat_prep,
375 .issue = io_symlinkat,
376 },
377 [IORING_OP_LINKAT] = {
378 .prep = io_linkat_prep,
379 .issue = io_linkat,
380 },
381 [IORING_OP_MSG_RING] = {
382 .needs_file = 1,
383 .iopoll = 1,
384 .prep = io_msg_ring_prep,
385 .issue = io_msg_ring,
386 },
387 [IORING_OP_FSETXATTR] = {
388 .needs_file = 1,
389 .prep = io_fsetxattr_prep,
390 .issue = io_fsetxattr,
391 },
392 [IORING_OP_SETXATTR] = {
393 .prep = io_setxattr_prep,
394 .issue = io_setxattr,
395 },
396 [IORING_OP_FGETXATTR] = {
397 .needs_file = 1,
398 .prep = io_fgetxattr_prep,
399 .issue = io_fgetxattr,
400 },
401 [IORING_OP_GETXATTR] = {
402 .prep = io_getxattr_prep,
403 .issue = io_getxattr,
404 },
405 [IORING_OP_SOCKET] = {
406 .audit_skip = 1,
407 #if defined(CONFIG_NET)
408 .prep = io_socket_prep,
409 .issue = io_socket,
410 #else
411 .prep = io_eopnotsupp_prep,
412 #endif
413 },
414 [IORING_OP_URING_CMD] = {
415 .needs_file = 1,
416 .plug = 1,
417 .iopoll = 1,
418 .iopoll_queue = 1,
419 .async_size = sizeof(struct io_async_cmd),
420 .prep = io_uring_cmd_prep,
421 .issue = io_uring_cmd,
422 },
423 [IORING_OP_SEND_ZC] = {
424 .needs_file = 1,
425 .unbound_nonreg_file = 1,
426 .pollout = 1,
427 .audit_skip = 1,
428 .ioprio = 1,
429 #if defined(CONFIG_NET)
430 .async_size = sizeof(struct io_async_msghdr),
431 .prep = io_send_zc_prep,
432 .issue = io_send_zc,
433 #else
434 .prep = io_eopnotsupp_prep,
435 #endif
436 },
437 [IORING_OP_SENDMSG_ZC] = {
438 .needs_file = 1,
439 .unbound_nonreg_file = 1,
440 .pollout = 1,
441 .ioprio = 1,
442 #if defined(CONFIG_NET)
443 .async_size = sizeof(struct io_async_msghdr),
444 .prep = io_send_zc_prep,
445 .issue = io_sendmsg_zc,
446 #else
447 .prep = io_eopnotsupp_prep,
448 #endif
449 },
450 [IORING_OP_READ_MULTISHOT] = {
451 .needs_file = 1,
452 .unbound_nonreg_file = 1,
453 .pollin = 1,
454 .buffer_select = 1,
455 .audit_skip = 1,
456 .async_size = sizeof(struct io_async_rw),
457 .prep = io_read_mshot_prep,
458 .issue = io_read_mshot,
459 },
460 [IORING_OP_WAITID] = {
461 .async_size = sizeof(struct io_waitid_async),
462 .prep = io_waitid_prep,
463 .issue = io_waitid,
464 },
465 [IORING_OP_FUTEX_WAIT] = {
466 #if defined(CONFIG_FUTEX)
467 .prep = io_futex_prep,
468 .issue = io_futex_wait,
469 #else
470 .prep = io_eopnotsupp_prep,
471 #endif
472 },
473 [IORING_OP_FUTEX_WAKE] = {
474 #if defined(CONFIG_FUTEX)
475 .prep = io_futex_prep,
476 .issue = io_futex_wake,
477 #else
478 .prep = io_eopnotsupp_prep,
479 #endif
480 },
481 [IORING_OP_FUTEX_WAITV] = {
482 #if defined(CONFIG_FUTEX)
483 .prep = io_futexv_prep,
484 .issue = io_futexv_wait,
485 #else
486 .prep = io_eopnotsupp_prep,
487 #endif
488 },
489 [IORING_OP_FIXED_FD_INSTALL] = {
490 .needs_file = 1,
491 .prep = io_install_fixed_fd_prep,
492 .issue = io_install_fixed_fd,
493 },
494 [IORING_OP_FTRUNCATE] = {
495 .needs_file = 1,
496 .hash_reg_file = 1,
497 .prep = io_ftruncate_prep,
498 .issue = io_ftruncate,
499 },
500 [IORING_OP_BIND] = {
501 #if defined(CONFIG_NET)
502 .needs_file = 1,
503 .prep = io_bind_prep,
504 .issue = io_bind,
505 .async_size = sizeof(struct io_async_msghdr),
506 #else
507 .prep = io_eopnotsupp_prep,
508 #endif
509 },
510 [IORING_OP_LISTEN] = {
511 #if defined(CONFIG_NET)
512 .needs_file = 1,
513 .prep = io_listen_prep,
514 .issue = io_listen,
515 .async_size = sizeof(struct io_async_msghdr),
516 #else
517 .prep = io_eopnotsupp_prep,
518 #endif
519 },
520 [IORING_OP_RECV_ZC] = {
521 .needs_file = 1,
522 .unbound_nonreg_file = 1,
523 .pollin = 1,
524 .ioprio = 1,
525 #if defined(CONFIG_NET)
526 .prep = io_recvzc_prep,
527 .issue = io_recvzc,
528 #else
529 .prep = io_eopnotsupp_prep,
530 #endif
531 },
532 [IORING_OP_EPOLL_WAIT] = {
533 .needs_file = 1,
534 .audit_skip = 1,
535 .pollin = 1,
536 #if defined(CONFIG_EPOLL)
537 .prep = io_epoll_wait_prep,
538 .issue = io_epoll_wait,
539 #else
540 .prep = io_eopnotsupp_prep,
541 #endif
542 },
543 [IORING_OP_READV_FIXED] = {
544 .needs_file = 1,
545 .unbound_nonreg_file = 1,
546 .pollin = 1,
547 .plug = 1,
548 .audit_skip = 1,
549 .ioprio = 1,
550 .iopoll = 1,
551 .iopoll_queue = 1,
552 .vectored = 1,
553 .async_size = sizeof(struct io_async_rw),
554 .prep = io_prep_readv_fixed,
555 .issue = io_read,
556 },
557 [IORING_OP_WRITEV_FIXED] = {
558 .needs_file = 1,
559 .hash_reg_file = 1,
560 .unbound_nonreg_file = 1,
561 .pollout = 1,
562 .plug = 1,
563 .audit_skip = 1,
564 .ioprio = 1,
565 .iopoll = 1,
566 .iopoll_queue = 1,
567 .vectored = 1,
568 .async_size = sizeof(struct io_async_rw),
569 .prep = io_prep_writev_fixed,
570 .issue = io_write,
571 },
572 };
573
574 const struct io_cold_def io_cold_defs[] = {
575 [IORING_OP_NOP] = {
576 .name = "NOP",
577 },
578 [IORING_OP_READV] = {
579 .name = "READV",
580 .cleanup = io_readv_writev_cleanup,
581 .fail = io_rw_fail,
582 },
583 [IORING_OP_WRITEV] = {
584 .name = "WRITEV",
585 .cleanup = io_readv_writev_cleanup,
586 .fail = io_rw_fail,
587 },
588 [IORING_OP_FSYNC] = {
589 .name = "FSYNC",
590 },
591 [IORING_OP_READ_FIXED] = {
592 .name = "READ_FIXED",
593 .cleanup = io_readv_writev_cleanup,
594 .fail = io_rw_fail,
595 },
596 [IORING_OP_WRITE_FIXED] = {
597 .name = "WRITE_FIXED",
598 .cleanup = io_readv_writev_cleanup,
599 .fail = io_rw_fail,
600 },
601 [IORING_OP_POLL_ADD] = {
602 .name = "POLL_ADD",
603 },
604 [IORING_OP_POLL_REMOVE] = {
605 .name = "POLL_REMOVE",
606 },
607 [IORING_OP_SYNC_FILE_RANGE] = {
608 .name = "SYNC_FILE_RANGE",
609 },
610 [IORING_OP_SENDMSG] = {
611 .name = "SENDMSG",
612 #if defined(CONFIG_NET)
613 .cleanup = io_sendmsg_recvmsg_cleanup,
614 .fail = io_sendrecv_fail,
615 #endif
616 },
617 [IORING_OP_RECVMSG] = {
618 .name = "RECVMSG",
619 #if defined(CONFIG_NET)
620 .cleanup = io_sendmsg_recvmsg_cleanup,
621 .fail = io_sendrecv_fail,
622 #endif
623 },
624 [IORING_OP_TIMEOUT] = {
625 .name = "TIMEOUT",
626 },
627 [IORING_OP_TIMEOUT_REMOVE] = {
628 .name = "TIMEOUT_REMOVE",
629 },
630 [IORING_OP_ACCEPT] = {
631 .name = "ACCEPT",
632 },
633 [IORING_OP_ASYNC_CANCEL] = {
634 .name = "ASYNC_CANCEL",
635 },
636 [IORING_OP_LINK_TIMEOUT] = {
637 .name = "LINK_TIMEOUT",
638 },
639 [IORING_OP_CONNECT] = {
640 .name = "CONNECT",
641 },
642 [IORING_OP_FALLOCATE] = {
643 .name = "FALLOCATE",
644 },
645 [IORING_OP_OPENAT] = {
646 .name = "OPENAT",
647 .cleanup = io_open_cleanup,
648 },
649 [IORING_OP_CLOSE] = {
650 .name = "CLOSE",
651 },
652 [IORING_OP_FILES_UPDATE] = {
653 .name = "FILES_UPDATE",
654 },
655 [IORING_OP_STATX] = {
656 .name = "STATX",
657 .cleanup = io_statx_cleanup,
658 },
659 [IORING_OP_READ] = {
660 .name = "READ",
661 .cleanup = io_readv_writev_cleanup,
662 .fail = io_rw_fail,
663 },
664 [IORING_OP_WRITE] = {
665 .name = "WRITE",
666 .cleanup = io_readv_writev_cleanup,
667 .fail = io_rw_fail,
668 },
669 [IORING_OP_FADVISE] = {
670 .name = "FADVISE",
671 },
672 [IORING_OP_MADVISE] = {
673 .name = "MADVISE",
674 },
675 [IORING_OP_SEND] = {
676 .name = "SEND",
677 #if defined(CONFIG_NET)
678 .cleanup = io_sendmsg_recvmsg_cleanup,
679 .fail = io_sendrecv_fail,
680 #endif
681 },
682 [IORING_OP_RECV] = {
683 .name = "RECV",
684 #if defined(CONFIG_NET)
685 .cleanup = io_sendmsg_recvmsg_cleanup,
686 .fail = io_sendrecv_fail,
687 #endif
688 },
689 [IORING_OP_OPENAT2] = {
690 .name = "OPENAT2",
691 .cleanup = io_open_cleanup,
692 },
693 [IORING_OP_EPOLL_CTL] = {
694 .name = "EPOLL",
695 },
696 [IORING_OP_SPLICE] = {
697 .name = "SPLICE",
698 .cleanup = io_splice_cleanup,
699 },
700 [IORING_OP_PROVIDE_BUFFERS] = {
701 .name = "PROVIDE_BUFFERS",
702 },
703 [IORING_OP_REMOVE_BUFFERS] = {
704 .name = "REMOVE_BUFFERS",
705 },
706 [IORING_OP_TEE] = {
707 .name = "TEE",
708 .cleanup = io_splice_cleanup,
709 },
710 [IORING_OP_SHUTDOWN] = {
711 .name = "SHUTDOWN",
712 },
713 [IORING_OP_RENAMEAT] = {
714 .name = "RENAMEAT",
715 .cleanup = io_renameat_cleanup,
716 },
717 [IORING_OP_UNLINKAT] = {
718 .name = "UNLINKAT",
719 .cleanup = io_unlinkat_cleanup,
720 },
721 [IORING_OP_MKDIRAT] = {
722 .name = "MKDIRAT",
723 .cleanup = io_mkdirat_cleanup,
724 },
725 [IORING_OP_SYMLINKAT] = {
726 .name = "SYMLINKAT",
727 .cleanup = io_link_cleanup,
728 },
729 [IORING_OP_LINKAT] = {
730 .name = "LINKAT",
731 .cleanup = io_link_cleanup,
732 },
733 [IORING_OP_MSG_RING] = {
734 .name = "MSG_RING",
735 .cleanup = io_msg_ring_cleanup,
736 },
737 [IORING_OP_FSETXATTR] = {
738 .name = "FSETXATTR",
739 .cleanup = io_xattr_cleanup,
740 },
741 [IORING_OP_SETXATTR] = {
742 .name = "SETXATTR",
743 .cleanup = io_xattr_cleanup,
744 },
745 [IORING_OP_FGETXATTR] = {
746 .name = "FGETXATTR",
747 .cleanup = io_xattr_cleanup,
748 },
749 [IORING_OP_GETXATTR] = {
750 .name = "GETXATTR",
751 .cleanup = io_xattr_cleanup,
752 },
753 [IORING_OP_SOCKET] = {
754 .name = "SOCKET",
755 },
756 [IORING_OP_URING_CMD] = {
757 .name = "URING_CMD",
758 .cleanup = io_uring_cmd_cleanup,
759 },
760 [IORING_OP_SEND_ZC] = {
761 .name = "SEND_ZC",
762 #if defined(CONFIG_NET)
763 .cleanup = io_send_zc_cleanup,
764 .fail = io_sendrecv_fail,
765 #endif
766 },
767 [IORING_OP_SENDMSG_ZC] = {
768 .name = "SENDMSG_ZC",
769 #if defined(CONFIG_NET)
770 .cleanup = io_send_zc_cleanup,
771 .fail = io_sendrecv_fail,
772 #endif
773 },
774 [IORING_OP_READ_MULTISHOT] = {
775 .name = "READ_MULTISHOT",
776 .cleanup = io_readv_writev_cleanup,
777 },
778 [IORING_OP_WAITID] = {
779 .name = "WAITID",
780 },
781 [IORING_OP_FUTEX_WAIT] = {
782 .name = "FUTEX_WAIT",
783 },
784 [IORING_OP_FUTEX_WAKE] = {
785 .name = "FUTEX_WAKE",
786 },
787 [IORING_OP_FUTEX_WAITV] = {
788 .name = "FUTEX_WAITV",
789 },
790 [IORING_OP_FIXED_FD_INSTALL] = {
791 .name = "FIXED_FD_INSTALL",
792 },
793 [IORING_OP_FTRUNCATE] = {
794 .name = "FTRUNCATE",
795 },
796 [IORING_OP_BIND] = {
797 .name = "BIND",
798 },
799 [IORING_OP_LISTEN] = {
800 .name = "LISTEN",
801 },
802 [IORING_OP_RECV_ZC] = {
803 .name = "RECV_ZC",
804 },
805 [IORING_OP_EPOLL_WAIT] = {
806 .name = "EPOLL_WAIT",
807 },
808 [IORING_OP_READV_FIXED] = {
809 .name = "READV_FIXED",
810 .cleanup = io_readv_writev_cleanup,
811 .fail = io_rw_fail,
812 },
813 [IORING_OP_WRITEV_FIXED] = {
814 .name = "WRITEV_FIXED",
815 .cleanup = io_readv_writev_cleanup,
816 .fail = io_rw_fail,
817 },
818 };
819
io_uring_get_opcode(u8 opcode)820 const char *io_uring_get_opcode(u8 opcode)
821 {
822 if (opcode < IORING_OP_LAST)
823 return io_cold_defs[opcode].name;
824 return "INVALID";
825 }
826
io_uring_op_supported(u8 opcode)827 bool io_uring_op_supported(u8 opcode)
828 {
829 if (opcode < IORING_OP_LAST &&
830 io_issue_defs[opcode].prep != io_eopnotsupp_prep)
831 return true;
832 return false;
833 }
834
io_uring_optable_init(void)835 void __init io_uring_optable_init(void)
836 {
837 int i;
838
839 BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs) != IORING_OP_LAST);
840 BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST);
841
842 for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) {
843 BUG_ON(!io_issue_defs[i].prep);
844 if (io_issue_defs[i].prep != io_eopnotsupp_prep)
845 BUG_ON(!io_issue_defs[i].issue);
846 WARN_ON_ONCE(!io_cold_defs[i].name);
847 }
848 }
849