1 // SPDX-License-Identifier: GPL-2.0
2 #ifndef IOU_ZC_RX_H
3 #define IOU_ZC_RX_H
4
5 #include <linux/io_uring_types.h>
6 #include <linux/dma-buf.h>
7 #include <linux/socket.h>
8 #include <net/page_pool/types.h>
9 #include <net/net_trackers.h>
10
11 #define ZCRX_SUPPORTED_REG_FLAGS (ZCRX_REG_IMPORT | ZCRX_REG_NODEV)
12 #define ZCRX_FEATURES (ZCRX_FEATURE_RX_PAGE_SIZE)
13
14 struct io_zcrx_mem {
15 unsigned long size;
16 bool is_dmabuf;
17
18 struct page **pages;
19 unsigned long nr_folios;
20 struct sg_table page_sg_table;
21 unsigned long account_pages;
22 struct sg_table *sgt;
23
24 struct dma_buf_attachment *attach;
25 struct dma_buf *dmabuf;
26 };
27
28 struct io_zcrx_area {
29 struct net_iov_area nia;
30 struct io_zcrx_ifq *ifq;
31 atomic_t *user_refs;
32
33 bool is_mapped;
34 u16 area_id;
35
36 /* freelist */
37 spinlock_t freelist_lock ____cacheline_aligned_in_smp;
38 u32 free_count;
39 u32 *freelist;
40
41 struct io_zcrx_mem mem;
42 };
43
44 struct zcrx_rq {
45 spinlock_t lock;
46 struct io_uring *ring;
47 struct io_uring_zcrx_rqe *rqes;
48 u32 cached_head;
49 u32 nr_entries;
50 };
51
52 struct io_zcrx_ifq {
53 struct io_zcrx_area *area;
54 unsigned niov_shift;
55 struct user_struct *user;
56 struct mm_struct *mm_account;
57 bool kern_readable;
58
59 struct zcrx_rq rq ____cacheline_aligned_in_smp;
60
61 u32 if_rxq;
62 struct device *dev;
63 struct net_device *netdev;
64 netdevice_tracker netdev_tracker;
65 refcount_t refs;
66 /* counts userspace facing users like io_uring */
67 refcount_t user_refs;
68
69 /*
70 * Page pool and net configuration lock, can be taken deeper in the
71 * net stack.
72 */
73 struct mutex pp_lock;
74 struct io_mapped_region rq_region;
75 };
76
77 #if defined(CONFIG_IO_URING_ZCRX)
78 int io_zcrx_ctrl(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_arg);
79 int io_register_zcrx(struct io_ring_ctx *ctx,
80 struct io_uring_zcrx_ifq_reg __user *arg);
81 void io_unregister_zcrx(struct io_ring_ctx *ctx);
82 void io_terminate_zcrx(struct io_ring_ctx *ctx);
83 int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
84 struct socket *sock, unsigned int flags,
85 unsigned issue_flags, unsigned int *len);
86 struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx,
87 unsigned int id);
88 #else
io_register_zcrx(struct io_ring_ctx * ctx,struct io_uring_zcrx_ifq_reg __user * arg)89 static inline int io_register_zcrx(struct io_ring_ctx *ctx,
90 struct io_uring_zcrx_ifq_reg __user *arg)
91 {
92 return -EOPNOTSUPP;
93 }
io_unregister_zcrx(struct io_ring_ctx * ctx)94 static inline void io_unregister_zcrx(struct io_ring_ctx *ctx)
95 {
96 }
io_terminate_zcrx(struct io_ring_ctx * ctx)97 static inline void io_terminate_zcrx(struct io_ring_ctx *ctx)
98 {
99 }
io_zcrx_recv(struct io_kiocb * req,struct io_zcrx_ifq * ifq,struct socket * sock,unsigned int flags,unsigned issue_flags,unsigned int * len)100 static inline int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
101 struct socket *sock, unsigned int flags,
102 unsigned issue_flags, unsigned int *len)
103 {
104 return -EOPNOTSUPP;
105 }
io_zcrx_get_region(struct io_ring_ctx * ctx,unsigned int id)106 static inline struct io_mapped_region *io_zcrx_get_region(struct io_ring_ctx *ctx,
107 unsigned int id)
108 {
109 return NULL;
110 }
io_zcrx_ctrl(struct io_ring_ctx * ctx,void __user * arg,unsigned nr_arg)111 static inline int io_zcrx_ctrl(struct io_ring_ctx *ctx,
112 void __user *arg, unsigned nr_arg)
113 {
114 return -EOPNOTSUPP;
115 }
116 #endif
117
118 int io_recvzc(struct io_kiocb *req, unsigned int issue_flags);
119 int io_recvzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
120
121 #endif
122