1 // SPDX-License-Identifier: GPL-2.0
2 #ifndef IOU_ZC_RX_H
3 #define IOU_ZC_RX_H
4
5 #include <linux/io_uring_types.h>
6 #include <linux/socket.h>
7 #include <net/page_pool/types.h>
8 #include <net/net_trackers.h>
9
10 struct io_zcrx_area {
11 struct net_iov_area nia;
12 struct io_zcrx_ifq *ifq;
13 atomic_t *user_refs;
14
15 bool is_mapped;
16 u16 area_id;
17 struct page **pages;
18 unsigned long nr_folios;
19
20 /* freelist */
21 spinlock_t freelist_lock ____cacheline_aligned_in_smp;
22 u32 free_count;
23 u32 *freelist;
24 };
25
26 struct io_zcrx_ifq {
27 struct io_ring_ctx *ctx;
28 struct io_zcrx_area *area;
29
30 spinlock_t rq_lock ____cacheline_aligned_in_smp;
31 struct io_uring *rq_ring;
32 struct io_uring_zcrx_rqe *rqes;
33 u32 cached_rq_head;
34 u32 rq_entries;
35
36 u32 if_rxq;
37 struct device *dev;
38 struct net_device *netdev;
39 netdevice_tracker netdev_tracker;
40 spinlock_t lock;
41 struct mutex dma_lock;
42 };
43
44 #if defined(CONFIG_IO_URING_ZCRX)
45 int io_register_zcrx_ifq(struct io_ring_ctx *ctx,
46 struct io_uring_zcrx_ifq_reg __user *arg);
47 void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx);
48 void io_shutdown_zcrx_ifqs(struct io_ring_ctx *ctx);
49 int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
50 struct socket *sock, unsigned int flags,
51 unsigned issue_flags, unsigned int *len);
52 #else
io_register_zcrx_ifq(struct io_ring_ctx * ctx,struct io_uring_zcrx_ifq_reg __user * arg)53 static inline int io_register_zcrx_ifq(struct io_ring_ctx *ctx,
54 struct io_uring_zcrx_ifq_reg __user *arg)
55 {
56 return -EOPNOTSUPP;
57 }
io_unregister_zcrx_ifqs(struct io_ring_ctx * ctx)58 static inline void io_unregister_zcrx_ifqs(struct io_ring_ctx *ctx)
59 {
60 }
io_shutdown_zcrx_ifqs(struct io_ring_ctx * ctx)61 static inline void io_shutdown_zcrx_ifqs(struct io_ring_ctx *ctx)
62 {
63 }
io_zcrx_recv(struct io_kiocb * req,struct io_zcrx_ifq * ifq,struct socket * sock,unsigned int flags,unsigned issue_flags,unsigned int * len)64 static inline int io_zcrx_recv(struct io_kiocb *req, struct io_zcrx_ifq *ifq,
65 struct socket *sock, unsigned int flags,
66 unsigned issue_flags, unsigned int *len)
67 {
68 return -EOPNOTSUPP;
69 }
70 #endif
71
72 int io_recvzc(struct io_kiocb *req, unsigned int issue_flags);
73 int io_recvzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
74
75 #endif
76