1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef TEST_XSK_H_
3 #define TEST_XSK_H_
4
5 #include <linux/ethtool.h>
6 #include <linux/if_xdp.h>
7
8 #include "../kselftest.h"
9 #include "xsk.h"
10
11 #ifndef SO_PREFER_BUSY_POLL
12 #define SO_PREFER_BUSY_POLL 69
13 #endif
14
15 #ifndef SO_BUSY_POLL_BUDGET
16 #define SO_BUSY_POLL_BUDGET 70
17 #endif
18
19 #define TEST_PASS 0
20 #define TEST_FAILURE -1
21 #define TEST_CONTINUE 1
22 #define TEST_SKIP 2
23
24 #define DEFAULT_PKT_CNT (4 * 1024)
25 #define DEFAULT_UMEM_BUFFERS (DEFAULT_PKT_CNT / 4)
26 #define HUGEPAGE_SIZE (2 * 1024 * 1024)
27 #define MIN_PKT_SIZE 64
28 #define MAX_ETH_PKT_SIZE 1518
29 #define MAX_INTERFACE_NAME_CHARS 16
30 #define MAX_TEST_NAME_SIZE 48
31 #define SOCK_RECONF_CTR 10
32 #define USLEEP_MAX 10000
33
34 #define MAX_SKB_FRAGS_PATH "/proc/sys/net/core/max_skb_frags"
35 #define SMP_CACHE_BYTES_PATH "/sys/devices/system/cpu/cpu0/cache/index0/coherency_line_size"
36
37 extern bool opt_verbose;
38 #define print_verbose(x...) do { if (opt_verbose) ksft_print_msg(x); } while (0)
39
40
ceil_u32(u32 a,u32 b)41 static inline u32 ceil_u32(u32 a, u32 b)
42 {
43 return (a + b - 1) / b;
44 }
45
ceil_u64(u64 a,u64 b)46 static inline u64 ceil_u64(u64 a, u64 b)
47 {
48 return (a + b - 1) / b;
49 }
50
read_procfs_val(const char * path)51 static inline unsigned int read_procfs_val(const char *path)
52 {
53 unsigned int read_val = 0;
54 FILE *file;
55
56 file = fopen(path, "r");
57 if (!file) {
58 ksft_print_msg("Error opening %s\n", path);
59 return 0;
60 }
61
62 if (fscanf(file, "%u", &read_val) != 1)
63 ksft_print_msg("Error reading %s\n", path);
64
65 fclose(file);
66 return read_val;
67 }
68
69 /* Simple test */
70 enum test_mode {
71 TEST_MODE_SKB,
72 TEST_MODE_DRV,
73 TEST_MODE_ZC,
74 TEST_MODE_ALL
75 };
76
77 struct ifobject;
78 struct test_spec;
79 typedef int (*validation_func_t)(struct ifobject *ifobj);
80 typedef void *(*thread_func_t)(void *arg);
81 typedef int (*test_func_t)(struct test_spec *test);
82
83 struct xsk_socket_info {
84 struct xsk_ring_cons rx;
85 struct xsk_ring_prod tx;
86 struct xsk_umem_info *umem;
87 struct xsk_socket *xsk;
88 struct pkt_stream *pkt_stream;
89 u32 outstanding_tx;
90 u32 rxqsize;
91 u32 batch_size;
92 u8 dst_mac[ETH_ALEN];
93 u8 src_mac[ETH_ALEN];
94 bool check_consumer;
95 };
96
97 int kick_rx(struct xsk_socket_info *xsk);
98 int kick_tx(struct xsk_socket_info *xsk);
99
100 struct xsk_umem_info {
101 struct xsk_ring_prod fq;
102 struct xsk_ring_cons cq;
103 struct xsk_umem *umem;
104 u64 next_buffer;
105 u32 num_frames;
106 u32 frame_headroom;
107 void *buffer;
108 u32 frame_size;
109 u32 base_addr;
110 u32 fill_size;
111 u32 comp_size;
112 bool unaligned_mode;
113 };
114
115 struct set_hw_ring {
116 u32 default_tx;
117 u32 default_rx;
118 };
119
120 int hw_ring_size_reset(struct ifobject *ifobj);
121
122 struct ifobject {
123 char ifname[MAX_INTERFACE_NAME_CHARS];
124 struct xsk_socket_info *xsk;
125 struct xsk_socket_info *xsk_arr;
126 struct xsk_umem_info *umem;
127 thread_func_t func_ptr;
128 validation_func_t validation_func;
129 struct xsk_xdp_progs *xdp_progs;
130 struct bpf_map *xskmap;
131 struct bpf_program *xdp_prog;
132 struct ethtool_ringparam ring;
133 struct set_hw_ring set_ring;
134 enum test_mode mode;
135 int ifindex;
136 int mtu;
137 u32 bind_flags;
138 u32 xdp_zc_max_segs;
139 u32 umem_tailroom;
140 u32 max_skb_frags;
141 bool tx_on;
142 bool rx_on;
143 bool use_poll;
144 bool busy_poll;
145 bool use_fill_ring;
146 bool release_rx;
147 bool shared_umem;
148 bool use_metadata;
149 bool unaligned_supp;
150 bool multi_buff_supp;
151 bool multi_buff_zc_supp;
152 bool hw_ring_size_supp;
153 };
154 struct ifobject *ifobject_create(void);
155 void ifobject_delete(struct ifobject *ifobj);
156 int init_iface(struct ifobject *ifobj, thread_func_t func_ptr);
157
158 int xsk_configure_umem(struct ifobject *ifobj, struct xsk_umem_info *umem, void *buffer, u64 size);
159 int xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_info *umem,
160 struct ifobject *ifobject, bool shared);
161
162
163 struct pkt {
164 int offset;
165 u32 len;
166 u32 pkt_nb;
167 bool valid;
168 u16 options;
169 };
170
171 struct pkt_stream {
172 u32 nb_pkts;
173 u32 current_pkt_nb;
174 struct pkt *pkts;
175 u32 max_pkt_len;
176 u32 nb_rx_pkts;
177 u32 nb_valid_entries;
178 bool verbatim;
179 };
180
pkt_continues(u32 options)181 static inline bool pkt_continues(u32 options)
182 {
183 return options & XDP_PKT_CONTD;
184 }
185
186 struct pkt_stream *pkt_stream_generate(u32 nb_pkts, u32 pkt_len);
187 void pkt_stream_delete(struct pkt_stream *pkt_stream);
188 void pkt_stream_reset(struct pkt_stream *pkt_stream);
189 void pkt_stream_restore_default(struct test_spec *test);
190
191 struct test_spec {
192 struct ifobject *ifobj_tx;
193 struct ifobject *ifobj_rx;
194 struct pkt_stream *tx_pkt_stream_default;
195 struct pkt_stream *rx_pkt_stream_default;
196 struct bpf_program *xdp_prog_rx;
197 struct bpf_program *xdp_prog_tx;
198 struct bpf_map *xskmap_rx;
199 struct bpf_map *xskmap_tx;
200 test_func_t test_func;
201 int mtu;
202 u16 total_steps;
203 u16 current_step;
204 u16 nb_sockets;
205 bool fail;
206 bool set_ring;
207 bool adjust_tail;
208 bool adjust_tail_support;
209 enum test_mode mode;
210 char name[MAX_TEST_NAME_SIZE];
211 };
212
213 #define busy_poll_string(test) (test)->ifobj_tx->busy_poll ? "BUSY-POLL " : ""
mode_string(struct test_spec * test)214 static inline char *mode_string(struct test_spec *test)
215 {
216 switch (test->mode) {
217 case TEST_MODE_SKB:
218 return "SKB";
219 case TEST_MODE_DRV:
220 return "DRV";
221 case TEST_MODE_ZC:
222 return "ZC";
223 default:
224 return "BOGUS";
225 }
226 }
227
228 void test_init(struct test_spec *test, struct ifobject *ifobj_tx,
229 struct ifobject *ifobj_rx, enum test_mode mode,
230 const struct test_spec *test_to_run);
231
232 int testapp_adjust_tail_grow(struct test_spec *test);
233 int testapp_adjust_tail_grow_mb(struct test_spec *test);
234 int testapp_adjust_tail_shrink(struct test_spec *test);
235 int testapp_adjust_tail_shrink_mb(struct test_spec *test);
236 int testapp_aligned_inv_desc(struct test_spec *test);
237 int testapp_aligned_inv_desc_2k_frame(struct test_spec *test);
238 int testapp_aligned_inv_desc_mb(struct test_spec *test);
239 int testapp_bidirectional(struct test_spec *test);
240 int testapp_headroom(struct test_spec *test);
241 int testapp_hw_sw_max_ring_size(struct test_spec *test);
242 int testapp_hw_sw_min_ring_size(struct test_spec *test);
243 int testapp_poll_rx(struct test_spec *test);
244 int testapp_poll_rxq_tmout(struct test_spec *test);
245 int testapp_poll_tx(struct test_spec *test);
246 int testapp_poll_txq_tmout(struct test_spec *test);
247 int testapp_send_receive(struct test_spec *test);
248 int testapp_send_receive_2k_frame(struct test_spec *test);
249 int testapp_send_receive_mb(struct test_spec *test);
250 int testapp_send_receive_unaligned(struct test_spec *test);
251 int testapp_send_receive_unaligned_mb(struct test_spec *test);
252 int testapp_single_pkt(struct test_spec *test);
253 int testapp_stats_fill_empty(struct test_spec *test);
254 int testapp_stats_rx_dropped(struct test_spec *test);
255 int testapp_stats_tx_invalid_descs(struct test_spec *test);
256 int testapp_stats_rx_full(struct test_spec *test);
257 int testapp_teardown(struct test_spec *test);
258 int testapp_too_many_frags(struct test_spec *test);
259 int testapp_tx_queue_consumer(struct test_spec *test);
260 int testapp_unaligned_inv_desc(struct test_spec *test);
261 int testapp_unaligned_inv_desc_4001_frame(struct test_spec *test);
262 int testapp_unaligned_inv_desc_mb(struct test_spec *test);
263 int testapp_xdp_drop(struct test_spec *test);
264 int testapp_xdp_metadata(struct test_spec *test);
265 int testapp_xdp_metadata_mb(struct test_spec *test);
266 int testapp_xdp_prog_cleanup(struct test_spec *test);
267 int testapp_xdp_shared_umem(struct test_spec *test);
268
269 void *worker_testapp_validate_rx(void *arg);
270 void *worker_testapp_validate_tx(void *arg);
271
272 static const struct test_spec tests[] = {
273 {.name = "SEND_RECEIVE", .test_func = testapp_send_receive},
274 {.name = "SEND_RECEIVE_2K_FRAME", .test_func = testapp_send_receive_2k_frame},
275 {.name = "SEND_RECEIVE_SINGLE_PKT", .test_func = testapp_single_pkt},
276 {.name = "POLL_RX", .test_func = testapp_poll_rx},
277 {.name = "POLL_TX", .test_func = testapp_poll_tx},
278 {.name = "POLL_RXQ_FULL", .test_func = testapp_poll_rxq_tmout},
279 {.name = "POLL_TXQ_FULL", .test_func = testapp_poll_txq_tmout},
280 {.name = "ALIGNED_INV_DESC", .test_func = testapp_aligned_inv_desc},
281 {.name = "ALIGNED_INV_DESC_2K_FRAME_SIZE", .test_func = testapp_aligned_inv_desc_2k_frame},
282 {.name = "UMEM_HEADROOM", .test_func = testapp_headroom},
283 {.name = "BIDIRECTIONAL", .test_func = testapp_bidirectional},
284 {.name = "STAT_RX_DROPPED", .test_func = testapp_stats_rx_dropped},
285 {.name = "STAT_TX_INVALID", .test_func = testapp_stats_tx_invalid_descs},
286 {.name = "STAT_RX_FULL", .test_func = testapp_stats_rx_full},
287 {.name = "STAT_FILL_EMPTY", .test_func = testapp_stats_fill_empty},
288 {.name = "XDP_PROG_CLEANUP", .test_func = testapp_xdp_prog_cleanup},
289 {.name = "XDP_DROP_HALF", .test_func = testapp_xdp_drop},
290 {.name = "XDP_SHARED_UMEM", .test_func = testapp_xdp_shared_umem},
291 {.name = "XDP_METADATA_COPY", .test_func = testapp_xdp_metadata},
292 {.name = "XDP_METADATA_COPY_MULTI_BUFF", .test_func = testapp_xdp_metadata_mb},
293 {.name = "ALIGNED_INV_DESC_MULTI_BUFF", .test_func = testapp_aligned_inv_desc_mb},
294 {.name = "TOO_MANY_FRAGS", .test_func = testapp_too_many_frags},
295 {.name = "XDP_ADJUST_TAIL_SHRINK", .test_func = testapp_adjust_tail_shrink},
296 {.name = "TX_QUEUE_CONSUMER", .test_func = testapp_tx_queue_consumer},
297 };
298
299 static const struct test_spec ci_skip_tests[] = {
300 /* Flaky tests */
301 {.name = "XDP_ADJUST_TAIL_SHRINK_MULTI_BUFF", .test_func = testapp_adjust_tail_shrink_mb},
302 {.name = "XDP_ADJUST_TAIL_GROW", .test_func = testapp_adjust_tail_grow},
303 {.name = "XDP_ADJUST_TAIL_GROW_MULTI_BUFF", .test_func = testapp_adjust_tail_grow_mb},
304 {.name = "SEND_RECEIVE_9K_PACKETS", .test_func = testapp_send_receive_mb},
305 /* Tests with huge page dependency */
306 {.name = "SEND_RECEIVE_UNALIGNED", .test_func = testapp_send_receive_unaligned},
307 {.name = "UNALIGNED_INV_DESC", .test_func = testapp_unaligned_inv_desc},
308 {.name = "UNALIGNED_INV_DESC_4001_FRAME_SIZE",
309 .test_func = testapp_unaligned_inv_desc_4001_frame},
310 {.name = "SEND_RECEIVE_UNALIGNED_9K_PACKETS",
311 .test_func = testapp_send_receive_unaligned_mb},
312 {.name = "UNALIGNED_INV_DESC_MULTI_BUFF", .test_func = testapp_unaligned_inv_desc_mb},
313 /* Test with HW ring size dependency */
314 {.name = "HW_SW_MIN_RING_SIZE", .test_func = testapp_hw_sw_min_ring_size},
315 {.name = "HW_SW_MAX_RING_SIZE", .test_func = testapp_hw_sw_max_ring_size},
316 /* Too long test */
317 {.name = "TEARDOWN", .test_func = testapp_teardown},
318 };
319
320
321 #endif /* TEST_XSK_H_ */
322