1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Authors:
4 * Copyright 2001, 2002 by Robert Olsson <robert.olsson@its.uu.se>
5 * Uppsala University and
6 * Swedish University of Agricultural Sciences
7 *
8 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
9 * Ben Greear <greearb@candelatech.com>
10 * Jens Låås <jens.laas@data.slu.se>
11 *
12 * A tool for loading the network with preconfigurated packets.
13 * The tool is implemented as a linux module. Parameters are output
14 * device, delay (to hard_xmit), number of packets, and whether
15 * to use multiple SKBs or just the same one.
16 * pktgen uses the installed interface's output routine.
17 *
18 * Additional hacking by:
19 *
20 * Jens.Laas@data.slu.se
21 * Improved by ANK. 010120.
22 * Improved by ANK even more. 010212.
23 * MAC address typo fixed. 010417 --ro
24 * Integrated. 020301 --DaveM
25 * Added multiskb option 020301 --DaveM
26 * Scaling of results. 020417--sigurdur@linpro.no
27 * Significant re-work of the module:
28 * * Convert to threaded model to more efficiently be able to transmit
29 * and receive on multiple interfaces at once.
30 * * Converted many counters to __u64 to allow longer runs.
31 * * Allow configuration of ranges, like min/max IP address, MACs,
32 * and UDP-ports, for both source and destination, and can
33 * set to use a random distribution or sequentially walk the range.
34 * * Can now change most values after starting.
35 * * Place 12-byte packet in UDP payload with magic number,
36 * sequence number, and timestamp.
37 * * Add receiver code that detects dropped pkts, re-ordered pkts, and
38 * latencies (with micro-second) precision.
39 * * Add IOCTL interface to easily get counters & configuration.
40 * --Ben Greear <greearb@candelatech.com>
41 *
42 * Renamed multiskb to clone_skb and cleaned up sending core for two distinct
43 * skb modes. A clone_skb=0 mode for Ben "ranges" work and a clone_skb != 0
44 * as a "fastpath" with a configurable number of clones after alloc's.
45 * clone_skb=0 means all packets are allocated this also means ranges time
46 * stamps etc can be used. clone_skb=100 means 1 malloc is followed by 100
47 * clones.
48 *
49 * Also moved to /proc/net/pktgen/
50 * --ro
51 *
52 * Sept 10: Fixed threading/locking. Lots of bone-headed and more clever
53 * mistakes. Also merged in DaveM's patch in the -pre6 patch.
54 * --Ben Greear <greearb@candelatech.com>
55 *
56 * Integrated to 2.5.x 021029 --Lucio Maciel (luciomaciel@zipmail.com.br)
57 *
58 * 021124 Finished major redesign and rewrite for new functionality.
59 * See Documentation/networking/pktgen.rst for how to use this.
60 *
61 * The new operation:
62 * For each CPU one thread/process is created at start. This process checks
63 * for running devices in the if_list and sends packets until count is 0 it
64 * also the thread checks the thread->control which is used for inter-process
65 * communication. controlling process "posts" operations to the threads this
66 * way.
67 * The if_list is RCU protected, and the if_lock remains to protect updating
68 * of if_list, from "add_device" as it invoked from userspace (via proc write).
69 *
70 * By design there should only be *one* "controlling" process. In practice
71 * multiple write accesses gives unpredictable result. Understood by "write"
72 * to /proc gives result code that should be read be the "writer".
73 * For practical use this should be no problem.
74 *
75 * Note when adding devices to a specific CPU there good idea to also assign
76 * /proc/irq/XX/smp_affinity so TX-interrupts gets bound to the same CPU.
77 * --ro
78 *
79 * Fix refcount off by one if first packet fails, potential null deref,
80 * memleak 030710- KJP
81 *
82 * First "ranges" functionality for ipv6 030726 --ro
83 *
84 * Included flow support. 030802 ANK.
85 *
86 * Fixed unaligned access on IA-64 Grant Grundler <grundler@parisc-linux.org>
87 *
88 * Remove if fix from added Harald Welte <laforge@netfilter.org> 040419
89 * ia64 compilation fix from Aron Griffis <aron@hp.com> 040604
90 *
91 * New xmit() return, do_div and misc clean up by Stephen Hemminger
92 * <shemminger@osdl.org> 040923
93 *
94 * Randy Dunlap fixed u64 printk compiler warning
95 *
96 * Remove FCS from BW calculation. Lennert Buytenhek <buytenh@wantstofly.org>
97 * New time handling. Lennert Buytenhek <buytenh@wantstofly.org> 041213
98 *
99 * Corrections from Nikolai Malykh (nmalykh@bilim.com)
100 * Removed unused flags F_SET_SRCMAC & F_SET_SRCIP 041230
101 *
102 * interruptible_sleep_on_timeout() replaced Nishanth Aravamudan <nacc@us.ibm.com>
103 * 050103
104 *
105 * MPLS support by Steven Whitehouse <steve@chygwyn.com>
106 *
107 * 802.1Q/Q-in-Q support by Francesco Fondelli (FF) <francesco.fondelli@gmail.com>
108 *
109 * Fixed src_mac command to set source mac of packet to value specified in
110 * command by Adit Ranadive <adit.262@gmail.com>
111 */
112
113 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
114
115 #include <linux/sys.h>
116 #include <linux/types.h>
117 #include <linux/minmax.h>
118 #include <linux/module.h>
119 #include <linux/moduleparam.h>
120 #include <linux/kernel.h>
121 #include <linux/mutex.h>
122 #include <linux/sched.h>
123 #include <linux/slab.h>
124 #include <linux/vmalloc.h>
125 #include <linux/unistd.h>
126 #include <linux/string.h>
127 #include <linux/ptrace.h>
128 #include <linux/errno.h>
129 #include <linux/hex.h>
130 #include <linux/ioport.h>
131 #include <linux/interrupt.h>
132 #include <linux/capability.h>
133 #include <linux/hrtimer.h>
134 #include <linux/freezer.h>
135 #include <linux/delay.h>
136 #include <linux/timer.h>
137 #include <linux/list.h>
138 #include <linux/init.h>
139 #include <linux/skbuff.h>
140 #include <linux/netdevice.h>
141 #include <linux/inet.h>
142 #include <linux/inetdevice.h>
143 #include <linux/rtnetlink.h>
144 #include <linux/if_arp.h>
145 #include <linux/if_vlan.h>
146 #include <linux/in.h>
147 #include <linux/ip.h>
148 #include <linux/ipv6.h>
149 #include <linux/udp.h>
150 #include <linux/proc_fs.h>
151 #include <linux/seq_file.h>
152 #include <linux/wait.h>
153 #include <linux/etherdevice.h>
154 #include <linux/kthread.h>
155 #include <linux/prefetch.h>
156 #include <linux/mmzone.h>
157 #include <net/net_namespace.h>
158 #include <net/checksum.h>
159 #include <net/ipv6.h>
160 #include <net/udp.h>
161 #include <net/ip6_checksum.h>
162 #include <net/addrconf.h>
163 #include <net/xfrm.h>
164 #include <net/netns/generic.h>
165 #include <asm/byteorder.h>
166 #include <linux/rcupdate.h>
167 #include <linux/bitops.h>
168 #include <linux/io.h>
169 #include <linux/timex.h>
170 #include <linux/uaccess.h>
171 #include <asm/dma.h>
172 #include <asm/div64.h> /* do_div */
173
174 #define VERSION "2.75"
175 #define IP_NAME_SZ 32
176 #define MAX_MPLS_LABELS 16 /* This is the max label stack depth */
177 #define MPLS_STACK_BOTTOM htonl(0x00000100)
178 /* Max number of internet mix entries that can be specified in imix_weights. */
179 #define MAX_IMIX_ENTRIES 20
180 #define IMIX_PRECISION 100 /* Precision of IMIX distribution */
181
182 #define func_enter() pr_debug("entering %s\n", __func__)
183
184 #define PKT_FLAGS \
185 pf(IPV6) /* Interface in IPV6 Mode */ \
186 pf(IPSRC_RND) /* IP-Src Random */ \
187 pf(IPDST_RND) /* IP-Dst Random */ \
188 pf(TXSIZE_RND) /* Transmit size is random */ \
189 pf(UDPSRC_RND) /* UDP-Src Random */ \
190 pf(UDPDST_RND) /* UDP-Dst Random */ \
191 pf(UDPCSUM) /* Include UDP checksum */ \
192 pf(NO_TIMESTAMP) /* Don't timestamp packets (default TS) */ \
193 pf(MPLS_RND) /* Random MPLS labels */ \
194 pf(QUEUE_MAP_RND) /* queue map Random */ \
195 pf(QUEUE_MAP_CPU) /* queue map mirrors smp_processor_id() */ \
196 pf(FLOW_SEQ) /* Sequential flows */ \
197 pf(IPSEC) /* ipsec on for flows */ \
198 pf(MACSRC_RND) /* MAC-Src Random */ \
199 pf(MACDST_RND) /* MAC-Dst Random */ \
200 pf(VID_RND) /* Random VLAN ID */ \
201 pf(SVID_RND) /* Random SVLAN ID */ \
202 pf(NODE) /* Node memory alloc*/ \
203 pf(SHARED) /* Shared SKB */ \
204
205 #define pf(flag) flag##_SHIFT,
206 enum pkt_flags {
207 PKT_FLAGS
208 };
209 #undef pf
210
211 /* Device flag bits */
212 #define pf(flag) static const __u32 F_##flag = (1<<flag##_SHIFT);
213 PKT_FLAGS
214 #undef pf
215
216 #define pf(flag) __stringify(flag),
217 static char *pkt_flag_names[] = {
218 PKT_FLAGS
219 };
220 #undef pf
221
222 #define NR_PKT_FLAGS ARRAY_SIZE(pkt_flag_names)
223
224 /* Thread control flag bits */
225 #define T_STOP (1<<0) /* Stop run */
226 #define T_RUN (1<<1) /* Start run */
227 #define T_REMDEVALL (1<<2) /* Remove all devs */
228 #define T_REMDEV (1<<3) /* Remove one dev */
229
230 /* Xmit modes */
231 #define M_START_XMIT 0 /* Default normal TX */
232 #define M_NETIF_RECEIVE 1 /* Inject packets into stack */
233 #define M_QUEUE_XMIT 2 /* Inject packet into qdisc */
234
235 /* If lock -- protects updating of if_list */
236 #define if_lock(t) mutex_lock(&(t->if_lock))
237 #define if_unlock(t) mutex_unlock(&(t->if_lock))
238
239 /* Used to help with determining the pkts on receive */
240 #define PKTGEN_MAGIC 0xbe9be955
241 #define PG_PROC_DIR "pktgen"
242 #define PGCTRL "pgctrl"
243
244 #define MAX_CFLOWS 65536
245
246 #define VLAN_TAG_SIZE(x) ((x)->vlan_id == 0xffff ? 0 : 4)
247 #define SVLAN_TAG_SIZE(x) ((x)->svlan_id == 0xffff ? 0 : 4)
248
249 struct imix_pkt {
250 u64 size;
251 u64 weight;
252 u64 count_so_far;
253 };
254
255 struct flow_state {
256 __be32 cur_daddr;
257 int count;
258 #ifdef CONFIG_XFRM
259 struct xfrm_state *x;
260 #endif
261 __u32 flags;
262 };
263
264 /* flow flag bits */
265 #define F_INIT (1<<0) /* flow has been initialized */
266
267 struct pktgen_dev {
268 /*
269 * Try to keep frequent/infrequent used vars. separated.
270 */
271 struct proc_dir_entry *entry; /* proc file */
272 struct pktgen_thread *pg_thread;/* the owner */
273 struct list_head list; /* chaining in the thread's run-queue */
274 struct rcu_head rcu; /* freed by RCU */
275
276 int running; /* if false, the test will stop */
277
278 /* If min != max, then we will either do a linear iteration, or
279 * we will do a random selection from within the range.
280 */
281 __u32 flags;
282 int xmit_mode;
283 int min_pkt_size;
284 int max_pkt_size;
285 int pkt_overhead; /* overhead for MPLS, VLANs, IPSEC etc */
286 int nfrags;
287 int removal_mark; /* non-zero => the device is marked for
288 * removal by worker thread
289 */
290
291 struct page *page;
292 u64 delay; /* nano-seconds */
293
294 __u64 count; /* Default No packets to send */
295 __u64 sofar; /* How many pkts we've sent so far */
296 __u64 tx_bytes; /* How many bytes we've transmitted */
297 __u64 errors; /* Errors when trying to transmit, */
298
299 /* runtime counters relating to clone_skb */
300
301 __u32 clone_count;
302 int last_ok; /* Was last skb sent?
303 * Or a failed transmit of some sort?
304 * This will keep sequence numbers in order
305 */
306 ktime_t next_tx;
307 ktime_t started_at;
308 ktime_t stopped_at;
309 u64 idle_acc; /* nano-seconds */
310
311 __u32 seq_num;
312
313 int clone_skb; /*
314 * Use multiple SKBs during packet gen.
315 * If this number is greater than 1, then
316 * that many copies of the same packet will be
317 * sent before a new packet is allocated.
318 * If you want to send 1024 identical packets
319 * before creating a new packet,
320 * set clone_skb to 1024.
321 */
322
323 char dst_min[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */
324 char dst_max[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */
325 char src_min[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */
326 char src_max[IP_NAME_SZ]; /* IP, ie 1.2.3.4 */
327
328 struct in6_addr in6_saddr;
329 struct in6_addr in6_daddr;
330 struct in6_addr cur_in6_daddr;
331 struct in6_addr cur_in6_saddr;
332 /* For ranges */
333 struct in6_addr min_in6_daddr;
334 struct in6_addr max_in6_daddr;
335 struct in6_addr min_in6_saddr;
336 struct in6_addr max_in6_saddr;
337
338 /* If we're doing ranges, random or incremental, then this
339 * defines the min/max for those ranges.
340 */
341 __be32 saddr_min; /* inclusive, source IP address */
342 __be32 saddr_max; /* exclusive, source IP address */
343 __be32 daddr_min; /* inclusive, dest IP address */
344 __be32 daddr_max; /* exclusive, dest IP address */
345
346 __u16 udp_src_min; /* inclusive, source UDP port */
347 __u16 udp_src_max; /* exclusive, source UDP port */
348 __u16 udp_dst_min; /* inclusive, dest UDP port */
349 __u16 udp_dst_max; /* exclusive, dest UDP port */
350
351 /* DSCP + ECN */
352 __u8 tos; /* six MSB of (former) IPv4 TOS
353 * are for dscp codepoint
354 */
355 __u8 traffic_class; /* ditto for the (former) Traffic Class in IPv6
356 * (see RFC 3260, sec. 4)
357 */
358
359 /* IMIX */
360 unsigned int n_imix_entries;
361 struct imix_pkt imix_entries[MAX_IMIX_ENTRIES];
362 /* Maps 0-IMIX_PRECISION range to imix_entry based on probability*/
363 __u8 imix_distribution[IMIX_PRECISION];
364
365 /* MPLS */
366 unsigned int nr_labels; /* Depth of stack, 0 = no MPLS */
367 __be32 labels[MAX_MPLS_LABELS];
368
369 /* VLAN/SVLAN (802.1Q/Q-in-Q) */
370 __u8 vlan_p;
371 __u8 vlan_cfi;
372 __u16 vlan_id; /* 0xffff means no vlan tag */
373
374 __u8 svlan_p;
375 __u8 svlan_cfi;
376 __u16 svlan_id; /* 0xffff means no svlan tag */
377
378 __u32 src_mac_count; /* How many MACs to iterate through */
379 __u32 dst_mac_count; /* How many MACs to iterate through */
380
381 unsigned char dst_mac[ETH_ALEN];
382 unsigned char src_mac[ETH_ALEN];
383
384 __u32 cur_dst_mac_offset;
385 __u32 cur_src_mac_offset;
386 __be32 cur_saddr;
387 __be32 cur_daddr;
388 __u16 ip_id;
389 __u16 cur_udp_dst;
390 __u16 cur_udp_src;
391 __u16 cur_queue_map;
392 __u32 cur_pkt_size;
393 __u32 last_pkt_size;
394
395 __u8 hh[14];
396 /* = {
397 * 0x00, 0x80, 0xC8, 0x79, 0xB3, 0xCB,
398 *
399 * We fill in SRC address later
400 * 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
401 * 0x08, 0x00
402 * };
403 */
404 __u16 pad; /* pad out the hh struct to an even 16 bytes */
405
406 struct sk_buff *skb; /* skb we are to transmit next, used for when we
407 * are transmitting the same one multiple times
408 */
409 struct net_device *odev; /* The out-going device.
410 * Note that the device should have it's
411 * pg_info pointer pointing back to this
412 * device.
413 * Set when the user specifies the out-going
414 * device name (not when the inject is
415 * started as it used to do.)
416 */
417 netdevice_tracker dev_tracker;
418 char odevname[32];
419 struct flow_state *flows;
420 unsigned int cflows; /* Concurrent flows (config) */
421 unsigned int lflow; /* Flow length (config) */
422 unsigned int nflows; /* accumulated flows (stats) */
423 unsigned int curfl; /* current sequenced flow (state)*/
424
425 u16 queue_map_min;
426 u16 queue_map_max;
427 __u32 skb_priority; /* skb priority field */
428 unsigned int burst; /* number of duplicated packets to burst */
429 int node; /* Memory node */
430
431 #ifdef CONFIG_XFRM
432 __u8 ipsmode; /* IPSEC mode (config) */
433 __u8 ipsproto; /* IPSEC type (config) */
434 __u32 spi;
435 struct xfrm_dst xdst;
436 struct dst_ops dstops;
437 #endif
438 char result[512];
439 };
440
441 struct pktgen_hdr {
442 __be32 pgh_magic;
443 __be32 seq_num;
444 __be32 tv_sec;
445 __be32 tv_usec;
446 };
447
448
449 static unsigned int pg_net_id __read_mostly;
450
451 struct pktgen_net {
452 struct net *net;
453 struct proc_dir_entry *proc_dir;
454 struct list_head pktgen_threads;
455 bool pktgen_exiting;
456 };
457
458 struct pktgen_thread {
459 struct mutex if_lock; /* for list of devices */
460 struct list_head if_list; /* All device here */
461 struct list_head th_list;
462 struct task_struct *tsk;
463 char result[512];
464
465 /* Field for thread to receive "posted" events terminate,
466 * stop ifs etc.
467 */
468
469 u32 control;
470 int cpu;
471
472 wait_queue_head_t queue;
473 struct completion start_done;
474 struct pktgen_net *net;
475 };
476
477 #define REMOVE 1
478 #define FIND 0
479
480 static const char version[] =
481 "Packet Generator for packet performance testing. Version: " VERSION "\n";
482
483 static int pktgen_remove_device(struct pktgen_thread *t, struct pktgen_dev *i);
484 static int pktgen_add_device(struct pktgen_thread *t, const char *ifname);
485 static struct pktgen_dev *pktgen_find_dev(struct pktgen_thread *t,
486 const char *ifname, bool exact);
487 static int pktgen_device_event(struct notifier_block *, unsigned long, void *);
488 static void pktgen_run_all_threads(struct pktgen_net *pn);
489 static void pktgen_reset_all_threads(struct pktgen_net *pn);
490 static void pktgen_stop_all_threads(struct pktgen_net *pn);
491
492 static void pktgen_stop(struct pktgen_thread *t);
493 static void pktgen_clear_counters(struct pktgen_dev *pkt_dev);
494 static void fill_imix_distribution(struct pktgen_dev *pkt_dev);
495
496 /* Module parameters, defaults. */
497 static int pg_count_d __read_mostly = 1000;
498 static int pg_delay_d __read_mostly;
499 static int pg_clone_skb_d __read_mostly;
500 static int debug __read_mostly;
501
502 static DEFINE_MUTEX(pktgen_thread_lock);
503
504 static struct notifier_block pktgen_notifier_block = {
505 .notifier_call = pktgen_device_event,
506 };
507
508 /*
509 * /proc handling functions
510 *
511 */
512
pgctrl_show(struct seq_file * seq,void * v)513 static int pgctrl_show(struct seq_file *seq, void *v)
514 {
515 seq_puts(seq, version);
516 return 0;
517 }
518
pgctrl_write(struct file * file,const char __user * buf,size_t count,loff_t * ppos)519 static ssize_t pgctrl_write(struct file *file, const char __user *buf,
520 size_t count, loff_t *ppos)
521 {
522 char data[128];
523 size_t max;
524 struct pktgen_net *pn = net_generic(current->nsproxy->net_ns, pg_net_id);
525
526 if (!capable(CAP_NET_ADMIN))
527 return -EPERM;
528
529 if (count < 1)
530 return -EINVAL;
531
532 max = min(count, sizeof(data) - 1);
533 if (copy_from_user(data, buf, max))
534 return -EFAULT;
535
536 if (data[max - 1] == '\n')
537 data[max - 1] = 0; /* strip trailing '\n', terminate string */
538 else
539 data[max] = 0; /* terminate string */
540
541 if (!strcmp(data, "stop"))
542 pktgen_stop_all_threads(pn);
543 else if (!strcmp(data, "start"))
544 pktgen_run_all_threads(pn);
545 else if (!strcmp(data, "reset"))
546 pktgen_reset_all_threads(pn);
547 else
548 return -EINVAL;
549
550 return count;
551 }
552
pgctrl_open(struct inode * inode,struct file * file)553 static int pgctrl_open(struct inode *inode, struct file *file)
554 {
555 return single_open(file, pgctrl_show, pde_data(inode));
556 }
557
558 static const struct proc_ops pktgen_proc_ops = {
559 .proc_open = pgctrl_open,
560 .proc_read = seq_read,
561 .proc_lseek = seq_lseek,
562 .proc_write = pgctrl_write,
563 .proc_release = single_release,
564 };
565
pktgen_if_show(struct seq_file * seq,void * v)566 static int pktgen_if_show(struct seq_file *seq, void *v)
567 {
568 const struct pktgen_dev *pkt_dev = seq->private;
569 ktime_t stopped;
570 unsigned int i;
571 u64 idle;
572
573 seq_printf(seq,
574 "Params: count %llu min_pkt_size: %u max_pkt_size: %u\n",
575 (unsigned long long)pkt_dev->count, pkt_dev->min_pkt_size,
576 pkt_dev->max_pkt_size);
577
578 if (pkt_dev->n_imix_entries > 0) {
579 seq_puts(seq, " imix_weights: ");
580 for (i = 0; i < pkt_dev->n_imix_entries; i++) {
581 seq_printf(seq, "%llu,%llu ",
582 pkt_dev->imix_entries[i].size,
583 pkt_dev->imix_entries[i].weight);
584 }
585 seq_puts(seq, "\n");
586 }
587
588 seq_printf(seq,
589 " frags: %d delay: %llu clone_skb: %d ifname: %s\n",
590 pkt_dev->nfrags, (unsigned long long) pkt_dev->delay,
591 pkt_dev->clone_skb, pkt_dev->odevname);
592
593 seq_printf(seq, " flows: %u flowlen: %u\n", pkt_dev->cflows,
594 pkt_dev->lflow);
595
596 seq_printf(seq,
597 " queue_map_min: %u queue_map_max: %u\n",
598 pkt_dev->queue_map_min,
599 pkt_dev->queue_map_max);
600
601 if (pkt_dev->skb_priority)
602 seq_printf(seq, " skb_priority: %u\n",
603 pkt_dev->skb_priority);
604
605 if (pkt_dev->flags & F_IPV6) {
606 seq_printf(seq,
607 " saddr: %pI6c min_saddr: %pI6c max_saddr: %pI6c\n"
608 " daddr: %pI6c min_daddr: %pI6c max_daddr: %pI6c\n",
609 &pkt_dev->in6_saddr,
610 &pkt_dev->min_in6_saddr, &pkt_dev->max_in6_saddr,
611 &pkt_dev->in6_daddr,
612 &pkt_dev->min_in6_daddr, &pkt_dev->max_in6_daddr);
613 } else {
614 seq_printf(seq,
615 " dst_min: %s dst_max: %s\n",
616 pkt_dev->dst_min, pkt_dev->dst_max);
617 seq_printf(seq,
618 " src_min: %s src_max: %s\n",
619 pkt_dev->src_min, pkt_dev->src_max);
620 }
621
622 seq_puts(seq, " src_mac: ");
623
624 seq_printf(seq, "%pM ",
625 is_zero_ether_addr(pkt_dev->src_mac) ?
626 pkt_dev->odev->dev_addr : pkt_dev->src_mac);
627
628 seq_puts(seq, "dst_mac: ");
629 seq_printf(seq, "%pM\n", pkt_dev->dst_mac);
630
631 seq_printf(seq,
632 " udp_src_min: %d udp_src_max: %d udp_dst_min: %d udp_dst_max: %d\n",
633 pkt_dev->udp_src_min, pkt_dev->udp_src_max,
634 pkt_dev->udp_dst_min, pkt_dev->udp_dst_max);
635
636 seq_printf(seq,
637 " src_mac_count: %d dst_mac_count: %d\n",
638 pkt_dev->src_mac_count, pkt_dev->dst_mac_count);
639
640 if (pkt_dev->nr_labels) {
641 seq_puts(seq, " mpls: ");
642 for (i = 0; i < pkt_dev->nr_labels; i++)
643 seq_printf(seq, "%08x%s", ntohl(pkt_dev->labels[i]),
644 i == pkt_dev->nr_labels-1 ? "\n" : ", ");
645 }
646
647 if (pkt_dev->vlan_id != 0xffff)
648 seq_printf(seq, " vlan_id: %u vlan_p: %u vlan_cfi: %u\n",
649 pkt_dev->vlan_id, pkt_dev->vlan_p,
650 pkt_dev->vlan_cfi);
651
652 if (pkt_dev->svlan_id != 0xffff)
653 seq_printf(seq, " svlan_id: %u vlan_p: %u vlan_cfi: %u\n",
654 pkt_dev->svlan_id, pkt_dev->svlan_p,
655 pkt_dev->svlan_cfi);
656
657 if (pkt_dev->tos)
658 seq_printf(seq, " tos: 0x%02x\n", pkt_dev->tos);
659
660 if (pkt_dev->traffic_class)
661 seq_printf(seq, " traffic_class: 0x%02x\n", pkt_dev->traffic_class);
662
663 if (pkt_dev->burst > 1)
664 seq_printf(seq, " burst: %d\n", pkt_dev->burst);
665
666 if (pkt_dev->node >= 0)
667 seq_printf(seq, " node: %d\n", pkt_dev->node);
668
669 if (pkt_dev->xmit_mode == M_NETIF_RECEIVE)
670 seq_puts(seq, " xmit_mode: netif_receive\n");
671 else if (pkt_dev->xmit_mode == M_QUEUE_XMIT)
672 seq_puts(seq, " xmit_mode: xmit_queue\n");
673
674 seq_puts(seq, " Flags: ");
675
676 for (i = 0; i < NR_PKT_FLAGS; i++) {
677 if (i == FLOW_SEQ_SHIFT)
678 if (!pkt_dev->cflows)
679 continue;
680
681 if (pkt_dev->flags & (1 << i)) {
682 seq_printf(seq, "%s ", pkt_flag_names[i]);
683 #ifdef CONFIG_XFRM
684 if (i == IPSEC_SHIFT && pkt_dev->spi)
685 seq_printf(seq, "spi:%u ", pkt_dev->spi);
686 #endif
687 } else if (i == FLOW_SEQ_SHIFT) {
688 seq_puts(seq, "FLOW_RND ");
689 }
690 }
691
692 seq_puts(seq, "\n");
693
694 /* not really stopped, more like last-running-at */
695 stopped = pkt_dev->running ? ktime_get() : pkt_dev->stopped_at;
696 idle = pkt_dev->idle_acc;
697 do_div(idle, NSEC_PER_USEC);
698
699 seq_printf(seq,
700 "Current:\n pkts-sofar: %llu errors: %llu\n",
701 (unsigned long long)pkt_dev->sofar,
702 (unsigned long long)pkt_dev->errors);
703
704 if (pkt_dev->n_imix_entries > 0) {
705 int i;
706
707 seq_puts(seq, " imix_size_counts: ");
708 for (i = 0; i < pkt_dev->n_imix_entries; i++) {
709 seq_printf(seq, "%llu,%llu ",
710 pkt_dev->imix_entries[i].size,
711 pkt_dev->imix_entries[i].count_so_far);
712 }
713 seq_puts(seq, "\n");
714 }
715
716 seq_printf(seq,
717 " started: %lluus stopped: %lluus idle: %lluus\n",
718 (unsigned long long) ktime_to_us(pkt_dev->started_at),
719 (unsigned long long) ktime_to_us(stopped),
720 (unsigned long long) idle);
721
722 seq_printf(seq,
723 " seq_num: %d cur_dst_mac_offset: %d cur_src_mac_offset: %d\n",
724 pkt_dev->seq_num, pkt_dev->cur_dst_mac_offset,
725 pkt_dev->cur_src_mac_offset);
726
727 if (pkt_dev->flags & F_IPV6) {
728 seq_printf(seq, " cur_saddr: %pI6c cur_daddr: %pI6c\n",
729 &pkt_dev->cur_in6_saddr,
730 &pkt_dev->cur_in6_daddr);
731 } else
732 seq_printf(seq, " cur_saddr: %pI4 cur_daddr: %pI4\n",
733 &pkt_dev->cur_saddr, &pkt_dev->cur_daddr);
734
735 seq_printf(seq, " cur_udp_dst: %d cur_udp_src: %d\n",
736 pkt_dev->cur_udp_dst, pkt_dev->cur_udp_src);
737
738 seq_printf(seq, " cur_queue_map: %u\n", pkt_dev->cur_queue_map);
739
740 seq_printf(seq, " flows: %u\n", pkt_dev->nflows);
741
742 if (pkt_dev->result[0])
743 seq_printf(seq, "Result: %s\n", pkt_dev->result);
744 else
745 seq_puts(seq, "Result: Idle\n");
746
747 return 0;
748 }
749
750
hex32_arg(const char __user * user_buffer,size_t maxlen,__u32 * num)751 static ssize_t hex32_arg(const char __user *user_buffer, size_t maxlen,
752 __u32 *num)
753 {
754 size_t i = 0;
755
756 *num = 0;
757
758 for (; i < maxlen; i++) {
759 int value;
760 char c;
761
762 if (get_user(c, &user_buffer[i]))
763 return -EFAULT;
764 value = hex_to_bin(c);
765 if (value >= 0) {
766 *num <<= 4;
767 *num |= value;
768 } else {
769 break;
770 }
771 }
772 return i;
773 }
774
count_trail_chars(const char __user * user_buffer,size_t maxlen)775 static ssize_t count_trail_chars(const char __user *user_buffer, size_t maxlen)
776 {
777 size_t i;
778
779 for (i = 0; i < maxlen; i++) {
780 char c;
781
782 if (get_user(c, &user_buffer[i]))
783 return -EFAULT;
784 switch (c) {
785 case '\"':
786 case '\n':
787 case '\r':
788 case '\t':
789 case ' ':
790 case '=':
791 break;
792 default:
793 goto done;
794 }
795 }
796 done:
797 return i;
798 }
799
num_arg(const char __user * user_buffer,size_t maxlen,unsigned long * num)800 static ssize_t num_arg(const char __user *user_buffer, size_t maxlen,
801 unsigned long *num)
802 {
803 size_t i;
804 *num = 0;
805
806 for (i = 0; i < maxlen; i++) {
807 char c;
808
809 if (get_user(c, &user_buffer[i]))
810 return -EFAULT;
811 if ((c >= '0') && (c <= '9')) {
812 *num *= 10;
813 *num += c - '0';
814 } else
815 break;
816 }
817 return i;
818 }
819
strn_len(const char __user * user_buffer,size_t maxlen)820 static ssize_t strn_len(const char __user *user_buffer, size_t maxlen)
821 {
822 size_t i;
823
824 for (i = 0; i < maxlen; i++) {
825 char c;
826
827 if (get_user(c, &user_buffer[i]))
828 return -EFAULT;
829 switch (c) {
830 case '\"':
831 case '\n':
832 case '\r':
833 case '\t':
834 case ' ':
835 case '=':
836 goto done_str;
837 default:
838 break;
839 }
840 }
841 done_str:
842 return i;
843 }
844
845 /* Parses imix entries from user buffer.
846 * The user buffer should consist of imix entries separated by spaces
847 * where each entry consists of size and weight delimited by commas.
848 * "size1,weight_1 size2,weight_2 ... size_n,weight_n" for example.
849 */
get_imix_entries(const char __user * buffer,size_t maxlen,struct pktgen_dev * pkt_dev)850 static ssize_t get_imix_entries(const char __user *buffer,
851 size_t maxlen,
852 struct pktgen_dev *pkt_dev)
853 {
854 size_t i = 0, max;
855 ssize_t len;
856 char c;
857
858 pkt_dev->n_imix_entries = 0;
859
860 do {
861 unsigned long weight;
862 unsigned long size;
863
864 if (pkt_dev->n_imix_entries >= MAX_IMIX_ENTRIES)
865 return -E2BIG;
866
867 if (i >= maxlen)
868 return -EINVAL;
869
870 max = min(10, maxlen - i);
871 len = num_arg(&buffer[i], max, &size);
872 if (len < 0)
873 return len;
874 i += len;
875 if (i >= maxlen)
876 return -EINVAL;
877 if (get_user(c, &buffer[i]))
878 return -EFAULT;
879 /* Check for comma between size_i and weight_i */
880 if (c != ',')
881 return -EINVAL;
882 i++;
883 if (i >= maxlen)
884 return -EINVAL;
885
886 if (size < 14 + 20 + 8)
887 size = 14 + 20 + 8;
888
889 max = min(10, maxlen - i);
890 len = num_arg(&buffer[i], max, &weight);
891 if (len < 0)
892 return len;
893 if (weight <= 0)
894 return -EINVAL;
895
896 pkt_dev->imix_entries[pkt_dev->n_imix_entries].size = size;
897 pkt_dev->imix_entries[pkt_dev->n_imix_entries].weight = weight;
898
899 i += len;
900 pkt_dev->n_imix_entries++;
901
902 if (i >= maxlen)
903 break;
904 if (get_user(c, &buffer[i]))
905 return -EFAULT;
906 i++;
907 } while (c == ' ');
908
909 return i;
910 }
911
get_labels(const char __user * buffer,size_t maxlen,struct pktgen_dev * pkt_dev)912 static ssize_t get_labels(const char __user *buffer,
913 size_t maxlen, struct pktgen_dev *pkt_dev)
914 {
915 unsigned int n = 0;
916 size_t i = 0, max;
917 ssize_t len;
918 char c;
919
920 pkt_dev->nr_labels = 0;
921 do {
922 __u32 tmp;
923
924 if (n >= MAX_MPLS_LABELS)
925 return -E2BIG;
926
927 if (i >= maxlen)
928 return -EINVAL;
929
930 max = min(8, maxlen - i);
931 len = hex32_arg(&buffer[i], max, &tmp);
932 if (len < 0)
933 return len;
934
935 /* return empty list in case of invalid input or zero value */
936 if (len == 0 || tmp == 0)
937 return maxlen;
938
939 pkt_dev->labels[n] = htonl(tmp);
940 if (pkt_dev->labels[n] & MPLS_STACK_BOTTOM)
941 pkt_dev->flags |= F_MPLS_RND;
942 i += len;
943 n++;
944 if (i >= maxlen)
945 break;
946 if (get_user(c, &buffer[i]))
947 return -EFAULT;
948 i++;
949 } while (c == ',');
950
951 pkt_dev->nr_labels = n;
952 return i;
953 }
954
pktgen_read_flag(const char * f,bool * disable)955 static __u32 pktgen_read_flag(const char *f, bool *disable)
956 {
957 __u32 i;
958
959 if (f[0] == '!') {
960 *disable = true;
961 f++;
962 }
963
964 for (i = 0; i < NR_PKT_FLAGS; i++) {
965 if (!IS_ENABLED(CONFIG_XFRM) && i == IPSEC_SHIFT)
966 continue;
967
968 /* allow only disabling ipv6 flag */
969 if (!*disable && i == IPV6_SHIFT)
970 continue;
971
972 if (strcmp(f, pkt_flag_names[i]) == 0)
973 return 1 << i;
974 }
975
976 if (strcmp(f, "FLOW_RND") == 0) {
977 *disable = !*disable;
978 return F_FLOW_SEQ;
979 }
980
981 return 0;
982 }
983
pktgen_if_write(struct file * file,const char __user * user_buffer,size_t count,loff_t * offset)984 static ssize_t pktgen_if_write(struct file *file,
985 const char __user *user_buffer, size_t count,
986 loff_t *offset)
987 {
988 struct seq_file *seq = file->private_data;
989 struct pktgen_dev *pkt_dev = seq->private;
990 size_t i, max;
991 ssize_t len;
992 char name[16], valstr[32];
993 unsigned long value = 0;
994 char *pg_result = NULL;
995 char buf[128];
996
997 pg_result = &(pkt_dev->result[0]);
998
999 if (count < 1) {
1000 pr_warn("wrong command format\n");
1001 return -EINVAL;
1002 }
1003
1004 max = count;
1005 len = count_trail_chars(user_buffer, max);
1006 if (len < 0) {
1007 pr_warn("illegal format\n");
1008 return len;
1009 }
1010 i = len;
1011
1012 /* Read variable name */
1013 max = min(sizeof(name) - 1, count - i);
1014 len = strn_len(&user_buffer[i], max);
1015 if (len < 0)
1016 return len;
1017
1018 memset(name, 0, sizeof(name));
1019 if (copy_from_user(name, &user_buffer[i], len))
1020 return -EFAULT;
1021 i += len;
1022
1023 max = count - i;
1024 len = count_trail_chars(&user_buffer[i], max);
1025 if (len < 0)
1026 return len;
1027
1028 i += len;
1029
1030 if (debug) {
1031 size_t copy = min_t(size_t, count + 1, 1024);
1032 char *tp = strndup_user(user_buffer, copy);
1033
1034 if (IS_ERR(tp))
1035 return PTR_ERR(tp);
1036
1037 pr_debug("%s,%zu buffer -:%s:-\n", name, count, tp);
1038 kfree(tp);
1039 }
1040
1041 if (!strcmp(name, "min_pkt_size")) {
1042 max = min(10, count - i);
1043 len = num_arg(&user_buffer[i], max, &value);
1044 if (len < 0)
1045 return len;
1046
1047 if (value < 14 + 20 + 8)
1048 value = 14 + 20 + 8;
1049 if (value != pkt_dev->min_pkt_size) {
1050 pkt_dev->min_pkt_size = value;
1051 pkt_dev->cur_pkt_size = value;
1052 }
1053 sprintf(pg_result, "OK: min_pkt_size=%d",
1054 pkt_dev->min_pkt_size);
1055 return count;
1056 }
1057
1058 if (!strcmp(name, "max_pkt_size")) {
1059 max = min(10, count - i);
1060 len = num_arg(&user_buffer[i], max, &value);
1061 if (len < 0)
1062 return len;
1063
1064 if (value < 14 + 20 + 8)
1065 value = 14 + 20 + 8;
1066 if (value != pkt_dev->max_pkt_size) {
1067 pkt_dev->max_pkt_size = value;
1068 pkt_dev->cur_pkt_size = value;
1069 }
1070 sprintf(pg_result, "OK: max_pkt_size=%d",
1071 pkt_dev->max_pkt_size);
1072 return count;
1073 }
1074
1075 /* Shortcut for min = max */
1076
1077 if (!strcmp(name, "pkt_size")) {
1078 max = min(10, count - i);
1079 len = num_arg(&user_buffer[i], max, &value);
1080 if (len < 0)
1081 return len;
1082
1083 if (value < 14 + 20 + 8)
1084 value = 14 + 20 + 8;
1085 if (value != pkt_dev->min_pkt_size) {
1086 pkt_dev->min_pkt_size = value;
1087 pkt_dev->max_pkt_size = value;
1088 pkt_dev->cur_pkt_size = value;
1089 }
1090 sprintf(pg_result, "OK: pkt_size=%d", pkt_dev->min_pkt_size);
1091 return count;
1092 }
1093
1094 if (!strcmp(name, "imix_weights")) {
1095 if (pkt_dev->clone_skb > 0)
1096 return -EINVAL;
1097
1098 max = count - i;
1099 len = get_imix_entries(&user_buffer[i], max, pkt_dev);
1100 if (len < 0)
1101 return len;
1102
1103 fill_imix_distribution(pkt_dev);
1104
1105 return count;
1106 }
1107
1108 if (!strcmp(name, "debug")) {
1109 max = min(10, count - i);
1110 len = num_arg(&user_buffer[i], max, &value);
1111 if (len < 0)
1112 return len;
1113
1114 debug = value;
1115 sprintf(pg_result, "OK: debug=%u", debug);
1116 return count;
1117 }
1118
1119 if (!strcmp(name, "frags")) {
1120 max = min(10, count - i);
1121 len = num_arg(&user_buffer[i], max, &value);
1122 if (len < 0)
1123 return len;
1124
1125 pkt_dev->nfrags = value;
1126 sprintf(pg_result, "OK: frags=%d", pkt_dev->nfrags);
1127 return count;
1128 }
1129 if (!strcmp(name, "delay")) {
1130 max = min(10, count - i);
1131 len = num_arg(&user_buffer[i], max, &value);
1132 if (len < 0)
1133 return len;
1134
1135 if (value == 0x7FFFFFFF)
1136 pkt_dev->delay = ULLONG_MAX;
1137 else
1138 pkt_dev->delay = (u64)value;
1139
1140 sprintf(pg_result, "OK: delay=%llu",
1141 (unsigned long long) pkt_dev->delay);
1142 return count;
1143 }
1144 if (!strcmp(name, "rate")) {
1145 max = min(10, count - i);
1146 len = num_arg(&user_buffer[i], max, &value);
1147 if (len < 0)
1148 return len;
1149
1150 if (!value)
1151 return -EINVAL;
1152 pkt_dev->delay = pkt_dev->min_pkt_size*8*NSEC_PER_USEC/value;
1153 if (debug)
1154 pr_info("Delay set at: %llu ns\n", pkt_dev->delay);
1155
1156 sprintf(pg_result, "OK: rate=%lu", value);
1157 return count;
1158 }
1159 if (!strcmp(name, "ratep")) {
1160 max = min(10, count - i);
1161 len = num_arg(&user_buffer[i], max, &value);
1162 if (len < 0)
1163 return len;
1164
1165 if (!value)
1166 return -EINVAL;
1167 pkt_dev->delay = NSEC_PER_SEC/value;
1168 if (debug)
1169 pr_info("Delay set at: %llu ns\n", pkt_dev->delay);
1170
1171 sprintf(pg_result, "OK: rate=%lu", value);
1172 return count;
1173 }
1174 if (!strcmp(name, "udp_src_min")) {
1175 max = min(10, count - i);
1176 len = num_arg(&user_buffer[i], max, &value);
1177 if (len < 0)
1178 return len;
1179
1180 if (value != pkt_dev->udp_src_min) {
1181 pkt_dev->udp_src_min = value;
1182 pkt_dev->cur_udp_src = value;
1183 }
1184 sprintf(pg_result, "OK: udp_src_min=%u", pkt_dev->udp_src_min);
1185 return count;
1186 }
1187 if (!strcmp(name, "udp_dst_min")) {
1188 max = min(10, count - i);
1189 len = num_arg(&user_buffer[i], max, &value);
1190 if (len < 0)
1191 return len;
1192
1193 if (value != pkt_dev->udp_dst_min) {
1194 pkt_dev->udp_dst_min = value;
1195 pkt_dev->cur_udp_dst = value;
1196 }
1197 sprintf(pg_result, "OK: udp_dst_min=%u", pkt_dev->udp_dst_min);
1198 return count;
1199 }
1200 if (!strcmp(name, "udp_src_max")) {
1201 max = min(10, count - i);
1202 len = num_arg(&user_buffer[i], max, &value);
1203 if (len < 0)
1204 return len;
1205
1206 if (value != pkt_dev->udp_src_max) {
1207 pkt_dev->udp_src_max = value;
1208 pkt_dev->cur_udp_src = value;
1209 }
1210 sprintf(pg_result, "OK: udp_src_max=%u", pkt_dev->udp_src_max);
1211 return count;
1212 }
1213 if (!strcmp(name, "udp_dst_max")) {
1214 max = min(10, count - i);
1215 len = num_arg(&user_buffer[i], max, &value);
1216 if (len < 0)
1217 return len;
1218
1219 if (value != pkt_dev->udp_dst_max) {
1220 pkt_dev->udp_dst_max = value;
1221 pkt_dev->cur_udp_dst = value;
1222 }
1223 sprintf(pg_result, "OK: udp_dst_max=%u", pkt_dev->udp_dst_max);
1224 return count;
1225 }
1226 if (!strcmp(name, "clone_skb")) {
1227 max = min(10, count - i);
1228 len = num_arg(&user_buffer[i], max, &value);
1229 if (len < 0)
1230 return len;
1231 /* clone_skb is not supported for netif_receive xmit_mode and
1232 * IMIX mode.
1233 */
1234 if ((value > 0) &&
1235 ((pkt_dev->xmit_mode == M_NETIF_RECEIVE) ||
1236 !(pkt_dev->odev->priv_flags & IFF_TX_SKB_SHARING)))
1237 return -EOPNOTSUPP;
1238 if (value > 0 && (pkt_dev->n_imix_entries > 0 ||
1239 !(pkt_dev->flags & F_SHARED)))
1240 return -EINVAL;
1241
1242 pkt_dev->clone_skb = value;
1243
1244 sprintf(pg_result, "OK: clone_skb=%d", pkt_dev->clone_skb);
1245 return count;
1246 }
1247 if (!strcmp(name, "count")) {
1248 max = min(10, count - i);
1249 len = num_arg(&user_buffer[i], max, &value);
1250 if (len < 0)
1251 return len;
1252
1253 pkt_dev->count = value;
1254 sprintf(pg_result, "OK: count=%llu",
1255 (unsigned long long)pkt_dev->count);
1256 return count;
1257 }
1258 if (!strcmp(name, "src_mac_count")) {
1259 max = min(10, count - i);
1260 len = num_arg(&user_buffer[i], max, &value);
1261 if (len < 0)
1262 return len;
1263
1264 if (pkt_dev->src_mac_count != value) {
1265 pkt_dev->src_mac_count = value;
1266 pkt_dev->cur_src_mac_offset = 0;
1267 }
1268 sprintf(pg_result, "OK: src_mac_count=%d",
1269 pkt_dev->src_mac_count);
1270 return count;
1271 }
1272 if (!strcmp(name, "dst_mac_count")) {
1273 max = min(10, count - i);
1274 len = num_arg(&user_buffer[i], max, &value);
1275 if (len < 0)
1276 return len;
1277
1278 if (pkt_dev->dst_mac_count != value) {
1279 pkt_dev->dst_mac_count = value;
1280 pkt_dev->cur_dst_mac_offset = 0;
1281 }
1282 sprintf(pg_result, "OK: dst_mac_count=%d",
1283 pkt_dev->dst_mac_count);
1284 return count;
1285 }
1286 if (!strcmp(name, "burst")) {
1287 max = min(10, count - i);
1288 len = num_arg(&user_buffer[i], max, &value);
1289 if (len < 0)
1290 return len;
1291
1292 if ((value > 1) &&
1293 ((pkt_dev->xmit_mode == M_QUEUE_XMIT) ||
1294 ((pkt_dev->xmit_mode == M_START_XMIT) &&
1295 (!(pkt_dev->odev->priv_flags & IFF_TX_SKB_SHARING)))))
1296 return -EOPNOTSUPP;
1297
1298 if (value > 1 && !(pkt_dev->flags & F_SHARED))
1299 return -EINVAL;
1300
1301 pkt_dev->burst = value < 1 ? 1 : value;
1302 sprintf(pg_result, "OK: burst=%u", pkt_dev->burst);
1303 return count;
1304 }
1305 if (!strcmp(name, "node")) {
1306 max = min(10, count - i);
1307 len = num_arg(&user_buffer[i], max, &value);
1308 if (len < 0)
1309 return len;
1310
1311 if (node_possible(value)) {
1312 pkt_dev->node = value;
1313 sprintf(pg_result, "OK: node=%d", pkt_dev->node);
1314 if (pkt_dev->page) {
1315 put_page(pkt_dev->page);
1316 pkt_dev->page = NULL;
1317 }
1318 } else {
1319 sprintf(pg_result, "ERROR: node not possible");
1320 }
1321 return count;
1322 }
1323 if (!strcmp(name, "xmit_mode")) {
1324 char f[32];
1325
1326 max = min(sizeof(f) - 1, count - i);
1327 len = strn_len(&user_buffer[i], max);
1328 if (len < 0)
1329 return len;
1330
1331 memset(f, 0, sizeof(f));
1332 if (copy_from_user(f, &user_buffer[i], len))
1333 return -EFAULT;
1334
1335 if (strcmp(f, "start_xmit") == 0) {
1336 pkt_dev->xmit_mode = M_START_XMIT;
1337 } else if (strcmp(f, "netif_receive") == 0) {
1338 /* clone_skb set earlier, not supported in this mode */
1339 if (pkt_dev->clone_skb > 0)
1340 return -EOPNOTSUPP;
1341
1342 pkt_dev->xmit_mode = M_NETIF_RECEIVE;
1343
1344 /* make sure new packet is allocated every time
1345 * pktgen_xmit() is called
1346 */
1347 pkt_dev->last_ok = 1;
1348 } else if (strcmp(f, "queue_xmit") == 0) {
1349 pkt_dev->xmit_mode = M_QUEUE_XMIT;
1350 pkt_dev->last_ok = 1;
1351 } else {
1352 sprintf(pg_result,
1353 "xmit_mode -:%s:- unknown\nAvailable modes: %s",
1354 f, "start_xmit, netif_receive\n");
1355 return count;
1356 }
1357 sprintf(pg_result, "OK: xmit_mode=%s", f);
1358 return count;
1359 }
1360 if (!strcmp(name, "flag")) {
1361 bool disable = false;
1362 __u32 flag;
1363 char f[32];
1364 char *end;
1365
1366 max = min(sizeof(f) - 1, count - i);
1367 len = strn_len(&user_buffer[i], max);
1368 if (len < 0)
1369 return len;
1370
1371 memset(f, 0, 32);
1372 if (copy_from_user(f, &user_buffer[i], len))
1373 return -EFAULT;
1374
1375 flag = pktgen_read_flag(f, &disable);
1376 if (flag) {
1377 if (disable) {
1378 /* If "clone_skb", or "burst" parameters are
1379 * configured, it means that the skb still
1380 * needs to be referenced by the pktgen, so
1381 * the skb must be shared.
1382 */
1383 if (flag == F_SHARED && (pkt_dev->clone_skb ||
1384 pkt_dev->burst > 1))
1385 return -EINVAL;
1386 pkt_dev->flags &= ~flag;
1387 } else {
1388 pkt_dev->flags |= flag;
1389 }
1390
1391 sprintf(pg_result, "OK: flags=0x%x", pkt_dev->flags);
1392 return count;
1393 }
1394
1395 /* Unknown flag */
1396 end = pkt_dev->result + sizeof(pkt_dev->result);
1397 pg_result += sprintf(pg_result,
1398 "Flag -:%s:- unknown\n"
1399 "Available flags, (prepend ! to un-set flag):\n", f);
1400
1401 for (int n = 0; n < NR_PKT_FLAGS && pg_result < end; n++) {
1402 if (!IS_ENABLED(CONFIG_XFRM) && n == IPSEC_SHIFT)
1403 continue;
1404 pg_result += snprintf(pg_result, end - pg_result,
1405 "%s, ", pkt_flag_names[n]);
1406 }
1407 if (!WARN_ON_ONCE(pg_result >= end)) {
1408 /* Remove the comma and whitespace at the end */
1409 *(pg_result - 2) = '\0';
1410 }
1411
1412 return count;
1413 }
1414 if (!strcmp(name, "dst_min") || !strcmp(name, "dst")) {
1415 max = min(sizeof(pkt_dev->dst_min) - 1, count - i);
1416 len = strn_len(&user_buffer[i], max);
1417 if (len < 0)
1418 return len;
1419
1420 if (copy_from_user(buf, &user_buffer[i], len))
1421 return -EFAULT;
1422 buf[len] = 0;
1423 if (strcmp(buf, pkt_dev->dst_min) != 0) {
1424 strscpy_pad(pkt_dev->dst_min, buf);
1425 pkt_dev->daddr_min = in_aton(pkt_dev->dst_min);
1426 pkt_dev->cur_daddr = pkt_dev->daddr_min;
1427 }
1428 if (debug)
1429 pr_debug("dst_min set to: %s\n", pkt_dev->dst_min);
1430
1431 sprintf(pg_result, "OK: dst_min=%s", pkt_dev->dst_min);
1432 return count;
1433 }
1434 if (!strcmp(name, "dst_max")) {
1435 max = min(sizeof(pkt_dev->dst_max) - 1, count - i);
1436 len = strn_len(&user_buffer[i], max);
1437 if (len < 0)
1438 return len;
1439
1440 if (copy_from_user(buf, &user_buffer[i], len))
1441 return -EFAULT;
1442 buf[len] = 0;
1443 if (strcmp(buf, pkt_dev->dst_max) != 0) {
1444 strscpy_pad(pkt_dev->dst_max, buf);
1445 pkt_dev->daddr_max = in_aton(pkt_dev->dst_max);
1446 pkt_dev->cur_daddr = pkt_dev->daddr_max;
1447 }
1448 if (debug)
1449 pr_debug("dst_max set to: %s\n", pkt_dev->dst_max);
1450
1451 sprintf(pg_result, "OK: dst_max=%s", pkt_dev->dst_max);
1452 return count;
1453 }
1454 if (!strcmp(name, "dst6")) {
1455 max = min(sizeof(buf) - 1, count - i);
1456 len = strn_len(&user_buffer[i], max);
1457 if (len < 0)
1458 return len;
1459
1460 pkt_dev->flags |= F_IPV6;
1461
1462 if (copy_from_user(buf, &user_buffer[i], len))
1463 return -EFAULT;
1464 buf[len] = 0;
1465
1466 in6_pton(buf, -1, pkt_dev->in6_daddr.s6_addr, -1, NULL);
1467 snprintf(buf, sizeof(buf), "%pI6c", &pkt_dev->in6_daddr);
1468
1469 pkt_dev->cur_in6_daddr = pkt_dev->in6_daddr;
1470
1471 if (debug)
1472 pr_debug("dst6 set to: %s\n", buf);
1473
1474 sprintf(pg_result, "OK: dst6=%s", buf);
1475 return count;
1476 }
1477 if (!strcmp(name, "dst6_min")) {
1478 max = min(sizeof(buf) - 1, count - i);
1479 len = strn_len(&user_buffer[i], max);
1480 if (len < 0)
1481 return len;
1482
1483 pkt_dev->flags |= F_IPV6;
1484
1485 if (copy_from_user(buf, &user_buffer[i], len))
1486 return -EFAULT;
1487 buf[len] = 0;
1488
1489 in6_pton(buf, -1, pkt_dev->min_in6_daddr.s6_addr, -1, NULL);
1490 snprintf(buf, sizeof(buf), "%pI6c", &pkt_dev->min_in6_daddr);
1491
1492 pkt_dev->cur_in6_daddr = pkt_dev->min_in6_daddr;
1493 if (debug)
1494 pr_debug("dst6_min set to: %s\n", buf);
1495
1496 sprintf(pg_result, "OK: dst6_min=%s", buf);
1497 return count;
1498 }
1499 if (!strcmp(name, "dst6_max")) {
1500 max = min(sizeof(buf) - 1, count - i);
1501 len = strn_len(&user_buffer[i], max);
1502 if (len < 0)
1503 return len;
1504
1505 pkt_dev->flags |= F_IPV6;
1506
1507 if (copy_from_user(buf, &user_buffer[i], len))
1508 return -EFAULT;
1509 buf[len] = 0;
1510
1511 in6_pton(buf, -1, pkt_dev->max_in6_daddr.s6_addr, -1, NULL);
1512 snprintf(buf, sizeof(buf), "%pI6c", &pkt_dev->max_in6_daddr);
1513
1514 if (debug)
1515 pr_debug("dst6_max set to: %s\n", buf);
1516
1517 sprintf(pg_result, "OK: dst6_max=%s", buf);
1518 return count;
1519 }
1520 if (!strcmp(name, "src6")) {
1521 max = min(sizeof(buf) - 1, count - i);
1522 len = strn_len(&user_buffer[i], max);
1523 if (len < 0)
1524 return len;
1525
1526 pkt_dev->flags |= F_IPV6;
1527
1528 if (copy_from_user(buf, &user_buffer[i], len))
1529 return -EFAULT;
1530 buf[len] = 0;
1531
1532 in6_pton(buf, -1, pkt_dev->in6_saddr.s6_addr, -1, NULL);
1533 snprintf(buf, sizeof(buf), "%pI6c", &pkt_dev->in6_saddr);
1534
1535 pkt_dev->cur_in6_saddr = pkt_dev->in6_saddr;
1536
1537 if (debug)
1538 pr_debug("src6 set to: %s\n", buf);
1539
1540 sprintf(pg_result, "OK: src6=%s", buf);
1541 return count;
1542 }
1543 if (!strcmp(name, "src_min")) {
1544 max = min(sizeof(pkt_dev->src_min) - 1, count - i);
1545 len = strn_len(&user_buffer[i], max);
1546 if (len < 0)
1547 return len;
1548
1549 if (copy_from_user(buf, &user_buffer[i], len))
1550 return -EFAULT;
1551 buf[len] = 0;
1552 if (strcmp(buf, pkt_dev->src_min) != 0) {
1553 strscpy_pad(pkt_dev->src_min, buf);
1554 pkt_dev->saddr_min = in_aton(pkt_dev->src_min);
1555 pkt_dev->cur_saddr = pkt_dev->saddr_min;
1556 }
1557 if (debug)
1558 pr_debug("src_min set to: %s\n", pkt_dev->src_min);
1559
1560 sprintf(pg_result, "OK: src_min=%s", pkt_dev->src_min);
1561 return count;
1562 }
1563 if (!strcmp(name, "src_max")) {
1564 max = min(sizeof(pkt_dev->src_max) - 1, count - i);
1565 len = strn_len(&user_buffer[i], max);
1566 if (len < 0)
1567 return len;
1568
1569 if (copy_from_user(buf, &user_buffer[i], len))
1570 return -EFAULT;
1571 buf[len] = 0;
1572 if (strcmp(buf, pkt_dev->src_max) != 0) {
1573 strscpy_pad(pkt_dev->src_max, buf);
1574 pkt_dev->saddr_max = in_aton(pkt_dev->src_max);
1575 pkt_dev->cur_saddr = pkt_dev->saddr_max;
1576 }
1577 if (debug)
1578 pr_debug("src_max set to: %s\n", pkt_dev->src_max);
1579
1580 sprintf(pg_result, "OK: src_max=%s", pkt_dev->src_max);
1581 return count;
1582 }
1583 if (!strcmp(name, "dst_mac")) {
1584 max = min(sizeof(valstr) - 1, count - i);
1585 len = strn_len(&user_buffer[i], max);
1586 if (len < 0)
1587 return len;
1588
1589 memset(valstr, 0, sizeof(valstr));
1590 if (copy_from_user(valstr, &user_buffer[i], len))
1591 return -EFAULT;
1592
1593 if (!mac_pton(valstr, pkt_dev->dst_mac))
1594 return -EINVAL;
1595 /* Set up Dest MAC */
1596 ether_addr_copy(&pkt_dev->hh[0], pkt_dev->dst_mac);
1597
1598 sprintf(pg_result, "OK: dstmac %pM", pkt_dev->dst_mac);
1599 return count;
1600 }
1601 if (!strcmp(name, "src_mac")) {
1602 max = min(sizeof(valstr) - 1, count - i);
1603 len = strn_len(&user_buffer[i], max);
1604 if (len < 0)
1605 return len;
1606
1607 memset(valstr, 0, sizeof(valstr));
1608 if (copy_from_user(valstr, &user_buffer[i], len))
1609 return -EFAULT;
1610
1611 if (!mac_pton(valstr, pkt_dev->src_mac))
1612 return -EINVAL;
1613 /* Set up Src MAC */
1614 ether_addr_copy(&pkt_dev->hh[6], pkt_dev->src_mac);
1615
1616 sprintf(pg_result, "OK: srcmac %pM", pkt_dev->src_mac);
1617 return count;
1618 }
1619
1620 if (!strcmp(name, "clear_counters")) {
1621 pktgen_clear_counters(pkt_dev);
1622 sprintf(pg_result, "OK: Clearing counters.\n");
1623 return count;
1624 }
1625
1626 if (!strcmp(name, "flows")) {
1627 max = min(10, count - i);
1628 len = num_arg(&user_buffer[i], max, &value);
1629 if (len < 0)
1630 return len;
1631
1632 if (value > MAX_CFLOWS)
1633 value = MAX_CFLOWS;
1634
1635 pkt_dev->cflows = value;
1636 sprintf(pg_result, "OK: flows=%u", pkt_dev->cflows);
1637 return count;
1638 }
1639 #ifdef CONFIG_XFRM
1640 if (!strcmp(name, "spi")) {
1641 max = min(10, count - i);
1642 len = num_arg(&user_buffer[i], max, &value);
1643 if (len < 0)
1644 return len;
1645
1646 pkt_dev->spi = value;
1647 sprintf(pg_result, "OK: spi=%u", pkt_dev->spi);
1648 return count;
1649 }
1650 #endif
1651 if (!strcmp(name, "flowlen")) {
1652 max = min(10, count - i);
1653 len = num_arg(&user_buffer[i], max, &value);
1654 if (len < 0)
1655 return len;
1656
1657 pkt_dev->lflow = value;
1658 sprintf(pg_result, "OK: flowlen=%u", pkt_dev->lflow);
1659 return count;
1660 }
1661
1662 if (!strcmp(name, "queue_map_min")) {
1663 max = min(5, count - i);
1664 len = num_arg(&user_buffer[i], max, &value);
1665 if (len < 0)
1666 return len;
1667
1668 pkt_dev->queue_map_min = value;
1669 sprintf(pg_result, "OK: queue_map_min=%u", pkt_dev->queue_map_min);
1670 return count;
1671 }
1672
1673 if (!strcmp(name, "queue_map_max")) {
1674 max = min(5, count - i);
1675 len = num_arg(&user_buffer[i], max, &value);
1676 if (len < 0)
1677 return len;
1678
1679 pkt_dev->queue_map_max = value;
1680 sprintf(pg_result, "OK: queue_map_max=%u", pkt_dev->queue_map_max);
1681 return count;
1682 }
1683
1684 if (!strcmp(name, "mpls")) {
1685 unsigned int n, cnt;
1686
1687 max = count - i;
1688 len = get_labels(&user_buffer[i], max, pkt_dev);
1689 if (len < 0)
1690 return len;
1691
1692 cnt = sprintf(pg_result, "OK: mpls=");
1693 for (n = 0; n < pkt_dev->nr_labels; n++)
1694 cnt += sprintf(pg_result + cnt,
1695 "%08x%s", ntohl(pkt_dev->labels[n]),
1696 n == pkt_dev->nr_labels-1 ? "" : ",");
1697
1698 if (pkt_dev->nr_labels && pkt_dev->vlan_id != 0xffff) {
1699 pkt_dev->vlan_id = 0xffff; /* turn off VLAN/SVLAN */
1700 pkt_dev->svlan_id = 0xffff;
1701
1702 if (debug)
1703 pr_debug("VLAN/SVLAN auto turned off\n");
1704 }
1705 return count;
1706 }
1707
1708 if (!strcmp(name, "vlan_id")) {
1709 max = min(4, count - i);
1710 len = num_arg(&user_buffer[i], max, &value);
1711 if (len < 0)
1712 return len;
1713
1714 if (value <= 4095) {
1715 pkt_dev->vlan_id = value; /* turn on VLAN */
1716
1717 if (debug)
1718 pr_debug("VLAN turned on\n");
1719
1720 if (debug && pkt_dev->nr_labels)
1721 pr_debug("MPLS auto turned off\n");
1722
1723 pkt_dev->nr_labels = 0; /* turn off MPLS */
1724 sprintf(pg_result, "OK: vlan_id=%u", pkt_dev->vlan_id);
1725 } else {
1726 pkt_dev->vlan_id = 0xffff; /* turn off VLAN/SVLAN */
1727 pkt_dev->svlan_id = 0xffff;
1728
1729 if (debug)
1730 pr_debug("VLAN/SVLAN turned off\n");
1731 }
1732 return count;
1733 }
1734
1735 if (!strcmp(name, "vlan_p")) {
1736 max = min(1, count - i);
1737 len = num_arg(&user_buffer[i], max, &value);
1738 if (len < 0)
1739 return len;
1740
1741 if ((value <= 7) && (pkt_dev->vlan_id != 0xffff)) {
1742 pkt_dev->vlan_p = value;
1743 sprintf(pg_result, "OK: vlan_p=%u", pkt_dev->vlan_p);
1744 } else {
1745 sprintf(pg_result, "ERROR: vlan_p must be 0-7");
1746 }
1747 return count;
1748 }
1749
1750 if (!strcmp(name, "vlan_cfi")) {
1751 max = min(1, count - i);
1752 len = num_arg(&user_buffer[i], max, &value);
1753 if (len < 0)
1754 return len;
1755
1756 if ((value <= 1) && (pkt_dev->vlan_id != 0xffff)) {
1757 pkt_dev->vlan_cfi = value;
1758 sprintf(pg_result, "OK: vlan_cfi=%u", pkt_dev->vlan_cfi);
1759 } else {
1760 sprintf(pg_result, "ERROR: vlan_cfi must be 0-1");
1761 }
1762 return count;
1763 }
1764
1765 if (!strcmp(name, "svlan_id")) {
1766 max = min(4, count - i);
1767 len = num_arg(&user_buffer[i], max, &value);
1768 if (len < 0)
1769 return len;
1770
1771 if ((value <= 4095) && ((pkt_dev->vlan_id != 0xffff))) {
1772 pkt_dev->svlan_id = value; /* turn on SVLAN */
1773
1774 if (debug)
1775 pr_debug("SVLAN turned on\n");
1776
1777 if (debug && pkt_dev->nr_labels)
1778 pr_debug("MPLS auto turned off\n");
1779
1780 pkt_dev->nr_labels = 0; /* turn off MPLS */
1781 sprintf(pg_result, "OK: svlan_id=%u", pkt_dev->svlan_id);
1782 } else {
1783 pkt_dev->vlan_id = 0xffff; /* turn off VLAN/SVLAN */
1784 pkt_dev->svlan_id = 0xffff;
1785
1786 if (debug)
1787 pr_debug("VLAN/SVLAN turned off\n");
1788 }
1789 return count;
1790 }
1791
1792 if (!strcmp(name, "svlan_p")) {
1793 max = min(1, count - i);
1794 len = num_arg(&user_buffer[i], max, &value);
1795 if (len < 0)
1796 return len;
1797
1798 if ((value <= 7) && (pkt_dev->svlan_id != 0xffff)) {
1799 pkt_dev->svlan_p = value;
1800 sprintf(pg_result, "OK: svlan_p=%u", pkt_dev->svlan_p);
1801 } else {
1802 sprintf(pg_result, "ERROR: svlan_p must be 0-7");
1803 }
1804 return count;
1805 }
1806
1807 if (!strcmp(name, "svlan_cfi")) {
1808 max = min(1, count - i);
1809 len = num_arg(&user_buffer[i], max, &value);
1810 if (len < 0)
1811 return len;
1812
1813 if ((value <= 1) && (pkt_dev->svlan_id != 0xffff)) {
1814 pkt_dev->svlan_cfi = value;
1815 sprintf(pg_result, "OK: svlan_cfi=%u", pkt_dev->svlan_cfi);
1816 } else {
1817 sprintf(pg_result, "ERROR: svlan_cfi must be 0-1");
1818 }
1819 return count;
1820 }
1821
1822 if (!strcmp(name, "tos")) {
1823 __u32 tmp_value;
1824
1825 max = min(2, count - i);
1826 len = hex32_arg(&user_buffer[i], max, &tmp_value);
1827 if (len < 0)
1828 return len;
1829
1830 if (len == 2) {
1831 pkt_dev->tos = tmp_value;
1832 sprintf(pg_result, "OK: tos=0x%02x", pkt_dev->tos);
1833 } else {
1834 sprintf(pg_result, "ERROR: tos must be 00-ff");
1835 }
1836 return count;
1837 }
1838
1839 if (!strcmp(name, "traffic_class")) {
1840 __u32 tmp_value;
1841
1842 max = min(2, count - i);
1843 len = hex32_arg(&user_buffer[i], max, &tmp_value);
1844 if (len < 0)
1845 return len;
1846
1847 if (len == 2) {
1848 pkt_dev->traffic_class = tmp_value;
1849 sprintf(pg_result, "OK: traffic_class=0x%02x", pkt_dev->traffic_class);
1850 } else {
1851 sprintf(pg_result, "ERROR: traffic_class must be 00-ff");
1852 }
1853 return count;
1854 }
1855
1856 if (!strcmp(name, "skb_priority")) {
1857 max = min(9, count - i);
1858 len = num_arg(&user_buffer[i], max, &value);
1859 if (len < 0)
1860 return len;
1861
1862 pkt_dev->skb_priority = value;
1863 sprintf(pg_result, "OK: skb_priority=%i",
1864 pkt_dev->skb_priority);
1865 return count;
1866 }
1867
1868 sprintf(pkt_dev->result, "No such parameter \"%s\"", name);
1869 return -EINVAL;
1870 }
1871
pktgen_if_open(struct inode * inode,struct file * file)1872 static int pktgen_if_open(struct inode *inode, struct file *file)
1873 {
1874 return single_open(file, pktgen_if_show, pde_data(inode));
1875 }
1876
1877 static const struct proc_ops pktgen_if_proc_ops = {
1878 .proc_open = pktgen_if_open,
1879 .proc_read = seq_read,
1880 .proc_lseek = seq_lseek,
1881 .proc_write = pktgen_if_write,
1882 .proc_release = single_release,
1883 };
1884
pktgen_thread_show(struct seq_file * seq,void * v)1885 static int pktgen_thread_show(struct seq_file *seq, void *v)
1886 {
1887 struct pktgen_thread *t = seq->private;
1888 const struct pktgen_dev *pkt_dev;
1889
1890 BUG_ON(!t);
1891
1892 seq_puts(seq, "Running: ");
1893
1894 rcu_read_lock();
1895 list_for_each_entry_rcu(pkt_dev, &t->if_list, list)
1896 if (pkt_dev->running)
1897 seq_printf(seq, "%s ", pkt_dev->odevname);
1898
1899 seq_puts(seq, "\nStopped: ");
1900
1901 list_for_each_entry_rcu(pkt_dev, &t->if_list, list)
1902 if (!pkt_dev->running)
1903 seq_printf(seq, "%s ", pkt_dev->odevname);
1904
1905 if (t->result[0])
1906 seq_printf(seq, "\nResult: %s\n", t->result);
1907 else
1908 seq_puts(seq, "\nResult: NA\n");
1909
1910 rcu_read_unlock();
1911
1912 return 0;
1913 }
1914
pktgen_thread_write(struct file * file,const char __user * user_buffer,size_t count,loff_t * offset)1915 static ssize_t pktgen_thread_write(struct file *file,
1916 const char __user *user_buffer,
1917 size_t count, loff_t *offset)
1918 {
1919 struct seq_file *seq = file->private_data;
1920 struct pktgen_thread *t = seq->private;
1921 size_t i, max;
1922 ssize_t len, ret;
1923 char name[40];
1924 char *pg_result;
1925
1926 if (count < 1) {
1927 // sprintf(pg_result, "Wrong command format");
1928 return -EINVAL;
1929 }
1930
1931 max = count;
1932 len = count_trail_chars(user_buffer, max);
1933 if (len < 0)
1934 return len;
1935
1936 i = len;
1937
1938 /* Read variable name */
1939 max = min(sizeof(name) - 1, count - i);
1940 len = strn_len(&user_buffer[i], max);
1941 if (len < 0)
1942 return len;
1943
1944 memset(name, 0, sizeof(name));
1945 if (copy_from_user(name, &user_buffer[i], len))
1946 return -EFAULT;
1947 i += len;
1948
1949 max = count - i;
1950 len = count_trail_chars(&user_buffer[i], max);
1951 if (len < 0)
1952 return len;
1953
1954 i += len;
1955
1956 if (debug)
1957 pr_debug("t=%s, count=%lu\n", name, (unsigned long)count);
1958
1959 if (!t) {
1960 pr_err("ERROR: No thread\n");
1961 ret = -EINVAL;
1962 goto out;
1963 }
1964
1965 pg_result = &(t->result[0]);
1966
1967 if (!strcmp(name, "add_device")) {
1968 char f[32];
1969
1970 memset(f, 0, 32);
1971 max = min(sizeof(f) - 1, count - i);
1972 len = strn_len(&user_buffer[i], max);
1973 if (len < 0) {
1974 ret = len;
1975 goto out;
1976 }
1977 if (copy_from_user(f, &user_buffer[i], len))
1978 return -EFAULT;
1979
1980 mutex_lock(&pktgen_thread_lock);
1981 ret = pktgen_add_device(t, f);
1982 mutex_unlock(&pktgen_thread_lock);
1983 if (!ret) {
1984 ret = count;
1985 sprintf(pg_result, "OK: add_device=%s", f);
1986 } else
1987 sprintf(pg_result, "ERROR: can not add device %s", f);
1988 goto out;
1989 }
1990
1991 if (!strcmp(name, "rem_device_all")) {
1992 mutex_lock(&pktgen_thread_lock);
1993 t->control |= T_REMDEVALL;
1994 mutex_unlock(&pktgen_thread_lock);
1995 schedule_timeout_interruptible(msecs_to_jiffies(125)); /* Propagate thread->control */
1996 ret = count;
1997 sprintf(pg_result, "OK: rem_device_all");
1998 goto out;
1999 }
2000
2001 if (!strcmp(name, "max_before_softirq")) {
2002 sprintf(pg_result, "OK: Note! max_before_softirq is obsoleted -- Do not use");
2003 ret = count;
2004 goto out;
2005 }
2006
2007 ret = -EINVAL;
2008 out:
2009 return ret;
2010 }
2011
pktgen_thread_open(struct inode * inode,struct file * file)2012 static int pktgen_thread_open(struct inode *inode, struct file *file)
2013 {
2014 return single_open(file, pktgen_thread_show, pde_data(inode));
2015 }
2016
2017 static const struct proc_ops pktgen_thread_proc_ops = {
2018 .proc_open = pktgen_thread_open,
2019 .proc_read = seq_read,
2020 .proc_lseek = seq_lseek,
2021 .proc_write = pktgen_thread_write,
2022 .proc_release = single_release,
2023 };
2024
2025 /* Think find or remove for NN */
__pktgen_NN_threads(const struct pktgen_net * pn,const char * ifname,int remove)2026 static struct pktgen_dev *__pktgen_NN_threads(const struct pktgen_net *pn,
2027 const char *ifname, int remove)
2028 {
2029 struct pktgen_thread *t;
2030 struct pktgen_dev *pkt_dev = NULL;
2031 bool exact = (remove == FIND);
2032
2033 list_for_each_entry(t, &pn->pktgen_threads, th_list) {
2034 pkt_dev = pktgen_find_dev(t, ifname, exact);
2035 if (pkt_dev) {
2036 if (remove) {
2037 pkt_dev->removal_mark = 1;
2038 t->control |= T_REMDEV;
2039 }
2040 break;
2041 }
2042 }
2043 return pkt_dev;
2044 }
2045
2046 /*
2047 * mark a device for removal
2048 */
pktgen_mark_device(const struct pktgen_net * pn,const char * ifname)2049 static void pktgen_mark_device(const struct pktgen_net *pn, const char *ifname)
2050 {
2051 struct pktgen_dev *pkt_dev = NULL;
2052 const int max_tries = 10, msec_per_try = 125;
2053 int i = 0;
2054
2055 mutex_lock(&pktgen_thread_lock);
2056 pr_debug("%s: marking %s for removal\n", __func__, ifname);
2057
2058 while (1) {
2059
2060 pkt_dev = __pktgen_NN_threads(pn, ifname, REMOVE);
2061 if (pkt_dev == NULL)
2062 break; /* success */
2063
2064 mutex_unlock(&pktgen_thread_lock);
2065 pr_debug("%s: waiting for %s to disappear....\n",
2066 __func__, ifname);
2067 schedule_timeout_interruptible(msecs_to_jiffies(msec_per_try));
2068 mutex_lock(&pktgen_thread_lock);
2069
2070 if (++i >= max_tries) {
2071 pr_err("%s: timed out after waiting %d msec for device %s to be removed\n",
2072 __func__, msec_per_try * i, ifname);
2073 break;
2074 }
2075
2076 }
2077
2078 mutex_unlock(&pktgen_thread_lock);
2079 }
2080
pktgen_change_name(const struct pktgen_net * pn,struct net_device * dev)2081 static void pktgen_change_name(const struct pktgen_net *pn, struct net_device *dev)
2082 {
2083 struct pktgen_thread *t;
2084
2085 mutex_lock(&pktgen_thread_lock);
2086
2087 list_for_each_entry(t, &pn->pktgen_threads, th_list) {
2088 struct pktgen_dev *pkt_dev;
2089
2090 if_lock(t);
2091 list_for_each_entry(pkt_dev, &t->if_list, list) {
2092 if (pkt_dev->odev != dev)
2093 continue;
2094
2095 proc_remove(pkt_dev->entry);
2096
2097 pkt_dev->entry = proc_create_data(dev->name, 0600,
2098 pn->proc_dir,
2099 &pktgen_if_proc_ops,
2100 pkt_dev);
2101 if (!pkt_dev->entry)
2102 pr_err("can't move proc entry for '%s'\n",
2103 dev->name);
2104 break;
2105 }
2106 if_unlock(t);
2107 }
2108 mutex_unlock(&pktgen_thread_lock);
2109 }
2110
pktgen_device_event(struct notifier_block * unused,unsigned long event,void * ptr)2111 static int pktgen_device_event(struct notifier_block *unused,
2112 unsigned long event, void *ptr)
2113 {
2114 struct net_device *dev = netdev_notifier_info_to_dev(ptr);
2115 struct pktgen_net *pn = net_generic(dev_net(dev), pg_net_id);
2116
2117 if (pn->pktgen_exiting)
2118 return NOTIFY_DONE;
2119
2120 /* It is OK that we do not hold the group lock right now,
2121 * as we run under the RTNL lock.
2122 */
2123
2124 switch (event) {
2125 case NETDEV_CHANGENAME:
2126 pktgen_change_name(pn, dev);
2127 break;
2128
2129 case NETDEV_UNREGISTER:
2130 pktgen_mark_device(pn, dev->name);
2131 break;
2132 }
2133
2134 return NOTIFY_DONE;
2135 }
2136
pktgen_dev_get_by_name(const struct pktgen_net * pn,struct pktgen_dev * pkt_dev,const char * ifname)2137 static struct net_device *pktgen_dev_get_by_name(const struct pktgen_net *pn,
2138 struct pktgen_dev *pkt_dev,
2139 const char *ifname)
2140 {
2141 char b[IFNAMSIZ+5];
2142 int i;
2143
2144 for (i = 0; ifname[i] != '@'; i++) {
2145 if (i == IFNAMSIZ)
2146 break;
2147
2148 b[i] = ifname[i];
2149 }
2150 b[i] = 0;
2151
2152 return dev_get_by_name(pn->net, b);
2153 }
2154
2155
2156 /* Associate pktgen_dev with a device. */
2157
pktgen_setup_dev(const struct pktgen_net * pn,struct pktgen_dev * pkt_dev,const char * ifname)2158 static int pktgen_setup_dev(const struct pktgen_net *pn,
2159 struct pktgen_dev *pkt_dev, const char *ifname)
2160 {
2161 struct net_device *odev;
2162 int err;
2163
2164 /* Clean old setups */
2165 if (pkt_dev->odev) {
2166 netdev_put(pkt_dev->odev, &pkt_dev->dev_tracker);
2167 pkt_dev->odev = NULL;
2168 }
2169
2170 odev = pktgen_dev_get_by_name(pn, pkt_dev, ifname);
2171 if (!odev) {
2172 pr_err("no such netdevice: \"%s\"\n", ifname);
2173 return -ENODEV;
2174 }
2175
2176 if (odev->type != ARPHRD_ETHER && odev->type != ARPHRD_LOOPBACK) {
2177 pr_err("not an ethernet or loopback device: \"%s\"\n", ifname);
2178 err = -EINVAL;
2179 } else if (!netif_running(odev)) {
2180 pr_err("device is down: \"%s\"\n", ifname);
2181 err = -ENETDOWN;
2182 } else {
2183 pkt_dev->odev = odev;
2184 netdev_tracker_alloc(odev, &pkt_dev->dev_tracker, GFP_KERNEL);
2185 return 0;
2186 }
2187
2188 dev_put(odev);
2189 return err;
2190 }
2191
2192 /* Read pkt_dev from the interface and set up internal pktgen_dev
2193 * structure to have the right information to create/send packets
2194 */
pktgen_setup_inject(struct pktgen_dev * pkt_dev)2195 static void pktgen_setup_inject(struct pktgen_dev *pkt_dev)
2196 {
2197 int ntxq;
2198
2199 if (!pkt_dev->odev) {
2200 pr_err("ERROR: pkt_dev->odev == NULL in setup_inject\n");
2201 sprintf(pkt_dev->result,
2202 "ERROR: pkt_dev->odev == NULL in setup_inject.\n");
2203 return;
2204 }
2205
2206 /* make sure that we don't pick a non-existing transmit queue */
2207 ntxq = pkt_dev->odev->real_num_tx_queues;
2208
2209 if (ntxq <= pkt_dev->queue_map_min) {
2210 pr_warn("WARNING: Requested queue_map_min (zero-based) (%d) exceeds valid range [0 - %d] for (%d) queues on %s, resetting\n",
2211 pkt_dev->queue_map_min, (ntxq ?: 1) - 1, ntxq,
2212 pkt_dev->odevname);
2213 pkt_dev->queue_map_min = (ntxq ?: 1) - 1;
2214 }
2215 if (pkt_dev->queue_map_max >= ntxq) {
2216 pr_warn("WARNING: Requested queue_map_max (zero-based) (%d) exceeds valid range [0 - %d] for (%d) queues on %s, resetting\n",
2217 pkt_dev->queue_map_max, (ntxq ?: 1) - 1, ntxq,
2218 pkt_dev->odevname);
2219 pkt_dev->queue_map_max = (ntxq ?: 1) - 1;
2220 }
2221
2222 /* Default to the interface's mac if not explicitly set. */
2223
2224 if (is_zero_ether_addr(pkt_dev->src_mac))
2225 ether_addr_copy(&(pkt_dev->hh[6]), pkt_dev->odev->dev_addr);
2226
2227 /* Set up Dest MAC */
2228 ether_addr_copy(&(pkt_dev->hh[0]), pkt_dev->dst_mac);
2229
2230 if (pkt_dev->flags & F_IPV6) {
2231 int i, set = 0, err = 1;
2232 struct inet6_dev *idev;
2233
2234 if (pkt_dev->min_pkt_size == 0) {
2235 pkt_dev->min_pkt_size = 14 + sizeof(struct ipv6hdr)
2236 + sizeof(struct udphdr)
2237 + sizeof(struct pktgen_hdr)
2238 + pkt_dev->pkt_overhead;
2239 }
2240
2241 for (i = 0; i < sizeof(struct in6_addr); i++)
2242 if (pkt_dev->cur_in6_saddr.s6_addr[i]) {
2243 set = 1;
2244 break;
2245 }
2246
2247 if (!set) {
2248
2249 /*
2250 * Use linklevel address if unconfigured.
2251 *
2252 * use ipv6_get_lladdr if/when it's get exported
2253 */
2254
2255 rcu_read_lock();
2256 idev = __in6_dev_get(pkt_dev->odev);
2257 if (idev) {
2258 struct inet6_ifaddr *ifp;
2259
2260 read_lock_bh(&idev->lock);
2261 list_for_each_entry(ifp, &idev->addr_list, if_list) {
2262 if ((ifp->scope & IFA_LINK) &&
2263 !(ifp->flags & IFA_F_TENTATIVE)) {
2264 pkt_dev->cur_in6_saddr = ifp->addr;
2265 err = 0;
2266 break;
2267 }
2268 }
2269 read_unlock_bh(&idev->lock);
2270 }
2271 rcu_read_unlock();
2272 if (err)
2273 pr_err("ERROR: IPv6 link address not available\n");
2274 }
2275 } else {
2276 if (pkt_dev->min_pkt_size == 0) {
2277 pkt_dev->min_pkt_size = 14 + sizeof(struct iphdr)
2278 + sizeof(struct udphdr)
2279 + sizeof(struct pktgen_hdr)
2280 + pkt_dev->pkt_overhead;
2281 }
2282
2283 pkt_dev->saddr_min = 0;
2284 pkt_dev->saddr_max = 0;
2285 if (strlen(pkt_dev->src_min) == 0) {
2286
2287 struct in_device *in_dev;
2288
2289 rcu_read_lock();
2290 in_dev = __in_dev_get_rcu(pkt_dev->odev);
2291 if (in_dev) {
2292 const struct in_ifaddr *ifa;
2293
2294 ifa = rcu_dereference(in_dev->ifa_list);
2295 if (ifa) {
2296 pkt_dev->saddr_min = ifa->ifa_address;
2297 pkt_dev->saddr_max = pkt_dev->saddr_min;
2298 }
2299 }
2300 rcu_read_unlock();
2301 } else {
2302 pkt_dev->saddr_min = in_aton(pkt_dev->src_min);
2303 pkt_dev->saddr_max = in_aton(pkt_dev->src_max);
2304 }
2305
2306 pkt_dev->daddr_min = in_aton(pkt_dev->dst_min);
2307 pkt_dev->daddr_max = in_aton(pkt_dev->dst_max);
2308 }
2309 /* Initialize current values. */
2310 pkt_dev->cur_pkt_size = pkt_dev->min_pkt_size;
2311 if (pkt_dev->min_pkt_size > pkt_dev->max_pkt_size)
2312 pkt_dev->max_pkt_size = pkt_dev->min_pkt_size;
2313
2314 pkt_dev->cur_dst_mac_offset = 0;
2315 pkt_dev->cur_src_mac_offset = 0;
2316 pkt_dev->cur_saddr = pkt_dev->saddr_min;
2317 pkt_dev->cur_daddr = pkt_dev->daddr_min;
2318 pkt_dev->cur_udp_dst = pkt_dev->udp_dst_min;
2319 pkt_dev->cur_udp_src = pkt_dev->udp_src_min;
2320 pkt_dev->nflows = 0;
2321 }
2322
2323
spin(struct pktgen_dev * pkt_dev,ktime_t spin_until)2324 static void spin(struct pktgen_dev *pkt_dev, ktime_t spin_until)
2325 {
2326 ktime_t start_time, end_time;
2327 s64 remaining;
2328 struct hrtimer_sleeper t;
2329
2330 hrtimer_setup_sleeper_on_stack(&t, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
2331 hrtimer_set_expires(&t.timer, spin_until);
2332
2333 remaining = ktime_to_ns(hrtimer_expires_remaining(&t.timer));
2334 if (remaining <= 0)
2335 goto out;
2336
2337 start_time = ktime_get();
2338 if (remaining < 100000) {
2339 /* for small delays (<100us), just loop until limit is reached */
2340 do {
2341 end_time = ktime_get();
2342 } while (ktime_compare(end_time, spin_until) < 0);
2343 } else {
2344 do {
2345 set_current_state(TASK_INTERRUPTIBLE);
2346 hrtimer_sleeper_start_expires(&t, HRTIMER_MODE_ABS);
2347
2348 if (likely(t.task))
2349 schedule();
2350
2351 hrtimer_cancel(&t.timer);
2352 } while (t.task && pkt_dev->running && !signal_pending(current));
2353 __set_current_state(TASK_RUNNING);
2354 end_time = ktime_get();
2355 }
2356
2357 pkt_dev->idle_acc += ktime_to_ns(ktime_sub(end_time, start_time));
2358 out:
2359 pkt_dev->next_tx = ktime_add_ns(spin_until, pkt_dev->delay);
2360 destroy_hrtimer_on_stack(&t.timer);
2361 }
2362
set_pkt_overhead(struct pktgen_dev * pkt_dev)2363 static inline void set_pkt_overhead(struct pktgen_dev *pkt_dev)
2364 {
2365 pkt_dev->pkt_overhead = 0;
2366 pkt_dev->pkt_overhead += pkt_dev->nr_labels*sizeof(u32);
2367 pkt_dev->pkt_overhead += VLAN_TAG_SIZE(pkt_dev);
2368 pkt_dev->pkt_overhead += SVLAN_TAG_SIZE(pkt_dev);
2369 }
2370
f_seen(const struct pktgen_dev * pkt_dev,int flow)2371 static inline int f_seen(const struct pktgen_dev *pkt_dev, int flow)
2372 {
2373 return !!(pkt_dev->flows[flow].flags & F_INIT);
2374 }
2375
f_pick(struct pktgen_dev * pkt_dev)2376 static inline int f_pick(struct pktgen_dev *pkt_dev)
2377 {
2378 int flow = pkt_dev->curfl;
2379
2380 if (pkt_dev->flags & F_FLOW_SEQ) {
2381 if (pkt_dev->flows[flow].count >= pkt_dev->lflow) {
2382 /* reset time */
2383 pkt_dev->flows[flow].count = 0;
2384 pkt_dev->flows[flow].flags = 0;
2385 pkt_dev->curfl += 1;
2386 if (pkt_dev->curfl >= pkt_dev->cflows)
2387 pkt_dev->curfl = 0; /*reset */
2388 }
2389 } else {
2390 flow = get_random_u32_below(pkt_dev->cflows);
2391 pkt_dev->curfl = flow;
2392
2393 if (pkt_dev->flows[flow].count > pkt_dev->lflow) {
2394 pkt_dev->flows[flow].count = 0;
2395 pkt_dev->flows[flow].flags = 0;
2396 }
2397 }
2398
2399 return pkt_dev->curfl;
2400 }
2401
2402
2403 /* If there was already an IPSEC SA, we keep it as is, else
2404 * we go look for it ...
2405 */
2406 #define DUMMY_MARK 0
get_ipsec_sa(struct pktgen_dev * pkt_dev,int flow)2407 static void get_ipsec_sa(struct pktgen_dev *pkt_dev, int flow)
2408 {
2409 #ifdef CONFIG_XFRM
2410 struct xfrm_state *x = pkt_dev->flows[flow].x;
2411 struct pktgen_net *pn = net_generic(dev_net(pkt_dev->odev), pg_net_id);
2412
2413 if (!x) {
2414
2415 if (pkt_dev->spi) {
2416 /* We need as quick as possible to find the right SA
2417 * Searching with minimum criteria to achieve, this.
2418 */
2419 x = xfrm_state_lookup_byspi(pn->net, htonl(pkt_dev->spi), AF_INET);
2420 } else {
2421 /* slow path: we don't already have xfrm_state */
2422 x = xfrm_stateonly_find(pn->net, DUMMY_MARK, 0,
2423 (xfrm_address_t *)&pkt_dev->cur_daddr,
2424 (xfrm_address_t *)&pkt_dev->cur_saddr,
2425 AF_INET,
2426 pkt_dev->ipsmode,
2427 pkt_dev->ipsproto, 0);
2428 }
2429 if (x) {
2430 pkt_dev->flows[flow].x = x;
2431 set_pkt_overhead(pkt_dev);
2432 pkt_dev->pkt_overhead += x->props.header_len;
2433 }
2434
2435 }
2436 #endif
2437 }
set_cur_queue_map(struct pktgen_dev * pkt_dev)2438 static void set_cur_queue_map(struct pktgen_dev *pkt_dev)
2439 {
2440 if (pkt_dev->flags & F_QUEUE_MAP_CPU)
2441 pkt_dev->cur_queue_map = smp_processor_id();
2442
2443 else if (pkt_dev->queue_map_min <= pkt_dev->queue_map_max) {
2444 __u16 t;
2445
2446 if (pkt_dev->flags & F_QUEUE_MAP_RND) {
2447 t = get_random_u32_inclusive(pkt_dev->queue_map_min,
2448 pkt_dev->queue_map_max);
2449 } else {
2450 t = pkt_dev->cur_queue_map + 1;
2451 if (t > pkt_dev->queue_map_max)
2452 t = pkt_dev->queue_map_min;
2453 }
2454 pkt_dev->cur_queue_map = t;
2455 }
2456 pkt_dev->cur_queue_map = pkt_dev->cur_queue_map % pkt_dev->odev->real_num_tx_queues;
2457 }
2458
2459 /* Increment/randomize headers according to flags and current values
2460 * for IP src/dest, UDP src/dst port, MAC-Addr src/dst
2461 */
mod_cur_headers(struct pktgen_dev * pkt_dev)2462 static void mod_cur_headers(struct pktgen_dev *pkt_dev)
2463 {
2464 __u32 imn;
2465 __u32 imx;
2466 int flow = 0;
2467
2468 if (pkt_dev->cflows)
2469 flow = f_pick(pkt_dev);
2470
2471 /* Deal with source MAC */
2472 if (pkt_dev->src_mac_count > 1) {
2473 __u32 mc;
2474 __u32 tmp;
2475
2476 if (pkt_dev->flags & F_MACSRC_RND)
2477 mc = get_random_u32_below(pkt_dev->src_mac_count);
2478 else {
2479 mc = pkt_dev->cur_src_mac_offset++;
2480 if (pkt_dev->cur_src_mac_offset >=
2481 pkt_dev->src_mac_count)
2482 pkt_dev->cur_src_mac_offset = 0;
2483 }
2484
2485 tmp = pkt_dev->src_mac[5] + (mc & 0xFF);
2486 pkt_dev->hh[11] = tmp;
2487 tmp = (pkt_dev->src_mac[4] + ((mc >> 8) & 0xFF) + (tmp >> 8));
2488 pkt_dev->hh[10] = tmp;
2489 tmp = (pkt_dev->src_mac[3] + ((mc >> 16) & 0xFF) + (tmp >> 8));
2490 pkt_dev->hh[9] = tmp;
2491 tmp = (pkt_dev->src_mac[2] + ((mc >> 24) & 0xFF) + (tmp >> 8));
2492 pkt_dev->hh[8] = tmp;
2493 tmp = (pkt_dev->src_mac[1] + (tmp >> 8));
2494 pkt_dev->hh[7] = tmp;
2495 }
2496
2497 /* Deal with Destination MAC */
2498 if (pkt_dev->dst_mac_count > 1) {
2499 __u32 mc;
2500 __u32 tmp;
2501
2502 if (pkt_dev->flags & F_MACDST_RND)
2503 mc = get_random_u32_below(pkt_dev->dst_mac_count);
2504
2505 else {
2506 mc = pkt_dev->cur_dst_mac_offset++;
2507 if (pkt_dev->cur_dst_mac_offset >=
2508 pkt_dev->dst_mac_count) {
2509 pkt_dev->cur_dst_mac_offset = 0;
2510 }
2511 }
2512
2513 tmp = pkt_dev->dst_mac[5] + (mc & 0xFF);
2514 pkt_dev->hh[5] = tmp;
2515 tmp = (pkt_dev->dst_mac[4] + ((mc >> 8) & 0xFF) + (tmp >> 8));
2516 pkt_dev->hh[4] = tmp;
2517 tmp = (pkt_dev->dst_mac[3] + ((mc >> 16) & 0xFF) + (tmp >> 8));
2518 pkt_dev->hh[3] = tmp;
2519 tmp = (pkt_dev->dst_mac[2] + ((mc >> 24) & 0xFF) + (tmp >> 8));
2520 pkt_dev->hh[2] = tmp;
2521 tmp = (pkt_dev->dst_mac[1] + (tmp >> 8));
2522 pkt_dev->hh[1] = tmp;
2523 }
2524
2525 if (pkt_dev->flags & F_MPLS_RND) {
2526 unsigned int i;
2527
2528 for (i = 0; i < pkt_dev->nr_labels; i++)
2529 if (pkt_dev->labels[i] & MPLS_STACK_BOTTOM)
2530 pkt_dev->labels[i] = MPLS_STACK_BOTTOM |
2531 ((__force __be32)get_random_u32() &
2532 htonl(0x000fffff));
2533 }
2534
2535 if ((pkt_dev->flags & F_VID_RND) && (pkt_dev->vlan_id != 0xffff)) {
2536 pkt_dev->vlan_id = get_random_u32_below(4096);
2537 }
2538
2539 if ((pkt_dev->flags & F_SVID_RND) && (pkt_dev->svlan_id != 0xffff)) {
2540 pkt_dev->svlan_id = get_random_u32_below(4096);
2541 }
2542
2543 if (pkt_dev->udp_src_min < pkt_dev->udp_src_max) {
2544 if (pkt_dev->flags & F_UDPSRC_RND)
2545 pkt_dev->cur_udp_src = get_random_u32_inclusive(pkt_dev->udp_src_min,
2546 pkt_dev->udp_src_max - 1);
2547
2548 else {
2549 pkt_dev->cur_udp_src++;
2550 if (pkt_dev->cur_udp_src >= pkt_dev->udp_src_max)
2551 pkt_dev->cur_udp_src = pkt_dev->udp_src_min;
2552 }
2553 }
2554
2555 if (pkt_dev->udp_dst_min < pkt_dev->udp_dst_max) {
2556 if (pkt_dev->flags & F_UDPDST_RND) {
2557 pkt_dev->cur_udp_dst = get_random_u32_inclusive(pkt_dev->udp_dst_min,
2558 pkt_dev->udp_dst_max - 1);
2559 } else {
2560 pkt_dev->cur_udp_dst++;
2561 if (pkt_dev->cur_udp_dst >= pkt_dev->udp_dst_max)
2562 pkt_dev->cur_udp_dst = pkt_dev->udp_dst_min;
2563 }
2564 }
2565
2566 if (!(pkt_dev->flags & F_IPV6)) {
2567
2568 imn = ntohl(pkt_dev->saddr_min);
2569 imx = ntohl(pkt_dev->saddr_max);
2570 if (imn < imx) {
2571 __u32 t;
2572
2573 if (pkt_dev->flags & F_IPSRC_RND)
2574 t = get_random_u32_inclusive(imn, imx - 1);
2575 else {
2576 t = ntohl(pkt_dev->cur_saddr);
2577 t++;
2578 if (t > imx)
2579 t = imn;
2580
2581 }
2582 pkt_dev->cur_saddr = htonl(t);
2583 }
2584
2585 if (pkt_dev->cflows && f_seen(pkt_dev, flow)) {
2586 pkt_dev->cur_daddr = pkt_dev->flows[flow].cur_daddr;
2587 } else {
2588 imn = ntohl(pkt_dev->daddr_min);
2589 imx = ntohl(pkt_dev->daddr_max);
2590 if (imn < imx) {
2591 __u32 t;
2592 __be32 s;
2593
2594 if (pkt_dev->flags & F_IPDST_RND) {
2595
2596 do {
2597 t = get_random_u32_inclusive(imn, imx - 1);
2598 s = htonl(t);
2599 } while (ipv4_is_loopback(s) ||
2600 ipv4_is_multicast(s) ||
2601 ipv4_is_lbcast(s) ||
2602 ipv4_is_zeronet(s) ||
2603 ipv4_is_local_multicast(s));
2604 pkt_dev->cur_daddr = s;
2605 } else {
2606 t = ntohl(pkt_dev->cur_daddr);
2607 t++;
2608 if (t > imx) {
2609 t = imn;
2610 }
2611 pkt_dev->cur_daddr = htonl(t);
2612 }
2613 }
2614 if (pkt_dev->cflows) {
2615 pkt_dev->flows[flow].flags |= F_INIT;
2616 pkt_dev->flows[flow].cur_daddr =
2617 pkt_dev->cur_daddr;
2618 if (pkt_dev->flags & F_IPSEC)
2619 get_ipsec_sa(pkt_dev, flow);
2620 pkt_dev->nflows++;
2621 }
2622 }
2623 } else { /* IPV6 * */
2624
2625 if (!ipv6_addr_any(&pkt_dev->min_in6_daddr)) {
2626 int i;
2627
2628 /* Only random destinations yet */
2629
2630 for (i = 0; i < 4; i++) {
2631 pkt_dev->cur_in6_daddr.s6_addr32[i] =
2632 (((__force __be32)get_random_u32() |
2633 pkt_dev->min_in6_daddr.s6_addr32[i]) &
2634 pkt_dev->max_in6_daddr.s6_addr32[i]);
2635 }
2636 }
2637 }
2638
2639 if (pkt_dev->min_pkt_size < pkt_dev->max_pkt_size) {
2640 __u32 t;
2641
2642 if (pkt_dev->flags & F_TXSIZE_RND) {
2643 t = get_random_u32_inclusive(pkt_dev->min_pkt_size,
2644 pkt_dev->max_pkt_size - 1);
2645 } else {
2646 t = pkt_dev->cur_pkt_size + 1;
2647 if (t > pkt_dev->max_pkt_size)
2648 t = pkt_dev->min_pkt_size;
2649 }
2650 pkt_dev->cur_pkt_size = t;
2651 } else if (pkt_dev->n_imix_entries > 0) {
2652 struct imix_pkt *entry;
2653 __u32 t = get_random_u32_below(IMIX_PRECISION);
2654 __u8 entry_index = pkt_dev->imix_distribution[t];
2655
2656 entry = &pkt_dev->imix_entries[entry_index];
2657 entry->count_so_far++;
2658 pkt_dev->cur_pkt_size = entry->size;
2659 }
2660
2661 set_cur_queue_map(pkt_dev);
2662
2663 pkt_dev->flows[flow].count++;
2664 }
2665
fill_imix_distribution(struct pktgen_dev * pkt_dev)2666 static void fill_imix_distribution(struct pktgen_dev *pkt_dev)
2667 {
2668 int cumulative_probabilites[MAX_IMIX_ENTRIES];
2669 int j = 0;
2670 __u64 cumulative_prob = 0;
2671 __u64 total_weight = 0;
2672 int i = 0;
2673
2674 for (i = 0; i < pkt_dev->n_imix_entries; i++)
2675 total_weight += pkt_dev->imix_entries[i].weight;
2676
2677 /* Fill cumulative_probabilites with sum of normalized probabilities */
2678 for (i = 0; i < pkt_dev->n_imix_entries - 1; i++) {
2679 cumulative_prob += div64_u64(pkt_dev->imix_entries[i].weight *
2680 IMIX_PRECISION,
2681 total_weight);
2682 cumulative_probabilites[i] = cumulative_prob;
2683 }
2684 cumulative_probabilites[pkt_dev->n_imix_entries - 1] = 100;
2685
2686 for (i = 0; i < IMIX_PRECISION; i++) {
2687 if (i == cumulative_probabilites[j])
2688 j++;
2689 pkt_dev->imix_distribution[i] = j;
2690 }
2691 }
2692
2693 #ifdef CONFIG_XFRM
2694 static u32 pktgen_dst_metrics[RTAX_MAX + 1] = {
2695
2696 [RTAX_HOPLIMIT] = 0x5, /* Set a static hoplimit */
2697 };
2698
pktgen_output_ipsec(struct sk_buff * skb,struct pktgen_dev * pkt_dev)2699 static int pktgen_output_ipsec(struct sk_buff *skb, struct pktgen_dev *pkt_dev)
2700 {
2701 struct xfrm_state *x = pkt_dev->flows[pkt_dev->curfl].x;
2702 int err = 0;
2703 struct net *net = dev_net(pkt_dev->odev);
2704
2705 if (!x)
2706 return 0;
2707 /* XXX: we dont support tunnel mode for now until
2708 * we resolve the dst issue
2709 */
2710 if ((x->props.mode != XFRM_MODE_TRANSPORT) && (pkt_dev->spi == 0))
2711 return 0;
2712
2713 /* But when user specify an valid SPI, transformation
2714 * supports both transport/tunnel mode + ESP/AH type.
2715 */
2716 if ((x->props.mode == XFRM_MODE_TUNNEL) && (pkt_dev->spi != 0))
2717 skb->_skb_refdst = (unsigned long)&pkt_dev->xdst.u.dst | SKB_DST_NOREF;
2718
2719 rcu_read_lock_bh();
2720 err = pktgen_xfrm_outer_mode_output(x, skb);
2721 rcu_read_unlock_bh();
2722 if (err) {
2723 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEMODEERROR);
2724 goto error;
2725 }
2726 err = x->type->output(x, skb);
2727 if (err) {
2728 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEPROTOERROR);
2729 goto error;
2730 }
2731 spin_lock_bh(&x->lock);
2732 x->curlft.bytes += skb->len;
2733 x->curlft.packets++;
2734 spin_unlock_bh(&x->lock);
2735 error:
2736 return err;
2737 }
2738
free_SAs(struct pktgen_dev * pkt_dev)2739 static void free_SAs(struct pktgen_dev *pkt_dev)
2740 {
2741 if (pkt_dev->cflows) {
2742 /* let go of the SAs if we have them */
2743 int i;
2744
2745 for (i = 0; i < pkt_dev->cflows; i++) {
2746 struct xfrm_state *x = pkt_dev->flows[i].x;
2747
2748 if (x) {
2749 xfrm_state_put(x);
2750 pkt_dev->flows[i].x = NULL;
2751 }
2752 }
2753 }
2754 }
2755
process_ipsec(struct pktgen_dev * pkt_dev,struct sk_buff * skb,__be16 protocol)2756 static int process_ipsec(struct pktgen_dev *pkt_dev,
2757 struct sk_buff *skb, __be16 protocol)
2758 {
2759 if (pkt_dev->flags & F_IPSEC) {
2760 struct xfrm_state *x = pkt_dev->flows[pkt_dev->curfl].x;
2761 int nhead = 0;
2762
2763 if (x) {
2764 struct ethhdr *eth;
2765 struct iphdr *iph;
2766 int ret;
2767
2768 nhead = x->props.header_len - skb_headroom(skb);
2769 if (nhead > 0) {
2770 ret = pskb_expand_head(skb, nhead, 0, GFP_ATOMIC);
2771 if (ret < 0) {
2772 pr_err("Error expanding ipsec packet %d\n",
2773 ret);
2774 goto err;
2775 }
2776 }
2777
2778 /* ipsec is not expecting ll header */
2779 skb_pull(skb, ETH_HLEN);
2780 ret = pktgen_output_ipsec(skb, pkt_dev);
2781 if (ret) {
2782 pr_err("Error creating ipsec packet %d\n", ret);
2783 goto err;
2784 }
2785 /* restore ll */
2786 eth = skb_push(skb, ETH_HLEN);
2787 memcpy(eth, pkt_dev->hh, 2 * ETH_ALEN);
2788 eth->h_proto = protocol;
2789
2790 /* Update IPv4 header len as well as checksum value */
2791 iph = ip_hdr(skb);
2792 iph->tot_len = htons(skb->len - ETH_HLEN);
2793 ip_send_check(iph);
2794 }
2795 }
2796 return 1;
2797 err:
2798 kfree_skb(skb);
2799 return 0;
2800 }
2801 #endif
2802
mpls_push(__be32 * mpls,struct pktgen_dev * pkt_dev)2803 static void mpls_push(__be32 *mpls, struct pktgen_dev *pkt_dev)
2804 {
2805 unsigned int i;
2806
2807 for (i = 0; i < pkt_dev->nr_labels; i++)
2808 *mpls++ = pkt_dev->labels[i] & ~MPLS_STACK_BOTTOM;
2809
2810 mpls--;
2811 *mpls |= MPLS_STACK_BOTTOM;
2812 }
2813
build_tci(unsigned int id,unsigned int cfi,unsigned int prio)2814 static inline __be16 build_tci(unsigned int id, unsigned int cfi,
2815 unsigned int prio)
2816 {
2817 return htons(id | (cfi << 12) | (prio << 13));
2818 }
2819
pktgen_finalize_skb(struct pktgen_dev * pkt_dev,struct sk_buff * skb,int datalen)2820 static void pktgen_finalize_skb(struct pktgen_dev *pkt_dev, struct sk_buff *skb,
2821 int datalen)
2822 {
2823 struct timespec64 timestamp;
2824 struct pktgen_hdr *pgh;
2825
2826 pgh = skb_put(skb, sizeof(*pgh));
2827 datalen -= sizeof(*pgh);
2828
2829 if (pkt_dev->nfrags <= 0) {
2830 skb_put_zero(skb, datalen);
2831 } else {
2832 int frags = pkt_dev->nfrags;
2833 int i, len;
2834 int frag_len;
2835
2836
2837 if (frags > MAX_SKB_FRAGS)
2838 frags = MAX_SKB_FRAGS;
2839 len = datalen - frags * PAGE_SIZE;
2840 if (len > 0) {
2841 skb_put_zero(skb, len);
2842 datalen = frags * PAGE_SIZE;
2843 }
2844
2845 i = 0;
2846 frag_len = min_t(int, datalen / frags, PAGE_SIZE);
2847 while (datalen > 0) {
2848 if (unlikely(!pkt_dev->page)) {
2849 int node = numa_node_id();
2850
2851 if (pkt_dev->node >= 0 && (pkt_dev->flags & F_NODE))
2852 node = pkt_dev->node;
2853 pkt_dev->page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0);
2854 if (!pkt_dev->page)
2855 break;
2856 }
2857 get_page(pkt_dev->page);
2858
2859 /*last fragment, fill rest of data*/
2860 if (i == (frags - 1))
2861 skb_frag_fill_page_desc(&skb_shinfo(skb)->frags[i],
2862 pkt_dev->page, 0,
2863 min(datalen, PAGE_SIZE));
2864 else
2865 skb_frag_fill_page_desc(&skb_shinfo(skb)->frags[i],
2866 pkt_dev->page, 0, frag_len);
2867
2868 datalen -= skb_frag_size(&skb_shinfo(skb)->frags[i]);
2869 skb->len += skb_frag_size(&skb_shinfo(skb)->frags[i]);
2870 skb->data_len += skb_frag_size(&skb_shinfo(skb)->frags[i]);
2871 i++;
2872 skb_shinfo(skb)->nr_frags = i;
2873 }
2874 }
2875
2876 /* Stamp the time, and sequence number,
2877 * convert them to network byte order
2878 */
2879 pgh->pgh_magic = htonl(PKTGEN_MAGIC);
2880 pgh->seq_num = htonl(pkt_dev->seq_num);
2881
2882 if (pkt_dev->flags & F_NO_TIMESTAMP) {
2883 pgh->tv_sec = 0;
2884 pgh->tv_usec = 0;
2885 } else {
2886 /*
2887 * pgh->tv_sec wraps in y2106 when interpreted as unsigned
2888 * as done by wireshark, or y2038 when interpreted as signed.
2889 * This is probably harmless, but if anyone wants to improve
2890 * it, we could introduce a variant that puts 64-bit nanoseconds
2891 * into the respective header bytes.
2892 * This would also be slightly faster to read.
2893 */
2894 ktime_get_real_ts64(×tamp);
2895 pgh->tv_sec = htonl(timestamp.tv_sec);
2896 pgh->tv_usec = htonl(timestamp.tv_nsec / NSEC_PER_USEC);
2897 }
2898 }
2899
pktgen_alloc_skb(struct net_device * dev,struct pktgen_dev * pkt_dev)2900 static struct sk_buff *pktgen_alloc_skb(struct net_device *dev,
2901 struct pktgen_dev *pkt_dev)
2902 {
2903 unsigned int extralen = LL_RESERVED_SPACE(dev);
2904 struct sk_buff *skb = NULL;
2905 unsigned int size;
2906
2907 size = pkt_dev->cur_pkt_size + 64 + extralen + pkt_dev->pkt_overhead;
2908 if (pkt_dev->flags & F_NODE) {
2909 int node = pkt_dev->node >= 0 ? pkt_dev->node : numa_node_id();
2910
2911 skb = __alloc_skb(NET_SKB_PAD + size, GFP_NOWAIT, 0, node);
2912 if (likely(skb)) {
2913 skb_reserve(skb, NET_SKB_PAD);
2914 skb->dev = dev;
2915 }
2916 } else {
2917 skb = __netdev_alloc_skb(dev, size, GFP_NOWAIT);
2918 }
2919
2920 /* the caller pre-fetches from skb->data and reserves for the mac hdr */
2921 if (likely(skb))
2922 skb_reserve(skb, extralen - 16);
2923
2924 return skb;
2925 }
2926
fill_packet_ipv4(struct net_device * odev,struct pktgen_dev * pkt_dev)2927 static struct sk_buff *fill_packet_ipv4(struct net_device *odev,
2928 struct pktgen_dev *pkt_dev)
2929 {
2930 struct sk_buff *skb = NULL;
2931 __u8 *eth;
2932 struct udphdr *udph;
2933 int datalen, iplen;
2934 struct iphdr *iph;
2935 __be16 protocol = htons(ETH_P_IP);
2936 __be32 *mpls;
2937 __be16 *vlan_tci = NULL; /* Encapsulates priority and VLAN ID */
2938 __be16 *vlan_encapsulated_proto = NULL; /* packet type ID field (or len) for VLAN tag */
2939 __be16 *svlan_tci = NULL; /* Encapsulates priority and SVLAN ID */
2940 __be16 *svlan_encapsulated_proto = NULL; /* packet type ID field (or len) for SVLAN tag */
2941 u16 queue_map;
2942
2943 if (pkt_dev->nr_labels)
2944 protocol = htons(ETH_P_MPLS_UC);
2945
2946 if (pkt_dev->vlan_id != 0xffff)
2947 protocol = htons(ETH_P_8021Q);
2948
2949 /* Update any of the values, used when we're incrementing various
2950 * fields.
2951 */
2952 mod_cur_headers(pkt_dev);
2953 queue_map = pkt_dev->cur_queue_map;
2954
2955 skb = pktgen_alloc_skb(odev, pkt_dev);
2956 if (!skb) {
2957 sprintf(pkt_dev->result, "No memory");
2958 return NULL;
2959 }
2960
2961 prefetchw(skb->data);
2962 skb_reserve(skb, 16);
2963
2964 /* Reserve for ethernet and IP header */
2965 eth = skb_push(skb, 14);
2966 mpls = skb_put(skb, pkt_dev->nr_labels * sizeof(__u32));
2967 if (pkt_dev->nr_labels)
2968 mpls_push(mpls, pkt_dev);
2969
2970 if (pkt_dev->vlan_id != 0xffff) {
2971 if (pkt_dev->svlan_id != 0xffff) {
2972 svlan_tci = skb_put(skb, sizeof(__be16));
2973 *svlan_tci = build_tci(pkt_dev->svlan_id,
2974 pkt_dev->svlan_cfi,
2975 pkt_dev->svlan_p);
2976 svlan_encapsulated_proto = skb_put(skb,
2977 sizeof(__be16));
2978 *svlan_encapsulated_proto = htons(ETH_P_8021Q);
2979 }
2980 vlan_tci = skb_put(skb, sizeof(__be16));
2981 *vlan_tci = build_tci(pkt_dev->vlan_id,
2982 pkt_dev->vlan_cfi,
2983 pkt_dev->vlan_p);
2984 vlan_encapsulated_proto = skb_put(skb, sizeof(__be16));
2985 *vlan_encapsulated_proto = htons(ETH_P_IP);
2986 }
2987
2988 skb_reset_mac_header(skb);
2989 skb_set_network_header(skb, skb->len);
2990 iph = skb_put(skb, sizeof(struct iphdr));
2991
2992 skb_set_transport_header(skb, skb->len);
2993 udph = skb_put(skb, sizeof(struct udphdr));
2994 skb_set_queue_mapping(skb, queue_map);
2995 skb->priority = pkt_dev->skb_priority;
2996
2997 memcpy(eth, pkt_dev->hh, 12);
2998 *(__be16 *)ð[12] = protocol;
2999
3000 /* Eth + IPh + UDPh + mpls */
3001 datalen = pkt_dev->cur_pkt_size - 14 - 20 - 8 -
3002 pkt_dev->pkt_overhead;
3003 if (datalen < 0 || datalen < sizeof(struct pktgen_hdr))
3004 datalen = sizeof(struct pktgen_hdr);
3005
3006 udph->source = htons(pkt_dev->cur_udp_src);
3007 udph->dest = htons(pkt_dev->cur_udp_dst);
3008 udph->len = htons(datalen + 8); /* DATA + udphdr */
3009 udph->check = 0;
3010
3011 iph->ihl = 5;
3012 iph->version = 4;
3013 iph->ttl = 32;
3014 iph->tos = pkt_dev->tos;
3015 iph->protocol = IPPROTO_UDP; /* UDP */
3016 iph->saddr = pkt_dev->cur_saddr;
3017 iph->daddr = pkt_dev->cur_daddr;
3018 iph->id = htons(pkt_dev->ip_id);
3019 pkt_dev->ip_id++;
3020 iph->frag_off = 0;
3021 iplen = 20 + 8 + datalen;
3022 iph->tot_len = htons(iplen);
3023 ip_send_check(iph);
3024 skb->protocol = protocol;
3025 skb->dev = odev;
3026 skb->pkt_type = PACKET_HOST;
3027
3028 pktgen_finalize_skb(pkt_dev, skb, datalen);
3029
3030 if (!(pkt_dev->flags & F_UDPCSUM)) {
3031 skb->ip_summed = CHECKSUM_NONE;
3032 } else if (odev->features & (NETIF_F_HW_CSUM | NETIF_F_IP_CSUM)) {
3033 skb->ip_summed = CHECKSUM_PARTIAL;
3034 skb->csum = 0;
3035 udp4_hwcsum(skb, iph->saddr, iph->daddr);
3036 } else {
3037 __wsum csum = skb_checksum(skb, skb_transport_offset(skb), datalen + 8, 0);
3038
3039 /* add protocol-dependent pseudo-header */
3040 udph->check = csum_tcpudp_magic(iph->saddr, iph->daddr,
3041 datalen + 8, IPPROTO_UDP, csum);
3042
3043 if (udph->check == 0)
3044 udph->check = CSUM_MANGLED_0;
3045 }
3046
3047 #ifdef CONFIG_XFRM
3048 if (!process_ipsec(pkt_dev, skb, protocol))
3049 return NULL;
3050 #endif
3051
3052 return skb;
3053 }
3054
fill_packet_ipv6(struct net_device * odev,struct pktgen_dev * pkt_dev)3055 static struct sk_buff *fill_packet_ipv6(struct net_device *odev,
3056 struct pktgen_dev *pkt_dev)
3057 {
3058 struct sk_buff *skb = NULL;
3059 __u8 *eth;
3060 struct udphdr *udph;
3061 int datalen, udplen;
3062 struct ipv6hdr *iph;
3063 __be16 protocol = htons(ETH_P_IPV6);
3064 __be32 *mpls;
3065 __be16 *vlan_tci = NULL; /* Encapsulates priority and VLAN ID */
3066 __be16 *vlan_encapsulated_proto = NULL; /* packet type ID field (or len) for VLAN tag */
3067 __be16 *svlan_tci = NULL; /* Encapsulates priority and SVLAN ID */
3068 __be16 *svlan_encapsulated_proto = NULL; /* packet type ID field (or len) for SVLAN tag */
3069 u16 queue_map;
3070
3071 if (pkt_dev->nr_labels)
3072 protocol = htons(ETH_P_MPLS_UC);
3073
3074 if (pkt_dev->vlan_id != 0xffff)
3075 protocol = htons(ETH_P_8021Q);
3076
3077 /* Update any of the values, used when we're incrementing various
3078 * fields.
3079 */
3080 mod_cur_headers(pkt_dev);
3081 queue_map = pkt_dev->cur_queue_map;
3082
3083 skb = pktgen_alloc_skb(odev, pkt_dev);
3084 if (!skb) {
3085 sprintf(pkt_dev->result, "No memory");
3086 return NULL;
3087 }
3088
3089 prefetchw(skb->data);
3090 skb_reserve(skb, 16);
3091
3092 /* Reserve for ethernet and IP header */
3093 eth = skb_push(skb, 14);
3094 mpls = skb_put(skb, pkt_dev->nr_labels * sizeof(__u32));
3095 if (pkt_dev->nr_labels)
3096 mpls_push(mpls, pkt_dev);
3097
3098 if (pkt_dev->vlan_id != 0xffff) {
3099 if (pkt_dev->svlan_id != 0xffff) {
3100 svlan_tci = skb_put(skb, sizeof(__be16));
3101 *svlan_tci = build_tci(pkt_dev->svlan_id,
3102 pkt_dev->svlan_cfi,
3103 pkt_dev->svlan_p);
3104 svlan_encapsulated_proto = skb_put(skb,
3105 sizeof(__be16));
3106 *svlan_encapsulated_proto = htons(ETH_P_8021Q);
3107 }
3108 vlan_tci = skb_put(skb, sizeof(__be16));
3109 *vlan_tci = build_tci(pkt_dev->vlan_id,
3110 pkt_dev->vlan_cfi,
3111 pkt_dev->vlan_p);
3112 vlan_encapsulated_proto = skb_put(skb, sizeof(__be16));
3113 *vlan_encapsulated_proto = htons(ETH_P_IPV6);
3114 }
3115
3116 skb_reset_mac_header(skb);
3117 skb_set_network_header(skb, skb->len);
3118 iph = skb_put(skb, sizeof(struct ipv6hdr));
3119
3120 skb_set_transport_header(skb, skb->len);
3121 udph = skb_put(skb, sizeof(struct udphdr));
3122 skb_set_queue_mapping(skb, queue_map);
3123 skb->priority = pkt_dev->skb_priority;
3124
3125 memcpy(eth, pkt_dev->hh, 12);
3126 *(__be16 *) ð[12] = protocol;
3127
3128 /* Eth + IPh + UDPh + mpls */
3129 datalen = pkt_dev->cur_pkt_size - 14 -
3130 sizeof(struct ipv6hdr) - sizeof(struct udphdr) -
3131 pkt_dev->pkt_overhead;
3132
3133 if (datalen < 0 || datalen < sizeof(struct pktgen_hdr)) {
3134 datalen = sizeof(struct pktgen_hdr);
3135 net_info_ratelimited("increased datalen to %d\n", datalen);
3136 }
3137
3138 udplen = datalen + sizeof(struct udphdr);
3139 udph->source = htons(pkt_dev->cur_udp_src);
3140 udph->dest = htons(pkt_dev->cur_udp_dst);
3141 udph->len = htons(udplen);
3142 udph->check = 0;
3143
3144 *(__be32 *) iph = htonl(0x60000000); /* Version + flow */
3145
3146 if (pkt_dev->traffic_class) {
3147 /* Version + traffic class + flow (0) */
3148 *(__be32 *)iph |= htonl(0x60000000 | (pkt_dev->traffic_class << 20));
3149 }
3150
3151 iph->hop_limit = 32;
3152
3153 iph->payload_len = htons(udplen);
3154 iph->nexthdr = IPPROTO_UDP;
3155
3156 iph->daddr = pkt_dev->cur_in6_daddr;
3157 iph->saddr = pkt_dev->cur_in6_saddr;
3158
3159 skb->protocol = protocol;
3160 skb->dev = odev;
3161 skb->pkt_type = PACKET_HOST;
3162
3163 pktgen_finalize_skb(pkt_dev, skb, datalen);
3164
3165 if (!(pkt_dev->flags & F_UDPCSUM)) {
3166 skb->ip_summed = CHECKSUM_NONE;
3167 } else if (odev->features & (NETIF_F_HW_CSUM | NETIF_F_IPV6_CSUM)) {
3168 skb->ip_summed = CHECKSUM_PARTIAL;
3169 skb->csum_start = skb_transport_header(skb) - skb->head;
3170 skb->csum_offset = offsetof(struct udphdr, check);
3171 udph->check = ~csum_ipv6_magic(&iph->saddr, &iph->daddr, udplen, IPPROTO_UDP, 0);
3172 } else {
3173 __wsum csum = skb_checksum(skb, skb_transport_offset(skb), udplen, 0);
3174
3175 /* add protocol-dependent pseudo-header */
3176 udph->check = csum_ipv6_magic(&iph->saddr, &iph->daddr, udplen, IPPROTO_UDP, csum);
3177
3178 if (udph->check == 0)
3179 udph->check = CSUM_MANGLED_0;
3180 }
3181
3182 return skb;
3183 }
3184
fill_packet(struct net_device * odev,struct pktgen_dev * pkt_dev)3185 static struct sk_buff *fill_packet(struct net_device *odev,
3186 struct pktgen_dev *pkt_dev)
3187 {
3188 if (pkt_dev->flags & F_IPV6)
3189 return fill_packet_ipv6(odev, pkt_dev);
3190 else
3191 return fill_packet_ipv4(odev, pkt_dev);
3192 }
3193
pktgen_clear_counters(struct pktgen_dev * pkt_dev)3194 static void pktgen_clear_counters(struct pktgen_dev *pkt_dev)
3195 {
3196 pkt_dev->seq_num = 1;
3197 pkt_dev->idle_acc = 0;
3198 pkt_dev->sofar = 0;
3199 pkt_dev->tx_bytes = 0;
3200 pkt_dev->errors = 0;
3201 }
3202
3203 /* Set up structure for sending pkts, clear counters */
3204
pktgen_run(struct pktgen_thread * t)3205 static void pktgen_run(struct pktgen_thread *t)
3206 {
3207 struct pktgen_dev *pkt_dev;
3208 int started = 0;
3209
3210 func_enter();
3211
3212 rcu_read_lock();
3213 list_for_each_entry_rcu(pkt_dev, &t->if_list, list) {
3214
3215 /*
3216 * setup odev and create initial packet.
3217 */
3218 pktgen_setup_inject(pkt_dev);
3219
3220 if (pkt_dev->odev) {
3221 pktgen_clear_counters(pkt_dev);
3222 pkt_dev->skb = NULL;
3223 pkt_dev->started_at = pkt_dev->next_tx = ktime_get();
3224
3225 set_pkt_overhead(pkt_dev);
3226
3227 strscpy(pkt_dev->result, "Starting");
3228 pkt_dev->running = 1; /* Cranke yeself! */
3229 started++;
3230 } else
3231 strscpy(pkt_dev->result, "Error starting");
3232 }
3233 rcu_read_unlock();
3234 if (started)
3235 t->control &= ~(T_STOP);
3236 }
3237
pktgen_handle_all_threads(struct pktgen_net * pn,u32 flags)3238 static void pktgen_handle_all_threads(struct pktgen_net *pn, u32 flags)
3239 {
3240 struct pktgen_thread *t;
3241
3242 mutex_lock(&pktgen_thread_lock);
3243
3244 list_for_each_entry(t, &pn->pktgen_threads, th_list)
3245 t->control |= (flags);
3246
3247 mutex_unlock(&pktgen_thread_lock);
3248 }
3249
pktgen_stop_all_threads(struct pktgen_net * pn)3250 static void pktgen_stop_all_threads(struct pktgen_net *pn)
3251 {
3252 func_enter();
3253
3254 pktgen_handle_all_threads(pn, T_STOP);
3255 }
3256
thread_is_running(const struct pktgen_thread * t)3257 static int thread_is_running(const struct pktgen_thread *t)
3258 {
3259 const struct pktgen_dev *pkt_dev;
3260
3261 rcu_read_lock();
3262 list_for_each_entry_rcu(pkt_dev, &t->if_list, list)
3263 if (pkt_dev->running) {
3264 rcu_read_unlock();
3265 return 1;
3266 }
3267 rcu_read_unlock();
3268 return 0;
3269 }
3270
pktgen_wait_thread_run(struct pktgen_thread * t)3271 static int pktgen_wait_thread_run(struct pktgen_thread *t)
3272 {
3273 while (thread_is_running(t)) {
3274
3275 /* note: 't' will still be around even after the unlock/lock
3276 * cycle because pktgen_thread threads are only cleared at
3277 * net exit
3278 */
3279 mutex_unlock(&pktgen_thread_lock);
3280 msleep_interruptible(100);
3281 mutex_lock(&pktgen_thread_lock);
3282
3283 if (signal_pending(current))
3284 goto signal;
3285 }
3286 return 1;
3287 signal:
3288 return 0;
3289 }
3290
pktgen_wait_all_threads_run(struct pktgen_net * pn)3291 static int pktgen_wait_all_threads_run(struct pktgen_net *pn)
3292 {
3293 struct pktgen_thread *t;
3294 int sig = 1;
3295
3296 /* prevent from racing with rmmod */
3297 if (!try_module_get(THIS_MODULE))
3298 return sig;
3299
3300 mutex_lock(&pktgen_thread_lock);
3301
3302 list_for_each_entry(t, &pn->pktgen_threads, th_list) {
3303 sig = pktgen_wait_thread_run(t);
3304 if (sig == 0)
3305 break;
3306 }
3307
3308 if (sig == 0)
3309 list_for_each_entry(t, &pn->pktgen_threads, th_list)
3310 t->control |= (T_STOP);
3311
3312 mutex_unlock(&pktgen_thread_lock);
3313 module_put(THIS_MODULE);
3314 return sig;
3315 }
3316
pktgen_run_all_threads(struct pktgen_net * pn)3317 static void pktgen_run_all_threads(struct pktgen_net *pn)
3318 {
3319 func_enter();
3320
3321 pktgen_handle_all_threads(pn, T_RUN);
3322
3323 /* Propagate thread->control */
3324 schedule_timeout_interruptible(msecs_to_jiffies(125));
3325
3326 pktgen_wait_all_threads_run(pn);
3327 }
3328
pktgen_reset_all_threads(struct pktgen_net * pn)3329 static void pktgen_reset_all_threads(struct pktgen_net *pn)
3330 {
3331 func_enter();
3332
3333 pktgen_handle_all_threads(pn, T_REMDEVALL);
3334
3335 /* Propagate thread->control */
3336 schedule_timeout_interruptible(msecs_to_jiffies(125));
3337
3338 pktgen_wait_all_threads_run(pn);
3339 }
3340
show_results(struct pktgen_dev * pkt_dev,int nr_frags)3341 static void show_results(struct pktgen_dev *pkt_dev, int nr_frags)
3342 {
3343 __u64 bps, mbps, pps;
3344 char *p = pkt_dev->result;
3345 ktime_t elapsed = ktime_sub(pkt_dev->stopped_at,
3346 pkt_dev->started_at);
3347 ktime_t idle = ns_to_ktime(pkt_dev->idle_acc);
3348
3349 p += sprintf(p, "OK: %llu(c%llu+d%llu) usec, %llu (%dbyte,%dfrags)\n",
3350 (unsigned long long)ktime_to_us(elapsed),
3351 (unsigned long long)ktime_to_us(ktime_sub(elapsed, idle)),
3352 (unsigned long long)ktime_to_us(idle),
3353 (unsigned long long)pkt_dev->sofar,
3354 pkt_dev->cur_pkt_size, nr_frags);
3355
3356 pps = div64_u64(pkt_dev->sofar * NSEC_PER_SEC,
3357 ktime_to_ns(elapsed));
3358
3359 if (pkt_dev->n_imix_entries > 0) {
3360 int i;
3361 struct imix_pkt *entry;
3362
3363 bps = 0;
3364 for (i = 0; i < pkt_dev->n_imix_entries; i++) {
3365 entry = &pkt_dev->imix_entries[i];
3366 bps += entry->size * entry->count_so_far;
3367 }
3368 bps = div64_u64(bps * 8 * NSEC_PER_SEC, ktime_to_ns(elapsed));
3369 } else {
3370 bps = pps * 8 * pkt_dev->cur_pkt_size;
3371 }
3372
3373 mbps = bps;
3374 do_div(mbps, 1000000);
3375 p += sprintf(p, " %llupps %lluMb/sec (%llubps) errors: %llu",
3376 (unsigned long long)pps,
3377 (unsigned long long)mbps,
3378 (unsigned long long)bps,
3379 (unsigned long long)pkt_dev->errors);
3380 }
3381
3382 /* Set stopped-at timer, remove from running list, do counters & statistics */
pktgen_stop_device(struct pktgen_dev * pkt_dev)3383 static int pktgen_stop_device(struct pktgen_dev *pkt_dev)
3384 {
3385 int nr_frags = pkt_dev->skb ? skb_shinfo(pkt_dev->skb)->nr_frags : -1;
3386
3387 if (!pkt_dev->running) {
3388 pr_warn("interface: %s is already stopped\n",
3389 pkt_dev->odevname);
3390 return -EINVAL;
3391 }
3392
3393 pkt_dev->running = 0;
3394 kfree_skb(pkt_dev->skb);
3395 pkt_dev->skb = NULL;
3396 pkt_dev->stopped_at = ktime_get();
3397
3398 show_results(pkt_dev, nr_frags);
3399
3400 return 0;
3401 }
3402
next_to_run(struct pktgen_thread * t)3403 static struct pktgen_dev *next_to_run(struct pktgen_thread *t)
3404 {
3405 struct pktgen_dev *pkt_dev, *best = NULL;
3406
3407 rcu_read_lock();
3408 list_for_each_entry_rcu(pkt_dev, &t->if_list, list) {
3409 if (!pkt_dev->running)
3410 continue;
3411 if (best == NULL)
3412 best = pkt_dev;
3413 else if (ktime_compare(pkt_dev->next_tx, best->next_tx) < 0)
3414 best = pkt_dev;
3415 }
3416 rcu_read_unlock();
3417
3418 return best;
3419 }
3420
pktgen_stop(struct pktgen_thread * t)3421 static void pktgen_stop(struct pktgen_thread *t)
3422 {
3423 struct pktgen_dev *pkt_dev;
3424
3425 func_enter();
3426
3427 rcu_read_lock();
3428
3429 list_for_each_entry_rcu(pkt_dev, &t->if_list, list) {
3430 pktgen_stop_device(pkt_dev);
3431 }
3432
3433 rcu_read_unlock();
3434 }
3435
3436 /*
3437 * one of our devices needs to be removed - find it
3438 * and remove it
3439 */
pktgen_rem_one_if(struct pktgen_thread * t)3440 static void pktgen_rem_one_if(struct pktgen_thread *t)
3441 {
3442 struct list_head *q, *n;
3443 struct pktgen_dev *cur;
3444
3445 func_enter();
3446
3447 list_for_each_safe(q, n, &t->if_list) {
3448 cur = list_entry(q, struct pktgen_dev, list);
3449
3450 if (!cur->removal_mark)
3451 continue;
3452
3453 kfree_skb(cur->skb);
3454 cur->skb = NULL;
3455
3456 pktgen_remove_device(t, cur);
3457
3458 break;
3459 }
3460 }
3461
pktgen_rem_all_ifs(struct pktgen_thread * t)3462 static void pktgen_rem_all_ifs(struct pktgen_thread *t)
3463 {
3464 struct list_head *q, *n;
3465 struct pktgen_dev *cur;
3466
3467 func_enter();
3468
3469 /* Remove all devices, free mem */
3470
3471 list_for_each_safe(q, n, &t->if_list) {
3472 cur = list_entry(q, struct pktgen_dev, list);
3473
3474 kfree_skb(cur->skb);
3475 cur->skb = NULL;
3476
3477 pktgen_remove_device(t, cur);
3478 }
3479 }
3480
pktgen_rem_thread(struct pktgen_thread * t)3481 static void pktgen_rem_thread(struct pktgen_thread *t)
3482 {
3483 /* Remove from the thread list */
3484 remove_proc_entry(t->tsk->comm, t->net->proc_dir);
3485 }
3486
pktgen_resched(struct pktgen_dev * pkt_dev)3487 static void pktgen_resched(struct pktgen_dev *pkt_dev)
3488 {
3489 ktime_t idle_start = ktime_get();
3490
3491 schedule();
3492 pkt_dev->idle_acc += ktime_to_ns(ktime_sub(ktime_get(), idle_start));
3493 }
3494
pktgen_wait_for_skb(struct pktgen_dev * pkt_dev)3495 static void pktgen_wait_for_skb(struct pktgen_dev *pkt_dev)
3496 {
3497 ktime_t idle_start = ktime_get();
3498
3499 while (refcount_read(&(pkt_dev->skb->users)) != 1) {
3500 if (signal_pending(current))
3501 break;
3502
3503 if (need_resched())
3504 pktgen_resched(pkt_dev);
3505 else
3506 cpu_relax();
3507 }
3508 pkt_dev->idle_acc += ktime_to_ns(ktime_sub(ktime_get(), idle_start));
3509 }
3510
pktgen_xmit(struct pktgen_dev * pkt_dev)3511 static void pktgen_xmit(struct pktgen_dev *pkt_dev)
3512 {
3513 bool skb_shared = !!(READ_ONCE(pkt_dev->flags) & F_SHARED);
3514 struct net_device *odev = pkt_dev->odev;
3515 struct netdev_queue *txq;
3516 unsigned int burst = 1;
3517 struct sk_buff *skb;
3518 int clone_skb = 0;
3519 int ret;
3520
3521 /* If 'skb_shared' is false, the read of possible
3522 * new values (if any) for 'burst' and 'clone_skb' will be skipped to
3523 * prevent some concurrent changes from slipping in. And the stabilized
3524 * config will be read in during the next run of pktgen_xmit.
3525 */
3526 if (skb_shared) {
3527 burst = READ_ONCE(pkt_dev->burst);
3528 clone_skb = READ_ONCE(pkt_dev->clone_skb);
3529 }
3530
3531 /* If device is offline, then don't send */
3532 if (unlikely(!netif_running(odev) || !netif_carrier_ok(odev))) {
3533 pktgen_stop_device(pkt_dev);
3534 return;
3535 }
3536
3537 /* This is max DELAY, this has special meaning of
3538 * "never transmit"
3539 */
3540 if (unlikely(pkt_dev->delay == ULLONG_MAX)) {
3541 pkt_dev->next_tx = ktime_add_ns(ktime_get(), ULONG_MAX);
3542 return;
3543 }
3544
3545 /* If no skb or clone count exhausted then get new one */
3546 if (!pkt_dev->skb || (pkt_dev->last_ok &&
3547 ++pkt_dev->clone_count >= clone_skb)) {
3548 /* build a new pkt */
3549 kfree_skb(pkt_dev->skb);
3550
3551 pkt_dev->skb = fill_packet(odev, pkt_dev);
3552 if (pkt_dev->skb == NULL) {
3553 pr_err("ERROR: couldn't allocate skb in fill_packet\n");
3554 schedule();
3555 pkt_dev->clone_count--; /* back out increment, OOM */
3556 return;
3557 }
3558 pkt_dev->last_pkt_size = pkt_dev->skb->len;
3559 pkt_dev->clone_count = 0; /* reset counter */
3560 }
3561
3562 if (pkt_dev->delay && pkt_dev->last_ok)
3563 spin(pkt_dev, pkt_dev->next_tx);
3564
3565 if (pkt_dev->xmit_mode == M_NETIF_RECEIVE) {
3566 skb = pkt_dev->skb;
3567 skb->protocol = eth_type_trans(skb, skb->dev);
3568 if (skb_shared)
3569 refcount_add(burst, &skb->users);
3570 local_bh_disable();
3571 do {
3572 ret = netif_receive_skb(skb);
3573 if (ret == NET_RX_DROP)
3574 pkt_dev->errors++;
3575 pkt_dev->sofar++;
3576 pkt_dev->seq_num++;
3577 if (unlikely(!skb_shared)) {
3578 pkt_dev->skb = NULL;
3579 break;
3580 }
3581 if (refcount_read(&skb->users) != burst) {
3582 /* skb was queued by rps/rfs or taps,
3583 * so cannot reuse this skb
3584 */
3585 WARN_ON(refcount_sub_and_test(burst - 1, &skb->users));
3586 /* get out of the loop and wait
3587 * until skb is consumed
3588 */
3589 break;
3590 }
3591 /* skb was 'freed' by stack, so clean few
3592 * bits and reuse it
3593 */
3594 skb_reset_redirect(skb);
3595 } while (--burst > 0);
3596 goto out; /* Skips xmit_mode M_START_XMIT */
3597 } else if (pkt_dev->xmit_mode == M_QUEUE_XMIT) {
3598 local_bh_disable();
3599 if (skb_shared)
3600 refcount_inc(&pkt_dev->skb->users);
3601
3602 ret = dev_queue_xmit(pkt_dev->skb);
3603
3604 if (!skb_shared && dev_xmit_complete(ret))
3605 pkt_dev->skb = NULL;
3606
3607 switch (ret) {
3608 case NET_XMIT_SUCCESS:
3609 pkt_dev->sofar++;
3610 pkt_dev->seq_num++;
3611 pkt_dev->tx_bytes += pkt_dev->last_pkt_size;
3612 break;
3613 case NET_XMIT_DROP:
3614 case NET_XMIT_CN:
3615 /* These are all valid return codes for a qdisc but
3616 * indicate packets are being dropped or will likely
3617 * be dropped soon.
3618 */
3619 case NETDEV_TX_BUSY:
3620 /* qdisc may call dev_hard_start_xmit directly in cases
3621 * where no queues exist e.g. loopback device, virtual
3622 * devices, etc. In this case we need to handle
3623 * NETDEV_TX_ codes.
3624 */
3625 default:
3626 pkt_dev->errors++;
3627 net_info_ratelimited("%s xmit error: %d\n",
3628 pkt_dev->odevname, ret);
3629 break;
3630 }
3631 goto out;
3632 }
3633
3634 txq = skb_get_tx_queue(odev, pkt_dev->skb);
3635
3636 local_bh_disable();
3637
3638 HARD_TX_LOCK(odev, txq, smp_processor_id());
3639
3640 if (unlikely(netif_xmit_frozen_or_drv_stopped(txq))) {
3641 pkt_dev->last_ok = 0;
3642 goto unlock;
3643 }
3644 if (skb_shared)
3645 refcount_add(burst, &pkt_dev->skb->users);
3646
3647 xmit_more:
3648 ret = netdev_start_xmit(pkt_dev->skb, odev, txq, --burst > 0);
3649
3650 if (!skb_shared && dev_xmit_complete(ret))
3651 pkt_dev->skb = NULL;
3652
3653 switch (ret) {
3654 case NETDEV_TX_OK:
3655 pkt_dev->last_ok = 1;
3656 pkt_dev->sofar++;
3657 pkt_dev->seq_num++;
3658 pkt_dev->tx_bytes += pkt_dev->last_pkt_size;
3659 if (burst > 0 && !netif_xmit_frozen_or_drv_stopped(txq))
3660 goto xmit_more;
3661 break;
3662 case NET_XMIT_DROP:
3663 case NET_XMIT_CN:
3664 /* skb has been consumed */
3665 pkt_dev->errors++;
3666 break;
3667 default: /* Drivers are not supposed to return other values! */
3668 net_info_ratelimited("%s xmit error: %d\n",
3669 pkt_dev->odevname, ret);
3670 pkt_dev->errors++;
3671 fallthrough;
3672 case NETDEV_TX_BUSY:
3673 /* Retry it next time */
3674 if (skb_shared)
3675 refcount_dec(&pkt_dev->skb->users);
3676 pkt_dev->last_ok = 0;
3677 }
3678 if (unlikely(burst))
3679 WARN_ON(refcount_sub_and_test(burst, &pkt_dev->skb->users));
3680 unlock:
3681 HARD_TX_UNLOCK(odev, txq);
3682
3683 out:
3684 local_bh_enable();
3685
3686 /* If pkt_dev->count is zero, then run forever */
3687 if ((pkt_dev->count != 0) && (pkt_dev->sofar >= pkt_dev->count)) {
3688 if (pkt_dev->skb)
3689 pktgen_wait_for_skb(pkt_dev);
3690
3691 /* Done with this */
3692 pktgen_stop_device(pkt_dev);
3693 }
3694 }
3695
3696 /*
3697 * Main loop of the thread goes here
3698 */
3699
pktgen_thread_worker(void * arg)3700 static int pktgen_thread_worker(void *arg)
3701 {
3702 struct pktgen_thread *t = arg;
3703 struct pktgen_dev *pkt_dev = NULL;
3704 int cpu = t->cpu;
3705
3706 WARN_ON_ONCE(smp_processor_id() != cpu);
3707
3708 init_waitqueue_head(&t->queue);
3709 complete(&t->start_done);
3710
3711 pr_debug("starting pktgen/%d: pid=%d\n", cpu, task_pid_nr(current));
3712
3713 set_freezable();
3714
3715 while (!kthread_should_stop()) {
3716 pkt_dev = next_to_run(t);
3717
3718 if (unlikely(!pkt_dev && t->control == 0)) {
3719 if (t->net->pktgen_exiting)
3720 break;
3721 wait_event_freezable_timeout(t->queue,
3722 t->control != 0, HZ / 10);
3723 continue;
3724 }
3725
3726 if (likely(pkt_dev)) {
3727 pktgen_xmit(pkt_dev);
3728
3729 if (need_resched())
3730 pktgen_resched(pkt_dev);
3731 else
3732 cpu_relax();
3733 }
3734
3735 if (t->control & T_STOP) {
3736 pktgen_stop(t);
3737 t->control &= ~(T_STOP);
3738 }
3739
3740 if (t->control & T_RUN) {
3741 pktgen_run(t);
3742 t->control &= ~(T_RUN);
3743 }
3744
3745 if (t->control & T_REMDEVALL) {
3746 pktgen_rem_all_ifs(t);
3747 t->control &= ~(T_REMDEVALL);
3748 }
3749
3750 if (t->control & T_REMDEV) {
3751 pktgen_rem_one_if(t);
3752 t->control &= ~(T_REMDEV);
3753 }
3754
3755 try_to_freeze();
3756 }
3757
3758 pr_debug("%s stopping all device\n", t->tsk->comm);
3759 pktgen_stop(t);
3760
3761 pr_debug("%s removing all device\n", t->tsk->comm);
3762 pktgen_rem_all_ifs(t);
3763
3764 pr_debug("%s removing thread\n", t->tsk->comm);
3765 pktgen_rem_thread(t);
3766
3767 return 0;
3768 }
3769
pktgen_find_dev(struct pktgen_thread * t,const char * ifname,bool exact)3770 static struct pktgen_dev *pktgen_find_dev(struct pktgen_thread *t,
3771 const char *ifname, bool exact)
3772 {
3773 struct pktgen_dev *p, *pkt_dev = NULL;
3774 size_t len = strlen(ifname);
3775
3776 rcu_read_lock();
3777 list_for_each_entry_rcu(p, &t->if_list, list)
3778 if (strncmp(p->odevname, ifname, len) == 0) {
3779 if (p->odevname[len]) {
3780 if (exact || p->odevname[len] != '@')
3781 continue;
3782 }
3783 pkt_dev = p;
3784 break;
3785 }
3786
3787 rcu_read_unlock();
3788 pr_debug("find_dev(%s) returning %p\n", ifname, pkt_dev);
3789 return pkt_dev;
3790 }
3791
3792 /*
3793 * Adds a dev at front of if_list.
3794 */
3795
add_dev_to_thread(struct pktgen_thread * t,struct pktgen_dev * pkt_dev)3796 static int add_dev_to_thread(struct pktgen_thread *t,
3797 struct pktgen_dev *pkt_dev)
3798 {
3799 int rv = 0;
3800
3801 /* This function cannot be called concurrently, as its called
3802 * under pktgen_thread_lock mutex, but it can run from
3803 * userspace on another CPU than the kthread. The if_lock()
3804 * is used here to sync with concurrent instances of
3805 * _rem_dev_from_if_list() invoked via kthread, which is also
3806 * updating the if_list
3807 */
3808 if_lock(t);
3809
3810 if (pkt_dev->pg_thread) {
3811 pr_err("ERROR: already assigned to a thread\n");
3812 rv = -EBUSY;
3813 goto out;
3814 }
3815
3816 pkt_dev->running = 0;
3817 pkt_dev->pg_thread = t;
3818 list_add_rcu(&pkt_dev->list, &t->if_list);
3819
3820 out:
3821 if_unlock(t);
3822 return rv;
3823 }
3824
3825 /* Called under thread lock */
3826
pktgen_add_device(struct pktgen_thread * t,const char * ifname)3827 static int pktgen_add_device(struct pktgen_thread *t, const char *ifname)
3828 {
3829 struct pktgen_dev *pkt_dev;
3830 int err;
3831 int node = cpu_to_node(t->cpu);
3832
3833 /* We don't allow a device to be on several threads */
3834
3835 pkt_dev = __pktgen_NN_threads(t->net, ifname, FIND);
3836 if (pkt_dev) {
3837 pr_err("ERROR: interface already used\n");
3838 return -EBUSY;
3839 }
3840
3841 pkt_dev = kzalloc_node(sizeof(struct pktgen_dev), GFP_KERNEL, node);
3842 if (!pkt_dev)
3843 return -ENOMEM;
3844
3845 strscpy(pkt_dev->odevname, ifname);
3846 pkt_dev->flows = vzalloc_node(array_size(MAX_CFLOWS,
3847 sizeof(struct flow_state)),
3848 node);
3849 if (pkt_dev->flows == NULL) {
3850 kfree(pkt_dev);
3851 return -ENOMEM;
3852 }
3853
3854 pkt_dev->removal_mark = 0;
3855 pkt_dev->nfrags = 0;
3856 pkt_dev->delay = pg_delay_d;
3857 pkt_dev->count = pg_count_d;
3858 pkt_dev->sofar = 0;
3859 pkt_dev->udp_src_min = 9; /* sink port */
3860 pkt_dev->udp_src_max = 9;
3861 pkt_dev->udp_dst_min = 9;
3862 pkt_dev->udp_dst_max = 9;
3863 pkt_dev->vlan_p = 0;
3864 pkt_dev->vlan_cfi = 0;
3865 pkt_dev->vlan_id = 0xffff;
3866 pkt_dev->svlan_p = 0;
3867 pkt_dev->svlan_cfi = 0;
3868 pkt_dev->svlan_id = 0xffff;
3869 pkt_dev->burst = 1;
3870 pkt_dev->node = NUMA_NO_NODE;
3871 pkt_dev->flags = F_SHARED; /* SKB shared by default */
3872
3873 err = pktgen_setup_dev(t->net, pkt_dev, ifname);
3874 if (err)
3875 goto out1;
3876 if (pkt_dev->odev->priv_flags & IFF_TX_SKB_SHARING)
3877 pkt_dev->clone_skb = pg_clone_skb_d;
3878
3879 pkt_dev->entry = proc_create_data(ifname, 0600, t->net->proc_dir,
3880 &pktgen_if_proc_ops, pkt_dev);
3881 if (!pkt_dev->entry) {
3882 pr_err("cannot create %s/%s procfs entry\n",
3883 PG_PROC_DIR, ifname);
3884 err = -EINVAL;
3885 goto out2;
3886 }
3887 #ifdef CONFIG_XFRM
3888 pkt_dev->ipsmode = XFRM_MODE_TRANSPORT;
3889 pkt_dev->ipsproto = IPPROTO_ESP;
3890
3891 /* xfrm tunnel mode needs additional dst to extract outer
3892 * ip header protocol/ttl/id field, here create a phony one.
3893 * instead of looking for a valid rt, which definitely hurting
3894 * performance under such circumstance.
3895 */
3896 pkt_dev->dstops.family = AF_INET;
3897 pkt_dev->xdst.u.dst.dev = pkt_dev->odev;
3898 dst_init_metrics(&pkt_dev->xdst.u.dst, pktgen_dst_metrics, false);
3899 pkt_dev->xdst.child = &pkt_dev->xdst.u.dst;
3900 pkt_dev->xdst.u.dst.ops = &pkt_dev->dstops;
3901 #endif
3902
3903 return add_dev_to_thread(t, pkt_dev);
3904 out2:
3905 netdev_put(pkt_dev->odev, &pkt_dev->dev_tracker);
3906 out1:
3907 #ifdef CONFIG_XFRM
3908 free_SAs(pkt_dev);
3909 #endif
3910 vfree(pkt_dev->flows);
3911 kfree(pkt_dev);
3912 return err;
3913 }
3914
pktgen_create_thread(int cpu,struct pktgen_net * pn)3915 static int __net_init pktgen_create_thread(int cpu, struct pktgen_net *pn)
3916 {
3917 struct pktgen_thread *t;
3918 struct proc_dir_entry *pe;
3919 struct task_struct *p;
3920
3921 t = kzalloc_node(sizeof(struct pktgen_thread), GFP_KERNEL,
3922 cpu_to_node(cpu));
3923 if (!t) {
3924 pr_err("ERROR: out of memory, can't create new thread\n");
3925 return -ENOMEM;
3926 }
3927
3928 mutex_init(&t->if_lock);
3929 t->cpu = cpu;
3930
3931 INIT_LIST_HEAD(&t->if_list);
3932
3933 list_add_tail(&t->th_list, &pn->pktgen_threads);
3934 init_completion(&t->start_done);
3935
3936 p = kthread_create_on_cpu(pktgen_thread_worker, t, cpu, "kpktgend_%d");
3937 if (IS_ERR(p)) {
3938 pr_err("kthread_create_on_node() failed for cpu %d\n", t->cpu);
3939 list_del(&t->th_list);
3940 kfree(t);
3941 return PTR_ERR(p);
3942 }
3943
3944 t->tsk = p;
3945
3946 pe = proc_create_data(t->tsk->comm, 0600, pn->proc_dir,
3947 &pktgen_thread_proc_ops, t);
3948 if (!pe) {
3949 pr_err("cannot create %s/%s procfs entry\n",
3950 PG_PROC_DIR, t->tsk->comm);
3951 kthread_stop(p);
3952 list_del(&t->th_list);
3953 kfree(t);
3954 return -EINVAL;
3955 }
3956
3957 t->net = pn;
3958 get_task_struct(p);
3959 wake_up_process(p);
3960 wait_for_completion(&t->start_done);
3961
3962 return 0;
3963 }
3964
3965 /*
3966 * Removes a device from the thread if_list.
3967 */
_rem_dev_from_if_list(struct pktgen_thread * t,struct pktgen_dev * pkt_dev)3968 static void _rem_dev_from_if_list(struct pktgen_thread *t,
3969 struct pktgen_dev *pkt_dev)
3970 {
3971 struct list_head *q, *n;
3972 struct pktgen_dev *p;
3973
3974 if_lock(t);
3975 list_for_each_safe(q, n, &t->if_list) {
3976 p = list_entry(q, struct pktgen_dev, list);
3977 if (p == pkt_dev)
3978 list_del_rcu(&p->list);
3979 }
3980 if_unlock(t);
3981 }
3982
pktgen_remove_device(struct pktgen_thread * t,struct pktgen_dev * pkt_dev)3983 static int pktgen_remove_device(struct pktgen_thread *t,
3984 struct pktgen_dev *pkt_dev)
3985 {
3986 pr_debug("remove_device pkt_dev=%p\n", pkt_dev);
3987
3988 if (pkt_dev->running) {
3989 pr_warn("WARNING: trying to remove a running interface, stopping it now\n");
3990 pktgen_stop_device(pkt_dev);
3991 }
3992
3993 /* Dis-associate from the interface */
3994
3995 if (pkt_dev->odev) {
3996 netdev_put(pkt_dev->odev, &pkt_dev->dev_tracker);
3997 pkt_dev->odev = NULL;
3998 }
3999
4000 /* Remove proc before if_list entry, because add_device uses
4001 * list to determine if interface already exist, avoid race
4002 * with proc_create_data()
4003 */
4004 proc_remove(pkt_dev->entry);
4005
4006 /* And update the thread if_list */
4007 _rem_dev_from_if_list(t, pkt_dev);
4008
4009 #ifdef CONFIG_XFRM
4010 free_SAs(pkt_dev);
4011 #endif
4012 vfree(pkt_dev->flows);
4013 if (pkt_dev->page)
4014 put_page(pkt_dev->page);
4015 kfree_rcu(pkt_dev, rcu);
4016 return 0;
4017 }
4018
pg_net_init(struct net * net)4019 static int __net_init pg_net_init(struct net *net)
4020 {
4021 struct pktgen_net *pn = net_generic(net, pg_net_id);
4022 struct proc_dir_entry *pe;
4023 int cpu, ret = 0;
4024
4025 pn->net = net;
4026 INIT_LIST_HEAD(&pn->pktgen_threads);
4027 pn->pktgen_exiting = false;
4028 pn->proc_dir = proc_mkdir(PG_PROC_DIR, pn->net->proc_net);
4029 if (!pn->proc_dir) {
4030 pr_warn("cannot create /proc/net/%s\n", PG_PROC_DIR);
4031 return -ENODEV;
4032 }
4033 pe = proc_create(PGCTRL, 0600, pn->proc_dir, &pktgen_proc_ops);
4034 if (pe == NULL) {
4035 pr_err("cannot create %s procfs entry\n", PGCTRL);
4036 ret = -EINVAL;
4037 goto remove;
4038 }
4039
4040 cpus_read_lock();
4041 for_each_online_cpu(cpu) {
4042 int err;
4043
4044 err = pktgen_create_thread(cpu, pn);
4045 if (err)
4046 pr_warn("Cannot create thread for cpu %d (%d)\n",
4047 cpu, err);
4048 }
4049 cpus_read_unlock();
4050
4051 if (list_empty(&pn->pktgen_threads)) {
4052 pr_err("Initialization failed for all threads\n");
4053 ret = -ENODEV;
4054 goto remove_entry;
4055 }
4056
4057 return 0;
4058
4059 remove_entry:
4060 remove_proc_entry(PGCTRL, pn->proc_dir);
4061 remove:
4062 remove_proc_entry(PG_PROC_DIR, pn->net->proc_net);
4063 return ret;
4064 }
4065
pg_net_exit(struct net * net)4066 static void __net_exit pg_net_exit(struct net *net)
4067 {
4068 struct pktgen_net *pn = net_generic(net, pg_net_id);
4069 struct pktgen_thread *t;
4070 struct list_head *q, *n;
4071 LIST_HEAD(list);
4072
4073 /* Stop all interfaces & threads */
4074 pn->pktgen_exiting = true;
4075
4076 mutex_lock(&pktgen_thread_lock);
4077 list_splice_init(&pn->pktgen_threads, &list);
4078 mutex_unlock(&pktgen_thread_lock);
4079
4080 list_for_each_safe(q, n, &list) {
4081 t = list_entry(q, struct pktgen_thread, th_list);
4082 list_del(&t->th_list);
4083 kthread_stop_put(t->tsk);
4084 kfree(t);
4085 }
4086
4087 remove_proc_entry(PGCTRL, pn->proc_dir);
4088 remove_proc_entry(PG_PROC_DIR, pn->net->proc_net);
4089 }
4090
4091 static struct pernet_operations pg_net_ops = {
4092 .init = pg_net_init,
4093 .exit = pg_net_exit,
4094 .id = &pg_net_id,
4095 .size = sizeof(struct pktgen_net),
4096 };
4097
pg_init(void)4098 static int __init pg_init(void)
4099 {
4100 int ret = 0;
4101
4102 pr_info("%s", version);
4103 ret = register_pernet_subsys(&pg_net_ops);
4104 if (ret)
4105 return ret;
4106 ret = register_netdevice_notifier(&pktgen_notifier_block);
4107 if (ret)
4108 unregister_pernet_subsys(&pg_net_ops);
4109
4110 return ret;
4111 }
4112
pg_cleanup(void)4113 static void __exit pg_cleanup(void)
4114 {
4115 unregister_netdevice_notifier(&pktgen_notifier_block);
4116 unregister_pernet_subsys(&pg_net_ops);
4117 /* Don't need rcu_barrier() due to use of kfree_rcu() */
4118 }
4119
4120 module_init(pg_init);
4121 module_exit(pg_cleanup);
4122
4123 MODULE_AUTHOR("Robert Olsson <robert.olsson@its.uu.se>");
4124 MODULE_DESCRIPTION("Packet Generator tool");
4125 MODULE_LICENSE("GPL");
4126 MODULE_VERSION(VERSION);
4127 module_param(pg_count_d, int, 0);
4128 MODULE_PARM_DESC(pg_count_d, "Default number of packets to inject");
4129 module_param(pg_delay_d, int, 0);
4130 MODULE_PARM_DESC(pg_delay_d, "Default delay between packets (nanoseconds)");
4131 module_param(pg_clone_skb_d, int, 0);
4132 MODULE_PARM_DESC(pg_clone_skb_d, "Default number of copies of the same packet");
4133 module_param(debug, int, 0);
4134 MODULE_PARM_DESC(debug, "Enable debugging of pktgen module");
4135