1 // SPDX-License-Identifier: GPL-2.0-only
2
3 #include <linux/netdevice.h>
4 #include <linux/notifier.h>
5 #include <linux/rtnetlink.h>
6 #include <net/busy_poll.h>
7 #include <net/net_namespace.h>
8 #include <net/netdev_queues.h>
9 #include <net/netdev_rx_queue.h>
10 #include <net/sock.h>
11 #include <net/xdp.h>
12 #include <net/xdp_sock.h>
13 #include <net/page_pool/memory_provider.h>
14
15 #include "dev.h"
16 #include "devmem.h"
17 #include "netdev-genl-gen.h"
18
19 struct netdev_nl_dump_ctx {
20 unsigned long ifindex;
21 unsigned int rxq_idx;
22 unsigned int txq_idx;
23 unsigned int napi_id;
24 };
25
netdev_dump_ctx(struct netlink_callback * cb)26 static struct netdev_nl_dump_ctx *netdev_dump_ctx(struct netlink_callback *cb)
27 {
28 NL_ASSERT_CTX_FITS(struct netdev_nl_dump_ctx);
29
30 return (struct netdev_nl_dump_ctx *)cb->ctx;
31 }
32
33 static int
netdev_nl_dev_fill(struct net_device * netdev,struct sk_buff * rsp,const struct genl_info * info)34 netdev_nl_dev_fill(struct net_device *netdev, struct sk_buff *rsp,
35 const struct genl_info *info)
36 {
37 u64 xsk_features = 0;
38 u64 xdp_rx_meta = 0;
39 void *hdr;
40
41 hdr = genlmsg_iput(rsp, info);
42 if (!hdr)
43 return -EMSGSIZE;
44
45 #define XDP_METADATA_KFUNC(_, flag, __, xmo) \
46 if (netdev->xdp_metadata_ops && netdev->xdp_metadata_ops->xmo) \
47 xdp_rx_meta |= flag;
48 XDP_METADATA_KFUNC_xxx
49 #undef XDP_METADATA_KFUNC
50
51 if (netdev->xsk_tx_metadata_ops) {
52 if (netdev->xsk_tx_metadata_ops->tmo_fill_timestamp)
53 xsk_features |= NETDEV_XSK_FLAGS_TX_TIMESTAMP;
54 if (netdev->xsk_tx_metadata_ops->tmo_request_checksum)
55 xsk_features |= NETDEV_XSK_FLAGS_TX_CHECKSUM;
56 if (netdev->xsk_tx_metadata_ops->tmo_request_launch_time)
57 xsk_features |= NETDEV_XSK_FLAGS_TX_LAUNCH_TIME_FIFO;
58 }
59
60 if (nla_put_u32(rsp, NETDEV_A_DEV_IFINDEX, netdev->ifindex) ||
61 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_FEATURES,
62 netdev->xdp_features, NETDEV_A_DEV_PAD) ||
63 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_RX_METADATA_FEATURES,
64 xdp_rx_meta, NETDEV_A_DEV_PAD) ||
65 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XSK_FEATURES,
66 xsk_features, NETDEV_A_DEV_PAD))
67 goto err_cancel_msg;
68
69 if (netdev->xdp_features & NETDEV_XDP_ACT_XSK_ZEROCOPY) {
70 if (nla_put_u32(rsp, NETDEV_A_DEV_XDP_ZC_MAX_SEGS,
71 netdev->xdp_zc_max_segs))
72 goto err_cancel_msg;
73 }
74
75 genlmsg_end(rsp, hdr);
76
77 return 0;
78
79 err_cancel_msg:
80 genlmsg_cancel(rsp, hdr);
81 return -EMSGSIZE;
82 }
83
84 static void
netdev_genl_dev_notify(struct net_device * netdev,int cmd)85 netdev_genl_dev_notify(struct net_device *netdev, int cmd)
86 {
87 struct genl_info info;
88 struct sk_buff *ntf;
89
90 if (!genl_has_listeners(&netdev_nl_family, dev_net(netdev),
91 NETDEV_NLGRP_MGMT))
92 return;
93
94 genl_info_init_ntf(&info, &netdev_nl_family, cmd);
95
96 ntf = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
97 if (!ntf)
98 return;
99
100 if (netdev_nl_dev_fill(netdev, ntf, &info)) {
101 nlmsg_free(ntf);
102 return;
103 }
104
105 genlmsg_multicast_netns(&netdev_nl_family, dev_net(netdev), ntf,
106 0, NETDEV_NLGRP_MGMT, GFP_KERNEL);
107 }
108
netdev_nl_dev_get_doit(struct sk_buff * skb,struct genl_info * info)109 int netdev_nl_dev_get_doit(struct sk_buff *skb, struct genl_info *info)
110 {
111 struct net_device *netdev;
112 struct sk_buff *rsp;
113 u32 ifindex;
114 int err;
115
116 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX))
117 return -EINVAL;
118
119 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]);
120
121 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
122 if (!rsp)
123 return -ENOMEM;
124
125 rtnl_lock();
126
127 netdev = __dev_get_by_index(genl_info_net(info), ifindex);
128 if (netdev)
129 err = netdev_nl_dev_fill(netdev, rsp, info);
130 else
131 err = -ENODEV;
132
133 rtnl_unlock();
134
135 if (err)
136 goto err_free_msg;
137
138 return genlmsg_reply(rsp, info);
139
140 err_free_msg:
141 nlmsg_free(rsp);
142 return err;
143 }
144
netdev_nl_dev_get_dumpit(struct sk_buff * skb,struct netlink_callback * cb)145 int netdev_nl_dev_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
146 {
147 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
148 struct net *net = sock_net(skb->sk);
149 struct net_device *netdev;
150 int err = 0;
151
152 rtnl_lock();
153 for_each_netdev_dump(net, netdev, ctx->ifindex) {
154 err = netdev_nl_dev_fill(netdev, skb, genl_info_dump(cb));
155 if (err < 0)
156 break;
157 }
158 rtnl_unlock();
159
160 return err;
161 }
162
163 static int
netdev_nl_napi_fill_one(struct sk_buff * rsp,struct napi_struct * napi,const struct genl_info * info)164 netdev_nl_napi_fill_one(struct sk_buff *rsp, struct napi_struct *napi,
165 const struct genl_info *info)
166 {
167 unsigned long irq_suspend_timeout;
168 unsigned long gro_flush_timeout;
169 u32 napi_defer_hard_irqs;
170 void *hdr;
171 pid_t pid;
172
173 if (!napi->dev->up)
174 return 0;
175
176 hdr = genlmsg_iput(rsp, info);
177 if (!hdr)
178 return -EMSGSIZE;
179
180 if (nla_put_u32(rsp, NETDEV_A_NAPI_ID, napi->napi_id))
181 goto nla_put_failure;
182
183 if (nla_put_u32(rsp, NETDEV_A_NAPI_IFINDEX, napi->dev->ifindex))
184 goto nla_put_failure;
185
186 if (napi->irq >= 0 && nla_put_u32(rsp, NETDEV_A_NAPI_IRQ, napi->irq))
187 goto nla_put_failure;
188
189 if (napi->thread) {
190 pid = task_pid_nr(napi->thread);
191 if (nla_put_u32(rsp, NETDEV_A_NAPI_PID, pid))
192 goto nla_put_failure;
193 }
194
195 napi_defer_hard_irqs = napi_get_defer_hard_irqs(napi);
196 if (nla_put_s32(rsp, NETDEV_A_NAPI_DEFER_HARD_IRQS,
197 napi_defer_hard_irqs))
198 goto nla_put_failure;
199
200 irq_suspend_timeout = napi_get_irq_suspend_timeout(napi);
201 if (nla_put_uint(rsp, NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT,
202 irq_suspend_timeout))
203 goto nla_put_failure;
204
205 gro_flush_timeout = napi_get_gro_flush_timeout(napi);
206 if (nla_put_uint(rsp, NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT,
207 gro_flush_timeout))
208 goto nla_put_failure;
209
210 genlmsg_end(rsp, hdr);
211
212 return 0;
213
214 nla_put_failure:
215 genlmsg_cancel(rsp, hdr);
216 return -EMSGSIZE;
217 }
218
netdev_nl_napi_get_doit(struct sk_buff * skb,struct genl_info * info)219 int netdev_nl_napi_get_doit(struct sk_buff *skb, struct genl_info *info)
220 {
221 struct napi_struct *napi;
222 struct sk_buff *rsp;
223 u32 napi_id;
224 int err;
225
226 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_NAPI_ID))
227 return -EINVAL;
228
229 napi_id = nla_get_u32(info->attrs[NETDEV_A_NAPI_ID]);
230
231 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
232 if (!rsp)
233 return -ENOMEM;
234
235 napi = netdev_napi_by_id_lock(genl_info_net(info), napi_id);
236 if (napi) {
237 err = netdev_nl_napi_fill_one(rsp, napi, info);
238 netdev_unlock(napi->dev);
239 } else {
240 NL_SET_BAD_ATTR(info->extack, info->attrs[NETDEV_A_NAPI_ID]);
241 err = -ENOENT;
242 }
243
244 if (err) {
245 goto err_free_msg;
246 } else if (!rsp->len) {
247 err = -ENOENT;
248 goto err_free_msg;
249 }
250
251 return genlmsg_reply(rsp, info);
252
253 err_free_msg:
254 nlmsg_free(rsp);
255 return err;
256 }
257
258 static int
netdev_nl_napi_dump_one(struct net_device * netdev,struct sk_buff * rsp,const struct genl_info * info,struct netdev_nl_dump_ctx * ctx)259 netdev_nl_napi_dump_one(struct net_device *netdev, struct sk_buff *rsp,
260 const struct genl_info *info,
261 struct netdev_nl_dump_ctx *ctx)
262 {
263 struct napi_struct *napi;
264 unsigned int prev_id;
265 int err = 0;
266
267 if (!netdev->up)
268 return err;
269
270 prev_id = UINT_MAX;
271 list_for_each_entry(napi, &netdev->napi_list, dev_list) {
272 if (!napi_id_valid(napi->napi_id))
273 continue;
274
275 /* Dump continuation below depends on the list being sorted */
276 WARN_ON_ONCE(napi->napi_id >= prev_id);
277 prev_id = napi->napi_id;
278
279 if (ctx->napi_id && napi->napi_id >= ctx->napi_id)
280 continue;
281
282 err = netdev_nl_napi_fill_one(rsp, napi, info);
283 if (err)
284 return err;
285 ctx->napi_id = napi->napi_id;
286 }
287 return err;
288 }
289
netdev_nl_napi_get_dumpit(struct sk_buff * skb,struct netlink_callback * cb)290 int netdev_nl_napi_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
291 {
292 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
293 const struct genl_info *info = genl_info_dump(cb);
294 struct net *net = sock_net(skb->sk);
295 struct net_device *netdev;
296 u32 ifindex = 0;
297 int err = 0;
298
299 if (info->attrs[NETDEV_A_NAPI_IFINDEX])
300 ifindex = nla_get_u32(info->attrs[NETDEV_A_NAPI_IFINDEX]);
301
302 if (ifindex) {
303 netdev = netdev_get_by_index_lock(net, ifindex);
304 if (netdev) {
305 err = netdev_nl_napi_dump_one(netdev, skb, info, ctx);
306 netdev_unlock(netdev);
307 } else {
308 err = -ENODEV;
309 }
310 } else {
311 for_each_netdev_lock_scoped(net, netdev, ctx->ifindex) {
312 err = netdev_nl_napi_dump_one(netdev, skb, info, ctx);
313 if (err < 0)
314 break;
315 ctx->napi_id = 0;
316 }
317 }
318
319 return err;
320 }
321
322 static int
netdev_nl_napi_set_config(struct napi_struct * napi,struct genl_info * info)323 netdev_nl_napi_set_config(struct napi_struct *napi, struct genl_info *info)
324 {
325 u64 irq_suspend_timeout = 0;
326 u64 gro_flush_timeout = 0;
327 u32 defer = 0;
328
329 if (info->attrs[NETDEV_A_NAPI_DEFER_HARD_IRQS]) {
330 defer = nla_get_u32(info->attrs[NETDEV_A_NAPI_DEFER_HARD_IRQS]);
331 napi_set_defer_hard_irqs(napi, defer);
332 }
333
334 if (info->attrs[NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT]) {
335 irq_suspend_timeout = nla_get_uint(info->attrs[NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT]);
336 napi_set_irq_suspend_timeout(napi, irq_suspend_timeout);
337 }
338
339 if (info->attrs[NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT]) {
340 gro_flush_timeout = nla_get_uint(info->attrs[NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT]);
341 napi_set_gro_flush_timeout(napi, gro_flush_timeout);
342 }
343
344 return 0;
345 }
346
netdev_nl_napi_set_doit(struct sk_buff * skb,struct genl_info * info)347 int netdev_nl_napi_set_doit(struct sk_buff *skb, struct genl_info *info)
348 {
349 struct napi_struct *napi;
350 unsigned int napi_id;
351 int err;
352
353 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_NAPI_ID))
354 return -EINVAL;
355
356 napi_id = nla_get_u32(info->attrs[NETDEV_A_NAPI_ID]);
357
358 napi = netdev_napi_by_id_lock(genl_info_net(info), napi_id);
359 if (napi) {
360 err = netdev_nl_napi_set_config(napi, info);
361 netdev_unlock(napi->dev);
362 } else {
363 NL_SET_BAD_ATTR(info->extack, info->attrs[NETDEV_A_NAPI_ID]);
364 err = -ENOENT;
365 }
366
367 return err;
368 }
369
nla_put_napi_id(struct sk_buff * skb,const struct napi_struct * napi)370 static int nla_put_napi_id(struct sk_buff *skb, const struct napi_struct *napi)
371 {
372 if (napi && napi_id_valid(napi->napi_id))
373 return nla_put_u32(skb, NETDEV_A_QUEUE_NAPI_ID, napi->napi_id);
374 return 0;
375 }
376
377 static int
netdev_nl_queue_fill_one(struct sk_buff * rsp,struct net_device * netdev,u32 q_idx,u32 q_type,const struct genl_info * info)378 netdev_nl_queue_fill_one(struct sk_buff *rsp, struct net_device *netdev,
379 u32 q_idx, u32 q_type, const struct genl_info *info)
380 {
381 struct pp_memory_provider_params *params;
382 struct netdev_rx_queue *rxq;
383 struct netdev_queue *txq;
384 void *hdr;
385
386 hdr = genlmsg_iput(rsp, info);
387 if (!hdr)
388 return -EMSGSIZE;
389
390 if (nla_put_u32(rsp, NETDEV_A_QUEUE_ID, q_idx) ||
391 nla_put_u32(rsp, NETDEV_A_QUEUE_TYPE, q_type) ||
392 nla_put_u32(rsp, NETDEV_A_QUEUE_IFINDEX, netdev->ifindex))
393 goto nla_put_failure;
394
395 switch (q_type) {
396 case NETDEV_QUEUE_TYPE_RX:
397 rxq = __netif_get_rx_queue(netdev, q_idx);
398 if (nla_put_napi_id(rsp, rxq->napi))
399 goto nla_put_failure;
400
401 params = &rxq->mp_params;
402 if (params->mp_ops &&
403 params->mp_ops->nl_fill(params->mp_priv, rsp, rxq))
404 goto nla_put_failure;
405 #ifdef CONFIG_XDP_SOCKETS
406 if (rxq->pool)
407 if (nla_put_empty_nest(rsp, NETDEV_A_QUEUE_XSK))
408 goto nla_put_failure;
409 #endif
410
411 break;
412 case NETDEV_QUEUE_TYPE_TX:
413 txq = netdev_get_tx_queue(netdev, q_idx);
414 if (nla_put_napi_id(rsp, txq->napi))
415 goto nla_put_failure;
416 #ifdef CONFIG_XDP_SOCKETS
417 if (txq->pool)
418 if (nla_put_empty_nest(rsp, NETDEV_A_QUEUE_XSK))
419 goto nla_put_failure;
420 #endif
421 break;
422 }
423
424 genlmsg_end(rsp, hdr);
425
426 return 0;
427
428 nla_put_failure:
429 genlmsg_cancel(rsp, hdr);
430 return -EMSGSIZE;
431 }
432
netdev_nl_queue_validate(struct net_device * netdev,u32 q_id,u32 q_type)433 static int netdev_nl_queue_validate(struct net_device *netdev, u32 q_id,
434 u32 q_type)
435 {
436 switch (q_type) {
437 case NETDEV_QUEUE_TYPE_RX:
438 if (q_id >= netdev->real_num_rx_queues)
439 return -EINVAL;
440 return 0;
441 case NETDEV_QUEUE_TYPE_TX:
442 if (q_id >= netdev->real_num_tx_queues)
443 return -EINVAL;
444 }
445 return 0;
446 }
447
448 static int
netdev_nl_queue_fill(struct sk_buff * rsp,struct net_device * netdev,u32 q_idx,u32 q_type,const struct genl_info * info)449 netdev_nl_queue_fill(struct sk_buff *rsp, struct net_device *netdev, u32 q_idx,
450 u32 q_type, const struct genl_info *info)
451 {
452 int err;
453
454 if (!netdev->up)
455 return -ENOENT;
456
457 err = netdev_nl_queue_validate(netdev, q_idx, q_type);
458 if (err)
459 return err;
460
461 return netdev_nl_queue_fill_one(rsp, netdev, q_idx, q_type, info);
462 }
463
netdev_nl_queue_get_doit(struct sk_buff * skb,struct genl_info * info)464 int netdev_nl_queue_get_doit(struct sk_buff *skb, struct genl_info *info)
465 {
466 u32 q_id, q_type, ifindex;
467 struct net_device *netdev;
468 struct sk_buff *rsp;
469 int err;
470
471 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_ID) ||
472 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_TYPE) ||
473 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_IFINDEX))
474 return -EINVAL;
475
476 q_id = nla_get_u32(info->attrs[NETDEV_A_QUEUE_ID]);
477 q_type = nla_get_u32(info->attrs[NETDEV_A_QUEUE_TYPE]);
478 ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]);
479
480 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
481 if (!rsp)
482 return -ENOMEM;
483
484 rtnl_lock();
485
486 netdev = netdev_get_by_index_lock(genl_info_net(info), ifindex);
487 if (netdev) {
488 err = netdev_nl_queue_fill(rsp, netdev, q_id, q_type, info);
489 netdev_unlock(netdev);
490 } else {
491 err = -ENODEV;
492 }
493
494 rtnl_unlock();
495
496 if (err)
497 goto err_free_msg;
498
499 return genlmsg_reply(rsp, info);
500
501 err_free_msg:
502 nlmsg_free(rsp);
503 return err;
504 }
505
506 static int
netdev_nl_queue_dump_one(struct net_device * netdev,struct sk_buff * rsp,const struct genl_info * info,struct netdev_nl_dump_ctx * ctx)507 netdev_nl_queue_dump_one(struct net_device *netdev, struct sk_buff *rsp,
508 const struct genl_info *info,
509 struct netdev_nl_dump_ctx *ctx)
510 {
511 int err = 0;
512
513 if (!netdev->up)
514 return err;
515
516 for (; ctx->rxq_idx < netdev->real_num_rx_queues; ctx->rxq_idx++) {
517 err = netdev_nl_queue_fill_one(rsp, netdev, ctx->rxq_idx,
518 NETDEV_QUEUE_TYPE_RX, info);
519 if (err)
520 return err;
521 }
522 for (; ctx->txq_idx < netdev->real_num_tx_queues; ctx->txq_idx++) {
523 err = netdev_nl_queue_fill_one(rsp, netdev, ctx->txq_idx,
524 NETDEV_QUEUE_TYPE_TX, info);
525 if (err)
526 return err;
527 }
528
529 return err;
530 }
531
netdev_nl_queue_get_dumpit(struct sk_buff * skb,struct netlink_callback * cb)532 int netdev_nl_queue_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
533 {
534 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
535 const struct genl_info *info = genl_info_dump(cb);
536 struct net *net = sock_net(skb->sk);
537 struct net_device *netdev;
538 u32 ifindex = 0;
539 int err = 0;
540
541 if (info->attrs[NETDEV_A_QUEUE_IFINDEX])
542 ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]);
543
544 rtnl_lock();
545 if (ifindex) {
546 netdev = netdev_get_by_index_lock(net, ifindex);
547 if (netdev) {
548 err = netdev_nl_queue_dump_one(netdev, skb, info, ctx);
549 netdev_unlock(netdev);
550 } else {
551 err = -ENODEV;
552 }
553 } else {
554 for_each_netdev_lock_scoped(net, netdev, ctx->ifindex) {
555 err = netdev_nl_queue_dump_one(netdev, skb, info, ctx);
556 if (err < 0)
557 break;
558 ctx->rxq_idx = 0;
559 ctx->txq_idx = 0;
560 }
561 }
562 rtnl_unlock();
563
564 return err;
565 }
566
567 #define NETDEV_STAT_NOT_SET (~0ULL)
568
netdev_nl_stats_add(void * _sum,const void * _add,size_t size)569 static void netdev_nl_stats_add(void *_sum, const void *_add, size_t size)
570 {
571 const u64 *add = _add;
572 u64 *sum = _sum;
573
574 while (size) {
575 if (*add != NETDEV_STAT_NOT_SET && *sum != NETDEV_STAT_NOT_SET)
576 *sum += *add;
577 sum++;
578 add++;
579 size -= 8;
580 }
581 }
582
netdev_stat_put(struct sk_buff * rsp,unsigned int attr_id,u64 value)583 static int netdev_stat_put(struct sk_buff *rsp, unsigned int attr_id, u64 value)
584 {
585 if (value == NETDEV_STAT_NOT_SET)
586 return 0;
587 return nla_put_uint(rsp, attr_id, value);
588 }
589
590 static int
netdev_nl_stats_write_rx(struct sk_buff * rsp,struct netdev_queue_stats_rx * rx)591 netdev_nl_stats_write_rx(struct sk_buff *rsp, struct netdev_queue_stats_rx *rx)
592 {
593 if (netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_PACKETS, rx->packets) ||
594 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_BYTES, rx->bytes) ||
595 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_ALLOC_FAIL, rx->alloc_fail) ||
596 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROPS, rx->hw_drops) ||
597 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROP_OVERRUNS, rx->hw_drop_overruns) ||
598 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_COMPLETE, rx->csum_complete) ||
599 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_UNNECESSARY, rx->csum_unnecessary) ||
600 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_NONE, rx->csum_none) ||
601 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_BAD, rx->csum_bad) ||
602 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_PACKETS, rx->hw_gro_packets) ||
603 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_BYTES, rx->hw_gro_bytes) ||
604 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_WIRE_PACKETS, rx->hw_gro_wire_packets) ||
605 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_WIRE_BYTES, rx->hw_gro_wire_bytes) ||
606 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROP_RATELIMITS, rx->hw_drop_ratelimits))
607 return -EMSGSIZE;
608 return 0;
609 }
610
611 static int
netdev_nl_stats_write_tx(struct sk_buff * rsp,struct netdev_queue_stats_tx * tx)612 netdev_nl_stats_write_tx(struct sk_buff *rsp, struct netdev_queue_stats_tx *tx)
613 {
614 if (netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_PACKETS, tx->packets) ||
615 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_BYTES, tx->bytes) ||
616 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROPS, tx->hw_drops) ||
617 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROP_ERRORS, tx->hw_drop_errors) ||
618 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_CSUM_NONE, tx->csum_none) ||
619 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_NEEDS_CSUM, tx->needs_csum) ||
620 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_PACKETS, tx->hw_gso_packets) ||
621 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_BYTES, tx->hw_gso_bytes) ||
622 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_WIRE_PACKETS, tx->hw_gso_wire_packets) ||
623 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_WIRE_BYTES, tx->hw_gso_wire_bytes) ||
624 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROP_RATELIMITS, tx->hw_drop_ratelimits) ||
625 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_STOP, tx->stop) ||
626 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_WAKE, tx->wake))
627 return -EMSGSIZE;
628 return 0;
629 }
630
631 static int
netdev_nl_stats_queue(struct net_device * netdev,struct sk_buff * rsp,u32 q_type,int i,const struct genl_info * info)632 netdev_nl_stats_queue(struct net_device *netdev, struct sk_buff *rsp,
633 u32 q_type, int i, const struct genl_info *info)
634 {
635 const struct netdev_stat_ops *ops = netdev->stat_ops;
636 struct netdev_queue_stats_rx rx;
637 struct netdev_queue_stats_tx tx;
638 void *hdr;
639
640 hdr = genlmsg_iput(rsp, info);
641 if (!hdr)
642 return -EMSGSIZE;
643 if (nla_put_u32(rsp, NETDEV_A_QSTATS_IFINDEX, netdev->ifindex) ||
644 nla_put_u32(rsp, NETDEV_A_QSTATS_QUEUE_TYPE, q_type) ||
645 nla_put_u32(rsp, NETDEV_A_QSTATS_QUEUE_ID, i))
646 goto nla_put_failure;
647
648 switch (q_type) {
649 case NETDEV_QUEUE_TYPE_RX:
650 memset(&rx, 0xff, sizeof(rx));
651 ops->get_queue_stats_rx(netdev, i, &rx);
652 if (!memchr_inv(&rx, 0xff, sizeof(rx)))
653 goto nla_cancel;
654 if (netdev_nl_stats_write_rx(rsp, &rx))
655 goto nla_put_failure;
656 break;
657 case NETDEV_QUEUE_TYPE_TX:
658 memset(&tx, 0xff, sizeof(tx));
659 ops->get_queue_stats_tx(netdev, i, &tx);
660 if (!memchr_inv(&tx, 0xff, sizeof(tx)))
661 goto nla_cancel;
662 if (netdev_nl_stats_write_tx(rsp, &tx))
663 goto nla_put_failure;
664 break;
665 }
666
667 genlmsg_end(rsp, hdr);
668 return 0;
669
670 nla_cancel:
671 genlmsg_cancel(rsp, hdr);
672 return 0;
673 nla_put_failure:
674 genlmsg_cancel(rsp, hdr);
675 return -EMSGSIZE;
676 }
677
678 static int
netdev_nl_stats_by_queue(struct net_device * netdev,struct sk_buff * rsp,const struct genl_info * info,struct netdev_nl_dump_ctx * ctx)679 netdev_nl_stats_by_queue(struct net_device *netdev, struct sk_buff *rsp,
680 const struct genl_info *info,
681 struct netdev_nl_dump_ctx *ctx)
682 {
683 const struct netdev_stat_ops *ops = netdev->stat_ops;
684 int i, err;
685
686 if (!(netdev->flags & IFF_UP))
687 return 0;
688
689 i = ctx->rxq_idx;
690 while (ops->get_queue_stats_rx && i < netdev->real_num_rx_queues) {
691 err = netdev_nl_stats_queue(netdev, rsp, NETDEV_QUEUE_TYPE_RX,
692 i, info);
693 if (err)
694 return err;
695 ctx->rxq_idx = ++i;
696 }
697 i = ctx->txq_idx;
698 while (ops->get_queue_stats_tx && i < netdev->real_num_tx_queues) {
699 err = netdev_nl_stats_queue(netdev, rsp, NETDEV_QUEUE_TYPE_TX,
700 i, info);
701 if (err)
702 return err;
703 ctx->txq_idx = ++i;
704 }
705
706 ctx->rxq_idx = 0;
707 ctx->txq_idx = 0;
708 return 0;
709 }
710
711 /**
712 * netdev_stat_queue_sum() - add up queue stats from range of queues
713 * @netdev: net_device
714 * @rx_start: index of the first Rx queue to query
715 * @rx_end: index after the last Rx queue (first *not* to query)
716 * @rx_sum: output Rx stats, should be already initialized
717 * @tx_start: index of the first Tx queue to query
718 * @tx_end: index after the last Tx queue (first *not* to query)
719 * @tx_sum: output Tx stats, should be already initialized
720 *
721 * Add stats from [start, end) range of queue IDs to *x_sum structs.
722 * The sum structs must be already initialized. Usually this
723 * helper is invoked from the .get_base_stats callbacks of drivers
724 * to account for stats of disabled queues. In that case the ranges
725 * are usually [netdev->real_num_*x_queues, netdev->num_*x_queues).
726 */
netdev_stat_queue_sum(struct net_device * netdev,int rx_start,int rx_end,struct netdev_queue_stats_rx * rx_sum,int tx_start,int tx_end,struct netdev_queue_stats_tx * tx_sum)727 void netdev_stat_queue_sum(struct net_device *netdev,
728 int rx_start, int rx_end,
729 struct netdev_queue_stats_rx *rx_sum,
730 int tx_start, int tx_end,
731 struct netdev_queue_stats_tx *tx_sum)
732 {
733 const struct netdev_stat_ops *ops;
734 struct netdev_queue_stats_rx rx;
735 struct netdev_queue_stats_tx tx;
736 int i;
737
738 ops = netdev->stat_ops;
739
740 for (i = rx_start; i < rx_end; i++) {
741 memset(&rx, 0xff, sizeof(rx));
742 if (ops->get_queue_stats_rx)
743 ops->get_queue_stats_rx(netdev, i, &rx);
744 netdev_nl_stats_add(rx_sum, &rx, sizeof(rx));
745 }
746 for (i = tx_start; i < tx_end; i++) {
747 memset(&tx, 0xff, sizeof(tx));
748 if (ops->get_queue_stats_tx)
749 ops->get_queue_stats_tx(netdev, i, &tx);
750 netdev_nl_stats_add(tx_sum, &tx, sizeof(tx));
751 }
752 }
753 EXPORT_SYMBOL(netdev_stat_queue_sum);
754
755 static int
netdev_nl_stats_by_netdev(struct net_device * netdev,struct sk_buff * rsp,const struct genl_info * info)756 netdev_nl_stats_by_netdev(struct net_device *netdev, struct sk_buff *rsp,
757 const struct genl_info *info)
758 {
759 struct netdev_queue_stats_rx rx_sum;
760 struct netdev_queue_stats_tx tx_sum;
761 void *hdr;
762
763 /* Netdev can't guarantee any complete counters */
764 if (!netdev->stat_ops->get_base_stats)
765 return 0;
766
767 memset(&rx_sum, 0xff, sizeof(rx_sum));
768 memset(&tx_sum, 0xff, sizeof(tx_sum));
769
770 netdev->stat_ops->get_base_stats(netdev, &rx_sum, &tx_sum);
771
772 /* The op was there, but nothing reported, don't bother */
773 if (!memchr_inv(&rx_sum, 0xff, sizeof(rx_sum)) &&
774 !memchr_inv(&tx_sum, 0xff, sizeof(tx_sum)))
775 return 0;
776
777 hdr = genlmsg_iput(rsp, info);
778 if (!hdr)
779 return -EMSGSIZE;
780 if (nla_put_u32(rsp, NETDEV_A_QSTATS_IFINDEX, netdev->ifindex))
781 goto nla_put_failure;
782
783 netdev_stat_queue_sum(netdev, 0, netdev->real_num_rx_queues, &rx_sum,
784 0, netdev->real_num_tx_queues, &tx_sum);
785
786 if (netdev_nl_stats_write_rx(rsp, &rx_sum) ||
787 netdev_nl_stats_write_tx(rsp, &tx_sum))
788 goto nla_put_failure;
789
790 genlmsg_end(rsp, hdr);
791 return 0;
792
793 nla_put_failure:
794 genlmsg_cancel(rsp, hdr);
795 return -EMSGSIZE;
796 }
797
798 static int
netdev_nl_qstats_get_dump_one(struct net_device * netdev,unsigned int scope,struct sk_buff * skb,const struct genl_info * info,struct netdev_nl_dump_ctx * ctx)799 netdev_nl_qstats_get_dump_one(struct net_device *netdev, unsigned int scope,
800 struct sk_buff *skb, const struct genl_info *info,
801 struct netdev_nl_dump_ctx *ctx)
802 {
803 if (!netdev->stat_ops)
804 return 0;
805
806 switch (scope) {
807 case 0:
808 return netdev_nl_stats_by_netdev(netdev, skb, info);
809 case NETDEV_QSTATS_SCOPE_QUEUE:
810 return netdev_nl_stats_by_queue(netdev, skb, info, ctx);
811 }
812
813 return -EINVAL; /* Should not happen, per netlink policy */
814 }
815
netdev_nl_qstats_get_dumpit(struct sk_buff * skb,struct netlink_callback * cb)816 int netdev_nl_qstats_get_dumpit(struct sk_buff *skb,
817 struct netlink_callback *cb)
818 {
819 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
820 const struct genl_info *info = genl_info_dump(cb);
821 struct net *net = sock_net(skb->sk);
822 struct net_device *netdev;
823 unsigned int ifindex;
824 unsigned int scope;
825 int err = 0;
826
827 scope = 0;
828 if (info->attrs[NETDEV_A_QSTATS_SCOPE])
829 scope = nla_get_uint(info->attrs[NETDEV_A_QSTATS_SCOPE]);
830
831 ifindex = 0;
832 if (info->attrs[NETDEV_A_QSTATS_IFINDEX])
833 ifindex = nla_get_u32(info->attrs[NETDEV_A_QSTATS_IFINDEX]);
834
835 rtnl_lock();
836 if (ifindex) {
837 netdev = __dev_get_by_index(net, ifindex);
838 if (netdev && netdev->stat_ops) {
839 err = netdev_nl_qstats_get_dump_one(netdev, scope, skb,
840 info, ctx);
841 } else {
842 NL_SET_BAD_ATTR(info->extack,
843 info->attrs[NETDEV_A_QSTATS_IFINDEX]);
844 err = netdev ? -EOPNOTSUPP : -ENODEV;
845 }
846 } else {
847 for_each_netdev_dump(net, netdev, ctx->ifindex) {
848 err = netdev_nl_qstats_get_dump_one(netdev, scope, skb,
849 info, ctx);
850 if (err < 0)
851 break;
852 }
853 }
854 rtnl_unlock();
855
856 return err;
857 }
858
netdev_nl_bind_rx_doit(struct sk_buff * skb,struct genl_info * info)859 int netdev_nl_bind_rx_doit(struct sk_buff *skb, struct genl_info *info)
860 {
861 struct nlattr *tb[ARRAY_SIZE(netdev_queue_id_nl_policy)];
862 struct net_devmem_dmabuf_binding *binding;
863 u32 ifindex, dmabuf_fd, rxq_idx;
864 struct netdev_nl_sock *priv;
865 struct net_device *netdev;
866 struct sk_buff *rsp;
867 struct nlattr *attr;
868 int rem, err = 0;
869 void *hdr;
870
871 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX) ||
872 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_FD) ||
873 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_QUEUES))
874 return -EINVAL;
875
876 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]);
877 dmabuf_fd = nla_get_u32(info->attrs[NETDEV_A_DMABUF_FD]);
878
879 priv = genl_sk_priv_get(&netdev_nl_family, NETLINK_CB(skb).sk);
880 if (IS_ERR(priv))
881 return PTR_ERR(priv);
882
883 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
884 if (!rsp)
885 return -ENOMEM;
886
887 hdr = genlmsg_iput(rsp, info);
888 if (!hdr) {
889 err = -EMSGSIZE;
890 goto err_genlmsg_free;
891 }
892
893 mutex_lock(&priv->lock);
894
895 err = 0;
896 netdev = netdev_get_by_index_lock(genl_info_net(info), ifindex);
897 if (!netdev) {
898 err = -ENODEV;
899 goto err_unlock_sock;
900 }
901 if (!netif_device_present(netdev))
902 err = -ENODEV;
903 else if (!netdev_need_ops_lock(netdev))
904 err = -EOPNOTSUPP;
905 if (err) {
906 NL_SET_BAD_ATTR(info->extack,
907 info->attrs[NETDEV_A_DEV_IFINDEX]);
908 goto err_unlock;
909 }
910
911 binding = net_devmem_bind_dmabuf(netdev, dmabuf_fd, info->extack);
912 if (IS_ERR(binding)) {
913 err = PTR_ERR(binding);
914 goto err_unlock;
915 }
916
917 nla_for_each_attr_type(attr, NETDEV_A_DMABUF_QUEUES,
918 genlmsg_data(info->genlhdr),
919 genlmsg_len(info->genlhdr), rem) {
920 err = nla_parse_nested(
921 tb, ARRAY_SIZE(netdev_queue_id_nl_policy) - 1, attr,
922 netdev_queue_id_nl_policy, info->extack);
923 if (err < 0)
924 goto err_unbind;
925
926 if (NL_REQ_ATTR_CHECK(info->extack, attr, tb, NETDEV_A_QUEUE_ID) ||
927 NL_REQ_ATTR_CHECK(info->extack, attr, tb, NETDEV_A_QUEUE_TYPE)) {
928 err = -EINVAL;
929 goto err_unbind;
930 }
931
932 if (nla_get_u32(tb[NETDEV_A_QUEUE_TYPE]) != NETDEV_QUEUE_TYPE_RX) {
933 NL_SET_BAD_ATTR(info->extack, tb[NETDEV_A_QUEUE_TYPE]);
934 err = -EINVAL;
935 goto err_unbind;
936 }
937
938 rxq_idx = nla_get_u32(tb[NETDEV_A_QUEUE_ID]);
939
940 err = net_devmem_bind_dmabuf_to_queue(netdev, rxq_idx, binding,
941 info->extack);
942 if (err)
943 goto err_unbind;
944 }
945
946 list_add(&binding->list, &priv->bindings);
947
948 nla_put_u32(rsp, NETDEV_A_DMABUF_ID, binding->id);
949 genlmsg_end(rsp, hdr);
950
951 err = genlmsg_reply(rsp, info);
952 if (err)
953 goto err_unbind;
954
955 netdev_unlock(netdev);
956
957 mutex_unlock(&priv->lock);
958
959 return 0;
960
961 err_unbind:
962 net_devmem_unbind_dmabuf(binding);
963 err_unlock:
964 netdev_unlock(netdev);
965 err_unlock_sock:
966 mutex_unlock(&priv->lock);
967 err_genlmsg_free:
968 nlmsg_free(rsp);
969 return err;
970 }
971
netdev_nl_sock_priv_init(struct netdev_nl_sock * priv)972 void netdev_nl_sock_priv_init(struct netdev_nl_sock *priv)
973 {
974 INIT_LIST_HEAD(&priv->bindings);
975 mutex_init(&priv->lock);
976 }
977
netdev_nl_sock_priv_destroy(struct netdev_nl_sock * priv)978 void netdev_nl_sock_priv_destroy(struct netdev_nl_sock *priv)
979 {
980 struct net_devmem_dmabuf_binding *binding;
981 struct net_devmem_dmabuf_binding *temp;
982 netdevice_tracker dev_tracker;
983 struct net_device *dev;
984
985 mutex_lock(&priv->lock);
986 list_for_each_entry_safe(binding, temp, &priv->bindings, list) {
987 mutex_lock(&binding->lock);
988 dev = binding->dev;
989 if (!dev) {
990 mutex_unlock(&binding->lock);
991 net_devmem_unbind_dmabuf(binding);
992 continue;
993 }
994 netdev_hold(dev, &dev_tracker, GFP_KERNEL);
995 mutex_unlock(&binding->lock);
996
997 netdev_lock(dev);
998 net_devmem_unbind_dmabuf(binding);
999 netdev_unlock(dev);
1000 netdev_put(dev, &dev_tracker);
1001 }
1002 mutex_unlock(&priv->lock);
1003 }
1004
netdev_genl_netdevice_event(struct notifier_block * nb,unsigned long event,void * ptr)1005 static int netdev_genl_netdevice_event(struct notifier_block *nb,
1006 unsigned long event, void *ptr)
1007 {
1008 struct net_device *netdev = netdev_notifier_info_to_dev(ptr);
1009
1010 switch (event) {
1011 case NETDEV_REGISTER:
1012 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_ADD_NTF);
1013 break;
1014 case NETDEV_UNREGISTER:
1015 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_DEL_NTF);
1016 break;
1017 case NETDEV_XDP_FEAT_CHANGE:
1018 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_CHANGE_NTF);
1019 break;
1020 }
1021
1022 return NOTIFY_OK;
1023 }
1024
1025 static struct notifier_block netdev_genl_nb = {
1026 .notifier_call = netdev_genl_netdevice_event,
1027 };
1028
netdev_genl_init(void)1029 static int __init netdev_genl_init(void)
1030 {
1031 int err;
1032
1033 err = register_netdevice_notifier(&netdev_genl_nb);
1034 if (err)
1035 return err;
1036
1037 err = genl_register_family(&netdev_nl_family);
1038 if (err)
1039 goto err_unreg_ntf;
1040
1041 return 0;
1042
1043 err_unreg_ntf:
1044 unregister_netdevice_notifier(&netdev_genl_nb);
1045 return err;
1046 }
1047
1048 subsys_initcall(netdev_genl_init);
1049