1 // SPDX-License-Identifier: GPL-2.0-only 2 3 #include <linux/netdevice.h> 4 #include <linux/notifier.h> 5 #include <linux/rtnetlink.h> 6 #include <net/busy_poll.h> 7 #include <net/net_namespace.h> 8 #include <net/netdev_queues.h> 9 #include <net/netdev_rx_queue.h> 10 #include <net/sock.h> 11 #include <net/xdp.h> 12 #include <net/xdp_sock.h> 13 #include <net/page_pool/memory_provider.h> 14 15 #include "dev.h" 16 #include "devmem.h" 17 #include "netdev-genl-gen.h" 18 19 struct netdev_nl_dump_ctx { 20 unsigned long ifindex; 21 unsigned int rxq_idx; 22 unsigned int txq_idx; 23 unsigned int napi_id; 24 }; 25 26 static struct netdev_nl_dump_ctx *netdev_dump_ctx(struct netlink_callback *cb) 27 { 28 NL_ASSERT_CTX_FITS(struct netdev_nl_dump_ctx); 29 30 return (struct netdev_nl_dump_ctx *)cb->ctx; 31 } 32 33 static int 34 netdev_nl_dev_fill(struct net_device *netdev, struct sk_buff *rsp, 35 const struct genl_info *info) 36 { 37 u64 xsk_features = 0; 38 u64 xdp_rx_meta = 0; 39 void *hdr; 40 41 netdev_assert_locked(netdev); /* note: rtnl_lock may not be held! */ 42 43 hdr = genlmsg_iput(rsp, info); 44 if (!hdr) 45 return -EMSGSIZE; 46 47 #define XDP_METADATA_KFUNC(_, flag, __, xmo) \ 48 if (netdev->xdp_metadata_ops && netdev->xdp_metadata_ops->xmo) \ 49 xdp_rx_meta |= flag; 50 XDP_METADATA_KFUNC_xxx 51 #undef XDP_METADATA_KFUNC 52 53 if (netdev->xsk_tx_metadata_ops) { 54 if (netdev->xsk_tx_metadata_ops->tmo_fill_timestamp) 55 xsk_features |= NETDEV_XSK_FLAGS_TX_TIMESTAMP; 56 if (netdev->xsk_tx_metadata_ops->tmo_request_checksum) 57 xsk_features |= NETDEV_XSK_FLAGS_TX_CHECKSUM; 58 if (netdev->xsk_tx_metadata_ops->tmo_request_launch_time) 59 xsk_features |= NETDEV_XSK_FLAGS_TX_LAUNCH_TIME_FIFO; 60 } 61 62 if (nla_put_u32(rsp, NETDEV_A_DEV_IFINDEX, netdev->ifindex) || 63 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_FEATURES, 64 netdev->xdp_features, NETDEV_A_DEV_PAD) || 65 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_RX_METADATA_FEATURES, 66 xdp_rx_meta, NETDEV_A_DEV_PAD) || 67 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XSK_FEATURES, 68 xsk_features, NETDEV_A_DEV_PAD)) 69 goto err_cancel_msg; 70 71 if (netdev->xdp_features & NETDEV_XDP_ACT_XSK_ZEROCOPY) { 72 if (nla_put_u32(rsp, NETDEV_A_DEV_XDP_ZC_MAX_SEGS, 73 netdev->xdp_zc_max_segs)) 74 goto err_cancel_msg; 75 } 76 77 genlmsg_end(rsp, hdr); 78 79 return 0; 80 81 err_cancel_msg: 82 genlmsg_cancel(rsp, hdr); 83 return -EMSGSIZE; 84 } 85 86 static void 87 netdev_genl_dev_notify(struct net_device *netdev, int cmd) 88 { 89 struct genl_info info; 90 struct sk_buff *ntf; 91 92 if (!genl_has_listeners(&netdev_nl_family, dev_net(netdev), 93 NETDEV_NLGRP_MGMT)) 94 return; 95 96 genl_info_init_ntf(&info, &netdev_nl_family, cmd); 97 98 ntf = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 99 if (!ntf) 100 return; 101 102 if (netdev_nl_dev_fill(netdev, ntf, &info)) { 103 nlmsg_free(ntf); 104 return; 105 } 106 107 genlmsg_multicast_netns(&netdev_nl_family, dev_net(netdev), ntf, 108 0, NETDEV_NLGRP_MGMT, GFP_KERNEL); 109 } 110 111 int netdev_nl_dev_get_doit(struct sk_buff *skb, struct genl_info *info) 112 { 113 struct net_device *netdev; 114 struct sk_buff *rsp; 115 u32 ifindex; 116 int err; 117 118 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX)) 119 return -EINVAL; 120 121 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]); 122 123 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 124 if (!rsp) 125 return -ENOMEM; 126 127 netdev = netdev_get_by_index_lock(genl_info_net(info), ifindex); 128 if (!netdev) { 129 err = -ENODEV; 130 goto err_free_msg; 131 } 132 133 err = netdev_nl_dev_fill(netdev, rsp, info); 134 netdev_unlock(netdev); 135 136 if (err) 137 goto err_free_msg; 138 139 return genlmsg_reply(rsp, info); 140 141 err_free_msg: 142 nlmsg_free(rsp); 143 return err; 144 } 145 146 int netdev_nl_dev_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb) 147 { 148 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb); 149 struct net *net = sock_net(skb->sk); 150 int err; 151 152 for_each_netdev_lock_scoped(net, netdev, ctx->ifindex) { 153 err = netdev_nl_dev_fill(netdev, skb, genl_info_dump(cb)); 154 if (err < 0) 155 return err; 156 } 157 158 return 0; 159 } 160 161 static int 162 netdev_nl_napi_fill_one(struct sk_buff *rsp, struct napi_struct *napi, 163 const struct genl_info *info) 164 { 165 unsigned long irq_suspend_timeout; 166 unsigned long gro_flush_timeout; 167 u32 napi_defer_hard_irqs; 168 void *hdr; 169 pid_t pid; 170 171 if (!napi->dev->up) 172 return 0; 173 174 hdr = genlmsg_iput(rsp, info); 175 if (!hdr) 176 return -EMSGSIZE; 177 178 if (nla_put_u32(rsp, NETDEV_A_NAPI_ID, napi->napi_id)) 179 goto nla_put_failure; 180 181 if (nla_put_u32(rsp, NETDEV_A_NAPI_IFINDEX, napi->dev->ifindex)) 182 goto nla_put_failure; 183 184 if (napi->irq >= 0 && nla_put_u32(rsp, NETDEV_A_NAPI_IRQ, napi->irq)) 185 goto nla_put_failure; 186 187 if (napi->thread) { 188 pid = task_pid_nr(napi->thread); 189 if (nla_put_u32(rsp, NETDEV_A_NAPI_PID, pid)) 190 goto nla_put_failure; 191 } 192 193 napi_defer_hard_irqs = napi_get_defer_hard_irqs(napi); 194 if (nla_put_s32(rsp, NETDEV_A_NAPI_DEFER_HARD_IRQS, 195 napi_defer_hard_irqs)) 196 goto nla_put_failure; 197 198 irq_suspend_timeout = napi_get_irq_suspend_timeout(napi); 199 if (nla_put_uint(rsp, NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT, 200 irq_suspend_timeout)) 201 goto nla_put_failure; 202 203 gro_flush_timeout = napi_get_gro_flush_timeout(napi); 204 if (nla_put_uint(rsp, NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT, 205 gro_flush_timeout)) 206 goto nla_put_failure; 207 208 genlmsg_end(rsp, hdr); 209 210 return 0; 211 212 nla_put_failure: 213 genlmsg_cancel(rsp, hdr); 214 return -EMSGSIZE; 215 } 216 217 int netdev_nl_napi_get_doit(struct sk_buff *skb, struct genl_info *info) 218 { 219 struct napi_struct *napi; 220 struct sk_buff *rsp; 221 u32 napi_id; 222 int err; 223 224 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_NAPI_ID)) 225 return -EINVAL; 226 227 napi_id = nla_get_u32(info->attrs[NETDEV_A_NAPI_ID]); 228 229 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 230 if (!rsp) 231 return -ENOMEM; 232 233 napi = netdev_napi_by_id_lock(genl_info_net(info), napi_id); 234 if (napi) { 235 err = netdev_nl_napi_fill_one(rsp, napi, info); 236 netdev_unlock(napi->dev); 237 } else { 238 NL_SET_BAD_ATTR(info->extack, info->attrs[NETDEV_A_NAPI_ID]); 239 err = -ENOENT; 240 } 241 242 if (err) { 243 goto err_free_msg; 244 } else if (!rsp->len) { 245 err = -ENOENT; 246 goto err_free_msg; 247 } 248 249 return genlmsg_reply(rsp, info); 250 251 err_free_msg: 252 nlmsg_free(rsp); 253 return err; 254 } 255 256 static int 257 netdev_nl_napi_dump_one(struct net_device *netdev, struct sk_buff *rsp, 258 const struct genl_info *info, 259 struct netdev_nl_dump_ctx *ctx) 260 { 261 struct napi_struct *napi; 262 unsigned int prev_id; 263 int err = 0; 264 265 if (!netdev->up) 266 return err; 267 268 prev_id = UINT_MAX; 269 list_for_each_entry(napi, &netdev->napi_list, dev_list) { 270 if (!napi_id_valid(napi->napi_id)) 271 continue; 272 273 /* Dump continuation below depends on the list being sorted */ 274 WARN_ON_ONCE(napi->napi_id >= prev_id); 275 prev_id = napi->napi_id; 276 277 if (ctx->napi_id && napi->napi_id >= ctx->napi_id) 278 continue; 279 280 err = netdev_nl_napi_fill_one(rsp, napi, info); 281 if (err) 282 return err; 283 ctx->napi_id = napi->napi_id; 284 } 285 return err; 286 } 287 288 int netdev_nl_napi_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb) 289 { 290 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb); 291 const struct genl_info *info = genl_info_dump(cb); 292 struct net *net = sock_net(skb->sk); 293 struct net_device *netdev; 294 u32 ifindex = 0; 295 int err = 0; 296 297 if (info->attrs[NETDEV_A_NAPI_IFINDEX]) 298 ifindex = nla_get_u32(info->attrs[NETDEV_A_NAPI_IFINDEX]); 299 300 if (ifindex) { 301 netdev = netdev_get_by_index_lock(net, ifindex); 302 if (netdev) { 303 err = netdev_nl_napi_dump_one(netdev, skb, info, ctx); 304 netdev_unlock(netdev); 305 } else { 306 err = -ENODEV; 307 } 308 } else { 309 for_each_netdev_lock_scoped(net, netdev, ctx->ifindex) { 310 err = netdev_nl_napi_dump_one(netdev, skb, info, ctx); 311 if (err < 0) 312 break; 313 ctx->napi_id = 0; 314 } 315 } 316 317 return err; 318 } 319 320 static int 321 netdev_nl_napi_set_config(struct napi_struct *napi, struct genl_info *info) 322 { 323 u64 irq_suspend_timeout = 0; 324 u64 gro_flush_timeout = 0; 325 u32 defer = 0; 326 327 if (info->attrs[NETDEV_A_NAPI_DEFER_HARD_IRQS]) { 328 defer = nla_get_u32(info->attrs[NETDEV_A_NAPI_DEFER_HARD_IRQS]); 329 napi_set_defer_hard_irqs(napi, defer); 330 } 331 332 if (info->attrs[NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT]) { 333 irq_suspend_timeout = nla_get_uint(info->attrs[NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT]); 334 napi_set_irq_suspend_timeout(napi, irq_suspend_timeout); 335 } 336 337 if (info->attrs[NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT]) { 338 gro_flush_timeout = nla_get_uint(info->attrs[NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT]); 339 napi_set_gro_flush_timeout(napi, gro_flush_timeout); 340 } 341 342 return 0; 343 } 344 345 int netdev_nl_napi_set_doit(struct sk_buff *skb, struct genl_info *info) 346 { 347 struct napi_struct *napi; 348 unsigned int napi_id; 349 int err; 350 351 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_NAPI_ID)) 352 return -EINVAL; 353 354 napi_id = nla_get_u32(info->attrs[NETDEV_A_NAPI_ID]); 355 356 napi = netdev_napi_by_id_lock(genl_info_net(info), napi_id); 357 if (napi) { 358 err = netdev_nl_napi_set_config(napi, info); 359 netdev_unlock(napi->dev); 360 } else { 361 NL_SET_BAD_ATTR(info->extack, info->attrs[NETDEV_A_NAPI_ID]); 362 err = -ENOENT; 363 } 364 365 return err; 366 } 367 368 static int nla_put_napi_id(struct sk_buff *skb, const struct napi_struct *napi) 369 { 370 if (napi && napi_id_valid(napi->napi_id)) 371 return nla_put_u32(skb, NETDEV_A_QUEUE_NAPI_ID, napi->napi_id); 372 return 0; 373 } 374 375 static int 376 netdev_nl_queue_fill_one(struct sk_buff *rsp, struct net_device *netdev, 377 u32 q_idx, u32 q_type, const struct genl_info *info) 378 { 379 struct pp_memory_provider_params *params; 380 struct netdev_rx_queue *rxq; 381 struct netdev_queue *txq; 382 void *hdr; 383 384 hdr = genlmsg_iput(rsp, info); 385 if (!hdr) 386 return -EMSGSIZE; 387 388 if (nla_put_u32(rsp, NETDEV_A_QUEUE_ID, q_idx) || 389 nla_put_u32(rsp, NETDEV_A_QUEUE_TYPE, q_type) || 390 nla_put_u32(rsp, NETDEV_A_QUEUE_IFINDEX, netdev->ifindex)) 391 goto nla_put_failure; 392 393 switch (q_type) { 394 case NETDEV_QUEUE_TYPE_RX: 395 rxq = __netif_get_rx_queue(netdev, q_idx); 396 if (nla_put_napi_id(rsp, rxq->napi)) 397 goto nla_put_failure; 398 399 params = &rxq->mp_params; 400 if (params->mp_ops && 401 params->mp_ops->nl_fill(params->mp_priv, rsp, rxq)) 402 goto nla_put_failure; 403 #ifdef CONFIG_XDP_SOCKETS 404 if (rxq->pool) 405 if (nla_put_empty_nest(rsp, NETDEV_A_QUEUE_XSK)) 406 goto nla_put_failure; 407 #endif 408 409 break; 410 case NETDEV_QUEUE_TYPE_TX: 411 txq = netdev_get_tx_queue(netdev, q_idx); 412 if (nla_put_napi_id(rsp, txq->napi)) 413 goto nla_put_failure; 414 #ifdef CONFIG_XDP_SOCKETS 415 if (txq->pool) 416 if (nla_put_empty_nest(rsp, NETDEV_A_QUEUE_XSK)) 417 goto nla_put_failure; 418 #endif 419 break; 420 } 421 422 genlmsg_end(rsp, hdr); 423 424 return 0; 425 426 nla_put_failure: 427 genlmsg_cancel(rsp, hdr); 428 return -EMSGSIZE; 429 } 430 431 static int netdev_nl_queue_validate(struct net_device *netdev, u32 q_id, 432 u32 q_type) 433 { 434 switch (q_type) { 435 case NETDEV_QUEUE_TYPE_RX: 436 if (q_id >= netdev->real_num_rx_queues) 437 return -EINVAL; 438 return 0; 439 case NETDEV_QUEUE_TYPE_TX: 440 if (q_id >= netdev->real_num_tx_queues) 441 return -EINVAL; 442 } 443 return 0; 444 } 445 446 static int 447 netdev_nl_queue_fill(struct sk_buff *rsp, struct net_device *netdev, u32 q_idx, 448 u32 q_type, const struct genl_info *info) 449 { 450 int err; 451 452 if (!netdev->up) 453 return -ENOENT; 454 455 err = netdev_nl_queue_validate(netdev, q_idx, q_type); 456 if (err) 457 return err; 458 459 return netdev_nl_queue_fill_one(rsp, netdev, q_idx, q_type, info); 460 } 461 462 int netdev_nl_queue_get_doit(struct sk_buff *skb, struct genl_info *info) 463 { 464 u32 q_id, q_type, ifindex; 465 struct net_device *netdev; 466 struct sk_buff *rsp; 467 int err; 468 469 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_ID) || 470 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_TYPE) || 471 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_IFINDEX)) 472 return -EINVAL; 473 474 q_id = nla_get_u32(info->attrs[NETDEV_A_QUEUE_ID]); 475 q_type = nla_get_u32(info->attrs[NETDEV_A_QUEUE_TYPE]); 476 ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]); 477 478 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 479 if (!rsp) 480 return -ENOMEM; 481 482 netdev = netdev_get_by_index_lock_ops_compat(genl_info_net(info), 483 ifindex); 484 if (netdev) { 485 err = netdev_nl_queue_fill(rsp, netdev, q_id, q_type, info); 486 netdev_unlock_ops_compat(netdev); 487 } else { 488 err = -ENODEV; 489 } 490 491 if (err) 492 goto err_free_msg; 493 494 return genlmsg_reply(rsp, info); 495 496 err_free_msg: 497 nlmsg_free(rsp); 498 return err; 499 } 500 501 static int 502 netdev_nl_queue_dump_one(struct net_device *netdev, struct sk_buff *rsp, 503 const struct genl_info *info, 504 struct netdev_nl_dump_ctx *ctx) 505 { 506 int err = 0; 507 508 if (!netdev->up) 509 return err; 510 511 for (; ctx->rxq_idx < netdev->real_num_rx_queues; ctx->rxq_idx++) { 512 err = netdev_nl_queue_fill_one(rsp, netdev, ctx->rxq_idx, 513 NETDEV_QUEUE_TYPE_RX, info); 514 if (err) 515 return err; 516 } 517 for (; ctx->txq_idx < netdev->real_num_tx_queues; ctx->txq_idx++) { 518 err = netdev_nl_queue_fill_one(rsp, netdev, ctx->txq_idx, 519 NETDEV_QUEUE_TYPE_TX, info); 520 if (err) 521 return err; 522 } 523 524 return err; 525 } 526 527 int netdev_nl_queue_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb) 528 { 529 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb); 530 const struct genl_info *info = genl_info_dump(cb); 531 struct net *net = sock_net(skb->sk); 532 struct net_device *netdev; 533 u32 ifindex = 0; 534 int err = 0; 535 536 if (info->attrs[NETDEV_A_QUEUE_IFINDEX]) 537 ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]); 538 539 if (ifindex) { 540 netdev = netdev_get_by_index_lock_ops_compat(net, ifindex); 541 if (netdev) { 542 err = netdev_nl_queue_dump_one(netdev, skb, info, ctx); 543 netdev_unlock_ops_compat(netdev); 544 } else { 545 err = -ENODEV; 546 } 547 } else { 548 for_each_netdev_lock_ops_compat_scoped(net, netdev, 549 ctx->ifindex) { 550 err = netdev_nl_queue_dump_one(netdev, skb, info, ctx); 551 if (err < 0) 552 break; 553 ctx->rxq_idx = 0; 554 ctx->txq_idx = 0; 555 } 556 } 557 558 return err; 559 } 560 561 #define NETDEV_STAT_NOT_SET (~0ULL) 562 563 static void netdev_nl_stats_add(void *_sum, const void *_add, size_t size) 564 { 565 const u64 *add = _add; 566 u64 *sum = _sum; 567 568 while (size) { 569 if (*add != NETDEV_STAT_NOT_SET && *sum != NETDEV_STAT_NOT_SET) 570 *sum += *add; 571 sum++; 572 add++; 573 size -= 8; 574 } 575 } 576 577 static int netdev_stat_put(struct sk_buff *rsp, unsigned int attr_id, u64 value) 578 { 579 if (value == NETDEV_STAT_NOT_SET) 580 return 0; 581 return nla_put_uint(rsp, attr_id, value); 582 } 583 584 static int 585 netdev_nl_stats_write_rx(struct sk_buff *rsp, struct netdev_queue_stats_rx *rx) 586 { 587 if (netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_PACKETS, rx->packets) || 588 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_BYTES, rx->bytes) || 589 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_ALLOC_FAIL, rx->alloc_fail) || 590 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROPS, rx->hw_drops) || 591 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROP_OVERRUNS, rx->hw_drop_overruns) || 592 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_COMPLETE, rx->csum_complete) || 593 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_UNNECESSARY, rx->csum_unnecessary) || 594 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_NONE, rx->csum_none) || 595 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_BAD, rx->csum_bad) || 596 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_PACKETS, rx->hw_gro_packets) || 597 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_BYTES, rx->hw_gro_bytes) || 598 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_WIRE_PACKETS, rx->hw_gro_wire_packets) || 599 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_WIRE_BYTES, rx->hw_gro_wire_bytes) || 600 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROP_RATELIMITS, rx->hw_drop_ratelimits)) 601 return -EMSGSIZE; 602 return 0; 603 } 604 605 static int 606 netdev_nl_stats_write_tx(struct sk_buff *rsp, struct netdev_queue_stats_tx *tx) 607 { 608 if (netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_PACKETS, tx->packets) || 609 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_BYTES, tx->bytes) || 610 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROPS, tx->hw_drops) || 611 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROP_ERRORS, tx->hw_drop_errors) || 612 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_CSUM_NONE, tx->csum_none) || 613 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_NEEDS_CSUM, tx->needs_csum) || 614 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_PACKETS, tx->hw_gso_packets) || 615 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_BYTES, tx->hw_gso_bytes) || 616 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_WIRE_PACKETS, tx->hw_gso_wire_packets) || 617 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_WIRE_BYTES, tx->hw_gso_wire_bytes) || 618 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROP_RATELIMITS, tx->hw_drop_ratelimits) || 619 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_STOP, tx->stop) || 620 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_WAKE, tx->wake)) 621 return -EMSGSIZE; 622 return 0; 623 } 624 625 static int 626 netdev_nl_stats_queue(struct net_device *netdev, struct sk_buff *rsp, 627 u32 q_type, int i, const struct genl_info *info) 628 { 629 const struct netdev_stat_ops *ops = netdev->stat_ops; 630 struct netdev_queue_stats_rx rx; 631 struct netdev_queue_stats_tx tx; 632 void *hdr; 633 634 hdr = genlmsg_iput(rsp, info); 635 if (!hdr) 636 return -EMSGSIZE; 637 if (nla_put_u32(rsp, NETDEV_A_QSTATS_IFINDEX, netdev->ifindex) || 638 nla_put_u32(rsp, NETDEV_A_QSTATS_QUEUE_TYPE, q_type) || 639 nla_put_u32(rsp, NETDEV_A_QSTATS_QUEUE_ID, i)) 640 goto nla_put_failure; 641 642 switch (q_type) { 643 case NETDEV_QUEUE_TYPE_RX: 644 memset(&rx, 0xff, sizeof(rx)); 645 ops->get_queue_stats_rx(netdev, i, &rx); 646 if (!memchr_inv(&rx, 0xff, sizeof(rx))) 647 goto nla_cancel; 648 if (netdev_nl_stats_write_rx(rsp, &rx)) 649 goto nla_put_failure; 650 break; 651 case NETDEV_QUEUE_TYPE_TX: 652 memset(&tx, 0xff, sizeof(tx)); 653 ops->get_queue_stats_tx(netdev, i, &tx); 654 if (!memchr_inv(&tx, 0xff, sizeof(tx))) 655 goto nla_cancel; 656 if (netdev_nl_stats_write_tx(rsp, &tx)) 657 goto nla_put_failure; 658 break; 659 } 660 661 genlmsg_end(rsp, hdr); 662 return 0; 663 664 nla_cancel: 665 genlmsg_cancel(rsp, hdr); 666 return 0; 667 nla_put_failure: 668 genlmsg_cancel(rsp, hdr); 669 return -EMSGSIZE; 670 } 671 672 static int 673 netdev_nl_stats_by_queue(struct net_device *netdev, struct sk_buff *rsp, 674 const struct genl_info *info, 675 struct netdev_nl_dump_ctx *ctx) 676 { 677 const struct netdev_stat_ops *ops = netdev->stat_ops; 678 int i, err; 679 680 if (!(netdev->flags & IFF_UP)) 681 return 0; 682 683 i = ctx->rxq_idx; 684 while (ops->get_queue_stats_rx && i < netdev->real_num_rx_queues) { 685 err = netdev_nl_stats_queue(netdev, rsp, NETDEV_QUEUE_TYPE_RX, 686 i, info); 687 if (err) 688 return err; 689 ctx->rxq_idx = ++i; 690 } 691 i = ctx->txq_idx; 692 while (ops->get_queue_stats_tx && i < netdev->real_num_tx_queues) { 693 err = netdev_nl_stats_queue(netdev, rsp, NETDEV_QUEUE_TYPE_TX, 694 i, info); 695 if (err) 696 return err; 697 ctx->txq_idx = ++i; 698 } 699 700 ctx->rxq_idx = 0; 701 ctx->txq_idx = 0; 702 return 0; 703 } 704 705 /** 706 * netdev_stat_queue_sum() - add up queue stats from range of queues 707 * @netdev: net_device 708 * @rx_start: index of the first Rx queue to query 709 * @rx_end: index after the last Rx queue (first *not* to query) 710 * @rx_sum: output Rx stats, should be already initialized 711 * @tx_start: index of the first Tx queue to query 712 * @tx_end: index after the last Tx queue (first *not* to query) 713 * @tx_sum: output Tx stats, should be already initialized 714 * 715 * Add stats from [start, end) range of queue IDs to *x_sum structs. 716 * The sum structs must be already initialized. Usually this 717 * helper is invoked from the .get_base_stats callbacks of drivers 718 * to account for stats of disabled queues. In that case the ranges 719 * are usually [netdev->real_num_*x_queues, netdev->num_*x_queues). 720 */ 721 void netdev_stat_queue_sum(struct net_device *netdev, 722 int rx_start, int rx_end, 723 struct netdev_queue_stats_rx *rx_sum, 724 int tx_start, int tx_end, 725 struct netdev_queue_stats_tx *tx_sum) 726 { 727 const struct netdev_stat_ops *ops; 728 struct netdev_queue_stats_rx rx; 729 struct netdev_queue_stats_tx tx; 730 int i; 731 732 ops = netdev->stat_ops; 733 734 for (i = rx_start; i < rx_end; i++) { 735 memset(&rx, 0xff, sizeof(rx)); 736 if (ops->get_queue_stats_rx) 737 ops->get_queue_stats_rx(netdev, i, &rx); 738 netdev_nl_stats_add(rx_sum, &rx, sizeof(rx)); 739 } 740 for (i = tx_start; i < tx_end; i++) { 741 memset(&tx, 0xff, sizeof(tx)); 742 if (ops->get_queue_stats_tx) 743 ops->get_queue_stats_tx(netdev, i, &tx); 744 netdev_nl_stats_add(tx_sum, &tx, sizeof(tx)); 745 } 746 } 747 EXPORT_SYMBOL(netdev_stat_queue_sum); 748 749 static int 750 netdev_nl_stats_by_netdev(struct net_device *netdev, struct sk_buff *rsp, 751 const struct genl_info *info) 752 { 753 struct netdev_queue_stats_rx rx_sum; 754 struct netdev_queue_stats_tx tx_sum; 755 void *hdr; 756 757 /* Netdev can't guarantee any complete counters */ 758 if (!netdev->stat_ops->get_base_stats) 759 return 0; 760 761 memset(&rx_sum, 0xff, sizeof(rx_sum)); 762 memset(&tx_sum, 0xff, sizeof(tx_sum)); 763 764 netdev->stat_ops->get_base_stats(netdev, &rx_sum, &tx_sum); 765 766 /* The op was there, but nothing reported, don't bother */ 767 if (!memchr_inv(&rx_sum, 0xff, sizeof(rx_sum)) && 768 !memchr_inv(&tx_sum, 0xff, sizeof(tx_sum))) 769 return 0; 770 771 hdr = genlmsg_iput(rsp, info); 772 if (!hdr) 773 return -EMSGSIZE; 774 if (nla_put_u32(rsp, NETDEV_A_QSTATS_IFINDEX, netdev->ifindex)) 775 goto nla_put_failure; 776 777 netdev_stat_queue_sum(netdev, 0, netdev->real_num_rx_queues, &rx_sum, 778 0, netdev->real_num_tx_queues, &tx_sum); 779 780 if (netdev_nl_stats_write_rx(rsp, &rx_sum) || 781 netdev_nl_stats_write_tx(rsp, &tx_sum)) 782 goto nla_put_failure; 783 784 genlmsg_end(rsp, hdr); 785 return 0; 786 787 nla_put_failure: 788 genlmsg_cancel(rsp, hdr); 789 return -EMSGSIZE; 790 } 791 792 static int 793 netdev_nl_qstats_get_dump_one(struct net_device *netdev, unsigned int scope, 794 struct sk_buff *skb, const struct genl_info *info, 795 struct netdev_nl_dump_ctx *ctx) 796 { 797 if (!netdev->stat_ops) 798 return 0; 799 800 switch (scope) { 801 case 0: 802 return netdev_nl_stats_by_netdev(netdev, skb, info); 803 case NETDEV_QSTATS_SCOPE_QUEUE: 804 return netdev_nl_stats_by_queue(netdev, skb, info, ctx); 805 } 806 807 return -EINVAL; /* Should not happen, per netlink policy */ 808 } 809 810 int netdev_nl_qstats_get_dumpit(struct sk_buff *skb, 811 struct netlink_callback *cb) 812 { 813 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb); 814 const struct genl_info *info = genl_info_dump(cb); 815 struct net *net = sock_net(skb->sk); 816 struct net_device *netdev; 817 unsigned int ifindex; 818 unsigned int scope; 819 int err = 0; 820 821 scope = 0; 822 if (info->attrs[NETDEV_A_QSTATS_SCOPE]) 823 scope = nla_get_uint(info->attrs[NETDEV_A_QSTATS_SCOPE]); 824 825 ifindex = 0; 826 if (info->attrs[NETDEV_A_QSTATS_IFINDEX]) 827 ifindex = nla_get_u32(info->attrs[NETDEV_A_QSTATS_IFINDEX]); 828 829 if (ifindex) { 830 netdev = netdev_get_by_index_lock_ops_compat(net, ifindex); 831 if (!netdev) { 832 NL_SET_BAD_ATTR(info->extack, 833 info->attrs[NETDEV_A_QSTATS_IFINDEX]); 834 return -ENODEV; 835 } 836 if (netdev->stat_ops) { 837 err = netdev_nl_qstats_get_dump_one(netdev, scope, skb, 838 info, ctx); 839 } else { 840 NL_SET_BAD_ATTR(info->extack, 841 info->attrs[NETDEV_A_QSTATS_IFINDEX]); 842 err = -EOPNOTSUPP; 843 } 844 netdev_unlock_ops_compat(netdev); 845 return err; 846 } 847 848 for_each_netdev_lock_ops_compat_scoped(net, netdev, ctx->ifindex) { 849 err = netdev_nl_qstats_get_dump_one(netdev, scope, skb, 850 info, ctx); 851 if (err < 0) 852 break; 853 } 854 855 return err; 856 } 857 858 int netdev_nl_bind_rx_doit(struct sk_buff *skb, struct genl_info *info) 859 { 860 struct nlattr *tb[ARRAY_SIZE(netdev_queue_id_nl_policy)]; 861 struct net_devmem_dmabuf_binding *binding; 862 u32 ifindex, dmabuf_fd, rxq_idx; 863 struct netdev_nl_sock *priv; 864 struct net_device *netdev; 865 struct sk_buff *rsp; 866 struct nlattr *attr; 867 int rem, err = 0; 868 void *hdr; 869 870 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX) || 871 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_FD) || 872 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_QUEUES)) 873 return -EINVAL; 874 875 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]); 876 dmabuf_fd = nla_get_u32(info->attrs[NETDEV_A_DMABUF_FD]); 877 878 priv = genl_sk_priv_get(&netdev_nl_family, NETLINK_CB(skb).sk); 879 if (IS_ERR(priv)) 880 return PTR_ERR(priv); 881 882 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 883 if (!rsp) 884 return -ENOMEM; 885 886 hdr = genlmsg_iput(rsp, info); 887 if (!hdr) { 888 err = -EMSGSIZE; 889 goto err_genlmsg_free; 890 } 891 892 mutex_lock(&priv->lock); 893 894 err = 0; 895 netdev = netdev_get_by_index_lock(genl_info_net(info), ifindex); 896 if (!netdev) { 897 err = -ENODEV; 898 goto err_unlock_sock; 899 } 900 if (!netif_device_present(netdev)) 901 err = -ENODEV; 902 else if (!netdev_need_ops_lock(netdev)) 903 err = -EOPNOTSUPP; 904 if (err) { 905 NL_SET_BAD_ATTR(info->extack, 906 info->attrs[NETDEV_A_DEV_IFINDEX]); 907 goto err_unlock; 908 } 909 910 binding = net_devmem_bind_dmabuf(netdev, DMA_FROM_DEVICE, dmabuf_fd, 911 info->extack); 912 if (IS_ERR(binding)) { 913 err = PTR_ERR(binding); 914 goto err_unlock; 915 } 916 917 nla_for_each_attr_type(attr, NETDEV_A_DMABUF_QUEUES, 918 genlmsg_data(info->genlhdr), 919 genlmsg_len(info->genlhdr), rem) { 920 err = nla_parse_nested( 921 tb, ARRAY_SIZE(netdev_queue_id_nl_policy) - 1, attr, 922 netdev_queue_id_nl_policy, info->extack); 923 if (err < 0) 924 goto err_unbind; 925 926 if (NL_REQ_ATTR_CHECK(info->extack, attr, tb, NETDEV_A_QUEUE_ID) || 927 NL_REQ_ATTR_CHECK(info->extack, attr, tb, NETDEV_A_QUEUE_TYPE)) { 928 err = -EINVAL; 929 goto err_unbind; 930 } 931 932 if (nla_get_u32(tb[NETDEV_A_QUEUE_TYPE]) != NETDEV_QUEUE_TYPE_RX) { 933 NL_SET_BAD_ATTR(info->extack, tb[NETDEV_A_QUEUE_TYPE]); 934 err = -EINVAL; 935 goto err_unbind; 936 } 937 938 rxq_idx = nla_get_u32(tb[NETDEV_A_QUEUE_ID]); 939 940 err = net_devmem_bind_dmabuf_to_queue(netdev, rxq_idx, binding, 941 info->extack); 942 if (err) 943 goto err_unbind; 944 } 945 946 list_add(&binding->list, &priv->bindings); 947 948 nla_put_u32(rsp, NETDEV_A_DMABUF_ID, binding->id); 949 genlmsg_end(rsp, hdr); 950 951 err = genlmsg_reply(rsp, info); 952 if (err) 953 goto err_unbind; 954 955 netdev_unlock(netdev); 956 957 mutex_unlock(&priv->lock); 958 959 return 0; 960 961 err_unbind: 962 net_devmem_unbind_dmabuf(binding); 963 err_unlock: 964 netdev_unlock(netdev); 965 err_unlock_sock: 966 mutex_unlock(&priv->lock); 967 err_genlmsg_free: 968 nlmsg_free(rsp); 969 return err; 970 } 971 972 int netdev_nl_bind_tx_doit(struct sk_buff *skb, struct genl_info *info) 973 { 974 struct net_devmem_dmabuf_binding *binding; 975 struct netdev_nl_sock *priv; 976 struct net_device *netdev; 977 u32 ifindex, dmabuf_fd; 978 struct sk_buff *rsp; 979 int err = 0; 980 void *hdr; 981 982 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX) || 983 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_FD)) 984 return -EINVAL; 985 986 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]); 987 dmabuf_fd = nla_get_u32(info->attrs[NETDEV_A_DMABUF_FD]); 988 989 priv = genl_sk_priv_get(&netdev_nl_family, NETLINK_CB(skb).sk); 990 if (IS_ERR(priv)) 991 return PTR_ERR(priv); 992 993 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL); 994 if (!rsp) 995 return -ENOMEM; 996 997 hdr = genlmsg_iput(rsp, info); 998 if (!hdr) { 999 err = -EMSGSIZE; 1000 goto err_genlmsg_free; 1001 } 1002 1003 mutex_lock(&priv->lock); 1004 1005 netdev = netdev_get_by_index_lock(genl_info_net(info), ifindex); 1006 if (!netdev) { 1007 err = -ENODEV; 1008 goto err_unlock_sock; 1009 } 1010 1011 if (!netif_device_present(netdev)) { 1012 err = -ENODEV; 1013 goto err_unlock_netdev; 1014 } 1015 1016 if (!netdev->netmem_tx) { 1017 err = -EOPNOTSUPP; 1018 NL_SET_ERR_MSG(info->extack, 1019 "Driver does not support netmem TX"); 1020 goto err_unlock_netdev; 1021 } 1022 1023 binding = net_devmem_bind_dmabuf(netdev, DMA_TO_DEVICE, dmabuf_fd, 1024 info->extack); 1025 if (IS_ERR(binding)) { 1026 err = PTR_ERR(binding); 1027 goto err_unlock_netdev; 1028 } 1029 1030 list_add(&binding->list, &priv->bindings); 1031 1032 nla_put_u32(rsp, NETDEV_A_DMABUF_ID, binding->id); 1033 genlmsg_end(rsp, hdr); 1034 1035 netdev_unlock(netdev); 1036 mutex_unlock(&priv->lock); 1037 1038 return genlmsg_reply(rsp, info); 1039 1040 err_unlock_netdev: 1041 netdev_unlock(netdev); 1042 err_unlock_sock: 1043 mutex_unlock(&priv->lock); 1044 err_genlmsg_free: 1045 nlmsg_free(rsp); 1046 return err; 1047 } 1048 1049 void netdev_nl_sock_priv_init(struct netdev_nl_sock *priv) 1050 { 1051 INIT_LIST_HEAD(&priv->bindings); 1052 mutex_init(&priv->lock); 1053 } 1054 1055 void netdev_nl_sock_priv_destroy(struct netdev_nl_sock *priv) 1056 { 1057 struct net_devmem_dmabuf_binding *binding; 1058 struct net_devmem_dmabuf_binding *temp; 1059 netdevice_tracker dev_tracker; 1060 struct net_device *dev; 1061 1062 mutex_lock(&priv->lock); 1063 list_for_each_entry_safe(binding, temp, &priv->bindings, list) { 1064 mutex_lock(&binding->lock); 1065 dev = binding->dev; 1066 if (!dev) { 1067 mutex_unlock(&binding->lock); 1068 net_devmem_unbind_dmabuf(binding); 1069 continue; 1070 } 1071 netdev_hold(dev, &dev_tracker, GFP_KERNEL); 1072 mutex_unlock(&binding->lock); 1073 1074 netdev_lock(dev); 1075 net_devmem_unbind_dmabuf(binding); 1076 netdev_unlock(dev); 1077 netdev_put(dev, &dev_tracker); 1078 } 1079 mutex_unlock(&priv->lock); 1080 } 1081 1082 static int netdev_genl_netdevice_event(struct notifier_block *nb, 1083 unsigned long event, void *ptr) 1084 { 1085 struct net_device *netdev = netdev_notifier_info_to_dev(ptr); 1086 1087 switch (event) { 1088 case NETDEV_REGISTER: 1089 netdev_lock_ops_to_full(netdev); 1090 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_ADD_NTF); 1091 netdev_unlock_full_to_ops(netdev); 1092 break; 1093 case NETDEV_UNREGISTER: 1094 netdev_lock(netdev); 1095 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_DEL_NTF); 1096 netdev_unlock(netdev); 1097 break; 1098 case NETDEV_XDP_FEAT_CHANGE: 1099 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_CHANGE_NTF); 1100 break; 1101 } 1102 1103 return NOTIFY_OK; 1104 } 1105 1106 static struct notifier_block netdev_genl_nb = { 1107 .notifier_call = netdev_genl_netdevice_event, 1108 }; 1109 1110 static int __init netdev_genl_init(void) 1111 { 1112 int err; 1113 1114 err = register_netdevice_notifier(&netdev_genl_nb); 1115 if (err) 1116 return err; 1117 1118 err = genl_register_family(&netdev_nl_family); 1119 if (err) 1120 goto err_unreg_ntf; 1121 1122 return 0; 1123 1124 err_unreg_ntf: 1125 unregister_netdevice_notifier(&netdev_genl_nb); 1126 return err; 1127 } 1128 1129 subsys_initcall(netdev_genl_init); 1130