Lines Matching +full:xdp +full:- +full:rx +full:- +full:metadata
1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
4 #include <linux/dma-mapping.h>
22 /* Per packet headroom in non-XDP mode. Present only for 1-frag packets. */
40 if (unlikely(buf->pg_refs < MIN_PAGE_REFS)) { in refresh_refs()
41 buf->pg_refs += EXTRA_PAGE_REFS; in refresh_refs()
42 page_ref_add(buf->page, EXTRA_PAGE_REFS); in refresh_refs()
46 /* Offer a buffer to the Rx buffer cache. The cache will hold the buffer if its
52 struct funeth_rx_cache *c = &q->cache; in cache_offer()
54 if (c->prod_cnt - c->cons_cnt <= c->mask && buf->node == numa_mem_id()) { in cache_offer()
55 c->bufs[c->prod_cnt & c->mask] = *buf; in cache_offer()
56 c->prod_cnt++; in cache_offer()
58 dma_unmap_page_attrs(q->dma_dev, buf->dma_addr, PAGE_SIZE, in cache_offer()
60 __page_frag_cache_drain(buf->page, buf->pg_refs); in cache_offer()
64 /* Get a page from the Rx buffer cache. We only consider the next available
69 struct funeth_rx_cache *c = &q->cache; in cache_get()
72 if (c->prod_cnt == c->cons_cnt) in cache_get()
75 buf = &c->bufs[c->cons_cnt & c->mask]; in cache_get()
76 if (page_ref_count(buf->page) == buf->pg_refs) { in cache_get()
77 dma_sync_single_for_device(q->dma_dev, buf->dma_addr, in cache_get()
80 buf->page = NULL; in cache_get()
82 c->cons_cnt++; in cache_get()
87 if (c->prod_cnt - c->cons_cnt > c->mask) { in cache_get()
88 dma_unmap_page_attrs(q->dma_dev, buf->dma_addr, PAGE_SIZE, in cache_get()
90 __page_frag_cache_drain(buf->page, buf->pg_refs); in cache_get()
91 buf->page = NULL; in cache_get()
92 c->cons_cnt++; in cache_get()
97 /* Allocate and DMA-map a page for receive. */
108 return -ENOMEM; in funeth_alloc_page()
110 rb->dma_addr = dma_map_page(q->dma_dev, p, 0, PAGE_SIZE, in funeth_alloc_page()
112 if (unlikely(dma_mapping_error(q->dma_dev, rb->dma_addr))) { in funeth_alloc_page()
115 return -ENOMEM; in funeth_alloc_page()
120 rb->page = p; in funeth_alloc_page()
121 rb->pg_refs = 1; in funeth_alloc_page()
123 rb->node = page_is_pfmemalloc(p) ? -1 : page_to_nid(p); in funeth_alloc_page()
129 if (rb->page) { in funeth_free_page()
130 dma_unmap_page(q->dma_dev, rb->dma_addr, PAGE_SIZE, in funeth_free_page()
132 __page_frag_cache_drain(rb->page, rb->pg_refs); in funeth_free_page()
133 rb->page = NULL; in funeth_free_page()
137 /* Run the XDP program assigned to an Rx queue.
146 struct xdp_buff xdp; in fun_run_xdp() local
150 xdp_init_buff(&xdp, ALIGN(skb_frag_size(frags), FUN_EPRQ_PKT_ALIGN), in fun_run_xdp()
151 &q->xdp_rxq); in fun_run_xdp()
152 xdp_prepare_buff(&xdp, buf_va, FUN_XDP_HEADROOM, skb_frag_size(frags) - in fun_run_xdp()
155 xdp_prog = READ_ONCE(q->xdp_prog); in fun_run_xdp()
156 act = bpf_prog_run_xdp(xdp_prog, &xdp); in fun_run_xdp()
161 skb_frag_size_set(frags, xdp.data_end - xdp.data); in fun_run_xdp()
162 skb_frag_off_add(frags, xdp.data - xdp.data_hard_start); in fun_run_xdp()
168 xdpf = xdp_convert_buff_to_frame(&xdp); in fun_run_xdp()
172 q->xdp_flush |= FUN_XDP_FLUSH_TX; in fun_run_xdp()
177 if (unlikely(xdp_do_redirect(q->netdev, &xdp, xdp_prog))) in fun_run_xdp()
180 q->xdp_flush |= FUN_XDP_FLUSH_REDIR; in fun_run_xdp()
183 bpf_warn_invalid_xdp_action(q->netdev, xdp_prog, act); in fun_run_xdp()
186 trace_xdp_exception(q->netdev, xdp_prog, act); in fun_run_xdp()
188 q->cur_buf->pg_refs++; /* return frags' page reference */ in fun_run_xdp()
192 q->cur_buf->pg_refs++; in fun_run_xdp()
199 return xdp.data; in fun_run_xdp()
202 /* A CQE contains a fixed completion structure along with optional metadata and
214 return cqe_info - FUNETH_CQE_INFO_OFFSET; in info_to_cqe()
237 /* Each received packet can be scattered across several Rx buffers or can
242 * - If the buffer at the head of an RQ has not been used it gets (part of) the
244 * - Otherwise, if the packet fully fits in the buffer's remaining space the
246 * - Otherwise, the packet goes into the next Rx buffer.
248 * This function returns the Rx buffer for a packet or fragment thereof of the
258 if (q->buf_offset + len <= PAGE_SIZE || !q->buf_offset) in get_buf()
264 * available and the current packet occupies @buf it is a multi-frag in get_buf()
267 if ((page_ref_count(buf->page) == buf->pg_refs && in get_buf()
268 buf->node == numa_mem_id()) || !q->spare_buf.page) { in get_buf()
269 dma_sync_single_for_device(q->dma_dev, buf->dma_addr, in get_buf()
274 *buf = q->spare_buf; in get_buf()
275 q->spare_buf.page = NULL; in get_buf()
276 q->rqes[q->rq_cons & q->rq_mask] = in get_buf()
277 FUN_EPRQ_RQBUF_INIT(buf->dma_addr); in get_buf()
279 q->buf_offset = 0; in get_buf()
280 q->rq_cons++; in get_buf()
281 return &q->bufs[q->rq_cons & q->rq_mask]; in get_buf()
284 /* Gather the page fragments making up the first Rx packet on @q. Its total
285 * length @tot_len includes optional head- and tail-rooms.
290 * A non-zero return value gives the caller permission to use references to the
299 struct funeth_rxbuf *buf = q->cur_buf; in fun_gather_pkt()
312 if (!q->spare_buf.page && in fun_gather_pkt()
313 funeth_alloc_page(q, &q->spare_buf, numa_mem_id(), in fun_gather_pkt()
318 PAGE_SIZE - q->buf_offset); in fun_gather_pkt()
319 dma_sync_single_for_cpu(q->dma_dev, in fun_gather_pkt()
320 buf->dma_addr + q->buf_offset, in fun_gather_pkt()
322 buf->pg_refs--; in fun_gather_pkt()
324 ref_ok |= buf->node; in fun_gather_pkt()
326 skb_frag_fill_page_desc(frags++, buf->page, q->buf_offset, in fun_gather_pkt()
329 tot_len -= frag_len; in fun_gather_pkt()
333 q->buf_offset = PAGE_SIZE; in fun_gather_pkt()
335 q->buf_offset = ALIGN(q->buf_offset + frag_len, FUN_EPRQ_PKT_ALIGN); in fun_gather_pkt()
336 q->cur_buf = buf; in fun_gather_pkt()
344 return d->hwtstamp_cfg.rx_filter == HWTSTAMP_FILTER_ALL; in rx_hwtstamp_enabled()
350 if (unlikely(q->cq_head == q->cq_mask)) { in advance_cq()
351 q->cq_head = 0; in advance_cq()
352 q->phase ^= 1; in advance_cq()
353 q->next_cqe_info = cqe_to_info(q->cqes); in advance_cq()
355 q->cq_head++; in advance_cq()
356 q->next_cqe_info += FUNETH_CQE_SIZE; in advance_cq()
358 prefetch(q->next_cqe_info); in advance_cq()
362 * fragments, run it through the optional XDP program, and if needed construct
367 const struct fun_eth_cqe *rxreq = info_to_cqe(q->next_cqe_info); in fun_handle_cqe_pkt()
368 unsigned int i, tot_len, pkt_len = be32_to_cpu(rxreq->pkt_len); in fun_handle_cqe_pkt()
369 struct net_device *ndev = q->netdev; in fun_handle_cqe_pkt()
379 u64_stats_update_begin(&q->syncp); in fun_handle_cqe_pkt()
380 q->stats.rx_pkts++; in fun_handle_cqe_pkt()
381 q->stats.rx_bytes += pkt_len; in fun_handle_cqe_pkt()
382 u64_stats_update_end(&q->syncp); in fun_handle_cqe_pkt()
386 /* account for head- and tail-room, present only for 1-buffer packets */ in fun_handle_cqe_pkt()
388 headroom = be16_to_cpu(rxreq->headroom); in fun_handle_cqe_pkt()
412 skb->protocol = eth_type_trans(skb, ndev); in fun_handle_cqe_pkt()
415 skb = napi_get_frags(q->napi); in fun_handle_cqe_pkt()
420 skb->pfmemalloc = 1; in fun_handle_cqe_pkt()
423 si->nr_frags = rxreq->nsgl; in fun_handle_cqe_pkt()
424 for (i = 0; i < si->nr_frags; i++) in fun_handle_cqe_pkt()
425 si->frags[i] = frags[i]; in fun_handle_cqe_pkt()
427 skb->len = pkt_len; in fun_handle_cqe_pkt()
428 skb->data_len = pkt_len; in fun_handle_cqe_pkt()
429 skb->truesize += round_up(pkt_len, FUN_EPRQ_PKT_ALIGN); in fun_handle_cqe_pkt()
432 skb_record_rx_queue(skb, q->qidx); in fun_handle_cqe_pkt()
433 cv = be16_to_cpu(rxreq->pkt_cv); in fun_handle_cqe_pkt()
434 if (likely((q->netdev->features & NETIF_F_RXHASH) && rxreq->hash)) in fun_handle_cqe_pkt()
435 skb_set_hash(skb, be32_to_cpu(rxreq->hash), in fun_handle_cqe_pkt()
437 if (likely((q->netdev->features & NETIF_F_RXCSUM) && rxreq->csum)) { in fun_handle_cqe_pkt()
439 skb->ip_summed = CHECKSUM_UNNECESSARY; in fun_handle_cqe_pkt()
440 skb->csum_level = be16_to_cpu(rxreq->csum) - 1; in fun_handle_cqe_pkt()
442 if (unlikely(rx_hwtstamp_enabled(q->netdev))) in fun_handle_cqe_pkt()
443 skb_hwtstamps(skb)->hwtstamp = be64_to_cpu(rxreq->timestamp); in fun_handle_cqe_pkt()
445 trace_funeth_rx(q, rxreq->nsgl, pkt_len, skb->hash, cv); in fun_handle_cqe_pkt()
447 gro_res = skb->data_len ? napi_gro_frags(q->napi) : in fun_handle_cqe_pkt()
448 napi_gro_receive(q->napi, skb); in fun_handle_cqe_pkt()
461 q->cur_buf->pg_refs++; in fun_handle_cqe_pkt()
462 for (i = 0; i < rxreq->nsgl - 1; i++) in fun_handle_cqe_pkt()
471 u16 sf_p = be16_to_cpu(ci->sf_p); in cqe_phase_mismatch()
481 struct funeth_priv *fp = netdev_priv(q->netdev); in fun_process_cqes()
484 xdpqs = rcu_dereference_bh(fp->xdpqs); in fun_process_cqes()
488 while (budget && !cqe_phase_mismatch(q->next_cqe_info, q->phase)) { in fun_process_cqes()
493 budget--; in fun_process_cqes()
496 if (unlikely(q->xdp_flush)) { in fun_process_cqes()
497 if (q->xdp_flush & FUN_XDP_FLUSH_TX) in fun_process_cqes()
499 if (q->xdp_flush & FUN_XDP_FLUSH_REDIR) in fun_process_cqes()
501 q->xdp_flush = 0; in fun_process_cqes()
507 /* NAPI handler for Rx queues. Calls the CQE processing loop and writes RQ/CQ
513 struct funeth_rxq *q = irq->rxq; in fun_rxq_napi_poll()
514 int work_done = budget - fun_process_cqes(q, budget); in fun_rxq_napi_poll()
515 u32 cq_db_val = q->cq_head; in fun_rxq_napi_poll()
520 cq_db_val |= q->irq_db_val; in fun_rxq_napi_poll()
522 /* check whether to post new Rx buffers */ in fun_rxq_napi_poll()
523 if (q->rq_cons - q->rq_cons_db >= q->rq_db_thres) { in fun_rxq_napi_poll()
524 u64_stats_update_begin(&q->syncp); in fun_rxq_napi_poll()
525 q->stats.rx_bufs += q->rq_cons - q->rq_cons_db; in fun_rxq_napi_poll()
526 u64_stats_update_end(&q->syncp); in fun_rxq_napi_poll()
527 q->rq_cons_db = q->rq_cons; in fun_rxq_napi_poll()
528 writel((q->rq_cons - 1) & q->rq_mask, q->rq_db); in fun_rxq_napi_poll()
531 writel(cq_db_val, q->cq_db); in fun_rxq_napi_poll()
535 /* Free the Rx buffers of an Rx queue. */
538 struct funeth_rxbuf *b = q->bufs; in fun_rxq_free_bufs()
541 for (i = 0; i <= q->rq_mask; i++, b++) in fun_rxq_free_bufs()
544 funeth_free_page(q, &q->spare_buf); in fun_rxq_free_bufs()
545 q->cur_buf = NULL; in fun_rxq_free_bufs()
548 /* Initially provision an Rx queue with Rx buffers. */
551 struct funeth_rxbuf *b = q->bufs; in fun_rxq_alloc_bufs()
554 for (i = 0; i <= q->rq_mask; i++, b++) { in fun_rxq_alloc_bufs()
557 return -ENOMEM; in fun_rxq_alloc_bufs()
559 q->rqes[i] = FUN_EPRQ_RQBUF_INIT(b->dma_addr); in fun_rxq_alloc_bufs()
561 q->cur_buf = q->bufs; in fun_rxq_alloc_bufs()
565 /* Initialize a used-buffer cache of the given depth. */
569 c->mask = depth - 1; in fun_rxq_init_cache()
570 c->bufs = kvzalloc_node(depth * sizeof(*c->bufs), GFP_KERNEL, node); in fun_rxq_init_cache()
571 return c->bufs ? 0 : -ENOMEM; in fun_rxq_init_cache()
574 /* Deallocate an Rx queue's used-buffer cache and its contents. */
577 struct funeth_rxbuf *b = q->cache.bufs; in fun_rxq_free_cache()
580 for (i = 0; i <= q->cache.mask; i++, b++) in fun_rxq_free_cache()
583 kvfree(q->cache.bufs); in fun_rxq_free_cache()
584 q->cache.bufs = NULL; in fun_rxq_free_cache()
589 struct funeth_priv *fp = netdev_priv(q->netdev); in fun_rxq_set_bpf()
595 if (headroom != q->headroom) { in fun_rxq_set_bpf()
600 0, q->hw_cqid, headroom); in fun_rxq_set_bpf()
601 err = fun_submit_admin_sync_cmd(fp->fdev, &cmd.common, NULL, 0, in fun_rxq_set_bpf()
605 q->headroom = headroom; in fun_rxq_set_bpf()
608 WRITE_ONCE(q->xdp_prog, prog); in fun_rxq_set_bpf()
612 /* Create an Rx queue, allocating the host memory it needs. */
621 int err = -ENOMEM; in fun_rxq_create_sw()
629 q->qidx = qidx; in fun_rxq_create_sw()
630 q->netdev = dev; in fun_rxq_create_sw()
631 q->cq_mask = ncqe - 1; in fun_rxq_create_sw()
632 q->rq_mask = nrqe - 1; in fun_rxq_create_sw()
633 q->numa_node = numa_node; in fun_rxq_create_sw()
634 q->rq_db_thres = nrqe / 4; in fun_rxq_create_sw()
635 u64_stats_init(&q->syncp); in fun_rxq_create_sw()
636 q->dma_dev = &fp->pdev->dev; in fun_rxq_create_sw()
638 q->rqes = fun_alloc_ring_mem(q->dma_dev, nrqe, sizeof(*q->rqes), in fun_rxq_create_sw()
639 sizeof(*q->bufs), false, numa_node, in fun_rxq_create_sw()
640 &q->rq_dma_addr, (void **)&q->bufs, NULL); in fun_rxq_create_sw()
641 if (!q->rqes) in fun_rxq_create_sw()
644 q->cqes = fun_alloc_ring_mem(q->dma_dev, ncqe, FUNETH_CQE_SIZE, 0, in fun_rxq_create_sw()
645 false, numa_node, &q->cq_dma_addr, NULL, in fun_rxq_create_sw()
647 if (!q->cqes) in fun_rxq_create_sw()
650 err = fun_rxq_init_cache(&q->cache, nrqe, numa_node); in fun_rxq_create_sw()
658 q->stats.rx_bufs = q->rq_mask; in fun_rxq_create_sw()
659 q->init_state = FUN_QSTATE_INIT_SW; in fun_rxq_create_sw()
665 dma_free_coherent(q->dma_dev, ncqe * FUNETH_CQE_SIZE, q->cqes, in fun_rxq_create_sw()
666 q->cq_dma_addr); in fun_rxq_create_sw()
668 fun_free_ring_mem(q->dma_dev, nrqe, sizeof(*q->rqes), false, q->rqes, in fun_rxq_create_sw()
669 q->rq_dma_addr, q->bufs); in fun_rxq_create_sw()
673 netdev_err(dev, "Unable to allocate memory for Rx queue %u\n", qidx); in fun_rxq_create_sw()
679 struct funeth_priv *fp = netdev_priv(q->netdev); in fun_rxq_free_sw()
683 fun_free_ring_mem(q->dma_dev, q->rq_mask + 1, sizeof(*q->rqes), false, in fun_rxq_free_sw()
684 q->rqes, q->rq_dma_addr, q->bufs); in fun_rxq_free_sw()
685 dma_free_coherent(q->dma_dev, (q->cq_mask + 1) * FUNETH_CQE_SIZE, in fun_rxq_free_sw()
686 q->cqes, q->cq_dma_addr); in fun_rxq_free_sw()
689 fp->rx_packets += q->stats.rx_pkts; in fun_rxq_free_sw()
690 fp->rx_bytes += q->stats.rx_bytes; in fun_rxq_free_sw()
691 fp->rx_dropped += q->stats.rx_map_err + q->stats.rx_mem_drops; in fun_rxq_free_sw()
696 /* Create an Rx queue's resources on the device. */
699 struct funeth_priv *fp = netdev_priv(q->netdev); in fun_rxq_create_dev()
700 unsigned int ncqe = q->cq_mask + 1; in fun_rxq_create_dev()
701 unsigned int nrqe = q->rq_mask + 1; in fun_rxq_create_dev()
704 err = xdp_rxq_info_reg(&q->xdp_rxq, q->netdev, q->qidx, in fun_rxq_create_dev()
705 irq->napi.napi_id); in fun_rxq_create_dev()
709 err = xdp_rxq_info_reg_mem_model(&q->xdp_rxq, MEM_TYPE_PAGE_SHARED, in fun_rxq_create_dev()
714 q->phase = 1; in fun_rxq_create_dev()
715 q->irq_cnt = 0; in fun_rxq_create_dev()
716 q->cq_head = 0; in fun_rxq_create_dev()
717 q->rq_cons = 0; in fun_rxq_create_dev()
718 q->rq_cons_db = 0; in fun_rxq_create_dev()
719 q->buf_offset = 0; in fun_rxq_create_dev()
720 q->napi = &irq->napi; in fun_rxq_create_dev()
721 q->irq_db_val = fp->cq_irq_db; in fun_rxq_create_dev()
722 q->next_cqe_info = cqe_to_info(q->cqes); in fun_rxq_create_dev()
724 q->xdp_prog = fp->xdp_prog; in fun_rxq_create_dev()
725 q->headroom = fp->xdp_prog ? FUN_XDP_HEADROOM : FUN_RX_HEADROOM; in fun_rxq_create_dev()
727 err = fun_sq_create(fp->fdev, FUN_ADMIN_RES_CREATE_FLAG_ALLOCATOR | in fun_rxq_create_dev()
729 FUN_HCI_ID_INVALID, 0, nrqe, q->rq_dma_addr, 0, 0, in fun_rxq_create_dev()
730 0, 0, fp->fdev->kern_end_qid, PAGE_SHIFT, in fun_rxq_create_dev()
731 &q->hw_sqid, &q->rq_db); in fun_rxq_create_dev()
735 err = fun_cq_create(fp->fdev, FUN_ADMIN_RES_CREATE_FLAG_ALLOCATOR | in fun_rxq_create_dev()
737 q->hw_sqid, ilog2(FUNETH_CQE_SIZE), ncqe, in fun_rxq_create_dev()
738 q->cq_dma_addr, q->headroom, FUN_RX_TAILROOM, 0, 0, in fun_rxq_create_dev()
739 irq->irq_idx, 0, fp->fdev->kern_end_qid, in fun_rxq_create_dev()
740 &q->hw_cqid, &q->cq_db); in fun_rxq_create_dev()
744 irq->rxq = q; in fun_rxq_create_dev()
745 writel(q->rq_mask, q->rq_db); in fun_rxq_create_dev()
746 q->init_state = FUN_QSTATE_INIT_FULL; in fun_rxq_create_dev()
748 netif_info(fp, ifup, q->netdev, in fun_rxq_create_dev()
749 "Rx queue %u, depth %u/%u, HW qid %u/%u, IRQ idx %u, node %d, headroom %u\n", in fun_rxq_create_dev()
750 q->qidx, ncqe, nrqe, q->hw_cqid, q->hw_sqid, irq->irq_idx, in fun_rxq_create_dev()
751 q->numa_node, q->headroom); in fun_rxq_create_dev()
755 fun_destroy_sq(fp->fdev, q->hw_sqid); in fun_rxq_create_dev()
757 xdp_rxq_info_unreg(&q->xdp_rxq); in fun_rxq_create_dev()
759 netdev_err(q->netdev, in fun_rxq_create_dev()
760 "Failed to create Rx queue %u on device, error %d\n", in fun_rxq_create_dev()
761 q->qidx, err); in fun_rxq_create_dev()
767 struct funeth_priv *fp = netdev_priv(q->netdev); in fun_rxq_free_dev()
770 if (q->init_state < FUN_QSTATE_INIT_FULL) in fun_rxq_free_dev()
773 irq = container_of(q->napi, struct fun_irq, napi); in fun_rxq_free_dev()
774 netif_info(fp, ifdown, q->netdev, in fun_rxq_free_dev()
775 "Freeing Rx queue %u (id %u/%u), IRQ %u\n", in fun_rxq_free_dev()
776 q->qidx, q->hw_cqid, q->hw_sqid, irq->irq_idx); in fun_rxq_free_dev()
778 irq->rxq = NULL; in fun_rxq_free_dev()
779 xdp_rxq_info_unreg(&q->xdp_rxq); in fun_rxq_free_dev()
780 fun_destroy_sq(fp->fdev, q->hw_sqid); in fun_rxq_free_dev()
781 fun_destroy_cq(fp->fdev, q->hw_cqid); in fun_rxq_free_dev()
782 q->init_state = FUN_QSTATE_INIT_SW; in fun_rxq_free_dev()
785 /* Create or advance an Rx queue, allocating all the host and device resources
801 if (q->init_state >= state) in funeth_rxq_create()
816 /* Free Rx queue resources until it reaches the target state. */