Lines Matching +full:queue +full:- +full:pkt +full:- +full:tx
1 // SPDX-License-Identifier: (GPL-2.0 OR MIT)
4 * Copyright (C) 2015-2021 Google, Inc.
20 iowrite32be(val, &priv->db_bar2[be32_to_cpu(q_resources->db_index)]); in gve_tx_put_doorbell()
26 struct gve_tx_ring *tx = &priv->tx[tx_qid]; in gve_xdp_tx_flush() local
28 gve_tx_put_doorbell(priv, tx->q_resources, tx->req); in gve_xdp_tx_flush()
32 * We copy skb payloads into the registered segment before writing Tx
33 * descriptors and ringing the Tx doorbell.
35 * gve_tx_fifo_* manages the Registered Segment as a FIFO - clients must
41 fifo->base = vmap(fifo->qpl->pages, fifo->qpl->num_entries, VM_MAP, in gve_tx_fifo_init()
43 if (unlikely(!fifo->base)) { in gve_tx_fifo_init()
44 netif_err(priv, drv, priv->dev, "Failed to vmap fifo, qpl_id = %d\n", in gve_tx_fifo_init()
45 fifo->qpl->id); in gve_tx_fifo_init()
46 return -ENOMEM; in gve_tx_fifo_init()
49 fifo->size = fifo->qpl->num_entries * PAGE_SIZE; in gve_tx_fifo_init()
50 atomic_set(&fifo->available, fifo->size); in gve_tx_fifo_init()
51 fifo->head = 0; in gve_tx_fifo_init()
57 WARN(atomic_read(&fifo->available) != fifo->size, in gve_tx_fifo_release()
58 "Releasing non-empty fifo"); in gve_tx_fifo_release()
60 vunmap(fifo->base); in gve_tx_fifo_release()
66 return (fifo->head + bytes < fifo->size) ? 0 : fifo->size - fifo->head; in gve_tx_fifo_pad_alloc_one_frag()
71 return (atomic_read(&fifo->available) <= bytes) ? false : true; in gve_tx_fifo_can_alloc()
74 /* gve_tx_alloc_fifo - Allocate fragment(s) from Tx FIFO
77 * @iov: Scatter-gather elements to fill with allocation fragment base/len
105 iov[0].iov_offset = fifo->head; in gve_tx_alloc_fifo()
107 fifo->head += bytes; in gve_tx_alloc_fifo()
109 if (fifo->head > fifo->size) { in gve_tx_alloc_fifo()
114 overflow = fifo->head - fifo->size; in gve_tx_alloc_fifo()
115 iov[0].iov_len -= overflow; in gve_tx_alloc_fifo()
119 fifo->head = overflow; in gve_tx_alloc_fifo()
122 /* Re-align to a cacheline boundary */ in gve_tx_alloc_fifo()
123 aligned_head = L1_CACHE_ALIGN(fifo->head); in gve_tx_alloc_fifo()
124 padding = aligned_head - fifo->head; in gve_tx_alloc_fifo()
125 iov[nfrags - 1].iov_padding = padding; in gve_tx_alloc_fifo()
126 atomic_sub(bytes + padding, &fifo->available); in gve_tx_alloc_fifo()
127 fifo->head = aligned_head; in gve_tx_alloc_fifo()
129 if (fifo->head == fifo->size) in gve_tx_alloc_fifo()
130 fifo->head = 0; in gve_tx_alloc_fifo()
135 /* gve_tx_free_fifo - Return space to Tx FIFO
141 atomic_add(bytes, &fifo->available); in gve_tx_free_fifo()
149 for (i = 0; i < ARRAY_SIZE(info->iov); i++) { in gve_tx_clear_buffer_state()
150 space_freed += info->iov[i].iov_len + info->iov[i].iov_padding; in gve_tx_clear_buffer_state()
151 info->iov[i].iov_len = 0; in gve_tx_clear_buffer_state()
152 info->iov[i].iov_padding = 0; in gve_tx_clear_buffer_state()
157 static int gve_clean_xdp_done(struct gve_priv *priv, struct gve_tx_ring *tx, in gve_clean_xdp_done() argument
161 u32 clean_end = tx->done + to_do; in gve_clean_xdp_done()
167 for (; tx->done < clean_end; tx->done++) { in gve_clean_xdp_done()
168 idx = tx->done & tx->mask; in gve_clean_xdp_done()
169 info = &tx->info[idx]; in gve_clean_xdp_done()
171 if (unlikely(!info->xdp.size)) in gve_clean_xdp_done()
174 bytes += info->xdp.size; in gve_clean_xdp_done()
176 xsk_complete += info->xdp.is_xsk; in gve_clean_xdp_done()
178 info->xdp.size = 0; in gve_clean_xdp_done()
179 if (info->xdp_frame) { in gve_clean_xdp_done()
180 xdp_return_frame(info->xdp_frame); in gve_clean_xdp_done()
181 info->xdp_frame = NULL; in gve_clean_xdp_done()
186 gve_tx_free_fifo(&tx->tx_fifo, space_freed); in gve_clean_xdp_done()
187 if (xsk_complete > 0 && tx->xsk_pool) in gve_clean_xdp_done()
188 xsk_tx_completed(tx->xsk_pool, xsk_complete); in gve_clean_xdp_done()
189 u64_stats_update_begin(&tx->statss); in gve_clean_xdp_done()
190 tx->bytes_done += bytes; in gve_clean_xdp_done()
191 tx->pkt_done += pkts; in gve_clean_xdp_done()
192 u64_stats_update_end(&tx->statss); in gve_clean_xdp_done()
196 static int gve_clean_tx_done(struct gve_priv *priv, struct gve_tx_ring *tx,
201 struct gve_tx_ring *tx = &priv->tx[idx]; in gve_tx_free_ring() local
202 struct device *hdev = &priv->pdev->dev; in gve_tx_free_ring()
207 slots = tx->mask + 1; in gve_tx_free_ring()
208 if (tx->q_num < priv->tx_cfg.num_queues) { in gve_tx_free_ring()
209 gve_clean_tx_done(priv, tx, priv->tx_desc_cnt, false); in gve_tx_free_ring()
210 netdev_tx_reset_queue(tx->netdev_txq); in gve_tx_free_ring()
212 gve_clean_xdp_done(priv, tx, priv->tx_desc_cnt); in gve_tx_free_ring()
215 dma_free_coherent(hdev, sizeof(*tx->q_resources), in gve_tx_free_ring()
216 tx->q_resources, tx->q_resources_bus); in gve_tx_free_ring()
217 tx->q_resources = NULL; in gve_tx_free_ring()
219 if (!tx->raw_addressing) { in gve_tx_free_ring()
220 gve_tx_fifo_release(priv, &tx->tx_fifo); in gve_tx_free_ring()
221 gve_unassign_qpl(priv, tx->tx_fifo.qpl->id); in gve_tx_free_ring()
222 tx->tx_fifo.qpl = NULL; in gve_tx_free_ring()
225 bytes = sizeof(*tx->desc) * slots; in gve_tx_free_ring()
226 dma_free_coherent(hdev, bytes, tx->desc, tx->bus); in gve_tx_free_ring()
227 tx->desc = NULL; in gve_tx_free_ring()
229 vfree(tx->info); in gve_tx_free_ring()
230 tx->info = NULL; in gve_tx_free_ring()
232 netif_dbg(priv, drv, priv->dev, "freed tx queue %d\n", idx); in gve_tx_free_ring()
237 struct gve_tx_ring *tx = &priv->tx[idx]; in gve_tx_alloc_ring() local
238 struct device *hdev = &priv->pdev->dev; in gve_tx_alloc_ring()
239 u32 slots = priv->tx_desc_cnt; in gve_tx_alloc_ring()
243 memset(tx, 0, sizeof(*tx)); in gve_tx_alloc_ring()
244 spin_lock_init(&tx->clean_lock); in gve_tx_alloc_ring()
245 spin_lock_init(&tx->xdp_lock); in gve_tx_alloc_ring()
246 tx->q_num = idx; in gve_tx_alloc_ring()
248 tx->mask = slots - 1; in gve_tx_alloc_ring()
251 tx->info = vcalloc(slots, sizeof(*tx->info)); in gve_tx_alloc_ring()
252 if (!tx->info) in gve_tx_alloc_ring()
253 return -ENOMEM; in gve_tx_alloc_ring()
255 /* alloc tx queue */ in gve_tx_alloc_ring()
256 bytes = sizeof(*tx->desc) * slots; in gve_tx_alloc_ring()
257 tx->desc = dma_alloc_coherent(hdev, bytes, &tx->bus, GFP_KERNEL); in gve_tx_alloc_ring()
258 if (!tx->desc) in gve_tx_alloc_ring()
261 tx->raw_addressing = priv->queue_format == GVE_GQI_RDA_FORMAT; in gve_tx_alloc_ring()
262 tx->dev = &priv->pdev->dev; in gve_tx_alloc_ring()
263 if (!tx->raw_addressing) { in gve_tx_alloc_ring()
264 tx->tx_fifo.qpl = gve_assign_tx_qpl(priv, idx); in gve_tx_alloc_ring()
265 if (!tx->tx_fifo.qpl) in gve_tx_alloc_ring()
267 /* map Tx FIFO */ in gve_tx_alloc_ring()
268 if (gve_tx_fifo_init(priv, &tx->tx_fifo)) in gve_tx_alloc_ring()
272 tx->q_resources = in gve_tx_alloc_ring()
274 sizeof(*tx->q_resources), in gve_tx_alloc_ring()
275 &tx->q_resources_bus, in gve_tx_alloc_ring()
277 if (!tx->q_resources) in gve_tx_alloc_ring()
280 netif_dbg(priv, drv, priv->dev, "tx[%d]->bus=%lx\n", idx, in gve_tx_alloc_ring()
281 (unsigned long)tx->bus); in gve_tx_alloc_ring()
282 if (idx < priv->tx_cfg.num_queues) in gve_tx_alloc_ring()
283 tx->netdev_txq = netdev_get_tx_queue(priv->dev, idx); in gve_tx_alloc_ring()
289 if (!tx->raw_addressing) in gve_tx_alloc_ring()
290 gve_tx_fifo_release(priv, &tx->tx_fifo); in gve_tx_alloc_ring()
292 if (!tx->raw_addressing) in gve_tx_alloc_ring()
293 gve_unassign_qpl(priv, tx->tx_fifo.qpl->id); in gve_tx_alloc_ring()
295 dma_free_coherent(hdev, bytes, tx->desc, tx->bus); in gve_tx_alloc_ring()
296 tx->desc = NULL; in gve_tx_alloc_ring()
298 vfree(tx->info); in gve_tx_alloc_ring()
299 tx->info = NULL; in gve_tx_alloc_ring()
300 return -ENOMEM; in gve_tx_alloc_ring()
311 netif_err(priv, drv, priv->dev, in gve_tx_alloc_rings()
312 "Failed to alloc tx ring=%d: err=%d\n", in gve_tx_alloc_rings()
335 /* gve_tx_avail - Calculates the number of slots available in the ring
336 * @tx: tx ring to check
340 * The capacity of the queue is mask + 1. We don't need to reserve an entry.
342 static inline u32 gve_tx_avail(struct gve_tx_ring *tx) in gve_tx_avail() argument
344 return tx->mask + 1 - (tx->req - tx->done); in gve_tx_avail()
347 static inline int gve_skb_fifo_bytes_required(struct gve_tx_ring *tx, in gve_skb_fifo_bytes_required() argument
355 min_t(int, GVE_GQ_TX_MIN_PKT_DESC_BYTES, skb->len); in gve_skb_fifo_bytes_required()
357 pad_bytes = gve_tx_fifo_pad_alloc_one_frag(&tx->tx_fifo, in gve_skb_fifo_bytes_required()
360 align_hdr_pad = L1_CACHE_ALIGN(hlen) - hlen; in gve_skb_fifo_bytes_required()
361 bytes = align_hdr_pad + pad_bytes + skb->len; in gve_skb_fifo_bytes_required()
376 if (info->skb) { in gve_tx_unmap_buf()
392 static inline bool gve_can_tx(struct gve_tx_ring *tx, int bytes_required) in gve_can_tx() argument
396 if (!tx->raw_addressing) in gve_can_tx()
397 can_alloc = gve_tx_fifo_can_alloc(&tx->tx_fifo, bytes_required); in gve_can_tx()
399 return (gve_tx_avail(tx) >= MAX_TX_DESC_NEEDED && can_alloc); in gve_can_tx()
404 /* Stops the queue if the skb cannot be transmitted. */
405 static int gve_maybe_stop_tx(struct gve_priv *priv, struct gve_tx_ring *tx, in gve_maybe_stop_tx() argument
413 if (!tx->raw_addressing) in gve_maybe_stop_tx()
414 bytes_required = gve_skb_fifo_bytes_required(tx, skb); in gve_maybe_stop_tx()
416 if (likely(gve_can_tx(tx, bytes_required))) in gve_maybe_stop_tx()
419 ret = -EBUSY; in gve_maybe_stop_tx()
420 spin_lock(&tx->clean_lock); in gve_maybe_stop_tx()
421 nic_done = gve_tx_load_event_counter(priv, tx); in gve_maybe_stop_tx()
422 to_do = nic_done - tx->done; in gve_maybe_stop_tx()
424 /* Only try to clean if there is hope for TX */ in gve_maybe_stop_tx()
425 if (to_do + gve_tx_avail(tx) >= MAX_TX_DESC_NEEDED) { in gve_maybe_stop_tx()
428 gve_clean_tx_done(priv, tx, to_do, false); in gve_maybe_stop_tx()
430 if (likely(gve_can_tx(tx, bytes_required))) in gve_maybe_stop_tx()
434 /* No space, so stop the queue */ in gve_maybe_stop_tx()
435 tx->stop_queue++; in gve_maybe_stop_tx()
436 netif_tx_stop_queue(tx->netdev_txq); in gve_maybe_stop_tx()
438 spin_unlock(&tx->clean_lock); in gve_maybe_stop_tx()
448 /* l4_hdr_offset and csum_offset are in units of 16-bit words */ in gve_tx_fill_pkt_desc()
450 pkt_desc->pkt.type_flags = GVE_TXD_TSO | GVE_TXF_L4CSUM; in gve_tx_fill_pkt_desc()
451 pkt_desc->pkt.l4_csum_offset = csum_offset >> 1; in gve_tx_fill_pkt_desc()
452 pkt_desc->pkt.l4_hdr_offset = l4_hdr_offset >> 1; in gve_tx_fill_pkt_desc()
454 pkt_desc->pkt.type_flags = GVE_TXD_STD | GVE_TXF_L4CSUM; in gve_tx_fill_pkt_desc()
455 pkt_desc->pkt.l4_csum_offset = csum_offset >> 1; in gve_tx_fill_pkt_desc()
456 pkt_desc->pkt.l4_hdr_offset = l4_hdr_offset >> 1; in gve_tx_fill_pkt_desc()
458 pkt_desc->pkt.type_flags = GVE_TXD_STD; in gve_tx_fill_pkt_desc()
459 pkt_desc->pkt.l4_csum_offset = 0; in gve_tx_fill_pkt_desc()
460 pkt_desc->pkt.l4_hdr_offset = 0; in gve_tx_fill_pkt_desc()
462 pkt_desc->pkt.desc_cnt = desc_cnt; in gve_tx_fill_pkt_desc()
463 pkt_desc->pkt.len = cpu_to_be16(pkt_len); in gve_tx_fill_pkt_desc()
464 pkt_desc->pkt.seg_len = cpu_to_be16(hlen); in gve_tx_fill_pkt_desc()
465 pkt_desc->pkt.seg_addr = cpu_to_be64(addr); in gve_tx_fill_pkt_desc()
471 BUILD_BUG_ON(sizeof(mtd_desc->mtd) != sizeof(mtd_desc->pkt)); in gve_tx_fill_mtd_desc()
473 mtd_desc->mtd.type_flags = GVE_TXD_MTD | GVE_MTD_SUBTYPE_PATH; in gve_tx_fill_mtd_desc()
474 mtd_desc->mtd.path_state = GVE_MTD_PATH_STATE_DEFAULT | in gve_tx_fill_mtd_desc()
476 mtd_desc->mtd.path_hash = cpu_to_be32(skb->hash); in gve_tx_fill_mtd_desc()
477 mtd_desc->mtd.reserved0 = 0; in gve_tx_fill_mtd_desc()
478 mtd_desc->mtd.reserved1 = 0; in gve_tx_fill_mtd_desc()
486 seg_desc->seg.type_flags = GVE_TXD_SEG; in gve_tx_fill_seg_desc()
489 seg_desc->seg.type_flags |= GVE_TXSF_IPV6; in gve_tx_fill_seg_desc()
490 seg_desc->seg.l3_offset = l3_offset >> 1; in gve_tx_fill_seg_desc()
491 seg_desc->seg.mss = cpu_to_be16(gso_size); in gve_tx_fill_seg_desc()
493 seg_desc->seg.seg_len = cpu_to_be16(len); in gve_tx_fill_seg_desc()
494 seg_desc->seg.seg_addr = cpu_to_be64(addr); in gve_tx_fill_seg_desc()
500 u64 last_page = (iov_offset + iov_len - 1) / PAGE_SIZE; in gve_dma_sync_for_device()
508 static int gve_tx_add_skb_copy(struct gve_priv *priv, struct gve_tx_ring *tx, struct sk_buff *skb) in gve_tx_add_skb_copy() argument
513 int mtd_desc_nr = !!skb->l4_hash; in gve_tx_add_skb_copy()
515 u32 idx = tx->req & tx->mask; in gve_tx_add_skb_copy()
521 info = &tx->info[idx]; in gve_tx_add_skb_copy()
522 pkt_desc = &tx->desc[idx]; in gve_tx_add_skb_copy()
529 min_t(int, GVE_GQ_TX_MIN_PKT_DESC_BYTES, skb->len); in gve_tx_add_skb_copy()
531 info->skb = skb; in gve_tx_add_skb_copy()
535 pad_bytes = gve_tx_fifo_pad_alloc_one_frag(&tx->tx_fifo, hlen); in gve_tx_add_skb_copy()
536 hdr_nfrags = gve_tx_alloc_fifo(&tx->tx_fifo, hlen + pad_bytes, in gve_tx_add_skb_copy()
537 &info->iov[0]); in gve_tx_add_skb_copy()
539 payload_nfrags = gve_tx_alloc_fifo(&tx->tx_fifo, skb->len - hlen, in gve_tx_add_skb_copy()
540 &info->iov[payload_iov]); in gve_tx_add_skb_copy()
542 gve_tx_fill_pkt_desc(pkt_desc, skb->csum_offset, skb->ip_summed, in gve_tx_add_skb_copy()
545 info->iov[hdr_nfrags - 1].iov_offset, skb->len); in gve_tx_add_skb_copy()
548 tx->tx_fifo.base + info->iov[hdr_nfrags - 1].iov_offset, in gve_tx_add_skb_copy()
550 gve_dma_sync_for_device(&priv->pdev->dev, tx->tx_fifo.qpl->page_buses, in gve_tx_add_skb_copy()
551 info->iov[hdr_nfrags - 1].iov_offset, in gve_tx_add_skb_copy()
552 info->iov[hdr_nfrags - 1].iov_len); in gve_tx_add_skb_copy()
556 next_idx = (tx->req + 1) & tx->mask; in gve_tx_add_skb_copy()
557 gve_tx_fill_mtd_desc(&tx->desc[next_idx], skb); in gve_tx_add_skb_copy()
561 next_idx = (tx->req + 1 + mtd_desc_nr + i - payload_iov) & tx->mask; in gve_tx_add_skb_copy()
562 seg_desc = &tx->desc[next_idx]; in gve_tx_add_skb_copy()
565 skb_shinfo(skb)->gso_size, in gve_tx_add_skb_copy()
567 info->iov[i].iov_len, in gve_tx_add_skb_copy()
568 info->iov[i].iov_offset); in gve_tx_add_skb_copy()
571 tx->tx_fifo.base + info->iov[i].iov_offset, in gve_tx_add_skb_copy()
572 info->iov[i].iov_len); in gve_tx_add_skb_copy()
573 gve_dma_sync_for_device(&priv->pdev->dev, tx->tx_fifo.qpl->page_buses, in gve_tx_add_skb_copy()
574 info->iov[i].iov_offset, in gve_tx_add_skb_copy()
575 info->iov[i].iov_len); in gve_tx_add_skb_copy()
576 copy_offset += info->iov[i].iov_len; in gve_tx_add_skb_copy()
582 static int gve_tx_add_skb_no_copy(struct gve_priv *priv, struct gve_tx_ring *tx, in gve_tx_add_skb_no_copy() argument
589 int mtd_desc_nr = !!skb->l4_hash; in gve_tx_add_skb_no_copy()
591 u32 idx = tx->req & tx->mask; in gve_tx_add_skb_no_copy()
596 info = &tx->info[idx]; in gve_tx_add_skb_no_copy()
597 pkt_desc = &tx->desc[idx]; in gve_tx_add_skb_no_copy()
602 * of the skb (which will contain the checksum because skb->csum_start and in gve_tx_add_skb_no_copy()
603 * skb->csum_offset are given relative to skb->head) in the first segment. in gve_tx_add_skb_no_copy()
608 info->skb = skb; in gve_tx_add_skb_no_copy()
610 addr = dma_map_single(tx->dev, skb->data, len, DMA_TO_DEVICE); in gve_tx_add_skb_no_copy()
611 if (unlikely(dma_mapping_error(tx->dev, addr))) { in gve_tx_add_skb_no_copy()
612 tx->dma_mapping_error++; in gve_tx_add_skb_no_copy()
618 num_descriptors = 1 + shinfo->nr_frags; in gve_tx_add_skb_no_copy()
624 gve_tx_fill_pkt_desc(pkt_desc, skb->csum_offset, skb->ip_summed, in gve_tx_add_skb_no_copy()
626 num_descriptors, hlen, addr, skb->len); in gve_tx_add_skb_no_copy()
629 idx = (idx + 1) & tx->mask; in gve_tx_add_skb_no_copy()
630 mtd_desc = &tx->desc[idx]; in gve_tx_add_skb_no_copy()
638 len -= hlen; in gve_tx_add_skb_no_copy()
640 idx = (idx + 1) & tx->mask; in gve_tx_add_skb_no_copy()
641 seg_desc = &tx->desc[idx]; in gve_tx_add_skb_no_copy()
643 skb_shinfo(skb)->gso_size, in gve_tx_add_skb_no_copy()
647 for (i = 0; i < shinfo->nr_frags; i++) { in gve_tx_add_skb_no_copy()
648 const skb_frag_t *frag = &shinfo->frags[i]; in gve_tx_add_skb_no_copy()
650 idx = (idx + 1) & tx->mask; in gve_tx_add_skb_no_copy()
651 seg_desc = &tx->desc[idx]; in gve_tx_add_skb_no_copy()
653 addr = skb_frag_dma_map(tx->dev, frag, 0, len, DMA_TO_DEVICE); in gve_tx_add_skb_no_copy()
654 if (unlikely(dma_mapping_error(tx->dev, addr))) { in gve_tx_add_skb_no_copy()
655 tx->dma_mapping_error++; in gve_tx_add_skb_no_copy()
658 tx->info[idx].skb = NULL; in gve_tx_add_skb_no_copy()
659 dma_unmap_len_set(&tx->info[idx], len, len); in gve_tx_add_skb_no_copy()
660 dma_unmap_addr_set(&tx->info[idx], dma, addr); in gve_tx_add_skb_no_copy()
663 skb_shinfo(skb)->gso_size, in gve_tx_add_skb_no_copy()
670 i += num_descriptors - shinfo->nr_frags; in gve_tx_add_skb_no_copy()
671 while (i--) { in gve_tx_add_skb_no_copy()
675 idx--; in gve_tx_add_skb_no_copy()
676 gve_tx_unmap_buf(tx->dev, &tx->info[idx & tx->mask]); in gve_tx_add_skb_no_copy()
679 tx->dropped_pkt++; in gve_tx_add_skb_no_copy()
686 struct gve_tx_ring *tx; in gve_tx() local
689 WARN(skb_get_queue_mapping(skb) >= priv->tx_cfg.num_queues, in gve_tx()
690 "skb queue index out of range"); in gve_tx()
691 tx = &priv->tx[skb_get_queue_mapping(skb)]; in gve_tx()
692 if (unlikely(gve_maybe_stop_tx(priv, tx, skb))) { in gve_tx()
693 /* We need to ring the txq doorbell -- we have stopped the Tx in gve_tx()
694 * queue for want of resources, but prior calls to gve_tx() in gve_tx()
698 gve_tx_put_doorbell(priv, tx->q_resources, tx->req); in gve_tx()
701 if (tx->raw_addressing) in gve_tx()
702 nsegs = gve_tx_add_skb_no_copy(priv, tx, skb); in gve_tx()
704 nsegs = gve_tx_add_skb_copy(priv, tx, skb); in gve_tx()
708 netdev_tx_sent_queue(tx->netdev_txq, skb->len); in gve_tx()
710 tx->req += nsegs; in gve_tx()
715 if (!netif_xmit_stopped(tx->netdev_txq) && netdev_xmit_more()) in gve_tx()
721 gve_tx_put_doorbell(priv, tx->q_resources, tx->req); in gve_tx()
725 static int gve_tx_fill_xdp(struct gve_priv *priv, struct gve_tx_ring *tx, in gve_tx_fill_xdp() argument
730 u32 reqi = tx->req; in gve_tx_fill_xdp()
732 pad = gve_tx_fifo_pad_alloc_one_frag(&tx->tx_fifo, len); in gve_tx_fill_xdp()
735 info = &tx->info[reqi & tx->mask]; in gve_tx_fill_xdp()
736 info->xdp_frame = frame_p; in gve_tx_fill_xdp()
737 info->xdp.size = len; in gve_tx_fill_xdp()
738 info->xdp.is_xsk = is_xsk; in gve_tx_fill_xdp()
740 nfrags = gve_tx_alloc_fifo(&tx->tx_fifo, pad + len, in gve_tx_fill_xdp()
741 &info->iov[0]); in gve_tx_fill_xdp()
743 ndescs = nfrags - iovi; in gve_tx_fill_xdp()
748 gve_tx_fill_pkt_desc(&tx->desc[reqi & tx->mask], 0, in gve_tx_fill_xdp()
750 info->iov[iovi].iov_len, in gve_tx_fill_xdp()
751 info->iov[iovi].iov_offset, len); in gve_tx_fill_xdp()
753 gve_tx_fill_seg_desc(&tx->desc[reqi & tx->mask], in gve_tx_fill_xdp()
755 info->iov[iovi].iov_len, in gve_tx_fill_xdp()
756 info->iov[iovi].iov_offset); in gve_tx_fill_xdp()
758 memcpy(tx->tx_fifo.base + info->iov[iovi].iov_offset, in gve_tx_fill_xdp()
759 data + offset, info->iov[iovi].iov_len); in gve_tx_fill_xdp()
760 gve_dma_sync_for_device(&priv->pdev->dev, in gve_tx_fill_xdp()
761 tx->tx_fifo.qpl->page_buses, in gve_tx_fill_xdp()
762 info->iov[iovi].iov_offset, in gve_tx_fill_xdp()
763 info->iov[iovi].iov_len); in gve_tx_fill_xdp()
764 offset += info->iov[iovi].iov_len; in gve_tx_fill_xdp()
776 struct gve_tx_ring *tx; in gve_xdp_xmit() local
780 return -EINVAL; in gve_xdp_xmit()
783 smp_processor_id() % priv->num_xdp_queues); in gve_xdp_xmit()
785 tx = &priv->tx[qid]; in gve_xdp_xmit()
787 spin_lock(&tx->xdp_lock); in gve_xdp_xmit()
789 err = gve_xdp_xmit_one(priv, tx, frames[i]->data, in gve_xdp_xmit()
790 frames[i]->len, frames[i]); in gve_xdp_xmit()
796 gve_tx_put_doorbell(priv, tx->q_resources, tx->req); in gve_xdp_xmit()
798 spin_unlock(&tx->xdp_lock); in gve_xdp_xmit()
800 u64_stats_update_begin(&tx->statss); in gve_xdp_xmit()
801 tx->xdp_xmit += n; in gve_xdp_xmit()
802 tx->xdp_xmit_errors += n - i; in gve_xdp_xmit()
803 u64_stats_update_end(&tx->statss); in gve_xdp_xmit()
808 int gve_xdp_xmit_one(struct gve_priv *priv, struct gve_tx_ring *tx, in gve_xdp_xmit_one() argument
813 if (!gve_can_tx(tx, len + GVE_GQ_TX_MIN_PKT_DESC_BYTES - 1)) in gve_xdp_xmit_one()
814 return -EBUSY; in gve_xdp_xmit_one()
816 nsegs = gve_tx_fill_xdp(priv, tx, data, len, frame_p, false); in gve_xdp_xmit_one()
817 tx->req += nsegs; in gve_xdp_xmit_one()
824 static int gve_clean_tx_done(struct gve_priv *priv, struct gve_tx_ring *tx, in gve_clean_tx_done() argument
835 idx = tx->done & tx->mask; in gve_clean_tx_done()
836 netif_info(priv, tx_done, priv->dev, in gve_clean_tx_done()
838 tx->q_num, __func__, idx, tx->req, tx->done); in gve_clean_tx_done()
839 info = &tx->info[idx]; in gve_clean_tx_done()
840 skb = info->skb; in gve_clean_tx_done()
843 if (tx->raw_addressing) in gve_clean_tx_done()
844 gve_tx_unmap_buf(tx->dev, info); in gve_clean_tx_done()
845 tx->done++; in gve_clean_tx_done()
848 info->skb = NULL; in gve_clean_tx_done()
849 bytes += skb->len; in gve_clean_tx_done()
852 if (tx->raw_addressing) in gve_clean_tx_done()
858 if (!tx->raw_addressing) in gve_clean_tx_done()
859 gve_tx_free_fifo(&tx->tx_fifo, space_freed); in gve_clean_tx_done()
860 u64_stats_update_begin(&tx->statss); in gve_clean_tx_done()
861 tx->bytes_done += bytes; in gve_clean_tx_done()
862 tx->pkt_done += pkts; in gve_clean_tx_done()
863 u64_stats_update_end(&tx->statss); in gve_clean_tx_done()
864 netdev_tx_completed_queue(tx->netdev_txq, pkts, bytes); in gve_clean_tx_done()
866 /* start the queue if we've stopped it */ in gve_clean_tx_done()
871 if (try_to_wake && netif_tx_queue_stopped(tx->netdev_txq) && in gve_clean_tx_done()
872 likely(gve_can_tx(tx, GVE_TX_START_THRESH))) { in gve_clean_tx_done()
873 tx->wake_queue++; in gve_clean_tx_done()
874 netif_tx_wake_queue(tx->netdev_txq); in gve_clean_tx_done()
881 struct gve_tx_ring *tx) in gve_tx_load_event_counter() argument
883 u32 counter_index = be32_to_cpu(tx->q_resources->counter_index); in gve_tx_load_event_counter()
884 __be32 counter = READ_ONCE(priv->counter_array[counter_index]); in gve_tx_load_event_counter()
889 static int gve_xsk_tx(struct gve_priv *priv, struct gve_tx_ring *tx, in gve_xsk_tx() argument
896 spin_lock(&tx->xdp_lock); in gve_xsk_tx()
898 if (!gve_can_tx(tx, GVE_TX_START_THRESH)) in gve_xsk_tx()
901 if (!xsk_tx_peek_desc(tx->xsk_pool, &desc)) { in gve_xsk_tx()
902 tx->xdp_xsk_done = tx->xdp_xsk_wakeup; in gve_xsk_tx()
906 data = xsk_buff_raw_get_data(tx->xsk_pool, desc.addr); in gve_xsk_tx()
907 nsegs = gve_tx_fill_xdp(priv, tx, data, desc.len, NULL, true); in gve_xsk_tx()
908 tx->req += nsegs; in gve_xsk_tx()
913 gve_tx_put_doorbell(priv, tx->q_resources, tx->req); in gve_xsk_tx()
914 xsk_tx_release(tx->xsk_pool); in gve_xsk_tx()
916 spin_unlock(&tx->xdp_lock); in gve_xsk_tx()
922 struct gve_priv *priv = block->priv; in gve_xdp_poll()
923 struct gve_tx_ring *tx = block->tx; in gve_xdp_poll() local
929 nic_done = gve_tx_load_event_counter(priv, tx); in gve_xdp_poll()
930 to_do = min_t(u32, (nic_done - tx->done), budget); in gve_xdp_poll()
931 gve_clean_xdp_done(priv, tx, to_do); in gve_xdp_poll()
932 repoll = nic_done != tx->done; in gve_xdp_poll()
934 if (tx->xsk_pool) { in gve_xdp_poll()
935 int sent = gve_xsk_tx(priv, tx, budget); in gve_xdp_poll()
937 u64_stats_update_begin(&tx->statss); in gve_xdp_poll()
938 tx->xdp_xsk_sent += sent; in gve_xdp_poll()
939 u64_stats_update_end(&tx->statss); in gve_xdp_poll()
941 if (xsk_uses_need_wakeup(tx->xsk_pool)) in gve_xdp_poll()
942 xsk_set_tx_need_wakeup(tx->xsk_pool); in gve_xdp_poll()
951 struct gve_priv *priv = block->priv; in gve_tx_poll()
952 struct gve_tx_ring *tx = block->tx; in gve_tx_poll() local
960 /* In TX path, it may try to clean completed pkts in order to xmit, in gve_tx_poll()
964 spin_lock(&tx->clean_lock); in gve_tx_poll()
966 nic_done = gve_tx_load_event_counter(priv, tx); in gve_tx_poll()
967 to_do = min_t(u32, (nic_done - tx->done), budget); in gve_tx_poll()
968 gve_clean_tx_done(priv, tx, to_do, true); in gve_tx_poll()
969 spin_unlock(&tx->clean_lock); in gve_tx_poll()
971 return nic_done != tx->done; in gve_tx_poll()
974 bool gve_tx_clean_pending(struct gve_priv *priv, struct gve_tx_ring *tx) in gve_tx_clean_pending() argument
976 u32 nic_done = gve_tx_load_event_counter(priv, tx); in gve_tx_clean_pending()
978 return nic_done != tx->done; in gve_tx_clean_pending()