Lines Matching full:nv
525 static void fbnic_clean_twq0(struct fbnic_napi_vector *nv, int napi_budget, in fbnic_clean_twq0() argument
561 fbnic_unmap_single_twd(nv->dev, &ring->desc[head]); in fbnic_clean_twq0()
567 fbnic_unmap_page_twd(nv->dev, &ring->desc[head]); in fbnic_clean_twq0()
583 txq = txring_txq(nv->napi.dev, ring); in fbnic_clean_twq0()
609 static void fbnic_clean_tsq(struct fbnic_napi_vector *nv, in fbnic_clean_tsq() argument
626 netdev_err(nv->napi.dev, in fbnic_clean_tsq()
638 fbn = netdev_priv(nv->napi.dev); in fbnic_clean_tsq()
680 struct fbnic_napi_vector *nv, int budget) in fbnic_page_pool_drain() argument
686 page_pool_put_unrefed_page(nv->page_pool, page, -1, !!budget); in fbnic_page_pool_drain()
691 static void fbnic_clean_twq(struct fbnic_napi_vector *nv, int napi_budget, in fbnic_clean_twq() argument
695 fbnic_clean_twq0(nv, napi_budget, &qt->sub0, false, head0); in fbnic_clean_twq()
697 fbnic_clean_twq0(nv, napi_budget, &qt->sub0, false, ts_head); in fbnic_clean_twq()
701 fbnic_clean_tcq(struct fbnic_napi_vector *nv, struct fbnic_q_triad *qt, in fbnic_clean_tcq() argument
734 fbnic_clean_tsq(nv, &qt->sub0, tcd, &ts_head, &head0); in fbnic_clean_tcq()
755 fbnic_clean_twq(nv, napi_budget, qt, ts_head, head0); in fbnic_clean_tcq()
758 static void fbnic_clean_bdq(struct fbnic_napi_vector *nv, int napi_budget, in fbnic_clean_bdq() argument
767 fbnic_page_pool_drain(ring, head, nv, napi_budget); in fbnic_clean_bdq()
797 static void fbnic_fill_bdq(struct fbnic_napi_vector *nv, struct fbnic_ring *bdq) in fbnic_fill_bdq() argument
808 page = page_pool_dev_alloc_pages(nv->page_pool); in fbnic_fill_bdq()
859 static void fbnic_pkt_prepare(struct fbnic_napi_vector *nv, u64 rcd, in fbnic_pkt_prepare() argument
885 dma_sync_single_range_for_cpu(nv->dev, page_pool_get_dma_addr(page), in fbnic_pkt_prepare()
900 static void fbnic_add_rx_frag(struct fbnic_napi_vector *nv, u64 rcd, in fbnic_add_rx_frag() argument
918 dma_sync_single_range_for_cpu(nv->dev, page_pool_get_dma_addr(page), in fbnic_add_rx_frag()
933 static void fbnic_put_pkt_buff(struct fbnic_napi_vector *nv, in fbnic_put_pkt_buff() argument
948 page_pool_put_full_page(nv->page_pool, page, !!budget); in fbnic_put_pkt_buff()
952 page_pool_put_full_page(nv->page_pool, page, !!budget); in fbnic_put_pkt_buff()
955 static struct sk_buff *fbnic_build_skb(struct fbnic_napi_vector *nv, in fbnic_build_skb() argument
993 skb->protocol = eth_type_trans(skb, nv->napi.dev); in fbnic_build_skb()
1009 static void fbnic_rx_tstamp(struct fbnic_napi_vector *nv, u64 rcd, in fbnic_rx_tstamp() argument
1018 fbn = netdev_priv(nv->napi.dev); in fbnic_rx_tstamp()
1026 static void fbnic_populate_skb_fields(struct fbnic_napi_vector *nv, in fbnic_populate_skb_fields() argument
1031 struct net_device *netdev = nv->napi.dev; in fbnic_populate_skb_fields()
1049 static int fbnic_clean_rcq(struct fbnic_napi_vector *nv, in fbnic_clean_rcq() argument
1079 fbnic_pkt_prepare(nv, rcd, pkt, qt); in fbnic_clean_rcq()
1084 fbnic_add_rx_frag(nv, rcd, pkt, qt); in fbnic_clean_rcq()
1092 fbnic_rx_tstamp(nv, rcd, pkt); in fbnic_clean_rcq()
1098 skb = fbnic_build_skb(nv, pkt); in fbnic_clean_rcq()
1102 fbnic_populate_skb_fields(nv, rcd, skb, qt, in fbnic_clean_rcq()
1109 napi_gro_receive(&nv->napi, skb); in fbnic_clean_rcq()
1118 fbnic_put_pkt_buff(nv, pkt, 1); in fbnic_clean_rcq()
1147 fbnic_clean_bdq(nv, budget, &qt->sub0, head0); in fbnic_clean_rcq()
1148 fbnic_fill_bdq(nv, &qt->sub0); in fbnic_clean_rcq()
1151 fbnic_clean_bdq(nv, budget, &qt->sub1, head1); in fbnic_clean_rcq()
1152 fbnic_fill_bdq(nv, &qt->sub1); in fbnic_clean_rcq()
1163 static void fbnic_nv_irq_disable(struct fbnic_napi_vector *nv) in fbnic_nv_irq_disable() argument
1165 struct fbnic_dev *fbd = nv->fbd; in fbnic_nv_irq_disable()
1166 u32 v_idx = nv->v_idx; in fbnic_nv_irq_disable()
1171 static void fbnic_nv_irq_rearm(struct fbnic_napi_vector *nv) in fbnic_nv_irq_rearm() argument
1173 struct fbnic_dev *fbd = nv->fbd; in fbnic_nv_irq_rearm()
1174 u32 v_idx = nv->v_idx; in fbnic_nv_irq_rearm()
1182 struct fbnic_napi_vector *nv = container_of(napi, in fbnic_poll() local
1187 for (i = 0; i < nv->txt_count; i++) in fbnic_poll()
1188 fbnic_clean_tcq(nv, &nv->qt[i], budget); in fbnic_poll()
1190 for (j = 0; j < nv->rxt_count; j++, i++) in fbnic_poll()
1191 work_done += fbnic_clean_rcq(nv, &nv->qt[i], budget); in fbnic_poll()
1197 fbnic_nv_irq_rearm(nv); in fbnic_poll()
1204 struct fbnic_napi_vector *nv = *(void **)data; in fbnic_msix_clean_rings() local
1206 napi_schedule_irqoff(&nv->napi); in fbnic_msix_clean_rings()
1273 struct fbnic_napi_vector *nv) in fbnic_free_napi_vector() argument
1275 struct fbnic_dev *fbd = nv->fbd; in fbnic_free_napi_vector()
1278 for (i = 0; i < nv->txt_count; i++) { in fbnic_free_napi_vector()
1279 fbnic_remove_tx_ring(fbn, &nv->qt[i].sub0); in fbnic_free_napi_vector()
1280 fbnic_remove_tx_ring(fbn, &nv->qt[i].cmpl); in fbnic_free_napi_vector()
1283 for (j = 0; j < nv->rxt_count; j++, i++) { in fbnic_free_napi_vector()
1284 fbnic_remove_rx_ring(fbn, &nv->qt[i].sub0); in fbnic_free_napi_vector()
1285 fbnic_remove_rx_ring(fbn, &nv->qt[i].sub1); in fbnic_free_napi_vector()
1286 fbnic_remove_rx_ring(fbn, &nv->qt[i].cmpl); in fbnic_free_napi_vector()
1289 fbnic_napi_free_irq(fbd, nv); in fbnic_free_napi_vector()
1290 page_pool_destroy(nv->page_pool); in fbnic_free_napi_vector()
1291 netif_napi_del(&nv->napi); in fbnic_free_napi_vector()
1292 fbn->napi[fbnic_napi_idx(nv)] = NULL; in fbnic_free_napi_vector()
1293 kfree(nv); in fbnic_free_napi_vector()
1309 struct fbnic_napi_vector *nv) in fbnic_alloc_nv_page_pool() argument
1314 .pool_size = (fbn->hpq_size + fbn->ppq_size) * nv->rxt_count, in fbnic_alloc_nv_page_pool()
1316 .dev = nv->dev, in fbnic_alloc_nv_page_pool()
1320 .napi = &nv->napi, in fbnic_alloc_nv_page_pool()
1341 nv->page_pool = pp; in fbnic_alloc_nv_page_pool()
1362 struct fbnic_napi_vector *nv; in fbnic_alloc_napi_vector() local
1376 nv = kzalloc(struct_size(nv, qt, qt_count), GFP_KERNEL); in fbnic_alloc_napi_vector()
1377 if (!nv) in fbnic_alloc_napi_vector()
1381 nv->txt_count = txt_count; in fbnic_alloc_napi_vector()
1382 nv->rxt_count = rxt_count; in fbnic_alloc_napi_vector()
1385 nv->fbd = fbd; in fbnic_alloc_napi_vector()
1386 nv->v_idx = v_idx; in fbnic_alloc_napi_vector()
1389 fbn->napi[fbnic_napi_idx(nv)] = nv; in fbnic_alloc_napi_vector()
1390 netif_napi_add(fbn->netdev, &nv->napi, fbnic_poll); in fbnic_alloc_napi_vector()
1393 netif_napi_set_irq(&nv->napi, in fbnic_alloc_napi_vector()
1394 pci_irq_vector(to_pci_dev(fbd->dev), nv->v_idx)); in fbnic_alloc_napi_vector()
1396 /* Tie nv back to PCIe dev */ in fbnic_alloc_napi_vector()
1397 nv->dev = fbd->dev; in fbnic_alloc_napi_vector()
1401 err = fbnic_alloc_nv_page_pool(fbn, nv); in fbnic_alloc_napi_vector()
1407 err = fbnic_napi_request_irq(fbd, nv); in fbnic_alloc_napi_vector()
1412 qt = nv->qt; in fbnic_alloc_napi_vector()
1467 page_pool_destroy(nv->page_pool); in fbnic_alloc_napi_vector()
1469 netif_napi_del(&nv->napi); in fbnic_alloc_napi_vector()
1470 fbn->napi[fbnic_napi_idx(nv)] = NULL; in fbnic_alloc_napi_vector()
1471 kfree(nv); in fbnic_alloc_napi_vector()
1734 struct fbnic_napi_vector *nv) in fbnic_free_nv_resources() argument
1739 for (i = 0; i < nv->txt_count; i++) in fbnic_free_nv_resources()
1740 fbnic_free_qt_resources(fbn, &nv->qt[i]); in fbnic_free_nv_resources()
1742 for (j = 0; j < nv->rxt_count; j++, i++) in fbnic_free_nv_resources()
1743 fbnic_free_qt_resources(fbn, &nv->qt[i]); in fbnic_free_nv_resources()
1747 struct fbnic_napi_vector *nv) in fbnic_alloc_nv_resources() argument
1752 for (i = 0; i < nv->txt_count; i++) { in fbnic_alloc_nv_resources()
1753 err = fbnic_alloc_tx_qt_resources(fbn, &nv->qt[i]); in fbnic_alloc_nv_resources()
1759 for (j = 0; j < nv->rxt_count; j++, i++) { in fbnic_alloc_nv_resources()
1760 err = fbnic_alloc_rx_qt_resources(fbn, &nv->qt[i]); in fbnic_alloc_nv_resources()
1769 fbnic_free_qt_resources(fbn, &nv->qt[i]); in fbnic_alloc_nv_resources()
1800 static void fbnic_set_netif_napi(struct fbnic_napi_vector *nv) in fbnic_set_netif_napi() argument
1805 for (i = 0; i < nv->txt_count; i++) { in fbnic_set_netif_napi()
1806 struct fbnic_q_triad *qt = &nv->qt[i]; in fbnic_set_netif_napi()
1808 netif_queue_set_napi(nv->napi.dev, qt->sub0.q_idx, in fbnic_set_netif_napi()
1809 NETDEV_QUEUE_TYPE_TX, &nv->napi); in fbnic_set_netif_napi()
1813 for (j = 0; j < nv->rxt_count; j++, i++) { in fbnic_set_netif_napi()
1814 struct fbnic_q_triad *qt = &nv->qt[i]; in fbnic_set_netif_napi()
1816 netif_queue_set_napi(nv->napi.dev, qt->cmpl.q_idx, in fbnic_set_netif_napi()
1817 NETDEV_QUEUE_TYPE_RX, &nv->napi); in fbnic_set_netif_napi()
1821 static void fbnic_reset_netif_napi(struct fbnic_napi_vector *nv) in fbnic_reset_netif_napi() argument
1826 for (i = 0; i < nv->txt_count; i++) { in fbnic_reset_netif_napi()
1827 struct fbnic_q_triad *qt = &nv->qt[i]; in fbnic_reset_netif_napi()
1829 netif_queue_set_napi(nv->napi.dev, qt->sub0.q_idx, in fbnic_reset_netif_napi()
1834 for (j = 0; j < nv->rxt_count; j++, i++) { in fbnic_reset_netif_napi()
1835 struct fbnic_q_triad *qt = &nv->qt[i]; in fbnic_reset_netif_napi()
1837 netif_queue_set_napi(nv->napi.dev, qt->cmpl.q_idx, in fbnic_reset_netif_napi()
1912 struct fbnic_napi_vector *nv = fbn->napi[i]; in fbnic_disable() local
1915 for (t = 0; t < nv->txt_count; t++) { in fbnic_disable()
1916 struct fbnic_q_triad *qt = &nv->qt[t]; in fbnic_disable()
1923 for (j = 0; j < nv->rxt_count; j++, t++) { in fbnic_disable()
1924 struct fbnic_q_triad *qt = &nv->qt[t]; in fbnic_disable()
2023 struct fbnic_napi_vector *nv = fbn->napi[i]; in fbnic_flush() local
2027 for (t = 0; t < nv->txt_count; t++) { in fbnic_flush()
2028 struct fbnic_q_triad *qt = &nv->qt[t]; in fbnic_flush()
2032 fbnic_clean_twq0(nv, 0, &qt->sub0, true, qt->sub0.tail); in fbnic_flush()
2042 tx_queue = netdev_get_tx_queue(nv->napi.dev, in fbnic_flush()
2048 for (j = 0; j < nv->rxt_count; j++, t++) { in fbnic_flush()
2049 struct fbnic_q_triad *qt = &nv->qt[t]; in fbnic_flush()
2052 fbnic_clean_bdq(nv, 0, &qt->sub0, qt->sub0.tail); in fbnic_flush()
2053 fbnic_clean_bdq(nv, 0, &qt->sub1, qt->sub1.tail); in fbnic_flush()
2058 fbnic_put_pkt_buff(nv, qt->cmpl.pkt, 0); in fbnic_flush()
2069 struct fbnic_napi_vector *nv = fbn->napi[i]; in fbnic_fill() local
2075 for (j = 0, t = nv->txt_count; j < nv->rxt_count; j++, t++) { in fbnic_fill()
2076 struct fbnic_q_triad *qt = &nv->qt[t]; in fbnic_fill()
2079 fbnic_fill_bdq(nv, &qt->sub0); in fbnic_fill()
2080 fbnic_fill_bdq(nv, &qt->sub1); in fbnic_fill()
2107 static void fbnic_enable_tcq(struct fbnic_napi_vector *nv, in fbnic_enable_tcq() argument
2128 fbnic_ring_wr32(tcq, FBNIC_QUEUE_TIM_CTL, nv->v_idx); in fbnic_enable_tcq()
2174 static void fbnic_config_drop_mode_rcq(struct fbnic_napi_vector *nv, in fbnic_config_drop_mode_rcq() argument
2202 void fbnic_config_txrx_usecs(struct fbnic_napi_vector *nv, u32 arm) in fbnic_config_txrx_usecs() argument
2204 struct fbnic_net *fbn = netdev_priv(nv->napi.dev); in fbnic_config_txrx_usecs()
2205 struct fbnic_dev *fbd = nv->fbd; in fbnic_config_txrx_usecs()
2213 fbnic_wr32(fbd, FBNIC_INTR_CQ_REARM(nv->v_idx), val); in fbnic_config_txrx_usecs()
2216 void fbnic_config_rx_frames(struct fbnic_napi_vector *nv) in fbnic_config_rx_frames() argument
2218 struct fbnic_net *fbn = netdev_priv(nv->napi.dev); in fbnic_config_rx_frames()
2221 for (i = nv->txt_count; i < nv->rxt_count + nv->txt_count; i++) { in fbnic_config_rx_frames()
2222 struct fbnic_q_triad *qt = &nv->qt[i]; in fbnic_config_rx_frames()
2224 fbnic_config_rim_threshold(&qt->cmpl, nv->v_idx, in fbnic_config_rx_frames()
2230 static void fbnic_enable_rcq(struct fbnic_napi_vector *nv, in fbnic_enable_rcq() argument
2233 struct fbnic_net *fbn = netdev_priv(nv->napi.dev); in fbnic_enable_rcq()
2237 fbnic_config_drop_mode_rcq(nv, rcq); in fbnic_enable_rcq()
2261 fbnic_config_rim_threshold(rcq, nv->v_idx, fbn->rx_max_frames * in fbnic_enable_rcq()
2275 struct fbnic_napi_vector *nv = fbn->napi[i]; in fbnic_enable() local
2279 for (t = 0; t < nv->txt_count; t++) { in fbnic_enable()
2280 struct fbnic_q_triad *qt = &nv->qt[t]; in fbnic_enable()
2283 fbnic_enable_tcq(nv, &qt->cmpl); in fbnic_enable()
2287 for (j = 0; j < nv->rxt_count; j++, t++) { in fbnic_enable()
2288 struct fbnic_q_triad *qt = &nv->qt[t]; in fbnic_enable()
2291 fbnic_config_drop_mode_rcq(nv, &qt->cmpl); in fbnic_enable()
2292 fbnic_enable_rcq(nv, &qt->cmpl); in fbnic_enable()
2299 static void fbnic_nv_irq_enable(struct fbnic_napi_vector *nv) in fbnic_nv_irq_enable() argument
2301 fbnic_config_txrx_usecs(nv, FBNIC_INTR_CQ_REARM_INTR_UNMASK); in fbnic_nv_irq_enable()
2311 struct fbnic_napi_vector *nv = fbn->napi[i]; in fbnic_napi_enable() local
2313 napi_enable(&nv->napi); in fbnic_napi_enable()
2315 fbnic_nv_irq_enable(nv); in fbnic_napi_enable()
2320 irqs[nv->v_idx / 32] |= BIT(nv->v_idx % 32); in fbnic_napi_enable()
2344 struct fbnic_napi_vector *nv = fbn->napi[i]; in fbnic_napi_depletion_check() local
2347 for (t = nv->txt_count, j = 0; j < nv->rxt_count; j++, t++) { in fbnic_napi_depletion_check()
2351 if (fbnic_desc_used(&nv->qt[t].sub0) < 4 || in fbnic_napi_depletion_check()
2352 fbnic_desc_used(&nv->qt[t].sub1) < 4) in fbnic_napi_depletion_check()
2353 irqs[nv->v_idx / 32] |= BIT(nv->v_idx % 32); in fbnic_napi_depletion_check()