Lines Matching +full:tx +full:- +full:rings +full:- +full:empty
1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2013 - 2019 Intel Corporation. */
18 "Copyright(c) 2013 - 2019 Intel Corporation.";
28 * fm10k_init_module - Driver Registration Routine
44 return -ENOMEM; in fm10k_init_module()
59 * fm10k_exit_module - Driver Exit Cleanup Routine
78 struct page *page = bi->page; in fm10k_alloc_mapped_page()
88 rx_ring->rx_stats.alloc_failed++; in fm10k_alloc_mapped_page()
93 dma = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE); in fm10k_alloc_mapped_page()
98 if (dma_mapping_error(rx_ring->dev, dma)) { in fm10k_alloc_mapped_page()
101 rx_ring->rx_stats.alloc_failed++; in fm10k_alloc_mapped_page()
105 bi->dma = dma; in fm10k_alloc_mapped_page()
106 bi->page = page; in fm10k_alloc_mapped_page()
107 bi->page_offset = 0; in fm10k_alloc_mapped_page()
113 * fm10k_alloc_rx_buffers - Replace used receive buffers
121 u16 i = rx_ring->next_to_use; in fm10k_alloc_rx_buffers()
128 bi = &rx_ring->rx_buffer[i]; in fm10k_alloc_rx_buffers()
129 i -= rx_ring->count; in fm10k_alloc_rx_buffers()
136 * because each write-back erases this info. in fm10k_alloc_rx_buffers()
138 rx_desc->q.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); in fm10k_alloc_rx_buffers()
145 bi = rx_ring->rx_buffer; in fm10k_alloc_rx_buffers()
146 i -= rx_ring->count; in fm10k_alloc_rx_buffers()
150 rx_desc->d.staterr = 0; in fm10k_alloc_rx_buffers()
152 cleaned_count--; in fm10k_alloc_rx_buffers()
155 i += rx_ring->count; in fm10k_alloc_rx_buffers()
157 if (rx_ring->next_to_use != i) { in fm10k_alloc_rx_buffers()
159 rx_ring->next_to_use = i; in fm10k_alloc_rx_buffers()
162 rx_ring->next_to_alloc = i; in fm10k_alloc_rx_buffers()
166 * applicable for weak-ordered memory model archs, in fm10k_alloc_rx_buffers()
167 * such as IA-64). in fm10k_alloc_rx_buffers()
172 writel(i, rx_ring->tail); in fm10k_alloc_rx_buffers()
177 * fm10k_reuse_rx_page - page flip buffer and store it back on the ring
187 u16 nta = rx_ring->next_to_alloc; in fm10k_reuse_rx_page()
189 new_buff = &rx_ring->rx_buffer[nta]; in fm10k_reuse_rx_page()
193 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; in fm10k_reuse_rx_page()
199 dma_sync_single_range_for_device(rx_ring->dev, old_buff->dma, in fm10k_reuse_rx_page()
200 old_buff->page_offset, in fm10k_reuse_rx_page()
209 /* avoid re-using remote and pfmemalloc pages */ in fm10k_can_reuse_rx_page()
219 rx_buffer->page_offset ^= FM10K_RX_BUFSZ; in fm10k_can_reuse_rx_page()
222 rx_buffer->page_offset += truesize; in fm10k_can_reuse_rx_page()
224 if (rx_buffer->page_offset > (PAGE_SIZE - FM10K_RX_BUFSZ)) in fm10k_can_reuse_rx_page()
237 * fm10k_add_rx_frag - Add contents of Rx buffer to sk_buff
243 * This function will add the data contained in rx_buffer->page to the skb.
256 struct page *page = rx_buffer->page; in fm10k_add_rx_frag()
257 unsigned char *va = page_address(page) + rx_buffer->page_offset; in fm10k_add_rx_frag()
271 /* page is reusable, we can reuse buffer as-is */ in fm10k_add_rx_frag()
281 * 60 bytes if the skb->len is less than 60 for skb_pad. in fm10k_add_rx_frag()
283 pull_len = eth_get_headlen(skb->dev, va, FM10K_RX_HDR_LEN); in fm10k_add_rx_frag()
290 size -= pull_len; in fm10k_add_rx_frag()
293 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, in fm10k_add_rx_frag()
303 unsigned int size = le16_to_cpu(rx_desc->w.length); in fm10k_fetch_rx_buffer()
307 rx_buffer = &rx_ring->rx_buffer[rx_ring->next_to_clean]; in fm10k_fetch_rx_buffer()
308 page = rx_buffer->page; in fm10k_fetch_rx_buffer()
313 rx_buffer->page_offset; in fm10k_fetch_rx_buffer()
319 skb = napi_alloc_skb(&rx_ring->q_vector->napi, in fm10k_fetch_rx_buffer()
322 rx_ring->rx_stats.alloc_failed++; in fm10k_fetch_rx_buffer()
326 /* we will be copying header into skb->data in in fm10k_fetch_rx_buffer()
330 prefetchw(skb->data); in fm10k_fetch_rx_buffer()
334 dma_sync_single_range_for_cpu(rx_ring->dev, in fm10k_fetch_rx_buffer()
335 rx_buffer->dma, in fm10k_fetch_rx_buffer()
336 rx_buffer->page_offset, in fm10k_fetch_rx_buffer()
346 dma_unmap_page(rx_ring->dev, rx_buffer->dma, in fm10k_fetch_rx_buffer()
351 rx_buffer->page = NULL; in fm10k_fetch_rx_buffer()
363 if (!(ring->netdev->features & NETIF_F_RXCSUM)) in fm10k_rx_checksum()
372 ring->rx_stats.csum_err++; in fm10k_rx_checksum()
378 skb->encapsulation = true; in fm10k_rx_checksum()
382 skb->ip_summed = CHECKSUM_UNNECESSARY; in fm10k_rx_checksum()
384 ring->rx_stats.csum_good++; in fm10k_rx_checksum()
399 if (!(ring->netdev->features & NETIF_F_RXHASH)) in fm10k_rx_hash()
402 rss_type = le16_to_cpu(rx_desc->w.pkt_info) & FM10K_RXD_RSSTYPE_MASK; in fm10k_rx_hash()
406 skb_set_hash(skb, le32_to_cpu(rx_desc->d.rss), in fm10k_rx_hash()
415 struct net_device *dev = rx_ring->netdev; in fm10k_type_trans()
416 struct fm10k_l2_accel *l2_accel = rcu_dereference_bh(rx_ring->l2_accel); in fm10k_type_trans()
420 u16 idx = le16_to_cpu(FM10K_CB(skb)->fi.w.dglort) - 1; in fm10k_type_trans()
422 idx -= l2_accel->dglort; in fm10k_type_trans()
423 if (idx < l2_accel->size && l2_accel->macvlan[idx]) in fm10k_type_trans()
424 dev = l2_accel->macvlan[idx]; in fm10k_type_trans()
431 skb_record_rx_queue(skb, rx_ring->queue_index); in fm10k_type_trans()
433 macvlan_count_rx(netdev_priv(dev), skb->len + ETH_HLEN, true, in fm10k_type_trans()
436 skb->protocol = eth_type_trans(skb, dev); in fm10k_type_trans()
440 * fm10k_process_skb_fields - Populate skb header fields from Rx descriptor
453 unsigned int len = skb->len; in fm10k_process_skb_fields()
459 FM10K_CB(skb)->tstamp = rx_desc->q.timestamp; in fm10k_process_skb_fields()
461 FM10K_CB(skb)->fi.w.vlan = rx_desc->w.vlan; in fm10k_process_skb_fields()
463 FM10K_CB(skb)->fi.d.glort = rx_desc->d.glort; in fm10k_process_skb_fields()
465 if (rx_desc->w.vlan) { in fm10k_process_skb_fields()
466 u16 vid = le16_to_cpu(rx_desc->w.vlan); in fm10k_process_skb_fields()
468 if ((vid & VLAN_VID_MASK) != rx_ring->vid) in fm10k_process_skb_fields()
481 * fm10k_is_non_eop - process handling of non-EOP buffers
488 * that this is in fact a non-EOP buffer.
493 u32 ntc = rx_ring->next_to_clean + 1; in fm10k_is_non_eop()
496 ntc = (ntc < rx_ring->count) ? ntc : 0; in fm10k_is_non_eop()
497 rx_ring->next_to_clean = ntc; in fm10k_is_non_eop()
508 * fm10k_cleanup_headers - Correct corrupted or empty headers
528 ((rxd)->w.csum_err & cpu_to_le16(bit)) in fm10k_cleanup_headers()
530 rx_ring->rx_stats.switch_errors++; in fm10k_cleanup_headers()
532 rx_ring->rx_stats.drops++; in fm10k_cleanup_headers()
534 rx_ring->rx_stats.pp_errors++; in fm10k_cleanup_headers()
536 rx_ring->rx_stats.link_errors++; in fm10k_cleanup_headers()
538 rx_ring->rx_stats.length_errors++; in fm10k_cleanup_headers()
540 rx_ring->rx_stats.errors++; in fm10k_cleanup_headers()
552 * fm10k_receive_skb - helper function to handle rx indications
559 napi_gro_receive(&q_vector->napi, skb); in fm10k_receive_skb()
566 struct sk_buff *skb = rx_ring->skb; in fm10k_clean_rx_irq()
579 rx_desc = FM10K_RX_DESC(rx_ring, rx_ring->next_to_clean); in fm10k_clean_rx_irq()
581 if (!rx_desc->d.staterr) in fm10k_clean_rx_irq()
599 /* fetch next buffer in frame if non-eop */ in fm10k_clean_rx_irq()
622 rx_ring->skb = skb; in fm10k_clean_rx_irq()
624 u64_stats_update_begin(&rx_ring->syncp); in fm10k_clean_rx_irq()
625 rx_ring->stats.packets += total_packets; in fm10k_clean_rx_irq()
626 rx_ring->stats.bytes += total_bytes; in fm10k_clean_rx_irq()
627 u64_stats_update_end(&rx_ring->syncp); in fm10k_clean_rx_irq()
628 q_vector->rx.total_packets += total_packets; in fm10k_clean_rx_irq()
629 q_vector->rx.total_bytes += total_bytes; in fm10k_clean_rx_irq()
637 struct fm10k_intfc *interface = netdev_priv(skb->dev); in fm10k_port_is_vxlan()
639 if (interface->vxlan_port != udp_hdr(skb)->dest) in fm10k_port_is_vxlan()
667 if (nvgre_hdr->flags & FM10K_NVGRE_RESERVED0_FLAGS) in fm10k_gre_is_nvgre()
671 if (nvgre_hdr->flags & NVGRE_TNI) in fm10k_gre_is_nvgre()
674 return (struct ethhdr *)(&nvgre_hdr->tni); in fm10k_gre_is_nvgre()
682 if (skb->inner_protocol_type != ENCAP_TYPE_ETHER || in fm10k_tx_encap_offload()
683 skb->inner_protocol != htons(ETH_P_TEB)) in fm10k_tx_encap_offload()
688 l4_hdr = ip_hdr(skb)->protocol; in fm10k_tx_encap_offload()
691 l4_hdr = ipv6_hdr(skb)->nexthdr; in fm10k_tx_encap_offload()
711 switch (eth_hdr->h_proto) { in fm10k_tx_encap_offload()
713 inner_l4_hdr = inner_ip_hdr(skb)->protocol; in fm10k_tx_encap_offload()
716 inner_l4_hdr = inner_ipv6_hdr(skb)->nexthdr; in fm10k_tx_encap_offload()
736 if (skb_inner_transport_header(skb) + inner_l4_hlen - in fm10k_tx_encap_offload()
740 return eth_hdr->h_proto; in fm10k_tx_encap_offload()
746 struct sk_buff *skb = first->skb; in fm10k_tso()
751 if (skb->ip_summed != CHECKSUM_PARTIAL) in fm10k_tso()
758 if (skb->encapsulation) { in fm10k_tso()
767 hdrlen = (th - skb->data) + (((struct tcphdr *)th)->doff << 2); in fm10k_tso()
769 first->tx_flags |= FM10K_TX_FLAGS_CSUM; in fm10k_tso()
772 first->gso_segs = skb_shinfo(skb)->gso_segs; in fm10k_tso()
773 first->bytecount += (first->gso_segs - 1) * hdrlen; in fm10k_tso()
775 /* populate Tx descriptor header size and mss */ in fm10k_tso()
776 tx_desc = FM10K_TX_DESC(tx_ring, tx_ring->next_to_use); in fm10k_tso()
777 tx_desc->hdrlen = hdrlen; in fm10k_tso()
778 tx_desc->mss = cpu_to_le16(skb_shinfo(skb)->gso_size); in fm10k_tso()
783 tx_ring->netdev->features &= ~NETIF_F_GSO_UDP_TUNNEL; in fm10k_tso()
785 netdev_err(tx_ring->netdev, in fm10k_tso()
787 return -1; in fm10k_tso()
793 struct sk_buff *skb = first->skb; in fm10k_tx_csum()
805 if (skb->ip_summed != CHECKSUM_PARTIAL) in fm10k_tx_csum()
808 if (skb->encapsulation) { in fm10k_tx_csum()
812 dev_warn(tx_ring->dev, in fm10k_tx_csum()
814 tx_ring->tx_stats.csum_err++; in fm10k_tx_csum()
828 l4_hdr = network_hdr.ipv4->protocol; in fm10k_tx_csum()
831 l4_hdr = network_hdr.ipv6->nexthdr; in fm10k_tx_csum()
832 if (likely((transport_hdr - network_hdr.raw) == in fm10k_tx_csum()
835 ipv6_skip_exthdr(skb, network_hdr.raw - skb->data + in fm10k_tx_csum()
850 if (skb->encapsulation) in fm10k_tx_csum()
855 dev_warn(tx_ring->dev, in fm10k_tx_csum()
860 tx_ring->tx_stats.csum_err++; in fm10k_tx_csum()
864 /* update TX checksum flag */ in fm10k_tx_csum()
865 first->tx_flags |= FM10K_TX_FLAGS_CSUM; in fm10k_tx_csum()
866 tx_ring->tx_stats.csum_good++; in fm10k_tx_csum()
869 /* populate Tx descriptor header size and mss */ in fm10k_tx_csum()
870 tx_desc = FM10K_TX_DESC(tx_ring, tx_ring->next_to_use); in fm10k_tx_csum()
871 tx_desc->hdrlen = 0; in fm10k_tx_csum()
872 tx_desc->mss = 0; in fm10k_tx_csum()
897 if ((++i & (FM10K_TXD_WB_FIFO_SIZE - 1)) == 0) in fm10k_tx_desc_push()
901 tx_desc->buffer_addr = cpu_to_le64(dma); in fm10k_tx_desc_push()
902 tx_desc->flags = desc_flags; in fm10k_tx_desc_push()
903 tx_desc->buflen = cpu_to_le16(size); in fm10k_tx_desc_push()
906 return i == tx_ring->count; in fm10k_tx_desc_push()
911 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); in __fm10k_maybe_stop_tx()
918 return -EBUSY; in __fm10k_maybe_stop_tx()
920 /* A reprieve! - use start_queue because it doesn't call schedule */ in __fm10k_maybe_stop_tx()
921 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index); in __fm10k_maybe_stop_tx()
922 ++tx_ring->tx_stats.restart_queue; in __fm10k_maybe_stop_tx()
936 struct sk_buff *skb = first->skb; in fm10k_tx_map()
943 u32 tx_flags = first->tx_flags; in fm10k_tx_map()
944 u16 i = tx_ring->next_to_use; in fm10k_tx_map()
951 tx_desc->vlan = cpu_to_le16(skb_vlan_tag_get(skb)); in fm10k_tx_map()
953 tx_desc->vlan = 0; in fm10k_tx_map()
956 data = skb->data; in fm10k_tx_map()
958 dma = dma_map_single(tx_ring->dev, data, size, DMA_TO_DEVICE); in fm10k_tx_map()
960 data_len = skb->data_len; in fm10k_tx_map()
963 for (frag = &skb_shinfo(skb)->frags[0];; frag++) { in fm10k_tx_map()
964 if (dma_mapping_error(tx_ring->dev, dma)) in fm10k_tx_map()
979 size -= FM10K_MAX_DATA_PER_TXD; in fm10k_tx_map()
992 data_len -= size; in fm10k_tx_map()
994 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, in fm10k_tx_map()
997 tx_buffer = &tx_ring->tx_buffer[i]; in fm10k_tx_map()
1007 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount); in fm10k_tx_map()
1010 skb_tx_timestamp(first->skb); in fm10k_tx_map()
1013 * are new descriptors to fetch. (Only applicable for weak-ordered in fm10k_tx_map()
1014 * memory model archs, such as IA-64). in fm10k_tx_map()
1022 first->next_to_watch = tx_desc; in fm10k_tx_map()
1024 tx_ring->next_to_use = i; in fm10k_tx_map()
1031 writel(i, tx_ring->tail); in fm10k_tx_map()
1036 dev_err(tx_ring->dev, "TX DMA map failed\n"); in fm10k_tx_map()
1040 tx_buffer = &tx_ring->tx_buffer[i]; in fm10k_tx_map()
1045 i = tx_ring->count; in fm10k_tx_map()
1046 i--; in fm10k_tx_map()
1049 tx_ring->next_to_use = i; in fm10k_tx_map()
1066 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) { in fm10k_xmit_frame_ring()
1067 skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; in fm10k_xmit_frame_ring()
1073 tx_ring->tx_stats.tx_busy++; in fm10k_xmit_frame_ring()
1078 first = &tx_ring->tx_buffer[tx_ring->next_to_use]; in fm10k_xmit_frame_ring()
1079 first->skb = skb; in fm10k_xmit_frame_ring()
1080 first->bytecount = max_t(unsigned int, skb->len, ETH_ZLEN); in fm10k_xmit_frame_ring()
1081 first->gso_segs = 1; in fm10k_xmit_frame_ring()
1084 first->tx_flags = tx_flags; in fm10k_xmit_frame_ring()
1097 dev_kfree_skb_any(first->skb); in fm10k_xmit_frame_ring()
1098 first->skb = NULL; in fm10k_xmit_frame_ring()
1105 return ring->stats.packets; in fm10k_get_tx_completed()
1109 * fm10k_get_tx_pending - how many Tx descriptors not processed
1115 struct fm10k_intfc *interface = ring->q_vector->interface; in fm10k_get_tx_pending()
1116 struct fm10k_hw *hw = &interface->hw; in fm10k_get_tx_pending()
1120 head = ring->next_to_clean; in fm10k_get_tx_pending()
1121 tail = ring->next_to_use; in fm10k_get_tx_pending()
1123 head = fm10k_read_reg(hw, FM10K_TDH(ring->reg_idx)); in fm10k_get_tx_pending()
1124 tail = fm10k_read_reg(hw, FM10K_TDT(ring->reg_idx)); in fm10k_get_tx_pending()
1127 return ((head <= tail) ? tail : tail + ring->count) - head; in fm10k_get_tx_pending()
1133 u32 tx_done_old = tx_ring->tx_stats.tx_done_old; in fm10k_check_tx_hang()
1148 tx_ring->tx_stats.tx_done_old = tx_done; in fm10k_check_tx_hang()
1150 clear_bit(__FM10K_HANG_CHECK_ARMED, tx_ring->state); in fm10k_check_tx_hang()
1156 return test_and_set_bit(__FM10K_HANG_CHECK_ARMED, tx_ring->state); in fm10k_check_tx_hang()
1160 * fm10k_tx_timeout_reset - initiate reset due to Tx timeout
1166 if (!test_bit(__FM10K_DOWN, interface->state)) { in fm10k_tx_timeout_reset()
1167 interface->tx_timeout_count++; in fm10k_tx_timeout_reset()
1168 set_bit(FM10K_FLAG_RESET_REQUESTED, interface->flags); in fm10k_tx_timeout_reset()
1174 * fm10k_clean_tx_irq - Reclaim resources after transmit completes
1176 * @tx_ring: tx ring to clean
1182 struct fm10k_intfc *interface = q_vector->interface; in fm10k_clean_tx_irq()
1186 unsigned int budget = q_vector->tx.work_limit; in fm10k_clean_tx_irq()
1187 unsigned int i = tx_ring->next_to_clean; in fm10k_clean_tx_irq()
1189 if (test_bit(__FM10K_DOWN, interface->state)) in fm10k_clean_tx_irq()
1192 tx_buffer = &tx_ring->tx_buffer[i]; in fm10k_clean_tx_irq()
1194 i -= tx_ring->count; in fm10k_clean_tx_irq()
1197 struct fm10k_tx_desc *eop_desc = tx_buffer->next_to_watch; in fm10k_clean_tx_irq()
1207 if (!(eop_desc->flags & FM10K_TXD_FLAG_DONE)) in fm10k_clean_tx_irq()
1211 tx_buffer->next_to_watch = NULL; in fm10k_clean_tx_irq()
1214 total_bytes += tx_buffer->bytecount; in fm10k_clean_tx_irq()
1215 total_packets += tx_buffer->gso_segs; in fm10k_clean_tx_irq()
1218 napi_consume_skb(tx_buffer->skb, napi_budget); in fm10k_clean_tx_irq()
1221 dma_unmap_single(tx_ring->dev, in fm10k_clean_tx_irq()
1227 tx_buffer->skb = NULL; in fm10k_clean_tx_irq()
1236 i -= tx_ring->count; in fm10k_clean_tx_irq()
1237 tx_buffer = tx_ring->tx_buffer; in fm10k_clean_tx_irq()
1243 dma_unmap_page(tx_ring->dev, in fm10k_clean_tx_irq()
1256 i -= tx_ring->count; in fm10k_clean_tx_irq()
1257 tx_buffer = tx_ring->tx_buffer; in fm10k_clean_tx_irq()
1261 /* issue prefetch for next Tx descriptor */ in fm10k_clean_tx_irq()
1265 budget--; in fm10k_clean_tx_irq()
1268 i += tx_ring->count; in fm10k_clean_tx_irq()
1269 tx_ring->next_to_clean = i; in fm10k_clean_tx_irq()
1270 u64_stats_update_begin(&tx_ring->syncp); in fm10k_clean_tx_irq()
1271 tx_ring->stats.bytes += total_bytes; in fm10k_clean_tx_irq()
1272 tx_ring->stats.packets += total_packets; in fm10k_clean_tx_irq()
1273 u64_stats_update_end(&tx_ring->syncp); in fm10k_clean_tx_irq()
1274 q_vector->tx.total_bytes += total_bytes; in fm10k_clean_tx_irq()
1275 q_vector->tx.total_packets += total_packets; in fm10k_clean_tx_irq()
1279 struct fm10k_hw *hw = &interface->hw; in fm10k_clean_tx_irq()
1281 netif_err(interface, drv, tx_ring->netdev, in fm10k_clean_tx_irq()
1282 "Detected Tx Unit Hang\n" in fm10k_clean_tx_irq()
1283 " Tx Queue <%d>\n" in fm10k_clean_tx_irq()
1287 tx_ring->queue_index, in fm10k_clean_tx_irq()
1288 fm10k_read_reg(hw, FM10K_TDH(tx_ring->reg_idx)), in fm10k_clean_tx_irq()
1289 fm10k_read_reg(hw, FM10K_TDT(tx_ring->reg_idx)), in fm10k_clean_tx_irq()
1290 tx_ring->next_to_use, i); in fm10k_clean_tx_irq()
1292 netif_stop_subqueue(tx_ring->netdev, in fm10k_clean_tx_irq()
1293 tx_ring->queue_index); in fm10k_clean_tx_irq()
1295 netif_info(interface, probe, tx_ring->netdev, in fm10k_clean_tx_irq()
1296 "tx hang %d detected on queue %d, resetting interface\n", in fm10k_clean_tx_irq()
1297 interface->tx_timeout_count + 1, in fm10k_clean_tx_irq()
1298 tx_ring->queue_index); in fm10k_clean_tx_irq()
1310 #define TX_WAKE_THRESHOLD min_t(u16, FM10K_MIN_TXD - 1, DESC_NEEDED * 2) in fm10k_clean_tx_irq()
1311 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) && in fm10k_clean_tx_irq()
1317 if (__netif_subqueue_stopped(tx_ring->netdev, in fm10k_clean_tx_irq()
1318 tx_ring->queue_index) && in fm10k_clean_tx_irq()
1319 !test_bit(__FM10K_DOWN, interface->state)) { in fm10k_clean_tx_irq()
1320 netif_wake_subqueue(tx_ring->netdev, in fm10k_clean_tx_irq()
1321 tx_ring->queue_index); in fm10k_clean_tx_irq()
1322 ++tx_ring->tx_stats.restart_queue; in fm10k_clean_tx_irq()
1330 * fm10k_update_itr - update the dynamic ITR value based on packet size
1337 * @ring_container: Container for rings to have ITR updated
1344 if (!ITR_IS_ADAPTIVE(ring_container->itr)) in fm10k_update_itr()
1347 packets = ring_container->total_packets; in fm10k_update_itr()
1351 avg_wire_size = ring_container->total_bytes / packets; in fm10k_update_itr()
1388 itr_round = READ_ONCE(ring_container->itr_scale) + 8; in fm10k_update_itr()
1389 avg_wire_size += BIT(itr_round) - 1; in fm10k_update_itr()
1393 ring_container->itr = avg_wire_size | FM10K_ITR_ADAPTIVE; in fm10k_update_itr()
1396 ring_container->total_bytes = 0; in fm10k_update_itr()
1397 ring_container->total_packets = 0; in fm10k_update_itr()
1402 /* Enable auto-mask and clear the current mask */ in fm10k_qv_enable()
1405 /* Update Tx ITR */ in fm10k_qv_enable()
1406 fm10k_update_itr(&q_vector->tx); in fm10k_qv_enable()
1409 fm10k_update_itr(&q_vector->rx); in fm10k_qv_enable()
1411 /* Store Tx itr in timer slot 0 */ in fm10k_qv_enable()
1412 itr |= (q_vector->tx.itr & FM10K_ITR_MAX); in fm10k_qv_enable()
1415 itr |= (q_vector->rx.itr & FM10K_ITR_MAX) << FM10K_ITR_INTERVAL1_SHIFT; in fm10k_qv_enable()
1418 writel(itr, q_vector->itr); in fm10k_qv_enable()
1429 fm10k_for_each_ring(ring, q_vector->tx) { in fm10k_poll()
1441 if (q_vector->rx.count > 1) in fm10k_poll()
1442 per_ring_budget = max(budget / q_vector->rx.count, 1); in fm10k_poll()
1446 fm10k_for_each_ring(ring, q_vector->rx) { in fm10k_poll()
1458 /* Exit the polling mode, but don't re-enable interrupts if stack might in fm10k_poll()
1459 * poll us due to busy-polling in fm10k_poll()
1464 return min(work_done, budget - 1); in fm10k_poll()
1468 * fm10k_set_qos_queues: Allocate queues for a QOS-enabled device
1480 struct net_device *dev = interface->netdev; in fm10k_set_qos_queues()
1485 /* Map queue offset and counts onto allocated tx queues */ in fm10k_set_qos_queues()
1492 f = &interface->ring_feature[RING_F_QOS]; in fm10k_set_qos_queues()
1493 f->indices = pcs; in fm10k_set_qos_queues()
1494 f->mask = BIT(fls(pcs - 1)) - 1; in fm10k_set_qos_queues()
1497 rss_i = interface->hw.mac.max_queues / pcs; in fm10k_set_qos_queues()
1498 rss_i = BIT(fls(rss_i) - 1); in fm10k_set_qos_queues()
1501 f = &interface->ring_feature[RING_F_RSS]; in fm10k_set_qos_queues()
1502 rss_i = min_t(u16, rss_i, f->limit); in fm10k_set_qos_queues()
1503 f->indices = rss_i; in fm10k_set_qos_queues()
1504 f->mask = BIT(fls(rss_i - 1)) - 1; in fm10k_set_qos_queues()
1510 interface->num_rx_queues = rss_i * pcs; in fm10k_set_qos_queues()
1511 interface->num_tx_queues = rss_i * pcs; in fm10k_set_qos_queues()
1521 * to allocate one Rx queue per CPU, and if available, one Tx queue per CPU.
1529 f = &interface->ring_feature[RING_F_RSS]; in fm10k_set_rss_queues()
1530 rss_i = min_t(u16, interface->hw.mac.max_queues, f->limit); in fm10k_set_rss_queues()
1533 f->indices = rss_i; in fm10k_set_rss_queues()
1534 f->mask = BIT(fls(rss_i - 1)) - 1; in fm10k_set_rss_queues()
1536 interface->num_rx_queues = rss_i; in fm10k_set_rss_queues()
1537 interface->num_tx_queues = rss_i; in fm10k_set_rss_queues()
1564 * fm10k_reset_num_queues - Reset the number of queues to zero
1572 interface->num_tx_queues = 0; in fm10k_reset_num_queues()
1573 interface->num_rx_queues = 0; in fm10k_reset_num_queues()
1574 interface->num_q_vectors = 0; in fm10k_reset_num_queues()
1578 * fm10k_alloc_q_vector - Allocate memory for a single interrupt vector
1582 * @txr_count: total number of Tx rings to allocate
1583 * @txr_idx: index of first Tx ring to allocate
1584 * @rxr_count: total number of Rx rings to allocate
1587 * We allocate one q_vector. If allocation fails we return -ENOMEM.
1600 /* allocate q_vector and rings */ in fm10k_alloc_q_vector()
1603 return -ENOMEM; in fm10k_alloc_q_vector()
1606 netif_napi_add(interface->netdev, &q_vector->napi, fm10k_poll); in fm10k_alloc_q_vector()
1609 interface->q_vector[v_idx] = q_vector; in fm10k_alloc_q_vector()
1610 q_vector->interface = interface; in fm10k_alloc_q_vector()
1611 q_vector->v_idx = v_idx; in fm10k_alloc_q_vector()
1613 /* initialize pointer to rings */ in fm10k_alloc_q_vector()
1614 ring = q_vector->ring; in fm10k_alloc_q_vector()
1616 /* save Tx ring container info */ in fm10k_alloc_q_vector()
1617 q_vector->tx.ring = ring; in fm10k_alloc_q_vector()
1618 q_vector->tx.work_limit = FM10K_DEFAULT_TX_WORK; in fm10k_alloc_q_vector()
1619 q_vector->tx.itr = interface->tx_itr; in fm10k_alloc_q_vector()
1620 q_vector->tx.itr_scale = interface->hw.mac.itr_scale; in fm10k_alloc_q_vector()
1621 q_vector->tx.count = txr_count; in fm10k_alloc_q_vector()
1625 ring->dev = &interface->pdev->dev; in fm10k_alloc_q_vector()
1626 ring->netdev = interface->netdev; in fm10k_alloc_q_vector()
1629 ring->q_vector = q_vector; in fm10k_alloc_q_vector()
1631 /* apply Tx specific ring traits */ in fm10k_alloc_q_vector()
1632 ring->count = interface->tx_ring_count; in fm10k_alloc_q_vector()
1633 ring->queue_index = txr_idx; in fm10k_alloc_q_vector()
1636 interface->tx_ring[txr_idx] = ring; in fm10k_alloc_q_vector()
1639 txr_count--; in fm10k_alloc_q_vector()
1647 q_vector->rx.ring = ring; in fm10k_alloc_q_vector()
1648 q_vector->rx.itr = interface->rx_itr; in fm10k_alloc_q_vector()
1649 q_vector->rx.itr_scale = interface->hw.mac.itr_scale; in fm10k_alloc_q_vector()
1650 q_vector->rx.count = rxr_count; in fm10k_alloc_q_vector()
1654 ring->dev = &interface->pdev->dev; in fm10k_alloc_q_vector()
1655 ring->netdev = interface->netdev; in fm10k_alloc_q_vector()
1656 rcu_assign_pointer(ring->l2_accel, interface->l2_accel); in fm10k_alloc_q_vector()
1659 ring->q_vector = q_vector; in fm10k_alloc_q_vector()
1662 ring->count = interface->rx_ring_count; in fm10k_alloc_q_vector()
1663 ring->queue_index = rxr_idx; in fm10k_alloc_q_vector()
1666 interface->rx_ring[rxr_idx] = ring; in fm10k_alloc_q_vector()
1669 rxr_count--; in fm10k_alloc_q_vector()
1682 * fm10k_free_q_vector - Free memory allocated for specific interrupt vector
1692 struct fm10k_q_vector *q_vector = interface->q_vector[v_idx]; in fm10k_free_q_vector()
1697 fm10k_for_each_ring(ring, q_vector->tx) in fm10k_free_q_vector()
1698 interface->tx_ring[ring->queue_index] = NULL; in fm10k_free_q_vector()
1700 fm10k_for_each_ring(ring, q_vector->rx) in fm10k_free_q_vector()
1701 interface->rx_ring[ring->queue_index] = NULL; in fm10k_free_q_vector()
1703 interface->q_vector[v_idx] = NULL; in fm10k_free_q_vector()
1704 netif_napi_del(&q_vector->napi); in fm10k_free_q_vector()
1709 * fm10k_alloc_q_vectors - Allocate memory for interrupt vectors
1713 * return -ENOMEM.
1717 unsigned int q_vectors = interface->num_q_vectors; in fm10k_alloc_q_vectors()
1718 unsigned int rxr_remaining = interface->num_rx_queues; in fm10k_alloc_q_vectors()
1719 unsigned int txr_remaining = interface->num_tx_queues; in fm10k_alloc_q_vectors()
1731 rxr_remaining--; in fm10k_alloc_q_vectors()
1737 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx); in fm10k_alloc_q_vectors()
1738 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx); in fm10k_alloc_q_vectors()
1748 rxr_remaining -= rqpv; in fm10k_alloc_q_vectors()
1749 txr_remaining -= tqpv; in fm10k_alloc_q_vectors()
1759 while (v_idx--) in fm10k_alloc_q_vectors()
1762 return -ENOMEM; in fm10k_alloc_q_vectors()
1766 * fm10k_free_q_vectors - Free memory allocated for interrupt vectors
1775 int v_idx = interface->num_q_vectors; in fm10k_free_q_vectors()
1779 while (v_idx--) in fm10k_free_q_vectors()
1784 * fm10k_reset_msix_capability - reset MSI-X capability
1787 * Reset the MSI-X capability back to its starting state
1791 pci_disable_msix(interface->pdev); in fm10k_reset_msix_capability()
1792 kfree(interface->msix_entries); in fm10k_reset_msix_capability()
1793 interface->msix_entries = NULL; in fm10k_reset_msix_capability()
1797 * fm10k_init_msix_capability - configure MSI-X capability
1805 struct fm10k_hw *hw = &interface->hw; in fm10k_init_msix_capability()
1808 /* It's easy to be greedy for MSI-X vectors, but it really in fm10k_init_msix_capability()
1814 v_budget = max(interface->num_rx_queues, interface->num_tx_queues); in fm10k_init_msix_capability()
1821 * hw.mac->max_msix_vectors vectors. With features in fm10k_init_msix_capability()
1822 * such as RSS and VMDq, we can easily surpass the number of Rx and Tx in fm10k_init_msix_capability()
1826 v_budget = min_t(int, v_budget, hw->mac.max_msix_vectors); in fm10k_init_msix_capability()
1828 /* A failure in MSI-X entry allocation is fatal. */ in fm10k_init_msix_capability()
1829 interface->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry), in fm10k_init_msix_capability()
1831 if (!interface->msix_entries) in fm10k_init_msix_capability()
1832 return -ENOMEM; in fm10k_init_msix_capability()
1836 interface->msix_entries[vector].entry = vector; in fm10k_init_msix_capability()
1838 /* Attempt to enable MSI-X with requested value */ in fm10k_init_msix_capability()
1839 v_budget = pci_enable_msix_range(interface->pdev, in fm10k_init_msix_capability()
1840 interface->msix_entries, in fm10k_init_msix_capability()
1844 kfree(interface->msix_entries); in fm10k_init_msix_capability()
1845 interface->msix_entries = NULL; in fm10k_init_msix_capability()
1850 interface->num_q_vectors = v_budget - NON_Q_VECTORS; in fm10k_init_msix_capability()
1856 * fm10k_cache_ring_qos - Descriptor ring to register mapping for QoS
1857 * @interface: Interface structure continaining rings and devices
1863 struct net_device *dev = interface->netdev; in fm10k_cache_ring_qos()
1865 u16 pc_stride = interface->ring_feature[RING_F_QOS].mask + 1; in fm10k_cache_ring_qos()
1871 rss_i = interface->ring_feature[RING_F_RSS].indices; in fm10k_cache_ring_qos()
1877 interface->tx_ring[offset + i]->reg_idx = q_idx; in fm10k_cache_ring_qos()
1878 interface->tx_ring[offset + i]->qos_pc = pc; in fm10k_cache_ring_qos()
1879 interface->rx_ring[offset + i]->reg_idx = q_idx; in fm10k_cache_ring_qos()
1880 interface->rx_ring[offset + i]->qos_pc = pc; in fm10k_cache_ring_qos()
1889 * fm10k_cache_ring_rss - Descriptor ring to register mapping for RSS
1890 * @interface: Interface structure continaining rings and devices
1898 for (i = 0; i < interface->num_rx_queues; i++) in fm10k_cache_ring_rss()
1899 interface->rx_ring[i]->reg_idx = i; in fm10k_cache_ring_rss()
1901 for (i = 0; i < interface->num_tx_queues; i++) in fm10k_cache_ring_rss()
1902 interface->tx_ring[i]->reg_idx = i; in fm10k_cache_ring_rss()
1906 * fm10k_assign_rings - Map rings to network devices
1907 * @interface: Interface structure containing rings and devices
1910 * devices so that they contain rings, and configure the rings so that
1923 u16 i, rss_i = interface->ring_feature[RING_F_RSS].indices; in fm10k_init_reta()
1929 if (netif_is_rxfh_configured(interface->netdev)) { in fm10k_init_reta()
1930 for (i = FM10K_RETA_SIZE; i--;) { in fm10k_init_reta()
1931 reta = interface->reta[i]; in fm10k_init_reta()
1939 dev_err(&interface->pdev->dev, in fm10k_init_reta()
1953 * fm10k_init_queueing_scheme - Determine proper queueing scheme
1957 * - Hardware queue count (num_*_queues)
1958 * - defined by miscellaneous hardware support/features (RSS, etc.)
1967 /* Configure MSI-X capability */ in fm10k_init_queueing_scheme()
1970 dev_err(&interface->pdev->dev, in fm10k_init_queueing_scheme()
1971 "Unable to initialize MSI-X capability\n"); in fm10k_init_queueing_scheme()
1978 dev_err(&interface->pdev->dev, in fm10k_init_queueing_scheme()
1983 /* Map rings to devices, and map devices to physical queues */ in fm10k_init_queueing_scheme()
1999 * fm10k_clear_queueing_scheme - Clear the current queueing scheme settings
2003 * to pre-load conditions