Lines Matching +full:reserved +full:- +full:cpu +full:- +full:vectors
1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2013 - 2019 Intel Corporation. */
18 "Copyright(c) 2013 - 2019 Intel Corporation.";
28 * fm10k_init_module - Driver Registration Routine
42 return -ENOMEM; in fm10k_init_module()
51 * fm10k_exit_module - Driver Exit Cleanup Routine
70 struct page *page = bi->page; in fm10k_alloc_mapped_page()
80 rx_ring->rx_stats.alloc_failed++; in fm10k_alloc_mapped_page()
85 dma = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE); in fm10k_alloc_mapped_page()
90 if (dma_mapping_error(rx_ring->dev, dma)) { in fm10k_alloc_mapped_page()
93 rx_ring->rx_stats.alloc_failed++; in fm10k_alloc_mapped_page()
97 bi->dma = dma; in fm10k_alloc_mapped_page()
98 bi->page = page; in fm10k_alloc_mapped_page()
99 bi->page_offset = 0; in fm10k_alloc_mapped_page()
105 * fm10k_alloc_rx_buffers - Replace used receive buffers
113 u16 i = rx_ring->next_to_use; in fm10k_alloc_rx_buffers()
120 bi = &rx_ring->rx_buffer[i]; in fm10k_alloc_rx_buffers()
121 i -= rx_ring->count; in fm10k_alloc_rx_buffers()
128 * because each write-back erases this info. in fm10k_alloc_rx_buffers()
130 rx_desc->q.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); in fm10k_alloc_rx_buffers()
137 bi = rx_ring->rx_buffer; in fm10k_alloc_rx_buffers()
138 i -= rx_ring->count; in fm10k_alloc_rx_buffers()
142 rx_desc->d.staterr = 0; in fm10k_alloc_rx_buffers()
144 cleaned_count--; in fm10k_alloc_rx_buffers()
147 i += rx_ring->count; in fm10k_alloc_rx_buffers()
149 if (rx_ring->next_to_use != i) { in fm10k_alloc_rx_buffers()
151 rx_ring->next_to_use = i; in fm10k_alloc_rx_buffers()
154 rx_ring->next_to_alloc = i; in fm10k_alloc_rx_buffers()
158 * applicable for weak-ordered memory model archs, in fm10k_alloc_rx_buffers()
159 * such as IA-64). in fm10k_alloc_rx_buffers()
164 writel(i, rx_ring->tail); in fm10k_alloc_rx_buffers()
169 * fm10k_reuse_rx_page - page flip buffer and store it back on the ring
179 u16 nta = rx_ring->next_to_alloc; in fm10k_reuse_rx_page()
181 new_buff = &rx_ring->rx_buffer[nta]; in fm10k_reuse_rx_page()
185 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; in fm10k_reuse_rx_page()
191 dma_sync_single_range_for_device(rx_ring->dev, old_buff->dma, in fm10k_reuse_rx_page()
192 old_buff->page_offset, in fm10k_reuse_rx_page()
206 /* avoid re-using remote pages */ in fm10k_can_reuse_rx_page()
216 rx_buffer->page_offset ^= FM10K_RX_BUFSZ; in fm10k_can_reuse_rx_page()
219 rx_buffer->page_offset += truesize; in fm10k_can_reuse_rx_page()
221 if (rx_buffer->page_offset > (PAGE_SIZE - FM10K_RX_BUFSZ)) in fm10k_can_reuse_rx_page()
234 * fm10k_add_rx_frag - Add contents of Rx buffer to sk_buff
240 * This function will add the data contained in rx_buffer->page to the skb.
253 struct page *page = rx_buffer->page; in fm10k_add_rx_frag()
254 unsigned char *va = page_address(page) + rx_buffer->page_offset; in fm10k_add_rx_frag()
268 /* page is not reserved, we can reuse buffer as-is */ in fm10k_add_rx_frag()
278 * 60 bytes if the skb->len is less than 60 for skb_pad. in fm10k_add_rx_frag()
280 pull_len = eth_get_headlen(skb->dev, va, FM10K_RX_HDR_LEN); in fm10k_add_rx_frag()
287 size -= pull_len; in fm10k_add_rx_frag()
290 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, in fm10k_add_rx_frag()
300 unsigned int size = le16_to_cpu(rx_desc->w.length); in fm10k_fetch_rx_buffer()
304 rx_buffer = &rx_ring->rx_buffer[rx_ring->next_to_clean]; in fm10k_fetch_rx_buffer()
305 page = rx_buffer->page; in fm10k_fetch_rx_buffer()
310 rx_buffer->page_offset; in fm10k_fetch_rx_buffer()
316 skb = napi_alloc_skb(&rx_ring->q_vector->napi, in fm10k_fetch_rx_buffer()
319 rx_ring->rx_stats.alloc_failed++; in fm10k_fetch_rx_buffer()
323 /* we will be copying header into skb->data in in fm10k_fetch_rx_buffer()
327 prefetchw(skb->data); in fm10k_fetch_rx_buffer()
330 /* we are reusing so sync this buffer for CPU use */ in fm10k_fetch_rx_buffer()
331 dma_sync_single_range_for_cpu(rx_ring->dev, in fm10k_fetch_rx_buffer()
332 rx_buffer->dma, in fm10k_fetch_rx_buffer()
333 rx_buffer->page_offset, in fm10k_fetch_rx_buffer()
343 dma_unmap_page(rx_ring->dev, rx_buffer->dma, in fm10k_fetch_rx_buffer()
348 rx_buffer->page = NULL; in fm10k_fetch_rx_buffer()
360 if (!(ring->netdev->features & NETIF_F_RXCSUM)) in fm10k_rx_checksum()
369 ring->rx_stats.csum_err++; in fm10k_rx_checksum()
375 skb->encapsulation = true; in fm10k_rx_checksum()
379 skb->ip_summed = CHECKSUM_UNNECESSARY; in fm10k_rx_checksum()
381 ring->rx_stats.csum_good++; in fm10k_rx_checksum()
396 if (!(ring->netdev->features & NETIF_F_RXHASH)) in fm10k_rx_hash()
399 rss_type = le16_to_cpu(rx_desc->w.pkt_info) & FM10K_RXD_RSSTYPE_MASK; in fm10k_rx_hash()
403 skb_set_hash(skb, le32_to_cpu(rx_desc->d.rss), in fm10k_rx_hash()
412 struct net_device *dev = rx_ring->netdev; in fm10k_type_trans()
413 struct fm10k_l2_accel *l2_accel = rcu_dereference_bh(rx_ring->l2_accel); in fm10k_type_trans()
417 u16 idx = le16_to_cpu(FM10K_CB(skb)->fi.w.dglort) - 1; in fm10k_type_trans()
419 idx -= l2_accel->dglort; in fm10k_type_trans()
420 if (idx < l2_accel->size && l2_accel->macvlan[idx]) in fm10k_type_trans()
421 dev = l2_accel->macvlan[idx]; in fm10k_type_trans()
428 skb_record_rx_queue(skb, rx_ring->queue_index); in fm10k_type_trans()
430 macvlan_count_rx(netdev_priv(dev), skb->len + ETH_HLEN, true, in fm10k_type_trans()
433 skb->protocol = eth_type_trans(skb, dev); in fm10k_type_trans()
437 * fm10k_process_skb_fields - Populate skb header fields from Rx descriptor
450 unsigned int len = skb->len; in fm10k_process_skb_fields()
456 FM10K_CB(skb)->tstamp = rx_desc->q.timestamp; in fm10k_process_skb_fields()
458 FM10K_CB(skb)->fi.w.vlan = rx_desc->w.vlan; in fm10k_process_skb_fields()
460 FM10K_CB(skb)->fi.d.glort = rx_desc->d.glort; in fm10k_process_skb_fields()
462 if (rx_desc->w.vlan) { in fm10k_process_skb_fields()
463 u16 vid = le16_to_cpu(rx_desc->w.vlan); in fm10k_process_skb_fields()
465 if ((vid & VLAN_VID_MASK) != rx_ring->vid) in fm10k_process_skb_fields()
478 * fm10k_is_non_eop - process handling of non-EOP buffers
485 * that this is in fact a non-EOP buffer.
490 u32 ntc = rx_ring->next_to_clean + 1; in fm10k_is_non_eop()
493 ntc = (ntc < rx_ring->count) ? ntc : 0; in fm10k_is_non_eop()
494 rx_ring->next_to_clean = ntc; in fm10k_is_non_eop()
505 * fm10k_cleanup_headers - Correct corrupted or empty headers
525 ((rxd)->w.csum_err & cpu_to_le16(bit)) in fm10k_cleanup_headers()
527 rx_ring->rx_stats.switch_errors++; in fm10k_cleanup_headers()
529 rx_ring->rx_stats.drops++; in fm10k_cleanup_headers()
531 rx_ring->rx_stats.pp_errors++; in fm10k_cleanup_headers()
533 rx_ring->rx_stats.link_errors++; in fm10k_cleanup_headers()
535 rx_ring->rx_stats.length_errors++; in fm10k_cleanup_headers()
537 rx_ring->rx_stats.errors++; in fm10k_cleanup_headers()
549 * fm10k_receive_skb - helper function to handle rx indications
556 napi_gro_receive(&q_vector->napi, skb); in fm10k_receive_skb()
563 struct sk_buff *skb = rx_ring->skb; in fm10k_clean_rx_irq()
576 rx_desc = FM10K_RX_DESC(rx_ring, rx_ring->next_to_clean); in fm10k_clean_rx_irq()
578 if (!rx_desc->d.staterr) in fm10k_clean_rx_irq()
596 /* fetch next buffer in frame if non-eop */ in fm10k_clean_rx_irq()
619 rx_ring->skb = skb; in fm10k_clean_rx_irq()
621 u64_stats_update_begin(&rx_ring->syncp); in fm10k_clean_rx_irq()
622 rx_ring->stats.packets += total_packets; in fm10k_clean_rx_irq()
623 rx_ring->stats.bytes += total_bytes; in fm10k_clean_rx_irq()
624 u64_stats_update_end(&rx_ring->syncp); in fm10k_clean_rx_irq()
625 q_vector->rx.total_packets += total_packets; in fm10k_clean_rx_irq()
626 q_vector->rx.total_bytes += total_bytes; in fm10k_clean_rx_irq()
634 struct fm10k_intfc *interface = netdev_priv(skb->dev); in fm10k_port_is_vxlan()
636 if (interface->vxlan_port != udp_hdr(skb)->dest) in fm10k_port_is_vxlan()
663 /* verify all reserved flags are 0 */ in fm10k_gre_is_nvgre()
664 if (nvgre_hdr->flags & FM10K_NVGRE_RESERVED0_FLAGS) in fm10k_gre_is_nvgre()
668 if (nvgre_hdr->flags & NVGRE_TNI) in fm10k_gre_is_nvgre()
671 return (struct ethhdr *)(&nvgre_hdr->tni); in fm10k_gre_is_nvgre()
679 if (skb->inner_protocol_type != ENCAP_TYPE_ETHER || in fm10k_tx_encap_offload()
680 skb->inner_protocol != htons(ETH_P_TEB)) in fm10k_tx_encap_offload()
685 l4_hdr = ip_hdr(skb)->protocol; in fm10k_tx_encap_offload()
688 l4_hdr = ipv6_hdr(skb)->nexthdr; in fm10k_tx_encap_offload()
708 switch (eth_hdr->h_proto) { in fm10k_tx_encap_offload()
710 inner_l4_hdr = inner_ip_hdr(skb)->protocol; in fm10k_tx_encap_offload()
713 inner_l4_hdr = inner_ipv6_hdr(skb)->nexthdr; in fm10k_tx_encap_offload()
733 if (skb_inner_transport_header(skb) + inner_l4_hlen - in fm10k_tx_encap_offload()
737 return eth_hdr->h_proto; in fm10k_tx_encap_offload()
743 struct sk_buff *skb = first->skb; in fm10k_tso()
748 if (skb->ip_summed != CHECKSUM_PARTIAL) in fm10k_tso()
755 if (skb->encapsulation) { in fm10k_tso()
764 hdrlen = (th - skb->data) + (((struct tcphdr *)th)->doff << 2); in fm10k_tso()
766 first->tx_flags |= FM10K_TX_FLAGS_CSUM; in fm10k_tso()
769 first->gso_segs = skb_shinfo(skb)->gso_segs; in fm10k_tso()
770 first->bytecount += (first->gso_segs - 1) * hdrlen; in fm10k_tso()
773 tx_desc = FM10K_TX_DESC(tx_ring, tx_ring->next_to_use); in fm10k_tso()
774 tx_desc->hdrlen = hdrlen; in fm10k_tso()
775 tx_desc->mss = cpu_to_le16(skb_shinfo(skb)->gso_size); in fm10k_tso()
780 tx_ring->netdev->features &= ~NETIF_F_GSO_UDP_TUNNEL; in fm10k_tso()
782 netdev_err(tx_ring->netdev, in fm10k_tso()
784 return -1; in fm10k_tso()
790 struct sk_buff *skb = first->skb; in fm10k_tx_csum()
802 if (skb->ip_summed != CHECKSUM_PARTIAL) in fm10k_tx_csum()
805 if (skb->encapsulation) { in fm10k_tx_csum()
809 dev_warn(tx_ring->dev, in fm10k_tx_csum()
811 tx_ring->tx_stats.csum_err++; in fm10k_tx_csum()
825 l4_hdr = network_hdr.ipv4->protocol; in fm10k_tx_csum()
828 l4_hdr = network_hdr.ipv6->nexthdr; in fm10k_tx_csum()
829 if (likely((transport_hdr - network_hdr.raw) == in fm10k_tx_csum()
832 ipv6_skip_exthdr(skb, network_hdr.raw - skb->data + in fm10k_tx_csum()
847 if (skb->encapsulation) in fm10k_tx_csum()
852 dev_warn(tx_ring->dev, in fm10k_tx_csum()
857 tx_ring->tx_stats.csum_err++; in fm10k_tx_csum()
862 first->tx_flags |= FM10K_TX_FLAGS_CSUM; in fm10k_tx_csum()
863 tx_ring->tx_stats.csum_good++; in fm10k_tx_csum()
867 tx_desc = FM10K_TX_DESC(tx_ring, tx_ring->next_to_use); in fm10k_tx_csum()
868 tx_desc->hdrlen = 0; in fm10k_tx_csum()
869 tx_desc->mss = 0; in fm10k_tx_csum()
894 if ((++i & (FM10K_TXD_WB_FIFO_SIZE - 1)) == 0) in fm10k_tx_desc_push()
898 tx_desc->buffer_addr = cpu_to_le64(dma); in fm10k_tx_desc_push()
899 tx_desc->flags = desc_flags; in fm10k_tx_desc_push()
900 tx_desc->buflen = cpu_to_le16(size); in fm10k_tx_desc_push()
903 return i == tx_ring->count; in fm10k_tx_desc_push()
908 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); in __fm10k_maybe_stop_tx()
913 /* Check again in a case another CPU has just made room available */ in __fm10k_maybe_stop_tx()
915 return -EBUSY; in __fm10k_maybe_stop_tx()
917 /* A reprieve! - use start_queue because it doesn't call schedule */ in __fm10k_maybe_stop_tx()
918 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index); in __fm10k_maybe_stop_tx()
919 ++tx_ring->tx_stats.restart_queue; in __fm10k_maybe_stop_tx()
933 struct sk_buff *skb = first->skb; in fm10k_tx_map()
940 u32 tx_flags = first->tx_flags; in fm10k_tx_map()
941 u16 i = tx_ring->next_to_use; in fm10k_tx_map()
948 tx_desc->vlan = cpu_to_le16(skb_vlan_tag_get(skb)); in fm10k_tx_map()
950 tx_desc->vlan = 0; in fm10k_tx_map()
953 data = skb->data; in fm10k_tx_map()
955 dma = dma_map_single(tx_ring->dev, data, size, DMA_TO_DEVICE); in fm10k_tx_map()
957 data_len = skb->data_len; in fm10k_tx_map()
960 for (frag = &skb_shinfo(skb)->frags[0];; frag++) { in fm10k_tx_map()
961 if (dma_mapping_error(tx_ring->dev, dma)) in fm10k_tx_map()
976 size -= FM10K_MAX_DATA_PER_TXD; in fm10k_tx_map()
989 data_len -= size; in fm10k_tx_map()
991 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, in fm10k_tx_map()
994 tx_buffer = &tx_ring->tx_buffer[i]; in fm10k_tx_map()
1004 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount); in fm10k_tx_map()
1007 skb_tx_timestamp(first->skb); in fm10k_tx_map()
1010 * are new descriptors to fetch. (Only applicable for weak-ordered in fm10k_tx_map()
1011 * memory model archs, such as IA-64). in fm10k_tx_map()
1019 first->next_to_watch = tx_desc; in fm10k_tx_map()
1021 tx_ring->next_to_use = i; in fm10k_tx_map()
1028 writel(i, tx_ring->tail); in fm10k_tx_map()
1033 dev_err(tx_ring->dev, "TX DMA map failed\n"); in fm10k_tx_map()
1037 tx_buffer = &tx_ring->tx_buffer[i]; in fm10k_tx_map()
1042 i = tx_ring->count; in fm10k_tx_map()
1043 i--; in fm10k_tx_map()
1046 tx_ring->next_to_use = i; in fm10k_tx_map()
1063 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) { in fm10k_xmit_frame_ring()
1064 skb_frag_t *frag = &skb_shinfo(skb)->frags[f]; in fm10k_xmit_frame_ring()
1070 tx_ring->tx_stats.tx_busy++; in fm10k_xmit_frame_ring()
1075 first = &tx_ring->tx_buffer[tx_ring->next_to_use]; in fm10k_xmit_frame_ring()
1076 first->skb = skb; in fm10k_xmit_frame_ring()
1077 first->bytecount = max_t(unsigned int, skb->len, ETH_ZLEN); in fm10k_xmit_frame_ring()
1078 first->gso_segs = 1; in fm10k_xmit_frame_ring()
1081 first->tx_flags = tx_flags; in fm10k_xmit_frame_ring()
1094 dev_kfree_skb_any(first->skb); in fm10k_xmit_frame_ring()
1095 first->skb = NULL; in fm10k_xmit_frame_ring()
1102 return ring->stats.packets; in fm10k_get_tx_completed()
1106 * fm10k_get_tx_pending - how many Tx descriptors not processed
1112 struct fm10k_intfc *interface = ring->q_vector->interface; in fm10k_get_tx_pending()
1113 struct fm10k_hw *hw = &interface->hw; in fm10k_get_tx_pending()
1117 head = ring->next_to_clean; in fm10k_get_tx_pending()
1118 tail = ring->next_to_use; in fm10k_get_tx_pending()
1120 head = fm10k_read_reg(hw, FM10K_TDH(ring->reg_idx)); in fm10k_get_tx_pending()
1121 tail = fm10k_read_reg(hw, FM10K_TDT(ring->reg_idx)); in fm10k_get_tx_pending()
1124 return ((head <= tail) ? tail : tail + ring->count) - head; in fm10k_get_tx_pending()
1130 u32 tx_done_old = tx_ring->tx_stats.tx_done_old; in fm10k_check_tx_hang()
1145 tx_ring->tx_stats.tx_done_old = tx_done; in fm10k_check_tx_hang()
1147 clear_bit(__FM10K_HANG_CHECK_ARMED, tx_ring->state); in fm10k_check_tx_hang()
1153 return test_and_set_bit(__FM10K_HANG_CHECK_ARMED, tx_ring->state); in fm10k_check_tx_hang()
1157 * fm10k_tx_timeout_reset - initiate reset due to Tx timeout
1163 if (!test_bit(__FM10K_DOWN, interface->state)) { in fm10k_tx_timeout_reset()
1164 interface->tx_timeout_count++; in fm10k_tx_timeout_reset()
1165 set_bit(FM10K_FLAG_RESET_REQUESTED, interface->flags); in fm10k_tx_timeout_reset()
1171 * fm10k_clean_tx_irq - Reclaim resources after transmit completes
1179 struct fm10k_intfc *interface = q_vector->interface; in fm10k_clean_tx_irq()
1183 unsigned int budget = q_vector->tx.work_limit; in fm10k_clean_tx_irq()
1184 unsigned int i = tx_ring->next_to_clean; in fm10k_clean_tx_irq()
1186 if (test_bit(__FM10K_DOWN, interface->state)) in fm10k_clean_tx_irq()
1189 tx_buffer = &tx_ring->tx_buffer[i]; in fm10k_clean_tx_irq()
1191 i -= tx_ring->count; in fm10k_clean_tx_irq()
1194 struct fm10k_tx_desc *eop_desc = tx_buffer->next_to_watch; in fm10k_clean_tx_irq()
1204 if (!(eop_desc->flags & FM10K_TXD_FLAG_DONE)) in fm10k_clean_tx_irq()
1208 tx_buffer->next_to_watch = NULL; in fm10k_clean_tx_irq()
1211 total_bytes += tx_buffer->bytecount; in fm10k_clean_tx_irq()
1212 total_packets += tx_buffer->gso_segs; in fm10k_clean_tx_irq()
1215 napi_consume_skb(tx_buffer->skb, napi_budget); in fm10k_clean_tx_irq()
1218 dma_unmap_single(tx_ring->dev, in fm10k_clean_tx_irq()
1224 tx_buffer->skb = NULL; in fm10k_clean_tx_irq()
1233 i -= tx_ring->count; in fm10k_clean_tx_irq()
1234 tx_buffer = tx_ring->tx_buffer; in fm10k_clean_tx_irq()
1240 dma_unmap_page(tx_ring->dev, in fm10k_clean_tx_irq()
1253 i -= tx_ring->count; in fm10k_clean_tx_irq()
1254 tx_buffer = tx_ring->tx_buffer; in fm10k_clean_tx_irq()
1262 budget--; in fm10k_clean_tx_irq()
1265 i += tx_ring->count; in fm10k_clean_tx_irq()
1266 tx_ring->next_to_clean = i; in fm10k_clean_tx_irq()
1267 u64_stats_update_begin(&tx_ring->syncp); in fm10k_clean_tx_irq()
1268 tx_ring->stats.bytes += total_bytes; in fm10k_clean_tx_irq()
1269 tx_ring->stats.packets += total_packets; in fm10k_clean_tx_irq()
1270 u64_stats_update_end(&tx_ring->syncp); in fm10k_clean_tx_irq()
1271 q_vector->tx.total_bytes += total_bytes; in fm10k_clean_tx_irq()
1272 q_vector->tx.total_packets += total_packets; in fm10k_clean_tx_irq()
1276 struct fm10k_hw *hw = &interface->hw; in fm10k_clean_tx_irq()
1278 netif_err(interface, drv, tx_ring->netdev, in fm10k_clean_tx_irq()
1284 tx_ring->queue_index, in fm10k_clean_tx_irq()
1285 fm10k_read_reg(hw, FM10K_TDH(tx_ring->reg_idx)), in fm10k_clean_tx_irq()
1286 fm10k_read_reg(hw, FM10K_TDT(tx_ring->reg_idx)), in fm10k_clean_tx_irq()
1287 tx_ring->next_to_use, i); in fm10k_clean_tx_irq()
1289 netif_stop_subqueue(tx_ring->netdev, in fm10k_clean_tx_irq()
1290 tx_ring->queue_index); in fm10k_clean_tx_irq()
1292 netif_info(interface, probe, tx_ring->netdev, in fm10k_clean_tx_irq()
1294 interface->tx_timeout_count + 1, in fm10k_clean_tx_irq()
1295 tx_ring->queue_index); in fm10k_clean_tx_irq()
1307 #define TX_WAKE_THRESHOLD min_t(u16, FM10K_MIN_TXD - 1, DESC_NEEDED * 2) in fm10k_clean_tx_irq()
1308 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) && in fm10k_clean_tx_irq()
1314 if (__netif_subqueue_stopped(tx_ring->netdev, in fm10k_clean_tx_irq()
1315 tx_ring->queue_index) && in fm10k_clean_tx_irq()
1316 !test_bit(__FM10K_DOWN, interface->state)) { in fm10k_clean_tx_irq()
1317 netif_wake_subqueue(tx_ring->netdev, in fm10k_clean_tx_irq()
1318 tx_ring->queue_index); in fm10k_clean_tx_irq()
1319 ++tx_ring->tx_stats.restart_queue; in fm10k_clean_tx_irq()
1327 * fm10k_update_itr - update the dynamic ITR value based on packet size
1341 if (!ITR_IS_ADAPTIVE(ring_container->itr)) in fm10k_update_itr()
1344 packets = ring_container->total_packets; in fm10k_update_itr()
1348 avg_wire_size = ring_container->total_bytes / packets; in fm10k_update_itr()
1385 itr_round = READ_ONCE(ring_container->itr_scale) + 8; in fm10k_update_itr()
1386 avg_wire_size += BIT(itr_round) - 1; in fm10k_update_itr()
1390 ring_container->itr = avg_wire_size | FM10K_ITR_ADAPTIVE; in fm10k_update_itr()
1393 ring_container->total_bytes = 0; in fm10k_update_itr()
1394 ring_container->total_packets = 0; in fm10k_update_itr()
1399 /* Enable auto-mask and clear the current mask */ in fm10k_qv_enable()
1403 fm10k_update_itr(&q_vector->tx); in fm10k_qv_enable()
1406 fm10k_update_itr(&q_vector->rx); in fm10k_qv_enable()
1409 itr |= (q_vector->tx.itr & FM10K_ITR_MAX); in fm10k_qv_enable()
1412 itr |= (q_vector->rx.itr & FM10K_ITR_MAX) << FM10K_ITR_INTERVAL1_SHIFT; in fm10k_qv_enable()
1415 writel(itr, q_vector->itr); in fm10k_qv_enable()
1426 fm10k_for_each_ring(ring, q_vector->tx) { in fm10k_poll()
1438 if (q_vector->rx.count > 1) in fm10k_poll()
1439 per_ring_budget = max(budget / q_vector->rx.count, 1); in fm10k_poll()
1443 fm10k_for_each_ring(ring, q_vector->rx) { in fm10k_poll()
1455 /* Exit the polling mode, but don't re-enable interrupts if stack might in fm10k_poll()
1456 * poll us due to busy-polling in fm10k_poll()
1461 return min(work_done, budget - 1); in fm10k_poll()
1465 * fm10k_set_qos_queues: Allocate queues for a QOS-enabled device
1477 struct net_device *dev = interface->netdev; in fm10k_set_qos_queues()
1489 f = &interface->ring_feature[RING_F_QOS]; in fm10k_set_qos_queues()
1490 f->indices = pcs; in fm10k_set_qos_queues()
1491 f->mask = BIT(fls(pcs - 1)) - 1; in fm10k_set_qos_queues()
1494 rss_i = interface->hw.mac.max_queues / pcs; in fm10k_set_qos_queues()
1495 rss_i = BIT(fls(rss_i) - 1); in fm10k_set_qos_queues()
1498 f = &interface->ring_feature[RING_F_RSS]; in fm10k_set_qos_queues()
1499 rss_i = min_t(u16, rss_i, f->limit); in fm10k_set_qos_queues()
1500 f->indices = rss_i; in fm10k_set_qos_queues()
1501 f->mask = BIT(fls(rss_i - 1)) - 1; in fm10k_set_qos_queues()
1507 interface->num_rx_queues = rss_i * pcs; in fm10k_set_qos_queues()
1508 interface->num_tx_queues = rss_i * pcs; in fm10k_set_qos_queues()
1518 * to allocate one Rx queue per CPU, and if available, one Tx queue per CPU.
1526 f = &interface->ring_feature[RING_F_RSS]; in fm10k_set_rss_queues()
1527 rss_i = min_t(u16, interface->hw.mac.max_queues, f->limit); in fm10k_set_rss_queues()
1530 f->indices = rss_i; in fm10k_set_rss_queues()
1531 f->mask = BIT(fls(rss_i - 1)) - 1; in fm10k_set_rss_queues()
1533 interface->num_rx_queues = rss_i; in fm10k_set_rss_queues()
1534 interface->num_tx_queues = rss_i; in fm10k_set_rss_queues()
1561 * fm10k_reset_num_queues - Reset the number of queues to zero
1569 interface->num_tx_queues = 0; in fm10k_reset_num_queues()
1570 interface->num_rx_queues = 0; in fm10k_reset_num_queues()
1571 interface->num_q_vectors = 0; in fm10k_reset_num_queues()
1575 * fm10k_alloc_q_vector - Allocate memory for a single interrupt vector
1584 * We allocate one q_vector. If allocation fails we return -ENOMEM.
1600 return -ENOMEM; in fm10k_alloc_q_vector()
1603 netif_napi_add(interface->netdev, &q_vector->napi, in fm10k_alloc_q_vector()
1607 interface->q_vector[v_idx] = q_vector; in fm10k_alloc_q_vector()
1608 q_vector->interface = interface; in fm10k_alloc_q_vector()
1609 q_vector->v_idx = v_idx; in fm10k_alloc_q_vector()
1612 ring = q_vector->ring; in fm10k_alloc_q_vector()
1615 q_vector->tx.ring = ring; in fm10k_alloc_q_vector()
1616 q_vector->tx.work_limit = FM10K_DEFAULT_TX_WORK; in fm10k_alloc_q_vector()
1617 q_vector->tx.itr = interface->tx_itr; in fm10k_alloc_q_vector()
1618 q_vector->tx.itr_scale = interface->hw.mac.itr_scale; in fm10k_alloc_q_vector()
1619 q_vector->tx.count = txr_count; in fm10k_alloc_q_vector()
1623 ring->dev = &interface->pdev->dev; in fm10k_alloc_q_vector()
1624 ring->netdev = interface->netdev; in fm10k_alloc_q_vector()
1627 ring->q_vector = q_vector; in fm10k_alloc_q_vector()
1630 ring->count = interface->tx_ring_count; in fm10k_alloc_q_vector()
1631 ring->queue_index = txr_idx; in fm10k_alloc_q_vector()
1634 interface->tx_ring[txr_idx] = ring; in fm10k_alloc_q_vector()
1637 txr_count--; in fm10k_alloc_q_vector()
1645 q_vector->rx.ring = ring; in fm10k_alloc_q_vector()
1646 q_vector->rx.itr = interface->rx_itr; in fm10k_alloc_q_vector()
1647 q_vector->rx.itr_scale = interface->hw.mac.itr_scale; in fm10k_alloc_q_vector()
1648 q_vector->rx.count = rxr_count; in fm10k_alloc_q_vector()
1652 ring->dev = &interface->pdev->dev; in fm10k_alloc_q_vector()
1653 ring->netdev = interface->netdev; in fm10k_alloc_q_vector()
1654 rcu_assign_pointer(ring->l2_accel, interface->l2_accel); in fm10k_alloc_q_vector()
1657 ring->q_vector = q_vector; in fm10k_alloc_q_vector()
1660 ring->count = interface->rx_ring_count; in fm10k_alloc_q_vector()
1661 ring->queue_index = rxr_idx; in fm10k_alloc_q_vector()
1664 interface->rx_ring[rxr_idx] = ring; in fm10k_alloc_q_vector()
1667 rxr_count--; in fm10k_alloc_q_vector()
1680 * fm10k_free_q_vector - Free memory allocated for specific interrupt vector
1690 struct fm10k_q_vector *q_vector = interface->q_vector[v_idx]; in fm10k_free_q_vector()
1695 fm10k_for_each_ring(ring, q_vector->tx) in fm10k_free_q_vector()
1696 interface->tx_ring[ring->queue_index] = NULL; in fm10k_free_q_vector()
1698 fm10k_for_each_ring(ring, q_vector->rx) in fm10k_free_q_vector()
1699 interface->rx_ring[ring->queue_index] = NULL; in fm10k_free_q_vector()
1701 interface->q_vector[v_idx] = NULL; in fm10k_free_q_vector()
1702 netif_napi_del(&q_vector->napi); in fm10k_free_q_vector()
1707 * fm10k_alloc_q_vectors - Allocate memory for interrupt vectors
1711 * return -ENOMEM.
1715 unsigned int q_vectors = interface->num_q_vectors; in fm10k_alloc_q_vectors()
1716 unsigned int rxr_remaining = interface->num_rx_queues; in fm10k_alloc_q_vectors()
1717 unsigned int txr_remaining = interface->num_tx_queues; in fm10k_alloc_q_vectors()
1729 rxr_remaining--; in fm10k_alloc_q_vectors()
1735 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx); in fm10k_alloc_q_vectors()
1736 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx); in fm10k_alloc_q_vectors()
1746 rxr_remaining -= rqpv; in fm10k_alloc_q_vectors()
1747 txr_remaining -= tqpv; in fm10k_alloc_q_vectors()
1757 while (v_idx--) in fm10k_alloc_q_vectors()
1760 return -ENOMEM; in fm10k_alloc_q_vectors()
1764 * fm10k_free_q_vectors - Free memory allocated for interrupt vectors
1773 int v_idx = interface->num_q_vectors; in fm10k_free_q_vectors()
1777 while (v_idx--) in fm10k_free_q_vectors()
1782 * f10k_reset_msix_capability - reset MSI-X capability
1785 * Reset the MSI-X capability back to its starting state
1789 pci_disable_msix(interface->pdev); in fm10k_reset_msix_capability()
1790 kfree(interface->msix_entries); in fm10k_reset_msix_capability()
1791 interface->msix_entries = NULL; in fm10k_reset_msix_capability()
1795 * f10k_init_msix_capability - configure MSI-X capability
1803 struct fm10k_hw *hw = &interface->hw; in fm10k_init_msix_capability()
1806 /* It's easy to be greedy for MSI-X vectors, but it really in fm10k_init_msix_capability()
1807 * doesn't do us much good if we have a lot more vectors in fm10k_init_msix_capability()
1808 * than CPU's. So let's be conservative and only ask for in fm10k_init_msix_capability()
1809 * (roughly) the same number of vectors as there are CPU's. in fm10k_init_msix_capability()
1810 * the default is to use pairs of vectors in fm10k_init_msix_capability()
1812 v_budget = max(interface->num_rx_queues, interface->num_tx_queues); in fm10k_init_msix_capability()
1815 /* account for vectors not related to queues */ in fm10k_init_msix_capability()
1819 * hw.mac->max_msix_vectors vectors. With features in fm10k_init_msix_capability()
1822 * those rare cases where the cpu count also exceeds our vector limit. in fm10k_init_msix_capability()
1824 v_budget = min_t(int, v_budget, hw->mac.max_msix_vectors); in fm10k_init_msix_capability()
1826 /* A failure in MSI-X entry allocation is fatal. */ in fm10k_init_msix_capability()
1827 interface->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry), in fm10k_init_msix_capability()
1829 if (!interface->msix_entries) in fm10k_init_msix_capability()
1830 return -ENOMEM; in fm10k_init_msix_capability()
1834 interface->msix_entries[vector].entry = vector; in fm10k_init_msix_capability()
1836 /* Attempt to enable MSI-X with requested value */ in fm10k_init_msix_capability()
1837 v_budget = pci_enable_msix_range(interface->pdev, in fm10k_init_msix_capability()
1838 interface->msix_entries, in fm10k_init_msix_capability()
1842 kfree(interface->msix_entries); in fm10k_init_msix_capability()
1843 interface->msix_entries = NULL; in fm10k_init_msix_capability()
1848 interface->num_q_vectors = v_budget - NON_Q_VECTORS; in fm10k_init_msix_capability()
1854 * fm10k_cache_ring_qos - Descriptor ring to register mapping for QoS
1861 struct net_device *dev = interface->netdev; in fm10k_cache_ring_qos()
1863 u16 pc_stride = interface->ring_feature[RING_F_QOS].mask + 1; in fm10k_cache_ring_qos()
1869 rss_i = interface->ring_feature[RING_F_RSS].indices; in fm10k_cache_ring_qos()
1875 interface->tx_ring[offset + i]->reg_idx = q_idx; in fm10k_cache_ring_qos()
1876 interface->tx_ring[offset + i]->qos_pc = pc; in fm10k_cache_ring_qos()
1877 interface->rx_ring[offset + i]->reg_idx = q_idx; in fm10k_cache_ring_qos()
1878 interface->rx_ring[offset + i]->qos_pc = pc; in fm10k_cache_ring_qos()
1887 * fm10k_cache_ring_rss - Descriptor ring to register mapping for RSS
1896 for (i = 0; i < interface->num_rx_queues; i++) in fm10k_cache_ring_rss()
1897 interface->rx_ring[i]->reg_idx = i; in fm10k_cache_ring_rss()
1899 for (i = 0; i < interface->num_tx_queues; i++) in fm10k_cache_ring_rss()
1900 interface->tx_ring[i]->reg_idx = i; in fm10k_cache_ring_rss()
1904 * fm10k_assign_rings - Map rings to network devices
1921 u16 i, rss_i = interface->ring_feature[RING_F_RSS].indices; in fm10k_init_reta()
1927 if (netif_is_rxfh_configured(interface->netdev)) { in fm10k_init_reta()
1928 for (i = FM10K_RETA_SIZE; i--;) { in fm10k_init_reta()
1929 reta = interface->reta[i]; in fm10k_init_reta()
1937 dev_err(&interface->pdev->dev, in fm10k_init_reta()
1951 * fm10k_init_queueing_scheme - Determine proper queueing scheme
1955 * - Hardware queue count (num_*_queues)
1956 * - defined by miscellaneous hardware support/features (RSS, etc.)
1965 /* Configure MSI-X capability */ in fm10k_init_queueing_scheme()
1968 dev_err(&interface->pdev->dev, in fm10k_init_queueing_scheme()
1969 "Unable to initialize MSI-X capability\n"); in fm10k_init_queueing_scheme()
1976 dev_err(&interface->pdev->dev, in fm10k_init_queueing_scheme()
1977 "Unable to allocate queue vectors\n"); in fm10k_init_queueing_scheme()
1997 * fm10k_clear_queueing_scheme - Clear the current queueing scheme settings
2001 * to pre-load conditions