1 // SPDX-License-Identifier: GPL-2.0
12  * ice_release_rx_desc - Store the new tail and head values
18 	u16 prev_ntu = rx_ring->next_to_use & ~0x7;  in ice_release_rx_desc()
20 	rx_ring->next_to_use = val;  in ice_release_rx_desc()
23 	rx_ring->next_to_alloc = val;  in ice_release_rx_desc()
34 		 * applicable for weak-ordered memory model archs,  in ice_release_rx_desc()
35 		 * such as IA-64).  in ice_release_rx_desc()
38 		writel(val, rx_ring->tail);  in ice_release_rx_desc()
43  * ice_get_rx_hash - get RX hash value from descriptor
52 	if (unlikely(rx_desc->wb.rxdid != ICE_RXDID_FLEX_NIC))  in ice_get_rx_hash()
56 	return le32_to_cpu(nic_mdid->rss_hash);  in ice_get_rx_hash()
60  * ice_rx_hash_to_skb - set the hash value in the skb
75 	if (!libeth_rx_pt_has_hash(rx_ring->netdev, decoded))  in ice_rx_hash_to_skb()
84  * ice_rx_gcs - Set generic checksum in skb
95 	skb->ip_summed = CHECKSUM_COMPLETE;  in ice_rx_gcs()
96 	csum = (__force u16)desc->raw_csum;  in ice_rx_gcs()
97 	skb->csum = csum_unfold((__force __sum16)swab16(csum));  in ice_rx_gcs()
101  * ice_rx_csum - Indicate in skb if checksum is good
107  * skb->protocol must be set before this function is called
118 	skb->ip_summed = CHECKSUM_NONE;  in ice_rx_csum()
121 	if (!libeth_rx_pt_has_checksum(ring->netdev, decoded))  in ice_rx_csum()
124 	rx_status0 = le16_to_cpu(rx_desc->wb.status_error0);  in ice_rx_csum()
125 	rx_status1 = le16_to_cpu(rx_desc->wb.status_error1);  in ice_rx_csum()
127 	if ((ring->flags & ICE_RX_FLAGS_RING_GCS) &&  in ice_rx_csum()
128 	    rx_desc->wb.rxdid == ICE_RXDID_FLEX_NIC &&  in ice_rx_csum()
136 	/* check if HW has decoded the packet and checksum */  in ice_rx_csum()
144 		ring->vsi->back->hw_rx_eipe_error++;  in ice_rx_csum()
170 		skb->csum_level = 1;  in ice_rx_csum()
172 	skb->ip_summed = CHECKSUM_UNNECESSARY;  in ice_rx_csum()
176 	ring->vsi->back->hw_csum_rx_error++;  in ice_rx_csum()
180  * ice_ptp_rx_hwts_to_skb - Put RX timestamp into skb
192 	u64 ts_ns = ice_ptp_get_rx_hwts(rx_desc, &rx_ring->pkt_ctx);  in ice_ptp_rx_hwts_to_skb()
194 	skb_hwtstamps(skb)->hwtstamp = ns_to_ktime(ts_ns);  in ice_ptp_rx_hwts_to_skb()
198  * ice_get_ptype - Read HW packet type from the descriptor
203 	return le16_to_cpu(rx_desc->wb.ptype_flex_flags0) &  in ice_get_ptype()
208  * ice_process_skb_fields - Populate skb header fields from Rx descriptor
226 	/* modifies the skb - consumes the enet header */  in ice_process_skb_fields()
227 	if (unlikely(rx_ring->flags & ICE_RX_FLAGS_MULTIDEV)) {  in ice_process_skb_fields()
232 			ice_repr_inc_rx_stats(netdev, skb->len);  in ice_process_skb_fields()
233 		skb->protocol = eth_type_trans(skb, netdev);  in ice_process_skb_fields()
235 		skb->protocol = eth_type_trans(skb, rx_ring->netdev);  in ice_process_skb_fields()
240 	if (rx_ring->ptp_rx)  in ice_process_skb_fields()
245  * ice_receive_skb - Send a completed packet up the stack
251  * gro receive functions (with/without VLAN tag)
256 	if ((vlan_tci & VLAN_VID_MASK) && rx_ring->vlan_proto)  in ice_receive_skb()
257 		__vlan_hwaccel_put_tag(skb, rx_ring->vlan_proto,  in ice_receive_skb()
260 	napi_gro_receive(&rx_ring->q_vector->napi, skb);  in ice_receive_skb()
264  * ice_clean_xdp_tx_buf - Free and unmap XDP Tx buffer
277 	switch (tx_buf->type) {  in ice_clean_xdp_tx_buf()
279 		page_frag_free(tx_buf->raw_buf);  in ice_clean_xdp_tx_buf()
282 		xdp_return_frame_bulk(tx_buf->xdpf, bq);  in ice_clean_xdp_tx_buf()
286 	tx_buf->type = ICE_TX_BUF_EMPTY;  in ice_clean_xdp_tx_buf()
290  * ice_clean_xdp_irq - Reclaim resources after transmit completes on XDP ring
296 	struct device *dev = xdp_ring->dev;  in ice_clean_xdp_irq()
297 	u32 ntc = xdp_ring->next_to_clean;  in ice_clean_xdp_irq()
299 	u32 cnt = xdp_ring->count;  in ice_clean_xdp_irq()
306 	idx = xdp_ring->tx_buf[ntc].rs_idx;  in ice_clean_xdp_irq()
308 	if (tx_desc->cmd_type_offset_bsz &  in ice_clean_xdp_irq()
311 			ready_frames = idx - ntc + 1;  in ice_clean_xdp_irq()
313 			ready_frames = idx + cnt - ntc + 1;  in ice_clean_xdp_irq()
324 		struct ice_tx_buf *tx_buf = &xdp_ring->tx_buf[ntc];  in ice_clean_xdp_irq()
328 		total_bytes += tx_buf->bytecount;  in ice_clean_xdp_irq()
329 		frags = tx_buf->nr_frags;  in ice_clean_xdp_irq()
332 		ready_frames -= frags + 1;  in ice_clean_xdp_irq()
340 			tx_buf = &xdp_ring->tx_buf[ntc];  in ice_clean_xdp_irq()
354 	tx_desc->cmd_type_offset_bsz = 0;  in ice_clean_xdp_irq()
355 	xdp_ring->next_to_clean = ntc;  in ice_clean_xdp_irq()
356 	xdp_ring->xdp_tx_active -= xdp_tx;  in ice_clean_xdp_irq()
363  * __ice_xmit_xdp_ring - submit frame to XDP ring for transmission
372 	u32 size = xdp->data_end - xdp->data;  in __ice_xmit_xdp_ring()
373 	struct device *dev = xdp_ring->dev;  in __ice_xmit_xdp_ring()
374 	u32 ntu = xdp_ring->next_to_use;  in __ice_xmit_xdp_ring()
378 	u32 cnt = xdp_ring->count;  in __ice_xmit_xdp_ring()
379 	void *data = xdp->data;  in __ice_xmit_xdp_ring()
393 		nr_frags = sinfo->nr_frags;  in __ice_xmit_xdp_ring()
399 	tx_head = &xdp_ring->tx_buf[ntu];  in __ice_xmit_xdp_ring()
414 			tx_buf->type = ICE_TX_BUF_FRAG;  in __ice_xmit_xdp_ring()
416 			tx_buf->type = ICE_TX_BUF_XDP_TX;  in __ice_xmit_xdp_ring()
417 			tx_buf->raw_buf = data;  in __ice_xmit_xdp_ring()
420 		tx_desc->buf_addr = cpu_to_le64(dma);  in __ice_xmit_xdp_ring()
421 		tx_desc->cmd_type_offset_bsz = ice_build_ctob(0, 0, size, 0);  in __ice_xmit_xdp_ring()
431 		tx_buf = &xdp_ring->tx_buf[ntu];  in __ice_xmit_xdp_ring()
433 		data = skb_frag_address(&sinfo->frags[frag]);  in __ice_xmit_xdp_ring()
434 		size = skb_frag_size(&sinfo->frags[frag]);  in __ice_xmit_xdp_ring()
439 	tx_head->bytecount = xdp_get_buff_len(xdp);  in __ice_xmit_xdp_ring()
440 	tx_head->nr_frags = nr_frags;  in __ice_xmit_xdp_ring()
443 		tx_head->type = ICE_TX_BUF_XDP_XMIT;  in __ice_xmit_xdp_ring()
444 		tx_head->xdpf = xdp->data_hard_start;  in __ice_xmit_xdp_ring()
448 	tx_desc->cmd_type_offset_bsz |=  in __ice_xmit_xdp_ring()
451 	xdp_ring->xdp_tx_active++;  in __ice_xmit_xdp_ring()
452 	xdp_ring->next_to_use = ntu;  in __ice_xmit_xdp_ring()
458 		tx_buf = &xdp_ring->tx_buf[ntu];  in __ice_xmit_xdp_ring()
467 		ntu--;  in __ice_xmit_xdp_ring()
472 	xdp_ring->ring_stats->tx_stats.tx_busy++;  in __ice_xmit_xdp_ring()
478  * ice_finalize_xdp_rx - Bump XDP Tx tail and/or flush redirect map
490 	struct ice_tx_buf *tx_buf = &xdp_ring->tx_buf[first_idx];  in ice_finalize_xdp_rx()
497 			spin_lock(&xdp_ring->tx_lock);  in ice_finalize_xdp_rx()
501 		tx_buf->rs_idx = ice_set_rs_bit(xdp_ring);  in ice_finalize_xdp_rx()
504 			spin_unlock(&xdp_ring->tx_lock);  in ice_finalize_xdp_rx()
509  * ice_xdp_rx_hw_ts - HW timestamp XDP hint handler
513  * Copy HW timestamp (if available) to the destination address.
519 	*ts_ns = ice_ptp_get_rx_hwts(xdp_ext->eop_desc,  in ice_xdp_rx_hw_ts()
520 				     xdp_ext->pkt_ctx);  in ice_xdp_rx_hw_ts()
522 		return -ENODATA;  in ice_xdp_rx_hw_ts()
528  * ice_xdp_rx_hash_type - Get XDP-specific hash type from the RX descriptor
538  * ice_xdp_rx_hash - RX hash XDP hint handler
550 	*hash = ice_get_rx_hash(xdp_ext->eop_desc);  in ice_xdp_rx_hash()
551 	*rss_type = ice_xdp_rx_hash_type(xdp_ext->eop_desc);  in ice_xdp_rx_hash()
553 		return -ENODATA;  in ice_xdp_rx_hash()
559  * ice_xdp_rx_vlan_tag - VLAN tag XDP hint handler
572 	*vlan_proto = xdp_ext->pkt_ctx->vlan_proto;  in ice_xdp_rx_vlan_tag()
574 		return -ENODATA;  in ice_xdp_rx_vlan_tag()
576 	*vlan_tci = ice_get_vlan_tci(xdp_ext->eop_desc);  in ice_xdp_rx_vlan_tag()
578 		return -ENODATA;  in ice_xdp_rx_vlan_tag()