1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
2 /* Copyright 2017-2019 NXP */
3 
4 #include "enetc.h"
5 #include <linux/bpf_trace.h>
6 #include <linux/clk.h>
7 #include <linux/tcp.h>
8 #include <linux/udp.h>
9 #include <linux/vmalloc.h>
10 #include <linux/ptp_classify.h>
11 #include <net/ip6_checksum.h>
12 #include <net/pkt_sched.h>
13 #include <net/tso.h>
14 
enetc_port_mac_rd(struct enetc_si * si,u32 reg)15 u32 enetc_port_mac_rd(struct enetc_si *si, u32 reg)
16 {
17 	return enetc_port_rd(&si->hw, reg);
18 }
19 EXPORT_SYMBOL_GPL(enetc_port_mac_rd);
20 
enetc_port_mac_wr(struct enetc_si * si,u32 reg,u32 val)21 void enetc_port_mac_wr(struct enetc_si *si, u32 reg, u32 val)
22 {
23 	enetc_port_wr(&si->hw, reg, val);
24 	if (si->hw_features & ENETC_SI_F_QBU)
25 		enetc_port_wr(&si->hw, reg + si->drvdata->pmac_offset, val);
26 }
27 EXPORT_SYMBOL_GPL(enetc_port_mac_wr);
28 
enetc_change_preemptible_tcs(struct enetc_ndev_priv * priv,u8 preemptible_tcs)29 static void enetc_change_preemptible_tcs(struct enetc_ndev_priv *priv,
30 					 u8 preemptible_tcs)
31 {
32 	if (!(priv->si->hw_features & ENETC_SI_F_QBU))
33 		return;
34 
35 	priv->preemptible_tcs = preemptible_tcs;
36 	enetc_mm_commit_preemptible_tcs(priv);
37 }
38 
enetc_num_stack_tx_queues(struct enetc_ndev_priv * priv)39 static int enetc_num_stack_tx_queues(struct enetc_ndev_priv *priv)
40 {
41 	int num_tx_rings = priv->num_tx_rings;
42 
43 	if (priv->xdp_prog)
44 		return num_tx_rings - num_possible_cpus();
45 
46 	return num_tx_rings;
47 }
48 
enetc_rx_ring_from_xdp_tx_ring(struct enetc_ndev_priv * priv,struct enetc_bdr * tx_ring)49 static struct enetc_bdr *enetc_rx_ring_from_xdp_tx_ring(struct enetc_ndev_priv *priv,
50 							struct enetc_bdr *tx_ring)
51 {
52 	int index = &priv->tx_ring[tx_ring->index] - priv->xdp_tx_ring;
53 
54 	return priv->rx_ring[index];
55 }
56 
enetc_tx_swbd_get_skb(struct enetc_tx_swbd * tx_swbd)57 static struct sk_buff *enetc_tx_swbd_get_skb(struct enetc_tx_swbd *tx_swbd)
58 {
59 	if (tx_swbd->is_xdp_tx || tx_swbd->is_xdp_redirect)
60 		return NULL;
61 
62 	return tx_swbd->skb;
63 }
64 
65 static struct xdp_frame *
enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd * tx_swbd)66 enetc_tx_swbd_get_xdp_frame(struct enetc_tx_swbd *tx_swbd)
67 {
68 	if (tx_swbd->is_xdp_redirect)
69 		return tx_swbd->xdp_frame;
70 
71 	return NULL;
72 }
73 
enetc_unmap_tx_buff(struct enetc_bdr * tx_ring,struct enetc_tx_swbd * tx_swbd)74 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring,
75 				struct enetc_tx_swbd *tx_swbd)
76 {
77 	/* For XDP_TX, pages come from RX, whereas for the other contexts where
78 	 * we have is_dma_page_set, those come from skb_frag_dma_map. We need
79 	 * to match the DMA mapping length, so we need to differentiate those.
80 	 */
81 	if (tx_swbd->is_dma_page)
82 		dma_unmap_page(tx_ring->dev, tx_swbd->dma,
83 			       tx_swbd->is_xdp_tx ? PAGE_SIZE : tx_swbd->len,
84 			       tx_swbd->dir);
85 	else
86 		dma_unmap_single(tx_ring->dev, tx_swbd->dma,
87 				 tx_swbd->len, tx_swbd->dir);
88 	tx_swbd->dma = 0;
89 }
90 
enetc_free_tx_frame(struct enetc_bdr * tx_ring,struct enetc_tx_swbd * tx_swbd)91 static void enetc_free_tx_frame(struct enetc_bdr *tx_ring,
92 				struct enetc_tx_swbd *tx_swbd)
93 {
94 	struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd);
95 	struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd);
96 
97 	if (tx_swbd->dma)
98 		enetc_unmap_tx_buff(tx_ring, tx_swbd);
99 
100 	if (xdp_frame) {
101 		xdp_return_frame(tx_swbd->xdp_frame);
102 		tx_swbd->xdp_frame = NULL;
103 	} else if (skb) {
104 		dev_kfree_skb_any(skb);
105 		tx_swbd->skb = NULL;
106 	}
107 }
108 
109 /* Let H/W know BD ring has been updated */
enetc_update_tx_ring_tail(struct enetc_bdr * tx_ring)110 static void enetc_update_tx_ring_tail(struct enetc_bdr *tx_ring)
111 {
112 	/* includes wmb() */
113 	enetc_wr_reg_hot(tx_ring->tpir, tx_ring->next_to_use);
114 }
115 
enetc_ptp_parse(struct sk_buff * skb,u8 * udp,u8 * msgtype,u8 * twostep,u16 * correction_offset,u16 * body_offset)116 static int enetc_ptp_parse(struct sk_buff *skb, u8 *udp,
117 			   u8 *msgtype, u8 *twostep,
118 			   u16 *correction_offset, u16 *body_offset)
119 {
120 	unsigned int ptp_class;
121 	struct ptp_header *hdr;
122 	unsigned int type;
123 	u8 *base;
124 
125 	ptp_class = ptp_classify_raw(skb);
126 	if (ptp_class == PTP_CLASS_NONE)
127 		return -EINVAL;
128 
129 	hdr = ptp_parse_header(skb, ptp_class);
130 	if (!hdr)
131 		return -EINVAL;
132 
133 	type = ptp_class & PTP_CLASS_PMASK;
134 	if (type == PTP_CLASS_IPV4 || type == PTP_CLASS_IPV6)
135 		*udp = 1;
136 	else
137 		*udp = 0;
138 
139 	*msgtype = ptp_get_msgtype(hdr, ptp_class);
140 	*twostep = hdr->flag_field[0] & 0x2;
141 
142 	base = skb_mac_header(skb);
143 	*correction_offset = (u8 *)&hdr->correction - base;
144 	*body_offset = (u8 *)hdr + sizeof(struct ptp_header) - base;
145 
146 	return 0;
147 }
148 
enetc_tx_csum_offload_check(struct sk_buff * skb)149 static bool enetc_tx_csum_offload_check(struct sk_buff *skb)
150 {
151 	switch (skb->csum_offset) {
152 	case offsetof(struct tcphdr, check):
153 	case offsetof(struct udphdr, check):
154 		return true;
155 	default:
156 		return false;
157 	}
158 }
159 
enetc_skb_is_ipv6(struct sk_buff * skb)160 static bool enetc_skb_is_ipv6(struct sk_buff *skb)
161 {
162 	return vlan_get_protocol(skb) == htons(ETH_P_IPV6);
163 }
164 
enetc_skb_is_tcp(struct sk_buff * skb)165 static bool enetc_skb_is_tcp(struct sk_buff *skb)
166 {
167 	return skb->csum_offset == offsetof(struct tcphdr, check);
168 }
169 
170 /**
171  * enetc_unwind_tx_frame() - Unwind the DMA mappings of a multi-buffer Tx frame
172  * @tx_ring: Pointer to the Tx ring on which the buffer descriptors are located
173  * @count: Number of Tx buffer descriptors which need to be unmapped
174  * @i: Index of the last successfully mapped Tx buffer descriptor
175  */
enetc_unwind_tx_frame(struct enetc_bdr * tx_ring,int count,int i)176 static void enetc_unwind_tx_frame(struct enetc_bdr *tx_ring, int count, int i)
177 {
178 	while (count--) {
179 		struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i];
180 
181 		enetc_free_tx_frame(tx_ring, tx_swbd);
182 		if (i == 0)
183 			i = tx_ring->bd_count;
184 		i--;
185 	}
186 }
187 
enetc_map_tx_buffs(struct enetc_bdr * tx_ring,struct sk_buff * skb)188 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb)
189 {
190 	bool do_vlan, do_onestep_tstamp = false, do_twostep_tstamp = false;
191 	struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev);
192 	struct enetc_hw *hw = &priv->si->hw;
193 	struct enetc_tx_swbd *tx_swbd;
194 	int len = skb_headlen(skb);
195 	union enetc_tx_bd temp_bd;
196 	u8 msgtype, twostep, udp;
197 	union enetc_tx_bd *txbd;
198 	u16 offset1, offset2;
199 	int i, count = 0;
200 	skb_frag_t *frag;
201 	unsigned int f;
202 	dma_addr_t dma;
203 	u8 flags = 0;
204 
205 	enetc_clear_tx_bd(&temp_bd);
206 	if (skb->ip_summed == CHECKSUM_PARTIAL) {
207 		/* Can not support TSD and checksum offload at the same time */
208 		if (priv->active_offloads & ENETC_F_TXCSUM &&
209 		    enetc_tx_csum_offload_check(skb) && !tx_ring->tsd_enable) {
210 			temp_bd.l3_aux0 = FIELD_PREP(ENETC_TX_BD_L3_START,
211 						     skb_network_offset(skb));
212 			temp_bd.l3_aux1 = FIELD_PREP(ENETC_TX_BD_L3_HDR_LEN,
213 						     skb_network_header_len(skb) / 4);
214 			temp_bd.l3_aux1 |= FIELD_PREP(ENETC_TX_BD_L3T,
215 						      enetc_skb_is_ipv6(skb));
216 			if (enetc_skb_is_tcp(skb))
217 				temp_bd.l4_aux = FIELD_PREP(ENETC_TX_BD_L4T,
218 							    ENETC_TXBD_L4T_TCP);
219 			else
220 				temp_bd.l4_aux = FIELD_PREP(ENETC_TX_BD_L4T,
221 							    ENETC_TXBD_L4T_UDP);
222 			flags |= ENETC_TXBD_FLAGS_CSUM_LSO | ENETC_TXBD_FLAGS_L4CS;
223 		} else if (skb_checksum_help(skb)) {
224 			return 0;
225 		}
226 	}
227 
228 	i = tx_ring->next_to_use;
229 	txbd = ENETC_TXBD(*tx_ring, i);
230 	prefetchw(txbd);
231 
232 	dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE);
233 	if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
234 		goto dma_err;
235 
236 	temp_bd.addr = cpu_to_le64(dma);
237 	temp_bd.buf_len = cpu_to_le16(len);
238 
239 	tx_swbd = &tx_ring->tx_swbd[i];
240 	tx_swbd->dma = dma;
241 	tx_swbd->len = len;
242 	tx_swbd->is_dma_page = 0;
243 	tx_swbd->dir = DMA_TO_DEVICE;
244 	count++;
245 
246 	do_vlan = skb_vlan_tag_present(skb);
247 	if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
248 		if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep, &offset1,
249 				    &offset2) ||
250 		    msgtype != PTP_MSGTYPE_SYNC || twostep)
251 			WARN_ONCE(1, "Bad packet for one-step timestamping\n");
252 		else
253 			do_onestep_tstamp = true;
254 	} else if (skb->cb[0] & ENETC_F_TX_TSTAMP) {
255 		do_twostep_tstamp = true;
256 	}
257 
258 	tx_swbd->do_twostep_tstamp = do_twostep_tstamp;
259 	tx_swbd->qbv_en = !!(priv->active_offloads & ENETC_F_QBV);
260 	tx_swbd->check_wb = tx_swbd->do_twostep_tstamp || tx_swbd->qbv_en;
261 
262 	if (do_vlan || do_onestep_tstamp || do_twostep_tstamp)
263 		flags |= ENETC_TXBD_FLAGS_EX;
264 
265 	if (tx_ring->tsd_enable)
266 		flags |= ENETC_TXBD_FLAGS_TSE | ENETC_TXBD_FLAGS_TXSTART;
267 
268 	/* first BD needs frm_len and offload flags set */
269 	temp_bd.frm_len = cpu_to_le16(skb->len);
270 	temp_bd.flags = flags;
271 
272 	if (flags & ENETC_TXBD_FLAGS_TSE)
273 		temp_bd.txstart = enetc_txbd_set_tx_start(skb->skb_mstamp_ns,
274 							  flags);
275 
276 	if (flags & ENETC_TXBD_FLAGS_EX) {
277 		u8 e_flags = 0;
278 		*txbd = temp_bd;
279 		enetc_clear_tx_bd(&temp_bd);
280 
281 		/* add extension BD for VLAN and/or timestamping */
282 		flags = 0;
283 		tx_swbd++;
284 		txbd++;
285 		i++;
286 		if (unlikely(i == tx_ring->bd_count)) {
287 			i = 0;
288 			tx_swbd = tx_ring->tx_swbd;
289 			txbd = ENETC_TXBD(*tx_ring, 0);
290 		}
291 		prefetchw(txbd);
292 
293 		if (do_vlan) {
294 			temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
295 			temp_bd.ext.tpid = 0; /* < C-TAG */
296 			e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS;
297 		}
298 
299 		if (do_onestep_tstamp) {
300 			__be32 new_sec_l, new_nsec;
301 			u32 lo, hi, nsec, val;
302 			__be16 new_sec_h;
303 			u8 *data;
304 			u64 sec;
305 
306 			lo = enetc_rd_hot(hw, ENETC_SICTR0);
307 			hi = enetc_rd_hot(hw, ENETC_SICTR1);
308 			sec = (u64)hi << 32 | lo;
309 			nsec = do_div(sec, 1000000000);
310 
311 			/* Configure extension BD */
312 			temp_bd.ext.tstamp = cpu_to_le32(lo & 0x3fffffff);
313 			e_flags |= ENETC_TXBD_E_FLAGS_ONE_STEP_PTP;
314 
315 			/* Update originTimestamp field of Sync packet
316 			 * - 48 bits seconds field
317 			 * - 32 bits nanseconds field
318 			 *
319 			 * In addition, the UDP checksum needs to be updated
320 			 * by software after updating originTimestamp field,
321 			 * otherwise the hardware will calculate the wrong
322 			 * checksum when updating the correction field and
323 			 * update it to the packet.
324 			 */
325 			data = skb_mac_header(skb);
326 			new_sec_h = htons((sec >> 32) & 0xffff);
327 			new_sec_l = htonl(sec & 0xffffffff);
328 			new_nsec = htonl(nsec);
329 			if (udp) {
330 				struct udphdr *uh = udp_hdr(skb);
331 				__be32 old_sec_l, old_nsec;
332 				__be16 old_sec_h;
333 
334 				old_sec_h = *(__be16 *)(data + offset2);
335 				inet_proto_csum_replace2(&uh->check, skb, old_sec_h,
336 							 new_sec_h, false);
337 
338 				old_sec_l = *(__be32 *)(data + offset2 + 2);
339 				inet_proto_csum_replace4(&uh->check, skb, old_sec_l,
340 							 new_sec_l, false);
341 
342 				old_nsec = *(__be32 *)(data + offset2 + 6);
343 				inet_proto_csum_replace4(&uh->check, skb, old_nsec,
344 							 new_nsec, false);
345 			}
346 
347 			*(__be16 *)(data + offset2) = new_sec_h;
348 			*(__be32 *)(data + offset2 + 2) = new_sec_l;
349 			*(__be32 *)(data + offset2 + 6) = new_nsec;
350 
351 			/* Configure single-step register */
352 			val = ENETC_PM0_SINGLE_STEP_EN;
353 			val |= ENETC_SET_SINGLE_STEP_OFFSET(offset1);
354 			if (udp)
355 				val |= ENETC_PM0_SINGLE_STEP_CH;
356 
357 			enetc_port_mac_wr(priv->si, ENETC_PM0_SINGLE_STEP,
358 					  val);
359 		} else if (do_twostep_tstamp) {
360 			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
361 			e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP;
362 		}
363 
364 		temp_bd.ext.e_flags = e_flags;
365 		count++;
366 	}
367 
368 	frag = &skb_shinfo(skb)->frags[0];
369 	for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) {
370 		len = skb_frag_size(frag);
371 		dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len,
372 				       DMA_TO_DEVICE);
373 		if (dma_mapping_error(tx_ring->dev, dma))
374 			goto dma_err;
375 
376 		*txbd = temp_bd;
377 		enetc_clear_tx_bd(&temp_bd);
378 
379 		flags = 0;
380 		tx_swbd++;
381 		txbd++;
382 		i++;
383 		if (unlikely(i == tx_ring->bd_count)) {
384 			i = 0;
385 			tx_swbd = tx_ring->tx_swbd;
386 			txbd = ENETC_TXBD(*tx_ring, 0);
387 		}
388 		prefetchw(txbd);
389 
390 		temp_bd.addr = cpu_to_le64(dma);
391 		temp_bd.buf_len = cpu_to_le16(len);
392 
393 		tx_swbd->dma = dma;
394 		tx_swbd->len = len;
395 		tx_swbd->is_dma_page = 1;
396 		tx_swbd->dir = DMA_TO_DEVICE;
397 		count++;
398 	}
399 
400 	/* last BD needs 'F' bit set */
401 	flags |= ENETC_TXBD_FLAGS_F;
402 	temp_bd.flags = flags;
403 	*txbd = temp_bd;
404 
405 	tx_ring->tx_swbd[i].is_eof = true;
406 	tx_ring->tx_swbd[i].skb = skb;
407 
408 	enetc_bdr_idx_inc(tx_ring, &i);
409 	tx_ring->next_to_use = i;
410 
411 	skb_tx_timestamp(skb);
412 
413 	enetc_update_tx_ring_tail(tx_ring);
414 
415 	return count;
416 
417 dma_err:
418 	dev_err(tx_ring->dev, "DMA map error");
419 
420 	enetc_unwind_tx_frame(tx_ring, count, i);
421 
422 	return 0;
423 }
424 
enetc_map_tx_tso_hdr(struct enetc_bdr * tx_ring,struct sk_buff * skb,struct enetc_tx_swbd * tx_swbd,union enetc_tx_bd * txbd,int * i,int hdr_len,int data_len)425 static int enetc_map_tx_tso_hdr(struct enetc_bdr *tx_ring, struct sk_buff *skb,
426 				struct enetc_tx_swbd *tx_swbd,
427 				union enetc_tx_bd *txbd, int *i, int hdr_len,
428 				int data_len)
429 {
430 	union enetc_tx_bd txbd_tmp;
431 	u8 flags = 0, e_flags = 0;
432 	dma_addr_t addr;
433 	int count = 1;
434 
435 	enetc_clear_tx_bd(&txbd_tmp);
436 	addr = tx_ring->tso_headers_dma + *i * TSO_HEADER_SIZE;
437 
438 	if (skb_vlan_tag_present(skb))
439 		flags |= ENETC_TXBD_FLAGS_EX;
440 
441 	txbd_tmp.addr = cpu_to_le64(addr);
442 	txbd_tmp.buf_len = cpu_to_le16(hdr_len);
443 
444 	/* first BD needs frm_len and offload flags set */
445 	txbd_tmp.frm_len = cpu_to_le16(hdr_len + data_len);
446 	txbd_tmp.flags = flags;
447 
448 	/* For the TSO header we do not set the dma address since we do not
449 	 * want it unmapped when we do cleanup. We still set len so that we
450 	 * count the bytes sent.
451 	 */
452 	tx_swbd->len = hdr_len;
453 	tx_swbd->do_twostep_tstamp = false;
454 	tx_swbd->check_wb = false;
455 
456 	/* Actually write the header in the BD */
457 	*txbd = txbd_tmp;
458 
459 	/* Add extension BD for VLAN */
460 	if (flags & ENETC_TXBD_FLAGS_EX) {
461 		/* Get the next BD */
462 		enetc_bdr_idx_inc(tx_ring, i);
463 		txbd = ENETC_TXBD(*tx_ring, *i);
464 		tx_swbd = &tx_ring->tx_swbd[*i];
465 		prefetchw(txbd);
466 
467 		/* Setup the VLAN fields */
468 		enetc_clear_tx_bd(&txbd_tmp);
469 		txbd_tmp.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
470 		txbd_tmp.ext.tpid = 0; /* < C-TAG */
471 		e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS;
472 
473 		/* Write the BD */
474 		txbd_tmp.ext.e_flags = e_flags;
475 		*txbd = txbd_tmp;
476 		count++;
477 	}
478 
479 	return count;
480 }
481 
enetc_map_tx_tso_data(struct enetc_bdr * tx_ring,struct sk_buff * skb,struct enetc_tx_swbd * tx_swbd,union enetc_tx_bd * txbd,char * data,int size,bool last_bd)482 static int enetc_map_tx_tso_data(struct enetc_bdr *tx_ring, struct sk_buff *skb,
483 				 struct enetc_tx_swbd *tx_swbd,
484 				 union enetc_tx_bd *txbd, char *data,
485 				 int size, bool last_bd)
486 {
487 	union enetc_tx_bd txbd_tmp;
488 	dma_addr_t addr;
489 	u8 flags = 0;
490 
491 	enetc_clear_tx_bd(&txbd_tmp);
492 
493 	addr = dma_map_single(tx_ring->dev, data, size, DMA_TO_DEVICE);
494 	if (unlikely(dma_mapping_error(tx_ring->dev, addr))) {
495 		netdev_err(tx_ring->ndev, "DMA map error\n");
496 		return -ENOMEM;
497 	}
498 
499 	if (last_bd) {
500 		flags |= ENETC_TXBD_FLAGS_F;
501 		tx_swbd->is_eof = 1;
502 	}
503 
504 	txbd_tmp.addr = cpu_to_le64(addr);
505 	txbd_tmp.buf_len = cpu_to_le16(size);
506 	txbd_tmp.flags = flags;
507 
508 	tx_swbd->dma = addr;
509 	tx_swbd->len = size;
510 	tx_swbd->dir = DMA_TO_DEVICE;
511 
512 	*txbd = txbd_tmp;
513 
514 	return 0;
515 }
516 
enetc_tso_hdr_csum(struct tso_t * tso,struct sk_buff * skb,char * hdr,int hdr_len,int * l4_hdr_len)517 static __wsum enetc_tso_hdr_csum(struct tso_t *tso, struct sk_buff *skb,
518 				 char *hdr, int hdr_len, int *l4_hdr_len)
519 {
520 	char *l4_hdr = hdr + skb_transport_offset(skb);
521 	int mac_hdr_len = skb_network_offset(skb);
522 
523 	if (tso->tlen != sizeof(struct udphdr)) {
524 		struct tcphdr *tcph = (struct tcphdr *)(l4_hdr);
525 
526 		tcph->check = 0;
527 	} else {
528 		struct udphdr *udph = (struct udphdr *)(l4_hdr);
529 
530 		udph->check = 0;
531 	}
532 
533 	/* Compute the IP checksum. This is necessary since tso_build_hdr()
534 	 * already incremented the IP ID field.
535 	 */
536 	if (!tso->ipv6) {
537 		struct iphdr *iph = (void *)(hdr + mac_hdr_len);
538 
539 		iph->check = 0;
540 		iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl);
541 	}
542 
543 	/* Compute the checksum over the L4 header. */
544 	*l4_hdr_len = hdr_len - skb_transport_offset(skb);
545 	return csum_partial(l4_hdr, *l4_hdr_len, 0);
546 }
547 
enetc_tso_complete_csum(struct enetc_bdr * tx_ring,struct tso_t * tso,struct sk_buff * skb,char * hdr,int len,__wsum sum)548 static void enetc_tso_complete_csum(struct enetc_bdr *tx_ring, struct tso_t *tso,
549 				    struct sk_buff *skb, char *hdr, int len,
550 				    __wsum sum)
551 {
552 	char *l4_hdr = hdr + skb_transport_offset(skb);
553 	__sum16 csum_final;
554 
555 	/* Complete the L4 checksum by appending the pseudo-header to the
556 	 * already computed checksum.
557 	 */
558 	if (!tso->ipv6)
559 		csum_final = csum_tcpudp_magic(ip_hdr(skb)->saddr,
560 					       ip_hdr(skb)->daddr,
561 					       len, ip_hdr(skb)->protocol, sum);
562 	else
563 		csum_final = csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
564 					     &ipv6_hdr(skb)->daddr,
565 					     len, ipv6_hdr(skb)->nexthdr, sum);
566 
567 	if (tso->tlen != sizeof(struct udphdr)) {
568 		struct tcphdr *tcph = (struct tcphdr *)(l4_hdr);
569 
570 		tcph->check = csum_final;
571 	} else {
572 		struct udphdr *udph = (struct udphdr *)(l4_hdr);
573 
574 		udph->check = csum_final;
575 	}
576 }
577 
enetc_lso_count_descs(const struct sk_buff * skb)578 static int enetc_lso_count_descs(const struct sk_buff *skb)
579 {
580 	/* 4 BDs: 1 BD for LSO header + 1 BD for extended BD + 1 BD
581 	 * for linear area data but not include LSO header, namely
582 	 * skb_headlen(skb) - lso_hdr_len (it may be 0, but that's
583 	 * okay, we only need to consider the worst case). And 1 BD
584 	 * for gap.
585 	 */
586 	return skb_shinfo(skb)->nr_frags + 4;
587 }
588 
enetc_lso_get_hdr_len(const struct sk_buff * skb)589 static int enetc_lso_get_hdr_len(const struct sk_buff *skb)
590 {
591 	int hdr_len, tlen;
592 
593 	tlen = skb_is_gso_tcp(skb) ? tcp_hdrlen(skb) : sizeof(struct udphdr);
594 	hdr_len = skb_transport_offset(skb) + tlen;
595 
596 	return hdr_len;
597 }
598 
enetc_lso_start(struct sk_buff * skb,struct enetc_lso_t * lso)599 static void enetc_lso_start(struct sk_buff *skb, struct enetc_lso_t *lso)
600 {
601 	lso->lso_seg_size = skb_shinfo(skb)->gso_size;
602 	lso->ipv6 = enetc_skb_is_ipv6(skb);
603 	lso->tcp = skb_is_gso_tcp(skb);
604 	lso->l3_hdr_len = skb_network_header_len(skb);
605 	lso->l3_start = skb_network_offset(skb);
606 	lso->hdr_len = enetc_lso_get_hdr_len(skb);
607 	lso->total_len = skb->len - lso->hdr_len;
608 }
609 
enetc_lso_map_hdr(struct enetc_bdr * tx_ring,struct sk_buff * skb,int * i,struct enetc_lso_t * lso)610 static void enetc_lso_map_hdr(struct enetc_bdr *tx_ring, struct sk_buff *skb,
611 			      int *i, struct enetc_lso_t *lso)
612 {
613 	union enetc_tx_bd txbd_tmp, *txbd;
614 	struct enetc_tx_swbd *tx_swbd;
615 	u16 frm_len, frm_len_ext;
616 	u8 flags, e_flags = 0;
617 	dma_addr_t addr;
618 	char *hdr;
619 
620 	/* Get the first BD of the LSO BDs chain */
621 	txbd = ENETC_TXBD(*tx_ring, *i);
622 	tx_swbd = &tx_ring->tx_swbd[*i];
623 	prefetchw(txbd);
624 
625 	/* Prepare LSO header: MAC + IP + TCP/UDP */
626 	hdr = tx_ring->tso_headers + *i * TSO_HEADER_SIZE;
627 	memcpy(hdr, skb->data, lso->hdr_len);
628 	addr = tx_ring->tso_headers_dma + *i * TSO_HEADER_SIZE;
629 
630 	/* {frm_len_ext, frm_len} indicates the total length of
631 	 * large transmit data unit. frm_len contains the 16 least
632 	 * significant bits and frm_len_ext contains the 4 most
633 	 * significant bits.
634 	 */
635 	frm_len = lso->total_len & 0xffff;
636 	frm_len_ext = (lso->total_len >> 16) & 0xf;
637 
638 	/* Set the flags of the first BD */
639 	flags = ENETC_TXBD_FLAGS_EX | ENETC_TXBD_FLAGS_CSUM_LSO |
640 		ENETC_TXBD_FLAGS_LSO | ENETC_TXBD_FLAGS_L4CS;
641 
642 	enetc_clear_tx_bd(&txbd_tmp);
643 	txbd_tmp.addr = cpu_to_le64(addr);
644 	txbd_tmp.hdr_len = cpu_to_le16(lso->hdr_len);
645 
646 	/* first BD needs frm_len and offload flags set */
647 	txbd_tmp.frm_len = cpu_to_le16(frm_len);
648 	txbd_tmp.flags = flags;
649 
650 	txbd_tmp.l3_aux0 = FIELD_PREP(ENETC_TX_BD_L3_START, lso->l3_start);
651 	/* l3_hdr_size in 32-bits (4 bytes) */
652 	txbd_tmp.l3_aux1 = FIELD_PREP(ENETC_TX_BD_L3_HDR_LEN,
653 				      lso->l3_hdr_len / 4);
654 	if (lso->ipv6)
655 		txbd_tmp.l3_aux1 |= ENETC_TX_BD_L3T;
656 	else
657 		txbd_tmp.l3_aux0 |= ENETC_TX_BD_IPCS;
658 
659 	txbd_tmp.l4_aux = FIELD_PREP(ENETC_TX_BD_L4T, lso->tcp ?
660 				     ENETC_TXBD_L4T_TCP : ENETC_TXBD_L4T_UDP);
661 
662 	/* For the LSO header we do not set the dma address since
663 	 * we do not want it unmapped when we do cleanup. We still
664 	 * set len so that we count the bytes sent.
665 	 */
666 	tx_swbd->len = lso->hdr_len;
667 	tx_swbd->do_twostep_tstamp = false;
668 	tx_swbd->check_wb = false;
669 
670 	/* Actually write the header in the BD */
671 	*txbd = txbd_tmp;
672 
673 	/* Get the next BD, and the next BD is extended BD */
674 	enetc_bdr_idx_inc(tx_ring, i);
675 	txbd = ENETC_TXBD(*tx_ring, *i);
676 	tx_swbd = &tx_ring->tx_swbd[*i];
677 	prefetchw(txbd);
678 
679 	enetc_clear_tx_bd(&txbd_tmp);
680 	if (skb_vlan_tag_present(skb)) {
681 		/* Setup the VLAN fields */
682 		txbd_tmp.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
683 		txbd_tmp.ext.tpid = ENETC_TPID_8021Q;
684 		e_flags = ENETC_TXBD_E_FLAGS_VLAN_INS;
685 	}
686 
687 	/* Write the BD */
688 	txbd_tmp.ext.e_flags = e_flags;
689 	txbd_tmp.ext.lso_sg_size = cpu_to_le16(lso->lso_seg_size);
690 	txbd_tmp.ext.frm_len_ext = cpu_to_le16(frm_len_ext);
691 	*txbd = txbd_tmp;
692 }
693 
enetc_lso_map_data(struct enetc_bdr * tx_ring,struct sk_buff * skb,int * i,struct enetc_lso_t * lso,int * count)694 static int enetc_lso_map_data(struct enetc_bdr *tx_ring, struct sk_buff *skb,
695 			      int *i, struct enetc_lso_t *lso, int *count)
696 {
697 	union enetc_tx_bd txbd_tmp, *txbd = NULL;
698 	struct enetc_tx_swbd *tx_swbd;
699 	skb_frag_t *frag;
700 	dma_addr_t dma;
701 	u8 flags = 0;
702 	int len, f;
703 
704 	len = skb_headlen(skb) - lso->hdr_len;
705 	if (len > 0) {
706 		dma = dma_map_single(tx_ring->dev, skb->data + lso->hdr_len,
707 				     len, DMA_TO_DEVICE);
708 		if (dma_mapping_error(tx_ring->dev, dma))
709 			return -ENOMEM;
710 
711 		enetc_bdr_idx_inc(tx_ring, i);
712 		txbd = ENETC_TXBD(*tx_ring, *i);
713 		tx_swbd = &tx_ring->tx_swbd[*i];
714 		prefetchw(txbd);
715 		*count += 1;
716 
717 		enetc_clear_tx_bd(&txbd_tmp);
718 		txbd_tmp.addr = cpu_to_le64(dma);
719 		txbd_tmp.buf_len = cpu_to_le16(len);
720 
721 		tx_swbd->dma = dma;
722 		tx_swbd->len = len;
723 		tx_swbd->is_dma_page = 0;
724 		tx_swbd->dir = DMA_TO_DEVICE;
725 	}
726 
727 	frag = &skb_shinfo(skb)->frags[0];
728 	for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) {
729 		if (txbd)
730 			*txbd = txbd_tmp;
731 
732 		len = skb_frag_size(frag);
733 		dma = skb_frag_dma_map(tx_ring->dev, frag);
734 		if (dma_mapping_error(tx_ring->dev, dma))
735 			return -ENOMEM;
736 
737 		/* Get the next BD */
738 		enetc_bdr_idx_inc(tx_ring, i);
739 		txbd = ENETC_TXBD(*tx_ring, *i);
740 		tx_swbd = &tx_ring->tx_swbd[*i];
741 		prefetchw(txbd);
742 		*count += 1;
743 
744 		enetc_clear_tx_bd(&txbd_tmp);
745 		txbd_tmp.addr = cpu_to_le64(dma);
746 		txbd_tmp.buf_len = cpu_to_le16(len);
747 
748 		tx_swbd->dma = dma;
749 		tx_swbd->len = len;
750 		tx_swbd->is_dma_page = 1;
751 		tx_swbd->dir = DMA_TO_DEVICE;
752 	}
753 
754 	/* Last BD needs 'F' bit set */
755 	flags |= ENETC_TXBD_FLAGS_F;
756 	txbd_tmp.flags = flags;
757 	*txbd = txbd_tmp;
758 
759 	tx_swbd->is_eof = 1;
760 	tx_swbd->skb = skb;
761 
762 	return 0;
763 }
764 
enetc_lso_hw_offload(struct enetc_bdr * tx_ring,struct sk_buff * skb)765 static int enetc_lso_hw_offload(struct enetc_bdr *tx_ring, struct sk_buff *skb)
766 {
767 	struct enetc_tx_swbd *tx_swbd;
768 	struct enetc_lso_t lso = {0};
769 	int err, i, count = 0;
770 
771 	/* Initialize the LSO handler */
772 	enetc_lso_start(skb, &lso);
773 	i = tx_ring->next_to_use;
774 
775 	enetc_lso_map_hdr(tx_ring, skb, &i, &lso);
776 	/* First BD and an extend BD */
777 	count += 2;
778 
779 	err = enetc_lso_map_data(tx_ring, skb, &i, &lso, &count);
780 	if (err)
781 		goto dma_err;
782 
783 	/* Go to the next BD */
784 	enetc_bdr_idx_inc(tx_ring, &i);
785 	tx_ring->next_to_use = i;
786 	enetc_update_tx_ring_tail(tx_ring);
787 
788 	return count;
789 
790 dma_err:
791 	do {
792 		tx_swbd = &tx_ring->tx_swbd[i];
793 		enetc_free_tx_frame(tx_ring, tx_swbd);
794 		if (i == 0)
795 			i = tx_ring->bd_count;
796 		i--;
797 	} while (--count);
798 
799 	return 0;
800 }
801 
enetc_map_tx_tso_buffs(struct enetc_bdr * tx_ring,struct sk_buff * skb)802 static int enetc_map_tx_tso_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb)
803 {
804 	struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev);
805 	int hdr_len, total_len, data_len;
806 	struct enetc_tx_swbd *tx_swbd;
807 	union enetc_tx_bd *txbd;
808 	struct tso_t tso;
809 	__wsum csum, csum2;
810 	int count = 0, pos;
811 	int err, i, bd_data_num;
812 
813 	/* Initialize the TSO handler, and prepare the first payload */
814 	hdr_len = tso_start(skb, &tso);
815 	total_len = skb->len - hdr_len;
816 	i = tx_ring->next_to_use;
817 
818 	while (total_len > 0) {
819 		char *hdr;
820 
821 		/* Get the BD */
822 		txbd = ENETC_TXBD(*tx_ring, i);
823 		tx_swbd = &tx_ring->tx_swbd[i];
824 		prefetchw(txbd);
825 
826 		/* Determine the length of this packet */
827 		data_len = min_t(int, skb_shinfo(skb)->gso_size, total_len);
828 		total_len -= data_len;
829 
830 		/* prepare packet headers: MAC + IP + TCP */
831 		hdr = tx_ring->tso_headers + i * TSO_HEADER_SIZE;
832 		tso_build_hdr(skb, hdr, &tso, data_len, total_len == 0);
833 
834 		/* compute the csum over the L4 header */
835 		csum = enetc_tso_hdr_csum(&tso, skb, hdr, hdr_len, &pos);
836 		count += enetc_map_tx_tso_hdr(tx_ring, skb, tx_swbd, txbd,
837 					      &i, hdr_len, data_len);
838 		bd_data_num = 0;
839 
840 		while (data_len > 0) {
841 			int size;
842 
843 			size = min_t(int, tso.size, data_len);
844 
845 			/* Advance the index in the BDR */
846 			enetc_bdr_idx_inc(tx_ring, &i);
847 			txbd = ENETC_TXBD(*tx_ring, i);
848 			tx_swbd = &tx_ring->tx_swbd[i];
849 			prefetchw(txbd);
850 
851 			/* Compute the checksum over this segment of data and
852 			 * add it to the csum already computed (over the L4
853 			 * header and possible other data segments).
854 			 */
855 			csum2 = csum_partial(tso.data, size, 0);
856 			csum = csum_block_add(csum, csum2, pos);
857 			pos += size;
858 
859 			err = enetc_map_tx_tso_data(tx_ring, skb, tx_swbd, txbd,
860 						    tso.data, size,
861 						    size == data_len);
862 			if (err) {
863 				if (i == 0)
864 					i = tx_ring->bd_count;
865 				i--;
866 
867 				goto err_map_data;
868 			}
869 
870 			data_len -= size;
871 			count++;
872 			bd_data_num++;
873 			tso_build_data(skb, &tso, size);
874 
875 			if (unlikely(bd_data_num >= priv->max_frags && data_len))
876 				goto err_chained_bd;
877 		}
878 
879 		enetc_tso_complete_csum(tx_ring, &tso, skb, hdr, pos, csum);
880 
881 		if (total_len == 0)
882 			tx_swbd->skb = skb;
883 
884 		/* Go to the next BD */
885 		enetc_bdr_idx_inc(tx_ring, &i);
886 	}
887 
888 	tx_ring->next_to_use = i;
889 	enetc_update_tx_ring_tail(tx_ring);
890 
891 	return count;
892 
893 err_map_data:
894 	dev_err(tx_ring->dev, "DMA map error");
895 
896 err_chained_bd:
897 	enetc_unwind_tx_frame(tx_ring, count, i);
898 
899 	return 0;
900 }
901 
enetc_start_xmit(struct sk_buff * skb,struct net_device * ndev)902 static netdev_tx_t enetc_start_xmit(struct sk_buff *skb,
903 				    struct net_device *ndev)
904 {
905 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
906 	struct enetc_bdr *tx_ring;
907 	int count;
908 
909 	/* Queue one-step Sync packet if already locked */
910 	if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
911 		if (test_and_set_bit_lock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS,
912 					  &priv->flags)) {
913 			skb_queue_tail(&priv->tx_skbs, skb);
914 			return NETDEV_TX_OK;
915 		}
916 	}
917 
918 	tx_ring = priv->tx_ring[skb->queue_mapping];
919 
920 	if (skb_is_gso(skb)) {
921 		/* LSO data unit lengths of up to 256KB are supported */
922 		if (priv->active_offloads & ENETC_F_LSO &&
923 		    (skb->len - enetc_lso_get_hdr_len(skb)) <=
924 		    ENETC_LSO_MAX_DATA_LEN) {
925 			if (enetc_bd_unused(tx_ring) < enetc_lso_count_descs(skb)) {
926 				netif_stop_subqueue(ndev, tx_ring->index);
927 				return NETDEV_TX_BUSY;
928 			}
929 
930 			count = enetc_lso_hw_offload(tx_ring, skb);
931 		} else {
932 			if (enetc_bd_unused(tx_ring) < tso_count_descs(skb)) {
933 				netif_stop_subqueue(ndev, tx_ring->index);
934 				return NETDEV_TX_BUSY;
935 			}
936 
937 			enetc_lock_mdio();
938 			count = enetc_map_tx_tso_buffs(tx_ring, skb);
939 			enetc_unlock_mdio();
940 		}
941 	} else {
942 		if (unlikely(skb_shinfo(skb)->nr_frags > priv->max_frags))
943 			if (unlikely(skb_linearize(skb)))
944 				goto drop_packet_err;
945 
946 		count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */
947 		if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) {
948 			netif_stop_subqueue(ndev, tx_ring->index);
949 			return NETDEV_TX_BUSY;
950 		}
951 
952 		enetc_lock_mdio();
953 		count = enetc_map_tx_buffs(tx_ring, skb);
954 		enetc_unlock_mdio();
955 	}
956 
957 	if (unlikely(!count))
958 		goto drop_packet_err;
959 
960 	if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED(priv->max_frags))
961 		netif_stop_subqueue(ndev, tx_ring->index);
962 
963 	return NETDEV_TX_OK;
964 
965 drop_packet_err:
966 	dev_kfree_skb_any(skb);
967 	return NETDEV_TX_OK;
968 }
969 
enetc_xmit(struct sk_buff * skb,struct net_device * ndev)970 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev)
971 {
972 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
973 	u8 udp, msgtype, twostep;
974 	u16 offset1, offset2;
975 
976 	/* Mark tx timestamp type on skb->cb[0] if requires */
977 	if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
978 	    (priv->active_offloads & ENETC_F_TX_TSTAMP_MASK)) {
979 		skb->cb[0] = priv->active_offloads & ENETC_F_TX_TSTAMP_MASK;
980 	} else {
981 		skb->cb[0] = 0;
982 	}
983 
984 	/* Fall back to two-step timestamp if not one-step Sync packet */
985 	if (skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP) {
986 		if (enetc_ptp_parse(skb, &udp, &msgtype, &twostep,
987 				    &offset1, &offset2) ||
988 		    msgtype != PTP_MSGTYPE_SYNC || twostep != 0)
989 			skb->cb[0] = ENETC_F_TX_TSTAMP;
990 	}
991 
992 	return enetc_start_xmit(skb, ndev);
993 }
994 EXPORT_SYMBOL_GPL(enetc_xmit);
995 
enetc_msix(int irq,void * data)996 static irqreturn_t enetc_msix(int irq, void *data)
997 {
998 	struct enetc_int_vector	*v = data;
999 	int i;
1000 
1001 	enetc_lock_mdio();
1002 
1003 	/* disable interrupts */
1004 	enetc_wr_reg_hot(v->rbier, 0);
1005 	enetc_wr_reg_hot(v->ricr1, v->rx_ictt);
1006 
1007 	for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
1008 		enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i), 0);
1009 
1010 	enetc_unlock_mdio();
1011 
1012 	napi_schedule(&v->napi);
1013 
1014 	return IRQ_HANDLED;
1015 }
1016 
enetc_rx_dim_work(struct work_struct * w)1017 static void enetc_rx_dim_work(struct work_struct *w)
1018 {
1019 	struct dim *dim = container_of(w, struct dim, work);
1020 	struct dim_cq_moder moder =
1021 		net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
1022 	struct enetc_int_vector	*v =
1023 		container_of(dim, struct enetc_int_vector, rx_dim);
1024 	struct enetc_ndev_priv *priv = netdev_priv(v->rx_ring.ndev);
1025 
1026 	v->rx_ictt = enetc_usecs_to_cycles(moder.usec, priv->sysclk_freq);
1027 	dim->state = DIM_START_MEASURE;
1028 }
1029 
enetc_rx_net_dim(struct enetc_int_vector * v)1030 static void enetc_rx_net_dim(struct enetc_int_vector *v)
1031 {
1032 	struct dim_sample dim_sample = {};
1033 
1034 	v->comp_cnt++;
1035 
1036 	if (!v->rx_napi_work)
1037 		return;
1038 
1039 	dim_update_sample(v->comp_cnt,
1040 			  v->rx_ring.stats.packets,
1041 			  v->rx_ring.stats.bytes,
1042 			  &dim_sample);
1043 	net_dim(&v->rx_dim, &dim_sample);
1044 }
1045 
enetc_bd_ready_count(struct enetc_bdr * tx_ring,int ci)1046 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci)
1047 {
1048 	int pi = enetc_rd_reg_hot(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK;
1049 
1050 	return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi;
1051 }
1052 
enetc_page_reusable(struct page * page)1053 static bool enetc_page_reusable(struct page *page)
1054 {
1055 	return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1);
1056 }
1057 
enetc_reuse_page(struct enetc_bdr * rx_ring,struct enetc_rx_swbd * old)1058 static void enetc_reuse_page(struct enetc_bdr *rx_ring,
1059 			     struct enetc_rx_swbd *old)
1060 {
1061 	struct enetc_rx_swbd *new;
1062 
1063 	new = &rx_ring->rx_swbd[rx_ring->next_to_alloc];
1064 
1065 	/* next buf that may reuse a page */
1066 	enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc);
1067 
1068 	/* copy page reference */
1069 	*new = *old;
1070 }
1071 
enetc_get_tx_tstamp(struct enetc_hw * hw,union enetc_tx_bd * txbd,u64 * tstamp)1072 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd,
1073 				u64 *tstamp)
1074 {
1075 	u32 lo, hi, tstamp_lo;
1076 
1077 	lo = enetc_rd_hot(hw, ENETC_SICTR0);
1078 	hi = enetc_rd_hot(hw, ENETC_SICTR1);
1079 	tstamp_lo = le32_to_cpu(txbd->wb.tstamp);
1080 	if (lo <= tstamp_lo)
1081 		hi -= 1;
1082 	*tstamp = (u64)hi << 32 | tstamp_lo;
1083 }
1084 
enetc_tstamp_tx(struct sk_buff * skb,u64 tstamp)1085 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp)
1086 {
1087 	struct skb_shared_hwtstamps shhwtstamps;
1088 
1089 	if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) {
1090 		memset(&shhwtstamps, 0, sizeof(shhwtstamps));
1091 		shhwtstamps.hwtstamp = ns_to_ktime(tstamp);
1092 		skb_txtime_consumed(skb);
1093 		skb_tstamp_tx(skb, &shhwtstamps);
1094 	}
1095 }
1096 
enetc_recycle_xdp_tx_buff(struct enetc_bdr * tx_ring,struct enetc_tx_swbd * tx_swbd)1097 static void enetc_recycle_xdp_tx_buff(struct enetc_bdr *tx_ring,
1098 				      struct enetc_tx_swbd *tx_swbd)
1099 {
1100 	struct enetc_ndev_priv *priv = netdev_priv(tx_ring->ndev);
1101 	struct enetc_rx_swbd rx_swbd = {
1102 		.dma = tx_swbd->dma,
1103 		.page = tx_swbd->page,
1104 		.page_offset = tx_swbd->page_offset,
1105 		.dir = tx_swbd->dir,
1106 		.len = tx_swbd->len,
1107 	};
1108 	struct enetc_bdr *rx_ring;
1109 
1110 	rx_ring = enetc_rx_ring_from_xdp_tx_ring(priv, tx_ring);
1111 
1112 	if (likely(enetc_swbd_unused(rx_ring))) {
1113 		enetc_reuse_page(rx_ring, &rx_swbd);
1114 
1115 		/* sync for use by the device */
1116 		dma_sync_single_range_for_device(rx_ring->dev, rx_swbd.dma,
1117 						 rx_swbd.page_offset,
1118 						 ENETC_RXB_DMA_SIZE_XDP,
1119 						 rx_swbd.dir);
1120 
1121 		rx_ring->stats.recycles++;
1122 	} else {
1123 		/* RX ring is already full, we need to unmap and free the
1124 		 * page, since there's nothing useful we can do with it.
1125 		 */
1126 		rx_ring->stats.recycle_failures++;
1127 
1128 		dma_unmap_page(rx_ring->dev, rx_swbd.dma, PAGE_SIZE,
1129 			       rx_swbd.dir);
1130 		__free_page(rx_swbd.page);
1131 	}
1132 
1133 	rx_ring->xdp.xdp_tx_in_flight--;
1134 }
1135 
enetc_clean_tx_ring(struct enetc_bdr * tx_ring,int napi_budget)1136 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget)
1137 {
1138 	int tx_frm_cnt = 0, tx_byte_cnt = 0, tx_win_drop = 0;
1139 	struct net_device *ndev = tx_ring->ndev;
1140 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
1141 	struct enetc_tx_swbd *tx_swbd;
1142 	int i, bds_to_clean;
1143 	bool do_twostep_tstamp;
1144 	u64 tstamp = 0;
1145 
1146 	i = tx_ring->next_to_clean;
1147 	tx_swbd = &tx_ring->tx_swbd[i];
1148 
1149 	bds_to_clean = enetc_bd_ready_count(tx_ring, i);
1150 
1151 	do_twostep_tstamp = false;
1152 
1153 	while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) {
1154 		struct xdp_frame *xdp_frame = enetc_tx_swbd_get_xdp_frame(tx_swbd);
1155 		struct sk_buff *skb = enetc_tx_swbd_get_skb(tx_swbd);
1156 		bool is_eof = tx_swbd->is_eof;
1157 
1158 		if (unlikely(tx_swbd->check_wb)) {
1159 			union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
1160 
1161 			if (txbd->flags & ENETC_TXBD_FLAGS_W &&
1162 			    tx_swbd->do_twostep_tstamp) {
1163 				enetc_get_tx_tstamp(&priv->si->hw, txbd,
1164 						    &tstamp);
1165 				do_twostep_tstamp = true;
1166 			}
1167 
1168 			if (tx_swbd->qbv_en &&
1169 			    txbd->wb.status & ENETC_TXBD_STATS_WIN)
1170 				tx_win_drop++;
1171 		}
1172 
1173 		if (tx_swbd->is_xdp_tx)
1174 			enetc_recycle_xdp_tx_buff(tx_ring, tx_swbd);
1175 		else if (likely(tx_swbd->dma))
1176 			enetc_unmap_tx_buff(tx_ring, tx_swbd);
1177 
1178 		if (xdp_frame) {
1179 			xdp_return_frame(xdp_frame);
1180 		} else if (skb) {
1181 			if (unlikely(skb->cb[0] & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)) {
1182 				/* Start work to release lock for next one-step
1183 				 * timestamping packet. And send one skb in
1184 				 * tx_skbs queue if has.
1185 				 */
1186 				schedule_work(&priv->tx_onestep_tstamp);
1187 			} else if (unlikely(do_twostep_tstamp)) {
1188 				enetc_tstamp_tx(skb, tstamp);
1189 				do_twostep_tstamp = false;
1190 			}
1191 			napi_consume_skb(skb, napi_budget);
1192 		}
1193 
1194 		tx_byte_cnt += tx_swbd->len;
1195 		/* Scrub the swbd here so we don't have to do that
1196 		 * when we reuse it during xmit
1197 		 */
1198 		memset(tx_swbd, 0, sizeof(*tx_swbd));
1199 
1200 		bds_to_clean--;
1201 		tx_swbd++;
1202 		i++;
1203 		if (unlikely(i == tx_ring->bd_count)) {
1204 			i = 0;
1205 			tx_swbd = tx_ring->tx_swbd;
1206 		}
1207 
1208 		/* BD iteration loop end */
1209 		if (is_eof) {
1210 			tx_frm_cnt++;
1211 			/* re-arm interrupt source */
1212 			enetc_wr_reg_hot(tx_ring->idr, BIT(tx_ring->index) |
1213 					 BIT(16 + tx_ring->index));
1214 		}
1215 
1216 		if (unlikely(!bds_to_clean))
1217 			bds_to_clean = enetc_bd_ready_count(tx_ring, i);
1218 	}
1219 
1220 	tx_ring->next_to_clean = i;
1221 	tx_ring->stats.packets += tx_frm_cnt;
1222 	tx_ring->stats.bytes += tx_byte_cnt;
1223 	tx_ring->stats.win_drop += tx_win_drop;
1224 
1225 	if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) &&
1226 		     __netif_subqueue_stopped(ndev, tx_ring->index) &&
1227 		     !test_bit(ENETC_TX_DOWN, &priv->flags) &&
1228 		     (enetc_bd_unused(tx_ring) >=
1229 		      ENETC_TXBDS_MAX_NEEDED(priv->max_frags)))) {
1230 		netif_wake_subqueue(ndev, tx_ring->index);
1231 	}
1232 
1233 	return tx_frm_cnt != ENETC_DEFAULT_TX_WORK;
1234 }
1235 
enetc_new_page(struct enetc_bdr * rx_ring,struct enetc_rx_swbd * rx_swbd)1236 static bool enetc_new_page(struct enetc_bdr *rx_ring,
1237 			   struct enetc_rx_swbd *rx_swbd)
1238 {
1239 	bool xdp = !!(rx_ring->xdp.prog);
1240 	struct page *page;
1241 	dma_addr_t addr;
1242 
1243 	page = dev_alloc_page();
1244 	if (unlikely(!page))
1245 		return false;
1246 
1247 	/* For XDP_TX, we forgo dma_unmap -> dma_map */
1248 	rx_swbd->dir = xdp ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
1249 
1250 	addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, rx_swbd->dir);
1251 	if (unlikely(dma_mapping_error(rx_ring->dev, addr))) {
1252 		__free_page(page);
1253 
1254 		return false;
1255 	}
1256 
1257 	rx_swbd->dma = addr;
1258 	rx_swbd->page = page;
1259 	rx_swbd->page_offset = rx_ring->buffer_offset;
1260 
1261 	return true;
1262 }
1263 
enetc_refill_rx_ring(struct enetc_bdr * rx_ring,const int buff_cnt)1264 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt)
1265 {
1266 	struct enetc_rx_swbd *rx_swbd;
1267 	union enetc_rx_bd *rxbd;
1268 	int i, j;
1269 
1270 	i = rx_ring->next_to_use;
1271 	rx_swbd = &rx_ring->rx_swbd[i];
1272 	rxbd = enetc_rxbd(rx_ring, i);
1273 
1274 	for (j = 0; j < buff_cnt; j++) {
1275 		/* try reuse page */
1276 		if (unlikely(!rx_swbd->page)) {
1277 			if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) {
1278 				rx_ring->stats.rx_alloc_errs++;
1279 				break;
1280 			}
1281 		}
1282 
1283 		/* update RxBD */
1284 		rxbd->w.addr = cpu_to_le64(rx_swbd->dma +
1285 					   rx_swbd->page_offset);
1286 		/* clear 'R" as well */
1287 		rxbd->r.lstatus = 0;
1288 
1289 		enetc_rxbd_next(rx_ring, &rxbd, &i);
1290 		rx_swbd = &rx_ring->rx_swbd[i];
1291 	}
1292 
1293 	if (likely(j)) {
1294 		rx_ring->next_to_alloc = i; /* keep track from page reuse */
1295 		rx_ring->next_to_use = i;
1296 
1297 		/* update ENETC's consumer index */
1298 		enetc_wr_reg_hot(rx_ring->rcir, rx_ring->next_to_use);
1299 	}
1300 
1301 	return j;
1302 }
1303 
enetc_get_rx_tstamp(struct net_device * ndev,union enetc_rx_bd * rxbd,struct sk_buff * skb)1304 static void enetc_get_rx_tstamp(struct net_device *ndev,
1305 				union enetc_rx_bd *rxbd,
1306 				struct sk_buff *skb)
1307 {
1308 	struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
1309 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
1310 	struct enetc_hw *hw = &priv->si->hw;
1311 	u32 lo, hi, tstamp_lo;
1312 	u64 tstamp;
1313 
1314 	if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) {
1315 		lo = enetc_rd_reg_hot(hw->reg + ENETC_SICTR0);
1316 		hi = enetc_rd_reg_hot(hw->reg + ENETC_SICTR1);
1317 		rxbd = enetc_rxbd_ext(rxbd);
1318 		tstamp_lo = le32_to_cpu(rxbd->ext.tstamp);
1319 		if (lo <= tstamp_lo)
1320 			hi -= 1;
1321 
1322 		tstamp = (u64)hi << 32 | tstamp_lo;
1323 		memset(shhwtstamps, 0, sizeof(*shhwtstamps));
1324 		shhwtstamps->hwtstamp = ns_to_ktime(tstamp);
1325 	}
1326 }
1327 
enetc_get_offloads(struct enetc_bdr * rx_ring,union enetc_rx_bd * rxbd,struct sk_buff * skb)1328 static void enetc_get_offloads(struct enetc_bdr *rx_ring,
1329 			       union enetc_rx_bd *rxbd, struct sk_buff *skb)
1330 {
1331 	struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
1332 
1333 	/* TODO: hashing */
1334 	if (rx_ring->ndev->features & NETIF_F_RXCSUM) {
1335 		u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum);
1336 
1337 		skb->csum = csum_unfold((__force __sum16)~htons(inet_csum));
1338 		skb->ip_summed = CHECKSUM_COMPLETE;
1339 	}
1340 
1341 	if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN) {
1342 		__be16 tpid = 0;
1343 
1344 		switch (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TPID) {
1345 		case 0:
1346 			tpid = htons(ETH_P_8021Q);
1347 			break;
1348 		case 1:
1349 			tpid = htons(ETH_P_8021AD);
1350 			break;
1351 		case 2:
1352 			tpid = htons(enetc_port_rd(&priv->si->hw,
1353 						   ENETC_PCVLANR1));
1354 			break;
1355 		case 3:
1356 			tpid = htons(enetc_port_rd(&priv->si->hw,
1357 						   ENETC_PCVLANR2));
1358 			break;
1359 		default:
1360 			break;
1361 		}
1362 
1363 		__vlan_hwaccel_put_tag(skb, tpid, le16_to_cpu(rxbd->r.vlan_opt));
1364 	}
1365 
1366 	if (IS_ENABLED(CONFIG_FSL_ENETC_PTP_CLOCK) &&
1367 	    (priv->active_offloads & ENETC_F_RX_TSTAMP))
1368 		enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb);
1369 }
1370 
1371 /* This gets called during the non-XDP NAPI poll cycle as well as on XDP_PASS,
1372  * so it needs to work with both DMA_FROM_DEVICE as well as DMA_BIDIRECTIONAL
1373  * mapped buffers.
1374  */
enetc_get_rx_buff(struct enetc_bdr * rx_ring,int i,u16 size)1375 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring,
1376 					       int i, u16 size)
1377 {
1378 	struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
1379 
1380 	dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma,
1381 				      rx_swbd->page_offset,
1382 				      size, rx_swbd->dir);
1383 	return rx_swbd;
1384 }
1385 
1386 /* Reuse the current page without performing half-page buffer flipping */
enetc_put_rx_buff(struct enetc_bdr * rx_ring,struct enetc_rx_swbd * rx_swbd)1387 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring,
1388 			      struct enetc_rx_swbd *rx_swbd)
1389 {
1390 	size_t buffer_size = ENETC_RXB_TRUESIZE - rx_ring->buffer_offset;
1391 
1392 	enetc_reuse_page(rx_ring, rx_swbd);
1393 
1394 	dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma,
1395 					 rx_swbd->page_offset,
1396 					 buffer_size, rx_swbd->dir);
1397 
1398 	rx_swbd->page = NULL;
1399 }
1400 
1401 /* Reuse the current page by performing half-page buffer flipping */
enetc_flip_rx_buff(struct enetc_bdr * rx_ring,struct enetc_rx_swbd * rx_swbd)1402 static void enetc_flip_rx_buff(struct enetc_bdr *rx_ring,
1403 			       struct enetc_rx_swbd *rx_swbd)
1404 {
1405 	if (likely(enetc_page_reusable(rx_swbd->page))) {
1406 		rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE;
1407 		page_ref_inc(rx_swbd->page);
1408 
1409 		enetc_put_rx_buff(rx_ring, rx_swbd);
1410 	} else {
1411 		dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE,
1412 			       rx_swbd->dir);
1413 		rx_swbd->page = NULL;
1414 	}
1415 }
1416 
enetc_map_rx_buff_to_skb(struct enetc_bdr * rx_ring,int i,u16 size)1417 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring,
1418 						int i, u16 size)
1419 {
1420 	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1421 	struct sk_buff *skb;
1422 	void *ba;
1423 
1424 	ba = page_address(rx_swbd->page) + rx_swbd->page_offset;
1425 	skb = build_skb(ba - rx_ring->buffer_offset, ENETC_RXB_TRUESIZE);
1426 	if (unlikely(!skb)) {
1427 		rx_ring->stats.rx_alloc_errs++;
1428 		return NULL;
1429 	}
1430 
1431 	skb_reserve(skb, rx_ring->buffer_offset);
1432 	__skb_put(skb, size);
1433 
1434 	enetc_flip_rx_buff(rx_ring, rx_swbd);
1435 
1436 	return skb;
1437 }
1438 
enetc_add_rx_buff_to_skb(struct enetc_bdr * rx_ring,int i,u16 size,struct sk_buff * skb)1439 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i,
1440 				     u16 size, struct sk_buff *skb)
1441 {
1442 	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1443 
1444 	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page,
1445 			rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE);
1446 
1447 	enetc_flip_rx_buff(rx_ring, rx_swbd);
1448 }
1449 
enetc_check_bd_errors_and_consume(struct enetc_bdr * rx_ring,u32 bd_status,union enetc_rx_bd ** rxbd,int * i)1450 static bool enetc_check_bd_errors_and_consume(struct enetc_bdr *rx_ring,
1451 					      u32 bd_status,
1452 					      union enetc_rx_bd **rxbd, int *i)
1453 {
1454 	if (likely(!(bd_status & ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK))))
1455 		return false;
1456 
1457 	enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]);
1458 	enetc_rxbd_next(rx_ring, rxbd, i);
1459 
1460 	while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
1461 		dma_rmb();
1462 		bd_status = le32_to_cpu((*rxbd)->r.lstatus);
1463 
1464 		enetc_put_rx_buff(rx_ring, &rx_ring->rx_swbd[*i]);
1465 		enetc_rxbd_next(rx_ring, rxbd, i);
1466 	}
1467 
1468 	rx_ring->ndev->stats.rx_dropped++;
1469 	rx_ring->ndev->stats.rx_errors++;
1470 
1471 	return true;
1472 }
1473 
enetc_build_skb(struct enetc_bdr * rx_ring,u32 bd_status,union enetc_rx_bd ** rxbd,int * i,int * cleaned_cnt,int buffer_size)1474 static struct sk_buff *enetc_build_skb(struct enetc_bdr *rx_ring,
1475 				       u32 bd_status, union enetc_rx_bd **rxbd,
1476 				       int *i, int *cleaned_cnt, int buffer_size)
1477 {
1478 	struct sk_buff *skb;
1479 	u16 size;
1480 
1481 	size = le16_to_cpu((*rxbd)->r.buf_len);
1482 	skb = enetc_map_rx_buff_to_skb(rx_ring, *i, size);
1483 	if (!skb)
1484 		return NULL;
1485 
1486 	enetc_get_offloads(rx_ring, *rxbd, skb);
1487 
1488 	(*cleaned_cnt)++;
1489 
1490 	enetc_rxbd_next(rx_ring, rxbd, i);
1491 
1492 	/* not last BD in frame? */
1493 	while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
1494 		bd_status = le32_to_cpu((*rxbd)->r.lstatus);
1495 		size = buffer_size;
1496 
1497 		if (bd_status & ENETC_RXBD_LSTATUS_F) {
1498 			dma_rmb();
1499 			size = le16_to_cpu((*rxbd)->r.buf_len);
1500 		}
1501 
1502 		enetc_add_rx_buff_to_skb(rx_ring, *i, size, skb);
1503 
1504 		(*cleaned_cnt)++;
1505 
1506 		enetc_rxbd_next(rx_ring, rxbd, i);
1507 	}
1508 
1509 	skb_record_rx_queue(skb, rx_ring->index);
1510 	skb->protocol = eth_type_trans(skb, rx_ring->ndev);
1511 
1512 	return skb;
1513 }
1514 
1515 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */
1516 
enetc_clean_rx_ring(struct enetc_bdr * rx_ring,struct napi_struct * napi,int work_limit)1517 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring,
1518 			       struct napi_struct *napi, int work_limit)
1519 {
1520 	int rx_frm_cnt = 0, rx_byte_cnt = 0;
1521 	int cleaned_cnt, i;
1522 
1523 	cleaned_cnt = enetc_bd_unused(rx_ring);
1524 	/* next descriptor to process */
1525 	i = rx_ring->next_to_clean;
1526 
1527 	while (likely(rx_frm_cnt < work_limit)) {
1528 		union enetc_rx_bd *rxbd;
1529 		struct sk_buff *skb;
1530 		u32 bd_status;
1531 
1532 		if (cleaned_cnt >= ENETC_RXBD_BUNDLE)
1533 			cleaned_cnt -= enetc_refill_rx_ring(rx_ring,
1534 							    cleaned_cnt);
1535 
1536 		rxbd = enetc_rxbd(rx_ring, i);
1537 		bd_status = le32_to_cpu(rxbd->r.lstatus);
1538 		if (!bd_status)
1539 			break;
1540 
1541 		enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index));
1542 		dma_rmb(); /* for reading other rxbd fields */
1543 
1544 		if (enetc_check_bd_errors_and_consume(rx_ring, bd_status,
1545 						      &rxbd, &i))
1546 			break;
1547 
1548 		skb = enetc_build_skb(rx_ring, bd_status, &rxbd, &i,
1549 				      &cleaned_cnt, ENETC_RXB_DMA_SIZE);
1550 		if (!skb)
1551 			break;
1552 
1553 		/* When set, the outer VLAN header is extracted and reported
1554 		 * in the receive buffer descriptor. So rx_byte_cnt should
1555 		 * add the length of the extracted VLAN header.
1556 		 */
1557 		if (bd_status & ENETC_RXBD_FLAG_VLAN)
1558 			rx_byte_cnt += VLAN_HLEN;
1559 		rx_byte_cnt += skb->len + ETH_HLEN;
1560 		rx_frm_cnt++;
1561 
1562 		napi_gro_receive(napi, skb);
1563 	}
1564 
1565 	rx_ring->next_to_clean = i;
1566 
1567 	rx_ring->stats.packets += rx_frm_cnt;
1568 	rx_ring->stats.bytes += rx_byte_cnt;
1569 
1570 	return rx_frm_cnt;
1571 }
1572 
enetc_xdp_map_tx_buff(struct enetc_bdr * tx_ring,int i,struct enetc_tx_swbd * tx_swbd,int frm_len)1573 static void enetc_xdp_map_tx_buff(struct enetc_bdr *tx_ring, int i,
1574 				  struct enetc_tx_swbd *tx_swbd,
1575 				  int frm_len)
1576 {
1577 	union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
1578 
1579 	prefetchw(txbd);
1580 
1581 	enetc_clear_tx_bd(txbd);
1582 	txbd->addr = cpu_to_le64(tx_swbd->dma + tx_swbd->page_offset);
1583 	txbd->buf_len = cpu_to_le16(tx_swbd->len);
1584 	txbd->frm_len = cpu_to_le16(frm_len);
1585 
1586 	memcpy(&tx_ring->tx_swbd[i], tx_swbd, sizeof(*tx_swbd));
1587 }
1588 
1589 /* Puts in the TX ring one XDP frame, mapped as an array of TX software buffer
1590  * descriptors.
1591  */
enetc_xdp_tx(struct enetc_bdr * tx_ring,struct enetc_tx_swbd * xdp_tx_arr,int num_tx_swbd)1592 static bool enetc_xdp_tx(struct enetc_bdr *tx_ring,
1593 			 struct enetc_tx_swbd *xdp_tx_arr, int num_tx_swbd)
1594 {
1595 	struct enetc_tx_swbd *tmp_tx_swbd = xdp_tx_arr;
1596 	int i, k, frm_len = tmp_tx_swbd->len;
1597 
1598 	if (unlikely(enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(num_tx_swbd)))
1599 		return false;
1600 
1601 	while (unlikely(!tmp_tx_swbd->is_eof)) {
1602 		tmp_tx_swbd++;
1603 		frm_len += tmp_tx_swbd->len;
1604 	}
1605 
1606 	i = tx_ring->next_to_use;
1607 
1608 	for (k = 0; k < num_tx_swbd; k++) {
1609 		struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[k];
1610 
1611 		enetc_xdp_map_tx_buff(tx_ring, i, xdp_tx_swbd, frm_len);
1612 
1613 		/* last BD needs 'F' bit set */
1614 		if (xdp_tx_swbd->is_eof) {
1615 			union enetc_tx_bd *txbd = ENETC_TXBD(*tx_ring, i);
1616 
1617 			txbd->flags = ENETC_TXBD_FLAGS_F;
1618 		}
1619 
1620 		enetc_bdr_idx_inc(tx_ring, &i);
1621 	}
1622 
1623 	tx_ring->next_to_use = i;
1624 
1625 	return true;
1626 }
1627 
enetc_xdp_frame_to_xdp_tx_swbd(struct enetc_bdr * tx_ring,struct enetc_tx_swbd * xdp_tx_arr,struct xdp_frame * xdp_frame)1628 static int enetc_xdp_frame_to_xdp_tx_swbd(struct enetc_bdr *tx_ring,
1629 					  struct enetc_tx_swbd *xdp_tx_arr,
1630 					  struct xdp_frame *xdp_frame)
1631 {
1632 	struct enetc_tx_swbd *xdp_tx_swbd = &xdp_tx_arr[0];
1633 	struct skb_shared_info *shinfo;
1634 	void *data = xdp_frame->data;
1635 	int len = xdp_frame->len;
1636 	skb_frag_t *frag;
1637 	dma_addr_t dma;
1638 	unsigned int f;
1639 	int n = 0;
1640 
1641 	dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE);
1642 	if (unlikely(dma_mapping_error(tx_ring->dev, dma))) {
1643 		netdev_err(tx_ring->ndev, "DMA map error\n");
1644 		return -1;
1645 	}
1646 
1647 	xdp_tx_swbd->dma = dma;
1648 	xdp_tx_swbd->dir = DMA_TO_DEVICE;
1649 	xdp_tx_swbd->len = len;
1650 	xdp_tx_swbd->is_xdp_redirect = true;
1651 	xdp_tx_swbd->is_eof = false;
1652 	xdp_tx_swbd->xdp_frame = NULL;
1653 
1654 	n++;
1655 
1656 	if (!xdp_frame_has_frags(xdp_frame))
1657 		goto out;
1658 
1659 	xdp_tx_swbd = &xdp_tx_arr[n];
1660 
1661 	shinfo = xdp_get_shared_info_from_frame(xdp_frame);
1662 
1663 	for (f = 0, frag = &shinfo->frags[0]; f < shinfo->nr_frags;
1664 	     f++, frag++) {
1665 		data = skb_frag_address(frag);
1666 		len = skb_frag_size(frag);
1667 
1668 		dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE);
1669 		if (unlikely(dma_mapping_error(tx_ring->dev, dma))) {
1670 			/* Undo the DMA mapping for all fragments */
1671 			while (--n >= 0)
1672 				enetc_unmap_tx_buff(tx_ring, &xdp_tx_arr[n]);
1673 
1674 			netdev_err(tx_ring->ndev, "DMA map error\n");
1675 			return -1;
1676 		}
1677 
1678 		xdp_tx_swbd->dma = dma;
1679 		xdp_tx_swbd->dir = DMA_TO_DEVICE;
1680 		xdp_tx_swbd->len = len;
1681 		xdp_tx_swbd->is_xdp_redirect = true;
1682 		xdp_tx_swbd->is_eof = false;
1683 		xdp_tx_swbd->xdp_frame = NULL;
1684 
1685 		n++;
1686 		xdp_tx_swbd = &xdp_tx_arr[n];
1687 	}
1688 out:
1689 	xdp_tx_arr[n - 1].is_eof = true;
1690 	xdp_tx_arr[n - 1].xdp_frame = xdp_frame;
1691 
1692 	return n;
1693 }
1694 
enetc_xdp_xmit(struct net_device * ndev,int num_frames,struct xdp_frame ** frames,u32 flags)1695 int enetc_xdp_xmit(struct net_device *ndev, int num_frames,
1696 		   struct xdp_frame **frames, u32 flags)
1697 {
1698 	struct enetc_tx_swbd xdp_redirect_arr[ENETC_MAX_SKB_FRAGS] = {0};
1699 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
1700 	struct enetc_bdr *tx_ring;
1701 	int xdp_tx_bd_cnt, i, k;
1702 	int xdp_tx_frm_cnt = 0;
1703 
1704 	if (unlikely(test_bit(ENETC_TX_DOWN, &priv->flags)))
1705 		return -ENETDOWN;
1706 
1707 	enetc_lock_mdio();
1708 
1709 	tx_ring = priv->xdp_tx_ring[smp_processor_id()];
1710 
1711 	prefetchw(ENETC_TXBD(*tx_ring, tx_ring->next_to_use));
1712 
1713 	for (k = 0; k < num_frames; k++) {
1714 		xdp_tx_bd_cnt = enetc_xdp_frame_to_xdp_tx_swbd(tx_ring,
1715 							       xdp_redirect_arr,
1716 							       frames[k]);
1717 		if (unlikely(xdp_tx_bd_cnt < 0))
1718 			break;
1719 
1720 		if (unlikely(!enetc_xdp_tx(tx_ring, xdp_redirect_arr,
1721 					   xdp_tx_bd_cnt))) {
1722 			for (i = 0; i < xdp_tx_bd_cnt; i++)
1723 				enetc_unmap_tx_buff(tx_ring,
1724 						    &xdp_redirect_arr[i]);
1725 			tx_ring->stats.xdp_tx_drops++;
1726 			break;
1727 		}
1728 
1729 		xdp_tx_frm_cnt++;
1730 	}
1731 
1732 	if (unlikely((flags & XDP_XMIT_FLUSH) || k != xdp_tx_frm_cnt))
1733 		enetc_update_tx_ring_tail(tx_ring);
1734 
1735 	tx_ring->stats.xdp_tx += xdp_tx_frm_cnt;
1736 
1737 	enetc_unlock_mdio();
1738 
1739 	return xdp_tx_frm_cnt;
1740 }
1741 EXPORT_SYMBOL_GPL(enetc_xdp_xmit);
1742 
enetc_map_rx_buff_to_xdp(struct enetc_bdr * rx_ring,int i,struct xdp_buff * xdp_buff,u16 size)1743 static void enetc_map_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i,
1744 				     struct xdp_buff *xdp_buff, u16 size)
1745 {
1746 	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1747 	void *hard_start = page_address(rx_swbd->page) + rx_swbd->page_offset;
1748 
1749 	/* To be used for XDP_TX */
1750 	rx_swbd->len = size;
1751 
1752 	xdp_prepare_buff(xdp_buff, hard_start - rx_ring->buffer_offset,
1753 			 rx_ring->buffer_offset, size, false);
1754 }
1755 
enetc_add_rx_buff_to_xdp(struct enetc_bdr * rx_ring,int i,u16 size,struct xdp_buff * xdp_buff)1756 static void enetc_add_rx_buff_to_xdp(struct enetc_bdr *rx_ring, int i,
1757 				     u16 size, struct xdp_buff *xdp_buff)
1758 {
1759 	struct skb_shared_info *shinfo = xdp_get_shared_info_from_buff(xdp_buff);
1760 	struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
1761 	skb_frag_t *frag;
1762 
1763 	/* To be used for XDP_TX */
1764 	rx_swbd->len = size;
1765 
1766 	if (!xdp_buff_has_frags(xdp_buff)) {
1767 		xdp_buff_set_frags_flag(xdp_buff);
1768 		shinfo->xdp_frags_size = size;
1769 		shinfo->nr_frags = 0;
1770 	} else {
1771 		shinfo->xdp_frags_size += size;
1772 	}
1773 
1774 	if (page_is_pfmemalloc(rx_swbd->page))
1775 		xdp_buff_set_frag_pfmemalloc(xdp_buff);
1776 
1777 	frag = &shinfo->frags[shinfo->nr_frags];
1778 	skb_frag_fill_page_desc(frag, rx_swbd->page, rx_swbd->page_offset,
1779 				size);
1780 
1781 	shinfo->nr_frags++;
1782 }
1783 
enetc_build_xdp_buff(struct enetc_bdr * rx_ring,u32 bd_status,union enetc_rx_bd ** rxbd,int * i,int * cleaned_cnt,struct xdp_buff * xdp_buff)1784 static void enetc_build_xdp_buff(struct enetc_bdr *rx_ring, u32 bd_status,
1785 				 union enetc_rx_bd **rxbd, int *i,
1786 				 int *cleaned_cnt, struct xdp_buff *xdp_buff)
1787 {
1788 	u16 size = le16_to_cpu((*rxbd)->r.buf_len);
1789 
1790 	xdp_init_buff(xdp_buff, ENETC_RXB_TRUESIZE, &rx_ring->xdp.rxq);
1791 
1792 	enetc_map_rx_buff_to_xdp(rx_ring, *i, xdp_buff, size);
1793 	(*cleaned_cnt)++;
1794 	enetc_rxbd_next(rx_ring, rxbd, i);
1795 
1796 	/* not last BD in frame? */
1797 	while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
1798 		bd_status = le32_to_cpu((*rxbd)->r.lstatus);
1799 		size = ENETC_RXB_DMA_SIZE_XDP;
1800 
1801 		if (bd_status & ENETC_RXBD_LSTATUS_F) {
1802 			dma_rmb();
1803 			size = le16_to_cpu((*rxbd)->r.buf_len);
1804 		}
1805 
1806 		enetc_add_rx_buff_to_xdp(rx_ring, *i, size, xdp_buff);
1807 		(*cleaned_cnt)++;
1808 		enetc_rxbd_next(rx_ring, rxbd, i);
1809 	}
1810 }
1811 
1812 /* Convert RX buffer descriptors to TX buffer descriptors. These will be
1813  * recycled back into the RX ring in enetc_clean_tx_ring.
1814  */
enetc_rx_swbd_to_xdp_tx_swbd(struct enetc_tx_swbd * xdp_tx_arr,struct enetc_bdr * rx_ring,int rx_ring_first,int rx_ring_last)1815 static int enetc_rx_swbd_to_xdp_tx_swbd(struct enetc_tx_swbd *xdp_tx_arr,
1816 					struct enetc_bdr *rx_ring,
1817 					int rx_ring_first, int rx_ring_last)
1818 {
1819 	int n = 0;
1820 
1821 	for (; rx_ring_first != rx_ring_last;
1822 	     n++, enetc_bdr_idx_inc(rx_ring, &rx_ring_first)) {
1823 		struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[rx_ring_first];
1824 		struct enetc_tx_swbd *tx_swbd = &xdp_tx_arr[n];
1825 
1826 		/* No need to dma_map, we already have DMA_BIDIRECTIONAL */
1827 		tx_swbd->dma = rx_swbd->dma;
1828 		tx_swbd->dir = rx_swbd->dir;
1829 		tx_swbd->page = rx_swbd->page;
1830 		tx_swbd->page_offset = rx_swbd->page_offset;
1831 		tx_swbd->len = rx_swbd->len;
1832 		tx_swbd->is_dma_page = true;
1833 		tx_swbd->is_xdp_tx = true;
1834 		tx_swbd->is_eof = false;
1835 	}
1836 
1837 	/* We rely on caller providing an rx_ring_last > rx_ring_first */
1838 	xdp_tx_arr[n - 1].is_eof = true;
1839 
1840 	return n;
1841 }
1842 
enetc_xdp_drop(struct enetc_bdr * rx_ring,int rx_ring_first,int rx_ring_last)1843 static void enetc_xdp_drop(struct enetc_bdr *rx_ring, int rx_ring_first,
1844 			   int rx_ring_last)
1845 {
1846 	while (rx_ring_first != rx_ring_last) {
1847 		enetc_put_rx_buff(rx_ring,
1848 				  &rx_ring->rx_swbd[rx_ring_first]);
1849 		enetc_bdr_idx_inc(rx_ring, &rx_ring_first);
1850 	}
1851 }
1852 
enetc_bulk_flip_buff(struct enetc_bdr * rx_ring,int rx_ring_first,int rx_ring_last)1853 static void enetc_bulk_flip_buff(struct enetc_bdr *rx_ring, int rx_ring_first,
1854 				 int rx_ring_last)
1855 {
1856 	while (rx_ring_first != rx_ring_last) {
1857 		enetc_flip_rx_buff(rx_ring,
1858 				   &rx_ring->rx_swbd[rx_ring_first]);
1859 		enetc_bdr_idx_inc(rx_ring, &rx_ring_first);
1860 	}
1861 }
1862 
enetc_clean_rx_ring_xdp(struct enetc_bdr * rx_ring,struct napi_struct * napi,int work_limit,struct bpf_prog * prog)1863 static int enetc_clean_rx_ring_xdp(struct enetc_bdr *rx_ring,
1864 				   struct napi_struct *napi, int work_limit,
1865 				   struct bpf_prog *prog)
1866 {
1867 	int xdp_tx_bd_cnt, xdp_tx_frm_cnt = 0, xdp_redirect_frm_cnt = 0;
1868 	struct enetc_tx_swbd xdp_tx_arr[ENETC_MAX_SKB_FRAGS] = {0};
1869 	struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
1870 	int rx_frm_cnt = 0, rx_byte_cnt = 0;
1871 	struct enetc_bdr *tx_ring;
1872 	int cleaned_cnt, i;
1873 	u32 xdp_act;
1874 
1875 	cleaned_cnt = enetc_bd_unused(rx_ring);
1876 	/* next descriptor to process */
1877 	i = rx_ring->next_to_clean;
1878 
1879 	while (likely(rx_frm_cnt < work_limit)) {
1880 		union enetc_rx_bd *rxbd, *orig_rxbd;
1881 		struct xdp_buff xdp_buff;
1882 		struct sk_buff *skb;
1883 		int orig_i, err;
1884 		u32 bd_status;
1885 
1886 		rxbd = enetc_rxbd(rx_ring, i);
1887 		bd_status = le32_to_cpu(rxbd->r.lstatus);
1888 		if (!bd_status)
1889 			break;
1890 
1891 		enetc_wr_reg_hot(rx_ring->idr, BIT(rx_ring->index));
1892 		dma_rmb(); /* for reading other rxbd fields */
1893 
1894 		if (enetc_check_bd_errors_and_consume(rx_ring, bd_status,
1895 						      &rxbd, &i))
1896 			break;
1897 
1898 		orig_rxbd = rxbd;
1899 		orig_i = i;
1900 
1901 		enetc_build_xdp_buff(rx_ring, bd_status, &rxbd, &i,
1902 				     &cleaned_cnt, &xdp_buff);
1903 
1904 		/* When set, the outer VLAN header is extracted and reported
1905 		 * in the receive buffer descriptor. So rx_byte_cnt should
1906 		 * add the length of the extracted VLAN header.
1907 		 */
1908 		if (bd_status & ENETC_RXBD_FLAG_VLAN)
1909 			rx_byte_cnt += VLAN_HLEN;
1910 		rx_byte_cnt += xdp_get_buff_len(&xdp_buff);
1911 
1912 		xdp_act = bpf_prog_run_xdp(prog, &xdp_buff);
1913 
1914 		switch (xdp_act) {
1915 		default:
1916 			bpf_warn_invalid_xdp_action(rx_ring->ndev, prog, xdp_act);
1917 			fallthrough;
1918 		case XDP_ABORTED:
1919 			trace_xdp_exception(rx_ring->ndev, prog, xdp_act);
1920 			fallthrough;
1921 		case XDP_DROP:
1922 			enetc_xdp_drop(rx_ring, orig_i, i);
1923 			rx_ring->stats.xdp_drops++;
1924 			break;
1925 		case XDP_PASS:
1926 			skb = xdp_build_skb_from_buff(&xdp_buff);
1927 			/* Probably under memory pressure, stop NAPI */
1928 			if (unlikely(!skb)) {
1929 				enetc_xdp_drop(rx_ring, orig_i, i);
1930 				rx_ring->stats.xdp_drops++;
1931 				goto out;
1932 			}
1933 
1934 			enetc_get_offloads(rx_ring, orig_rxbd, skb);
1935 
1936 			/* These buffers are about to be owned by the stack.
1937 			 * Update our buffer cache (the rx_swbd array elements)
1938 			 * with their other page halves.
1939 			 */
1940 			enetc_bulk_flip_buff(rx_ring, orig_i, i);
1941 
1942 			napi_gro_receive(napi, skb);
1943 			break;
1944 		case XDP_TX:
1945 			tx_ring = priv->xdp_tx_ring[rx_ring->index];
1946 			if (unlikely(test_bit(ENETC_TX_DOWN, &priv->flags))) {
1947 				enetc_xdp_drop(rx_ring, orig_i, i);
1948 				tx_ring->stats.xdp_tx_drops++;
1949 				break;
1950 			}
1951 
1952 			xdp_tx_bd_cnt = enetc_rx_swbd_to_xdp_tx_swbd(xdp_tx_arr,
1953 								     rx_ring,
1954 								     orig_i, i);
1955 
1956 			if (!enetc_xdp_tx(tx_ring, xdp_tx_arr, xdp_tx_bd_cnt)) {
1957 				enetc_xdp_drop(rx_ring, orig_i, i);
1958 				tx_ring->stats.xdp_tx_drops++;
1959 			} else {
1960 				tx_ring->stats.xdp_tx++;
1961 				rx_ring->xdp.xdp_tx_in_flight += xdp_tx_bd_cnt;
1962 				xdp_tx_frm_cnt++;
1963 				/* The XDP_TX enqueue was successful, so we
1964 				 * need to scrub the RX software BDs because
1965 				 * the ownership of the buffers no longer
1966 				 * belongs to the RX ring, and we must prevent
1967 				 * enetc_refill_rx_ring() from reusing
1968 				 * rx_swbd->page.
1969 				 */
1970 				while (orig_i != i) {
1971 					rx_ring->rx_swbd[orig_i].page = NULL;
1972 					enetc_bdr_idx_inc(rx_ring, &orig_i);
1973 				}
1974 			}
1975 			break;
1976 		case XDP_REDIRECT:
1977 			err = xdp_do_redirect(rx_ring->ndev, &xdp_buff, prog);
1978 			if (unlikely(err)) {
1979 				enetc_xdp_drop(rx_ring, orig_i, i);
1980 				rx_ring->stats.xdp_redirect_failures++;
1981 			} else {
1982 				enetc_bulk_flip_buff(rx_ring, orig_i, i);
1983 				xdp_redirect_frm_cnt++;
1984 				rx_ring->stats.xdp_redirect++;
1985 			}
1986 		}
1987 
1988 		rx_frm_cnt++;
1989 	}
1990 
1991 out:
1992 	rx_ring->next_to_clean = i;
1993 
1994 	rx_ring->stats.packets += rx_frm_cnt;
1995 	rx_ring->stats.bytes += rx_byte_cnt;
1996 
1997 	if (xdp_redirect_frm_cnt)
1998 		xdp_do_flush();
1999 
2000 	if (xdp_tx_frm_cnt)
2001 		enetc_update_tx_ring_tail(tx_ring);
2002 
2003 	if (cleaned_cnt > rx_ring->xdp.xdp_tx_in_flight)
2004 		enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring) -
2005 				     rx_ring->xdp.xdp_tx_in_flight);
2006 
2007 	return rx_frm_cnt;
2008 }
2009 
enetc_poll(struct napi_struct * napi,int budget)2010 static int enetc_poll(struct napi_struct *napi, int budget)
2011 {
2012 	struct enetc_int_vector
2013 		*v = container_of(napi, struct enetc_int_vector, napi);
2014 	struct enetc_bdr *rx_ring = &v->rx_ring;
2015 	struct bpf_prog *prog;
2016 	bool complete = true;
2017 	int work_done;
2018 	int i;
2019 
2020 	enetc_lock_mdio();
2021 
2022 	for (i = 0; i < v->count_tx_rings; i++)
2023 		if (!enetc_clean_tx_ring(&v->tx_ring[i], budget))
2024 			complete = false;
2025 
2026 	prog = rx_ring->xdp.prog;
2027 	if (prog)
2028 		work_done = enetc_clean_rx_ring_xdp(rx_ring, napi, budget, prog);
2029 	else
2030 		work_done = enetc_clean_rx_ring(rx_ring, napi, budget);
2031 	if (work_done == budget)
2032 		complete = false;
2033 	if (work_done)
2034 		v->rx_napi_work = true;
2035 
2036 	if (!complete) {
2037 		enetc_unlock_mdio();
2038 		return budget;
2039 	}
2040 
2041 	napi_complete_done(napi, work_done);
2042 
2043 	if (likely(v->rx_dim_en))
2044 		enetc_rx_net_dim(v);
2045 
2046 	v->rx_napi_work = false;
2047 
2048 	/* enable interrupts */
2049 	enetc_wr_reg_hot(v->rbier, ENETC_RBIER_RXTIE);
2050 
2051 	for_each_set_bit(i, &v->tx_rings_map, ENETC_MAX_NUM_TXQS)
2052 		enetc_wr_reg_hot(v->tbier_base + ENETC_BDR_OFF(i),
2053 				 ENETC_TBIER_TXTIE);
2054 
2055 	enetc_unlock_mdio();
2056 
2057 	return work_done;
2058 }
2059 
2060 /* Probing and Init */
2061 #define ENETC_MAX_RFS_SIZE 64
enetc_get_si_caps(struct enetc_si * si)2062 void enetc_get_si_caps(struct enetc_si *si)
2063 {
2064 	struct enetc_hw *hw = &si->hw;
2065 	u32 val;
2066 
2067 	/* find out how many of various resources we have to work with */
2068 	val = enetc_rd(hw, ENETC_SICAPR0);
2069 	si->num_rx_rings = (val >> 16) & 0xff;
2070 	si->num_tx_rings = val & 0xff;
2071 
2072 	val = enetc_rd(hw, ENETC_SIPCAPR0);
2073 	if (val & ENETC_SIPCAPR0_RFS) {
2074 		val = enetc_rd(hw, ENETC_SIRFSCAPR);
2075 		si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val);
2076 		si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE);
2077 	} else {
2078 		/* ENETC which not supports RFS */
2079 		si->num_fs_entries = 0;
2080 	}
2081 
2082 	si->num_rss = 0;
2083 	val = enetc_rd(hw, ENETC_SIPCAPR0);
2084 	if (val & ENETC_SIPCAPR0_RSS) {
2085 		u32 rss;
2086 
2087 		rss = enetc_rd(hw, ENETC_SIRSSCAPR);
2088 		si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(rss);
2089 	}
2090 
2091 	if (val & ENETC_SIPCAPR0_LSO)
2092 		si->hw_features |= ENETC_SI_F_LSO;
2093 }
2094 EXPORT_SYMBOL_GPL(enetc_get_si_caps);
2095 
enetc_dma_alloc_bdr(struct enetc_bdr_resource * res)2096 static int enetc_dma_alloc_bdr(struct enetc_bdr_resource *res)
2097 {
2098 	size_t bd_base_size = res->bd_count * res->bd_size;
2099 
2100 	res->bd_base = dma_alloc_coherent(res->dev, bd_base_size,
2101 					  &res->bd_dma_base, GFP_KERNEL);
2102 	if (!res->bd_base)
2103 		return -ENOMEM;
2104 
2105 	/* h/w requires 128B alignment */
2106 	if (!IS_ALIGNED(res->bd_dma_base, 128)) {
2107 		dma_free_coherent(res->dev, bd_base_size, res->bd_base,
2108 				  res->bd_dma_base);
2109 		return -EINVAL;
2110 	}
2111 
2112 	return 0;
2113 }
2114 
enetc_dma_free_bdr(const struct enetc_bdr_resource * res)2115 static void enetc_dma_free_bdr(const struct enetc_bdr_resource *res)
2116 {
2117 	size_t bd_base_size = res->bd_count * res->bd_size;
2118 
2119 	dma_free_coherent(res->dev, bd_base_size, res->bd_base,
2120 			  res->bd_dma_base);
2121 }
2122 
enetc_alloc_tx_resource(struct enetc_bdr_resource * res,struct device * dev,size_t bd_count)2123 static int enetc_alloc_tx_resource(struct enetc_bdr_resource *res,
2124 				   struct device *dev, size_t bd_count)
2125 {
2126 	int err;
2127 
2128 	res->dev = dev;
2129 	res->bd_count = bd_count;
2130 	res->bd_size = sizeof(union enetc_tx_bd);
2131 
2132 	res->tx_swbd = vcalloc(bd_count, sizeof(*res->tx_swbd));
2133 	if (!res->tx_swbd)
2134 		return -ENOMEM;
2135 
2136 	err = enetc_dma_alloc_bdr(res);
2137 	if (err)
2138 		goto err_alloc_bdr;
2139 
2140 	res->tso_headers = dma_alloc_coherent(dev, bd_count * TSO_HEADER_SIZE,
2141 					      &res->tso_headers_dma,
2142 					      GFP_KERNEL);
2143 	if (!res->tso_headers) {
2144 		err = -ENOMEM;
2145 		goto err_alloc_tso;
2146 	}
2147 
2148 	return 0;
2149 
2150 err_alloc_tso:
2151 	enetc_dma_free_bdr(res);
2152 err_alloc_bdr:
2153 	vfree(res->tx_swbd);
2154 	res->tx_swbd = NULL;
2155 
2156 	return err;
2157 }
2158 
enetc_free_tx_resource(const struct enetc_bdr_resource * res)2159 static void enetc_free_tx_resource(const struct enetc_bdr_resource *res)
2160 {
2161 	dma_free_coherent(res->dev, res->bd_count * TSO_HEADER_SIZE,
2162 			  res->tso_headers, res->tso_headers_dma);
2163 	enetc_dma_free_bdr(res);
2164 	vfree(res->tx_swbd);
2165 }
2166 
2167 static struct enetc_bdr_resource *
enetc_alloc_tx_resources(struct enetc_ndev_priv * priv)2168 enetc_alloc_tx_resources(struct enetc_ndev_priv *priv)
2169 {
2170 	struct enetc_bdr_resource *tx_res;
2171 	int i, err;
2172 
2173 	tx_res = kcalloc(priv->num_tx_rings, sizeof(*tx_res), GFP_KERNEL);
2174 	if (!tx_res)
2175 		return ERR_PTR(-ENOMEM);
2176 
2177 	for (i = 0; i < priv->num_tx_rings; i++) {
2178 		struct enetc_bdr *tx_ring = priv->tx_ring[i];
2179 
2180 		err = enetc_alloc_tx_resource(&tx_res[i], tx_ring->dev,
2181 					      tx_ring->bd_count);
2182 		if (err)
2183 			goto fail;
2184 	}
2185 
2186 	return tx_res;
2187 
2188 fail:
2189 	while (i-- > 0)
2190 		enetc_free_tx_resource(&tx_res[i]);
2191 
2192 	kfree(tx_res);
2193 
2194 	return ERR_PTR(err);
2195 }
2196 
enetc_free_tx_resources(const struct enetc_bdr_resource * tx_res,size_t num_resources)2197 static void enetc_free_tx_resources(const struct enetc_bdr_resource *tx_res,
2198 				    size_t num_resources)
2199 {
2200 	size_t i;
2201 
2202 	for (i = 0; i < num_resources; i++)
2203 		enetc_free_tx_resource(&tx_res[i]);
2204 
2205 	kfree(tx_res);
2206 }
2207 
enetc_alloc_rx_resource(struct enetc_bdr_resource * res,struct device * dev,size_t bd_count,bool extended)2208 static int enetc_alloc_rx_resource(struct enetc_bdr_resource *res,
2209 				   struct device *dev, size_t bd_count,
2210 				   bool extended)
2211 {
2212 	int err;
2213 
2214 	res->dev = dev;
2215 	res->bd_count = bd_count;
2216 	res->bd_size = sizeof(union enetc_rx_bd);
2217 	if (extended)
2218 		res->bd_size *= 2;
2219 
2220 	res->rx_swbd = vcalloc(bd_count, sizeof(struct enetc_rx_swbd));
2221 	if (!res->rx_swbd)
2222 		return -ENOMEM;
2223 
2224 	err = enetc_dma_alloc_bdr(res);
2225 	if (err) {
2226 		vfree(res->rx_swbd);
2227 		return err;
2228 	}
2229 
2230 	return 0;
2231 }
2232 
enetc_free_rx_resource(const struct enetc_bdr_resource * res)2233 static void enetc_free_rx_resource(const struct enetc_bdr_resource *res)
2234 {
2235 	enetc_dma_free_bdr(res);
2236 	vfree(res->rx_swbd);
2237 }
2238 
2239 static struct enetc_bdr_resource *
enetc_alloc_rx_resources(struct enetc_ndev_priv * priv,bool extended)2240 enetc_alloc_rx_resources(struct enetc_ndev_priv *priv, bool extended)
2241 {
2242 	struct enetc_bdr_resource *rx_res;
2243 	int i, err;
2244 
2245 	rx_res = kcalloc(priv->num_rx_rings, sizeof(*rx_res), GFP_KERNEL);
2246 	if (!rx_res)
2247 		return ERR_PTR(-ENOMEM);
2248 
2249 	for (i = 0; i < priv->num_rx_rings; i++) {
2250 		struct enetc_bdr *rx_ring = priv->rx_ring[i];
2251 
2252 		err = enetc_alloc_rx_resource(&rx_res[i], rx_ring->dev,
2253 					      rx_ring->bd_count, extended);
2254 		if (err)
2255 			goto fail;
2256 	}
2257 
2258 	return rx_res;
2259 
2260 fail:
2261 	while (i-- > 0)
2262 		enetc_free_rx_resource(&rx_res[i]);
2263 
2264 	kfree(rx_res);
2265 
2266 	return ERR_PTR(err);
2267 }
2268 
enetc_free_rx_resources(const struct enetc_bdr_resource * rx_res,size_t num_resources)2269 static void enetc_free_rx_resources(const struct enetc_bdr_resource *rx_res,
2270 				    size_t num_resources)
2271 {
2272 	size_t i;
2273 
2274 	for (i = 0; i < num_resources; i++)
2275 		enetc_free_rx_resource(&rx_res[i]);
2276 
2277 	kfree(rx_res);
2278 }
2279 
enetc_assign_tx_resource(struct enetc_bdr * tx_ring,const struct enetc_bdr_resource * res)2280 static void enetc_assign_tx_resource(struct enetc_bdr *tx_ring,
2281 				     const struct enetc_bdr_resource *res)
2282 {
2283 	tx_ring->bd_base = res ? res->bd_base : NULL;
2284 	tx_ring->bd_dma_base = res ? res->bd_dma_base : 0;
2285 	tx_ring->tx_swbd = res ? res->tx_swbd : NULL;
2286 	tx_ring->tso_headers = res ? res->tso_headers : NULL;
2287 	tx_ring->tso_headers_dma = res ? res->tso_headers_dma : 0;
2288 }
2289 
enetc_assign_rx_resource(struct enetc_bdr * rx_ring,const struct enetc_bdr_resource * res)2290 static void enetc_assign_rx_resource(struct enetc_bdr *rx_ring,
2291 				     const struct enetc_bdr_resource *res)
2292 {
2293 	rx_ring->bd_base = res ? res->bd_base : NULL;
2294 	rx_ring->bd_dma_base = res ? res->bd_dma_base : 0;
2295 	rx_ring->rx_swbd = res ? res->rx_swbd : NULL;
2296 }
2297 
enetc_assign_tx_resources(struct enetc_ndev_priv * priv,const struct enetc_bdr_resource * res)2298 static void enetc_assign_tx_resources(struct enetc_ndev_priv *priv,
2299 				      const struct enetc_bdr_resource *res)
2300 {
2301 	int i;
2302 
2303 	if (priv->tx_res)
2304 		enetc_free_tx_resources(priv->tx_res, priv->num_tx_rings);
2305 
2306 	for (i = 0; i < priv->num_tx_rings; i++) {
2307 		enetc_assign_tx_resource(priv->tx_ring[i],
2308 					 res ? &res[i] : NULL);
2309 	}
2310 
2311 	priv->tx_res = res;
2312 }
2313 
enetc_assign_rx_resources(struct enetc_ndev_priv * priv,const struct enetc_bdr_resource * res)2314 static void enetc_assign_rx_resources(struct enetc_ndev_priv *priv,
2315 				      const struct enetc_bdr_resource *res)
2316 {
2317 	int i;
2318 
2319 	if (priv->rx_res)
2320 		enetc_free_rx_resources(priv->rx_res, priv->num_rx_rings);
2321 
2322 	for (i = 0; i < priv->num_rx_rings; i++) {
2323 		enetc_assign_rx_resource(priv->rx_ring[i],
2324 					 res ? &res[i] : NULL);
2325 	}
2326 
2327 	priv->rx_res = res;
2328 }
2329 
enetc_free_tx_ring(struct enetc_bdr * tx_ring)2330 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring)
2331 {
2332 	int i;
2333 
2334 	for (i = 0; i < tx_ring->bd_count; i++) {
2335 		struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i];
2336 
2337 		enetc_free_tx_frame(tx_ring, tx_swbd);
2338 	}
2339 }
2340 
enetc_free_rx_ring(struct enetc_bdr * rx_ring)2341 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring)
2342 {
2343 	int i;
2344 
2345 	for (i = 0; i < rx_ring->bd_count; i++) {
2346 		struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
2347 
2348 		if (!rx_swbd->page)
2349 			continue;
2350 
2351 		dma_unmap_page(rx_ring->dev, rx_swbd->dma, PAGE_SIZE,
2352 			       rx_swbd->dir);
2353 		__free_page(rx_swbd->page);
2354 		rx_swbd->page = NULL;
2355 	}
2356 }
2357 
enetc_free_rxtx_rings(struct enetc_ndev_priv * priv)2358 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv)
2359 {
2360 	int i;
2361 
2362 	for (i = 0; i < priv->num_rx_rings; i++)
2363 		enetc_free_rx_ring(priv->rx_ring[i]);
2364 
2365 	for (i = 0; i < priv->num_tx_rings; i++)
2366 		enetc_free_tx_ring(priv->tx_ring[i]);
2367 }
2368 
enetc_setup_default_rss_table(struct enetc_si * si,int num_groups)2369 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups)
2370 {
2371 	int *rss_table;
2372 	int i;
2373 
2374 	rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL);
2375 	if (!rss_table)
2376 		return -ENOMEM;
2377 
2378 	/* Set up RSS table defaults */
2379 	for (i = 0; i < si->num_rss; i++)
2380 		rss_table[i] = i % num_groups;
2381 
2382 	enetc_set_rss_table(si, rss_table, si->num_rss);
2383 
2384 	kfree(rss_table);
2385 
2386 	return 0;
2387 }
2388 
enetc_set_lso_flags_mask(struct enetc_hw * hw)2389 static void enetc_set_lso_flags_mask(struct enetc_hw *hw)
2390 {
2391 	enetc_wr(hw, ENETC4_SILSOSFMR0,
2392 		 SILSOSFMR0_VAL_SET(ENETC4_TCP_NL_SEG_FLAGS_DMASK,
2393 				    ENETC4_TCP_NL_SEG_FLAGS_DMASK));
2394 	enetc_wr(hw, ENETC4_SILSOSFMR1, 0);
2395 }
2396 
enetc_configure_si(struct enetc_ndev_priv * priv)2397 int enetc_configure_si(struct enetc_ndev_priv *priv)
2398 {
2399 	struct enetc_si *si = priv->si;
2400 	struct enetc_hw *hw = &si->hw;
2401 	int err;
2402 
2403 	/* set SI cache attributes */
2404 	enetc_wr(hw, ENETC_SICAR0,
2405 		 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT);
2406 	enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI);
2407 	/* enable SI */
2408 	enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN);
2409 
2410 	if (si->hw_features & ENETC_SI_F_LSO)
2411 		enetc_set_lso_flags_mask(hw);
2412 
2413 	/* TODO: RSS support for i.MX95 will be supported later, and the
2414 	 * is_enetc_rev1() condition will be removed
2415 	 */
2416 	if (si->num_rss && is_enetc_rev1(si)) {
2417 		err = enetc_setup_default_rss_table(si, priv->num_rx_rings);
2418 		if (err)
2419 			return err;
2420 	}
2421 
2422 	return 0;
2423 }
2424 EXPORT_SYMBOL_GPL(enetc_configure_si);
2425 
enetc_init_si_rings_params(struct enetc_ndev_priv * priv)2426 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv)
2427 {
2428 	struct enetc_si *si = priv->si;
2429 	int cpus = num_online_cpus();
2430 
2431 	priv->tx_bd_count = ENETC_TX_RING_DEFAULT_SIZE;
2432 	priv->rx_bd_count = ENETC_RX_RING_DEFAULT_SIZE;
2433 
2434 	/* Enable all available TX rings in order to configure as many
2435 	 * priorities as possible, when needed.
2436 	 * TODO: Make # of TX rings run-time configurable
2437 	 */
2438 	priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings);
2439 	priv->num_tx_rings = si->num_tx_rings;
2440 	priv->bdr_int_num = priv->num_rx_rings;
2441 	priv->ic_mode = ENETC_IC_RX_ADAPTIVE | ENETC_IC_TX_MANUAL;
2442 	priv->tx_ictt = enetc_usecs_to_cycles(600, priv->sysclk_freq);
2443 }
2444 EXPORT_SYMBOL_GPL(enetc_init_si_rings_params);
2445 
enetc_alloc_si_resources(struct enetc_ndev_priv * priv)2446 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv)
2447 {
2448 	struct enetc_si *si = priv->si;
2449 
2450 	priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules),
2451 				  GFP_KERNEL);
2452 	if (!priv->cls_rules)
2453 		return -ENOMEM;
2454 
2455 	return 0;
2456 }
2457 EXPORT_SYMBOL_GPL(enetc_alloc_si_resources);
2458 
enetc_free_si_resources(struct enetc_ndev_priv * priv)2459 void enetc_free_si_resources(struct enetc_ndev_priv *priv)
2460 {
2461 	kfree(priv->cls_rules);
2462 }
2463 EXPORT_SYMBOL_GPL(enetc_free_si_resources);
2464 
enetc_setup_txbdr(struct enetc_hw * hw,struct enetc_bdr * tx_ring)2465 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
2466 {
2467 	int idx = tx_ring->index;
2468 	u32 tbmr;
2469 
2470 	enetc_txbdr_wr(hw, idx, ENETC_TBBAR0,
2471 		       lower_32_bits(tx_ring->bd_dma_base));
2472 
2473 	enetc_txbdr_wr(hw, idx, ENETC_TBBAR1,
2474 		       upper_32_bits(tx_ring->bd_dma_base));
2475 
2476 	WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */
2477 	enetc_txbdr_wr(hw, idx, ENETC_TBLENR,
2478 		       ENETC_RTBLENR_LEN(tx_ring->bd_count));
2479 
2480 	/* clearing PI/CI registers for Tx not supported, adjust sw indexes */
2481 	tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR);
2482 	tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR);
2483 
2484 	/* enable Tx ints by setting pkt thr to 1 */
2485 	enetc_txbdr_wr(hw, idx, ENETC_TBICR0, ENETC_TBICR0_ICEN | 0x1);
2486 
2487 	tbmr = ENETC_TBMR_SET_PRIO(tx_ring->prio);
2488 	if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX)
2489 		tbmr |= ENETC_TBMR_VIH;
2490 
2491 	/* enable ring */
2492 	enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr);
2493 
2494 	tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR);
2495 	tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR);
2496 	tx_ring->idr = hw->reg + ENETC_SITXIDR;
2497 }
2498 
enetc_setup_rxbdr(struct enetc_hw * hw,struct enetc_bdr * rx_ring,bool extended)2499 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring,
2500 			      bool extended)
2501 {
2502 	int idx = rx_ring->index;
2503 	u32 rbmr = 0;
2504 
2505 	enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0,
2506 		       lower_32_bits(rx_ring->bd_dma_base));
2507 
2508 	enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1,
2509 		       upper_32_bits(rx_ring->bd_dma_base));
2510 
2511 	WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */
2512 	enetc_rxbdr_wr(hw, idx, ENETC_RBLENR,
2513 		       ENETC_RTBLENR_LEN(rx_ring->bd_count));
2514 
2515 	if (rx_ring->xdp.prog)
2516 		enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE_XDP);
2517 	else
2518 		enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE);
2519 
2520 	/* Also prepare the consumer index in case page allocation never
2521 	 * succeeds. In that case, hardware will never advance producer index
2522 	 * to match consumer index, and will drop all frames.
2523 	 */
2524 	enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0);
2525 	enetc_rxbdr_wr(hw, idx, ENETC_RBCIR, 1);
2526 
2527 	/* enable Rx ints by setting pkt thr to 1 */
2528 	enetc_rxbdr_wr(hw, idx, ENETC_RBICR0, ENETC_RBICR0_ICEN | 0x1);
2529 
2530 	rx_ring->ext_en = extended;
2531 	if (rx_ring->ext_en)
2532 		rbmr |= ENETC_RBMR_BDS;
2533 
2534 	if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX)
2535 		rbmr |= ENETC_RBMR_VTE;
2536 
2537 	rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR);
2538 	rx_ring->idr = hw->reg + ENETC_SIRXIDR;
2539 
2540 	rx_ring->next_to_clean = 0;
2541 	rx_ring->next_to_use = 0;
2542 	rx_ring->next_to_alloc = 0;
2543 
2544 	enetc_lock_mdio();
2545 	enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring));
2546 	enetc_unlock_mdio();
2547 
2548 	enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr);
2549 }
2550 
enetc_setup_bdrs(struct enetc_ndev_priv * priv,bool extended)2551 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv, bool extended)
2552 {
2553 	struct enetc_hw *hw = &priv->si->hw;
2554 	int i;
2555 
2556 	for (i = 0; i < priv->num_tx_rings; i++)
2557 		enetc_setup_txbdr(hw, priv->tx_ring[i]);
2558 
2559 	for (i = 0; i < priv->num_rx_rings; i++)
2560 		enetc_setup_rxbdr(hw, priv->rx_ring[i], extended);
2561 }
2562 
enetc_enable_txbdr(struct enetc_hw * hw,struct enetc_bdr * tx_ring)2563 static void enetc_enable_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
2564 {
2565 	int idx = tx_ring->index;
2566 	u32 tbmr;
2567 
2568 	tbmr = enetc_txbdr_rd(hw, idx, ENETC_TBMR);
2569 	tbmr |= ENETC_TBMR_EN;
2570 	enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr);
2571 }
2572 
enetc_enable_rxbdr(struct enetc_hw * hw,struct enetc_bdr * rx_ring)2573 static void enetc_enable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
2574 {
2575 	int idx = rx_ring->index;
2576 	u32 rbmr;
2577 
2578 	rbmr = enetc_rxbdr_rd(hw, idx, ENETC_RBMR);
2579 	rbmr |= ENETC_RBMR_EN;
2580 	enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr);
2581 }
2582 
enetc_enable_rx_bdrs(struct enetc_ndev_priv * priv)2583 static void enetc_enable_rx_bdrs(struct enetc_ndev_priv *priv)
2584 {
2585 	struct enetc_hw *hw = &priv->si->hw;
2586 	int i;
2587 
2588 	for (i = 0; i < priv->num_rx_rings; i++)
2589 		enetc_enable_rxbdr(hw, priv->rx_ring[i]);
2590 }
2591 
enetc_enable_tx_bdrs(struct enetc_ndev_priv * priv)2592 static void enetc_enable_tx_bdrs(struct enetc_ndev_priv *priv)
2593 {
2594 	struct enetc_hw *hw = &priv->si->hw;
2595 	int i;
2596 
2597 	for (i = 0; i < priv->num_tx_rings; i++)
2598 		enetc_enable_txbdr(hw, priv->tx_ring[i]);
2599 }
2600 
enetc_disable_rxbdr(struct enetc_hw * hw,struct enetc_bdr * rx_ring)2601 static void enetc_disable_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
2602 {
2603 	int idx = rx_ring->index;
2604 
2605 	/* disable EN bit on ring */
2606 	enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0);
2607 }
2608 
enetc_disable_txbdr(struct enetc_hw * hw,struct enetc_bdr * rx_ring)2609 static void enetc_disable_txbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
2610 {
2611 	int idx = rx_ring->index;
2612 
2613 	/* disable EN bit on ring */
2614 	enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0);
2615 }
2616 
enetc_disable_rx_bdrs(struct enetc_ndev_priv * priv)2617 static void enetc_disable_rx_bdrs(struct enetc_ndev_priv *priv)
2618 {
2619 	struct enetc_hw *hw = &priv->si->hw;
2620 	int i;
2621 
2622 	for (i = 0; i < priv->num_rx_rings; i++)
2623 		enetc_disable_rxbdr(hw, priv->rx_ring[i]);
2624 }
2625 
enetc_disable_tx_bdrs(struct enetc_ndev_priv * priv)2626 static void enetc_disable_tx_bdrs(struct enetc_ndev_priv *priv)
2627 {
2628 	struct enetc_hw *hw = &priv->si->hw;
2629 	int i;
2630 
2631 	for (i = 0; i < priv->num_tx_rings; i++)
2632 		enetc_disable_txbdr(hw, priv->tx_ring[i]);
2633 }
2634 
enetc_wait_txbdr(struct enetc_hw * hw,struct enetc_bdr * tx_ring)2635 static void enetc_wait_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
2636 {
2637 	int delay = 8, timeout = 100;
2638 	int idx = tx_ring->index;
2639 
2640 	/* wait for busy to clear */
2641 	while (delay < timeout &&
2642 	       enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) {
2643 		msleep(delay);
2644 		delay *= 2;
2645 	}
2646 
2647 	if (delay >= timeout)
2648 		netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n",
2649 			    idx);
2650 }
2651 
enetc_wait_bdrs(struct enetc_ndev_priv * priv)2652 static void enetc_wait_bdrs(struct enetc_ndev_priv *priv)
2653 {
2654 	struct enetc_hw *hw = &priv->si->hw;
2655 	int i;
2656 
2657 	for (i = 0; i < priv->num_tx_rings; i++)
2658 		enetc_wait_txbdr(hw, priv->tx_ring[i]);
2659 }
2660 
enetc_setup_irqs(struct enetc_ndev_priv * priv)2661 static int enetc_setup_irqs(struct enetc_ndev_priv *priv)
2662 {
2663 	struct pci_dev *pdev = priv->si->pdev;
2664 	struct enetc_hw *hw = &priv->si->hw;
2665 	int i, j, err;
2666 
2667 	for (i = 0; i < priv->bdr_int_num; i++) {
2668 		int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
2669 		struct enetc_int_vector *v = priv->int_vector[i];
2670 		int entry = ENETC_BDR_INT_BASE_IDX + i;
2671 
2672 		snprintf(v->name, sizeof(v->name), "%s-rxtx%d",
2673 			 priv->ndev->name, i);
2674 		err = request_irq(irq, enetc_msix, IRQF_NO_AUTOEN, v->name, v);
2675 		if (err) {
2676 			dev_err(priv->dev, "request_irq() failed!\n");
2677 			goto irq_err;
2678 		}
2679 
2680 		v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER);
2681 		v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER);
2682 		v->ricr1 = hw->reg + ENETC_BDR(RX, i, ENETC_RBICR1);
2683 
2684 		enetc_wr(hw, ENETC_SIMSIRRV(i), entry);
2685 
2686 		for (j = 0; j < v->count_tx_rings; j++) {
2687 			int idx = v->tx_ring[j].index;
2688 
2689 			enetc_wr(hw, ENETC_SIMSITRV(idx), entry);
2690 		}
2691 		irq_set_affinity_hint(irq, get_cpu_mask(i % num_online_cpus()));
2692 	}
2693 
2694 	return 0;
2695 
2696 irq_err:
2697 	while (i--) {
2698 		int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
2699 
2700 		irq_set_affinity_hint(irq, NULL);
2701 		free_irq(irq, priv->int_vector[i]);
2702 	}
2703 
2704 	return err;
2705 }
2706 
enetc_free_irqs(struct enetc_ndev_priv * priv)2707 static void enetc_free_irqs(struct enetc_ndev_priv *priv)
2708 {
2709 	struct pci_dev *pdev = priv->si->pdev;
2710 	int i;
2711 
2712 	for (i = 0; i < priv->bdr_int_num; i++) {
2713 		int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
2714 
2715 		irq_set_affinity_hint(irq, NULL);
2716 		free_irq(irq, priv->int_vector[i]);
2717 	}
2718 }
2719 
enetc_setup_interrupts(struct enetc_ndev_priv * priv)2720 static void enetc_setup_interrupts(struct enetc_ndev_priv *priv)
2721 {
2722 	struct enetc_hw *hw = &priv->si->hw;
2723 	u32 icpt, ictt;
2724 	int i;
2725 
2726 	/* enable Tx & Rx event indication */
2727 	if (priv->ic_mode &
2728 	    (ENETC_IC_RX_MANUAL | ENETC_IC_RX_ADAPTIVE)) {
2729 		icpt = ENETC_RBICR0_SET_ICPT(ENETC_RXIC_PKTTHR);
2730 		/* init to non-0 minimum, will be adjusted later */
2731 		ictt = 0x1;
2732 	} else {
2733 		icpt = 0x1; /* enable Rx ints by setting pkt thr to 1 */
2734 		ictt = 0;
2735 	}
2736 
2737 	for (i = 0; i < priv->num_rx_rings; i++) {
2738 		enetc_rxbdr_wr(hw, i, ENETC_RBICR1, ictt);
2739 		enetc_rxbdr_wr(hw, i, ENETC_RBICR0, ENETC_RBICR0_ICEN | icpt);
2740 		enetc_rxbdr_wr(hw, i, ENETC_RBIER, ENETC_RBIER_RXTIE);
2741 	}
2742 
2743 	if (priv->ic_mode & ENETC_IC_TX_MANUAL)
2744 		icpt = ENETC_TBICR0_SET_ICPT(ENETC_TXIC_PKTTHR);
2745 	else
2746 		icpt = 0x1; /* enable Tx ints by setting pkt thr to 1 */
2747 
2748 	for (i = 0; i < priv->num_tx_rings; i++) {
2749 		enetc_txbdr_wr(hw, i, ENETC_TBICR1, priv->tx_ictt);
2750 		enetc_txbdr_wr(hw, i, ENETC_TBICR0, ENETC_TBICR0_ICEN | icpt);
2751 		enetc_txbdr_wr(hw, i, ENETC_TBIER, ENETC_TBIER_TXTIE);
2752 	}
2753 }
2754 
enetc_clear_interrupts(struct enetc_ndev_priv * priv)2755 static void enetc_clear_interrupts(struct enetc_ndev_priv *priv)
2756 {
2757 	struct enetc_hw *hw = &priv->si->hw;
2758 	int i;
2759 
2760 	for (i = 0; i < priv->num_tx_rings; i++)
2761 		enetc_txbdr_wr(hw, i, ENETC_TBIER, 0);
2762 
2763 	for (i = 0; i < priv->num_rx_rings; i++)
2764 		enetc_rxbdr_wr(hw, i, ENETC_RBIER, 0);
2765 }
2766 
enetc_phylink_connect(struct net_device * ndev)2767 static int enetc_phylink_connect(struct net_device *ndev)
2768 {
2769 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2770 	struct ethtool_keee edata;
2771 	int err;
2772 
2773 	if (!priv->phylink) {
2774 		/* phy-less mode */
2775 		netif_carrier_on(ndev);
2776 		return 0;
2777 	}
2778 
2779 	err = phylink_of_phy_connect(priv->phylink, priv->dev->of_node, 0);
2780 	if (err) {
2781 		dev_err(&ndev->dev, "could not attach to PHY\n");
2782 		return err;
2783 	}
2784 
2785 	/* disable EEE autoneg, until ENETC driver supports it */
2786 	memset(&edata, 0, sizeof(struct ethtool_keee));
2787 	phylink_ethtool_set_eee(priv->phylink, &edata);
2788 
2789 	phylink_start(priv->phylink);
2790 
2791 	return 0;
2792 }
2793 
enetc_tx_onestep_tstamp(struct work_struct * work)2794 static void enetc_tx_onestep_tstamp(struct work_struct *work)
2795 {
2796 	struct enetc_ndev_priv *priv;
2797 	struct sk_buff *skb;
2798 
2799 	priv = container_of(work, struct enetc_ndev_priv, tx_onestep_tstamp);
2800 
2801 	netif_tx_lock_bh(priv->ndev);
2802 
2803 	clear_bit_unlock(ENETC_TX_ONESTEP_TSTAMP_IN_PROGRESS, &priv->flags);
2804 	skb = skb_dequeue(&priv->tx_skbs);
2805 	if (skb)
2806 		enetc_start_xmit(skb, priv->ndev);
2807 
2808 	netif_tx_unlock_bh(priv->ndev);
2809 }
2810 
enetc_tx_onestep_tstamp_init(struct enetc_ndev_priv * priv)2811 static void enetc_tx_onestep_tstamp_init(struct enetc_ndev_priv *priv)
2812 {
2813 	INIT_WORK(&priv->tx_onestep_tstamp, enetc_tx_onestep_tstamp);
2814 	skb_queue_head_init(&priv->tx_skbs);
2815 }
2816 
enetc_start(struct net_device * ndev)2817 void enetc_start(struct net_device *ndev)
2818 {
2819 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2820 	int i;
2821 
2822 	enetc_setup_interrupts(priv);
2823 
2824 	for (i = 0; i < priv->bdr_int_num; i++) {
2825 		int irq = pci_irq_vector(priv->si->pdev,
2826 					 ENETC_BDR_INT_BASE_IDX + i);
2827 
2828 		napi_enable(&priv->int_vector[i]->napi);
2829 		enable_irq(irq);
2830 	}
2831 
2832 	enetc_enable_tx_bdrs(priv);
2833 
2834 	enetc_enable_rx_bdrs(priv);
2835 
2836 	netif_tx_start_all_queues(ndev);
2837 
2838 	clear_bit(ENETC_TX_DOWN, &priv->flags);
2839 }
2840 EXPORT_SYMBOL_GPL(enetc_start);
2841 
enetc_open(struct net_device * ndev)2842 int enetc_open(struct net_device *ndev)
2843 {
2844 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2845 	struct enetc_bdr_resource *tx_res, *rx_res;
2846 	bool extended;
2847 	int err;
2848 
2849 	extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP);
2850 
2851 	err = clk_prepare_enable(priv->ref_clk);
2852 	if (err)
2853 		return err;
2854 
2855 	err = enetc_setup_irqs(priv);
2856 	if (err)
2857 		goto err_setup_irqs;
2858 
2859 	err = enetc_phylink_connect(ndev);
2860 	if (err)
2861 		goto err_phy_connect;
2862 
2863 	tx_res = enetc_alloc_tx_resources(priv);
2864 	if (IS_ERR(tx_res)) {
2865 		err = PTR_ERR(tx_res);
2866 		goto err_alloc_tx;
2867 	}
2868 
2869 	rx_res = enetc_alloc_rx_resources(priv, extended);
2870 	if (IS_ERR(rx_res)) {
2871 		err = PTR_ERR(rx_res);
2872 		goto err_alloc_rx;
2873 	}
2874 
2875 	enetc_tx_onestep_tstamp_init(priv);
2876 	enetc_assign_tx_resources(priv, tx_res);
2877 	enetc_assign_rx_resources(priv, rx_res);
2878 	enetc_setup_bdrs(priv, extended);
2879 	enetc_start(ndev);
2880 
2881 	return 0;
2882 
2883 err_alloc_rx:
2884 	enetc_free_tx_resources(tx_res, priv->num_tx_rings);
2885 err_alloc_tx:
2886 	if (priv->phylink)
2887 		phylink_disconnect_phy(priv->phylink);
2888 err_phy_connect:
2889 	enetc_free_irqs(priv);
2890 err_setup_irqs:
2891 	clk_disable_unprepare(priv->ref_clk);
2892 
2893 	return err;
2894 }
2895 EXPORT_SYMBOL_GPL(enetc_open);
2896 
enetc_stop(struct net_device * ndev)2897 void enetc_stop(struct net_device *ndev)
2898 {
2899 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2900 	int i;
2901 
2902 	set_bit(ENETC_TX_DOWN, &priv->flags);
2903 
2904 	netif_tx_stop_all_queues(ndev);
2905 
2906 	enetc_disable_rx_bdrs(priv);
2907 
2908 	enetc_wait_bdrs(priv);
2909 
2910 	enetc_disable_tx_bdrs(priv);
2911 
2912 	for (i = 0; i < priv->bdr_int_num; i++) {
2913 		int irq = pci_irq_vector(priv->si->pdev,
2914 					 ENETC_BDR_INT_BASE_IDX + i);
2915 
2916 		disable_irq(irq);
2917 		napi_synchronize(&priv->int_vector[i]->napi);
2918 		napi_disable(&priv->int_vector[i]->napi);
2919 	}
2920 
2921 	enetc_clear_interrupts(priv);
2922 }
2923 EXPORT_SYMBOL_GPL(enetc_stop);
2924 
enetc_close(struct net_device * ndev)2925 int enetc_close(struct net_device *ndev)
2926 {
2927 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
2928 
2929 	enetc_stop(ndev);
2930 
2931 	if (priv->phylink) {
2932 		phylink_stop(priv->phylink);
2933 		phylink_disconnect_phy(priv->phylink);
2934 	} else {
2935 		netif_carrier_off(ndev);
2936 	}
2937 
2938 	enetc_free_rxtx_rings(priv);
2939 
2940 	/* Avoids dangling pointers and also frees old resources */
2941 	enetc_assign_rx_resources(priv, NULL);
2942 	enetc_assign_tx_resources(priv, NULL);
2943 
2944 	enetc_free_irqs(priv);
2945 	clk_disable_unprepare(priv->ref_clk);
2946 
2947 	return 0;
2948 }
2949 EXPORT_SYMBOL_GPL(enetc_close);
2950 
enetc_reconfigure(struct enetc_ndev_priv * priv,bool extended,int (* cb)(struct enetc_ndev_priv * priv,void * ctx),void * ctx)2951 static int enetc_reconfigure(struct enetc_ndev_priv *priv, bool extended,
2952 			     int (*cb)(struct enetc_ndev_priv *priv, void *ctx),
2953 			     void *ctx)
2954 {
2955 	struct enetc_bdr_resource *tx_res, *rx_res;
2956 	int err;
2957 
2958 	ASSERT_RTNL();
2959 
2960 	/* If the interface is down, run the callback right away,
2961 	 * without reconfiguration.
2962 	 */
2963 	if (!netif_running(priv->ndev)) {
2964 		if (cb) {
2965 			err = cb(priv, ctx);
2966 			if (err)
2967 				return err;
2968 		}
2969 
2970 		return 0;
2971 	}
2972 
2973 	tx_res = enetc_alloc_tx_resources(priv);
2974 	if (IS_ERR(tx_res)) {
2975 		err = PTR_ERR(tx_res);
2976 		goto out;
2977 	}
2978 
2979 	rx_res = enetc_alloc_rx_resources(priv, extended);
2980 	if (IS_ERR(rx_res)) {
2981 		err = PTR_ERR(rx_res);
2982 		goto out_free_tx_res;
2983 	}
2984 
2985 	enetc_stop(priv->ndev);
2986 	enetc_free_rxtx_rings(priv);
2987 
2988 	/* Interface is down, run optional callback now */
2989 	if (cb) {
2990 		err = cb(priv, ctx);
2991 		if (err)
2992 			goto out_restart;
2993 	}
2994 
2995 	enetc_assign_tx_resources(priv, tx_res);
2996 	enetc_assign_rx_resources(priv, rx_res);
2997 	enetc_setup_bdrs(priv, extended);
2998 	enetc_start(priv->ndev);
2999 
3000 	return 0;
3001 
3002 out_restart:
3003 	enetc_setup_bdrs(priv, extended);
3004 	enetc_start(priv->ndev);
3005 	enetc_free_rx_resources(rx_res, priv->num_rx_rings);
3006 out_free_tx_res:
3007 	enetc_free_tx_resources(tx_res, priv->num_tx_rings);
3008 out:
3009 	return err;
3010 }
3011 
enetc_debug_tx_ring_prios(struct enetc_ndev_priv * priv)3012 static void enetc_debug_tx_ring_prios(struct enetc_ndev_priv *priv)
3013 {
3014 	int i;
3015 
3016 	for (i = 0; i < priv->num_tx_rings; i++)
3017 		netdev_dbg(priv->ndev, "TX ring %d prio %d\n", i,
3018 			   priv->tx_ring[i]->prio);
3019 }
3020 
enetc_reset_tc_mqprio(struct net_device * ndev)3021 void enetc_reset_tc_mqprio(struct net_device *ndev)
3022 {
3023 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
3024 	struct enetc_hw *hw = &priv->si->hw;
3025 	struct enetc_bdr *tx_ring;
3026 	int num_stack_tx_queues;
3027 	int i;
3028 
3029 	num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
3030 
3031 	netdev_reset_tc(ndev);
3032 	netif_set_real_num_tx_queues(ndev, num_stack_tx_queues);
3033 	priv->min_num_stack_tx_queues = num_possible_cpus();
3034 
3035 	/* Reset all ring priorities to 0 */
3036 	for (i = 0; i < priv->num_tx_rings; i++) {
3037 		tx_ring = priv->tx_ring[i];
3038 		tx_ring->prio = 0;
3039 		enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio);
3040 	}
3041 
3042 	enetc_debug_tx_ring_prios(priv);
3043 
3044 	enetc_change_preemptible_tcs(priv, 0);
3045 }
3046 EXPORT_SYMBOL_GPL(enetc_reset_tc_mqprio);
3047 
enetc_setup_tc_mqprio(struct net_device * ndev,void * type_data)3048 int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data)
3049 {
3050 	struct tc_mqprio_qopt_offload *mqprio = type_data;
3051 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
3052 	struct tc_mqprio_qopt *qopt = &mqprio->qopt;
3053 	struct enetc_hw *hw = &priv->si->hw;
3054 	int num_stack_tx_queues = 0;
3055 	struct enetc_bdr *tx_ring;
3056 	u8 num_tc = qopt->num_tc;
3057 	int offset, count;
3058 	int err, tc, q;
3059 
3060 	if (!num_tc) {
3061 		enetc_reset_tc_mqprio(ndev);
3062 		return 0;
3063 	}
3064 
3065 	err = netdev_set_num_tc(ndev, num_tc);
3066 	if (err)
3067 		return err;
3068 
3069 	for (tc = 0; tc < num_tc; tc++) {
3070 		offset = qopt->offset[tc];
3071 		count = qopt->count[tc];
3072 		num_stack_tx_queues += count;
3073 
3074 		err = netdev_set_tc_queue(ndev, tc, count, offset);
3075 		if (err)
3076 			goto err_reset_tc;
3077 
3078 		for (q = offset; q < offset + count; q++) {
3079 			tx_ring = priv->tx_ring[q];
3080 			/* The prio_tc_map is skb_tx_hash()'s way of selecting
3081 			 * between TX queues based on skb->priority. As such,
3082 			 * there's nothing to offload based on it.
3083 			 * Make the mqprio "traffic class" be the priority of
3084 			 * this ring group, and leave the Tx IPV to traffic
3085 			 * class mapping as its default mapping value of 1:1.
3086 			 */
3087 			tx_ring->prio = tc;
3088 			enetc_set_bdr_prio(hw, tx_ring->index, tx_ring->prio);
3089 		}
3090 	}
3091 
3092 	err = netif_set_real_num_tx_queues(ndev, num_stack_tx_queues);
3093 	if (err)
3094 		goto err_reset_tc;
3095 
3096 	priv->min_num_stack_tx_queues = num_stack_tx_queues;
3097 
3098 	enetc_debug_tx_ring_prios(priv);
3099 
3100 	enetc_change_preemptible_tcs(priv, mqprio->preemptible_tcs);
3101 
3102 	return 0;
3103 
3104 err_reset_tc:
3105 	enetc_reset_tc_mqprio(ndev);
3106 	return err;
3107 }
3108 EXPORT_SYMBOL_GPL(enetc_setup_tc_mqprio);
3109 
enetc_reconfigure_xdp_cb(struct enetc_ndev_priv * priv,void * ctx)3110 static int enetc_reconfigure_xdp_cb(struct enetc_ndev_priv *priv, void *ctx)
3111 {
3112 	struct bpf_prog *old_prog, *prog = ctx;
3113 	int num_stack_tx_queues;
3114 	int err, i;
3115 
3116 	old_prog = xchg(&priv->xdp_prog, prog);
3117 
3118 	num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
3119 	err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues);
3120 	if (err) {
3121 		xchg(&priv->xdp_prog, old_prog);
3122 		return err;
3123 	}
3124 
3125 	if (old_prog)
3126 		bpf_prog_put(old_prog);
3127 
3128 	for (i = 0; i < priv->num_rx_rings; i++) {
3129 		struct enetc_bdr *rx_ring = priv->rx_ring[i];
3130 
3131 		rx_ring->xdp.prog = prog;
3132 
3133 		if (prog)
3134 			rx_ring->buffer_offset = XDP_PACKET_HEADROOM;
3135 		else
3136 			rx_ring->buffer_offset = ENETC_RXB_PAD;
3137 	}
3138 
3139 	return 0;
3140 }
3141 
enetc_setup_xdp_prog(struct net_device * ndev,struct bpf_prog * prog,struct netlink_ext_ack * extack)3142 static int enetc_setup_xdp_prog(struct net_device *ndev, struct bpf_prog *prog,
3143 				struct netlink_ext_ack *extack)
3144 {
3145 	int num_xdp_tx_queues = prog ? num_possible_cpus() : 0;
3146 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
3147 	bool extended;
3148 
3149 	if (priv->min_num_stack_tx_queues + num_xdp_tx_queues >
3150 	    priv->num_tx_rings) {
3151 		NL_SET_ERR_MSG_FMT_MOD(extack,
3152 				       "Reserving %d XDP TXQs leaves under %d for stack (total %d)",
3153 				       num_xdp_tx_queues,
3154 				       priv->min_num_stack_tx_queues,
3155 				       priv->num_tx_rings);
3156 		return -EBUSY;
3157 	}
3158 
3159 	extended = !!(priv->active_offloads & ENETC_F_RX_TSTAMP);
3160 
3161 	/* The buffer layout is changing, so we need to drain the old
3162 	 * RX buffers and seed new ones.
3163 	 */
3164 	return enetc_reconfigure(priv, extended, enetc_reconfigure_xdp_cb, prog);
3165 }
3166 
enetc_setup_bpf(struct net_device * ndev,struct netdev_bpf * bpf)3167 int enetc_setup_bpf(struct net_device *ndev, struct netdev_bpf *bpf)
3168 {
3169 	switch (bpf->command) {
3170 	case XDP_SETUP_PROG:
3171 		return enetc_setup_xdp_prog(ndev, bpf->prog, bpf->extack);
3172 	default:
3173 		return -EINVAL;
3174 	}
3175 
3176 	return 0;
3177 }
3178 EXPORT_SYMBOL_GPL(enetc_setup_bpf);
3179 
enetc_get_stats(struct net_device * ndev)3180 struct net_device_stats *enetc_get_stats(struct net_device *ndev)
3181 {
3182 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
3183 	struct net_device_stats *stats = &ndev->stats;
3184 	unsigned long packets = 0, bytes = 0;
3185 	unsigned long tx_dropped = 0;
3186 	int i;
3187 
3188 	for (i = 0; i < priv->num_rx_rings; i++) {
3189 		packets += priv->rx_ring[i]->stats.packets;
3190 		bytes	+= priv->rx_ring[i]->stats.bytes;
3191 	}
3192 
3193 	stats->rx_packets = packets;
3194 	stats->rx_bytes = bytes;
3195 	bytes = 0;
3196 	packets = 0;
3197 
3198 	for (i = 0; i < priv->num_tx_rings; i++) {
3199 		packets += priv->tx_ring[i]->stats.packets;
3200 		bytes	+= priv->tx_ring[i]->stats.bytes;
3201 		tx_dropped += priv->tx_ring[i]->stats.win_drop;
3202 	}
3203 
3204 	stats->tx_packets = packets;
3205 	stats->tx_bytes = bytes;
3206 	stats->tx_dropped = tx_dropped;
3207 
3208 	return stats;
3209 }
3210 EXPORT_SYMBOL_GPL(enetc_get_stats);
3211 
enetc_set_rss(struct net_device * ndev,int en)3212 static int enetc_set_rss(struct net_device *ndev, int en)
3213 {
3214 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
3215 	struct enetc_hw *hw = &priv->si->hw;
3216 	u32 reg;
3217 
3218 	enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings);
3219 
3220 	reg = enetc_rd(hw, ENETC_SIMR);
3221 	reg &= ~ENETC_SIMR_RSSE;
3222 	reg |= (en) ? ENETC_SIMR_RSSE : 0;
3223 	enetc_wr(hw, ENETC_SIMR, reg);
3224 
3225 	return 0;
3226 }
3227 
enetc_enable_rxvlan(struct net_device * ndev,bool en)3228 static void enetc_enable_rxvlan(struct net_device *ndev, bool en)
3229 {
3230 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
3231 	struct enetc_hw *hw = &priv->si->hw;
3232 	int i;
3233 
3234 	for (i = 0; i < priv->num_rx_rings; i++)
3235 		enetc_bdr_enable_rxvlan(hw, i, en);
3236 }
3237 
enetc_enable_txvlan(struct net_device * ndev,bool en)3238 static void enetc_enable_txvlan(struct net_device *ndev, bool en)
3239 {
3240 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
3241 	struct enetc_hw *hw = &priv->si->hw;
3242 	int i;
3243 
3244 	for (i = 0; i < priv->num_tx_rings; i++)
3245 		enetc_bdr_enable_txvlan(hw, i, en);
3246 }
3247 
enetc_set_features(struct net_device * ndev,netdev_features_t features)3248 void enetc_set_features(struct net_device *ndev, netdev_features_t features)
3249 {
3250 	netdev_features_t changed = ndev->features ^ features;
3251 
3252 	if (changed & NETIF_F_RXHASH)
3253 		enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH));
3254 
3255 	if (changed & NETIF_F_HW_VLAN_CTAG_RX)
3256 		enetc_enable_rxvlan(ndev,
3257 				    !!(features & NETIF_F_HW_VLAN_CTAG_RX));
3258 
3259 	if (changed & NETIF_F_HW_VLAN_CTAG_TX)
3260 		enetc_enable_txvlan(ndev,
3261 				    !!(features & NETIF_F_HW_VLAN_CTAG_TX));
3262 }
3263 EXPORT_SYMBOL_GPL(enetc_set_features);
3264 
enetc_hwtstamp_set(struct net_device * ndev,struct ifreq * ifr)3265 static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr)
3266 {
3267 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
3268 	int err, new_offloads = priv->active_offloads;
3269 	struct hwtstamp_config config;
3270 
3271 	if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
3272 		return -EFAULT;
3273 
3274 	switch (config.tx_type) {
3275 	case HWTSTAMP_TX_OFF:
3276 		new_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
3277 		break;
3278 	case HWTSTAMP_TX_ON:
3279 		new_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
3280 		new_offloads |= ENETC_F_TX_TSTAMP;
3281 		break;
3282 	case HWTSTAMP_TX_ONESTEP_SYNC:
3283 		if (!enetc_si_is_pf(priv->si))
3284 			return -EOPNOTSUPP;
3285 
3286 		new_offloads &= ~ENETC_F_TX_TSTAMP_MASK;
3287 		new_offloads |= ENETC_F_TX_ONESTEP_SYNC_TSTAMP;
3288 		break;
3289 	default:
3290 		return -ERANGE;
3291 	}
3292 
3293 	switch (config.rx_filter) {
3294 	case HWTSTAMP_FILTER_NONE:
3295 		new_offloads &= ~ENETC_F_RX_TSTAMP;
3296 		break;
3297 	default:
3298 		new_offloads |= ENETC_F_RX_TSTAMP;
3299 		config.rx_filter = HWTSTAMP_FILTER_ALL;
3300 	}
3301 
3302 	if ((new_offloads ^ priv->active_offloads) & ENETC_F_RX_TSTAMP) {
3303 		bool extended = !!(new_offloads & ENETC_F_RX_TSTAMP);
3304 
3305 		err = enetc_reconfigure(priv, extended, NULL, NULL);
3306 		if (err)
3307 			return err;
3308 	}
3309 
3310 	priv->active_offloads = new_offloads;
3311 
3312 	return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
3313 	       -EFAULT : 0;
3314 }
3315 
enetc_hwtstamp_get(struct net_device * ndev,struct ifreq * ifr)3316 static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr)
3317 {
3318 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
3319 	struct hwtstamp_config config;
3320 
3321 	config.flags = 0;
3322 
3323 	if (priv->active_offloads & ENETC_F_TX_ONESTEP_SYNC_TSTAMP)
3324 		config.tx_type = HWTSTAMP_TX_ONESTEP_SYNC;
3325 	else if (priv->active_offloads & ENETC_F_TX_TSTAMP)
3326 		config.tx_type = HWTSTAMP_TX_ON;
3327 	else
3328 		config.tx_type = HWTSTAMP_TX_OFF;
3329 
3330 	config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ?
3331 			    HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE;
3332 
3333 	return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
3334 	       -EFAULT : 0;
3335 }
3336 
enetc_ioctl(struct net_device * ndev,struct ifreq * rq,int cmd)3337 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd)
3338 {
3339 	struct enetc_ndev_priv *priv = netdev_priv(ndev);
3340 
3341 	if (IS_ENABLED(CONFIG_FSL_ENETC_PTP_CLOCK)) {
3342 		if (cmd == SIOCSHWTSTAMP)
3343 			return enetc_hwtstamp_set(ndev, rq);
3344 		if (cmd == SIOCGHWTSTAMP)
3345 			return enetc_hwtstamp_get(ndev, rq);
3346 	}
3347 
3348 	if (!priv->phylink)
3349 		return -EOPNOTSUPP;
3350 
3351 	return phylink_mii_ioctl(priv->phylink, rq, cmd);
3352 }
3353 EXPORT_SYMBOL_GPL(enetc_ioctl);
3354 
enetc_int_vector_init(struct enetc_ndev_priv * priv,int i,int v_tx_rings)3355 static int enetc_int_vector_init(struct enetc_ndev_priv *priv, int i,
3356 				 int v_tx_rings)
3357 {
3358 	struct enetc_int_vector *v;
3359 	struct enetc_bdr *bdr;
3360 	int j, err;
3361 
3362 	v = kzalloc(struct_size(v, tx_ring, v_tx_rings), GFP_KERNEL);
3363 	if (!v)
3364 		return -ENOMEM;
3365 
3366 	priv->int_vector[i] = v;
3367 	bdr = &v->rx_ring;
3368 	bdr->index = i;
3369 	bdr->ndev = priv->ndev;
3370 	bdr->dev = priv->dev;
3371 	bdr->bd_count = priv->rx_bd_count;
3372 	bdr->buffer_offset = ENETC_RXB_PAD;
3373 	priv->rx_ring[i] = bdr;
3374 
3375 	err = __xdp_rxq_info_reg(&bdr->xdp.rxq, priv->ndev, i, 0,
3376 				 ENETC_RXB_DMA_SIZE_XDP);
3377 	if (err)
3378 		goto free_vector;
3379 
3380 	err = xdp_rxq_info_reg_mem_model(&bdr->xdp.rxq, MEM_TYPE_PAGE_SHARED,
3381 					 NULL);
3382 	if (err) {
3383 		xdp_rxq_info_unreg(&bdr->xdp.rxq);
3384 		goto free_vector;
3385 	}
3386 
3387 	/* init defaults for adaptive IC */
3388 	if (priv->ic_mode & ENETC_IC_RX_ADAPTIVE) {
3389 		v->rx_ictt = 0x1;
3390 		v->rx_dim_en = true;
3391 	}
3392 
3393 	INIT_WORK(&v->rx_dim.work, enetc_rx_dim_work);
3394 	netif_napi_add(priv->ndev, &v->napi, enetc_poll);
3395 	v->count_tx_rings = v_tx_rings;
3396 
3397 	for (j = 0; j < v_tx_rings; j++) {
3398 		int idx;
3399 
3400 		/* default tx ring mapping policy */
3401 		idx = priv->bdr_int_num * j + i;
3402 		__set_bit(idx, &v->tx_rings_map);
3403 		bdr = &v->tx_ring[j];
3404 		bdr->index = idx;
3405 		bdr->ndev = priv->ndev;
3406 		bdr->dev = priv->dev;
3407 		bdr->bd_count = priv->tx_bd_count;
3408 		priv->tx_ring[idx] = bdr;
3409 	}
3410 
3411 	return 0;
3412 
3413 free_vector:
3414 	priv->rx_ring[i] = NULL;
3415 	priv->int_vector[i] = NULL;
3416 	kfree(v);
3417 
3418 	return err;
3419 }
3420 
enetc_int_vector_destroy(struct enetc_ndev_priv * priv,int i)3421 static void enetc_int_vector_destroy(struct enetc_ndev_priv *priv, int i)
3422 {
3423 	struct enetc_int_vector *v = priv->int_vector[i];
3424 	struct enetc_bdr *rx_ring = &v->rx_ring;
3425 	int j, tx_ring_index;
3426 
3427 	xdp_rxq_info_unreg_mem_model(&rx_ring->xdp.rxq);
3428 	xdp_rxq_info_unreg(&rx_ring->xdp.rxq);
3429 	netif_napi_del(&v->napi);
3430 	cancel_work_sync(&v->rx_dim.work);
3431 
3432 	for (j = 0; j < v->count_tx_rings; j++) {
3433 		tx_ring_index = priv->bdr_int_num * j + i;
3434 		priv->tx_ring[tx_ring_index] = NULL;
3435 	}
3436 
3437 	priv->rx_ring[i] = NULL;
3438 	priv->int_vector[i] = NULL;
3439 	kfree(v);
3440 }
3441 
enetc_alloc_msix(struct enetc_ndev_priv * priv)3442 int enetc_alloc_msix(struct enetc_ndev_priv *priv)
3443 {
3444 	struct pci_dev *pdev = priv->si->pdev;
3445 	int v_tx_rings, v_remainder;
3446 	int num_stack_tx_queues;
3447 	int first_xdp_tx_ring;
3448 	int i, n, err, nvec;
3449 
3450 	nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num;
3451 	/* allocate MSIX for both messaging and Rx/Tx interrupts */
3452 	n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX);
3453 
3454 	if (n < 0)
3455 		return n;
3456 
3457 	if (n != nvec)
3458 		return -EPERM;
3459 
3460 	/* # of tx rings per int vector */
3461 	v_tx_rings = priv->num_tx_rings / priv->bdr_int_num;
3462 	v_remainder = priv->num_tx_rings % priv->bdr_int_num;
3463 
3464 	for (i = 0; i < priv->bdr_int_num; i++) {
3465 		/* Distribute the remaining TX rings to the first v_remainder
3466 		 * interrupt vectors
3467 		 */
3468 		int num_tx_rings = i < v_remainder ? v_tx_rings + 1 : v_tx_rings;
3469 
3470 		err = enetc_int_vector_init(priv, i, num_tx_rings);
3471 		if (err)
3472 			goto fail;
3473 	}
3474 
3475 	num_stack_tx_queues = enetc_num_stack_tx_queues(priv);
3476 
3477 	err = netif_set_real_num_tx_queues(priv->ndev, num_stack_tx_queues);
3478 	if (err)
3479 		goto fail;
3480 
3481 	err = netif_set_real_num_rx_queues(priv->ndev, priv->num_rx_rings);
3482 	if (err)
3483 		goto fail;
3484 
3485 	priv->min_num_stack_tx_queues = num_possible_cpus();
3486 	first_xdp_tx_ring = priv->num_tx_rings - num_possible_cpus();
3487 	priv->xdp_tx_ring = &priv->tx_ring[first_xdp_tx_ring];
3488 
3489 	return 0;
3490 
3491 fail:
3492 	while (i--)
3493 		enetc_int_vector_destroy(priv, i);
3494 
3495 	pci_free_irq_vectors(pdev);
3496 
3497 	return err;
3498 }
3499 EXPORT_SYMBOL_GPL(enetc_alloc_msix);
3500 
enetc_free_msix(struct enetc_ndev_priv * priv)3501 void enetc_free_msix(struct enetc_ndev_priv *priv)
3502 {
3503 	int i;
3504 
3505 	for (i = 0; i < priv->bdr_int_num; i++)
3506 		enetc_int_vector_destroy(priv, i);
3507 
3508 	/* disable all MSIX for this device */
3509 	pci_free_irq_vectors(priv->si->pdev);
3510 }
3511 EXPORT_SYMBOL_GPL(enetc_free_msix);
3512 
enetc_kfree_si(struct enetc_si * si)3513 static void enetc_kfree_si(struct enetc_si *si)
3514 {
3515 	char *p = (char *)si - si->pad;
3516 
3517 	kfree(p);
3518 }
3519 
enetc_detect_errata(struct enetc_si * si)3520 static void enetc_detect_errata(struct enetc_si *si)
3521 {
3522 	if (si->pdev->revision == ENETC_REV1)
3523 		si->errata = ENETC_ERR_VLAN_ISOL | ENETC_ERR_UCMCSWP;
3524 }
3525 
enetc_pci_probe(struct pci_dev * pdev,const char * name,int sizeof_priv)3526 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv)
3527 {
3528 	struct enetc_si *si, *p;
3529 	struct enetc_hw *hw;
3530 	size_t alloc_size;
3531 	int err, len;
3532 
3533 	pcie_flr(pdev);
3534 	err = pci_enable_device_mem(pdev);
3535 	if (err)
3536 		return dev_err_probe(&pdev->dev, err, "device enable failed\n");
3537 
3538 	/* set up for high or low dma */
3539 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
3540 	if (err) {
3541 		dev_err(&pdev->dev, "DMA configuration failed: 0x%x\n", err);
3542 		goto err_dma;
3543 	}
3544 
3545 	err = pci_request_mem_regions(pdev, name);
3546 	if (err) {
3547 		dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err);
3548 		goto err_pci_mem_reg;
3549 	}
3550 
3551 	pci_set_master(pdev);
3552 
3553 	alloc_size = sizeof(struct enetc_si);
3554 	if (sizeof_priv) {
3555 		/* align priv to 32B */
3556 		alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN);
3557 		alloc_size += sizeof_priv;
3558 	}
3559 	/* force 32B alignment for enetc_si */
3560 	alloc_size += ENETC_SI_ALIGN - 1;
3561 
3562 	p = kzalloc(alloc_size, GFP_KERNEL);
3563 	if (!p) {
3564 		err = -ENOMEM;
3565 		goto err_alloc_si;
3566 	}
3567 
3568 	si = PTR_ALIGN(p, ENETC_SI_ALIGN);
3569 	si->pad = (char *)si - (char *)p;
3570 
3571 	pci_set_drvdata(pdev, si);
3572 	si->pdev = pdev;
3573 	hw = &si->hw;
3574 
3575 	len = pci_resource_len(pdev, ENETC_BAR_REGS);
3576 	hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len);
3577 	if (!hw->reg) {
3578 		err = -ENXIO;
3579 		dev_err(&pdev->dev, "ioremap() failed\n");
3580 		goto err_ioremap;
3581 	}
3582 	if (len > ENETC_PORT_BASE)
3583 		hw->port = hw->reg + ENETC_PORT_BASE;
3584 	if (len > ENETC_GLOBAL_BASE)
3585 		hw->global = hw->reg + ENETC_GLOBAL_BASE;
3586 
3587 	enetc_detect_errata(si);
3588 
3589 	return 0;
3590 
3591 err_ioremap:
3592 	enetc_kfree_si(si);
3593 err_alloc_si:
3594 	pci_release_mem_regions(pdev);
3595 err_pci_mem_reg:
3596 err_dma:
3597 	pci_disable_device(pdev);
3598 
3599 	return err;
3600 }
3601 EXPORT_SYMBOL_GPL(enetc_pci_probe);
3602 
enetc_pci_remove(struct pci_dev * pdev)3603 void enetc_pci_remove(struct pci_dev *pdev)
3604 {
3605 	struct enetc_si *si = pci_get_drvdata(pdev);
3606 	struct enetc_hw *hw = &si->hw;
3607 
3608 	iounmap(hw->reg);
3609 	enetc_kfree_si(si);
3610 	pci_release_mem_regions(pdev);
3611 	pci_disable_device(pdev);
3612 }
3613 EXPORT_SYMBOL_GPL(enetc_pci_remove);
3614 
3615 static const struct enetc_drvdata enetc_pf_data = {
3616 	.sysclk_freq = ENETC_CLK_400M,
3617 	.pmac_offset = ENETC_PMAC_OFFSET,
3618 	.max_frags = ENETC_MAX_SKB_FRAGS,
3619 	.eth_ops = &enetc_pf_ethtool_ops,
3620 };
3621 
3622 static const struct enetc_drvdata enetc4_pf_data = {
3623 	.sysclk_freq = ENETC_CLK_333M,
3624 	.tx_csum = true,
3625 	.max_frags = ENETC4_MAX_SKB_FRAGS,
3626 	.pmac_offset = ENETC4_PMAC_OFFSET,
3627 	.eth_ops = &enetc4_pf_ethtool_ops,
3628 };
3629 
3630 static const struct enetc_drvdata enetc_vf_data = {
3631 	.sysclk_freq = ENETC_CLK_400M,
3632 	.max_frags = ENETC_MAX_SKB_FRAGS,
3633 	.eth_ops = &enetc_vf_ethtool_ops,
3634 };
3635 
3636 static const struct enetc_platform_info enetc_info[] = {
3637 	{ .revision = ENETC_REV_1_0,
3638 	  .dev_id = ENETC_DEV_ID_PF,
3639 	  .data = &enetc_pf_data,
3640 	},
3641 	{ .revision = ENETC_REV_4_1,
3642 	  .dev_id = NXP_ENETC_PF_DEV_ID,
3643 	  .data = &enetc4_pf_data,
3644 	},
3645 	{ .revision = ENETC_REV_1_0,
3646 	  .dev_id = ENETC_DEV_ID_VF,
3647 	  .data = &enetc_vf_data,
3648 	},
3649 };
3650 
enetc_get_driver_data(struct enetc_si * si)3651 int enetc_get_driver_data(struct enetc_si *si)
3652 {
3653 	u16 dev_id = si->pdev->device;
3654 	int i;
3655 
3656 	for (i = 0; i < ARRAY_SIZE(enetc_info); i++) {
3657 		if (si->revision == enetc_info[i].revision &&
3658 		    dev_id == enetc_info[i].dev_id) {
3659 			si->drvdata = enetc_info[i].data;
3660 
3661 			return 0;
3662 		}
3663 	}
3664 
3665 	return -ERANGE;
3666 }
3667 EXPORT_SYMBOL_GPL(enetc_get_driver_data);
3668 
3669 MODULE_DESCRIPTION("NXP ENETC Ethernet driver");
3670 MODULE_LICENSE("Dual BSD/GPL");
3671