1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell RVU Ethernet driver
3  *
4  * Copyright (C) 2020 Marvell.
5  *
6  */
7 
8 #include <linux/etherdevice.h>
9 #include <net/ip.h>
10 #include <net/tso.h>
11 #include <linux/bpf.h>
12 #include <linux/bpf_trace.h>
13 #include <net/ip6_checksum.h>
14 #include <net/xfrm.h>
15 #include <net/xdp.h>
16 
17 #include "otx2_reg.h"
18 #include "otx2_common.h"
19 #include "otx2_struct.h"
20 #include "otx2_txrx.h"
21 #include "otx2_ptp.h"
22 #include "cn10k.h"
23 #include "otx2_xsk.h"
24 
25 #define CQE_ADDR(CQ, idx) ((CQ)->cqe_base + ((CQ)->cqe_size * (idx)))
26 #define PTP_PORT	        0x13F
27 /* PTPv2 header Original Timestamp starts at byte offset 34 and
28  * contains 6 byte seconds field and 4 byte nano seconds field.
29  */
30 #define PTP_SYNC_SEC_OFFSET	34
31 
32 DEFINE_STATIC_KEY_FALSE(cn10k_ipsec_sa_enabled);
33 
otx2_get_free_sqe(struct otx2_snd_queue * sq)34 static int otx2_get_free_sqe(struct otx2_snd_queue *sq)
35 {
36 	return (sq->cons_head - sq->head - 1 + sq->sqe_cnt)
37 		& (sq->sqe_cnt - 1);
38 }
39 
40 static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf,
41 				     struct bpf_prog *prog,
42 				     struct nix_cqe_rx_s *cqe,
43 				     struct otx2_cq_queue *cq,
44 				     u32 *metasize, bool *need_xdp_flush);
45 
otx2_sq_set_sqe_base(struct otx2_snd_queue * sq,struct sk_buff * skb)46 static void otx2_sq_set_sqe_base(struct otx2_snd_queue *sq,
47 				 struct sk_buff *skb)
48 {
49 	if (static_branch_unlikely(&cn10k_ipsec_sa_enabled) &&
50 	    (xfrm_offload(skb)))
51 		sq->sqe_base = sq->sqe_ring->base + sq->sqe_size +
52 				(sq->head * (sq->sqe_size * 2));
53 	else
54 		sq->sqe_base = sq->sqe->base;
55 }
56 
otx2_nix_cq_op_status(struct otx2_nic * pfvf,struct otx2_cq_queue * cq)57 static int otx2_nix_cq_op_status(struct otx2_nic *pfvf,
58 				 struct otx2_cq_queue *cq)
59 {
60 	u64 incr = (u64)(cq->cq_idx) << 32;
61 	u64 status;
62 
63 	status = otx2_atomic64_fetch_add(incr, pfvf->cq_op_addr);
64 
65 	if (unlikely(status & BIT_ULL(CQ_OP_STAT_OP_ERR) ||
66 		     status & BIT_ULL(CQ_OP_STAT_CQ_ERR))) {
67 		dev_err(pfvf->dev, "CQ stopped due to error");
68 		return -EINVAL;
69 	}
70 
71 	cq->cq_tail = status & 0xFFFFF;
72 	cq->cq_head = (status >> 20) & 0xFFFFF;
73 	if (cq->cq_tail < cq->cq_head)
74 		cq->pend_cqe = (cq->cqe_cnt - cq->cq_head) +
75 				cq->cq_tail;
76 	else
77 		cq->pend_cqe = cq->cq_tail - cq->cq_head;
78 
79 	return 0;
80 }
81 
otx2_get_next_cqe(struct otx2_cq_queue * cq)82 static struct nix_cqe_hdr_s *otx2_get_next_cqe(struct otx2_cq_queue *cq)
83 {
84 	struct nix_cqe_hdr_s *cqe_hdr;
85 
86 	cqe_hdr = (struct nix_cqe_hdr_s *)CQE_ADDR(cq, cq->cq_head);
87 	if (cqe_hdr->cqe_type == NIX_XQE_TYPE_INVALID)
88 		return NULL;
89 
90 	cq->cq_head++;
91 	cq->cq_head &= (cq->cqe_cnt - 1);
92 
93 	return cqe_hdr;
94 }
95 
frag_num(unsigned int i)96 static unsigned int frag_num(unsigned int i)
97 {
98 #ifdef __BIG_ENDIAN
99 	return (i & ~3) + 3 - (i & 3);
100 #else
101 	return i;
102 #endif
103 }
104 
otx2_xdp_snd_pkt_handler(struct otx2_nic * pfvf,struct otx2_snd_queue * sq,struct nix_cqe_tx_s * cqe,int * xsk_frames)105 static void otx2_xdp_snd_pkt_handler(struct otx2_nic *pfvf,
106 				     struct otx2_snd_queue *sq,
107 				     struct nix_cqe_tx_s *cqe,
108 				     int *xsk_frames)
109 {
110 	struct nix_send_comp_s *snd_comp = &cqe->comp;
111 	struct sg_list *sg;
112 
113 	sg = &sq->sg[snd_comp->sqe_id];
114 	if (sg->flags & OTX2_AF_XDP_FRAME) {
115 		(*xsk_frames)++;
116 		return;
117 	}
118 
119 	if (sg->flags & OTX2_XDP_REDIRECT)
120 		otx2_dma_unmap_page(pfvf, sg->dma_addr[0], sg->size[0], DMA_TO_DEVICE);
121 	xdp_return_frame((struct xdp_frame *)sg->skb);
122 	sg->skb = (u64)NULL;
123 }
124 
otx2_snd_pkt_handler(struct otx2_nic * pfvf,struct otx2_cq_queue * cq,struct otx2_snd_queue * sq,struct nix_cqe_tx_s * cqe,int budget,int * tx_pkts,int * tx_bytes)125 static void otx2_snd_pkt_handler(struct otx2_nic *pfvf,
126 				 struct otx2_cq_queue *cq,
127 				 struct otx2_snd_queue *sq,
128 				 struct nix_cqe_tx_s *cqe,
129 				 int budget, int *tx_pkts, int *tx_bytes)
130 {
131 	struct nix_send_comp_s *snd_comp = &cqe->comp;
132 	struct skb_shared_hwtstamps ts;
133 	struct sk_buff *skb = NULL;
134 	u64 timestamp, tsns;
135 	struct sg_list *sg;
136 	int err;
137 
138 	if (unlikely(snd_comp->status) && netif_msg_tx_err(pfvf))
139 		net_err_ratelimited("%s: TX%d: Error in send CQ status:%x\n",
140 				    pfvf->netdev->name, cq->cint_idx,
141 				    snd_comp->status);
142 
143 	sg = &sq->sg[snd_comp->sqe_id];
144 	skb = (struct sk_buff *)sg->skb;
145 	if (unlikely(!skb))
146 		return;
147 
148 	if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) {
149 		timestamp = ((u64 *)sq->timestamps->base)[snd_comp->sqe_id];
150 		if (timestamp != 1) {
151 			timestamp = pfvf->ptp->convert_tx_ptp_tstmp(timestamp);
152 			err = otx2_ptp_tstamp2time(pfvf, timestamp, &tsns);
153 			if (!err) {
154 				memset(&ts, 0, sizeof(ts));
155 				ts.hwtstamp = ns_to_ktime(tsns);
156 				skb_tstamp_tx(skb, &ts);
157 			}
158 		}
159 	}
160 
161 	*tx_bytes += skb->len;
162 	(*tx_pkts)++;
163 	otx2_dma_unmap_skb_frags(pfvf, sg);
164 	napi_consume_skb(skb, budget);
165 	sg->skb = (u64)NULL;
166 }
167 
otx2_set_rxtstamp(struct otx2_nic * pfvf,struct sk_buff * skb,void * data)168 static void otx2_set_rxtstamp(struct otx2_nic *pfvf,
169 			      struct sk_buff *skb, void *data)
170 {
171 	u64 timestamp, tsns;
172 	int err;
173 
174 	if (!(pfvf->flags & OTX2_FLAG_RX_TSTAMP_ENABLED))
175 		return;
176 
177 	timestamp = pfvf->ptp->convert_rx_ptp_tstmp(*(u64 *)data);
178 	/* The first 8 bytes is the timestamp */
179 	err = otx2_ptp_tstamp2time(pfvf, timestamp, &tsns);
180 	if (err)
181 		return;
182 
183 	skb_hwtstamps(skb)->hwtstamp = ns_to_ktime(tsns);
184 }
185 
otx2_skb_add_frag(struct otx2_nic * pfvf,struct sk_buff * skb,u64 iova,int len,struct nix_rx_parse_s * parse,int qidx)186 static bool otx2_skb_add_frag(struct otx2_nic *pfvf, struct sk_buff *skb,
187 			      u64 iova, int len, struct nix_rx_parse_s *parse,
188 			      int qidx)
189 {
190 	struct page *page;
191 	int off = 0;
192 	void *va;
193 
194 	va = phys_to_virt(otx2_iova_to_phys(pfvf->iommu_domain, iova));
195 
196 	if (likely(!skb_shinfo(skb)->nr_frags)) {
197 		/* Check if data starts at some nonzero offset
198 		 * from the start of the buffer.  For now the
199 		 * only possible offset is 8 bytes in the case
200 		 * where packet is prepended by a timestamp.
201 		 */
202 		if (parse->laptr) {
203 			otx2_set_rxtstamp(pfvf, skb, va);
204 			off = OTX2_HW_TIMESTAMP_LEN;
205 		}
206 	}
207 
208 	page = virt_to_page(va);
209 	if (likely(skb_shinfo(skb)->nr_frags < MAX_SKB_FRAGS)) {
210 		skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,
211 				va - page_address(page) + off,
212 				len - off, pfvf->rbsize);
213 		return true;
214 	}
215 
216 	/* If more than MAX_SKB_FRAGS fragments are received then
217 	 * give back those buffer pointers to hardware for reuse.
218 	 */
219 	pfvf->hw_ops->aura_freeptr(pfvf, qidx, iova & ~0x07ULL);
220 
221 	return false;
222 }
223 
otx2_set_rxhash(struct otx2_nic * pfvf,struct nix_cqe_rx_s * cqe,struct sk_buff * skb)224 static void otx2_set_rxhash(struct otx2_nic *pfvf,
225 			    struct nix_cqe_rx_s *cqe, struct sk_buff *skb)
226 {
227 	enum pkt_hash_types hash_type = PKT_HASH_TYPE_NONE;
228 	struct otx2_rss_info *rss;
229 	u32 hash = 0;
230 
231 	if (!(pfvf->netdev->features & NETIF_F_RXHASH))
232 		return;
233 
234 	rss = &pfvf->hw.rss_info;
235 	if (rss->flowkey_cfg) {
236 		if (rss->flowkey_cfg &
237 		    ~(NIX_FLOW_KEY_TYPE_IPV4 | NIX_FLOW_KEY_TYPE_IPV6))
238 			hash_type = PKT_HASH_TYPE_L4;
239 		else
240 			hash_type = PKT_HASH_TYPE_L3;
241 		hash = cqe->hdr.flow_tag;
242 	}
243 	skb_set_hash(skb, hash, hash_type);
244 }
245 
otx2_free_rcv_seg(struct otx2_nic * pfvf,struct nix_cqe_rx_s * cqe,int qidx)246 static void otx2_free_rcv_seg(struct otx2_nic *pfvf, struct nix_cqe_rx_s *cqe,
247 			      int qidx)
248 {
249 	struct nix_rx_sg_s *sg = &cqe->sg;
250 	void *end, *start;
251 	u64 *seg_addr;
252 	int seg;
253 
254 	start = (void *)sg;
255 	end = start + ((cqe->parse.desc_sizem1 + 1) * 16);
256 	while (start < end) {
257 		sg = (struct nix_rx_sg_s *)start;
258 		seg_addr = &sg->seg_addr;
259 		for (seg = 0; seg < sg->segs; seg++, seg_addr++)
260 			pfvf->hw_ops->aura_freeptr(pfvf, qidx,
261 						   *seg_addr & ~0x07ULL);
262 		start += sizeof(*sg);
263 	}
264 }
265 
otx2_check_rcv_errors(struct otx2_nic * pfvf,struct nix_cqe_rx_s * cqe,int qidx)266 static bool otx2_check_rcv_errors(struct otx2_nic *pfvf,
267 				  struct nix_cqe_rx_s *cqe, int qidx)
268 {
269 	struct otx2_drv_stats *stats = &pfvf->hw.drv_stats;
270 	struct nix_rx_parse_s *parse = &cqe->parse;
271 
272 	if (netif_msg_rx_err(pfvf))
273 		netdev_err(pfvf->netdev,
274 			   "RQ%d: Error pkt with errlev:0x%x errcode:0x%x\n",
275 			   qidx, parse->errlev, parse->errcode);
276 
277 	if (parse->errlev == NPC_ERRLVL_RE) {
278 		switch (parse->errcode) {
279 		case ERRCODE_FCS:
280 		case ERRCODE_FCS_RCV:
281 			atomic_inc(&stats->rx_fcs_errs);
282 			break;
283 		case ERRCODE_UNDERSIZE:
284 			atomic_inc(&stats->rx_undersize_errs);
285 			break;
286 		case ERRCODE_OVERSIZE:
287 			atomic_inc(&stats->rx_oversize_errs);
288 			break;
289 		case ERRCODE_OL2_LEN_MISMATCH:
290 			atomic_inc(&stats->rx_len_errs);
291 			break;
292 		default:
293 			atomic_inc(&stats->rx_other_errs);
294 			break;
295 		}
296 	} else if (parse->errlev == NPC_ERRLVL_NIX) {
297 		switch (parse->errcode) {
298 		case ERRCODE_OL3_LEN:
299 		case ERRCODE_OL4_LEN:
300 		case ERRCODE_IL3_LEN:
301 		case ERRCODE_IL4_LEN:
302 			atomic_inc(&stats->rx_len_errs);
303 			break;
304 		case ERRCODE_OL4_CSUM:
305 		case ERRCODE_IL4_CSUM:
306 			atomic_inc(&stats->rx_csum_errs);
307 			break;
308 		default:
309 			atomic_inc(&stats->rx_other_errs);
310 			break;
311 		}
312 	} else {
313 		atomic_inc(&stats->rx_other_errs);
314 		/* For now ignore all the NPC parser errors and
315 		 * pass the packets to stack.
316 		 */
317 		return false;
318 	}
319 
320 	/* If RXALL is enabled pass on packets to stack. */
321 	if (pfvf->netdev->features & NETIF_F_RXALL)
322 		return false;
323 
324 	/* Free buffer back to pool */
325 	if (cqe->sg.segs)
326 		otx2_free_rcv_seg(pfvf, cqe, qidx);
327 	return true;
328 }
329 
otx2_rcv_pkt_handler(struct otx2_nic * pfvf,struct napi_struct * napi,struct otx2_cq_queue * cq,struct nix_cqe_rx_s * cqe,bool * need_xdp_flush)330 static void otx2_rcv_pkt_handler(struct otx2_nic *pfvf,
331 				 struct napi_struct *napi,
332 				 struct otx2_cq_queue *cq,
333 				 struct nix_cqe_rx_s *cqe, bool *need_xdp_flush)
334 {
335 	struct nix_rx_parse_s *parse = &cqe->parse;
336 	struct nix_rx_sg_s *sg = &cqe->sg;
337 	struct sk_buff *skb = NULL;
338 	void *end, *start;
339 	u32 metasize = 0;
340 	u64 *seg_addr;
341 	u16 *seg_size;
342 	int seg;
343 
344 	if (unlikely(parse->errlev || parse->errcode)) {
345 		if (otx2_check_rcv_errors(pfvf, cqe, cq->cq_idx))
346 			return;
347 	}
348 
349 	if (pfvf->xdp_prog)
350 		if (otx2_xdp_rcv_pkt_handler(pfvf, pfvf->xdp_prog, cqe, cq,
351 					     &metasize, need_xdp_flush))
352 			return;
353 
354 	skb = napi_get_frags(napi);
355 	if (unlikely(!skb))
356 		return;
357 
358 	start = (void *)sg;
359 	end = start + ((cqe->parse.desc_sizem1 + 1) * 16);
360 	while (start < end) {
361 		sg = (struct nix_rx_sg_s *)start;
362 		seg_addr = &sg->seg_addr;
363 		seg_size = (void *)sg;
364 		for (seg = 0; seg < sg->segs; seg++, seg_addr++) {
365 			if (otx2_skb_add_frag(pfvf, skb, *seg_addr,
366 					      seg_size[seg], parse, cq->cq_idx))
367 				cq->pool_ptrs++;
368 		}
369 		start += sizeof(*sg);
370 	}
371 	otx2_set_rxhash(pfvf, cqe, skb);
372 
373 	if (!(pfvf->flags & OTX2_FLAG_REP_MODE_ENABLED)) {
374 		skb_record_rx_queue(skb, cq->cq_idx);
375 		if (pfvf->netdev->features & NETIF_F_RXCSUM)
376 			skb->ip_summed = CHECKSUM_UNNECESSARY;
377 	}
378 
379 	if (pfvf->flags & OTX2_FLAG_TC_MARK_ENABLED)
380 		skb->mark = parse->match_id;
381 
382 	skb_mark_for_recycle(skb);
383 	if (metasize)
384 		skb_metadata_set(skb, metasize);
385 
386 	napi_gro_frags(napi);
387 }
388 
otx2_rx_napi_handler(struct otx2_nic * pfvf,struct napi_struct * napi,struct otx2_cq_queue * cq,int budget)389 static int otx2_rx_napi_handler(struct otx2_nic *pfvf,
390 				struct napi_struct *napi,
391 				struct otx2_cq_queue *cq, int budget)
392 {
393 	bool need_xdp_flush = false;
394 	struct nix_cqe_rx_s *cqe;
395 	int processed_cqe = 0;
396 
397 	if (cq->pend_cqe >= budget)
398 		goto process_cqe;
399 
400 	if (otx2_nix_cq_op_status(pfvf, cq) || !cq->pend_cqe)
401 		return 0;
402 
403 process_cqe:
404 	while (likely(processed_cqe < budget) && cq->pend_cqe) {
405 		cqe = (struct nix_cqe_rx_s *)CQE_ADDR(cq, cq->cq_head);
406 		if (cqe->hdr.cqe_type == NIX_XQE_TYPE_INVALID ||
407 		    !cqe->sg.seg_addr) {
408 			if (!processed_cqe)
409 				return 0;
410 			break;
411 		}
412 		cq->cq_head++;
413 		cq->cq_head &= (cq->cqe_cnt - 1);
414 
415 		otx2_rcv_pkt_handler(pfvf, napi, cq, cqe, &need_xdp_flush);
416 
417 		cqe->hdr.cqe_type = NIX_XQE_TYPE_INVALID;
418 		cqe->sg.seg_addr = 0x00;
419 		processed_cqe++;
420 		cq->pend_cqe--;
421 	}
422 	if (need_xdp_flush)
423 		xdp_do_flush();
424 
425 	/* Free CQEs to HW */
426 	otx2_write64(pfvf, NIX_LF_CQ_OP_DOOR,
427 		     ((u64)cq->cq_idx << 32) | processed_cqe);
428 
429 	return processed_cqe;
430 }
431 
otx2_refill_pool_ptrs(void * dev,struct otx2_cq_queue * cq)432 int otx2_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq)
433 {
434 	struct otx2_nic *pfvf = dev;
435 	int cnt = cq->pool_ptrs;
436 	dma_addr_t bufptr;
437 
438 	while (cq->pool_ptrs) {
439 		if (otx2_alloc_buffer(pfvf, cq, &bufptr))
440 			break;
441 		otx2_aura_freeptr(pfvf, cq->cq_idx, bufptr + OTX2_HEAD_ROOM);
442 		cq->pool_ptrs--;
443 	}
444 
445 	return cnt - cq->pool_ptrs;
446 }
447 
otx2_zc_submit_pkts(struct otx2_nic * pfvf,struct xsk_buff_pool * xsk_pool,int * xsk_frames,int qidx,int budget)448 static void otx2_zc_submit_pkts(struct otx2_nic *pfvf, struct xsk_buff_pool *xsk_pool,
449 				int *xsk_frames, int qidx, int budget)
450 {
451 	if (*xsk_frames)
452 		xsk_tx_completed(xsk_pool, *xsk_frames);
453 
454 	if (xsk_uses_need_wakeup(xsk_pool))
455 		xsk_set_tx_need_wakeup(xsk_pool);
456 
457 	otx2_zc_napi_handler(pfvf, xsk_pool, qidx, budget);
458 }
459 
otx2_tx_napi_handler(struct otx2_nic * pfvf,struct otx2_cq_queue * cq,int budget)460 static int otx2_tx_napi_handler(struct otx2_nic *pfvf,
461 				struct otx2_cq_queue *cq, int budget)
462 {
463 	int tx_pkts = 0, tx_bytes = 0, qidx;
464 	struct otx2_snd_queue *sq;
465 	struct nix_cqe_tx_s *cqe;
466 	struct net_device *ndev;
467 	int processed_cqe = 0;
468 	int xsk_frames = 0;
469 
470 	qidx = cq->cq_idx - pfvf->hw.rx_queues;
471 	sq = &pfvf->qset.sq[qidx];
472 
473 	if (cq->pend_cqe >= budget)
474 		goto process_cqe;
475 
476 	if (otx2_nix_cq_op_status(pfvf, cq) || !cq->pend_cqe) {
477 		if (sq->xsk_pool)
478 			otx2_zc_submit_pkts(pfvf, sq->xsk_pool, &xsk_frames,
479 					    qidx, budget);
480 		return 0;
481 	}
482 
483 process_cqe:
484 
485 	while (likely(processed_cqe < budget) && cq->pend_cqe) {
486 		cqe = (struct nix_cqe_tx_s *)otx2_get_next_cqe(cq);
487 		if (unlikely(!cqe)) {
488 			if (!processed_cqe)
489 				return 0;
490 			break;
491 		}
492 
493 		if (cq->cq_type == CQ_XDP)
494 			otx2_xdp_snd_pkt_handler(pfvf, sq, cqe, &xsk_frames);
495 		else
496 			otx2_snd_pkt_handler(pfvf, cq, &pfvf->qset.sq[qidx],
497 					     cqe, budget, &tx_pkts, &tx_bytes);
498 
499 		cqe->hdr.cqe_type = NIX_XQE_TYPE_INVALID;
500 		processed_cqe++;
501 		cq->pend_cqe--;
502 
503 		sq->cons_head++;
504 		sq->cons_head &= (sq->sqe_cnt - 1);
505 	}
506 
507 	/* Free CQEs to HW */
508 	otx2_write64(pfvf, NIX_LF_CQ_OP_DOOR,
509 		     ((u64)cq->cq_idx << 32) | processed_cqe);
510 
511 #if IS_ENABLED(CONFIG_RVU_ESWITCH)
512 	if (pfvf->flags & OTX2_FLAG_REP_MODE_ENABLED)
513 		ndev = pfvf->reps[qidx]->netdev;
514 	else
515 #endif
516 		ndev = pfvf->netdev;
517 
518 	if (likely(tx_pkts)) {
519 		struct netdev_queue *txq;
520 
521 		qidx = cq->cq_idx - pfvf->hw.rx_queues;
522 
523 		if (qidx >= pfvf->hw.tx_queues)
524 			qidx -= pfvf->hw.xdp_queues;
525 		if (pfvf->flags & OTX2_FLAG_REP_MODE_ENABLED)
526 			qidx = 0;
527 		txq = netdev_get_tx_queue(ndev, qidx);
528 		netdev_tx_completed_queue(txq, tx_pkts, tx_bytes);
529 		/* Check if queue was stopped earlier due to ring full */
530 		smp_mb();
531 		if (netif_tx_queue_stopped(txq) &&
532 		    netif_carrier_ok(ndev))
533 			netif_tx_wake_queue(txq);
534 	}
535 
536 	if (sq->xsk_pool)
537 		otx2_zc_submit_pkts(pfvf, sq->xsk_pool, &xsk_frames, qidx, budget);
538 
539 	return 0;
540 }
541 
otx2_adjust_adaptive_coalese(struct otx2_nic * pfvf,struct otx2_cq_poll * cq_poll)542 static void otx2_adjust_adaptive_coalese(struct otx2_nic *pfvf, struct otx2_cq_poll *cq_poll)
543 {
544 	struct dim_sample dim_sample = { 0 };
545 	u64 rx_frames, rx_bytes;
546 	u64 tx_frames, tx_bytes;
547 
548 	rx_frames = OTX2_GET_RX_STATS(RX_BCAST) + OTX2_GET_RX_STATS(RX_MCAST) +
549 		OTX2_GET_RX_STATS(RX_UCAST);
550 	rx_bytes = OTX2_GET_RX_STATS(RX_OCTS);
551 	tx_bytes = OTX2_GET_TX_STATS(TX_OCTS);
552 	tx_frames = OTX2_GET_TX_STATS(TX_UCAST);
553 
554 	dim_update_sample(pfvf->napi_events,
555 			  rx_frames + tx_frames,
556 			  rx_bytes + tx_bytes,
557 			  &dim_sample);
558 	net_dim(&cq_poll->dim, &dim_sample);
559 }
560 
otx2_napi_handler(struct napi_struct * napi,int budget)561 int otx2_napi_handler(struct napi_struct *napi, int budget)
562 {
563 	struct otx2_cq_queue *rx_cq = NULL;
564 	struct otx2_cq_queue *cq = NULL;
565 	struct otx2_pool *pool = NULL;
566 	struct otx2_cq_poll *cq_poll;
567 	int workdone = 0, cq_idx, i;
568 	struct otx2_qset *qset;
569 	struct otx2_nic *pfvf;
570 	int filled_cnt = -1;
571 
572 	cq_poll = container_of(napi, struct otx2_cq_poll, napi);
573 	pfvf = (struct otx2_nic *)cq_poll->dev;
574 	qset = &pfvf->qset;
575 
576 	for (i = 0; i < CQS_PER_CINT; i++) {
577 		cq_idx = cq_poll->cq_ids[i];
578 		if (unlikely(cq_idx == CINT_INVALID_CQ))
579 			continue;
580 		cq = &qset->cq[cq_idx];
581 		if (cq->cq_type == CQ_RX) {
582 			rx_cq = cq;
583 			workdone += otx2_rx_napi_handler(pfvf, napi,
584 							 cq, budget);
585 		} else {
586 			workdone += otx2_tx_napi_handler(pfvf, cq, budget);
587 		}
588 	}
589 
590 	if (rx_cq && rx_cq->pool_ptrs)
591 		filled_cnt = pfvf->hw_ops->refill_pool_ptrs(pfvf, rx_cq);
592 
593 	/* Clear the IRQ */
594 	otx2_write64(pfvf, NIX_LF_CINTX_INT(cq_poll->cint_idx), BIT_ULL(0));
595 
596 	if (workdone < budget && napi_complete_done(napi, workdone)) {
597 		/* If interface is going down, don't re-enable IRQ */
598 		if (pfvf->flags & OTX2_FLAG_INTF_DOWN)
599 			return workdone;
600 
601 		/* Adjust irq coalese using net_dim */
602 		if (pfvf->flags & OTX2_FLAG_ADPTV_INT_COAL_ENABLED)
603 			otx2_adjust_adaptive_coalese(pfvf, cq_poll);
604 
605 		if (likely(cq))
606 			pool = &pfvf->qset.pool[cq->cq_idx];
607 
608 		if (unlikely(!filled_cnt)) {
609 			struct refill_work *work;
610 			struct delayed_work *dwork;
611 
612 			if (likely(cq)) {
613 				work = &pfvf->refill_wrk[cq->cq_idx];
614 				dwork = &work->pool_refill_work;
615 				/* Schedule a task if no other task is running */
616 				if (!cq->refill_task_sched) {
617 					work->napi = napi;
618 					cq->refill_task_sched = true;
619 					schedule_delayed_work(dwork,
620 							      msecs_to_jiffies(100));
621 				}
622 				/* Call wake-up for not able to fill buffers */
623 				if (pool->xsk_pool)
624 					xsk_set_rx_need_wakeup(pool->xsk_pool);
625 			}
626 		} else {
627 			/* Clear wake-up, since buffers are filled successfully */
628 			if (pool && pool->xsk_pool)
629 				xsk_clear_rx_need_wakeup(pool->xsk_pool);
630 			/* Re-enable interrupts */
631 			otx2_write64(pfvf,
632 				     NIX_LF_CINTX_ENA_W1S(cq_poll->cint_idx),
633 				     BIT_ULL(0));
634 		}
635 	}
636 	return workdone;
637 }
638 EXPORT_SYMBOL(otx2_napi_handler);
639 
otx2_sqe_flush(void * dev,struct otx2_snd_queue * sq,int size,int qidx)640 void otx2_sqe_flush(void *dev, struct otx2_snd_queue *sq,
641 		    int size, int qidx)
642 {
643 	u64 status;
644 
645 	/* Packet data stores should finish before SQE is flushed to HW */
646 	dma_wmb();
647 
648 	do {
649 		memcpy(sq->lmt_addr, sq->sqe_base, size);
650 		status = otx2_lmt_flush(sq->io_addr);
651 	} while (status == 0);
652 
653 	sq->head++;
654 	sq->head &= (sq->sqe_cnt - 1);
655 }
656 
657 /* Add SQE scatter/gather subdescriptor structure */
otx2_sqe_add_sg(struct otx2_nic * pfvf,struct otx2_snd_queue * sq,struct sk_buff * skb,int num_segs,int * offset)658 static bool otx2_sqe_add_sg(struct otx2_nic *pfvf, struct otx2_snd_queue *sq,
659 			    struct sk_buff *skb, int num_segs, int *offset)
660 {
661 	struct nix_sqe_sg_s *sg = NULL;
662 	u64 dma_addr, *iova = NULL;
663 	u16 *sg_lens = NULL;
664 	int seg, len;
665 
666 	sq->sg[sq->head].num_segs = 0;
667 
668 	for (seg = 0; seg < num_segs; seg++) {
669 		if ((seg % MAX_SEGS_PER_SG) == 0) {
670 			sg = (struct nix_sqe_sg_s *)(sq->sqe_base + *offset);
671 			sg->ld_type = NIX_SEND_LDTYPE_LDD;
672 			sg->subdc = NIX_SUBDC_SG;
673 			sg->segs = 0;
674 			sg_lens = (void *)sg;
675 			iova = (void *)sg + sizeof(*sg);
676 			/* Next subdc always starts at a 16byte boundary.
677 			 * So if sg->segs is whether 2 or 3, offset += 16bytes.
678 			 */
679 			if ((num_segs - seg) >= (MAX_SEGS_PER_SG - 1))
680 				*offset += sizeof(*sg) + (3 * sizeof(u64));
681 			else
682 				*offset += sizeof(*sg) + sizeof(u64);
683 		}
684 		dma_addr = otx2_dma_map_skb_frag(pfvf, skb, seg, &len);
685 		if (dma_mapping_error(pfvf->dev, dma_addr))
686 			return false;
687 
688 		sg_lens[frag_num(seg % MAX_SEGS_PER_SG)] = len;
689 		sg->segs++;
690 		*iova++ = dma_addr;
691 
692 		/* Save DMA mapping info for later unmapping */
693 		sq->sg[sq->head].dma_addr[seg] = dma_addr;
694 		sq->sg[sq->head].size[seg] = len;
695 		sq->sg[sq->head].num_segs++;
696 	}
697 
698 	sq->sg[sq->head].skb = (u64)skb;
699 	return true;
700 }
701 
702 /* Add SQE extended header subdescriptor */
otx2_sqe_add_ext(struct otx2_nic * pfvf,struct otx2_snd_queue * sq,struct sk_buff * skb,int * offset)703 static void otx2_sqe_add_ext(struct otx2_nic *pfvf, struct otx2_snd_queue *sq,
704 			     struct sk_buff *skb, int *offset)
705 {
706 	struct nix_sqe_ext_s *ext;
707 
708 	ext = (struct nix_sqe_ext_s *)(sq->sqe_base + *offset);
709 	ext->subdc = NIX_SUBDC_EXT;
710 	if (skb_shinfo(skb)->gso_size) {
711 		ext->lso = 1;
712 		ext->lso_sb = skb_tcp_all_headers(skb);
713 		ext->lso_mps = skb_shinfo(skb)->gso_size;
714 
715 		/* Only TSOv4 and TSOv6 GSO offloads are supported */
716 		if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
717 			ext->lso_format = pfvf->hw.lso_tsov4_idx;
718 
719 			/* HW adds payload size to 'ip_hdr->tot_len' while
720 			 * sending TSO segment, hence set payload length
721 			 * in IP header of the packet to just header length.
722 			 */
723 			ip_hdr(skb)->tot_len =
724 				htons(ext->lso_sb - skb_network_offset(skb));
725 		} else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) {
726 			ext->lso_format = pfvf->hw.lso_tsov6_idx;
727 			ipv6_hdr(skb)->payload_len = htons(tcp_hdrlen(skb));
728 		} else if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) {
729 			__be16 l3_proto = vlan_get_protocol(skb);
730 			struct udphdr *udph = udp_hdr(skb);
731 			__be16 iplen;
732 
733 			ext->lso_sb = skb_transport_offset(skb) +
734 					sizeof(struct udphdr);
735 
736 			/* HW adds payload size to length fields in IP and
737 			 * UDP headers while segmentation, hence adjust the
738 			 * lengths to just header sizes.
739 			 */
740 			iplen = htons(ext->lso_sb - skb_network_offset(skb));
741 			if (l3_proto == htons(ETH_P_IP)) {
742 				ip_hdr(skb)->tot_len = iplen;
743 				ext->lso_format = pfvf->hw.lso_udpv4_idx;
744 			} else {
745 				ipv6_hdr(skb)->payload_len = iplen;
746 				ext->lso_format = pfvf->hw.lso_udpv6_idx;
747 			}
748 
749 			udph->len = htons(sizeof(struct udphdr));
750 		}
751 	} else if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) {
752 		ext->tstmp = 1;
753 	}
754 
755 #define OTX2_VLAN_PTR_OFFSET     (ETH_HLEN - ETH_TLEN)
756 	if (skb_vlan_tag_present(skb)) {
757 		if (skb->vlan_proto == htons(ETH_P_8021Q)) {
758 			ext->vlan1_ins_ena = 1;
759 			ext->vlan1_ins_ptr = OTX2_VLAN_PTR_OFFSET;
760 			ext->vlan1_ins_tci = skb_vlan_tag_get(skb);
761 		} else if (skb->vlan_proto == htons(ETH_P_8021AD)) {
762 			ext->vlan0_ins_ena = 1;
763 			ext->vlan0_ins_ptr = OTX2_VLAN_PTR_OFFSET;
764 			ext->vlan0_ins_tci = skb_vlan_tag_get(skb);
765 		}
766 	}
767 
768 	*offset += sizeof(*ext);
769 }
770 
otx2_sqe_add_mem(struct otx2_snd_queue * sq,int * offset,int alg,u64 iova,int ptp_offset,u64 base_ns,bool udp_csum_crt)771 static void otx2_sqe_add_mem(struct otx2_snd_queue *sq, int *offset,
772 			     int alg, u64 iova, int ptp_offset,
773 			     u64 base_ns, bool udp_csum_crt)
774 {
775 	struct nix_sqe_mem_s *mem;
776 
777 	mem = (struct nix_sqe_mem_s *)(sq->sqe_base + *offset);
778 	mem->subdc = NIX_SUBDC_MEM;
779 	mem->alg = alg;
780 	mem->wmem = 1; /* wait for the memory operation */
781 	mem->addr = iova;
782 
783 	if (ptp_offset) {
784 		mem->start_offset = ptp_offset;
785 		mem->udp_csum_crt = !!udp_csum_crt;
786 		mem->base_ns = base_ns;
787 		mem->step_type = 1;
788 	}
789 
790 	*offset += sizeof(*mem);
791 }
792 
793 /* Add SQE header subdescriptor structure */
otx2_sqe_add_hdr(struct otx2_nic * pfvf,struct otx2_snd_queue * sq,struct nix_sqe_hdr_s * sqe_hdr,struct sk_buff * skb,u16 qidx)794 static void otx2_sqe_add_hdr(struct otx2_nic *pfvf, struct otx2_snd_queue *sq,
795 			     struct nix_sqe_hdr_s *sqe_hdr,
796 			     struct sk_buff *skb, u16 qidx)
797 {
798 	int proto = 0;
799 
800 	/* Check if SQE was framed before, if yes then no need to
801 	 * set these constants again and again.
802 	 */
803 	if (!sqe_hdr->total) {
804 		/* Don't free Tx buffers to Aura */
805 		sqe_hdr->df = 1;
806 		sqe_hdr->aura = sq->aura_id;
807 		/* Post a CQE Tx after pkt transmission */
808 		sqe_hdr->pnc = 1;
809 		sqe_hdr->sq = (qidx >=  pfvf->hw.tx_queues) ?
810 			       qidx + pfvf->hw.xdp_queues : qidx;
811 	}
812 	sqe_hdr->total = skb->len;
813 	/* Set SQE identifier which will be used later for freeing SKB */
814 	sqe_hdr->sqe_id = sq->head;
815 
816 	/* Offload TCP/UDP checksum to HW */
817 	if (skb->ip_summed == CHECKSUM_PARTIAL) {
818 		sqe_hdr->ol3ptr = skb_network_offset(skb);
819 		sqe_hdr->ol4ptr = skb_transport_offset(skb);
820 		/* get vlan protocol Ethertype */
821 		if (eth_type_vlan(skb->protocol))
822 			skb->protocol = vlan_get_protocol(skb);
823 
824 		if (skb->protocol == htons(ETH_P_IP)) {
825 			proto = ip_hdr(skb)->protocol;
826 			/* In case of TSO, HW needs this to be explicitly set.
827 			 * So set this always, instead of adding a check.
828 			 */
829 			sqe_hdr->ol3type = NIX_SENDL3TYPE_IP4_CKSUM;
830 		} else if (skb->protocol == htons(ETH_P_IPV6)) {
831 			proto = ipv6_hdr(skb)->nexthdr;
832 			sqe_hdr->ol3type = NIX_SENDL3TYPE_IP6;
833 		}
834 
835 		if (proto == IPPROTO_TCP)
836 			sqe_hdr->ol4type = NIX_SENDL4TYPE_TCP_CKSUM;
837 		else if (proto == IPPROTO_UDP)
838 			sqe_hdr->ol4type = NIX_SENDL4TYPE_UDP_CKSUM;
839 	}
840 }
841 
otx2_dma_map_tso_skb(struct otx2_nic * pfvf,struct otx2_snd_queue * sq,struct sk_buff * skb,int sqe,int hdr_len)842 static int otx2_dma_map_tso_skb(struct otx2_nic *pfvf,
843 				struct otx2_snd_queue *sq,
844 				struct sk_buff *skb, int sqe, int hdr_len)
845 {
846 	int num_segs = skb_shinfo(skb)->nr_frags + 1;
847 	struct sg_list *sg = &sq->sg[sqe];
848 	u64 dma_addr;
849 	int seg, len;
850 
851 	sg->num_segs = 0;
852 
853 	/* Get payload length at skb->data */
854 	len = skb_headlen(skb) - hdr_len;
855 
856 	for (seg = 0; seg < num_segs; seg++) {
857 		/* Skip skb->data, if there is no payload */
858 		if (!seg && !len)
859 			continue;
860 		dma_addr = otx2_dma_map_skb_frag(pfvf, skb, seg, &len);
861 		if (dma_mapping_error(pfvf->dev, dma_addr))
862 			goto unmap;
863 
864 		/* Save DMA mapping info for later unmapping */
865 		sg->dma_addr[sg->num_segs] = dma_addr;
866 		sg->size[sg->num_segs] = len;
867 		sg->num_segs++;
868 	}
869 	return 0;
870 unmap:
871 	otx2_dma_unmap_skb_frags(pfvf, sg);
872 	return -EINVAL;
873 }
874 
otx2_tso_frag_dma_addr(struct otx2_snd_queue * sq,struct sk_buff * skb,int seg,u64 seg_addr,int hdr_len,int sqe)875 static u64 otx2_tso_frag_dma_addr(struct otx2_snd_queue *sq,
876 				  struct sk_buff *skb, int seg,
877 				  u64 seg_addr, int hdr_len, int sqe)
878 {
879 	struct sg_list *sg = &sq->sg[sqe];
880 	const skb_frag_t *frag;
881 	int offset;
882 
883 	if (seg < 0)
884 		return sg->dma_addr[0] + (seg_addr - (u64)skb->data);
885 
886 	frag = &skb_shinfo(skb)->frags[seg];
887 	offset = seg_addr - (u64)skb_frag_address(frag);
888 	if (skb_headlen(skb) - hdr_len)
889 		seg++;
890 	return sg->dma_addr[seg] + offset;
891 }
892 
otx2_sqe_tso_add_sg(struct otx2_snd_queue * sq,struct sg_list * list,int * offset)893 static void otx2_sqe_tso_add_sg(struct otx2_snd_queue *sq,
894 				struct sg_list *list, int *offset)
895 {
896 	struct nix_sqe_sg_s *sg = NULL;
897 	u16 *sg_lens = NULL;
898 	u64 *iova = NULL;
899 	int seg;
900 
901 	/* Add SG descriptors with buffer addresses */
902 	for (seg = 0; seg < list->num_segs; seg++) {
903 		if ((seg % MAX_SEGS_PER_SG) == 0) {
904 			sg = (struct nix_sqe_sg_s *)(sq->sqe_base + *offset);
905 			sg->ld_type = NIX_SEND_LDTYPE_LDD;
906 			sg->subdc = NIX_SUBDC_SG;
907 			sg->segs = 0;
908 			sg_lens = (void *)sg;
909 			iova = (void *)sg + sizeof(*sg);
910 			/* Next subdc always starts at a 16byte boundary.
911 			 * So if sg->segs is whether 2 or 3, offset += 16bytes.
912 			 */
913 			if ((list->num_segs - seg) >= (MAX_SEGS_PER_SG - 1))
914 				*offset += sizeof(*sg) + (3 * sizeof(u64));
915 			else
916 				*offset += sizeof(*sg) + sizeof(u64);
917 		}
918 		sg_lens[frag_num(seg % MAX_SEGS_PER_SG)] = list->size[seg];
919 		*iova++ = list->dma_addr[seg];
920 		sg->segs++;
921 	}
922 }
923 
otx2_sq_append_tso(struct otx2_nic * pfvf,struct otx2_snd_queue * sq,struct sk_buff * skb,u16 qidx)924 static void otx2_sq_append_tso(struct otx2_nic *pfvf, struct otx2_snd_queue *sq,
925 			       struct sk_buff *skb, u16 qidx)
926 {
927 	struct netdev_queue *txq = netdev_get_tx_queue(pfvf->netdev, qidx);
928 	int hdr_len, tcp_data, seg_len, pkt_len, offset;
929 	struct nix_sqe_hdr_s *sqe_hdr;
930 	int first_sqe = sq->head;
931 	struct sg_list list;
932 	struct tso_t tso;
933 
934 	hdr_len = tso_start(skb, &tso);
935 
936 	/* Map SKB's fragments to DMA.
937 	 * It's done here to avoid mapping for every TSO segment's packet.
938 	 */
939 	if (otx2_dma_map_tso_skb(pfvf, sq, skb, first_sqe, hdr_len)) {
940 		dev_kfree_skb_any(skb);
941 		return;
942 	}
943 
944 	netdev_tx_sent_queue(txq, skb->len);
945 
946 	tcp_data = skb->len - hdr_len;
947 	while (tcp_data > 0) {
948 		char *hdr;
949 
950 		seg_len = min_t(int, skb_shinfo(skb)->gso_size, tcp_data);
951 		tcp_data -= seg_len;
952 
953 		/* Set SQE's SEND_HDR */
954 		memset(sq->sqe_base, 0, sq->sqe_size);
955 		sqe_hdr = (struct nix_sqe_hdr_s *)(sq->sqe_base);
956 		otx2_sqe_add_hdr(pfvf, sq, sqe_hdr, skb, qidx);
957 		offset = sizeof(*sqe_hdr);
958 
959 		/* Add TSO segment's pkt header */
960 		hdr = sq->tso_hdrs->base + (sq->head * TSO_HEADER_SIZE);
961 		tso_build_hdr(skb, hdr, &tso, seg_len, tcp_data == 0);
962 		list.dma_addr[0] =
963 			sq->tso_hdrs->iova + (sq->head * TSO_HEADER_SIZE);
964 		list.size[0] = hdr_len;
965 		list.num_segs = 1;
966 
967 		/* Add TSO segment's payload data fragments */
968 		pkt_len = hdr_len;
969 		while (seg_len > 0) {
970 			int size;
971 
972 			size = min_t(int, tso.size, seg_len);
973 
974 			list.size[list.num_segs] = size;
975 			list.dma_addr[list.num_segs] =
976 				otx2_tso_frag_dma_addr(sq, skb,
977 						       tso.next_frag_idx - 1,
978 						       (u64)tso.data, hdr_len,
979 						       first_sqe);
980 			list.num_segs++;
981 			pkt_len += size;
982 			seg_len -= size;
983 			tso_build_data(skb, &tso, size);
984 		}
985 		sqe_hdr->total = pkt_len;
986 		otx2_sqe_tso_add_sg(sq, &list, &offset);
987 
988 		/* DMA mappings and skb needs to be freed only after last
989 		 * TSO segment is transmitted out. So set 'PNC' only for
990 		 * last segment. Also point last segment's sqe_id to first
991 		 * segment's SQE index where skb address and DMA mappings
992 		 * are saved.
993 		 */
994 		if (!tcp_data) {
995 			sqe_hdr->pnc = 1;
996 			sqe_hdr->sqe_id = first_sqe;
997 			sq->sg[first_sqe].skb = (u64)skb;
998 		} else {
999 			sqe_hdr->pnc = 0;
1000 		}
1001 
1002 		sqe_hdr->sizem1 = (offset / 16) - 1;
1003 
1004 		/* Flush SQE to HW */
1005 		pfvf->hw_ops->sqe_flush(pfvf, sq, offset, qidx);
1006 	}
1007 }
1008 
is_hw_tso_supported(struct otx2_nic * pfvf,struct sk_buff * skb)1009 static bool is_hw_tso_supported(struct otx2_nic *pfvf,
1010 				struct sk_buff *skb)
1011 {
1012 	int payload_len, last_seg_size;
1013 
1014 	if (test_bit(HW_TSO, &pfvf->hw.cap_flag))
1015 		return true;
1016 
1017 	/* On 96xx A0, HW TSO not supported */
1018 	if (!is_96xx_B0(pfvf->pdev))
1019 		return false;
1020 
1021 	/* HW has an issue due to which when the payload of the last LSO
1022 	 * segment is shorter than 16 bytes, some header fields may not
1023 	 * be correctly modified, hence don't offload such TSO segments.
1024 	 */
1025 
1026 	payload_len = skb->len - skb_tcp_all_headers(skb);
1027 	last_seg_size = payload_len % skb_shinfo(skb)->gso_size;
1028 	if (last_seg_size && last_seg_size < 16)
1029 		return false;
1030 
1031 	return true;
1032 }
1033 
otx2_get_sqe_count(struct otx2_nic * pfvf,struct sk_buff * skb)1034 static int otx2_get_sqe_count(struct otx2_nic *pfvf, struct sk_buff *skb)
1035 {
1036 	if (!skb_shinfo(skb)->gso_size)
1037 		return 1;
1038 
1039 	/* HW TSO */
1040 	if (is_hw_tso_supported(pfvf, skb))
1041 		return 1;
1042 
1043 	/* SW TSO */
1044 	return skb_shinfo(skb)->gso_segs;
1045 }
1046 
otx2_validate_network_transport(struct sk_buff * skb)1047 static bool otx2_validate_network_transport(struct sk_buff *skb)
1048 {
1049 	if ((ip_hdr(skb)->protocol == IPPROTO_UDP) ||
1050 	    (ipv6_hdr(skb)->nexthdr == IPPROTO_UDP)) {
1051 		struct udphdr *udph = udp_hdr(skb);
1052 
1053 		if (udph->source == htons(PTP_PORT) &&
1054 		    udph->dest == htons(PTP_PORT))
1055 			return true;
1056 	}
1057 
1058 	return false;
1059 }
1060 
otx2_ptp_is_sync(struct sk_buff * skb,int * offset,bool * udp_csum_crt)1061 static bool otx2_ptp_is_sync(struct sk_buff *skb, int *offset, bool *udp_csum_crt)
1062 {
1063 	struct ethhdr *eth = (struct ethhdr *)(skb->data);
1064 	u16 nix_offload_hlen = 0, inner_vhlen = 0;
1065 	bool udp_hdr_present = false, is_sync;
1066 	u8 *data = skb->data, *msgtype;
1067 	__be16 proto = eth->h_proto;
1068 	int network_depth = 0;
1069 
1070 	/* NIX is programmed to offload outer  VLAN header
1071 	 * in case of single vlan protocol field holds Network header ETH_IP/V6
1072 	 * in case of stacked vlan protocol field holds Inner vlan (8100)
1073 	 */
1074 	if (skb->dev->features & NETIF_F_HW_VLAN_CTAG_TX &&
1075 	    skb->dev->features & NETIF_F_HW_VLAN_STAG_TX) {
1076 		if (skb->vlan_proto == htons(ETH_P_8021AD)) {
1077 			/* Get vlan protocol */
1078 			proto = __vlan_get_protocol(skb, eth->h_proto, NULL);
1079 			/* SKB APIs like skb_transport_offset does not include
1080 			 * offloaded vlan header length. Need to explicitly add
1081 			 * the length
1082 			 */
1083 			nix_offload_hlen = VLAN_HLEN;
1084 			inner_vhlen = VLAN_HLEN;
1085 		} else if (skb->vlan_proto == htons(ETH_P_8021Q)) {
1086 			nix_offload_hlen = VLAN_HLEN;
1087 		}
1088 	} else if (eth_type_vlan(eth->h_proto)) {
1089 		proto = __vlan_get_protocol(skb, eth->h_proto, &network_depth);
1090 	}
1091 
1092 	switch (ntohs(proto)) {
1093 	case ETH_P_1588:
1094 		if (network_depth)
1095 			*offset = network_depth;
1096 		else
1097 			*offset = ETH_HLEN + nix_offload_hlen +
1098 				  inner_vhlen;
1099 		break;
1100 	case ETH_P_IP:
1101 	case ETH_P_IPV6:
1102 		if (!otx2_validate_network_transport(skb))
1103 			return false;
1104 
1105 		*offset = nix_offload_hlen + skb_transport_offset(skb) +
1106 			  sizeof(struct udphdr);
1107 		udp_hdr_present = true;
1108 
1109 	}
1110 
1111 	msgtype = data + *offset;
1112 	/* Check PTP messageId is SYNC or not */
1113 	is_sync = !(*msgtype & 0xf);
1114 	if (is_sync)
1115 		*udp_csum_crt = udp_hdr_present;
1116 	else
1117 		*offset = 0;
1118 
1119 	return is_sync;
1120 }
1121 
otx2_set_txtstamp(struct otx2_nic * pfvf,struct sk_buff * skb,struct otx2_snd_queue * sq,int * offset)1122 static void otx2_set_txtstamp(struct otx2_nic *pfvf, struct sk_buff *skb,
1123 			      struct otx2_snd_queue *sq, int *offset)
1124 {
1125 	struct ethhdr	*eth = (struct ethhdr *)(skb->data);
1126 	struct ptpv2_tstamp *origin_tstamp;
1127 	bool udp_csum_crt = false;
1128 	unsigned int udphoff;
1129 	struct timespec64 ts;
1130 	int ptp_offset = 0;
1131 	__wsum skb_csum;
1132 	u64 iova;
1133 
1134 	if (unlikely(!skb_shinfo(skb)->gso_size &&
1135 		     (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))) {
1136 		if (unlikely(pfvf->flags & OTX2_FLAG_PTP_ONESTEP_SYNC &&
1137 			     otx2_ptp_is_sync(skb, &ptp_offset, &udp_csum_crt))) {
1138 			origin_tstamp = (struct ptpv2_tstamp *)
1139 					((u8 *)skb->data + ptp_offset +
1140 					 PTP_SYNC_SEC_OFFSET);
1141 			ts = ns_to_timespec64(pfvf->ptp->tstamp);
1142 			origin_tstamp->seconds_msb = htons((ts.tv_sec >> 32) & 0xffff);
1143 			origin_tstamp->seconds_lsb = htonl(ts.tv_sec & 0xffffffff);
1144 			origin_tstamp->nanoseconds = htonl(ts.tv_nsec);
1145 			/* Point to correction field in PTP packet */
1146 			ptp_offset += 8;
1147 
1148 			/* When user disables hw checksum, stack calculates the csum,
1149 			 * but it does not cover ptp timestamp which is added later.
1150 			 * Recalculate the checksum manually considering the timestamp.
1151 			 */
1152 			if (udp_csum_crt) {
1153 				struct udphdr *uh = udp_hdr(skb);
1154 
1155 				if (skb->ip_summed != CHECKSUM_PARTIAL && uh->check != 0) {
1156 					udphoff = skb_transport_offset(skb);
1157 					uh->check = 0;
1158 					skb_csum = skb_checksum(skb, udphoff, skb->len - udphoff,
1159 								0);
1160 					if (ntohs(eth->h_proto) == ETH_P_IPV6)
1161 						uh->check = csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
1162 									    &ipv6_hdr(skb)->daddr,
1163 									    skb->len - udphoff,
1164 									    ipv6_hdr(skb)->nexthdr,
1165 									    skb_csum);
1166 					else
1167 						uh->check = csum_tcpudp_magic(ip_hdr(skb)->saddr,
1168 									      ip_hdr(skb)->daddr,
1169 									      skb->len - udphoff,
1170 									      IPPROTO_UDP,
1171 									      skb_csum);
1172 				}
1173 			}
1174 		} else {
1175 			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1176 		}
1177 		iova = sq->timestamps->iova + (sq->head * sizeof(u64));
1178 		otx2_sqe_add_mem(sq, offset, NIX_SENDMEMALG_E_SETTSTMP, iova,
1179 				 ptp_offset, pfvf->ptp->base_ns, udp_csum_crt);
1180 	} else {
1181 		skb_tx_timestamp(skb);
1182 	}
1183 }
1184 
otx2_sq_append_skb(void * dev,struct netdev_queue * txq,struct otx2_snd_queue * sq,struct sk_buff * skb,u16 qidx)1185 bool otx2_sq_append_skb(void *dev, struct netdev_queue *txq,
1186 			struct otx2_snd_queue *sq,
1187 			struct sk_buff *skb, u16 qidx)
1188 {
1189 	int offset, num_segs, free_desc;
1190 	struct nix_sqe_hdr_s *sqe_hdr;
1191 	struct otx2_nic *pfvf = dev;
1192 	bool ret;
1193 
1194 	/* Check if there is enough room between producer
1195 	 * and consumer index.
1196 	 */
1197 	free_desc = otx2_get_free_sqe(sq);
1198 	if (free_desc < sq->sqe_thresh)
1199 		return false;
1200 
1201 	if (free_desc < otx2_get_sqe_count(pfvf, skb))
1202 		return false;
1203 
1204 	num_segs = skb_shinfo(skb)->nr_frags + 1;
1205 
1206 	/* If SKB doesn't fit in a single SQE, linearize it.
1207 	 * TODO: Consider adding JUMP descriptor instead.
1208 	 */
1209 
1210 	if (unlikely(num_segs > OTX2_MAX_FRAGS_IN_SQE)) {
1211 		if (__skb_linearize(skb)) {
1212 			dev_kfree_skb_any(skb);
1213 			return true;
1214 		}
1215 		num_segs = skb_shinfo(skb)->nr_frags + 1;
1216 	}
1217 
1218 	if (skb_shinfo(skb)->gso_size && !is_hw_tso_supported(pfvf, skb)) {
1219 		/* Insert vlan tag before giving pkt to tso */
1220 		if (skb_vlan_tag_present(skb)) {
1221 			skb = __vlan_hwaccel_push_inside(skb);
1222 			if (!skb)
1223 				return true;
1224 		}
1225 		otx2_sq_append_tso(pfvf, sq, skb, qidx);
1226 		return true;
1227 	}
1228 
1229 	/* Set sqe base address */
1230 	otx2_sq_set_sqe_base(sq, skb);
1231 
1232 	/* Set SQE's SEND_HDR.
1233 	 * Do not clear the first 64bit as it contains constant info.
1234 	 */
1235 	memset(sq->sqe_base + 8, 0, sq->sqe_size - 8);
1236 	sqe_hdr = (struct nix_sqe_hdr_s *)(sq->sqe_base);
1237 	otx2_sqe_add_hdr(pfvf, sq, sqe_hdr, skb, qidx);
1238 	offset = sizeof(*sqe_hdr);
1239 
1240 	/* Add extended header if needed */
1241 	otx2_sqe_add_ext(pfvf, sq, skb, &offset);
1242 
1243 	/* Add SG subdesc with data frags */
1244 	if (static_branch_unlikely(&cn10k_ipsec_sa_enabled) &&
1245 	    (xfrm_offload(skb)))
1246 		ret = otx2_sqe_add_sg_ipsec(pfvf, sq, skb, num_segs, &offset);
1247 	else
1248 		ret = otx2_sqe_add_sg(pfvf, sq, skb, num_segs, &offset);
1249 
1250 	if (!ret) {
1251 		otx2_dma_unmap_skb_frags(pfvf, &sq->sg[sq->head]);
1252 		return false;
1253 	}
1254 
1255 	otx2_set_txtstamp(pfvf, skb, sq, &offset);
1256 
1257 	sqe_hdr->sizem1 = (offset / 16) - 1;
1258 
1259 	if (static_branch_unlikely(&cn10k_ipsec_sa_enabled) &&
1260 	    (xfrm_offload(skb)))
1261 		return cn10k_ipsec_transmit(pfvf, txq, sq, skb, num_segs,
1262 					    offset);
1263 
1264 	netdev_tx_sent_queue(txq, skb->len);
1265 
1266 	/* Flush SQE to HW */
1267 	pfvf->hw_ops->sqe_flush(pfvf, sq, offset, qidx);
1268 	return true;
1269 }
1270 EXPORT_SYMBOL(otx2_sq_append_skb);
1271 
otx2_cleanup_rx_cqes(struct otx2_nic * pfvf,struct otx2_cq_queue * cq,int qidx)1272 void otx2_cleanup_rx_cqes(struct otx2_nic *pfvf, struct otx2_cq_queue *cq, int qidx)
1273 {
1274 	struct nix_cqe_rx_s *cqe;
1275 	struct otx2_pool *pool;
1276 	int processed_cqe = 0;
1277 	u16 pool_id;
1278 	u64 iova;
1279 
1280 	pool_id = otx2_get_pool_idx(pfvf, AURA_NIX_RQ, qidx);
1281 	pool = &pfvf->qset.pool[pool_id];
1282 
1283 	if (pfvf->xdp_prog) {
1284 		if (pool->page_pool)
1285 			xdp_rxq_info_unreg_mem_model(&cq->xdp_rxq);
1286 
1287 		xdp_rxq_info_unreg(&cq->xdp_rxq);
1288 	}
1289 
1290 	if (otx2_nix_cq_op_status(pfvf, cq) || !cq->pend_cqe)
1291 		return;
1292 
1293 	while (cq->pend_cqe) {
1294 		cqe = (struct nix_cqe_rx_s *)otx2_get_next_cqe(cq);
1295 		processed_cqe++;
1296 		cq->pend_cqe--;
1297 
1298 		if (!cqe)
1299 			continue;
1300 		if (cqe->sg.segs > 1) {
1301 			otx2_free_rcv_seg(pfvf, cqe, cq->cq_idx);
1302 			continue;
1303 		}
1304 		iova = cqe->sg.seg_addr - OTX2_HEAD_ROOM;
1305 
1306 		otx2_free_bufs(pfvf, pool, iova, pfvf->rbsize);
1307 	}
1308 
1309 	/* Free CQEs to HW */
1310 	otx2_write64(pfvf, NIX_LF_CQ_OP_DOOR,
1311 		     ((u64)cq->cq_idx << 32) | processed_cqe);
1312 }
1313 
otx2_cleanup_tx_cqes(struct otx2_nic * pfvf,struct otx2_cq_queue * cq)1314 void otx2_cleanup_tx_cqes(struct otx2_nic *pfvf, struct otx2_cq_queue *cq)
1315 {
1316 	int tx_pkts = 0, tx_bytes = 0;
1317 	struct sk_buff *skb = NULL;
1318 	struct otx2_snd_queue *sq;
1319 	struct nix_cqe_tx_s *cqe;
1320 	struct netdev_queue *txq;
1321 	int processed_cqe = 0;
1322 	struct sg_list *sg;
1323 	int qidx;
1324 
1325 	qidx = cq->cq_idx - pfvf->hw.rx_queues;
1326 	sq = &pfvf->qset.sq[qidx];
1327 
1328 	if (otx2_nix_cq_op_status(pfvf, cq) || !cq->pend_cqe)
1329 		return;
1330 
1331 	while (cq->pend_cqe) {
1332 		cqe = (struct nix_cqe_tx_s *)otx2_get_next_cqe(cq);
1333 		processed_cqe++;
1334 		cq->pend_cqe--;
1335 
1336 		if (!cqe)
1337 			continue;
1338 		sg = &sq->sg[cqe->comp.sqe_id];
1339 		skb = (struct sk_buff *)sg->skb;
1340 		if (skb) {
1341 			tx_bytes += skb->len;
1342 			tx_pkts++;
1343 			otx2_dma_unmap_skb_frags(pfvf, sg);
1344 			dev_kfree_skb_any(skb);
1345 			sg->skb = (u64)NULL;
1346 		}
1347 	}
1348 
1349 	if (likely(tx_pkts)) {
1350 		if (qidx >= pfvf->hw.tx_queues)
1351 			qidx -= pfvf->hw.xdp_queues;
1352 		txq = netdev_get_tx_queue(pfvf->netdev, qidx);
1353 		netdev_tx_completed_queue(txq, tx_pkts, tx_bytes);
1354 	}
1355 	/* Free CQEs to HW */
1356 	otx2_write64(pfvf, NIX_LF_CQ_OP_DOOR,
1357 		     ((u64)cq->cq_idx << 32) | processed_cqe);
1358 }
1359 
otx2_rxtx_enable(struct otx2_nic * pfvf,bool enable)1360 int otx2_rxtx_enable(struct otx2_nic *pfvf, bool enable)
1361 {
1362 	struct msg_req *msg;
1363 	int err;
1364 
1365 	mutex_lock(&pfvf->mbox.lock);
1366 	if (enable)
1367 		msg = otx2_mbox_alloc_msg_nix_lf_start_rx(&pfvf->mbox);
1368 	else
1369 		msg = otx2_mbox_alloc_msg_nix_lf_stop_rx(&pfvf->mbox);
1370 
1371 	if (!msg) {
1372 		mutex_unlock(&pfvf->mbox.lock);
1373 		return -ENOMEM;
1374 	}
1375 
1376 	err = otx2_sync_mbox_msg(&pfvf->mbox);
1377 	mutex_unlock(&pfvf->mbox.lock);
1378 	return err;
1379 }
1380 
otx2_free_pending_sqe(struct otx2_nic * pfvf)1381 void otx2_free_pending_sqe(struct otx2_nic *pfvf)
1382 {
1383 	int tx_pkts = 0, tx_bytes = 0;
1384 	struct sk_buff *skb = NULL;
1385 	struct otx2_snd_queue *sq;
1386 	struct netdev_queue *txq;
1387 	struct sg_list *sg;
1388 	int sq_idx, sqe;
1389 
1390 	for (sq_idx = 0; sq_idx < pfvf->hw.tx_queues; sq_idx++) {
1391 		sq = &pfvf->qset.sq[sq_idx];
1392 		for (sqe = 0; sqe < sq->sqe_cnt; sqe++) {
1393 			sg = &sq->sg[sqe];
1394 			skb = (struct sk_buff *)sg->skb;
1395 			if (skb) {
1396 				tx_bytes += skb->len;
1397 				tx_pkts++;
1398 				otx2_dma_unmap_skb_frags(pfvf, sg);
1399 				dev_kfree_skb_any(skb);
1400 				sg->skb = (u64)NULL;
1401 			}
1402 		}
1403 
1404 		if (!tx_pkts)
1405 			continue;
1406 		txq = netdev_get_tx_queue(pfvf->netdev, sq_idx);
1407 		netdev_tx_completed_queue(txq, tx_pkts, tx_bytes);
1408 		tx_pkts = 0;
1409 		tx_bytes = 0;
1410 	}
1411 }
1412 
otx2_xdp_sqe_add_sg(struct otx2_snd_queue * sq,struct xdp_frame * xdpf,u64 dma_addr,int len,int * offset,u16 flags)1413 static void otx2_xdp_sqe_add_sg(struct otx2_snd_queue *sq,
1414 				struct xdp_frame *xdpf,
1415 				u64 dma_addr, int len, int *offset, u16 flags)
1416 {
1417 	struct nix_sqe_sg_s *sg = NULL;
1418 	u64 *iova = NULL;
1419 
1420 	sg = (struct nix_sqe_sg_s *)(sq->sqe_base + *offset);
1421 	sg->ld_type = NIX_SEND_LDTYPE_LDD;
1422 	sg->subdc = NIX_SUBDC_SG;
1423 	sg->segs = 1;
1424 	sg->seg1_size = len;
1425 	iova = (void *)sg + sizeof(*sg);
1426 	*iova = dma_addr;
1427 	*offset += sizeof(*sg) + sizeof(u64);
1428 
1429 	sq->sg[sq->head].dma_addr[0] = dma_addr;
1430 	sq->sg[sq->head].size[0] = len;
1431 	sq->sg[sq->head].num_segs = 1;
1432 	sq->sg[sq->head].flags = flags;
1433 	sq->sg[sq->head].skb = (u64)xdpf;
1434 }
1435 
otx2_read_free_sqe(struct otx2_nic * pfvf,u16 qidx)1436 int otx2_read_free_sqe(struct otx2_nic *pfvf, u16 qidx)
1437 {
1438 	struct otx2_snd_queue *sq;
1439 	int free_sqe;
1440 
1441 	sq = &pfvf->qset.sq[qidx];
1442 	free_sqe = otx2_get_free_sqe(sq);
1443 	if (free_sqe < sq->sqe_thresh) {
1444 		netdev_warn(pfvf->netdev, "No free sqe for Send queue%d\n", qidx);
1445 		return 0;
1446 	}
1447 
1448 	return free_sqe - sq->sqe_thresh;
1449 }
1450 
otx2_xdp_sq_append_pkt(struct otx2_nic * pfvf,struct xdp_frame * xdpf,u64 iova,int len,u16 qidx,u16 flags)1451 bool otx2_xdp_sq_append_pkt(struct otx2_nic *pfvf, struct xdp_frame *xdpf,
1452 			    u64 iova, int len, u16 qidx, u16 flags)
1453 {
1454 	struct nix_sqe_hdr_s *sqe_hdr;
1455 	struct otx2_snd_queue *sq;
1456 	int offset, free_sqe;
1457 
1458 	sq = &pfvf->qset.sq[qidx];
1459 	free_sqe = otx2_get_free_sqe(sq);
1460 	if (free_sqe < sq->sqe_thresh)
1461 		return false;
1462 
1463 	memset(sq->sqe_base + 8, 0, sq->sqe_size - 8);
1464 
1465 	sqe_hdr = (struct nix_sqe_hdr_s *)(sq->sqe_base);
1466 
1467 	if (!sqe_hdr->total) {
1468 		sqe_hdr->aura = sq->aura_id;
1469 		sqe_hdr->df = 1;
1470 		sqe_hdr->sq = qidx;
1471 		sqe_hdr->pnc = 1;
1472 	}
1473 	sqe_hdr->total = len;
1474 	sqe_hdr->sqe_id = sq->head;
1475 
1476 	offset = sizeof(*sqe_hdr);
1477 
1478 	otx2_xdp_sqe_add_sg(sq, xdpf, iova, len, &offset, flags);
1479 	sqe_hdr->sizem1 = (offset / 16) - 1;
1480 	pfvf->hw_ops->sqe_flush(pfvf, sq, offset, qidx);
1481 
1482 	return true;
1483 }
1484 
otx2_xdp_rcv_pkt_handler(struct otx2_nic * pfvf,struct bpf_prog * prog,struct nix_cqe_rx_s * cqe,struct otx2_cq_queue * cq,u32 * metasize,bool * need_xdp_flush)1485 static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf,
1486 				     struct bpf_prog *prog,
1487 				     struct nix_cqe_rx_s *cqe,
1488 				     struct otx2_cq_queue *cq,
1489 				     u32 *metasize, bool *need_xdp_flush)
1490 {
1491 	struct xdp_buff xdp, *xsk_buff = NULL;
1492 	unsigned char *hard_start;
1493 	struct otx2_pool *pool;
1494 	struct xdp_frame *xdpf;
1495 	int qidx = cq->cq_idx;
1496 	struct page *page;
1497 	u64 iova, pa;
1498 	u32 act;
1499 	int err;
1500 
1501 	pool = &pfvf->qset.pool[qidx];
1502 
1503 	if (pool->xsk_pool) {
1504 		xsk_buff = pool->xdp[--cq->rbpool->xdp_top];
1505 		if (!xsk_buff)
1506 			return false;
1507 
1508 		xsk_buff->data_end = xsk_buff->data + cqe->sg.seg_size;
1509 		act = bpf_prog_run_xdp(prog, xsk_buff);
1510 		goto handle_xdp_verdict;
1511 	}
1512 
1513 	iova = cqe->sg.seg_addr - OTX2_HEAD_ROOM;
1514 	pa = otx2_iova_to_phys(pfvf->iommu_domain, iova);
1515 	page = virt_to_page(phys_to_virt(pa));
1516 
1517 	xdp_init_buff(&xdp, pfvf->rbsize, &cq->xdp_rxq);
1518 
1519 	hard_start = (unsigned char *)phys_to_virt(pa);
1520 	xdp_prepare_buff(&xdp, hard_start, OTX2_HEAD_ROOM,
1521 			 cqe->sg.seg_size, true);
1522 
1523 	act = bpf_prog_run_xdp(prog, &xdp);
1524 
1525 handle_xdp_verdict:
1526 	switch (act) {
1527 	case XDP_PASS:
1528 		*metasize = xdp.data - xdp.data_meta;
1529 		break;
1530 	case XDP_TX:
1531 		qidx += pfvf->hw.tx_queues;
1532 		cq->pool_ptrs++;
1533 		xdpf = xdp_convert_buff_to_frame(&xdp);
1534 		return otx2_xdp_sq_append_pkt(pfvf, xdpf,
1535 					      cqe->sg.seg_addr,
1536 					      cqe->sg.seg_size,
1537 					      qidx, OTX2_XDP_TX);
1538 	case XDP_REDIRECT:
1539 		cq->pool_ptrs++;
1540 		if (xsk_buff) {
1541 			err = xdp_do_redirect(pfvf->netdev, xsk_buff, prog);
1542 			if (!err) {
1543 				*need_xdp_flush = true;
1544 				return true;
1545 			}
1546 			return false;
1547 		}
1548 
1549 		err = xdp_do_redirect(pfvf->netdev, &xdp, prog);
1550 		if (!err) {
1551 			*need_xdp_flush = true;
1552 			return true;
1553 		}
1554 
1555 		otx2_dma_unmap_page(pfvf, iova, pfvf->rbsize,
1556 				    DMA_FROM_DEVICE);
1557 		xdpf = xdp_convert_buff_to_frame(&xdp);
1558 		xdp_return_frame(xdpf);
1559 		break;
1560 	default:
1561 		bpf_warn_invalid_xdp_action(pfvf->netdev, prog, act);
1562 		fallthrough;
1563 	case XDP_ABORTED:
1564 		if (act == XDP_ABORTED)
1565 			trace_xdp_exception(pfvf->netdev, prog, act);
1566 		fallthrough;
1567 	case XDP_DROP:
1568 		cq->pool_ptrs++;
1569 		if (xsk_buff) {
1570 			xsk_buff_free(xsk_buff);
1571 		} else if (page->pp) {
1572 			page_pool_recycle_direct(pool->page_pool, page);
1573 		} else {
1574 			otx2_dma_unmap_page(pfvf, iova, pfvf->rbsize,
1575 					    DMA_FROM_DEVICE);
1576 			put_page(page);
1577 		}
1578 		return true;
1579 	}
1580 	return false;
1581 }
1582