1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell RVU Ethernet driver
3 *
4 * Copyright (C) 2020 Marvell.
5 *
6 */
7
8 #include <linux/etherdevice.h>
9 #include <net/ip.h>
10 #include <net/tso.h>
11 #include <linux/bpf.h>
12 #include <linux/bpf_trace.h>
13 #include <net/ip6_checksum.h>
14 #include <net/xfrm.h>
15 #include <net/xdp.h>
16
17 #include "otx2_reg.h"
18 #include "otx2_common.h"
19 #include "otx2_struct.h"
20 #include "otx2_txrx.h"
21 #include "otx2_ptp.h"
22 #include "cn10k.h"
23 #include "otx2_xsk.h"
24
25 #define CQE_ADDR(CQ, idx) ((CQ)->cqe_base + ((CQ)->cqe_size * (idx)))
26 #define PTP_PORT 0x13F
27 /* PTPv2 header Original Timestamp starts at byte offset 34 and
28 * contains 6 byte seconds field and 4 byte nano seconds field.
29 */
30 #define PTP_SYNC_SEC_OFFSET 34
31
32 DEFINE_STATIC_KEY_FALSE(cn10k_ipsec_sa_enabled);
33
otx2_get_free_sqe(struct otx2_snd_queue * sq)34 static int otx2_get_free_sqe(struct otx2_snd_queue *sq)
35 {
36 return (sq->cons_head - sq->head - 1 + sq->sqe_cnt)
37 & (sq->sqe_cnt - 1);
38 }
39
40 static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf,
41 struct bpf_prog *prog,
42 struct nix_cqe_rx_s *cqe,
43 struct otx2_cq_queue *cq,
44 u32 *metasize, bool *need_xdp_flush);
45
otx2_sq_set_sqe_base(struct otx2_snd_queue * sq,struct sk_buff * skb)46 static void otx2_sq_set_sqe_base(struct otx2_snd_queue *sq,
47 struct sk_buff *skb)
48 {
49 if (static_branch_unlikely(&cn10k_ipsec_sa_enabled) &&
50 (xfrm_offload(skb)))
51 sq->sqe_base = sq->sqe_ring->base + sq->sqe_size +
52 (sq->head * (sq->sqe_size * 2));
53 else
54 sq->sqe_base = sq->sqe->base;
55 }
56
otx2_nix_cq_op_status(struct otx2_nic * pfvf,struct otx2_cq_queue * cq)57 static int otx2_nix_cq_op_status(struct otx2_nic *pfvf,
58 struct otx2_cq_queue *cq)
59 {
60 u64 incr = (u64)(cq->cq_idx) << 32;
61 u64 status;
62
63 status = otx2_atomic64_fetch_add(incr, pfvf->cq_op_addr);
64
65 if (unlikely(status & BIT_ULL(CQ_OP_STAT_OP_ERR) ||
66 status & BIT_ULL(CQ_OP_STAT_CQ_ERR))) {
67 dev_err(pfvf->dev, "CQ stopped due to error");
68 return -EINVAL;
69 }
70
71 cq->cq_tail = status & 0xFFFFF;
72 cq->cq_head = (status >> 20) & 0xFFFFF;
73 if (cq->cq_tail < cq->cq_head)
74 cq->pend_cqe = (cq->cqe_cnt - cq->cq_head) +
75 cq->cq_tail;
76 else
77 cq->pend_cqe = cq->cq_tail - cq->cq_head;
78
79 return 0;
80 }
81
otx2_get_next_cqe(struct otx2_cq_queue * cq)82 static struct nix_cqe_hdr_s *otx2_get_next_cqe(struct otx2_cq_queue *cq)
83 {
84 struct nix_cqe_hdr_s *cqe_hdr;
85
86 cqe_hdr = (struct nix_cqe_hdr_s *)CQE_ADDR(cq, cq->cq_head);
87 if (cqe_hdr->cqe_type == NIX_XQE_TYPE_INVALID)
88 return NULL;
89
90 cq->cq_head++;
91 cq->cq_head &= (cq->cqe_cnt - 1);
92
93 return cqe_hdr;
94 }
95
frag_num(unsigned int i)96 static unsigned int frag_num(unsigned int i)
97 {
98 #ifdef __BIG_ENDIAN
99 return (i & ~3) + 3 - (i & 3);
100 #else
101 return i;
102 #endif
103 }
104
otx2_xdp_snd_pkt_handler(struct otx2_nic * pfvf,struct otx2_snd_queue * sq,struct nix_cqe_tx_s * cqe,int * xsk_frames)105 static void otx2_xdp_snd_pkt_handler(struct otx2_nic *pfvf,
106 struct otx2_snd_queue *sq,
107 struct nix_cqe_tx_s *cqe,
108 int *xsk_frames)
109 {
110 struct nix_send_comp_s *snd_comp = &cqe->comp;
111 struct sg_list *sg;
112
113 sg = &sq->sg[snd_comp->sqe_id];
114 if (sg->flags & OTX2_AF_XDP_FRAME) {
115 (*xsk_frames)++;
116 return;
117 }
118
119 if (sg->flags & OTX2_XDP_REDIRECT)
120 otx2_dma_unmap_page(pfvf, sg->dma_addr[0], sg->size[0], DMA_TO_DEVICE);
121 xdp_return_frame((struct xdp_frame *)sg->skb);
122 sg->skb = (u64)NULL;
123 }
124
otx2_snd_pkt_handler(struct otx2_nic * pfvf,struct otx2_cq_queue * cq,struct otx2_snd_queue * sq,struct nix_cqe_tx_s * cqe,int budget,int * tx_pkts,int * tx_bytes)125 static void otx2_snd_pkt_handler(struct otx2_nic *pfvf,
126 struct otx2_cq_queue *cq,
127 struct otx2_snd_queue *sq,
128 struct nix_cqe_tx_s *cqe,
129 int budget, int *tx_pkts, int *tx_bytes)
130 {
131 struct nix_send_comp_s *snd_comp = &cqe->comp;
132 struct skb_shared_hwtstamps ts;
133 struct sk_buff *skb = NULL;
134 u64 timestamp, tsns;
135 struct sg_list *sg;
136 int err;
137
138 if (unlikely(snd_comp->status) && netif_msg_tx_err(pfvf))
139 net_err_ratelimited("%s: TX%d: Error in send CQ status:%x\n",
140 pfvf->netdev->name, cq->cint_idx,
141 snd_comp->status);
142
143 sg = &sq->sg[snd_comp->sqe_id];
144 skb = (struct sk_buff *)sg->skb;
145 if (unlikely(!skb))
146 return;
147
148 if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) {
149 timestamp = ((u64 *)sq->timestamps->base)[snd_comp->sqe_id];
150 if (timestamp != 1) {
151 timestamp = pfvf->ptp->convert_tx_ptp_tstmp(timestamp);
152 err = otx2_ptp_tstamp2time(pfvf, timestamp, &tsns);
153 if (!err) {
154 memset(&ts, 0, sizeof(ts));
155 ts.hwtstamp = ns_to_ktime(tsns);
156 skb_tstamp_tx(skb, &ts);
157 }
158 }
159 }
160
161 *tx_bytes += skb->len;
162 (*tx_pkts)++;
163 otx2_dma_unmap_skb_frags(pfvf, sg);
164 napi_consume_skb(skb, budget);
165 sg->skb = (u64)NULL;
166 }
167
otx2_set_rxtstamp(struct otx2_nic * pfvf,struct sk_buff * skb,void * data)168 static void otx2_set_rxtstamp(struct otx2_nic *pfvf,
169 struct sk_buff *skb, void *data)
170 {
171 u64 timestamp, tsns;
172 int err;
173
174 if (!(pfvf->flags & OTX2_FLAG_RX_TSTAMP_ENABLED))
175 return;
176
177 timestamp = pfvf->ptp->convert_rx_ptp_tstmp(*(u64 *)data);
178 /* The first 8 bytes is the timestamp */
179 err = otx2_ptp_tstamp2time(pfvf, timestamp, &tsns);
180 if (err)
181 return;
182
183 skb_hwtstamps(skb)->hwtstamp = ns_to_ktime(tsns);
184 }
185
otx2_skb_add_frag(struct otx2_nic * pfvf,struct sk_buff * skb,u64 iova,int len,struct nix_rx_parse_s * parse,int qidx)186 static bool otx2_skb_add_frag(struct otx2_nic *pfvf, struct sk_buff *skb,
187 u64 iova, int len, struct nix_rx_parse_s *parse,
188 int qidx)
189 {
190 struct page *page;
191 int off = 0;
192 void *va;
193
194 va = phys_to_virt(otx2_iova_to_phys(pfvf->iommu_domain, iova));
195
196 if (likely(!skb_shinfo(skb)->nr_frags)) {
197 /* Check if data starts at some nonzero offset
198 * from the start of the buffer. For now the
199 * only possible offset is 8 bytes in the case
200 * where packet is prepended by a timestamp.
201 */
202 if (parse->laptr) {
203 otx2_set_rxtstamp(pfvf, skb, va);
204 off = OTX2_HW_TIMESTAMP_LEN;
205 }
206 }
207
208 page = virt_to_page(va);
209 if (likely(skb_shinfo(skb)->nr_frags < MAX_SKB_FRAGS)) {
210 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,
211 va - page_address(page) + off,
212 len - off, pfvf->rbsize);
213 return true;
214 }
215
216 /* If more than MAX_SKB_FRAGS fragments are received then
217 * give back those buffer pointers to hardware for reuse.
218 */
219 pfvf->hw_ops->aura_freeptr(pfvf, qidx, iova & ~0x07ULL);
220
221 return false;
222 }
223
otx2_set_rxhash(struct otx2_nic * pfvf,struct nix_cqe_rx_s * cqe,struct sk_buff * skb)224 static void otx2_set_rxhash(struct otx2_nic *pfvf,
225 struct nix_cqe_rx_s *cqe, struct sk_buff *skb)
226 {
227 enum pkt_hash_types hash_type = PKT_HASH_TYPE_NONE;
228 struct otx2_rss_info *rss;
229 u32 hash = 0;
230
231 if (!(pfvf->netdev->features & NETIF_F_RXHASH))
232 return;
233
234 rss = &pfvf->hw.rss_info;
235 if (rss->flowkey_cfg) {
236 if (rss->flowkey_cfg &
237 ~(NIX_FLOW_KEY_TYPE_IPV4 | NIX_FLOW_KEY_TYPE_IPV6))
238 hash_type = PKT_HASH_TYPE_L4;
239 else
240 hash_type = PKT_HASH_TYPE_L3;
241 hash = cqe->hdr.flow_tag;
242 }
243 skb_set_hash(skb, hash, hash_type);
244 }
245
otx2_free_rcv_seg(struct otx2_nic * pfvf,struct nix_cqe_rx_s * cqe,int qidx)246 static void otx2_free_rcv_seg(struct otx2_nic *pfvf, struct nix_cqe_rx_s *cqe,
247 int qidx)
248 {
249 struct nix_rx_sg_s *sg = &cqe->sg;
250 void *end, *start;
251 u64 *seg_addr;
252 int seg;
253
254 start = (void *)sg;
255 end = start + ((cqe->parse.desc_sizem1 + 1) * 16);
256 while (start < end) {
257 sg = (struct nix_rx_sg_s *)start;
258 seg_addr = &sg->seg_addr;
259 for (seg = 0; seg < sg->segs; seg++, seg_addr++)
260 pfvf->hw_ops->aura_freeptr(pfvf, qidx,
261 *seg_addr & ~0x07ULL);
262 start += sizeof(*sg);
263 }
264 }
265
otx2_check_rcv_errors(struct otx2_nic * pfvf,struct nix_cqe_rx_s * cqe,int qidx)266 static bool otx2_check_rcv_errors(struct otx2_nic *pfvf,
267 struct nix_cqe_rx_s *cqe, int qidx)
268 {
269 struct otx2_drv_stats *stats = &pfvf->hw.drv_stats;
270 struct nix_rx_parse_s *parse = &cqe->parse;
271
272 if (netif_msg_rx_err(pfvf))
273 netdev_err(pfvf->netdev,
274 "RQ%d: Error pkt with errlev:0x%x errcode:0x%x\n",
275 qidx, parse->errlev, parse->errcode);
276
277 if (parse->errlev == NPC_ERRLVL_RE) {
278 switch (parse->errcode) {
279 case ERRCODE_FCS:
280 case ERRCODE_FCS_RCV:
281 atomic_inc(&stats->rx_fcs_errs);
282 break;
283 case ERRCODE_UNDERSIZE:
284 atomic_inc(&stats->rx_undersize_errs);
285 break;
286 case ERRCODE_OVERSIZE:
287 atomic_inc(&stats->rx_oversize_errs);
288 break;
289 case ERRCODE_OL2_LEN_MISMATCH:
290 atomic_inc(&stats->rx_len_errs);
291 break;
292 default:
293 atomic_inc(&stats->rx_other_errs);
294 break;
295 }
296 } else if (parse->errlev == NPC_ERRLVL_NIX) {
297 switch (parse->errcode) {
298 case ERRCODE_OL3_LEN:
299 case ERRCODE_OL4_LEN:
300 case ERRCODE_IL3_LEN:
301 case ERRCODE_IL4_LEN:
302 atomic_inc(&stats->rx_len_errs);
303 break;
304 case ERRCODE_OL4_CSUM:
305 case ERRCODE_IL4_CSUM:
306 atomic_inc(&stats->rx_csum_errs);
307 break;
308 default:
309 atomic_inc(&stats->rx_other_errs);
310 break;
311 }
312 } else {
313 atomic_inc(&stats->rx_other_errs);
314 /* For now ignore all the NPC parser errors and
315 * pass the packets to stack.
316 */
317 return false;
318 }
319
320 /* If RXALL is enabled pass on packets to stack. */
321 if (pfvf->netdev->features & NETIF_F_RXALL)
322 return false;
323
324 /* Free buffer back to pool */
325 if (cqe->sg.segs)
326 otx2_free_rcv_seg(pfvf, cqe, qidx);
327 return true;
328 }
329
otx2_rcv_pkt_handler(struct otx2_nic * pfvf,struct napi_struct * napi,struct otx2_cq_queue * cq,struct nix_cqe_rx_s * cqe,bool * need_xdp_flush)330 static void otx2_rcv_pkt_handler(struct otx2_nic *pfvf,
331 struct napi_struct *napi,
332 struct otx2_cq_queue *cq,
333 struct nix_cqe_rx_s *cqe, bool *need_xdp_flush)
334 {
335 struct nix_rx_parse_s *parse = &cqe->parse;
336 struct nix_rx_sg_s *sg = &cqe->sg;
337 struct sk_buff *skb = NULL;
338 void *end, *start;
339 u32 metasize = 0;
340 u64 *seg_addr;
341 u16 *seg_size;
342 int seg;
343
344 if (unlikely(parse->errlev || parse->errcode)) {
345 if (otx2_check_rcv_errors(pfvf, cqe, cq->cq_idx))
346 return;
347 }
348
349 if (pfvf->xdp_prog)
350 if (otx2_xdp_rcv_pkt_handler(pfvf, pfvf->xdp_prog, cqe, cq,
351 &metasize, need_xdp_flush))
352 return;
353
354 skb = napi_get_frags(napi);
355 if (unlikely(!skb))
356 return;
357
358 start = (void *)sg;
359 end = start + ((cqe->parse.desc_sizem1 + 1) * 16);
360 while (start < end) {
361 sg = (struct nix_rx_sg_s *)start;
362 seg_addr = &sg->seg_addr;
363 seg_size = (void *)sg;
364 for (seg = 0; seg < sg->segs; seg++, seg_addr++) {
365 if (otx2_skb_add_frag(pfvf, skb, *seg_addr,
366 seg_size[seg], parse, cq->cq_idx))
367 cq->pool_ptrs++;
368 }
369 start += sizeof(*sg);
370 }
371 otx2_set_rxhash(pfvf, cqe, skb);
372
373 if (!(pfvf->flags & OTX2_FLAG_REP_MODE_ENABLED)) {
374 skb_record_rx_queue(skb, cq->cq_idx);
375 if (pfvf->netdev->features & NETIF_F_RXCSUM)
376 skb->ip_summed = CHECKSUM_UNNECESSARY;
377 }
378
379 if (pfvf->flags & OTX2_FLAG_TC_MARK_ENABLED)
380 skb->mark = parse->match_id;
381
382 skb_mark_for_recycle(skb);
383 if (metasize)
384 skb_metadata_set(skb, metasize);
385
386 napi_gro_frags(napi);
387 }
388
otx2_rx_napi_handler(struct otx2_nic * pfvf,struct napi_struct * napi,struct otx2_cq_queue * cq,int budget)389 static int otx2_rx_napi_handler(struct otx2_nic *pfvf,
390 struct napi_struct *napi,
391 struct otx2_cq_queue *cq, int budget)
392 {
393 bool need_xdp_flush = false;
394 struct nix_cqe_rx_s *cqe;
395 int processed_cqe = 0;
396
397 if (cq->pend_cqe >= budget)
398 goto process_cqe;
399
400 if (otx2_nix_cq_op_status(pfvf, cq) || !cq->pend_cqe)
401 return 0;
402
403 process_cqe:
404 while (likely(processed_cqe < budget) && cq->pend_cqe) {
405 cqe = (struct nix_cqe_rx_s *)CQE_ADDR(cq, cq->cq_head);
406 if (cqe->hdr.cqe_type == NIX_XQE_TYPE_INVALID ||
407 !cqe->sg.seg_addr) {
408 if (!processed_cqe)
409 return 0;
410 break;
411 }
412 cq->cq_head++;
413 cq->cq_head &= (cq->cqe_cnt - 1);
414
415 otx2_rcv_pkt_handler(pfvf, napi, cq, cqe, &need_xdp_flush);
416
417 cqe->hdr.cqe_type = NIX_XQE_TYPE_INVALID;
418 cqe->sg.seg_addr = 0x00;
419 processed_cqe++;
420 cq->pend_cqe--;
421 }
422 if (need_xdp_flush)
423 xdp_do_flush();
424
425 /* Free CQEs to HW */
426 otx2_write64(pfvf, NIX_LF_CQ_OP_DOOR,
427 ((u64)cq->cq_idx << 32) | processed_cqe);
428
429 return processed_cqe;
430 }
431
otx2_refill_pool_ptrs(void * dev,struct otx2_cq_queue * cq)432 int otx2_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq)
433 {
434 struct otx2_nic *pfvf = dev;
435 int cnt = cq->pool_ptrs;
436 dma_addr_t bufptr;
437
438 while (cq->pool_ptrs) {
439 if (otx2_alloc_buffer(pfvf, cq, &bufptr))
440 break;
441 otx2_aura_freeptr(pfvf, cq->cq_idx, bufptr + OTX2_HEAD_ROOM);
442 cq->pool_ptrs--;
443 }
444
445 return cnt - cq->pool_ptrs;
446 }
447
otx2_zc_submit_pkts(struct otx2_nic * pfvf,struct xsk_buff_pool * xsk_pool,int * xsk_frames,int qidx,int budget)448 static void otx2_zc_submit_pkts(struct otx2_nic *pfvf, struct xsk_buff_pool *xsk_pool,
449 int *xsk_frames, int qidx, int budget)
450 {
451 if (*xsk_frames)
452 xsk_tx_completed(xsk_pool, *xsk_frames);
453
454 if (xsk_uses_need_wakeup(xsk_pool))
455 xsk_set_tx_need_wakeup(xsk_pool);
456
457 otx2_zc_napi_handler(pfvf, xsk_pool, qidx, budget);
458 }
459
otx2_tx_napi_handler(struct otx2_nic * pfvf,struct otx2_cq_queue * cq,int budget)460 static int otx2_tx_napi_handler(struct otx2_nic *pfvf,
461 struct otx2_cq_queue *cq, int budget)
462 {
463 int tx_pkts = 0, tx_bytes = 0, qidx;
464 struct otx2_snd_queue *sq;
465 struct nix_cqe_tx_s *cqe;
466 struct net_device *ndev;
467 int processed_cqe = 0;
468 int xsk_frames = 0;
469
470 qidx = cq->cq_idx - pfvf->hw.rx_queues;
471 sq = &pfvf->qset.sq[qidx];
472
473 if (cq->pend_cqe >= budget)
474 goto process_cqe;
475
476 if (otx2_nix_cq_op_status(pfvf, cq) || !cq->pend_cqe) {
477 if (sq->xsk_pool)
478 otx2_zc_submit_pkts(pfvf, sq->xsk_pool, &xsk_frames,
479 qidx, budget);
480 return 0;
481 }
482
483 process_cqe:
484
485 while (likely(processed_cqe < budget) && cq->pend_cqe) {
486 cqe = (struct nix_cqe_tx_s *)otx2_get_next_cqe(cq);
487 if (unlikely(!cqe)) {
488 if (!processed_cqe)
489 return 0;
490 break;
491 }
492
493 if (cq->cq_type == CQ_XDP)
494 otx2_xdp_snd_pkt_handler(pfvf, sq, cqe, &xsk_frames);
495 else
496 otx2_snd_pkt_handler(pfvf, cq, &pfvf->qset.sq[qidx],
497 cqe, budget, &tx_pkts, &tx_bytes);
498
499 cqe->hdr.cqe_type = NIX_XQE_TYPE_INVALID;
500 processed_cqe++;
501 cq->pend_cqe--;
502
503 sq->cons_head++;
504 sq->cons_head &= (sq->sqe_cnt - 1);
505 }
506
507 /* Free CQEs to HW */
508 otx2_write64(pfvf, NIX_LF_CQ_OP_DOOR,
509 ((u64)cq->cq_idx << 32) | processed_cqe);
510
511 #if IS_ENABLED(CONFIG_RVU_ESWITCH)
512 if (pfvf->flags & OTX2_FLAG_REP_MODE_ENABLED)
513 ndev = pfvf->reps[qidx]->netdev;
514 else
515 #endif
516 ndev = pfvf->netdev;
517
518 if (likely(tx_pkts)) {
519 struct netdev_queue *txq;
520
521 qidx = cq->cq_idx - pfvf->hw.rx_queues;
522
523 if (qidx >= pfvf->hw.tx_queues)
524 qidx -= pfvf->hw.xdp_queues;
525 if (pfvf->flags & OTX2_FLAG_REP_MODE_ENABLED)
526 qidx = 0;
527 txq = netdev_get_tx_queue(ndev, qidx);
528 netdev_tx_completed_queue(txq, tx_pkts, tx_bytes);
529 /* Check if queue was stopped earlier due to ring full */
530 smp_mb();
531 if (netif_tx_queue_stopped(txq) &&
532 netif_carrier_ok(ndev))
533 netif_tx_wake_queue(txq);
534 }
535
536 if (sq->xsk_pool)
537 otx2_zc_submit_pkts(pfvf, sq->xsk_pool, &xsk_frames, qidx, budget);
538
539 return 0;
540 }
541
otx2_adjust_adaptive_coalese(struct otx2_nic * pfvf,struct otx2_cq_poll * cq_poll)542 static void otx2_adjust_adaptive_coalese(struct otx2_nic *pfvf, struct otx2_cq_poll *cq_poll)
543 {
544 struct dim_sample dim_sample = { 0 };
545 u64 rx_frames, rx_bytes;
546 u64 tx_frames, tx_bytes;
547
548 rx_frames = OTX2_GET_RX_STATS(RX_BCAST) + OTX2_GET_RX_STATS(RX_MCAST) +
549 OTX2_GET_RX_STATS(RX_UCAST);
550 rx_bytes = OTX2_GET_RX_STATS(RX_OCTS);
551 tx_bytes = OTX2_GET_TX_STATS(TX_OCTS);
552 tx_frames = OTX2_GET_TX_STATS(TX_UCAST);
553
554 dim_update_sample(pfvf->napi_events,
555 rx_frames + tx_frames,
556 rx_bytes + tx_bytes,
557 &dim_sample);
558 net_dim(&cq_poll->dim, &dim_sample);
559 }
560
otx2_napi_handler(struct napi_struct * napi,int budget)561 int otx2_napi_handler(struct napi_struct *napi, int budget)
562 {
563 struct otx2_cq_queue *rx_cq = NULL;
564 struct otx2_cq_queue *cq = NULL;
565 struct otx2_pool *pool = NULL;
566 struct otx2_cq_poll *cq_poll;
567 int workdone = 0, cq_idx, i;
568 struct otx2_qset *qset;
569 struct otx2_nic *pfvf;
570 int filled_cnt = -1;
571
572 cq_poll = container_of(napi, struct otx2_cq_poll, napi);
573 pfvf = (struct otx2_nic *)cq_poll->dev;
574 qset = &pfvf->qset;
575
576 for (i = 0; i < CQS_PER_CINT; i++) {
577 cq_idx = cq_poll->cq_ids[i];
578 if (unlikely(cq_idx == CINT_INVALID_CQ))
579 continue;
580 cq = &qset->cq[cq_idx];
581 if (cq->cq_type == CQ_RX) {
582 rx_cq = cq;
583 workdone += otx2_rx_napi_handler(pfvf, napi,
584 cq, budget);
585 } else {
586 workdone += otx2_tx_napi_handler(pfvf, cq, budget);
587 }
588 }
589
590 if (rx_cq && rx_cq->pool_ptrs)
591 filled_cnt = pfvf->hw_ops->refill_pool_ptrs(pfvf, rx_cq);
592
593 /* Clear the IRQ */
594 otx2_write64(pfvf, NIX_LF_CINTX_INT(cq_poll->cint_idx), BIT_ULL(0));
595
596 if (workdone < budget && napi_complete_done(napi, workdone)) {
597 /* If interface is going down, don't re-enable IRQ */
598 if (pfvf->flags & OTX2_FLAG_INTF_DOWN)
599 return workdone;
600
601 /* Adjust irq coalese using net_dim */
602 if (pfvf->flags & OTX2_FLAG_ADPTV_INT_COAL_ENABLED)
603 otx2_adjust_adaptive_coalese(pfvf, cq_poll);
604
605 if (likely(cq))
606 pool = &pfvf->qset.pool[cq->cq_idx];
607
608 if (unlikely(!filled_cnt)) {
609 struct refill_work *work;
610 struct delayed_work *dwork;
611
612 if (likely(cq)) {
613 work = &pfvf->refill_wrk[cq->cq_idx];
614 dwork = &work->pool_refill_work;
615 /* Schedule a task if no other task is running */
616 if (!cq->refill_task_sched) {
617 work->napi = napi;
618 cq->refill_task_sched = true;
619 schedule_delayed_work(dwork,
620 msecs_to_jiffies(100));
621 }
622 /* Call wake-up for not able to fill buffers */
623 if (pool->xsk_pool)
624 xsk_set_rx_need_wakeup(pool->xsk_pool);
625 }
626 } else {
627 /* Clear wake-up, since buffers are filled successfully */
628 if (pool && pool->xsk_pool)
629 xsk_clear_rx_need_wakeup(pool->xsk_pool);
630 /* Re-enable interrupts */
631 otx2_write64(pfvf,
632 NIX_LF_CINTX_ENA_W1S(cq_poll->cint_idx),
633 BIT_ULL(0));
634 }
635 }
636 return workdone;
637 }
638 EXPORT_SYMBOL(otx2_napi_handler);
639
otx2_sqe_flush(void * dev,struct otx2_snd_queue * sq,int size,int qidx)640 void otx2_sqe_flush(void *dev, struct otx2_snd_queue *sq,
641 int size, int qidx)
642 {
643 u64 status;
644
645 /* Packet data stores should finish before SQE is flushed to HW */
646 dma_wmb();
647
648 do {
649 memcpy(sq->lmt_addr, sq->sqe_base, size);
650 status = otx2_lmt_flush(sq->io_addr);
651 } while (status == 0);
652
653 sq->head++;
654 sq->head &= (sq->sqe_cnt - 1);
655 }
656
657 /* Add SQE scatter/gather subdescriptor structure */
otx2_sqe_add_sg(struct otx2_nic * pfvf,struct otx2_snd_queue * sq,struct sk_buff * skb,int num_segs,int * offset)658 static bool otx2_sqe_add_sg(struct otx2_nic *pfvf, struct otx2_snd_queue *sq,
659 struct sk_buff *skb, int num_segs, int *offset)
660 {
661 struct nix_sqe_sg_s *sg = NULL;
662 u64 dma_addr, *iova = NULL;
663 u16 *sg_lens = NULL;
664 int seg, len;
665
666 sq->sg[sq->head].num_segs = 0;
667
668 for (seg = 0; seg < num_segs; seg++) {
669 if ((seg % MAX_SEGS_PER_SG) == 0) {
670 sg = (struct nix_sqe_sg_s *)(sq->sqe_base + *offset);
671 sg->ld_type = NIX_SEND_LDTYPE_LDD;
672 sg->subdc = NIX_SUBDC_SG;
673 sg->segs = 0;
674 sg_lens = (void *)sg;
675 iova = (void *)sg + sizeof(*sg);
676 /* Next subdc always starts at a 16byte boundary.
677 * So if sg->segs is whether 2 or 3, offset += 16bytes.
678 */
679 if ((num_segs - seg) >= (MAX_SEGS_PER_SG - 1))
680 *offset += sizeof(*sg) + (3 * sizeof(u64));
681 else
682 *offset += sizeof(*sg) + sizeof(u64);
683 }
684 dma_addr = otx2_dma_map_skb_frag(pfvf, skb, seg, &len);
685 if (dma_mapping_error(pfvf->dev, dma_addr))
686 return false;
687
688 sg_lens[frag_num(seg % MAX_SEGS_PER_SG)] = len;
689 sg->segs++;
690 *iova++ = dma_addr;
691
692 /* Save DMA mapping info for later unmapping */
693 sq->sg[sq->head].dma_addr[seg] = dma_addr;
694 sq->sg[sq->head].size[seg] = len;
695 sq->sg[sq->head].num_segs++;
696 }
697
698 sq->sg[sq->head].skb = (u64)skb;
699 return true;
700 }
701
702 /* Add SQE extended header subdescriptor */
otx2_sqe_add_ext(struct otx2_nic * pfvf,struct otx2_snd_queue * sq,struct sk_buff * skb,int * offset)703 static void otx2_sqe_add_ext(struct otx2_nic *pfvf, struct otx2_snd_queue *sq,
704 struct sk_buff *skb, int *offset)
705 {
706 struct nix_sqe_ext_s *ext;
707
708 ext = (struct nix_sqe_ext_s *)(sq->sqe_base + *offset);
709 ext->subdc = NIX_SUBDC_EXT;
710 if (skb_shinfo(skb)->gso_size) {
711 ext->lso = 1;
712 ext->lso_sb = skb_tcp_all_headers(skb);
713 ext->lso_mps = skb_shinfo(skb)->gso_size;
714
715 /* Only TSOv4 and TSOv6 GSO offloads are supported */
716 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
717 ext->lso_format = pfvf->hw.lso_tsov4_idx;
718
719 /* HW adds payload size to 'ip_hdr->tot_len' while
720 * sending TSO segment, hence set payload length
721 * in IP header of the packet to just header length.
722 */
723 ip_hdr(skb)->tot_len =
724 htons(ext->lso_sb - skb_network_offset(skb));
725 } else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) {
726 ext->lso_format = pfvf->hw.lso_tsov6_idx;
727 ipv6_hdr(skb)->payload_len = htons(tcp_hdrlen(skb));
728 } else if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) {
729 __be16 l3_proto = vlan_get_protocol(skb);
730 struct udphdr *udph = udp_hdr(skb);
731 __be16 iplen;
732
733 ext->lso_sb = skb_transport_offset(skb) +
734 sizeof(struct udphdr);
735
736 /* HW adds payload size to length fields in IP and
737 * UDP headers while segmentation, hence adjust the
738 * lengths to just header sizes.
739 */
740 iplen = htons(ext->lso_sb - skb_network_offset(skb));
741 if (l3_proto == htons(ETH_P_IP)) {
742 ip_hdr(skb)->tot_len = iplen;
743 ext->lso_format = pfvf->hw.lso_udpv4_idx;
744 } else {
745 ipv6_hdr(skb)->payload_len = iplen;
746 ext->lso_format = pfvf->hw.lso_udpv6_idx;
747 }
748
749 udph->len = htons(sizeof(struct udphdr));
750 }
751 } else if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) {
752 ext->tstmp = 1;
753 }
754
755 #define OTX2_VLAN_PTR_OFFSET (ETH_HLEN - ETH_TLEN)
756 if (skb_vlan_tag_present(skb)) {
757 if (skb->vlan_proto == htons(ETH_P_8021Q)) {
758 ext->vlan1_ins_ena = 1;
759 ext->vlan1_ins_ptr = OTX2_VLAN_PTR_OFFSET;
760 ext->vlan1_ins_tci = skb_vlan_tag_get(skb);
761 } else if (skb->vlan_proto == htons(ETH_P_8021AD)) {
762 ext->vlan0_ins_ena = 1;
763 ext->vlan0_ins_ptr = OTX2_VLAN_PTR_OFFSET;
764 ext->vlan0_ins_tci = skb_vlan_tag_get(skb);
765 }
766 }
767
768 *offset += sizeof(*ext);
769 }
770
otx2_sqe_add_mem(struct otx2_snd_queue * sq,int * offset,int alg,u64 iova,int ptp_offset,u64 base_ns,bool udp_csum_crt)771 static void otx2_sqe_add_mem(struct otx2_snd_queue *sq, int *offset,
772 int alg, u64 iova, int ptp_offset,
773 u64 base_ns, bool udp_csum_crt)
774 {
775 struct nix_sqe_mem_s *mem;
776
777 mem = (struct nix_sqe_mem_s *)(sq->sqe_base + *offset);
778 mem->subdc = NIX_SUBDC_MEM;
779 mem->alg = alg;
780 mem->wmem = 1; /* wait for the memory operation */
781 mem->addr = iova;
782
783 if (ptp_offset) {
784 mem->start_offset = ptp_offset;
785 mem->udp_csum_crt = !!udp_csum_crt;
786 mem->base_ns = base_ns;
787 mem->step_type = 1;
788 }
789
790 *offset += sizeof(*mem);
791 }
792
793 /* Add SQE header subdescriptor structure */
otx2_sqe_add_hdr(struct otx2_nic * pfvf,struct otx2_snd_queue * sq,struct nix_sqe_hdr_s * sqe_hdr,struct sk_buff * skb,u16 qidx)794 static void otx2_sqe_add_hdr(struct otx2_nic *pfvf, struct otx2_snd_queue *sq,
795 struct nix_sqe_hdr_s *sqe_hdr,
796 struct sk_buff *skb, u16 qidx)
797 {
798 int proto = 0;
799
800 /* Check if SQE was framed before, if yes then no need to
801 * set these constants again and again.
802 */
803 if (!sqe_hdr->total) {
804 /* Don't free Tx buffers to Aura */
805 sqe_hdr->df = 1;
806 sqe_hdr->aura = sq->aura_id;
807 /* Post a CQE Tx after pkt transmission */
808 sqe_hdr->pnc = 1;
809 sqe_hdr->sq = (qidx >= pfvf->hw.tx_queues) ?
810 qidx + pfvf->hw.xdp_queues : qidx;
811 }
812 sqe_hdr->total = skb->len;
813 /* Set SQE identifier which will be used later for freeing SKB */
814 sqe_hdr->sqe_id = sq->head;
815
816 /* Offload TCP/UDP checksum to HW */
817 if (skb->ip_summed == CHECKSUM_PARTIAL) {
818 sqe_hdr->ol3ptr = skb_network_offset(skb);
819 sqe_hdr->ol4ptr = skb_transport_offset(skb);
820 /* get vlan protocol Ethertype */
821 if (eth_type_vlan(skb->protocol))
822 skb->protocol = vlan_get_protocol(skb);
823
824 if (skb->protocol == htons(ETH_P_IP)) {
825 proto = ip_hdr(skb)->protocol;
826 /* In case of TSO, HW needs this to be explicitly set.
827 * So set this always, instead of adding a check.
828 */
829 sqe_hdr->ol3type = NIX_SENDL3TYPE_IP4_CKSUM;
830 } else if (skb->protocol == htons(ETH_P_IPV6)) {
831 proto = ipv6_hdr(skb)->nexthdr;
832 sqe_hdr->ol3type = NIX_SENDL3TYPE_IP6;
833 }
834
835 if (proto == IPPROTO_TCP)
836 sqe_hdr->ol4type = NIX_SENDL4TYPE_TCP_CKSUM;
837 else if (proto == IPPROTO_UDP)
838 sqe_hdr->ol4type = NIX_SENDL4TYPE_UDP_CKSUM;
839 }
840 }
841
otx2_dma_map_tso_skb(struct otx2_nic * pfvf,struct otx2_snd_queue * sq,struct sk_buff * skb,int sqe,int hdr_len)842 static int otx2_dma_map_tso_skb(struct otx2_nic *pfvf,
843 struct otx2_snd_queue *sq,
844 struct sk_buff *skb, int sqe, int hdr_len)
845 {
846 int num_segs = skb_shinfo(skb)->nr_frags + 1;
847 struct sg_list *sg = &sq->sg[sqe];
848 u64 dma_addr;
849 int seg, len;
850
851 sg->num_segs = 0;
852
853 /* Get payload length at skb->data */
854 len = skb_headlen(skb) - hdr_len;
855
856 for (seg = 0; seg < num_segs; seg++) {
857 /* Skip skb->data, if there is no payload */
858 if (!seg && !len)
859 continue;
860 dma_addr = otx2_dma_map_skb_frag(pfvf, skb, seg, &len);
861 if (dma_mapping_error(pfvf->dev, dma_addr))
862 goto unmap;
863
864 /* Save DMA mapping info for later unmapping */
865 sg->dma_addr[sg->num_segs] = dma_addr;
866 sg->size[sg->num_segs] = len;
867 sg->num_segs++;
868 }
869 return 0;
870 unmap:
871 otx2_dma_unmap_skb_frags(pfvf, sg);
872 return -EINVAL;
873 }
874
otx2_tso_frag_dma_addr(struct otx2_snd_queue * sq,struct sk_buff * skb,int seg,u64 seg_addr,int hdr_len,int sqe)875 static u64 otx2_tso_frag_dma_addr(struct otx2_snd_queue *sq,
876 struct sk_buff *skb, int seg,
877 u64 seg_addr, int hdr_len, int sqe)
878 {
879 struct sg_list *sg = &sq->sg[sqe];
880 const skb_frag_t *frag;
881 int offset;
882
883 if (seg < 0)
884 return sg->dma_addr[0] + (seg_addr - (u64)skb->data);
885
886 frag = &skb_shinfo(skb)->frags[seg];
887 offset = seg_addr - (u64)skb_frag_address(frag);
888 if (skb_headlen(skb) - hdr_len)
889 seg++;
890 return sg->dma_addr[seg] + offset;
891 }
892
otx2_sqe_tso_add_sg(struct otx2_snd_queue * sq,struct sg_list * list,int * offset)893 static void otx2_sqe_tso_add_sg(struct otx2_snd_queue *sq,
894 struct sg_list *list, int *offset)
895 {
896 struct nix_sqe_sg_s *sg = NULL;
897 u16 *sg_lens = NULL;
898 u64 *iova = NULL;
899 int seg;
900
901 /* Add SG descriptors with buffer addresses */
902 for (seg = 0; seg < list->num_segs; seg++) {
903 if ((seg % MAX_SEGS_PER_SG) == 0) {
904 sg = (struct nix_sqe_sg_s *)(sq->sqe_base + *offset);
905 sg->ld_type = NIX_SEND_LDTYPE_LDD;
906 sg->subdc = NIX_SUBDC_SG;
907 sg->segs = 0;
908 sg_lens = (void *)sg;
909 iova = (void *)sg + sizeof(*sg);
910 /* Next subdc always starts at a 16byte boundary.
911 * So if sg->segs is whether 2 or 3, offset += 16bytes.
912 */
913 if ((list->num_segs - seg) >= (MAX_SEGS_PER_SG - 1))
914 *offset += sizeof(*sg) + (3 * sizeof(u64));
915 else
916 *offset += sizeof(*sg) + sizeof(u64);
917 }
918 sg_lens[frag_num(seg % MAX_SEGS_PER_SG)] = list->size[seg];
919 *iova++ = list->dma_addr[seg];
920 sg->segs++;
921 }
922 }
923
otx2_sq_append_tso(struct otx2_nic * pfvf,struct otx2_snd_queue * sq,struct sk_buff * skb,u16 qidx)924 static void otx2_sq_append_tso(struct otx2_nic *pfvf, struct otx2_snd_queue *sq,
925 struct sk_buff *skb, u16 qidx)
926 {
927 struct netdev_queue *txq = netdev_get_tx_queue(pfvf->netdev, qidx);
928 int hdr_len, tcp_data, seg_len, pkt_len, offset;
929 struct nix_sqe_hdr_s *sqe_hdr;
930 int first_sqe = sq->head;
931 struct sg_list list;
932 struct tso_t tso;
933
934 hdr_len = tso_start(skb, &tso);
935
936 /* Map SKB's fragments to DMA.
937 * It's done here to avoid mapping for every TSO segment's packet.
938 */
939 if (otx2_dma_map_tso_skb(pfvf, sq, skb, first_sqe, hdr_len)) {
940 dev_kfree_skb_any(skb);
941 return;
942 }
943
944 netdev_tx_sent_queue(txq, skb->len);
945
946 tcp_data = skb->len - hdr_len;
947 while (tcp_data > 0) {
948 char *hdr;
949
950 seg_len = min_t(int, skb_shinfo(skb)->gso_size, tcp_data);
951 tcp_data -= seg_len;
952
953 /* Set SQE's SEND_HDR */
954 memset(sq->sqe_base, 0, sq->sqe_size);
955 sqe_hdr = (struct nix_sqe_hdr_s *)(sq->sqe_base);
956 otx2_sqe_add_hdr(pfvf, sq, sqe_hdr, skb, qidx);
957 offset = sizeof(*sqe_hdr);
958
959 /* Add TSO segment's pkt header */
960 hdr = sq->tso_hdrs->base + (sq->head * TSO_HEADER_SIZE);
961 tso_build_hdr(skb, hdr, &tso, seg_len, tcp_data == 0);
962 list.dma_addr[0] =
963 sq->tso_hdrs->iova + (sq->head * TSO_HEADER_SIZE);
964 list.size[0] = hdr_len;
965 list.num_segs = 1;
966
967 /* Add TSO segment's payload data fragments */
968 pkt_len = hdr_len;
969 while (seg_len > 0) {
970 int size;
971
972 size = min_t(int, tso.size, seg_len);
973
974 list.size[list.num_segs] = size;
975 list.dma_addr[list.num_segs] =
976 otx2_tso_frag_dma_addr(sq, skb,
977 tso.next_frag_idx - 1,
978 (u64)tso.data, hdr_len,
979 first_sqe);
980 list.num_segs++;
981 pkt_len += size;
982 seg_len -= size;
983 tso_build_data(skb, &tso, size);
984 }
985 sqe_hdr->total = pkt_len;
986 otx2_sqe_tso_add_sg(sq, &list, &offset);
987
988 /* DMA mappings and skb needs to be freed only after last
989 * TSO segment is transmitted out. So set 'PNC' only for
990 * last segment. Also point last segment's sqe_id to first
991 * segment's SQE index where skb address and DMA mappings
992 * are saved.
993 */
994 if (!tcp_data) {
995 sqe_hdr->pnc = 1;
996 sqe_hdr->sqe_id = first_sqe;
997 sq->sg[first_sqe].skb = (u64)skb;
998 } else {
999 sqe_hdr->pnc = 0;
1000 }
1001
1002 sqe_hdr->sizem1 = (offset / 16) - 1;
1003
1004 /* Flush SQE to HW */
1005 pfvf->hw_ops->sqe_flush(pfvf, sq, offset, qidx);
1006 }
1007 }
1008
is_hw_tso_supported(struct otx2_nic * pfvf,struct sk_buff * skb)1009 static bool is_hw_tso_supported(struct otx2_nic *pfvf,
1010 struct sk_buff *skb)
1011 {
1012 int payload_len, last_seg_size;
1013
1014 if (test_bit(HW_TSO, &pfvf->hw.cap_flag))
1015 return true;
1016
1017 /* On 96xx A0, HW TSO not supported */
1018 if (!is_96xx_B0(pfvf->pdev))
1019 return false;
1020
1021 /* HW has an issue due to which when the payload of the last LSO
1022 * segment is shorter than 16 bytes, some header fields may not
1023 * be correctly modified, hence don't offload such TSO segments.
1024 */
1025
1026 payload_len = skb->len - skb_tcp_all_headers(skb);
1027 last_seg_size = payload_len % skb_shinfo(skb)->gso_size;
1028 if (last_seg_size && last_seg_size < 16)
1029 return false;
1030
1031 return true;
1032 }
1033
otx2_get_sqe_count(struct otx2_nic * pfvf,struct sk_buff * skb)1034 static int otx2_get_sqe_count(struct otx2_nic *pfvf, struct sk_buff *skb)
1035 {
1036 if (!skb_shinfo(skb)->gso_size)
1037 return 1;
1038
1039 /* HW TSO */
1040 if (is_hw_tso_supported(pfvf, skb))
1041 return 1;
1042
1043 /* SW TSO */
1044 return skb_shinfo(skb)->gso_segs;
1045 }
1046
otx2_validate_network_transport(struct sk_buff * skb)1047 static bool otx2_validate_network_transport(struct sk_buff *skb)
1048 {
1049 if ((ip_hdr(skb)->protocol == IPPROTO_UDP) ||
1050 (ipv6_hdr(skb)->nexthdr == IPPROTO_UDP)) {
1051 struct udphdr *udph = udp_hdr(skb);
1052
1053 if (udph->source == htons(PTP_PORT) &&
1054 udph->dest == htons(PTP_PORT))
1055 return true;
1056 }
1057
1058 return false;
1059 }
1060
otx2_ptp_is_sync(struct sk_buff * skb,int * offset,bool * udp_csum_crt)1061 static bool otx2_ptp_is_sync(struct sk_buff *skb, int *offset, bool *udp_csum_crt)
1062 {
1063 struct ethhdr *eth = (struct ethhdr *)(skb->data);
1064 u16 nix_offload_hlen = 0, inner_vhlen = 0;
1065 bool udp_hdr_present = false, is_sync;
1066 u8 *data = skb->data, *msgtype;
1067 __be16 proto = eth->h_proto;
1068 int network_depth = 0;
1069
1070 /* NIX is programmed to offload outer VLAN header
1071 * in case of single vlan protocol field holds Network header ETH_IP/V6
1072 * in case of stacked vlan protocol field holds Inner vlan (8100)
1073 */
1074 if (skb->dev->features & NETIF_F_HW_VLAN_CTAG_TX &&
1075 skb->dev->features & NETIF_F_HW_VLAN_STAG_TX) {
1076 if (skb->vlan_proto == htons(ETH_P_8021AD)) {
1077 /* Get vlan protocol */
1078 proto = __vlan_get_protocol(skb, eth->h_proto, NULL);
1079 /* SKB APIs like skb_transport_offset does not include
1080 * offloaded vlan header length. Need to explicitly add
1081 * the length
1082 */
1083 nix_offload_hlen = VLAN_HLEN;
1084 inner_vhlen = VLAN_HLEN;
1085 } else if (skb->vlan_proto == htons(ETH_P_8021Q)) {
1086 nix_offload_hlen = VLAN_HLEN;
1087 }
1088 } else if (eth_type_vlan(eth->h_proto)) {
1089 proto = __vlan_get_protocol(skb, eth->h_proto, &network_depth);
1090 }
1091
1092 switch (ntohs(proto)) {
1093 case ETH_P_1588:
1094 if (network_depth)
1095 *offset = network_depth;
1096 else
1097 *offset = ETH_HLEN + nix_offload_hlen +
1098 inner_vhlen;
1099 break;
1100 case ETH_P_IP:
1101 case ETH_P_IPV6:
1102 if (!otx2_validate_network_transport(skb))
1103 return false;
1104
1105 *offset = nix_offload_hlen + skb_transport_offset(skb) +
1106 sizeof(struct udphdr);
1107 udp_hdr_present = true;
1108
1109 }
1110
1111 msgtype = data + *offset;
1112 /* Check PTP messageId is SYNC or not */
1113 is_sync = !(*msgtype & 0xf);
1114 if (is_sync)
1115 *udp_csum_crt = udp_hdr_present;
1116 else
1117 *offset = 0;
1118
1119 return is_sync;
1120 }
1121
otx2_set_txtstamp(struct otx2_nic * pfvf,struct sk_buff * skb,struct otx2_snd_queue * sq,int * offset)1122 static void otx2_set_txtstamp(struct otx2_nic *pfvf, struct sk_buff *skb,
1123 struct otx2_snd_queue *sq, int *offset)
1124 {
1125 struct ethhdr *eth = (struct ethhdr *)(skb->data);
1126 struct ptpv2_tstamp *origin_tstamp;
1127 bool udp_csum_crt = false;
1128 unsigned int udphoff;
1129 struct timespec64 ts;
1130 int ptp_offset = 0;
1131 __wsum skb_csum;
1132 u64 iova;
1133
1134 if (unlikely(!skb_shinfo(skb)->gso_size &&
1135 (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))) {
1136 if (unlikely(pfvf->flags & OTX2_FLAG_PTP_ONESTEP_SYNC &&
1137 otx2_ptp_is_sync(skb, &ptp_offset, &udp_csum_crt))) {
1138 origin_tstamp = (struct ptpv2_tstamp *)
1139 ((u8 *)skb->data + ptp_offset +
1140 PTP_SYNC_SEC_OFFSET);
1141 ts = ns_to_timespec64(pfvf->ptp->tstamp);
1142 origin_tstamp->seconds_msb = htons((ts.tv_sec >> 32) & 0xffff);
1143 origin_tstamp->seconds_lsb = htonl(ts.tv_sec & 0xffffffff);
1144 origin_tstamp->nanoseconds = htonl(ts.tv_nsec);
1145 /* Point to correction field in PTP packet */
1146 ptp_offset += 8;
1147
1148 /* When user disables hw checksum, stack calculates the csum,
1149 * but it does not cover ptp timestamp which is added later.
1150 * Recalculate the checksum manually considering the timestamp.
1151 */
1152 if (udp_csum_crt) {
1153 struct udphdr *uh = udp_hdr(skb);
1154
1155 if (skb->ip_summed != CHECKSUM_PARTIAL && uh->check != 0) {
1156 udphoff = skb_transport_offset(skb);
1157 uh->check = 0;
1158 skb_csum = skb_checksum(skb, udphoff, skb->len - udphoff,
1159 0);
1160 if (ntohs(eth->h_proto) == ETH_P_IPV6)
1161 uh->check = csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
1162 &ipv6_hdr(skb)->daddr,
1163 skb->len - udphoff,
1164 ipv6_hdr(skb)->nexthdr,
1165 skb_csum);
1166 else
1167 uh->check = csum_tcpudp_magic(ip_hdr(skb)->saddr,
1168 ip_hdr(skb)->daddr,
1169 skb->len - udphoff,
1170 IPPROTO_UDP,
1171 skb_csum);
1172 }
1173 }
1174 } else {
1175 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1176 }
1177 iova = sq->timestamps->iova + (sq->head * sizeof(u64));
1178 otx2_sqe_add_mem(sq, offset, NIX_SENDMEMALG_E_SETTSTMP, iova,
1179 ptp_offset, pfvf->ptp->base_ns, udp_csum_crt);
1180 } else {
1181 skb_tx_timestamp(skb);
1182 }
1183 }
1184
otx2_sq_append_skb(void * dev,struct netdev_queue * txq,struct otx2_snd_queue * sq,struct sk_buff * skb,u16 qidx)1185 bool otx2_sq_append_skb(void *dev, struct netdev_queue *txq,
1186 struct otx2_snd_queue *sq,
1187 struct sk_buff *skb, u16 qidx)
1188 {
1189 int offset, num_segs, free_desc;
1190 struct nix_sqe_hdr_s *sqe_hdr;
1191 struct otx2_nic *pfvf = dev;
1192 bool ret;
1193
1194 /* Check if there is enough room between producer
1195 * and consumer index.
1196 */
1197 free_desc = otx2_get_free_sqe(sq);
1198 if (free_desc < sq->sqe_thresh)
1199 return false;
1200
1201 if (free_desc < otx2_get_sqe_count(pfvf, skb))
1202 return false;
1203
1204 num_segs = skb_shinfo(skb)->nr_frags + 1;
1205
1206 /* If SKB doesn't fit in a single SQE, linearize it.
1207 * TODO: Consider adding JUMP descriptor instead.
1208 */
1209
1210 if (unlikely(num_segs > OTX2_MAX_FRAGS_IN_SQE)) {
1211 if (__skb_linearize(skb)) {
1212 dev_kfree_skb_any(skb);
1213 return true;
1214 }
1215 num_segs = skb_shinfo(skb)->nr_frags + 1;
1216 }
1217
1218 if (skb_shinfo(skb)->gso_size && !is_hw_tso_supported(pfvf, skb)) {
1219 /* Insert vlan tag before giving pkt to tso */
1220 if (skb_vlan_tag_present(skb)) {
1221 skb = __vlan_hwaccel_push_inside(skb);
1222 if (!skb)
1223 return true;
1224 }
1225 otx2_sq_append_tso(pfvf, sq, skb, qidx);
1226 return true;
1227 }
1228
1229 /* Set sqe base address */
1230 otx2_sq_set_sqe_base(sq, skb);
1231
1232 /* Set SQE's SEND_HDR.
1233 * Do not clear the first 64bit as it contains constant info.
1234 */
1235 memset(sq->sqe_base + 8, 0, sq->sqe_size - 8);
1236 sqe_hdr = (struct nix_sqe_hdr_s *)(sq->sqe_base);
1237 otx2_sqe_add_hdr(pfvf, sq, sqe_hdr, skb, qidx);
1238 offset = sizeof(*sqe_hdr);
1239
1240 /* Add extended header if needed */
1241 otx2_sqe_add_ext(pfvf, sq, skb, &offset);
1242
1243 /* Add SG subdesc with data frags */
1244 if (static_branch_unlikely(&cn10k_ipsec_sa_enabled) &&
1245 (xfrm_offload(skb)))
1246 ret = otx2_sqe_add_sg_ipsec(pfvf, sq, skb, num_segs, &offset);
1247 else
1248 ret = otx2_sqe_add_sg(pfvf, sq, skb, num_segs, &offset);
1249
1250 if (!ret) {
1251 otx2_dma_unmap_skb_frags(pfvf, &sq->sg[sq->head]);
1252 return false;
1253 }
1254
1255 otx2_set_txtstamp(pfvf, skb, sq, &offset);
1256
1257 sqe_hdr->sizem1 = (offset / 16) - 1;
1258
1259 if (static_branch_unlikely(&cn10k_ipsec_sa_enabled) &&
1260 (xfrm_offload(skb)))
1261 return cn10k_ipsec_transmit(pfvf, txq, sq, skb, num_segs,
1262 offset);
1263
1264 netdev_tx_sent_queue(txq, skb->len);
1265
1266 /* Flush SQE to HW */
1267 pfvf->hw_ops->sqe_flush(pfvf, sq, offset, qidx);
1268 return true;
1269 }
1270 EXPORT_SYMBOL(otx2_sq_append_skb);
1271
otx2_cleanup_rx_cqes(struct otx2_nic * pfvf,struct otx2_cq_queue * cq,int qidx)1272 void otx2_cleanup_rx_cqes(struct otx2_nic *pfvf, struct otx2_cq_queue *cq, int qidx)
1273 {
1274 struct nix_cqe_rx_s *cqe;
1275 struct otx2_pool *pool;
1276 int processed_cqe = 0;
1277 u16 pool_id;
1278 u64 iova;
1279
1280 pool_id = otx2_get_pool_idx(pfvf, AURA_NIX_RQ, qidx);
1281 pool = &pfvf->qset.pool[pool_id];
1282
1283 if (pfvf->xdp_prog) {
1284 if (pool->page_pool)
1285 xdp_rxq_info_unreg_mem_model(&cq->xdp_rxq);
1286
1287 xdp_rxq_info_unreg(&cq->xdp_rxq);
1288 }
1289
1290 if (otx2_nix_cq_op_status(pfvf, cq) || !cq->pend_cqe)
1291 return;
1292
1293 while (cq->pend_cqe) {
1294 cqe = (struct nix_cqe_rx_s *)otx2_get_next_cqe(cq);
1295 processed_cqe++;
1296 cq->pend_cqe--;
1297
1298 if (!cqe)
1299 continue;
1300 if (cqe->sg.segs > 1) {
1301 otx2_free_rcv_seg(pfvf, cqe, cq->cq_idx);
1302 continue;
1303 }
1304 iova = cqe->sg.seg_addr - OTX2_HEAD_ROOM;
1305
1306 otx2_free_bufs(pfvf, pool, iova, pfvf->rbsize);
1307 }
1308
1309 /* Free CQEs to HW */
1310 otx2_write64(pfvf, NIX_LF_CQ_OP_DOOR,
1311 ((u64)cq->cq_idx << 32) | processed_cqe);
1312 }
1313
otx2_cleanup_tx_cqes(struct otx2_nic * pfvf,struct otx2_cq_queue * cq)1314 void otx2_cleanup_tx_cqes(struct otx2_nic *pfvf, struct otx2_cq_queue *cq)
1315 {
1316 int tx_pkts = 0, tx_bytes = 0;
1317 struct sk_buff *skb = NULL;
1318 struct otx2_snd_queue *sq;
1319 struct nix_cqe_tx_s *cqe;
1320 struct netdev_queue *txq;
1321 int processed_cqe = 0;
1322 struct sg_list *sg;
1323 int qidx;
1324
1325 qidx = cq->cq_idx - pfvf->hw.rx_queues;
1326 sq = &pfvf->qset.sq[qidx];
1327
1328 if (otx2_nix_cq_op_status(pfvf, cq) || !cq->pend_cqe)
1329 return;
1330
1331 while (cq->pend_cqe) {
1332 cqe = (struct nix_cqe_tx_s *)otx2_get_next_cqe(cq);
1333 processed_cqe++;
1334 cq->pend_cqe--;
1335
1336 if (!cqe)
1337 continue;
1338 sg = &sq->sg[cqe->comp.sqe_id];
1339 skb = (struct sk_buff *)sg->skb;
1340 if (skb) {
1341 tx_bytes += skb->len;
1342 tx_pkts++;
1343 otx2_dma_unmap_skb_frags(pfvf, sg);
1344 dev_kfree_skb_any(skb);
1345 sg->skb = (u64)NULL;
1346 }
1347 }
1348
1349 if (likely(tx_pkts)) {
1350 if (qidx >= pfvf->hw.tx_queues)
1351 qidx -= pfvf->hw.xdp_queues;
1352 txq = netdev_get_tx_queue(pfvf->netdev, qidx);
1353 netdev_tx_completed_queue(txq, tx_pkts, tx_bytes);
1354 }
1355 /* Free CQEs to HW */
1356 otx2_write64(pfvf, NIX_LF_CQ_OP_DOOR,
1357 ((u64)cq->cq_idx << 32) | processed_cqe);
1358 }
1359
otx2_rxtx_enable(struct otx2_nic * pfvf,bool enable)1360 int otx2_rxtx_enable(struct otx2_nic *pfvf, bool enable)
1361 {
1362 struct msg_req *msg;
1363 int err;
1364
1365 mutex_lock(&pfvf->mbox.lock);
1366 if (enable)
1367 msg = otx2_mbox_alloc_msg_nix_lf_start_rx(&pfvf->mbox);
1368 else
1369 msg = otx2_mbox_alloc_msg_nix_lf_stop_rx(&pfvf->mbox);
1370
1371 if (!msg) {
1372 mutex_unlock(&pfvf->mbox.lock);
1373 return -ENOMEM;
1374 }
1375
1376 err = otx2_sync_mbox_msg(&pfvf->mbox);
1377 mutex_unlock(&pfvf->mbox.lock);
1378 return err;
1379 }
1380
otx2_free_pending_sqe(struct otx2_nic * pfvf)1381 void otx2_free_pending_sqe(struct otx2_nic *pfvf)
1382 {
1383 int tx_pkts = 0, tx_bytes = 0;
1384 struct sk_buff *skb = NULL;
1385 struct otx2_snd_queue *sq;
1386 struct netdev_queue *txq;
1387 struct sg_list *sg;
1388 int sq_idx, sqe;
1389
1390 for (sq_idx = 0; sq_idx < pfvf->hw.tx_queues; sq_idx++) {
1391 sq = &pfvf->qset.sq[sq_idx];
1392 for (sqe = 0; sqe < sq->sqe_cnt; sqe++) {
1393 sg = &sq->sg[sqe];
1394 skb = (struct sk_buff *)sg->skb;
1395 if (skb) {
1396 tx_bytes += skb->len;
1397 tx_pkts++;
1398 otx2_dma_unmap_skb_frags(pfvf, sg);
1399 dev_kfree_skb_any(skb);
1400 sg->skb = (u64)NULL;
1401 }
1402 }
1403
1404 if (!tx_pkts)
1405 continue;
1406 txq = netdev_get_tx_queue(pfvf->netdev, sq_idx);
1407 netdev_tx_completed_queue(txq, tx_pkts, tx_bytes);
1408 tx_pkts = 0;
1409 tx_bytes = 0;
1410 }
1411 }
1412
otx2_xdp_sqe_add_sg(struct otx2_snd_queue * sq,struct xdp_frame * xdpf,u64 dma_addr,int len,int * offset,u16 flags)1413 static void otx2_xdp_sqe_add_sg(struct otx2_snd_queue *sq,
1414 struct xdp_frame *xdpf,
1415 u64 dma_addr, int len, int *offset, u16 flags)
1416 {
1417 struct nix_sqe_sg_s *sg = NULL;
1418 u64 *iova = NULL;
1419
1420 sg = (struct nix_sqe_sg_s *)(sq->sqe_base + *offset);
1421 sg->ld_type = NIX_SEND_LDTYPE_LDD;
1422 sg->subdc = NIX_SUBDC_SG;
1423 sg->segs = 1;
1424 sg->seg1_size = len;
1425 iova = (void *)sg + sizeof(*sg);
1426 *iova = dma_addr;
1427 *offset += sizeof(*sg) + sizeof(u64);
1428
1429 sq->sg[sq->head].dma_addr[0] = dma_addr;
1430 sq->sg[sq->head].size[0] = len;
1431 sq->sg[sq->head].num_segs = 1;
1432 sq->sg[sq->head].flags = flags;
1433 sq->sg[sq->head].skb = (u64)xdpf;
1434 }
1435
otx2_read_free_sqe(struct otx2_nic * pfvf,u16 qidx)1436 int otx2_read_free_sqe(struct otx2_nic *pfvf, u16 qidx)
1437 {
1438 struct otx2_snd_queue *sq;
1439 int free_sqe;
1440
1441 sq = &pfvf->qset.sq[qidx];
1442 free_sqe = otx2_get_free_sqe(sq);
1443 if (free_sqe < sq->sqe_thresh) {
1444 netdev_warn(pfvf->netdev, "No free sqe for Send queue%d\n", qidx);
1445 return 0;
1446 }
1447
1448 return free_sqe - sq->sqe_thresh;
1449 }
1450
otx2_xdp_sq_append_pkt(struct otx2_nic * pfvf,struct xdp_frame * xdpf,u64 iova,int len,u16 qidx,u16 flags)1451 bool otx2_xdp_sq_append_pkt(struct otx2_nic *pfvf, struct xdp_frame *xdpf,
1452 u64 iova, int len, u16 qidx, u16 flags)
1453 {
1454 struct nix_sqe_hdr_s *sqe_hdr;
1455 struct otx2_snd_queue *sq;
1456 int offset, free_sqe;
1457
1458 sq = &pfvf->qset.sq[qidx];
1459 free_sqe = otx2_get_free_sqe(sq);
1460 if (free_sqe < sq->sqe_thresh)
1461 return false;
1462
1463 memset(sq->sqe_base + 8, 0, sq->sqe_size - 8);
1464
1465 sqe_hdr = (struct nix_sqe_hdr_s *)(sq->sqe_base);
1466
1467 if (!sqe_hdr->total) {
1468 sqe_hdr->aura = sq->aura_id;
1469 sqe_hdr->df = 1;
1470 sqe_hdr->sq = qidx;
1471 sqe_hdr->pnc = 1;
1472 }
1473 sqe_hdr->total = len;
1474 sqe_hdr->sqe_id = sq->head;
1475
1476 offset = sizeof(*sqe_hdr);
1477
1478 otx2_xdp_sqe_add_sg(sq, xdpf, iova, len, &offset, flags);
1479 sqe_hdr->sizem1 = (offset / 16) - 1;
1480 pfvf->hw_ops->sqe_flush(pfvf, sq, offset, qidx);
1481
1482 return true;
1483 }
1484
otx2_xdp_rcv_pkt_handler(struct otx2_nic * pfvf,struct bpf_prog * prog,struct nix_cqe_rx_s * cqe,struct otx2_cq_queue * cq,u32 * metasize,bool * need_xdp_flush)1485 static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf,
1486 struct bpf_prog *prog,
1487 struct nix_cqe_rx_s *cqe,
1488 struct otx2_cq_queue *cq,
1489 u32 *metasize, bool *need_xdp_flush)
1490 {
1491 struct xdp_buff xdp, *xsk_buff = NULL;
1492 unsigned char *hard_start;
1493 struct otx2_pool *pool;
1494 struct xdp_frame *xdpf;
1495 int qidx = cq->cq_idx;
1496 struct page *page;
1497 u64 iova, pa;
1498 u32 act;
1499 int err;
1500
1501 pool = &pfvf->qset.pool[qidx];
1502
1503 if (pool->xsk_pool) {
1504 xsk_buff = pool->xdp[--cq->rbpool->xdp_top];
1505 if (!xsk_buff)
1506 return false;
1507
1508 xsk_buff->data_end = xsk_buff->data + cqe->sg.seg_size;
1509 act = bpf_prog_run_xdp(prog, xsk_buff);
1510 goto handle_xdp_verdict;
1511 }
1512
1513 iova = cqe->sg.seg_addr - OTX2_HEAD_ROOM;
1514 pa = otx2_iova_to_phys(pfvf->iommu_domain, iova);
1515 page = virt_to_page(phys_to_virt(pa));
1516
1517 xdp_init_buff(&xdp, pfvf->rbsize, &cq->xdp_rxq);
1518
1519 hard_start = (unsigned char *)phys_to_virt(pa);
1520 xdp_prepare_buff(&xdp, hard_start, OTX2_HEAD_ROOM,
1521 cqe->sg.seg_size, true);
1522
1523 act = bpf_prog_run_xdp(prog, &xdp);
1524
1525 handle_xdp_verdict:
1526 switch (act) {
1527 case XDP_PASS:
1528 *metasize = xdp.data - xdp.data_meta;
1529 break;
1530 case XDP_TX:
1531 qidx += pfvf->hw.tx_queues;
1532 cq->pool_ptrs++;
1533 xdpf = xdp_convert_buff_to_frame(&xdp);
1534 return otx2_xdp_sq_append_pkt(pfvf, xdpf,
1535 cqe->sg.seg_addr,
1536 cqe->sg.seg_size,
1537 qidx, OTX2_XDP_TX);
1538 case XDP_REDIRECT:
1539 cq->pool_ptrs++;
1540 if (xsk_buff) {
1541 err = xdp_do_redirect(pfvf->netdev, xsk_buff, prog);
1542 if (!err) {
1543 *need_xdp_flush = true;
1544 return true;
1545 }
1546 return false;
1547 }
1548
1549 err = xdp_do_redirect(pfvf->netdev, &xdp, prog);
1550 if (!err) {
1551 *need_xdp_flush = true;
1552 return true;
1553 }
1554
1555 otx2_dma_unmap_page(pfvf, iova, pfvf->rbsize,
1556 DMA_FROM_DEVICE);
1557 xdpf = xdp_convert_buff_to_frame(&xdp);
1558 xdp_return_frame(xdpf);
1559 break;
1560 default:
1561 bpf_warn_invalid_xdp_action(pfvf->netdev, prog, act);
1562 fallthrough;
1563 case XDP_ABORTED:
1564 if (act == XDP_ABORTED)
1565 trace_xdp_exception(pfvf->netdev, prog, act);
1566 fallthrough;
1567 case XDP_DROP:
1568 cq->pool_ptrs++;
1569 if (xsk_buff) {
1570 xsk_buff_free(xsk_buff);
1571 } else if (page->pp) {
1572 page_pool_recycle_direct(pool->page_pool, page);
1573 } else {
1574 otx2_dma_unmap_page(pfvf, iova, pfvf->rbsize,
1575 DMA_FROM_DEVICE);
1576 put_page(page);
1577 }
1578 return true;
1579 }
1580 return false;
1581 }
1582