1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell RVU Ethernet driver
3 *
4 * Copyright (C) 2020 Marvell.
5 *
6 */
7
8 #include <linux/etherdevice.h>
9 #include <net/ip.h>
10 #include <net/tso.h>
11 #include <linux/bpf.h>
12 #include <linux/bpf_trace.h>
13 #include <net/ip6_checksum.h>
14 #include <net/xfrm.h>
15 #include <net/xdp.h>
16
17 #include "otx2_reg.h"
18 #include "otx2_common.h"
19 #include "otx2_struct.h"
20 #include "otx2_txrx.h"
21 #include "otx2_ptp.h"
22 #include "cn10k.h"
23 #include "otx2_xsk.h"
24
25 #define CQE_ADDR(CQ, idx) ((CQ)->cqe_base + ((CQ)->cqe_size * (idx)))
26 #define PTP_PORT 0x13F
27 /* PTPv2 header Original Timestamp starts at byte offset 34 and
28 * contains 6 byte seconds field and 4 byte nano seconds field.
29 */
30 #define PTP_SYNC_SEC_OFFSET 34
31
32 DEFINE_STATIC_KEY_FALSE(cn10k_ipsec_sa_enabled);
33
otx2_get_free_sqe(struct otx2_snd_queue * sq)34 static int otx2_get_free_sqe(struct otx2_snd_queue *sq)
35 {
36 return (sq->cons_head - sq->head - 1 + sq->sqe_cnt)
37 & (sq->sqe_cnt - 1);
38 }
39
40 static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf,
41 struct bpf_prog *prog,
42 struct nix_cqe_rx_s *cqe,
43 struct otx2_cq_queue *cq,
44 u32 *metasize, bool *need_xdp_flush);
45
otx2_sq_set_sqe_base(struct otx2_snd_queue * sq,struct sk_buff * skb)46 static void otx2_sq_set_sqe_base(struct otx2_snd_queue *sq,
47 struct sk_buff *skb)
48 {
49 if (static_branch_unlikely(&cn10k_ipsec_sa_enabled) &&
50 (xfrm_offload(skb)))
51 sq->sqe_base = sq->sqe_ring->base + sq->sqe_size +
52 (sq->head * (sq->sqe_size * 2));
53 else
54 sq->sqe_base = sq->sqe->base;
55 }
56
otx2_nix_cq_op_status(struct otx2_nic * pfvf,struct otx2_cq_queue * cq)57 static int otx2_nix_cq_op_status(struct otx2_nic *pfvf,
58 struct otx2_cq_queue *cq)
59 {
60 u64 incr = (u64)(cq->cq_idx) << 32;
61 u64 status;
62
63 status = otx2_atomic64_fetch_add(incr, pfvf->cq_op_addr);
64
65 if (unlikely(status & BIT_ULL(CQ_OP_STAT_OP_ERR) ||
66 status & BIT_ULL(CQ_OP_STAT_CQ_ERR))) {
67 dev_err(pfvf->dev, "CQ stopped due to error");
68 return -EINVAL;
69 }
70
71 cq->cq_tail = status & 0xFFFFF;
72 cq->cq_head = (status >> 20) & 0xFFFFF;
73 if (cq->cq_tail < cq->cq_head)
74 cq->pend_cqe = (cq->cqe_cnt - cq->cq_head) +
75 cq->cq_tail;
76 else
77 cq->pend_cqe = cq->cq_tail - cq->cq_head;
78
79 return 0;
80 }
81
otx2_get_next_cqe(struct otx2_cq_queue * cq)82 static struct nix_cqe_hdr_s *otx2_get_next_cqe(struct otx2_cq_queue *cq)
83 {
84 struct nix_cqe_hdr_s *cqe_hdr;
85
86 cqe_hdr = (struct nix_cqe_hdr_s *)CQE_ADDR(cq, cq->cq_head);
87 if (cqe_hdr->cqe_type == NIX_XQE_TYPE_INVALID)
88 return NULL;
89
90 cq->cq_head++;
91 cq->cq_head &= (cq->cqe_cnt - 1);
92
93 return cqe_hdr;
94 }
95
frag_num(unsigned int i)96 static unsigned int frag_num(unsigned int i)
97 {
98 #ifdef __BIG_ENDIAN
99 return (i & ~3) + 3 - (i & 3);
100 #else
101 return i;
102 #endif
103 }
104
otx2_xdp_snd_pkt_handler(struct otx2_nic * pfvf,struct otx2_snd_queue * sq,struct nix_cqe_tx_s * cqe,int * xsk_frames)105 static void otx2_xdp_snd_pkt_handler(struct otx2_nic *pfvf,
106 struct otx2_snd_queue *sq,
107 struct nix_cqe_tx_s *cqe,
108 int *xsk_frames)
109 {
110 struct nix_send_comp_s *snd_comp = &cqe->comp;
111 struct sg_list *sg;
112
113 sg = &sq->sg[snd_comp->sqe_id];
114 if (sg->flags & OTX2_AF_XDP_FRAME) {
115 (*xsk_frames)++;
116 return;
117 }
118
119 if (sg->flags & OTX2_XDP_REDIRECT)
120 otx2_dma_unmap_page(pfvf, sg->dma_addr[0], sg->size[0], DMA_TO_DEVICE);
121 xdp_return_frame((struct xdp_frame *)sg->skb);
122 sg->skb = (u64)NULL;
123 }
124
otx2_snd_pkt_handler(struct otx2_nic * pfvf,struct otx2_cq_queue * cq,struct otx2_snd_queue * sq,struct nix_cqe_tx_s * cqe,int budget,int * tx_pkts,int * tx_bytes)125 static void otx2_snd_pkt_handler(struct otx2_nic *pfvf,
126 struct otx2_cq_queue *cq,
127 struct otx2_snd_queue *sq,
128 struct nix_cqe_tx_s *cqe,
129 int budget, int *tx_pkts, int *tx_bytes)
130 {
131 struct nix_send_comp_s *snd_comp = &cqe->comp;
132 struct skb_shared_hwtstamps ts;
133 struct sk_buff *skb = NULL;
134 u64 timestamp, tsns;
135 struct sg_list *sg;
136 int err;
137
138 if (unlikely(snd_comp->status) && netif_msg_tx_err(pfvf))
139 net_err_ratelimited("%s: TX%d: Error in send CQ status:%x\n",
140 pfvf->netdev->name, cq->cint_idx,
141 snd_comp->status);
142
143 sg = &sq->sg[snd_comp->sqe_id];
144 skb = (struct sk_buff *)sg->skb;
145 if (unlikely(!skb))
146 return;
147
148 if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) {
149 timestamp = ((u64 *)sq->timestamps->base)[snd_comp->sqe_id];
150 if (timestamp != 1) {
151 timestamp = pfvf->ptp->convert_tx_ptp_tstmp(timestamp);
152 err = otx2_ptp_tstamp2time(pfvf, timestamp, &tsns);
153 if (!err) {
154 memset(&ts, 0, sizeof(ts));
155 ts.hwtstamp = ns_to_ktime(tsns);
156 skb_tstamp_tx(skb, &ts);
157 }
158 }
159 }
160
161 *tx_bytes += skb->len;
162 (*tx_pkts)++;
163 otx2_dma_unmap_skb_frags(pfvf, sg);
164 napi_consume_skb(skb, budget);
165 sg->skb = (u64)NULL;
166 }
167
otx2_set_rxtstamp(struct otx2_nic * pfvf,struct sk_buff * skb,void * data)168 static void otx2_set_rxtstamp(struct otx2_nic *pfvf,
169 struct sk_buff *skb, void *data)
170 {
171 u64 timestamp, tsns;
172 int err;
173
174 if (!(pfvf->flags & OTX2_FLAG_RX_TSTAMP_ENABLED))
175 return;
176
177 timestamp = pfvf->ptp->convert_rx_ptp_tstmp(*(u64 *)data);
178 /* The first 8 bytes is the timestamp */
179 err = otx2_ptp_tstamp2time(pfvf, timestamp, &tsns);
180 if (err)
181 return;
182
183 skb_hwtstamps(skb)->hwtstamp = ns_to_ktime(tsns);
184 }
185
otx2_skb_add_frag(struct otx2_nic * pfvf,struct sk_buff * skb,u64 iova,int len,struct nix_rx_parse_s * parse,int qidx)186 static bool otx2_skb_add_frag(struct otx2_nic *pfvf, struct sk_buff *skb,
187 u64 iova, int len, struct nix_rx_parse_s *parse,
188 int qidx)
189 {
190 struct page *page;
191 int off = 0;
192 void *va;
193
194 va = phys_to_virt(otx2_iova_to_phys(pfvf->iommu_domain, iova));
195
196 if (likely(!skb_shinfo(skb)->nr_frags)) {
197 /* Check if data starts at some nonzero offset
198 * from the start of the buffer. For now the
199 * only possible offset is 8 bytes in the case
200 * where packet is prepended by a timestamp.
201 */
202 if (parse->laptr) {
203 otx2_set_rxtstamp(pfvf, skb, va);
204 off = OTX2_HW_TIMESTAMP_LEN;
205 }
206 }
207
208 page = virt_to_page(va);
209 if (likely(skb_shinfo(skb)->nr_frags < MAX_SKB_FRAGS)) {
210 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,
211 va - page_address(page) + off,
212 len - off, pfvf->rbsize);
213 return true;
214 }
215
216 /* If more than MAX_SKB_FRAGS fragments are received then
217 * give back those buffer pointers to hardware for reuse.
218 */
219 pfvf->hw_ops->aura_freeptr(pfvf, qidx, iova & ~0x07ULL);
220
221 return false;
222 }
223
otx2_set_rxhash(struct otx2_nic * pfvf,struct nix_cqe_rx_s * cqe,struct sk_buff * skb)224 static void otx2_set_rxhash(struct otx2_nic *pfvf,
225 struct nix_cqe_rx_s *cqe, struct sk_buff *skb)
226 {
227 enum pkt_hash_types hash_type = PKT_HASH_TYPE_NONE;
228 struct otx2_rss_info *rss;
229 u32 hash = 0;
230
231 if (!(pfvf->netdev->features & NETIF_F_RXHASH))
232 return;
233
234 rss = &pfvf->hw.rss_info;
235 if (rss->flowkey_cfg) {
236 if (rss->flowkey_cfg &
237 ~(NIX_FLOW_KEY_TYPE_IPV4 | NIX_FLOW_KEY_TYPE_IPV6))
238 hash_type = PKT_HASH_TYPE_L4;
239 else
240 hash_type = PKT_HASH_TYPE_L3;
241 hash = cqe->hdr.flow_tag;
242 }
243 skb_set_hash(skb, hash, hash_type);
244 }
245
otx2_free_rcv_seg(struct otx2_nic * pfvf,struct nix_cqe_rx_s * cqe,int qidx)246 static void otx2_free_rcv_seg(struct otx2_nic *pfvf, struct nix_cqe_rx_s *cqe,
247 int qidx)
248 {
249 struct nix_rx_sg_s *sg = &cqe->sg;
250 void *end, *start;
251 u64 *seg_addr;
252 int seg;
253
254 start = (void *)sg;
255 end = start + ((cqe->parse.desc_sizem1 + 1) * 16);
256 while (start < end) {
257 sg = (struct nix_rx_sg_s *)start;
258 seg_addr = &sg->seg_addr;
259 for (seg = 0; seg < sg->segs; seg++, seg_addr++)
260 pfvf->hw_ops->aura_freeptr(pfvf, qidx,
261 *seg_addr & ~0x07ULL);
262 start += sizeof(*sg);
263 }
264 }
265
otx2_check_rcv_errors(struct otx2_nic * pfvf,struct nix_cqe_rx_s * cqe,int qidx)266 static bool otx2_check_rcv_errors(struct otx2_nic *pfvf,
267 struct nix_cqe_rx_s *cqe, int qidx)
268 {
269 struct otx2_drv_stats *stats = &pfvf->hw.drv_stats;
270 struct nix_rx_parse_s *parse = &cqe->parse;
271
272 if (netif_msg_rx_err(pfvf))
273 netdev_err(pfvf->netdev,
274 "RQ%d: Error pkt with errlev:0x%x errcode:0x%x\n",
275 qidx, parse->errlev, parse->errcode);
276
277 if (parse->errlev == NPC_ERRLVL_RE) {
278 switch (parse->errcode) {
279 case ERRCODE_FCS:
280 case ERRCODE_FCS_RCV:
281 atomic_inc(&stats->rx_fcs_errs);
282 break;
283 case ERRCODE_UNDERSIZE:
284 atomic_inc(&stats->rx_undersize_errs);
285 break;
286 case ERRCODE_OVERSIZE:
287 atomic_inc(&stats->rx_oversize_errs);
288 break;
289 case ERRCODE_OL2_LEN_MISMATCH:
290 atomic_inc(&stats->rx_len_errs);
291 break;
292 default:
293 atomic_inc(&stats->rx_other_errs);
294 break;
295 }
296 } else if (parse->errlev == NPC_ERRLVL_NIX) {
297 switch (parse->errcode) {
298 case ERRCODE_OL3_LEN:
299 case ERRCODE_OL4_LEN:
300 case ERRCODE_IL3_LEN:
301 case ERRCODE_IL4_LEN:
302 atomic_inc(&stats->rx_len_errs);
303 break;
304 case ERRCODE_OL4_CSUM:
305 case ERRCODE_IL4_CSUM:
306 atomic_inc(&stats->rx_csum_errs);
307 break;
308 default:
309 atomic_inc(&stats->rx_other_errs);
310 break;
311 }
312 } else {
313 atomic_inc(&stats->rx_other_errs);
314 /* For now ignore all the NPC parser errors and
315 * pass the packets to stack.
316 */
317 return false;
318 }
319
320 /* If RXALL is enabled pass on packets to stack. */
321 if (pfvf->netdev->features & NETIF_F_RXALL)
322 return false;
323
324 /* Free buffer back to pool */
325 if (cqe->sg.segs)
326 otx2_free_rcv_seg(pfvf, cqe, qidx);
327 return true;
328 }
329
otx2_rcv_pkt_handler(struct otx2_nic * pfvf,struct napi_struct * napi,struct otx2_cq_queue * cq,struct nix_cqe_rx_s * cqe,bool * need_xdp_flush)330 static void otx2_rcv_pkt_handler(struct otx2_nic *pfvf,
331 struct napi_struct *napi,
332 struct otx2_cq_queue *cq,
333 struct nix_cqe_rx_s *cqe, bool *need_xdp_flush)
334 {
335 struct nix_rx_parse_s *parse = &cqe->parse;
336 struct nix_rx_sg_s *sg = &cqe->sg;
337 struct sk_buff *skb = NULL;
338 u64 *word = (u64 *)parse;
339 void *end, *start;
340 u32 metasize = 0;
341 u64 *seg_addr;
342 u16 *seg_size;
343 int seg;
344
345 if (unlikely(parse->errlev || parse->errcode)) {
346 if (otx2_check_rcv_errors(pfvf, cqe, cq->cq_idx)) {
347 trace_otx2_parse_dump(pfvf->pdev, "Err:", word);
348 return;
349 }
350 }
351 trace_otx2_parse_dump(pfvf->pdev, "", word);
352
353 if (pfvf->xdp_prog)
354 if (otx2_xdp_rcv_pkt_handler(pfvf, pfvf->xdp_prog, cqe, cq,
355 &metasize, need_xdp_flush))
356 return;
357
358 skb = napi_get_frags(napi);
359 if (unlikely(!skb))
360 return;
361
362 start = (void *)sg;
363 end = start + ((cqe->parse.desc_sizem1 + 1) * 16);
364 while (start < end) {
365 sg = (struct nix_rx_sg_s *)start;
366 seg_addr = &sg->seg_addr;
367 seg_size = (void *)sg;
368 for (seg = 0; seg < sg->segs; seg++, seg_addr++) {
369 if (otx2_skb_add_frag(pfvf, skb, *seg_addr,
370 seg_size[seg], parse, cq->cq_idx))
371 cq->pool_ptrs++;
372 }
373 start += sizeof(*sg);
374 }
375 otx2_set_rxhash(pfvf, cqe, skb);
376
377 if (!(pfvf->flags & OTX2_FLAG_REP_MODE_ENABLED)) {
378 skb_record_rx_queue(skb, cq->cq_idx);
379 if (pfvf->netdev->features & NETIF_F_RXCSUM)
380 skb->ip_summed = CHECKSUM_UNNECESSARY;
381 }
382
383 if (pfvf->flags & OTX2_FLAG_TC_MARK_ENABLED)
384 skb->mark = parse->match_id;
385
386 skb_mark_for_recycle(skb);
387 if (metasize)
388 skb_metadata_set(skb, metasize);
389
390 napi_gro_frags(napi);
391 }
392
otx2_rx_napi_handler(struct otx2_nic * pfvf,struct napi_struct * napi,struct otx2_cq_queue * cq,int budget)393 static int otx2_rx_napi_handler(struct otx2_nic *pfvf,
394 struct napi_struct *napi,
395 struct otx2_cq_queue *cq, int budget)
396 {
397 bool need_xdp_flush = false;
398 struct nix_cqe_rx_s *cqe;
399 int processed_cqe = 0;
400
401 if (cq->pend_cqe >= budget)
402 goto process_cqe;
403
404 if (otx2_nix_cq_op_status(pfvf, cq) || !cq->pend_cqe)
405 return 0;
406
407 process_cqe:
408 while (likely(processed_cqe < budget) && cq->pend_cqe) {
409 cqe = (struct nix_cqe_rx_s *)CQE_ADDR(cq, cq->cq_head);
410 if (cqe->hdr.cqe_type == NIX_XQE_TYPE_INVALID ||
411 !cqe->sg.seg_addr) {
412 if (!processed_cqe)
413 return 0;
414 break;
415 }
416 cq->cq_head++;
417 cq->cq_head &= (cq->cqe_cnt - 1);
418
419 otx2_rcv_pkt_handler(pfvf, napi, cq, cqe, &need_xdp_flush);
420
421 cqe->hdr.cqe_type = NIX_XQE_TYPE_INVALID;
422 cqe->sg.seg_addr = 0x00;
423 processed_cqe++;
424 cq->pend_cqe--;
425 }
426 if (need_xdp_flush)
427 xdp_do_flush();
428
429 /* Free CQEs to HW */
430 otx2_write64(pfvf, NIX_LF_CQ_OP_DOOR,
431 ((u64)cq->cq_idx << 32) | processed_cqe);
432
433 return processed_cqe;
434 }
435
otx2_refill_pool_ptrs(void * dev,struct otx2_cq_queue * cq)436 int otx2_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq)
437 {
438 struct otx2_nic *pfvf = dev;
439 int cnt = cq->pool_ptrs;
440 dma_addr_t bufptr;
441
442 while (cq->pool_ptrs) {
443 if (otx2_alloc_buffer(pfvf, cq, &bufptr))
444 break;
445 otx2_aura_freeptr(pfvf, cq->cq_idx, bufptr + OTX2_HEAD_ROOM);
446 cq->pool_ptrs--;
447 }
448
449 return cnt - cq->pool_ptrs;
450 }
451
otx2_zc_submit_pkts(struct otx2_nic * pfvf,struct xsk_buff_pool * xsk_pool,int * xsk_frames,int qidx,int budget)452 static void otx2_zc_submit_pkts(struct otx2_nic *pfvf, struct xsk_buff_pool *xsk_pool,
453 int *xsk_frames, int qidx, int budget)
454 {
455 if (*xsk_frames)
456 xsk_tx_completed(xsk_pool, *xsk_frames);
457
458 if (xsk_uses_need_wakeup(xsk_pool))
459 xsk_set_tx_need_wakeup(xsk_pool);
460
461 otx2_zc_napi_handler(pfvf, xsk_pool, qidx, budget);
462 }
463
otx2_tx_napi_handler(struct otx2_nic * pfvf,struct otx2_cq_queue * cq,int budget)464 static int otx2_tx_napi_handler(struct otx2_nic *pfvf,
465 struct otx2_cq_queue *cq, int budget)
466 {
467 int tx_pkts = 0, tx_bytes = 0, qidx;
468 struct otx2_snd_queue *sq;
469 struct nix_cqe_tx_s *cqe;
470 struct net_device *ndev;
471 int processed_cqe = 0;
472 int xsk_frames = 0;
473
474 qidx = cq->cq_idx - pfvf->hw.rx_queues;
475 sq = &pfvf->qset.sq[qidx];
476
477 if (cq->pend_cqe >= budget)
478 goto process_cqe;
479
480 if (otx2_nix_cq_op_status(pfvf, cq) || !cq->pend_cqe) {
481 if (sq->xsk_pool)
482 otx2_zc_submit_pkts(pfvf, sq->xsk_pool, &xsk_frames,
483 qidx, budget);
484 return 0;
485 }
486
487 process_cqe:
488
489 while (likely(processed_cqe < budget) && cq->pend_cqe) {
490 cqe = (struct nix_cqe_tx_s *)otx2_get_next_cqe(cq);
491 if (unlikely(!cqe)) {
492 if (!processed_cqe)
493 return 0;
494 break;
495 }
496
497 if (cq->cq_type == CQ_XDP)
498 otx2_xdp_snd_pkt_handler(pfvf, sq, cqe, &xsk_frames);
499 else
500 otx2_snd_pkt_handler(pfvf, cq, &pfvf->qset.sq[qidx],
501 cqe, budget, &tx_pkts, &tx_bytes);
502
503 cqe->hdr.cqe_type = NIX_XQE_TYPE_INVALID;
504 processed_cqe++;
505 cq->pend_cqe--;
506
507 sq->cons_head++;
508 sq->cons_head &= (sq->sqe_cnt - 1);
509 }
510
511 /* Free CQEs to HW */
512 otx2_write64(pfvf, NIX_LF_CQ_OP_DOOR,
513 ((u64)cq->cq_idx << 32) | processed_cqe);
514
515 #if IS_ENABLED(CONFIG_RVU_ESWITCH)
516 if (pfvf->flags & OTX2_FLAG_REP_MODE_ENABLED)
517 ndev = pfvf->reps[qidx]->netdev;
518 else
519 #endif
520 ndev = pfvf->netdev;
521
522 if (likely(tx_pkts)) {
523 struct netdev_queue *txq;
524
525 qidx = cq->cq_idx - pfvf->hw.rx_queues;
526
527 if (qidx >= pfvf->hw.tx_queues)
528 qidx -= pfvf->hw.xdp_queues;
529 if (pfvf->flags & OTX2_FLAG_REP_MODE_ENABLED)
530 qidx = 0;
531 txq = netdev_get_tx_queue(ndev, qidx);
532 netdev_tx_completed_queue(txq, tx_pkts, tx_bytes);
533 /* Check if queue was stopped earlier due to ring full */
534 smp_mb();
535 if (netif_tx_queue_stopped(txq) &&
536 netif_carrier_ok(ndev))
537 netif_tx_wake_queue(txq);
538 }
539
540 if (sq->xsk_pool)
541 otx2_zc_submit_pkts(pfvf, sq->xsk_pool, &xsk_frames, qidx, budget);
542
543 return 0;
544 }
545
otx2_adjust_adaptive_coalese(struct otx2_nic * pfvf,struct otx2_cq_poll * cq_poll)546 static void otx2_adjust_adaptive_coalese(struct otx2_nic *pfvf, struct otx2_cq_poll *cq_poll)
547 {
548 struct dim_sample dim_sample = { 0 };
549 u64 rx_frames, rx_bytes;
550 u64 tx_frames, tx_bytes;
551
552 rx_frames = OTX2_GET_RX_STATS(RX_BCAST) + OTX2_GET_RX_STATS(RX_MCAST) +
553 OTX2_GET_RX_STATS(RX_UCAST);
554 rx_bytes = OTX2_GET_RX_STATS(RX_OCTS);
555 tx_bytes = OTX2_GET_TX_STATS(TX_OCTS);
556 tx_frames = OTX2_GET_TX_STATS(TX_UCAST);
557
558 dim_update_sample(pfvf->napi_events,
559 rx_frames + tx_frames,
560 rx_bytes + tx_bytes,
561 &dim_sample);
562 net_dim(&cq_poll->dim, &dim_sample);
563 }
564
otx2_napi_handler(struct napi_struct * napi,int budget)565 int otx2_napi_handler(struct napi_struct *napi, int budget)
566 {
567 struct otx2_cq_queue *rx_cq = NULL;
568 struct otx2_cq_queue *cq = NULL;
569 struct otx2_pool *pool = NULL;
570 struct otx2_cq_poll *cq_poll;
571 int workdone = 0, cq_idx, i;
572 struct otx2_qset *qset;
573 struct otx2_nic *pfvf;
574 int filled_cnt = -1;
575
576 cq_poll = container_of(napi, struct otx2_cq_poll, napi);
577 pfvf = (struct otx2_nic *)cq_poll->dev;
578 qset = &pfvf->qset;
579
580 for (i = 0; i < CQS_PER_CINT; i++) {
581 cq_idx = cq_poll->cq_ids[i];
582 if (unlikely(cq_idx == CINT_INVALID_CQ))
583 continue;
584 cq = &qset->cq[cq_idx];
585 if (cq->cq_type == CQ_RX) {
586 rx_cq = cq;
587 workdone += otx2_rx_napi_handler(pfvf, napi,
588 cq, budget);
589 } else {
590 workdone += otx2_tx_napi_handler(pfvf, cq, budget);
591 }
592 }
593
594 if (rx_cq && rx_cq->pool_ptrs)
595 filled_cnt = pfvf->hw_ops->refill_pool_ptrs(pfvf, rx_cq);
596
597 /* Clear the IRQ */
598 otx2_write64(pfvf, NIX_LF_CINTX_INT(cq_poll->cint_idx), BIT_ULL(0));
599
600 if (workdone < budget && napi_complete_done(napi, workdone)) {
601 /* If interface is going down, don't re-enable IRQ */
602 if (pfvf->flags & OTX2_FLAG_INTF_DOWN)
603 return workdone;
604
605 /* Adjust irq coalese using net_dim */
606 if (pfvf->flags & OTX2_FLAG_ADPTV_INT_COAL_ENABLED)
607 otx2_adjust_adaptive_coalese(pfvf, cq_poll);
608
609 if (likely(cq))
610 pool = &pfvf->qset.pool[cq->cq_idx];
611
612 if (unlikely(!filled_cnt)) {
613 struct refill_work *work;
614 struct delayed_work *dwork;
615
616 if (likely(cq)) {
617 work = &pfvf->refill_wrk[cq->cq_idx];
618 dwork = &work->pool_refill_work;
619 /* Schedule a task if no other task is running */
620 if (!cq->refill_task_sched) {
621 work->napi = napi;
622 cq->refill_task_sched = true;
623 schedule_delayed_work(dwork,
624 msecs_to_jiffies(100));
625 }
626 /* Call wake-up for not able to fill buffers */
627 if (pool->xsk_pool)
628 xsk_set_rx_need_wakeup(pool->xsk_pool);
629 }
630 } else {
631 /* Clear wake-up, since buffers are filled successfully */
632 if (pool && pool->xsk_pool)
633 xsk_clear_rx_need_wakeup(pool->xsk_pool);
634 /* Re-enable interrupts */
635 otx2_write64(pfvf,
636 NIX_LF_CINTX_ENA_W1S(cq_poll->cint_idx),
637 BIT_ULL(0));
638 }
639 }
640 return workdone;
641 }
642 EXPORT_SYMBOL(otx2_napi_handler);
643
otx2_sqe_flush(void * dev,struct otx2_snd_queue * sq,int size,int qidx)644 void otx2_sqe_flush(void *dev, struct otx2_snd_queue *sq,
645 int size, int qidx)
646 {
647 u64 status;
648
649 /* Packet data stores should finish before SQE is flushed to HW */
650 dma_wmb();
651
652 do {
653 memcpy(sq->lmt_addr, sq->sqe_base, size);
654 status = otx2_lmt_flush(sq->io_addr);
655 } while (status == 0);
656
657 sq->head++;
658 sq->head &= (sq->sqe_cnt - 1);
659 }
660
661 /* Add SQE scatter/gather subdescriptor structure */
otx2_sqe_add_sg(struct otx2_nic * pfvf,struct otx2_snd_queue * sq,struct sk_buff * skb,int num_segs,int * offset)662 static bool otx2_sqe_add_sg(struct otx2_nic *pfvf, struct otx2_snd_queue *sq,
663 struct sk_buff *skb, int num_segs, int *offset)
664 {
665 struct nix_sqe_sg_s *sg = NULL;
666 u64 dma_addr, *iova = NULL;
667 u16 *sg_lens = NULL;
668 int seg, len;
669
670 sq->sg[sq->head].num_segs = 0;
671
672 for (seg = 0; seg < num_segs; seg++) {
673 if ((seg % MAX_SEGS_PER_SG) == 0) {
674 sg = (struct nix_sqe_sg_s *)(sq->sqe_base + *offset);
675 sg->ld_type = NIX_SEND_LDTYPE_LDD;
676 sg->subdc = NIX_SUBDC_SG;
677 sg->segs = 0;
678 sg_lens = (void *)sg;
679 iova = (void *)sg + sizeof(*sg);
680 /* Next subdc always starts at a 16byte boundary.
681 * So if sg->segs is whether 2 or 3, offset += 16bytes.
682 */
683 if ((num_segs - seg) >= (MAX_SEGS_PER_SG - 1))
684 *offset += sizeof(*sg) + (3 * sizeof(u64));
685 else
686 *offset += sizeof(*sg) + sizeof(u64);
687 }
688 dma_addr = otx2_dma_map_skb_frag(pfvf, skb, seg, &len);
689 if (dma_mapping_error(pfvf->dev, dma_addr))
690 return false;
691
692 sg_lens[frag_num(seg % MAX_SEGS_PER_SG)] = len;
693 sg->segs++;
694 *iova++ = dma_addr;
695
696 /* Save DMA mapping info for later unmapping */
697 sq->sg[sq->head].dma_addr[seg] = dma_addr;
698 sq->sg[sq->head].size[seg] = len;
699 sq->sg[sq->head].num_segs++;
700 }
701
702 sq->sg[sq->head].skb = (u64)skb;
703 return true;
704 }
705
706 /* Add SQE extended header subdescriptor */
otx2_sqe_add_ext(struct otx2_nic * pfvf,struct otx2_snd_queue * sq,struct sk_buff * skb,int * offset)707 static void otx2_sqe_add_ext(struct otx2_nic *pfvf, struct otx2_snd_queue *sq,
708 struct sk_buff *skb, int *offset)
709 {
710 struct nix_sqe_ext_s *ext;
711
712 ext = (struct nix_sqe_ext_s *)(sq->sqe_base + *offset);
713 ext->subdc = NIX_SUBDC_EXT;
714 if (skb_shinfo(skb)->gso_size) {
715 ext->lso = 1;
716 ext->lso_sb = skb_tcp_all_headers(skb);
717 ext->lso_mps = skb_shinfo(skb)->gso_size;
718
719 /* Only TSOv4 and TSOv6 GSO offloads are supported */
720 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
721 ext->lso_format = pfvf->hw.lso_tsov4_idx;
722
723 /* HW adds payload size to 'ip_hdr->tot_len' while
724 * sending TSO segment, hence set payload length
725 * in IP header of the packet to just header length.
726 */
727 ip_hdr(skb)->tot_len =
728 htons(ext->lso_sb - skb_network_offset(skb));
729 } else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) {
730 ext->lso_format = pfvf->hw.lso_tsov6_idx;
731 ipv6_hdr(skb)->payload_len = htons(tcp_hdrlen(skb));
732 } else if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) {
733 __be16 l3_proto = vlan_get_protocol(skb);
734 struct udphdr *udph = udp_hdr(skb);
735 __be16 iplen;
736
737 ext->lso_sb = skb_transport_offset(skb) +
738 sizeof(struct udphdr);
739
740 /* HW adds payload size to length fields in IP and
741 * UDP headers while segmentation, hence adjust the
742 * lengths to just header sizes.
743 */
744 iplen = htons(ext->lso_sb - skb_network_offset(skb));
745 if (l3_proto == htons(ETH_P_IP)) {
746 ip_hdr(skb)->tot_len = iplen;
747 ext->lso_format = pfvf->hw.lso_udpv4_idx;
748 } else {
749 ipv6_hdr(skb)->payload_len = iplen;
750 ext->lso_format = pfvf->hw.lso_udpv6_idx;
751 }
752
753 udph->len = htons(sizeof(struct udphdr));
754 }
755 } else if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) {
756 ext->tstmp = 1;
757 }
758
759 #define OTX2_VLAN_PTR_OFFSET (ETH_HLEN - ETH_TLEN)
760 if (skb_vlan_tag_present(skb)) {
761 if (skb->vlan_proto == htons(ETH_P_8021Q)) {
762 ext->vlan1_ins_ena = 1;
763 ext->vlan1_ins_ptr = OTX2_VLAN_PTR_OFFSET;
764 ext->vlan1_ins_tci = skb_vlan_tag_get(skb);
765 } else if (skb->vlan_proto == htons(ETH_P_8021AD)) {
766 ext->vlan0_ins_ena = 1;
767 ext->vlan0_ins_ptr = OTX2_VLAN_PTR_OFFSET;
768 ext->vlan0_ins_tci = skb_vlan_tag_get(skb);
769 }
770 }
771
772 *offset += sizeof(*ext);
773 }
774
otx2_sqe_add_mem(struct otx2_snd_queue * sq,int * offset,int alg,u64 iova,int ptp_offset,u64 base_ns,bool udp_csum_crt)775 static void otx2_sqe_add_mem(struct otx2_snd_queue *sq, int *offset,
776 int alg, u64 iova, int ptp_offset,
777 u64 base_ns, bool udp_csum_crt)
778 {
779 struct nix_sqe_mem_s *mem;
780
781 mem = (struct nix_sqe_mem_s *)(sq->sqe_base + *offset);
782 mem->subdc = NIX_SUBDC_MEM;
783 mem->alg = alg;
784 mem->wmem = 1; /* wait for the memory operation */
785 mem->addr = iova;
786
787 if (ptp_offset) {
788 mem->start_offset = ptp_offset;
789 mem->udp_csum_crt = !!udp_csum_crt;
790 mem->base_ns = base_ns;
791 mem->step_type = 1;
792 }
793
794 *offset += sizeof(*mem);
795 }
796
797 /* Add SQE header subdescriptor structure */
otx2_sqe_add_hdr(struct otx2_nic * pfvf,struct otx2_snd_queue * sq,struct nix_sqe_hdr_s * sqe_hdr,struct sk_buff * skb,u16 qidx)798 static void otx2_sqe_add_hdr(struct otx2_nic *pfvf, struct otx2_snd_queue *sq,
799 struct nix_sqe_hdr_s *sqe_hdr,
800 struct sk_buff *skb, u16 qidx)
801 {
802 int proto = 0;
803
804 /* Check if SQE was framed before, if yes then no need to
805 * set these constants again and again.
806 */
807 if (!sqe_hdr->total) {
808 /* Don't free Tx buffers to Aura */
809 sqe_hdr->df = 1;
810 sqe_hdr->aura = sq->aura_id;
811 /* Post a CQE Tx after pkt transmission */
812 sqe_hdr->pnc = 1;
813 sqe_hdr->sq = (qidx >= pfvf->hw.tx_queues) ?
814 qidx + pfvf->hw.xdp_queues : qidx;
815 }
816 sqe_hdr->total = skb->len;
817 /* Set SQE identifier which will be used later for freeing SKB */
818 sqe_hdr->sqe_id = sq->head;
819
820 /* Offload TCP/UDP checksum to HW */
821 if (skb->ip_summed == CHECKSUM_PARTIAL) {
822 sqe_hdr->ol3ptr = skb_network_offset(skb);
823 sqe_hdr->ol4ptr = skb_transport_offset(skb);
824 /* get vlan protocol Ethertype */
825 if (eth_type_vlan(skb->protocol))
826 skb->protocol = vlan_get_protocol(skb);
827
828 if (skb->protocol == htons(ETH_P_IP)) {
829 proto = ip_hdr(skb)->protocol;
830 /* In case of TSO, HW needs this to be explicitly set.
831 * So set this always, instead of adding a check.
832 */
833 sqe_hdr->ol3type = NIX_SENDL3TYPE_IP4_CKSUM;
834 } else if (skb->protocol == htons(ETH_P_IPV6)) {
835 proto = ipv6_hdr(skb)->nexthdr;
836 sqe_hdr->ol3type = NIX_SENDL3TYPE_IP6;
837 }
838
839 if (proto == IPPROTO_TCP)
840 sqe_hdr->ol4type = NIX_SENDL4TYPE_TCP_CKSUM;
841 else if (proto == IPPROTO_UDP)
842 sqe_hdr->ol4type = NIX_SENDL4TYPE_UDP_CKSUM;
843 }
844 }
845
otx2_dma_map_tso_skb(struct otx2_nic * pfvf,struct otx2_snd_queue * sq,struct sk_buff * skb,int sqe,int hdr_len)846 static int otx2_dma_map_tso_skb(struct otx2_nic *pfvf,
847 struct otx2_snd_queue *sq,
848 struct sk_buff *skb, int sqe, int hdr_len)
849 {
850 int num_segs = skb_shinfo(skb)->nr_frags + 1;
851 struct sg_list *sg = &sq->sg[sqe];
852 u64 dma_addr;
853 int seg, len;
854
855 sg->num_segs = 0;
856
857 /* Get payload length at skb->data */
858 len = skb_headlen(skb) - hdr_len;
859
860 for (seg = 0; seg < num_segs; seg++) {
861 /* Skip skb->data, if there is no payload */
862 if (!seg && !len)
863 continue;
864 dma_addr = otx2_dma_map_skb_frag(pfvf, skb, seg, &len);
865 if (dma_mapping_error(pfvf->dev, dma_addr))
866 goto unmap;
867
868 /* Save DMA mapping info for later unmapping */
869 sg->dma_addr[sg->num_segs] = dma_addr;
870 sg->size[sg->num_segs] = len;
871 sg->num_segs++;
872 }
873 return 0;
874 unmap:
875 otx2_dma_unmap_skb_frags(pfvf, sg);
876 return -EINVAL;
877 }
878
otx2_tso_frag_dma_addr(struct otx2_snd_queue * sq,struct sk_buff * skb,int seg,u64 seg_addr,int hdr_len,int sqe)879 static u64 otx2_tso_frag_dma_addr(struct otx2_snd_queue *sq,
880 struct sk_buff *skb, int seg,
881 u64 seg_addr, int hdr_len, int sqe)
882 {
883 struct sg_list *sg = &sq->sg[sqe];
884 const skb_frag_t *frag;
885 int offset;
886
887 if (seg < 0)
888 return sg->dma_addr[0] + (seg_addr - (u64)skb->data);
889
890 frag = &skb_shinfo(skb)->frags[seg];
891 offset = seg_addr - (u64)skb_frag_address(frag);
892 if (skb_headlen(skb) - hdr_len)
893 seg++;
894 return sg->dma_addr[seg] + offset;
895 }
896
otx2_sqe_tso_add_sg(struct otx2_snd_queue * sq,struct sg_list * list,int * offset)897 static void otx2_sqe_tso_add_sg(struct otx2_snd_queue *sq,
898 struct sg_list *list, int *offset)
899 {
900 struct nix_sqe_sg_s *sg = NULL;
901 u16 *sg_lens = NULL;
902 u64 *iova = NULL;
903 int seg;
904
905 /* Add SG descriptors with buffer addresses */
906 for (seg = 0; seg < list->num_segs; seg++) {
907 if ((seg % MAX_SEGS_PER_SG) == 0) {
908 sg = (struct nix_sqe_sg_s *)(sq->sqe_base + *offset);
909 sg->ld_type = NIX_SEND_LDTYPE_LDD;
910 sg->subdc = NIX_SUBDC_SG;
911 sg->segs = 0;
912 sg_lens = (void *)sg;
913 iova = (void *)sg + sizeof(*sg);
914 /* Next subdc always starts at a 16byte boundary.
915 * So if sg->segs is whether 2 or 3, offset += 16bytes.
916 */
917 if ((list->num_segs - seg) >= (MAX_SEGS_PER_SG - 1))
918 *offset += sizeof(*sg) + (3 * sizeof(u64));
919 else
920 *offset += sizeof(*sg) + sizeof(u64);
921 }
922 sg_lens[frag_num(seg % MAX_SEGS_PER_SG)] = list->size[seg];
923 *iova++ = list->dma_addr[seg];
924 sg->segs++;
925 }
926 }
927
otx2_sq_append_tso(struct otx2_nic * pfvf,struct otx2_snd_queue * sq,struct sk_buff * skb,u16 qidx)928 static void otx2_sq_append_tso(struct otx2_nic *pfvf, struct otx2_snd_queue *sq,
929 struct sk_buff *skb, u16 qidx)
930 {
931 struct netdev_queue *txq = netdev_get_tx_queue(pfvf->netdev, qidx);
932 int hdr_len, tcp_data, seg_len, pkt_len, offset;
933 struct nix_sqe_hdr_s *sqe_hdr;
934 int first_sqe = sq->head;
935 struct sg_list list;
936 struct tso_t tso;
937
938 hdr_len = tso_start(skb, &tso);
939
940 /* Map SKB's fragments to DMA.
941 * It's done here to avoid mapping for every TSO segment's packet.
942 */
943 if (otx2_dma_map_tso_skb(pfvf, sq, skb, first_sqe, hdr_len)) {
944 dev_kfree_skb_any(skb);
945 return;
946 }
947
948 netdev_tx_sent_queue(txq, skb->len);
949
950 tcp_data = skb->len - hdr_len;
951 while (tcp_data > 0) {
952 char *hdr;
953
954 seg_len = min_t(int, skb_shinfo(skb)->gso_size, tcp_data);
955 tcp_data -= seg_len;
956
957 /* Set SQE's SEND_HDR */
958 memset(sq->sqe_base, 0, sq->sqe_size);
959 sqe_hdr = (struct nix_sqe_hdr_s *)(sq->sqe_base);
960 otx2_sqe_add_hdr(pfvf, sq, sqe_hdr, skb, qidx);
961 offset = sizeof(*sqe_hdr);
962
963 /* Add TSO segment's pkt header */
964 hdr = sq->tso_hdrs->base + (sq->head * TSO_HEADER_SIZE);
965 tso_build_hdr(skb, hdr, &tso, seg_len, tcp_data == 0);
966 list.dma_addr[0] =
967 sq->tso_hdrs->iova + (sq->head * TSO_HEADER_SIZE);
968 list.size[0] = hdr_len;
969 list.num_segs = 1;
970
971 /* Add TSO segment's payload data fragments */
972 pkt_len = hdr_len;
973 while (seg_len > 0) {
974 int size;
975
976 size = min_t(int, tso.size, seg_len);
977
978 list.size[list.num_segs] = size;
979 list.dma_addr[list.num_segs] =
980 otx2_tso_frag_dma_addr(sq, skb,
981 tso.next_frag_idx - 1,
982 (u64)tso.data, hdr_len,
983 first_sqe);
984 list.num_segs++;
985 pkt_len += size;
986 seg_len -= size;
987 tso_build_data(skb, &tso, size);
988 }
989 sqe_hdr->total = pkt_len;
990 otx2_sqe_tso_add_sg(sq, &list, &offset);
991
992 /* DMA mappings and skb needs to be freed only after last
993 * TSO segment is transmitted out. So set 'PNC' only for
994 * last segment. Also point last segment's sqe_id to first
995 * segment's SQE index where skb address and DMA mappings
996 * are saved.
997 */
998 if (!tcp_data) {
999 sqe_hdr->pnc = 1;
1000 sqe_hdr->sqe_id = first_sqe;
1001 sq->sg[first_sqe].skb = (u64)skb;
1002 } else {
1003 sqe_hdr->pnc = 0;
1004 }
1005
1006 sqe_hdr->sizem1 = (offset / 16) - 1;
1007
1008 /* Flush SQE to HW */
1009 pfvf->hw_ops->sqe_flush(pfvf, sq, offset, qidx);
1010 }
1011 }
1012
is_hw_tso_supported(struct otx2_nic * pfvf,struct sk_buff * skb)1013 static bool is_hw_tso_supported(struct otx2_nic *pfvf,
1014 struct sk_buff *skb)
1015 {
1016 int payload_len, last_seg_size;
1017
1018 if (test_bit(HW_TSO, &pfvf->hw.cap_flag))
1019 return true;
1020
1021 /* On 96xx A0, HW TSO not supported */
1022 if (!is_96xx_B0(pfvf->pdev))
1023 return false;
1024
1025 /* HW has an issue due to which when the payload of the last LSO
1026 * segment is shorter than 16 bytes, some header fields may not
1027 * be correctly modified, hence don't offload such TSO segments.
1028 */
1029
1030 payload_len = skb->len - skb_tcp_all_headers(skb);
1031 last_seg_size = payload_len % skb_shinfo(skb)->gso_size;
1032 if (last_seg_size && last_seg_size < 16)
1033 return false;
1034
1035 return true;
1036 }
1037
otx2_get_sqe_count(struct otx2_nic * pfvf,struct sk_buff * skb)1038 static int otx2_get_sqe_count(struct otx2_nic *pfvf, struct sk_buff *skb)
1039 {
1040 if (!skb_shinfo(skb)->gso_size)
1041 return 1;
1042
1043 /* HW TSO */
1044 if (is_hw_tso_supported(pfvf, skb))
1045 return 1;
1046
1047 /* SW TSO */
1048 return skb_shinfo(skb)->gso_segs;
1049 }
1050
otx2_validate_network_transport(struct sk_buff * skb)1051 static bool otx2_validate_network_transport(struct sk_buff *skb)
1052 {
1053 if ((ip_hdr(skb)->protocol == IPPROTO_UDP) ||
1054 (ipv6_hdr(skb)->nexthdr == IPPROTO_UDP)) {
1055 struct udphdr *udph = udp_hdr(skb);
1056
1057 if (udph->source == htons(PTP_PORT) &&
1058 udph->dest == htons(PTP_PORT))
1059 return true;
1060 }
1061
1062 return false;
1063 }
1064
otx2_ptp_is_sync(struct sk_buff * skb,int * offset,bool * udp_csum_crt)1065 static bool otx2_ptp_is_sync(struct sk_buff *skb, int *offset, bool *udp_csum_crt)
1066 {
1067 struct ethhdr *eth = (struct ethhdr *)(skb->data);
1068 u16 nix_offload_hlen = 0, inner_vhlen = 0;
1069 bool udp_hdr_present = false, is_sync;
1070 u8 *data = skb->data, *msgtype;
1071 __be16 proto = eth->h_proto;
1072 int network_depth = 0;
1073
1074 /* NIX is programmed to offload outer VLAN header
1075 * in case of single vlan protocol field holds Network header ETH_IP/V6
1076 * in case of stacked vlan protocol field holds Inner vlan (8100)
1077 */
1078 if (skb->dev->features & NETIF_F_HW_VLAN_CTAG_TX &&
1079 skb->dev->features & NETIF_F_HW_VLAN_STAG_TX) {
1080 if (skb->vlan_proto == htons(ETH_P_8021AD)) {
1081 /* Get vlan protocol */
1082 proto = __vlan_get_protocol(skb, eth->h_proto, NULL);
1083 /* SKB APIs like skb_transport_offset does not include
1084 * offloaded vlan header length. Need to explicitly add
1085 * the length
1086 */
1087 nix_offload_hlen = VLAN_HLEN;
1088 inner_vhlen = VLAN_HLEN;
1089 } else if (skb->vlan_proto == htons(ETH_P_8021Q)) {
1090 nix_offload_hlen = VLAN_HLEN;
1091 }
1092 } else if (eth_type_vlan(eth->h_proto)) {
1093 proto = __vlan_get_protocol(skb, eth->h_proto, &network_depth);
1094 }
1095
1096 switch (ntohs(proto)) {
1097 case ETH_P_1588:
1098 if (network_depth)
1099 *offset = network_depth;
1100 else
1101 *offset = ETH_HLEN + nix_offload_hlen +
1102 inner_vhlen;
1103 break;
1104 case ETH_P_IP:
1105 case ETH_P_IPV6:
1106 if (!otx2_validate_network_transport(skb))
1107 return false;
1108
1109 *offset = nix_offload_hlen + skb_transport_offset(skb) +
1110 sizeof(struct udphdr);
1111 udp_hdr_present = true;
1112
1113 }
1114
1115 msgtype = data + *offset;
1116 /* Check PTP messageId is SYNC or not */
1117 is_sync = !(*msgtype & 0xf);
1118 if (is_sync)
1119 *udp_csum_crt = udp_hdr_present;
1120 else
1121 *offset = 0;
1122
1123 return is_sync;
1124 }
1125
otx2_set_txtstamp(struct otx2_nic * pfvf,struct sk_buff * skb,struct otx2_snd_queue * sq,int * offset)1126 static void otx2_set_txtstamp(struct otx2_nic *pfvf, struct sk_buff *skb,
1127 struct otx2_snd_queue *sq, int *offset)
1128 {
1129 struct ethhdr *eth = (struct ethhdr *)(skb->data);
1130 struct ptpv2_tstamp *origin_tstamp;
1131 bool udp_csum_crt = false;
1132 unsigned int udphoff;
1133 struct timespec64 ts;
1134 int ptp_offset = 0;
1135 __wsum skb_csum;
1136 u64 iova;
1137
1138 if (unlikely(!skb_shinfo(skb)->gso_size &&
1139 (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))) {
1140 if (unlikely(pfvf->flags & OTX2_FLAG_PTP_ONESTEP_SYNC &&
1141 otx2_ptp_is_sync(skb, &ptp_offset, &udp_csum_crt))) {
1142 origin_tstamp = (struct ptpv2_tstamp *)
1143 ((u8 *)skb->data + ptp_offset +
1144 PTP_SYNC_SEC_OFFSET);
1145 ts = ns_to_timespec64(pfvf->ptp->tstamp);
1146 origin_tstamp->seconds_msb = htons((ts.tv_sec >> 32) & 0xffff);
1147 origin_tstamp->seconds_lsb = htonl(ts.tv_sec & 0xffffffff);
1148 origin_tstamp->nanoseconds = htonl(ts.tv_nsec);
1149 /* Point to correction field in PTP packet */
1150 ptp_offset += 8;
1151
1152 /* When user disables hw checksum, stack calculates the csum,
1153 * but it does not cover ptp timestamp which is added later.
1154 * Recalculate the checksum manually considering the timestamp.
1155 */
1156 if (udp_csum_crt) {
1157 struct udphdr *uh = udp_hdr(skb);
1158
1159 if (skb->ip_summed != CHECKSUM_PARTIAL && uh->check != 0) {
1160 udphoff = skb_transport_offset(skb);
1161 uh->check = 0;
1162 skb_csum = skb_checksum(skb, udphoff, skb->len - udphoff,
1163 0);
1164 if (ntohs(eth->h_proto) == ETH_P_IPV6)
1165 uh->check = csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
1166 &ipv6_hdr(skb)->daddr,
1167 skb->len - udphoff,
1168 ipv6_hdr(skb)->nexthdr,
1169 skb_csum);
1170 else
1171 uh->check = csum_tcpudp_magic(ip_hdr(skb)->saddr,
1172 ip_hdr(skb)->daddr,
1173 skb->len - udphoff,
1174 IPPROTO_UDP,
1175 skb_csum);
1176 }
1177 }
1178 } else {
1179 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1180 }
1181 iova = sq->timestamps->iova + (sq->head * sizeof(u64));
1182 otx2_sqe_add_mem(sq, offset, NIX_SENDMEMALG_E_SETTSTMP, iova,
1183 ptp_offset, pfvf->ptp->base_ns, udp_csum_crt);
1184 } else {
1185 skb_tx_timestamp(skb);
1186 }
1187 }
1188
otx2_sq_append_skb(void * dev,struct netdev_queue * txq,struct otx2_snd_queue * sq,struct sk_buff * skb,u16 qidx)1189 bool otx2_sq_append_skb(void *dev, struct netdev_queue *txq,
1190 struct otx2_snd_queue *sq,
1191 struct sk_buff *skb, u16 qidx)
1192 {
1193 int offset, num_segs, free_desc;
1194 struct nix_sqe_hdr_s *sqe_hdr;
1195 struct otx2_nic *pfvf = dev;
1196 bool ret;
1197
1198 /* Check if there is enough room between producer
1199 * and consumer index.
1200 */
1201 free_desc = otx2_get_free_sqe(sq);
1202 if (free_desc < sq->sqe_thresh)
1203 return false;
1204
1205 if (free_desc < otx2_get_sqe_count(pfvf, skb))
1206 return false;
1207
1208 num_segs = skb_shinfo(skb)->nr_frags + 1;
1209
1210 /* If SKB doesn't fit in a single SQE, linearize it.
1211 * TODO: Consider adding JUMP descriptor instead.
1212 */
1213
1214 if (unlikely(num_segs > OTX2_MAX_FRAGS_IN_SQE)) {
1215 if (__skb_linearize(skb)) {
1216 dev_kfree_skb_any(skb);
1217 return true;
1218 }
1219 num_segs = skb_shinfo(skb)->nr_frags + 1;
1220 }
1221
1222 if (skb_shinfo(skb)->gso_size && !is_hw_tso_supported(pfvf, skb)) {
1223 /* Insert vlan tag before giving pkt to tso */
1224 if (skb_vlan_tag_present(skb)) {
1225 skb = __vlan_hwaccel_push_inside(skb);
1226 if (!skb)
1227 return true;
1228 }
1229 otx2_sq_append_tso(pfvf, sq, skb, qidx);
1230 return true;
1231 }
1232
1233 /* Set sqe base address */
1234 otx2_sq_set_sqe_base(sq, skb);
1235
1236 /* Set SQE's SEND_HDR.
1237 * Do not clear the first 64bit as it contains constant info.
1238 */
1239 memset(sq->sqe_base + 8, 0, sq->sqe_size - 8);
1240 sqe_hdr = (struct nix_sqe_hdr_s *)(sq->sqe_base);
1241 otx2_sqe_add_hdr(pfvf, sq, sqe_hdr, skb, qidx);
1242 offset = sizeof(*sqe_hdr);
1243
1244 /* Add extended header if needed */
1245 otx2_sqe_add_ext(pfvf, sq, skb, &offset);
1246
1247 /* Add SG subdesc with data frags */
1248 if (static_branch_unlikely(&cn10k_ipsec_sa_enabled) &&
1249 (xfrm_offload(skb)))
1250 ret = otx2_sqe_add_sg_ipsec(pfvf, sq, skb, num_segs, &offset);
1251 else
1252 ret = otx2_sqe_add_sg(pfvf, sq, skb, num_segs, &offset);
1253
1254 if (!ret) {
1255 otx2_dma_unmap_skb_frags(pfvf, &sq->sg[sq->head]);
1256 return false;
1257 }
1258
1259 otx2_set_txtstamp(pfvf, skb, sq, &offset);
1260
1261 sqe_hdr->sizem1 = (offset / 16) - 1;
1262
1263 if (static_branch_unlikely(&cn10k_ipsec_sa_enabled) &&
1264 (xfrm_offload(skb)))
1265 return cn10k_ipsec_transmit(pfvf, txq, sq, skb, num_segs,
1266 offset);
1267
1268 netdev_tx_sent_queue(txq, skb->len);
1269
1270 /* Flush SQE to HW */
1271 pfvf->hw_ops->sqe_flush(pfvf, sq, offset, qidx);
1272 return true;
1273 }
1274 EXPORT_SYMBOL(otx2_sq_append_skb);
1275
otx2_cleanup_rx_cqes(struct otx2_nic * pfvf,struct otx2_cq_queue * cq,int qidx)1276 void otx2_cleanup_rx_cqes(struct otx2_nic *pfvf, struct otx2_cq_queue *cq, int qidx)
1277 {
1278 struct nix_cqe_rx_s *cqe;
1279 struct otx2_pool *pool;
1280 int processed_cqe = 0;
1281 u16 pool_id;
1282 u64 iova;
1283
1284 pool_id = otx2_get_pool_idx(pfvf, AURA_NIX_RQ, qidx);
1285 pool = &pfvf->qset.pool[pool_id];
1286
1287 if (pfvf->xdp_prog) {
1288 if (pool->page_pool)
1289 xdp_rxq_info_unreg_mem_model(&cq->xdp_rxq);
1290
1291 xdp_rxq_info_unreg(&cq->xdp_rxq);
1292 }
1293
1294 if (otx2_nix_cq_op_status(pfvf, cq) || !cq->pend_cqe)
1295 return;
1296
1297 while (cq->pend_cqe) {
1298 cqe = (struct nix_cqe_rx_s *)otx2_get_next_cqe(cq);
1299 processed_cqe++;
1300 cq->pend_cqe--;
1301
1302 if (!cqe)
1303 continue;
1304 if (cqe->sg.segs > 1) {
1305 otx2_free_rcv_seg(pfvf, cqe, cq->cq_idx);
1306 continue;
1307 }
1308 iova = cqe->sg.seg_addr - OTX2_HEAD_ROOM;
1309
1310 otx2_free_bufs(pfvf, pool, iova, pfvf->rbsize);
1311 }
1312
1313 /* Free CQEs to HW */
1314 otx2_write64(pfvf, NIX_LF_CQ_OP_DOOR,
1315 ((u64)cq->cq_idx << 32) | processed_cqe);
1316 }
1317
otx2_cleanup_tx_cqes(struct otx2_nic * pfvf,struct otx2_cq_queue * cq)1318 void otx2_cleanup_tx_cqes(struct otx2_nic *pfvf, struct otx2_cq_queue *cq)
1319 {
1320 int tx_pkts = 0, tx_bytes = 0;
1321 struct sk_buff *skb = NULL;
1322 struct otx2_snd_queue *sq;
1323 struct nix_cqe_tx_s *cqe;
1324 struct netdev_queue *txq;
1325 int processed_cqe = 0;
1326 struct sg_list *sg;
1327 int qidx;
1328
1329 qidx = cq->cq_idx - pfvf->hw.rx_queues;
1330 sq = &pfvf->qset.sq[qidx];
1331
1332 if (otx2_nix_cq_op_status(pfvf, cq) || !cq->pend_cqe)
1333 return;
1334
1335 while (cq->pend_cqe) {
1336 cqe = (struct nix_cqe_tx_s *)otx2_get_next_cqe(cq);
1337 processed_cqe++;
1338 cq->pend_cqe--;
1339
1340 if (!cqe)
1341 continue;
1342 sg = &sq->sg[cqe->comp.sqe_id];
1343 skb = (struct sk_buff *)sg->skb;
1344 if (skb) {
1345 tx_bytes += skb->len;
1346 tx_pkts++;
1347 otx2_dma_unmap_skb_frags(pfvf, sg);
1348 dev_kfree_skb_any(skb);
1349 sg->skb = (u64)NULL;
1350 }
1351 }
1352
1353 if (likely(tx_pkts)) {
1354 if (qidx >= pfvf->hw.tx_queues)
1355 qidx -= pfvf->hw.xdp_queues;
1356 txq = netdev_get_tx_queue(pfvf->netdev, qidx);
1357 netdev_tx_completed_queue(txq, tx_pkts, tx_bytes);
1358 }
1359 /* Free CQEs to HW */
1360 otx2_write64(pfvf, NIX_LF_CQ_OP_DOOR,
1361 ((u64)cq->cq_idx << 32) | processed_cqe);
1362 }
1363
otx2_rxtx_enable(struct otx2_nic * pfvf,bool enable)1364 int otx2_rxtx_enable(struct otx2_nic *pfvf, bool enable)
1365 {
1366 struct msg_req *msg;
1367 int err;
1368
1369 mutex_lock(&pfvf->mbox.lock);
1370 if (enable)
1371 msg = otx2_mbox_alloc_msg_nix_lf_start_rx(&pfvf->mbox);
1372 else
1373 msg = otx2_mbox_alloc_msg_nix_lf_stop_rx(&pfvf->mbox);
1374
1375 if (!msg) {
1376 mutex_unlock(&pfvf->mbox.lock);
1377 return -ENOMEM;
1378 }
1379
1380 err = otx2_sync_mbox_msg(&pfvf->mbox);
1381 mutex_unlock(&pfvf->mbox.lock);
1382 return err;
1383 }
1384
otx2_free_pending_sqe(struct otx2_nic * pfvf)1385 void otx2_free_pending_sqe(struct otx2_nic *pfvf)
1386 {
1387 int tx_pkts = 0, tx_bytes = 0;
1388 struct sk_buff *skb = NULL;
1389 struct otx2_snd_queue *sq;
1390 struct netdev_queue *txq;
1391 struct sg_list *sg;
1392 int sq_idx, sqe;
1393
1394 for (sq_idx = 0; sq_idx < pfvf->hw.tx_queues; sq_idx++) {
1395 sq = &pfvf->qset.sq[sq_idx];
1396 for (sqe = 0; sqe < sq->sqe_cnt; sqe++) {
1397 sg = &sq->sg[sqe];
1398 skb = (struct sk_buff *)sg->skb;
1399 if (skb) {
1400 tx_bytes += skb->len;
1401 tx_pkts++;
1402 otx2_dma_unmap_skb_frags(pfvf, sg);
1403 dev_kfree_skb_any(skb);
1404 sg->skb = (u64)NULL;
1405 }
1406 }
1407
1408 if (!tx_pkts)
1409 continue;
1410 txq = netdev_get_tx_queue(pfvf->netdev, sq_idx);
1411 netdev_tx_completed_queue(txq, tx_pkts, tx_bytes);
1412 tx_pkts = 0;
1413 tx_bytes = 0;
1414 }
1415 }
1416
otx2_xdp_sqe_add_sg(struct otx2_snd_queue * sq,struct xdp_frame * xdpf,u64 dma_addr,int len,int * offset,u16 flags)1417 void otx2_xdp_sqe_add_sg(struct otx2_snd_queue *sq, struct xdp_frame *xdpf,
1418 u64 dma_addr, int len, int *offset, u16 flags)
1419 {
1420 struct nix_sqe_sg_s *sg = NULL;
1421 u64 *iova = NULL;
1422
1423 sg = (struct nix_sqe_sg_s *)(sq->sqe_base + *offset);
1424 sg->ld_type = NIX_SEND_LDTYPE_LDD;
1425 sg->subdc = NIX_SUBDC_SG;
1426 sg->segs = 1;
1427 sg->seg1_size = len;
1428 iova = (void *)sg + sizeof(*sg);
1429 *iova = dma_addr;
1430 *offset += sizeof(*sg) + sizeof(u64);
1431
1432 sq->sg[sq->head].dma_addr[0] = dma_addr;
1433 sq->sg[sq->head].size[0] = len;
1434 sq->sg[sq->head].num_segs = 1;
1435 sq->sg[sq->head].flags = flags;
1436 sq->sg[sq->head].skb = (u64)xdpf;
1437 }
1438
otx2_read_free_sqe(struct otx2_nic * pfvf,u16 qidx)1439 int otx2_read_free_sqe(struct otx2_nic *pfvf, u16 qidx)
1440 {
1441 struct otx2_snd_queue *sq;
1442 int free_sqe;
1443
1444 sq = &pfvf->qset.sq[qidx];
1445 free_sqe = otx2_get_free_sqe(sq);
1446 if (free_sqe < sq->sqe_thresh) {
1447 netdev_warn(pfvf->netdev, "No free sqe for Send queue%d\n", qidx);
1448 return 0;
1449 }
1450
1451 return free_sqe - sq->sqe_thresh;
1452 }
1453
otx2_xdp_sq_append_pkt(struct otx2_nic * pfvf,struct xdp_frame * xdpf,u64 iova,int len,u16 qidx,u16 flags)1454 bool otx2_xdp_sq_append_pkt(struct otx2_nic *pfvf, struct xdp_frame *xdpf,
1455 u64 iova, int len, u16 qidx, u16 flags)
1456 {
1457 struct nix_sqe_hdr_s *sqe_hdr;
1458 struct otx2_snd_queue *sq;
1459 int offset, free_sqe;
1460
1461 sq = &pfvf->qset.sq[qidx];
1462 free_sqe = otx2_get_free_sqe(sq);
1463 if (free_sqe < sq->sqe_thresh)
1464 return false;
1465
1466 memset(sq->sqe_base + 8, 0, sq->sqe_size - 8);
1467
1468 sqe_hdr = (struct nix_sqe_hdr_s *)(sq->sqe_base);
1469
1470 if (!sqe_hdr->total) {
1471 sqe_hdr->aura = sq->aura_id;
1472 sqe_hdr->df = 1;
1473 sqe_hdr->sq = qidx;
1474 sqe_hdr->pnc = 1;
1475 }
1476 sqe_hdr->total = len;
1477 sqe_hdr->sqe_id = sq->head;
1478
1479 offset = sizeof(*sqe_hdr);
1480
1481 otx2_xdp_sqe_add_sg(sq, xdpf, iova, len, &offset, flags);
1482 sqe_hdr->sizem1 = (offset / 16) - 1;
1483 pfvf->hw_ops->sqe_flush(pfvf, sq, offset, qidx);
1484
1485 return true;
1486 }
1487
otx2_xdp_rcv_pkt_handler(struct otx2_nic * pfvf,struct bpf_prog * prog,struct nix_cqe_rx_s * cqe,struct otx2_cq_queue * cq,u32 * metasize,bool * need_xdp_flush)1488 static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf,
1489 struct bpf_prog *prog,
1490 struct nix_cqe_rx_s *cqe,
1491 struct otx2_cq_queue *cq,
1492 u32 *metasize, bool *need_xdp_flush)
1493 {
1494 struct xdp_buff xdp, *xsk_buff = NULL;
1495 unsigned char *hard_start;
1496 struct otx2_pool *pool;
1497 struct xdp_frame *xdpf;
1498 int qidx = cq->cq_idx;
1499 struct page *page;
1500 u64 iova, pa;
1501 u32 act;
1502 int err;
1503
1504 pool = &pfvf->qset.pool[qidx];
1505
1506 if (pool->xsk_pool) {
1507 xsk_buff = pool->xdp[--cq->rbpool->xdp_top];
1508 if (!xsk_buff)
1509 return false;
1510
1511 xsk_buff->data_end = xsk_buff->data + cqe->sg.seg_size;
1512 act = bpf_prog_run_xdp(prog, xsk_buff);
1513 goto handle_xdp_verdict;
1514 }
1515
1516 iova = cqe->sg.seg_addr - OTX2_HEAD_ROOM;
1517 pa = otx2_iova_to_phys(pfvf->iommu_domain, iova);
1518 page = virt_to_page(phys_to_virt(pa));
1519
1520 xdp_init_buff(&xdp, pfvf->rbsize, &cq->xdp_rxq);
1521
1522 hard_start = (unsigned char *)phys_to_virt(pa);
1523 xdp_prepare_buff(&xdp, hard_start, OTX2_HEAD_ROOM,
1524 cqe->sg.seg_size, true);
1525
1526 act = bpf_prog_run_xdp(prog, &xdp);
1527
1528 handle_xdp_verdict:
1529 switch (act) {
1530 case XDP_PASS:
1531 *metasize = xdp.data - xdp.data_meta;
1532 break;
1533 case XDP_TX:
1534 qidx += pfvf->hw.tx_queues;
1535 cq->pool_ptrs++;
1536 xdpf = xdp_convert_buff_to_frame(&xdp);
1537 return otx2_xdp_sq_append_pkt(pfvf, xdpf,
1538 cqe->sg.seg_addr,
1539 cqe->sg.seg_size,
1540 qidx, OTX2_XDP_TX);
1541 case XDP_REDIRECT:
1542 cq->pool_ptrs++;
1543 if (xsk_buff) {
1544 err = xdp_do_redirect(pfvf->netdev, xsk_buff, prog);
1545 if (!err) {
1546 *need_xdp_flush = true;
1547 return true;
1548 }
1549 return false;
1550 }
1551
1552 err = xdp_do_redirect(pfvf->netdev, &xdp, prog);
1553 if (!err) {
1554 *need_xdp_flush = true;
1555 return true;
1556 }
1557
1558 otx2_dma_unmap_page(pfvf, iova, pfvf->rbsize,
1559 DMA_FROM_DEVICE);
1560 xdpf = xdp_convert_buff_to_frame(&xdp);
1561 xdp_return_frame(xdpf);
1562 break;
1563 default:
1564 bpf_warn_invalid_xdp_action(pfvf->netdev, prog, act);
1565 fallthrough;
1566 case XDP_ABORTED:
1567 if (act == XDP_ABORTED)
1568 trace_xdp_exception(pfvf->netdev, prog, act);
1569 fallthrough;
1570 case XDP_DROP:
1571 cq->pool_ptrs++;
1572 if (xsk_buff) {
1573 xsk_buff_free(xsk_buff);
1574 } else if (pp_page_to_nmdesc(page)->pp) {
1575 page_pool_recycle_direct(pool->page_pool, page);
1576 } else {
1577 otx2_dma_unmap_page(pfvf, iova, pfvf->rbsize,
1578 DMA_FROM_DEVICE);
1579 put_page(page);
1580 }
1581 return true;
1582 }
1583 return false;
1584 }
1585