1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell RVU Physical Function ethernet driver
3 *
4 * Copyright (C) 2020 Marvell.
5 *
6 */
7
8 #include <linux/module.h>
9 #include <linux/interrupt.h>
10 #include <linux/pci.h>
11 #include <linux/etherdevice.h>
12 #include <linux/of.h>
13 #include <linux/if_vlan.h>
14 #include <linux/iommu.h>
15 #include <net/ip.h>
16 #include <linux/bpf.h>
17 #include <linux/bpf_trace.h>
18 #include <linux/bitfield.h>
19 #include <net/page_pool/types.h>
20
21 #include "otx2_reg.h"
22 #include "otx2_common.h"
23 #include "otx2_txrx.h"
24 #include "otx2_struct.h"
25 #include "otx2_ptp.h"
26 #include "cn10k.h"
27 #include "qos.h"
28 #include <rvu_trace.h>
29
30 #define DRV_NAME "rvu_nicpf"
31 #define DRV_STRING "Marvell RVU NIC Physical Function Driver"
32
33 /* Supported devices */
34 static const struct pci_device_id otx2_pf_id_table[] = {
35 { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_OCTEONTX2_RVU_PF) },
36 { 0, } /* end of table */
37 };
38
39 MODULE_AUTHOR("Sunil Goutham <sgoutham@marvell.com>");
40 MODULE_DESCRIPTION(DRV_STRING);
41 MODULE_LICENSE("GPL v2");
42 MODULE_DEVICE_TABLE(pci, otx2_pf_id_table);
43
44 static void otx2_vf_link_event_task(struct work_struct *work);
45
46 enum {
47 TYPE_PFAF,
48 TYPE_PFVF,
49 };
50
51 static int otx2_config_hw_tx_tstamp(struct otx2_nic *pfvf, bool enable);
52 static int otx2_config_hw_rx_tstamp(struct otx2_nic *pfvf, bool enable);
53
otx2_change_mtu(struct net_device * netdev,int new_mtu)54 static int otx2_change_mtu(struct net_device *netdev, int new_mtu)
55 {
56 struct otx2_nic *pf = netdev_priv(netdev);
57 bool if_up = netif_running(netdev);
58 int err = 0;
59
60 if (pf->xdp_prog && new_mtu > MAX_XDP_MTU) {
61 netdev_warn(netdev, "Jumbo frames not yet supported with XDP, current MTU %d.\n",
62 netdev->mtu);
63 return -EINVAL;
64 }
65 if (if_up)
66 otx2_stop(netdev);
67
68 netdev_info(netdev, "Changing MTU from %d to %d\n",
69 netdev->mtu, new_mtu);
70 netdev->mtu = new_mtu;
71
72 if (if_up)
73 err = otx2_open(netdev);
74
75 return err;
76 }
77
otx2_disable_flr_me_intr(struct otx2_nic * pf)78 static void otx2_disable_flr_me_intr(struct otx2_nic *pf)
79 {
80 int irq, vfs = pf->total_vfs;
81
82 /* Disable VFs ME interrupts */
83 otx2_write64(pf, RVU_PF_VFME_INT_ENA_W1CX(0), INTR_MASK(vfs));
84 irq = pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFME0);
85 free_irq(irq, pf);
86
87 /* Disable VFs FLR interrupts */
88 otx2_write64(pf, RVU_PF_VFFLR_INT_ENA_W1CX(0), INTR_MASK(vfs));
89 irq = pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFFLR0);
90 free_irq(irq, pf);
91
92 if (vfs <= 64)
93 return;
94
95 otx2_write64(pf, RVU_PF_VFME_INT_ENA_W1CX(1), INTR_MASK(vfs - 64));
96 irq = pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFME1);
97 free_irq(irq, pf);
98
99 otx2_write64(pf, RVU_PF_VFFLR_INT_ENA_W1CX(1), INTR_MASK(vfs - 64));
100 irq = pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFFLR1);
101 free_irq(irq, pf);
102 }
103
otx2_flr_wq_destroy(struct otx2_nic * pf)104 static void otx2_flr_wq_destroy(struct otx2_nic *pf)
105 {
106 if (!pf->flr_wq)
107 return;
108 destroy_workqueue(pf->flr_wq);
109 pf->flr_wq = NULL;
110 devm_kfree(pf->dev, pf->flr_wrk);
111 }
112
otx2_flr_handler(struct work_struct * work)113 static void otx2_flr_handler(struct work_struct *work)
114 {
115 struct flr_work *flrwork = container_of(work, struct flr_work, work);
116 struct otx2_nic *pf = flrwork->pf;
117 struct mbox *mbox = &pf->mbox;
118 struct msg_req *req;
119 int vf, reg = 0;
120
121 vf = flrwork - pf->flr_wrk;
122
123 mutex_lock(&mbox->lock);
124 req = otx2_mbox_alloc_msg_vf_flr(mbox);
125 if (!req) {
126 mutex_unlock(&mbox->lock);
127 return;
128 }
129 req->hdr.pcifunc &= RVU_PFVF_FUNC_MASK;
130 req->hdr.pcifunc |= (vf + 1) & RVU_PFVF_FUNC_MASK;
131
132 if (!otx2_sync_mbox_msg(&pf->mbox)) {
133 if (vf >= 64) {
134 reg = 1;
135 vf = vf - 64;
136 }
137 /* clear transcation pending bit */
138 otx2_write64(pf, RVU_PF_VFTRPENDX(reg), BIT_ULL(vf));
139 otx2_write64(pf, RVU_PF_VFFLR_INT_ENA_W1SX(reg), BIT_ULL(vf));
140 }
141
142 mutex_unlock(&mbox->lock);
143 }
144
otx2_pf_flr_intr_handler(int irq,void * pf_irq)145 static irqreturn_t otx2_pf_flr_intr_handler(int irq, void *pf_irq)
146 {
147 struct otx2_nic *pf = (struct otx2_nic *)pf_irq;
148 int reg, dev, vf, start_vf, num_reg = 1;
149 u64 intr;
150
151 if (pf->total_vfs > 64)
152 num_reg = 2;
153
154 for (reg = 0; reg < num_reg; reg++) {
155 intr = otx2_read64(pf, RVU_PF_VFFLR_INTX(reg));
156 if (!intr)
157 continue;
158 start_vf = 64 * reg;
159 for (vf = 0; vf < 64; vf++) {
160 if (!(intr & BIT_ULL(vf)))
161 continue;
162 dev = vf + start_vf;
163 queue_work(pf->flr_wq, &pf->flr_wrk[dev].work);
164 /* Clear interrupt */
165 otx2_write64(pf, RVU_PF_VFFLR_INTX(reg), BIT_ULL(vf));
166 /* Disable the interrupt */
167 otx2_write64(pf, RVU_PF_VFFLR_INT_ENA_W1CX(reg),
168 BIT_ULL(vf));
169 }
170 }
171 return IRQ_HANDLED;
172 }
173
otx2_pf_me_intr_handler(int irq,void * pf_irq)174 static irqreturn_t otx2_pf_me_intr_handler(int irq, void *pf_irq)
175 {
176 struct otx2_nic *pf = (struct otx2_nic *)pf_irq;
177 int vf, reg, num_reg = 1;
178 u64 intr;
179
180 if (pf->total_vfs > 64)
181 num_reg = 2;
182
183 for (reg = 0; reg < num_reg; reg++) {
184 intr = otx2_read64(pf, RVU_PF_VFME_INTX(reg));
185 if (!intr)
186 continue;
187 for (vf = 0; vf < 64; vf++) {
188 if (!(intr & BIT_ULL(vf)))
189 continue;
190 /* clear trpend bit */
191 otx2_write64(pf, RVU_PF_VFTRPENDX(reg), BIT_ULL(vf));
192 /* clear interrupt */
193 otx2_write64(pf, RVU_PF_VFME_INTX(reg), BIT_ULL(vf));
194 }
195 }
196 return IRQ_HANDLED;
197 }
198
otx2_register_flr_me_intr(struct otx2_nic * pf,int numvfs)199 static int otx2_register_flr_me_intr(struct otx2_nic *pf, int numvfs)
200 {
201 struct otx2_hw *hw = &pf->hw;
202 char *irq_name;
203 int ret;
204
205 /* Register ME interrupt handler*/
206 irq_name = &hw->irq_name[RVU_PF_INT_VEC_VFME0 * NAME_SIZE];
207 snprintf(irq_name, NAME_SIZE, "RVUPF%d_ME0", rvu_get_pf(pf->pcifunc));
208 ret = request_irq(pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFME0),
209 otx2_pf_me_intr_handler, 0, irq_name, pf);
210 if (ret) {
211 dev_err(pf->dev,
212 "RVUPF: IRQ registration failed for ME0\n");
213 }
214
215 /* Register FLR interrupt handler */
216 irq_name = &hw->irq_name[RVU_PF_INT_VEC_VFFLR0 * NAME_SIZE];
217 snprintf(irq_name, NAME_SIZE, "RVUPF%d_FLR0", rvu_get_pf(pf->pcifunc));
218 ret = request_irq(pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFFLR0),
219 otx2_pf_flr_intr_handler, 0, irq_name, pf);
220 if (ret) {
221 dev_err(pf->dev,
222 "RVUPF: IRQ registration failed for FLR0\n");
223 return ret;
224 }
225
226 if (numvfs > 64) {
227 irq_name = &hw->irq_name[RVU_PF_INT_VEC_VFME1 * NAME_SIZE];
228 snprintf(irq_name, NAME_SIZE, "RVUPF%d_ME1",
229 rvu_get_pf(pf->pcifunc));
230 ret = request_irq(pci_irq_vector
231 (pf->pdev, RVU_PF_INT_VEC_VFME1),
232 otx2_pf_me_intr_handler, 0, irq_name, pf);
233 if (ret) {
234 dev_err(pf->dev,
235 "RVUPF: IRQ registration failed for ME1\n");
236 }
237 irq_name = &hw->irq_name[RVU_PF_INT_VEC_VFFLR1 * NAME_SIZE];
238 snprintf(irq_name, NAME_SIZE, "RVUPF%d_FLR1",
239 rvu_get_pf(pf->pcifunc));
240 ret = request_irq(pci_irq_vector
241 (pf->pdev, RVU_PF_INT_VEC_VFFLR1),
242 otx2_pf_flr_intr_handler, 0, irq_name, pf);
243 if (ret) {
244 dev_err(pf->dev,
245 "RVUPF: IRQ registration failed for FLR1\n");
246 return ret;
247 }
248 }
249
250 /* Enable ME interrupt for all VFs*/
251 otx2_write64(pf, RVU_PF_VFME_INTX(0), INTR_MASK(numvfs));
252 otx2_write64(pf, RVU_PF_VFME_INT_ENA_W1SX(0), INTR_MASK(numvfs));
253
254 /* Enable FLR interrupt for all VFs*/
255 otx2_write64(pf, RVU_PF_VFFLR_INTX(0), INTR_MASK(numvfs));
256 otx2_write64(pf, RVU_PF_VFFLR_INT_ENA_W1SX(0), INTR_MASK(numvfs));
257
258 if (numvfs > 64) {
259 numvfs -= 64;
260
261 otx2_write64(pf, RVU_PF_VFME_INTX(1), INTR_MASK(numvfs));
262 otx2_write64(pf, RVU_PF_VFME_INT_ENA_W1SX(1),
263 INTR_MASK(numvfs));
264
265 otx2_write64(pf, RVU_PF_VFFLR_INTX(1), INTR_MASK(numvfs));
266 otx2_write64(pf, RVU_PF_VFFLR_INT_ENA_W1SX(1),
267 INTR_MASK(numvfs));
268 }
269 return 0;
270 }
271
otx2_pf_flr_init(struct otx2_nic * pf,int num_vfs)272 static int otx2_pf_flr_init(struct otx2_nic *pf, int num_vfs)
273 {
274 int vf;
275
276 pf->flr_wq = alloc_ordered_workqueue("otx2_pf_flr_wq", WQ_HIGHPRI);
277 if (!pf->flr_wq)
278 return -ENOMEM;
279
280 pf->flr_wrk = devm_kcalloc(pf->dev, num_vfs,
281 sizeof(struct flr_work), GFP_KERNEL);
282 if (!pf->flr_wrk) {
283 destroy_workqueue(pf->flr_wq);
284 return -ENOMEM;
285 }
286
287 for (vf = 0; vf < num_vfs; vf++) {
288 pf->flr_wrk[vf].pf = pf;
289 INIT_WORK(&pf->flr_wrk[vf].work, otx2_flr_handler);
290 }
291
292 return 0;
293 }
294
otx2_queue_work(struct mbox * mw,struct workqueue_struct * mbox_wq,int first,int mdevs,u64 intr,int type)295 static void otx2_queue_work(struct mbox *mw, struct workqueue_struct *mbox_wq,
296 int first, int mdevs, u64 intr, int type)
297 {
298 struct otx2_mbox_dev *mdev;
299 struct otx2_mbox *mbox;
300 struct mbox_hdr *hdr;
301 int i;
302
303 for (i = first; i < mdevs; i++) {
304 /* start from 0 */
305 if (!(intr & BIT_ULL(i - first)))
306 continue;
307
308 mbox = &mw->mbox;
309 mdev = &mbox->dev[i];
310 if (type == TYPE_PFAF)
311 otx2_sync_mbox_bbuf(mbox, i);
312 hdr = mdev->mbase + mbox->rx_start;
313 /* The hdr->num_msgs is set to zero immediately in the interrupt
314 * handler to ensure that it holds a correct value next time
315 * when the interrupt handler is called.
316 * pf->mbox.num_msgs holds the data for use in pfaf_mbox_handler
317 * pf>mbox.up_num_msgs holds the data for use in
318 * pfaf_mbox_up_handler.
319 */
320 if (hdr->num_msgs) {
321 mw[i].num_msgs = hdr->num_msgs;
322 hdr->num_msgs = 0;
323 if (type == TYPE_PFAF)
324 memset(mbox->hwbase + mbox->rx_start, 0,
325 ALIGN(sizeof(struct mbox_hdr),
326 sizeof(u64)));
327
328 queue_work(mbox_wq, &mw[i].mbox_wrk);
329 }
330
331 mbox = &mw->mbox_up;
332 mdev = &mbox->dev[i];
333 if (type == TYPE_PFAF)
334 otx2_sync_mbox_bbuf(mbox, i);
335 hdr = mdev->mbase + mbox->rx_start;
336 if (hdr->num_msgs) {
337 mw[i].up_num_msgs = hdr->num_msgs;
338 hdr->num_msgs = 0;
339 if (type == TYPE_PFAF)
340 memset(mbox->hwbase + mbox->rx_start, 0,
341 ALIGN(sizeof(struct mbox_hdr),
342 sizeof(u64)));
343
344 queue_work(mbox_wq, &mw[i].mbox_up_wrk);
345 }
346 }
347 }
348
otx2_forward_msg_pfvf(struct otx2_mbox_dev * mdev,struct otx2_mbox * pfvf_mbox,void * bbuf_base,int devid)349 static void otx2_forward_msg_pfvf(struct otx2_mbox_dev *mdev,
350 struct otx2_mbox *pfvf_mbox, void *bbuf_base,
351 int devid)
352 {
353 struct otx2_mbox_dev *src_mdev = mdev;
354 int offset;
355
356 /* Msgs are already copied, trigger VF's mbox irq */
357 smp_wmb();
358
359 offset = pfvf_mbox->trigger | (devid << pfvf_mbox->tr_shift);
360 writeq(1, (void __iomem *)pfvf_mbox->reg_base + offset);
361
362 /* Restore VF's mbox bounce buffer region address */
363 src_mdev->mbase = bbuf_base;
364 }
365
otx2_forward_vf_mbox_msgs(struct otx2_nic * pf,struct otx2_mbox * src_mbox,int dir,int vf,int num_msgs)366 static int otx2_forward_vf_mbox_msgs(struct otx2_nic *pf,
367 struct otx2_mbox *src_mbox,
368 int dir, int vf, int num_msgs)
369 {
370 struct otx2_mbox_dev *src_mdev, *dst_mdev;
371 struct mbox_hdr *mbox_hdr;
372 struct mbox_hdr *req_hdr;
373 struct mbox *dst_mbox;
374 int dst_size, err;
375
376 if (dir == MBOX_DIR_PFAF) {
377 /* Set VF's mailbox memory as PF's bounce buffer memory, so
378 * that explicit copying of VF's msgs to PF=>AF mbox region
379 * and AF=>PF responses to VF's mbox region can be avoided.
380 */
381 src_mdev = &src_mbox->dev[vf];
382 mbox_hdr = src_mbox->hwbase +
383 src_mbox->rx_start + (vf * MBOX_SIZE);
384
385 dst_mbox = &pf->mbox;
386 dst_size = dst_mbox->mbox.tx_size -
387 ALIGN(sizeof(*mbox_hdr), MBOX_MSG_ALIGN);
388 /* Check if msgs fit into destination area and has valid size */
389 if (mbox_hdr->msg_size > dst_size || !mbox_hdr->msg_size)
390 return -EINVAL;
391
392 dst_mdev = &dst_mbox->mbox.dev[0];
393
394 mutex_lock(&pf->mbox.lock);
395 dst_mdev->mbase = src_mdev->mbase;
396 dst_mdev->msg_size = mbox_hdr->msg_size;
397 dst_mdev->num_msgs = num_msgs;
398 err = otx2_sync_mbox_msg(dst_mbox);
399 /* Error code -EIO indicate there is a communication failure
400 * to the AF. Rest of the error codes indicate that AF processed
401 * VF messages and set the error codes in response messages
402 * (if any) so simply forward responses to VF.
403 */
404 if (err == -EIO) {
405 dev_warn(pf->dev,
406 "AF not responding to VF%d messages\n", vf);
407 /* restore PF mbase and exit */
408 dst_mdev->mbase = pf->mbox.bbuf_base;
409 mutex_unlock(&pf->mbox.lock);
410 return err;
411 }
412 /* At this point, all the VF messages sent to AF are acked
413 * with proper responses and responses are copied to VF
414 * mailbox hence raise interrupt to VF.
415 */
416 req_hdr = (struct mbox_hdr *)(dst_mdev->mbase +
417 dst_mbox->mbox.rx_start);
418 req_hdr->num_msgs = num_msgs;
419
420 otx2_forward_msg_pfvf(dst_mdev, &pf->mbox_pfvf[0].mbox,
421 pf->mbox.bbuf_base, vf);
422 mutex_unlock(&pf->mbox.lock);
423 } else if (dir == MBOX_DIR_PFVF_UP) {
424 src_mdev = &src_mbox->dev[0];
425 mbox_hdr = src_mbox->hwbase + src_mbox->rx_start;
426 req_hdr = (struct mbox_hdr *)(src_mdev->mbase +
427 src_mbox->rx_start);
428 req_hdr->num_msgs = num_msgs;
429
430 dst_mbox = &pf->mbox_pfvf[0];
431 dst_size = dst_mbox->mbox_up.tx_size -
432 ALIGN(sizeof(*mbox_hdr), MBOX_MSG_ALIGN);
433 /* Check if msgs fit into destination area */
434 if (mbox_hdr->msg_size > dst_size)
435 return -EINVAL;
436
437 dst_mdev = &dst_mbox->mbox_up.dev[vf];
438 dst_mdev->mbase = src_mdev->mbase;
439 dst_mdev->msg_size = mbox_hdr->msg_size;
440 dst_mdev->num_msgs = mbox_hdr->num_msgs;
441 err = otx2_sync_mbox_up_msg(dst_mbox, vf);
442 if (err) {
443 dev_warn(pf->dev,
444 "VF%d is not responding to mailbox\n", vf);
445 return err;
446 }
447 } else if (dir == MBOX_DIR_VFPF_UP) {
448 req_hdr = (struct mbox_hdr *)(src_mbox->dev[0].mbase +
449 src_mbox->rx_start);
450 req_hdr->num_msgs = num_msgs;
451 otx2_forward_msg_pfvf(&pf->mbox_pfvf->mbox_up.dev[vf],
452 &pf->mbox.mbox_up,
453 pf->mbox_pfvf[vf].bbuf_base,
454 0);
455 }
456
457 return 0;
458 }
459
otx2_pfvf_mbox_handler(struct work_struct * work)460 static void otx2_pfvf_mbox_handler(struct work_struct *work)
461 {
462 struct mbox_msghdr *msg = NULL;
463 int offset, vf_idx, id, err;
464 struct otx2_mbox_dev *mdev;
465 struct mbox_hdr *req_hdr;
466 struct otx2_mbox *mbox;
467 struct mbox *vf_mbox;
468 struct otx2_nic *pf;
469
470 vf_mbox = container_of(work, struct mbox, mbox_wrk);
471 pf = vf_mbox->pfvf;
472 vf_idx = vf_mbox - pf->mbox_pfvf;
473
474 mbox = &pf->mbox_pfvf[0].mbox;
475 mdev = &mbox->dev[vf_idx];
476 req_hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start);
477
478 offset = ALIGN(sizeof(*req_hdr), MBOX_MSG_ALIGN);
479
480 for (id = 0; id < vf_mbox->num_msgs; id++) {
481 msg = (struct mbox_msghdr *)(mdev->mbase + mbox->rx_start +
482 offset);
483
484 if (msg->sig != OTX2_MBOX_REQ_SIG)
485 goto inval_msg;
486
487 /* Set VF's number in each of the msg */
488 msg->pcifunc &= RVU_PFVF_FUNC_MASK;
489 msg->pcifunc |= (vf_idx + 1) & RVU_PFVF_FUNC_MASK;
490 offset = msg->next_msgoff;
491 }
492 err = otx2_forward_vf_mbox_msgs(pf, mbox, MBOX_DIR_PFAF, vf_idx,
493 vf_mbox->num_msgs);
494 if (err)
495 goto inval_msg;
496 return;
497
498 inval_msg:
499 otx2_reply_invalid_msg(mbox, vf_idx, 0, msg->id);
500 otx2_mbox_msg_send(mbox, vf_idx);
501 }
502
otx2_pfvf_mbox_up_handler(struct work_struct * work)503 static void otx2_pfvf_mbox_up_handler(struct work_struct *work)
504 {
505 struct mbox *vf_mbox = container_of(work, struct mbox, mbox_up_wrk);
506 struct otx2_nic *pf = vf_mbox->pfvf;
507 struct otx2_mbox_dev *mdev;
508 int offset, id, vf_idx = 0;
509 struct mbox_hdr *rsp_hdr;
510 struct mbox_msghdr *msg;
511 struct otx2_mbox *mbox;
512
513 vf_idx = vf_mbox - pf->mbox_pfvf;
514 mbox = &pf->mbox_pfvf[0].mbox_up;
515 mdev = &mbox->dev[vf_idx];
516
517 rsp_hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start);
518 offset = mbox->rx_start + ALIGN(sizeof(*rsp_hdr), MBOX_MSG_ALIGN);
519
520 for (id = 0; id < vf_mbox->up_num_msgs; id++) {
521 msg = mdev->mbase + offset;
522
523 if (msg->id >= MBOX_MSG_MAX) {
524 dev_err(pf->dev,
525 "Mbox msg with unknown ID 0x%x\n", msg->id);
526 goto end;
527 }
528
529 if (msg->sig != OTX2_MBOX_RSP_SIG) {
530 dev_err(pf->dev,
531 "Mbox msg with wrong signature %x, ID 0x%x\n",
532 msg->sig, msg->id);
533 goto end;
534 }
535
536 switch (msg->id) {
537 case MBOX_MSG_CGX_LINK_EVENT:
538 break;
539 default:
540 if (msg->rc)
541 dev_err(pf->dev,
542 "Mbox msg response has err %d, ID 0x%x\n",
543 msg->rc, msg->id);
544 break;
545 }
546
547 end:
548 offset = mbox->rx_start + msg->next_msgoff;
549 if (mdev->msgs_acked == (vf_mbox->up_num_msgs - 1))
550 __otx2_mbox_reset(mbox, 0);
551 mdev->msgs_acked++;
552 }
553 }
554
otx2_pfvf_mbox_intr_handler(int irq,void * pf_irq)555 static irqreturn_t otx2_pfvf_mbox_intr_handler(int irq, void *pf_irq)
556 {
557 struct otx2_nic *pf = (struct otx2_nic *)(pf_irq);
558 int vfs = pf->total_vfs;
559 struct mbox *mbox;
560 u64 intr;
561
562 mbox = pf->mbox_pfvf;
563 /* Handle VF interrupts */
564 if (vfs > 64) {
565 intr = otx2_read64(pf, RVU_PF_VFPF_MBOX_INTX(1));
566 otx2_write64(pf, RVU_PF_VFPF_MBOX_INTX(1), intr);
567 otx2_queue_work(mbox, pf->mbox_pfvf_wq, 64, vfs, intr,
568 TYPE_PFVF);
569 if (intr)
570 trace_otx2_msg_interrupt(mbox->mbox.pdev, "VF(s) to PF", intr);
571 vfs = 64;
572 }
573
574 intr = otx2_read64(pf, RVU_PF_VFPF_MBOX_INTX(0));
575 otx2_write64(pf, RVU_PF_VFPF_MBOX_INTX(0), intr);
576
577 otx2_queue_work(mbox, pf->mbox_pfvf_wq, 0, vfs, intr, TYPE_PFVF);
578
579 if (intr)
580 trace_otx2_msg_interrupt(mbox->mbox.pdev, "VF(s) to PF", intr);
581
582 return IRQ_HANDLED;
583 }
584
otx2_pfvf_mbox_init(struct otx2_nic * pf,int numvfs)585 static int otx2_pfvf_mbox_init(struct otx2_nic *pf, int numvfs)
586 {
587 void __iomem *hwbase;
588 struct mbox *mbox;
589 int err, vf;
590 u64 base;
591
592 if (!numvfs)
593 return -EINVAL;
594
595 pf->mbox_pfvf = devm_kcalloc(&pf->pdev->dev, numvfs,
596 sizeof(struct mbox), GFP_KERNEL);
597 if (!pf->mbox_pfvf)
598 return -ENOMEM;
599
600 pf->mbox_pfvf_wq = alloc_ordered_workqueue("otx2_pfvf_mailbox",
601 WQ_HIGHPRI | WQ_MEM_RECLAIM);
602 if (!pf->mbox_pfvf_wq)
603 return -ENOMEM;
604
605 /* On CN10K platform, PF <-> VF mailbox region follows after
606 * PF <-> AF mailbox region.
607 */
608 if (test_bit(CN10K_MBOX, &pf->hw.cap_flag))
609 base = pci_resource_start(pf->pdev, PCI_MBOX_BAR_NUM) +
610 MBOX_SIZE;
611 else
612 base = readq((void __iomem *)((u64)pf->reg_base +
613 RVU_PF_VF_BAR4_ADDR));
614
615 hwbase = ioremap_wc(base, MBOX_SIZE * pf->total_vfs);
616 if (!hwbase) {
617 err = -ENOMEM;
618 goto free_wq;
619 }
620
621 mbox = &pf->mbox_pfvf[0];
622 err = otx2_mbox_init(&mbox->mbox, hwbase, pf->pdev, pf->reg_base,
623 MBOX_DIR_PFVF, numvfs);
624 if (err)
625 goto free_iomem;
626
627 err = otx2_mbox_init(&mbox->mbox_up, hwbase, pf->pdev, pf->reg_base,
628 MBOX_DIR_PFVF_UP, numvfs);
629 if (err)
630 goto free_iomem;
631
632 for (vf = 0; vf < numvfs; vf++) {
633 mbox->pfvf = pf;
634 INIT_WORK(&mbox->mbox_wrk, otx2_pfvf_mbox_handler);
635 INIT_WORK(&mbox->mbox_up_wrk, otx2_pfvf_mbox_up_handler);
636 mbox++;
637 }
638
639 return 0;
640
641 free_iomem:
642 if (hwbase)
643 iounmap(hwbase);
644 free_wq:
645 destroy_workqueue(pf->mbox_pfvf_wq);
646 return err;
647 }
648
otx2_pfvf_mbox_destroy(struct otx2_nic * pf)649 static void otx2_pfvf_mbox_destroy(struct otx2_nic *pf)
650 {
651 struct mbox *mbox = &pf->mbox_pfvf[0];
652
653 if (!mbox)
654 return;
655
656 if (pf->mbox_pfvf_wq) {
657 destroy_workqueue(pf->mbox_pfvf_wq);
658 pf->mbox_pfvf_wq = NULL;
659 }
660
661 if (mbox->mbox.hwbase)
662 iounmap(mbox->mbox.hwbase);
663
664 otx2_mbox_destroy(&mbox->mbox);
665 }
666
otx2_enable_pfvf_mbox_intr(struct otx2_nic * pf,int numvfs)667 static void otx2_enable_pfvf_mbox_intr(struct otx2_nic *pf, int numvfs)
668 {
669 /* Clear PF <=> VF mailbox IRQ */
670 otx2_write64(pf, RVU_PF_VFPF_MBOX_INTX(0), ~0ull);
671 otx2_write64(pf, RVU_PF_VFPF_MBOX_INTX(1), ~0ull);
672
673 /* Enable PF <=> VF mailbox IRQ */
674 otx2_write64(pf, RVU_PF_VFPF_MBOX_INT_ENA_W1SX(0), INTR_MASK(numvfs));
675 if (numvfs > 64) {
676 numvfs -= 64;
677 otx2_write64(pf, RVU_PF_VFPF_MBOX_INT_ENA_W1SX(1),
678 INTR_MASK(numvfs));
679 }
680 }
681
otx2_disable_pfvf_mbox_intr(struct otx2_nic * pf,int numvfs)682 static void otx2_disable_pfvf_mbox_intr(struct otx2_nic *pf, int numvfs)
683 {
684 int vector;
685
686 /* Disable PF <=> VF mailbox IRQ */
687 otx2_write64(pf, RVU_PF_VFPF_MBOX_INT_ENA_W1CX(0), ~0ull);
688 otx2_write64(pf, RVU_PF_VFPF_MBOX_INT_ENA_W1CX(1), ~0ull);
689
690 otx2_write64(pf, RVU_PF_VFPF_MBOX_INTX(0), ~0ull);
691 vector = pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFPF_MBOX0);
692 free_irq(vector, pf);
693
694 if (numvfs > 64) {
695 otx2_write64(pf, RVU_PF_VFPF_MBOX_INTX(1), ~0ull);
696 vector = pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFPF_MBOX1);
697 free_irq(vector, pf);
698 }
699 }
700
otx2_register_pfvf_mbox_intr(struct otx2_nic * pf,int numvfs)701 static int otx2_register_pfvf_mbox_intr(struct otx2_nic *pf, int numvfs)
702 {
703 struct otx2_hw *hw = &pf->hw;
704 char *irq_name;
705 int err;
706
707 /* Register MBOX0 interrupt handler */
708 irq_name = &hw->irq_name[RVU_PF_INT_VEC_VFPF_MBOX0 * NAME_SIZE];
709 if (pf->pcifunc)
710 snprintf(irq_name, NAME_SIZE,
711 "RVUPF%d_VF Mbox0", rvu_get_pf(pf->pcifunc));
712 else
713 snprintf(irq_name, NAME_SIZE, "RVUPF_VF Mbox0");
714 err = request_irq(pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_VFPF_MBOX0),
715 otx2_pfvf_mbox_intr_handler, 0, irq_name, pf);
716 if (err) {
717 dev_err(pf->dev,
718 "RVUPF: IRQ registration failed for PFVF mbox0 irq\n");
719 return err;
720 }
721
722 if (numvfs > 64) {
723 /* Register MBOX1 interrupt handler */
724 irq_name = &hw->irq_name[RVU_PF_INT_VEC_VFPF_MBOX1 * NAME_SIZE];
725 if (pf->pcifunc)
726 snprintf(irq_name, NAME_SIZE,
727 "RVUPF%d_VF Mbox1", rvu_get_pf(pf->pcifunc));
728 else
729 snprintf(irq_name, NAME_SIZE, "RVUPF_VF Mbox1");
730 err = request_irq(pci_irq_vector(pf->pdev,
731 RVU_PF_INT_VEC_VFPF_MBOX1),
732 otx2_pfvf_mbox_intr_handler,
733 0, irq_name, pf);
734 if (err) {
735 dev_err(pf->dev,
736 "RVUPF: IRQ registration failed for PFVF mbox1 irq\n");
737 return err;
738 }
739 }
740
741 otx2_enable_pfvf_mbox_intr(pf, numvfs);
742
743 return 0;
744 }
745
otx2_process_pfaf_mbox_msg(struct otx2_nic * pf,struct mbox_msghdr * msg)746 static void otx2_process_pfaf_mbox_msg(struct otx2_nic *pf,
747 struct mbox_msghdr *msg)
748 {
749 int devid;
750
751 if (msg->id >= MBOX_MSG_MAX) {
752 dev_err(pf->dev,
753 "Mbox msg with unknown ID 0x%x\n", msg->id);
754 return;
755 }
756
757 if (msg->sig != OTX2_MBOX_RSP_SIG) {
758 dev_err(pf->dev,
759 "Mbox msg with wrong signature %x, ID 0x%x\n",
760 msg->sig, msg->id);
761 return;
762 }
763
764 /* message response heading VF */
765 devid = msg->pcifunc & RVU_PFVF_FUNC_MASK;
766 if (devid) {
767 struct otx2_vf_config *config = &pf->vf_configs[devid - 1];
768 struct delayed_work *dwork;
769
770 switch (msg->id) {
771 case MBOX_MSG_NIX_LF_START_RX:
772 config->intf_down = false;
773 dwork = &config->link_event_work;
774 schedule_delayed_work(dwork, msecs_to_jiffies(100));
775 break;
776 case MBOX_MSG_NIX_LF_STOP_RX:
777 config->intf_down = true;
778 break;
779 }
780
781 return;
782 }
783
784 switch (msg->id) {
785 case MBOX_MSG_READY:
786 pf->pcifunc = msg->pcifunc;
787 break;
788 case MBOX_MSG_MSIX_OFFSET:
789 mbox_handler_msix_offset(pf, (struct msix_offset_rsp *)msg);
790 break;
791 case MBOX_MSG_NPA_LF_ALLOC:
792 mbox_handler_npa_lf_alloc(pf, (struct npa_lf_alloc_rsp *)msg);
793 break;
794 case MBOX_MSG_NIX_LF_ALLOC:
795 mbox_handler_nix_lf_alloc(pf, (struct nix_lf_alloc_rsp *)msg);
796 break;
797 case MBOX_MSG_NIX_BP_ENABLE:
798 mbox_handler_nix_bp_enable(pf, (struct nix_bp_cfg_rsp *)msg);
799 break;
800 case MBOX_MSG_CGX_STATS:
801 mbox_handler_cgx_stats(pf, (struct cgx_stats_rsp *)msg);
802 break;
803 case MBOX_MSG_CGX_FEC_STATS:
804 mbox_handler_cgx_fec_stats(pf, (struct cgx_fec_stats_rsp *)msg);
805 break;
806 default:
807 if (msg->rc)
808 dev_err(pf->dev,
809 "Mbox msg response has err %d, ID 0x%x\n",
810 msg->rc, msg->id);
811 break;
812 }
813 }
814
otx2_pfaf_mbox_handler(struct work_struct * work)815 static void otx2_pfaf_mbox_handler(struct work_struct *work)
816 {
817 struct otx2_mbox_dev *mdev;
818 struct mbox_hdr *rsp_hdr;
819 struct mbox_msghdr *msg;
820 struct otx2_mbox *mbox;
821 struct mbox *af_mbox;
822 struct otx2_nic *pf;
823 int offset, id;
824
825 af_mbox = container_of(work, struct mbox, mbox_wrk);
826 mbox = &af_mbox->mbox;
827 mdev = &mbox->dev[0];
828 rsp_hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start);
829
830 offset = mbox->rx_start + ALIGN(sizeof(*rsp_hdr), MBOX_MSG_ALIGN);
831 pf = af_mbox->pfvf;
832
833 for (id = 0; id < af_mbox->num_msgs; id++) {
834 msg = (struct mbox_msghdr *)(mdev->mbase + offset);
835 otx2_process_pfaf_mbox_msg(pf, msg);
836 offset = mbox->rx_start + msg->next_msgoff;
837 if (mdev->msgs_acked == (af_mbox->num_msgs - 1))
838 __otx2_mbox_reset(mbox, 0);
839 mdev->msgs_acked++;
840 }
841
842 }
843
otx2_handle_link_event(struct otx2_nic * pf)844 static void otx2_handle_link_event(struct otx2_nic *pf)
845 {
846 struct cgx_link_user_info *linfo = &pf->linfo;
847 struct net_device *netdev = pf->netdev;
848
849 pr_info("%s NIC Link is %s %d Mbps %s duplex\n", netdev->name,
850 linfo->link_up ? "UP" : "DOWN", linfo->speed,
851 linfo->full_duplex ? "Full" : "Half");
852 if (linfo->link_up) {
853 netif_carrier_on(netdev);
854 netif_tx_start_all_queues(netdev);
855 } else {
856 netif_tx_stop_all_queues(netdev);
857 netif_carrier_off(netdev);
858 }
859 }
860
otx2_mbox_up_handler_mcs_intr_notify(struct otx2_nic * pf,struct mcs_intr_info * event,struct msg_rsp * rsp)861 int otx2_mbox_up_handler_mcs_intr_notify(struct otx2_nic *pf,
862 struct mcs_intr_info *event,
863 struct msg_rsp *rsp)
864 {
865 cn10k_handle_mcs_event(pf, event);
866
867 return 0;
868 }
869
otx2_mbox_up_handler_cgx_link_event(struct otx2_nic * pf,struct cgx_link_info_msg * msg,struct msg_rsp * rsp)870 int otx2_mbox_up_handler_cgx_link_event(struct otx2_nic *pf,
871 struct cgx_link_info_msg *msg,
872 struct msg_rsp *rsp)
873 {
874 int i;
875
876 /* Copy the link info sent by AF */
877 pf->linfo = msg->link_info;
878
879 /* notify VFs about link event */
880 for (i = 0; i < pci_num_vf(pf->pdev); i++) {
881 struct otx2_vf_config *config = &pf->vf_configs[i];
882 struct delayed_work *dwork = &config->link_event_work;
883
884 if (config->intf_down)
885 continue;
886
887 schedule_delayed_work(dwork, msecs_to_jiffies(100));
888 }
889
890 /* interface has not been fully configured yet */
891 if (pf->flags & OTX2_FLAG_INTF_DOWN)
892 return 0;
893
894 otx2_handle_link_event(pf);
895 return 0;
896 }
897
otx2_process_mbox_msg_up(struct otx2_nic * pf,struct mbox_msghdr * req)898 static int otx2_process_mbox_msg_up(struct otx2_nic *pf,
899 struct mbox_msghdr *req)
900 {
901 /* Check if valid, if not reply with a invalid msg */
902 if (req->sig != OTX2_MBOX_REQ_SIG) {
903 otx2_reply_invalid_msg(&pf->mbox.mbox_up, 0, 0, req->id);
904 return -ENODEV;
905 }
906
907 switch (req->id) {
908 #define M(_name, _id, _fn_name, _req_type, _rsp_type) \
909 case _id: { \
910 struct _rsp_type *rsp; \
911 int err; \
912 \
913 rsp = (struct _rsp_type *)otx2_mbox_alloc_msg( \
914 &pf->mbox.mbox_up, 0, \
915 sizeof(struct _rsp_type)); \
916 if (!rsp) \
917 return -ENOMEM; \
918 \
919 rsp->hdr.id = _id; \
920 rsp->hdr.sig = OTX2_MBOX_RSP_SIG; \
921 rsp->hdr.pcifunc = 0; \
922 rsp->hdr.rc = 0; \
923 \
924 err = otx2_mbox_up_handler_ ## _fn_name( \
925 pf, (struct _req_type *)req, rsp); \
926 return err; \
927 }
928 MBOX_UP_CGX_MESSAGES
929 MBOX_UP_MCS_MESSAGES
930 #undef M
931 break;
932 default:
933 otx2_reply_invalid_msg(&pf->mbox.mbox_up, 0, 0, req->id);
934 return -ENODEV;
935 }
936 return 0;
937 }
938
otx2_pfaf_mbox_up_handler(struct work_struct * work)939 static void otx2_pfaf_mbox_up_handler(struct work_struct *work)
940 {
941 struct mbox *af_mbox = container_of(work, struct mbox, mbox_up_wrk);
942 struct otx2_mbox *mbox = &af_mbox->mbox_up;
943 struct otx2_mbox_dev *mdev = &mbox->dev[0];
944 struct otx2_nic *pf = af_mbox->pfvf;
945 int offset, id, devid = 0;
946 struct mbox_hdr *rsp_hdr;
947 struct mbox_msghdr *msg;
948
949 rsp_hdr = (struct mbox_hdr *)(mdev->mbase + mbox->rx_start);
950
951 offset = mbox->rx_start + ALIGN(sizeof(*rsp_hdr), MBOX_MSG_ALIGN);
952
953 for (id = 0; id < af_mbox->up_num_msgs; id++) {
954 msg = (struct mbox_msghdr *)(mdev->mbase + offset);
955
956 devid = msg->pcifunc & RVU_PFVF_FUNC_MASK;
957 /* Skip processing VF's messages */
958 if (!devid)
959 otx2_process_mbox_msg_up(pf, msg);
960 offset = mbox->rx_start + msg->next_msgoff;
961 }
962 if (devid) {
963 otx2_forward_vf_mbox_msgs(pf, &pf->mbox.mbox_up,
964 MBOX_DIR_PFVF_UP, devid - 1,
965 af_mbox->up_num_msgs);
966 return;
967 }
968
969 otx2_mbox_msg_send(mbox, 0);
970 }
971
otx2_pfaf_mbox_intr_handler(int irq,void * pf_irq)972 static irqreturn_t otx2_pfaf_mbox_intr_handler(int irq, void *pf_irq)
973 {
974 struct otx2_nic *pf = (struct otx2_nic *)pf_irq;
975 struct mbox *mbox;
976
977 /* Clear the IRQ */
978 otx2_write64(pf, RVU_PF_INT, BIT_ULL(0));
979
980 mbox = &pf->mbox;
981
982 trace_otx2_msg_interrupt(mbox->mbox.pdev, "AF to PF", BIT_ULL(0));
983
984 otx2_queue_work(mbox, pf->mbox_wq, 0, 1, 1, TYPE_PFAF);
985
986 return IRQ_HANDLED;
987 }
988
otx2_disable_mbox_intr(struct otx2_nic * pf)989 static void otx2_disable_mbox_intr(struct otx2_nic *pf)
990 {
991 int vector = pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_AFPF_MBOX);
992
993 /* Disable AF => PF mailbox IRQ */
994 otx2_write64(pf, RVU_PF_INT_ENA_W1C, BIT_ULL(0));
995 free_irq(vector, pf);
996 }
997
otx2_register_mbox_intr(struct otx2_nic * pf,bool probe_af)998 static int otx2_register_mbox_intr(struct otx2_nic *pf, bool probe_af)
999 {
1000 struct otx2_hw *hw = &pf->hw;
1001 struct msg_req *req;
1002 char *irq_name;
1003 int err;
1004
1005 /* Register mailbox interrupt handler */
1006 irq_name = &hw->irq_name[RVU_PF_INT_VEC_AFPF_MBOX * NAME_SIZE];
1007 snprintf(irq_name, NAME_SIZE, "RVUPFAF Mbox");
1008 err = request_irq(pci_irq_vector(pf->pdev, RVU_PF_INT_VEC_AFPF_MBOX),
1009 otx2_pfaf_mbox_intr_handler, 0, irq_name, pf);
1010 if (err) {
1011 dev_err(pf->dev,
1012 "RVUPF: IRQ registration failed for PFAF mbox irq\n");
1013 return err;
1014 }
1015
1016 /* Enable mailbox interrupt for msgs coming from AF.
1017 * First clear to avoid spurious interrupts, if any.
1018 */
1019 otx2_write64(pf, RVU_PF_INT, BIT_ULL(0));
1020 otx2_write64(pf, RVU_PF_INT_ENA_W1S, BIT_ULL(0));
1021
1022 if (!probe_af)
1023 return 0;
1024
1025 /* Check mailbox communication with AF */
1026 req = otx2_mbox_alloc_msg_ready(&pf->mbox);
1027 if (!req) {
1028 otx2_disable_mbox_intr(pf);
1029 return -ENOMEM;
1030 }
1031 err = otx2_sync_mbox_msg(&pf->mbox);
1032 if (err) {
1033 dev_warn(pf->dev,
1034 "AF not responding to mailbox, deferring probe\n");
1035 otx2_disable_mbox_intr(pf);
1036 return -EPROBE_DEFER;
1037 }
1038
1039 return 0;
1040 }
1041
otx2_pfaf_mbox_destroy(struct otx2_nic * pf)1042 static void otx2_pfaf_mbox_destroy(struct otx2_nic *pf)
1043 {
1044 struct mbox *mbox = &pf->mbox;
1045
1046 if (pf->mbox_wq) {
1047 destroy_workqueue(pf->mbox_wq);
1048 pf->mbox_wq = NULL;
1049 }
1050
1051 if (mbox->mbox.hwbase)
1052 iounmap((void __iomem *)mbox->mbox.hwbase);
1053
1054 otx2_mbox_destroy(&mbox->mbox);
1055 otx2_mbox_destroy(&mbox->mbox_up);
1056 }
1057
otx2_pfaf_mbox_init(struct otx2_nic * pf)1058 static int otx2_pfaf_mbox_init(struct otx2_nic *pf)
1059 {
1060 struct mbox *mbox = &pf->mbox;
1061 void __iomem *hwbase;
1062 int err;
1063
1064 mbox->pfvf = pf;
1065 pf->mbox_wq = alloc_ordered_workqueue("otx2_pfaf_mailbox",
1066 WQ_HIGHPRI | WQ_MEM_RECLAIM);
1067 if (!pf->mbox_wq)
1068 return -ENOMEM;
1069
1070 /* Mailbox is a reserved memory (in RAM) region shared between
1071 * admin function (i.e AF) and this PF, shouldn't be mapped as
1072 * device memory to allow unaligned accesses.
1073 */
1074 hwbase = ioremap_wc(pci_resource_start(pf->pdev, PCI_MBOX_BAR_NUM),
1075 MBOX_SIZE);
1076 if (!hwbase) {
1077 dev_err(pf->dev, "Unable to map PFAF mailbox region\n");
1078 err = -ENOMEM;
1079 goto exit;
1080 }
1081
1082 err = otx2_mbox_init(&mbox->mbox, hwbase, pf->pdev, pf->reg_base,
1083 MBOX_DIR_PFAF, 1);
1084 if (err)
1085 goto exit;
1086
1087 err = otx2_mbox_init(&mbox->mbox_up, hwbase, pf->pdev, pf->reg_base,
1088 MBOX_DIR_PFAF_UP, 1);
1089 if (err)
1090 goto exit;
1091
1092 err = otx2_mbox_bbuf_init(mbox, pf->pdev);
1093 if (err)
1094 goto exit;
1095
1096 INIT_WORK(&mbox->mbox_wrk, otx2_pfaf_mbox_handler);
1097 INIT_WORK(&mbox->mbox_up_wrk, otx2_pfaf_mbox_up_handler);
1098 mutex_init(&mbox->lock);
1099
1100 return 0;
1101 exit:
1102 otx2_pfaf_mbox_destroy(pf);
1103 return err;
1104 }
1105
otx2_cgx_config_linkevents(struct otx2_nic * pf,bool enable)1106 static int otx2_cgx_config_linkevents(struct otx2_nic *pf, bool enable)
1107 {
1108 struct msg_req *msg;
1109 int err;
1110
1111 mutex_lock(&pf->mbox.lock);
1112 if (enable)
1113 msg = otx2_mbox_alloc_msg_cgx_start_linkevents(&pf->mbox);
1114 else
1115 msg = otx2_mbox_alloc_msg_cgx_stop_linkevents(&pf->mbox);
1116
1117 if (!msg) {
1118 mutex_unlock(&pf->mbox.lock);
1119 return -ENOMEM;
1120 }
1121
1122 err = otx2_sync_mbox_msg(&pf->mbox);
1123 mutex_unlock(&pf->mbox.lock);
1124 return err;
1125 }
1126
otx2_cgx_config_loopback(struct otx2_nic * pf,bool enable)1127 static int otx2_cgx_config_loopback(struct otx2_nic *pf, bool enable)
1128 {
1129 struct msg_req *msg;
1130 int err;
1131
1132 if (enable && !bitmap_empty(pf->flow_cfg->dmacflt_bmap,
1133 pf->flow_cfg->dmacflt_max_flows))
1134 netdev_warn(pf->netdev,
1135 "CGX/RPM internal loopback might not work as DMAC filters are active\n");
1136
1137 mutex_lock(&pf->mbox.lock);
1138 if (enable)
1139 msg = otx2_mbox_alloc_msg_cgx_intlbk_enable(&pf->mbox);
1140 else
1141 msg = otx2_mbox_alloc_msg_cgx_intlbk_disable(&pf->mbox);
1142
1143 if (!msg) {
1144 mutex_unlock(&pf->mbox.lock);
1145 return -ENOMEM;
1146 }
1147
1148 err = otx2_sync_mbox_msg(&pf->mbox);
1149 mutex_unlock(&pf->mbox.lock);
1150 return err;
1151 }
1152
otx2_set_real_num_queues(struct net_device * netdev,int tx_queues,int rx_queues)1153 int otx2_set_real_num_queues(struct net_device *netdev,
1154 int tx_queues, int rx_queues)
1155 {
1156 int err;
1157
1158 err = netif_set_real_num_tx_queues(netdev, tx_queues);
1159 if (err) {
1160 netdev_err(netdev,
1161 "Failed to set no of Tx queues: %d\n", tx_queues);
1162 return err;
1163 }
1164
1165 err = netif_set_real_num_rx_queues(netdev, rx_queues);
1166 if (err)
1167 netdev_err(netdev,
1168 "Failed to set no of Rx queues: %d\n", rx_queues);
1169 return err;
1170 }
1171 EXPORT_SYMBOL(otx2_set_real_num_queues);
1172
1173 static char *nix_sqoperr_e_str[NIX_SQOPERR_MAX] = {
1174 "NIX_SQOPERR_OOR",
1175 "NIX_SQOPERR_CTX_FAULT",
1176 "NIX_SQOPERR_CTX_POISON",
1177 "NIX_SQOPERR_DISABLED",
1178 "NIX_SQOPERR_SIZE_ERR",
1179 "NIX_SQOPERR_OFLOW",
1180 "NIX_SQOPERR_SQB_NULL",
1181 "NIX_SQOPERR_SQB_FAULT",
1182 "NIX_SQOPERR_SQE_SZ_ZERO",
1183 };
1184
1185 static char *nix_mnqerr_e_str[NIX_MNQERR_MAX] = {
1186 "NIX_MNQERR_SQ_CTX_FAULT",
1187 "NIX_MNQERR_SQ_CTX_POISON",
1188 "NIX_MNQERR_SQB_FAULT",
1189 "NIX_MNQERR_SQB_POISON",
1190 "NIX_MNQERR_TOTAL_ERR",
1191 "NIX_MNQERR_LSO_ERR",
1192 "NIX_MNQERR_CQ_QUERY_ERR",
1193 "NIX_MNQERR_MAX_SQE_SIZE_ERR",
1194 "NIX_MNQERR_MAXLEN_ERR",
1195 "NIX_MNQERR_SQE_SIZEM1_ZERO",
1196 };
1197
1198 static char *nix_snd_status_e_str[NIX_SND_STATUS_MAX] = {
1199 [NIX_SND_STATUS_GOOD] = "NIX_SND_STATUS_GOOD",
1200 [NIX_SND_STATUS_SQ_CTX_FAULT] = "NIX_SND_STATUS_SQ_CTX_FAULT",
1201 [NIX_SND_STATUS_SQ_CTX_POISON] = "NIX_SND_STATUS_SQ_CTX_POISON",
1202 [NIX_SND_STATUS_SQB_FAULT] = "NIX_SND_STATUS_SQB_FAULT",
1203 [NIX_SND_STATUS_SQB_POISON] = "NIX_SND_STATUS_SQB_POISON",
1204 [NIX_SND_STATUS_HDR_ERR] = "NIX_SND_STATUS_HDR_ERR",
1205 [NIX_SND_STATUS_EXT_ERR] = "NIX_SND_STATUS_EXT_ERR",
1206 [NIX_SND_STATUS_JUMP_FAULT] = "NIX_SND_STATUS_JUMP_FAULT",
1207 [NIX_SND_STATUS_JUMP_POISON] = "NIX_SND_STATUS_JUMP_POISON",
1208 [NIX_SND_STATUS_CRC_ERR] = "NIX_SND_STATUS_CRC_ERR",
1209 [NIX_SND_STATUS_IMM_ERR] = "NIX_SND_STATUS_IMM_ERR",
1210 [NIX_SND_STATUS_SG_ERR] = "NIX_SND_STATUS_SG_ERR",
1211 [NIX_SND_STATUS_MEM_ERR] = "NIX_SND_STATUS_MEM_ERR",
1212 [NIX_SND_STATUS_INVALID_SUBDC] = "NIX_SND_STATUS_INVALID_SUBDC",
1213 [NIX_SND_STATUS_SUBDC_ORDER_ERR] = "NIX_SND_STATUS_SUBDC_ORDER_ERR",
1214 [NIX_SND_STATUS_DATA_FAULT] = "NIX_SND_STATUS_DATA_FAULT",
1215 [NIX_SND_STATUS_DATA_POISON] = "NIX_SND_STATUS_DATA_POISON",
1216 [NIX_SND_STATUS_NPC_DROP_ACTION] = "NIX_SND_STATUS_NPC_DROP_ACTION",
1217 [NIX_SND_STATUS_LOCK_VIOL] = "NIX_SND_STATUS_LOCK_VIOL",
1218 [NIX_SND_STATUS_NPC_UCAST_CHAN_ERR] = "NIX_SND_STAT_NPC_UCAST_CHAN_ERR",
1219 [NIX_SND_STATUS_NPC_MCAST_CHAN_ERR] = "NIX_SND_STAT_NPC_MCAST_CHAN_ERR",
1220 [NIX_SND_STATUS_NPC_MCAST_ABORT] = "NIX_SND_STATUS_NPC_MCAST_ABORT",
1221 [NIX_SND_STATUS_NPC_VTAG_PTR_ERR] = "NIX_SND_STATUS_NPC_VTAG_PTR_ERR",
1222 [NIX_SND_STATUS_NPC_VTAG_SIZE_ERR] = "NIX_SND_STATUS_NPC_VTAG_SIZE_ERR",
1223 [NIX_SND_STATUS_SEND_MEM_FAULT] = "NIX_SND_STATUS_SEND_MEM_FAULT",
1224 [NIX_SND_STATUS_SEND_STATS_ERR] = "NIX_SND_STATUS_SEND_STATS_ERR",
1225 };
1226
otx2_q_intr_handler(int irq,void * data)1227 static irqreturn_t otx2_q_intr_handler(int irq, void *data)
1228 {
1229 struct otx2_nic *pf = data;
1230 struct otx2_snd_queue *sq;
1231 u64 val, *ptr;
1232 u64 qidx = 0;
1233
1234 /* CQ */
1235 for (qidx = 0; qidx < pf->qset.cq_cnt; qidx++) {
1236 ptr = otx2_get_regaddr(pf, NIX_LF_CQ_OP_INT);
1237 val = otx2_atomic64_add((qidx << 44), ptr);
1238
1239 otx2_write64(pf, NIX_LF_CQ_OP_INT, (qidx << 44) |
1240 (val & NIX_CQERRINT_BITS));
1241 if (!(val & (NIX_CQERRINT_BITS | BIT_ULL(42))))
1242 continue;
1243
1244 if (val & BIT_ULL(42)) {
1245 netdev_err(pf->netdev,
1246 "CQ%lld: error reading NIX_LF_CQ_OP_INT, NIX_LF_ERR_INT 0x%llx\n",
1247 qidx, otx2_read64(pf, NIX_LF_ERR_INT));
1248 } else {
1249 if (val & BIT_ULL(NIX_CQERRINT_DOOR_ERR))
1250 netdev_err(pf->netdev, "CQ%lld: Doorbell error",
1251 qidx);
1252 if (val & BIT_ULL(NIX_CQERRINT_CQE_FAULT))
1253 netdev_err(pf->netdev,
1254 "CQ%lld: Memory fault on CQE write to LLC/DRAM",
1255 qidx);
1256 }
1257
1258 schedule_work(&pf->reset_task);
1259 }
1260
1261 /* SQ */
1262 for (qidx = 0; qidx < otx2_get_total_tx_queues(pf); qidx++) {
1263 u64 sq_op_err_dbg, mnq_err_dbg, snd_err_dbg;
1264 u8 sq_op_err_code, mnq_err_code, snd_err_code;
1265
1266 sq = &pf->qset.sq[qidx];
1267 if (!sq->sqb_ptrs)
1268 continue;
1269
1270 /* Below debug registers captures first errors corresponding to
1271 * those registers. We don't have to check against SQ qid as
1272 * these are fatal errors.
1273 */
1274
1275 ptr = otx2_get_regaddr(pf, NIX_LF_SQ_OP_INT);
1276 val = otx2_atomic64_add((qidx << 44), ptr);
1277 otx2_write64(pf, NIX_LF_SQ_OP_INT, (qidx << 44) |
1278 (val & NIX_SQINT_BITS));
1279
1280 if (val & BIT_ULL(42)) {
1281 netdev_err(pf->netdev,
1282 "SQ%lld: error reading NIX_LF_SQ_OP_INT, NIX_LF_ERR_INT 0x%llx\n",
1283 qidx, otx2_read64(pf, NIX_LF_ERR_INT));
1284 goto done;
1285 }
1286
1287 sq_op_err_dbg = otx2_read64(pf, NIX_LF_SQ_OP_ERR_DBG);
1288 if (!(sq_op_err_dbg & BIT(44)))
1289 goto chk_mnq_err_dbg;
1290
1291 sq_op_err_code = FIELD_GET(GENMASK(7, 0), sq_op_err_dbg);
1292 netdev_err(pf->netdev,
1293 "SQ%lld: NIX_LF_SQ_OP_ERR_DBG(0x%llx) err=%s(%#x)\n",
1294 qidx, sq_op_err_dbg,
1295 nix_sqoperr_e_str[sq_op_err_code],
1296 sq_op_err_code);
1297
1298 otx2_write64(pf, NIX_LF_SQ_OP_ERR_DBG, BIT_ULL(44));
1299
1300 if (sq_op_err_code == NIX_SQOPERR_SQB_NULL)
1301 goto chk_mnq_err_dbg;
1302
1303 /* Err is not NIX_SQOPERR_SQB_NULL, call aq function to read SQ structure.
1304 * TODO: But we are in irq context. How to call mbox functions which does sleep
1305 */
1306
1307 chk_mnq_err_dbg:
1308 mnq_err_dbg = otx2_read64(pf, NIX_LF_MNQ_ERR_DBG);
1309 if (!(mnq_err_dbg & BIT(44)))
1310 goto chk_snd_err_dbg;
1311
1312 mnq_err_code = FIELD_GET(GENMASK(7, 0), mnq_err_dbg);
1313 netdev_err(pf->netdev,
1314 "SQ%lld: NIX_LF_MNQ_ERR_DBG(0x%llx) err=%s(%#x)\n",
1315 qidx, mnq_err_dbg, nix_mnqerr_e_str[mnq_err_code],
1316 mnq_err_code);
1317 otx2_write64(pf, NIX_LF_MNQ_ERR_DBG, BIT_ULL(44));
1318
1319 chk_snd_err_dbg:
1320 snd_err_dbg = otx2_read64(pf, NIX_LF_SEND_ERR_DBG);
1321 if (snd_err_dbg & BIT(44)) {
1322 snd_err_code = FIELD_GET(GENMASK(7, 0), snd_err_dbg);
1323 netdev_err(pf->netdev,
1324 "SQ%lld: NIX_LF_SND_ERR_DBG:0x%llx err=%s(%#x)\n",
1325 qidx, snd_err_dbg,
1326 nix_snd_status_e_str[snd_err_code],
1327 snd_err_code);
1328 otx2_write64(pf, NIX_LF_SEND_ERR_DBG, BIT_ULL(44));
1329 }
1330
1331 done:
1332 /* Print values and reset */
1333 if (val & BIT_ULL(NIX_SQINT_SQB_ALLOC_FAIL))
1334 netdev_err(pf->netdev, "SQ%lld: SQB allocation failed",
1335 qidx);
1336
1337 schedule_work(&pf->reset_task);
1338 }
1339
1340 return IRQ_HANDLED;
1341 }
1342
otx2_cq_intr_handler(int irq,void * cq_irq)1343 static irqreturn_t otx2_cq_intr_handler(int irq, void *cq_irq)
1344 {
1345 struct otx2_cq_poll *cq_poll = (struct otx2_cq_poll *)cq_irq;
1346 struct otx2_nic *pf = (struct otx2_nic *)cq_poll->dev;
1347 int qidx = cq_poll->cint_idx;
1348
1349 /* Disable interrupts.
1350 *
1351 * Completion interrupts behave in a level-triggered interrupt
1352 * fashion, and hence have to be cleared only after it is serviced.
1353 */
1354 otx2_write64(pf, NIX_LF_CINTX_ENA_W1C(qidx), BIT_ULL(0));
1355
1356 /* Schedule NAPI */
1357 pf->napi_events++;
1358 napi_schedule_irqoff(&cq_poll->napi);
1359
1360 return IRQ_HANDLED;
1361 }
1362
otx2_disable_napi(struct otx2_nic * pf)1363 static void otx2_disable_napi(struct otx2_nic *pf)
1364 {
1365 struct otx2_qset *qset = &pf->qset;
1366 struct otx2_cq_poll *cq_poll;
1367 int qidx;
1368
1369 for (qidx = 0; qidx < pf->hw.cint_cnt; qidx++) {
1370 cq_poll = &qset->napi[qidx];
1371 cancel_work_sync(&cq_poll->dim.work);
1372 napi_disable(&cq_poll->napi);
1373 netif_napi_del(&cq_poll->napi);
1374 }
1375 }
1376
otx2_free_cq_res(struct otx2_nic * pf)1377 static void otx2_free_cq_res(struct otx2_nic *pf)
1378 {
1379 struct otx2_qset *qset = &pf->qset;
1380 struct otx2_cq_queue *cq;
1381 int qidx;
1382
1383 /* Disable CQs */
1384 otx2_ctx_disable(&pf->mbox, NIX_AQ_CTYPE_CQ, false);
1385 for (qidx = 0; qidx < qset->cq_cnt; qidx++) {
1386 cq = &qset->cq[qidx];
1387 qmem_free(pf->dev, cq->cqe);
1388 }
1389 }
1390
otx2_free_sq_res(struct otx2_nic * pf)1391 static void otx2_free_sq_res(struct otx2_nic *pf)
1392 {
1393 struct otx2_qset *qset = &pf->qset;
1394 struct otx2_snd_queue *sq;
1395 int qidx;
1396
1397 /* Disable SQs */
1398 otx2_ctx_disable(&pf->mbox, NIX_AQ_CTYPE_SQ, false);
1399 /* Free SQB pointers */
1400 otx2_sq_free_sqbs(pf);
1401 for (qidx = 0; qidx < otx2_get_total_tx_queues(pf); qidx++) {
1402 sq = &qset->sq[qidx];
1403 /* Skip freeing Qos queues if they are not initialized */
1404 if (!sq->sqe)
1405 continue;
1406 qmem_free(pf->dev, sq->sqe);
1407 qmem_free(pf->dev, sq->tso_hdrs);
1408 kfree(sq->sg);
1409 kfree(sq->sqb_ptrs);
1410 }
1411 }
1412
otx2_get_rbuf_size(struct otx2_nic * pf,int mtu)1413 static int otx2_get_rbuf_size(struct otx2_nic *pf, int mtu)
1414 {
1415 int frame_size;
1416 int total_size;
1417 int rbuf_size;
1418
1419 if (pf->hw.rbuf_len)
1420 return ALIGN(pf->hw.rbuf_len, OTX2_ALIGN) + OTX2_HEAD_ROOM;
1421
1422 /* The data transferred by NIX to memory consists of actual packet
1423 * plus additional data which has timestamp and/or EDSA/HIGIG2
1424 * headers if interface is configured in corresponding modes.
1425 * NIX transfers entire data using 6 segments/buffers and writes
1426 * a CQE_RX descriptor with those segment addresses. First segment
1427 * has additional data prepended to packet. Also software omits a
1428 * headroom of 128 bytes in each segment. Hence the total size of
1429 * memory needed to receive a packet with 'mtu' is:
1430 * frame size = mtu + additional data;
1431 * memory = frame_size + headroom * 6;
1432 * each receive buffer size = memory / 6;
1433 */
1434 frame_size = mtu + OTX2_ETH_HLEN + OTX2_HW_TIMESTAMP_LEN;
1435 total_size = frame_size + OTX2_HEAD_ROOM * 6;
1436 rbuf_size = total_size / 6;
1437
1438 return ALIGN(rbuf_size, 2048);
1439 }
1440
otx2_init_hw_resources(struct otx2_nic * pf)1441 static int otx2_init_hw_resources(struct otx2_nic *pf)
1442 {
1443 struct nix_lf_free_req *free_req;
1444 struct mbox *mbox = &pf->mbox;
1445 struct otx2_hw *hw = &pf->hw;
1446 struct msg_req *req;
1447 int err = 0, lvl;
1448
1449 /* Set required NPA LF's pool counts
1450 * Auras and Pools are used in a 1:1 mapping,
1451 * so, aura count = pool count.
1452 */
1453 hw->rqpool_cnt = hw->rx_queues;
1454 hw->sqpool_cnt = otx2_get_total_tx_queues(pf);
1455 hw->pool_cnt = hw->rqpool_cnt + hw->sqpool_cnt;
1456
1457 /* Maximum hardware supported transmit length */
1458 pf->tx_max_pktlen = pf->netdev->max_mtu + OTX2_ETH_HLEN;
1459
1460 pf->rbsize = otx2_get_rbuf_size(pf, pf->netdev->mtu);
1461
1462 mutex_lock(&mbox->lock);
1463 /* NPA init */
1464 err = otx2_config_npa(pf);
1465 if (err)
1466 goto exit;
1467
1468 /* NIX init */
1469 err = otx2_config_nix(pf);
1470 if (err)
1471 goto err_free_npa_lf;
1472
1473 /* Enable backpressure for CGX mapped PF/VFs */
1474 if (!is_otx2_lbkvf(pf->pdev))
1475 otx2_nix_config_bp(pf, true);
1476
1477 /* Init Auras and pools used by NIX RQ, for free buffer ptrs */
1478 err = otx2_rq_aura_pool_init(pf);
1479 if (err) {
1480 mutex_unlock(&mbox->lock);
1481 goto err_free_nix_lf;
1482 }
1483 /* Init Auras and pools used by NIX SQ, for queueing SQEs */
1484 err = otx2_sq_aura_pool_init(pf);
1485 if (err) {
1486 mutex_unlock(&mbox->lock);
1487 goto err_free_rq_ptrs;
1488 }
1489
1490 err = otx2_txsch_alloc(pf);
1491 if (err) {
1492 mutex_unlock(&mbox->lock);
1493 goto err_free_sq_ptrs;
1494 }
1495
1496 #ifdef CONFIG_DCB
1497 if (pf->pfc_en) {
1498 err = otx2_pfc_txschq_alloc(pf);
1499 if (err) {
1500 mutex_unlock(&mbox->lock);
1501 goto err_free_sq_ptrs;
1502 }
1503 }
1504 #endif
1505
1506 err = otx2_config_nix_queues(pf);
1507 if (err) {
1508 mutex_unlock(&mbox->lock);
1509 goto err_free_txsch;
1510 }
1511
1512 for (lvl = 0; lvl < NIX_TXSCH_LVL_CNT; lvl++) {
1513 err = otx2_txschq_config(pf, lvl, 0, false);
1514 if (err) {
1515 mutex_unlock(&mbox->lock);
1516 goto err_free_nix_queues;
1517 }
1518 }
1519
1520 #ifdef CONFIG_DCB
1521 if (pf->pfc_en) {
1522 err = otx2_pfc_txschq_config(pf);
1523 if (err) {
1524 mutex_unlock(&mbox->lock);
1525 goto err_free_nix_queues;
1526 }
1527 }
1528 #endif
1529
1530 mutex_unlock(&mbox->lock);
1531 return err;
1532
1533 err_free_nix_queues:
1534 otx2_free_sq_res(pf);
1535 otx2_free_cq_res(pf);
1536 otx2_ctx_disable(mbox, NIX_AQ_CTYPE_RQ, false);
1537 err_free_txsch:
1538 otx2_txschq_stop(pf);
1539 err_free_sq_ptrs:
1540 otx2_sq_free_sqbs(pf);
1541 err_free_rq_ptrs:
1542 otx2_free_aura_ptr(pf, AURA_NIX_RQ);
1543 otx2_ctx_disable(mbox, NPA_AQ_CTYPE_POOL, true);
1544 otx2_ctx_disable(mbox, NPA_AQ_CTYPE_AURA, true);
1545 otx2_aura_pool_free(pf);
1546 err_free_nix_lf:
1547 mutex_lock(&mbox->lock);
1548 free_req = otx2_mbox_alloc_msg_nix_lf_free(mbox);
1549 if (free_req) {
1550 free_req->flags = NIX_LF_DISABLE_FLOWS;
1551 if (otx2_sync_mbox_msg(mbox))
1552 dev_err(pf->dev, "%s failed to free nixlf\n", __func__);
1553 }
1554 err_free_npa_lf:
1555 /* Reset NPA LF */
1556 req = otx2_mbox_alloc_msg_npa_lf_free(mbox);
1557 if (req) {
1558 if (otx2_sync_mbox_msg(mbox))
1559 dev_err(pf->dev, "%s failed to free npalf\n", __func__);
1560 }
1561 exit:
1562 mutex_unlock(&mbox->lock);
1563 return err;
1564 }
1565
otx2_free_hw_resources(struct otx2_nic * pf)1566 static void otx2_free_hw_resources(struct otx2_nic *pf)
1567 {
1568 struct otx2_qset *qset = &pf->qset;
1569 struct nix_lf_free_req *free_req;
1570 struct mbox *mbox = &pf->mbox;
1571 struct otx2_cq_queue *cq;
1572 struct otx2_pool *pool;
1573 struct msg_req *req;
1574 int pool_id;
1575 int qidx;
1576
1577 /* Ensure all SQE are processed */
1578 otx2_sqb_flush(pf);
1579
1580 /* Stop transmission */
1581 otx2_txschq_stop(pf);
1582
1583 #ifdef CONFIG_DCB
1584 if (pf->pfc_en)
1585 otx2_pfc_txschq_stop(pf);
1586 #endif
1587
1588 otx2_clean_qos_queues(pf);
1589
1590 mutex_lock(&mbox->lock);
1591 /* Disable backpressure */
1592 if (!(pf->pcifunc & RVU_PFVF_FUNC_MASK))
1593 otx2_nix_config_bp(pf, false);
1594 mutex_unlock(&mbox->lock);
1595
1596 /* Disable RQs */
1597 otx2_ctx_disable(mbox, NIX_AQ_CTYPE_RQ, false);
1598
1599 /*Dequeue all CQEs */
1600 for (qidx = 0; qidx < qset->cq_cnt; qidx++) {
1601 cq = &qset->cq[qidx];
1602 if (cq->cq_type == CQ_RX)
1603 otx2_cleanup_rx_cqes(pf, cq, qidx);
1604 else
1605 otx2_cleanup_tx_cqes(pf, cq);
1606 }
1607 otx2_free_pending_sqe(pf);
1608
1609 otx2_free_sq_res(pf);
1610
1611 /* Free RQ buffer pointers*/
1612 otx2_free_aura_ptr(pf, AURA_NIX_RQ);
1613
1614 for (qidx = 0; qidx < pf->hw.rx_queues; qidx++) {
1615 pool_id = otx2_get_pool_idx(pf, AURA_NIX_RQ, qidx);
1616 pool = &pf->qset.pool[pool_id];
1617 page_pool_destroy(pool->page_pool);
1618 pool->page_pool = NULL;
1619 }
1620
1621 otx2_free_cq_res(pf);
1622
1623 /* Free all ingress bandwidth profiles allocated */
1624 cn10k_free_all_ipolicers(pf);
1625
1626 mutex_lock(&mbox->lock);
1627 /* Reset NIX LF */
1628 free_req = otx2_mbox_alloc_msg_nix_lf_free(mbox);
1629 if (free_req) {
1630 free_req->flags = NIX_LF_DISABLE_FLOWS;
1631 if (!(pf->flags & OTX2_FLAG_PF_SHUTDOWN))
1632 free_req->flags |= NIX_LF_DONT_FREE_TX_VTAG;
1633 if (otx2_sync_mbox_msg(mbox))
1634 dev_err(pf->dev, "%s failed to free nixlf\n", __func__);
1635 }
1636 mutex_unlock(&mbox->lock);
1637
1638 /* Disable NPA Pool and Aura hw context */
1639 otx2_ctx_disable(mbox, NPA_AQ_CTYPE_POOL, true);
1640 otx2_ctx_disable(mbox, NPA_AQ_CTYPE_AURA, true);
1641 otx2_aura_pool_free(pf);
1642
1643 mutex_lock(&mbox->lock);
1644 /* Reset NPA LF */
1645 req = otx2_mbox_alloc_msg_npa_lf_free(mbox);
1646 if (req) {
1647 if (otx2_sync_mbox_msg(mbox))
1648 dev_err(pf->dev, "%s failed to free npalf\n", __func__);
1649 }
1650 mutex_unlock(&mbox->lock);
1651 }
1652
otx2_promisc_use_mce_list(struct otx2_nic * pfvf)1653 static bool otx2_promisc_use_mce_list(struct otx2_nic *pfvf)
1654 {
1655 int vf;
1656
1657 /* The AF driver will determine whether to allow the VF netdev or not */
1658 if (is_otx2_vf(pfvf->pcifunc))
1659 return true;
1660
1661 /* check if there are any trusted VFs associated with the PF netdev */
1662 for (vf = 0; vf < pci_num_vf(pfvf->pdev); vf++)
1663 if (pfvf->vf_configs[vf].trusted)
1664 return true;
1665 return false;
1666 }
1667
otx2_do_set_rx_mode(struct otx2_nic * pf)1668 static void otx2_do_set_rx_mode(struct otx2_nic *pf)
1669 {
1670 struct net_device *netdev = pf->netdev;
1671 struct nix_rx_mode *req;
1672 bool promisc = false;
1673
1674 if (!(netdev->flags & IFF_UP))
1675 return;
1676
1677 if ((netdev->flags & IFF_PROMISC) ||
1678 (netdev_uc_count(netdev) > OTX2_MAX_UNICAST_FLOWS)) {
1679 promisc = true;
1680 }
1681
1682 /* Write unicast address to mcam entries or del from mcam */
1683 if (!promisc && netdev->priv_flags & IFF_UNICAST_FLT)
1684 __dev_uc_sync(netdev, otx2_add_macfilter, otx2_del_macfilter);
1685
1686 mutex_lock(&pf->mbox.lock);
1687 req = otx2_mbox_alloc_msg_nix_set_rx_mode(&pf->mbox);
1688 if (!req) {
1689 mutex_unlock(&pf->mbox.lock);
1690 return;
1691 }
1692
1693 req->mode = NIX_RX_MODE_UCAST;
1694
1695 if (promisc)
1696 req->mode |= NIX_RX_MODE_PROMISC;
1697 if (netdev->flags & (IFF_ALLMULTI | IFF_MULTICAST))
1698 req->mode |= NIX_RX_MODE_ALLMULTI;
1699
1700 if (otx2_promisc_use_mce_list(pf))
1701 req->mode |= NIX_RX_MODE_USE_MCE;
1702
1703 otx2_sync_mbox_msg(&pf->mbox);
1704 mutex_unlock(&pf->mbox.lock);
1705 }
1706
otx2_set_irq_coalesce(struct otx2_nic * pfvf)1707 static void otx2_set_irq_coalesce(struct otx2_nic *pfvf)
1708 {
1709 int cint;
1710
1711 for (cint = 0; cint < pfvf->hw.cint_cnt; cint++)
1712 otx2_config_irq_coalescing(pfvf, cint);
1713 }
1714
otx2_dim_work(struct work_struct * w)1715 static void otx2_dim_work(struct work_struct *w)
1716 {
1717 struct dim_cq_moder cur_moder;
1718 struct otx2_cq_poll *cq_poll;
1719 struct otx2_nic *pfvf;
1720 struct dim *dim;
1721
1722 dim = container_of(w, struct dim, work);
1723 cur_moder = net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
1724 cq_poll = container_of(dim, struct otx2_cq_poll, dim);
1725 pfvf = (struct otx2_nic *)cq_poll->dev;
1726 pfvf->hw.cq_time_wait = (cur_moder.usec > CQ_TIMER_THRESH_MAX) ?
1727 CQ_TIMER_THRESH_MAX : cur_moder.usec;
1728 pfvf->hw.cq_ecount_wait = (cur_moder.pkts > NAPI_POLL_WEIGHT) ?
1729 NAPI_POLL_WEIGHT : cur_moder.pkts;
1730 otx2_set_irq_coalesce(pfvf);
1731 dim->state = DIM_START_MEASURE;
1732 }
1733
otx2_open(struct net_device * netdev)1734 int otx2_open(struct net_device *netdev)
1735 {
1736 struct otx2_nic *pf = netdev_priv(netdev);
1737 struct otx2_cq_poll *cq_poll = NULL;
1738 struct otx2_qset *qset = &pf->qset;
1739 int err = 0, qidx, vec;
1740 char *irq_name;
1741
1742 netif_carrier_off(netdev);
1743
1744 /* RQ and SQs are mapped to different CQs,
1745 * so find out max CQ IRQs (i.e CINTs) needed.
1746 */
1747 pf->hw.non_qos_queues = pf->hw.tx_queues + pf->hw.xdp_queues;
1748 pf->hw.cint_cnt = max3(pf->hw.rx_queues, pf->hw.tx_queues,
1749 pf->hw.tc_tx_queues);
1750
1751 pf->qset.cq_cnt = pf->hw.rx_queues + otx2_get_total_tx_queues(pf);
1752
1753 qset->napi = kcalloc(pf->hw.cint_cnt, sizeof(*cq_poll), GFP_KERNEL);
1754 if (!qset->napi)
1755 return -ENOMEM;
1756
1757 /* CQ size of RQ */
1758 qset->rqe_cnt = qset->rqe_cnt ? qset->rqe_cnt : Q_COUNT(Q_SIZE_256);
1759 /* CQ size of SQ */
1760 qset->sqe_cnt = qset->sqe_cnt ? qset->sqe_cnt : Q_COUNT(Q_SIZE_4K);
1761
1762 err = -ENOMEM;
1763 qset->cq = kcalloc(pf->qset.cq_cnt,
1764 sizeof(struct otx2_cq_queue), GFP_KERNEL);
1765 if (!qset->cq)
1766 goto err_free_mem;
1767
1768 qset->sq = kcalloc(otx2_get_total_tx_queues(pf),
1769 sizeof(struct otx2_snd_queue), GFP_KERNEL);
1770 if (!qset->sq)
1771 goto err_free_mem;
1772
1773 qset->rq = kcalloc(pf->hw.rx_queues,
1774 sizeof(struct otx2_rcv_queue), GFP_KERNEL);
1775 if (!qset->rq)
1776 goto err_free_mem;
1777
1778 err = otx2_init_hw_resources(pf);
1779 if (err)
1780 goto err_free_mem;
1781
1782 /* Register NAPI handler */
1783 for (qidx = 0; qidx < pf->hw.cint_cnt; qidx++) {
1784 cq_poll = &qset->napi[qidx];
1785 cq_poll->cint_idx = qidx;
1786 /* RQ0 & SQ0 are mapped to CINT0 and so on..
1787 * 'cq_ids[0]' points to RQ's CQ and
1788 * 'cq_ids[1]' points to SQ's CQ and
1789 * 'cq_ids[2]' points to XDP's CQ and
1790 */
1791 cq_poll->cq_ids[CQ_RX] =
1792 (qidx < pf->hw.rx_queues) ? qidx : CINT_INVALID_CQ;
1793 cq_poll->cq_ids[CQ_TX] = (qidx < pf->hw.tx_queues) ?
1794 qidx + pf->hw.rx_queues : CINT_INVALID_CQ;
1795 if (pf->xdp_prog)
1796 cq_poll->cq_ids[CQ_XDP] = (qidx < pf->hw.xdp_queues) ?
1797 (qidx + pf->hw.rx_queues +
1798 pf->hw.tx_queues) :
1799 CINT_INVALID_CQ;
1800 else
1801 cq_poll->cq_ids[CQ_XDP] = CINT_INVALID_CQ;
1802
1803 cq_poll->cq_ids[CQ_QOS] = (qidx < pf->hw.tc_tx_queues) ?
1804 (qidx + pf->hw.rx_queues +
1805 pf->hw.non_qos_queues) :
1806 CINT_INVALID_CQ;
1807
1808 cq_poll->dev = (void *)pf;
1809 cq_poll->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_CQE;
1810 INIT_WORK(&cq_poll->dim.work, otx2_dim_work);
1811 netif_napi_add(netdev, &cq_poll->napi, otx2_napi_handler);
1812 napi_enable(&cq_poll->napi);
1813 }
1814
1815 /* Set maximum frame size allowed in HW */
1816 err = otx2_hw_set_mtu(pf, netdev->mtu);
1817 if (err)
1818 goto err_disable_napi;
1819
1820 /* Setup segmentation algorithms, if failed, clear offload capability */
1821 otx2_setup_segmentation(pf);
1822
1823 /* Initialize RSS */
1824 err = otx2_rss_init(pf);
1825 if (err)
1826 goto err_disable_napi;
1827
1828 /* Register Queue IRQ handlers */
1829 vec = pf->hw.nix_msixoff + NIX_LF_QINT_VEC_START;
1830 irq_name = &pf->hw.irq_name[vec * NAME_SIZE];
1831
1832 snprintf(irq_name, NAME_SIZE, "%s-qerr", pf->netdev->name);
1833
1834 err = request_irq(pci_irq_vector(pf->pdev, vec),
1835 otx2_q_intr_handler, 0, irq_name, pf);
1836 if (err) {
1837 dev_err(pf->dev,
1838 "RVUPF%d: IRQ registration failed for QERR\n",
1839 rvu_get_pf(pf->pcifunc));
1840 goto err_disable_napi;
1841 }
1842
1843 /* Enable QINT IRQ */
1844 otx2_write64(pf, NIX_LF_QINTX_ENA_W1S(0), BIT_ULL(0));
1845
1846 /* Register CQ IRQ handlers */
1847 vec = pf->hw.nix_msixoff + NIX_LF_CINT_VEC_START;
1848 for (qidx = 0; qidx < pf->hw.cint_cnt; qidx++) {
1849 irq_name = &pf->hw.irq_name[vec * NAME_SIZE];
1850
1851 snprintf(irq_name, NAME_SIZE, "%s-rxtx-%d", pf->netdev->name,
1852 qidx);
1853
1854 err = request_irq(pci_irq_vector(pf->pdev, vec),
1855 otx2_cq_intr_handler, 0, irq_name,
1856 &qset->napi[qidx]);
1857 if (err) {
1858 dev_err(pf->dev,
1859 "RVUPF%d: IRQ registration failed for CQ%d\n",
1860 rvu_get_pf(pf->pcifunc), qidx);
1861 goto err_free_cints;
1862 }
1863 vec++;
1864
1865 otx2_config_irq_coalescing(pf, qidx);
1866
1867 /* Enable CQ IRQ */
1868 otx2_write64(pf, NIX_LF_CINTX_INT(qidx), BIT_ULL(0));
1869 otx2_write64(pf, NIX_LF_CINTX_ENA_W1S(qidx), BIT_ULL(0));
1870 }
1871
1872 otx2_set_cints_affinity(pf);
1873
1874 if (pf->flags & OTX2_FLAG_RX_VLAN_SUPPORT)
1875 otx2_enable_rxvlan(pf, true);
1876
1877 /* When reinitializing enable time stamping if it is enabled before */
1878 if (pf->flags & OTX2_FLAG_TX_TSTAMP_ENABLED) {
1879 pf->flags &= ~OTX2_FLAG_TX_TSTAMP_ENABLED;
1880 otx2_config_hw_tx_tstamp(pf, true);
1881 }
1882 if (pf->flags & OTX2_FLAG_RX_TSTAMP_ENABLED) {
1883 pf->flags &= ~OTX2_FLAG_RX_TSTAMP_ENABLED;
1884 otx2_config_hw_rx_tstamp(pf, true);
1885 }
1886
1887 pf->flags &= ~OTX2_FLAG_INTF_DOWN;
1888 /* 'intf_down' may be checked on any cpu */
1889 smp_wmb();
1890
1891 /* Enable QoS configuration before starting tx queues */
1892 otx2_qos_config_txschq(pf);
1893
1894 /* we have already received link status notification */
1895 if (pf->linfo.link_up && !(pf->pcifunc & RVU_PFVF_FUNC_MASK))
1896 otx2_handle_link_event(pf);
1897
1898 /* Install DMAC Filters */
1899 if (pf->flags & OTX2_FLAG_DMACFLTR_SUPPORT)
1900 otx2_dmacflt_reinstall_flows(pf);
1901
1902 otx2_tc_apply_ingress_police_rules(pf);
1903
1904 err = otx2_rxtx_enable(pf, true);
1905 /* If a mbox communication error happens at this point then interface
1906 * will end up in a state such that it is in down state but hardware
1907 * mcam entries are enabled to receive the packets. Hence disable the
1908 * packet I/O.
1909 */
1910 if (err == EIO)
1911 goto err_disable_rxtx;
1912 else if (err)
1913 goto err_tx_stop_queues;
1914
1915 otx2_do_set_rx_mode(pf);
1916
1917 return 0;
1918
1919 err_disable_rxtx:
1920 otx2_rxtx_enable(pf, false);
1921 err_tx_stop_queues:
1922 netif_tx_stop_all_queues(netdev);
1923 netif_carrier_off(netdev);
1924 pf->flags |= OTX2_FLAG_INTF_DOWN;
1925 err_free_cints:
1926 otx2_free_cints(pf, qidx);
1927 vec = pci_irq_vector(pf->pdev,
1928 pf->hw.nix_msixoff + NIX_LF_QINT_VEC_START);
1929 otx2_write64(pf, NIX_LF_QINTX_ENA_W1C(0), BIT_ULL(0));
1930 free_irq(vec, pf);
1931 err_disable_napi:
1932 otx2_disable_napi(pf);
1933 otx2_free_hw_resources(pf);
1934 err_free_mem:
1935 kfree(qset->sq);
1936 kfree(qset->cq);
1937 kfree(qset->rq);
1938 kfree(qset->napi);
1939 return err;
1940 }
1941 EXPORT_SYMBOL(otx2_open);
1942
otx2_stop(struct net_device * netdev)1943 int otx2_stop(struct net_device *netdev)
1944 {
1945 struct otx2_nic *pf = netdev_priv(netdev);
1946 struct otx2_cq_poll *cq_poll = NULL;
1947 struct otx2_qset *qset = &pf->qset;
1948 struct otx2_rss_info *rss;
1949 int qidx, vec, wrk;
1950
1951 /* If the DOWN flag is set resources are already freed */
1952 if (pf->flags & OTX2_FLAG_INTF_DOWN)
1953 return 0;
1954
1955 netif_carrier_off(netdev);
1956 netif_tx_stop_all_queues(netdev);
1957
1958 pf->flags |= OTX2_FLAG_INTF_DOWN;
1959 /* 'intf_down' may be checked on any cpu */
1960 smp_wmb();
1961
1962 /* First stop packet Rx/Tx */
1963 otx2_rxtx_enable(pf, false);
1964
1965 /* Clear RSS enable flag */
1966 rss = &pf->hw.rss_info;
1967 rss->enable = false;
1968 if (!netif_is_rxfh_configured(netdev))
1969 kfree(rss->rss_ctx[DEFAULT_RSS_CONTEXT_GROUP]);
1970
1971 /* Cleanup Queue IRQ */
1972 vec = pci_irq_vector(pf->pdev,
1973 pf->hw.nix_msixoff + NIX_LF_QINT_VEC_START);
1974 otx2_write64(pf, NIX_LF_QINTX_ENA_W1C(0), BIT_ULL(0));
1975 free_irq(vec, pf);
1976
1977 /* Cleanup CQ NAPI and IRQ */
1978 vec = pf->hw.nix_msixoff + NIX_LF_CINT_VEC_START;
1979 for (qidx = 0; qidx < pf->hw.cint_cnt; qidx++) {
1980 /* Disable interrupt */
1981 otx2_write64(pf, NIX_LF_CINTX_ENA_W1C(qidx), BIT_ULL(0));
1982
1983 synchronize_irq(pci_irq_vector(pf->pdev, vec));
1984
1985 cq_poll = &qset->napi[qidx];
1986 napi_synchronize(&cq_poll->napi);
1987 vec++;
1988 }
1989
1990 netif_tx_disable(netdev);
1991
1992 for (wrk = 0; wrk < pf->qset.cq_cnt; wrk++)
1993 cancel_delayed_work_sync(&pf->refill_wrk[wrk].pool_refill_work);
1994 devm_kfree(pf->dev, pf->refill_wrk);
1995
1996 otx2_free_hw_resources(pf);
1997 otx2_free_cints(pf, pf->hw.cint_cnt);
1998 otx2_disable_napi(pf);
1999
2000 for (qidx = 0; qidx < netdev->num_tx_queues; qidx++)
2001 netdev_tx_reset_queue(netdev_get_tx_queue(netdev, qidx));
2002
2003
2004 kfree(qset->sq);
2005 kfree(qset->cq);
2006 kfree(qset->rq);
2007 kfree(qset->napi);
2008 /* Do not clear RQ/SQ ringsize settings */
2009 memset_startat(qset, 0, sqe_cnt);
2010 return 0;
2011 }
2012 EXPORT_SYMBOL(otx2_stop);
2013
otx2_xmit(struct sk_buff * skb,struct net_device * netdev)2014 static netdev_tx_t otx2_xmit(struct sk_buff *skb, struct net_device *netdev)
2015 {
2016 struct otx2_nic *pf = netdev_priv(netdev);
2017 int qidx = skb_get_queue_mapping(skb);
2018 struct otx2_snd_queue *sq;
2019 struct netdev_queue *txq;
2020 int sq_idx;
2021
2022 /* XDP SQs are not mapped with TXQs
2023 * advance qid to derive correct sq mapped with QOS
2024 */
2025 sq_idx = (qidx >= pf->hw.tx_queues) ? (qidx + pf->hw.xdp_queues) : qidx;
2026
2027 /* Check for minimum and maximum packet length */
2028 if (skb->len <= ETH_HLEN ||
2029 (!skb_shinfo(skb)->gso_size && skb->len > pf->tx_max_pktlen)) {
2030 dev_kfree_skb(skb);
2031 return NETDEV_TX_OK;
2032 }
2033
2034 sq = &pf->qset.sq[sq_idx];
2035 txq = netdev_get_tx_queue(netdev, qidx);
2036
2037 if (!otx2_sq_append_skb(netdev, sq, skb, qidx)) {
2038 netif_tx_stop_queue(txq);
2039
2040 /* Check again, incase SQBs got freed up */
2041 smp_mb();
2042 if (((sq->num_sqbs - *sq->aura_fc_addr) * sq->sqe_per_sqb)
2043 > sq->sqe_thresh)
2044 netif_tx_wake_queue(txq);
2045
2046 return NETDEV_TX_BUSY;
2047 }
2048
2049 return NETDEV_TX_OK;
2050 }
2051
otx2_qos_select_htb_queue(struct otx2_nic * pf,struct sk_buff * skb,u16 htb_maj_id)2052 static int otx2_qos_select_htb_queue(struct otx2_nic *pf, struct sk_buff *skb,
2053 u16 htb_maj_id)
2054 {
2055 u16 classid;
2056
2057 if ((TC_H_MAJ(skb->priority) >> 16) == htb_maj_id)
2058 classid = TC_H_MIN(skb->priority);
2059 else
2060 classid = READ_ONCE(pf->qos.defcls);
2061
2062 if (!classid)
2063 return 0;
2064
2065 return otx2_get_txq_by_classid(pf, classid);
2066 }
2067
otx2_select_queue(struct net_device * netdev,struct sk_buff * skb,struct net_device * sb_dev)2068 u16 otx2_select_queue(struct net_device *netdev, struct sk_buff *skb,
2069 struct net_device *sb_dev)
2070 {
2071 struct otx2_nic *pf = netdev_priv(netdev);
2072 bool qos_enabled;
2073 #ifdef CONFIG_DCB
2074 u8 vlan_prio;
2075 #endif
2076 int txq;
2077
2078 qos_enabled = netdev->real_num_tx_queues > pf->hw.tx_queues;
2079 if (unlikely(qos_enabled)) {
2080 /* This smp_load_acquire() pairs with smp_store_release() in
2081 * otx2_qos_root_add() called from htb offload root creation
2082 */
2083 u16 htb_maj_id = smp_load_acquire(&pf->qos.maj_id);
2084
2085 if (unlikely(htb_maj_id)) {
2086 txq = otx2_qos_select_htb_queue(pf, skb, htb_maj_id);
2087 if (txq > 0)
2088 return txq;
2089 goto process_pfc;
2090 }
2091 }
2092
2093 process_pfc:
2094 #ifdef CONFIG_DCB
2095 if (!skb_vlan_tag_present(skb))
2096 goto pick_tx;
2097
2098 vlan_prio = skb->vlan_tci >> 13;
2099 if ((vlan_prio > pf->hw.tx_queues - 1) ||
2100 !pf->pfc_alloc_status[vlan_prio])
2101 goto pick_tx;
2102
2103 return vlan_prio;
2104
2105 pick_tx:
2106 #endif
2107 txq = netdev_pick_tx(netdev, skb, NULL);
2108 if (unlikely(qos_enabled))
2109 return txq % pf->hw.tx_queues;
2110
2111 return txq;
2112 }
2113 EXPORT_SYMBOL(otx2_select_queue);
2114
otx2_fix_features(struct net_device * dev,netdev_features_t features)2115 static netdev_features_t otx2_fix_features(struct net_device *dev,
2116 netdev_features_t features)
2117 {
2118 if (features & NETIF_F_HW_VLAN_CTAG_RX)
2119 features |= NETIF_F_HW_VLAN_STAG_RX;
2120 else
2121 features &= ~NETIF_F_HW_VLAN_STAG_RX;
2122
2123 return features;
2124 }
2125
otx2_set_rx_mode(struct net_device * netdev)2126 static void otx2_set_rx_mode(struct net_device *netdev)
2127 {
2128 struct otx2_nic *pf = netdev_priv(netdev);
2129
2130 queue_work(pf->otx2_wq, &pf->rx_mode_work);
2131 }
2132
otx2_rx_mode_wrk_handler(struct work_struct * work)2133 static void otx2_rx_mode_wrk_handler(struct work_struct *work)
2134 {
2135 struct otx2_nic *pf = container_of(work, struct otx2_nic, rx_mode_work);
2136
2137 otx2_do_set_rx_mode(pf);
2138 }
2139
otx2_set_features(struct net_device * netdev,netdev_features_t features)2140 static int otx2_set_features(struct net_device *netdev,
2141 netdev_features_t features)
2142 {
2143 netdev_features_t changed = features ^ netdev->features;
2144 struct otx2_nic *pf = netdev_priv(netdev);
2145
2146 if ((changed & NETIF_F_LOOPBACK) && netif_running(netdev))
2147 return otx2_cgx_config_loopback(pf,
2148 features & NETIF_F_LOOPBACK);
2149
2150 if ((changed & NETIF_F_HW_VLAN_CTAG_RX) && netif_running(netdev))
2151 return otx2_enable_rxvlan(pf,
2152 features & NETIF_F_HW_VLAN_CTAG_RX);
2153
2154 return otx2_handle_ntuple_tc_features(netdev, features);
2155 }
2156
otx2_reset_task(struct work_struct * work)2157 static void otx2_reset_task(struct work_struct *work)
2158 {
2159 struct otx2_nic *pf = container_of(work, struct otx2_nic, reset_task);
2160
2161 if (!netif_running(pf->netdev))
2162 return;
2163
2164 rtnl_lock();
2165 otx2_stop(pf->netdev);
2166 pf->reset_count++;
2167 otx2_open(pf->netdev);
2168 netif_trans_update(pf->netdev);
2169 rtnl_unlock();
2170 }
2171
otx2_config_hw_rx_tstamp(struct otx2_nic * pfvf,bool enable)2172 static int otx2_config_hw_rx_tstamp(struct otx2_nic *pfvf, bool enable)
2173 {
2174 struct msg_req *req;
2175 int err;
2176
2177 if (pfvf->flags & OTX2_FLAG_RX_TSTAMP_ENABLED && enable)
2178 return 0;
2179
2180 mutex_lock(&pfvf->mbox.lock);
2181 if (enable)
2182 req = otx2_mbox_alloc_msg_cgx_ptp_rx_enable(&pfvf->mbox);
2183 else
2184 req = otx2_mbox_alloc_msg_cgx_ptp_rx_disable(&pfvf->mbox);
2185 if (!req) {
2186 mutex_unlock(&pfvf->mbox.lock);
2187 return -ENOMEM;
2188 }
2189
2190 err = otx2_sync_mbox_msg(&pfvf->mbox);
2191 if (err) {
2192 mutex_unlock(&pfvf->mbox.lock);
2193 return err;
2194 }
2195
2196 mutex_unlock(&pfvf->mbox.lock);
2197 if (enable)
2198 pfvf->flags |= OTX2_FLAG_RX_TSTAMP_ENABLED;
2199 else
2200 pfvf->flags &= ~OTX2_FLAG_RX_TSTAMP_ENABLED;
2201 return 0;
2202 }
2203
otx2_config_hw_tx_tstamp(struct otx2_nic * pfvf,bool enable)2204 static int otx2_config_hw_tx_tstamp(struct otx2_nic *pfvf, bool enable)
2205 {
2206 struct msg_req *req;
2207 int err;
2208
2209 if (pfvf->flags & OTX2_FLAG_TX_TSTAMP_ENABLED && enable)
2210 return 0;
2211
2212 mutex_lock(&pfvf->mbox.lock);
2213 if (enable)
2214 req = otx2_mbox_alloc_msg_nix_lf_ptp_tx_enable(&pfvf->mbox);
2215 else
2216 req = otx2_mbox_alloc_msg_nix_lf_ptp_tx_disable(&pfvf->mbox);
2217 if (!req) {
2218 mutex_unlock(&pfvf->mbox.lock);
2219 return -ENOMEM;
2220 }
2221
2222 err = otx2_sync_mbox_msg(&pfvf->mbox);
2223 if (err) {
2224 mutex_unlock(&pfvf->mbox.lock);
2225 return err;
2226 }
2227
2228 mutex_unlock(&pfvf->mbox.lock);
2229 if (enable)
2230 pfvf->flags |= OTX2_FLAG_TX_TSTAMP_ENABLED;
2231 else
2232 pfvf->flags &= ~OTX2_FLAG_TX_TSTAMP_ENABLED;
2233 return 0;
2234 }
2235
otx2_config_hwtstamp(struct net_device * netdev,struct ifreq * ifr)2236 int otx2_config_hwtstamp(struct net_device *netdev, struct ifreq *ifr)
2237 {
2238 struct otx2_nic *pfvf = netdev_priv(netdev);
2239 struct hwtstamp_config config;
2240
2241 if (!pfvf->ptp)
2242 return -ENODEV;
2243
2244 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
2245 return -EFAULT;
2246
2247 switch (config.tx_type) {
2248 case HWTSTAMP_TX_OFF:
2249 if (pfvf->flags & OTX2_FLAG_PTP_ONESTEP_SYNC)
2250 pfvf->flags &= ~OTX2_FLAG_PTP_ONESTEP_SYNC;
2251
2252 cancel_delayed_work(&pfvf->ptp->synctstamp_work);
2253 otx2_config_hw_tx_tstamp(pfvf, false);
2254 break;
2255 case HWTSTAMP_TX_ONESTEP_SYNC:
2256 if (!test_bit(CN10K_PTP_ONESTEP, &pfvf->hw.cap_flag))
2257 return -ERANGE;
2258 pfvf->flags |= OTX2_FLAG_PTP_ONESTEP_SYNC;
2259 schedule_delayed_work(&pfvf->ptp->synctstamp_work,
2260 msecs_to_jiffies(500));
2261 fallthrough;
2262 case HWTSTAMP_TX_ON:
2263 otx2_config_hw_tx_tstamp(pfvf, true);
2264 break;
2265 default:
2266 return -ERANGE;
2267 }
2268
2269 switch (config.rx_filter) {
2270 case HWTSTAMP_FILTER_NONE:
2271 otx2_config_hw_rx_tstamp(pfvf, false);
2272 break;
2273 case HWTSTAMP_FILTER_ALL:
2274 case HWTSTAMP_FILTER_SOME:
2275 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
2276 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
2277 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
2278 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
2279 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
2280 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
2281 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
2282 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
2283 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
2284 case HWTSTAMP_FILTER_PTP_V2_EVENT:
2285 case HWTSTAMP_FILTER_PTP_V2_SYNC:
2286 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
2287 otx2_config_hw_rx_tstamp(pfvf, true);
2288 config.rx_filter = HWTSTAMP_FILTER_ALL;
2289 break;
2290 default:
2291 return -ERANGE;
2292 }
2293
2294 memcpy(&pfvf->tstamp, &config, sizeof(config));
2295
2296 return copy_to_user(ifr->ifr_data, &config,
2297 sizeof(config)) ? -EFAULT : 0;
2298 }
2299 EXPORT_SYMBOL(otx2_config_hwtstamp);
2300
otx2_ioctl(struct net_device * netdev,struct ifreq * req,int cmd)2301 int otx2_ioctl(struct net_device *netdev, struct ifreq *req, int cmd)
2302 {
2303 struct otx2_nic *pfvf = netdev_priv(netdev);
2304 struct hwtstamp_config *cfg = &pfvf->tstamp;
2305
2306 switch (cmd) {
2307 case SIOCSHWTSTAMP:
2308 return otx2_config_hwtstamp(netdev, req);
2309 case SIOCGHWTSTAMP:
2310 return copy_to_user(req->ifr_data, cfg,
2311 sizeof(*cfg)) ? -EFAULT : 0;
2312 default:
2313 return -EOPNOTSUPP;
2314 }
2315 }
2316 EXPORT_SYMBOL(otx2_ioctl);
2317
otx2_do_set_vf_mac(struct otx2_nic * pf,int vf,const u8 * mac)2318 static int otx2_do_set_vf_mac(struct otx2_nic *pf, int vf, const u8 *mac)
2319 {
2320 struct npc_install_flow_req *req;
2321 int err;
2322
2323 mutex_lock(&pf->mbox.lock);
2324 req = otx2_mbox_alloc_msg_npc_install_flow(&pf->mbox);
2325 if (!req) {
2326 err = -ENOMEM;
2327 goto out;
2328 }
2329
2330 ether_addr_copy(req->packet.dmac, mac);
2331 eth_broadcast_addr((u8 *)&req->mask.dmac);
2332 req->features = BIT_ULL(NPC_DMAC);
2333 req->channel = pf->hw.rx_chan_base;
2334 req->intf = NIX_INTF_RX;
2335 req->default_rule = 1;
2336 req->append = 1;
2337 req->vf = vf + 1;
2338 req->op = NIX_RX_ACTION_DEFAULT;
2339
2340 err = otx2_sync_mbox_msg(&pf->mbox);
2341 out:
2342 mutex_unlock(&pf->mbox.lock);
2343 return err;
2344 }
2345
otx2_set_vf_mac(struct net_device * netdev,int vf,u8 * mac)2346 static int otx2_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
2347 {
2348 struct otx2_nic *pf = netdev_priv(netdev);
2349 struct pci_dev *pdev = pf->pdev;
2350 struct otx2_vf_config *config;
2351 int ret;
2352
2353 if (!netif_running(netdev))
2354 return -EAGAIN;
2355
2356 if (vf >= pf->total_vfs)
2357 return -EINVAL;
2358
2359 if (!is_valid_ether_addr(mac))
2360 return -EINVAL;
2361
2362 config = &pf->vf_configs[vf];
2363 ether_addr_copy(config->mac, mac);
2364
2365 ret = otx2_do_set_vf_mac(pf, vf, mac);
2366 if (ret == 0)
2367 dev_info(&pdev->dev,
2368 "Load/Reload VF driver\n");
2369
2370 return ret;
2371 }
2372
otx2_do_set_vf_vlan(struct otx2_nic * pf,int vf,u16 vlan,u8 qos,__be16 proto)2373 static int otx2_do_set_vf_vlan(struct otx2_nic *pf, int vf, u16 vlan, u8 qos,
2374 __be16 proto)
2375 {
2376 struct otx2_flow_config *flow_cfg = pf->flow_cfg;
2377 struct nix_vtag_config_rsp *vtag_rsp;
2378 struct npc_delete_flow_req *del_req;
2379 struct nix_vtag_config *vtag_req;
2380 struct npc_install_flow_req *req;
2381 struct otx2_vf_config *config;
2382 int err = 0;
2383 u32 idx;
2384
2385 config = &pf->vf_configs[vf];
2386
2387 if (!vlan && !config->vlan)
2388 goto out;
2389
2390 mutex_lock(&pf->mbox.lock);
2391
2392 /* free old tx vtag entry */
2393 if (config->vlan) {
2394 vtag_req = otx2_mbox_alloc_msg_nix_vtag_cfg(&pf->mbox);
2395 if (!vtag_req) {
2396 err = -ENOMEM;
2397 goto out;
2398 }
2399 vtag_req->cfg_type = 0;
2400 vtag_req->tx.free_vtag0 = 1;
2401 vtag_req->tx.vtag0_idx = config->tx_vtag_idx;
2402
2403 err = otx2_sync_mbox_msg(&pf->mbox);
2404 if (err)
2405 goto out;
2406 }
2407
2408 if (!vlan && config->vlan) {
2409 /* rx */
2410 del_req = otx2_mbox_alloc_msg_npc_delete_flow(&pf->mbox);
2411 if (!del_req) {
2412 err = -ENOMEM;
2413 goto out;
2414 }
2415 idx = ((vf * OTX2_PER_VF_VLAN_FLOWS) + OTX2_VF_VLAN_RX_INDEX);
2416 del_req->entry =
2417 flow_cfg->def_ent[flow_cfg->vf_vlan_offset + idx];
2418 err = otx2_sync_mbox_msg(&pf->mbox);
2419 if (err)
2420 goto out;
2421
2422 /* tx */
2423 del_req = otx2_mbox_alloc_msg_npc_delete_flow(&pf->mbox);
2424 if (!del_req) {
2425 err = -ENOMEM;
2426 goto out;
2427 }
2428 idx = ((vf * OTX2_PER_VF_VLAN_FLOWS) + OTX2_VF_VLAN_TX_INDEX);
2429 del_req->entry =
2430 flow_cfg->def_ent[flow_cfg->vf_vlan_offset + idx];
2431 err = otx2_sync_mbox_msg(&pf->mbox);
2432
2433 goto out;
2434 }
2435
2436 /* rx */
2437 req = otx2_mbox_alloc_msg_npc_install_flow(&pf->mbox);
2438 if (!req) {
2439 err = -ENOMEM;
2440 goto out;
2441 }
2442
2443 idx = ((vf * OTX2_PER_VF_VLAN_FLOWS) + OTX2_VF_VLAN_RX_INDEX);
2444 req->entry = flow_cfg->def_ent[flow_cfg->vf_vlan_offset + idx];
2445 req->packet.vlan_tci = htons(vlan);
2446 req->mask.vlan_tci = htons(VLAN_VID_MASK);
2447 /* af fills the destination mac addr */
2448 eth_broadcast_addr((u8 *)&req->mask.dmac);
2449 req->features = BIT_ULL(NPC_OUTER_VID) | BIT_ULL(NPC_DMAC);
2450 req->channel = pf->hw.rx_chan_base;
2451 req->intf = NIX_INTF_RX;
2452 req->vf = vf + 1;
2453 req->op = NIX_RX_ACTION_DEFAULT;
2454 req->vtag0_valid = true;
2455 req->vtag0_type = NIX_AF_LFX_RX_VTAG_TYPE7;
2456 req->set_cntr = 1;
2457
2458 err = otx2_sync_mbox_msg(&pf->mbox);
2459 if (err)
2460 goto out;
2461
2462 /* tx */
2463 vtag_req = otx2_mbox_alloc_msg_nix_vtag_cfg(&pf->mbox);
2464 if (!vtag_req) {
2465 err = -ENOMEM;
2466 goto out;
2467 }
2468
2469 /* configure tx vtag params */
2470 vtag_req->vtag_size = VTAGSIZE_T4;
2471 vtag_req->cfg_type = 0; /* tx vlan cfg */
2472 vtag_req->tx.cfg_vtag0 = 1;
2473 vtag_req->tx.vtag0 = ((u64)ntohs(proto) << 16) | vlan;
2474
2475 err = otx2_sync_mbox_msg(&pf->mbox);
2476 if (err)
2477 goto out;
2478
2479 vtag_rsp = (struct nix_vtag_config_rsp *)otx2_mbox_get_rsp
2480 (&pf->mbox.mbox, 0, &vtag_req->hdr);
2481 if (IS_ERR(vtag_rsp)) {
2482 err = PTR_ERR(vtag_rsp);
2483 goto out;
2484 }
2485 config->tx_vtag_idx = vtag_rsp->vtag0_idx;
2486
2487 req = otx2_mbox_alloc_msg_npc_install_flow(&pf->mbox);
2488 if (!req) {
2489 err = -ENOMEM;
2490 goto out;
2491 }
2492
2493 eth_zero_addr((u8 *)&req->mask.dmac);
2494 idx = ((vf * OTX2_PER_VF_VLAN_FLOWS) + OTX2_VF_VLAN_TX_INDEX);
2495 req->entry = flow_cfg->def_ent[flow_cfg->vf_vlan_offset + idx];
2496 req->features = BIT_ULL(NPC_DMAC);
2497 req->channel = pf->hw.tx_chan_base;
2498 req->intf = NIX_INTF_TX;
2499 req->vf = vf + 1;
2500 req->op = NIX_TX_ACTIONOP_UCAST_DEFAULT;
2501 req->vtag0_def = vtag_rsp->vtag0_idx;
2502 req->vtag0_op = VTAG_INSERT;
2503 req->set_cntr = 1;
2504
2505 err = otx2_sync_mbox_msg(&pf->mbox);
2506 out:
2507 config->vlan = vlan;
2508 mutex_unlock(&pf->mbox.lock);
2509 return err;
2510 }
2511
otx2_set_vf_vlan(struct net_device * netdev,int vf,u16 vlan,u8 qos,__be16 proto)2512 static int otx2_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos,
2513 __be16 proto)
2514 {
2515 struct otx2_nic *pf = netdev_priv(netdev);
2516 struct pci_dev *pdev = pf->pdev;
2517
2518 if (!netif_running(netdev))
2519 return -EAGAIN;
2520
2521 if (vf >= pci_num_vf(pdev))
2522 return -EINVAL;
2523
2524 /* qos is currently unsupported */
2525 if (vlan >= VLAN_N_VID || qos)
2526 return -EINVAL;
2527
2528 if (proto != htons(ETH_P_8021Q))
2529 return -EPROTONOSUPPORT;
2530
2531 if (!(pf->flags & OTX2_FLAG_VF_VLAN_SUPPORT))
2532 return -EOPNOTSUPP;
2533
2534 return otx2_do_set_vf_vlan(pf, vf, vlan, qos, proto);
2535 }
2536
otx2_get_vf_config(struct net_device * netdev,int vf,struct ifla_vf_info * ivi)2537 static int otx2_get_vf_config(struct net_device *netdev, int vf,
2538 struct ifla_vf_info *ivi)
2539 {
2540 struct otx2_nic *pf = netdev_priv(netdev);
2541 struct pci_dev *pdev = pf->pdev;
2542 struct otx2_vf_config *config;
2543
2544 if (!netif_running(netdev))
2545 return -EAGAIN;
2546
2547 if (vf >= pci_num_vf(pdev))
2548 return -EINVAL;
2549
2550 config = &pf->vf_configs[vf];
2551 ivi->vf = vf;
2552 ether_addr_copy(ivi->mac, config->mac);
2553 ivi->vlan = config->vlan;
2554 ivi->trusted = config->trusted;
2555
2556 return 0;
2557 }
2558
otx2_xdp_xmit_tx(struct otx2_nic * pf,struct xdp_frame * xdpf,int qidx)2559 static int otx2_xdp_xmit_tx(struct otx2_nic *pf, struct xdp_frame *xdpf,
2560 int qidx)
2561 {
2562 struct page *page;
2563 u64 dma_addr;
2564 int err = 0;
2565
2566 dma_addr = otx2_dma_map_page(pf, virt_to_page(xdpf->data),
2567 offset_in_page(xdpf->data), xdpf->len,
2568 DMA_TO_DEVICE);
2569 if (dma_mapping_error(pf->dev, dma_addr))
2570 return -ENOMEM;
2571
2572 err = otx2_xdp_sq_append_pkt(pf, dma_addr, xdpf->len, qidx);
2573 if (!err) {
2574 otx2_dma_unmap_page(pf, dma_addr, xdpf->len, DMA_TO_DEVICE);
2575 page = virt_to_page(xdpf->data);
2576 put_page(page);
2577 return -ENOMEM;
2578 }
2579 return 0;
2580 }
2581
otx2_xdp_xmit(struct net_device * netdev,int n,struct xdp_frame ** frames,u32 flags)2582 static int otx2_xdp_xmit(struct net_device *netdev, int n,
2583 struct xdp_frame **frames, u32 flags)
2584 {
2585 struct otx2_nic *pf = netdev_priv(netdev);
2586 int qidx = smp_processor_id();
2587 struct otx2_snd_queue *sq;
2588 int drops = 0, i;
2589
2590 if (!netif_running(netdev))
2591 return -ENETDOWN;
2592
2593 qidx += pf->hw.tx_queues;
2594 sq = pf->xdp_prog ? &pf->qset.sq[qidx] : NULL;
2595
2596 /* Abort xmit if xdp queue is not */
2597 if (unlikely(!sq))
2598 return -ENXIO;
2599
2600 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
2601 return -EINVAL;
2602
2603 for (i = 0; i < n; i++) {
2604 struct xdp_frame *xdpf = frames[i];
2605 int err;
2606
2607 err = otx2_xdp_xmit_tx(pf, xdpf, qidx);
2608 if (err)
2609 drops++;
2610 }
2611 return n - drops;
2612 }
2613
otx2_xdp_setup(struct otx2_nic * pf,struct bpf_prog * prog)2614 static int otx2_xdp_setup(struct otx2_nic *pf, struct bpf_prog *prog)
2615 {
2616 struct net_device *dev = pf->netdev;
2617 bool if_up = netif_running(pf->netdev);
2618 struct bpf_prog *old_prog;
2619
2620 if (prog && dev->mtu > MAX_XDP_MTU) {
2621 netdev_warn(dev, "Jumbo frames not yet supported with XDP\n");
2622 return -EOPNOTSUPP;
2623 }
2624
2625 if (if_up)
2626 otx2_stop(pf->netdev);
2627
2628 old_prog = xchg(&pf->xdp_prog, prog);
2629
2630 if (old_prog)
2631 bpf_prog_put(old_prog);
2632
2633 if (pf->xdp_prog)
2634 bpf_prog_add(pf->xdp_prog, pf->hw.rx_queues - 1);
2635
2636 /* Network stack and XDP shared same rx queues.
2637 * Use separate tx queues for XDP and network stack.
2638 */
2639 if (pf->xdp_prog) {
2640 pf->hw.xdp_queues = pf->hw.rx_queues;
2641 xdp_features_set_redirect_target(dev, false);
2642 } else {
2643 pf->hw.xdp_queues = 0;
2644 xdp_features_clear_redirect_target(dev);
2645 }
2646
2647 if (if_up)
2648 otx2_open(pf->netdev);
2649
2650 return 0;
2651 }
2652
otx2_xdp(struct net_device * netdev,struct netdev_bpf * xdp)2653 static int otx2_xdp(struct net_device *netdev, struct netdev_bpf *xdp)
2654 {
2655 struct otx2_nic *pf = netdev_priv(netdev);
2656
2657 switch (xdp->command) {
2658 case XDP_SETUP_PROG:
2659 return otx2_xdp_setup(pf, xdp->prog);
2660 default:
2661 return -EINVAL;
2662 }
2663 }
2664
otx2_set_vf_permissions(struct otx2_nic * pf,int vf,int req_perm)2665 static int otx2_set_vf_permissions(struct otx2_nic *pf, int vf,
2666 int req_perm)
2667 {
2668 struct set_vf_perm *req;
2669 int rc;
2670
2671 mutex_lock(&pf->mbox.lock);
2672 req = otx2_mbox_alloc_msg_set_vf_perm(&pf->mbox);
2673 if (!req) {
2674 rc = -ENOMEM;
2675 goto out;
2676 }
2677
2678 /* Let AF reset VF permissions as sriov is disabled */
2679 if (req_perm == OTX2_RESET_VF_PERM) {
2680 req->flags |= RESET_VF_PERM;
2681 } else if (req_perm == OTX2_TRUSTED_VF) {
2682 if (pf->vf_configs[vf].trusted)
2683 req->flags |= VF_TRUSTED;
2684 }
2685
2686 req->vf = vf;
2687 rc = otx2_sync_mbox_msg(&pf->mbox);
2688 out:
2689 mutex_unlock(&pf->mbox.lock);
2690 return rc;
2691 }
2692
otx2_ndo_set_vf_trust(struct net_device * netdev,int vf,bool enable)2693 static int otx2_ndo_set_vf_trust(struct net_device *netdev, int vf,
2694 bool enable)
2695 {
2696 struct otx2_nic *pf = netdev_priv(netdev);
2697 struct pci_dev *pdev = pf->pdev;
2698 int rc;
2699
2700 if (vf >= pci_num_vf(pdev))
2701 return -EINVAL;
2702
2703 if (pf->vf_configs[vf].trusted == enable)
2704 return 0;
2705
2706 pf->vf_configs[vf].trusted = enable;
2707 rc = otx2_set_vf_permissions(pf, vf, OTX2_TRUSTED_VF);
2708
2709 if (rc) {
2710 pf->vf_configs[vf].trusted = !enable;
2711 } else {
2712 netdev_info(pf->netdev, "VF %d is %strusted\n",
2713 vf, enable ? "" : "not ");
2714 otx2_set_rx_mode(netdev);
2715 }
2716
2717 return rc;
2718 }
2719
2720 static const struct net_device_ops otx2_netdev_ops = {
2721 .ndo_open = otx2_open,
2722 .ndo_stop = otx2_stop,
2723 .ndo_start_xmit = otx2_xmit,
2724 .ndo_select_queue = otx2_select_queue,
2725 .ndo_fix_features = otx2_fix_features,
2726 .ndo_set_mac_address = otx2_set_mac_address,
2727 .ndo_change_mtu = otx2_change_mtu,
2728 .ndo_set_rx_mode = otx2_set_rx_mode,
2729 .ndo_set_features = otx2_set_features,
2730 .ndo_tx_timeout = otx2_tx_timeout,
2731 .ndo_get_stats64 = otx2_get_stats64,
2732 .ndo_eth_ioctl = otx2_ioctl,
2733 .ndo_set_vf_mac = otx2_set_vf_mac,
2734 .ndo_set_vf_vlan = otx2_set_vf_vlan,
2735 .ndo_get_vf_config = otx2_get_vf_config,
2736 .ndo_bpf = otx2_xdp,
2737 .ndo_xdp_xmit = otx2_xdp_xmit,
2738 .ndo_setup_tc = otx2_setup_tc,
2739 .ndo_set_vf_trust = otx2_ndo_set_vf_trust,
2740 };
2741
otx2_wq_init(struct otx2_nic * pf)2742 static int otx2_wq_init(struct otx2_nic *pf)
2743 {
2744 pf->otx2_wq = create_singlethread_workqueue("otx2_wq");
2745 if (!pf->otx2_wq)
2746 return -ENOMEM;
2747
2748 INIT_WORK(&pf->rx_mode_work, otx2_rx_mode_wrk_handler);
2749 INIT_WORK(&pf->reset_task, otx2_reset_task);
2750 return 0;
2751 }
2752
otx2_check_pf_usable(struct otx2_nic * nic)2753 static int otx2_check_pf_usable(struct otx2_nic *nic)
2754 {
2755 u64 rev;
2756
2757 rev = otx2_read64(nic, RVU_PF_BLOCK_ADDRX_DISC(BLKADDR_RVUM));
2758 rev = (rev >> 12) & 0xFF;
2759 /* Check if AF has setup revision for RVUM block,
2760 * otherwise this driver probe should be deferred
2761 * until AF driver comes up.
2762 */
2763 if (!rev) {
2764 dev_warn(nic->dev,
2765 "AF is not initialized, deferring probe\n");
2766 return -EPROBE_DEFER;
2767 }
2768 return 0;
2769 }
2770
otx2_realloc_msix_vectors(struct otx2_nic * pf)2771 static int otx2_realloc_msix_vectors(struct otx2_nic *pf)
2772 {
2773 struct otx2_hw *hw = &pf->hw;
2774 int num_vec, err;
2775
2776 /* NPA interrupts are inot registered, so alloc only
2777 * upto NIX vector offset.
2778 */
2779 num_vec = hw->nix_msixoff;
2780 num_vec += NIX_LF_CINT_VEC_START + hw->max_queues;
2781
2782 otx2_disable_mbox_intr(pf);
2783 pci_free_irq_vectors(hw->pdev);
2784 err = pci_alloc_irq_vectors(hw->pdev, num_vec, num_vec, PCI_IRQ_MSIX);
2785 if (err < 0) {
2786 dev_err(pf->dev, "%s: Failed to realloc %d IRQ vectors\n",
2787 __func__, num_vec);
2788 return err;
2789 }
2790
2791 return otx2_register_mbox_intr(pf, false);
2792 }
2793
otx2_sriov_vfcfg_init(struct otx2_nic * pf)2794 static int otx2_sriov_vfcfg_init(struct otx2_nic *pf)
2795 {
2796 int i;
2797
2798 pf->vf_configs = devm_kcalloc(pf->dev, pf->total_vfs,
2799 sizeof(struct otx2_vf_config),
2800 GFP_KERNEL);
2801 if (!pf->vf_configs)
2802 return -ENOMEM;
2803
2804 for (i = 0; i < pf->total_vfs; i++) {
2805 pf->vf_configs[i].pf = pf;
2806 pf->vf_configs[i].intf_down = true;
2807 pf->vf_configs[i].trusted = false;
2808 INIT_DELAYED_WORK(&pf->vf_configs[i].link_event_work,
2809 otx2_vf_link_event_task);
2810 }
2811
2812 return 0;
2813 }
2814
otx2_sriov_vfcfg_cleanup(struct otx2_nic * pf)2815 static void otx2_sriov_vfcfg_cleanup(struct otx2_nic *pf)
2816 {
2817 int i;
2818
2819 if (!pf->vf_configs)
2820 return;
2821
2822 for (i = 0; i < pf->total_vfs; i++) {
2823 cancel_delayed_work_sync(&pf->vf_configs[i].link_event_work);
2824 otx2_set_vf_permissions(pf, i, OTX2_RESET_VF_PERM);
2825 }
2826 }
2827
otx2_probe(struct pci_dev * pdev,const struct pci_device_id * id)2828 static int otx2_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2829 {
2830 struct device *dev = &pdev->dev;
2831 int err, qcount, qos_txqs;
2832 struct net_device *netdev;
2833 struct otx2_nic *pf;
2834 struct otx2_hw *hw;
2835 int num_vec;
2836
2837 err = pcim_enable_device(pdev);
2838 if (err) {
2839 dev_err(dev, "Failed to enable PCI device\n");
2840 return err;
2841 }
2842
2843 err = pci_request_regions(pdev, DRV_NAME);
2844 if (err) {
2845 dev_err(dev, "PCI request regions failed 0x%x\n", err);
2846 return err;
2847 }
2848
2849 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(48));
2850 if (err) {
2851 dev_err(dev, "DMA mask config failed, abort\n");
2852 goto err_release_regions;
2853 }
2854
2855 pci_set_master(pdev);
2856
2857 /* Set number of queues */
2858 qcount = min_t(int, num_online_cpus(), OTX2_MAX_CQ_CNT);
2859 qos_txqs = min_t(int, qcount, OTX2_QOS_MAX_LEAF_NODES);
2860
2861 netdev = alloc_etherdev_mqs(sizeof(*pf), qcount + qos_txqs, qcount);
2862 if (!netdev) {
2863 err = -ENOMEM;
2864 goto err_release_regions;
2865 }
2866
2867 pci_set_drvdata(pdev, netdev);
2868 SET_NETDEV_DEV(netdev, &pdev->dev);
2869 pf = netdev_priv(netdev);
2870 pf->netdev = netdev;
2871 pf->pdev = pdev;
2872 pf->dev = dev;
2873 pf->total_vfs = pci_sriov_get_totalvfs(pdev);
2874 pf->flags |= OTX2_FLAG_INTF_DOWN;
2875
2876 hw = &pf->hw;
2877 hw->pdev = pdev;
2878 hw->rx_queues = qcount;
2879 hw->tx_queues = qcount;
2880 hw->non_qos_queues = qcount;
2881 hw->max_queues = qcount;
2882 hw->rbuf_len = OTX2_DEFAULT_RBUF_LEN;
2883 /* Use CQE of 128 byte descriptor size by default */
2884 hw->xqe_size = 128;
2885
2886 num_vec = pci_msix_vec_count(pdev);
2887 hw->irq_name = devm_kmalloc_array(&hw->pdev->dev, num_vec, NAME_SIZE,
2888 GFP_KERNEL);
2889 if (!hw->irq_name) {
2890 err = -ENOMEM;
2891 goto err_free_netdev;
2892 }
2893
2894 hw->affinity_mask = devm_kcalloc(&hw->pdev->dev, num_vec,
2895 sizeof(cpumask_var_t), GFP_KERNEL);
2896 if (!hw->affinity_mask) {
2897 err = -ENOMEM;
2898 goto err_free_netdev;
2899 }
2900
2901 /* Map CSRs */
2902 pf->reg_base = pcim_iomap(pdev, PCI_CFG_REG_BAR_NUM, 0);
2903 if (!pf->reg_base) {
2904 dev_err(dev, "Unable to map physical function CSRs, aborting\n");
2905 err = -ENOMEM;
2906 goto err_free_netdev;
2907 }
2908
2909 err = otx2_check_pf_usable(pf);
2910 if (err)
2911 goto err_free_netdev;
2912
2913 err = pci_alloc_irq_vectors(hw->pdev, RVU_PF_INT_VEC_CNT,
2914 RVU_PF_INT_VEC_CNT, PCI_IRQ_MSIX);
2915 if (err < 0) {
2916 dev_err(dev, "%s: Failed to alloc %d IRQ vectors\n",
2917 __func__, num_vec);
2918 goto err_free_netdev;
2919 }
2920
2921 otx2_setup_dev_hw_settings(pf);
2922
2923 /* Init PF <=> AF mailbox stuff */
2924 err = otx2_pfaf_mbox_init(pf);
2925 if (err)
2926 goto err_free_irq_vectors;
2927
2928 /* Register mailbox interrupt */
2929 err = otx2_register_mbox_intr(pf, true);
2930 if (err)
2931 goto err_mbox_destroy;
2932
2933 /* Request AF to attach NPA and NIX LFs to this PF.
2934 * NIX and NPA LFs are needed for this PF to function as a NIC.
2935 */
2936 err = otx2_attach_npa_nix(pf);
2937 if (err)
2938 goto err_disable_mbox_intr;
2939
2940 err = otx2_realloc_msix_vectors(pf);
2941 if (err)
2942 goto err_detach_rsrc;
2943
2944 err = otx2_set_real_num_queues(netdev, hw->tx_queues, hw->rx_queues);
2945 if (err)
2946 goto err_detach_rsrc;
2947
2948 err = cn10k_lmtst_init(pf);
2949 if (err)
2950 goto err_detach_rsrc;
2951
2952 /* Assign default mac address */
2953 otx2_get_mac_from_af(netdev);
2954
2955 /* Don't check for error. Proceed without ptp */
2956 otx2_ptp_init(pf);
2957
2958 /* NPA's pool is a stack to which SW frees buffer pointers via Aura.
2959 * HW allocates buffer pointer from stack and uses it for DMA'ing
2960 * ingress packet. In some scenarios HW can free back allocated buffer
2961 * pointers to pool. This makes it impossible for SW to maintain a
2962 * parallel list where physical addresses of buffer pointers (IOVAs)
2963 * given to HW can be saved for later reference.
2964 *
2965 * So the only way to convert Rx packet's buffer address is to use
2966 * IOMMU's iova_to_phys() handler which translates the address by
2967 * walking through the translation tables.
2968 */
2969 pf->iommu_domain = iommu_get_domain_for_dev(dev);
2970
2971 netdev->hw_features = (NETIF_F_RXCSUM | NETIF_F_IP_CSUM |
2972 NETIF_F_IPV6_CSUM | NETIF_F_RXHASH |
2973 NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
2974 NETIF_F_GSO_UDP_L4);
2975 netdev->features |= netdev->hw_features;
2976
2977 err = otx2_mcam_flow_init(pf);
2978 if (err)
2979 goto err_ptp_destroy;
2980
2981 err = cn10k_mcs_init(pf);
2982 if (err)
2983 goto err_del_mcam_entries;
2984
2985 if (pf->flags & OTX2_FLAG_NTUPLE_SUPPORT)
2986 netdev->hw_features |= NETIF_F_NTUPLE;
2987
2988 if (pf->flags & OTX2_FLAG_UCAST_FLTR_SUPPORT)
2989 netdev->priv_flags |= IFF_UNICAST_FLT;
2990
2991 /* Support TSO on tag interface */
2992 netdev->vlan_features |= netdev->features;
2993 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX |
2994 NETIF_F_HW_VLAN_STAG_TX;
2995 if (pf->flags & OTX2_FLAG_RX_VLAN_SUPPORT)
2996 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX |
2997 NETIF_F_HW_VLAN_STAG_RX;
2998 netdev->features |= netdev->hw_features;
2999
3000 /* HW supports tc offload but mutually exclusive with n-tuple filters */
3001 if (pf->flags & OTX2_FLAG_TC_FLOWER_SUPPORT)
3002 netdev->hw_features |= NETIF_F_HW_TC;
3003
3004 netdev->hw_features |= NETIF_F_LOOPBACK | NETIF_F_RXALL;
3005
3006 netif_set_tso_max_segs(netdev, OTX2_MAX_GSO_SEGS);
3007 netdev->watchdog_timeo = OTX2_TX_TIMEOUT;
3008
3009 netdev->netdev_ops = &otx2_netdev_ops;
3010 netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT;
3011
3012 netdev->min_mtu = OTX2_MIN_MTU;
3013 netdev->max_mtu = otx2_get_max_mtu(pf);
3014
3015 err = register_netdev(netdev);
3016 if (err) {
3017 dev_err(dev, "Failed to register netdevice\n");
3018 goto err_mcs_free;
3019 }
3020
3021 err = otx2_wq_init(pf);
3022 if (err)
3023 goto err_unreg_netdev;
3024
3025 otx2_set_ethtool_ops(netdev);
3026
3027 err = otx2_init_tc(pf);
3028 if (err)
3029 goto err_mcam_flow_del;
3030
3031 err = otx2_register_dl(pf);
3032 if (err)
3033 goto err_mcam_flow_del;
3034
3035 /* Initialize SR-IOV resources */
3036 err = otx2_sriov_vfcfg_init(pf);
3037 if (err)
3038 goto err_pf_sriov_init;
3039
3040 /* Enable link notifications */
3041 otx2_cgx_config_linkevents(pf, true);
3042
3043 #ifdef CONFIG_DCB
3044 err = otx2_dcbnl_set_ops(netdev);
3045 if (err)
3046 goto err_pf_sriov_init;
3047 #endif
3048
3049 otx2_qos_init(pf, qos_txqs);
3050
3051 return 0;
3052
3053 err_pf_sriov_init:
3054 otx2_shutdown_tc(pf);
3055 err_mcam_flow_del:
3056 otx2_mcam_flow_del(pf);
3057 err_unreg_netdev:
3058 unregister_netdev(netdev);
3059 err_mcs_free:
3060 cn10k_mcs_free(pf);
3061 err_del_mcam_entries:
3062 otx2_mcam_flow_del(pf);
3063 err_ptp_destroy:
3064 otx2_ptp_destroy(pf);
3065 err_detach_rsrc:
3066 if (pf->hw.lmt_info)
3067 free_percpu(pf->hw.lmt_info);
3068 if (test_bit(CN10K_LMTST, &pf->hw.cap_flag))
3069 qmem_free(pf->dev, pf->dync_lmt);
3070 otx2_detach_resources(&pf->mbox);
3071 err_disable_mbox_intr:
3072 otx2_disable_mbox_intr(pf);
3073 err_mbox_destroy:
3074 otx2_pfaf_mbox_destroy(pf);
3075 err_free_irq_vectors:
3076 pci_free_irq_vectors(hw->pdev);
3077 err_free_netdev:
3078 pci_set_drvdata(pdev, NULL);
3079 free_netdev(netdev);
3080 err_release_regions:
3081 pci_release_regions(pdev);
3082 return err;
3083 }
3084
otx2_vf_link_event_task(struct work_struct * work)3085 static void otx2_vf_link_event_task(struct work_struct *work)
3086 {
3087 struct otx2_vf_config *config;
3088 struct cgx_link_info_msg *req;
3089 struct mbox_msghdr *msghdr;
3090 struct otx2_nic *pf;
3091 int vf_idx;
3092
3093 config = container_of(work, struct otx2_vf_config,
3094 link_event_work.work);
3095 vf_idx = config - config->pf->vf_configs;
3096 pf = config->pf;
3097
3098 msghdr = otx2_mbox_alloc_msg_rsp(&pf->mbox_pfvf[0].mbox_up, vf_idx,
3099 sizeof(*req), sizeof(struct msg_rsp));
3100 if (!msghdr) {
3101 dev_err(pf->dev, "Failed to create VF%d link event\n", vf_idx);
3102 return;
3103 }
3104
3105 req = (struct cgx_link_info_msg *)msghdr;
3106 req->hdr.id = MBOX_MSG_CGX_LINK_EVENT;
3107 req->hdr.sig = OTX2_MBOX_REQ_SIG;
3108 memcpy(&req->link_info, &pf->linfo, sizeof(req->link_info));
3109
3110 otx2_sync_mbox_up_msg(&pf->mbox_pfvf[0], vf_idx);
3111 }
3112
otx2_sriov_enable(struct pci_dev * pdev,int numvfs)3113 static int otx2_sriov_enable(struct pci_dev *pdev, int numvfs)
3114 {
3115 struct net_device *netdev = pci_get_drvdata(pdev);
3116 struct otx2_nic *pf = netdev_priv(netdev);
3117 int ret;
3118
3119 /* Init PF <=> VF mailbox stuff */
3120 ret = otx2_pfvf_mbox_init(pf, numvfs);
3121 if (ret)
3122 return ret;
3123
3124 ret = otx2_register_pfvf_mbox_intr(pf, numvfs);
3125 if (ret)
3126 goto free_mbox;
3127
3128 ret = otx2_pf_flr_init(pf, numvfs);
3129 if (ret)
3130 goto free_intr;
3131
3132 ret = otx2_register_flr_me_intr(pf, numvfs);
3133 if (ret)
3134 goto free_flr;
3135
3136 ret = pci_enable_sriov(pdev, numvfs);
3137 if (ret)
3138 goto free_flr_intr;
3139
3140 return numvfs;
3141 free_flr_intr:
3142 otx2_disable_flr_me_intr(pf);
3143 free_flr:
3144 otx2_flr_wq_destroy(pf);
3145 free_intr:
3146 otx2_disable_pfvf_mbox_intr(pf, numvfs);
3147 free_mbox:
3148 otx2_pfvf_mbox_destroy(pf);
3149 return ret;
3150 }
3151
otx2_sriov_disable(struct pci_dev * pdev)3152 static int otx2_sriov_disable(struct pci_dev *pdev)
3153 {
3154 struct net_device *netdev = pci_get_drvdata(pdev);
3155 struct otx2_nic *pf = netdev_priv(netdev);
3156 int numvfs = pci_num_vf(pdev);
3157
3158 if (!numvfs)
3159 return 0;
3160
3161 pci_disable_sriov(pdev);
3162
3163 otx2_disable_flr_me_intr(pf);
3164 otx2_flr_wq_destroy(pf);
3165 otx2_disable_pfvf_mbox_intr(pf, numvfs);
3166 otx2_pfvf_mbox_destroy(pf);
3167
3168 return 0;
3169 }
3170
otx2_sriov_configure(struct pci_dev * pdev,int numvfs)3171 static int otx2_sriov_configure(struct pci_dev *pdev, int numvfs)
3172 {
3173 if (numvfs == 0)
3174 return otx2_sriov_disable(pdev);
3175 else
3176 return otx2_sriov_enable(pdev, numvfs);
3177 }
3178
otx2_remove(struct pci_dev * pdev)3179 static void otx2_remove(struct pci_dev *pdev)
3180 {
3181 struct net_device *netdev = pci_get_drvdata(pdev);
3182 struct otx2_nic *pf;
3183
3184 if (!netdev)
3185 return;
3186
3187 pf = netdev_priv(netdev);
3188
3189 pf->flags |= OTX2_FLAG_PF_SHUTDOWN;
3190
3191 if (pf->flags & OTX2_FLAG_TX_TSTAMP_ENABLED)
3192 otx2_config_hw_tx_tstamp(pf, false);
3193 if (pf->flags & OTX2_FLAG_RX_TSTAMP_ENABLED)
3194 otx2_config_hw_rx_tstamp(pf, false);
3195
3196 /* Disable 802.3x pause frames */
3197 if (pf->flags & OTX2_FLAG_RX_PAUSE_ENABLED ||
3198 (pf->flags & OTX2_FLAG_TX_PAUSE_ENABLED)) {
3199 pf->flags &= ~OTX2_FLAG_RX_PAUSE_ENABLED;
3200 pf->flags &= ~OTX2_FLAG_TX_PAUSE_ENABLED;
3201 otx2_config_pause_frm(pf);
3202 }
3203
3204 #ifdef CONFIG_DCB
3205 /* Disable PFC config */
3206 if (pf->pfc_en) {
3207 pf->pfc_en = 0;
3208 otx2_config_priority_flow_ctrl(pf);
3209 }
3210 #endif
3211 cancel_work_sync(&pf->reset_task);
3212 /* Disable link notifications */
3213 otx2_cgx_config_linkevents(pf, false);
3214
3215 otx2_unregister_dl(pf);
3216 unregister_netdev(netdev);
3217 cn10k_mcs_free(pf);
3218 otx2_sriov_disable(pf->pdev);
3219 otx2_sriov_vfcfg_cleanup(pf);
3220 if (pf->otx2_wq)
3221 destroy_workqueue(pf->otx2_wq);
3222
3223 otx2_ptp_destroy(pf);
3224 otx2_mcam_flow_del(pf);
3225 otx2_shutdown_tc(pf);
3226 otx2_shutdown_qos(pf);
3227 otx2_detach_resources(&pf->mbox);
3228 if (pf->hw.lmt_info)
3229 free_percpu(pf->hw.lmt_info);
3230 if (test_bit(CN10K_LMTST, &pf->hw.cap_flag))
3231 qmem_free(pf->dev, pf->dync_lmt);
3232 otx2_disable_mbox_intr(pf);
3233 otx2_pfaf_mbox_destroy(pf);
3234 pci_free_irq_vectors(pf->pdev);
3235 pci_set_drvdata(pdev, NULL);
3236 free_netdev(netdev);
3237
3238 pci_release_regions(pdev);
3239 }
3240
3241 static struct pci_driver otx2_pf_driver = {
3242 .name = DRV_NAME,
3243 .id_table = otx2_pf_id_table,
3244 .probe = otx2_probe,
3245 .shutdown = otx2_remove,
3246 .remove = otx2_remove,
3247 .sriov_configure = otx2_sriov_configure
3248 };
3249
otx2_rvupf_init_module(void)3250 static int __init otx2_rvupf_init_module(void)
3251 {
3252 pr_info("%s: %s\n", DRV_NAME, DRV_STRING);
3253
3254 return pci_register_driver(&otx2_pf_driver);
3255 }
3256
otx2_rvupf_cleanup_module(void)3257 static void __exit otx2_rvupf_cleanup_module(void)
3258 {
3259 pci_unregister_driver(&otx2_pf_driver);
3260 }
3261
3262 module_init(otx2_rvupf_init_module);
3263 module_exit(otx2_rvupf_cleanup_module);
3264