1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (C) 2013 Emulex
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions are met:
9 *
10 * 1. Redistributions of source code must retain the above copyright notice,
11 * this list of conditions and the following disclaimer.
12 *
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * 3. Neither the name of the Emulex Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived from
19 * this software without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31 * POSSIBILITY OF SUCH DAMAGE.
32 *
33 * Contact Information:
34 * freebsd-drivers@emulex.com
35 *
36 * Emulex
37 * 3333 Susan Street
38 * Costa Mesa, CA 92626
39 */
40
41
42 #include "opt_inet6.h"
43 #include "opt_inet.h"
44
45 #include "oce_if.h"
46 #include "oce_user.h"
47
48 #define is_tso_pkt(m) (m->m_pkthdr.csum_flags & CSUM_TSO)
49
50 /* UE Status Low CSR */
51 static char *ue_status_low_desc[] = {
52 "CEV",
53 "CTX",
54 "DBUF",
55 "ERX",
56 "Host",
57 "MPU",
58 "NDMA",
59 "PTC ",
60 "RDMA ",
61 "RXF ",
62 "RXIPS ",
63 "RXULP0 ",
64 "RXULP1 ",
65 "RXULP2 ",
66 "TIM ",
67 "TPOST ",
68 "TPRE ",
69 "TXIPS ",
70 "TXULP0 ",
71 "TXULP1 ",
72 "UC ",
73 "WDMA ",
74 "TXULP2 ",
75 "HOST1 ",
76 "P0_OB_LINK ",
77 "P1_OB_LINK ",
78 "HOST_GPIO ",
79 "MBOX ",
80 "AXGMAC0",
81 "AXGMAC1",
82 "JTAG",
83 "MPU_INTPEND"
84 };
85
86 /* UE Status High CSR */
87 static char *ue_status_hi_desc[] = {
88 "LPCMEMHOST",
89 "MGMT_MAC",
90 "PCS0ONLINE",
91 "MPU_IRAM",
92 "PCS1ONLINE",
93 "PCTL0",
94 "PCTL1",
95 "PMEM",
96 "RR",
97 "TXPB",
98 "RXPP",
99 "XAUI",
100 "TXP",
101 "ARM",
102 "IPC",
103 "HOST2",
104 "HOST3",
105 "HOST4",
106 "HOST5",
107 "HOST6",
108 "HOST7",
109 "HOST8",
110 "HOST9",
111 "NETC",
112 "Unknown",
113 "Unknown",
114 "Unknown",
115 "Unknown",
116 "Unknown",
117 "Unknown",
118 "Unknown",
119 "Unknown"
120 };
121
122 struct oce_common_cqe_info{
123 uint8_t vtp:1;
124 uint8_t l4_cksum_pass:1;
125 uint8_t ip_cksum_pass:1;
126 uint8_t ipv6_frame:1;
127 uint8_t qnq:1;
128 uint8_t rsvd:3;
129 uint8_t num_frags;
130 uint16_t pkt_size;
131 uint16_t vtag;
132 };
133
134 /* Driver entry points prototypes */
135 static int oce_probe(device_t dev);
136 static int oce_attach(device_t dev);
137 static int oce_detach(device_t dev);
138 static int oce_shutdown(device_t dev);
139 static int oce_ioctl(if_t ifp, u_long command, caddr_t data);
140 static void oce_init(void *xsc);
141 static int oce_multiq_start(if_t ifp, struct mbuf *m);
142 static void oce_multiq_flush(if_t ifp);
143
144 /* Driver interrupt routines protypes */
145 static void oce_intr(void *arg, int pending);
146 static int oce_setup_intr(POCE_SOFTC sc);
147 static int oce_fast_isr(void *arg);
148 static int oce_alloc_intr(POCE_SOFTC sc, int vector,
149 void (*isr) (void *arg, int pending));
150
151 /* Media callbacks prototypes */
152 static void oce_media_status(if_t ifp, struct ifmediareq *req);
153 static int oce_media_change(if_t ifp);
154
155 /* Transmit routines prototypes */
156 static int oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index);
157 static void oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq);
158 static void oce_process_tx_completion(struct oce_wq *wq);
159 static int oce_multiq_transmit(if_t ifp, struct mbuf *m,
160 struct oce_wq *wq);
161
162 /* Receive routines prototypes */
163 static int oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe);
164 static int oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe);
165 static void oce_rx(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe);
166 static void oce_check_rx_bufs(POCE_SOFTC sc, uint32_t num_cqes, struct oce_rq *rq);
167 static uint16_t oce_rq_handler_lro(void *arg);
168 static void oce_correct_header(struct mbuf *m, struct nic_hwlro_cqe_part1 *cqe1, struct nic_hwlro_cqe_part2 *cqe2);
169 static void oce_rx_lro(struct oce_rq *rq, struct nic_hwlro_singleton_cqe *cqe, struct nic_hwlro_cqe_part2 *cqe2);
170 static void oce_rx_mbuf_chain(struct oce_rq *rq, struct oce_common_cqe_info *cqe_info, struct mbuf **m);
171
172 /* Helper function prototypes in this file */
173 static void oce_attach_ifp(POCE_SOFTC sc);
174 static void oce_add_vlan(void *arg, if_t ifp, uint16_t vtag);
175 static void oce_del_vlan(void *arg, if_t ifp, uint16_t vtag);
176 static int oce_vid_config(POCE_SOFTC sc);
177 static void oce_mac_addr_set(POCE_SOFTC sc);
178 static int oce_handle_passthrough(if_t ifp, caddr_t data);
179 static void oce_local_timer(void *arg);
180 static void oce_if_deactivate(POCE_SOFTC sc);
181 static void oce_if_activate(POCE_SOFTC sc);
182 static void setup_max_queues_want(POCE_SOFTC sc);
183 static void update_queues_got(POCE_SOFTC sc);
184 static void process_link_state(POCE_SOFTC sc,
185 struct oce_async_cqe_link_state *acqe);
186 static int oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m);
187 static void oce_get_config(POCE_SOFTC sc);
188 static struct mbuf *oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete);
189 static void oce_read_env_variables(POCE_SOFTC sc);
190
191 /* IP specific */
192 #if defined(INET6) || defined(INET)
193 static int oce_init_lro(POCE_SOFTC sc);
194 static struct mbuf * oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp);
195 #endif
196
197 static device_method_t oce_dispatch[] = {
198 DEVMETHOD(device_probe, oce_probe),
199 DEVMETHOD(device_attach, oce_attach),
200 DEVMETHOD(device_detach, oce_detach),
201 DEVMETHOD(device_shutdown, oce_shutdown),
202
203 DEVMETHOD_END
204 };
205
206 static driver_t oce_driver = {
207 "oce",
208 oce_dispatch,
209 sizeof(OCE_SOFTC)
210 };
211
212 /* global vars */
213 const char component_revision[32] = {"///" COMPONENT_REVISION "///"};
214
215 /* Module capabilites and parameters */
216 uint32_t oce_max_rsp_handled = OCE_MAX_RSP_HANDLED;
217 uint32_t oce_enable_rss = OCE_MODCAP_RSS;
218 uint32_t oce_rq_buf_size = 2048;
219
220 TUNABLE_INT("hw.oce.max_rsp_handled", &oce_max_rsp_handled);
221 TUNABLE_INT("hw.oce.enable_rss", &oce_enable_rss);
222
223 /* Supported devices table */
224 static uint32_t supportedDevices[] = {
225 (PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE2,
226 (PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE3,
227 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_BE3,
228 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201,
229 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201_VF,
230 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_SH
231 };
232
233 DRIVER_MODULE(oce, pci, oce_driver, 0, 0);
234 MODULE_PNP_INFO("W32:vendor/device", pci, oce, supportedDevices,
235 nitems(supportedDevices));
236 MODULE_DEPEND(oce, pci, 1, 1, 1);
237 MODULE_DEPEND(oce, ether, 1, 1, 1);
238 MODULE_VERSION(oce, 1);
239
240 POCE_SOFTC softc_head = NULL;
241 POCE_SOFTC softc_tail = NULL;
242
243 struct oce_rdma_if *oce_rdma_if = NULL;
244
245 /*****************************************************************************
246 * Driver entry points functions *
247 *****************************************************************************/
248
249 static int
oce_probe(device_t dev)250 oce_probe(device_t dev)
251 {
252 uint16_t vendor = 0;
253 uint16_t device = 0;
254 int i = 0;
255 POCE_SOFTC sc;
256
257 sc = device_get_softc(dev);
258 bzero(sc, sizeof(OCE_SOFTC));
259 sc->dev = dev;
260
261 vendor = pci_get_vendor(dev);
262 device = pci_get_device(dev);
263
264 for (i = 0; i < (sizeof(supportedDevices) / sizeof(uint32_t)); i++) {
265 if (vendor == ((supportedDevices[i] >> 16) & 0xffff)) {
266 if (device == (supportedDevices[i] & 0xffff)) {
267 device_set_descf(dev,
268 "%s:%s", "Emulex CNA NIC function",
269 component_revision);
270
271 switch (device) {
272 case PCI_PRODUCT_BE2:
273 sc->flags |= OCE_FLAGS_BE2;
274 break;
275 case PCI_PRODUCT_BE3:
276 sc->flags |= OCE_FLAGS_BE3;
277 break;
278 case PCI_PRODUCT_XE201:
279 case PCI_PRODUCT_XE201_VF:
280 sc->flags |= OCE_FLAGS_XE201;
281 break;
282 case PCI_PRODUCT_SH:
283 sc->flags |= OCE_FLAGS_SH;
284 break;
285 default:
286 return ENXIO;
287 }
288 return BUS_PROBE_DEFAULT;
289 }
290 }
291 }
292
293 return ENXIO;
294 }
295
296 static int
oce_attach(device_t dev)297 oce_attach(device_t dev)
298 {
299 POCE_SOFTC sc;
300 int rc = 0;
301
302 sc = device_get_softc(dev);
303
304 rc = oce_hw_pci_alloc(sc);
305 if (rc)
306 return rc;
307
308 sc->tx_ring_size = OCE_TX_RING_SIZE;
309 sc->rx_ring_size = OCE_RX_RING_SIZE;
310 /* receive fragment size should be multiple of 2K */
311 sc->rq_frag_size = ((oce_rq_buf_size / 2048) * 2048);
312 sc->flow_control = OCE_DEFAULT_FLOW_CONTROL;
313 sc->promisc = OCE_DEFAULT_PROMISCUOUS;
314
315 LOCK_CREATE(&sc->bmbx_lock, "Mailbox_lock");
316 LOCK_CREATE(&sc->dev_lock, "Device_lock");
317
318 /* initialise the hardware */
319 rc = oce_hw_init(sc);
320 if (rc)
321 goto pci_res_free;
322
323 oce_read_env_variables(sc);
324
325 oce_get_config(sc);
326
327 setup_max_queues_want(sc);
328
329 rc = oce_setup_intr(sc);
330 if (rc)
331 goto mbox_free;
332
333 rc = oce_queue_init_all(sc);
334 if (rc)
335 goto intr_free;
336
337 oce_attach_ifp(sc);
338
339 #if defined(INET6) || defined(INET)
340 rc = oce_init_lro(sc);
341 if (rc)
342 goto ifp_free;
343 #endif
344
345 rc = oce_hw_start(sc);
346 if (rc)
347 goto lro_free;
348
349 sc->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
350 oce_add_vlan, sc, EVENTHANDLER_PRI_FIRST);
351 sc->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
352 oce_del_vlan, sc, EVENTHANDLER_PRI_FIRST);
353
354 rc = oce_stats_init(sc);
355 if (rc)
356 goto vlan_free;
357
358 oce_add_sysctls(sc);
359
360 callout_init(&sc->timer, CALLOUT_MPSAFE);
361 rc = callout_reset(&sc->timer, 2 * hz, oce_local_timer, sc);
362 if (rc)
363 goto stats_free;
364
365 sc->next =NULL;
366 if (softc_tail != NULL) {
367 softc_tail->next = sc;
368 } else {
369 softc_head = sc;
370 }
371 softc_tail = sc;
372
373 return 0;
374
375 stats_free:
376 callout_drain(&sc->timer);
377 oce_stats_free(sc);
378 vlan_free:
379 if (sc->vlan_attach)
380 EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach);
381 if (sc->vlan_detach)
382 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach);
383 oce_hw_intr_disable(sc);
384 lro_free:
385 #if defined(INET6) || defined(INET)
386 oce_free_lro(sc);
387 ifp_free:
388 #endif
389 ether_ifdetach(sc->ifp);
390 if_free(sc->ifp);
391 oce_queue_release_all(sc);
392 intr_free:
393 oce_intr_free(sc);
394 mbox_free:
395 oce_dma_free(sc, &sc->bsmbx);
396 pci_res_free:
397 oce_hw_pci_free(sc);
398 LOCK_DESTROY(&sc->dev_lock);
399 LOCK_DESTROY(&sc->bmbx_lock);
400 return rc;
401
402 }
403
404 static int
oce_detach(device_t dev)405 oce_detach(device_t dev)
406 {
407 POCE_SOFTC sc = device_get_softc(dev);
408 POCE_SOFTC poce_sc_tmp, *ppoce_sc_tmp1, poce_sc_tmp2 = NULL;
409
410 poce_sc_tmp = softc_head;
411 ppoce_sc_tmp1 = &softc_head;
412 while (poce_sc_tmp != NULL) {
413 if (poce_sc_tmp == sc) {
414 *ppoce_sc_tmp1 = sc->next;
415 if (sc->next == NULL) {
416 softc_tail = poce_sc_tmp2;
417 }
418 break;
419 }
420 poce_sc_tmp2 = poce_sc_tmp;
421 ppoce_sc_tmp1 = &poce_sc_tmp->next;
422 poce_sc_tmp = poce_sc_tmp->next;
423 }
424
425 LOCK(&sc->dev_lock);
426 oce_if_deactivate(sc);
427 UNLOCK(&sc->dev_lock);
428
429 callout_drain(&sc->timer);
430
431 if (sc->vlan_attach != NULL)
432 EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach);
433 if (sc->vlan_detach != NULL)
434 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach);
435
436 ether_ifdetach(sc->ifp);
437
438 if_free(sc->ifp);
439
440 oce_hw_shutdown(sc);
441
442 bus_generic_detach(dev);
443
444 return 0;
445 }
446
447 static int
oce_shutdown(device_t dev)448 oce_shutdown(device_t dev)
449 {
450 int rc;
451
452 rc = oce_detach(dev);
453
454 return rc;
455 }
456
457 static int
oce_ioctl(if_t ifp,u_long command,caddr_t data)458 oce_ioctl(if_t ifp, u_long command, caddr_t data)
459 {
460 struct ifreq *ifr = (struct ifreq *)data;
461 POCE_SOFTC sc = if_getsoftc(ifp);
462 struct ifi2creq i2c;
463 uint8_t offset = 0;
464 int rc = 0;
465 uint32_t u;
466
467 switch (command) {
468 case SIOCGIFMEDIA:
469 rc = ifmedia_ioctl(ifp, ifr, &sc->media, command);
470 break;
471
472 case SIOCSIFMTU:
473 if (ifr->ifr_mtu > OCE_MAX_MTU)
474 rc = EINVAL;
475 else
476 if_setmtu(ifp, ifr->ifr_mtu);
477 break;
478
479 case SIOCSIFFLAGS:
480 if (if_getflags(ifp) & IFF_UP) {
481 if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) {
482 if_setdrvflagbits(ifp, IFF_DRV_RUNNING, 0);
483 oce_init(sc);
484 }
485 device_printf(sc->dev, "Interface Up\n");
486 } else {
487 LOCK(&sc->dev_lock);
488
489 if_setdrvflagbits(sc->ifp, 0,
490 IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
491 oce_if_deactivate(sc);
492
493 UNLOCK(&sc->dev_lock);
494
495 device_printf(sc->dev, "Interface Down\n");
496 }
497
498 if ((if_getflags(ifp) & IFF_PROMISC) && !sc->promisc) {
499 if (!oce_rxf_set_promiscuous(sc, (1 | (1 << 1))))
500 sc->promisc = TRUE;
501 } else if (!(if_getflags(ifp) & IFF_PROMISC) && sc->promisc) {
502 if (!oce_rxf_set_promiscuous(sc, 0))
503 sc->promisc = FALSE;
504 }
505
506 break;
507
508 case SIOCADDMULTI:
509 case SIOCDELMULTI:
510 rc = oce_hw_update_multicast(sc);
511 if (rc)
512 device_printf(sc->dev,
513 "Update multicast address failed\n");
514 break;
515
516 case SIOCSIFCAP:
517 u = ifr->ifr_reqcap ^ if_getcapenable(ifp);
518
519 if (u & IFCAP_TXCSUM) {
520 if_togglecapenable(ifp, IFCAP_TXCSUM);
521 if_togglehwassist(ifp, (CSUM_TCP | CSUM_UDP | CSUM_IP));
522
523 if (IFCAP_TSO & if_getcapenable(ifp) &&
524 !(IFCAP_TXCSUM & if_getcapenable(ifp))) {
525 u &= ~IFCAP_TSO;
526 if_setcapenablebit(ifp, 0, IFCAP_TSO);
527 if_sethwassistbits(ifp, 0, CSUM_TSO);
528 if_printf(ifp,
529 "TSO disabled due to -txcsum.\n");
530 }
531 }
532
533 if (u & IFCAP_RXCSUM)
534 if_togglecapenable(ifp, IFCAP_RXCSUM);
535
536 if (u & IFCAP_TSO4) {
537 if_togglecapenable(ifp, IFCAP_TSO4);
538
539 if (IFCAP_TSO & if_getcapenable(ifp)) {
540 if (IFCAP_TXCSUM & if_getcapenable(ifp))
541 if_sethwassistbits(ifp, CSUM_TSO, 0);
542 else {
543 if_setcapenablebit(ifp, 0, IFCAP_TSO);
544 if_sethwassistbits(ifp, 0, CSUM_TSO);
545 if_printf(ifp,
546 "Enable txcsum first.\n");
547 rc = EAGAIN;
548 }
549 } else
550 if_sethwassistbits(ifp, 0, CSUM_TSO);
551 }
552
553 if (u & IFCAP_VLAN_HWTAGGING)
554 if_togglecapenable(ifp, IFCAP_VLAN_HWTAGGING);
555
556 if (u & IFCAP_VLAN_HWFILTER) {
557 if_togglecapenable(ifp, IFCAP_VLAN_HWFILTER);
558 oce_vid_config(sc);
559 }
560 #if defined(INET6) || defined(INET)
561 if (u & IFCAP_LRO) {
562 if_togglecapenable(ifp, IFCAP_LRO);
563 if(sc->enable_hwlro) {
564 if(if_getcapenable(ifp) & IFCAP_LRO) {
565 rc = oce_mbox_nic_set_iface_lro_config(sc, 1);
566 }else {
567 rc = oce_mbox_nic_set_iface_lro_config(sc, 0);
568 }
569 }
570 }
571 #endif
572
573 break;
574
575 case SIOCGI2C:
576 rc = copyin(ifr_data_get_ptr(ifr), &i2c, sizeof(i2c));
577 if (rc)
578 break;
579
580 if (i2c.dev_addr == PAGE_NUM_A0) {
581 offset = i2c.offset;
582 } else if (i2c.dev_addr == PAGE_NUM_A2) {
583 offset = TRANSCEIVER_A0_SIZE + i2c.offset;
584 } else {
585 rc = EINVAL;
586 break;
587 }
588
589 if (i2c.len > sizeof(i2c.data) ||
590 i2c.len + offset > sizeof(sfp_vpd_dump_buffer)) {
591 rc = EINVAL;
592 break;
593 }
594
595 rc = oce_mbox_read_transrecv_data(sc, i2c.dev_addr);
596 if (rc) {
597 rc = -rc;
598 break;
599 }
600
601 memcpy(&i2c.data[0], &sfp_vpd_dump_buffer[offset], i2c.len);
602
603 rc = copyout(&i2c, ifr_data_get_ptr(ifr), sizeof(i2c));
604 break;
605
606 case SIOCGPRIVATE_0:
607 rc = priv_check(curthread, PRIV_DRIVER);
608 if (rc != 0)
609 break;
610 rc = oce_handle_passthrough(ifp, data);
611 break;
612 default:
613 rc = ether_ioctl(ifp, command, data);
614 break;
615 }
616
617 return rc;
618 }
619
620 static void
oce_init(void * arg)621 oce_init(void *arg)
622 {
623 POCE_SOFTC sc = arg;
624
625 LOCK(&sc->dev_lock);
626
627 if (if_getflags(sc->ifp) & IFF_UP) {
628 oce_if_deactivate(sc);
629 oce_if_activate(sc);
630 }
631
632 UNLOCK(&sc->dev_lock);
633
634 }
635
636 static int
oce_multiq_start(if_t ifp,struct mbuf * m)637 oce_multiq_start(if_t ifp, struct mbuf *m)
638 {
639 POCE_SOFTC sc = if_getsoftc(ifp);
640 struct oce_wq *wq = NULL;
641 int queue_index = 0;
642 int status = 0;
643
644 if (!sc->link_status)
645 return ENXIO;
646
647 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
648 queue_index = m->m_pkthdr.flowid % sc->nwqs;
649
650 wq = sc->wq[queue_index];
651
652 LOCK(&wq->tx_lock);
653 status = oce_multiq_transmit(ifp, m, wq);
654 UNLOCK(&wq->tx_lock);
655
656 return status;
657
658 }
659
660 static void
oce_multiq_flush(if_t ifp)661 oce_multiq_flush(if_t ifp)
662 {
663 POCE_SOFTC sc = if_getsoftc(ifp);
664 struct mbuf *m;
665 int i = 0;
666
667 for (i = 0; i < sc->nwqs; i++) {
668 while ((m = buf_ring_dequeue_sc(sc->wq[i]->br)) != NULL)
669 m_freem(m);
670 }
671 if_qflush(ifp);
672 }
673
674 /*****************************************************************************
675 * Driver interrupt routines functions *
676 *****************************************************************************/
677
678 static void
oce_intr(void * arg,int pending)679 oce_intr(void *arg, int pending)
680 {
681
682 POCE_INTR_INFO ii = (POCE_INTR_INFO) arg;
683 POCE_SOFTC sc = ii->sc;
684 struct oce_eq *eq = ii->eq;
685 struct oce_eqe *eqe;
686 struct oce_cq *cq = NULL;
687 int i, num_eqes = 0;
688
689 bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map,
690 BUS_DMASYNC_POSTWRITE);
691 do {
692 eqe = RING_GET_CONSUMER_ITEM_VA(eq->ring, struct oce_eqe);
693 if (eqe->evnt == 0)
694 break;
695 eqe->evnt = 0;
696 bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map,
697 BUS_DMASYNC_POSTWRITE);
698 RING_GET(eq->ring, 1);
699 num_eqes++;
700
701 } while (TRUE);
702
703 if (!num_eqes)
704 goto eq_arm; /* Spurious */
705
706 /* Clear EQ entries, but dont arm */
707 oce_arm_eq(sc, eq->eq_id, num_eqes, FALSE, FALSE);
708
709 /* Process TX, RX and MCC. But dont arm CQ*/
710 for (i = 0; i < eq->cq_valid; i++) {
711 cq = eq->cq[i];
712 (*cq->cq_handler)(cq->cb_arg);
713 }
714
715 /* Arm all cqs connected to this EQ */
716 for (i = 0; i < eq->cq_valid; i++) {
717 cq = eq->cq[i];
718 oce_arm_cq(sc, cq->cq_id, 0, TRUE);
719 }
720
721 eq_arm:
722 oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE);
723
724 return;
725 }
726
727 static int
oce_setup_intr(POCE_SOFTC sc)728 oce_setup_intr(POCE_SOFTC sc)
729 {
730 int rc = 0, use_intx = 0;
731 int vector = 0, req_vectors = 0;
732 int tot_req_vectors, tot_vectors;
733
734 if (is_rss_enabled(sc))
735 req_vectors = MAX((sc->nrqs - 1), sc->nwqs);
736 else
737 req_vectors = 1;
738
739 tot_req_vectors = req_vectors;
740 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) {
741 if (req_vectors > 1) {
742 tot_req_vectors += OCE_RDMA_VECTORS;
743 sc->roce_intr_count = OCE_RDMA_VECTORS;
744 }
745 }
746
747 if (sc->flags & OCE_FLAGS_MSIX_CAPABLE) {
748 sc->intr_count = req_vectors;
749 tot_vectors = tot_req_vectors;
750 rc = pci_alloc_msix(sc->dev, &tot_vectors);
751 if (rc != 0) {
752 use_intx = 1;
753 pci_release_msi(sc->dev);
754 } else {
755 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) {
756 if (tot_vectors < tot_req_vectors) {
757 if (sc->intr_count < (2 * OCE_RDMA_VECTORS)) {
758 sc->roce_intr_count = (tot_vectors / 2);
759 }
760 sc->intr_count = tot_vectors - sc->roce_intr_count;
761 }
762 } else {
763 sc->intr_count = tot_vectors;
764 }
765 sc->flags |= OCE_FLAGS_USING_MSIX;
766 }
767 } else
768 use_intx = 1;
769
770 if (use_intx)
771 sc->intr_count = 1;
772
773 /* Scale number of queues based on intr we got */
774 update_queues_got(sc);
775
776 if (use_intx) {
777 device_printf(sc->dev, "Using legacy interrupt\n");
778 rc = oce_alloc_intr(sc, vector, oce_intr);
779 if (rc)
780 goto error;
781 } else {
782 for (; vector < sc->intr_count; vector++) {
783 rc = oce_alloc_intr(sc, vector, oce_intr);
784 if (rc)
785 goto error;
786 }
787 }
788
789 return 0;
790 error:
791 oce_intr_free(sc);
792 return rc;
793 }
794
795 static int
oce_fast_isr(void * arg)796 oce_fast_isr(void *arg)
797 {
798 POCE_INTR_INFO ii = (POCE_INTR_INFO) arg;
799 POCE_SOFTC sc = ii->sc;
800
801 if (ii->eq == NULL)
802 return FILTER_STRAY;
803
804 oce_arm_eq(sc, ii->eq->eq_id, 0, FALSE, TRUE);
805
806 taskqueue_enqueue(ii->tq, &ii->task);
807
808 ii->eq->intr++;
809
810 return FILTER_HANDLED;
811 }
812
813 static int
oce_alloc_intr(POCE_SOFTC sc,int vector,void (* isr)(void * arg,int pending))814 oce_alloc_intr(POCE_SOFTC sc, int vector, void (*isr) (void *arg, int pending))
815 {
816 POCE_INTR_INFO ii;
817 int rc = 0, rr;
818
819 if (vector >= OCE_MAX_EQ)
820 return (EINVAL);
821
822 ii = &sc->intrs[vector];
823
824 /* Set the resource id for the interrupt.
825 * MSIx is vector + 1 for the resource id,
826 * INTx is 0 for the resource id.
827 */
828 if (sc->flags & OCE_FLAGS_USING_MSIX)
829 rr = vector + 1;
830 else
831 rr = 0;
832 ii->intr_res = bus_alloc_resource_any(sc->dev,
833 SYS_RES_IRQ,
834 &rr, RF_ACTIVE|RF_SHAREABLE);
835 ii->irq_rr = rr;
836 if (ii->intr_res == NULL) {
837 device_printf(sc->dev,
838 "Could not allocate interrupt\n");
839 rc = ENXIO;
840 return rc;
841 }
842
843 TASK_INIT(&ii->task, 0, isr, ii);
844 ii->vector = vector;
845 sprintf(ii->task_name, "oce_task[%d]", ii->vector);
846 ii->tq = taskqueue_create_fast(ii->task_name,
847 M_NOWAIT,
848 taskqueue_thread_enqueue,
849 &ii->tq);
850 taskqueue_start_threads(&ii->tq, 1, PI_NET, "%s taskq",
851 device_get_nameunit(sc->dev));
852
853 ii->sc = sc;
854 rc = bus_setup_intr(sc->dev,
855 ii->intr_res,
856 INTR_TYPE_NET,
857 oce_fast_isr, NULL, ii, &ii->tag);
858 return rc;
859
860 }
861
862 void
oce_intr_free(POCE_SOFTC sc)863 oce_intr_free(POCE_SOFTC sc)
864 {
865 int i = 0;
866
867 for (i = 0; i < sc->intr_count; i++) {
868
869 if (sc->intrs[i].tag != NULL)
870 bus_teardown_intr(sc->dev, sc->intrs[i].intr_res,
871 sc->intrs[i].tag);
872 if (sc->intrs[i].tq != NULL)
873 taskqueue_free(sc->intrs[i].tq);
874
875 if (sc->intrs[i].intr_res != NULL)
876 bus_release_resource(sc->dev, SYS_RES_IRQ,
877 sc->intrs[i].irq_rr,
878 sc->intrs[i].intr_res);
879 sc->intrs[i].tag = NULL;
880 sc->intrs[i].intr_res = NULL;
881 }
882
883 if (sc->flags & OCE_FLAGS_USING_MSIX)
884 pci_release_msi(sc->dev);
885
886 }
887
888 /******************************************************************************
889 * Media callbacks functions *
890 ******************************************************************************/
891
892 static void
oce_media_status(if_t ifp,struct ifmediareq * req)893 oce_media_status(if_t ifp, struct ifmediareq *req)
894 {
895 POCE_SOFTC sc = (POCE_SOFTC) if_getsoftc(ifp);
896
897 req->ifm_status = IFM_AVALID;
898 req->ifm_active = IFM_ETHER;
899
900 if (sc->link_status == 1)
901 req->ifm_status |= IFM_ACTIVE;
902 else
903 return;
904
905 switch (sc->link_speed) {
906 case 1: /* 10 Mbps */
907 req->ifm_active |= IFM_10_T | IFM_FDX;
908 sc->speed = 10;
909 break;
910 case 2: /* 100 Mbps */
911 req->ifm_active |= IFM_100_TX | IFM_FDX;
912 sc->speed = 100;
913 break;
914 case 3: /* 1 Gbps */
915 req->ifm_active |= IFM_1000_T | IFM_FDX;
916 sc->speed = 1000;
917 break;
918 case 4: /* 10 Gbps */
919 req->ifm_active |= IFM_10G_SR | IFM_FDX;
920 sc->speed = 10000;
921 break;
922 case 5: /* 20 Gbps */
923 req->ifm_active |= IFM_10G_SR | IFM_FDX;
924 sc->speed = 20000;
925 break;
926 case 6: /* 25 Gbps */
927 req->ifm_active |= IFM_10G_SR | IFM_FDX;
928 sc->speed = 25000;
929 break;
930 case 7: /* 40 Gbps */
931 req->ifm_active |= IFM_40G_SR4 | IFM_FDX;
932 sc->speed = 40000;
933 break;
934 default:
935 sc->speed = 0;
936 break;
937 }
938
939 return;
940 }
941
942 int
oce_media_change(if_t ifp)943 oce_media_change(if_t ifp)
944 {
945 return 0;
946 }
947
oce_is_pkt_dest_bmc(POCE_SOFTC sc,struct mbuf * m,boolean_t * os2bmc,struct mbuf ** m_new)948 static void oce_is_pkt_dest_bmc(POCE_SOFTC sc,
949 struct mbuf *m, boolean_t *os2bmc,
950 struct mbuf **m_new)
951 {
952 struct ether_header *eh = NULL;
953
954 eh = mtod(m, struct ether_header *);
955
956 if (!is_os2bmc_enabled(sc) || *os2bmc) {
957 *os2bmc = FALSE;
958 goto done;
959 }
960 if (!ETHER_IS_MULTICAST(eh->ether_dhost))
961 goto done;
962
963 if (is_mc_allowed_on_bmc(sc, eh) ||
964 is_bc_allowed_on_bmc(sc, eh) ||
965 is_arp_allowed_on_bmc(sc, ntohs(eh->ether_type))) {
966 *os2bmc = TRUE;
967 goto done;
968 }
969
970 if (mtod(m, struct ip *)->ip_p == IPPROTO_IPV6) {
971 struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *);
972 uint8_t nexthdr = ip6->ip6_nxt;
973 if (nexthdr == IPPROTO_ICMPV6) {
974 struct icmp6_hdr *icmp6 = (struct icmp6_hdr *)(ip6 + 1);
975 switch (icmp6->icmp6_type) {
976 case ND_ROUTER_ADVERT:
977 *os2bmc = is_ipv6_ra_filt_enabled(sc);
978 goto done;
979 case ND_NEIGHBOR_ADVERT:
980 *os2bmc = is_ipv6_na_filt_enabled(sc);
981 goto done;
982 default:
983 break;
984 }
985 }
986 }
987
988 if (mtod(m, struct ip *)->ip_p == IPPROTO_UDP) {
989 struct ip *ip = mtod(m, struct ip *);
990 int iphlen = ip->ip_hl << 2;
991 struct udphdr *uh = (struct udphdr *)((caddr_t)ip + iphlen);
992 switch (uh->uh_dport) {
993 case DHCP_CLIENT_PORT:
994 *os2bmc = is_dhcp_client_filt_enabled(sc);
995 goto done;
996 case DHCP_SERVER_PORT:
997 *os2bmc = is_dhcp_srvr_filt_enabled(sc);
998 goto done;
999 case NET_BIOS_PORT1:
1000 case NET_BIOS_PORT2:
1001 *os2bmc = is_nbios_filt_enabled(sc);
1002 goto done;
1003 case DHCPV6_RAS_PORT:
1004 *os2bmc = is_ipv6_ras_filt_enabled(sc);
1005 goto done;
1006 default:
1007 break;
1008 }
1009 }
1010 done:
1011 if (*os2bmc) {
1012 *m_new = m_dup(m, M_NOWAIT);
1013 if (!*m_new) {
1014 *os2bmc = FALSE;
1015 return;
1016 }
1017 *m_new = oce_insert_vlan_tag(sc, *m_new, NULL);
1018 }
1019 }
1020
1021 /*****************************************************************************
1022 * Transmit routines functions *
1023 *****************************************************************************/
1024
1025 static int
oce_tx(POCE_SOFTC sc,struct mbuf ** mpp,int wq_index)1026 oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index)
1027 {
1028 int rc = 0, i, retry_cnt = 0;
1029 bus_dma_segment_t segs[OCE_MAX_TX_ELEMENTS];
1030 struct mbuf *m, *m_temp, *m_new = NULL;
1031 struct oce_wq *wq = sc->wq[wq_index];
1032 struct oce_packet_desc *pd;
1033 struct oce_nic_hdr_wqe *nichdr;
1034 struct oce_nic_frag_wqe *nicfrag;
1035 struct ether_header *eh = NULL;
1036 int num_wqes;
1037 uint32_t reg_value;
1038 boolean_t complete = TRUE;
1039 boolean_t os2bmc = FALSE;
1040
1041 m = *mpp;
1042 if (!m)
1043 return EINVAL;
1044
1045 if (!(m->m_flags & M_PKTHDR)) {
1046 rc = ENXIO;
1047 goto free_ret;
1048 }
1049
1050 /* Don't allow non-TSO packets longer than MTU */
1051 if (!is_tso_pkt(m)) {
1052 eh = mtod(m, struct ether_header *);
1053 if(m->m_pkthdr.len > ETHER_MAX_FRAME(sc->ifp, eh->ether_type, FALSE))
1054 goto free_ret;
1055 }
1056
1057 if(oce_tx_asic_stall_verify(sc, m)) {
1058 m = oce_insert_vlan_tag(sc, m, &complete);
1059 if(!m) {
1060 device_printf(sc->dev, "Insertion unsuccessful\n");
1061 return 0;
1062 }
1063 }
1064
1065 /* Lancer, SH ASIC has a bug wherein Packets that are 32 bytes or less
1066 * may cause a transmit stall on that port. So the work-around is to
1067 * pad short packets (<= 32 bytes) to a 36-byte length.
1068 */
1069 if(IS_SH(sc) || IS_XE201(sc) ) {
1070 if(m->m_pkthdr.len <= 32) {
1071 char buf[36];
1072 bzero((void *)buf, 36);
1073 m_append(m, (36 - m->m_pkthdr.len), buf);
1074 }
1075 }
1076
1077 tx_start:
1078 if (m->m_pkthdr.csum_flags & CSUM_TSO) {
1079 /* consolidate packet buffers for TSO/LSO segment offload */
1080 #if defined(INET6) || defined(INET)
1081 m = oce_tso_setup(sc, mpp);
1082 #else
1083 m = NULL;
1084 #endif
1085 if (m == NULL) {
1086 rc = ENXIO;
1087 goto free_ret;
1088 }
1089 }
1090
1091 pd = &wq->pckts[wq->pkt_desc_head];
1092
1093 retry:
1094 rc = bus_dmamap_load_mbuf_sg(wq->tag,
1095 pd->map,
1096 m, segs, &pd->nsegs, BUS_DMA_NOWAIT);
1097 if (rc == 0) {
1098 num_wqes = pd->nsegs + 1;
1099 if (IS_BE(sc) || IS_SH(sc)) {
1100 /*Dummy required only for BE3.*/
1101 if (num_wqes & 1)
1102 num_wqes++;
1103 }
1104 if (num_wqes >= RING_NUM_FREE(wq->ring)) {
1105 bus_dmamap_unload(wq->tag, pd->map);
1106 return EBUSY;
1107 }
1108 atomic_store_rel_int(&wq->pkt_desc_head,
1109 (wq->pkt_desc_head + 1) % \
1110 OCE_WQ_PACKET_ARRAY_SIZE);
1111 bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_PREWRITE);
1112 pd->mbuf = m;
1113
1114 nichdr =
1115 RING_GET_PRODUCER_ITEM_VA(wq->ring, struct oce_nic_hdr_wqe);
1116 nichdr->u0.dw[0] = 0;
1117 nichdr->u0.dw[1] = 0;
1118 nichdr->u0.dw[2] = 0;
1119 nichdr->u0.dw[3] = 0;
1120
1121 nichdr->u0.s.complete = complete;
1122 nichdr->u0.s.mgmt = os2bmc;
1123 nichdr->u0.s.event = 1;
1124 nichdr->u0.s.crc = 1;
1125 nichdr->u0.s.forward = 0;
1126 nichdr->u0.s.ipcs = (m->m_pkthdr.csum_flags & CSUM_IP) ? 1 : 0;
1127 nichdr->u0.s.udpcs =
1128 (m->m_pkthdr.csum_flags & CSUM_UDP) ? 1 : 0;
1129 nichdr->u0.s.tcpcs =
1130 (m->m_pkthdr.csum_flags & CSUM_TCP) ? 1 : 0;
1131 nichdr->u0.s.num_wqe = num_wqes;
1132 nichdr->u0.s.total_length = m->m_pkthdr.len;
1133
1134 if (m->m_flags & M_VLANTAG) {
1135 nichdr->u0.s.vlan = 1; /*Vlan present*/
1136 nichdr->u0.s.vlan_tag = m->m_pkthdr.ether_vtag;
1137 }
1138
1139 if (m->m_pkthdr.csum_flags & CSUM_TSO) {
1140 if (m->m_pkthdr.tso_segsz) {
1141 nichdr->u0.s.lso = 1;
1142 nichdr->u0.s.lso_mss = m->m_pkthdr.tso_segsz;
1143 }
1144 if (!IS_BE(sc) || !IS_SH(sc))
1145 nichdr->u0.s.ipcs = 1;
1146 }
1147
1148 RING_PUT(wq->ring, 1);
1149 atomic_add_int(&wq->ring->num_used, 1);
1150
1151 for (i = 0; i < pd->nsegs; i++) {
1152 nicfrag =
1153 RING_GET_PRODUCER_ITEM_VA(wq->ring,
1154 struct oce_nic_frag_wqe);
1155 nicfrag->u0.s.rsvd0 = 0;
1156 nicfrag->u0.s.frag_pa_hi = ADDR_HI(segs[i].ds_addr);
1157 nicfrag->u0.s.frag_pa_lo = ADDR_LO(segs[i].ds_addr);
1158 nicfrag->u0.s.frag_len = segs[i].ds_len;
1159 pd->wqe_idx = wq->ring->pidx;
1160 RING_PUT(wq->ring, 1);
1161 atomic_add_int(&wq->ring->num_used, 1);
1162 }
1163 if (num_wqes > (pd->nsegs + 1)) {
1164 nicfrag =
1165 RING_GET_PRODUCER_ITEM_VA(wq->ring,
1166 struct oce_nic_frag_wqe);
1167 nicfrag->u0.dw[0] = 0;
1168 nicfrag->u0.dw[1] = 0;
1169 nicfrag->u0.dw[2] = 0;
1170 nicfrag->u0.dw[3] = 0;
1171 pd->wqe_idx = wq->ring->pidx;
1172 RING_PUT(wq->ring, 1);
1173 atomic_add_int(&wq->ring->num_used, 1);
1174 pd->nsegs++;
1175 }
1176
1177 if_inc_counter(sc->ifp, IFCOUNTER_OPACKETS, 1);
1178 wq->tx_stats.tx_reqs++;
1179 wq->tx_stats.tx_wrbs += num_wqes;
1180 wq->tx_stats.tx_bytes += m->m_pkthdr.len;
1181 wq->tx_stats.tx_pkts++;
1182
1183 bus_dmamap_sync(wq->ring->dma.tag, wq->ring->dma.map,
1184 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1185 reg_value = (num_wqes << 16) | wq->wq_id;
1186
1187 /* if os2bmc is not enabled or if the pkt is already tagged as
1188 bmc, do nothing
1189 */
1190 oce_is_pkt_dest_bmc(sc, m, &os2bmc, &m_new);
1191
1192 if_inc_counter(sc->ifp, IFCOUNTER_OBYTES, m->m_pkthdr.len);
1193 if (m->m_flags & M_MCAST)
1194 if_inc_counter(sc->ifp, IFCOUNTER_OMCASTS, 1);
1195 ETHER_BPF_MTAP(sc->ifp, m);
1196
1197 OCE_WRITE_REG32(sc, db, wq->db_offset, reg_value);
1198
1199 } else if (rc == EFBIG) {
1200 if (retry_cnt == 0) {
1201 m_temp = m_defrag(m, M_NOWAIT);
1202 if (m_temp == NULL)
1203 goto free_ret;
1204 m = m_temp;
1205 *mpp = m_temp;
1206 retry_cnt = retry_cnt + 1;
1207 goto retry;
1208 } else
1209 goto free_ret;
1210 } else if (rc == ENOMEM)
1211 return rc;
1212 else
1213 goto free_ret;
1214
1215 if (os2bmc) {
1216 m = m_new;
1217 goto tx_start;
1218 }
1219
1220 return 0;
1221
1222 free_ret:
1223 m_freem(*mpp);
1224 *mpp = NULL;
1225 return rc;
1226 }
1227
1228 static void
oce_process_tx_completion(struct oce_wq * wq)1229 oce_process_tx_completion(struct oce_wq *wq)
1230 {
1231 struct oce_packet_desc *pd;
1232 POCE_SOFTC sc = (POCE_SOFTC) wq->parent;
1233 struct mbuf *m;
1234
1235 pd = &wq->pckts[wq->pkt_desc_tail];
1236 atomic_store_rel_int(&wq->pkt_desc_tail,
1237 (wq->pkt_desc_tail + 1) % OCE_WQ_PACKET_ARRAY_SIZE);
1238 atomic_subtract_int(&wq->ring->num_used, pd->nsegs + 1);
1239 bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_POSTWRITE);
1240 bus_dmamap_unload(wq->tag, pd->map);
1241
1242 m = pd->mbuf;
1243 m_freem(m);
1244 pd->mbuf = NULL;
1245
1246 if (if_getdrvflags(sc->ifp) & IFF_DRV_OACTIVE) {
1247 if (wq->ring->num_used < (wq->ring->num_items / 2)) {
1248 if_setdrvflagbits(sc->ifp, 0, (IFF_DRV_OACTIVE));
1249 oce_tx_restart(sc, wq);
1250 }
1251 }
1252 }
1253
1254 static void
oce_tx_restart(POCE_SOFTC sc,struct oce_wq * wq)1255 oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq)
1256 {
1257
1258 if ((if_getdrvflags(sc->ifp) & IFF_DRV_RUNNING) != IFF_DRV_RUNNING)
1259 return;
1260
1261 if (!drbr_empty(sc->ifp, wq->br))
1262 taskqueue_enqueue(taskqueue_swi, &wq->txtask);
1263
1264 }
1265
1266 #if defined(INET6) || defined(INET)
1267 static struct mbuf *
oce_tso_setup(POCE_SOFTC sc,struct mbuf ** mpp)1268 oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp)
1269 {
1270 struct mbuf *m;
1271 #ifdef INET
1272 struct ip *ip;
1273 #endif
1274 #ifdef INET6
1275 struct ip6_hdr *ip6;
1276 #endif
1277 struct ether_vlan_header *eh;
1278 struct tcphdr *th;
1279 uint16_t etype;
1280 int total_len = 0, ehdrlen = 0;
1281
1282 m = *mpp;
1283
1284 if (M_WRITABLE(m) == 0) {
1285 m = m_dup(*mpp, M_NOWAIT);
1286 if (!m)
1287 return NULL;
1288 m_freem(*mpp);
1289 *mpp = m;
1290 }
1291
1292 eh = mtod(m, struct ether_vlan_header *);
1293 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
1294 etype = ntohs(eh->evl_proto);
1295 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
1296 } else {
1297 etype = ntohs(eh->evl_encap_proto);
1298 ehdrlen = ETHER_HDR_LEN;
1299 }
1300
1301 switch (etype) {
1302 #ifdef INET
1303 case ETHERTYPE_IP:
1304 ip = (struct ip *)(m->m_data + ehdrlen);
1305 if (ip->ip_p != IPPROTO_TCP)
1306 return NULL;
1307 th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2));
1308
1309 total_len = ehdrlen + (ip->ip_hl << 2) + (th->th_off << 2);
1310 break;
1311 #endif
1312 #ifdef INET6
1313 case ETHERTYPE_IPV6:
1314 ip6 = (struct ip6_hdr *)(m->m_data + ehdrlen);
1315 if (ip6->ip6_nxt != IPPROTO_TCP)
1316 return NULL;
1317 th = (struct tcphdr *)((caddr_t)ip6 + sizeof(struct ip6_hdr));
1318
1319 total_len = ehdrlen + sizeof(struct ip6_hdr) + (th->th_off << 2);
1320 break;
1321 #endif
1322 default:
1323 return NULL;
1324 }
1325
1326 m = m_pullup(m, total_len);
1327 *mpp = m;
1328 return m;
1329 }
1330 #endif /* INET6 || INET */
1331
1332 void
oce_tx_task(void * arg,int npending)1333 oce_tx_task(void *arg, int npending)
1334 {
1335 struct oce_wq *wq = arg;
1336 POCE_SOFTC sc = wq->parent;
1337 if_t ifp = sc->ifp;
1338 int rc = 0;
1339
1340 LOCK(&wq->tx_lock);
1341 rc = oce_multiq_transmit(ifp, NULL, wq);
1342 if (rc) {
1343 device_printf(sc->dev,
1344 "TX[%d] restart failed\n", wq->queue_index);
1345 }
1346 UNLOCK(&wq->tx_lock);
1347 }
1348
1349 void
oce_start(if_t ifp)1350 oce_start(if_t ifp)
1351 {
1352 POCE_SOFTC sc = if_getsoftc(ifp);
1353 struct mbuf *m;
1354 int rc = 0;
1355 int def_q = 0; /* Defualt tx queue is 0*/
1356
1357 if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
1358 IFF_DRV_RUNNING)
1359 return;
1360
1361 if (!sc->link_status)
1362 return;
1363
1364 while (true) {
1365 m = if_dequeue(sc->ifp);
1366 if (m == NULL)
1367 break;
1368
1369 LOCK(&sc->wq[def_q]->tx_lock);
1370 rc = oce_tx(sc, &m, def_q);
1371 UNLOCK(&sc->wq[def_q]->tx_lock);
1372 if (rc) {
1373 if (m != NULL) {
1374 sc->wq[def_q]->tx_stats.tx_stops ++;
1375 if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0);
1376 if_sendq_prepend(ifp, m);
1377 m = NULL;
1378 }
1379 break;
1380 }
1381 }
1382 }
1383
1384 /* Handle the Completion Queue for transmit */
1385 uint16_t
oce_wq_handler(void * arg)1386 oce_wq_handler(void *arg)
1387 {
1388 struct oce_wq *wq = (struct oce_wq *)arg;
1389 POCE_SOFTC sc = wq->parent;
1390 struct oce_cq *cq = wq->cq;
1391 struct oce_nic_tx_cqe *cqe;
1392 int num_cqes = 0;
1393
1394 LOCK(&wq->tx_compl_lock);
1395 bus_dmamap_sync(cq->ring->dma.tag,
1396 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
1397 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe);
1398 while (cqe->u0.dw[3]) {
1399 DW_SWAP((uint32_t *) cqe, sizeof(oce_wq_cqe));
1400
1401 wq->ring->cidx = cqe->u0.s.wqe_index + 1;
1402 if (wq->ring->cidx >= wq->ring->num_items)
1403 wq->ring->cidx -= wq->ring->num_items;
1404
1405 oce_process_tx_completion(wq);
1406 wq->tx_stats.tx_compl++;
1407 cqe->u0.dw[3] = 0;
1408 RING_GET(cq->ring, 1);
1409 bus_dmamap_sync(cq->ring->dma.tag,
1410 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
1411 cqe =
1412 RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe);
1413 num_cqes++;
1414 }
1415
1416 if (num_cqes)
1417 oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE);
1418
1419 UNLOCK(&wq->tx_compl_lock);
1420 return num_cqes;
1421 }
1422
1423 static int
oce_multiq_transmit(if_t ifp,struct mbuf * m,struct oce_wq * wq)1424 oce_multiq_transmit(if_t ifp, struct mbuf *m, struct oce_wq *wq)
1425 {
1426 POCE_SOFTC sc = if_getsoftc(ifp);
1427 int status = 0, queue_index = 0;
1428 struct mbuf *next = NULL;
1429 struct buf_ring *br = NULL;
1430
1431 br = wq->br;
1432 queue_index = wq->queue_index;
1433
1434 if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
1435 IFF_DRV_RUNNING) {
1436 if (m != NULL)
1437 status = drbr_enqueue(ifp, br, m);
1438 return status;
1439 }
1440
1441 if (m != NULL) {
1442 if ((status = drbr_enqueue(ifp, br, m)) != 0)
1443 return status;
1444 }
1445 while ((next = drbr_peek(ifp, br)) != NULL) {
1446 if (oce_tx(sc, &next, queue_index)) {
1447 if (next == NULL) {
1448 drbr_advance(ifp, br);
1449 } else {
1450 drbr_putback(ifp, br, next);
1451 wq->tx_stats.tx_stops ++;
1452 if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0);
1453 }
1454 break;
1455 }
1456 drbr_advance(ifp, br);
1457 }
1458
1459 return 0;
1460 }
1461
1462 /*****************************************************************************
1463 * Receive routines functions *
1464 *****************************************************************************/
1465
1466 static void
oce_correct_header(struct mbuf * m,struct nic_hwlro_cqe_part1 * cqe1,struct nic_hwlro_cqe_part2 * cqe2)1467 oce_correct_header(struct mbuf *m, struct nic_hwlro_cqe_part1 *cqe1, struct nic_hwlro_cqe_part2 *cqe2)
1468 {
1469 uint32_t *p;
1470 struct ether_header *eh = NULL;
1471 struct tcphdr *tcp_hdr = NULL;
1472 struct ip *ip4_hdr = NULL;
1473 struct ip6_hdr *ip6 = NULL;
1474 uint32_t payload_len = 0;
1475
1476 eh = mtod(m, struct ether_header *);
1477 /* correct IP header */
1478 if(!cqe2->ipv6_frame) {
1479 ip4_hdr = (struct ip *)((char*)eh + sizeof(struct ether_header));
1480 ip4_hdr->ip_ttl = cqe2->frame_lifespan;
1481 ip4_hdr->ip_len = htons(cqe2->coalesced_size - sizeof(struct ether_header));
1482 tcp_hdr = (struct tcphdr *)((char*)ip4_hdr + sizeof(struct ip));
1483 }else {
1484 ip6 = (struct ip6_hdr *)((char*)eh + sizeof(struct ether_header));
1485 ip6->ip6_ctlun.ip6_un1.ip6_un1_hlim = cqe2->frame_lifespan;
1486 payload_len = cqe2->coalesced_size - sizeof(struct ether_header)
1487 - sizeof(struct ip6_hdr);
1488 ip6->ip6_ctlun.ip6_un1.ip6_un1_plen = htons(payload_len);
1489 tcp_hdr = (struct tcphdr *)((char*)ip6 + sizeof(struct ip6_hdr));
1490 }
1491
1492 /* correct tcp header */
1493 tcp_hdr->th_ack = htonl(cqe2->tcp_ack_num);
1494 if(cqe2->push) {
1495 tcp_set_flags(tcp_hdr, tcp_get_flags(tcp_hdr) | TH_PUSH);
1496 }
1497 tcp_hdr->th_win = htons(cqe2->tcp_window);
1498 tcp_hdr->th_sum = 0xffff;
1499 if(cqe2->ts_opt) {
1500 p = (uint32_t *)((char*)tcp_hdr + sizeof(struct tcphdr) + 2);
1501 *p = cqe1->tcp_timestamp_val;
1502 *(p+1) = cqe1->tcp_timestamp_ecr;
1503 }
1504
1505 return;
1506 }
1507
1508 static void
oce_rx_mbuf_chain(struct oce_rq * rq,struct oce_common_cqe_info * cqe_info,struct mbuf ** m)1509 oce_rx_mbuf_chain(struct oce_rq *rq, struct oce_common_cqe_info *cqe_info, struct mbuf **m)
1510 {
1511 POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1512 uint32_t i = 0, frag_len = 0;
1513 uint32_t len = cqe_info->pkt_size;
1514 struct oce_packet_desc *pd;
1515 struct mbuf *tail = NULL;
1516
1517 for (i = 0; i < cqe_info->num_frags; i++) {
1518 if (rq->ring->cidx == rq->ring->pidx) {
1519 device_printf(sc->dev,
1520 "oce_rx_mbuf_chain: Invalid RX completion - Queue is empty\n");
1521 return;
1522 }
1523 pd = &rq->pckts[rq->ring->cidx];
1524
1525 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE);
1526 bus_dmamap_unload(rq->tag, pd->map);
1527 RING_GET(rq->ring, 1);
1528 rq->pending--;
1529
1530 frag_len = (len > rq->cfg.frag_size) ? rq->cfg.frag_size : len;
1531 pd->mbuf->m_len = frag_len;
1532
1533 if (tail != NULL) {
1534 /* additional fragments */
1535 pd->mbuf->m_flags &= ~M_PKTHDR;
1536 tail->m_next = pd->mbuf;
1537 if(rq->islro)
1538 tail->m_nextpkt = NULL;
1539 tail = pd->mbuf;
1540 } else {
1541 /* first fragment, fill out much of the packet header */
1542 pd->mbuf->m_pkthdr.len = len;
1543 if(rq->islro)
1544 pd->mbuf->m_nextpkt = NULL;
1545 pd->mbuf->m_pkthdr.csum_flags = 0;
1546 if (IF_CSUM_ENABLED(sc)) {
1547 if (cqe_info->l4_cksum_pass) {
1548 if(!cqe_info->ipv6_frame) { /* IPV4 */
1549 pd->mbuf->m_pkthdr.csum_flags |=
1550 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
1551 }else { /* IPV6 frame */
1552 if(rq->islro) {
1553 pd->mbuf->m_pkthdr.csum_flags |=
1554 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
1555 }
1556 }
1557 pd->mbuf->m_pkthdr.csum_data = 0xffff;
1558 }
1559 if (cqe_info->ip_cksum_pass) {
1560 pd->mbuf->m_pkthdr.csum_flags |=
1561 (CSUM_IP_CHECKED|CSUM_IP_VALID);
1562 }
1563 }
1564 *m = tail = pd->mbuf;
1565 }
1566 pd->mbuf = NULL;
1567 len -= frag_len;
1568 }
1569
1570 return;
1571 }
1572
1573 static void
oce_rx_lro(struct oce_rq * rq,struct nic_hwlro_singleton_cqe * cqe,struct nic_hwlro_cqe_part2 * cqe2)1574 oce_rx_lro(struct oce_rq *rq, struct nic_hwlro_singleton_cqe *cqe, struct nic_hwlro_cqe_part2 *cqe2)
1575 {
1576 POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1577 struct nic_hwlro_cqe_part1 *cqe1 = NULL;
1578 struct mbuf *m = NULL;
1579 struct oce_common_cqe_info cq_info;
1580
1581 /* parse cqe */
1582 if(cqe2 == NULL) {
1583 cq_info.pkt_size = cqe->pkt_size;
1584 cq_info.vtag = cqe->vlan_tag;
1585 cq_info.l4_cksum_pass = cqe->l4_cksum_pass;
1586 cq_info.ip_cksum_pass = cqe->ip_cksum_pass;
1587 cq_info.ipv6_frame = cqe->ipv6_frame;
1588 cq_info.vtp = cqe->vtp;
1589 cq_info.qnq = cqe->qnq;
1590 }else {
1591 cqe1 = (struct nic_hwlro_cqe_part1 *)cqe;
1592 cq_info.pkt_size = cqe2->coalesced_size;
1593 cq_info.vtag = cqe2->vlan_tag;
1594 cq_info.l4_cksum_pass = cqe2->l4_cksum_pass;
1595 cq_info.ip_cksum_pass = cqe2->ip_cksum_pass;
1596 cq_info.ipv6_frame = cqe2->ipv6_frame;
1597 cq_info.vtp = cqe2->vtp;
1598 cq_info.qnq = cqe1->qnq;
1599 }
1600
1601 cq_info.vtag = BSWAP_16(cq_info.vtag);
1602
1603 cq_info.num_frags = cq_info.pkt_size / rq->cfg.frag_size;
1604 if(cq_info.pkt_size % rq->cfg.frag_size)
1605 cq_info.num_frags++;
1606
1607 oce_rx_mbuf_chain(rq, &cq_info, &m);
1608
1609 if (m) {
1610 if(cqe2) {
1611 //assert(cqe2->valid != 0);
1612
1613 //assert(cqe2->cqe_type != 2);
1614 oce_correct_header(m, cqe1, cqe2);
1615 }
1616
1617 m->m_pkthdr.rcvif = sc->ifp;
1618 if (rq->queue_index)
1619 m->m_pkthdr.flowid = (rq->queue_index - 1);
1620 else
1621 m->m_pkthdr.flowid = rq->queue_index;
1622 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
1623
1624 /* This deternies if vlan tag is Valid */
1625 if (cq_info.vtp) {
1626 if (sc->function_mode & FNM_FLEX10_MODE) {
1627 /* FLEX10. If QnQ is not set, neglect VLAN */
1628 if (cq_info.qnq) {
1629 m->m_pkthdr.ether_vtag = cq_info.vtag;
1630 m->m_flags |= M_VLANTAG;
1631 }
1632 } else if (sc->pvid != (cq_info.vtag & VLAN_VID_MASK)) {
1633 /* In UMC mode generally pvid will be striped by
1634 hw. But in some cases we have seen it comes
1635 with pvid. So if pvid == vlan, neglect vlan.
1636 */
1637 m->m_pkthdr.ether_vtag = cq_info.vtag;
1638 m->m_flags |= M_VLANTAG;
1639 }
1640 }
1641 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1);
1642
1643 if_input(sc->ifp, m);
1644
1645 /* Update rx stats per queue */
1646 rq->rx_stats.rx_pkts++;
1647 rq->rx_stats.rx_bytes += cq_info.pkt_size;
1648 rq->rx_stats.rx_frags += cq_info.num_frags;
1649 rq->rx_stats.rx_ucast_pkts++;
1650 }
1651 return;
1652 }
1653
1654 static void
oce_rx(struct oce_rq * rq,struct oce_nic_rx_cqe * cqe)1655 oce_rx(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe)
1656 {
1657 POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1658 int len;
1659 struct mbuf *m = NULL;
1660 struct oce_common_cqe_info cq_info;
1661 uint16_t vtag = 0;
1662
1663 /* Is it a flush compl that has no data */
1664 if(!cqe->u0.s.num_fragments)
1665 goto exit;
1666
1667 len = cqe->u0.s.pkt_size;
1668 if (!len) {
1669 /*partial DMA workaround for Lancer*/
1670 oce_discard_rx_comp(rq, cqe->u0.s.num_fragments);
1671 goto exit;
1672 }
1673
1674 if (!oce_cqe_portid_valid(sc, cqe)) {
1675 oce_discard_rx_comp(rq, cqe->u0.s.num_fragments);
1676 goto exit;
1677 }
1678
1679 /* Get vlan_tag value */
1680 if(IS_BE(sc) || IS_SH(sc))
1681 vtag = BSWAP_16(cqe->u0.s.vlan_tag);
1682 else
1683 vtag = cqe->u0.s.vlan_tag;
1684
1685 cq_info.l4_cksum_pass = cqe->u0.s.l4_cksum_pass;
1686 cq_info.ip_cksum_pass = cqe->u0.s.ip_cksum_pass;
1687 cq_info.ipv6_frame = cqe->u0.s.ip_ver;
1688 cq_info.num_frags = cqe->u0.s.num_fragments;
1689 cq_info.pkt_size = cqe->u0.s.pkt_size;
1690
1691 oce_rx_mbuf_chain(rq, &cq_info, &m);
1692
1693 if (m) {
1694 m->m_pkthdr.rcvif = sc->ifp;
1695 if (rq->queue_index)
1696 m->m_pkthdr.flowid = (rq->queue_index - 1);
1697 else
1698 m->m_pkthdr.flowid = rq->queue_index;
1699 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
1700
1701 /* This deternies if vlan tag is Valid */
1702 if (oce_cqe_vtp_valid(sc, cqe)) {
1703 if (sc->function_mode & FNM_FLEX10_MODE) {
1704 /* FLEX10. If QnQ is not set, neglect VLAN */
1705 if (cqe->u0.s.qnq) {
1706 m->m_pkthdr.ether_vtag = vtag;
1707 m->m_flags |= M_VLANTAG;
1708 }
1709 } else if (sc->pvid != (vtag & VLAN_VID_MASK)) {
1710 /* In UMC mode generally pvid will be striped by
1711 hw. But in some cases we have seen it comes
1712 with pvid. So if pvid == vlan, neglect vlan.
1713 */
1714 m->m_pkthdr.ether_vtag = vtag;
1715 m->m_flags |= M_VLANTAG;
1716 }
1717 }
1718
1719 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1);
1720 #if defined(INET6) || defined(INET)
1721 /* Try to queue to LRO */
1722 if (IF_LRO_ENABLED(sc) &&
1723 (cqe->u0.s.ip_cksum_pass) &&
1724 (cqe->u0.s.l4_cksum_pass) &&
1725 (!cqe->u0.s.ip_ver) &&
1726 (rq->lro.lro_cnt != 0)) {
1727 if (tcp_lro_rx(&rq->lro, m, 0) == 0) {
1728 rq->lro_pkts_queued ++;
1729 goto post_done;
1730 }
1731 /* If LRO posting fails then try to post to STACK */
1732 }
1733 #endif
1734
1735 if_input(sc->ifp, m);
1736 #if defined(INET6) || defined(INET)
1737 post_done:
1738 #endif
1739 /* Update rx stats per queue */
1740 rq->rx_stats.rx_pkts++;
1741 rq->rx_stats.rx_bytes += cqe->u0.s.pkt_size;
1742 rq->rx_stats.rx_frags += cqe->u0.s.num_fragments;
1743 if (cqe->u0.s.pkt_type == OCE_MULTICAST_PACKET)
1744 rq->rx_stats.rx_mcast_pkts++;
1745 if (cqe->u0.s.pkt_type == OCE_UNICAST_PACKET)
1746 rq->rx_stats.rx_ucast_pkts++;
1747 }
1748 exit:
1749 return;
1750 }
1751
1752 void
oce_discard_rx_comp(struct oce_rq * rq,int num_frags)1753 oce_discard_rx_comp(struct oce_rq *rq, int num_frags)
1754 {
1755 uint32_t i = 0;
1756 struct oce_packet_desc *pd;
1757 POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1758
1759 for (i = 0; i < num_frags; i++) {
1760 if (rq->ring->cidx == rq->ring->pidx) {
1761 device_printf(sc->dev,
1762 "oce_discard_rx_comp: Invalid RX completion - Queue is empty\n");
1763 return;
1764 }
1765 pd = &rq->pckts[rq->ring->cidx];
1766 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE);
1767 bus_dmamap_unload(rq->tag, pd->map);
1768 if (pd->mbuf != NULL) {
1769 m_freem(pd->mbuf);
1770 pd->mbuf = NULL;
1771 }
1772
1773 RING_GET(rq->ring, 1);
1774 rq->pending--;
1775 }
1776 }
1777
1778 static int
oce_cqe_vtp_valid(POCE_SOFTC sc,struct oce_nic_rx_cqe * cqe)1779 oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe)
1780 {
1781 struct oce_nic_rx_cqe_v1 *cqe_v1;
1782 int vtp = 0;
1783
1784 if (sc->be3_native) {
1785 cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe;
1786 vtp = cqe_v1->u0.s.vlan_tag_present;
1787 } else
1788 vtp = cqe->u0.s.vlan_tag_present;
1789
1790 return vtp;
1791
1792 }
1793
1794 static int
oce_cqe_portid_valid(POCE_SOFTC sc,struct oce_nic_rx_cqe * cqe)1795 oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe)
1796 {
1797 struct oce_nic_rx_cqe_v1 *cqe_v1;
1798 int port_id = 0;
1799
1800 if (sc->be3_native && (IS_BE(sc) || IS_SH(sc))) {
1801 cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe;
1802 port_id = cqe_v1->u0.s.port;
1803 if (sc->port_id != port_id)
1804 return 0;
1805 } else
1806 ;/* For BE3 legacy and Lancer this is dummy */
1807
1808 return 1;
1809
1810 }
1811
1812 #if defined(INET6) || defined(INET)
1813 void
oce_rx_flush_lro(struct oce_rq * rq)1814 oce_rx_flush_lro(struct oce_rq *rq)
1815 {
1816 struct lro_ctrl *lro = &rq->lro;
1817 POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1818
1819 if (!IF_LRO_ENABLED(sc))
1820 return;
1821
1822 tcp_lro_flush_all(lro);
1823 rq->lro_pkts_queued = 0;
1824
1825 return;
1826 }
1827
1828 static int
oce_init_lro(POCE_SOFTC sc)1829 oce_init_lro(POCE_SOFTC sc)
1830 {
1831 struct lro_ctrl *lro = NULL;
1832 int i = 0, rc = 0;
1833
1834 for (i = 0; i < sc->nrqs; i++) {
1835 lro = &sc->rq[i]->lro;
1836 rc = tcp_lro_init(lro);
1837 if (rc != 0) {
1838 device_printf(sc->dev, "LRO init failed\n");
1839 return rc;
1840 }
1841 lro->ifp = sc->ifp;
1842 }
1843
1844 return rc;
1845 }
1846
1847 void
oce_free_lro(POCE_SOFTC sc)1848 oce_free_lro(POCE_SOFTC sc)
1849 {
1850 struct lro_ctrl *lro = NULL;
1851 int i = 0;
1852
1853 for (i = 0; i < sc->nrqs; i++) {
1854 lro = &sc->rq[i]->lro;
1855 if (lro)
1856 tcp_lro_free(lro);
1857 }
1858 }
1859 #endif
1860
1861 int
oce_alloc_rx_bufs(struct oce_rq * rq,int count)1862 oce_alloc_rx_bufs(struct oce_rq *rq, int count)
1863 {
1864 POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1865 int i, rc;
1866 struct oce_packet_desc *pd;
1867 bus_dma_segment_t segs[6];
1868 int nsegs, added = 0;
1869 struct oce_nic_rqe *rqe;
1870 pd_rxulp_db_t rxdb_reg;
1871 uint32_t val = 0;
1872 uint32_t oce_max_rq_posts = 64;
1873
1874 bzero(&rxdb_reg, sizeof(pd_rxulp_db_t));
1875 for (i = 0; i < count; i++) {
1876 pd = &rq->pckts[rq->ring->pidx];
1877 pd->mbuf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, oce_rq_buf_size);
1878 if (pd->mbuf == NULL) {
1879 device_printf(sc->dev, "mbuf allocation failed, size = %d\n",oce_rq_buf_size);
1880 break;
1881 }
1882 pd->mbuf->m_nextpkt = NULL;
1883
1884 pd->mbuf->m_len = pd->mbuf->m_pkthdr.len = rq->cfg.frag_size;
1885
1886 rc = bus_dmamap_load_mbuf_sg(rq->tag,
1887 pd->map,
1888 pd->mbuf,
1889 segs, &nsegs, BUS_DMA_NOWAIT);
1890 if (rc) {
1891 m_free(pd->mbuf);
1892 device_printf(sc->dev, "bus_dmamap_load_mbuf_sg failed rc = %d\n", rc);
1893 break;
1894 }
1895
1896 if (nsegs != 1) {
1897 i--;
1898 continue;
1899 }
1900
1901 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_PREREAD);
1902
1903 rqe = RING_GET_PRODUCER_ITEM_VA(rq->ring, struct oce_nic_rqe);
1904 rqe->u0.s.frag_pa_hi = ADDR_HI(segs[0].ds_addr);
1905 rqe->u0.s.frag_pa_lo = ADDR_LO(segs[0].ds_addr);
1906 DW_SWAP(u32ptr(rqe), sizeof(struct oce_nic_rqe));
1907 RING_PUT(rq->ring, 1);
1908 added++;
1909 rq->pending++;
1910 }
1911 oce_max_rq_posts = sc->enable_hwlro ? OCE_HWLRO_MAX_RQ_POSTS : OCE_MAX_RQ_POSTS;
1912 if (added != 0) {
1913 for (i = added / oce_max_rq_posts; i > 0; i--) {
1914 rxdb_reg.bits.num_posted = oce_max_rq_posts;
1915 rxdb_reg.bits.qid = rq->rq_id;
1916 if(rq->islro) {
1917 val |= rq->rq_id & DB_LRO_RQ_ID_MASK;
1918 val |= oce_max_rq_posts << 16;
1919 OCE_WRITE_REG32(sc, db, DB_OFFSET, val);
1920 }else {
1921 OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0);
1922 }
1923 added -= oce_max_rq_posts;
1924 }
1925 if (added > 0) {
1926 rxdb_reg.bits.qid = rq->rq_id;
1927 rxdb_reg.bits.num_posted = added;
1928 if(rq->islro) {
1929 val |= rq->rq_id & DB_LRO_RQ_ID_MASK;
1930 val |= added << 16;
1931 OCE_WRITE_REG32(sc, db, DB_OFFSET, val);
1932 }else {
1933 OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0);
1934 }
1935 }
1936 }
1937
1938 return 0;
1939 }
1940
1941 static void
oce_check_rx_bufs(POCE_SOFTC sc,uint32_t num_cqes,struct oce_rq * rq)1942 oce_check_rx_bufs(POCE_SOFTC sc, uint32_t num_cqes, struct oce_rq *rq)
1943 {
1944 if (num_cqes) {
1945 oce_arm_cq(sc, rq->cq->cq_id, num_cqes, FALSE);
1946 if(!sc->enable_hwlro) {
1947 if((OCE_RQ_PACKET_ARRAY_SIZE - rq->pending) > 1)
1948 oce_alloc_rx_bufs(rq, ((OCE_RQ_PACKET_ARRAY_SIZE - rq->pending) - 1));
1949 }else {
1950 if ((OCE_RQ_PACKET_ARRAY_SIZE -1 - rq->pending) > 64)
1951 oce_alloc_rx_bufs(rq, 64);
1952 }
1953 }
1954
1955 return;
1956 }
1957
1958 uint16_t
oce_rq_handler_lro(void * arg)1959 oce_rq_handler_lro(void *arg)
1960 {
1961 struct oce_rq *rq = (struct oce_rq *)arg;
1962 struct oce_cq *cq = rq->cq;
1963 POCE_SOFTC sc = rq->parent;
1964 struct nic_hwlro_singleton_cqe *cqe;
1965 struct nic_hwlro_cqe_part2 *cqe2;
1966 int num_cqes = 0;
1967
1968 LOCK(&rq->rx_lock);
1969 bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
1970 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe);
1971 while (cqe->valid) {
1972 if(cqe->cqe_type == 0) { /* singleton cqe */
1973 /* we should not get singleton cqe after cqe1 on same rq */
1974 if(rq->cqe_firstpart != NULL) {
1975 device_printf(sc->dev, "Got singleton cqe after cqe1 \n");
1976 goto exit_rq_handler_lro;
1977 }
1978 if(cqe->error != 0) {
1979 rq->rx_stats.rxcp_err++;
1980 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1);
1981 }
1982 oce_rx_lro(rq, cqe, NULL);
1983 rq->rx_stats.rx_compl++;
1984 cqe->valid = 0;
1985 RING_GET(cq->ring, 1);
1986 num_cqes++;
1987 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled))
1988 break;
1989 }else if(cqe->cqe_type == 0x1) { /* first part */
1990 /* we should not get cqe1 after cqe1 on same rq */
1991 if(rq->cqe_firstpart != NULL) {
1992 device_printf(sc->dev, "Got cqe1 after cqe1 \n");
1993 goto exit_rq_handler_lro;
1994 }
1995 rq->cqe_firstpart = (struct nic_hwlro_cqe_part1 *)cqe;
1996 RING_GET(cq->ring, 1);
1997 }else if(cqe->cqe_type == 0x2) { /* second part */
1998 cqe2 = (struct nic_hwlro_cqe_part2 *)cqe;
1999 if(cqe2->error != 0) {
2000 rq->rx_stats.rxcp_err++;
2001 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1);
2002 }
2003 /* We should not get cqe2 without cqe1 */
2004 if(rq->cqe_firstpart == NULL) {
2005 device_printf(sc->dev, "Got cqe2 without cqe1 \n");
2006 goto exit_rq_handler_lro;
2007 }
2008 oce_rx_lro(rq, (struct nic_hwlro_singleton_cqe *)rq->cqe_firstpart, cqe2);
2009
2010 rq->rx_stats.rx_compl++;
2011 rq->cqe_firstpart->valid = 0;
2012 cqe2->valid = 0;
2013 rq->cqe_firstpart = NULL;
2014
2015 RING_GET(cq->ring, 1);
2016 num_cqes += 2;
2017 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled))
2018 break;
2019 }
2020
2021 bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2022 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe);
2023 }
2024 oce_check_rx_bufs(sc, num_cqes, rq);
2025 exit_rq_handler_lro:
2026 UNLOCK(&rq->rx_lock);
2027 return 0;
2028 }
2029
2030 /* Handle the Completion Queue for receive */
2031 uint16_t
oce_rq_handler(void * arg)2032 oce_rq_handler(void *arg)
2033 {
2034 struct epoch_tracker et;
2035 struct oce_rq *rq = (struct oce_rq *)arg;
2036 struct oce_cq *cq = rq->cq;
2037 POCE_SOFTC sc = rq->parent;
2038 struct oce_nic_rx_cqe *cqe;
2039 int num_cqes = 0;
2040
2041 NET_EPOCH_ENTER(et);
2042 if(rq->islro) {
2043 oce_rq_handler_lro(arg);
2044 NET_EPOCH_EXIT(et);
2045 return 0;
2046 }
2047 LOCK(&rq->rx_lock);
2048 bus_dmamap_sync(cq->ring->dma.tag,
2049 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2050 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe);
2051 while (cqe->u0.dw[2]) {
2052 DW_SWAP((uint32_t *) cqe, sizeof(oce_rq_cqe));
2053
2054 if (cqe->u0.s.error == 0) {
2055 oce_rx(rq, cqe);
2056 } else {
2057 rq->rx_stats.rxcp_err++;
2058 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1);
2059 /* Post L3/L4 errors to stack.*/
2060 oce_rx(rq, cqe);
2061 }
2062 rq->rx_stats.rx_compl++;
2063 cqe->u0.dw[2] = 0;
2064
2065 #if defined(INET6) || defined(INET)
2066 if (IF_LRO_ENABLED(sc) && rq->lro_pkts_queued >= 16) {
2067 oce_rx_flush_lro(rq);
2068 }
2069 #endif
2070
2071 RING_GET(cq->ring, 1);
2072 bus_dmamap_sync(cq->ring->dma.tag,
2073 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2074 cqe =
2075 RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe);
2076 num_cqes++;
2077 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled))
2078 break;
2079 }
2080
2081 #if defined(INET6) || defined(INET)
2082 if (IF_LRO_ENABLED(sc))
2083 oce_rx_flush_lro(rq);
2084 #endif
2085
2086 oce_check_rx_bufs(sc, num_cqes, rq);
2087 UNLOCK(&rq->rx_lock);
2088 NET_EPOCH_EXIT(et);
2089 return 0;
2090
2091 }
2092
2093 /*****************************************************************************
2094 * Helper function prototypes in this file *
2095 *****************************************************************************/
2096
2097 static void
oce_attach_ifp(POCE_SOFTC sc)2098 oce_attach_ifp(POCE_SOFTC sc)
2099 {
2100
2101 sc->ifp = if_alloc(IFT_ETHER);
2102
2103 ifmedia_init(&sc->media, IFM_IMASK, oce_media_change, oce_media_status);
2104 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2105 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO);
2106
2107 if_setflags(sc->ifp, IFF_BROADCAST | IFF_MULTICAST);
2108 if_setioctlfn(sc->ifp, oce_ioctl);
2109 if_setstartfn(sc->ifp, oce_start);
2110 if_setinitfn(sc->ifp, oce_init);
2111 if_setmtu(sc->ifp, ETHERMTU);
2112 if_setsoftc(sc->ifp, sc);
2113 if_settransmitfn(sc->ifp, oce_multiq_start);
2114 if_setqflushfn(sc->ifp, oce_multiq_flush);
2115
2116 if_initname(sc->ifp,
2117 device_get_name(sc->dev), device_get_unit(sc->dev));
2118
2119 if_setsendqlen(sc->ifp, OCE_MAX_TX_DESC - 1);
2120 if_setsendqready(sc->ifp);
2121
2122 if_sethwassist(sc->ifp, OCE_IF_HWASSIST);
2123 if_sethwassistbits(sc->ifp, CSUM_TSO, 0);
2124 if_sethwassistbits(sc->ifp, (CSUM_IP | CSUM_TCP | CSUM_UDP), 0);
2125
2126 if_setcapabilities(sc->ifp, OCE_IF_CAPABILITIES);
2127 if_setcapabilitiesbit(sc->ifp, IFCAP_HWCSUM, 0);
2128 if_setcapabilitiesbit(sc->ifp, IFCAP_VLAN_HWFILTER, 0);
2129
2130 #if defined(INET6) || defined(INET)
2131 if_setcapabilitiesbit(sc->ifp, IFCAP_TSO, 0);
2132 if_setcapabilitiesbit(sc->ifp, IFCAP_LRO, 0);
2133 if_setcapabilitiesbit(sc->ifp, IFCAP_VLAN_HWTSO, 0);
2134 #endif
2135
2136 if_setcapenable(sc->ifp, if_getcapabilities(sc->ifp));
2137 if_setbaudrate(sc->ifp, IF_Gbps(10));
2138
2139 if_sethwtsomax(sc->ifp, 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN));
2140 if_sethwtsomaxsegcount(sc->ifp, OCE_MAX_TX_ELEMENTS);
2141 if_sethwtsomaxsegsize(sc->ifp, 4096);
2142
2143 ether_ifattach(sc->ifp, sc->macaddr.mac_addr);
2144 }
2145
2146 static void
oce_add_vlan(void * arg,if_t ifp,uint16_t vtag)2147 oce_add_vlan(void *arg, if_t ifp, uint16_t vtag)
2148 {
2149 POCE_SOFTC sc = if_getsoftc(ifp);
2150
2151 if (if_getsoftc(ifp) != arg)
2152 return;
2153 if ((vtag == 0) || (vtag > 4095))
2154 return;
2155
2156 sc->vlan_tag[vtag] = 1;
2157 sc->vlans_added++;
2158 if (sc->vlans_added <= (sc->max_vlans + 1))
2159 oce_vid_config(sc);
2160 }
2161
2162 static void
oce_del_vlan(void * arg,if_t ifp,uint16_t vtag)2163 oce_del_vlan(void *arg, if_t ifp, uint16_t vtag)
2164 {
2165 POCE_SOFTC sc = if_getsoftc(ifp);
2166
2167 if (if_getsoftc(ifp) != arg)
2168 return;
2169 if ((vtag == 0) || (vtag > 4095))
2170 return;
2171
2172 sc->vlan_tag[vtag] = 0;
2173 sc->vlans_added--;
2174 oce_vid_config(sc);
2175 }
2176
2177 /*
2178 * A max of 64 vlans can be configured in BE. If the user configures
2179 * more, place the card in vlan promiscuous mode.
2180 */
2181 static int
oce_vid_config(POCE_SOFTC sc)2182 oce_vid_config(POCE_SOFTC sc)
2183 {
2184 struct normal_vlan vtags[MAX_VLANFILTER_SIZE];
2185 uint16_t ntags = 0, i;
2186 int status = 0;
2187
2188 if ((sc->vlans_added <= MAX_VLANFILTER_SIZE) &&
2189 (if_getcapenable(sc->ifp) & IFCAP_VLAN_HWFILTER)) {
2190 for (i = 0; i < MAX_VLANS; i++) {
2191 if (sc->vlan_tag[i]) {
2192 vtags[ntags].vtag = i;
2193 ntags++;
2194 }
2195 }
2196 if (ntags)
2197 status = oce_config_vlan(sc, (uint8_t) sc->if_id,
2198 vtags, ntags, 1, 0);
2199 } else
2200 status = oce_config_vlan(sc, (uint8_t) sc->if_id,
2201 NULL, 0, 1, 1);
2202 return status;
2203 }
2204
2205 static void
oce_mac_addr_set(POCE_SOFTC sc)2206 oce_mac_addr_set(POCE_SOFTC sc)
2207 {
2208 uint32_t old_pmac_id = sc->pmac_id;
2209 int status = 0;
2210
2211 status = bcmp((if_getlladdr(sc->ifp)), sc->macaddr.mac_addr,
2212 sc->macaddr.size_of_struct);
2213 if (!status)
2214 return;
2215
2216 status = oce_mbox_macaddr_add(sc, (uint8_t *)(if_getlladdr(sc->ifp)),
2217 sc->if_id, &sc->pmac_id);
2218 if (!status) {
2219 status = oce_mbox_macaddr_del(sc, sc->if_id, old_pmac_id);
2220 bcopy((if_getlladdr(sc->ifp)), sc->macaddr.mac_addr,
2221 sc->macaddr.size_of_struct);
2222 }
2223 if (status)
2224 device_printf(sc->dev, "Failed update macaddress\n");
2225
2226 }
2227
2228 static int
oce_handle_passthrough(if_t ifp,caddr_t data)2229 oce_handle_passthrough(if_t ifp, caddr_t data)
2230 {
2231 POCE_SOFTC sc = if_getsoftc(ifp);
2232 struct ifreq *ifr = (struct ifreq *)data;
2233 int rc = ENXIO;
2234 char cookie[32] = {0};
2235 void *priv_data = ifr_data_get_ptr(ifr);
2236 void *ioctl_ptr;
2237 uint32_t req_size;
2238 struct mbx_hdr req;
2239 OCE_DMA_MEM dma_mem;
2240
2241 if (copyin(priv_data, cookie, strlen(IOCTL_COOKIE)))
2242 return EFAULT;
2243
2244 if (memcmp(cookie, IOCTL_COOKIE, strlen(IOCTL_COOKIE)))
2245 return EINVAL;
2246
2247 ioctl_ptr = (char *)priv_data + strlen(IOCTL_COOKIE);
2248 if (copyin(ioctl_ptr, &req, sizeof(struct mbx_hdr)))
2249 return EFAULT;
2250
2251 req_size = le32toh(req.u0.req.request_length);
2252 if (req_size > 65536)
2253 return EINVAL;
2254
2255 req_size += sizeof(struct mbx_hdr);
2256 rc = oce_dma_alloc(sc, req_size, &dma_mem, 0);
2257 if (rc)
2258 return ENOMEM;
2259
2260 if (copyin(ioctl_ptr, OCE_DMAPTR(&dma_mem,char), req_size)) {
2261 rc = EFAULT;
2262 goto dma_free;
2263 }
2264
2265 rc = oce_pass_through_mbox(sc, &dma_mem, req_size);
2266 if (rc) {
2267 rc = EIO;
2268 goto dma_free;
2269 }
2270
2271 if (copyout(OCE_DMAPTR(&dma_mem,char), ioctl_ptr, req_size)) {
2272 rc = EFAULT;
2273 goto dma_free;
2274 }
2275
2276 /*
2277 firmware is filling all the attributes for this ioctl except
2278 the driver version..so fill it
2279 */
2280 if(req.u0.rsp.opcode == OPCODE_COMMON_GET_CNTL_ATTRIBUTES) {
2281 struct mbx_common_get_cntl_attr *fw_cmd =
2282 (struct mbx_common_get_cntl_attr *)ioctl_ptr;
2283 _Static_assert(sizeof(COMPONENT_REVISION) <=
2284 sizeof(fw_cmd->params.rsp.cntl_attr_info.hba_attr.drv_ver_str),
2285 "driver version string too long");
2286
2287 rc = copyout(COMPONENT_REVISION,
2288 fw_cmd->params.rsp.cntl_attr_info.hba_attr.drv_ver_str,
2289 sizeof(COMPONENT_REVISION));
2290 }
2291
2292 dma_free:
2293 oce_dma_free(sc, &dma_mem);
2294 return rc;
2295
2296 }
2297
2298 static void
oce_eqd_set_periodic(POCE_SOFTC sc)2299 oce_eqd_set_periodic(POCE_SOFTC sc)
2300 {
2301 struct oce_set_eqd set_eqd[OCE_MAX_EQ];
2302 struct oce_aic_obj *aic;
2303 struct oce_eq *eqo;
2304 uint64_t now = 0, delta;
2305 int eqd, i, num = 0;
2306 uint32_t tx_reqs = 0, rxpkts = 0, pps;
2307 struct oce_wq *wq;
2308 struct oce_rq *rq;
2309
2310 #define ticks_to_msecs(t) (1000 * (t) / hz)
2311
2312 for (i = 0 ; i < sc->neqs; i++) {
2313 eqo = sc->eq[i];
2314 aic = &sc->aic_obj[i];
2315 /* When setting the static eq delay from the user space */
2316 if (!aic->enable) {
2317 if (aic->ticks)
2318 aic->ticks = 0;
2319 eqd = aic->et_eqd;
2320 goto modify_eqd;
2321 }
2322
2323 if (i == 0) {
2324 rq = sc->rq[0];
2325 rxpkts = rq->rx_stats.rx_pkts;
2326 } else
2327 rxpkts = 0;
2328 if (i + 1 < sc->nrqs) {
2329 rq = sc->rq[i + 1];
2330 rxpkts += rq->rx_stats.rx_pkts;
2331 }
2332 if (i < sc->nwqs) {
2333 wq = sc->wq[i];
2334 tx_reqs = wq->tx_stats.tx_reqs;
2335 } else
2336 tx_reqs = 0;
2337 now = ticks;
2338
2339 if (!aic->ticks || now < aic->ticks ||
2340 rxpkts < aic->prev_rxpkts || tx_reqs < aic->prev_txreqs) {
2341 aic->prev_rxpkts = rxpkts;
2342 aic->prev_txreqs = tx_reqs;
2343 aic->ticks = now;
2344 continue;
2345 }
2346
2347 delta = ticks_to_msecs(now - aic->ticks);
2348
2349 pps = (((uint32_t)(rxpkts - aic->prev_rxpkts) * 1000) / delta) +
2350 (((uint32_t)(tx_reqs - aic->prev_txreqs) * 1000) / delta);
2351 eqd = (pps / 15000) << 2;
2352 if (eqd < 8)
2353 eqd = 0;
2354
2355 /* Make sure that the eq delay is in the known range */
2356 eqd = min(eqd, aic->max_eqd);
2357 eqd = max(eqd, aic->min_eqd);
2358
2359 aic->prev_rxpkts = rxpkts;
2360 aic->prev_txreqs = tx_reqs;
2361 aic->ticks = now;
2362
2363 modify_eqd:
2364 if (eqd != aic->cur_eqd) {
2365 set_eqd[num].delay_multiplier = (eqd * 65)/100;
2366 set_eqd[num].eq_id = eqo->eq_id;
2367 aic->cur_eqd = eqd;
2368 num++;
2369 }
2370 }
2371
2372 /* Is there atleast one eq that needs to be modified? */
2373 for(i = 0; i < num; i += 8) {
2374 if((num - i) >=8 )
2375 oce_mbox_eqd_modify_periodic(sc, &set_eqd[i], 8);
2376 else
2377 oce_mbox_eqd_modify_periodic(sc, &set_eqd[i], (num - i));
2378 }
2379
2380 }
2381
oce_detect_hw_error(POCE_SOFTC sc)2382 static void oce_detect_hw_error(POCE_SOFTC sc)
2383 {
2384
2385 uint32_t ue_low = 0, ue_high = 0, ue_low_mask = 0, ue_high_mask = 0;
2386 uint32_t sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0;
2387 uint32_t i;
2388
2389 if (sc->hw_error)
2390 return;
2391
2392 if (IS_XE201(sc)) {
2393 sliport_status = OCE_READ_REG32(sc, db, SLIPORT_STATUS_OFFSET);
2394 if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2395 sliport_err1 = OCE_READ_REG32(sc, db, SLIPORT_ERROR1_OFFSET);
2396 sliport_err2 = OCE_READ_REG32(sc, db, SLIPORT_ERROR2_OFFSET);
2397 }
2398 } else {
2399 ue_low = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW);
2400 ue_high = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HIGH);
2401 ue_low_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW_MASK);
2402 ue_high_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HI_MASK);
2403
2404 ue_low = (ue_low & ~ue_low_mask);
2405 ue_high = (ue_high & ~ue_high_mask);
2406 }
2407
2408 /* On certain platforms BE hardware can indicate spurious UEs.
2409 * Allow the h/w to stop working completely in case of a real UE.
2410 * Hence not setting the hw_error for UE detection.
2411 */
2412 if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2413 sc->hw_error = TRUE;
2414 device_printf(sc->dev, "Error detected in the card\n");
2415 }
2416
2417 if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2418 device_printf(sc->dev,
2419 "ERR: sliport status 0x%x\n", sliport_status);
2420 device_printf(sc->dev,
2421 "ERR: sliport error1 0x%x\n", sliport_err1);
2422 device_printf(sc->dev,
2423 "ERR: sliport error2 0x%x\n", sliport_err2);
2424 }
2425
2426 if (ue_low) {
2427 for (i = 0; ue_low; ue_low >>= 1, i++) {
2428 if (ue_low & 1)
2429 device_printf(sc->dev, "UE: %s bit set\n",
2430 ue_status_low_desc[i]);
2431 }
2432 }
2433
2434 if (ue_high) {
2435 for (i = 0; ue_high; ue_high >>= 1, i++) {
2436 if (ue_high & 1)
2437 device_printf(sc->dev, "UE: %s bit set\n",
2438 ue_status_hi_desc[i]);
2439 }
2440 }
2441
2442 }
2443
2444 static void
oce_local_timer(void * arg)2445 oce_local_timer(void *arg)
2446 {
2447 POCE_SOFTC sc = arg;
2448 int i = 0;
2449
2450 oce_detect_hw_error(sc);
2451 oce_refresh_nic_stats(sc);
2452 oce_refresh_queue_stats(sc);
2453 oce_mac_addr_set(sc);
2454
2455 /* TX Watch Dog*/
2456 for (i = 0; i < sc->nwqs; i++)
2457 oce_tx_restart(sc, sc->wq[i]);
2458
2459 /* calculate and set the eq delay for optimal interrupt rate */
2460 if (IS_BE(sc) || IS_SH(sc))
2461 oce_eqd_set_periodic(sc);
2462
2463 callout_reset(&sc->timer, hz, oce_local_timer, sc);
2464 }
2465
2466 static void
oce_tx_compl_clean(POCE_SOFTC sc)2467 oce_tx_compl_clean(POCE_SOFTC sc)
2468 {
2469 struct oce_wq *wq;
2470 int i = 0, timeo = 0, num_wqes = 0;
2471 int pending_txqs = sc->nwqs;
2472
2473 /* Stop polling for compls when HW has been silent for 10ms or
2474 * hw_error or no outstanding completions expected
2475 */
2476 do {
2477 pending_txqs = sc->nwqs;
2478
2479 for_all_wq_queues(sc, wq, i) {
2480 num_wqes = oce_wq_handler(wq);
2481
2482 if(num_wqes)
2483 timeo = 0;
2484
2485 if(!wq->ring->num_used)
2486 pending_txqs--;
2487 }
2488
2489 if (pending_txqs == 0 || ++timeo > 10 || sc->hw_error)
2490 break;
2491
2492 DELAY(1000);
2493 } while (TRUE);
2494
2495 for_all_wq_queues(sc, wq, i) {
2496 while(wq->ring->num_used) {
2497 LOCK(&wq->tx_compl_lock);
2498 oce_process_tx_completion(wq);
2499 UNLOCK(&wq->tx_compl_lock);
2500 }
2501 }
2502
2503 }
2504
2505 /* NOTE : This should only be called holding
2506 * DEVICE_LOCK.
2507 */
2508 static void
oce_if_deactivate(POCE_SOFTC sc)2509 oce_if_deactivate(POCE_SOFTC sc)
2510 {
2511 int i;
2512 struct oce_rq *rq;
2513 struct oce_wq *wq;
2514 struct oce_eq *eq;
2515
2516 if_setdrvflagbits(sc->ifp, 0, (IFF_DRV_RUNNING | IFF_DRV_OACTIVE));
2517
2518 oce_tx_compl_clean(sc);
2519
2520 /* Stop intrs and finish any bottom halves pending */
2521 oce_hw_intr_disable(sc);
2522
2523 /* Since taskqueue_drain takes a Gaint Lock, We should not acquire
2524 any other lock. So unlock device lock and require after
2525 completing taskqueue_drain.
2526 */
2527 UNLOCK(&sc->dev_lock);
2528 for (i = 0; i < sc->intr_count; i++) {
2529 if (sc->intrs[i].tq != NULL) {
2530 taskqueue_drain(sc->intrs[i].tq, &sc->intrs[i].task);
2531 }
2532 }
2533 LOCK(&sc->dev_lock);
2534
2535 /* Delete RX queue in card with flush param */
2536 oce_stop_rx(sc);
2537
2538 /* Invalidate any pending cq and eq entries*/
2539 for_all_evnt_queues(sc, eq, i)
2540 oce_drain_eq(eq);
2541 for_all_rq_queues(sc, rq, i)
2542 oce_drain_rq_cq(rq);
2543 for_all_wq_queues(sc, wq, i)
2544 oce_drain_wq_cq(wq);
2545
2546 /* But still we need to get MCC aync events.
2547 So enable intrs and also arm first EQ
2548 */
2549 oce_hw_intr_enable(sc);
2550 oce_arm_eq(sc, sc->eq[0]->eq_id, 0, TRUE, FALSE);
2551
2552 DELAY(10);
2553 }
2554
2555 static void
oce_if_activate(POCE_SOFTC sc)2556 oce_if_activate(POCE_SOFTC sc)
2557 {
2558 struct oce_eq *eq;
2559 struct oce_rq *rq;
2560 struct oce_wq *wq;
2561 int i, rc = 0;
2562
2563 if_setdrvflagbits(sc->ifp, IFF_DRV_RUNNING , 0);
2564
2565 oce_hw_intr_disable(sc);
2566
2567 oce_start_rx(sc);
2568
2569 for_all_rq_queues(sc, rq, i) {
2570 rc = oce_start_rq(rq);
2571 if (rc)
2572 device_printf(sc->dev, "Unable to start RX\n");
2573 }
2574
2575 for_all_wq_queues(sc, wq, i) {
2576 rc = oce_start_wq(wq);
2577 if (rc)
2578 device_printf(sc->dev, "Unable to start TX\n");
2579 }
2580
2581 for_all_evnt_queues(sc, eq, i)
2582 oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE);
2583
2584 oce_hw_intr_enable(sc);
2585
2586 }
2587
2588 static void
process_link_state(POCE_SOFTC sc,struct oce_async_cqe_link_state * acqe)2589 process_link_state(POCE_SOFTC sc, struct oce_async_cqe_link_state *acqe)
2590 {
2591 /* Update Link status */
2592 if ((acqe->u0.s.link_status & ~ASYNC_EVENT_LOGICAL) ==
2593 ASYNC_EVENT_LINK_UP) {
2594 sc->link_status = ASYNC_EVENT_LINK_UP;
2595 if_link_state_change(sc->ifp, LINK_STATE_UP);
2596 } else {
2597 sc->link_status = ASYNC_EVENT_LINK_DOWN;
2598 if_link_state_change(sc->ifp, LINK_STATE_DOWN);
2599 }
2600 }
2601
oce_async_grp5_osbmc_process(POCE_SOFTC sc,struct oce_async_evt_grp5_os2bmc * evt)2602 static void oce_async_grp5_osbmc_process(POCE_SOFTC sc,
2603 struct oce_async_evt_grp5_os2bmc *evt)
2604 {
2605 DW_SWAP(evt, sizeof(struct oce_async_evt_grp5_os2bmc));
2606 if (evt->u.s.mgmt_enable)
2607 sc->flags |= OCE_FLAGS_OS2BMC;
2608 else
2609 return;
2610
2611 sc->bmc_filt_mask = evt->u.s.arp_filter;
2612 sc->bmc_filt_mask |= (evt->u.s.dhcp_client_filt << 1);
2613 sc->bmc_filt_mask |= (evt->u.s.dhcp_server_filt << 2);
2614 sc->bmc_filt_mask |= (evt->u.s.net_bios_filt << 3);
2615 sc->bmc_filt_mask |= (evt->u.s.bcast_filt << 4);
2616 sc->bmc_filt_mask |= (evt->u.s.ipv6_nbr_filt << 5);
2617 sc->bmc_filt_mask |= (evt->u.s.ipv6_ra_filt << 6);
2618 sc->bmc_filt_mask |= (evt->u.s.ipv6_ras_filt << 7);
2619 sc->bmc_filt_mask |= (evt->u.s.mcast_filt << 8);
2620 }
2621
oce_process_grp5_events(POCE_SOFTC sc,struct oce_mq_cqe * cqe)2622 static void oce_process_grp5_events(POCE_SOFTC sc, struct oce_mq_cqe *cqe)
2623 {
2624 struct oce_async_event_grp5_pvid_state *gcqe;
2625 struct oce_async_evt_grp5_os2bmc *bmccqe;
2626
2627 switch (cqe->u0.s.async_type) {
2628 case ASYNC_EVENT_PVID_STATE:
2629 /* GRP5 PVID */
2630 gcqe = (struct oce_async_event_grp5_pvid_state *)cqe;
2631 if (gcqe->enabled)
2632 sc->pvid = gcqe->tag & VLAN_VID_MASK;
2633 else
2634 sc->pvid = 0;
2635 break;
2636 case ASYNC_EVENT_OS2BMC:
2637 bmccqe = (struct oce_async_evt_grp5_os2bmc *)cqe;
2638 oce_async_grp5_osbmc_process(sc, bmccqe);
2639 break;
2640 default:
2641 break;
2642 }
2643 }
2644
2645 /* Handle the Completion Queue for the Mailbox/Async notifications */
2646 uint16_t
oce_mq_handler(void * arg)2647 oce_mq_handler(void *arg)
2648 {
2649 struct oce_mq *mq = (struct oce_mq *)arg;
2650 POCE_SOFTC sc = mq->parent;
2651 struct oce_cq *cq = mq->cq;
2652 int num_cqes = 0, evt_type = 0, optype = 0;
2653 struct oce_mq_cqe *cqe;
2654 struct oce_async_cqe_link_state *acqe;
2655 struct oce_async_event_qnq *dbgcqe;
2656
2657 bus_dmamap_sync(cq->ring->dma.tag,
2658 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2659 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe);
2660
2661 while (cqe->u0.dw[3]) {
2662 DW_SWAP((uint32_t *) cqe, sizeof(oce_mq_cqe));
2663 if (cqe->u0.s.async_event) {
2664 evt_type = cqe->u0.s.event_type;
2665 optype = cqe->u0.s.async_type;
2666 if (evt_type == ASYNC_EVENT_CODE_LINK_STATE) {
2667 /* Link status evt */
2668 acqe = (struct oce_async_cqe_link_state *)cqe;
2669 process_link_state(sc, acqe);
2670 } else if (evt_type == ASYNC_EVENT_GRP5) {
2671 oce_process_grp5_events(sc, cqe);
2672 } else if (evt_type == ASYNC_EVENT_CODE_DEBUG &&
2673 optype == ASYNC_EVENT_DEBUG_QNQ) {
2674 dbgcqe = (struct oce_async_event_qnq *)cqe;
2675 if(dbgcqe->valid)
2676 sc->qnqid = dbgcqe->vlan_tag;
2677 sc->qnq_debug_event = TRUE;
2678 }
2679 }
2680 cqe->u0.dw[3] = 0;
2681 RING_GET(cq->ring, 1);
2682 bus_dmamap_sync(cq->ring->dma.tag,
2683 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2684 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe);
2685 num_cqes++;
2686 }
2687
2688 if (num_cqes)
2689 oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE);
2690
2691 return 0;
2692 }
2693
2694 static void
setup_max_queues_want(POCE_SOFTC sc)2695 setup_max_queues_want(POCE_SOFTC sc)
2696 {
2697 /* Check if it is FLEX machine. Is so dont use RSS */
2698 if ((sc->function_mode & FNM_FLEX10_MODE) ||
2699 (sc->function_mode & FNM_UMC_MODE) ||
2700 (sc->function_mode & FNM_VNIC_MODE) ||
2701 (!is_rss_enabled(sc)) ||
2702 IS_BE2(sc)) {
2703 sc->nrqs = 1;
2704 sc->nwqs = 1;
2705 } else {
2706 sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1;
2707 sc->nwqs = MIN(OCE_NCPUS, sc->nrssqs);
2708 }
2709
2710 if (IS_BE2(sc) && is_rss_enabled(sc))
2711 sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1;
2712 }
2713
2714 static void
update_queues_got(POCE_SOFTC sc)2715 update_queues_got(POCE_SOFTC sc)
2716 {
2717 if (is_rss_enabled(sc)) {
2718 sc->nrqs = sc->intr_count + 1;
2719 sc->nwqs = sc->intr_count;
2720 } else {
2721 sc->nrqs = 1;
2722 sc->nwqs = 1;
2723 }
2724
2725 if (IS_BE2(sc))
2726 sc->nwqs = 1;
2727 }
2728
2729 static int
oce_check_ipv6_ext_hdr(struct mbuf * m)2730 oce_check_ipv6_ext_hdr(struct mbuf *m)
2731 {
2732 struct ether_header *eh = mtod(m, struct ether_header *);
2733 caddr_t m_datatemp = m->m_data;
2734
2735 if (eh->ether_type == htons(ETHERTYPE_IPV6)) {
2736 m->m_data += sizeof(struct ether_header);
2737 struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *);
2738
2739 if((ip6->ip6_nxt != IPPROTO_TCP) && \
2740 (ip6->ip6_nxt != IPPROTO_UDP)){
2741 struct ip6_ext *ip6e = NULL;
2742 m->m_data += sizeof(struct ip6_hdr);
2743
2744 ip6e = (struct ip6_ext *) mtod(m, struct ip6_ext *);
2745 if(ip6e->ip6e_len == 0xff) {
2746 m->m_data = m_datatemp;
2747 return TRUE;
2748 }
2749 }
2750 m->m_data = m_datatemp;
2751 }
2752 return FALSE;
2753 }
2754
2755 static int
is_be3_a1(POCE_SOFTC sc)2756 is_be3_a1(POCE_SOFTC sc)
2757 {
2758 if((sc->flags & OCE_FLAGS_BE3) && ((sc->asic_revision & 0xFF) < 2)) {
2759 return TRUE;
2760 }
2761 return FALSE;
2762 }
2763
2764 static struct mbuf *
oce_insert_vlan_tag(POCE_SOFTC sc,struct mbuf * m,boolean_t * complete)2765 oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete)
2766 {
2767 uint16_t vlan_tag = 0;
2768
2769 if(!M_WRITABLE(m))
2770 return NULL;
2771
2772 /* Embed vlan tag in the packet if it is not part of it */
2773 if(m->m_flags & M_VLANTAG) {
2774 vlan_tag = EVL_VLANOFTAG(m->m_pkthdr.ether_vtag);
2775 m->m_flags &= ~M_VLANTAG;
2776 }
2777
2778 /* if UMC, ignore vlan tag insertion and instead insert pvid */
2779 if(sc->pvid) {
2780 if(!vlan_tag)
2781 vlan_tag = sc->pvid;
2782 if (complete)
2783 *complete = FALSE;
2784 }
2785
2786 if(vlan_tag) {
2787 m = ether_vlanencap(m, vlan_tag);
2788 }
2789
2790 if(sc->qnqid) {
2791 m = ether_vlanencap(m, sc->qnqid);
2792
2793 if (complete)
2794 *complete = FALSE;
2795 }
2796 return m;
2797 }
2798
2799 static int
oce_tx_asic_stall_verify(POCE_SOFTC sc,struct mbuf * m)2800 oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m)
2801 {
2802 if(is_be3_a1(sc) && IS_QNQ_OR_UMC(sc) && \
2803 oce_check_ipv6_ext_hdr(m)) {
2804 return TRUE;
2805 }
2806 return FALSE;
2807 }
2808
2809 static void
oce_get_config(POCE_SOFTC sc)2810 oce_get_config(POCE_SOFTC sc)
2811 {
2812 int rc = 0;
2813 uint32_t max_rss = 0;
2814
2815 if ((IS_BE(sc) || IS_SH(sc)) && (!sc->be3_native))
2816 max_rss = OCE_LEGACY_MODE_RSS;
2817 else
2818 max_rss = OCE_MAX_RSS;
2819
2820 if (!IS_BE(sc)) {
2821 rc = oce_get_profile_config(sc, max_rss);
2822 if (rc) {
2823 sc->nwqs = OCE_MAX_WQ;
2824 sc->nrssqs = max_rss;
2825 sc->nrqs = sc->nrssqs + 1;
2826 }
2827 }
2828 else { /* For BE3 don't rely on fw for determining the resources */
2829 sc->nrssqs = max_rss;
2830 sc->nrqs = sc->nrssqs + 1;
2831 sc->nwqs = OCE_MAX_WQ;
2832 sc->max_vlans = MAX_VLANFILTER_SIZE;
2833 }
2834 }
2835
2836 static void
oce_rdma_close(void)2837 oce_rdma_close(void)
2838 {
2839 if (oce_rdma_if != NULL) {
2840 oce_rdma_if = NULL;
2841 }
2842 }
2843
2844 static void
oce_get_mac_addr(POCE_SOFTC sc,uint8_t * macaddr)2845 oce_get_mac_addr(POCE_SOFTC sc, uint8_t *macaddr)
2846 {
2847 memcpy(macaddr, sc->macaddr.mac_addr, 6);
2848 }
2849
2850 int
oce_register_rdma(POCE_RDMA_INFO rdma_info,POCE_RDMA_IF rdma_if)2851 oce_register_rdma(POCE_RDMA_INFO rdma_info, POCE_RDMA_IF rdma_if)
2852 {
2853 POCE_SOFTC sc;
2854 struct oce_dev_info di;
2855 int i;
2856
2857 if ((rdma_info == NULL) || (rdma_if == NULL)) {
2858 return -EINVAL;
2859 }
2860
2861 if ((rdma_info->size != OCE_RDMA_INFO_SIZE) ||
2862 (rdma_if->size != OCE_RDMA_IF_SIZE)) {
2863 return -ENXIO;
2864 }
2865
2866 rdma_info->close = oce_rdma_close;
2867 rdma_info->mbox_post = oce_mbox_post;
2868 rdma_info->common_req_hdr_init = mbx_common_req_hdr_init;
2869 rdma_info->get_mac_addr = oce_get_mac_addr;
2870
2871 oce_rdma_if = rdma_if;
2872
2873 sc = softc_head;
2874 while (sc != NULL) {
2875 if (oce_rdma_if->announce != NULL) {
2876 memset(&di, 0, sizeof(di));
2877 di.dev = sc->dev;
2878 di.softc = sc;
2879 di.ifp = sc->ifp;
2880 di.db_bhandle = sc->db_bhandle;
2881 di.db_btag = sc->db_btag;
2882 di.db_page_size = 4096;
2883 if (sc->flags & OCE_FLAGS_USING_MSIX) {
2884 di.intr_mode = OCE_INTERRUPT_MODE_MSIX;
2885 } else if (sc->flags & OCE_FLAGS_USING_MSI) {
2886 di.intr_mode = OCE_INTERRUPT_MODE_MSI;
2887 } else {
2888 di.intr_mode = OCE_INTERRUPT_MODE_INTX;
2889 }
2890 di.dev_family = OCE_GEN2_FAMILY; // fixme: must detect skyhawk
2891 if (di.intr_mode != OCE_INTERRUPT_MODE_INTX) {
2892 di.msix.num_vectors = sc->intr_count + sc->roce_intr_count;
2893 di.msix.start_vector = sc->intr_count;
2894 for (i=0; i<di.msix.num_vectors; i++) {
2895 di.msix.vector_list[i] = sc->intrs[i].vector;
2896 }
2897 } else {
2898 }
2899 memcpy(di.mac_addr, sc->macaddr.mac_addr, 6);
2900 di.vendor_id = pci_get_vendor(sc->dev);
2901 di.dev_id = pci_get_device(sc->dev);
2902
2903 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) {
2904 di.flags |= OCE_RDMA_INFO_RDMA_SUPPORTED;
2905 }
2906
2907 rdma_if->announce(&di);
2908 sc = sc->next;
2909 }
2910 }
2911
2912 return 0;
2913 }
2914
2915 static void
oce_read_env_variables(POCE_SOFTC sc)2916 oce_read_env_variables( POCE_SOFTC sc )
2917 {
2918 char *value = NULL;
2919 int rc = 0;
2920
2921 /* read if user wants to enable hwlro or swlro */
2922 //value = getenv("oce_enable_hwlro");
2923 if(value && IS_SH(sc)) {
2924 sc->enable_hwlro = strtol(value, NULL, 10);
2925 if(sc->enable_hwlro) {
2926 rc = oce_mbox_nic_query_lro_capabilities(sc, NULL, NULL);
2927 if(rc) {
2928 device_printf(sc->dev, "no hardware lro support\n");
2929 device_printf(sc->dev, "software lro enabled\n");
2930 sc->enable_hwlro = 0;
2931 }else {
2932 device_printf(sc->dev, "hardware lro enabled\n");
2933 oce_max_rsp_handled = 32;
2934 }
2935 }else {
2936 device_printf(sc->dev, "software lro enabled\n");
2937 }
2938 }else {
2939 sc->enable_hwlro = 0;
2940 }
2941
2942 /* read mbuf size */
2943 //value = getenv("oce_rq_buf_size");
2944 if(value && IS_SH(sc)) {
2945 oce_rq_buf_size = strtol(value, NULL, 10);
2946 switch(oce_rq_buf_size) {
2947 case 2048:
2948 case 4096:
2949 case 9216:
2950 case 16384:
2951 break;
2952
2953 default:
2954 device_printf(sc->dev, " Supported oce_rq_buf_size values are 2K, 4K, 9K, 16K \n");
2955 oce_rq_buf_size = 2048;
2956 }
2957 }
2958
2959 return;
2960 }
2961