Lines Matching +full:pcie +full:- +full:mirror
1 // SPDX-License-Identifier: GPL-2.0-only
9 #include <linux/dma-mapping.h>
95 regmap_update_bits(dev->hw->regs, reg, mask | val, val); in wed_m32()
131 return readl(dev->wlan.base + reg); in wifi_r32()
137 writel(val, dev->wlan.base + reg); in wifi_w32()
157 if (!mtk_wed_is_v3_or_greater(dev->hw)) in mtk_wdma_v3_rx_reset()
166 dev_err(dev->hw->dev, "rx reset failed\n"); in mtk_wdma_v3_rx_reset()
171 dev_err(dev->hw->dev, "rx reset failed\n"); in mtk_wdma_v3_rx_reset()
179 dev_err(dev->hw->dev, "rx reset failed\n"); in mtk_wdma_v3_rx_reset()
184 dev_err(dev->hw->dev, "rx reset failed\n"); in mtk_wdma_v3_rx_reset()
246 dev_err(dev->hw->dev, "rx reset failed\n"); in mtk_wdma_rx_reset()
252 for (i = 0; i < ARRAY_SIZE(dev->rx_wdma); i++) { in mtk_wdma_rx_reset()
253 if (dev->rx_wdma[i].desc) in mtk_wdma_rx_reset()
285 if (!mtk_wed_is_v3_or_greater(dev->hw)) in mtk_wdma_v3_tx_reset()
294 dev_err(dev->hw->dev, "tx reset failed\n"); in mtk_wdma_v3_tx_reset()
299 dev_err(dev->hw->dev, "tx reset failed\n"); in mtk_wdma_v3_tx_reset()
307 dev_err(dev->hw->dev, "tx reset failed\n"); in mtk_wdma_v3_tx_reset()
312 dev_err(dev->hw->dev, "tx reset failed\n"); in mtk_wdma_v3_tx_reset()
367 dev_err(dev->hw->dev, "tx reset failed\n"); in mtk_wdma_tx_reset()
373 for (i = 0; i < ARRAY_SIZE(dev->tx_wdma); i++) in mtk_wdma_tx_reset()
398 struct mtk_wed_wo *wo = dev->hw->wed_wo; in mtk_wed_wo_reset()
414 dev_err(dev->hw->dev, "failed to disable wed-wo\n"); in mtk_wed_wo_reset()
419 switch (dev->hw->index) { in mtk_wed_wo_reset()
452 dev = hw->wed_dev; in mtk_wed_fe_reset()
453 if (!dev || !dev->wlan.reset) in mtk_wed_fe_reset()
457 err = dev->wlan.reset(dev); in mtk_wed_fe_reset()
459 dev_err(dev->dev, "wlan reset failed: %d\n", err); in mtk_wed_fe_reset()
478 dev = hw->wed_dev; in mtk_wed_fe_reset_complete()
479 if (!dev || !dev->wlan.reset_complete) in mtk_wed_fe_reset_complete()
482 dev->wlan.reset_complete(dev); in mtk_wed_fe_reset_complete()
494 if (dev->wlan.bus_type == MTK_WED_BUS_PCIE) { in mtk_wed_assign()
495 hw = hw_list[pci_domain_nr(dev->wlan.pci_dev->bus)]; in mtk_wed_assign()
499 if (!hw->wed_dev) in mtk_wed_assign()
505 /* MT7986 WED devices do not have any pcie slot restrictions */ in mtk_wed_assign()
507 /* MT7986 PCIE or AXI */ in mtk_wed_assign()
510 if (hw && !hw->wed_dev) in mtk_wed_assign()
517 hw->wed_dev = dev; in mtk_wed_assign()
524 struct mtk_wed_hw *hw = dev->hw; in mtk_wed_amsdu_buffer_alloc()
531 wed_amsdu = devm_kcalloc(hw->dev, MTK_WED_AMSDU_NPAGES, in mtk_wed_amsdu_buffer_alloc()
534 return -ENOMEM; in mtk_wed_amsdu_buffer_alloc()
548 wed_amsdu[i].txd_phy = dma_map_single(hw->dev, ptr, in mtk_wed_amsdu_buffer_alloc()
551 if (dma_mapping_error(hw->dev, wed_amsdu[i].txd_phy)) in mtk_wed_amsdu_buffer_alloc()
554 dev->hw->wed_amsdu = wed_amsdu; in mtk_wed_amsdu_buffer_alloc()
559 for (i--; i >= 0; i--) in mtk_wed_amsdu_buffer_alloc()
560 dma_unmap_single(hw->dev, wed_amsdu[i].txd_phy, in mtk_wed_amsdu_buffer_alloc()
562 return -ENOMEM; in mtk_wed_amsdu_buffer_alloc()
568 struct mtk_wed_amsdu *wed_amsdu = dev->hw->wed_amsdu; in mtk_wed_amsdu_free_buffer()
575 dma_unmap_single(dev->hw->dev, wed_amsdu[i].txd_phy, in mtk_wed_amsdu_free_buffer()
585 struct mtk_wed_amsdu *wed_amsdu = dev->hw->wed_amsdu; in mtk_wed_amsdu_init()
599 dev->wlan.amsdu_max_len >> 8) | in mtk_wed_amsdu_init()
601 dev->wlan.amsdu_max_subframes)); in mtk_wed_amsdu_init()
608 dev_err(dev->hw->dev, "amsdu initialization failed\n"); in mtk_wed_amsdu_init()
614 FIELD_PREP(MTK_WED_AMSDU_HIFTXD_SRC, dev->hw->index)); in mtk_wed_amsdu_init()
625 if (dev->wlan.id == 0x7991) in mtk_wed_amsdu_init()
636 u32 desc_size = dev->hw->soc->tx_ring_desc_size; in mtk_wed_tx_buffer_alloc()
638 int token = dev->wlan.token_start; in mtk_wed_tx_buffer_alloc()
643 if (!mtk_wed_is_v3_or_greater(dev->hw)) { in mtk_wed_tx_buffer_alloc()
644 ring_size = dev->wlan.nbuf & ~(MTK_WED_BUF_PER_PAGE - 1); in mtk_wed_tx_buffer_alloc()
645 dev->tx_buf_ring.size = ring_size; in mtk_wed_tx_buffer_alloc()
647 dev->tx_buf_ring.size = MTK_WED_TX_BM_DMA_SIZE; in mtk_wed_tx_buffer_alloc()
650 n_pages = dev->tx_buf_ring.size / MTK_WED_BUF_PER_PAGE; in mtk_wed_tx_buffer_alloc()
654 return -ENOMEM; in mtk_wed_tx_buffer_alloc()
656 dev->tx_buf_ring.pages = page_list; in mtk_wed_tx_buffer_alloc()
658 desc_ptr = dma_alloc_coherent(dev->hw->dev, in mtk_wed_tx_buffer_alloc()
659 dev->tx_buf_ring.size * desc_size, in mtk_wed_tx_buffer_alloc()
662 return -ENOMEM; in mtk_wed_tx_buffer_alloc()
664 dev->tx_buf_ring.desc = desc_ptr; in mtk_wed_tx_buffer_alloc()
665 dev->tx_buf_ring.desc_phys = desc_phys; in mtk_wed_tx_buffer_alloc()
675 return -ENOMEM; in mtk_wed_tx_buffer_alloc()
677 page_phys = dma_map_page(dev->hw->dev, page, 0, PAGE_SIZE, in mtk_wed_tx_buffer_alloc()
679 if (dma_mapping_error(dev->hw->dev, page_phys)) { in mtk_wed_tx_buffer_alloc()
681 return -ENOMEM; in mtk_wed_tx_buffer_alloc()
686 dma_sync_single_for_cpu(dev->hw->dev, page_phys, PAGE_SIZE, in mtk_wed_tx_buffer_alloc()
696 desc->buf0 = cpu_to_le32(buf_phys); in mtk_wed_tx_buffer_alloc()
697 if (!mtk_wed_is_v3_or_greater(dev->hw)) { in mtk_wed_tx_buffer_alloc()
700 txd_size = dev->wlan.init_buf(buf, buf_phys, in mtk_wed_tx_buffer_alloc()
702 desc->buf1 = cpu_to_le32(buf_phys + txd_size); in mtk_wed_tx_buffer_alloc()
704 if (mtk_wed_is_v1(dev->hw)) in mtk_wed_tx_buffer_alloc()
707 MTK_WED_BUF_SIZE - txd_size); in mtk_wed_tx_buffer_alloc()
711 MTK_WED_BUF_SIZE - txd_size); in mtk_wed_tx_buffer_alloc()
712 desc->info = 0; in mtk_wed_tx_buffer_alloc()
716 desc->ctrl = cpu_to_le32(ctrl); in mtk_wed_tx_buffer_alloc()
723 dma_sync_single_for_device(dev->hw->dev, page_phys, PAGE_SIZE, in mtk_wed_tx_buffer_alloc()
733 struct mtk_wed_buf *page_list = dev->tx_buf_ring.pages; in mtk_wed_free_tx_buffer()
734 struct mtk_wed_hw *hw = dev->hw; in mtk_wed_free_tx_buffer()
740 if (!dev->tx_buf_ring.desc) in mtk_wed_free_tx_buffer()
743 for (i = 0; i < dev->tx_buf_ring.size; i += MTK_WED_BUF_PER_PAGE) { in mtk_wed_free_tx_buffer()
750 dma_unmap_page(dev->hw->dev, page_phy, PAGE_SIZE, in mtk_wed_free_tx_buffer()
755 dma_free_coherent(dev->hw->dev, in mtk_wed_free_tx_buffer()
756 dev->tx_buf_ring.size * hw->soc->tx_ring_desc_size, in mtk_wed_free_tx_buffer()
757 dev->tx_buf_ring.desc, in mtk_wed_free_tx_buffer()
758 dev->tx_buf_ring.desc_phys); in mtk_wed_free_tx_buffer()
773 if (!dev->wlan.hw_rro) in mtk_wed_hwrro_buffer_alloc()
778 return -ENOMEM; in mtk_wed_hwrro_buffer_alloc()
780 dev->hw_rro.size = dev->wlan.rx_nbuf & ~(MTK_WED_BUF_PER_PAGE - 1); in mtk_wed_hwrro_buffer_alloc()
781 dev->hw_rro.pages = page_list; in mtk_wed_hwrro_buffer_alloc()
782 desc = dma_alloc_coherent(dev->hw->dev, in mtk_wed_hwrro_buffer_alloc()
783 dev->wlan.rx_nbuf * sizeof(*desc), in mtk_wed_hwrro_buffer_alloc()
786 return -ENOMEM; in mtk_wed_hwrro_buffer_alloc()
788 dev->hw_rro.desc = desc; in mtk_wed_hwrro_buffer_alloc()
789 dev->hw_rro.desc_phys = desc_phys; in mtk_wed_hwrro_buffer_alloc()
798 return -ENOMEM; in mtk_wed_hwrro_buffer_alloc()
800 page_phys = dma_map_page(dev->hw->dev, page, 0, PAGE_SIZE, in mtk_wed_hwrro_buffer_alloc()
802 if (dma_mapping_error(dev->hw->dev, page_phys)) { in mtk_wed_hwrro_buffer_alloc()
804 return -ENOMEM; in mtk_wed_hwrro_buffer_alloc()
809 dma_sync_single_for_cpu(dev->hw->dev, page_phys, PAGE_SIZE, in mtk_wed_hwrro_buffer_alloc()
814 desc->buf0 = cpu_to_le32(buf_phys); in mtk_wed_hwrro_buffer_alloc()
815 desc->token = cpu_to_le32(RX_DMA_PREP_ADDR64(buf_phys)); in mtk_wed_hwrro_buffer_alloc()
820 dma_sync_single_for_device(dev->hw->dev, page_phys, PAGE_SIZE, in mtk_wed_hwrro_buffer_alloc()
833 dev->rx_buf_ring.size = dev->wlan.rx_nbuf; in mtk_wed_rx_buffer_alloc()
834 desc = dma_alloc_coherent(dev->hw->dev, in mtk_wed_rx_buffer_alloc()
835 dev->wlan.rx_nbuf * sizeof(*desc), in mtk_wed_rx_buffer_alloc()
838 return -ENOMEM; in mtk_wed_rx_buffer_alloc()
840 dev->rx_buf_ring.desc = desc; in mtk_wed_rx_buffer_alloc()
841 dev->rx_buf_ring.desc_phys = desc_phys; in mtk_wed_rx_buffer_alloc()
842 dev->wlan.init_rx_buf(dev, dev->wlan.rx_npkt); in mtk_wed_rx_buffer_alloc()
850 struct mtk_wed_buf *page_list = dev->hw_rro.pages; in mtk_wed_hwrro_free_buffer()
851 struct mtk_wed_bm_desc *desc = dev->hw_rro.desc; in mtk_wed_hwrro_free_buffer()
854 if (!dev->wlan.hw_rro) in mtk_wed_hwrro_free_buffer()
870 dma_unmap_page(dev->hw->dev, buf_addr, PAGE_SIZE, in mtk_wed_hwrro_free_buffer()
875 dma_free_coherent(dev->hw->dev, dev->hw_rro.size * sizeof(*desc), in mtk_wed_hwrro_free_buffer()
876 desc, dev->hw_rro.desc_phys); in mtk_wed_hwrro_free_buffer()
885 struct mtk_wed_bm_desc *desc = dev->rx_buf_ring.desc; in mtk_wed_free_rx_buffer()
890 dev->wlan.release_rx_buf(dev); in mtk_wed_free_rx_buffer()
891 dma_free_coherent(dev->hw->dev, dev->rx_buf_ring.size * sizeof(*desc), in mtk_wed_free_rx_buffer()
892 desc, dev->rx_buf_ring.desc_phys); in mtk_wed_free_rx_buffer()
900 if (!mtk_wed_get_rx_capa(dev) || !dev->wlan.hw_rro) in mtk_wed_hwrro_init()
906 wed_w32(dev, MTK_WED_RRO_PG_BM_BASE, dev->hw_rro.desc_phys); in mtk_wed_hwrro_init()
921 FIELD_PREP(MTK_WED_RX_BM_RX_DMAD_SDL0, dev->wlan.rx_size)); in mtk_wed_rx_buffer_hw_init()
922 wed_w32(dev, MTK_WED_RX_BM_BASE, dev->rx_buf_ring.desc_phys); in mtk_wed_rx_buffer_hw_init()
924 FIELD_PREP(MTK_WED_RX_BM_SW_TAIL, dev->wlan.rx_npkt)); in mtk_wed_rx_buffer_hw_init()
935 if (!ring->desc) in mtk_wed_free_ring()
938 dma_free_coherent(dev->hw->dev, ring->size * ring->desc_size, in mtk_wed_free_ring()
939 ring->desc, ring->desc_phys); in mtk_wed_free_ring()
946 mtk_wed_free_ring(dev, &dev->rro.ring); in mtk_wed_free_rx_rings()
954 for (i = 0; i < ARRAY_SIZE(dev->tx_ring); i++) in mtk_wed_free_tx_rings()
955 mtk_wed_free_ring(dev, &dev->tx_ring[i]); in mtk_wed_free_tx_rings()
956 for (i = 0; i < ARRAY_SIZE(dev->rx_wdma); i++) in mtk_wed_free_tx_rings()
957 mtk_wed_free_ring(dev, &dev->rx_wdma[i]); in mtk_wed_free_tx_rings()
965 switch (dev->hw->version) { in mtk_wed_set_ext_int()
983 if (!dev->hw->num_flows) in mtk_wed_set_ext_int()
993 if (!mtk_wed_is_v2(dev->hw)) in mtk_wed_set_512_support()
1014 u32 cur_idx = readl(ring->wpdma + MTK_WED_RING_OFS_CPU_IDX); in mtk_wed_check_wfdma_rx_fill()
1016 if (cur_idx == MTK_WED_RX_RING_SIZE - 1) in mtk_wed_check_wfdma_rx_fill()
1023 dev_err(dev->hw->dev, "rx dma enable failed\n"); in mtk_wed_check_wfdma_rx_fill()
1024 return -ETIMEDOUT; in mtk_wed_check_wfdma_rx_fill()
1048 if (mtk_wed_is_v1(dev->hw)) { in mtk_wed_dma_disable()
1049 regmap_write(dev->hw->mirror, dev->hw->index * 4, 0); in mtk_wed_dma_disable()
1062 if (mtk_wed_is_v3_or_greater(dev->hw) && in mtk_wed_dma_disable()
1104 if (mtk_wed_is_v1(dev->hw)) in mtk_wed_deinit()
1112 if (mtk_wed_is_v3_or_greater(dev->hw)) { in mtk_wed_deinit()
1124 struct mtk_wed_hw *hw = dev->hw; in __mtk_wed_detach()
1135 if (hw->wed_wo) in __mtk_wed_detach()
1138 if (hw->wed_wo) in __mtk_wed_detach()
1142 if (dev->wlan.bus_type == MTK_WED_BUS_PCIE) { in __mtk_wed_detach()
1145 wlan_node = dev->wlan.pci_dev->dev.of_node; in __mtk_wed_detach()
1146 if (of_dma_is_coherent(wlan_node) && hw->hifsys) in __mtk_wed_detach()
1147 regmap_update_bits(hw->hifsys, HIFSYS_DMA_AG_MAP, in __mtk_wed_detach()
1148 BIT(hw->index), BIT(hw->index)); in __mtk_wed_detach()
1151 if ((!hw_list[!hw->index] || !hw_list[!hw->index]->wed_dev) && in __mtk_wed_detach()
1152 hw->eth->dma_dev != hw->eth->dev) in __mtk_wed_detach()
1153 mtk_eth_set_dma_device(hw->eth, hw->eth->dev); in __mtk_wed_detach()
1158 hw->wed_dev = NULL; in __mtk_wed_detach()
1172 switch (dev->wlan.bus_type) { in mtk_wed_bus_init()
1174 struct device_node *np = dev->hw->eth->dev->of_node; in mtk_wed_bus_init()
1176 if (mtk_wed_is_v2(dev->hw)) { in mtk_wed_bus_init()
1180 "mediatek,wed-pcie"); in mtk_wed_bus_init()
1187 if (dev->wlan.msi) { in mtk_wed_bus_init()
1189 dev->hw->pcie_base | 0xc08); in mtk_wed_bus_init()
1191 dev->hw->pcie_base | 0xc04); in mtk_wed_bus_init()
1195 dev->hw->pcie_base | 0x180); in mtk_wed_bus_init()
1197 dev->hw->pcie_base | 0x184); in mtk_wed_bus_init()
1204 /* pcie interrupt control: pola/source selection */ in mtk_wed_bus_init()
1209 dev->hw->index)); in mtk_wed_bus_init()
1227 if (mtk_wed_is_v1(dev->hw)) { in mtk_wed_set_wpdma()
1228 wed_w32(dev, MTK_WED_WPDMA_CFG_BASE, dev->wlan.wpdma_phys); in mtk_wed_set_wpdma()
1234 wed_w32(dev, MTK_WED_WPDMA_CFG_BASE, dev->wlan.wpdma_int); in mtk_wed_set_wpdma()
1235 wed_w32(dev, MTK_WED_WPDMA_CFG_INT_MASK, dev->wlan.wpdma_mask); in mtk_wed_set_wpdma()
1236 wed_w32(dev, MTK_WED_WPDMA_CFG_TX, dev->wlan.wpdma_tx); in mtk_wed_set_wpdma()
1237 wed_w32(dev, MTK_WED_WPDMA_CFG_TX_FREE, dev->wlan.wpdma_txfree); in mtk_wed_set_wpdma()
1242 wed_w32(dev, MTK_WED_WPDMA_RX_GLO_CFG, dev->wlan.wpdma_rx_glo); in mtk_wed_set_wpdma()
1243 wed_w32(dev, dev->hw->soc->regmap.wpdma_rx_ring0, dev->wlan.wpdma_rx); in mtk_wed_set_wpdma()
1245 if (!dev->wlan.hw_rro) in mtk_wed_set_wpdma()
1248 wed_w32(dev, MTK_WED_RRO_RX_D_CFG(0), dev->wlan.wpdma_rx_rro[0]); in mtk_wed_set_wpdma()
1249 wed_w32(dev, MTK_WED_RRO_RX_D_CFG(1), dev->wlan.wpdma_rx_rro[1]); in mtk_wed_set_wpdma()
1252 dev->wlan.wpdma_rx_pg + i * 0x10); in mtk_wed_set_wpdma()
1265 if (!mtk_wed_is_v3_or_greater(dev->hw)) { in mtk_wed_hw_init_early()
1273 if (mtk_wed_is_v1(dev->hw)) { in mtk_wed_hw_init_early()
1274 u32 offset = dev->hw->index ? 0x04000400 : 0; in mtk_wed_hw_init_early()
1284 MTK_PCIE_BASE(dev->hw->index)); in mtk_wed_hw_init_early()
1286 wed_w32(dev, MTK_WED_WDMA_CFG_BASE, dev->hw->wdma_phy); in mtk_wed_hw_init_early()
1306 ring->desc = dma_alloc_coherent(dev->hw->dev, in mtk_wed_rro_ring_alloc()
1307 size * sizeof(*ring->desc), in mtk_wed_rro_ring_alloc()
1308 &ring->desc_phys, GFP_KERNEL); in mtk_wed_rro_ring_alloc()
1309 if (!ring->desc) in mtk_wed_rro_ring_alloc()
1310 return -ENOMEM; in mtk_wed_rro_ring_alloc()
1312 ring->desc_size = sizeof(*ring->desc); in mtk_wed_rro_ring_alloc()
1313 ring->size = size; in mtk_wed_rro_ring_alloc()
1326 index = of_property_match_string(dev->hw->node, "memory-region-names", in mtk_wed_rro_alloc()
1327 "wo-dlm"); in mtk_wed_rro_alloc()
1331 np = of_parse_phandle(dev->hw->node, "memory-region", index); in mtk_wed_rro_alloc()
1333 return -ENODEV; in mtk_wed_rro_alloc()
1339 return -ENODEV; in mtk_wed_rro_alloc()
1341 dev->rro.miod_phys = rmem->base; in mtk_wed_rro_alloc()
1342 dev->rro.fdbk_phys = MTK_WED_MIOD_COUNT + dev->rro.miod_phys; in mtk_wed_rro_alloc()
1344 return mtk_wed_rro_ring_alloc(dev, &dev->rro.ring, in mtk_wed_rro_alloc()
1351 struct mtk_wed_wo *wo = dev->hw->wed_wo; in mtk_wed_rro_cfg()
1388 wed_w32(dev, MTK_WED_RROQM_MIOD_CTRL0, dev->rro.miod_phys); in mtk_wed_rro_hw_init()
1391 wed_w32(dev, MTK_WED_RROQM_FDBK_CTRL0, dev->rro.fdbk_phys); in mtk_wed_rro_hw_init()
1395 wed_w32(dev, MTK_WED_RROQ_BASE_L, dev->rro.ring.desc_phys); in mtk_wed_rro_hw_init()
1402 wed_w32(dev, MTK_WED_RROQM_MIOD_CTRL2, MTK_WED_MIOD_CNT - 1); in mtk_wed_rro_hw_init()
1418 if (mtk_wed_is_v2(dev->hw)) { in mtk_wed_route_qm_hw_init()
1423 0x3 + dev->hw->index)); in mtk_wed_route_qm_hw_init()
1428 0x3 + dev->hw->index)); in mtk_wed_route_qm_hw_init()
1437 if (dev->init_done) in mtk_wed_hw_init()
1440 dev->init_done = true; in mtk_wed_hw_init()
1443 wed_w32(dev, MTK_WED_TX_BM_BASE, dev->tx_buf_ring.desc_phys); in mtk_wed_hw_init()
1446 if (mtk_wed_is_v1(dev->hw)) { in mtk_wed_hw_init()
1450 dev->tx_buf_ring.size / 128) | in mtk_wed_hw_init()
1456 } else if (mtk_wed_is_v2(dev->hw)) { in mtk_wed_hw_init()
1460 dev->tx_buf_ring.size / 128) | in mtk_wed_hw_init()
1472 dev->tx_buf_ring.size / 128) | in mtk_wed_hw_init()
1474 dev->tx_buf_ring.size / 128)); in mtk_wed_hw_init()
1477 wed_w32(dev, dev->hw->soc->regmap.tx_bm_tkid, in mtk_wed_hw_init()
1478 FIELD_PREP(MTK_WED_TX_BM_TKID_START, dev->wlan.token_start) | in mtk_wed_hw_init()
1480 dev->wlan.token_start + dev->wlan.nbuf - 1)); in mtk_wed_hw_init()
1484 if (mtk_wed_is_v3_or_greater(dev->hw)) { in mtk_wed_hw_init()
1492 dev->wlan.nbuf / 128) | in mtk_wed_hw_init()
1494 dev->wlan.nbuf / 128)); in mtk_wed_hw_init()
1504 if (mtk_wed_is_v1(dev->hw)) { in mtk_wed_hw_init()
1538 if (!mtk_wed_is_v1(dev->hw)) in mtk_wed_hw_init()
1545 void *head = (void *)ring->desc; in mtk_wed_ring_reset()
1551 desc = (struct mtk_wdma_desc *)(head + i * ring->desc_size); in mtk_wed_ring_reset()
1552 desc->buf0 = 0; in mtk_wed_ring_reset()
1554 desc->ctrl = cpu_to_le32(MTK_WDMA_DESC_CTRL_DMA_DONE); in mtk_wed_ring_reset()
1556 desc->ctrl = cpu_to_le32(MTK_WFDMA_DESC_CTRL_TO_HOST); in mtk_wed_ring_reset()
1557 desc->buf1 = 0; in mtk_wed_ring_reset()
1558 desc->info = 0; in mtk_wed_ring_reset()
1565 struct mtk_wed_wo *wo = dev->hw->wed_wo; in mtk_wed_rx_reset()
1575 if (dev->wlan.hw_rro) { in mtk_wed_rx_reset()
1585 if (!ret && mtk_wed_is_v3_or_greater(dev->hw)) in mtk_wed_rx_reset()
1592 if (mtk_wed_is_v3_or_greater(dev->hw)) { in mtk_wed_rx_reset()
1629 if (dev->wlan.hw_rro) { in mtk_wed_rx_reset()
1656 } else if (mtk_wed_is_v3_or_greater(dev->hw)) { in mtk_wed_rx_reset()
1669 if (mtk_wed_is_v3_or_greater(dev->hw)) in mtk_wed_rx_reset()
1685 dev->hw->soc->regmap.reset_idx_rx_mask); in mtk_wed_rx_reset()
1695 if (dev->wlan.hw_rro) { in mtk_wed_rx_reset()
1712 for (i = 0; i < ARRAY_SIZE(dev->rx_ring); i++) { in mtk_wed_rx_reset()
1713 if (!dev->rx_ring[i].desc) in mtk_wed_rx_reset()
1716 mtk_wed_ring_reset(&dev->rx_ring[i], MTK_WED_RX_RING_SIZE, in mtk_wed_rx_reset()
1732 for (i = 0; i < ARRAY_SIZE(dev->tx_ring); i++) { in mtk_wed_reset_dma()
1733 if (!dev->tx_ring[i].desc) in mtk_wed_reset_dma()
1736 mtk_wed_ring_reset(&dev->tx_ring[i], MTK_WED_TX_RING_SIZE, in mtk_wed_reset_dma()
1748 dev->hw->soc->regmap.reset_idx_tx_mask); in mtk_wed_reset_dma()
1754 if (mtk_wed_is_v3_or_greater(dev->hw)) { in mtk_wed_reset_dma()
1767 if (!busy && mtk_wed_is_v3_or_greater(dev->hw)) in mtk_wed_reset_dma()
1775 if (mtk_wed_is_v3_or_greater(dev->hw)) { in mtk_wed_reset_dma()
1804 if (mtk_wed_is_v1(dev->hw)) in mtk_wed_reset_dma()
1832 if (mtk_wed_is_v3_or_greater(dev->hw)) in mtk_wed_reset_dma()
1841 dev->init_done = false; in mtk_wed_reset_dma()
1842 if (mtk_wed_is_v1(dev->hw)) in mtk_wed_reset_dma()
1850 if (mtk_wed_is_v3_or_greater(dev->hw)) { in mtk_wed_reset_dma()
1864 ring->desc = dma_alloc_coherent(dev->hw->dev, size * desc_size, in mtk_wed_ring_alloc()
1865 &ring->desc_phys, GFP_KERNEL); in mtk_wed_ring_alloc()
1866 if (!ring->desc) in mtk_wed_ring_alloc()
1867 return -ENOMEM; in mtk_wed_ring_alloc()
1869 ring->desc_size = desc_size; in mtk_wed_ring_alloc()
1870 ring->size = size; in mtk_wed_ring_alloc()
1882 if (idx >= ARRAY_SIZE(dev->rx_wdma)) in mtk_wed_wdma_rx_ring_setup()
1883 return -EINVAL; in mtk_wed_wdma_rx_ring_setup()
1885 wdma = &dev->rx_wdma[idx]; in mtk_wed_wdma_rx_ring_setup()
1887 dev->hw->soc->wdma_desc_size, true)) in mtk_wed_wdma_rx_ring_setup()
1888 return -ENOMEM; in mtk_wed_wdma_rx_ring_setup()
1891 wdma->desc_phys); in mtk_wed_wdma_rx_ring_setup()
1897 wdma->desc_phys); in mtk_wed_wdma_rx_ring_setup()
1910 if (idx >= ARRAY_SIZE(dev->tx_wdma)) in mtk_wed_wdma_tx_ring_setup()
1911 return -EINVAL; in mtk_wed_wdma_tx_ring_setup()
1913 wdma = &dev->tx_wdma[idx]; in mtk_wed_wdma_tx_ring_setup()
1915 dev->hw->soc->wdma_desc_size, true)) in mtk_wed_wdma_tx_ring_setup()
1916 return -ENOMEM; in mtk_wed_wdma_tx_ring_setup()
1918 if (mtk_wed_is_v3_or_greater(dev->hw)) { in mtk_wed_wdma_tx_ring_setup()
1919 struct mtk_wdma_desc *desc = wdma->desc; in mtk_wed_wdma_tx_ring_setup()
1923 desc->buf0 = 0; in mtk_wed_wdma_tx_ring_setup()
1924 desc->ctrl = cpu_to_le32(MTK_WDMA_DESC_CTRL_DMA_DONE); in mtk_wed_wdma_tx_ring_setup()
1925 desc->buf1 = 0; in mtk_wed_wdma_tx_ring_setup()
1926 desc->info = cpu_to_le32(MTK_WDMA_TXD0_DESC_INFO_DMA_DONE); in mtk_wed_wdma_tx_ring_setup()
1928 desc->buf0 = 0; in mtk_wed_wdma_tx_ring_setup()
1929 desc->ctrl = cpu_to_le32(MTK_WDMA_DESC_CTRL_DMA_DONE); in mtk_wed_wdma_tx_ring_setup()
1930 desc->buf1 = 0; in mtk_wed_wdma_tx_ring_setup()
1931 desc->info = cpu_to_le32(MTK_WDMA_TXD1_DESC_INFO_DMA_DONE); in mtk_wed_wdma_tx_ring_setup()
1937 wdma->desc_phys); in mtk_wed_wdma_tx_ring_setup()
1948 wdma->desc_phys); in mtk_wed_wdma_tx_ring_setup()
1964 struct mtk_eth *eth = dev->hw->eth; in mtk_wed_ppe_check()
1975 skb->protocol = eh->h_proto; in mtk_wed_ppe_check()
1976 mtk_ppe_check_skb(eth->ppe[dev->hw->index], skb, hash); in mtk_wed_ppe_check()
1991 if (mtk_wed_is_v1(dev->hw)) { in mtk_wed_configure_irq()
2001 if (mtk_wed_is_v3_or_greater(dev->hw)) in mtk_wed_configure_irq()
2011 dev->wlan.tx_tbit[0]) | in mtk_wed_configure_irq()
2013 dev->wlan.tx_tbit[1])); in mtk_wed_configure_irq()
2020 dev->wlan.txfree_tbit)); in mtk_wed_configure_irq()
2029 dev->wlan.rx_tbit[0]) | in mtk_wed_configure_irq()
2031 dev->wlan.rx_tbit[1])); in mtk_wed_configure_irq()
2040 dev->wdma_idx)); in mtk_wed_configure_irq()
2057 if (!mtk_wed_is_v3_or_greater(dev->hw)) { in mtk_wed_dma_enable()
2083 if (mtk_wed_is_v1(dev->hw)) { in mtk_wed_dma_enable()
2093 if (mtk_wed_is_v3_or_greater(dev->hw)) { in mtk_wed_dma_enable()
2129 if (mtk_wed_is_v3_or_greater(dev->hw)) { in mtk_wed_dma_enable()
2141 struct mtk_wed_ring *ring = &dev->rx_ring[i]; in mtk_wed_dma_enable()
2144 if (!(ring->flags & MTK_WED_RING_CONFIGURED)) in mtk_wed_dma_enable()
2148 dev_err(dev->hw->dev, in mtk_wed_dma_enable()
2154 dev->wlan.wpdma_rx_glo - in mtk_wed_dma_enable()
2155 dev->wlan.phy_base) | MTK_WFMDA_RX_DMA_EN; in mtk_wed_dma_enable()
2157 dev->wlan.wpdma_rx_glo - dev->wlan.phy_base, in mtk_wed_dma_enable()
2170 if (!mtk_wed_get_rx_capa(dev) || !dev->wlan.hw_rro) in mtk_wed_start_hw_rro()
2189 dev->wlan.rro_rx_tbit[0]) | in mtk_wed_start_hw_rro()
2191 dev->wlan.rro_rx_tbit[1])); in mtk_wed_start_hw_rro()
2201 dev->wlan.rx_pg_tbit[0]) | in mtk_wed_start_hw_rro()
2203 dev->wlan.rx_pg_tbit[1]) | in mtk_wed_start_hw_rro()
2205 dev->wlan.rx_pg_tbit[2])); in mtk_wed_start_hw_rro()
2214 struct mtk_wed_ring *ring = &dev->rx_rro_ring[i]; in mtk_wed_start_hw_rro()
2216 if (!(ring->flags & MTK_WED_RING_CONFIGURED)) in mtk_wed_start_hw_rro()
2220 dev_err(dev->hw->dev, in mtk_wed_start_hw_rro()
2225 struct mtk_wed_ring *ring = &dev->rx_page_ring[i]; in mtk_wed_start_hw_rro()
2227 if (!(ring->flags & MTK_WED_RING_CONFIGURED)) in mtk_wed_start_hw_rro()
2231 dev_err(dev->hw->dev, in mtk_wed_start_hw_rro()
2240 struct mtk_wed_ring *ring = &dev->rx_rro_ring[idx]; in mtk_wed_rro_rx_ring_setup()
2242 ring->wpdma = regs; in mtk_wed_rro_rx_ring_setup()
2247 ring->flags |= MTK_WED_RING_CONFIGURED; in mtk_wed_rro_rx_ring_setup()
2253 struct mtk_wed_ring *ring = &dev->rx_page_ring[idx]; in mtk_wed_msdu_pg_rx_ring_setup()
2255 ring->wpdma = regs; in mtk_wed_msdu_pg_rx_ring_setup()
2260 ring->flags |= MTK_WED_RING_CONFIGURED; in mtk_wed_msdu_pg_rx_ring_setup()
2266 struct mtk_wed_ring *ring = &dev->ind_cmd_ring; in mtk_wed_ind_rx_ring_setup()
2270 ring->wpdma = regs; in mtk_wed_ind_rx_ring_setup()
2278 wed_w32(dev, MTK_WED_RRO_CFG0, dev->wlan.phy_base + in mtk_wed_ind_rx_ring_setup()
2279 dev->wlan.ind_cmd.ack_sn_addr); in mtk_wed_ind_rx_ring_setup()
2282 dev->wlan.ind_cmd.win_size) | in mtk_wed_ind_rx_ring_setup()
2284 dev->wlan.ind_cmd.particular_sid)); in mtk_wed_ind_rx_ring_setup()
2288 dev->wlan.ind_cmd.particular_se_phys); in mtk_wed_ind_rx_ring_setup()
2290 for (i = 0; i < dev->wlan.ind_cmd.se_group_nums; i++) { in mtk_wed_ind_rx_ring_setup()
2292 dev->wlan.ind_cmd.addr_elem_phys[i] >> 4); in mtk_wed_ind_rx_ring_setup()
2300 dev_err(dev->hw->dev, in mtk_wed_ind_rx_ring_setup()
2305 for (i = 0; i < dev->wlan.ind_cmd.particular_sid; i++) { in mtk_wed_ind_rx_ring_setup()
2317 dev_err(dev->hw->dev, in mtk_wed_ind_rx_ring_setup()
2335 for (i = 0; i < ARRAY_SIZE(dev->rx_wdma); i++) in mtk_wed_start()
2336 if (!dev->rx_wdma[i].desc) in mtk_wed_start()
2344 if (mtk_wed_is_v1(dev->hw)) { in mtk_wed_start()
2345 u32 val = dev->wlan.wpdma_phys | MTK_PCIE_MIRROR_MAP_EN | in mtk_wed_start()
2347 dev->hw->index); in mtk_wed_start()
2349 val |= BIT(0) | (BIT(1) * !!dev->hw->index); in mtk_wed_start()
2350 regmap_write(dev->hw->mirror, dev->hw->index * 4, val); in mtk_wed_start()
2361 if (mtk_wed_is_v3_or_greater(dev->hw)) { in mtk_wed_start()
2371 mtk_wed_set_512_support(dev, dev->wlan.wcid_512); in mtk_wed_start()
2375 dev->running = true; in mtk_wed_start()
2389 if ((dev->wlan.bus_type == MTK_WED_BUS_PCIE && in mtk_wed_attach()
2390 pci_domain_nr(dev->wlan.pci_dev->bus) > 1) || in mtk_wed_attach()
2392 ret = -ENODEV; in mtk_wed_attach()
2404 ret = -ENODEV; in mtk_wed_attach()
2408 device = dev->wlan.bus_type == MTK_WED_BUS_PCIE in mtk_wed_attach()
2409 ? &dev->wlan.pci_dev->dev in mtk_wed_attach()
2410 : &dev->wlan.platform_dev->dev; in mtk_wed_attach()
2412 hw->index, hw->version); in mtk_wed_attach()
2414 dev->hw = hw; in mtk_wed_attach()
2415 dev->dev = hw->dev; in mtk_wed_attach()
2416 dev->irq = hw->irq; in mtk_wed_attach()
2417 dev->wdma_idx = hw->index; in mtk_wed_attach()
2418 dev->version = hw->version; in mtk_wed_attach()
2419 dev->hw->pcie_base = mtk_wed_get_pcie_base(dev); in mtk_wed_attach()
2421 if (hw->eth->dma_dev == hw->eth->dev && in mtk_wed_attach()
2422 of_dma_is_coherent(hw->eth->dev->of_node)) in mtk_wed_attach()
2423 mtk_eth_set_dma_device(hw->eth, hw->dev); in mtk_wed_attach()
2441 regmap_update_bits(hw->hifsys, HIFSYS_DMA_AG_MAP, in mtk_wed_attach()
2442 BIT(hw->index), 0); in mtk_wed_attach()
2444 dev->rev_id = wed_r32(dev, MTK_WED_REV_ID); in mtk_wed_attach()
2450 dev_err(dev->hw->dev, "failed to attach wed device\n"); in mtk_wed_attach()
2463 struct mtk_wed_ring *ring = &dev->tx_ring[idx]; in mtk_wed_tx_ring_setup()
2477 if (WARN_ON(idx >= ARRAY_SIZE(dev->tx_ring))) in mtk_wed_tx_ring_setup()
2478 return -EINVAL; in mtk_wed_tx_ring_setup()
2481 sizeof(*ring->desc), true)) in mtk_wed_tx_ring_setup()
2482 return -ENOMEM; in mtk_wed_tx_ring_setup()
2486 return -ENOMEM; in mtk_wed_tx_ring_setup()
2488 ring->reg_base = MTK_WED_RING_TX(idx); in mtk_wed_tx_ring_setup()
2489 ring->wpdma = regs; in mtk_wed_tx_ring_setup()
2491 if (mtk_wed_is_v3_or_greater(dev->hw) && idx == 1) { in mtk_wed_tx_ring_setup()
2508 /* WED -> WPDMA */ in mtk_wed_tx_ring_setup()
2509 wpdma_tx_w32(dev, idx, MTK_WED_RING_OFS_BASE, ring->desc_phys); in mtk_wed_tx_ring_setup()
2514 ring->desc_phys); in mtk_wed_tx_ring_setup()
2525 struct mtk_wed_ring *ring = &dev->txfree_ring; in mtk_wed_txfree_ring_setup()
2526 int i, index = mtk_wed_is_v1(dev->hw); in mtk_wed_txfree_ring_setup()
2533 ring->reg_base = MTK_WED_RING_RX(index); in mtk_wed_txfree_ring_setup()
2534 ring->wpdma = regs; in mtk_wed_txfree_ring_setup()
2550 struct mtk_wed_ring *ring = &dev->rx_ring[idx]; in mtk_wed_rx_ring_setup()
2552 if (WARN_ON(idx >= ARRAY_SIZE(dev->rx_ring))) in mtk_wed_rx_ring_setup()
2553 return -EINVAL; in mtk_wed_rx_ring_setup()
2556 sizeof(*ring->desc), false)) in mtk_wed_rx_ring_setup()
2557 return -ENOMEM; in mtk_wed_rx_ring_setup()
2561 return -ENOMEM; in mtk_wed_rx_ring_setup()
2563 ring->reg_base = MTK_WED_RING_RX_DATA(idx); in mtk_wed_rx_ring_setup()
2564 ring->wpdma = regs; in mtk_wed_rx_ring_setup()
2565 ring->flags |= MTK_WED_RING_CONFIGURED; in mtk_wed_rx_ring_setup()
2567 /* WPDMA -> WED */ in mtk_wed_rx_ring_setup()
2568 wpdma_rx_w32(dev, idx, MTK_WED_RING_OFS_BASE, ring->desc_phys); in mtk_wed_rx_ring_setup()
2572 ring->desc_phys); in mtk_wed_rx_ring_setup()
2584 if (mtk_wed_is_v3_or_greater(dev->hw)) in mtk_wed_irq_get()
2593 if (!dev->hw->num_flows) in mtk_wed_irq_get()
2596 pr_err("mtk_wed%d: error status=%08x\n", dev->hw->index, val); in mtk_wed_irq_get()
2608 if (!dev->running) in mtk_wed_irq_set_mask()
2622 if (!hw || !hw->wed_dev) { in mtk_wed_flow_add()
2623 ret = -ENODEV; in mtk_wed_flow_add()
2627 if (!hw->wed_dev->wlan.offload_enable) in mtk_wed_flow_add()
2630 if (hw->num_flows) { in mtk_wed_flow_add()
2631 hw->num_flows++; in mtk_wed_flow_add()
2635 ret = hw->wed_dev->wlan.offload_enable(hw->wed_dev); in mtk_wed_flow_add()
2637 hw->num_flows++; in mtk_wed_flow_add()
2638 mtk_wed_set_ext_int(hw->wed_dev, true); in mtk_wed_flow_add()
2652 if (!hw || !hw->wed_dev) in mtk_wed_flow_remove()
2655 if (!hw->wed_dev->wlan.offload_disable) in mtk_wed_flow_remove()
2658 if (--hw->num_flows) in mtk_wed_flow_remove()
2661 hw->wed_dev->wlan.offload_disable(hw->wed_dev); in mtk_wed_flow_remove()
2662 mtk_wed_set_ext_int(hw->wed_dev, true); in mtk_wed_flow_remove()
2673 struct mtk_wed_hw *hw = priv->hw; in mtk_wed_setup_tc_block_cb()
2675 if (!tc_can_offload(priv->dev)) in mtk_wed_setup_tc_block_cb()
2676 return -EOPNOTSUPP; in mtk_wed_setup_tc_block_cb()
2679 return -EOPNOTSUPP; in mtk_wed_setup_tc_block_cb()
2681 return mtk_flow_offload_cmd(hw->eth, cls, hw->index); in mtk_wed_setup_tc_block_cb()
2691 struct mtk_eth *eth = hw->eth; in mtk_wed_setup_tc_block()
2694 if (!eth->soc->offload_version) in mtk_wed_setup_tc_block()
2695 return -EOPNOTSUPP; in mtk_wed_setup_tc_block()
2697 if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS) in mtk_wed_setup_tc_block()
2698 return -EOPNOTSUPP; in mtk_wed_setup_tc_block()
2701 f->driver_block_list = &block_cb_list; in mtk_wed_setup_tc_block()
2703 switch (f->command) { in mtk_wed_setup_tc_block()
2705 block_cb = flow_block_cb_lookup(f->block, cb, dev); in mtk_wed_setup_tc_block()
2713 return -ENOMEM; in mtk_wed_setup_tc_block()
2715 priv->hw = hw; in mtk_wed_setup_tc_block()
2716 priv->dev = dev; in mtk_wed_setup_tc_block()
2725 list_add_tail(&block_cb->driver_list, &block_cb_list); in mtk_wed_setup_tc_block()
2728 block_cb = flow_block_cb_lookup(f->block, cb, dev); in mtk_wed_setup_tc_block()
2730 return -ENOENT; in mtk_wed_setup_tc_block()
2734 list_del(&block_cb->driver_list); in mtk_wed_setup_tc_block()
2735 kfree(block_cb->cb_priv); in mtk_wed_setup_tc_block()
2739 return -EOPNOTSUPP; in mtk_wed_setup_tc_block()
2747 struct mtk_wed_hw *hw = wed->hw; in mtk_wed_setup_tc()
2750 return -EOPNOTSUPP; in mtk_wed_setup_tc()
2757 return -EOPNOTSUPP; in mtk_wed_setup_tc()
2786 struct device_node *eth_np = eth->dev->of_node; in mtk_wed_add_hw()
2799 get_device(&pdev->dev); in mtk_wed_add_hw()
2819 hw->node = np; in mtk_wed_add_hw()
2820 hw->regs = regs; in mtk_wed_add_hw()
2821 hw->eth = eth; in mtk_wed_add_hw()
2822 hw->dev = &pdev->dev; in mtk_wed_add_hw()
2823 hw->wdma_phy = wdma_phy; in mtk_wed_add_hw()
2824 hw->wdma = wdma; in mtk_wed_add_hw()
2825 hw->index = index; in mtk_wed_add_hw()
2826 hw->irq = irq; in mtk_wed_add_hw()
2827 hw->version = eth->soc->version; in mtk_wed_add_hw()
2829 switch (hw->version) { in mtk_wed_add_hw()
2831 hw->soc = &mt7986_data; in mtk_wed_add_hw()
2834 hw->soc = &mt7988_data; in mtk_wed_add_hw()
2838 hw->mirror = syscon_regmap_lookup_by_phandle(eth_np, in mtk_wed_add_hw()
2839 "mediatek,pcie-mirror"); in mtk_wed_add_hw()
2840 hw->hifsys = syscon_regmap_lookup_by_phandle(eth_np, in mtk_wed_add_hw()
2842 if (IS_ERR(hw->mirror) || IS_ERR(hw->hifsys)) { in mtk_wed_add_hw()
2848 regmap_write(hw->mirror, 0, 0); in mtk_wed_add_hw()
2849 regmap_write(hw->mirror, 4, 0); in mtk_wed_add_hw()
2851 hw->soc = &mt7622_data; in mtk_wed_add_hw()
2866 put_device(&pdev->dev); in mtk_wed_add_hw()
2887 debugfs_remove(hw->debugfs_dir); in mtk_wed_exit()
2888 put_device(hw->dev); in mtk_wed_exit()
2889 of_node_put(hw->node); in mtk_wed_exit()