1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2025 AIROHA Inc
4 * Author: Lorenzo Bianconi <lorenzo@kernel.org>
5 */
6 #include <linux/kernel.h>
7 #include <net/flow_offload.h>
8 #include <net/pkt_cls.h>
9
10 #include "mt76.h"
11 #include "dma.h"
12 #include "mt76_connac.h"
13
14 #define MT76_NPU_RX_BUF_SIZE (1800 + \
15 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
16
mt76_npu_fill_rx_queue(struct mt76_dev * dev,struct mt76_queue * q)17 int mt76_npu_fill_rx_queue(struct mt76_dev *dev, struct mt76_queue *q)
18 {
19 int nframes = 0;
20
21 while (q->queued < q->ndesc - 1) {
22 struct airoha_npu_rx_dma_desc *desc = (void *)q->desc;
23 struct mt76_queue_entry *e = &q->entry[q->head];
24 struct page *page;
25 int offset;
26
27 e->buf = mt76_get_page_pool_buf(q, &offset, q->buf_size);
28 if (!e->buf)
29 break;
30
31 e->dma_len[0] = SKB_WITH_OVERHEAD(q->buf_size);
32 page = virt_to_head_page(e->buf);
33 e->dma_addr[0] = page_pool_get_dma_addr(page) + offset;
34
35 memset(&desc[q->head], 0, sizeof(*desc));
36 desc[q->head].addr = e->dma_addr[0];
37
38 q->head = (q->head + 1) % q->ndesc;
39 q->queued++;
40 nframes++;
41 }
42
43 return nframes;
44 }
45
mt76_npu_queue_cleanup(struct mt76_dev * dev,struct mt76_queue * q)46 void mt76_npu_queue_cleanup(struct mt76_dev *dev, struct mt76_queue *q)
47 {
48 spin_lock_bh(&q->lock);
49 while (q->queued > 0) {
50 struct mt76_queue_entry *e = &q->entry[q->tail];
51
52 dma_sync_single_for_cpu(dev->dma_dev, e->dma_addr[0],
53 e->dma_len[0],
54 page_pool_get_dma_dir(q->page_pool));
55 mt76_put_page_pool_buf(e->buf, false);
56 q->tail = (q->tail + 1) % q->ndesc;
57 q->queued--;
58 }
59 spin_unlock_bh(&q->lock);
60 }
61
mt76_npu_dequeue(struct mt76_dev * dev,struct mt76_queue * q,u32 * info)62 static struct sk_buff *mt76_npu_dequeue(struct mt76_dev *dev,
63 struct mt76_queue *q,
64 u32 *info)
65 {
66 struct airoha_npu_rx_dma_desc *desc = (void *)q->desc;
67 int i, nframes, index = q->tail;
68 struct sk_buff *skb = NULL;
69
70 nframes = FIELD_GET(NPU_RX_DMA_PKT_COUNT_MASK, desc[index].info);
71 nframes = max_t(int, nframes, 1);
72
73 for (i = 0; i < nframes; i++) {
74 struct mt76_queue_entry *e = &q->entry[index];
75 int len = FIELD_GET(NPU_RX_DMA_DESC_CUR_LEN_MASK,
76 desc[index].ctrl);
77
78 if (!FIELD_GET(NPU_RX_DMA_DESC_DONE_MASK, desc[index].ctrl)) {
79 dev_kfree_skb(skb);
80 return NULL;
81 }
82
83 dma_sync_single_for_cpu(dev->dma_dev, e->dma_addr[0],
84 e->dma_len[0],
85 page_pool_get_dma_dir(q->page_pool));
86
87 if (!skb) {
88 skb = napi_build_skb(e->buf, q->buf_size);
89 if (!skb)
90 return NULL;
91
92 __skb_put(skb, len);
93 skb_reset_mac_header(skb);
94 skb_mark_for_recycle(skb);
95 } else {
96 struct skb_shared_info *shinfo = skb_shinfo(skb);
97 struct page *page = virt_to_head_page(e->buf);
98 int nr_frags = shinfo->nr_frags;
99
100 if (nr_frags < ARRAY_SIZE(shinfo->frags))
101 skb_add_rx_frag(skb, nr_frags, page,
102 e->buf - page_address(page),
103 len, q->buf_size);
104 }
105
106 *info = desc[index].info;
107 index = (index + 1) % q->ndesc;
108 }
109 q->tail = index;
110 q->queued -= i;
111 Q_WRITE(q, dma_idx, q->tail);
112
113 return skb;
114 }
115
mt76_npu_check_ppe(struct mt76_dev * dev,struct sk_buff * skb,u32 info)116 void mt76_npu_check_ppe(struct mt76_dev *dev, struct sk_buff *skb,
117 u32 info)
118 {
119 struct airoha_ppe_dev *ppe_dev;
120 u16 reason, hash;
121
122 if (!mt76_npu_device_active(dev))
123 return;
124
125 rcu_read_lock();
126
127 ppe_dev = rcu_dereference(dev->mmio.ppe_dev);
128 if (!ppe_dev)
129 goto out;
130
131 hash = FIELD_GET(NPU_RX_DMA_FOE_ID_MASK, info);
132 skb_set_hash(skb, hash, PKT_HASH_TYPE_L4);
133
134 reason = FIELD_GET(NPU_RX_DMA_CRSN_MASK, info);
135 if (reason == PPE_CPU_REASON_HIT_UNBIND_RATE_REACHED) {
136 skb_set_mac_header(skb, 0);
137 airoha_ppe_dev_check_skb(ppe_dev, skb, hash, true);
138 }
139 out:
140 rcu_read_unlock();
141 }
142 EXPORT_SYMBOL_GPL(mt76_npu_check_ppe);
143
mt76_npu_rx_poll(struct napi_struct * napi,int budget)144 static int mt76_npu_rx_poll(struct napi_struct *napi, int budget)
145 {
146 struct mt76_dev *dev = mt76_priv(napi->dev);
147 enum mt76_rxq_id qid = napi - dev->napi;
148 struct airoha_npu *npu;
149 int done = 0;
150
151 rcu_read_lock();
152
153 npu = rcu_dereference(dev->mmio.npu);
154 if (!npu)
155 goto out;
156
157 while (done < budget) {
158 struct sk_buff *skb;
159 u32 info = 0;
160
161 skb = mt76_npu_dequeue(dev, &dev->q_rx[qid], &info);
162 if (!skb)
163 break;
164
165 dev->drv->rx_skb(dev, qid, skb, &info);
166 mt76_rx_poll_complete(dev, qid, napi);
167 done++;
168 }
169
170 mt76_npu_fill_rx_queue(dev, &dev->q_rx[qid]);
171 out:
172 if (done < budget && napi_complete(napi))
173 dev->drv->rx_poll_complete(dev, qid);
174
175 rcu_read_unlock();
176
177 return done;
178 }
179
mt76_npu_irq_handler(int irq,void * q_instance)180 static irqreturn_t mt76_npu_irq_handler(int irq, void *q_instance)
181 {
182 struct mt76_queue *q = q_instance;
183 struct mt76_dev *dev = q->dev;
184 int qid = q - &dev->q_rx[0];
185 int index = qid - MT_RXQ_NPU0;
186 struct airoha_npu *npu;
187 u32 status;
188
189 rcu_read_lock();
190
191 npu = rcu_dereference(dev->mmio.npu);
192 if (!npu)
193 goto out;
194
195 status = airoha_npu_wlan_get_irq_status(npu, index);
196 airoha_npu_wlan_set_irq_status(npu, status);
197
198 airoha_npu_wlan_disable_irq(npu, index);
199 napi_schedule(&dev->napi[qid]);
200 out:
201 rcu_read_unlock();
202
203 return IRQ_HANDLED;
204 }
205
mt76_npu_dma_add_buf(struct mt76_phy * phy,struct mt76_queue * q,struct sk_buff * skb,struct mt76_queue_buf * buf,void * txwi_ptr)206 int mt76_npu_dma_add_buf(struct mt76_phy *phy, struct mt76_queue *q,
207 struct sk_buff *skb, struct mt76_queue_buf *buf,
208 void *txwi_ptr)
209 {
210 u16 txwi_len = min_t(u16, phy->dev->drv->txwi_size, NPU_TXWI_LEN);
211 struct airoha_npu_tx_dma_desc *desc = (void *)q->desc;
212 int ret;
213
214 /* TODO: Take into account unlinear skbs */
215 memcpy(desc[q->head].txwi, txwi_ptr, txwi_len);
216 desc[q->head].addr = buf->addr;
217 desc[q->head].ctrl = FIELD_PREP(NPU_TX_DMA_DESC_VEND_LEN_MASK, txwi_len) |
218 FIELD_PREP(NPU_TX_DMA_DESC_LEN_MASK, skb->len) |
219 NPU_TX_DMA_DESC_DONE_MASK;
220
221 ret = q->head;
222 q->entry[q->head].skip_buf0 = true;
223 q->entry[q->head].skip_buf1 = true;
224 q->entry[q->head].txwi = NULL;
225 q->entry[q->head].skb = NULL;
226 q->entry[q->head].wcid = 0xffff;
227
228 q->head = (q->head + 1) % q->ndesc;
229 q->queued++;
230
231 return ret;
232 }
233
mt76_npu_txdesc_cleanup(struct mt76_queue * q,int index)234 void mt76_npu_txdesc_cleanup(struct mt76_queue *q, int index)
235 {
236 struct airoha_npu_tx_dma_desc *desc = (void *)q->desc;
237
238 if (!mt76_queue_is_npu_tx(q))
239 return;
240
241 desc[index].ctrl &= ~NPU_TX_DMA_DESC_DONE_MASK;
242 }
243
mt76_npu_queue_setup(struct mt76_dev * dev,struct mt76_queue * q)244 void mt76_npu_queue_setup(struct mt76_dev *dev, struct mt76_queue *q)
245 {
246 int qid = FIELD_GET(MT_QFLAG_WED_RING, q->flags);
247 bool xmit = mt76_queue_is_npu_tx(q);
248 struct airoha_npu *npu;
249
250 if (!mt76_queue_is_npu(q))
251 return;
252
253 npu = rcu_dereference_protected(dev->mmio.npu, &dev->mutex);
254 if (npu)
255 q->wed_regs = airoha_npu_wlan_get_queue_addr(npu, qid, xmit);
256 }
257
mt76_npu_rx_queue_init(struct mt76_dev * dev,struct mt76_queue * q)258 int mt76_npu_rx_queue_init(struct mt76_dev *dev, struct mt76_queue *q)
259 {
260 int err, irq, qid = q - &dev->q_rx[0];
261 int size, index = qid - MT_RXQ_NPU0;
262 struct airoha_npu *npu;
263 const char *name;
264
265 mutex_lock(&dev->mutex);
266
267 npu = rcu_dereference_protected(dev->mmio.npu, &dev->mutex);
268 irq = npu && index < ARRAY_SIZE(npu->irqs) ? npu->irqs[index]
269 : -EINVAL;
270 if (irq < 0) {
271 err = irq;
272 goto out;
273 }
274
275 q->flags = MT_NPU_Q_RX(index);
276 size = qid == MT_RXQ_NPU1 ? NPU_RX1_DESC_NUM : NPU_RX0_DESC_NUM;
277 err = dev->queue_ops->alloc(dev, q, 0, size,
278 MT76_NPU_RX_BUF_SIZE, 0);
279 if (err)
280 goto out;
281
282 name = devm_kasprintf(dev->dev, GFP_KERNEL, "mt76-npu.%d", index);
283 if (!name) {
284 err = -ENOMEM;
285 goto out;
286 }
287
288 err = devm_request_irq(dev->dev, irq, mt76_npu_irq_handler,
289 IRQF_SHARED, name, q);
290 if (err)
291 goto out;
292
293 netif_napi_add(dev->napi_dev, &dev->napi[qid], mt76_npu_rx_poll);
294 mt76_npu_fill_rx_queue(dev, q);
295 napi_enable(&dev->napi[qid]);
296 out:
297 mutex_unlock(&dev->mutex);
298
299 return err;
300 }
301 EXPORT_SYMBOL_GPL(mt76_npu_rx_queue_init);
302
mt76_npu_setup_tc_block_cb(enum tc_setup_type type,void * type_data,void * cb_priv)303 static int mt76_npu_setup_tc_block_cb(enum tc_setup_type type,
304 void *type_data, void *cb_priv)
305 {
306 struct mt76_phy *phy = cb_priv;
307 struct mt76_dev *dev = phy->dev;
308 struct airoha_ppe_dev *ppe_dev;
309 int err = -EOPNOTSUPP;
310
311 if (type != TC_SETUP_CLSFLOWER)
312 return -EOPNOTSUPP;
313
314 mutex_lock(&dev->mutex);
315
316 ppe_dev = rcu_dereference_protected(dev->mmio.ppe_dev, &dev->mutex);
317 if (ppe_dev)
318 err = airoha_ppe_dev_setup_tc_block_cb(ppe_dev, type_data);
319
320 mutex_unlock(&dev->mutex);
321
322 return err;
323 }
324
mt76_npu_setup_tc_block(struct mt76_phy * phy,struct net_device * dev,struct flow_block_offload * f)325 static int mt76_npu_setup_tc_block(struct mt76_phy *phy,
326 struct net_device *dev,
327 struct flow_block_offload *f)
328 {
329 flow_setup_cb_t *cb = mt76_npu_setup_tc_block_cb;
330 static LIST_HEAD(block_cb_list);
331 struct flow_block_cb *block_cb;
332
333 if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
334 return -EOPNOTSUPP;
335
336 if (!tc_can_offload(dev))
337 return -EOPNOTSUPP;
338
339 f->driver_block_list = &block_cb_list;
340 switch (f->command) {
341 case FLOW_BLOCK_BIND:
342 block_cb = flow_block_cb_lookup(f->block, cb, dev);
343 if (block_cb) {
344 flow_block_cb_incref(block_cb);
345 return 0;
346 }
347
348 block_cb = flow_block_cb_alloc(cb, dev, phy, NULL);
349 if (IS_ERR(block_cb))
350 return PTR_ERR(block_cb);
351
352 flow_block_cb_incref(block_cb);
353 flow_block_cb_add(block_cb, f);
354 list_add_tail(&block_cb->driver_list, &block_cb_list);
355 return 0;
356 case FLOW_BLOCK_UNBIND:
357 block_cb = flow_block_cb_lookup(f->block, cb, dev);
358 if (!block_cb)
359 return -ENOENT;
360
361 if (!flow_block_cb_decref(block_cb)) {
362 flow_block_cb_remove(block_cb, f);
363 list_del(&block_cb->driver_list);
364 }
365 return 0;
366 default:
367 return -EOPNOTSUPP;
368 }
369 }
370
mt76_npu_net_setup_tc(struct ieee80211_hw * hw,struct ieee80211_vif * vif,struct net_device * dev,enum tc_setup_type type,void * type_data)371 int mt76_npu_net_setup_tc(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
372 struct net_device *dev, enum tc_setup_type type,
373 void *type_data)
374 {
375 struct mt76_phy *phy = hw->priv;
376
377 if (!tc_can_offload(dev))
378 return -EOPNOTSUPP;
379
380 if (!mt76_npu_device_active(phy->dev))
381 return -EOPNOTSUPP;
382
383 switch (type) {
384 case TC_SETUP_BLOCK:
385 case TC_SETUP_FT:
386 return mt76_npu_setup_tc_block(phy, dev, type_data);
387 default:
388 return -EOPNOTSUPP;
389 }
390 }
391 EXPORT_SYMBOL_GPL(mt76_npu_net_setup_tc);
392
mt76_npu_disable_irqs(struct mt76_dev * dev)393 void mt76_npu_disable_irqs(struct mt76_dev *dev)
394 {
395 struct airoha_npu *npu;
396 int i;
397
398 rcu_read_lock();
399
400 npu = rcu_dereference(dev->mmio.npu);
401 if (!npu)
402 goto unlock;
403
404 for (i = MT_RXQ_NPU0; i <= MT_RXQ_NPU1; i++) {
405 int qid = i - MT_RXQ_NPU0;
406 u32 status;
407
408 status = airoha_npu_wlan_get_irq_status(npu, qid);
409 airoha_npu_wlan_set_irq_status(npu, status);
410 airoha_npu_wlan_disable_irq(npu, qid);
411 }
412 unlock:
413 rcu_read_unlock();
414 }
415 EXPORT_SYMBOL_GPL(mt76_npu_disable_irqs);
416
mt76_npu_init(struct mt76_dev * dev,phys_addr_t phy_addr,int type)417 int mt76_npu_init(struct mt76_dev *dev, phys_addr_t phy_addr, int type)
418 {
419 struct airoha_ppe_dev *ppe_dev;
420 struct airoha_npu *npu;
421 int err = 0;
422
423 /* NPU offloading is only supported by MT7992 */
424 if (!is_mt7992(dev))
425 return 0;
426
427 mutex_lock(&dev->mutex);
428
429 npu = airoha_npu_get(dev->dev);
430 if (IS_ERR(npu)) {
431 request_module("airoha-npu");
432 npu = airoha_npu_get(dev->dev);
433 }
434
435 if (IS_ERR(npu)) {
436 err = PTR_ERR(npu);
437 goto error_unlock;
438 }
439
440 ppe_dev = airoha_ppe_get_dev(dev->dev);
441 if (IS_ERR(ppe_dev)) {
442 request_module("airoha-eth");
443 ppe_dev = airoha_ppe_get_dev(dev->dev);
444 }
445
446 if (IS_ERR(ppe_dev)) {
447 err = PTR_ERR(ppe_dev);
448 goto error_npu_put;
449 }
450
451 err = airoha_npu_wlan_init_reserved_memory(npu);
452 if (err)
453 goto error_ppe_put;
454
455 dev->dma_dev = npu->dev;
456 dev->mmio.phy_addr = phy_addr;
457 dev->mmio.npu_type = type;
458 /* NPU offloading requires HW-RRO for RX packet reordering. */
459 dev->hwrro_mode = MT76_HWRRO_V3_1;
460
461 rcu_assign_pointer(dev->mmio.npu, npu);
462 rcu_assign_pointer(dev->mmio.ppe_dev, ppe_dev);
463 synchronize_rcu();
464
465 mutex_unlock(&dev->mutex);
466
467 return 0;
468
469 error_ppe_put:
470 airoha_ppe_put_dev(ppe_dev);
471 error_npu_put:
472 airoha_npu_put(npu);
473 error_unlock:
474 mutex_unlock(&dev->mutex);
475
476 return err;
477 }
478 EXPORT_SYMBOL_GPL(mt76_npu_init);
479
mt76_npu_deinit(struct mt76_dev * dev)480 void mt76_npu_deinit(struct mt76_dev *dev)
481 {
482 struct airoha_ppe_dev *ppe_dev;
483 struct airoha_npu *npu;
484
485 mutex_lock(&dev->mutex);
486
487 npu = rcu_replace_pointer(dev->mmio.npu, NULL,
488 lockdep_is_held(&dev->mutex));
489 if (npu)
490 airoha_npu_put(npu);
491
492 ppe_dev = rcu_replace_pointer(dev->mmio.ppe_dev, NULL,
493 lockdep_is_held(&dev->mutex));
494 if (ppe_dev)
495 airoha_ppe_put_dev(ppe_dev);
496
497 mutex_unlock(&dev->mutex);
498
499 mt76_npu_queue_cleanup(dev, &dev->q_rx[MT_RXQ_NPU0]);
500 mt76_npu_queue_cleanup(dev, &dev->q_rx[MT_RXQ_NPU1]);
501 }
502