1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2025 AIROHA Inc
4 * Author: Lorenzo Bianconi <lorenzo@kernel.org>
5 */
6
7 #include <linux/devcoredump.h>
8 #include <linux/firmware.h>
9 #include <linux/platform_device.h>
10 #include <linux/of_net.h>
11 #include <linux/of_platform.h>
12 #include <linux/of_reserved_mem.h>
13 #include <linux/regmap.h>
14
15 #include "airoha_npu.h"
16
17 #define NPU_EN7581_FIRMWARE_DATA "airoha/en7581_npu_data.bin"
18 #define NPU_EN7581_FIRMWARE_RV32 "airoha/en7581_npu_rv32.bin"
19 #define NPU_EN7581_FIRMWARE_RV32_MAX_SIZE 0x200000
20 #define NPU_EN7581_FIRMWARE_DATA_MAX_SIZE 0x10000
21 #define NPU_DUMP_SIZE 512
22
23 #define REG_NPU_LOCAL_SRAM 0x0
24
25 #define NPU_PC_BASE_ADDR 0x305000
26 #define REG_PC_DBG(_n) (0x305000 + ((_n) * 0x100))
27
28 #define NPU_CLUSTER_BASE_ADDR 0x306000
29
30 #define REG_CR_BOOT_TRIGGER (NPU_CLUSTER_BASE_ADDR + 0x000)
31 #define REG_CR_BOOT_CONFIG (NPU_CLUSTER_BASE_ADDR + 0x004)
32 #define REG_CR_BOOT_BASE(_n) (NPU_CLUSTER_BASE_ADDR + 0x020 + ((_n) << 2))
33
34 #define NPU_MBOX_BASE_ADDR 0x30c000
35
36 #define REG_CR_MBOX_INT_STATUS (NPU_MBOX_BASE_ADDR + 0x000)
37 #define MBOX_INT_STATUS_MASK BIT(8)
38
39 #define REG_CR_MBOX_INT_MASK(_n) (NPU_MBOX_BASE_ADDR + 0x004 + ((_n) << 2))
40 #define REG_CR_MBQ0_CTRL(_n) (NPU_MBOX_BASE_ADDR + 0x030 + ((_n) << 2))
41 #define REG_CR_MBQ8_CTRL(_n) (NPU_MBOX_BASE_ADDR + 0x0b0 + ((_n) << 2))
42 #define REG_CR_NPU_MIB(_n) (NPU_MBOX_BASE_ADDR + 0x140 + ((_n) << 2))
43
44 #define NPU_TIMER_BASE_ADDR 0x310100
45 #define REG_WDT_TIMER_CTRL(_n) (NPU_TIMER_BASE_ADDR + ((_n) * 0x100))
46 #define WDT_EN_MASK BIT(25)
47 #define WDT_INTR_MASK BIT(21)
48
49 enum {
50 NPU_OP_SET = 1,
51 NPU_OP_SET_NO_WAIT,
52 NPU_OP_GET,
53 NPU_OP_GET_NO_WAIT,
54 };
55
56 enum {
57 NPU_FUNC_WIFI,
58 NPU_FUNC_TUNNEL,
59 NPU_FUNC_NOTIFY,
60 NPU_FUNC_DBA,
61 NPU_FUNC_TR471,
62 NPU_FUNC_PPE,
63 };
64
65 enum {
66 NPU_MBOX_ERROR,
67 NPU_MBOX_SUCCESS,
68 };
69
70 enum {
71 PPE_FUNC_SET_WAIT,
72 PPE_FUNC_SET_WAIT_HWNAT_INIT,
73 PPE_FUNC_SET_WAIT_HWNAT_DEINIT,
74 PPE_FUNC_SET_WAIT_API,
75 };
76
77 enum {
78 PPE2_SRAM_SET_ENTRY,
79 PPE_SRAM_SET_ENTRY,
80 PPE_SRAM_SET_VAL,
81 PPE_SRAM_RESET_VAL,
82 };
83
84 enum {
85 QDMA_WAN_ETHER = 1,
86 QDMA_WAN_PON_XDSL,
87 };
88
89 #define MBOX_MSG_FUNC_ID GENMASK(14, 11)
90 #define MBOX_MSG_STATIC_BUF BIT(5)
91 #define MBOX_MSG_STATUS GENMASK(4, 2)
92 #define MBOX_MSG_DONE BIT(1)
93 #define MBOX_MSG_WAIT_RSP BIT(0)
94
95 #define PPE_TYPE_L2B_IPV4 2
96 #define PPE_TYPE_L2B_IPV4_IPV6 3
97
98 struct ppe_mbox_data {
99 u32 func_type;
100 u32 func_id;
101 union {
102 struct {
103 u8 cds;
104 u8 xpon_hal_api;
105 u8 wan_xsi;
106 u8 ct_joyme4;
107 u8 max_packet;
108 u8 rsv[3];
109 u32 ppe_type;
110 u32 wan_mode;
111 u32 wan_sel;
112 } init_info;
113 struct {
114 u32 func_id;
115 u32 size;
116 u32 data;
117 } set_info;
118 };
119 };
120
airoha_npu_send_msg(struct airoha_npu * npu,int func_id,void * p,int size)121 static int airoha_npu_send_msg(struct airoha_npu *npu, int func_id,
122 void *p, int size)
123 {
124 u16 core = 0; /* FIXME */
125 u32 val, offset = core << 4;
126 dma_addr_t dma_addr;
127 void *addr;
128 int ret;
129
130 addr = kmemdup(p, size, GFP_ATOMIC);
131 if (!addr)
132 return -ENOMEM;
133
134 dma_addr = dma_map_single(npu->dev, addr, size, DMA_TO_DEVICE);
135 ret = dma_mapping_error(npu->dev, dma_addr);
136 if (ret)
137 goto out;
138
139 spin_lock_bh(&npu->cores[core].lock);
140
141 regmap_write(npu->regmap, REG_CR_MBQ0_CTRL(0) + offset, dma_addr);
142 regmap_write(npu->regmap, REG_CR_MBQ0_CTRL(1) + offset, size);
143 regmap_read(npu->regmap, REG_CR_MBQ0_CTRL(2) + offset, &val);
144 regmap_write(npu->regmap, REG_CR_MBQ0_CTRL(2) + offset, val + 1);
145 val = FIELD_PREP(MBOX_MSG_FUNC_ID, func_id) | MBOX_MSG_WAIT_RSP;
146 regmap_write(npu->regmap, REG_CR_MBQ0_CTRL(3) + offset, val);
147
148 ret = regmap_read_poll_timeout_atomic(npu->regmap,
149 REG_CR_MBQ0_CTRL(3) + offset,
150 val, (val & MBOX_MSG_DONE),
151 100, 100 * MSEC_PER_SEC);
152 if (!ret && FIELD_GET(MBOX_MSG_STATUS, val) != NPU_MBOX_SUCCESS)
153 ret = -EINVAL;
154
155 spin_unlock_bh(&npu->cores[core].lock);
156
157 dma_unmap_single(npu->dev, dma_addr, size, DMA_TO_DEVICE);
158 out:
159 kfree(addr);
160
161 return ret;
162 }
163
airoha_npu_run_firmware(struct device * dev,void __iomem * base,struct reserved_mem * rmem)164 static int airoha_npu_run_firmware(struct device *dev, void __iomem *base,
165 struct reserved_mem *rmem)
166 {
167 const struct firmware *fw;
168 void __iomem *addr;
169 int ret;
170
171 ret = request_firmware(&fw, NPU_EN7581_FIRMWARE_RV32, dev);
172 if (ret)
173 return ret == -ENOENT ? -EPROBE_DEFER : ret;
174
175 if (fw->size > NPU_EN7581_FIRMWARE_RV32_MAX_SIZE) {
176 dev_err(dev, "%s: fw size too overlimit (%zu)\n",
177 NPU_EN7581_FIRMWARE_RV32, fw->size);
178 ret = -E2BIG;
179 goto out;
180 }
181
182 addr = devm_ioremap(dev, rmem->base, rmem->size);
183 if (!addr) {
184 ret = -ENOMEM;
185 goto out;
186 }
187
188 memcpy_toio(addr, fw->data, fw->size);
189 release_firmware(fw);
190
191 ret = request_firmware(&fw, NPU_EN7581_FIRMWARE_DATA, dev);
192 if (ret)
193 return ret == -ENOENT ? -EPROBE_DEFER : ret;
194
195 if (fw->size > NPU_EN7581_FIRMWARE_DATA_MAX_SIZE) {
196 dev_err(dev, "%s: fw size too overlimit (%zu)\n",
197 NPU_EN7581_FIRMWARE_DATA, fw->size);
198 ret = -E2BIG;
199 goto out;
200 }
201
202 memcpy_toio(base + REG_NPU_LOCAL_SRAM, fw->data, fw->size);
203 out:
204 release_firmware(fw);
205
206 return ret;
207 }
208
airoha_npu_mbox_handler(int irq,void * npu_instance)209 static irqreturn_t airoha_npu_mbox_handler(int irq, void *npu_instance)
210 {
211 struct airoha_npu *npu = npu_instance;
212
213 /* clear mbox interrupt status */
214 regmap_write(npu->regmap, REG_CR_MBOX_INT_STATUS,
215 MBOX_INT_STATUS_MASK);
216
217 /* acknowledge npu */
218 regmap_update_bits(npu->regmap, REG_CR_MBQ8_CTRL(3),
219 MBOX_MSG_STATUS | MBOX_MSG_DONE, MBOX_MSG_DONE);
220
221 return IRQ_HANDLED;
222 }
223
airoha_npu_wdt_work(struct work_struct * work)224 static void airoha_npu_wdt_work(struct work_struct *work)
225 {
226 struct airoha_npu_core *core;
227 struct airoha_npu *npu;
228 void *dump;
229 u32 val[3];
230 int c;
231
232 core = container_of(work, struct airoha_npu_core, wdt_work);
233 npu = core->npu;
234
235 dump = vzalloc(NPU_DUMP_SIZE);
236 if (!dump)
237 return;
238
239 c = core - &npu->cores[0];
240 regmap_bulk_read(npu->regmap, REG_PC_DBG(c), val, ARRAY_SIZE(val));
241 snprintf(dump, NPU_DUMP_SIZE, "PC: %08x SP: %08x LR: %08x\n",
242 val[0], val[1], val[2]);
243
244 dev_coredumpv(npu->dev, dump, NPU_DUMP_SIZE, GFP_KERNEL);
245 }
246
airoha_npu_wdt_handler(int irq,void * core_instance)247 static irqreturn_t airoha_npu_wdt_handler(int irq, void *core_instance)
248 {
249 struct airoha_npu_core *core = core_instance;
250 struct airoha_npu *npu = core->npu;
251 int c = core - &npu->cores[0];
252 u32 val;
253
254 regmap_set_bits(npu->regmap, REG_WDT_TIMER_CTRL(c), WDT_INTR_MASK);
255 if (!regmap_read(npu->regmap, REG_WDT_TIMER_CTRL(c), &val) &&
256 FIELD_GET(WDT_EN_MASK, val))
257 schedule_work(&core->wdt_work);
258
259 return IRQ_HANDLED;
260 }
261
airoha_npu_ppe_init(struct airoha_npu * npu)262 static int airoha_npu_ppe_init(struct airoha_npu *npu)
263 {
264 struct ppe_mbox_data ppe_data = {
265 .func_type = NPU_OP_SET,
266 .func_id = PPE_FUNC_SET_WAIT_HWNAT_INIT,
267 .init_info = {
268 .ppe_type = PPE_TYPE_L2B_IPV4_IPV6,
269 .wan_mode = QDMA_WAN_ETHER,
270 },
271 };
272
273 return airoha_npu_send_msg(npu, NPU_FUNC_PPE, &ppe_data,
274 sizeof(struct ppe_mbox_data));
275 }
276
airoha_npu_ppe_deinit(struct airoha_npu * npu)277 static int airoha_npu_ppe_deinit(struct airoha_npu *npu)
278 {
279 struct ppe_mbox_data ppe_data = {
280 .func_type = NPU_OP_SET,
281 .func_id = PPE_FUNC_SET_WAIT_HWNAT_DEINIT,
282 };
283
284 return airoha_npu_send_msg(npu, NPU_FUNC_PPE, &ppe_data,
285 sizeof(struct ppe_mbox_data));
286 }
287
airoha_npu_ppe_flush_sram_entries(struct airoha_npu * npu,dma_addr_t foe_addr,int sram_num_entries)288 static int airoha_npu_ppe_flush_sram_entries(struct airoha_npu *npu,
289 dma_addr_t foe_addr,
290 int sram_num_entries)
291 {
292 struct ppe_mbox_data ppe_data = {
293 .func_type = NPU_OP_SET,
294 .func_id = PPE_FUNC_SET_WAIT_API,
295 .set_info = {
296 .func_id = PPE_SRAM_RESET_VAL,
297 .data = foe_addr,
298 .size = sram_num_entries,
299 },
300 };
301
302 return airoha_npu_send_msg(npu, NPU_FUNC_PPE, &ppe_data,
303 sizeof(struct ppe_mbox_data));
304 }
305
airoha_npu_foe_commit_entry(struct airoha_npu * npu,dma_addr_t foe_addr,u32 entry_size,u32 hash,bool ppe2)306 static int airoha_npu_foe_commit_entry(struct airoha_npu *npu,
307 dma_addr_t foe_addr,
308 u32 entry_size, u32 hash, bool ppe2)
309 {
310 struct ppe_mbox_data ppe_data = {
311 .func_type = NPU_OP_SET,
312 .func_id = PPE_FUNC_SET_WAIT_API,
313 .set_info = {
314 .data = foe_addr,
315 .size = entry_size,
316 },
317 };
318 int err;
319
320 ppe_data.set_info.func_id = ppe2 ? PPE2_SRAM_SET_ENTRY
321 : PPE_SRAM_SET_ENTRY;
322
323 err = airoha_npu_send_msg(npu, NPU_FUNC_PPE, &ppe_data,
324 sizeof(struct ppe_mbox_data));
325 if (err)
326 return err;
327
328 ppe_data.set_info.func_id = PPE_SRAM_SET_VAL;
329 ppe_data.set_info.data = hash;
330 ppe_data.set_info.size = sizeof(u32);
331
332 return airoha_npu_send_msg(npu, NPU_FUNC_PPE, &ppe_data,
333 sizeof(struct ppe_mbox_data));
334 }
335
airoha_npu_get(struct device * dev)336 struct airoha_npu *airoha_npu_get(struct device *dev)
337 {
338 struct platform_device *pdev;
339 struct device_node *np;
340 struct airoha_npu *npu;
341
342 np = of_parse_phandle(dev->of_node, "airoha,npu", 0);
343 if (!np)
344 return ERR_PTR(-ENODEV);
345
346 pdev = of_find_device_by_node(np);
347 of_node_put(np);
348
349 if (!pdev) {
350 dev_err(dev, "cannot find device node %s\n", np->name);
351 return ERR_PTR(-ENODEV);
352 }
353
354 if (!try_module_get(THIS_MODULE)) {
355 dev_err(dev, "failed to get the device driver module\n");
356 npu = ERR_PTR(-ENODEV);
357 goto error_pdev_put;
358 }
359
360 npu = platform_get_drvdata(pdev);
361 if (!npu) {
362 npu = ERR_PTR(-ENODEV);
363 goto error_module_put;
364 }
365
366 if (!device_link_add(dev, &pdev->dev, DL_FLAG_AUTOREMOVE_SUPPLIER)) {
367 dev_err(&pdev->dev,
368 "failed to create device link to consumer %s\n",
369 dev_name(dev));
370 npu = ERR_PTR(-EINVAL);
371 goto error_module_put;
372 }
373
374 return npu;
375
376 error_module_put:
377 module_put(THIS_MODULE);
378 error_pdev_put:
379 platform_device_put(pdev);
380
381 return npu;
382 }
383 EXPORT_SYMBOL_GPL(airoha_npu_get);
384
airoha_npu_put(struct airoha_npu * npu)385 void airoha_npu_put(struct airoha_npu *npu)
386 {
387 module_put(THIS_MODULE);
388 put_device(npu->dev);
389 }
390 EXPORT_SYMBOL_GPL(airoha_npu_put);
391
392 static const struct of_device_id of_airoha_npu_match[] = {
393 { .compatible = "airoha,en7581-npu" },
394 { /* sentinel */ }
395 };
396 MODULE_DEVICE_TABLE(of, of_airoha_npu_match);
397
398 static const struct regmap_config regmap_config = {
399 .name = "npu",
400 .reg_bits = 32,
401 .val_bits = 32,
402 .reg_stride = 4,
403 .disable_locking = true,
404 };
405
airoha_npu_probe(struct platform_device * pdev)406 static int airoha_npu_probe(struct platform_device *pdev)
407 {
408 struct device *dev = &pdev->dev;
409 struct reserved_mem *rmem;
410 struct airoha_npu *npu;
411 struct device_node *np;
412 void __iomem *base;
413 int i, irq, err;
414
415 base = devm_platform_ioremap_resource(pdev, 0);
416 if (IS_ERR(base))
417 return PTR_ERR(base);
418
419 npu = devm_kzalloc(dev, sizeof(*npu), GFP_KERNEL);
420 if (!npu)
421 return -ENOMEM;
422
423 npu->dev = dev;
424 npu->ops.ppe_init = airoha_npu_ppe_init;
425 npu->ops.ppe_deinit = airoha_npu_ppe_deinit;
426 npu->ops.ppe_flush_sram_entries = airoha_npu_ppe_flush_sram_entries;
427 npu->ops.ppe_foe_commit_entry = airoha_npu_foe_commit_entry;
428
429 npu->regmap = devm_regmap_init_mmio(dev, base, ®map_config);
430 if (IS_ERR(npu->regmap))
431 return PTR_ERR(npu->regmap);
432
433 np = of_parse_phandle(dev->of_node, "memory-region", 0);
434 if (!np)
435 return -ENODEV;
436
437 rmem = of_reserved_mem_lookup(np);
438 of_node_put(np);
439
440 if (!rmem)
441 return -ENODEV;
442
443 irq = platform_get_irq(pdev, 0);
444 if (irq < 0)
445 return irq;
446
447 err = devm_request_irq(dev, irq, airoha_npu_mbox_handler,
448 IRQF_SHARED, "airoha-npu-mbox", npu);
449 if (err)
450 return err;
451
452 for (i = 0; i < ARRAY_SIZE(npu->cores); i++) {
453 struct airoha_npu_core *core = &npu->cores[i];
454
455 spin_lock_init(&core->lock);
456 core->npu = npu;
457
458 irq = platform_get_irq(pdev, i + 1);
459 if (irq < 0)
460 return irq;
461
462 err = devm_request_irq(dev, irq, airoha_npu_wdt_handler,
463 IRQF_SHARED, "airoha-npu-wdt", core);
464 if (err)
465 return err;
466
467 INIT_WORK(&core->wdt_work, airoha_npu_wdt_work);
468 }
469
470 err = dma_set_coherent_mask(dev, DMA_BIT_MASK(32));
471 if (err)
472 return err;
473
474 err = airoha_npu_run_firmware(dev, base, rmem);
475 if (err)
476 return dev_err_probe(dev, err, "failed to run npu firmware\n");
477
478 regmap_write(npu->regmap, REG_CR_NPU_MIB(10),
479 rmem->base + NPU_EN7581_FIRMWARE_RV32_MAX_SIZE);
480 regmap_write(npu->regmap, REG_CR_NPU_MIB(11), 0x40000); /* SRAM 256K */
481 regmap_write(npu->regmap, REG_CR_NPU_MIB(12), 0);
482 regmap_write(npu->regmap, REG_CR_NPU_MIB(21), 1);
483 msleep(100);
484
485 /* setting booting address */
486 for (i = 0; i < NPU_NUM_CORES; i++)
487 regmap_write(npu->regmap, REG_CR_BOOT_BASE(i), rmem->base);
488 usleep_range(1000, 2000);
489
490 /* enable NPU cores */
491 /* do not start core3 since it is used for WiFi offloading */
492 regmap_write(npu->regmap, REG_CR_BOOT_CONFIG, 0xf7);
493 regmap_write(npu->regmap, REG_CR_BOOT_TRIGGER, 0x1);
494 msleep(100);
495
496 platform_set_drvdata(pdev, npu);
497
498 return 0;
499 }
500
airoha_npu_remove(struct platform_device * pdev)501 static void airoha_npu_remove(struct platform_device *pdev)
502 {
503 struct airoha_npu *npu = platform_get_drvdata(pdev);
504 int i;
505
506 for (i = 0; i < ARRAY_SIZE(npu->cores); i++)
507 cancel_work_sync(&npu->cores[i].wdt_work);
508 }
509
510 static struct platform_driver airoha_npu_driver = {
511 .probe = airoha_npu_probe,
512 .remove = airoha_npu_remove,
513 .driver = {
514 .name = "airoha-npu",
515 .of_match_table = of_airoha_npu_match,
516 },
517 };
518 module_platform_driver(airoha_npu_driver);
519
520 MODULE_LICENSE("GPL");
521 MODULE_AUTHOR("Lorenzo Bianconi <lorenzo@kernel.org>");
522 MODULE_DESCRIPTION("Airoha Network Processor Unit driver");
523