1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2025 AIROHA Inc 4 * Author: Lorenzo Bianconi <lorenzo@kernel.org> 5 */ 6 7 #include <linux/devcoredump.h> 8 #include <linux/firmware.h> 9 #include <linux/platform_device.h> 10 #include <linux/of_net.h> 11 #include <linux/of_platform.h> 12 #include <linux/of_reserved_mem.h> 13 #include <linux/regmap.h> 14 15 #include "airoha_eth.h" 16 #include "airoha_npu.h" 17 18 #define NPU_EN7581_FIRMWARE_DATA "airoha/en7581_npu_data.bin" 19 #define NPU_EN7581_FIRMWARE_RV32 "airoha/en7581_npu_rv32.bin" 20 #define NPU_EN7581_FIRMWARE_RV32_MAX_SIZE 0x200000 21 #define NPU_EN7581_FIRMWARE_DATA_MAX_SIZE 0x10000 22 #define NPU_DUMP_SIZE 512 23 24 #define REG_NPU_LOCAL_SRAM 0x0 25 26 #define NPU_PC_BASE_ADDR 0x305000 27 #define REG_PC_DBG(_n) (0x305000 + ((_n) * 0x100)) 28 29 #define NPU_CLUSTER_BASE_ADDR 0x306000 30 31 #define REG_CR_BOOT_TRIGGER (NPU_CLUSTER_BASE_ADDR + 0x000) 32 #define REG_CR_BOOT_CONFIG (NPU_CLUSTER_BASE_ADDR + 0x004) 33 #define REG_CR_BOOT_BASE(_n) (NPU_CLUSTER_BASE_ADDR + 0x020 + ((_n) << 2)) 34 35 #define NPU_MBOX_BASE_ADDR 0x30c000 36 37 #define REG_CR_MBOX_INT_STATUS (NPU_MBOX_BASE_ADDR + 0x000) 38 #define MBOX_INT_STATUS_MASK BIT(8) 39 40 #define REG_CR_MBOX_INT_MASK(_n) (NPU_MBOX_BASE_ADDR + 0x004 + ((_n) << 2)) 41 #define REG_CR_MBQ0_CTRL(_n) (NPU_MBOX_BASE_ADDR + 0x030 + ((_n) << 2)) 42 #define REG_CR_MBQ8_CTRL(_n) (NPU_MBOX_BASE_ADDR + 0x0b0 + ((_n) << 2)) 43 #define REG_CR_NPU_MIB(_n) (NPU_MBOX_BASE_ADDR + 0x140 + ((_n) << 2)) 44 45 #define NPU_TIMER_BASE_ADDR 0x310100 46 #define REG_WDT_TIMER_CTRL(_n) (NPU_TIMER_BASE_ADDR + ((_n) * 0x100)) 47 #define WDT_EN_MASK BIT(25) 48 #define WDT_INTR_MASK BIT(21) 49 50 enum { 51 NPU_OP_SET = 1, 52 NPU_OP_SET_NO_WAIT, 53 NPU_OP_GET, 54 NPU_OP_GET_NO_WAIT, 55 }; 56 57 enum { 58 NPU_FUNC_WIFI, 59 NPU_FUNC_TUNNEL, 60 NPU_FUNC_NOTIFY, 61 NPU_FUNC_DBA, 62 NPU_FUNC_TR471, 63 NPU_FUNC_PPE, 64 }; 65 66 enum { 67 NPU_MBOX_ERROR, 68 NPU_MBOX_SUCCESS, 69 }; 70 71 enum { 72 PPE_FUNC_SET_WAIT, 73 PPE_FUNC_SET_WAIT_HWNAT_INIT, 74 PPE_FUNC_SET_WAIT_HWNAT_DEINIT, 75 PPE_FUNC_SET_WAIT_API, 76 PPE_FUNC_SET_WAIT_FLOW_STATS_SETUP, 77 }; 78 79 enum { 80 PPE2_SRAM_SET_ENTRY, 81 PPE_SRAM_SET_ENTRY, 82 PPE_SRAM_SET_VAL, 83 PPE_SRAM_RESET_VAL, 84 }; 85 86 enum { 87 QDMA_WAN_ETHER = 1, 88 QDMA_WAN_PON_XDSL, 89 }; 90 91 #define MBOX_MSG_FUNC_ID GENMASK(14, 11) 92 #define MBOX_MSG_STATIC_BUF BIT(5) 93 #define MBOX_MSG_STATUS GENMASK(4, 2) 94 #define MBOX_MSG_DONE BIT(1) 95 #define MBOX_MSG_WAIT_RSP BIT(0) 96 97 #define PPE_TYPE_L2B_IPV4 2 98 #define PPE_TYPE_L2B_IPV4_IPV6 3 99 100 struct ppe_mbox_data { 101 u32 func_type; 102 u32 func_id; 103 union { 104 struct { 105 u8 cds; 106 u8 xpon_hal_api; 107 u8 wan_xsi; 108 u8 ct_joyme4; 109 u8 max_packet; 110 u8 rsv[3]; 111 u32 ppe_type; 112 u32 wan_mode; 113 u32 wan_sel; 114 } init_info; 115 struct { 116 u32 func_id; 117 u32 size; 118 u32 data; 119 } set_info; 120 struct { 121 u32 npu_stats_addr; 122 u32 foe_stats_addr; 123 } stats_info; 124 }; 125 }; 126 127 static int airoha_npu_send_msg(struct airoha_npu *npu, int func_id, 128 void *p, int size) 129 { 130 u16 core = 0; /* FIXME */ 131 u32 val, offset = core << 4; 132 dma_addr_t dma_addr; 133 int ret; 134 135 dma_addr = dma_map_single(npu->dev, p, size, DMA_TO_DEVICE); 136 ret = dma_mapping_error(npu->dev, dma_addr); 137 if (ret) 138 return ret; 139 140 spin_lock_bh(&npu->cores[core].lock); 141 142 regmap_write(npu->regmap, REG_CR_MBQ0_CTRL(0) + offset, dma_addr); 143 regmap_write(npu->regmap, REG_CR_MBQ0_CTRL(1) + offset, size); 144 regmap_read(npu->regmap, REG_CR_MBQ0_CTRL(2) + offset, &val); 145 regmap_write(npu->regmap, REG_CR_MBQ0_CTRL(2) + offset, val + 1); 146 val = FIELD_PREP(MBOX_MSG_FUNC_ID, func_id) | MBOX_MSG_WAIT_RSP; 147 regmap_write(npu->regmap, REG_CR_MBQ0_CTRL(3) + offset, val); 148 149 ret = regmap_read_poll_timeout_atomic(npu->regmap, 150 REG_CR_MBQ0_CTRL(3) + offset, 151 val, (val & MBOX_MSG_DONE), 152 100, 100 * MSEC_PER_SEC); 153 if (!ret && FIELD_GET(MBOX_MSG_STATUS, val) != NPU_MBOX_SUCCESS) 154 ret = -EINVAL; 155 156 spin_unlock_bh(&npu->cores[core].lock); 157 158 dma_unmap_single(npu->dev, dma_addr, size, DMA_TO_DEVICE); 159 160 return ret; 161 } 162 163 static int airoha_npu_run_firmware(struct device *dev, void __iomem *base, 164 struct reserved_mem *rmem) 165 { 166 const struct firmware *fw; 167 void __iomem *addr; 168 int ret; 169 170 ret = request_firmware(&fw, NPU_EN7581_FIRMWARE_RV32, dev); 171 if (ret) 172 return ret == -ENOENT ? -EPROBE_DEFER : ret; 173 174 if (fw->size > NPU_EN7581_FIRMWARE_RV32_MAX_SIZE) { 175 dev_err(dev, "%s: fw size too overlimit (%zu)\n", 176 NPU_EN7581_FIRMWARE_RV32, fw->size); 177 ret = -E2BIG; 178 goto out; 179 } 180 181 addr = devm_ioremap(dev, rmem->base, rmem->size); 182 if (!addr) { 183 ret = -ENOMEM; 184 goto out; 185 } 186 187 memcpy_toio(addr, fw->data, fw->size); 188 release_firmware(fw); 189 190 ret = request_firmware(&fw, NPU_EN7581_FIRMWARE_DATA, dev); 191 if (ret) 192 return ret == -ENOENT ? -EPROBE_DEFER : ret; 193 194 if (fw->size > NPU_EN7581_FIRMWARE_DATA_MAX_SIZE) { 195 dev_err(dev, "%s: fw size too overlimit (%zu)\n", 196 NPU_EN7581_FIRMWARE_DATA, fw->size); 197 ret = -E2BIG; 198 goto out; 199 } 200 201 memcpy_toio(base + REG_NPU_LOCAL_SRAM, fw->data, fw->size); 202 out: 203 release_firmware(fw); 204 205 return ret; 206 } 207 208 static irqreturn_t airoha_npu_mbox_handler(int irq, void *npu_instance) 209 { 210 struct airoha_npu *npu = npu_instance; 211 212 /* clear mbox interrupt status */ 213 regmap_write(npu->regmap, REG_CR_MBOX_INT_STATUS, 214 MBOX_INT_STATUS_MASK); 215 216 /* acknowledge npu */ 217 regmap_update_bits(npu->regmap, REG_CR_MBQ8_CTRL(3), 218 MBOX_MSG_STATUS | MBOX_MSG_DONE, MBOX_MSG_DONE); 219 220 return IRQ_HANDLED; 221 } 222 223 static void airoha_npu_wdt_work(struct work_struct *work) 224 { 225 struct airoha_npu_core *core; 226 struct airoha_npu *npu; 227 void *dump; 228 u32 val[3]; 229 int c; 230 231 core = container_of(work, struct airoha_npu_core, wdt_work); 232 npu = core->npu; 233 234 dump = vzalloc(NPU_DUMP_SIZE); 235 if (!dump) 236 return; 237 238 c = core - &npu->cores[0]; 239 regmap_bulk_read(npu->regmap, REG_PC_DBG(c), val, ARRAY_SIZE(val)); 240 snprintf(dump, NPU_DUMP_SIZE, "PC: %08x SP: %08x LR: %08x\n", 241 val[0], val[1], val[2]); 242 243 dev_coredumpv(npu->dev, dump, NPU_DUMP_SIZE, GFP_KERNEL); 244 } 245 246 static irqreturn_t airoha_npu_wdt_handler(int irq, void *core_instance) 247 { 248 struct airoha_npu_core *core = core_instance; 249 struct airoha_npu *npu = core->npu; 250 int c = core - &npu->cores[0]; 251 u32 val; 252 253 regmap_set_bits(npu->regmap, REG_WDT_TIMER_CTRL(c), WDT_INTR_MASK); 254 if (!regmap_read(npu->regmap, REG_WDT_TIMER_CTRL(c), &val) && 255 FIELD_GET(WDT_EN_MASK, val)) 256 schedule_work(&core->wdt_work); 257 258 return IRQ_HANDLED; 259 } 260 261 static int airoha_npu_ppe_init(struct airoha_npu *npu) 262 { 263 struct ppe_mbox_data *ppe_data; 264 int err; 265 266 ppe_data = kzalloc(sizeof(*ppe_data), GFP_KERNEL); 267 if (!ppe_data) 268 return -ENOMEM; 269 270 ppe_data->func_type = NPU_OP_SET; 271 ppe_data->func_id = PPE_FUNC_SET_WAIT_HWNAT_INIT; 272 ppe_data->init_info.ppe_type = PPE_TYPE_L2B_IPV4_IPV6; 273 ppe_data->init_info.wan_mode = QDMA_WAN_ETHER; 274 275 err = airoha_npu_send_msg(npu, NPU_FUNC_PPE, ppe_data, 276 sizeof(*ppe_data)); 277 kfree(ppe_data); 278 279 return err; 280 } 281 282 static int airoha_npu_ppe_deinit(struct airoha_npu *npu) 283 { 284 struct ppe_mbox_data *ppe_data; 285 int err; 286 287 ppe_data = kzalloc(sizeof(*ppe_data), GFP_KERNEL); 288 if (!ppe_data) 289 return -ENOMEM; 290 291 ppe_data->func_type = NPU_OP_SET; 292 ppe_data->func_id = PPE_FUNC_SET_WAIT_HWNAT_DEINIT; 293 294 err = airoha_npu_send_msg(npu, NPU_FUNC_PPE, ppe_data, 295 sizeof(*ppe_data)); 296 kfree(ppe_data); 297 298 return err; 299 } 300 301 static int airoha_npu_ppe_flush_sram_entries(struct airoha_npu *npu, 302 dma_addr_t foe_addr, 303 int sram_num_entries) 304 { 305 struct ppe_mbox_data *ppe_data; 306 int err; 307 308 ppe_data = kzalloc(sizeof(*ppe_data), GFP_KERNEL); 309 if (!ppe_data) 310 return -ENOMEM; 311 312 ppe_data->func_type = NPU_OP_SET; 313 ppe_data->func_id = PPE_FUNC_SET_WAIT_API; 314 ppe_data->set_info.func_id = PPE_SRAM_RESET_VAL; 315 ppe_data->set_info.data = foe_addr; 316 ppe_data->set_info.size = sram_num_entries; 317 318 err = airoha_npu_send_msg(npu, NPU_FUNC_PPE, ppe_data, 319 sizeof(*ppe_data)); 320 kfree(ppe_data); 321 322 return err; 323 } 324 325 static int airoha_npu_foe_commit_entry(struct airoha_npu *npu, 326 dma_addr_t foe_addr, 327 u32 entry_size, u32 hash, bool ppe2) 328 { 329 struct ppe_mbox_data *ppe_data; 330 int err; 331 332 ppe_data = kzalloc(sizeof(*ppe_data), GFP_ATOMIC); 333 if (!ppe_data) 334 return -ENOMEM; 335 336 ppe_data->func_type = NPU_OP_SET; 337 ppe_data->func_id = PPE_FUNC_SET_WAIT_API; 338 ppe_data->set_info.data = foe_addr; 339 ppe_data->set_info.size = entry_size; 340 ppe_data->set_info.func_id = ppe2 ? PPE2_SRAM_SET_ENTRY 341 : PPE_SRAM_SET_ENTRY; 342 343 err = airoha_npu_send_msg(npu, NPU_FUNC_PPE, ppe_data, 344 sizeof(*ppe_data)); 345 if (err) 346 goto out; 347 348 ppe_data->set_info.func_id = PPE_SRAM_SET_VAL; 349 ppe_data->set_info.data = hash; 350 ppe_data->set_info.size = sizeof(u32); 351 352 err = airoha_npu_send_msg(npu, NPU_FUNC_PPE, ppe_data, 353 sizeof(*ppe_data)); 354 out: 355 kfree(ppe_data); 356 357 return err; 358 } 359 360 static int airoha_npu_stats_setup(struct airoha_npu *npu, 361 dma_addr_t foe_stats_addr) 362 { 363 int err, size = PPE_STATS_NUM_ENTRIES * sizeof(*npu->stats); 364 struct ppe_mbox_data *ppe_data; 365 366 if (!size) /* flow stats are disabled */ 367 return 0; 368 369 ppe_data = kzalloc(sizeof(*ppe_data), GFP_ATOMIC); 370 if (!ppe_data) 371 return -ENOMEM; 372 373 ppe_data->func_type = NPU_OP_SET; 374 ppe_data->func_id = PPE_FUNC_SET_WAIT_FLOW_STATS_SETUP; 375 ppe_data->stats_info.foe_stats_addr = foe_stats_addr; 376 377 err = airoha_npu_send_msg(npu, NPU_FUNC_PPE, ppe_data, 378 sizeof(*ppe_data)); 379 if (err) 380 goto out; 381 382 npu->stats = devm_ioremap(npu->dev, 383 ppe_data->stats_info.npu_stats_addr, 384 size); 385 if (!npu->stats) 386 err = -ENOMEM; 387 out: 388 kfree(ppe_data); 389 390 return err; 391 } 392 393 struct airoha_npu *airoha_npu_get(struct device *dev, dma_addr_t *stats_addr) 394 { 395 struct platform_device *pdev; 396 struct device_node *np; 397 struct airoha_npu *npu; 398 399 np = of_parse_phandle(dev->of_node, "airoha,npu", 0); 400 if (!np) 401 return ERR_PTR(-ENODEV); 402 403 pdev = of_find_device_by_node(np); 404 405 if (!pdev) { 406 dev_err(dev, "cannot find device node %s\n", np->name); 407 of_node_put(np); 408 return ERR_PTR(-ENODEV); 409 } 410 of_node_put(np); 411 412 if (!try_module_get(THIS_MODULE)) { 413 dev_err(dev, "failed to get the device driver module\n"); 414 npu = ERR_PTR(-ENODEV); 415 goto error_pdev_put; 416 } 417 418 npu = platform_get_drvdata(pdev); 419 if (!npu) { 420 npu = ERR_PTR(-ENODEV); 421 goto error_module_put; 422 } 423 424 if (!device_link_add(dev, &pdev->dev, DL_FLAG_AUTOREMOVE_SUPPLIER)) { 425 dev_err(&pdev->dev, 426 "failed to create device link to consumer %s\n", 427 dev_name(dev)); 428 npu = ERR_PTR(-EINVAL); 429 goto error_module_put; 430 } 431 432 if (stats_addr) { 433 int err; 434 435 err = airoha_npu_stats_setup(npu, *stats_addr); 436 if (err) { 437 dev_err(dev, "failed to allocate npu stats buffer\n"); 438 npu = ERR_PTR(err); 439 goto error_module_put; 440 } 441 } 442 443 return npu; 444 445 error_module_put: 446 module_put(THIS_MODULE); 447 error_pdev_put: 448 platform_device_put(pdev); 449 450 return npu; 451 } 452 EXPORT_SYMBOL_GPL(airoha_npu_get); 453 454 void airoha_npu_put(struct airoha_npu *npu) 455 { 456 module_put(THIS_MODULE); 457 put_device(npu->dev); 458 } 459 EXPORT_SYMBOL_GPL(airoha_npu_put); 460 461 static const struct of_device_id of_airoha_npu_match[] = { 462 { .compatible = "airoha,en7581-npu" }, 463 { /* sentinel */ } 464 }; 465 MODULE_DEVICE_TABLE(of, of_airoha_npu_match); 466 467 static const struct regmap_config regmap_config = { 468 .name = "npu", 469 .reg_bits = 32, 470 .val_bits = 32, 471 .reg_stride = 4, 472 .disable_locking = true, 473 }; 474 475 static int airoha_npu_probe(struct platform_device *pdev) 476 { 477 struct device *dev = &pdev->dev; 478 struct reserved_mem *rmem; 479 struct airoha_npu *npu; 480 struct device_node *np; 481 void __iomem *base; 482 int i, irq, err; 483 484 base = devm_platform_ioremap_resource(pdev, 0); 485 if (IS_ERR(base)) 486 return PTR_ERR(base); 487 488 npu = devm_kzalloc(dev, sizeof(*npu), GFP_KERNEL); 489 if (!npu) 490 return -ENOMEM; 491 492 npu->dev = dev; 493 npu->ops.ppe_init = airoha_npu_ppe_init; 494 npu->ops.ppe_deinit = airoha_npu_ppe_deinit; 495 npu->ops.ppe_flush_sram_entries = airoha_npu_ppe_flush_sram_entries; 496 npu->ops.ppe_foe_commit_entry = airoha_npu_foe_commit_entry; 497 498 npu->regmap = devm_regmap_init_mmio(dev, base, ®map_config); 499 if (IS_ERR(npu->regmap)) 500 return PTR_ERR(npu->regmap); 501 502 np = of_parse_phandle(dev->of_node, "memory-region", 0); 503 if (!np) 504 return -ENODEV; 505 506 rmem = of_reserved_mem_lookup(np); 507 of_node_put(np); 508 509 if (!rmem) 510 return -ENODEV; 511 512 irq = platform_get_irq(pdev, 0); 513 if (irq < 0) 514 return irq; 515 516 err = devm_request_irq(dev, irq, airoha_npu_mbox_handler, 517 IRQF_SHARED, "airoha-npu-mbox", npu); 518 if (err) 519 return err; 520 521 for (i = 0; i < ARRAY_SIZE(npu->cores); i++) { 522 struct airoha_npu_core *core = &npu->cores[i]; 523 524 spin_lock_init(&core->lock); 525 core->npu = npu; 526 527 irq = platform_get_irq(pdev, i + 1); 528 if (irq < 0) 529 return irq; 530 531 err = devm_request_irq(dev, irq, airoha_npu_wdt_handler, 532 IRQF_SHARED, "airoha-npu-wdt", core); 533 if (err) 534 return err; 535 536 INIT_WORK(&core->wdt_work, airoha_npu_wdt_work); 537 } 538 539 err = dma_set_coherent_mask(dev, DMA_BIT_MASK(32)); 540 if (err) 541 return err; 542 543 err = airoha_npu_run_firmware(dev, base, rmem); 544 if (err) 545 return dev_err_probe(dev, err, "failed to run npu firmware\n"); 546 547 regmap_write(npu->regmap, REG_CR_NPU_MIB(10), 548 rmem->base + NPU_EN7581_FIRMWARE_RV32_MAX_SIZE); 549 regmap_write(npu->regmap, REG_CR_NPU_MIB(11), 0x40000); /* SRAM 256K */ 550 regmap_write(npu->regmap, REG_CR_NPU_MIB(12), 0); 551 regmap_write(npu->regmap, REG_CR_NPU_MIB(21), 1); 552 msleep(100); 553 554 /* setting booting address */ 555 for (i = 0; i < NPU_NUM_CORES; i++) 556 regmap_write(npu->regmap, REG_CR_BOOT_BASE(i), rmem->base); 557 usleep_range(1000, 2000); 558 559 /* enable NPU cores */ 560 /* do not start core3 since it is used for WiFi offloading */ 561 regmap_write(npu->regmap, REG_CR_BOOT_CONFIG, 0xf7); 562 regmap_write(npu->regmap, REG_CR_BOOT_TRIGGER, 0x1); 563 msleep(100); 564 565 platform_set_drvdata(pdev, npu); 566 567 return 0; 568 } 569 570 static void airoha_npu_remove(struct platform_device *pdev) 571 { 572 struct airoha_npu *npu = platform_get_drvdata(pdev); 573 int i; 574 575 for (i = 0; i < ARRAY_SIZE(npu->cores); i++) 576 cancel_work_sync(&npu->cores[i].wdt_work); 577 } 578 579 static struct platform_driver airoha_npu_driver = { 580 .probe = airoha_npu_probe, 581 .remove = airoha_npu_remove, 582 .driver = { 583 .name = "airoha-npu", 584 .of_match_table = of_airoha_npu_match, 585 }, 586 }; 587 module_platform_driver(airoha_npu_driver); 588 589 MODULE_LICENSE("GPL"); 590 MODULE_AUTHOR("Lorenzo Bianconi <lorenzo@kernel.org>"); 591 MODULE_DESCRIPTION("Airoha Network Processor Unit driver"); 592