Lines Matching +full:axi +full:- +full:spi +full:- +full:engine +full:- +full:1
1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2020-2022 HabanaLabs, Ltd.
32 #define GAUDI2_RESET_WAIT_MSEC 1 /* 1ms */
34 #define GAUDI2_PLDM_RESET_WAIT_MSEC 1000 /* 1s */
40 #define GAUDI2_PLDM_TEST_QUEUE_WAIT_USEC 1000000 /* 1s */
45 * since the code already has built-in support for binning of up to MAX_FAULTY_TPCS TPCs
49 #define MAX_CLUSTER_BINNING_FAULTY_TPCS 1
50 #define MAX_FAULTY_XBARS 1
51 #define MAX_FAULTY_EDMAS 1
52 #define MAX_FAULTY_DECODERS 1
120 #define MMU_RANGE_INV_ASID_EN_SHIFT 1
126 #define GAUDI2_PMMU_SPI_SEI_ENABLE_MASK GENMASK(GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE - 2, 0)
127 #define GAUDI2_HMMU_SPI_SEI_ENABLE_MASK GENMASK(GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE - 1, 0)
131 #define GAUDI2_VDEC_MSIX_ENTRIES (GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM - \
132 GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM + 1)
134 #define ENGINE_ID_DCORE_OFFSET (GAUDI2_DCORE1_ENGINE_ID_EDMA_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0)
164 /* HW scrambles only bits 0-25 */
281 {RAZWI_INITIATOR_ID_X_Y(1, 5, 1), mmSFT0_HBW_RTR_IF1_RTR_CTRL_BASE,
283 {RAZWI_INITIATOR_ID_X_Y(1, 5, 1), mmSFT0_HBW_RTR_IF0_RTR_CTRL_BASE,
289 {RAZWI_INITIATOR_ID_X_Y(1, 10, 1), mmSFT2_HBW_RTR_IF0_RTR_CTRL_BASE,
291 {RAZWI_INITIATOR_ID_X_Y(1, 10, 1), mmSFT2_HBW_RTR_IF1_RTR_CTRL_BASE,
297 {RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE,
301 {RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE,
305 {RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE,
309 {RAZWI_INITIATOR_ID_X_Y(1, 5, 0), mmDCORE0_RTR0_CTRL_BASE,
313 {RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE,
317 {RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE,
321 {RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE,
325 {RAZWI_INITIATOR_ID_X_Y(1, 10, 0), mmDCORE2_RTR0_CTRL_BASE,
436 [1] = HBM_ID1,
737 "AXI SPLIT SEI Status"
786 "PQ AXI HBW error",
787 "CQ AXI HBW error",
788 "CP AXI HBW error",
791 "CP AXI LBW error",
795 "FENCE 1 inc over max value and clipped",
799 "FENCE 1 dec under min value and clipped",
808 "CQ AXI HBW error",
809 "CP AXI HBW error",
812 "CP AXI LBW error",
816 "FENCE 1 inc over max value and clipped",
820 "FENCE 1 dec under min value and clipped",
837 "MSG AXI LBW returned with error"
902 "wap sei (wbc axi err)",
935 "TRANSPOSE ENGINE DESC FIFO OVERFLOW",
946 "TRANSPOSE ENGINE DESC FIFO OVERFLOW",
959 {"armed monitor write got BRESP (SLVERR or DECERR)", "AXI id"},
976 "AXI drain HBW",
977 "AXI drain LBW",
1948 "gaudi2 vdec 1_0", "gaudi2 vdec 1_0 abnormal",
1949 "gaudi2 vdec 1_1", "gaudi2 vdec 1_1 abnormal",
1993 static const u32 gaudi2_tpc_initiator_hbw_rtr_id[NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES + 1] = {
2001 static const u32 gaudi2_tpc_initiator_lbw_rtr_id[NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES + 1] = {
2112 * and read global errors. Most HW blocks are addressable and those who aren't (N/A)-
2135 {HBM_MC_SPI_THR_ENG_MASK, "temperature-based throttling engaged"},
2136 {HBM_MC_SPI_THR_DIS_ENG_MASK, "temperature-based throttling disengaged"},
2159 {"page fault", 1}, /* INTERRUPT_CLR[1] */
2160 {"page access", 1}, /* INTERRUPT_CLR[1] */
2163 {"mmu rei0", -1}, /* no clear register bit */
2164 {"mmu rei1", -1}, /* no clear register bit */
2165 {"stlb rei0", -1}, /* no clear register bit */
2166 {"stlb rei1", -1}, /* no clear register bit */
2235 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_iterate_tpcs()
2240 ctx->rc = 0; in gaudi2_iterate_tpcs()
2246 if (!(prop->tpc_enabled_mask & BIT(tpc_seq))) in gaudi2_iterate_tpcs()
2251 ctx->fn(hdev, dcore, inst, offset, ctx); in gaudi2_iterate_tpcs()
2252 if (ctx->rc) { in gaudi2_iterate_tpcs()
2253 dev_err(hdev->dev, "TPC iterator failed for DCORE%d TPC%d\n", in gaudi2_iterate_tpcs()
2260 if (!(prop->tpc_enabled_mask & BIT(TPC_ID_DCORE0_TPC6))) in gaudi2_iterate_tpcs()
2264 offset = DCORE_TPC_OFFSET * (NUM_DCORE0_TPC - 1); in gaudi2_iterate_tpcs()
2265 ctx->fn(hdev, 0, NUM_DCORE0_TPC - 1, offset, ctx); in gaudi2_iterate_tpcs()
2266 if (ctx->rc) in gaudi2_iterate_tpcs()
2267 dev_err(hdev->dev, "TPC iterator failed for DCORE0 TPC6\n"); in gaudi2_iterate_tpcs()
2280 struct asic_fixed_properties *prop = &hdev->asic_prop; in set_number_of_functional_hbms()
2281 u8 faulty_hbms = hweight64(hdev->dram_binning); in set_number_of_functional_hbms()
2285 dev_dbg(hdev->dev, "All HBM are in use (no binning)\n"); in set_number_of_functional_hbms()
2286 prop->num_functional_hbms = GAUDI2_HBM_NUM; in set_number_of_functional_hbms()
2297 dev_err(hdev->dev, in set_number_of_functional_hbms()
2299 MAX_FAULTY_HBMS, hdev->dram_binning); in set_number_of_functional_hbms()
2300 return -EINVAL; in set_number_of_functional_hbms()
2305 * GAUDI2_HBM_NUM - 1. in set_number_of_functional_hbms()
2307 prop->num_functional_hbms = GAUDI2_HBM_NUM - faulty_hbms; in set_number_of_functional_hbms()
2313 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_set_dram_properties()
2319 return -EINVAL; in gaudi2_set_dram_properties()
2326 basic_hbm_page_size = prop->num_functional_hbms * SZ_8M; in gaudi2_set_dram_properties()
2327 prop->dram_page_size = GAUDI2_COMPENSATE_TLB_PAGE_SIZE_FACTOR * basic_hbm_page_size; in gaudi2_set_dram_properties()
2328 prop->device_mem_alloc_default_page_size = prop->dram_page_size; in gaudi2_set_dram_properties()
2329 prop->dram_size = prop->num_functional_hbms * SZ_16G; in gaudi2_set_dram_properties()
2330 prop->dram_base_address = DRAM_PHYS_BASE; in gaudi2_set_dram_properties()
2331 prop->dram_end_address = prop->dram_base_address + prop->dram_size; in gaudi2_set_dram_properties()
2332 prop->dram_supports_virtual_memory = true; in gaudi2_set_dram_properties()
2334 prop->dram_user_base_address = DRAM_PHYS_BASE + prop->dram_page_size; in gaudi2_set_dram_properties()
2335 prop->dram_hints_align_mask = ~GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK; in gaudi2_set_dram_properties()
2336 prop->hints_dram_reserved_va_range.start_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HBM_START; in gaudi2_set_dram_properties()
2337 prop->hints_dram_reserved_va_range.end_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HBM_END; in gaudi2_set_dram_properties()
2345 * 1. partition the virtual address space to DRAM-page (whole) pages. in gaudi2_set_dram_properties()
2347 * 2. limit the amount of virtual address space we got from 1 above to in gaudi2_set_dram_properties()
2356 prop->dmmu.start_addr = prop->dram_base_address + in gaudi2_set_dram_properties()
2357 (prop->dram_page_size * in gaudi2_set_dram_properties()
2358 DIV_ROUND_UP_SECTOR_T(prop->dram_size, prop->dram_page_size)); in gaudi2_set_dram_properties()
2360 prop->dmmu.end_addr = prop->dmmu.start_addr + prop->dram_page_size * in gaudi2_set_dram_properties()
2361 div_u64((VA_HBM_SPACE_END - prop->dmmu.start_addr), prop->dmmu.page_size); in gaudi2_set_dram_properties()
2368 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_set_fixed_properties()
2373 prop->max_queues = GAUDI2_QUEUE_ID_SIZE; in gaudi2_set_fixed_properties()
2374 prop->hw_queues_props = kcalloc(prop->max_queues, sizeof(struct hw_queue_properties), in gaudi2_set_fixed_properties()
2377 if (!prop->hw_queues_props) in gaudi2_set_fixed_properties()
2378 return -ENOMEM; in gaudi2_set_fixed_properties()
2380 q_props = prop->hw_queues_props; in gaudi2_set_fixed_properties()
2389 q_props[i].supports_sync_stream = 1; in gaudi2_set_fixed_properties()
2397 q_props[GAUDI2_QUEUE_ID_CPU_PQ].driver_only = 1; in gaudi2_set_fixed_properties()
2400 prop->cache_line_size = DEVICE_CACHE_LINE_SIZE; in gaudi2_set_fixed_properties()
2401 prop->cfg_base_address = CFG_BASE; in gaudi2_set_fixed_properties()
2402 prop->device_dma_offset_for_host_access = HOST_PHYS_BASE_0; in gaudi2_set_fixed_properties()
2403 prop->host_base_address = HOST_PHYS_BASE_0; in gaudi2_set_fixed_properties()
2404 prop->host_end_address = prop->host_base_address + HOST_PHYS_SIZE_0; in gaudi2_set_fixed_properties()
2405 prop->max_pending_cs = GAUDI2_MAX_PENDING_CS; in gaudi2_set_fixed_properties()
2406 prop->completion_queues_count = GAUDI2_RESERVED_CQ_NUMBER; in gaudi2_set_fixed_properties()
2407 prop->user_dec_intr_count = NUMBER_OF_DEC; in gaudi2_set_fixed_properties()
2408 prop->user_interrupt_count = GAUDI2_IRQ_NUM_USER_LAST - GAUDI2_IRQ_NUM_USER_FIRST + 1; in gaudi2_set_fixed_properties()
2409 prop->completion_mode = HL_COMPLETION_MODE_CS; in gaudi2_set_fixed_properties()
2410 prop->sync_stream_first_sob = GAUDI2_RESERVED_SOB_NUMBER; in gaudi2_set_fixed_properties()
2411 prop->sync_stream_first_mon = GAUDI2_RESERVED_MON_NUMBER; in gaudi2_set_fixed_properties()
2413 prop->sram_base_address = SRAM_BASE_ADDR; in gaudi2_set_fixed_properties()
2414 prop->sram_size = SRAM_SIZE; in gaudi2_set_fixed_properties()
2415 prop->sram_end_address = prop->sram_base_address + prop->sram_size; in gaudi2_set_fixed_properties()
2416 prop->sram_user_base_address = prop->sram_base_address + SRAM_USER_BASE_OFFSET; in gaudi2_set_fixed_properties()
2418 prop->hints_range_reservation = true; in gaudi2_set_fixed_properties()
2420 prop->rotator_enabled_mask = BIT(NUM_OF_ROT) - 1; in gaudi2_set_fixed_properties()
2422 if (hdev->pldm) in gaudi2_set_fixed_properties()
2423 prop->mmu_pgt_size = 0x800000; /* 8MB */ in gaudi2_set_fixed_properties()
2425 prop->mmu_pgt_size = MMU_PAGE_TABLES_INITIAL_SIZE; in gaudi2_set_fixed_properties()
2427 prop->mmu_pte_size = HL_PTE_SIZE; in gaudi2_set_fixed_properties()
2428 prop->mmu_hop_table_size = HOP_TABLE_SIZE_512_PTE; in gaudi2_set_fixed_properties()
2429 prop->mmu_hop0_tables_total_size = HOP0_512_PTE_TABLES_TOTAL_SIZE; in gaudi2_set_fixed_properties()
2431 prop->dmmu.hop_shifts[MMU_HOP0] = DHOP0_SHIFT; in gaudi2_set_fixed_properties()
2432 prop->dmmu.hop_shifts[MMU_HOP1] = DHOP1_SHIFT; in gaudi2_set_fixed_properties()
2433 prop->dmmu.hop_shifts[MMU_HOP2] = DHOP2_SHIFT; in gaudi2_set_fixed_properties()
2434 prop->dmmu.hop_shifts[MMU_HOP3] = DHOP3_SHIFT; in gaudi2_set_fixed_properties()
2435 prop->dmmu.hop_shifts[MMU_HOP4] = DHOP4_SHIFT; in gaudi2_set_fixed_properties()
2436 prop->dmmu.hop_masks[MMU_HOP0] = DHOP0_MASK; in gaudi2_set_fixed_properties()
2437 prop->dmmu.hop_masks[MMU_HOP1] = DHOP1_MASK; in gaudi2_set_fixed_properties()
2438 prop->dmmu.hop_masks[MMU_HOP2] = DHOP2_MASK; in gaudi2_set_fixed_properties()
2439 prop->dmmu.hop_masks[MMU_HOP3] = DHOP3_MASK; in gaudi2_set_fixed_properties()
2440 prop->dmmu.hop_masks[MMU_HOP4] = DHOP4_MASK; in gaudi2_set_fixed_properties()
2441 prop->dmmu.page_size = PAGE_SIZE_1GB; in gaudi2_set_fixed_properties()
2442 prop->dmmu.num_hops = MMU_ARCH_6_HOPS; in gaudi2_set_fixed_properties()
2443 prop->dmmu.last_mask = LAST_MASK; in gaudi2_set_fixed_properties()
2444 prop->dmmu.host_resident = 1; in gaudi2_set_fixed_properties()
2445 prop->dmmu.hop_table_size = prop->mmu_hop_table_size; in gaudi2_set_fixed_properties()
2446 prop->dmmu.hop0_tables_total_size = prop->mmu_hop0_tables_total_size; in gaudi2_set_fixed_properties()
2454 prop->dram_size = (GAUDI2_HBM_NUM - 1) * SZ_16G; in gaudi2_set_fixed_properties()
2456 hdev->pmmu_huge_range = true; in gaudi2_set_fixed_properties()
2457 prop->pmmu.host_resident = 1; in gaudi2_set_fixed_properties()
2458 prop->pmmu.num_hops = MMU_ARCH_6_HOPS; in gaudi2_set_fixed_properties()
2459 prop->pmmu.last_mask = LAST_MASK; in gaudi2_set_fixed_properties()
2460 prop->pmmu.hop_table_size = prop->mmu_hop_table_size; in gaudi2_set_fixed_properties()
2461 prop->pmmu.hop0_tables_total_size = prop->mmu_hop0_tables_total_size; in gaudi2_set_fixed_properties()
2463 prop->hints_host_reserved_va_range.start_addr = RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START; in gaudi2_set_fixed_properties()
2464 prop->hints_host_reserved_va_range.end_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HOST_END; in gaudi2_set_fixed_properties()
2465 prop->hints_host_hpage_reserved_va_range.start_addr = in gaudi2_set_fixed_properties()
2467 prop->hints_host_hpage_reserved_va_range.end_addr = in gaudi2_set_fixed_properties()
2471 prop->pmmu.hop_shifts[MMU_HOP0] = HOP0_SHIFT_64K; in gaudi2_set_fixed_properties()
2472 prop->pmmu.hop_shifts[MMU_HOP1] = HOP1_SHIFT_64K; in gaudi2_set_fixed_properties()
2473 prop->pmmu.hop_shifts[MMU_HOP2] = HOP2_SHIFT_64K; in gaudi2_set_fixed_properties()
2474 prop->pmmu.hop_shifts[MMU_HOP3] = HOP3_SHIFT_64K; in gaudi2_set_fixed_properties()
2475 prop->pmmu.hop_shifts[MMU_HOP4] = HOP4_SHIFT_64K; in gaudi2_set_fixed_properties()
2476 prop->pmmu.hop_shifts[MMU_HOP5] = HOP5_SHIFT_64K; in gaudi2_set_fixed_properties()
2477 prop->pmmu.hop_masks[MMU_HOP0] = HOP0_MASK_64K; in gaudi2_set_fixed_properties()
2478 prop->pmmu.hop_masks[MMU_HOP1] = HOP1_MASK_64K; in gaudi2_set_fixed_properties()
2479 prop->pmmu.hop_masks[MMU_HOP2] = HOP2_MASK_64K; in gaudi2_set_fixed_properties()
2480 prop->pmmu.hop_masks[MMU_HOP3] = HOP3_MASK_64K; in gaudi2_set_fixed_properties()
2481 prop->pmmu.hop_masks[MMU_HOP4] = HOP4_MASK_64K; in gaudi2_set_fixed_properties()
2482 prop->pmmu.hop_masks[MMU_HOP5] = HOP5_MASK_64K; in gaudi2_set_fixed_properties()
2483 prop->pmmu.start_addr = VA_HOST_SPACE_PAGE_START; in gaudi2_set_fixed_properties()
2484 prop->pmmu.end_addr = VA_HOST_SPACE_PAGE_END; in gaudi2_set_fixed_properties()
2485 prop->pmmu.page_size = PAGE_SIZE_64KB; in gaudi2_set_fixed_properties()
2488 memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu)); in gaudi2_set_fixed_properties()
2489 prop->pmmu_huge.page_size = PAGE_SIZE_16MB; in gaudi2_set_fixed_properties()
2490 prop->pmmu_huge.start_addr = VA_HOST_SPACE_HPAGE_START; in gaudi2_set_fixed_properties()
2491 prop->pmmu_huge.end_addr = VA_HOST_SPACE_HPAGE_END; in gaudi2_set_fixed_properties()
2493 prop->pmmu.hop_shifts[MMU_HOP0] = HOP0_SHIFT_4K; in gaudi2_set_fixed_properties()
2494 prop->pmmu.hop_shifts[MMU_HOP1] = HOP1_SHIFT_4K; in gaudi2_set_fixed_properties()
2495 prop->pmmu.hop_shifts[MMU_HOP2] = HOP2_SHIFT_4K; in gaudi2_set_fixed_properties()
2496 prop->pmmu.hop_shifts[MMU_HOP3] = HOP3_SHIFT_4K; in gaudi2_set_fixed_properties()
2497 prop->pmmu.hop_shifts[MMU_HOP4] = HOP4_SHIFT_4K; in gaudi2_set_fixed_properties()
2498 prop->pmmu.hop_shifts[MMU_HOP5] = HOP5_SHIFT_4K; in gaudi2_set_fixed_properties()
2499 prop->pmmu.hop_masks[MMU_HOP0] = HOP0_MASK_4K; in gaudi2_set_fixed_properties()
2500 prop->pmmu.hop_masks[MMU_HOP1] = HOP1_MASK_4K; in gaudi2_set_fixed_properties()
2501 prop->pmmu.hop_masks[MMU_HOP2] = HOP2_MASK_4K; in gaudi2_set_fixed_properties()
2502 prop->pmmu.hop_masks[MMU_HOP3] = HOP3_MASK_4K; in gaudi2_set_fixed_properties()
2503 prop->pmmu.hop_masks[MMU_HOP4] = HOP4_MASK_4K; in gaudi2_set_fixed_properties()
2504 prop->pmmu.hop_masks[MMU_HOP5] = HOP5_MASK_4K; in gaudi2_set_fixed_properties()
2505 prop->pmmu.start_addr = VA_HOST_SPACE_PAGE_START; in gaudi2_set_fixed_properties()
2506 prop->pmmu.end_addr = VA_HOST_SPACE_PAGE_END; in gaudi2_set_fixed_properties()
2507 prop->pmmu.page_size = PAGE_SIZE_4KB; in gaudi2_set_fixed_properties()
2510 memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu)); in gaudi2_set_fixed_properties()
2511 prop->pmmu_huge.page_size = PAGE_SIZE_2MB; in gaudi2_set_fixed_properties()
2512 prop->pmmu_huge.start_addr = VA_HOST_SPACE_HPAGE_START; in gaudi2_set_fixed_properties()
2513 prop->pmmu_huge.end_addr = VA_HOST_SPACE_HPAGE_END; in gaudi2_set_fixed_properties()
2516 prop->max_num_of_engines = GAUDI2_ENGINE_ID_SIZE; in gaudi2_set_fixed_properties()
2517 prop->num_engine_cores = CPU_ID_MAX; in gaudi2_set_fixed_properties()
2518 prop->cfg_size = CFG_SIZE; in gaudi2_set_fixed_properties()
2519 prop->max_asid = MAX_ASID; in gaudi2_set_fixed_properties()
2520 prop->num_of_events = GAUDI2_EVENT_SIZE; in gaudi2_set_fixed_properties()
2522 prop->supports_engine_modes = true; in gaudi2_set_fixed_properties()
2524 prop->dc_power_default = DC_POWER_DEFAULT; in gaudi2_set_fixed_properties()
2526 prop->cb_pool_cb_cnt = GAUDI2_CB_POOL_CB_CNT; in gaudi2_set_fixed_properties()
2527 prop->cb_pool_cb_size = GAUDI2_CB_POOL_CB_SIZE; in gaudi2_set_fixed_properties()
2528 prop->pcie_dbi_base_address = CFG_BASE + mmPCIE_DBI_BASE; in gaudi2_set_fixed_properties()
2529 prop->pcie_aux_dbi_reg_addr = CFG_BASE + mmPCIE_AUX_DBI; in gaudi2_set_fixed_properties()
2531 strscpy_pad(prop->cpucp_info.card_name, GAUDI2_DEFAULT_CARD_NAME, CARD_NAME_MAX_LEN); in gaudi2_set_fixed_properties()
2533 prop->mme_master_slave_mode = 1; in gaudi2_set_fixed_properties()
2535 prop->first_available_user_sob[0] = GAUDI2_RESERVED_SOB_NUMBER + in gaudi2_set_fixed_properties()
2538 prop->first_available_user_mon[0] = GAUDI2_RESERVED_MON_NUMBER + in gaudi2_set_fixed_properties()
2541 prop->first_available_user_interrupt = GAUDI2_IRQ_NUM_USER_FIRST; in gaudi2_set_fixed_properties()
2542 prop->tpc_interrupt_id = GAUDI2_IRQ_NUM_TPC_ASSERT; in gaudi2_set_fixed_properties()
2543 prop->eq_interrupt_id = GAUDI2_IRQ_NUM_EVENT_QUEUE; in gaudi2_set_fixed_properties()
2545 prop->first_available_cq[0] = GAUDI2_RESERVED_CQ_NUMBER; in gaudi2_set_fixed_properties()
2547 prop->fw_cpu_boot_dev_sts0_valid = false; in gaudi2_set_fixed_properties()
2548 prop->fw_cpu_boot_dev_sts1_valid = false; in gaudi2_set_fixed_properties()
2549 prop->hard_reset_done_by_fw = false; in gaudi2_set_fixed_properties()
2550 prop->gic_interrupts_enable = true; in gaudi2_set_fixed_properties()
2552 prop->server_type = HL_SERVER_TYPE_UNKNOWN; in gaudi2_set_fixed_properties()
2554 prop->max_dec = NUMBER_OF_DEC; in gaudi2_set_fixed_properties()
2556 prop->clk_pll_index = HL_GAUDI2_MME_PLL; in gaudi2_set_fixed_properties()
2558 prop->dma_mask = 64; in gaudi2_set_fixed_properties()
2560 prop->hbw_flush_reg = mmPCIE_WRAP_SPECIAL_GLBL_SPARE_0; in gaudi2_set_fixed_properties()
2575 hdev->rmmio = hdev->pcie_bar[SRAM_CFG_BAR_ID] + (CFG_BASE - STM_FLASH_BASE_ADDR); in gaudi2_pci_bars_map()
2582 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_set_hbm_bar_base()
2587 if ((gaudi2) && (gaudi2->dram_bar_cur_addr == addr)) in gaudi2_set_hbm_bar_base()
2590 if (hdev->asic_prop.iatu_done_by_fw) in gaudi2_set_hbm_bar_base()
2593 /* Inbound Region 2 - Bar 4 - Point to DRAM */ in gaudi2_set_hbm_bar_base()
2602 old_addr = gaudi2->dram_bar_cur_addr; in gaudi2_set_hbm_bar_base()
2603 gaudi2->dram_bar_cur_addr = addr; in gaudi2_set_hbm_bar_base()
2616 if (hdev->asic_prop.iatu_done_by_fw) in gaudi2_init_iatu()
2619 /* Temporary inbound Region 0 - Bar 0 - Point to CFG in gaudi2_init_iatu()
2626 inbound_region.addr = STM_FLASH_BASE_ADDR - STM_FLASH_ALIGNED_OFF; in gaudi2_init_iatu()
2635 hdev->pcie_bar_phys[SRAM_CFG_BAR_ID] = (u64)bar_addr_high << 32 | bar_addr_low; in gaudi2_init_iatu()
2637 /* Inbound Region 0 - Bar 0 - Point to CFG */ in gaudi2_init_iatu()
2647 /* Inbound Region 1 - Bar 0 - Point to BAR0_RESERVED + SRAM */ in gaudi2_init_iatu()
2653 rc = hl_pci_set_inbound_region(hdev, 1, &inbound_region); in gaudi2_init_iatu()
2657 /* Inbound Region 2 - Bar 4 - Point to DRAM */ in gaudi2_init_iatu()
2665 /* Outbound Region 0 - Point to Host */ in gaudi2_init_iatu()
2680 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_tpc_binning_init_prop()
2686 if (hweight64(hdev->tpc_binning) > MAX_CLUSTER_BINNING_FAULTY_TPCS) { in gaudi2_tpc_binning_init_prop()
2687 dev_err(hdev->dev, "TPC binning is supported for max of %d faulty TPCs, provided mask 0x%llx\n", in gaudi2_tpc_binning_init_prop()
2689 hdev->tpc_binning); in gaudi2_tpc_binning_init_prop()
2690 return -EINVAL; in gaudi2_tpc_binning_init_prop()
2693 prop->tpc_binning_mask = hdev->tpc_binning; in gaudi2_tpc_binning_init_prop()
2694 prop->tpc_enabled_mask = GAUDI2_TPC_FULL_MASK; in gaudi2_tpc_binning_init_prop()
2701 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_set_tpc_binning_masks()
2702 struct hw_queue_properties *q_props = prop->hw_queues_props; in gaudi2_set_tpc_binning_masks()
2711 tpc_binning_mask = prop->tpc_binning_mask; in gaudi2_set_tpc_binning_masks()
2731 * Coverity complains about possible out-of-bound access in in gaudi2_set_tpc_binning_masks()
2735 dev_err(hdev->dev, in gaudi2_set_tpc_binning_masks()
2738 return -EINVAL; in gaudi2_set_tpc_binning_masks()
2743 clear_bit(subst_seq, (unsigned long *)&prop->tpc_enabled_mask); in gaudi2_set_tpc_binning_masks()
2746 q_props[qid_base].binned = 1; in gaudi2_set_tpc_binning_masks()
2747 q_props[qid_base + 1].binned = 1; in gaudi2_set_tpc_binning_masks()
2748 q_props[qid_base + 2].binned = 1; in gaudi2_set_tpc_binning_masks()
2749 q_props[qid_base + 3].binned = 1; in gaudi2_set_tpc_binning_masks()
2759 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_set_dec_binning_masks()
2762 num_faulty = hweight32(hdev->decoder_binning); in gaudi2_set_dec_binning_masks()
2769 …dev_err(hdev->dev, "decoder binning is supported for max of single faulty decoder, provided mask 0… in gaudi2_set_dec_binning_masks()
2770 hdev->decoder_binning); in gaudi2_set_dec_binning_masks()
2771 return -EINVAL; in gaudi2_set_dec_binning_masks()
2774 prop->decoder_binning_mask = (hdev->decoder_binning & GAUDI2_DECODER_FULL_MASK); in gaudi2_set_dec_binning_masks()
2776 if (prop->decoder_binning_mask) in gaudi2_set_dec_binning_masks()
2777 prop->decoder_enabled_mask = (GAUDI2_DECODER_FULL_MASK & ~BIT(DEC_ID_PCIE_VDEC1)); in gaudi2_set_dec_binning_masks()
2779 prop->decoder_enabled_mask = GAUDI2_DECODER_FULL_MASK; in gaudi2_set_dec_binning_masks()
2786 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_set_dram_binning_masks()
2789 if (!hdev->dram_binning) { in gaudi2_set_dram_binning_masks()
2790 prop->dram_binning_mask = 0; in gaudi2_set_dram_binning_masks()
2791 prop->dram_enabled_mask = GAUDI2_DRAM_FULL_MASK; in gaudi2_set_dram_binning_masks()
2796 prop->faulty_dram_cluster_map |= hdev->dram_binning; in gaudi2_set_dram_binning_masks()
2797 prop->dram_binning_mask = hdev->dram_binning; in gaudi2_set_dram_binning_masks()
2798 prop->dram_enabled_mask = GAUDI2_DRAM_FULL_MASK & ~BIT(HBM_ID5); in gaudi2_set_dram_binning_masks()
2803 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_set_edma_binning_masks()
2807 num_faulty = hweight32(hdev->edma_binning); in gaudi2_set_edma_binning_masks()
2814 dev_err(hdev->dev, in gaudi2_set_edma_binning_masks()
2816 hdev->edma_binning); in gaudi2_set_edma_binning_masks()
2817 return -EINVAL; in gaudi2_set_edma_binning_masks()
2820 if (!hdev->edma_binning) { in gaudi2_set_edma_binning_masks()
2821 prop->edma_binning_mask = 0; in gaudi2_set_edma_binning_masks()
2822 prop->edma_enabled_mask = GAUDI2_EDMA_FULL_MASK; in gaudi2_set_edma_binning_masks()
2826 seq = __ffs((unsigned long)hdev->edma_binning); in gaudi2_set_edma_binning_masks()
2829 prop->faulty_dram_cluster_map |= BIT(edma_to_hbm_cluster[seq]); in gaudi2_set_edma_binning_masks()
2830 prop->edma_binning_mask = hdev->edma_binning; in gaudi2_set_edma_binning_masks()
2831 prop->edma_enabled_mask = GAUDI2_EDMA_FULL_MASK & ~BIT(EDMA_ID_DCORE3_INSTANCE1); in gaudi2_set_edma_binning_masks()
2834 q_props = prop->hw_queues_props; in gaudi2_set_edma_binning_masks()
2835 q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_0].binned = 1; in gaudi2_set_edma_binning_masks()
2836 q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_1].binned = 1; in gaudi2_set_edma_binning_masks()
2837 q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_2].binned = 1; in gaudi2_set_edma_binning_masks()
2838 q_props[GAUDI2_QUEUE_ID_DCORE3_EDMA_1_3].binned = 1; in gaudi2_set_edma_binning_masks()
2845 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_set_xbar_edge_enable_mask()
2850 prop->xbar_edge_enabled_mask = GAUDI2_XBAR_EDGE_FULL_MASK; in gaudi2_set_xbar_edge_enable_mask()
2865 dev_err(hdev->dev, "we cannot have more than %d faulty XBAR EDGE\n", in gaudi2_set_xbar_edge_enable_mask()
2867 return -EINVAL; in gaudi2_set_xbar_edge_enable_mask()
2873 prop->faulty_dram_cluster_map |= BIT(xbar_edge_to_hbm_cluster[seq]); in gaudi2_set_xbar_edge_enable_mask()
2874 prop->xbar_edge_enabled_mask = (~xbar_edge_iso_mask) & GAUDI2_XBAR_EDGE_FULL_MASK; in gaudi2_set_xbar_edge_enable_mask()
2886 * If more than single cluster is faulty- the chip is unusable in gaudi2_set_cluster_binning_masks_common()
2888 hdev->asic_prop.faulty_dram_cluster_map = 0; in gaudi2_set_cluster_binning_masks_common()
2902 hdev->asic_prop.hmmu_hif_enabled_mask = GAUDI2_HIF_HMMU_FULL_MASK; in gaudi2_set_cluster_binning_masks_common()
2909 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_set_cluster_binning_masks()
2912 rc = gaudi2_set_cluster_binning_masks_common(hdev, prop->cpucp_info.xbar_binning_mask); in gaudi2_set_cluster_binning_masks()
2917 if (prop->faulty_dram_cluster_map) { in gaudi2_set_cluster_binning_masks()
2918 u8 cluster_seq = __ffs((unsigned long)prop->faulty_dram_cluster_map); in gaudi2_set_cluster_binning_masks()
2920 prop->hmmu_hif_enabled_mask = cluster_hmmu_hif_enabled_mask[cluster_seq]; in gaudi2_set_cluster_binning_masks()
2947 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_cpucp_info_get()
2948 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_cpucp_info_get()
2953 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) in gaudi2_cpucp_info_get()
2959 if (hdev->reset_info.in_compute_reset) in gaudi2_cpucp_info_get()
2967 dram_size = le64_to_cpu(prop->cpucp_info.dram_size); in gaudi2_cpucp_info_get()
2971 if ((dram_size != ((GAUDI2_HBM_NUM - 1) * SZ_16G)) && in gaudi2_cpucp_info_get()
2973 dev_err(hdev->dev, in gaudi2_cpucp_info_get()
2975 dram_size, prop->dram_size); in gaudi2_cpucp_info_get()
2976 dram_size = prop->dram_size; in gaudi2_cpucp_info_get()
2979 prop->dram_size = dram_size; in gaudi2_cpucp_info_get()
2980 prop->dram_end_address = prop->dram_base_address + dram_size; in gaudi2_cpucp_info_get()
2983 if (!strlen(prop->cpucp_info.card_name)) in gaudi2_cpucp_info_get()
2984 strscpy_pad(prop->cpucp_info.card_name, GAUDI2_DEFAULT_CARD_NAME, in gaudi2_cpucp_info_get()
2988 hdev->dram_binning = prop->cpucp_info.dram_binning_mask; in gaudi2_cpucp_info_get()
2989 hdev->edma_binning = prop->cpucp_info.edma_binning_mask; in gaudi2_cpucp_info_get()
2990 hdev->tpc_binning = le64_to_cpu(prop->cpucp_info.tpc_binning_mask); in gaudi2_cpucp_info_get()
2991 hdev->decoder_binning = lower_32_bits(le64_to_cpu(prop->cpucp_info.decoder_binning_mask)); in gaudi2_cpucp_info_get()
2993 dev_dbg(hdev->dev, "Read binning masks: tpc: 0x%llx, dram: 0x%llx, edma: 0x%x, dec: 0x%x\n", in gaudi2_cpucp_info_get()
2994 hdev->tpc_binning, hdev->dram_binning, hdev->edma_binning, in gaudi2_cpucp_info_get()
2995 hdev->decoder_binning); in gaudi2_cpucp_info_get()
3001 rc = hdev->asic_funcs->set_dram_properties(hdev); in gaudi2_cpucp_info_get()
3005 rc = hdev->asic_funcs->set_binning_masks(hdev); in gaudi2_cpucp_info_get()
3013 prop->max_power_default = (u64) max_power; in gaudi2_cpucp_info_get()
3020 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_fetch_psoc_frequency()
3024 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) in gaudi2_fetch_psoc_frequency()
3031 hdev->asic_prop.psoc_timestamp_frequency = pll_freq_arr[3]; in gaudi2_fetch_psoc_frequency()
3038 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_early_init()
3039 struct pci_dev *pdev = hdev->pdev; in gaudi2_early_init()
3051 dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n", in gaudi2_early_init()
3053 rc = -ENODEV; in gaudi2_early_init()
3059 dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n", in gaudi2_early_init()
3061 rc = -ENODEV; in gaudi2_early_init()
3065 prop->dram_pci_bar_size = pci_resource_len(pdev, DRAM_BAR_ID); in gaudi2_early_init()
3066 hdev->dram_pci_bar_start = pci_resource_start(pdev, DRAM_BAR_ID); in gaudi2_early_init()
3071 if (hdev->pldm) in gaudi2_early_init()
3072 hdev->asic_prop.iatu_done_by_fw = false; in gaudi2_early_init()
3074 hdev->asic_prop.iatu_done_by_fw = true; in gaudi2_early_init()
3081 * version to determine whether we run with a security-enabled firmware in gaudi2_early_init()
3085 if (hdev->reset_on_preboot_fail) in gaudi2_early_init()
3087 hdev->asic_funcs->hw_fini(hdev, true, false); in gaudi2_early_init()
3092 dev_dbg(hdev->dev, "H/W state is dirty, must reset before initializing\n"); in gaudi2_early_init()
3093 rc = hdev->asic_funcs->hw_fini(hdev, true, false); in gaudi2_early_init()
3095 dev_err(hdev->dev, "failed to reset HW in dirty state (%d)\n", rc); in gaudi2_early_init()
3105 kfree(hdev->asic_prop.hw_queues_props); in gaudi2_early_init()
3111 kfree(hdev->asic_prop.hw_queues_props); in gaudi2_early_fini()
3139 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi2_init_arcs()
3140 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_arcs()
3160 !(hdev->nic_ports_mask & BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0))) in gaudi2_init_arcs()
3163 if (gaudi2_is_arc_tpc_owned(arc_id) && !(gaudi2->tpc_hw_cap_initialized & in gaudi2_init_arcs()
3164 BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0))) in gaudi2_init_arcs()
3171 hdev->asic_prop.engine_core_interrupt_reg_addr = in gaudi2_init_arcs()
3172 CFG_BASE + le32_to_cpu(dyn_regs->eng_arc_irq_ctrl); in gaudi2_init_arcs()
3201 reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_MME_ARC_UPPER_DCCM_EN_VAL_MASK, 1); in gaudi2_scrub_arc_dccm()
3243 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_late_init()
3246 hdev->asic_prop.supports_advanced_cpucp_rc = true; in gaudi2_late_init()
3249 gaudi2->virt_msix_db_dma_addr); in gaudi2_late_init()
3251 dev_err(hdev->dev, "Failed to enable PCI access from CPU\n"); in gaudi2_late_init()
3257 dev_err(hdev->dev, "Failed to fetch psoc frequency\n"); in gaudi2_late_init()
3265 dev_err(hdev->dev, "Failed to scrub arcs DCCM\n"); in gaudi2_late_init()
3286 struct user_mapped_block *blocks = gaudi2->mapped_blocks; in gaudi2_user_mapped_dec_init()
3302 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_user_mapped_blocks_init()
3303 struct user_mapped_block *blocks = gaudi2->mapped_blocks; in gaudi2_user_mapped_blocks_init()
3320 blocks[NUM_ARC_CPUS + 1].address = mmARC_FARM_ARC1_ACP_ENG_BASE; in gaudi2_user_mapped_blocks_init()
3321 blocks[NUM_ARC_CPUS + 1].size = HL_BLOCK_SIZE; in gaudi2_user_mapped_blocks_init()
3360 for (i = 1; i < NUM_OF_DCORES; ++i) { in gaudi2_user_mapped_blocks_init()
3361 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1)].size = SM_OBJS_BLOCK_SIZE; in gaudi2_user_mapped_blocks_init()
3362 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1) + 1].size = HL_BLOCK_SIZE; in gaudi2_user_mapped_blocks_init()
3364 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1)].address = in gaudi2_user_mapped_blocks_init()
3367 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1) + 1].address = in gaudi2_user_mapped_blocks_init()
3378 /* The device ARC works with 32-bits addresses, and because there is a single HW register in gaudi2_alloc_cpu_accessible_dma_mem()
3387 rc = -ENOMEM; in gaudi2_alloc_cpu_accessible_dma_mem()
3391 end_addr = dma_addr_arr[i] + HL_CPU_ACCESSIBLE_MEM_SIZE - 1; in gaudi2_alloc_cpu_accessible_dma_mem()
3397 dev_err(hdev->dev, in gaudi2_alloc_cpu_accessible_dma_mem()
3399 rc = -EFAULT; in gaudi2_alloc_cpu_accessible_dma_mem()
3403 hdev->cpu_accessible_dma_mem = virt_addr_arr[i]; in gaudi2_alloc_cpu_accessible_dma_mem()
3404 hdev->cpu_accessible_dma_address = dma_addr_arr[i]; in gaudi2_alloc_cpu_accessible_dma_mem()
3416 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_set_pci_memory_regions()
3420 region = &hdev->pci_mem_region[PCI_REGION_CFG]; in gaudi2_set_pci_memory_regions()
3421 region->region_base = CFG_BASE; in gaudi2_set_pci_memory_regions()
3422 region->region_size = CFG_SIZE; in gaudi2_set_pci_memory_regions()
3423 region->offset_in_bar = CFG_BASE - STM_FLASH_BASE_ADDR; in gaudi2_set_pci_memory_regions()
3424 region->bar_size = CFG_BAR_SIZE; in gaudi2_set_pci_memory_regions()
3425 region->bar_id = SRAM_CFG_BAR_ID; in gaudi2_set_pci_memory_regions()
3426 region->used = 1; in gaudi2_set_pci_memory_regions()
3429 region = &hdev->pci_mem_region[PCI_REGION_SRAM]; in gaudi2_set_pci_memory_regions()
3430 region->region_base = SRAM_BASE_ADDR; in gaudi2_set_pci_memory_regions()
3431 region->region_size = SRAM_SIZE; in gaudi2_set_pci_memory_regions()
3432 region->offset_in_bar = CFG_REGION_SIZE + BAR0_RSRVD_SIZE; in gaudi2_set_pci_memory_regions()
3433 region->bar_size = CFG_BAR_SIZE; in gaudi2_set_pci_memory_regions()
3434 region->bar_id = SRAM_CFG_BAR_ID; in gaudi2_set_pci_memory_regions()
3435 region->used = 1; in gaudi2_set_pci_memory_regions()
3438 region = &hdev->pci_mem_region[PCI_REGION_DRAM]; in gaudi2_set_pci_memory_regions()
3439 region->region_base = DRAM_PHYS_BASE; in gaudi2_set_pci_memory_regions()
3440 region->region_size = hdev->asic_prop.dram_size; in gaudi2_set_pci_memory_regions()
3441 region->offset_in_bar = 0; in gaudi2_set_pci_memory_regions()
3442 region->bar_size = prop->dram_pci_bar_size; in gaudi2_set_pci_memory_regions()
3443 region->bar_id = DRAM_BAR_ID; in gaudi2_set_pci_memory_regions()
3444 region->used = 1; in gaudi2_set_pci_memory_regions()
3449 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_user_interrupt_setup()
3453 HL_USR_INTR_STRUCT_INIT(hdev->tpc_interrupt, hdev, 0, HL_USR_INTERRUPT_TPC); in gaudi2_user_interrupt_setup()
3456 HL_USR_INTR_STRUCT_INIT(hdev->unexpected_error_interrupt, hdev, 0, in gaudi2_user_interrupt_setup()
3460 HL_USR_INTR_STRUCT_INIT(hdev->common_user_cq_interrupt, hdev, in gaudi2_user_interrupt_setup()
3464 HL_USR_INTR_STRUCT_INIT(hdev->common_decoder_interrupt, hdev, in gaudi2_user_interrupt_setup()
3478 HL_USR_INTR_STRUCT_INIT(hdev->user_interrupt[j], hdev, i, in gaudi2_user_interrupt_setup()
3481 for (i = GAUDI2_IRQ_NUM_USER_FIRST, k = 0 ; k < prop->user_interrupt_count; i++, j++, k++) in gaudi2_user_interrupt_setup()
3482 HL_USR_INTR_STRUCT_INIT(hdev->user_interrupt[j], hdev, i, HL_USR_INTERRUPT_CQ); in gaudi2_user_interrupt_setup()
3489 return rand ? rand : 1; in gaudi2_get_non_zero_random_int()
3494 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_special_blocks_free()
3496 &prop->skip_special_blocks_cfg; in gaudi2_special_blocks_free()
3498 kfree(prop->special_blocks); in gaudi2_special_blocks_free()
3499 kfree(skip_special_blocks_cfg->block_types); in gaudi2_special_blocks_free()
3500 kfree(skip_special_blocks_cfg->block_ranges); in gaudi2_special_blocks_free()
3517 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_special_blocks_config()
3521 prop->glbl_err_cause_num = GAUDI2_NUM_OF_GLBL_ERR_CAUSE; in gaudi2_special_blocks_config()
3522 prop->num_of_special_blocks = ARRAY_SIZE(gaudi2_special_blocks); in gaudi2_special_blocks_config()
3523 prop->special_blocks = kmalloc_array(prop->num_of_special_blocks, in gaudi2_special_blocks_config()
3524 sizeof(*prop->special_blocks), GFP_KERNEL); in gaudi2_special_blocks_config()
3525 if (!prop->special_blocks) in gaudi2_special_blocks_config()
3526 return -ENOMEM; in gaudi2_special_blocks_config()
3528 for (i = 0 ; i < prop->num_of_special_blocks ; i++) in gaudi2_special_blocks_config()
3529 memcpy(&prop->special_blocks[i], &gaudi2_special_blocks[i], in gaudi2_special_blocks_config()
3530 sizeof(*prop->special_blocks)); in gaudi2_special_blocks_config()
3533 memset(&prop->skip_special_blocks_cfg, 0, sizeof(prop->skip_special_blocks_cfg)); in gaudi2_special_blocks_config()
3534 prop->skip_special_blocks_cfg.skip_block_hook = gaudi2_special_block_skip; in gaudi2_special_blocks_config()
3537 prop->skip_special_blocks_cfg.block_types = in gaudi2_special_blocks_config()
3540 if (!prop->skip_special_blocks_cfg.block_types) { in gaudi2_special_blocks_config()
3541 rc = -ENOMEM; in gaudi2_special_blocks_config()
3545 memcpy(prop->skip_special_blocks_cfg.block_types, gaudi2_iterator_skip_block_types, in gaudi2_special_blocks_config()
3548 prop->skip_special_blocks_cfg.block_types_len = in gaudi2_special_blocks_config()
3553 prop->skip_special_blocks_cfg.block_ranges = in gaudi2_special_blocks_config()
3556 if (!prop->skip_special_blocks_cfg.block_ranges) { in gaudi2_special_blocks_config()
3557 rc = -ENOMEM; in gaudi2_special_blocks_config()
3562 memcpy(&prop->skip_special_blocks_cfg.block_ranges[i], in gaudi2_special_blocks_config()
3566 prop->skip_special_blocks_cfg.block_ranges_len = in gaudi2_special_blocks_config()
3573 kfree(prop->skip_special_blocks_cfg.block_types); in gaudi2_special_blocks_config()
3575 kfree(prop->special_blocks); in gaudi2_special_blocks_config()
3587 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_test_queues_msgs_free()
3588 struct gaudi2_queues_test_info *msg_info = gaudi2->queues_test_info; in gaudi2_test_queues_msgs_free()
3592 /* bail-out if this is an allocation failure point */ in gaudi2_test_queues_msgs_free()
3603 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_test_queues_msgs_alloc()
3604 struct gaudi2_queues_test_info *msg_info = gaudi2->queues_test_info; in gaudi2_test_queues_msgs_alloc()
3607 /* allocate a message-short buf for each Q we intend to test */ in gaudi2_test_queues_msgs_alloc()
3613 dev_err(hdev->dev, in gaudi2_test_queues_msgs_alloc()
3615 rc = -ENOMEM; in gaudi2_test_queues_msgs_alloc()
3629 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_sw_init()
3636 return -ENOMEM; in gaudi2_sw_init()
3642 if (gaudi2->num_of_valid_hw_events == GAUDI2_EVENT_SIZE) { in gaudi2_sw_init()
3643 dev_err(hdev->dev, "H/W events array exceeds the limit of %u events\n", in gaudi2_sw_init()
3645 rc = -EINVAL; in gaudi2_sw_init()
3649 gaudi2->hw_events[gaudi2->num_of_valid_hw_events++] = gaudi2_irq_map_table[i].fc_id; in gaudi2_sw_init()
3653 gaudi2->lfsr_rand_seeds[i] = gaudi2_get_non_zero_random_int(); in gaudi2_sw_init()
3655 gaudi2->cpucp_info_get = gaudi2_cpucp_info_get; in gaudi2_sw_init()
3657 hdev->asic_specific = gaudi2; in gaudi2_sw_init()
3660 * Use DEVICE_CACHE_LINE_SIZE for alignment since the NIC memory-mapped in gaudi2_sw_init()
3663 hdev->dma_pool = dma_pool_create(dev_name(hdev->dev), &hdev->pdev->dev, in gaudi2_sw_init()
3665 if (!hdev->dma_pool) { in gaudi2_sw_init()
3666 dev_err(hdev->dev, "failed to create DMA pool\n"); in gaudi2_sw_init()
3667 rc = -ENOMEM; in gaudi2_sw_init()
3675 hdev->cpu_accessible_dma_pool = gen_pool_create(ilog2(32), -1); in gaudi2_sw_init()
3676 if (!hdev->cpu_accessible_dma_pool) { in gaudi2_sw_init()
3677 dev_err(hdev->dev, "Failed to create CPU accessible DMA pool\n"); in gaudi2_sw_init()
3678 rc = -ENOMEM; in gaudi2_sw_init()
3682 rc = gen_pool_add(hdev->cpu_accessible_dma_pool, (uintptr_t) hdev->cpu_accessible_dma_mem, in gaudi2_sw_init()
3683 HL_CPU_ACCESSIBLE_MEM_SIZE, -1); in gaudi2_sw_init()
3685 dev_err(hdev->dev, "Failed to add memory to CPU accessible DMA pool\n"); in gaudi2_sw_init()
3686 rc = -EFAULT; in gaudi2_sw_init()
3690 gaudi2->virt_msix_db_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev, prop->pmmu.page_size, in gaudi2_sw_init()
3691 &gaudi2->virt_msix_db_dma_addr); in gaudi2_sw_init()
3692 if (!gaudi2->virt_msix_db_cpu_addr) { in gaudi2_sw_init()
3693 dev_err(hdev->dev, "Failed to allocate DMA memory for virtual MSI-X doorbell\n"); in gaudi2_sw_init()
3694 rc = -ENOMEM; in gaudi2_sw_init()
3698 spin_lock_init(&gaudi2->hw_queues_lock); in gaudi2_sw_init()
3700 gaudi2->scratchpad_kernel_address = hl_asic_dma_alloc_coherent(hdev, PAGE_SIZE, in gaudi2_sw_init()
3701 &gaudi2->scratchpad_bus_address, in gaudi2_sw_init()
3703 if (!gaudi2->scratchpad_kernel_address) { in gaudi2_sw_init()
3704 rc = -ENOMEM; in gaudi2_sw_init()
3713 hdev->supports_coresight = true; in gaudi2_sw_init()
3714 hdev->supports_sync_stream = true; in gaudi2_sw_init()
3715 hdev->supports_cb_mapping = true; in gaudi2_sw_init()
3716 hdev->supports_wait_for_multi_cs = false; in gaudi2_sw_init()
3718 prop->supports_compute_reset = true; in gaudi2_sw_init()
3721 if (hl_is_fw_sw_ver_below(hdev, 1, 11)) in gaudi2_sw_init()
3722 hdev->event_queue.check_eqe_index = false; in gaudi2_sw_init()
3724 hdev->event_queue.check_eqe_index = true; in gaudi2_sw_init()
3726 hdev->asic_funcs->set_pci_memory_regions(hdev); in gaudi2_sw_init()
3741 hl_asic_dma_free_coherent(hdev, PAGE_SIZE, gaudi2->scratchpad_kernel_address, in gaudi2_sw_init()
3742 gaudi2->scratchpad_bus_address); in gaudi2_sw_init()
3744 hl_cpu_accessible_dma_pool_free(hdev, prop->pmmu.page_size, gaudi2->virt_msix_db_cpu_addr); in gaudi2_sw_init()
3746 gen_pool_destroy(hdev->cpu_accessible_dma_pool); in gaudi2_sw_init()
3748 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem, in gaudi2_sw_init()
3749 hdev->cpu_accessible_dma_address); in gaudi2_sw_init()
3751 dma_pool_destroy(hdev->dma_pool); in gaudi2_sw_init()
3759 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_sw_fini()
3760 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_sw_fini()
3766 hl_cpu_accessible_dma_pool_free(hdev, prop->pmmu.page_size, gaudi2->virt_msix_db_cpu_addr); in gaudi2_sw_fini()
3768 gen_pool_destroy(hdev->cpu_accessible_dma_pool); in gaudi2_sw_fini()
3770 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem, in gaudi2_sw_fini()
3771 hdev->cpu_accessible_dma_address); in gaudi2_sw_fini()
3773 hl_asic_dma_free_coherent(hdev, PAGE_SIZE, gaudi2->scratchpad_kernel_address, in gaudi2_sw_fini()
3774 gaudi2->scratchpad_bus_address); in gaudi2_sw_fini()
3776 dma_pool_destroy(hdev->dma_pool); in gaudi2_sw_fini()
3806 * gaudi2_clear_qm_fence_counters_common - clear QM's fence counters
3823 size = mmPDMA0_QM_CP_BARRIER_CFG - mmPDMA0_QM_CP_FENCE0_CNT_0; in gaudi2_clear_qm_fence_counters_common()
3846 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_stop_dma_qmans()
3849 if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK)) in gaudi2_stop_dma_qmans()
3857 if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK)) in gaudi2_stop_dma_qmans()
3865 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq))) in gaudi2_stop_dma_qmans()
3879 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_stop_mme_qmans()
3882 offset = mmDCORE1_MME_QM_BASE - mmDCORE0_MME_QM_BASE; in gaudi2_stop_mme_qmans()
3885 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i))) in gaudi2_stop_mme_qmans()
3894 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_stop_tpc_qmans()
3898 if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK)) in gaudi2_stop_tpc_qmans()
3902 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i))) in gaudi2_stop_tpc_qmans()
3912 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_stop_rot_qmans()
3916 if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK)) in gaudi2_stop_rot_qmans()
3920 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i))) in gaudi2_stop_rot_qmans()
3930 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_stop_nic_qmans()
3934 if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK)) in gaudi2_stop_nic_qmans()
3940 if (!(hdev->nic_ports_mask & BIT(i))) in gaudi2_stop_nic_qmans()
3958 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_dma_stall()
3961 if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK)) in gaudi2_dma_stall()
3968 if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK)) in gaudi2_dma_stall()
3976 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq))) in gaudi2_dma_stall()
3990 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_mme_stall()
3993 offset = mmDCORE1_MME_CTRL_LO_QM_STALL - mmDCORE0_MME_CTRL_LO_QM_STALL; in gaudi2_mme_stall()
3996 if (gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i)) in gaudi2_mme_stall()
3997 WREG32(mmDCORE0_MME_CTRL_LO_QM_STALL + (i * offset), 1); in gaudi2_mme_stall()
4002 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_tpc_stall()
4006 if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK)) in gaudi2_tpc_stall()
4010 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i))) in gaudi2_tpc_stall()
4014 WREG32(reg_base + TPC_CFG_STALL_OFFSET, 1); in gaudi2_tpc_stall()
4020 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_rotator_stall()
4024 if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK)) in gaudi2_rotator_stall()
4032 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i))) in gaudi2_rotator_stall()
4046 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_disable_dma_qmans()
4049 if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK)) in gaudi2_disable_dma_qmans()
4056 if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK)) in gaudi2_disable_dma_qmans()
4064 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq))) in gaudi2_disable_dma_qmans()
4078 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_disable_mme_qmans()
4081 offset = mmDCORE1_MME_QM_BASE - mmDCORE0_MME_QM_BASE; in gaudi2_disable_mme_qmans()
4084 if (gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i)) in gaudi2_disable_mme_qmans()
4090 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_disable_tpc_qmans()
4094 if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK)) in gaudi2_disable_tpc_qmans()
4098 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i))) in gaudi2_disable_tpc_qmans()
4108 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_disable_rot_qmans()
4112 if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK)) in gaudi2_disable_rot_qmans()
4116 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i))) in gaudi2_disable_rot_qmans()
4126 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_disable_nic_qmans()
4130 if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK)) in gaudi2_disable_nic_qmans()
4136 if (!(hdev->nic_ports_mask & BIT(i))) in gaudi2_disable_nic_qmans()
4149 /* Zero the lower/upper parts of the 64-bit counter */ in gaudi2_enable_timestamp()
4154 WREG32(mmPSOC_TIMESTAMP_BASE, 1); in gaudi2_enable_timestamp()
4171 return gaudi2_vdec_irq_name[irq_number - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM]; in gaudi2_irq_name()
4191 irq = pci_irq_vector(hdev->pdev, i); in gaudi2_dec_disable_msix()
4192 relative_idx = i - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM; in gaudi2_dec_disable_msix()
4194 dec = hdev->dec + relative_idx / 2; in gaudi2_dec_disable_msix()
4202 (void *) &hdev->user_interrupt[dec->core_id])); in gaudi2_dec_disable_msix()
4215 irq = pci_irq_vector(hdev->pdev, i); in gaudi2_dec_enable_msix()
4216 relative_idx = i - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM; in gaudi2_dec_enable_msix()
4225 dec = hdev->dec + relative_idx / 2; in gaudi2_dec_enable_msix()
4231 (void *) &hdev->user_interrupt[dec->core_id]); in gaudi2_dec_enable_msix()
4235 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in gaudi2_dec_enable_msix()
4249 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_enable_msix()
4250 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_enable_msix()
4254 if (gaudi2->hw_cap_initialized & HW_CAP_MSIX) in gaudi2_enable_msix()
4257 rc = pci_alloc_irq_vectors(hdev->pdev, GAUDI2_MSIX_ENTRIES, GAUDI2_MSIX_ENTRIES, in gaudi2_enable_msix()
4260 dev_err(hdev->dev, "MSI-X: Failed to enable support -- %d/%d\n", in gaudi2_enable_msix()
4265 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION); in gaudi2_enable_msix()
4266 cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_CS_COMPLETION]; in gaudi2_enable_msix()
4269 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in gaudi2_enable_msix()
4273 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE); in gaudi2_enable_msix()
4275 &hdev->event_queue); in gaudi2_enable_msix()
4277 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in gaudi2_enable_msix()
4283 dev_err(hdev->dev, "Failed to enable decoder IRQ"); in gaudi2_enable_msix()
4287 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT); in gaudi2_enable_msix()
4290 &hdev->tpc_interrupt); in gaudi2_enable_msix()
4292 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in gaudi2_enable_msix()
4296 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR); in gaudi2_enable_msix()
4299 &hdev->unexpected_error_interrupt); in gaudi2_enable_msix()
4301 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in gaudi2_enable_msix()
4305 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count, user_irq_init_cnt = 0; in gaudi2_enable_msix()
4306 user_irq_init_cnt < prop->user_interrupt_count; in gaudi2_enable_msix()
4309 irq = pci_irq_vector(hdev->pdev, i); in gaudi2_enable_msix()
4311 &hdev->user_interrupt[j]); in gaudi2_enable_msix()
4313 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in gaudi2_enable_msix()
4318 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EQ_ERROR); in gaudi2_enable_msix()
4323 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in gaudi2_enable_msix()
4327 gaudi2->hw_cap_initialized |= HW_CAP_MSIX; in gaudi2_enable_msix()
4332 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count; in gaudi2_enable_msix()
4335 irq = pci_irq_vector(hdev->pdev, i); in gaudi2_enable_msix()
4336 free_irq(irq, &hdev->user_interrupt[j]); in gaudi2_enable_msix()
4338 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR); in gaudi2_enable_msix()
4339 free_irq(irq, &hdev->unexpected_error_interrupt); in gaudi2_enable_msix()
4341 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT); in gaudi2_enable_msix()
4342 free_irq(irq, &hdev->tpc_interrupt); in gaudi2_enable_msix()
4344 gaudi2_dec_disable_msix(hdev, GAUDI2_IRQ_NUM_DEC_LAST + 1); in gaudi2_enable_msix()
4346 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE); in gaudi2_enable_msix()
4350 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION); in gaudi2_enable_msix()
4354 pci_free_irq_vectors(hdev->pdev); in gaudi2_enable_msix()
4361 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_sync_irqs()
4365 if (!(gaudi2->hw_cap_initialized & HW_CAP_MSIX)) in gaudi2_sync_irqs()
4369 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION)); in gaudi2_sync_irqs()
4372 irq = pci_irq_vector(hdev->pdev, i); in gaudi2_sync_irqs()
4376 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT)); in gaudi2_sync_irqs()
4377 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR)); in gaudi2_sync_irqs()
4379 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = 0 ; j < hdev->asic_prop.user_interrupt_count; in gaudi2_sync_irqs()
4381 irq = pci_irq_vector(hdev->pdev, i); in gaudi2_sync_irqs()
4385 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE)); in gaudi2_sync_irqs()
4386 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EQ_ERROR)); in gaudi2_sync_irqs()
4391 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_disable_msix()
4392 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_disable_msix()
4396 if (!(gaudi2->hw_cap_initialized & HW_CAP_MSIX)) in gaudi2_disable_msix()
4401 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE); in gaudi2_disable_msix()
4402 free_irq(irq, &hdev->event_queue); in gaudi2_disable_msix()
4404 gaudi2_dec_disable_msix(hdev, GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM + 1); in gaudi2_disable_msix()
4406 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT); in gaudi2_disable_msix()
4407 free_irq(irq, &hdev->tpc_interrupt); in gaudi2_disable_msix()
4409 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR); in gaudi2_disable_msix()
4410 free_irq(irq, &hdev->unexpected_error_interrupt); in gaudi2_disable_msix()
4412 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count, k = 0; in gaudi2_disable_msix()
4413 k < hdev->asic_prop.user_interrupt_count ; i++, j++, k++) { in gaudi2_disable_msix()
4415 irq = pci_irq_vector(hdev->pdev, i); in gaudi2_disable_msix()
4416 free_irq(irq, &hdev->user_interrupt[j]); in gaudi2_disable_msix()
4419 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION); in gaudi2_disable_msix()
4420 cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_CS_COMPLETION]; in gaudi2_disable_msix()
4423 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EQ_ERROR); in gaudi2_disable_msix()
4426 pci_free_irq_vectors(hdev->pdev); in gaudi2_disable_msix()
4428 gaudi2->hw_cap_initialized &= ~HW_CAP_MSIX; in gaudi2_disable_msix()
4438 if (hdev->pldm) in gaudi2_stop_dcore_dec()
4445 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit))) in gaudi2_stop_dcore_dec()
4465 dev_err(hdev->dev, in gaudi2_stop_dcore_dec()
4478 if (hdev->pldm) in gaudi2_stop_pcie_dec()
4485 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit))) in gaudi2_stop_pcie_dec()
4505 dev_err(hdev->dev, in gaudi2_stop_pcie_dec()
4513 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_stop_dec()
4516 if ((gaudi2->dec_hw_cap_initialized & HW_CAP_DEC_MASK) == 0) in gaudi2_stop_dec()
4531 reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_RUN_HALT_REQ_RUN_REQ_MASK, 1); in gaudi2_set_arc_running_mode()
4533 reg_val = FIELD_PREP(ARC_FARM_ARC0_AUX_RUN_HALT_REQ_HALT_REQ_MASK, 1); in gaudi2_set_arc_running_mode()
4553 if (hdev->pldm) in gaudi2_verify_arc_running_mode()
4577 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_reset_arcs()
4590 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_nic_qmans_manual_flush()
4594 if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK)) in gaudi2_nic_qmans_manual_flush()
4600 if (!(hdev->nic_ports_mask & BIT(i))) in gaudi2_nic_qmans_manual_flush()
4622 dev_err(hdev->dev, "failed to %s arc: %d\n", in gaudi2_set_engine_cores()
4625 return -1; in gaudi2_set_engine_cores()
4635 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_set_tpc_engine_mode()
4638 if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK)) in gaudi2_set_tpc_engine_mode()
4642 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + tpc_id))) in gaudi2_set_tpc_engine_mode()
4648 (engine_command == HL_ENGINE_STALL) ? 1 : 0); in gaudi2_set_tpc_engine_mode()
4662 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_set_mme_engine_mode()
4666 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + mme_id))) in gaudi2_set_mme_engine_mode()
4672 (engine_command == HL_ENGINE_STALL) ? 1 : 0); in gaudi2_set_mme_engine_mode()
4680 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_set_edma_engine_mode()
4683 if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK)) in gaudi2_set_edma_engine_mode()
4687 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + edma_id))) in gaudi2_set_edma_engine_mode()
4693 (engine_command == HL_ENGINE_STALL) ? 1 : 0); in gaudi2_set_edma_engine_mode()
4740 dev_err(hdev->dev, "Invalid engine ID %u\n", engine_ids[i]); in gaudi2_set_engine_modes()
4741 return -EINVAL; in gaudi2_set_engine_modes()
4761 dev_err(hdev->dev, "failed to execute command id %u\n", engine_command); in gaudi2_set_engines()
4762 return -EINVAL; in gaudi2_set_engines()
4770 if (hdev->pldm) in gaudi2_halt_engines()
4820 struct pre_fw_load_props *pre_fw_load = &hdev->fw_loader.pre_fw_load; in gaudi2_init_firmware_preload_params()
4822 pre_fw_load->cpu_boot_status_reg = mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS; in gaudi2_init_firmware_preload_params()
4823 pre_fw_load->sts_boot_dev_sts0_reg = mmCPU_BOOT_DEV_STS0; in gaudi2_init_firmware_preload_params()
4824 pre_fw_load->sts_boot_dev_sts1_reg = mmCPU_BOOT_DEV_STS1; in gaudi2_init_firmware_preload_params()
4825 pre_fw_load->boot_err0_reg = mmCPU_BOOT_ERR0; in gaudi2_init_firmware_preload_params()
4826 pre_fw_load->boot_err1_reg = mmCPU_BOOT_ERR1; in gaudi2_init_firmware_preload_params()
4827 pre_fw_load->wait_for_preboot_timeout = GAUDI2_PREBOOT_REQ_TIMEOUT_USEC; in gaudi2_init_firmware_preload_params()
4828 pre_fw_load->wait_for_preboot_extended_timeout = in gaudi2_init_firmware_preload_params()
4834 struct fw_load_mgr *fw_loader = &hdev->fw_loader; in gaudi2_init_firmware_loader()
4839 fw_loader->fw_comp_loaded = FW_TYPE_NONE; in gaudi2_init_firmware_loader()
4840 fw_loader->boot_fit_img.image_name = GAUDI2_BOOT_FIT_FILE; in gaudi2_init_firmware_loader()
4841 fw_loader->linux_img.image_name = GAUDI2_LINUX_FW_FILE; in gaudi2_init_firmware_loader()
4842 fw_loader->boot_fit_timeout = GAUDI2_BOOT_FIT_REQ_TIMEOUT_USEC; in gaudi2_init_firmware_loader()
4843 fw_loader->skip_bmc = false; in gaudi2_init_firmware_loader()
4844 fw_loader->sram_bar_id = SRAM_CFG_BAR_ID; in gaudi2_init_firmware_loader()
4845 fw_loader->dram_bar_id = DRAM_BAR_ID; in gaudi2_init_firmware_loader()
4846 fw_loader->cpu_timeout = GAUDI2_CPU_TIMEOUT_USEC; in gaudi2_init_firmware_loader()
4850 * hard-coded). in later stages of the protocol those values will be in gaudi2_init_firmware_loader()
4852 * will always be up-to-date in gaudi2_init_firmware_loader()
4854 dynamic_loader = &hdev->fw_loader.dynamic_loader; in gaudi2_init_firmware_loader()
4855 dyn_regs = &dynamic_loader->comm_desc.cpu_dyn_regs; in gaudi2_init_firmware_loader()
4856 dyn_regs->kmd_msg_to_cpu = cpu_to_le32(mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU); in gaudi2_init_firmware_loader()
4857 dyn_regs->cpu_cmd_status_to_host = cpu_to_le32(mmCPU_CMD_STATUS_TO_HOST); in gaudi2_init_firmware_loader()
4858 dynamic_loader->wait_for_bl_timeout = GAUDI2_WAIT_FOR_BL_TIMEOUT_USEC; in gaudi2_init_firmware_loader()
4863 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_cpu()
4866 if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU)) in gaudi2_init_cpu()
4869 if (gaudi2->hw_cap_initialized & HW_CAP_CPU) in gaudi2_init_cpu()
4876 gaudi2->hw_cap_initialized |= HW_CAP_CPU; in gaudi2_init_cpu()
4883 struct hl_hw_queue *cpu_pq = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ]; in gaudi2_init_cpu_queues()
4884 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_init_cpu_queues()
4885 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_cpu_queues()
4891 if (!hdev->cpu_queues_enable) in gaudi2_init_cpu_queues()
4894 if (gaudi2->hw_cap_initialized & HW_CAP_CPU_Q) in gaudi2_init_cpu_queues()
4897 eq = &hdev->event_queue; in gaudi2_init_cpu_queues()
4899 dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi2_init_cpu_queues()
4901 WREG32(mmCPU_IF_PQ_BASE_ADDR_LOW, lower_32_bits(cpu_pq->bus_address)); in gaudi2_init_cpu_queues()
4902 WREG32(mmCPU_IF_PQ_BASE_ADDR_HIGH, upper_32_bits(cpu_pq->bus_address)); in gaudi2_init_cpu_queues()
4904 WREG32(mmCPU_IF_EQ_BASE_ADDR_LOW, lower_32_bits(eq->bus_address)); in gaudi2_init_cpu_queues()
4905 WREG32(mmCPU_IF_EQ_BASE_ADDR_HIGH, upper_32_bits(eq->bus_address)); in gaudi2_init_cpu_queues()
4907 WREG32(mmCPU_IF_CQ_BASE_ADDR_LOW, lower_32_bits(hdev->cpu_accessible_dma_address)); in gaudi2_init_cpu_queues()
4908 WREG32(mmCPU_IF_CQ_BASE_ADDR_HIGH, upper_32_bits(hdev->cpu_accessible_dma_address)); in gaudi2_init_cpu_queues()
4923 WREG32(le32_to_cpu(dyn_regs->gic_host_pi_upd_irq), in gaudi2_init_cpu_queues()
4935 dev_err(hdev->dev, "Failed to communicate with device CPU (timeout)\n"); in gaudi2_init_cpu_queues()
4936 return -EIO; in gaudi2_init_cpu_queues()
4940 if (prop->fw_cpu_boot_dev_sts0_valid) in gaudi2_init_cpu_queues()
4941 prop->fw_app_cpu_boot_dev_sts0 = RREG32(mmCPU_BOOT_DEV_STS0); in gaudi2_init_cpu_queues()
4943 if (prop->fw_cpu_boot_dev_sts1_valid) in gaudi2_init_cpu_queues()
4944 prop->fw_app_cpu_boot_dev_sts1 = RREG32(mmCPU_BOOT_DEV_STS1); in gaudi2_init_cpu_queues()
4946 gaudi2->hw_cap_initialized |= HW_CAP_CPU_Q; in gaudi2_init_cpu_queues()
4957 q = &hdev->kernel_queues[queue_id_base + pq_id]; in gaudi2_init_qman_pq()
4961 lower_32_bits(q->bus_address)); in gaudi2_init_qman_pq()
4963 upper_32_bits(q->bus_address)); in gaudi2_init_qman_pq()
4995 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_qman_pqc()
5006 lower_32_bits(gaudi2->scratchpad_bus_address)); in gaudi2_init_qman_pqc()
5008 upper_32_bits(gaudi2->scratchpad_bus_address)); in gaudi2_init_qman_pqc()
5019 WREG32(reg_base + QM_PQC_CFG_OFFSET, 1 << PDMA0_QM_PQC_CFG_EN_SHIFT); in gaudi2_init_qman_pqc()
5024 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi2_get_dyn_sp_reg()
5037 sp_reg_addr = le32_to_cpu(dyn_regs->gic_dma_qm_irq_ctrl); in gaudi2_get_dyn_sp_reg()
5046 sp_reg_addr = le32_to_cpu(dyn_regs->gic_mme_qm_irq_ctrl); in gaudi2_get_dyn_sp_reg()
5055 sp_reg_addr = le32_to_cpu(dyn_regs->gic_tpc_qm_irq_ctrl); in gaudi2_get_dyn_sp_reg()
5058 sp_reg_addr = le32_to_cpu(dyn_regs->gic_rot_qm_irq_ctrl); in gaudi2_get_dyn_sp_reg()
5061 sp_reg_addr = le32_to_cpu(dyn_regs->gic_nic_qm_irq_ctrl); in gaudi2_get_dyn_sp_reg()
5064 dev_err(hdev->dev, "Unexpected h/w queue %d\n", queue_id_base); in gaudi2_get_dyn_sp_reg()
5113 hdev->kernel_queues[queue_id_base + pq_id].cq_id = GAUDI2_RESERVED_CQ_CS_COMPLETION; in gaudi2_init_qman()
5128 prot = 1 << ARC_FARM_KDMA_PROT_ERR_VAL_SHIFT; in gaudi2_init_dma_core()
5130 prot |= 1 << ARC_FARM_KDMA_PROT_VAL_SHIFT; in gaudi2_init_dma_core()
5134 dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi2_init_dma_core()
5135 irq_handler_offset = le32_to_cpu(dyn_regs->gic_dma_core_irq_ctrl); in gaudi2_init_dma_core()
5148 WREG32(reg_base + DMA_CORE_CFG_0_OFFSET, 1 << ARC_FARM_KDMA_CFG_0_EN_SHIFT); in gaudi2_init_dma_core()
5153 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_kdma()
5156 if ((gaudi2->hw_cap_initialized & HW_CAP_KDMA) == HW_CAP_KDMA) in gaudi2_init_kdma()
5163 gaudi2->hw_cap_initialized |= HW_CAP_KDMA; in gaudi2_init_kdma()
5168 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_pdma()
5171 if ((gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK) == HW_CAP_PDMA_MASK) in gaudi2_init_pdma()
5186 gaudi2->hw_cap_initialized |= HW_CAP_PDMA_MASK; in gaudi2_init_pdma()
5205 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_init_edma()
5206 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_edma()
5209 if ((gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK) == HW_CAP_EDMA_MASK) in gaudi2_init_edma()
5216 if (!(prop->edma_enabled_mask & BIT(seq))) in gaudi2_init_edma()
5221 gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_EDMA_SHIFT + seq); in gaudi2_init_edma()
5227 * gaudi2_arm_monitors_for_virt_msix_db() - Arm monitors for writing to the virtual MSI-X doorbell.
5234 * write directly to the HBW host memory of the virtual MSI-X doorbell.
5239 * completion, by decrementing the sync object value and re-arming the monitor.
5245 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_arm_monitors_for_virt_msix_db()
5254 * 1. Write interrupt ID to the virtual MSI-X doorbell (master monitor) in gaudi2_arm_monitors_for_virt_msix_db()
5255 * 2. Decrement SOB value by 1. in gaudi2_arm_monitors_for_virt_msix_db()
5256 * 3. Re-arm the master monitor. in gaudi2_arm_monitors_for_virt_msix_db()
5261 /* 2nd monitor: Decrement SOB value by 1 */ in gaudi2_arm_monitors_for_virt_msix_db()
5268 payload = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_VAL_MASK, 0x7FFF) | /* "-1" */ in gaudi2_arm_monitors_for_virt_msix_db()
5269 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_SIGN_MASK, 1) | in gaudi2_arm_monitors_for_virt_msix_db()
5270 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_INC_MASK, 1); in gaudi2_arm_monitors_for_virt_msix_db()
5273 /* 3rd monitor: Re-arm the master monitor */ in gaudi2_arm_monitors_for_virt_msix_db()
5286 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_ARM_SOD_MASK, 1); in gaudi2_arm_monitors_for_virt_msix_db()
5291 /* 1st monitor (master): Write interrupt ID to the virtual MSI-X doorbell */ in gaudi2_arm_monitors_for_virt_msix_db()
5297 addr = gaudi2->virt_msix_db_dma_addr; in gaudi2_arm_monitors_for_virt_msix_db()
5310 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_prepare_sm_for_virt_msix_db()
5314 if (!(prop->decoder_enabled_mask & BIT(decoder_id))) in gaudi2_prepare_sm_for_virt_msix_db()
5324 interrupt_id += 1; in gaudi2_prepare_sm_for_virt_msix_db()
5331 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_sm()
5337 reg_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_CQ_EN_MASK, 1); in gaudi2_init_sm()
5338 reg_val |= FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_LBW_EN_MASK, 1); in gaudi2_init_sm()
5344 reg_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_MON_CONFIG_CQ_EN_MASK, 1); in gaudi2_init_sm()
5347 /* Init CQ0 DB - configure the monitor to trigger MSI-X interrupt */ in gaudi2_init_sm()
5348 WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0, lower_32_bits(gaudi2->virt_msix_db_dma_addr)); in gaudi2_init_sm()
5349 WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0, upper_32_bits(gaudi2->virt_msix_db_dma_addr)); in gaudi2_init_sm()
5354 hdev->completion_queue[i].bus_address; in gaudi2_init_sm()
5368 /* Initialize sync objects and monitors which are used for the virtual MSI-X doorbell */ in gaudi2_init_sm()
5374 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_mme_acc()
5379 reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_SRC_POS_INF_MASK, 1); in gaudi2_init_mme_acc()
5380 reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_SRC_NEG_INF_MASK, 1); in gaudi2_init_mme_acc()
5381 reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_SRC_NAN_MASK, 1); in gaudi2_init_mme_acc()
5382 reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_RESULT_POS_INF_MASK, 1); in gaudi2_init_mme_acc()
5383 reg_val |= FIELD_PREP(MME_ACC_INTR_MASK_AP_RESULT_NEG_INF_MASK, 1); in gaudi2_init_mme_acc()
5390 WREG32(reg_base + MME_ACC_AP_LFSR_SEED_WDATA_OFFSET, gaudi2->lfsr_rand_seeds[i]); in gaudi2_init_mme_acc()
5403 case 1: in gaudi2_init_dcore_mme()
5413 dev_err(hdev->dev, "Invalid dcore id %u\n", dcore_id); in gaudi2_init_dcore_mme()
5428 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_mme()
5431 if ((gaudi2->hw_cap_initialized & HW_CAP_MME_MASK) == HW_CAP_MME_MASK) in gaudi2_init_mme()
5437 gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_MME_SHIFT + i); in gaudi2_init_mme()
5458 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_tpc_config()
5459 struct gaudi2_tpc_init_cfg_data *cfg_data = ctx->data; in gaudi2_init_tpc_config()
5463 queue_id_base = cfg_data->dcore_tpc_qid_base[dcore] + (inst * NUM_OF_PQ_PER_QMAN); in gaudi2_init_tpc_config()
5465 if (dcore == 0 && inst == (NUM_DCORE0_TPC - 1)) in gaudi2_init_tpc_config()
5474 gaudi2->tpc_hw_cap_initialized |= BIT_ULL(HW_CAP_TPC_SHIFT + seq); in gaudi2_init_tpc_config()
5479 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_tpc()
5483 if (!hdev->asic_prop.tpc_enabled_mask) in gaudi2_init_tpc()
5486 if ((gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK) == HW_CAP_TPC_MASK) in gaudi2_init_tpc()
5490 init_cfg_data.dcore_tpc_qid_base[1] = GAUDI2_QUEUE_ID_DCORE1_TPC_0_0; in gaudi2_init_tpc()
5500 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_rotator()
5509 gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_ROT_SHIFT + i); in gaudi2_init_rotator()
5532 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_dec()
5536 if (!hdev->asic_prop.decoder_enabled_mask) in gaudi2_init_dec()
5539 if ((gaudi2->dec_hw_cap_initialized & HW_CAP_DEC_MASK) == HW_CAP_DEC_MASK) in gaudi2_init_dec()
5546 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit))) in gaudi2_init_dec()
5556 gaudi2->dec_hw_cap_initialized |= BIT_ULL(HW_CAP_DEC_SHIFT + dec_bit); in gaudi2_init_dec()
5561 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit))) in gaudi2_init_dec()
5569 gaudi2->dec_hw_cap_initialized |= BIT_ULL(HW_CAP_DEC_SHIFT + dec_bit); in gaudi2_init_dec()
5579 if (hdev->pldm || !hdev->pdev) in gaudi2_mmu_update_asid_hop0_addr()
5598 dev_err(hdev->dev, "Timeout during MMU hop0 config of asid %d\n", asid); in gaudi2_mmu_update_asid_hop0_addr()
5625 timeout_usec = (hdev->pldm) ? GAUDI2_PLDM_MMU_TIMEOUT_USEC : in gaudi2_mmu_invalidate_cache_status_poll()
5629 if (inv_params->flags & MMU_OP_CLEAR_MEMCACHE) { in gaudi2_mmu_invalidate_cache_status_poll()
5648 if (inv_params->flags & MMU_OP_SKIP_LOW_CACHE_INV) in gaudi2_mmu_invalidate_cache_status_poll()
5651 start_offset = inv_params->range_invalidation ? in gaudi2_mmu_invalidate_cache_status_poll()
5667 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_is_hmmu_enabled()
5672 if (gaudi2->hw_cap_initialized & hw_cap) in gaudi2_is_hmmu_enabled()
5692 if (inv_params->range_invalidation) { in gaudi2_mmu_invalidate_cache_trigger()
5699 u64 start = inv_params->start_va - 1; in gaudi2_mmu_invalidate_cache_trigger()
5710 inv_params->end_va >> MMU_RANGE_INV_VA_LSB_SHIFT); in gaudi2_mmu_invalidate_cache_trigger()
5713 inv_params->end_va >> MMU_RANGE_INV_VA_MSB_SHIFT); in gaudi2_mmu_invalidate_cache_trigger()
5719 inv_params->inv_start_val, inv_params->flags); in gaudi2_mmu_invalidate_cache_trigger()
5775 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_mmu_invalidate_cache()
5779 if (hdev->reset_info.hard_reset_pending) in gaudi2_mmu_invalidate_cache()
5783 invld_params.inv_start_val = 1; in gaudi2_mmu_invalidate_cache()
5785 if ((flags & MMU_OP_USERPTR) && (gaudi2->hw_cap_initialized & HW_CAP_PMMU)) { in gaudi2_mmu_invalidate_cache()
5802 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_mmu_invalidate_cache_range()
5807 if (hdev->reset_info.hard_reset_pending) in gaudi2_mmu_invalidate_cache_range()
5810 inv_start_val = (1 << MMU_RANGE_INV_EN_SHIFT | in gaudi2_mmu_invalidate_cache_range()
5811 1 << MMU_RANGE_INV_ASID_EN_SHIFT | in gaudi2_mmu_invalidate_cache_range()
5816 if ((flags & MMU_OP_USERPTR) && (gaudi2->hw_cap_initialized & HW_CAP_PMMU)) { in gaudi2_mmu_invalidate_cache_range()
5828 invld_params.inv_start_val = 1; in gaudi2_mmu_invalidate_cache_range()
5852 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_mmu_update_hop0_addr()
5854 u32 asid, max_asid = prop->max_asid; in gaudi2_mmu_update_hop0_addr()
5858 if (hdev->pldm) in gaudi2_mmu_update_hop0_addr()
5862 hop0_addr = hdev->mmu_priv.hr.mmu_asid_hop0[asid].phys_addr; in gaudi2_mmu_update_hop0_addr()
5865 dev_err(hdev->dev, "failed to set hop0 addr for asid %d\n", asid); in gaudi2_mmu_update_hop0_addr()
5878 if (hdev->pldm || !hdev->pdev) in gaudi2_mmu_init_common()
5883 WREG32(stlb_base + STLB_INV_ALL_START_OFFSET, 1); in gaudi2_mmu_init_common()
5894 dev_notice_ratelimited(hdev->dev, "Timeout when waiting for MMU SRAM init\n"); in gaudi2_mmu_init_common()
5911 dev_notice_ratelimited(hdev->dev, "Timeout when waiting for MMU invalidate all\n"); in gaudi2_mmu_init_common()
5913 WREG32(mmu_base + MMU_ENABLE_OFFSET, 1); in gaudi2_mmu_init_common()
5920 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_pci_mmu_init()
5924 if (gaudi2->hw_cap_initialized & HW_CAP_PMMU) in gaudi2_pci_mmu_init()
5951 1), in gaudi2_pci_mmu_init()
5963 gaudi2->hw_cap_initialized |= HW_CAP_PMMU; in gaudi2_pci_mmu_init()
5971 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_dcore_hmmu_init()
5972 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_dcore_hmmu_init()
5984 if ((gaudi2->hw_cap_initialized & hw_cap) || !(prop->hmmu_hif_enabled_mask & BIT(dmmu_seq))) in gaudi2_dcore_hmmu_init()
6006 RMWREG32(stlb_base + STLB_HOP_CONFIGURATION_OFFSET, 1, in gaudi2_dcore_hmmu_init()
6015 gaudi2->hw_cap_initialized |= hw_cap; in gaudi2_dcore_hmmu_init()
6051 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_hw_init()
6068 if (hdev->asic_prop.iatu_done_by_fw) in gaudi2_hw_init()
6069 gaudi2->dram_bar_cur_addr = DRAM_PHYS_BASE; in gaudi2_hw_init()
6072 * Before pushing u-boot/linux to device, need to set the hbm bar to in gaudi2_hw_init()
6076 dev_err(hdev->dev, "failed to map HBM bar to DRAM base address\n"); in gaudi2_hw_init()
6077 return -EIO; in gaudi2_hw_init()
6082 dev_err(hdev->dev, "failed to initialize CPU\n"); in gaudi2_hw_init()
6091 dev_err(hdev->dev, "failed to initialize CPU H/W queues %d\n", rc); in gaudi2_hw_init()
6095 rc = gaudi2->cpucp_info_get(hdev); in gaudi2_hw_init()
6097 dev_err(hdev->dev, "Failed to get cpucp info\n"); in gaudi2_hw_init()
6140 * gaudi2_send_hard_reset_cmd - common function to handle reset
6149 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi2_send_hard_reset_cmd()
6151 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_send_hard_reset_cmd()
6154 preboot_only = (hdev->fw_loader.fw_comp_loaded == FW_TYPE_PREBOOT_CPU); in gaudi2_send_hard_reset_cmd()
6155 heartbeat_reset = (hdev->reset_info.curr_reset_cause == HL_RESET_CAUSE_HEARTBEAT); in gaudi2_send_hard_reset_cmd()
6170 if (gaudi2 && (gaudi2->hw_cap_initialized & HW_CAP_CPU) && in gaudi2_send_hard_reset_cmd()
6176 * 1. FW reset: FW initiate the reset sequence in gaudi2_send_hard_reset_cmd()
6183 WREG32(le32_to_cpu(dyn_regs->gic_host_halt_irq), in gaudi2_send_hard_reset_cmd()
6193 * For the case in which we are working with Linux/Bootfit this is a hail-mary in gaudi2_send_hard_reset_cmd()
6207 if (hdev->asic_prop.hard_reset_done_by_fw) in gaudi2_send_hard_reset_cmd()
6215 * gaudi2_execute_hard_reset - execute hard reset by driver/FW
6223 if (hdev->asic_prop.hard_reset_done_by_fw) { in gaudi2_execute_hard_reset()
6236 WREG32(mmPSOC_RESET_CONF_SW_ALL_RST, 1); in gaudi2_execute_hard_reset()
6254 dev_err(hdev->dev, "Timeout while waiting for FW to complete soft reset (0x%x)\n", in gaudi2_get_soft_rst_done_indication()
6260 * gaudi2_execute_soft_reset - execute soft reset by driver/FW
6274 if (hl_is_fw_sw_ver_below(hdev, 1, 10)) { in gaudi2_execute_soft_reset()
6293 gaudi2_write_rr_to_all_lbw_rtrs(hdev, RR_TYPE_LONG, NUM_LONG_LBW_RR - 1, in gaudi2_execute_soft_reset()
6296 gaudi2_write_rr_to_all_lbw_rtrs(hdev, RR_TYPE_LONG, NUM_LONG_LBW_RR - 2, in gaudi2_execute_soft_reset()
6300 WREG32(mmPSOC_RESET_CONF_SOFT_RST, 1); in gaudi2_execute_soft_reset()
6322 dev_err(hdev->dev, "Timeout while waiting for device to reset 0x%x\n", reg_val); in gaudi2_poll_btm_indication()
6327 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_hw_fini()
6332 if (hdev->pldm) { in gaudi2_hw_fini()
6347 driver_performs_reset = !hdev->asic_prop.hard_reset_done_by_fw; in gaudi2_hw_fini()
6356 driver_performs_reset = (hdev->fw_components == FW_TYPE_PREBOOT_CPU && in gaudi2_hw_fini()
6357 !hdev->asic_prop.fw_security_enabled); in gaudi2_hw_fini()
6368 * 1. FW update is being triggered in gaudi2_hw_fini()
6369 * - setting the dirty bit in gaudi2_hw_fini()
6372 * - dirty bit cleared in gaudi2_hw_fini()
6373 * - BTM indication cleared in gaudi2_hw_fini()
6374 * - preboot ready indication cleared in gaudi2_hw_fini()
6376 * - BTM indication will be set in gaudi2_hw_fini()
6377 * - BIST test performed and another reset triggered in gaudi2_hw_fini()
6388 if (hdev->fw_components & FW_TYPE_PREBOOT_CPU) in gaudi2_hw_fini()
6397 gaudi2->dec_hw_cap_initialized &= ~(HW_CAP_DEC_MASK); in gaudi2_hw_fini()
6398 gaudi2->tpc_hw_cap_initialized &= ~(HW_CAP_TPC_MASK); in gaudi2_hw_fini()
6401 * Clear NIC capability mask in order for driver to re-configure in gaudi2_hw_fini()
6402 * NIC QMANs. NIC ports will not be re-configured during soft in gaudi2_hw_fini()
6405 gaudi2->nic_hw_cap_initialized &= ~(HW_CAP_NIC_MASK); in gaudi2_hw_fini()
6408 gaudi2->hw_cap_initialized &= in gaudi2_hw_fini()
6415 memset(gaudi2->events_stat, 0, sizeof(gaudi2->events_stat)); in gaudi2_hw_fini()
6417 gaudi2->hw_cap_initialized &= in gaudi2_hw_fini()
6431 dev_err(hdev->dev, "Failed to disable PCI access from CPU\n"); in gaudi2_suspend()
6451 rc = dma_mmap_coherent(hdev->dev, vma, cpu_addr, dma_addr, size); in gaudi2_mmap()
6453 dev_err(hdev->dev, "dma_mmap_coherent error %d", rc); in gaudi2_mmap()
6457 rc = remap_pfn_range(vma, vma->vm_start, in gaudi2_mmap()
6459 size, vma->vm_page_prot); in gaudi2_mmap()
6461 dev_err(hdev->dev, "remap_pfn_range error %d", rc); in gaudi2_mmap()
6470 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_is_queue_enabled()
6484 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0) >> 2); in gaudi2_is_queue_enabled()
6488 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0) >> 2); in gaudi2_is_queue_enabled()
6492 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0) >> 2); in gaudi2_is_queue_enabled()
6496 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0) >> 2); in gaudi2_is_queue_enabled()
6504 hw_test_cap_bit = HW_CAP_MME_SHIFT + 1; in gaudi2_is_queue_enabled()
6517 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE0_TPC_0_0) >> 2); in gaudi2_is_queue_enabled()
6521 return !!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(0)); in gaudi2_is_queue_enabled()
6526 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE1_TPC_0_0) >> 2); in gaudi2_is_queue_enabled()
6531 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE2_TPC_0_0) >> 2); in gaudi2_is_queue_enabled()
6536 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE3_TPC_0_0) >> 2); in gaudi2_is_queue_enabled()
6544 hw_test_cap_bit = HW_CAP_ROT_SHIFT + ((hw_queue_id - GAUDI2_QUEUE_ID_ROT_0_0) >> 2); in gaudi2_is_queue_enabled()
6548 hw_nic_cap_bit = HW_CAP_NIC_SHIFT + ((hw_queue_id - GAUDI2_QUEUE_ID_NIC_0_0) >> 2); in gaudi2_is_queue_enabled()
6552 return !!(gaudi2->nic_hw_cap_initialized & BIT_ULL(0)); in gaudi2_is_queue_enabled()
6556 return !!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q); in gaudi2_is_queue_enabled()
6563 return !!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(hw_tpc_cap_bit)); in gaudi2_is_queue_enabled()
6566 return !!(gaudi2->nic_hw_cap_initialized & BIT_ULL(hw_nic_cap_bit)); in gaudi2_is_queue_enabled()
6571 return !!(gaudi2->hw_cap_initialized & hw_cap_mask); in gaudi2_is_queue_enabled()
6576 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_is_arc_enabled()
6581 return !!(gaudi2->active_hw_arc & BIT_ULL(arc_id)); in gaudi2_is_arc_enabled()
6584 return !!(gaudi2->active_tpc_arc & BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0)); in gaudi2_is_arc_enabled()
6587 return !!(gaudi2->active_nic_arc & BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0)); in gaudi2_is_arc_enabled()
6596 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_clr_arc_id_cap()
6601 gaudi2->active_hw_arc &= ~(BIT_ULL(arc_id)); in gaudi2_clr_arc_id_cap()
6605 gaudi2->active_tpc_arc &= ~(BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0)); in gaudi2_clr_arc_id_cap()
6609 gaudi2->active_nic_arc &= ~(BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0)); in gaudi2_clr_arc_id_cap()
6619 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_set_arc_id_cap()
6624 gaudi2->active_hw_arc |= BIT_ULL(arc_id); in gaudi2_set_arc_id_cap()
6628 gaudi2->active_tpc_arc |= BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0); in gaudi2_set_arc_id_cap()
6632 gaudi2->active_nic_arc |= BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0); in gaudi2_set_arc_id_cap()
6642 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi2_ring_doorbell()
6647 * QMAN has 4 successive PQ_PI registers, 1 for each of the QMAN PQs. in gaudi2_ring_doorbell()
6666 WREG32(le32_to_cpu(dyn_regs->gic_host_pi_upd_irq), in gaudi2_ring_doorbell()
6677 pqe[1] = pbd[1]; in gaudi2_pqe_write()
6683 return dma_alloc_coherent(&hdev->pdev->dev, size, dma_handle, flags); in gaudi2_dma_alloc_coherent()
6689 dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, dma_handle); in gaudi2_dma_free_coherent()
6695 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_send_cpu_message()
6697 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) { in gaudi2_send_cpu_message()
6715 return dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle); in gaudi2_dma_pool_zalloc()
6720 dma_pool_free(hdev->dma_pool, vaddr, dma_addr); in gaudi2_dma_pool_free()
6736 struct asic_fixed_properties *asic_prop = &hdev->asic_prop; in gaudi2_validate_cb_address()
6737 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_validate_cb_address()
6739 if (!gaudi2_is_queue_enabled(hdev, parser->hw_queue_id)) { in gaudi2_validate_cb_address()
6740 dev_err(hdev->dev, "h/w queue %d is disabled\n", parser->hw_queue_id); in gaudi2_validate_cb_address()
6741 return -EINVAL; in gaudi2_validate_cb_address()
6746 if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, in gaudi2_validate_cb_address()
6747 parser->user_cb_size, in gaudi2_validate_cb_address()
6748 asic_prop->sram_user_base_address, in gaudi2_validate_cb_address()
6749 asic_prop->sram_end_address)) in gaudi2_validate_cb_address()
6752 if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, in gaudi2_validate_cb_address()
6753 parser->user_cb_size, in gaudi2_validate_cb_address()
6754 asic_prop->dram_user_base_address, in gaudi2_validate_cb_address()
6755 asic_prop->dram_end_address)) in gaudi2_validate_cb_address()
6758 if ((gaudi2->hw_cap_initialized & HW_CAP_DMMU_MASK) && in gaudi2_validate_cb_address()
6759 hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, in gaudi2_validate_cb_address()
6760 parser->user_cb_size, in gaudi2_validate_cb_address()
6761 asic_prop->dmmu.start_addr, in gaudi2_validate_cb_address()
6762 asic_prop->dmmu.end_addr)) in gaudi2_validate_cb_address()
6765 if (gaudi2->hw_cap_initialized & HW_CAP_PMMU) { in gaudi2_validate_cb_address()
6766 if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, in gaudi2_validate_cb_address()
6767 parser->user_cb_size, in gaudi2_validate_cb_address()
6768 asic_prop->pmmu.start_addr, in gaudi2_validate_cb_address()
6769 asic_prop->pmmu.end_addr) || in gaudi2_validate_cb_address()
6771 (u64) (uintptr_t) parser->user_cb, in gaudi2_validate_cb_address()
6772 parser->user_cb_size, in gaudi2_validate_cb_address()
6773 asic_prop->pmmu_huge.start_addr, in gaudi2_validate_cb_address()
6774 asic_prop->pmmu_huge.end_addr)) in gaudi2_validate_cb_address()
6777 } else if (gaudi2_host_phys_addr_valid((u64) (uintptr_t) parser->user_cb)) { in gaudi2_validate_cb_address()
6778 if (!hdev->pdev) in gaudi2_validate_cb_address()
6781 if (!device_iommu_mapped(&hdev->pdev->dev)) in gaudi2_validate_cb_address()
6785 dev_err(hdev->dev, "CB address %p + 0x%x for internal QMAN is not valid\n", in gaudi2_validate_cb_address()
6786 parser->user_cb, parser->user_cb_size); in gaudi2_validate_cb_address()
6788 return -EFAULT; in gaudi2_validate_cb_address()
6793 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_cs_parser()
6795 if (!parser->is_kernel_allocated_cb) in gaudi2_cs_parser()
6798 if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU)) { in gaudi2_cs_parser()
6799 dev_err(hdev->dev, "PMMU not initialized - Unsupported mode in Gaudi2\n"); in gaudi2_cs_parser()
6800 return -EINVAL; in gaudi2_cs_parser()
6808 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_send_heartbeat()
6810 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) in gaudi2_send_heartbeat()
6853 mask = ~(1 << (sob_id & 0x7)); in gaudi2_arm_cq_monitor()
6854 mode = 1; /* comparison mode is "equal to" */ in gaudi2_arm_cq_monitor()
6876 GAUDI2_RESERVED_CQ_KDMA_COMPLETION, 1, 1); in gaudi2_send_job_to_kdma()
6881 comp_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_INC_MASK, 1) | in gaudi2_send_job_to_kdma()
6882 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_VAL_MASK, 1); in gaudi2_send_job_to_kdma()
6893 commit_mask = FIELD_PREP(ARC_FARM_KDMA_CTX_COMMIT_LIN_MASK, 1) | in gaudi2_send_job_to_kdma()
6894 FIELD_PREP(ARC_FARM_KDMA_CTX_COMMIT_WR_COMP_EN_MASK, 1); in gaudi2_send_job_to_kdma()
6897 commit_mask |= FIELD_PREP(ARC_FARM_KDMA_CTX_COMMIT_MEM_SET_MASK, 1); in gaudi2_send_job_to_kdma()
6902 cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_KDMA_COMPLETION]; in gaudi2_send_job_to_kdma()
6903 cq_base = cq->kernel_address; in gaudi2_send_job_to_kdma()
6904 polling_addr = (u32 *)&cq_base[cq->ci]; in gaudi2_send_job_to_kdma()
6906 if (hdev->pldm) in gaudi2_send_job_to_kdma()
6907 /* for each 1MB 20 second of timeout */ in gaudi2_send_job_to_kdma()
6908 timeout = ((size / SZ_1M) + 1) * USEC_PER_SEC * 20; in gaudi2_send_job_to_kdma()
6917 (status == 1), in gaudi2_send_job_to_kdma()
6925 dev_err(hdev->dev, "Timeout while waiting for KDMA to be idle\n"); in gaudi2_send_job_to_kdma()
6926 WREG32(mmARC_FARM_KDMA_CFG_1, 1 << ARC_FARM_KDMA_CFG_1_HALT_SHIFT); in gaudi2_send_job_to_kdma()
6930 cq->ci = hl_cq_inc_ptr(cq->ci); in gaudi2_send_job_to_kdma()
6952 WREG32(reg_base + QM_PQC_CFG_OFFSET, 1 << PDMA0_QM_PQC_CFG_EN_SHIFT); in gaudi2_qman_set_test_mode()
6958 return hdev->asic_prop.first_available_user_sob[0] + in gaudi2_test_queue_hw_queue_id_to_sob_id()
6959 hw_queue_id - GAUDI2_QUEUE_ID_PDMA_0_0; in gaudi2_test_queue_hw_queue_id_to_sob_id()
6975 u32 tmp, sob_base = 1; in gaudi2_test_queue_send_msg_short()
6976 struct packet_msg_short *msg_short_pkt = msg_info->kern_addr; in gaudi2_test_queue_send_msg_short()
6981 (1 << GAUDI2_PKT_CTL_EB_SHIFT) | in gaudi2_test_queue_send_msg_short()
6982 (1 << GAUDI2_PKT_CTL_MB_SHIFT) | in gaudi2_test_queue_send_msg_short()
6986 msg_short_pkt->value = cpu_to_le32(sob_val); in gaudi2_test_queue_send_msg_short()
6987 msg_short_pkt->ctl = cpu_to_le32(tmp); in gaudi2_test_queue_send_msg_short()
6989 rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, pkt_size, msg_info->dma_addr); in gaudi2_test_queue_send_msg_short()
6991 dev_err(hdev->dev, in gaudi2_test_queue_send_msg_short()
7004 if (hdev->pldm) in gaudi2_test_queue_wait_completion()
7017 if (rc == -ETIMEDOUT) { in gaudi2_test_queue_wait_completion()
7018 dev_err(hdev->dev, "H/W queue %d test failed (SOB_OBJ_0 == 0x%x)\n", in gaudi2_test_queue_wait_completion()
7020 rc = -EIO; in gaudi2_test_queue_wait_completion()
7028 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_test_cpu_queue()
7034 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) in gaudi2_test_cpu_queue()
7042 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_test_queues()
7052 msg_info = &gaudi2->queues_test_info[i - GAUDI2_QUEUE_ID_PDMA_0_0]; in gaudi2_test_queues()
7071 /* chip is not usable, no need for cleanups, just bail-out with error */ in gaudi2_test_queues()
7084 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_compute_reset_late_init()
7092 dev_err(hdev->dev, "Failed to scrub arcs DCCM\n"); in gaudi2_compute_reset_late_init()
7099 irq_arr_size = gaudi2->num_of_valid_hw_events * sizeof(gaudi2->hw_events[0]); in gaudi2_compute_reset_late_init()
7100 return hl_fw_unmask_irq_arr(hdev, gaudi2->hw_events, irq_arr_size); in gaudi2_compute_reset_late_init()
7107 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_get_edma_idle_status()
7109 const char *edma_fmt = "%-6d%-6d%-9s%#-14x%#-15x%#x\n"; in gaudi2_get_edma_idle_status()
7117 "---- ---- ------- ------------ ------------- -------------\n"); in gaudi2_get_edma_idle_status()
7123 if (!(prop->edma_enabled_mask & BIT(seq))) in gaudi2_get_edma_idle_status()
7158 const char *pdma_fmt = "%-6d%-9s%#-14x%#-15x%#x\n"; in gaudi2_get_pdma_idle_status()
7166 "---- ------- ------------ ------------- -------------\n"); in gaudi2_get_pdma_idle_status()
7197 const char *nic_fmt = "%-5d%-9s%#-14x%#-12x\n"; in gaudi2_get_nic_idle_status()
7204 if (e && hdev->nic_ports_mask) in gaudi2_get_nic_idle_status()
7207 "--- ------- ------------ ----------\n"); in gaudi2_get_nic_idle_status()
7210 if (!(i & 1)) in gaudi2_get_nic_idle_status()
7215 if (!(hdev->nic_ports_mask & BIT(i))) in gaudi2_get_nic_idle_status()
7244 const char *mme_fmt = "%-5d%-6s%-9s%#-14x%#x\n"; in gaudi2_get_mme_idle_status()
7252 "--- ---- ------- ------------ ---------------\n"); in gaudi2_get_mme_idle_status()
7285 struct gaudi2_tpc_idle_data *idle_data = ctx->data; in gaudi2_is_tpc_engine_idle()
7290 if ((dcore == 0) && (inst == (NUM_DCORE0_TPC - 1))) in gaudi2_is_tpc_engine_idle()
7303 *(idle_data->is_idle) &= is_eng_idle; in gaudi2_is_tpc_engine_idle()
7305 if (idle_data->mask && !is_eng_idle) in gaudi2_is_tpc_engine_idle()
7306 set_bit(engine_idx, idle_data->mask); in gaudi2_is_tpc_engine_idle()
7308 if (idle_data->e) in gaudi2_is_tpc_engine_idle()
7309 hl_engine_data_sprintf(idle_data->e, in gaudi2_is_tpc_engine_idle()
7310 idle_data->tpc_fmt, dcore, inst, in gaudi2_is_tpc_engine_idle()
7318 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_get_tpc_idle_status()
7323 .tpc_fmt = "%-6d%-5d%-9s%#-14x%#-12x%#x\n", in gaudi2_get_tpc_idle_status()
7333 if (e && prop->tpc_enabled_mask) in gaudi2_get_tpc_idle_status()
7336 "---- --- ------- ------------ ---------- ------\n"); in gaudi2_get_tpc_idle_status()
7346 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_get_decoder_idle_status()
7348 const char *pcie_dec_fmt = "%-10d%-9s%#x\n"; in gaudi2_get_decoder_idle_status()
7349 const char *dec_fmt = "%-6d%-5d%-9s%#x\n"; in gaudi2_get_decoder_idle_status()
7356 if (e && (prop->decoder_enabled_mask & (~PCIE_DEC_EN_MASK))) in gaudi2_get_decoder_idle_status()
7359 "---- --- ------- ---------------\n"); in gaudi2_get_decoder_idle_status()
7363 dec_enabled_bit = 1 << (i * NUM_OF_DEC_PER_DCORE + j); in gaudi2_get_decoder_idle_status()
7364 if (!(prop->decoder_enabled_mask & dec_enabled_bit)) in gaudi2_get_decoder_idle_status()
7384 if (e && (prop->decoder_enabled_mask & PCIE_DEC_EN_MASK)) in gaudi2_get_decoder_idle_status()
7387 "-------- ------- ---------------\n"); in gaudi2_get_decoder_idle_status()
7392 if (!(prop->decoder_enabled_mask & BIT(dec_enabled_bit))) in gaudi2_get_decoder_idle_status()
7415 const char *rot_fmt = "%-6d%-5d%-9s%#-14x%#-14x%#x\n"; in gaudi2_get_rotator_idle_status()
7425 "---- --- ------- ------------ ------------ ----------\n"); in gaudi2_get_rotator_idle_status()
7467 __acquires(&gaudi2->hw_queues_lock) in gaudi2_hw_queues_lock()
7469 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_hw_queues_lock()
7471 spin_lock(&gaudi2->hw_queues_lock); in gaudi2_hw_queues_lock()
7475 __releases(&gaudi2->hw_queues_lock) in gaudi2_hw_queues_unlock()
7477 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_hw_queues_unlock()
7479 spin_unlock(&gaudi2->hw_queues_lock); in gaudi2_hw_queues_unlock()
7484 return hdev->pdev->device; in gaudi2_get_pci_id()
7489 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_get_eeprom_data()
7491 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) in gaudi2_get_eeprom_data()
7504 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_get_events_stat()
7507 *size = (u32) sizeof(gaudi2->events_stat_aggregate); in gaudi2_get_events_stat()
7508 return gaudi2->events_stat_aggregate; in gaudi2_get_events_stat()
7511 *size = (u32) sizeof(gaudi2->events_stat); in gaudi2_get_events_stat()
7512 return gaudi2->events_stat; in gaudi2_get_events_stat()
7518 u32 offset = (mmDCORE0_VDEC1_BRDG_CTRL_BASE - mmDCORE0_VDEC0_BRDG_CTRL_BASE) * in gaudi2_mmu_vdec_dcore_prepare()
7541 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_mmu_dcore_prepare()
7548 if (prop->edma_enabled_mask & BIT(edma_seq_base)) { in gaudi2_mmu_dcore_prepare()
7555 if (prop->edma_enabled_mask & BIT(edma_seq_base + 1)) { in gaudi2_mmu_dcore_prepare()
7565 * Sync Mngrs on dcores 1 - 3 are exposed to user, so must use user ASID in gaudi2_mmu_dcore_prepare()
7601 if (prop->decoder_enabled_mask & BIT(dcore_id * NUM_OF_DEC_PER_DCORE + vdec_id)) in gaudi2_mmu_dcore_prepare()
7609 u32 offset = (mmPCIE_VDEC1_BRDG_CTRL_BASE - mmPCIE_VDEC0_BRDG_CTRL_BASE) * shared_vdec_id; in gudi2_mmu_vdec_shared_prepare()
7630 u32 offset = (mmARC_FARM_ARC1_DUP_ENG_BASE - mmARC_FARM_ARC0_DUP_ENG_BASE) * arc_farm_id; in gudi2_mmu_arc_farm_arc_dup_eng_prepare()
7684 if (hdev->fw_components & FW_TYPE_BOOT_CPU) in gaudi2_arc_mmu_prepare_all()
7702 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_mmu_shared_prepare()
7730 if (prop->decoder_enabled_mask & BIT(NUM_OF_DCORES * NUM_OF_DEC_PER_DCORE + 0)) in gaudi2_mmu_shared_prepare()
7733 if (prop->decoder_enabled_mask & BIT(NUM_OF_DCORES * NUM_OF_DEC_PER_DCORE + 1)) in gaudi2_mmu_shared_prepare()
7734 gudi2_mmu_vdec_shared_prepare(hdev, 1, rw_asid, 0); in gaudi2_mmu_shared_prepare()
7750 struct gaudi2_tpc_mmu_data *mmu_data = ctx->data; in gaudi2_tpc_mmu_prepare()
7753 WREG32(mmDCORE0_TPC0_CFG_AXUSER_HB_ASID + offset, mmu_data->rw_asid); in gaudi2_tpc_mmu_prepare()
7755 WREG32(mmDCORE0_TPC0_QM_AXUSER_NONSECURED_HB_ASID + offset, mmu_data->rw_asid); in gaudi2_tpc_mmu_prepare()
7761 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_mmu_prepare()
7770 dev_crit(hdev->dev, "asid %u is too big\n", asid); in gaudi2_mmu_prepare()
7771 return -EINVAL; in gaudi2_mmu_prepare()
7774 if (!(gaudi2->hw_cap_initialized & HW_CAP_MMU_MASK)) in gaudi2_mmu_prepare()
7798 /* return in case of NIC status event - these events are received periodically and not as in is_info_event()
7820 dev_err_ratelimited(hdev->dev, "%s: %pV\n", in gaudi2_print_event()
7824 dev_err(hdev->dev, "%s: %pV\n", in gaudi2_print_event()
7839 if (!hl_is_fw_sw_ver_below(hdev, 1, 12)) in gaudi2_handle_ecc_event()
7842 ecc_address = le64_to_cpu(ecc_data->ecc_address); in gaudi2_handle_ecc_event()
7843 ecc_syndrome = le64_to_cpu(ecc_data->ecc_syndrom); in gaudi2_handle_ecc_event()
7844 memory_wrapper_idx = ecc_data->memory_wrapper_idx; in gaudi2_handle_ecc_event()
7847 block_id = le16_to_cpu(ecc_data->block_id); in gaudi2_handle_ecc_event()
7848 gaudi2_print_event(hdev, event_type, !ecc_data->is_critical, in gaudi2_handle_ecc_event()
7851 ecc_data->is_critical); in gaudi2_handle_ecc_event()
7853 gaudi2_print_event(hdev, event_type, !ecc_data->is_critical, in gaudi2_handle_ecc_event()
7855 ecc_address, ecc_syndrome, memory_wrapper_idx, ecc_data->is_critical); in gaudi2_handle_ecc_event()
7858 return !!ecc_data->is_critical; in gaudi2_handle_ecc_event()
7863 struct undefined_opcode_info *undef_opcode = &hdev->captured_err_info.undef_opcode; in handle_lower_qman_data_on_err()
7869 is_arc_cq = FIELD_GET(PDMA0_QM_CP_STS_CUR_CQ_MASK, cp_sts); /* 0 - legacy CQ, 1 - ARC_CQ */ in handle_lower_qman_data_on_err()
7887 dev_info(hdev->dev, in handle_lower_qman_data_on_err()
7891 if (undef_opcode->write_enable) { in handle_lower_qman_data_on_err()
7893 undef_opcode->timestamp = ktime_get(); in handle_lower_qman_data_on_err()
7894 undef_opcode->cq_addr = cq_ptr; in handle_lower_qman_data_on_err()
7895 undef_opcode->cq_size = cq_size; in handle_lower_qman_data_on_err()
7896 undef_opcode->engine_id = engine_id; in handle_lower_qman_data_on_err()
7897 undef_opcode->stream_id = QMAN_STREAMS; in handle_lower_qman_data_on_err()
7898 undef_opcode->write_enable = 0; in handle_lower_qman_data_on_err()
7909 glbl_sts_addr = qman_base + (mmDCORE0_TPC0_QM_GLBL_ERR_STS_0 - mmDCORE0_TPC0_QM_BASE); in gaudi2_handle_qman_err_generic()
7910 arb_err_addr = qman_base + (mmDCORE0_TPC0_QM_ARB_ERR_CAUSE - mmDCORE0_TPC0_QM_BASE); in gaudi2_handle_qman_err_generic()
7913 for (i = 0 ; i < QMAN_STREAMS + 1 ; i++) { in gaudi2_handle_qman_err_generic()
7984 hl_handle_razwi(hdev, (u64)razwi_hi << 32 | razwi_lo, &eng_id, 1, in gaudi2_razwi_rr_hbw_shared_printf_info()
7987 dev_err_ratelimited(hdev->dev, in gaudi2_razwi_rr_hbw_shared_printf_info()
7988 "%s-RAZWI SHARED RR HBW %s error, address %#llx, Initiator coordinates 0x%x\n", in gaudi2_razwi_rr_hbw_shared_printf_info()
8011 hl_handle_razwi(hdev, razwi_addr, &eng_id, 1, rd_wr_flag | HL_RAZWI_LBW, event_mask); in gaudi2_razwi_rr_lbw_shared_printf_info()
8012 dev_err_ratelimited(hdev->dev, in gaudi2_razwi_rr_lbw_shared_printf_info()
8013 …"%s-RAZWI SHARED RR LBW %s error, mstr_if 0x%llx, captured address 0x%llX Initiator coordinates 0x… in gaudi2_razwi_rr_lbw_shared_printf_info()
8027 (GAUDI2_DCORE0_ENGINE_ID_TPC_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0)); in gaudi2_razwi_calc_engine_id()
8030 return ((GAUDI2_DCORE0_ENGINE_ID_MME - GAUDI2_DCORE0_ENGINE_ID_EDMA_0) + in gaudi2_razwi_calc_engine_id()
8052 (GAUDI2_DCORE0_ENGINE_ID_DEC_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0)); in gaudi2_razwi_calc_engine_id()
8083 if (hdev->tpc_binning) { in gaudi2_ack_module_razwi_event_handler()
8084 binned_idx = __ffs(hdev->tpc_binning); in gaudi2_ack_module_razwi_event_handler()
8091 if (hl_is_fw_sw_ver_below(hdev, 1, 9) && in gaudi2_ack_module_razwi_event_handler()
8092 !hdev->asic_prop.fw_security_enabled && in gaudi2_ack_module_razwi_event_handler()
8093 ((module_idx == 0) || (module_idx == 1))) in gaudi2_ack_module_razwi_event_handler()
8156 if (hdev->decoder_binning) { in gaudi2_ack_module_razwi_event_handler()
8157 binned_idx = __ffs(hdev->decoder_binning); in gaudi2_ack_module_razwi_event_handler()
8187 (((s32)lbw_rtr_id - hbw_rtr_id) * DCORE_RTR_OFFSET); in gaudi2_ack_module_razwi_event_handler()
8232 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_check_if_razwi_happened()
8236 for (mod_idx = 0 ; mod_idx < (NUM_OF_TPC_PER_DCORE * NUM_OF_DCORES + 1) ; mod_idx++) { in gaudi2_check_if_razwi_happened()
8237 if (prop->tpc_enabled_mask & BIT(mod_idx)) in gaudi2_check_if_razwi_happened()
8249 if (prop->edma_enabled_mask & BIT(mod_idx)) in gaudi2_check_if_razwi_happened()
8258 if (hdev->nic_ports_mask & BIT(mod_idx)) in gaudi2_check_if_razwi_happened()
8259 gaudi2_ack_module_razwi_event_handler(hdev, RAZWI_NIC, mod_idx >> 1, 0, in gaudi2_check_if_razwi_happened()
8264 if (prop->decoder_enabled_mask & BIT(mod_idx)) in gaudi2_check_if_razwi_happened()
8288 PSOC_RAZWI_ENG_STR_SIZE - str_size, "%s", in gaudi2_psoc_razwi_get_engines()
8292 PSOC_RAZWI_ENG_STR_SIZE - str_size, " or %s", in gaudi2_psoc_razwi_get_engines()
8329 dev_err(hdev->dev, in gaudi2_handle_psoc_razwi_happened()
8343 dev_err(hdev->dev, in gaudi2_handle_psoc_razwi_happened()
8355 dev_err(hdev->dev, in gaudi2_handle_psoc_razwi_happened()
8367 dev_err(hdev->dev, in gaudi2_handle_psoc_razwi_happened()
8375 /* In common case the loop will break, when there is only one engine id, or in gaudi2_handle_psoc_razwi_happened()
8393 if (hdev->pldm || !(hdev->fw_components & FW_TYPE_LINUX)) { in gaudi2_ack_psoc_razwi_event_handler()
8401 dev_err_ratelimited(hdev->dev, in gaudi2_ack_psoc_razwi_event_handler()
8412 dev_err_ratelimited(hdev->dev, in gaudi2_ack_psoc_razwi_event_handler()
8417 if (hdev->pldm || !(hdev->fw_components & FW_TYPE_LINUX)) in gaudi2_ack_psoc_razwi_event_handler()
8453 index = event_type - GAUDI2_EVENT_TPC0_AXI_ERR_RSP; in gaudi2_handle_qm_sei_err()
8467 index = (event_type - GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE) / in gaudi2_handle_qm_sei_err()
8468 (GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE - in gaudi2_handle_qm_sei_err()
8475 index = event_type - GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP; in gaudi2_handle_qm_sei_err()
8481 index = event_type - GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE; in gaudi2_handle_qm_sei_err()
8514 index = event_type - GAUDI2_EVENT_TPC0_QM; in gaudi2_handle_qman_err()
8519 index = event_type - GAUDI2_EVENT_TPC6_QM; in gaudi2_handle_qman_err()
8524 index = event_type - GAUDI2_EVENT_TPC12_QM; in gaudi2_handle_qman_err()
8529 index = event_type - GAUDI2_EVENT_TPC18_QM; in gaudi2_handle_qman_err()
8559 index = 1; in gaudi2_handle_qman_err()
8616 /* Handle EDMA QM SEI here because there is no AXI error response event for EDMA */ in gaudi2_handle_qman_err()
8681 u64 intr_cause_data = le64_to_cpu(razwi_with_intr_cause->intr_cause.intr_cause_data); in gaudi2_handle_rot_err()
8703 u64 intr_cause_data = le64_to_cpu(razwi_with_intr_cause->intr_cause.intr_cause_data); in gaudi2_tpc_ack_interrupts()
8735 (dec_index - NUM_OF_VDEC_PER_DCORE * NUM_OF_DCORES); in gaudi2_handle_dec_err()
8752 /* Write 1 clear errors */ in gaudi2_handle_dec_err()
8821 /* check if RAZWI happened on WAP0/1 */ in gaudi2_handle_mme_wap_err()
8837 /* If an AXI read or write error is received, an error is reported and in gaudi2_handle_kdma_core_event()
8839 * register of the KDMA engine, the reported error is always HBW even if in gaudi2_handle_kdma_core_event()
8984 dev_err_ratelimited(hdev->dev, "PMMU page fault on va 0x%llx\n", addr); in gaudi2_handle_page_error()
8989 dev_err_ratelimited(hdev->dev, "HMMU page fault on va range 0x%llx - 0x%llx\n", in gaudi2_handle_page_error()
9016 dev_err_ratelimited(hdev->dev, "%s access error on va 0x%llx\n", in gaudi2_handle_access_error()
9036 else if (i == 1) in gaudi2_handle_mmu_spi_sei_generic()
9097 dev_err_ratelimited(hdev->dev, "SM%u err. err cause: CQ_INTR. queue index: %u\n", in gaudi2_handle_sm_err()
9122 dcore = 1; in get_hmmu_base()
9128 index_in_dcore = 1; in get_hmmu_base()
9132 dcore = 1; in get_hmmu_base()
9133 index_in_dcore = 1; in get_hmmu_base()
9162 dcore = 1; in get_hmmu_base()
9172 dcore = 1; in get_hmmu_base()
9188 index_in_dcore = 1; in get_hmmu_base()
9193 index_in_dcore = 1; in get_hmmu_base()
9241 dev_err_ratelimited(hdev->dev, in gaudi2_hbm_sei_handle_read_err()
9247 addr = le32_to_cpu(rd_err_data->dbg_rd_err_addr.rd_addr_val); in gaudi2_hbm_sei_handle_read_err()
9248 dev_err_ratelimited(hdev->dev, in gaudi2_hbm_sei_handle_read_err()
9258 if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) & in gaudi2_hbm_sei_handle_read_err()
9260 dev_err_ratelimited(hdev->dev, "Beat%d ECC SERR: DM: %#x, Syndrome: %#x\n", in gaudi2_hbm_sei_handle_read_err()
9262 le32_to_cpu(rd_err_data->dbg_rd_err_dm), in gaudi2_hbm_sei_handle_read_err()
9263 le32_to_cpu(rd_err_data->dbg_rd_err_syndrome)); in gaudi2_hbm_sei_handle_read_err()
9265 if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) & in gaudi2_hbm_sei_handle_read_err()
9267 dev_err_ratelimited(hdev->dev, "Beat%d ECC DERR: DM: %#x, Syndrome: %#x\n", in gaudi2_hbm_sei_handle_read_err()
9269 le32_to_cpu(rd_err_data->dbg_rd_err_dm), in gaudi2_hbm_sei_handle_read_err()
9270 le32_to_cpu(rd_err_data->dbg_rd_err_syndrome)); in gaudi2_hbm_sei_handle_read_err()
9275 if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) & in gaudi2_hbm_sei_handle_read_err()
9277 dev_err_ratelimited(hdev->dev, in gaudi2_hbm_sei_handle_read_err()
9280 le32_to_cpu(rd_err_data->dbg_rd_err_dm), in gaudi2_hbm_sei_handle_read_err()
9281 (le32_to_cpu(rd_err_data->dbg_rd_err_misc) & in gaudi2_hbm_sei_handle_read_err()
9287 dev_err_ratelimited(hdev->dev, "Beat%d DQ data:\n", beat); in gaudi2_hbm_sei_handle_read_err()
9288 dev_err_ratelimited(hdev->dev, "\t0x%08x\n", in gaudi2_hbm_sei_handle_read_err()
9289 le32_to_cpu(rd_err_data->dbg_rd_err_data[beat * 2])); in gaudi2_hbm_sei_handle_read_err()
9290 dev_err_ratelimited(hdev->dev, "\t0x%08x\n", in gaudi2_hbm_sei_handle_read_err()
9291 le32_to_cpu(rd_err_data->dbg_rd_err_data[beat * 2 + 1])); in gaudi2_hbm_sei_handle_read_err()
9300 struct hbm_sei_wr_cmd_address *wr_cmd_addr = wr_par_err_data->dbg_last_wr_cmds; in gaudi2_hbm_sei_print_wr_par_info()
9301 u32 i, curr_addr, derr = wr_par_err_data->dbg_derr; in gaudi2_hbm_sei_print_wr_par_info()
9303 dev_err_ratelimited(hdev->dev, "WRITE PARITY ERROR count: %d\n", err_cnt); in gaudi2_hbm_sei_print_wr_par_info()
9305 dev_err_ratelimited(hdev->dev, "CK-0 DERR: 0x%02x, CK-1 DERR: 0x%02x\n", in gaudi2_hbm_sei_print_wr_par_info()
9308 /* JIRA H6-3286 - the following prints may not be valid */ in gaudi2_hbm_sei_print_wr_par_info()
9309 dev_err_ratelimited(hdev->dev, "Last latched write commands addresses:\n"); in gaudi2_hbm_sei_print_wr_par_info()
9312 dev_err_ratelimited(hdev->dev, in gaudi2_hbm_sei_print_wr_par_info()
9325 __le32 *col_cmd = ca_par_err_data->dbg_col; in gaudi2_hbm_sei_print_ca_par_info()
9326 __le16 *row_cmd = ca_par_err_data->dbg_row; in gaudi2_hbm_sei_print_ca_par_info()
9329 dev_err_ratelimited(hdev->dev, "CA ERROR count: %d\n", err_cnt); in gaudi2_hbm_sei_print_ca_par_info()
9331 dev_err_ratelimited(hdev->dev, "Last latched C&R bus commands:\n"); in gaudi2_hbm_sei_print_ca_par_info()
9333 dev_err_ratelimited(hdev->dev, "cmd%u: ROW(0x%04x) COL(0x%05x)\n", i, in gaudi2_hbm_sei_print_ca_par_info()
9345 hbm_id = (event_type - GAUDI2_EVENT_HBM0_MC0_SEI_SEVERE) / 4; in gaudi2_handle_hbm_mc_sei_err()
9346 mc_id = ((event_type - GAUDI2_EVENT_HBM0_MC0_SEI_SEVERE) / 2) % 2; in gaudi2_handle_hbm_mc_sei_err()
9348 cause_idx = sei_data->hdr.sei_cause; in gaudi2_handle_hbm_mc_sei_err()
9349 if (cause_idx > GAUDI2_NUM_OF_HBM_SEI_CAUSE - 1) { in gaudi2_handle_hbm_mc_sei_err()
9356 gaudi2_print_event(hdev, event_type, !sei_data->hdr.is_critical, in gaudi2_handle_hbm_mc_sei_err()
9357 "System %s Error Interrupt - HBM(%u) MC(%u) MC_CH(%u) MC_PC(%u). Error cause: %s", in gaudi2_handle_hbm_mc_sei_err()
9358 sei_data->hdr.is_critical ? "Critical" : "Non-critical", in gaudi2_handle_hbm_mc_sei_err()
9359 hbm_id, mc_id, sei_data->hdr.mc_channel, sei_data->hdr.mc_pseudo_channel, in gaudi2_handle_hbm_mc_sei_err()
9362 /* Print error-specific info */ in gaudi2_handle_hbm_mc_sei_err()
9369 gaudi2_hbm_sei_print_ca_par_info(hdev, &sei_data->ca_parity_even_info, in gaudi2_handle_hbm_mc_sei_err()
9370 le32_to_cpu(sei_data->hdr.cnt)); in gaudi2_handle_hbm_mc_sei_err()
9375 gaudi2_hbm_sei_print_ca_par_info(hdev, &sei_data->ca_parity_odd_info, in gaudi2_handle_hbm_mc_sei_err()
9376 le32_to_cpu(sei_data->hdr.cnt)); in gaudi2_handle_hbm_mc_sei_err()
9381 gaudi2_hbm_sei_print_wr_par_info(hdev, &sei_data->wr_parity_info, in gaudi2_handle_hbm_mc_sei_err()
9382 le32_to_cpu(sei_data->hdr.cnt)); in gaudi2_handle_hbm_mc_sei_err()
9391 &sei_data->read_err_info, in gaudi2_handle_hbm_mc_sei_err()
9392 le32_to_cpu(sei_data->hdr.cnt)); in gaudi2_handle_hbm_mc_sei_err()
9399 require_hard_reset |= !!sei_data->hdr.is_critical; in gaudi2_handle_hbm_mc_sei_err()
9410 return 1; in gaudi2_handle_hbm_cattrip()
9422 dev_dbg(hdev->dev, "HBM spi event: notification cause(%s)\n", in gaudi2_handle_hbm_mc_spi()
9434 mutex_lock(&hdev->clk_throttling.lock); in gaudi2_print_clk_change_info()
9438 hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_POWER; in gaudi2_print_clk_change_info()
9439 hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_POWER; in gaudi2_print_clk_change_info()
9440 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].start = ktime_get(); in gaudi2_print_clk_change_info()
9441 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = zero_time; in gaudi2_print_clk_change_info()
9442 dev_dbg_ratelimited(hdev->dev, "Clock throttling due to power consumption\n"); in gaudi2_print_clk_change_info()
9446 hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_POWER; in gaudi2_print_clk_change_info()
9447 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = ktime_get(); in gaudi2_print_clk_change_info()
9448 dev_dbg_ratelimited(hdev->dev, "Power envelop is safe, back to optimal clock\n"); in gaudi2_print_clk_change_info()
9452 hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_THERMAL; in gaudi2_print_clk_change_info()
9453 hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_THERMAL; in gaudi2_print_clk_change_info()
9454 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].start = ktime_get(); in gaudi2_print_clk_change_info()
9455 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = zero_time; in gaudi2_print_clk_change_info()
9457 dev_info_ratelimited(hdev->dev, "Clock throttling due to overheating\n"); in gaudi2_print_clk_change_info()
9461 hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_THERMAL; in gaudi2_print_clk_change_info()
9462 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = ktime_get(); in gaudi2_print_clk_change_info()
9464 dev_info_ratelimited(hdev->dev, "Thermal envelop is safe, back to optimal clock\n"); in gaudi2_print_clk_change_info()
9468 dev_err(hdev->dev, "Received invalid clock change event %d\n", event_type); in gaudi2_print_clk_change_info()
9472 mutex_unlock(&hdev->clk_throttling.lock); in gaudi2_print_clk_change_info()
9478 struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ]; in gaudi2_print_out_of_sync_info()
9482 le32_to_cpu(sync_err->pi), le32_to_cpu(sync_err->ci), in gaudi2_print_out_of_sync_info()
9483 q->pi, atomic_read(&q->ci)); in gaudi2_print_out_of_sync_info()
9504 "pcie msi-x gen denied due to vector num check failure, vec(0x%X)", in gaudi2_handle_pcie_p2p_msix()
9519 cause = le64_to_cpu(drain_data->intr_cause.intr_cause_data); in gaudi2_handle_pcie_drain()
9520 lbw_rd = le64_to_cpu(drain_data->drain_rd_addr_lbw); in gaudi2_handle_pcie_drain()
9521 lbw_wr = le64_to_cpu(drain_data->drain_wr_addr_lbw); in gaudi2_handle_pcie_drain()
9522 hbw_rd = le64_to_cpu(drain_data->drain_rd_addr_hbw); in gaudi2_handle_pcie_drain()
9523 hbw_wr = le64_to_cpu(drain_data->drain_wr_addr_hbw); in gaudi2_handle_pcie_drain()
9526 dev_err_ratelimited(hdev->dev, in gaudi2_handle_pcie_drain()
9527 "PCIE AXI drain LBW completed, read_err %u, write_err %u\n", in gaudi2_handle_pcie_drain()
9532 if (cause & BIT_ULL(1)) { in gaudi2_handle_pcie_drain()
9533 dev_err_ratelimited(hdev->dev, in gaudi2_handle_pcie_drain()
9534 "PCIE AXI drain HBW completed, raddr %#llx, waddr %#llx\n", in gaudi2_handle_pcie_drain()
9549 dev_err_ratelimited(hdev->dev, "PSOC %s completed\n", in gaudi2_handle_psoc_drain()
9563 struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ]; in gaudi2_print_cpu_pkt_failure_info()
9567 le32_to_cpu(sync_err->pi), le32_to_cpu(sync_err->ci), q->pi, atomic_read(&q->ci)); in gaudi2_print_cpu_pkt_failure_info()
9577 intr_type = le32_to_cpu(data->intr_type); in hl_arc_event_handle()
9578 engine_id = le32_to_cpu(data->engine_id); in hl_arc_event_handle()
9579 payload = le64_to_cpu(data->payload); in hl_arc_event_handle()
9587 engine_id, intr_type, q->queue_index); in hl_arc_event_handle()
9588 return 1; in hl_arc_event_handle()
9602 index = event_type - GAUDI2_EVENT_TPC0_AXI_ERR_RSP; in event_id_to_engine_id()
9606 index = event_type - GAUDI2_EVENT_TPC0_QM; in event_id_to_engine_id()
9618 index = 1; in event_id_to_engine_id()
9648 index = event_type - GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE; in event_id_to_engine_id()
9652 index = (event_type - GAUDI2_EVENT_DEC0_SPI) >> 1; in event_id_to_engine_id()
9656 index = event_type - GAUDI2_EVENT_NIC0_AXI_ERROR_RESPONSE; in event_id_to_engine_id()
9659 index = event_type - GAUDI2_EVENT_NIC0_QM0; in event_id_to_engine_id()
9662 index = event_type - GAUDI2_EVENT_NIC0_BMON_SPMU; in event_id_to_engine_id()
9665 index = (event_type - GAUDI2_EVENT_TPC0_BMON_SPMU) >> 1; in event_id_to_engine_id()
9718 return GAUDI2_DCORE1_ENGINE_ID_TPC_0 + index - TPC_ID_DCORE1_TPC0; in event_id_to_engine_id()
9720 return GAUDI2_DCORE2_ENGINE_ID_TPC_0 + index - TPC_ID_DCORE2_TPC0; in event_id_to_engine_id()
9722 return GAUDI2_DCORE3_ENGINE_ID_TPC_0 + index - TPC_ID_DCORE3_TPC0; in event_id_to_engine_id()
9762 hdev->eq_heartbeat_received = true; in hl_eq_heartbeat_event_handle()
9767 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_handle_eqe()
9773 ctl = le32_to_cpu(eq_entry->hdr.ctl); in gaudi2_handle_eqe()
9777 dev_err(hdev->dev, "Event type %u exceeds maximum of %u", in gaudi2_handle_eqe()
9778 event_type, GAUDI2_EVENT_SIZE - 1); in gaudi2_handle_eqe()
9782 gaudi2->events_stat[event_type]++; in gaudi2_handle_eqe()
9783 gaudi2->events_stat_aggregate[event_type]++; in gaudi2_handle_eqe()
9791 reset_required = gaudi2_handle_ecc_event(hdev, event_type, &eq_entry->ecc_data); in gaudi2_handle_eqe()
9792 is_critical = eq_entry->ecc_data.is_critical; in gaudi2_handle_eqe()
9824 index = event_type - GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE; in gaudi2_handle_eqe()
9826 &eq_entry->razwi_with_intr_cause, &event_mask); in gaudi2_handle_eqe()
9832 index = event_type - GAUDI2_EVENT_TPC0_AXI_ERR_RSP; in gaudi2_handle_eqe()
9834 &eq_entry->razwi_with_intr_cause, &event_mask); in gaudi2_handle_eqe()
9840 index = event_type - GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE; in gaudi2_handle_eqe()
9870 index = (event_type - GAUDI2_EVENT_TPC0_KERNEL_ERR) / in gaudi2_handle_eqe()
9871 (GAUDI2_EVENT_TPC1_KERNEL_ERR - GAUDI2_EVENT_TPC0_KERNEL_ERR); in gaudi2_handle_eqe()
9873 &eq_entry->razwi_with_intr_cause, &event_mask); in gaudi2_handle_eqe()
9887 index = (event_type - GAUDI2_EVENT_DEC0_SPI) / in gaudi2_handle_eqe()
9888 (GAUDI2_EVENT_DEC1_SPI - GAUDI2_EVENT_DEC0_SPI); in gaudi2_handle_eqe()
9897 index = (event_type - GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE) / in gaudi2_handle_eqe()
9898 (GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE - in gaudi2_handle_eqe()
9909 index = (event_type - GAUDI2_EVENT_MME0_QMAN_SW_ERROR) / in gaudi2_handle_eqe()
9910 (GAUDI2_EVENT_MME1_QMAN_SW_ERROR - in gaudi2_handle_eqe()
9920 index = (event_type - GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID) / in gaudi2_handle_eqe()
9921 (GAUDI2_EVENT_MME1_WAP_SOURCE_RESULT_INVALID - in gaudi2_handle_eqe()
9930 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); in gaudi2_handle_eqe()
9936 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); in gaudi2_handle_eqe()
9942 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); in gaudi2_handle_eqe()
9948 le64_to_cpu(eq_entry->intr_cause.intr_cause_data), &event_mask); in gaudi2_handle_eqe()
9964 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); in gaudi2_handle_eqe()
9971 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); in gaudi2_handle_eqe()
9983 if (gaudi2_handle_hbm_mc_sei_err(hdev, event_type, &eq_entry->sei_data)) { in gaudi2_handle_eqe()
9992 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); in gaudi2_handle_eqe()
9998 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); in gaudi2_handle_eqe()
10003 error_count = gaudi2_handle_pcie_drain(hdev, &eq_entry->pcie_drain_ind_data); in gaudi2_handle_eqe()
10006 if (hl_is_fw_sw_ver_equal_or_greater(hdev, 1, 13)) in gaudi2_handle_eqe()
10012 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); in gaudi2_handle_eqe()
10121 gaudi2_print_out_of_sync_info(hdev, event_type, &eq_entry->pkt_sync_err); in gaudi2_handle_eqe()
10130 /* Do nothing- FW will handle it */ in gaudi2_handle_eqe()
10139 index = event_type - GAUDI2_EVENT_SM0_AXI_ERROR_RESPONSE; in gaudi2_handle_eqe()
10150 dev_info(hdev->dev, "CPLD shutdown cause, reset reason: 0x%llx\n", in gaudi2_handle_eqe()
10151 le64_to_cpu(eq_entry->data[0])); in gaudi2_handle_eqe()
10156 dev_err(hdev->dev, "CPLD shutdown event, reset reason: 0x%llx\n", in gaudi2_handle_eqe()
10157 le64_to_cpu(eq_entry->data[0])); in gaudi2_handle_eqe()
10163 gaudi2_print_cpu_pkt_failure_info(hdev, event_type, &eq_entry->pkt_sync_err); in gaudi2_handle_eqe()
10170 error_count = hl_arc_event_handle(hdev, event_type, &eq_entry->arc_data); in gaudi2_handle_eqe()
10188 dev_info_ratelimited(hdev->dev, "%s event received\n", in gaudi2_handle_eqe()
10198 dev_err_ratelimited(hdev->dev, "Cannot find handler for event %d\n", in gaudi2_handle_eqe()
10223 if (hdev->hard_reset_on_fw_events || in gaudi2_handle_eqe()
10224 (hdev->asic_prop.fw_security_enabled && is_critical)) in gaudi2_handle_eqe()
10238 if (hdev->asic_prop.fw_security_enabled && is_critical) { in gaudi2_handle_eqe()
10260 ctl |= FIELD_PREP(GAUDI2_PKT_LIN_DMA_CTL_MEMSET_MASK, 1); in gaudi2_memset_memory_chunk_using_edma_qm()
10261 ctl |= FIELD_PREP(GAUDI2_PKT_LIN_DMA_CTL_WRCOMP_MASK, 1); in gaudi2_memset_memory_chunk_using_edma_qm()
10262 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_EB_MASK, 1); in gaudi2_memset_memory_chunk_using_edma_qm()
10264 lin_dma_pkt->ctl = cpu_to_le32(ctl); in gaudi2_memset_memory_chunk_using_edma_qm()
10265 lin_dma_pkt->src_addr = cpu_to_le64(val); in gaudi2_memset_memory_chunk_using_edma_qm()
10266 lin_dma_pkt->dst_addr = cpu_to_le64(addr); in gaudi2_memset_memory_chunk_using_edma_qm()
10267 lin_dma_pkt->tsize = cpu_to_le32(size); in gaudi2_memset_memory_chunk_using_edma_qm()
10273 dev_err(hdev->dev, "Failed to send lin dma packet to H/W queue %d\n", in gaudi2_memset_memory_chunk_using_edma_qm()
10288 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_memset_device_memory()
10293 if (prop->edma_enabled_mask == 0) { in gaudi2_memset_device_memory()
10294 dev_info(hdev->dev, "non of the EDMA engines is enabled - skip dram scrubbing\n"); in gaudi2_memset_device_memory()
10295 return -EIO; in gaudi2_memset_device_memory()
10298 sob_offset = hdev->asic_prop.first_available_user_sob[0] * 4; in gaudi2_memset_device_memory()
10301 comp_val = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_INC_MASK, 1) | in gaudi2_memset_device_memory()
10302 FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_VAL_MASK, 1); in gaudi2_memset_device_memory()
10303 mmubp = FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_WR_MASK, 1) | in gaudi2_memset_device_memory()
10304 FIELD_PREP(ARC_FARM_KDMA_CTX_AXUSER_HB_MMU_BP_RD_MASK, 1); in gaudi2_memset_device_memory()
10313 return -ENOMEM; in gaudi2_memset_device_memory()
10316 * set mmu bypass for the scrubbing - all ddmas are configured the same so save in gaudi2_memset_device_memory()
10327 if (!(prop->edma_enabled_mask & BIT(edma_bit))) in gaudi2_memset_device_memory()
10350 if (!(prop->edma_enabled_mask & BIT(edma_bit))) in gaudi2_memset_device_memory()
10353 chunk_size = min_t(u64, SZ_2G, end_addr - cur_addr); in gaudi2_memset_device_memory()
10373 dev_err(hdev->dev, "DMA Timeout during HBM scrubbing\n"); in gaudi2_memset_device_memory()
10382 if (!(prop->edma_enabled_mask & BIT(edma_bit))) in gaudi2_memset_device_memory()
10403 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_scrub_device_dram()
10404 u64 size = prop->dram_end_address - prop->dram_user_base_address; in gaudi2_scrub_device_dram()
10406 rc = gaudi2_memset_device_memory(hdev, prop->dram_user_base_address, size, val); in gaudi2_scrub_device_dram()
10409 dev_err(hdev->dev, "Failed to scrub dram, address: 0x%llx size: %llu\n", in gaudi2_scrub_device_dram()
10410 prop->dram_user_base_address, size); in gaudi2_scrub_device_dram()
10417 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_scrub_device_mem()
10418 u64 val = hdev->memory_scrub_val; in gaudi2_scrub_device_mem()
10421 if (!hdev->memory_scrub) in gaudi2_scrub_device_mem()
10425 addr = prop->sram_user_base_address; in gaudi2_scrub_device_mem()
10426 size = hdev->pldm ? 0x10000 : (prop->sram_size - SRAM_USER_BASE_OFFSET); in gaudi2_scrub_device_mem()
10427 dev_dbg(hdev->dev, "Scrubbing SRAM: 0x%09llx - 0x%09llx, val: 0x%llx\n", in gaudi2_scrub_device_mem()
10431 dev_err(hdev->dev, "scrubbing SRAM failed (%d)\n", rc); in gaudi2_scrub_device_mem()
10438 dev_err(hdev->dev, "scrubbing DRAM failed (%d)\n", rc); in gaudi2_scrub_device_mem()
10451 offset = hdev->asic_prop.first_available_cq[0] * 4; in gaudi2_restore_user_sm_registers()
10458 size = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 - in gaudi2_restore_user_sm_registers()
10475 size = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 - mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0; in gaudi2_restore_user_sm_registers()
10477 for (dcore_id = 1 ; dcore_id < NUM_OF_DCORES ; dcore_id++) { in gaudi2_restore_user_sm_registers()
10493 offset = hdev->asic_prop.first_available_user_mon[0] * 4; in gaudi2_restore_user_sm_registers()
10495 val = 1 << DCORE0_SYNC_MNGR_OBJS_MON_STATUS_PROT_SHIFT; in gaudi2_restore_user_sm_registers()
10496 size = mmDCORE0_SYNC_MNGR_OBJS_SM_SEC_0 - (mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0 + offset); in gaudi2_restore_user_sm_registers()
10506 size = mmDCORE0_SYNC_MNGR_OBJS_SM_SEC_0 - mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0; in gaudi2_restore_user_sm_registers()
10508 for (dcore_id = 1 ; dcore_id < NUM_OF_DCORES ; dcore_id++) { in gaudi2_restore_user_sm_registers()
10515 offset = hdev->asic_prop.first_available_user_sob[0] * 4; in gaudi2_restore_user_sm_registers()
10518 size = mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 - in gaudi2_restore_user_sm_registers()
10525 size = mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 - mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0; in gaudi2_restore_user_sm_registers()
10527 for (dcore_id = 1 ; dcore_id < NUM_OF_DCORES ; dcore_id++) { in gaudi2_restore_user_sm_registers()
10586 u64 block_base = cfg_ctx->base + block_idx * cfg_ctx->block_off; in gaudi2_init_block_instances()
10590 for (i = 0 ; i < cfg_ctx->instances ; i++) { in gaudi2_init_block_instances()
10591 seq = block_idx * cfg_ctx->instances + i; in gaudi2_init_block_instances()
10594 if (!(cfg_ctx->enabled_mask & BIT_ULL(seq))) in gaudi2_init_block_instances()
10597 cfg_ctx->instance_cfg_fn(hdev, block_base + i * cfg_ctx->instance_off, in gaudi2_init_block_instances()
10598 cfg_ctx->data); in gaudi2_init_block_instances()
10607 cfg_ctx->enabled_mask = mask; in gaudi2_init_blocks_with_mask()
10609 for (i = 0 ; i < cfg_ctx->blocks ; i++) in gaudi2_init_blocks_with_mask()
10630 dev_err(hdev->dev, "No ctx available\n"); in gaudi2_debugfs_read_dma()
10631 return -EINVAL; in gaudi2_debugfs_read_dma()
10638 dev_err(hdev->dev, "Failed to allocate memory for KDMA read\n"); in gaudi2_debugfs_read_dma()
10639 rc = -ENOMEM; in gaudi2_debugfs_read_dma()
10647 dev_err(hdev->dev, "Failed to reserve vmem on asic\n"); in gaudi2_debugfs_read_dma()
10648 rc = -ENOMEM; in gaudi2_debugfs_read_dma()
10653 mutex_lock(&hdev->mmu_lock); in gaudi2_debugfs_read_dma()
10657 dev_err(hdev->dev, "Failed to create mapping on asic mmu\n"); in gaudi2_debugfs_read_dma()
10663 ctx->asid, reserved_va_base, SZ_2M); in gaudi2_debugfs_read_dma()
10669 mutex_unlock(&hdev->mmu_lock); in gaudi2_debugfs_read_dma()
10672 gaudi2_kdma_set_mmbp_asid(hdev, false, ctx->asid); in gaudi2_debugfs_read_dma()
10693 size_left -= SZ_2M; in gaudi2_debugfs_read_dma()
10698 mutex_lock(&hdev->mmu_lock); in gaudi2_debugfs_read_dma()
10705 ctx->asid, reserved_va_base, SZ_2M); in gaudi2_debugfs_read_dma()
10708 mutex_unlock(&hdev->mmu_lock); in gaudi2_debugfs_read_dma()
10720 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_internal_cb_pool_init()
10723 if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU)) in gaudi2_internal_cb_pool_init()
10726 hdev->internal_cb_pool_virt_addr = hl_asic_dma_alloc_coherent(hdev, in gaudi2_internal_cb_pool_init()
10728 &hdev->internal_cb_pool_dma_addr, in gaudi2_internal_cb_pool_init()
10731 if (!hdev->internal_cb_pool_virt_addr) in gaudi2_internal_cb_pool_init()
10732 return -ENOMEM; in gaudi2_internal_cb_pool_init()
10737 hdev->internal_cb_pool = gen_pool_create(min_alloc_order, -1); in gaudi2_internal_cb_pool_init()
10738 if (!hdev->internal_cb_pool) { in gaudi2_internal_cb_pool_init()
10739 dev_err(hdev->dev, "Failed to create internal CB pool\n"); in gaudi2_internal_cb_pool_init()
10740 rc = -ENOMEM; in gaudi2_internal_cb_pool_init()
10744 rc = gen_pool_add(hdev->internal_cb_pool, (uintptr_t) hdev->internal_cb_pool_virt_addr, in gaudi2_internal_cb_pool_init()
10745 HOST_SPACE_INTERNAL_CB_SZ, -1); in gaudi2_internal_cb_pool_init()
10747 dev_err(hdev->dev, "Failed to add memory to internal CB pool\n"); in gaudi2_internal_cb_pool_init()
10748 rc = -EFAULT; in gaudi2_internal_cb_pool_init()
10752 hdev->internal_cb_va_base = hl_reserve_va_block(hdev, ctx, HL_VA_RANGE_TYPE_HOST, in gaudi2_internal_cb_pool_init()
10755 if (!hdev->internal_cb_va_base) { in gaudi2_internal_cb_pool_init()
10756 rc = -ENOMEM; in gaudi2_internal_cb_pool_init()
10760 mutex_lock(&hdev->mmu_lock); in gaudi2_internal_cb_pool_init()
10762 rc = hl_mmu_map_contiguous(ctx, hdev->internal_cb_va_base, hdev->internal_cb_pool_dma_addr, in gaudi2_internal_cb_pool_init()
10771 mutex_unlock(&hdev->mmu_lock); in gaudi2_internal_cb_pool_init()
10776 hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); in gaudi2_internal_cb_pool_init()
10778 mutex_unlock(&hdev->mmu_lock); in gaudi2_internal_cb_pool_init()
10779 hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); in gaudi2_internal_cb_pool_init()
10781 gen_pool_destroy(hdev->internal_cb_pool); in gaudi2_internal_cb_pool_init()
10783 hl_asic_dma_free_coherent(hdev, HOST_SPACE_INTERNAL_CB_SZ, hdev->internal_cb_pool_virt_addr, in gaudi2_internal_cb_pool_init()
10784 hdev->internal_cb_pool_dma_addr); in gaudi2_internal_cb_pool_init()
10791 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_internal_cb_pool_fini()
10793 if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU)) in gaudi2_internal_cb_pool_fini()
10796 mutex_lock(&hdev->mmu_lock); in gaudi2_internal_cb_pool_fini()
10797 hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); in gaudi2_internal_cb_pool_fini()
10798 hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); in gaudi2_internal_cb_pool_fini()
10800 mutex_unlock(&hdev->mmu_lock); in gaudi2_internal_cb_pool_fini()
10802 gen_pool_destroy(hdev->internal_cb_pool); in gaudi2_internal_cb_pool_fini()
10804 hl_asic_dma_free_coherent(hdev, HOST_SPACE_INTERNAL_CB_SZ, hdev->internal_cb_pool_virt_addr, in gaudi2_internal_cb_pool_fini()
10805 hdev->internal_cb_pool_dma_addr); in gaudi2_internal_cb_pool_fini()
10816 struct hl_device *hdev = ctx->hdev; in gaudi2_map_virtual_msix_doorbell_memory()
10817 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_map_virtual_msix_doorbell_memory()
10818 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_map_virtual_msix_doorbell_memory()
10822 gaudi2->virt_msix_db_dma_addr, prop->pmmu.page_size, true); in gaudi2_map_virtual_msix_doorbell_memory()
10824 dev_err(hdev->dev, "Failed to map VA %#llx for virtual MSI-X doorbell memory\n", in gaudi2_map_virtual_msix_doorbell_memory()
10832 struct hl_device *hdev = ctx->hdev; in gaudi2_unmap_virtual_msix_doorbell_memory()
10833 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_unmap_virtual_msix_doorbell_memory()
10837 prop->pmmu.page_size, true); in gaudi2_unmap_virtual_msix_doorbell_memory()
10839 dev_err(hdev->dev, "Failed to unmap VA %#llx of virtual MSI-X doorbell memory\n", in gaudi2_unmap_virtual_msix_doorbell_memory()
10847 if (ctx->asid == HL_KERNEL_ASID_ID) in gaudi2_ctx_init()
10850 rc = gaudi2_mmu_prepare(ctx->hdev, ctx->asid); in gaudi2_ctx_init()
10857 if (ctx->hdev->reset_upon_device_release) in gaudi2_ctx_init()
10858 gaudi2_restore_nic_qm_registers(ctx->hdev); in gaudi2_ctx_init()
10860 gaudi2_restore_user_registers(ctx->hdev); in gaudi2_ctx_init()
10862 rc = gaudi2_internal_cb_pool_init(ctx->hdev, ctx); in gaudi2_ctx_init()
10868 gaudi2_internal_cb_pool_fini(ctx->hdev, ctx); in gaudi2_ctx_init()
10875 if (ctx->asid == HL_KERNEL_ASID_ID) in gaudi2_ctx_fini()
10878 gaudi2_internal_cb_pool_fini(ctx->hdev, ctx); in gaudi2_ctx_fini()
10885 struct hl_device *hdev = cs->ctx->hdev; in gaudi2_pre_schedule_cs()
10886 int index = cs->sequence & (hdev->asic_prop.max_pending_cs - 1); in gaudi2_pre_schedule_cs()
10897 * generates MSI-X interrupt. in gaudi2_pre_schedule_cs()
10901 mon_payload = (1 << CQ_ENTRY_SHADOW_INDEX_VALID_SHIFT) | in gaudi2_pre_schedule_cs()
10902 (1 << CQ_ENTRY_READY_SHIFT) | index; in gaudi2_pre_schedule_cs()
10905 cs->jobs_cnt); in gaudi2_pre_schedule_cs()
10921 pkt = (struct packet_msg_short *) (uintptr_t) (cb->kernel_address + size); in gaudi2_gen_signal_cb()
10924 /* Inc by 1, Mode ADD */ in gaudi2_gen_signal_cb()
10925 value = FIELD_PREP(GAUDI2_PKT_SHORT_VAL_SOB_SYNC_VAL_MASK, 1); in gaudi2_gen_signal_cb()
10926 value |= FIELD_PREP(GAUDI2_PKT_SHORT_VAL_SOB_MOD_MASK, 1); in gaudi2_gen_signal_cb()
10929 ctl |= FIELD_PREP(GAUDI2_PKT_SHORT_CTL_BASE_MASK, 1); /* SOB base */ in gaudi2_gen_signal_cb()
10932 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 1); in gaudi2_gen_signal_cb()
10934 pkt->value = cpu_to_le32(value); in gaudi2_gen_signal_cb()
10935 pkt->ctl = cpu_to_le32(ctl); in gaudi2_gen_signal_cb()
10952 pkt->value = cpu_to_le32(value); in gaudi2_add_mon_msg_short()
10953 pkt->ctl = cpu_to_le32(ctl); in gaudi2_add_mon_msg_short()
10965 dev_err(hdev->dev, "sob_base %u (mask %#x) is not valid\n", sob_base, sob_mask); in gaudi2_add_arm_monitor_pkt()
10980 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 1); in gaudi2_add_arm_monitor_pkt()
10982 pkt->value = cpu_to_le32(value); in gaudi2_add_arm_monitor_pkt()
10983 pkt->ctl = cpu_to_le32(ctl); in gaudi2_add_arm_monitor_pkt()
10994 cfg = FIELD_PREP(GAUDI2_PKT_FENCE_CFG_DEC_VAL_MASK, 1); in gaudi2_add_fence_pkt()
10995 cfg |= FIELD_PREP(GAUDI2_PKT_FENCE_CFG_TARGET_VAL_MASK, 1); in gaudi2_add_fence_pkt()
11000 ctl |= FIELD_PREP(GAUDI2_PKT_CTL_MB_MASK, 1); in gaudi2_add_fence_pkt()
11002 pkt->cfg = cpu_to_le32(cfg); in gaudi2_add_fence_pkt()
11003 pkt->ctl = cpu_to_le32(ctl); in gaudi2_add_fence_pkt()
11010 struct hl_cb *cb = prop->data; in gaudi2_gen_wait_cb()
11011 void *buf = (void *) (uintptr_t) (cb->kernel_address); in gaudi2_gen_wait_cb()
11014 u32 stream_index, size = prop->size; in gaudi2_gen_wait_cb()
11017 stream_index = prop->q_idx % 4; in gaudi2_gen_wait_cb()
11018 fence_addr = CFG_BASE + gaudi2_qm_blocks_bases[prop->q_idx] + in gaudi2_gen_wait_cb()
11028 msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + prop->mon_id * 4) - in gaudi2_gen_wait_cb()
11034 msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + prop->mon_id * 4) - in gaudi2_gen_wait_cb()
11043 msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + prop->mon_id * 4) - in gaudi2_gen_wait_cb()
11046 size += gaudi2_add_mon_msg_short(buf + size, 1, msg_addr_offset); in gaudi2_gen_wait_cb()
11049 msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + prop->mon_id * 4) - monitor_base; in gaudi2_gen_wait_cb()
11051 size += gaudi2_add_arm_monitor_pkt(hdev, buf + size, prop->sob_base, prop->sob_mask, in gaudi2_gen_wait_cb()
11052 prop->sob_val, msg_addr_offset); in gaudi2_gen_wait_cb()
11064 dev_dbg(hdev->dev, "reset SOB, q_idx: %d, sob_id: %d\n", hw_sob->q_idx, hw_sob->sob_id); in gaudi2_reset_sob()
11066 WREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + hw_sob->sob_id * 4, 0); in gaudi2_reset_sob()
11068 kref_init(&hw_sob->kref); in gaudi2_reset_sob()
11091 return -EINVAL; in gaudi2_collective_wait_create_jobs()
11095 * hl_mmu_scramble - converts a dram (non power of 2) page-size aligned address
11096 * to DMMU page-size address (64MB) before mapping it in
11108 * PA1 0x3000000 VA1 0x9C000000 SVA1= (VA1/48M)*64M 0xD0000000 <- PA1/48M 0x1
11109 * PA2 0x9000000 VA2 0x9F000000 SVA2= (VA2/48M)*64M 0xD4000000 <- PA2/48M 0x3
11114 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_mmu_scramble_addr()
11122 divisor = prop->num_functional_hbms * GAUDI2_HBM_MMU_SCRM_MEM_SIZE; in gaudi2_mmu_scramble_addr()
11134 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_mmu_descramble_addr()
11142 divisor = prop->num_functional_hbms * GAUDI2_HBM_MMU_SCRM_MEM_SIZE; in gaudi2_mmu_descramble_addr()
11158 dev_err(hdev->dev, "Unexpected core number %d for DEC\n", core_id); in gaudi2_get_dec_base_addr()
11179 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_get_hw_block_id()
11183 if (block_addr == CFG_BASE + gaudi2->mapped_blocks[i].address) { in gaudi2_get_hw_block_id()
11186 *block_size = gaudi2->mapped_blocks[i].size; in gaudi2_get_hw_block_id()
11191 dev_err(hdev->dev, "Invalid block address %#llx", block_addr); in gaudi2_get_hw_block_id()
11193 return -EINVAL; in gaudi2_get_hw_block_id()
11199 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_block_mmap()
11205 dev_err(hdev->dev, "Invalid block id %u", block_id); in gaudi2_block_mmap()
11206 return -EINVAL; in gaudi2_block_mmap()
11210 if (block_size != gaudi2->mapped_blocks[block_id].size) { in gaudi2_block_mmap()
11211 dev_err(hdev->dev, "Invalid block size %u", block_size); in gaudi2_block_mmap()
11212 return -EINVAL; in gaudi2_block_mmap()
11215 offset_in_bar = CFG_BASE + gaudi2->mapped_blocks[block_id].address - STM_FLASH_BASE_ADDR; in gaudi2_block_mmap()
11217 address = pci_resource_start(hdev->pdev, SRAM_CFG_BAR_ID) + offset_in_bar; in gaudi2_block_mmap()
11222 rc = remap_pfn_range(vma, vma->vm_start, address >> PAGE_SHIFT, in gaudi2_block_mmap()
11223 block_size, vma->vm_page_prot); in gaudi2_block_mmap()
11225 dev_err(hdev->dev, "remap_pfn_range error %d", rc); in gaudi2_block_mmap()
11232 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_enable_events_from_fw()
11234 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi2_enable_events_from_fw()
11235 u32 irq_handler_offset = le32_to_cpu(dyn_regs->gic_host_ints_irq); in gaudi2_enable_events_from_fw()
11237 if (gaudi2->hw_cap_initialized & HW_CAP_CPU_Q) in gaudi2_enable_events_from_fw()
11297 return -EINVAL; in gaudi2_get_mmu_base()
11306 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_ack_mmu_error()
11309 if (!(gaudi2->hw_cap_initialized & mmu_id)) in gaudi2_ack_mmu_error()
11359 default: return -EINVAL; in gaudi2_map_pll_idx_to_fw_idx()
11402 hdev->state_dump_specs.props = gaudi2_state_dump_specs_props; in gaudi2_state_dump_init()
11403 hdev->state_dump_specs.funcs = gaudi2_state_dump_funcs; in gaudi2_state_dump_init()
11426 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_mmu_get_real_page_size()
11430 if (page_size % mmu_prop->page_size) in gaudi2_mmu_get_real_page_size()
11433 *real_page_size = mmu_prop->page_size; in gaudi2_mmu_get_real_page_size()
11437 if ((page_size % prop->dram_page_size) || (prop->dram_page_size > mmu_prop->page_size)) in gaudi2_mmu_get_real_page_size()
11448 *real_page_size = prop->dram_page_size; in gaudi2_mmu_get_real_page_size()
11453 dev_err(hdev->dev, "page size of %u is not %uKB aligned, can't map\n", in gaudi2_mmu_get_real_page_size()
11454 page_size, mmu_prop->page_size >> 10); in gaudi2_mmu_get_real_page_size()
11455 return -EFAULT; in gaudi2_mmu_get_real_page_size()
11460 return -EOPNOTSUPP; in gaudi2_get_monitor_dump()
11465 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_send_device_activity()
11467 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) in gaudi2_send_device_activity()
11575 hdev->asic_funcs = &gaudi2_funcs; in gaudi2_set_asic_funcs()