Lines Matching +full:max +full:- +full:outbound +full:- +full:regions
1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2020-2022 HabanaLabs, Ltd.
45 * since the code already has built-in support for binning of up to MAX_FAULTY_TPCS TPCs
47 * for MAX faulty TPCs which reflects the cluster binning requirements
126 #define GAUDI2_PMMU_SPI_SEI_ENABLE_MASK GENMASK(GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE - 2, 0)
127 #define GAUDI2_HMMU_SPI_SEI_ENABLE_MASK GENMASK(GAUDI2_NUM_OF_MMU_SPI_SEI_CAUSE - 1, 0)
131 #define GAUDI2_VDEC_MSIX_ENTRIES (GAUDI2_IRQ_NUM_SHARED_DEC1_ABNRM - \
134 #define ENGINE_ID_DCORE_OFFSET (GAUDI2_DCORE1_ENGINE_ID_EDMA_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0)
164 /* HW scrambles only bits 0-25 */
794 "FENCE 0 inc over max value and clipped",
795 "FENCE 1 inc over max value and clipped",
796 "FENCE 2 inc over max value and clipped",
797 "FENCE 3 inc over max value and clipped",
815 "FENCE 0 inc over max value and clipped",
816 "FENCE 1 inc over max value and clipped",
817 "FENCE 2 inc over max value and clipped",
818 "FENCE 3 inc over max value and clipped",
2112 * and read global errors. Most HW blocks are addressable and those who aren't (N/A)-
2135 {HBM_MC_SPI_THR_ENG_MASK, "temperature-based throttling engaged"},
2136 {HBM_MC_SPI_THR_DIS_ENG_MASK, "temperature-based throttling disengaged"},
2163 {"mmu rei0", -1}, /* no clear register bit */
2164 {"mmu rei1", -1}, /* no clear register bit */
2165 {"stlb rei0", -1}, /* no clear register bit */
2166 {"stlb rei1", -1}, /* no clear register bit */
2235 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_iterate_tpcs()
2240 ctx->rc = 0; in gaudi2_iterate_tpcs()
2246 if (!(prop->tpc_enabled_mask & BIT(tpc_seq))) in gaudi2_iterate_tpcs()
2251 ctx->fn(hdev, dcore, inst, offset, ctx); in gaudi2_iterate_tpcs()
2252 if (ctx->rc) { in gaudi2_iterate_tpcs()
2253 dev_err(hdev->dev, "TPC iterator failed for DCORE%d TPC%d\n", in gaudi2_iterate_tpcs()
2260 if (!(prop->tpc_enabled_mask & BIT(TPC_ID_DCORE0_TPC6))) in gaudi2_iterate_tpcs()
2264 offset = DCORE_TPC_OFFSET * (NUM_DCORE0_TPC - 1); in gaudi2_iterate_tpcs()
2265 ctx->fn(hdev, 0, NUM_DCORE0_TPC - 1, offset, ctx); in gaudi2_iterate_tpcs()
2266 if (ctx->rc) in gaudi2_iterate_tpcs()
2267 dev_err(hdev->dev, "TPC iterator failed for DCORE0 TPC6\n"); in gaudi2_iterate_tpcs()
2280 struct asic_fixed_properties *prop = &hdev->asic_prop; in set_number_of_functional_hbms()
2281 u8 faulty_hbms = hweight64(hdev->dram_binning); in set_number_of_functional_hbms()
2285 dev_dbg(hdev->dev, "All HBM are in use (no binning)\n"); in set_number_of_functional_hbms()
2286 prop->num_functional_hbms = GAUDI2_HBM_NUM; in set_number_of_functional_hbms()
2297 dev_err(hdev->dev, in set_number_of_functional_hbms()
2298 "HBM binning supports max of %d faulty HBMs, supplied mask 0x%llx.\n", in set_number_of_functional_hbms()
2299 MAX_FAULTY_HBMS, hdev->dram_binning); in set_number_of_functional_hbms()
2300 return -EINVAL; in set_number_of_functional_hbms()
2305 * GAUDI2_HBM_NUM - 1. in set_number_of_functional_hbms()
2307 prop->num_functional_hbms = GAUDI2_HBM_NUM - faulty_hbms; in set_number_of_functional_hbms()
2313 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_set_dram_properties()
2319 return -EINVAL; in gaudi2_set_dram_properties()
2326 basic_hbm_page_size = prop->num_functional_hbms * SZ_8M; in gaudi2_set_dram_properties()
2327 prop->dram_page_size = GAUDI2_COMPENSATE_TLB_PAGE_SIZE_FACTOR * basic_hbm_page_size; in gaudi2_set_dram_properties()
2328 prop->device_mem_alloc_default_page_size = prop->dram_page_size; in gaudi2_set_dram_properties()
2329 prop->dram_size = prop->num_functional_hbms * SZ_16G; in gaudi2_set_dram_properties()
2330 prop->dram_base_address = DRAM_PHYS_BASE; in gaudi2_set_dram_properties()
2331 prop->dram_end_address = prop->dram_base_address + prop->dram_size; in gaudi2_set_dram_properties()
2332 prop->dram_supports_virtual_memory = true; in gaudi2_set_dram_properties()
2334 prop->dram_user_base_address = DRAM_PHYS_BASE + prop->dram_page_size; in gaudi2_set_dram_properties()
2335 prop->dram_hints_align_mask = ~GAUDI2_HBM_MMU_SCRM_ADDRESS_MASK; in gaudi2_set_dram_properties()
2336 prop->hints_dram_reserved_va_range.start_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HBM_START; in gaudi2_set_dram_properties()
2337 prop->hints_dram_reserved_va_range.end_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HBM_END; in gaudi2_set_dram_properties()
2345 * 1. partition the virtual address space to DRAM-page (whole) pages. in gaudi2_set_dram_properties()
2356 prop->dmmu.start_addr = prop->dram_base_address + in gaudi2_set_dram_properties()
2357 (prop->dram_page_size * in gaudi2_set_dram_properties()
2358 DIV_ROUND_UP_SECTOR_T(prop->dram_size, prop->dram_page_size)); in gaudi2_set_dram_properties()
2360 prop->dmmu.end_addr = prop->dmmu.start_addr + prop->dram_page_size * in gaudi2_set_dram_properties()
2361 div_u64((VA_HBM_SPACE_END - prop->dmmu.start_addr), prop->dmmu.page_size); in gaudi2_set_dram_properties()
2368 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_set_fixed_properties()
2373 prop->max_queues = GAUDI2_QUEUE_ID_SIZE; in gaudi2_set_fixed_properties()
2374 prop->hw_queues_props = kcalloc(prop->max_queues, sizeof(struct hw_queue_properties), in gaudi2_set_fixed_properties()
2377 if (!prop->hw_queues_props) in gaudi2_set_fixed_properties()
2378 return -ENOMEM; in gaudi2_set_fixed_properties()
2380 q_props = prop->hw_queues_props; in gaudi2_set_fixed_properties()
2400 prop->cache_line_size = DEVICE_CACHE_LINE_SIZE; in gaudi2_set_fixed_properties()
2401 prop->cfg_base_address = CFG_BASE; in gaudi2_set_fixed_properties()
2402 prop->device_dma_offset_for_host_access = HOST_PHYS_BASE_0; in gaudi2_set_fixed_properties()
2403 prop->host_base_address = HOST_PHYS_BASE_0; in gaudi2_set_fixed_properties()
2404 prop->host_end_address = prop->host_base_address + HOST_PHYS_SIZE_0; in gaudi2_set_fixed_properties()
2405 prop->max_pending_cs = GAUDI2_MAX_PENDING_CS; in gaudi2_set_fixed_properties()
2406 prop->completion_queues_count = GAUDI2_RESERVED_CQ_NUMBER; in gaudi2_set_fixed_properties()
2407 prop->user_dec_intr_count = NUMBER_OF_DEC; in gaudi2_set_fixed_properties()
2408 prop->user_interrupt_count = GAUDI2_IRQ_NUM_USER_LAST - GAUDI2_IRQ_NUM_USER_FIRST + 1; in gaudi2_set_fixed_properties()
2409 prop->completion_mode = HL_COMPLETION_MODE_CS; in gaudi2_set_fixed_properties()
2410 prop->sync_stream_first_sob = GAUDI2_RESERVED_SOB_NUMBER; in gaudi2_set_fixed_properties()
2411 prop->sync_stream_first_mon = GAUDI2_RESERVED_MON_NUMBER; in gaudi2_set_fixed_properties()
2413 prop->sram_base_address = SRAM_BASE_ADDR; in gaudi2_set_fixed_properties()
2414 prop->sram_size = SRAM_SIZE; in gaudi2_set_fixed_properties()
2415 prop->sram_end_address = prop->sram_base_address + prop->sram_size; in gaudi2_set_fixed_properties()
2416 prop->sram_user_base_address = prop->sram_base_address + SRAM_USER_BASE_OFFSET; in gaudi2_set_fixed_properties()
2418 prop->hints_range_reservation = true; in gaudi2_set_fixed_properties()
2420 prop->rotator_enabled_mask = BIT(NUM_OF_ROT) - 1; in gaudi2_set_fixed_properties()
2422 if (hdev->pldm) in gaudi2_set_fixed_properties()
2423 prop->mmu_pgt_size = 0x800000; /* 8MB */ in gaudi2_set_fixed_properties()
2425 prop->mmu_pgt_size = MMU_PAGE_TABLES_INITIAL_SIZE; in gaudi2_set_fixed_properties()
2427 prop->mmu_pte_size = HL_PTE_SIZE; in gaudi2_set_fixed_properties()
2428 prop->mmu_hop_table_size = HOP_TABLE_SIZE_512_PTE; in gaudi2_set_fixed_properties()
2429 prop->mmu_hop0_tables_total_size = HOP0_512_PTE_TABLES_TOTAL_SIZE; in gaudi2_set_fixed_properties()
2431 prop->dmmu.hop_shifts[MMU_HOP0] = DHOP0_SHIFT; in gaudi2_set_fixed_properties()
2432 prop->dmmu.hop_shifts[MMU_HOP1] = DHOP1_SHIFT; in gaudi2_set_fixed_properties()
2433 prop->dmmu.hop_shifts[MMU_HOP2] = DHOP2_SHIFT; in gaudi2_set_fixed_properties()
2434 prop->dmmu.hop_shifts[MMU_HOP3] = DHOP3_SHIFT; in gaudi2_set_fixed_properties()
2435 prop->dmmu.hop_shifts[MMU_HOP4] = DHOP4_SHIFT; in gaudi2_set_fixed_properties()
2436 prop->dmmu.hop_masks[MMU_HOP0] = DHOP0_MASK; in gaudi2_set_fixed_properties()
2437 prop->dmmu.hop_masks[MMU_HOP1] = DHOP1_MASK; in gaudi2_set_fixed_properties()
2438 prop->dmmu.hop_masks[MMU_HOP2] = DHOP2_MASK; in gaudi2_set_fixed_properties()
2439 prop->dmmu.hop_masks[MMU_HOP3] = DHOP3_MASK; in gaudi2_set_fixed_properties()
2440 prop->dmmu.hop_masks[MMU_HOP4] = DHOP4_MASK; in gaudi2_set_fixed_properties()
2441 prop->dmmu.page_size = PAGE_SIZE_1GB; in gaudi2_set_fixed_properties()
2442 prop->dmmu.num_hops = MMU_ARCH_6_HOPS; in gaudi2_set_fixed_properties()
2443 prop->dmmu.last_mask = LAST_MASK; in gaudi2_set_fixed_properties()
2444 prop->dmmu.host_resident = 1; in gaudi2_set_fixed_properties()
2445 prop->dmmu.hop_table_size = prop->mmu_hop_table_size; in gaudi2_set_fixed_properties()
2446 prop->dmmu.hop0_tables_total_size = prop->mmu_hop0_tables_total_size; in gaudi2_set_fixed_properties()
2454 prop->dram_size = (GAUDI2_HBM_NUM - 1) * SZ_16G; in gaudi2_set_fixed_properties()
2456 hdev->pmmu_huge_range = true; in gaudi2_set_fixed_properties()
2457 prop->pmmu.host_resident = 1; in gaudi2_set_fixed_properties()
2458 prop->pmmu.num_hops = MMU_ARCH_6_HOPS; in gaudi2_set_fixed_properties()
2459 prop->pmmu.last_mask = LAST_MASK; in gaudi2_set_fixed_properties()
2460 prop->pmmu.hop_table_size = prop->mmu_hop_table_size; in gaudi2_set_fixed_properties()
2461 prop->pmmu.hop0_tables_total_size = prop->mmu_hop0_tables_total_size; in gaudi2_set_fixed_properties()
2463 prop->hints_host_reserved_va_range.start_addr = RESERVED_VA_FOR_VIRTUAL_MSIX_DOORBELL_START; in gaudi2_set_fixed_properties()
2464 prop->hints_host_reserved_va_range.end_addr = RESERVED_VA_RANGE_FOR_ARC_ON_HOST_END; in gaudi2_set_fixed_properties()
2465 prop->hints_host_hpage_reserved_va_range.start_addr = in gaudi2_set_fixed_properties()
2467 prop->hints_host_hpage_reserved_va_range.end_addr = in gaudi2_set_fixed_properties()
2471 prop->pmmu.hop_shifts[MMU_HOP0] = HOP0_SHIFT_64K; in gaudi2_set_fixed_properties()
2472 prop->pmmu.hop_shifts[MMU_HOP1] = HOP1_SHIFT_64K; in gaudi2_set_fixed_properties()
2473 prop->pmmu.hop_shifts[MMU_HOP2] = HOP2_SHIFT_64K; in gaudi2_set_fixed_properties()
2474 prop->pmmu.hop_shifts[MMU_HOP3] = HOP3_SHIFT_64K; in gaudi2_set_fixed_properties()
2475 prop->pmmu.hop_shifts[MMU_HOP4] = HOP4_SHIFT_64K; in gaudi2_set_fixed_properties()
2476 prop->pmmu.hop_shifts[MMU_HOP5] = HOP5_SHIFT_64K; in gaudi2_set_fixed_properties()
2477 prop->pmmu.hop_masks[MMU_HOP0] = HOP0_MASK_64K; in gaudi2_set_fixed_properties()
2478 prop->pmmu.hop_masks[MMU_HOP1] = HOP1_MASK_64K; in gaudi2_set_fixed_properties()
2479 prop->pmmu.hop_masks[MMU_HOP2] = HOP2_MASK_64K; in gaudi2_set_fixed_properties()
2480 prop->pmmu.hop_masks[MMU_HOP3] = HOP3_MASK_64K; in gaudi2_set_fixed_properties()
2481 prop->pmmu.hop_masks[MMU_HOP4] = HOP4_MASK_64K; in gaudi2_set_fixed_properties()
2482 prop->pmmu.hop_masks[MMU_HOP5] = HOP5_MASK_64K; in gaudi2_set_fixed_properties()
2483 prop->pmmu.start_addr = VA_HOST_SPACE_PAGE_START; in gaudi2_set_fixed_properties()
2484 prop->pmmu.end_addr = VA_HOST_SPACE_PAGE_END; in gaudi2_set_fixed_properties()
2485 prop->pmmu.page_size = PAGE_SIZE_64KB; in gaudi2_set_fixed_properties()
2488 memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu)); in gaudi2_set_fixed_properties()
2489 prop->pmmu_huge.page_size = PAGE_SIZE_16MB; in gaudi2_set_fixed_properties()
2490 prop->pmmu_huge.start_addr = VA_HOST_SPACE_HPAGE_START; in gaudi2_set_fixed_properties()
2491 prop->pmmu_huge.end_addr = VA_HOST_SPACE_HPAGE_END; in gaudi2_set_fixed_properties()
2493 prop->pmmu.hop_shifts[MMU_HOP0] = HOP0_SHIFT_4K; in gaudi2_set_fixed_properties()
2494 prop->pmmu.hop_shifts[MMU_HOP1] = HOP1_SHIFT_4K; in gaudi2_set_fixed_properties()
2495 prop->pmmu.hop_shifts[MMU_HOP2] = HOP2_SHIFT_4K; in gaudi2_set_fixed_properties()
2496 prop->pmmu.hop_shifts[MMU_HOP3] = HOP3_SHIFT_4K; in gaudi2_set_fixed_properties()
2497 prop->pmmu.hop_shifts[MMU_HOP4] = HOP4_SHIFT_4K; in gaudi2_set_fixed_properties()
2498 prop->pmmu.hop_shifts[MMU_HOP5] = HOP5_SHIFT_4K; in gaudi2_set_fixed_properties()
2499 prop->pmmu.hop_masks[MMU_HOP0] = HOP0_MASK_4K; in gaudi2_set_fixed_properties()
2500 prop->pmmu.hop_masks[MMU_HOP1] = HOP1_MASK_4K; in gaudi2_set_fixed_properties()
2501 prop->pmmu.hop_masks[MMU_HOP2] = HOP2_MASK_4K; in gaudi2_set_fixed_properties()
2502 prop->pmmu.hop_masks[MMU_HOP3] = HOP3_MASK_4K; in gaudi2_set_fixed_properties()
2503 prop->pmmu.hop_masks[MMU_HOP4] = HOP4_MASK_4K; in gaudi2_set_fixed_properties()
2504 prop->pmmu.hop_masks[MMU_HOP5] = HOP5_MASK_4K; in gaudi2_set_fixed_properties()
2505 prop->pmmu.start_addr = VA_HOST_SPACE_PAGE_START; in gaudi2_set_fixed_properties()
2506 prop->pmmu.end_addr = VA_HOST_SPACE_PAGE_END; in gaudi2_set_fixed_properties()
2507 prop->pmmu.page_size = PAGE_SIZE_4KB; in gaudi2_set_fixed_properties()
2510 memcpy(&prop->pmmu_huge, &prop->pmmu, sizeof(prop->pmmu)); in gaudi2_set_fixed_properties()
2511 prop->pmmu_huge.page_size = PAGE_SIZE_2MB; in gaudi2_set_fixed_properties()
2512 prop->pmmu_huge.start_addr = VA_HOST_SPACE_HPAGE_START; in gaudi2_set_fixed_properties()
2513 prop->pmmu_huge.end_addr = VA_HOST_SPACE_HPAGE_END; in gaudi2_set_fixed_properties()
2516 prop->max_num_of_engines = GAUDI2_ENGINE_ID_SIZE; in gaudi2_set_fixed_properties()
2517 prop->num_engine_cores = CPU_ID_MAX; in gaudi2_set_fixed_properties()
2518 prop->cfg_size = CFG_SIZE; in gaudi2_set_fixed_properties()
2519 prop->max_asid = MAX_ASID; in gaudi2_set_fixed_properties()
2520 prop->num_of_events = GAUDI2_EVENT_SIZE; in gaudi2_set_fixed_properties()
2522 prop->supports_engine_modes = true; in gaudi2_set_fixed_properties()
2524 prop->dc_power_default = DC_POWER_DEFAULT; in gaudi2_set_fixed_properties()
2526 prop->cb_pool_cb_cnt = GAUDI2_CB_POOL_CB_CNT; in gaudi2_set_fixed_properties()
2527 prop->cb_pool_cb_size = GAUDI2_CB_POOL_CB_SIZE; in gaudi2_set_fixed_properties()
2528 prop->pcie_dbi_base_address = CFG_BASE + mmPCIE_DBI_BASE; in gaudi2_set_fixed_properties()
2529 prop->pcie_aux_dbi_reg_addr = CFG_BASE + mmPCIE_AUX_DBI; in gaudi2_set_fixed_properties()
2531 strscpy_pad(prop->cpucp_info.card_name, GAUDI2_DEFAULT_CARD_NAME, CARD_NAME_MAX_LEN); in gaudi2_set_fixed_properties()
2533 prop->mme_master_slave_mode = 1; in gaudi2_set_fixed_properties()
2535 prop->first_available_user_sob[0] = GAUDI2_RESERVED_SOB_NUMBER + in gaudi2_set_fixed_properties()
2538 prop->first_available_user_mon[0] = GAUDI2_RESERVED_MON_NUMBER + in gaudi2_set_fixed_properties()
2541 prop->first_available_user_interrupt = GAUDI2_IRQ_NUM_USER_FIRST; in gaudi2_set_fixed_properties()
2542 prop->tpc_interrupt_id = GAUDI2_IRQ_NUM_TPC_ASSERT; in gaudi2_set_fixed_properties()
2543 prop->eq_interrupt_id = GAUDI2_IRQ_NUM_EVENT_QUEUE; in gaudi2_set_fixed_properties()
2545 prop->first_available_cq[0] = GAUDI2_RESERVED_CQ_NUMBER; in gaudi2_set_fixed_properties()
2547 prop->fw_cpu_boot_dev_sts0_valid = false; in gaudi2_set_fixed_properties()
2548 prop->fw_cpu_boot_dev_sts1_valid = false; in gaudi2_set_fixed_properties()
2549 prop->hard_reset_done_by_fw = false; in gaudi2_set_fixed_properties()
2550 prop->gic_interrupts_enable = true; in gaudi2_set_fixed_properties()
2552 prop->server_type = HL_SERVER_TYPE_UNKNOWN; in gaudi2_set_fixed_properties()
2554 prop->max_dec = NUMBER_OF_DEC; in gaudi2_set_fixed_properties()
2556 prop->clk_pll_index = HL_GAUDI2_MME_PLL; in gaudi2_set_fixed_properties()
2558 prop->dma_mask = 64; in gaudi2_set_fixed_properties()
2560 prop->hbw_flush_reg = mmPCIE_WRAP_SPECIAL_GLBL_SPARE_0; in gaudi2_set_fixed_properties()
2575 hdev->rmmio = hdev->pcie_bar[SRAM_CFG_BAR_ID] + (CFG_BASE - STM_FLASH_BASE_ADDR); in gaudi2_pci_bars_map()
2582 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_set_hbm_bar_base()
2587 if ((gaudi2) && (gaudi2->dram_bar_cur_addr == addr)) in gaudi2_set_hbm_bar_base()
2590 if (hdev->asic_prop.iatu_done_by_fw) in gaudi2_set_hbm_bar_base()
2593 /* Inbound Region 2 - Bar 4 - Point to DRAM */ in gaudi2_set_hbm_bar_base()
2602 old_addr = gaudi2->dram_bar_cur_addr; in gaudi2_set_hbm_bar_base()
2603 gaudi2->dram_bar_cur_addr = addr; in gaudi2_set_hbm_bar_base()
2616 if (hdev->asic_prop.iatu_done_by_fw) in gaudi2_init_iatu()
2619 /* Temporary inbound Region 0 - Bar 0 - Point to CFG in gaudi2_init_iatu()
2626 inbound_region.addr = STM_FLASH_BASE_ADDR - STM_FLASH_ALIGNED_OFF; in gaudi2_init_iatu()
2635 hdev->pcie_bar_phys[SRAM_CFG_BAR_ID] = (u64)bar_addr_high << 32 | bar_addr_low; in gaudi2_init_iatu()
2637 /* Inbound Region 0 - Bar 0 - Point to CFG */ in gaudi2_init_iatu()
2647 /* Inbound Region 1 - Bar 0 - Point to BAR0_RESERVED + SRAM */ in gaudi2_init_iatu()
2657 /* Inbound Region 2 - Bar 4 - Point to DRAM */ in gaudi2_init_iatu()
2665 /* Outbound Region 0 - Point to Host */ in gaudi2_init_iatu()
2680 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_tpc_binning_init_prop()
2686 if (hweight64(hdev->tpc_binning) > MAX_CLUSTER_BINNING_FAULTY_TPCS) { in gaudi2_tpc_binning_init_prop()
2687 dev_err(hdev->dev, "TPC binning is supported for max of %d faulty TPCs, provided mask 0x%llx\n", in gaudi2_tpc_binning_init_prop()
2689 hdev->tpc_binning); in gaudi2_tpc_binning_init_prop()
2690 return -EINVAL; in gaudi2_tpc_binning_init_prop()
2693 prop->tpc_binning_mask = hdev->tpc_binning; in gaudi2_tpc_binning_init_prop()
2694 prop->tpc_enabled_mask = GAUDI2_TPC_FULL_MASK; in gaudi2_tpc_binning_init_prop()
2701 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_set_tpc_binning_masks()
2702 struct hw_queue_properties *q_props = prop->hw_queues_props; in gaudi2_set_tpc_binning_masks()
2711 tpc_binning_mask = prop->tpc_binning_mask; in gaudi2_set_tpc_binning_masks()
2731 * Coverity complains about possible out-of-bound access in in gaudi2_set_tpc_binning_masks()
2735 dev_err(hdev->dev, in gaudi2_set_tpc_binning_masks()
2738 return -EINVAL; in gaudi2_set_tpc_binning_masks()
2743 clear_bit(subst_seq, (unsigned long *)&prop->tpc_enabled_mask); in gaudi2_set_tpc_binning_masks()
2759 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_set_dec_binning_masks()
2762 num_faulty = hweight32(hdev->decoder_binning); in gaudi2_set_dec_binning_masks()
2769 …dev_err(hdev->dev, "decoder binning is supported for max of single faulty decoder, provided mask 0… in gaudi2_set_dec_binning_masks()
2770 hdev->decoder_binning); in gaudi2_set_dec_binning_masks()
2771 return -EINVAL; in gaudi2_set_dec_binning_masks()
2774 prop->decoder_binning_mask = (hdev->decoder_binning & GAUDI2_DECODER_FULL_MASK); in gaudi2_set_dec_binning_masks()
2776 if (prop->decoder_binning_mask) in gaudi2_set_dec_binning_masks()
2777 prop->decoder_enabled_mask = (GAUDI2_DECODER_FULL_MASK & ~BIT(DEC_ID_PCIE_VDEC1)); in gaudi2_set_dec_binning_masks()
2779 prop->decoder_enabled_mask = GAUDI2_DECODER_FULL_MASK; in gaudi2_set_dec_binning_masks()
2786 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_set_dram_binning_masks()
2789 if (!hdev->dram_binning) { in gaudi2_set_dram_binning_masks()
2790 prop->dram_binning_mask = 0; in gaudi2_set_dram_binning_masks()
2791 prop->dram_enabled_mask = GAUDI2_DRAM_FULL_MASK; in gaudi2_set_dram_binning_masks()
2796 prop->faulty_dram_cluster_map |= hdev->dram_binning; in gaudi2_set_dram_binning_masks()
2797 prop->dram_binning_mask = hdev->dram_binning; in gaudi2_set_dram_binning_masks()
2798 prop->dram_enabled_mask = GAUDI2_DRAM_FULL_MASK & ~BIT(HBM_ID5); in gaudi2_set_dram_binning_masks()
2803 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_set_edma_binning_masks()
2807 num_faulty = hweight32(hdev->edma_binning); in gaudi2_set_edma_binning_masks()
2814 dev_err(hdev->dev, in gaudi2_set_edma_binning_masks()
2815 "EDMA binning is supported for max of single faulty EDMA, provided mask 0x%x\n", in gaudi2_set_edma_binning_masks()
2816 hdev->edma_binning); in gaudi2_set_edma_binning_masks()
2817 return -EINVAL; in gaudi2_set_edma_binning_masks()
2820 if (!hdev->edma_binning) { in gaudi2_set_edma_binning_masks()
2821 prop->edma_binning_mask = 0; in gaudi2_set_edma_binning_masks()
2822 prop->edma_enabled_mask = GAUDI2_EDMA_FULL_MASK; in gaudi2_set_edma_binning_masks()
2826 seq = __ffs((unsigned long)hdev->edma_binning); in gaudi2_set_edma_binning_masks()
2829 prop->faulty_dram_cluster_map |= BIT(edma_to_hbm_cluster[seq]); in gaudi2_set_edma_binning_masks()
2830 prop->edma_binning_mask = hdev->edma_binning; in gaudi2_set_edma_binning_masks()
2831 prop->edma_enabled_mask = GAUDI2_EDMA_FULL_MASK & ~BIT(EDMA_ID_DCORE3_INSTANCE1); in gaudi2_set_edma_binning_masks()
2834 q_props = prop->hw_queues_props; in gaudi2_set_edma_binning_masks()
2845 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_set_xbar_edge_enable_mask()
2850 prop->xbar_edge_enabled_mask = GAUDI2_XBAR_EDGE_FULL_MASK; in gaudi2_set_xbar_edge_enable_mask()
2865 dev_err(hdev->dev, "we cannot have more than %d faulty XBAR EDGE\n", in gaudi2_set_xbar_edge_enable_mask()
2867 return -EINVAL; in gaudi2_set_xbar_edge_enable_mask()
2873 prop->faulty_dram_cluster_map |= BIT(xbar_edge_to_hbm_cluster[seq]); in gaudi2_set_xbar_edge_enable_mask()
2874 prop->xbar_edge_enabled_mask = (~xbar_edge_iso_mask) & GAUDI2_XBAR_EDGE_FULL_MASK; in gaudi2_set_xbar_edge_enable_mask()
2886 * If more than single cluster is faulty- the chip is unusable in gaudi2_set_cluster_binning_masks_common()
2888 hdev->asic_prop.faulty_dram_cluster_map = 0; in gaudi2_set_cluster_binning_masks_common()
2902 hdev->asic_prop.hmmu_hif_enabled_mask = GAUDI2_HIF_HMMU_FULL_MASK; in gaudi2_set_cluster_binning_masks_common()
2909 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_set_cluster_binning_masks()
2912 rc = gaudi2_set_cluster_binning_masks_common(hdev, prop->cpucp_info.xbar_binning_mask); in gaudi2_set_cluster_binning_masks()
2917 if (prop->faulty_dram_cluster_map) { in gaudi2_set_cluster_binning_masks()
2918 u8 cluster_seq = __ffs((unsigned long)prop->faulty_dram_cluster_map); in gaudi2_set_cluster_binning_masks()
2920 prop->hmmu_hif_enabled_mask = cluster_hmmu_hif_enabled_mask[cluster_seq]; in gaudi2_set_cluster_binning_masks()
2947 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_cpucp_info_get()
2948 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_cpucp_info_get()
2953 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) in gaudi2_cpucp_info_get()
2959 if (hdev->reset_info.in_compute_reset) in gaudi2_cpucp_info_get()
2967 dram_size = le64_to_cpu(prop->cpucp_info.dram_size); in gaudi2_cpucp_info_get()
2971 if ((dram_size != ((GAUDI2_HBM_NUM - 1) * SZ_16G)) && in gaudi2_cpucp_info_get()
2973 dev_err(hdev->dev, in gaudi2_cpucp_info_get()
2975 dram_size, prop->dram_size); in gaudi2_cpucp_info_get()
2976 dram_size = prop->dram_size; in gaudi2_cpucp_info_get()
2979 prop->dram_size = dram_size; in gaudi2_cpucp_info_get()
2980 prop->dram_end_address = prop->dram_base_address + dram_size; in gaudi2_cpucp_info_get()
2983 if (!strlen(prop->cpucp_info.card_name)) in gaudi2_cpucp_info_get()
2984 strscpy_pad(prop->cpucp_info.card_name, GAUDI2_DEFAULT_CARD_NAME, in gaudi2_cpucp_info_get()
2988 hdev->dram_binning = prop->cpucp_info.dram_binning_mask; in gaudi2_cpucp_info_get()
2989 hdev->edma_binning = prop->cpucp_info.edma_binning_mask; in gaudi2_cpucp_info_get()
2990 hdev->tpc_binning = le64_to_cpu(prop->cpucp_info.tpc_binning_mask); in gaudi2_cpucp_info_get()
2991 hdev->decoder_binning = lower_32_bits(le64_to_cpu(prop->cpucp_info.decoder_binning_mask)); in gaudi2_cpucp_info_get()
2993 dev_dbg(hdev->dev, "Read binning masks: tpc: 0x%llx, dram: 0x%llx, edma: 0x%x, dec: 0x%x\n", in gaudi2_cpucp_info_get()
2994 hdev->tpc_binning, hdev->dram_binning, hdev->edma_binning, in gaudi2_cpucp_info_get()
2995 hdev->decoder_binning); in gaudi2_cpucp_info_get()
3001 rc = hdev->asic_funcs->set_dram_properties(hdev); in gaudi2_cpucp_info_get()
3005 rc = hdev->asic_funcs->set_binning_masks(hdev); in gaudi2_cpucp_info_get()
3013 prop->max_power_default = (u64) max_power; in gaudi2_cpucp_info_get()
3020 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_fetch_psoc_frequency()
3024 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) in gaudi2_fetch_psoc_frequency()
3031 hdev->asic_prop.psoc_timestamp_frequency = pll_freq_arr[3]; in gaudi2_fetch_psoc_frequency()
3038 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_early_init()
3039 struct pci_dev *pdev = hdev->pdev; in gaudi2_early_init()
3051 dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n", in gaudi2_early_init()
3053 rc = -ENODEV; in gaudi2_early_init()
3059 dev_err(hdev->dev, "Not " HL_NAME "? BAR %d size %pa, expecting %llu\n", in gaudi2_early_init()
3061 rc = -ENODEV; in gaudi2_early_init()
3065 prop->dram_pci_bar_size = pci_resource_len(pdev, DRAM_BAR_ID); in gaudi2_early_init()
3066 hdev->dram_pci_bar_start = pci_resource_start(pdev, DRAM_BAR_ID); in gaudi2_early_init()
3071 if (hdev->pldm) in gaudi2_early_init()
3072 hdev->asic_prop.iatu_done_by_fw = false; in gaudi2_early_init()
3074 hdev->asic_prop.iatu_done_by_fw = true; in gaudi2_early_init()
3081 * version to determine whether we run with a security-enabled firmware in gaudi2_early_init()
3085 if (hdev->reset_on_preboot_fail) in gaudi2_early_init()
3087 hdev->asic_funcs->hw_fini(hdev, true, false); in gaudi2_early_init()
3092 dev_dbg(hdev->dev, "H/W state is dirty, must reset before initializing\n"); in gaudi2_early_init()
3093 rc = hdev->asic_funcs->hw_fini(hdev, true, false); in gaudi2_early_init()
3095 dev_err(hdev->dev, "failed to reset HW in dirty state (%d)\n", rc); in gaudi2_early_init()
3105 kfree(hdev->asic_prop.hw_queues_props); in gaudi2_early_init()
3111 kfree(hdev->asic_prop.hw_queues_props); in gaudi2_early_fini()
3139 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi2_init_arcs()
3140 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_arcs()
3160 !(hdev->nic_ports_mask & BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0))) in gaudi2_init_arcs()
3163 if (gaudi2_is_arc_tpc_owned(arc_id) && !(gaudi2->tpc_hw_cap_initialized & in gaudi2_init_arcs()
3164 BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0))) in gaudi2_init_arcs()
3171 hdev->asic_prop.engine_core_interrupt_reg_addr = in gaudi2_init_arcs()
3172 CFG_BASE + le32_to_cpu(dyn_regs->eng_arc_irq_ctrl); in gaudi2_init_arcs()
3243 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_late_init()
3246 hdev->asic_prop.supports_advanced_cpucp_rc = true; in gaudi2_late_init()
3249 gaudi2->virt_msix_db_dma_addr); in gaudi2_late_init()
3251 dev_err(hdev->dev, "Failed to enable PCI access from CPU\n"); in gaudi2_late_init()
3257 dev_err(hdev->dev, "Failed to fetch psoc frequency\n"); in gaudi2_late_init()
3265 dev_err(hdev->dev, "Failed to scrub arcs DCCM\n"); in gaudi2_late_init()
3286 struct user_mapped_block *blocks = gaudi2->mapped_blocks; in gaudi2_user_mapped_dec_init()
3302 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_user_mapped_blocks_init()
3303 struct user_mapped_block *blocks = gaudi2->mapped_blocks; in gaudi2_user_mapped_blocks_init()
3361 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1)].size = SM_OBJS_BLOCK_SIZE; in gaudi2_user_mapped_blocks_init()
3362 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1) + 1].size = HL_BLOCK_SIZE; in gaudi2_user_mapped_blocks_init()
3364 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1)].address = in gaudi2_user_mapped_blocks_init()
3367 blocks[USR_MAPPED_BLK_SM_START_IDX + 2 * (i - 1) + 1].address = in gaudi2_user_mapped_blocks_init()
3378 /* The device ARC works with 32-bits addresses, and because there is a single HW register in gaudi2_alloc_cpu_accessible_dma_mem()
3387 rc = -ENOMEM; in gaudi2_alloc_cpu_accessible_dma_mem()
3391 end_addr = dma_addr_arr[i] + HL_CPU_ACCESSIBLE_MEM_SIZE - 1; in gaudi2_alloc_cpu_accessible_dma_mem()
3397 dev_err(hdev->dev, in gaudi2_alloc_cpu_accessible_dma_mem()
3399 rc = -EFAULT; in gaudi2_alloc_cpu_accessible_dma_mem()
3403 hdev->cpu_accessible_dma_mem = virt_addr_arr[i]; in gaudi2_alloc_cpu_accessible_dma_mem()
3404 hdev->cpu_accessible_dma_address = dma_addr_arr[i]; in gaudi2_alloc_cpu_accessible_dma_mem()
3416 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_set_pci_memory_regions()
3420 region = &hdev->pci_mem_region[PCI_REGION_CFG]; in gaudi2_set_pci_memory_regions()
3421 region->region_base = CFG_BASE; in gaudi2_set_pci_memory_regions()
3422 region->region_size = CFG_SIZE; in gaudi2_set_pci_memory_regions()
3423 region->offset_in_bar = CFG_BASE - STM_FLASH_BASE_ADDR; in gaudi2_set_pci_memory_regions()
3424 region->bar_size = CFG_BAR_SIZE; in gaudi2_set_pci_memory_regions()
3425 region->bar_id = SRAM_CFG_BAR_ID; in gaudi2_set_pci_memory_regions()
3426 region->used = 1; in gaudi2_set_pci_memory_regions()
3429 region = &hdev->pci_mem_region[PCI_REGION_SRAM]; in gaudi2_set_pci_memory_regions()
3430 region->region_base = SRAM_BASE_ADDR; in gaudi2_set_pci_memory_regions()
3431 region->region_size = SRAM_SIZE; in gaudi2_set_pci_memory_regions()
3432 region->offset_in_bar = CFG_REGION_SIZE + BAR0_RSRVD_SIZE; in gaudi2_set_pci_memory_regions()
3433 region->bar_size = CFG_BAR_SIZE; in gaudi2_set_pci_memory_regions()
3434 region->bar_id = SRAM_CFG_BAR_ID; in gaudi2_set_pci_memory_regions()
3435 region->used = 1; in gaudi2_set_pci_memory_regions()
3438 region = &hdev->pci_mem_region[PCI_REGION_DRAM]; in gaudi2_set_pci_memory_regions()
3439 region->region_base = DRAM_PHYS_BASE; in gaudi2_set_pci_memory_regions()
3440 region->region_size = hdev->asic_prop.dram_size; in gaudi2_set_pci_memory_regions()
3441 region->offset_in_bar = 0; in gaudi2_set_pci_memory_regions()
3442 region->bar_size = prop->dram_pci_bar_size; in gaudi2_set_pci_memory_regions()
3443 region->bar_id = DRAM_BAR_ID; in gaudi2_set_pci_memory_regions()
3444 region->used = 1; in gaudi2_set_pci_memory_regions()
3449 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_user_interrupt_setup()
3453 HL_USR_INTR_STRUCT_INIT(hdev->tpc_interrupt, hdev, 0, HL_USR_INTERRUPT_TPC); in gaudi2_user_interrupt_setup()
3456 HL_USR_INTR_STRUCT_INIT(hdev->unexpected_error_interrupt, hdev, 0, in gaudi2_user_interrupt_setup()
3460 HL_USR_INTR_STRUCT_INIT(hdev->common_user_cq_interrupt, hdev, in gaudi2_user_interrupt_setup()
3464 HL_USR_INTR_STRUCT_INIT(hdev->common_decoder_interrupt, hdev, in gaudi2_user_interrupt_setup()
3478 HL_USR_INTR_STRUCT_INIT(hdev->user_interrupt[j], hdev, i, in gaudi2_user_interrupt_setup()
3481 for (i = GAUDI2_IRQ_NUM_USER_FIRST, k = 0 ; k < prop->user_interrupt_count; i++, j++, k++) in gaudi2_user_interrupt_setup()
3482 HL_USR_INTR_STRUCT_INIT(hdev->user_interrupt[j], hdev, i, HL_USR_INTERRUPT_CQ); in gaudi2_user_interrupt_setup()
3494 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_special_blocks_free()
3496 &prop->skip_special_blocks_cfg; in gaudi2_special_blocks_free()
3498 kfree(prop->special_blocks); in gaudi2_special_blocks_free()
3499 kfree(skip_special_blocks_cfg->block_types); in gaudi2_special_blocks_free()
3500 kfree(skip_special_blocks_cfg->block_ranges); in gaudi2_special_blocks_free()
3517 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_special_blocks_config()
3521 prop->glbl_err_cause_num = GAUDI2_NUM_OF_GLBL_ERR_CAUSE; in gaudi2_special_blocks_config()
3522 prop->num_of_special_blocks = ARRAY_SIZE(gaudi2_special_blocks); in gaudi2_special_blocks_config()
3523 prop->special_blocks = kmalloc_array(prop->num_of_special_blocks, in gaudi2_special_blocks_config()
3524 sizeof(*prop->special_blocks), GFP_KERNEL); in gaudi2_special_blocks_config()
3525 if (!prop->special_blocks) in gaudi2_special_blocks_config()
3526 return -ENOMEM; in gaudi2_special_blocks_config()
3528 for (i = 0 ; i < prop->num_of_special_blocks ; i++) in gaudi2_special_blocks_config()
3529 memcpy(&prop->special_blocks[i], &gaudi2_special_blocks[i], in gaudi2_special_blocks_config()
3530 sizeof(*prop->special_blocks)); in gaudi2_special_blocks_config()
3533 memset(&prop->skip_special_blocks_cfg, 0, sizeof(prop->skip_special_blocks_cfg)); in gaudi2_special_blocks_config()
3534 prop->skip_special_blocks_cfg.skip_block_hook = gaudi2_special_block_skip; in gaudi2_special_blocks_config()
3537 prop->skip_special_blocks_cfg.block_types = in gaudi2_special_blocks_config()
3540 if (!prop->skip_special_blocks_cfg.block_types) { in gaudi2_special_blocks_config()
3541 rc = -ENOMEM; in gaudi2_special_blocks_config()
3545 memcpy(prop->skip_special_blocks_cfg.block_types, gaudi2_iterator_skip_block_types, in gaudi2_special_blocks_config()
3548 prop->skip_special_blocks_cfg.block_types_len = in gaudi2_special_blocks_config()
3553 prop->skip_special_blocks_cfg.block_ranges = in gaudi2_special_blocks_config()
3556 if (!prop->skip_special_blocks_cfg.block_ranges) { in gaudi2_special_blocks_config()
3557 rc = -ENOMEM; in gaudi2_special_blocks_config()
3562 memcpy(&prop->skip_special_blocks_cfg.block_ranges[i], in gaudi2_special_blocks_config()
3566 prop->skip_special_blocks_cfg.block_ranges_len = in gaudi2_special_blocks_config()
3573 kfree(prop->skip_special_blocks_cfg.block_types); in gaudi2_special_blocks_config()
3575 kfree(prop->special_blocks); in gaudi2_special_blocks_config()
3587 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_test_queues_msgs_free()
3588 struct gaudi2_queues_test_info *msg_info = gaudi2->queues_test_info; in gaudi2_test_queues_msgs_free()
3592 /* bail-out if this is an allocation failure point */ in gaudi2_test_queues_msgs_free()
3603 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_test_queues_msgs_alloc()
3604 struct gaudi2_queues_test_info *msg_info = gaudi2->queues_test_info; in gaudi2_test_queues_msgs_alloc()
3607 /* allocate a message-short buf for each Q we intend to test */ in gaudi2_test_queues_msgs_alloc()
3613 dev_err(hdev->dev, in gaudi2_test_queues_msgs_alloc()
3615 rc = -ENOMEM; in gaudi2_test_queues_msgs_alloc()
3629 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_sw_init()
3636 return -ENOMEM; in gaudi2_sw_init()
3642 if (gaudi2->num_of_valid_hw_events == GAUDI2_EVENT_SIZE) { in gaudi2_sw_init()
3643 dev_err(hdev->dev, "H/W events array exceeds the limit of %u events\n", in gaudi2_sw_init()
3645 rc = -EINVAL; in gaudi2_sw_init()
3649 gaudi2->hw_events[gaudi2->num_of_valid_hw_events++] = gaudi2_irq_map_table[i].fc_id; in gaudi2_sw_init()
3653 gaudi2->lfsr_rand_seeds[i] = gaudi2_get_non_zero_random_int(); in gaudi2_sw_init()
3655 gaudi2->cpucp_info_get = gaudi2_cpucp_info_get; in gaudi2_sw_init()
3657 hdev->asic_specific = gaudi2; in gaudi2_sw_init()
3660 * Use DEVICE_CACHE_LINE_SIZE for alignment since the NIC memory-mapped in gaudi2_sw_init()
3663 hdev->dma_pool = dma_pool_create(dev_name(hdev->dev), &hdev->pdev->dev, in gaudi2_sw_init()
3665 if (!hdev->dma_pool) { in gaudi2_sw_init()
3666 dev_err(hdev->dev, "failed to create DMA pool\n"); in gaudi2_sw_init()
3667 rc = -ENOMEM; in gaudi2_sw_init()
3675 hdev->cpu_accessible_dma_pool = gen_pool_create(ilog2(32), -1); in gaudi2_sw_init()
3676 if (!hdev->cpu_accessible_dma_pool) { in gaudi2_sw_init()
3677 dev_err(hdev->dev, "Failed to create CPU accessible DMA pool\n"); in gaudi2_sw_init()
3678 rc = -ENOMEM; in gaudi2_sw_init()
3682 rc = gen_pool_add(hdev->cpu_accessible_dma_pool, (uintptr_t) hdev->cpu_accessible_dma_mem, in gaudi2_sw_init()
3683 HL_CPU_ACCESSIBLE_MEM_SIZE, -1); in gaudi2_sw_init()
3685 dev_err(hdev->dev, "Failed to add memory to CPU accessible DMA pool\n"); in gaudi2_sw_init()
3686 rc = -EFAULT; in gaudi2_sw_init()
3690 gaudi2->virt_msix_db_cpu_addr = hl_cpu_accessible_dma_pool_alloc(hdev, prop->pmmu.page_size, in gaudi2_sw_init()
3691 &gaudi2->virt_msix_db_dma_addr); in gaudi2_sw_init()
3692 if (!gaudi2->virt_msix_db_cpu_addr) { in gaudi2_sw_init()
3693 dev_err(hdev->dev, "Failed to allocate DMA memory for virtual MSI-X doorbell\n"); in gaudi2_sw_init()
3694 rc = -ENOMEM; in gaudi2_sw_init()
3698 spin_lock_init(&gaudi2->hw_queues_lock); in gaudi2_sw_init()
3700 gaudi2->scratchpad_kernel_address = hl_asic_dma_alloc_coherent(hdev, PAGE_SIZE, in gaudi2_sw_init()
3701 &gaudi2->scratchpad_bus_address, in gaudi2_sw_init()
3703 if (!gaudi2->scratchpad_kernel_address) { in gaudi2_sw_init()
3704 rc = -ENOMEM; in gaudi2_sw_init()
3713 hdev->supports_coresight = true; in gaudi2_sw_init()
3714 hdev->supports_sync_stream = true; in gaudi2_sw_init()
3715 hdev->supports_cb_mapping = true; in gaudi2_sw_init()
3716 hdev->supports_wait_for_multi_cs = false; in gaudi2_sw_init()
3718 prop->supports_compute_reset = true; in gaudi2_sw_init()
3722 hdev->event_queue.check_eqe_index = false; in gaudi2_sw_init()
3724 hdev->event_queue.check_eqe_index = true; in gaudi2_sw_init()
3726 hdev->asic_funcs->set_pci_memory_regions(hdev); in gaudi2_sw_init()
3741 hl_asic_dma_free_coherent(hdev, PAGE_SIZE, gaudi2->scratchpad_kernel_address, in gaudi2_sw_init()
3742 gaudi2->scratchpad_bus_address); in gaudi2_sw_init()
3744 hl_cpu_accessible_dma_pool_free(hdev, prop->pmmu.page_size, gaudi2->virt_msix_db_cpu_addr); in gaudi2_sw_init()
3746 gen_pool_destroy(hdev->cpu_accessible_dma_pool); in gaudi2_sw_init()
3748 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem, in gaudi2_sw_init()
3749 hdev->cpu_accessible_dma_address); in gaudi2_sw_init()
3751 dma_pool_destroy(hdev->dma_pool); in gaudi2_sw_init()
3759 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_sw_fini()
3760 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_sw_fini()
3766 hl_cpu_accessible_dma_pool_free(hdev, prop->pmmu.page_size, gaudi2->virt_msix_db_cpu_addr); in gaudi2_sw_fini()
3768 gen_pool_destroy(hdev->cpu_accessible_dma_pool); in gaudi2_sw_fini()
3770 hl_asic_dma_free_coherent(hdev, HL_CPU_ACCESSIBLE_MEM_SIZE, hdev->cpu_accessible_dma_mem, in gaudi2_sw_fini()
3771 hdev->cpu_accessible_dma_address); in gaudi2_sw_fini()
3773 hl_asic_dma_free_coherent(hdev, PAGE_SIZE, gaudi2->scratchpad_kernel_address, in gaudi2_sw_fini()
3774 gaudi2->scratchpad_bus_address); in gaudi2_sw_fini()
3776 dma_pool_destroy(hdev->dma_pool); in gaudi2_sw_fini()
3806 * gaudi2_clear_qm_fence_counters_common - clear QM's fence counters
3823 size = mmPDMA0_QM_CP_BARRIER_CFG - mmPDMA0_QM_CP_FENCE0_CNT_0; in gaudi2_clear_qm_fence_counters_common()
3846 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_stop_dma_qmans()
3849 if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK)) in gaudi2_stop_dma_qmans()
3857 if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK)) in gaudi2_stop_dma_qmans()
3865 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq))) in gaudi2_stop_dma_qmans()
3879 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_stop_mme_qmans()
3882 offset = mmDCORE1_MME_QM_BASE - mmDCORE0_MME_QM_BASE; in gaudi2_stop_mme_qmans()
3885 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i))) in gaudi2_stop_mme_qmans()
3894 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_stop_tpc_qmans()
3898 if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK)) in gaudi2_stop_tpc_qmans()
3902 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i))) in gaudi2_stop_tpc_qmans()
3912 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_stop_rot_qmans()
3916 if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK)) in gaudi2_stop_rot_qmans()
3920 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i))) in gaudi2_stop_rot_qmans()
3930 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_stop_nic_qmans()
3934 if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK)) in gaudi2_stop_nic_qmans()
3940 if (!(hdev->nic_ports_mask & BIT(i))) in gaudi2_stop_nic_qmans()
3958 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_dma_stall()
3961 if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK)) in gaudi2_dma_stall()
3968 if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK)) in gaudi2_dma_stall()
3976 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq))) in gaudi2_dma_stall()
3990 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_mme_stall()
3993 offset = mmDCORE1_MME_CTRL_LO_QM_STALL - mmDCORE0_MME_CTRL_LO_QM_STALL; in gaudi2_mme_stall()
3996 if (gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i)) in gaudi2_mme_stall()
4002 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_tpc_stall()
4006 if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK)) in gaudi2_tpc_stall()
4010 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i))) in gaudi2_tpc_stall()
4020 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_rotator_stall()
4024 if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK)) in gaudi2_rotator_stall()
4032 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i))) in gaudi2_rotator_stall()
4046 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_disable_dma_qmans()
4049 if (!(gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK)) in gaudi2_disable_dma_qmans()
4056 if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK)) in gaudi2_disable_dma_qmans()
4064 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + seq))) in gaudi2_disable_dma_qmans()
4078 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_disable_mme_qmans()
4081 offset = mmDCORE1_MME_QM_BASE - mmDCORE0_MME_QM_BASE; in gaudi2_disable_mme_qmans()
4084 if (gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + i)) in gaudi2_disable_mme_qmans()
4090 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_disable_tpc_qmans()
4094 if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK)) in gaudi2_disable_tpc_qmans()
4098 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + i))) in gaudi2_disable_tpc_qmans()
4108 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_disable_rot_qmans()
4112 if (!(gaudi2->hw_cap_initialized & HW_CAP_ROT_MASK)) in gaudi2_disable_rot_qmans()
4116 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_ROT_SHIFT + i))) in gaudi2_disable_rot_qmans()
4126 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_disable_nic_qmans()
4130 if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK)) in gaudi2_disable_nic_qmans()
4136 if (!(hdev->nic_ports_mask & BIT(i))) in gaudi2_disable_nic_qmans()
4149 /* Zero the lower/upper parts of the 64-bit counter */ in gaudi2_enable_timestamp()
4171 return gaudi2_vdec_irq_name[irq_number - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM]; in gaudi2_irq_name()
4191 irq = pci_irq_vector(hdev->pdev, i); in gaudi2_dec_disable_msix()
4192 relative_idx = i - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM; in gaudi2_dec_disable_msix()
4194 dec = hdev->dec + relative_idx / 2; in gaudi2_dec_disable_msix()
4202 (void *) &hdev->user_interrupt[dec->core_id])); in gaudi2_dec_disable_msix()
4215 irq = pci_irq_vector(hdev->pdev, i); in gaudi2_dec_enable_msix()
4216 relative_idx = i - GAUDI2_IRQ_NUM_DCORE0_DEC0_NRM; in gaudi2_dec_enable_msix()
4225 dec = hdev->dec + relative_idx / 2; in gaudi2_dec_enable_msix()
4231 (void *) &hdev->user_interrupt[dec->core_id]); in gaudi2_dec_enable_msix()
4235 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in gaudi2_dec_enable_msix()
4249 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_enable_msix()
4250 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_enable_msix()
4254 if (gaudi2->hw_cap_initialized & HW_CAP_MSIX) in gaudi2_enable_msix()
4257 rc = pci_alloc_irq_vectors(hdev->pdev, GAUDI2_MSIX_ENTRIES, GAUDI2_MSIX_ENTRIES, in gaudi2_enable_msix()
4260 dev_err(hdev->dev, "MSI-X: Failed to enable support -- %d/%d\n", in gaudi2_enable_msix()
4265 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION); in gaudi2_enable_msix()
4266 cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_CS_COMPLETION]; in gaudi2_enable_msix()
4269 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in gaudi2_enable_msix()
4273 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE); in gaudi2_enable_msix()
4275 &hdev->event_queue); in gaudi2_enable_msix()
4277 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in gaudi2_enable_msix()
4283 dev_err(hdev->dev, "Failed to enable decoder IRQ"); in gaudi2_enable_msix()
4287 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT); in gaudi2_enable_msix()
4290 &hdev->tpc_interrupt); in gaudi2_enable_msix()
4292 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in gaudi2_enable_msix()
4296 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR); in gaudi2_enable_msix()
4299 &hdev->unexpected_error_interrupt); in gaudi2_enable_msix()
4301 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in gaudi2_enable_msix()
4305 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count, user_irq_init_cnt = 0; in gaudi2_enable_msix()
4306 user_irq_init_cnt < prop->user_interrupt_count; in gaudi2_enable_msix()
4309 irq = pci_irq_vector(hdev->pdev, i); in gaudi2_enable_msix()
4311 &hdev->user_interrupt[j]); in gaudi2_enable_msix()
4313 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in gaudi2_enable_msix()
4318 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EQ_ERROR); in gaudi2_enable_msix()
4323 dev_err(hdev->dev, "Failed to request IRQ %d", irq); in gaudi2_enable_msix()
4327 gaudi2->hw_cap_initialized |= HW_CAP_MSIX; in gaudi2_enable_msix()
4332 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count; in gaudi2_enable_msix()
4335 irq = pci_irq_vector(hdev->pdev, i); in gaudi2_enable_msix()
4336 free_irq(irq, &hdev->user_interrupt[j]); in gaudi2_enable_msix()
4338 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR); in gaudi2_enable_msix()
4339 free_irq(irq, &hdev->unexpected_error_interrupt); in gaudi2_enable_msix()
4341 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT); in gaudi2_enable_msix()
4342 free_irq(irq, &hdev->tpc_interrupt); in gaudi2_enable_msix()
4346 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE); in gaudi2_enable_msix()
4350 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION); in gaudi2_enable_msix()
4354 pci_free_irq_vectors(hdev->pdev); in gaudi2_enable_msix()
4361 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_sync_irqs()
4365 if (!(gaudi2->hw_cap_initialized & HW_CAP_MSIX)) in gaudi2_sync_irqs()
4369 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION)); in gaudi2_sync_irqs()
4372 irq = pci_irq_vector(hdev->pdev, i); in gaudi2_sync_irqs()
4376 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT)); in gaudi2_sync_irqs()
4377 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR)); in gaudi2_sync_irqs()
4379 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = 0 ; j < hdev->asic_prop.user_interrupt_count; in gaudi2_sync_irqs()
4381 irq = pci_irq_vector(hdev->pdev, i); in gaudi2_sync_irqs()
4385 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE)); in gaudi2_sync_irqs()
4386 synchronize_irq(pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EQ_ERROR)); in gaudi2_sync_irqs()
4391 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_disable_msix()
4392 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_disable_msix()
4396 if (!(gaudi2->hw_cap_initialized & HW_CAP_MSIX)) in gaudi2_disable_msix()
4401 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EVENT_QUEUE); in gaudi2_disable_msix()
4402 free_irq(irq, &hdev->event_queue); in gaudi2_disable_msix()
4406 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_TPC_ASSERT); in gaudi2_disable_msix()
4407 free_irq(irq, &hdev->tpc_interrupt); in gaudi2_disable_msix()
4409 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_UNEXPECTED_ERROR); in gaudi2_disable_msix()
4410 free_irq(irq, &hdev->unexpected_error_interrupt); in gaudi2_disable_msix()
4412 for (i = GAUDI2_IRQ_NUM_USER_FIRST, j = prop->user_dec_intr_count, k = 0; in gaudi2_disable_msix()
4413 k < hdev->asic_prop.user_interrupt_count ; i++, j++, k++) { in gaudi2_disable_msix()
4415 irq = pci_irq_vector(hdev->pdev, i); in gaudi2_disable_msix()
4416 free_irq(irq, &hdev->user_interrupt[j]); in gaudi2_disable_msix()
4419 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_COMPLETION); in gaudi2_disable_msix()
4420 cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_CS_COMPLETION]; in gaudi2_disable_msix()
4423 irq = pci_irq_vector(hdev->pdev, GAUDI2_IRQ_NUM_EQ_ERROR); in gaudi2_disable_msix()
4426 pci_free_irq_vectors(hdev->pdev); in gaudi2_disable_msix()
4428 gaudi2->hw_cap_initialized &= ~HW_CAP_MSIX; in gaudi2_disable_msix()
4438 if (hdev->pldm) in gaudi2_stop_dcore_dec()
4445 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit))) in gaudi2_stop_dcore_dec()
4465 dev_err(hdev->dev, in gaudi2_stop_dcore_dec()
4478 if (hdev->pldm) in gaudi2_stop_pcie_dec()
4485 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit))) in gaudi2_stop_pcie_dec()
4505 dev_err(hdev->dev, in gaudi2_stop_pcie_dec()
4513 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_stop_dec()
4516 if ((gaudi2->dec_hw_cap_initialized & HW_CAP_DEC_MASK) == 0) in gaudi2_stop_dec()
4553 if (hdev->pldm) in gaudi2_verify_arc_running_mode()
4577 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_reset_arcs()
4590 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_nic_qmans_manual_flush()
4594 if (!(gaudi2->nic_hw_cap_initialized & HW_CAP_NIC_MASK)) in gaudi2_nic_qmans_manual_flush()
4600 if (!(hdev->nic_ports_mask & BIT(i))) in gaudi2_nic_qmans_manual_flush()
4622 dev_err(hdev->dev, "failed to %s arc: %d\n", in gaudi2_set_engine_cores()
4625 return -1; in gaudi2_set_engine_cores()
4635 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_set_tpc_engine_mode()
4638 if (!(gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK)) in gaudi2_set_tpc_engine_mode()
4642 if (!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(HW_CAP_TPC_SHIFT + tpc_id))) in gaudi2_set_tpc_engine_mode()
4662 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_set_mme_engine_mode()
4666 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_MME_SHIFT + mme_id))) in gaudi2_set_mme_engine_mode()
4680 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_set_edma_engine_mode()
4683 if (!(gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK)) in gaudi2_set_edma_engine_mode()
4687 if (!(gaudi2->hw_cap_initialized & BIT_ULL(HW_CAP_EDMA_SHIFT + edma_id))) in gaudi2_set_edma_engine_mode()
4740 dev_err(hdev->dev, "Invalid engine ID %u\n", engine_ids[i]); in gaudi2_set_engine_modes()
4741 return -EINVAL; in gaudi2_set_engine_modes()
4761 dev_err(hdev->dev, "failed to execute command id %u\n", engine_command); in gaudi2_set_engines()
4762 return -EINVAL; in gaudi2_set_engines()
4770 if (hdev->pldm) in gaudi2_halt_engines()
4820 struct pre_fw_load_props *pre_fw_load = &hdev->fw_loader.pre_fw_load; in gaudi2_init_firmware_preload_params()
4822 pre_fw_load->cpu_boot_status_reg = mmPSOC_GLOBAL_CONF_CPU_BOOT_STATUS; in gaudi2_init_firmware_preload_params()
4823 pre_fw_load->sts_boot_dev_sts0_reg = mmCPU_BOOT_DEV_STS0; in gaudi2_init_firmware_preload_params()
4824 pre_fw_load->sts_boot_dev_sts1_reg = mmCPU_BOOT_DEV_STS1; in gaudi2_init_firmware_preload_params()
4825 pre_fw_load->boot_err0_reg = mmCPU_BOOT_ERR0; in gaudi2_init_firmware_preload_params()
4826 pre_fw_load->boot_err1_reg = mmCPU_BOOT_ERR1; in gaudi2_init_firmware_preload_params()
4827 pre_fw_load->wait_for_preboot_timeout = GAUDI2_PREBOOT_REQ_TIMEOUT_USEC; in gaudi2_init_firmware_preload_params()
4828 pre_fw_load->wait_for_preboot_extended_timeout = in gaudi2_init_firmware_preload_params()
4834 struct fw_load_mgr *fw_loader = &hdev->fw_loader; in gaudi2_init_firmware_loader()
4839 fw_loader->fw_comp_loaded = FW_TYPE_NONE; in gaudi2_init_firmware_loader()
4840 fw_loader->boot_fit_img.image_name = GAUDI2_BOOT_FIT_FILE; in gaudi2_init_firmware_loader()
4841 fw_loader->linux_img.image_name = GAUDI2_LINUX_FW_FILE; in gaudi2_init_firmware_loader()
4842 fw_loader->boot_fit_timeout = GAUDI2_BOOT_FIT_REQ_TIMEOUT_USEC; in gaudi2_init_firmware_loader()
4843 fw_loader->skip_bmc = false; in gaudi2_init_firmware_loader()
4844 fw_loader->sram_bar_id = SRAM_CFG_BAR_ID; in gaudi2_init_firmware_loader()
4845 fw_loader->dram_bar_id = DRAM_BAR_ID; in gaudi2_init_firmware_loader()
4846 fw_loader->cpu_timeout = GAUDI2_CPU_TIMEOUT_USEC; in gaudi2_init_firmware_loader()
4850 * hard-coded). in later stages of the protocol those values will be in gaudi2_init_firmware_loader()
4852 * will always be up-to-date in gaudi2_init_firmware_loader()
4854 dynamic_loader = &hdev->fw_loader.dynamic_loader; in gaudi2_init_firmware_loader()
4855 dyn_regs = &dynamic_loader->comm_desc.cpu_dyn_regs; in gaudi2_init_firmware_loader()
4856 dyn_regs->kmd_msg_to_cpu = cpu_to_le32(mmPSOC_GLOBAL_CONF_KMD_MSG_TO_CPU); in gaudi2_init_firmware_loader()
4857 dyn_regs->cpu_cmd_status_to_host = cpu_to_le32(mmCPU_CMD_STATUS_TO_HOST); in gaudi2_init_firmware_loader()
4858 dynamic_loader->wait_for_bl_timeout = GAUDI2_WAIT_FOR_BL_TIMEOUT_USEC; in gaudi2_init_firmware_loader()
4863 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_cpu()
4866 if (!(hdev->fw_components & FW_TYPE_PREBOOT_CPU)) in gaudi2_init_cpu()
4869 if (gaudi2->hw_cap_initialized & HW_CAP_CPU) in gaudi2_init_cpu()
4876 gaudi2->hw_cap_initialized |= HW_CAP_CPU; in gaudi2_init_cpu()
4883 struct hl_hw_queue *cpu_pq = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ]; in gaudi2_init_cpu_queues()
4884 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_init_cpu_queues()
4885 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_cpu_queues()
4891 if (!hdev->cpu_queues_enable) in gaudi2_init_cpu_queues()
4894 if (gaudi2->hw_cap_initialized & HW_CAP_CPU_Q) in gaudi2_init_cpu_queues()
4897 eq = &hdev->event_queue; in gaudi2_init_cpu_queues()
4899 dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi2_init_cpu_queues()
4901 WREG32(mmCPU_IF_PQ_BASE_ADDR_LOW, lower_32_bits(cpu_pq->bus_address)); in gaudi2_init_cpu_queues()
4902 WREG32(mmCPU_IF_PQ_BASE_ADDR_HIGH, upper_32_bits(cpu_pq->bus_address)); in gaudi2_init_cpu_queues()
4904 WREG32(mmCPU_IF_EQ_BASE_ADDR_LOW, lower_32_bits(eq->bus_address)); in gaudi2_init_cpu_queues()
4905 WREG32(mmCPU_IF_EQ_BASE_ADDR_HIGH, upper_32_bits(eq->bus_address)); in gaudi2_init_cpu_queues()
4907 WREG32(mmCPU_IF_CQ_BASE_ADDR_LOW, lower_32_bits(hdev->cpu_accessible_dma_address)); in gaudi2_init_cpu_queues()
4908 WREG32(mmCPU_IF_CQ_BASE_ADDR_HIGH, upper_32_bits(hdev->cpu_accessible_dma_address)); in gaudi2_init_cpu_queues()
4923 WREG32(le32_to_cpu(dyn_regs->gic_host_pi_upd_irq), in gaudi2_init_cpu_queues()
4935 dev_err(hdev->dev, "Failed to communicate with device CPU (timeout)\n"); in gaudi2_init_cpu_queues()
4936 return -EIO; in gaudi2_init_cpu_queues()
4940 if (prop->fw_cpu_boot_dev_sts0_valid) in gaudi2_init_cpu_queues()
4941 prop->fw_app_cpu_boot_dev_sts0 = RREG32(mmCPU_BOOT_DEV_STS0); in gaudi2_init_cpu_queues()
4943 if (prop->fw_cpu_boot_dev_sts1_valid) in gaudi2_init_cpu_queues()
4944 prop->fw_app_cpu_boot_dev_sts1 = RREG32(mmCPU_BOOT_DEV_STS1); in gaudi2_init_cpu_queues()
4946 gaudi2->hw_cap_initialized |= HW_CAP_CPU_Q; in gaudi2_init_cpu_queues()
4957 q = &hdev->kernel_queues[queue_id_base + pq_id]; in gaudi2_init_qman_pq()
4961 lower_32_bits(q->bus_address)); in gaudi2_init_qman_pq()
4963 upper_32_bits(q->bus_address)); in gaudi2_init_qman_pq()
4995 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_qman_pqc()
5006 lower_32_bits(gaudi2->scratchpad_bus_address)); in gaudi2_init_qman_pqc()
5008 upper_32_bits(gaudi2->scratchpad_bus_address)); in gaudi2_init_qman_pqc()
5024 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi2_get_dyn_sp_reg()
5037 sp_reg_addr = le32_to_cpu(dyn_regs->gic_dma_qm_irq_ctrl); in gaudi2_get_dyn_sp_reg()
5046 sp_reg_addr = le32_to_cpu(dyn_regs->gic_mme_qm_irq_ctrl); in gaudi2_get_dyn_sp_reg()
5055 sp_reg_addr = le32_to_cpu(dyn_regs->gic_tpc_qm_irq_ctrl); in gaudi2_get_dyn_sp_reg()
5058 sp_reg_addr = le32_to_cpu(dyn_regs->gic_rot_qm_irq_ctrl); in gaudi2_get_dyn_sp_reg()
5061 sp_reg_addr = le32_to_cpu(dyn_regs->gic_nic_qm_irq_ctrl); in gaudi2_get_dyn_sp_reg()
5064 dev_err(hdev->dev, "Unexpected h/w queue %d\n", queue_id_base); in gaudi2_get_dyn_sp_reg()
5113 hdev->kernel_queues[queue_id_base + pq_id].cq_id = GAUDI2_RESERVED_CQ_CS_COMPLETION; in gaudi2_init_qman()
5134 dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi2_init_dma_core()
5135 irq_handler_offset = le32_to_cpu(dyn_regs->gic_dma_core_irq_ctrl); in gaudi2_init_dma_core()
5153 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_kdma()
5156 if ((gaudi2->hw_cap_initialized & HW_CAP_KDMA) == HW_CAP_KDMA) in gaudi2_init_kdma()
5163 gaudi2->hw_cap_initialized |= HW_CAP_KDMA; in gaudi2_init_kdma()
5168 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_pdma()
5171 if ((gaudi2->hw_cap_initialized & HW_CAP_PDMA_MASK) == HW_CAP_PDMA_MASK) in gaudi2_init_pdma()
5186 gaudi2->hw_cap_initialized |= HW_CAP_PDMA_MASK; in gaudi2_init_pdma()
5205 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_init_edma()
5206 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_edma()
5209 if ((gaudi2->hw_cap_initialized & HW_CAP_EDMA_MASK) == HW_CAP_EDMA_MASK) in gaudi2_init_edma()
5216 if (!(prop->edma_enabled_mask & BIT(seq))) in gaudi2_init_edma()
5221 gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_EDMA_SHIFT + seq); in gaudi2_init_edma()
5227 * gaudi2_arm_monitors_for_virt_msix_db() - Arm monitors for writing to the virtual MSI-X doorbell.
5234 * write directly to the HBW host memory of the virtual MSI-X doorbell.
5239 * completion, by decrementing the sync object value and re-arming the monitor.
5245 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_arm_monitors_for_virt_msix_db()
5254 * 1. Write interrupt ID to the virtual MSI-X doorbell (master monitor) in gaudi2_arm_monitors_for_virt_msix_db()
5256 * 3. Re-arm the master monitor. in gaudi2_arm_monitors_for_virt_msix_db()
5268 payload = FIELD_PREP(DCORE0_SYNC_MNGR_OBJS_SOB_OBJ_VAL_MASK, 0x7FFF) | /* "-1" */ in gaudi2_arm_monitors_for_virt_msix_db()
5273 /* 3rd monitor: Re-arm the master monitor */ in gaudi2_arm_monitors_for_virt_msix_db()
5291 /* 1st monitor (master): Write interrupt ID to the virtual MSI-X doorbell */ in gaudi2_arm_monitors_for_virt_msix_db()
5297 addr = gaudi2->virt_msix_db_dma_addr; in gaudi2_arm_monitors_for_virt_msix_db()
5310 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_prepare_sm_for_virt_msix_db()
5314 if (!(prop->decoder_enabled_mask & BIT(decoder_id))) in gaudi2_prepare_sm_for_virt_msix_db()
5331 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_sm()
5347 /* Init CQ0 DB - configure the monitor to trigger MSI-X interrupt */ in gaudi2_init_sm()
5348 WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0, lower_32_bits(gaudi2->virt_msix_db_dma_addr)); in gaudi2_init_sm()
5349 WREG32(mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0, upper_32_bits(gaudi2->virt_msix_db_dma_addr)); in gaudi2_init_sm()
5354 hdev->completion_queue[i].bus_address; in gaudi2_init_sm()
5368 /* Initialize sync objects and monitors which are used for the virtual MSI-X doorbell */ in gaudi2_init_sm()
5374 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_mme_acc()
5390 WREG32(reg_base + MME_ACC_AP_LFSR_SEED_WDATA_OFFSET, gaudi2->lfsr_rand_seeds[i]); in gaudi2_init_mme_acc()
5413 dev_err(hdev->dev, "Invalid dcore id %u\n", dcore_id); in gaudi2_init_dcore_mme()
5428 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_mme()
5431 if ((gaudi2->hw_cap_initialized & HW_CAP_MME_MASK) == HW_CAP_MME_MASK) in gaudi2_init_mme()
5437 gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_MME_SHIFT + i); in gaudi2_init_mme()
5458 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_tpc_config()
5459 struct gaudi2_tpc_init_cfg_data *cfg_data = ctx->data; in gaudi2_init_tpc_config()
5463 queue_id_base = cfg_data->dcore_tpc_qid_base[dcore] + (inst * NUM_OF_PQ_PER_QMAN); in gaudi2_init_tpc_config()
5465 if (dcore == 0 && inst == (NUM_DCORE0_TPC - 1)) in gaudi2_init_tpc_config()
5474 gaudi2->tpc_hw_cap_initialized |= BIT_ULL(HW_CAP_TPC_SHIFT + seq); in gaudi2_init_tpc_config()
5479 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_tpc()
5483 if (!hdev->asic_prop.tpc_enabled_mask) in gaudi2_init_tpc()
5486 if ((gaudi2->tpc_hw_cap_initialized & HW_CAP_TPC_MASK) == HW_CAP_TPC_MASK) in gaudi2_init_tpc()
5500 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_rotator()
5509 gaudi2->hw_cap_initialized |= BIT_ULL(HW_CAP_ROT_SHIFT + i); in gaudi2_init_rotator()
5532 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_init_dec()
5536 if (!hdev->asic_prop.decoder_enabled_mask) in gaudi2_init_dec()
5539 if ((gaudi2->dec_hw_cap_initialized & HW_CAP_DEC_MASK) == HW_CAP_DEC_MASK) in gaudi2_init_dec()
5546 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit))) in gaudi2_init_dec()
5556 gaudi2->dec_hw_cap_initialized |= BIT_ULL(HW_CAP_DEC_SHIFT + dec_bit); in gaudi2_init_dec()
5561 if (!(hdev->asic_prop.decoder_enabled_mask & BIT(dec_bit))) in gaudi2_init_dec()
5569 gaudi2->dec_hw_cap_initialized |= BIT_ULL(HW_CAP_DEC_SHIFT + dec_bit); in gaudi2_init_dec()
5579 if (hdev->pldm || !hdev->pdev) in gaudi2_mmu_update_asid_hop0_addr()
5598 dev_err(hdev->dev, "Timeout during MMU hop0 config of asid %d\n", asid); in gaudi2_mmu_update_asid_hop0_addr()
5625 timeout_usec = (hdev->pldm) ? GAUDI2_PLDM_MMU_TIMEOUT_USEC : in gaudi2_mmu_invalidate_cache_status_poll()
5629 if (inv_params->flags & MMU_OP_CLEAR_MEMCACHE) { in gaudi2_mmu_invalidate_cache_status_poll()
5648 if (inv_params->flags & MMU_OP_SKIP_LOW_CACHE_INV) in gaudi2_mmu_invalidate_cache_status_poll()
5651 start_offset = inv_params->range_invalidation ? in gaudi2_mmu_invalidate_cache_status_poll()
5667 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_is_hmmu_enabled()
5672 if (gaudi2->hw_cap_initialized & hw_cap) in gaudi2_is_hmmu_enabled()
5692 if (inv_params->range_invalidation) { in gaudi2_mmu_invalidate_cache_trigger()
5699 u64 start = inv_params->start_va - 1; in gaudi2_mmu_invalidate_cache_trigger()
5710 inv_params->end_va >> MMU_RANGE_INV_VA_LSB_SHIFT); in gaudi2_mmu_invalidate_cache_trigger()
5713 inv_params->end_va >> MMU_RANGE_INV_VA_MSB_SHIFT); in gaudi2_mmu_invalidate_cache_trigger()
5719 inv_params->inv_start_val, inv_params->flags); in gaudi2_mmu_invalidate_cache_trigger()
5775 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_mmu_invalidate_cache()
5779 if (hdev->reset_info.hard_reset_pending) in gaudi2_mmu_invalidate_cache()
5785 if ((flags & MMU_OP_USERPTR) && (gaudi2->hw_cap_initialized & HW_CAP_PMMU)) { in gaudi2_mmu_invalidate_cache()
5802 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_mmu_invalidate_cache_range()
5807 if (hdev->reset_info.hard_reset_pending) in gaudi2_mmu_invalidate_cache_range()
5816 if ((flags & MMU_OP_USERPTR) && (gaudi2->hw_cap_initialized & HW_CAP_PMMU)) { in gaudi2_mmu_invalidate_cache_range()
5852 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_mmu_update_hop0_addr()
5854 u32 asid, max_asid = prop->max_asid; in gaudi2_mmu_update_hop0_addr()
5858 if (hdev->pldm) in gaudi2_mmu_update_hop0_addr()
5862 hop0_addr = hdev->mmu_priv.hr.mmu_asid_hop0[asid].phys_addr; in gaudi2_mmu_update_hop0_addr()
5865 dev_err(hdev->dev, "failed to set hop0 addr for asid %d\n", asid); in gaudi2_mmu_update_hop0_addr()
5878 if (hdev->pldm || !hdev->pdev) in gaudi2_mmu_init_common()
5894 dev_notice_ratelimited(hdev->dev, "Timeout when waiting for MMU SRAM init\n"); in gaudi2_mmu_init_common()
5911 dev_notice_ratelimited(hdev->dev, "Timeout when waiting for MMU invalidate all\n"); in gaudi2_mmu_init_common()
5920 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_pci_mmu_init()
5924 if (gaudi2->hw_cap_initialized & HW_CAP_PMMU) in gaudi2_pci_mmu_init()
5963 gaudi2->hw_cap_initialized |= HW_CAP_PMMU; in gaudi2_pci_mmu_init()
5971 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_dcore_hmmu_init()
5972 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_dcore_hmmu_init()
5984 if ((gaudi2->hw_cap_initialized & hw_cap) || !(prop->hmmu_hif_enabled_mask & BIT(dmmu_seq))) in gaudi2_dcore_hmmu_init()
6015 gaudi2->hw_cap_initialized |= hw_cap; in gaudi2_dcore_hmmu_init()
6051 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_hw_init()
6068 if (hdev->asic_prop.iatu_done_by_fw) in gaudi2_hw_init()
6069 gaudi2->dram_bar_cur_addr = DRAM_PHYS_BASE; in gaudi2_hw_init()
6072 * Before pushing u-boot/linux to device, need to set the hbm bar to in gaudi2_hw_init()
6076 dev_err(hdev->dev, "failed to map HBM bar to DRAM base address\n"); in gaudi2_hw_init()
6077 return -EIO; in gaudi2_hw_init()
6082 dev_err(hdev->dev, "failed to initialize CPU\n"); in gaudi2_hw_init()
6091 dev_err(hdev->dev, "failed to initialize CPU H/W queues %d\n", rc); in gaudi2_hw_init()
6095 rc = gaudi2->cpucp_info_get(hdev); in gaudi2_hw_init()
6097 dev_err(hdev->dev, "Failed to get cpucp info\n"); in gaudi2_hw_init()
6140 * gaudi2_send_hard_reset_cmd - common function to handle reset
6149 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi2_send_hard_reset_cmd()
6151 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_send_hard_reset_cmd()
6154 preboot_only = (hdev->fw_loader.fw_comp_loaded == FW_TYPE_PREBOOT_CPU); in gaudi2_send_hard_reset_cmd()
6155 heartbeat_reset = (hdev->reset_info.curr_reset_cause == HL_RESET_CAUSE_HEARTBEAT); in gaudi2_send_hard_reset_cmd()
6170 if (gaudi2 && (gaudi2->hw_cap_initialized & HW_CAP_CPU) && in gaudi2_send_hard_reset_cmd()
6183 WREG32(le32_to_cpu(dyn_regs->gic_host_halt_irq), in gaudi2_send_hard_reset_cmd()
6193 * For the case in which we are working with Linux/Bootfit this is a hail-mary in gaudi2_send_hard_reset_cmd()
6207 if (hdev->asic_prop.hard_reset_done_by_fw) in gaudi2_send_hard_reset_cmd()
6215 * gaudi2_execute_hard_reset - execute hard reset by driver/FW
6223 if (hdev->asic_prop.hard_reset_done_by_fw) { in gaudi2_execute_hard_reset()
6254 dev_err(hdev->dev, "Timeout while waiting for FW to complete soft reset (0x%x)\n", in gaudi2_get_soft_rst_done_indication()
6260 * gaudi2_execute_soft_reset - execute soft reset by driver/FW
6293 gaudi2_write_rr_to_all_lbw_rtrs(hdev, RR_TYPE_LONG, NUM_LONG_LBW_RR - 1, in gaudi2_execute_soft_reset()
6296 gaudi2_write_rr_to_all_lbw_rtrs(hdev, RR_TYPE_LONG, NUM_LONG_LBW_RR - 2, in gaudi2_execute_soft_reset()
6322 dev_err(hdev->dev, "Timeout while waiting for device to reset 0x%x\n", reg_val); in gaudi2_poll_btm_indication()
6327 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_hw_fini()
6332 if (hdev->pldm) { in gaudi2_hw_fini()
6347 driver_performs_reset = !hdev->asic_prop.hard_reset_done_by_fw; in gaudi2_hw_fini()
6356 driver_performs_reset = (hdev->fw_components == FW_TYPE_PREBOOT_CPU && in gaudi2_hw_fini()
6357 !hdev->asic_prop.fw_security_enabled); in gaudi2_hw_fini()
6369 * - setting the dirty bit in gaudi2_hw_fini()
6372 * - dirty bit cleared in gaudi2_hw_fini()
6373 * - BTM indication cleared in gaudi2_hw_fini()
6374 * - preboot ready indication cleared in gaudi2_hw_fini()
6376 * - BTM indication will be set in gaudi2_hw_fini()
6377 * - BIST test performed and another reset triggered in gaudi2_hw_fini()
6388 if (hdev->fw_components & FW_TYPE_PREBOOT_CPU) in gaudi2_hw_fini()
6397 gaudi2->dec_hw_cap_initialized &= ~(HW_CAP_DEC_MASK); in gaudi2_hw_fini()
6398 gaudi2->tpc_hw_cap_initialized &= ~(HW_CAP_TPC_MASK); in gaudi2_hw_fini()
6401 * Clear NIC capability mask in order for driver to re-configure in gaudi2_hw_fini()
6402 * NIC QMANs. NIC ports will not be re-configured during soft in gaudi2_hw_fini()
6405 gaudi2->nic_hw_cap_initialized &= ~(HW_CAP_NIC_MASK); in gaudi2_hw_fini()
6408 gaudi2->hw_cap_initialized &= in gaudi2_hw_fini()
6415 memset(gaudi2->events_stat, 0, sizeof(gaudi2->events_stat)); in gaudi2_hw_fini()
6417 gaudi2->hw_cap_initialized &= in gaudi2_hw_fini()
6431 dev_err(hdev->dev, "Failed to disable PCI access from CPU\n"); in gaudi2_suspend()
6451 rc = dma_mmap_coherent(hdev->dev, vma, cpu_addr, dma_addr, size); in gaudi2_mmap()
6453 dev_err(hdev->dev, "dma_mmap_coherent error %d", rc); in gaudi2_mmap()
6457 rc = remap_pfn_range(vma, vma->vm_start, in gaudi2_mmap()
6459 size, vma->vm_page_prot); in gaudi2_mmap()
6461 dev_err(hdev->dev, "remap_pfn_range error %d", rc); in gaudi2_mmap()
6470 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_is_queue_enabled()
6484 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE0_EDMA_0_0) >> 2); in gaudi2_is_queue_enabled()
6488 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE1_EDMA_0_0) >> 2); in gaudi2_is_queue_enabled()
6492 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE2_EDMA_0_0) >> 2); in gaudi2_is_queue_enabled()
6496 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE3_EDMA_0_0) >> 2); in gaudi2_is_queue_enabled()
6517 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE0_TPC_0_0) >> 2); in gaudi2_is_queue_enabled()
6521 return !!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(0)); in gaudi2_is_queue_enabled()
6526 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE1_TPC_0_0) >> 2); in gaudi2_is_queue_enabled()
6531 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE2_TPC_0_0) >> 2); in gaudi2_is_queue_enabled()
6536 ((hw_queue_id - GAUDI2_QUEUE_ID_DCORE3_TPC_0_0) >> 2); in gaudi2_is_queue_enabled()
6544 hw_test_cap_bit = HW_CAP_ROT_SHIFT + ((hw_queue_id - GAUDI2_QUEUE_ID_ROT_0_0) >> 2); in gaudi2_is_queue_enabled()
6548 hw_nic_cap_bit = HW_CAP_NIC_SHIFT + ((hw_queue_id - GAUDI2_QUEUE_ID_NIC_0_0) >> 2); in gaudi2_is_queue_enabled()
6552 return !!(gaudi2->nic_hw_cap_initialized & BIT_ULL(0)); in gaudi2_is_queue_enabled()
6556 return !!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q); in gaudi2_is_queue_enabled()
6563 return !!(gaudi2->tpc_hw_cap_initialized & BIT_ULL(hw_tpc_cap_bit)); in gaudi2_is_queue_enabled()
6566 return !!(gaudi2->nic_hw_cap_initialized & BIT_ULL(hw_nic_cap_bit)); in gaudi2_is_queue_enabled()
6571 return !!(gaudi2->hw_cap_initialized & hw_cap_mask); in gaudi2_is_queue_enabled()
6576 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_is_arc_enabled()
6581 return !!(gaudi2->active_hw_arc & BIT_ULL(arc_id)); in gaudi2_is_arc_enabled()
6584 return !!(gaudi2->active_tpc_arc & BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0)); in gaudi2_is_arc_enabled()
6587 return !!(gaudi2->active_nic_arc & BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0)); in gaudi2_is_arc_enabled()
6596 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_clr_arc_id_cap()
6601 gaudi2->active_hw_arc &= ~(BIT_ULL(arc_id)); in gaudi2_clr_arc_id_cap()
6605 gaudi2->active_tpc_arc &= ~(BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0)); in gaudi2_clr_arc_id_cap()
6609 gaudi2->active_nic_arc &= ~(BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0)); in gaudi2_clr_arc_id_cap()
6619 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_set_arc_id_cap()
6624 gaudi2->active_hw_arc |= BIT_ULL(arc_id); in gaudi2_set_arc_id_cap()
6628 gaudi2->active_tpc_arc |= BIT_ULL(arc_id - CPU_ID_TPC_QMAN_ARC0); in gaudi2_set_arc_id_cap()
6632 gaudi2->active_nic_arc |= BIT_ULL(arc_id - CPU_ID_NIC_QMAN_ARC0); in gaudi2_set_arc_id_cap()
6642 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi2_ring_doorbell()
6666 WREG32(le32_to_cpu(dyn_regs->gic_host_pi_upd_irq), in gaudi2_ring_doorbell()
6683 return dma_alloc_coherent(&hdev->pdev->dev, size, dma_handle, flags); in gaudi2_dma_alloc_coherent()
6689 dma_free_coherent(&hdev->pdev->dev, size, cpu_addr, dma_handle); in gaudi2_dma_free_coherent()
6695 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_send_cpu_message()
6697 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) { in gaudi2_send_cpu_message()
6715 return dma_pool_zalloc(hdev->dma_pool, mem_flags, dma_handle); in gaudi2_dma_pool_zalloc()
6720 dma_pool_free(hdev->dma_pool, vaddr, dma_addr); in gaudi2_dma_pool_free()
6736 struct asic_fixed_properties *asic_prop = &hdev->asic_prop; in gaudi2_validate_cb_address()
6737 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_validate_cb_address()
6739 if (!gaudi2_is_queue_enabled(hdev, parser->hw_queue_id)) { in gaudi2_validate_cb_address()
6740 dev_err(hdev->dev, "h/w queue %d is disabled\n", parser->hw_queue_id); in gaudi2_validate_cb_address()
6741 return -EINVAL; in gaudi2_validate_cb_address()
6746 if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, in gaudi2_validate_cb_address()
6747 parser->user_cb_size, in gaudi2_validate_cb_address()
6748 asic_prop->sram_user_base_address, in gaudi2_validate_cb_address()
6749 asic_prop->sram_end_address)) in gaudi2_validate_cb_address()
6752 if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, in gaudi2_validate_cb_address()
6753 parser->user_cb_size, in gaudi2_validate_cb_address()
6754 asic_prop->dram_user_base_address, in gaudi2_validate_cb_address()
6755 asic_prop->dram_end_address)) in gaudi2_validate_cb_address()
6758 if ((gaudi2->hw_cap_initialized & HW_CAP_DMMU_MASK) && in gaudi2_validate_cb_address()
6759 hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, in gaudi2_validate_cb_address()
6760 parser->user_cb_size, in gaudi2_validate_cb_address()
6761 asic_prop->dmmu.start_addr, in gaudi2_validate_cb_address()
6762 asic_prop->dmmu.end_addr)) in gaudi2_validate_cb_address()
6765 if (gaudi2->hw_cap_initialized & HW_CAP_PMMU) { in gaudi2_validate_cb_address()
6766 if (hl_mem_area_inside_range((u64) (uintptr_t) parser->user_cb, in gaudi2_validate_cb_address()
6767 parser->user_cb_size, in gaudi2_validate_cb_address()
6768 asic_prop->pmmu.start_addr, in gaudi2_validate_cb_address()
6769 asic_prop->pmmu.end_addr) || in gaudi2_validate_cb_address()
6771 (u64) (uintptr_t) parser->user_cb, in gaudi2_validate_cb_address()
6772 parser->user_cb_size, in gaudi2_validate_cb_address()
6773 asic_prop->pmmu_huge.start_addr, in gaudi2_validate_cb_address()
6774 asic_prop->pmmu_huge.end_addr)) in gaudi2_validate_cb_address()
6777 } else if (gaudi2_host_phys_addr_valid((u64) (uintptr_t) parser->user_cb)) { in gaudi2_validate_cb_address()
6778 if (!hdev->pdev) in gaudi2_validate_cb_address()
6781 if (!device_iommu_mapped(&hdev->pdev->dev)) in gaudi2_validate_cb_address()
6785 dev_err(hdev->dev, "CB address %p + 0x%x for internal QMAN is not valid\n", in gaudi2_validate_cb_address()
6786 parser->user_cb, parser->user_cb_size); in gaudi2_validate_cb_address()
6788 return -EFAULT; in gaudi2_validate_cb_address()
6793 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_cs_parser()
6795 if (!parser->is_kernel_allocated_cb) in gaudi2_cs_parser()
6798 if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU)) { in gaudi2_cs_parser()
6799 dev_err(hdev->dev, "PMMU not initialized - Unsupported mode in Gaudi2\n"); in gaudi2_cs_parser()
6800 return -EINVAL; in gaudi2_cs_parser()
6808 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_send_heartbeat()
6810 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) in gaudi2_send_heartbeat()
6902 cq = &hdev->completion_queue[GAUDI2_RESERVED_CQ_KDMA_COMPLETION]; in gaudi2_send_job_to_kdma()
6903 cq_base = cq->kernel_address; in gaudi2_send_job_to_kdma()
6904 polling_addr = (u32 *)&cq_base[cq->ci]; in gaudi2_send_job_to_kdma()
6906 if (hdev->pldm) in gaudi2_send_job_to_kdma()
6925 dev_err(hdev->dev, "Timeout while waiting for KDMA to be idle\n"); in gaudi2_send_job_to_kdma()
6930 cq->ci = hl_cq_inc_ptr(cq->ci); in gaudi2_send_job_to_kdma()
6958 return hdev->asic_prop.first_available_user_sob[0] + in gaudi2_test_queue_hw_queue_id_to_sob_id()
6959 hw_queue_id - GAUDI2_QUEUE_ID_PDMA_0_0; in gaudi2_test_queue_hw_queue_id_to_sob_id()
6976 struct packet_msg_short *msg_short_pkt = msg_info->kern_addr; in gaudi2_test_queue_send_msg_short()
6986 msg_short_pkt->value = cpu_to_le32(sob_val); in gaudi2_test_queue_send_msg_short()
6987 msg_short_pkt->ctl = cpu_to_le32(tmp); in gaudi2_test_queue_send_msg_short()
6989 rc = hl_hw_queue_send_cb_no_cmpl(hdev, hw_queue_id, pkt_size, msg_info->dma_addr); in gaudi2_test_queue_send_msg_short()
6991 dev_err(hdev->dev, in gaudi2_test_queue_send_msg_short()
7004 if (hdev->pldm) in gaudi2_test_queue_wait_completion()
7017 if (rc == -ETIMEDOUT) { in gaudi2_test_queue_wait_completion()
7018 dev_err(hdev->dev, "H/W queue %d test failed (SOB_OBJ_0 == 0x%x)\n", in gaudi2_test_queue_wait_completion()
7020 rc = -EIO; in gaudi2_test_queue_wait_completion()
7028 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_test_cpu_queue()
7034 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) in gaudi2_test_cpu_queue()
7042 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_test_queues()
7052 msg_info = &gaudi2->queues_test_info[i - GAUDI2_QUEUE_ID_PDMA_0_0]; in gaudi2_test_queues()
7071 /* chip is not usable, no need for cleanups, just bail-out with error */ in gaudi2_test_queues()
7084 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_compute_reset_late_init()
7092 dev_err(hdev->dev, "Failed to scrub arcs DCCM\n"); in gaudi2_compute_reset_late_init()
7099 irq_arr_size = gaudi2->num_of_valid_hw_events * sizeof(gaudi2->hw_events[0]); in gaudi2_compute_reset_late_init()
7100 return hl_fw_unmask_irq_arr(hdev, gaudi2->hw_events, irq_arr_size); in gaudi2_compute_reset_late_init()
7107 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_get_edma_idle_status()
7109 const char *edma_fmt = "%-6d%-6d%-9s%#-14x%#-15x%#x\n"; in gaudi2_get_edma_idle_status()
7117 "---- ---- ------- ------------ ------------- -------------\n"); in gaudi2_get_edma_idle_status()
7123 if (!(prop->edma_enabled_mask & BIT(seq))) in gaudi2_get_edma_idle_status()
7158 const char *pdma_fmt = "%-6d%-9s%#-14x%#-15x%#x\n"; in gaudi2_get_pdma_idle_status()
7166 "---- ------- ------------ ------------- -------------\n"); in gaudi2_get_pdma_idle_status()
7197 const char *nic_fmt = "%-5d%-9s%#-14x%#-12x\n"; in gaudi2_get_nic_idle_status()
7204 if (e && hdev->nic_ports_mask) in gaudi2_get_nic_idle_status()
7207 "--- ------- ------------ ----------\n"); in gaudi2_get_nic_idle_status()
7215 if (!(hdev->nic_ports_mask & BIT(i))) in gaudi2_get_nic_idle_status()
7244 const char *mme_fmt = "%-5d%-6s%-9s%#-14x%#x\n"; in gaudi2_get_mme_idle_status()
7252 "--- ---- ------- ------------ ---------------\n"); in gaudi2_get_mme_idle_status()
7285 struct gaudi2_tpc_idle_data *idle_data = ctx->data; in gaudi2_is_tpc_engine_idle()
7290 if ((dcore == 0) && (inst == (NUM_DCORE0_TPC - 1))) in gaudi2_is_tpc_engine_idle()
7303 *(idle_data->is_idle) &= is_eng_idle; in gaudi2_is_tpc_engine_idle()
7305 if (idle_data->mask && !is_eng_idle) in gaudi2_is_tpc_engine_idle()
7306 set_bit(engine_idx, idle_data->mask); in gaudi2_is_tpc_engine_idle()
7308 if (idle_data->e) in gaudi2_is_tpc_engine_idle()
7309 hl_engine_data_sprintf(idle_data->e, in gaudi2_is_tpc_engine_idle()
7310 idle_data->tpc_fmt, dcore, inst, in gaudi2_is_tpc_engine_idle()
7318 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_get_tpc_idle_status()
7323 .tpc_fmt = "%-6d%-5d%-9s%#-14x%#-12x%#x\n", in gaudi2_get_tpc_idle_status()
7333 if (e && prop->tpc_enabled_mask) in gaudi2_get_tpc_idle_status()
7336 "---- --- ------- ------------ ---------- ------\n"); in gaudi2_get_tpc_idle_status()
7346 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_get_decoder_idle_status()
7348 const char *pcie_dec_fmt = "%-10d%-9s%#x\n"; in gaudi2_get_decoder_idle_status()
7349 const char *dec_fmt = "%-6d%-5d%-9s%#x\n"; in gaudi2_get_decoder_idle_status()
7356 if (e && (prop->decoder_enabled_mask & (~PCIE_DEC_EN_MASK))) in gaudi2_get_decoder_idle_status()
7359 "---- --- ------- ---------------\n"); in gaudi2_get_decoder_idle_status()
7364 if (!(prop->decoder_enabled_mask & dec_enabled_bit)) in gaudi2_get_decoder_idle_status()
7384 if (e && (prop->decoder_enabled_mask & PCIE_DEC_EN_MASK)) in gaudi2_get_decoder_idle_status()
7387 "-------- ------- ---------------\n"); in gaudi2_get_decoder_idle_status()
7392 if (!(prop->decoder_enabled_mask & BIT(dec_enabled_bit))) in gaudi2_get_decoder_idle_status()
7415 const char *rot_fmt = "%-6d%-5d%-9s%#-14x%#-14x%#x\n"; in gaudi2_get_rotator_idle_status()
7425 "---- --- ------- ------------ ------------ ----------\n"); in gaudi2_get_rotator_idle_status()
7467 __acquires(&gaudi2->hw_queues_lock) in gaudi2_hw_queues_lock()
7469 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_hw_queues_lock()
7471 spin_lock(&gaudi2->hw_queues_lock); in gaudi2_hw_queues_lock()
7475 __releases(&gaudi2->hw_queues_lock) in gaudi2_hw_queues_unlock()
7477 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_hw_queues_unlock()
7479 spin_unlock(&gaudi2->hw_queues_lock); in gaudi2_hw_queues_unlock()
7484 return hdev->pdev->device; in gaudi2_get_pci_id()
7489 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_get_eeprom_data()
7491 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) in gaudi2_get_eeprom_data()
7504 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_get_events_stat()
7507 *size = (u32) sizeof(gaudi2->events_stat_aggregate); in gaudi2_get_events_stat()
7508 return gaudi2->events_stat_aggregate; in gaudi2_get_events_stat()
7511 *size = (u32) sizeof(gaudi2->events_stat); in gaudi2_get_events_stat()
7512 return gaudi2->events_stat; in gaudi2_get_events_stat()
7518 u32 offset = (mmDCORE0_VDEC1_BRDG_CTRL_BASE - mmDCORE0_VDEC0_BRDG_CTRL_BASE) * in gaudi2_mmu_vdec_dcore_prepare()
7541 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_mmu_dcore_prepare()
7548 if (prop->edma_enabled_mask & BIT(edma_seq_base)) { in gaudi2_mmu_dcore_prepare()
7555 if (prop->edma_enabled_mask & BIT(edma_seq_base + 1)) { in gaudi2_mmu_dcore_prepare()
7565 * Sync Mngrs on dcores 1 - 3 are exposed to user, so must use user ASID in gaudi2_mmu_dcore_prepare()
7601 if (prop->decoder_enabled_mask & BIT(dcore_id * NUM_OF_DEC_PER_DCORE + vdec_id)) in gaudi2_mmu_dcore_prepare()
7609 u32 offset = (mmPCIE_VDEC1_BRDG_CTRL_BASE - mmPCIE_VDEC0_BRDG_CTRL_BASE) * shared_vdec_id; in gudi2_mmu_vdec_shared_prepare()
7630 u32 offset = (mmARC_FARM_ARC1_DUP_ENG_BASE - mmARC_FARM_ARC0_DUP_ENG_BASE) * arc_farm_id; in gudi2_mmu_arc_farm_arc_dup_eng_prepare()
7642 /* Enable MMU and configure asid for all relevant ARC regions */ in gaudi2_arc_mmu_prepare()
7684 if (hdev->fw_components & FW_TYPE_BOOT_CPU) in gaudi2_arc_mmu_prepare_all()
7702 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_mmu_shared_prepare()
7730 if (prop->decoder_enabled_mask & BIT(NUM_OF_DCORES * NUM_OF_DEC_PER_DCORE + 0)) in gaudi2_mmu_shared_prepare()
7733 if (prop->decoder_enabled_mask & BIT(NUM_OF_DCORES * NUM_OF_DEC_PER_DCORE + 1)) in gaudi2_mmu_shared_prepare()
7750 struct gaudi2_tpc_mmu_data *mmu_data = ctx->data; in gaudi2_tpc_mmu_prepare()
7753 WREG32(mmDCORE0_TPC0_CFG_AXUSER_HB_ASID + offset, mmu_data->rw_asid); in gaudi2_tpc_mmu_prepare()
7755 WREG32(mmDCORE0_TPC0_QM_AXUSER_NONSECURED_HB_ASID + offset, mmu_data->rw_asid); in gaudi2_tpc_mmu_prepare()
7761 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_mmu_prepare()
7770 dev_crit(hdev->dev, "asid %u is too big\n", asid); in gaudi2_mmu_prepare()
7771 return -EINVAL; in gaudi2_mmu_prepare()
7774 if (!(gaudi2->hw_cap_initialized & HW_CAP_MMU_MASK)) in gaudi2_mmu_prepare()
7798 /* return in case of NIC status event - these events are received periodically and not as in is_info_event()
7820 dev_err_ratelimited(hdev->dev, "%s: %pV\n", in gaudi2_print_event()
7824 dev_err(hdev->dev, "%s: %pV\n", in gaudi2_print_event()
7842 ecc_address = le64_to_cpu(ecc_data->ecc_address); in gaudi2_handle_ecc_event()
7843 ecc_syndrome = le64_to_cpu(ecc_data->ecc_syndrom); in gaudi2_handle_ecc_event()
7844 memory_wrapper_idx = ecc_data->memory_wrapper_idx; in gaudi2_handle_ecc_event()
7847 block_id = le16_to_cpu(ecc_data->block_id); in gaudi2_handle_ecc_event()
7848 gaudi2_print_event(hdev, event_type, !ecc_data->is_critical, in gaudi2_handle_ecc_event()
7851 ecc_data->is_critical); in gaudi2_handle_ecc_event()
7853 gaudi2_print_event(hdev, event_type, !ecc_data->is_critical, in gaudi2_handle_ecc_event()
7855 ecc_address, ecc_syndrome, memory_wrapper_idx, ecc_data->is_critical); in gaudi2_handle_ecc_event()
7858 return !!ecc_data->is_critical; in gaudi2_handle_ecc_event()
7863 struct undefined_opcode_info *undef_opcode = &hdev->captured_err_info.undef_opcode; in handle_lower_qman_data_on_err()
7869 is_arc_cq = FIELD_GET(PDMA0_QM_CP_STS_CUR_CQ_MASK, cp_sts); /* 0 - legacy CQ, 1 - ARC_CQ */ in handle_lower_qman_data_on_err()
7887 dev_info(hdev->dev, in handle_lower_qman_data_on_err()
7891 if (undef_opcode->write_enable) { in handle_lower_qman_data_on_err()
7893 undef_opcode->timestamp = ktime_get(); in handle_lower_qman_data_on_err()
7894 undef_opcode->cq_addr = cq_ptr; in handle_lower_qman_data_on_err()
7895 undef_opcode->cq_size = cq_size; in handle_lower_qman_data_on_err()
7896 undef_opcode->engine_id = engine_id; in handle_lower_qman_data_on_err()
7897 undef_opcode->stream_id = QMAN_STREAMS; in handle_lower_qman_data_on_err()
7898 undef_opcode->write_enable = 0; in handle_lower_qman_data_on_err()
7909 glbl_sts_addr = qman_base + (mmDCORE0_TPC0_QM_GLBL_ERR_STS_0 - mmDCORE0_TPC0_QM_BASE); in gaudi2_handle_qman_err_generic()
7910 arb_err_addr = qman_base + (mmDCORE0_TPC0_QM_ARB_ERR_CAUSE - mmDCORE0_TPC0_QM_BASE); in gaudi2_handle_qman_err_generic()
7987 dev_err_ratelimited(hdev->dev, in gaudi2_razwi_rr_hbw_shared_printf_info()
7988 "%s-RAZWI SHARED RR HBW %s error, address %#llx, Initiator coordinates 0x%x\n", in gaudi2_razwi_rr_hbw_shared_printf_info()
8012 dev_err_ratelimited(hdev->dev, in gaudi2_razwi_rr_lbw_shared_printf_info()
8013 …"%s-RAZWI SHARED RR LBW %s error, mstr_if 0x%llx, captured address 0x%llX Initiator coordinates 0x… in gaudi2_razwi_rr_lbw_shared_printf_info()
8027 (GAUDI2_DCORE0_ENGINE_ID_TPC_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0)); in gaudi2_razwi_calc_engine_id()
8030 return ((GAUDI2_DCORE0_ENGINE_ID_MME - GAUDI2_DCORE0_ENGINE_ID_EDMA_0) + in gaudi2_razwi_calc_engine_id()
8052 (GAUDI2_DCORE0_ENGINE_ID_DEC_0 - GAUDI2_DCORE0_ENGINE_ID_EDMA_0)); in gaudi2_razwi_calc_engine_id()
8083 if (hdev->tpc_binning) { in gaudi2_ack_module_razwi_event_handler()
8084 binned_idx = __ffs(hdev->tpc_binning); in gaudi2_ack_module_razwi_event_handler()
8092 !hdev->asic_prop.fw_security_enabled && in gaudi2_ack_module_razwi_event_handler()
8156 if (hdev->decoder_binning) { in gaudi2_ack_module_razwi_event_handler()
8157 binned_idx = __ffs(hdev->decoder_binning); in gaudi2_ack_module_razwi_event_handler()
8187 (((s32)lbw_rtr_id - hbw_rtr_id) * DCORE_RTR_OFFSET); in gaudi2_ack_module_razwi_event_handler()
8232 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_check_if_razwi_happened()
8237 if (prop->tpc_enabled_mask & BIT(mod_idx)) in gaudi2_check_if_razwi_happened()
8249 if (prop->edma_enabled_mask & BIT(mod_idx)) in gaudi2_check_if_razwi_happened()
8258 if (hdev->nic_ports_mask & BIT(mod_idx)) in gaudi2_check_if_razwi_happened()
8264 if (prop->decoder_enabled_mask & BIT(mod_idx)) in gaudi2_check_if_razwi_happened()
8288 PSOC_RAZWI_ENG_STR_SIZE - str_size, "%s", in gaudi2_psoc_razwi_get_engines()
8292 PSOC_RAZWI_ENG_STR_SIZE - str_size, " or %s", in gaudi2_psoc_razwi_get_engines()
8329 dev_err(hdev->dev, in gaudi2_handle_psoc_razwi_happened()
8343 dev_err(hdev->dev, in gaudi2_handle_psoc_razwi_happened()
8355 dev_err(hdev->dev, in gaudi2_handle_psoc_razwi_happened()
8367 dev_err(hdev->dev, in gaudi2_handle_psoc_razwi_happened()
8393 if (hdev->pldm || !(hdev->fw_components & FW_TYPE_LINUX)) { in gaudi2_ack_psoc_razwi_event_handler()
8401 dev_err_ratelimited(hdev->dev, in gaudi2_ack_psoc_razwi_event_handler()
8412 dev_err_ratelimited(hdev->dev, in gaudi2_ack_psoc_razwi_event_handler()
8417 if (hdev->pldm || !(hdev->fw_components & FW_TYPE_LINUX)) in gaudi2_ack_psoc_razwi_event_handler()
8453 index = event_type - GAUDI2_EVENT_TPC0_AXI_ERR_RSP; in gaudi2_handle_qm_sei_err()
8467 index = (event_type - GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE) / in gaudi2_handle_qm_sei_err()
8468 (GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE - in gaudi2_handle_qm_sei_err()
8475 index = event_type - GAUDI2_EVENT_PDMA_CH0_AXI_ERR_RSP; in gaudi2_handle_qm_sei_err()
8481 index = event_type - GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE; in gaudi2_handle_qm_sei_err()
8514 index = event_type - GAUDI2_EVENT_TPC0_QM; in gaudi2_handle_qman_err()
8519 index = event_type - GAUDI2_EVENT_TPC6_QM; in gaudi2_handle_qman_err()
8524 index = event_type - GAUDI2_EVENT_TPC12_QM; in gaudi2_handle_qman_err()
8529 index = event_type - GAUDI2_EVENT_TPC18_QM; in gaudi2_handle_qman_err()
8681 u64 intr_cause_data = le64_to_cpu(razwi_with_intr_cause->intr_cause.intr_cause_data); in gaudi2_handle_rot_err()
8703 u64 intr_cause_data = le64_to_cpu(razwi_with_intr_cause->intr_cause.intr_cause_data); in gaudi2_tpc_ack_interrupts()
8735 (dec_index - NUM_OF_VDEC_PER_DCORE * NUM_OF_DCORES); in gaudi2_handle_dec_err()
8984 dev_err_ratelimited(hdev->dev, "PMMU page fault on va 0x%llx\n", addr); in gaudi2_handle_page_error()
8989 dev_err_ratelimited(hdev->dev, "HMMU page fault on va range 0x%llx - 0x%llx\n", in gaudi2_handle_page_error()
9016 dev_err_ratelimited(hdev->dev, "%s access error on va 0x%llx\n", in gaudi2_handle_access_error()
9097 dev_err_ratelimited(hdev->dev, "SM%u err. err cause: CQ_INTR. queue index: %u\n", in gaudi2_handle_sm_err()
9241 dev_err_ratelimited(hdev->dev, in gaudi2_hbm_sei_handle_read_err()
9247 addr = le32_to_cpu(rd_err_data->dbg_rd_err_addr.rd_addr_val); in gaudi2_hbm_sei_handle_read_err()
9248 dev_err_ratelimited(hdev->dev, in gaudi2_hbm_sei_handle_read_err()
9258 if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) & in gaudi2_hbm_sei_handle_read_err()
9260 dev_err_ratelimited(hdev->dev, "Beat%d ECC SERR: DM: %#x, Syndrome: %#x\n", in gaudi2_hbm_sei_handle_read_err()
9262 le32_to_cpu(rd_err_data->dbg_rd_err_dm), in gaudi2_hbm_sei_handle_read_err()
9263 le32_to_cpu(rd_err_data->dbg_rd_err_syndrome)); in gaudi2_hbm_sei_handle_read_err()
9265 if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) & in gaudi2_hbm_sei_handle_read_err()
9267 dev_err_ratelimited(hdev->dev, "Beat%d ECC DERR: DM: %#x, Syndrome: %#x\n", in gaudi2_hbm_sei_handle_read_err()
9269 le32_to_cpu(rd_err_data->dbg_rd_err_dm), in gaudi2_hbm_sei_handle_read_err()
9270 le32_to_cpu(rd_err_data->dbg_rd_err_syndrome)); in gaudi2_hbm_sei_handle_read_err()
9275 if (le32_to_cpu(rd_err_data->dbg_rd_err_misc) & in gaudi2_hbm_sei_handle_read_err()
9277 dev_err_ratelimited(hdev->dev, in gaudi2_hbm_sei_handle_read_err()
9280 le32_to_cpu(rd_err_data->dbg_rd_err_dm), in gaudi2_hbm_sei_handle_read_err()
9281 (le32_to_cpu(rd_err_data->dbg_rd_err_misc) & in gaudi2_hbm_sei_handle_read_err()
9287 dev_err_ratelimited(hdev->dev, "Beat%d DQ data:\n", beat); in gaudi2_hbm_sei_handle_read_err()
9288 dev_err_ratelimited(hdev->dev, "\t0x%08x\n", in gaudi2_hbm_sei_handle_read_err()
9289 le32_to_cpu(rd_err_data->dbg_rd_err_data[beat * 2])); in gaudi2_hbm_sei_handle_read_err()
9290 dev_err_ratelimited(hdev->dev, "\t0x%08x\n", in gaudi2_hbm_sei_handle_read_err()
9291 le32_to_cpu(rd_err_data->dbg_rd_err_data[beat * 2 + 1])); in gaudi2_hbm_sei_handle_read_err()
9300 struct hbm_sei_wr_cmd_address *wr_cmd_addr = wr_par_err_data->dbg_last_wr_cmds; in gaudi2_hbm_sei_print_wr_par_info()
9301 u32 i, curr_addr, derr = wr_par_err_data->dbg_derr; in gaudi2_hbm_sei_print_wr_par_info()
9303 dev_err_ratelimited(hdev->dev, "WRITE PARITY ERROR count: %d\n", err_cnt); in gaudi2_hbm_sei_print_wr_par_info()
9305 dev_err_ratelimited(hdev->dev, "CK-0 DERR: 0x%02x, CK-1 DERR: 0x%02x\n", in gaudi2_hbm_sei_print_wr_par_info()
9308 /* JIRA H6-3286 - the following prints may not be valid */ in gaudi2_hbm_sei_print_wr_par_info()
9309 dev_err_ratelimited(hdev->dev, "Last latched write commands addresses:\n"); in gaudi2_hbm_sei_print_wr_par_info()
9312 dev_err_ratelimited(hdev->dev, in gaudi2_hbm_sei_print_wr_par_info()
9325 __le32 *col_cmd = ca_par_err_data->dbg_col; in gaudi2_hbm_sei_print_ca_par_info()
9326 __le16 *row_cmd = ca_par_err_data->dbg_row; in gaudi2_hbm_sei_print_ca_par_info()
9329 dev_err_ratelimited(hdev->dev, "CA ERROR count: %d\n", err_cnt); in gaudi2_hbm_sei_print_ca_par_info()
9331 dev_err_ratelimited(hdev->dev, "Last latched C&R bus commands:\n"); in gaudi2_hbm_sei_print_ca_par_info()
9333 dev_err_ratelimited(hdev->dev, "cmd%u: ROW(0x%04x) COL(0x%05x)\n", i, in gaudi2_hbm_sei_print_ca_par_info()
9345 hbm_id = (event_type - GAUDI2_EVENT_HBM0_MC0_SEI_SEVERE) / 4; in gaudi2_handle_hbm_mc_sei_err()
9346 mc_id = ((event_type - GAUDI2_EVENT_HBM0_MC0_SEI_SEVERE) / 2) % 2; in gaudi2_handle_hbm_mc_sei_err()
9348 cause_idx = sei_data->hdr.sei_cause; in gaudi2_handle_hbm_mc_sei_err()
9349 if (cause_idx > GAUDI2_NUM_OF_HBM_SEI_CAUSE - 1) { in gaudi2_handle_hbm_mc_sei_err()
9356 gaudi2_print_event(hdev, event_type, !sei_data->hdr.is_critical, in gaudi2_handle_hbm_mc_sei_err()
9357 "System %s Error Interrupt - HBM(%u) MC(%u) MC_CH(%u) MC_PC(%u). Error cause: %s", in gaudi2_handle_hbm_mc_sei_err()
9358 sei_data->hdr.is_critical ? "Critical" : "Non-critical", in gaudi2_handle_hbm_mc_sei_err()
9359 hbm_id, mc_id, sei_data->hdr.mc_channel, sei_data->hdr.mc_pseudo_channel, in gaudi2_handle_hbm_mc_sei_err()
9362 /* Print error-specific info */ in gaudi2_handle_hbm_mc_sei_err()
9369 gaudi2_hbm_sei_print_ca_par_info(hdev, &sei_data->ca_parity_even_info, in gaudi2_handle_hbm_mc_sei_err()
9370 le32_to_cpu(sei_data->hdr.cnt)); in gaudi2_handle_hbm_mc_sei_err()
9375 gaudi2_hbm_sei_print_ca_par_info(hdev, &sei_data->ca_parity_odd_info, in gaudi2_handle_hbm_mc_sei_err()
9376 le32_to_cpu(sei_data->hdr.cnt)); in gaudi2_handle_hbm_mc_sei_err()
9381 gaudi2_hbm_sei_print_wr_par_info(hdev, &sei_data->wr_parity_info, in gaudi2_handle_hbm_mc_sei_err()
9382 le32_to_cpu(sei_data->hdr.cnt)); in gaudi2_handle_hbm_mc_sei_err()
9391 &sei_data->read_err_info, in gaudi2_handle_hbm_mc_sei_err()
9392 le32_to_cpu(sei_data->hdr.cnt)); in gaudi2_handle_hbm_mc_sei_err()
9399 require_hard_reset |= !!sei_data->hdr.is_critical; in gaudi2_handle_hbm_mc_sei_err()
9422 dev_dbg(hdev->dev, "HBM spi event: notification cause(%s)\n", in gaudi2_handle_hbm_mc_spi()
9434 mutex_lock(&hdev->clk_throttling.lock); in gaudi2_print_clk_change_info()
9438 hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_POWER; in gaudi2_print_clk_change_info()
9439 hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_POWER; in gaudi2_print_clk_change_info()
9440 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].start = ktime_get(); in gaudi2_print_clk_change_info()
9441 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = zero_time; in gaudi2_print_clk_change_info()
9442 dev_dbg_ratelimited(hdev->dev, "Clock throttling due to power consumption\n"); in gaudi2_print_clk_change_info()
9446 hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_POWER; in gaudi2_print_clk_change_info()
9447 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_POWER].end = ktime_get(); in gaudi2_print_clk_change_info()
9448 dev_dbg_ratelimited(hdev->dev, "Power envelop is safe, back to optimal clock\n"); in gaudi2_print_clk_change_info()
9452 hdev->clk_throttling.current_reason |= HL_CLK_THROTTLE_THERMAL; in gaudi2_print_clk_change_info()
9453 hdev->clk_throttling.aggregated_reason |= HL_CLK_THROTTLE_THERMAL; in gaudi2_print_clk_change_info()
9454 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].start = ktime_get(); in gaudi2_print_clk_change_info()
9455 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = zero_time; in gaudi2_print_clk_change_info()
9457 dev_info_ratelimited(hdev->dev, "Clock throttling due to overheating\n"); in gaudi2_print_clk_change_info()
9461 hdev->clk_throttling.current_reason &= ~HL_CLK_THROTTLE_THERMAL; in gaudi2_print_clk_change_info()
9462 hdev->clk_throttling.timestamp[HL_CLK_THROTTLE_TYPE_THERMAL].end = ktime_get(); in gaudi2_print_clk_change_info()
9464 dev_info_ratelimited(hdev->dev, "Thermal envelop is safe, back to optimal clock\n"); in gaudi2_print_clk_change_info()
9468 dev_err(hdev->dev, "Received invalid clock change event %d\n", event_type); in gaudi2_print_clk_change_info()
9472 mutex_unlock(&hdev->clk_throttling.lock); in gaudi2_print_clk_change_info()
9478 struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ]; in gaudi2_print_out_of_sync_info()
9482 le32_to_cpu(sync_err->pi), le32_to_cpu(sync_err->ci), in gaudi2_print_out_of_sync_info()
9483 q->pi, atomic_read(&q->ci)); in gaudi2_print_out_of_sync_info()
9504 "pcie msi-x gen denied due to vector num check failure, vec(0x%X)", in gaudi2_handle_pcie_p2p_msix()
9519 cause = le64_to_cpu(drain_data->intr_cause.intr_cause_data); in gaudi2_handle_pcie_drain()
9520 lbw_rd = le64_to_cpu(drain_data->drain_rd_addr_lbw); in gaudi2_handle_pcie_drain()
9521 lbw_wr = le64_to_cpu(drain_data->drain_wr_addr_lbw); in gaudi2_handle_pcie_drain()
9522 hbw_rd = le64_to_cpu(drain_data->drain_rd_addr_hbw); in gaudi2_handle_pcie_drain()
9523 hbw_wr = le64_to_cpu(drain_data->drain_wr_addr_hbw); in gaudi2_handle_pcie_drain()
9526 dev_err_ratelimited(hdev->dev, in gaudi2_handle_pcie_drain()
9533 dev_err_ratelimited(hdev->dev, in gaudi2_handle_pcie_drain()
9549 dev_err_ratelimited(hdev->dev, "PSOC %s completed\n", in gaudi2_handle_psoc_drain()
9563 struct hl_hw_queue *q = &hdev->kernel_queues[GAUDI2_QUEUE_ID_CPU_PQ]; in gaudi2_print_cpu_pkt_failure_info()
9567 le32_to_cpu(sync_err->pi), le32_to_cpu(sync_err->ci), q->pi, atomic_read(&q->ci)); in gaudi2_print_cpu_pkt_failure_info()
9577 intr_type = le32_to_cpu(data->intr_type); in hl_arc_event_handle()
9578 engine_id = le32_to_cpu(data->engine_id); in hl_arc_event_handle()
9579 payload = le64_to_cpu(data->payload); in hl_arc_event_handle()
9587 engine_id, intr_type, q->queue_index); in hl_arc_event_handle()
9602 index = event_type - GAUDI2_EVENT_TPC0_AXI_ERR_RSP; in event_id_to_engine_id()
9606 index = event_type - GAUDI2_EVENT_TPC0_QM; in event_id_to_engine_id()
9648 index = event_type - GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE; in event_id_to_engine_id()
9652 index = (event_type - GAUDI2_EVENT_DEC0_SPI) >> 1; in event_id_to_engine_id()
9656 index = event_type - GAUDI2_EVENT_NIC0_AXI_ERROR_RESPONSE; in event_id_to_engine_id()
9659 index = event_type - GAUDI2_EVENT_NIC0_QM0; in event_id_to_engine_id()
9662 index = event_type - GAUDI2_EVENT_NIC0_BMON_SPMU; in event_id_to_engine_id()
9665 index = (event_type - GAUDI2_EVENT_TPC0_BMON_SPMU) >> 1; in event_id_to_engine_id()
9718 return GAUDI2_DCORE1_ENGINE_ID_TPC_0 + index - TPC_ID_DCORE1_TPC0; in event_id_to_engine_id()
9720 return GAUDI2_DCORE2_ENGINE_ID_TPC_0 + index - TPC_ID_DCORE2_TPC0; in event_id_to_engine_id()
9722 return GAUDI2_DCORE3_ENGINE_ID_TPC_0 + index - TPC_ID_DCORE3_TPC0; in event_id_to_engine_id()
9762 hdev->eq_heartbeat_received = true; in hl_eq_heartbeat_event_handle()
9767 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_handle_eqe()
9773 ctl = le32_to_cpu(eq_entry->hdr.ctl); in gaudi2_handle_eqe()
9777 dev_err(hdev->dev, "Event type %u exceeds maximum of %u", in gaudi2_handle_eqe()
9778 event_type, GAUDI2_EVENT_SIZE - 1); in gaudi2_handle_eqe()
9782 gaudi2->events_stat[event_type]++; in gaudi2_handle_eqe()
9783 gaudi2->events_stat_aggregate[event_type]++; in gaudi2_handle_eqe()
9791 reset_required = gaudi2_handle_ecc_event(hdev, event_type, &eq_entry->ecc_data); in gaudi2_handle_eqe()
9792 is_critical = eq_entry->ecc_data.is_critical; in gaudi2_handle_eqe()
9824 index = event_type - GAUDI2_EVENT_ROTATOR0_AXI_ERROR_RESPONSE; in gaudi2_handle_eqe()
9826 &eq_entry->razwi_with_intr_cause, &event_mask); in gaudi2_handle_eqe()
9832 index = event_type - GAUDI2_EVENT_TPC0_AXI_ERR_RSP; in gaudi2_handle_eqe()
9834 &eq_entry->razwi_with_intr_cause, &event_mask); in gaudi2_handle_eqe()
9840 index = event_type - GAUDI2_EVENT_DEC0_AXI_ERR_RSPONSE; in gaudi2_handle_eqe()
9870 index = (event_type - GAUDI2_EVENT_TPC0_KERNEL_ERR) / in gaudi2_handle_eqe()
9871 (GAUDI2_EVENT_TPC1_KERNEL_ERR - GAUDI2_EVENT_TPC0_KERNEL_ERR); in gaudi2_handle_eqe()
9873 &eq_entry->razwi_with_intr_cause, &event_mask); in gaudi2_handle_eqe()
9887 index = (event_type - GAUDI2_EVENT_DEC0_SPI) / in gaudi2_handle_eqe()
9888 (GAUDI2_EVENT_DEC1_SPI - GAUDI2_EVENT_DEC0_SPI); in gaudi2_handle_eqe()
9897 index = (event_type - GAUDI2_EVENT_MME0_CTRL_AXI_ERROR_RESPONSE) / in gaudi2_handle_eqe()
9898 (GAUDI2_EVENT_MME1_CTRL_AXI_ERROR_RESPONSE - in gaudi2_handle_eqe()
9909 index = (event_type - GAUDI2_EVENT_MME0_QMAN_SW_ERROR) / in gaudi2_handle_eqe()
9910 (GAUDI2_EVENT_MME1_QMAN_SW_ERROR - in gaudi2_handle_eqe()
9920 index = (event_type - GAUDI2_EVENT_MME0_WAP_SOURCE_RESULT_INVALID) / in gaudi2_handle_eqe()
9921 (GAUDI2_EVENT_MME1_WAP_SOURCE_RESULT_INVALID - in gaudi2_handle_eqe()
9930 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); in gaudi2_handle_eqe()
9936 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); in gaudi2_handle_eqe()
9942 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); in gaudi2_handle_eqe()
9948 le64_to_cpu(eq_entry->intr_cause.intr_cause_data), &event_mask); in gaudi2_handle_eqe()
9964 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); in gaudi2_handle_eqe()
9971 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); in gaudi2_handle_eqe()
9983 if (gaudi2_handle_hbm_mc_sei_err(hdev, event_type, &eq_entry->sei_data)) { in gaudi2_handle_eqe()
9992 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); in gaudi2_handle_eqe()
9998 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); in gaudi2_handle_eqe()
10003 error_count = gaudi2_handle_pcie_drain(hdev, &eq_entry->pcie_drain_ind_data); in gaudi2_handle_eqe()
10012 le64_to_cpu(eq_entry->intr_cause.intr_cause_data)); in gaudi2_handle_eqe()
10121 gaudi2_print_out_of_sync_info(hdev, event_type, &eq_entry->pkt_sync_err); in gaudi2_handle_eqe()
10130 /* Do nothing- FW will handle it */ in gaudi2_handle_eqe()
10139 index = event_type - GAUDI2_EVENT_SM0_AXI_ERROR_RESPONSE; in gaudi2_handle_eqe()
10150 dev_info(hdev->dev, "CPLD shutdown cause, reset reason: 0x%llx\n", in gaudi2_handle_eqe()
10151 le64_to_cpu(eq_entry->data[0])); in gaudi2_handle_eqe()
10156 dev_err(hdev->dev, "CPLD shutdown event, reset reason: 0x%llx\n", in gaudi2_handle_eqe()
10157 le64_to_cpu(eq_entry->data[0])); in gaudi2_handle_eqe()
10163 gaudi2_print_cpu_pkt_failure_info(hdev, event_type, &eq_entry->pkt_sync_err); in gaudi2_handle_eqe()
10170 error_count = hl_arc_event_handle(hdev, event_type, &eq_entry->arc_data); in gaudi2_handle_eqe()
10188 dev_info_ratelimited(hdev->dev, "%s event received\n", in gaudi2_handle_eqe()
10198 dev_err_ratelimited(hdev->dev, "Cannot find handler for event %d\n", in gaudi2_handle_eqe()
10223 if (hdev->hard_reset_on_fw_events || in gaudi2_handle_eqe()
10224 (hdev->asic_prop.fw_security_enabled && is_critical)) in gaudi2_handle_eqe()
10238 if (hdev->asic_prop.fw_security_enabled && is_critical) { in gaudi2_handle_eqe()
10264 lin_dma_pkt->ctl = cpu_to_le32(ctl); in gaudi2_memset_memory_chunk_using_edma_qm()
10265 lin_dma_pkt->src_addr = cpu_to_le64(val); in gaudi2_memset_memory_chunk_using_edma_qm()
10266 lin_dma_pkt->dst_addr = cpu_to_le64(addr); in gaudi2_memset_memory_chunk_using_edma_qm()
10267 lin_dma_pkt->tsize = cpu_to_le32(size); in gaudi2_memset_memory_chunk_using_edma_qm()
10273 dev_err(hdev->dev, "Failed to send lin dma packet to H/W queue %d\n", in gaudi2_memset_memory_chunk_using_edma_qm()
10288 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_memset_device_memory()
10293 if (prop->edma_enabled_mask == 0) { in gaudi2_memset_device_memory()
10294 dev_info(hdev->dev, "non of the EDMA engines is enabled - skip dram scrubbing\n"); in gaudi2_memset_device_memory()
10295 return -EIO; in gaudi2_memset_device_memory()
10298 sob_offset = hdev->asic_prop.first_available_user_sob[0] * 4; in gaudi2_memset_device_memory()
10313 return -ENOMEM; in gaudi2_memset_device_memory()
10316 * set mmu bypass for the scrubbing - all ddmas are configured the same so save in gaudi2_memset_device_memory()
10327 if (!(prop->edma_enabled_mask & BIT(edma_bit))) in gaudi2_memset_device_memory()
10350 if (!(prop->edma_enabled_mask & BIT(edma_bit))) in gaudi2_memset_device_memory()
10353 chunk_size = min_t(u64, SZ_2G, end_addr - cur_addr); in gaudi2_memset_device_memory()
10373 dev_err(hdev->dev, "DMA Timeout during HBM scrubbing\n"); in gaudi2_memset_device_memory()
10382 if (!(prop->edma_enabled_mask & BIT(edma_bit))) in gaudi2_memset_device_memory()
10403 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_scrub_device_dram()
10404 u64 size = prop->dram_end_address - prop->dram_user_base_address; in gaudi2_scrub_device_dram()
10406 rc = gaudi2_memset_device_memory(hdev, prop->dram_user_base_address, size, val); in gaudi2_scrub_device_dram()
10409 dev_err(hdev->dev, "Failed to scrub dram, address: 0x%llx size: %llu\n", in gaudi2_scrub_device_dram()
10410 prop->dram_user_base_address, size); in gaudi2_scrub_device_dram()
10417 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_scrub_device_mem()
10418 u64 val = hdev->memory_scrub_val; in gaudi2_scrub_device_mem()
10421 if (!hdev->memory_scrub) in gaudi2_scrub_device_mem()
10425 addr = prop->sram_user_base_address; in gaudi2_scrub_device_mem()
10426 size = hdev->pldm ? 0x10000 : (prop->sram_size - SRAM_USER_BASE_OFFSET); in gaudi2_scrub_device_mem()
10427 dev_dbg(hdev->dev, "Scrubbing SRAM: 0x%09llx - 0x%09llx, val: 0x%llx\n", in gaudi2_scrub_device_mem()
10431 dev_err(hdev->dev, "scrubbing SRAM failed (%d)\n", rc); in gaudi2_scrub_device_mem()
10438 dev_err(hdev->dev, "scrubbing DRAM failed (%d)\n", rc); in gaudi2_scrub_device_mem()
10451 offset = hdev->asic_prop.first_available_cq[0] * 4; in gaudi2_restore_user_sm_registers()
10458 size = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 - in gaudi2_restore_user_sm_registers()
10475 size = mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_H_0 - mmDCORE0_SYNC_MNGR_GLBL_LBW_ADDR_L_0; in gaudi2_restore_user_sm_registers()
10493 offset = hdev->asic_prop.first_available_user_mon[0] * 4; in gaudi2_restore_user_sm_registers()
10496 size = mmDCORE0_SYNC_MNGR_OBJS_SM_SEC_0 - (mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0 + offset); in gaudi2_restore_user_sm_registers()
10506 size = mmDCORE0_SYNC_MNGR_OBJS_SM_SEC_0 - mmDCORE0_SYNC_MNGR_OBJS_MON_STATUS_0; in gaudi2_restore_user_sm_registers()
10515 offset = hdev->asic_prop.first_available_user_sob[0] * 4; in gaudi2_restore_user_sm_registers()
10518 size = mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 - in gaudi2_restore_user_sm_registers()
10525 size = mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 - mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0; in gaudi2_restore_user_sm_registers()
10586 u64 block_base = cfg_ctx->base + block_idx * cfg_ctx->block_off; in gaudi2_init_block_instances()
10590 for (i = 0 ; i < cfg_ctx->instances ; i++) { in gaudi2_init_block_instances()
10591 seq = block_idx * cfg_ctx->instances + i; in gaudi2_init_block_instances()
10594 if (!(cfg_ctx->enabled_mask & BIT_ULL(seq))) in gaudi2_init_block_instances()
10597 cfg_ctx->instance_cfg_fn(hdev, block_base + i * cfg_ctx->instance_off, in gaudi2_init_block_instances()
10598 cfg_ctx->data); in gaudi2_init_block_instances()
10607 cfg_ctx->enabled_mask = mask; in gaudi2_init_blocks_with_mask()
10609 for (i = 0 ; i < cfg_ctx->blocks ; i++) in gaudi2_init_blocks_with_mask()
10630 dev_err(hdev->dev, "No ctx available\n"); in gaudi2_debugfs_read_dma()
10631 return -EINVAL; in gaudi2_debugfs_read_dma()
10638 dev_err(hdev->dev, "Failed to allocate memory for KDMA read\n"); in gaudi2_debugfs_read_dma()
10639 rc = -ENOMEM; in gaudi2_debugfs_read_dma()
10647 dev_err(hdev->dev, "Failed to reserve vmem on asic\n"); in gaudi2_debugfs_read_dma()
10648 rc = -ENOMEM; in gaudi2_debugfs_read_dma()
10653 mutex_lock(&hdev->mmu_lock); in gaudi2_debugfs_read_dma()
10657 dev_err(hdev->dev, "Failed to create mapping on asic mmu\n"); in gaudi2_debugfs_read_dma()
10663 ctx->asid, reserved_va_base, SZ_2M); in gaudi2_debugfs_read_dma()
10669 mutex_unlock(&hdev->mmu_lock); in gaudi2_debugfs_read_dma()
10672 gaudi2_kdma_set_mmbp_asid(hdev, false, ctx->asid); in gaudi2_debugfs_read_dma()
10693 size_left -= SZ_2M; in gaudi2_debugfs_read_dma()
10698 mutex_lock(&hdev->mmu_lock); in gaudi2_debugfs_read_dma()
10705 ctx->asid, reserved_va_base, SZ_2M); in gaudi2_debugfs_read_dma()
10708 mutex_unlock(&hdev->mmu_lock); in gaudi2_debugfs_read_dma()
10720 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_internal_cb_pool_init()
10723 if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU)) in gaudi2_internal_cb_pool_init()
10726 hdev->internal_cb_pool_virt_addr = hl_asic_dma_alloc_coherent(hdev, in gaudi2_internal_cb_pool_init()
10728 &hdev->internal_cb_pool_dma_addr, in gaudi2_internal_cb_pool_init()
10731 if (!hdev->internal_cb_pool_virt_addr) in gaudi2_internal_cb_pool_init()
10732 return -ENOMEM; in gaudi2_internal_cb_pool_init()
10737 hdev->internal_cb_pool = gen_pool_create(min_alloc_order, -1); in gaudi2_internal_cb_pool_init()
10738 if (!hdev->internal_cb_pool) { in gaudi2_internal_cb_pool_init()
10739 dev_err(hdev->dev, "Failed to create internal CB pool\n"); in gaudi2_internal_cb_pool_init()
10740 rc = -ENOMEM; in gaudi2_internal_cb_pool_init()
10744 rc = gen_pool_add(hdev->internal_cb_pool, (uintptr_t) hdev->internal_cb_pool_virt_addr, in gaudi2_internal_cb_pool_init()
10745 HOST_SPACE_INTERNAL_CB_SZ, -1); in gaudi2_internal_cb_pool_init()
10747 dev_err(hdev->dev, "Failed to add memory to internal CB pool\n"); in gaudi2_internal_cb_pool_init()
10748 rc = -EFAULT; in gaudi2_internal_cb_pool_init()
10752 hdev->internal_cb_va_base = hl_reserve_va_block(hdev, ctx, HL_VA_RANGE_TYPE_HOST, in gaudi2_internal_cb_pool_init()
10755 if (!hdev->internal_cb_va_base) { in gaudi2_internal_cb_pool_init()
10756 rc = -ENOMEM; in gaudi2_internal_cb_pool_init()
10760 mutex_lock(&hdev->mmu_lock); in gaudi2_internal_cb_pool_init()
10762 rc = hl_mmu_map_contiguous(ctx, hdev->internal_cb_va_base, hdev->internal_cb_pool_dma_addr, in gaudi2_internal_cb_pool_init()
10771 mutex_unlock(&hdev->mmu_lock); in gaudi2_internal_cb_pool_init()
10776 hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); in gaudi2_internal_cb_pool_init()
10778 mutex_unlock(&hdev->mmu_lock); in gaudi2_internal_cb_pool_init()
10779 hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); in gaudi2_internal_cb_pool_init()
10781 gen_pool_destroy(hdev->internal_cb_pool); in gaudi2_internal_cb_pool_init()
10783 hl_asic_dma_free_coherent(hdev, HOST_SPACE_INTERNAL_CB_SZ, hdev->internal_cb_pool_virt_addr, in gaudi2_internal_cb_pool_init()
10784 hdev->internal_cb_pool_dma_addr); in gaudi2_internal_cb_pool_init()
10791 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_internal_cb_pool_fini()
10793 if (!(gaudi2->hw_cap_initialized & HW_CAP_PMMU)) in gaudi2_internal_cb_pool_fini()
10796 mutex_lock(&hdev->mmu_lock); in gaudi2_internal_cb_pool_fini()
10797 hl_mmu_unmap_contiguous(ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); in gaudi2_internal_cb_pool_fini()
10798 hl_unreserve_va_block(hdev, ctx, hdev->internal_cb_va_base, HOST_SPACE_INTERNAL_CB_SZ); in gaudi2_internal_cb_pool_fini()
10800 mutex_unlock(&hdev->mmu_lock); in gaudi2_internal_cb_pool_fini()
10802 gen_pool_destroy(hdev->internal_cb_pool); in gaudi2_internal_cb_pool_fini()
10804 hl_asic_dma_free_coherent(hdev, HOST_SPACE_INTERNAL_CB_SZ, hdev->internal_cb_pool_virt_addr, in gaudi2_internal_cb_pool_fini()
10805 hdev->internal_cb_pool_dma_addr); in gaudi2_internal_cb_pool_fini()
10816 struct hl_device *hdev = ctx->hdev; in gaudi2_map_virtual_msix_doorbell_memory()
10817 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_map_virtual_msix_doorbell_memory()
10818 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_map_virtual_msix_doorbell_memory()
10822 gaudi2->virt_msix_db_dma_addr, prop->pmmu.page_size, true); in gaudi2_map_virtual_msix_doorbell_memory()
10824 dev_err(hdev->dev, "Failed to map VA %#llx for virtual MSI-X doorbell memory\n", in gaudi2_map_virtual_msix_doorbell_memory()
10832 struct hl_device *hdev = ctx->hdev; in gaudi2_unmap_virtual_msix_doorbell_memory()
10833 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_unmap_virtual_msix_doorbell_memory()
10837 prop->pmmu.page_size, true); in gaudi2_unmap_virtual_msix_doorbell_memory()
10839 dev_err(hdev->dev, "Failed to unmap VA %#llx of virtual MSI-X doorbell memory\n", in gaudi2_unmap_virtual_msix_doorbell_memory()
10847 if (ctx->asid == HL_KERNEL_ASID_ID) in gaudi2_ctx_init()
10850 rc = gaudi2_mmu_prepare(ctx->hdev, ctx->asid); in gaudi2_ctx_init()
10857 if (ctx->hdev->reset_upon_device_release) in gaudi2_ctx_init()
10858 gaudi2_restore_nic_qm_registers(ctx->hdev); in gaudi2_ctx_init()
10860 gaudi2_restore_user_registers(ctx->hdev); in gaudi2_ctx_init()
10862 rc = gaudi2_internal_cb_pool_init(ctx->hdev, ctx); in gaudi2_ctx_init()
10868 gaudi2_internal_cb_pool_fini(ctx->hdev, ctx); in gaudi2_ctx_init()
10875 if (ctx->asid == HL_KERNEL_ASID_ID) in gaudi2_ctx_fini()
10878 gaudi2_internal_cb_pool_fini(ctx->hdev, ctx); in gaudi2_ctx_fini()
10885 struct hl_device *hdev = cs->ctx->hdev; in gaudi2_pre_schedule_cs()
10886 int index = cs->sequence & (hdev->asic_prop.max_pending_cs - 1); in gaudi2_pre_schedule_cs()
10897 * generates MSI-X interrupt. in gaudi2_pre_schedule_cs()
10905 cs->jobs_cnt); in gaudi2_pre_schedule_cs()
10921 pkt = (struct packet_msg_short *) (uintptr_t) (cb->kernel_address + size); in gaudi2_gen_signal_cb()
10934 pkt->value = cpu_to_le32(value); in gaudi2_gen_signal_cb()
10935 pkt->ctl = cpu_to_le32(ctl); in gaudi2_gen_signal_cb()
10952 pkt->value = cpu_to_le32(value); in gaudi2_add_mon_msg_short()
10953 pkt->ctl = cpu_to_le32(ctl); in gaudi2_add_mon_msg_short()
10965 dev_err(hdev->dev, "sob_base %u (mask %#x) is not valid\n", sob_base, sob_mask); in gaudi2_add_arm_monitor_pkt()
10982 pkt->value = cpu_to_le32(value); in gaudi2_add_arm_monitor_pkt()
10983 pkt->ctl = cpu_to_le32(ctl); in gaudi2_add_arm_monitor_pkt()
11002 pkt->cfg = cpu_to_le32(cfg); in gaudi2_add_fence_pkt()
11003 pkt->ctl = cpu_to_le32(ctl); in gaudi2_add_fence_pkt()
11010 struct hl_cb *cb = prop->data; in gaudi2_gen_wait_cb()
11011 void *buf = (void *) (uintptr_t) (cb->kernel_address); in gaudi2_gen_wait_cb()
11014 u32 stream_index, size = prop->size; in gaudi2_gen_wait_cb()
11017 stream_index = prop->q_idx % 4; in gaudi2_gen_wait_cb()
11018 fence_addr = CFG_BASE + gaudi2_qm_blocks_bases[prop->q_idx] + in gaudi2_gen_wait_cb()
11028 msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0 + prop->mon_id * 4) - in gaudi2_gen_wait_cb()
11034 msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_ADDRH_0 + prop->mon_id * 4) - in gaudi2_gen_wait_cb()
11043 msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_PAY_DATA_0 + prop->mon_id * 4) - in gaudi2_gen_wait_cb()
11049 msg_addr_offset = (mmDCORE0_SYNC_MNGR_OBJS_MON_ARM_0 + prop->mon_id * 4) - monitor_base; in gaudi2_gen_wait_cb()
11051 size += gaudi2_add_arm_monitor_pkt(hdev, buf + size, prop->sob_base, prop->sob_mask, in gaudi2_gen_wait_cb()
11052 prop->sob_val, msg_addr_offset); in gaudi2_gen_wait_cb()
11064 dev_dbg(hdev->dev, "reset SOB, q_idx: %d, sob_id: %d\n", hw_sob->q_idx, hw_sob->sob_id); in gaudi2_reset_sob()
11066 WREG32(mmDCORE0_SYNC_MNGR_OBJS_SOB_OBJ_0 + hw_sob->sob_id * 4, 0); in gaudi2_reset_sob()
11068 kref_init(&hw_sob->kref); in gaudi2_reset_sob()
11091 return -EINVAL; in gaudi2_collective_wait_create_jobs()
11095 * hl_mmu_scramble - converts a dram (non power of 2) page-size aligned address
11096 * to DMMU page-size address (64MB) before mapping it in
11108 * PA1 0x3000000 VA1 0x9C000000 SVA1= (VA1/48M)*64M 0xD0000000 <- PA1/48M 0x1
11109 * PA2 0x9000000 VA2 0x9F000000 SVA2= (VA2/48M)*64M 0xD4000000 <- PA2/48M 0x3
11114 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_mmu_scramble_addr()
11122 divisor = prop->num_functional_hbms * GAUDI2_HBM_MMU_SCRM_MEM_SIZE; in gaudi2_mmu_scramble_addr()
11134 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_mmu_descramble_addr()
11142 divisor = prop->num_functional_hbms * GAUDI2_HBM_MMU_SCRM_MEM_SIZE; in gaudi2_mmu_descramble_addr()
11158 dev_err(hdev->dev, "Unexpected core number %d for DEC\n", core_id); in gaudi2_get_dec_base_addr()
11179 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_get_hw_block_id()
11183 if (block_addr == CFG_BASE + gaudi2->mapped_blocks[i].address) { in gaudi2_get_hw_block_id()
11186 *block_size = gaudi2->mapped_blocks[i].size; in gaudi2_get_hw_block_id()
11191 dev_err(hdev->dev, "Invalid block address %#llx", block_addr); in gaudi2_get_hw_block_id()
11193 return -EINVAL; in gaudi2_get_hw_block_id()
11199 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_block_mmap()
11205 dev_err(hdev->dev, "Invalid block id %u", block_id); in gaudi2_block_mmap()
11206 return -EINVAL; in gaudi2_block_mmap()
11210 if (block_size != gaudi2->mapped_blocks[block_id].size) { in gaudi2_block_mmap()
11211 dev_err(hdev->dev, "Invalid block size %u", block_size); in gaudi2_block_mmap()
11212 return -EINVAL; in gaudi2_block_mmap()
11215 offset_in_bar = CFG_BASE + gaudi2->mapped_blocks[block_id].address - STM_FLASH_BASE_ADDR; in gaudi2_block_mmap()
11217 address = pci_resource_start(hdev->pdev, SRAM_CFG_BAR_ID) + offset_in_bar; in gaudi2_block_mmap()
11222 rc = remap_pfn_range(vma, vma->vm_start, address >> PAGE_SHIFT, in gaudi2_block_mmap()
11223 block_size, vma->vm_page_prot); in gaudi2_block_mmap()
11225 dev_err(hdev->dev, "remap_pfn_range error %d", rc); in gaudi2_block_mmap()
11232 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_enable_events_from_fw()
11234 struct cpu_dyn_regs *dyn_regs = &hdev->fw_loader.dynamic_loader.comm_desc.cpu_dyn_regs; in gaudi2_enable_events_from_fw()
11235 u32 irq_handler_offset = le32_to_cpu(dyn_regs->gic_host_ints_irq); in gaudi2_enable_events_from_fw()
11237 if (gaudi2->hw_cap_initialized & HW_CAP_CPU_Q) in gaudi2_enable_events_from_fw()
11297 return -EINVAL; in gaudi2_get_mmu_base()
11306 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_ack_mmu_error()
11309 if (!(gaudi2->hw_cap_initialized & mmu_id)) in gaudi2_ack_mmu_error()
11359 default: return -EINVAL; in gaudi2_map_pll_idx_to_fw_idx()
11402 hdev->state_dump_specs.props = gaudi2_state_dump_specs_props; in gaudi2_state_dump_init()
11403 hdev->state_dump_specs.funcs = gaudi2_state_dump_funcs; in gaudi2_state_dump_init()
11426 struct asic_fixed_properties *prop = &hdev->asic_prop; in gaudi2_mmu_get_real_page_size()
11430 if (page_size % mmu_prop->page_size) in gaudi2_mmu_get_real_page_size()
11433 *real_page_size = mmu_prop->page_size; in gaudi2_mmu_get_real_page_size()
11437 if ((page_size % prop->dram_page_size) || (prop->dram_page_size > mmu_prop->page_size)) in gaudi2_mmu_get_real_page_size()
11448 *real_page_size = prop->dram_page_size; in gaudi2_mmu_get_real_page_size()
11453 dev_err(hdev->dev, "page size of %u is not %uKB aligned, can't map\n", in gaudi2_mmu_get_real_page_size()
11454 page_size, mmu_prop->page_size >> 10); in gaudi2_mmu_get_real_page_size()
11455 return -EFAULT; in gaudi2_mmu_get_real_page_size()
11460 return -EOPNOTSUPP; in gaudi2_get_monitor_dump()
11465 struct gaudi2_device *gaudi2 = hdev->asic_specific; in gaudi2_send_device_activity()
11467 if (!(gaudi2->hw_cap_initialized & HW_CAP_CPU_Q)) in gaudi2_send_device_activity()
11575 hdev->asic_funcs = &gaudi2_funcs; in gaudi2_set_asic_funcs()