Lines Matching +full:ctx +full:- +full:asid
1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2016-2022 HabanaLabs, Ltd.
15 * hl_mmu_get_funcs() - get MMU functions structure
25 return &hdev->mmu_func[pgt_residency]; in hl_mmu_get_funcs()
30 struct asic_fixed_properties *prop = &hdev->asic_prop; in hl_is_dram_va()
32 return hl_mem_area_inside_range(virt_addr, prop->dmmu.page_size, in hl_is_dram_va()
33 prop->dmmu.start_addr, in hl_is_dram_va()
34 prop->dmmu.end_addr); in hl_is_dram_va()
38 * hl_mmu_init() - initialize the MMU module.
41 * Return: 0 for success, non-zero for failure.
45 int rc = -EOPNOTSUPP; in hl_mmu_init()
47 if (hdev->mmu_disable) in hl_mmu_init()
50 mutex_init(&hdev->mmu_lock); in hl_mmu_init()
52 if (hdev->mmu_func[MMU_DR_PGT].init != NULL) { in hl_mmu_init()
53 rc = hdev->mmu_func[MMU_DR_PGT].init(hdev); in hl_mmu_init()
58 if (hdev->mmu_func[MMU_HR_PGT].init != NULL) { in hl_mmu_init()
59 rc = hdev->mmu_func[MMU_HR_PGT].init(hdev); in hl_mmu_init()
67 if (hdev->mmu_func[MMU_DR_PGT].fini != NULL) in hl_mmu_init()
68 hdev->mmu_func[MMU_DR_PGT].fini(hdev); in hl_mmu_init()
74 * hl_mmu_fini() - release the MMU module.
78 * - Disable MMU in H/W.
79 * - Free the pgt_infos pool.
85 if (hdev->mmu_disable) in hl_mmu_fini()
88 if (hdev->mmu_func[MMU_DR_PGT].fini != NULL) in hl_mmu_fini()
89 hdev->mmu_func[MMU_DR_PGT].fini(hdev); in hl_mmu_fini()
91 if (hdev->mmu_func[MMU_HR_PGT].fini != NULL) in hl_mmu_fini()
92 hdev->mmu_func[MMU_HR_PGT].fini(hdev); in hl_mmu_fini()
94 mutex_destroy(&hdev->mmu_lock); in hl_mmu_fini()
98 * hl_mmu_ctx_init() - initialize a context for using the MMU module.
99 * @ctx: pointer to the context structure to initialize.
103 * Return: 0 on success, non-zero otherwise.
105 int hl_mmu_ctx_init(struct hl_ctx *ctx) in hl_mmu_ctx_init() argument
107 struct hl_device *hdev = ctx->hdev; in hl_mmu_ctx_init()
108 int rc = -EOPNOTSUPP; in hl_mmu_ctx_init()
110 if (hdev->mmu_disable) in hl_mmu_ctx_init()
113 if (hdev->mmu_func[MMU_DR_PGT].ctx_init != NULL) { in hl_mmu_ctx_init()
114 rc = hdev->mmu_func[MMU_DR_PGT].ctx_init(ctx); in hl_mmu_ctx_init()
119 if (hdev->mmu_func[MMU_HR_PGT].ctx_init != NULL) { in hl_mmu_ctx_init()
120 rc = hdev->mmu_func[MMU_HR_PGT].ctx_init(ctx); in hl_mmu_ctx_init()
128 if (hdev->mmu_func[MMU_DR_PGT].fini != NULL) in hl_mmu_ctx_init()
129 hdev->mmu_func[MMU_DR_PGT].fini(hdev); in hl_mmu_ctx_init()
135 * hl_mmu_ctx_fini - disable a ctx from using the mmu module
137 * @ctx: pointer to the context structure
140 * - Free any pgts which were not freed yet
141 * - Free the mutex
142 * - Free DRAM default page mapping hops
144 void hl_mmu_ctx_fini(struct hl_ctx *ctx) in hl_mmu_ctx_fini() argument
146 struct hl_device *hdev = ctx->hdev; in hl_mmu_ctx_fini()
148 if (hdev->mmu_disable) in hl_mmu_ctx_fini()
151 if (hdev->mmu_func[MMU_DR_PGT].ctx_fini != NULL) in hl_mmu_ctx_fini()
152 hdev->mmu_func[MMU_DR_PGT].ctx_fini(ctx); in hl_mmu_ctx_fini()
154 if (hdev->mmu_func[MMU_HR_PGT].ctx_fini != NULL) in hl_mmu_ctx_fini()
155 hdev->mmu_func[MMU_HR_PGT].ctx_fini(ctx); in hl_mmu_ctx_fini()
159 * hl_mmu_get_real_page_size - get real page size to use in map/unmap operation
171 * 1. it shall not be called directly- only from mmu_func structure instance
179 * size is bigger, we break it to sub-pages and map them separately. in hl_mmu_get_real_page_size()
181 if ((page_size % mmu_prop->page_size) == 0) { in hl_mmu_get_real_page_size()
182 *real_page_size = mmu_prop->page_size; in hl_mmu_get_real_page_size()
186 dev_err(hdev->dev, "page size of %u is not %uKB aligned, can't map\n", in hl_mmu_get_real_page_size()
187 page_size, mmu_prop->page_size >> 10); in hl_mmu_get_real_page_size()
189 return -EFAULT; in hl_mmu_get_real_page_size()
195 struct asic_fixed_properties *prop = &hdev->asic_prop; in hl_mmu_get_prop()
198 return &prop->dmmu; in hl_mmu_get_prop()
199 else if ((page_size % prop->pmmu_huge.page_size) == 0) in hl_mmu_get_prop()
200 return &prop->pmmu_huge; in hl_mmu_get_prop()
202 return &prop->pmmu; in hl_mmu_get_prop()
206 * hl_mmu_unmap_page - unmaps a virtual addr
208 * @ctx: pointer to the context structure
214 * - Check that the virt addr is mapped
215 * - Unmap the virt addr and frees pgts if possible
216 * - Returns 0 on success, -EINVAL if the given addr is not mapped
226 int hl_mmu_unmap_page(struct hl_ctx *ctx, u64 virt_addr, u32 page_size, bool flush_pte) in hl_mmu_unmap_page() argument
228 struct hl_device *hdev = ctx->hdev; in hl_mmu_unmap_page()
236 if (hdev->mmu_disable) in hl_mmu_unmap_page()
242 pgt_residency = mmu_prop->host_resident ? MMU_HR_PGT : MMU_DR_PGT; in hl_mmu_unmap_page()
245 rc = hdev->asic_funcs->mmu_get_real_page_size(hdev, mmu_prop, page_size, &real_page_size, in hl_mmu_unmap_page()
254 rc = mmu_funcs->unmap(ctx, real_virt_addr, is_dram_addr); in hl_mmu_unmap_page()
262 mmu_funcs->flush(ctx); in hl_mmu_unmap_page()
265 trace_habanalabs_mmu_unmap(hdev->dev, virt_addr, 0, page_size, flush_pte); in hl_mmu_unmap_page()
271 * hl_mmu_map_page - maps a virtual addr to physical addr
273 * @ctx: pointer to the context structure
280 * - Check that the virt addr is not mapped
281 * - Allocate pgts as necessary in order to map the virt addr to the phys
282 * - Returns 0 on success, -EINVAL if addr is already mapped, or -ENOMEM.
292 int hl_mmu_map_page(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr, u32 page_size, in hl_mmu_map_page() argument
296 struct hl_device *hdev = ctx->hdev; in hl_mmu_map_page()
304 if (hdev->mmu_disable) in hl_mmu_map_page()
310 pgt_residency = mmu_prop->host_resident ? MMU_HR_PGT : MMU_DR_PGT; in hl_mmu_map_page()
313 rc = hdev->asic_funcs->mmu_get_real_page_size(hdev, mmu_prop, page_size, &real_page_size, in hl_mmu_map_page()
324 ((hdev->asic_funcs->scramble_addr(hdev, phys_addr) & in hl_mmu_map_page()
325 (mmu_prop->page_size - 1)) || in hl_mmu_map_page()
326 (hdev->asic_funcs->scramble_addr(hdev, virt_addr) & in hl_mmu_map_page()
327 (mmu_prop->page_size - 1)))) || in hl_mmu_map_page()
328 (!is_dram_addr && ((phys_addr & (real_page_size - 1)) || in hl_mmu_map_page()
329 (virt_addr & (real_page_size - 1))))) in hl_mmu_map_page()
330 dev_crit(hdev->dev, in hl_mmu_map_page()
339 rc = mmu_funcs->map(ctx, real_virt_addr, real_phys_addr, real_page_size, in hl_mmu_map_page()
350 mmu_funcs->flush(ctx); in hl_mmu_map_page()
352 trace_habanalabs_mmu_map(hdev->dev, virt_addr, phys_addr, page_size, flush_pte); in hl_mmu_map_page()
359 if (mmu_funcs->unmap(ctx, real_virt_addr, is_dram_addr)) in hl_mmu_map_page()
360 dev_warn_ratelimited(hdev->dev, in hl_mmu_map_page()
366 mmu_funcs->flush(ctx); in hl_mmu_map_page()
372 * hl_mmu_map_contiguous - implements a wrapper for hl_mmu_map_page
375 * @ctx: pointer to the context structure
381 int hl_mmu_map_contiguous(struct hl_ctx *ctx, u64 virt_addr, in hl_mmu_map_contiguous() argument
384 struct hl_device *hdev = ctx->hdev; in hl_mmu_map_contiguous()
385 struct asic_fixed_properties *prop = &hdev->asic_prop; in hl_mmu_map_contiguous()
392 prop->dmmu.start_addr, prop->dmmu.end_addr)) in hl_mmu_map_contiguous()
393 page_size = prop->dmmu.page_size; in hl_mmu_map_contiguous()
395 prop->pmmu.start_addr, prop->pmmu.end_addr)) in hl_mmu_map_contiguous()
396 page_size = prop->pmmu.page_size; in hl_mmu_map_contiguous()
398 prop->pmmu_huge.start_addr, prop->pmmu_huge.end_addr)) in hl_mmu_map_contiguous()
399 page_size = prop->pmmu_huge.page_size; in hl_mmu_map_contiguous()
401 return -EINVAL; in hl_mmu_map_contiguous()
407 rc = hl_mmu_map_page(ctx, curr_va, curr_pa, page_size, in hl_mmu_map_contiguous()
410 dev_err(hdev->dev, in hl_mmu_map_contiguous()
413 /* last mapping failed so don't try to unmap it - reduce off by page_size */ in hl_mmu_map_contiguous()
414 off -= page_size; in hl_mmu_map_contiguous()
422 for (; off >= 0 ; off -= page_size) { in hl_mmu_map_contiguous()
424 flush_pte = (off - (s32) page_size) < 0; in hl_mmu_map_contiguous()
425 if (hl_mmu_unmap_page(ctx, curr_va, page_size, flush_pte)) in hl_mmu_map_contiguous()
426 dev_warn_ratelimited(hdev->dev, in hl_mmu_map_contiguous()
434 * hl_mmu_unmap_contiguous - implements a wrapper for hl_mmu_unmap_page
437 * @ctx: pointer to the context structure
442 int hl_mmu_unmap_contiguous(struct hl_ctx *ctx, u64 virt_addr, u32 size) in hl_mmu_unmap_contiguous() argument
444 struct hl_device *hdev = ctx->hdev; in hl_mmu_unmap_contiguous()
445 struct asic_fixed_properties *prop = &hdev->asic_prop; in hl_mmu_unmap_contiguous()
452 prop->dmmu.start_addr, prop->dmmu.end_addr)) in hl_mmu_unmap_contiguous()
453 page_size = prop->dmmu.page_size; in hl_mmu_unmap_contiguous()
455 prop->pmmu.start_addr, prop->pmmu.end_addr)) in hl_mmu_unmap_contiguous()
456 page_size = prop->pmmu.page_size; in hl_mmu_unmap_contiguous()
458 prop->pmmu_huge.start_addr, prop->pmmu_huge.end_addr)) in hl_mmu_unmap_contiguous()
459 page_size = prop->pmmu_huge.page_size; in hl_mmu_unmap_contiguous()
461 return -EINVAL; in hl_mmu_unmap_contiguous()
466 rc = hl_mmu_unmap_page(ctx, curr_va, page_size, flush_pte); in hl_mmu_unmap_contiguous()
468 dev_warn_ratelimited(hdev->dev, in hl_mmu_unmap_contiguous()
475 static void hl_mmu_pa_page_with_offset(struct hl_ctx *ctx, u64 virt_addr, in hl_mmu_pa_page_with_offset() argument
479 struct asic_fixed_properties *prop = &ctx->hdev->asic_prop; in hl_mmu_pa_page_with_offset()
484 if (hops->unscrambled_paddr) in hl_mmu_pa_page_with_offset()
485 tmp_phys_addr = hops->unscrambled_paddr; in hl_mmu_pa_page_with_offset()
487 tmp_phys_addr = hops->hop_info[hops->used_hops - 1].hop_pte_val; in hl_mmu_pa_page_with_offset()
489 if (hops->range_type == HL_VA_RANGE_TYPE_HOST_HUGE) in hl_mmu_pa_page_with_offset()
490 mmu_prop = &prop->pmmu_huge; in hl_mmu_pa_page_with_offset()
491 else if (hops->range_type == HL_VA_RANGE_TYPE_HOST) in hl_mmu_pa_page_with_offset()
492 mmu_prop = &prop->pmmu; in hl_mmu_pa_page_with_offset()
494 mmu_prop = &prop->dmmu; in hl_mmu_pa_page_with_offset()
496 if ((hops->range_type == HL_VA_RANGE_TYPE_DRAM) && in hl_mmu_pa_page_with_offset()
497 !is_power_of_2(prop->dram_page_size)) { in hl_mmu_pa_page_with_offset()
508 dram_page_size = prop->dram_page_size; in hl_mmu_pa_page_with_offset()
509 dram_base = prop->dram_base_address; in hl_mmu_pa_page_with_offset()
510 abs_phys_addr = tmp_phys_addr - dram_base; in hl_mmu_pa_page_with_offset()
511 abs_virt_addr = virt_addr - dram_base; in hl_mmu_pa_page_with_offset()
523 hop_shift = mmu_prop->hop_shifts[hops->used_hops - 1]; in hl_mmu_pa_page_with_offset()
524 offset_mask = (1ull << hop_shift) - 1; in hl_mmu_pa_page_with_offset()
531 int hl_mmu_va_to_pa(struct hl_ctx *ctx, u64 virt_addr, u64 *phys_addr) in hl_mmu_va_to_pa() argument
538 rc = hl_mmu_get_tlb_info(ctx, virt_addr, &hops); in hl_mmu_va_to_pa()
542 hl_mmu_pa_page_with_offset(ctx, virt_addr, &hops, phys_addr); in hl_mmu_va_to_pa()
547 int hl_mmu_get_tlb_info(struct hl_ctx *ctx, u64 virt_addr, in hl_mmu_get_tlb_info() argument
550 struct hl_device *hdev = ctx->hdev; in hl_mmu_get_tlb_info()
557 if (hdev->mmu_disable) in hl_mmu_get_tlb_info()
558 return -EOPNOTSUPP; in hl_mmu_get_tlb_info()
560 prop = &hdev->asic_prop; in hl_mmu_get_tlb_info()
561 hops->scrambled_vaddr = virt_addr; /* assume no scrambling */ in hl_mmu_get_tlb_info()
563 is_dram_addr = hl_mem_area_inside_range(virt_addr, prop->dmmu.page_size, in hl_mmu_get_tlb_info()
564 prop->dmmu.start_addr, in hl_mmu_get_tlb_info()
565 prop->dmmu.end_addr); in hl_mmu_get_tlb_info()
567 /* host-residency is the same in PMMU and PMMU huge, no need to distinguish here */ in hl_mmu_get_tlb_info()
568 mmu_prop = is_dram_addr ? &prop->dmmu : &prop->pmmu; in hl_mmu_get_tlb_info()
569 pgt_residency = mmu_prop->host_resident ? MMU_HR_PGT : MMU_DR_PGT; in hl_mmu_get_tlb_info()
572 mutex_lock(&hdev->mmu_lock); in hl_mmu_get_tlb_info()
573 rc = mmu_funcs->get_tlb_info(ctx, virt_addr, hops); in hl_mmu_get_tlb_info()
574 mutex_unlock(&hdev->mmu_lock); in hl_mmu_get_tlb_info()
580 if (hops->unscrambled_paddr) in hl_mmu_get_tlb_info()
581 hl_mmu_pa_page_with_offset(ctx, virt_addr, hops, &hops->unscrambled_paddr); in hl_mmu_get_tlb_info()
588 if (hdev->mmu_disable) in hl_mmu_if_set_funcs()
591 switch (hdev->asic_type) { in hl_mmu_if_set_funcs()
595 hl_mmu_v1_set_funcs(hdev, &hdev->mmu_func[MMU_DR_PGT]); in hl_mmu_if_set_funcs()
601 hl_mmu_v2_hr_set_funcs(hdev, &hdev->mmu_func[MMU_HR_PGT]); in hl_mmu_if_set_funcs()
604 dev_err(hdev->dev, "Unrecognized ASIC type %d\n", in hl_mmu_if_set_funcs()
605 hdev->asic_type); in hl_mmu_if_set_funcs()
606 return -EOPNOTSUPP; in hl_mmu_if_set_funcs()
613 * hl_mmu_scramble_addr() - The generic mmu address scrambling routine.
625 * hl_mmu_descramble_addr() - The generic mmu address descrambling
630 * Return: The un-scrambled address.
641 rc = hdev->asic_funcs->mmu_invalidate_cache(hdev, is_hard, flags); in hl_mmu_invalidate_cache()
643 dev_err_ratelimited(hdev->dev, in hl_mmu_invalidate_cache()
651 u32 flags, u32 asid, u64 va, u64 size) in hl_mmu_invalidate_cache_range() argument
655 rc = hdev->asic_funcs->mmu_invalidate_cache_range(hdev, is_hard, flags, in hl_mmu_invalidate_cache_range()
656 asid, va, size); in hl_mmu_invalidate_cache_range()
658 dev_err_ratelimited(hdev->dev, in hl_mmu_invalidate_cache_range()
668 struct hl_ctx *ctx = pfw->ctx; in hl_mmu_prefetch_work_function() local
669 struct hl_device *hdev = ctx->hdev; in hl_mmu_prefetch_work_function()
674 mutex_lock(&hdev->mmu_lock); in hl_mmu_prefetch_work_function()
676 hdev->asic_funcs->mmu_prefetch_cache_range(ctx, pfw->flags, pfw->asid, pfw->va, pfw->size); in hl_mmu_prefetch_work_function()
678 mutex_unlock(&hdev->mmu_lock); in hl_mmu_prefetch_work_function()
682 * context was taken in the common mmu prefetch function- see comment there about in hl_mmu_prefetch_work_function()
685 hl_ctx_put(ctx); in hl_mmu_prefetch_work_function()
689 int hl_mmu_prefetch_cache_range(struct hl_ctx *ctx, u32 flags, u32 asid, u64 va, u64 size) in hl_mmu_prefetch_cache_range() argument
695 return -ENOMEM; in hl_mmu_prefetch_cache_range()
697 INIT_WORK(&handle_prefetch_work->prefetch_work, hl_mmu_prefetch_work_function); in hl_mmu_prefetch_cache_range()
698 handle_prefetch_work->ctx = ctx; in hl_mmu_prefetch_cache_range()
699 handle_prefetch_work->va = va; in hl_mmu_prefetch_cache_range()
700 handle_prefetch_work->size = size; in hl_mmu_prefetch_cache_range()
701 handle_prefetch_work->flags = flags; in hl_mmu_prefetch_cache_range()
702 handle_prefetch_work->asid = asid; in hl_mmu_prefetch_cache_range()
708 hl_ctx_get(ctx); in hl_mmu_prefetch_cache_range()
709 queue_work(ctx->hdev->prefetch_wq, &handle_prefetch_work->prefetch_work); in hl_mmu_prefetch_cache_range()
714 u64 hl_mmu_get_next_hop_addr(struct hl_ctx *ctx, u64 curr_pte) in hl_mmu_get_next_hop_addr() argument
720 * hl_mmu_get_hop_pte_phys_addr() - extract PTE address from HOP
721 * @ctx: pointer to the context structure to initialize.
729 u64 hl_mmu_get_hop_pte_phys_addr(struct hl_ctx *ctx, struct hl_mmu_properties *mmu_prop, in hl_mmu_get_hop_pte_phys_addr() argument
734 if (hop_idx >= mmu_prop->num_hops) { in hl_mmu_get_hop_pte_phys_addr()
735 dev_err_ratelimited(ctx->hdev->dev, "Invalid hop index %d\n", hop_idx); in hl_mmu_get_hop_pte_phys_addr()
739 shift = mmu_prop->hop_shifts[hop_idx]; in hl_mmu_get_hop_pte_phys_addr()
740 mask = mmu_prop->hop_masks[hop_idx]; in hl_mmu_get_hop_pte_phys_addr()
742 return hop_addr + ctx->hdev->asic_prop.mmu_pte_size * ((virt_addr & mask) >> shift); in hl_mmu_get_hop_pte_phys_addr()
751 hl_asic_dma_free_coherent(hdev, (chunk->end_addr - chunk->start_addr) + 1, in mmu_dma_mem_free_from_chunk()
752 (void *)chunk->start_addr, chunk->phys_addr); in mmu_dma_mem_free_from_chunk()
755 void hl_mmu_hr_flush(struct hl_ctx *ctx) in hl_mmu_hr_flush() argument
762 * hl_mmu_hr_pool_destroy() - destroy genpool
768 * - free entries allocated for shadow HOP0
769 * - free pool chunks
770 * - free pool
775 struct asic_fixed_properties *prop = &hdev->asic_prop; in hl_mmu_hr_pool_destroy()
776 struct gen_pool **pool = &hr_priv->mmu_pgt_pool; in hl_mmu_hr_pool_destroy()
778 int asid; in hl_mmu_hr_pool_destroy() local
784 if (hr_priv->mmu_asid_hop0) { in hl_mmu_hr_pool_destroy()
785 for (asid = 0 ; asid < prop->max_asid ; asid++) { in hl_mmu_hr_pool_destroy()
786 hop0_pgt = &hr_priv->mmu_asid_hop0[asid]; in hl_mmu_hr_pool_destroy()
787 if (ZERO_OR_NULL_PTR(hop0_pgt->virt_addr)) in hl_mmu_hr_pool_destroy()
790 gen_pool_free(*pool, (uintptr_t) hop0_pgt->virt_addr, hop_table_size); in hl_mmu_hr_pool_destroy()
805 * hl_mmu_hr_init() - initialize the MMU module.
811 * @return 0 on success otherwise non-zero error code
814 * - Create a pool of pages for pgt_infos.
815 * - Create a shadow table for pgt
820 struct asic_fixed_properties *prop = &hdev->asic_prop; in hl_mmu_hr_init()
831 * allocations with higher-order alignment restrictions in hl_mmu_hr_init()
833 hr_priv->mmu_pgt_pool = gen_pool_create(PAGE_SHIFT, -1); in hl_mmu_hr_init()
834 if (ZERO_OR_NULL_PTR(hr_priv->mmu_pgt_pool)) { in hl_mmu_hr_init()
835 dev_err(hdev->dev, "Failed to create hr page pool\n"); in hl_mmu_hr_init()
836 return -ENOMEM; in hl_mmu_hr_init()
839 hr_priv->mmu_asid_hop0 = kvcalloc(prop->max_asid, sizeof(struct pgt_info), GFP_KERNEL); in hl_mmu_hr_init()
840 if (ZERO_OR_NULL_PTR(hr_priv->mmu_asid_hop0)) { in hl_mmu_hr_init()
841 dev_err(hdev->dev, "Failed to allocate hr-mmu hop0 table\n"); in hl_mmu_hr_init()
842 rc = -ENOMEM; in hl_mmu_hr_init()
851 dev_err(hdev->dev, in hl_mmu_hr_init()
852 "Failed to allocate memory for host-resident page pool\n"); in hl_mmu_hr_init()
853 rc = -ENOMEM; in hl_mmu_hr_init()
857 rc = gen_pool_add_virt(hr_priv->mmu_pgt_pool, virt_addr, (phys_addr_t) dma_addr, in hl_mmu_hr_init()
858 pool_chunk_size, -1); in hl_mmu_hr_init()
860 dev_err(hdev->dev, "Failed to fill host-resident page pool\n"); in hl_mmu_hr_init()
865 for (i = 0 ; i < prop->max_asid ; i++) { in hl_mmu_hr_init()
866 hop0_pgt = &hr_priv->mmu_asid_hop0[i]; in hl_mmu_hr_init()
867 hop0_pgt->virt_addr = (uintptr_t) in hl_mmu_hr_init()
868 gen_pool_dma_zalloc_align(hr_priv->mmu_pgt_pool, in hl_mmu_hr_init()
870 (dma_addr_t *) &hop0_pgt->phys_addr, in hl_mmu_hr_init()
872 if (!hop0_pgt->virt_addr) { in hl_mmu_hr_init()
873 dev_err(hdev->dev, "Failed to allocate HOP from pgt pool\n"); in hl_mmu_hr_init()
874 rc = -ENOMEM; in hl_mmu_hr_init()
885 if (!ZERO_OR_NULL_PTR(hr_priv->mmu_asid_hop0)) in hl_mmu_hr_init()
886 kvfree(hr_priv->mmu_asid_hop0); in hl_mmu_hr_init()
892 * hl_mmu_hr_fini() - release the MMU module.
898 * - Disable MMU in H/W.
899 * - Free the pgt_infos pool.
909 if (!ZERO_OR_NULL_PTR(hr_priv->mmu_asid_hop0)) { in hl_mmu_hr_fini()
910 kvfree(hr_priv->mmu_asid_hop0); in hl_mmu_hr_fini()
916 hr_priv->mmu_asid_hop0 = NULL; in hl_mmu_hr_fini()
921 * hl_mmu_hr_free_hop_remove_pgt() - free HOP and remove PGT from hash
929 gen_pool_free(hr_priv->mmu_pgt_pool, pgt_info->virt_addr, hop_table_size); in hl_mmu_hr_free_hop_remove_pgt()
930 hash_del(&pgt_info->node); in hl_mmu_hr_free_hop_remove_pgt()
935 * hl_mmu_hr_pte_phys_to_virt() - translate PTE phys addr to virt addr
936 * @ctx: pointer to the context structure
946 u64 hl_mmu_hr_pte_phys_to_virt(struct hl_ctx *ctx, struct pgt_info *pgt, in hl_mmu_hr_pte_phys_to_virt() argument
949 u64 page_mask = (hop_table_size - 1); in hl_mmu_hr_pte_phys_to_virt()
952 return pgt->virt_addr + pte_offset; in hl_mmu_hr_pte_phys_to_virt()
956 * hl_mmu_hr_write_pte() - write HR PTE
957 * @ctx: pointer to the context structure
963 void hl_mmu_hr_write_pte(struct hl_ctx *ctx, struct pgt_info *pgt_info, u64 phys_pte_addr, in hl_mmu_hr_write_pte() argument
970 u64 virt_addr = hl_mmu_hr_pte_phys_to_virt(ctx, pgt_info, phys_pte_addr, hop_table_size); in hl_mmu_hr_write_pte()
976 * hl_mmu_hr_clear_pte() - clear HR PTE
977 * @ctx: pointer to the context structure
982 void hl_mmu_hr_clear_pte(struct hl_ctx *ctx, struct pgt_info *pgt_info, u64 phys_pte_addr, in hl_mmu_hr_clear_pte() argument
986 hl_mmu_hr_write_pte(ctx, pgt_info, phys_pte_addr, 0, hop_table_size); in hl_mmu_hr_clear_pte()
990 * hl_mmu_hr_put_pte() - put HR PTE and remove it if necessary (no more PTEs)
991 * @ctx: pointer to the context structure
998 int hl_mmu_hr_put_pte(struct hl_ctx *ctx, struct pgt_info *pgt_info, in hl_mmu_hr_put_pte() argument
1004 pgt_info->num_of_ptes--; in hl_mmu_hr_put_pte()
1010 num_of_ptes_left = pgt_info->num_of_ptes; in hl_mmu_hr_put_pte()
1018 * hl_mmu_hr_get_pte() - increase PGT PTE count
1019 * @ctx: pointer to the context structure
1023 void hl_mmu_hr_get_pte(struct hl_ctx *ctx, struct hl_hr_mmu_funcs *hr_func, u64 phys_hop_addr) in hl_mmu_hr_get_pte() argument
1025 hr_func->get_pgt_info(ctx, phys_hop_addr)->num_of_ptes++; in hl_mmu_hr_get_pte()
1029 * hl_mmu_hr_get_next_hop_pgt_info() - get pgt_info structure for the next HOP
1030 * @ctx: pointer to the context structure.
1036 struct pgt_info *hl_mmu_hr_get_next_hop_pgt_info(struct hl_ctx *ctx, in hl_mmu_hr_get_next_hop_pgt_info() argument
1040 u64 next_hop_phys_addr = hl_mmu_get_next_hop_addr(ctx, curr_pte); in hl_mmu_hr_get_next_hop_pgt_info()
1045 return hr_func->get_pgt_info(ctx, next_hop_phys_addr); in hl_mmu_hr_get_next_hop_pgt_info()
1049 * hl_mmu_hr_alloc_hop() - allocate HOP
1050 * @ctx: pointer to the context structure.
1057 struct pgt_info *hl_mmu_hr_alloc_hop(struct hl_ctx *ctx, struct hl_mmu_hr_priv *hr_priv, in hl_mmu_hr_alloc_hop() argument
1061 struct hl_device *hdev = ctx->hdev; in hl_mmu_hr_alloc_hop()
1072 virt_addr = gen_pool_dma_zalloc_align(hr_priv->mmu_pgt_pool, in hl_mmu_hr_alloc_hop()
1073 mmu_prop->hop_table_size, in hl_mmu_hr_alloc_hop()
1075 mmu_prop->hop_table_size); in hl_mmu_hr_alloc_hop()
1079 /* No memory in pool - get some and try again */ in hl_mmu_hr_alloc_hop()
1085 if (gen_pool_add_virt(hr_priv->mmu_pgt_pool, (unsigned long)virt_addr, in hl_mmu_hr_alloc_hop()
1086 phys_addr, SZ_2M, -1)) { in hl_mmu_hr_alloc_hop()
1094 dev_err(hdev->dev, "failed to allocate page\n"); in hl_mmu_hr_alloc_hop()
1098 pgt_info->phys_addr = phys_addr; in hl_mmu_hr_alloc_hop()
1099 pgt_info->shadow_addr = (unsigned long) NULL; in hl_mmu_hr_alloc_hop()
1100 pgt_info->virt_addr = (unsigned long)virt_addr; in hl_mmu_hr_alloc_hop()
1101 pgt_info->ctx = ctx; in hl_mmu_hr_alloc_hop()
1102 pgt_info->num_of_ptes = 0; in hl_mmu_hr_alloc_hop()
1103 hr_func->add_pgt_info(ctx, pgt_info, phys_addr); in hl_mmu_hr_alloc_hop()
1114 * hl_mmu_hr_get_alloc_next_hop() - get the next HOP, allocate it if it does not exist
1115 * @ctx: pointer to the context structure.
1124 struct pgt_info *hl_mmu_hr_get_alloc_next_hop(struct hl_ctx *ctx, in hl_mmu_hr_get_alloc_next_hop() argument
1130 u64 hop_addr = hl_mmu_get_next_hop_addr(ctx, curr_pte); in hl_mmu_hr_get_alloc_next_hop()
1133 return hr_func->get_pgt_info(ctx, hop_addr); in hl_mmu_hr_get_alloc_next_hop()
1136 return hl_mmu_hr_alloc_hop(ctx, hr_priv, hr_func, mmu_prop); in hl_mmu_hr_get_alloc_next_hop()
1140 * hl_mmu_hr_get_tlb_info() - get the TLB info (info for a specific mapping)
1141 * @ctx: pointer to the context structure.
1148 int hl_mmu_hr_get_tlb_info(struct hl_ctx *ctx, u64 virt_addr, struct hl_mmu_hop_info *hops, in hl_mmu_hr_get_tlb_info() argument
1153 struct hl_device *hdev = ctx->hdev; in hl_mmu_hr_get_tlb_info()
1158 rc = hr_func->get_tlb_mapping_params(hdev, &mmu_prop, hops, virt_addr, &is_huge); in hl_mmu_hr_get_tlb_info()
1162 used_hops = mmu_prop->num_hops; in hl_mmu_hr_get_tlb_info()
1166 used_hops--; in hl_mmu_hr_get_tlb_info()
1168 hops->scrambled_vaddr = hdev->asic_funcs->scramble_addr(hdev, virt_addr); in hl_mmu_hr_get_tlb_info()
1172 hops_pgt_info[i] = hr_func->get_hop0_pgt_info(ctx); in hl_mmu_hr_get_tlb_info()
1174 hops_pgt_info[i] = hl_mmu_hr_get_next_hop_pgt_info(ctx, hr_func, in hl_mmu_hr_get_tlb_info()
1175 hops->hop_info[i - 1].hop_pte_val); in hl_mmu_hr_get_tlb_info()
1178 return -EFAULT; in hl_mmu_hr_get_tlb_info()
1180 hops->hop_info[i].hop_addr = hops_pgt_info[i]->phys_addr; in hl_mmu_hr_get_tlb_info()
1181 hops->hop_info[i].hop_pte_addr = in hl_mmu_hr_get_tlb_info()
1182 hl_mmu_get_hop_pte_phys_addr(ctx, mmu_prop, i, in hl_mmu_hr_get_tlb_info()
1183 hops->hop_info[i].hop_addr, in hl_mmu_hr_get_tlb_info()
1184 hops->scrambled_vaddr); in hl_mmu_hr_get_tlb_info()
1185 hops->hop_info[i].hop_pte_val = *(u64 *) (uintptr_t) in hl_mmu_hr_get_tlb_info()
1186 hl_mmu_hr_pte_phys_to_virt(ctx, hops_pgt_info[i], in hl_mmu_hr_get_tlb_info()
1187 hops->hop_info[i].hop_pte_addr, in hl_mmu_hr_get_tlb_info()
1188 mmu_prop->hop_table_size); in hl_mmu_hr_get_tlb_info()
1190 if (!(hops->hop_info[i].hop_pte_val & PAGE_PRESENT_MASK)) in hl_mmu_hr_get_tlb_info()
1191 return -EFAULT; in hl_mmu_hr_get_tlb_info()
1193 if (hops->hop_info[i].hop_pte_val & mmu_prop->last_mask) in hl_mmu_hr_get_tlb_info()
1198 if (i == mmu_prop->num_hops) in hl_mmu_hr_get_tlb_info()
1199 return -EFAULT; in hl_mmu_hr_get_tlb_info()
1201 if (hops->scrambled_vaddr != virt_addr) in hl_mmu_hr_get_tlb_info()
1202 hops->unscrambled_paddr = hdev->asic_funcs->descramble_addr in hl_mmu_hr_get_tlb_info()
1203 (hdev, hops->hop_info[i].hop_pte_val); in hl_mmu_hr_get_tlb_info()
1205 hops->unscrambled_paddr = hops->hop_info[i].hop_pte_val; in hl_mmu_hr_get_tlb_info()
1207 hops->used_hops = i + 1; in hl_mmu_hr_get_tlb_info()