Lines Matching +full:width +full:- +full:mm
1 // SPDX-License-Identifier: GPL-2.0
37 atomic64_inc(&job->hwctx->job_free_cnt); in aie2_job_release()
38 wake_up(&job->hwctx->priv->job_free_wq); in aie2_job_release()
39 if (job->out_fence) in aie2_job_release()
40 dma_fence_put(job->out_fence); in aie2_job_release()
46 kref_put(&job->refcnt, aie2_job_release); in aie2_job_put()
53 drm_sched_stop(&hwctx->priv->sched, bad_job); in aie2_hwctx_stop()
54 aie2_destroy_context(xdna->dev_handle, hwctx); in aie2_hwctx_stop()
59 struct amdxdna_gem_obj *heap = hwctx->priv->heap; in aie2_hwctx_restart()
62 ret = aie2_create_context(xdna->dev_handle, hwctx); in aie2_hwctx_restart()
68 ret = aie2_map_host_buf(xdna->dev_handle, hwctx->fw_ctx_id, in aie2_hwctx_restart()
69 heap->mem.userptr, heap->mem.size); in aie2_hwctx_restart()
75 if (hwctx->status != HWCTX_STAT_READY) { in aie2_hwctx_restart()
76 XDNA_DBG(xdna, "hwctx is not ready, status %d", hwctx->status); in aie2_hwctx_restart()
87 drm_sched_start(&hwctx->priv->sched, 0); in aie2_hwctx_restart()
88 XDNA_DBG(xdna, "%s restarted, ret %d", hwctx->name, ret); in aie2_hwctx_restart()
94 struct amdxdna_dev *xdna = client->xdna; in aie2_restart_ctx()
98 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); in aie2_restart_ctx()
99 mutex_lock(&client->hwctx_lock); in aie2_restart_ctx()
101 if (hwctx->status != HWCTX_STAT_STOP) in aie2_restart_ctx()
104 hwctx->status = hwctx->old_status; in aie2_restart_ctx()
105 XDNA_DBG(xdna, "Resetting %s", hwctx->name); in aie2_restart_ctx()
108 mutex_unlock(&client->hwctx_lock); in aie2_restart_ctx()
116 fence = drm_syncobj_fence_get(hwctx->priv->syncobj); in aie2_cmd_get_out_fence()
135 fence = aie2_cmd_get_out_fence(hwctx, hwctx->priv->seq - 1); in aie2_hwctx_wait_for_idle()
146 struct amdxdna_dev *xdna = hwctx->client->xdna; in aie2_hwctx_suspend()
153 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); in aie2_hwctx_suspend()
156 hwctx->old_status = hwctx->status; in aie2_hwctx_suspend()
157 hwctx->status = HWCTX_STAT_STOP; in aie2_hwctx_suspend()
162 struct amdxdna_dev *xdna = hwctx->client->xdna; in aie2_hwctx_resume()
169 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); in aie2_hwctx_resume()
170 hwctx->status = hwctx->old_status; in aie2_hwctx_resume()
177 struct dma_fence *fence = job->fence; in aie2_sched_notify()
179 trace_xdna_job(&job->base, job->hwctx->name, "signaled fence", job->seq); in aie2_sched_notify()
180 job->hwctx->priv->completed++; in aie2_sched_notify()
183 up(&job->hwctx->priv->job_sem); in aie2_sched_notify()
184 job->job_done = true; in aie2_sched_notify()
186 mmput_async(job->mm); in aie2_sched_notify()
198 cmd_abo = job->cmd_bo; in aie2_sched_resp_handler()
205 ret = -EINVAL; in aie2_sched_resp_handler()
210 XDNA_DBG(job->hwctx->client->xdna, "Resp status 0x%x", status); in aie2_sched_resp_handler()
232 ret = -EINVAL; in aie2_sched_nocmd_resp_handler()
237 XDNA_DBG(job->hwctx->client->xdna, "Resp status 0x%x", status); in aie2_sched_nocmd_resp_handler()
255 cmd_abo = job->cmd_bo; in aie2_sched_cmdlist_resp_handler()
258 ret = -EINVAL; in aie2_sched_cmdlist_resp_handler()
263 xdna = job->hwctx->client->xdna; in aie2_sched_cmdlist_resp_handler()
278 ret = -EINVAL; in aie2_sched_cmdlist_resp_handler()
286 cc->error_index = fail_cmd_idx; in aie2_sched_cmdlist_resp_handler()
287 if (cc->error_index >= cc->command_count) in aie2_sched_cmdlist_resp_handler()
288 cc->error_index = 0; in aie2_sched_cmdlist_resp_handler()
299 struct amdxdna_gem_obj *cmd_abo = job->cmd_bo; in aie2_sched_job_run()
300 struct amdxdna_hwctx *hwctx = job->hwctx; in aie2_sched_job_run()
304 if (!mmget_not_zero(job->mm)) in aie2_sched_job_run()
305 return ERR_PTR(-ESRCH); in aie2_sched_job_run()
307 kref_get(&job->refcnt); in aie2_sched_job_run()
308 fence = dma_fence_get(job->fence); in aie2_sched_job_run()
326 dma_fence_put(job->fence); in aie2_sched_job_run()
328 mmput(job->mm); in aie2_sched_job_run()
331 trace_xdna_job(sched_job, hwctx->name, "sent to device", job->seq); in aie2_sched_job_run()
339 struct amdxdna_hwctx *hwctx = job->hwctx; in aie2_sched_job_free()
341 trace_xdna_job(sched_job, hwctx->name, "job free", job->seq); in aie2_sched_job_free()
342 if (!job->job_done) in aie2_sched_job_free()
343 up(&hwctx->priv->job_sem); in aie2_sched_job_free()
353 struct amdxdna_hwctx *hwctx = job->hwctx; in aie2_sched_job_timedout()
356 xdna = hwctx->client->xdna; in aie2_sched_job_timedout()
357 trace_xdna_job(sched_job, hwctx->name, "job timedout", job->seq); in aie2_sched_job_timedout()
358 mutex_lock(&xdna->dev_lock); in aie2_sched_job_timedout()
362 mutex_unlock(&xdna->dev_lock); in aie2_sched_job_timedout()
375 struct amdxdna_dev *xdna = hwctx->client->xdna; in aie2_hwctx_col_list()
378 u32 width = 1, entries = 0; in aie2_hwctx_col_list() local
381 if (!hwctx->num_tiles) { in aie2_hwctx_col_list()
383 return -EINVAL; in aie2_hwctx_col_list()
386 ndev = xdna->dev_handle; in aie2_hwctx_col_list()
387 if (unlikely(!ndev->metadata.core.row_count)) { in aie2_hwctx_col_list()
389 return -EINVAL; in aie2_hwctx_col_list()
392 hwctx->num_col = hwctx->num_tiles / ndev->metadata.core.row_count; in aie2_hwctx_col_list()
393 if (!hwctx->num_col || hwctx->num_col > ndev->total_col) { in aie2_hwctx_col_list()
394 XDNA_ERR(xdna, "Invalid num_col %d", hwctx->num_col); in aie2_hwctx_col_list()
395 return -EINVAL; in aie2_hwctx_col_list()
398 if (ndev->priv->col_align == COL_ALIGN_NATURE) in aie2_hwctx_col_list()
399 width = hwctx->num_col; in aie2_hwctx_col_list()
402 * In range [start, end], find out columns that is multiple of width. in aie2_hwctx_col_list()
407 start = xdna->dev_info->first_col; in aie2_hwctx_col_list()
408 end = ndev->total_col - hwctx->num_col; in aie2_hwctx_col_list()
413 first = start + (width - start % width) % width; in aie2_hwctx_col_list()
414 last = end - end % width; in aie2_hwctx_col_list()
416 entries = (last - first) / width + 1; in aie2_hwctx_col_list()
421 XDNA_ERR(xdna, "Start %d end %d width %d", in aie2_hwctx_col_list()
422 start, end, width); in aie2_hwctx_col_list()
423 return -EINVAL; in aie2_hwctx_col_list()
426 hwctx->col_list = kmalloc_array(entries, sizeof(*hwctx->col_list), GFP_KERNEL); in aie2_hwctx_col_list()
427 if (!hwctx->col_list) in aie2_hwctx_col_list()
428 return -ENOMEM; in aie2_hwctx_col_list()
430 hwctx->col_list_len = entries; in aie2_hwctx_col_list()
431 hwctx->col_list[0] = first; in aie2_hwctx_col_list()
433 hwctx->col_list[i] = hwctx->col_list[i - 1] + width; in aie2_hwctx_col_list()
435 print_hex_dump_debug("col_list: ", DUMP_PREFIX_OFFSET, 16, 4, hwctx->col_list, in aie2_hwctx_col_list()
436 entries * sizeof(*hwctx->col_list), false); in aie2_hwctx_col_list()
442 struct amdxdna_dev *xdna = hwctx->client->xdna; in aie2_alloc_resource()
448 return -ENOMEM; in aie2_alloc_resource()
450 xrs_req->cdo.start_cols = hwctx->col_list; in aie2_alloc_resource()
451 xrs_req->cdo.cols_len = hwctx->col_list_len; in aie2_alloc_resource()
452 xrs_req->cdo.ncols = hwctx->num_col; in aie2_alloc_resource()
453 xrs_req->cdo.qos_cap.opc = hwctx->max_opc; in aie2_alloc_resource()
455 xrs_req->rqos.gops = hwctx->qos.gops; in aie2_alloc_resource()
456 xrs_req->rqos.fps = hwctx->qos.fps; in aie2_alloc_resource()
457 xrs_req->rqos.dma_bw = hwctx->qos.dma_bandwidth; in aie2_alloc_resource()
458 xrs_req->rqos.latency = hwctx->qos.latency; in aie2_alloc_resource()
459 xrs_req->rqos.exec_time = hwctx->qos.frame_exec_time; in aie2_alloc_resource()
460 xrs_req->rqos.priority = hwctx->qos.priority; in aie2_alloc_resource()
462 xrs_req->rid = (uintptr_t)hwctx; in aie2_alloc_resource()
464 ret = xrs_allocate_resource(xdna->xrs_hdl, xrs_req, hwctx); in aie2_alloc_resource()
474 struct amdxdna_dev *xdna = hwctx->client->xdna; in aie2_release_resource()
477 ret = xrs_release_resource(xdna->xrs_hdl, (uintptr_t)hwctx); in aie2_release_resource()
484 struct amdxdna_dev *xdna = hwctx->client->xdna; in aie2_ctx_syncobj_create()
485 struct drm_file *filp = hwctx->client->filp; in aie2_ctx_syncobj_create()
490 hwctx->syncobj_hdl = AMDXDNA_INVALID_FENCE_HANDLE; in aie2_ctx_syncobj_create()
503 hwctx->priv->syncobj = syncobj; in aie2_ctx_syncobj_create()
504 hwctx->syncobj_hdl = hdl; in aie2_ctx_syncobj_create()
515 drm_syncobj_put(hwctx->priv->syncobj); in aie2_ctx_syncobj_destroy()
520 struct amdxdna_client *client = hwctx->client; in aie2_hwctx_init()
521 struct amdxdna_dev *xdna = client->xdna; in aie2_hwctx_init()
527 .name = hwctx->name, in aie2_hwctx_init()
528 .dev = xdna->ddev.dev, in aie2_hwctx_init()
536 priv = kzalloc(sizeof(*hwctx->priv), GFP_KERNEL); in aie2_hwctx_init()
538 return -ENOMEM; in aie2_hwctx_init()
539 hwctx->priv = priv; in aie2_hwctx_init()
541 mutex_lock(&client->mm_lock); in aie2_hwctx_init()
542 heap = client->dev_heap; in aie2_hwctx_init()
545 mutex_unlock(&client->mm_lock); in aie2_hwctx_init()
546 ret = -ENOENT; in aie2_hwctx_init()
550 mutex_unlock(&client->mm_lock); in aie2_hwctx_init()
551 priv->heap = heap; in aie2_hwctx_init()
552 sema_init(&priv->job_sem, HWCTX_MAX_CMDS); in aie2_hwctx_init()
560 for (i = 0; i < ARRAY_SIZE(priv->cmd_buf); i++) { in aie2_hwctx_init()
569 abo = amdxdna_drm_alloc_dev_bo(&xdna->ddev, &args, client->filp, true); in aie2_hwctx_init()
576 i, abo->mem.dev_addr, abo->mem.size); in aie2_hwctx_init()
577 priv->cmd_buf[i] = abo; in aie2_hwctx_init()
580 sched = &priv->sched; in aie2_hwctx_init()
581 mutex_init(&priv->io_lock); in aie2_hwctx_init()
584 might_lock(&priv->io_lock); in aie2_hwctx_init()
593 ret = drm_sched_entity_init(&priv->entity, DRM_SCHED_PRIORITY_NORMAL, in aie2_hwctx_init()
612 ret = aie2_map_host_buf(xdna->dev_handle, hwctx->fw_ctx_id, in aie2_hwctx_init()
613 heap->mem.userptr, heap->mem.size); in aie2_hwctx_init()
625 hwctx->status = HWCTX_STAT_INIT; in aie2_hwctx_init()
626 ndev = xdna->dev_handle; in aie2_hwctx_init()
627 ndev->hwctx_num++; in aie2_hwctx_init()
628 init_waitqueue_head(&priv->job_free_wq); in aie2_hwctx_init()
630 XDNA_DBG(xdna, "hwctx %s init completed", hwctx->name); in aie2_hwctx_init()
637 kfree(hwctx->col_list); in aie2_hwctx_init()
639 drm_sched_entity_destroy(&priv->entity); in aie2_hwctx_init()
641 drm_sched_fini(&priv->sched); in aie2_hwctx_init()
643 for (i = 0; i < ARRAY_SIZE(priv->cmd_buf); i++) { in aie2_hwctx_init()
644 if (!priv->cmd_buf[i]) in aie2_hwctx_init()
646 drm_gem_object_put(to_gobj(priv->cmd_buf[i])); in aie2_hwctx_init()
662 xdna = hwctx->client->xdna; in aie2_hwctx_fini()
663 ndev = xdna->dev_handle; in aie2_hwctx_fini()
664 ndev->hwctx_num--; in aie2_hwctx_fini()
666 XDNA_DBG(xdna, "%s sequence number %lld", hwctx->name, hwctx->priv->seq); in aie2_hwctx_fini()
667 drm_sched_entity_destroy(&hwctx->priv->entity); in aie2_hwctx_fini()
675 wait_event(hwctx->priv->job_free_wq, in aie2_hwctx_fini()
676 atomic64_read(&hwctx->job_submit_cnt) == in aie2_hwctx_fini()
677 atomic64_read(&hwctx->job_free_cnt)); in aie2_hwctx_fini()
679 drm_sched_fini(&hwctx->priv->sched); in aie2_hwctx_fini()
682 for (idx = 0; idx < ARRAY_SIZE(hwctx->priv->cmd_buf); idx++) in aie2_hwctx_fini()
683 drm_gem_object_put(to_gobj(hwctx->priv->cmd_buf[idx])); in aie2_hwctx_fini()
684 amdxdna_gem_unpin(hwctx->priv->heap); in aie2_hwctx_fini()
685 drm_gem_object_put(to_gobj(hwctx->priv->heap)); in aie2_hwctx_fini()
687 mutex_destroy(&hwctx->priv->io_lock); in aie2_hwctx_fini()
688 kfree(hwctx->col_list); in aie2_hwctx_fini()
689 kfree(hwctx->priv); in aie2_hwctx_fini()
690 kfree(hwctx->cus); in aie2_hwctx_fini()
696 struct amdxdna_dev *xdna = hwctx->client->xdna; in aie2_hwctx_cu_config()
700 XDNA_DBG(xdna, "Config %d CU to %s", config->num_cus, hwctx->name); in aie2_hwctx_cu_config()
701 if (XDNA_MBZ_DBG(xdna, config->pad, sizeof(config->pad))) in aie2_hwctx_cu_config()
702 return -EINVAL; in aie2_hwctx_cu_config()
704 if (hwctx->status != HWCTX_STAT_INIT) { in aie2_hwctx_cu_config()
705 XDNA_ERR(xdna, "Not support re-config CU"); in aie2_hwctx_cu_config()
706 return -EINVAL; in aie2_hwctx_cu_config()
709 if (!config->num_cus) { in aie2_hwctx_cu_config()
711 return -EINVAL; in aie2_hwctx_cu_config()
714 total_size = struct_size(config, cu_configs, config->num_cus); in aie2_hwctx_cu_config()
717 return -EINVAL; in aie2_hwctx_cu_config()
720 hwctx->cus = kmemdup(config, total_size, GFP_KERNEL); in aie2_hwctx_cu_config()
721 if (!hwctx->cus) in aie2_hwctx_cu_config()
722 return -ENOMEM; in aie2_hwctx_cu_config()
731 hwctx->status = HWCTX_STAT_READY; in aie2_hwctx_cu_config()
736 kfree(hwctx->cus); in aie2_hwctx_cu_config()
737 hwctx->cus = NULL; in aie2_hwctx_cu_config()
743 struct amdxdna_dev *xdna = hwctx->client->xdna; in aie2_hwctx_config()
745 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock)); in aie2_hwctx_config()
751 return -EOPNOTSUPP; in aie2_hwctx_config()
754 return -EOPNOTSUPP; in aie2_hwctx_config()
760 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); in aie2_populate_range()
761 struct mm_struct *mm = abo->mem.notifier.mm; in aie2_populate_range() local
767 abo->mem.userptr, abo->mem.size); in aie2_populate_range()
768 range.notifier = &abo->mem.notifier; in aie2_populate_range()
769 range.start = abo->mem.userptr; in aie2_populate_range()
770 range.end = abo->mem.userptr + abo->mem.size; in aie2_populate_range()
771 range.hmm_pfns = abo->mem.pfns; in aie2_populate_range()
774 if (!mmget_not_zero(mm)) in aie2_populate_range()
775 return -EFAULT; in aie2_populate_range()
779 range.notifier_seq = mmu_interval_read_begin(&abo->mem.notifier); in aie2_populate_range()
780 mmap_read_lock(mm); in aie2_populate_range()
782 mmap_read_unlock(mm); in aie2_populate_range()
785 ret = -ETIME; in aie2_populate_range()
789 if (ret == -EBUSY) in aie2_populate_range()
795 down_read(&xdna->notifier_lock); in aie2_populate_range()
796 if (mmu_interval_read_retry(&abo->mem.notifier, range.notifier_seq)) { in aie2_populate_range()
797 up_read(&xdna->notifier_lock); in aie2_populate_range()
800 abo->mem.map_invalid = false; in aie2_populate_range()
801 up_read(&xdna->notifier_lock); in aie2_populate_range()
804 mmput(mm); in aie2_populate_range()
810 struct amdxdna_dev *xdna = hwctx->client->xdna; in aie2_cmd_submit()
817 ret = down_interruptible(&hwctx->priv->job_sem); in aie2_cmd_submit()
826 ret = -ENOMEM; in aie2_cmd_submit()
830 ret = drm_sched_job_init(&job->base, &hwctx->priv->entity, 1, hwctx); in aie2_cmd_submit()
837 ret = drm_gem_lock_reservations(job->bos, job->bo_cnt, &acquire_ctx); in aie2_cmd_submit()
843 for (i = 0; i < job->bo_cnt; i++) { in aie2_cmd_submit()
844 ret = dma_resv_reserve_fences(job->bos[i]->resv, 1); in aie2_cmd_submit()
847 drm_gem_unlock_reservations(job->bos, job->bo_cnt, &acquire_ctx); in aie2_cmd_submit()
852 down_read(&xdna->notifier_lock); in aie2_cmd_submit()
853 for (i = 0; i < job->bo_cnt; i++) { in aie2_cmd_submit()
854 abo = to_xdna_obj(job->bos[i]); in aie2_cmd_submit()
855 if (abo->mem.map_invalid) { in aie2_cmd_submit()
856 up_read(&xdna->notifier_lock); in aie2_cmd_submit()
857 drm_gem_unlock_reservations(job->bos, job->bo_cnt, &acquire_ctx); in aie2_cmd_submit()
862 ret = -ETIME; in aie2_cmd_submit()
873 mutex_lock(&hwctx->priv->io_lock); in aie2_cmd_submit()
874 drm_sched_job_arm(&job->base); in aie2_cmd_submit()
875 job->out_fence = dma_fence_get(&job->base.s_fence->finished); in aie2_cmd_submit()
876 for (i = 0; i < job->bo_cnt; i++) in aie2_cmd_submit()
877 dma_resv_add_fence(job->bos[i]->resv, job->out_fence, DMA_RESV_USAGE_WRITE); in aie2_cmd_submit()
878 job->seq = hwctx->priv->seq++; in aie2_cmd_submit()
879 kref_get(&job->refcnt); in aie2_cmd_submit()
880 drm_sched_entity_push_job(&job->base); in aie2_cmd_submit()
882 *seq = job->seq; in aie2_cmd_submit()
883 drm_syncobj_add_point(hwctx->priv->syncobj, chain, job->out_fence, *seq); in aie2_cmd_submit()
884 mutex_unlock(&hwctx->priv->io_lock); in aie2_cmd_submit()
886 up_read(&xdna->notifier_lock); in aie2_cmd_submit()
887 drm_gem_unlock_reservations(job->bos, job->bo_cnt, &acquire_ctx); in aie2_cmd_submit()
890 atomic64_inc(&hwctx->job_submit_cnt); in aie2_cmd_submit()
895 drm_sched_job_cleanup(&job->base); in aie2_cmd_submit()
899 up(&hwctx->priv->job_sem); in aie2_cmd_submit()
900 job->job_done = true; in aie2_cmd_submit()
907 struct amdxdna_dev *xdna = to_xdna_dev(to_gobj(abo)->dev); in aie2_hmm_invalidate()
911 down_write(&xdna->notifier_lock); in aie2_hmm_invalidate()
912 abo->mem.map_invalid = true; in aie2_hmm_invalidate()
913 mmu_interval_set_seq(&abo->mem.notifier, cur_seq); in aie2_hmm_invalidate()
914 up_write(&xdna->notifier_lock); in aie2_hmm_invalidate()
915 ret = dma_resv_wait_timeout(gobj->resv, DMA_RESV_USAGE_BOOKKEEP, in aie2_hmm_invalidate()
917 if (!ret || ret == -ERESTARTSYS) in aie2_hmm_invalidate()