xref: /linux/drivers/accel/ivpu/ivpu_gem.c (revision 6315d93541f8a5f77c5ef5c4f25233e66d189603) 
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2020-2023 Intel Corporation
4  */
5 
6 #include <linux/dma-buf.h>
7 #include <linux/highmem.h>
8 #include <linux/module.h>
9 #include <linux/set_memory.h>
10 #include <linux/xarray.h>
11 
12 #include <drm/drm_cache.h>
13 #include <drm/drm_debugfs.h>
14 #include <drm/drm_file.h>
15 #include <drm/drm_utils.h>
16 
17 #include "ivpu_drv.h"
18 #include "ivpu_gem.h"
19 #include "ivpu_hw.h"
20 #include "ivpu_mmu.h"
21 #include "ivpu_mmu_context.h"
22 
23 MODULE_IMPORT_NS("DMA_BUF");
24 
25 static const struct drm_gem_object_funcs ivpu_gem_funcs;
26 
27 static inline void ivpu_dbg_bo(struct ivpu_device *vdev, struct ivpu_bo *bo, const char *action)
28 {
29 	ivpu_dbg(vdev, BO,
30 		 "%6s: bo %8p vpu_addr %9llx size %8zu ctx %d has_pages %d dma_mapped %d mmu_mapped %d wc %d imported %d\n",
31 		 action, bo, bo->vpu_addr, ivpu_bo_size(bo), bo->ctx ? bo->ctx->id : 0,
32 		 (bool)bo->base.pages, (bool)bo->base.sgt, bo->mmu_mapped, bo->base.map_wc,
33 		 (bool)bo->base.base.import_attach);
34 }
35 
36 /*
37  * ivpu_bo_pin() - pin the backing physical pages and map them to VPU.
38  *
39  * This function pins physical memory pages, then maps the physical pages
40  * to IOMMU address space and finally updates the VPU MMU page tables
41  * to allow the VPU to translate VPU address to IOMMU address.
42  */
43 int __must_check ivpu_bo_pin(struct ivpu_bo *bo)
44 {
45 	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
46 	int ret = 0;
47 
48 	mutex_lock(&bo->lock);
49 
50 	ivpu_dbg_bo(vdev, bo, "pin");
51 	drm_WARN_ON(&vdev->drm, !bo->ctx);
52 
53 	if (!bo->mmu_mapped) {
54 		struct sg_table *sgt = drm_gem_shmem_get_pages_sgt(&bo->base);
55 
56 		if (IS_ERR(sgt)) {
57 			ret = PTR_ERR(sgt);
58 			ivpu_err(vdev, "Failed to map BO in IOMMU: %d\n", ret);
59 			goto unlock;
60 		}
61 
62 		ret = ivpu_mmu_context_map_sgt(vdev, bo->ctx, bo->vpu_addr, sgt,
63 					       ivpu_bo_is_snooped(bo));
64 		if (ret) {
65 			ivpu_err(vdev, "Failed to map BO in MMU: %d\n", ret);
66 			goto unlock;
67 		}
68 		bo->mmu_mapped = true;
69 	}
70 
71 unlock:
72 	mutex_unlock(&bo->lock);
73 
74 	return ret;
75 }
76 
77 static int
78 ivpu_bo_alloc_vpu_addr(struct ivpu_bo *bo, struct ivpu_mmu_context *ctx,
79 		       const struct ivpu_addr_range *range)
80 {
81 	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
82 	int idx, ret;
83 
84 	if (!drm_dev_enter(&vdev->drm, &idx))
85 		return -ENODEV;
86 
87 	mutex_lock(&bo->lock);
88 
89 	ret = ivpu_mmu_context_insert_node(ctx, range, ivpu_bo_size(bo), &bo->mm_node);
90 	if (!ret) {
91 		bo->ctx = ctx;
92 		bo->vpu_addr = bo->mm_node.start;
93 	} else {
94 		ivpu_err(vdev, "Failed to add BO to context %u: %d\n", ctx->id, ret);
95 	}
96 
97 	ivpu_dbg_bo(vdev, bo, "alloc");
98 
99 	mutex_unlock(&bo->lock);
100 
101 	drm_dev_exit(idx);
102 
103 	return ret;
104 }
105 
106 static void ivpu_bo_unbind_locked(struct ivpu_bo *bo)
107 {
108 	struct ivpu_device *vdev = ivpu_bo_to_vdev(bo);
109 
110 	lockdep_assert(lockdep_is_held(&bo->lock) || !kref_read(&bo->base.base.refcount));
111 
112 	if (bo->mmu_mapped) {
113 		drm_WARN_ON(&vdev->drm, !bo->ctx);
114 		drm_WARN_ON(&vdev->drm, !bo->vpu_addr);
115 		drm_WARN_ON(&vdev->drm, !bo->base.sgt);
116 		ivpu_mmu_context_unmap_sgt(vdev, bo->ctx, bo->vpu_addr, bo->base.sgt);
117 		bo->mmu_mapped = false;
118 	}
119 
120 	if (bo->ctx) {
121 		ivpu_mmu_context_remove_node(bo->ctx, &bo->mm_node);
122 		bo->ctx = NULL;
123 	}
124 
125 	if (bo->base.base.import_attach)
126 		return;
127 
128 	dma_resv_lock(bo->base.base.resv, NULL);
129 	if (bo->base.sgt) {
130 		dma_unmap_sgtable(vdev->drm.dev, bo->base.sgt, DMA_BIDIRECTIONAL, 0);
131 		sg_free_table(bo->base.sgt);
132 		kfree(bo->base.sgt);
133 		bo->base.sgt = NULL;
134 	}
135 	dma_resv_unlock(bo->base.base.resv);
136 }
137 
138 void ivpu_bo_unbind_all_bos_from_context(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx)
139 {
140 	struct ivpu_bo *bo;
141 
142 	if (drm_WARN_ON(&vdev->drm, !ctx))
143 		return;
144 
145 	mutex_lock(&vdev->bo_list_lock);
146 	list_for_each_entry(bo, &vdev->bo_list, bo_list_node) {
147 		mutex_lock(&bo->lock);
148 		if (bo->ctx == ctx) {
149 			ivpu_dbg_bo(vdev, bo, "unbind");
150 			ivpu_bo_unbind_locked(bo);
151 		}
152 		mutex_unlock(&bo->lock);
153 	}
154 	mutex_unlock(&vdev->bo_list_lock);
155 }
156 
157 struct drm_gem_object *ivpu_gem_create_object(struct drm_device *dev, size_t size)
158 {
159 	struct ivpu_bo *bo;
160 
161 	if (size == 0 || !PAGE_ALIGNED(size))
162 		return ERR_PTR(-EINVAL);
163 
164 	bo = kzalloc(sizeof(*bo), GFP_KERNEL);
165 	if (!bo)
166 		return ERR_PTR(-ENOMEM);
167 
168 	bo->base.base.funcs = &ivpu_gem_funcs;
169 	bo->base.pages_mark_dirty_on_put = true; /* VPU can dirty a BO anytime */
170 
171 	INIT_LIST_HEAD(&bo->bo_list_node);
172 	mutex_init(&bo->lock);
173 
174 	return &bo->base.base;
175 }
176 
177 struct drm_gem_object *ivpu_gem_prime_import(struct drm_device *dev,
178 					     struct dma_buf *dma_buf)
179 {
180 	struct device *attach_dev = dev->dev;
181 	struct dma_buf_attachment *attach;
182 	struct sg_table *sgt;
183 	struct drm_gem_object *obj;
184 	int ret;
185 
186 	attach = dma_buf_attach(dma_buf, attach_dev);
187 	if (IS_ERR(attach))
188 		return ERR_CAST(attach);
189 
190 	get_dma_buf(dma_buf);
191 
192 	sgt = dma_buf_map_attachment_unlocked(attach, DMA_BIDIRECTIONAL);
193 	if (IS_ERR(sgt)) {
194 		ret = PTR_ERR(sgt);
195 		goto fail_detach;
196 	}
197 
198 	obj = drm_gem_shmem_prime_import_sg_table(dev, attach, sgt);
199 	if (IS_ERR(obj)) {
200 		ret = PTR_ERR(obj);
201 		goto fail_unmap;
202 	}
203 
204 	obj->import_attach = attach;
205 	obj->resv = dma_buf->resv;
206 
207 	return obj;
208 
209 fail_unmap:
210 	dma_buf_unmap_attachment_unlocked(attach, sgt, DMA_BIDIRECTIONAL);
211 fail_detach:
212 	dma_buf_detach(dma_buf, attach);
213 	dma_buf_put(dma_buf);
214 
215 	return ERR_PTR(ret);
216 }
217 
218 static struct ivpu_bo *ivpu_bo_alloc(struct ivpu_device *vdev, u64 size, u32 flags)
219 {
220 	struct drm_gem_shmem_object *shmem;
221 	struct ivpu_bo *bo;
222 
223 	switch (flags & DRM_IVPU_BO_CACHE_MASK) {
224 	case DRM_IVPU_BO_CACHED:
225 	case DRM_IVPU_BO_WC:
226 		break;
227 	default:
228 		return ERR_PTR(-EINVAL);
229 	}
230 
231 	shmem = drm_gem_shmem_create(&vdev->drm, size);
232 	if (IS_ERR(shmem))
233 		return ERR_CAST(shmem);
234 
235 	bo = to_ivpu_bo(&shmem->base);
236 	bo->base.map_wc = flags & DRM_IVPU_BO_WC;
237 	bo->flags = flags;
238 
239 	mutex_lock(&vdev->bo_list_lock);
240 	list_add_tail(&bo->bo_list_node, &vdev->bo_list);
241 	mutex_unlock(&vdev->bo_list_lock);
242 
243 	return bo;
244 }
245 
246 static int ivpu_gem_bo_open(struct drm_gem_object *obj, struct drm_file *file)
247 {
248 	struct ivpu_file_priv *file_priv = file->driver_priv;
249 	struct ivpu_device *vdev = file_priv->vdev;
250 	struct ivpu_bo *bo = to_ivpu_bo(obj);
251 	struct ivpu_addr_range *range;
252 
253 	if (bo->ctx) {
254 		ivpu_warn(vdev, "Can't add BO to ctx %u: already in ctx %u\n",
255 			  file_priv->ctx.id, bo->ctx->id);
256 		return -EALREADY;
257 	}
258 
259 	if (bo->flags & DRM_IVPU_BO_SHAVE_MEM)
260 		range = &vdev->hw->ranges.shave;
261 	else if (bo->flags & DRM_IVPU_BO_DMA_MEM)
262 		range = &vdev->hw->ranges.dma;
263 	else
264 		range = &vdev->hw->ranges.user;
265 
266 	return ivpu_bo_alloc_vpu_addr(bo, &file_priv->ctx, range);
267 }
268 
269 static void ivpu_gem_bo_free(struct drm_gem_object *obj)
270 {
271 	struct ivpu_device *vdev = to_ivpu_device(obj->dev);
272 	struct ivpu_bo *bo = to_ivpu_bo(obj);
273 
274 	ivpu_dbg_bo(vdev, bo, "free");
275 
276 	mutex_lock(&vdev->bo_list_lock);
277 	list_del(&bo->bo_list_node);
278 	mutex_unlock(&vdev->bo_list_lock);
279 
280 	drm_WARN_ON(&vdev->drm, !dma_resv_test_signaled(obj->resv, DMA_RESV_USAGE_READ));
281 
282 	ivpu_bo_unbind_locked(bo);
283 	mutex_destroy(&bo->lock);
284 
285 	drm_WARN_ON(obj->dev, bo->base.pages_use_count > 1);
286 	drm_gem_shmem_free(&bo->base);
287 }
288 
289 static const struct drm_gem_object_funcs ivpu_gem_funcs = {
290 	.free = ivpu_gem_bo_free,
291 	.open = ivpu_gem_bo_open,
292 	.print_info = drm_gem_shmem_object_print_info,
293 	.pin = drm_gem_shmem_object_pin,
294 	.unpin = drm_gem_shmem_object_unpin,
295 	.get_sg_table = drm_gem_shmem_object_get_sg_table,
296 	.vmap = drm_gem_shmem_object_vmap,
297 	.vunmap = drm_gem_shmem_object_vunmap,
298 	.mmap = drm_gem_shmem_object_mmap,
299 	.vm_ops = &drm_gem_shmem_vm_ops,
300 };
301 
302 int ivpu_bo_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
303 {
304 	struct ivpu_file_priv *file_priv = file->driver_priv;
305 	struct ivpu_device *vdev = file_priv->vdev;
306 	struct drm_ivpu_bo_create *args = data;
307 	u64 size = PAGE_ALIGN(args->size);
308 	struct ivpu_bo *bo;
309 	int ret;
310 
311 	if (args->flags & ~DRM_IVPU_BO_FLAGS)
312 		return -EINVAL;
313 
314 	if (size == 0)
315 		return -EINVAL;
316 
317 	bo = ivpu_bo_alloc(vdev, size, args->flags);
318 	if (IS_ERR(bo)) {
319 		ivpu_err(vdev, "Failed to allocate BO: %pe (ctx %u size %llu flags 0x%x)",
320 			 bo, file_priv->ctx.id, args->size, args->flags);
321 		return PTR_ERR(bo);
322 	}
323 
324 	ret = drm_gem_handle_create(file, &bo->base.base, &args->handle);
325 	if (!ret)
326 		args->vpu_addr = bo->vpu_addr;
327 
328 	drm_gem_object_put(&bo->base.base);
329 
330 	return ret;
331 }
332 
333 struct ivpu_bo *
334 ivpu_bo_create(struct ivpu_device *vdev, struct ivpu_mmu_context *ctx,
335 	       struct ivpu_addr_range *range, u64 size, u32 flags)
336 {
337 	struct iosys_map map;
338 	struct ivpu_bo *bo;
339 	int ret;
340 
341 	if (drm_WARN_ON(&vdev->drm, !range))
342 		return NULL;
343 
344 	drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->start));
345 	drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(range->end));
346 	drm_WARN_ON(&vdev->drm, !PAGE_ALIGNED(size));
347 
348 	bo = ivpu_bo_alloc(vdev, size, flags);
349 	if (IS_ERR(bo)) {
350 		ivpu_err(vdev, "Failed to allocate BO: %pe (vpu_addr 0x%llx size %llu flags 0x%x)",
351 			 bo, range->start, size, flags);
352 		return NULL;
353 	}
354 
355 	ret = ivpu_bo_alloc_vpu_addr(bo, ctx, range);
356 	if (ret)
357 		goto err_put;
358 
359 	ret = ivpu_bo_pin(bo);
360 	if (ret)
361 		goto err_put;
362 
363 	if (flags & DRM_IVPU_BO_MAPPABLE) {
364 		dma_resv_lock(bo->base.base.resv, NULL);
365 		ret = drm_gem_shmem_vmap(&bo->base, &map);
366 		dma_resv_unlock(bo->base.base.resv);
367 
368 		if (ret)
369 			goto err_put;
370 	}
371 
372 	return bo;
373 
374 err_put:
375 	drm_gem_object_put(&bo->base.base);
376 	return NULL;
377 }
378 
379 struct ivpu_bo *ivpu_bo_create_global(struct ivpu_device *vdev, u64 size, u32 flags)
380 {
381 	return ivpu_bo_create(vdev, &vdev->gctx, &vdev->hw->ranges.global, size, flags);
382 }
383 
384 void ivpu_bo_free(struct ivpu_bo *bo)
385 {
386 	struct iosys_map map = IOSYS_MAP_INIT_VADDR(bo->base.vaddr);
387 
388 	if (bo->flags & DRM_IVPU_BO_MAPPABLE) {
389 		dma_resv_lock(bo->base.base.resv, NULL);
390 		drm_gem_shmem_vunmap(&bo->base, &map);
391 		dma_resv_unlock(bo->base.base.resv);
392 	}
393 
394 	drm_gem_object_put(&bo->base.base);
395 }
396 
397 int ivpu_bo_info_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
398 {
399 	struct drm_ivpu_bo_info *args = data;
400 	struct drm_gem_object *obj;
401 	struct ivpu_bo *bo;
402 	int ret = 0;
403 
404 	obj = drm_gem_object_lookup(file, args->handle);
405 	if (!obj)
406 		return -ENOENT;
407 
408 	bo = to_ivpu_bo(obj);
409 
410 	mutex_lock(&bo->lock);
411 	args->flags = bo->flags;
412 	args->mmap_offset = drm_vma_node_offset_addr(&obj->vma_node);
413 	args->vpu_addr = bo->vpu_addr;
414 	args->size = obj->size;
415 	mutex_unlock(&bo->lock);
416 
417 	drm_gem_object_put(obj);
418 	return ret;
419 }
420 
421 int ivpu_bo_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
422 {
423 	struct drm_ivpu_bo_wait *args = data;
424 	struct drm_gem_object *obj;
425 	unsigned long timeout;
426 	long ret;
427 
428 	timeout = drm_timeout_abs_to_jiffies(args->timeout_ns);
429 
430 	/* Add 1 jiffy to ensure the wait function never times out before intended timeout_ns */
431 	timeout += 1;
432 
433 	obj = drm_gem_object_lookup(file, args->handle);
434 	if (!obj)
435 		return -EINVAL;
436 
437 	ret = dma_resv_wait_timeout(obj->resv, DMA_RESV_USAGE_READ, true, timeout);
438 	if (ret == 0) {
439 		ret = -ETIMEDOUT;
440 	} else if (ret > 0) {
441 		ret = 0;
442 		args->job_status = to_ivpu_bo(obj)->job_status;
443 	}
444 
445 	drm_gem_object_put(obj);
446 
447 	return ret;
448 }
449 
450 static void ivpu_bo_print_info(struct ivpu_bo *bo, struct drm_printer *p)
451 {
452 	mutex_lock(&bo->lock);
453 
454 	drm_printf(p, "%-9p %-3u 0x%-12llx %-10lu 0x%-8x %-4u",
455 		   bo, bo->ctx ? bo->ctx->id : 0, bo->vpu_addr, bo->base.base.size,
456 		   bo->flags, kref_read(&bo->base.base.refcount));
457 
458 	if (bo->base.pages)
459 		drm_printf(p, " has_pages");
460 
461 	if (bo->mmu_mapped)
462 		drm_printf(p, " mmu_mapped");
463 
464 	if (bo->base.base.import_attach)
465 		drm_printf(p, " imported");
466 
467 	drm_printf(p, "\n");
468 
469 	mutex_unlock(&bo->lock);
470 }
471 
472 void ivpu_bo_list(struct drm_device *dev, struct drm_printer *p)
473 {
474 	struct ivpu_device *vdev = to_ivpu_device(dev);
475 	struct ivpu_bo *bo;
476 
477 	drm_printf(p, "%-9s %-3s %-14s %-10s %-10s %-4s %s\n",
478 		   "bo", "ctx", "vpu_addr", "size", "flags", "refs", "attribs");
479 
480 	mutex_lock(&vdev->bo_list_lock);
481 	list_for_each_entry(bo, &vdev->bo_list, bo_list_node)
482 		ivpu_bo_print_info(bo, p);
483 	mutex_unlock(&vdev->bo_list_lock);
484 }
485 
486 void ivpu_bo_list_print(struct drm_device *dev)
487 {
488 	struct drm_printer p = drm_info_printer(dev->dev);
489 
490 	ivpu_bo_list(dev, &p);
491 }
492