1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2022-2024, Advanced Micro Devices, Inc.
4 */
5
6 #include <drm/amdxdna_accel.h>
7 #include <drm/drm_device.h>
8 #include <drm/drm_drv.h>
9 #include <drm/drm_file.h>
10 #include <drm/drm_gem.h>
11 #include <drm/drm_gem_shmem_helper.h>
12 #include <drm/drm_print.h>
13 #include <drm/gpu_scheduler.h>
14 #include <linux/xarray.h>
15 #include <trace/events/amdxdna.h>
16
17 #include "amdxdna_ctx.h"
18 #include "amdxdna_gem.h"
19 #include "amdxdna_pci_drv.h"
20
21 #define MAX_HWCTX_ID 255
22 #define MAX_ARG_COUNT 4095
23
24 struct amdxdna_fence {
25 struct dma_fence base;
26 spinlock_t lock; /* for base */
27 struct amdxdna_hwctx *hwctx;
28 };
29
amdxdna_fence_get_driver_name(struct dma_fence * fence)30 static const char *amdxdna_fence_get_driver_name(struct dma_fence *fence)
31 {
32 return KBUILD_MODNAME;
33 }
34
amdxdna_fence_get_timeline_name(struct dma_fence * fence)35 static const char *amdxdna_fence_get_timeline_name(struct dma_fence *fence)
36 {
37 struct amdxdna_fence *xdna_fence;
38
39 xdna_fence = container_of(fence, struct amdxdna_fence, base);
40
41 return xdna_fence->hwctx->name;
42 }
43
44 static const struct dma_fence_ops fence_ops = {
45 .get_driver_name = amdxdna_fence_get_driver_name,
46 .get_timeline_name = amdxdna_fence_get_timeline_name,
47 };
48
amdxdna_fence_create(struct amdxdna_hwctx * hwctx)49 static struct dma_fence *amdxdna_fence_create(struct amdxdna_hwctx *hwctx)
50 {
51 struct amdxdna_fence *fence;
52
53 fence = kzalloc(sizeof(*fence), GFP_KERNEL);
54 if (!fence)
55 return NULL;
56
57 fence->hwctx = hwctx;
58 spin_lock_init(&fence->lock);
59 dma_fence_init(&fence->base, &fence_ops, &fence->lock, hwctx->id, 0);
60 return &fence->base;
61 }
62
amdxdna_hwctx_suspend(struct amdxdna_client * client)63 void amdxdna_hwctx_suspend(struct amdxdna_client *client)
64 {
65 struct amdxdna_dev *xdna = client->xdna;
66 struct amdxdna_hwctx *hwctx;
67 unsigned long hwctx_id;
68
69 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
70 mutex_lock(&client->hwctx_lock);
71 amdxdna_for_each_hwctx(client, hwctx_id, hwctx)
72 xdna->dev_info->ops->hwctx_suspend(hwctx);
73 mutex_unlock(&client->hwctx_lock);
74 }
75
amdxdna_hwctx_resume(struct amdxdna_client * client)76 void amdxdna_hwctx_resume(struct amdxdna_client *client)
77 {
78 struct amdxdna_dev *xdna = client->xdna;
79 struct amdxdna_hwctx *hwctx;
80 unsigned long hwctx_id;
81
82 drm_WARN_ON(&xdna->ddev, !mutex_is_locked(&xdna->dev_lock));
83 mutex_lock(&client->hwctx_lock);
84 amdxdna_for_each_hwctx(client, hwctx_id, hwctx)
85 xdna->dev_info->ops->hwctx_resume(hwctx);
86 mutex_unlock(&client->hwctx_lock);
87 }
88
amdxdna_hwctx_destroy_rcu(struct amdxdna_hwctx * hwctx,struct srcu_struct * ss)89 static void amdxdna_hwctx_destroy_rcu(struct amdxdna_hwctx *hwctx,
90 struct srcu_struct *ss)
91 {
92 struct amdxdna_dev *xdna = hwctx->client->xdna;
93
94 synchronize_srcu(ss);
95
96 /* At this point, user is not able to submit new commands */
97 mutex_lock(&xdna->dev_lock);
98 xdna->dev_info->ops->hwctx_fini(hwctx);
99 mutex_unlock(&xdna->dev_lock);
100
101 kfree(hwctx->name);
102 kfree(hwctx);
103 }
104
amdxdna_cmd_get_payload(struct amdxdna_gem_obj * abo,u32 * size)105 void *amdxdna_cmd_get_payload(struct amdxdna_gem_obj *abo, u32 *size)
106 {
107 struct amdxdna_cmd *cmd = abo->mem.kva;
108 u32 num_masks, count;
109
110 if (amdxdna_cmd_get_op(abo) == ERT_CMD_CHAIN)
111 num_masks = 0;
112 else
113 num_masks = 1 + FIELD_GET(AMDXDNA_CMD_EXTRA_CU_MASK, cmd->header);
114
115 if (size) {
116 count = FIELD_GET(AMDXDNA_CMD_COUNT, cmd->header);
117 if (unlikely(count <= num_masks)) {
118 *size = 0;
119 return NULL;
120 }
121 *size = (count - num_masks) * sizeof(u32);
122 }
123 return &cmd->data[num_masks];
124 }
125
amdxdna_cmd_get_cu_idx(struct amdxdna_gem_obj * abo)126 int amdxdna_cmd_get_cu_idx(struct amdxdna_gem_obj *abo)
127 {
128 struct amdxdna_cmd *cmd = abo->mem.kva;
129 u32 num_masks, i;
130 u32 *cu_mask;
131
132 if (amdxdna_cmd_get_op(abo) == ERT_CMD_CHAIN)
133 return -1;
134
135 num_masks = 1 + FIELD_GET(AMDXDNA_CMD_EXTRA_CU_MASK, cmd->header);
136 cu_mask = cmd->data;
137 for (i = 0; i < num_masks; i++) {
138 if (cu_mask[i])
139 return ffs(cu_mask[i]) - 1;
140 }
141
142 return -1;
143 }
144
145 /*
146 * This should be called in close() and remove(). DO NOT call in other syscalls.
147 * This guarantee that when hwctx and resources will be released, if user
148 * doesn't call amdxdna_drm_destroy_hwctx_ioctl.
149 */
amdxdna_hwctx_remove_all(struct amdxdna_client * client)150 void amdxdna_hwctx_remove_all(struct amdxdna_client *client)
151 {
152 struct amdxdna_hwctx *hwctx;
153 unsigned long hwctx_id;
154
155 mutex_lock(&client->hwctx_lock);
156 amdxdna_for_each_hwctx(client, hwctx_id, hwctx) {
157 XDNA_DBG(client->xdna, "PID %d close HW context %d",
158 client->pid, hwctx->id);
159 xa_erase(&client->hwctx_xa, hwctx->id);
160 mutex_unlock(&client->hwctx_lock);
161 amdxdna_hwctx_destroy_rcu(hwctx, &client->hwctx_srcu);
162 mutex_lock(&client->hwctx_lock);
163 }
164 mutex_unlock(&client->hwctx_lock);
165 }
166
amdxdna_drm_create_hwctx_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)167 int amdxdna_drm_create_hwctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
168 {
169 struct amdxdna_client *client = filp->driver_priv;
170 struct amdxdna_drm_create_hwctx *args = data;
171 struct amdxdna_dev *xdna = to_xdna_dev(dev);
172 struct amdxdna_hwctx *hwctx;
173 int ret, idx;
174
175 if (args->ext || args->ext_flags)
176 return -EINVAL;
177
178 if (!drm_dev_enter(dev, &idx))
179 return -ENODEV;
180
181 hwctx = kzalloc(sizeof(*hwctx), GFP_KERNEL);
182 if (!hwctx) {
183 ret = -ENOMEM;
184 goto exit;
185 }
186
187 if (copy_from_user(&hwctx->qos, u64_to_user_ptr(args->qos_p), sizeof(hwctx->qos))) {
188 XDNA_ERR(xdna, "Access QoS info failed");
189 ret = -EFAULT;
190 goto free_hwctx;
191 }
192
193 hwctx->client = client;
194 hwctx->fw_ctx_id = -1;
195 hwctx->num_tiles = args->num_tiles;
196 hwctx->mem_size = args->mem_size;
197 hwctx->max_opc = args->max_opc;
198 ret = xa_alloc_cyclic(&client->hwctx_xa, &hwctx->id, hwctx,
199 XA_LIMIT(AMDXDNA_INVALID_CTX_HANDLE + 1, MAX_HWCTX_ID),
200 &client->next_hwctxid, GFP_KERNEL);
201 if (ret < 0) {
202 XDNA_ERR(xdna, "Allocate hwctx ID failed, ret %d", ret);
203 goto free_hwctx;
204 }
205
206 hwctx->name = kasprintf(GFP_KERNEL, "hwctx.%d.%d", client->pid, hwctx->id);
207 if (!hwctx->name) {
208 ret = -ENOMEM;
209 goto rm_id;
210 }
211
212 mutex_lock(&xdna->dev_lock);
213 ret = xdna->dev_info->ops->hwctx_init(hwctx);
214 if (ret) {
215 mutex_unlock(&xdna->dev_lock);
216 XDNA_ERR(xdna, "Init hwctx failed, ret %d", ret);
217 goto free_name;
218 }
219 args->handle = hwctx->id;
220 args->syncobj_handle = hwctx->syncobj_hdl;
221 mutex_unlock(&xdna->dev_lock);
222
223 atomic64_set(&hwctx->job_submit_cnt, 0);
224 atomic64_set(&hwctx->job_free_cnt, 0);
225 XDNA_DBG(xdna, "PID %d create HW context %d, ret %d", client->pid, args->handle, ret);
226 drm_dev_exit(idx);
227 return 0;
228
229 free_name:
230 kfree(hwctx->name);
231 rm_id:
232 xa_erase(&client->hwctx_xa, hwctx->id);
233 free_hwctx:
234 kfree(hwctx);
235 exit:
236 drm_dev_exit(idx);
237 return ret;
238 }
239
amdxdna_drm_destroy_hwctx_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)240 int amdxdna_drm_destroy_hwctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
241 {
242 struct amdxdna_client *client = filp->driver_priv;
243 struct amdxdna_drm_destroy_hwctx *args = data;
244 struct amdxdna_dev *xdna = to_xdna_dev(dev);
245 struct amdxdna_hwctx *hwctx;
246 int ret = 0, idx;
247
248 if (XDNA_MBZ_DBG(xdna, &args->pad, sizeof(args->pad)))
249 return -EINVAL;
250
251 if (!drm_dev_enter(dev, &idx))
252 return -ENODEV;
253
254 hwctx = xa_erase(&client->hwctx_xa, args->handle);
255 if (!hwctx) {
256 ret = -EINVAL;
257 XDNA_DBG(xdna, "PID %d HW context %d not exist",
258 client->pid, args->handle);
259 goto out;
260 }
261
262 /*
263 * The pushed jobs are handled by DRM scheduler during destroy.
264 * SRCU to synchronize with exec command ioctls.
265 */
266 amdxdna_hwctx_destroy_rcu(hwctx, &client->hwctx_srcu);
267
268 XDNA_DBG(xdna, "PID %d destroyed HW context %d", client->pid, args->handle);
269 out:
270 drm_dev_exit(idx);
271 return ret;
272 }
273
amdxdna_drm_config_hwctx_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)274 int amdxdna_drm_config_hwctx_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
275 {
276 struct amdxdna_client *client = filp->driver_priv;
277 struct amdxdna_drm_config_hwctx *args = data;
278 struct amdxdna_dev *xdna = to_xdna_dev(dev);
279 struct amdxdna_hwctx *hwctx;
280 int ret, idx;
281 u32 buf_size;
282 void *buf;
283 u64 val;
284
285 if (XDNA_MBZ_DBG(xdna, &args->pad, sizeof(args->pad)))
286 return -EINVAL;
287
288 if (!xdna->dev_info->ops->hwctx_config)
289 return -EOPNOTSUPP;
290
291 val = args->param_val;
292 buf_size = args->param_val_size;
293
294 switch (args->param_type) {
295 case DRM_AMDXDNA_HWCTX_CONFIG_CU:
296 /* For those types that param_val is pointer */
297 if (buf_size > PAGE_SIZE) {
298 XDNA_ERR(xdna, "Config CU param buffer too large");
299 return -E2BIG;
300 }
301
302 /* Hwctx needs to keep buf */
303 buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
304 if (!buf)
305 return -ENOMEM;
306
307 if (copy_from_user(buf, u64_to_user_ptr(val), buf_size)) {
308 kfree(buf);
309 return -EFAULT;
310 }
311
312 break;
313 case DRM_AMDXDNA_HWCTX_ASSIGN_DBG_BUF:
314 case DRM_AMDXDNA_HWCTX_REMOVE_DBG_BUF:
315 /* For those types that param_val is a value */
316 buf = NULL;
317 buf_size = 0;
318 break;
319 default:
320 XDNA_DBG(xdna, "Unknown HW context config type %d", args->param_type);
321 return -EINVAL;
322 }
323
324 mutex_lock(&xdna->dev_lock);
325 idx = srcu_read_lock(&client->hwctx_srcu);
326 hwctx = xa_load(&client->hwctx_xa, args->handle);
327 if (!hwctx) {
328 XDNA_DBG(xdna, "PID %d failed to get hwctx %d", client->pid, args->handle);
329 ret = -EINVAL;
330 goto unlock_srcu;
331 }
332
333 ret = xdna->dev_info->ops->hwctx_config(hwctx, args->param_type, val, buf, buf_size);
334
335 unlock_srcu:
336 srcu_read_unlock(&client->hwctx_srcu, idx);
337 mutex_unlock(&xdna->dev_lock);
338 kfree(buf);
339 return ret;
340 }
341
342 static void
amdxdna_arg_bos_put(struct amdxdna_sched_job * job)343 amdxdna_arg_bos_put(struct amdxdna_sched_job *job)
344 {
345 int i;
346
347 for (i = 0; i < job->bo_cnt; i++) {
348 if (!job->bos[i])
349 break;
350 drm_gem_object_put(job->bos[i]);
351 }
352 }
353
354 static int
amdxdna_arg_bos_lookup(struct amdxdna_client * client,struct amdxdna_sched_job * job,u32 * bo_hdls,u32 bo_cnt)355 amdxdna_arg_bos_lookup(struct amdxdna_client *client,
356 struct amdxdna_sched_job *job,
357 u32 *bo_hdls, u32 bo_cnt)
358 {
359 struct drm_gem_object *gobj;
360 int i, ret;
361
362 job->bo_cnt = bo_cnt;
363 for (i = 0; i < job->bo_cnt; i++) {
364 struct amdxdna_gem_obj *abo;
365
366 gobj = drm_gem_object_lookup(client->filp, bo_hdls[i]);
367 if (!gobj) {
368 ret = -ENOENT;
369 goto put_shmem_bo;
370 }
371 abo = to_xdna_obj(gobj);
372
373 mutex_lock(&abo->lock);
374 if (abo->pinned) {
375 mutex_unlock(&abo->lock);
376 job->bos[i] = gobj;
377 continue;
378 }
379
380 ret = amdxdna_gem_pin_nolock(abo);
381 if (ret) {
382 mutex_unlock(&abo->lock);
383 drm_gem_object_put(gobj);
384 goto put_shmem_bo;
385 }
386 abo->pinned = true;
387 mutex_unlock(&abo->lock);
388
389 job->bos[i] = gobj;
390 }
391
392 return 0;
393
394 put_shmem_bo:
395 amdxdna_arg_bos_put(job);
396 return ret;
397 }
398
amdxdna_sched_job_cleanup(struct amdxdna_sched_job * job)399 void amdxdna_sched_job_cleanup(struct amdxdna_sched_job *job)
400 {
401 trace_amdxdna_debug_point(job->hwctx->name, job->seq, "job release");
402 amdxdna_arg_bos_put(job);
403 amdxdna_gem_put_obj(job->cmd_bo);
404 }
405
amdxdna_cmd_submit(struct amdxdna_client * client,u32 cmd_bo_hdl,u32 * arg_bo_hdls,u32 arg_bo_cnt,u32 hwctx_hdl,u64 * seq)406 int amdxdna_cmd_submit(struct amdxdna_client *client,
407 u32 cmd_bo_hdl, u32 *arg_bo_hdls, u32 arg_bo_cnt,
408 u32 hwctx_hdl, u64 *seq)
409 {
410 struct amdxdna_dev *xdna = client->xdna;
411 struct amdxdna_sched_job *job;
412 struct amdxdna_hwctx *hwctx;
413 int ret, idx;
414
415 XDNA_DBG(xdna, "Command BO hdl %d, Arg BO count %d", cmd_bo_hdl, arg_bo_cnt);
416 job = kzalloc(struct_size(job, bos, arg_bo_cnt), GFP_KERNEL);
417 if (!job)
418 return -ENOMEM;
419
420 if (cmd_bo_hdl != AMDXDNA_INVALID_BO_HANDLE) {
421 job->cmd_bo = amdxdna_gem_get_obj(client, cmd_bo_hdl, AMDXDNA_BO_CMD);
422 if (!job->cmd_bo) {
423 XDNA_ERR(xdna, "Failed to get cmd bo from %d", cmd_bo_hdl);
424 ret = -EINVAL;
425 goto free_job;
426 }
427 } else {
428 job->cmd_bo = NULL;
429 }
430
431 ret = amdxdna_arg_bos_lookup(client, job, arg_bo_hdls, arg_bo_cnt);
432 if (ret) {
433 XDNA_ERR(xdna, "Argument BOs lookup failed, ret %d", ret);
434 goto cmd_put;
435 }
436
437 idx = srcu_read_lock(&client->hwctx_srcu);
438 hwctx = xa_load(&client->hwctx_xa, hwctx_hdl);
439 if (!hwctx) {
440 XDNA_DBG(xdna, "PID %d failed to get hwctx %d",
441 client->pid, hwctx_hdl);
442 ret = -EINVAL;
443 goto unlock_srcu;
444 }
445
446 if (hwctx->status != HWCTX_STAT_READY) {
447 XDNA_ERR(xdna, "HW Context is not ready");
448 ret = -EINVAL;
449 goto unlock_srcu;
450 }
451
452 job->hwctx = hwctx;
453 job->mm = current->mm;
454
455 job->fence = amdxdna_fence_create(hwctx);
456 if (!job->fence) {
457 XDNA_ERR(xdna, "Failed to create fence");
458 ret = -ENOMEM;
459 goto unlock_srcu;
460 }
461 kref_init(&job->refcnt);
462
463 ret = xdna->dev_info->ops->cmd_submit(hwctx, job, seq);
464 if (ret)
465 goto put_fence;
466
467 /*
468 * The amdxdna_hwctx_destroy_rcu() will release hwctx and associated
469 * resource after synchronize_srcu(). The submitted jobs should be
470 * handled by the queue, for example DRM scheduler, in device layer.
471 * For here we can unlock SRCU.
472 */
473 srcu_read_unlock(&client->hwctx_srcu, idx);
474 trace_amdxdna_debug_point(hwctx->name, *seq, "job pushed");
475
476 return 0;
477
478 put_fence:
479 dma_fence_put(job->fence);
480 unlock_srcu:
481 srcu_read_unlock(&client->hwctx_srcu, idx);
482 amdxdna_arg_bos_put(job);
483 cmd_put:
484 amdxdna_gem_put_obj(job->cmd_bo);
485 free_job:
486 kfree(job);
487 return ret;
488 }
489
490 /*
491 * The submit command ioctl submits a command to firmware. One firmware command
492 * may contain multiple command BOs for processing as a whole.
493 * The command sequence number is returned which can be used for wait command ioctl.
494 */
amdxdna_drm_submit_execbuf(struct amdxdna_client * client,struct amdxdna_drm_exec_cmd * args)495 static int amdxdna_drm_submit_execbuf(struct amdxdna_client *client,
496 struct amdxdna_drm_exec_cmd *args)
497 {
498 struct amdxdna_dev *xdna = client->xdna;
499 u32 *arg_bo_hdls;
500 u32 cmd_bo_hdl;
501 int ret;
502
503 if (!args->arg_count || args->arg_count > MAX_ARG_COUNT) {
504 XDNA_ERR(xdna, "Invalid arg bo count %d", args->arg_count);
505 return -EINVAL;
506 }
507
508 /* Only support single command for now. */
509 if (args->cmd_count != 1) {
510 XDNA_ERR(xdna, "Invalid cmd bo count %d", args->cmd_count);
511 return -EINVAL;
512 }
513
514 cmd_bo_hdl = (u32)args->cmd_handles;
515 arg_bo_hdls = kcalloc(args->arg_count, sizeof(u32), GFP_KERNEL);
516 if (!arg_bo_hdls)
517 return -ENOMEM;
518 ret = copy_from_user(arg_bo_hdls, u64_to_user_ptr(args->args),
519 args->arg_count * sizeof(u32));
520 if (ret) {
521 ret = -EFAULT;
522 goto free_cmd_bo_hdls;
523 }
524
525 ret = amdxdna_cmd_submit(client, cmd_bo_hdl, arg_bo_hdls,
526 args->arg_count, args->hwctx, &args->seq);
527 if (ret)
528 XDNA_DBG(xdna, "Submit cmds failed, ret %d", ret);
529
530 free_cmd_bo_hdls:
531 kfree(arg_bo_hdls);
532 if (!ret)
533 XDNA_DBG(xdna, "Pushed cmd %lld to scheduler", args->seq);
534 return ret;
535 }
536
amdxdna_drm_submit_cmd_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)537 int amdxdna_drm_submit_cmd_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
538 {
539 struct amdxdna_client *client = filp->driver_priv;
540 struct amdxdna_drm_exec_cmd *args = data;
541
542 if (args->ext || args->ext_flags)
543 return -EINVAL;
544
545 switch (args->type) {
546 case AMDXDNA_CMD_SUBMIT_EXEC_BUF:
547 return amdxdna_drm_submit_execbuf(client, args);
548 }
549
550 XDNA_ERR(client->xdna, "Invalid command type %d", args->type);
551 return -EINVAL;
552 }
553