1 // SPDX-License-Identifier: GPL-2.0 or MIT
2 /* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
3 /* Copyright 2023 Collabora ltd. */
4
5 #include <drm/drm_debugfs.h>
6 #include <drm/drm_drv.h>
7 #include <drm/drm_exec.h>
8 #include <drm/drm_gpuvm.h>
9 #include <drm/drm_managed.h>
10 #include <drm/gpu_scheduler.h>
11 #include <drm/panthor_drm.h>
12
13 #include <linux/atomic.h>
14 #include <linux/bitfield.h>
15 #include <linux/delay.h>
16 #include <linux/dma-mapping.h>
17 #include <linux/interrupt.h>
18 #include <linux/io.h>
19 #include <linux/iopoll.h>
20 #include <linux/io-pgtable.h>
21 #include <linux/iommu.h>
22 #include <linux/kmemleak.h>
23 #include <linux/platform_device.h>
24 #include <linux/pm_runtime.h>
25 #include <linux/rwsem.h>
26 #include <linux/sched.h>
27 #include <linux/shmem_fs.h>
28 #include <linux/sizes.h>
29
30 #include "panthor_device.h"
31 #include "panthor_gem.h"
32 #include "panthor_heap.h"
33 #include "panthor_mmu.h"
34 #include "panthor_regs.h"
35 #include "panthor_sched.h"
36
37 #define MAX_AS_SLOTS 32
38
39 struct panthor_vm;
40
41 /**
42 * struct panthor_as_slot - Address space slot
43 */
44 struct panthor_as_slot {
45 /** @vm: VM bound to this slot. NULL is no VM is bound. */
46 struct panthor_vm *vm;
47 };
48
49 /**
50 * struct panthor_mmu - MMU related data
51 */
52 struct panthor_mmu {
53 /** @irq: The MMU irq. */
54 struct panthor_irq irq;
55
56 /**
57 * @as: Address space related fields.
58 *
59 * The GPU has a limited number of address spaces (AS) slots, forcing
60 * us to re-assign them to re-assign slots on-demand.
61 */
62 struct {
63 /** @as.slots_lock: Lock protecting access to all other AS fields. */
64 struct mutex slots_lock;
65
66 /** @as.alloc_mask: Bitmask encoding the allocated slots. */
67 unsigned long alloc_mask;
68
69 /** @as.faulty_mask: Bitmask encoding the faulty slots. */
70 unsigned long faulty_mask;
71
72 /** @as.slots: VMs currently bound to the AS slots. */
73 struct panthor_as_slot slots[MAX_AS_SLOTS];
74
75 /**
76 * @as.lru_list: List of least recently used VMs.
77 *
78 * We use this list to pick a VM to evict when all slots are
79 * used.
80 *
81 * There should be no more active VMs than there are AS slots,
82 * so this LRU is just here to keep VMs bound until there's
83 * a need to release a slot, thus avoid unnecessary TLB/cache
84 * flushes.
85 */
86 struct list_head lru_list;
87 } as;
88
89 /** @vm: VMs management fields */
90 struct {
91 /** @vm.lock: Lock protecting access to list. */
92 struct mutex lock;
93
94 /** @vm.list: List containing all VMs. */
95 struct list_head list;
96
97 /** @vm.reset_in_progress: True if a reset is in progress. */
98 bool reset_in_progress;
99
100 /** @vm.wq: Workqueue used for the VM_BIND queues. */
101 struct workqueue_struct *wq;
102 } vm;
103 };
104
105 /**
106 * struct panthor_vm_pool - VM pool object
107 */
108 struct panthor_vm_pool {
109 /** @xa: Array used for VM handle tracking. */
110 struct xarray xa;
111 };
112
113 /**
114 * struct panthor_vma - GPU mapping object
115 *
116 * This is used to track GEM mappings in GPU space.
117 */
118 struct panthor_vma {
119 /** @base: Inherits from drm_gpuva. */
120 struct drm_gpuva base;
121
122 /** @node: Used to implement deferred release of VMAs. */
123 struct list_head node;
124
125 /**
126 * @flags: Combination of drm_panthor_vm_bind_op_flags.
127 *
128 * Only map related flags are accepted.
129 */
130 u32 flags;
131 };
132
133 /**
134 * struct panthor_vm_op_ctx - VM operation context
135 *
136 * With VM operations potentially taking place in a dma-signaling path, we
137 * need to make sure everything that might require resource allocation is
138 * pre-allocated upfront. This is what this operation context is far.
139 *
140 * We also collect resources that have been freed, so we can release them
141 * asynchronously, and let the VM_BIND scheduler process the next VM_BIND
142 * request.
143 */
144 struct panthor_vm_op_ctx {
145 /** @rsvd_page_tables: Pages reserved for the MMU page table update. */
146 struct {
147 /** @rsvd_page_tables.count: Number of pages reserved. */
148 u32 count;
149
150 /** @rsvd_page_tables.ptr: Point to the first unused page in the @pages table. */
151 u32 ptr;
152
153 /**
154 * @rsvd_page_tables.pages: Array of pages to be used for an MMU page table update.
155 *
156 * After an VM operation, there might be free pages left in this array.
157 * They should be returned to the pt_cache as part of the op_ctx cleanup.
158 */
159 void **pages;
160 } rsvd_page_tables;
161
162 /**
163 * @preallocated_vmas: Pre-allocated VMAs to handle the remap case.
164 *
165 * Partial unmap requests or map requests overlapping existing mappings will
166 * trigger a remap call, which need to register up to three panthor_vma objects
167 * (one for the new mapping, and two for the previous and next mappings).
168 */
169 struct panthor_vma *preallocated_vmas[3];
170
171 /** @flags: Combination of drm_panthor_vm_bind_op_flags. */
172 u32 flags;
173
174 /** @va: Virtual range targeted by the VM operation. */
175 struct {
176 /** @va.addr: Start address. */
177 u64 addr;
178
179 /** @va.range: Range size. */
180 u64 range;
181 } va;
182
183 /**
184 * @returned_vmas: List of panthor_vma objects returned after a VM operation.
185 *
186 * For unmap operations, this will contain all VMAs that were covered by the
187 * specified VA range.
188 *
189 * For map operations, this will contain all VMAs that previously mapped to
190 * the specified VA range.
191 *
192 * Those VMAs, and the resources they point to will be released as part of
193 * the op_ctx cleanup operation.
194 */
195 struct list_head returned_vmas;
196
197 /** @map: Fields specific to a map operation. */
198 struct {
199 /** @map.vm_bo: Buffer object to map. */
200 struct drm_gpuvm_bo *vm_bo;
201
202 /** @map.bo_offset: Offset in the buffer object. */
203 u64 bo_offset;
204
205 /**
206 * @map.sgt: sg-table pointing to pages backing the GEM object.
207 *
208 * This is gathered at job creation time, such that we don't have
209 * to allocate in ::run_job().
210 */
211 struct sg_table *sgt;
212
213 /**
214 * @map.new_vma: The new VMA object that will be inserted to the VA tree.
215 */
216 struct panthor_vma *new_vma;
217 } map;
218 };
219
220 /**
221 * struct panthor_vm - VM object
222 *
223 * A VM is an object representing a GPU (or MCU) virtual address space.
224 * It embeds the MMU page table for this address space, a tree containing
225 * all the virtual mappings of GEM objects, and other things needed to manage
226 * the VM.
227 *
228 * Except for the MCU VM, which is managed by the kernel, all other VMs are
229 * created by userspace and mostly managed by userspace, using the
230 * %DRM_IOCTL_PANTHOR_VM_BIND ioctl.
231 *
232 * A portion of the virtual address space is reserved for kernel objects,
233 * like heap chunks, and userspace gets to decide how much of the virtual
234 * address space is left to the kernel (half of the virtual address space
235 * by default).
236 */
237 struct panthor_vm {
238 /**
239 * @base: Inherit from drm_gpuvm.
240 *
241 * We delegate all the VA management to the common drm_gpuvm framework
242 * and only implement hooks to update the MMU page table.
243 */
244 struct drm_gpuvm base;
245
246 /**
247 * @sched: Scheduler used for asynchronous VM_BIND request.
248 *
249 * We use a 1:1 scheduler here.
250 */
251 struct drm_gpu_scheduler sched;
252
253 /**
254 * @entity: Scheduling entity representing the VM_BIND queue.
255 *
256 * There's currently one bind queue per VM. It doesn't make sense to
257 * allow more given the VM operations are serialized anyway.
258 */
259 struct drm_sched_entity entity;
260
261 /** @ptdev: Device. */
262 struct panthor_device *ptdev;
263
264 /** @memattr: Value to program to the AS_MEMATTR register. */
265 u64 memattr;
266
267 /** @pgtbl_ops: Page table operations. */
268 struct io_pgtable_ops *pgtbl_ops;
269
270 /** @root_page_table: Stores the root page table pointer. */
271 void *root_page_table;
272
273 /**
274 * @op_lock: Lock used to serialize operations on a VM.
275 *
276 * The serialization of jobs queued to the VM_BIND queue is already
277 * taken care of by drm_sched, but we need to serialize synchronous
278 * and asynchronous VM_BIND request. This is what this lock is for.
279 */
280 struct mutex op_lock;
281
282 /**
283 * @op_ctx: The context attached to the currently executing VM operation.
284 *
285 * NULL when no operation is in progress.
286 */
287 struct panthor_vm_op_ctx *op_ctx;
288
289 /**
290 * @mm: Memory management object representing the auto-VA/kernel-VA.
291 *
292 * Used to auto-allocate VA space for kernel-managed objects (tiler
293 * heaps, ...).
294 *
295 * For the MCU VM, this is managing the VA range that's used to map
296 * all shared interfaces.
297 *
298 * For user VMs, the range is specified by userspace, and must not
299 * exceed half of the VA space addressable.
300 */
301 struct drm_mm mm;
302
303 /** @mm_lock: Lock protecting the @mm field. */
304 struct mutex mm_lock;
305
306 /** @kernel_auto_va: Automatic VA-range for kernel BOs. */
307 struct {
308 /** @kernel_auto_va.start: Start of the automatic VA-range for kernel BOs. */
309 u64 start;
310
311 /** @kernel_auto_va.size: Size of the automatic VA-range for kernel BOs. */
312 u64 end;
313 } kernel_auto_va;
314
315 /** @as: Address space related fields. */
316 struct {
317 /**
318 * @as.id: ID of the address space this VM is bound to.
319 *
320 * A value of -1 means the VM is inactive/not bound.
321 */
322 int id;
323
324 /** @as.active_cnt: Number of active users of this VM. */
325 refcount_t active_cnt;
326
327 /**
328 * @as.lru_node: Used to instead the VM in the panthor_mmu::as::lru_list.
329 *
330 * Active VMs should not be inserted in the LRU list.
331 */
332 struct list_head lru_node;
333 } as;
334
335 /**
336 * @heaps: Tiler heap related fields.
337 */
338 struct {
339 /**
340 * @heaps.pool: The heap pool attached to this VM.
341 *
342 * Will stay NULL until someone creates a heap context on this VM.
343 */
344 struct panthor_heap_pool *pool;
345
346 /** @heaps.lock: Lock used to protect access to @pool. */
347 struct mutex lock;
348 } heaps;
349
350 /** @node: Used to insert the VM in the panthor_mmu::vm::list. */
351 struct list_head node;
352
353 /** @for_mcu: True if this is the MCU VM. */
354 bool for_mcu;
355
356 /**
357 * @destroyed: True if the VM was destroyed.
358 *
359 * No further bind requests should be queued to a destroyed VM.
360 */
361 bool destroyed;
362
363 /**
364 * @unusable: True if the VM has turned unusable because something
365 * bad happened during an asynchronous request.
366 *
367 * We don't try to recover from such failures, because this implies
368 * informing userspace about the specific operation that failed, and
369 * hoping the userspace driver can replay things from there. This all
370 * sounds very complicated for little gain.
371 *
372 * Instead, we should just flag the VM as unusable, and fail any
373 * further request targeting this VM.
374 *
375 * We also provide a way to query a VM state, so userspace can destroy
376 * it and create a new one.
377 *
378 * As an analogy, this would be mapped to a VK_ERROR_DEVICE_LOST
379 * situation, where the logical device needs to be re-created.
380 */
381 bool unusable;
382
383 /**
384 * @unhandled_fault: Unhandled fault happened.
385 *
386 * This should be reported to the scheduler, and the queue/group be
387 * flagged as faulty as a result.
388 */
389 bool unhandled_fault;
390 };
391
392 /**
393 * struct panthor_vm_bind_job - VM bind job
394 */
395 struct panthor_vm_bind_job {
396 /** @base: Inherit from drm_sched_job. */
397 struct drm_sched_job base;
398
399 /** @refcount: Reference count. */
400 struct kref refcount;
401
402 /** @cleanup_op_ctx_work: Work used to cleanup the VM operation context. */
403 struct work_struct cleanup_op_ctx_work;
404
405 /** @vm: VM targeted by the VM operation. */
406 struct panthor_vm *vm;
407
408 /** @ctx: Operation context. */
409 struct panthor_vm_op_ctx ctx;
410 };
411
412 /*
413 * @pt_cache: Cache used to allocate MMU page tables.
414 *
415 * The pre-allocation pattern forces us to over-allocate to plan for
416 * the worst case scenario, and return the pages we didn't use.
417 *
418 * Having a kmem_cache allows us to speed allocations.
419 */
420 static struct kmem_cache *pt_cache;
421
422 /**
423 * alloc_pt() - Custom page table allocator
424 * @cookie: Cookie passed at page table allocation time.
425 * @size: Size of the page table. This size should be fixed,
426 * and determined at creation time based on the granule size.
427 * @gfp: GFP flags.
428 *
429 * We want a custom allocator so we can use a cache for page table
430 * allocations and amortize the cost of the over-reservation that's
431 * done to allow asynchronous VM operations.
432 *
433 * Return: non-NULL on success, NULL if the allocation failed for any
434 * reason.
435 */
alloc_pt(void * cookie,size_t size,gfp_t gfp)436 static void *alloc_pt(void *cookie, size_t size, gfp_t gfp)
437 {
438 struct panthor_vm *vm = cookie;
439 void *page;
440
441 /* Allocation of the root page table happening during init. */
442 if (unlikely(!vm->root_page_table)) {
443 struct page *p;
444
445 drm_WARN_ON(&vm->ptdev->base, vm->op_ctx);
446 p = alloc_pages_node(dev_to_node(vm->ptdev->base.dev),
447 gfp | __GFP_ZERO, get_order(size));
448 page = p ? page_address(p) : NULL;
449 vm->root_page_table = page;
450 return page;
451 }
452
453 /* We're not supposed to have anything bigger than 4k here, because we picked a
454 * 4k granule size at init time.
455 */
456 if (drm_WARN_ON(&vm->ptdev->base, size != SZ_4K))
457 return NULL;
458
459 /* We must have some op_ctx attached to the VM and it must have at least one
460 * free page.
461 */
462 if (drm_WARN_ON(&vm->ptdev->base, !vm->op_ctx) ||
463 drm_WARN_ON(&vm->ptdev->base,
464 vm->op_ctx->rsvd_page_tables.ptr >= vm->op_ctx->rsvd_page_tables.count))
465 return NULL;
466
467 page = vm->op_ctx->rsvd_page_tables.pages[vm->op_ctx->rsvd_page_tables.ptr++];
468 memset(page, 0, SZ_4K);
469
470 /* Page table entries don't use virtual addresses, which trips out
471 * kmemleak. kmemleak_alloc_phys() might work, but physical addresses
472 * are mixed with other fields, and I fear kmemleak won't detect that
473 * either.
474 *
475 * Let's just ignore memory passed to the page-table driver for now.
476 */
477 kmemleak_ignore(page);
478 return page;
479 }
480
481 /**
482 * free_pt() - Custom page table free function
483 * @cookie: Cookie passed at page table allocation time.
484 * @data: Page table to free.
485 * @size: Size of the page table. This size should be fixed,
486 * and determined at creation time based on the granule size.
487 */
free_pt(void * cookie,void * data,size_t size)488 static void free_pt(void *cookie, void *data, size_t size)
489 {
490 struct panthor_vm *vm = cookie;
491
492 if (unlikely(vm->root_page_table == data)) {
493 free_pages((unsigned long)data, get_order(size));
494 vm->root_page_table = NULL;
495 return;
496 }
497
498 if (drm_WARN_ON(&vm->ptdev->base, size != SZ_4K))
499 return;
500
501 /* Return the page to the pt_cache. */
502 kmem_cache_free(pt_cache, data);
503 }
504
wait_ready(struct panthor_device * ptdev,u32 as_nr)505 static int wait_ready(struct panthor_device *ptdev, u32 as_nr)
506 {
507 int ret;
508 u32 val;
509
510 /* Wait for the MMU status to indicate there is no active command, in
511 * case one is pending.
512 */
513 ret = readl_relaxed_poll_timeout_atomic(ptdev->iomem + AS_STATUS(as_nr),
514 val, !(val & AS_STATUS_AS_ACTIVE),
515 10, 100000);
516
517 if (ret) {
518 panthor_device_schedule_reset(ptdev);
519 drm_err(&ptdev->base, "AS_ACTIVE bit stuck\n");
520 }
521
522 return ret;
523 }
524
write_cmd(struct panthor_device * ptdev,u32 as_nr,u32 cmd)525 static int write_cmd(struct panthor_device *ptdev, u32 as_nr, u32 cmd)
526 {
527 int status;
528
529 /* write AS_COMMAND when MMU is ready to accept another command */
530 status = wait_ready(ptdev, as_nr);
531 if (!status)
532 gpu_write(ptdev, AS_COMMAND(as_nr), cmd);
533
534 return status;
535 }
536
lock_region(struct panthor_device * ptdev,u32 as_nr,u64 region_start,u64 size)537 static void lock_region(struct panthor_device *ptdev, u32 as_nr,
538 u64 region_start, u64 size)
539 {
540 u8 region_width;
541 u64 region;
542 u64 region_end = region_start + size;
543
544 if (!size)
545 return;
546
547 /*
548 * The locked region is a naturally aligned power of 2 block encoded as
549 * log2 minus(1).
550 * Calculate the desired start/end and look for the highest bit which
551 * differs. The smallest naturally aligned block must include this bit
552 * change, the desired region starts with this bit (and subsequent bits)
553 * zeroed and ends with the bit (and subsequent bits) set to one.
554 */
555 region_width = max(fls64(region_start ^ (region_end - 1)),
556 const_ilog2(AS_LOCK_REGION_MIN_SIZE)) - 1;
557
558 /*
559 * Mask off the low bits of region_start (which would be ignored by
560 * the hardware anyway)
561 */
562 region_start &= GENMASK_ULL(63, region_width);
563
564 region = region_width | region_start;
565
566 /* Lock the region that needs to be updated */
567 gpu_write(ptdev, AS_LOCKADDR_LO(as_nr), lower_32_bits(region));
568 gpu_write(ptdev, AS_LOCKADDR_HI(as_nr), upper_32_bits(region));
569 write_cmd(ptdev, as_nr, AS_COMMAND_LOCK);
570 }
571
mmu_hw_do_operation_locked(struct panthor_device * ptdev,int as_nr,u64 iova,u64 size,u32 op)572 static int mmu_hw_do_operation_locked(struct panthor_device *ptdev, int as_nr,
573 u64 iova, u64 size, u32 op)
574 {
575 lockdep_assert_held(&ptdev->mmu->as.slots_lock);
576
577 if (as_nr < 0)
578 return 0;
579
580 /*
581 * If the AS number is greater than zero, then we can be sure
582 * the device is up and running, so we don't need to explicitly
583 * power it up
584 */
585
586 if (op != AS_COMMAND_UNLOCK)
587 lock_region(ptdev, as_nr, iova, size);
588
589 /* Run the MMU operation */
590 write_cmd(ptdev, as_nr, op);
591
592 /* Wait for the flush to complete */
593 return wait_ready(ptdev, as_nr);
594 }
595
mmu_hw_do_operation(struct panthor_vm * vm,u64 iova,u64 size,u32 op)596 static int mmu_hw_do_operation(struct panthor_vm *vm,
597 u64 iova, u64 size, u32 op)
598 {
599 struct panthor_device *ptdev = vm->ptdev;
600 int ret;
601
602 mutex_lock(&ptdev->mmu->as.slots_lock);
603 ret = mmu_hw_do_operation_locked(ptdev, vm->as.id, iova, size, op);
604 mutex_unlock(&ptdev->mmu->as.slots_lock);
605
606 return ret;
607 }
608
panthor_mmu_as_enable(struct panthor_device * ptdev,u32 as_nr,u64 transtab,u64 transcfg,u64 memattr)609 static int panthor_mmu_as_enable(struct panthor_device *ptdev, u32 as_nr,
610 u64 transtab, u64 transcfg, u64 memattr)
611 {
612 int ret;
613
614 ret = mmu_hw_do_operation_locked(ptdev, as_nr, 0, ~0ULL, AS_COMMAND_FLUSH_MEM);
615 if (ret)
616 return ret;
617
618 gpu_write(ptdev, AS_TRANSTAB_LO(as_nr), lower_32_bits(transtab));
619 gpu_write(ptdev, AS_TRANSTAB_HI(as_nr), upper_32_bits(transtab));
620
621 gpu_write(ptdev, AS_MEMATTR_LO(as_nr), lower_32_bits(memattr));
622 gpu_write(ptdev, AS_MEMATTR_HI(as_nr), upper_32_bits(memattr));
623
624 gpu_write(ptdev, AS_TRANSCFG_LO(as_nr), lower_32_bits(transcfg));
625 gpu_write(ptdev, AS_TRANSCFG_HI(as_nr), upper_32_bits(transcfg));
626
627 return write_cmd(ptdev, as_nr, AS_COMMAND_UPDATE);
628 }
629
panthor_mmu_as_disable(struct panthor_device * ptdev,u32 as_nr)630 static int panthor_mmu_as_disable(struct panthor_device *ptdev, u32 as_nr)
631 {
632 int ret;
633
634 ret = mmu_hw_do_operation_locked(ptdev, as_nr, 0, ~0ULL, AS_COMMAND_FLUSH_MEM);
635 if (ret)
636 return ret;
637
638 gpu_write(ptdev, AS_TRANSTAB_LO(as_nr), 0);
639 gpu_write(ptdev, AS_TRANSTAB_HI(as_nr), 0);
640
641 gpu_write(ptdev, AS_MEMATTR_LO(as_nr), 0);
642 gpu_write(ptdev, AS_MEMATTR_HI(as_nr), 0);
643
644 gpu_write(ptdev, AS_TRANSCFG_LO(as_nr), AS_TRANSCFG_ADRMODE_UNMAPPED);
645 gpu_write(ptdev, AS_TRANSCFG_HI(as_nr), 0);
646
647 return write_cmd(ptdev, as_nr, AS_COMMAND_UPDATE);
648 }
649
panthor_mmu_fault_mask(struct panthor_device * ptdev,u32 value)650 static u32 panthor_mmu_fault_mask(struct panthor_device *ptdev, u32 value)
651 {
652 /* Bits 16 to 31 mean REQ_COMPLETE. */
653 return value & GENMASK(15, 0);
654 }
655
panthor_mmu_as_fault_mask(struct panthor_device * ptdev,u32 as)656 static u32 panthor_mmu_as_fault_mask(struct panthor_device *ptdev, u32 as)
657 {
658 return BIT(as);
659 }
660
661 /**
662 * panthor_vm_has_unhandled_faults() - Check if a VM has unhandled faults
663 * @vm: VM to check.
664 *
665 * Return: true if the VM has unhandled faults, false otherwise.
666 */
panthor_vm_has_unhandled_faults(struct panthor_vm * vm)667 bool panthor_vm_has_unhandled_faults(struct panthor_vm *vm)
668 {
669 return vm->unhandled_fault;
670 }
671
672 /**
673 * panthor_vm_is_unusable() - Check if the VM is still usable
674 * @vm: VM to check.
675 *
676 * Return: true if the VM is unusable, false otherwise.
677 */
panthor_vm_is_unusable(struct panthor_vm * vm)678 bool panthor_vm_is_unusable(struct panthor_vm *vm)
679 {
680 return vm->unusable;
681 }
682
panthor_vm_release_as_locked(struct panthor_vm * vm)683 static void panthor_vm_release_as_locked(struct panthor_vm *vm)
684 {
685 struct panthor_device *ptdev = vm->ptdev;
686
687 lockdep_assert_held(&ptdev->mmu->as.slots_lock);
688
689 if (drm_WARN_ON(&ptdev->base, vm->as.id < 0))
690 return;
691
692 ptdev->mmu->as.slots[vm->as.id].vm = NULL;
693 clear_bit(vm->as.id, &ptdev->mmu->as.alloc_mask);
694 refcount_set(&vm->as.active_cnt, 0);
695 list_del_init(&vm->as.lru_node);
696 vm->as.id = -1;
697 }
698
699 /**
700 * panthor_vm_active() - Flag a VM as active
701 * @vm: VM to flag as active.
702 *
703 * Assigns an address space to a VM so it can be used by the GPU/MCU.
704 *
705 * Return: 0 on success, a negative error code otherwise.
706 */
panthor_vm_active(struct panthor_vm * vm)707 int panthor_vm_active(struct panthor_vm *vm)
708 {
709 struct panthor_device *ptdev = vm->ptdev;
710 u32 va_bits = GPU_MMU_FEATURES_VA_BITS(ptdev->gpu_info.mmu_features);
711 struct io_pgtable_cfg *cfg = &io_pgtable_ops_to_pgtable(vm->pgtbl_ops)->cfg;
712 int ret = 0, as, cookie;
713 u64 transtab, transcfg;
714
715 if (!drm_dev_enter(&ptdev->base, &cookie))
716 return -ENODEV;
717
718 if (refcount_inc_not_zero(&vm->as.active_cnt))
719 goto out_dev_exit;
720
721 mutex_lock(&ptdev->mmu->as.slots_lock);
722
723 if (refcount_inc_not_zero(&vm->as.active_cnt))
724 goto out_unlock;
725
726 as = vm->as.id;
727 if (as >= 0) {
728 /* Unhandled pagefault on this AS, the MMU was disabled. We need to
729 * re-enable the MMU after clearing+unmasking the AS interrupts.
730 */
731 if (ptdev->mmu->as.faulty_mask & panthor_mmu_as_fault_mask(ptdev, as))
732 goto out_enable_as;
733
734 goto out_make_active;
735 }
736
737 /* Check for a free AS */
738 if (vm->for_mcu) {
739 drm_WARN_ON(&ptdev->base, ptdev->mmu->as.alloc_mask & BIT(0));
740 as = 0;
741 } else {
742 as = ffz(ptdev->mmu->as.alloc_mask | BIT(0));
743 }
744
745 if (!(BIT(as) & ptdev->gpu_info.as_present)) {
746 struct panthor_vm *lru_vm;
747
748 lru_vm = list_first_entry_or_null(&ptdev->mmu->as.lru_list,
749 struct panthor_vm,
750 as.lru_node);
751 if (drm_WARN_ON(&ptdev->base, !lru_vm)) {
752 ret = -EBUSY;
753 goto out_unlock;
754 }
755
756 drm_WARN_ON(&ptdev->base, refcount_read(&lru_vm->as.active_cnt));
757 as = lru_vm->as.id;
758 panthor_vm_release_as_locked(lru_vm);
759 }
760
761 /* Assign the free or reclaimed AS to the FD */
762 vm->as.id = as;
763 set_bit(as, &ptdev->mmu->as.alloc_mask);
764 ptdev->mmu->as.slots[as].vm = vm;
765
766 out_enable_as:
767 transtab = cfg->arm_lpae_s1_cfg.ttbr;
768 transcfg = AS_TRANSCFG_PTW_MEMATTR_WB |
769 AS_TRANSCFG_PTW_RA |
770 AS_TRANSCFG_ADRMODE_AARCH64_4K |
771 AS_TRANSCFG_INA_BITS(55 - va_bits);
772 if (ptdev->coherent)
773 transcfg |= AS_TRANSCFG_PTW_SH_OS;
774
775 /* If the VM is re-activated, we clear the fault. */
776 vm->unhandled_fault = false;
777
778 /* Unhandled pagefault on this AS, clear the fault and re-enable interrupts
779 * before enabling the AS.
780 */
781 if (ptdev->mmu->as.faulty_mask & panthor_mmu_as_fault_mask(ptdev, as)) {
782 gpu_write(ptdev, MMU_INT_CLEAR, panthor_mmu_as_fault_mask(ptdev, as));
783 ptdev->mmu->as.faulty_mask &= ~panthor_mmu_as_fault_mask(ptdev, as);
784 gpu_write(ptdev, MMU_INT_MASK, ~ptdev->mmu->as.faulty_mask);
785 }
786
787 ret = panthor_mmu_as_enable(vm->ptdev, vm->as.id, transtab, transcfg, vm->memattr);
788
789 out_make_active:
790 if (!ret) {
791 refcount_set(&vm->as.active_cnt, 1);
792 list_del_init(&vm->as.lru_node);
793 }
794
795 out_unlock:
796 mutex_unlock(&ptdev->mmu->as.slots_lock);
797
798 out_dev_exit:
799 drm_dev_exit(cookie);
800 return ret;
801 }
802
803 /**
804 * panthor_vm_idle() - Flag a VM idle
805 * @vm: VM to flag as idle.
806 *
807 * When we know the GPU is done with the VM (no more jobs to process),
808 * we can relinquish the AS slot attached to this VM, if any.
809 *
810 * We don't release the slot immediately, but instead place the VM in
811 * the LRU list, so it can be evicted if another VM needs an AS slot.
812 * This way, VMs keep attached to the AS they were given until we run
813 * out of free slot, limiting the number of MMU operations (TLB flush
814 * and other AS updates).
815 */
panthor_vm_idle(struct panthor_vm * vm)816 void panthor_vm_idle(struct panthor_vm *vm)
817 {
818 struct panthor_device *ptdev = vm->ptdev;
819
820 if (!refcount_dec_and_mutex_lock(&vm->as.active_cnt, &ptdev->mmu->as.slots_lock))
821 return;
822
823 if (!drm_WARN_ON(&ptdev->base, vm->as.id == -1 || !list_empty(&vm->as.lru_node)))
824 list_add_tail(&vm->as.lru_node, &ptdev->mmu->as.lru_list);
825
826 refcount_set(&vm->as.active_cnt, 0);
827 mutex_unlock(&ptdev->mmu->as.slots_lock);
828 }
829
panthor_vm_page_size(struct panthor_vm * vm)830 u32 panthor_vm_page_size(struct panthor_vm *vm)
831 {
832 const struct io_pgtable *pgt = io_pgtable_ops_to_pgtable(vm->pgtbl_ops);
833 u32 pg_shift = ffs(pgt->cfg.pgsize_bitmap) - 1;
834
835 return 1u << pg_shift;
836 }
837
panthor_vm_stop(struct panthor_vm * vm)838 static void panthor_vm_stop(struct panthor_vm *vm)
839 {
840 drm_sched_stop(&vm->sched, NULL);
841 }
842
panthor_vm_start(struct panthor_vm * vm)843 static void panthor_vm_start(struct panthor_vm *vm)
844 {
845 drm_sched_start(&vm->sched, 0);
846 }
847
848 /**
849 * panthor_vm_as() - Get the AS slot attached to a VM
850 * @vm: VM to get the AS slot of.
851 *
852 * Return: -1 if the VM is not assigned an AS slot yet, >= 0 otherwise.
853 */
panthor_vm_as(struct panthor_vm * vm)854 int panthor_vm_as(struct panthor_vm *vm)
855 {
856 return vm->as.id;
857 }
858
get_pgsize(u64 addr,size_t size,size_t * count)859 static size_t get_pgsize(u64 addr, size_t size, size_t *count)
860 {
861 /*
862 * io-pgtable only operates on multiple pages within a single table
863 * entry, so we need to split at boundaries of the table size, i.e.
864 * the next block size up. The distance from address A to the next
865 * boundary of block size B is logically B - A % B, but in unsigned
866 * two's complement where B is a power of two we get the equivalence
867 * B - A % B == (B - A) % B == (n * B - A) % B, and choose n = 0 :)
868 */
869 size_t blk_offset = -addr % SZ_2M;
870
871 if (blk_offset || size < SZ_2M) {
872 *count = min_not_zero(blk_offset, size) / SZ_4K;
873 return SZ_4K;
874 }
875 blk_offset = -addr % SZ_1G ?: SZ_1G;
876 *count = min(blk_offset, size) / SZ_2M;
877 return SZ_2M;
878 }
879
panthor_vm_flush_range(struct panthor_vm * vm,u64 iova,u64 size)880 static int panthor_vm_flush_range(struct panthor_vm *vm, u64 iova, u64 size)
881 {
882 struct panthor_device *ptdev = vm->ptdev;
883 int ret = 0, cookie;
884
885 if (vm->as.id < 0)
886 return 0;
887
888 /* If the device is unplugged, we just silently skip the flush. */
889 if (!drm_dev_enter(&ptdev->base, &cookie))
890 return 0;
891
892 ret = mmu_hw_do_operation(vm, iova, size, AS_COMMAND_FLUSH_PT);
893
894 drm_dev_exit(cookie);
895 return ret;
896 }
897
898 /**
899 * panthor_vm_flush_all() - Flush L2 caches for the entirety of a VM's AS
900 * @vm: VM whose cache to flush
901 *
902 * Return: 0 on success, a negative error code if flush failed.
903 */
panthor_vm_flush_all(struct panthor_vm * vm)904 int panthor_vm_flush_all(struct panthor_vm *vm)
905 {
906 return panthor_vm_flush_range(vm, vm->base.mm_start, vm->base.mm_range);
907 }
908
panthor_vm_unmap_pages(struct panthor_vm * vm,u64 iova,u64 size)909 static int panthor_vm_unmap_pages(struct panthor_vm *vm, u64 iova, u64 size)
910 {
911 struct panthor_device *ptdev = vm->ptdev;
912 struct io_pgtable_ops *ops = vm->pgtbl_ops;
913 u64 offset = 0;
914
915 drm_dbg(&ptdev->base, "unmap: as=%d, iova=%llx, len=%llx", vm->as.id, iova, size);
916
917 while (offset < size) {
918 size_t unmapped_sz = 0, pgcount;
919 size_t pgsize = get_pgsize(iova + offset, size - offset, &pgcount);
920
921 unmapped_sz = ops->unmap_pages(ops, iova + offset, pgsize, pgcount, NULL);
922
923 if (drm_WARN_ON(&ptdev->base, unmapped_sz != pgsize * pgcount)) {
924 drm_err(&ptdev->base, "failed to unmap range %llx-%llx (requested range %llx-%llx)\n",
925 iova + offset + unmapped_sz,
926 iova + offset + pgsize * pgcount,
927 iova, iova + size);
928 panthor_vm_flush_range(vm, iova, offset + unmapped_sz);
929 return -EINVAL;
930 }
931 offset += unmapped_sz;
932 }
933
934 return panthor_vm_flush_range(vm, iova, size);
935 }
936
937 static int
panthor_vm_map_pages(struct panthor_vm * vm,u64 iova,int prot,struct sg_table * sgt,u64 offset,u64 size)938 panthor_vm_map_pages(struct panthor_vm *vm, u64 iova, int prot,
939 struct sg_table *sgt, u64 offset, u64 size)
940 {
941 struct panthor_device *ptdev = vm->ptdev;
942 unsigned int count;
943 struct scatterlist *sgl;
944 struct io_pgtable_ops *ops = vm->pgtbl_ops;
945 u64 start_iova = iova;
946 int ret;
947
948 if (!size)
949 return 0;
950
951 for_each_sgtable_dma_sg(sgt, sgl, count) {
952 dma_addr_t paddr = sg_dma_address(sgl);
953 size_t len = sg_dma_len(sgl);
954
955 if (len <= offset) {
956 offset -= len;
957 continue;
958 }
959
960 paddr += offset;
961 len -= offset;
962 len = min_t(size_t, len, size);
963 size -= len;
964
965 drm_dbg(&ptdev->base, "map: as=%d, iova=%llx, paddr=%pad, len=%zx",
966 vm->as.id, iova, &paddr, len);
967
968 while (len) {
969 size_t pgcount, mapped = 0;
970 size_t pgsize = get_pgsize(iova | paddr, len, &pgcount);
971
972 ret = ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot,
973 GFP_KERNEL, &mapped);
974 iova += mapped;
975 paddr += mapped;
976 len -= mapped;
977
978 if (drm_WARN_ON(&ptdev->base, !ret && !mapped))
979 ret = -ENOMEM;
980
981 if (ret) {
982 /* If something failed, unmap what we've already mapped before
983 * returning. The unmap call is not supposed to fail.
984 */
985 drm_WARN_ON(&ptdev->base,
986 panthor_vm_unmap_pages(vm, start_iova,
987 iova - start_iova));
988 return ret;
989 }
990 }
991
992 if (!size)
993 break;
994
995 offset = 0;
996 }
997
998 return panthor_vm_flush_range(vm, start_iova, iova - start_iova);
999 }
1000
flags_to_prot(u32 flags)1001 static int flags_to_prot(u32 flags)
1002 {
1003 int prot = 0;
1004
1005 if (flags & DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC)
1006 prot |= IOMMU_NOEXEC;
1007
1008 if (!(flags & DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED))
1009 prot |= IOMMU_CACHE;
1010
1011 if (flags & DRM_PANTHOR_VM_BIND_OP_MAP_READONLY)
1012 prot |= IOMMU_READ;
1013 else
1014 prot |= IOMMU_READ | IOMMU_WRITE;
1015
1016 return prot;
1017 }
1018
1019 /**
1020 * panthor_vm_alloc_va() - Allocate a region in the auto-va space
1021 * @vm: VM to allocate a region on.
1022 * @va: start of the VA range. Can be PANTHOR_VM_KERNEL_AUTO_VA if the user
1023 * wants the VA to be automatically allocated from the auto-VA range.
1024 * @size: size of the VA range.
1025 * @va_node: drm_mm_node to initialize. Must be zero-initialized.
1026 *
1027 * Some GPU objects, like heap chunks, are fully managed by the kernel and
1028 * need to be mapped to the userspace VM, in the region reserved for kernel
1029 * objects.
1030 *
1031 * This function takes care of allocating a region in the kernel auto-VA space.
1032 *
1033 * Return: 0 on success, an error code otherwise.
1034 */
1035 int
panthor_vm_alloc_va(struct panthor_vm * vm,u64 va,u64 size,struct drm_mm_node * va_node)1036 panthor_vm_alloc_va(struct panthor_vm *vm, u64 va, u64 size,
1037 struct drm_mm_node *va_node)
1038 {
1039 ssize_t vm_pgsz = panthor_vm_page_size(vm);
1040 int ret;
1041
1042 if (!size || !IS_ALIGNED(size, vm_pgsz))
1043 return -EINVAL;
1044
1045 if (va != PANTHOR_VM_KERNEL_AUTO_VA && !IS_ALIGNED(va, vm_pgsz))
1046 return -EINVAL;
1047
1048 mutex_lock(&vm->mm_lock);
1049 if (va != PANTHOR_VM_KERNEL_AUTO_VA) {
1050 va_node->start = va;
1051 va_node->size = size;
1052 ret = drm_mm_reserve_node(&vm->mm, va_node);
1053 } else {
1054 ret = drm_mm_insert_node_in_range(&vm->mm, va_node, size,
1055 size >= SZ_2M ? SZ_2M : SZ_4K,
1056 0, vm->kernel_auto_va.start,
1057 vm->kernel_auto_va.end,
1058 DRM_MM_INSERT_BEST);
1059 }
1060 mutex_unlock(&vm->mm_lock);
1061
1062 return ret;
1063 }
1064
1065 /**
1066 * panthor_vm_free_va() - Free a region allocated with panthor_vm_alloc_va()
1067 * @vm: VM to free the region on.
1068 * @va_node: Memory node representing the region to free.
1069 */
panthor_vm_free_va(struct panthor_vm * vm,struct drm_mm_node * va_node)1070 void panthor_vm_free_va(struct panthor_vm *vm, struct drm_mm_node *va_node)
1071 {
1072 mutex_lock(&vm->mm_lock);
1073 drm_mm_remove_node(va_node);
1074 mutex_unlock(&vm->mm_lock);
1075 }
1076
panthor_vm_bo_put(struct drm_gpuvm_bo * vm_bo)1077 static void panthor_vm_bo_put(struct drm_gpuvm_bo *vm_bo)
1078 {
1079 struct panthor_gem_object *bo = to_panthor_bo(vm_bo->obj);
1080 struct drm_gpuvm *vm = vm_bo->vm;
1081 bool unpin;
1082
1083 /* We must retain the GEM before calling drm_gpuvm_bo_put(),
1084 * otherwise the mutex might be destroyed while we hold it.
1085 * Same goes for the VM, since we take the VM resv lock.
1086 */
1087 drm_gem_object_get(&bo->base.base);
1088 drm_gpuvm_get(vm);
1089
1090 /* We take the resv lock to protect against concurrent accesses to the
1091 * gpuvm evicted/extobj lists that are modified in
1092 * drm_gpuvm_bo_destroy(), which is called if drm_gpuvm_bo_put()
1093 * releases sthe last vm_bo reference.
1094 * We take the BO GPUVA list lock to protect the vm_bo removal from the
1095 * GEM vm_bo list.
1096 */
1097 dma_resv_lock(drm_gpuvm_resv(vm), NULL);
1098 mutex_lock(&bo->gpuva_list_lock);
1099 unpin = drm_gpuvm_bo_put(vm_bo);
1100 mutex_unlock(&bo->gpuva_list_lock);
1101 dma_resv_unlock(drm_gpuvm_resv(vm));
1102
1103 /* If the vm_bo object was destroyed, release the pin reference that
1104 * was hold by this object.
1105 */
1106 if (unpin && !bo->base.base.import_attach)
1107 drm_gem_shmem_unpin(&bo->base);
1108
1109 drm_gpuvm_put(vm);
1110 drm_gem_object_put(&bo->base.base);
1111 }
1112
panthor_vm_cleanup_op_ctx(struct panthor_vm_op_ctx * op_ctx,struct panthor_vm * vm)1113 static void panthor_vm_cleanup_op_ctx(struct panthor_vm_op_ctx *op_ctx,
1114 struct panthor_vm *vm)
1115 {
1116 struct panthor_vma *vma, *tmp_vma;
1117
1118 u32 remaining_pt_count = op_ctx->rsvd_page_tables.count -
1119 op_ctx->rsvd_page_tables.ptr;
1120
1121 if (remaining_pt_count) {
1122 kmem_cache_free_bulk(pt_cache, remaining_pt_count,
1123 op_ctx->rsvd_page_tables.pages +
1124 op_ctx->rsvd_page_tables.ptr);
1125 }
1126
1127 kfree(op_ctx->rsvd_page_tables.pages);
1128
1129 if (op_ctx->map.vm_bo)
1130 panthor_vm_bo_put(op_ctx->map.vm_bo);
1131
1132 for (u32 i = 0; i < ARRAY_SIZE(op_ctx->preallocated_vmas); i++)
1133 kfree(op_ctx->preallocated_vmas[i]);
1134
1135 list_for_each_entry_safe(vma, tmp_vma, &op_ctx->returned_vmas, node) {
1136 list_del(&vma->node);
1137 panthor_vm_bo_put(vma->base.vm_bo);
1138 kfree(vma);
1139 }
1140 }
1141
1142 static struct panthor_vma *
panthor_vm_op_ctx_get_vma(struct panthor_vm_op_ctx * op_ctx)1143 panthor_vm_op_ctx_get_vma(struct panthor_vm_op_ctx *op_ctx)
1144 {
1145 for (u32 i = 0; i < ARRAY_SIZE(op_ctx->preallocated_vmas); i++) {
1146 struct panthor_vma *vma = op_ctx->preallocated_vmas[i];
1147
1148 if (vma) {
1149 op_ctx->preallocated_vmas[i] = NULL;
1150 return vma;
1151 }
1152 }
1153
1154 return NULL;
1155 }
1156
1157 static int
panthor_vm_op_ctx_prealloc_vmas(struct panthor_vm_op_ctx * op_ctx)1158 panthor_vm_op_ctx_prealloc_vmas(struct panthor_vm_op_ctx *op_ctx)
1159 {
1160 u32 vma_count;
1161
1162 switch (op_ctx->flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) {
1163 case DRM_PANTHOR_VM_BIND_OP_TYPE_MAP:
1164 /* One VMA for the new mapping, and two more VMAs for the remap case
1165 * which might contain both a prev and next VA.
1166 */
1167 vma_count = 3;
1168 break;
1169
1170 case DRM_PANTHOR_VM_BIND_OP_TYPE_UNMAP:
1171 /* Partial unmaps might trigger a remap with either a prev or a next VA,
1172 * but not both.
1173 */
1174 vma_count = 1;
1175 break;
1176
1177 default:
1178 return 0;
1179 }
1180
1181 for (u32 i = 0; i < vma_count; i++) {
1182 struct panthor_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL);
1183
1184 if (!vma)
1185 return -ENOMEM;
1186
1187 op_ctx->preallocated_vmas[i] = vma;
1188 }
1189
1190 return 0;
1191 }
1192
1193 #define PANTHOR_VM_BIND_OP_MAP_FLAGS \
1194 (DRM_PANTHOR_VM_BIND_OP_MAP_READONLY | \
1195 DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC | \
1196 DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED | \
1197 DRM_PANTHOR_VM_BIND_OP_TYPE_MASK)
1198
panthor_vm_prepare_map_op_ctx(struct panthor_vm_op_ctx * op_ctx,struct panthor_vm * vm,struct panthor_gem_object * bo,u64 offset,u64 size,u64 va,u32 flags)1199 static int panthor_vm_prepare_map_op_ctx(struct panthor_vm_op_ctx *op_ctx,
1200 struct panthor_vm *vm,
1201 struct panthor_gem_object *bo,
1202 u64 offset,
1203 u64 size, u64 va,
1204 u32 flags)
1205 {
1206 struct drm_gpuvm_bo *preallocated_vm_bo;
1207 struct sg_table *sgt = NULL;
1208 u64 pt_count;
1209 int ret;
1210
1211 if (!bo)
1212 return -EINVAL;
1213
1214 if ((flags & ~PANTHOR_VM_BIND_OP_MAP_FLAGS) ||
1215 (flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) != DRM_PANTHOR_VM_BIND_OP_TYPE_MAP)
1216 return -EINVAL;
1217
1218 /* Make sure the VA and size are aligned and in-bounds. */
1219 if (size > bo->base.base.size || offset > bo->base.base.size - size)
1220 return -EINVAL;
1221
1222 /* If the BO has an exclusive VM attached, it can't be mapped to other VMs. */
1223 if (bo->exclusive_vm_root_gem &&
1224 bo->exclusive_vm_root_gem != panthor_vm_root_gem(vm))
1225 return -EINVAL;
1226
1227 memset(op_ctx, 0, sizeof(*op_ctx));
1228 INIT_LIST_HEAD(&op_ctx->returned_vmas);
1229 op_ctx->flags = flags;
1230 op_ctx->va.range = size;
1231 op_ctx->va.addr = va;
1232
1233 ret = panthor_vm_op_ctx_prealloc_vmas(op_ctx);
1234 if (ret)
1235 goto err_cleanup;
1236
1237 if (!bo->base.base.import_attach) {
1238 /* Pre-reserve the BO pages, so the map operation doesn't have to
1239 * allocate.
1240 */
1241 ret = drm_gem_shmem_pin(&bo->base);
1242 if (ret)
1243 goto err_cleanup;
1244 }
1245
1246 sgt = drm_gem_shmem_get_pages_sgt(&bo->base);
1247 if (IS_ERR(sgt)) {
1248 if (!bo->base.base.import_attach)
1249 drm_gem_shmem_unpin(&bo->base);
1250
1251 ret = PTR_ERR(sgt);
1252 goto err_cleanup;
1253 }
1254
1255 op_ctx->map.sgt = sgt;
1256
1257 preallocated_vm_bo = drm_gpuvm_bo_create(&vm->base, &bo->base.base);
1258 if (!preallocated_vm_bo) {
1259 if (!bo->base.base.import_attach)
1260 drm_gem_shmem_unpin(&bo->base);
1261
1262 ret = -ENOMEM;
1263 goto err_cleanup;
1264 }
1265
1266 /* drm_gpuvm_bo_obtain_prealloc() will call drm_gpuvm_bo_put() on our
1267 * pre-allocated BO if the <BO,VM> association exists. Given we
1268 * only have one ref on preallocated_vm_bo, drm_gpuvm_bo_destroy() will
1269 * be called immediately, and we have to hold the VM resv lock when
1270 * calling this function.
1271 */
1272 dma_resv_lock(panthor_vm_resv(vm), NULL);
1273 mutex_lock(&bo->gpuva_list_lock);
1274 op_ctx->map.vm_bo = drm_gpuvm_bo_obtain_prealloc(preallocated_vm_bo);
1275 mutex_unlock(&bo->gpuva_list_lock);
1276 dma_resv_unlock(panthor_vm_resv(vm));
1277
1278 /* If the a vm_bo for this <VM,BO> combination exists, it already
1279 * retains a pin ref, and we can release the one we took earlier.
1280 *
1281 * If our pre-allocated vm_bo is picked, it now retains the pin ref,
1282 * which will be released in panthor_vm_bo_put().
1283 */
1284 if (preallocated_vm_bo != op_ctx->map.vm_bo &&
1285 !bo->base.base.import_attach)
1286 drm_gem_shmem_unpin(&bo->base);
1287
1288 op_ctx->map.bo_offset = offset;
1289
1290 /* L1, L2 and L3 page tables.
1291 * We could optimize L3 allocation by iterating over the sgt and merging
1292 * 2M contiguous blocks, but it's simpler to over-provision and return
1293 * the pages if they're not used.
1294 */
1295 pt_count = ((ALIGN(va + size, 1ull << 39) - ALIGN_DOWN(va, 1ull << 39)) >> 39) +
1296 ((ALIGN(va + size, 1ull << 30) - ALIGN_DOWN(va, 1ull << 30)) >> 30) +
1297 ((ALIGN(va + size, 1ull << 21) - ALIGN_DOWN(va, 1ull << 21)) >> 21);
1298
1299 op_ctx->rsvd_page_tables.pages = kcalloc(pt_count,
1300 sizeof(*op_ctx->rsvd_page_tables.pages),
1301 GFP_KERNEL);
1302 if (!op_ctx->rsvd_page_tables.pages) {
1303 ret = -ENOMEM;
1304 goto err_cleanup;
1305 }
1306
1307 ret = kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, pt_count,
1308 op_ctx->rsvd_page_tables.pages);
1309 op_ctx->rsvd_page_tables.count = ret;
1310 if (ret != pt_count) {
1311 ret = -ENOMEM;
1312 goto err_cleanup;
1313 }
1314
1315 /* Insert BO into the extobj list last, when we know nothing can fail. */
1316 dma_resv_lock(panthor_vm_resv(vm), NULL);
1317 drm_gpuvm_bo_extobj_add(op_ctx->map.vm_bo);
1318 dma_resv_unlock(panthor_vm_resv(vm));
1319
1320 return 0;
1321
1322 err_cleanup:
1323 panthor_vm_cleanup_op_ctx(op_ctx, vm);
1324 return ret;
1325 }
1326
panthor_vm_prepare_unmap_op_ctx(struct panthor_vm_op_ctx * op_ctx,struct panthor_vm * vm,u64 va,u64 size)1327 static int panthor_vm_prepare_unmap_op_ctx(struct panthor_vm_op_ctx *op_ctx,
1328 struct panthor_vm *vm,
1329 u64 va, u64 size)
1330 {
1331 u32 pt_count = 0;
1332 int ret;
1333
1334 memset(op_ctx, 0, sizeof(*op_ctx));
1335 INIT_LIST_HEAD(&op_ctx->returned_vmas);
1336 op_ctx->va.range = size;
1337 op_ctx->va.addr = va;
1338 op_ctx->flags = DRM_PANTHOR_VM_BIND_OP_TYPE_UNMAP;
1339
1340 /* Pre-allocate L3 page tables to account for the split-2M-block
1341 * situation on unmap.
1342 */
1343 if (va != ALIGN(va, SZ_2M))
1344 pt_count++;
1345
1346 if (va + size != ALIGN(va + size, SZ_2M) &&
1347 ALIGN(va + size, SZ_2M) != ALIGN(va, SZ_2M))
1348 pt_count++;
1349
1350 ret = panthor_vm_op_ctx_prealloc_vmas(op_ctx);
1351 if (ret)
1352 goto err_cleanup;
1353
1354 if (pt_count) {
1355 op_ctx->rsvd_page_tables.pages = kcalloc(pt_count,
1356 sizeof(*op_ctx->rsvd_page_tables.pages),
1357 GFP_KERNEL);
1358 if (!op_ctx->rsvd_page_tables.pages) {
1359 ret = -ENOMEM;
1360 goto err_cleanup;
1361 }
1362
1363 ret = kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, pt_count,
1364 op_ctx->rsvd_page_tables.pages);
1365 if (ret != pt_count) {
1366 ret = -ENOMEM;
1367 goto err_cleanup;
1368 }
1369 op_ctx->rsvd_page_tables.count = pt_count;
1370 }
1371
1372 return 0;
1373
1374 err_cleanup:
1375 panthor_vm_cleanup_op_ctx(op_ctx, vm);
1376 return ret;
1377 }
1378
panthor_vm_prepare_sync_only_op_ctx(struct panthor_vm_op_ctx * op_ctx,struct panthor_vm * vm)1379 static void panthor_vm_prepare_sync_only_op_ctx(struct panthor_vm_op_ctx *op_ctx,
1380 struct panthor_vm *vm)
1381 {
1382 memset(op_ctx, 0, sizeof(*op_ctx));
1383 INIT_LIST_HEAD(&op_ctx->returned_vmas);
1384 op_ctx->flags = DRM_PANTHOR_VM_BIND_OP_TYPE_SYNC_ONLY;
1385 }
1386
1387 /**
1388 * panthor_vm_get_bo_for_va() - Get the GEM object mapped at a virtual address
1389 * @vm: VM to look into.
1390 * @va: Virtual address to search for.
1391 * @bo_offset: Offset of the GEM object mapped at this virtual address.
1392 * Only valid on success.
1393 *
1394 * The object returned by this function might no longer be mapped when the
1395 * function returns. It's the caller responsibility to ensure there's no
1396 * concurrent map/unmap operations making the returned value invalid, or
1397 * make sure it doesn't matter if the object is no longer mapped.
1398 *
1399 * Return: A valid pointer on success, an ERR_PTR() otherwise.
1400 */
1401 struct panthor_gem_object *
panthor_vm_get_bo_for_va(struct panthor_vm * vm,u64 va,u64 * bo_offset)1402 panthor_vm_get_bo_for_va(struct panthor_vm *vm, u64 va, u64 *bo_offset)
1403 {
1404 struct panthor_gem_object *bo = ERR_PTR(-ENOENT);
1405 struct drm_gpuva *gpuva;
1406 struct panthor_vma *vma;
1407
1408 /* Take the VM lock to prevent concurrent map/unmap operations. */
1409 mutex_lock(&vm->op_lock);
1410 gpuva = drm_gpuva_find_first(&vm->base, va, 1);
1411 vma = gpuva ? container_of(gpuva, struct panthor_vma, base) : NULL;
1412 if (vma && vma->base.gem.obj) {
1413 drm_gem_object_get(vma->base.gem.obj);
1414 bo = to_panthor_bo(vma->base.gem.obj);
1415 *bo_offset = vma->base.gem.offset + (va - vma->base.va.addr);
1416 }
1417 mutex_unlock(&vm->op_lock);
1418
1419 return bo;
1420 }
1421
1422 #define PANTHOR_VM_MIN_KERNEL_VA_SIZE SZ_256M
1423
1424 static u64
panthor_vm_create_get_user_va_range(const struct drm_panthor_vm_create * args,u64 full_va_range)1425 panthor_vm_create_get_user_va_range(const struct drm_panthor_vm_create *args,
1426 u64 full_va_range)
1427 {
1428 u64 user_va_range;
1429
1430 /* Make sure we have a minimum amount of VA space for kernel objects. */
1431 if (full_va_range < PANTHOR_VM_MIN_KERNEL_VA_SIZE)
1432 return 0;
1433
1434 if (args->user_va_range) {
1435 /* Use the user provided value if != 0. */
1436 user_va_range = args->user_va_range;
1437 } else if (TASK_SIZE_OF(current) < full_va_range) {
1438 /* If the task VM size is smaller than the GPU VA range, pick this
1439 * as our default user VA range, so userspace can CPU/GPU map buffers
1440 * at the same address.
1441 */
1442 user_va_range = TASK_SIZE_OF(current);
1443 } else {
1444 /* If the GPU VA range is smaller than the task VM size, we
1445 * just have to live with the fact we won't be able to map
1446 * all buffers at the same GPU/CPU address.
1447 *
1448 * If the GPU VA range is bigger than 4G (more than 32-bit of
1449 * VA), we split the range in two, and assign half of it to
1450 * the user and the other half to the kernel, if it's not, we
1451 * keep the kernel VA space as small as possible.
1452 */
1453 user_va_range = full_va_range > SZ_4G ?
1454 full_va_range / 2 :
1455 full_va_range - PANTHOR_VM_MIN_KERNEL_VA_SIZE;
1456 }
1457
1458 if (full_va_range - PANTHOR_VM_MIN_KERNEL_VA_SIZE < user_va_range)
1459 user_va_range = full_va_range - PANTHOR_VM_MIN_KERNEL_VA_SIZE;
1460
1461 return user_va_range;
1462 }
1463
1464 #define PANTHOR_VM_CREATE_FLAGS 0
1465
1466 static int
panthor_vm_create_check_args(const struct panthor_device * ptdev,const struct drm_panthor_vm_create * args,u64 * kernel_va_start,u64 * kernel_va_range)1467 panthor_vm_create_check_args(const struct panthor_device *ptdev,
1468 const struct drm_panthor_vm_create *args,
1469 u64 *kernel_va_start, u64 *kernel_va_range)
1470 {
1471 u32 va_bits = GPU_MMU_FEATURES_VA_BITS(ptdev->gpu_info.mmu_features);
1472 u64 full_va_range = 1ull << va_bits;
1473 u64 user_va_range;
1474
1475 if (args->flags & ~PANTHOR_VM_CREATE_FLAGS)
1476 return -EINVAL;
1477
1478 user_va_range = panthor_vm_create_get_user_va_range(args, full_va_range);
1479 if (!user_va_range || (args->user_va_range && args->user_va_range > user_va_range))
1480 return -EINVAL;
1481
1482 /* Pick a kernel VA range that's a power of two, to have a clear split. */
1483 *kernel_va_range = rounddown_pow_of_two(full_va_range - user_va_range);
1484 *kernel_va_start = full_va_range - *kernel_va_range;
1485 return 0;
1486 }
1487
1488 /*
1489 * Only 32 VMs per open file. If that becomes a limiting factor, we can
1490 * increase this number.
1491 */
1492 #define PANTHOR_MAX_VMS_PER_FILE 32
1493
1494 /**
1495 * panthor_vm_pool_create_vm() - Create a VM
1496 * @ptdev: The panthor device
1497 * @pool: The VM to create this VM on.
1498 * @args: VM creation args.
1499 *
1500 * Return: a positive VM ID on success, a negative error code otherwise.
1501 */
panthor_vm_pool_create_vm(struct panthor_device * ptdev,struct panthor_vm_pool * pool,struct drm_panthor_vm_create * args)1502 int panthor_vm_pool_create_vm(struct panthor_device *ptdev,
1503 struct panthor_vm_pool *pool,
1504 struct drm_panthor_vm_create *args)
1505 {
1506 u64 kernel_va_start, kernel_va_range;
1507 struct panthor_vm *vm;
1508 int ret;
1509 u32 id;
1510
1511 ret = panthor_vm_create_check_args(ptdev, args, &kernel_va_start, &kernel_va_range);
1512 if (ret)
1513 return ret;
1514
1515 vm = panthor_vm_create(ptdev, false, kernel_va_start, kernel_va_range,
1516 kernel_va_start, kernel_va_range);
1517 if (IS_ERR(vm))
1518 return PTR_ERR(vm);
1519
1520 ret = xa_alloc(&pool->xa, &id, vm,
1521 XA_LIMIT(1, PANTHOR_MAX_VMS_PER_FILE), GFP_KERNEL);
1522
1523 if (ret) {
1524 panthor_vm_put(vm);
1525 return ret;
1526 }
1527
1528 args->user_va_range = kernel_va_start;
1529 return id;
1530 }
1531
panthor_vm_destroy(struct panthor_vm * vm)1532 static void panthor_vm_destroy(struct panthor_vm *vm)
1533 {
1534 if (!vm)
1535 return;
1536
1537 vm->destroyed = true;
1538
1539 mutex_lock(&vm->heaps.lock);
1540 panthor_heap_pool_destroy(vm->heaps.pool);
1541 vm->heaps.pool = NULL;
1542 mutex_unlock(&vm->heaps.lock);
1543
1544 drm_WARN_ON(&vm->ptdev->base,
1545 panthor_vm_unmap_range(vm, vm->base.mm_start, vm->base.mm_range));
1546 panthor_vm_put(vm);
1547 }
1548
1549 /**
1550 * panthor_vm_pool_destroy_vm() - Destroy a VM.
1551 * @pool: VM pool.
1552 * @handle: VM handle.
1553 *
1554 * This function doesn't free the VM object or its resources, it just kills
1555 * all mappings, and makes sure nothing can be mapped after that point.
1556 *
1557 * If there was any active jobs at the time this function is called, these
1558 * jobs should experience page faults and be killed as a result.
1559 *
1560 * The VM resources are freed when the last reference on the VM object is
1561 * dropped.
1562 *
1563 * Return: %0 for success, negative errno value for failure
1564 */
panthor_vm_pool_destroy_vm(struct panthor_vm_pool * pool,u32 handle)1565 int panthor_vm_pool_destroy_vm(struct panthor_vm_pool *pool, u32 handle)
1566 {
1567 struct panthor_vm *vm;
1568
1569 vm = xa_erase(&pool->xa, handle);
1570
1571 panthor_vm_destroy(vm);
1572
1573 return vm ? 0 : -EINVAL;
1574 }
1575
1576 /**
1577 * panthor_vm_pool_get_vm() - Retrieve VM object bound to a VM handle
1578 * @pool: VM pool to check.
1579 * @handle: Handle of the VM to retrieve.
1580 *
1581 * Return: A valid pointer if the VM exists, NULL otherwise.
1582 */
1583 struct panthor_vm *
panthor_vm_pool_get_vm(struct panthor_vm_pool * pool,u32 handle)1584 panthor_vm_pool_get_vm(struct panthor_vm_pool *pool, u32 handle)
1585 {
1586 struct panthor_vm *vm;
1587
1588 xa_lock(&pool->xa);
1589 vm = panthor_vm_get(xa_load(&pool->xa, handle));
1590 xa_unlock(&pool->xa);
1591
1592 return vm;
1593 }
1594
1595 /**
1596 * panthor_vm_pool_destroy() - Destroy a VM pool.
1597 * @pfile: File.
1598 *
1599 * Destroy all VMs in the pool, and release the pool resources.
1600 *
1601 * Note that VMs can outlive the pool they were created from if other
1602 * objects hold a reference to there VMs.
1603 */
panthor_vm_pool_destroy(struct panthor_file * pfile)1604 void panthor_vm_pool_destroy(struct panthor_file *pfile)
1605 {
1606 struct panthor_vm *vm;
1607 unsigned long i;
1608
1609 if (!pfile->vms)
1610 return;
1611
1612 xa_for_each(&pfile->vms->xa, i, vm)
1613 panthor_vm_destroy(vm);
1614
1615 xa_destroy(&pfile->vms->xa);
1616 kfree(pfile->vms);
1617 }
1618
1619 /**
1620 * panthor_vm_pool_create() - Create a VM pool
1621 * @pfile: File.
1622 *
1623 * Return: 0 on success, a negative error code otherwise.
1624 */
panthor_vm_pool_create(struct panthor_file * pfile)1625 int panthor_vm_pool_create(struct panthor_file *pfile)
1626 {
1627 pfile->vms = kzalloc(sizeof(*pfile->vms), GFP_KERNEL);
1628 if (!pfile->vms)
1629 return -ENOMEM;
1630
1631 xa_init_flags(&pfile->vms->xa, XA_FLAGS_ALLOC1);
1632 return 0;
1633 }
1634
1635 /* dummy TLB ops, the real TLB flush happens in panthor_vm_flush_range() */
mmu_tlb_flush_all(void * cookie)1636 static void mmu_tlb_flush_all(void *cookie)
1637 {
1638 }
1639
mmu_tlb_flush_walk(unsigned long iova,size_t size,size_t granule,void * cookie)1640 static void mmu_tlb_flush_walk(unsigned long iova, size_t size, size_t granule, void *cookie)
1641 {
1642 }
1643
1644 static const struct iommu_flush_ops mmu_tlb_ops = {
1645 .tlb_flush_all = mmu_tlb_flush_all,
1646 .tlb_flush_walk = mmu_tlb_flush_walk,
1647 };
1648
access_type_name(struct panthor_device * ptdev,u32 fault_status)1649 static const char *access_type_name(struct panthor_device *ptdev,
1650 u32 fault_status)
1651 {
1652 switch (fault_status & AS_FAULTSTATUS_ACCESS_TYPE_MASK) {
1653 case AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC:
1654 return "ATOMIC";
1655 case AS_FAULTSTATUS_ACCESS_TYPE_READ:
1656 return "READ";
1657 case AS_FAULTSTATUS_ACCESS_TYPE_WRITE:
1658 return "WRITE";
1659 case AS_FAULTSTATUS_ACCESS_TYPE_EX:
1660 return "EXECUTE";
1661 default:
1662 drm_WARN_ON(&ptdev->base, 1);
1663 return NULL;
1664 }
1665 }
1666
panthor_mmu_irq_handler(struct panthor_device * ptdev,u32 status)1667 static void panthor_mmu_irq_handler(struct panthor_device *ptdev, u32 status)
1668 {
1669 bool has_unhandled_faults = false;
1670
1671 status = panthor_mmu_fault_mask(ptdev, status);
1672 while (status) {
1673 u32 as = ffs(status | (status >> 16)) - 1;
1674 u32 mask = panthor_mmu_as_fault_mask(ptdev, as);
1675 u32 new_int_mask;
1676 u64 addr;
1677 u32 fault_status;
1678 u32 exception_type;
1679 u32 access_type;
1680 u32 source_id;
1681
1682 fault_status = gpu_read(ptdev, AS_FAULTSTATUS(as));
1683 addr = gpu_read(ptdev, AS_FAULTADDRESS_LO(as));
1684 addr |= (u64)gpu_read(ptdev, AS_FAULTADDRESS_HI(as)) << 32;
1685
1686 /* decode the fault status */
1687 exception_type = fault_status & 0xFF;
1688 access_type = (fault_status >> 8) & 0x3;
1689 source_id = (fault_status >> 16);
1690
1691 mutex_lock(&ptdev->mmu->as.slots_lock);
1692
1693 ptdev->mmu->as.faulty_mask |= mask;
1694 new_int_mask =
1695 panthor_mmu_fault_mask(ptdev, ~ptdev->mmu->as.faulty_mask);
1696
1697 /* terminal fault, print info about the fault */
1698 drm_err(&ptdev->base,
1699 "Unhandled Page fault in AS%d at VA 0x%016llX\n"
1700 "raw fault status: 0x%X\n"
1701 "decoded fault status: %s\n"
1702 "exception type 0x%X: %s\n"
1703 "access type 0x%X: %s\n"
1704 "source id 0x%X\n",
1705 as, addr,
1706 fault_status,
1707 (fault_status & (1 << 10) ? "DECODER FAULT" : "SLAVE FAULT"),
1708 exception_type, panthor_exception_name(ptdev, exception_type),
1709 access_type, access_type_name(ptdev, fault_status),
1710 source_id);
1711
1712 /* Ignore MMU interrupts on this AS until it's been
1713 * re-enabled.
1714 */
1715 ptdev->mmu->irq.mask = new_int_mask;
1716 gpu_write(ptdev, MMU_INT_MASK, new_int_mask);
1717
1718 if (ptdev->mmu->as.slots[as].vm)
1719 ptdev->mmu->as.slots[as].vm->unhandled_fault = true;
1720
1721 /* Disable the MMU to kill jobs on this AS. */
1722 panthor_mmu_as_disable(ptdev, as);
1723 mutex_unlock(&ptdev->mmu->as.slots_lock);
1724
1725 status &= ~mask;
1726 has_unhandled_faults = true;
1727 }
1728
1729 if (has_unhandled_faults)
1730 panthor_sched_report_mmu_fault(ptdev);
1731 }
1732 PANTHOR_IRQ_HANDLER(mmu, MMU, panthor_mmu_irq_handler);
1733
1734 /**
1735 * panthor_mmu_suspend() - Suspend the MMU logic
1736 * @ptdev: Device.
1737 *
1738 * All we do here is de-assign the AS slots on all active VMs, so things
1739 * get flushed to the main memory, and no further access to these VMs are
1740 * possible.
1741 *
1742 * We also suspend the MMU IRQ.
1743 */
panthor_mmu_suspend(struct panthor_device * ptdev)1744 void panthor_mmu_suspend(struct panthor_device *ptdev)
1745 {
1746 mutex_lock(&ptdev->mmu->as.slots_lock);
1747 for (u32 i = 0; i < ARRAY_SIZE(ptdev->mmu->as.slots); i++) {
1748 struct panthor_vm *vm = ptdev->mmu->as.slots[i].vm;
1749
1750 if (vm) {
1751 drm_WARN_ON(&ptdev->base, panthor_mmu_as_disable(ptdev, i));
1752 panthor_vm_release_as_locked(vm);
1753 }
1754 }
1755 mutex_unlock(&ptdev->mmu->as.slots_lock);
1756
1757 panthor_mmu_irq_suspend(&ptdev->mmu->irq);
1758 }
1759
1760 /**
1761 * panthor_mmu_resume() - Resume the MMU logic
1762 * @ptdev: Device.
1763 *
1764 * Resume the IRQ.
1765 *
1766 * We don't re-enable previously active VMs. We assume other parts of the
1767 * driver will call panthor_vm_active() on the VMs they intend to use.
1768 */
panthor_mmu_resume(struct panthor_device * ptdev)1769 void panthor_mmu_resume(struct panthor_device *ptdev)
1770 {
1771 mutex_lock(&ptdev->mmu->as.slots_lock);
1772 ptdev->mmu->as.alloc_mask = 0;
1773 ptdev->mmu->as.faulty_mask = 0;
1774 mutex_unlock(&ptdev->mmu->as.slots_lock);
1775
1776 panthor_mmu_irq_resume(&ptdev->mmu->irq, panthor_mmu_fault_mask(ptdev, ~0));
1777 }
1778
1779 /**
1780 * panthor_mmu_pre_reset() - Prepare for a reset
1781 * @ptdev: Device.
1782 *
1783 * Suspend the IRQ, and make sure all VM_BIND queues are stopped, so we
1784 * don't get asked to do a VM operation while the GPU is down.
1785 *
1786 * We don't cleanly shutdown the AS slots here, because the reset might
1787 * come from an AS_ACTIVE_BIT stuck situation.
1788 */
panthor_mmu_pre_reset(struct panthor_device * ptdev)1789 void panthor_mmu_pre_reset(struct panthor_device *ptdev)
1790 {
1791 struct panthor_vm *vm;
1792
1793 panthor_mmu_irq_suspend(&ptdev->mmu->irq);
1794
1795 mutex_lock(&ptdev->mmu->vm.lock);
1796 ptdev->mmu->vm.reset_in_progress = true;
1797 list_for_each_entry(vm, &ptdev->mmu->vm.list, node)
1798 panthor_vm_stop(vm);
1799 mutex_unlock(&ptdev->mmu->vm.lock);
1800 }
1801
1802 /**
1803 * panthor_mmu_post_reset() - Restore things after a reset
1804 * @ptdev: Device.
1805 *
1806 * Put the MMU logic back in action after a reset. That implies resuming the
1807 * IRQ and re-enabling the VM_BIND queues.
1808 */
panthor_mmu_post_reset(struct panthor_device * ptdev)1809 void panthor_mmu_post_reset(struct panthor_device *ptdev)
1810 {
1811 struct panthor_vm *vm;
1812
1813 mutex_lock(&ptdev->mmu->as.slots_lock);
1814
1815 /* Now that the reset is effective, we can assume that none of the
1816 * AS slots are setup, and clear the faulty flags too.
1817 */
1818 ptdev->mmu->as.alloc_mask = 0;
1819 ptdev->mmu->as.faulty_mask = 0;
1820
1821 for (u32 i = 0; i < ARRAY_SIZE(ptdev->mmu->as.slots); i++) {
1822 struct panthor_vm *vm = ptdev->mmu->as.slots[i].vm;
1823
1824 if (vm)
1825 panthor_vm_release_as_locked(vm);
1826 }
1827
1828 mutex_unlock(&ptdev->mmu->as.slots_lock);
1829
1830 panthor_mmu_irq_resume(&ptdev->mmu->irq, panthor_mmu_fault_mask(ptdev, ~0));
1831
1832 /* Restart the VM_BIND queues. */
1833 mutex_lock(&ptdev->mmu->vm.lock);
1834 list_for_each_entry(vm, &ptdev->mmu->vm.list, node) {
1835 panthor_vm_start(vm);
1836 }
1837 ptdev->mmu->vm.reset_in_progress = false;
1838 mutex_unlock(&ptdev->mmu->vm.lock);
1839 }
1840
panthor_vm_free(struct drm_gpuvm * gpuvm)1841 static void panthor_vm_free(struct drm_gpuvm *gpuvm)
1842 {
1843 struct panthor_vm *vm = container_of(gpuvm, struct panthor_vm, base);
1844 struct panthor_device *ptdev = vm->ptdev;
1845
1846 mutex_lock(&vm->heaps.lock);
1847 if (drm_WARN_ON(&ptdev->base, vm->heaps.pool))
1848 panthor_heap_pool_destroy(vm->heaps.pool);
1849 mutex_unlock(&vm->heaps.lock);
1850 mutex_destroy(&vm->heaps.lock);
1851
1852 mutex_lock(&ptdev->mmu->vm.lock);
1853 list_del(&vm->node);
1854 /* Restore the scheduler state so we can call drm_sched_entity_destroy()
1855 * and drm_sched_fini(). If get there, that means we have no job left
1856 * and no new jobs can be queued, so we can start the scheduler without
1857 * risking interfering with the reset.
1858 */
1859 if (ptdev->mmu->vm.reset_in_progress)
1860 panthor_vm_start(vm);
1861 mutex_unlock(&ptdev->mmu->vm.lock);
1862
1863 drm_sched_entity_destroy(&vm->entity);
1864 drm_sched_fini(&vm->sched);
1865
1866 mutex_lock(&ptdev->mmu->as.slots_lock);
1867 if (vm->as.id >= 0) {
1868 int cookie;
1869
1870 if (drm_dev_enter(&ptdev->base, &cookie)) {
1871 panthor_mmu_as_disable(ptdev, vm->as.id);
1872 drm_dev_exit(cookie);
1873 }
1874
1875 ptdev->mmu->as.slots[vm->as.id].vm = NULL;
1876 clear_bit(vm->as.id, &ptdev->mmu->as.alloc_mask);
1877 list_del(&vm->as.lru_node);
1878 }
1879 mutex_unlock(&ptdev->mmu->as.slots_lock);
1880
1881 free_io_pgtable_ops(vm->pgtbl_ops);
1882
1883 drm_mm_takedown(&vm->mm);
1884 kfree(vm);
1885 }
1886
1887 /**
1888 * panthor_vm_put() - Release a reference on a VM
1889 * @vm: VM to release the reference on. Can be NULL.
1890 */
panthor_vm_put(struct panthor_vm * vm)1891 void panthor_vm_put(struct panthor_vm *vm)
1892 {
1893 drm_gpuvm_put(vm ? &vm->base : NULL);
1894 }
1895
1896 /**
1897 * panthor_vm_get() - Get a VM reference
1898 * @vm: VM to get the reference on. Can be NULL.
1899 *
1900 * Return: @vm value.
1901 */
panthor_vm_get(struct panthor_vm * vm)1902 struct panthor_vm *panthor_vm_get(struct panthor_vm *vm)
1903 {
1904 if (vm)
1905 drm_gpuvm_get(&vm->base);
1906
1907 return vm;
1908 }
1909
1910 /**
1911 * panthor_vm_get_heap_pool() - Get the heap pool attached to a VM
1912 * @vm: VM to query the heap pool on.
1913 * @create: True if the heap pool should be created when it doesn't exist.
1914 *
1915 * Heap pools are per-VM. This function allows one to retrieve the heap pool
1916 * attached to a VM.
1917 *
1918 * If no heap pool exists yet, and @create is true, we create one.
1919 *
1920 * The returned panthor_heap_pool should be released with panthor_heap_pool_put().
1921 *
1922 * Return: A valid pointer on success, an ERR_PTR() otherwise.
1923 */
panthor_vm_get_heap_pool(struct panthor_vm * vm,bool create)1924 struct panthor_heap_pool *panthor_vm_get_heap_pool(struct panthor_vm *vm, bool create)
1925 {
1926 struct panthor_heap_pool *pool;
1927
1928 mutex_lock(&vm->heaps.lock);
1929 if (!vm->heaps.pool && create) {
1930 if (vm->destroyed)
1931 pool = ERR_PTR(-EINVAL);
1932 else
1933 pool = panthor_heap_pool_create(vm->ptdev, vm);
1934
1935 if (!IS_ERR(pool))
1936 vm->heaps.pool = panthor_heap_pool_get(pool);
1937 } else {
1938 pool = panthor_heap_pool_get(vm->heaps.pool);
1939 if (!pool)
1940 pool = ERR_PTR(-ENOENT);
1941 }
1942 mutex_unlock(&vm->heaps.lock);
1943
1944 return pool;
1945 }
1946
1947 /**
1948 * panthor_vm_heaps_sizes() - Calculate size of all heap chunks across all
1949 * heaps over all the heap pools in a VM
1950 * @pfile: File.
1951 * @stats: Memory stats to be updated.
1952 *
1953 * Calculate all heap chunk sizes in all heap pools bound to a VM. If the VM
1954 * is active, record the size as active as well.
1955 */
panthor_vm_heaps_sizes(struct panthor_file * pfile,struct drm_memory_stats * stats)1956 void panthor_vm_heaps_sizes(struct panthor_file *pfile, struct drm_memory_stats *stats)
1957 {
1958 struct panthor_vm *vm;
1959 unsigned long i;
1960
1961 if (!pfile->vms)
1962 return;
1963
1964 xa_lock(&pfile->vms->xa);
1965 xa_for_each(&pfile->vms->xa, i, vm) {
1966 size_t size = panthor_heap_pool_size(vm->heaps.pool);
1967 stats->resident += size;
1968 if (vm->as.id >= 0)
1969 stats->active += size;
1970 }
1971 xa_unlock(&pfile->vms->xa);
1972 }
1973
mair_to_memattr(u64 mair,bool coherent)1974 static u64 mair_to_memattr(u64 mair, bool coherent)
1975 {
1976 u64 memattr = 0;
1977 u32 i;
1978
1979 for (i = 0; i < 8; i++) {
1980 u8 in_attr = mair >> (8 * i), out_attr;
1981 u8 outer = in_attr >> 4, inner = in_attr & 0xf;
1982
1983 /* For caching to be enabled, inner and outer caching policy
1984 * have to be both write-back, if one of them is write-through
1985 * or non-cacheable, we just choose non-cacheable. Device
1986 * memory is also translated to non-cacheable.
1987 */
1988 if (!(outer & 3) || !(outer & 4) || !(inner & 4)) {
1989 out_attr = AS_MEMATTR_AARCH64_INNER_OUTER_NC |
1990 AS_MEMATTR_AARCH64_SH_MIDGARD_INNER |
1991 AS_MEMATTR_AARCH64_INNER_ALLOC_EXPL(false, false);
1992 } else {
1993 out_attr = AS_MEMATTR_AARCH64_INNER_OUTER_WB |
1994 AS_MEMATTR_AARCH64_INNER_ALLOC_EXPL(inner & 1, inner & 2);
1995 /* Use SH_MIDGARD_INNER mode when device isn't coherent,
1996 * so SH_IS, which is used when IOMMU_CACHE is set, maps
1997 * to Mali's internal-shareable mode. As per the Mali
1998 * Spec, inner and outer-shareable modes aren't allowed
1999 * for WB memory when coherency is disabled.
2000 * Use SH_CPU_INNER mode when coherency is enabled, so
2001 * that SH_IS actually maps to the standard definition of
2002 * inner-shareable.
2003 */
2004 if (!coherent)
2005 out_attr |= AS_MEMATTR_AARCH64_SH_MIDGARD_INNER;
2006 else
2007 out_attr |= AS_MEMATTR_AARCH64_SH_CPU_INNER;
2008 }
2009
2010 memattr |= (u64)out_attr << (8 * i);
2011 }
2012
2013 return memattr;
2014 }
2015
panthor_vma_link(struct panthor_vm * vm,struct panthor_vma * vma,struct drm_gpuvm_bo * vm_bo)2016 static void panthor_vma_link(struct panthor_vm *vm,
2017 struct panthor_vma *vma,
2018 struct drm_gpuvm_bo *vm_bo)
2019 {
2020 struct panthor_gem_object *bo = to_panthor_bo(vma->base.gem.obj);
2021
2022 mutex_lock(&bo->gpuva_list_lock);
2023 drm_gpuva_link(&vma->base, vm_bo);
2024 drm_WARN_ON(&vm->ptdev->base, drm_gpuvm_bo_put(vm_bo));
2025 mutex_unlock(&bo->gpuva_list_lock);
2026 }
2027
panthor_vma_unlink(struct panthor_vm * vm,struct panthor_vma * vma)2028 static void panthor_vma_unlink(struct panthor_vm *vm,
2029 struct panthor_vma *vma)
2030 {
2031 struct panthor_gem_object *bo = to_panthor_bo(vma->base.gem.obj);
2032 struct drm_gpuvm_bo *vm_bo = drm_gpuvm_bo_get(vma->base.vm_bo);
2033
2034 mutex_lock(&bo->gpuva_list_lock);
2035 drm_gpuva_unlink(&vma->base);
2036 mutex_unlock(&bo->gpuva_list_lock);
2037
2038 /* drm_gpuva_unlink() release the vm_bo, but we manually retained it
2039 * when entering this function, so we can implement deferred VMA
2040 * destruction. Re-assign it here.
2041 */
2042 vma->base.vm_bo = vm_bo;
2043 list_add_tail(&vma->node, &vm->op_ctx->returned_vmas);
2044 }
2045
panthor_vma_init(struct panthor_vma * vma,u32 flags)2046 static void panthor_vma_init(struct panthor_vma *vma, u32 flags)
2047 {
2048 INIT_LIST_HEAD(&vma->node);
2049 vma->flags = flags;
2050 }
2051
2052 #define PANTHOR_VM_MAP_FLAGS \
2053 (DRM_PANTHOR_VM_BIND_OP_MAP_READONLY | \
2054 DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC | \
2055 DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED)
2056
panthor_gpuva_sm_step_map(struct drm_gpuva_op * op,void * priv)2057 static int panthor_gpuva_sm_step_map(struct drm_gpuva_op *op, void *priv)
2058 {
2059 struct panthor_vm *vm = priv;
2060 struct panthor_vm_op_ctx *op_ctx = vm->op_ctx;
2061 struct panthor_vma *vma = panthor_vm_op_ctx_get_vma(op_ctx);
2062 int ret;
2063
2064 if (!vma)
2065 return -EINVAL;
2066
2067 panthor_vma_init(vma, op_ctx->flags & PANTHOR_VM_MAP_FLAGS);
2068
2069 ret = panthor_vm_map_pages(vm, op->map.va.addr, flags_to_prot(vma->flags),
2070 op_ctx->map.sgt, op->map.gem.offset,
2071 op->map.va.range);
2072 if (ret)
2073 return ret;
2074
2075 /* Ref owned by the mapping now, clear the obj field so we don't release the
2076 * pinning/obj ref behind GPUVA's back.
2077 */
2078 drm_gpuva_map(&vm->base, &vma->base, &op->map);
2079 panthor_vma_link(vm, vma, op_ctx->map.vm_bo);
2080 op_ctx->map.vm_bo = NULL;
2081 return 0;
2082 }
2083
panthor_gpuva_sm_step_remap(struct drm_gpuva_op * op,void * priv)2084 static int panthor_gpuva_sm_step_remap(struct drm_gpuva_op *op,
2085 void *priv)
2086 {
2087 struct panthor_vma *unmap_vma = container_of(op->remap.unmap->va, struct panthor_vma, base);
2088 struct panthor_vm *vm = priv;
2089 struct panthor_vm_op_ctx *op_ctx = vm->op_ctx;
2090 struct panthor_vma *prev_vma = NULL, *next_vma = NULL;
2091 u64 unmap_start, unmap_range;
2092 int ret;
2093
2094 drm_gpuva_op_remap_to_unmap_range(&op->remap, &unmap_start, &unmap_range);
2095 ret = panthor_vm_unmap_pages(vm, unmap_start, unmap_range);
2096 if (ret)
2097 return ret;
2098
2099 if (op->remap.prev) {
2100 prev_vma = panthor_vm_op_ctx_get_vma(op_ctx);
2101 panthor_vma_init(prev_vma, unmap_vma->flags);
2102 }
2103
2104 if (op->remap.next) {
2105 next_vma = panthor_vm_op_ctx_get_vma(op_ctx);
2106 panthor_vma_init(next_vma, unmap_vma->flags);
2107 }
2108
2109 drm_gpuva_remap(prev_vma ? &prev_vma->base : NULL,
2110 next_vma ? &next_vma->base : NULL,
2111 &op->remap);
2112
2113 if (prev_vma) {
2114 /* panthor_vma_link() transfers the vm_bo ownership to
2115 * the VMA object. Since the vm_bo we're passing is still
2116 * owned by the old mapping which will be released when this
2117 * mapping is destroyed, we need to grab a ref here.
2118 */
2119 panthor_vma_link(vm, prev_vma,
2120 drm_gpuvm_bo_get(op->remap.unmap->va->vm_bo));
2121 }
2122
2123 if (next_vma) {
2124 panthor_vma_link(vm, next_vma,
2125 drm_gpuvm_bo_get(op->remap.unmap->va->vm_bo));
2126 }
2127
2128 panthor_vma_unlink(vm, unmap_vma);
2129 return 0;
2130 }
2131
panthor_gpuva_sm_step_unmap(struct drm_gpuva_op * op,void * priv)2132 static int panthor_gpuva_sm_step_unmap(struct drm_gpuva_op *op,
2133 void *priv)
2134 {
2135 struct panthor_vma *unmap_vma = container_of(op->unmap.va, struct panthor_vma, base);
2136 struct panthor_vm *vm = priv;
2137 int ret;
2138
2139 ret = panthor_vm_unmap_pages(vm, unmap_vma->base.va.addr,
2140 unmap_vma->base.va.range);
2141 if (drm_WARN_ON(&vm->ptdev->base, ret))
2142 return ret;
2143
2144 drm_gpuva_unmap(&op->unmap);
2145 panthor_vma_unlink(vm, unmap_vma);
2146 return 0;
2147 }
2148
2149 static const struct drm_gpuvm_ops panthor_gpuvm_ops = {
2150 .vm_free = panthor_vm_free,
2151 .sm_step_map = panthor_gpuva_sm_step_map,
2152 .sm_step_remap = panthor_gpuva_sm_step_remap,
2153 .sm_step_unmap = panthor_gpuva_sm_step_unmap,
2154 };
2155
2156 /**
2157 * panthor_vm_resv() - Get the dma_resv object attached to a VM.
2158 * @vm: VM to get the dma_resv of.
2159 *
2160 * Return: A dma_resv object.
2161 */
panthor_vm_resv(struct panthor_vm * vm)2162 struct dma_resv *panthor_vm_resv(struct panthor_vm *vm)
2163 {
2164 return drm_gpuvm_resv(&vm->base);
2165 }
2166
panthor_vm_root_gem(struct panthor_vm * vm)2167 struct drm_gem_object *panthor_vm_root_gem(struct panthor_vm *vm)
2168 {
2169 if (!vm)
2170 return NULL;
2171
2172 return vm->base.r_obj;
2173 }
2174
2175 static int
panthor_vm_exec_op(struct panthor_vm * vm,struct panthor_vm_op_ctx * op,bool flag_vm_unusable_on_failure)2176 panthor_vm_exec_op(struct panthor_vm *vm, struct panthor_vm_op_ctx *op,
2177 bool flag_vm_unusable_on_failure)
2178 {
2179 u32 op_type = op->flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK;
2180 int ret;
2181
2182 if (op_type == DRM_PANTHOR_VM_BIND_OP_TYPE_SYNC_ONLY)
2183 return 0;
2184
2185 mutex_lock(&vm->op_lock);
2186 vm->op_ctx = op;
2187 switch (op_type) {
2188 case DRM_PANTHOR_VM_BIND_OP_TYPE_MAP:
2189 if (vm->unusable) {
2190 ret = -EINVAL;
2191 break;
2192 }
2193
2194 ret = drm_gpuvm_sm_map(&vm->base, vm, op->va.addr, op->va.range,
2195 op->map.vm_bo->obj, op->map.bo_offset);
2196 break;
2197
2198 case DRM_PANTHOR_VM_BIND_OP_TYPE_UNMAP:
2199 ret = drm_gpuvm_sm_unmap(&vm->base, vm, op->va.addr, op->va.range);
2200 break;
2201
2202 default:
2203 ret = -EINVAL;
2204 break;
2205 }
2206
2207 if (ret && flag_vm_unusable_on_failure)
2208 vm->unusable = true;
2209
2210 vm->op_ctx = NULL;
2211 mutex_unlock(&vm->op_lock);
2212
2213 return ret;
2214 }
2215
2216 static struct dma_fence *
panthor_vm_bind_run_job(struct drm_sched_job * sched_job)2217 panthor_vm_bind_run_job(struct drm_sched_job *sched_job)
2218 {
2219 struct panthor_vm_bind_job *job = container_of(sched_job, struct panthor_vm_bind_job, base);
2220 bool cookie;
2221 int ret;
2222
2223 /* Not only we report an error whose result is propagated to the
2224 * drm_sched finished fence, but we also flag the VM as unusable, because
2225 * a failure in the async VM_BIND results in an inconsistent state. VM needs
2226 * to be destroyed and recreated.
2227 */
2228 cookie = dma_fence_begin_signalling();
2229 ret = panthor_vm_exec_op(job->vm, &job->ctx, true);
2230 dma_fence_end_signalling(cookie);
2231
2232 return ret ? ERR_PTR(ret) : NULL;
2233 }
2234
panthor_vm_bind_job_release(struct kref * kref)2235 static void panthor_vm_bind_job_release(struct kref *kref)
2236 {
2237 struct panthor_vm_bind_job *job = container_of(kref, struct panthor_vm_bind_job, refcount);
2238
2239 if (job->base.s_fence)
2240 drm_sched_job_cleanup(&job->base);
2241
2242 panthor_vm_cleanup_op_ctx(&job->ctx, job->vm);
2243 panthor_vm_put(job->vm);
2244 kfree(job);
2245 }
2246
2247 /**
2248 * panthor_vm_bind_job_put() - Release a VM_BIND job reference
2249 * @sched_job: Job to release the reference on.
2250 */
panthor_vm_bind_job_put(struct drm_sched_job * sched_job)2251 void panthor_vm_bind_job_put(struct drm_sched_job *sched_job)
2252 {
2253 struct panthor_vm_bind_job *job =
2254 container_of(sched_job, struct panthor_vm_bind_job, base);
2255
2256 if (sched_job)
2257 kref_put(&job->refcount, panthor_vm_bind_job_release);
2258 }
2259
2260 static void
panthor_vm_bind_free_job(struct drm_sched_job * sched_job)2261 panthor_vm_bind_free_job(struct drm_sched_job *sched_job)
2262 {
2263 struct panthor_vm_bind_job *job =
2264 container_of(sched_job, struct panthor_vm_bind_job, base);
2265
2266 drm_sched_job_cleanup(sched_job);
2267
2268 /* Do the heavy cleanups asynchronously, so we're out of the
2269 * dma-signaling path and can acquire dma-resv locks safely.
2270 */
2271 queue_work(panthor_cleanup_wq, &job->cleanup_op_ctx_work);
2272 }
2273
2274 static enum drm_gpu_sched_stat
panthor_vm_bind_timedout_job(struct drm_sched_job * sched_job)2275 panthor_vm_bind_timedout_job(struct drm_sched_job *sched_job)
2276 {
2277 WARN(1, "VM_BIND ops are synchronous for now, there should be no timeout!");
2278 return DRM_GPU_SCHED_STAT_NOMINAL;
2279 }
2280
2281 static const struct drm_sched_backend_ops panthor_vm_bind_ops = {
2282 .run_job = panthor_vm_bind_run_job,
2283 .free_job = panthor_vm_bind_free_job,
2284 .timedout_job = panthor_vm_bind_timedout_job,
2285 };
2286
2287 /**
2288 * panthor_vm_create() - Create a VM
2289 * @ptdev: Device.
2290 * @for_mcu: True if this is the FW MCU VM.
2291 * @kernel_va_start: Start of the range reserved for kernel BO mapping.
2292 * @kernel_va_size: Size of the range reserved for kernel BO mapping.
2293 * @auto_kernel_va_start: Start of the auto-VA kernel range.
2294 * @auto_kernel_va_size: Size of the auto-VA kernel range.
2295 *
2296 * Return: A valid pointer on success, an ERR_PTR() otherwise.
2297 */
2298 struct panthor_vm *
panthor_vm_create(struct panthor_device * ptdev,bool for_mcu,u64 kernel_va_start,u64 kernel_va_size,u64 auto_kernel_va_start,u64 auto_kernel_va_size)2299 panthor_vm_create(struct panthor_device *ptdev, bool for_mcu,
2300 u64 kernel_va_start, u64 kernel_va_size,
2301 u64 auto_kernel_va_start, u64 auto_kernel_va_size)
2302 {
2303 u32 va_bits = GPU_MMU_FEATURES_VA_BITS(ptdev->gpu_info.mmu_features);
2304 u32 pa_bits = GPU_MMU_FEATURES_PA_BITS(ptdev->gpu_info.mmu_features);
2305 u64 full_va_range = 1ull << va_bits;
2306 struct drm_gem_object *dummy_gem;
2307 struct drm_gpu_scheduler *sched;
2308 const struct drm_sched_init_args sched_args = {
2309 .ops = &panthor_vm_bind_ops,
2310 .submit_wq = ptdev->mmu->vm.wq,
2311 .num_rqs = 1,
2312 .credit_limit = 1,
2313 /* Bind operations are synchronous for now, no timeout needed. */
2314 .timeout = MAX_SCHEDULE_TIMEOUT,
2315 .name = "panthor-vm-bind",
2316 .dev = ptdev->base.dev,
2317 };
2318 struct io_pgtable_cfg pgtbl_cfg;
2319 u64 mair, min_va, va_range;
2320 struct panthor_vm *vm;
2321 int ret;
2322
2323 vm = kzalloc(sizeof(*vm), GFP_KERNEL);
2324 if (!vm)
2325 return ERR_PTR(-ENOMEM);
2326
2327 /* We allocate a dummy GEM for the VM. */
2328 dummy_gem = drm_gpuvm_resv_object_alloc(&ptdev->base);
2329 if (!dummy_gem) {
2330 ret = -ENOMEM;
2331 goto err_free_vm;
2332 }
2333
2334 mutex_init(&vm->heaps.lock);
2335 vm->for_mcu = for_mcu;
2336 vm->ptdev = ptdev;
2337 mutex_init(&vm->op_lock);
2338
2339 if (for_mcu) {
2340 /* CSF MCU is a cortex M7, and can only address 4G */
2341 min_va = 0;
2342 va_range = SZ_4G;
2343 } else {
2344 min_va = 0;
2345 va_range = full_va_range;
2346 }
2347
2348 mutex_init(&vm->mm_lock);
2349 drm_mm_init(&vm->mm, kernel_va_start, kernel_va_size);
2350 vm->kernel_auto_va.start = auto_kernel_va_start;
2351 vm->kernel_auto_va.end = vm->kernel_auto_va.start + auto_kernel_va_size - 1;
2352
2353 INIT_LIST_HEAD(&vm->node);
2354 INIT_LIST_HEAD(&vm->as.lru_node);
2355 vm->as.id = -1;
2356 refcount_set(&vm->as.active_cnt, 0);
2357
2358 pgtbl_cfg = (struct io_pgtable_cfg) {
2359 .pgsize_bitmap = SZ_4K | SZ_2M,
2360 .ias = va_bits,
2361 .oas = pa_bits,
2362 .coherent_walk = ptdev->coherent,
2363 .tlb = &mmu_tlb_ops,
2364 .iommu_dev = ptdev->base.dev,
2365 .alloc = alloc_pt,
2366 .free = free_pt,
2367 };
2368
2369 vm->pgtbl_ops = alloc_io_pgtable_ops(ARM_64_LPAE_S1, &pgtbl_cfg, vm);
2370 if (!vm->pgtbl_ops) {
2371 ret = -EINVAL;
2372 goto err_mm_takedown;
2373 }
2374
2375 ret = drm_sched_init(&vm->sched, &sched_args);
2376 if (ret)
2377 goto err_free_io_pgtable;
2378
2379 sched = &vm->sched;
2380 ret = drm_sched_entity_init(&vm->entity, 0, &sched, 1, NULL);
2381 if (ret)
2382 goto err_sched_fini;
2383
2384 mair = io_pgtable_ops_to_pgtable(vm->pgtbl_ops)->cfg.arm_lpae_s1_cfg.mair;
2385 vm->memattr = mair_to_memattr(mair, ptdev->coherent);
2386
2387 mutex_lock(&ptdev->mmu->vm.lock);
2388 list_add_tail(&vm->node, &ptdev->mmu->vm.list);
2389
2390 /* If a reset is in progress, stop the scheduler. */
2391 if (ptdev->mmu->vm.reset_in_progress)
2392 panthor_vm_stop(vm);
2393 mutex_unlock(&ptdev->mmu->vm.lock);
2394
2395 /* We intentionally leave the reserved range to zero, because we want kernel VMAs
2396 * to be handled the same way user VMAs are.
2397 */
2398 drm_gpuvm_init(&vm->base, for_mcu ? "panthor-MCU-VM" : "panthor-GPU-VM",
2399 DRM_GPUVM_RESV_PROTECTED, &ptdev->base, dummy_gem,
2400 min_va, va_range, 0, 0, &panthor_gpuvm_ops);
2401 drm_gem_object_put(dummy_gem);
2402 return vm;
2403
2404 err_sched_fini:
2405 drm_sched_fini(&vm->sched);
2406
2407 err_free_io_pgtable:
2408 free_io_pgtable_ops(vm->pgtbl_ops);
2409
2410 err_mm_takedown:
2411 drm_mm_takedown(&vm->mm);
2412 drm_gem_object_put(dummy_gem);
2413
2414 err_free_vm:
2415 kfree(vm);
2416 return ERR_PTR(ret);
2417 }
2418
2419 static int
panthor_vm_bind_prepare_op_ctx(struct drm_file * file,struct panthor_vm * vm,const struct drm_panthor_vm_bind_op * op,struct panthor_vm_op_ctx * op_ctx)2420 panthor_vm_bind_prepare_op_ctx(struct drm_file *file,
2421 struct panthor_vm *vm,
2422 const struct drm_panthor_vm_bind_op *op,
2423 struct panthor_vm_op_ctx *op_ctx)
2424 {
2425 ssize_t vm_pgsz = panthor_vm_page_size(vm);
2426 struct drm_gem_object *gem;
2427 int ret;
2428
2429 /* Aligned on page size. */
2430 if (!IS_ALIGNED(op->va | op->size, vm_pgsz))
2431 return -EINVAL;
2432
2433 switch (op->flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) {
2434 case DRM_PANTHOR_VM_BIND_OP_TYPE_MAP:
2435 gem = drm_gem_object_lookup(file, op->bo_handle);
2436 ret = panthor_vm_prepare_map_op_ctx(op_ctx, vm,
2437 gem ? to_panthor_bo(gem) : NULL,
2438 op->bo_offset,
2439 op->size,
2440 op->va,
2441 op->flags);
2442 drm_gem_object_put(gem);
2443 return ret;
2444
2445 case DRM_PANTHOR_VM_BIND_OP_TYPE_UNMAP:
2446 if (op->flags & ~DRM_PANTHOR_VM_BIND_OP_TYPE_MASK)
2447 return -EINVAL;
2448
2449 if (op->bo_handle || op->bo_offset)
2450 return -EINVAL;
2451
2452 return panthor_vm_prepare_unmap_op_ctx(op_ctx, vm, op->va, op->size);
2453
2454 case DRM_PANTHOR_VM_BIND_OP_TYPE_SYNC_ONLY:
2455 if (op->flags & ~DRM_PANTHOR_VM_BIND_OP_TYPE_MASK)
2456 return -EINVAL;
2457
2458 if (op->bo_handle || op->bo_offset)
2459 return -EINVAL;
2460
2461 if (op->va || op->size)
2462 return -EINVAL;
2463
2464 if (!op->syncs.count)
2465 return -EINVAL;
2466
2467 panthor_vm_prepare_sync_only_op_ctx(op_ctx, vm);
2468 return 0;
2469
2470 default:
2471 return -EINVAL;
2472 }
2473 }
2474
panthor_vm_bind_job_cleanup_op_ctx_work(struct work_struct * work)2475 static void panthor_vm_bind_job_cleanup_op_ctx_work(struct work_struct *work)
2476 {
2477 struct panthor_vm_bind_job *job =
2478 container_of(work, struct panthor_vm_bind_job, cleanup_op_ctx_work);
2479
2480 panthor_vm_bind_job_put(&job->base);
2481 }
2482
2483 /**
2484 * panthor_vm_bind_job_create() - Create a VM_BIND job
2485 * @file: File.
2486 * @vm: VM targeted by the VM_BIND job.
2487 * @op: VM operation data.
2488 *
2489 * Return: A valid pointer on success, an ERR_PTR() otherwise.
2490 */
2491 struct drm_sched_job *
panthor_vm_bind_job_create(struct drm_file * file,struct panthor_vm * vm,const struct drm_panthor_vm_bind_op * op)2492 panthor_vm_bind_job_create(struct drm_file *file,
2493 struct panthor_vm *vm,
2494 const struct drm_panthor_vm_bind_op *op)
2495 {
2496 struct panthor_vm_bind_job *job;
2497 int ret;
2498
2499 if (!vm)
2500 return ERR_PTR(-EINVAL);
2501
2502 if (vm->destroyed || vm->unusable)
2503 return ERR_PTR(-EINVAL);
2504
2505 job = kzalloc(sizeof(*job), GFP_KERNEL);
2506 if (!job)
2507 return ERR_PTR(-ENOMEM);
2508
2509 ret = panthor_vm_bind_prepare_op_ctx(file, vm, op, &job->ctx);
2510 if (ret) {
2511 kfree(job);
2512 return ERR_PTR(ret);
2513 }
2514
2515 INIT_WORK(&job->cleanup_op_ctx_work, panthor_vm_bind_job_cleanup_op_ctx_work);
2516 kref_init(&job->refcount);
2517 job->vm = panthor_vm_get(vm);
2518
2519 ret = drm_sched_job_init(&job->base, &vm->entity, 1, vm);
2520 if (ret)
2521 goto err_put_job;
2522
2523 return &job->base;
2524
2525 err_put_job:
2526 panthor_vm_bind_job_put(&job->base);
2527 return ERR_PTR(ret);
2528 }
2529
2530 /**
2531 * panthor_vm_bind_job_prepare_resvs() - Prepare VM_BIND job dma_resvs
2532 * @exec: The locking/preparation context.
2533 * @sched_job: The job to prepare resvs on.
2534 *
2535 * Locks and prepare the VM resv.
2536 *
2537 * If this is a map operation, locks and prepares the GEM resv.
2538 *
2539 * Return: 0 on success, a negative error code otherwise.
2540 */
panthor_vm_bind_job_prepare_resvs(struct drm_exec * exec,struct drm_sched_job * sched_job)2541 int panthor_vm_bind_job_prepare_resvs(struct drm_exec *exec,
2542 struct drm_sched_job *sched_job)
2543 {
2544 struct panthor_vm_bind_job *job = container_of(sched_job, struct panthor_vm_bind_job, base);
2545 int ret;
2546
2547 /* Acquire the VM lock an reserve a slot for this VM bind job. */
2548 ret = drm_gpuvm_prepare_vm(&job->vm->base, exec, 1);
2549 if (ret)
2550 return ret;
2551
2552 if (job->ctx.map.vm_bo) {
2553 /* Lock/prepare the GEM being mapped. */
2554 ret = drm_exec_prepare_obj(exec, job->ctx.map.vm_bo->obj, 1);
2555 if (ret)
2556 return ret;
2557 }
2558
2559 return 0;
2560 }
2561
2562 /**
2563 * panthor_vm_bind_job_update_resvs() - Update the resv objects touched by a job
2564 * @exec: drm_exec context.
2565 * @sched_job: Job to update the resvs on.
2566 */
panthor_vm_bind_job_update_resvs(struct drm_exec * exec,struct drm_sched_job * sched_job)2567 void panthor_vm_bind_job_update_resvs(struct drm_exec *exec,
2568 struct drm_sched_job *sched_job)
2569 {
2570 struct panthor_vm_bind_job *job = container_of(sched_job, struct panthor_vm_bind_job, base);
2571
2572 /* Explicit sync => we just register our job finished fence as bookkeep. */
2573 drm_gpuvm_resv_add_fence(&job->vm->base, exec,
2574 &sched_job->s_fence->finished,
2575 DMA_RESV_USAGE_BOOKKEEP,
2576 DMA_RESV_USAGE_BOOKKEEP);
2577 }
2578
panthor_vm_update_resvs(struct panthor_vm * vm,struct drm_exec * exec,struct dma_fence * fence,enum dma_resv_usage private_usage,enum dma_resv_usage extobj_usage)2579 void panthor_vm_update_resvs(struct panthor_vm *vm, struct drm_exec *exec,
2580 struct dma_fence *fence,
2581 enum dma_resv_usage private_usage,
2582 enum dma_resv_usage extobj_usage)
2583 {
2584 drm_gpuvm_resv_add_fence(&vm->base, exec, fence, private_usage, extobj_usage);
2585 }
2586
2587 /**
2588 * panthor_vm_bind_exec_sync_op() - Execute a VM_BIND operation synchronously.
2589 * @file: File.
2590 * @vm: VM targeted by the VM operation.
2591 * @op: Data describing the VM operation.
2592 *
2593 * Return: 0 on success, a negative error code otherwise.
2594 */
panthor_vm_bind_exec_sync_op(struct drm_file * file,struct panthor_vm * vm,struct drm_panthor_vm_bind_op * op)2595 int panthor_vm_bind_exec_sync_op(struct drm_file *file,
2596 struct panthor_vm *vm,
2597 struct drm_panthor_vm_bind_op *op)
2598 {
2599 struct panthor_vm_op_ctx op_ctx;
2600 int ret;
2601
2602 /* No sync objects allowed on synchronous operations. */
2603 if (op->syncs.count)
2604 return -EINVAL;
2605
2606 if (!op->size)
2607 return 0;
2608
2609 ret = panthor_vm_bind_prepare_op_ctx(file, vm, op, &op_ctx);
2610 if (ret)
2611 return ret;
2612
2613 ret = panthor_vm_exec_op(vm, &op_ctx, false);
2614 panthor_vm_cleanup_op_ctx(&op_ctx, vm);
2615
2616 return ret;
2617 }
2618
2619 /**
2620 * panthor_vm_map_bo_range() - Map a GEM object range to a VM
2621 * @vm: VM to map the GEM to.
2622 * @bo: GEM object to map.
2623 * @offset: Offset in the GEM object.
2624 * @size: Size to map.
2625 * @va: Virtual address to map the object to.
2626 * @flags: Combination of drm_panthor_vm_bind_op_flags flags.
2627 * Only map-related flags are valid.
2628 *
2629 * Internal use only. For userspace requests, use
2630 * panthor_vm_bind_exec_sync_op() instead.
2631 *
2632 * Return: 0 on success, a negative error code otherwise.
2633 */
panthor_vm_map_bo_range(struct panthor_vm * vm,struct panthor_gem_object * bo,u64 offset,u64 size,u64 va,u32 flags)2634 int panthor_vm_map_bo_range(struct panthor_vm *vm, struct panthor_gem_object *bo,
2635 u64 offset, u64 size, u64 va, u32 flags)
2636 {
2637 struct panthor_vm_op_ctx op_ctx;
2638 int ret;
2639
2640 ret = panthor_vm_prepare_map_op_ctx(&op_ctx, vm, bo, offset, size, va, flags);
2641 if (ret)
2642 return ret;
2643
2644 ret = panthor_vm_exec_op(vm, &op_ctx, false);
2645 panthor_vm_cleanup_op_ctx(&op_ctx, vm);
2646
2647 return ret;
2648 }
2649
2650 /**
2651 * panthor_vm_unmap_range() - Unmap a portion of the VA space
2652 * @vm: VM to unmap the region from.
2653 * @va: Virtual address to unmap. Must be 4k aligned.
2654 * @size: Size of the region to unmap. Must be 4k aligned.
2655 *
2656 * Internal use only. For userspace requests, use
2657 * panthor_vm_bind_exec_sync_op() instead.
2658 *
2659 * Return: 0 on success, a negative error code otherwise.
2660 */
panthor_vm_unmap_range(struct panthor_vm * vm,u64 va,u64 size)2661 int panthor_vm_unmap_range(struct panthor_vm *vm, u64 va, u64 size)
2662 {
2663 struct panthor_vm_op_ctx op_ctx;
2664 int ret;
2665
2666 ret = panthor_vm_prepare_unmap_op_ctx(&op_ctx, vm, va, size);
2667 if (ret)
2668 return ret;
2669
2670 ret = panthor_vm_exec_op(vm, &op_ctx, false);
2671 panthor_vm_cleanup_op_ctx(&op_ctx, vm);
2672
2673 return ret;
2674 }
2675
2676 /**
2677 * panthor_vm_prepare_mapped_bos_resvs() - Prepare resvs on VM BOs.
2678 * @exec: Locking/preparation context.
2679 * @vm: VM targeted by the GPU job.
2680 * @slot_count: Number of slots to reserve.
2681 *
2682 * GPU jobs assume all BOs bound to the VM at the time the job is submitted
2683 * are available when the job is executed. In order to guarantee that, we
2684 * need to reserve a slot on all BOs mapped to a VM and update this slot with
2685 * the job fence after its submission.
2686 *
2687 * Return: 0 on success, a negative error code otherwise.
2688 */
panthor_vm_prepare_mapped_bos_resvs(struct drm_exec * exec,struct panthor_vm * vm,u32 slot_count)2689 int panthor_vm_prepare_mapped_bos_resvs(struct drm_exec *exec, struct panthor_vm *vm,
2690 u32 slot_count)
2691 {
2692 int ret;
2693
2694 /* Acquire the VM lock and reserve a slot for this GPU job. */
2695 ret = drm_gpuvm_prepare_vm(&vm->base, exec, slot_count);
2696 if (ret)
2697 return ret;
2698
2699 return drm_gpuvm_prepare_objects(&vm->base, exec, slot_count);
2700 }
2701
2702 /**
2703 * panthor_mmu_unplug() - Unplug the MMU logic
2704 * @ptdev: Device.
2705 *
2706 * No access to the MMU regs should be done after this function is called.
2707 * We suspend the IRQ and disable all VMs to guarantee that.
2708 */
panthor_mmu_unplug(struct panthor_device * ptdev)2709 void panthor_mmu_unplug(struct panthor_device *ptdev)
2710 {
2711 if (!IS_ENABLED(CONFIG_PM) || pm_runtime_active(ptdev->base.dev))
2712 panthor_mmu_irq_suspend(&ptdev->mmu->irq);
2713
2714 mutex_lock(&ptdev->mmu->as.slots_lock);
2715 for (u32 i = 0; i < ARRAY_SIZE(ptdev->mmu->as.slots); i++) {
2716 struct panthor_vm *vm = ptdev->mmu->as.slots[i].vm;
2717
2718 if (vm) {
2719 drm_WARN_ON(&ptdev->base, panthor_mmu_as_disable(ptdev, i));
2720 panthor_vm_release_as_locked(vm);
2721 }
2722 }
2723 mutex_unlock(&ptdev->mmu->as.slots_lock);
2724 }
2725
panthor_mmu_release_wq(struct drm_device * ddev,void * res)2726 static void panthor_mmu_release_wq(struct drm_device *ddev, void *res)
2727 {
2728 destroy_workqueue(res);
2729 }
2730
2731 /**
2732 * panthor_mmu_init() - Initialize the MMU logic.
2733 * @ptdev: Device.
2734 *
2735 * Return: 0 on success, a negative error code otherwise.
2736 */
panthor_mmu_init(struct panthor_device * ptdev)2737 int panthor_mmu_init(struct panthor_device *ptdev)
2738 {
2739 u32 va_bits = GPU_MMU_FEATURES_VA_BITS(ptdev->gpu_info.mmu_features);
2740 struct panthor_mmu *mmu;
2741 int ret, irq;
2742
2743 mmu = drmm_kzalloc(&ptdev->base, sizeof(*mmu), GFP_KERNEL);
2744 if (!mmu)
2745 return -ENOMEM;
2746
2747 INIT_LIST_HEAD(&mmu->as.lru_list);
2748
2749 ret = drmm_mutex_init(&ptdev->base, &mmu->as.slots_lock);
2750 if (ret)
2751 return ret;
2752
2753 INIT_LIST_HEAD(&mmu->vm.list);
2754 ret = drmm_mutex_init(&ptdev->base, &mmu->vm.lock);
2755 if (ret)
2756 return ret;
2757
2758 ptdev->mmu = mmu;
2759
2760 irq = platform_get_irq_byname(to_platform_device(ptdev->base.dev), "mmu");
2761 if (irq <= 0)
2762 return -ENODEV;
2763
2764 ret = panthor_request_mmu_irq(ptdev, &mmu->irq, irq,
2765 panthor_mmu_fault_mask(ptdev, ~0));
2766 if (ret)
2767 return ret;
2768
2769 mmu->vm.wq = alloc_workqueue("panthor-vm-bind", WQ_UNBOUND, 0);
2770 if (!mmu->vm.wq)
2771 return -ENOMEM;
2772
2773 /* On 32-bit kernels, the VA space is limited by the io_pgtable_ops abstraction,
2774 * which passes iova as an unsigned long. Patch the mmu_features to reflect this
2775 * limitation.
2776 */
2777 if (va_bits > BITS_PER_LONG) {
2778 ptdev->gpu_info.mmu_features &= ~GENMASK(7, 0);
2779 ptdev->gpu_info.mmu_features |= BITS_PER_LONG;
2780 }
2781
2782 return drmm_add_action_or_reset(&ptdev->base, panthor_mmu_release_wq, mmu->vm.wq);
2783 }
2784
2785 #ifdef CONFIG_DEBUG_FS
show_vm_gpuvas(struct panthor_vm * vm,struct seq_file * m)2786 static int show_vm_gpuvas(struct panthor_vm *vm, struct seq_file *m)
2787 {
2788 int ret;
2789
2790 mutex_lock(&vm->op_lock);
2791 ret = drm_debugfs_gpuva_info(m, &vm->base);
2792 mutex_unlock(&vm->op_lock);
2793
2794 return ret;
2795 }
2796
show_each_vm(struct seq_file * m,void * arg)2797 static int show_each_vm(struct seq_file *m, void *arg)
2798 {
2799 struct drm_info_node *node = (struct drm_info_node *)m->private;
2800 struct drm_device *ddev = node->minor->dev;
2801 struct panthor_device *ptdev = container_of(ddev, struct panthor_device, base);
2802 int (*show)(struct panthor_vm *, struct seq_file *) = node->info_ent->data;
2803 struct panthor_vm *vm;
2804 int ret = 0;
2805
2806 mutex_lock(&ptdev->mmu->vm.lock);
2807 list_for_each_entry(vm, &ptdev->mmu->vm.list, node) {
2808 ret = show(vm, m);
2809 if (ret < 0)
2810 break;
2811
2812 seq_puts(m, "\n");
2813 }
2814 mutex_unlock(&ptdev->mmu->vm.lock);
2815
2816 return ret;
2817 }
2818
2819 static struct drm_info_list panthor_mmu_debugfs_list[] = {
2820 DRM_DEBUGFS_GPUVA_INFO(show_each_vm, show_vm_gpuvas),
2821 };
2822
2823 /**
2824 * panthor_mmu_debugfs_init() - Initialize MMU debugfs entries
2825 * @minor: Minor.
2826 */
panthor_mmu_debugfs_init(struct drm_minor * minor)2827 void panthor_mmu_debugfs_init(struct drm_minor *minor)
2828 {
2829 drm_debugfs_create_files(panthor_mmu_debugfs_list,
2830 ARRAY_SIZE(panthor_mmu_debugfs_list),
2831 minor->debugfs_root, minor);
2832 }
2833 #endif /* CONFIG_DEBUG_FS */
2834
2835 /**
2836 * panthor_mmu_pt_cache_init() - Initialize the page table cache.
2837 *
2838 * Return: 0 on success, a negative error code otherwise.
2839 */
panthor_mmu_pt_cache_init(void)2840 int panthor_mmu_pt_cache_init(void)
2841 {
2842 pt_cache = kmem_cache_create("panthor-mmu-pt", SZ_4K, SZ_4K, 0, NULL);
2843 if (!pt_cache)
2844 return -ENOMEM;
2845
2846 return 0;
2847 }
2848
2849 /**
2850 * panthor_mmu_pt_cache_fini() - Destroy the page table cache.
2851 */
panthor_mmu_pt_cache_fini(void)2852 void panthor_mmu_pt_cache_fini(void)
2853 {
2854 kmem_cache_destroy(pt_cache);
2855 }
2856