1 // SPDX-License-Identifier: GPL-2.0 or MIT
2 /* Copyright 2023 Collabora ltd. */
3
4 #include <drm/drm_drv.h>
5 #include <drm/drm_exec.h>
6 #include <drm/drm_gem_shmem_helper.h>
7 #include <drm/drm_managed.h>
8 #include <drm/gpu_scheduler.h>
9 #include <drm/panthor_drm.h>
10
11 #include <linux/build_bug.h>
12 #include <linux/cleanup.h>
13 #include <linux/clk.h>
14 #include <linux/delay.h>
15 #include <linux/dma-mapping.h>
16 #include <linux/dma-resv.h>
17 #include <linux/firmware.h>
18 #include <linux/interrupt.h>
19 #include <linux/io.h>
20 #include <linux/iopoll.h>
21 #include <linux/iosys-map.h>
22 #include <linux/module.h>
23 #include <linux/platform_device.h>
24 #include <linux/pm_runtime.h>
25
26 #include "panthor_devfreq.h"
27 #include "panthor_device.h"
28 #include "panthor_fw.h"
29 #include "panthor_gem.h"
30 #include "panthor_gpu.h"
31 #include "panthor_heap.h"
32 #include "panthor_mmu.h"
33 #include "panthor_regs.h"
34 #include "panthor_sched.h"
35
36 /**
37 * DOC: Scheduler
38 *
39 * Mali CSF hardware adopts a firmware-assisted scheduling model, where
40 * the firmware takes care of scheduling aspects, to some extent.
41 *
42 * The scheduling happens at the scheduling group level, each group
43 * contains 1 to N queues (N is FW/hardware dependent, and exposed
44 * through the firmware interface). Each queue is assigned a command
45 * stream ring buffer, which serves as a way to get jobs submitted to
46 * the GPU, among other things.
47 *
48 * The firmware can schedule a maximum of M groups (M is FW/hardware
49 * dependent, and exposed through the firmware interface). Passed
50 * this maximum number of groups, the kernel must take care of
51 * rotating the groups passed to the firmware so every group gets
52 * a chance to have his queues scheduled for execution.
53 *
54 * The current implementation only supports with kernel-mode queues.
55 * In other terms, userspace doesn't have access to the ring-buffer.
56 * Instead, userspace passes indirect command stream buffers that are
57 * called from the queue ring-buffer by the kernel using a pre-defined
58 * sequence of command stream instructions to ensure the userspace driver
59 * always gets consistent results (cache maintenance,
60 * synchronization, ...).
61 *
62 * We rely on the drm_gpu_scheduler framework to deal with job
63 * dependencies and submission. As any other driver dealing with a
64 * FW-scheduler, we use the 1:1 entity:scheduler mode, such that each
65 * entity has its own job scheduler. When a job is ready to be executed
66 * (all its dependencies are met), it is pushed to the appropriate
67 * queue ring-buffer, and the group is scheduled for execution if it
68 * wasn't already active.
69 *
70 * Kernel-side group scheduling is timeslice-based. When we have less
71 * groups than there are slots, the periodic tick is disabled and we
72 * just let the FW schedule the active groups. When there are more
73 * groups than slots, we let each group a chance to execute stuff for
74 * a given amount of time, and then re-evaluate and pick new groups
75 * to schedule. The group selection algorithm is based on
76 * priority+round-robin.
77 *
78 * Even though user-mode queues is out of the scope right now, the
79 * current design takes them into account by avoiding any guess on the
80 * group/queue state that would be based on information we wouldn't have
81 * if userspace was in charge of the ring-buffer. That's also one of the
82 * reason we don't do 'cooperative' scheduling (encoding FW group slot
83 * reservation as dma_fence that would be returned from the
84 * drm_gpu_scheduler::prepare_job() hook, and treating group rotation as
85 * a queue of waiters, ordered by job submission order). This approach
86 * would work for kernel-mode queues, but would make user-mode queues a
87 * lot more complicated to retrofit.
88 */
89
90 #define JOB_TIMEOUT_MS 5000
91
92 #define MAX_CSG_PRIO 0xf
93
94 #define NUM_INSTRS_PER_CACHE_LINE (64 / sizeof(u64))
95 #define MAX_INSTRS_PER_JOB 24
96
97 struct panthor_group;
98
99 /**
100 * struct panthor_csg_slot - Command stream group slot
101 *
102 * This represents a FW slot for a scheduling group.
103 */
104 struct panthor_csg_slot {
105 /** @group: Scheduling group bound to this slot. */
106 struct panthor_group *group;
107
108 /** @priority: Group priority. */
109 u8 priority;
110
111 /**
112 * @idle: True if the group bound to this slot is idle.
113 *
114 * A group is idle when it has nothing waiting for execution on
115 * all its queues, or when queues are blocked waiting for something
116 * to happen (synchronization object).
117 */
118 bool idle;
119 };
120
121 /**
122 * enum panthor_csg_priority - Group priority
123 */
124 enum panthor_csg_priority {
125 /** @PANTHOR_CSG_PRIORITY_LOW: Low priority group. */
126 PANTHOR_CSG_PRIORITY_LOW = 0,
127
128 /** @PANTHOR_CSG_PRIORITY_MEDIUM: Medium priority group. */
129 PANTHOR_CSG_PRIORITY_MEDIUM,
130
131 /** @PANTHOR_CSG_PRIORITY_HIGH: High priority group. */
132 PANTHOR_CSG_PRIORITY_HIGH,
133
134 /**
135 * @PANTHOR_CSG_PRIORITY_RT: Real-time priority group.
136 *
137 * Real-time priority allows one to preempt scheduling of other
138 * non-real-time groups. When such a group becomes executable,
139 * it will evict the group with the lowest non-rt priority if
140 * there's no free group slot available.
141 */
142 PANTHOR_CSG_PRIORITY_RT,
143
144 /** @PANTHOR_CSG_PRIORITY_COUNT: Number of priority levels. */
145 PANTHOR_CSG_PRIORITY_COUNT,
146 };
147
148 /**
149 * struct panthor_scheduler - Object used to manage the scheduler
150 */
151 struct panthor_scheduler {
152 /** @ptdev: Device. */
153 struct panthor_device *ptdev;
154
155 /**
156 * @wq: Workqueue used by our internal scheduler logic and
157 * drm_gpu_scheduler.
158 *
159 * Used for the scheduler tick, group update or other kind of FW
160 * event processing that can't be handled in the threaded interrupt
161 * path. Also passed to the drm_gpu_scheduler instances embedded
162 * in panthor_queue.
163 */
164 struct workqueue_struct *wq;
165
166 /**
167 * @heap_alloc_wq: Workqueue used to schedule tiler_oom works.
168 *
169 * We have a queue dedicated to heap chunk allocation works to avoid
170 * blocking the rest of the scheduler if the allocation tries to
171 * reclaim memory.
172 */
173 struct workqueue_struct *heap_alloc_wq;
174
175 /** @tick_work: Work executed on a scheduling tick. */
176 struct delayed_work tick_work;
177
178 /**
179 * @sync_upd_work: Work used to process synchronization object updates.
180 *
181 * We use this work to unblock queues/groups that were waiting on a
182 * synchronization object.
183 */
184 struct work_struct sync_upd_work;
185
186 /**
187 * @fw_events_work: Work used to process FW events outside the interrupt path.
188 *
189 * Even if the interrupt is threaded, we need any event processing
190 * that require taking the panthor_scheduler::lock to be processed
191 * outside the interrupt path so we don't block the tick logic when
192 * it calls panthor_fw_{csg,wait}_wait_acks(). Since most of the
193 * event processing requires taking this lock, we just delegate all
194 * FW event processing to the scheduler workqueue.
195 */
196 struct work_struct fw_events_work;
197
198 /**
199 * @fw_events: Bitmask encoding pending FW events.
200 */
201 atomic_t fw_events;
202
203 /**
204 * @resched_target: When the next tick should occur.
205 *
206 * Expressed in jiffies.
207 */
208 u64 resched_target;
209
210 /**
211 * @last_tick: When the last tick occurred.
212 *
213 * Expressed in jiffies.
214 */
215 u64 last_tick;
216
217 /** @tick_period: Tick period in jiffies. */
218 u64 tick_period;
219
220 /**
221 * @lock: Lock protecting access to all the scheduler fields.
222 *
223 * Should be taken in the tick work, the irq handler, and anywhere the @groups
224 * fields are touched.
225 */
226 struct mutex lock;
227
228 /** @groups: Various lists used to classify groups. */
229 struct {
230 /**
231 * @runnable: Runnable group lists.
232 *
233 * When a group has queues that want to execute something,
234 * its panthor_group::run_node should be inserted here.
235 *
236 * One list per-priority.
237 */
238 struct list_head runnable[PANTHOR_CSG_PRIORITY_COUNT];
239
240 /**
241 * @idle: Idle group lists.
242 *
243 * When all queues of a group are idle (either because they
244 * have nothing to execute, or because they are blocked), the
245 * panthor_group::run_node field should be inserted here.
246 *
247 * One list per-priority.
248 */
249 struct list_head idle[PANTHOR_CSG_PRIORITY_COUNT];
250
251 /**
252 * @waiting: List of groups whose queues are blocked on a
253 * synchronization object.
254 *
255 * Insert panthor_group::wait_node here when a group is waiting
256 * for synchronization objects to be signaled.
257 *
258 * This list is evaluated in the @sync_upd_work work.
259 */
260 struct list_head waiting;
261 } groups;
262
263 /**
264 * @csg_slots: FW command stream group slots.
265 */
266 struct panthor_csg_slot csg_slots[MAX_CSGS];
267
268 /** @csg_slot_count: Number of command stream group slots exposed by the FW. */
269 u32 csg_slot_count;
270
271 /** @cs_slot_count: Number of command stream slot per group slot exposed by the FW. */
272 u32 cs_slot_count;
273
274 /** @as_slot_count: Number of address space slots supported by the MMU. */
275 u32 as_slot_count;
276
277 /** @used_csg_slot_count: Number of command stream group slot currently used. */
278 u32 used_csg_slot_count;
279
280 /** @sb_slot_count: Number of scoreboard slots. */
281 u32 sb_slot_count;
282
283 /**
284 * @might_have_idle_groups: True if an active group might have become idle.
285 *
286 * This will force a tick, so other runnable groups can be scheduled if one
287 * or more active groups became idle.
288 */
289 bool might_have_idle_groups;
290
291 /** @pm: Power management related fields. */
292 struct {
293 /** @has_ref: True if the scheduler owns a runtime PM reference. */
294 bool has_ref;
295 } pm;
296
297 /** @reset: Reset related fields. */
298 struct {
299 /** @lock: Lock protecting the other reset fields. */
300 struct mutex lock;
301
302 /**
303 * @in_progress: True if a reset is in progress.
304 *
305 * Set to true in panthor_sched_pre_reset() and back to false in
306 * panthor_sched_post_reset().
307 */
308 atomic_t in_progress;
309
310 /**
311 * @stopped_groups: List containing all groups that were stopped
312 * before a reset.
313 *
314 * Insert panthor_group::run_node in the pre_reset path.
315 */
316 struct list_head stopped_groups;
317 } reset;
318 };
319
320 /**
321 * struct panthor_syncobj_32b - 32-bit FW synchronization object
322 */
323 struct panthor_syncobj_32b {
324 /** @seqno: Sequence number. */
325 u32 seqno;
326
327 /**
328 * @status: Status.
329 *
330 * Not zero on failure.
331 */
332 u32 status;
333 };
334
335 /**
336 * struct panthor_syncobj_64b - 64-bit FW synchronization object
337 */
338 struct panthor_syncobj_64b {
339 /** @seqno: Sequence number. */
340 u64 seqno;
341
342 /**
343 * @status: Status.
344 *
345 * Not zero on failure.
346 */
347 u32 status;
348
349 /** @pad: MBZ. */
350 u32 pad;
351 };
352
353 /**
354 * struct panthor_queue - Execution queue
355 */
356 struct panthor_queue {
357 /** @scheduler: DRM scheduler used for this queue. */
358 struct drm_gpu_scheduler scheduler;
359
360 /** @entity: DRM scheduling entity used for this queue. */
361 struct drm_sched_entity entity;
362
363 /**
364 * @remaining_time: Time remaining before the job timeout expires.
365 *
366 * The job timeout is suspended when the queue is not scheduled by the
367 * FW. Every time we suspend the timer, we need to save the remaining
368 * time so we can restore it later on.
369 */
370 unsigned long remaining_time;
371
372 /** @timeout_suspended: True if the job timeout was suspended. */
373 bool timeout_suspended;
374
375 /**
376 * @doorbell_id: Doorbell assigned to this queue.
377 *
378 * Right now, all groups share the same doorbell, and the doorbell ID
379 * is assigned to group_slot + 1 when the group is assigned a slot. But
380 * we might decide to provide fine grained doorbell assignment at some
381 * point, so don't have to wake up all queues in a group every time one
382 * of them is updated.
383 */
384 u8 doorbell_id;
385
386 /**
387 * @priority: Priority of the queue inside the group.
388 *
389 * Must be less than 16 (Only 4 bits available).
390 */
391 u8 priority;
392 #define CSF_MAX_QUEUE_PRIO GENMASK(3, 0)
393
394 /** @ringbuf: Command stream ring-buffer. */
395 struct panthor_kernel_bo *ringbuf;
396
397 /** @iface: Firmware interface. */
398 struct {
399 /** @mem: FW memory allocated for this interface. */
400 struct panthor_kernel_bo *mem;
401
402 /** @input: Input interface. */
403 struct panthor_fw_ringbuf_input_iface *input;
404
405 /** @output: Output interface. */
406 const struct panthor_fw_ringbuf_output_iface *output;
407
408 /** @input_fw_va: FW virtual address of the input interface buffer. */
409 u32 input_fw_va;
410
411 /** @output_fw_va: FW virtual address of the output interface buffer. */
412 u32 output_fw_va;
413 } iface;
414
415 /**
416 * @syncwait: Stores information about the synchronization object this
417 * queue is waiting on.
418 */
419 struct {
420 /** @gpu_va: GPU address of the synchronization object. */
421 u64 gpu_va;
422
423 /** @ref: Reference value to compare against. */
424 u64 ref;
425
426 /** @gt: True if this is a greater-than test. */
427 bool gt;
428
429 /** @sync64: True if this is a 64-bit sync object. */
430 bool sync64;
431
432 /** @bo: Buffer object holding the synchronization object. */
433 struct drm_gem_object *obj;
434
435 /** @offset: Offset of the synchronization object inside @bo. */
436 u64 offset;
437
438 /**
439 * @kmap: Kernel mapping of the buffer object holding the
440 * synchronization object.
441 */
442 void *kmap;
443 } syncwait;
444
445 /** @fence_ctx: Fence context fields. */
446 struct {
447 /** @lock: Used to protect access to all fences allocated by this context. */
448 spinlock_t lock;
449
450 /**
451 * @id: Fence context ID.
452 *
453 * Allocated with dma_fence_context_alloc().
454 */
455 u64 id;
456
457 /** @seqno: Sequence number of the last initialized fence. */
458 atomic64_t seqno;
459
460 /**
461 * @last_fence: Fence of the last submitted job.
462 *
463 * We return this fence when we get an empty command stream.
464 * This way, we are guaranteed that all earlier jobs have completed
465 * when drm_sched_job::s_fence::finished without having to feed
466 * the CS ring buffer with a dummy job that only signals the fence.
467 */
468 struct dma_fence *last_fence;
469
470 /**
471 * @in_flight_jobs: List containing all in-flight jobs.
472 *
473 * Used to keep track and signal panthor_job::done_fence when the
474 * synchronization object attached to the queue is signaled.
475 */
476 struct list_head in_flight_jobs;
477 } fence_ctx;
478
479 /** @profiling: Job profiling data slots and access information. */
480 struct {
481 /** @slots: Kernel BO holding the slots. */
482 struct panthor_kernel_bo *slots;
483
484 /** @slot_count: Number of jobs ringbuffer can hold at once. */
485 u32 slot_count;
486
487 /** @seqno: Index of the next available profiling information slot. */
488 u32 seqno;
489 } profiling;
490 };
491
492 /**
493 * enum panthor_group_state - Scheduling group state.
494 */
495 enum panthor_group_state {
496 /** @PANTHOR_CS_GROUP_CREATED: Group was created, but not scheduled yet. */
497 PANTHOR_CS_GROUP_CREATED,
498
499 /** @PANTHOR_CS_GROUP_ACTIVE: Group is currently scheduled. */
500 PANTHOR_CS_GROUP_ACTIVE,
501
502 /**
503 * @PANTHOR_CS_GROUP_SUSPENDED: Group was scheduled at least once, but is
504 * inactive/suspended right now.
505 */
506 PANTHOR_CS_GROUP_SUSPENDED,
507
508 /**
509 * @PANTHOR_CS_GROUP_TERMINATED: Group was terminated.
510 *
511 * Can no longer be scheduled. The only allowed action is a destruction.
512 */
513 PANTHOR_CS_GROUP_TERMINATED,
514
515 /**
516 * @PANTHOR_CS_GROUP_UNKNOWN_STATE: Group is an unknown state.
517 *
518 * The FW returned an inconsistent state. The group is flagged unusable
519 * and can no longer be scheduled. The only allowed action is a
520 * destruction.
521 *
522 * When that happens, we also schedule a FW reset, to start from a fresh
523 * state.
524 */
525 PANTHOR_CS_GROUP_UNKNOWN_STATE,
526 };
527
528 /**
529 * struct panthor_group - Scheduling group object
530 */
531 struct panthor_group {
532 /** @refcount: Reference count */
533 struct kref refcount;
534
535 /** @ptdev: Device. */
536 struct panthor_device *ptdev;
537
538 /** @vm: VM bound to the group. */
539 struct panthor_vm *vm;
540
541 /** @compute_core_mask: Mask of shader cores that can be used for compute jobs. */
542 u64 compute_core_mask;
543
544 /** @fragment_core_mask: Mask of shader cores that can be used for fragment jobs. */
545 u64 fragment_core_mask;
546
547 /** @tiler_core_mask: Mask of tiler cores that can be used for tiler jobs. */
548 u64 tiler_core_mask;
549
550 /** @max_compute_cores: Maximum number of shader cores used for compute jobs. */
551 u8 max_compute_cores;
552
553 /** @max_fragment_cores: Maximum number of shader cores used for fragment jobs. */
554 u8 max_fragment_cores;
555
556 /** @max_tiler_cores: Maximum number of tiler cores used for tiler jobs. */
557 u8 max_tiler_cores;
558
559 /** @priority: Group priority (check panthor_csg_priority). */
560 u8 priority;
561
562 /** @blocked_queues: Bitmask reflecting the blocked queues. */
563 u32 blocked_queues;
564
565 /** @idle_queues: Bitmask reflecting the idle queues. */
566 u32 idle_queues;
567
568 /** @fatal_lock: Lock used to protect access to fatal fields. */
569 spinlock_t fatal_lock;
570
571 /** @fatal_queues: Bitmask reflecting the queues that hit a fatal exception. */
572 u32 fatal_queues;
573
574 /** @tiler_oom: Mask of queues that have a tiler OOM event to process. */
575 atomic_t tiler_oom;
576
577 /** @queue_count: Number of queues in this group. */
578 u32 queue_count;
579
580 /** @queues: Queues owned by this group. */
581 struct panthor_queue *queues[MAX_CS_PER_CSG];
582
583 /**
584 * @csg_id: ID of the FW group slot.
585 *
586 * -1 when the group is not scheduled/active.
587 */
588 int csg_id;
589
590 /**
591 * @destroyed: True when the group has been destroyed.
592 *
593 * If a group is destroyed it becomes useless: no further jobs can be submitted
594 * to its queues. We simply wait for all references to be dropped so we can
595 * release the group object.
596 */
597 bool destroyed;
598
599 /**
600 * @timedout: True when a timeout occurred on any of the queues owned by
601 * this group.
602 *
603 * Timeouts can be reported by drm_sched or by the FW. If a reset is required,
604 * and the group can't be suspended, this also leads to a timeout. In any case,
605 * any timeout situation is unrecoverable, and the group becomes useless. We
606 * simply wait for all references to be dropped so we can release the group
607 * object.
608 */
609 bool timedout;
610
611 /**
612 * @innocent: True when the group becomes unusable because the group suspension
613 * failed during a reset.
614 *
615 * Sometimes the FW was put in a bad state by other groups, causing the group
616 * suspension happening in the reset path to fail. In that case, we consider the
617 * group innocent.
618 */
619 bool innocent;
620
621 /**
622 * @syncobjs: Pool of per-queue synchronization objects.
623 *
624 * One sync object per queue. The position of the sync object is
625 * determined by the queue index.
626 */
627 struct panthor_kernel_bo *syncobjs;
628
629 /** @fdinfo: Per-file info exposed through /proc/<process>/fdinfo */
630 struct {
631 /** @data: Total sampled values for jobs in queues from this group. */
632 struct panthor_gpu_usage data;
633
634 /**
635 * @fdinfo.lock: Spinlock to govern concurrent access from drm file's fdinfo
636 * callback and job post-completion processing function
637 */
638 spinlock_t lock;
639
640 /** @fdinfo.kbo_sizes: Aggregate size of private kernel BO's held by the group. */
641 size_t kbo_sizes;
642 } fdinfo;
643
644 /** @state: Group state. */
645 enum panthor_group_state state;
646
647 /**
648 * @suspend_buf: Suspend buffer.
649 *
650 * Stores the state of the group and its queues when a group is suspended.
651 * Used at resume time to restore the group in its previous state.
652 *
653 * The size of the suspend buffer is exposed through the FW interface.
654 */
655 struct panthor_kernel_bo *suspend_buf;
656
657 /**
658 * @protm_suspend_buf: Protection mode suspend buffer.
659 *
660 * Stores the state of the group and its queues when a group that's in
661 * protection mode is suspended.
662 *
663 * Used at resume time to restore the group in its previous state.
664 *
665 * The size of the protection mode suspend buffer is exposed through the
666 * FW interface.
667 */
668 struct panthor_kernel_bo *protm_suspend_buf;
669
670 /** @sync_upd_work: Work used to check/signal job fences. */
671 struct work_struct sync_upd_work;
672
673 /** @tiler_oom_work: Work used to process tiler OOM events happening on this group. */
674 struct work_struct tiler_oom_work;
675
676 /** @term_work: Work used to finish the group termination procedure. */
677 struct work_struct term_work;
678
679 /**
680 * @release_work: Work used to release group resources.
681 *
682 * We need to postpone the group release to avoid a deadlock when
683 * the last ref is released in the tick work.
684 */
685 struct work_struct release_work;
686
687 /**
688 * @run_node: Node used to insert the group in the
689 * panthor_group::groups::{runnable,idle} and
690 * panthor_group::reset.stopped_groups lists.
691 */
692 struct list_head run_node;
693
694 /**
695 * @wait_node: Node used to insert the group in the
696 * panthor_group::groups::waiting list.
697 */
698 struct list_head wait_node;
699 };
700
701 struct panthor_job_profiling_data {
702 struct {
703 u64 before;
704 u64 after;
705 } cycles;
706
707 struct {
708 u64 before;
709 u64 after;
710 } time;
711 };
712
713 /**
714 * group_queue_work() - Queue a group work
715 * @group: Group to queue the work for.
716 * @wname: Work name.
717 *
718 * Grabs a ref and queue a work item to the scheduler workqueue. If
719 * the work was already queued, we release the reference we grabbed.
720 *
721 * Work callbacks must release the reference we grabbed here.
722 */
723 #define group_queue_work(group, wname) \
724 do { \
725 group_get(group); \
726 if (!queue_work((group)->ptdev->scheduler->wq, &(group)->wname ## _work)) \
727 group_put(group); \
728 } while (0)
729
730 /**
731 * sched_queue_work() - Queue a scheduler work.
732 * @sched: Scheduler object.
733 * @wname: Work name.
734 *
735 * Conditionally queues a scheduler work if no reset is pending/in-progress.
736 */
737 #define sched_queue_work(sched, wname) \
738 do { \
739 if (!atomic_read(&(sched)->reset.in_progress) && \
740 !panthor_device_reset_is_pending((sched)->ptdev)) \
741 queue_work((sched)->wq, &(sched)->wname ## _work); \
742 } while (0)
743
744 /**
745 * sched_queue_delayed_work() - Queue a scheduler delayed work.
746 * @sched: Scheduler object.
747 * @wname: Work name.
748 * @delay: Work delay in jiffies.
749 *
750 * Conditionally queues a scheduler delayed work if no reset is
751 * pending/in-progress.
752 */
753 #define sched_queue_delayed_work(sched, wname, delay) \
754 do { \
755 if (!atomic_read(&sched->reset.in_progress) && \
756 !panthor_device_reset_is_pending((sched)->ptdev)) \
757 mod_delayed_work((sched)->wq, &(sched)->wname ## _work, delay); \
758 } while (0)
759
760 /*
761 * We currently set the maximum of groups per file to an arbitrary low value.
762 * But this can be updated if we need more.
763 */
764 #define MAX_GROUPS_PER_POOL 128
765
766 /**
767 * struct panthor_group_pool - Group pool
768 *
769 * Each file get assigned a group pool.
770 */
771 struct panthor_group_pool {
772 /** @xa: Xarray used to manage group handles. */
773 struct xarray xa;
774 };
775
776 /**
777 * struct panthor_job - Used to manage GPU job
778 */
779 struct panthor_job {
780 /** @base: Inherit from drm_sched_job. */
781 struct drm_sched_job base;
782
783 /** @refcount: Reference count. */
784 struct kref refcount;
785
786 /** @group: Group of the queue this job will be pushed to. */
787 struct panthor_group *group;
788
789 /** @queue_idx: Index of the queue inside @group. */
790 u32 queue_idx;
791
792 /** @call_info: Information about the userspace command stream call. */
793 struct {
794 /** @start: GPU address of the userspace command stream. */
795 u64 start;
796
797 /** @size: Size of the userspace command stream. */
798 u32 size;
799
800 /**
801 * @latest_flush: Flush ID at the time the userspace command
802 * stream was built.
803 *
804 * Needed for the flush reduction mechanism.
805 */
806 u32 latest_flush;
807 } call_info;
808
809 /** @ringbuf: Position of this job is in the ring buffer. */
810 struct {
811 /** @start: Start offset. */
812 u64 start;
813
814 /** @end: End offset. */
815 u64 end;
816 } ringbuf;
817
818 /**
819 * @node: Used to insert the job in the panthor_queue::fence_ctx::in_flight_jobs
820 * list.
821 */
822 struct list_head node;
823
824 /** @done_fence: Fence signaled when the job is finished or cancelled. */
825 struct dma_fence *done_fence;
826
827 /** @profiling: Job profiling information. */
828 struct {
829 /** @mask: Current device job profiling enablement bitmask. */
830 u32 mask;
831
832 /** @slot: Job index in the profiling slots BO. */
833 u32 slot;
834 } profiling;
835 };
836
837 static void
panthor_queue_put_syncwait_obj(struct panthor_queue * queue)838 panthor_queue_put_syncwait_obj(struct panthor_queue *queue)
839 {
840 if (queue->syncwait.kmap) {
841 struct iosys_map map = IOSYS_MAP_INIT_VADDR(queue->syncwait.kmap);
842
843 drm_gem_vunmap_unlocked(queue->syncwait.obj, &map);
844 queue->syncwait.kmap = NULL;
845 }
846
847 drm_gem_object_put(queue->syncwait.obj);
848 queue->syncwait.obj = NULL;
849 }
850
851 static void *
panthor_queue_get_syncwait_obj(struct panthor_group * group,struct panthor_queue * queue)852 panthor_queue_get_syncwait_obj(struct panthor_group *group, struct panthor_queue *queue)
853 {
854 struct panthor_device *ptdev = group->ptdev;
855 struct panthor_gem_object *bo;
856 struct iosys_map map;
857 int ret;
858
859 if (queue->syncwait.kmap)
860 return queue->syncwait.kmap + queue->syncwait.offset;
861
862 bo = panthor_vm_get_bo_for_va(group->vm,
863 queue->syncwait.gpu_va,
864 &queue->syncwait.offset);
865 if (drm_WARN_ON(&ptdev->base, IS_ERR_OR_NULL(bo)))
866 goto err_put_syncwait_obj;
867
868 queue->syncwait.obj = &bo->base.base;
869 ret = drm_gem_vmap_unlocked(queue->syncwait.obj, &map);
870 if (drm_WARN_ON(&ptdev->base, ret))
871 goto err_put_syncwait_obj;
872
873 queue->syncwait.kmap = map.vaddr;
874 if (drm_WARN_ON(&ptdev->base, !queue->syncwait.kmap))
875 goto err_put_syncwait_obj;
876
877 return queue->syncwait.kmap + queue->syncwait.offset;
878
879 err_put_syncwait_obj:
880 panthor_queue_put_syncwait_obj(queue);
881 return NULL;
882 }
883
group_free_queue(struct panthor_group * group,struct panthor_queue * queue)884 static void group_free_queue(struct panthor_group *group, struct panthor_queue *queue)
885 {
886 if (IS_ERR_OR_NULL(queue))
887 return;
888
889 if (queue->entity.fence_context)
890 drm_sched_entity_destroy(&queue->entity);
891
892 if (queue->scheduler.ops)
893 drm_sched_fini(&queue->scheduler);
894
895 panthor_queue_put_syncwait_obj(queue);
896
897 panthor_kernel_bo_destroy(queue->ringbuf);
898 panthor_kernel_bo_destroy(queue->iface.mem);
899 panthor_kernel_bo_destroy(queue->profiling.slots);
900
901 /* Release the last_fence we were holding, if any. */
902 dma_fence_put(queue->fence_ctx.last_fence);
903
904 kfree(queue);
905 }
906
group_release_work(struct work_struct * work)907 static void group_release_work(struct work_struct *work)
908 {
909 struct panthor_group *group = container_of(work,
910 struct panthor_group,
911 release_work);
912 u32 i;
913
914 for (i = 0; i < group->queue_count; i++)
915 group_free_queue(group, group->queues[i]);
916
917 panthor_kernel_bo_destroy(group->suspend_buf);
918 panthor_kernel_bo_destroy(group->protm_suspend_buf);
919 panthor_kernel_bo_destroy(group->syncobjs);
920
921 panthor_vm_put(group->vm);
922 kfree(group);
923 }
924
group_release(struct kref * kref)925 static void group_release(struct kref *kref)
926 {
927 struct panthor_group *group = container_of(kref,
928 struct panthor_group,
929 refcount);
930 struct panthor_device *ptdev = group->ptdev;
931
932 drm_WARN_ON(&ptdev->base, group->csg_id >= 0);
933 drm_WARN_ON(&ptdev->base, !list_empty(&group->run_node));
934 drm_WARN_ON(&ptdev->base, !list_empty(&group->wait_node));
935
936 queue_work(panthor_cleanup_wq, &group->release_work);
937 }
938
group_put(struct panthor_group * group)939 static void group_put(struct panthor_group *group)
940 {
941 if (group)
942 kref_put(&group->refcount, group_release);
943 }
944
945 static struct panthor_group *
group_get(struct panthor_group * group)946 group_get(struct panthor_group *group)
947 {
948 if (group)
949 kref_get(&group->refcount);
950
951 return group;
952 }
953
954 /**
955 * group_bind_locked() - Bind a group to a group slot
956 * @group: Group.
957 * @csg_id: Slot.
958 *
959 * Return: 0 on success, a negative error code otherwise.
960 */
961 static int
group_bind_locked(struct panthor_group * group,u32 csg_id)962 group_bind_locked(struct panthor_group *group, u32 csg_id)
963 {
964 struct panthor_device *ptdev = group->ptdev;
965 struct panthor_csg_slot *csg_slot;
966 int ret;
967
968 lockdep_assert_held(&ptdev->scheduler->lock);
969
970 if (drm_WARN_ON(&ptdev->base, group->csg_id != -1 || csg_id >= MAX_CSGS ||
971 ptdev->scheduler->csg_slots[csg_id].group))
972 return -EINVAL;
973
974 ret = panthor_vm_active(group->vm);
975 if (ret)
976 return ret;
977
978 csg_slot = &ptdev->scheduler->csg_slots[csg_id];
979 group_get(group);
980 group->csg_id = csg_id;
981
982 /* Dummy doorbell allocation: doorbell is assigned to the group and
983 * all queues use the same doorbell.
984 *
985 * TODO: Implement LRU-based doorbell assignment, so the most often
986 * updated queues get their own doorbell, thus avoiding useless checks
987 * on queues belonging to the same group that are rarely updated.
988 */
989 for (u32 i = 0; i < group->queue_count; i++)
990 group->queues[i]->doorbell_id = csg_id + 1;
991
992 csg_slot->group = group;
993
994 return 0;
995 }
996
997 /**
998 * group_unbind_locked() - Unbind a group from a slot.
999 * @group: Group to unbind.
1000 *
1001 * Return: 0 on success, a negative error code otherwise.
1002 */
1003 static int
group_unbind_locked(struct panthor_group * group)1004 group_unbind_locked(struct panthor_group *group)
1005 {
1006 struct panthor_device *ptdev = group->ptdev;
1007 struct panthor_csg_slot *slot;
1008
1009 lockdep_assert_held(&ptdev->scheduler->lock);
1010
1011 if (drm_WARN_ON(&ptdev->base, group->csg_id < 0 || group->csg_id >= MAX_CSGS))
1012 return -EINVAL;
1013
1014 if (drm_WARN_ON(&ptdev->base, group->state == PANTHOR_CS_GROUP_ACTIVE))
1015 return -EINVAL;
1016
1017 slot = &ptdev->scheduler->csg_slots[group->csg_id];
1018 panthor_vm_idle(group->vm);
1019 group->csg_id = -1;
1020
1021 /* Tiler OOM events will be re-issued next time the group is scheduled. */
1022 atomic_set(&group->tiler_oom, 0);
1023 cancel_work(&group->tiler_oom_work);
1024
1025 for (u32 i = 0; i < group->queue_count; i++)
1026 group->queues[i]->doorbell_id = -1;
1027
1028 slot->group = NULL;
1029
1030 group_put(group);
1031 return 0;
1032 }
1033
1034 /**
1035 * cs_slot_prog_locked() - Program a queue slot
1036 * @ptdev: Device.
1037 * @csg_id: Group slot ID.
1038 * @cs_id: Queue slot ID.
1039 *
1040 * Program a queue slot with the queue information so things can start being
1041 * executed on this queue.
1042 *
1043 * The group slot must have a group bound to it already (group_bind_locked()).
1044 */
1045 static void
cs_slot_prog_locked(struct panthor_device * ptdev,u32 csg_id,u32 cs_id)1046 cs_slot_prog_locked(struct panthor_device *ptdev, u32 csg_id, u32 cs_id)
1047 {
1048 struct panthor_queue *queue = ptdev->scheduler->csg_slots[csg_id].group->queues[cs_id];
1049 struct panthor_fw_cs_iface *cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);
1050
1051 lockdep_assert_held(&ptdev->scheduler->lock);
1052
1053 queue->iface.input->extract = queue->iface.output->extract;
1054 drm_WARN_ON(&ptdev->base, queue->iface.input->insert < queue->iface.input->extract);
1055
1056 cs_iface->input->ringbuf_base = panthor_kernel_bo_gpuva(queue->ringbuf);
1057 cs_iface->input->ringbuf_size = panthor_kernel_bo_size(queue->ringbuf);
1058 cs_iface->input->ringbuf_input = queue->iface.input_fw_va;
1059 cs_iface->input->ringbuf_output = queue->iface.output_fw_va;
1060 cs_iface->input->config = CS_CONFIG_PRIORITY(queue->priority) |
1061 CS_CONFIG_DOORBELL(queue->doorbell_id);
1062 cs_iface->input->ack_irq_mask = ~0;
1063 panthor_fw_update_reqs(cs_iface, req,
1064 CS_IDLE_SYNC_WAIT |
1065 CS_IDLE_EMPTY |
1066 CS_STATE_START |
1067 CS_EXTRACT_EVENT,
1068 CS_IDLE_SYNC_WAIT |
1069 CS_IDLE_EMPTY |
1070 CS_STATE_MASK |
1071 CS_EXTRACT_EVENT);
1072 if (queue->iface.input->insert != queue->iface.input->extract && queue->timeout_suspended) {
1073 drm_sched_resume_timeout(&queue->scheduler, queue->remaining_time);
1074 queue->timeout_suspended = false;
1075 }
1076 }
1077
1078 /**
1079 * cs_slot_reset_locked() - Reset a queue slot
1080 * @ptdev: Device.
1081 * @csg_id: Group slot.
1082 * @cs_id: Queue slot.
1083 *
1084 * Change the queue slot state to STOP and suspend the queue timeout if
1085 * the queue is not blocked.
1086 *
1087 * The group slot must have a group bound to it (group_bind_locked()).
1088 */
1089 static int
cs_slot_reset_locked(struct panthor_device * ptdev,u32 csg_id,u32 cs_id)1090 cs_slot_reset_locked(struct panthor_device *ptdev, u32 csg_id, u32 cs_id)
1091 {
1092 struct panthor_fw_cs_iface *cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);
1093 struct panthor_group *group = ptdev->scheduler->csg_slots[csg_id].group;
1094 struct panthor_queue *queue = group->queues[cs_id];
1095
1096 lockdep_assert_held(&ptdev->scheduler->lock);
1097
1098 panthor_fw_update_reqs(cs_iface, req,
1099 CS_STATE_STOP,
1100 CS_STATE_MASK);
1101
1102 /* If the queue is blocked, we want to keep the timeout running, so
1103 * we can detect unbounded waits and kill the group when that happens.
1104 */
1105 if (!(group->blocked_queues & BIT(cs_id)) && !queue->timeout_suspended) {
1106 queue->remaining_time = drm_sched_suspend_timeout(&queue->scheduler);
1107 queue->timeout_suspended = true;
1108 WARN_ON(queue->remaining_time > msecs_to_jiffies(JOB_TIMEOUT_MS));
1109 }
1110
1111 return 0;
1112 }
1113
1114 /**
1115 * csg_slot_sync_priority_locked() - Synchronize the group slot priority
1116 * @ptdev: Device.
1117 * @csg_id: Group slot ID.
1118 *
1119 * Group slot priority update happens asynchronously. When we receive a
1120 * %CSG_ENDPOINT_CONFIG, we know the update is effective, and can
1121 * reflect it to our panthor_csg_slot object.
1122 */
1123 static void
csg_slot_sync_priority_locked(struct panthor_device * ptdev,u32 csg_id)1124 csg_slot_sync_priority_locked(struct panthor_device *ptdev, u32 csg_id)
1125 {
1126 struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id];
1127 struct panthor_fw_csg_iface *csg_iface;
1128
1129 lockdep_assert_held(&ptdev->scheduler->lock);
1130
1131 csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
1132 csg_slot->priority = (csg_iface->input->endpoint_req & CSG_EP_REQ_PRIORITY_MASK) >> 28;
1133 }
1134
1135 /**
1136 * cs_slot_sync_queue_state_locked() - Synchronize the queue slot priority
1137 * @ptdev: Device.
1138 * @csg_id: Group slot.
1139 * @cs_id: Queue slot.
1140 *
1141 * Queue state is updated on group suspend or STATUS_UPDATE event.
1142 */
1143 static void
cs_slot_sync_queue_state_locked(struct panthor_device * ptdev,u32 csg_id,u32 cs_id)1144 cs_slot_sync_queue_state_locked(struct panthor_device *ptdev, u32 csg_id, u32 cs_id)
1145 {
1146 struct panthor_group *group = ptdev->scheduler->csg_slots[csg_id].group;
1147 struct panthor_queue *queue = group->queues[cs_id];
1148 struct panthor_fw_cs_iface *cs_iface =
1149 panthor_fw_get_cs_iface(group->ptdev, csg_id, cs_id);
1150
1151 u32 status_wait_cond;
1152
1153 switch (cs_iface->output->status_blocked_reason) {
1154 case CS_STATUS_BLOCKED_REASON_UNBLOCKED:
1155 if (queue->iface.input->insert == queue->iface.output->extract &&
1156 cs_iface->output->status_scoreboards == 0)
1157 group->idle_queues |= BIT(cs_id);
1158 break;
1159
1160 case CS_STATUS_BLOCKED_REASON_SYNC_WAIT:
1161 if (list_empty(&group->wait_node)) {
1162 list_move_tail(&group->wait_node,
1163 &group->ptdev->scheduler->groups.waiting);
1164 }
1165
1166 /* The queue is only blocked if there's no deferred operation
1167 * pending, which can be checked through the scoreboard status.
1168 */
1169 if (!cs_iface->output->status_scoreboards)
1170 group->blocked_queues |= BIT(cs_id);
1171
1172 queue->syncwait.gpu_va = cs_iface->output->status_wait_sync_ptr;
1173 queue->syncwait.ref = cs_iface->output->status_wait_sync_value;
1174 status_wait_cond = cs_iface->output->status_wait & CS_STATUS_WAIT_SYNC_COND_MASK;
1175 queue->syncwait.gt = status_wait_cond == CS_STATUS_WAIT_SYNC_COND_GT;
1176 if (cs_iface->output->status_wait & CS_STATUS_WAIT_SYNC_64B) {
1177 u64 sync_val_hi = cs_iface->output->status_wait_sync_value_hi;
1178
1179 queue->syncwait.sync64 = true;
1180 queue->syncwait.ref |= sync_val_hi << 32;
1181 } else {
1182 queue->syncwait.sync64 = false;
1183 }
1184 break;
1185
1186 default:
1187 /* Other reasons are not blocking. Consider the queue as runnable
1188 * in those cases.
1189 */
1190 break;
1191 }
1192 }
1193
1194 static void
csg_slot_sync_queues_state_locked(struct panthor_device * ptdev,u32 csg_id)1195 csg_slot_sync_queues_state_locked(struct panthor_device *ptdev, u32 csg_id)
1196 {
1197 struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id];
1198 struct panthor_group *group = csg_slot->group;
1199 u32 i;
1200
1201 lockdep_assert_held(&ptdev->scheduler->lock);
1202
1203 group->idle_queues = 0;
1204 group->blocked_queues = 0;
1205
1206 for (i = 0; i < group->queue_count; i++) {
1207 if (group->queues[i])
1208 cs_slot_sync_queue_state_locked(ptdev, csg_id, i);
1209 }
1210 }
1211
1212 static void
csg_slot_sync_state_locked(struct panthor_device * ptdev,u32 csg_id)1213 csg_slot_sync_state_locked(struct panthor_device *ptdev, u32 csg_id)
1214 {
1215 struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id];
1216 struct panthor_fw_csg_iface *csg_iface;
1217 struct panthor_group *group;
1218 enum panthor_group_state new_state, old_state;
1219 u32 csg_state;
1220
1221 lockdep_assert_held(&ptdev->scheduler->lock);
1222
1223 csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
1224 group = csg_slot->group;
1225
1226 if (!group)
1227 return;
1228
1229 old_state = group->state;
1230 csg_state = csg_iface->output->ack & CSG_STATE_MASK;
1231 switch (csg_state) {
1232 case CSG_STATE_START:
1233 case CSG_STATE_RESUME:
1234 new_state = PANTHOR_CS_GROUP_ACTIVE;
1235 break;
1236 case CSG_STATE_TERMINATE:
1237 new_state = PANTHOR_CS_GROUP_TERMINATED;
1238 break;
1239 case CSG_STATE_SUSPEND:
1240 new_state = PANTHOR_CS_GROUP_SUSPENDED;
1241 break;
1242 default:
1243 /* The unknown state might be caused by a FW state corruption,
1244 * which means the group metadata can't be trusted anymore, and
1245 * the SUSPEND operation might propagate the corruption to the
1246 * suspend buffers. Flag the group state as unknown to make
1247 * sure it's unusable after that point.
1248 */
1249 drm_err(&ptdev->base, "Invalid state on CSG %d (state=%d)",
1250 csg_id, csg_state);
1251 new_state = PANTHOR_CS_GROUP_UNKNOWN_STATE;
1252 break;
1253 }
1254
1255 if (old_state == new_state)
1256 return;
1257
1258 /* The unknown state might be caused by a FW issue, reset the FW to
1259 * take a fresh start.
1260 */
1261 if (new_state == PANTHOR_CS_GROUP_UNKNOWN_STATE)
1262 panthor_device_schedule_reset(ptdev);
1263
1264 if (new_state == PANTHOR_CS_GROUP_SUSPENDED)
1265 csg_slot_sync_queues_state_locked(ptdev, csg_id);
1266
1267 if (old_state == PANTHOR_CS_GROUP_ACTIVE) {
1268 u32 i;
1269
1270 /* Reset the queue slots so we start from a clean
1271 * state when starting/resuming a new group on this
1272 * CSG slot. No wait needed here, and no ringbell
1273 * either, since the CS slot will only be re-used
1274 * on the next CSG start operation.
1275 */
1276 for (i = 0; i < group->queue_count; i++) {
1277 if (group->queues[i])
1278 cs_slot_reset_locked(ptdev, csg_id, i);
1279 }
1280 }
1281
1282 group->state = new_state;
1283 }
1284
1285 static int
csg_slot_prog_locked(struct panthor_device * ptdev,u32 csg_id,u32 priority)1286 csg_slot_prog_locked(struct panthor_device *ptdev, u32 csg_id, u32 priority)
1287 {
1288 struct panthor_fw_csg_iface *csg_iface;
1289 struct panthor_csg_slot *csg_slot;
1290 struct panthor_group *group;
1291 u32 queue_mask = 0, i;
1292
1293 lockdep_assert_held(&ptdev->scheduler->lock);
1294
1295 if (priority > MAX_CSG_PRIO)
1296 return -EINVAL;
1297
1298 if (drm_WARN_ON(&ptdev->base, csg_id >= MAX_CSGS))
1299 return -EINVAL;
1300
1301 csg_slot = &ptdev->scheduler->csg_slots[csg_id];
1302 group = csg_slot->group;
1303 if (!group || group->state == PANTHOR_CS_GROUP_ACTIVE)
1304 return 0;
1305
1306 csg_iface = panthor_fw_get_csg_iface(group->ptdev, csg_id);
1307
1308 for (i = 0; i < group->queue_count; i++) {
1309 if (group->queues[i]) {
1310 cs_slot_prog_locked(ptdev, csg_id, i);
1311 queue_mask |= BIT(i);
1312 }
1313 }
1314
1315 csg_iface->input->allow_compute = group->compute_core_mask;
1316 csg_iface->input->allow_fragment = group->fragment_core_mask;
1317 csg_iface->input->allow_other = group->tiler_core_mask;
1318 csg_iface->input->endpoint_req = CSG_EP_REQ_COMPUTE(group->max_compute_cores) |
1319 CSG_EP_REQ_FRAGMENT(group->max_fragment_cores) |
1320 CSG_EP_REQ_TILER(group->max_tiler_cores) |
1321 CSG_EP_REQ_PRIORITY(priority);
1322 csg_iface->input->config = panthor_vm_as(group->vm);
1323
1324 if (group->suspend_buf)
1325 csg_iface->input->suspend_buf = panthor_kernel_bo_gpuva(group->suspend_buf);
1326 else
1327 csg_iface->input->suspend_buf = 0;
1328
1329 if (group->protm_suspend_buf) {
1330 csg_iface->input->protm_suspend_buf =
1331 panthor_kernel_bo_gpuva(group->protm_suspend_buf);
1332 } else {
1333 csg_iface->input->protm_suspend_buf = 0;
1334 }
1335
1336 csg_iface->input->ack_irq_mask = ~0;
1337 panthor_fw_toggle_reqs(csg_iface, doorbell_req, doorbell_ack, queue_mask);
1338 return 0;
1339 }
1340
1341 static void
cs_slot_process_fatal_event_locked(struct panthor_device * ptdev,u32 csg_id,u32 cs_id)1342 cs_slot_process_fatal_event_locked(struct panthor_device *ptdev,
1343 u32 csg_id, u32 cs_id)
1344 {
1345 struct panthor_scheduler *sched = ptdev->scheduler;
1346 struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id];
1347 struct panthor_group *group = csg_slot->group;
1348 struct panthor_fw_cs_iface *cs_iface;
1349 u32 fatal;
1350 u64 info;
1351
1352 lockdep_assert_held(&sched->lock);
1353
1354 cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);
1355 fatal = cs_iface->output->fatal;
1356 info = cs_iface->output->fatal_info;
1357
1358 if (group)
1359 group->fatal_queues |= BIT(cs_id);
1360
1361 if (CS_EXCEPTION_TYPE(fatal) == DRM_PANTHOR_EXCEPTION_CS_UNRECOVERABLE) {
1362 /* If this exception is unrecoverable, queue a reset, and make
1363 * sure we stop scheduling groups until the reset has happened.
1364 */
1365 panthor_device_schedule_reset(ptdev);
1366 cancel_delayed_work(&sched->tick_work);
1367 } else {
1368 sched_queue_delayed_work(sched, tick, 0);
1369 }
1370
1371 drm_warn(&ptdev->base,
1372 "CSG slot %d CS slot: %d\n"
1373 "CS_FATAL.EXCEPTION_TYPE: 0x%x (%s)\n"
1374 "CS_FATAL.EXCEPTION_DATA: 0x%x\n"
1375 "CS_FATAL_INFO.EXCEPTION_DATA: 0x%llx\n",
1376 csg_id, cs_id,
1377 (unsigned int)CS_EXCEPTION_TYPE(fatal),
1378 panthor_exception_name(ptdev, CS_EXCEPTION_TYPE(fatal)),
1379 (unsigned int)CS_EXCEPTION_DATA(fatal),
1380 info);
1381 }
1382
1383 static void
cs_slot_process_fault_event_locked(struct panthor_device * ptdev,u32 csg_id,u32 cs_id)1384 cs_slot_process_fault_event_locked(struct panthor_device *ptdev,
1385 u32 csg_id, u32 cs_id)
1386 {
1387 struct panthor_scheduler *sched = ptdev->scheduler;
1388 struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id];
1389 struct panthor_group *group = csg_slot->group;
1390 struct panthor_queue *queue = group && cs_id < group->queue_count ?
1391 group->queues[cs_id] : NULL;
1392 struct panthor_fw_cs_iface *cs_iface;
1393 u32 fault;
1394 u64 info;
1395
1396 lockdep_assert_held(&sched->lock);
1397
1398 cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);
1399 fault = cs_iface->output->fault;
1400 info = cs_iface->output->fault_info;
1401
1402 if (queue && CS_EXCEPTION_TYPE(fault) == DRM_PANTHOR_EXCEPTION_CS_INHERIT_FAULT) {
1403 u64 cs_extract = queue->iface.output->extract;
1404 struct panthor_job *job;
1405
1406 spin_lock(&queue->fence_ctx.lock);
1407 list_for_each_entry(job, &queue->fence_ctx.in_flight_jobs, node) {
1408 if (cs_extract >= job->ringbuf.end)
1409 continue;
1410
1411 if (cs_extract < job->ringbuf.start)
1412 break;
1413
1414 dma_fence_set_error(job->done_fence, -EINVAL);
1415 }
1416 spin_unlock(&queue->fence_ctx.lock);
1417 }
1418
1419 drm_warn(&ptdev->base,
1420 "CSG slot %d CS slot: %d\n"
1421 "CS_FAULT.EXCEPTION_TYPE: 0x%x (%s)\n"
1422 "CS_FAULT.EXCEPTION_DATA: 0x%x\n"
1423 "CS_FAULT_INFO.EXCEPTION_DATA: 0x%llx\n",
1424 csg_id, cs_id,
1425 (unsigned int)CS_EXCEPTION_TYPE(fault),
1426 panthor_exception_name(ptdev, CS_EXCEPTION_TYPE(fault)),
1427 (unsigned int)CS_EXCEPTION_DATA(fault),
1428 info);
1429 }
1430
group_process_tiler_oom(struct panthor_group * group,u32 cs_id)1431 static int group_process_tiler_oom(struct panthor_group *group, u32 cs_id)
1432 {
1433 struct panthor_device *ptdev = group->ptdev;
1434 struct panthor_scheduler *sched = ptdev->scheduler;
1435 u32 renderpasses_in_flight, pending_frag_count;
1436 struct panthor_heap_pool *heaps = NULL;
1437 u64 heap_address, new_chunk_va = 0;
1438 u32 vt_start, vt_end, frag_end;
1439 int ret, csg_id;
1440
1441 mutex_lock(&sched->lock);
1442 csg_id = group->csg_id;
1443 if (csg_id >= 0) {
1444 struct panthor_fw_cs_iface *cs_iface;
1445
1446 cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);
1447 heaps = panthor_vm_get_heap_pool(group->vm, false);
1448 heap_address = cs_iface->output->heap_address;
1449 vt_start = cs_iface->output->heap_vt_start;
1450 vt_end = cs_iface->output->heap_vt_end;
1451 frag_end = cs_iface->output->heap_frag_end;
1452 renderpasses_in_flight = vt_start - frag_end;
1453 pending_frag_count = vt_end - frag_end;
1454 }
1455 mutex_unlock(&sched->lock);
1456
1457 /* The group got scheduled out, we stop here. We will get a new tiler OOM event
1458 * when it's scheduled again.
1459 */
1460 if (unlikely(csg_id < 0))
1461 return 0;
1462
1463 if (IS_ERR(heaps) || frag_end > vt_end || vt_end >= vt_start) {
1464 ret = -EINVAL;
1465 } else {
1466 /* We do the allocation without holding the scheduler lock to avoid
1467 * blocking the scheduling.
1468 */
1469 ret = panthor_heap_grow(heaps, heap_address,
1470 renderpasses_in_flight,
1471 pending_frag_count, &new_chunk_va);
1472 }
1473
1474 /* If the heap context doesn't have memory for us, we want to let the
1475 * FW try to reclaim memory by waiting for fragment jobs to land or by
1476 * executing the tiler OOM exception handler, which is supposed to
1477 * implement incremental rendering.
1478 */
1479 if (ret && ret != -ENOMEM) {
1480 drm_warn(&ptdev->base, "Failed to extend the tiler heap\n");
1481 group->fatal_queues |= BIT(cs_id);
1482 sched_queue_delayed_work(sched, tick, 0);
1483 goto out_put_heap_pool;
1484 }
1485
1486 mutex_lock(&sched->lock);
1487 csg_id = group->csg_id;
1488 if (csg_id >= 0) {
1489 struct panthor_fw_csg_iface *csg_iface;
1490 struct panthor_fw_cs_iface *cs_iface;
1491
1492 csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
1493 cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);
1494
1495 cs_iface->input->heap_start = new_chunk_va;
1496 cs_iface->input->heap_end = new_chunk_va;
1497 panthor_fw_update_reqs(cs_iface, req, cs_iface->output->ack, CS_TILER_OOM);
1498 panthor_fw_toggle_reqs(csg_iface, doorbell_req, doorbell_ack, BIT(cs_id));
1499 panthor_fw_ring_csg_doorbells(ptdev, BIT(csg_id));
1500 }
1501 mutex_unlock(&sched->lock);
1502
1503 /* We allocated a chunck, but couldn't link it to the heap
1504 * context because the group was scheduled out while we were
1505 * allocating memory. We need to return this chunk to the heap.
1506 */
1507 if (unlikely(csg_id < 0 && new_chunk_va))
1508 panthor_heap_return_chunk(heaps, heap_address, new_chunk_va);
1509
1510 ret = 0;
1511
1512 out_put_heap_pool:
1513 panthor_heap_pool_put(heaps);
1514 return ret;
1515 }
1516
group_tiler_oom_work(struct work_struct * work)1517 static void group_tiler_oom_work(struct work_struct *work)
1518 {
1519 struct panthor_group *group =
1520 container_of(work, struct panthor_group, tiler_oom_work);
1521 u32 tiler_oom = atomic_xchg(&group->tiler_oom, 0);
1522
1523 while (tiler_oom) {
1524 u32 cs_id = ffs(tiler_oom) - 1;
1525
1526 group_process_tiler_oom(group, cs_id);
1527 tiler_oom &= ~BIT(cs_id);
1528 }
1529
1530 group_put(group);
1531 }
1532
1533 static void
cs_slot_process_tiler_oom_event_locked(struct panthor_device * ptdev,u32 csg_id,u32 cs_id)1534 cs_slot_process_tiler_oom_event_locked(struct panthor_device *ptdev,
1535 u32 csg_id, u32 cs_id)
1536 {
1537 struct panthor_scheduler *sched = ptdev->scheduler;
1538 struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id];
1539 struct panthor_group *group = csg_slot->group;
1540
1541 lockdep_assert_held(&sched->lock);
1542
1543 if (drm_WARN_ON(&ptdev->base, !group))
1544 return;
1545
1546 atomic_or(BIT(cs_id), &group->tiler_oom);
1547
1548 /* We don't use group_queue_work() here because we want to queue the
1549 * work item to the heap_alloc_wq.
1550 */
1551 group_get(group);
1552 if (!queue_work(sched->heap_alloc_wq, &group->tiler_oom_work))
1553 group_put(group);
1554 }
1555
cs_slot_process_irq_locked(struct panthor_device * ptdev,u32 csg_id,u32 cs_id)1556 static bool cs_slot_process_irq_locked(struct panthor_device *ptdev,
1557 u32 csg_id, u32 cs_id)
1558 {
1559 struct panthor_fw_cs_iface *cs_iface;
1560 u32 req, ack, events;
1561
1562 lockdep_assert_held(&ptdev->scheduler->lock);
1563
1564 cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id);
1565 req = cs_iface->input->req;
1566 ack = cs_iface->output->ack;
1567 events = (req ^ ack) & CS_EVT_MASK;
1568
1569 if (events & CS_FATAL)
1570 cs_slot_process_fatal_event_locked(ptdev, csg_id, cs_id);
1571
1572 if (events & CS_FAULT)
1573 cs_slot_process_fault_event_locked(ptdev, csg_id, cs_id);
1574
1575 if (events & CS_TILER_OOM)
1576 cs_slot_process_tiler_oom_event_locked(ptdev, csg_id, cs_id);
1577
1578 /* We don't acknowledge the TILER_OOM event since its handling is
1579 * deferred to a separate work.
1580 */
1581 panthor_fw_update_reqs(cs_iface, req, ack, CS_FATAL | CS_FAULT);
1582
1583 return (events & (CS_FAULT | CS_TILER_OOM)) != 0;
1584 }
1585
csg_slot_sync_idle_state_locked(struct panthor_device * ptdev,u32 csg_id)1586 static void csg_slot_sync_idle_state_locked(struct panthor_device *ptdev, u32 csg_id)
1587 {
1588 struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id];
1589 struct panthor_fw_csg_iface *csg_iface;
1590
1591 lockdep_assert_held(&ptdev->scheduler->lock);
1592
1593 csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
1594 csg_slot->idle = csg_iface->output->status_state & CSG_STATUS_STATE_IS_IDLE;
1595 }
1596
csg_slot_process_idle_event_locked(struct panthor_device * ptdev,u32 csg_id)1597 static void csg_slot_process_idle_event_locked(struct panthor_device *ptdev, u32 csg_id)
1598 {
1599 struct panthor_scheduler *sched = ptdev->scheduler;
1600
1601 lockdep_assert_held(&sched->lock);
1602
1603 sched->might_have_idle_groups = true;
1604
1605 /* Schedule a tick so we can evict idle groups and schedule non-idle
1606 * ones. This will also update runtime PM and devfreq busy/idle states,
1607 * so the device can lower its frequency or get suspended.
1608 */
1609 sched_queue_delayed_work(sched, tick, 0);
1610 }
1611
csg_slot_sync_update_locked(struct panthor_device * ptdev,u32 csg_id)1612 static void csg_slot_sync_update_locked(struct panthor_device *ptdev,
1613 u32 csg_id)
1614 {
1615 struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id];
1616 struct panthor_group *group = csg_slot->group;
1617
1618 lockdep_assert_held(&ptdev->scheduler->lock);
1619
1620 if (group)
1621 group_queue_work(group, sync_upd);
1622
1623 sched_queue_work(ptdev->scheduler, sync_upd);
1624 }
1625
1626 static void
csg_slot_process_progress_timer_event_locked(struct panthor_device * ptdev,u32 csg_id)1627 csg_slot_process_progress_timer_event_locked(struct panthor_device *ptdev, u32 csg_id)
1628 {
1629 struct panthor_scheduler *sched = ptdev->scheduler;
1630 struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id];
1631 struct panthor_group *group = csg_slot->group;
1632
1633 lockdep_assert_held(&sched->lock);
1634
1635 drm_warn(&ptdev->base, "CSG slot %d progress timeout\n", csg_id);
1636
1637 group = csg_slot->group;
1638 if (!drm_WARN_ON(&ptdev->base, !group))
1639 group->timedout = true;
1640
1641 sched_queue_delayed_work(sched, tick, 0);
1642 }
1643
sched_process_csg_irq_locked(struct panthor_device * ptdev,u32 csg_id)1644 static void sched_process_csg_irq_locked(struct panthor_device *ptdev, u32 csg_id)
1645 {
1646 u32 req, ack, cs_irq_req, cs_irq_ack, cs_irqs, csg_events;
1647 struct panthor_fw_csg_iface *csg_iface;
1648 u32 ring_cs_db_mask = 0;
1649
1650 lockdep_assert_held(&ptdev->scheduler->lock);
1651
1652 if (drm_WARN_ON(&ptdev->base, csg_id >= ptdev->scheduler->csg_slot_count))
1653 return;
1654
1655 csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
1656 req = READ_ONCE(csg_iface->input->req);
1657 ack = READ_ONCE(csg_iface->output->ack);
1658 cs_irq_req = READ_ONCE(csg_iface->output->cs_irq_req);
1659 cs_irq_ack = READ_ONCE(csg_iface->input->cs_irq_ack);
1660 csg_events = (req ^ ack) & CSG_EVT_MASK;
1661
1662 /* There may not be any pending CSG/CS interrupts to process */
1663 if (req == ack && cs_irq_req == cs_irq_ack)
1664 return;
1665
1666 /* Immediately set IRQ_ACK bits to be same as the IRQ_REQ bits before
1667 * examining the CS_ACK & CS_REQ bits. This would ensure that Host
1668 * doesn't miss an interrupt for the CS in the race scenario where
1669 * whilst Host is servicing an interrupt for the CS, firmware sends
1670 * another interrupt for that CS.
1671 */
1672 csg_iface->input->cs_irq_ack = cs_irq_req;
1673
1674 panthor_fw_update_reqs(csg_iface, req, ack,
1675 CSG_SYNC_UPDATE |
1676 CSG_IDLE |
1677 CSG_PROGRESS_TIMER_EVENT);
1678
1679 if (csg_events & CSG_IDLE)
1680 csg_slot_process_idle_event_locked(ptdev, csg_id);
1681
1682 if (csg_events & CSG_PROGRESS_TIMER_EVENT)
1683 csg_slot_process_progress_timer_event_locked(ptdev, csg_id);
1684
1685 cs_irqs = cs_irq_req ^ cs_irq_ack;
1686 while (cs_irqs) {
1687 u32 cs_id = ffs(cs_irqs) - 1;
1688
1689 if (cs_slot_process_irq_locked(ptdev, csg_id, cs_id))
1690 ring_cs_db_mask |= BIT(cs_id);
1691
1692 cs_irqs &= ~BIT(cs_id);
1693 }
1694
1695 if (csg_events & CSG_SYNC_UPDATE)
1696 csg_slot_sync_update_locked(ptdev, csg_id);
1697
1698 if (ring_cs_db_mask)
1699 panthor_fw_toggle_reqs(csg_iface, doorbell_req, doorbell_ack, ring_cs_db_mask);
1700
1701 panthor_fw_ring_csg_doorbells(ptdev, BIT(csg_id));
1702 }
1703
sched_process_idle_event_locked(struct panthor_device * ptdev)1704 static void sched_process_idle_event_locked(struct panthor_device *ptdev)
1705 {
1706 struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev);
1707
1708 lockdep_assert_held(&ptdev->scheduler->lock);
1709
1710 /* Acknowledge the idle event and schedule a tick. */
1711 panthor_fw_update_reqs(glb_iface, req, glb_iface->output->ack, GLB_IDLE);
1712 sched_queue_delayed_work(ptdev->scheduler, tick, 0);
1713 }
1714
1715 /**
1716 * sched_process_global_irq_locked() - Process the scheduling part of a global IRQ
1717 * @ptdev: Device.
1718 */
sched_process_global_irq_locked(struct panthor_device * ptdev)1719 static void sched_process_global_irq_locked(struct panthor_device *ptdev)
1720 {
1721 struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev);
1722 u32 req, ack, evts;
1723
1724 lockdep_assert_held(&ptdev->scheduler->lock);
1725
1726 req = READ_ONCE(glb_iface->input->req);
1727 ack = READ_ONCE(glb_iface->output->ack);
1728 evts = (req ^ ack) & GLB_EVT_MASK;
1729
1730 if (evts & GLB_IDLE)
1731 sched_process_idle_event_locked(ptdev);
1732 }
1733
process_fw_events_work(struct work_struct * work)1734 static void process_fw_events_work(struct work_struct *work)
1735 {
1736 struct panthor_scheduler *sched = container_of(work, struct panthor_scheduler,
1737 fw_events_work);
1738 u32 events = atomic_xchg(&sched->fw_events, 0);
1739 struct panthor_device *ptdev = sched->ptdev;
1740
1741 mutex_lock(&sched->lock);
1742
1743 if (events & JOB_INT_GLOBAL_IF) {
1744 sched_process_global_irq_locked(ptdev);
1745 events &= ~JOB_INT_GLOBAL_IF;
1746 }
1747
1748 while (events) {
1749 u32 csg_id = ffs(events) - 1;
1750
1751 sched_process_csg_irq_locked(ptdev, csg_id);
1752 events &= ~BIT(csg_id);
1753 }
1754
1755 mutex_unlock(&sched->lock);
1756 }
1757
1758 /**
1759 * panthor_sched_report_fw_events() - Report FW events to the scheduler.
1760 */
panthor_sched_report_fw_events(struct panthor_device * ptdev,u32 events)1761 void panthor_sched_report_fw_events(struct panthor_device *ptdev, u32 events)
1762 {
1763 if (!ptdev->scheduler)
1764 return;
1765
1766 atomic_or(events, &ptdev->scheduler->fw_events);
1767 sched_queue_work(ptdev->scheduler, fw_events);
1768 }
1769
fence_get_driver_name(struct dma_fence * fence)1770 static const char *fence_get_driver_name(struct dma_fence *fence)
1771 {
1772 return "panthor";
1773 }
1774
queue_fence_get_timeline_name(struct dma_fence * fence)1775 static const char *queue_fence_get_timeline_name(struct dma_fence *fence)
1776 {
1777 return "queue-fence";
1778 }
1779
1780 static const struct dma_fence_ops panthor_queue_fence_ops = {
1781 .get_driver_name = fence_get_driver_name,
1782 .get_timeline_name = queue_fence_get_timeline_name,
1783 };
1784
1785 struct panthor_csg_slots_upd_ctx {
1786 u32 update_mask;
1787 u32 timedout_mask;
1788 struct {
1789 u32 value;
1790 u32 mask;
1791 } requests[MAX_CSGS];
1792 };
1793
csgs_upd_ctx_init(struct panthor_csg_slots_upd_ctx * ctx)1794 static void csgs_upd_ctx_init(struct panthor_csg_slots_upd_ctx *ctx)
1795 {
1796 memset(ctx, 0, sizeof(*ctx));
1797 }
1798
csgs_upd_ctx_queue_reqs(struct panthor_device * ptdev,struct panthor_csg_slots_upd_ctx * ctx,u32 csg_id,u32 value,u32 mask)1799 static void csgs_upd_ctx_queue_reqs(struct panthor_device *ptdev,
1800 struct panthor_csg_slots_upd_ctx *ctx,
1801 u32 csg_id, u32 value, u32 mask)
1802 {
1803 if (drm_WARN_ON(&ptdev->base, !mask) ||
1804 drm_WARN_ON(&ptdev->base, csg_id >= ptdev->scheduler->csg_slot_count))
1805 return;
1806
1807 ctx->requests[csg_id].value = (ctx->requests[csg_id].value & ~mask) | (value & mask);
1808 ctx->requests[csg_id].mask |= mask;
1809 ctx->update_mask |= BIT(csg_id);
1810 }
1811
csgs_upd_ctx_apply_locked(struct panthor_device * ptdev,struct panthor_csg_slots_upd_ctx * ctx)1812 static int csgs_upd_ctx_apply_locked(struct panthor_device *ptdev,
1813 struct panthor_csg_slots_upd_ctx *ctx)
1814 {
1815 struct panthor_scheduler *sched = ptdev->scheduler;
1816 u32 update_slots = ctx->update_mask;
1817
1818 lockdep_assert_held(&sched->lock);
1819
1820 if (!ctx->update_mask)
1821 return 0;
1822
1823 while (update_slots) {
1824 struct panthor_fw_csg_iface *csg_iface;
1825 u32 csg_id = ffs(update_slots) - 1;
1826
1827 update_slots &= ~BIT(csg_id);
1828 csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
1829 panthor_fw_update_reqs(csg_iface, req,
1830 ctx->requests[csg_id].value,
1831 ctx->requests[csg_id].mask);
1832 }
1833
1834 panthor_fw_ring_csg_doorbells(ptdev, ctx->update_mask);
1835
1836 update_slots = ctx->update_mask;
1837 while (update_slots) {
1838 struct panthor_fw_csg_iface *csg_iface;
1839 u32 csg_id = ffs(update_slots) - 1;
1840 u32 req_mask = ctx->requests[csg_id].mask, acked;
1841 int ret;
1842
1843 update_slots &= ~BIT(csg_id);
1844 csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
1845
1846 ret = panthor_fw_csg_wait_acks(ptdev, csg_id, req_mask, &acked, 100);
1847
1848 if (acked & CSG_ENDPOINT_CONFIG)
1849 csg_slot_sync_priority_locked(ptdev, csg_id);
1850
1851 if (acked & CSG_STATE_MASK)
1852 csg_slot_sync_state_locked(ptdev, csg_id);
1853
1854 if (acked & CSG_STATUS_UPDATE) {
1855 csg_slot_sync_queues_state_locked(ptdev, csg_id);
1856 csg_slot_sync_idle_state_locked(ptdev, csg_id);
1857 }
1858
1859 if (ret && acked != req_mask &&
1860 ((csg_iface->input->req ^ csg_iface->output->ack) & req_mask) != 0) {
1861 drm_err(&ptdev->base, "CSG %d update request timedout", csg_id);
1862 ctx->timedout_mask |= BIT(csg_id);
1863 }
1864 }
1865
1866 if (ctx->timedout_mask)
1867 return -ETIMEDOUT;
1868
1869 return 0;
1870 }
1871
1872 struct panthor_sched_tick_ctx {
1873 struct list_head old_groups[PANTHOR_CSG_PRIORITY_COUNT];
1874 struct list_head groups[PANTHOR_CSG_PRIORITY_COUNT];
1875 u32 idle_group_count;
1876 u32 group_count;
1877 enum panthor_csg_priority min_priority;
1878 struct panthor_vm *vms[MAX_CS_PER_CSG];
1879 u32 as_count;
1880 bool immediate_tick;
1881 u32 csg_upd_failed_mask;
1882 };
1883
1884 static bool
tick_ctx_is_full(const struct panthor_scheduler * sched,const struct panthor_sched_tick_ctx * ctx)1885 tick_ctx_is_full(const struct panthor_scheduler *sched,
1886 const struct panthor_sched_tick_ctx *ctx)
1887 {
1888 return ctx->group_count == sched->csg_slot_count;
1889 }
1890
1891 static bool
group_is_idle(struct panthor_group * group)1892 group_is_idle(struct panthor_group *group)
1893 {
1894 struct panthor_device *ptdev = group->ptdev;
1895 u32 inactive_queues;
1896
1897 if (group->csg_id >= 0)
1898 return ptdev->scheduler->csg_slots[group->csg_id].idle;
1899
1900 inactive_queues = group->idle_queues | group->blocked_queues;
1901 return hweight32(inactive_queues) == group->queue_count;
1902 }
1903
1904 static bool
group_can_run(struct panthor_group * group)1905 group_can_run(struct panthor_group *group)
1906 {
1907 return group->state != PANTHOR_CS_GROUP_TERMINATED &&
1908 group->state != PANTHOR_CS_GROUP_UNKNOWN_STATE &&
1909 !group->destroyed && group->fatal_queues == 0 &&
1910 !group->timedout;
1911 }
1912
1913 static void
tick_ctx_pick_groups_from_list(const struct panthor_scheduler * sched,struct panthor_sched_tick_ctx * ctx,struct list_head * queue,bool skip_idle_groups,bool owned_by_tick_ctx)1914 tick_ctx_pick_groups_from_list(const struct panthor_scheduler *sched,
1915 struct panthor_sched_tick_ctx *ctx,
1916 struct list_head *queue,
1917 bool skip_idle_groups,
1918 bool owned_by_tick_ctx)
1919 {
1920 struct panthor_group *group, *tmp;
1921
1922 if (tick_ctx_is_full(sched, ctx))
1923 return;
1924
1925 list_for_each_entry_safe(group, tmp, queue, run_node) {
1926 u32 i;
1927
1928 if (!group_can_run(group))
1929 continue;
1930
1931 if (skip_idle_groups && group_is_idle(group))
1932 continue;
1933
1934 for (i = 0; i < ctx->as_count; i++) {
1935 if (ctx->vms[i] == group->vm)
1936 break;
1937 }
1938
1939 if (i == ctx->as_count && ctx->as_count == sched->as_slot_count)
1940 continue;
1941
1942 if (!owned_by_tick_ctx)
1943 group_get(group);
1944
1945 list_move_tail(&group->run_node, &ctx->groups[group->priority]);
1946 ctx->group_count++;
1947 if (group_is_idle(group))
1948 ctx->idle_group_count++;
1949
1950 if (i == ctx->as_count)
1951 ctx->vms[ctx->as_count++] = group->vm;
1952
1953 if (ctx->min_priority > group->priority)
1954 ctx->min_priority = group->priority;
1955
1956 if (tick_ctx_is_full(sched, ctx))
1957 return;
1958 }
1959 }
1960
1961 static void
tick_ctx_insert_old_group(struct panthor_scheduler * sched,struct panthor_sched_tick_ctx * ctx,struct panthor_group * group,bool full_tick)1962 tick_ctx_insert_old_group(struct panthor_scheduler *sched,
1963 struct panthor_sched_tick_ctx *ctx,
1964 struct panthor_group *group,
1965 bool full_tick)
1966 {
1967 struct panthor_csg_slot *csg_slot = &sched->csg_slots[group->csg_id];
1968 struct panthor_group *other_group;
1969
1970 if (!full_tick) {
1971 list_add_tail(&group->run_node, &ctx->old_groups[group->priority]);
1972 return;
1973 }
1974
1975 /* Rotate to make sure groups with lower CSG slot
1976 * priorities have a chance to get a higher CSG slot
1977 * priority next time they get picked. This priority
1978 * has an impact on resource request ordering, so it's
1979 * important to make sure we don't let one group starve
1980 * all other groups with the same group priority.
1981 */
1982 list_for_each_entry(other_group,
1983 &ctx->old_groups[csg_slot->group->priority],
1984 run_node) {
1985 struct panthor_csg_slot *other_csg_slot = &sched->csg_slots[other_group->csg_id];
1986
1987 if (other_csg_slot->priority > csg_slot->priority) {
1988 list_add_tail(&csg_slot->group->run_node, &other_group->run_node);
1989 return;
1990 }
1991 }
1992
1993 list_add_tail(&group->run_node, &ctx->old_groups[group->priority]);
1994 }
1995
1996 static void
tick_ctx_init(struct panthor_scheduler * sched,struct panthor_sched_tick_ctx * ctx,bool full_tick)1997 tick_ctx_init(struct panthor_scheduler *sched,
1998 struct panthor_sched_tick_ctx *ctx,
1999 bool full_tick)
2000 {
2001 struct panthor_device *ptdev = sched->ptdev;
2002 struct panthor_csg_slots_upd_ctx upd_ctx;
2003 int ret;
2004 u32 i;
2005
2006 memset(ctx, 0, sizeof(*ctx));
2007 csgs_upd_ctx_init(&upd_ctx);
2008
2009 ctx->min_priority = PANTHOR_CSG_PRIORITY_COUNT;
2010 for (i = 0; i < ARRAY_SIZE(ctx->groups); i++) {
2011 INIT_LIST_HEAD(&ctx->groups[i]);
2012 INIT_LIST_HEAD(&ctx->old_groups[i]);
2013 }
2014
2015 for (i = 0; i < sched->csg_slot_count; i++) {
2016 struct panthor_csg_slot *csg_slot = &sched->csg_slots[i];
2017 struct panthor_group *group = csg_slot->group;
2018 struct panthor_fw_csg_iface *csg_iface;
2019
2020 if (!group)
2021 continue;
2022
2023 csg_iface = panthor_fw_get_csg_iface(ptdev, i);
2024 group_get(group);
2025
2026 /* If there was unhandled faults on the VM, force processing of
2027 * CSG IRQs, so we can flag the faulty queue.
2028 */
2029 if (panthor_vm_has_unhandled_faults(group->vm)) {
2030 sched_process_csg_irq_locked(ptdev, i);
2031
2032 /* No fatal fault reported, flag all queues as faulty. */
2033 if (!group->fatal_queues)
2034 group->fatal_queues |= GENMASK(group->queue_count - 1, 0);
2035 }
2036
2037 tick_ctx_insert_old_group(sched, ctx, group, full_tick);
2038 csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, i,
2039 csg_iface->output->ack ^ CSG_STATUS_UPDATE,
2040 CSG_STATUS_UPDATE);
2041 }
2042
2043 ret = csgs_upd_ctx_apply_locked(ptdev, &upd_ctx);
2044 if (ret) {
2045 panthor_device_schedule_reset(ptdev);
2046 ctx->csg_upd_failed_mask |= upd_ctx.timedout_mask;
2047 }
2048 }
2049
2050 static void
group_term_post_processing(struct panthor_group * group)2051 group_term_post_processing(struct panthor_group *group)
2052 {
2053 struct panthor_job *job, *tmp;
2054 LIST_HEAD(faulty_jobs);
2055 bool cookie;
2056 u32 i = 0;
2057
2058 if (drm_WARN_ON(&group->ptdev->base, group_can_run(group)))
2059 return;
2060
2061 cookie = dma_fence_begin_signalling();
2062 for (i = 0; i < group->queue_count; i++) {
2063 struct panthor_queue *queue = group->queues[i];
2064 struct panthor_syncobj_64b *syncobj;
2065 int err;
2066
2067 if (group->fatal_queues & BIT(i))
2068 err = -EINVAL;
2069 else if (group->timedout)
2070 err = -ETIMEDOUT;
2071 else
2072 err = -ECANCELED;
2073
2074 if (!queue)
2075 continue;
2076
2077 spin_lock(&queue->fence_ctx.lock);
2078 list_for_each_entry_safe(job, tmp, &queue->fence_ctx.in_flight_jobs, node) {
2079 list_move_tail(&job->node, &faulty_jobs);
2080 dma_fence_set_error(job->done_fence, err);
2081 dma_fence_signal_locked(job->done_fence);
2082 }
2083 spin_unlock(&queue->fence_ctx.lock);
2084
2085 /* Manually update the syncobj seqno to unblock waiters. */
2086 syncobj = group->syncobjs->kmap + (i * sizeof(*syncobj));
2087 syncobj->status = ~0;
2088 syncobj->seqno = atomic64_read(&queue->fence_ctx.seqno);
2089 sched_queue_work(group->ptdev->scheduler, sync_upd);
2090 }
2091 dma_fence_end_signalling(cookie);
2092
2093 list_for_each_entry_safe(job, tmp, &faulty_jobs, node) {
2094 list_del_init(&job->node);
2095 panthor_job_put(&job->base);
2096 }
2097 }
2098
group_term_work(struct work_struct * work)2099 static void group_term_work(struct work_struct *work)
2100 {
2101 struct panthor_group *group =
2102 container_of(work, struct panthor_group, term_work);
2103
2104 group_term_post_processing(group);
2105 group_put(group);
2106 }
2107
2108 static void
tick_ctx_cleanup(struct panthor_scheduler * sched,struct panthor_sched_tick_ctx * ctx)2109 tick_ctx_cleanup(struct panthor_scheduler *sched,
2110 struct panthor_sched_tick_ctx *ctx)
2111 {
2112 struct panthor_device *ptdev = sched->ptdev;
2113 struct panthor_group *group, *tmp;
2114 u32 i;
2115
2116 for (i = 0; i < ARRAY_SIZE(ctx->old_groups); i++) {
2117 list_for_each_entry_safe(group, tmp, &ctx->old_groups[i], run_node) {
2118 /* If everything went fine, we should only have groups
2119 * to be terminated in the old_groups lists.
2120 */
2121 drm_WARN_ON(&ptdev->base, !ctx->csg_upd_failed_mask &&
2122 group_can_run(group));
2123
2124 if (!group_can_run(group)) {
2125 list_del_init(&group->run_node);
2126 list_del_init(&group->wait_node);
2127 group_queue_work(group, term);
2128 } else if (group->csg_id >= 0) {
2129 list_del_init(&group->run_node);
2130 } else {
2131 list_move(&group->run_node,
2132 group_is_idle(group) ?
2133 &sched->groups.idle[group->priority] :
2134 &sched->groups.runnable[group->priority]);
2135 }
2136 group_put(group);
2137 }
2138 }
2139
2140 for (i = 0; i < ARRAY_SIZE(ctx->groups); i++) {
2141 /* If everything went fine, the groups to schedule lists should
2142 * be empty.
2143 */
2144 drm_WARN_ON(&ptdev->base,
2145 !ctx->csg_upd_failed_mask && !list_empty(&ctx->groups[i]));
2146
2147 list_for_each_entry_safe(group, tmp, &ctx->groups[i], run_node) {
2148 if (group->csg_id >= 0) {
2149 list_del_init(&group->run_node);
2150 } else {
2151 list_move(&group->run_node,
2152 group_is_idle(group) ?
2153 &sched->groups.idle[group->priority] :
2154 &sched->groups.runnable[group->priority]);
2155 }
2156 group_put(group);
2157 }
2158 }
2159 }
2160
2161 static void
tick_ctx_apply(struct panthor_scheduler * sched,struct panthor_sched_tick_ctx * ctx)2162 tick_ctx_apply(struct panthor_scheduler *sched, struct panthor_sched_tick_ctx *ctx)
2163 {
2164 struct panthor_group *group, *tmp;
2165 struct panthor_device *ptdev = sched->ptdev;
2166 struct panthor_csg_slot *csg_slot;
2167 int prio, new_csg_prio = MAX_CSG_PRIO, i;
2168 u32 free_csg_slots = 0;
2169 struct panthor_csg_slots_upd_ctx upd_ctx;
2170 int ret;
2171
2172 csgs_upd_ctx_init(&upd_ctx);
2173
2174 for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) {
2175 /* Suspend or terminate evicted groups. */
2176 list_for_each_entry(group, &ctx->old_groups[prio], run_node) {
2177 bool term = !group_can_run(group);
2178 int csg_id = group->csg_id;
2179
2180 if (drm_WARN_ON(&ptdev->base, csg_id < 0))
2181 continue;
2182
2183 csg_slot = &sched->csg_slots[csg_id];
2184 csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id,
2185 term ? CSG_STATE_TERMINATE : CSG_STATE_SUSPEND,
2186 CSG_STATE_MASK);
2187 }
2188
2189 /* Update priorities on already running groups. */
2190 list_for_each_entry(group, &ctx->groups[prio], run_node) {
2191 struct panthor_fw_csg_iface *csg_iface;
2192 int csg_id = group->csg_id;
2193
2194 if (csg_id < 0) {
2195 new_csg_prio--;
2196 continue;
2197 }
2198
2199 csg_slot = &sched->csg_slots[csg_id];
2200 csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
2201 if (csg_slot->priority == new_csg_prio) {
2202 new_csg_prio--;
2203 continue;
2204 }
2205
2206 panthor_fw_update_reqs(csg_iface, endpoint_req,
2207 CSG_EP_REQ_PRIORITY(new_csg_prio),
2208 CSG_EP_REQ_PRIORITY_MASK);
2209 csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id,
2210 csg_iface->output->ack ^ CSG_ENDPOINT_CONFIG,
2211 CSG_ENDPOINT_CONFIG);
2212 new_csg_prio--;
2213 }
2214 }
2215
2216 ret = csgs_upd_ctx_apply_locked(ptdev, &upd_ctx);
2217 if (ret) {
2218 panthor_device_schedule_reset(ptdev);
2219 ctx->csg_upd_failed_mask |= upd_ctx.timedout_mask;
2220 return;
2221 }
2222
2223 /* Unbind evicted groups. */
2224 for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) {
2225 list_for_each_entry(group, &ctx->old_groups[prio], run_node) {
2226 /* This group is gone. Process interrupts to clear
2227 * any pending interrupts before we start the new
2228 * group.
2229 */
2230 if (group->csg_id >= 0)
2231 sched_process_csg_irq_locked(ptdev, group->csg_id);
2232
2233 group_unbind_locked(group);
2234 }
2235 }
2236
2237 for (i = 0; i < sched->csg_slot_count; i++) {
2238 if (!sched->csg_slots[i].group)
2239 free_csg_slots |= BIT(i);
2240 }
2241
2242 csgs_upd_ctx_init(&upd_ctx);
2243 new_csg_prio = MAX_CSG_PRIO;
2244
2245 /* Start new groups. */
2246 for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) {
2247 list_for_each_entry(group, &ctx->groups[prio], run_node) {
2248 int csg_id = group->csg_id;
2249 struct panthor_fw_csg_iface *csg_iface;
2250
2251 if (csg_id >= 0) {
2252 new_csg_prio--;
2253 continue;
2254 }
2255
2256 csg_id = ffs(free_csg_slots) - 1;
2257 if (drm_WARN_ON(&ptdev->base, csg_id < 0))
2258 break;
2259
2260 csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id);
2261 csg_slot = &sched->csg_slots[csg_id];
2262 group_bind_locked(group, csg_id);
2263 csg_slot_prog_locked(ptdev, csg_id, new_csg_prio--);
2264 csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id,
2265 group->state == PANTHOR_CS_GROUP_SUSPENDED ?
2266 CSG_STATE_RESUME : CSG_STATE_START,
2267 CSG_STATE_MASK);
2268 csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id,
2269 csg_iface->output->ack ^ CSG_ENDPOINT_CONFIG,
2270 CSG_ENDPOINT_CONFIG);
2271 free_csg_slots &= ~BIT(csg_id);
2272 }
2273 }
2274
2275 ret = csgs_upd_ctx_apply_locked(ptdev, &upd_ctx);
2276 if (ret) {
2277 panthor_device_schedule_reset(ptdev);
2278 ctx->csg_upd_failed_mask |= upd_ctx.timedout_mask;
2279 return;
2280 }
2281
2282 for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) {
2283 list_for_each_entry_safe(group, tmp, &ctx->groups[prio], run_node) {
2284 list_del_init(&group->run_node);
2285
2286 /* If the group has been destroyed while we were
2287 * scheduling, ask for an immediate tick to
2288 * re-evaluate as soon as possible and get rid of
2289 * this dangling group.
2290 */
2291 if (group->destroyed)
2292 ctx->immediate_tick = true;
2293 group_put(group);
2294 }
2295
2296 /* Return evicted groups to the idle or run queues. Groups
2297 * that can no longer be run (because they've been destroyed
2298 * or experienced an unrecoverable error) will be scheduled
2299 * for destruction in tick_ctx_cleanup().
2300 */
2301 list_for_each_entry_safe(group, tmp, &ctx->old_groups[prio], run_node) {
2302 if (!group_can_run(group))
2303 continue;
2304
2305 if (group_is_idle(group))
2306 list_move_tail(&group->run_node, &sched->groups.idle[prio]);
2307 else
2308 list_move_tail(&group->run_node, &sched->groups.runnable[prio]);
2309 group_put(group);
2310 }
2311 }
2312
2313 sched->used_csg_slot_count = ctx->group_count;
2314 sched->might_have_idle_groups = ctx->idle_group_count > 0;
2315 }
2316
2317 static u64
tick_ctx_update_resched_target(struct panthor_scheduler * sched,const struct panthor_sched_tick_ctx * ctx)2318 tick_ctx_update_resched_target(struct panthor_scheduler *sched,
2319 const struct panthor_sched_tick_ctx *ctx)
2320 {
2321 /* We had space left, no need to reschedule until some external event happens. */
2322 if (!tick_ctx_is_full(sched, ctx))
2323 goto no_tick;
2324
2325 /* If idle groups were scheduled, no need to wake up until some external
2326 * event happens (group unblocked, new job submitted, ...).
2327 */
2328 if (ctx->idle_group_count)
2329 goto no_tick;
2330
2331 if (drm_WARN_ON(&sched->ptdev->base, ctx->min_priority >= PANTHOR_CSG_PRIORITY_COUNT))
2332 goto no_tick;
2333
2334 /* If there are groups of the same priority waiting, we need to
2335 * keep the scheduler ticking, otherwise, we'll just wait for
2336 * new groups with higher priority to be queued.
2337 */
2338 if (!list_empty(&sched->groups.runnable[ctx->min_priority])) {
2339 u64 resched_target = sched->last_tick + sched->tick_period;
2340
2341 if (time_before64(sched->resched_target, sched->last_tick) ||
2342 time_before64(resched_target, sched->resched_target))
2343 sched->resched_target = resched_target;
2344
2345 return sched->resched_target - sched->last_tick;
2346 }
2347
2348 no_tick:
2349 sched->resched_target = U64_MAX;
2350 return U64_MAX;
2351 }
2352
tick_work(struct work_struct * work)2353 static void tick_work(struct work_struct *work)
2354 {
2355 struct panthor_scheduler *sched = container_of(work, struct panthor_scheduler,
2356 tick_work.work);
2357 struct panthor_device *ptdev = sched->ptdev;
2358 struct panthor_sched_tick_ctx ctx;
2359 u64 remaining_jiffies = 0, resched_delay;
2360 u64 now = get_jiffies_64();
2361 int prio, ret, cookie;
2362
2363 if (!drm_dev_enter(&ptdev->base, &cookie))
2364 return;
2365
2366 ret = panthor_device_resume_and_get(ptdev);
2367 if (drm_WARN_ON(&ptdev->base, ret))
2368 goto out_dev_exit;
2369
2370 if (time_before64(now, sched->resched_target))
2371 remaining_jiffies = sched->resched_target - now;
2372
2373 mutex_lock(&sched->lock);
2374 if (panthor_device_reset_is_pending(sched->ptdev))
2375 goto out_unlock;
2376
2377 tick_ctx_init(sched, &ctx, remaining_jiffies != 0);
2378 if (ctx.csg_upd_failed_mask)
2379 goto out_cleanup_ctx;
2380
2381 if (remaining_jiffies) {
2382 /* Scheduling forced in the middle of a tick. Only RT groups
2383 * can preempt non-RT ones. Currently running RT groups can't be
2384 * preempted.
2385 */
2386 for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1;
2387 prio >= 0 && !tick_ctx_is_full(sched, &ctx);
2388 prio--) {
2389 tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio],
2390 true, true);
2391 if (prio == PANTHOR_CSG_PRIORITY_RT) {
2392 tick_ctx_pick_groups_from_list(sched, &ctx,
2393 &sched->groups.runnable[prio],
2394 true, false);
2395 }
2396 }
2397 }
2398
2399 /* First pick non-idle groups */
2400 for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1;
2401 prio >= 0 && !tick_ctx_is_full(sched, &ctx);
2402 prio--) {
2403 tick_ctx_pick_groups_from_list(sched, &ctx, &sched->groups.runnable[prio],
2404 true, false);
2405 tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio], true, true);
2406 }
2407
2408 /* If we have free CSG slots left, pick idle groups */
2409 for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1;
2410 prio >= 0 && !tick_ctx_is_full(sched, &ctx);
2411 prio--) {
2412 /* Check the old_group queue first to avoid reprogramming the slots */
2413 tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio], false, true);
2414 tick_ctx_pick_groups_from_list(sched, &ctx, &sched->groups.idle[prio],
2415 false, false);
2416 }
2417
2418 tick_ctx_apply(sched, &ctx);
2419 if (ctx.csg_upd_failed_mask)
2420 goto out_cleanup_ctx;
2421
2422 if (ctx.idle_group_count == ctx.group_count) {
2423 panthor_devfreq_record_idle(sched->ptdev);
2424 if (sched->pm.has_ref) {
2425 pm_runtime_put_autosuspend(ptdev->base.dev);
2426 sched->pm.has_ref = false;
2427 }
2428 } else {
2429 panthor_devfreq_record_busy(sched->ptdev);
2430 if (!sched->pm.has_ref) {
2431 pm_runtime_get(ptdev->base.dev);
2432 sched->pm.has_ref = true;
2433 }
2434 }
2435
2436 sched->last_tick = now;
2437 resched_delay = tick_ctx_update_resched_target(sched, &ctx);
2438 if (ctx.immediate_tick)
2439 resched_delay = 0;
2440
2441 if (resched_delay != U64_MAX)
2442 sched_queue_delayed_work(sched, tick, resched_delay);
2443
2444 out_cleanup_ctx:
2445 tick_ctx_cleanup(sched, &ctx);
2446
2447 out_unlock:
2448 mutex_unlock(&sched->lock);
2449 pm_runtime_mark_last_busy(ptdev->base.dev);
2450 pm_runtime_put_autosuspend(ptdev->base.dev);
2451
2452 out_dev_exit:
2453 drm_dev_exit(cookie);
2454 }
2455
panthor_queue_eval_syncwait(struct panthor_group * group,u8 queue_idx)2456 static int panthor_queue_eval_syncwait(struct panthor_group *group, u8 queue_idx)
2457 {
2458 struct panthor_queue *queue = group->queues[queue_idx];
2459 union {
2460 struct panthor_syncobj_64b sync64;
2461 struct panthor_syncobj_32b sync32;
2462 } *syncobj;
2463 bool result;
2464 u64 value;
2465
2466 syncobj = panthor_queue_get_syncwait_obj(group, queue);
2467 if (!syncobj)
2468 return -EINVAL;
2469
2470 value = queue->syncwait.sync64 ?
2471 syncobj->sync64.seqno :
2472 syncobj->sync32.seqno;
2473
2474 if (queue->syncwait.gt)
2475 result = value > queue->syncwait.ref;
2476 else
2477 result = value <= queue->syncwait.ref;
2478
2479 if (result)
2480 panthor_queue_put_syncwait_obj(queue);
2481
2482 return result;
2483 }
2484
sync_upd_work(struct work_struct * work)2485 static void sync_upd_work(struct work_struct *work)
2486 {
2487 struct panthor_scheduler *sched = container_of(work,
2488 struct panthor_scheduler,
2489 sync_upd_work);
2490 struct panthor_group *group, *tmp;
2491 bool immediate_tick = false;
2492
2493 mutex_lock(&sched->lock);
2494 list_for_each_entry_safe(group, tmp, &sched->groups.waiting, wait_node) {
2495 u32 tested_queues = group->blocked_queues;
2496 u32 unblocked_queues = 0;
2497
2498 while (tested_queues) {
2499 u32 cs_id = ffs(tested_queues) - 1;
2500 int ret;
2501
2502 ret = panthor_queue_eval_syncwait(group, cs_id);
2503 drm_WARN_ON(&group->ptdev->base, ret < 0);
2504 if (ret)
2505 unblocked_queues |= BIT(cs_id);
2506
2507 tested_queues &= ~BIT(cs_id);
2508 }
2509
2510 if (unblocked_queues) {
2511 group->blocked_queues &= ~unblocked_queues;
2512
2513 if (group->csg_id < 0) {
2514 list_move(&group->run_node,
2515 &sched->groups.runnable[group->priority]);
2516 if (group->priority == PANTHOR_CSG_PRIORITY_RT)
2517 immediate_tick = true;
2518 }
2519 }
2520
2521 if (!group->blocked_queues)
2522 list_del_init(&group->wait_node);
2523 }
2524 mutex_unlock(&sched->lock);
2525
2526 if (immediate_tick)
2527 sched_queue_delayed_work(sched, tick, 0);
2528 }
2529
group_schedule_locked(struct panthor_group * group,u32 queue_mask)2530 static void group_schedule_locked(struct panthor_group *group, u32 queue_mask)
2531 {
2532 struct panthor_device *ptdev = group->ptdev;
2533 struct panthor_scheduler *sched = ptdev->scheduler;
2534 struct list_head *queue = &sched->groups.runnable[group->priority];
2535 u64 delay_jiffies = 0;
2536 bool was_idle;
2537 u64 now;
2538
2539 if (!group_can_run(group))
2540 return;
2541
2542 /* All updated queues are blocked, no need to wake up the scheduler. */
2543 if ((queue_mask & group->blocked_queues) == queue_mask)
2544 return;
2545
2546 was_idle = group_is_idle(group);
2547 group->idle_queues &= ~queue_mask;
2548
2549 /* Don't mess up with the lists if we're in a middle of a reset. */
2550 if (atomic_read(&sched->reset.in_progress))
2551 return;
2552
2553 if (was_idle && !group_is_idle(group))
2554 list_move_tail(&group->run_node, queue);
2555
2556 /* RT groups are preemptive. */
2557 if (group->priority == PANTHOR_CSG_PRIORITY_RT) {
2558 sched_queue_delayed_work(sched, tick, 0);
2559 return;
2560 }
2561
2562 /* Some groups might be idle, force an immediate tick to
2563 * re-evaluate.
2564 */
2565 if (sched->might_have_idle_groups) {
2566 sched_queue_delayed_work(sched, tick, 0);
2567 return;
2568 }
2569
2570 /* Scheduler is ticking, nothing to do. */
2571 if (sched->resched_target != U64_MAX) {
2572 /* If there are free slots, force immediating ticking. */
2573 if (sched->used_csg_slot_count < sched->csg_slot_count)
2574 sched_queue_delayed_work(sched, tick, 0);
2575
2576 return;
2577 }
2578
2579 /* Scheduler tick was off, recalculate the resched_target based on the
2580 * last tick event, and queue the scheduler work.
2581 */
2582 now = get_jiffies_64();
2583 sched->resched_target = sched->last_tick + sched->tick_period;
2584 if (sched->used_csg_slot_count == sched->csg_slot_count &&
2585 time_before64(now, sched->resched_target))
2586 delay_jiffies = min_t(unsigned long, sched->resched_target - now, ULONG_MAX);
2587
2588 sched_queue_delayed_work(sched, tick, delay_jiffies);
2589 }
2590
queue_stop(struct panthor_queue * queue,struct panthor_job * bad_job)2591 static void queue_stop(struct panthor_queue *queue,
2592 struct panthor_job *bad_job)
2593 {
2594 drm_sched_stop(&queue->scheduler, bad_job ? &bad_job->base : NULL);
2595 }
2596
queue_start(struct panthor_queue * queue)2597 static void queue_start(struct panthor_queue *queue)
2598 {
2599 struct panthor_job *job;
2600
2601 /* Re-assign the parent fences. */
2602 list_for_each_entry(job, &queue->scheduler.pending_list, base.list)
2603 job->base.s_fence->parent = dma_fence_get(job->done_fence);
2604
2605 drm_sched_start(&queue->scheduler, 0);
2606 }
2607
panthor_group_stop(struct panthor_group * group)2608 static void panthor_group_stop(struct panthor_group *group)
2609 {
2610 struct panthor_scheduler *sched = group->ptdev->scheduler;
2611
2612 lockdep_assert_held(&sched->reset.lock);
2613
2614 for (u32 i = 0; i < group->queue_count; i++)
2615 queue_stop(group->queues[i], NULL);
2616
2617 group_get(group);
2618 list_move_tail(&group->run_node, &sched->reset.stopped_groups);
2619 }
2620
panthor_group_start(struct panthor_group * group)2621 static void panthor_group_start(struct panthor_group *group)
2622 {
2623 struct panthor_scheduler *sched = group->ptdev->scheduler;
2624
2625 lockdep_assert_held(&group->ptdev->scheduler->reset.lock);
2626
2627 for (u32 i = 0; i < group->queue_count; i++)
2628 queue_start(group->queues[i]);
2629
2630 if (group_can_run(group)) {
2631 list_move_tail(&group->run_node,
2632 group_is_idle(group) ?
2633 &sched->groups.idle[group->priority] :
2634 &sched->groups.runnable[group->priority]);
2635 } else {
2636 list_del_init(&group->run_node);
2637 list_del_init(&group->wait_node);
2638 group_queue_work(group, term);
2639 }
2640
2641 group_put(group);
2642 }
2643
panthor_sched_immediate_tick(struct panthor_device * ptdev)2644 static void panthor_sched_immediate_tick(struct panthor_device *ptdev)
2645 {
2646 struct panthor_scheduler *sched = ptdev->scheduler;
2647
2648 sched_queue_delayed_work(sched, tick, 0);
2649 }
2650
2651 /**
2652 * panthor_sched_report_mmu_fault() - Report MMU faults to the scheduler.
2653 */
panthor_sched_report_mmu_fault(struct panthor_device * ptdev)2654 void panthor_sched_report_mmu_fault(struct panthor_device *ptdev)
2655 {
2656 /* Force a tick to immediately kill faulty groups. */
2657 if (ptdev->scheduler)
2658 panthor_sched_immediate_tick(ptdev);
2659 }
2660
panthor_sched_resume(struct panthor_device * ptdev)2661 void panthor_sched_resume(struct panthor_device *ptdev)
2662 {
2663 /* Force a tick to re-evaluate after a resume. */
2664 panthor_sched_immediate_tick(ptdev);
2665 }
2666
panthor_sched_suspend(struct panthor_device * ptdev)2667 void panthor_sched_suspend(struct panthor_device *ptdev)
2668 {
2669 struct panthor_scheduler *sched = ptdev->scheduler;
2670 struct panthor_csg_slots_upd_ctx upd_ctx;
2671 struct panthor_group *group;
2672 u32 suspended_slots;
2673 u32 i;
2674
2675 mutex_lock(&sched->lock);
2676 csgs_upd_ctx_init(&upd_ctx);
2677 for (i = 0; i < sched->csg_slot_count; i++) {
2678 struct panthor_csg_slot *csg_slot = &sched->csg_slots[i];
2679
2680 if (csg_slot->group) {
2681 csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, i,
2682 group_can_run(csg_slot->group) ?
2683 CSG_STATE_SUSPEND : CSG_STATE_TERMINATE,
2684 CSG_STATE_MASK);
2685 }
2686 }
2687
2688 suspended_slots = upd_ctx.update_mask;
2689
2690 csgs_upd_ctx_apply_locked(ptdev, &upd_ctx);
2691 suspended_slots &= ~upd_ctx.timedout_mask;
2692
2693 if (upd_ctx.timedout_mask) {
2694 u32 slot_mask = upd_ctx.timedout_mask;
2695
2696 drm_err(&ptdev->base, "CSG suspend failed, escalating to termination");
2697 csgs_upd_ctx_init(&upd_ctx);
2698 while (slot_mask) {
2699 u32 csg_id = ffs(slot_mask) - 1;
2700 struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id];
2701
2702 /* If the group was still usable before that point, we consider
2703 * it innocent.
2704 */
2705 if (group_can_run(csg_slot->group))
2706 csg_slot->group->innocent = true;
2707
2708 /* We consider group suspension failures as fatal and flag the
2709 * group as unusable by setting timedout=true.
2710 */
2711 csg_slot->group->timedout = true;
2712
2713 csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id,
2714 CSG_STATE_TERMINATE,
2715 CSG_STATE_MASK);
2716 slot_mask &= ~BIT(csg_id);
2717 }
2718
2719 csgs_upd_ctx_apply_locked(ptdev, &upd_ctx);
2720
2721 slot_mask = upd_ctx.timedout_mask;
2722 while (slot_mask) {
2723 u32 csg_id = ffs(slot_mask) - 1;
2724 struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id];
2725
2726 /* Terminate command timedout, but the soft-reset will
2727 * automatically terminate all active groups, so let's
2728 * force the state to halted here.
2729 */
2730 if (csg_slot->group->state != PANTHOR_CS_GROUP_TERMINATED)
2731 csg_slot->group->state = PANTHOR_CS_GROUP_TERMINATED;
2732 slot_mask &= ~BIT(csg_id);
2733 }
2734 }
2735
2736 /* Flush L2 and LSC caches to make sure suspend state is up-to-date.
2737 * If the flush fails, flag all queues for termination.
2738 */
2739 if (suspended_slots) {
2740 bool flush_caches_failed = false;
2741 u32 slot_mask = suspended_slots;
2742
2743 if (panthor_gpu_flush_caches(ptdev, CACHE_CLEAN, CACHE_CLEAN, 0))
2744 flush_caches_failed = true;
2745
2746 while (slot_mask) {
2747 u32 csg_id = ffs(slot_mask) - 1;
2748 struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id];
2749
2750 if (flush_caches_failed)
2751 csg_slot->group->state = PANTHOR_CS_GROUP_TERMINATED;
2752 else
2753 csg_slot_sync_update_locked(ptdev, csg_id);
2754
2755 slot_mask &= ~BIT(csg_id);
2756 }
2757 }
2758
2759 for (i = 0; i < sched->csg_slot_count; i++) {
2760 struct panthor_csg_slot *csg_slot = &sched->csg_slots[i];
2761
2762 group = csg_slot->group;
2763 if (!group)
2764 continue;
2765
2766 group_get(group);
2767
2768 if (group->csg_id >= 0)
2769 sched_process_csg_irq_locked(ptdev, group->csg_id);
2770
2771 group_unbind_locked(group);
2772
2773 drm_WARN_ON(&group->ptdev->base, !list_empty(&group->run_node));
2774
2775 if (group_can_run(group)) {
2776 list_add(&group->run_node,
2777 &sched->groups.idle[group->priority]);
2778 } else {
2779 /* We don't bother stopping the scheduler if the group is
2780 * faulty, the group termination work will finish the job.
2781 */
2782 list_del_init(&group->wait_node);
2783 group_queue_work(group, term);
2784 }
2785 group_put(group);
2786 }
2787 mutex_unlock(&sched->lock);
2788 }
2789
panthor_sched_pre_reset(struct panthor_device * ptdev)2790 void panthor_sched_pre_reset(struct panthor_device *ptdev)
2791 {
2792 struct panthor_scheduler *sched = ptdev->scheduler;
2793 struct panthor_group *group, *group_tmp;
2794 u32 i;
2795
2796 mutex_lock(&sched->reset.lock);
2797 atomic_set(&sched->reset.in_progress, true);
2798
2799 /* Cancel all scheduler works. Once this is done, these works can't be
2800 * scheduled again until the reset operation is complete.
2801 */
2802 cancel_work_sync(&sched->sync_upd_work);
2803 cancel_delayed_work_sync(&sched->tick_work);
2804
2805 panthor_sched_suspend(ptdev);
2806
2807 /* Stop all groups that might still accept jobs, so we don't get passed
2808 * new jobs while we're resetting.
2809 */
2810 for (i = 0; i < ARRAY_SIZE(sched->groups.runnable); i++) {
2811 /* All groups should be in the idle lists. */
2812 drm_WARN_ON(&ptdev->base, !list_empty(&sched->groups.runnable[i]));
2813 list_for_each_entry_safe(group, group_tmp, &sched->groups.runnable[i], run_node)
2814 panthor_group_stop(group);
2815 }
2816
2817 for (i = 0; i < ARRAY_SIZE(sched->groups.idle); i++) {
2818 list_for_each_entry_safe(group, group_tmp, &sched->groups.idle[i], run_node)
2819 panthor_group_stop(group);
2820 }
2821
2822 mutex_unlock(&sched->reset.lock);
2823 }
2824
panthor_sched_post_reset(struct panthor_device * ptdev,bool reset_failed)2825 void panthor_sched_post_reset(struct panthor_device *ptdev, bool reset_failed)
2826 {
2827 struct panthor_scheduler *sched = ptdev->scheduler;
2828 struct panthor_group *group, *group_tmp;
2829
2830 mutex_lock(&sched->reset.lock);
2831
2832 list_for_each_entry_safe(group, group_tmp, &sched->reset.stopped_groups, run_node) {
2833 /* Consider all previously running group as terminated if the
2834 * reset failed.
2835 */
2836 if (reset_failed)
2837 group->state = PANTHOR_CS_GROUP_TERMINATED;
2838
2839 panthor_group_start(group);
2840 }
2841
2842 /* We're done resetting the GPU, clear the reset.in_progress bit so we can
2843 * kick the scheduler.
2844 */
2845 atomic_set(&sched->reset.in_progress, false);
2846 mutex_unlock(&sched->reset.lock);
2847
2848 /* No need to queue a tick and update syncs if the reset failed. */
2849 if (!reset_failed) {
2850 sched_queue_delayed_work(sched, tick, 0);
2851 sched_queue_work(sched, sync_upd);
2852 }
2853 }
2854
update_fdinfo_stats(struct panthor_job * job)2855 static void update_fdinfo_stats(struct panthor_job *job)
2856 {
2857 struct panthor_group *group = job->group;
2858 struct panthor_queue *queue = group->queues[job->queue_idx];
2859 struct panthor_gpu_usage *fdinfo = &group->fdinfo.data;
2860 struct panthor_job_profiling_data *slots = queue->profiling.slots->kmap;
2861 struct panthor_job_profiling_data *data = &slots[job->profiling.slot];
2862
2863 scoped_guard(spinlock, &group->fdinfo.lock) {
2864 if (job->profiling.mask & PANTHOR_DEVICE_PROFILING_CYCLES)
2865 fdinfo->cycles += data->cycles.after - data->cycles.before;
2866 if (job->profiling.mask & PANTHOR_DEVICE_PROFILING_TIMESTAMP)
2867 fdinfo->time += data->time.after - data->time.before;
2868 }
2869 }
2870
panthor_fdinfo_gather_group_samples(struct panthor_file * pfile)2871 void panthor_fdinfo_gather_group_samples(struct panthor_file *pfile)
2872 {
2873 struct panthor_group_pool *gpool = pfile->groups;
2874 struct panthor_group *group;
2875 unsigned long i;
2876
2877 if (IS_ERR_OR_NULL(gpool))
2878 return;
2879
2880 xa_lock(&gpool->xa);
2881 xa_for_each(&gpool->xa, i, group) {
2882 guard(spinlock)(&group->fdinfo.lock);
2883 pfile->stats.cycles += group->fdinfo.data.cycles;
2884 pfile->stats.time += group->fdinfo.data.time;
2885 group->fdinfo.data.cycles = 0;
2886 group->fdinfo.data.time = 0;
2887 }
2888 xa_unlock(&gpool->xa);
2889 }
2890
group_sync_upd_work(struct work_struct * work)2891 static void group_sync_upd_work(struct work_struct *work)
2892 {
2893 struct panthor_group *group =
2894 container_of(work, struct panthor_group, sync_upd_work);
2895 struct panthor_job *job, *job_tmp;
2896 LIST_HEAD(done_jobs);
2897 u32 queue_idx;
2898 bool cookie;
2899
2900 cookie = dma_fence_begin_signalling();
2901 for (queue_idx = 0; queue_idx < group->queue_count; queue_idx++) {
2902 struct panthor_queue *queue = group->queues[queue_idx];
2903 struct panthor_syncobj_64b *syncobj;
2904
2905 if (!queue)
2906 continue;
2907
2908 syncobj = group->syncobjs->kmap + (queue_idx * sizeof(*syncobj));
2909
2910 spin_lock(&queue->fence_ctx.lock);
2911 list_for_each_entry_safe(job, job_tmp, &queue->fence_ctx.in_flight_jobs, node) {
2912 if (syncobj->seqno < job->done_fence->seqno)
2913 break;
2914
2915 list_move_tail(&job->node, &done_jobs);
2916 dma_fence_signal_locked(job->done_fence);
2917 }
2918 spin_unlock(&queue->fence_ctx.lock);
2919 }
2920 dma_fence_end_signalling(cookie);
2921
2922 list_for_each_entry_safe(job, job_tmp, &done_jobs, node) {
2923 if (job->profiling.mask)
2924 update_fdinfo_stats(job);
2925 list_del_init(&job->node);
2926 panthor_job_put(&job->base);
2927 }
2928
2929 group_put(group);
2930 }
2931
2932 struct panthor_job_ringbuf_instrs {
2933 u64 buffer[MAX_INSTRS_PER_JOB];
2934 u32 count;
2935 };
2936
2937 struct panthor_job_instr {
2938 u32 profile_mask;
2939 u64 instr;
2940 };
2941
2942 #define JOB_INSTR(__prof, __instr) \
2943 { \
2944 .profile_mask = __prof, \
2945 .instr = __instr, \
2946 }
2947
2948 static void
copy_instrs_to_ringbuf(struct panthor_queue * queue,struct panthor_job * job,struct panthor_job_ringbuf_instrs * instrs)2949 copy_instrs_to_ringbuf(struct panthor_queue *queue,
2950 struct panthor_job *job,
2951 struct panthor_job_ringbuf_instrs *instrs)
2952 {
2953 u64 ringbuf_size = panthor_kernel_bo_size(queue->ringbuf);
2954 u64 start = job->ringbuf.start & (ringbuf_size - 1);
2955 u64 size, written;
2956
2957 /*
2958 * We need to write a whole slot, including any trailing zeroes
2959 * that may come at the end of it. Also, because instrs.buffer has
2960 * been zero-initialised, there's no need to pad it with 0's
2961 */
2962 instrs->count = ALIGN(instrs->count, NUM_INSTRS_PER_CACHE_LINE);
2963 size = instrs->count * sizeof(u64);
2964 WARN_ON(size > ringbuf_size);
2965 written = min(ringbuf_size - start, size);
2966
2967 memcpy(queue->ringbuf->kmap + start, instrs->buffer, written);
2968
2969 if (written < size)
2970 memcpy(queue->ringbuf->kmap,
2971 &instrs->buffer[written / sizeof(u64)],
2972 size - written);
2973 }
2974
2975 struct panthor_job_cs_params {
2976 u32 profile_mask;
2977 u64 addr_reg; u64 val_reg;
2978 u64 cycle_reg; u64 time_reg;
2979 u64 sync_addr; u64 times_addr;
2980 u64 cs_start; u64 cs_size;
2981 u32 last_flush; u32 waitall_mask;
2982 };
2983
2984 static void
get_job_cs_params(struct panthor_job * job,struct panthor_job_cs_params * params)2985 get_job_cs_params(struct panthor_job *job, struct panthor_job_cs_params *params)
2986 {
2987 struct panthor_group *group = job->group;
2988 struct panthor_queue *queue = group->queues[job->queue_idx];
2989 struct panthor_device *ptdev = group->ptdev;
2990 struct panthor_scheduler *sched = ptdev->scheduler;
2991
2992 params->addr_reg = ptdev->csif_info.cs_reg_count -
2993 ptdev->csif_info.unpreserved_cs_reg_count;
2994 params->val_reg = params->addr_reg + 2;
2995 params->cycle_reg = params->addr_reg;
2996 params->time_reg = params->val_reg;
2997
2998 params->sync_addr = panthor_kernel_bo_gpuva(group->syncobjs) +
2999 job->queue_idx * sizeof(struct panthor_syncobj_64b);
3000 params->times_addr = panthor_kernel_bo_gpuva(queue->profiling.slots) +
3001 (job->profiling.slot * sizeof(struct panthor_job_profiling_data));
3002 params->waitall_mask = GENMASK(sched->sb_slot_count - 1, 0);
3003
3004 params->cs_start = job->call_info.start;
3005 params->cs_size = job->call_info.size;
3006 params->last_flush = job->call_info.latest_flush;
3007
3008 params->profile_mask = job->profiling.mask;
3009 }
3010
3011 #define JOB_INSTR_ALWAYS(instr) \
3012 JOB_INSTR(PANTHOR_DEVICE_PROFILING_DISABLED, (instr))
3013 #define JOB_INSTR_TIMESTAMP(instr) \
3014 JOB_INSTR(PANTHOR_DEVICE_PROFILING_TIMESTAMP, (instr))
3015 #define JOB_INSTR_CYCLES(instr) \
3016 JOB_INSTR(PANTHOR_DEVICE_PROFILING_CYCLES, (instr))
3017
3018 static void
prepare_job_instrs(const struct panthor_job_cs_params * params,struct panthor_job_ringbuf_instrs * instrs)3019 prepare_job_instrs(const struct panthor_job_cs_params *params,
3020 struct panthor_job_ringbuf_instrs *instrs)
3021 {
3022 const struct panthor_job_instr instr_seq[] = {
3023 /* MOV32 rX+2, cs.latest_flush */
3024 JOB_INSTR_ALWAYS((2ull << 56) | (params->val_reg << 48) | params->last_flush),
3025 /* FLUSH_CACHE2.clean_inv_all.no_wait.signal(0) rX+2 */
3026 JOB_INSTR_ALWAYS((36ull << 56) | (0ull << 48) | (params->val_reg << 40) |
3027 (0 << 16) | 0x233),
3028 /* MOV48 rX:rX+1, cycles_offset */
3029 JOB_INSTR_CYCLES((1ull << 56) | (params->cycle_reg << 48) |
3030 (params->times_addr +
3031 offsetof(struct panthor_job_profiling_data, cycles.before))),
3032 /* STORE_STATE cycles */
3033 JOB_INSTR_CYCLES((40ull << 56) | (params->cycle_reg << 40) | (1ll << 32)),
3034 /* MOV48 rX:rX+1, time_offset */
3035 JOB_INSTR_TIMESTAMP((1ull << 56) | (params->time_reg << 48) |
3036 (params->times_addr +
3037 offsetof(struct panthor_job_profiling_data, time.before))),
3038 /* STORE_STATE timer */
3039 JOB_INSTR_TIMESTAMP((40ull << 56) | (params->time_reg << 40) | (0ll << 32)),
3040 /* MOV48 rX:rX+1, cs.start */
3041 JOB_INSTR_ALWAYS((1ull << 56) | (params->addr_reg << 48) | params->cs_start),
3042 /* MOV32 rX+2, cs.size */
3043 JOB_INSTR_ALWAYS((2ull << 56) | (params->val_reg << 48) | params->cs_size),
3044 /* WAIT(0) => waits for FLUSH_CACHE2 instruction */
3045 JOB_INSTR_ALWAYS((3ull << 56) | (1 << 16)),
3046 /* CALL rX:rX+1, rX+2 */
3047 JOB_INSTR_ALWAYS((32ull << 56) | (params->addr_reg << 40) |
3048 (params->val_reg << 32)),
3049 /* MOV48 rX:rX+1, cycles_offset */
3050 JOB_INSTR_CYCLES((1ull << 56) | (params->cycle_reg << 48) |
3051 (params->times_addr +
3052 offsetof(struct panthor_job_profiling_data, cycles.after))),
3053 /* STORE_STATE cycles */
3054 JOB_INSTR_CYCLES((40ull << 56) | (params->cycle_reg << 40) | (1ll << 32)),
3055 /* MOV48 rX:rX+1, time_offset */
3056 JOB_INSTR_TIMESTAMP((1ull << 56) | (params->time_reg << 48) |
3057 (params->times_addr +
3058 offsetof(struct panthor_job_profiling_data, time.after))),
3059 /* STORE_STATE timer */
3060 JOB_INSTR_TIMESTAMP((40ull << 56) | (params->time_reg << 40) | (0ll << 32)),
3061 /* MOV48 rX:rX+1, sync_addr */
3062 JOB_INSTR_ALWAYS((1ull << 56) | (params->addr_reg << 48) | params->sync_addr),
3063 /* MOV48 rX+2, #1 */
3064 JOB_INSTR_ALWAYS((1ull << 56) | (params->val_reg << 48) | 1),
3065 /* WAIT(all) */
3066 JOB_INSTR_ALWAYS((3ull << 56) | (params->waitall_mask << 16)),
3067 /* SYNC_ADD64.system_scope.propage_err.nowait rX:rX+1, rX+2*/
3068 JOB_INSTR_ALWAYS((51ull << 56) | (0ull << 48) | (params->addr_reg << 40) |
3069 (params->val_reg << 32) | (0 << 16) | 1),
3070 /* ERROR_BARRIER, so we can recover from faults at job boundaries. */
3071 JOB_INSTR_ALWAYS((47ull << 56)),
3072 };
3073 u32 pad;
3074
3075 instrs->count = 0;
3076
3077 /* NEED to be cacheline aligned to please the prefetcher. */
3078 static_assert(sizeof(instrs->buffer) % 64 == 0,
3079 "panthor_job_ringbuf_instrs::buffer is not aligned on a cacheline");
3080
3081 /* Make sure we have enough storage to store the whole sequence. */
3082 static_assert(ALIGN(ARRAY_SIZE(instr_seq), NUM_INSTRS_PER_CACHE_LINE) ==
3083 ARRAY_SIZE(instrs->buffer),
3084 "instr_seq vs panthor_job_ringbuf_instrs::buffer size mismatch");
3085
3086 for (u32 i = 0; i < ARRAY_SIZE(instr_seq); i++) {
3087 /* If the profile mask of this instruction is not enabled, skip it. */
3088 if (instr_seq[i].profile_mask &&
3089 !(instr_seq[i].profile_mask & params->profile_mask))
3090 continue;
3091
3092 instrs->buffer[instrs->count++] = instr_seq[i].instr;
3093 }
3094
3095 pad = ALIGN(instrs->count, NUM_INSTRS_PER_CACHE_LINE);
3096 memset(&instrs->buffer[instrs->count], 0,
3097 (pad - instrs->count) * sizeof(instrs->buffer[0]));
3098 instrs->count = pad;
3099 }
3100
calc_job_credits(u32 profile_mask)3101 static u32 calc_job_credits(u32 profile_mask)
3102 {
3103 struct panthor_job_ringbuf_instrs instrs;
3104 struct panthor_job_cs_params params = {
3105 .profile_mask = profile_mask,
3106 };
3107
3108 prepare_job_instrs(¶ms, &instrs);
3109 return instrs.count;
3110 }
3111
3112 static struct dma_fence *
queue_run_job(struct drm_sched_job * sched_job)3113 queue_run_job(struct drm_sched_job *sched_job)
3114 {
3115 struct panthor_job *job = container_of(sched_job, struct panthor_job, base);
3116 struct panthor_group *group = job->group;
3117 struct panthor_queue *queue = group->queues[job->queue_idx];
3118 struct panthor_device *ptdev = group->ptdev;
3119 struct panthor_scheduler *sched = ptdev->scheduler;
3120 struct panthor_job_ringbuf_instrs instrs;
3121 struct panthor_job_cs_params cs_params;
3122 struct dma_fence *done_fence;
3123 int ret;
3124
3125 /* Stream size is zero, nothing to do except making sure all previously
3126 * submitted jobs are done before we signal the
3127 * drm_sched_job::s_fence::finished fence.
3128 */
3129 if (!job->call_info.size) {
3130 job->done_fence = dma_fence_get(queue->fence_ctx.last_fence);
3131 return dma_fence_get(job->done_fence);
3132 }
3133
3134 ret = panthor_device_resume_and_get(ptdev);
3135 if (drm_WARN_ON(&ptdev->base, ret))
3136 return ERR_PTR(ret);
3137
3138 mutex_lock(&sched->lock);
3139 if (!group_can_run(group)) {
3140 done_fence = ERR_PTR(-ECANCELED);
3141 goto out_unlock;
3142 }
3143
3144 dma_fence_init(job->done_fence,
3145 &panthor_queue_fence_ops,
3146 &queue->fence_ctx.lock,
3147 queue->fence_ctx.id,
3148 atomic64_inc_return(&queue->fence_ctx.seqno));
3149
3150 job->profiling.slot = queue->profiling.seqno++;
3151 if (queue->profiling.seqno == queue->profiling.slot_count)
3152 queue->profiling.seqno = 0;
3153
3154 job->ringbuf.start = queue->iface.input->insert;
3155
3156 get_job_cs_params(job, &cs_params);
3157 prepare_job_instrs(&cs_params, &instrs);
3158 copy_instrs_to_ringbuf(queue, job, &instrs);
3159
3160 job->ringbuf.end = job->ringbuf.start + (instrs.count * sizeof(u64));
3161
3162 panthor_job_get(&job->base);
3163 spin_lock(&queue->fence_ctx.lock);
3164 list_add_tail(&job->node, &queue->fence_ctx.in_flight_jobs);
3165 spin_unlock(&queue->fence_ctx.lock);
3166
3167 /* Make sure the ring buffer is updated before the INSERT
3168 * register.
3169 */
3170 wmb();
3171
3172 queue->iface.input->extract = queue->iface.output->extract;
3173 queue->iface.input->insert = job->ringbuf.end;
3174
3175 if (group->csg_id < 0) {
3176 /* If the queue is blocked, we want to keep the timeout running, so we
3177 * can detect unbounded waits and kill the group when that happens.
3178 * Otherwise, we suspend the timeout so the time we spend waiting for
3179 * a CSG slot is not counted.
3180 */
3181 if (!(group->blocked_queues & BIT(job->queue_idx)) &&
3182 !queue->timeout_suspended) {
3183 queue->remaining_time = drm_sched_suspend_timeout(&queue->scheduler);
3184 queue->timeout_suspended = true;
3185 }
3186
3187 group_schedule_locked(group, BIT(job->queue_idx));
3188 } else {
3189 gpu_write(ptdev, CSF_DOORBELL(queue->doorbell_id), 1);
3190 if (!sched->pm.has_ref &&
3191 !(group->blocked_queues & BIT(job->queue_idx))) {
3192 pm_runtime_get(ptdev->base.dev);
3193 sched->pm.has_ref = true;
3194 }
3195 panthor_devfreq_record_busy(sched->ptdev);
3196 }
3197
3198 /* Update the last fence. */
3199 dma_fence_put(queue->fence_ctx.last_fence);
3200 queue->fence_ctx.last_fence = dma_fence_get(job->done_fence);
3201
3202 done_fence = dma_fence_get(job->done_fence);
3203
3204 out_unlock:
3205 mutex_unlock(&sched->lock);
3206 pm_runtime_mark_last_busy(ptdev->base.dev);
3207 pm_runtime_put_autosuspend(ptdev->base.dev);
3208
3209 return done_fence;
3210 }
3211
3212 static enum drm_gpu_sched_stat
queue_timedout_job(struct drm_sched_job * sched_job)3213 queue_timedout_job(struct drm_sched_job *sched_job)
3214 {
3215 struct panthor_job *job = container_of(sched_job, struct panthor_job, base);
3216 struct panthor_group *group = job->group;
3217 struct panthor_device *ptdev = group->ptdev;
3218 struct panthor_scheduler *sched = ptdev->scheduler;
3219 struct panthor_queue *queue = group->queues[job->queue_idx];
3220
3221 drm_warn(&ptdev->base, "job timeout\n");
3222
3223 drm_WARN_ON(&ptdev->base, atomic_read(&sched->reset.in_progress));
3224
3225 queue_stop(queue, job);
3226
3227 mutex_lock(&sched->lock);
3228 group->timedout = true;
3229 if (group->csg_id >= 0) {
3230 sched_queue_delayed_work(ptdev->scheduler, tick, 0);
3231 } else {
3232 /* Remove from the run queues, so the scheduler can't
3233 * pick the group on the next tick.
3234 */
3235 list_del_init(&group->run_node);
3236 list_del_init(&group->wait_node);
3237
3238 group_queue_work(group, term);
3239 }
3240 mutex_unlock(&sched->lock);
3241
3242 queue_start(queue);
3243
3244 return DRM_GPU_SCHED_STAT_NOMINAL;
3245 }
3246
queue_free_job(struct drm_sched_job * sched_job)3247 static void queue_free_job(struct drm_sched_job *sched_job)
3248 {
3249 drm_sched_job_cleanup(sched_job);
3250 panthor_job_put(sched_job);
3251 }
3252
3253 static const struct drm_sched_backend_ops panthor_queue_sched_ops = {
3254 .run_job = queue_run_job,
3255 .timedout_job = queue_timedout_job,
3256 .free_job = queue_free_job,
3257 };
3258
calc_profiling_ringbuf_num_slots(struct panthor_device * ptdev,u32 cs_ringbuf_size)3259 static u32 calc_profiling_ringbuf_num_slots(struct panthor_device *ptdev,
3260 u32 cs_ringbuf_size)
3261 {
3262 u32 min_profiled_job_instrs = U32_MAX;
3263 u32 last_flag = fls(PANTHOR_DEVICE_PROFILING_ALL);
3264
3265 /*
3266 * We want to calculate the minimum size of a profiled job's CS,
3267 * because since they need additional instructions for the sampling
3268 * of performance metrics, they might take up further slots in
3269 * the queue's ringbuffer. This means we might not need as many job
3270 * slots for keeping track of their profiling information. What we
3271 * need is the maximum number of slots we should allocate to this end,
3272 * which matches the maximum number of profiled jobs we can place
3273 * simultaneously in the queue's ring buffer.
3274 * That has to be calculated separately for every single job profiling
3275 * flag, but not in the case job profiling is disabled, since unprofiled
3276 * jobs don't need to keep track of this at all.
3277 */
3278 for (u32 i = 0; i < last_flag; i++) {
3279 min_profiled_job_instrs =
3280 min(min_profiled_job_instrs, calc_job_credits(BIT(i)));
3281 }
3282
3283 return DIV_ROUND_UP(cs_ringbuf_size, min_profiled_job_instrs * sizeof(u64));
3284 }
3285
3286 static struct panthor_queue *
group_create_queue(struct panthor_group * group,const struct drm_panthor_queue_create * args)3287 group_create_queue(struct panthor_group *group,
3288 const struct drm_panthor_queue_create *args)
3289 {
3290 const struct drm_sched_init_args sched_args = {
3291 .ops = &panthor_queue_sched_ops,
3292 .submit_wq = group->ptdev->scheduler->wq,
3293 .num_rqs = 1,
3294 /*
3295 * The credit limit argument tells us the total number of
3296 * instructions across all CS slots in the ringbuffer, with
3297 * some jobs requiring twice as many as others, depending on
3298 * their profiling status.
3299 */
3300 .credit_limit = args->ringbuf_size / sizeof(u64),
3301 .timeout = msecs_to_jiffies(JOB_TIMEOUT_MS),
3302 .timeout_wq = group->ptdev->reset.wq,
3303 .name = "panthor-queue",
3304 .dev = group->ptdev->base.dev,
3305 };
3306 struct drm_gpu_scheduler *drm_sched;
3307 struct panthor_queue *queue;
3308 int ret;
3309
3310 if (args->pad[0] || args->pad[1] || args->pad[2])
3311 return ERR_PTR(-EINVAL);
3312
3313 if (args->ringbuf_size < SZ_4K || args->ringbuf_size > SZ_64K ||
3314 !is_power_of_2(args->ringbuf_size))
3315 return ERR_PTR(-EINVAL);
3316
3317 if (args->priority > CSF_MAX_QUEUE_PRIO)
3318 return ERR_PTR(-EINVAL);
3319
3320 queue = kzalloc(sizeof(*queue), GFP_KERNEL);
3321 if (!queue)
3322 return ERR_PTR(-ENOMEM);
3323
3324 queue->fence_ctx.id = dma_fence_context_alloc(1);
3325 spin_lock_init(&queue->fence_ctx.lock);
3326 INIT_LIST_HEAD(&queue->fence_ctx.in_flight_jobs);
3327
3328 queue->priority = args->priority;
3329
3330 queue->ringbuf = panthor_kernel_bo_create(group->ptdev, group->vm,
3331 args->ringbuf_size,
3332 DRM_PANTHOR_BO_NO_MMAP,
3333 DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC |
3334 DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED,
3335 PANTHOR_VM_KERNEL_AUTO_VA);
3336 if (IS_ERR(queue->ringbuf)) {
3337 ret = PTR_ERR(queue->ringbuf);
3338 goto err_free_queue;
3339 }
3340
3341 ret = panthor_kernel_bo_vmap(queue->ringbuf);
3342 if (ret)
3343 goto err_free_queue;
3344
3345 queue->iface.mem = panthor_fw_alloc_queue_iface_mem(group->ptdev,
3346 &queue->iface.input,
3347 &queue->iface.output,
3348 &queue->iface.input_fw_va,
3349 &queue->iface.output_fw_va);
3350 if (IS_ERR(queue->iface.mem)) {
3351 ret = PTR_ERR(queue->iface.mem);
3352 goto err_free_queue;
3353 }
3354
3355 queue->profiling.slot_count =
3356 calc_profiling_ringbuf_num_slots(group->ptdev, args->ringbuf_size);
3357
3358 queue->profiling.slots =
3359 panthor_kernel_bo_create(group->ptdev, group->vm,
3360 queue->profiling.slot_count *
3361 sizeof(struct panthor_job_profiling_data),
3362 DRM_PANTHOR_BO_NO_MMAP,
3363 DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC |
3364 DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED,
3365 PANTHOR_VM_KERNEL_AUTO_VA);
3366
3367 if (IS_ERR(queue->profiling.slots)) {
3368 ret = PTR_ERR(queue->profiling.slots);
3369 goto err_free_queue;
3370 }
3371
3372 ret = panthor_kernel_bo_vmap(queue->profiling.slots);
3373 if (ret)
3374 goto err_free_queue;
3375
3376 ret = drm_sched_init(&queue->scheduler, &sched_args);
3377 if (ret)
3378 goto err_free_queue;
3379
3380 drm_sched = &queue->scheduler;
3381 ret = drm_sched_entity_init(&queue->entity, 0, &drm_sched, 1, NULL);
3382
3383 return queue;
3384
3385 err_free_queue:
3386 group_free_queue(group, queue);
3387 return ERR_PTR(ret);
3388 }
3389
add_group_kbo_sizes(struct panthor_device * ptdev,struct panthor_group * group)3390 static void add_group_kbo_sizes(struct panthor_device *ptdev,
3391 struct panthor_group *group)
3392 {
3393 struct panthor_queue *queue;
3394 int i;
3395
3396 if (drm_WARN_ON(&ptdev->base, IS_ERR_OR_NULL(group)))
3397 return;
3398 if (drm_WARN_ON(&ptdev->base, ptdev != group->ptdev))
3399 return;
3400
3401 group->fdinfo.kbo_sizes += group->suspend_buf->obj->size;
3402 group->fdinfo.kbo_sizes += group->protm_suspend_buf->obj->size;
3403 group->fdinfo.kbo_sizes += group->syncobjs->obj->size;
3404
3405 for (i = 0; i < group->queue_count; i++) {
3406 queue = group->queues[i];
3407 group->fdinfo.kbo_sizes += queue->ringbuf->obj->size;
3408 group->fdinfo.kbo_sizes += queue->iface.mem->obj->size;
3409 group->fdinfo.kbo_sizes += queue->profiling.slots->obj->size;
3410 }
3411 }
3412
3413 #define MAX_GROUPS_PER_POOL 128
3414
panthor_group_create(struct panthor_file * pfile,const struct drm_panthor_group_create * group_args,const struct drm_panthor_queue_create * queue_args)3415 int panthor_group_create(struct panthor_file *pfile,
3416 const struct drm_panthor_group_create *group_args,
3417 const struct drm_panthor_queue_create *queue_args)
3418 {
3419 struct panthor_device *ptdev = pfile->ptdev;
3420 struct panthor_group_pool *gpool = pfile->groups;
3421 struct panthor_scheduler *sched = ptdev->scheduler;
3422 struct panthor_fw_csg_iface *csg_iface = panthor_fw_get_csg_iface(ptdev, 0);
3423 struct panthor_group *group = NULL;
3424 u32 gid, i, suspend_size;
3425 int ret;
3426
3427 if (group_args->pad)
3428 return -EINVAL;
3429
3430 if (group_args->priority >= PANTHOR_CSG_PRIORITY_COUNT)
3431 return -EINVAL;
3432
3433 if ((group_args->compute_core_mask & ~ptdev->gpu_info.shader_present) ||
3434 (group_args->fragment_core_mask & ~ptdev->gpu_info.shader_present) ||
3435 (group_args->tiler_core_mask & ~ptdev->gpu_info.tiler_present))
3436 return -EINVAL;
3437
3438 if (hweight64(group_args->compute_core_mask) < group_args->max_compute_cores ||
3439 hweight64(group_args->fragment_core_mask) < group_args->max_fragment_cores ||
3440 hweight64(group_args->tiler_core_mask) < group_args->max_tiler_cores)
3441 return -EINVAL;
3442
3443 group = kzalloc(sizeof(*group), GFP_KERNEL);
3444 if (!group)
3445 return -ENOMEM;
3446
3447 spin_lock_init(&group->fatal_lock);
3448 kref_init(&group->refcount);
3449 group->state = PANTHOR_CS_GROUP_CREATED;
3450 group->csg_id = -1;
3451
3452 group->ptdev = ptdev;
3453 group->max_compute_cores = group_args->max_compute_cores;
3454 group->compute_core_mask = group_args->compute_core_mask;
3455 group->max_fragment_cores = group_args->max_fragment_cores;
3456 group->fragment_core_mask = group_args->fragment_core_mask;
3457 group->max_tiler_cores = group_args->max_tiler_cores;
3458 group->tiler_core_mask = group_args->tiler_core_mask;
3459 group->priority = group_args->priority;
3460
3461 INIT_LIST_HEAD(&group->wait_node);
3462 INIT_LIST_HEAD(&group->run_node);
3463 INIT_WORK(&group->term_work, group_term_work);
3464 INIT_WORK(&group->sync_upd_work, group_sync_upd_work);
3465 INIT_WORK(&group->tiler_oom_work, group_tiler_oom_work);
3466 INIT_WORK(&group->release_work, group_release_work);
3467
3468 group->vm = panthor_vm_pool_get_vm(pfile->vms, group_args->vm_id);
3469 if (!group->vm) {
3470 ret = -EINVAL;
3471 goto err_put_group;
3472 }
3473
3474 suspend_size = csg_iface->control->suspend_size;
3475 group->suspend_buf = panthor_fw_alloc_suspend_buf_mem(ptdev, suspend_size);
3476 if (IS_ERR(group->suspend_buf)) {
3477 ret = PTR_ERR(group->suspend_buf);
3478 group->suspend_buf = NULL;
3479 goto err_put_group;
3480 }
3481
3482 suspend_size = csg_iface->control->protm_suspend_size;
3483 group->protm_suspend_buf = panthor_fw_alloc_suspend_buf_mem(ptdev, suspend_size);
3484 if (IS_ERR(group->protm_suspend_buf)) {
3485 ret = PTR_ERR(group->protm_suspend_buf);
3486 group->protm_suspend_buf = NULL;
3487 goto err_put_group;
3488 }
3489
3490 group->syncobjs = panthor_kernel_bo_create(ptdev, group->vm,
3491 group_args->queues.count *
3492 sizeof(struct panthor_syncobj_64b),
3493 DRM_PANTHOR_BO_NO_MMAP,
3494 DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC |
3495 DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED,
3496 PANTHOR_VM_KERNEL_AUTO_VA);
3497 if (IS_ERR(group->syncobjs)) {
3498 ret = PTR_ERR(group->syncobjs);
3499 goto err_put_group;
3500 }
3501
3502 ret = panthor_kernel_bo_vmap(group->syncobjs);
3503 if (ret)
3504 goto err_put_group;
3505
3506 memset(group->syncobjs->kmap, 0,
3507 group_args->queues.count * sizeof(struct panthor_syncobj_64b));
3508
3509 for (i = 0; i < group_args->queues.count; i++) {
3510 group->queues[i] = group_create_queue(group, &queue_args[i]);
3511 if (IS_ERR(group->queues[i])) {
3512 ret = PTR_ERR(group->queues[i]);
3513 group->queues[i] = NULL;
3514 goto err_put_group;
3515 }
3516
3517 group->queue_count++;
3518 }
3519
3520 group->idle_queues = GENMASK(group->queue_count - 1, 0);
3521
3522 ret = xa_alloc(&gpool->xa, &gid, group, XA_LIMIT(1, MAX_GROUPS_PER_POOL), GFP_KERNEL);
3523 if (ret)
3524 goto err_put_group;
3525
3526 mutex_lock(&sched->reset.lock);
3527 if (atomic_read(&sched->reset.in_progress)) {
3528 panthor_group_stop(group);
3529 } else {
3530 mutex_lock(&sched->lock);
3531 list_add_tail(&group->run_node,
3532 &sched->groups.idle[group->priority]);
3533 mutex_unlock(&sched->lock);
3534 }
3535 mutex_unlock(&sched->reset.lock);
3536
3537 add_group_kbo_sizes(group->ptdev, group);
3538 spin_lock_init(&group->fdinfo.lock);
3539
3540 return gid;
3541
3542 err_put_group:
3543 group_put(group);
3544 return ret;
3545 }
3546
panthor_group_destroy(struct panthor_file * pfile,u32 group_handle)3547 int panthor_group_destroy(struct panthor_file *pfile, u32 group_handle)
3548 {
3549 struct panthor_group_pool *gpool = pfile->groups;
3550 struct panthor_device *ptdev = pfile->ptdev;
3551 struct panthor_scheduler *sched = ptdev->scheduler;
3552 struct panthor_group *group;
3553
3554 group = xa_erase(&gpool->xa, group_handle);
3555 if (!group)
3556 return -EINVAL;
3557
3558 for (u32 i = 0; i < group->queue_count; i++) {
3559 if (group->queues[i])
3560 drm_sched_entity_destroy(&group->queues[i]->entity);
3561 }
3562
3563 mutex_lock(&sched->reset.lock);
3564 mutex_lock(&sched->lock);
3565 group->destroyed = true;
3566 if (group->csg_id >= 0) {
3567 sched_queue_delayed_work(sched, tick, 0);
3568 } else if (!atomic_read(&sched->reset.in_progress)) {
3569 /* Remove from the run queues, so the scheduler can't
3570 * pick the group on the next tick.
3571 */
3572 list_del_init(&group->run_node);
3573 list_del_init(&group->wait_node);
3574 group_queue_work(group, term);
3575 }
3576 mutex_unlock(&sched->lock);
3577 mutex_unlock(&sched->reset.lock);
3578
3579 group_put(group);
3580 return 0;
3581 }
3582
group_from_handle(struct panthor_group_pool * pool,u32 group_handle)3583 static struct panthor_group *group_from_handle(struct panthor_group_pool *pool,
3584 u32 group_handle)
3585 {
3586 struct panthor_group *group;
3587
3588 xa_lock(&pool->xa);
3589 group = group_get(xa_load(&pool->xa, group_handle));
3590 xa_unlock(&pool->xa);
3591
3592 return group;
3593 }
3594
panthor_group_get_state(struct panthor_file * pfile,struct drm_panthor_group_get_state * get_state)3595 int panthor_group_get_state(struct panthor_file *pfile,
3596 struct drm_panthor_group_get_state *get_state)
3597 {
3598 struct panthor_group_pool *gpool = pfile->groups;
3599 struct panthor_device *ptdev = pfile->ptdev;
3600 struct panthor_scheduler *sched = ptdev->scheduler;
3601 struct panthor_group *group;
3602
3603 if (get_state->pad)
3604 return -EINVAL;
3605
3606 group = group_from_handle(gpool, get_state->group_handle);
3607 if (!group)
3608 return -EINVAL;
3609
3610 memset(get_state, 0, sizeof(*get_state));
3611
3612 mutex_lock(&sched->lock);
3613 if (group->timedout)
3614 get_state->state |= DRM_PANTHOR_GROUP_STATE_TIMEDOUT;
3615 if (group->fatal_queues) {
3616 get_state->state |= DRM_PANTHOR_GROUP_STATE_FATAL_FAULT;
3617 get_state->fatal_queues = group->fatal_queues;
3618 }
3619 if (group->innocent)
3620 get_state->state |= DRM_PANTHOR_GROUP_STATE_INNOCENT;
3621 mutex_unlock(&sched->lock);
3622
3623 group_put(group);
3624 return 0;
3625 }
3626
panthor_group_pool_create(struct panthor_file * pfile)3627 int panthor_group_pool_create(struct panthor_file *pfile)
3628 {
3629 struct panthor_group_pool *gpool;
3630
3631 gpool = kzalloc(sizeof(*gpool), GFP_KERNEL);
3632 if (!gpool)
3633 return -ENOMEM;
3634
3635 xa_init_flags(&gpool->xa, XA_FLAGS_ALLOC1);
3636 pfile->groups = gpool;
3637 return 0;
3638 }
3639
panthor_group_pool_destroy(struct panthor_file * pfile)3640 void panthor_group_pool_destroy(struct panthor_file *pfile)
3641 {
3642 struct panthor_group_pool *gpool = pfile->groups;
3643 struct panthor_group *group;
3644 unsigned long i;
3645
3646 if (IS_ERR_OR_NULL(gpool))
3647 return;
3648
3649 xa_for_each(&gpool->xa, i, group)
3650 panthor_group_destroy(pfile, i);
3651
3652 xa_destroy(&gpool->xa);
3653 kfree(gpool);
3654 pfile->groups = NULL;
3655 }
3656
3657 /**
3658 * panthor_fdinfo_gather_group_mem_info() - Retrieve aggregate size of all private kernel BO's
3659 * belonging to all the groups owned by an open Panthor file
3660 * @pfile: File.
3661 * @stats: Memory statistics to be updated.
3662 *
3663 */
3664 void
panthor_fdinfo_gather_group_mem_info(struct panthor_file * pfile,struct drm_memory_stats * stats)3665 panthor_fdinfo_gather_group_mem_info(struct panthor_file *pfile,
3666 struct drm_memory_stats *stats)
3667 {
3668 struct panthor_group_pool *gpool = pfile->groups;
3669 struct panthor_group *group;
3670 unsigned long i;
3671
3672 if (IS_ERR_OR_NULL(gpool))
3673 return;
3674
3675 xa_lock(&gpool->xa);
3676 xa_for_each(&gpool->xa, i, group) {
3677 stats->resident += group->fdinfo.kbo_sizes;
3678 if (group->csg_id >= 0)
3679 stats->active += group->fdinfo.kbo_sizes;
3680 }
3681 xa_unlock(&gpool->xa);
3682 }
3683
job_release(struct kref * ref)3684 static void job_release(struct kref *ref)
3685 {
3686 struct panthor_job *job = container_of(ref, struct panthor_job, refcount);
3687
3688 drm_WARN_ON(&job->group->ptdev->base, !list_empty(&job->node));
3689
3690 if (job->base.s_fence)
3691 drm_sched_job_cleanup(&job->base);
3692
3693 if (job->done_fence && job->done_fence->ops)
3694 dma_fence_put(job->done_fence);
3695 else
3696 dma_fence_free(job->done_fence);
3697
3698 group_put(job->group);
3699
3700 kfree(job);
3701 }
3702
panthor_job_get(struct drm_sched_job * sched_job)3703 struct drm_sched_job *panthor_job_get(struct drm_sched_job *sched_job)
3704 {
3705 if (sched_job) {
3706 struct panthor_job *job = container_of(sched_job, struct panthor_job, base);
3707
3708 kref_get(&job->refcount);
3709 }
3710
3711 return sched_job;
3712 }
3713
panthor_job_put(struct drm_sched_job * sched_job)3714 void panthor_job_put(struct drm_sched_job *sched_job)
3715 {
3716 struct panthor_job *job = container_of(sched_job, struct panthor_job, base);
3717
3718 if (sched_job)
3719 kref_put(&job->refcount, job_release);
3720 }
3721
panthor_job_vm(struct drm_sched_job * sched_job)3722 struct panthor_vm *panthor_job_vm(struct drm_sched_job *sched_job)
3723 {
3724 struct panthor_job *job = container_of(sched_job, struct panthor_job, base);
3725
3726 return job->group->vm;
3727 }
3728
3729 struct drm_sched_job *
panthor_job_create(struct panthor_file * pfile,u16 group_handle,const struct drm_panthor_queue_submit * qsubmit)3730 panthor_job_create(struct panthor_file *pfile,
3731 u16 group_handle,
3732 const struct drm_panthor_queue_submit *qsubmit)
3733 {
3734 struct panthor_group_pool *gpool = pfile->groups;
3735 struct panthor_job *job;
3736 u32 credits;
3737 int ret;
3738
3739 if (qsubmit->pad)
3740 return ERR_PTR(-EINVAL);
3741
3742 /* If stream_addr is zero, so stream_size should be. */
3743 if ((qsubmit->stream_size == 0) != (qsubmit->stream_addr == 0))
3744 return ERR_PTR(-EINVAL);
3745
3746 /* Make sure the address is aligned on 64-byte (cacheline) and the size is
3747 * aligned on 8-byte (instruction size).
3748 */
3749 if ((qsubmit->stream_addr & 63) || (qsubmit->stream_size & 7))
3750 return ERR_PTR(-EINVAL);
3751
3752 /* bits 24:30 must be zero. */
3753 if (qsubmit->latest_flush & GENMASK(30, 24))
3754 return ERR_PTR(-EINVAL);
3755
3756 job = kzalloc(sizeof(*job), GFP_KERNEL);
3757 if (!job)
3758 return ERR_PTR(-ENOMEM);
3759
3760 kref_init(&job->refcount);
3761 job->queue_idx = qsubmit->queue_index;
3762 job->call_info.size = qsubmit->stream_size;
3763 job->call_info.start = qsubmit->stream_addr;
3764 job->call_info.latest_flush = qsubmit->latest_flush;
3765 INIT_LIST_HEAD(&job->node);
3766
3767 job->group = group_from_handle(gpool, group_handle);
3768 if (!job->group) {
3769 ret = -EINVAL;
3770 goto err_put_job;
3771 }
3772
3773 if (!group_can_run(job->group)) {
3774 ret = -EINVAL;
3775 goto err_put_job;
3776 }
3777
3778 if (job->queue_idx >= job->group->queue_count ||
3779 !job->group->queues[job->queue_idx]) {
3780 ret = -EINVAL;
3781 goto err_put_job;
3782 }
3783
3784 /* Empty command streams don't need a fence, they'll pick the one from
3785 * the previously submitted job.
3786 */
3787 if (job->call_info.size) {
3788 job->done_fence = kzalloc(sizeof(*job->done_fence), GFP_KERNEL);
3789 if (!job->done_fence) {
3790 ret = -ENOMEM;
3791 goto err_put_job;
3792 }
3793 }
3794
3795 job->profiling.mask = pfile->ptdev->profile_mask;
3796 credits = calc_job_credits(job->profiling.mask);
3797 if (credits == 0) {
3798 ret = -EINVAL;
3799 goto err_put_job;
3800 }
3801
3802 ret = drm_sched_job_init(&job->base,
3803 &job->group->queues[job->queue_idx]->entity,
3804 credits, job->group);
3805 if (ret)
3806 goto err_put_job;
3807
3808 return &job->base;
3809
3810 err_put_job:
3811 panthor_job_put(&job->base);
3812 return ERR_PTR(ret);
3813 }
3814
panthor_job_update_resvs(struct drm_exec * exec,struct drm_sched_job * sched_job)3815 void panthor_job_update_resvs(struct drm_exec *exec, struct drm_sched_job *sched_job)
3816 {
3817 struct panthor_job *job = container_of(sched_job, struct panthor_job, base);
3818
3819 panthor_vm_update_resvs(job->group->vm, exec, &sched_job->s_fence->finished,
3820 DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_BOOKKEEP);
3821 }
3822
panthor_sched_unplug(struct panthor_device * ptdev)3823 void panthor_sched_unplug(struct panthor_device *ptdev)
3824 {
3825 struct panthor_scheduler *sched = ptdev->scheduler;
3826
3827 cancel_delayed_work_sync(&sched->tick_work);
3828
3829 mutex_lock(&sched->lock);
3830 if (sched->pm.has_ref) {
3831 pm_runtime_put(ptdev->base.dev);
3832 sched->pm.has_ref = false;
3833 }
3834 mutex_unlock(&sched->lock);
3835 }
3836
panthor_sched_fini(struct drm_device * ddev,void * res)3837 static void panthor_sched_fini(struct drm_device *ddev, void *res)
3838 {
3839 struct panthor_scheduler *sched = res;
3840 int prio;
3841
3842 if (!sched || !sched->csg_slot_count)
3843 return;
3844
3845 cancel_delayed_work_sync(&sched->tick_work);
3846
3847 if (sched->wq)
3848 destroy_workqueue(sched->wq);
3849
3850 if (sched->heap_alloc_wq)
3851 destroy_workqueue(sched->heap_alloc_wq);
3852
3853 for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) {
3854 drm_WARN_ON(ddev, !list_empty(&sched->groups.runnable[prio]));
3855 drm_WARN_ON(ddev, !list_empty(&sched->groups.idle[prio]));
3856 }
3857
3858 drm_WARN_ON(ddev, !list_empty(&sched->groups.waiting));
3859 }
3860
panthor_sched_init(struct panthor_device * ptdev)3861 int panthor_sched_init(struct panthor_device *ptdev)
3862 {
3863 struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev);
3864 struct panthor_fw_csg_iface *csg_iface = panthor_fw_get_csg_iface(ptdev, 0);
3865 struct panthor_fw_cs_iface *cs_iface = panthor_fw_get_cs_iface(ptdev, 0, 0);
3866 struct panthor_scheduler *sched;
3867 u32 gpu_as_count, num_groups;
3868 int prio, ret;
3869
3870 sched = drmm_kzalloc(&ptdev->base, sizeof(*sched), GFP_KERNEL);
3871 if (!sched)
3872 return -ENOMEM;
3873
3874 /* The highest bit in JOB_INT_* is reserved for globabl IRQs. That
3875 * leaves 31 bits for CSG IRQs, hence the MAX_CSGS clamp here.
3876 */
3877 num_groups = min_t(u32, MAX_CSGS, glb_iface->control->group_num);
3878
3879 /* The FW-side scheduler might deadlock if two groups with the same
3880 * priority try to access a set of resources that overlaps, with part
3881 * of the resources being allocated to one group and the other part to
3882 * the other group, both groups waiting for the remaining resources to
3883 * be allocated. To avoid that, it is recommended to assign each CSG a
3884 * different priority. In theory we could allow several groups to have
3885 * the same CSG priority if they don't request the same resources, but
3886 * that makes the scheduling logic more complicated, so let's clamp
3887 * the number of CSG slots to MAX_CSG_PRIO + 1 for now.
3888 */
3889 num_groups = min_t(u32, MAX_CSG_PRIO + 1, num_groups);
3890
3891 /* We need at least one AS for the MCU and one for the GPU contexts. */
3892 gpu_as_count = hweight32(ptdev->gpu_info.as_present & GENMASK(31, 1));
3893 if (!gpu_as_count) {
3894 drm_err(&ptdev->base, "Not enough AS (%d, expected at least 2)",
3895 gpu_as_count + 1);
3896 return -EINVAL;
3897 }
3898
3899 sched->ptdev = ptdev;
3900 sched->sb_slot_count = CS_FEATURES_SCOREBOARDS(cs_iface->control->features);
3901 sched->csg_slot_count = num_groups;
3902 sched->cs_slot_count = csg_iface->control->stream_num;
3903 sched->as_slot_count = gpu_as_count;
3904 ptdev->csif_info.csg_slot_count = sched->csg_slot_count;
3905 ptdev->csif_info.cs_slot_count = sched->cs_slot_count;
3906 ptdev->csif_info.scoreboard_slot_count = sched->sb_slot_count;
3907
3908 sched->last_tick = 0;
3909 sched->resched_target = U64_MAX;
3910 sched->tick_period = msecs_to_jiffies(10);
3911 INIT_DELAYED_WORK(&sched->tick_work, tick_work);
3912 INIT_WORK(&sched->sync_upd_work, sync_upd_work);
3913 INIT_WORK(&sched->fw_events_work, process_fw_events_work);
3914
3915 ret = drmm_mutex_init(&ptdev->base, &sched->lock);
3916 if (ret)
3917 return ret;
3918
3919 for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) {
3920 INIT_LIST_HEAD(&sched->groups.runnable[prio]);
3921 INIT_LIST_HEAD(&sched->groups.idle[prio]);
3922 }
3923 INIT_LIST_HEAD(&sched->groups.waiting);
3924
3925 ret = drmm_mutex_init(&ptdev->base, &sched->reset.lock);
3926 if (ret)
3927 return ret;
3928
3929 INIT_LIST_HEAD(&sched->reset.stopped_groups);
3930
3931 /* sched->heap_alloc_wq will be used for heap chunk allocation on
3932 * tiler OOM events, which means we can't use the same workqueue for
3933 * the scheduler because works queued by the scheduler are in
3934 * the dma-signalling path. Allocate a dedicated heap_alloc_wq to
3935 * work around this limitation.
3936 *
3937 * FIXME: Ultimately, what we need is a failable/non-blocking GEM
3938 * allocation path that we can call when a heap OOM is reported. The
3939 * FW is smart enough to fall back on other methods if the kernel can't
3940 * allocate memory, and fail the tiling job if none of these
3941 * countermeasures worked.
3942 *
3943 * Set WQ_MEM_RECLAIM on sched->wq to unblock the situation when the
3944 * system is running out of memory.
3945 */
3946 sched->heap_alloc_wq = alloc_workqueue("panthor-heap-alloc", WQ_UNBOUND, 0);
3947 sched->wq = alloc_workqueue("panthor-csf-sched", WQ_MEM_RECLAIM | WQ_UNBOUND, 0);
3948 if (!sched->wq || !sched->heap_alloc_wq) {
3949 panthor_sched_fini(&ptdev->base, sched);
3950 drm_err(&ptdev->base, "Failed to allocate the workqueues");
3951 return -ENOMEM;
3952 }
3953
3954 ret = drmm_add_action_or_reset(&ptdev->base, panthor_sched_fini, sched);
3955 if (ret)
3956 return ret;
3957
3958 ptdev->scheduler = sched;
3959 return 0;
3960 }
3961