xref: /linux/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c (revision ab93e0dd72c37d378dd936f031ffb83ff2bd87ce)
1 /*
2  * Copyright 2009 Jerome Glisse.
3  * All Rights Reserved.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the
7  * "Software"), to deal in the Software without restriction, including
8  * without limitation the rights to use, copy, modify, merge, publish,
9  * distribute, sub license, and/or sell copies of the Software, and to
10  * permit persons to whom the Software is furnished to do so, subject to
11  * the following conditions:
12  *
13  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15  * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16  * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17  * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19  * USE OR OTHER DEALINGS IN THE SOFTWARE.
20  *
21  * The above copyright notice and this permission notice (including the
22  * next paragraph) shall be included in all copies or substantial portions
23  * of the Software.
24  *
25  */
26 /*
27  * Authors:
28  *    Jerome Glisse <glisse@freedesktop.org>
29  *    Dave Airlie
30  */
31 #include <linux/seq_file.h>
32 #include <linux/atomic.h>
33 #include <linux/wait.h>
34 #include <linux/kref.h>
35 #include <linux/slab.h>
36 #include <linux/firmware.h>
37 #include <linux/pm_runtime.h>
38 
39 #include <drm/drm_drv.h>
40 #include "amdgpu.h"
41 #include "amdgpu_trace.h"
42 #include "amdgpu_reset.h"
43 
44 /*
45  * Cast helper
46  */
47 static const struct dma_fence_ops amdgpu_fence_ops;
48 static const struct dma_fence_ops amdgpu_job_fence_ops;
to_amdgpu_fence(struct dma_fence * f)49 static inline struct amdgpu_fence *to_amdgpu_fence(struct dma_fence *f)
50 {
51 	struct amdgpu_fence *__f = container_of(f, struct amdgpu_fence, base);
52 
53 	if (__f->base.ops == &amdgpu_fence_ops ||
54 	    __f->base.ops == &amdgpu_job_fence_ops)
55 		return __f;
56 
57 	return NULL;
58 }
59 
60 /**
61  * amdgpu_fence_write - write a fence value
62  *
63  * @ring: ring the fence is associated with
64  * @seq: sequence number to write
65  *
66  * Writes a fence value to memory (all asics).
67  */
amdgpu_fence_write(struct amdgpu_ring * ring,u32 seq)68 static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq)
69 {
70 	struct amdgpu_fence_driver *drv = &ring->fence_drv;
71 
72 	if (drv->cpu_addr)
73 		*drv->cpu_addr = cpu_to_le32(seq);
74 }
75 
76 /**
77  * amdgpu_fence_read - read a fence value
78  *
79  * @ring: ring the fence is associated with
80  *
81  * Reads a fence value from memory (all asics).
82  * Returns the value of the fence read from memory.
83  */
amdgpu_fence_read(struct amdgpu_ring * ring)84 static u32 amdgpu_fence_read(struct amdgpu_ring *ring)
85 {
86 	struct amdgpu_fence_driver *drv = &ring->fence_drv;
87 	u32 seq = 0;
88 
89 	if (drv->cpu_addr)
90 		seq = le32_to_cpu(*drv->cpu_addr);
91 	else
92 		seq = atomic_read(&drv->last_seq);
93 
94 	return seq;
95 }
96 
97 /**
98  * amdgpu_fence_emit - emit a fence on the requested ring
99  *
100  * @ring: ring the fence is associated with
101  * @f: resulting fence object
102  * @af: amdgpu fence input
103  * @flags: flags to pass into the subordinate .emit_fence() call
104  *
105  * Emits a fence command on the requested ring (all asics).
106  * Returns 0 on success, -ENOMEM on failure.
107  */
amdgpu_fence_emit(struct amdgpu_ring * ring,struct dma_fence ** f,struct amdgpu_fence * af,unsigned int flags)108 int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f,
109 		      struct amdgpu_fence *af, unsigned int flags)
110 {
111 	struct amdgpu_device *adev = ring->adev;
112 	struct dma_fence *fence;
113 	struct amdgpu_fence *am_fence;
114 	struct dma_fence __rcu **ptr;
115 	uint32_t seq;
116 	int r;
117 
118 	if (!af) {
119 		/* create a separate hw fence */
120 		am_fence = kzalloc(sizeof(*am_fence), GFP_KERNEL);
121 		if (!am_fence)
122 			return -ENOMEM;
123 		am_fence->context = 0;
124 	} else {
125 		am_fence = af;
126 	}
127 	fence = &am_fence->base;
128 	am_fence->ring = ring;
129 
130 	seq = ++ring->fence_drv.sync_seq;
131 	am_fence->seq = seq;
132 	if (af) {
133 		dma_fence_init(fence, &amdgpu_job_fence_ops,
134 			       &ring->fence_drv.lock,
135 			       adev->fence_context + ring->idx, seq);
136 		/* Against remove in amdgpu_job_{free, free_cb} */
137 		dma_fence_get(fence);
138 	} else {
139 		dma_fence_init(fence, &amdgpu_fence_ops,
140 			       &ring->fence_drv.lock,
141 			       adev->fence_context + ring->idx, seq);
142 	}
143 
144 	amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
145 			       seq, flags | AMDGPU_FENCE_FLAG_INT);
146 	amdgpu_fence_save_wptr(fence);
147 	pm_runtime_get_noresume(adev_to_drm(adev)->dev);
148 	ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];
149 	if (unlikely(rcu_dereference_protected(*ptr, 1))) {
150 		struct dma_fence *old;
151 
152 		rcu_read_lock();
153 		old = dma_fence_get_rcu_safe(ptr);
154 		rcu_read_unlock();
155 
156 		if (old) {
157 			r = dma_fence_wait(old, false);
158 			dma_fence_put(old);
159 			if (r)
160 				return r;
161 		}
162 	}
163 
164 	to_amdgpu_fence(fence)->start_timestamp = ktime_get();
165 
166 	/* This function can't be called concurrently anyway, otherwise
167 	 * emitting the fence would mess up the hardware ring buffer.
168 	 */
169 	rcu_assign_pointer(*ptr, dma_fence_get(fence));
170 
171 	*f = fence;
172 
173 	return 0;
174 }
175 
176 /**
177  * amdgpu_fence_emit_polling - emit a fence on the requeste ring
178  *
179  * @ring: ring the fence is associated with
180  * @s: resulting sequence number
181  * @timeout: the timeout for waiting in usecs
182  *
183  * Emits a fence command on the requested ring (all asics).
184  * Used For polling fence.
185  * Returns 0 on success, -ENOMEM on failure.
186  */
amdgpu_fence_emit_polling(struct amdgpu_ring * ring,uint32_t * s,uint32_t timeout)187 int amdgpu_fence_emit_polling(struct amdgpu_ring *ring, uint32_t *s,
188 			      uint32_t timeout)
189 {
190 	uint32_t seq;
191 	signed long r;
192 
193 	if (!s)
194 		return -EINVAL;
195 
196 	seq = ++ring->fence_drv.sync_seq;
197 	r = amdgpu_fence_wait_polling(ring,
198 				      seq - ring->fence_drv.num_fences_mask,
199 				      timeout);
200 	if (r < 1)
201 		return -ETIMEDOUT;
202 
203 	amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,
204 			       seq, 0);
205 
206 	*s = seq;
207 
208 	return 0;
209 }
210 
211 /**
212  * amdgpu_fence_schedule_fallback - schedule fallback check
213  *
214  * @ring: pointer to struct amdgpu_ring
215  *
216  * Start a timer as fallback to our interrupts.
217  */
amdgpu_fence_schedule_fallback(struct amdgpu_ring * ring)218 static void amdgpu_fence_schedule_fallback(struct amdgpu_ring *ring)
219 {
220 	mod_timer(&ring->fence_drv.fallback_timer,
221 		  jiffies + AMDGPU_FENCE_JIFFIES_TIMEOUT);
222 }
223 
224 /**
225  * amdgpu_fence_process - check for fence activity
226  *
227  * @ring: pointer to struct amdgpu_ring
228  *
229  * Checks the current fence value and calculates the last
230  * signalled fence value. Wakes the fence queue if the
231  * sequence number has increased.
232  *
233  * Returns true if fence was processed
234  */
amdgpu_fence_process(struct amdgpu_ring * ring)235 bool amdgpu_fence_process(struct amdgpu_ring *ring)
236 {
237 	struct amdgpu_fence_driver *drv = &ring->fence_drv;
238 	struct amdgpu_device *adev = ring->adev;
239 	uint32_t seq, last_seq;
240 
241 	do {
242 		last_seq = atomic_read(&ring->fence_drv.last_seq);
243 		seq = amdgpu_fence_read(ring);
244 
245 	} while (atomic_cmpxchg(&drv->last_seq, last_seq, seq) != last_seq);
246 
247 	if (timer_delete(&ring->fence_drv.fallback_timer) &&
248 	    seq != ring->fence_drv.sync_seq)
249 		amdgpu_fence_schedule_fallback(ring);
250 
251 	if (unlikely(seq == last_seq))
252 		return false;
253 
254 	last_seq &= drv->num_fences_mask;
255 	seq &= drv->num_fences_mask;
256 
257 	do {
258 		struct dma_fence *fence, **ptr;
259 		struct amdgpu_fence *am_fence;
260 
261 		++last_seq;
262 		last_seq &= drv->num_fences_mask;
263 		ptr = &drv->fences[last_seq];
264 
265 		/* There is always exactly one thread signaling this fence slot */
266 		fence = rcu_dereference_protected(*ptr, 1);
267 		RCU_INIT_POINTER(*ptr, NULL);
268 
269 		if (!fence)
270 			continue;
271 
272 		/* Save the wptr in the fence driver so we know what the last processed
273 		 * wptr was.  This is required for re-emitting the ring state for
274 		 * queues that are reset but are not guilty and thus have no guilty fence.
275 		 */
276 		am_fence = container_of(fence, struct amdgpu_fence, base);
277 		drv->signalled_wptr = am_fence->wptr;
278 		dma_fence_signal(fence);
279 		dma_fence_put(fence);
280 		pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
281 		pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
282 	} while (last_seq != seq);
283 
284 	return true;
285 }
286 
287 /**
288  * amdgpu_fence_fallback - fallback for hardware interrupts
289  *
290  * @t: timer context used to obtain the pointer to ring structure
291  *
292  * Checks for fence activity.
293  */
amdgpu_fence_fallback(struct timer_list * t)294 static void amdgpu_fence_fallback(struct timer_list *t)
295 {
296 	struct amdgpu_ring *ring = timer_container_of(ring, t,
297 						      fence_drv.fallback_timer);
298 
299 	if (amdgpu_fence_process(ring))
300 		dev_warn(ring->adev->dev,
301 			 "Fence fallback timer expired on ring %s\n",
302 			 ring->name);
303 }
304 
305 /**
306  * amdgpu_fence_wait_empty - wait for all fences to signal
307  *
308  * @ring: ring index the fence is associated with
309  *
310  * Wait for all fences on the requested ring to signal (all asics).
311  * Returns 0 if the fences have passed, error for all other cases.
312  */
amdgpu_fence_wait_empty(struct amdgpu_ring * ring)313 int amdgpu_fence_wait_empty(struct amdgpu_ring *ring)
314 {
315 	uint64_t seq = READ_ONCE(ring->fence_drv.sync_seq);
316 	struct dma_fence *fence, **ptr;
317 	int r;
318 
319 	if (!seq)
320 		return 0;
321 
322 	ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];
323 	rcu_read_lock();
324 	fence = rcu_dereference(*ptr);
325 	if (!fence || !dma_fence_get_rcu(fence)) {
326 		rcu_read_unlock();
327 		return 0;
328 	}
329 	rcu_read_unlock();
330 
331 	r = dma_fence_wait(fence, false);
332 	dma_fence_put(fence);
333 	return r;
334 }
335 
336 /**
337  * amdgpu_fence_wait_polling - busy wait for givn sequence number
338  *
339  * @ring: ring index the fence is associated with
340  * @wait_seq: sequence number to wait
341  * @timeout: the timeout for waiting in usecs
342  *
343  * Wait for all fences on the requested ring to signal (all asics).
344  * Returns left time if no timeout, 0 or minus if timeout.
345  */
amdgpu_fence_wait_polling(struct amdgpu_ring * ring,uint32_t wait_seq,signed long timeout)346 signed long amdgpu_fence_wait_polling(struct amdgpu_ring *ring,
347 				      uint32_t wait_seq,
348 				      signed long timeout)
349 {
350 
351 	while ((int32_t)(wait_seq - amdgpu_fence_read(ring)) > 0 && timeout > 0) {
352 		udelay(2);
353 		timeout -= 2;
354 	}
355 	return timeout > 0 ? timeout : 0;
356 }
357 /**
358  * amdgpu_fence_count_emitted - get the count of emitted fences
359  *
360  * @ring: ring the fence is associated with
361  *
362  * Get the number of fences emitted on the requested ring (all asics).
363  * Returns the number of emitted fences on the ring.  Used by the
364  * dynpm code to ring track activity.
365  */
amdgpu_fence_count_emitted(struct amdgpu_ring * ring)366 unsigned int amdgpu_fence_count_emitted(struct amdgpu_ring *ring)
367 {
368 	uint64_t emitted;
369 
370 	/* We are not protected by ring lock when reading the last sequence
371 	 * but it's ok to report slightly wrong fence count here.
372 	 */
373 	emitted = 0x100000000ull;
374 	emitted -= atomic_read(&ring->fence_drv.last_seq);
375 	emitted += READ_ONCE(ring->fence_drv.sync_seq);
376 	return lower_32_bits(emitted);
377 }
378 
379 /**
380  * amdgpu_fence_last_unsignaled_time_us - the time fence emitted until now
381  * @ring: ring the fence is associated with
382  *
383  * Find the earliest fence unsignaled until now, calculate the time delta
384  * between the time fence emitted and now.
385  */
amdgpu_fence_last_unsignaled_time_us(struct amdgpu_ring * ring)386 u64 amdgpu_fence_last_unsignaled_time_us(struct amdgpu_ring *ring)
387 {
388 	struct amdgpu_fence_driver *drv = &ring->fence_drv;
389 	struct dma_fence *fence;
390 	uint32_t last_seq, sync_seq;
391 
392 	last_seq = atomic_read(&ring->fence_drv.last_seq);
393 	sync_seq = READ_ONCE(ring->fence_drv.sync_seq);
394 	if (last_seq == sync_seq)
395 		return 0;
396 
397 	++last_seq;
398 	last_seq &= drv->num_fences_mask;
399 	fence = drv->fences[last_seq];
400 	if (!fence)
401 		return 0;
402 
403 	return ktime_us_delta(ktime_get(),
404 		to_amdgpu_fence(fence)->start_timestamp);
405 }
406 
407 /**
408  * amdgpu_fence_update_start_timestamp - update the timestamp of the fence
409  * @ring: ring the fence is associated with
410  * @seq: the fence seq number to update.
411  * @timestamp: the start timestamp to update.
412  *
413  * The function called at the time the fence and related ib is about to
414  * resubmit to gpu in MCBP scenario. Thus we do not consider race condition
415  * with amdgpu_fence_process to modify the same fence.
416  */
amdgpu_fence_update_start_timestamp(struct amdgpu_ring * ring,uint32_t seq,ktime_t timestamp)417 void amdgpu_fence_update_start_timestamp(struct amdgpu_ring *ring, uint32_t seq, ktime_t timestamp)
418 {
419 	struct amdgpu_fence_driver *drv = &ring->fence_drv;
420 	struct dma_fence *fence;
421 
422 	seq &= drv->num_fences_mask;
423 	fence = drv->fences[seq];
424 	if (!fence)
425 		return;
426 
427 	to_amdgpu_fence(fence)->start_timestamp = timestamp;
428 }
429 
430 /**
431  * amdgpu_fence_driver_start_ring - make the fence driver
432  * ready for use on the requested ring.
433  *
434  * @ring: ring to start the fence driver on
435  * @irq_src: interrupt source to use for this ring
436  * @irq_type: interrupt type to use for this ring
437  *
438  * Make the fence driver ready for processing (all asics).
439  * Not all asics have all rings, so each asic will only
440  * start the fence driver on the rings it has.
441  * Returns 0 for success, errors for failure.
442  */
amdgpu_fence_driver_start_ring(struct amdgpu_ring * ring,struct amdgpu_irq_src * irq_src,unsigned int irq_type)443 int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,
444 				   struct amdgpu_irq_src *irq_src,
445 				   unsigned int irq_type)
446 {
447 	struct amdgpu_device *adev = ring->adev;
448 	uint64_t index;
449 
450 	if (ring->funcs->type != AMDGPU_RING_TYPE_UVD) {
451 		ring->fence_drv.cpu_addr = ring->fence_cpu_addr;
452 		ring->fence_drv.gpu_addr = ring->fence_gpu_addr;
453 	} else {
454 		/* put fence directly behind firmware */
455 		index = ALIGN(adev->uvd.fw->size, 8);
456 		ring->fence_drv.cpu_addr = adev->uvd.inst[ring->me].cpu_addr + index;
457 		ring->fence_drv.gpu_addr = adev->uvd.inst[ring->me].gpu_addr + index;
458 	}
459 	amdgpu_fence_write(ring, atomic_read(&ring->fence_drv.last_seq));
460 
461 	ring->fence_drv.irq_src = irq_src;
462 	ring->fence_drv.irq_type = irq_type;
463 	ring->fence_drv.initialized = true;
464 
465 	DRM_DEV_DEBUG(adev->dev, "fence driver on ring %s use gpu addr 0x%016llx\n",
466 		      ring->name, ring->fence_drv.gpu_addr);
467 	return 0;
468 }
469 
470 /**
471  * amdgpu_fence_driver_init_ring - init the fence driver
472  * for the requested ring.
473  *
474  * @ring: ring to init the fence driver on
475  *
476  * Init the fence driver for the requested ring (all asics).
477  * Helper function for amdgpu_fence_driver_init().
478  */
amdgpu_fence_driver_init_ring(struct amdgpu_ring * ring)479 int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring)
480 {
481 	struct amdgpu_device *adev = ring->adev;
482 
483 	if (!adev)
484 		return -EINVAL;
485 
486 	if (!is_power_of_2(ring->num_hw_submission))
487 		return -EINVAL;
488 
489 	ring->fence_drv.cpu_addr = NULL;
490 	ring->fence_drv.gpu_addr = 0;
491 	ring->fence_drv.sync_seq = 0;
492 	atomic_set(&ring->fence_drv.last_seq, 0);
493 	ring->fence_drv.initialized = false;
494 
495 	timer_setup(&ring->fence_drv.fallback_timer, amdgpu_fence_fallback, 0);
496 
497 	ring->fence_drv.num_fences_mask = ring->num_hw_submission * 2 - 1;
498 	spin_lock_init(&ring->fence_drv.lock);
499 	ring->fence_drv.fences = kcalloc(ring->num_hw_submission * 2, sizeof(void *),
500 					 GFP_KERNEL);
501 
502 	if (!ring->fence_drv.fences)
503 		return -ENOMEM;
504 
505 	return 0;
506 }
507 
508 /**
509  * amdgpu_fence_driver_sw_init - init the fence driver
510  * for all possible rings.
511  *
512  * @adev: amdgpu device pointer
513  *
514  * Init the fence driver for all possible rings (all asics).
515  * Not all asics have all rings, so each asic will only
516  * start the fence driver on the rings it has using
517  * amdgpu_fence_driver_start_ring().
518  * Returns 0 for success.
519  */
amdgpu_fence_driver_sw_init(struct amdgpu_device * adev)520 int amdgpu_fence_driver_sw_init(struct amdgpu_device *adev)
521 {
522 	return 0;
523 }
524 
525 /**
526  * amdgpu_fence_need_ring_interrupt_restore - helper function to check whether
527  * fence driver interrupts need to be restored.
528  *
529  * @ring: ring that to be checked
530  *
531  * Interrupts for rings that belong to GFX IP don't need to be restored
532  * when the target power state is s0ix.
533  *
534  * Return true if need to restore interrupts, false otherwise.
535  */
amdgpu_fence_need_ring_interrupt_restore(struct amdgpu_ring * ring)536 static bool amdgpu_fence_need_ring_interrupt_restore(struct amdgpu_ring *ring)
537 {
538 	struct amdgpu_device *adev = ring->adev;
539 	bool is_gfx_power_domain = false;
540 
541 	switch (ring->funcs->type) {
542 	case AMDGPU_RING_TYPE_SDMA:
543 	/* SDMA 5.x+ is part of GFX power domain so it's covered by GFXOFF */
544 		if (amdgpu_ip_version(adev, SDMA0_HWIP, 0) >=
545 		    IP_VERSION(5, 0, 0))
546 			is_gfx_power_domain = true;
547 		break;
548 	case AMDGPU_RING_TYPE_GFX:
549 	case AMDGPU_RING_TYPE_COMPUTE:
550 	case AMDGPU_RING_TYPE_KIQ:
551 	case AMDGPU_RING_TYPE_MES:
552 		is_gfx_power_domain = true;
553 		break;
554 	default:
555 		break;
556 	}
557 
558 	return !(adev->in_s0ix && is_gfx_power_domain);
559 }
560 
561 /**
562  * amdgpu_fence_driver_hw_fini - tear down the fence driver
563  * for all possible rings.
564  *
565  * @adev: amdgpu device pointer
566  *
567  * Tear down the fence driver for all possible rings (all asics).
568  */
amdgpu_fence_driver_hw_fini(struct amdgpu_device * adev)569 void amdgpu_fence_driver_hw_fini(struct amdgpu_device *adev)
570 {
571 	int i, r;
572 
573 	for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
574 		struct amdgpu_ring *ring = adev->rings[i];
575 
576 		if (!ring || !ring->fence_drv.initialized)
577 			continue;
578 
579 		/* You can't wait for HW to signal if it's gone */
580 		if (!drm_dev_is_unplugged(adev_to_drm(adev)))
581 			r = amdgpu_fence_wait_empty(ring);
582 		else
583 			r = -ENODEV;
584 		/* no need to trigger GPU reset as we are unloading */
585 		if (r)
586 			amdgpu_fence_driver_force_completion(ring);
587 
588 		if (!drm_dev_is_unplugged(adev_to_drm(adev)) &&
589 		    ring->fence_drv.irq_src &&
590 		    amdgpu_fence_need_ring_interrupt_restore(ring))
591 			amdgpu_irq_put(adev, ring->fence_drv.irq_src,
592 				       ring->fence_drv.irq_type);
593 
594 		timer_delete_sync(&ring->fence_drv.fallback_timer);
595 	}
596 }
597 
598 /* Will either stop and flush handlers for amdgpu interrupt or reanble it */
amdgpu_fence_driver_isr_toggle(struct amdgpu_device * adev,bool stop)599 void amdgpu_fence_driver_isr_toggle(struct amdgpu_device *adev, bool stop)
600 {
601 	int i;
602 
603 	for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
604 		struct amdgpu_ring *ring = adev->rings[i];
605 
606 		if (!ring || !ring->fence_drv.initialized || !ring->fence_drv.irq_src)
607 			continue;
608 
609 		if (stop)
610 			disable_irq(adev->irq.irq);
611 		else
612 			enable_irq(adev->irq.irq);
613 	}
614 }
615 
amdgpu_fence_driver_sw_fini(struct amdgpu_device * adev)616 void amdgpu_fence_driver_sw_fini(struct amdgpu_device *adev)
617 {
618 	unsigned int i, j;
619 
620 	for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
621 		struct amdgpu_ring *ring = adev->rings[i];
622 
623 		if (!ring || !ring->fence_drv.initialized)
624 			continue;
625 
626 		/*
627 		 * Notice we check for sched.ops since there's some
628 		 * override on the meaning of sched.ready by amdgpu.
629 		 * The natural check would be sched.ready, which is
630 		 * set as drm_sched_init() finishes...
631 		 */
632 		if (ring->sched.ops)
633 			drm_sched_fini(&ring->sched);
634 
635 		for (j = 0; j <= ring->fence_drv.num_fences_mask; ++j)
636 			dma_fence_put(ring->fence_drv.fences[j]);
637 		kfree(ring->fence_drv.fences);
638 		ring->fence_drv.fences = NULL;
639 		ring->fence_drv.initialized = false;
640 	}
641 }
642 
643 /**
644  * amdgpu_fence_driver_hw_init - enable the fence driver
645  * for all possible rings.
646  *
647  * @adev: amdgpu device pointer
648  *
649  * Enable the fence driver for all possible rings (all asics).
650  * Not all asics have all rings, so each asic will only
651  * start the fence driver on the rings it has using
652  * amdgpu_fence_driver_start_ring().
653  * Returns 0 for success.
654  */
amdgpu_fence_driver_hw_init(struct amdgpu_device * adev)655 void amdgpu_fence_driver_hw_init(struct amdgpu_device *adev)
656 {
657 	int i;
658 
659 	for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
660 		struct amdgpu_ring *ring = adev->rings[i];
661 
662 		if (!ring || !ring->fence_drv.initialized)
663 			continue;
664 
665 		/* enable the interrupt */
666 		if (ring->fence_drv.irq_src &&
667 		    amdgpu_fence_need_ring_interrupt_restore(ring))
668 			amdgpu_irq_get(adev, ring->fence_drv.irq_src,
669 				       ring->fence_drv.irq_type);
670 	}
671 }
672 
673 /**
674  * amdgpu_fence_driver_clear_job_fences - clear job embedded fences of ring
675  *
676  * @ring: fence of the ring to be cleared
677  *
678  */
amdgpu_fence_driver_clear_job_fences(struct amdgpu_ring * ring)679 void amdgpu_fence_driver_clear_job_fences(struct amdgpu_ring *ring)
680 {
681 	int i;
682 	struct dma_fence *old, **ptr;
683 
684 	for (i = 0; i <= ring->fence_drv.num_fences_mask; i++) {
685 		ptr = &ring->fence_drv.fences[i];
686 		old = rcu_dereference_protected(*ptr, 1);
687 		if (old && old->ops == &amdgpu_job_fence_ops) {
688 			struct amdgpu_job *job;
689 
690 			/* For non-scheduler bad job, i.e. failed ib test, we need to signal
691 			 * it right here or we won't be able to track them in fence_drv
692 			 * and they will remain unsignaled during sa_bo free.
693 			 */
694 			job = container_of(old, struct amdgpu_job, hw_fence.base);
695 			if (!job->base.s_fence && !dma_fence_is_signaled(old))
696 				dma_fence_signal(old);
697 			RCU_INIT_POINTER(*ptr, NULL);
698 			dma_fence_put(old);
699 		}
700 	}
701 }
702 
703 /**
704  * amdgpu_fence_driver_set_error - set error code on fences
705  * @ring: the ring which contains the fences
706  * @error: the error code to set
707  *
708  * Set an error code to all the fences pending on the ring.
709  */
amdgpu_fence_driver_set_error(struct amdgpu_ring * ring,int error)710 void amdgpu_fence_driver_set_error(struct amdgpu_ring *ring, int error)
711 {
712 	struct amdgpu_fence_driver *drv = &ring->fence_drv;
713 	unsigned long flags;
714 
715 	spin_lock_irqsave(&drv->lock, flags);
716 	for (unsigned int i = 0; i <= drv->num_fences_mask; ++i) {
717 		struct dma_fence *fence;
718 
719 		fence = rcu_dereference_protected(drv->fences[i],
720 						  lockdep_is_held(&drv->lock));
721 		if (fence && !dma_fence_is_signaled_locked(fence))
722 			dma_fence_set_error(fence, error);
723 	}
724 	spin_unlock_irqrestore(&drv->lock, flags);
725 }
726 
727 /**
728  * amdgpu_fence_driver_force_completion - force signal latest fence of ring
729  *
730  * @ring: fence of the ring to signal
731  *
732  */
amdgpu_fence_driver_force_completion(struct amdgpu_ring * ring)733 void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring)
734 {
735 	amdgpu_fence_driver_set_error(ring, -ECANCELED);
736 	amdgpu_fence_write(ring, ring->fence_drv.sync_seq);
737 	amdgpu_fence_process(ring);
738 }
739 
740 
741 /**
742  * Kernel queue reset handling
743  *
744  * The driver can reset individual queues for most engines, but those queues
745  * may contain work from multiple contexts.  Resetting the queue will reset
746  * lose all of that state.  In order to minimize the collateral damage, the
747  * driver will save the ring contents which are not associated with the guilty
748  * context prior to resetting the queue.  After resetting the queue the queue
749  * contents from the other contexts is re-emitted to the rings so that it can
750  * be processed by the engine.  To handle this, we save the queue's write
751  * pointer (wptr) in the fences associated with each context.  If we get a
752  * queue timeout, we can then use the wptrs from the fences to determine
753  * which data needs to be saved out of the queue's ring buffer.
754  */
755 
756 /**
757  * amdgpu_fence_driver_guilty_force_completion - force signal of specified sequence
758  *
759  * @fence: fence of the ring to signal
760  *
761  */
amdgpu_fence_driver_guilty_force_completion(struct amdgpu_fence * fence)762 void amdgpu_fence_driver_guilty_force_completion(struct amdgpu_fence *fence)
763 {
764 	dma_fence_set_error(&fence->base, -ETIME);
765 	amdgpu_fence_write(fence->ring, fence->seq);
766 	amdgpu_fence_process(fence->ring);
767 }
768 
amdgpu_fence_save_wptr(struct dma_fence * fence)769 void amdgpu_fence_save_wptr(struct dma_fence *fence)
770 {
771 	struct amdgpu_fence *am_fence = container_of(fence, struct amdgpu_fence, base);
772 
773 	am_fence->wptr = am_fence->ring->wptr;
774 }
775 
amdgpu_ring_backup_unprocessed_command(struct amdgpu_ring * ring,u64 start_wptr,u32 end_wptr)776 static void amdgpu_ring_backup_unprocessed_command(struct amdgpu_ring *ring,
777 						   u64 start_wptr, u32 end_wptr)
778 {
779 	unsigned int first_idx = start_wptr & ring->buf_mask;
780 	unsigned int last_idx = end_wptr & ring->buf_mask;
781 	unsigned int i;
782 
783 	/* Backup the contents of the ring buffer. */
784 	for (i = first_idx; i != last_idx; ++i, i &= ring->buf_mask)
785 		ring->ring_backup[ring->ring_backup_entries_to_copy++] = ring->ring[i];
786 }
787 
amdgpu_ring_backup_unprocessed_commands(struct amdgpu_ring * ring,struct amdgpu_fence * guilty_fence)788 void amdgpu_ring_backup_unprocessed_commands(struct amdgpu_ring *ring,
789 					     struct amdgpu_fence *guilty_fence)
790 {
791 	struct dma_fence *unprocessed;
792 	struct dma_fence __rcu **ptr;
793 	struct amdgpu_fence *fence;
794 	u64 wptr, i, seqno;
795 
796 	seqno = amdgpu_fence_read(ring);
797 	wptr = ring->fence_drv.signalled_wptr;
798 	ring->ring_backup_entries_to_copy = 0;
799 
800 	for (i = seqno + 1; i <= ring->fence_drv.sync_seq; ++i) {
801 		ptr = &ring->fence_drv.fences[i & ring->fence_drv.num_fences_mask];
802 		rcu_read_lock();
803 		unprocessed = rcu_dereference(*ptr);
804 
805 		if (unprocessed && !dma_fence_is_signaled(unprocessed)) {
806 			fence = container_of(unprocessed, struct amdgpu_fence, base);
807 
808 			/* save everything if the ring is not guilty, otherwise
809 			 * just save the content from other contexts.
810 			 */
811 			if (!guilty_fence || (fence->context != guilty_fence->context))
812 				amdgpu_ring_backup_unprocessed_command(ring, wptr,
813 								       fence->wptr);
814 			wptr = fence->wptr;
815 		}
816 		rcu_read_unlock();
817 	}
818 }
819 
820 /*
821  * Common fence implementation
822  */
823 
amdgpu_fence_get_driver_name(struct dma_fence * fence)824 static const char *amdgpu_fence_get_driver_name(struct dma_fence *fence)
825 {
826 	return "amdgpu";
827 }
828 
amdgpu_fence_get_timeline_name(struct dma_fence * f)829 static const char *amdgpu_fence_get_timeline_name(struct dma_fence *f)
830 {
831 	return (const char *)to_amdgpu_fence(f)->ring->name;
832 }
833 
amdgpu_job_fence_get_timeline_name(struct dma_fence * f)834 static const char *amdgpu_job_fence_get_timeline_name(struct dma_fence *f)
835 {
836 	struct amdgpu_job *job = container_of(f, struct amdgpu_job, hw_fence.base);
837 
838 	return (const char *)to_amdgpu_ring(job->base.sched)->name;
839 }
840 
841 /**
842  * amdgpu_fence_enable_signaling - enable signalling on fence
843  * @f: fence
844  *
845  * This function is called with fence_queue lock held, and adds a callback
846  * to fence_queue that checks if this fence is signaled, and if so it
847  * signals the fence and removes itself.
848  */
amdgpu_fence_enable_signaling(struct dma_fence * f)849 static bool amdgpu_fence_enable_signaling(struct dma_fence *f)
850 {
851 	if (!timer_pending(&to_amdgpu_fence(f)->ring->fence_drv.fallback_timer))
852 		amdgpu_fence_schedule_fallback(to_amdgpu_fence(f)->ring);
853 
854 	return true;
855 }
856 
857 /**
858  * amdgpu_job_fence_enable_signaling - enable signalling on job fence
859  * @f: fence
860  *
861  * This is the simliar function with amdgpu_fence_enable_signaling above, it
862  * only handles the job embedded fence.
863  */
amdgpu_job_fence_enable_signaling(struct dma_fence * f)864 static bool amdgpu_job_fence_enable_signaling(struct dma_fence *f)
865 {
866 	struct amdgpu_job *job = container_of(f, struct amdgpu_job, hw_fence.base);
867 
868 	if (!timer_pending(&to_amdgpu_ring(job->base.sched)->fence_drv.fallback_timer))
869 		amdgpu_fence_schedule_fallback(to_amdgpu_ring(job->base.sched));
870 
871 	return true;
872 }
873 
874 /**
875  * amdgpu_fence_free - free up the fence memory
876  *
877  * @rcu: RCU callback head
878  *
879  * Free up the fence memory after the RCU grace period.
880  */
amdgpu_fence_free(struct rcu_head * rcu)881 static void amdgpu_fence_free(struct rcu_head *rcu)
882 {
883 	struct dma_fence *f = container_of(rcu, struct dma_fence, rcu);
884 
885 	/* free fence_slab if it's separated fence*/
886 	kfree(to_amdgpu_fence(f));
887 }
888 
889 /**
890  * amdgpu_job_fence_free - free up the job with embedded fence
891  *
892  * @rcu: RCU callback head
893  *
894  * Free up the job with embedded fence after the RCU grace period.
895  */
amdgpu_job_fence_free(struct rcu_head * rcu)896 static void amdgpu_job_fence_free(struct rcu_head *rcu)
897 {
898 	struct dma_fence *f = container_of(rcu, struct dma_fence, rcu);
899 
900 	/* free job if fence has a parent job */
901 	kfree(container_of(f, struct amdgpu_job, hw_fence.base));
902 }
903 
904 /**
905  * amdgpu_fence_release - callback that fence can be freed
906  *
907  * @f: fence
908  *
909  * This function is called when the reference count becomes zero.
910  * It just RCU schedules freeing up the fence.
911  */
amdgpu_fence_release(struct dma_fence * f)912 static void amdgpu_fence_release(struct dma_fence *f)
913 {
914 	call_rcu(&f->rcu, amdgpu_fence_free);
915 }
916 
917 /**
918  * amdgpu_job_fence_release - callback that job embedded fence can be freed
919  *
920  * @f: fence
921  *
922  * This is the simliar function with amdgpu_fence_release above, it
923  * only handles the job embedded fence.
924  */
amdgpu_job_fence_release(struct dma_fence * f)925 static void amdgpu_job_fence_release(struct dma_fence *f)
926 {
927 	call_rcu(&f->rcu, amdgpu_job_fence_free);
928 }
929 
930 static const struct dma_fence_ops amdgpu_fence_ops = {
931 	.get_driver_name = amdgpu_fence_get_driver_name,
932 	.get_timeline_name = amdgpu_fence_get_timeline_name,
933 	.enable_signaling = amdgpu_fence_enable_signaling,
934 	.release = amdgpu_fence_release,
935 };
936 
937 static const struct dma_fence_ops amdgpu_job_fence_ops = {
938 	.get_driver_name = amdgpu_fence_get_driver_name,
939 	.get_timeline_name = amdgpu_job_fence_get_timeline_name,
940 	.enable_signaling = amdgpu_job_fence_enable_signaling,
941 	.release = amdgpu_job_fence_release,
942 };
943 
944 /*
945  * Fence debugfs
946  */
947 #if defined(CONFIG_DEBUG_FS)
amdgpu_debugfs_fence_info_show(struct seq_file * m,void * unused)948 static int amdgpu_debugfs_fence_info_show(struct seq_file *m, void *unused)
949 {
950 	struct amdgpu_device *adev = m->private;
951 	int i;
952 
953 	for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
954 		struct amdgpu_ring *ring = adev->rings[i];
955 
956 		if (!ring || !ring->fence_drv.initialized)
957 			continue;
958 
959 		amdgpu_fence_process(ring);
960 
961 		seq_printf(m, "--- ring %d (%s) ---\n", i, ring->name);
962 		seq_printf(m, "Last signaled fence          0x%08x\n",
963 			   atomic_read(&ring->fence_drv.last_seq));
964 		seq_printf(m, "Last emitted                 0x%08x\n",
965 			   ring->fence_drv.sync_seq);
966 
967 		if (ring->funcs->type == AMDGPU_RING_TYPE_GFX ||
968 		    ring->funcs->type == AMDGPU_RING_TYPE_SDMA) {
969 			seq_printf(m, "Last signaled trailing fence 0x%08x\n",
970 				   le32_to_cpu(*ring->trail_fence_cpu_addr));
971 			seq_printf(m, "Last emitted                 0x%08x\n",
972 				   ring->trail_seq);
973 		}
974 
975 		if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)
976 			continue;
977 
978 		/* set in CP_VMID_PREEMPT and preemption occurred */
979 		seq_printf(m, "Last preempted               0x%08x\n",
980 			   le32_to_cpu(*(ring->fence_drv.cpu_addr + 2)));
981 		/* set in CP_VMID_RESET and reset occurred */
982 		seq_printf(m, "Last reset                   0x%08x\n",
983 			   le32_to_cpu(*(ring->fence_drv.cpu_addr + 4)));
984 		/* Both preemption and reset occurred */
985 		seq_printf(m, "Last both                    0x%08x\n",
986 			   le32_to_cpu(*(ring->fence_drv.cpu_addr + 6)));
987 	}
988 	return 0;
989 }
990 
991 /*
992  * amdgpu_debugfs_gpu_recover - manually trigger a gpu reset & recover
993  *
994  * Manually trigger a gpu reset at the next fence wait.
995  */
gpu_recover_get(void * data,u64 * val)996 static int gpu_recover_get(void *data, u64 *val)
997 {
998 	struct amdgpu_device *adev = (struct amdgpu_device *)data;
999 	struct drm_device *dev = adev_to_drm(adev);
1000 	int r;
1001 
1002 	r = pm_runtime_get_sync(dev->dev);
1003 	if (r < 0) {
1004 		pm_runtime_put_autosuspend(dev->dev);
1005 		return 0;
1006 	}
1007 
1008 	if (amdgpu_reset_domain_schedule(adev->reset_domain, &adev->reset_work))
1009 		flush_work(&adev->reset_work);
1010 
1011 	*val = atomic_read(&adev->reset_domain->reset_res);
1012 
1013 	pm_runtime_mark_last_busy(dev->dev);
1014 	pm_runtime_put_autosuspend(dev->dev);
1015 
1016 	return 0;
1017 }
1018 
1019 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_fence_info);
1020 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_debugfs_gpu_recover_fops, gpu_recover_get, NULL,
1021 			 "%lld\n");
1022 
amdgpu_debugfs_reset_work(struct work_struct * work)1023 static void amdgpu_debugfs_reset_work(struct work_struct *work)
1024 {
1025 	struct amdgpu_device *adev = container_of(work, struct amdgpu_device,
1026 						  reset_work);
1027 
1028 	struct amdgpu_reset_context reset_context;
1029 
1030 	memset(&reset_context, 0, sizeof(reset_context));
1031 
1032 	reset_context.method = AMD_RESET_METHOD_NONE;
1033 	reset_context.reset_req_dev = adev;
1034 	reset_context.src = AMDGPU_RESET_SRC_USER;
1035 	set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
1036 	set_bit(AMDGPU_SKIP_COREDUMP, &reset_context.flags);
1037 
1038 	amdgpu_device_gpu_recover(adev, NULL, &reset_context);
1039 }
1040 
1041 #endif
1042 
amdgpu_debugfs_fence_init(struct amdgpu_device * adev)1043 void amdgpu_debugfs_fence_init(struct amdgpu_device *adev)
1044 {
1045 #if defined(CONFIG_DEBUG_FS)
1046 	struct drm_minor *minor = adev_to_drm(adev)->primary;
1047 	struct dentry *root = minor->debugfs_root;
1048 
1049 	debugfs_create_file("amdgpu_fence_info", 0444, root, adev,
1050 			    &amdgpu_debugfs_fence_info_fops);
1051 
1052 	if (!amdgpu_sriov_vf(adev)) {
1053 
1054 		INIT_WORK(&adev->reset_work, amdgpu_debugfs_reset_work);
1055 		debugfs_create_file("amdgpu_gpu_recover", 0444, root, adev,
1056 				    &amdgpu_debugfs_gpu_recover_fops);
1057 	}
1058 #endif
1059 }
1060 
1061