58  * @ring: ring the fence is associated with
63 static void amdgpu_fence_write(struct amdgpu_ring *ring, u32 seq)  in amdgpu_fence_write()  argument
65 	struct amdgpu_fence_driver *drv = &ring->fence_drv;  in amdgpu_fence_write()
74  * @ring: ring the fence is associated with
79 static u32 amdgpu_fence_read(struct amdgpu_ring *ring)  in amdgpu_fence_read()  argument
81 	struct amdgpu_fence_driver *drv = &ring->fence_drv;  in amdgpu_fence_read()
94 	af->fence_wptr_start = af->ring->wptr;  in amdgpu_fence_save_fence_wptr_start()
99 	af->fence_wptr_end = af->ring->wptr;  in amdgpu_fence_save_fence_wptr_end()
103  * amdgpu_fence_emit - emit a fence on the requested ring
105  * @ring: ring the fence is associated with
109  * Emits a fence command on the requested ring (all asics).
112 int amdgpu_fence_emit(struct amdgpu_ring *ring, struct amdgpu_fence *af,  in amdgpu_fence_emit()  argument
115 	struct amdgpu_device *adev = ring->adev;  in amdgpu_fence_emit()
122 	af->ring = ring;  in amdgpu_fence_emit()
124 	seq = ++ring->fence_drv.sync_seq;  in amdgpu_fence_emit()
126 		       &ring->fence_drv.lock,  in amdgpu_fence_emit()
127 		       adev->fence_context + ring->idx, seq);  in amdgpu_fence_emit()
130 	amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,  in amdgpu_fence_emit()
135 	ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];  in amdgpu_fence_emit()
154 	 * emitting the fence would mess up the hardware ring buffer.  in amdgpu_fence_emit()
162  * amdgpu_fence_emit_polling - emit a fence on the requeste ring
164  * @ring: ring the fence is associated with
168  * Emits a fence command on the requested ring (all asics).
172 int amdgpu_fence_emit_polling(struct amdgpu_ring *ring, uint32_t *s,  in amdgpu_fence_emit_polling()  argument
181 	seq = ++ring->fence_drv.sync_seq;  in amdgpu_fence_emit_polling()
182 	r = amdgpu_fence_wait_polling(ring,  in amdgpu_fence_emit_polling()
183 				      seq - ring->fence_drv.num_fences_mask,  in amdgpu_fence_emit_polling()
188 	amdgpu_ring_emit_fence(ring, ring->fence_drv.gpu_addr,  in amdgpu_fence_emit_polling()
199  * @ring: pointer to struct amdgpu_ring
203 static void amdgpu_fence_schedule_fallback(struct amdgpu_ring *ring)  in amdgpu_fence_schedule_fallback()  argument
205 	mod_timer(&ring->fence_drv.fallback_timer,  in amdgpu_fence_schedule_fallback()
212  * @ring: pointer to struct amdgpu_ring
220 bool amdgpu_fence_process(struct amdgpu_ring *ring)  in amdgpu_fence_process()  argument
222 	struct amdgpu_fence_driver *drv = &ring->fence_drv;  in amdgpu_fence_process()
223 	struct amdgpu_device *adev = ring->adev;  in amdgpu_fence_process()
227 		last_seq = atomic_read(&ring->fence_drv.last_seq);  in amdgpu_fence_process()
228 		seq = amdgpu_fence_read(ring);  in amdgpu_fence_process()
232 	if (timer_delete(&ring->fence_drv.fallback_timer) &&  in amdgpu_fence_process()
233 	    seq != ring->fence_drv.sync_seq)  in amdgpu_fence_process()
234 		amdgpu_fence_schedule_fallback(ring);  in amdgpu_fence_process()
258 		 * wptr was.  This is required for re-emitting the ring state for  in amdgpu_fence_process()
274  * @t: timer context used to obtain the pointer to ring structure
280 	struct amdgpu_ring *ring = timer_container_of(ring, t,  in amdgpu_fence_fallback()  local
283 	if (amdgpu_fence_process(ring))  in amdgpu_fence_fallback()
284 		dev_warn(ring->adev->dev,  in amdgpu_fence_fallback()
285 			 "Fence fallback timer expired on ring %s\n",  in amdgpu_fence_fallback()
286 			 ring->name);  in amdgpu_fence_fallback()
292  * @ring: ring index the fence is associated with
294  * Wait for all fences on the requested ring to signal (all asics).
297 int amdgpu_fence_wait_empty(struct amdgpu_ring *ring)  in amdgpu_fence_wait_empty()  argument
299 	uint64_t seq = READ_ONCE(ring->fence_drv.sync_seq);  in amdgpu_fence_wait_empty()
306 	ptr = &ring->fence_drv.fences[seq & ring->fence_drv.num_fences_mask];  in amdgpu_fence_wait_empty()
323  * @ring: ring index the fence is associated with
327  * Wait for all fences on the requested ring to signal (all asics).
330 signed long amdgpu_fence_wait_polling(struct amdgpu_ring *ring,  in amdgpu_fence_wait_polling()  argument
335 	while ((int32_t)(wait_seq - amdgpu_fence_read(ring)) > 0 && timeout > 0) {  in amdgpu_fence_wait_polling()
344  * @ring: ring the fence is associated with
346  * Get the number of fences emitted on the requested ring (all asics).
347  * Returns the number of emitted fences on the ring.  Used by the
348  * dynpm code to ring track activity.
350 unsigned int amdgpu_fence_count_emitted(struct amdgpu_ring *ring)  in amdgpu_fence_count_emitted()  argument
354 	/* We are not protected by ring lock when reading the last sequence  in amdgpu_fence_count_emitted()
358 	emitted -= atomic_read(&ring->fence_drv.last_seq);  in amdgpu_fence_count_emitted()
359 	emitted += READ_ONCE(ring->fence_drv.sync_seq);  in amdgpu_fence_count_emitted()
365  * @ring: ring the fence is associated with
370 u64 amdgpu_fence_last_unsignaled_time_us(struct amdgpu_ring *ring)  in amdgpu_fence_last_unsignaled_time_us()  argument
372 	struct amdgpu_fence_driver *drv = &ring->fence_drv;  in amdgpu_fence_last_unsignaled_time_us()
376 	last_seq = atomic_read(&ring->fence_drv.last_seq);  in amdgpu_fence_last_unsignaled_time_us()
377 	sync_seq = READ_ONCE(ring->fence_drv.sync_seq);  in amdgpu_fence_last_unsignaled_time_us()
393  * @ring: ring the fence is associated with
401 void amdgpu_fence_update_start_timestamp(struct amdgpu_ring *ring, uint32_t seq, ktime_t timestamp)  in amdgpu_fence_update_start_timestamp()  argument
403 	struct amdgpu_fence_driver *drv = &ring->fence_drv;  in amdgpu_fence_update_start_timestamp()
416  * ready for use on the requested ring.
418  * @ring: ring to start the fence driver on
419  * @irq_src: interrupt source to use for this ring
420  * @irq_type: interrupt type to use for this ring
427 int amdgpu_fence_driver_start_ring(struct amdgpu_ring *ring,  in amdgpu_fence_driver_start_ring()  argument
431 	struct amdgpu_device *adev = ring->adev;  in amdgpu_fence_driver_start_ring()
434 	if (ring->funcs->type != AMDGPU_RING_TYPE_UVD) {  in amdgpu_fence_driver_start_ring()
435 		ring->fence_drv.cpu_addr = ring->fence_cpu_addr;  in amdgpu_fence_driver_start_ring()
436 		ring->fence_drv.gpu_addr = ring->fence_gpu_addr;  in amdgpu_fence_driver_start_ring()
440 		ring->fence_drv.cpu_addr = adev->uvd.inst[ring->me].cpu_addr + index;  in amdgpu_fence_driver_start_ring()
441 		ring->fence_drv.gpu_addr = adev->uvd.inst[ring->me].gpu_addr + index;  in amdgpu_fence_driver_start_ring()
443 	amdgpu_fence_write(ring, atomic_read(&ring->fence_drv.last_seq));  in amdgpu_fence_driver_start_ring()
445 	ring->fence_drv.irq_src = irq_src;  in amdgpu_fence_driver_start_ring()
446 	ring->fence_drv.irq_type = irq_type;  in amdgpu_fence_driver_start_ring()
447 	ring->fence_drv.initialized = true;  in amdgpu_fence_driver_start_ring()
449 	DRM_DEV_DEBUG(adev->dev, "fence driver on ring %s use gpu addr 0x%016llx\n",  in amdgpu_fence_driver_start_ring()
450 		      ring->name, ring->fence_drv.gpu_addr);  in amdgpu_fence_driver_start_ring()
456  * for the requested ring.
458  * @ring: ring to init the fence driver on
460  * Init the fence driver for the requested ring (all asics).
463 int amdgpu_fence_driver_init_ring(struct amdgpu_ring *ring)  in amdgpu_fence_driver_init_ring()  argument
465 	struct amdgpu_device *adev = ring->adev;  in amdgpu_fence_driver_init_ring()
470 	if (!is_power_of_2(ring->num_hw_submission))  in amdgpu_fence_driver_init_ring()
473 	ring->fence_drv.cpu_addr = NULL;  in amdgpu_fence_driver_init_ring()
474 	ring->fence_drv.gpu_addr = 0;  in amdgpu_fence_driver_init_ring()
475 	ring->fence_drv.sync_seq = 0;  in amdgpu_fence_driver_init_ring()
476 	atomic_set(&ring->fence_drv.last_seq, 0);  in amdgpu_fence_driver_init_ring()
477 	ring->fence_drv.initialized = false;  in amdgpu_fence_driver_init_ring()
479 	timer_setup(&ring->fence_drv.fallback_timer, amdgpu_fence_fallback, 0);  in amdgpu_fence_driver_init_ring()
481 	ring->fence_drv.num_fences_mask = ring->num_hw_submission * 2 - 1;  in amdgpu_fence_driver_init_ring()
482 	spin_lock_init(&ring->fence_drv.lock);  in amdgpu_fence_driver_init_ring()
483 	ring->fence_drv.fences = kcalloc(ring->num_hw_submission * 2, sizeof(void *),  in amdgpu_fence_driver_init_ring()
486 	if (!ring->fence_drv.fences)  in amdgpu_fence_driver_init_ring()
513  * @ring: ring that to be checked
520 static bool amdgpu_fence_need_ring_interrupt_restore(struct amdgpu_ring *ring)  in amdgpu_fence_need_ring_interrupt_restore()  argument
522 	struct amdgpu_device *adev = ring->adev;  in amdgpu_fence_need_ring_interrupt_restore()
525 	switch (ring->funcs->type) {  in amdgpu_fence_need_ring_interrupt_restore()
558 		struct amdgpu_ring *ring = adev->rings[i];  in amdgpu_fence_driver_hw_fini()  local
560 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_fence_driver_hw_fini()
565 			r = amdgpu_fence_wait_empty(ring);  in amdgpu_fence_driver_hw_fini()
570 			amdgpu_fence_driver_force_completion(ring);  in amdgpu_fence_driver_hw_fini()
573 		    ring->fence_drv.irq_src &&  in amdgpu_fence_driver_hw_fini()
574 		    amdgpu_fence_need_ring_interrupt_restore(ring))  in amdgpu_fence_driver_hw_fini()
575 			amdgpu_irq_put(adev, ring->fence_drv.irq_src,  in amdgpu_fence_driver_hw_fini()
576 				       ring->fence_drv.irq_type);  in amdgpu_fence_driver_hw_fini()
578 		timer_delete_sync(&ring->fence_drv.fallback_timer);  in amdgpu_fence_driver_hw_fini()
588 		struct amdgpu_ring *ring = adev->rings[i];  in amdgpu_fence_driver_isr_toggle()  local
590 		if (!ring || !ring->fence_drv.initialized || !ring->fence_drv.irq_src)  in amdgpu_fence_driver_isr_toggle()
605 		struct amdgpu_ring *ring = adev->rings[i];  in amdgpu_fence_driver_sw_fini()  local
607 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_fence_driver_sw_fini()
616 		if (ring->sched.ops)  in amdgpu_fence_driver_sw_fini()
617 			drm_sched_fini(&ring->sched);  in amdgpu_fence_driver_sw_fini()
619 		for (j = 0; j <= ring->fence_drv.num_fences_mask; ++j)  in amdgpu_fence_driver_sw_fini()
620 			dma_fence_put(ring->fence_drv.fences[j]);  in amdgpu_fence_driver_sw_fini()
621 		kfree(ring->fence_drv.fences);  in amdgpu_fence_driver_sw_fini()
622 		ring->fence_drv.fences = NULL;  in amdgpu_fence_driver_sw_fini()
623 		ring->fence_drv.initialized = false;  in amdgpu_fence_driver_sw_fini()
644 		struct amdgpu_ring *ring = adev->rings[i];  in amdgpu_fence_driver_hw_init()  local
646 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_fence_driver_hw_init()
650 		if (ring->fence_drv.irq_src &&  in amdgpu_fence_driver_hw_init()
651 		    amdgpu_fence_need_ring_interrupt_restore(ring))  in amdgpu_fence_driver_hw_init()
652 			amdgpu_irq_get(adev, ring->fence_drv.irq_src,  in amdgpu_fence_driver_hw_init()
653 				       ring->fence_drv.irq_type);  in amdgpu_fence_driver_hw_init()
659  * @ring: the ring which contains the fences
662  * Set an error code to all the fences pending on the ring.
664 void amdgpu_fence_driver_set_error(struct amdgpu_ring *ring, int error)  in amdgpu_fence_driver_set_error()  argument
666 	struct amdgpu_fence_driver *drv = &ring->fence_drv;  in amdgpu_fence_driver_set_error()
682  * amdgpu_fence_driver_force_completion - force signal latest fence of ring
684  * @ring: fence of the ring to signal
687 void amdgpu_fence_driver_force_completion(struct amdgpu_ring *ring)  in amdgpu_fence_driver_force_completion()  argument
689 	amdgpu_fence_driver_set_error(ring, -ECANCELED);  in amdgpu_fence_driver_force_completion()
690 	amdgpu_fence_write(ring, ring->fence_drv.sync_seq);  in amdgpu_fence_driver_force_completion()
691 	amdgpu_fence_process(ring);  in amdgpu_fence_driver_force_completion()
701  * driver will save the ring contents which are not associated with the guilty
707  * which data needs to be saved out of the queue's ring buffer.
713  * @af: fence of the ring to update
721 	struct amdgpu_ring *ring = af->ring;  in amdgpu_fence_driver_update_timedout_fence_state()  local
726 	last_seq = amdgpu_fence_read(ring) & ring->fence_drv.num_fences_mask;  in amdgpu_fence_driver_update_timedout_fence_state()
727 	seq = ring->fence_drv.sync_seq & ring->fence_drv.num_fences_mask;  in amdgpu_fence_driver_update_timedout_fence_state()
730 	spin_lock_irqsave(&ring->fence_drv.lock, flags);  in amdgpu_fence_driver_update_timedout_fence_state()
733 		last_seq &= ring->fence_drv.num_fences_mask;  in amdgpu_fence_driver_update_timedout_fence_state()
735 		ptr = &ring->fence_drv.fences[last_seq];  in amdgpu_fence_driver_update_timedout_fence_state()
751 	spin_unlock_irqrestore(&ring->fence_drv.lock, flags);  in amdgpu_fence_driver_update_timedout_fence_state()
755 		amdgpu_fence_driver_force_completion(af->ring);  in amdgpu_fence_driver_update_timedout_fence_state()
756 		af->ring->ring_backup_entries_to_copy = 0;  in amdgpu_fence_driver_update_timedout_fence_state()
762 	af->wptr = af->ring->wptr;  in amdgpu_fence_save_wptr()
765 static void amdgpu_ring_backup_unprocessed_command(struct amdgpu_ring *ring,  in amdgpu_ring_backup_unprocessed_command()  argument
768 	unsigned int first_idx = start_wptr & ring->buf_mask;  in amdgpu_ring_backup_unprocessed_command()
769 	unsigned int last_idx = end_wptr & ring->buf_mask;  in amdgpu_ring_backup_unprocessed_command()
772 	/* Backup the contents of the ring buffer. */  in amdgpu_ring_backup_unprocessed_command()
773 	for (i = first_idx; i != last_idx; ++i, i &= ring->buf_mask)  in amdgpu_ring_backup_unprocessed_command()
774 		ring->ring_backup[ring->ring_backup_entries_to_copy++] = ring->ring[i];  in amdgpu_ring_backup_unprocessed_command()
777 void amdgpu_ring_backup_unprocessed_commands(struct amdgpu_ring *ring,  in amdgpu_ring_backup_unprocessed_commands()  argument
786 	last_seq = amdgpu_fence_read(ring) & ring->fence_drv.num_fences_mask;  in amdgpu_ring_backup_unprocessed_commands()
787 	seq = ring->fence_drv.sync_seq & ring->fence_drv.num_fences_mask;  in amdgpu_ring_backup_unprocessed_commands()
788 	wptr = ring->fence_drv.signalled_wptr;  in amdgpu_ring_backup_unprocessed_commands()
789 	ring->ring_backup_entries_to_copy = 0;  in amdgpu_ring_backup_unprocessed_commands()
793 		last_seq &= ring->fence_drv.num_fences_mask;  in amdgpu_ring_backup_unprocessed_commands()
795 		ptr = &ring->fence_drv.fences[last_seq];  in amdgpu_ring_backup_unprocessed_commands()
802 			/* save everything if the ring is not guilty, otherwise  in amdgpu_ring_backup_unprocessed_commands()
807 				amdgpu_ring_backup_unprocessed_command(ring, wptr,  in amdgpu_ring_backup_unprocessed_commands()
811 				amdgpu_ring_backup_unprocessed_command(ring,  in amdgpu_ring_backup_unprocessed_commands()
833 	return (const char *)to_amdgpu_fence(f)->ring->name;  in amdgpu_fence_get_timeline_name()
846 	if (!timer_pending(&to_amdgpu_fence(f)->ring->fence_drv.fallback_timer))  in amdgpu_fence_enable_signaling()
847 		amdgpu_fence_schedule_fallback(to_amdgpu_fence(f)->ring);  in amdgpu_fence_enable_signaling()
897 		struct amdgpu_ring *ring = adev->rings[i];  in amdgpu_debugfs_fence_info_show()  local
899 		if (!ring || !ring->fence_drv.initialized)  in amdgpu_debugfs_fence_info_show()
902 		amdgpu_fence_process(ring);  in amdgpu_debugfs_fence_info_show()
904 		seq_printf(m, "--- ring %d (%s) ---\n", i, ring->name);  in amdgpu_debugfs_fence_info_show()
906 			   atomic_read(&ring->fence_drv.last_seq));  in amdgpu_debugfs_fence_info_show()
908 			   ring->fence_drv.sync_seq);  in amdgpu_debugfs_fence_info_show()
910 		if (ring->funcs->type == AMDGPU_RING_TYPE_GFX ||  in amdgpu_debugfs_fence_info_show()
911 		    ring->funcs->type == AMDGPU_RING_TYPE_SDMA) {  in amdgpu_debugfs_fence_info_show()
913 				   le32_to_cpu(*ring->trail_fence_cpu_addr));  in amdgpu_debugfs_fence_info_show()
915 				   ring->trail_seq);  in amdgpu_debugfs_fence_info_show()
918 		if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)  in amdgpu_debugfs_fence_info_show()
923 			   le32_to_cpu(*(ring->fence_drv.cpu_addr + 2)));  in amdgpu_debugfs_fence_info_show()
926 			   le32_to_cpu(*(ring->fence_drv.cpu_addr + 4)));  in amdgpu_debugfs_fence_info_show()
929 			   le32_to_cpu(*(ring->fence_drv.cpu_addr + 6)));  in amdgpu_debugfs_fence_info_show()