43 static inline struct amdgpu_mux_entry *amdgpu_ring_mux_sw_entry(struct amdgpu_ring_mux *mux,  in amdgpu_ring_mux_sw_entry()  argument
46 	return ring->entry_index < mux->ring_entry_size ?  in amdgpu_ring_mux_sw_entry()
47 			&mux->ring_entry[ring->entry_index] : NULL;  in amdgpu_ring_mux_sw_entry()
51 static void amdgpu_ring_mux_copy_pkt_from_sw_ring(struct amdgpu_ring_mux *mux,  in amdgpu_ring_mux_copy_pkt_from_sw_ring()  argument
56 	struct amdgpu_ring *real_ring = mux->real_ring;  in amdgpu_ring_mux_copy_pkt_from_sw_ring()
76 static void amdgpu_mux_resubmit_chunks(struct amdgpu_ring_mux *mux)  in amdgpu_mux_resubmit_chunks()  argument
84 	if (!mux->s_resubmit)  in amdgpu_mux_resubmit_chunks()
87 	for (i = 0; i < mux->num_ring_entries; i++) {  in amdgpu_mux_resubmit_chunks()
88 		if (mux->ring_entry[i].ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT) {  in amdgpu_mux_resubmit_chunks()
89 			e = &mux->ring_entry[i];  in amdgpu_mux_resubmit_chunks()
100 	seq = mux->seqno_to_resubmit;  in amdgpu_mux_resubmit_chunks()
118 				amdgpu_ring_mux_copy_pkt_from_sw_ring(mux, e->ring,  in amdgpu_mux_resubmit_chunks()
121 				mux->wptr_resubmit = chunk->end;  in amdgpu_mux_resubmit_chunks()
122 				amdgpu_ring_commit(mux->real_ring);  in amdgpu_mux_resubmit_chunks()
127 	del_timer(&mux->resubmit_timer);  in amdgpu_mux_resubmit_chunks()
128 	mux->s_resubmit = false;  in amdgpu_mux_resubmit_chunks()
131 static void amdgpu_ring_mux_schedule_resubmit(struct amdgpu_ring_mux *mux)  in amdgpu_ring_mux_schedule_resubmit()  argument
133 	mod_timer(&mux->resubmit_timer, jiffies + AMDGPU_MUX_RESUBMIT_JIFFIES_TIMEOUT);  in amdgpu_ring_mux_schedule_resubmit()
138 	struct amdgpu_ring_mux *mux = from_timer(mux, t, resubmit_timer);  in amdgpu_mux_resubmit_fallback()  local
140 	if (!spin_trylock(&mux->lock)) {  in amdgpu_mux_resubmit_fallback()
141 		amdgpu_ring_mux_schedule_resubmit(mux);  in amdgpu_mux_resubmit_fallback()
145 	amdgpu_mux_resubmit_chunks(mux);  in amdgpu_mux_resubmit_fallback()
146 	spin_unlock(&mux->lock);  in amdgpu_mux_resubmit_fallback()
149 int amdgpu_ring_mux_init(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring,  in amdgpu_ring_mux_init()  argument
152 	mux->real_ring = ring;  in amdgpu_ring_mux_init()
153 	mux->num_ring_entries = 0;  in amdgpu_ring_mux_init()
155 	mux->ring_entry = kcalloc(entry_size, sizeof(struct amdgpu_mux_entry), GFP_KERNEL);  in amdgpu_ring_mux_init()
156 	if (!mux->ring_entry)  in amdgpu_ring_mux_init()
159 	mux->ring_entry_size = entry_size;  in amdgpu_ring_mux_init()
160 	mux->s_resubmit = false;  in amdgpu_ring_mux_init()
170 	spin_lock_init(&mux->lock);  in amdgpu_ring_mux_init()
171 	timer_setup(&mux->resubmit_timer, amdgpu_mux_resubmit_fallback, 0);  in amdgpu_ring_mux_init()
176 void amdgpu_ring_mux_fini(struct amdgpu_ring_mux *mux)  in amdgpu_ring_mux_fini()  argument
182 	for (i = 0; i < mux->num_ring_entries; i++) {  in amdgpu_ring_mux_fini()
183 		e = &mux->ring_entry[i];  in amdgpu_ring_mux_fini()
190 	kfree(mux->ring_entry);  in amdgpu_ring_mux_fini()
191 	mux->ring_entry = NULL;  in amdgpu_ring_mux_fini()
192 	mux->num_ring_entries = 0;  in amdgpu_ring_mux_fini()
193 	mux->ring_entry_size = 0;  in amdgpu_ring_mux_fini()
196 int amdgpu_ring_mux_add_sw_ring(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring)  in amdgpu_ring_mux_add_sw_ring()  argument
200 	if (mux->num_ring_entries >= mux->ring_entry_size) {  in amdgpu_ring_mux_add_sw_ring()
205 	e = &mux->ring_entry[mux->num_ring_entries];  in amdgpu_ring_mux_add_sw_ring()
206 	ring->entry_index = mux->num_ring_entries;  in amdgpu_ring_mux_add_sw_ring()
210 	mux->num_ring_entries += 1;  in amdgpu_ring_mux_add_sw_ring()
214 void amdgpu_ring_mux_set_wptr(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring, u64 wptr)  in amdgpu_ring_mux_set_wptr()  argument
218 	spin_lock(&mux->lock);  in amdgpu_ring_mux_set_wptr()
221 		amdgpu_mux_resubmit_chunks(mux);  in amdgpu_ring_mux_set_wptr()
223 	e = amdgpu_ring_mux_sw_entry(mux, ring);  in amdgpu_ring_mux_set_wptr()
226 		spin_unlock(&mux->lock);  in amdgpu_ring_mux_set_wptr()
231 	if (ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT && mux->pending_trailing_fence_signaled) {  in amdgpu_ring_mux_set_wptr()
232 		spin_unlock(&mux->lock);  in amdgpu_ring_mux_set_wptr()
238 	if (ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT && e->sw_cptr < mux->wptr_resubmit)  in amdgpu_ring_mux_set_wptr()
239 		e->sw_cptr = mux->wptr_resubmit;  in amdgpu_ring_mux_set_wptr()
241 	e->start_ptr_in_hw_ring = mux->real_ring->wptr;  in amdgpu_ring_mux_set_wptr()
244 	if (ring->hw_prio > AMDGPU_RING_PRIO_DEFAULT || mux->wptr_resubmit < wptr) {  in amdgpu_ring_mux_set_wptr()
245 		amdgpu_ring_mux_copy_pkt_from_sw_ring(mux, ring, e->sw_cptr, wptr);  in amdgpu_ring_mux_set_wptr()
246 		e->end_ptr_in_hw_ring = mux->real_ring->wptr;  in amdgpu_ring_mux_set_wptr()
247 		amdgpu_ring_commit(mux->real_ring);  in amdgpu_ring_mux_set_wptr()
249 		e->end_ptr_in_hw_ring = mux->real_ring->wptr;  in amdgpu_ring_mux_set_wptr()
251 	spin_unlock(&mux->lock);  in amdgpu_ring_mux_set_wptr()
254 u64 amdgpu_ring_mux_get_wptr(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring)  in amdgpu_ring_mux_get_wptr()  argument
258 	e = amdgpu_ring_mux_sw_entry(mux, ring);  in amdgpu_ring_mux_get_wptr()
269  * @mux: the multiplexer the software rings attach to
283 u64 amdgpu_ring_mux_get_rptr(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring)  in amdgpu_ring_mux_get_rptr()  argument
288 	e = amdgpu_ring_mux_sw_entry(mux, ring);  in amdgpu_ring_mux_get_rptr()
294 	readp = amdgpu_ring_get_rptr(mux->real_ring);  in amdgpu_ring_mux_get_rptr()
296 	start = e->start_ptr_in_hw_ring & mux->real_ring->buf_mask;  in amdgpu_ring_mux_get_rptr()
297 	end = e->end_ptr_in_hw_ring & mux->real_ring->buf_mask;  in amdgpu_ring_mux_get_rptr()
300 			readp += mux->real_ring->ring_size >> 2;  in amdgpu_ring_mux_get_rptr()
301 		end += mux->real_ring->ring_size >> 2;  in amdgpu_ring_mux_get_rptr()
320 	struct amdgpu_ring_mux *mux = &adev->gfx.muxer;  in amdgpu_sw_ring_get_rptr_gfx()  local
323 	return amdgpu_ring_mux_get_rptr(mux, ring);  in amdgpu_sw_ring_get_rptr_gfx()
329 	struct amdgpu_ring_mux *mux = &adev->gfx.muxer;  in amdgpu_sw_ring_get_wptr_gfx()  local
332 	return amdgpu_ring_mux_get_wptr(mux, ring);  in amdgpu_sw_ring_get_wptr_gfx()
338 	struct amdgpu_ring_mux *mux = &adev->gfx.muxer;  in amdgpu_sw_ring_set_wptr_gfx()  local
341 	amdgpu_ring_mux_set_wptr(mux, ring, ring->wptr);  in amdgpu_sw_ring_set_wptr_gfx()
363 static int amdgpu_mcbp_scan(struct amdgpu_ring_mux *mux)  in amdgpu_mcbp_scan()  argument
369 	for (i = 0; i < mux->num_ring_entries; i++) {  in amdgpu_mcbp_scan()
370 		ring = mux->ring_entry[i].ring;  in amdgpu_mcbp_scan()
379 	return need_preempt && !mux->s_resubmit;  in amdgpu_mcbp_scan()
383 static int amdgpu_mcbp_trigger_preempt(struct amdgpu_ring_mux *mux)  in amdgpu_mcbp_trigger_preempt()  argument
387 	spin_lock(&mux->lock);  in amdgpu_mcbp_trigger_preempt()
388 	mux->pending_trailing_fence_signaled = true;  in amdgpu_mcbp_trigger_preempt()
389 	r = amdgpu_ring_preempt_ib(mux->real_ring);  in amdgpu_mcbp_trigger_preempt()
390 	spin_unlock(&mux->lock);  in amdgpu_mcbp_trigger_preempt()
397 	struct amdgpu_ring_mux *mux = &adev->gfx.muxer;  in amdgpu_sw_ring_ib_begin()  local
401 		if (amdgpu_mcbp_scan(mux) > 0)  in amdgpu_sw_ring_ib_begin()
402 			amdgpu_mcbp_trigger_preempt(mux);  in amdgpu_sw_ring_ib_begin()
406 	amdgpu_ring_mux_start_ib(mux, ring);  in amdgpu_sw_ring_ib_begin()
412 	struct amdgpu_ring_mux *mux = &adev->gfx.muxer;  in amdgpu_sw_ring_ib_end()  local
417 	amdgpu_ring_mux_end_ib(mux, ring);  in amdgpu_sw_ring_ib_end()
423 	struct amdgpu_ring_mux *mux = &adev->gfx.muxer;  in amdgpu_sw_ring_ib_mark_offset()  local
431 	amdgpu_ring_mux_ib_mark_offset(mux, ring, offset, type);  in amdgpu_sw_ring_ib_mark_offset()
434 void amdgpu_ring_mux_start_ib(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring)  in amdgpu_ring_mux_start_ib()  argument
439 	spin_lock(&mux->lock);  in amdgpu_ring_mux_start_ib()
440 	amdgpu_mux_resubmit_chunks(mux);  in amdgpu_ring_mux_start_ib()
441 	spin_unlock(&mux->lock);  in amdgpu_ring_mux_start_ib()
443 	e = amdgpu_ring_mux_sw_entry(mux, ring);  in amdgpu_ring_mux_start_ib()
463 static void scan_and_remove_signaled_chunk(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring)  in scan_and_remove_signaled_chunk()  argument
469 	e = amdgpu_ring_mux_sw_entry(mux, ring);  in scan_and_remove_signaled_chunk()
485 void amdgpu_ring_mux_ib_mark_offset(struct amdgpu_ring_mux *mux,  in amdgpu_ring_mux_ib_mark_offset()  argument
492 	e = amdgpu_ring_mux_sw_entry(mux, ring);  in amdgpu_ring_mux_ib_mark_offset()
520 void amdgpu_ring_mux_end_ib(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring)  in amdgpu_ring_mux_end_ib()  argument
525 	e = amdgpu_ring_mux_sw_entry(mux, ring);  in amdgpu_ring_mux_end_ib()
540 	scan_and_remove_signaled_chunk(mux, ring);  in amdgpu_ring_mux_end_ib()
543 bool amdgpu_mcbp_handle_trailing_fence_irq(struct amdgpu_ring_mux *mux)  in amdgpu_mcbp_handle_trailing_fence_irq()  argument
549 	if (!mux->pending_trailing_fence_signaled)  in amdgpu_mcbp_handle_trailing_fence_irq()
552 	if (mux->real_ring->trail_seq != le32_to_cpu(*mux->real_ring->trail_fence_cpu_addr))  in amdgpu_mcbp_handle_trailing_fence_irq()
555 	for (i = 0; i < mux->num_ring_entries; i++) {  in amdgpu_mcbp_handle_trailing_fence_irq()
556 		e = &mux->ring_entry[i];  in amdgpu_mcbp_handle_trailing_fence_irq()
570 		mux->s_resubmit = true;  in amdgpu_mcbp_handle_trailing_fence_irq()
571 		mux->seqno_to_resubmit = ring->fence_drv.sync_seq;  in amdgpu_mcbp_handle_trailing_fence_irq()
572 		amdgpu_ring_mux_schedule_resubmit(mux);  in amdgpu_mcbp_handle_trailing_fence_irq()
575 	mux->pending_trailing_fence_signaled = false;  in amdgpu_mcbp_handle_trailing_fence_irq()