Lines Matching +full:mux +full:-
43 static inline struct amdgpu_mux_entry *amdgpu_ring_mux_sw_entry(struct amdgpu_ring_mux *mux, in amdgpu_ring_mux_sw_entry() argument
46 return ring->entry_index < mux->ring_entry_size ? in amdgpu_ring_mux_sw_entry()
47 &mux->ring_entry[ring->entry_index] : NULL; in amdgpu_ring_mux_sw_entry()
51 static void amdgpu_ring_mux_copy_pkt_from_sw_ring(struct amdgpu_ring_mux *mux, in amdgpu_ring_mux_copy_pkt_from_sw_ring() argument
56 struct amdgpu_ring *real_ring = mux->real_ring; in amdgpu_ring_mux_copy_pkt_from_sw_ring()
58 start = s_start & ring->buf_mask; in amdgpu_ring_mux_copy_pkt_from_sw_ring()
59 end = s_end & ring->buf_mask; in amdgpu_ring_mux_copy_pkt_from_sw_ring()
66 amdgpu_ring_alloc(real_ring, (ring->ring_size >> 2) + end - start); in amdgpu_ring_mux_copy_pkt_from_sw_ring()
67 amdgpu_ring_write_multiple(real_ring, (void *)&ring->ring[start], in amdgpu_ring_mux_copy_pkt_from_sw_ring()
68 (ring->ring_size >> 2) - start); in amdgpu_ring_mux_copy_pkt_from_sw_ring()
69 amdgpu_ring_write_multiple(real_ring, (void *)&ring->ring[0], end); in amdgpu_ring_mux_copy_pkt_from_sw_ring()
71 amdgpu_ring_alloc(real_ring, end - start); in amdgpu_ring_mux_copy_pkt_from_sw_ring()
72 amdgpu_ring_write_multiple(real_ring, (void *)&ring->ring[start], end - start); in amdgpu_ring_mux_copy_pkt_from_sw_ring()
76 static void amdgpu_mux_resubmit_chunks(struct amdgpu_ring_mux *mux) in amdgpu_mux_resubmit_chunks() argument
84 if (!mux->s_resubmit) in amdgpu_mux_resubmit_chunks()
87 for (i = 0; i < mux->num_ring_entries; i++) { in amdgpu_mux_resubmit_chunks()
88 if (mux->ring_entry[i].ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT) { in amdgpu_mux_resubmit_chunks()
89 e = &mux->ring_entry[i]; in amdgpu_mux_resubmit_chunks()
99 last_seq = atomic_read(&e->ring->fence_drv.last_seq); in amdgpu_mux_resubmit_chunks()
100 seq = mux->seqno_to_resubmit; in amdgpu_mux_resubmit_chunks()
103 list_for_each_entry(chunk, &e->list, entry) { in amdgpu_mux_resubmit_chunks()
104 if (chunk->sync_seq > last_seq && chunk->sync_seq <= seq) { in amdgpu_mux_resubmit_chunks()
105 amdgpu_fence_update_start_timestamp(e->ring, in amdgpu_mux_resubmit_chunks()
106 chunk->sync_seq, in amdgpu_mux_resubmit_chunks()
108 if (chunk->sync_seq == in amdgpu_mux_resubmit_chunks()
109 le32_to_cpu(*(e->ring->fence_drv.cpu_addr + 2))) { in amdgpu_mux_resubmit_chunks()
110 if (chunk->cntl_offset <= e->ring->buf_mask) in amdgpu_mux_resubmit_chunks()
111 amdgpu_ring_patch_cntl(e->ring, in amdgpu_mux_resubmit_chunks()
112 chunk->cntl_offset); in amdgpu_mux_resubmit_chunks()
113 if (chunk->ce_offset <= e->ring->buf_mask) in amdgpu_mux_resubmit_chunks()
114 amdgpu_ring_patch_ce(e->ring, chunk->ce_offset); in amdgpu_mux_resubmit_chunks()
115 if (chunk->de_offset <= e->ring->buf_mask) in amdgpu_mux_resubmit_chunks()
116 amdgpu_ring_patch_de(e->ring, chunk->de_offset); in amdgpu_mux_resubmit_chunks()
118 amdgpu_ring_mux_copy_pkt_from_sw_ring(mux, e->ring, in amdgpu_mux_resubmit_chunks()
119 chunk->start, in amdgpu_mux_resubmit_chunks()
120 chunk->end); in amdgpu_mux_resubmit_chunks()
121 mux->wptr_resubmit = chunk->end; in amdgpu_mux_resubmit_chunks()
122 amdgpu_ring_commit(mux->real_ring); in amdgpu_mux_resubmit_chunks()
127 del_timer(&mux->resubmit_timer); in amdgpu_mux_resubmit_chunks()
128 mux->s_resubmit = false; in amdgpu_mux_resubmit_chunks()
131 static void amdgpu_ring_mux_schedule_resubmit(struct amdgpu_ring_mux *mux) in amdgpu_ring_mux_schedule_resubmit() argument
133 mod_timer(&mux->resubmit_timer, jiffies + AMDGPU_MUX_RESUBMIT_JIFFIES_TIMEOUT); in amdgpu_ring_mux_schedule_resubmit()
138 struct amdgpu_ring_mux *mux = from_timer(mux, t, resubmit_timer); in amdgpu_mux_resubmit_fallback() local
140 if (!spin_trylock(&mux->lock)) { in amdgpu_mux_resubmit_fallback()
141 amdgpu_ring_mux_schedule_resubmit(mux); in amdgpu_mux_resubmit_fallback()
145 amdgpu_mux_resubmit_chunks(mux); in amdgpu_mux_resubmit_fallback()
146 spin_unlock(&mux->lock); in amdgpu_mux_resubmit_fallback()
149 int amdgpu_ring_mux_init(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring, in amdgpu_ring_mux_init() argument
152 mux->real_ring = ring; in amdgpu_ring_mux_init()
153 mux->num_ring_entries = 0; in amdgpu_ring_mux_init()
155 mux->ring_entry = kcalloc(entry_size, sizeof(struct amdgpu_mux_entry), GFP_KERNEL); in amdgpu_ring_mux_init()
156 if (!mux->ring_entry) in amdgpu_ring_mux_init()
157 return -ENOMEM; in amdgpu_ring_mux_init()
159 mux->ring_entry_size = entry_size; in amdgpu_ring_mux_init()
160 mux->s_resubmit = false; in amdgpu_ring_mux_init()
167 return -ENOMEM; in amdgpu_ring_mux_init()
170 spin_lock_init(&mux->lock); in amdgpu_ring_mux_init()
171 timer_setup(&mux->resubmit_timer, amdgpu_mux_resubmit_fallback, 0); in amdgpu_ring_mux_init()
176 void amdgpu_ring_mux_fini(struct amdgpu_ring_mux *mux) in amdgpu_ring_mux_fini() argument
182 for (i = 0; i < mux->num_ring_entries; i++) { in amdgpu_ring_mux_fini()
183 e = &mux->ring_entry[i]; in amdgpu_ring_mux_fini()
184 list_for_each_entry_safe(chunk, chunk2, &e->list, entry) { in amdgpu_ring_mux_fini()
185 list_del(&chunk->entry); in amdgpu_ring_mux_fini()
190 kfree(mux->ring_entry); in amdgpu_ring_mux_fini()
191 mux->ring_entry = NULL; in amdgpu_ring_mux_fini()
192 mux->num_ring_entries = 0; in amdgpu_ring_mux_fini()
193 mux->ring_entry_size = 0; in amdgpu_ring_mux_fini()
196 int amdgpu_ring_mux_add_sw_ring(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring) in amdgpu_ring_mux_add_sw_ring() argument
200 if (mux->num_ring_entries >= mux->ring_entry_size) { in amdgpu_ring_mux_add_sw_ring()
202 return -ENOENT; in amdgpu_ring_mux_add_sw_ring()
205 e = &mux->ring_entry[mux->num_ring_entries]; in amdgpu_ring_mux_add_sw_ring()
206 ring->entry_index = mux->num_ring_entries; in amdgpu_ring_mux_add_sw_ring()
207 e->ring = ring; in amdgpu_ring_mux_add_sw_ring()
209 INIT_LIST_HEAD(&e->list); in amdgpu_ring_mux_add_sw_ring()
210 mux->num_ring_entries += 1; in amdgpu_ring_mux_add_sw_ring()
214 void amdgpu_ring_mux_set_wptr(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring, u64 wptr) in amdgpu_ring_mux_set_wptr() argument
218 spin_lock(&mux->lock); in amdgpu_ring_mux_set_wptr()
220 if (ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT) in amdgpu_ring_mux_set_wptr()
221 amdgpu_mux_resubmit_chunks(mux); in amdgpu_ring_mux_set_wptr()
223 e = amdgpu_ring_mux_sw_entry(mux, ring); in amdgpu_ring_mux_set_wptr()
226 spin_unlock(&mux->lock); in amdgpu_ring_mux_set_wptr()
231 if (ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT && mux->pending_trailing_fence_signaled) { in amdgpu_ring_mux_set_wptr()
232 spin_unlock(&mux->lock); in amdgpu_ring_mux_set_wptr()
236 e->sw_cptr = e->sw_wptr; in amdgpu_ring_mux_set_wptr()
238 if (ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT && e->sw_cptr < mux->wptr_resubmit) in amdgpu_ring_mux_set_wptr()
239 e->sw_cptr = mux->wptr_resubmit; in amdgpu_ring_mux_set_wptr()
240 e->sw_wptr = wptr; in amdgpu_ring_mux_set_wptr()
241 e->start_ptr_in_hw_ring = mux->real_ring->wptr; in amdgpu_ring_mux_set_wptr()
244 if (ring->hw_prio > AMDGPU_RING_PRIO_DEFAULT || mux->wptr_resubmit < wptr) { in amdgpu_ring_mux_set_wptr()
245 amdgpu_ring_mux_copy_pkt_from_sw_ring(mux, ring, e->sw_cptr, wptr); in amdgpu_ring_mux_set_wptr()
246 e->end_ptr_in_hw_ring = mux->real_ring->wptr; in amdgpu_ring_mux_set_wptr()
247 amdgpu_ring_commit(mux->real_ring); in amdgpu_ring_mux_set_wptr()
249 e->end_ptr_in_hw_ring = mux->real_ring->wptr; in amdgpu_ring_mux_set_wptr()
251 spin_unlock(&mux->lock); in amdgpu_ring_mux_set_wptr()
254 u64 amdgpu_ring_mux_get_wptr(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring) in amdgpu_ring_mux_get_wptr() argument
258 e = amdgpu_ring_mux_sw_entry(mux, ring); in amdgpu_ring_mux_get_wptr()
264 return e->sw_wptr; in amdgpu_ring_mux_get_wptr()
268 * amdgpu_ring_mux_get_rptr - get the readptr of the software ring
269 * @mux: the multiplexer the software rings attach to
283 u64 amdgpu_ring_mux_get_rptr(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring) in amdgpu_ring_mux_get_rptr() argument
288 e = amdgpu_ring_mux_sw_entry(mux, ring); in amdgpu_ring_mux_get_rptr()
294 readp = amdgpu_ring_get_rptr(mux->real_ring); in amdgpu_ring_mux_get_rptr()
296 start = e->start_ptr_in_hw_ring & mux->real_ring->buf_mask; in amdgpu_ring_mux_get_rptr()
297 end = e->end_ptr_in_hw_ring & mux->real_ring->buf_mask; in amdgpu_ring_mux_get_rptr()
300 readp += mux->real_ring->ring_size >> 2; in amdgpu_ring_mux_get_rptr()
301 end += mux->real_ring->ring_size >> 2; in amdgpu_ring_mux_get_rptr()
305 offset = readp - start; in amdgpu_ring_mux_get_rptr()
306 e->sw_rptr = (e->sw_cptr + offset) & ring->buf_mask; in amdgpu_ring_mux_get_rptr()
308 e->sw_rptr = e->sw_cptr; in amdgpu_ring_mux_get_rptr()
311 e->sw_rptr = e->sw_wptr; in amdgpu_ring_mux_get_rptr()
314 return e->sw_rptr; in amdgpu_ring_mux_get_rptr()
319 struct amdgpu_device *adev = ring->adev; in amdgpu_sw_ring_get_rptr_gfx()
320 struct amdgpu_ring_mux *mux = &adev->gfx.muxer; in amdgpu_sw_ring_get_rptr_gfx() local
322 WARN_ON(!ring->is_sw_ring); in amdgpu_sw_ring_get_rptr_gfx()
323 return amdgpu_ring_mux_get_rptr(mux, ring); in amdgpu_sw_ring_get_rptr_gfx()
328 struct amdgpu_device *adev = ring->adev; in amdgpu_sw_ring_get_wptr_gfx()
329 struct amdgpu_ring_mux *mux = &adev->gfx.muxer; in amdgpu_sw_ring_get_wptr_gfx() local
331 WARN_ON(!ring->is_sw_ring); in amdgpu_sw_ring_get_wptr_gfx()
332 return amdgpu_ring_mux_get_wptr(mux, ring); in amdgpu_sw_ring_get_wptr_gfx()
337 struct amdgpu_device *adev = ring->adev; in amdgpu_sw_ring_set_wptr_gfx()
338 struct amdgpu_ring_mux *mux = &adev->gfx.muxer; in amdgpu_sw_ring_set_wptr_gfx() local
340 WARN_ON(!ring->is_sw_ring); in amdgpu_sw_ring_set_wptr_gfx()
341 amdgpu_ring_mux_set_wptr(mux, ring, ring->wptr); in amdgpu_sw_ring_set_wptr_gfx()
347 WARN_ON(!ring->is_sw_ring); in amdgpu_sw_ring_insert_nop()
363 static int amdgpu_mcbp_scan(struct amdgpu_ring_mux *mux) in amdgpu_mcbp_scan() argument
369 for (i = 0; i < mux->num_ring_entries; i++) { in amdgpu_mcbp_scan()
370 ring = mux->ring_entry[i].ring; in amdgpu_mcbp_scan()
371 if (ring->hw_prio > AMDGPU_RING_PRIO_DEFAULT && in amdgpu_mcbp_scan()
374 if (ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT && in amdgpu_mcbp_scan()
379 return need_preempt && !mux->s_resubmit; in amdgpu_mcbp_scan()
382 /* Trigger Mid-Command Buffer Preemption (MCBP) and find if we need to resubmit. */
383 static int amdgpu_mcbp_trigger_preempt(struct amdgpu_ring_mux *mux) in amdgpu_mcbp_trigger_preempt() argument
387 spin_lock(&mux->lock); in amdgpu_mcbp_trigger_preempt()
388 mux->pending_trailing_fence_signaled = true; in amdgpu_mcbp_trigger_preempt()
389 r = amdgpu_ring_preempt_ib(mux->real_ring); in amdgpu_mcbp_trigger_preempt()
390 spin_unlock(&mux->lock); in amdgpu_mcbp_trigger_preempt()
396 struct amdgpu_device *adev = ring->adev; in amdgpu_sw_ring_ib_begin()
397 struct amdgpu_ring_mux *mux = &adev->gfx.muxer; in amdgpu_sw_ring_ib_begin() local
399 WARN_ON(!ring->is_sw_ring); in amdgpu_sw_ring_ib_begin()
400 if (adev->gfx.mcbp && ring->hw_prio > AMDGPU_RING_PRIO_DEFAULT) { in amdgpu_sw_ring_ib_begin()
401 if (amdgpu_mcbp_scan(mux) > 0) in amdgpu_sw_ring_ib_begin()
402 amdgpu_mcbp_trigger_preempt(mux); in amdgpu_sw_ring_ib_begin()
406 amdgpu_ring_mux_start_ib(mux, ring); in amdgpu_sw_ring_ib_begin()
411 struct amdgpu_device *adev = ring->adev; in amdgpu_sw_ring_ib_end()
412 struct amdgpu_ring_mux *mux = &adev->gfx.muxer; in amdgpu_sw_ring_ib_end() local
414 WARN_ON(!ring->is_sw_ring); in amdgpu_sw_ring_ib_end()
415 if (ring->hw_prio > AMDGPU_RING_PRIO_DEFAULT) in amdgpu_sw_ring_ib_end()
417 amdgpu_ring_mux_end_ib(mux, ring); in amdgpu_sw_ring_ib_end()
422 struct amdgpu_device *adev = ring->adev; in amdgpu_sw_ring_ib_mark_offset()
423 struct amdgpu_ring_mux *mux = &adev->gfx.muxer; in amdgpu_sw_ring_ib_mark_offset() local
426 if (ring->hw_prio > AMDGPU_RING_PRIO_DEFAULT) in amdgpu_sw_ring_ib_mark_offset()
429 offset = ring->wptr & ring->buf_mask; in amdgpu_sw_ring_ib_mark_offset()
431 amdgpu_ring_mux_ib_mark_offset(mux, ring, offset, type); in amdgpu_sw_ring_ib_mark_offset()
434 void amdgpu_ring_mux_start_ib(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring) in amdgpu_ring_mux_start_ib() argument
439 spin_lock(&mux->lock); in amdgpu_ring_mux_start_ib()
440 amdgpu_mux_resubmit_chunks(mux); in amdgpu_ring_mux_start_ib()
441 spin_unlock(&mux->lock); in amdgpu_ring_mux_start_ib()
443 e = amdgpu_ring_mux_sw_entry(mux, ring); in amdgpu_ring_mux_start_ib()
455 chunk->start = ring->wptr; in amdgpu_ring_mux_start_ib()
457 chunk->cntl_offset = ring->buf_mask + 1; in amdgpu_ring_mux_start_ib()
458 chunk->de_offset = ring->buf_mask + 1; in amdgpu_ring_mux_start_ib()
459 chunk->ce_offset = ring->buf_mask + 1; in amdgpu_ring_mux_start_ib()
460 list_add_tail(&chunk->entry, &e->list); in amdgpu_ring_mux_start_ib()
463 static void scan_and_remove_signaled_chunk(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring) in scan_and_remove_signaled_chunk() argument
469 e = amdgpu_ring_mux_sw_entry(mux, ring); in scan_and_remove_signaled_chunk()
475 last_seq = atomic_read(&ring->fence_drv.last_seq); in scan_and_remove_signaled_chunk()
477 list_for_each_entry_safe(chunk, tmp, &e->list, entry) { in scan_and_remove_signaled_chunk()
478 if (chunk->sync_seq <= last_seq) { in scan_and_remove_signaled_chunk()
479 list_del(&chunk->entry); in scan_and_remove_signaled_chunk()
485 void amdgpu_ring_mux_ib_mark_offset(struct amdgpu_ring_mux *mux, in amdgpu_ring_mux_ib_mark_offset() argument
492 e = amdgpu_ring_mux_sw_entry(mux, ring); in amdgpu_ring_mux_ib_mark_offset()
498 chunk = list_last_entry(&e->list, struct amdgpu_mux_chunk, entry); in amdgpu_ring_mux_ib_mark_offset()
506 chunk->cntl_offset = offset; in amdgpu_ring_mux_ib_mark_offset()
509 chunk->de_offset = offset; in amdgpu_ring_mux_ib_mark_offset()
512 chunk->ce_offset = offset; in amdgpu_ring_mux_ib_mark_offset()
520 void amdgpu_ring_mux_end_ib(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring) in amdgpu_ring_mux_end_ib() argument
525 e = amdgpu_ring_mux_sw_entry(mux, ring); in amdgpu_ring_mux_end_ib()
531 chunk = list_last_entry(&e->list, struct amdgpu_mux_chunk, entry); in amdgpu_ring_mux_end_ib()
537 chunk->end = ring->wptr; in amdgpu_ring_mux_end_ib()
538 chunk->sync_seq = READ_ONCE(ring->fence_drv.sync_seq); in amdgpu_ring_mux_end_ib()
540 scan_and_remove_signaled_chunk(mux, ring); in amdgpu_ring_mux_end_ib()
543 bool amdgpu_mcbp_handle_trailing_fence_irq(struct amdgpu_ring_mux *mux) in amdgpu_mcbp_handle_trailing_fence_irq() argument
549 if (!mux->pending_trailing_fence_signaled) in amdgpu_mcbp_handle_trailing_fence_irq()
552 if (mux->real_ring->trail_seq != le32_to_cpu(*mux->real_ring->trail_fence_cpu_addr)) in amdgpu_mcbp_handle_trailing_fence_irq()
555 for (i = 0; i < mux->num_ring_entries; i++) { in amdgpu_mcbp_handle_trailing_fence_irq()
556 e = &mux->ring_entry[i]; in amdgpu_mcbp_handle_trailing_fence_irq()
557 if (e->ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT) { in amdgpu_mcbp_handle_trailing_fence_irq()
558 ring = e->ring; in amdgpu_mcbp_handle_trailing_fence_irq()
570 mux->s_resubmit = true; in amdgpu_mcbp_handle_trailing_fence_irq()
571 mux->seqno_to_resubmit = ring->fence_drv.sync_seq; in amdgpu_mcbp_handle_trailing_fence_irq()
572 amdgpu_ring_mux_schedule_resubmit(mux); in amdgpu_mcbp_handle_trailing_fence_irq()
575 mux->pending_trailing_fence_signaled = false; in amdgpu_mcbp_handle_trailing_fence_irq()