Lines Matching +full:mux +full:-
43 static inline struct amdgpu_mux_entry *amdgpu_ring_mux_sw_entry(struct amdgpu_ring_mux *mux, in amdgpu_ring_mux_sw_entry() argument
46 return ring->entry_index < mux->ring_entry_size ? in amdgpu_ring_mux_sw_entry()
47 &mux->ring_entry[ring->entry_index] : NULL; in amdgpu_ring_mux_sw_entry()
51 static void amdgpu_ring_mux_copy_pkt_from_sw_ring(struct amdgpu_ring_mux *mux, in amdgpu_ring_mux_copy_pkt_from_sw_ring() argument
56 struct amdgpu_ring *real_ring = mux->real_ring; in amdgpu_ring_mux_copy_pkt_from_sw_ring()
58 start = s_start & ring->buf_mask; in amdgpu_ring_mux_copy_pkt_from_sw_ring()
59 end = s_end & ring->buf_mask; in amdgpu_ring_mux_copy_pkt_from_sw_ring()
66 amdgpu_ring_alloc(real_ring, (ring->ring_size >> 2) + end - start); in amdgpu_ring_mux_copy_pkt_from_sw_ring()
67 amdgpu_ring_write_multiple(real_ring, (void *)&ring->ring[start], in amdgpu_ring_mux_copy_pkt_from_sw_ring()
68 (ring->ring_size >> 2) - start); in amdgpu_ring_mux_copy_pkt_from_sw_ring()
69 amdgpu_ring_write_multiple(real_ring, (void *)&ring->ring[0], end); in amdgpu_ring_mux_copy_pkt_from_sw_ring()
71 amdgpu_ring_alloc(real_ring, end - start); in amdgpu_ring_mux_copy_pkt_from_sw_ring()
72 amdgpu_ring_write_multiple(real_ring, (void *)&ring->ring[start], end - start); in amdgpu_ring_mux_copy_pkt_from_sw_ring()
76 static void amdgpu_mux_resubmit_chunks(struct amdgpu_ring_mux *mux) in amdgpu_mux_resubmit_chunks() argument
84 if (!mux->s_resubmit) in amdgpu_mux_resubmit_chunks()
87 for (i = 0; i < mux->num_ring_entries; i++) { in amdgpu_mux_resubmit_chunks()
88 if (mux->ring_entry[i].ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT) { in amdgpu_mux_resubmit_chunks()
89 e = &mux->ring_entry[i]; in amdgpu_mux_resubmit_chunks()
99 last_seq = atomic_read(&e->ring->fence_drv.last_seq); in amdgpu_mux_resubmit_chunks()
100 seq = mux->seqno_to_resubmit; in amdgpu_mux_resubmit_chunks()
103 list_for_each_entry(chunk, &e->list, entry) { in amdgpu_mux_resubmit_chunks()
104 if (chunk->sync_seq > last_seq && chunk->sync_seq <= seq) { in amdgpu_mux_resubmit_chunks()
105 amdgpu_fence_update_start_timestamp(e->ring, in amdgpu_mux_resubmit_chunks()
106 chunk->sync_seq, in amdgpu_mux_resubmit_chunks()
108 if (chunk->sync_seq == in amdgpu_mux_resubmit_chunks()
109 le32_to_cpu(*(e->ring->fence_drv.cpu_addr + 2))) { in amdgpu_mux_resubmit_chunks()
110 if (chunk->cntl_offset <= e->ring->buf_mask) in amdgpu_mux_resubmit_chunks()
111 amdgpu_ring_patch_cntl(e->ring, in amdgpu_mux_resubmit_chunks()
112 chunk->cntl_offset); in amdgpu_mux_resubmit_chunks()
113 if (chunk->ce_offset <= e->ring->buf_mask) in amdgpu_mux_resubmit_chunks()
114 amdgpu_ring_patch_ce(e->ring, chunk->ce_offset); in amdgpu_mux_resubmit_chunks()
115 if (chunk->de_offset <= e->ring->buf_mask) in amdgpu_mux_resubmit_chunks()
116 amdgpu_ring_patch_de(e->ring, chunk->de_offset); in amdgpu_mux_resubmit_chunks()
118 amdgpu_ring_mux_copy_pkt_from_sw_ring(mux, e->ring, in amdgpu_mux_resubmit_chunks()
119 chunk->start, in amdgpu_mux_resubmit_chunks()
120 chunk->end); in amdgpu_mux_resubmit_chunks()
121 mux->wptr_resubmit = chunk->end; in amdgpu_mux_resubmit_chunks()
122 amdgpu_ring_commit(mux->real_ring); in amdgpu_mux_resubmit_chunks()
127 timer_delete(&mux->resubmit_timer); in amdgpu_mux_resubmit_chunks()
128 mux->s_resubmit = false; in amdgpu_mux_resubmit_chunks()
131 static void amdgpu_ring_mux_schedule_resubmit(struct amdgpu_ring_mux *mux) in amdgpu_ring_mux_schedule_resubmit() argument
133 mod_timer(&mux->resubmit_timer, jiffies + AMDGPU_MUX_RESUBMIT_JIFFIES_TIMEOUT); in amdgpu_ring_mux_schedule_resubmit()
138 struct amdgpu_ring_mux *mux = from_timer(mux, t, resubmit_timer); in amdgpu_mux_resubmit_fallback() local
140 if (!spin_trylock(&mux->lock)) { in amdgpu_mux_resubmit_fallback()
141 amdgpu_ring_mux_schedule_resubmit(mux); in amdgpu_mux_resubmit_fallback()
145 amdgpu_mux_resubmit_chunks(mux); in amdgpu_mux_resubmit_fallback()
146 spin_unlock(&mux->lock); in amdgpu_mux_resubmit_fallback()
149 int amdgpu_ring_mux_init(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring, in amdgpu_ring_mux_init() argument
152 mux->real_ring = ring; in amdgpu_ring_mux_init()
153 mux->num_ring_entries = 0; in amdgpu_ring_mux_init()
155 mux->ring_entry = kcalloc(entry_size, sizeof(struct amdgpu_mux_entry), GFP_KERNEL); in amdgpu_ring_mux_init()
156 if (!mux->ring_entry) in amdgpu_ring_mux_init()
157 return -ENOMEM; in amdgpu_ring_mux_init()
159 mux->ring_entry_size = entry_size; in amdgpu_ring_mux_init()
160 mux->s_resubmit = false; in amdgpu_ring_mux_init()
165 return -ENOMEM; in amdgpu_ring_mux_init()
168 spin_lock_init(&mux->lock); in amdgpu_ring_mux_init()
169 timer_setup(&mux->resubmit_timer, amdgpu_mux_resubmit_fallback, 0); in amdgpu_ring_mux_init()
174 void amdgpu_ring_mux_fini(struct amdgpu_ring_mux *mux) in amdgpu_ring_mux_fini() argument
180 for (i = 0; i < mux->num_ring_entries; i++) { in amdgpu_ring_mux_fini()
181 e = &mux->ring_entry[i]; in amdgpu_ring_mux_fini()
182 list_for_each_entry_safe(chunk, chunk2, &e->list, entry) { in amdgpu_ring_mux_fini()
183 list_del(&chunk->entry); in amdgpu_ring_mux_fini()
188 kfree(mux->ring_entry); in amdgpu_ring_mux_fini()
189 mux->ring_entry = NULL; in amdgpu_ring_mux_fini()
190 mux->num_ring_entries = 0; in amdgpu_ring_mux_fini()
191 mux->ring_entry_size = 0; in amdgpu_ring_mux_fini()
194 int amdgpu_ring_mux_add_sw_ring(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring) in amdgpu_ring_mux_add_sw_ring() argument
198 if (mux->num_ring_entries >= mux->ring_entry_size) { in amdgpu_ring_mux_add_sw_ring()
200 return -ENOENT; in amdgpu_ring_mux_add_sw_ring()
203 e = &mux->ring_entry[mux->num_ring_entries]; in amdgpu_ring_mux_add_sw_ring()
204 ring->entry_index = mux->num_ring_entries; in amdgpu_ring_mux_add_sw_ring()
205 e->ring = ring; in amdgpu_ring_mux_add_sw_ring()
207 INIT_LIST_HEAD(&e->list); in amdgpu_ring_mux_add_sw_ring()
208 mux->num_ring_entries += 1; in amdgpu_ring_mux_add_sw_ring()
212 void amdgpu_ring_mux_set_wptr(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring, u64 wptr) in amdgpu_ring_mux_set_wptr() argument
216 spin_lock(&mux->lock); in amdgpu_ring_mux_set_wptr()
218 if (ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT) in amdgpu_ring_mux_set_wptr()
219 amdgpu_mux_resubmit_chunks(mux); in amdgpu_ring_mux_set_wptr()
221 e = amdgpu_ring_mux_sw_entry(mux, ring); in amdgpu_ring_mux_set_wptr()
224 spin_unlock(&mux->lock); in amdgpu_ring_mux_set_wptr()
229 if (ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT && mux->pending_trailing_fence_signaled) { in amdgpu_ring_mux_set_wptr()
230 spin_unlock(&mux->lock); in amdgpu_ring_mux_set_wptr()
234 e->sw_cptr = e->sw_wptr; in amdgpu_ring_mux_set_wptr()
236 if (ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT && e->sw_cptr < mux->wptr_resubmit) in amdgpu_ring_mux_set_wptr()
237 e->sw_cptr = mux->wptr_resubmit; in amdgpu_ring_mux_set_wptr()
238 e->sw_wptr = wptr; in amdgpu_ring_mux_set_wptr()
239 e->start_ptr_in_hw_ring = mux->real_ring->wptr; in amdgpu_ring_mux_set_wptr()
242 if (ring->hw_prio > AMDGPU_RING_PRIO_DEFAULT || mux->wptr_resubmit < wptr) { in amdgpu_ring_mux_set_wptr()
243 amdgpu_ring_mux_copy_pkt_from_sw_ring(mux, ring, e->sw_cptr, wptr); in amdgpu_ring_mux_set_wptr()
244 e->end_ptr_in_hw_ring = mux->real_ring->wptr; in amdgpu_ring_mux_set_wptr()
245 amdgpu_ring_commit(mux->real_ring); in amdgpu_ring_mux_set_wptr()
247 e->end_ptr_in_hw_ring = mux->real_ring->wptr; in amdgpu_ring_mux_set_wptr()
249 spin_unlock(&mux->lock); in amdgpu_ring_mux_set_wptr()
252 u64 amdgpu_ring_mux_get_wptr(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring) in amdgpu_ring_mux_get_wptr() argument
256 e = amdgpu_ring_mux_sw_entry(mux, ring); in amdgpu_ring_mux_get_wptr()
262 return e->sw_wptr; in amdgpu_ring_mux_get_wptr()
266 * amdgpu_ring_mux_get_rptr - get the readptr of the software ring
267 * @mux: the multiplexer the software rings attach to
281 u64 amdgpu_ring_mux_get_rptr(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring) in amdgpu_ring_mux_get_rptr() argument
286 e = amdgpu_ring_mux_sw_entry(mux, ring); in amdgpu_ring_mux_get_rptr()
292 readp = amdgpu_ring_get_rptr(mux->real_ring); in amdgpu_ring_mux_get_rptr()
294 start = e->start_ptr_in_hw_ring & mux->real_ring->buf_mask; in amdgpu_ring_mux_get_rptr()
295 end = e->end_ptr_in_hw_ring & mux->real_ring->buf_mask; in amdgpu_ring_mux_get_rptr()
298 readp += mux->real_ring->ring_size >> 2; in amdgpu_ring_mux_get_rptr()
299 end += mux->real_ring->ring_size >> 2; in amdgpu_ring_mux_get_rptr()
303 offset = readp - start; in amdgpu_ring_mux_get_rptr()
304 e->sw_rptr = (e->sw_cptr + offset) & ring->buf_mask; in amdgpu_ring_mux_get_rptr()
306 e->sw_rptr = e->sw_cptr; in amdgpu_ring_mux_get_rptr()
309 e->sw_rptr = e->sw_wptr; in amdgpu_ring_mux_get_rptr()
312 return e->sw_rptr; in amdgpu_ring_mux_get_rptr()
317 struct amdgpu_device *adev = ring->adev; in amdgpu_sw_ring_get_rptr_gfx()
318 struct amdgpu_ring_mux *mux = &adev->gfx.muxer; in amdgpu_sw_ring_get_rptr_gfx() local
320 WARN_ON(!ring->is_sw_ring); in amdgpu_sw_ring_get_rptr_gfx()
321 return amdgpu_ring_mux_get_rptr(mux, ring); in amdgpu_sw_ring_get_rptr_gfx()
326 struct amdgpu_device *adev = ring->adev; in amdgpu_sw_ring_get_wptr_gfx()
327 struct amdgpu_ring_mux *mux = &adev->gfx.muxer; in amdgpu_sw_ring_get_wptr_gfx() local
329 WARN_ON(!ring->is_sw_ring); in amdgpu_sw_ring_get_wptr_gfx()
330 return amdgpu_ring_mux_get_wptr(mux, ring); in amdgpu_sw_ring_get_wptr_gfx()
335 struct amdgpu_device *adev = ring->adev; in amdgpu_sw_ring_set_wptr_gfx()
336 struct amdgpu_ring_mux *mux = &adev->gfx.muxer; in amdgpu_sw_ring_set_wptr_gfx() local
338 WARN_ON(!ring->is_sw_ring); in amdgpu_sw_ring_set_wptr_gfx()
339 amdgpu_ring_mux_set_wptr(mux, ring, ring->wptr); in amdgpu_sw_ring_set_wptr_gfx()
345 WARN_ON(!ring->is_sw_ring); in amdgpu_sw_ring_insert_nop()
361 static int amdgpu_mcbp_scan(struct amdgpu_ring_mux *mux) in amdgpu_mcbp_scan() argument
367 for (i = 0; i < mux->num_ring_entries; i++) { in amdgpu_mcbp_scan()
368 ring = mux->ring_entry[i].ring; in amdgpu_mcbp_scan()
369 if (ring->hw_prio > AMDGPU_RING_PRIO_DEFAULT && in amdgpu_mcbp_scan()
372 if (ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT && in amdgpu_mcbp_scan()
377 return need_preempt && !mux->s_resubmit; in amdgpu_mcbp_scan()
380 /* Trigger Mid-Command Buffer Preemption (MCBP) and find if we need to resubmit. */
381 static int amdgpu_mcbp_trigger_preempt(struct amdgpu_ring_mux *mux) in amdgpu_mcbp_trigger_preempt() argument
385 spin_lock(&mux->lock); in amdgpu_mcbp_trigger_preempt()
386 mux->pending_trailing_fence_signaled = true; in amdgpu_mcbp_trigger_preempt()
387 r = amdgpu_ring_preempt_ib(mux->real_ring); in amdgpu_mcbp_trigger_preempt()
388 spin_unlock(&mux->lock); in amdgpu_mcbp_trigger_preempt()
394 struct amdgpu_device *adev = ring->adev; in amdgpu_sw_ring_ib_begin()
395 struct amdgpu_ring_mux *mux = &adev->gfx.muxer; in amdgpu_sw_ring_ib_begin() local
397 WARN_ON(!ring->is_sw_ring); in amdgpu_sw_ring_ib_begin()
398 if (adev->gfx.mcbp && ring->hw_prio > AMDGPU_RING_PRIO_DEFAULT) { in amdgpu_sw_ring_ib_begin()
399 if (amdgpu_mcbp_scan(mux) > 0) in amdgpu_sw_ring_ib_begin()
400 amdgpu_mcbp_trigger_preempt(mux); in amdgpu_sw_ring_ib_begin()
404 amdgpu_ring_mux_start_ib(mux, ring); in amdgpu_sw_ring_ib_begin()
409 struct amdgpu_device *adev = ring->adev; in amdgpu_sw_ring_ib_end()
410 struct amdgpu_ring_mux *mux = &adev->gfx.muxer; in amdgpu_sw_ring_ib_end() local
412 WARN_ON(!ring->is_sw_ring); in amdgpu_sw_ring_ib_end()
413 if (adev->gfx.mcbp && ring->hw_prio > AMDGPU_RING_PRIO_DEFAULT) in amdgpu_sw_ring_ib_end()
415 amdgpu_ring_mux_end_ib(mux, ring); in amdgpu_sw_ring_ib_end()
420 struct amdgpu_device *adev = ring->adev; in amdgpu_sw_ring_ib_mark_offset()
421 struct amdgpu_ring_mux *mux = &adev->gfx.muxer; in amdgpu_sw_ring_ib_mark_offset() local
424 if (ring->hw_prio > AMDGPU_RING_PRIO_DEFAULT) in amdgpu_sw_ring_ib_mark_offset()
427 offset = ring->wptr & ring->buf_mask; in amdgpu_sw_ring_ib_mark_offset()
429 amdgpu_ring_mux_ib_mark_offset(mux, ring, offset, type); in amdgpu_sw_ring_ib_mark_offset()
432 void amdgpu_ring_mux_start_ib(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring) in amdgpu_ring_mux_start_ib() argument
437 spin_lock(&mux->lock); in amdgpu_ring_mux_start_ib()
438 amdgpu_mux_resubmit_chunks(mux); in amdgpu_ring_mux_start_ib()
439 spin_unlock(&mux->lock); in amdgpu_ring_mux_start_ib()
441 e = amdgpu_ring_mux_sw_entry(mux, ring); in amdgpu_ring_mux_start_ib()
453 chunk->start = ring->wptr; in amdgpu_ring_mux_start_ib()
455 chunk->cntl_offset = ring->buf_mask + 1; in amdgpu_ring_mux_start_ib()
456 chunk->de_offset = ring->buf_mask + 1; in amdgpu_ring_mux_start_ib()
457 chunk->ce_offset = ring->buf_mask + 1; in amdgpu_ring_mux_start_ib()
458 list_add_tail(&chunk->entry, &e->list); in amdgpu_ring_mux_start_ib()
461 static void scan_and_remove_signaled_chunk(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring) in scan_and_remove_signaled_chunk() argument
467 e = amdgpu_ring_mux_sw_entry(mux, ring); in scan_and_remove_signaled_chunk()
473 last_seq = atomic_read(&ring->fence_drv.last_seq); in scan_and_remove_signaled_chunk()
475 list_for_each_entry_safe(chunk, tmp, &e->list, entry) { in scan_and_remove_signaled_chunk()
476 if (chunk->sync_seq <= last_seq) { in scan_and_remove_signaled_chunk()
477 list_del(&chunk->entry); in scan_and_remove_signaled_chunk()
483 void amdgpu_ring_mux_ib_mark_offset(struct amdgpu_ring_mux *mux, in amdgpu_ring_mux_ib_mark_offset() argument
490 e = amdgpu_ring_mux_sw_entry(mux, ring); in amdgpu_ring_mux_ib_mark_offset()
496 chunk = list_last_entry(&e->list, struct amdgpu_mux_chunk, entry); in amdgpu_ring_mux_ib_mark_offset()
504 chunk->cntl_offset = offset; in amdgpu_ring_mux_ib_mark_offset()
507 chunk->de_offset = offset; in amdgpu_ring_mux_ib_mark_offset()
510 chunk->ce_offset = offset; in amdgpu_ring_mux_ib_mark_offset()
518 void amdgpu_ring_mux_end_ib(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring) in amdgpu_ring_mux_end_ib() argument
523 e = amdgpu_ring_mux_sw_entry(mux, ring); in amdgpu_ring_mux_end_ib()
529 chunk = list_last_entry(&e->list, struct amdgpu_mux_chunk, entry); in amdgpu_ring_mux_end_ib()
535 chunk->end = ring->wptr; in amdgpu_ring_mux_end_ib()
536 chunk->sync_seq = READ_ONCE(ring->fence_drv.sync_seq); in amdgpu_ring_mux_end_ib()
538 scan_and_remove_signaled_chunk(mux, ring); in amdgpu_ring_mux_end_ib()
541 bool amdgpu_mcbp_handle_trailing_fence_irq(struct amdgpu_ring_mux *mux) in amdgpu_mcbp_handle_trailing_fence_irq() argument
547 if (!mux->pending_trailing_fence_signaled) in amdgpu_mcbp_handle_trailing_fence_irq()
550 if (mux->real_ring->trail_seq != le32_to_cpu(*mux->real_ring->trail_fence_cpu_addr)) in amdgpu_mcbp_handle_trailing_fence_irq()
553 for (i = 0; i < mux->num_ring_entries; i++) { in amdgpu_mcbp_handle_trailing_fence_irq()
554 e = &mux->ring_entry[i]; in amdgpu_mcbp_handle_trailing_fence_irq()
555 if (e->ring->hw_prio <= AMDGPU_RING_PRIO_DEFAULT) { in amdgpu_mcbp_handle_trailing_fence_irq()
556 ring = e->ring; in amdgpu_mcbp_handle_trailing_fence_irq()
568 mux->s_resubmit = true; in amdgpu_mcbp_handle_trailing_fence_irq()
569 mux->seqno_to_resubmit = ring->fence_drv.sync_seq; in amdgpu_mcbp_handle_trailing_fence_irq()
570 amdgpu_ring_mux_schedule_resubmit(mux); in amdgpu_mcbp_handle_trailing_fence_irq()
573 mux->pending_trailing_fence_signaled = false; in amdgpu_mcbp_handle_trailing_fence_irq()