Lines Matching +full:generic +full:- +full:xhci
1 // SPDX-License-Identifier: GPL-2.0
3 * xHCI host controller driver
26 * until you reach a non-link TRB.
59 #include <linux/dma-mapping.h>
60 #include "xhci.h"
61 #include "xhci-trace.h"
63 static int queue_command(struct xhci_hcd *xhci, struct xhci_command *cmd,
76 if (!seg || !trb || trb < seg->trbs) in xhci_trb_virt_to_dma()
79 segment_offset = trb - seg->trbs; in xhci_trb_virt_to_dma()
82 return seg->dma + (segment_offset * sizeof(*trb)); in xhci_trb_virt_to_dma()
87 return TRB_TYPE_NOOP_LE32(trb->generic.field[3]); in trb_is_noop()
92 return TRB_TYPE_LINK_LE32(trb->link.control); in trb_is_link()
97 return trb == &seg->trbs[TRBS_PER_SEGMENT - 1]; in last_trb_on_seg()
103 return last_trb_on_seg(seg, trb) && (seg->next == ring->first_seg); in last_trb_on_ring()
108 return le32_to_cpu(trb->link.control) & LINK_TOGGLE; in link_trb_toggles_cycle()
113 struct urb_priv *urb_priv = td->urb->hcpriv; in last_td_in_urb()
115 return urb_priv->num_tds_done == urb_priv->num_tds; in last_td_in_urb()
120 return ((le32_to_cpu(ring->dequeue->event_cmd.flags) & TRB_CYCLE) == in unhandled_event_trb()
121 ring->cycle_state); in unhandled_event_trb()
126 struct urb_priv *urb_priv = urb->hcpriv; in inc_td_cnt()
128 urb_priv->num_tds_done++; in inc_td_cnt()
135 trb->link.control &= cpu_to_le32(~TRB_CHAIN); in trb_to_noop()
137 trb->generic.field[0] = 0; in trb_to_noop()
138 trb->generic.field[1] = 0; in trb_to_noop()
139 trb->generic.field[2] = 0; in trb_to_noop()
141 trb->generic.field[3] &= cpu_to_le32(TRB_CYCLE); in trb_to_noop()
142 trb->generic.field[3] |= cpu_to_le32(TRB_TYPE(noop_type)); in trb_to_noop()
154 *seg = (*seg)->next; in next_trb()
155 *trb = ((*seg)->trbs); in next_trb()
164 void inc_deq(struct xhci_hcd *xhci, struct xhci_ring *ring) in inc_deq() argument
169 if (ring->type == TYPE_EVENT) { in inc_deq()
170 if (!last_trb_on_seg(ring->deq_seg, ring->dequeue)) { in inc_deq()
171 ring->dequeue++; in inc_deq()
174 if (last_trb_on_ring(ring, ring->deq_seg, ring->dequeue)) in inc_deq()
175 ring->cycle_state ^= 1; in inc_deq()
176 ring->deq_seg = ring->deq_seg->next; in inc_deq()
177 ring->dequeue = ring->deq_seg->trbs; in inc_deq()
185 if (!trb_is_link(ring->dequeue)) { in inc_deq()
186 if (last_trb_on_seg(ring->deq_seg, ring->dequeue)) in inc_deq()
187 xhci_warn(xhci, "Missing link TRB at end of segment\n"); in inc_deq()
189 ring->dequeue++; in inc_deq()
192 while (trb_is_link(ring->dequeue)) { in inc_deq()
193 ring->deq_seg = ring->deq_seg->next; in inc_deq()
194 ring->dequeue = ring->deq_seg->trbs; in inc_deq()
198 if (link_trb_count++ > ring->num_segs) { in inc_deq()
199 xhci_warn(xhci, "Ring is an endless link TRB loop\n"); in inc_deq()
210 static void inc_enq_past_link(struct xhci_hcd *xhci, struct xhci_ring *ring, u32 chain) in inc_enq_past_link() argument
214 while (trb_is_link(ring->enqueue)) { in inc_enq_past_link()
220 * xHCI hardware can't handle the chain bit being cleared on a link TRB. in inc_enq_past_link()
225 if (!xhci_link_chain_quirk(xhci, ring->type)) { in inc_enq_past_link()
226 ring->enqueue->link.control &= cpu_to_le32(~TRB_CHAIN); in inc_enq_past_link()
227 ring->enqueue->link.control |= cpu_to_le32(chain); in inc_enq_past_link()
232 ring->enqueue->link.control ^= cpu_to_le32(TRB_CYCLE); in inc_enq_past_link()
235 if (link_trb_toggles_cycle(ring->enqueue)) in inc_enq_past_link()
236 ring->cycle_state ^= 1; in inc_enq_past_link()
238 ring->enq_seg = ring->enq_seg->next; in inc_enq_past_link()
239 ring->enqueue = ring->enq_seg->trbs; in inc_enq_past_link()
243 if (link_trb_count++ > ring->num_segs) { in inc_enq_past_link()
244 xhci_warn(xhci, "Link TRB loop at enqueue\n"); in inc_enq_past_link()
261 static void inc_enq(struct xhci_hcd *xhci, struct xhci_ring *ring, in inc_enq() argument
266 chain = le32_to_cpu(ring->enqueue->generic.field[3]) & TRB_CHAIN; in inc_enq()
268 if (last_trb_on_seg(ring->enq_seg, ring->enqueue)) { in inc_enq()
269 xhci_err(xhci, "Tried to move enqueue past ring segment\n"); in inc_enq()
273 ring->enqueue++; in inc_enq()
281 if (trb_is_link(ring->enqueue) && (chain || more_trbs_coming)) in inc_enq()
282 inc_enq_past_link(xhci, ring, chain); in inc_enq()
296 start_dma = xhci_trb_virt_to_dma(td->start_seg, td->start_trb); in trb_in_td()
297 cur_seg = td->start_seg; in trb_in_td()
304 &cur_seg->trbs[TRBS_PER_SEGMENT - 1]); in trb_in_td()
306 end_trb_dma = xhci_trb_virt_to_dma(cur_seg, td->end_trb); in trb_in_td()
319 (suspect_dma >= cur_seg->dma && in trb_in_td()
329 cur_seg = cur_seg->next; in trb_in_td()
330 start_dma = xhci_trb_virt_to_dma(cur_seg, &cur_seg->trbs[0]); in trb_in_td()
331 } while (cur_seg != td->start_seg); in trb_in_td()
344 struct xhci_segment *enq_seg = ring->enq_seg; in xhci_num_trbs_free()
345 union xhci_trb *enq = ring->enqueue; in xhci_num_trbs_free()
352 enq_seg = enq_seg->next; in xhci_num_trbs_free()
353 enq = enq_seg->trbs; in xhci_num_trbs_free()
357 if (enq == ring->dequeue) in xhci_num_trbs_free()
358 return ring->num_segs * (TRBS_PER_SEGMENT - 1); in xhci_num_trbs_free()
361 if (ring->deq_seg == enq_seg && ring->dequeue >= enq) in xhci_num_trbs_free()
362 return free + (ring->dequeue - enq); in xhci_num_trbs_free()
363 last_on_seg = &enq_seg->trbs[TRBS_PER_SEGMENT - 1]; in xhci_num_trbs_free()
364 free += last_on_seg - enq; in xhci_num_trbs_free()
365 enq_seg = enq_seg->next; in xhci_num_trbs_free()
366 enq = enq_seg->trbs; in xhci_num_trbs_free()
367 } while (i++ < ring->num_segs); in xhci_num_trbs_free()
378 static unsigned int xhci_ring_expansion_needed(struct xhci_hcd *xhci, struct xhci_ring *ring, in xhci_ring_expansion_needed() argument
386 enq_used = ring->enqueue - ring->enq_seg->trbs; in xhci_ring_expansion_needed()
389 trbs_past_seg = enq_used + num_trbs - (TRBS_PER_SEGMENT - 1); in xhci_ring_expansion_needed()
401 if (trb_is_link(ring->enqueue) && ring->enq_seg->next->trbs == ring->dequeue) in xhci_ring_expansion_needed()
404 new_segs = 1 + (trbs_past_seg / (TRBS_PER_SEGMENT - 1)); in xhci_ring_expansion_needed()
405 seg = ring->enq_seg; in xhci_ring_expansion_needed()
408 seg = seg->next; in xhci_ring_expansion_needed()
409 if (seg == ring->deq_seg) { in xhci_ring_expansion_needed()
410 xhci_dbg(xhci, "Adding %d trbs requires expanding ring by %d segments\n", in xhci_ring_expansion_needed()
414 new_segs--; in xhci_ring_expansion_needed()
421 void xhci_ring_cmd_db(struct xhci_hcd *xhci) in xhci_ring_cmd_db() argument
423 if (!(xhci->cmd_ring_state & CMD_RING_STATE_RUNNING)) in xhci_ring_cmd_db()
426 xhci_dbg(xhci, "// Ding dong!\n"); in xhci_ring_cmd_db()
430 writel(DB_VALUE_HOST, &xhci->dba->doorbell[0]); in xhci_ring_cmd_db()
432 readl(&xhci->dba->doorbell[0]); in xhci_ring_cmd_db()
435 static bool xhci_mod_cmd_timer(struct xhci_hcd *xhci) in xhci_mod_cmd_timer() argument
437 return mod_delayed_work(system_wq, &xhci->cmd_timer, in xhci_mod_cmd_timer()
438 msecs_to_jiffies(xhci->current_cmd->timeout_ms)); in xhci_mod_cmd_timer()
441 static struct xhci_command *xhci_next_queued_cmd(struct xhci_hcd *xhci) in xhci_next_queued_cmd() argument
443 return list_first_entry_or_null(&xhci->cmd_list, struct xhci_command, in xhci_next_queued_cmd()
448 * Turn all commands on command ring with status set to "aborted" to no-op trbs.
450 * This must be called with command ring stopped and xhci->lock held.
452 static void xhci_handle_stopped_cmd_ring(struct xhci_hcd *xhci, in xhci_handle_stopped_cmd_ring() argument
457 /* Turn all aborted commands in list to no-ops, then restart */ in xhci_handle_stopped_cmd_ring()
458 list_for_each_entry(i_cmd, &xhci->cmd_list, cmd_list) { in xhci_handle_stopped_cmd_ring()
460 if (i_cmd->status != COMP_COMMAND_ABORTED) in xhci_handle_stopped_cmd_ring()
463 i_cmd->status = COMP_COMMAND_RING_STOPPED; in xhci_handle_stopped_cmd_ring()
465 xhci_dbg(xhci, "Turn aborted command %p to no-op\n", in xhci_handle_stopped_cmd_ring()
466 i_cmd->command_trb); in xhci_handle_stopped_cmd_ring()
468 trb_to_noop(i_cmd->command_trb, TRB_CMD_NOOP); in xhci_handle_stopped_cmd_ring()
472 * completion event is received for these no-op commands in xhci_handle_stopped_cmd_ring()
476 xhci->cmd_ring_state = CMD_RING_STATE_RUNNING; in xhci_handle_stopped_cmd_ring()
479 if ((xhci->cmd_ring->dequeue != xhci->cmd_ring->enqueue) && in xhci_handle_stopped_cmd_ring()
480 !(xhci->xhc_state & XHCI_STATE_DYING)) { in xhci_handle_stopped_cmd_ring()
481 xhci->current_cmd = cur_cmd; in xhci_handle_stopped_cmd_ring()
483 xhci_mod_cmd_timer(xhci); in xhci_handle_stopped_cmd_ring()
484 xhci_ring_cmd_db(xhci); in xhci_handle_stopped_cmd_ring()
488 /* Must be called with xhci->lock held, releases and acquires lock back */
489 static int xhci_abort_cmd_ring(struct xhci_hcd *xhci, unsigned long flags) in xhci_abort_cmd_ring() argument
491 struct xhci_segment *new_seg = xhci->cmd_ring->deq_seg; in xhci_abort_cmd_ring()
492 union xhci_trb *new_deq = xhci->cmd_ring->dequeue; in xhci_abort_cmd_ring()
496 xhci_dbg(xhci, "Abort command ring\n"); in xhci_abort_cmd_ring()
498 reinit_completion(&xhci->cmd_ring_stop_completion); in xhci_abort_cmd_ring()
513 xhci_write_64(xhci, crcr | CMD_RING_ABORT, &xhci->op_regs->cmd_ring); in xhci_abort_cmd_ring()
515 /* Section 4.6.1.2 of xHCI 1.0 spec says software should also time the in xhci_abort_cmd_ring()
517 * seconds then driver handles it as if host died (-ENODEV). in xhci_abort_cmd_ring()
518 * In the future we should distinguish between -ENODEV and -ETIMEDOUT in xhci_abort_cmd_ring()
519 * and try to recover a -ETIMEDOUT with a host controller reset. in xhci_abort_cmd_ring()
521 ret = xhci_handshake_check_state(xhci, &xhci->op_regs->cmd_ring, in xhci_abort_cmd_ring()
525 xhci_err(xhci, "Abort failed to stop command ring: %d\n", ret); in xhci_abort_cmd_ring()
526 xhci_halt(xhci); in xhci_abort_cmd_ring()
527 xhci_hc_died(xhci); in xhci_abort_cmd_ring()
536 spin_unlock_irqrestore(&xhci->lock, flags); in xhci_abort_cmd_ring()
537 ret = wait_for_completion_timeout(&xhci->cmd_ring_stop_completion, in xhci_abort_cmd_ring()
539 spin_lock_irqsave(&xhci->lock, flags); in xhci_abort_cmd_ring()
541 xhci_dbg(xhci, "No stop event for abort, ring start fail?\n"); in xhci_abort_cmd_ring()
542 xhci_cleanup_command_queue(xhci); in xhci_abort_cmd_ring()
544 xhci_handle_stopped_cmd_ring(xhci, xhci_next_queued_cmd(xhci)); in xhci_abort_cmd_ring()
549 void xhci_ring_ep_doorbell(struct xhci_hcd *xhci, in xhci_ring_ep_doorbell() argument
554 __le32 __iomem *db_addr = &xhci->dba->doorbell[slot_id]; in xhci_ring_ep_doorbell()
555 struct xhci_virt_ep *ep = &xhci->devs[slot_id]->eps[ep_index]; in xhci_ring_ep_doorbell()
556 unsigned int ep_state = ep->ep_state; in xhci_ring_ep_doorbell()
576 static void ring_doorbell_for_active_rings(struct xhci_hcd *xhci, in ring_doorbell_for_active_rings() argument
583 ep = &xhci->devs[slot_id]->eps[ep_index]; in ring_doorbell_for_active_rings()
586 if (!(ep->ep_state & EP_HAS_STREAMS)) { in ring_doorbell_for_active_rings()
587 if (ep->ring && !(list_empty(&ep->ring->td_list))) in ring_doorbell_for_active_rings()
588 xhci_ring_ep_doorbell(xhci, slot_id, ep_index, 0); in ring_doorbell_for_active_rings()
592 for (stream_id = 1; stream_id < ep->stream_info->num_streams; in ring_doorbell_for_active_rings()
594 struct xhci_stream_info *stream_info = ep->stream_info; in ring_doorbell_for_active_rings()
595 if (!list_empty(&stream_info->stream_rings[stream_id]->td_list)) in ring_doorbell_for_active_rings()
596 xhci_ring_ep_doorbell(xhci, slot_id, ep_index, in ring_doorbell_for_active_rings()
601 void xhci_ring_doorbell_for_active_rings(struct xhci_hcd *xhci, in xhci_ring_doorbell_for_active_rings() argument
605 ring_doorbell_for_active_rings(xhci, slot_id, ep_index); in xhci_ring_doorbell_for_active_rings()
608 static struct xhci_virt_ep *xhci_get_virt_ep(struct xhci_hcd *xhci, in xhci_get_virt_ep() argument
613 xhci_warn(xhci, "Invalid slot_id %u\n", slot_id); in xhci_get_virt_ep()
617 xhci_warn(xhci, "Invalid endpoint index %u\n", ep_index); in xhci_get_virt_ep()
620 if (!xhci->devs[slot_id]) { in xhci_get_virt_ep()
621 xhci_warn(xhci, "No xhci virt device for slot_id %u\n", slot_id); in xhci_get_virt_ep()
625 return &xhci->devs[slot_id]->eps[ep_index]; in xhci_get_virt_ep()
628 static struct xhci_ring *xhci_virt_ep_to_ring(struct xhci_hcd *xhci, in xhci_virt_ep_to_ring() argument
633 if (!(ep->ep_state & EP_HAS_STREAMS)) in xhci_virt_ep_to_ring()
634 return ep->ring; in xhci_virt_ep_to_ring()
636 if (!ep->stream_info) in xhci_virt_ep_to_ring()
639 if (stream_id == 0 || stream_id >= ep->stream_info->num_streams) { in xhci_virt_ep_to_ring()
640 xhci_warn(xhci, "Invalid stream_id %u request for slot_id %u ep_index %u\n", in xhci_virt_ep_to_ring()
641 stream_id, ep->vdev->slot_id, ep->ep_index); in xhci_virt_ep_to_ring()
645 return ep->stream_info->stream_rings[stream_id]; in xhci_virt_ep_to_ring()
652 struct xhci_ring *xhci_triad_to_transfer_ring(struct xhci_hcd *xhci, in xhci_triad_to_transfer_ring() argument
658 ep = xhci_get_virt_ep(xhci, slot_id, ep_index); in xhci_triad_to_transfer_ring()
662 return xhci_virt_ep_to_ring(xhci, ep, stream_id); in xhci_triad_to_transfer_ring()
672 static u64 xhci_get_hw_deq(struct xhci_hcd *xhci, struct xhci_virt_device *vdev, in xhci_get_hw_deq() argument
679 ep = &vdev->eps[ep_index]; in xhci_get_hw_deq()
681 if (ep->ep_state & EP_HAS_STREAMS) { in xhci_get_hw_deq()
682 st_ctx = &ep->stream_info->stream_ctx_array[stream_id]; in xhci_get_hw_deq()
683 return le64_to_cpu(st_ctx->stream_ring); in xhci_get_hw_deq()
685 ep_ctx = xhci_get_ep_ctx(xhci, vdev->out_ctx, ep_index); in xhci_get_hw_deq()
686 return le64_to_cpu(ep_ctx->deq); in xhci_get_hw_deq()
689 static int xhci_move_dequeue_past_td(struct xhci_hcd *xhci, in xhci_move_dequeue_past_td() argument
693 struct xhci_virt_device *dev = xhci->devs[slot_id]; in xhci_move_dequeue_past_td()
694 struct xhci_virt_ep *ep = &dev->eps[ep_index]; in xhci_move_dequeue_past_td()
707 ep_ring = xhci_triad_to_transfer_ring(xhci, slot_id, in xhci_move_dequeue_past_td()
710 xhci_warn(xhci, "WARN can't find new dequeue, invalid stream ID %u\n", in xhci_move_dequeue_past_td()
712 return -ENODEV; in xhci_move_dequeue_past_td()
715 hw_dequeue = xhci_get_hw_deq(xhci, dev, ep_index, stream_id); in xhci_move_dequeue_past_td()
716 new_seg = ep_ring->deq_seg; in xhci_move_dequeue_past_td()
717 new_deq = ep_ring->dequeue; in xhci_move_dequeue_past_td()
718 new_cycle = le32_to_cpu(td->end_trb->generic.field[3]) & TRB_CYCLE; in xhci_move_dequeue_past_td()
732 if (new_deq == td->end_trb) in xhci_move_dequeue_past_td()
742 if (new_deq == ep->ring->dequeue) { in xhci_move_dequeue_past_td()
743 xhci_err(xhci, "Error: Failed finding new dequeue state\n"); in xhci_move_dequeue_past_td()
744 return -EINVAL; in xhci_move_dequeue_past_td()
752 xhci_warn(xhci, "Can't find dma of new dequeue ptr\n"); in xhci_move_dequeue_past_td()
753 xhci_warn(xhci, "deq seg = %p, deq ptr = %p\n", new_seg, new_deq); in xhci_move_dequeue_past_td()
754 return -EINVAL; in xhci_move_dequeue_past_td()
757 if ((ep->ep_state & SET_DEQ_PENDING)) { in xhci_move_dequeue_past_td()
758 xhci_warn(xhci, "Set TR Deq already pending, don't submit for 0x%pad\n", in xhci_move_dequeue_past_td()
760 return -EBUSY; in xhci_move_dequeue_past_td()
764 cmd = xhci_alloc_command(xhci, false, GFP_ATOMIC); in xhci_move_dequeue_past_td()
766 xhci_warn(xhci, "Can't alloc Set TR Deq cmd 0x%pad\n", &addr); in xhci_move_dequeue_past_td()
767 return -ENOMEM; in xhci_move_dequeue_past_td()
772 ret = queue_command(xhci, cmd, in xhci_move_dequeue_past_td()
778 xhci_free_command(xhci, cmd); in xhci_move_dequeue_past_td()
781 ep->queued_deq_seg = new_seg; in xhci_move_dequeue_past_td()
782 ep->queued_deq_ptr = new_deq; in xhci_move_dequeue_past_td()
784 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_move_dequeue_past_td()
792 ep->ep_state |= SET_DEQ_PENDING; in xhci_move_dequeue_past_td()
793 xhci_ring_cmd_db(xhci); in xhci_move_dequeue_past_td()
803 struct xhci_segment *seg = td->start_seg; in td_to_noop()
804 union xhci_trb *trb = td->start_trb; in td_to_noop()
810 if (flip_cycle && trb != td->start_trb && trb != td->end_trb) in td_to_noop()
811 trb->generic.field[3] ^= cpu_to_le32(TRB_CYCLE); in td_to_noop()
813 if (trb == td->end_trb) in td_to_noop()
820 static void xhci_giveback_urb_in_irq(struct xhci_hcd *xhci, in xhci_giveback_urb_in_irq() argument
823 struct urb *urb = cur_td->urb; in xhci_giveback_urb_in_irq()
824 struct urb_priv *urb_priv = urb->hcpriv; in xhci_giveback_urb_in_irq()
825 struct usb_hcd *hcd = bus_to_hcd(urb->dev->bus); in xhci_giveback_urb_in_irq()
827 if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) { in xhci_giveback_urb_in_irq()
828 xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs--; in xhci_giveback_urb_in_irq()
829 if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs == 0) { in xhci_giveback_urb_in_irq()
830 if (xhci->quirks & XHCI_AMD_PLL_FIX) in xhci_giveback_urb_in_irq()
840 static void xhci_unmap_td_bounce_buffer(struct xhci_hcd *xhci, in xhci_unmap_td_bounce_buffer() argument
843 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_unmap_td_bounce_buffer()
844 struct xhci_segment *seg = td->bounce_seg; in xhci_unmap_td_bounce_buffer()
845 struct urb *urb = td->urb; in xhci_unmap_td_bounce_buffer()
852 dma_unmap_single(dev, seg->bounce_dma, ring->bounce_buf_len, in xhci_unmap_td_bounce_buffer()
857 dma_unmap_single(dev, seg->bounce_dma, ring->bounce_buf_len, in xhci_unmap_td_bounce_buffer()
860 if (urb->num_sgs) { in xhci_unmap_td_bounce_buffer()
861 len = sg_pcopy_from_buffer(urb->sg, urb->num_sgs, seg->bounce_buf, in xhci_unmap_td_bounce_buffer()
862 seg->bounce_len, seg->bounce_offs); in xhci_unmap_td_bounce_buffer()
863 if (len != seg->bounce_len) in xhci_unmap_td_bounce_buffer()
864 xhci_warn(xhci, "WARN Wrong bounce buffer read length: %zu != %d\n", in xhci_unmap_td_bounce_buffer()
865 len, seg->bounce_len); in xhci_unmap_td_bounce_buffer()
867 memcpy(urb->transfer_buffer + seg->bounce_offs, seg->bounce_buf, in xhci_unmap_td_bounce_buffer()
868 seg->bounce_len); in xhci_unmap_td_bounce_buffer()
870 seg->bounce_len = 0; in xhci_unmap_td_bounce_buffer()
871 seg->bounce_offs = 0; in xhci_unmap_td_bounce_buffer()
874 static void xhci_td_cleanup(struct xhci_hcd *xhci, struct xhci_td *td, in xhci_td_cleanup() argument
880 urb = td->urb; in xhci_td_cleanup()
883 xhci_unmap_td_bounce_buffer(xhci, ep_ring, td); in xhci_td_cleanup()
887 * length, urb->actual_length will be a very big number (since it's in xhci_td_cleanup()
890 if (urb->actual_length > urb->transfer_buffer_length) { in xhci_td_cleanup()
891 xhci_warn(xhci, "URB req %u and actual %u transfer length mismatch\n", in xhci_td_cleanup()
892 urb->transfer_buffer_length, urb->actual_length); in xhci_td_cleanup()
893 urb->actual_length = 0; in xhci_td_cleanup()
897 if (!list_empty(&td->td_list)) in xhci_td_cleanup()
898 list_del_init(&td->td_list); in xhci_td_cleanup()
900 if (!list_empty(&td->cancelled_td_list)) in xhci_td_cleanup()
901 list_del_init(&td->cancelled_td_list); in xhci_td_cleanup()
906 if ((urb->actual_length != urb->transfer_buffer_length && in xhci_td_cleanup()
907 (urb->transfer_flags & URB_SHORT_NOT_OK)) || in xhci_td_cleanup()
908 (status != 0 && !usb_endpoint_xfer_isoc(&urb->ep->desc))) in xhci_td_cleanup()
909 xhci_dbg(xhci, "Giveback URB %p, len = %d, expected = %d, status = %d\n", in xhci_td_cleanup()
910 urb, urb->actual_length, in xhci_td_cleanup()
911 urb->transfer_buffer_length, status); in xhci_td_cleanup()
914 if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) in xhci_td_cleanup()
916 xhci_giveback_urb_in_irq(xhci, td, status); in xhci_td_cleanup()
921 static void xhci_dequeue_td(struct xhci_hcd *xhci, struct xhci_td *td, struct xhci_ring *ring, in xhci_dequeue_td() argument
924 ring->dequeue = td->end_trb; in xhci_dequeue_td()
925 ring->deq_seg = td->end_seg; in xhci_dequeue_td()
926 inc_deq(xhci, ring); in xhci_dequeue_td()
928 xhci_td_cleanup(xhci, td, ring, status); in xhci_dequeue_td()
937 list_for_each_entry_safe(td, tmp_td, &ep->cancelled_td_list, in xhci_giveback_invalidated_tds()
940 ring = xhci_urb_to_transfer_ring(ep->xhci, td->urb); in xhci_giveback_invalidated_tds()
942 if (td->cancel_status == TD_CLEARED) { in xhci_giveback_invalidated_tds()
943 xhci_dbg(ep->xhci, "%s: Giveback cancelled URB %p TD\n", in xhci_giveback_invalidated_tds()
944 __func__, td->urb); in xhci_giveback_invalidated_tds()
945 xhci_td_cleanup(ep->xhci, td, ring, td->status); in xhci_giveback_invalidated_tds()
947 xhci_dbg(ep->xhci, "%s: Keep cancelled URB %p TD as cancel_status is %d\n", in xhci_giveback_invalidated_tds()
948 __func__, td->urb, td->cancel_status); in xhci_giveback_invalidated_tds()
950 if (ep->xhci->xhc_state & XHCI_STATE_DYING) in xhci_giveback_invalidated_tds()
955 static int xhci_reset_halted_ep(struct xhci_hcd *xhci, unsigned int slot_id, in xhci_reset_halted_ep() argument
961 command = xhci_alloc_command(xhci, false, GFP_ATOMIC); in xhci_reset_halted_ep()
963 ret = -ENOMEM; in xhci_reset_halted_ep()
967 xhci_dbg(xhci, "%s-reset ep %u, slot %u\n", in xhci_reset_halted_ep()
971 ret = xhci_queue_reset_ep(xhci, command, slot_id, ep_index, reset_type); in xhci_reset_halted_ep()
974 xhci_err(xhci, "ERROR queuing reset endpoint for slot %d ep_index %d, %d\n", in xhci_reset_halted_ep()
979 static int xhci_handle_halted_endpoint(struct xhci_hcd *xhci, in xhci_handle_halted_endpoint() argument
984 unsigned int slot_id = ep->vdev->slot_id; in xhci_handle_halted_endpoint()
991 if (ep->vdev->flags & VDEV_PORT_ERROR) in xhci_handle_halted_endpoint()
992 return -ENODEV; in xhci_handle_halted_endpoint()
996 ep->ep_state |= EP_HARD_CLEAR_TOGGLE; in xhci_handle_halted_endpoint()
997 if (td && list_empty(&td->cancelled_td_list)) { in xhci_handle_halted_endpoint()
998 list_add_tail(&td->cancelled_td_list, &ep->cancelled_td_list); in xhci_handle_halted_endpoint()
999 td->cancel_status = TD_HALTED; in xhci_handle_halted_endpoint()
1003 if (ep->ep_state & EP_HALTED) { in xhci_handle_halted_endpoint()
1004 xhci_dbg(xhci, "Reset ep command for ep_index %d already pending\n", in xhci_handle_halted_endpoint()
1005 ep->ep_index); in xhci_handle_halted_endpoint()
1009 err = xhci_reset_halted_ep(xhci, slot_id, ep->ep_index, reset_type); in xhci_handle_halted_endpoint()
1013 ep->ep_state |= EP_HALTED; in xhci_handle_halted_endpoint()
1015 xhci_ring_cmd_db(xhci); in xhci_handle_halted_endpoint()
1022 * We have the xHCI lock, so nothing can modify this list until we drop it.
1023 * We're also in the event handler, so we can't get re-interrupted if another
1031 struct xhci_hcd *xhci; in xhci_invalidate_cancelled_tds() local
1037 unsigned int slot_id = ep->vdev->slot_id; in xhci_invalidate_cancelled_tds()
1044 if (ep->ep_state & SET_DEQ_PENDING) in xhci_invalidate_cancelled_tds()
1047 xhci = ep->xhci; in xhci_invalidate_cancelled_tds()
1049 list_for_each_entry_safe(td, tmp_td, &ep->cancelled_td_list, cancelled_td_list) { in xhci_invalidate_cancelled_tds()
1050 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_invalidate_cancelled_tds()
1053 td->start_seg, td->start_trb), in xhci_invalidate_cancelled_tds()
1054 td->urb->stream_id, td->urb); in xhci_invalidate_cancelled_tds()
1055 list_del_init(&td->td_list); in xhci_invalidate_cancelled_tds()
1056 ring = xhci_urb_to_transfer_ring(xhci, td->urb); in xhci_invalidate_cancelled_tds()
1058 xhci_warn(xhci, "WARN Cancelled URB %p has invalid stream ID %u.\n", in xhci_invalidate_cancelled_tds()
1059 td->urb, td->urb->stream_id); in xhci_invalidate_cancelled_tds()
1068 hw_deq = xhci_get_hw_deq(xhci, ep->vdev, ep->ep_index, in xhci_invalidate_cancelled_tds()
1069 td->urb->stream_id); in xhci_invalidate_cancelled_tds()
1072 if (td->cancel_status == TD_HALTED || trb_in_td(td, hw_deq)) { in xhci_invalidate_cancelled_tds()
1073 switch (td->cancel_status) { in xhci_invalidate_cancelled_tds()
1074 case TD_CLEARED: /* TD is already no-op */ in xhci_invalidate_cancelled_tds()
1081 if (cached_td->urb->stream_id != td->urb->stream_id) { in xhci_invalidate_cancelled_tds()
1083 xhci_dbg(xhci, in xhci_invalidate_cancelled_tds()
1085 td->urb->stream_id, td->urb); in xhci_invalidate_cancelled_tds()
1086 td->cancel_status = TD_CLEARING_CACHE_DEFERRED; in xhci_invalidate_cancelled_tds()
1091 xhci_warn(xhci, in xhci_invalidate_cancelled_tds()
1093 td->urb, cached_td->urb, in xhci_invalidate_cancelled_tds()
1094 td->urb->stream_id); in xhci_invalidate_cancelled_tds()
1096 cached_td->cancel_status = TD_CLEARED; in xhci_invalidate_cancelled_tds()
1099 td->cancel_status = TD_CLEARING_CACHE; in xhci_invalidate_cancelled_tds()
1105 td->cancel_status = TD_CLEARED; in xhci_invalidate_cancelled_tds()
1113 err = xhci_move_dequeue_past_td(xhci, slot_id, ep->ep_index, in xhci_invalidate_cancelled_tds()
1114 cached_td->urb->stream_id, in xhci_invalidate_cancelled_tds()
1117 /* Failed to move past cached td, just set cached TDs to no-op */ in xhci_invalidate_cancelled_tds()
1118 list_for_each_entry_safe(td, tmp_td, &ep->cancelled_td_list, cancelled_td_list) { in xhci_invalidate_cancelled_tds()
1124 if (td->cancel_status != TD_CLEARING_CACHE && in xhci_invalidate_cancelled_tds()
1125 td->cancel_status != TD_CLEARING_CACHE_DEFERRED) in xhci_invalidate_cancelled_tds()
1127 xhci_warn(xhci, "Failed to clear cancelled cached URB %p, mark clear anyway\n", in xhci_invalidate_cancelled_tds()
1128 td->urb); in xhci_invalidate_cancelled_tds()
1130 td->cancel_status = TD_CLEARED; in xhci_invalidate_cancelled_tds()
1141 * Call under xhci->lock on a stopped endpoint.
1151 * Only call for non-running rings without streams.
1158 if (!list_empty(&ep->ring->td_list)) { /* Not streams compatible */ in find_halted_td()
1159 hw_deq = xhci_get_hw_deq(ep->xhci, ep->vdev, ep->ep_index, 0); in find_halted_td()
1161 td = list_first_entry(&ep->ring->td_list, struct xhci_td, td_list); in find_halted_td()
1175 * 2. Otherwise, we turn all the TRBs in the TD into No-op TRBs (with the chain
1178 static void xhci_handle_cmd_stop_ep(struct xhci_hcd *xhci, int slot_id, in xhci_handle_cmd_stop_ep() argument
1189 if (unlikely(TRB_TO_SUSPEND_PORT(le32_to_cpu(trb->generic.field[3])))) { in xhci_handle_cmd_stop_ep()
1190 if (!xhci->devs[slot_id]) in xhci_handle_cmd_stop_ep()
1191 xhci_warn(xhci, "Stop endpoint command completion for disabled slot %u\n", in xhci_handle_cmd_stop_ep()
1196 ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3])); in xhci_handle_cmd_stop_ep()
1197 ep = xhci_get_virt_ep(xhci, slot_id, ep_index); in xhci_handle_cmd_stop_ep()
1201 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep_index); in xhci_handle_cmd_stop_ep()
1214 * Proper error code is unknown here, it would be -EPIPE if device side in xhci_handle_cmd_stop_ep()
1215 * of enadpoit halted (aka STALL), and -EPROTO if not (transaction error) in xhci_handle_cmd_stop_ep()
1216 * We use -EPROTO, if device is stalled it should return a stall error on in xhci_handle_cmd_stop_ep()
1217 * next transfer, which then will return -EPIPE, and device side stall is in xhci_handle_cmd_stop_ep()
1222 xhci_dbg(xhci, "Stop ep completion raced with stall\n"); in xhci_handle_cmd_stop_ep()
1227 if (ep->ep_state & EP_HALTED) in xhci_handle_cmd_stop_ep()
1230 if (ep->ep_state & EP_HAS_STREAMS) { in xhci_handle_cmd_stop_ep()
1236 td->status = -EPROTO; in xhci_handle_cmd_stop_ep()
1239 err = xhci_handle_halted_endpoint(xhci, ep, td, reset_type); in xhci_handle_cmd_stop_ep()
1240 xhci_dbg(xhci, "Stop ep completion resetting ep, status %d\n", err); in xhci_handle_cmd_stop_ep()
1245 ep->ep_state &= ~EP_STOP_CMD_PENDING; in xhci_handle_cmd_stop_ep()
1249 * Per xHCI 4.6.9, Stop Endpoint command on a Stopped in xhci_handle_cmd_stop_ep()
1256 if (ep->ep_state & EP_HALTED) in xhci_handle_cmd_stop_ep()
1271 xhci_dbg(xhci, "Stop ep completion ctx error, ctx_state %d\n", in xhci_handle_cmd_stop_ep()
1277 if (time_is_before_jiffies(ep->stop_time + msecs_to_jiffies(100))) in xhci_handle_cmd_stop_ep()
1280 command = xhci_alloc_command(xhci, false, GFP_ATOMIC); in xhci_handle_cmd_stop_ep()
1282 ep->ep_state &= ~EP_STOP_CMD_PENDING; in xhci_handle_cmd_stop_ep()
1285 xhci_queue_stop_endpoint(xhci, command, slot_id, ep_index, 0); in xhci_handle_cmd_stop_ep()
1286 xhci_ring_cmd_db(xhci); in xhci_handle_cmd_stop_ep()
1296 ep->ep_state &= ~EP_STOP_CMD_PENDING; in xhci_handle_cmd_stop_ep()
1300 ring_doorbell_for_active_rings(xhci, slot_id, ep_index); in xhci_handle_cmd_stop_ep()
1303 static void xhci_kill_ring_urbs(struct xhci_hcd *xhci, struct xhci_ring *ring) in xhci_kill_ring_urbs() argument
1308 list_for_each_entry_safe(cur_td, tmp, &ring->td_list, td_list) { in xhci_kill_ring_urbs()
1309 list_del_init(&cur_td->td_list); in xhci_kill_ring_urbs()
1311 if (!list_empty(&cur_td->cancelled_td_list)) in xhci_kill_ring_urbs()
1312 list_del_init(&cur_td->cancelled_td_list); in xhci_kill_ring_urbs()
1314 xhci_unmap_td_bounce_buffer(xhci, ring, cur_td); in xhci_kill_ring_urbs()
1316 inc_td_cnt(cur_td->urb); in xhci_kill_ring_urbs()
1318 xhci_giveback_urb_in_irq(xhci, cur_td, -ESHUTDOWN); in xhci_kill_ring_urbs()
1322 static void xhci_kill_endpoint_urbs(struct xhci_hcd *xhci, in xhci_kill_endpoint_urbs() argument
1330 ep = xhci_get_virt_ep(xhci, slot_id, ep_index); in xhci_kill_endpoint_urbs()
1334 if ((ep->ep_state & EP_HAS_STREAMS) || in xhci_kill_endpoint_urbs()
1335 (ep->ep_state & EP_GETTING_NO_STREAMS)) { in xhci_kill_endpoint_urbs()
1338 for (stream_id = 1; stream_id < ep->stream_info->num_streams; in xhci_kill_endpoint_urbs()
1340 ring = ep->stream_info->stream_rings[stream_id]; in xhci_kill_endpoint_urbs()
1344 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_kill_endpoint_urbs()
1347 xhci_kill_ring_urbs(xhci, ring); in xhci_kill_endpoint_urbs()
1350 ring = ep->ring; in xhci_kill_endpoint_urbs()
1353 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_kill_endpoint_urbs()
1356 xhci_kill_ring_urbs(xhci, ring); in xhci_kill_endpoint_urbs()
1359 list_for_each_entry_safe(cur_td, tmp, &ep->cancelled_td_list, in xhci_kill_endpoint_urbs()
1361 list_del_init(&cur_td->cancelled_td_list); in xhci_kill_endpoint_urbs()
1362 inc_td_cnt(cur_td->urb); in xhci_kill_endpoint_urbs()
1365 xhci_giveback_urb_in_irq(xhci, cur_td, -ESHUTDOWN); in xhci_kill_endpoint_urbs()
1375 * Call with xhci->lock held.
1376 * lock is relased and re-acquired while giving back urb.
1378 void xhci_hc_died(struct xhci_hcd *xhci) in xhci_hc_died() argument
1382 if (xhci->xhc_state & XHCI_STATE_DYING) in xhci_hc_died()
1385 xhci_err(xhci, "xHCI host controller not responding, assume dead\n"); in xhci_hc_died()
1386 xhci->xhc_state |= XHCI_STATE_DYING; in xhci_hc_died()
1388 xhci_cleanup_command_queue(xhci); in xhci_hc_died()
1391 for (i = 0; i <= HCS_MAX_SLOTS(xhci->hcs_params1); i++) { in xhci_hc_died()
1392 if (!xhci->devs[i]) in xhci_hc_died()
1395 xhci_kill_endpoint_urbs(xhci, i, j); in xhci_hc_died()
1399 if (!(xhci->xhc_state & XHCI_STATE_REMOVING)) in xhci_hc_died()
1400 usb_hc_died(xhci_to_hcd(xhci)); in xhci_hc_died()
1410 static void xhci_handle_cmd_set_deq(struct xhci_hcd *xhci, int slot_id, in xhci_handle_cmd_set_deq() argument
1422 ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3])); in xhci_handle_cmd_set_deq()
1423 stream_id = TRB_TO_STREAM_ID(le32_to_cpu(trb->generic.field[2])); in xhci_handle_cmd_set_deq()
1424 ep = xhci_get_virt_ep(xhci, slot_id, ep_index); in xhci_handle_cmd_set_deq()
1428 ep_ring = xhci_virt_ep_to_ring(xhci, ep, stream_id); in xhci_handle_cmd_set_deq()
1430 xhci_warn(xhci, "WARN Set TR deq ptr command for freed stream ID %u\n", in xhci_handle_cmd_set_deq()
1436 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep_index); in xhci_handle_cmd_set_deq()
1437 slot_ctx = xhci_get_slot_ctx(xhci, ep->vdev->out_ctx); in xhci_handle_cmd_set_deq()
1441 if (ep->ep_state & EP_HAS_STREAMS) { in xhci_handle_cmd_set_deq()
1442 stream_ctx = &ep->stream_info->stream_ctx_array[stream_id]; in xhci_handle_cmd_set_deq()
1443 trace_xhci_handle_cmd_set_deq_stream(ep->stream_info, stream_id); in xhci_handle_cmd_set_deq()
1452 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd invalid because of stream ID configuration\n"); in xhci_handle_cmd_set_deq()
1455 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd failed due to incorrect slot or ep state.\n"); in xhci_handle_cmd_set_deq()
1457 slot_state = le32_to_cpu(slot_ctx->dev_state); in xhci_handle_cmd_set_deq()
1459 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_handle_cmd_set_deq()
1464 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd failed because slot %u was not enabled.\n", in xhci_handle_cmd_set_deq()
1468 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd with unknown completion code of %u.\n", in xhci_handle_cmd_set_deq()
1481 if (ep->ep_state & EP_HAS_STREAMS) { in xhci_handle_cmd_set_deq()
1482 deq = le64_to_cpu(stream_ctx->stream_ring) & SCTX_DEQ_MASK; in xhci_handle_cmd_set_deq()
1485 * Cadence xHCI controllers store some endpoint state in xhci_handle_cmd_set_deq()
1493 if (xhci->quirks & XHCI_CDNS_SCTX_QUIRK) { in xhci_handle_cmd_set_deq()
1494 stream_ctx->reserved[0] = 0; in xhci_handle_cmd_set_deq()
1495 stream_ctx->reserved[1] = 0; in xhci_handle_cmd_set_deq()
1498 deq = le64_to_cpu(ep_ctx->deq) & ~EP_CTX_CYCLE_MASK; in xhci_handle_cmd_set_deq()
1500 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_handle_cmd_set_deq()
1502 if (xhci_trb_virt_to_dma(ep->queued_deq_seg, in xhci_handle_cmd_set_deq()
1503 ep->queued_deq_ptr) == deq) { in xhci_handle_cmd_set_deq()
1507 ep_ring->deq_seg = ep->queued_deq_seg; in xhci_handle_cmd_set_deq()
1508 ep_ring->dequeue = ep->queued_deq_ptr; in xhci_handle_cmd_set_deq()
1510 xhci_warn(xhci, "Mismatch between completed Set TR Deq Ptr command & xHCI internal state.\n"); in xhci_handle_cmd_set_deq()
1511 xhci_warn(xhci, "ep deq seg = %p, deq ptr = %p\n", in xhci_handle_cmd_set_deq()
1512 ep->queued_deq_seg, ep->queued_deq_ptr); in xhci_handle_cmd_set_deq()
1516 list_for_each_entry_safe(td, tmp_td, &ep->cancelled_td_list, in xhci_handle_cmd_set_deq()
1518 ep_ring = xhci_urb_to_transfer_ring(ep->xhci, td->urb); in xhci_handle_cmd_set_deq()
1519 if (td->cancel_status == TD_CLEARING_CACHE) { in xhci_handle_cmd_set_deq()
1520 td->cancel_status = TD_CLEARED; in xhci_handle_cmd_set_deq()
1521 xhci_dbg(ep->xhci, "%s: Giveback cancelled URB %p TD\n", in xhci_handle_cmd_set_deq()
1522 __func__, td->urb); in xhci_handle_cmd_set_deq()
1523 xhci_td_cleanup(ep->xhci, td, ep_ring, td->status); in xhci_handle_cmd_set_deq()
1525 xhci_dbg(ep->xhci, "%s: Keep cancelled URB %p TD as cancel_status is %d\n", in xhci_handle_cmd_set_deq()
1526 __func__, td->urb, td->cancel_status); in xhci_handle_cmd_set_deq()
1530 ep->ep_state &= ~SET_DEQ_PENDING; in xhci_handle_cmd_set_deq()
1531 ep->queued_deq_seg = NULL; in xhci_handle_cmd_set_deq()
1532 ep->queued_deq_ptr = NULL; in xhci_handle_cmd_set_deq()
1535 if (!list_empty(&ep->cancelled_td_list)) { in xhci_handle_cmd_set_deq()
1536 xhci_dbg(ep->xhci, "%s: Pending TDs to clear, continuing with invalidation\n", in xhci_handle_cmd_set_deq()
1540 ring_doorbell_for_active_rings(xhci, slot_id, ep_index); in xhci_handle_cmd_set_deq()
1545 xhci_dbg(ep->xhci, "%s: All TDs cleared, ring doorbell\n", __func__); in xhci_handle_cmd_set_deq()
1546 ring_doorbell_for_active_rings(xhci, slot_id, ep_index); in xhci_handle_cmd_set_deq()
1550 static void xhci_handle_cmd_reset_ep(struct xhci_hcd *xhci, int slot_id, in xhci_handle_cmd_reset_ep() argument
1557 ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3])); in xhci_handle_cmd_reset_ep()
1558 ep = xhci_get_virt_ep(xhci, slot_id, ep_index); in xhci_handle_cmd_reset_ep()
1562 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep_index); in xhci_handle_cmd_reset_ep()
1568 xhci_dbg_trace(xhci, trace_xhci_dbg_reset_ep, in xhci_handle_cmd_reset_ep()
1575 ep->ep_state &= ~EP_HALTED; in xhci_handle_cmd_reset_ep()
1580 if ((le32_to_cpu(trb->generic.field[3])) & TRB_TSP) in xhci_handle_cmd_reset_ep()
1581 ring_doorbell_for_active_rings(xhci, slot_id, ep_index); in xhci_handle_cmd_reset_ep()
1588 command->slot_id = slot_id; in xhci_handle_cmd_enable_slot()
1590 command->slot_id = 0; in xhci_handle_cmd_enable_slot()
1593 static void xhci_handle_cmd_disable_slot(struct xhci_hcd *xhci, int slot_id) in xhci_handle_cmd_disable_slot() argument
1598 virt_dev = xhci->devs[slot_id]; in xhci_handle_cmd_disable_slot()
1602 slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx); in xhci_handle_cmd_disable_slot()
1605 if (xhci->quirks & XHCI_EP_LIMIT_QUIRK) in xhci_handle_cmd_disable_slot()
1607 xhci_free_device_endpoint_resources(xhci, virt_dev, true); in xhci_handle_cmd_disable_slot()
1610 static void xhci_handle_cmd_config_ep(struct xhci_hcd *xhci, int slot_id) in xhci_handle_cmd_config_ep() argument
1623 virt_dev = xhci->devs[slot_id]; in xhci_handle_cmd_config_ep()
1626 ctrl_ctx = xhci_get_input_control_ctx(virt_dev->in_ctx); in xhci_handle_cmd_config_ep()
1628 xhci_warn(xhci, "Could not get input context, bad type.\n"); in xhci_handle_cmd_config_ep()
1632 add_flags = le32_to_cpu(ctrl_ctx->add_flags); in xhci_handle_cmd_config_ep()
1635 ep_index = xhci_last_valid_endpoint(add_flags) - 1; in xhci_handle_cmd_config_ep()
1637 ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->out_ctx, ep_index); in xhci_handle_cmd_config_ep()
1643 static void xhci_handle_cmd_addr_dev(struct xhci_hcd *xhci, int slot_id) in xhci_handle_cmd_addr_dev() argument
1648 vdev = xhci->devs[slot_id]; in xhci_handle_cmd_addr_dev()
1651 slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx); in xhci_handle_cmd_addr_dev()
1655 static void xhci_handle_cmd_reset_dev(struct xhci_hcd *xhci, int slot_id) in xhci_handle_cmd_reset_dev() argument
1660 vdev = xhci->devs[slot_id]; in xhci_handle_cmd_reset_dev()
1662 xhci_warn(xhci, "Reset device command completion for disabled slot %u\n", in xhci_handle_cmd_reset_dev()
1666 slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx); in xhci_handle_cmd_reset_dev()
1669 xhci_dbg(xhci, "Completed reset device command.\n"); in xhci_handle_cmd_reset_dev()
1672 static void xhci_handle_cmd_nec_get_fw(struct xhci_hcd *xhci, in xhci_handle_cmd_nec_get_fw() argument
1675 if (!(xhci->quirks & XHCI_NEC_HOST)) { in xhci_handle_cmd_nec_get_fw()
1676 xhci_warn(xhci, "WARN NEC_GET_FW command on non-NEC host\n"); in xhci_handle_cmd_nec_get_fw()
1679 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks, in xhci_handle_cmd_nec_get_fw()
1681 NEC_FW_MAJOR(le32_to_cpu(event->status)), in xhci_handle_cmd_nec_get_fw()
1682 NEC_FW_MINOR(le32_to_cpu(event->status))); in xhci_handle_cmd_nec_get_fw()
1687 list_del(&cmd->cmd_list); in xhci_complete_del_and_free_cmd()
1689 if (cmd->completion) { in xhci_complete_del_and_free_cmd()
1690 cmd->status = comp_code; in xhci_complete_del_and_free_cmd()
1691 cmd->comp_param = comp_param; in xhci_complete_del_and_free_cmd()
1692 complete(cmd->completion); in xhci_complete_del_and_free_cmd()
1698 void xhci_cleanup_command_queue(struct xhci_hcd *xhci) in xhci_cleanup_command_queue() argument
1701 xhci->current_cmd = NULL; in xhci_cleanup_command_queue()
1702 list_for_each_entry_safe(cur_cmd, tmp_cmd, &xhci->cmd_list, cmd_list) in xhci_cleanup_command_queue()
1708 struct xhci_hcd *xhci; in xhci_handle_command_timeout() local
1715 xhci = container_of(to_delayed_work(work), struct xhci_hcd, cmd_timer); in xhci_handle_command_timeout()
1717 spin_lock_irqsave(&xhci->lock, flags); in xhci_handle_command_timeout()
1723 if (!xhci->current_cmd || delayed_work_pending(&xhci->cmd_timer)) { in xhci_handle_command_timeout()
1724 spin_unlock_irqrestore(&xhci->lock, flags); in xhci_handle_command_timeout()
1728 cmd_field3 = le32_to_cpu(xhci->current_cmd->command_trb->generic.field[3]); in xhci_handle_command_timeout()
1729 usbsts = readl(&xhci->op_regs->status); in xhci_handle_command_timeout()
1730 xhci_dbg(xhci, "Command timeout, USBSTS:%s\n", xhci_decode_usbsts(str, usbsts)); in xhci_handle_command_timeout()
1732 /* Bail out and tear down xhci if a stop endpoint command failed */ in xhci_handle_command_timeout()
1736 xhci_warn(xhci, "xHCI host not responding to stop endpoint command\n"); in xhci_handle_command_timeout()
1738 ep = xhci_get_virt_ep(xhci, TRB_TO_SLOT_ID(cmd_field3), in xhci_handle_command_timeout()
1741 ep->ep_state &= ~EP_STOP_CMD_PENDING; in xhci_handle_command_timeout()
1743 xhci_halt(xhci); in xhci_handle_command_timeout()
1744 xhci_hc_died(xhci); in xhci_handle_command_timeout()
1749 xhci->current_cmd->status = COMP_COMMAND_ABORTED; in xhci_handle_command_timeout()
1752 hw_ring_state = xhci_read_64(xhci, &xhci->op_regs->cmd_ring); in xhci_handle_command_timeout()
1754 xhci_hc_died(xhci); in xhci_handle_command_timeout()
1758 if ((xhci->cmd_ring_state & CMD_RING_STATE_RUNNING) && in xhci_handle_command_timeout()
1761 xhci->cmd_ring_state = CMD_RING_STATE_ABORTED; in xhci_handle_command_timeout()
1762 xhci_dbg(xhci, "Command timeout\n"); in xhci_handle_command_timeout()
1763 xhci_abort_cmd_ring(xhci, flags); in xhci_handle_command_timeout()
1768 if (xhci->xhc_state & XHCI_STATE_REMOVING) { in xhci_handle_command_timeout()
1769 xhci_dbg(xhci, "host removed, ring start fail?\n"); in xhci_handle_command_timeout()
1770 xhci_cleanup_command_queue(xhci); in xhci_handle_command_timeout()
1776 xhci_dbg(xhci, "Command timeout on stopped ring\n"); in xhci_handle_command_timeout()
1777 xhci_handle_stopped_cmd_ring(xhci, xhci->current_cmd); in xhci_handle_command_timeout()
1780 spin_unlock_irqrestore(&xhci->lock, flags); in xhci_handle_command_timeout()
1784 static void handle_cmd_completion(struct xhci_hcd *xhci, in handle_cmd_completion() argument
1787 unsigned int slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags)); in handle_cmd_completion()
1788 u32 status = le32_to_cpu(event->status); in handle_cmd_completion()
1797 xhci_warn(xhci, "Invalid slot_id %u\n", slot_id); in handle_cmd_completion()
1801 cmd_dma = le64_to_cpu(event->cmd_trb); in handle_cmd_completion()
1802 cmd_trb = xhci->cmd_ring->dequeue; in handle_cmd_completion()
1804 trace_xhci_handle_command(xhci->cmd_ring, &cmd_trb->generic, cmd_dma); in handle_cmd_completion()
1806 cmd_comp_code = GET_COMP_CODE(le32_to_cpu(event->status)); in handle_cmd_completion()
1810 complete_all(&xhci->cmd_ring_stop_completion); in handle_cmd_completion()
1814 cmd_dequeue_dma = xhci_trb_virt_to_dma(xhci->cmd_ring->deq_seg, in handle_cmd_completion()
1821 xhci_warn(xhci, in handle_cmd_completion()
1826 cmd = list_first_entry(&xhci->cmd_list, struct xhci_command, cmd_list); in handle_cmd_completion()
1828 cancel_delayed_work(&xhci->cmd_timer); in handle_cmd_completion()
1830 if (cmd->command_trb != xhci->cmd_ring->dequeue) { in handle_cmd_completion()
1831 xhci_err(xhci, in handle_cmd_completion()
1843 xhci->cmd_ring_state = CMD_RING_STATE_STOPPED; in handle_cmd_completion()
1844 if (cmd->status == COMP_COMMAND_ABORTED) { in handle_cmd_completion()
1845 if (xhci->current_cmd == cmd) in handle_cmd_completion()
1846 xhci->current_cmd = NULL; in handle_cmd_completion()
1851 cmd_type = TRB_FIELD_TO_TYPE(le32_to_cpu(cmd_trb->generic.field[3])); in handle_cmd_completion()
1857 xhci_handle_cmd_disable_slot(xhci, slot_id); in handle_cmd_completion()
1860 if (!cmd->completion) in handle_cmd_completion()
1861 xhci_handle_cmd_config_ep(xhci, slot_id); in handle_cmd_completion()
1866 xhci_handle_cmd_addr_dev(xhci, slot_id); in handle_cmd_completion()
1870 le32_to_cpu(cmd_trb->generic.field[3]))); in handle_cmd_completion()
1871 if (!cmd->completion) in handle_cmd_completion()
1872 xhci_handle_cmd_stop_ep(xhci, slot_id, cmd_trb, in handle_cmd_completion()
1877 le32_to_cpu(cmd_trb->generic.field[3]))); in handle_cmd_completion()
1878 xhci_handle_cmd_set_deq(xhci, slot_id, cmd_trb, cmd_comp_code); in handle_cmd_completion()
1881 /* Is this an aborted command turned to NO-OP? */ in handle_cmd_completion()
1882 if (cmd->status == COMP_COMMAND_RING_STOPPED) in handle_cmd_completion()
1887 le32_to_cpu(cmd_trb->generic.field[3]))); in handle_cmd_completion()
1888 xhci_handle_cmd_reset_ep(xhci, slot_id, cmd_trb, cmd_comp_code); in handle_cmd_completion()
1892 * Use the SLOT_ID from the command TRB instead (xhci 4.6.11) in handle_cmd_completion()
1895 le32_to_cpu(cmd_trb->generic.field[3])); in handle_cmd_completion()
1896 xhci_handle_cmd_reset_dev(xhci, slot_id); in handle_cmd_completion()
1899 xhci_handle_cmd_nec_get_fw(xhci, event); in handle_cmd_completion()
1903 xhci_info(xhci, "INFO unknown command type %d\n", cmd_type); in handle_cmd_completion()
1908 if (!list_is_singular(&xhci->cmd_list)) { in handle_cmd_completion()
1909 xhci->current_cmd = list_first_entry(&cmd->cmd_list, in handle_cmd_completion()
1911 xhci_mod_cmd_timer(xhci); in handle_cmd_completion()
1912 } else if (xhci->current_cmd == cmd) { in handle_cmd_completion()
1913 xhci->current_cmd = NULL; in handle_cmd_completion()
1919 inc_deq(xhci, xhci->cmd_ring); in handle_cmd_completion()
1922 static void handle_vendor_event(struct xhci_hcd *xhci, in handle_vendor_event() argument
1925 xhci_dbg(xhci, "Vendor specific event TRB type = %u\n", trb_type); in handle_vendor_event()
1926 if (trb_type == TRB_NEC_CMD_COMP && (xhci->quirks & XHCI_NEC_HOST)) in handle_vendor_event()
1927 handle_cmd_completion(xhci, &event->event_cmd); in handle_vendor_event()
1930 static void handle_device_notification(struct xhci_hcd *xhci, in handle_device_notification() argument
1936 slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->generic.field[3])); in handle_device_notification()
1937 if (!xhci->devs[slot_id]) { in handle_device_notification()
1938 xhci_warn(xhci, "Device Notification event for " in handle_device_notification()
1943 xhci_dbg(xhci, "Device Wake Notification event for slot ID %u\n", in handle_device_notification()
1945 udev = xhci->devs[slot_id]->udev; in handle_device_notification()
1946 if (udev && udev->parent) in handle_device_notification()
1947 usb_wakeup_notification(udev->parent, udev->portnum); in handle_device_notification()
1951 * Quirk hanlder for errata seen on Cavium ThunderX2 processor XHCI
1953 * As per ThunderX2errata-129 USB 2 device may come up as USB 1
1962 static void xhci_cavium_reset_phy_quirk(struct xhci_hcd *xhci) in xhci_cavium_reset_phy_quirk() argument
1964 struct usb_hcd *hcd = xhci_to_hcd(xhci); in xhci_cavium_reset_phy_quirk()
1970 writel(0x6F, hcd->regs + 0x1048); in xhci_cavium_reset_phy_quirk()
1972 /* De-assert the PHY reset */ in xhci_cavium_reset_phy_quirk()
1973 writel(0x7F, hcd->regs + 0x1048); in xhci_cavium_reset_phy_quirk()
1975 pll_lock_check = readl(hcd->regs + 0x1070); in xhci_cavium_reset_phy_quirk()
1976 } while (!(pll_lock_check & 0x1) && --retry_count); in xhci_cavium_reset_phy_quirk()
1979 static void handle_port_status(struct xhci_hcd *xhci, union xhci_trb *event) in handle_port_status() argument
1991 if (GET_COMP_CODE(le32_to_cpu(event->generic.field[2])) != COMP_SUCCESS) in handle_port_status()
1992 xhci_warn(xhci, in handle_port_status()
1995 port_id = GET_PORT_ID(le32_to_cpu(event->generic.field[0])); in handle_port_status()
1996 max_ports = HCS_MAX_PORTS(xhci->hcs_params1); in handle_port_status()
1999 xhci_warn(xhci, "Port change event with invalid port ID %d\n", in handle_port_status()
2004 port = &xhci->hw_ports[port_id - 1]; in handle_port_status()
2005 if (!port || !port->rhub || port->hcd_portnum == DUPLICATE_ENTRY) { in handle_port_status()
2006 xhci_warn(xhci, "Port change event, no port for port ID %u\n", in handle_port_status()
2013 if (port->rhub == &xhci->usb3_rhub && xhci->shared_hcd == NULL) { in handle_port_status()
2014 xhci_dbg(xhci, "ignore port event for removed USB3 hcd\n"); in handle_port_status()
2019 hcd = port->rhub->hcd; in handle_port_status()
2020 bus_state = &port->rhub->bus_state; in handle_port_status()
2021 hcd_portnum = port->hcd_portnum; in handle_port_status()
2022 portsc = readl(port->addr); in handle_port_status()
2024 xhci_dbg(xhci, "Port change event, %d-%d, id %d, portsc: 0x%x\n", in handle_port_status()
2025 hcd->self.busnum, hcd_portnum + 1, port_id, portsc); in handle_port_status()
2029 if (hcd->state == HC_STATE_SUSPENDED) { in handle_port_status()
2030 xhci_dbg(xhci, "resume root hub\n"); in handle_port_status()
2034 if (hcd->speed >= HCD_USB3 && in handle_port_status()
2036 if (port->slot_id && xhci->devs[port->slot_id]) in handle_port_status()
2037 xhci->devs[port->slot_id]->flags |= VDEV_PORT_ERROR; in handle_port_status()
2041 xhci_dbg(xhci, "port resume event for port %d\n", port_id); in handle_port_status()
2043 cmd_reg = readl(&xhci->op_regs->command); in handle_port_status()
2045 xhci_warn(xhci, "xHC is not running.\n"); in handle_port_status()
2050 xhci_dbg(xhci, "remote wake SS port %d\n", port_id); in handle_port_status()
2055 bus_state->port_remote_wakeup |= 1 << hcd_portnum; in handle_port_status()
2056 xhci_test_and_clear_bit(xhci, port, PORT_PLC); in handle_port_status()
2057 usb_hcd_start_port_resume(&hcd->self, hcd_portnum); in handle_port_status()
2058 xhci_set_link_state(xhci, port, XDEV_U0); in handle_port_status()
2064 } else if (!test_bit(hcd_portnum, &bus_state->resuming_ports)) { in handle_port_status()
2065 xhci_dbg(xhci, "resume HS port %d\n", port_id); in handle_port_status()
2066 port->resume_timestamp = jiffies + in handle_port_status()
2068 set_bit(hcd_portnum, &bus_state->resuming_ports); in handle_port_status()
2071 * usb device auto-resume latency around ~40ms. in handle_port_status()
2073 set_bit(HCD_FLAG_POLL_RH, &hcd->flags); in handle_port_status()
2074 mod_timer(&hcd->rh_timer, in handle_port_status()
2075 port->resume_timestamp); in handle_port_status()
2076 usb_hcd_start_port_resume(&hcd->self, hcd_portnum); in handle_port_status()
2086 xhci_dbg(xhci, "resume SS port %d finished\n", port_id); in handle_port_status()
2087 complete(&port->u3exit_done); in handle_port_status()
2090 * U3Exit state after a host-initiated resume. If it's a device in handle_port_status()
2095 if (port->slot_id && xhci->devs[port->slot_id]) in handle_port_status()
2096 xhci_ring_device(xhci, port->slot_id); in handle_port_status()
2097 if (bus_state->port_remote_wakeup & (1 << hcd_portnum)) { in handle_port_status()
2098 xhci_test_and_clear_bit(xhci, port, PORT_PLC); in handle_port_status()
2099 usb_wakeup_notification(hcd->self.root_hub, in handle_port_status()
2107 * Check to see if xhci-hub.c is waiting on RExit to U0 transition (or in handle_port_status()
2111 if (hcd->speed < HCD_USB3 && port->rexit_active) { in handle_port_status()
2112 complete(&port->rexit_done); in handle_port_status()
2113 port->rexit_active = false; in handle_port_status()
2118 if (hcd->speed < HCD_USB3) { in handle_port_status()
2119 xhci_test_and_clear_bit(xhci, port, PORT_PLC); in handle_port_status()
2120 if ((xhci->quirks & XHCI_RESET_PLL_ON_DISCONNECT) && in handle_port_status()
2122 xhci_cavium_reset_phy_quirk(xhci); in handle_port_status()
2135 * xHCI port-status-change events occur when the "or" of all the in handle_port_status()
2136 * status-change bits in the portsc register changes from 0 to 1. in handle_port_status()
2141 xhci_dbg(xhci, "%s: starting usb%d port polling.\n", in handle_port_status()
2142 __func__, hcd->self.busnum); in handle_port_status()
2143 set_bit(HCD_FLAG_POLL_RH, &hcd->flags); in handle_port_status()
2144 spin_unlock(&xhci->lock); in handle_port_status()
2147 spin_lock(&xhci->lock); in handle_port_status()
2150 static void xhci_clear_hub_tt_buffer(struct xhci_hcd *xhci, struct xhci_td *td, in xhci_clear_hub_tt_buffer() argument
2154 * As part of low/full-speed endpoint-halt processing in xhci_clear_hub_tt_buffer()
2157 if (td->urb->dev->tt && !usb_pipeint(td->urb->pipe) && in xhci_clear_hub_tt_buffer()
2158 (td->urb->dev->tt->hub != xhci_to_hcd(xhci)->self.root_hub) && in xhci_clear_hub_tt_buffer()
2159 !(ep->ep_state & EP_CLEARING_TT)) { in xhci_clear_hub_tt_buffer()
2160 ep->ep_state |= EP_CLEARING_TT; in xhci_clear_hub_tt_buffer()
2161 td->urb->ep->hcpriv = td->urb->dev; in xhci_clear_hub_tt_buffer()
2162 if (usb_hub_clear_tt_buffer(td->urb)) in xhci_clear_hub_tt_buffer()
2163 ep->ep_state &= ~EP_CLEARING_TT; in xhci_clear_hub_tt_buffer()
2168 * Check if xhci internal endpoint state has gone to a "halt" state due to an
2197 int xhci_is_vendor_info_code(struct xhci_hcd *xhci, unsigned int trb_comp_code) in xhci_is_vendor_info_code() argument
2201 * treat as not-an-error. in xhci_is_vendor_info_code()
2203 xhci_dbg(xhci, "Vendor defined info completion code %u\n", in xhci_is_vendor_info_code()
2205 xhci_dbg(xhci, "Treating code as success.\n"); in xhci_is_vendor_info_code()
2211 static void finish_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep, in finish_td() argument
2217 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep->ep_index); in finish_td()
2249 if ((ep->ep_state & EP_HALTED) && in finish_td()
2250 !list_empty(&td->cancelled_td_list)) { in finish_td()
2251 xhci_dbg(xhci, "Already resolving halted ep for 0x%llx\n", in finish_td()
2253 td->start_seg, td->start_trb)); in finish_td()
2260 xhci_clear_hub_tt_buffer(xhci, td, ep); in finish_td()
2261 xhci_handle_halted_endpoint(xhci, ep, td, EP_HARD_RESET); in finish_td()
2265 * xhci internal endpoint state will go to a "halt" state for in finish_td()
2274 if (ep->ep_index != 0) in finish_td()
2275 xhci_clear_hub_tt_buffer(xhci, td, ep); in finish_td()
2277 xhci_handle_halted_endpoint(xhci, ep, td, EP_HARD_RESET); in finish_td()
2284 xhci_dequeue_td(xhci, td, ep_ring, td->status); in finish_td()
2291 union xhci_trb *trb = td->start_trb; in sum_trb_lengths()
2292 struct xhci_segment *seg = td->start_seg; in sum_trb_lengths()
2296 sum += TRB_LEN(le32_to_cpu(trb->generic.field[2])); in sum_trb_lengths()
2304 static void process_ctrl_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep, in process_ctrl_td() argument
2313 trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(ep_trb->generic.field[3])); in process_ctrl_td()
2314 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep->ep_index); in process_ctrl_td()
2315 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); in process_ctrl_td()
2316 requested = td->urb->transfer_buffer_length; in process_ctrl_td()
2317 remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)); in process_ctrl_td()
2322 xhci_warn(xhci, "WARN: Success on ctrl %s TRB without IOC set?\n", in process_ctrl_td()
2324 td->status = -ESHUTDOWN; in process_ctrl_td()
2327 td->status = 0; in process_ctrl_td()
2330 td->status = 0; in process_ctrl_td()
2334 td->urb->actual_length = remaining; in process_ctrl_td()
2336 xhci_warn(xhci, "WARN: Stopped Short Packet on ctrl setup or status TRB\n"); in process_ctrl_td()
2341 td->urb->actual_length = 0; in process_ctrl_td()
2345 td->urb->actual_length = requested - remaining; in process_ctrl_td()
2348 td->urb->actual_length = requested; in process_ctrl_td()
2351 xhci_warn(xhci, "WARN: unexpected TRB Type %d\n", in process_ctrl_td()
2360 xhci_dbg(xhci, "TRB error %u, halted endpoint index = %u\n", in process_ctrl_td()
2361 trb_comp_code, ep->ep_index); in process_ctrl_td()
2366 td->urb->actual_length = requested - remaining; in process_ctrl_td()
2367 else if (!td->urb_length_set) in process_ctrl_td()
2368 td->urb->actual_length = 0; in process_ctrl_td()
2382 td->urb_length_set = true; in process_ctrl_td()
2383 td->urb->actual_length = requested - remaining; in process_ctrl_td()
2384 xhci_dbg(xhci, "Waiting for status stage event\n"); in process_ctrl_td()
2389 if (!td->urb_length_set) in process_ctrl_td()
2390 td->urb->actual_length = requested; in process_ctrl_td()
2393 finish_td(xhci, ep, ep_ring, td, trb_comp_code); in process_ctrl_td()
2399 static void process_isoc_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep, in process_isoc_td() argument
2411 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); in process_isoc_td()
2412 urb_priv = td->urb->hcpriv; in process_isoc_td()
2413 idx = urb_priv->num_tds_done; in process_isoc_td()
2414 frame = &td->urb->iso_frame_desc[idx]; in process_isoc_td()
2415 requested = frame->length; in process_isoc_td()
2416 remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)); in process_isoc_td()
2417 ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2])); in process_isoc_td()
2418 short_framestatus = td->urb->transfer_flags & URB_SHORT_NOT_OK ? in process_isoc_td()
2419 -EREMOTEIO : 0; in process_isoc_td()
2424 /* Don't overwrite status if TD had an error, see xHCI 4.9.1 */ in process_isoc_td()
2425 if (td->error_mid_td) in process_isoc_td()
2428 frame->status = short_framestatus; in process_isoc_td()
2432 frame->status = 0; in process_isoc_td()
2435 frame->status = short_framestatus; in process_isoc_td()
2439 frame->status = -ECOMM; in process_isoc_td()
2445 frame->status = -EOVERFLOW; in process_isoc_td()
2446 if (ep_trb != td->end_trb) in process_isoc_td()
2447 td->error_mid_td = true; in process_isoc_td()
2450 frame->status = -EXDEV; in process_isoc_td()
2452 if (ep_trb != td->end_trb) in process_isoc_td()
2453 td->error_mid_td = true; in process_isoc_td()
2457 frame->status = -EPROTO; in process_isoc_td()
2460 frame->status = -EPROTO; in process_isoc_td()
2462 if (ep_trb != td->end_trb) in process_isoc_td()
2463 td->error_mid_td = true; in process_isoc_td()
2470 frame->status = short_framestatus; in process_isoc_td()
2481 frame->status = -1; in process_isoc_td()
2485 if (td->urb_length_set) in process_isoc_td()
2489 frame->actual_length = sum_trb_lengths(td, ep_trb) + in process_isoc_td()
2490 ep_trb_len - remaining; in process_isoc_td()
2492 frame->actual_length = requested; in process_isoc_td()
2494 td->urb->actual_length += frame->actual_length; in process_isoc_td()
2498 if (td->error_mid_td && ep_trb != td->end_trb) { in process_isoc_td()
2499 xhci_dbg(xhci, "Error mid isoc TD, wait for final completion event\n"); in process_isoc_td()
2500 td->urb_length_set = true; in process_isoc_td()
2503 finish_td(xhci, ep, ep_ring, td, trb_comp_code); in process_isoc_td()
2506 static void skip_isoc_td(struct xhci_hcd *xhci, struct xhci_td *td, in skip_isoc_td() argument
2513 urb_priv = td->urb->hcpriv; in skip_isoc_td()
2514 idx = urb_priv->num_tds_done; in skip_isoc_td()
2515 frame = &td->urb->iso_frame_desc[idx]; in skip_isoc_td()
2518 frame->status = -EXDEV; in skip_isoc_td()
2521 frame->actual_length = 0; in skip_isoc_td()
2523 xhci_dequeue_td(xhci, td, ep->ring, status); in skip_isoc_td()
2529 static void process_bulk_intr_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep, in process_bulk_intr_td() argument
2537 slot_ctx = xhci_get_slot_ctx(xhci, ep->vdev->out_ctx); in process_bulk_intr_td()
2538 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); in process_bulk_intr_td()
2539 remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)); in process_bulk_intr_td()
2540 ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2])); in process_bulk_intr_td()
2541 requested = td->urb->transfer_buffer_length; in process_bulk_intr_td()
2545 ep->err_count = 0; in process_bulk_intr_td()
2547 if (ep_trb != td->end_trb || remaining) { in process_bulk_intr_td()
2548 xhci_warn(xhci, "WARN Successful completion on short TX\n"); in process_bulk_intr_td()
2549 xhci_dbg(xhci, "ep %#x - asked for %d bytes, %d bytes untransferred\n", in process_bulk_intr_td()
2550 td->urb->ep->desc.bEndpointAddress, in process_bulk_intr_td()
2553 td->status = 0; in process_bulk_intr_td()
2556 td->status = 0; in process_bulk_intr_td()
2559 td->urb->actual_length = remaining; in process_bulk_intr_td()
2563 td->urb->actual_length = sum_trb_lengths(td, ep_trb); in process_bulk_intr_td()
2566 if (xhci->quirks & XHCI_NO_SOFT_RETRY || in process_bulk_intr_td()
2567 (ep->err_count++ > MAX_SOFT_RETRY) || in process_bulk_intr_td()
2568 le32_to_cpu(slot_ctx->tt_info) & TT_SLOT) in process_bulk_intr_td()
2571 td->status = 0; in process_bulk_intr_td()
2573 xhci_handle_halted_endpoint(xhci, ep, td, EP_SOFT_RESET); in process_bulk_intr_td()
2580 if (ep_trb == td->end_trb) in process_bulk_intr_td()
2581 td->urb->actual_length = requested - remaining; in process_bulk_intr_td()
2583 td->urb->actual_length = in process_bulk_intr_td()
2585 ep_trb_len - remaining; in process_bulk_intr_td()
2588 xhci_warn(xhci, "bad transfer trb length %d in event trb\n", in process_bulk_intr_td()
2590 td->urb->actual_length = 0; in process_bulk_intr_td()
2593 finish_td(xhci, ep, ep_ring, td, trb_comp_code); in process_bulk_intr_td()
2596 /* Transfer events which don't point to a transfer TRB, see xhci 4.17.4 */
2597 static int handle_transferless_tx_event(struct xhci_hcd *xhci, struct xhci_virt_ep *ep, in handle_transferless_tx_event() argument
2605 xhci_dbg(xhci, "Stream transaction error ep %u no id\n", ep->ep_index); in handle_transferless_tx_event()
2606 if (ep->err_count++ > MAX_SOFT_RETRY) in handle_transferless_tx_event()
2607 xhci_handle_halted_endpoint(xhci, ep, NULL, EP_HARD_RESET); in handle_transferless_tx_event()
2609 xhci_handle_halted_endpoint(xhci, ep, NULL, EP_SOFT_RESET); in handle_transferless_tx_event()
2616 xhci_err(xhci, "Transfer event %u for unknown stream ring slot %u ep %u\n", in handle_transferless_tx_event()
2617 trb_comp_code, ep->vdev->slot_id, ep->ep_index); in handle_transferless_tx_event()
2618 return -ENODEV; in handle_transferless_tx_event()
2623 static bool xhci_spurious_success_tx_event(struct xhci_hcd *xhci, in xhci_spurious_success_tx_event() argument
2626 switch (ring->old_trb_comp_code) { in xhci_spurious_success_tx_event()
2628 return xhci->quirks & XHCI_SPURIOUS_SUCCESS; in xhci_spurious_success_tx_event()
2632 return xhci->quirks & XHCI_ETRON_HOST && in xhci_spurious_success_tx_event()
2633 ring->type == TYPE_ISOC; in xhci_spurious_success_tx_event()
2644 static int handle_tx_event(struct xhci_hcd *xhci, in handle_tx_event() argument
2656 int status = -EINPROGRESS; in handle_tx_event()
2661 slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags)); in handle_tx_event()
2662 ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1; in handle_tx_event()
2663 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); in handle_tx_event()
2664 ep_trb_dma = le64_to_cpu(event->buffer); in handle_tx_event()
2666 ep = xhci_get_virt_ep(xhci, slot_id, ep_index); in handle_tx_event()
2668 xhci_err(xhci, "ERROR Invalid Transfer event\n"); in handle_tx_event()
2673 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep_index); in handle_tx_event()
2676 xhci_err(xhci, in handle_tx_event()
2683 return handle_transferless_tx_event(xhci, ep, trb_comp_code); in handle_tx_event()
2691 if (EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)) != 0) { in handle_tx_event()
2693 xhci_dbg(xhci, "Successful completion on short TX for slot %u ep %u with last td comp code %d\n", in handle_tx_event()
2694 slot_id, ep_index, ep_ring->old_trb_comp_code); in handle_tx_event()
2701 xhci_dbg(xhci, "Stopped on Transfer TRB for slot %u ep %u\n", in handle_tx_event()
2705 xhci_dbg(xhci, in handle_tx_event()
2706 "Stopped on No-op or Link TRB for slot %u ep %u\n", in handle_tx_event()
2710 xhci_dbg(xhci, in handle_tx_event()
2716 xhci_dbg(xhci, "Stalled endpoint for slot %u ep %u\n", slot_id, in handle_tx_event()
2718 status = -EPIPE; in handle_tx_event()
2721 xhci_dbg(xhci, "Split transaction error for slot %u ep %u\n", in handle_tx_event()
2723 status = -EPROTO; in handle_tx_event()
2726 xhci_dbg(xhci, "Transfer error for slot %u ep %u on endpoint\n", in handle_tx_event()
2728 status = -EPROTO; in handle_tx_event()
2731 xhci_dbg(xhci, "Babble error for slot %u ep %u on endpoint\n", in handle_tx_event()
2733 status = -EOVERFLOW; in handle_tx_event()
2737 xhci_warn(xhci, in handle_tx_event()
2740 status = -EILSEQ; in handle_tx_event()
2744 xhci_warn(xhci, in handle_tx_event()
2747 status = -ENOSR; in handle_tx_event()
2750 xhci_warn(xhci, in handle_tx_event()
2755 xhci_warn(xhci, in handle_tx_event()
2765 xhci_dbg(xhci, "Underrun event on slot %u ep %u\n", slot_id, ep_index); in handle_tx_event()
2769 xhci_dbg(xhci, "Overrun event on slot %u ep %u\n", slot_id, ep_index); in handle_tx_event()
2779 ep->skip = true; in handle_tx_event()
2780 xhci_dbg(xhci, in handle_tx_event()
2785 ep->skip = true; in handle_tx_event()
2786 xhci_dbg(xhci, in handle_tx_event()
2793 xhci_warn(xhci, in handle_tx_event()
2796 status = -EPROTO; in handle_tx_event()
2799 if (xhci_is_vendor_info_code(xhci, trb_comp_code)) { in handle_tx_event()
2803 xhci_warn(xhci, in handle_tx_event()
2806 if (ep->skip) in handle_tx_event()
2812 * xhci 4.10.2 states isoc endpoints should continue in handle_tx_event()
2816 * xhci 4.9.1 states that if there are errors in mult-TRB in handle_tx_event()
2824 td = list_first_entry_or_null(&ep_ring->td_list, struct xhci_td, td_list); in handle_tx_event()
2826 if (td && td->error_mid_td && !trb_in_td(td, ep_trb_dma)) { in handle_tx_event()
2827 xhci_dbg(xhci, "Missing TD completion event after mid TD error\n"); in handle_tx_event()
2828 xhci_dequeue_td(xhci, td, ep_ring, td->status); in handle_tx_event()
2835 if (list_empty(&ep_ring->td_list)) { in handle_tx_event()
2845 !xhci_spurious_success_tx_event(xhci, ep_ring)) { in handle_tx_event()
2846 xhci_warn(xhci, "Event TRB for slot %u ep %u with no TDs queued\n", in handle_tx_event()
2850 ep->skip = false; in handle_tx_event()
2855 td = list_first_entry(&ep_ring->td_list, struct xhci_td, in handle_tx_event()
2863 if (ep->skip && usb_endpoint_xfer_isoc(&td->urb->ep->desc)) { in handle_tx_event()
2868 skip_isoc_td(xhci, td, ep, status); in handle_tx_event()
2870 if (!list_empty(&ep_ring->td_list)) { in handle_tx_event()
2873 * If we are here, we are on xHCI 1.0 host with no in handle_tx_event()
2878 xhci_dbg(xhci, "Skipped one TD for slot %u ep %u", in handle_tx_event()
2885 xhci_dbg(xhci, "All TDs skipped for slot %u ep %u. Clear skip flag.\n", in handle_tx_event()
2887 ep->skip = false; in handle_tx_event()
2898 * TD pointed by 'ep_ring->dequeue' because that the hardware dequeue in handle_tx_event()
2912 if (xhci_spurious_success_tx_event(xhci, ep_ring)) { in handle_tx_event()
2913 xhci_dbg(xhci, "Spurious event dma %pad, comp_code %u after %u\n", in handle_tx_event()
2914 &ep_trb_dma, trb_comp_code, ep_ring->old_trb_comp_code); in handle_tx_event()
2915 ep_ring->old_trb_comp_code = 0; in handle_tx_event()
2923 if (ep->skip) { in handle_tx_event()
2924 xhci_dbg(xhci, in handle_tx_event()
2927 ep->skip = false; in handle_tx_event()
2931 * If ep->skip is set, it means there are missed tds on the in handle_tx_event()
2936 } while (ep->skip); in handle_tx_event()
2938 ep_ring->old_trb_comp_code = trb_comp_code; in handle_tx_event()
2944 ep_trb = &ep_seg->trbs[(ep_trb_dma - ep_seg->dma) / sizeof(*ep_trb)]; in handle_tx_event()
2948 * No-op TRB could trigger interrupts in a case where a URB was killed in handle_tx_event()
2957 td->status = status; in handle_tx_event()
2960 if (usb_endpoint_xfer_control(&td->urb->ep->desc)) in handle_tx_event()
2961 process_ctrl_td(xhci, ep, ep_ring, td, ep_trb, event); in handle_tx_event()
2962 else if (usb_endpoint_xfer_isoc(&td->urb->ep->desc)) in handle_tx_event()
2963 process_isoc_td(xhci, ep, ep_ring, td, ep_trb, event); in handle_tx_event()
2965 process_bulk_intr_td(xhci, ep, ep_ring, td, ep_trb, event); in handle_tx_event()
2970 xhci_handle_halted_endpoint(xhci, ep, td, EP_HARD_RESET); in handle_tx_event()
2975 xhci_err(xhci, "Event dma %pad for ep %d status %d not part of TD at %016llx - %016llx\n", in handle_tx_event()
2977 (unsigned long long)xhci_trb_virt_to_dma(td->start_seg, td->start_trb), in handle_tx_event()
2978 (unsigned long long)xhci_trb_virt_to_dma(td->end_seg, td->end_trb)); in handle_tx_event()
2980 xhci_for_each_ring_seg(ep_ring->first_seg, ep_seg) in handle_tx_event()
2981 xhci_warn(xhci, "Ring seg %u dma %pad\n", ep_seg->num, &ep_seg->dma); in handle_tx_event()
2983 return -ESHUTDOWN; in handle_tx_event()
2986 xhci_err(xhci, "@%016llx %08x %08x %08x %08x\n", in handle_tx_event()
2988 ir->event_ring->deq_seg, in handle_tx_event()
2989 ir->event_ring->dequeue), in handle_tx_event()
2990 lower_32_bits(le64_to_cpu(event->buffer)), in handle_tx_event()
2991 upper_32_bits(le64_to_cpu(event->buffer)), in handle_tx_event()
2992 le32_to_cpu(event->transfer_len), in handle_tx_event()
2993 le32_to_cpu(event->flags)); in handle_tx_event()
2994 return -ENODEV; in handle_tx_event()
2998 * This function handles one OS-owned event on the event ring. It may drop
2999 * xhci->lock between event processing (e.g. to pass up port status changes).
3001 static int xhci_handle_event_trb(struct xhci_hcd *xhci, struct xhci_interrupter *ir, in xhci_handle_event_trb() argument
3006 trace_xhci_handle_event(ir->event_ring, &event->generic, in xhci_handle_event_trb()
3007 xhci_trb_virt_to_dma(ir->event_ring->deq_seg, in xhci_handle_event_trb()
3008 ir->event_ring->dequeue)); in xhci_handle_event_trb()
3015 trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(event->event_cmd.flags)); in xhci_handle_event_trb()
3020 handle_cmd_completion(xhci, &event->event_cmd); in xhci_handle_event_trb()
3023 handle_port_status(xhci, event); in xhci_handle_event_trb()
3026 handle_tx_event(xhci, ir, &event->trans_event); in xhci_handle_event_trb()
3029 handle_device_notification(xhci, event); in xhci_handle_event_trb()
3033 handle_vendor_event(xhci, event, trb_type); in xhci_handle_event_trb()
3035 xhci_warn(xhci, "ERROR unknown event type %d\n", trb_type); in xhci_handle_event_trb()
3037 /* Any of the above functions may drop and re-acquire the lock, so check in xhci_handle_event_trb()
3038 * to make sure a watchdog timer didn't mark the host as non-responsive. in xhci_handle_event_trb()
3040 if (xhci->xhc_state & XHCI_STATE_DYING) { in xhci_handle_event_trb()
3041 xhci_dbg(xhci, "xHCI host dying, returning from event handler.\n"); in xhci_handle_event_trb()
3042 return -ENODEV; in xhci_handle_event_trb()
3050 * - When all events have finished
3051 * - To avoid "Event Ring Full Error" condition
3053 static void xhci_update_erst_dequeue(struct xhci_hcd *xhci, in xhci_update_erst_dequeue() argument
3060 temp_64 = xhci_read_64(xhci, &ir->ir_set->erst_dequeue); in xhci_update_erst_dequeue()
3061 deq = xhci_trb_virt_to_dma(ir->event_ring->deq_seg, in xhci_update_erst_dequeue()
3062 ir->event_ring->dequeue); in xhci_update_erst_dequeue()
3064 xhci_warn(xhci, "WARN something wrong with SW event ring dequeue ptr\n"); in xhci_update_erst_dequeue()
3073 temp_64 = ir->event_ring->deq_seg->num & ERST_DESI_MASK; in xhci_update_erst_dequeue()
3079 xhci_write_64(xhci, temp_64, &ir->ir_set->erst_dequeue); in xhci_update_erst_dequeue()
3085 if (!ir->ip_autoclear) { in xhci_clear_interrupt_pending()
3088 irq_pending = readl(&ir->ir_set->irq_pending); in xhci_clear_interrupt_pending()
3090 writel(irq_pending, &ir->ir_set->irq_pending); in xhci_clear_interrupt_pending()
3095 * Handle all OS-owned events on an interrupter event ring. It may drop
3096 * and reaquire xhci->lock between event processing.
3098 static int xhci_handle_events(struct xhci_hcd *xhci, struct xhci_interrupter *ir) in xhci_handle_events() argument
3107 if (!ir->event_ring || !ir->event_ring->dequeue) { in xhci_handle_events()
3108 xhci_err(xhci, "ERROR interrupter event ring not ready\n"); in xhci_handle_events()
3109 return -ENOMEM; in xhci_handle_events()
3112 if (xhci->xhc_state & XHCI_STATE_DYING || in xhci_handle_events()
3113 xhci->xhc_state & XHCI_STATE_HALTED) { in xhci_handle_events()
3114 xhci_dbg(xhci, "xHCI dying, ignoring interrupt. Shouldn't IRQs be disabled?\n"); in xhci_handle_events()
3117 temp = xhci_read_64(xhci, &ir->ir_set->erst_dequeue); in xhci_handle_events()
3118 xhci_write_64(xhci, temp | ERST_EHB, &ir->ir_set->erst_dequeue); in xhci_handle_events()
3119 return -ENODEV; in xhci_handle_events()
3123 while (unhandled_event_trb(ir->event_ring)) { in xhci_handle_events()
3124 err = xhci_handle_event_trb(xhci, ir, ir->event_ring->dequeue); in xhci_handle_events()
3131 xhci_update_erst_dequeue(xhci, ir, false); in xhci_handle_events()
3133 if (ir->isoc_bei_interval > AVOID_BEI_INTERVAL_MIN) in xhci_handle_events()
3134 ir->isoc_bei_interval = ir->isoc_bei_interval / 2; in xhci_handle_events()
3140 inc_deq(xhci, ir->event_ring); in xhci_handle_events()
3146 xhci_update_erst_dequeue(xhci, ir, true); in xhci_handle_events()
3152 * xHCI spec says we can get an interrupt, and if the HC has an error condition,
3158 struct xhci_hcd *xhci = hcd_to_xhci(hcd); in xhci_irq() local
3162 spin_lock(&xhci->lock); in xhci_irq()
3164 status = readl(&xhci->op_regs->status); in xhci_irq()
3166 xhci_hc_died(xhci); in xhci_irq()
3176 xhci_warn(xhci, "WARNING: Host Controller Error\n"); in xhci_irq()
3181 xhci_warn(xhci, "WARNING: Host System Error\n"); in xhci_irq()
3182 xhci_halt(xhci); in xhci_irq()
3188 * so we can receive interrupts from other MSI-X interrupters. in xhci_irq()
3192 writel(status, &xhci->op_regs->status); in xhci_irq()
3195 xhci_handle_events(xhci, xhci->interrupters[0]); in xhci_irq()
3197 spin_unlock(&xhci->lock); in xhci_irq()
3211 * Generic function for queueing a TRB on a ring.
3217 static void queue_trb(struct xhci_hcd *xhci, struct xhci_ring *ring, in queue_trb() argument
3223 trb = &ring->enqueue->generic; in queue_trb()
3224 trb->field[0] = cpu_to_le32(field1); in queue_trb()
3225 trb->field[1] = cpu_to_le32(field2); in queue_trb()
3226 trb->field[2] = cpu_to_le32(field3); in queue_trb()
3229 trb->field[3] = cpu_to_le32(field4); in queue_trb()
3232 xhci_trb_virt_to_dma(ring->enq_seg, ring->enqueue)); in queue_trb()
3234 inc_enq(xhci, ring, more_trbs_coming); in queue_trb()
3241 static int prepare_ring(struct xhci_hcd *xhci, struct xhci_ring *ep_ring, in prepare_ring() argument
3253 xhci_warn(xhci, "WARN urb submitted to disabled ep\n"); in prepare_ring()
3254 return -ENOENT; in prepare_ring()
3256 xhci_warn(xhci, "WARN waiting for error on ep to be cleared\n"); in prepare_ring()
3258 /* XXX not sure if this should be -ENOENT or not */ in prepare_ring()
3259 return -EINVAL; in prepare_ring()
3261 xhci_dbg(xhci, "WARN halted endpoint, queueing URB anyway.\n"); in prepare_ring()
3267 xhci_err(xhci, "ERROR unknown endpoint state for ep\n"); in prepare_ring()
3272 return -EINVAL; in prepare_ring()
3275 if (ep_ring != xhci->cmd_ring) { in prepare_ring()
3276 new_segs = xhci_ring_expansion_needed(xhci, ep_ring, num_trbs); in prepare_ring()
3278 xhci_err(xhci, "Do not support expand command ring\n"); in prepare_ring()
3279 return -ENOMEM; in prepare_ring()
3283 xhci_dbg_trace(xhci, trace_xhci_dbg_ring_expansion, in prepare_ring()
3285 if (xhci_ring_expansion(xhci, ep_ring, new_segs, mem_flags)) { in prepare_ring()
3286 xhci_err(xhci, "Ring expansion failed\n"); in prepare_ring()
3287 return -ENOMEM; in prepare_ring()
3292 if (trb_is_link(ep_ring->enqueue)) in prepare_ring()
3293 inc_enq_past_link(xhci, ep_ring, 0); in prepare_ring()
3295 if (last_trb_on_seg(ep_ring->enq_seg, ep_ring->enqueue)) { in prepare_ring()
3296 xhci_warn(xhci, "Missing link TRB at end of ring segment\n"); in prepare_ring()
3297 return -EINVAL; in prepare_ring()
3303 static int prepare_transfer(struct xhci_hcd *xhci, in prepare_transfer() argument
3316 struct xhci_ep_ctx *ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index); in prepare_transfer()
3318 ep_ring = xhci_triad_to_transfer_ring(xhci, xdev->slot_id, ep_index, in prepare_transfer()
3321 xhci_dbg(xhci, "Can't prepare ring for bad stream ID %u\n", in prepare_transfer()
3323 return -EINVAL; in prepare_transfer()
3326 ret = prepare_ring(xhci, ep_ring, GET_EP_CTX_STATE(ep_ctx), in prepare_transfer()
3331 urb_priv = urb->hcpriv; in prepare_transfer()
3332 td = &urb_priv->td[td_index]; in prepare_transfer()
3334 INIT_LIST_HEAD(&td->td_list); in prepare_transfer()
3335 INIT_LIST_HEAD(&td->cancelled_td_list); in prepare_transfer()
3338 ret = usb_hcd_link_urb_to_ep(bus_to_hcd(urb->dev->bus), urb); in prepare_transfer()
3343 td->urb = urb; in prepare_transfer()
3345 list_add_tail(&td->td_list, &ep_ring->td_list); in prepare_transfer()
3346 td->start_seg = ep_ring->enq_seg; in prepare_transfer()
3347 td->start_trb = ep_ring->enqueue; in prepare_transfer()
3356 num_trbs = DIV_ROUND_UP(len + (addr & (TRB_MAX_BUFF_SIZE - 1)), in count_trbs()
3366 return count_trbs(urb->transfer_dma, urb->transfer_buffer_length); in count_trbs_needed()
3374 full_len = urb->transfer_buffer_length; in count_sg_trbs_needed()
3376 for_each_sg(urb->sg, sg, urb->num_mapped_sgs, i) { in count_sg_trbs_needed()
3380 full_len -= len; in count_sg_trbs_needed()
3392 addr = (u64) (urb->transfer_dma + urb->iso_frame_desc[i].offset); in count_isoc_trbs_needed()
3393 len = urb->iso_frame_desc[i].length; in count_isoc_trbs_needed()
3400 if (unlikely(running_total != urb->transfer_buffer_length)) in check_trb_math()
3401 dev_err(&urb->dev->dev, "%s - ep %#x - Miscalculated tx length, " in check_trb_math()
3404 urb->ep->desc.bEndpointAddress, in check_trb_math()
3406 urb->transfer_buffer_length, in check_trb_math()
3407 urb->transfer_buffer_length); in check_trb_math()
3410 static void giveback_first_trb(struct xhci_hcd *xhci, int slot_id, in giveback_first_trb() argument
3420 start_trb->field[3] |= cpu_to_le32(start_cycle); in giveback_first_trb()
3422 start_trb->field[3] &= cpu_to_le32(~TRB_CYCLE); in giveback_first_trb()
3423 xhci_ring_ep_doorbell(xhci, slot_id, ep_index, stream_id); in giveback_first_trb()
3431 xhci_interval = EP_INTERVAL_TO_UFRAMES(le32_to_cpu(ep_ctx->ep_info)); in check_interval()
3432 ep_interval = urb->interval; in check_interval()
3435 if (urb->dev->speed == USB_SPEED_LOW || in check_interval()
3436 urb->dev->speed == USB_SPEED_FULL) in check_interval()
3443 dev_dbg_ratelimited(&urb->dev->dev, in check_interval()
3444 "Driver uses different interval (%d microframe%s) than xHCI (%d microframe%s)\n", in check_interval()
3447 urb->interval = xhci_interval; in check_interval()
3449 if (urb->dev->speed == USB_SPEED_LOW || in check_interval()
3450 urb->dev->speed == USB_SPEED_FULL) in check_interval()
3451 urb->interval /= 8; in check_interval()
3456 * xHCI uses normal TRBs for both bulk and interrupt. When the interrupt
3461 int xhci_queue_intr_tx(struct xhci_hcd *xhci, gfp_t mem_flags, in xhci_queue_intr_tx() argument
3466 ep_ctx = xhci_get_ep_ctx(xhci, xhci->devs[slot_id]->out_ctx, ep_index); in xhci_queue_intr_tx()
3469 return xhci_queue_bulk_tx(xhci, mem_flags, urb, slot_id, ep_index); in xhci_queue_intr_tx()
3473 * For xHCI 1.0 host controllers, TD size is the number of max packet sized
3482 * TD size = total_packet_count - packets_transferred
3484 * For xHCI 0.96 and older, TD size field should be the remaining bytes
3492 static u32 xhci_td_remainder(struct xhci_hcd *xhci, int transferred, in xhci_td_remainder() argument
3498 /* MTK xHCI 0.96 contains some features from 1.0 */ in xhci_td_remainder()
3499 if (xhci->hci_version < 0x100 && !(xhci->quirks & XHCI_MTK_HOST)) in xhci_td_remainder()
3500 return ((td_total_len - transferred) >> 10); in xhci_td_remainder()
3502 /* One TRB with a zero-length data packet. */ in xhci_td_remainder()
3507 /* for MTK xHCI 0.96, TD size include this TRB, but not in 1.x */ in xhci_td_remainder()
3508 if ((xhci->quirks & XHCI_MTK_HOST) && (xhci->hci_version < 0x100)) in xhci_td_remainder()
3511 maxp = usb_endpoint_maxp(&urb->ep->desc); in xhci_td_remainder()
3515 return (total_packet_count - ((transferred + trb_buff_len) / maxp)); in xhci_td_remainder()
3519 static int xhci_align_td(struct xhci_hcd *xhci, struct urb *urb, u32 enqd_len, in xhci_align_td() argument
3522 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_align_td()
3528 max_pkt = usb_endpoint_maxp(&urb->ep->desc); in xhci_align_td()
3535 xhci_dbg(xhci, "Unaligned %d bytes, buff len %d\n", in xhci_align_td()
3540 *trb_buff_len -= unalign; in xhci_align_td()
3541 xhci_dbg(xhci, "split align, new buff len %d\n", *trb_buff_len); in xhci_align_td()
3550 new_buff_len = max_pkt - (enqd_len % max_pkt); in xhci_align_td()
3552 if (new_buff_len > (urb->transfer_buffer_length - enqd_len)) in xhci_align_td()
3553 new_buff_len = (urb->transfer_buffer_length - enqd_len); in xhci_align_td()
3557 if (urb->num_sgs) { in xhci_align_td()
3558 len = sg_pcopy_to_buffer(urb->sg, urb->num_sgs, in xhci_align_td()
3559 seg->bounce_buf, new_buff_len, enqd_len); in xhci_align_td()
3561 xhci_warn(xhci, "WARN Wrong bounce buffer write length: %zu != %d\n", in xhci_align_td()
3564 memcpy(seg->bounce_buf, urb->transfer_buffer + enqd_len, new_buff_len); in xhci_align_td()
3567 seg->bounce_dma = dma_map_single(dev, seg->bounce_buf, in xhci_align_td()
3570 seg->bounce_dma = dma_map_single(dev, seg->bounce_buf, in xhci_align_td()
3574 if (dma_mapping_error(dev, seg->bounce_dma)) { in xhci_align_td()
3576 xhci_warn(xhci, "Failed mapping bounce buffer, not aligning\n"); in xhci_align_td()
3580 seg->bounce_len = new_buff_len; in xhci_align_td()
3581 seg->bounce_offs = enqd_len; in xhci_align_td()
3583 xhci_dbg(xhci, "Bounce align, new buff len %d\n", *trb_buff_len); in xhci_align_td()
3588 /* This is very similar to what ehci-q.c qtd_fill() does */
3589 int xhci_queue_bulk_tx(struct xhci_hcd *xhci, gfp_t mem_flags, in xhci_queue_bulk_tx() argument
3607 ring = xhci_urb_to_transfer_ring(xhci, urb); in xhci_queue_bulk_tx()
3609 return -EINVAL; in xhci_queue_bulk_tx()
3611 full_len = urb->transfer_buffer_length; in xhci_queue_bulk_tx()
3613 if (urb->num_sgs && !(urb->transfer_flags & URB_DMA_MAP_SINGLE)) { in xhci_queue_bulk_tx()
3614 num_sgs = urb->num_mapped_sgs; in xhci_queue_bulk_tx()
3615 sg = urb->sg; in xhci_queue_bulk_tx()
3621 addr = (u64) urb->transfer_dma; in xhci_queue_bulk_tx()
3624 ret = prepare_transfer(xhci, xhci->devs[slot_id], in xhci_queue_bulk_tx()
3625 ep_index, urb->stream_id, in xhci_queue_bulk_tx()
3630 urb_priv = urb->hcpriv; in xhci_queue_bulk_tx()
3632 /* Deal with URB_ZERO_PACKET - need one more td/trb */ in xhci_queue_bulk_tx()
3633 if (urb->transfer_flags & URB_ZERO_PACKET && urb_priv->num_tds > 1) in xhci_queue_bulk_tx()
3636 td = &urb_priv->td[0]; in xhci_queue_bulk_tx()
3643 start_trb = &ring->enqueue->generic; in xhci_queue_bulk_tx()
3644 start_cycle = ring->cycle_state; in xhci_queue_bulk_tx()
3647 /* Queue the TRBs, even if they are zero-length */ in xhci_queue_bulk_tx()
3657 trb_buff_len = full_len - enqd_len; in xhci_queue_bulk_tx()
3665 field |= ring->cycle_state; in xhci_queue_bulk_tx()
3672 if (trb_is_link(ring->enqueue + 1)) { in xhci_queue_bulk_tx()
3673 if (xhci_align_td(xhci, urb, enqd_len, in xhci_queue_bulk_tx()
3675 ring->enq_seg)) { in xhci_queue_bulk_tx()
3676 send_addr = ring->enq_seg->bounce_dma; in xhci_queue_bulk_tx()
3678 td->bounce_seg = ring->enq_seg; in xhci_queue_bulk_tx()
3686 td->end_trb = ring->enqueue; in xhci_queue_bulk_tx()
3687 td->end_seg = ring->enq_seg; in xhci_queue_bulk_tx()
3689 memcpy(&send_addr, urb->transfer_buffer, in xhci_queue_bulk_tx()
3701 remainder = xhci_td_remainder(xhci, enqd_len, trb_buff_len, in xhci_queue_bulk_tx()
3708 queue_trb(xhci, ring, more_trbs_coming | need_zero_pkt, in xhci_queue_bulk_tx()
3718 --num_sgs; in xhci_queue_bulk_tx()
3719 sent_len -= block_len; in xhci_queue_bulk_tx()
3727 block_len -= sent_len; in xhci_queue_bulk_tx()
3732 ret = prepare_transfer(xhci, xhci->devs[slot_id], in xhci_queue_bulk_tx()
3733 ep_index, urb->stream_id, in xhci_queue_bulk_tx()
3735 urb_priv->td[1].end_trb = ring->enqueue; in xhci_queue_bulk_tx()
3736 urb_priv->td[1].end_seg = ring->enq_seg; in xhci_queue_bulk_tx()
3737 field = TRB_TYPE(TRB_NORMAL) | ring->cycle_state | TRB_IOC; in xhci_queue_bulk_tx()
3738 queue_trb(xhci, ring, 0, 0, 0, TRB_INTR_TARGET(0), field); in xhci_queue_bulk_tx()
3742 giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id, in xhci_queue_bulk_tx()
3747 /* Caller must have locked xhci->lock */
3748 int xhci_queue_ctrl_tx(struct xhci_hcd *xhci, gfp_t mem_flags, in xhci_queue_ctrl_tx() argument
3761 ep_ring = xhci_urb_to_transfer_ring(xhci, urb); in xhci_queue_ctrl_tx()
3763 return -EINVAL; in xhci_queue_ctrl_tx()
3769 if (!urb->setup_packet) in xhci_queue_ctrl_tx()
3770 return -EINVAL; in xhci_queue_ctrl_tx()
3772 if ((xhci->quirks & XHCI_ETRON_HOST) && in xhci_queue_ctrl_tx()
3773 urb->dev->speed >= USB_SPEED_SUPER) { in xhci_queue_ctrl_tx()
3779 if (last_trb_on_seg(ep_ring->enq_seg, ep_ring->enqueue + 1)) { in xhci_queue_ctrl_tx()
3780 field = TRB_TYPE(TRB_TR_NOOP) | ep_ring->cycle_state; in xhci_queue_ctrl_tx()
3781 queue_trb(xhci, ep_ring, false, 0, 0, in xhci_queue_ctrl_tx()
3793 if (urb->transfer_buffer_length > 0) in xhci_queue_ctrl_tx()
3795 ret = prepare_transfer(xhci, xhci->devs[slot_id], in xhci_queue_ctrl_tx()
3796 ep_index, urb->stream_id, in xhci_queue_ctrl_tx()
3801 urb_priv = urb->hcpriv; in xhci_queue_ctrl_tx()
3802 td = &urb_priv->td[0]; in xhci_queue_ctrl_tx()
3809 start_trb = &ep_ring->enqueue->generic; in xhci_queue_ctrl_tx()
3810 start_cycle = ep_ring->cycle_state; in xhci_queue_ctrl_tx()
3812 /* Queue setup TRB - see section 6.4.1.2.1 */ in xhci_queue_ctrl_tx()
3814 setup = (struct usb_ctrlrequest *) urb->setup_packet; in xhci_queue_ctrl_tx()
3820 /* xHCI 1.0/1.1 6.4.1.2.1: Transfer Type field */ in xhci_queue_ctrl_tx()
3821 if ((xhci->hci_version >= 0x100) || (xhci->quirks & XHCI_MTK_HOST)) { in xhci_queue_ctrl_tx()
3822 if (urb->transfer_buffer_length > 0) { in xhci_queue_ctrl_tx()
3823 if (setup->bRequestType & USB_DIR_IN) in xhci_queue_ctrl_tx()
3830 queue_trb(xhci, ep_ring, true, in xhci_queue_ctrl_tx()
3831 setup->bRequestType | setup->bRequest << 8 | le16_to_cpu(setup->wValue) << 16, in xhci_queue_ctrl_tx()
3832 le16_to_cpu(setup->wIndex) | le16_to_cpu(setup->wLength) << 16, in xhci_queue_ctrl_tx()
3844 if (urb->transfer_buffer_length > 0) { in xhci_queue_ctrl_tx()
3849 memcpy(&addr, urb->transfer_buffer, in xhci_queue_ctrl_tx()
3850 urb->transfer_buffer_length); in xhci_queue_ctrl_tx()
3854 addr = (u64) urb->transfer_dma; in xhci_queue_ctrl_tx()
3857 remainder = xhci_td_remainder(xhci, 0, in xhci_queue_ctrl_tx()
3858 urb->transfer_buffer_length, in xhci_queue_ctrl_tx()
3859 urb->transfer_buffer_length, in xhci_queue_ctrl_tx()
3861 length_field = TRB_LEN(urb->transfer_buffer_length) | in xhci_queue_ctrl_tx()
3864 if (setup->bRequestType & USB_DIR_IN) in xhci_queue_ctrl_tx()
3866 queue_trb(xhci, ep_ring, true, in xhci_queue_ctrl_tx()
3870 field | ep_ring->cycle_state); in xhci_queue_ctrl_tx()
3874 td->end_trb = ep_ring->enqueue; in xhci_queue_ctrl_tx()
3875 td->end_seg = ep_ring->enq_seg; in xhci_queue_ctrl_tx()
3877 /* Queue status TRB - see Table 7 and sections 4.11.2.2 and 6.4.1.2.3 */ in xhci_queue_ctrl_tx()
3879 if (urb->transfer_buffer_length > 0 && setup->bRequestType & USB_DIR_IN) in xhci_queue_ctrl_tx()
3883 queue_trb(xhci, ep_ring, false, in xhci_queue_ctrl_tx()
3888 field | TRB_IOC | TRB_TYPE(TRB_STATUS) | ep_ring->cycle_state); in xhci_queue_ctrl_tx()
3890 giveback_first_trb(xhci, slot_id, ep_index, 0, in xhci_queue_ctrl_tx()
3901 * zero. Only xHCI 1.0 host controllers support this field.
3903 static unsigned int xhci_get_burst_count(struct xhci_hcd *xhci, in xhci_get_burst_count() argument
3908 if (xhci->hci_version < 0x100 || urb->dev->speed < USB_SPEED_SUPER) in xhci_get_burst_count()
3911 max_burst = urb->ep->ss_ep_comp.bMaxBurst; in xhci_get_burst_count()
3912 return DIV_ROUND_UP(total_packet_count, max_burst + 1) - 1; in xhci_get_burst_count()
3923 static unsigned int xhci_get_last_burst_packet_count(struct xhci_hcd *xhci, in xhci_get_last_burst_packet_count() argument
3929 if (xhci->hci_version < 0x100) in xhci_get_last_burst_packet_count()
3932 if (urb->dev->speed >= USB_SPEED_SUPER) { in xhci_get_last_burst_packet_count()
3934 max_burst = urb->ep->ss_ep_comp.bMaxBurst; in xhci_get_last_burst_packet_count()
3937 * number of packets, but the TLBPC field is zero-based. in xhci_get_last_burst_packet_count()
3941 return residue - 1; in xhci_get_last_burst_packet_count()
3945 return total_packet_count - 1; in xhci_get_last_burst_packet_count()
3955 static int xhci_get_isoc_frame_id(struct xhci_hcd *xhci, in xhci_get_isoc_frame_id() argument
3961 if (urb->dev->speed == USB_SPEED_LOW || in xhci_get_isoc_frame_id()
3962 urb->dev->speed == USB_SPEED_FULL) in xhci_get_isoc_frame_id()
3963 start_frame = urb->start_frame + index * urb->interval; in xhci_get_isoc_frame_id()
3965 start_frame = (urb->start_frame + index * urb->interval) >> 3; in xhci_get_isoc_frame_id()
3975 ist = HCS_IST(xhci->hcs_params2) & 0x7; in xhci_get_isoc_frame_id()
3976 if (HCS_IST(xhci->hcs_params2) & (1 << 3)) in xhci_get_isoc_frame_id()
3992 current_frame_id = readl(&xhci->run_regs->microframe_index); in xhci_get_isoc_frame_id()
4003 ret = -EINVAL; in xhci_get_isoc_frame_id()
4007 ret = -EINVAL; in xhci_get_isoc_frame_id()
4009 ret = -EINVAL; in xhci_get_isoc_frame_id()
4013 if (ret == -EINVAL || start_frame == start_frame_id) { in xhci_get_isoc_frame_id()
4015 if (urb->dev->speed == USB_SPEED_LOW || in xhci_get_isoc_frame_id()
4016 urb->dev->speed == USB_SPEED_FULL) in xhci_get_isoc_frame_id()
4017 urb->start_frame = start_frame; in xhci_get_isoc_frame_id()
4019 urb->start_frame = start_frame << 3; in xhci_get_isoc_frame_id()
4025 xhci_warn(xhci, "Frame ID %d (reg %d, index %d) beyond range (%d, %d)\n", in xhci_get_isoc_frame_id()
4028 xhci_warn(xhci, "Ignore frame ID field, use SIA bit instead\n"); in xhci_get_isoc_frame_id()
4036 static bool trb_block_event_intr(struct xhci_hcd *xhci, int num_tds, int i, in trb_block_event_intr() argument
4039 if (xhci->hci_version < 0x100) in trb_block_event_intr()
4042 if (i == num_tds - 1) in trb_block_event_intr()
4048 if (i && ir->isoc_bei_interval && xhci->quirks & XHCI_AVOID_BEI) in trb_block_event_intr()
4049 return !!(i % ir->isoc_bei_interval); in trb_block_event_intr()
4055 static int xhci_queue_isoc_tx(struct xhci_hcd *xhci, gfp_t mem_flags, in xhci_queue_isoc_tx() argument
4074 xep = &xhci->devs[slot_id]->eps[ep_index]; in xhci_queue_isoc_tx()
4075 ep_ring = xhci->devs[slot_id]->eps[ep_index].ring; in xhci_queue_isoc_tx()
4076 ir = xhci->interrupters[0]; in xhci_queue_isoc_tx()
4078 num_tds = urb->number_of_packets; in xhci_queue_isoc_tx()
4080 xhci_dbg(xhci, "Isoc URB with zero packets?\n"); in xhci_queue_isoc_tx()
4081 return -EINVAL; in xhci_queue_isoc_tx()
4083 start_addr = (u64) urb->transfer_dma; in xhci_queue_isoc_tx()
4084 start_trb = &ep_ring->enqueue->generic; in xhci_queue_isoc_tx()
4085 start_cycle = ep_ring->cycle_state; in xhci_queue_isoc_tx()
4087 urb_priv = urb->hcpriv; in xhci_queue_isoc_tx()
4088 /* Queue the TRBs for each TD, even if they are zero-length */ in xhci_queue_isoc_tx()
4096 addr = start_addr + urb->iso_frame_desc[i].offset; in xhci_queue_isoc_tx()
4097 td_len = urb->iso_frame_desc[i].length; in xhci_queue_isoc_tx()
4099 max_pkt = usb_endpoint_maxp(&urb->ep->desc); in xhci_queue_isoc_tx()
4102 /* A zero-length transfer still involves at least one packet. */ in xhci_queue_isoc_tx()
4105 burst_count = xhci_get_burst_count(xhci, urb, total_pkt_count); in xhci_queue_isoc_tx()
4106 last_burst_pkt_count = xhci_get_last_burst_packet_count(xhci, in xhci_queue_isoc_tx()
4111 ret = prepare_transfer(xhci, xhci->devs[slot_id], ep_index, in xhci_queue_isoc_tx()
4112 urb->stream_id, trbs_per_td, urb, i, mem_flags); in xhci_queue_isoc_tx()
4118 td = &urb_priv->td[i]; in xhci_queue_isoc_tx()
4121 if (!(urb->transfer_flags & URB_ISO_ASAP) && in xhci_queue_isoc_tx()
4122 HCC_CFC(xhci->hcc_params)) { in xhci_queue_isoc_tx()
4123 frame_id = xhci_get_isoc_frame_id(xhci, urb, i); in xhci_queue_isoc_tx()
4135 (i ? ep_ring->cycle_state : !start_cycle); in xhci_queue_isoc_tx()
4137 /* xhci 1.1 with ETE uses TD_Size field for TBC, old is Rsvdz */ in xhci_queue_isoc_tx()
4138 if (!xep->use_extended_tbc) in xhci_queue_isoc_tx()
4148 ep_ring->cycle_state; in xhci_queue_isoc_tx()
4155 if (j < trbs_per_td - 1) { in xhci_queue_isoc_tx()
4160 td->end_trb = ep_ring->enqueue; in xhci_queue_isoc_tx()
4161 td->end_seg = ep_ring->enq_seg; in xhci_queue_isoc_tx()
4163 if (trb_block_event_intr(xhci, num_tds, i, ir)) in xhci_queue_isoc_tx()
4172 remainder = xhci_td_remainder(xhci, running_total, in xhci_queue_isoc_tx()
4179 /* xhci 1.1 with ETE uses TD Size field for TBC */ in xhci_queue_isoc_tx()
4180 if (first_trb && xep->use_extended_tbc) in xhci_queue_isoc_tx()
4186 queue_trb(xhci, ep_ring, more_trbs_coming, in xhci_queue_isoc_tx()
4194 td_remain_len -= trb_buff_len; in xhci_queue_isoc_tx()
4199 xhci_err(xhci, "ISOC TD length unmatch\n"); in xhci_queue_isoc_tx()
4200 ret = -EINVAL; in xhci_queue_isoc_tx()
4206 if (HCC_CFC(xhci->hcc_params)) in xhci_queue_isoc_tx()
4207 xep->next_frame_id = urb->start_frame + num_tds * urb->interval; in xhci_queue_isoc_tx()
4209 if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs == 0) { in xhci_queue_isoc_tx()
4210 if (xhci->quirks & XHCI_AMD_PLL_FIX) in xhci_queue_isoc_tx()
4213 xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs++; in xhci_queue_isoc_tx()
4215 giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id, in xhci_queue_isoc_tx()
4221 for (i--; i >= 0; i--) in xhci_queue_isoc_tx()
4222 list_del_init(&urb_priv->td[i].td_list); in xhci_queue_isoc_tx()
4225 * into No-ops with a software-owned cycle bit. That way the hardware in xhci_queue_isoc_tx()
4227 * overwrite them. td->start_trb and td->start_seg are already set. in xhci_queue_isoc_tx()
4229 urb_priv->td[0].end_trb = ep_ring->enqueue; in xhci_queue_isoc_tx()
4231 td_to_noop(&urb_priv->td[0], true); in xhci_queue_isoc_tx()
4234 ep_ring->enqueue = urb_priv->td[0].start_trb; in xhci_queue_isoc_tx()
4235 ep_ring->enq_seg = urb_priv->td[0].start_seg; in xhci_queue_isoc_tx()
4236 ep_ring->cycle_state = start_cycle; in xhci_queue_isoc_tx()
4237 usb_hcd_unlink_urb_from_ep(bus_to_hcd(urb->dev->bus), urb); in xhci_queue_isoc_tx()
4244 * Update interval as xhci_queue_intr_tx does. Use xhci frame_index to
4245 * update urb->start_frame if URB_ISO_ASAP is set in transfer_flags or
4248 int xhci_queue_isoc_tx_prepare(struct xhci_hcd *xhci, gfp_t mem_flags, in xhci_queue_isoc_tx_prepare() argument
4260 xdev = xhci->devs[slot_id]; in xhci_queue_isoc_tx_prepare()
4261 xep = &xhci->devs[slot_id]->eps[ep_index]; in xhci_queue_isoc_tx_prepare()
4262 ep_ring = xdev->eps[ep_index].ring; in xhci_queue_isoc_tx_prepare()
4263 ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index); in xhci_queue_isoc_tx_prepare()
4266 num_tds = urb->number_of_packets; in xhci_queue_isoc_tx_prepare()
4273 ret = prepare_ring(xhci, ep_ring, GET_EP_CTX_STATE(ep_ctx), in xhci_queue_isoc_tx_prepare()
4284 /* Calculate the start frame and put it in urb->start_frame. */ in xhci_queue_isoc_tx_prepare()
4285 if (HCC_CFC(xhci->hcc_params) && !list_empty(&ep_ring->td_list)) { in xhci_queue_isoc_tx_prepare()
4287 urb->start_frame = xep->next_frame_id; in xhci_queue_isoc_tx_prepare()
4292 start_frame = readl(&xhci->run_regs->microframe_index); in xhci_queue_isoc_tx_prepare()
4298 ist = HCS_IST(xhci->hcs_params2) & 0x7; in xhci_queue_isoc_tx_prepare()
4299 if (HCS_IST(xhci->hcs_params2) & (1 << 3)) in xhci_queue_isoc_tx_prepare()
4308 if (urb->dev->speed == USB_SPEED_LOW || in xhci_queue_isoc_tx_prepare()
4309 urb->dev->speed == USB_SPEED_FULL) { in xhci_queue_isoc_tx_prepare()
4310 start_frame = roundup(start_frame, urb->interval << 3); in xhci_queue_isoc_tx_prepare()
4311 urb->start_frame = start_frame >> 3; in xhci_queue_isoc_tx_prepare()
4313 start_frame = roundup(start_frame, urb->interval); in xhci_queue_isoc_tx_prepare()
4314 urb->start_frame = start_frame; in xhci_queue_isoc_tx_prepare()
4319 return xhci_queue_isoc_tx(xhci, mem_flags, urb, slot_id, ep_index); in xhci_queue_isoc_tx_prepare()
4324 /* Generic function for queueing a command TRB on the command ring.
4329 * Don't decrement xhci->cmd_ring_reserved_trbs after we've queued the TRB
4332 static int queue_command(struct xhci_hcd *xhci, struct xhci_command *cmd, in queue_command() argument
4336 int reserved_trbs = xhci->cmd_ring_reserved_trbs; in queue_command()
4339 if ((xhci->xhc_state & XHCI_STATE_DYING) || in queue_command()
4340 (xhci->xhc_state & XHCI_STATE_HALTED)) { in queue_command()
4341 xhci_dbg(xhci, "xHCI dying or halted, can't queue_command\n"); in queue_command()
4342 return -ESHUTDOWN; in queue_command()
4348 ret = prepare_ring(xhci, xhci->cmd_ring, EP_STATE_RUNNING, in queue_command()
4351 xhci_err(xhci, "ERR: No room for command on command ring\n"); in queue_command()
4353 xhci_err(xhci, "ERR: Reserved TRB counting for " in queue_command()
4358 cmd->command_trb = xhci->cmd_ring->enqueue; in queue_command()
4361 if (list_empty(&xhci->cmd_list)) { in queue_command()
4362 xhci->current_cmd = cmd; in queue_command()
4363 xhci_mod_cmd_timer(xhci); in queue_command()
4366 list_add_tail(&cmd->cmd_list, &xhci->cmd_list); in queue_command()
4368 queue_trb(xhci, xhci->cmd_ring, false, field1, field2, field3, in queue_command()
4369 field4 | xhci->cmd_ring->cycle_state); in queue_command()
4374 int xhci_queue_slot_control(struct xhci_hcd *xhci, struct xhci_command *cmd, in xhci_queue_slot_control() argument
4377 return queue_command(xhci, cmd, 0, 0, 0, in xhci_queue_slot_control()
4382 int xhci_queue_address_device(struct xhci_hcd *xhci, struct xhci_command *cmd, in xhci_queue_address_device() argument
4385 return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr), in xhci_queue_address_device()
4391 int xhci_queue_vendor_command(struct xhci_hcd *xhci, struct xhci_command *cmd, in xhci_queue_vendor_command() argument
4394 return queue_command(xhci, cmd, field1, field2, field3, field4, false); in xhci_queue_vendor_command()
4398 int xhci_queue_reset_device(struct xhci_hcd *xhci, struct xhci_command *cmd, in xhci_queue_reset_device() argument
4401 return queue_command(xhci, cmd, 0, 0, 0, in xhci_queue_reset_device()
4407 int xhci_queue_configure_endpoint(struct xhci_hcd *xhci, in xhci_queue_configure_endpoint() argument
4411 return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr), in xhci_queue_configure_endpoint()
4418 int xhci_queue_evaluate_context(struct xhci_hcd *xhci, struct xhci_command *cmd, in xhci_queue_evaluate_context() argument
4421 return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr), in xhci_queue_evaluate_context()
4431 int xhci_queue_stop_endpoint(struct xhci_hcd *xhci, struct xhci_command *cmd, in xhci_queue_stop_endpoint() argument
4439 return queue_command(xhci, cmd, 0, 0, 0, in xhci_queue_stop_endpoint()
4443 int xhci_queue_reset_ep(struct xhci_hcd *xhci, struct xhci_command *cmd, in xhci_queue_reset_ep() argument
4454 return queue_command(xhci, cmd, 0, 0, 0, in xhci_queue_reset_ep()