Lines Matching +full:lpm +full:- +full:nyet +full:- +full:threshold
1 // SPDX-License-Identifier: GPL-2.0
3 * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
5 * Copyright (C) 2010-2011 Texas Instruments Incorporated - https://www.ti.com
20 #include <linux/dma-mapping.h>
30 #define DWC3_ALIGN_FRAME(d, n) (((d)->frame_number + ((d)->interval * (n))) \
31 & ~((d)->interval - 1))
34 * dwc3_gadget_set_test_mode - enables usb2 test modes
39 * success or -EINVAL if wrong Test Selector is passed.
45 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in dwc3_gadget_set_test_mode()
57 return -EINVAL; in dwc3_gadget_set_test_mode()
66 * dwc3_gadget_get_link_state - gets current state of usb link
70 * return the link state on success (>= 0) or -ETIMEDOUT.
76 reg = dwc3_readl(dwc->regs, DWC3_DSTS); in dwc3_gadget_get_link_state()
82 * dwc3_gadget_set_link_state - sets usb link to a particular state
87 * return 0 on success or -ETIMEDOUT.
99 while (--retries) { in dwc3_gadget_set_link_state()
100 reg = dwc3_readl(dwc->regs, DWC3_DSTS); in dwc3_gadget_set_link_state()
108 return -ETIMEDOUT; in dwc3_gadget_set_link_state()
111 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in dwc3_gadget_set_link_state()
115 dwc3_writel(dwc->regs, DWC3_DCTL, reg); in dwc3_gadget_set_link_state()
119 dwc3_writel(dwc->regs, DWC3_DCTL, reg); in dwc3_gadget_set_link_state()
130 while (--retries) { in dwc3_gadget_set_link_state()
131 reg = dwc3_readl(dwc->regs, DWC3_DSTS); in dwc3_gadget_set_link_state()
139 return -ETIMEDOUT; in dwc3_gadget_set_link_state()
146 if (dwc->ep0state != EP0_SETUP_PHASE) { in dwc3_ep0_reset_state()
147 dir = !!dwc->ep0_expect_in; in dwc3_ep0_reset_state()
148 if (dwc->ep0state == EP0_DATA_PHASE) in dwc3_ep0_reset_state()
149 dwc3_ep0_end_control_data(dwc, dwc->eps[dir]); in dwc3_ep0_reset_state()
151 dwc3_ep0_end_control_data(dwc, dwc->eps[!dir]); in dwc3_ep0_reset_state()
153 dwc->eps[0]->trb_enqueue = 0; in dwc3_ep0_reset_state()
154 dwc->eps[1]->trb_enqueue = 0; in dwc3_ep0_reset_state()
161 * dwc3_ep_inc_trb - increment a trb index.
171 if (*index == (DWC3_TRB_NUM - 1)) in dwc3_ep_inc_trb()
176 * dwc3_ep_inc_enq - increment endpoint's enqueue pointer
181 dwc3_ep_inc_trb(&dep->trb_enqueue); in dwc3_ep_inc_enq()
185 * dwc3_ep_inc_deq - increment endpoint's dequeue pointer
190 dwc3_ep_inc_trb(&dep->trb_dequeue); in dwc3_ep_inc_deq()
196 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_del_and_unmap_request()
198 list_del(&req->list); in dwc3_gadget_del_and_unmap_request()
199 req->remaining = 0; in dwc3_gadget_del_and_unmap_request()
200 req->num_trbs = 0; in dwc3_gadget_del_and_unmap_request()
202 if (req->request.status == -EINPROGRESS) in dwc3_gadget_del_and_unmap_request()
203 req->request.status = status; in dwc3_gadget_del_and_unmap_request()
205 if (req->trb) in dwc3_gadget_del_and_unmap_request()
206 usb_gadget_unmap_request_by_dev(dwc->sysdev, in dwc3_gadget_del_and_unmap_request()
207 &req->request, req->direction); in dwc3_gadget_del_and_unmap_request()
209 req->trb = NULL; in dwc3_gadget_del_and_unmap_request()
212 if (dep->number > 1) in dwc3_gadget_del_and_unmap_request()
213 pm_runtime_put(dwc->dev); in dwc3_gadget_del_and_unmap_request()
217 * dwc3_gadget_giveback - call struct usb_request's ->complete callback
223 * function will unmap @req and call its ->complete() callback to notify upper
229 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_giveback()
232 req->status = DWC3_REQUEST_STATUS_COMPLETED; in dwc3_gadget_giveback()
234 spin_unlock(&dwc->lock); in dwc3_gadget_giveback()
235 usb_gadget_giveback_request(&dep->endpoint, &req->request); in dwc3_gadget_giveback()
236 spin_lock(&dwc->lock); in dwc3_gadget_giveback()
240 * dwc3_send_gadget_generic_command - issue a generic command for the controller
256 dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param); in dwc3_send_gadget_generic_command()
257 dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT); in dwc3_send_gadget_generic_command()
260 reg = dwc3_readl(dwc->regs, DWC3_DGCMD); in dwc3_send_gadget_generic_command()
264 ret = -EINVAL; in dwc3_send_gadget_generic_command()
267 } while (--timeout); in dwc3_send_gadget_generic_command()
270 ret = -ETIMEDOUT; in dwc3_send_gadget_generic_command()
271 status = -ETIMEDOUT; in dwc3_send_gadget_generic_command()
280 * dwc3_send_gadget_ep_cmd - issue an endpoint command
308 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc; in dwc3_send_gadget_ep_cmd()
309 struct dwc3 *dwc = dep->dwc; in dwc3_send_gadget_ep_cmd()
315 int ret = -EINVAL; in dwc3_send_gadget_ep_cmd()
327 if (dwc->gadget->speed <= USB_SPEED_HIGH || in dwc3_send_gadget_ep_cmd()
329 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0)); in dwc3_send_gadget_ep_cmd()
341 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg); in dwc3_send_gadget_ep_cmd()
351 dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0); in dwc3_send_gadget_ep_cmd()
352 dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1); in dwc3_send_gadget_ep_cmd()
353 dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2); in dwc3_send_gadget_ep_cmd()
377 dwc3_writel(dep->regs, DWC3_DEPCMD, cmd); in dwc3_send_gadget_ep_cmd()
387 reg = dwc3_readl(dep->regs, DWC3_DEPCMD); in dwc3_send_gadget_ep_cmd()
396 dev_WARN(dwc->dev, "No resource for %s\n", in dwc3_send_gadget_ep_cmd()
397 dep->name); in dwc3_send_gadget_ep_cmd()
398 ret = -EINVAL; in dwc3_send_gadget_ep_cmd()
408 * Instead of always returning -EINVAL, let's in dwc3_send_gadget_ep_cmd()
410 * the case by returning -EAGAIN. in dwc3_send_gadget_ep_cmd()
412 ret = -EAGAIN; in dwc3_send_gadget_ep_cmd()
415 dev_WARN(dwc->dev, "UNKNOWN cmd status\n"); in dwc3_send_gadget_ep_cmd()
420 } while (--timeout); in dwc3_send_gadget_ep_cmd()
423 ret = -ETIMEDOUT; in dwc3_send_gadget_ep_cmd()
424 cmd_status = -ETIMEDOUT; in dwc3_send_gadget_ep_cmd()
432 dep->flags |= DWC3_EP_TRANSFER_STARTED; in dwc3_send_gadget_ep_cmd()
434 if (ret != -ETIMEDOUT) in dwc3_send_gadget_ep_cmd()
443 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0)); in dwc3_send_gadget_ep_cmd()
445 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg); in dwc3_send_gadget_ep_cmd()
453 struct dwc3 *dwc = dep->dwc; in dwc3_send_clear_stall_ep_cmd()
461 * some (non-compliant) hosts may not send ACK TPs for pending in dwc3_send_clear_stall_ep_cmd()
465 if (dep->direction && in dwc3_send_clear_stall_ep_cmd()
467 (dwc->gadget->speed >= USB_SPEED_SUPER)) in dwc3_send_clear_stall_ep_cmd()
478 u32 offset = (char *) trb - (char *) dep->trb_pool; in dwc3_trb_dma_offset()
480 return dep->trb_pool_dma + offset; in dwc3_trb_dma_offset()
485 struct dwc3 *dwc = dep->dwc; in dwc3_alloc_trb_pool()
487 if (dep->trb_pool) in dwc3_alloc_trb_pool()
490 dep->trb_pool = dma_alloc_coherent(dwc->sysdev, in dwc3_alloc_trb_pool()
492 &dep->trb_pool_dma, GFP_KERNEL); in dwc3_alloc_trb_pool()
493 if (!dep->trb_pool) { in dwc3_alloc_trb_pool()
494 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n", in dwc3_alloc_trb_pool()
495 dep->name); in dwc3_alloc_trb_pool()
496 return -ENOMEM; in dwc3_alloc_trb_pool()
504 struct dwc3 *dwc = dep->dwc; in dwc3_free_trb_pool()
506 dma_free_coherent(dwc->sysdev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM, in dwc3_free_trb_pool()
507 dep->trb_pool, dep->trb_pool_dma); in dwc3_free_trb_pool()
509 dep->trb_pool = NULL; in dwc3_free_trb_pool()
510 dep->trb_pool_dma = 0; in dwc3_free_trb_pool()
518 if (dep->flags & DWC3_EP_RESOURCE_ALLOCATED) in dwc3_gadget_set_xfer_resource()
530 dep->flags |= DWC3_EP_RESOURCE_ALLOCATED; in dwc3_gadget_set_xfer_resource()
535 * dwc3_gadget_start_config - reset endpoint resources
540 * part of the power-on/soft-reset initialization.
542 * Set resource_index=2 to reset only non-control endpoints' resources. Do this
554 return -EINVAL; in dwc3_gadget_start_config()
560 ret = dwc3_send_gadget_ep_cmd(dwc->eps[0], cmd, ¶ms); in dwc3_gadget_start_config()
565 for (i = resource_index; i < dwc->num_eps; i++) { in dwc3_gadget_start_config()
566 dep = dwc->eps[i]; in dwc3_gadget_start_config()
570 dep->flags &= ~DWC3_EP_RESOURCE_ALLOCATED; in dwc3_gadget_start_config()
581 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_set_ep_config()
583 comp_desc = dep->endpoint.comp_desc; in dwc3_gadget_set_ep_config()
584 desc = dep->endpoint.desc; in dwc3_gadget_set_ep_config()
592 if (dwc->gadget->speed >= USB_SPEED_SUPER) { in dwc3_gadget_set_ep_config()
593 u32 burst = dep->endpoint.maxburst; in dwc3_gadget_set_ep_config()
595 params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1); in dwc3_gadget_set_ep_config()
600 params.param2 |= dep->saved_state; in dwc3_gadget_set_ep_config()
605 if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc)) in dwc3_gadget_set_ep_config()
612 dep->stream_capable = true; in dwc3_gadget_set_ep_config()
624 params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number); in dwc3_gadget_set_ep_config()
630 if (dep->direction) in dwc3_gadget_set_ep_config()
631 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1); in dwc3_gadget_set_ep_config()
633 if (desc->bInterval) { in dwc3_gadget_set_ep_config()
644 bInterval_m1 = min_t(u8, desc->bInterval - 1, 13); in dwc3_gadget_set_ep_config()
647 dwc->gadget->speed == USB_SPEED_FULL) in dwc3_gadget_set_ep_config()
648 dep->interval = desc->bInterval; in dwc3_gadget_set_ep_config()
650 dep->interval = 1 << (desc->bInterval - 1); in dwc3_gadget_set_ep_config()
659 * dwc3_gadget_calc_tx_fifo_size - calculates the txfifo size value
694 * dwc3_gadget_calc_ram_depth - calculates the ram depth for txfifo
704 is_single_port_ram = DWC3_SPRAM_TYPE(dwc->hwparams.hwparams1); in dwc3_gadget_calc_ram_depth()
710 ram_depth = is_single_port_ram ? DWC3_RAM0_DEPTH(dwc->hwparams.hwparams6) : in dwc3_gadget_calc_ram_depth()
711 DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7); in dwc3_gadget_calc_ram_depth()
716 * at a non-zero address. in dwc3_gadget_calc_ram_depth()
721 /* Check if TXFIFOs start at non-zero addr */ in dwc3_gadget_calc_ram_depth()
722 reg = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0)); in dwc3_gadget_calc_ram_depth()
725 ram_depth -= (fifo_0_start >> 16); in dwc3_gadget_calc_ram_depth()
732 * dwc3_gadget_clear_tx_fifos - Clears txfifo allocation
745 if (!dwc->do_fifo_resize) in dwc3_gadget_clear_tx_fifos()
749 dep = dwc->eps[1]; in dwc3_gadget_clear_tx_fifos()
750 size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0)); in dwc3_gadget_clear_tx_fifos()
756 dwc->last_fifo_depth = fifo_depth; in dwc3_gadget_clear_tx_fifos()
758 for (num = 3; num < min_t(int, dwc->num_eps, DWC3_ENDPOINTS_NUM); num += 2) { in dwc3_gadget_clear_tx_fifos()
759 dep = dwc->eps[num]; in dwc3_gadget_clear_tx_fifos()
765 dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1)) & in dwc3_gadget_clear_tx_fifos()
768 dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1), size); in dwc3_gadget_clear_tx_fifos()
769 dep->flags &= ~DWC3_EP_TXFIFO_RESIZED; in dwc3_gadget_clear_tx_fifos()
771 dwc->num_ep_resized = 0; in dwc3_gadget_clear_tx_fifos()
775 * dwc3_gadget_resize_tx_fifos - reallocate fifo spaces for current use-case
783 * on the configured size for RAM1 - which contains TxFifo -,
797 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_resize_tx_fifos()
808 if (!dwc->do_fifo_resize) in dwc3_gadget_resize_tx_fifos()
812 if (!usb_endpoint_dir_in(dep->endpoint.desc) || dep->number <= 1) in dwc3_gadget_resize_tx_fifos()
816 if (dep->flags & DWC3_EP_TXFIFO_RESIZED) in dwc3_gadget_resize_tx_fifos()
821 switch (dwc->gadget->speed) { in dwc3_gadget_resize_tx_fifos()
824 if (usb_endpoint_xfer_bulk(dep->endpoint.desc) || in dwc3_gadget_resize_tx_fifos()
825 usb_endpoint_xfer_isoc(dep->endpoint.desc)) in dwc3_gadget_resize_tx_fifos()
827 dep->endpoint.maxburst, in dwc3_gadget_resize_tx_fifos()
828 dwc->tx_fifo_resize_max_num); in dwc3_gadget_resize_tx_fifos()
831 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) { in dwc3_gadget_resize_tx_fifos()
833 usb_endpoint_maxp_mult(dep->endpoint.desc) + 1, in dwc3_gadget_resize_tx_fifos()
834 dwc->tx_fifo_resize_max_num); in dwc3_gadget_resize_tx_fifos()
839 if (usb_endpoint_xfer_bulk(dep->endpoint.desc)) in dwc3_gadget_resize_tx_fifos()
850 num_in_ep = dwc->max_cfg_eps; in dwc3_gadget_resize_tx_fifos()
851 num_in_ep -= dwc->num_ep_resized; in dwc3_gadget_resize_tx_fifos()
855 remaining = ram_depth - min_depth - dwc->last_fifo_depth; in dwc3_gadget_resize_tx_fifos()
862 fifo_size = (num_fifos - 1) * fifo; in dwc3_gadget_resize_tx_fifos()
870 /* Check if TXFIFOs start at non-zero addr */ in dwc3_gadget_resize_tx_fifos()
871 tmp = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0)); in dwc3_gadget_resize_tx_fifos()
874 fifo_size |= (fifo_0_start + (dwc->last_fifo_depth << 16)); in dwc3_gadget_resize_tx_fifos()
876 dwc->last_fifo_depth += DWC3_GTXFIFOSIZ_TXFDEP(fifo_size); in dwc3_gadget_resize_tx_fifos()
878 dwc->last_fifo_depth += DWC31_GTXFIFOSIZ_TXFDEP(fifo_size); in dwc3_gadget_resize_tx_fifos()
881 if (dwc->last_fifo_depth >= ram_depth) { in dwc3_gadget_resize_tx_fifos()
882 dev_err(dwc->dev, "Fifosize(%d) > RAM size(%d) %s depth:%d\n", in dwc3_gadget_resize_tx_fifos()
883 dwc->last_fifo_depth, ram_depth, in dwc3_gadget_resize_tx_fifos()
884 dep->endpoint.name, fifo_size); in dwc3_gadget_resize_tx_fifos()
890 dwc->last_fifo_depth -= fifo_size; in dwc3_gadget_resize_tx_fifos()
891 return -ENOMEM; in dwc3_gadget_resize_tx_fifos()
894 dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1), fifo_size); in dwc3_gadget_resize_tx_fifos()
895 dep->flags |= DWC3_EP_TXFIFO_RESIZED; in dwc3_gadget_resize_tx_fifos()
896 dwc->num_ep_resized++; in dwc3_gadget_resize_tx_fifos()
902 * __dwc3_gadget_ep_enable - initializes a hw endpoint
911 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc; in __dwc3_gadget_ep_enable()
912 struct dwc3 *dwc = dep->dwc; in __dwc3_gadget_ep_enable()
917 if (!(dep->flags & DWC3_EP_ENABLED)) { in __dwc3_gadget_ep_enable()
927 if (!(dep->flags & DWC3_EP_RESOURCE_ALLOCATED)) { in __dwc3_gadget_ep_enable()
933 if (!(dep->flags & DWC3_EP_ENABLED)) { in __dwc3_gadget_ep_enable()
937 dep->type = usb_endpoint_type(desc); in __dwc3_gadget_ep_enable()
938 dep->flags |= DWC3_EP_ENABLED; in __dwc3_gadget_ep_enable()
940 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA); in __dwc3_gadget_ep_enable()
941 reg |= DWC3_DALEPENA_EP(dep->number); in __dwc3_gadget_ep_enable()
942 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg); in __dwc3_gadget_ep_enable()
944 dep->trb_dequeue = 0; in __dwc3_gadget_ep_enable()
945 dep->trb_enqueue = 0; in __dwc3_gadget_ep_enable()
951 memset(dep->trb_pool, 0, in __dwc3_gadget_ep_enable()
955 trb_st_hw = &dep->trb_pool[0]; in __dwc3_gadget_ep_enable()
957 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1]; in __dwc3_gadget_ep_enable()
958 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw)); in __dwc3_gadget_ep_enable()
959 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw)); in __dwc3_gadget_ep_enable()
960 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB; in __dwc3_gadget_ep_enable()
961 trb_link->ctrl |= DWC3_TRB_CTRL_HWO; in __dwc3_gadget_ep_enable()
965 * Issue StartTransfer here with no-op TRB so we can always rely on No in __dwc3_gadget_ep_enable()
976 trb = &dep->trb_pool[0]; in __dwc3_gadget_ep_enable()
988 if (dep->stream_capable) { in __dwc3_gadget_ep_enable()
995 * no-op TRB as normal, but end it immediately. As a in __dwc3_gadget_ep_enable()
1012 if (!dep->direction || in __dwc3_gadget_ep_enable()
1013 !(dwc->hwparams.hwparams9 & in __dwc3_gadget_ep_enable()
1015 dep->flags |= DWC3_EP_FORCE_RESTART_STREAM; in __dwc3_gadget_ep_enable()
1032 if (dep->flags & DWC3_EP_DELAY_STOP) in dwc3_remove_requests()
1035 /* - giveback all requests to gadget driver */ in dwc3_remove_requests()
1036 while (!list_empty(&dep->started_list)) { in dwc3_remove_requests()
1037 req = next_request(&dep->started_list); in dwc3_remove_requests()
1042 while (!list_empty(&dep->pending_list)) { in dwc3_remove_requests()
1043 req = next_request(&dep->pending_list); in dwc3_remove_requests()
1048 while (!list_empty(&dep->cancelled_list)) { in dwc3_remove_requests()
1049 req = next_request(&dep->cancelled_list); in dwc3_remove_requests()
1056 * __dwc3_gadget_ep_disable - disables a hw endpoint
1067 struct dwc3 *dwc = dep->dwc; in __dwc3_gadget_ep_disable()
1074 if (dep->flags & DWC3_EP_STALL) in __dwc3_gadget_ep_disable()
1077 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA); in __dwc3_gadget_ep_disable()
1078 reg &= ~DWC3_DALEPENA_EP(dep->number); in __dwc3_gadget_ep_disable()
1079 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg); in __dwc3_gadget_ep_disable()
1081 dwc3_remove_requests(dwc, dep, -ESHUTDOWN); in __dwc3_gadget_ep_disable()
1083 dep->stream_capable = false; in __dwc3_gadget_ep_disable()
1084 dep->type = 0; in __dwc3_gadget_ep_disable()
1091 if (dep->flags & DWC3_EP_DELAY_STOP) in __dwc3_gadget_ep_disable()
1093 dep->flags &= mask; in __dwc3_gadget_ep_disable()
1095 /* Clear out the ep descriptors for non-ep0 */ in __dwc3_gadget_ep_disable()
1096 if (dep->number > 1) { in __dwc3_gadget_ep_disable()
1097 dep->endpoint.comp_desc = NULL; in __dwc3_gadget_ep_disable()
1098 dep->endpoint.desc = NULL; in __dwc3_gadget_ep_disable()
1104 /* -------------------------------------------------------------------------- */
1109 return -EINVAL; in dwc3_gadget_ep0_enable()
1114 return -EINVAL; in dwc3_gadget_ep0_disable()
1117 /* -------------------------------------------------------------------------- */
1127 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) { in dwc3_gadget_ep_enable()
1129 return -EINVAL; in dwc3_gadget_ep_enable()
1132 if (!desc->wMaxPacketSize) { in dwc3_gadget_ep_enable()
1134 return -EINVAL; in dwc3_gadget_ep_enable()
1138 dwc = dep->dwc; in dwc3_gadget_ep_enable()
1140 if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED, in dwc3_gadget_ep_enable()
1142 dep->name)) in dwc3_gadget_ep_enable()
1145 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_ep_enable()
1147 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_ep_enable()
1161 return -EINVAL; in dwc3_gadget_ep_disable()
1165 dwc = dep->dwc; in dwc3_gadget_ep_disable()
1167 if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED), in dwc3_gadget_ep_disable()
1169 dep->name)) in dwc3_gadget_ep_disable()
1172 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_ep_disable()
1174 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_ep_disable()
1189 req->direction = dep->direction; in dwc3_gadget_ep_alloc_request()
1190 req->epnum = dep->number; in dwc3_gadget_ep_alloc_request()
1191 req->dep = dep; in dwc3_gadget_ep_alloc_request()
1192 req->status = DWC3_REQUEST_STATUS_UNKNOWN; in dwc3_gadget_ep_alloc_request()
1196 return &req->request; in dwc3_gadget_ep_alloc_request()
1209 * dwc3_ep_prev_trb - returns the previous TRB in the ring
1222 tmp = DWC3_TRB_NUM - 1; in dwc3_ep_prev_trb()
1224 return &dep->trb_pool[tmp - 1]; in dwc3_ep_prev_trb()
1233 * or empty. It's considered full when there are DWC3_TRB_NUM-1 of TRBs in dwc3_calc_trbs_left()
1236 if (dep->trb_enqueue == dep->trb_dequeue) { in dwc3_calc_trbs_left()
1243 req = next_request(&dep->started_list); in dwc3_calc_trbs_left()
1244 if (req && req->num_trbs) in dwc3_calc_trbs_left()
1247 return DWC3_TRB_NUM - 1; in dwc3_calc_trbs_left()
1250 trbs_left = dep->trb_dequeue - dep->trb_enqueue; in dwc3_calc_trbs_left()
1251 trbs_left &= (DWC3_TRB_NUM - 1); in dwc3_calc_trbs_left()
1253 if (dep->trb_dequeue < dep->trb_enqueue) in dwc3_calc_trbs_left()
1254 trbs_left--; in dwc3_calc_trbs_left()
1260 * dwc3_prepare_one_trb - setup one TRB from one request
1276 unsigned int stream_id = req->request.stream_id; in dwc3_prepare_one_trb()
1277 unsigned int short_not_ok = req->request.short_not_ok; in dwc3_prepare_one_trb()
1278 unsigned int no_interrupt = req->request.no_interrupt; in dwc3_prepare_one_trb()
1279 unsigned int is_last = req->request.is_last; in dwc3_prepare_one_trb()
1280 struct dwc3 *dwc = dep->dwc; in dwc3_prepare_one_trb()
1281 struct usb_gadget *gadget = dwc->gadget; in dwc3_prepare_one_trb()
1282 enum usb_device_speed speed = gadget->speed; in dwc3_prepare_one_trb()
1285 dma = dep->dwc->bounce_addr; in dwc3_prepare_one_trb()
1286 else if (req->request.num_sgs > 0) in dwc3_prepare_one_trb()
1287 dma = sg_dma_address(req->start_sg); in dwc3_prepare_one_trb()
1289 dma = req->request.dma; in dwc3_prepare_one_trb()
1291 trb = &dep->trb_pool[dep->trb_enqueue]; in dwc3_prepare_one_trb()
1293 if (!req->trb) { in dwc3_prepare_one_trb()
1295 req->trb = trb; in dwc3_prepare_one_trb()
1296 req->trb_dma = dwc3_trb_dma_offset(dep, trb); in dwc3_prepare_one_trb()
1299 req->num_trbs++; in dwc3_prepare_one_trb()
1301 trb->size = DWC3_TRB_SIZE_LENGTH(trb_length); in dwc3_prepare_one_trb()
1302 trb->bpl = lower_32_bits(dma); in dwc3_prepare_one_trb()
1303 trb->bph = upper_32_bits(dma); in dwc3_prepare_one_trb()
1305 switch (usb_endpoint_type(dep->endpoint.desc)) { in dwc3_prepare_one_trb()
1307 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP; in dwc3_prepare_one_trb()
1312 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST; in dwc3_prepare_one_trb()
1328 * - DATA0 in dwc3_prepare_one_trb()
1331 * - DATA1, DATA0 in dwc3_prepare_one_trb()
1334 * - DATA2, DATA1, DATA0 in dwc3_prepare_one_trb()
1337 struct usb_ep *ep = &dep->endpoint; in dwc3_prepare_one_trb()
1339 unsigned int maxp = usb_endpoint_maxp(ep->desc); in dwc3_prepare_one_trb()
1341 if (req->request.length <= (2 * maxp)) in dwc3_prepare_one_trb()
1342 mult--; in dwc3_prepare_one_trb()
1344 if (req->request.length <= maxp) in dwc3_prepare_one_trb()
1345 mult--; in dwc3_prepare_one_trb()
1347 trb->size |= DWC3_TRB_SIZE_PCM1(mult); in dwc3_prepare_one_trb()
1350 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS; in dwc3_prepare_one_trb()
1354 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI; in dwc3_prepare_one_trb()
1359 trb->ctrl = DWC3_TRBCTL_NORMAL; in dwc3_prepare_one_trb()
1366 dev_WARN(dwc->dev, "Unknown endpoint type %d\n", in dwc3_prepare_one_trb()
1367 usb_endpoint_type(dep->endpoint.desc)); in dwc3_prepare_one_trb()
1374 if (usb_endpoint_dir_out(dep->endpoint.desc)) { in dwc3_prepare_one_trb()
1375 if (!dep->stream_capable) in dwc3_prepare_one_trb()
1376 trb->ctrl |= DWC3_TRB_CTRL_CSP; in dwc3_prepare_one_trb()
1379 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI; in dwc3_prepare_one_trb()
1383 if (dep->stream_capable && DWC3_MST_CAPABLE(&dwc->hwparams)) in dwc3_prepare_one_trb()
1384 trb->ctrl |= DWC3_TRB_CTRL_CSP; in dwc3_prepare_one_trb()
1387 trb->ctrl |= DWC3_TRB_CTRL_IOC; in dwc3_prepare_one_trb()
1390 trb->ctrl |= DWC3_TRB_CTRL_CHN; in dwc3_prepare_one_trb()
1391 else if (dep->stream_capable && is_last && in dwc3_prepare_one_trb()
1392 !DWC3_MST_CAPABLE(&dwc->hwparams)) in dwc3_prepare_one_trb()
1393 trb->ctrl |= DWC3_TRB_CTRL_LST; in dwc3_prepare_one_trb()
1395 if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable) in dwc3_prepare_one_trb()
1396 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id); in dwc3_prepare_one_trb()
1406 * However there is a possibility of CPU re-ordering here which can cause in dwc3_prepare_one_trb()
1408 * Add a write memory barrier to prevent CPU re-ordering. in dwc3_prepare_one_trb()
1411 trb->ctrl |= DWC3_TRB_CTRL_HWO; in dwc3_prepare_one_trb()
1420 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc); in dwc3_needs_extra_trb()
1421 unsigned int rem = req->request.length % maxp; in dwc3_needs_extra_trb()
1423 if ((req->request.length && req->request.zero && !rem && in dwc3_needs_extra_trb()
1424 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) || in dwc3_needs_extra_trb()
1425 (!req->direction && rem)) in dwc3_needs_extra_trb()
1432 * dwc3_prepare_last_sg - prepare TRBs for the last SG entry
1444 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc); in dwc3_prepare_last_sg()
1445 unsigned int rem = req->request.length % maxp; in dwc3_prepare_last_sg()
1458 if (req->direction || req->request.length) in dwc3_prepare_last_sg()
1463 if ((!req->direction && !req->request.length) || needs_extra_trb) in dwc3_prepare_last_sg()
1465 req->direction ? 0 : maxp - rem, in dwc3_prepare_last_sg()
1474 struct scatterlist *sg = req->start_sg; in dwc3_prepare_trbs_sg()
1477 unsigned int length = req->request.length; in dwc3_prepare_trbs_sg()
1478 unsigned int remaining = req->num_pending_sgs; in dwc3_prepare_trbs_sg()
1479 unsigned int num_queued_sgs = req->request.num_mapped_sgs - remaining; in dwc3_prepare_trbs_sg()
1480 unsigned int num_trbs = req->num_trbs; in dwc3_prepare_trbs_sg()
1487 for_each_sg(req->request.sg, s, num_queued_sgs, i) in dwc3_prepare_trbs_sg()
1488 length -= sg_dma_len(s); in dwc3_prepare_trbs_sg()
1498 length -= trb_length; in dwc3_prepare_trbs_sg()
1507 if ((i == remaining - 1) || !length) in dwc3_prepare_trbs_sg()
1529 list_for_each_entry(r, &dep->started_list, list) { in dwc3_prepare_trbs_sg()
1530 if (r != req && !r->request.no_interrupt) in dwc3_prepare_trbs_sg()
1550 req->start_sg = sg_next(s); in dwc3_prepare_trbs_sg()
1552 req->num_pending_sgs--; in dwc3_prepare_trbs_sg()
1560 req->num_pending_sgs = 0; in dwc3_prepare_trbs_sg()
1568 return req->num_trbs - num_trbs; in dwc3_prepare_trbs_sg()
1574 return dwc3_prepare_last_sg(dep, req, req->request.length, 0); in dwc3_prepare_trbs_linear()
1578 * dwc3_prepare_trbs - setup TRBs from requests
1604 list_for_each_entry(req, &dep->started_list, list) { in dwc3_prepare_trbs()
1605 if (req->num_pending_sgs > 0) { in dwc3_prepare_trbs()
1607 if (!ret || req->num_pending_sgs) in dwc3_prepare_trbs()
1619 if (dep->stream_capable && req->request.is_last && in dwc3_prepare_trbs()
1620 !DWC3_MST_CAPABLE(&dep->dwc->hwparams)) in dwc3_prepare_trbs()
1624 list_for_each_entry_safe(req, n, &dep->pending_list, list) { in dwc3_prepare_trbs()
1625 struct dwc3 *dwc = dep->dwc; in dwc3_prepare_trbs()
1627 ret = usb_gadget_map_request_by_dev(dwc->sysdev, &req->request, in dwc3_prepare_trbs()
1628 dep->direction); in dwc3_prepare_trbs()
1632 req->start_sg = req->request.sg; in dwc3_prepare_trbs()
1633 req->num_pending_sgs = req->request.num_mapped_sgs; in dwc3_prepare_trbs()
1635 if (req->num_pending_sgs > 0) { in dwc3_prepare_trbs()
1637 if (req->num_pending_sgs) in dwc3_prepare_trbs()
1651 if (dep->stream_capable && req->request.is_last && in dwc3_prepare_trbs()
1652 !DWC3_MST_CAPABLE(&dwc->hwparams)) in dwc3_prepare_trbs()
1678 starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED); in __dwc3_gadget_kick_transfer()
1687 req = next_request(&dep->started_list); in __dwc3_gadget_kick_transfer()
1689 dep->flags |= DWC3_EP_PENDING_REQUEST; in __dwc3_gadget_kick_transfer()
1696 params.param0 = upper_32_bits(req->trb_dma); in __dwc3_gadget_kick_transfer()
1697 params.param1 = lower_32_bits(req->trb_dma); in __dwc3_gadget_kick_transfer()
1700 if (dep->stream_capable) in __dwc3_gadget_kick_transfer()
1701 cmd |= DWC3_DEPCMD_PARAM(req->request.stream_id); in __dwc3_gadget_kick_transfer()
1703 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) in __dwc3_gadget_kick_transfer()
1704 cmd |= DWC3_DEPCMD_PARAM(dep->frame_number); in __dwc3_gadget_kick_transfer()
1707 DWC3_DEPCMD_PARAM(dep->resource_index); in __dwc3_gadget_kick_transfer()
1714 if (ret == -EAGAIN) in __dwc3_gadget_kick_transfer()
1719 list_for_each_entry_safe(req, tmp, &dep->started_list, list) in __dwc3_gadget_kick_transfer()
1723 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING)) in __dwc3_gadget_kick_transfer()
1729 if (dep->stream_capable && req->request.is_last && in __dwc3_gadget_kick_transfer()
1730 !DWC3_MST_CAPABLE(&dep->dwc->hwparams)) in __dwc3_gadget_kick_transfer()
1731 dep->flags |= DWC3_EP_WAIT_TRANSFER_COMPLETE; in __dwc3_gadget_kick_transfer()
1740 reg = dwc3_readl(dwc->regs, DWC3_DSTS); in __dwc3_gadget_get_frame()
1745 * __dwc3_stop_active_transfer - stop the current active transfer
1764 cmd |= DWC3_DEPCMD_PARAM(dep->resource_index); in __dwc3_stop_active_transfer()
1773 if (ret == -ETIMEDOUT && dep->dwc->ep0state != EP0_SETUP_PHASE) { in __dwc3_stop_active_transfer()
1774 dep->flags |= DWC3_EP_DELAY_STOP; in __dwc3_stop_active_transfer()
1778 dep->resource_index = 0; in __dwc3_stop_active_transfer()
1781 dep->flags &= ~DWC3_EP_TRANSFER_STARTED; in __dwc3_stop_active_transfer()
1783 dep->flags |= DWC3_EP_END_TRANSFER_PENDING; in __dwc3_stop_active_transfer()
1785 dep->flags &= ~DWC3_EP_DELAY_STOP; in __dwc3_stop_active_transfer()
1790 * dwc3_gadget_start_isoc_quirk - workaround invalid frame number
1793 * This function tests for the correct combination of BIT[15:14] from the 16-bit
1797 * In DWC_usb31 version 1.70a-ea06 and prior, for highspeed and fullspeed
1798 * isochronous IN, BIT[15:14] of the 16-bit microframe number reported by the
1802 * internal 16-bit microframe, the START TRANSFER command will pass and the
1805 * other conditions, the START TRANSFER command will fail with bus-expiry.
1810 * (or 2 seconds). 4 seconds into the future will result in a bus-expiry status.
1813 * command status will result in a 2-second delay start. The smaller BIT[15:14]
1837 while (dep->combo_num < 2) { in dwc3_gadget_start_isoc_quirk()
1844 * 4 uframes in the future with BIT[15:14] as dep->combo_num in dwc3_gadget_start_isoc_quirk()
1846 test_frame_number = dep->frame_number & DWC3_FRNUMBER_MASK; in dwc3_gadget_start_isoc_quirk()
1847 test_frame_number |= dep->combo_num << 14; in dwc3_gadget_start_isoc_quirk()
1848 test_frame_number += max_t(u32, 4, dep->interval); in dwc3_gadget_start_isoc_quirk()
1850 params.param0 = upper_32_bits(dep->dwc->bounce_addr); in dwc3_gadget_start_isoc_quirk()
1851 params.param1 = lower_32_bits(dep->dwc->bounce_addr); in dwc3_gadget_start_isoc_quirk()
1857 /* Redo if some other failure beside bus-expiry is received */ in dwc3_gadget_start_isoc_quirk()
1858 if (cmd_status && cmd_status != -EAGAIN) { in dwc3_gadget_start_isoc_quirk()
1859 dep->start_cmd_status = 0; in dwc3_gadget_start_isoc_quirk()
1860 dep->combo_num = 0; in dwc3_gadget_start_isoc_quirk()
1865 if (dep->combo_num == 0) in dwc3_gadget_start_isoc_quirk()
1866 dep->start_cmd_status = cmd_status; in dwc3_gadget_start_isoc_quirk()
1868 dep->combo_num++; in dwc3_gadget_start_isoc_quirk()
1881 test0 = (dep->start_cmd_status == 0); in dwc3_gadget_start_isoc_quirk()
1885 dep->combo_num = 1; in dwc3_gadget_start_isoc_quirk()
1887 dep->combo_num = 2; in dwc3_gadget_start_isoc_quirk()
1889 dep->combo_num = 3; in dwc3_gadget_start_isoc_quirk()
1891 dep->combo_num = 0; in dwc3_gadget_start_isoc_quirk()
1893 dep->frame_number &= DWC3_FRNUMBER_MASK; in dwc3_gadget_start_isoc_quirk()
1894 dep->frame_number |= dep->combo_num << 14; in dwc3_gadget_start_isoc_quirk()
1895 dep->frame_number += max_t(u32, 4, dep->interval); in dwc3_gadget_start_isoc_quirk()
1898 dep->start_cmd_status = 0; in dwc3_gadget_start_isoc_quirk()
1899 dep->combo_num = 0; in dwc3_gadget_start_isoc_quirk()
1906 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc; in __dwc3_gadget_start_isoc()
1907 struct dwc3 *dwc = dep->dwc; in __dwc3_gadget_start_isoc()
1911 if (list_empty(&dep->pending_list) && in __dwc3_gadget_start_isoc()
1912 list_empty(&dep->started_list)) { in __dwc3_gadget_start_isoc()
1913 dep->flags |= DWC3_EP_PENDING_REQUEST; in __dwc3_gadget_start_isoc()
1914 return -EAGAIN; in __dwc3_gadget_start_isoc()
1917 if (!dwc->dis_start_transfer_quirk && in __dwc3_gadget_start_isoc()
1920 if (dwc->gadget->speed <= USB_SPEED_HIGH && dep->direction) in __dwc3_gadget_start_isoc()
1924 if (desc->bInterval <= 14 && in __dwc3_gadget_start_isoc()
1925 dwc->gadget->speed >= USB_SPEED_HIGH) { in __dwc3_gadget_start_isoc()
1928 (dep->frame_number & DWC3_FRNUMBER_MASK); in __dwc3_gadget_start_isoc()
1939 dep->frame_number = (dep->frame_number & ~DWC3_FRNUMBER_MASK) | in __dwc3_gadget_start_isoc()
1942 dep->frame_number += BIT(14); in __dwc3_gadget_start_isoc()
1949 if (desc->bInterval < 3) in __dwc3_gadget_start_isoc()
1950 future_interval += 3 - desc->bInterval; in __dwc3_gadget_start_isoc()
1952 dep->frame_number = DWC3_ALIGN_FRAME(dep, future_interval); in __dwc3_gadget_start_isoc()
1955 if (ret != -EAGAIN) in __dwc3_gadget_start_isoc()
1960 * After a number of unsuccessful start attempts due to bus-expiry in __dwc3_gadget_start_isoc()
1964 if (ret == -EAGAIN) in __dwc3_gadget_start_isoc()
1972 struct dwc3 *dwc = dep->dwc; in __dwc3_gadget_ep_queue()
1974 if (!dep->endpoint.desc || !dwc->pullups_connected || !dwc->connected) { in __dwc3_gadget_ep_queue()
1975 dev_dbg(dwc->dev, "%s: can't queue to disabled endpoint\n", in __dwc3_gadget_ep_queue()
1976 dep->name); in __dwc3_gadget_ep_queue()
1977 return -ESHUTDOWN; in __dwc3_gadget_ep_queue()
1980 if (WARN(req->dep != dep, "request %p belongs to '%s'\n", in __dwc3_gadget_ep_queue()
1981 &req->request, req->dep->name)) in __dwc3_gadget_ep_queue()
1982 return -EINVAL; in __dwc3_gadget_ep_queue()
1984 if (WARN(req->status < DWC3_REQUEST_STATUS_COMPLETED, in __dwc3_gadget_ep_queue()
1986 dep->name, &req->request)) in __dwc3_gadget_ep_queue()
1987 return -EINVAL; in __dwc3_gadget_ep_queue()
1989 pm_runtime_get(dwc->dev); in __dwc3_gadget_ep_queue()
1991 req->request.actual = 0; in __dwc3_gadget_ep_queue()
1992 req->request.status = -EINPROGRESS; in __dwc3_gadget_ep_queue()
1996 list_add_tail(&req->list, &dep->pending_list); in __dwc3_gadget_ep_queue()
1997 req->status = DWC3_REQUEST_STATUS_QUEUED; in __dwc3_gadget_ep_queue()
1999 if (dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE) in __dwc3_gadget_ep_queue()
2006 if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) || in __dwc3_gadget_ep_queue()
2007 (dep->flags & DWC3_EP_WEDGE) || in __dwc3_gadget_ep_queue()
2008 (dep->flags & DWC3_EP_DELAY_STOP) || in __dwc3_gadget_ep_queue()
2009 (dep->flags & DWC3_EP_STALL)) { in __dwc3_gadget_ep_queue()
2010 dep->flags |= DWC3_EP_DELAY_START; in __dwc3_gadget_ep_queue()
2017 * (micro-)frame number. in __dwc3_gadget_ep_queue()
2022 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) { in __dwc3_gadget_ep_queue()
2023 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) { in __dwc3_gadget_ep_queue()
2024 if ((dep->flags & DWC3_EP_PENDING_REQUEST)) in __dwc3_gadget_ep_queue()
2041 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_ep_queue()
2047 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_ep_queue()
2049 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_ep_queue()
2058 /* If req->trb is not set, then the request has not started */ in dwc3_gadget_ep_skip_trbs()
2059 if (!req->trb) in dwc3_gadget_ep_skip_trbs()
2072 for (i = 0; i < req->num_trbs; i++) { in dwc3_gadget_ep_skip_trbs()
2075 trb = &dep->trb_pool[dep->trb_dequeue]; in dwc3_gadget_ep_skip_trbs()
2076 trb->ctrl &= ~DWC3_TRB_CTRL_HWO; in dwc3_gadget_ep_skip_trbs()
2080 req->num_trbs = 0; in dwc3_gadget_ep_skip_trbs()
2086 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_ep_cleanup_cancelled_requests()
2088 while (!list_empty(&dep->cancelled_list)) { in dwc3_gadget_ep_cleanup_cancelled_requests()
2089 req = next_request(&dep->cancelled_list); in dwc3_gadget_ep_cleanup_cancelled_requests()
2091 switch (req->status) { in dwc3_gadget_ep_cleanup_cancelled_requests()
2093 dwc3_gadget_giveback(dep, req, -ESHUTDOWN); in dwc3_gadget_ep_cleanup_cancelled_requests()
2096 dwc3_gadget_giveback(dep, req, -ECONNRESET); in dwc3_gadget_ep_cleanup_cancelled_requests()
2099 dwc3_gadget_giveback(dep, req, -EPIPE); in dwc3_gadget_ep_cleanup_cancelled_requests()
2102 dev_err(dwc->dev, "request cancelled with wrong reason:%d\n", req->status); in dwc3_gadget_ep_cleanup_cancelled_requests()
2103 dwc3_gadget_giveback(dep, req, -ECONNRESET); in dwc3_gadget_ep_cleanup_cancelled_requests()
2110 if (!dep->endpoint.desc) in dwc3_gadget_ep_cleanup_cancelled_requests()
2122 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_ep_dequeue()
2129 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_ep_dequeue()
2131 list_for_each_entry(r, &dep->cancelled_list, list) { in dwc3_gadget_ep_dequeue()
2136 list_for_each_entry(r, &dep->pending_list, list) { in dwc3_gadget_ep_dequeue()
2145 if (dep->number > 1) in dwc3_gadget_ep_dequeue()
2146 dwc3_gadget_giveback(dep, req, -ECONNRESET); in dwc3_gadget_ep_dequeue()
2153 list_for_each_entry(r, &dep->started_list, list) { in dwc3_gadget_ep_dequeue()
2164 list_for_each_entry_safe(r, t, &dep->started_list, list) in dwc3_gadget_ep_dequeue()
2168 dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE; in dwc3_gadget_ep_dequeue()
2174 dev_err(dwc->dev, "request %p was not queued to %s\n", in dwc3_gadget_ep_dequeue()
2175 request, ep->name); in dwc3_gadget_ep_dequeue()
2176 ret = -EINVAL; in dwc3_gadget_ep_dequeue()
2178 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_ep_dequeue()
2186 struct dwc3 *dwc = dep->dwc; in __dwc3_gadget_ep_set_halt()
2191 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) { in __dwc3_gadget_ep_set_halt()
2192 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name); in __dwc3_gadget_ep_set_halt()
2193 return -EINVAL; in __dwc3_gadget_ep_set_halt()
2204 if (dep->number > 1) in __dwc3_gadget_ep_set_halt()
2205 trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue); in __dwc3_gadget_ep_set_halt()
2207 trb = &dwc->ep0_trb[dep->trb_enqueue]; in __dwc3_gadget_ep_set_halt()
2209 transfer_in_flight = trb->ctrl & DWC3_TRB_CTRL_HWO; in __dwc3_gadget_ep_set_halt()
2210 started = !list_empty(&dep->started_list); in __dwc3_gadget_ep_set_halt()
2212 if (!protocol && ((dep->direction && transfer_in_flight) || in __dwc3_gadget_ep_set_halt()
2213 (!dep->direction && started))) { in __dwc3_gadget_ep_set_halt()
2214 return -EAGAIN; in __dwc3_gadget_ep_set_halt()
2220 dev_err(dwc->dev, "failed to set STALL on %s\n", in __dwc3_gadget_ep_set_halt()
2221 dep->name); in __dwc3_gadget_ep_set_halt()
2223 dep->flags |= DWC3_EP_STALL; in __dwc3_gadget_ep_set_halt()
2230 if (dep->number <= 1) { in __dwc3_gadget_ep_set_halt()
2231 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE); in __dwc3_gadget_ep_set_halt()
2237 list_for_each_entry_safe(req, tmp, &dep->started_list, list) in __dwc3_gadget_ep_set_halt()
2240 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING || in __dwc3_gadget_ep_set_halt()
2241 (dep->flags & DWC3_EP_DELAY_STOP)) { in __dwc3_gadget_ep_set_halt()
2242 dep->flags |= DWC3_EP_PENDING_CLEAR_STALL; in __dwc3_gadget_ep_set_halt()
2244 dwc->clear_stall_protocol = dep->number; in __dwc3_gadget_ep_set_halt()
2253 dev_err(dwc->dev, "failed to clear STALL on %s\n", in __dwc3_gadget_ep_set_halt()
2254 dep->name); in __dwc3_gadget_ep_set_halt()
2258 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE); in __dwc3_gadget_ep_set_halt()
2260 if ((dep->flags & DWC3_EP_DELAY_START) && in __dwc3_gadget_ep_set_halt()
2261 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) in __dwc3_gadget_ep_set_halt()
2264 dep->flags &= ~DWC3_EP_DELAY_START; in __dwc3_gadget_ep_set_halt()
2273 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_ep_set_halt()
2279 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_ep_set_halt()
2281 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_ep_set_halt()
2289 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_ep_set_wedge()
2293 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_ep_set_wedge()
2294 dep->flags |= DWC3_EP_WEDGE; in dwc3_gadget_ep_set_wedge()
2296 if (dep->number == 0 || dep->number == 1) in dwc3_gadget_ep_set_wedge()
2300 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_ep_set_wedge()
2305 /* -------------------------------------------------------------------------- */
2335 /* -------------------------------------------------------------------------- */
2344 reg = dwc3_readl(dwc->regs, DWC3_DEVTEN); in dwc3_gadget_enable_linksts_evts()
2350 dwc3_writel(dwc->regs, DWC3_DEVTEN, reg); in dwc3_gadget_enable_linksts_evts()
2373 reg = dwc3_readl(dwc->regs, DWC3_DSTS); in __dwc3_gadget_wakeup()
2386 return -EINVAL; in __dwc3_gadget_wakeup()
2393 dev_err(dwc->dev, "failed to put link in Recovery\n"); in __dwc3_gadget_wakeup()
2401 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in __dwc3_gadget_wakeup()
2403 dwc3_writel(dwc->regs, DWC3_DCTL, reg); in __dwc3_gadget_wakeup()
2419 if (!dwc->wakeup_configured) { in dwc3_gadget_wakeup()
2420 dev_err(dwc->dev, "remote wakeup not configured\n"); in dwc3_gadget_wakeup()
2421 return -EINVAL; in dwc3_gadget_wakeup()
2424 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_wakeup()
2425 if (!dwc->gadget->wakeup_armed) { in dwc3_gadget_wakeup()
2426 dev_err(dwc->dev, "not armed for remote wakeup\n"); in dwc3_gadget_wakeup()
2427 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_wakeup()
2428 return -EINVAL; in dwc3_gadget_wakeup()
2432 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_wakeup()
2446 if (!dwc->wakeup_configured) { in dwc3_gadget_func_wakeup()
2447 dev_err(dwc->dev, "remote wakeup not configured\n"); in dwc3_gadget_func_wakeup()
2448 return -EINVAL; in dwc3_gadget_func_wakeup()
2451 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_func_wakeup()
2458 dwc->wakeup_pending_funcs |= BIT(intf_id); in dwc3_gadget_func_wakeup()
2460 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_func_wakeup()
2468 dev_err(dwc->dev, "function remote wakeup failed, ret:%d\n", ret); in dwc3_gadget_func_wakeup()
2470 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_func_wakeup()
2480 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_set_remote_wakeup()
2481 dwc->wakeup_configured = !!set; in dwc3_gadget_set_remote_wakeup()
2482 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_set_remote_wakeup()
2493 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_set_selfpowered()
2494 g->is_selfpowered = !!is_selfpowered; in dwc3_gadget_set_selfpowered()
2495 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_set_selfpowered()
2504 for (epnum = 2; epnum < dwc->num_eps; epnum++) { in dwc3_stop_active_transfers()
2507 dep = dwc->eps[epnum]; in dwc3_stop_active_transfers()
2511 dwc3_remove_requests(dwc, dep, -ESHUTDOWN); in dwc3_stop_active_transfers()
2517 enum usb_ssp_rate ssp_rate = dwc->gadget_ssp_rate; in __dwc3_gadget_set_ssp_rate()
2521 ssp_rate = dwc->max_ssp_rate; in __dwc3_gadget_set_ssp_rate()
2523 reg = dwc3_readl(dwc->regs, DWC3_DCFG); in __dwc3_gadget_set_ssp_rate()
2529 else if (dwc->max_ssp_rate != USB_SSP_GEN_1x2) in __dwc3_gadget_set_ssp_rate()
2533 dwc->max_ssp_rate != USB_SSP_GEN_2x1) in __dwc3_gadget_set_ssp_rate()
2536 dwc3_writel(dwc->regs, DWC3_DCFG, reg); in __dwc3_gadget_set_ssp_rate()
2544 speed = dwc->gadget_max_speed; in __dwc3_gadget_set_speed()
2545 if (speed == USB_SPEED_UNKNOWN || speed > dwc->maximum_speed) in __dwc3_gadget_set_speed()
2546 speed = dwc->maximum_speed; in __dwc3_gadget_set_speed()
2554 reg = dwc3_readl(dwc->regs, DWC3_DCFG); in __dwc3_gadget_set_speed()
2560 * bit if we try to force the IP to USB2-only mode. in __dwc3_gadget_set_speed()
2571 !dwc->dis_metastability_quirk) { in __dwc3_gadget_set_speed()
2591 dev_err(dwc->dev, "invalid speed (%d)\n", speed); in __dwc3_gadget_set_speed()
2605 dwc3_writel(dwc->regs, DWC3_DCFG, reg); in __dwc3_gadget_set_speed()
2614 if (pm_runtime_suspended(dwc->dev)) in dwc3_gadget_run_stop()
2630 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0)); in dwc3_gadget_run_stop()
2642 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg); in dwc3_gadget_run_stop()
2644 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in dwc3_gadget_run_stop()
2656 dwc->pullups_connected = true; in dwc3_gadget_run_stop()
2660 dwc->pullups_connected = false; in dwc3_gadget_run_stop()
2667 reg = dwc3_readl(dwc->regs, DWC3_DSTS); in dwc3_gadget_run_stop()
2669 } while (--timeout && !(!is_on ^ !reg)); in dwc3_gadget_run_stop()
2672 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0)); in dwc3_gadget_run_stop()
2674 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg); in dwc3_gadget_run_stop()
2678 return -ETIMEDOUT; in dwc3_gadget_run_stop()
2692 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_soft_disconnect()
2693 if (!dwc->pullups_connected) { in dwc3_gadget_soft_disconnect()
2694 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_soft_disconnect()
2698 dwc->connected = false; in dwc3_gadget_soft_disconnect()
2704 if (dwc->delayed_status) in dwc3_gadget_soft_disconnect()
2709 * Section 4.1.8 Table 4-7, it states that for a device-initiated in dwc3_gadget_soft_disconnect()
2715 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_soft_disconnect()
2724 if (dwc->ep0state != EP0_SETUP_PHASE) { in dwc3_gadget_soft_disconnect()
2725 reinit_completion(&dwc->ep0_in_setup); in dwc3_gadget_soft_disconnect()
2727 ret = wait_for_completion_timeout(&dwc->ep0_in_setup, in dwc3_gadget_soft_disconnect()
2730 dev_warn(dwc->dev, "wait for SETUP phase timed out\n"); in dwc3_gadget_soft_disconnect()
2731 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_soft_disconnect()
2733 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_soft_disconnect()
2752 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_soft_disconnect()
2754 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_soft_disconnect()
2756 usb_gadget_set_state(dwc->gadget, USB_STATE_NOTATTACHED); in dwc3_gadget_soft_disconnect()
2768 * device-initiated disconnect requires a core soft reset in dwc3_gadget_soft_connect()
2787 dwc->softconnect = is_on; in dwc3_gadget_pullup()
2795 pm_runtime_barrier(dwc->dev); in dwc3_gadget_pullup()
2796 if (pm_runtime_suspended(dwc->dev)) in dwc3_gadget_pullup()
2805 ret = pm_runtime_get_sync(dwc->dev); in dwc3_gadget_pullup()
2807 pm_runtime_put(dwc->dev); in dwc3_gadget_pullup()
2809 pm_runtime_set_suspended(dwc->dev); in dwc3_gadget_pullup()
2813 if (dwc->pullups_connected == is_on) { in dwc3_gadget_pullup()
2814 pm_runtime_put(dwc->dev); in dwc3_gadget_pullup()
2818 synchronize_irq(dwc->irq_gadget); in dwc3_gadget_pullup()
2825 pm_runtime_put(dwc->dev); in dwc3_gadget_pullup()
2846 /* On 2.30a and above this bit enables U3/L2-L1 Suspend Events */ in dwc3_gadget_enable_irq()
2850 dwc3_writel(dwc->regs, DWC3_DEVTEN, reg); in dwc3_gadget_enable_irq()
2856 dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00); in dwc3_gadget_disable_irq()
2863 * dwc3_gadget_setup_nump - calculate and initialize NUMP field of %DWC3_DCFG
2876 * RxFIFO Size = (RAM2_DEPTH * MDWIDTH / 8) - 24 - 16;
2890 ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7); in dwc3_gadget_setup_nump()
2893 nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024; in dwc3_gadget_setup_nump()
2897 reg = dwc3_readl(dwc->regs, DWC3_DCFG); in dwc3_gadget_setup_nump()
2900 dwc3_writel(dwc->regs, DWC3_DCFG, reg); in dwc3_gadget_setup_nump()
2910 * Use IMOD if enabled via dwc->imod_interval. Otherwise, if in __dwc3_gadget_start()
2913 if (dwc->imod_interval) { in __dwc3_gadget_start()
2914 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval); in __dwc3_gadget_start()
2915 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB); in __dwc3_gadget_start()
2917 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), 0); in __dwc3_gadget_start()
2927 reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG); in __dwc3_gadget_start()
2933 dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg); in __dwc3_gadget_start()
2944 reg = dwc3_readl(dwc->regs, DWC3_DCFG); in __dwc3_gadget_start()
2946 dwc3_writel(dwc->regs, DWC3_DCFG, reg); in __dwc3_gadget_start()
2949 if (DWC3_MST_CAPABLE(&dwc->hwparams)) { in __dwc3_gadget_start()
2950 reg = dwc3_readl(dwc->regs, DWC3_DCFG1); in __dwc3_gadget_start()
2952 dwc3_writel(dwc->regs, DWC3_DCFG1, reg); in __dwc3_gadget_start()
2960 dev_err(dwc->dev, "failed to config endpoints\n"); in __dwc3_gadget_start()
2964 dep = dwc->eps[0]; in __dwc3_gadget_start()
2965 dep->flags = 0; in __dwc3_gadget_start()
2968 dev_err(dwc->dev, "failed to enable %s\n", dep->name); in __dwc3_gadget_start()
2972 dep = dwc->eps[1]; in __dwc3_gadget_start()
2973 dep->flags = 0; in __dwc3_gadget_start()
2976 dev_err(dwc->dev, "failed to enable %s\n", dep->name); in __dwc3_gadget_start()
2981 dwc->ep0state = EP0_SETUP_PHASE; in __dwc3_gadget_start()
2982 dwc->ep0_bounced = false; in __dwc3_gadget_start()
2983 dwc->link_state = DWC3_LINK_STATE_SS_DIS; in __dwc3_gadget_start()
2984 dwc->delayed_status = false; in __dwc3_gadget_start()
2993 __dwc3_gadget_ep_disable(dwc->eps[0]); in __dwc3_gadget_start()
3007 irq = dwc->irq_gadget; in dwc3_gadget_start()
3009 IRQF_SHARED, "dwc3", dwc->ev_buf); in dwc3_gadget_start()
3011 dev_err(dwc->dev, "failed to request irq #%d --> %d\n", in dwc3_gadget_start()
3016 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_start()
3017 dwc->gadget_driver = driver; in dwc3_gadget_start()
3018 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_start()
3020 if (dwc->sys_wakeup) in dwc3_gadget_start()
3021 device_wakeup_enable(dwc->sysdev); in dwc3_gadget_start()
3029 __dwc3_gadget_ep_disable(dwc->eps[0]); in __dwc3_gadget_stop()
3030 __dwc3_gadget_ep_disable(dwc->eps[1]); in __dwc3_gadget_stop()
3038 if (dwc->sys_wakeup) in dwc3_gadget_stop()
3039 device_wakeup_disable(dwc->sysdev); in dwc3_gadget_stop()
3041 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_stop()
3042 dwc->gadget_driver = NULL; in dwc3_gadget_stop()
3043 dwc->max_cfg_eps = 0; in dwc3_gadget_stop()
3044 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_stop()
3046 free_irq(dwc->irq_gadget, dwc->ev_buf); in dwc3_gadget_stop()
3056 params->besl_baseline = USB_DEFAULT_BESL_UNSPECIFIED; in dwc3_gadget_config_params()
3057 params->besl_deep = USB_DEFAULT_BESL_UNSPECIFIED; in dwc3_gadget_config_params()
3060 if (!dwc->dis_enblslpm_quirk) { in dwc3_gadget_config_params()
3070 params->besl_baseline = 1; in dwc3_gadget_config_params()
3071 if (dwc->is_utmi_l1_suspend) in dwc3_gadget_config_params()
3072 params->besl_deep = in dwc3_gadget_config_params()
3073 clamp_t(u8, dwc->hird_threshold, 2, 15); in dwc3_gadget_config_params()
3077 if (dwc->dis_u1_entry_quirk) in dwc3_gadget_config_params()
3078 params->bU1devExitLat = 0; in dwc3_gadget_config_params()
3080 params->bU1devExitLat = DWC3_DEFAULT_U1_DEV_EXIT_LAT; in dwc3_gadget_config_params()
3083 if (dwc->dis_u2_entry_quirk) in dwc3_gadget_config_params()
3084 params->bU2DevExitLat = 0; in dwc3_gadget_config_params()
3086 params->bU2DevExitLat = in dwc3_gadget_config_params()
3096 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_set_speed()
3097 dwc->gadget_max_speed = speed; in dwc3_gadget_set_speed()
3098 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_set_speed()
3107 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_set_ssp_rate()
3108 dwc->gadget_max_speed = USB_SPEED_SUPER_PLUS; in dwc3_gadget_set_ssp_rate()
3109 dwc->gadget_ssp_rate = rate; in dwc3_gadget_set_ssp_rate()
3110 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_set_ssp_rate()
3119 if (dwc->usb2_phy) in dwc3_gadget_vbus_draw()
3120 return usb_phy_set_power(dwc->usb2_phy, mA); in dwc3_gadget_vbus_draw()
3122 if (!dwc->usb_psy) in dwc3_gadget_vbus_draw()
3123 return -EOPNOTSUPP; in dwc3_gadget_vbus_draw()
3126 ret = power_supply_set_property(dwc->usb_psy, POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT, &val); in dwc3_gadget_vbus_draw()
3132 * dwc3_gadget_check_config - ensure dwc3 can support the USB configuration
3149 if (!dwc->do_fifo_resize) in dwc3_gadget_check_config()
3152 list_for_each_entry(ep, &g->ep_list, ep_list) { in dwc3_gadget_check_config()
3154 if (ep->claimed && (ep->address & USB_DIR_IN)) in dwc3_gadget_check_config()
3158 if (ep_num <= dwc->max_cfg_eps) in dwc3_gadget_check_config()
3162 dwc->max_cfg_eps = ep_num; in dwc3_gadget_check_config()
3164 fifo_size = dwc3_gadget_calc_tx_fifo_size(dwc, dwc->max_cfg_eps); in dwc3_gadget_check_config()
3166 fifo_size += dwc->max_cfg_eps; in dwc3_gadget_check_config()
3171 return -ENOMEM; in dwc3_gadget_check_config()
3181 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_async_callbacks()
3182 dwc->async_callbacks = enable; in dwc3_gadget_async_callbacks()
3183 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_async_callbacks()
3203 /* -------------------------------------------------------------------------- */
3207 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_init_control_endpoint()
3209 usb_ep_set_maxpacket_limit(&dep->endpoint, 512); in dwc3_gadget_init_control_endpoint()
3210 dep->endpoint.maxburst = 1; in dwc3_gadget_init_control_endpoint()
3211 dep->endpoint.ops = &dwc3_gadget_ep0_ops; in dwc3_gadget_init_control_endpoint()
3212 if (!dep->direction) in dwc3_gadget_init_control_endpoint()
3213 dwc->gadget->ep0 = &dep->endpoint; in dwc3_gadget_init_control_endpoint()
3215 dep->endpoint.caps.type_control = true; in dwc3_gadget_init_control_endpoint()
3222 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_init_in_endpoint()
3232 size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1)); in dwc3_gadget_init_in_endpoint()
3243 * maxpacket = mdwidth * (fifo_size - 1); in dwc3_gadget_init_in_endpoint()
3247 * maxpacket = mdwidth * ((fifo_size - 1) - 1) - mdwidth; in dwc3_gadget_init_in_endpoint()
3250 maxpacket = mdwidth * (size - 1); in dwc3_gadget_init_in_endpoint()
3252 maxpacket = mdwidth * ((size - 1) - 1) - mdwidth; in dwc3_gadget_init_in_endpoint()
3256 usb_ep_set_maxpacket_limit(&dep->endpoint, size); in dwc3_gadget_init_in_endpoint()
3258 dep->endpoint.max_streams = 16; in dwc3_gadget_init_in_endpoint()
3259 dep->endpoint.ops = &dwc3_gadget_ep_ops; in dwc3_gadget_init_in_endpoint()
3260 list_add_tail(&dep->endpoint.ep_list, in dwc3_gadget_init_in_endpoint()
3261 &dwc->gadget->ep_list); in dwc3_gadget_init_in_endpoint()
3262 dep->endpoint.caps.type_iso = true; in dwc3_gadget_init_in_endpoint()
3263 dep->endpoint.caps.type_bulk = true; in dwc3_gadget_init_in_endpoint()
3264 dep->endpoint.caps.type_int = true; in dwc3_gadget_init_in_endpoint()
3271 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_init_out_endpoint()
3281 size = dwc3_readl(dwc->regs, DWC3_GRXFIFOSIZ(0)); in dwc3_gadget_init_out_endpoint()
3298 size -= (3 * 8) + 16; in dwc3_gadget_init_out_endpoint()
3304 usb_ep_set_maxpacket_limit(&dep->endpoint, size); in dwc3_gadget_init_out_endpoint()
3305 dep->endpoint.max_streams = 16; in dwc3_gadget_init_out_endpoint()
3306 dep->endpoint.ops = &dwc3_gadget_ep_ops; in dwc3_gadget_init_out_endpoint()
3307 list_add_tail(&dep->endpoint.ep_list, in dwc3_gadget_init_out_endpoint()
3308 &dwc->gadget->ep_list); in dwc3_gadget_init_out_endpoint()
3309 dep->endpoint.caps.type_iso = true; in dwc3_gadget_init_out_endpoint()
3310 dep->endpoint.caps.type_bulk = true; in dwc3_gadget_init_out_endpoint()
3311 dep->endpoint.caps.type_int = true; in dwc3_gadget_init_out_endpoint()
3320 struct dwc3 *dwc = dep->dwc; in dwc3_nostream_work()
3323 spin_lock_irqsave(&dwc->lock, flags); in dwc3_nostream_work()
3324 if (dep->flags & DWC3_EP_STREAM_PRIMED) in dwc3_nostream_work()
3327 if ((dep->flags & DWC3_EP_IGNORE_NEXT_NOSTREAM) || in dwc3_nostream_work()
3328 (!DWC3_MST_CAPABLE(&dwc->hwparams) && in dwc3_nostream_work()
3329 !(dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE))) in dwc3_nostream_work()
3348 dwc3_send_gadget_generic_command(dwc, cmd, dep->number); in dwc3_nostream_work()
3350 dep->flags |= DWC3_EP_DELAY_START; in dwc3_nostream_work()
3352 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_nostream_work()
3356 dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM; in dwc3_nostream_work()
3357 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_nostream_work()
3369 return -ENOMEM; in dwc3_gadget_init_endpoint()
3371 dep->dwc = dwc; in dwc3_gadget_init_endpoint()
3372 dep->number = epnum; in dwc3_gadget_init_endpoint()
3373 dep->direction = direction; in dwc3_gadget_init_endpoint()
3374 dep->regs = dwc->regs + DWC3_DEP_BASE(epnum); in dwc3_gadget_init_endpoint()
3375 dwc->eps[epnum] = dep; in dwc3_gadget_init_endpoint()
3376 dep->combo_num = 0; in dwc3_gadget_init_endpoint()
3377 dep->start_cmd_status = 0; in dwc3_gadget_init_endpoint()
3379 snprintf(dep->name, sizeof(dep->name), "ep%u%s", num, in dwc3_gadget_init_endpoint()
3382 dep->endpoint.name = dep->name; in dwc3_gadget_init_endpoint()
3384 if (!(dep->number > 1)) { in dwc3_gadget_init_endpoint()
3385 dep->endpoint.desc = &dwc3_gadget_ep0_desc; in dwc3_gadget_init_endpoint()
3386 dep->endpoint.comp_desc = NULL; in dwc3_gadget_init_endpoint()
3399 dep->endpoint.caps.dir_in = direction; in dwc3_gadget_init_endpoint()
3400 dep->endpoint.caps.dir_out = !direction; in dwc3_gadget_init_endpoint()
3402 INIT_LIST_HEAD(&dep->pending_list); in dwc3_gadget_init_endpoint()
3403 INIT_LIST_HEAD(&dep->started_list); in dwc3_gadget_init_endpoint()
3404 INIT_LIST_HEAD(&dep->cancelled_list); in dwc3_gadget_init_endpoint()
3405 INIT_DELAYED_WORK(&dep->nostream_work, dwc3_nostream_work); in dwc3_gadget_init_endpoint()
3418 if (!device_property_present(dwc->dev, propname)) in dwc3_gadget_get_reserved_endpoints()
3421 ret = device_property_count_u8(dwc->dev, propname); in dwc3_gadget_get_reserved_endpoints()
3426 ret = device_property_read_u8_array(dwc->dev, propname, eps, min(num, count)); in dwc3_gadget_get_reserved_endpoints()
3435 const char *propname = "snps,reserved-endpoints"; in dwc3_gadget_init_endpoints()
3442 INIT_LIST_HEAD(&dwc->gadget->ep_list); in dwc3_gadget_init_endpoints()
3447 dev_err(dwc->dev, "failed to read %s\n", propname); in dwc3_gadget_init_endpoints()
3474 dep = dwc->eps[epnum]; in dwc3_gadget_free_endpoints()
3479 * bi-directional USB endpoint 0. in dwc3_gadget_free_endpoints()
3488 list_del(&dep->endpoint.ep_list); in dwc3_gadget_free_endpoints()
3496 /* -------------------------------------------------------------------------- */
3507 req->num_trbs--; in dwc3_gadget_ep_reclaim_completed_trb()
3519 if (chain && (trb->ctrl & DWC3_TRB_CTRL_HWO)) in dwc3_gadget_ep_reclaim_completed_trb()
3520 trb->ctrl &= ~DWC3_TRB_CTRL_HWO; in dwc3_gadget_ep_reclaim_completed_trb()
3524 * have the Isoc-First type. Track and report its interval frame number. in dwc3_gadget_ep_reclaim_completed_trb()
3526 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && in dwc3_gadget_ep_reclaim_completed_trb()
3527 (trb->ctrl & DWC3_TRBCTL_ISOCHRONOUS_FIRST)) { in dwc3_gadget_ep_reclaim_completed_trb()
3530 frame_number = DWC3_TRB_CTRL_GET_SID_SOFN(trb->ctrl); in dwc3_gadget_ep_reclaim_completed_trb()
3531 frame_number &= ~(dep->interval - 1); in dwc3_gadget_ep_reclaim_completed_trb()
3532 req->request.frame_number = frame_number; in dwc3_gadget_ep_reclaim_completed_trb()
3538 * TRB. Don't add it to req->remaining calculation. in dwc3_gadget_ep_reclaim_completed_trb()
3540 if (trb->bpl == lower_32_bits(dep->dwc->bounce_addr) && in dwc3_gadget_ep_reclaim_completed_trb()
3541 trb->bph == upper_32_bits(dep->dwc->bounce_addr)) { in dwc3_gadget_ep_reclaim_completed_trb()
3542 trb->ctrl &= ~DWC3_TRB_CTRL_HWO; in dwc3_gadget_ep_reclaim_completed_trb()
3546 count = trb->size & DWC3_TRB_SIZE_MASK; in dwc3_gadget_ep_reclaim_completed_trb()
3547 req->remaining += count; in dwc3_gadget_ep_reclaim_completed_trb()
3549 if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN) in dwc3_gadget_ep_reclaim_completed_trb()
3552 if (event->status & DEPEVT_STATUS_SHORT && !chain) in dwc3_gadget_ep_reclaim_completed_trb()
3555 if ((trb->ctrl & DWC3_TRB_CTRL_ISP_IMI) && in dwc3_gadget_ep_reclaim_completed_trb()
3556 DWC3_TRB_SIZE_TRBSTS(trb->size) == DWC3_TRBSTS_MISSED_ISOC) in dwc3_gadget_ep_reclaim_completed_trb()
3559 if ((trb->ctrl & DWC3_TRB_CTRL_IOC) || in dwc3_gadget_ep_reclaim_completed_trb()
3560 (trb->ctrl & DWC3_TRB_CTRL_LST)) in dwc3_gadget_ep_reclaim_completed_trb()
3571 unsigned int num_completed_trbs = req->num_trbs; in dwc3_gadget_ep_reclaim_trb_sg()
3576 trb = &dep->trb_pool[dep->trb_dequeue]; in dwc3_gadget_ep_reclaim_trb_sg()
3580 !!(trb->ctrl & DWC3_TRB_CTRL_CHN)); in dwc3_gadget_ep_reclaim_trb_sg()
3590 return req->num_pending_sgs == 0 && req->num_trbs == 0; in dwc3_gadget_ep_request_completed()
3602 req->request.actual = req->request.length - req->remaining; in dwc3_gadget_ep_cleanup_completed_request()
3613 if (req->request.no_interrupt) { in dwc3_gadget_ep_cleanup_completed_request()
3616 trb = dwc3_ep_prev_trb(dep, dep->trb_dequeue); in dwc3_gadget_ep_cleanup_completed_request()
3617 switch (DWC3_TRB_SIZE_TRBSTS(trb->size)) { in dwc3_gadget_ep_cleanup_completed_request()
3620 request_status = -EXDEV; in dwc3_gadget_ep_cleanup_completed_request()
3646 while (!list_empty(&dep->started_list)) { in dwc3_gadget_ep_cleanup_completed_requests()
3649 req = next_request(&dep->started_list); in dwc3_gadget_ep_cleanup_completed_requests()
3658 if (!dep->endpoint.desc) in dwc3_gadget_ep_cleanup_completed_requests()
3666 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_ep_should_continue()
3668 if (!dep->endpoint.desc || !dwc->pullups_connected || in dwc3_gadget_ep_should_continue()
3669 !dwc->connected) in dwc3_gadget_ep_should_continue()
3672 if (!list_empty(&dep->pending_list)) in dwc3_gadget_ep_should_continue()
3679 req = next_request(&dep->started_list); in dwc3_gadget_ep_should_continue()
3689 dep->frame_number = event->parameters; in dwc3_gadget_endpoint_frame_from_event()
3695 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_endpoint_trbs_complete()
3700 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING) in dwc3_gadget_endpoint_trbs_complete()
3703 if (!dep->endpoint.desc) in dwc3_gadget_endpoint_trbs_complete()
3706 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && in dwc3_gadget_endpoint_trbs_complete()
3707 list_empty(&dep->started_list) && in dwc3_gadget_endpoint_trbs_complete()
3708 (list_empty(&dep->pending_list) || status == -EXDEV)) in dwc3_gadget_endpoint_trbs_complete()
3716 * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround. in dwc3_gadget_endpoint_trbs_complete()
3724 dep = dwc->eps[i]; in dwc3_gadget_endpoint_trbs_complete()
3728 if (!(dep->flags & DWC3_EP_ENABLED)) in dwc3_gadget_endpoint_trbs_complete()
3731 if (!list_empty(&dep->started_list)) in dwc3_gadget_endpoint_trbs_complete()
3735 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in dwc3_gadget_endpoint_trbs_complete()
3736 reg |= dwc->u1u2; in dwc3_gadget_endpoint_trbs_complete()
3737 dwc3_writel(dwc->regs, DWC3_DCTL, reg); in dwc3_gadget_endpoint_trbs_complete()
3739 dwc->u1u2 = 0; in dwc3_gadget_endpoint_trbs_complete()
3750 if (!dep->endpoint.desc) in dwc3_gadget_endpoint_transfer_in_progress()
3753 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) in dwc3_gadget_endpoint_transfer_in_progress()
3756 if (event->status & DEPEVT_STATUS_BUSERR) in dwc3_gadget_endpoint_transfer_in_progress()
3757 status = -ECONNRESET; in dwc3_gadget_endpoint_transfer_in_progress()
3759 if (event->status & DEPEVT_STATUS_MISSED_ISOC) in dwc3_gadget_endpoint_transfer_in_progress()
3760 status = -EXDEV; in dwc3_gadget_endpoint_transfer_in_progress()
3770 dep->flags &= ~DWC3_EP_TRANSFER_STARTED; in dwc3_gadget_endpoint_transfer_complete()
3772 if (event->status & DEPEVT_STATUS_BUSERR) in dwc3_gadget_endpoint_transfer_complete()
3773 status = -ECONNRESET; in dwc3_gadget_endpoint_transfer_complete()
3776 dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE; in dwc3_gadget_endpoint_transfer_complete()
3792 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING) in dwc3_gadget_endpoint_transfer_not_ready()
3801 u8 cmd = DEPEVT_PARAMETER_CMD(event->parameters); in dwc3_gadget_endpoint_command_complete()
3811 if (dep->stream_capable) in dwc3_gadget_endpoint_command_complete()
3812 dep->flags |= DWC3_EP_IGNORE_NEXT_NOSTREAM; in dwc3_gadget_endpoint_command_complete()
3814 dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING; in dwc3_gadget_endpoint_command_complete()
3815 dep->flags &= ~DWC3_EP_TRANSFER_STARTED; in dwc3_gadget_endpoint_command_complete()
3818 if (dep->flags & DWC3_EP_PENDING_CLEAR_STALL) { in dwc3_gadget_endpoint_command_complete()
3819 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_endpoint_command_complete()
3821 dep->flags &= ~DWC3_EP_PENDING_CLEAR_STALL; in dwc3_gadget_endpoint_command_complete()
3823 struct usb_ep *ep0 = &dwc->eps[0]->endpoint; in dwc3_gadget_endpoint_command_complete()
3825 dev_err(dwc->dev, "failed to clear STALL on %s\n", dep->name); in dwc3_gadget_endpoint_command_complete()
3826 if (dwc->delayed_status) in dwc3_gadget_endpoint_command_complete()
3831 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE); in dwc3_gadget_endpoint_command_complete()
3832 if (dwc->clear_stall_protocol == dep->number) in dwc3_gadget_endpoint_command_complete()
3836 if ((dep->flags & DWC3_EP_DELAY_START) && in dwc3_gadget_endpoint_command_complete()
3837 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) in dwc3_gadget_endpoint_command_complete()
3840 dep->flags &= ~DWC3_EP_DELAY_START; in dwc3_gadget_endpoint_command_complete()
3846 if (event->status == DEPEVT_STREAMEVT_FOUND) { in dwc3_gadget_endpoint_stream_event()
3847 cancel_delayed_work(&dep->nostream_work); in dwc3_gadget_endpoint_stream_event()
3848 dep->flags |= DWC3_EP_STREAM_PRIMED; in dwc3_gadget_endpoint_stream_event()
3849 dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM; in dwc3_gadget_endpoint_stream_event()
3854 switch (event->parameters) { in dwc3_gadget_endpoint_stream_event()
3856 cancel_delayed_work(&dep->nostream_work); in dwc3_gadget_endpoint_stream_event()
3857 dep->flags |= DWC3_EP_STREAM_PRIMED; in dwc3_gadget_endpoint_stream_event()
3858 dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM; in dwc3_gadget_endpoint_stream_event()
3861 dep->flags &= ~DWC3_EP_STREAM_PRIMED; in dwc3_gadget_endpoint_stream_event()
3862 if (dep->flags & DWC3_EP_FORCE_RESTART_STREAM) in dwc3_gadget_endpoint_stream_event()
3863 queue_delayed_work(system_wq, &dep->nostream_work, in dwc3_gadget_endpoint_stream_event()
3873 u8 epnum = event->endpoint_number; in dwc3_endpoint_interrupt()
3875 dep = dwc->eps[epnum]; in dwc3_endpoint_interrupt()
3877 dev_warn(dwc->dev, "spurious event, endpoint %u is not allocated\n", epnum); in dwc3_endpoint_interrupt()
3881 if (!(dep->flags & DWC3_EP_ENABLED)) { in dwc3_endpoint_interrupt()
3882 if ((epnum > 1) && !(dep->flags & DWC3_EP_TRANSFER_STARTED)) in dwc3_endpoint_interrupt()
3886 if ((event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT) && in dwc3_endpoint_interrupt()
3887 !(epnum <= 1 && event->endpoint_event == DWC3_DEPEVT_XFERCOMPLETE)) in dwc3_endpoint_interrupt()
3896 switch (event->endpoint_event) { in dwc3_endpoint_interrupt()
3915 dev_err(dwc->dev, "unknown endpoint event %d\n", event->endpoint_event); in dwc3_endpoint_interrupt()
3922 if (dwc->async_callbacks && dwc->gadget_driver->disconnect) { in dwc3_disconnect_gadget()
3923 spin_unlock(&dwc->lock); in dwc3_disconnect_gadget()
3924 dwc->gadget_driver->disconnect(dwc->gadget); in dwc3_disconnect_gadget()
3925 spin_lock(&dwc->lock); in dwc3_disconnect_gadget()
3931 if (dwc->async_callbacks && dwc->gadget_driver->suspend) { in dwc3_suspend_gadget()
3932 spin_unlock(&dwc->lock); in dwc3_suspend_gadget()
3933 dwc->gadget_driver->suspend(dwc->gadget); in dwc3_suspend_gadget()
3934 spin_lock(&dwc->lock); in dwc3_suspend_gadget()
3940 if (dwc->async_callbacks && dwc->gadget_driver->resume) { in dwc3_resume_gadget()
3941 spin_unlock(&dwc->lock); in dwc3_resume_gadget()
3942 dwc->gadget_driver->resume(dwc->gadget); in dwc3_resume_gadget()
3943 spin_lock(&dwc->lock); in dwc3_resume_gadget()
3949 if (!dwc->gadget_driver) in dwc3_reset_gadget()
3952 if (dwc->async_callbacks && dwc->gadget->speed != USB_SPEED_UNKNOWN) { in dwc3_reset_gadget()
3953 spin_unlock(&dwc->lock); in dwc3_reset_gadget()
3954 usb_gadget_udc_reset(dwc->gadget, dwc->gadget_driver); in dwc3_reset_gadget()
3955 spin_lock(&dwc->lock); in dwc3_reset_gadget()
3962 struct dwc3 *dwc = dep->dwc; in dwc3_stop_active_transfer()
3970 if (dep->number <= 1 && dwc->ep0state != EP0_DATA_PHASE) in dwc3_stop_active_transfer()
3973 if (interrupt && (dep->flags & DWC3_EP_DELAY_STOP)) in dwc3_stop_active_transfer()
3976 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED) || in dwc3_stop_active_transfer()
3977 (dep->flags & DWC3_EP_END_TRANSFER_PENDING)) in dwc3_stop_active_transfer()
3987 if (dwc->ep0state != EP0_SETUP_PHASE && !dwc->delayed_status) { in dwc3_stop_active_transfer()
3988 dep->flags |= DWC3_EP_DELAY_STOP; in dwc3_stop_active_transfer()
4034 dep = dwc->eps[epnum]; in dwc3_clear_stall_all_ep()
4038 if (!(dep->flags & DWC3_EP_STALL)) in dwc3_clear_stall_all_ep()
4041 dep->flags &= ~DWC3_EP_STALL; in dwc3_clear_stall_all_ep()
4052 dwc->suspended = false; in dwc3_gadget_disconnect_interrupt()
4056 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in dwc3_gadget_disconnect_interrupt()
4061 dwc->connected = false; in dwc3_gadget_disconnect_interrupt()
4065 dwc->gadget->speed = USB_SPEED_UNKNOWN; in dwc3_gadget_disconnect_interrupt()
4066 dwc->setup_packet_pending = false; in dwc3_gadget_disconnect_interrupt()
4067 dwc->gadget->wakeup_armed = false; in dwc3_gadget_disconnect_interrupt()
4069 usb_gadget_set_state(dwc->gadget, USB_STATE_NOTATTACHED); in dwc3_gadget_disconnect_interrupt()
4076 * interrupt to set dwc->connected to FALSE. in dwc3_gadget_disconnect_interrupt()
4078 pm_request_idle(dwc->dev); in dwc3_gadget_disconnect_interrupt()
4085 dwc->suspended = false; in dwc3_gadget_reset_interrupt()
4094 dwc->connected = false; in dwc3_gadget_reset_interrupt()
4123 if (dwc->setup_packet_pending) in dwc3_gadget_reset_interrupt()
4138 * Section 4.1.2 Table 4-2, it states that during a USB reset, the SW in dwc3_gadget_reset_interrupt()
4143 dwc->connected = true; in dwc3_gadget_reset_interrupt()
4145 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in dwc3_gadget_reset_interrupt()
4148 dwc->test_mode = false; in dwc3_gadget_reset_interrupt()
4149 dwc->gadget->wakeup_armed = false; in dwc3_gadget_reset_interrupt()
4154 reg = dwc3_readl(dwc->regs, DWC3_DCFG); in dwc3_gadget_reset_interrupt()
4156 dwc3_writel(dwc->regs, DWC3_DCFG, reg); in dwc3_gadget_reset_interrupt()
4167 if (!dwc->softconnect) in dwc3_gadget_conndone_interrupt()
4170 reg = dwc3_readl(dwc->regs, DWC3_DSTS); in dwc3_gadget_conndone_interrupt()
4172 dwc->speed = speed; in dwc3_gadget_conndone_interrupt()
4177 dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN; in dwc3_gadget_conndone_interrupt()
4191 dwc->gadget->ep0->maxpacket = 512; in dwc3_gadget_conndone_interrupt()
4192 dwc->gadget->speed = USB_SPEED_SUPER_PLUS; in dwc3_gadget_conndone_interrupt()
4195 dwc->gadget->ssp_rate = USB_SSP_GEN_2x2; in dwc3_gadget_conndone_interrupt()
4197 dwc->gadget->ssp_rate = USB_SSP_GEN_2x1; in dwc3_gadget_conndone_interrupt()
4217 dwc->gadget->ep0->maxpacket = 512; in dwc3_gadget_conndone_interrupt()
4218 dwc->gadget->speed = USB_SPEED_SUPER; in dwc3_gadget_conndone_interrupt()
4221 dwc->gadget->speed = USB_SPEED_SUPER_PLUS; in dwc3_gadget_conndone_interrupt()
4222 dwc->gadget->ssp_rate = USB_SSP_GEN_1x2; in dwc3_gadget_conndone_interrupt()
4227 dwc->gadget->ep0->maxpacket = 64; in dwc3_gadget_conndone_interrupt()
4228 dwc->gadget->speed = USB_SPEED_HIGH; in dwc3_gadget_conndone_interrupt()
4232 dwc->gadget->ep0->maxpacket = 64; in dwc3_gadget_conndone_interrupt()
4233 dwc->gadget->speed = USB_SPEED_FULL; in dwc3_gadget_conndone_interrupt()
4237 dwc->eps[1]->endpoint.maxpacket = dwc->gadget->ep0->maxpacket; in dwc3_gadget_conndone_interrupt()
4239 /* Enable USB2 LPM Capability */ in dwc3_gadget_conndone_interrupt()
4242 !dwc->usb2_gadget_lpm_disable && in dwc3_gadget_conndone_interrupt()
4245 reg = dwc3_readl(dwc->regs, DWC3_DCFG); in dwc3_gadget_conndone_interrupt()
4247 dwc3_writel(dwc->regs, DWC3_DCFG, reg); in dwc3_gadget_conndone_interrupt()
4249 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in dwc3_gadget_conndone_interrupt()
4252 reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold | in dwc3_gadget_conndone_interrupt()
4253 (dwc->is_utmi_l1_suspend << 4)); in dwc3_gadget_conndone_interrupt()
4256 * When dwc3 revisions >= 2.40a, LPM Erratum is enabled and in dwc3_gadget_conndone_interrupt()
4258 * BESL value in the LPM token is less than or equal to LPM in dwc3_gadget_conndone_interrupt()
4259 * NYET threshold. in dwc3_gadget_conndone_interrupt()
4261 WARN_ONCE(DWC3_VER_IS_PRIOR(DWC3, 240A) && dwc->has_lpm_erratum, in dwc3_gadget_conndone_interrupt()
4262 "LPM Erratum not available on dwc3 revisions < 2.40a\n"); in dwc3_gadget_conndone_interrupt()
4264 if (dwc->has_lpm_erratum && !DWC3_VER_IS_PRIOR(DWC3, 240A)) { in dwc3_gadget_conndone_interrupt()
4266 reg |= DWC3_DCTL_NYET_THRES(dwc->lpm_nyet_threshold); in dwc3_gadget_conndone_interrupt()
4271 if (dwc->usb2_gadget_lpm_disable) { in dwc3_gadget_conndone_interrupt()
4272 reg = dwc3_readl(dwc->regs, DWC3_DCFG); in dwc3_gadget_conndone_interrupt()
4274 dwc3_writel(dwc->regs, DWC3_DCFG, reg); in dwc3_gadget_conndone_interrupt()
4277 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in dwc3_gadget_conndone_interrupt()
4282 dep = dwc->eps[0]; in dwc3_gadget_conndone_interrupt()
4285 dev_err(dwc->dev, "failed to enable %s\n", dep->name); in dwc3_gadget_conndone_interrupt()
4289 dep = dwc->eps[1]; in dwc3_gadget_conndone_interrupt()
4292 dev_err(dwc->dev, "failed to enable %s\n", dep->name); in dwc3_gadget_conndone_interrupt()
4307 dwc->suspended = false; in dwc3_gadget_wakeup_interrupt()
4314 if (dwc->async_callbacks && dwc->gadget_driver->resume) { in dwc3_gadget_wakeup_interrupt()
4315 spin_unlock(&dwc->lock); in dwc3_gadget_wakeup_interrupt()
4316 dwc->gadget_driver->resume(dwc->gadget); in dwc3_gadget_wakeup_interrupt()
4317 spin_lock(&dwc->lock); in dwc3_gadget_wakeup_interrupt()
4320 dwc->link_state = evtinfo & DWC3_LINK_STATE_MASK; in dwc3_gadget_wakeup_interrupt()
4334 * host-initiated U3 exit. in dwc3_gadget_linksts_change_interrupt()
4345 * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation in dwc3_gadget_linksts_change_interrupt()
4348 pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1); in dwc3_gadget_linksts_change_interrupt()
4351 if ((dwc->link_state == DWC3_LINK_STATE_U3) && in dwc3_gadget_linksts_change_interrupt()
4372 * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us in dwc3_gadget_linksts_change_interrupt()
4380 switch (dwc->link_state) { in dwc3_gadget_linksts_change_interrupt()
4383 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in dwc3_gadget_linksts_change_interrupt()
4389 if (!dwc->u1u2) in dwc3_gadget_linksts_change_interrupt()
4390 dwc->u1u2 = reg & u1u2; in dwc3_gadget_linksts_change_interrupt()
4405 if (dwc->gadget->wakeup_armed || dwc->wakeup_pending_funcs) { in dwc3_gadget_linksts_change_interrupt()
4408 dwc->suspended = false; in dwc3_gadget_linksts_change_interrupt()
4412 if (dwc->speed == USB_SPEED_SUPER) in dwc3_gadget_linksts_change_interrupt()
4427 dwc->link_state = next; in dwc3_gadget_linksts_change_interrupt()
4430 while (dwc->wakeup_pending_funcs && (next == DWC3_LINK_STATE_U0)) { in dwc3_gadget_linksts_change_interrupt()
4431 intf_id = ffs(dwc->wakeup_pending_funcs) - 1; in dwc3_gadget_linksts_change_interrupt()
4436 dev_err(dwc->dev, "Failed to send DN wake for intf %d\n", intf_id); in dwc3_gadget_linksts_change_interrupt()
4438 dwc->wakeup_pending_funcs &= ~BIT(intf_id); in dwc3_gadget_linksts_change_interrupt()
4447 if (!dwc->suspended && next == DWC3_LINK_STATE_U3) { in dwc3_gadget_suspend_interrupt()
4448 dwc->suspended = true; in dwc3_gadget_suspend_interrupt()
4452 dwc->link_state = next; in dwc3_gadget_suspend_interrupt()
4458 switch (event->type) { in dwc3_gadget_interrupt()
4469 dwc3_gadget_wakeup_interrupt(dwc, event->event_info); in dwc3_gadget_interrupt()
4472 dev_WARN_ONCE(dwc->dev, true, "unexpected hibernation event\n"); in dwc3_gadget_interrupt()
4475 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info); in dwc3_gadget_interrupt()
4480 dwc3_gadget_suspend_interrupt(dwc, event->event_info); in dwc3_gadget_interrupt()
4488 dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type); in dwc3_gadget_interrupt()
4495 trace_dwc3_event(event->raw, dwc); in dwc3_process_event_entry()
4497 if (!event->type.is_devspec) in dwc3_process_event_entry()
4498 dwc3_endpoint_interrupt(dwc, &event->depevt); in dwc3_process_event_entry()
4499 else if (event->type.type == DWC3_EVENT_TYPE_DEV) in dwc3_process_event_entry()
4500 dwc3_gadget_interrupt(dwc, &event->devt); in dwc3_process_event_entry()
4502 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw); in dwc3_process_event_entry()
4507 struct dwc3 *dwc = evt->dwc; in dwc3_process_event_buf()
4511 left = evt->count; in dwc3_process_event_buf()
4513 if (!(evt->flags & DWC3_EVENT_PENDING)) in dwc3_process_event_buf()
4519 event.raw = *(u32 *) (evt->cache + evt->lpos); in dwc3_process_event_buf()
4532 evt->lpos = (evt->lpos + 4) % evt->length; in dwc3_process_event_buf()
4533 left -= 4; in dwc3_process_event_buf()
4536 evt->count = 0; in dwc3_process_event_buf()
4540 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), in dwc3_process_event_buf()
4541 DWC3_GEVNTSIZ_SIZE(evt->length)); in dwc3_process_event_buf()
4543 evt->flags &= ~DWC3_EVENT_PENDING; in dwc3_process_event_buf()
4550 if (dwc->imod_interval) { in dwc3_process_event_buf()
4551 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB); in dwc3_process_event_buf()
4552 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval); in dwc3_process_event_buf()
4561 struct dwc3 *dwc = evt->dwc; in dwc3_thread_interrupt()
4566 spin_lock_irqsave(&dwc->lock, flags); in dwc3_thread_interrupt()
4568 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_thread_interrupt()
4576 struct dwc3 *dwc = evt->dwc; in dwc3_check_event_buf()
4580 if (pm_runtime_suspended(dwc->dev)) { in dwc3_check_event_buf()
4581 dwc->pending_events = true; in dwc3_check_event_buf()
4587 pm_runtime_get(dwc->dev); in dwc3_check_event_buf()
4588 disable_irq_nosync(dwc->irq_gadget); in dwc3_check_event_buf()
4593 * With PCIe legacy interrupt, test shows that top-half irq handler can in dwc3_check_event_buf()
4594 * be called again after HW interrupt deassertion. Check if bottom-half in dwc3_check_event_buf()
4598 if (evt->flags & DWC3_EVENT_PENDING) in dwc3_check_event_buf()
4601 count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0)); in dwc3_check_event_buf()
4606 if (count > evt->length) { in dwc3_check_event_buf()
4607 dev_err_ratelimited(dwc->dev, "invalid count(%u) > evt->length(%u)\n", in dwc3_check_event_buf()
4608 count, evt->length); in dwc3_check_event_buf()
4612 evt->count = count; in dwc3_check_event_buf()
4613 evt->flags |= DWC3_EVENT_PENDING; in dwc3_check_event_buf()
4616 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), in dwc3_check_event_buf()
4617 DWC3_GEVNTSIZ_INTMASK | DWC3_GEVNTSIZ_SIZE(evt->length)); in dwc3_check_event_buf()
4619 amount = min(count, evt->length - evt->lpos); in dwc3_check_event_buf()
4620 memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount); in dwc3_check_event_buf()
4623 memcpy(evt->cache, evt->buf, count - amount); in dwc3_check_event_buf()
4625 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count); in dwc3_check_event_buf()
4639 struct platform_device *dwc3_pdev = to_platform_device(dwc->dev); in dwc3_gadget_get_irq()
4646 if (irq == -EPROBE_DEFER) in dwc3_gadget_get_irq()
4653 if (irq == -EPROBE_DEFER) in dwc3_gadget_get_irq()
4670 * dwc3_gadget_init - initializes gadget related registers
4687 dwc->irq_gadget = irq; in dwc3_gadget_init()
4689 dwc->ep0_trb = dma_alloc_coherent(dwc->sysdev, in dwc3_gadget_init()
4690 sizeof(*dwc->ep0_trb) * 2, in dwc3_gadget_init()
4691 &dwc->ep0_trb_addr, GFP_KERNEL); in dwc3_gadget_init()
4692 if (!dwc->ep0_trb) { in dwc3_gadget_init()
4693 dev_err(dwc->dev, "failed to allocate ep0 trb\n"); in dwc3_gadget_init()
4694 ret = -ENOMEM; in dwc3_gadget_init()
4698 dwc->setup_buf = kzalloc(DWC3_EP0_SETUP_SIZE, GFP_KERNEL); in dwc3_gadget_init()
4699 if (!dwc->setup_buf) { in dwc3_gadget_init()
4700 ret = -ENOMEM; in dwc3_gadget_init()
4704 dwc->bounce = dma_alloc_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, in dwc3_gadget_init()
4705 &dwc->bounce_addr, GFP_KERNEL); in dwc3_gadget_init()
4706 if (!dwc->bounce) { in dwc3_gadget_init()
4707 ret = -ENOMEM; in dwc3_gadget_init()
4711 init_completion(&dwc->ep0_in_setup); in dwc3_gadget_init()
4712 dwc->gadget = kzalloc(sizeof(struct usb_gadget), GFP_KERNEL); in dwc3_gadget_init()
4713 if (!dwc->gadget) { in dwc3_gadget_init()
4714 ret = -ENOMEM; in dwc3_gadget_init()
4719 usb_initialize_gadget(dwc->dev, dwc->gadget, dwc_gadget_release); in dwc3_gadget_init()
4720 dev = &dwc->gadget->dev; in dwc3_gadget_init()
4721 dev->platform_data = dwc; in dwc3_gadget_init()
4722 dwc->gadget->ops = &dwc3_gadget_ops; in dwc3_gadget_init()
4723 dwc->gadget->speed = USB_SPEED_UNKNOWN; in dwc3_gadget_init()
4724 dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN; in dwc3_gadget_init()
4725 dwc->gadget->sg_supported = true; in dwc3_gadget_init()
4726 dwc->gadget->name = "dwc3-gadget"; in dwc3_gadget_init()
4727 dwc->gadget->lpm_capable = !dwc->usb2_gadget_lpm_disable; in dwc3_gadget_init()
4728 dwc->gadget->wakeup_capable = true; in dwc3_gadget_init()
4744 * composite.c that we are USB 2.0 + LPM ECN. in dwc3_gadget_init()
4747 !dwc->dis_metastability_quirk) in dwc3_gadget_init()
4748 dev_info(dwc->dev, "changing max_speed on rev %08x\n", in dwc3_gadget_init()
4749 dwc->revision); in dwc3_gadget_init()
4751 dwc->gadget->max_speed = dwc->maximum_speed; in dwc3_gadget_init()
4752 dwc->gadget->max_ssp_rate = dwc->max_ssp_rate; in dwc3_gadget_init()
4759 ret = dwc3_gadget_init_endpoints(dwc, dwc->num_eps); in dwc3_gadget_init()
4763 ret = usb_add_gadget(dwc->gadget); in dwc3_gadget_init()
4765 dev_err(dwc->dev, "failed to add gadget\n"); in dwc3_gadget_init()
4769 if (DWC3_IP_IS(DWC32) && dwc->maximum_speed == USB_SPEED_SUPER_PLUS) in dwc3_gadget_init()
4770 dwc3_gadget_set_ssp_rate(dwc->gadget, dwc->max_ssp_rate); in dwc3_gadget_init()
4772 dwc3_gadget_set_speed(dwc->gadget, dwc->maximum_speed); in dwc3_gadget_init()
4775 if (dwc->sys_wakeup) in dwc3_gadget_init()
4776 device_wakeup_disable(dwc->sysdev); in dwc3_gadget_init()
4783 usb_put_gadget(dwc->gadget); in dwc3_gadget_init()
4784 dwc->gadget = NULL; in dwc3_gadget_init()
4786 dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce, in dwc3_gadget_init()
4787 dwc->bounce_addr); in dwc3_gadget_init()
4790 kfree(dwc->setup_buf); in dwc3_gadget_init()
4793 dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2, in dwc3_gadget_init()
4794 dwc->ep0_trb, dwc->ep0_trb_addr); in dwc3_gadget_init()
4800 /* -------------------------------------------------------------------------- */
4804 if (!dwc->gadget) in dwc3_gadget_exit()
4808 usb_del_gadget(dwc->gadget); in dwc3_gadget_exit()
4810 usb_put_gadget(dwc->gadget); in dwc3_gadget_exit()
4811 dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce, in dwc3_gadget_exit()
4812 dwc->bounce_addr); in dwc3_gadget_exit()
4813 kfree(dwc->setup_buf); in dwc3_gadget_exit()
4814 dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2, in dwc3_gadget_exit()
4815 dwc->ep0_trb, dwc->ep0_trb_addr); in dwc3_gadget_exit()
4827 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_suspend()
4828 if (dwc->gadget_driver) in dwc3_gadget_suspend()
4830 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_suspend()
4840 if (dwc->softconnect) in dwc3_gadget_suspend()
4848 if (!dwc->gadget_driver || !dwc->softconnect) in dwc3_gadget_resume()