1 // SPDX-License-Identifier: GPL-2.0
3  * mtu3_qmu.c - Queue Management Unit driver for device controller
34 #define GET_GPD_HWO(gpd)	(le32_to_cpu((gpd)->dw0_info) & GPD_FLAGS_HWO)
41 	((mtu)->gen2cp) ? GPD_RX_BUF_LEN_EL(x_) : GPD_RX_BUF_LEN_OG(x_); \
49 	((mtu)->gen2cp) ? GPD_DATA_LEN_EL(x_) : GPD_DATA_LEN_OG(x_); \
60 	((mtu)->gen2cp) ? GPD_EXT_NGP_EL(x_) : GPD_EXT_NGP_OG(x_); \
66 	((mtu)->gen2cp) ? GPD_EXT_BUF_EL(x_) : GPD_EXT_BUF_OG(x_); \
122 	dma_addr_t dma_base = ring->dma;  in gpd_dma_to_virt()
123 	struct qmu_gpd *gpd_head = ring->start;  in gpd_dma_to_virt()
124 	u32 offset = (dma_addr - dma_base) / sizeof(*gpd_head);  in gpd_dma_to_virt()
135 	dma_addr_t dma_base = ring->dma;  in gpd_virt_to_dma()
136 	struct qmu_gpd *gpd_head = ring->start;  in gpd_virt_to_dma()
139 	offset = gpd - gpd_head;  in gpd_virt_to_dma()
148 	ring->start = gpd;  in gpd_ring_init()
149 	ring->enqueue = gpd;  in gpd_ring_init()
150 	ring->dequeue = gpd;  in gpd_ring_init()
151 	ring->end = gpd + MAX_GPD_NUM - 1;  in gpd_ring_init()
156 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;  in reset_gpd_list()
157 	struct qmu_gpd *gpd = ring->start;  in reset_gpd_list()
160 		gpd->dw0_info &= cpu_to_le32(~GPD_FLAGS_HWO);  in reset_gpd_list()
168 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;  in mtu3_gpd_ring_alloc()
171 	gpd = dma_pool_zalloc(mep->mtu->qmu_gpd_pool, GFP_ATOMIC, &ring->dma);  in mtu3_gpd_ring_alloc()
173 		return -ENOMEM;  in mtu3_gpd_ring_alloc()
182 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;  in mtu3_gpd_ring_free()
184 	dma_pool_free(mep->mtu->qmu_gpd_pool,  in mtu3_gpd_ring_free()
185 			ring->start, ring->dma);  in mtu3_gpd_ring_free()
191 	struct mtu3 *mtu = mep->mtu;  in mtu3_qmu_resume()
192 	void __iomem *mbase = mtu->mac_base;  in mtu3_qmu_resume()
193 	int epnum = mep->epnum;  in mtu3_qmu_resume()
196 	offset = mep->is_in ? USB_QMU_TQCSR(epnum) : USB_QMU_RQCSR(epnum);  in mtu3_qmu_resume()
205 	if (ring->enqueue < ring->end)  in advance_enq_gpd()
206 		ring->enqueue++;  in advance_enq_gpd()
208 		ring->enqueue = ring->start;  in advance_enq_gpd()
210 	return ring->enqueue;  in advance_enq_gpd()
216 	if (ring->dequeue < ring->end)  in advance_deq_gpd()
217 		ring->dequeue++;  in advance_deq_gpd()
219 		ring->dequeue = ring->start;  in advance_deq_gpd()
221 	return ring->dequeue;  in advance_deq_gpd()
227 	struct qmu_gpd *enq = ring->enqueue;  in gpd_ring_empty()
230 	if (ring->enqueue < ring->end)  in gpd_ring_empty()
233 		next = ring->start;  in gpd_ring_empty()
236 	return next == ring->dequeue;  in gpd_ring_empty()
241 	return gpd_ring_empty(&mep->gpd_ring);  in mtu3_prepare_transfer()
247 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;  in mtu3_prepare_tx_gpd()
248 	struct qmu_gpd *gpd = ring->enqueue;  in mtu3_prepare_tx_gpd()
249 	struct usb_request *req = &mreq->request;  in mtu3_prepare_tx_gpd()
250 	struct mtu3 *mtu = mep->mtu;  in mtu3_prepare_tx_gpd()
254 	gpd->dw0_info = 0;	/* SW own it */  in mtu3_prepare_tx_gpd()
255 	gpd->buffer = cpu_to_le32(lower_32_bits(req->dma));  in mtu3_prepare_tx_gpd()
256 	ext_addr = GPD_EXT_BUF(mtu, upper_32_bits(req->dma));  in mtu3_prepare_tx_gpd()
257 	gpd->dw3_info = cpu_to_le32(GPD_DATA_LEN(mtu, req->length));  in mtu3_prepare_tx_gpd()
262 	dev_dbg(mep->mtu->dev, "TX-EP%d queue gpd=%p, enq=%p, qdma=%pad\n",  in mtu3_prepare_tx_gpd()
263 		mep->epnum, gpd, enq, &enq_dma);  in mtu3_prepare_tx_gpd()
265 	enq->dw0_info &= cpu_to_le32(~GPD_FLAGS_HWO);  in mtu3_prepare_tx_gpd()
266 	gpd->next_gpd = cpu_to_le32(lower_32_bits(enq_dma));  in mtu3_prepare_tx_gpd()
268 	gpd->dw0_info = cpu_to_le32(ext_addr);  in mtu3_prepare_tx_gpd()
270 	if (req->zero) {  in mtu3_prepare_tx_gpd()
271 		if (mtu->gen2cp)  in mtu3_prepare_tx_gpd()
272 			gpd->dw0_info |= cpu_to_le32(GPD_FLAGS_ZLP);  in mtu3_prepare_tx_gpd()
274 			gpd->dw3_info |= cpu_to_le32(GPD_EXT_FLAG_ZLP);  in mtu3_prepare_tx_gpd()
279 	gpd->dw0_info |= cpu_to_le32(GPD_FLAGS_IOC | GPD_FLAGS_HWO);  in mtu3_prepare_tx_gpd()
281 	mreq->gpd = gpd;  in mtu3_prepare_tx_gpd()
290 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;  in mtu3_prepare_rx_gpd()
291 	struct qmu_gpd *gpd = ring->enqueue;  in mtu3_prepare_rx_gpd()
292 	struct usb_request *req = &mreq->request;  in mtu3_prepare_rx_gpd()
293 	struct mtu3 *mtu = mep->mtu;  in mtu3_prepare_rx_gpd()
297 	gpd->dw0_info = 0;	/* SW own it */  in mtu3_prepare_rx_gpd()
298 	gpd->buffer = cpu_to_le32(lower_32_bits(req->dma));  in mtu3_prepare_rx_gpd()
299 	ext_addr = GPD_EXT_BUF(mtu, upper_32_bits(req->dma));  in mtu3_prepare_rx_gpd()
300 	gpd->dw0_info = cpu_to_le32(GPD_RX_BUF_LEN(mtu, req->length));  in mtu3_prepare_rx_gpd()
305 	dev_dbg(mep->mtu->dev, "RX-EP%d queue gpd=%p, enq=%p, qdma=%pad\n",  in mtu3_prepare_rx_gpd()
306 		mep->epnum, gpd, enq, &enq_dma);  in mtu3_prepare_rx_gpd()
308 	enq->dw0_info &= cpu_to_le32(~GPD_FLAGS_HWO);  in mtu3_prepare_rx_gpd()
309 	gpd->next_gpd = cpu_to_le32(lower_32_bits(enq_dma));  in mtu3_prepare_rx_gpd()
311 	gpd->dw3_info = cpu_to_le32(ext_addr);  in mtu3_prepare_rx_gpd()
314 	gpd->dw0_info |= cpu_to_le32(GPD_FLAGS_IOC | GPD_FLAGS_HWO);  in mtu3_prepare_rx_gpd()
316 	mreq->gpd = gpd;  in mtu3_prepare_rx_gpd()
325 	if (mep->is_in)  in mtu3_insert_gpd()
333 	struct mtu3 *mtu = mep->mtu;  in mtu3_qmu_start()
334 	void __iomem *mbase = mtu->mac_base;  in mtu3_qmu_start()
335 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;  in mtu3_qmu_start()
336 	u8 epnum = mep->epnum;  in mtu3_qmu_start()
338 	if (mep->is_in) {  in mtu3_qmu_start()
340 		write_txq_start_addr(mbase, epnum, ring->dma);  in mtu3_qmu_start()
348 			dev_warn(mtu->dev, "Tx %d Active Now!\n", epnum);  in mtu3_qmu_start()
354 		write_rxq_start_addr(mbase, epnum, ring->dma);  in mtu3_qmu_start()
365 			dev_warn(mtu->dev, "Rx %d Active Now!\n", epnum);  in mtu3_qmu_start()
377 	struct mtu3 *mtu = mep->mtu;  in mtu3_qmu_stop()
378 	void __iomem *mbase = mtu->mac_base;  in mtu3_qmu_stop()
379 	int epnum = mep->epnum;  in mtu3_qmu_stop()
384 	qcsr = mep->is_in ? USB_QMU_TQCSR(epnum) : USB_QMU_RQCSR(epnum);  in mtu3_qmu_stop()
387 		dev_dbg(mtu->dev, "%s's qmu is inactive now!\n", mep->name);  in mtu3_qmu_stop()
392 	if (mep->is_in)  in mtu3_qmu_stop()
398 		dev_err(mtu->dev, "stop %s's qmu failed\n", mep->name);  in mtu3_qmu_stop()
403 	if (mep->is_in)  in mtu3_qmu_stop()
406 	dev_dbg(mtu->dev, "%s's qmu stop now!\n", mep->name);  in mtu3_qmu_stop()
412 	dev_dbg(mep->mtu->dev, "%s flush QMU %s\n", __func__,  in mtu3_qmu_flush()
413 		((mep->is_in) ? "TX" : "RX"));  in mtu3_qmu_flush()
423  * a length error interrupt, and in the ISR sends a ZLP by BMU.
427 	struct mtu3_ep *mep = mtu->in_eps + epnum;  in qmu_tx_zlp_error_handler()
428 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;  in qmu_tx_zlp_error_handler()
429 	void __iomem *mbase = mtu->mac_base;  in qmu_tx_zlp_error_handler()
437 	if (mreq && mreq->request.length != 0)  in qmu_tx_zlp_error_handler()
443 	if (GPD_DATA_LEN(mtu, le32_to_cpu(gpd_current->dw3_info)) != 0) {  in qmu_tx_zlp_error_handler()
444 		dev_err(mtu->dev, "TX EP%d buffer length error(!=0)\n", epnum);  in qmu_tx_zlp_error_handler()
448 	dev_dbg(mtu->dev, "%s send ZLP for req=%p\n", __func__, mreq);  in qmu_tx_zlp_error_handler()
451 	mtu3_clrbits(mbase, MU3D_EP_TXCR0(mep->epnum), TX_DMAREQEN);  in qmu_tx_zlp_error_handler()
453 	ret = readl_poll_timeout_atomic(mbase + MU3D_EP_TXCR0(mep->epnum),  in qmu_tx_zlp_error_handler()
456 		dev_err(mtu->dev, "%s wait for fifo empty fail\n", __func__);  in qmu_tx_zlp_error_handler()
459 	mtu3_setbits(mbase, MU3D_EP_TXCR0(mep->epnum), TX_TXPKTRDY);  in qmu_tx_zlp_error_handler()
463 	gpd_current->dw0_info |= cpu_to_le32(GPD_FLAGS_BPS | GPD_FLAGS_HWO);  in qmu_tx_zlp_error_handler()
466 	mtu3_setbits(mbase, MU3D_EP_TXCR0(mep->epnum), TX_DMAREQEN);  in qmu_tx_zlp_error_handler()
471  * when rx error happens (except zlperr), QMU will stop, and RQCPR saves
472  * the GPD encountered error, Done irq will arise after resuming QMU again.
476 	struct mtu3_ep *mep = mtu->out_eps + epnum;  in qmu_error_rx()
477 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;  in qmu_error_rx()
482 	cur_gpd_dma = read_rxq_cur_addr(mtu->mac_base, epnum);  in qmu_error_rx()
486 	if (!mreq || mreq->gpd != gpd_current) {  in qmu_error_rx()
487 		dev_err(mtu->dev, "no correct RX req is found\n");  in qmu_error_rx()
491 	mreq->request.status = -EAGAIN;  in qmu_error_rx()
494 	gpd_current->dw0_info |= cpu_to_le32(GPD_FLAGS_BPS | GPD_FLAGS_HWO);  in qmu_error_rx()
497 	dev_dbg(mtu->dev, "%s EP%d, current=%p, req=%p\n",  in qmu_error_rx()
503  * queue_tx --> qmu_interrupt(clear interrupt pending, schedule tasklet)-->
504  * queue_tx --> process_tasklet(meanwhile, the second one is transferred,
505  * tasklet process both of them)-->qmu_interrupt for second one.
510 	struct mtu3_ep *mep = mtu->in_eps + epnum;  in qmu_done_tx()
511 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;  in qmu_done_tx()
512 	void __iomem *mbase = mtu->mac_base;  in qmu_done_tx()
513 	struct qmu_gpd *gpd = ring->dequeue;  in qmu_done_tx()
523 	dev_dbg(mtu->dev, "%s EP%d, last=%p, current=%p, enq=%p\n",  in qmu_done_tx()
524 		__func__, epnum, gpd, gpd_current, ring->enqueue);  in qmu_done_tx()
530 		if (mreq == NULL || mreq->gpd != gpd) {  in qmu_done_tx()
531 			dev_err(mtu->dev, "no correct TX req is found\n");  in qmu_done_tx()
535 		request = &mreq->request;  in qmu_done_tx()
536 		request->actual = GPD_DATA_LEN(mtu, le32_to_cpu(gpd->dw3_info));  in qmu_done_tx()
543 	dev_dbg(mtu->dev, "%s EP%d, deq=%p, enq=%p, complete\n",  in qmu_done_tx()
544 		__func__, epnum, ring->dequeue, ring->enqueue);  in qmu_done_tx()
550 	struct mtu3_ep *mep = mtu->out_eps + epnum;  in qmu_done_rx()
551 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;  in qmu_done_rx()
552 	void __iomem *mbase = mtu->mac_base;  in qmu_done_rx()
553 	struct qmu_gpd *gpd = ring->dequeue;  in qmu_done_rx()
562 	dev_dbg(mtu->dev, "%s EP%d, last=%p, current=%p, enq=%p\n",  in qmu_done_rx()
563 		__func__, epnum, gpd, gpd_current, ring->enqueue);  in qmu_done_rx()
569 		if (mreq == NULL || mreq->gpd != gpd) {  in qmu_done_rx()
570 			dev_err(mtu->dev, "no correct RX req is found\n");  in qmu_done_rx()
573 		req = &mreq->request;  in qmu_done_rx()
575 		req->actual = GPD_DATA_LEN(mtu, le32_to_cpu(gpd->dw3_info));  in qmu_done_rx()
582 	dev_dbg(mtu->dev, "%s EP%d, deq=%p, enq=%p, complete\n",  in qmu_done_rx()
583 		__func__, epnum, ring->dequeue, ring->enqueue);  in qmu_done_rx()
590 	for (i = 1; i < mtu->num_eps; i++) {  in qmu_done_isr()
600 	void __iomem *mbase = mtu->mac_base;  in qmu_exception_isr()
608 		for (i = 1; i < mtu->num_eps; i++) {  in qmu_exception_isr()
610 				dev_err(mtu->dev, "Rx %d CS error!\n", i);  in qmu_exception_isr()
613 				dev_err(mtu->dev, "RX %d Length error\n", i);  in qmu_exception_isr()
622 		for (i = 1; i < mtu->num_eps; i++) {  in qmu_exception_isr()
624 				dev_dbg(mtu->dev, "RX EP%d Recv ZLP\n", i);  in qmu_exception_isr()
631 		for (i = 1; i < mtu->num_eps; i++) {  in qmu_exception_isr()
633 				dev_err(mtu->dev, "Tx %d checksum error!\n", i);  in qmu_exception_isr()
644 	void __iomem *mbase = mtu->mac_base;  in mtu3_qmu_isr()
655 	dev_dbg(mtu->dev, "=== QMUdone[tx=%x, rx=%x] QMUexp[%x] ===\n",  in mtu3_qmu_isr()
674 	mtu->qmu_gpd_pool = dma_pool_create("QMU_GPD", mtu->dev,  in mtu3_qmu_init()
677 	if (!mtu->qmu_gpd_pool)  in mtu3_qmu_init()
678 		return -ENOMEM;  in mtu3_qmu_init()
685 	dma_pool_destroy(mtu->qmu_gpd_pool);  in mtu3_qmu_exit()