98 				 struct usbhs_fifo *fifo);
99 static struct dma_chan *usbhsf_dma_chan_get(struct usbhs_fifo *fifo,
107 	struct usbhs_fifo *fifo = usbhs_pipe_to_fifo(pipe);  in usbhs_pkt_pop()  local
121 		if (fifo)  in usbhs_pkt_pop()
122 			chan = usbhsf_dma_chan_get(fifo, pkt);  in usbhs_pkt_pop()
133 	if (fifo)  in usbhs_pkt_pop()
134 		usbhsf_fifo_unselect(pipe, fifo);  in usbhs_pkt_pop()
245  *		FIFO ctrl
248 				   struct usbhs_fifo *fifo)  in usbhsf_send_terminator()  argument
252 	usbhs_bset(priv, fifo->ctr, BVAL, BVAL);  in usbhsf_send_terminator()
256 			       struct usbhs_fifo *fifo)  in usbhsf_fifo_barrier()  argument
258 	/* The FIFO port is accessible */  in usbhsf_fifo_barrier()
259 	if (usbhs_read(priv, fifo->ctr) & FRDY)  in usbhsf_fifo_barrier()
266 			      struct usbhs_fifo *fifo)  in usbhsf_fifo_clear()  argument
280 			ret = usbhsf_fifo_barrier(priv, fifo);  in usbhsf_fifo_clear()
288 		usbhs_write(priv, fifo->ctr, BCLR);  in usbhsf_fifo_clear()
292 			       struct usbhs_fifo *fifo)  in usbhsf_fifo_rcv_len()  argument
294 	return usbhs_read(priv, fifo->ctr) & DTLN_MASK;  in usbhsf_fifo_rcv_len()
298 				 struct usbhs_fifo *fifo)  in usbhsf_fifo_unselect()  argument
303 	usbhs_write(priv, fifo->sel, 0);  in usbhsf_fifo_unselect()
307 			      struct usbhs_fifo *fifo,  in usbhsf_fifo_select()  argument
317 	    usbhsf_fifo_is_busy(fifo))  in usbhsf_fifo_select()
328 	usbhs_write(priv, fifo->sel, base | MBW_32);  in usbhsf_fifo_select()
332 		if (base == (mask & usbhs_read(priv, fifo->sel))) {  in usbhsf_fifo_select()
333 			usbhs_pipe_select_fifo(pipe, fifo);  in usbhsf_fifo_select()
339 	dev_err(dev, "fifo select error\n");  in usbhsf_fifo_select()
351 	struct usbhs_fifo *fifo = usbhsf_get_cfifo(priv); /* CFIFO */  in usbhs_dcp_dir_switch_to_write()  local
357 	ret = usbhsf_fifo_select(pipe, fifo, 1);  in usbhs_dcp_dir_switch_to_write()
365 	usbhsf_fifo_clear(pipe, fifo);  in usbhs_dcp_dir_switch_to_write()
366 	usbhsf_send_terminator(pipe, fifo);  in usbhs_dcp_dir_switch_to_write()
368 	usbhsf_fifo_unselect(pipe, fifo);  in usbhs_dcp_dir_switch_to_write()
380 	struct usbhs_fifo *fifo = usbhsf_get_cfifo(priv); /* CFIFO */  in usbhs_dcp_dir_switch_to_read()  local
386 	ret = usbhsf_fifo_select(pipe, fifo, 0);  in usbhs_dcp_dir_switch_to_read()
393 	usbhsf_fifo_clear(pipe, fifo);  in usbhs_dcp_dir_switch_to_read()
395 	usbhsf_fifo_unselect(pipe, fifo);  in usbhs_dcp_dir_switch_to_read()
458 	struct usbhs_fifo *fifo = usbhsf_get_cfifo(priv);  in usbhsf_dcp_data_stage_prepare_pop()  local
466 	 *  - clear fifo  in usbhsf_dcp_data_stage_prepare_pop()
473 	usbhsf_fifo_select(pipe, fifo, 0);  in usbhsf_dcp_data_stage_prepare_pop()
474 	usbhsf_fifo_clear(pipe, fifo);  in usbhsf_dcp_data_stage_prepare_pop()
475 	usbhsf_fifo_unselect(pipe, fifo);  in usbhsf_dcp_data_stage_prepare_pop()
497 	struct usbhs_fifo *fifo = usbhsf_get_cfifo(priv); /* CFIFO */  in usbhsf_pio_try_push()  local
498 	void __iomem *addr = priv->base + fifo->port;  in usbhsf_pio_try_push()
510 	ret = usbhsf_fifo_select(pipe, fifo, 1);  in usbhsf_pio_try_push()
521 	ret = usbhsf_fifo_barrier(priv, fifo);  in usbhsf_pio_try_push()
567 		usbhsf_send_terminator(pipe, fifo);  in usbhsf_pio_try_push()
577 	usbhsf_fifo_unselect(pipe, fifo);  in usbhsf_pio_try_push()
582 	usbhsf_fifo_unselect(pipe, fifo);  in usbhsf_pio_try_push()
614 	struct usbhs_fifo *fifo = usbhsf_get_cfifo(priv);  in usbhsf_prepare_pop()  local
629 		usbhsf_fifo_clear(pipe, fifo);  in usbhsf_prepare_pop()
644 	struct usbhs_fifo *fifo = usbhsf_get_cfifo(priv); /* CFIFO */  in usbhsf_pio_try_pop()  local
645 	void __iomem *addr = priv->base + fifo->port;  in usbhsf_pio_try_pop()
653 	ret = usbhsf_fifo_select(pipe, fifo, 0);  in usbhsf_pio_try_pop()
657 	ret = usbhsf_fifo_barrier(priv, fifo);  in usbhsf_pio_try_pop()
661 	rcv_len = usbhsf_fifo_rcv_len(priv, fifo);  in usbhsf_pio_try_pop()
694 	 * "Operation" - "FIFO Buffer Memory" - "FIFO Port Function"  in usbhsf_pio_try_pop()
698 		usbhsf_fifo_clear(pipe, fifo);  in usbhsf_pio_try_pop()
727 	usbhsf_fifo_unselect(pipe, fifo);  in usbhsf_pio_try_pop()
755  *		DMA fifo functions
757 static struct dma_chan *usbhsf_dma_chan_get(struct usbhs_fifo *fifo,  in usbhsf_dma_chan_get()  argument
761 		return fifo->tx_chan;  in usbhsf_dma_chan_get()
764 		return fifo->rx_chan;  in usbhsf_dma_chan_get()
772 	struct usbhs_fifo *fifo;  in usbhsf_get_dma_fifo()  local
775 	usbhs_for_each_dfifo(priv, fifo, i) {  in usbhsf_get_dma_fifo()
776 		if (usbhsf_dma_chan_get(fifo, pkt) &&  in usbhsf_get_dma_fifo()
777 		    !usbhsf_fifo_is_busy(fifo))  in usbhsf_get_dma_fifo()
778 			return fifo;  in usbhsf_get_dma_fifo()
787 			      struct usbhs_fifo *fifo,  in __usbhsf_dma_ctrl()  argument
792 	usbhs_bset(priv, fifo->sel, DREQE, dreqe);  in __usbhsf_dma_ctrl()
800 	struct usbhs_fifo *fifo = usbhs_pipe_to_fifo(pipe);  in __usbhsf_dma_map_ctrl()  local
801 	struct dma_chan *chan = usbhsf_dma_chan_get(fifo, pkt);  in __usbhsf_dma_map_ctrl()
811 	struct usbhs_fifo *fifo;  in usbhsf_dma_xfer_preparing()  local
819 	fifo = usbhs_pipe_to_fifo(pipe);  in usbhsf_dma_xfer_preparing()
820 	if (!fifo)  in usbhsf_dma_xfer_preparing()
823 	chan = usbhsf_dma_chan_get(fifo, pkt);  in usbhsf_dma_xfer_preparing()
842 		fifo->name, usbhs_pipe_number(pipe), pkt->length, pkt->zero);  in usbhsf_dma_xfer_preparing()
847 	usbhsf_dma_start(pipe, fifo);  in usbhsf_dma_xfer_preparing()
870 	struct usbhs_fifo *fifo;  in usbhsf_dma_prepare_push()  local
897 	/* get enable DMA fifo */  in usbhsf_dma_prepare_push()
898 	fifo = usbhsf_get_dma_fifo(priv, pkt);  in usbhsf_dma_prepare_push()
899 	if (!fifo)  in usbhsf_dma_prepare_push()
902 	ret = usbhsf_fifo_select(pipe, fifo, 0);  in usbhsf_dma_prepare_push()
923 	usbhsf_fifo_unselect(pipe, fifo);  in usbhsf_dma_prepare_push()
949 	usbhsf_dma_stop(pipe, pipe->fifo);  in usbhsf_dma_push_done()
951 	usbhsf_fifo_unselect(pipe, pipe->fifo);  in usbhsf_dma_push_done()
982 	struct usbhs_fifo *fifo;  in usbhsf_dma_prepare_pop_with_usb_dmac()  local
993 	fifo = usbhsf_get_dma_fifo(priv, pkt);  in usbhsf_dma_prepare_pop_with_usb_dmac()
994 	if (!fifo)  in usbhsf_dma_prepare_pop_with_usb_dmac()
1006 	ret = usbhsf_fifo_select(pipe, fifo, 0);  in usbhsf_dma_prepare_pop_with_usb_dmac()
1029 	usbhsf_fifo_unselect(pipe, fifo);  in usbhsf_dma_prepare_pop_with_usb_dmac()
1055 	struct usbhs_fifo *fifo;  in usbhsf_dma_try_pop_with_rx_irq()  local
1064 	/* get enable DMA fifo */  in usbhsf_dma_try_pop_with_rx_irq()
1065 	fifo = usbhsf_get_dma_fifo(priv, pkt);  in usbhsf_dma_try_pop_with_rx_irq()
1066 	if (!fifo)  in usbhsf_dma_try_pop_with_rx_irq()
1072 	ret = usbhsf_fifo_select(pipe, fifo, 0);  in usbhsf_dma_try_pop_with_rx_irq()
1077 	len = usbhsf_fifo_rcv_len(priv, fifo);  in usbhsf_dma_try_pop_with_rx_irq()
1085 	ret = usbhsf_fifo_barrier(priv, fifo);  in usbhsf_dma_try_pop_with_rx_irq()
1109 	usbhsf_fifo_unselect(pipe, fifo);  in usbhsf_dma_try_pop_with_rx_irq()
1134 	usbhsf_dma_stop(pipe, pipe->fifo);  in usbhsf_dma_pop_done_with_rx_irq()
1136 	usbhsf_fifo_unselect(pipe, pipe->fifo);  in usbhsf_dma_pop_done_with_rx_irq()
1176 	struct usbhs_fifo *fifo = usbhs_pipe_to_fifo(pipe);  in usbhsf_dma_pop_done_with_usb_dmac()  local
1177 	struct dma_chan *chan = usbhsf_dma_chan_get(fifo, pkt);  in usbhsf_dma_pop_done_with_usb_dmac()
1187 	rcv_len = usbhsf_fifo_rcv_len(priv, fifo);  in usbhsf_dma_pop_done_with_usb_dmac()
1188 	usbhsf_fifo_clear(pipe, fifo);  in usbhsf_dma_pop_done_with_usb_dmac()
1192 	usbhsf_dma_stop(pipe, fifo);  in usbhsf_dma_pop_done_with_usb_dmac()
1194 	usbhsf_fifo_unselect(pipe, pipe->fifo);  in usbhsf_dma_pop_done_with_usb_dmac()
1238 static void usbhsf_dma_quit(struct usbhs_priv *priv, struct usbhs_fifo *fifo)  in usbhsf_dma_quit()  argument
1240 	if (fifo->tx_chan)  in usbhsf_dma_quit()
1241 		dma_release_channel(fifo->tx_chan);  in usbhsf_dma_quit()
1242 	if (fifo->rx_chan)  in usbhsf_dma_quit()
1243 		dma_release_channel(fifo->rx_chan);  in usbhsf_dma_quit()
1245 	fifo->tx_chan = NULL;  in usbhsf_dma_quit()
1246 	fifo->rx_chan = NULL;  in usbhsf_dma_quit()
1249 static void usbhsf_dma_init_pdev(struct usbhs_fifo *fifo)  in usbhsf_dma_init_pdev()  argument
1255 	fifo->tx_chan = dma_request_channel(mask, usbhsf_dma_filter,  in usbhsf_dma_init_pdev()
1256 					    &fifo->tx_slave);  in usbhsf_dma_init_pdev()
1260 	fifo->rx_chan = dma_request_channel(mask, usbhsf_dma_filter,  in usbhsf_dma_init_pdev()
1261 					    &fifo->rx_slave);  in usbhsf_dma_init_pdev()
1264 static void usbhsf_dma_init_dt(struct device *dev, struct usbhs_fifo *fifo,  in usbhsf_dma_init_dt()  argument
1276 		fifo->tx_chan = dma_request_chan(dev, name);  in usbhsf_dma_init_dt()
1277 		if (IS_ERR(fifo->tx_chan))  in usbhsf_dma_init_dt()
1278 			fifo->tx_chan = NULL;  in usbhsf_dma_init_dt()
1280 		fifo->rx_chan = dma_request_chan(dev, name);  in usbhsf_dma_init_dt()
1281 		if (IS_ERR(fifo->rx_chan))  in usbhsf_dma_init_dt()
1282 			fifo->rx_chan = NULL;  in usbhsf_dma_init_dt()
1286 static void usbhsf_dma_init(struct usbhs_priv *priv, struct usbhs_fifo *fifo,  in usbhsf_dma_init()  argument
1292 		usbhsf_dma_init_dt(dev, fifo, channel);  in usbhsf_dma_init()
1294 		usbhsf_dma_init_pdev(fifo);  in usbhsf_dma_init()
1296 	if (fifo->tx_chan || fifo->rx_chan)  in usbhsf_dma_init()
1298 			 fifo->name,  in usbhsf_dma_init()
1299 			 fifo->tx_chan ? "[TX]" : "    ",  in usbhsf_dma_init()
1300 			 fifo->rx_chan ? "[RX]" : "    ");  in usbhsf_dma_init()
1385 	struct usbhs_fifo *fifo = usbhsf_get_cfifo(priv); /* CFIFO */  in usbhs_fifo_clear_dcp()  local
1387 	/* clear DCP FIFO of transmission */  in usbhs_fifo_clear_dcp()
1388 	if (usbhsf_fifo_select(pipe, fifo, 1) < 0)  in usbhs_fifo_clear_dcp()
1390 	usbhsf_fifo_clear(pipe, fifo);  in usbhs_fifo_clear_dcp()
1391 	usbhsf_fifo_unselect(pipe, fifo);  in usbhs_fifo_clear_dcp()
1393 	/* clear DCP FIFO of reception */  in usbhs_fifo_clear_dcp()
1394 	if (usbhsf_fifo_select(pipe, fifo, 0) < 0)  in usbhs_fifo_clear_dcp()
1396 	usbhsf_fifo_clear(pipe, fifo);  in usbhs_fifo_clear_dcp()
1397 	usbhsf_fifo_unselect(pipe, fifo);  in usbhs_fifo_clear_dcp()
1401  *		fifo init
1430 #define __USBHS_DFIFO_INIT(priv, fifo, channel, fifo_port)		\  argument
1432 	fifo = usbhsf_get_dnfifo(priv, channel);			\
1433 	fifo->name	= "D"#channel"FIFO";				\
1434 	fifo->port	= fifo_port;					\
1435 	fifo->sel	= D##channel##FIFOSEL;				\
1436 	fifo->ctr	= D##channel##FIFOCTR;				\
1437 	fifo->tx_slave.shdma_slave.slave_id =				\
1439 	fifo->rx_slave.shdma_slave.slave_id =				\
1441 	usbhsf_dma_init(priv, fifo, channel);				\
1444 #define USBHS_DFIFO_INIT(priv, fifo, channel)				\  argument
1445 		__USBHS_DFIFO_INIT(priv, fifo, channel, D##channel##FIFO)
1446 #define USBHS_DFIFO_INIT_NO_PORT(priv, fifo, channel)			\  argument
1447 		__USBHS_DFIFO_INIT(priv, fifo, channel, 0)
1451 	struct usbhs_fifo *fifo;  in usbhs_fifo_probe()  local
1454 	fifo = usbhsf_get_cfifo(priv);  in usbhs_fifo_probe()
1455 	fifo->name	= "CFIFO";  in usbhs_fifo_probe()
1456 	fifo->port	= CFIFO;  in usbhs_fifo_probe()
1457 	fifo->sel	= CFIFOSEL;  in usbhs_fifo_probe()
1458 	fifo->ctr	= CFIFOCTR;  in usbhs_fifo_probe()
1461 	USBHS_DFIFO_INIT(priv, fifo, 0);  in usbhs_fifo_probe()
1462 	USBHS_DFIFO_INIT(priv, fifo, 1);  in usbhs_fifo_probe()
1463 	USBHS_DFIFO_INIT_NO_PORT(priv, fifo, 2);  in usbhs_fifo_probe()
1464 	USBHS_DFIFO_INIT_NO_PORT(priv, fifo, 3);  in usbhs_fifo_probe()
1471 	struct usbhs_fifo *fifo;  in usbhs_fifo_remove()  local
1474 	usbhs_for_each_dfifo(priv, fifo, i)  in usbhs_fifo_remove()
1475 		usbhsf_dma_quit(priv, fifo);  in usbhs_fifo_remove()