Lines Matching +full:rx +full:- +full:tx
1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (C) 2005-2006 by Texas Instruments
6 * For now it's DaVinci-only, but CPPI isn't specific to DaVinci or USB.
21 /* CPPI DMA status 7-mar-2006:
23 * - See musb_{host,gadget}.c for more info
25 * - Correct RX DMA generally forces the engine into irq-per-packet mode,
26 * which can easily saturate the CPU under non-mass-storage loads.
28 * NOTES 24-aug-2006 (2.6.18-rc4):
30 * - peripheral RXDMA wedged in a test with packets of length 512/512/1.
33 * and RX DMA0: 4 left, 80000000 8feff880, 8feff860 8feff860; 8f321401
34 * 004001ff 00000001 .. 8feff860) Host was just getting NAKed on tx
38 * evidently also directly update the RX and TX CSRs ... so audit all
44 * more simply, switch to a global freelist not per-channel ones.
63 struct cppi_descriptor *bd = c->freelist; in cppi_bd_alloc()
66 c->freelist = bd->next; in cppi_bd_alloc()
75 bd->next = c->freelist; in cppi_bd_free()
76 c->freelist = bd; in cppi_bd_free()
85 /* zero out entire rx state RAM entry for the channel */
86 static void cppi_reset_rx(struct cppi_rx_stateram __iomem *rx) in cppi_reset_rx() argument
88 musb_writel(&rx->rx_skipbytes, 0, 0); in cppi_reset_rx()
89 musb_writel(&rx->rx_head, 0, 0); in cppi_reset_rx()
90 musb_writel(&rx->rx_sop, 0, 0); in cppi_reset_rx()
91 musb_writel(&rx->rx_current, 0, 0); in cppi_reset_rx()
92 musb_writel(&rx->rx_buf_current, 0, 0); in cppi_reset_rx()
93 musb_writel(&rx->rx_len_len, 0, 0); in cppi_reset_rx()
94 musb_writel(&rx->rx_cnt_cnt, 0, 0); in cppi_reset_rx()
97 /* zero out entire tx state RAM entry for the channel */
98 static void cppi_reset_tx(struct cppi_tx_stateram __iomem *tx, u32 ptr) in cppi_reset_tx() argument
100 musb_writel(&tx->tx_head, 0, 0); in cppi_reset_tx()
101 musb_writel(&tx->tx_buf, 0, 0); in cppi_reset_tx()
102 musb_writel(&tx->tx_current, 0, 0); in cppi_reset_tx()
103 musb_writel(&tx->tx_buf_current, 0, 0); in cppi_reset_tx()
104 musb_writel(&tx->tx_info, 0, 0); in cppi_reset_tx()
105 musb_writel(&tx->tx_rem_len, 0, 0); in cppi_reset_tx()
106 /* musb_writel(&tx->tx_dummy, 0, 0); */ in cppi_reset_tx()
107 musb_writel(&tx->tx_complete, 0, ptr); in cppi_reset_tx()
115 c->head = NULL; in cppi_pool_init()
116 c->tail = NULL; in cppi_pool_init()
117 c->last_processed = NULL; in cppi_pool_init()
118 c->channel.status = MUSB_DMA_STATUS_UNKNOWN; in cppi_pool_init()
119 c->controller = cppi; in cppi_pool_init()
120 c->is_rndis = 0; in cppi_pool_init()
121 c->freelist = NULL; in cppi_pool_init()
128 bd = dma_pool_alloc(cppi->pool, GFP_KERNEL, &dma); in cppi_pool_init()
129 bd->dma = dma; in cppi_pool_init()
138 struct cppi *cppi = c->controller; in cppi_pool_free()
141 (void) cppi_channel_abort(&c->channel); in cppi_pool_free()
142 c->channel.status = MUSB_DMA_STATUS_UNKNOWN; in cppi_pool_free()
143 c->controller = NULL; in cppi_pool_free()
146 bd = c->last_processed; in cppi_pool_free()
149 dma_pool_free(cppi->pool, bd, bd->dma); in cppi_pool_free()
152 c->last_processed = NULL; in cppi_pool_free()
161 for (i = 0; i < ARRAY_SIZE(controller->tx); i++) { in cppi_controller_start()
162 controller->tx[i].transmit = true; in cppi_controller_start()
163 controller->tx[i].index = i; in cppi_controller_start()
165 for (i = 0; i < ARRAY_SIZE(controller->rx); i++) { in cppi_controller_start()
166 controller->rx[i].transmit = false; in cppi_controller_start()
167 controller->rx[i].index = i; in cppi_controller_start()
171 for (i = 0; i < ARRAY_SIZE(controller->tx); i++) in cppi_controller_start()
172 cppi_pool_init(controller, controller->tx + i); in cppi_controller_start()
173 for (i = 0; i < ARRAY_SIZE(controller->rx); i++) in cppi_controller_start()
174 cppi_pool_init(controller, controller->rx + i); in cppi_controller_start()
176 tibase = controller->tibase; in cppi_controller_start()
177 INIT_LIST_HEAD(&controller->tx_complete); in cppi_controller_start()
179 /* initialise tx/rx channel head pointers to zero */ in cppi_controller_start()
180 for (i = 0; i < ARRAY_SIZE(controller->tx); i++) { in cppi_controller_start()
181 struct cppi_channel *tx_ch = controller->tx + i; in cppi_controller_start()
182 struct cppi_tx_stateram __iomem *tx; in cppi_controller_start() local
184 INIT_LIST_HEAD(&tx_ch->tx_complete); in cppi_controller_start()
186 tx = tibase + DAVINCI_TXCPPI_STATERAM_OFFSET(i); in cppi_controller_start()
187 tx_ch->state_ram = tx; in cppi_controller_start()
188 cppi_reset_tx(tx, 0); in cppi_controller_start()
190 for (i = 0; i < ARRAY_SIZE(controller->rx); i++) { in cppi_controller_start()
191 struct cppi_channel *rx_ch = controller->rx + i; in cppi_controller_start()
192 struct cppi_rx_stateram __iomem *rx; in cppi_controller_start() local
194 INIT_LIST_HEAD(&rx_ch->tx_complete); in cppi_controller_start()
196 rx = tibase + DAVINCI_RXCPPI_STATERAM_OFFSET(i); in cppi_controller_start()
197 rx_ch->state_ram = rx; in cppi_controller_start()
198 cppi_reset_rx(rx); in cppi_controller_start()
207 /* enable tx/rx CPPI control */ in cppi_controller_start()
211 /* disable RNDIS mode, also host rx RNDIS autorequest */ in cppi_controller_start()
219 * De-Init the DMA controller as necessary.
228 musb = controller->controller.musb; in cppi_controller_stop()
230 tibase = controller->tibase; in cppi_controller_stop()
237 musb_dbg(musb, "Tearing down RX and TX Channels"); in cppi_controller_stop()
238 for (i = 0; i < ARRAY_SIZE(controller->tx); i++) { in cppi_controller_stop()
240 controller->tx[i].last_processed = NULL; in cppi_controller_stop()
241 cppi_pool_free(controller->tx + i); in cppi_controller_stop()
243 for (i = 0; i < ARRAY_SIZE(controller->rx); i++) in cppi_controller_stop()
244 cppi_pool_free(controller->rx + i); in cppi_controller_stop()
246 /* in Tx Case proper teardown is supported. We resort to disabling in cppi_controller_stop()
247 * Tx/Rx CPPI after cleanup of Tx channels. Before TX teardown is in cppi_controller_stop()
248 * complete TX CPPI cannot be disabled. in cppi_controller_stop()
250 /*disable tx/rx cppi */ in cppi_controller_stop()
257 * Except for TX irqs, where dma done != fifo empty and reusable ...
261 * REVISIT same issue applies to pure PIO usage too, and non-cppi dma...
276 * each transfer direction of a non-control endpoint, so allocating
291 tibase = controller->tibase; in cppi_channel_allocate()
292 musb = c->musb; in cppi_channel_allocate()
294 /* ep0 doesn't use DMA; remember cppi indices are 0..N-1 */ in cppi_channel_allocate()
295 index = ep->epnum - 1; in cppi_channel_allocate()
298 * probably disable the non-CPPI irq until we need it. in cppi_channel_allocate()
301 if (index >= ARRAY_SIZE(controller->tx)) { in cppi_channel_allocate()
305 cppi_ch = controller->tx + index; in cppi_channel_allocate()
307 if (index >= ARRAY_SIZE(controller->rx)) { in cppi_channel_allocate()
311 cppi_ch = controller->rx + index; in cppi_channel_allocate()
312 core_rxirq_disable(tibase, ep->epnum); in cppi_channel_allocate()
318 if (cppi_ch->hw_ep) in cppi_channel_allocate()
319 musb_dbg(musb, "re-allocating DMA%d %cX channel %p", in cppi_channel_allocate()
321 cppi_ch->hw_ep = ep; in cppi_channel_allocate()
322 cppi_ch->channel.status = MUSB_DMA_STATUS_FREE; in cppi_channel_allocate()
323 cppi_ch->channel.max_len = 0x7fffffff; in cppi_channel_allocate()
326 return &cppi_ch->channel; in cppi_channel_allocate()
338 tibase = c->controller->tibase; in cppi_channel_release()
339 if (!c->hw_ep) in cppi_channel_release()
340 musb_dbg(c->controller->controller.musb, in cppi_channel_release()
342 else if (!c->transmit) in cppi_channel_release()
343 core_rxirq_enable(tibase, c->index + 1); in cppi_channel_release()
346 c->hw_ep = NULL; in cppi_channel_release()
347 channel->status = MUSB_DMA_STATUS_UNKNOWN; in cppi_channel_release()
354 void __iomem *base = c->controller->mregs; in cppi_dump_rx()
355 struct cppi_rx_stateram __iomem *rx = c->state_ram; in cppi_dump_rx() local
357 musb_ep_select(base, c->index + 1); in cppi_dump_rx()
359 musb_dbg(c->controller->controller.musb, in cppi_dump_rx()
360 "RX DMA%d%s: %d left, csr %04x, " in cppi_dump_rx()
363 c->index, tag, in cppi_dump_rx()
364 musb_readl(c->controller->tibase, in cppi_dump_rx()
365 DAVINCI_RXCPPI_BUFCNT0_REG + 4 * c->index), in cppi_dump_rx()
366 musb_readw(c->hw_ep->regs, MUSB_RXCSR), in cppi_dump_rx()
368 musb_readl(&rx->rx_skipbytes, 0), in cppi_dump_rx()
369 musb_readl(&rx->rx_head, 0), in cppi_dump_rx()
370 musb_readl(&rx->rx_sop, 0), in cppi_dump_rx()
371 musb_readl(&rx->rx_current, 0), in cppi_dump_rx()
373 musb_readl(&rx->rx_buf_current, 0), in cppi_dump_rx()
374 musb_readl(&rx->rx_len_len, 0), in cppi_dump_rx()
375 musb_readl(&rx->rx_cnt_cnt, 0), in cppi_dump_rx()
376 musb_readl(&rx->rx_complete, 0) in cppi_dump_rx()
384 void __iomem *base = c->controller->mregs; in cppi_dump_tx()
385 struct cppi_tx_stateram __iomem *tx = c->state_ram; in cppi_dump_tx() local
387 musb_ep_select(base, c->index + 1); in cppi_dump_tx()
389 musb_dbg(c->controller->controller.musb, in cppi_dump_tx()
390 "TX DMA%d%s: csr %04x, " in cppi_dump_tx()
393 c->index, tag, in cppi_dump_tx()
394 musb_readw(c->hw_ep->regs, MUSB_TXCSR), in cppi_dump_tx()
396 musb_readl(&tx->tx_head, 0), in cppi_dump_tx()
397 musb_readl(&tx->tx_buf, 0), in cppi_dump_tx()
398 musb_readl(&tx->tx_current, 0), in cppi_dump_tx()
399 musb_readl(&tx->tx_buf_current, 0), in cppi_dump_tx()
401 musb_readl(&tx->tx_info, 0), in cppi_dump_tx()
402 musb_readl(&tx->tx_rem_len, 0), in cppi_dump_tx()
404 musb_readl(&tx->tx_complete, 0) in cppi_dump_tx()
414 if (c->is_rndis != is_rndis) { in cppi_rndis_update()
416 u32 temp = 1 << (c->index); in cppi_rndis_update()
425 c->is_rndis = is_rndis; in cppi_rndis_update()
433 tag, bd->dma, in cppi_dump_rxbd()
434 bd->hw_next, bd->hw_bufp, bd->hw_off_len, in cppi_dump_rxbd()
435 bd->hw_options); in cppi_dump_rxbd()
438 static void cppi_dump_rxq(int level, const char *tag, struct cppi_channel *rx) in cppi_dump_rxq() argument
442 cppi_dump_rx(level, rx, tag); in cppi_dump_rxq()
443 if (rx->last_processed) in cppi_dump_rxq()
444 cppi_dump_rxbd("last", rx->last_processed); in cppi_dump_rxq()
445 for (bd = rx->head; bd; bd = bd->next) in cppi_dump_rxq()
450 /* NOTE: DaVinci autoreq is ignored except for host side "RNDIS" mode RX;
451 * so we won't ever use it (see "CPPI RX Woes" below).
453 static inline int cppi_autoreq_update(struct cppi_channel *rx, in cppi_autoreq_update() argument
464 val = tmp & ~((0x3) << (rx->index * 2)); in cppi_autoreq_update()
472 val |= ((0x3) << (rx->index * 2)); in cppi_autoreq_update()
473 n_bds--; in cppi_autoreq_update()
475 /* one segment, autoreq "all-but-last" */ in cppi_autoreq_update()
476 val |= ((0x1) << (rx->index * 2)); in cppi_autoreq_update()
490 } while (n-- > 0); in cppi_autoreq_update()
495 if (n_bds && rx->channel.actual_len) { in cppi_autoreq_update()
496 void __iomem *regs = rx->hw_ep->regs; in cppi_autoreq_update()
512 * - RX builds new queues each time, to help handle routine "early
516 * - for now, TX reuses the same queue of BDs every time
526 * in the TX paths. (RX can't do this, see below.)
528 * That's true even in the CPPI- friendly iso case, where most urbs have
530 * "transparent" DMA model is always correct, even on the RX side.
534 * CPPI TX:
536 * TX is a lot more reasonable than RX; it doesn't need to run in
537 * irq-per-packet mode very often. RNDIS mode seems to behave too
538 * (except how it handles the exactly-N-packets case). Building a
542 * The main issue with TX mode RNDIS relates to transfer lengths that
553 * On TX, "transparent" mode works ... although experiments have shown
561 cppi_next_tx_segment(struct musb *musb, struct cppi_channel *tx) in cppi_next_tx_segment() argument
563 unsigned maxpacket = tx->maxpacket; in cppi_next_tx_segment()
564 dma_addr_t addr = tx->buf_dma + tx->offset; in cppi_next_tx_segment()
565 size_t length = tx->buf_len - tx->offset; in cppi_next_tx_segment()
569 struct cppi_tx_stateram __iomem *tx_ram = tx->state_ram; in cppi_next_tx_segment()
572 /* TX can use the CPPI "rndis" mode, where we can probably fit this in cppi_next_tx_segment()
594 musb_dbg(musb, "TX DMA%d, pktSz %d %s bds %d dma 0x%llx len %u", in cppi_next_tx_segment()
595 tx->index, in cppi_next_tx_segment()
601 cppi_rndis_update(tx, 0, musb->ctrl_base, rndis); in cppi_next_tx_segment()
609 bd = tx->freelist; in cppi_next_tx_segment()
610 tx->head = bd; in cppi_next_tx_segment()
611 tx->last_processed = NULL; in cppi_next_tx_segment()
613 /* FIXME use BD pool like RX side does, and just queue in cppi_next_tx_segment()
622 if (++i < n_bds && bd->next) in cppi_next_tx_segment()
623 bd->hw_next = bd->next->dma; in cppi_next_tx_segment()
625 bd->hw_next = 0; in cppi_next_tx_segment()
627 bd->hw_bufp = tx->buf_dma + tx->offset; in cppi_next_tx_segment()
632 if ((tx->offset + maxpacket) <= tx->buf_len) { in cppi_next_tx_segment()
633 tx->offset += maxpacket; in cppi_next_tx_segment()
634 bd->hw_off_len = maxpacket; in cppi_next_tx_segment()
635 bd->hw_options = CPPI_SOP_SET | CPPI_EOP_SET in cppi_next_tx_segment()
641 partial_len = tx->buf_len - tx->offset; in cppi_next_tx_segment()
642 tx->offset = tx->buf_len; in cppi_next_tx_segment()
643 bd->hw_off_len = partial_len; in cppi_next_tx_segment()
645 bd->hw_options = CPPI_SOP_SET | CPPI_EOP_SET in cppi_next_tx_segment()
648 bd->hw_options |= CPPI_ZERO_SET; in cppi_next_tx_segment()
652 bd, bd->hw_next, bd->hw_bufp, in cppi_next_tx_segment()
653 bd->hw_off_len, bd->hw_options); in cppi_next_tx_segment()
656 tx->tail = bd; in cppi_next_tx_segment()
657 bd = bd->next; in cppi_next_tx_segment()
660 /* BDs live in DMA-coherent memory, but writes might be pending */ in cppi_next_tx_segment()
664 musb_writel(&tx_ram->tx_head, 0, (u32)tx->freelist->dma); in cppi_next_tx_segment()
666 cppi_dump_tx(5, tx, "/S"); in cppi_next_tx_segment()
670 * CPPI RX Woes:
672 * Consider a 1KB bulk RX buffer in two scenarios: (a) it's fed two 300 byte
673 * packets back-to-back, and (b) it's fed two 512 byte packets back-to-back.
680 * - RX queues in "rndis" mode -- one single BD -- handle (a) correctly, but
683 * not a "true" RNDIS mode. In the RNDIS protocol short-packet termination
684 * is optional, and it's fine if peripherals -- not hosts! -- pad messages
685 * out to end-of-buffer. Standard PCI host controller DMA descriptors
688 * - RX queues in "transparent" mode -- two BDs with 512 bytes each -- have
689 * converse problems: (b) is handled right, but (a) loses badly. CPPI RX
693 * are intended as outputs for RX queues, not inputs...)
695 * - A variant of "transparent" mode -- one BD at a time -- is the only way to
700 * So how to get rid of IRQ-per-packet? The transparent multi-BD case could
707 * is appropriate -- which do NOT include RNDIS host drivers, but do include
708 * the CDC Ethernet driver! -- and the documentation is incomplete/wrong.
709 * So we can't _ever_ use RX RNDIS mode ... except by using a heuristic
712 * Leaving only "transparent" mode; we avoid multi-bd modes in almost all
727 * - less than 64KB (max per cppi descriptor)
728 * - not a multiple of 4096 (g_zero default, full reads typical)
729 * - N (>1) packets long, ditto (full reads not EXPECTED)
731 * try rx rndis mode
743 MODULE_PARM_DESC(cppi_rx_rndis, "enable/disable RX RNDIS heuristic");
747 * cppi_next_rx_segment - dma read for the next chunk of a buffer
749 * @rx: dma channel
754 * See above notes about why we can't use multi-BD RX queues except in
762 cppi_next_rx_segment(struct musb *musb, struct cppi_channel *rx, int onepacket) in cppi_next_rx_segment() argument
764 unsigned maxpacket = rx->maxpacket; in cppi_next_rx_segment()
765 dma_addr_t addr = rx->buf_dma + rx->offset; in cppi_next_rx_segment()
766 size_t length = rx->buf_len - rx->offset; in cppi_next_rx_segment()
770 void __iomem *tibase = musb->ctrl_base; in cppi_next_rx_segment()
772 struct cppi_rx_stateram __iomem *rx_ram = rx->state_ram; in cppi_next_rx_segment()
785 && (length & (maxpacket - 1)) == 0) { in cppi_next_rx_segment()
809 n_bds = cppi_autoreq_update(rx, tibase, onepacket, n_bds); in cppi_next_rx_segment()
811 cppi_rndis_update(rx, 1, musb->ctrl_base, is_rndis); in cppi_next_rx_segment()
815 musb_dbg(musb, "RX DMA%d seg, maxp %d %s bds %d (cnt %d) " in cppi_next_rx_segment()
817 rx->index, maxpacket, in cppi_next_rx_segment()
823 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4)) in cppi_next_rx_segment()
826 rx->channel.actual_len, rx->buf_len); in cppi_next_rx_segment()
831 bd = cppi_bd_alloc(rx); in cppi_next_rx_segment()
832 rx->head = bd; in cppi_next_rx_segment()
839 bd = cppi_bd_alloc(rx); in cppi_next_rx_segment()
842 tail->next = bd; in cppi_next_rx_segment()
843 tail->hw_next = bd->dma; in cppi_next_rx_segment()
845 bd->hw_next = 0; in cppi_next_rx_segment()
853 bd->hw_bufp = addr; in cppi_next_rx_segment()
855 rx->offset += bd_len; in cppi_next_rx_segment()
857 bd->hw_off_len = (0 /*offset*/ << 16) + bd_len; in cppi_next_rx_segment()
858 bd->buflen = bd_len; in cppi_next_rx_segment()
860 bd->hw_options = CPPI_OWN_SET | (i == 0 ? length : 0); in cppi_next_rx_segment()
861 length -= bd_len; in cppi_next_rx_segment()
866 WARNING("rx dma%d -- no BDs? need %d\n", rx->index, n_bds); in cppi_next_rx_segment()
869 WARNING("rx dma%d -- only %d of %d BDs\n", rx->index, i, n_bds); in cppi_next_rx_segment()
871 tail->next = NULL; in cppi_next_rx_segment()
872 tail->hw_next = 0; in cppi_next_rx_segment()
874 bd = rx->head; in cppi_next_rx_segment()
875 rx->tail = tail; in cppi_next_rx_segment()
882 bd->hw_options |= CPPI_SOP_SET; in cppi_next_rx_segment()
883 tail->hw_options |= CPPI_EOP_SET; in cppi_next_rx_segment()
885 for (d = rx->head; d; d = d->next) in cppi_next_rx_segment()
889 tail = rx->last_processed; in cppi_next_rx_segment()
891 tail->next = bd; in cppi_next_rx_segment()
892 tail->hw_next = bd->dma; in cppi_next_rx_segment()
895 core_rxirq_enable(tibase, rx->index + 1); in cppi_next_rx_segment()
897 /* BDs live in DMA-coherent memory, but writes might be pending */ in cppi_next_rx_segment()
903 musb_writel(&rx_ram->rx_head, 0, bd->dma); in cppi_next_rx_segment()
910 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4)) in cppi_next_rx_segment()
915 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4), in cppi_next_rx_segment()
917 else if (n_bds > (i - 3)) in cppi_next_rx_segment()
919 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4), in cppi_next_rx_segment()
920 n_bds - (i - 3)); in cppi_next_rx_segment()
923 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4)) in cppi_next_rx_segment()
926 musb_dbg(musb, "bufcnt%d underrun - %d (for %d)", in cppi_next_rx_segment()
927 rx->index, i, n_bds); in cppi_next_rx_segment()
929 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4), in cppi_next_rx_segment()
933 cppi_dump_rx(4, rx, "/S"); in cppi_next_rx_segment()
937 * cppi_channel_program - program channel for data transfer
940 * @mode: For RX, 1 unless the usb protocol driver promised to treat
942 * For TX, ignored because of RNDIS mode races/glitches.
956 controller = cppi_ch->controller; in cppi_channel_program()
957 musb = controller->controller.musb; in cppi_channel_program()
959 switch (ch->status) { in cppi_channel_program()
964 cppi_ch->transmit ? 'T' : 'R', in cppi_channel_program()
965 cppi_ch->index); in cppi_channel_program()
970 cppi_ch->transmit ? 'T' : 'R', in cppi_channel_program()
971 cppi_ch->index); in cppi_channel_program()
976 cppi_ch->transmit ? 'T' : 'R', in cppi_channel_program()
977 cppi_ch->index); in cppi_channel_program()
983 ch->status = MUSB_DMA_STATUS_BUSY; in cppi_channel_program()
986 cppi_ch->buf_dma = dma_addr; in cppi_channel_program()
987 cppi_ch->offset = 0; in cppi_channel_program()
988 cppi_ch->maxpacket = maxpacket; in cppi_channel_program()
989 cppi_ch->buf_len = len; in cppi_channel_program()
990 cppi_ch->channel.actual_len = 0; in cppi_channel_program()
992 /* TX channel? or RX? */ in cppi_channel_program()
993 if (cppi_ch->transmit) in cppi_channel_program()
1003 struct cppi_channel *rx = &cppi->rx[ch]; in cppi_rx_scan() local
1004 struct cppi_rx_stateram __iomem *state = rx->state_ram; in cppi_rx_scan()
1006 struct cppi_descriptor *last = rx->last_processed; in cppi_rx_scan()
1011 void __iomem *regs = rx->hw_ep->regs; in cppi_rx_scan()
1012 struct musb *musb = cppi->controller.musb; in cppi_rx_scan()
1014 cppi_dump_rx(6, rx, "/K"); in cppi_rx_scan()
1016 bd = last ? last->next : rx->head; in cppi_rx_scan()
1021 for (i = 0, safe2ack = musb_readl(&state->rx_complete, 0); in cppi_rx_scan()
1023 i++, bd = bd->next) { in cppi_rx_scan()
1028 if (!completed && (bd->hw_options & CPPI_OWN_SET)) in cppi_rx_scan()
1033 (unsigned long long)bd->dma, bd->hw_next, bd->hw_bufp, in cppi_rx_scan()
1034 bd->hw_off_len, bd->hw_options, in cppi_rx_scan()
1035 rx->channel.actual_len); in cppi_rx_scan()
1038 if ((bd->hw_options & CPPI_SOP_SET) && !completed) in cppi_rx_scan()
1039 len = bd->hw_off_len & CPPI_RECV_PKTLEN_MASK; in cppi_rx_scan()
1043 if (bd->hw_options & CPPI_EOQ_MASK) in cppi_rx_scan()
1046 if (!completed && len < bd->buflen) { in cppi_rx_scan()
1053 musb_dbg(musb, "rx short %d/%d (%d)", in cppi_rx_scan()
1054 len, bd->buflen, in cppi_rx_scan()
1055 rx->channel.actual_len); in cppi_rx_scan()
1066 if (bd->dma == safe2ack) { in cppi_rx_scan()
1067 musb_writel(&state->rx_complete, 0, safe2ack); in cppi_rx_scan()
1068 safe2ack = musb_readl(&state->rx_complete, 0); in cppi_rx_scan()
1070 if (bd->dma == safe2ack) in cppi_rx_scan()
1074 rx->channel.actual_len += len; in cppi_rx_scan()
1076 cppi_bd_free(rx, last); in cppi_rx_scan()
1079 /* stop scanning on end-of-segment */ in cppi_rx_scan()
1080 if (bd->hw_next == 0) in cppi_rx_scan()
1083 rx->last_processed = last; in cppi_rx_scan()
1089 if (safe2ack == 0 || safe2ack == rx->last_processed->dma) in cppi_rx_scan()
1090 musb_writel(&state->rx_complete, 0, safe2ack); in cppi_rx_scan()
1092 cppi_bd_free(rx, last); in cppi_rx_scan()
1093 rx->last_processed = NULL; in cppi_rx_scan()
1098 WARN_ON(rx->head); in cppi_rx_scan()
1100 musb_ep_select(cppi->mregs, rx->index + 1); in cppi_rx_scan()
1104 rx->index, in cppi_rx_scan()
1105 rx->head, rx->tail, in cppi_rx_scan()
1106 rx->last_processed in cppi_rx_scan()
1108 rx->last_processed->dma in cppi_rx_scan()
1112 cppi_dump_rxq(4, "/what?", rx); in cppi_rx_scan()
1118 rx->head = bd; in cppi_rx_scan()
1123 csr = musb_readw(rx->hw_ep->regs, MUSB_RXCSR); in cppi_rx_scan()
1124 if (is_host_active(cppi->controller.musb) in cppi_rx_scan()
1130 csr = musb_readw(rx->hw_ep->regs, MUSB_RXCSR); in cppi_rx_scan()
1133 rx->head = NULL; in cppi_rx_scan()
1134 rx->tail = NULL; in cppi_rx_scan()
1137 cppi_dump_rx(6, rx, completed ? "/completed" : "/cleaned"); in cppi_rx_scan()
1147 u32 rx, tx; in cppi_interrupt() local
1151 cppi = container_of(musb->dma_controller, struct cppi, controller); in cppi_interrupt()
1152 if (cppi->irq) in cppi_interrupt()
1153 spin_lock_irqsave(&musb->lock, flags); in cppi_interrupt()
1155 tibase = musb->ctrl_base; in cppi_interrupt()
1157 tx = musb_readl(tibase, DAVINCI_TXCPPI_MASKED_REG); in cppi_interrupt()
1158 rx = musb_readl(tibase, DAVINCI_RXCPPI_MASKED_REG); in cppi_interrupt()
1160 if (!tx && !rx) { in cppi_interrupt()
1161 if (cppi->irq) in cppi_interrupt()
1162 spin_unlock_irqrestore(&musb->lock, flags); in cppi_interrupt()
1166 musb_dbg(musb, "CPPI IRQ Tx%x Rx%x", tx, rx); in cppi_interrupt()
1168 /* process TX channels */ in cppi_interrupt()
1169 for (index = 0; tx; tx = tx >> 1, index++) { in cppi_interrupt()
1175 if (!(tx & 1)) in cppi_interrupt()
1178 tx_ch = cppi->tx + index; in cppi_interrupt()
1179 tx_ram = tx_ch->state_ram; in cppi_interrupt()
1187 bd = tx_ch->head; in cppi_interrupt()
1195 musb_writel(&tx_ram->tx_complete, 0, 0); in cppi_interrupt()
1201 i++, bd = bd->next) { in cppi_interrupt()
1206 if (bd->hw_options & CPPI_OWN_SET) in cppi_interrupt()
1210 bd, bd->hw_next, bd->hw_bufp, in cppi_interrupt()
1211 bd->hw_off_len, bd->hw_options); in cppi_interrupt()
1213 len = bd->hw_off_len & CPPI_BUFFER_LEN_MASK; in cppi_interrupt()
1214 tx_ch->channel.actual_len += len; in cppi_interrupt()
1216 tx_ch->last_processed = bd; in cppi_interrupt()
1222 * REVISIT use the same ack strategy as rx in cppi_interrupt()
1226 /* if ((bd->hw_options & CPPI_EOQ_MASK)) */ in cppi_interrupt()
1227 musb_writel(&tx_ram->tx_complete, 0, bd->dma); in cppi_interrupt()
1229 /* stop scanning on end-of-segment */ in cppi_interrupt()
1230 if (bd->hw_next == 0) in cppi_interrupt()
1239 if (tx_ch->offset >= tx_ch->buf_len) { in cppi_interrupt()
1240 tx_ch->head = NULL; in cppi_interrupt()
1241 tx_ch->tail = NULL; in cppi_interrupt()
1242 tx_ch->channel.status = MUSB_DMA_STATUS_FREE; in cppi_interrupt()
1244 hw_ep = tx_ch->hw_ep; in cppi_interrupt()
1255 tx_ch->head = bd; in cppi_interrupt()
1258 /* Start processing the RX block */ in cppi_interrupt()
1259 for (index = 0; rx; rx = rx >> 1, index++) { in cppi_interrupt()
1261 if (rx & 1) { in cppi_interrupt()
1264 rx_ch = cppi->rx + index; in cppi_interrupt()
1271 if (rx_ch->channel.actual_len != rx_ch->buf_len in cppi_interrupt()
1272 && rx_ch->channel.actual_len in cppi_interrupt()
1273 == rx_ch->offset) { in cppi_interrupt()
1279 rx_ch->channel.status = MUSB_DMA_STATUS_FREE; in cppi_interrupt()
1281 hw_ep = rx_ch->hw_ep; in cppi_interrupt()
1288 /* write to CPPI EOI register to re-enable interrupts */ in cppi_interrupt()
1291 if (cppi->irq) in cppi_interrupt()
1292 spin_unlock_irqrestore(&musb->lock, flags); in cppi_interrupt()
1303 struct device *dev = musb->controller; in cppi_dma_controller_create()
1311 controller->mregs = mregs; in cppi_dma_controller_create()
1312 controller->tibase = mregs - DAVINCI_BASE_OFFSET; in cppi_dma_controller_create()
1314 controller->controller.musb = musb; in cppi_dma_controller_create()
1315 controller->controller.channel_alloc = cppi_channel_allocate; in cppi_dma_controller_create()
1316 controller->controller.channel_release = cppi_channel_release; in cppi_dma_controller_create()
1317 controller->controller.channel_program = cppi_channel_program; in cppi_dma_controller_create()
1318 controller->controller.channel_abort = cppi_channel_abort; in cppi_dma_controller_create()
1320 /* NOTE: allocating from on-chip SRAM would give the least in cppi_dma_controller_create()
1325 controller->pool = dma_pool_create("cppi", in cppi_dma_controller_create()
1326 controller->controller.musb->controller, in cppi_dma_controller_create()
1329 if (!controller->pool) { in cppi_dma_controller_create()
1335 if (request_irq(irq, cppi_interrupt, 0, "cppi-dma", musb)) { in cppi_dma_controller_create()
1337 musb_dma_controller_destroy(&controller->controller); in cppi_dma_controller_create()
1340 controller->irq = irq; in cppi_dma_controller_create()
1344 return &controller->controller; in cppi_dma_controller_create()
1349 * Destroy a previously-instantiated DMA controller.
1359 if (cppi->irq) in cppi_dma_controller_destroy()
1360 free_irq(cppi->irq, cppi->controller.musb); in cppi_dma_controller_destroy()
1363 dma_pool_destroy(cppi->pool); in cppi_dma_controller_destroy()
1384 controller = cppi_ch->controller; in cppi_channel_abort()
1386 switch (channel->status) { in cppi_channel_abort()
1389 /* from RX or TX fault irq handler */ in cppi_channel_abort()
1392 regs = cppi_ch->hw_ep->regs; in cppi_channel_abort()
1398 return -EINVAL; in cppi_channel_abort()
1401 if (!cppi_ch->transmit && cppi_ch->head) in cppi_channel_abort()
1404 mbase = controller->mregs; in cppi_channel_abort()
1405 tibase = controller->tibase; in cppi_channel_abort()
1407 queue = cppi_ch->head; in cppi_channel_abort()
1408 cppi_ch->head = NULL; in cppi_channel_abort()
1409 cppi_ch->tail = NULL; in cppi_channel_abort()
1415 musb_ep_select(mbase, cppi_ch->index + 1); in cppi_channel_abort()
1417 if (cppi_ch->transmit) { in cppi_channel_abort()
1427 musb_writel(tibase, DAVINCI_TXCPPI_TEAR_REG, cppi_ch->index); in cppi_channel_abort()
1429 tx_ram = cppi_ch->state_ram; in cppi_channel_abort()
1431 value = musb_readl(&tx_ram->tx_complete, 0); in cppi_channel_abort()
1452 cppi_ch->head = NULL; in cppi_channel_abort()
1453 musb_writel(&tx_ram->tx_complete, 0, 1); in cppi_channel_abort()
1456 /* REVISIT tx side _should_ clean up the same way in cppi_channel_abort()
1457 * as the RX side ... this does no cleanup at all! in cppi_channel_abort()
1460 } else /* RX */ { in cppi_channel_abort()
1466 * current RX DMA state iff any pending fifo transfer is done. in cppi_channel_abort()
1469 core_rxirq_disable(tibase, cppi_ch->index + 1); in cppi_channel_abort()
1472 if (is_host_active(cppi_ch->controller->controller.musb)) { in cppi_channel_abort()
1474 value &= ~((0x3) << (cppi_ch->index * 2)); in cppi_channel_abort()
1481 if (is_host_active(cppi_ch->controller->controller.musb)) { in cppi_channel_abort()
1493 * We can't use bit zero of stateram->rx_sop, since that in cppi_channel_abort()
1497 if (channel->status == MUSB_DMA_STATUS_BUSY) in cppi_channel_abort()
1503 cppi_rx_scan(controller, cppi_ch->index); in cppi_channel_abort()
1508 * RX channels to quiesce (how??) and then temporarily in cppi_channel_abort()
1516 cppi_reset_rx(cppi_ch->state_ram); in cppi_channel_abort()
1524 cppi_bd_free(cppi_ch, cppi_ch->last_processed); in cppi_channel_abort()
1525 cppi_ch->last_processed = NULL; in cppi_channel_abort()
1528 struct cppi_descriptor *tmp = queue->next; in cppi_channel_abort()
1535 channel->status = MUSB_DMA_STATUS_FREE; in cppi_channel_abort()
1536 cppi_ch->buf_dma = 0; in cppi_channel_abort()
1537 cppi_ch->offset = 0; in cppi_channel_abort()
1538 cppi_ch->buf_len = 0; in cppi_channel_abort()
1539 cppi_ch->maxpacket = 0; in cppi_channel_abort()