Lines Matching +full:rx +full:- +full:tx
2 * Copyright (C) 2005-2006 by Texas Instruments
5 * For now it's DaVinci-only, but CPPI isn't specific to DaVinci or USB.
18 /* CPPI DMA status 7-mar-2006:
20 * - See musb_{host,gadget}.c for more info
22 * - Correct RX DMA generally forces the engine into irq-per-packet mode,
23 * which can easily saturate the CPU under non-mass-storage loads.
25 * NOTES 24-aug-2006 (2.6.18-rc4):
27 * - peripheral RXDMA wedged in a test with packets of length 512/512/1.
30 * and RX DMA0: 4 left, 80000000 8feff880, 8feff860 8feff860; 8f321401
31 * 004001ff 00000001 .. 8feff860) Host was just getting NAKed on tx
35 * evidently also directly update the RX and TX CSRs ... so audit all
41 * more simply, switch to a global freelist not per-channel ones.
60 struct cppi_descriptor *bd = c->freelist; in cppi_bd_alloc()
63 c->freelist = bd->next; in cppi_bd_alloc()
72 bd->next = c->freelist; in cppi_bd_free()
73 c->freelist = bd; in cppi_bd_free()
82 /* zero out entire rx state RAM entry for the channel */
83 static void cppi_reset_rx(struct cppi_rx_stateram __iomem *rx) in cppi_reset_rx() argument
85 musb_writel(&rx->rx_skipbytes, 0, 0); in cppi_reset_rx()
86 musb_writel(&rx->rx_head, 0, 0); in cppi_reset_rx()
87 musb_writel(&rx->rx_sop, 0, 0); in cppi_reset_rx()
88 musb_writel(&rx->rx_current, 0, 0); in cppi_reset_rx()
89 musb_writel(&rx->rx_buf_current, 0, 0); in cppi_reset_rx()
90 musb_writel(&rx->rx_len_len, 0, 0); in cppi_reset_rx()
91 musb_writel(&rx->rx_cnt_cnt, 0, 0); in cppi_reset_rx()
94 /* zero out entire tx state RAM entry for the channel */
95 static void cppi_reset_tx(struct cppi_tx_stateram __iomem *tx, u32 ptr) in cppi_reset_tx() argument
97 musb_writel(&tx->tx_head, 0, 0); in cppi_reset_tx()
98 musb_writel(&tx->tx_buf, 0, 0); in cppi_reset_tx()
99 musb_writel(&tx->tx_current, 0, 0); in cppi_reset_tx()
100 musb_writel(&tx->tx_buf_current, 0, 0); in cppi_reset_tx()
101 musb_writel(&tx->tx_info, 0, 0); in cppi_reset_tx()
102 musb_writel(&tx->tx_rem_len, 0, 0); in cppi_reset_tx()
103 /* musb_writel(&tx->tx_dummy, 0, 0); */ in cppi_reset_tx()
104 musb_writel(&tx->tx_complete, 0, ptr); in cppi_reset_tx()
112 c->head = NULL; in cppi_pool_init()
113 c->tail = NULL; in cppi_pool_init()
114 c->last_processed = NULL; in cppi_pool_init()
115 c->channel.status = MUSB_DMA_STATUS_UNKNOWN; in cppi_pool_init()
116 c->controller = cppi; in cppi_pool_init()
117 c->is_rndis = 0; in cppi_pool_init()
118 c->freelist = NULL; in cppi_pool_init()
125 bd = dma_pool_alloc(cppi->pool, GFP_KERNEL, &dma); in cppi_pool_init()
126 bd->dma = dma; in cppi_pool_init()
135 struct cppi *cppi = c->controller; in cppi_pool_free()
138 (void) cppi_channel_abort(&c->channel); in cppi_pool_free()
139 c->channel.status = MUSB_DMA_STATUS_UNKNOWN; in cppi_pool_free()
140 c->controller = NULL; in cppi_pool_free()
143 bd = c->last_processed; in cppi_pool_free()
146 dma_pool_free(cppi->pool, bd, bd->dma); in cppi_pool_free()
149 c->last_processed = NULL; in cppi_pool_free()
161 for (i = 0; i < ARRAY_SIZE(controller->tx); i++) { in cppi_controller_start()
162 controller->tx[i].transmit = true; in cppi_controller_start()
163 controller->tx[i].index = i; in cppi_controller_start()
165 for (i = 0; i < ARRAY_SIZE(controller->rx); i++) { in cppi_controller_start()
166 controller->rx[i].transmit = false; in cppi_controller_start()
167 controller->rx[i].index = i; in cppi_controller_start()
171 for (i = 0; i < ARRAY_SIZE(controller->tx); i++) in cppi_controller_start()
172 cppi_pool_init(controller, controller->tx + i); in cppi_controller_start()
173 for (i = 0; i < ARRAY_SIZE(controller->rx); i++) in cppi_controller_start()
174 cppi_pool_init(controller, controller->rx + i); in cppi_controller_start()
176 tibase = controller->tibase; in cppi_controller_start()
177 INIT_LIST_HEAD(&controller->tx_complete); in cppi_controller_start()
179 /* initialise tx/rx channel head pointers to zero */ in cppi_controller_start()
180 for (i = 0; i < ARRAY_SIZE(controller->tx); i++) { in cppi_controller_start()
181 struct cppi_channel *tx_ch = controller->tx + i; in cppi_controller_start()
182 struct cppi_tx_stateram __iomem *tx; in cppi_controller_start() local
184 INIT_LIST_HEAD(&tx_ch->tx_complete); in cppi_controller_start()
186 tx = tibase + DAVINCI_TXCPPI_STATERAM_OFFSET(i); in cppi_controller_start()
187 tx_ch->state_ram = tx; in cppi_controller_start()
188 cppi_reset_tx(tx, 0); in cppi_controller_start()
190 for (i = 0; i < ARRAY_SIZE(controller->rx); i++) { in cppi_controller_start()
191 struct cppi_channel *rx_ch = controller->rx + i; in cppi_controller_start()
192 struct cppi_rx_stateram __iomem *rx; in cppi_controller_start() local
194 INIT_LIST_HEAD(&rx_ch->tx_complete); in cppi_controller_start()
196 rx = tibase + DAVINCI_RXCPPI_STATERAM_OFFSET(i); in cppi_controller_start()
197 rx_ch->state_ram = rx; in cppi_controller_start()
198 cppi_reset_rx(rx); in cppi_controller_start()
207 /* enable tx/rx CPPI control */ in cppi_controller_start()
211 /* disable RNDIS mode, also host rx RNDIS autorequest */ in cppi_controller_start()
221 * De-Init the DMA controller as necessary.
232 musb = controller->musb; in cppi_controller_stop()
234 tibase = controller->tibase; in cppi_controller_stop()
241 dev_dbg(musb->controller, "Tearing down RX and TX Channels\n"); in cppi_controller_stop()
242 for (i = 0; i < ARRAY_SIZE(controller->tx); i++) { in cppi_controller_stop()
244 controller->tx[i].last_processed = NULL; in cppi_controller_stop()
245 cppi_pool_free(controller->tx + i); in cppi_controller_stop()
247 for (i = 0; i < ARRAY_SIZE(controller->rx); i++) in cppi_controller_stop()
248 cppi_pool_free(controller->rx + i); in cppi_controller_stop()
250 /* in Tx Case proper teardown is supported. We resort to disabling in cppi_controller_stop()
251 * Tx/Rx CPPI after cleanup of Tx channels. Before TX teardown is in cppi_controller_stop()
252 * complete TX CPPI cannot be disabled. in cppi_controller_stop()
254 /*disable tx/rx cppi */ in cppi_controller_stop()
263 * Except for TX irqs, where dma done != fifo empty and reusable ...
267 * REVISIT same issue applies to pure PIO usage too, and non-cppi dma...
282 * each transfer direction of a non-control endpoint, so allocating
297 tibase = controller->tibase; in cppi_channel_allocate()
298 musb = controller->musb; in cppi_channel_allocate()
300 /* ep0 doesn't use DMA; remember cppi indices are 0..N-1 */ in cppi_channel_allocate()
301 index = ep->epnum - 1; in cppi_channel_allocate()
304 * probably disable the non-CPPI irq until we need it. in cppi_channel_allocate()
307 if (index >= ARRAY_SIZE(controller->tx)) { in cppi_channel_allocate()
308 dev_dbg(musb->controller, "no %cX%d CPPI channel\n", 'T', index); in cppi_channel_allocate()
311 cppi_ch = controller->tx + index; in cppi_channel_allocate()
313 if (index >= ARRAY_SIZE(controller->rx)) { in cppi_channel_allocate()
314 dev_dbg(musb->controller, "no %cX%d CPPI channel\n", 'R', index); in cppi_channel_allocate()
317 cppi_ch = controller->rx + index; in cppi_channel_allocate()
318 core_rxirq_disable(tibase, ep->epnum); in cppi_channel_allocate()
324 if (cppi_ch->hw_ep) in cppi_channel_allocate()
325 dev_dbg(musb->controller, "re-allocating DMA%d %cX channel %p\n", in cppi_channel_allocate()
327 cppi_ch->hw_ep = ep; in cppi_channel_allocate()
328 cppi_ch->channel.status = MUSB_DMA_STATUS_FREE; in cppi_channel_allocate()
329 cppi_ch->channel.max_len = 0x7fffffff; in cppi_channel_allocate()
331 dev_dbg(musb->controller, "Allocate CPPI%d %cX\n", index, transmit ? 'T' : 'R'); in cppi_channel_allocate()
332 return &cppi_ch->channel; in cppi_channel_allocate()
344 tibase = c->controller->tibase; in cppi_channel_release()
345 if (!c->hw_ep) in cppi_channel_release()
346 dev_dbg(c->controller->musb->controller, in cppi_channel_release()
348 else if (!c->transmit) in cppi_channel_release()
349 core_rxirq_enable(tibase, c->index + 1); in cppi_channel_release()
352 c->hw_ep = NULL; in cppi_channel_release()
353 channel->status = MUSB_DMA_STATUS_UNKNOWN; in cppi_channel_release()
360 void __iomem *base = c->controller->mregs; in cppi_dump_rx()
361 struct cppi_rx_stateram __iomem *rx = c->state_ram; in cppi_dump_rx() local
363 musb_ep_select(base, c->index + 1); in cppi_dump_rx()
365 dev_dbg(c->controller->musb->controller, in cppi_dump_rx()
366 "RX DMA%d%s: %d left, csr %04x, " in cppi_dump_rx()
370 c->index, tag, in cppi_dump_rx()
371 musb_readl(c->controller->tibase, in cppi_dump_rx()
372 DAVINCI_RXCPPI_BUFCNT0_REG + 4 * c->index), in cppi_dump_rx()
373 musb_readw(c->hw_ep->regs, MUSB_RXCSR), in cppi_dump_rx()
375 musb_readl(&rx->rx_skipbytes, 0), in cppi_dump_rx()
376 musb_readl(&rx->rx_head, 0), in cppi_dump_rx()
377 musb_readl(&rx->rx_sop, 0), in cppi_dump_rx()
378 musb_readl(&rx->rx_current, 0), in cppi_dump_rx()
380 musb_readl(&rx->rx_buf_current, 0), in cppi_dump_rx()
381 musb_readl(&rx->rx_len_len, 0), in cppi_dump_rx()
382 musb_readl(&rx->rx_cnt_cnt, 0), in cppi_dump_rx()
383 musb_readl(&rx->rx_complete, 0) in cppi_dump_rx()
391 void __iomem *base = c->controller->mregs; in cppi_dump_tx()
392 struct cppi_tx_stateram __iomem *tx = c->state_ram; in cppi_dump_tx() local
394 musb_ep_select(base, c->index + 1); in cppi_dump_tx()
396 dev_dbg(c->controller->musb->controller, in cppi_dump_tx()
397 "TX DMA%d%s: csr %04x, " in cppi_dump_tx()
401 c->index, tag, in cppi_dump_tx()
402 musb_readw(c->hw_ep->regs, MUSB_TXCSR), in cppi_dump_tx()
404 musb_readl(&tx->tx_head, 0), in cppi_dump_tx()
405 musb_readl(&tx->tx_buf, 0), in cppi_dump_tx()
406 musb_readl(&tx->tx_current, 0), in cppi_dump_tx()
407 musb_readl(&tx->tx_buf_current, 0), in cppi_dump_tx()
409 musb_readl(&tx->tx_info, 0), in cppi_dump_tx()
410 musb_readl(&tx->tx_rem_len, 0), in cppi_dump_tx()
412 musb_readl(&tx->tx_complete, 0) in cppi_dump_tx()
422 if (c->is_rndis != is_rndis) { in cppi_rndis_update()
424 u32 temp = 1 << (c->index); in cppi_rndis_update()
433 c->is_rndis = is_rndis; in cppi_rndis_update()
442 tag, bd->dma, in cppi_dump_rxbd()
443 bd->hw_next, bd->hw_bufp, bd->hw_off_len, in cppi_dump_rxbd()
444 bd->hw_options); in cppi_dump_rxbd()
448 static void cppi_dump_rxq(int level, const char *tag, struct cppi_channel *rx) in cppi_dump_rxq() argument
455 cppi_dump_rx(level, rx, tag); in cppi_dump_rxq()
456 if (rx->last_processed) in cppi_dump_rxq()
457 cppi_dump_rxbd("last", rx->last_processed); in cppi_dump_rxq()
458 for (bd = rx->head; bd; bd = bd->next) in cppi_dump_rxq()
464 /* NOTE: DaVinci autoreq is ignored except for host side "RNDIS" mode RX;
465 * so we won't ever use it (see "CPPI RX Woes" below).
467 static inline int cppi_autoreq_update(struct cppi_channel *rx, in cppi_autoreq_update() argument
478 val = tmp & ~((0x3) << (rx->index * 2)); in cppi_autoreq_update()
486 val |= ((0x3) << (rx->index * 2)); in cppi_autoreq_update()
487 n_bds--; in cppi_autoreq_update()
489 /* one segment, autoreq "all-but-last" */ in cppi_autoreq_update()
490 val |= ((0x1) << (rx->index * 2)); in cppi_autoreq_update()
504 } while (n-- > 0); in cppi_autoreq_update()
509 if (n_bds && rx->channel.actual_len) { in cppi_autoreq_update()
510 void __iomem *regs = rx->hw_ep->regs; in cppi_autoreq_update()
526 * - RX builds new queues each time, to help handle routine "early
530 * - for now, TX reuses the same queue of BDs every time
540 * in the TX paths. (RX can't do this, see below.)
542 * That's true even in the CPPI- friendly iso case, where most urbs have
544 * "transparent" DMA model is always correct, even on the RX side.
548 * CPPI TX:
550 * TX is a lot more reasonable than RX; it doesn't need to run in
551 * irq-per-packet mode very often. RNDIS mode seems to behave too
552 * (except how it handles the exactly-N-packets case). Building a
556 * The main issue with TX mode RNDIS relates to transfer lengths that
567 * On TX, "transparent" mode works ... although experiments have shown
575 cppi_next_tx_segment(struct musb *musb, struct cppi_channel *tx) in cppi_next_tx_segment() argument
577 unsigned maxpacket = tx->maxpacket; in cppi_next_tx_segment()
578 dma_addr_t addr = tx->buf_dma + tx->offset; in cppi_next_tx_segment()
579 size_t length = tx->buf_len - tx->offset; in cppi_next_tx_segment()
583 struct cppi_tx_stateram __iomem *tx_ram = tx->state_ram; in cppi_next_tx_segment()
586 /* TX can use the CPPI "rndis" mode, where we can probably fit this in cppi_next_tx_segment()
607 dev_dbg(musb->controller, "TX DMA%d, pktSz %d %s bds %d dma 0x%llx len %u\n", in cppi_next_tx_segment()
608 tx->index, in cppi_next_tx_segment()
614 cppi_rndis_update(tx, 0, musb->ctrl_base, rndis); in cppi_next_tx_segment()
622 bd = tx->freelist; in cppi_next_tx_segment()
623 tx->head = bd; in cppi_next_tx_segment()
624 tx->last_processed = NULL; in cppi_next_tx_segment()
626 /* FIXME use BD pool like RX side does, and just queue in cppi_next_tx_segment()
635 if (++i < n_bds && bd->next) in cppi_next_tx_segment()
636 bd->hw_next = bd->next->dma; in cppi_next_tx_segment()
638 bd->hw_next = 0; in cppi_next_tx_segment()
640 bd->hw_bufp = tx->buf_dma + tx->offset; in cppi_next_tx_segment()
645 if ((tx->offset + maxpacket) <= tx->buf_len) { in cppi_next_tx_segment()
646 tx->offset += maxpacket; in cppi_next_tx_segment()
647 bd->hw_off_len = maxpacket; in cppi_next_tx_segment()
648 bd->hw_options = CPPI_SOP_SET | CPPI_EOP_SET in cppi_next_tx_segment()
654 partial_len = tx->buf_len - tx->offset; in cppi_next_tx_segment()
655 tx->offset = tx->buf_len; in cppi_next_tx_segment()
656 bd->hw_off_len = partial_len; in cppi_next_tx_segment()
658 bd->hw_options = CPPI_SOP_SET | CPPI_EOP_SET in cppi_next_tx_segment()
661 bd->hw_options |= CPPI_ZERO_SET; in cppi_next_tx_segment()
664 dev_dbg(musb->controller, "TXBD %p: nxt %08x buf %08x len %04x opt %08x\n", in cppi_next_tx_segment()
665 bd, bd->hw_next, bd->hw_bufp, in cppi_next_tx_segment()
666 bd->hw_off_len, bd->hw_options); in cppi_next_tx_segment()
669 tx->tail = bd; in cppi_next_tx_segment()
670 bd = bd->next; in cppi_next_tx_segment()
673 /* BDs live in DMA-coherent memory, but writes might be pending */ in cppi_next_tx_segment()
677 musb_writel(&tx_ram->tx_head, 0, (u32)tx->freelist->dma); in cppi_next_tx_segment()
679 cppi_dump_tx(5, tx, "/S"); in cppi_next_tx_segment()
683 * CPPI RX Woes:
685 * Consider a 1KB bulk RX buffer in two scenarios: (a) it's fed two 300 byte
686 * packets back-to-back, and (b) it's fed two 512 byte packets back-to-back.
693 * - RX queues in "rndis" mode -- one single BD -- handle (a) correctly, but
696 * not a "true" RNDIS mode. In the RNDIS protocol short-packet termination
697 * is optional, and it's fine if peripherals -- not hosts! -- pad messages
698 * out to end-of-buffer. Standard PCI host controller DMA descriptors
701 * - RX queues in "transparent" mode -- two BDs with 512 bytes each -- have
702 * converse problems: (b) is handled right, but (a) loses badly. CPPI RX
706 * are intended as outputs for RX queues, not inputs...)
708 * - A variant of "transparent" mode -- one BD at a time -- is the only way to
713 * So how to get rid of IRQ-per-packet? The transparent multi-BD case could
720 * is appropriate -- which do NOT include RNDIS host drivers, but do include
721 * the CDC Ethernet driver! -- and the documentation is incomplete/wrong.
722 * So we can't _ever_ use RX RNDIS mode ... except by using a heuristic
725 * Leaving only "transparent" mode; we avoid multi-bd modes in almost all
740 * - less than 64KB (max per cppi descriptor)
741 * - not a multiple of 4096 (g_zero default, full reads typical)
742 * - N (>1) packets long, ditto (full reads not EXPECTED)
744 * try rx rndis mode
756 MODULE_PARM_DESC(cppi_rx_rndis, "enable/disable RX RNDIS heuristic");
760 * cppi_next_rx_segment - dma read for the next chunk of a buffer
762 * @rx: dma channel
767 * See above notes about why we can't use multi-BD RX queues except in
775 cppi_next_rx_segment(struct musb *musb, struct cppi_channel *rx, int onepacket) in cppi_next_rx_segment() argument
777 unsigned maxpacket = rx->maxpacket; in cppi_next_rx_segment()
778 dma_addr_t addr = rx->buf_dma + rx->offset; in cppi_next_rx_segment()
779 size_t length = rx->buf_len - rx->offset; in cppi_next_rx_segment()
783 void __iomem *tibase = musb->ctrl_base; in cppi_next_rx_segment()
785 struct cppi_rx_stateram __iomem *rx_ram = rx->state_ram; in cppi_next_rx_segment()
797 && (length & (maxpacket - 1)) == 0) { in cppi_next_rx_segment()
823 n_bds = cppi_autoreq_update(rx, tibase, onepacket, n_bds); in cppi_next_rx_segment()
825 cppi_rndis_update(rx, 1, musb->ctrl_base, is_rndis); in cppi_next_rx_segment()
829 dev_dbg(musb->controller, "RX DMA%d seg, maxp %d %s bds %d (cnt %d) " in cppi_next_rx_segment()
831 rx->index, maxpacket, in cppi_next_rx_segment()
837 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4)) in cppi_next_rx_segment()
840 rx->channel.actual_len, rx->buf_len); in cppi_next_rx_segment()
845 bd = cppi_bd_alloc(rx); in cppi_next_rx_segment()
846 rx->head = bd; in cppi_next_rx_segment()
853 bd = cppi_bd_alloc(rx); in cppi_next_rx_segment()
856 tail->next = bd; in cppi_next_rx_segment()
857 tail->hw_next = bd->dma; in cppi_next_rx_segment()
859 bd->hw_next = 0; in cppi_next_rx_segment()
867 bd->hw_bufp = addr; in cppi_next_rx_segment()
869 rx->offset += bd_len; in cppi_next_rx_segment()
871 bd->hw_off_len = (0 /*offset*/ << 16) + bd_len; in cppi_next_rx_segment()
872 bd->buflen = bd_len; in cppi_next_rx_segment()
874 bd->hw_options = CPPI_OWN_SET | (i == 0 ? length : 0); in cppi_next_rx_segment()
875 length -= bd_len; in cppi_next_rx_segment()
880 WARNING("rx dma%d -- no BDs? need %d\n", rx->index, n_bds); in cppi_next_rx_segment()
883 WARNING("rx dma%d -- only %d of %d BDs\n", rx->index, i, n_bds); in cppi_next_rx_segment()
885 tail->next = NULL; in cppi_next_rx_segment()
886 tail->hw_next = 0; in cppi_next_rx_segment()
888 bd = rx->head; in cppi_next_rx_segment()
889 rx->tail = tail; in cppi_next_rx_segment()
896 bd->hw_options |= CPPI_SOP_SET; in cppi_next_rx_segment()
897 tail->hw_options |= CPPI_EOP_SET; in cppi_next_rx_segment()
903 for (d = rx->head; d; d = d->next) in cppi_next_rx_segment()
909 tail = rx->last_processed; in cppi_next_rx_segment()
911 tail->next = bd; in cppi_next_rx_segment()
912 tail->hw_next = bd->dma; in cppi_next_rx_segment()
915 core_rxirq_enable(tibase, rx->index + 1); in cppi_next_rx_segment()
917 /* BDs live in DMA-coherent memory, but writes might be pending */ in cppi_next_rx_segment()
923 musb_writel(&rx_ram->rx_head, 0, bd->dma); in cppi_next_rx_segment()
930 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4)) in cppi_next_rx_segment()
935 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4), in cppi_next_rx_segment()
937 else if (n_bds > (i - 3)) in cppi_next_rx_segment()
939 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4), in cppi_next_rx_segment()
940 n_bds - (i - 3)); in cppi_next_rx_segment()
943 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4)) in cppi_next_rx_segment()
946 dev_dbg(musb->controller, "bufcnt%d underrun - %d (for %d)\n", in cppi_next_rx_segment()
947 rx->index, i, n_bds); in cppi_next_rx_segment()
949 DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4), in cppi_next_rx_segment()
953 cppi_dump_rx(4, rx, "/S"); in cppi_next_rx_segment()
957 * cppi_channel_program - program channel for data transfer
960 * @mode: For RX, 1 unless the usb protocol driver promised to treat
962 * For TX, ignored because of RNDIS mode races/glitches.
976 controller = cppi_ch->controller; in cppi_channel_program()
977 musb = controller->musb; in cppi_channel_program()
979 switch (ch->status) { in cppi_channel_program()
984 cppi_ch->transmit ? 'T' : 'R', in cppi_channel_program()
985 cppi_ch->index); in cppi_channel_program()
990 cppi_ch->transmit ? 'T' : 'R', in cppi_channel_program()
991 cppi_ch->index); in cppi_channel_program()
995 dev_dbg(musb->controller, "%cX DMA%d not allocated!\n", in cppi_channel_program()
996 cppi_ch->transmit ? 'T' : 'R', in cppi_channel_program()
997 cppi_ch->index); in cppi_channel_program()
1003 ch->status = MUSB_DMA_STATUS_BUSY; in cppi_channel_program()
1006 cppi_ch->buf_dma = dma_addr; in cppi_channel_program()
1007 cppi_ch->offset = 0; in cppi_channel_program()
1008 cppi_ch->maxpacket = maxpacket; in cppi_channel_program()
1009 cppi_ch->buf_len = len; in cppi_channel_program()
1010 cppi_ch->channel.actual_len = 0; in cppi_channel_program()
1012 /* TX channel? or RX? */ in cppi_channel_program()
1013 if (cppi_ch->transmit) in cppi_channel_program()
1023 struct cppi_channel *rx = &cppi->rx[ch]; in cppi_rx_scan() local
1024 struct cppi_rx_stateram __iomem *state = rx->state_ram; in cppi_rx_scan()
1026 struct cppi_descriptor *last = rx->last_processed; in cppi_rx_scan()
1031 void __iomem *regs = rx->hw_ep->regs; in cppi_rx_scan()
1032 struct musb *musb = cppi->musb; in cppi_rx_scan()
1034 cppi_dump_rx(6, rx, "/K"); in cppi_rx_scan()
1036 bd = last ? last->next : rx->head; in cppi_rx_scan()
1041 for (i = 0, safe2ack = musb_readl(&state->rx_complete, 0); in cppi_rx_scan()
1043 i++, bd = bd->next) { in cppi_rx_scan()
1048 if (!completed && (bd->hw_options & CPPI_OWN_SET)) in cppi_rx_scan()
1051 dev_dbg(musb->controller, "C/RXBD %llx: nxt %08x buf %08x " in cppi_rx_scan()
1053 (unsigned long long)bd->dma, bd->hw_next, bd->hw_bufp, in cppi_rx_scan()
1054 bd->hw_off_len, bd->hw_options, in cppi_rx_scan()
1055 rx->channel.actual_len); in cppi_rx_scan()
1058 if ((bd->hw_options & CPPI_SOP_SET) && !completed) in cppi_rx_scan()
1059 len = bd->hw_off_len & CPPI_RECV_PKTLEN_MASK; in cppi_rx_scan()
1063 if (bd->hw_options & CPPI_EOQ_MASK) in cppi_rx_scan()
1066 if (!completed && len < bd->buflen) { in cppi_rx_scan()
1073 dev_dbg(musb->controller, "rx short %d/%d (%d)\n", in cppi_rx_scan()
1074 len, bd->buflen, in cppi_rx_scan()
1075 rx->channel.actual_len); in cppi_rx_scan()
1086 if (bd->dma == safe2ack) { in cppi_rx_scan()
1087 musb_writel(&state->rx_complete, 0, safe2ack); in cppi_rx_scan()
1088 safe2ack = musb_readl(&state->rx_complete, 0); in cppi_rx_scan()
1090 if (bd->dma == safe2ack) in cppi_rx_scan()
1094 rx->channel.actual_len += len; in cppi_rx_scan()
1096 cppi_bd_free(rx, last); in cppi_rx_scan()
1099 /* stop scanning on end-of-segment */ in cppi_rx_scan()
1100 if (bd->hw_next == 0) in cppi_rx_scan()
1103 rx->last_processed = last; in cppi_rx_scan()
1109 if (safe2ack == 0 || safe2ack == rx->last_processed->dma) in cppi_rx_scan()
1110 musb_writel(&state->rx_complete, 0, safe2ack); in cppi_rx_scan()
1112 cppi_bd_free(rx, last); in cppi_rx_scan()
1113 rx->last_processed = NULL; in cppi_rx_scan()
1118 WARN_ON(rx->head); in cppi_rx_scan()
1120 musb_ep_select(cppi->mregs, rx->index + 1); in cppi_rx_scan()
1123 dev_dbg(musb->controller, "list%d %p/%p, last %llx%s, csr %04x\n", in cppi_rx_scan()
1124 rx->index, in cppi_rx_scan()
1125 rx->head, rx->tail, in cppi_rx_scan()
1126 rx->last_processed in cppi_rx_scan()
1128 rx->last_processed->dma in cppi_rx_scan()
1132 cppi_dump_rxq(4, "/what?", rx); in cppi_rx_scan()
1138 rx->head = bd; in cppi_rx_scan()
1143 csr = musb_readw(rx->hw_ep->regs, MUSB_RXCSR); in cppi_rx_scan()
1144 if (is_host_active(cppi->musb) in cppi_rx_scan()
1150 csr = musb_readw(rx->hw_ep->regs, MUSB_RXCSR); in cppi_rx_scan()
1153 rx->head = NULL; in cppi_rx_scan()
1154 rx->tail = NULL; in cppi_rx_scan()
1157 cppi_dump_rx(6, rx, completed ? "/completed" : "/cleaned"); in cppi_rx_scan()
1167 u32 rx, tx; in cppi_interrupt() local
1171 cppi = container_of(musb->dma_controller, struct cppi, controller); in cppi_interrupt()
1172 if (cppi->irq) in cppi_interrupt()
1173 spin_lock_irqsave(&musb->lock, flags); in cppi_interrupt()
1175 tibase = musb->ctrl_base; in cppi_interrupt()
1177 tx = musb_readl(tibase, DAVINCI_TXCPPI_MASKED_REG); in cppi_interrupt()
1178 rx = musb_readl(tibase, DAVINCI_RXCPPI_MASKED_REG); in cppi_interrupt()
1180 if (!tx && !rx) { in cppi_interrupt()
1181 if (cppi->irq) in cppi_interrupt()
1182 spin_unlock_irqrestore(&musb->lock, flags); in cppi_interrupt()
1186 dev_dbg(musb->controller, "CPPI IRQ Tx%x Rx%x\n", tx, rx); in cppi_interrupt()
1188 /* process TX channels */ in cppi_interrupt()
1189 for (index = 0; tx; tx = tx >> 1, index++) { in cppi_interrupt()
1195 if (!(tx & 1)) in cppi_interrupt()
1198 tx_ch = cppi->tx + index; in cppi_interrupt()
1199 tx_ram = tx_ch->state_ram; in cppi_interrupt()
1207 bd = tx_ch->head; in cppi_interrupt()
1214 dev_dbg(musb->controller, "null BD\n"); in cppi_interrupt()
1215 musb_writel(&tx_ram->tx_complete, 0, 0); in cppi_interrupt()
1221 i++, bd = bd->next) { in cppi_interrupt()
1226 if (bd->hw_options & CPPI_OWN_SET) in cppi_interrupt()
1229 dev_dbg(musb->controller, "C/TXBD %p n %x b %x off %x opt %x\n", in cppi_interrupt()
1230 bd, bd->hw_next, bd->hw_bufp, in cppi_interrupt()
1231 bd->hw_off_len, bd->hw_options); in cppi_interrupt()
1233 len = bd->hw_off_len & CPPI_BUFFER_LEN_MASK; in cppi_interrupt()
1234 tx_ch->channel.actual_len += len; in cppi_interrupt()
1236 tx_ch->last_processed = bd; in cppi_interrupt()
1242 * REVISIT use the same ack strategy as rx in cppi_interrupt()
1246 /* if ((bd->hw_options & CPPI_EOQ_MASK)) */ in cppi_interrupt()
1247 musb_writel(&tx_ram->tx_complete, 0, bd->dma); in cppi_interrupt()
1249 /* stop scanning on end-of-segment */ in cppi_interrupt()
1250 if (bd->hw_next == 0) in cppi_interrupt()
1259 if (tx_ch->offset >= tx_ch->buf_len) { in cppi_interrupt()
1260 tx_ch->head = NULL; in cppi_interrupt()
1261 tx_ch->tail = NULL; in cppi_interrupt()
1262 tx_ch->channel.status = MUSB_DMA_STATUS_FREE; in cppi_interrupt()
1264 hw_ep = tx_ch->hw_ep; in cppi_interrupt()
1275 tx_ch->head = bd; in cppi_interrupt()
1278 /* Start processing the RX block */ in cppi_interrupt()
1279 for (index = 0; rx; rx = rx >> 1, index++) { in cppi_interrupt()
1281 if (rx & 1) { in cppi_interrupt()
1284 rx_ch = cppi->rx + index; in cppi_interrupt()
1291 if (rx_ch->channel.actual_len != rx_ch->buf_len in cppi_interrupt()
1292 && rx_ch->channel.actual_len in cppi_interrupt()
1293 == rx_ch->offset) { in cppi_interrupt()
1299 rx_ch->channel.status = MUSB_DMA_STATUS_FREE; in cppi_interrupt()
1301 hw_ep = rx_ch->hw_ep; in cppi_interrupt()
1308 /* write to CPPI EOI register to re-enable interrupts */ in cppi_interrupt()
1311 if (cppi->irq) in cppi_interrupt()
1312 spin_unlock_irqrestore(&musb->lock, flags); in cppi_interrupt()
1322 struct device *dev = musb->controller; in dma_controller_create()
1330 controller->mregs = mregs; in dma_controller_create()
1331 controller->tibase = mregs - DAVINCI_BASE_OFFSET; in dma_controller_create()
1333 controller->musb = musb; in dma_controller_create()
1334 controller->controller.start = cppi_controller_start; in dma_controller_create()
1335 controller->controller.stop = cppi_controller_stop; in dma_controller_create()
1336 controller->controller.channel_alloc = cppi_channel_allocate; in dma_controller_create()
1337 controller->controller.channel_release = cppi_channel_release; in dma_controller_create()
1338 controller->controller.channel_program = cppi_channel_program; in dma_controller_create()
1339 controller->controller.channel_abort = cppi_channel_abort; in dma_controller_create()
1341 /* NOTE: allocating from on-chip SRAM would give the least in dma_controller_create()
1346 controller->pool = dma_pool_create("cppi", in dma_controller_create()
1347 controller->musb->controller, in dma_controller_create()
1350 if (!controller->pool) { in dma_controller_create()
1356 if (request_irq(irq, cppi_interrupt, 0, "cppi-dma", musb)) { in dma_controller_create()
1358 dma_controller_destroy(&controller->controller); in dma_controller_create()
1361 controller->irq = irq; in dma_controller_create()
1364 return &controller->controller; in dma_controller_create()
1368 * Destroy a previously-instantiated DMA controller.
1376 if (cppi->irq) in dma_controller_destroy()
1377 free_irq(cppi->irq, cppi->musb); in dma_controller_destroy()
1380 dma_pool_destroy(cppi->pool); in dma_controller_destroy()
1400 controller = cppi_ch->controller; in cppi_channel_abort()
1402 switch (channel->status) { in cppi_channel_abort()
1405 /* from RX or TX fault irq handler */ in cppi_channel_abort()
1408 regs = cppi_ch->hw_ep->regs; in cppi_channel_abort()
1414 return -EINVAL; in cppi_channel_abort()
1417 if (!cppi_ch->transmit && cppi_ch->head) in cppi_channel_abort()
1420 mbase = controller->mregs; in cppi_channel_abort()
1421 tibase = controller->tibase; in cppi_channel_abort()
1423 queue = cppi_ch->head; in cppi_channel_abort()
1424 cppi_ch->head = NULL; in cppi_channel_abort()
1425 cppi_ch->tail = NULL; in cppi_channel_abort()
1431 musb_ep_select(mbase, cppi_ch->index + 1); in cppi_channel_abort()
1433 if (cppi_ch->transmit) { in cppi_channel_abort()
1443 musb_writel(tibase, DAVINCI_TXCPPI_TEAR_REG, cppi_ch->index); in cppi_channel_abort()
1445 tx_ram = cppi_ch->state_ram; in cppi_channel_abort()
1447 value = musb_readl(&tx_ram->tx_complete, 0); in cppi_channel_abort()
1468 cppi_ch->head = NULL; in cppi_channel_abort()
1469 musb_writel(&tx_ram->tx_complete, 0, 1); in cppi_channel_abort()
1472 /* REVISIT tx side _should_ clean up the same way in cppi_channel_abort()
1473 * as the RX side ... this does no cleanup at all! in cppi_channel_abort()
1476 } else /* RX */ { in cppi_channel_abort()
1482 * current RX DMA state iff any pending fifo transfer is done. in cppi_channel_abort()
1485 core_rxirq_disable(tibase, cppi_ch->index + 1); in cppi_channel_abort()
1488 if (is_host_active(cppi_ch->controller->musb)) { in cppi_channel_abort()
1490 value &= ~((0x3) << (cppi_ch->index * 2)); in cppi_channel_abort()
1497 if (is_host_active(cppi_ch->controller->musb)) { in cppi_channel_abort()
1509 * We can't use bit zero of stateram->rx_sop, since that in cppi_channel_abort()
1513 if (channel->status == MUSB_DMA_STATUS_BUSY) in cppi_channel_abort()
1519 cppi_rx_scan(controller, cppi_ch->index); in cppi_channel_abort()
1524 * RX channels to quiesce (how??) and then temporarily in cppi_channel_abort()
1532 cppi_reset_rx(cppi_ch->state_ram); in cppi_channel_abort()
1540 cppi_bd_free(cppi_ch, cppi_ch->last_processed); in cppi_channel_abort()
1541 cppi_ch->last_processed = NULL; in cppi_channel_abort()
1544 struct cppi_descriptor *tmp = queue->next; in cppi_channel_abort()
1551 channel->status = MUSB_DMA_STATUS_FREE; in cppi_channel_abort()
1552 cppi_ch->buf_dma = 0; in cppi_channel_abort()
1553 cppi_ch->offset = 0; in cppi_channel_abort()
1554 cppi_ch->buf_len = 0; in cppi_channel_abort()
1555 cppi_ch->maxpacket = 0; in cppi_channel_abort()