1 /*
2 * arch/arm/mach-tegra/dma.c
3 *
4 * System DMA driver for NVIDIA Tegra SoCs
5 *
6 * Copyright (c) 2008-2009, NVIDIA Corporation.
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2 of the License, or
11 * (at your option) any later version.
12 *
13 * This program is distributed in the hope that it will be useful, but WITHOUT
14 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 * more details.
17 *
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, write to the Free Software Foundation, Inc.,
20 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
21 */
22
23 #include <linux/io.h>
24 #include <linux/interrupt.h>
25 #include <linux/module.h>
26 #include <linux/spinlock.h>
27 #include <linux/err.h>
28 #include <linux/irq.h>
29 #include <linux/delay.h>
30 #include <linux/clk.h>
31 #include <mach/dma.h>
32 #include <mach/irqs.h>
33 #include <mach/iomap.h>
34 #include <mach/suspend.h>
35
36 #define APB_DMA_GEN 0x000
37 #define GEN_ENABLE (1<<31)
38
39 #define APB_DMA_CNTRL 0x010
40
41 #define APB_DMA_IRQ_MASK 0x01c
42
43 #define APB_DMA_IRQ_MASK_SET 0x020
44
45 #define APB_DMA_CHAN_CSR 0x000
46 #define CSR_ENB (1<<31)
47 #define CSR_IE_EOC (1<<30)
48 #define CSR_HOLD (1<<29)
49 #define CSR_DIR (1<<28)
50 #define CSR_ONCE (1<<27)
51 #define CSR_FLOW (1<<21)
52 #define CSR_REQ_SEL_SHIFT 16
53 #define CSR_REQ_SEL_MASK (0x1F<<CSR_REQ_SEL_SHIFT)
54 #define CSR_REQ_SEL_INVALID (31<<CSR_REQ_SEL_SHIFT)
55 #define CSR_WCOUNT_SHIFT 2
56 #define CSR_WCOUNT_MASK 0xFFFC
57
58 #define APB_DMA_CHAN_STA 0x004
59 #define STA_BUSY (1<<31)
60 #define STA_ISE_EOC (1<<30)
61 #define STA_HALT (1<<29)
62 #define STA_PING_PONG (1<<28)
63 #define STA_COUNT_SHIFT 2
64 #define STA_COUNT_MASK 0xFFFC
65
66 #define APB_DMA_CHAN_AHB_PTR 0x010
67
68 #define APB_DMA_CHAN_AHB_SEQ 0x014
69 #define AHB_SEQ_INTR_ENB (1<<31)
70 #define AHB_SEQ_BUS_WIDTH_SHIFT 28
71 #define AHB_SEQ_BUS_WIDTH_MASK (0x7<<AHB_SEQ_BUS_WIDTH_SHIFT)
72 #define AHB_SEQ_BUS_WIDTH_8 (0<<AHB_SEQ_BUS_WIDTH_SHIFT)
73 #define AHB_SEQ_BUS_WIDTH_16 (1<<AHB_SEQ_BUS_WIDTH_SHIFT)
74 #define AHB_SEQ_BUS_WIDTH_32 (2<<AHB_SEQ_BUS_WIDTH_SHIFT)
75 #define AHB_SEQ_BUS_WIDTH_64 (3<<AHB_SEQ_BUS_WIDTH_SHIFT)
76 #define AHB_SEQ_BUS_WIDTH_128 (4<<AHB_SEQ_BUS_WIDTH_SHIFT)
77 #define AHB_SEQ_DATA_SWAP (1<<27)
78 #define AHB_SEQ_BURST_MASK (0x7<<24)
79 #define AHB_SEQ_BURST_1 (4<<24)
80 #define AHB_SEQ_BURST_4 (5<<24)
81 #define AHB_SEQ_BURST_8 (6<<24)
82 #define AHB_SEQ_DBL_BUF (1<<19)
83 #define AHB_SEQ_WRAP_SHIFT 16
84 #define AHB_SEQ_WRAP_MASK (0x7<<AHB_SEQ_WRAP_SHIFT)
85
86 #define APB_DMA_CHAN_APB_PTR 0x018
87
88 #define APB_DMA_CHAN_APB_SEQ 0x01c
89 #define APB_SEQ_BUS_WIDTH_SHIFT 28
90 #define APB_SEQ_BUS_WIDTH_MASK (0x7<<APB_SEQ_BUS_WIDTH_SHIFT)
91 #define APB_SEQ_BUS_WIDTH_8 (0<<APB_SEQ_BUS_WIDTH_SHIFT)
92 #define APB_SEQ_BUS_WIDTH_16 (1<<APB_SEQ_BUS_WIDTH_SHIFT)
93 #define APB_SEQ_BUS_WIDTH_32 (2<<APB_SEQ_BUS_WIDTH_SHIFT)
94 #define APB_SEQ_BUS_WIDTH_64 (3<<APB_SEQ_BUS_WIDTH_SHIFT)
95 #define APB_SEQ_BUS_WIDTH_128 (4<<APB_SEQ_BUS_WIDTH_SHIFT)
96 #define APB_SEQ_DATA_SWAP (1<<27)
97 #define APB_SEQ_WRAP_SHIFT 16
98 #define APB_SEQ_WRAP_MASK (0x7<<APB_SEQ_WRAP_SHIFT)
99
100 #define TEGRA_SYSTEM_DMA_CH_NR 16
101 #define TEGRA_SYSTEM_DMA_AVP_CH_NUM 4
102 #define TEGRA_SYSTEM_DMA_CH_MIN 0
103 #define TEGRA_SYSTEM_DMA_CH_MAX \
104 (TEGRA_SYSTEM_DMA_CH_NR - TEGRA_SYSTEM_DMA_AVP_CH_NUM - 1)
105
106 #define NV_DMA_MAX_TRASFER_SIZE 0x10000
107
108 static const unsigned int ahb_addr_wrap_table[8] = {
109 0, 32, 64, 128, 256, 512, 1024, 2048
110 };
111
112 static const unsigned int apb_addr_wrap_table[8] = {
113 0, 1, 2, 4, 8, 16, 32, 64
114 };
115
116 static const unsigned int bus_width_table[5] = {
117 8, 16, 32, 64, 128
118 };
119
120 #define TEGRA_DMA_NAME_SIZE 16
121 struct tegra_dma_channel {
122 struct list_head list;
123 int id;
124 spinlock_t lock;
125 char name[TEGRA_DMA_NAME_SIZE];
126 void __iomem *addr;
127 int mode;
128 int irq;
129 int req_transfer_count;
130 };
131
132 #define NV_DMA_MAX_CHANNELS 32
133
134 static bool tegra_dma_initialized;
135 static DEFINE_MUTEX(tegra_dma_lock);
136
137 static DECLARE_BITMAP(channel_usage, NV_DMA_MAX_CHANNELS);
138 static struct tegra_dma_channel dma_channels[NV_DMA_MAX_CHANNELS];
139
140 static void tegra_dma_update_hw(struct tegra_dma_channel *ch,
141 struct tegra_dma_req *req);
142 static void tegra_dma_update_hw_partial(struct tegra_dma_channel *ch,
143 struct tegra_dma_req *req);
144 static void tegra_dma_stop(struct tegra_dma_channel *ch);
145
tegra_dma_flush(struct tegra_dma_channel * ch)146 void tegra_dma_flush(struct tegra_dma_channel *ch)
147 {
148 }
149 EXPORT_SYMBOL(tegra_dma_flush);
150
tegra_dma_dequeue(struct tegra_dma_channel * ch)151 void tegra_dma_dequeue(struct tegra_dma_channel *ch)
152 {
153 struct tegra_dma_req *req;
154
155 if (tegra_dma_is_empty(ch))
156 return;
157
158 req = list_entry(ch->list.next, typeof(*req), node);
159
160 tegra_dma_dequeue_req(ch, req);
161 return;
162 }
163
tegra_dma_stop(struct tegra_dma_channel * ch)164 static void tegra_dma_stop(struct tegra_dma_channel *ch)
165 {
166 u32 csr;
167 u32 status;
168
169 csr = readl(ch->addr + APB_DMA_CHAN_CSR);
170 csr &= ~CSR_IE_EOC;
171 writel(csr, ch->addr + APB_DMA_CHAN_CSR);
172
173 csr &= ~CSR_ENB;
174 writel(csr, ch->addr + APB_DMA_CHAN_CSR);
175
176 status = readl(ch->addr + APB_DMA_CHAN_STA);
177 if (status & STA_ISE_EOC)
178 writel(status, ch->addr + APB_DMA_CHAN_STA);
179 }
180
tegra_dma_cancel(struct tegra_dma_channel * ch)181 static int tegra_dma_cancel(struct tegra_dma_channel *ch)
182 {
183 u32 csr;
184 unsigned long irq_flags;
185
186 spin_lock_irqsave(&ch->lock, irq_flags);
187 while (!list_empty(&ch->list))
188 list_del(ch->list.next);
189
190 csr = readl(ch->addr + APB_DMA_CHAN_CSR);
191 csr &= ~CSR_REQ_SEL_MASK;
192 csr |= CSR_REQ_SEL_INVALID;
193 writel(csr, ch->addr + APB_DMA_CHAN_CSR);
194
195 tegra_dma_stop(ch);
196
197 spin_unlock_irqrestore(&ch->lock, irq_flags);
198 return 0;
199 }
200
tegra_dma_dequeue_req(struct tegra_dma_channel * ch,struct tegra_dma_req * _req)201 int tegra_dma_dequeue_req(struct tegra_dma_channel *ch,
202 struct tegra_dma_req *_req)
203 {
204 unsigned int csr;
205 unsigned int status;
206 struct tegra_dma_req *req = NULL;
207 int found = 0;
208 unsigned long irq_flags;
209 int to_transfer;
210 int req_transfer_count;
211
212 spin_lock_irqsave(&ch->lock, irq_flags);
213 list_for_each_entry(req, &ch->list, node) {
214 if (req == _req) {
215 list_del(&req->node);
216 found = 1;
217 break;
218 }
219 }
220 if (!found) {
221 spin_unlock_irqrestore(&ch->lock, irq_flags);
222 return 0;
223 }
224
225 /* STOP the DMA and get the transfer count.
226 * Getting the transfer count is tricky.
227 * - Change the source selector to invalid to stop the DMA from
228 * FIFO to memory.
229 * - Read the status register to know the number of pending
230 * bytes to be transferred.
231 * - Finally stop or program the DMA to the next buffer in the
232 * list.
233 */
234 csr = readl(ch->addr + APB_DMA_CHAN_CSR);
235 csr &= ~CSR_REQ_SEL_MASK;
236 csr |= CSR_REQ_SEL_INVALID;
237 writel(csr, ch->addr + APB_DMA_CHAN_CSR);
238
239 /* Get the transfer count */
240 status = readl(ch->addr + APB_DMA_CHAN_STA);
241 to_transfer = (status & STA_COUNT_MASK) >> STA_COUNT_SHIFT;
242 req_transfer_count = ch->req_transfer_count;
243 req_transfer_count += 1;
244 to_transfer += 1;
245
246 req->bytes_transferred = req_transfer_count;
247
248 if (status & STA_BUSY)
249 req->bytes_transferred -= to_transfer;
250
251 /* In continuous transfer mode, DMA only tracks the count of the
252 * half DMA buffer. So, if the DMA already finished half the DMA
253 * then add the half buffer to the completed count.
254 *
255 * FIXME: There can be a race here. What if the req to
256 * dequue happens at the same time as the DMA just moved to
257 * the new buffer and SW didn't yet received the interrupt?
258 */
259 if (ch->mode & TEGRA_DMA_MODE_CONTINOUS)
260 if (req->buffer_status == TEGRA_DMA_REQ_BUF_STATUS_HALF_FULL)
261 req->bytes_transferred += req_transfer_count;
262
263 req->bytes_transferred *= 4;
264
265 tegra_dma_stop(ch);
266 if (!list_empty(&ch->list)) {
267 /* if the list is not empty, queue the next request */
268 struct tegra_dma_req *next_req;
269 next_req = list_entry(ch->list.next,
270 typeof(*next_req), node);
271 tegra_dma_update_hw(ch, next_req);
272 }
273 req->status = -TEGRA_DMA_REQ_ERROR_ABORTED;
274
275 spin_unlock_irqrestore(&ch->lock, irq_flags);
276
277 /* Callback should be called without any lock */
278 req->complete(req);
279 return 0;
280 }
281 EXPORT_SYMBOL(tegra_dma_dequeue_req);
282
tegra_dma_is_empty(struct tegra_dma_channel * ch)283 bool tegra_dma_is_empty(struct tegra_dma_channel *ch)
284 {
285 unsigned long irq_flags;
286 bool is_empty;
287
288 spin_lock_irqsave(&ch->lock, irq_flags);
289 if (list_empty(&ch->list))
290 is_empty = true;
291 else
292 is_empty = false;
293 spin_unlock_irqrestore(&ch->lock, irq_flags);
294 return is_empty;
295 }
296 EXPORT_SYMBOL(tegra_dma_is_empty);
297
tegra_dma_is_req_inflight(struct tegra_dma_channel * ch,struct tegra_dma_req * _req)298 bool tegra_dma_is_req_inflight(struct tegra_dma_channel *ch,
299 struct tegra_dma_req *_req)
300 {
301 unsigned long irq_flags;
302 struct tegra_dma_req *req;
303
304 spin_lock_irqsave(&ch->lock, irq_flags);
305 list_for_each_entry(req, &ch->list, node) {
306 if (req == _req) {
307 spin_unlock_irqrestore(&ch->lock, irq_flags);
308 return true;
309 }
310 }
311 spin_unlock_irqrestore(&ch->lock, irq_flags);
312 return false;
313 }
314 EXPORT_SYMBOL(tegra_dma_is_req_inflight);
315
tegra_dma_enqueue_req(struct tegra_dma_channel * ch,struct tegra_dma_req * req)316 int tegra_dma_enqueue_req(struct tegra_dma_channel *ch,
317 struct tegra_dma_req *req)
318 {
319 unsigned long irq_flags;
320 struct tegra_dma_req *_req;
321 int start_dma = 0;
322
323 if (req->size > NV_DMA_MAX_TRASFER_SIZE ||
324 req->source_addr & 0x3 || req->dest_addr & 0x3) {
325 pr_err("Invalid DMA request for channel %d\n", ch->id);
326 return -EINVAL;
327 }
328
329 spin_lock_irqsave(&ch->lock, irq_flags);
330
331 list_for_each_entry(_req, &ch->list, node) {
332 if (req == _req) {
333 spin_unlock_irqrestore(&ch->lock, irq_flags);
334 return -EEXIST;
335 }
336 }
337
338 req->bytes_transferred = 0;
339 req->status = 0;
340 req->buffer_status = 0;
341 if (list_empty(&ch->list))
342 start_dma = 1;
343
344 list_add_tail(&req->node, &ch->list);
345
346 if (start_dma)
347 tegra_dma_update_hw(ch, req);
348
349 spin_unlock_irqrestore(&ch->lock, irq_flags);
350
351 return 0;
352 }
353 EXPORT_SYMBOL(tegra_dma_enqueue_req);
354
tegra_dma_allocate_channel(int mode)355 struct tegra_dma_channel *tegra_dma_allocate_channel(int mode)
356 {
357 int channel;
358 struct tegra_dma_channel *ch = NULL;
359
360 if (WARN_ON(!tegra_dma_initialized))
361 return NULL;
362
363 mutex_lock(&tegra_dma_lock);
364
365 /* first channel is the shared channel */
366 if (mode & TEGRA_DMA_SHARED) {
367 channel = TEGRA_SYSTEM_DMA_CH_MIN;
368 } else {
369 channel = find_first_zero_bit(channel_usage,
370 ARRAY_SIZE(dma_channels));
371 if (channel >= ARRAY_SIZE(dma_channels))
372 goto out;
373 }
374 __set_bit(channel, channel_usage);
375 ch = &dma_channels[channel];
376 ch->mode = mode;
377
378 out:
379 mutex_unlock(&tegra_dma_lock);
380 return ch;
381 }
382 EXPORT_SYMBOL(tegra_dma_allocate_channel);
383
tegra_dma_free_channel(struct tegra_dma_channel * ch)384 void tegra_dma_free_channel(struct tegra_dma_channel *ch)
385 {
386 if (ch->mode & TEGRA_DMA_SHARED)
387 return;
388 tegra_dma_cancel(ch);
389 mutex_lock(&tegra_dma_lock);
390 __clear_bit(ch->id, channel_usage);
391 mutex_unlock(&tegra_dma_lock);
392 }
393 EXPORT_SYMBOL(tegra_dma_free_channel);
394
tegra_dma_update_hw_partial(struct tegra_dma_channel * ch,struct tegra_dma_req * req)395 static void tegra_dma_update_hw_partial(struct tegra_dma_channel *ch,
396 struct tegra_dma_req *req)
397 {
398 u32 apb_ptr;
399 u32 ahb_ptr;
400
401 if (req->to_memory) {
402 apb_ptr = req->source_addr;
403 ahb_ptr = req->dest_addr;
404 } else {
405 apb_ptr = req->dest_addr;
406 ahb_ptr = req->source_addr;
407 }
408 writel(apb_ptr, ch->addr + APB_DMA_CHAN_APB_PTR);
409 writel(ahb_ptr, ch->addr + APB_DMA_CHAN_AHB_PTR);
410
411 req->status = TEGRA_DMA_REQ_INFLIGHT;
412 return;
413 }
414
tegra_dma_update_hw(struct tegra_dma_channel * ch,struct tegra_dma_req * req)415 static void tegra_dma_update_hw(struct tegra_dma_channel *ch,
416 struct tegra_dma_req *req)
417 {
418 int ahb_addr_wrap;
419 int apb_addr_wrap;
420 int ahb_bus_width;
421 int apb_bus_width;
422 int index;
423
424 u32 ahb_seq;
425 u32 apb_seq;
426 u32 ahb_ptr;
427 u32 apb_ptr;
428 u32 csr;
429
430 csr = CSR_IE_EOC | CSR_FLOW;
431 ahb_seq = AHB_SEQ_INTR_ENB | AHB_SEQ_BURST_1;
432 apb_seq = 0;
433
434 csr |= req->req_sel << CSR_REQ_SEL_SHIFT;
435
436 /* One shot mode is always single buffered,
437 * continuous mode is always double buffered
438 * */
439 if (ch->mode & TEGRA_DMA_MODE_ONESHOT) {
440 csr |= CSR_ONCE;
441 ch->req_transfer_count = (req->size >> 2) - 1;
442 } else {
443 ahb_seq |= AHB_SEQ_DBL_BUF;
444
445 /* In double buffered mode, we set the size to half the
446 * requested size and interrupt when half the buffer
447 * is full */
448 ch->req_transfer_count = (req->size >> 3) - 1;
449 }
450
451 csr |= ch->req_transfer_count << CSR_WCOUNT_SHIFT;
452
453 if (req->to_memory) {
454 apb_ptr = req->source_addr;
455 ahb_ptr = req->dest_addr;
456
457 apb_addr_wrap = req->source_wrap;
458 ahb_addr_wrap = req->dest_wrap;
459 apb_bus_width = req->source_bus_width;
460 ahb_bus_width = req->dest_bus_width;
461
462 } else {
463 csr |= CSR_DIR;
464 apb_ptr = req->dest_addr;
465 ahb_ptr = req->source_addr;
466
467 apb_addr_wrap = req->dest_wrap;
468 ahb_addr_wrap = req->source_wrap;
469 apb_bus_width = req->dest_bus_width;
470 ahb_bus_width = req->source_bus_width;
471 }
472
473 apb_addr_wrap >>= 2;
474 ahb_addr_wrap >>= 2;
475
476 /* set address wrap for APB size */
477 index = 0;
478 do {
479 if (apb_addr_wrap_table[index] == apb_addr_wrap)
480 break;
481 index++;
482 } while (index < ARRAY_SIZE(apb_addr_wrap_table));
483 BUG_ON(index == ARRAY_SIZE(apb_addr_wrap_table));
484 apb_seq |= index << APB_SEQ_WRAP_SHIFT;
485
486 /* set address wrap for AHB size */
487 index = 0;
488 do {
489 if (ahb_addr_wrap_table[index] == ahb_addr_wrap)
490 break;
491 index++;
492 } while (index < ARRAY_SIZE(ahb_addr_wrap_table));
493 BUG_ON(index == ARRAY_SIZE(ahb_addr_wrap_table));
494 ahb_seq |= index << AHB_SEQ_WRAP_SHIFT;
495
496 for (index = 0; index < ARRAY_SIZE(bus_width_table); index++) {
497 if (bus_width_table[index] == ahb_bus_width)
498 break;
499 }
500 BUG_ON(index == ARRAY_SIZE(bus_width_table));
501 ahb_seq |= index << AHB_SEQ_BUS_WIDTH_SHIFT;
502
503 for (index = 0; index < ARRAY_SIZE(bus_width_table); index++) {
504 if (bus_width_table[index] == apb_bus_width)
505 break;
506 }
507 BUG_ON(index == ARRAY_SIZE(bus_width_table));
508 apb_seq |= index << APB_SEQ_BUS_WIDTH_SHIFT;
509
510 writel(csr, ch->addr + APB_DMA_CHAN_CSR);
511 writel(apb_seq, ch->addr + APB_DMA_CHAN_APB_SEQ);
512 writel(apb_ptr, ch->addr + APB_DMA_CHAN_APB_PTR);
513 writel(ahb_seq, ch->addr + APB_DMA_CHAN_AHB_SEQ);
514 writel(ahb_ptr, ch->addr + APB_DMA_CHAN_AHB_PTR);
515
516 csr |= CSR_ENB;
517 writel(csr, ch->addr + APB_DMA_CHAN_CSR);
518
519 req->status = TEGRA_DMA_REQ_INFLIGHT;
520 }
521
handle_oneshot_dma(struct tegra_dma_channel * ch)522 static void handle_oneshot_dma(struct tegra_dma_channel *ch)
523 {
524 struct tegra_dma_req *req;
525 unsigned long irq_flags;
526
527 spin_lock_irqsave(&ch->lock, irq_flags);
528 if (list_empty(&ch->list)) {
529 spin_unlock_irqrestore(&ch->lock, irq_flags);
530 return;
531 }
532
533 req = list_entry(ch->list.next, typeof(*req), node);
534 if (req) {
535 int bytes_transferred;
536
537 bytes_transferred = ch->req_transfer_count;
538 bytes_transferred += 1;
539 bytes_transferred <<= 2;
540
541 list_del(&req->node);
542 req->bytes_transferred = bytes_transferred;
543 req->status = TEGRA_DMA_REQ_SUCCESS;
544
545 spin_unlock_irqrestore(&ch->lock, irq_flags);
546 /* Callback should be called without any lock */
547 pr_debug("%s: transferred %d bytes\n", __func__,
548 req->bytes_transferred);
549 req->complete(req);
550 spin_lock_irqsave(&ch->lock, irq_flags);
551 }
552
553 if (!list_empty(&ch->list)) {
554 req = list_entry(ch->list.next, typeof(*req), node);
555 /* the complete function we just called may have enqueued
556 another req, in which case dma has already started */
557 if (req->status != TEGRA_DMA_REQ_INFLIGHT)
558 tegra_dma_update_hw(ch, req);
559 }
560 spin_unlock_irqrestore(&ch->lock, irq_flags);
561 }
562
handle_continuous_dma(struct tegra_dma_channel * ch)563 static void handle_continuous_dma(struct tegra_dma_channel *ch)
564 {
565 struct tegra_dma_req *req;
566 unsigned long irq_flags;
567
568 spin_lock_irqsave(&ch->lock, irq_flags);
569 if (list_empty(&ch->list)) {
570 spin_unlock_irqrestore(&ch->lock, irq_flags);
571 return;
572 }
573
574 req = list_entry(ch->list.next, typeof(*req), node);
575 if (req) {
576 if (req->buffer_status == TEGRA_DMA_REQ_BUF_STATUS_EMPTY) {
577 bool is_dma_ping_complete;
578 is_dma_ping_complete = (readl(ch->addr + APB_DMA_CHAN_STA)
579 & STA_PING_PONG) ? true : false;
580 if (req->to_memory)
581 is_dma_ping_complete = !is_dma_ping_complete;
582 /* Out of sync - Release current buffer */
583 if (!is_dma_ping_complete) {
584 int bytes_transferred;
585
586 bytes_transferred = ch->req_transfer_count;
587 bytes_transferred += 1;
588 bytes_transferred <<= 3;
589 req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_FULL;
590 req->bytes_transferred = bytes_transferred;
591 req->status = TEGRA_DMA_REQ_SUCCESS;
592 tegra_dma_stop(ch);
593
594 if (!list_is_last(&req->node, &ch->list)) {
595 struct tegra_dma_req *next_req;
596
597 next_req = list_entry(req->node.next,
598 typeof(*next_req), node);
599 tegra_dma_update_hw(ch, next_req);
600 }
601
602 list_del(&req->node);
603
604 /* DMA lock is NOT held when callbak is called */
605 spin_unlock_irqrestore(&ch->lock, irq_flags);
606 req->complete(req);
607 return;
608 }
609 /* Load the next request into the hardware, if available
610 * */
611 if (!list_is_last(&req->node, &ch->list)) {
612 struct tegra_dma_req *next_req;
613
614 next_req = list_entry(req->node.next,
615 typeof(*next_req), node);
616 tegra_dma_update_hw_partial(ch, next_req);
617 }
618 req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_HALF_FULL;
619 req->status = TEGRA_DMA_REQ_SUCCESS;
620 /* DMA lock is NOT held when callback is called */
621 spin_unlock_irqrestore(&ch->lock, irq_flags);
622 if (likely(req->threshold))
623 req->threshold(req);
624 return;
625
626 } else if (req->buffer_status ==
627 TEGRA_DMA_REQ_BUF_STATUS_HALF_FULL) {
628 /* Callback when the buffer is completely full (i.e on
629 * the second interrupt */
630 int bytes_transferred;
631
632 bytes_transferred = ch->req_transfer_count;
633 bytes_transferred += 1;
634 bytes_transferred <<= 3;
635
636 req->buffer_status = TEGRA_DMA_REQ_BUF_STATUS_FULL;
637 req->bytes_transferred = bytes_transferred;
638 req->status = TEGRA_DMA_REQ_SUCCESS;
639 list_del(&req->node);
640
641 /* DMA lock is NOT held when callbak is called */
642 spin_unlock_irqrestore(&ch->lock, irq_flags);
643 req->complete(req);
644 return;
645
646 } else {
647 BUG();
648 }
649 }
650 spin_unlock_irqrestore(&ch->lock, irq_flags);
651 }
652
dma_isr(int irq,void * data)653 static irqreturn_t dma_isr(int irq, void *data)
654 {
655 struct tegra_dma_channel *ch = data;
656 unsigned long status;
657
658 status = readl(ch->addr + APB_DMA_CHAN_STA);
659 if (status & STA_ISE_EOC)
660 writel(status, ch->addr + APB_DMA_CHAN_STA);
661 else {
662 pr_warning("Got a spurious ISR for DMA channel %d\n", ch->id);
663 return IRQ_HANDLED;
664 }
665 return IRQ_WAKE_THREAD;
666 }
667
dma_thread_fn(int irq,void * data)668 static irqreturn_t dma_thread_fn(int irq, void *data)
669 {
670 struct tegra_dma_channel *ch = data;
671
672 if (ch->mode & TEGRA_DMA_MODE_ONESHOT)
673 handle_oneshot_dma(ch);
674 else
675 handle_continuous_dma(ch);
676
677
678 return IRQ_HANDLED;
679 }
680
tegra_dma_init(void)681 int __init tegra_dma_init(void)
682 {
683 int ret = 0;
684 int i;
685 unsigned int irq;
686 void __iomem *addr;
687 struct clk *c;
688
689 bitmap_fill(channel_usage, NV_DMA_MAX_CHANNELS);
690
691 c = clk_get_sys("tegra-dma", NULL);
692 if (IS_ERR(c)) {
693 pr_err("Unable to get clock for APB DMA\n");
694 ret = PTR_ERR(c);
695 goto fail;
696 }
697 ret = clk_enable(c);
698 if (ret != 0) {
699 pr_err("Unable to enable clock for APB DMA\n");
700 goto fail;
701 }
702
703 addr = IO_ADDRESS(TEGRA_APB_DMA_BASE);
704 writel(GEN_ENABLE, addr + APB_DMA_GEN);
705 writel(0, addr + APB_DMA_CNTRL);
706 writel(0xFFFFFFFFul >> (31 - TEGRA_SYSTEM_DMA_CH_MAX),
707 addr + APB_DMA_IRQ_MASK_SET);
708
709 for (i = TEGRA_SYSTEM_DMA_CH_MIN; i <= TEGRA_SYSTEM_DMA_CH_MAX; i++) {
710 struct tegra_dma_channel *ch = &dma_channels[i];
711
712 ch->id = i;
713 snprintf(ch->name, TEGRA_DMA_NAME_SIZE, "dma_channel_%d", i);
714
715 ch->addr = IO_ADDRESS(TEGRA_APB_DMA_CH0_BASE +
716 TEGRA_APB_DMA_CH0_SIZE * i);
717
718 spin_lock_init(&ch->lock);
719 INIT_LIST_HEAD(&ch->list);
720
721 irq = INT_APB_DMA_CH0 + i;
722 ret = request_threaded_irq(irq, dma_isr, dma_thread_fn, 0,
723 dma_channels[i].name, ch);
724 if (ret) {
725 pr_err("Failed to register IRQ %d for DMA %d\n",
726 irq, i);
727 goto fail;
728 }
729 ch->irq = irq;
730
731 __clear_bit(i, channel_usage);
732 }
733 /* mark the shared channel allocated */
734 __set_bit(TEGRA_SYSTEM_DMA_CH_MIN, channel_usage);
735
736 tegra_dma_initialized = true;
737
738 return 0;
739 fail:
740 writel(0, addr + APB_DMA_GEN);
741 for (i = TEGRA_SYSTEM_DMA_CH_MIN; i <= TEGRA_SYSTEM_DMA_CH_MAX; i++) {
742 struct tegra_dma_channel *ch = &dma_channels[i];
743 if (ch->irq)
744 free_irq(ch->irq, ch);
745 }
746 return ret;
747 }
748 postcore_initcall(tegra_dma_init);
749
750 #ifdef CONFIG_PM
751 static u32 apb_dma[5*TEGRA_SYSTEM_DMA_CH_NR + 3];
752
tegra_dma_suspend(void)753 void tegra_dma_suspend(void)
754 {
755 void __iomem *addr = IO_ADDRESS(TEGRA_APB_DMA_BASE);
756 u32 *ctx = apb_dma;
757 int i;
758
759 *ctx++ = readl(addr + APB_DMA_GEN);
760 *ctx++ = readl(addr + APB_DMA_CNTRL);
761 *ctx++ = readl(addr + APB_DMA_IRQ_MASK);
762
763 for (i = 0; i < TEGRA_SYSTEM_DMA_CH_NR; i++) {
764 addr = IO_ADDRESS(TEGRA_APB_DMA_CH0_BASE +
765 TEGRA_APB_DMA_CH0_SIZE * i);
766
767 *ctx++ = readl(addr + APB_DMA_CHAN_CSR);
768 *ctx++ = readl(addr + APB_DMA_CHAN_AHB_PTR);
769 *ctx++ = readl(addr + APB_DMA_CHAN_AHB_SEQ);
770 *ctx++ = readl(addr + APB_DMA_CHAN_APB_PTR);
771 *ctx++ = readl(addr + APB_DMA_CHAN_APB_SEQ);
772 }
773 }
774
tegra_dma_resume(void)775 void tegra_dma_resume(void)
776 {
777 void __iomem *addr = IO_ADDRESS(TEGRA_APB_DMA_BASE);
778 u32 *ctx = apb_dma;
779 int i;
780
781 writel(*ctx++, addr + APB_DMA_GEN);
782 writel(*ctx++, addr + APB_DMA_CNTRL);
783 writel(*ctx++, addr + APB_DMA_IRQ_MASK);
784
785 for (i = 0; i < TEGRA_SYSTEM_DMA_CH_NR; i++) {
786 addr = IO_ADDRESS(TEGRA_APB_DMA_CH0_BASE +
787 TEGRA_APB_DMA_CH0_SIZE * i);
788
789 writel(*ctx++, addr + APB_DMA_CHAN_CSR);
790 writel(*ctx++, addr + APB_DMA_CHAN_AHB_PTR);
791 writel(*ctx++, addr + APB_DMA_CHAN_AHB_SEQ);
792 writel(*ctx++, addr + APB_DMA_CHAN_APB_PTR);
793 writel(*ctx++, addr + APB_DMA_CHAN_APB_SEQ);
794 }
795 }
796
797 #endif
798