1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2 /* Copyright (c) 2010-2012 Broadcom. All rights reserved. */
3
4 #include <linux/kernel.h>
5 #include <linux/types.h>
6 #include <linux/errno.h>
7 #include <linux/interrupt.h>
8 #include <linux/pagemap.h>
9 #include <linux/dma-mapping.h>
10 #include <linux/io.h>
11 #include <linux/platform_device.h>
12 #include <linux/uaccess.h>
13 #include <linux/mm.h>
14 #include <linux/of.h>
15 #include <linux/slab.h>
16 #include <soc/bcm2835/raspberrypi-firmware.h>
17
18 #define TOTAL_SLOTS (VCHIQ_SLOT_ZERO_SLOTS + 2 * 32)
19
20 #include "vchiq_arm.h"
21 #include "vchiq_connected.h"
22 #include "vchiq_pagelist.h"
23
24 #define MAX_FRAGMENTS (VCHIQ_NUM_CURRENT_BULKS * 2)
25
26 #define VCHIQ_PLATFORM_FRAGMENTS_OFFSET_IDX 0
27 #define VCHIQ_PLATFORM_FRAGMENTS_COUNT_IDX 1
28
29 #define BELL0 0x00
30 #define BELL2 0x08
31
32 struct vchiq_2835_state {
33 int inited;
34 struct vchiq_arm_state arm_state;
35 };
36
37 struct vchiq_pagelist_info {
38 struct pagelist *pagelist;
39 size_t pagelist_buffer_size;
40 dma_addr_t dma_addr;
41 enum dma_data_direction dma_dir;
42 unsigned int num_pages;
43 unsigned int pages_need_release;
44 struct page **pages;
45 struct scatterlist *scatterlist;
46 unsigned int scatterlist_mapped;
47 };
48
49 static void __iomem *g_regs;
50 /* This value is the size of the L2 cache lines as understood by the
51 * VPU firmware, which determines the required alignment of the
52 * offsets/sizes in pagelists.
53 *
54 * Modern VPU firmware looks for a DT "cache-line-size" property in
55 * the VCHIQ node and will overwrite it with the actual L2 cache size,
56 * which the kernel must then respect. That property was rejected
57 * upstream, so we have to use the VPU firmware's compatibility value
58 * of 32.
59 */
60 static unsigned int g_cache_line_size = 32;
61 static unsigned int g_fragments_size;
62 static char *g_fragments_base;
63 static char *g_free_fragments;
64 static struct semaphore g_free_fragments_sema;
65 static struct device *g_dev;
66
67 static DEFINE_SEMAPHORE(g_free_fragments_mutex);
68
69 static irqreturn_t
70 vchiq_doorbell_irq(int irq, void *dev_id);
71
72 static struct vchiq_pagelist_info *
73 create_pagelist(char *buf, char __user *ubuf, size_t count, unsigned short type);
74
75 static void
76 free_pagelist(struct vchiq_pagelist_info *pagelistinfo,
77 int actual);
78
vchiq_platform_init(struct platform_device * pdev,struct vchiq_state * state)79 int vchiq_platform_init(struct platform_device *pdev, struct vchiq_state *state)
80 {
81 struct device *dev = &pdev->dev;
82 struct vchiq_drvdata *drvdata = platform_get_drvdata(pdev);
83 struct rpi_firmware *fw = drvdata->fw;
84 struct vchiq_slot_zero *vchiq_slot_zero;
85 void *slot_mem;
86 dma_addr_t slot_phys;
87 u32 channelbase;
88 int slot_mem_size, frag_mem_size;
89 int err, irq, i;
90
91 /*
92 * VCHI messages between the CPU and firmware use
93 * 32-bit bus addresses.
94 */
95 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
96
97 if (err < 0)
98 return err;
99
100 g_cache_line_size = drvdata->cache_line_size;
101 g_fragments_size = 2 * g_cache_line_size;
102
103 /* Allocate space for the channels in coherent memory */
104 slot_mem_size = PAGE_ALIGN(TOTAL_SLOTS * VCHIQ_SLOT_SIZE);
105 frag_mem_size = PAGE_ALIGN(g_fragments_size * MAX_FRAGMENTS);
106
107 slot_mem = dmam_alloc_coherent(dev, slot_mem_size + frag_mem_size,
108 &slot_phys, GFP_KERNEL);
109 if (!slot_mem) {
110 dev_err(dev, "could not allocate DMA memory\n");
111 return -ENOMEM;
112 }
113
114 WARN_ON(((unsigned long)slot_mem & (PAGE_SIZE - 1)) != 0);
115
116 vchiq_slot_zero = vchiq_init_slots(slot_mem, slot_mem_size);
117 if (!vchiq_slot_zero)
118 return -EINVAL;
119
120 vchiq_slot_zero->platform_data[VCHIQ_PLATFORM_FRAGMENTS_OFFSET_IDX] =
121 (int)slot_phys + slot_mem_size;
122 vchiq_slot_zero->platform_data[VCHIQ_PLATFORM_FRAGMENTS_COUNT_IDX] =
123 MAX_FRAGMENTS;
124
125 g_fragments_base = (char *)slot_mem + slot_mem_size;
126
127 g_free_fragments = g_fragments_base;
128 for (i = 0; i < (MAX_FRAGMENTS - 1); i++) {
129 *(char **)&g_fragments_base[i*g_fragments_size] =
130 &g_fragments_base[(i + 1)*g_fragments_size];
131 }
132 *(char **)&g_fragments_base[i * g_fragments_size] = NULL;
133 sema_init(&g_free_fragments_sema, MAX_FRAGMENTS);
134
135 if (vchiq_init_state(state, vchiq_slot_zero) != VCHIQ_SUCCESS)
136 return -EINVAL;
137
138 g_regs = devm_platform_ioremap_resource(pdev, 0);
139 if (IS_ERR(g_regs))
140 return PTR_ERR(g_regs);
141
142 irq = platform_get_irq(pdev, 0);
143 if (irq <= 0)
144 return irq;
145
146 err = devm_request_irq(dev, irq, vchiq_doorbell_irq, IRQF_IRQPOLL,
147 "VCHIQ doorbell", state);
148 if (err) {
149 dev_err(dev, "failed to register irq=%d\n", irq);
150 return err;
151 }
152
153 /* Send the base address of the slots to VideoCore */
154 channelbase = slot_phys;
155 err = rpi_firmware_property(fw, RPI_FIRMWARE_VCHIQ_INIT,
156 &channelbase, sizeof(channelbase));
157 if (err || channelbase) {
158 dev_err(dev, "failed to set channelbase\n");
159 return err ? : -ENXIO;
160 }
161
162 g_dev = dev;
163 vchiq_log_info(vchiq_arm_log_level,
164 "vchiq_init - done (slots %pK, phys %pad)",
165 vchiq_slot_zero, &slot_phys);
166
167 vchiq_call_connected_callbacks();
168
169 return 0;
170 }
171
172 enum vchiq_status
vchiq_platform_init_state(struct vchiq_state * state)173 vchiq_platform_init_state(struct vchiq_state *state)
174 {
175 enum vchiq_status status = VCHIQ_SUCCESS;
176 struct vchiq_2835_state *platform_state;
177
178 state->platform_state = kzalloc(sizeof(*platform_state), GFP_KERNEL);
179 if (!state->platform_state)
180 return VCHIQ_ERROR;
181
182 platform_state = (struct vchiq_2835_state *)state->platform_state;
183
184 platform_state->inited = 1;
185 status = vchiq_arm_init_state(state, &platform_state->arm_state);
186
187 if (status != VCHIQ_SUCCESS)
188 platform_state->inited = 0;
189
190 return status;
191 }
192
193 struct vchiq_arm_state*
vchiq_platform_get_arm_state(struct vchiq_state * state)194 vchiq_platform_get_arm_state(struct vchiq_state *state)
195 {
196 struct vchiq_2835_state *platform_state;
197
198 platform_state = (struct vchiq_2835_state *)state->platform_state;
199
200 WARN_ON_ONCE(!platform_state->inited);
201
202 return &platform_state->arm_state;
203 }
204
205 void
remote_event_signal(struct remote_event * event)206 remote_event_signal(struct remote_event *event)
207 {
208 wmb();
209
210 event->fired = 1;
211
212 dsb(sy); /* data barrier operation */
213
214 if (event->armed)
215 writel(0, g_regs + BELL2); /* trigger vc interrupt */
216 }
217
218 enum vchiq_status
vchiq_prepare_bulk_data(struct vchiq_bulk * bulk,void * offset,void __user * uoffset,int size,int dir)219 vchiq_prepare_bulk_data(struct vchiq_bulk *bulk, void *offset,
220 void __user *uoffset, int size, int dir)
221 {
222 struct vchiq_pagelist_info *pagelistinfo;
223
224 pagelistinfo = create_pagelist(offset, uoffset, size,
225 (dir == VCHIQ_BULK_RECEIVE)
226 ? PAGELIST_READ
227 : PAGELIST_WRITE);
228
229 if (!pagelistinfo)
230 return VCHIQ_ERROR;
231
232 bulk->data = pagelistinfo->dma_addr;
233
234 /*
235 * Store the pagelistinfo address in remote_data,
236 * which isn't used by the slave.
237 */
238 bulk->remote_data = pagelistinfo;
239
240 return VCHIQ_SUCCESS;
241 }
242
243 void
vchiq_complete_bulk(struct vchiq_bulk * bulk)244 vchiq_complete_bulk(struct vchiq_bulk *bulk)
245 {
246 if (bulk && bulk->remote_data && bulk->actual)
247 free_pagelist((struct vchiq_pagelist_info *)bulk->remote_data,
248 bulk->actual);
249 }
250
vchiq_dump_platform_state(void * dump_context)251 int vchiq_dump_platform_state(void *dump_context)
252 {
253 char buf[80];
254 int len;
255
256 len = snprintf(buf, sizeof(buf),
257 " Platform: 2835 (VC master)");
258 return vchiq_dump(dump_context, buf, len + 1);
259 }
260
261 /*
262 * Local functions
263 */
264
265 static irqreturn_t
vchiq_doorbell_irq(int irq,void * dev_id)266 vchiq_doorbell_irq(int irq, void *dev_id)
267 {
268 struct vchiq_state *state = dev_id;
269 irqreturn_t ret = IRQ_NONE;
270 unsigned int status;
271
272 /* Read (and clear) the doorbell */
273 status = readl(g_regs + BELL0);
274
275 if (status & 0x4) { /* Was the doorbell rung? */
276 remote_event_pollall(state);
277 ret = IRQ_HANDLED;
278 }
279
280 return ret;
281 }
282
283 static void
cleanup_pagelistinfo(struct vchiq_pagelist_info * pagelistinfo)284 cleanup_pagelistinfo(struct vchiq_pagelist_info *pagelistinfo)
285 {
286 if (pagelistinfo->scatterlist_mapped) {
287 dma_unmap_sg(g_dev, pagelistinfo->scatterlist,
288 pagelistinfo->num_pages, pagelistinfo->dma_dir);
289 }
290
291 if (pagelistinfo->pages_need_release)
292 unpin_user_pages(pagelistinfo->pages, pagelistinfo->num_pages);
293
294 dma_free_coherent(g_dev, pagelistinfo->pagelist_buffer_size,
295 pagelistinfo->pagelist, pagelistinfo->dma_addr);
296 }
297
298 /* There is a potential problem with partial cache lines (pages?)
299 * at the ends of the block when reading. If the CPU accessed anything in
300 * the same line (page?) then it may have pulled old data into the cache,
301 * obscuring the new data underneath. We can solve this by transferring the
302 * partial cache lines separately, and allowing the ARM to copy into the
303 * cached area.
304 */
305
306 static struct vchiq_pagelist_info *
create_pagelist(char * buf,char __user * ubuf,size_t count,unsigned short type)307 create_pagelist(char *buf, char __user *ubuf,
308 size_t count, unsigned short type)
309 {
310 struct pagelist *pagelist;
311 struct vchiq_pagelist_info *pagelistinfo;
312 struct page **pages;
313 u32 *addrs;
314 unsigned int num_pages, offset, i, k;
315 int actual_pages;
316 size_t pagelist_size;
317 struct scatterlist *scatterlist, *sg;
318 int dma_buffers;
319 dma_addr_t dma_addr;
320
321 if (count >= INT_MAX - PAGE_SIZE)
322 return NULL;
323
324 if (buf)
325 offset = (uintptr_t)buf & (PAGE_SIZE - 1);
326 else
327 offset = (uintptr_t)ubuf & (PAGE_SIZE - 1);
328 num_pages = DIV_ROUND_UP(count + offset, PAGE_SIZE);
329
330 if (num_pages > (SIZE_MAX - sizeof(struct pagelist) -
331 sizeof(struct vchiq_pagelist_info)) /
332 (sizeof(u32) + sizeof(pages[0]) +
333 sizeof(struct scatterlist)))
334 return NULL;
335
336 pagelist_size = sizeof(struct pagelist) +
337 (num_pages * sizeof(u32)) +
338 (num_pages * sizeof(pages[0]) +
339 (num_pages * sizeof(struct scatterlist))) +
340 sizeof(struct vchiq_pagelist_info);
341
342 /* Allocate enough storage to hold the page pointers and the page
343 * list
344 */
345 pagelist = dma_alloc_coherent(g_dev, pagelist_size, &dma_addr,
346 GFP_KERNEL);
347
348 vchiq_log_trace(vchiq_arm_log_level, "%s - %pK", __func__, pagelist);
349
350 if (!pagelist)
351 return NULL;
352
353 addrs = pagelist->addrs;
354 pages = (struct page **)(addrs + num_pages);
355 scatterlist = (struct scatterlist *)(pages + num_pages);
356 pagelistinfo = (struct vchiq_pagelist_info *)
357 (scatterlist + num_pages);
358
359 pagelist->length = count;
360 pagelist->type = type;
361 pagelist->offset = offset;
362
363 /* Populate the fields of the pagelistinfo structure */
364 pagelistinfo->pagelist = pagelist;
365 pagelistinfo->pagelist_buffer_size = pagelist_size;
366 pagelistinfo->dma_addr = dma_addr;
367 pagelistinfo->dma_dir = (type == PAGELIST_WRITE) ?
368 DMA_TO_DEVICE : DMA_FROM_DEVICE;
369 pagelistinfo->num_pages = num_pages;
370 pagelistinfo->pages_need_release = 0;
371 pagelistinfo->pages = pages;
372 pagelistinfo->scatterlist = scatterlist;
373 pagelistinfo->scatterlist_mapped = 0;
374
375 if (buf) {
376 unsigned long length = count;
377 unsigned int off = offset;
378
379 for (actual_pages = 0; actual_pages < num_pages;
380 actual_pages++) {
381 struct page *pg =
382 vmalloc_to_page((buf +
383 (actual_pages * PAGE_SIZE)));
384 size_t bytes = PAGE_SIZE - off;
385
386 if (!pg) {
387 cleanup_pagelistinfo(pagelistinfo);
388 return NULL;
389 }
390
391 if (bytes > length)
392 bytes = length;
393 pages[actual_pages] = pg;
394 length -= bytes;
395 off = 0;
396 }
397 /* do not try and release vmalloc pages */
398 } else {
399 actual_pages = pin_user_pages_fast(
400 (unsigned long)ubuf & PAGE_MASK,
401 num_pages,
402 type == PAGELIST_READ,
403 pages);
404
405 if (actual_pages != num_pages) {
406 vchiq_log_info(vchiq_arm_log_level,
407 "%s - only %d/%d pages locked",
408 __func__, actual_pages, num_pages);
409
410 /* This is probably due to the process being killed */
411 if (actual_pages > 0)
412 unpin_user_pages(pages, actual_pages);
413 cleanup_pagelistinfo(pagelistinfo);
414 return NULL;
415 }
416 /* release user pages */
417 pagelistinfo->pages_need_release = 1;
418 }
419
420 /*
421 * Initialize the scatterlist so that the magic cookie
422 * is filled if debugging is enabled
423 */
424 sg_init_table(scatterlist, num_pages);
425 /* Now set the pages for each scatterlist */
426 for (i = 0; i < num_pages; i++) {
427 unsigned int len = PAGE_SIZE - offset;
428
429 if (len > count)
430 len = count;
431 sg_set_page(scatterlist + i, pages[i], len, offset);
432 offset = 0;
433 count -= len;
434 }
435
436 dma_buffers = dma_map_sg(g_dev,
437 scatterlist,
438 num_pages,
439 pagelistinfo->dma_dir);
440
441 if (dma_buffers == 0) {
442 cleanup_pagelistinfo(pagelistinfo);
443 return NULL;
444 }
445
446 pagelistinfo->scatterlist_mapped = 1;
447
448 /* Combine adjacent blocks for performance */
449 k = 0;
450 for_each_sg(scatterlist, sg, dma_buffers, i) {
451 u32 len = sg_dma_len(sg);
452 u32 addr = sg_dma_address(sg);
453
454 /* Note: addrs is the address + page_count - 1
455 * The firmware expects blocks after the first to be page-
456 * aligned and a multiple of the page size
457 */
458 WARN_ON(len == 0);
459 WARN_ON(i && (i != (dma_buffers - 1)) && (len & ~PAGE_MASK));
460 WARN_ON(i && (addr & ~PAGE_MASK));
461 if (k > 0 &&
462 ((addrs[k - 1] & PAGE_MASK) +
463 (((addrs[k - 1] & ~PAGE_MASK) + 1) << PAGE_SHIFT))
464 == (addr & PAGE_MASK))
465 addrs[k - 1] += ((len + PAGE_SIZE - 1) >> PAGE_SHIFT);
466 else
467 addrs[k++] = (addr & PAGE_MASK) |
468 (((len + PAGE_SIZE - 1) >> PAGE_SHIFT) - 1);
469 }
470
471 /* Partial cache lines (fragments) require special measures */
472 if ((type == PAGELIST_READ) &&
473 ((pagelist->offset & (g_cache_line_size - 1)) ||
474 ((pagelist->offset + pagelist->length) &
475 (g_cache_line_size - 1)))) {
476 char *fragments;
477
478 if (down_interruptible(&g_free_fragments_sema)) {
479 cleanup_pagelistinfo(pagelistinfo);
480 return NULL;
481 }
482
483 WARN_ON(!g_free_fragments);
484
485 down(&g_free_fragments_mutex);
486 fragments = g_free_fragments;
487 WARN_ON(!fragments);
488 g_free_fragments = *(char **) g_free_fragments;
489 up(&g_free_fragments_mutex);
490 pagelist->type = PAGELIST_READ_WITH_FRAGMENTS +
491 (fragments - g_fragments_base) / g_fragments_size;
492 }
493
494 return pagelistinfo;
495 }
496
497 static void
free_pagelist(struct vchiq_pagelist_info * pagelistinfo,int actual)498 free_pagelist(struct vchiq_pagelist_info *pagelistinfo,
499 int actual)
500 {
501 struct pagelist *pagelist = pagelistinfo->pagelist;
502 struct page **pages = pagelistinfo->pages;
503 unsigned int num_pages = pagelistinfo->num_pages;
504
505 vchiq_log_trace(vchiq_arm_log_level, "%s - %pK, %d",
506 __func__, pagelistinfo->pagelist, actual);
507
508 /*
509 * NOTE: dma_unmap_sg must be called before the
510 * cpu can touch any of the data/pages.
511 */
512 dma_unmap_sg(g_dev, pagelistinfo->scatterlist,
513 pagelistinfo->num_pages, pagelistinfo->dma_dir);
514 pagelistinfo->scatterlist_mapped = 0;
515
516 /* Deal with any partial cache lines (fragments) */
517 if (pagelist->type >= PAGELIST_READ_WITH_FRAGMENTS) {
518 char *fragments = g_fragments_base +
519 (pagelist->type - PAGELIST_READ_WITH_FRAGMENTS) *
520 g_fragments_size;
521 int head_bytes, tail_bytes;
522
523 head_bytes = (g_cache_line_size - pagelist->offset) &
524 (g_cache_line_size - 1);
525 tail_bytes = (pagelist->offset + actual) &
526 (g_cache_line_size - 1);
527
528 if ((actual >= 0) && (head_bytes != 0)) {
529 if (head_bytes > actual)
530 head_bytes = actual;
531
532 memcpy((char *)kmap(pages[0]) +
533 pagelist->offset,
534 fragments,
535 head_bytes);
536 kunmap(pages[0]);
537 }
538 if ((actual >= 0) && (head_bytes < actual) &&
539 (tail_bytes != 0)) {
540 memcpy((char *)kmap(pages[num_pages - 1]) +
541 ((pagelist->offset + actual) &
542 (PAGE_SIZE - 1) & ~(g_cache_line_size - 1)),
543 fragments + g_cache_line_size,
544 tail_bytes);
545 kunmap(pages[num_pages - 1]);
546 }
547
548 down(&g_free_fragments_mutex);
549 *(char **)fragments = g_free_fragments;
550 g_free_fragments = fragments;
551 up(&g_free_fragments_mutex);
552 up(&g_free_fragments_sema);
553 }
554
555 /* Need to mark all the pages dirty. */
556 if (pagelist->type != PAGELIST_WRITE &&
557 pagelistinfo->pages_need_release) {
558 unsigned int i;
559
560 for (i = 0; i < num_pages; i++)
561 set_page_dirty(pages[i]);
562 }
563
564 cleanup_pagelistinfo(pagelistinfo);
565 }
566