Lines Matching +full:fastrpc +full:- +full:compute +full:- +full:cb
1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2011-2018, The Linux Foundation. All rights reserved.
7 #include <linux/dma-buf.h>
8 #include <linux/dma-mapping.h>
9 #include <linux/dma-resv.h>
23 #include <uapi/misc/fastrpc.h>
43 #define FASTRPC_DEVICE_NAME "fastrpc"
316 if (map->table) { in fastrpc_free_map()
317 if (map->attr & FASTRPC_ATTR_SECUREMAP) { in fastrpc_free_map()
319 int vmid = map->fl->cctx->vmperms[0].vmid; in fastrpc_free_map()
325 err = qcom_scm_assign_mem(map->phys, map->size, in fastrpc_free_map()
328 dev_err(map->fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d", in fastrpc_free_map()
329 map->phys, map->size, err); in fastrpc_free_map()
333 dma_buf_unmap_attachment_unlocked(map->attach, map->table, in fastrpc_free_map()
335 dma_buf_detach(map->buf, map->attach); in fastrpc_free_map()
336 dma_buf_put(map->buf); in fastrpc_free_map()
339 if (map->fl) { in fastrpc_free_map()
340 spin_lock(&map->fl->lock); in fastrpc_free_map()
341 list_del(&map->node); in fastrpc_free_map()
342 spin_unlock(&map->fl->lock); in fastrpc_free_map()
343 map->fl = NULL; in fastrpc_free_map()
352 kref_put(&map->refcount, fastrpc_free_map); in fastrpc_map_put()
358 return -ENOENT; in fastrpc_map_get()
360 return kref_get_unless_zero(&map->refcount) ? 0 : -ENOENT; in fastrpc_map_get()
367 struct fastrpc_session_ctx *sess = fl->sctx; in fastrpc_map_lookup()
369 int ret = -ENOENT; in fastrpc_map_lookup()
371 spin_lock(&fl->lock); in fastrpc_map_lookup()
372 list_for_each_entry(map, &fl->maps, node) { in fastrpc_map_lookup()
373 if (map->fd != fd) in fastrpc_map_lookup()
379 dev_dbg(sess->dev, "%s: Failed to get map fd=%d ret=%d\n", in fastrpc_map_lookup()
389 spin_unlock(&fl->lock); in fastrpc_map_lookup()
396 dma_free_coherent(buf->dev, buf->size, buf->virt, in fastrpc_buf_free()
397 FASTRPC_PHYS(buf->phys)); in fastrpc_buf_free()
408 return -ENOMEM; in __fastrpc_buf_alloc()
410 INIT_LIST_HEAD(&buf->attachments); in __fastrpc_buf_alloc()
411 INIT_LIST_HEAD(&buf->node); in __fastrpc_buf_alloc()
412 mutex_init(&buf->lock); in __fastrpc_buf_alloc()
414 buf->fl = fl; in __fastrpc_buf_alloc()
415 buf->virt = NULL; in __fastrpc_buf_alloc()
416 buf->phys = 0; in __fastrpc_buf_alloc()
417 buf->size = size; in __fastrpc_buf_alloc()
418 buf->dev = dev; in __fastrpc_buf_alloc()
419 buf->raddr = 0; in __fastrpc_buf_alloc()
421 buf->virt = dma_alloc_coherent(dev, buf->size, (dma_addr_t *)&buf->phys, in __fastrpc_buf_alloc()
423 if (!buf->virt) { in __fastrpc_buf_alloc()
424 mutex_destroy(&buf->lock); in __fastrpc_buf_alloc()
426 return -ENOMEM; in __fastrpc_buf_alloc()
446 if (fl->sctx && fl->sctx->sid) in fastrpc_buf_alloc()
447 buf->phys += ((u64)fl->sctx->sid << 32); in fastrpc_buf_alloc()
455 struct device *rdev = &fl->cctx->rpdev->dev; in fastrpc_remote_heap_alloc()
471 kref_get(&cctx->refcount); in fastrpc_channel_ctx_get()
476 kref_put(&cctx->refcount, fastrpc_channel_ctx_free); in fastrpc_channel_ctx_put()
487 cctx = ctx->cctx; in fastrpc_context_free()
489 for (i = 0; i < ctx->nbufs; i++) in fastrpc_context_free()
490 fastrpc_map_put(ctx->maps[i]); in fastrpc_context_free()
492 if (ctx->buf) in fastrpc_context_free()
493 fastrpc_buf_free(ctx->buf); in fastrpc_context_free()
495 spin_lock_irqsave(&cctx->lock, flags); in fastrpc_context_free()
496 idr_remove(&cctx->ctx_idr, ctx->ctxid >> 4); in fastrpc_context_free()
497 spin_unlock_irqrestore(&cctx->lock, flags); in fastrpc_context_free()
499 kfree(ctx->maps); in fastrpc_context_free()
500 kfree(ctx->olaps); in fastrpc_context_free()
508 kref_get(&ctx->refcount); in fastrpc_context_get()
513 kref_put(&ctx->refcount, fastrpc_context_free); in fastrpc_context_put()
524 #define CMP(aa, bb) ((aa) == (bb) ? 0 : (aa) < (bb) ? -1 : 1)
530 int st = CMP(pa->start, pb->start); in olaps_cmp()
532 int ed = CMP(pb->end, pa->end); in olaps_cmp()
542 for (i = 0; i < ctx->nbufs; ++i) { in fastrpc_get_buff_overlaps()
543 ctx->olaps[i].start = ctx->args[i].ptr; in fastrpc_get_buff_overlaps()
544 ctx->olaps[i].end = ctx->olaps[i].start + ctx->args[i].length; in fastrpc_get_buff_overlaps()
545 ctx->olaps[i].raix = i; in fastrpc_get_buff_overlaps()
548 sort(ctx->olaps, ctx->nbufs, sizeof(*ctx->olaps), olaps_cmp, NULL); in fastrpc_get_buff_overlaps()
550 for (i = 0; i < ctx->nbufs; ++i) { in fastrpc_get_buff_overlaps()
552 if (ctx->olaps[i].start < max_end) { in fastrpc_get_buff_overlaps()
553 ctx->olaps[i].mstart = max_end; in fastrpc_get_buff_overlaps()
554 ctx->olaps[i].mend = ctx->olaps[i].end; in fastrpc_get_buff_overlaps()
555 ctx->olaps[i].offset = max_end - ctx->olaps[i].start; in fastrpc_get_buff_overlaps()
557 if (ctx->olaps[i].end > max_end) { in fastrpc_get_buff_overlaps()
558 max_end = ctx->olaps[i].end; in fastrpc_get_buff_overlaps()
560 ctx->olaps[i].mend = 0; in fastrpc_get_buff_overlaps()
561 ctx->olaps[i].mstart = 0; in fastrpc_get_buff_overlaps()
565 ctx->olaps[i].mend = ctx->olaps[i].end; in fastrpc_get_buff_overlaps()
566 ctx->olaps[i].mstart = ctx->olaps[i].start; in fastrpc_get_buff_overlaps()
567 ctx->olaps[i].offset = 0; in fastrpc_get_buff_overlaps()
568 max_end = ctx->olaps[i].end; in fastrpc_get_buff_overlaps()
577 struct fastrpc_channel_ctx *cctx = user->cctx; in fastrpc_context_alloc()
584 return ERR_PTR(-ENOMEM); in fastrpc_context_alloc()
586 INIT_LIST_HEAD(&ctx->node); in fastrpc_context_alloc()
587 ctx->fl = user; in fastrpc_context_alloc()
588 ctx->nscalars = REMOTE_SCALARS_LENGTH(sc); in fastrpc_context_alloc()
589 ctx->nbufs = REMOTE_SCALARS_INBUFS(sc) + in fastrpc_context_alloc()
592 if (ctx->nscalars) { in fastrpc_context_alloc()
593 ctx->maps = kcalloc(ctx->nscalars, in fastrpc_context_alloc()
594 sizeof(*ctx->maps), GFP_KERNEL); in fastrpc_context_alloc()
595 if (!ctx->maps) { in fastrpc_context_alloc()
597 return ERR_PTR(-ENOMEM); in fastrpc_context_alloc()
599 ctx->olaps = kcalloc(ctx->nscalars, in fastrpc_context_alloc()
600 sizeof(*ctx->olaps), GFP_KERNEL); in fastrpc_context_alloc()
601 if (!ctx->olaps) { in fastrpc_context_alloc()
602 kfree(ctx->maps); in fastrpc_context_alloc()
604 return ERR_PTR(-ENOMEM); in fastrpc_context_alloc()
606 ctx->args = args; in fastrpc_context_alloc()
613 ctx->sc = sc; in fastrpc_context_alloc()
614 ctx->retval = -1; in fastrpc_context_alloc()
615 ctx->pid = current->pid; in fastrpc_context_alloc()
616 ctx->tgid = user->tgid; in fastrpc_context_alloc()
617 ctx->cctx = cctx; in fastrpc_context_alloc()
618 init_completion(&ctx->work); in fastrpc_context_alloc()
619 INIT_WORK(&ctx->put_work, fastrpc_context_put_wq); in fastrpc_context_alloc()
621 spin_lock(&user->lock); in fastrpc_context_alloc()
622 list_add_tail(&ctx->node, &user->pending); in fastrpc_context_alloc()
623 spin_unlock(&user->lock); in fastrpc_context_alloc()
625 spin_lock_irqsave(&cctx->lock, flags); in fastrpc_context_alloc()
626 ret = idr_alloc_cyclic(&cctx->ctx_idr, ctx, 1, in fastrpc_context_alloc()
629 spin_unlock_irqrestore(&cctx->lock, flags); in fastrpc_context_alloc()
632 ctx->ctxid = ret << 4; in fastrpc_context_alloc()
633 spin_unlock_irqrestore(&cctx->lock, flags); in fastrpc_context_alloc()
635 kref_init(&ctx->refcount); in fastrpc_context_alloc()
639 spin_lock(&user->lock); in fastrpc_context_alloc()
640 list_del(&ctx->node); in fastrpc_context_alloc()
641 spin_unlock(&user->lock); in fastrpc_context_alloc()
643 kfree(ctx->maps); in fastrpc_context_alloc()
644 kfree(ctx->olaps); in fastrpc_context_alloc()
654 struct fastrpc_dma_buf_attachment *a = attachment->priv; in fastrpc_map_dma_buf()
658 table = &a->sgt; in fastrpc_map_dma_buf()
660 ret = dma_map_sgtable(attachment->dev, table, dir, 0); in fastrpc_map_dma_buf()
670 dma_unmap_sgtable(attach->dev, table, dir, 0); in fastrpc_unmap_dma_buf()
675 struct fastrpc_buf *buffer = dmabuf->priv; in fastrpc_release()
684 struct fastrpc_buf *buffer = dmabuf->priv; in fastrpc_dma_buf_attach()
689 return -ENOMEM; in fastrpc_dma_buf_attach()
691 ret = dma_get_sgtable(buffer->dev, &a->sgt, buffer->virt, in fastrpc_dma_buf_attach()
692 FASTRPC_PHYS(buffer->phys), buffer->size); in fastrpc_dma_buf_attach()
694 dev_err(buffer->dev, "failed to get scatterlist from DMA API\n"); in fastrpc_dma_buf_attach()
696 return -EINVAL; in fastrpc_dma_buf_attach()
699 a->dev = attachment->dev; in fastrpc_dma_buf_attach()
700 INIT_LIST_HEAD(&a->node); in fastrpc_dma_buf_attach()
701 attachment->priv = a; in fastrpc_dma_buf_attach()
703 mutex_lock(&buffer->lock); in fastrpc_dma_buf_attach()
704 list_add(&a->node, &buffer->attachments); in fastrpc_dma_buf_attach()
705 mutex_unlock(&buffer->lock); in fastrpc_dma_buf_attach()
713 struct fastrpc_dma_buf_attachment *a = attachment->priv; in fastrpc_dma_buf_detatch()
714 struct fastrpc_buf *buffer = dmabuf->priv; in fastrpc_dma_buf_detatch()
716 mutex_lock(&buffer->lock); in fastrpc_dma_buf_detatch()
717 list_del(&a->node); in fastrpc_dma_buf_detatch()
718 mutex_unlock(&buffer->lock); in fastrpc_dma_buf_detatch()
719 sg_free_table(&a->sgt); in fastrpc_dma_buf_detatch()
725 struct fastrpc_buf *buf = dmabuf->priv; in fastrpc_vmap()
727 iosys_map_set_vaddr(map, buf->virt); in fastrpc_vmap()
735 struct fastrpc_buf *buf = dmabuf->priv; in fastrpc_mmap()
736 size_t size = vma->vm_end - vma->vm_start; in fastrpc_mmap()
738 dma_resv_assert_held(dmabuf->resv); in fastrpc_mmap()
740 return dma_mmap_coherent(buf->dev, vma, buf->virt, in fastrpc_mmap()
741 FASTRPC_PHYS(buf->phys), size); in fastrpc_mmap()
757 struct fastrpc_session_ctx *sess = fl->sctx; in fastrpc_map_create()
767 return -ENOMEM; in fastrpc_map_create()
769 INIT_LIST_HEAD(&map->node); in fastrpc_map_create()
770 kref_init(&map->refcount); in fastrpc_map_create()
772 map->fl = fl; in fastrpc_map_create()
773 map->fd = fd; in fastrpc_map_create()
774 map->buf = dma_buf_get(fd); in fastrpc_map_create()
775 if (IS_ERR(map->buf)) { in fastrpc_map_create()
776 err = PTR_ERR(map->buf); in fastrpc_map_create()
780 map->attach = dma_buf_attach(map->buf, sess->dev); in fastrpc_map_create()
781 if (IS_ERR(map->attach)) { in fastrpc_map_create()
782 dev_err(sess->dev, "Failed to attach dmabuf\n"); in fastrpc_map_create()
783 err = PTR_ERR(map->attach); in fastrpc_map_create()
787 table = dma_buf_map_attachment_unlocked(map->attach, DMA_BIDIRECTIONAL); in fastrpc_map_create()
792 map->table = table; in fastrpc_map_create()
795 map->phys = sg_phys(map->table->sgl); in fastrpc_map_create()
797 map->phys = sg_dma_address(map->table->sgl); in fastrpc_map_create()
798 map->phys += ((u64)fl->sctx->sid << 32); in fastrpc_map_create()
800 map->size = len; in fastrpc_map_create()
801 map->va = sg_virt(map->table->sgl); in fastrpc_map_create()
802 map->len = len; in fastrpc_map_create()
814 dst_perms[1].vmid = fl->cctx->vmperms[0].vmid; in fastrpc_map_create()
816 map->attr = attr; in fastrpc_map_create()
817 err = qcom_scm_assign_mem(map->phys, (u64)map->size, &src_perms, dst_perms, 2); in fastrpc_map_create()
819 dev_err(sess->dev, "Failed to assign memory with phys 0x%llx size 0x%llx err %d", in fastrpc_map_create()
820 map->phys, map->size, err); in fastrpc_map_create()
824 spin_lock(&fl->lock); in fastrpc_map_create()
825 list_add_tail(&map->node, &fl->maps); in fastrpc_map_create()
826 spin_unlock(&fl->lock); in fastrpc_map_create()
832 dma_buf_detach(map->buf, map->attach); in fastrpc_map_create()
834 dma_buf_put(map->buf); in fastrpc_map_create()
842 * Fastrpc payload buffer with metadata looks like:
845 * +---------------------------------+
848 * | (0 - N) |
849 * +---------------------------------+
852 * | (0 - N) |
853 * +---------------------------------+
856 * | (0 - N) |
857 * +---------------------------------+
860 * +---------------------------------+
862 * +---------------------------------+
864 * | (0-N) |
865 * +---------------------------------+
874 sizeof(struct fastrpc_phy_page)) * ctx->nscalars + in fastrpc_get_meta_size()
887 for (oix = 0; oix < ctx->nbufs; oix++) { in fastrpc_get_payload_size()
888 int i = ctx->olaps[oix].raix; in fastrpc_get_payload_size()
890 if (ctx->args[i].fd == 0 || ctx->args[i].fd == -1) { in fastrpc_get_payload_size()
892 if (ctx->olaps[oix].offset == 0) in fastrpc_get_payload_size()
895 size += (ctx->olaps[oix].mend - ctx->olaps[oix].mstart); in fastrpc_get_payload_size()
904 struct device *dev = ctx->fl->sctx->dev; in fastrpc_create_maps()
907 for (i = 0; i < ctx->nscalars; ++i) { in fastrpc_create_maps()
909 if (ctx->args[i].fd == 0 || ctx->args[i].fd == -1 || in fastrpc_create_maps()
910 ctx->args[i].length == 0) in fastrpc_create_maps()
913 err = fastrpc_map_create(ctx->fl, ctx->args[i].fd, in fastrpc_create_maps()
914 ctx->args[i].length, ctx->args[i].attr, &ctx->maps[i]); in fastrpc_create_maps()
917 return -EINVAL; in fastrpc_create_maps()
936 struct device *dev = ctx->fl->sctx->dev; in fastrpc_get_args()
946 inbufs = REMOTE_SCALARS_INBUFS(ctx->sc); in fastrpc_get_args()
954 ctx->msg_sz = pkt_size; in fastrpc_get_args()
956 err = fastrpc_buf_alloc(ctx->fl, dev, pkt_size, &ctx->buf); in fastrpc_get_args()
960 memset(ctx->buf->virt, 0, pkt_size); in fastrpc_get_args()
961 rpra = ctx->buf->virt; in fastrpc_get_args()
962 list = fastrpc_invoke_buf_start(rpra, ctx->nscalars); in fastrpc_get_args()
963 pages = fastrpc_phy_page_start(list, ctx->nscalars); in fastrpc_get_args()
964 args = (uintptr_t)ctx->buf->virt + metalen; in fastrpc_get_args()
965 rlen = pkt_size - metalen; in fastrpc_get_args()
966 ctx->rpra = rpra; in fastrpc_get_args()
968 for (oix = 0; oix < ctx->nbufs; ++oix) { in fastrpc_get_args()
971 i = ctx->olaps[oix].raix; in fastrpc_get_args()
972 len = ctx->args[i].length; in fastrpc_get_args()
982 if (ctx->maps[i]) { in fastrpc_get_args()
985 rpra[i].buf.pv = (u64) ctx->args[i].ptr; in fastrpc_get_args()
986 pages[i].addr = ctx->maps[i]->phys; in fastrpc_get_args()
988 mmap_read_lock(current->mm); in fastrpc_get_args()
989 vma = find_vma(current->mm, ctx->args[i].ptr); in fastrpc_get_args()
991 pages[i].addr += ctx->args[i].ptr - in fastrpc_get_args()
992 vma->vm_start; in fastrpc_get_args()
993 mmap_read_unlock(current->mm); in fastrpc_get_args()
995 pg_start = (ctx->args[i].ptr & PAGE_MASK) >> PAGE_SHIFT; in fastrpc_get_args()
996 pg_end = ((ctx->args[i].ptr + len - 1) & PAGE_MASK) >> in fastrpc_get_args()
998 pages[i].size = (pg_end - pg_start + 1) * PAGE_SIZE; in fastrpc_get_args()
1002 if (ctx->olaps[oix].offset == 0) { in fastrpc_get_args()
1003 rlen -= ALIGN(args, FASTRPC_ALIGN) - args; in fastrpc_get_args()
1007 mlen = ctx->olaps[oix].mend - ctx->olaps[oix].mstart; in fastrpc_get_args()
1012 rpra[i].buf.pv = args - ctx->olaps[oix].offset; in fastrpc_get_args()
1013 pages[i].addr = ctx->buf->phys - in fastrpc_get_args()
1014 ctx->olaps[oix].offset + in fastrpc_get_args()
1015 (pkt_size - rlen); in fastrpc_get_args()
1019 pg_end = ((args + len - 1) & PAGE_MASK) >> PAGE_SHIFT; in fastrpc_get_args()
1020 pages[i].size = (pg_end - pg_start + 1) * PAGE_SIZE; in fastrpc_get_args()
1022 rlen -= mlen; in fastrpc_get_args()
1025 if (i < inbufs && !ctx->maps[i]) { in fastrpc_get_args()
1027 void *src = (void *)(uintptr_t)ctx->args[i].ptr; in fastrpc_get_args()
1032 err = -EFAULT; in fastrpc_get_args()
1041 for (i = ctx->nbufs; i < ctx->nscalars; ++i) { in fastrpc_get_args()
1042 list[i].num = ctx->args[i].length ? 1 : 0; in fastrpc_get_args()
1044 if (ctx->maps[i]) { in fastrpc_get_args()
1045 pages[i].addr = ctx->maps[i]->phys; in fastrpc_get_args()
1046 pages[i].size = ctx->maps[i]->size; in fastrpc_get_args()
1048 rpra[i].dma.fd = ctx->args[i].fd; in fastrpc_get_args()
1049 rpra[i].dma.len = ctx->args[i].length; in fastrpc_get_args()
1050 rpra[i].dma.offset = (u64) ctx->args[i].ptr; in fastrpc_get_args()
1063 union fastrpc_remote_arg *rpra = ctx->rpra; in fastrpc_put_args()
1064 struct fastrpc_user *fl = ctx->fl; in fastrpc_put_args()
1071 inbufs = REMOTE_SCALARS_INBUFS(ctx->sc); in fastrpc_put_args()
1072 outbufs = REMOTE_SCALARS_OUTBUFS(ctx->sc); in fastrpc_put_args()
1073 handles = REMOTE_SCALARS_INHANDLES(ctx->sc) + REMOTE_SCALARS_OUTHANDLES(ctx->sc); in fastrpc_put_args()
1074 list = fastrpc_invoke_buf_start(rpra, ctx->nscalars); in fastrpc_put_args()
1075 pages = fastrpc_phy_page_start(list, ctx->nscalars); in fastrpc_put_args()
1078 for (i = inbufs; i < ctx->nbufs; ++i) { in fastrpc_put_args()
1079 if (!ctx->maps[i]) { in fastrpc_put_args()
1081 void *dst = (void *)(uintptr_t)ctx->args[i].ptr; in fastrpc_put_args()
1086 return -EFAULT; in fastrpc_put_args()
1109 struct fastrpc_user *fl = ctx->fl; in fastrpc_invoke_send()
1110 struct fastrpc_msg *msg = &ctx->msg; in fastrpc_invoke_send()
1113 cctx = fl->cctx; in fastrpc_invoke_send()
1114 msg->pid = fl->tgid; in fastrpc_invoke_send()
1115 msg->tid = current->pid; in fastrpc_invoke_send()
1118 msg->pid = 0; in fastrpc_invoke_send()
1120 msg->ctx = ctx->ctxid | fl->pd; in fastrpc_invoke_send()
1121 msg->handle = handle; in fastrpc_invoke_send()
1122 msg->sc = ctx->sc; in fastrpc_invoke_send()
1123 msg->addr = ctx->buf ? ctx->buf->phys : 0; in fastrpc_invoke_send()
1124 msg->size = roundup(ctx->msg_sz, PAGE_SIZE); in fastrpc_invoke_send()
1127 ret = rpmsg_send(cctx->rpdev->ept, (void *)msg, sizeof(*msg)); in fastrpc_invoke_send()
1145 if (!fl->sctx) in fastrpc_internal_invoke()
1146 return -EINVAL; in fastrpc_internal_invoke()
1148 if (!fl->cctx->rpdev) in fastrpc_internal_invoke()
1149 return -EPIPE; in fastrpc_internal_invoke()
1152 …dev_warn_ratelimited(fl->sctx->dev, "user app trying to send a kernel RPC message (%d)\n", handle… in fastrpc_internal_invoke()
1153 return -EPERM; in fastrpc_internal_invoke()
1167 err = fastrpc_invoke_send(fl->sctx, ctx, kernel, handle); in fastrpc_internal_invoke()
1172 if (!wait_for_completion_timeout(&ctx->work, 10 * HZ)) in fastrpc_internal_invoke()
1173 err = -ETIMEDOUT; in fastrpc_internal_invoke()
1175 err = wait_for_completion_interruptible(&ctx->work); in fastrpc_internal_invoke()
1189 err = ctx->retval; in fastrpc_internal_invoke()
1194 if (err != -ERESTARTSYS && err != -ETIMEDOUT) { in fastrpc_internal_invoke()
1195 /* We are done with this compute context */ in fastrpc_internal_invoke()
1196 spin_lock(&fl->lock); in fastrpc_internal_invoke()
1197 list_del(&ctx->node); in fastrpc_internal_invoke()
1198 spin_unlock(&fl->lock); in fastrpc_internal_invoke()
1202 if (err == -ERESTARTSYS) { in fastrpc_internal_invoke()
1203 list_for_each_entry_safe(buf, b, &fl->mmaps, node) { in fastrpc_internal_invoke()
1204 list_del(&buf->node); in fastrpc_internal_invoke()
1205 list_add_tail(&buf->node, &fl->cctx->invoke_interrupted_mmaps); in fastrpc_internal_invoke()
1210 dev_dbg(fl->sctx->dev, "Error: Invoke Failed %d\n", err); in fastrpc_internal_invoke()
1217 /* Check if the device node is non-secure and channel is secure*/ in is_session_rejected()
1218 if (!fl->is_secure_dev && fl->cctx->secure) { in is_session_rejected()
1224 if (!fl->cctx->unsigned_support || !unsigned_pd_request) { in is_session_rejected()
1225 dev_err(&fl->cctx->rpdev->dev, "Error: Untrusted application trying to offload to signed PD"); in is_session_rejected()
1250 return -ENOMEM; in fastrpc_init_create_static_process()
1253 err = -EFAULT; in fastrpc_init_create_static_process()
1258 err = -EINVAL; in fastrpc_init_create_static_process()
1264 err = -ENOMEM; in fastrpc_init_create_static_process()
1269 err = -EFAULT; in fastrpc_init_create_static_process()
1273 if (!fl->cctx->remote_heap) { in fastrpc_init_create_static_process()
1274 err = fastrpc_remote_heap_alloc(fl, fl->sctx->dev, init.memlen, in fastrpc_init_create_static_process()
1275 &fl->cctx->remote_heap); in fastrpc_init_create_static_process()
1280 if (fl->cctx->vmcount) { in fastrpc_init_create_static_process()
1283 err = qcom_scm_assign_mem(fl->cctx->remote_heap->phys, in fastrpc_init_create_static_process()
1284 (u64)fl->cctx->remote_heap->size, in fastrpc_init_create_static_process()
1286 fl->cctx->vmperms, fl->cctx->vmcount); in fastrpc_init_create_static_process()
1288 dev_err(fl->sctx->dev, "Failed to assign memory with phys 0x%llx size 0x%llx err %d", in fastrpc_init_create_static_process()
1289 fl->cctx->remote_heap->phys, fl->cctx->remote_heap->size, err); in fastrpc_init_create_static_process()
1295 inbuf.pgid = fl->tgid; in fastrpc_init_create_static_process()
1298 fl->pd = USER_PD; in fastrpc_init_create_static_process()
1302 args[0].fd = -1; in fastrpc_init_create_static_process()
1306 args[1].fd = -1; in fastrpc_init_create_static_process()
1308 pages[0].addr = fl->cctx->remote_heap->phys; in fastrpc_init_create_static_process()
1309 pages[0].size = fl->cctx->remote_heap->size; in fastrpc_init_create_static_process()
1313 args[2].fd = -1; in fastrpc_init_create_static_process()
1326 if (fl->cctx->vmcount) { in fastrpc_init_create_static_process()
1331 for (i = 0; i < fl->cctx->vmcount; i++) in fastrpc_init_create_static_process()
1332 src_perms |= BIT(fl->cctx->vmperms[i].vmid); in fastrpc_init_create_static_process()
1336 err = qcom_scm_assign_mem(fl->cctx->remote_heap->phys, in fastrpc_init_create_static_process()
1337 (u64)fl->cctx->remote_heap->size, in fastrpc_init_create_static_process()
1340 dev_err(fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d", in fastrpc_init_create_static_process()
1341 fl->cctx->remote_heap->phys, fl->cctx->remote_heap->size, err); in fastrpc_init_create_static_process()
1344 fastrpc_buf_free(fl->cctx->remote_heap); in fastrpc_init_create_static_process()
1376 return -ENOMEM; in fastrpc_init_create_process()
1379 err = -EFAULT; in fastrpc_init_create_process()
1387 err = -ECONNREFUSED; in fastrpc_init_create_process()
1392 err = -EINVAL; in fastrpc_init_create_process()
1396 inbuf.pgid = fl->tgid; in fastrpc_init_create_process()
1397 inbuf.namelen = strlen(current->comm) + 1; in fastrpc_init_create_process()
1402 fl->pd = USER_PD; in fastrpc_init_create_process()
1412 err = fastrpc_buf_alloc(fl, fl->sctx->dev, memlen, in fastrpc_init_create_process()
1417 fl->init_mem = imem; in fastrpc_init_create_process()
1420 args[0].fd = -1; in fastrpc_init_create_process()
1422 args[1].ptr = (u64)(uintptr_t)current->comm; in fastrpc_init_create_process()
1424 args[1].fd = -1; in fastrpc_init_create_process()
1430 pages[0].addr = imem->phys; in fastrpc_init_create_process()
1431 pages[0].size = imem->size; in fastrpc_init_create_process()
1435 args[3].fd = -1; in fastrpc_init_create_process()
1439 args[4].fd = -1; in fastrpc_init_create_process()
1443 args[5].fd = -1; in fastrpc_init_create_process()
1459 fl->init_mem = NULL; in fastrpc_init_create_process()
1476 spin_lock_irqsave(&cctx->lock, flags); in fastrpc_session_alloc()
1477 for (i = 0; i < cctx->sesscount; i++) { in fastrpc_session_alloc()
1478 if (!cctx->session[i].used && cctx->session[i].valid) { in fastrpc_session_alloc()
1479 cctx->session[i].used = true; in fastrpc_session_alloc()
1480 session = &cctx->session[i]; in fastrpc_session_alloc()
1484 spin_unlock_irqrestore(&cctx->lock, flags); in fastrpc_session_alloc()
1494 spin_lock_irqsave(&cctx->lock, flags); in fastrpc_session_free()
1495 session->used = false; in fastrpc_session_free()
1496 spin_unlock_irqrestore(&cctx->lock, flags); in fastrpc_session_free()
1505 tgid = fl->tgid; in fastrpc_release_current_dsp_process()
1508 args[0].fd = -1; in fastrpc_release_current_dsp_process()
1517 struct fastrpc_user *fl = (struct fastrpc_user *)file->private_data; in fastrpc_device_release()
1518 struct fastrpc_channel_ctx *cctx = fl->cctx; in fastrpc_device_release()
1526 spin_lock_irqsave(&cctx->lock, flags); in fastrpc_device_release()
1527 list_del(&fl->user); in fastrpc_device_release()
1528 spin_unlock_irqrestore(&cctx->lock, flags); in fastrpc_device_release()
1530 if (fl->init_mem) in fastrpc_device_release()
1531 fastrpc_buf_free(fl->init_mem); in fastrpc_device_release()
1533 list_for_each_entry_safe(ctx, n, &fl->pending, node) { in fastrpc_device_release()
1534 list_del(&ctx->node); in fastrpc_device_release()
1538 list_for_each_entry_safe(map, m, &fl->maps, node) in fastrpc_device_release()
1541 list_for_each_entry_safe(buf, b, &fl->mmaps, node) { in fastrpc_device_release()
1542 list_del(&buf->node); in fastrpc_device_release()
1546 fastrpc_session_free(cctx, fl->sctx); in fastrpc_device_release()
1549 mutex_destroy(&fl->mutex); in fastrpc_device_release()
1551 file->private_data = NULL; in fastrpc_device_release()
1563 fdevice = miscdev_to_fdevice(filp->private_data); in fastrpc_device_open()
1564 cctx = fdevice->cctx; in fastrpc_device_open()
1568 return -ENOMEM; in fastrpc_device_open()
1573 filp->private_data = fl; in fastrpc_device_open()
1574 spin_lock_init(&fl->lock); in fastrpc_device_open()
1575 mutex_init(&fl->mutex); in fastrpc_device_open()
1576 INIT_LIST_HEAD(&fl->pending); in fastrpc_device_open()
1577 INIT_LIST_HEAD(&fl->maps); in fastrpc_device_open()
1578 INIT_LIST_HEAD(&fl->mmaps); in fastrpc_device_open()
1579 INIT_LIST_HEAD(&fl->user); in fastrpc_device_open()
1580 fl->tgid = current->tgid; in fastrpc_device_open()
1581 fl->cctx = cctx; in fastrpc_device_open()
1582 fl->is_secure_dev = fdevice->secure; in fastrpc_device_open()
1584 fl->sctx = fastrpc_session_alloc(cctx); in fastrpc_device_open()
1585 if (!fl->sctx) { in fastrpc_device_open()
1586 dev_err(&cctx->rpdev->dev, "No session available\n"); in fastrpc_device_open()
1587 mutex_destroy(&fl->mutex); in fastrpc_device_open()
1590 return -EBUSY; in fastrpc_device_open()
1593 spin_lock_irqsave(&cctx->lock, flags); in fastrpc_device_open()
1594 list_add_tail(&fl->user, &cctx->users); in fastrpc_device_open()
1595 spin_unlock_irqrestore(&cctx->lock, flags); in fastrpc_device_open()
1608 return -EFAULT; in fastrpc_dmabuf_alloc()
1610 err = fastrpc_buf_alloc(fl, fl->sctx->dev, bp.size, &buf); in fastrpc_dmabuf_alloc()
1617 buf->dmabuf = dma_buf_export(&exp_info); in fastrpc_dmabuf_alloc()
1618 if (IS_ERR(buf->dmabuf)) { in fastrpc_dmabuf_alloc()
1619 err = PTR_ERR(buf->dmabuf); in fastrpc_dmabuf_alloc()
1624 bp.fd = dma_buf_fd(buf->dmabuf, O_ACCMODE); in fastrpc_dmabuf_alloc()
1626 dma_buf_put(buf->dmabuf); in fastrpc_dmabuf_alloc()
1627 return -EINVAL; in fastrpc_dmabuf_alloc()
1639 return -EFAULT; in fastrpc_dmabuf_alloc()
1648 int tgid = fl->tgid; in fastrpc_init_attach()
1653 args[0].fd = -1; in fastrpc_init_attach()
1655 fl->pd = pd; in fastrpc_init_attach()
1669 return -EFAULT; in fastrpc_invoke()
1676 return -ENOMEM; in fastrpc_invoke()
1681 return -EFAULT; in fastrpc_invoke()
1701 args[0].fd = -1; in fastrpc_get_info_from_dsp()
1704 args[1].fd = -1; in fastrpc_get_info_from_dsp()
1705 fl->pd = USER_PD; in fastrpc_get_info_from_dsp()
1714 struct fastrpc_channel_ctx *cctx = fl->cctx; in fastrpc_get_info_from_kernel()
1715 uint32_t attribute_id = cap->attribute_id; in fastrpc_get_info_from_kernel()
1718 uint32_t domain = cap->domain; in fastrpc_get_info_from_kernel()
1721 spin_lock_irqsave(&cctx->lock, flags); in fastrpc_get_info_from_kernel()
1723 if (cctx->valid_attributes) { in fastrpc_get_info_from_kernel()
1724 spin_unlock_irqrestore(&cctx->lock, flags); in fastrpc_get_info_from_kernel()
1727 spin_unlock_irqrestore(&cctx->lock, flags); in fastrpc_get_info_from_kernel()
1731 return -ENOMEM; in fastrpc_get_info_from_kernel()
1735 dev_info(&cctx->rpdev->dev, in fastrpc_get_info_from_kernel()
1738 return -EOPNOTSUPP; in fastrpc_get_info_from_kernel()
1740 dev_err(&cctx->rpdev->dev, "Error: dsp information is incorrect err: %d\n", err); in fastrpc_get_info_from_kernel()
1745 spin_lock_irqsave(&cctx->lock, flags); in fastrpc_get_info_from_kernel()
1746 memcpy(cctx->dsp_attributes, dsp_attributes, FASTRPC_MAX_DSP_ATTRIBUTES_LEN); in fastrpc_get_info_from_kernel()
1747 cctx->valid_attributes = true; in fastrpc_get_info_from_kernel()
1748 spin_unlock_irqrestore(&cctx->lock, flags); in fastrpc_get_info_from_kernel()
1751 cap->capability = cctx->dsp_attributes[attribute_id]; in fastrpc_get_info_from_kernel()
1761 return -EFAULT; in fastrpc_get_dsp_info()
1765 dev_err(&fl->cctx->rpdev->dev, "Error: Invalid domain id:%d, err:%d\n", in fastrpc_get_dsp_info()
1767 return -ECHRNG; in fastrpc_get_dsp_info()
1770 /* Fastrpc Capablities does not support modem domain */ in fastrpc_get_dsp_info()
1772 dev_err(&fl->cctx->rpdev->dev, "Error: modem not supported %d\n", err); in fastrpc_get_dsp_info()
1773 return -ECHRNG; in fastrpc_get_dsp_info()
1777 dev_err(&fl->cctx->rpdev->dev, "Error: invalid attribute: %d, err: %d\n", in fastrpc_get_dsp_info()
1779 return -EOVERFLOW; in fastrpc_get_dsp_info()
1787 return -EFAULT; in fastrpc_get_dsp_info()
1796 struct device *dev = fl->sctx->dev; in fastrpc_req_munmap_impl()
1800 req_msg.pgid = fl->tgid; in fastrpc_req_munmap_impl()
1801 req_msg.size = buf->size; in fastrpc_req_munmap_impl()
1802 req_msg.vaddr = buf->raddr; in fastrpc_req_munmap_impl()
1811 dev_dbg(dev, "unmmap\tpt 0x%09lx OK\n", buf->raddr); in fastrpc_req_munmap_impl()
1812 spin_lock(&fl->lock); in fastrpc_req_munmap_impl()
1813 list_del(&buf->node); in fastrpc_req_munmap_impl()
1814 spin_unlock(&fl->lock); in fastrpc_req_munmap_impl()
1817 dev_err(dev, "unmmap\tpt 0x%09lx ERROR\n", buf->raddr); in fastrpc_req_munmap_impl()
1827 struct device *dev = fl->sctx->dev; in fastrpc_req_munmap()
1830 return -EFAULT; in fastrpc_req_munmap()
1832 spin_lock(&fl->lock); in fastrpc_req_munmap()
1833 list_for_each_entry_safe(iter, b, &fl->mmaps, node) { in fastrpc_req_munmap()
1834 if ((iter->raddr == req.vaddrout) && (iter->size == req.size)) { in fastrpc_req_munmap()
1839 spin_unlock(&fl->lock); in fastrpc_req_munmap()
1844 return -EINVAL; in fastrpc_req_munmap()
1858 struct device *dev = fl->sctx->dev; in fastrpc_req_mmap()
1863 return -EFAULT; in fastrpc_req_mmap()
1868 return -EINVAL; in fastrpc_req_mmap()
1873 return -EINVAL; in fastrpc_req_mmap()
1886 req_msg.pgid = fl->tgid; in fastrpc_req_mmap()
1894 pages.addr = buf->phys; in fastrpc_req_mmap()
1895 pages.size = buf->size; in fastrpc_req_mmap()
1907 dev_err(dev, "mmap error (len 0x%08llx)\n", buf->size); in fastrpc_req_mmap()
1912 buf->raddr = (uintptr_t) rsp_msg.vaddr; in fastrpc_req_mmap()
1918 if (req.flags == ADSP_MMAP_REMOTE_HEAP_ADDR && fl->cctx->vmcount) { in fastrpc_req_mmap()
1921 err = qcom_scm_assign_mem(buf->phys, (u64)buf->size, in fastrpc_req_mmap()
1922 &src_perms, fl->cctx->vmperms, fl->cctx->vmcount); in fastrpc_req_mmap()
1924 dev_err(fl->sctx->dev, "Failed to assign memory phys 0x%llx size 0x%llx err %d", in fastrpc_req_mmap()
1925 buf->phys, buf->size, err); in fastrpc_req_mmap()
1930 spin_lock(&fl->lock); in fastrpc_req_mmap()
1931 list_add_tail(&buf->node, &fl->mmaps); in fastrpc_req_mmap()
1932 spin_unlock(&fl->lock); in fastrpc_req_mmap()
1935 err = -EFAULT; in fastrpc_req_mmap()
1940 buf->raddr, buf->size); in fastrpc_req_mmap()
1959 struct device *dev = fl->sctx->dev; in fastrpc_req_mem_unmap_impl()
1961 spin_lock(&fl->lock); in fastrpc_req_mem_unmap_impl()
1962 list_for_each_entry_safe(iter, m, &fl->maps, node) { in fastrpc_req_mem_unmap_impl()
1963 if ((req->fd < 0 || iter->fd == req->fd) && (iter->raddr == req->vaddr)) { in fastrpc_req_mem_unmap_impl()
1969 spin_unlock(&fl->lock); in fastrpc_req_mem_unmap_impl()
1973 return -EINVAL; in fastrpc_req_mem_unmap_impl()
1976 req_msg.pgid = fl->tgid; in fastrpc_req_mem_unmap_impl()
1977 req_msg.len = map->len; in fastrpc_req_mem_unmap_impl()
1978 req_msg.vaddrin = map->raddr; in fastrpc_req_mem_unmap_impl()
1979 req_msg.fd = map->fd; in fastrpc_req_mem_unmap_impl()
1988 dev_err(dev, "unmmap\tpt fd = %d, 0x%09llx error\n", map->fd, map->raddr); in fastrpc_req_mem_unmap_impl()
2001 return -EFAULT; in fastrpc_req_mem_unmap()
2014 struct device *dev = fl->sctx->dev; in fastrpc_req_mem_map()
2020 return -EFAULT; in fastrpc_req_mem_map()
2029 req_msg.pgid = fl->tgid; in fastrpc_req_mem_map()
2033 map->va = (void *) (uintptr_t) req.vaddrin; in fastrpc_req_mem_map()
2041 pages.addr = map->phys; in fastrpc_req_mem_map()
2042 pages.size = map->size; in fastrpc_req_mem_map()
2057 req.fd, req.vaddrin, map->size); in fastrpc_req_mem_map()
2062 map->raddr = rsp_msg.vaddr; in fastrpc_req_mem_map()
2070 req_unmap.length = map->size; in fastrpc_req_mem_map()
2072 return -EFAULT; in fastrpc_req_mem_map()
2086 struct fastrpc_user *fl = (struct fastrpc_user *)file->private_data; in fastrpc_device_ioctl()
2125 err = -ENOTTY; in fastrpc_device_ioctl()
2143 struct device *dev = &pdev->dev; in fastrpc_cb_probe()
2148 cctx = dev_get_drvdata(dev->parent); in fastrpc_cb_probe()
2150 return -EINVAL; in fastrpc_cb_probe()
2152 of_property_read_u32(dev->of_node, "qcom,nsessions", &sessions); in fastrpc_cb_probe()
2154 spin_lock_irqsave(&cctx->lock, flags); in fastrpc_cb_probe()
2155 if (cctx->sesscount >= FASTRPC_MAX_SESSIONS) { in fastrpc_cb_probe()
2156 dev_err(&pdev->dev, "too many sessions\n"); in fastrpc_cb_probe()
2157 spin_unlock_irqrestore(&cctx->lock, flags); in fastrpc_cb_probe()
2158 return -ENOSPC; in fastrpc_cb_probe()
2160 sess = &cctx->session[cctx->sesscount++]; in fastrpc_cb_probe()
2161 sess->used = false; in fastrpc_cb_probe()
2162 sess->valid = true; in fastrpc_cb_probe()
2163 sess->dev = dev; in fastrpc_cb_probe()
2166 if (of_property_read_u32(dev->of_node, "reg", &sess->sid)) in fastrpc_cb_probe()
2167 dev_info(dev, "FastRPC Session ID not specified in DT\n"); in fastrpc_cb_probe()
2173 if (cctx->sesscount >= FASTRPC_MAX_SESSIONS) in fastrpc_cb_probe()
2175 dup_sess = &cctx->session[cctx->sesscount++]; in fastrpc_cb_probe()
2179 spin_unlock_irqrestore(&cctx->lock, flags); in fastrpc_cb_probe()
2182 dev_err(dev, "32-bit DMA enable failed\n"); in fastrpc_cb_probe()
2191 struct fastrpc_channel_ctx *cctx = dev_get_drvdata(pdev->dev.parent); in fastrpc_cb_remove()
2192 struct fastrpc_session_ctx *sess = dev_get_drvdata(&pdev->dev); in fastrpc_cb_remove()
2196 spin_lock_irqsave(&cctx->lock, flags); in fastrpc_cb_remove()
2198 if (cctx->session[i].sid == sess->sid) { in fastrpc_cb_remove()
2199 cctx->session[i].valid = false; in fastrpc_cb_remove()
2200 cctx->sesscount--; in fastrpc_cb_remove()
2203 spin_unlock_irqrestore(&cctx->lock, flags); in fastrpc_cb_remove()
2209 { .compatible = "qcom,fastrpc-compute-cb", },
2217 .name = "qcom,fastrpc-cb",
2231 return -ENOMEM; in fastrpc_device_register()
2233 fdev->secure = is_secured; in fastrpc_device_register()
2234 fdev->cctx = cctx; in fastrpc_device_register()
2235 fdev->miscdev.minor = MISC_DYNAMIC_MINOR; in fastrpc_device_register()
2236 fdev->miscdev.fops = &fastrpc_fops; in fastrpc_device_register()
2237 fdev->miscdev.name = devm_kasprintf(dev, GFP_KERNEL, "fastrpc-%s%s", in fastrpc_device_register()
2238 domain, is_secured ? "-secure" : ""); in fastrpc_device_register()
2239 if (!fdev->miscdev.name) in fastrpc_device_register()
2240 return -ENOMEM; in fastrpc_device_register()
2242 err = misc_register(&fdev->miscdev); in fastrpc_device_register()
2245 cctx->secure_fdevice = fdev; in fastrpc_device_register()
2247 cctx->fdevice = fdev; in fastrpc_device_register()
2255 struct device *rdev = &rpdev->dev; in fastrpc_rpmsg_probe()
2257 int i, err, domain_id = -1, vmcount; in fastrpc_rpmsg_probe()
2262 err = of_property_read_string(rdev->of_node, "label", &domain); in fastrpc_rpmsg_probe()
2264 dev_info(rdev, "FastRPC Domain not specified in DT\n"); in fastrpc_rpmsg_probe()
2276 dev_info(rdev, "FastRPC Invalid Domain ID %d\n", domain_id); in fastrpc_rpmsg_probe()
2277 return -EINVAL; in fastrpc_rpmsg_probe()
2280 if (of_reserved_mem_device_init_by_idx(rdev, rdev->of_node, 0)) in fastrpc_rpmsg_probe()
2281 dev_info(rdev, "no reserved DMA memory for FASTRPC\n"); in fastrpc_rpmsg_probe()
2283 vmcount = of_property_read_variable_u32_array(rdev->of_node, in fastrpc_rpmsg_probe()
2288 return -EPROBE_DEFER; in fastrpc_rpmsg_probe()
2292 return -ENOMEM; in fastrpc_rpmsg_probe()
2295 data->vmcount = vmcount; in fastrpc_rpmsg_probe()
2296 for (i = 0; i < data->vmcount; i++) { in fastrpc_rpmsg_probe()
2297 data->vmperms[i].vmid = vmids[i]; in fastrpc_rpmsg_probe()
2298 data->vmperms[i].perm = QCOM_SCM_PERM_RWX; in fastrpc_rpmsg_probe()
2302 secure_dsp = !(of_property_read_bool(rdev->of_node, "qcom,non-secure-domain")); in fastrpc_rpmsg_probe()
2303 data->secure = secure_dsp; in fastrpc_rpmsg_probe()
2310 data->unsigned_support = false; in fastrpc_rpmsg_probe()
2316 data->unsigned_support = true; in fastrpc_rpmsg_probe()
2327 err = -EINVAL; in fastrpc_rpmsg_probe()
2331 kref_init(&data->refcount); in fastrpc_rpmsg_probe()
2333 dev_set_drvdata(&rpdev->dev, data); in fastrpc_rpmsg_probe()
2334 rdev->dma_mask = &data->dma_mask; in fastrpc_rpmsg_probe()
2336 INIT_LIST_HEAD(&data->users); in fastrpc_rpmsg_probe()
2337 INIT_LIST_HEAD(&data->invoke_interrupted_mmaps); in fastrpc_rpmsg_probe()
2338 spin_lock_init(&data->lock); in fastrpc_rpmsg_probe()
2339 idr_init(&data->ctx_idr); in fastrpc_rpmsg_probe()
2340 data->domain_id = domain_id; in fastrpc_rpmsg_probe()
2341 data->rpdev = rpdev; in fastrpc_rpmsg_probe()
2343 err = of_platform_populate(rdev->of_node, NULL, NULL, rdev); in fastrpc_rpmsg_probe()
2350 if (data->fdevice) in fastrpc_rpmsg_probe()
2351 misc_deregister(&data->fdevice->miscdev); in fastrpc_rpmsg_probe()
2352 if (data->secure_fdevice) in fastrpc_rpmsg_probe()
2353 misc_deregister(&data->secure_fdevice->miscdev); in fastrpc_rpmsg_probe()
2364 spin_lock(&user->lock); in fastrpc_notify_users()
2365 list_for_each_entry(ctx, &user->pending, node) { in fastrpc_notify_users()
2366 ctx->retval = -EPIPE; in fastrpc_notify_users()
2367 complete(&ctx->work); in fastrpc_notify_users()
2369 spin_unlock(&user->lock); in fastrpc_notify_users()
2374 struct fastrpc_channel_ctx *cctx = dev_get_drvdata(&rpdev->dev); in fastrpc_rpmsg_remove()
2380 spin_lock_irqsave(&cctx->lock, flags); in fastrpc_rpmsg_remove()
2381 cctx->rpdev = NULL; in fastrpc_rpmsg_remove()
2382 list_for_each_entry(user, &cctx->users, user) in fastrpc_rpmsg_remove()
2384 spin_unlock_irqrestore(&cctx->lock, flags); in fastrpc_rpmsg_remove()
2386 if (cctx->fdevice) in fastrpc_rpmsg_remove()
2387 misc_deregister(&cctx->fdevice->miscdev); in fastrpc_rpmsg_remove()
2389 if (cctx->secure_fdevice) in fastrpc_rpmsg_remove()
2390 misc_deregister(&cctx->secure_fdevice->miscdev); in fastrpc_rpmsg_remove()
2392 list_for_each_entry_safe(buf, b, &cctx->invoke_interrupted_mmaps, node) in fastrpc_rpmsg_remove()
2393 list_del(&buf->node); in fastrpc_rpmsg_remove()
2395 if (cctx->remote_heap) in fastrpc_rpmsg_remove()
2396 fastrpc_buf_free(cctx->remote_heap); in fastrpc_rpmsg_remove()
2398 of_platform_depopulate(&rpdev->dev); in fastrpc_rpmsg_remove()
2406 struct fastrpc_channel_ctx *cctx = dev_get_drvdata(&rpdev->dev); in fastrpc_rpmsg_callback()
2413 return -EINVAL; in fastrpc_rpmsg_callback()
2415 ctxid = ((rsp->ctx & FASTRPC_CTXID_MASK) >> 4); in fastrpc_rpmsg_callback()
2417 spin_lock_irqsave(&cctx->lock, flags); in fastrpc_rpmsg_callback()
2418 ctx = idr_find(&cctx->ctx_idr, ctxid); in fastrpc_rpmsg_callback()
2419 spin_unlock_irqrestore(&cctx->lock, flags); in fastrpc_rpmsg_callback()
2422 dev_err(&rpdev->dev, "No context ID matches response\n"); in fastrpc_rpmsg_callback()
2423 return -ENOENT; in fastrpc_rpmsg_callback()
2426 ctx->retval = rsp->retval; in fastrpc_rpmsg_callback()
2427 complete(&ctx->work); in fastrpc_rpmsg_callback()
2434 schedule_work(&ctx->put_work); in fastrpc_rpmsg_callback()
2440 { .compatible = "qcom,fastrpc" },
2450 .name = "qcom,fastrpc",
2461 pr_err("fastrpc: failed to register cb driver\n"); in fastrpc_init()
2467 pr_err("fastrpc: failed to register rpmsg driver\n"); in fastrpc_init()