1c3bab824SSuman Anna // SPDX-License-Identifier: GPL-2.0 2bcabbccaSOhad Ben-Cohen /* 3bcabbccaSOhad Ben-Cohen * Virtio-based remote processor messaging bus 4bcabbccaSOhad Ben-Cohen * 5bcabbccaSOhad Ben-Cohen * Copyright (C) 2011 Texas Instruments, Inc. 6bcabbccaSOhad Ben-Cohen * Copyright (C) 2011 Google, Inc. 7bcabbccaSOhad Ben-Cohen * 8bcabbccaSOhad Ben-Cohen * Ohad Ben-Cohen <ohad@wizery.com> 9bcabbccaSOhad Ben-Cohen * Brian Swetland <swetland@google.com> 10bcabbccaSOhad Ben-Cohen */ 11bcabbccaSOhad Ben-Cohen 12bcabbccaSOhad Ben-Cohen #define pr_fmt(fmt) "%s: " fmt, __func__ 13bcabbccaSOhad Ben-Cohen 146de1c933SLoic Pallardy #include <linux/dma-mapping.h> 156de1c933SLoic Pallardy #include <linux/idr.h> 166de1c933SLoic Pallardy #include <linux/jiffies.h> 17bcabbccaSOhad Ben-Cohen #include <linux/kernel.h> 18bcabbccaSOhad Ben-Cohen #include <linux/module.h> 196de1c933SLoic Pallardy #include <linux/mutex.h> 206de1c933SLoic Pallardy #include <linux/of_device.h> 216de1c933SLoic Pallardy #include <linux/rpmsg.h> 226de1c933SLoic Pallardy #include <linux/scatterlist.h> 236de1c933SLoic Pallardy #include <linux/slab.h> 246de1c933SLoic Pallardy #include <linux/sched.h> 25bcabbccaSOhad Ben-Cohen #include <linux/virtio.h> 26111d1089SGuennadi Liakhovetski #include <linux/virtio_byteorder.h> 27bcabbccaSOhad Ben-Cohen #include <linux/virtio_ids.h> 28bcabbccaSOhad Ben-Cohen #include <linux/virtio_config.h> 29bcabbccaSOhad Ben-Cohen #include <linux/wait.h> 30bcabbccaSOhad Ben-Cohen 318b881c07SBjorn Andersson #include "rpmsg_internal.h" 328b881c07SBjorn Andersson 33bcabbccaSOhad Ben-Cohen /** 34bcabbccaSOhad Ben-Cohen * struct virtproc_info - virtual remote processor state 35bcabbccaSOhad Ben-Cohen * @vdev: the virtio device 36bcabbccaSOhad Ben-Cohen * @rvq: rx virtqueue 37bcabbccaSOhad Ben-Cohen * @svq: tx virtqueue 38bcabbccaSOhad Ben-Cohen * @rbufs: kernel address of rx buffers 39bcabbccaSOhad Ben-Cohen * @sbufs: kernel address of tx buffers 40b1b98914SSuman Anna * @num_bufs: total number of buffers for rx and tx 41f93848f9SLoic Pallardy * @buf_size: size of one rx or tx buffer 42bcabbccaSOhad Ben-Cohen * @last_sbuf: index of last tx buffer used 43bcabbccaSOhad Ben-Cohen * @bufs_dma: dma base addr of the buffers 44bcabbccaSOhad Ben-Cohen * @tx_lock: protects svq, sbufs and sleepers, to allow concurrent senders. 45bcabbccaSOhad Ben-Cohen * sending a message might require waking up a dozing remote 46bcabbccaSOhad Ben-Cohen * processor, which involves sleeping, hence the mutex. 47bcabbccaSOhad Ben-Cohen * @endpoints: idr of local endpoints, allows fast retrieval 48bcabbccaSOhad Ben-Cohen * @endpoints_lock: lock of the endpoints set 49bcabbccaSOhad Ben-Cohen * @sendq: wait queue of sending contexts waiting for a tx buffers 50bcabbccaSOhad Ben-Cohen * @sleepers: number of senders that are waiting for a tx buffer 51bcabbccaSOhad Ben-Cohen * @ns_ept: the bus's name service endpoint 52bcabbccaSOhad Ben-Cohen * 53bcabbccaSOhad Ben-Cohen * This structure stores the rpmsg state of a given virtio remote processor 54bcabbccaSOhad Ben-Cohen * device (there might be several virtio proc devices for each physical 55bcabbccaSOhad Ben-Cohen * remote processor). 56bcabbccaSOhad Ben-Cohen */ 57bcabbccaSOhad Ben-Cohen struct virtproc_info { 58bcabbccaSOhad Ben-Cohen struct virtio_device *vdev; 59bcabbccaSOhad Ben-Cohen struct virtqueue *rvq, *svq; 60bcabbccaSOhad Ben-Cohen void *rbufs, *sbufs; 61b1b98914SSuman Anna unsigned int num_bufs; 62f93848f9SLoic Pallardy unsigned int buf_size; 63bcabbccaSOhad Ben-Cohen int last_sbuf; 64bcabbccaSOhad Ben-Cohen dma_addr_t bufs_dma; 65bcabbccaSOhad Ben-Cohen struct mutex tx_lock; 66bcabbccaSOhad Ben-Cohen struct idr endpoints; 67bcabbccaSOhad Ben-Cohen struct mutex endpoints_lock; 68bcabbccaSOhad Ben-Cohen wait_queue_head_t sendq; 69bcabbccaSOhad Ben-Cohen atomic_t sleepers; 70bcabbccaSOhad Ben-Cohen struct rpmsg_endpoint *ns_ept; 71bcabbccaSOhad Ben-Cohen }; 72bcabbccaSOhad Ben-Cohen 73e88dae5dSBjorn Andersson /* The feature bitmap for virtio rpmsg */ 74e88dae5dSBjorn Andersson #define VIRTIO_RPMSG_F_NS 0 /* RP supports name service notifications */ 75e88dae5dSBjorn Andersson 76e88dae5dSBjorn Andersson /** 77e88dae5dSBjorn Andersson * struct rpmsg_hdr - common header for all rpmsg messages 78e88dae5dSBjorn Andersson * @src: source address 79e88dae5dSBjorn Andersson * @dst: destination address 80e88dae5dSBjorn Andersson * @reserved: reserved for future use 81e88dae5dSBjorn Andersson * @len: length of payload (in bytes) 82e88dae5dSBjorn Andersson * @flags: message flags 83e88dae5dSBjorn Andersson * @data: @len bytes of message payload data 84e88dae5dSBjorn Andersson * 85e88dae5dSBjorn Andersson * Every message sent(/received) on the rpmsg bus begins with this header. 86e88dae5dSBjorn Andersson */ 87e88dae5dSBjorn Andersson struct rpmsg_hdr { 88111d1089SGuennadi Liakhovetski __virtio32 src; 89111d1089SGuennadi Liakhovetski __virtio32 dst; 90111d1089SGuennadi Liakhovetski __virtio32 reserved; 91111d1089SGuennadi Liakhovetski __virtio16 len; 92111d1089SGuennadi Liakhovetski __virtio16 flags; 934f05fc33SGustavo A. R. Silva u8 data[]; 94e88dae5dSBjorn Andersson } __packed; 95e88dae5dSBjorn Andersson 96e88dae5dSBjorn Andersson /** 97e88dae5dSBjorn Andersson * struct rpmsg_ns_msg - dynamic name service announcement message 98e88dae5dSBjorn Andersson * @name: name of remote service that is published 99e88dae5dSBjorn Andersson * @addr: address of remote service that is published 100e88dae5dSBjorn Andersson * @flags: indicates whether service is created or destroyed 101e88dae5dSBjorn Andersson * 102e88dae5dSBjorn Andersson * This message is sent across to publish a new service, or announce 103e88dae5dSBjorn Andersson * about its removal. When we receive these messages, an appropriate 104e88dae5dSBjorn Andersson * rpmsg channel (i.e device) is created/destroyed. In turn, the ->probe() 105e88dae5dSBjorn Andersson * or ->remove() handler of the appropriate rpmsg driver will be invoked 106e88dae5dSBjorn Andersson * (if/as-soon-as one is registered). 107e88dae5dSBjorn Andersson */ 108e88dae5dSBjorn Andersson struct rpmsg_ns_msg { 109e88dae5dSBjorn Andersson char name[RPMSG_NAME_SIZE]; 110111d1089SGuennadi Liakhovetski __virtio32 addr; 111111d1089SGuennadi Liakhovetski __virtio32 flags; 112e88dae5dSBjorn Andersson } __packed; 113e88dae5dSBjorn Andersson 114e88dae5dSBjorn Andersson /** 115e88dae5dSBjorn Andersson * enum rpmsg_ns_flags - dynamic name service announcement flags 116e88dae5dSBjorn Andersson * 117e88dae5dSBjorn Andersson * @RPMSG_NS_CREATE: a new remote service was just created 118e88dae5dSBjorn Andersson * @RPMSG_NS_DESTROY: a known remote service was just destroyed 119e88dae5dSBjorn Andersson */ 120e88dae5dSBjorn Andersson enum rpmsg_ns_flags { 121e88dae5dSBjorn Andersson RPMSG_NS_CREATE = 0, 122e88dae5dSBjorn Andersson RPMSG_NS_DESTROY = 1, 123e88dae5dSBjorn Andersson }; 124e88dae5dSBjorn Andersson 1253bf950ffSBjorn Andersson /** 126*6c09ea0bSArnaud Pouliquen * struct virtio_rpmsg_channel - rpmsg channel descriptor 127*6c09ea0bSArnaud Pouliquen * @rpdev: the rpmsg channel device 128*6c09ea0bSArnaud Pouliquen * @vrp: the virtio remote processor device this channel belongs to 129*6c09ea0bSArnaud Pouliquen * 130*6c09ea0bSArnaud Pouliquen * This structure stores the channel that links the rpmsg device to the virtio 131*6c09ea0bSArnaud Pouliquen * remote processor device. 1323bf950ffSBjorn Andersson */ 1333bf950ffSBjorn Andersson struct virtio_rpmsg_channel { 1343bf950ffSBjorn Andersson struct rpmsg_device rpdev; 1353bf950ffSBjorn Andersson 1363bf950ffSBjorn Andersson struct virtproc_info *vrp; 1373bf950ffSBjorn Andersson }; 1383bf950ffSBjorn Andersson 1393bf950ffSBjorn Andersson #define to_virtio_rpmsg_channel(_rpdev) \ 1403bf950ffSBjorn Andersson container_of(_rpdev, struct virtio_rpmsg_channel, rpdev) 1413bf950ffSBjorn Andersson 142bcabbccaSOhad Ben-Cohen /* 143b1b98914SSuman Anna * We're allocating buffers of 512 bytes each for communications. The 144b1b98914SSuman Anna * number of buffers will be computed from the number of buffers supported 145b1b98914SSuman Anna * by the vring, upto a maximum of 512 buffers (256 in each direction). 146bcabbccaSOhad Ben-Cohen * 147bcabbccaSOhad Ben-Cohen * Each buffer will have 16 bytes for the msg header and 496 bytes for 148bcabbccaSOhad Ben-Cohen * the payload. 149bcabbccaSOhad Ben-Cohen * 150b1b98914SSuman Anna * This will utilize a maximum total space of 256KB for the buffers. 151bcabbccaSOhad Ben-Cohen * 152bcabbccaSOhad Ben-Cohen * We might also want to add support for user-provided buffers in time. 153bcabbccaSOhad Ben-Cohen * This will allow bigger buffer size flexibility, and can also be used 154bcabbccaSOhad Ben-Cohen * to achieve zero-copy messaging. 155bcabbccaSOhad Ben-Cohen * 156bcabbccaSOhad Ben-Cohen * Note that these numbers are purely a decision of this driver - we 157bcabbccaSOhad Ben-Cohen * can change this without changing anything in the firmware of the remote 158bcabbccaSOhad Ben-Cohen * processor. 159bcabbccaSOhad Ben-Cohen */ 160b1b98914SSuman Anna #define MAX_RPMSG_NUM_BUFS (512) 161f93848f9SLoic Pallardy #define MAX_RPMSG_BUF_SIZE (512) 162bcabbccaSOhad Ben-Cohen 163bcabbccaSOhad Ben-Cohen /* 164bcabbccaSOhad Ben-Cohen * Local addresses are dynamically allocated on-demand. 165bcabbccaSOhad Ben-Cohen * We do not dynamically assign addresses from the low 1024 range, 166bcabbccaSOhad Ben-Cohen * in order to reserve that address range for predefined services. 167bcabbccaSOhad Ben-Cohen */ 168bcabbccaSOhad Ben-Cohen #define RPMSG_RESERVED_ADDRESSES (1024) 169bcabbccaSOhad Ben-Cohen 170bcabbccaSOhad Ben-Cohen /* Address 53 is reserved for advertising remote services */ 171bcabbccaSOhad Ben-Cohen #define RPMSG_NS_ADDR (53) 172bcabbccaSOhad Ben-Cohen 1738a228ecfSBjorn Andersson static void virtio_rpmsg_destroy_ept(struct rpmsg_endpoint *ept); 1748a228ecfSBjorn Andersson static int virtio_rpmsg_send(struct rpmsg_endpoint *ept, void *data, int len); 1758a228ecfSBjorn Andersson static int virtio_rpmsg_sendto(struct rpmsg_endpoint *ept, void *data, int len, 1768a228ecfSBjorn Andersson u32 dst); 1778a228ecfSBjorn Andersson static int virtio_rpmsg_send_offchannel(struct rpmsg_endpoint *ept, u32 src, 1788a228ecfSBjorn Andersson u32 dst, void *data, int len); 1798a228ecfSBjorn Andersson static int virtio_rpmsg_trysend(struct rpmsg_endpoint *ept, void *data, int len); 1808a228ecfSBjorn Andersson static int virtio_rpmsg_trysendto(struct rpmsg_endpoint *ept, void *data, 1818a228ecfSBjorn Andersson int len, u32 dst); 1828a228ecfSBjorn Andersson static int virtio_rpmsg_trysend_offchannel(struct rpmsg_endpoint *ept, u32 src, 1838a228ecfSBjorn Andersson u32 dst, void *data, int len); 184bcabbccaSOhad Ben-Cohen 1858a228ecfSBjorn Andersson static const struct rpmsg_endpoint_ops virtio_endpoint_ops = { 1868a228ecfSBjorn Andersson .destroy_ept = virtio_rpmsg_destroy_ept, 1878a228ecfSBjorn Andersson .send = virtio_rpmsg_send, 1888a228ecfSBjorn Andersson .sendto = virtio_rpmsg_sendto, 1898a228ecfSBjorn Andersson .send_offchannel = virtio_rpmsg_send_offchannel, 1908a228ecfSBjorn Andersson .trysend = virtio_rpmsg_trysend, 1918a228ecfSBjorn Andersson .trysendto = virtio_rpmsg_trysendto, 1928a228ecfSBjorn Andersson .trysend_offchannel = virtio_rpmsg_trysend_offchannel, 1938a228ecfSBjorn Andersson }; 1948a228ecfSBjorn Andersson 1955a081caaSOhad Ben-Cohen /** 1969dd87c2aSLoic Pallardy * rpmsg_sg_init - initialize scatterlist according to cpu address location 1979dd87c2aSLoic Pallardy * @sg: scatterlist to fill 1989dd87c2aSLoic Pallardy * @cpu_addr: virtual address of the buffer 1999dd87c2aSLoic Pallardy * @len: buffer length 2009dd87c2aSLoic Pallardy * 2019dd87c2aSLoic Pallardy * An internal function filling scatterlist according to virtual address 2029dd87c2aSLoic Pallardy * location (in vmalloc or in kernel). 2039dd87c2aSLoic Pallardy */ 2049dd87c2aSLoic Pallardy static void 2059dd87c2aSLoic Pallardy rpmsg_sg_init(struct scatterlist *sg, void *cpu_addr, unsigned int len) 2069dd87c2aSLoic Pallardy { 2079dd87c2aSLoic Pallardy if (is_vmalloc_addr(cpu_addr)) { 2089dd87c2aSLoic Pallardy sg_init_table(sg, 1); 2099dd87c2aSLoic Pallardy sg_set_page(sg, vmalloc_to_page(cpu_addr), len, 2109dd87c2aSLoic Pallardy offset_in_page(cpu_addr)); 2119dd87c2aSLoic Pallardy } else { 2129dd87c2aSLoic Pallardy WARN_ON(!virt_addr_valid(cpu_addr)); 2139dd87c2aSLoic Pallardy sg_init_one(sg, cpu_addr, len); 2149dd87c2aSLoic Pallardy } 2159dd87c2aSLoic Pallardy } 2169dd87c2aSLoic Pallardy 2179dd87c2aSLoic Pallardy /** 2185a081caaSOhad Ben-Cohen * __ept_release() - deallocate an rpmsg endpoint 2195a081caaSOhad Ben-Cohen * @kref: the ept's reference count 2205a081caaSOhad Ben-Cohen * 2215a081caaSOhad Ben-Cohen * This function deallocates an ept, and is invoked when its @kref refcount 2225a081caaSOhad Ben-Cohen * drops to zero. 2235a081caaSOhad Ben-Cohen * 2245a081caaSOhad Ben-Cohen * Never invoke this function directly! 2255a081caaSOhad Ben-Cohen */ 2265a081caaSOhad Ben-Cohen static void __ept_release(struct kref *kref) 2275a081caaSOhad Ben-Cohen { 2285a081caaSOhad Ben-Cohen struct rpmsg_endpoint *ept = container_of(kref, struct rpmsg_endpoint, 2295a081caaSOhad Ben-Cohen refcount); 2305a081caaSOhad Ben-Cohen /* 2315a081caaSOhad Ben-Cohen * At this point no one holds a reference to ept anymore, 2325a081caaSOhad Ben-Cohen * so we can directly free it 2335a081caaSOhad Ben-Cohen */ 2345a081caaSOhad Ben-Cohen kfree(ept); 2355a081caaSOhad Ben-Cohen } 2365a081caaSOhad Ben-Cohen 237bcabbccaSOhad Ben-Cohen /* for more info, see below documentation of rpmsg_create_ept() */ 238bcabbccaSOhad Ben-Cohen static struct rpmsg_endpoint *__rpmsg_create_ept(struct virtproc_info *vrp, 23992e1de51SBjorn Andersson struct rpmsg_device *rpdev, 2400963679cSAnna, Suman rpmsg_rx_cb_t cb, 241bcabbccaSOhad Ben-Cohen void *priv, u32 addr) 242bcabbccaSOhad Ben-Cohen { 243d0ffce77STejun Heo int id_min, id_max, id; 244bcabbccaSOhad Ben-Cohen struct rpmsg_endpoint *ept; 245bcabbccaSOhad Ben-Cohen struct device *dev = rpdev ? &rpdev->dev : &vrp->vdev->dev; 246bcabbccaSOhad Ben-Cohen 247bcabbccaSOhad Ben-Cohen ept = kzalloc(sizeof(*ept), GFP_KERNEL); 248a8bb3fd9SAnna, Suman if (!ept) 249bcabbccaSOhad Ben-Cohen return NULL; 250bcabbccaSOhad Ben-Cohen 2515a081caaSOhad Ben-Cohen kref_init(&ept->refcount); 25215fd943aSOhad Ben-Cohen mutex_init(&ept->cb_lock); 2535a081caaSOhad Ben-Cohen 254bcabbccaSOhad Ben-Cohen ept->rpdev = rpdev; 255bcabbccaSOhad Ben-Cohen ept->cb = cb; 256bcabbccaSOhad Ben-Cohen ept->priv = priv; 2578a228ecfSBjorn Andersson ept->ops = &virtio_endpoint_ops; 258bcabbccaSOhad Ben-Cohen 259bcabbccaSOhad Ben-Cohen /* do we need to allocate a local address ? */ 260d0ffce77STejun Heo if (addr == RPMSG_ADDR_ANY) { 261d0ffce77STejun Heo id_min = RPMSG_RESERVED_ADDRESSES; 262d0ffce77STejun Heo id_max = 0; 263d0ffce77STejun Heo } else { 264d0ffce77STejun Heo id_min = addr; 265d0ffce77STejun Heo id_max = addr + 1; 266d0ffce77STejun Heo } 267bcabbccaSOhad Ben-Cohen 268bcabbccaSOhad Ben-Cohen mutex_lock(&vrp->endpoints_lock); 269bcabbccaSOhad Ben-Cohen 270bcabbccaSOhad Ben-Cohen /* bind the endpoint to an rpmsg address (and allocate one if needed) */ 271d0ffce77STejun Heo id = idr_alloc(&vrp->endpoints, ept, id_min, id_max, GFP_KERNEL); 272d0ffce77STejun Heo if (id < 0) { 273d0ffce77STejun Heo dev_err(dev, "idr_alloc failed: %d\n", id); 274bcabbccaSOhad Ben-Cohen goto free_ept; 275bcabbccaSOhad Ben-Cohen } 276d0ffce77STejun Heo ept->addr = id; 277bcabbccaSOhad Ben-Cohen 278bcabbccaSOhad Ben-Cohen mutex_unlock(&vrp->endpoints_lock); 279bcabbccaSOhad Ben-Cohen 280bcabbccaSOhad Ben-Cohen return ept; 281bcabbccaSOhad Ben-Cohen 282bcabbccaSOhad Ben-Cohen free_ept: 283bcabbccaSOhad Ben-Cohen mutex_unlock(&vrp->endpoints_lock); 2845a081caaSOhad Ben-Cohen kref_put(&ept->refcount, __ept_release); 285bcabbccaSOhad Ben-Cohen return NULL; 286bcabbccaSOhad Ben-Cohen } 287bcabbccaSOhad Ben-Cohen 28836b72c7dSBjorn Andersson static struct rpmsg_endpoint *virtio_rpmsg_create_ept(struct rpmsg_device *rpdev, 28936b72c7dSBjorn Andersson rpmsg_rx_cb_t cb, 29036b72c7dSBjorn Andersson void *priv, 29136b72c7dSBjorn Andersson struct rpmsg_channel_info chinfo) 29236b72c7dSBjorn Andersson { 2933bf950ffSBjorn Andersson struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev); 2943bf950ffSBjorn Andersson 2953bf950ffSBjorn Andersson return __rpmsg_create_ept(vch->vrp, rpdev, cb, priv, chinfo.src); 29636b72c7dSBjorn Andersson } 29736b72c7dSBjorn Andersson 298bcabbccaSOhad Ben-Cohen /** 299fa2d7795SOhad Ben-Cohen * __rpmsg_destroy_ept() - destroy an existing rpmsg endpoint 300fa2d7795SOhad Ben-Cohen * @vrp: virtproc which owns this ept 301fa2d7795SOhad Ben-Cohen * @ept: endpoing to destroy 302fa2d7795SOhad Ben-Cohen * 303fa2d7795SOhad Ben-Cohen * An internal function which destroy an ept without assuming it is 304fa2d7795SOhad Ben-Cohen * bound to an rpmsg channel. This is needed for handling the internal 305fa2d7795SOhad Ben-Cohen * name service endpoint, which isn't bound to an rpmsg channel. 306fa2d7795SOhad Ben-Cohen * See also __rpmsg_create_ept(). 307fa2d7795SOhad Ben-Cohen */ 308fa2d7795SOhad Ben-Cohen static void 309fa2d7795SOhad Ben-Cohen __rpmsg_destroy_ept(struct virtproc_info *vrp, struct rpmsg_endpoint *ept) 310fa2d7795SOhad Ben-Cohen { 31115fd943aSOhad Ben-Cohen /* make sure new inbound messages can't find this ept anymore */ 312fa2d7795SOhad Ben-Cohen mutex_lock(&vrp->endpoints_lock); 313fa2d7795SOhad Ben-Cohen idr_remove(&vrp->endpoints, ept->addr); 314fa2d7795SOhad Ben-Cohen mutex_unlock(&vrp->endpoints_lock); 315fa2d7795SOhad Ben-Cohen 31615fd943aSOhad Ben-Cohen /* make sure in-flight inbound messages won't invoke cb anymore */ 31715fd943aSOhad Ben-Cohen mutex_lock(&ept->cb_lock); 31815fd943aSOhad Ben-Cohen ept->cb = NULL; 31915fd943aSOhad Ben-Cohen mutex_unlock(&ept->cb_lock); 32015fd943aSOhad Ben-Cohen 3215a081caaSOhad Ben-Cohen kref_put(&ept->refcount, __ept_release); 322fa2d7795SOhad Ben-Cohen } 323fa2d7795SOhad Ben-Cohen 3248a228ecfSBjorn Andersson static void virtio_rpmsg_destroy_ept(struct rpmsg_endpoint *ept) 3258a228ecfSBjorn Andersson { 3263bf950ffSBjorn Andersson struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(ept->rpdev); 3273bf950ffSBjorn Andersson 3283bf950ffSBjorn Andersson __rpmsg_destroy_ept(vch->vrp, ept); 3298a228ecfSBjorn Andersson } 3308a228ecfSBjorn Andersson 33136b72c7dSBjorn Andersson static int virtio_rpmsg_announce_create(struct rpmsg_device *rpdev) 33236b72c7dSBjorn Andersson { 3333bf950ffSBjorn Andersson struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev); 3343bf950ffSBjorn Andersson struct virtproc_info *vrp = vch->vrp; 33536b72c7dSBjorn Andersson struct device *dev = &rpdev->dev; 33636b72c7dSBjorn Andersson int err = 0; 33736b72c7dSBjorn Andersson 338bcabbccaSOhad Ben-Cohen /* need to tell remote processor's name service about this channel ? */ 339b2599ebfSHenri Roosen if (rpdev->announce && rpdev->ept && 340bcabbccaSOhad Ben-Cohen virtio_has_feature(vrp->vdev, VIRTIO_RPMSG_F_NS)) { 341bcabbccaSOhad Ben-Cohen struct rpmsg_ns_msg nsm; 342bcabbccaSOhad Ben-Cohen 343bcabbccaSOhad Ben-Cohen strncpy(nsm.name, rpdev->id.name, RPMSG_NAME_SIZE); 344111d1089SGuennadi Liakhovetski nsm.addr = cpu_to_virtio32(vrp->vdev, rpdev->ept->addr); 345111d1089SGuennadi Liakhovetski nsm.flags = cpu_to_virtio32(vrp->vdev, RPMSG_NS_CREATE); 346bcabbccaSOhad Ben-Cohen 3472a48d732SBjorn Andersson err = rpmsg_sendto(rpdev->ept, &nsm, sizeof(nsm), RPMSG_NS_ADDR); 348bcabbccaSOhad Ben-Cohen if (err) 349bcabbccaSOhad Ben-Cohen dev_err(dev, "failed to announce service %d\n", err); 350bcabbccaSOhad Ben-Cohen } 351bcabbccaSOhad Ben-Cohen 352bcabbccaSOhad Ben-Cohen return err; 353bcabbccaSOhad Ben-Cohen } 354bcabbccaSOhad Ben-Cohen 35536b72c7dSBjorn Andersson static int virtio_rpmsg_announce_destroy(struct rpmsg_device *rpdev) 356bcabbccaSOhad Ben-Cohen { 3573bf950ffSBjorn Andersson struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev); 3583bf950ffSBjorn Andersson struct virtproc_info *vrp = vch->vrp; 35936b72c7dSBjorn Andersson struct device *dev = &rpdev->dev; 360bcabbccaSOhad Ben-Cohen int err = 0; 361bcabbccaSOhad Ben-Cohen 362bcabbccaSOhad Ben-Cohen /* tell remote processor's name service we're removing this channel */ 363b2599ebfSHenri Roosen if (rpdev->announce && rpdev->ept && 364bcabbccaSOhad Ben-Cohen virtio_has_feature(vrp->vdev, VIRTIO_RPMSG_F_NS)) { 365bcabbccaSOhad Ben-Cohen struct rpmsg_ns_msg nsm; 366bcabbccaSOhad Ben-Cohen 367bcabbccaSOhad Ben-Cohen strncpy(nsm.name, rpdev->id.name, RPMSG_NAME_SIZE); 368111d1089SGuennadi Liakhovetski nsm.addr = cpu_to_virtio32(vrp->vdev, rpdev->ept->addr); 369111d1089SGuennadi Liakhovetski nsm.flags = cpu_to_virtio32(vrp->vdev, RPMSG_NS_DESTROY); 370bcabbccaSOhad Ben-Cohen 3712a48d732SBjorn Andersson err = rpmsg_sendto(rpdev->ept, &nsm, sizeof(nsm), RPMSG_NS_ADDR); 372bcabbccaSOhad Ben-Cohen if (err) 373bcabbccaSOhad Ben-Cohen dev_err(dev, "failed to announce service %d\n", err); 374bcabbccaSOhad Ben-Cohen } 375bcabbccaSOhad Ben-Cohen 37636b72c7dSBjorn Andersson return err; 37736b72c7dSBjorn Andersson } 37836b72c7dSBjorn Andersson 37936b72c7dSBjorn Andersson static const struct rpmsg_device_ops virtio_rpmsg_ops = { 38036b72c7dSBjorn Andersson .create_ept = virtio_rpmsg_create_ept, 38136b72c7dSBjorn Andersson .announce_create = virtio_rpmsg_announce_create, 38236b72c7dSBjorn Andersson .announce_destroy = virtio_rpmsg_announce_destroy, 38336b72c7dSBjorn Andersson }; 38436b72c7dSBjorn Andersson 385b0b03b81SBjorn Andersson static void virtio_rpmsg_release_device(struct device *dev) 386b0b03b81SBjorn Andersson { 387b0b03b81SBjorn Andersson struct rpmsg_device *rpdev = to_rpmsg_device(dev); 388b0b03b81SBjorn Andersson struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev); 389b0b03b81SBjorn Andersson 390b0b03b81SBjorn Andersson kfree(vch); 391b0b03b81SBjorn Andersson } 392b0b03b81SBjorn Andersson 393bcabbccaSOhad Ben-Cohen /* 394bcabbccaSOhad Ben-Cohen * create an rpmsg channel using its name and address info. 395bcabbccaSOhad Ben-Cohen * this function will be used to create both static and dynamic 396bcabbccaSOhad Ben-Cohen * channels. 397bcabbccaSOhad Ben-Cohen */ 39892e1de51SBjorn Andersson static struct rpmsg_device *rpmsg_create_channel(struct virtproc_info *vrp, 399bcabbccaSOhad Ben-Cohen struct rpmsg_channel_info *chinfo) 400bcabbccaSOhad Ben-Cohen { 4013bf950ffSBjorn Andersson struct virtio_rpmsg_channel *vch; 40292e1de51SBjorn Andersson struct rpmsg_device *rpdev; 403bcabbccaSOhad Ben-Cohen struct device *tmp, *dev = &vrp->vdev->dev; 404bcabbccaSOhad Ben-Cohen int ret; 405bcabbccaSOhad Ben-Cohen 406bcabbccaSOhad Ben-Cohen /* make sure a similar channel doesn't already exist */ 4078b881c07SBjorn Andersson tmp = rpmsg_find_device(dev, chinfo); 408bcabbccaSOhad Ben-Cohen if (tmp) { 409bcabbccaSOhad Ben-Cohen /* decrement the matched device's refcount back */ 410bcabbccaSOhad Ben-Cohen put_device(tmp); 411bcabbccaSOhad Ben-Cohen dev_err(dev, "channel %s:%x:%x already exist\n", 412bcabbccaSOhad Ben-Cohen chinfo->name, chinfo->src, chinfo->dst); 413bcabbccaSOhad Ben-Cohen return NULL; 414bcabbccaSOhad Ben-Cohen } 415bcabbccaSOhad Ben-Cohen 4163bf950ffSBjorn Andersson vch = kzalloc(sizeof(*vch), GFP_KERNEL); 4173bf950ffSBjorn Andersson if (!vch) 418bcabbccaSOhad Ben-Cohen return NULL; 419bcabbccaSOhad Ben-Cohen 4203bf950ffSBjorn Andersson /* Link the channel to our vrp */ 4213bf950ffSBjorn Andersson vch->vrp = vrp; 4223bf950ffSBjorn Andersson 4233bf950ffSBjorn Andersson /* Assign public information to the rpmsg_device */ 4243bf950ffSBjorn Andersson rpdev = &vch->rpdev; 425bcabbccaSOhad Ben-Cohen rpdev->src = chinfo->src; 426bcabbccaSOhad Ben-Cohen rpdev->dst = chinfo->dst; 42736b72c7dSBjorn Andersson rpdev->ops = &virtio_rpmsg_ops; 428bcabbccaSOhad Ben-Cohen 429bcabbccaSOhad Ben-Cohen /* 430bcabbccaSOhad Ben-Cohen * rpmsg server channels has predefined local address (for now), 431bcabbccaSOhad Ben-Cohen * and their existence needs to be announced remotely 432bcabbccaSOhad Ben-Cohen */ 433c8ced113SAndrew F. Davis rpdev->announce = rpdev->src != RPMSG_ADDR_ANY; 434bcabbccaSOhad Ben-Cohen 435bcabbccaSOhad Ben-Cohen strncpy(rpdev->id.name, chinfo->name, RPMSG_NAME_SIZE); 436bcabbccaSOhad Ben-Cohen 4376eed598aSBjorn Andersson rpdev->dev.parent = &vrp->vdev->dev; 438b0b03b81SBjorn Andersson rpdev->dev.release = virtio_rpmsg_release_device; 4396eed598aSBjorn Andersson ret = rpmsg_register_device(rpdev); 4406eed598aSBjorn Andersson if (ret) 4416eed598aSBjorn Andersson return NULL; 4426eed598aSBjorn Andersson 4436eed598aSBjorn Andersson return rpdev; 4446eed598aSBjorn Andersson } 4456eed598aSBjorn Andersson 446bcabbccaSOhad Ben-Cohen /* super simple buffer "allocator" that is just enough for now */ 447bcabbccaSOhad Ben-Cohen static void *get_a_tx_buf(struct virtproc_info *vrp) 448bcabbccaSOhad Ben-Cohen { 449bcabbccaSOhad Ben-Cohen unsigned int len; 450bcabbccaSOhad Ben-Cohen void *ret; 451bcabbccaSOhad Ben-Cohen 452bcabbccaSOhad Ben-Cohen /* support multiple concurrent senders */ 453bcabbccaSOhad Ben-Cohen mutex_lock(&vrp->tx_lock); 454bcabbccaSOhad Ben-Cohen 455bcabbccaSOhad Ben-Cohen /* 456bcabbccaSOhad Ben-Cohen * either pick the next unused tx buffer 457bcabbccaSOhad Ben-Cohen * (half of our buffers are used for sending messages) 458bcabbccaSOhad Ben-Cohen */ 459b1b98914SSuman Anna if (vrp->last_sbuf < vrp->num_bufs / 2) 460f93848f9SLoic Pallardy ret = vrp->sbufs + vrp->buf_size * vrp->last_sbuf++; 461bcabbccaSOhad Ben-Cohen /* or recycle a used one */ 462bcabbccaSOhad Ben-Cohen else 463bcabbccaSOhad Ben-Cohen ret = virtqueue_get_buf(vrp->svq, &len); 464bcabbccaSOhad Ben-Cohen 465bcabbccaSOhad Ben-Cohen mutex_unlock(&vrp->tx_lock); 466bcabbccaSOhad Ben-Cohen 467bcabbccaSOhad Ben-Cohen return ret; 468bcabbccaSOhad Ben-Cohen } 469bcabbccaSOhad Ben-Cohen 470bcabbccaSOhad Ben-Cohen /** 471bcabbccaSOhad Ben-Cohen * rpmsg_upref_sleepers() - enable "tx-complete" interrupts, if needed 472bcabbccaSOhad Ben-Cohen * @vrp: virtual remote processor state 473bcabbccaSOhad Ben-Cohen * 474bcabbccaSOhad Ben-Cohen * This function is called before a sender is blocked, waiting for 475bcabbccaSOhad Ben-Cohen * a tx buffer to become available. 476bcabbccaSOhad Ben-Cohen * 477bcabbccaSOhad Ben-Cohen * If we already have blocking senders, this function merely increases 478bcabbccaSOhad Ben-Cohen * the "sleepers" reference count, and exits. 479bcabbccaSOhad Ben-Cohen * 480bcabbccaSOhad Ben-Cohen * Otherwise, if this is the first sender to block, we also enable 481bcabbccaSOhad Ben-Cohen * virtio's tx callbacks, so we'd be immediately notified when a tx 482bcabbccaSOhad Ben-Cohen * buffer is consumed (we rely on virtio's tx callback in order 483bcabbccaSOhad Ben-Cohen * to wake up sleeping senders as soon as a tx buffer is used by the 484bcabbccaSOhad Ben-Cohen * remote processor). 485bcabbccaSOhad Ben-Cohen */ 486bcabbccaSOhad Ben-Cohen static void rpmsg_upref_sleepers(struct virtproc_info *vrp) 487bcabbccaSOhad Ben-Cohen { 488bcabbccaSOhad Ben-Cohen /* support multiple concurrent senders */ 489bcabbccaSOhad Ben-Cohen mutex_lock(&vrp->tx_lock); 490bcabbccaSOhad Ben-Cohen 491bcabbccaSOhad Ben-Cohen /* are we the first sleeping context waiting for tx buffers ? */ 492bcabbccaSOhad Ben-Cohen if (atomic_inc_return(&vrp->sleepers) == 1) 493bcabbccaSOhad Ben-Cohen /* enable "tx-complete" interrupts before dozing off */ 494bcabbccaSOhad Ben-Cohen virtqueue_enable_cb(vrp->svq); 495bcabbccaSOhad Ben-Cohen 496bcabbccaSOhad Ben-Cohen mutex_unlock(&vrp->tx_lock); 497bcabbccaSOhad Ben-Cohen } 498bcabbccaSOhad Ben-Cohen 499bcabbccaSOhad Ben-Cohen /** 500bcabbccaSOhad Ben-Cohen * rpmsg_downref_sleepers() - disable "tx-complete" interrupts, if needed 501bcabbccaSOhad Ben-Cohen * @vrp: virtual remote processor state 502bcabbccaSOhad Ben-Cohen * 503bcabbccaSOhad Ben-Cohen * This function is called after a sender, that waited for a tx buffer 504bcabbccaSOhad Ben-Cohen * to become available, is unblocked. 505bcabbccaSOhad Ben-Cohen * 506bcabbccaSOhad Ben-Cohen * If we still have blocking senders, this function merely decreases 507bcabbccaSOhad Ben-Cohen * the "sleepers" reference count, and exits. 508bcabbccaSOhad Ben-Cohen * 509bcabbccaSOhad Ben-Cohen * Otherwise, if there are no more blocking senders, we also disable 510bcabbccaSOhad Ben-Cohen * virtio's tx callbacks, to avoid the overhead incurred with handling 511bcabbccaSOhad Ben-Cohen * those (now redundant) interrupts. 512bcabbccaSOhad Ben-Cohen */ 513bcabbccaSOhad Ben-Cohen static void rpmsg_downref_sleepers(struct virtproc_info *vrp) 514bcabbccaSOhad Ben-Cohen { 515bcabbccaSOhad Ben-Cohen /* support multiple concurrent senders */ 516bcabbccaSOhad Ben-Cohen mutex_lock(&vrp->tx_lock); 517bcabbccaSOhad Ben-Cohen 518bcabbccaSOhad Ben-Cohen /* are we the last sleeping context waiting for tx buffers ? */ 519bcabbccaSOhad Ben-Cohen if (atomic_dec_and_test(&vrp->sleepers)) 520bcabbccaSOhad Ben-Cohen /* disable "tx-complete" interrupts */ 521bcabbccaSOhad Ben-Cohen virtqueue_disable_cb(vrp->svq); 522bcabbccaSOhad Ben-Cohen 523bcabbccaSOhad Ben-Cohen mutex_unlock(&vrp->tx_lock); 524bcabbccaSOhad Ben-Cohen } 525bcabbccaSOhad Ben-Cohen 526bcabbccaSOhad Ben-Cohen /** 527bcabbccaSOhad Ben-Cohen * rpmsg_send_offchannel_raw() - send a message across to the remote processor 528bcabbccaSOhad Ben-Cohen * @rpdev: the rpmsg channel 529bcabbccaSOhad Ben-Cohen * @src: source address 530bcabbccaSOhad Ben-Cohen * @dst: destination address 531bcabbccaSOhad Ben-Cohen * @data: payload of message 532bcabbccaSOhad Ben-Cohen * @len: length of payload 533bcabbccaSOhad Ben-Cohen * @wait: indicates whether caller should block in case no TX buffers available 534bcabbccaSOhad Ben-Cohen * 535bcabbccaSOhad Ben-Cohen * This function is the base implementation for all of the rpmsg sending API. 536bcabbccaSOhad Ben-Cohen * 537bcabbccaSOhad Ben-Cohen * It will send @data of length @len to @dst, and say it's from @src. The 538bcabbccaSOhad Ben-Cohen * message will be sent to the remote processor which the @rpdev channel 539bcabbccaSOhad Ben-Cohen * belongs to. 540bcabbccaSOhad Ben-Cohen * 541bcabbccaSOhad Ben-Cohen * The message is sent using one of the TX buffers that are available for 542bcabbccaSOhad Ben-Cohen * communication with this remote processor. 543bcabbccaSOhad Ben-Cohen * 544bcabbccaSOhad Ben-Cohen * If @wait is true, the caller will be blocked until either a TX buffer is 545bcabbccaSOhad Ben-Cohen * available, or 15 seconds elapses (we don't want callers to 546bcabbccaSOhad Ben-Cohen * sleep indefinitely due to misbehaving remote processors), and in that 547bcabbccaSOhad Ben-Cohen * case -ERESTARTSYS is returned. The number '15' itself was picked 548bcabbccaSOhad Ben-Cohen * arbitrarily; there's little point in asking drivers to provide a timeout 549bcabbccaSOhad Ben-Cohen * value themselves. 550bcabbccaSOhad Ben-Cohen * 551bcabbccaSOhad Ben-Cohen * Otherwise, if @wait is false, and there are no TX buffers available, 552bcabbccaSOhad Ben-Cohen * the function will immediately fail, and -ENOMEM will be returned. 553bcabbccaSOhad Ben-Cohen * 554bcabbccaSOhad Ben-Cohen * Normally drivers shouldn't use this function directly; instead, drivers 555bcabbccaSOhad Ben-Cohen * should use the appropriate rpmsg_{try}send{to, _offchannel} API 556bcabbccaSOhad Ben-Cohen * (see include/linux/rpmsg.h). 557bcabbccaSOhad Ben-Cohen * 558bcabbccaSOhad Ben-Cohen * Returns 0 on success and an appropriate error value on failure. 559bcabbccaSOhad Ben-Cohen */ 5608a228ecfSBjorn Andersson static int rpmsg_send_offchannel_raw(struct rpmsg_device *rpdev, 5618a228ecfSBjorn Andersson u32 src, u32 dst, 562bcabbccaSOhad Ben-Cohen void *data, int len, bool wait) 563bcabbccaSOhad Ben-Cohen { 5643bf950ffSBjorn Andersson struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev); 5653bf950ffSBjorn Andersson struct virtproc_info *vrp = vch->vrp; 566bcabbccaSOhad Ben-Cohen struct device *dev = &rpdev->dev; 567bcabbccaSOhad Ben-Cohen struct scatterlist sg; 568bcabbccaSOhad Ben-Cohen struct rpmsg_hdr *msg; 569bcabbccaSOhad Ben-Cohen int err; 570bcabbccaSOhad Ben-Cohen 571bcabbccaSOhad Ben-Cohen /* bcasting isn't allowed */ 572bcabbccaSOhad Ben-Cohen if (src == RPMSG_ADDR_ANY || dst == RPMSG_ADDR_ANY) { 573bcabbccaSOhad Ben-Cohen dev_err(dev, "invalid addr (src 0x%x, dst 0x%x)\n", src, dst); 574bcabbccaSOhad Ben-Cohen return -EINVAL; 575bcabbccaSOhad Ben-Cohen } 576bcabbccaSOhad Ben-Cohen 577bcabbccaSOhad Ben-Cohen /* 578bcabbccaSOhad Ben-Cohen * We currently use fixed-sized buffers, and therefore the payload 579bcabbccaSOhad Ben-Cohen * length is limited. 580bcabbccaSOhad Ben-Cohen * 581bcabbccaSOhad Ben-Cohen * One of the possible improvements here is either to support 582bcabbccaSOhad Ben-Cohen * user-provided buffers (and then we can also support zero-copy 583bcabbccaSOhad Ben-Cohen * messaging), or to improve the buffer allocator, to support 584bcabbccaSOhad Ben-Cohen * variable-length buffer sizes. 585bcabbccaSOhad Ben-Cohen */ 586f93848f9SLoic Pallardy if (len > vrp->buf_size - sizeof(struct rpmsg_hdr)) { 587bcabbccaSOhad Ben-Cohen dev_err(dev, "message is too big (%d)\n", len); 588bcabbccaSOhad Ben-Cohen return -EMSGSIZE; 589bcabbccaSOhad Ben-Cohen } 590bcabbccaSOhad Ben-Cohen 591bcabbccaSOhad Ben-Cohen /* grab a buffer */ 592bcabbccaSOhad Ben-Cohen msg = get_a_tx_buf(vrp); 593bcabbccaSOhad Ben-Cohen if (!msg && !wait) 594bcabbccaSOhad Ben-Cohen return -ENOMEM; 595bcabbccaSOhad Ben-Cohen 596bcabbccaSOhad Ben-Cohen /* no free buffer ? wait for one (but bail after 15 seconds) */ 597bcabbccaSOhad Ben-Cohen while (!msg) { 598bcabbccaSOhad Ben-Cohen /* enable "tx-complete" interrupts, if not already enabled */ 599bcabbccaSOhad Ben-Cohen rpmsg_upref_sleepers(vrp); 600bcabbccaSOhad Ben-Cohen 601bcabbccaSOhad Ben-Cohen /* 602bcabbccaSOhad Ben-Cohen * sleep until a free buffer is available or 15 secs elapse. 603bcabbccaSOhad Ben-Cohen * the timeout period is not configurable because there's 604bcabbccaSOhad Ben-Cohen * little point in asking drivers to specify that. 605bcabbccaSOhad Ben-Cohen * if later this happens to be required, it'd be easy to add. 606bcabbccaSOhad Ben-Cohen */ 607bcabbccaSOhad Ben-Cohen err = wait_event_interruptible_timeout(vrp->sendq, 608bcabbccaSOhad Ben-Cohen (msg = get_a_tx_buf(vrp)), 609bcabbccaSOhad Ben-Cohen msecs_to_jiffies(15000)); 610bcabbccaSOhad Ben-Cohen 611bcabbccaSOhad Ben-Cohen /* disable "tx-complete" interrupts if we're the last sleeper */ 612bcabbccaSOhad Ben-Cohen rpmsg_downref_sleepers(vrp); 613bcabbccaSOhad Ben-Cohen 614bcabbccaSOhad Ben-Cohen /* timeout ? */ 615bcabbccaSOhad Ben-Cohen if (!err) { 616bcabbccaSOhad Ben-Cohen dev_err(dev, "timeout waiting for a tx buffer\n"); 617bcabbccaSOhad Ben-Cohen return -ERESTARTSYS; 618bcabbccaSOhad Ben-Cohen } 619bcabbccaSOhad Ben-Cohen } 620bcabbccaSOhad Ben-Cohen 621111d1089SGuennadi Liakhovetski msg->len = cpu_to_virtio16(vrp->vdev, len); 622bcabbccaSOhad Ben-Cohen msg->flags = 0; 623111d1089SGuennadi Liakhovetski msg->src = cpu_to_virtio32(vrp->vdev, src); 624111d1089SGuennadi Liakhovetski msg->dst = cpu_to_virtio32(vrp->vdev, dst); 625bcabbccaSOhad Ben-Cohen msg->reserved = 0; 626bcabbccaSOhad Ben-Cohen memcpy(msg->data, data, len); 627bcabbccaSOhad Ben-Cohen 628bcabbccaSOhad Ben-Cohen dev_dbg(dev, "TX From 0x%x, To 0x%x, Len %d, Flags %d, Reserved %d\n", 629111d1089SGuennadi Liakhovetski src, dst, len, msg->flags, msg->reserved); 630211e3a93SAnna, Suman #if defined(CONFIG_DYNAMIC_DEBUG) 631211e3a93SAnna, Suman dynamic_hex_dump("rpmsg_virtio TX: ", DUMP_PREFIX_NONE, 16, 1, 632111d1089SGuennadi Liakhovetski msg, sizeof(*msg) + len, true); 633211e3a93SAnna, Suman #endif 634bcabbccaSOhad Ben-Cohen 6359dd87c2aSLoic Pallardy rpmsg_sg_init(&sg, msg, sizeof(*msg) + len); 636bcabbccaSOhad Ben-Cohen 637bcabbccaSOhad Ben-Cohen mutex_lock(&vrp->tx_lock); 638bcabbccaSOhad Ben-Cohen 639bcabbccaSOhad Ben-Cohen /* add message to the remote processor's virtqueue */ 640cee51d69SRusty Russell err = virtqueue_add_outbuf(vrp->svq, &sg, 1, msg, GFP_KERNEL); 64157e1a373SRusty Russell if (err) { 642bcabbccaSOhad Ben-Cohen /* 643bcabbccaSOhad Ben-Cohen * need to reclaim the buffer here, otherwise it's lost 644bcabbccaSOhad Ben-Cohen * (memory won't leak, but rpmsg won't use it again for TX). 645bcabbccaSOhad Ben-Cohen * this will wait for a buffer management overhaul. 646bcabbccaSOhad Ben-Cohen */ 647cee51d69SRusty Russell dev_err(dev, "virtqueue_add_outbuf failed: %d\n", err); 648bcabbccaSOhad Ben-Cohen goto out; 649bcabbccaSOhad Ben-Cohen } 650bcabbccaSOhad Ben-Cohen 651bcabbccaSOhad Ben-Cohen /* tell the remote processor it has a pending message to read */ 652bcabbccaSOhad Ben-Cohen virtqueue_kick(vrp->svq); 653bcabbccaSOhad Ben-Cohen out: 654bcabbccaSOhad Ben-Cohen mutex_unlock(&vrp->tx_lock); 655bcabbccaSOhad Ben-Cohen return err; 656bcabbccaSOhad Ben-Cohen } 657bcabbccaSOhad Ben-Cohen 6588a228ecfSBjorn Andersson static int virtio_rpmsg_send(struct rpmsg_endpoint *ept, void *data, int len) 6598a228ecfSBjorn Andersson { 6608a228ecfSBjorn Andersson struct rpmsg_device *rpdev = ept->rpdev; 6618a228ecfSBjorn Andersson u32 src = ept->addr, dst = rpdev->dst; 6628a228ecfSBjorn Andersson 6638a228ecfSBjorn Andersson return rpmsg_send_offchannel_raw(rpdev, src, dst, data, len, true); 6648a228ecfSBjorn Andersson } 6658a228ecfSBjorn Andersson 6668a228ecfSBjorn Andersson static int virtio_rpmsg_sendto(struct rpmsg_endpoint *ept, void *data, int len, 6678a228ecfSBjorn Andersson u32 dst) 6688a228ecfSBjorn Andersson { 6698a228ecfSBjorn Andersson struct rpmsg_device *rpdev = ept->rpdev; 6708a228ecfSBjorn Andersson u32 src = ept->addr; 6718a228ecfSBjorn Andersson 6728a228ecfSBjorn Andersson return rpmsg_send_offchannel_raw(rpdev, src, dst, data, len, true); 6738a228ecfSBjorn Andersson } 6748a228ecfSBjorn Andersson 6758a228ecfSBjorn Andersson static int virtio_rpmsg_send_offchannel(struct rpmsg_endpoint *ept, u32 src, 6768a228ecfSBjorn Andersson u32 dst, void *data, int len) 6778a228ecfSBjorn Andersson { 6788a228ecfSBjorn Andersson struct rpmsg_device *rpdev = ept->rpdev; 6798a228ecfSBjorn Andersson 6808a228ecfSBjorn Andersson return rpmsg_send_offchannel_raw(rpdev, src, dst, data, len, true); 6818a228ecfSBjorn Andersson } 6828a228ecfSBjorn Andersson 6838a228ecfSBjorn Andersson static int virtio_rpmsg_trysend(struct rpmsg_endpoint *ept, void *data, int len) 6848a228ecfSBjorn Andersson { 6858a228ecfSBjorn Andersson struct rpmsg_device *rpdev = ept->rpdev; 6868a228ecfSBjorn Andersson u32 src = ept->addr, dst = rpdev->dst; 6878a228ecfSBjorn Andersson 6888a228ecfSBjorn Andersson return rpmsg_send_offchannel_raw(rpdev, src, dst, data, len, false); 6898a228ecfSBjorn Andersson } 6908a228ecfSBjorn Andersson 6918a228ecfSBjorn Andersson static int virtio_rpmsg_trysendto(struct rpmsg_endpoint *ept, void *data, 6928a228ecfSBjorn Andersson int len, u32 dst) 6938a228ecfSBjorn Andersson { 6948a228ecfSBjorn Andersson struct rpmsg_device *rpdev = ept->rpdev; 6958a228ecfSBjorn Andersson u32 src = ept->addr; 6968a228ecfSBjorn Andersson 6978a228ecfSBjorn Andersson return rpmsg_send_offchannel_raw(rpdev, src, dst, data, len, false); 6988a228ecfSBjorn Andersson } 6998a228ecfSBjorn Andersson 7008a228ecfSBjorn Andersson static int virtio_rpmsg_trysend_offchannel(struct rpmsg_endpoint *ept, u32 src, 7018a228ecfSBjorn Andersson u32 dst, void *data, int len) 7028a228ecfSBjorn Andersson { 7038a228ecfSBjorn Andersson struct rpmsg_device *rpdev = ept->rpdev; 7048a228ecfSBjorn Andersson 7058a228ecfSBjorn Andersson return rpmsg_send_offchannel_raw(rpdev, src, dst, data, len, false); 7068a228ecfSBjorn Andersson } 7078a228ecfSBjorn Andersson 7081aa7d6a5SRobert Tivy static int rpmsg_recv_single(struct virtproc_info *vrp, struct device *dev, 7091aa7d6a5SRobert Tivy struct rpmsg_hdr *msg, unsigned int len) 710bcabbccaSOhad Ben-Cohen { 711bcabbccaSOhad Ben-Cohen struct rpmsg_endpoint *ept; 712bcabbccaSOhad Ben-Cohen struct scatterlist sg; 713111d1089SGuennadi Liakhovetski unsigned int msg_len = virtio16_to_cpu(vrp->vdev, msg->len); 714bcabbccaSOhad Ben-Cohen int err; 715bcabbccaSOhad Ben-Cohen 716bcabbccaSOhad Ben-Cohen dev_dbg(dev, "From: 0x%x, To: 0x%x, Len: %d, Flags: %d, Reserved: %d\n", 717111d1089SGuennadi Liakhovetski virtio32_to_cpu(vrp->vdev, msg->src), 718111d1089SGuennadi Liakhovetski virtio32_to_cpu(vrp->vdev, msg->dst), msg_len, 719111d1089SGuennadi Liakhovetski virtio16_to_cpu(vrp->vdev, msg->flags), 720111d1089SGuennadi Liakhovetski virtio32_to_cpu(vrp->vdev, msg->reserved)); 721211e3a93SAnna, Suman #if defined(CONFIG_DYNAMIC_DEBUG) 722211e3a93SAnna, Suman dynamic_hex_dump("rpmsg_virtio RX: ", DUMP_PREFIX_NONE, 16, 1, 723111d1089SGuennadi Liakhovetski msg, sizeof(*msg) + msg_len, true); 724211e3a93SAnna, Suman #endif 725bcabbccaSOhad Ben-Cohen 7269648224eSOhad Ben-Cohen /* 7279648224eSOhad Ben-Cohen * We currently use fixed-sized buffers, so trivially sanitize 7289648224eSOhad Ben-Cohen * the reported payload length. 7299648224eSOhad Ben-Cohen */ 730f93848f9SLoic Pallardy if (len > vrp->buf_size || 731111d1089SGuennadi Liakhovetski msg_len > (len - sizeof(struct rpmsg_hdr))) { 732111d1089SGuennadi Liakhovetski dev_warn(dev, "inbound msg too big: (%d, %d)\n", len, msg_len); 7331aa7d6a5SRobert Tivy return -EINVAL; 7349648224eSOhad Ben-Cohen } 7359648224eSOhad Ben-Cohen 736bcabbccaSOhad Ben-Cohen /* use the dst addr to fetch the callback of the appropriate user */ 737bcabbccaSOhad Ben-Cohen mutex_lock(&vrp->endpoints_lock); 7385a081caaSOhad Ben-Cohen 739111d1089SGuennadi Liakhovetski ept = idr_find(&vrp->endpoints, virtio32_to_cpu(vrp->vdev, msg->dst)); 7405a081caaSOhad Ben-Cohen 7415a081caaSOhad Ben-Cohen /* let's make sure no one deallocates ept while we use it */ 7425a081caaSOhad Ben-Cohen if (ept) 7435a081caaSOhad Ben-Cohen kref_get(&ept->refcount); 7445a081caaSOhad Ben-Cohen 745bcabbccaSOhad Ben-Cohen mutex_unlock(&vrp->endpoints_lock); 746bcabbccaSOhad Ben-Cohen 74715fd943aSOhad Ben-Cohen if (ept) { 74815fd943aSOhad Ben-Cohen /* make sure ept->cb doesn't go away while we use it */ 74915fd943aSOhad Ben-Cohen mutex_lock(&ept->cb_lock); 75015fd943aSOhad Ben-Cohen 75115fd943aSOhad Ben-Cohen if (ept->cb) 752111d1089SGuennadi Liakhovetski ept->cb(ept->rpdev, msg->data, msg_len, ept->priv, 753111d1089SGuennadi Liakhovetski virtio32_to_cpu(vrp->vdev, msg->src)); 75415fd943aSOhad Ben-Cohen 75515fd943aSOhad Ben-Cohen mutex_unlock(&ept->cb_lock); 756bcabbccaSOhad Ben-Cohen 7575a081caaSOhad Ben-Cohen /* farewell, ept, we don't need you anymore */ 7585a081caaSOhad Ben-Cohen kref_put(&ept->refcount, __ept_release); 75915fd943aSOhad Ben-Cohen } else 7608a168ca7SMasanari Iida dev_warn(dev, "msg received with no recipient\n"); 7615a081caaSOhad Ben-Cohen 762f1d9e9c7SOhad Ben-Cohen /* publish the real size of the buffer */ 7639dd87c2aSLoic Pallardy rpmsg_sg_init(&sg, msg, vrp->buf_size); 764bcabbccaSOhad Ben-Cohen 765bcabbccaSOhad Ben-Cohen /* add the buffer back to the remote processor's virtqueue */ 766cee51d69SRusty Russell err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, msg, GFP_KERNEL); 767bcabbccaSOhad Ben-Cohen if (err < 0) { 768bcabbccaSOhad Ben-Cohen dev_err(dev, "failed to add a virtqueue buffer: %d\n", err); 7691aa7d6a5SRobert Tivy return err; 7701aa7d6a5SRobert Tivy } 7711aa7d6a5SRobert Tivy 7721aa7d6a5SRobert Tivy return 0; 7731aa7d6a5SRobert Tivy } 7741aa7d6a5SRobert Tivy 7751aa7d6a5SRobert Tivy /* called when an rx buffer is used, and it's time to digest a message */ 7761aa7d6a5SRobert Tivy static void rpmsg_recv_done(struct virtqueue *rvq) 7771aa7d6a5SRobert Tivy { 7781aa7d6a5SRobert Tivy struct virtproc_info *vrp = rvq->vdev->priv; 7791aa7d6a5SRobert Tivy struct device *dev = &rvq->vdev->dev; 7801aa7d6a5SRobert Tivy struct rpmsg_hdr *msg; 7811aa7d6a5SRobert Tivy unsigned int len, msgs_received = 0; 7821aa7d6a5SRobert Tivy int err; 7831aa7d6a5SRobert Tivy 7841aa7d6a5SRobert Tivy msg = virtqueue_get_buf(rvq, &len); 7851aa7d6a5SRobert Tivy if (!msg) { 7861aa7d6a5SRobert Tivy dev_err(dev, "uhm, incoming signal, but no used buffer ?\n"); 787bcabbccaSOhad Ben-Cohen return; 788bcabbccaSOhad Ben-Cohen } 789bcabbccaSOhad Ben-Cohen 7901aa7d6a5SRobert Tivy while (msg) { 7911aa7d6a5SRobert Tivy err = rpmsg_recv_single(vrp, dev, msg, len); 7921aa7d6a5SRobert Tivy if (err) 7931aa7d6a5SRobert Tivy break; 7941aa7d6a5SRobert Tivy 7951aa7d6a5SRobert Tivy msgs_received++; 7961aa7d6a5SRobert Tivy 7971aa7d6a5SRobert Tivy msg = virtqueue_get_buf(rvq, &len); 7986c49fbe3SLee Jones } 7991aa7d6a5SRobert Tivy 8001aa7d6a5SRobert Tivy dev_dbg(dev, "Received %u messages\n", msgs_received); 8011aa7d6a5SRobert Tivy 802bcabbccaSOhad Ben-Cohen /* tell the remote processor we added another available rx buffer */ 8031aa7d6a5SRobert Tivy if (msgs_received) 804bcabbccaSOhad Ben-Cohen virtqueue_kick(vrp->rvq); 805bcabbccaSOhad Ben-Cohen } 806bcabbccaSOhad Ben-Cohen 807bcabbccaSOhad Ben-Cohen /* 808bcabbccaSOhad Ben-Cohen * This is invoked whenever the remote processor completed processing 809bcabbccaSOhad Ben-Cohen * a TX msg we just sent it, and the buffer is put back to the used ring. 810bcabbccaSOhad Ben-Cohen * 811bcabbccaSOhad Ben-Cohen * Normally, though, we suppress this "tx complete" interrupt in order to 812bcabbccaSOhad Ben-Cohen * avoid the incurred overhead. 813bcabbccaSOhad Ben-Cohen */ 814bcabbccaSOhad Ben-Cohen static void rpmsg_xmit_done(struct virtqueue *svq) 815bcabbccaSOhad Ben-Cohen { 816bcabbccaSOhad Ben-Cohen struct virtproc_info *vrp = svq->vdev->priv; 817bcabbccaSOhad Ben-Cohen 818bcabbccaSOhad Ben-Cohen dev_dbg(&svq->vdev->dev, "%s\n", __func__); 819bcabbccaSOhad Ben-Cohen 820bcabbccaSOhad Ben-Cohen /* wake up potential senders that are waiting for a tx buffer */ 821bcabbccaSOhad Ben-Cohen wake_up_interruptible(&vrp->sendq); 822bcabbccaSOhad Ben-Cohen } 823bcabbccaSOhad Ben-Cohen 824bcabbccaSOhad Ben-Cohen /* invoked when a name service announcement arrives */ 8254b83c52aSBjorn Andersson static int rpmsg_ns_cb(struct rpmsg_device *rpdev, void *data, int len, 826bcabbccaSOhad Ben-Cohen void *priv, u32 src) 827bcabbccaSOhad Ben-Cohen { 828bcabbccaSOhad Ben-Cohen struct rpmsg_ns_msg *msg = data; 82992e1de51SBjorn Andersson struct rpmsg_device *newch; 830bcabbccaSOhad Ben-Cohen struct rpmsg_channel_info chinfo; 831bcabbccaSOhad Ben-Cohen struct virtproc_info *vrp = priv; 832bcabbccaSOhad Ben-Cohen struct device *dev = &vrp->vdev->dev; 833bcabbccaSOhad Ben-Cohen int ret; 834bcabbccaSOhad Ben-Cohen 835211e3a93SAnna, Suman #if defined(CONFIG_DYNAMIC_DEBUG) 836211e3a93SAnna, Suman dynamic_hex_dump("NS announcement: ", DUMP_PREFIX_NONE, 16, 1, 837211e3a93SAnna, Suman data, len, true); 838211e3a93SAnna, Suman #endif 839bcabbccaSOhad Ben-Cohen 840bcabbccaSOhad Ben-Cohen if (len != sizeof(*msg)) { 841bcabbccaSOhad Ben-Cohen dev_err(dev, "malformed ns msg (%d)\n", len); 8424b83c52aSBjorn Andersson return -EINVAL; 843bcabbccaSOhad Ben-Cohen } 844bcabbccaSOhad Ben-Cohen 845bcabbccaSOhad Ben-Cohen /* 846bcabbccaSOhad Ben-Cohen * the name service ept does _not_ belong to a real rpmsg channel, 847bcabbccaSOhad Ben-Cohen * and is handled by the rpmsg bus itself. 848bcabbccaSOhad Ben-Cohen * for sanity reasons, make sure a valid rpdev has _not_ sneaked 849bcabbccaSOhad Ben-Cohen * in somehow. 850bcabbccaSOhad Ben-Cohen */ 851bcabbccaSOhad Ben-Cohen if (rpdev) { 852bcabbccaSOhad Ben-Cohen dev_err(dev, "anomaly: ns ept has an rpdev handle\n"); 8534b83c52aSBjorn Andersson return -EINVAL; 854bcabbccaSOhad Ben-Cohen } 855bcabbccaSOhad Ben-Cohen 856bcabbccaSOhad Ben-Cohen /* don't trust the remote processor for null terminating the name */ 857bcabbccaSOhad Ben-Cohen msg->name[RPMSG_NAME_SIZE - 1] = '\0'; 858bcabbccaSOhad Ben-Cohen 859bcabbccaSOhad Ben-Cohen strncpy(chinfo.name, msg->name, sizeof(chinfo.name)); 860bcabbccaSOhad Ben-Cohen chinfo.src = RPMSG_ADDR_ANY; 861111d1089SGuennadi Liakhovetski chinfo.dst = virtio32_to_cpu(vrp->vdev, msg->addr); 862bcabbccaSOhad Ben-Cohen 863111d1089SGuennadi Liakhovetski dev_info(dev, "%sing channel %s addr 0x%x\n", 864111d1089SGuennadi Liakhovetski virtio32_to_cpu(vrp->vdev, msg->flags) & RPMSG_NS_DESTROY ? 865111d1089SGuennadi Liakhovetski "destroy" : "creat", msg->name, chinfo.dst); 866111d1089SGuennadi Liakhovetski 867111d1089SGuennadi Liakhovetski if (virtio32_to_cpu(vrp->vdev, msg->flags) & RPMSG_NS_DESTROY) { 8685e619b48SBjorn Andersson ret = rpmsg_unregister_device(&vrp->vdev->dev, &chinfo); 869bcabbccaSOhad Ben-Cohen if (ret) 870bcabbccaSOhad Ben-Cohen dev_err(dev, "rpmsg_destroy_channel failed: %d\n", ret); 871bcabbccaSOhad Ben-Cohen } else { 872bcabbccaSOhad Ben-Cohen newch = rpmsg_create_channel(vrp, &chinfo); 873bcabbccaSOhad Ben-Cohen if (!newch) 874bcabbccaSOhad Ben-Cohen dev_err(dev, "rpmsg_create_channel failed\n"); 875bcabbccaSOhad Ben-Cohen } 8764b83c52aSBjorn Andersson 8774b83c52aSBjorn Andersson return 0; 878bcabbccaSOhad Ben-Cohen } 879bcabbccaSOhad Ben-Cohen 880bcabbccaSOhad Ben-Cohen static int rpmsg_probe(struct virtio_device *vdev) 881bcabbccaSOhad Ben-Cohen { 882bcabbccaSOhad Ben-Cohen vq_callback_t *vq_cbs[] = { rpmsg_recv_done, rpmsg_xmit_done }; 883f7ad26ffSStefan Hajnoczi static const char * const names[] = { "input", "output" }; 884bcabbccaSOhad Ben-Cohen struct virtqueue *vqs[2]; 885bcabbccaSOhad Ben-Cohen struct virtproc_info *vrp; 886bcabbccaSOhad Ben-Cohen void *bufs_va; 887bcabbccaSOhad Ben-Cohen int err = 0, i; 888b1b98914SSuman Anna size_t total_buf_space; 88971e4b8bfSMichael S. Tsirkin bool notify; 890bcabbccaSOhad Ben-Cohen 891bcabbccaSOhad Ben-Cohen vrp = kzalloc(sizeof(*vrp), GFP_KERNEL); 892bcabbccaSOhad Ben-Cohen if (!vrp) 893bcabbccaSOhad Ben-Cohen return -ENOMEM; 894bcabbccaSOhad Ben-Cohen 895bcabbccaSOhad Ben-Cohen vrp->vdev = vdev; 896bcabbccaSOhad Ben-Cohen 897bcabbccaSOhad Ben-Cohen idr_init(&vrp->endpoints); 898bcabbccaSOhad Ben-Cohen mutex_init(&vrp->endpoints_lock); 899bcabbccaSOhad Ben-Cohen mutex_init(&vrp->tx_lock); 900bcabbccaSOhad Ben-Cohen init_waitqueue_head(&vrp->sendq); 901bcabbccaSOhad Ben-Cohen 902bcabbccaSOhad Ben-Cohen /* We expect two virtqueues, rx and tx (and in this order) */ 9039b2bbdb2SMichael S. Tsirkin err = virtio_find_vqs(vdev, 2, vqs, vq_cbs, names, NULL); 904bcabbccaSOhad Ben-Cohen if (err) 905bcabbccaSOhad Ben-Cohen goto free_vrp; 906bcabbccaSOhad Ben-Cohen 907bcabbccaSOhad Ben-Cohen vrp->rvq = vqs[0]; 908bcabbccaSOhad Ben-Cohen vrp->svq = vqs[1]; 909bcabbccaSOhad Ben-Cohen 910b1b98914SSuman Anna /* we expect symmetric tx/rx vrings */ 911b1b98914SSuman Anna WARN_ON(virtqueue_get_vring_size(vrp->rvq) != 912b1b98914SSuman Anna virtqueue_get_vring_size(vrp->svq)); 913b1b98914SSuman Anna 914b1b98914SSuman Anna /* we need less buffers if vrings are small */ 915b1b98914SSuman Anna if (virtqueue_get_vring_size(vrp->rvq) < MAX_RPMSG_NUM_BUFS / 2) 916b1b98914SSuman Anna vrp->num_bufs = virtqueue_get_vring_size(vrp->rvq) * 2; 917b1b98914SSuman Anna else 918b1b98914SSuman Anna vrp->num_bufs = MAX_RPMSG_NUM_BUFS; 919b1b98914SSuman Anna 920f93848f9SLoic Pallardy vrp->buf_size = MAX_RPMSG_BUF_SIZE; 921f93848f9SLoic Pallardy 922f93848f9SLoic Pallardy total_buf_space = vrp->num_bufs * vrp->buf_size; 923b1b98914SSuman Anna 924bcabbccaSOhad Ben-Cohen /* allocate coherent memory for the buffers */ 925d999b622SLoic Pallardy bufs_va = dma_alloc_coherent(vdev->dev.parent, 926b1b98914SSuman Anna total_buf_space, &vrp->bufs_dma, 927b1b98914SSuman Anna GFP_KERNEL); 9283119b487SWei Yongjun if (!bufs_va) { 9293119b487SWei Yongjun err = -ENOMEM; 930bcabbccaSOhad Ben-Cohen goto vqs_del; 9313119b487SWei Yongjun } 932bcabbccaSOhad Ben-Cohen 933de4064afSSuman Anna dev_dbg(&vdev->dev, "buffers: va %pK, dma %pad\n", 9348d95b322SAnna, Suman bufs_va, &vrp->bufs_dma); 935bcabbccaSOhad Ben-Cohen 936bcabbccaSOhad Ben-Cohen /* half of the buffers is dedicated for RX */ 937bcabbccaSOhad Ben-Cohen vrp->rbufs = bufs_va; 938bcabbccaSOhad Ben-Cohen 939bcabbccaSOhad Ben-Cohen /* and half is dedicated for TX */ 940b1b98914SSuman Anna vrp->sbufs = bufs_va + total_buf_space / 2; 941bcabbccaSOhad Ben-Cohen 942bcabbccaSOhad Ben-Cohen /* set up the receive buffers */ 943b1b98914SSuman Anna for (i = 0; i < vrp->num_bufs / 2; i++) { 944bcabbccaSOhad Ben-Cohen struct scatterlist sg; 945f93848f9SLoic Pallardy void *cpu_addr = vrp->rbufs + i * vrp->buf_size; 946bcabbccaSOhad Ben-Cohen 9479dd87c2aSLoic Pallardy rpmsg_sg_init(&sg, cpu_addr, vrp->buf_size); 948bcabbccaSOhad Ben-Cohen 949cee51d69SRusty Russell err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, cpu_addr, 950bcabbccaSOhad Ben-Cohen GFP_KERNEL); 95157e1a373SRusty Russell WARN_ON(err); /* sanity check; this can't really happen */ 952bcabbccaSOhad Ben-Cohen } 953bcabbccaSOhad Ben-Cohen 954bcabbccaSOhad Ben-Cohen /* suppress "tx-complete" interrupts */ 955bcabbccaSOhad Ben-Cohen virtqueue_disable_cb(vrp->svq); 956bcabbccaSOhad Ben-Cohen 957bcabbccaSOhad Ben-Cohen vdev->priv = vrp; 958bcabbccaSOhad Ben-Cohen 959bcabbccaSOhad Ben-Cohen /* if supported by the remote processor, enable the name service */ 960bcabbccaSOhad Ben-Cohen if (virtio_has_feature(vdev, VIRTIO_RPMSG_F_NS)) { 961bcabbccaSOhad Ben-Cohen /* a dedicated endpoint handles the name service msgs */ 962bcabbccaSOhad Ben-Cohen vrp->ns_ept = __rpmsg_create_ept(vrp, NULL, rpmsg_ns_cb, 963bcabbccaSOhad Ben-Cohen vrp, RPMSG_NS_ADDR); 964bcabbccaSOhad Ben-Cohen if (!vrp->ns_ept) { 965bcabbccaSOhad Ben-Cohen dev_err(&vdev->dev, "failed to create the ns ept\n"); 966bcabbccaSOhad Ben-Cohen err = -ENOMEM; 967bcabbccaSOhad Ben-Cohen goto free_coherent; 968bcabbccaSOhad Ben-Cohen } 969bcabbccaSOhad Ben-Cohen } 970bcabbccaSOhad Ben-Cohen 97171e4b8bfSMichael S. Tsirkin /* 97271e4b8bfSMichael S. Tsirkin * Prepare to kick but don't notify yet - we can't do this before 97371e4b8bfSMichael S. Tsirkin * device is ready. 97471e4b8bfSMichael S. Tsirkin */ 97571e4b8bfSMichael S. Tsirkin notify = virtqueue_kick_prepare(vrp->rvq); 97671e4b8bfSMichael S. Tsirkin 97771e4b8bfSMichael S. Tsirkin /* From this point on, we can notify and get callbacks. */ 97871e4b8bfSMichael S. Tsirkin virtio_device_ready(vdev); 97971e4b8bfSMichael S. Tsirkin 980bcabbccaSOhad Ben-Cohen /* tell the remote processor it can start sending messages */ 98171e4b8bfSMichael S. Tsirkin /* 98271e4b8bfSMichael S. Tsirkin * this might be concurrent with callbacks, but we are only 98371e4b8bfSMichael S. Tsirkin * doing notify, not a full kick here, so that's ok. 98471e4b8bfSMichael S. Tsirkin */ 98571e4b8bfSMichael S. Tsirkin if (notify) 98671e4b8bfSMichael S. Tsirkin virtqueue_notify(vrp->rvq); 987bcabbccaSOhad Ben-Cohen 988bcabbccaSOhad Ben-Cohen dev_info(&vdev->dev, "rpmsg host is online\n"); 989bcabbccaSOhad Ben-Cohen 990bcabbccaSOhad Ben-Cohen return 0; 991bcabbccaSOhad Ben-Cohen 992bcabbccaSOhad Ben-Cohen free_coherent: 993d999b622SLoic Pallardy dma_free_coherent(vdev->dev.parent, total_buf_space, 994eeb0074fSFernando Guzman Lugo bufs_va, vrp->bufs_dma); 995bcabbccaSOhad Ben-Cohen vqs_del: 996bcabbccaSOhad Ben-Cohen vdev->config->del_vqs(vrp->vdev); 997bcabbccaSOhad Ben-Cohen free_vrp: 998bcabbccaSOhad Ben-Cohen kfree(vrp); 999bcabbccaSOhad Ben-Cohen return err; 1000bcabbccaSOhad Ben-Cohen } 1001bcabbccaSOhad Ben-Cohen 1002bcabbccaSOhad Ben-Cohen static int rpmsg_remove_device(struct device *dev, void *data) 1003bcabbccaSOhad Ben-Cohen { 1004bcabbccaSOhad Ben-Cohen device_unregister(dev); 1005bcabbccaSOhad Ben-Cohen 1006bcabbccaSOhad Ben-Cohen return 0; 1007bcabbccaSOhad Ben-Cohen } 1008bcabbccaSOhad Ben-Cohen 10090fe763c5SGreg Kroah-Hartman static void rpmsg_remove(struct virtio_device *vdev) 1010bcabbccaSOhad Ben-Cohen { 1011bcabbccaSOhad Ben-Cohen struct virtproc_info *vrp = vdev->priv; 1012f93848f9SLoic Pallardy size_t total_buf_space = vrp->num_bufs * vrp->buf_size; 1013bcabbccaSOhad Ben-Cohen int ret; 1014bcabbccaSOhad Ben-Cohen 1015bcabbccaSOhad Ben-Cohen vdev->config->reset(vdev); 1016bcabbccaSOhad Ben-Cohen 1017bcabbccaSOhad Ben-Cohen ret = device_for_each_child(&vdev->dev, NULL, rpmsg_remove_device); 1018bcabbccaSOhad Ben-Cohen if (ret) 1019bcabbccaSOhad Ben-Cohen dev_warn(&vdev->dev, "can't remove rpmsg device: %d\n", ret); 1020bcabbccaSOhad Ben-Cohen 1021fa2d7795SOhad Ben-Cohen if (vrp->ns_ept) 1022fa2d7795SOhad Ben-Cohen __rpmsg_destroy_ept(vrp, vrp->ns_ept); 1023fa2d7795SOhad Ben-Cohen 1024bcabbccaSOhad Ben-Cohen idr_destroy(&vrp->endpoints); 1025bcabbccaSOhad Ben-Cohen 1026bcabbccaSOhad Ben-Cohen vdev->config->del_vqs(vrp->vdev); 1027bcabbccaSOhad Ben-Cohen 1028d999b622SLoic Pallardy dma_free_coherent(vdev->dev.parent, total_buf_space, 1029bcabbccaSOhad Ben-Cohen vrp->rbufs, vrp->bufs_dma); 1030bcabbccaSOhad Ben-Cohen 1031bcabbccaSOhad Ben-Cohen kfree(vrp); 1032bcabbccaSOhad Ben-Cohen } 1033bcabbccaSOhad Ben-Cohen 1034bcabbccaSOhad Ben-Cohen static struct virtio_device_id id_table[] = { 1035bcabbccaSOhad Ben-Cohen { VIRTIO_ID_RPMSG, VIRTIO_DEV_ANY_ID }, 1036bcabbccaSOhad Ben-Cohen { 0 }, 1037bcabbccaSOhad Ben-Cohen }; 1038bcabbccaSOhad Ben-Cohen 1039bcabbccaSOhad Ben-Cohen static unsigned int features[] = { 1040bcabbccaSOhad Ben-Cohen VIRTIO_RPMSG_F_NS, 1041bcabbccaSOhad Ben-Cohen }; 1042bcabbccaSOhad Ben-Cohen 1043bcabbccaSOhad Ben-Cohen static struct virtio_driver virtio_ipc_driver = { 1044bcabbccaSOhad Ben-Cohen .feature_table = features, 1045bcabbccaSOhad Ben-Cohen .feature_table_size = ARRAY_SIZE(features), 1046bcabbccaSOhad Ben-Cohen .driver.name = KBUILD_MODNAME, 1047bcabbccaSOhad Ben-Cohen .driver.owner = THIS_MODULE, 1048bcabbccaSOhad Ben-Cohen .id_table = id_table, 1049bcabbccaSOhad Ben-Cohen .probe = rpmsg_probe, 10500fe763c5SGreg Kroah-Hartman .remove = rpmsg_remove, 1051bcabbccaSOhad Ben-Cohen }; 1052bcabbccaSOhad Ben-Cohen 1053bcabbccaSOhad Ben-Cohen static int __init rpmsg_init(void) 1054bcabbccaSOhad Ben-Cohen { 1055bcabbccaSOhad Ben-Cohen int ret; 1056bcabbccaSOhad Ben-Cohen 1057bcabbccaSOhad Ben-Cohen ret = register_virtio_driver(&virtio_ipc_driver); 10585e619b48SBjorn Andersson if (ret) 1059bcabbccaSOhad Ben-Cohen pr_err("failed to register virtio driver: %d\n", ret); 1060bcabbccaSOhad Ben-Cohen 1061bcabbccaSOhad Ben-Cohen return ret; 1062bcabbccaSOhad Ben-Cohen } 106396342526SFederico Fuga subsys_initcall(rpmsg_init); 1064bcabbccaSOhad Ben-Cohen 1065bcabbccaSOhad Ben-Cohen static void __exit rpmsg_fini(void) 1066bcabbccaSOhad Ben-Cohen { 1067bcabbccaSOhad Ben-Cohen unregister_virtio_driver(&virtio_ipc_driver); 1068bcabbccaSOhad Ben-Cohen } 1069bcabbccaSOhad Ben-Cohen module_exit(rpmsg_fini); 1070bcabbccaSOhad Ben-Cohen 1071bcabbccaSOhad Ben-Cohen MODULE_DEVICE_TABLE(virtio, id_table); 1072bcabbccaSOhad Ben-Cohen MODULE_DESCRIPTION("Virtio-based remote processor messaging bus"); 1073bcabbccaSOhad Ben-Cohen MODULE_LICENSE("GPL v2"); 1074