131638bcaSCyrill Gorcunov #include "kvm/virtio-pci-dev.h" 2b5ee1ea7SAsias He #include "kvm/virtio-net.h" 34f56d42cSAsias He #include "kvm/virtio.h" 44f56d42cSAsias He #include "kvm/types.h" 54f56d42cSAsias He #include "kvm/mutex.h" 64f56d42cSAsias He #include "kvm/util.h" 74f56d42cSAsias He #include "kvm/kvm.h" 82449f6e3SSasha Levin #include "kvm/irq.h" 9b5ee1ea7SAsias He #include "kvm/uip.h" 10cb83de6fSSasha Levin #include "kvm/guest_compat.h" 114f56d42cSAsias He 129ed67cdcSSasha Levin #include <linux/vhost.h> 134f56d42cSAsias He #include <linux/virtio_net.h> 144f56d42cSAsias He #include <linux/if_tun.h> 1578a2a3e2SSasha Levin #include <linux/types.h> 16c229370aSIngo Molnar 17c229370aSIngo Molnar #include <arpa/inet.h> 184f56d42cSAsias He #include <net/if.h> 19c229370aSIngo Molnar 20c229370aSIngo Molnar #include <unistd.h> 214f56d42cSAsias He #include <fcntl.h> 22c229370aSIngo Molnar 23cb7202c1SSasha Levin #include <sys/socket.h> 24c229370aSIngo Molnar #include <sys/ioctl.h> 25c229370aSIngo Molnar #include <sys/types.h> 2673b7d038SAmos Kong #include <sys/wait.h> 279ed67cdcSSasha Levin #include <sys/eventfd.h> 284f56d42cSAsias He 294059ad8bSAsias He #define VIRTIO_NET_QUEUE_SIZE 256 306585ffb7SSasha Levin #define VIRTIO_NET_NUM_QUEUES 8 314f56d42cSAsias He 32b4fdde6dSAsias He struct net_dev; 33b4fdde6dSAsias He 34b4fdde6dSAsias He struct net_dev_operations { 35b4fdde6dSAsias He int (*rx)(struct iovec *iov, u16 in, struct net_dev *ndev); 36b4fdde6dSAsias He int (*tx)(struct iovec *iov, u16 in, struct net_dev *ndev); 37b4fdde6dSAsias He }; 38b4fdde6dSAsias He 398626798bSAsias He struct net_dev { 40d3476f7dSSasha Levin struct mutex mutex; 4102eca50cSAsias He struct virtio_device vdev; 429a6d73f1SSasha Levin struct list_head list; 434f56d42cSAsias He 446585ffb7SSasha Levin struct virt_queue vqs[VIRTIO_NET_NUM_QUEUES * 2 + 1]; 45c229370aSIngo Molnar struct virtio_net_config config; 4679052597SSasha Levin u32 features, rx_vqs, tx_vqs, queue_pairs; 474f56d42cSAsias He 486585ffb7SSasha Levin pthread_t io_thread[VIRTIO_NET_NUM_QUEUES * 2 + 1]; 496585ffb7SSasha Levin struct mutex io_lock[VIRTIO_NET_NUM_QUEUES * 2 + 1]; 506585ffb7SSasha Levin pthread_cond_t io_cond[VIRTIO_NET_NUM_QUEUES * 2 + 1]; 51c4aa7c02SPekka Enberg 529ed67cdcSSasha Levin int vhost_fd; 534f56d42cSAsias He int tap_fd; 544f56d42cSAsias He char tap_name[IFNAMSIZ]; 55bb1a32f1SAsias He 56bb1a32f1SAsias He int mode; 57bb1a32f1SAsias He 58b5ee1ea7SAsias He struct uip_info info; 59b4fdde6dSAsias He struct net_dev_operations *ops; 609a6d73f1SSasha Levin struct kvm *kvm; 614f56d42cSAsias He }; 624f56d42cSAsias He 639a6d73f1SSasha Levin static LIST_HEAD(ndevs); 64312c62d1SSasha Levin static int compat_id = -1; 654f56d42cSAsias He 66c4aa7c02SPekka Enberg static void *virtio_net_rx_thread(void *p) 674f56d42cSAsias He { 684f56d42cSAsias He struct iovec iov[VIRTIO_NET_QUEUE_SIZE]; 694f56d42cSAsias He struct virt_queue *vq; 7043835ac9SSasha Levin struct kvm *kvm; 719a6d73f1SSasha Levin struct net_dev *ndev = p; 723fdf659dSSasha Levin u16 out, in; 733fdf659dSSasha Levin u16 head; 744f56d42cSAsias He int len; 756585ffb7SSasha Levin u32 id; 766585ffb7SSasha Levin 776585ffb7SSasha Levin mutex_lock(&ndev->mutex); 786585ffb7SSasha Levin id = ndev->rx_vqs++ * 2; 796585ffb7SSasha Levin mutex_unlock(&ndev->mutex); 804f56d42cSAsias He 81a4d8c55eSSasha Levin kvm__set_thread_name("virtio-net-rx"); 82a4d8c55eSSasha Levin 839a6d73f1SSasha Levin kvm = ndev->kvm; 846585ffb7SSasha Levin vq = &ndev->vqs[id]; 85c4aa7c02SPekka Enberg 86c4aa7c02SPekka Enberg while (1) { 876585ffb7SSasha Levin mutex_lock(&ndev->io_lock[id]); 88c4aa7c02SPekka Enberg if (!virt_queue__available(vq)) 896585ffb7SSasha Levin pthread_cond_wait(&ndev->io_cond[id], &ndev->io_lock[id].mutex); 906585ffb7SSasha Levin mutex_unlock(&ndev->io_lock[id]); 914f56d42cSAsias He 924f56d42cSAsias He while (virt_queue__available(vq)) { 9343835ac9SSasha Levin head = virt_queue__get_iov(vq, iov, &out, &in, kvm); 949a6d73f1SSasha Levin len = ndev->ops->rx(iov, in, ndev); 95246c8347SAsias He virt_queue__set_used_elem(vq, head, len); 967f5ffaf5SAsias He 97c4aa7c02SPekka Enberg /* We should interrupt guest right now, otherwise latency is huge. */ 986585ffb7SSasha Levin if (virtio_queue__should_signal(vq)) 996585ffb7SSasha Levin ndev->vdev.ops->signal_vq(kvm, &ndev->vdev, id); 1004f56d42cSAsias He } 101c4aa7c02SPekka Enberg } 102c4aa7c02SPekka Enberg 103c4aa7c02SPekka Enberg pthread_exit(NULL); 104c4aa7c02SPekka Enberg return NULL; 105c4aa7c02SPekka Enberg 106c4aa7c02SPekka Enberg } 107c4aa7c02SPekka Enberg 108c4aa7c02SPekka Enberg static void *virtio_net_tx_thread(void *p) 1094f56d42cSAsias He { 1104f56d42cSAsias He struct iovec iov[VIRTIO_NET_QUEUE_SIZE]; 1114f56d42cSAsias He struct virt_queue *vq; 11243835ac9SSasha Levin struct kvm *kvm; 1139a6d73f1SSasha Levin struct net_dev *ndev = p; 1143fdf659dSSasha Levin u16 out, in; 1153fdf659dSSasha Levin u16 head; 1164f56d42cSAsias He int len; 1176585ffb7SSasha Levin u32 id; 1186585ffb7SSasha Levin 1196585ffb7SSasha Levin mutex_lock(&ndev->mutex); 1206585ffb7SSasha Levin id = ndev->tx_vqs++ * 2 + 1; 1216585ffb7SSasha Levin mutex_unlock(&ndev->mutex); 1224f56d42cSAsias He 123a4d8c55eSSasha Levin kvm__set_thread_name("virtio-net-tx"); 124a4d8c55eSSasha Levin 1259a6d73f1SSasha Levin kvm = ndev->kvm; 1266585ffb7SSasha Levin vq = &ndev->vqs[id]; 127c4aa7c02SPekka Enberg 128c4aa7c02SPekka Enberg while (1) { 1296585ffb7SSasha Levin mutex_lock(&ndev->io_lock[id]); 130c4aa7c02SPekka Enberg if (!virt_queue__available(vq)) 1316585ffb7SSasha Levin pthread_cond_wait(&ndev->io_cond[id], &ndev->io_lock[id].mutex); 1326585ffb7SSasha Levin mutex_unlock(&ndev->io_lock[id]); 1334f56d42cSAsias He 1344f56d42cSAsias He while (virt_queue__available(vq)) { 13543835ac9SSasha Levin head = virt_queue__get_iov(vq, iov, &out, &in, kvm); 1369a6d73f1SSasha Levin len = ndev->ops->tx(iov, out, ndev); 1374f56d42cSAsias He virt_queue__set_used_elem(vq, head, len); 1384f56d42cSAsias He } 1394f56d42cSAsias He 1406585ffb7SSasha Levin if (virtio_queue__should_signal(vq)) 1416585ffb7SSasha Levin ndev->vdev.ops->signal_vq(kvm, &ndev->vdev, id); 1424f56d42cSAsias He } 1434f56d42cSAsias He 144c4aa7c02SPekka Enberg pthread_exit(NULL); 145407475bfSPekka Enberg 146c4aa7c02SPekka Enberg return NULL; 147c4aa7c02SPekka Enberg 148c4aa7c02SPekka Enberg } 149407475bfSPekka Enberg 1506585ffb7SSasha Levin static virtio_net_ctrl_ack virtio_net_handle_mq(struct kvm* kvm, struct net_dev *ndev, struct virtio_net_ctrl_hdr *ctrl) 1516585ffb7SSasha Levin { 1526585ffb7SSasha Levin /* Not much to do here */ 1536585ffb7SSasha Levin return VIRTIO_NET_OK; 1546585ffb7SSasha Levin } 1556585ffb7SSasha Levin 1566585ffb7SSasha Levin static void *virtio_net_ctrl_thread(void *p) 157abd4a801SSasha Levin { 158abd4a801SSasha Levin struct iovec iov[VIRTIO_NET_QUEUE_SIZE]; 159abd4a801SSasha Levin u16 out, in, head; 1606585ffb7SSasha Levin struct net_dev *ndev = p; 1616585ffb7SSasha Levin struct kvm *kvm = ndev->kvm; 16279052597SSasha Levin u32 id = ndev->queue_pairs * 2; 1636585ffb7SSasha Levin struct virt_queue *vq = &ndev->vqs[id]; 164abd4a801SSasha Levin struct virtio_net_ctrl_hdr *ctrl; 165abd4a801SSasha Levin virtio_net_ctrl_ack *ack; 166abd4a801SSasha Levin 1676585ffb7SSasha Levin while (1) { 1686585ffb7SSasha Levin mutex_lock(&ndev->io_lock[id]); 1696585ffb7SSasha Levin if (!virt_queue__available(vq)) 1706585ffb7SSasha Levin pthread_cond_wait(&ndev->io_cond[id], &ndev->io_lock[id].mutex); 1716585ffb7SSasha Levin mutex_unlock(&ndev->io_lock[id]); 1726585ffb7SSasha Levin 1736585ffb7SSasha Levin while (virt_queue__available(vq)) { 1746585ffb7SSasha Levin head = virt_queue__get_iov(&ndev->vqs[id], iov, &out, &in, kvm); 175abd4a801SSasha Levin ctrl = iov[0].iov_base; 176abd4a801SSasha Levin ack = iov[out].iov_base; 177abd4a801SSasha Levin 178abd4a801SSasha Levin switch (ctrl->class) { 1796585ffb7SSasha Levin case VIRTIO_NET_CTRL_MQ: 1806585ffb7SSasha Levin *ack = virtio_net_handle_mq(kvm, ndev, ctrl); 1816585ffb7SSasha Levin break; 182abd4a801SSasha Levin default: 183abd4a801SSasha Levin *ack = VIRTIO_NET_ERR; 184abd4a801SSasha Levin break; 185abd4a801SSasha Levin } 1866585ffb7SSasha Levin virt_queue__set_used_elem(&ndev->vqs[id], head, iov[out].iov_len); 1876585ffb7SSasha Levin } 188abd4a801SSasha Levin 1896585ffb7SSasha Levin if (virtio_queue__should_signal(&ndev->vqs[id])) 1906585ffb7SSasha Levin ndev->vdev.ops->signal_vq(kvm, &ndev->vdev, id); 1916585ffb7SSasha Levin } 192abd4a801SSasha Levin 1936585ffb7SSasha Levin pthread_exit(NULL); 194abd4a801SSasha Levin 1956585ffb7SSasha Levin return NULL; 196abd4a801SSasha Levin } 197abd4a801SSasha Levin 1989a6d73f1SSasha Levin static void virtio_net_handle_callback(struct kvm *kvm, struct net_dev *ndev, int queue) 1994f56d42cSAsias He { 20079052597SSasha Levin if ((u32)queue >= (ndev->queue_pairs * 2 + 1)) { 2019a6d73f1SSasha Levin pr_warning("Unknown queue index %u", queue); 2026585ffb7SSasha Levin return; 203c4aa7c02SPekka Enberg } 2046585ffb7SSasha Levin 2056585ffb7SSasha Levin mutex_lock(&ndev->io_lock[queue]); 2066585ffb7SSasha Levin pthread_cond_signal(&ndev->io_cond[queue]); 2076585ffb7SSasha Levin mutex_unlock(&ndev->io_lock[queue]); 2084f56d42cSAsias He } 2094f56d42cSAsias He 2109a6d73f1SSasha Levin static bool virtio_net__tap_init(const struct virtio_net_params *params, 2119a6d73f1SSasha Levin struct net_dev *ndev) 2124f56d42cSAsias He { 213cb7202c1SSasha Levin int sock = socket(AF_INET, SOCK_STREAM, 0); 214f715177dSAsias He int pid, status, offload, hdr_len; 215cb7202c1SSasha Levin struct sockaddr_in sin = {0}; 216246c8347SAsias He struct ifreq ifr; 2174f56d42cSAsias He 218f19edd1eSSasha Levin /* Did the user already gave us the FD? */ 219f19edd1eSSasha Levin if (params->fd) { 220f19edd1eSSasha Levin ndev->tap_fd = params->fd; 221f19edd1eSSasha Levin return 1; 222f19edd1eSSasha Levin } 223f19edd1eSSasha Levin 2249a6d73f1SSasha Levin ndev->tap_fd = open("/dev/net/tun", O_RDWR); 2259a6d73f1SSasha Levin if (ndev->tap_fd < 0) { 2264542f276SCyrill Gorcunov pr_warning("Unable to open /dev/net/tun"); 2273b02f580SSasha Levin goto fail; 2283b02f580SSasha Levin } 2294f56d42cSAsias He 2304f56d42cSAsias He memset(&ifr, 0, sizeof(ifr)); 231246c8347SAsias He ifr.ifr_flags = IFF_TAP | IFF_NO_PI | IFF_VNET_HDR; 2329a6d73f1SSasha Levin if (ioctl(ndev->tap_fd, TUNSETIFF, &ifr) < 0) { 2334542f276SCyrill Gorcunov pr_warning("Config tap device error. Are you root?"); 2343b02f580SSasha Levin goto fail; 2353b02f580SSasha Levin } 2364f56d42cSAsias He 2379a6d73f1SSasha Levin strncpy(ndev->tap_name, ifr.ifr_name, sizeof(ndev->tap_name)); 2384f56d42cSAsias He 2399a6d73f1SSasha Levin if (ioctl(ndev->tap_fd, TUNSETNOCSUM, 1) < 0) { 2404542f276SCyrill Gorcunov pr_warning("Config tap device TUNSETNOCSUM error"); 241246c8347SAsias He goto fail; 242246c8347SAsias He } 243246c8347SAsias He 244246c8347SAsias He hdr_len = sizeof(struct virtio_net_hdr); 2459a6d73f1SSasha Levin if (ioctl(ndev->tap_fd, TUNSETVNETHDRSZ, &hdr_len) < 0) 2464542f276SCyrill Gorcunov pr_warning("Config tap device TUNSETVNETHDRSZ error"); 247246c8347SAsias He 248246c8347SAsias He offload = TUN_F_CSUM | TUN_F_TSO4 | TUN_F_TSO6 | TUN_F_UFO; 2499a6d73f1SSasha Levin if (ioctl(ndev->tap_fd, TUNSETOFFLOAD, offload) < 0) { 2504542f276SCyrill Gorcunov pr_warning("Config tap device TUNSETOFFLOAD error"); 251246c8347SAsias He goto fail; 252246c8347SAsias He } 2534f56d42cSAsias He 25473b7d038SAmos Kong if (strcmp(params->script, "none")) { 25573b7d038SAmos Kong pid = fork(); 25673b7d038SAmos Kong if (pid == 0) { 2579a6d73f1SSasha Levin execl(params->script, params->script, ndev->tap_name, NULL); 25873b7d038SAmos Kong _exit(1); 25973b7d038SAmos Kong } else { 26073b7d038SAmos Kong waitpid(pid, &status, 0); 26173b7d038SAmos Kong if (WIFEXITED(status) && WEXITSTATUS(status) != 0) { 2624542f276SCyrill Gorcunov pr_warning("Fail to setup tap by %s", params->script); 26373b7d038SAmos Kong goto fail; 26473b7d038SAmos Kong } 26573b7d038SAmos Kong } 26673b7d038SAmos Kong } else { 267cb7202c1SSasha Levin memset(&ifr, 0, sizeof(ifr)); 2689a6d73f1SSasha Levin strncpy(ifr.ifr_name, ndev->tap_name, sizeof(ndev->tap_name)); 269bdfcfca6SSasha Levin sin.sin_addr.s_addr = inet_addr(params->host_ip); 270cb7202c1SSasha Levin memcpy(&(ifr.ifr_addr), &sin, sizeof(ifr.ifr_addr)); 271cb7202c1SSasha Levin ifr.ifr_addr.sa_family = AF_INET; 2723b02f580SSasha Levin if (ioctl(sock, SIOCSIFADDR, &ifr) < 0) { 2734542f276SCyrill Gorcunov pr_warning("Could not set ip address on tap device"); 2743b02f580SSasha Levin goto fail; 2753b02f580SSasha Levin } 27673b7d038SAmos Kong } 277cb7202c1SSasha Levin 278cb7202c1SSasha Levin memset(&ifr, 0, sizeof(ifr)); 2799a6d73f1SSasha Levin strncpy(ifr.ifr_name, ndev->tap_name, sizeof(ndev->tap_name)); 280cb7202c1SSasha Levin ioctl(sock, SIOCGIFFLAGS, &ifr); 281cb7202c1SSasha Levin ifr.ifr_flags |= IFF_UP | IFF_RUNNING; 282cb7202c1SSasha Levin if (ioctl(sock, SIOCSIFFLAGS, &ifr) < 0) 2834542f276SCyrill Gorcunov pr_warning("Could not bring tap device up"); 284cb7202c1SSasha Levin 285cb7202c1SSasha Levin close(sock); 2863b02f580SSasha Levin 2873b02f580SSasha Levin return 1; 2883b02f580SSasha Levin 2893b02f580SSasha Levin fail: 2903b02f580SSasha Levin if (sock >= 0) 2913b02f580SSasha Levin close(sock); 2929a6d73f1SSasha Levin if (ndev->tap_fd >= 0) 2939a6d73f1SSasha Levin close(ndev->tap_fd); 2943b02f580SSasha Levin 2953b02f580SSasha Levin return 0; 2964f56d42cSAsias He } 2974f56d42cSAsias He 298b4fdde6dSAsias He static inline int tap_ops_tx(struct iovec *iov, u16 out, struct net_dev *ndev) 299b4fdde6dSAsias He { 300b4fdde6dSAsias He return writev(ndev->tap_fd, iov, out); 301b4fdde6dSAsias He } 302b4fdde6dSAsias He 303b4fdde6dSAsias He static inline int tap_ops_rx(struct iovec *iov, u16 in, struct net_dev *ndev) 304b4fdde6dSAsias He { 305b4fdde6dSAsias He return readv(ndev->tap_fd, iov, in); 306b4fdde6dSAsias He } 307b4fdde6dSAsias He 308b4fdde6dSAsias He static inline int uip_ops_tx(struct iovec *iov, u16 out, struct net_dev *ndev) 309b4fdde6dSAsias He { 310b4fdde6dSAsias He return uip_tx(iov, out, &ndev->info); 311b4fdde6dSAsias He } 312b4fdde6dSAsias He 313b4fdde6dSAsias He static inline int uip_ops_rx(struct iovec *iov, u16 in, struct net_dev *ndev) 314b4fdde6dSAsias He { 315b4fdde6dSAsias He return uip_rx(iov, in, &ndev->info); 316b4fdde6dSAsias He } 317b4fdde6dSAsias He 318b4fdde6dSAsias He static struct net_dev_operations tap_ops = { 319b4fdde6dSAsias He .rx = tap_ops_rx, 320b4fdde6dSAsias He .tx = tap_ops_tx, 321b4fdde6dSAsias He }; 322b4fdde6dSAsias He 323b4fdde6dSAsias He static struct net_dev_operations uip_ops = { 324b4fdde6dSAsias He .rx = uip_ops_rx, 325b4fdde6dSAsias He .tx = uip_ops_tx, 326b4fdde6dSAsias He }; 327b4fdde6dSAsias He 328c5ae742bSSasha Levin static u8 *get_config(struct kvm *kvm, void *dev) 32978a2a3e2SSasha Levin { 33078a2a3e2SSasha Levin struct net_dev *ndev = dev; 33178a2a3e2SSasha Levin 332c5ae742bSSasha Levin return ((u8 *)(&ndev->config)); 33378a2a3e2SSasha Levin } 33478a2a3e2SSasha Levin 33578a2a3e2SSasha Levin static u32 get_host_features(struct kvm *kvm, void *dev) 33678a2a3e2SSasha Levin { 33779052597SSasha Levin struct net_dev *ndev = dev; 33879052597SSasha Levin 33978a2a3e2SSasha Levin return 1UL << VIRTIO_NET_F_MAC 34078a2a3e2SSasha Levin | 1UL << VIRTIO_NET_F_CSUM 34178a2a3e2SSasha Levin | 1UL << VIRTIO_NET_F_HOST_UFO 34278a2a3e2SSasha Levin | 1UL << VIRTIO_NET_F_HOST_TSO4 34378a2a3e2SSasha Levin | 1UL << VIRTIO_NET_F_HOST_TSO6 34478a2a3e2SSasha Levin | 1UL << VIRTIO_NET_F_GUEST_UFO 34578a2a3e2SSasha Levin | 1UL << VIRTIO_NET_F_GUEST_TSO4 34692c1f37bSSasha Levin | 1UL << VIRTIO_NET_F_GUEST_TSO6 347754c8ce3SSasha Levin | 1UL << VIRTIO_RING_F_EVENT_IDX 348abd4a801SSasha Levin | 1UL << VIRTIO_RING_F_INDIRECT_DESC 3496585ffb7SSasha Levin | 1UL << VIRTIO_NET_F_CTRL_VQ 35079052597SSasha Levin | 1UL << (ndev->queue_pairs > 1 ? VIRTIO_NET_F_MQ : 0); 35178a2a3e2SSasha Levin } 35278a2a3e2SSasha Levin 35378a2a3e2SSasha Levin static void set_guest_features(struct kvm *kvm, void *dev, u32 features) 35478a2a3e2SSasha Levin { 35578a2a3e2SSasha Levin struct net_dev *ndev = dev; 35678a2a3e2SSasha Levin 35778a2a3e2SSasha Levin ndev->features = features; 35878a2a3e2SSasha Levin } 35978a2a3e2SSasha Levin 360*9e56ec14SSasha Levin static bool is_ctrl_vq(struct net_dev *ndev, u32 vq) 361*9e56ec14SSasha Levin { 362*9e56ec14SSasha Levin return vq == (u32)(ndev->queue_pairs * 2); 363*9e56ec14SSasha Levin } 364*9e56ec14SSasha Levin 365c59ba304SWill Deacon static int init_vq(struct kvm *kvm, void *dev, u32 vq, u32 page_size, u32 align, 366c59ba304SWill Deacon u32 pfn) 36778a2a3e2SSasha Levin { 3689ed67cdcSSasha Levin struct vhost_vring_state state = { .index = vq }; 3699ed67cdcSSasha Levin struct vhost_vring_addr addr; 37078a2a3e2SSasha Levin struct net_dev *ndev = dev; 37178a2a3e2SSasha Levin struct virt_queue *queue; 37278a2a3e2SSasha Levin void *p; 3739ed67cdcSSasha Levin int r; 37478a2a3e2SSasha Levin 375312c62d1SSasha Levin compat__remove_message(compat_id); 37678a2a3e2SSasha Levin 37778a2a3e2SSasha Levin queue = &ndev->vqs[vq]; 37878a2a3e2SSasha Levin queue->pfn = pfn; 379c59ba304SWill Deacon p = guest_flat_to_host(kvm, queue->pfn * page_size); 38078a2a3e2SSasha Levin 381c59ba304SWill Deacon vring_init(&queue->vring, VIRTIO_NET_QUEUE_SIZE, p, align); 38278a2a3e2SSasha Levin 3836585ffb7SSasha Levin mutex_init(&ndev->io_lock[vq]); 3846585ffb7SSasha Levin pthread_cond_init(&ndev->io_cond[vq], NULL); 385*9e56ec14SSasha Levin if (is_ctrl_vq(ndev, vq)) { 3866585ffb7SSasha Levin pthread_create(&ndev->io_thread[vq], NULL, virtio_net_ctrl_thread, ndev); 387*9e56ec14SSasha Levin 388*9e56ec14SSasha Levin return 0; 389*9e56ec14SSasha Levin } else if (ndev->vhost_fd == 0 ) { 390*9e56ec14SSasha Levin if (vq & 1) 3916585ffb7SSasha Levin pthread_create(&ndev->io_thread[vq], NULL, virtio_net_tx_thread, ndev); 3926585ffb7SSasha Levin else 3936585ffb7SSasha Levin pthread_create(&ndev->io_thread[vq], NULL, virtio_net_rx_thread, ndev); 3946585ffb7SSasha Levin 39578a2a3e2SSasha Levin return 0; 3966585ffb7SSasha Levin } 3979ed67cdcSSasha Levin 3989ed67cdcSSasha Levin state.num = queue->vring.num; 3999ed67cdcSSasha Levin r = ioctl(ndev->vhost_fd, VHOST_SET_VRING_NUM, &state); 4009ed67cdcSSasha Levin if (r < 0) 4019ed67cdcSSasha Levin die_perror("VHOST_SET_VRING_NUM failed"); 4029ed67cdcSSasha Levin state.num = 0; 4039ed67cdcSSasha Levin r = ioctl(ndev->vhost_fd, VHOST_SET_VRING_BASE, &state); 4049ed67cdcSSasha Levin if (r < 0) 4059ed67cdcSSasha Levin die_perror("VHOST_SET_VRING_BASE failed"); 4069ed67cdcSSasha Levin 4079ed67cdcSSasha Levin addr = (struct vhost_vring_addr) { 4089ed67cdcSSasha Levin .index = vq, 4099ed67cdcSSasha Levin .desc_user_addr = (u64)(unsigned long)queue->vring.desc, 4109ed67cdcSSasha Levin .avail_user_addr = (u64)(unsigned long)queue->vring.avail, 4119ed67cdcSSasha Levin .used_user_addr = (u64)(unsigned long)queue->vring.used, 4129ed67cdcSSasha Levin }; 4139ed67cdcSSasha Levin 4149ed67cdcSSasha Levin r = ioctl(ndev->vhost_fd, VHOST_SET_VRING_ADDR, &addr); 4159ed67cdcSSasha Levin if (r < 0) 4169ed67cdcSSasha Levin die_perror("VHOST_SET_VRING_ADDR failed"); 4179ed67cdcSSasha Levin 4189ed67cdcSSasha Levin return 0; 4199ed67cdcSSasha Levin } 4209ed67cdcSSasha Levin 4219ed67cdcSSasha Levin static void notify_vq_gsi(struct kvm *kvm, void *dev, u32 vq, u32 gsi) 4229ed67cdcSSasha Levin { 4239ed67cdcSSasha Levin struct net_dev *ndev = dev; 4249ed67cdcSSasha Levin struct kvm_irqfd irq; 4259ed67cdcSSasha Levin struct vhost_vring_file file; 4269ed67cdcSSasha Levin int r; 4279ed67cdcSSasha Levin 4289ed67cdcSSasha Levin if (ndev->vhost_fd == 0) 4299ed67cdcSSasha Levin return; 4309ed67cdcSSasha Levin 4319ed67cdcSSasha Levin irq = (struct kvm_irqfd) { 4329ed67cdcSSasha Levin .gsi = gsi, 4339ed67cdcSSasha Levin .fd = eventfd(0, 0), 4349ed67cdcSSasha Levin }; 4359ed67cdcSSasha Levin file = (struct vhost_vring_file) { 4369ed67cdcSSasha Levin .index = vq, 4379ed67cdcSSasha Levin .fd = irq.fd, 4389ed67cdcSSasha Levin }; 4399ed67cdcSSasha Levin 4409ed67cdcSSasha Levin r = ioctl(kvm->vm_fd, KVM_IRQFD, &irq); 4419ed67cdcSSasha Levin if (r < 0) 4429ed67cdcSSasha Levin die_perror("KVM_IRQFD failed"); 4439ed67cdcSSasha Levin 4449ed67cdcSSasha Levin r = ioctl(ndev->vhost_fd, VHOST_SET_VRING_CALL, &file); 4459ed67cdcSSasha Levin if (r < 0) 4469ed67cdcSSasha Levin die_perror("VHOST_SET_VRING_CALL failed"); 4479ed67cdcSSasha Levin file.fd = ndev->tap_fd; 4489ed67cdcSSasha Levin r = ioctl(ndev->vhost_fd, VHOST_NET_SET_BACKEND, &file); 4499ed67cdcSSasha Levin if (r != 0) 4509ed67cdcSSasha Levin die("VHOST_NET_SET_BACKEND failed %d", errno); 4519ed67cdcSSasha Levin 4529ed67cdcSSasha Levin } 4539ed67cdcSSasha Levin 4549ed67cdcSSasha Levin static void notify_vq_eventfd(struct kvm *kvm, void *dev, u32 vq, u32 efd) 4559ed67cdcSSasha Levin { 4569ed67cdcSSasha Levin struct net_dev *ndev = dev; 4579ed67cdcSSasha Levin struct vhost_vring_file file = { 4589ed67cdcSSasha Levin .index = vq, 4599ed67cdcSSasha Levin .fd = efd, 4609ed67cdcSSasha Levin }; 4619ed67cdcSSasha Levin int r; 4629ed67cdcSSasha Levin 463*9e56ec14SSasha Levin if (ndev->vhost_fd == 0 || is_ctrl_vq(ndev, vq)) 4649ed67cdcSSasha Levin return; 4659ed67cdcSSasha Levin 4669ed67cdcSSasha Levin r = ioctl(ndev->vhost_fd, VHOST_SET_VRING_KICK, &file); 4679ed67cdcSSasha Levin if (r < 0) 4689ed67cdcSSasha Levin die_perror("VHOST_SET_VRING_KICK failed"); 46978a2a3e2SSasha Levin } 47078a2a3e2SSasha Levin 47178a2a3e2SSasha Levin static int notify_vq(struct kvm *kvm, void *dev, u32 vq) 47278a2a3e2SSasha Levin { 4739a6d73f1SSasha Levin struct net_dev *ndev = dev; 4749a6d73f1SSasha Levin 4759a6d73f1SSasha Levin virtio_net_handle_callback(kvm, ndev, vq); 47678a2a3e2SSasha Levin 47778a2a3e2SSasha Levin return 0; 47878a2a3e2SSasha Levin } 47978a2a3e2SSasha Levin 48078a2a3e2SSasha Levin static int get_pfn_vq(struct kvm *kvm, void *dev, u32 vq) 48178a2a3e2SSasha Levin { 48278a2a3e2SSasha Levin struct net_dev *ndev = dev; 48378a2a3e2SSasha Levin 48478a2a3e2SSasha Levin return ndev->vqs[vq].pfn; 48578a2a3e2SSasha Levin } 48678a2a3e2SSasha Levin 48778a2a3e2SSasha Levin static int get_size_vq(struct kvm *kvm, void *dev, u32 vq) 48878a2a3e2SSasha Levin { 4890f4f51a8SAsias He /* FIXME: dynamic */ 49078a2a3e2SSasha Levin return VIRTIO_NET_QUEUE_SIZE; 49178a2a3e2SSasha Levin } 49278a2a3e2SSasha Levin 4930f4f51a8SAsias He static int set_size_vq(struct kvm *kvm, void *dev, u32 vq, int size) 4940f4f51a8SAsias He { 4950f4f51a8SAsias He /* FIXME: dynamic */ 4960f4f51a8SAsias He return size; 4970f4f51a8SAsias He } 4980f4f51a8SAsias He 4991c47ce69SSasha Levin static struct virtio_ops net_dev_virtio_ops = (struct virtio_ops) { 5001c47ce69SSasha Levin .get_config = get_config, 5011c47ce69SSasha Levin .get_host_features = get_host_features, 5021c47ce69SSasha Levin .set_guest_features = set_guest_features, 5031c47ce69SSasha Levin .init_vq = init_vq, 5041c47ce69SSasha Levin .get_pfn_vq = get_pfn_vq, 5051c47ce69SSasha Levin .get_size_vq = get_size_vq, 5060f4f51a8SAsias He .set_size_vq = set_size_vq, 5070f4f51a8SAsias He .notify_vq = notify_vq, 5089ed67cdcSSasha Levin .notify_vq_gsi = notify_vq_gsi, 5099ed67cdcSSasha Levin .notify_vq_eventfd = notify_vq_eventfd, 5101c47ce69SSasha Levin }; 5111c47ce69SSasha Levin 5129ed67cdcSSasha Levin static void virtio_net__vhost_init(struct kvm *kvm, struct net_dev *ndev) 5139ed67cdcSSasha Levin { 51492c1f37bSSasha Levin u64 features = 1UL << VIRTIO_RING_F_EVENT_IDX; 5159ed67cdcSSasha Levin struct vhost_memory *mem; 5169ed67cdcSSasha Levin int r; 5179ed67cdcSSasha Levin 5189ed67cdcSSasha Levin ndev->vhost_fd = open("/dev/vhost-net", O_RDWR); 5199ed67cdcSSasha Levin if (ndev->vhost_fd < 0) 5209ed67cdcSSasha Levin die_perror("Failed openning vhost-net device"); 5219ed67cdcSSasha Levin 5228b795457SAsias He mem = calloc(1, sizeof(*mem) + sizeof(struct vhost_memory_region)); 5239ed67cdcSSasha Levin if (mem == NULL) 5249ed67cdcSSasha Levin die("Failed allocating memory for vhost memory map"); 5259ed67cdcSSasha Levin 5269ed67cdcSSasha Levin mem->nregions = 1; 5279ed67cdcSSasha Levin mem->regions[0] = (struct vhost_memory_region) { 5289ed67cdcSSasha Levin .guest_phys_addr = 0, 5299ed67cdcSSasha Levin .memory_size = kvm->ram_size, 530b4108023SAsias He .userspace_addr = (unsigned long)kvm->ram_start, 5319ed67cdcSSasha Levin }; 5329ed67cdcSSasha Levin 5339ed67cdcSSasha Levin r = ioctl(ndev->vhost_fd, VHOST_SET_OWNER); 5349ed67cdcSSasha Levin if (r != 0) 5359ed67cdcSSasha Levin die_perror("VHOST_SET_OWNER failed"); 5369ed67cdcSSasha Levin 5379ed67cdcSSasha Levin r = ioctl(ndev->vhost_fd, VHOST_SET_FEATURES, &features); 5389ed67cdcSSasha Levin if (r != 0) 5399ed67cdcSSasha Levin die_perror("VHOST_SET_FEATURES failed"); 5409ed67cdcSSasha Levin r = ioctl(ndev->vhost_fd, VHOST_SET_MEM_TABLE, mem); 5419ed67cdcSSasha Levin if (r != 0) 5429ed67cdcSSasha Levin die_perror("VHOST_SET_MEM_TABLE failed"); 543627d6874SAsias He 544627d6874SAsias He ndev->vdev.use_vhost = true; 545627d6874SAsias He 5469ed67cdcSSasha Levin free(mem); 5479ed67cdcSSasha Levin } 5489ed67cdcSSasha Levin 5495f225124SSasha Levin static inline void str_to_mac(const char *str, char *mac) 5505f225124SSasha Levin { 5515f225124SSasha Levin sscanf(str, "%hhx:%hhx:%hhx:%hhx:%hhx:%hhx", 5525f225124SSasha Levin mac, mac+1, mac+2, mac+3, mac+4, mac+5); 5535f225124SSasha Levin } 5544346fd8fSSasha Levin static int set_net_param(struct kvm *kvm, struct virtio_net_params *p, 5554346fd8fSSasha Levin const char *param, const char *val) 5565f225124SSasha Levin { 5575f225124SSasha Levin if (strcmp(param, "guest_mac") == 0) { 5585f225124SSasha Levin str_to_mac(val, p->guest_mac); 5595f225124SSasha Levin } else if (strcmp(param, "mode") == 0) { 5605f225124SSasha Levin if (!strncmp(val, "user", 4)) { 5615f225124SSasha Levin int i; 5625f225124SSasha Levin 5635f225124SSasha Levin for (i = 0; i < kvm->cfg.num_net_devices; i++) 5645f225124SSasha Levin if (kvm->cfg.net_params[i].mode == NET_MODE_USER) 5655f225124SSasha Levin die("Only one usermode network device allowed at a time"); 5665f225124SSasha Levin p->mode = NET_MODE_USER; 5675f225124SSasha Levin } else if (!strncmp(val, "tap", 3)) { 5685f225124SSasha Levin p->mode = NET_MODE_TAP; 5695f225124SSasha Levin } else if (!strncmp(val, "none", 4)) { 5705f225124SSasha Levin kvm->cfg.no_net = 1; 5715f225124SSasha Levin return -1; 5725f225124SSasha Levin } else 5735f225124SSasha Levin die("Unknown network mode %s, please use user, tap or none", kvm->cfg.network); 5745f225124SSasha Levin } else if (strcmp(param, "script") == 0) { 5755f225124SSasha Levin p->script = strdup(val); 5765f225124SSasha Levin } else if (strcmp(param, "guest_ip") == 0) { 5775f225124SSasha Levin p->guest_ip = strdup(val); 5785f225124SSasha Levin } else if (strcmp(param, "host_ip") == 0) { 5795f225124SSasha Levin p->host_ip = strdup(val); 5805f225124SSasha Levin } else if (strcmp(param, "trans") == 0) { 5815f225124SSasha Levin p->trans = strdup(val); 5825f225124SSasha Levin } else if (strcmp(param, "vhost") == 0) { 5835f225124SSasha Levin p->vhost = atoi(val); 5845f225124SSasha Levin } else if (strcmp(param, "fd") == 0) { 5855f225124SSasha Levin p->fd = atoi(val); 58679052597SSasha Levin } else if (strcmp(param, "mq") == 0) { 58779052597SSasha Levin p->mq = atoi(val); 5885f225124SSasha Levin } else 5895f225124SSasha Levin die("Unknown network parameter %s", param); 5905f225124SSasha Levin 5915f225124SSasha Levin return 0; 5925f225124SSasha Levin } 5935f225124SSasha Levin 5945f225124SSasha Levin int netdev_parser(const struct option *opt, const char *arg, int unset) 5955f225124SSasha Levin { 5965f225124SSasha Levin struct virtio_net_params p; 5975f225124SSasha Levin char *buf = NULL, *cmd = NULL, *cur = NULL; 5985f225124SSasha Levin bool on_cmd = true; 5995f225124SSasha Levin struct kvm *kvm = opt->ptr; 6005f225124SSasha Levin 6015f225124SSasha Levin if (arg) { 6025f225124SSasha Levin buf = strdup(arg); 6035f225124SSasha Levin if (buf == NULL) 6045f225124SSasha Levin die("Failed allocating new net buffer"); 6055f225124SSasha Levin cur = strtok(buf, ",="); 6065f225124SSasha Levin } 6075f225124SSasha Levin 6085f225124SSasha Levin p = (struct virtio_net_params) { 6095f225124SSasha Levin .guest_ip = DEFAULT_GUEST_ADDR, 6105f225124SSasha Levin .host_ip = DEFAULT_HOST_ADDR, 6115f225124SSasha Levin .script = DEFAULT_SCRIPT, 6125f225124SSasha Levin .mode = NET_MODE_TAP, 6135f225124SSasha Levin }; 6145f225124SSasha Levin 6155f225124SSasha Levin str_to_mac(DEFAULT_GUEST_MAC, p.guest_mac); 6165f225124SSasha Levin p.guest_mac[5] += kvm->cfg.num_net_devices; 6175f225124SSasha Levin 6185f225124SSasha Levin while (cur) { 6195f225124SSasha Levin if (on_cmd) { 6205f225124SSasha Levin cmd = cur; 6215f225124SSasha Levin } else { 6224346fd8fSSasha Levin if (set_net_param(kvm, &p, cmd, cur) < 0) 6235f225124SSasha Levin goto done; 6245f225124SSasha Levin } 6255f225124SSasha Levin on_cmd = !on_cmd; 6265f225124SSasha Levin 6275f225124SSasha Levin cur = strtok(NULL, ",="); 6285f225124SSasha Levin }; 6295f225124SSasha Levin 6305f225124SSasha Levin kvm->cfg.num_net_devices++; 6315f225124SSasha Levin 6325f225124SSasha Levin kvm->cfg.net_params = realloc(kvm->cfg.net_params, kvm->cfg.num_net_devices * sizeof(*kvm->cfg.net_params)); 6335f225124SSasha Levin if (kvm->cfg.net_params == NULL) 6345f225124SSasha Levin die("Failed adding new network device"); 6355f225124SSasha Levin 6365f225124SSasha Levin kvm->cfg.net_params[kvm->cfg.num_net_devices - 1] = p; 6375f225124SSasha Levin 6385f225124SSasha Levin done: 6395f225124SSasha Levin free(buf); 6405f225124SSasha Levin return 0; 6415f225124SSasha Levin } 6425f225124SSasha Levin 6435f225124SSasha Levin static int virtio_net__init_one(struct virtio_net_params *params) 6444f56d42cSAsias He { 645b5ee1ea7SAsias He int i; 6469a6d73f1SSasha Levin struct net_dev *ndev; 6479a6d73f1SSasha Levin 6489a6d73f1SSasha Levin ndev = calloc(1, sizeof(struct net_dev)); 6499a6d73f1SSasha Levin if (ndev == NULL) 6505f225124SSasha Levin return -ENOMEM; 6519a6d73f1SSasha Levin 6529a6d73f1SSasha Levin list_add_tail(&ndev->list, &ndevs); 6539a6d73f1SSasha Levin 6549a6d73f1SSasha Levin ndev->kvm = params->kvm; 6559a6d73f1SSasha Levin 6569a6d73f1SSasha Levin mutex_init(&ndev->mutex); 65779052597SSasha Levin ndev->queue_pairs = max(1, min(VIRTIO_NET_NUM_QUEUES, params->mq)); 6589a6d73f1SSasha Levin ndev->config.status = VIRTIO_NET_S_LINK_UP; 65979052597SSasha Levin if (ndev->queue_pairs > 1) 66079052597SSasha Levin ndev->config.max_virtqueue_pairs = ndev->queue_pairs; 66179052597SSasha Levin 6620c54698eSAsias He for (i = 0 ; i < 6 ; i++) { 6639a6d73f1SSasha Levin ndev->config.mac[i] = params->guest_mac[i]; 6649a6d73f1SSasha Levin ndev->info.guest_mac.addr[i] = params->guest_mac[i]; 6659a6d73f1SSasha Levin ndev->info.host_mac.addr[i] = params->host_mac[i]; 6660c54698eSAsias He } 667f715177dSAsias He 6689a6d73f1SSasha Levin ndev->mode = params->mode; 6699a6d73f1SSasha Levin if (ndev->mode == NET_MODE_TAP) { 6709a6d73f1SSasha Levin if (!virtio_net__tap_init(params, ndev)) 671db051127SMichael Ellerman die_perror("You have requested a TAP device, but creation of one has failed because"); 6729a6d73f1SSasha Levin ndev->ops = &tap_ops; 673b4fdde6dSAsias He } else { 6749a6d73f1SSasha Levin ndev->info.host_ip = ntohl(inet_addr(params->host_ip)); 6759a6d73f1SSasha Levin ndev->info.guest_ip = ntohl(inet_addr(params->guest_ip)); 6769a6d73f1SSasha Levin ndev->info.guest_netmask = ntohl(inet_addr("255.255.255.0")); 6779a6d73f1SSasha Levin ndev->info.buf_nr = 20, 678643f6c08SSasha Levin ndev->info.vnet_hdr_len = sizeof(struct virtio_net_hdr); 6799a6d73f1SSasha Levin uip_init(&ndev->info); 6809a6d73f1SSasha Levin ndev->ops = &uip_ops; 681b4fdde6dSAsias He } 682b5ee1ea7SAsias He 68369205aa1SAsias He if (params->trans && strcmp(params->trans, "mmio") == 0) 6844346fd8fSSasha Levin virtio_init(params->kvm, ndev, &ndev->vdev, &net_dev_virtio_ops, 68569205aa1SAsias He VIRTIO_MMIO, PCI_DEVICE_ID_VIRTIO_NET, VIRTIO_ID_NET, PCI_CLASS_NET); 68669205aa1SAsias He else 6874346fd8fSSasha Levin virtio_init(params->kvm, ndev, &ndev->vdev, &net_dev_virtio_ops, 68802eca50cSAsias He VIRTIO_PCI, PCI_DEVICE_ID_VIRTIO_NET, VIRTIO_ID_NET, PCI_CLASS_NET); 68927ab67f5SSasha Levin 6909ed67cdcSSasha Levin if (params->vhost) 6919ed67cdcSSasha Levin virtio_net__vhost_init(params->kvm, ndev); 692cb83de6fSSasha Levin 693d278197dSAsias He if (compat_id == -1) 69452f34d2cSAsias He compat_id = virtio_compat_add_message("virtio-net", "CONFIG_VIRTIO_NET"); 6955f225124SSasha Levin 6965f225124SSasha Levin return 0; 6975f225124SSasha Levin } 6985f225124SSasha Levin 6995f225124SSasha Levin int virtio_net__init(struct kvm *kvm) 7005f225124SSasha Levin { 7015f225124SSasha Levin int i; 7025f225124SSasha Levin 7035f225124SSasha Levin for (i = 0; i < kvm->cfg.num_net_devices; i++) { 7045f225124SSasha Levin kvm->cfg.net_params[i].kvm = kvm; 7055f225124SSasha Levin virtio_net__init_one(&kvm->cfg.net_params[i]); 7065f225124SSasha Levin } 7075f225124SSasha Levin 7085f225124SSasha Levin if (kvm->cfg.num_net_devices == 0 && kvm->cfg.no_net == 0) { 7095f225124SSasha Levin struct virtio_net_params net_params; 7105f225124SSasha Levin 7115f225124SSasha Levin net_params = (struct virtio_net_params) { 7125f225124SSasha Levin .guest_ip = kvm->cfg.guest_ip, 7135f225124SSasha Levin .host_ip = kvm->cfg.host_ip, 7145f225124SSasha Levin .kvm = kvm, 7155f225124SSasha Levin .script = kvm->cfg.script, 7165f225124SSasha Levin .mode = NET_MODE_USER, 7175f225124SSasha Levin }; 7185f225124SSasha Levin str_to_mac(kvm->cfg.guest_mac, net_params.guest_mac); 7195f225124SSasha Levin str_to_mac(kvm->cfg.host_mac, net_params.host_mac); 7205f225124SSasha Levin 7215f225124SSasha Levin virtio_net__init_one(&net_params); 7225f225124SSasha Levin } 7235f225124SSasha Levin 7245f225124SSasha Levin return 0; 7255f225124SSasha Levin } 72649a8afd1SSasha Levin virtio_dev_init(virtio_net__init); 7275f225124SSasha Levin 7285f225124SSasha Levin int virtio_net__exit(struct kvm *kvm) 7295f225124SSasha Levin { 7305f225124SSasha Levin return 0; 7314f56d42cSAsias He } 73249a8afd1SSasha Levin virtio_dev_exit(virtio_net__exit); 733