131638bcaSCyrill Gorcunov #include "kvm/virtio-pci-dev.h" 2b5ee1ea7SAsias He #include "kvm/virtio-net.h" 34f56d42cSAsias He #include "kvm/virtio.h" 44f56d42cSAsias He #include "kvm/types.h" 54f56d42cSAsias He #include "kvm/mutex.h" 64f56d42cSAsias He #include "kvm/util.h" 74f56d42cSAsias He #include "kvm/kvm.h" 82449f6e3SSasha Levin #include "kvm/irq.h" 9b5ee1ea7SAsias He #include "kvm/uip.h" 10cb83de6fSSasha Levin #include "kvm/guest_compat.h" 114f56d42cSAsias He 129ed67cdcSSasha Levin #include <linux/vhost.h> 134f56d42cSAsias He #include <linux/virtio_net.h> 144f56d42cSAsias He #include <linux/if_tun.h> 1578a2a3e2SSasha Levin #include <linux/types.h> 16c229370aSIngo Molnar 17c229370aSIngo Molnar #include <arpa/inet.h> 184f56d42cSAsias He #include <net/if.h> 19c229370aSIngo Molnar 20c229370aSIngo Molnar #include <unistd.h> 214f56d42cSAsias He #include <fcntl.h> 22c229370aSIngo Molnar 23cb7202c1SSasha Levin #include <sys/socket.h> 24c229370aSIngo Molnar #include <sys/ioctl.h> 25c229370aSIngo Molnar #include <sys/types.h> 2673b7d038SAmos Kong #include <sys/wait.h> 279ed67cdcSSasha Levin #include <sys/eventfd.h> 284f56d42cSAsias He 294059ad8bSAsias He #define VIRTIO_NET_QUEUE_SIZE 256 306585ffb7SSasha Levin #define VIRTIO_NET_NUM_QUEUES 8 314f56d42cSAsias He 32b4fdde6dSAsias He struct net_dev; 33b4fdde6dSAsias He 34b4fdde6dSAsias He struct net_dev_operations { 35b4fdde6dSAsias He int (*rx)(struct iovec *iov, u16 in, struct net_dev *ndev); 36b4fdde6dSAsias He int (*tx)(struct iovec *iov, u16 in, struct net_dev *ndev); 37b4fdde6dSAsias He }; 38b4fdde6dSAsias He 398626798bSAsias He struct net_dev { 40d3476f7dSSasha Levin struct mutex mutex; 4102eca50cSAsias He struct virtio_device vdev; 429a6d73f1SSasha Levin struct list_head list; 434f56d42cSAsias He 446585ffb7SSasha Levin struct virt_queue vqs[VIRTIO_NET_NUM_QUEUES * 2 + 1]; 45c229370aSIngo Molnar struct virtio_net_config config; 46*79052597SSasha Levin u32 features, rx_vqs, tx_vqs, queue_pairs; 474f56d42cSAsias He 486585ffb7SSasha Levin pthread_t io_thread[VIRTIO_NET_NUM_QUEUES * 2 + 1]; 496585ffb7SSasha Levin struct mutex io_lock[VIRTIO_NET_NUM_QUEUES * 2 + 1]; 506585ffb7SSasha Levin pthread_cond_t io_cond[VIRTIO_NET_NUM_QUEUES * 2 + 1]; 51c4aa7c02SPekka Enberg 529ed67cdcSSasha Levin int vhost_fd; 534f56d42cSAsias He int tap_fd; 544f56d42cSAsias He char tap_name[IFNAMSIZ]; 55bb1a32f1SAsias He 56bb1a32f1SAsias He int mode; 57bb1a32f1SAsias He 58b5ee1ea7SAsias He struct uip_info info; 59b4fdde6dSAsias He struct net_dev_operations *ops; 609a6d73f1SSasha Levin struct kvm *kvm; 614f56d42cSAsias He }; 624f56d42cSAsias He 639a6d73f1SSasha Levin static LIST_HEAD(ndevs); 64312c62d1SSasha Levin static int compat_id = -1; 654f56d42cSAsias He 66c4aa7c02SPekka Enberg static void *virtio_net_rx_thread(void *p) 674f56d42cSAsias He { 684f56d42cSAsias He struct iovec iov[VIRTIO_NET_QUEUE_SIZE]; 694f56d42cSAsias He struct virt_queue *vq; 7043835ac9SSasha Levin struct kvm *kvm; 719a6d73f1SSasha Levin struct net_dev *ndev = p; 723fdf659dSSasha Levin u16 out, in; 733fdf659dSSasha Levin u16 head; 744f56d42cSAsias He int len; 756585ffb7SSasha Levin u32 id; 766585ffb7SSasha Levin 776585ffb7SSasha Levin mutex_lock(&ndev->mutex); 786585ffb7SSasha Levin id = ndev->rx_vqs++ * 2; 796585ffb7SSasha Levin mutex_unlock(&ndev->mutex); 804f56d42cSAsias He 81a4d8c55eSSasha Levin kvm__set_thread_name("virtio-net-rx"); 82a4d8c55eSSasha Levin 839a6d73f1SSasha Levin kvm = ndev->kvm; 846585ffb7SSasha Levin vq = &ndev->vqs[id]; 85c4aa7c02SPekka Enberg 86c4aa7c02SPekka Enberg while (1) { 876585ffb7SSasha Levin mutex_lock(&ndev->io_lock[id]); 88c4aa7c02SPekka Enberg if (!virt_queue__available(vq)) 896585ffb7SSasha Levin pthread_cond_wait(&ndev->io_cond[id], &ndev->io_lock[id].mutex); 906585ffb7SSasha Levin mutex_unlock(&ndev->io_lock[id]); 914f56d42cSAsias He 924f56d42cSAsias He while (virt_queue__available(vq)) { 9343835ac9SSasha Levin head = virt_queue__get_iov(vq, iov, &out, &in, kvm); 949a6d73f1SSasha Levin len = ndev->ops->rx(iov, in, ndev); 95246c8347SAsias He virt_queue__set_used_elem(vq, head, len); 967f5ffaf5SAsias He 97c4aa7c02SPekka Enberg /* We should interrupt guest right now, otherwise latency is huge. */ 986585ffb7SSasha Levin if (virtio_queue__should_signal(vq)) 996585ffb7SSasha Levin ndev->vdev.ops->signal_vq(kvm, &ndev->vdev, id); 1004f56d42cSAsias He } 101c4aa7c02SPekka Enberg } 102c4aa7c02SPekka Enberg 103c4aa7c02SPekka Enberg pthread_exit(NULL); 104c4aa7c02SPekka Enberg return NULL; 105c4aa7c02SPekka Enberg 106c4aa7c02SPekka Enberg } 107c4aa7c02SPekka Enberg 108c4aa7c02SPekka Enberg static void *virtio_net_tx_thread(void *p) 1094f56d42cSAsias He { 1104f56d42cSAsias He struct iovec iov[VIRTIO_NET_QUEUE_SIZE]; 1114f56d42cSAsias He struct virt_queue *vq; 11243835ac9SSasha Levin struct kvm *kvm; 1139a6d73f1SSasha Levin struct net_dev *ndev = p; 1143fdf659dSSasha Levin u16 out, in; 1153fdf659dSSasha Levin u16 head; 1164f56d42cSAsias He int len; 1176585ffb7SSasha Levin u32 id; 1186585ffb7SSasha Levin 1196585ffb7SSasha Levin mutex_lock(&ndev->mutex); 1206585ffb7SSasha Levin id = ndev->tx_vqs++ * 2 + 1; 1216585ffb7SSasha Levin mutex_unlock(&ndev->mutex); 1224f56d42cSAsias He 123a4d8c55eSSasha Levin kvm__set_thread_name("virtio-net-tx"); 124a4d8c55eSSasha Levin 1259a6d73f1SSasha Levin kvm = ndev->kvm; 1266585ffb7SSasha Levin vq = &ndev->vqs[id]; 127c4aa7c02SPekka Enberg 128c4aa7c02SPekka Enberg while (1) { 1296585ffb7SSasha Levin mutex_lock(&ndev->io_lock[id]); 130c4aa7c02SPekka Enberg if (!virt_queue__available(vq)) 1316585ffb7SSasha Levin pthread_cond_wait(&ndev->io_cond[id], &ndev->io_lock[id].mutex); 1326585ffb7SSasha Levin mutex_unlock(&ndev->io_lock[id]); 1334f56d42cSAsias He 1344f56d42cSAsias He while (virt_queue__available(vq)) { 13543835ac9SSasha Levin head = virt_queue__get_iov(vq, iov, &out, &in, kvm); 1369a6d73f1SSasha Levin len = ndev->ops->tx(iov, out, ndev); 1374f56d42cSAsias He virt_queue__set_used_elem(vq, head, len); 1384f56d42cSAsias He } 1394f56d42cSAsias He 1406585ffb7SSasha Levin if (virtio_queue__should_signal(vq)) 1416585ffb7SSasha Levin ndev->vdev.ops->signal_vq(kvm, &ndev->vdev, id); 1424f56d42cSAsias He } 1434f56d42cSAsias He 144c4aa7c02SPekka Enberg pthread_exit(NULL); 145407475bfSPekka Enberg 146c4aa7c02SPekka Enberg return NULL; 147c4aa7c02SPekka Enberg 148c4aa7c02SPekka Enberg } 149407475bfSPekka Enberg 1506585ffb7SSasha Levin static virtio_net_ctrl_ack virtio_net_handle_mq(struct kvm* kvm, struct net_dev *ndev, struct virtio_net_ctrl_hdr *ctrl) 1516585ffb7SSasha Levin { 1526585ffb7SSasha Levin /* Not much to do here */ 1536585ffb7SSasha Levin return VIRTIO_NET_OK; 1546585ffb7SSasha Levin } 1556585ffb7SSasha Levin 1566585ffb7SSasha Levin static void *virtio_net_ctrl_thread(void *p) 157abd4a801SSasha Levin { 158abd4a801SSasha Levin struct iovec iov[VIRTIO_NET_QUEUE_SIZE]; 159abd4a801SSasha Levin u16 out, in, head; 1606585ffb7SSasha Levin struct net_dev *ndev = p; 1616585ffb7SSasha Levin struct kvm *kvm = ndev->kvm; 162*79052597SSasha Levin u32 id = ndev->queue_pairs * 2; 1636585ffb7SSasha Levin struct virt_queue *vq = &ndev->vqs[id]; 164abd4a801SSasha Levin struct virtio_net_ctrl_hdr *ctrl; 165abd4a801SSasha Levin virtio_net_ctrl_ack *ack; 166abd4a801SSasha Levin 1676585ffb7SSasha Levin while (1) { 1686585ffb7SSasha Levin mutex_lock(&ndev->io_lock[id]); 1696585ffb7SSasha Levin if (!virt_queue__available(vq)) 1706585ffb7SSasha Levin pthread_cond_wait(&ndev->io_cond[id], &ndev->io_lock[id].mutex); 1716585ffb7SSasha Levin mutex_unlock(&ndev->io_lock[id]); 1726585ffb7SSasha Levin 1736585ffb7SSasha Levin while (virt_queue__available(vq)) { 1746585ffb7SSasha Levin head = virt_queue__get_iov(&ndev->vqs[id], iov, &out, &in, kvm); 175abd4a801SSasha Levin ctrl = iov[0].iov_base; 176abd4a801SSasha Levin ack = iov[out].iov_base; 177abd4a801SSasha Levin 178abd4a801SSasha Levin switch (ctrl->class) { 1796585ffb7SSasha Levin case VIRTIO_NET_CTRL_MQ: 1806585ffb7SSasha Levin *ack = virtio_net_handle_mq(kvm, ndev, ctrl); 1816585ffb7SSasha Levin break; 182abd4a801SSasha Levin default: 183abd4a801SSasha Levin *ack = VIRTIO_NET_ERR; 184abd4a801SSasha Levin break; 185abd4a801SSasha Levin } 1866585ffb7SSasha Levin virt_queue__set_used_elem(&ndev->vqs[id], head, iov[out].iov_len); 1876585ffb7SSasha Levin } 188abd4a801SSasha Levin 1896585ffb7SSasha Levin if (virtio_queue__should_signal(&ndev->vqs[id])) 1906585ffb7SSasha Levin ndev->vdev.ops->signal_vq(kvm, &ndev->vdev, id); 1916585ffb7SSasha Levin } 192abd4a801SSasha Levin 1936585ffb7SSasha Levin pthread_exit(NULL); 194abd4a801SSasha Levin 1956585ffb7SSasha Levin return NULL; 196abd4a801SSasha Levin } 197abd4a801SSasha Levin 1989a6d73f1SSasha Levin static void virtio_net_handle_callback(struct kvm *kvm, struct net_dev *ndev, int queue) 1994f56d42cSAsias He { 200*79052597SSasha Levin if ((u32)queue >= (ndev->queue_pairs * 2 + 1)) { 2019a6d73f1SSasha Levin pr_warning("Unknown queue index %u", queue); 2026585ffb7SSasha Levin return; 203c4aa7c02SPekka Enberg } 2046585ffb7SSasha Levin 2056585ffb7SSasha Levin mutex_lock(&ndev->io_lock[queue]); 2066585ffb7SSasha Levin pthread_cond_signal(&ndev->io_cond[queue]); 2076585ffb7SSasha Levin mutex_unlock(&ndev->io_lock[queue]); 2084f56d42cSAsias He } 2094f56d42cSAsias He 2109a6d73f1SSasha Levin static bool virtio_net__tap_init(const struct virtio_net_params *params, 2119a6d73f1SSasha Levin struct net_dev *ndev) 2124f56d42cSAsias He { 213cb7202c1SSasha Levin int sock = socket(AF_INET, SOCK_STREAM, 0); 214f715177dSAsias He int pid, status, offload, hdr_len; 215cb7202c1SSasha Levin struct sockaddr_in sin = {0}; 216246c8347SAsias He struct ifreq ifr; 2174f56d42cSAsias He 218f19edd1eSSasha Levin /* Did the user already gave us the FD? */ 219f19edd1eSSasha Levin if (params->fd) { 220f19edd1eSSasha Levin ndev->tap_fd = params->fd; 221f19edd1eSSasha Levin return 1; 222f19edd1eSSasha Levin } 223f19edd1eSSasha Levin 2249a6d73f1SSasha Levin ndev->tap_fd = open("/dev/net/tun", O_RDWR); 2259a6d73f1SSasha Levin if (ndev->tap_fd < 0) { 2264542f276SCyrill Gorcunov pr_warning("Unable to open /dev/net/tun"); 2273b02f580SSasha Levin goto fail; 2283b02f580SSasha Levin } 2294f56d42cSAsias He 2304f56d42cSAsias He memset(&ifr, 0, sizeof(ifr)); 231246c8347SAsias He ifr.ifr_flags = IFF_TAP | IFF_NO_PI | IFF_VNET_HDR; 2329a6d73f1SSasha Levin if (ioctl(ndev->tap_fd, TUNSETIFF, &ifr) < 0) { 2334542f276SCyrill Gorcunov pr_warning("Config tap device error. Are you root?"); 2343b02f580SSasha Levin goto fail; 2353b02f580SSasha Levin } 2364f56d42cSAsias He 2379a6d73f1SSasha Levin strncpy(ndev->tap_name, ifr.ifr_name, sizeof(ndev->tap_name)); 2384f56d42cSAsias He 2399a6d73f1SSasha Levin if (ioctl(ndev->tap_fd, TUNSETNOCSUM, 1) < 0) { 2404542f276SCyrill Gorcunov pr_warning("Config tap device TUNSETNOCSUM error"); 241246c8347SAsias He goto fail; 242246c8347SAsias He } 243246c8347SAsias He 244246c8347SAsias He hdr_len = sizeof(struct virtio_net_hdr); 2459a6d73f1SSasha Levin if (ioctl(ndev->tap_fd, TUNSETVNETHDRSZ, &hdr_len) < 0) 2464542f276SCyrill Gorcunov pr_warning("Config tap device TUNSETVNETHDRSZ error"); 247246c8347SAsias He 248246c8347SAsias He offload = TUN_F_CSUM | TUN_F_TSO4 | TUN_F_TSO6 | TUN_F_UFO; 2499a6d73f1SSasha Levin if (ioctl(ndev->tap_fd, TUNSETOFFLOAD, offload) < 0) { 2504542f276SCyrill Gorcunov pr_warning("Config tap device TUNSETOFFLOAD error"); 251246c8347SAsias He goto fail; 252246c8347SAsias He } 2534f56d42cSAsias He 25473b7d038SAmos Kong if (strcmp(params->script, "none")) { 25573b7d038SAmos Kong pid = fork(); 25673b7d038SAmos Kong if (pid == 0) { 2579a6d73f1SSasha Levin execl(params->script, params->script, ndev->tap_name, NULL); 25873b7d038SAmos Kong _exit(1); 25973b7d038SAmos Kong } else { 26073b7d038SAmos Kong waitpid(pid, &status, 0); 26173b7d038SAmos Kong if (WIFEXITED(status) && WEXITSTATUS(status) != 0) { 2624542f276SCyrill Gorcunov pr_warning("Fail to setup tap by %s", params->script); 26373b7d038SAmos Kong goto fail; 26473b7d038SAmos Kong } 26573b7d038SAmos Kong } 26673b7d038SAmos Kong } else { 267cb7202c1SSasha Levin memset(&ifr, 0, sizeof(ifr)); 2689a6d73f1SSasha Levin strncpy(ifr.ifr_name, ndev->tap_name, sizeof(ndev->tap_name)); 269bdfcfca6SSasha Levin sin.sin_addr.s_addr = inet_addr(params->host_ip); 270cb7202c1SSasha Levin memcpy(&(ifr.ifr_addr), &sin, sizeof(ifr.ifr_addr)); 271cb7202c1SSasha Levin ifr.ifr_addr.sa_family = AF_INET; 2723b02f580SSasha Levin if (ioctl(sock, SIOCSIFADDR, &ifr) < 0) { 2734542f276SCyrill Gorcunov pr_warning("Could not set ip address on tap device"); 2743b02f580SSasha Levin goto fail; 2753b02f580SSasha Levin } 27673b7d038SAmos Kong } 277cb7202c1SSasha Levin 278cb7202c1SSasha Levin memset(&ifr, 0, sizeof(ifr)); 2799a6d73f1SSasha Levin strncpy(ifr.ifr_name, ndev->tap_name, sizeof(ndev->tap_name)); 280cb7202c1SSasha Levin ioctl(sock, SIOCGIFFLAGS, &ifr); 281cb7202c1SSasha Levin ifr.ifr_flags |= IFF_UP | IFF_RUNNING; 282cb7202c1SSasha Levin if (ioctl(sock, SIOCSIFFLAGS, &ifr) < 0) 2834542f276SCyrill Gorcunov pr_warning("Could not bring tap device up"); 284cb7202c1SSasha Levin 285cb7202c1SSasha Levin close(sock); 2863b02f580SSasha Levin 2873b02f580SSasha Levin return 1; 2883b02f580SSasha Levin 2893b02f580SSasha Levin fail: 2903b02f580SSasha Levin if (sock >= 0) 2913b02f580SSasha Levin close(sock); 2929a6d73f1SSasha Levin if (ndev->tap_fd >= 0) 2939a6d73f1SSasha Levin close(ndev->tap_fd); 2943b02f580SSasha Levin 2953b02f580SSasha Levin return 0; 2964f56d42cSAsias He } 2974f56d42cSAsias He 298b4fdde6dSAsias He static inline int tap_ops_tx(struct iovec *iov, u16 out, struct net_dev *ndev) 299b4fdde6dSAsias He { 300b4fdde6dSAsias He return writev(ndev->tap_fd, iov, out); 301b4fdde6dSAsias He } 302b4fdde6dSAsias He 303b4fdde6dSAsias He static inline int tap_ops_rx(struct iovec *iov, u16 in, struct net_dev *ndev) 304b4fdde6dSAsias He { 305b4fdde6dSAsias He return readv(ndev->tap_fd, iov, in); 306b4fdde6dSAsias He } 307b4fdde6dSAsias He 308b4fdde6dSAsias He static inline int uip_ops_tx(struct iovec *iov, u16 out, struct net_dev *ndev) 309b4fdde6dSAsias He { 310b4fdde6dSAsias He return uip_tx(iov, out, &ndev->info); 311b4fdde6dSAsias He } 312b4fdde6dSAsias He 313b4fdde6dSAsias He static inline int uip_ops_rx(struct iovec *iov, u16 in, struct net_dev *ndev) 314b4fdde6dSAsias He { 315b4fdde6dSAsias He return uip_rx(iov, in, &ndev->info); 316b4fdde6dSAsias He } 317b4fdde6dSAsias He 318b4fdde6dSAsias He static struct net_dev_operations tap_ops = { 319b4fdde6dSAsias He .rx = tap_ops_rx, 320b4fdde6dSAsias He .tx = tap_ops_tx, 321b4fdde6dSAsias He }; 322b4fdde6dSAsias He 323b4fdde6dSAsias He static struct net_dev_operations uip_ops = { 324b4fdde6dSAsias He .rx = uip_ops_rx, 325b4fdde6dSAsias He .tx = uip_ops_tx, 326b4fdde6dSAsias He }; 327b4fdde6dSAsias He 328c5ae742bSSasha Levin static u8 *get_config(struct kvm *kvm, void *dev) 32978a2a3e2SSasha Levin { 33078a2a3e2SSasha Levin struct net_dev *ndev = dev; 33178a2a3e2SSasha Levin 332c5ae742bSSasha Levin return ((u8 *)(&ndev->config)); 33378a2a3e2SSasha Levin } 33478a2a3e2SSasha Levin 33578a2a3e2SSasha Levin static u32 get_host_features(struct kvm *kvm, void *dev) 33678a2a3e2SSasha Levin { 337*79052597SSasha Levin struct net_dev *ndev = dev; 338*79052597SSasha Levin 33978a2a3e2SSasha Levin return 1UL << VIRTIO_NET_F_MAC 34078a2a3e2SSasha Levin | 1UL << VIRTIO_NET_F_CSUM 34178a2a3e2SSasha Levin | 1UL << VIRTIO_NET_F_HOST_UFO 34278a2a3e2SSasha Levin | 1UL << VIRTIO_NET_F_HOST_TSO4 34378a2a3e2SSasha Levin | 1UL << VIRTIO_NET_F_HOST_TSO6 34478a2a3e2SSasha Levin | 1UL << VIRTIO_NET_F_GUEST_UFO 34578a2a3e2SSasha Levin | 1UL << VIRTIO_NET_F_GUEST_TSO4 34692c1f37bSSasha Levin | 1UL << VIRTIO_NET_F_GUEST_TSO6 347754c8ce3SSasha Levin | 1UL << VIRTIO_RING_F_EVENT_IDX 348abd4a801SSasha Levin | 1UL << VIRTIO_RING_F_INDIRECT_DESC 3496585ffb7SSasha Levin | 1UL << VIRTIO_NET_F_CTRL_VQ 350*79052597SSasha Levin | 1UL << (ndev->queue_pairs > 1 ? VIRTIO_NET_F_MQ : 0); 35178a2a3e2SSasha Levin } 35278a2a3e2SSasha Levin 35378a2a3e2SSasha Levin static void set_guest_features(struct kvm *kvm, void *dev, u32 features) 35478a2a3e2SSasha Levin { 35578a2a3e2SSasha Levin struct net_dev *ndev = dev; 35678a2a3e2SSasha Levin 35778a2a3e2SSasha Levin ndev->features = features; 35878a2a3e2SSasha Levin } 35978a2a3e2SSasha Levin 360c59ba304SWill Deacon static int init_vq(struct kvm *kvm, void *dev, u32 vq, u32 page_size, u32 align, 361c59ba304SWill Deacon u32 pfn) 36278a2a3e2SSasha Levin { 3639ed67cdcSSasha Levin struct vhost_vring_state state = { .index = vq }; 3649ed67cdcSSasha Levin struct vhost_vring_addr addr; 36578a2a3e2SSasha Levin struct net_dev *ndev = dev; 36678a2a3e2SSasha Levin struct virt_queue *queue; 36778a2a3e2SSasha Levin void *p; 3689ed67cdcSSasha Levin int r; 36978a2a3e2SSasha Levin 370312c62d1SSasha Levin compat__remove_message(compat_id); 37178a2a3e2SSasha Levin 37278a2a3e2SSasha Levin queue = &ndev->vqs[vq]; 37378a2a3e2SSasha Levin queue->pfn = pfn; 374c59ba304SWill Deacon p = guest_flat_to_host(kvm, queue->pfn * page_size); 37578a2a3e2SSasha Levin 376c59ba304SWill Deacon vring_init(&queue->vring, VIRTIO_NET_QUEUE_SIZE, p, align); 37778a2a3e2SSasha Levin 3786585ffb7SSasha Levin mutex_init(&ndev->io_lock[vq]); 3796585ffb7SSasha Levin pthread_cond_init(&ndev->io_cond[vq], NULL); 3806585ffb7SSasha Levin if (ndev->vhost_fd == 0) { 381*79052597SSasha Levin if (vq == (u32)(ndev->queue_pairs * 2)) 3826585ffb7SSasha Levin pthread_create(&ndev->io_thread[vq], NULL, virtio_net_ctrl_thread, ndev); 3836585ffb7SSasha Levin else if (vq & 1) 3846585ffb7SSasha Levin pthread_create(&ndev->io_thread[vq], NULL, virtio_net_tx_thread, ndev); 3856585ffb7SSasha Levin else 3866585ffb7SSasha Levin pthread_create(&ndev->io_thread[vq], NULL, virtio_net_rx_thread, ndev); 3876585ffb7SSasha Levin 38878a2a3e2SSasha Levin return 0; 3896585ffb7SSasha Levin } 3909ed67cdcSSasha Levin 3919ed67cdcSSasha Levin state.num = queue->vring.num; 3929ed67cdcSSasha Levin r = ioctl(ndev->vhost_fd, VHOST_SET_VRING_NUM, &state); 3939ed67cdcSSasha Levin if (r < 0) 3949ed67cdcSSasha Levin die_perror("VHOST_SET_VRING_NUM failed"); 3959ed67cdcSSasha Levin state.num = 0; 3969ed67cdcSSasha Levin r = ioctl(ndev->vhost_fd, VHOST_SET_VRING_BASE, &state); 3979ed67cdcSSasha Levin if (r < 0) 3989ed67cdcSSasha Levin die_perror("VHOST_SET_VRING_BASE failed"); 3999ed67cdcSSasha Levin 4009ed67cdcSSasha Levin addr = (struct vhost_vring_addr) { 4019ed67cdcSSasha Levin .index = vq, 4029ed67cdcSSasha Levin .desc_user_addr = (u64)(unsigned long)queue->vring.desc, 4039ed67cdcSSasha Levin .avail_user_addr = (u64)(unsigned long)queue->vring.avail, 4049ed67cdcSSasha Levin .used_user_addr = (u64)(unsigned long)queue->vring.used, 4059ed67cdcSSasha Levin }; 4069ed67cdcSSasha Levin 4079ed67cdcSSasha Levin r = ioctl(ndev->vhost_fd, VHOST_SET_VRING_ADDR, &addr); 4089ed67cdcSSasha Levin if (r < 0) 4099ed67cdcSSasha Levin die_perror("VHOST_SET_VRING_ADDR failed"); 4109ed67cdcSSasha Levin 4119ed67cdcSSasha Levin return 0; 4129ed67cdcSSasha Levin } 4139ed67cdcSSasha Levin 4149ed67cdcSSasha Levin static void notify_vq_gsi(struct kvm *kvm, void *dev, u32 vq, u32 gsi) 4159ed67cdcSSasha Levin { 4169ed67cdcSSasha Levin struct net_dev *ndev = dev; 4179ed67cdcSSasha Levin struct kvm_irqfd irq; 4189ed67cdcSSasha Levin struct vhost_vring_file file; 4199ed67cdcSSasha Levin int r; 4209ed67cdcSSasha Levin 4219ed67cdcSSasha Levin if (ndev->vhost_fd == 0) 4229ed67cdcSSasha Levin return; 4239ed67cdcSSasha Levin 4249ed67cdcSSasha Levin irq = (struct kvm_irqfd) { 4259ed67cdcSSasha Levin .gsi = gsi, 4269ed67cdcSSasha Levin .fd = eventfd(0, 0), 4279ed67cdcSSasha Levin }; 4289ed67cdcSSasha Levin file = (struct vhost_vring_file) { 4299ed67cdcSSasha Levin .index = vq, 4309ed67cdcSSasha Levin .fd = irq.fd, 4319ed67cdcSSasha Levin }; 4329ed67cdcSSasha Levin 4339ed67cdcSSasha Levin r = ioctl(kvm->vm_fd, KVM_IRQFD, &irq); 4349ed67cdcSSasha Levin if (r < 0) 4359ed67cdcSSasha Levin die_perror("KVM_IRQFD failed"); 4369ed67cdcSSasha Levin 4379ed67cdcSSasha Levin r = ioctl(ndev->vhost_fd, VHOST_SET_VRING_CALL, &file); 4389ed67cdcSSasha Levin if (r < 0) 4399ed67cdcSSasha Levin die_perror("VHOST_SET_VRING_CALL failed"); 4409ed67cdcSSasha Levin file.fd = ndev->tap_fd; 4419ed67cdcSSasha Levin r = ioctl(ndev->vhost_fd, VHOST_NET_SET_BACKEND, &file); 4429ed67cdcSSasha Levin if (r != 0) 4439ed67cdcSSasha Levin die("VHOST_NET_SET_BACKEND failed %d", errno); 4449ed67cdcSSasha Levin 4459ed67cdcSSasha Levin } 4469ed67cdcSSasha Levin 4479ed67cdcSSasha Levin static void notify_vq_eventfd(struct kvm *kvm, void *dev, u32 vq, u32 efd) 4489ed67cdcSSasha Levin { 4499ed67cdcSSasha Levin struct net_dev *ndev = dev; 4509ed67cdcSSasha Levin struct vhost_vring_file file = { 4519ed67cdcSSasha Levin .index = vq, 4529ed67cdcSSasha Levin .fd = efd, 4539ed67cdcSSasha Levin }; 4549ed67cdcSSasha Levin int r; 4559ed67cdcSSasha Levin 4569ed67cdcSSasha Levin if (ndev->vhost_fd == 0) 4579ed67cdcSSasha Levin return; 4589ed67cdcSSasha Levin 4599ed67cdcSSasha Levin r = ioctl(ndev->vhost_fd, VHOST_SET_VRING_KICK, &file); 4609ed67cdcSSasha Levin if (r < 0) 4619ed67cdcSSasha Levin die_perror("VHOST_SET_VRING_KICK failed"); 46278a2a3e2SSasha Levin } 46378a2a3e2SSasha Levin 46478a2a3e2SSasha Levin static int notify_vq(struct kvm *kvm, void *dev, u32 vq) 46578a2a3e2SSasha Levin { 4669a6d73f1SSasha Levin struct net_dev *ndev = dev; 4679a6d73f1SSasha Levin 4689a6d73f1SSasha Levin virtio_net_handle_callback(kvm, ndev, vq); 46978a2a3e2SSasha Levin 47078a2a3e2SSasha Levin return 0; 47178a2a3e2SSasha Levin } 47278a2a3e2SSasha Levin 47378a2a3e2SSasha Levin static int get_pfn_vq(struct kvm *kvm, void *dev, u32 vq) 47478a2a3e2SSasha Levin { 47578a2a3e2SSasha Levin struct net_dev *ndev = dev; 47678a2a3e2SSasha Levin 47778a2a3e2SSasha Levin return ndev->vqs[vq].pfn; 47878a2a3e2SSasha Levin } 47978a2a3e2SSasha Levin 48078a2a3e2SSasha Levin static int get_size_vq(struct kvm *kvm, void *dev, u32 vq) 48178a2a3e2SSasha Levin { 4820f4f51a8SAsias He /* FIXME: dynamic */ 48378a2a3e2SSasha Levin return VIRTIO_NET_QUEUE_SIZE; 48478a2a3e2SSasha Levin } 48578a2a3e2SSasha Levin 4860f4f51a8SAsias He static int set_size_vq(struct kvm *kvm, void *dev, u32 vq, int size) 4870f4f51a8SAsias He { 4880f4f51a8SAsias He /* FIXME: dynamic */ 4890f4f51a8SAsias He return size; 4900f4f51a8SAsias He } 4910f4f51a8SAsias He 4921c47ce69SSasha Levin static struct virtio_ops net_dev_virtio_ops = (struct virtio_ops) { 4931c47ce69SSasha Levin .get_config = get_config, 4941c47ce69SSasha Levin .get_host_features = get_host_features, 4951c47ce69SSasha Levin .set_guest_features = set_guest_features, 4961c47ce69SSasha Levin .init_vq = init_vq, 4971c47ce69SSasha Levin .get_pfn_vq = get_pfn_vq, 4981c47ce69SSasha Levin .get_size_vq = get_size_vq, 4990f4f51a8SAsias He .set_size_vq = set_size_vq, 5000f4f51a8SAsias He .notify_vq = notify_vq, 5019ed67cdcSSasha Levin .notify_vq_gsi = notify_vq_gsi, 5029ed67cdcSSasha Levin .notify_vq_eventfd = notify_vq_eventfd, 5031c47ce69SSasha Levin }; 5041c47ce69SSasha Levin 5059ed67cdcSSasha Levin static void virtio_net__vhost_init(struct kvm *kvm, struct net_dev *ndev) 5069ed67cdcSSasha Levin { 50792c1f37bSSasha Levin u64 features = 1UL << VIRTIO_RING_F_EVENT_IDX; 5089ed67cdcSSasha Levin struct vhost_memory *mem; 5099ed67cdcSSasha Levin int r; 5109ed67cdcSSasha Levin 5119ed67cdcSSasha Levin ndev->vhost_fd = open("/dev/vhost-net", O_RDWR); 5129ed67cdcSSasha Levin if (ndev->vhost_fd < 0) 5139ed67cdcSSasha Levin die_perror("Failed openning vhost-net device"); 5149ed67cdcSSasha Levin 5158b795457SAsias He mem = calloc(1, sizeof(*mem) + sizeof(struct vhost_memory_region)); 5169ed67cdcSSasha Levin if (mem == NULL) 5179ed67cdcSSasha Levin die("Failed allocating memory for vhost memory map"); 5189ed67cdcSSasha Levin 5199ed67cdcSSasha Levin mem->nregions = 1; 5209ed67cdcSSasha Levin mem->regions[0] = (struct vhost_memory_region) { 5219ed67cdcSSasha Levin .guest_phys_addr = 0, 5229ed67cdcSSasha Levin .memory_size = kvm->ram_size, 523b4108023SAsias He .userspace_addr = (unsigned long)kvm->ram_start, 5249ed67cdcSSasha Levin }; 5259ed67cdcSSasha Levin 5269ed67cdcSSasha Levin r = ioctl(ndev->vhost_fd, VHOST_SET_OWNER); 5279ed67cdcSSasha Levin if (r != 0) 5289ed67cdcSSasha Levin die_perror("VHOST_SET_OWNER failed"); 5299ed67cdcSSasha Levin 5309ed67cdcSSasha Levin r = ioctl(ndev->vhost_fd, VHOST_SET_FEATURES, &features); 5319ed67cdcSSasha Levin if (r != 0) 5329ed67cdcSSasha Levin die_perror("VHOST_SET_FEATURES failed"); 5339ed67cdcSSasha Levin r = ioctl(ndev->vhost_fd, VHOST_SET_MEM_TABLE, mem); 5349ed67cdcSSasha Levin if (r != 0) 5359ed67cdcSSasha Levin die_perror("VHOST_SET_MEM_TABLE failed"); 536627d6874SAsias He 537627d6874SAsias He ndev->vdev.use_vhost = true; 538627d6874SAsias He 5399ed67cdcSSasha Levin free(mem); 5409ed67cdcSSasha Levin } 5419ed67cdcSSasha Levin 5425f225124SSasha Levin static inline void str_to_mac(const char *str, char *mac) 5435f225124SSasha Levin { 5445f225124SSasha Levin sscanf(str, "%hhx:%hhx:%hhx:%hhx:%hhx:%hhx", 5455f225124SSasha Levin mac, mac+1, mac+2, mac+3, mac+4, mac+5); 5465f225124SSasha Levin } 5474346fd8fSSasha Levin static int set_net_param(struct kvm *kvm, struct virtio_net_params *p, 5484346fd8fSSasha Levin const char *param, const char *val) 5495f225124SSasha Levin { 5505f225124SSasha Levin if (strcmp(param, "guest_mac") == 0) { 5515f225124SSasha Levin str_to_mac(val, p->guest_mac); 5525f225124SSasha Levin } else if (strcmp(param, "mode") == 0) { 5535f225124SSasha Levin if (!strncmp(val, "user", 4)) { 5545f225124SSasha Levin int i; 5555f225124SSasha Levin 5565f225124SSasha Levin for (i = 0; i < kvm->cfg.num_net_devices; i++) 5575f225124SSasha Levin if (kvm->cfg.net_params[i].mode == NET_MODE_USER) 5585f225124SSasha Levin die("Only one usermode network device allowed at a time"); 5595f225124SSasha Levin p->mode = NET_MODE_USER; 5605f225124SSasha Levin } else if (!strncmp(val, "tap", 3)) { 5615f225124SSasha Levin p->mode = NET_MODE_TAP; 5625f225124SSasha Levin } else if (!strncmp(val, "none", 4)) { 5635f225124SSasha Levin kvm->cfg.no_net = 1; 5645f225124SSasha Levin return -1; 5655f225124SSasha Levin } else 5665f225124SSasha Levin die("Unknown network mode %s, please use user, tap or none", kvm->cfg.network); 5675f225124SSasha Levin } else if (strcmp(param, "script") == 0) { 5685f225124SSasha Levin p->script = strdup(val); 5695f225124SSasha Levin } else if (strcmp(param, "guest_ip") == 0) { 5705f225124SSasha Levin p->guest_ip = strdup(val); 5715f225124SSasha Levin } else if (strcmp(param, "host_ip") == 0) { 5725f225124SSasha Levin p->host_ip = strdup(val); 5735f225124SSasha Levin } else if (strcmp(param, "trans") == 0) { 5745f225124SSasha Levin p->trans = strdup(val); 5755f225124SSasha Levin } else if (strcmp(param, "vhost") == 0) { 5765f225124SSasha Levin p->vhost = atoi(val); 5775f225124SSasha Levin } else if (strcmp(param, "fd") == 0) { 5785f225124SSasha Levin p->fd = atoi(val); 579*79052597SSasha Levin } else if (strcmp(param, "mq") == 0) { 580*79052597SSasha Levin p->mq = atoi(val); 5815f225124SSasha Levin } else 5825f225124SSasha Levin die("Unknown network parameter %s", param); 5835f225124SSasha Levin 5845f225124SSasha Levin return 0; 5855f225124SSasha Levin } 5865f225124SSasha Levin 5875f225124SSasha Levin int netdev_parser(const struct option *opt, const char *arg, int unset) 5885f225124SSasha Levin { 5895f225124SSasha Levin struct virtio_net_params p; 5905f225124SSasha Levin char *buf = NULL, *cmd = NULL, *cur = NULL; 5915f225124SSasha Levin bool on_cmd = true; 5925f225124SSasha Levin struct kvm *kvm = opt->ptr; 5935f225124SSasha Levin 5945f225124SSasha Levin if (arg) { 5955f225124SSasha Levin buf = strdup(arg); 5965f225124SSasha Levin if (buf == NULL) 5975f225124SSasha Levin die("Failed allocating new net buffer"); 5985f225124SSasha Levin cur = strtok(buf, ",="); 5995f225124SSasha Levin } 6005f225124SSasha Levin 6015f225124SSasha Levin p = (struct virtio_net_params) { 6025f225124SSasha Levin .guest_ip = DEFAULT_GUEST_ADDR, 6035f225124SSasha Levin .host_ip = DEFAULT_HOST_ADDR, 6045f225124SSasha Levin .script = DEFAULT_SCRIPT, 6055f225124SSasha Levin .mode = NET_MODE_TAP, 6065f225124SSasha Levin }; 6075f225124SSasha Levin 6085f225124SSasha Levin str_to_mac(DEFAULT_GUEST_MAC, p.guest_mac); 6095f225124SSasha Levin p.guest_mac[5] += kvm->cfg.num_net_devices; 6105f225124SSasha Levin 6115f225124SSasha Levin while (cur) { 6125f225124SSasha Levin if (on_cmd) { 6135f225124SSasha Levin cmd = cur; 6145f225124SSasha Levin } else { 6154346fd8fSSasha Levin if (set_net_param(kvm, &p, cmd, cur) < 0) 6165f225124SSasha Levin goto done; 6175f225124SSasha Levin } 6185f225124SSasha Levin on_cmd = !on_cmd; 6195f225124SSasha Levin 6205f225124SSasha Levin cur = strtok(NULL, ",="); 6215f225124SSasha Levin }; 6225f225124SSasha Levin 6235f225124SSasha Levin kvm->cfg.num_net_devices++; 6245f225124SSasha Levin 6255f225124SSasha Levin kvm->cfg.net_params = realloc(kvm->cfg.net_params, kvm->cfg.num_net_devices * sizeof(*kvm->cfg.net_params)); 6265f225124SSasha Levin if (kvm->cfg.net_params == NULL) 6275f225124SSasha Levin die("Failed adding new network device"); 6285f225124SSasha Levin 6295f225124SSasha Levin kvm->cfg.net_params[kvm->cfg.num_net_devices - 1] = p; 6305f225124SSasha Levin 6315f225124SSasha Levin done: 6325f225124SSasha Levin free(buf); 6335f225124SSasha Levin return 0; 6345f225124SSasha Levin } 6355f225124SSasha Levin 6365f225124SSasha Levin static int virtio_net__init_one(struct virtio_net_params *params) 6374f56d42cSAsias He { 638b5ee1ea7SAsias He int i; 6399a6d73f1SSasha Levin struct net_dev *ndev; 6409a6d73f1SSasha Levin 6419a6d73f1SSasha Levin ndev = calloc(1, sizeof(struct net_dev)); 6429a6d73f1SSasha Levin if (ndev == NULL) 6435f225124SSasha Levin return -ENOMEM; 6449a6d73f1SSasha Levin 6459a6d73f1SSasha Levin list_add_tail(&ndev->list, &ndevs); 6469a6d73f1SSasha Levin 6479a6d73f1SSasha Levin ndev->kvm = params->kvm; 6489a6d73f1SSasha Levin 6499a6d73f1SSasha Levin mutex_init(&ndev->mutex); 650*79052597SSasha Levin ndev->queue_pairs = max(1, min(VIRTIO_NET_NUM_QUEUES, params->mq)); 6519a6d73f1SSasha Levin ndev->config.status = VIRTIO_NET_S_LINK_UP; 652*79052597SSasha Levin if (ndev->queue_pairs > 1) 653*79052597SSasha Levin ndev->config.max_virtqueue_pairs = ndev->queue_pairs; 654*79052597SSasha Levin 6550c54698eSAsias He for (i = 0 ; i < 6 ; i++) { 6569a6d73f1SSasha Levin ndev->config.mac[i] = params->guest_mac[i]; 6579a6d73f1SSasha Levin ndev->info.guest_mac.addr[i] = params->guest_mac[i]; 6589a6d73f1SSasha Levin ndev->info.host_mac.addr[i] = params->host_mac[i]; 6590c54698eSAsias He } 660f715177dSAsias He 6619a6d73f1SSasha Levin ndev->mode = params->mode; 6629a6d73f1SSasha Levin if (ndev->mode == NET_MODE_TAP) { 6639a6d73f1SSasha Levin if (!virtio_net__tap_init(params, ndev)) 664db051127SMichael Ellerman die_perror("You have requested a TAP device, but creation of one has failed because"); 6659a6d73f1SSasha Levin ndev->ops = &tap_ops; 666b4fdde6dSAsias He } else { 6679a6d73f1SSasha Levin ndev->info.host_ip = ntohl(inet_addr(params->host_ip)); 6689a6d73f1SSasha Levin ndev->info.guest_ip = ntohl(inet_addr(params->guest_ip)); 6699a6d73f1SSasha Levin ndev->info.guest_netmask = ntohl(inet_addr("255.255.255.0")); 6709a6d73f1SSasha Levin ndev->info.buf_nr = 20, 6719a6d73f1SSasha Levin uip_init(&ndev->info); 6729a6d73f1SSasha Levin ndev->ops = &uip_ops; 673b4fdde6dSAsias He } 674b5ee1ea7SAsias He 67569205aa1SAsias He if (params->trans && strcmp(params->trans, "mmio") == 0) 6764346fd8fSSasha Levin virtio_init(params->kvm, ndev, &ndev->vdev, &net_dev_virtio_ops, 67769205aa1SAsias He VIRTIO_MMIO, PCI_DEVICE_ID_VIRTIO_NET, VIRTIO_ID_NET, PCI_CLASS_NET); 67869205aa1SAsias He else 6794346fd8fSSasha Levin virtio_init(params->kvm, ndev, &ndev->vdev, &net_dev_virtio_ops, 68002eca50cSAsias He VIRTIO_PCI, PCI_DEVICE_ID_VIRTIO_NET, VIRTIO_ID_NET, PCI_CLASS_NET); 68127ab67f5SSasha Levin 6829ed67cdcSSasha Levin if (params->vhost) 6839ed67cdcSSasha Levin virtio_net__vhost_init(params->kvm, ndev); 684cb83de6fSSasha Levin 685d278197dSAsias He if (compat_id == -1) 68652f34d2cSAsias He compat_id = virtio_compat_add_message("virtio-net", "CONFIG_VIRTIO_NET"); 6875f225124SSasha Levin 6885f225124SSasha Levin return 0; 6895f225124SSasha Levin } 6905f225124SSasha Levin 6915f225124SSasha Levin int virtio_net__init(struct kvm *kvm) 6925f225124SSasha Levin { 6935f225124SSasha Levin int i; 6945f225124SSasha Levin 6955f225124SSasha Levin for (i = 0; i < kvm->cfg.num_net_devices; i++) { 6965f225124SSasha Levin kvm->cfg.net_params[i].kvm = kvm; 6975f225124SSasha Levin virtio_net__init_one(&kvm->cfg.net_params[i]); 6985f225124SSasha Levin } 6995f225124SSasha Levin 7005f225124SSasha Levin if (kvm->cfg.num_net_devices == 0 && kvm->cfg.no_net == 0) { 7015f225124SSasha Levin struct virtio_net_params net_params; 7025f225124SSasha Levin 7035f225124SSasha Levin net_params = (struct virtio_net_params) { 7045f225124SSasha Levin .guest_ip = kvm->cfg.guest_ip, 7055f225124SSasha Levin .host_ip = kvm->cfg.host_ip, 7065f225124SSasha Levin .kvm = kvm, 7075f225124SSasha Levin .script = kvm->cfg.script, 7085f225124SSasha Levin .mode = NET_MODE_USER, 7095f225124SSasha Levin }; 7105f225124SSasha Levin str_to_mac(kvm->cfg.guest_mac, net_params.guest_mac); 7115f225124SSasha Levin str_to_mac(kvm->cfg.host_mac, net_params.host_mac); 7125f225124SSasha Levin 7135f225124SSasha Levin virtio_net__init_one(&net_params); 7145f225124SSasha Levin } 7155f225124SSasha Levin 7165f225124SSasha Levin return 0; 7175f225124SSasha Levin } 71849a8afd1SSasha Levin virtio_dev_init(virtio_net__init); 7195f225124SSasha Levin 7205f225124SSasha Levin int virtio_net__exit(struct kvm *kvm) 7215f225124SSasha Levin { 7225f225124SSasha Levin return 0; 7234f56d42cSAsias He } 72449a8afd1SSasha Levin virtio_dev_exit(virtio_net__exit); 725