drivers/net/tap.c

71  * through sock_hold(&q->sk). When the macvlan_dev goes away first,
72  * q->vlan becomes inaccessible. When the files gets closed,
82 			    struct tap_queue *q)
88 	if (q->enabled)
92 	rcu_assign_pointer(tap->taps[tap->numvtaps], q);
93 	q->queue_index = tap->numvtaps;
94 	q->enabled = true;
103 			 struct tap_queue *q)
108 	rcu_assign_pointer(q->tap, tap);
109 	rcu_assign_pointer(tap->taps[tap->numvtaps], q);
110 	sock_hold(&q->sk);
112 	q->file = file;
113 	q->queue_index = tap->numvtaps;
114 	q->enabled = true;
115 	file->private_data = q;
116 	list_add_tail(&q->next, &tap->queue_list);
124 static int tap_disable_queue(struct tap_queue *q)
130 	if (!q->enabled)
133 	tap = rtnl_dereference(q->tap);
136 		int index = q->queue_index;
143 		q->enabled = false;
159 static void tap_put_queue(struct tap_queue *q)
164 	tap = rtnl_dereference(q->tap);
167 		if (q->enabled)
168 			BUG_ON(tap_disable_queue(q));
171 		RCU_INIT_POINTER(q->tap, NULL);
172 		sock_put(&q->sk);
173 		list_del_init(&q->next);
179 	sock_put(&q->sk);
237 	struct tap_queue *q, *tmp;
240 	list_for_each_entry_safe(q, tmp, &tap->queue_list, next) {
241 		list_del_init(&q->next);
242 		RCU_INIT_POINTER(q->tap, NULL);
243 		if (q->enabled)
246 		sock_put(&q->sk);
260 	struct tap_queue *q;
268 	q = tap_get_queue(tap, skb);
269 	if (!q)
278 	if (q->flags & IFF_VNET_HDR)
290 			if (ptr_ring_produce(&q->ring, skb)) {
300 			if (ptr_ring_produce(&q->ring, skb)) {
319 		if (ptr_ring_produce(&q->ring, skb)) {
326 	wake_up_interruptible_poll(sk_sleep(&q->sk), EPOLLIN | EPOLLRDNORM | EPOLLRDBAND);
441 	struct tap_queue *q = container_of(sk, struct tap_queue, sk);
443 	ptr_ring_cleanup(&q->ring, __skb_array_destroy_skb);
450 	struct tap_queue *q;
459 	q = (struct tap_queue *)sk_alloc(net, AF_UNSPEC, GFP_KERNEL,
461 	if (!q)
463 	if (ptr_ring_init(&q->ring, tap->dev->tx_queue_len, GFP_KERNEL)) {
464 		sk_free(&q->sk);
468 	init_waitqueue_head(&q->sock.wq.wait);
469 	q->sock.type = SOCK_RAW;
470 	q->sock.state = SS_CONNECTED;
471 	q->sock.file = file;
472 	q->sock.ops = &tap_socket_ops;
473 	sock_init_data_uid(&q->sock, &q->sk, current_fsuid());
474 	q->sk.sk_write_space = tap_sock_write_space;
475 	q->sk.sk_destruct = tap_sock_destruct;
476 	q->flags = IFF_VNET_HDR | IFF_NO_PI | IFF_TAP;
477 	q->vnet_hdr_sz = sizeof(struct virtio_net_hdr);
487 		sock_set_flag(&q->sk, SOCK_ZEROCOPY);
489 	err = tap_set_queue(tap, file, q);
504 	sock_put(&q->sk);
515 	struct tap_queue *q = file->private_data;
516 	tap_put_queue(q);
522 	struct tap_queue *q = file->private_data;
525 	if (!q)
529 	poll_wait(file, &q->sock.wq.wait, wait);
531 	if (!ptr_ring_empty(&q->ring))
534 	if (sock_writeable(&q->sk) ||
535 	    (!test_and_set_bit(SOCKWQ_ASYNC_NOSPACE, &q->sock.flags) &&
536 	     sock_writeable(&q->sk)))
572 static ssize_t tap_get_user(struct tap_queue *q, void *msg_control,
590 	if (q->flags & IFF_VNET_HDR) {
591 		vnet_hdr_len = READ_ONCE(q->vnet_hdr_sz);
593 		hdr_len = tun_vnet_hdr_get(vnet_hdr_len, q->flags, from, &vnet_hdr);
606 	if (msg_control && sock_flag(&q->sk, SOCK_ZEROCOPY)) {
622 	skb = tap_alloc_skb(&q->sk, TAP_RESERVE, copylen,
642 	tap = rcu_dereference(q->tap);
651 		err = tun_vnet_hdr_to_skb(q->flags, skb, &vnet_hdr);
683 	tap = rcu_dereference(q->tap);
694 	struct tap_queue *q = file->private_data;
700 	return tap_get_user(q, NULL, from, noblock);
704 static ssize_t tap_put_user(struct tap_queue *q,
713 	if (q->flags & IFF_VNET_HDR) {
716 		vnet_hdr_len = READ_ONCE(q->vnet_hdr_sz);
718 		ret = tun_vnet_hdr_from_skb(q->flags, NULL, skb, &vnet_hdr);
756 static ssize_t tap_do_read(struct tap_queue *q,
773 			prepare_to_wait(sk_sleep(&q->sk), &wait,
777 		skb = ptr_ring_consume(&q->ring);
792 		finish_wait(sk_sleep(&q->sk), &wait);
796 		ret = tap_put_user(q, skb, to);
808 	struct tap_queue *q = file->private_data;
815 	ret = tap_do_read(q, to, noblock, NULL);
822 static struct tap_dev *tap_get_tap_dev(struct tap_queue *q)
827 	tap = rtnl_dereference(q->tap);
841 	struct tap_queue *q = file->private_data;
845 	tap = tap_get_tap_dev(q);
850 		ret = tap_enable_queue(tap, file, q);
852 		ret = tap_disable_queue(q);
860 static int set_offload(struct tap_queue *q, unsigned long arg)
866 	tap = rtnl_dereference(q->tap);
919 	struct tap_queue *q = file->private_data;
940 			q->flags = (q->flags & ~TAP_IFFEATURES) | u;
946 		tap = tap_get_tap_dev(q);
953 		u = q->flags;
980 		q->sk.sk_sndbuf = s;
991 		ret = set_offload(q, arg);
997 		tap = tap_get_tap_dev(q);
1016 		tap = tap_get_tap_dev(q);
1030 		return tun_vnet_ioctl(&q->vnet_hdr_sz, &q->flags, cmd, sp);
1045 static int tap_get_user_xdp(struct tap_queue *q, struct xdp_buff *xdp)
1059 	if (q->flags & IFF_VNET_HDR)
1060 		vnet_hdr_len = READ_ONCE(q->vnet_hdr_sz);
1076 		err = tun_vnet_hdr_to_skb(q->flags, skb, gso);
1087 	tap = rcu_dereference(q->tap);
1103 	tap = rcu_dereference(q->tap);
1113 	struct tap_queue *q = container_of(sock, struct tap_queue, sock);
1122 			tap_get_user_xdp(q, xdp);
1127 	return tap_get_user(q, ctl ? ctl->ptr : NULL, &m->msg_iter,
1134 	struct tap_queue *q = container_of(sock, struct tap_queue, sock);
1141 	ret = tap_do_read(q, &m->msg_iter, flags & MSG_DONTWAIT, skb);
1151 	struct tap_queue *q = container_of(sock, struct tap_queue,
1153 	return PTR_RING_PEEK_CALL(&q->ring, __skb_array_len_with_tag);
1169 	struct tap_queue *q;
1172 	q = file->private_data;
1173 	if (!q)
1175 	return &q->sock;
1181 	struct tap_queue *q;
1185 	q = file->private_data;
1186 	if (!q)
1188 	return &q->ring;
1195 	struct tap_queue *q;
1204 	list_for_each_entry(q, &tap->queue_list, next)
1205 		rings[i++] = &q->ring;