Lines Matching +full:operating +full:- +full:points +full:- +full:v2
3 * operating system. INET is implemented using the BSD Socket
6 * PACKET - implements raw packet sockets.
21 * Alan Cox : Re-commented the code.
29 * Alan Cox : New buffers. Use sk->mac.raw.
99 - if device has no dev->hard_header routine, it adds and removes ll header
101 but higher levels still should reserve dev->hard_header_len.
105 - packet socket receives packets with pulled ll header,
109 -----------
111 Incoming, dev->hard_header!=NULL
112 mac_header -> ll header
113 data -> data
115 Outgoing, dev->hard_header!=NULL
116 mac_header -> ll header
117 data -> ll header
119 Incoming, dev->hard_header==NULL
120 mac_header -> UNKNOWN position. It is very likely, that it points to ll
123 data -> data
125 Outgoing, dev->hard_header==NULL
126 mac_header -> data. ll header is still not built!
127 data -> data
130 If dev->hard_header==NULL we are unlikely to restore sensible ll header.
134 ------------
136 dev->hard_header != NULL
137 mac_header -> ll header
138 data -> ll header
140 dev->hard_header == NULL (ll header is added by device, we cannot control it)
141 mac_header -> data
142 data -> data
179 /* kbdq - kernel block descriptor queue */
190 * trick to see if user-space has caught up
237 #define BLOCK_STATUS(x) ((x)->hdr.bh1.block_status)
238 #define BLOCK_NUM_PKTS(x) ((x)->hdr.bh1.num_pkts)
239 #define BLOCK_O2FP(x) ((x)->hdr.bh1.offset_to_first_pkt)
240 #define BLOCK_LEN(x) ((x)->hdr.bh1.blk_len)
241 #define BLOCK_SNUM(x) ((x)->hdr.bh1.seq_num)
242 #define BLOCK_O2PRIV(x) ((x)->offset_to_priv)
327 #define PACKET_SKB_CB(__skb) ((struct packet_skb_cb *)((__skb)->cb))
329 #define GET_PBDQC_FROM_RB(x) ((struct tpacket_kbdq_core *)(&(x)->prb_bdqc))
331 ((struct tpacket_block_desc *)((x)->pkbdq[(bid)].buffer))
333 ((struct tpacket_block_desc *)((x)->pkbdq[(x)->kactive_blk_num].buffer))
335 (((x)->kactive_blk_num < ((x)->knum_blocks-1)) ? \
336 ((x)->kactive_blk_num+1) : 0)
346 /* register_prot_hook must be invoked with the po->bind_lock held,
353 if (!po->running) { in register_prot_hook()
354 if (po->fanout) in register_prot_hook()
357 dev_add_pack(&po->prot_hook); in register_prot_hook()
359 po->running = 1; in register_prot_hook()
363 /* {,__}unregister_prot_hook() must be invoked with the po->bind_lock
365 * the po->bind_lock and do a synchronize_net to make sure no
367 * of po->prot_hook. If the sync parameter is false, it is the
374 po->running = 0; in __unregister_prot_hook()
375 if (po->fanout) in __unregister_prot_hook()
378 __dev_remove_pack(&po->prot_hook); in __unregister_prot_hook()
382 spin_unlock(&po->bind_lock); in __unregister_prot_hook()
384 spin_lock(&po->bind_lock); in __unregister_prot_hook()
392 if (po->running) in unregister_prot_hook()
412 switch (po->tp_version) { in __packet_set_status()
414 h.h1->tp_status = status; in __packet_set_status()
415 flush_dcache_page(pgv_to_page(&h.h1->tp_status)); in __packet_set_status()
418 h.h2->tp_status = status; in __packet_set_status()
419 flush_dcache_page(pgv_to_page(&h.h2->tp_status)); in __packet_set_status()
441 switch (po->tp_version) { in __packet_get_status()
443 flush_dcache_page(pgv_to_page(&h.h1->tp_status)); in __packet_get_status()
444 return h.h1->tp_status; in __packet_get_status()
446 flush_dcache_page(pgv_to_page(&h.h2->tp_status)); in __packet_get_status()
447 return h.h2->tp_status; in __packet_get_status()
468 pg_vec_pos = position / rb->frames_per_block; in packet_lookup_frame()
469 frame_offset = position % rb->frames_per_block; in packet_lookup_frame()
471 h.raw = rb->pg_vec[pg_vec_pos].buffer + in packet_lookup_frame()
472 (frame_offset * rb->frame_size); in packet_lookup_frame()
484 return packet_lookup_frame(po, rb, rb->head, status); in packet_current_frame()
489 del_timer_sync(&pkc->retire_blk_timer); in prb_del_retire_blk_timer()
498 pkc = tx_ring ? &po->tx_ring.prb_bdqc : &po->rx_ring.prb_bdqc; in prb_shutdown_retire_blk_timer()
500 spin_lock(&rb_queue->lock); in prb_shutdown_retire_blk_timer()
501 pkc->delete_blk_timer = 1; in prb_shutdown_retire_blk_timer()
502 spin_unlock(&rb_queue->lock); in prb_shutdown_retire_blk_timer()
511 init_timer(&pkc->retire_blk_timer); in prb_init_blk_timer()
512 pkc->retire_blk_timer.data = (long)po; in prb_init_blk_timer()
513 pkc->retire_blk_timer.function = func; in prb_init_blk_timer()
514 pkc->retire_blk_timer.expires = jiffies; in prb_init_blk_timer()
524 pkc = tx_ring ? &po->tx_ring.prb_bdqc : &po->rx_ring.prb_bdqc; in prb_setup_retire_blk_timer()
537 dev = __dev_get_by_index(sock_net(&po->sk), po->ifindex); in prb_calc_retire_blk_tmo()
580 p1->feature_req_word = req_u->req3.tp_feature_req_word; in prb_init_ft_ops()
588 struct tpacket_kbdq_core *p1 = &rb->prb_bdqc; in init_prb_bdqc()
593 p1->knxt_seq_num = 1; in init_prb_bdqc()
594 p1->pkbdq = pg_vec; in init_prb_bdqc()
596 p1->pkblk_start = (char *)pg_vec[0].buffer; in init_prb_bdqc()
597 p1->kblk_size = req_u->req3.tp_block_size; in init_prb_bdqc()
598 p1->knum_blocks = req_u->req3.tp_block_nr; in init_prb_bdqc()
599 p1->hdrlen = po->tp_hdrlen; in init_prb_bdqc()
600 p1->version = po->tp_version; in init_prb_bdqc()
601 p1->last_kactive_blk_num = 0; in init_prb_bdqc()
602 po->stats_u.stats3.tp_freeze_q_cnt = 0; in init_prb_bdqc()
603 if (req_u->req3.tp_retire_blk_tov) in init_prb_bdqc()
604 p1->retire_blk_tov = req_u->req3.tp_retire_blk_tov; in init_prb_bdqc()
606 p1->retire_blk_tov = prb_calc_retire_blk_tmo(po, in init_prb_bdqc()
607 req_u->req3.tp_block_size); in init_prb_bdqc()
608 p1->tov_in_jiffies = msecs_to_jiffies(p1->retire_blk_tov); in init_prb_bdqc()
609 p1->blk_sizeof_priv = req_u->req3.tp_sizeof_priv; in init_prb_bdqc()
621 mod_timer(&pkc->retire_blk_timer, in _prb_refresh_rx_retire_blk_timer()
622 jiffies + pkc->tov_in_jiffies); in _prb_refresh_rx_retire_blk_timer()
623 pkc->last_kactive_blk_num = pkc->kactive_blk_num; in _prb_refresh_rx_retire_blk_timer()
630 * on packet-by-packet basis.
632 * With a 1MB block-size, on a 1Gbps line, it will take
645 * a) line-speed and b) block-size.
652 struct tpacket_kbdq_core *pkc = &po->rx_ring.prb_bdqc; in prb_retire_rx_blk_timer_expired()
656 spin_lock(&po->sk.sk_receive_queue.lock); in prb_retire_rx_blk_timer_expired()
661 if (unlikely(pkc->delete_blk_timer)) in prb_retire_rx_blk_timer_expired()
674 while (atomic_read(&pkc->blk_fill_in_prog)) { in prb_retire_rx_blk_timer_expired()
680 if (pkc->last_kactive_blk_num == pkc->kactive_blk_num) { in prb_retire_rx_blk_timer_expired()
688 /* Case 1. Queue was frozen because user-space was in prb_retire_rx_blk_timer_expired()
693 * Ok, user-space is still behind. in prb_retire_rx_blk_timer_expired()
698 /* Case 2. queue was frozen,user-space caught up, in prb_retire_rx_blk_timer_expired()
703 * Thawing/timer-refresh is a side effect. in prb_retire_rx_blk_timer_expired()
715 spin_unlock(&po->sk.sk_receive_queue.lock); in prb_retire_rx_blk_timer_expired()
731 end = (u8 *)PAGE_ALIGN((unsigned long)pkc1->pkblk_end); in prb_flush_block()
768 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1; in prb_close_block()
770 if (po->stats.tp_drops) in prb_close_block()
773 last_pkt = (struct tpacket3_hdr *)pkc1->prev; in prb_close_block()
774 last_pkt->tp_next_offset = 0; in prb_close_block()
778 h1->ts_last_pkt.ts_sec = last_pkt->tp_sec; in prb_close_block()
779 h1->ts_last_pkt.ts_nsec = last_pkt->tp_nsec; in prb_close_block()
781 /* Ok, we tmo'd - so get the current time */ in prb_close_block()
784 h1->ts_last_pkt.ts_sec = ts.tv_sec; in prb_close_block()
785 h1->ts_last_pkt.ts_nsec = ts.tv_nsec; in prb_close_block()
793 pkc1->kactive_blk_num = GET_NEXT_PRB_BLK_NUM(pkc1); in prb_close_block()
798 pkc->reset_pending_on_curr_blk = 0; in prb_thaw_queue()
812 struct tpacket_hdr_v1 *h1 = &pbd1->hdr.bh1; in prb_open_block()
821 BLOCK_SNUM(pbd1) = pkc1->knxt_seq_num++; in prb_open_block()
823 BLOCK_LEN(pbd1) = BLK_PLUS_PRIV(pkc1->blk_sizeof_priv); in prb_open_block()
825 h1->ts_first_pkt.ts_sec = ts.tv_sec; in prb_open_block()
826 h1->ts_first_pkt.ts_nsec = ts.tv_nsec; in prb_open_block()
827 pkc1->pkblk_start = (char *)pbd1; in prb_open_block()
828 pkc1->nxt_offset = (char *)(pkc1->pkblk_start + in prb_open_block()
829 BLK_PLUS_PRIV(pkc1->blk_sizeof_priv)); in prb_open_block()
830 BLOCK_O2FP(pbd1) = (__u32)BLK_PLUS_PRIV(pkc1->blk_sizeof_priv); in prb_open_block()
832 pbd1->version = pkc1->version; in prb_open_block()
833 pkc1->prev = pkc1->nxt_offset; in prb_open_block()
834 pkc1->pkblk_end = pkc1->pkblk_start + pkc1->kblk_size; in prb_open_block()
844 pbd1, BLOCK_STATUS(pbd1), pkc1->kactive_blk_num); in prb_open_block()
854 * 4) user-space is either sleeping or processing block '0'.
856 * it will close block-7,loop around and try to fill block '0'.
857 * call-flow:
861 * |->(BLOCK_STATUS == USER) evaluates to true
862 * 5.1) Since block-0 is currently in-use, we just freeze the queue.
867 * re-open block-0 in near future.
869 * case and __packet_lookup_frame_in_block will check if block-0
870 * is free and can now be re-used.
875 pkc->reset_pending_on_curr_blk = 1; in prb_freeze_queue()
876 po->stats_u.stats3.tp_freeze_q_cnt++; in prb_freeze_queue()
909 return (void *)pkc->nxt_offset; in prb_dispatch_next_block()
921 * cpu-0 and tpacket_rcv() got invoked on cpu-1, didn't in prb_retire_current_block()
926 * the timer-handler already handled this case. in prb_retire_current_block()
929 while (atomic_read(&pkc->blk_fill_in_prog)) { in prb_retire_current_block()
938 WARN(1, "ERROR-pbd[%d]:%p\n", pkc->kactive_blk_num, pbd); in prb_retire_current_block()
951 return pkc->reset_pending_on_curr_blk; in prb_queue_frozen()
957 atomic_dec(&pkc->blk_fill_in_prog); in prb_clear_blk_fill_status()
963 ppd->hv1.tp_rxhash = skb_get_rxhash(pkc->skb); in prb_fill_rxhash()
969 ppd->hv1.tp_rxhash = 0; in prb_clear_rxhash()
975 if (vlan_tx_tag_present(pkc->skb)) { in prb_fill_vlan_info()
976 ppd->hv1.tp_vlan_tci = vlan_tx_tag_get(pkc->skb); in prb_fill_vlan_info()
977 ppd->tp_status = TP_STATUS_VLAN_VALID; in prb_fill_vlan_info()
979 ppd->hv1.tp_vlan_tci = ppd->tp_status = 0; in prb_fill_vlan_info()
988 if (pkc->feature_req_word & TP_FT_REQ_FILL_RXHASH) in prb_run_all_ft_ops()
1002 ppd->tp_next_offset = TOTAL_PKT_LEN_INCL_ALIGN(len); in prb_fill_curr_block()
1003 pkc->prev = curr; in prb_fill_curr_block()
1004 pkc->nxt_offset += TOTAL_PKT_LEN_INCL_ALIGN(len); in prb_fill_curr_block()
1007 atomic_inc(&pkc->blk_fill_in_prog); in prb_fill_curr_block()
1011 /* Assumes caller has the sk->rx_queue.lock */
1022 pkc = GET_PBDQC_FROM_RB(((struct packet_ring_buffer *)&po->rx_ring)); in __packet_lookup_frame_in_block()
1029 * is still in_use by user-space. in __packet_lookup_frame_in_block()
1036 * Ok, the block was released by user-space. in __packet_lookup_frame_in_block()
1046 curr = pkc->nxt_offset; in __packet_lookup_frame_in_block()
1047 pkc->skb = skb; in __packet_lookup_frame_in_block()
1048 end = (char *) ((char *)pbd + pkc->kblk_size); in __packet_lookup_frame_in_block()
1079 switch (po->tp_version) { in packet_current_rx_frame()
1082 curr = packet_lookup_frame(po, &po->rx_ring, in packet_current_rx_frame()
1083 po->rx_ring.head, status); in packet_current_rx_frame()
1110 if (rb->prb_bdqc.kactive_blk_num) in prb_previous_blk_num()
1111 prev = rb->prb_bdqc.kactive_blk_num-1; in prb_previous_blk_num()
1113 prev = rb->prb_bdqc.knum_blocks-1; in prb_previous_blk_num()
1130 if (po->tp_version <= TPACKET_V2) in packet_previous_rx_frame()
1139 switch (po->tp_version) { in packet_increment_rx_head()
1155 unsigned int previous = rb->head ? rb->head - 1 : rb->frame_max; in packet_previous_frame()
1161 buff->head = buff->head != buff->frame_max ? buff->head+1 : 0; in packet_increment_head()
1166 skb_queue_purge(&sk->sk_error_queue); in packet_sock_destruct()
1168 WARN_ON(atomic_read(&sk->sk_rmem_alloc)); in packet_sock_destruct()
1169 WARN_ON(atomic_read(&sk->sk_wmem_alloc)); in packet_sock_destruct()
1181 int x = atomic_read(&f->rr_cur) + 1; in fanout_rr_next()
1191 u32 idx, hash = skb->rxhash; in fanout_demux_hash()
1195 return f->arr[idx]; in fanout_demux_hash()
1202 cur = atomic_read(&f->rr_cur); in fanout_demux_lb()
1203 while ((old = atomic_cmpxchg(&f->rr_cur, cur, in fanout_demux_lb()
1206 return f->arr[cur]; in fanout_demux_lb()
1213 return f->arr[cpu % num]; in fanout_demux_cpu()
1219 struct packet_fanout *f = pt->af_packet_priv; in packet_rcv_fanout()
1220 unsigned int num = f->num_members; in packet_rcv_fanout()
1224 if (!net_eq(dev_net(dev), read_pnet(&f->net)) || in packet_rcv_fanout()
1230 switch (f->type) { in packet_rcv_fanout()
1233 if (f->defrag) { in packet_rcv_fanout()
1251 return po->prot_hook.func(skb, dev, &po->prot_hook, orig_dev); in packet_rcv_fanout()
1259 struct packet_fanout *f = po->fanout; in __fanout_link()
1261 spin_lock(&f->lock); in __fanout_link()
1262 f->arr[f->num_members] = sk; in __fanout_link()
1264 f->num_members++; in __fanout_link()
1265 spin_unlock(&f->lock); in __fanout_link()
1270 struct packet_fanout *f = po->fanout; in __fanout_unlink()
1273 spin_lock(&f->lock); in __fanout_unlink()
1274 for (i = 0; i < f->num_members; i++) { in __fanout_unlink()
1275 if (f->arr[i] == sk) in __fanout_unlink()
1278 BUG_ON(i >= f->num_members); in __fanout_unlink()
1279 f->arr[i] = f->arr[f->num_members - 1]; in __fanout_unlink()
1280 f->num_members--; in __fanout_unlink()
1281 spin_unlock(&f->lock); in __fanout_unlink()
1298 return -EINVAL; in fanout_add()
1301 if (!po->running) in fanout_add()
1302 return -EINVAL; in fanout_add()
1304 if (po->fanout) in fanout_add()
1305 return -EALREADY; in fanout_add()
1310 if (f->id == id && in fanout_add()
1311 read_pnet(&f->net) == sock_net(sk)) { in fanout_add()
1316 err = -EINVAL; in fanout_add()
1317 if (match && match->defrag != defrag) in fanout_add()
1320 err = -ENOMEM; in fanout_add()
1324 write_pnet(&match->net, sock_net(sk)); in fanout_add()
1325 match->id = id; in fanout_add()
1326 match->type = type; in fanout_add()
1327 match->defrag = defrag; in fanout_add()
1328 atomic_set(&match->rr_cur, 0); in fanout_add()
1329 INIT_LIST_HEAD(&match->list); in fanout_add()
1330 spin_lock_init(&match->lock); in fanout_add()
1331 atomic_set(&match->sk_ref, 0); in fanout_add()
1332 match->prot_hook.type = po->prot_hook.type; in fanout_add()
1333 match->prot_hook.dev = po->prot_hook.dev; in fanout_add()
1334 match->prot_hook.func = packet_rcv_fanout; in fanout_add()
1335 match->prot_hook.af_packet_priv = match; in fanout_add()
1336 dev_add_pack(&match->prot_hook); in fanout_add()
1337 list_add(&match->list, &fanout_list); in fanout_add()
1339 err = -EINVAL; in fanout_add()
1340 if (match->type == type && in fanout_add()
1341 match->prot_hook.type == po->prot_hook.type && in fanout_add()
1342 match->prot_hook.dev == po->prot_hook.dev) { in fanout_add()
1343 err = -ENOSPC; in fanout_add()
1344 if (atomic_read(&match->sk_ref) < PACKET_FANOUT_MAX) { in fanout_add()
1345 __dev_remove_pack(&po->prot_hook); in fanout_add()
1346 po->fanout = match; in fanout_add()
1347 atomic_inc(&match->sk_ref); in fanout_add()
1362 f = po->fanout; in fanout_release()
1366 po->fanout = NULL; in fanout_release()
1369 if (atomic_dec_and_test(&f->sk_ref)) { in fanout_release()
1370 list_del(&f->list); in fanout_release()
1371 dev_remove_pack(&f->prot_hook); in fanout_release()
1392 sk = pt->af_packet_priv; in packet_rcv_spkt()
1401 * For outgoing ones skb->data == skb_mac_header(skb) in packet_rcv_spkt()
1405 if (skb->pkt_type == PACKET_LOOPBACK) in packet_rcv_spkt()
1421 spkt = &PACKET_SKB_CB(skb)->sa.pkt; in packet_rcv_spkt()
1423 skb_push(skb, skb->data - skb_mac_header(skb)); in packet_rcv_spkt()
1429 spkt->spkt_family = dev->type; in packet_rcv_spkt()
1430 strlcpy(spkt->spkt_device, dev->name, sizeof(spkt->spkt_device)); in packet_rcv_spkt()
1431 spkt->spkt_protocol = skb->protocol; in packet_rcv_spkt()
1456 struct sock *sk = sock->sk; in packet_sendmsg_spkt()
1457 struct sockaddr_pkt *saddr = (struct sockaddr_pkt *)msg->msg_name; in packet_sendmsg_spkt()
1468 if (msg->msg_namelen < sizeof(struct sockaddr)) in packet_sendmsg_spkt()
1469 return -EINVAL; in packet_sendmsg_spkt()
1470 if (msg->msg_namelen == sizeof(struct sockaddr_pkt)) in packet_sendmsg_spkt()
1471 proto = saddr->spkt_protocol; in packet_sendmsg_spkt()
1473 return -ENOTCONN; /* SOCK_PACKET must be sent giving an address */ in packet_sendmsg_spkt()
1479 saddr->spkt_device[13] = 0; in packet_sendmsg_spkt()
1482 dev = dev_get_by_name_rcu(sock_net(sk), saddr->spkt_device); in packet_sendmsg_spkt()
1483 err = -ENODEV; in packet_sendmsg_spkt()
1487 err = -ENETDOWN; in packet_sendmsg_spkt()
1488 if (!(dev->flags & IFF_UP)) in packet_sendmsg_spkt()
1496 err = -EMSGSIZE; in packet_sendmsg_spkt()
1497 if (len > dev->mtu + dev->hard_header_len + VLAN_HLEN) in packet_sendmsg_spkt()
1502 int tlen = dev->needed_tailroom; in packet_sendmsg_spkt()
1503 unsigned int hhlen = dev->header_ops ? dev->hard_header_len : 0; in packet_sendmsg_spkt()
1508 return -ENOBUFS; in packet_sendmsg_spkt()
1518 skb->data -= hhlen; in packet_sendmsg_spkt()
1519 skb->tail -= hhlen; in packet_sendmsg_spkt()
1523 err = memcpy_fromiovec(skb_put(skb, len), msg->msg_iov, len); in packet_sendmsg_spkt()
1529 if (len > (dev->mtu + dev->hard_header_len)) { in packet_sendmsg_spkt()
1531 * double-check this now that we have the actual in packet_sendmsg_spkt()
1537 if (ehdr->h_proto != htons(ETH_P_8021Q)) { in packet_sendmsg_spkt()
1538 err = -EMSGSIZE; in packet_sendmsg_spkt()
1543 skb->protocol = proto; in packet_sendmsg_spkt()
1544 skb->dev = dev; in packet_sendmsg_spkt()
1545 skb->priority = sk->sk_priority; in packet_sendmsg_spkt()
1546 skb->mark = sk->sk_mark; in packet_sendmsg_spkt()
1547 err = sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags); in packet_sendmsg_spkt()
1569 filter = rcu_dereference(sk->sk_filter); in run_filter()
1581 * Note tricky part: we DO mangle shared skb! skb->data, skb->len
1582 * and skb->cb are mangled. It works because (and until) packets
1595 u8 *skb_head = skb->data; in packet_rcv()
1596 int skb_len = skb->len; in packet_rcv()
1599 if (skb->pkt_type == PACKET_LOOPBACK) in packet_rcv()
1602 sk = pt->af_packet_priv; in packet_rcv()
1608 skb->dev = dev; in packet_rcv()
1610 if (dev->header_ops) { in packet_rcv()
1618 if (sk->sk_type != SOCK_DGRAM) in packet_rcv()
1619 skb_push(skb, skb->data - skb_mac_header(skb)); in packet_rcv()
1620 else if (skb->pkt_type == PACKET_OUTGOING) { in packet_rcv()
1626 snaplen = skb->len; in packet_rcv()
1634 if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf) in packet_rcv()
1642 if (skb_head != skb->data) { in packet_rcv()
1643 skb->data = skb_head; in packet_rcv()
1644 skb->len = skb_len; in packet_rcv()
1650 BUILD_BUG_ON(sizeof(*PACKET_SKB_CB(skb)) + MAX_ADDR_LEN - 8 > in packet_rcv()
1651 sizeof(skb->cb)); in packet_rcv()
1653 sll = &PACKET_SKB_CB(skb)->sa.ll; in packet_rcv()
1654 sll->sll_family = AF_PACKET; in packet_rcv()
1655 sll->sll_hatype = dev->type; in packet_rcv()
1656 sll->sll_protocol = skb->protocol; in packet_rcv()
1657 sll->sll_pkttype = skb->pkt_type; in packet_rcv()
1658 if (unlikely(po->origdev)) in packet_rcv()
1659 sll->sll_ifindex = orig_dev->ifindex; in packet_rcv()
1661 sll->sll_ifindex = dev->ifindex; in packet_rcv()
1663 sll->sll_halen = dev_parse_header(skb, sll->sll_addr); in packet_rcv()
1665 PACKET_SKB_CB(skb)->origlen = skb->len; in packet_rcv()
1671 skb->dev = NULL; in packet_rcv()
1677 spin_lock(&sk->sk_receive_queue.lock); in packet_rcv()
1678 po->stats.tp_packets++; in packet_rcv()
1679 skb->dropcount = atomic_read(&sk->sk_drops); in packet_rcv()
1680 __skb_queue_tail(&sk->sk_receive_queue, skb); in packet_rcv()
1681 spin_unlock(&sk->sk_receive_queue.lock); in packet_rcv()
1682 sk->sk_data_ready(sk, skb->len); in packet_rcv()
1686 spin_lock(&sk->sk_receive_queue.lock); in packet_rcv()
1687 po->stats.tp_drops++; in packet_rcv()
1688 atomic_inc(&sk->sk_drops); in packet_rcv()
1689 spin_unlock(&sk->sk_receive_queue.lock); in packet_rcv()
1692 if (skb_head != skb->data && skb_shared(skb)) { in packet_rcv()
1693 skb->data = skb_head; in packet_rcv()
1694 skb->len = skb_len; in packet_rcv()
1713 u8 *skb_head = skb->data; in tpacket_rcv()
1714 int skb_len = skb->len; in tpacket_rcv()
1723 if (skb->pkt_type == PACKET_LOOPBACK) in tpacket_rcv()
1726 sk = pt->af_packet_priv; in tpacket_rcv()
1732 if (dev->header_ops) { in tpacket_rcv()
1733 if (sk->sk_type != SOCK_DGRAM) in tpacket_rcv()
1734 skb_push(skb, skb->data - skb_mac_header(skb)); in tpacket_rcv()
1735 else if (skb->pkt_type == PACKET_OUTGOING) { in tpacket_rcv()
1741 if (skb->ip_summed == CHECKSUM_PARTIAL) in tpacket_rcv()
1744 snaplen = skb->len; in tpacket_rcv()
1752 if (sk->sk_type == SOCK_DGRAM) { in tpacket_rcv()
1753 macoff = netoff = TPACKET_ALIGN(po->tp_hdrlen) + 16 + in tpacket_rcv()
1754 po->tp_reserve; in tpacket_rcv()
1757 netoff = TPACKET_ALIGN(po->tp_hdrlen + in tpacket_rcv()
1759 po->tp_reserve; in tpacket_rcv()
1760 macoff = netoff - maclen; in tpacket_rcv()
1762 if (po->tp_version <= TPACKET_V2) { in tpacket_rcv()
1763 if (macoff + snaplen > po->rx_ring.frame_size) { in tpacket_rcv()
1764 if (po->copy_thresh && in tpacket_rcv()
1765 atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf) { in tpacket_rcv()
1770 skb_head = skb->data; in tpacket_rcv()
1775 snaplen = po->rx_ring.frame_size - macoff; in tpacket_rcv()
1780 spin_lock(&sk->sk_receive_queue.lock); in tpacket_rcv()
1785 if (po->tp_version <= TPACKET_V2) { in tpacket_rcv()
1786 packet_increment_rx_head(po, &po->rx_ring); in tpacket_rcv()
1789 * because it's COR - Clear On Read. in tpacket_rcv()
1790 * Anyways, moving it for V1/V2 only as V3 doesn't need this in tpacket_rcv()
1793 if (po->stats.tp_drops) in tpacket_rcv()
1796 po->stats.tp_packets++; in tpacket_rcv()
1799 __skb_queue_tail(&sk->sk_receive_queue, copy_skb); in tpacket_rcv()
1801 spin_unlock(&sk->sk_receive_queue.lock); in tpacket_rcv()
1805 switch (po->tp_version) { in tpacket_rcv()
1807 h.h1->tp_len = skb->len; in tpacket_rcv()
1808 h.h1->tp_snaplen = snaplen; in tpacket_rcv()
1809 h.h1->tp_mac = macoff; in tpacket_rcv()
1810 h.h1->tp_net = netoff; in tpacket_rcv()
1811 if ((po->tp_tstamp & SOF_TIMESTAMPING_SYS_HARDWARE) in tpacket_rcv()
1812 && shhwtstamps->syststamp.tv64) in tpacket_rcv()
1813 tv = ktime_to_timeval(shhwtstamps->syststamp); in tpacket_rcv()
1814 else if ((po->tp_tstamp & SOF_TIMESTAMPING_RAW_HARDWARE) in tpacket_rcv()
1815 && shhwtstamps->hwtstamp.tv64) in tpacket_rcv()
1816 tv = ktime_to_timeval(shhwtstamps->hwtstamp); in tpacket_rcv()
1817 else if (skb->tstamp.tv64) in tpacket_rcv()
1818 tv = ktime_to_timeval(skb->tstamp); in tpacket_rcv()
1821 h.h1->tp_sec = tv.tv_sec; in tpacket_rcv()
1822 h.h1->tp_usec = tv.tv_usec; in tpacket_rcv()
1826 h.h2->tp_len = skb->len; in tpacket_rcv()
1827 h.h2->tp_snaplen = snaplen; in tpacket_rcv()
1828 h.h2->tp_mac = macoff; in tpacket_rcv()
1829 h.h2->tp_net = netoff; in tpacket_rcv()
1830 if ((po->tp_tstamp & SOF_TIMESTAMPING_SYS_HARDWARE) in tpacket_rcv()
1831 && shhwtstamps->syststamp.tv64) in tpacket_rcv()
1832 ts = ktime_to_timespec(shhwtstamps->syststamp); in tpacket_rcv()
1833 else if ((po->tp_tstamp & SOF_TIMESTAMPING_RAW_HARDWARE) in tpacket_rcv()
1834 && shhwtstamps->hwtstamp.tv64) in tpacket_rcv()
1835 ts = ktime_to_timespec(shhwtstamps->hwtstamp); in tpacket_rcv()
1836 else if (skb->tstamp.tv64) in tpacket_rcv()
1837 ts = ktime_to_timespec(skb->tstamp); in tpacket_rcv()
1840 h.h2->tp_sec = ts.tv_sec; in tpacket_rcv()
1841 h.h2->tp_nsec = ts.tv_nsec; in tpacket_rcv()
1843 h.h2->tp_vlan_tci = vlan_tx_tag_get(skb); in tpacket_rcv()
1846 h.h2->tp_vlan_tci = 0; in tpacket_rcv()
1848 h.h2->tp_padding = 0; in tpacket_rcv()
1855 h.h3->tp_status |= status; in tpacket_rcv()
1856 h.h3->tp_len = skb->len; in tpacket_rcv()
1857 h.h3->tp_snaplen = snaplen; in tpacket_rcv()
1858 h.h3->tp_mac = macoff; in tpacket_rcv()
1859 h.h3->tp_net = netoff; in tpacket_rcv()
1860 if ((po->tp_tstamp & SOF_TIMESTAMPING_SYS_HARDWARE) in tpacket_rcv()
1861 && shhwtstamps->syststamp.tv64) in tpacket_rcv()
1862 ts = ktime_to_timespec(shhwtstamps->syststamp); in tpacket_rcv()
1863 else if ((po->tp_tstamp & SOF_TIMESTAMPING_RAW_HARDWARE) in tpacket_rcv()
1864 && shhwtstamps->hwtstamp.tv64) in tpacket_rcv()
1865 ts = ktime_to_timespec(shhwtstamps->hwtstamp); in tpacket_rcv()
1866 else if (skb->tstamp.tv64) in tpacket_rcv()
1867 ts = ktime_to_timespec(skb->tstamp); in tpacket_rcv()
1870 h.h3->tp_sec = ts.tv_sec; in tpacket_rcv()
1871 h.h3->tp_nsec = ts.tv_nsec; in tpacket_rcv()
1879 sll->sll_halen = dev_parse_header(skb, sll->sll_addr); in tpacket_rcv()
1880 sll->sll_family = AF_PACKET; in tpacket_rcv()
1881 sll->sll_hatype = dev->type; in tpacket_rcv()
1882 sll->sll_protocol = skb->protocol; in tpacket_rcv()
1883 sll->sll_pkttype = skb->pkt_type; in tpacket_rcv()
1884 if (unlikely(po->origdev)) in tpacket_rcv()
1885 sll->sll_ifindex = orig_dev->ifindex; in tpacket_rcv()
1887 sll->sll_ifindex = dev->ifindex; in tpacket_rcv()
1894 if (po->tp_version <= TPACKET_V2) { in tpacket_rcv()
1903 if (po->tp_version <= TPACKET_V2) in tpacket_rcv()
1906 prb_clear_blk_fill_status(&po->rx_ring); in tpacket_rcv()
1908 sk->sk_data_ready(sk, 0); in tpacket_rcv()
1911 if (skb_head != skb->data && skb_shared(skb)) { in tpacket_rcv()
1912 skb->data = skb_head; in tpacket_rcv()
1913 skb->len = skb_len; in tpacket_rcv()
1920 po->stats.tp_drops++; in tpacket_rcv()
1921 spin_unlock(&sk->sk_receive_queue.lock); in tpacket_rcv()
1923 sk->sk_data_ready(sk, 0); in tpacket_rcv()
1930 struct packet_sock *po = pkt_sk(skb->sk); in tpacket_destruct_skb()
1933 if (likely(po->tx_ring.pg_vec)) { in tpacket_destruct_skb()
1934 ph = skb_shinfo(skb)->destructor_arg; in tpacket_destruct_skb()
1936 BUG_ON(atomic_read(&po->tx_ring.pending) == 0); in tpacket_destruct_skb()
1937 atomic_dec(&po->tx_ring.pending); in tpacket_destruct_skb()
1954 struct socket *sock = po->sk.sk_socket; in tpacket_fill_skb()
1961 skb->protocol = proto; in tpacket_fill_skb()
1962 skb->dev = dev; in tpacket_fill_skb()
1963 skb->priority = po->sk.sk_priority; in tpacket_fill_skb()
1964 skb->mark = po->sk.sk_mark; in tpacket_fill_skb()
1965 skb_shinfo(skb)->destructor_arg = ph.raw; in tpacket_fill_skb()
1967 switch (po->tp_version) { in tpacket_fill_skb()
1969 tp_len = ph.h2->tp_len; in tpacket_fill_skb()
1972 tp_len = ph.h1->tp_len; in tpacket_fill_skb()
1977 return -EMSGSIZE; in tpacket_fill_skb()
1983 data = ph.raw + po->tp_hdrlen - sizeof(struct sockaddr_ll); in tpacket_fill_skb()
1986 if (sock->type == SOCK_DGRAM) { in tpacket_fill_skb()
1990 return -EINVAL; in tpacket_fill_skb()
1991 } else if (dev->hard_header_len) { in tpacket_fill_skb()
1993 if (unlikely(tp_len <= dev->hard_header_len)) { in tpacket_fill_skb()
1995 tp_len, dev->hard_header_len); in tpacket_fill_skb()
1996 return -EINVAL; in tpacket_fill_skb()
1999 skb_push(skb, dev->hard_header_len); in tpacket_fill_skb()
2001 dev->hard_header_len); in tpacket_fill_skb()
2005 data += dev->hard_header_len; in tpacket_fill_skb()
2006 to_write -= dev->hard_header_len; in tpacket_fill_skb()
2009 err = -EFAULT; in tpacket_fill_skb()
2011 len_max = PAGE_SIZE - offset; in tpacket_fill_skb()
2014 skb->data_len = to_write; in tpacket_fill_skb()
2015 skb->len += to_write; in tpacket_fill_skb()
2016 skb->truesize += to_write; in tpacket_fill_skb()
2017 atomic_add(to_write, &po->sk.sk_wmem_alloc); in tpacket_fill_skb()
2020 nr_frags = skb_shinfo(skb)->nr_frags; in tpacket_fill_skb()
2025 return -EFAULT; in tpacket_fill_skb()
2033 to_write -= len; in tpacket_fill_skb()
2050 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name; in tpacket_snd()
2057 mutex_lock(&po->pg_vec_lock); in tpacket_snd()
2059 err = -EBUSY; in tpacket_snd()
2061 dev = po->prot_hook.dev; in tpacket_snd()
2062 proto = po->num; in tpacket_snd()
2065 err = -EINVAL; in tpacket_snd()
2066 if (msg->msg_namelen < sizeof(struct sockaddr_ll)) in tpacket_snd()
2068 if (msg->msg_namelen < (saddr->sll_halen in tpacket_snd()
2072 proto = saddr->sll_protocol; in tpacket_snd()
2073 addr = saddr->sll_addr; in tpacket_snd()
2074 dev = dev_get_by_index(sock_net(&po->sk), saddr->sll_ifindex); in tpacket_snd()
2078 err = -ENXIO; in tpacket_snd()
2082 reserve = dev->hard_header_len; in tpacket_snd()
2084 err = -ENETDOWN; in tpacket_snd()
2085 if (unlikely(!(dev->flags & IFF_UP))) in tpacket_snd()
2088 size_max = po->tx_ring.frame_size in tpacket_snd()
2089 - (po->tp_hdrlen - sizeof(struct sockaddr_ll)); in tpacket_snd()
2091 if (size_max > dev->mtu + reserve) in tpacket_snd()
2092 size_max = dev->mtu + reserve; in tpacket_snd()
2095 ph = packet_current_frame(po, &po->tx_ring, in tpacket_snd()
2105 tlen = dev->needed_tailroom; in tpacket_snd()
2106 skb = sock_alloc_send_skb(&po->sk, in tpacket_snd()
2117 if (po->tp_loss) { in tpacket_snd()
2120 packet_increment_head(&po->tx_ring); in tpacket_snd()
2130 skb->destructor = tpacket_destruct_skb; in tpacket_snd()
2132 atomic_inc(&po->tx_ring.pending); in tpacket_snd()
2150 packet_increment_head(&po->tx_ring); in tpacket_snd()
2153 ((!(msg->msg_flags & MSG_DONTWAIT)) && in tpacket_snd()
2154 (atomic_read(&po->tx_ring.pending)))) in tpacket_snd()
2167 mutex_unlock(&po->pg_vec_lock); in tpacket_snd()
2182 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock, in packet_alloc_skb()
2189 skb->data_len = len - linear; in packet_alloc_skb()
2190 skb->len += len - linear; in packet_alloc_skb()
2198 struct sock *sk = sock->sk; in packet_snd()
2199 struct sockaddr_ll *saddr = (struct sockaddr_ll *)msg->msg_name; in packet_snd()
2218 dev = po->prot_hook.dev; in packet_snd()
2219 proto = po->num; in packet_snd()
2222 err = -EINVAL; in packet_snd()
2223 if (msg->msg_namelen < sizeof(struct sockaddr_ll)) in packet_snd()
2225 if (msg->msg_namelen < (saddr->sll_halen + offsetof(struct sockaddr_ll, sll_addr))) in packet_snd()
2227 proto = saddr->sll_protocol; in packet_snd()
2228 addr = saddr->sll_addr; in packet_snd()
2229 dev = dev_get_by_index(sock_net(sk), saddr->sll_ifindex); in packet_snd()
2233 err = -ENXIO; in packet_snd()
2236 if (sock->type == SOCK_RAW) in packet_snd()
2237 reserve = dev->hard_header_len; in packet_snd()
2239 err = -ENETDOWN; in packet_snd()
2240 if (!(dev->flags & IFF_UP)) in packet_snd()
2243 if (po->has_vnet_hdr) { in packet_snd()
2246 err = -EINVAL; in packet_snd()
2250 len -= vnet_hdr_len; in packet_snd()
2252 err = memcpy_fromiovec((void *)&vnet_hdr, msg->msg_iov, in packet_snd()
2263 err = -EINVAL; in packet_snd()
2291 err = -EMSGSIZE; in packet_snd()
2292 if (!gso_type && (len > dev->mtu + reserve + VLAN_HLEN)) in packet_snd()
2295 err = -ENOBUFS; in packet_snd()
2297 tlen = dev->needed_tailroom; in packet_snd()
2299 msg->msg_flags & MSG_DONTWAIT, &err); in packet_snd()
2305 err = -EINVAL; in packet_snd()
2306 if (sock->type == SOCK_DGRAM && in packet_snd()
2310 /* Returns -EFAULT on error */ in packet_snd()
2311 err = skb_copy_datagram_from_iovec(skb, offset, msg->msg_iov, 0, len); in packet_snd()
2314 err = sock_tx_timestamp(sk, &skb_shinfo(skb)->tx_flags); in packet_snd()
2318 if (!gso_type && (len > dev->mtu + reserve)) { in packet_snd()
2320 * double-check this now that we have the actual in packet_snd()
2326 if (ehdr->h_proto != htons(ETH_P_8021Q)) { in packet_snd()
2327 err = -EMSGSIZE; in packet_snd()
2332 skb->protocol = proto; in packet_snd()
2333 skb->dev = dev; in packet_snd()
2334 skb->priority = sk->sk_priority; in packet_snd()
2335 skb->mark = sk->sk_mark; in packet_snd()
2337 if (po->has_vnet_hdr) { in packet_snd()
2341 err = -EINVAL; in packet_snd()
2346 skb_shinfo(skb)->gso_size = vnet_hdr.gso_size; in packet_snd()
2347 skb_shinfo(skb)->gso_type = gso_type; in packet_snd()
2350 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY; in packet_snd()
2351 skb_shinfo(skb)->gso_segs = 0; in packet_snd()
2381 struct sock *sk = sock->sk; in packet_sendmsg()
2383 if (po->tx_ring.pg_vec) in packet_sendmsg()
2396 struct sock *sk = sock->sk; in packet_release()
2407 spin_lock_bh(&net->packet.sklist_lock); in packet_release()
2409 sock_prot_inuse_add(net, sk->sk_prot, -1); in packet_release()
2410 spin_unlock_bh(&net->packet.sklist_lock); in packet_release()
2412 spin_lock(&po->bind_lock); in packet_release()
2414 if (po->prot_hook.dev) { in packet_release()
2415 dev_put(po->prot_hook.dev); in packet_release()
2416 po->prot_hook.dev = NULL; in packet_release()
2418 spin_unlock(&po->bind_lock); in packet_release()
2424 if (po->rx_ring.pg_vec) in packet_release()
2427 if (po->tx_ring.pg_vec) in packet_release()
2437 sock->sk = NULL; in packet_release()
2441 skb_queue_purge(&sk->sk_receive_queue); in packet_release()
2456 if (po->fanout) { in packet_do_bind()
2460 return -EINVAL; in packet_do_bind()
2465 spin_lock(&po->bind_lock); in packet_do_bind()
2467 po->num = protocol; in packet_do_bind()
2468 po->prot_hook.type = protocol; in packet_do_bind()
2469 if (po->prot_hook.dev) in packet_do_bind()
2470 dev_put(po->prot_hook.dev); in packet_do_bind()
2471 po->prot_hook.dev = dev; in packet_do_bind()
2473 po->ifindex = dev ? dev->ifindex : 0; in packet_do_bind()
2478 if (!dev || (dev->flags & IFF_UP)) { in packet_do_bind()
2481 sk->sk_err = ENETDOWN; in packet_do_bind()
2483 sk->sk_error_report(sk); in packet_do_bind()
2487 spin_unlock(&po->bind_lock); in packet_do_bind()
2499 struct sock *sk = sock->sk; in packet_bind_spkt()
2502 int err = -ENODEV; in packet_bind_spkt()
2509 return -EINVAL; in packet_bind_spkt()
2510 strlcpy(name, uaddr->sa_data, sizeof(name)); in packet_bind_spkt()
2514 err = packet_do_bind(sk, dev, pkt_sk(sk)->num); in packet_bind_spkt()
2521 struct sock *sk = sock->sk; in packet_bind()
2531 return -EINVAL; in packet_bind()
2532 if (sll->sll_family != AF_PACKET) in packet_bind()
2533 return -EINVAL; in packet_bind()
2535 if (sll->sll_ifindex) { in packet_bind()
2536 err = -ENODEV; in packet_bind()
2537 dev = dev_get_by_index(sock_net(sk), sll->sll_ifindex); in packet_bind()
2541 err = packet_do_bind(sk, dev, sll->sll_protocol ? : pkt_sk(sk)->num); in packet_bind()
2566 return -EPERM; in packet_create()
2567 if (sock->type != SOCK_DGRAM && sock->type != SOCK_RAW && in packet_create()
2568 sock->type != SOCK_PACKET) in packet_create()
2569 return -ESOCKTNOSUPPORT; in packet_create()
2571 sock->state = SS_UNCONNECTED; in packet_create()
2573 err = -ENOBUFS; in packet_create()
2578 sock->ops = &packet_ops; in packet_create()
2579 if (sock->type == SOCK_PACKET) in packet_create()
2580 sock->ops = &packet_ops_spkt; in packet_create()
2585 sk->sk_family = PF_PACKET; in packet_create()
2586 po->num = proto; in packet_create()
2588 sk->sk_destruct = packet_sock_destruct; in packet_create()
2595 spin_lock_init(&po->bind_lock); in packet_create()
2596 mutex_init(&po->pg_vec_lock); in packet_create()
2597 po->prot_hook.func = packet_rcv; in packet_create()
2599 if (sock->type == SOCK_PACKET) in packet_create()
2600 po->prot_hook.func = packet_rcv_spkt; in packet_create()
2602 po->prot_hook.af_packet_priv = sk; in packet_create()
2605 po->prot_hook.type = proto; in packet_create()
2609 spin_lock_bh(&net->packet.sklist_lock); in packet_create()
2610 sk_add_node_rcu(sk, &net->packet.sklist); in packet_create()
2612 spin_unlock_bh(&net->packet.sklist_lock); in packet_create()
2625 err = -EAGAIN; in packet_recv_error()
2626 skb = skb_dequeue(&sk->sk_error_queue); in packet_recv_error()
2630 copied = skb->len; in packet_recv_error()
2632 msg->msg_flags |= MSG_TRUNC; in packet_recv_error()
2635 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied); in packet_recv_error()
2643 sizeof(serr->ee), &serr->ee); in packet_recv_error()
2645 msg->msg_flags |= MSG_ERRQUEUE; in packet_recv_error()
2649 spin_lock_bh(&sk->sk_error_queue.lock); in packet_recv_error()
2650 sk->sk_err = 0; in packet_recv_error()
2651 if ((skb2 = skb_peek(&sk->sk_error_queue)) != NULL) { in packet_recv_error()
2652 sk->sk_err = SKB_EXT_ERR(skb2)->ee.ee_errno; in packet_recv_error()
2653 spin_unlock_bh(&sk->sk_error_queue.lock); in packet_recv_error()
2654 sk->sk_error_report(sk); in packet_recv_error()
2656 spin_unlock_bh(&sk->sk_error_queue.lock); in packet_recv_error()
2672 struct sock *sk = sock->sk; in packet_recvmsg()
2678 err = -EINVAL; in packet_recvmsg()
2684 if (pkt_sk(sk)->ifindex < 0) in packet_recvmsg()
2685 return -ENODEV; in packet_recvmsg()
2695 * of horrible races and re-entrancy so we can forget about it in packet_recvmsg()
2713 if (pkt_sk(sk)->has_vnet_hdr) { in packet_recvmsg()
2716 err = -EINVAL; in packet_recvmsg()
2721 len -= vnet_hdr_len; in packet_recvmsg()
2728 vnet_hdr.gso_size = sinfo->gso_size; in packet_recvmsg()
2729 if (sinfo->gso_type & SKB_GSO_TCPV4) in packet_recvmsg()
2731 else if (sinfo->gso_type & SKB_GSO_TCPV6) in packet_recvmsg()
2733 else if (sinfo->gso_type & SKB_GSO_UDP) in packet_recvmsg()
2735 else if (sinfo->gso_type & SKB_GSO_FCOE) in packet_recvmsg()
2739 if (sinfo->gso_type & SKB_GSO_TCP_ECN) in packet_recvmsg()
2744 if (skb->ip_summed == CHECKSUM_PARTIAL) { in packet_recvmsg()
2747 vnet_hdr.csum_offset = skb->csum_offset; in packet_recvmsg()
2748 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) { in packet_recvmsg()
2752 err = memcpy_toiovec(msg->msg_iov, (void *)&vnet_hdr, in packet_recvmsg()
2763 sll = &PACKET_SKB_CB(skb)->sa.ll; in packet_recvmsg()
2764 if (sock->type == SOCK_PACKET) in packet_recvmsg()
2765 msg->msg_namelen = sizeof(struct sockaddr_pkt); in packet_recvmsg()
2767 msg->msg_namelen = sll->sll_halen + offsetof(struct sockaddr_ll, sll_addr); in packet_recvmsg()
2774 copied = skb->len; in packet_recvmsg()
2777 msg->msg_flags |= MSG_TRUNC; in packet_recvmsg()
2780 err = skb_copy_datagram_iovec(skb, 0, msg->msg_iov, copied); in packet_recvmsg()
2786 if (msg->msg_name) in packet_recvmsg()
2787 memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa, in packet_recvmsg()
2788 msg->msg_namelen); in packet_recvmsg()
2790 if (pkt_sk(sk)->auxdata) { in packet_recvmsg()
2794 if (skb->ip_summed == CHECKSUM_PARTIAL) in packet_recvmsg()
2796 aux.tp_len = PACKET_SKB_CB(skb)->origlen; in packet_recvmsg()
2797 aux.tp_snaplen = skb->len; in packet_recvmsg()
2812 * hides all the races and re-entrancy issues from us. in packet_recvmsg()
2814 err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied); in packet_recvmsg()
2826 struct sock *sk = sock->sk; in packet_getname_spkt()
2829 return -EOPNOTSUPP; in packet_getname_spkt()
2831 uaddr->sa_family = AF_PACKET; in packet_getname_spkt()
2833 dev = dev_get_by_index_rcu(sock_net(sk), pkt_sk(sk)->ifindex); in packet_getname_spkt()
2835 strncpy(uaddr->sa_data, dev->name, 14); in packet_getname_spkt()
2837 memset(uaddr->sa_data, 0, 14); in packet_getname_spkt()
2848 struct sock *sk = sock->sk; in packet_getname()
2853 return -EOPNOTSUPP; in packet_getname()
2855 sll->sll_family = AF_PACKET; in packet_getname()
2856 sll->sll_ifindex = po->ifindex; in packet_getname()
2857 sll->sll_protocol = po->num; in packet_getname()
2858 sll->sll_pkttype = 0; in packet_getname()
2860 dev = dev_get_by_index_rcu(sock_net(sk), po->ifindex); in packet_getname()
2862 sll->sll_hatype = dev->type; in packet_getname()
2863 sll->sll_halen = dev->addr_len; in packet_getname()
2864 memcpy(sll->sll_addr, dev->dev_addr, dev->addr_len); in packet_getname()
2866 sll->sll_hatype = 0; /* Bad: we have no ARPHRD_UNSPEC */ in packet_getname()
2867 sll->sll_halen = 0; in packet_getname()
2870 *uaddr_len = offsetof(struct sockaddr_ll, sll_addr) + sll->sll_halen; in packet_getname()
2878 switch (i->type) { in packet_dev_mc()
2880 if (i->alen != dev->addr_len) in packet_dev_mc()
2881 return -EINVAL; in packet_dev_mc()
2883 return dev_mc_add(dev, i->addr); in packet_dev_mc()
2885 return dev_mc_del(dev, i->addr); in packet_dev_mc()
2894 if (i->alen != dev->addr_len) in packet_dev_mc()
2895 return -EINVAL; in packet_dev_mc()
2897 return dev_uc_add(dev, i->addr); in packet_dev_mc()
2899 return dev_uc_del(dev, i->addr); in packet_dev_mc()
2909 for ( ; i; i = i->next) { in packet_dev_mclist()
2910 if (i->ifindex == dev->ifindex) in packet_dev_mclist()
2924 err = -ENODEV; in packet_mc_add()
2925 dev = __dev_get_by_index(sock_net(sk), mreq->mr_ifindex); in packet_mc_add()
2929 err = -EINVAL; in packet_mc_add()
2930 if (mreq->mr_alen > dev->addr_len) in packet_mc_add()
2933 err = -ENOBUFS; in packet_mc_add()
2939 for (ml = po->mclist; ml; ml = ml->next) { in packet_mc_add()
2940 if (ml->ifindex == mreq->mr_ifindex && in packet_mc_add()
2941 ml->type == mreq->mr_type && in packet_mc_add()
2942 ml->alen == mreq->mr_alen && in packet_mc_add()
2943 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) { in packet_mc_add()
2944 ml->count++; in packet_mc_add()
2951 i->type = mreq->mr_type; in packet_mc_add()
2952 i->ifindex = mreq->mr_ifindex; in packet_mc_add()
2953 i->alen = mreq->mr_alen; in packet_mc_add()
2954 memcpy(i->addr, mreq->mr_address, i->alen); in packet_mc_add()
2955 i->count = 1; in packet_mc_add()
2956 i->next = po->mclist; in packet_mc_add()
2957 po->mclist = i; in packet_mc_add()
2960 po->mclist = i->next; in packet_mc_add()
2975 for (mlp = &pkt_sk(sk)->mclist; (ml = *mlp) != NULL; mlp = &ml->next) { in packet_mc_drop()
2976 if (ml->ifindex == mreq->mr_ifindex && in packet_mc_drop()
2977 ml->type == mreq->mr_type && in packet_mc_drop()
2978 ml->alen == mreq->mr_alen && in packet_mc_drop()
2979 memcmp(ml->addr, mreq->mr_address, ml->alen) == 0) { in packet_mc_drop()
2980 if (--ml->count == 0) { in packet_mc_drop()
2982 *mlp = ml->next; in packet_mc_drop()
2983 dev = __dev_get_by_index(sock_net(sk), ml->ifindex); in packet_mc_drop()
2985 packet_dev_mc(dev, ml, -1); in packet_mc_drop()
2993 return -EADDRNOTAVAIL; in packet_mc_drop()
3001 if (!po->mclist) in packet_flush_mclist()
3005 while ((ml = po->mclist) != NULL) { in packet_flush_mclist()
3008 po->mclist = ml->next; in packet_flush_mclist()
3009 dev = __dev_get_by_index(sock_net(sk), ml->ifindex); in packet_flush_mclist()
3011 packet_dev_mc(dev, ml, -1); in packet_flush_mclist()
3020 struct sock *sk = sock->sk; in packet_setsockopt()
3025 return -ENOPROTOOPT; in packet_setsockopt()
3035 return -EINVAL; in packet_setsockopt()
3039 return -EFAULT; in packet_setsockopt()
3041 return -EINVAL; in packet_setsockopt()
3055 switch (po->tp_version) { in packet_setsockopt()
3066 return -EINVAL; in packet_setsockopt()
3067 if (pkt_sk(sk)->has_vnet_hdr) in packet_setsockopt()
3068 return -EINVAL; in packet_setsockopt()
3070 return -EFAULT; in packet_setsockopt()
3079 return -EINVAL; in packet_setsockopt()
3081 return -EFAULT; in packet_setsockopt()
3083 pkt_sk(sk)->copy_thresh = val; in packet_setsockopt()
3091 return -EINVAL; in packet_setsockopt()
3092 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) in packet_setsockopt()
3093 return -EBUSY; in packet_setsockopt()
3095 return -EFAULT; in packet_setsockopt()
3100 po->tp_version = val; in packet_setsockopt()
3103 return -EINVAL; in packet_setsockopt()
3111 return -EINVAL; in packet_setsockopt()
3112 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) in packet_setsockopt()
3113 return -EBUSY; in packet_setsockopt()
3115 return -EFAULT; in packet_setsockopt()
3116 po->tp_reserve = val; in packet_setsockopt()
3124 return -EINVAL; in packet_setsockopt()
3125 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) in packet_setsockopt()
3126 return -EBUSY; in packet_setsockopt()
3128 return -EFAULT; in packet_setsockopt()
3129 po->tp_loss = !!val; in packet_setsockopt()
3137 return -EINVAL; in packet_setsockopt()
3139 return -EFAULT; in packet_setsockopt()
3141 po->auxdata = !!val; in packet_setsockopt()
3149 return -EINVAL; in packet_setsockopt()
3151 return -EFAULT; in packet_setsockopt()
3153 po->origdev = !!val; in packet_setsockopt()
3160 if (sock->type != SOCK_RAW) in packet_setsockopt()
3161 return -EINVAL; in packet_setsockopt()
3162 if (po->rx_ring.pg_vec || po->tx_ring.pg_vec) in packet_setsockopt()
3163 return -EBUSY; in packet_setsockopt()
3165 return -EINVAL; in packet_setsockopt()
3167 return -EFAULT; in packet_setsockopt()
3169 po->has_vnet_hdr = !!val; in packet_setsockopt()
3177 return -EINVAL; in packet_setsockopt()
3179 return -EFAULT; in packet_setsockopt()
3181 po->tp_tstamp = val; in packet_setsockopt()
3189 return -EINVAL; in packet_setsockopt()
3191 return -EFAULT; in packet_setsockopt()
3196 return -ENOPROTOOPT; in packet_setsockopt()
3205 struct sock *sk = sock->sk; in packet_getsockopt()
3212 return -ENOPROTOOPT; in packet_getsockopt()
3215 return -EFAULT; in packet_getsockopt()
3218 return -EINVAL; in packet_getsockopt()
3222 if (po->tp_version == TPACKET_V3) { in packet_getsockopt()
3228 spin_lock_bh(&sk->sk_receive_queue.lock); in packet_getsockopt()
3229 if (po->tp_version == TPACKET_V3) { in packet_getsockopt()
3230 memcpy(&st_u.stats3, &po->stats, in packet_getsockopt()
3233 po->stats_u.stats3.tp_freeze_q_cnt; in packet_getsockopt()
3234 st_u.stats3.tp_packets += po->stats.tp_drops; in packet_getsockopt()
3237 st = po->stats; in packet_getsockopt()
3241 memset(&po->stats, 0, sizeof(st)); in packet_getsockopt()
3242 spin_unlock_bh(&sk->sk_receive_queue.lock); in packet_getsockopt()
3247 val = po->auxdata; in packet_getsockopt()
3254 val = po->origdev; in packet_getsockopt()
3261 val = po->has_vnet_hdr; in packet_getsockopt()
3268 val = po->tp_version; in packet_getsockopt()
3275 return -EFAULT; in packet_getsockopt()
3287 return -EINVAL; in packet_getsockopt()
3294 val = po->tp_reserve; in packet_getsockopt()
3300 val = po->tp_loss; in packet_getsockopt()
3306 val = po->tp_tstamp; in packet_getsockopt()
3312 val = (po->fanout ? in packet_getsockopt()
3313 ((u32)po->fanout->id | in packet_getsockopt()
3314 ((u32)po->fanout->type << 16)) : in packet_getsockopt()
3319 return -ENOPROTOOPT; in packet_getsockopt()
3323 return -EFAULT; in packet_getsockopt()
3325 return -EFAULT; in packet_getsockopt()
3338 sk_for_each_rcu(sk, node, &net->packet.sklist) { in packet_notifier()
3343 if (po->mclist) in packet_notifier()
3344 packet_dev_mclist(dev, po->mclist, -1); in packet_notifier()
3348 if (dev->ifindex == po->ifindex) { in packet_notifier()
3349 spin_lock(&po->bind_lock); in packet_notifier()
3350 if (po->running) { in packet_notifier()
3352 sk->sk_err = ENETDOWN; in packet_notifier()
3354 sk->sk_error_report(sk); in packet_notifier()
3357 po->ifindex = -1; in packet_notifier()
3358 if (po->prot_hook.dev) in packet_notifier()
3359 dev_put(po->prot_hook.dev); in packet_notifier()
3360 po->prot_hook.dev = NULL; in packet_notifier()
3362 spin_unlock(&po->bind_lock); in packet_notifier()
3366 if (dev->ifindex == po->ifindex) { in packet_notifier()
3367 spin_lock(&po->bind_lock); in packet_notifier()
3368 if (po->num) in packet_notifier()
3370 spin_unlock(&po->bind_lock); in packet_notifier()
3383 struct sock *sk = sock->sk; in packet_ioctl()
3397 spin_lock_bh(&sk->sk_receive_queue.lock); in packet_ioctl()
3398 skb = skb_peek(&sk->sk_receive_queue); in packet_ioctl()
3400 amount = skb->len; in packet_ioctl()
3401 spin_unlock_bh(&sk->sk_receive_queue.lock); in packet_ioctl()
3428 return -ENOIOCTLCMD; in packet_ioctl()
3436 struct sock *sk = sock->sk; in packet_poll()
3440 spin_lock_bh(&sk->sk_receive_queue.lock); in packet_poll()
3441 if (po->rx_ring.pg_vec) { in packet_poll()
3442 if (!packet_previous_rx_frame(po, &po->rx_ring, in packet_poll()
3446 spin_unlock_bh(&sk->sk_receive_queue.lock); in packet_poll()
3447 spin_lock_bh(&sk->sk_write_queue.lock); in packet_poll()
3448 if (po->tx_ring.pg_vec) { in packet_poll()
3449 if (packet_current_frame(po, &po->tx_ring, TP_STATUS_AVAILABLE)) in packet_poll()
3452 spin_unlock_bh(&sk->sk_write_queue.lock); in packet_poll()
3463 struct file *file = vma->vm_file; in packet_mm_open()
3464 struct socket *sock = file->private_data; in packet_mm_open()
3465 struct sock *sk = sock->sk; in packet_mm_open()
3468 atomic_inc(&pkt_sk(sk)->mapped); in packet_mm_open()
3473 struct file *file = vma->vm_file; in packet_mm_close()
3474 struct socket *sock = file->private_data; in packet_mm_close()
3475 struct sock *sk = sock->sk; in packet_mm_close()
3478 atomic_dec(&pkt_sk(sk)->mapped); in packet_mm_close()
3539 unsigned int block_nr = req->tp_block_nr; in alloc_pg_vec()
3571 int err = -EINVAL; in packet_set_ring()
3573 struct tpacket_req *req = &req_u->req; in packet_set_ring()
3575 /* Opening a Tx-ring is NOT supported in TPACKET_V3 */ in packet_set_ring()
3576 if (!closing && tx_ring && (po->tp_version > TPACKET_V2)) { in packet_set_ring()
3577 WARN(1, "Tx-ring is not supported.\n"); in packet_set_ring()
3581 rb = tx_ring ? &po->tx_ring : &po->rx_ring; in packet_set_ring()
3582 rb_queue = tx_ring ? &sk->sk_write_queue : &sk->sk_receive_queue; in packet_set_ring()
3584 err = -EBUSY; in packet_set_ring()
3586 if (atomic_read(&po->mapped)) in packet_set_ring()
3588 if (atomic_read(&rb->pending)) in packet_set_ring()
3592 if (req->tp_block_nr) { in packet_set_ring()
3594 err = -EBUSY; in packet_set_ring()
3595 if (unlikely(rb->pg_vec)) in packet_set_ring()
3598 switch (po->tp_version) { in packet_set_ring()
3600 po->tp_hdrlen = TPACKET_HDRLEN; in packet_set_ring()
3603 po->tp_hdrlen = TPACKET2_HDRLEN; in packet_set_ring()
3606 po->tp_hdrlen = TPACKET3_HDRLEN; in packet_set_ring()
3610 err = -EINVAL; in packet_set_ring()
3611 if (unlikely((int)req->tp_block_size <= 0)) in packet_set_ring()
3613 if (unlikely(req->tp_block_size & (PAGE_SIZE - 1))) in packet_set_ring()
3615 if (unlikely(req->tp_frame_size < po->tp_hdrlen + in packet_set_ring()
3616 po->tp_reserve)) in packet_set_ring()
3618 if (unlikely(req->tp_frame_size & (TPACKET_ALIGNMENT - 1))) in packet_set_ring()
3621 rb->frames_per_block = req->tp_block_size/req->tp_frame_size; in packet_set_ring()
3622 if (unlikely(rb->frames_per_block <= 0)) in packet_set_ring()
3624 if (unlikely((rb->frames_per_block * req->tp_block_nr) != in packet_set_ring()
3625 req->tp_frame_nr)) in packet_set_ring()
3628 err = -ENOMEM; in packet_set_ring()
3629 order = get_order(req->tp_block_size); in packet_set_ring()
3633 switch (po->tp_version) { in packet_set_ring()
3647 err = -EINVAL; in packet_set_ring()
3648 if (unlikely(req->tp_frame_nr)) in packet_set_ring()
3655 spin_lock(&po->bind_lock); in packet_set_ring()
3656 was_running = po->running; in packet_set_ring()
3657 num = po->num; in packet_set_ring()
3659 po->num = 0; in packet_set_ring()
3662 spin_unlock(&po->bind_lock); in packet_set_ring()
3666 err = -EBUSY; in packet_set_ring()
3667 mutex_lock(&po->pg_vec_lock); in packet_set_ring()
3668 if (closing || atomic_read(&po->mapped) == 0) { in packet_set_ring()
3670 spin_lock_bh(&rb_queue->lock); in packet_set_ring()
3671 swap(rb->pg_vec, pg_vec); in packet_set_ring()
3672 rb->frame_max = (req->tp_frame_nr - 1); in packet_set_ring()
3673 rb->head = 0; in packet_set_ring()
3674 rb->frame_size = req->tp_frame_size; in packet_set_ring()
3675 spin_unlock_bh(&rb_queue->lock); in packet_set_ring()
3677 swap(rb->pg_vec_order, order); in packet_set_ring()
3678 swap(rb->pg_vec_len, req->tp_block_nr); in packet_set_ring()
3680 rb->pg_vec_pages = req->tp_block_size/PAGE_SIZE; in packet_set_ring()
3681 po->prot_hook.func = (po->rx_ring.pg_vec) ? in packet_set_ring()
3684 if (atomic_read(&po->mapped)) in packet_set_ring()
3686 atomic_read(&po->mapped)); in packet_set_ring()
3688 mutex_unlock(&po->pg_vec_lock); in packet_set_ring()
3690 spin_lock(&po->bind_lock); in packet_set_ring()
3692 po->num = num; in packet_set_ring()
3695 spin_unlock(&po->bind_lock); in packet_set_ring()
3696 if (closing && (po->tp_version > TPACKET_V2)) { in packet_set_ring()
3697 /* Because we don't support block-based V3 on tx-ring */ in packet_set_ring()
3704 free_pg_vec(pg_vec, order, req->tp_block_nr); in packet_set_ring()
3712 struct sock *sk = sock->sk; in packet_mmap()
3717 int err = -EINVAL; in packet_mmap()
3720 if (vma->vm_pgoff) in packet_mmap()
3721 return -EINVAL; in packet_mmap()
3723 mutex_lock(&po->pg_vec_lock); in packet_mmap()
3726 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) { in packet_mmap()
3727 if (rb->pg_vec) { in packet_mmap()
3728 expected_size += rb->pg_vec_len in packet_mmap()
3729 * rb->pg_vec_pages in packet_mmap()
3737 size = vma->vm_end - vma->vm_start; in packet_mmap()
3741 start = vma->vm_start; in packet_mmap()
3742 for (rb = &po->rx_ring; rb <= &po->tx_ring; rb++) { in packet_mmap()
3743 if (rb->pg_vec == NULL) in packet_mmap()
3746 for (i = 0; i < rb->pg_vec_len; i++) { in packet_mmap()
3748 void *kaddr = rb->pg_vec[i].buffer; in packet_mmap()
3751 for (pg_num = 0; pg_num < rb->pg_vec_pages; pg_num++) { in packet_mmap()
3762 atomic_inc(&po->mapped); in packet_mmap()
3763 vma->vm_ops = &packet_mmap_ops; in packet_mmap()
3767 mutex_unlock(&po->pg_vec_lock); in packet_mmap()
3831 return seq_hlist_start_head_rcu(&net->packet.sklist, *pos); in packet_seq_start()
3837 return seq_hlist_next_rcu(v, &net->packet.sklist, pos); in packet_seq_next()
3855 "%pK %-6d %-4d %04x %-5d %1d %-6u %-6u %-6lu\n", in packet_seq_show()
3857 atomic_read(&s->sk_refcnt), in packet_seq_show()
3858 s->sk_type, in packet_seq_show()
3859 ntohs(po->num), in packet_seq_show()
3860 po->ifindex, in packet_seq_show()
3861 po->running, in packet_seq_show()
3862 atomic_read(&s->sk_rmem_alloc), in packet_seq_show()
3895 spin_lock_init(&net->packet.sklist_lock); in packet_net_init()
3896 INIT_HLIST_HEAD(&net->packet.sklist); in packet_net_init()
3899 return -ENOMEM; in packet_net_init()