13 void xp_add_xsk(struct xsk_buff_pool *pool, struct xdp_sock *xs)  in xp_add_xsk()  argument
20 	spin_lock_irqsave(&pool->xsk_tx_list_lock, flags);  in xp_add_xsk()
21 	list_add_rcu(&xs->tx_list, &pool->xsk_tx_list);  in xp_add_xsk()
22 	spin_unlock_irqrestore(&pool->xsk_tx_list_lock, flags);  in xp_add_xsk()
25 void xp_del_xsk(struct xsk_buff_pool *pool, struct xdp_sock *xs)  in xp_del_xsk()  argument
32 	spin_lock_irqsave(&pool->xsk_tx_list_lock, flags);  in xp_del_xsk()
34 	spin_unlock_irqrestore(&pool->xsk_tx_list_lock, flags);  in xp_del_xsk()
37 void xp_destroy(struct xsk_buff_pool *pool)  in xp_destroy()  argument
39 	if (!pool)  in xp_destroy()
42 	kvfree(pool->tx_descs);  in xp_destroy()
43 	kvfree(pool->heads);  in xp_destroy()
44 	kvfree(pool);  in xp_destroy()
47 int xp_alloc_tx_descs(struct xsk_buff_pool *pool, struct xdp_sock *xs)  in xp_alloc_tx_descs()  argument
49 	pool->tx_descs = kvcalloc(xs->tx->nentries, sizeof(*pool->tx_descs),  in xp_alloc_tx_descs()
51 	if (!pool->tx_descs)  in xp_alloc_tx_descs()
61 	struct xsk_buff_pool *pool;  in xp_create_and_assign_umem()  local
66 	pool = kvzalloc(struct_size(pool, free_heads, entries),	GFP_KERNEL);  in xp_create_and_assign_umem()
67 	if (!pool)  in xp_create_and_assign_umem()
70 	pool->heads = kvcalloc(umem->chunks, sizeof(*pool->heads), GFP_KERNEL);  in xp_create_and_assign_umem()
71 	if (!pool->heads)  in xp_create_and_assign_umem()
75 		if (xp_alloc_tx_descs(pool, xs))  in xp_create_and_assign_umem()
78 	pool->chunk_mask = ~((u64)umem->chunk_size - 1);  in xp_create_and_assign_umem()
79 	pool->addrs_cnt = umem->size;  in xp_create_and_assign_umem()
80 	pool->heads_cnt = umem->chunks;  in xp_create_and_assign_umem()
81 	pool->free_heads_cnt = umem->chunks;  in xp_create_and_assign_umem()
82 	pool->headroom = umem->headroom;  in xp_create_and_assign_umem()
83 	pool->chunk_size = umem->chunk_size;  in xp_create_and_assign_umem()
84 	pool->chunk_shift = ffs(umem->chunk_size) - 1;  in xp_create_and_assign_umem()
85 	pool->unaligned = unaligned;  in xp_create_and_assign_umem()
86 	pool->frame_len = umem->chunk_size - umem->headroom -  in xp_create_and_assign_umem()
88 	pool->umem = umem;  in xp_create_and_assign_umem()
89 	pool->addrs = umem->addrs;  in xp_create_and_assign_umem()
90 	pool->tx_metadata_len = umem->tx_metadata_len;  in xp_create_and_assign_umem()
91 	pool->tx_sw_csum = umem->flags & XDP_UMEM_TX_SW_CSUM;  in xp_create_and_assign_umem()
92 	spin_lock_init(&pool->rx_lock);  in xp_create_and_assign_umem()
93 	INIT_LIST_HEAD(&pool->free_list);  in xp_create_and_assign_umem()
94 	INIT_LIST_HEAD(&pool->xskb_list);  in xp_create_and_assign_umem()
95 	INIT_LIST_HEAD(&pool->xsk_tx_list);  in xp_create_and_assign_umem()
96 	spin_lock_init(&pool->xsk_tx_list_lock);  in xp_create_and_assign_umem()
97 	spin_lock_init(&pool->cq_lock);  in xp_create_and_assign_umem()
98 	refcount_set(&pool->users, 1);  in xp_create_and_assign_umem()
100 	pool->fq = xs->fq_tmp;  in xp_create_and_assign_umem()
101 	pool->cq = xs->cq_tmp;  in xp_create_and_assign_umem()
103 	for (i = 0; i < pool->free_heads_cnt; i++) {  in xp_create_and_assign_umem()
104 		xskb = &pool->heads[i];  in xp_create_and_assign_umem()
105 		xskb->pool = pool;  in xp_create_and_assign_umem()
108 		if (pool->unaligned)  in xp_create_and_assign_umem()
109 			pool->free_heads[i] = xskb;  in xp_create_and_assign_umem()
111 			xp_init_xskb_addr(xskb, pool, (u64)i * pool->chunk_size);  in xp_create_and_assign_umem()
114 	return pool;  in xp_create_and_assign_umem()
117 	xp_destroy(pool);  in xp_create_and_assign_umem()
121 void xp_set_rxq_info(struct xsk_buff_pool *pool, struct xdp_rxq_info *rxq)  in xp_set_rxq_info()  argument
125 	for (i = 0; i < pool->heads_cnt; i++)  in xp_set_rxq_info()
126 		pool->heads[i].xdp.rxq = rxq;  in xp_set_rxq_info()
130 void xp_fill_cb(struct xsk_buff_pool *pool, struct xsk_cb_desc *desc)  in xp_fill_cb()  argument
134 	for (i = 0; i < pool->heads_cnt; i++) {  in xp_fill_cb()
135 		struct xdp_buff_xsk *xskb = &pool->heads[i];  in xp_fill_cb()
142 static void xp_disable_drv_zc(struct xsk_buff_pool *pool)  in xp_disable_drv_zc()  argument
149 	if (pool->umem->zc) {  in xp_disable_drv_zc()
151 		bpf.xsk.pool = NULL;  in xp_disable_drv_zc()
152 		bpf.xsk.queue_id = pool->queue_id;  in xp_disable_drv_zc()
154 		err = pool->netdev->netdev_ops->ndo_bpf(pool->netdev, &bpf);  in xp_disable_drv_zc()
165 int xp_assign_dev(struct xsk_buff_pool *pool,  in xp_assign_dev()  argument
183 	pool->netdev = netdev;  in xp_assign_dev()
184 	pool->queue_id = queue_id;  in xp_assign_dev()
185 	err = xsk_reg_pool_at_qid(netdev, pool, queue_id);  in xp_assign_dev()
190 		pool->umem->flags |= XDP_UMEM_SG_FLAG;  in xp_assign_dev()
193 		pool->uses_need_wakeup = true;  in xp_assign_dev()
198 	pool->cached_need_wakeup = XDP_WAKEUP_TX;  in xp_assign_dev()
222 	bpf.xsk.pool = pool;  in xp_assign_dev()
230 	if (!pool->dma_pages) {  in xp_assign_dev()
235 	pool->umem->zc = true;  in xp_assign_dev()
236 	pool->xdp_zc_max_segs = netdev->xdp_zc_max_segs;  in xp_assign_dev()
240 	xp_disable_drv_zc(pool);  in xp_assign_dev()
251 int xp_assign_dev_shared(struct xsk_buff_pool *pool, struct xdp_sock *umem_xs,  in xp_assign_dev_shared()  argument
258 	if (!pool->fq || !pool->cq)  in xp_assign_dev_shared()
262 	if (umem_xs->pool->uses_need_wakeup)  in xp_assign_dev_shared()
265 	return xp_assign_dev(pool, dev, queue_id, flags);  in xp_assign_dev_shared()
268 void xp_clear_dev(struct xsk_buff_pool *pool)  in xp_clear_dev()  argument
270 	if (!pool->netdev)  in xp_clear_dev()
273 	xp_disable_drv_zc(pool);  in xp_clear_dev()
274 	xsk_clear_pool_at_qid(pool->netdev, pool->queue_id);  in xp_clear_dev()
275 	dev_put(pool->netdev);  in xp_clear_dev()
276 	pool->netdev = NULL;  in xp_clear_dev()
281 	struct xsk_buff_pool *pool = container_of(work, struct xsk_buff_pool,  in xp_release_deferred()  local
285 	xp_clear_dev(pool);  in xp_release_deferred()
288 	if (pool->fq) {  in xp_release_deferred()
289 		xskq_destroy(pool->fq);  in xp_release_deferred()
290 		pool->fq = NULL;  in xp_release_deferred()
293 	if (pool->cq) {  in xp_release_deferred()
294 		xskq_destroy(pool->cq);  in xp_release_deferred()
295 		pool->cq = NULL;  in xp_release_deferred()
298 	xdp_put_umem(pool->umem, false);  in xp_release_deferred()
299 	xp_destroy(pool);  in xp_release_deferred()
302 void xp_get_pool(struct xsk_buff_pool *pool)  in xp_get_pool()  argument
304 	refcount_inc(&pool->users);  in xp_get_pool()
307 bool xp_put_pool(struct xsk_buff_pool *pool)  in xp_put_pool()  argument
309 	if (!pool)  in xp_put_pool()
312 	if (refcount_dec_and_test(&pool->users)) {  in xp_put_pool()
313 		INIT_WORK(&pool->work, xp_release_deferred);  in xp_put_pool()
314 		schedule_work(&pool->work);  in xp_put_pool()
321 static struct xsk_dma_map *xp_find_dma_map(struct xsk_buff_pool *pool)  in xp_find_dma_map()  argument
325 	list_for_each_entry(dma_map, &pool->umem->xsk_dma_list, list) {  in xp_find_dma_map()
326 		if (dma_map->netdev == pool->netdev)  in xp_find_dma_map()
381 void xp_dma_unmap(struct xsk_buff_pool *pool, unsigned long attrs)  in xp_dma_unmap()  argument
385 	if (!pool->dma_pages)  in xp_dma_unmap()
388 	dma_map = xp_find_dma_map(pool);  in xp_dma_unmap()
397 	kvfree(pool->dma_pages);  in xp_dma_unmap()
398 	pool->dma_pages = NULL;  in xp_dma_unmap()
399 	pool->dma_pages_cnt = 0;  in xp_dma_unmap()
400 	pool->dev = NULL;  in xp_dma_unmap()
416 static int xp_init_dma_info(struct xsk_buff_pool *pool, struct xsk_dma_map *dma_map)  in xp_init_dma_info()  argument
418 	if (!pool->unaligned) {  in xp_init_dma_info()
421 		for (i = 0; i < pool->heads_cnt; i++) {  in xp_init_dma_info()
422 			struct xdp_buff_xsk *xskb = &pool->heads[i];  in xp_init_dma_info()
425 			orig_addr = xskb->xdp.data_hard_start - pool->addrs - pool->headroom;  in xp_init_dma_info()
426 			xp_init_xskb_dma(xskb, pool, dma_map->dma_pages, orig_addr);  in xp_init_dma_info()
430 	pool->dma_pages = kvcalloc(dma_map->dma_pages_cnt, sizeof(*pool->dma_pages), GFP_KERNEL);  in xp_init_dma_info()
431 	if (!pool->dma_pages)  in xp_init_dma_info()
434 	pool->dev = dma_map->dev;  in xp_init_dma_info()
435 	pool->dma_pages_cnt = dma_map->dma_pages_cnt;  in xp_init_dma_info()
436 	memcpy(pool->dma_pages, dma_map->dma_pages,  in xp_init_dma_info()
437 	       pool->dma_pages_cnt * sizeof(*pool->dma_pages));  in xp_init_dma_info()
442 int xp_dma_map(struct xsk_buff_pool *pool, struct device *dev,  in xp_dma_map()  argument
450 	dma_map = xp_find_dma_map(pool);  in xp_dma_map()
452 		err = xp_init_dma_info(pool, dma_map);  in xp_dma_map()
460 	dma_map = xp_create_dma_map(dev, pool->netdev, nr_pages, pool->umem);  in xp_dma_map()
474 	if (pool->unaligned)  in xp_dma_map()
477 	err = xp_init_dma_info(pool, dma_map);  in xp_dma_map()
487 static bool xp_addr_crosses_non_contig_pg(struct xsk_buff_pool *pool,  in xp_addr_crosses_non_contig_pg()  argument
490 	return xp_desc_crosses_non_contig_pg(pool, addr, pool->chunk_size);  in xp_addr_crosses_non_contig_pg()
493 static bool xp_check_unaligned(struct xsk_buff_pool *pool, u64 *addr)  in xp_check_unaligned()  argument
496 	if (*addr >= pool->addrs_cnt ||  in xp_check_unaligned()
497 	    *addr + pool->chunk_size > pool->addrs_cnt ||  in xp_check_unaligned()
498 	    xp_addr_crosses_non_contig_pg(pool, *addr))  in xp_check_unaligned()
503 static bool xp_check_aligned(struct xsk_buff_pool *pool, u64 *addr)  in xp_check_aligned()  argument
505 	*addr = xp_aligned_extract_addr(pool, *addr);  in xp_check_aligned()
506 	return *addr < pool->addrs_cnt;  in xp_check_aligned()
509 static struct xdp_buff_xsk *xp_get_xskb(struct xsk_buff_pool *pool, u64 addr)  in xp_get_xskb()  argument
513 	if (pool->unaligned) {  in xp_get_xskb()
514 		xskb = pool->free_heads[--pool->free_heads_cnt];  in xp_get_xskb()
515 		xp_init_xskb_addr(xskb, pool, addr);  in xp_get_xskb()
516 		if (pool->dma_pages)  in xp_get_xskb()
517 			xp_init_xskb_dma(xskb, pool, pool->dma_pages, addr);  in xp_get_xskb()
519 		xskb = &pool->heads[xp_aligned_extract_idx(pool, addr)];  in xp_get_xskb()
525 static struct xdp_buff_xsk *__xp_alloc(struct xsk_buff_pool *pool)  in __xp_alloc()  argument
531 	if (pool->free_heads_cnt == 0)  in __xp_alloc()
535 		if (!xskq_cons_peek_addr_unchecked(pool->fq, &addr)) {  in __xp_alloc()
536 			pool->fq->queue_empty_descs++;  in __xp_alloc()
540 		ok = pool->unaligned ? xp_check_unaligned(pool, &addr) :  in __xp_alloc()
541 		     xp_check_aligned(pool, &addr);  in __xp_alloc()
543 			pool->fq->invalid_descs++;  in __xp_alloc()
544 			xskq_cons_release(pool->fq);  in __xp_alloc()
550 	xskb = xp_get_xskb(pool, addr);  in __xp_alloc()
552 	xskq_cons_release(pool->fq);  in __xp_alloc()
556 struct xdp_buff *xp_alloc(struct xsk_buff_pool *pool)  in xp_alloc()  argument
560 	if (!pool->free_list_cnt) {  in xp_alloc()
561 		xskb = __xp_alloc(pool);  in xp_alloc()
565 		pool->free_list_cnt--;  in xp_alloc()
566 		xskb = list_first_entry(&pool->free_list, struct xdp_buff_xsk,  in xp_alloc()
575 	if (pool->dev)  in xp_alloc()
576 		xp_dma_sync_for_device(pool, xskb->dma, pool->frame_len);  in xp_alloc()
582 static u32 xp_alloc_new_from_fq(struct xsk_buff_pool *pool, struct xdp_buff **xdp, u32 max)  in xp_alloc_new_from_fq()  argument
586 	if (max > pool->free_heads_cnt)  in xp_alloc_new_from_fq()
587 		max = pool->free_heads_cnt;  in xp_alloc_new_from_fq()
588 	max = xskq_cons_nb_entries(pool->fq, max);  in xp_alloc_new_from_fq()
590 	cached_cons = pool->fq->cached_cons;  in xp_alloc_new_from_fq()
598 		__xskq_cons_read_addr_unchecked(pool->fq, cached_cons++, &addr);  in xp_alloc_new_from_fq()
600 		ok = pool->unaligned ? xp_check_unaligned(pool, &addr) :  in xp_alloc_new_from_fq()
601 			xp_check_aligned(pool, &addr);  in xp_alloc_new_from_fq()
603 			pool->fq->invalid_descs++;  in xp_alloc_new_from_fq()
608 		xskb = xp_get_xskb(pool, addr);  in xp_alloc_new_from_fq()
614 	xskq_cons_release_n(pool->fq, max);  in xp_alloc_new_from_fq()
618 static u32 xp_alloc_reused(struct xsk_buff_pool *pool, struct xdp_buff **xdp, u32 nb_entries)  in xp_alloc_reused()  argument
623 	nb_entries = min_t(u32, nb_entries, pool->free_list_cnt);  in xp_alloc_reused()
627 		xskb = list_first_entry(&pool->free_list, struct xdp_buff_xsk, list_node);  in xp_alloc_reused()
633 	pool->free_list_cnt -= nb_entries;  in xp_alloc_reused()
638 static u32 xp_alloc_slow(struct xsk_buff_pool *pool, struct xdp_buff **xdp,  in xp_alloc_slow()  argument
646 		buff = xp_alloc(pool);  in xp_alloc_slow()
656 u32 xp_alloc_batch(struct xsk_buff_pool *pool, struct xdp_buff **xdp, u32 max)  in xp_alloc_batch()  argument
660 	if (unlikely(pool->dev && dma_dev_need_sync(pool->dev)))  in xp_alloc_batch()
661 		return xp_alloc_slow(pool, xdp, max);  in xp_alloc_batch()
663 	if (unlikely(pool->free_list_cnt)) {  in xp_alloc_batch()
664 		nb_entries1 = xp_alloc_reused(pool, xdp, max);  in xp_alloc_batch()
672 	nb_entries2 = xp_alloc_new_from_fq(pool, xdp, max);  in xp_alloc_batch()
674 		pool->fq->queue_empty_descs++;  in xp_alloc_batch()
680 bool xp_can_alloc(struct xsk_buff_pool *pool, u32 count)  in xp_can_alloc()  argument
684 	if (pool->free_list_cnt >= count)  in xp_can_alloc()
687 	req_count = count - pool->free_list_cnt;  in xp_can_alloc()
688 	avail_count = xskq_cons_nb_entries(pool->fq, req_count);  in xp_can_alloc()
690 		pool->fq->queue_empty_descs++;  in xp_can_alloc()
701 	xskb->pool->free_list_cnt++;  in xp_free()
702 	list_add(&xskb->list_node, &xskb->pool->free_list);  in xp_free()
706 static u64 __xp_raw_get_addr(const struct xsk_buff_pool *pool, u64 addr)  in __xp_raw_get_addr()  argument
708 	return pool->unaligned ? xp_unaligned_add_offset_to_addr(addr) : addr;  in __xp_raw_get_addr()
711 static void *__xp_raw_get_data(const struct xsk_buff_pool *pool, u64 addr)  in __xp_raw_get_data()  argument
713 	return pool->addrs + addr;  in __xp_raw_get_data()
716 void *xp_raw_get_data(struct xsk_buff_pool *pool, u64 addr)  in xp_raw_get_data()  argument
718 	return __xp_raw_get_data(pool, __xp_raw_get_addr(pool, addr));  in xp_raw_get_data()
722 static dma_addr_t __xp_raw_get_dma(const struct xsk_buff_pool *pool, u64 addr)  in __xp_raw_get_dma()  argument
724 	return (pool->dma_pages[addr >> PAGE_SHIFT] &  in __xp_raw_get_dma()
729 dma_addr_t xp_raw_get_dma(struct xsk_buff_pool *pool, u64 addr)  in xp_raw_get_dma()  argument
731 	return __xp_raw_get_dma(pool, __xp_raw_get_addr(pool, addr));  in xp_raw_get_dma()
737  * @pool: XSk buff pool desc address belongs to
747 struct xdp_desc_ctx xp_raw_get_ctx(const struct xsk_buff_pool *pool, u64 addr)  in xp_raw_get_ctx()  argument
751 	addr = __xp_raw_get_addr(pool, addr);  in xp_raw_get_ctx()
753 	ret.dma = __xp_raw_get_dma(pool, addr);  in xp_raw_get_ctx()
754 	ret.meta = __xsk_buff_get_metadata(pool, __xp_raw_get_data(pool, addr));  in xp_raw_get_ctx()