amazon/ena/ena_netdev.c

54 	struct ena_ring *tx_ring;
63 	tx_ring = &adapter->tx_ring[txqueue];
65 	time_since_last_napi = jiffies_to_usecs(jiffies - tx_ring->tx_stats.last_napi_jiffies);
66 	napi_scheduled = !!(tx_ring->napi->state & NAPIF_STATE_SCHED);
193 		txr = &adapter->tx_ring[i];
223 			rxr->xdp_ring = &adapter->tx_ring[i + adapter->num_io_queues];
236 	struct ena_ring *tx_ring = &adapter->tx_ring[qid];
240 	if (tx_ring->tx_buffer_info) {
246 	size = sizeof(struct ena_tx_buffer) * tx_ring->ring_size;
249 	tx_ring->tx_buffer_info = vzalloc_node(size, node);
250 	if (!tx_ring->tx_buffer_info) {
251 		tx_ring->tx_buffer_info = vzalloc(size);
252 		if (!tx_ring->tx_buffer_info)
256 	size = sizeof(u16) * tx_ring->ring_size;
257 	tx_ring->free_ids = vzalloc_node(size, node);
258 	if (!tx_ring->free_ids) {
259 		tx_ring->free_ids = vzalloc(size);
260 		if (!tx_ring->free_ids)
264 	size = tx_ring->tx_max_header_size;
265 	tx_ring->push_buf_intermediate_buf = vzalloc_node(size, node);
266 	if (!tx_ring->push_buf_intermediate_buf) {
267 		tx_ring->push_buf_intermediate_buf = vzalloc(size);
268 		if (!tx_ring->push_buf_intermediate_buf)
273 	for (i = 0; i < tx_ring->ring_size; i++)
274 		tx_ring->free_ids[i] = i;
277 	memset(&tx_ring->tx_stats, 0x0, sizeof(tx_ring->tx_stats));
279 	tx_ring->next_to_use = 0;
280 	tx_ring->next_to_clean = 0;
281 	tx_ring->cpu = ena_irq->cpu;
282 	tx_ring->numa_node = node;
286 	vfree(tx_ring->free_ids);
287 	tx_ring->free_ids = NULL;
289 	vfree(tx_ring->tx_buffer_info);
290 	tx_ring->tx_buffer_info = NULL;
303 	struct ena_ring *tx_ring = &adapter->tx_ring[qid];
305 	vfree(tx_ring->tx_buffer_info);
306 	tx_ring->tx_buffer_info = NULL;
308 	vfree(tx_ring->free_ids);
309 	tx_ring->free_ids = NULL;
311 	vfree(tx_ring->push_buf_intermediate_buf);
312 	tx_ring->push_buf_intermediate_buf = NULL;
662 void ena_unmap_tx_buff(struct ena_ring *tx_ring,
676 		dma_unmap_single(tx_ring->dev,
686 		dma_unmap_page(tx_ring->dev, dma_unmap_addr(ena_buf, paddr),
693  * @tx_ring: TX ring for which buffers be freed
695 static void ena_free_tx_bufs(struct ena_ring *tx_ring)
701 	is_xdp_ring = ENA_IS_XDP_INDEX(tx_ring->adapter, tx_ring->qid);
703 	for (i = 0; i < tx_ring->ring_size; i++) {
704 		struct ena_tx_buffer *tx_info = &tx_ring->tx_buffer_info[i];
710 			netif_notice(tx_ring->adapter, ifdown, tx_ring->netdev,
712 				     tx_ring->qid, i);
715 			netif_dbg(tx_ring->adapter, ifdown, tx_ring->netdev,
717 				  tx_ring->qid, i);
720 		ena_unmap_tx_buff(tx_ring, tx_info);
729 		netdev_tx_reset_queue(netdev_get_tx_queue(tx_ring->netdev,
730 							  tx_ring->qid));
735 	struct ena_ring *tx_ring;
739 		tx_ring = &adapter->tx_ring[i];
740 		ena_free_tx_bufs(tx_ring);
796 static int validate_tx_req_id(struct ena_ring *tx_ring, u16 req_id)
800 	tx_info = &tx_ring->tx_buffer_info[req_id];
804 	return handle_invalid_req_id(tx_ring, req_id, tx_info, false);
807 static int ena_clean_tx_irq(struct ena_ring *tx_ring, u32 budget)
818 	next_to_clean = tx_ring->next_to_clean;
819 	txq = netdev_get_tx_queue(tx_ring->netdev, tx_ring->qid);
825 		rc = ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq,
829 				handle_invalid_req_id(tx_ring, req_id, NULL, false);
834 		rc = validate_tx_req_id(tx_ring, req_id);
838 		tx_info = &tx_ring->tx_buffer_info[req_id];
847 		ena_unmap_tx_buff(tx_ring, tx_info);
849 		netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev,
850 			  "tx_poll: q %d skb %p completed\n", tx_ring->qid,
858 		tx_ring->free_ids[next_to_clean] = req_id;
860 						     tx_ring->ring_size);
863 	tx_ring->next_to_clean = next_to_clean;
864 	ena_com_comp_ack(tx_ring->ena_com_io_sq, total_done);
868 	netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev,
870 		  tx_ring->qid, tx_pkts);
877 	above_thresh = ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
882 			ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
885 		    test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags)) {
887 			ena_increase_stat(&tx_ring->tx_stats.queue_wakeup, 1,
888 					  &tx_ring->syncp);
1368 void ena_unmask_interrupt(struct ena_ring *tx_ring,
1371 	u32 rx_interval = tx_ring->smoothed_interval;
1387 				tx_ring->smoothed_interval,
1390 	ena_increase_stat(&tx_ring->tx_stats.unmask_interrupt, 1,
1391 			  &tx_ring->syncp);
1398 	ena_com_unmask_intr(tx_ring->ena_com_io_cq, &intr_reg);
1401 void ena_update_ring_numa_node(struct ena_ring *tx_ring,
1408 	if (likely(tx_ring->cpu == cpu))
1411 	tx_ring->cpu = cpu;
1417 	if (likely(tx_ring->numa_node == numa_node))
1423 		ena_com_update_numa_node(tx_ring->ena_com_io_cq, numa_node);
1424 		tx_ring->numa_node = numa_node;
1440 	struct ena_ring *tx_ring, *rx_ring;
1447 	tx_ring = ena_napi->tx_ring;
1450 	tx_budget = tx_ring->ring_size / ENA_TX_POLL_BUDGET_DIVIDER;
1452 	if (!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags) ||
1453 	    test_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags)) {
1458 	tx_work_done = ena_clean_tx_irq(tx_ring, tx_budget);
1468 	if (unlikely(!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags) ||
1469 		     test_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags))) {
1489 			ena_update_ring_numa_node(tx_ring, rx_ring);
1490 			ena_unmask_interrupt(tx_ring, rx_ring);
1498 	u64_stats_update_begin(&tx_ring->syncp);
1499 	tx_ring->tx_stats.napi_comp += napi_comp_call;
1500 	tx_ring->tx_stats.tx_poll++;
1501 	u64_stats_update_end(&tx_ring->syncp);
1503 	tx_ring->tx_stats.last_napi_jiffies = jiffies;
1773 		struct ena_ring *rx_ring, *tx_ring;
1778 		tx_ring = &adapter->tx_ring[i];
1789 		napi->tx_ring = tx_ring;
1893 	struct ena_ring *tx_ring;
1900 	tx_ring = &adapter->tx_ring[qid];
1910 	ctx.queue_size = tx_ring->ring_size;
1911 	ctx.numa_node = tx_ring->numa_node;
1922 				     &tx_ring->ena_com_io_sq,
1923 				     &tx_ring->ena_com_io_cq);
1932 	ena_com_update_numa_node(tx_ring->ena_com_io_cq, ctx.numa_node);
2040 		adapter->tx_ring[i].ring_size = new_tx_size;
2114 		cur_tx_ring_size = adapter->tx_ring[0].ring_size;
2194 		ena_unmask_interrupt(&adapter->tx_ring[i],
2472 static int ena_check_and_linearize_skb(struct ena_ring *tx_ring,
2480 	if (num_frags < tx_ring->sgl_size)
2483 	if ((num_frags == tx_ring->sgl_size) &&
2484 	    (header_len < tx_ring->tx_max_header_size))
2487 	ena_increase_stat(&tx_ring->tx_stats.linearize, 1, &tx_ring->syncp);
2491 		ena_increase_stat(&tx_ring->tx_stats.linearize_failed, 1,
2492 				  &tx_ring->syncp);
2498 static int ena_tx_map_skb(struct ena_ring *tx_ring,
2504 	struct ena_adapter *adapter = tx_ring->adapter;
2516 	if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
2527 		push_len = min_t(u32, skb->len, tx_ring->tx_max_header_size);
2529 					       tx_ring->push_buf_intermediate_buf);
2532 			ena_increase_stat(&tx_ring->tx_stats.llq_buffer_copy, 1,
2533 					  &tx_ring->syncp);
2540 				    tx_ring->tx_max_header_size);
2548 		dma = dma_map_single(tx_ring->dev, skb->data + push_len,
2550 		if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
2575 		dma = skb_frag_dma_map(tx_ring->dev, frag, delta,
2577 		if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
2590 	ena_increase_stat(&tx_ring->tx_stats.dma_mapping_err, 1,
2591 			  &tx_ring->syncp);
2597 	ena_unmap_tx_buff(tx_ring, tx_info);
2608 	struct ena_ring *tx_ring;
2617 	tx_ring = &adapter->tx_ring[qid];
2620 	rc = ena_check_and_linearize_skb(tx_ring, skb);
2624 	next_to_use = tx_ring->next_to_use;
2625 	req_id = tx_ring->free_ids[next_to_use];
2626 	tx_info = &tx_ring->tx_buffer_info[req_id];
2631 	rc = ena_tx_map_skb(tx_ring, tx_info, skb, &push_hdr, &header_len);
2643 	ena_tx_csum(&ena_tx_ctx, skb, tx_ring->disable_meta_caching);
2646 			     tx_ring,
2660 	if (unlikely(!ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
2661 						   tx_ring->sgl_size + 2))) {
2666 		ena_increase_stat(&tx_ring->tx_stats.queue_stop, 1,
2667 				  &tx_ring->syncp);
2679 		if (ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
2682 			ena_increase_stat(&tx_ring->tx_stats.queue_wakeup, 1,
2683 					  &tx_ring->syncp);
2693 		ena_ring_tx_doorbell(tx_ring);
2698 	ena_unmap_tx_buff(tx_ring, tx_info);
2810 	struct ena_ring *rx_ring, *tx_ring;
2823 		tx_ring = &adapter->tx_ring[i];
2826 			start = u64_stats_fetch_begin(&tx_ring->syncp);
2827 			packets = tx_ring->tx_stats.cnt;
2828 			bytes = tx_ring->tx_stats.bytes;
2829 		} while (u64_stats_fetch_retry(&tx_ring->syncp, start));
2900 	if (adapter->tx_ring->ring_size)
2901 		tx_queue_size = adapter->tx_ring->ring_size;
3317 		txr = &adapter->tx_ring[i];
3417 					  struct ena_ring *tx_ring)
3419 	struct ena_napi *ena_napi = container_of(tx_ring->napi, struct ena_napi, napi);
3432 	for (i = 0; i < tx_ring->ring_size; i++) {
3433 		tx_buf = &tx_ring->tx_buffer_info[i];
3449 				  tx_ring->qid);
3459 				jiffies_to_usecs(jiffies - tx_ring->tx_stats.last_napi_jiffies);
3481 				     tx_ring->qid, i, time_since_last_napi, napi_scheduled);
3500 	ena_increase_stat(&tx_ring->tx_stats.missed_tx, missed_tx,
3501 			  &tx_ring->syncp);
3508 	struct ena_ring *tx_ring;
3534 		tx_ring = &adapter->tx_ring[qid];
3537 		rc = check_missing_comp_in_tx_queue(adapter, tx_ring);