1 /* Broadcom NetXtreme-C/E network driver.
3  * Copyright (c) 2014-2016 Broadcom Corporation
4  * Copyright (c) 2016-2018 Broadcom Limited
32 #include <linux/io-64-nonatomic-lo-hi.h>
74 	(((txr)->tx_napi_idx << TX_OPAQUE_RING_SHIFT) |			\
75 	 ((bds) << TX_OPAQUE_BDS_SHIFT) | ((idx) & (bp)->tx_ring_mask))
83 				 (bp)->tx_ring_mask)
89 #define TX_MAX_FRAGS		(TX_MAX_BD_CNT - 2)
191 	(le32_to_cpu((txcmp)->sq_cons_idx) & TX_CMP_SQ_CONS_IDX_MASK)
211 	(le32_to_cpu((tscmp)->tx_ts_cmp_ts_ns_lo) |			\
212 	 ((u64)(le32_to_cpu((tscmp)->tx_ts_cmp_flags_type) &		\
216 	(((tscmp)->tx_ts_cmp_flags_type & cpu_to_le32(TX_TS_CMP_FLAGS_ERROR)) &&\
217 	 ((tscmp)->tx_ts_cmp_errors_v & cpu_to_le32(TX_TS_CMP_TS_INVALID_ERR)))
271 	((rxcmp)->rx_cmp_len_flags_type & cpu_to_le32(RX_CMP_FLAGS_RSS_VALID))
276 	(((le32_to_cpu((rxcmp)->rx_cmp_misc_v1) & RX_CMP_RSS_HASH_TYPE) >>\
280 	(le32_to_cpu((rxcmp)->rx_cmp_len_flags_type) & RX_CMP_FLAGS_ITYPES_MASK)
283 	((le32_to_cpu((rxcmp)->rx_cmp_misc_v1) & RX_CMP_V3_RSS_EXT_OP_LEGACY) >>\
287 	((le32_to_cpu((rxcmp)->rx_cmp_misc_v1) & RX_CMP_V3_RSS_EXT_OP_NEW) >>\
291 	(((bp)->rss_cap & BNXT_RSS_CAP_RSS_TCAM) ?		\
301 	((rxcmp)->rx_cmp_misc_v1 & cpu_to_le32(RX_CMP_METADATA1_VALID))
304 	(le32_to_cpu((rxcmp)->rx_cmp_misc_v1) & RX_CMP_METADATA1_TPID_SEL)
370 	    (((rxcmp1)->rx_cmp_flags2 &	RX_CMP_L4_CS_BITS) &&		\
371 	     !((rxcmp1)->rx_cmp_cfa_code_errors_v2 & RX_CMP_L4_CS_ERR_BITS))
374 	    ((le32_to_cpu((rxcmp1)->rx_cmp_flags2) &			\
378 	((le32_to_cpu((rxcmpl1)->rx_cmp_cfa_code_errors_v2) &		\
382 	((le32_to_cpu((rxcmp1)->rx_cmp_cfa_code_errors_v2) &		\
399 	((le32_to_cpu((rx_agg)->rx_agg_cmp_v) &		\
444 	((rx_tpa_start)->rx_tpa_start_cmp_len_flags_type &		\
448 	(((le32_to_cpu((rx_tpa_start)->rx_tpa_start_cmp_misc_v1) &	\
453 	(((le32_to_cpu((rx_tpa_start)->rx_tpa_start_cmp_misc_v1) &	\
458 	((le32_to_cpu((rx_tpa_start)->rx_tpa_start_cmp_misc_v1) &	\
462 	((le32_to_cpu((rx_tpa_start)->rx_tpa_start_cmp_misc_v1) &	\
466 	((rx_tpa_start)->rx_tpa_start_cmp_len_flags_type &		\
470 	((rx_tpa_start)->rx_tpa_start_cmp_misc_v1 &			\
474 	(le32_to_cpu((rx_tpa_start)->rx_tpa_start_cmp_misc_v1) &	\
509 	((le32_to_cpu((rx_tpa_start)->rx_tpa_start_cmp_cfa_code_v2) &	\
513 	(!!((rx_tpa_start)->rx_tpa_start_cmp_flags2 &		\
517 	((le32_to_cpu((rx_tpa_start)->rx_tpa_start_cmp_cfa_code_v2) &	\
522 	((le32_to_cpu((rx_tpa_start)->rx_tpa_start_cmp_cfa_code_v2) &	\
563 	((le32_to_cpu((rx_tpa_end)->rx_tpa_end_cmp_misc_v1) &		\
567 	((le32_to_cpu((rx_tpa_end)->rx_tpa_end_cmp_misc_v1) &		\
571 	((le32_to_cpu((rx_tpa_end)->rx_tpa_end_cmp_misc_v1) &		\
575 	((le32_to_cpu((rx_tpa_end)->rx_tpa_end_cmp_misc_v1) &		\
579 	((le32_to_cpu((rx_tpa_end)->rx_tpa_end_cmp_misc_v1) &		\
587 	((rx_tpa_end)->rx_tpa_end_cmp_len_flags_type &			\
591 	(!!((rx_tpa_end)->rx_tpa_end_cmp_tsdelta &			\
620 	((rx_tpa_end_ext)->rx_tpa_end_cmp_errors_v2 &			\
624 	((le32_to_cpu((rx_tpa_end_ext)->rx_tpa_end_cmp_dup_acks) &	\
629 	((le32_to_cpu((rx_tpa_end_ext)->rx_tpa_end_cmp_dup_acks) &	\
690 	(((cpr)->cp_ring_type << BNXT_NQ_HDL_TYPE_SHIFT) | (cpr)->cp_idx)
709 /* 64-bit doorbell */
737 #define INVALID_HW_RING_ID	((u16)-1)
754 /* The RXBD length is 16-bit so we can only support page sizes < 64K */
765 /* First RX buffer page in XDP multi-buf mode
767  * +-------------------------------------------------------------------------+
768  * | XDP_PACKET_HEADROOM | bp->rx_buf_use_size              | skb_shared_info|
769  * | (bp->rx_dma_offset) |                                  |                |
770  * +-------------------------------------------------------------------------+
773 	((unsigned int)PAGE_SIZE - VLAN_ETH_HLEN - NET_IP_ALIGN -	\
776 	(BNXT_MAX_PAGE_MODE_MTU_SBUF - \
786 #define MAX_TPA_P5_MASK	(MAX_TPA_P5 - 1)
817 #define BNXT_MAX_RX_DESC_CNT		(RX_DESC_CNT * MAX_RX_PAGES - 1)
818 #define BNXT_MAX_RX_DESC_CNT_JUM_ENA	(RX_DESC_CNT * MAX_RX_PAGES_AGG_ENA - 1)
819 #define BNXT_MAX_RX_JUM_DESC_CNT	(RX_DESC_CNT * MAX_RX_AGG_PAGES - 1)
820 #define BNXT_MAX_TX_DESC_CNT		(TX_DESC_CNT * MAX_TX_PAGES - 1)
827 #define RX_RING(bp, x)	(((x) & (bp)->rx_ring_mask) >> (BNXT_PAGE_SHIFT - 4))
828 #define RX_AGG_RING(bp, x)	(((x) & (bp)->rx_agg_ring_mask) >>	\
829 				 (BNXT_PAGE_SHIFT - 4))
830 #define RX_IDX(x)	((x) & (RX_DESC_CNT - 1))
832 #define TX_RING(bp, x)	(((x) & (bp)->tx_ring_mask) >> (BNXT_PAGE_SHIFT - 4))
833 #define TX_IDX(x)	((x) & (TX_DESC_CNT - 1))
835 #define CP_RING(x)	(((x) & ~(CP_DESC_CNT - 1)) >> (BNXT_PAGE_SHIFT - 4))
836 #define CP_IDX(x)	((x) & (CP_DESC_CNT - 1))
839 	(!!((txcmp)->tx_cmp_errors_v & cpu_to_le32(TX_CMP_V)) ==	\
840 	 !((raw_cons) & bp->cp_bit))
843 	(!!((rxcmp1)->rx_cmp_cfa_code_errors_v2 & cpu_to_le32(RX_CMP_V)) ==\
844 	 !((raw_cons) & bp->cp_bit))
847 	(!!((agg)->rx_agg_cmp_v & cpu_to_le32(RX_AGG_CMP_V)) ==	\
848 	 !((raw_cons) & bp->cp_bit))
851 	(!!((nqcmp)->v & cpu_to_le32(NQ_CN_V)) == !((raw_cons) & bp->cp_bit))
854 	(le32_to_cpu((txcmp)->tx_cmp_flags_type) & CMP_TYPE)
857 	(le32_to_cpu((rxcmp)->rx_cmp_len_flags_type) & RX_CMP_CMP_TYPE)
859 #define RING_RX(bp, idx)	((idx) & (bp)->rx_ring_mask)
862 #define RING_RX_AGG(bp, idx)	((idx) & (bp)->rx_agg_ring_mask)
865 #define RING_TX(bp, idx)	((idx) & (bp)->tx_ring_mask)
870 #define RING_CMP(idx)		((idx) & bp->cp_ring_mask)
967 #define DB_EPOCH(db, idx)	(((idx) & (db)->db_epoch_mask) <<	\
968 				 ((db)->db_epoch_shift))
972 #define DB_RING_IDX(db, idx)	(((idx) & (db)->db_ring_mask) |		\
1209 	for (iter = 0, txr = (bnapi)->tx_ring[0]; txr;	\
1210 	     txr = (iter < BNXT_MAX_TXR_PER_NAPI - 1) ?	\
1211 	     (bnapi)->tx_ring[++iter] : NULL)
1256 #define INVALID_STATS_CTX_ID	-1
1508 	__le32	preemphasis;  member
1553 #define BNXT_LINK_IS_UP(bp)	((bp)->link_info.link_state == BNXT_LINK_STATE_UP)
1655 	u32			preemphasis;  member
1720 	((link_info)->support_pam4_speeds)
1774 	((bp->flags & BNXT_FLAG_CHIP_P5_PLUS) ? 0x80000000 : 0xb1000000)
1840 	/* Stat counter mask (width) */
1870 #define MAX_CTX_BYTES_MASK	(MAX_CTX_BYTES - 1)
1974 #define BNXT_CTX_INV	((u16)-1)
2148 	if (!bs_trace->wrapped &&  in bnxt_bs_trace_check_wrap()
2149 	    *bs_trace->magic_byte != BNXT_TRACE_BUF_MAGIC_BYTE)  in bnxt_bs_trace_check_wrap()
2150 		bs_trace->wrapped = 1;  in bnxt_bs_trace_check_wrap()
2151 	bs_trace->last_offset = offset;  in bnxt_bs_trace_check_wrap()
2286 #define BNXT_PF(bp)		(!((bp)->flags & BNXT_FLAG_VF))
2287 #define BNXT_VF(bp)		((bp)->flags & BNXT_FLAG_VF)
2289 #define	BNXT_VF_IS_TRUSTED(bp)	((bp)->vf.flags & BNXT_VF_TRUST)
2293 #define BNXT_NPAR(bp)		((bp)->port_partition_type)
2294 #define BNXT_MH(bp)		((bp)->flags & BNXT_FLAG_MULTI_HOST)
2297 				 ((bp)->phy_flags & BNXT_PHY_FL_SHARED_PORT_CFG))
2300 				 (bp)->link_info.phy_state == BNXT_PHY_STATE_ENABLED)
2301 #define BNXT_CHIP_TYPE_NITRO_A0(bp) ((bp)->flags & BNXT_FLAG_CHIP_NITRO_A0)
2302 #define BNXT_RX_PAGE_MODE(bp)	((bp)->flags & BNXT_FLAG_RX_PAGE_MODE)
2304 				 (!((bp)->flags & BNXT_FLAG_CHIP_P5_PLUS) ||\
2305 				  (bp)->max_tpa_v2) && !is_kdump_kernel())
2306 #define BNXT_RX_JUMBO_MODE(bp)	((bp)->flags & BNXT_FLAG_JUMBO)
2309 	((bp)->chip_num == CHIP_NUM_57608)
2312 	((bp)->chip_num == CHIP_NUM_57508 ||	\
2313 	 (bp)->chip_num == CHIP_NUM_57504 ||	\
2314 	 (bp)->chip_num == CHIP_NUM_57502)
2322 	(BNXT_CHIP_NUM_57X1X((bp)->chip_num) ||	\
2323 	 BNXT_CHIP_NUM_5745X((bp)->chip_num) ||	\
2324 	 BNXT_CHIP_NUM_588XX((bp)->chip_num) ||	\
2325 	 (BNXT_CHIP_NUM_58700((bp)->chip_num) &&	\
2330 	(BNXT_CHIP_NUM_57X0X((bp)->chip_num) ||	\
2450 	(test_bit(BNXT_STATE_FW_FATAL_COND, &(bp)->state) ||	\
2451 	 pci_channel_offline((bp)->pdev))
2512 #define BNXT_NEW_RM(bp)		((bp)->fw_cap & BNXT_FW_CAP_NEW_RM)
2514 				 ((bp)->fw_cap & BNXT_FW_CAP_PTP_RTC))
2516 	(BNXT_PF(bp) && ((bp)->fw_cap & BNXT_FW_CAP_CFA_RFS_RING_TBL_IDX_V3))
2520 	 ((bp)->rss_cap & BNXT_RSS_CAP_MULTI_RSS_CTX))
2523 	 ((bp)->fw_cap & BNXT_FW_CAP_VNIC_RE_FLUSH))
2525 	((bp)->fw_cap & BNXT_FW_CAP_ENABLE_RDMA_SRIOV)
2527 	((bp)->fw_cap & BNXT_FW_CAP_ROCE_VF_RESC_MGMT_SUPPORTED)
2529 	((bp)->fw_cap & BNXT_FW_CAP_SW_MAX_RESOURCE_LIMITS)
2557 #define PHY_VER_STR_LEN         (FW_VER_STR_LEN - BC_HWRM_STR_LEN)
2564 #define BNXT_FW_MAJ(bp)		((bp)->fw_ver_code >> 48)
2565 #define BNXT_FW_BLD(bp)		(((bp)->fw_ver_code >> 16) & 0xffff)
2639 	/* ensure atomic 64-bit doorbell writes on 32-bit systems. */
2648 #define BNXT_NTP_FLTR_HASH_MASK	(BNXT_NTP_FLTR_HASH_SIZE - 1)
2658 #define BNXT_L2_FLTR_HASH_MASK	(BNXT_L2_FLTR_HASH_SIZE - 1)
2714 	/* devlink interface and vf-rep structs */
2718 	struct bnxt_vf_rep	**vf_reps; /* array of vf-rep ptrs */
2719 	u16			*cfa_code_map; /* cfa_code -> vf_idx map */
2803 	u32 used = READ_ONCE(txr->tx_prod) - READ_ONCE(txr->tx_cons);  in bnxt_tx_avail()
2805 	return bp->tx_ring_size - (used & bp->tx_ring_mask);  in bnxt_tx_avail()
2812 	spin_lock(&bp->db_lock);  in bnxt_writeq()
2814 	spin_unlock(&bp->db_lock);  in bnxt_writeq()
2824 	spin_lock(&bp->db_lock);  in bnxt_writeq_relaxed()
2826 	spin_unlock(&bp->db_lock);  in bnxt_writeq_relaxed()
2836 	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {  in bnxt_db_write_relaxed()
2837 		bnxt_writeq_relaxed(bp, db->db_key64 | DB_RING_IDX(db, idx),  in bnxt_db_write_relaxed()
2838 				    db->doorbell);  in bnxt_db_write_relaxed()
2840 		u32 db_val = db->db_key32 | DB_RING_IDX(db, idx);  in bnxt_db_write_relaxed()
2842 		writel_relaxed(db_val, db->doorbell);  in bnxt_db_write_relaxed()
2843 		if (bp->flags & BNXT_FLAG_DOUBLE_DB)  in bnxt_db_write_relaxed()
2844 			writel_relaxed(db_val, db->doorbell);  in bnxt_db_write_relaxed()
2852 	if (bp->flags & BNXT_FLAG_CHIP_P5_PLUS) {  in bnxt_db_write()
2853 		bnxt_writeq(bp, db->db_key64 | DB_RING_IDX(db, idx),  in bnxt_db_write()
2854 			    db->doorbell);  in bnxt_db_write()
2856 		u32 db_val = db->db_key32 | DB_RING_IDX(db, idx);  in bnxt_db_write()
2858 		writel(db_val, db->doorbell);  in bnxt_db_write()
2859 		if (bp->flags & BNXT_FLAG_DOUBLE_DB)  in bnxt_db_write()
2860 			writel(db_val, db->doorbell);  in bnxt_db_write()
2868 	return BNXT_PF(bp) && (bp->pf.active_vfs || bp->sriov_cfg);  in bnxt_sriov_cfg()