1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* Texas Instruments ICSSG Ethernet driver
3 *
4 * Copyright (C) 2018-2022 Texas Instruments Incorporated - https://www.ti.com/
5 *
6 */
7
8 #ifndef __NET_TI_ICSSG_PRUETH_H
9 #define __NET_TI_ICSSG_PRUETH_H
10
11 #include <linux/bpf.h>
12 #include <linux/bpf_trace.h>
13 #include <linux/etherdevice.h>
14 #include <linux/genalloc.h>
15 #include <linux/if_vlan.h>
16 #include <linux/if_hsr.h>
17 #include <linux/interrupt.h>
18 #include <linux/kernel.h>
19 #include <linux/mfd/syscon.h>
20 #include <linux/module.h>
21 #include <linux/mutex.h>
22 #include <linux/net_tstamp.h>
23 #include <linux/of.h>
24 #include <linux/of_irq.h>
25 #include <linux/of_mdio.h>
26 #include <linux/of_net.h>
27 #include <linux/of_platform.h>
28 #include <linux/phy.h>
29 #include <linux/remoteproc/pruss.h>
30 #include <linux/pruss_driver.h>
31 #include <linux/ptp_clock_kernel.h>
32 #include <linux/remoteproc.h>
33
34 #include <linux/dma-mapping.h>
35 #include <linux/dma/ti-cppi5.h>
36 #include <linux/dma/k3-udma-glue.h>
37
38 #include <net/devlink.h>
39 #include <net/xdp.h>
40 #include <net/page_pool/helpers.h>
41 #include <net/xsk_buff_pool.h>
42 #include <net/xdp_sock_drv.h>
43
44 #include "icssg_config.h"
45 #include "icss_iep.h"
46 #include "icssg_switch_map.h"
47
48 #define PRUETH_MAX_MTU (2000 - ETH_HLEN - ETH_FCS_LEN)
49 #define PRUETH_MIN_PKT_SIZE (VLAN_ETH_ZLEN)
50 #define PRUETH_MAX_PKT_SIZE (PRUETH_MAX_MTU + ETH_HLEN + ETH_FCS_LEN)
51
52 #define ICSS_SLICE0 0
53 #define ICSS_SLICE1 1
54
55 #define ICSS_FW_PRU 0
56 #define ICSS_FW_RTU 1
57
58 #define ICSSG_MAX_RFLOWS 8 /* per slice */
59
60 #define ICSSG_NUM_PA_STATS 32
61 #define ICSSG_NUM_MIIG_STATS 60
62 /* Number of ICSSG related stats */
63 #define ICSSG_NUM_STATS (ICSSG_NUM_MIIG_STATS + ICSSG_NUM_PA_STATS)
64 #define ICSSG_NUM_STANDARD_STATS 31
65 #define ICSSG_NUM_ETHTOOL_STATS (ICSSG_NUM_STATS - ICSSG_NUM_STANDARD_STATS)
66
67 #define IEP_DEFAULT_CYCLE_TIME_NS 1000000 /* 1 ms */
68
69 #define PRUETH_UNDIRECTED_PKT_DST_TAG 0
70 #define PRUETH_UNDIRECTED_PKT_TAG_INS BIT(30)
71
72 /* Firmware status codes */
73 #define ICSS_HS_FW_READY 0x55555555
74 #define ICSS_HS_FW_DEAD 0xDEAD0000 /* lower 16 bits contain error code */
75
76 /* Firmware command codes */
77 #define ICSS_HS_CMD_BUSY 0x40000000
78 #define ICSS_HS_CMD_DONE 0x80000000
79 #define ICSS_HS_CMD_CANCEL 0x10000000
80
81 /* Firmware commands */
82 #define ICSS_CMD_SPAD 0x20
83 #define ICSS_CMD_RXTX 0x10
84 #define ICSS_CMD_ADD_FDB 0x1
85 #define ICSS_CMD_DEL_FDB 0x2
86 #define ICSS_CMD_SET_RUN 0x4
87 #define ICSS_CMD_GET_FDB_SLOT 0x5
88 #define ICSS_CMD_ENABLE_VLAN 0x5
89 #define ICSS_CMD_DISABLE_VLAN 0x6
90 #define ICSS_CMD_ADD_FILTER 0x7
91 #define ICSS_CMD_ADD_MAC 0x8
92
93 /* VLAN Filtering Related MACROs */
94 #define PRUETH_DFLT_VLAN_HSR 1
95 #define PRUETH_DFLT_VLAN_SW 1
96 #define PRUETH_DFLT_VLAN_MAC 0
97 #define MAX_VLAN_ID 256
98
99 /* In switch mode there are 3 real ports i.e. 3 mac addrs.
100 * however Linux sees only the host side port. The other 2 ports
101 * are the switch ports.
102 * In emac mode there are 2 real ports i.e. 2 mac addrs.
103 * Linux sees both the ports.
104 */
105 enum prueth_port {
106 PRUETH_PORT_HOST = 0, /* host side port */
107 PRUETH_PORT_MII0, /* physical port RG/SG MII 0 */
108 PRUETH_PORT_MII1, /* physical port RG/SG MII 1 */
109 PRUETH_PORT_INVALID, /* Invalid prueth port */
110 };
111
112 enum prueth_mac {
113 PRUETH_MAC0 = 0,
114 PRUETH_MAC1,
115 PRUETH_NUM_MACS,
116 PRUETH_MAC_INVALID,
117 };
118
119 struct prueth_tx_chn {
120 struct device *dma_dev;
121 struct napi_struct napi_tx;
122 struct k3_cppi_desc_pool *desc_pool;
123 struct k3_udma_glue_tx_channel *tx_chn;
124 struct prueth_emac *emac;
125 u32 id;
126 u32 descs_num;
127 unsigned int irq;
128 char name[32];
129 struct hrtimer tx_hrtimer;
130 unsigned long tx_pace_timeout_ns;
131 struct xsk_buff_pool *xsk_pool;
132 bool irq_disabled;
133 };
134
135 struct prueth_rx_chn {
136 struct device *dev;
137 struct device *dma_dev;
138 struct k3_cppi_desc_pool *desc_pool;
139 struct k3_udma_glue_rx_channel *rx_chn;
140 u32 descs_num;
141 unsigned int irq[ICSSG_MAX_RFLOWS]; /* separate irq per flow */
142 char name[32];
143 struct page_pool *pg_pool;
144 struct xdp_rxq_info xdp_rxq;
145 struct xsk_buff_pool *xsk_pool;
146 bool irq_disabled;
147 };
148
149 enum prueth_swdata_type {
150 PRUETH_SWDATA_INVALID = 0,
151 PRUETH_SWDATA_SKB,
152 PRUETH_SWDATA_PAGE,
153 PRUETH_SWDATA_CMD,
154 PRUETH_SWDATA_XDPF,
155 PRUETH_SWDATA_XSK,
156 };
157
158 enum prueth_tx_buff_type {
159 PRUETH_TX_BUFF_TYPE_XDP_TX,
160 PRUETH_TX_BUFF_TYPE_XDP_NDO,
161 };
162
163 struct prueth_swdata {
164 enum prueth_swdata_type type;
165 union prueth_data {
166 struct sk_buff *skb;
167 struct page *page;
168 u32 cmd;
169 struct xdp_frame *xdpf;
170 struct xdp_buff *xdp;
171 } data;
172 };
173
174 /* There are 4 Tx DMA channels, but the highest priority is CH3 (thread 3)
175 * and lower three are lower priority channels or threads.
176 */
177 #define PRUETH_MAX_TX_QUEUES 4
178
179 #define PRUETH_MAX_TX_TS_REQUESTS 50 /* Max simultaneous TX_TS requests */
180
181 /* XDP BPF state */
182 #define ICSSG_XDP_PASS 0
183 #define ICSSG_XDP_CONSUMED BIT(0)
184 #define ICSSG_XDP_TX BIT(1)
185 #define ICSSG_XDP_REDIR BIT(2)
186
187 /* Minimum coalesce time in usecs for both Tx and Rx */
188 #define ICSSG_MIN_COALESCE_USECS 20
189
190 /* data for each emac port */
191 struct prueth_emac {
192 bool is_sr1;
193 struct prueth *prueth;
194 struct net_device *ndev;
195 u8 mac_addr[6];
196 struct napi_struct napi_rx;
197 u32 msg_enable;
198
199 int link;
200 int speed;
201 int duplex;
202
203 const char *phy_id;
204 struct device_node *phy_node;
205 phy_interface_t phy_if;
206 enum prueth_port port_id;
207 struct icss_iep *iep;
208 unsigned int rx_ts_enabled : 1;
209 unsigned int tx_ts_enabled : 1;
210 unsigned int half_duplex : 1;
211
212 /* DMA related */
213 struct prueth_tx_chn tx_chns[PRUETH_MAX_TX_QUEUES];
214 struct completion tdown_complete;
215 atomic_t tdown_cnt;
216 struct prueth_rx_chn rx_chns;
217 int rx_flow_id_base;
218 int tx_ch_num;
219
220 /* SR1.0 Management channel */
221 struct prueth_rx_chn rx_mgm_chn;
222 int rx_mgm_flow_id_base;
223
224 spinlock_t lock; /* serialize access */
225
226 /* TX HW Timestamping */
227 /* TX TS cookie will be index to the tx_ts_skb array */
228 struct sk_buff *tx_ts_skb[PRUETH_MAX_TX_TS_REQUESTS];
229 atomic_t tx_ts_pending;
230 int tx_ts_irq;
231
232 u8 cmd_seq;
233 /* shutdown related */
234 __le32 cmd_data[4];
235 struct completion cmd_complete;
236 /* Mutex to serialize access to firmware command interface */
237 struct mutex cmd_lock;
238 struct work_struct rx_mode_work;
239
240 struct pruss_mem_region dram;
241
242 bool offload_fwd_mark;
243 int port_vlan;
244
245 struct delayed_work stats_work;
246 u64 stats[ICSSG_NUM_MIIG_STATS];
247 u64 pa_stats[ICSSG_NUM_PA_STATS];
248
249 /* RX IRQ Coalescing Related */
250 struct hrtimer rx_hrtimer;
251 unsigned long rx_pace_timeout_ns;
252
253 struct netdev_hw_addr_list vlan_mcast_list[MAX_VLAN_ID];
254 struct bpf_prog *xdp_prog;
255 struct xdp_attachment_info xdpi;
256 int xsk_qid;
257 };
258
259 /* The buf includes headroom compatible with both skb and xdpf */
260 #define PRUETH_HEADROOM_NA (max(XDP_PACKET_HEADROOM, NET_SKB_PAD) + NET_IP_ALIGN)
261 #define PRUETH_HEADROOM ALIGN(PRUETH_HEADROOM_NA, sizeof(long))
262
263 /**
264 * struct prueth_pdata - PRUeth platform data
265 * @fdqring_mode: Free desc queue mode
266 * @quirk_10m_link_issue: 10M link detect errata
267 * @switch_mode: switch firmware support
268 * @banked_ms_ram: banked memory support
269 */
270 struct prueth_pdata {
271 enum k3_ring_mode fdqring_mode;
272 u32 quirk_10m_link_issue:1;
273 u32 switch_mode:1;
274 u32 banked_ms_ram:1;
275 };
276
277 struct icssg_firmwares {
278 const char *pru;
279 const char *rtu;
280 const char *txpru;
281 };
282
283 /**
284 * struct prueth - PRUeth structure
285 * @dev: device
286 * @pruss: pruss handle
287 * @pru: rproc instances of PRUs
288 * @rtu: rproc instances of RTUs
289 * @txpru: rproc instances of TX_PRUs
290 * @shram: PRUSS shared RAM region
291 * @sram_pool: MSMC RAM pool for buffers
292 * @msmcram: MSMC RAM region
293 * @eth_node: DT node for the port
294 * @emac: private EMAC data structure
295 * @registered_netdevs: list of registered netdevs
296 * @miig_rt: regmap to mii_g_rt block
297 * @mii_rt: regmap to mii_rt block
298 * @pa_stats: regmap to pa_stats block
299 * @pru_id: ID for each of the PRUs
300 * @pdev: pointer to ICSSG platform device
301 * @pdata: pointer to platform data for ICSSG driver
302 * @icssg_hwcmdseq: seq counter or HWQ messages
303 * @emacs_initialized: num of EMACs/ext ports that are up/running
304 * @iep0: pointer to IEP0 device
305 * @iep1: pointer to IEP1 device
306 * @vlan_tbl: VLAN-FID table pointer
307 * @hw_bridge_dev: pointer to HW bridge net device
308 * @hsr_dev: pointer to the HSR net device
309 * @hsr_prp_version: enum to store the protocol version of hsr master
310 * @br_members: bitmask of bridge member ports
311 * @hsr_members: bitmask of hsr member ports
312 * @prueth_netdevice_nb: netdevice notifier block
313 * @prueth_switchdev_nb: switchdev notifier block
314 * @prueth_switchdev_bl_nb: switchdev blocking notifier block
315 * @is_switch_mode: flag to indicate if device is in Switch mode
316 * @is_hsr_offload_mode: flag to indicate if device is in hsr offload mode
317 * @is_switchmode_supported: indicates platform support for switch mode
318 * @switch_id: ID for mapping switch ports to bridge
319 * @default_vlan: Default VLAN for host
320 * @icssg_emac_firmwares: Firmware names for EMAC mode, indexed per MAC
321 * @icssg_switch_firmwares: Firmware names for SWITCH mode, indexed per MAC
322 * @icssg_hsr_firmwares: Firmware names for HSR mode, indexed per MAC
323 * @icssg_prp_firmwares: Firmware names for PRP mode, indexed per MAC
324 */
325 struct prueth {
326 struct device *dev;
327 struct pruss *pruss;
328 struct rproc *pru[PRUSS_NUM_PRUS];
329 struct rproc *rtu[PRUSS_NUM_PRUS];
330 struct rproc *txpru[PRUSS_NUM_PRUS];
331 struct pruss_mem_region shram;
332 struct gen_pool *sram_pool;
333 struct pruss_mem_region msmcram;
334
335 struct device_node *eth_node[PRUETH_NUM_MACS];
336 struct prueth_emac *emac[PRUETH_NUM_MACS];
337 struct net_device *registered_netdevs[PRUETH_NUM_MACS];
338 struct regmap *miig_rt;
339 struct regmap *mii_rt;
340 struct regmap *pa_stats;
341
342 enum pruss_pru_id pru_id[PRUSS_NUM_PRUS];
343 struct platform_device *pdev;
344 struct prueth_pdata pdata;
345 u8 icssg_hwcmdseq;
346 int emacs_initialized;
347 struct icss_iep *iep0;
348 struct icss_iep *iep1;
349 struct prueth_vlan_tbl *vlan_tbl;
350
351 struct net_device *hw_bridge_dev;
352 struct net_device *hsr_dev;
353 enum hsr_version hsr_prp_version;
354 u8 br_members;
355 u8 hsr_members;
356 struct notifier_block prueth_netdevice_nb;
357 struct notifier_block prueth_switchdev_nb;
358 struct notifier_block prueth_switchdev_bl_nb;
359 bool is_switch_mode;
360 bool is_hsr_offload_mode;
361 bool is_switchmode_supported;
362 unsigned char switch_id[MAX_PHYS_ITEM_ID_LEN];
363 int default_vlan;
364 /** @vtbl_lock: Lock for vtbl in shared memory */
365 spinlock_t vtbl_lock;
366 /** @stats_lock: Lock for reading icssg stats */
367 spinlock_t stats_lock;
368 struct icssg_firmwares icssg_emac_firmwares[PRUETH_NUM_MACS];
369 struct icssg_firmwares icssg_switch_firmwares[PRUETH_NUM_MACS];
370 struct icssg_firmwares icssg_hsr_firmwares[PRUETH_NUM_MACS];
371 struct icssg_firmwares icssg_prp_firmwares[PRUETH_NUM_MACS];
372 };
373
374 struct emac_tx_ts_response {
375 u32 reserved[2];
376 u32 cookie;
377 u32 lo_ts;
378 u32 hi_ts;
379 };
380
381 struct emac_tx_ts_response_sr1 {
382 __le32 lo_ts;
383 __le32 hi_ts;
384 __le32 reserved;
385 __le32 cookie;
386 };
387
388 /* get PRUSS SLICE number from prueth_emac */
prueth_emac_slice(struct prueth_emac * emac)389 static inline int prueth_emac_slice(struct prueth_emac *emac)
390 {
391 switch (emac->port_id) {
392 case PRUETH_PORT_MII0:
393 return ICSS_SLICE0;
394 case PRUETH_PORT_MII1:
395 return ICSS_SLICE1;
396 default:
397 return -EINVAL;
398 }
399 }
400
401 extern const struct ethtool_ops icssg_ethtool_ops;
402 extern const struct dev_pm_ops prueth_dev_pm_ops;
403
icssg_read_time(const void __iomem * addr)404 static inline u64 icssg_read_time(const void __iomem *addr)
405 {
406 u32 low, high;
407
408 do {
409 high = readl(addr + 4);
410 low = readl(addr);
411 } while (high != readl(addr + 4));
412
413 return low + ((u64)high << 32);
414 }
415
416 /* Classifier helpers */
417 void icssg_class_set_mac_addr(struct regmap *miig_rt, int slice, u8 *mac);
418 void icssg_class_set_host_mac_addr(struct regmap *miig_rt, const u8 *mac);
419 void icssg_class_disable(struct regmap *miig_rt, int slice);
420 void icssg_class_default(struct regmap *miig_rt, int slice, bool allmulti,
421 bool is_sr1);
422 void icssg_class_promiscuous_sr1(struct regmap *miig_rt, int slice);
423 void icssg_class_add_mcast_sr1(struct regmap *miig_rt, int slice,
424 struct net_device *ndev);
425 void icssg_ft1_set_mac_addr(struct regmap *miig_rt, int slice, u8 *mac_addr);
426
427 /* config helpers */
428 void icssg_config_ipg(struct prueth_emac *emac);
429 int icssg_config(struct prueth *prueth, struct prueth_emac *emac,
430 int slice);
431 int icssg_set_port_state(struct prueth_emac *emac,
432 enum icssg_port_state_cmd state);
433 void icssg_config_set_speed(struct prueth_emac *emac);
434 void icssg_config_half_duplex(struct prueth_emac *emac);
435 void icssg_init_emac_mode(struct prueth *prueth);
436 void icssg_init_fw_offload_mode(struct prueth *prueth);
437
438 /* Buffer queue helpers */
439 int icssg_queue_pop(struct prueth *prueth, u8 queue);
440 void icssg_queue_push(struct prueth *prueth, int queue, u16 addr);
441 u32 icssg_queue_level(struct prueth *prueth, int queue);
442
443 int icssg_send_fdb_msg(struct prueth_emac *emac, struct mgmt_cmd *cmd,
444 struct mgmt_cmd_rsp *rsp);
445 int icssg_fdb_add_del(struct prueth_emac *emac, const unsigned char *addr,
446 u8 vid, u8 fid_c2, bool add);
447 int icssg_fdb_lookup(struct prueth_emac *emac, const unsigned char *addr,
448 u8 vid);
449 void icssg_vtbl_modify(struct prueth_emac *emac, u8 vid, u8 port_mask,
450 u8 untag_mask, bool add);
451 u16 icssg_get_pvid(struct prueth_emac *emac);
452 void icssg_set_pvid(struct prueth *prueth, u8 vid, u8 port);
453 int emac_fdb_flow_id_updated(struct prueth_emac *emac);
454 #define prueth_napi_to_tx_chn(pnapi) \
455 container_of(pnapi, struct prueth_tx_chn, napi_tx)
456
457 void icssg_stats_work_handler(struct work_struct *work);
458 void emac_update_hardware_stats(struct prueth_emac *emac);
459 int emac_get_stat_by_name(struct prueth_emac *emac, char *stat_name);
460
461 /* Common functions */
462 void prueth_cleanup_rx_chns(struct prueth_emac *emac,
463 struct prueth_rx_chn *rx_chn,
464 int max_rflows);
465 void prueth_cleanup_tx_chns(struct prueth_emac *emac);
466 void prueth_ndev_del_tx_napi(struct prueth_emac *emac, int num);
467 void prueth_xmit_free(struct prueth_tx_chn *tx_chn,
468 struct cppi5_host_desc_t *desc);
469 int emac_tx_complete_packets(struct prueth_emac *emac, int chn,
470 int budget, bool *tdown);
471 int prueth_ndev_add_tx_napi(struct prueth_emac *emac);
472 int prueth_init_tx_chns(struct prueth_emac *emac);
473 int prueth_init_rx_chns(struct prueth_emac *emac,
474 struct prueth_rx_chn *rx_chn,
475 char *name, u32 max_rflows,
476 u32 max_desc_num);
477 int prueth_dma_rx_push_mapped(struct prueth_emac *emac,
478 struct prueth_rx_chn *rx_chn,
479 struct page *page, u32 buf_len);
480 unsigned int prueth_rxbuf_total_len(unsigned int len);
481 void emac_rx_timestamp(struct prueth_emac *emac,
482 struct sk_buff *skb, u32 *psdata);
483 enum netdev_tx icssg_ndo_start_xmit(struct sk_buff *skb, struct net_device *ndev);
484 irqreturn_t prueth_rx_irq(int irq, void *dev_id);
485 void prueth_cleanup_tx_ts(struct prueth_emac *emac);
486 int icssg_napi_rx_poll(struct napi_struct *napi_rx, int budget);
487 int prueth_prepare_rx_chan(struct prueth_emac *emac,
488 struct prueth_rx_chn *chn,
489 int buf_size);
490 void prueth_reset_tx_chan(struct prueth_emac *emac, int ch_num,
491 bool free_skb);
492 void prueth_reset_rx_chan(struct prueth_rx_chn *chn,
493 int num_flows, bool disable);
494 void icssg_ndo_tx_timeout(struct net_device *ndev, unsigned int txqueue);
495 int icssg_ndo_get_ts_config(struct net_device *ndev,
496 struct kernel_hwtstamp_config *config);
497 int icssg_ndo_set_ts_config(struct net_device *ndev,
498 struct kernel_hwtstamp_config *config,
499 struct netlink_ext_ack *extack);
500 void icssg_ndo_get_stats64(struct net_device *ndev,
501 struct rtnl_link_stats64 *stats);
502 int icssg_ndo_get_phys_port_name(struct net_device *ndev, char *name,
503 size_t len);
504 int prueth_node_port(struct device_node *eth_node);
505 int prueth_node_mac(struct device_node *eth_node);
506 void prueth_netdev_exit(struct prueth *prueth,
507 struct device_node *eth_node);
508 int prueth_get_cores(struct prueth *prueth, int slice, bool is_sr1);
509 void prueth_put_cores(struct prueth *prueth, int slice);
510
511 /* Revision specific helper */
512 u64 icssg_ts_to_ns(u32 hi_sw, u32 hi, u32 lo, u32 cycle_time_ns);
513 u32 emac_xmit_xdp_frame(struct prueth_emac *emac,
514 struct xdp_frame *xdpf,
515 unsigned int q_idx,
516 enum prueth_tx_buff_type buff_type);
517 void prueth_rx_cleanup(void *data, dma_addr_t desc_dma);
518 void prueth_tx_cleanup(void *data, dma_addr_t desc_dma);
519 int prueth_xsk_wakeup(struct net_device *ndev, u32 qid, u32 flags);
prueth_xdp_is_enabled(struct prueth_emac * emac)520 static inline bool prueth_xdp_is_enabled(struct prueth_emac *emac)
521 {
522 return !!READ_ONCE(emac->xdp_prog);
523 }
524
525 #endif /* __NET_TI_ICSSG_PRUETH_H */
526