xref: /linux/drivers/net/wireless/mediatek/mt76/mt7996/main.c (revision 91a4855d6c03e770e42f17c798a36a3c46e63de2)
1 // SPDX-License-Identifier: BSD-3-Clause-Clear
2 /*
3  * Copyright (C) 2022 MediaTek Inc.
4  */
5 
6 #include "mt7996.h"
7 #include "mcu.h"
8 #include "mac.h"
9 
10 int mt7996_run(struct mt7996_phy *phy)
11 {
12 	struct mt7996_dev *dev = phy->dev;
13 	int ret;
14 
15 	mt7996_mac_enable_nf(dev, phy->mt76->band_idx);
16 
17 	ret = mt7996_mcu_set_rts_thresh(phy, 0x92b);
18 	if (ret)
19 		return ret;
20 
21 	ret = mt7996_mcu_set_radio_en(phy, true);
22 	if (ret)
23 		return ret;
24 
25 	ret = mt7996_mcu_set_chan_info(phy, UNI_CHANNEL_RX_PATH);
26 	if (ret)
27 		return ret;
28 
29 	ret = mt7996_mcu_set_thermal_throttling(phy, MT7996_THERMAL_THROTTLE_MAX);
30 	if (ret)
31 		return ret;
32 
33 	ret = mt7996_mcu_set_thermal_protect(phy, true);
34 	if (ret)
35 		return ret;
36 
37 	set_bit(MT76_STATE_RUNNING, &phy->mt76->state);
38 
39 	ieee80211_queue_delayed_work(dev->mphy.hw, &phy->mt76->mac_work,
40 				     MT7996_WATCHDOG_TIME);
41 
42 	if (!phy->counter_reset) {
43 		mt7996_mac_reset_counters(phy);
44 		phy->counter_reset = true;
45 	}
46 
47 	return 0;
48 }
49 
50 static int mt7996_start(struct ieee80211_hw *hw)
51 {
52 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
53 	int ret;
54 
55 	flush_work(&dev->init_work);
56 
57 	mutex_lock(&dev->mt76.mutex);
58 	ret = mt7996_mcu_set_hdr_trans(dev, true);
59 	if (!ret && !is_mt7996(&dev->mt76)) {
60 		u8 queue = mt76_connac_lmac_mapping(IEEE80211_AC_VI);
61 
62 		ret = mt7996_mcu_cp_support(dev, queue);
63 	}
64 	mutex_unlock(&dev->mt76.mutex);
65 
66 	return ret;
67 }
68 
69 static void mt7996_stop_phy(struct mt7996_phy *phy)
70 {
71 	struct mt7996_dev *dev;
72 
73 	if (!phy || !test_bit(MT76_STATE_RUNNING, &phy->mt76->state))
74 		return;
75 
76 	dev = phy->dev;
77 
78 	cancel_delayed_work_sync(&phy->mt76->mac_work);
79 
80 	mutex_lock(&dev->mt76.mutex);
81 
82 	mt7996_mcu_rdd_resume_tx(phy);
83 	mt7996_mcu_set_radio_en(phy, false);
84 
85 	clear_bit(MT76_STATE_RUNNING, &phy->mt76->state);
86 
87 	mutex_unlock(&dev->mt76.mutex);
88 }
89 
90 static void mt7996_stop(struct ieee80211_hw *hw, bool suspend)
91 {
92 }
93 
94 static inline int get_free_idx(u64 mask, u8 start, u8 end)
95 {
96 	if (~mask & GENMASK_ULL(end, start))
97 		return __ffs64(~mask & GENMASK_ULL(end, start)) + 1;
98 	return 0;
99 }
100 
101 static int get_omac_idx(enum nl80211_iftype type, u64 mask)
102 {
103 	int i;
104 
105 	switch (type) {
106 	case NL80211_IFTYPE_MESH_POINT:
107 	case NL80211_IFTYPE_ADHOC:
108 	case NL80211_IFTYPE_STATION:
109 		/* prefer hw bssid slot 1-3 */
110 		i = get_free_idx(mask, HW_BSSID_1, HW_BSSID_3);
111 		if (i)
112 			return i - 1;
113 
114 		if (type != NL80211_IFTYPE_STATION)
115 			break;
116 
117 		i = get_free_idx(mask, EXT_BSSID_1, EXT_BSSID_MAX);
118 		if (i)
119 			return i - 1;
120 
121 		if (~mask & BIT(HW_BSSID_0))
122 			return HW_BSSID_0;
123 
124 		break;
125 	case NL80211_IFTYPE_MONITOR:
126 	case NL80211_IFTYPE_AP:
127 		/* ap uses hw bssid 0 and ext bssid */
128 		if (~mask & BIT(HW_BSSID_0))
129 			return HW_BSSID_0;
130 
131 		i = get_free_idx(mask, EXT_BSSID_1, EXT_BSSID_MAX);
132 		if (i)
133 			return i - 1;
134 
135 		break;
136 	default:
137 		WARN_ON(1);
138 		break;
139 	}
140 
141 	return -1;
142 }
143 
144 static int get_own_mld_idx(u64 mask, bool group_mld)
145 {
146 	u8 start = group_mld ? 0 : 16;
147 	u8 end = group_mld ? 15 : 63;
148 	int idx;
149 
150 	idx = get_free_idx(mask, start, end);
151 	if (idx)
152 		return idx - 1;
153 
154 	/* If the 16-63 range is not available, perform another lookup in the
155 	 * range 0-15
156 	 */
157 	if (!group_mld) {
158 		idx = get_free_idx(mask, 0, 15);
159 		if (idx)
160 			return idx - 1;
161 	}
162 
163 	return -EINVAL;
164 }
165 
166 static void
167 mt7996_init_bitrate_mask(struct ieee80211_vif *vif, struct mt7996_vif_link *mlink)
168 {
169 	int i;
170 
171 	for (i = 0; i < ARRAY_SIZE(mlink->bitrate_mask.control); i++) {
172 		mlink->bitrate_mask.control[i].gi = NL80211_TXRATE_DEFAULT_GI;
173 		mlink->bitrate_mask.control[i].he_gi = 0xff;
174 		mlink->bitrate_mask.control[i].he_ltf = 0xff;
175 		mlink->bitrate_mask.control[i].legacy = GENMASK(31, 0);
176 		memset(mlink->bitrate_mask.control[i].ht_mcs, 0xff,
177 		       sizeof(mlink->bitrate_mask.control[i].ht_mcs));
178 		memset(mlink->bitrate_mask.control[i].vht_mcs, 0xff,
179 		       sizeof(mlink->bitrate_mask.control[i].vht_mcs));
180 		memset(mlink->bitrate_mask.control[i].he_mcs, 0xff,
181 		       sizeof(mlink->bitrate_mask.control[i].he_mcs));
182 	}
183 }
184 
185 static int
186 mt7996_set_hw_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
187 		  struct ieee80211_vif *vif, struct ieee80211_sta *sta,
188 		  unsigned int link_id, struct ieee80211_key_conf *key)
189 {
190 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
191 	struct ieee80211_bss_conf *link_conf;
192 	struct mt7996_sta_link *msta_link;
193 	struct mt7996_vif_link *link;
194 	int idx = key->keyidx;
195 	u8 *wcid_keyidx;
196 	bool is_bigtk;
197 	int err;
198 
199 	link = mt7996_vif_link(dev, vif, link_id);
200 	if (!link)
201 		return 0;
202 
203 	if (!mt7996_vif_link_phy(link))
204 		return 0;
205 
206 	if (sta) {
207 		struct mt7996_sta *msta;
208 
209 		msta = (struct mt7996_sta *)sta->drv_priv;
210 		msta_link = mt76_dereference(msta->link[link_id],
211 					     &dev->mt76);
212 		if (!msta_link)
213 			return 0;
214 
215 		if (!msta_link->wcid.sta)
216 			return -EOPNOTSUPP;
217 	} else {
218 		msta_link = &link->msta_link;
219 	}
220 	wcid_keyidx = &msta_link->wcid.hw_key_idx;
221 
222 	is_bigtk = key->keyidx == 6 || key->keyidx == 7;
223 	switch (key->cipher) {
224 	case WLAN_CIPHER_SUITE_AES_CMAC:
225 	case WLAN_CIPHER_SUITE_BIP_CMAC_256:
226 	case WLAN_CIPHER_SUITE_BIP_GMAC_128:
227 	case WLAN_CIPHER_SUITE_BIP_GMAC_256:
228 		if (is_bigtk) {
229 			wcid_keyidx = &msta_link->wcid.hw_key_idx2;
230 			key->flags |= IEEE80211_KEY_FLAG_GENERATE_MMIE;
231 		}
232 		break;
233 	default:
234 		break;
235 	}
236 
237 	link_conf = link_conf_dereference_protected(vif, link_id);
238 	if (!link_conf)
239 		link_conf = &vif->bss_conf;
240 
241 	if (cmd == SET_KEY && !sta && !link->mt76.cipher) {
242 		struct mt7996_phy *phy = mt7996_vif_link_phy(link);
243 
244 		link->mt76.cipher =
245 			mt76_connac_mcu_get_cipher(key->cipher);
246 		if (phy)
247 			mt7996_mcu_add_bss_info(phy, vif, link_conf,
248 						&link->mt76, msta_link, true);
249 	}
250 
251 	if (cmd == SET_KEY)
252 		*wcid_keyidx = idx;
253 	else if (idx == *wcid_keyidx)
254 		*wcid_keyidx = -1;
255 
256 	/* only do remove key for BIGTK */
257 	if (cmd != SET_KEY && !is_bigtk)
258 		return 0;
259 
260 	mt76_wcid_key_setup(&dev->mt76, &msta_link->wcid, key);
261 
262 	err = mt7996_mcu_add_key(&dev->mt76, link, key,
263 				 MCU_WMWA_UNI_CMD(STA_REC_UPDATE),
264 				 &msta_link->wcid, cmd);
265 
266 	/* remove and add beacon in order to enable beacon protection */
267 	if (cmd == SET_KEY && is_bigtk && link_conf->enable_beacon) {
268 		mt7996_mcu_add_beacon(hw, vif, link_conf, false);
269 		mt7996_mcu_add_beacon(hw, vif, link_conf, true);
270 	}
271 
272 	return err;
273 }
274 
275 struct mt7996_key_iter_data {
276     enum set_key_cmd cmd;
277     unsigned int link_id;
278 };
279 
280 static void
281 mt7996_key_iter(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
282 		struct ieee80211_sta *sta, struct ieee80211_key_conf *key,
283 		void *data)
284 {
285 	struct mt7996_key_iter_data *it = data;
286 
287 	if (sta)
288 		return;
289 
290 	WARN_ON(mt7996_set_hw_key(hw, it->cmd, vif, NULL, it->link_id, key));
291 }
292 
293 int mt7996_vif_link_add(struct mt76_phy *mphy, struct ieee80211_vif *vif,
294 			struct ieee80211_bss_conf *link_conf,
295 			struct mt76_vif_link *mlink)
296 {
297 	struct mt7996_vif_link *link = container_of(mlink, struct mt7996_vif_link, mt76);
298 	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
299 	struct mt7996_sta_link *msta_link = &link->msta_link;
300 	struct mt7996_phy *phy = mphy->priv;
301 	struct mt7996_dev *dev = phy->dev;
302 	u8 band_idx = phy->mt76->band_idx;
303 	struct mt7996_key_iter_data it = {
304 		.cmd = SET_KEY,
305 		.link_id = link_conf->link_id,
306 	};
307 	int mld_idx, idx, ret;
308 
309 	if ((mvif->mt76.valid_links & BIT(link_conf->link_id)) &&
310 	    !mlink->offchannel) {
311 		if (vif->type == NL80211_IFTYPE_AP)
312 			return mt7996_mcu_mld_link_oper(dev, link_conf, link,
313 							true);
314 		return 0;
315 	}
316 
317 	mlink->idx = __ffs64(~dev->mt76.vif_mask);
318 	if (mlink->idx >= mt7996_max_interface_num(dev))
319 		return -ENOSPC;
320 
321 	idx = get_omac_idx(vif->type, phy->omac_mask);
322 	if (idx < 0)
323 		return -ENOSPC;
324 
325 	mld_idx = get_own_mld_idx(dev->mld_idx_mask, false);
326 	if (mld_idx < 0)
327 		return -ENOSPC;
328 
329 	link->mld_idx = mld_idx;
330 	mlink->omac_idx = idx;
331 	mlink->band_idx = band_idx;
332 	mlink->wmm_idx = vif->type == NL80211_IFTYPE_AP ? 0 : 3;
333 	mlink->wcid = &msta_link->wcid;
334 	mlink->wcid->offchannel = mlink->offchannel;
335 
336 	ret = mt7996_mcu_add_dev_info(phy, vif, link_conf, mlink, true);
337 	if (ret)
338 		return ret;
339 
340 	dev->mt76.vif_mask |= BIT_ULL(mlink->idx);
341 	dev->mld_idx_mask |= BIT_ULL(link->mld_idx);
342 	phy->omac_mask |= BIT_ULL(mlink->omac_idx);
343 
344 	idx = MT7996_WTBL_RESERVED - mlink->idx;
345 
346 	INIT_LIST_HEAD(&msta_link->rc_list);
347 	msta_link->wcid.idx = idx;
348 	msta_link->wcid.link_id = link_conf->link_id;
349 	msta_link->wcid.link_valid = ieee80211_vif_is_mld(vif);
350 	msta_link->wcid.tx_info |= MT_WCID_TX_INFO_SET;
351 	mt76_wcid_init(&msta_link->wcid, band_idx);
352 
353 	mt7996_mac_wtbl_update(dev, idx,
354 			       MT_WTBL_UPDATE_ADM_COUNT_CLEAR);
355 
356 	if (vif->type != NL80211_IFTYPE_AP &&
357 	    (!mlink->omac_idx || mlink->omac_idx > 3))
358 		vif->offload_flags = 0;
359 
360 	if (phy->mt76->chandef.chan->band != NL80211_BAND_2GHZ)
361 		mlink->basic_rates_idx = MT7996_BASIC_RATES_TBL + 4;
362 	else
363 		mlink->basic_rates_idx = MT7996_BASIC_RATES_TBL;
364 
365 	mt7996_init_bitrate_mask(vif, link);
366 
367 	mt7996_mcu_add_bss_info(phy, vif, link_conf, mlink, msta_link, true);
368 	/* defer the first STA_REC of BMC entry to BSS_CHANGED_BSSID for STA
369 	 * interface, since firmware only records BSSID when the entry is new
370 	 */
371 	if (vif->type != NL80211_IFTYPE_STATION)
372 		mt7996_mcu_add_sta(dev, link_conf, NULL, link, NULL,
373 				   CONN_STATE_PORT_SECURE, true);
374 	rcu_assign_pointer(dev->mt76.wcid[idx], &msta_link->wcid);
375 
376 	ieee80211_iter_keys(mphy->hw, vif, mt7996_key_iter, &it);
377 
378 	if (!mlink->wcid->offchannel) {
379 		if (vif->txq &&
380 		    mvif->mt76.deflink_id == IEEE80211_LINK_UNSPECIFIED) {
381 			struct mt76_txq *mtxq;
382 
383 			mtxq = (struct mt76_txq *)vif->txq->drv_priv;
384 			mvif->mt76.deflink_id = link_conf->link_id;
385 			mtxq->wcid = idx;
386 		}
387 		mvif->mt76.valid_links |= BIT(link_conf->link_id);
388 	}
389 
390 	if (vif->type == NL80211_IFTYPE_STATION) {
391 		vif->driver_flags |= IEEE80211_VIF_BEACON_FILTER;
392 
393 		if (vif->cfg.assoc && link_conf->beacon_int) {
394 			mlink->beacon_mon_interval =
395 				msecs_to_jiffies(ieee80211_tu_to_usec(
396 					link_conf->beacon_int) / 1000);
397 			WRITE_ONCE(mlink->beacon_mon_last, jiffies);
398 		}
399 	}
400 
401 	return 0;
402 }
403 
404 static void mt7996_vif_link_destroy(struct mt7996_phy *phy,
405 				    struct mt7996_vif_link *link,
406 				    struct ieee80211_vif *vif,
407 				    struct ieee80211_bss_conf *link_conf)
408 {
409 	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
410 	struct mt7996_sta_link *msta_link = &link->msta_link;
411 	unsigned int link_id = msta_link->wcid.link_id;
412 	struct mt76_vif_link *mlink = &link->mt76;
413 	struct mt7996_key_iter_data it = {
414 		.cmd = SET_KEY,
415 		.link_id = link_id,
416 	};
417 	struct mt7996_dev *dev = phy->dev;
418 	int idx = msta_link->wcid.idx;
419 
420 	if (!link_conf)
421 		link_conf = &vif->bss_conf;
422 
423 	if (!mlink->wcid->offchannel)
424 		ieee80211_iter_keys(phy->mt76->hw, vif, mt7996_key_iter, &it);
425 
426 	mt7996_mcu_add_sta(dev, link_conf, NULL, link, NULL,
427 			   CONN_STATE_DISCONNECT, false);
428 	mt7996_mcu_add_bss_info(phy, vif, link_conf, mlink, msta_link, false);
429 	mt7996_mcu_add_dev_info(phy, vif, link_conf, mlink, false);
430 
431 	rcu_assign_pointer(dev->mt76.wcid[idx], NULL);
432 
433 	dev->mt76.vif_mask &= ~BIT_ULL(mlink->idx);
434 	dev->mld_idx_mask &= ~BIT_ULL(link->mld_idx);
435 	phy->omac_mask &= ~BIT_ULL(mlink->omac_idx);
436 	if (!mlink->wcid->offchannel)
437 		mvif->mt76.valid_links &= ~BIT(link_id);
438 
439 	spin_lock_bh(&dev->mt76.sta_poll_lock);
440 	if (!list_empty(&msta_link->wcid.poll_list))
441 		list_del_init(&msta_link->wcid.poll_list);
442 	spin_unlock_bh(&dev->mt76.sta_poll_lock);
443 
444 	mt76_wcid_cleanup(&dev->mt76, &msta_link->wcid);
445 
446 	if (mlink != (struct mt76_vif_link *)vif->drv_priv &&
447 	    !mlink->wcid->offchannel) {
448 		rcu_assign_pointer(mlink->mvif->link[link_id], NULL);
449 		kfree_rcu(mlink, rcu_head);
450 	}
451 }
452 
453 void mt7996_vif_link_remove(struct mt76_phy *mphy, struct ieee80211_vif *vif,
454 			    struct ieee80211_bss_conf *link_conf,
455 			    struct mt76_vif_link *mlink)
456 {
457 	struct mt7996_vif_link *link = container_of(mlink, struct mt7996_vif_link, mt76);
458 	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
459 	struct mt7996_sta_link *msta_link = &link->msta_link;
460 	unsigned int link_id = msta_link->wcid.link_id;
461 	struct mt7996_phy *phy = mphy->priv;
462 
463 	/* Hw requires to destroy active links tearing down the interface, so
464 	 * postpone it removing the interface.
465 	 */
466 	if (mlink->wcid->offchannel) {
467 		mt7996_vif_link_destroy(phy, link, vif, link_conf);
468 	} else {
469 		if (vif->type == NL80211_IFTYPE_AP) {
470 			mt7996_mcu_mld_reconf_stop_link(phy->dev, vif,
471 							BIT(link_id));
472 			mt7996_mcu_mld_link_oper(phy->dev, link_conf, link,
473 						 false);
474 		}
475 
476 		if (vif->txq && mvif->mt76.deflink_id == link_id) {
477 			struct ieee80211_bss_conf *iter;
478 			struct mt76_txq *mtxq;
479 
480 			mvif->mt76.deflink_id = IEEE80211_LINK_UNSPECIFIED;
481 			mtxq = (struct mt76_txq *)vif->txq->drv_priv;
482 			/* Primary link will be removed, look for a new one */
483 			for_each_vif_active_link(vif, iter, link_id) {
484 				if (link_id == msta_link->wcid.link_id)
485 					continue;
486 
487 				link = mt7996_vif_link(phy->dev, vif, link_id);
488 				if (!link)
489 					continue;
490 
491 				mtxq->wcid = link->msta_link.wcid.idx;
492 				mvif->mt76.deflink_id = link_id;
493 				break;
494 			}
495 		}
496 	}
497 }
498 
499 static void mt7996_phy_set_rxfilter(struct mt7996_phy *phy)
500 {
501 	struct mt7996_dev *dev = phy->dev;
502 	u32 ctl_flags = MT_WF_RFCR1_DROP_ACK |
503 			MT_WF_RFCR1_DROP_BF_POLL |
504 			MT_WF_RFCR1_DROP_BA |
505 			MT_WF_RFCR1_DROP_CFEND |
506 			MT_WF_RFCR1_DROP_CFACK;
507 	u32 filter = phy->rxfilter;
508 
509 	if (filter & MT_WF_RFCR_DROP_OTHER_UC) {
510 		filter |= MT_WF_RFCR_DROP_CTS |
511 			  MT_WF_RFCR_DROP_RTS |
512 			  MT_WF_RFCR_DROP_CTL_RSV |
513 			  MT_WF_RFCR_DROP_FCSFAIL;
514 	}
515 
516 	mt76_wr(dev, MT_WF_RFCR(phy->mt76->band_idx), filter);
517 	if (filter & MT_WF_RFCR_DROP_CTL_RSV)
518 		mt76_set(dev, MT_WF_RFCR1(phy->mt76->band_idx), ctl_flags);
519 	else
520 		mt76_clear(dev, MT_WF_RFCR1(phy->mt76->band_idx), ctl_flags);
521 }
522 
523 static void mt7996_set_monitor(struct mt7996_phy *phy, bool enabled)
524 {
525 	struct mt7996_dev *dev;
526 
527 	if (!phy)
528 		return;
529 
530 	dev = phy->dev;
531 
532 	if (enabled == !(phy->rxfilter & MT_WF_RFCR_DROP_OTHER_UC))
533 		return;
534 
535 	if (!enabled)
536 		phy->rxfilter |= MT_WF_RFCR_DROP_OTHER_UC;
537 	else
538 		phy->rxfilter &= ~MT_WF_RFCR_DROP_OTHER_UC;
539 
540 	mt76_rmw_field(dev, MT_DMA_DCR0(phy->mt76->band_idx),
541 		       MT_DMA_DCR0_RXD_G5_EN, enabled);
542 	mt76_rmw_field(dev, MT_MDP_DCR0,
543 		       MT_MDP_DCR0_RX_HDR_TRANS_EN, !enabled);
544 	mt7996_phy_set_rxfilter(phy);
545 	mt7996_mcu_set_sniffer_mode(phy, enabled);
546 }
547 
548 static int mt7996_add_interface(struct ieee80211_hw *hw,
549 				struct ieee80211_vif *vif)
550 {
551 	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
552 	struct wireless_dev *wdev = ieee80211_vif_to_wdev(vif);
553 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
554 	int i, err = 0;
555 
556 	mutex_lock(&dev->mt76.mutex);
557 
558 	for (i = 0; i < MT7996_MAX_RADIOS; i++) {
559 		struct mt7996_phy *phy = dev->radio_phy[i];
560 
561 		if (!phy || !(wdev->radio_mask & BIT(i)) ||
562 		    test_bit(MT76_STATE_RUNNING, &phy->mt76->state))
563 			continue;
564 
565 		err = mt7996_run(phy);
566 		if (err)
567 			goto out;
568 
569 		if (vif->type == NL80211_IFTYPE_MONITOR)
570 			mt7996_set_monitor(phy, true);
571 	}
572 
573 	mt76_vif_init(vif, &mvif->mt76);
574 
575 	vif->offload_flags |= IEEE80211_OFFLOAD_ENCAP_4ADDR;
576 	mvif->mt76.deflink_id = IEEE80211_LINK_UNSPECIFIED;
577 
578 out:
579 	mutex_unlock(&dev->mt76.mutex);
580 
581 	return err;
582 }
583 
584 struct mt7996_radio_data {
585 	u32 active_mask;
586 	u32 monitor_mask;
587 };
588 
589 static void mt7996_remove_iter(void *data, u8 *mac, struct ieee80211_vif *vif)
590 {
591 	struct wireless_dev *wdev = ieee80211_vif_to_wdev(vif);
592 	struct mt7996_radio_data *rdata = data;
593 
594 	rdata->active_mask |= wdev->radio_mask;
595 	if (vif->type == NL80211_IFTYPE_MONITOR)
596 		rdata->monitor_mask |= wdev->radio_mask;
597 }
598 
599 static void mt7996_remove_interface(struct ieee80211_hw *hw,
600 				    struct ieee80211_vif *vif)
601 {
602 	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
603 	unsigned long rem_links = mvif->mt76.valid_links;
604 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
605 	struct mt7996_radio_data rdata = {};
606 	unsigned int link_id;
607 	int i;
608 
609 	/* Remove all active links */
610 	for_each_set_bit(link_id, &rem_links, IEEE80211_MLD_MAX_NUM_LINKS) {
611 		struct mt7996_vif_link *link;
612 		struct mt7996_phy *phy;
613 
614 		link = mt7996_vif_link(dev, vif, link_id);
615 		if (!link)
616 			continue;
617 
618 		phy = __mt7996_phy(dev, link->msta_link.wcid.phy_idx);
619 		if (!phy)
620 			continue;
621 
622 		mt7996_vif_link_destroy(phy, link, vif, NULL);
623 	}
624 
625 	ieee80211_iterate_active_interfaces_mtx(hw, 0, mt7996_remove_iter,
626 						&rdata);
627 	mt76_vif_cleanup(&dev->mt76, vif);
628 
629 	for (i = 0; i < MT7996_MAX_RADIOS; i++) {
630 		struct mt7996_phy *phy = dev->radio_phy[i];
631 
632 		if (!phy)
633 			continue;
634 		if (!(rdata.monitor_mask & BIT(i)))
635 			mt7996_set_monitor(phy, false);
636 		if (!(rdata.active_mask & BIT(i)))
637 			mt7996_stop_phy(phy);
638 	}
639 }
640 
641 int mt7996_set_channel(struct mt76_phy *mphy)
642 {
643 	struct mt7996_phy *phy = mphy->priv;
644 	int ret;
645 
646 	if (mphy->offchannel)
647 		mt7996_mac_update_beacons(phy);
648 
649 	ret = mt7996_mcu_set_chan_info(phy, UNI_CHANNEL_SWITCH);
650 	if (ret)
651 		goto out;
652 
653 	ret = mt7996_mcu_set_chan_info(phy, UNI_CHANNEL_RX_PATH);
654 	if (ret)
655 		goto out;
656 
657 	ret = mt7996_mcu_set_txpower_sku(phy);
658 	if (ret)
659 		goto out;
660 
661 	ret = mt7996_dfs_init_radar_detector(phy);
662 	mt7996_mac_cca_stats_reset(phy);
663 
664 	mt7996_mac_reset_counters(phy);
665 	phy->noise = 0;
666 	if (!mphy->offchannel)
667 		mt7996_mac_update_beacons(phy);
668 
669 out:
670 	ieee80211_queue_delayed_work(mphy->hw, &mphy->mac_work,
671 				     MT7996_WATCHDOG_TIME);
672 
673 	return ret;
674 }
675 
676 static int mt7996_set_key(struct ieee80211_hw *hw, enum set_key_cmd cmd,
677 			  struct ieee80211_vif *vif, struct ieee80211_sta *sta,
678 			  struct ieee80211_key_conf *key)
679 {
680 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
681 	unsigned int link_id;
682 	unsigned long links;
683 	int err = 0;
684 
685 	/* The hardware does not support per-STA RX GTK, fallback
686 	 * to software mode for these.
687 	 */
688 	if ((vif->type == NL80211_IFTYPE_ADHOC ||
689 	     vif->type == NL80211_IFTYPE_MESH_POINT) &&
690 	    (key->cipher == WLAN_CIPHER_SUITE_TKIP ||
691 	     key->cipher == WLAN_CIPHER_SUITE_CCMP) &&
692 	    !(key->flags & IEEE80211_KEY_FLAG_PAIRWISE))
693 		return -EOPNOTSUPP;
694 
695 	/* fall back to sw encryption for unsupported ciphers */
696 	switch (key->cipher) {
697 	case WLAN_CIPHER_SUITE_TKIP:
698 	case WLAN_CIPHER_SUITE_CCMP:
699 	case WLAN_CIPHER_SUITE_CCMP_256:
700 	case WLAN_CIPHER_SUITE_GCMP:
701 	case WLAN_CIPHER_SUITE_GCMP_256:
702 	case WLAN_CIPHER_SUITE_SMS4:
703 		break;
704 	case WLAN_CIPHER_SUITE_AES_CMAC:
705 	case WLAN_CIPHER_SUITE_BIP_CMAC_256:
706 	case WLAN_CIPHER_SUITE_BIP_GMAC_128:
707 	case WLAN_CIPHER_SUITE_BIP_GMAC_256:
708 		if (key->keyidx == 6 || key->keyidx == 7)
709 			break;
710 		fallthrough;
711 	case WLAN_CIPHER_SUITE_WEP40:
712 	case WLAN_CIPHER_SUITE_WEP104:
713 	default:
714 		return -EOPNOTSUPP;
715 	}
716 
717 	mutex_lock(&dev->mt76.mutex);
718 
719 	if (key->link_id >= 0)
720 		links = BIT(key->link_id);
721 	else if (sta && sta->valid_links)
722 		links = sta->valid_links;
723 	else if (vif->valid_links)
724 		links = vif->valid_links;
725 	else
726 		links = BIT(0);
727 
728 	for_each_set_bit(link_id, &links, IEEE80211_MLD_MAX_NUM_LINKS) {
729 		err = mt7996_set_hw_key(hw, cmd, vif, sta, link_id, key);
730 		if (err)
731 			break;
732 	}
733 	mutex_unlock(&dev->mt76.mutex);
734 
735 	return err;
736 }
737 
738 static int mt7996_config(struct ieee80211_hw *hw, int radio_idx, u32 changed)
739 {
740 	return 0;
741 }
742 
743 static int
744 mt7996_conf_tx(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
745 	       unsigned int link_id, u16 queue,
746 	       const struct ieee80211_tx_queue_params *params)
747 {
748 	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
749 	struct mt7996_vif_link_info *link_info = &mvif->link_info[link_id];
750 	static const u8 mq_to_aci[] = {
751 		[IEEE80211_AC_VO] = 3,
752 		[IEEE80211_AC_VI] = 2,
753 		[IEEE80211_AC_BE] = 0,
754 		[IEEE80211_AC_BK] = 1,
755 	};
756 
757 	/* firmware uses access class index */
758 	link_info->queue_params[mq_to_aci[queue]] = *params;
759 	/* no need to update right away, we'll get BSS_CHANGED_QOS */
760 
761 	return 0;
762 }
763 
764 static void mt7996_configure_filter(struct ieee80211_hw *hw,
765 				    unsigned int changed_flags,
766 				    unsigned int *total_flags,
767 				    u64 multicast)
768 {
769 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
770 	struct mt7996_phy *phy;
771 	u32 filter_mask = 0, filter_set = 0;
772 	u32 flags = 0;
773 
774 #define MT76_FILTER(_flag, _hw) do {				\
775 		flags |= *total_flags & FIF_##_flag;		\
776 		filter_mask |= (_hw);				\
777 		filter_set |= !(flags & FIF_##_flag) * (_hw);	\
778 	} while (0)
779 
780 	mutex_lock(&dev->mt76.mutex);
781 
782 	MT76_FILTER(OTHER_BSS, MT_WF_RFCR_DROP_OTHER_TIM |
783 			       MT_WF_RFCR_DROP_A3_MAC |
784 			       MT_WF_RFCR_DROP_A3_BSSID);
785 
786 	MT76_FILTER(FCSFAIL, MT_WF_RFCR_DROP_FCSFAIL);
787 
788 	MT76_FILTER(CONTROL, MT_WF_RFCR_DROP_CTS |
789 			     MT_WF_RFCR_DROP_RTS |
790 			     MT_WF_RFCR_DROP_CTL_RSV);
791 
792 	*total_flags = flags;
793 
794 	mt7996_for_each_phy(dev, phy) {
795 		phy->rxfilter &= ~(MT_WF_RFCR_DROP_OTHER_BSS |
796 				   MT_WF_RFCR_DROP_OTHER_BEACON |
797 				   MT_WF_RFCR_DROP_FRAME_REPORT |
798 				   MT_WF_RFCR_DROP_PROBEREQ |
799 				   MT_WF_RFCR_DROP_MCAST_FILTERED |
800 				   MT_WF_RFCR_DROP_MCAST |
801 				   MT_WF_RFCR_DROP_BCAST |
802 				   MT_WF_RFCR_DROP_DUPLICATE |
803 				   MT_WF_RFCR_DROP_A2_BSSID |
804 				   MT_WF_RFCR_DROP_UNWANTED_CTL |
805 				   MT_WF_RFCR_DROP_STBC_MULTI |
806 				   filter_mask);
807 		phy->rxfilter |= filter_set;
808 		mt7996_phy_set_rxfilter(phy);
809 	}
810 
811 	mutex_unlock(&dev->mt76.mutex);
812 }
813 
814 static int
815 mt7996_get_txpower(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
816 		   unsigned int link_id, int *dbm)
817 {
818 	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
819 	struct mt7996_phy *phy = mt7996_vif_link_phy(&mvif->deflink);
820 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
821 	struct wireless_dev *wdev;
822 	int n_chains, delta, i;
823 
824 	if (!phy) {
825 		wdev = ieee80211_vif_to_wdev(vif);
826 		for (i = 0; i < hw->wiphy->n_radio; i++)
827 			if (wdev->radio_mask & BIT(i))
828 				phy = dev->radio_phy[i];
829 
830 		if (!phy)
831 			return -EINVAL;
832 	}
833 
834 	n_chains = hweight16(phy->mt76->chainmask);
835 	delta = mt76_tx_power_path_delta(n_chains);
836 	*dbm = DIV_ROUND_UP(phy->mt76->txpower_cur + delta, 2);
837 
838 	return 0;
839 }
840 
841 static u8
842 mt7996_get_rates_table(struct mt7996_phy *phy, struct ieee80211_bss_conf *conf,
843 		       bool beacon, bool mcast)
844 {
845 	struct mt7996_dev *dev = phy->dev;
846 	struct mt76_vif_link *mvif = mt76_vif_conf_link(&dev->mt76, conf->vif, conf);
847 	u16 rate;
848 	u8 i, idx;
849 
850 	rate = mt76_connac2_mac_tx_rate_val(phy->mt76, conf, beacon, mcast);
851 
852 	if (beacon) {
853 		/* odd index for driver, even index for firmware */
854 		idx = MT7996_BEACON_RATES_TBL + 2 * phy->mt76->band_idx;
855 		if (phy->beacon_rate != rate)
856 			mt7996_mcu_set_fixed_rate_table(phy, idx, rate, beacon);
857 
858 		return idx;
859 	}
860 
861 	idx = FIELD_GET(MT_TX_RATE_IDX, rate);
862 	for (i = 0; i < ARRAY_SIZE(mt76_rates); i++)
863 		if ((mt76_rates[i].hw_value & GENMASK(7, 0)) == idx)
864 			return MT7996_BASIC_RATES_TBL + 2 * i;
865 
866 	return mvif->basic_rates_idx;
867 }
868 
869 static void
870 mt7996_update_mu_group(struct ieee80211_hw *hw, struct mt7996_vif_link *link,
871 		       struct ieee80211_bss_conf *info)
872 {
873 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
874 	u8 band = link->mt76.band_idx;
875 	u32 *mu;
876 
877 	mu = (u32 *)info->mu_group.membership;
878 	mt76_wr(dev, MT_WF_PHYRX_BAND_GID_TAB_VLD0(band), mu[0]);
879 	mt76_wr(dev, MT_WF_PHYRX_BAND_GID_TAB_VLD1(band), mu[1]);
880 
881 	mu = (u32 *)info->mu_group.position;
882 	mt76_wr(dev, MT_WF_PHYRX_BAND_GID_TAB_POS0(band), mu[0]);
883 	mt76_wr(dev, MT_WF_PHYRX_BAND_GID_TAB_POS1(band), mu[1]);
884 	mt76_wr(dev, MT_WF_PHYRX_BAND_GID_TAB_POS2(band), mu[2]);
885 	mt76_wr(dev, MT_WF_PHYRX_BAND_GID_TAB_POS3(band), mu[3]);
886 }
887 
888 static void
889 mt7996_vif_cfg_changed(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
890 		       u64 changed)
891 {
892 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
893 
894 	mutex_lock(&dev->mt76.mutex);
895 
896 	if ((changed & BSS_CHANGED_ASSOC) && vif->cfg.assoc) {
897 		struct ieee80211_bss_conf *link_conf;
898 		unsigned long link_id;
899 
900 		for_each_vif_active_link(vif, link_conf, link_id) {
901 			struct mt7996_vif_link *link;
902 			struct mt7996_phy *phy;
903 
904 			link = mt7996_vif_link(dev, vif, link_id);
905 			if (!link)
906 				continue;
907 
908 			if (vif->type == NL80211_IFTYPE_STATION) {
909 				link->mt76.beacon_mon_interval =
910 					msecs_to_jiffies(ieee80211_tu_to_usec(
911 						link_conf->beacon_int) / 1000);
912 				WRITE_ONCE(link->mt76.beacon_mon_last, jiffies);
913 			}
914 
915 			phy = mt7996_vif_link_phy(link);
916 			if (!phy)
917 				continue;
918 
919 			mt7996_mcu_add_bss_info(phy, vif, link_conf,
920 						&link->mt76, &link->msta_link,
921 						true);
922 			mt7996_mcu_add_sta(dev, link_conf, NULL, link, NULL,
923 					   CONN_STATE_PORT_SECURE,
924 					   !!(changed & BSS_CHANGED_BSSID));
925 		}
926 	}
927 
928 	if ((changed & BSS_CHANGED_ASSOC) && !vif->cfg.assoc &&
929 	    vif->type == NL80211_IFTYPE_STATION) {
930 		struct ieee80211_bss_conf *link_conf;
931 		unsigned long link_id;
932 
933 		for_each_vif_active_link(vif, link_conf, link_id) {
934 			struct mt7996_vif_link *link;
935 
936 			link = mt7996_vif_link(dev, vif, link_id);
937 			if (link)
938 				link->mt76.beacon_mon_interval = 0;
939 		}
940 	}
941 
942 	mutex_unlock(&dev->mt76.mutex);
943 }
944 
945 static void
946 mt7996_link_info_changed(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
947 			 struct ieee80211_bss_conf *info, u64 changed)
948 {
949 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
950 	struct mt7996_vif_link *link;
951 	struct mt7996_phy *phy;
952 	struct mt76_phy *mphy;
953 
954 	mutex_lock(&dev->mt76.mutex);
955 
956 	link = mt7996_vif_conf_link(dev, vif, info);
957 	if (!link)
958 		goto out;
959 
960 	mphy = mt76_vif_link_phy(&link->mt76);
961 	if (!mphy)
962 		goto out;
963 
964 	phy = mphy->priv;
965 
966 	/* station mode uses BSSID to map the wlan entry to a peer,
967 	 * and then peer references bss_info_rfch to set bandwidth cap.
968 	 */
969 	if ((changed & BSS_CHANGED_BSSID && !is_zero_ether_addr(info->bssid)) ||
970 	    (changed & BSS_CHANGED_BEACON_ENABLED && info->enable_beacon)) {
971 		mt7996_mcu_add_bss_info(phy, vif, info, &link->mt76,
972 					&link->msta_link, true);
973 		mt7996_mcu_add_sta(dev, info, NULL, link, NULL,
974 				   CONN_STATE_PORT_SECURE,
975 				   !!(changed & BSS_CHANGED_BSSID));
976 	}
977 
978 	if (changed & BSS_CHANGED_HT || changed & BSS_CHANGED_ERP_CTS_PROT)
979 		mt7996_mcu_set_protection(phy, link, info->ht_operation_mode,
980 					  info->use_cts_prot);
981 
982 	if (changed & BSS_CHANGED_ERP_SLOT) {
983 		int slottime = info->use_short_slot ? 9 : 20;
984 
985 		if (slottime != phy->slottime) {
986 			phy->slottime = slottime;
987 			mt7996_mcu_set_timing(phy, vif, info);
988 		}
989 	}
990 
991 	if (changed & BSS_CHANGED_MCAST_RATE)
992 		link->mt76.mcast_rates_idx =
993 			mt7996_get_rates_table(phy, info, false, true);
994 
995 	if (changed & BSS_CHANGED_BASIC_RATES)
996 		link->mt76.basic_rates_idx =
997 			mt7996_get_rates_table(phy, info, false, false);
998 
999 	/* ensure that enable txcmd_mode after bss_info */
1000 	if (changed & (BSS_CHANGED_QOS | BSS_CHANGED_BEACON_ENABLED))
1001 		mt7996_mcu_set_tx(dev, vif, info);
1002 
1003 	if (changed & BSS_CHANGED_HE_OBSS_PD)
1004 		mt7996_mcu_add_obss_spr(phy, link, &info->he_obss_pd);
1005 
1006 	if (changed & BSS_CHANGED_HE_BSS_COLOR) {
1007 		if ((vif->type == NL80211_IFTYPE_AP &&
1008 		    link->mt76.omac_idx <= HW_BSSID_MAX) ||
1009 		   vif->type == NL80211_IFTYPE_STATION)
1010 			mt7996_mcu_update_bss_color(dev, &link->mt76,
1011 						    &info->he_bss_color);
1012 	}
1013 
1014 	if (changed & (BSS_CHANGED_BEACON |
1015 		       BSS_CHANGED_BEACON_ENABLED)) {
1016 		link->mt76.beacon_rates_idx =
1017 			mt7996_get_rates_table(phy, info, true, false);
1018 
1019 		mt7996_mcu_add_beacon(hw, vif, info, info->enable_beacon);
1020 	}
1021 
1022 	if (changed & (BSS_CHANGED_UNSOL_BCAST_PROBE_RESP |
1023 		       BSS_CHANGED_FILS_DISCOVERY))
1024 		mt7996_mcu_beacon_inband_discov(dev, info, link, changed);
1025 
1026 	if (changed & BSS_CHANGED_MU_GROUPS)
1027 		mt7996_update_mu_group(hw, link, info);
1028 
1029 	if (changed & BSS_CHANGED_TXPOWER &&
1030 	    info->txpower != phy->txpower) {
1031 		phy->txpower = info->txpower;
1032 		mt7996_mcu_set_txpower_sku(phy);
1033 	}
1034 
1035 out:
1036 	mutex_unlock(&dev->mt76.mutex);
1037 }
1038 
1039 static void
1040 mt7996_channel_switch_beacon(struct ieee80211_hw *hw,
1041 			     struct ieee80211_vif *vif,
1042 			     struct cfg80211_chan_def *chandef)
1043 {
1044 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
1045 	struct mt7996_phy *phy = mt7996_band_phy(dev, chandef->chan->band);
1046 	struct ieee80211_bss_conf *link_conf;
1047 	unsigned int link_id;
1048 
1049 	mutex_lock(&dev->mt76.mutex);
1050 
1051 	for_each_vif_active_link(vif, link_conf, link_id) {
1052 		struct mt7996_vif_link *link;
1053 		struct mt7996_phy *link_phy;
1054 
1055 		link = mt7996_vif_link(dev, vif, link_id);
1056 		if (!link)
1057 			continue;
1058 
1059 		link_phy = mt7996_vif_link_phy(link);
1060 		if (link_phy != phy)
1061 			continue;
1062 
1063 		/* Reset beacon when channel switch triggered during CAC to let
1064 		 * FW correctly perform CSA countdown
1065 		 */
1066 		if (!cfg80211_reg_can_beacon(hw->wiphy, &phy->mt76->chandef,
1067 					     vif->type))
1068 			mt7996_mcu_add_beacon(hw, vif, link_conf, false);
1069 
1070 		mt7996_mcu_add_beacon(hw, vif, link_conf, true);
1071 		break;
1072 	}
1073 
1074 	mutex_unlock(&dev->mt76.mutex);
1075 }
1076 
1077 static int
1078 mt7996_post_channel_switch(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
1079 			   struct ieee80211_bss_conf *link_conf)
1080 {
1081 	struct cfg80211_chan_def *chandef = &link_conf->chanreq.oper;
1082 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
1083 	struct mt7996_phy *phy = mt7996_band_phy(dev, chandef->chan->band);
1084 	struct mt7996_vif_link *link;
1085 	int ret = -EINVAL;
1086 
1087 	mutex_lock(&dev->mt76.mutex);
1088 
1089 	link = mt7996_vif_conf_link(dev, vif, link_conf);
1090 	if (!link)
1091 		goto out;
1092 
1093 	ret = mt7996_mcu_update_bss_rfch(phy, link);
1094 	if (ret)
1095 		goto out;
1096 
1097 	ieee80211_iterate_stations_mtx(hw, mt7996_mcu_update_sta_rec_bw, link);
1098 
1099 	ret = mt7996_mcu_rdd_resume_tx(phy);
1100 
1101 out:
1102 	mutex_unlock(&dev->mt76.mutex);
1103 
1104 	return ret;
1105 }
1106 
1107 static void
1108 mt7996_sta_init_txq_wcid(struct ieee80211_sta *sta, int idx)
1109 {
1110 	int i;
1111 
1112 	for (i = 0; i < ARRAY_SIZE(sta->txq); i++) {
1113 		struct mt76_txq *mtxq;
1114 
1115 		if (!sta->txq[i])
1116 			continue;
1117 
1118 		mtxq = (struct mt76_txq *)sta->txq[i]->drv_priv;
1119 		mtxq->wcid = idx;
1120 	}
1121 }
1122 
1123 static int
1124 mt7996_mac_sta_init_link(struct mt7996_dev *dev,
1125 			 struct ieee80211_bss_conf *link_conf,
1126 			 struct ieee80211_link_sta *link_sta,
1127 			 struct mt7996_vif_link *link, unsigned int link_id)
1128 {
1129 	struct ieee80211_sta *sta = link_sta->sta;
1130 	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
1131 	struct mt7996_phy *phy = mt7996_vif_link_phy(link);
1132 	struct mt7996_sta_link *msta_link;
1133 	int idx;
1134 
1135 	if (!phy)
1136 		return -EINVAL;
1137 
1138 	idx = mt76_wcid_alloc(dev->mt76.wcid_mask, MT7996_WTBL_STA);
1139 	if (idx < 0)
1140 		return -ENOSPC;
1141 
1142 	if (msta->deflink_id == IEEE80211_LINK_UNSPECIFIED) {
1143 		msta_link = &msta->deflink;
1144 		msta->deflink_id = link_id;
1145 		msta->seclink_id = msta->deflink_id;
1146 		mt7996_sta_init_txq_wcid(sta, idx);
1147 	} else {
1148 		msta_link = kzalloc_obj(*msta_link);
1149 		if (!msta_link)
1150 			return -ENOMEM;
1151 
1152 		if (msta->seclink_id == msta->deflink_id &&
1153 		    (sta->valid_links & ~BIT(msta->deflink_id)))
1154 			msta->seclink_id = __ffs(sta->valid_links &
1155 						 ~BIT(msta->deflink_id));
1156 	}
1157 
1158 	INIT_LIST_HEAD(&msta_link->rc_list);
1159 	INIT_LIST_HEAD(&msta_link->wcid.poll_list);
1160 	msta_link->sta = msta;
1161 	msta_link->wcid.sta = 1;
1162 	msta_link->wcid.idx = idx;
1163 	msta_link->wcid.link_id = link_id;
1164 	msta_link->wcid.link_valid = !!sta->valid_links;
1165 	msta_link->wcid.def_wcid = &msta->deflink.wcid;
1166 
1167 	ewma_avg_signal_init(&msta_link->avg_ack_signal);
1168 	ewma_signal_init(&msta_link->wcid.rssi);
1169 
1170 	rcu_assign_pointer(msta->link[link_id], msta_link);
1171 
1172 	mt7996_mac_wtbl_update(dev, idx, MT_WTBL_UPDATE_ADM_COUNT_CLEAR);
1173 	mt7996_mcu_add_sta(dev, link_conf, link_sta, link, msta_link,
1174 			   CONN_STATE_DISCONNECT, true);
1175 
1176 	rcu_assign_pointer(dev->mt76.wcid[idx], &msta_link->wcid);
1177 	mt76_wcid_init(&msta_link->wcid, phy->mt76->band_idx);
1178 
1179 	return 0;
1180 }
1181 
1182 void mt7996_mac_sta_remove_link(struct mt7996_dev *dev,
1183 				struct ieee80211_sta *sta,
1184 				unsigned int link_id, bool flush)
1185 {
1186 	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
1187 	struct mt7996_sta_link *msta_link;
1188 
1189 	msta_link = mt76_dereference(msta->link[link_id], &dev->mt76);
1190 	if (!msta_link)
1191 		return;
1192 
1193 	spin_lock_bh(&dev->mt76.sta_poll_lock);
1194 	if (!list_empty(&msta_link->wcid.poll_list))
1195 		list_del_init(&msta_link->wcid.poll_list);
1196 	if (!list_empty(&msta_link->rc_list))
1197 		list_del_init(&msta_link->rc_list);
1198 	spin_unlock_bh(&dev->mt76.sta_poll_lock);
1199 
1200 	mt76_wcid_cleanup(&dev->mt76, &msta_link->wcid);
1201 
1202 	if (msta_link->wcid.link_valid) {
1203 		struct mt7996_phy *phy;
1204 
1205 		mt7996_mac_wtbl_update(dev, msta_link->wcid.idx,
1206 				       MT_WTBL_UPDATE_ADM_COUNT_CLEAR);
1207 
1208 		phy = __mt7996_phy(dev, msta_link->wcid.phy_idx);
1209 		if (phy)
1210 			phy->mt76->num_sta--;
1211 
1212 		if (msta->deflink_id == link_id) {
1213 			msta->deflink_id = IEEE80211_LINK_UNSPECIFIED;
1214 			if (msta->seclink_id == link_id) {
1215 				/* no secondary link available */
1216 				msta->seclink_id = msta->deflink_id;
1217 			} else {
1218 				struct mt7996_sta_link *msta_seclink;
1219 
1220 				/* switch to the secondary link */
1221 				msta_seclink = mt76_dereference(
1222 						msta->link[msta->seclink_id],
1223 						&dev->mt76);
1224 				if (msta_seclink) {
1225 					msta->deflink_id = msta->seclink_id;
1226 					mt7996_sta_init_txq_wcid(sta,
1227 						msta_seclink->wcid.idx);
1228 				}
1229 			}
1230 		} else if (msta->seclink_id == link_id) {
1231 			msta->seclink_id = msta->deflink_id;
1232 		}
1233 		msta_link->wcid.link_valid = false;
1234 	}
1235 
1236 	if (flush) {
1237 		rcu_assign_pointer(msta->link[link_id], NULL);
1238 		rcu_assign_pointer(dev->mt76.wcid[msta_link->wcid.idx], NULL);
1239 		mt76_wcid_mask_clear(dev->mt76.wcid_mask, msta_link->wcid.idx);
1240 		if (msta_link != &msta->deflink)
1241 			kfree_rcu(msta_link, rcu_head);
1242 	}
1243 }
1244 
1245 static void
1246 mt7996_mac_sta_remove_links(struct mt7996_dev *dev, struct ieee80211_vif *vif,
1247 			    struct ieee80211_sta *sta, unsigned long links,
1248 			    bool flush)
1249 {
1250 	unsigned int link_id;
1251 
1252 	for_each_set_bit(link_id, &links, IEEE80211_MLD_MAX_NUM_LINKS)
1253 		mt7996_mac_sta_remove_link(dev, sta, link_id, flush);
1254 }
1255 
1256 static int
1257 mt7996_mac_sta_add_links(struct mt7996_dev *dev, struct ieee80211_vif *vif,
1258 			 struct ieee80211_sta *sta, unsigned long new_links)
1259 {
1260 	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
1261 	unsigned int link_id;
1262 	int err = 0;
1263 
1264 	for_each_set_bit(link_id, &new_links, IEEE80211_MLD_MAX_NUM_LINKS) {
1265 		struct ieee80211_bss_conf *link_conf;
1266 		struct ieee80211_link_sta *link_sta;
1267 		struct mt7996_sta_link *msta_link;
1268 		struct mt7996_vif_link *link;
1269 		struct mt76_phy *mphy;
1270 
1271 		msta_link = mt76_dereference(msta->link[link_id], &dev->mt76);
1272 		if (msta_link) {
1273 			msta_link->wcid.link_valid = true;
1274 			continue;
1275 		}
1276 
1277 		link_conf = link_conf_dereference_protected(vif, link_id);
1278 		if (!link_conf) {
1279 			err = -EINVAL;
1280 			goto error_unlink;
1281 		}
1282 
1283 		link = mt7996_vif_link(dev, vif, link_id);
1284 		if (!link) {
1285 			err = -EINVAL;
1286 			goto error_unlink;
1287 		}
1288 
1289 		link_sta = link_sta_dereference_protected(sta, link_id);
1290 		if (!link_sta) {
1291 			err = -EINVAL;
1292 			goto error_unlink;
1293 		}
1294 
1295 		mphy = mt76_vif_link_phy(&link->mt76);
1296 		if (!mphy) {
1297 			err = -EINVAL;
1298 			goto error_unlink;
1299 		}
1300 
1301 		err = mt7996_mac_sta_init_link(dev, link_conf, link_sta, link,
1302 					       link_id);
1303 		if (err)
1304 			goto error_unlink;
1305 
1306 		mphy->num_sta++;
1307 	}
1308 
1309 	return 0;
1310 
1311 error_unlink:
1312 	mt7996_mac_sta_remove_links(dev, vif, sta, new_links, true);
1313 
1314 	return err;
1315 }
1316 
1317 static int
1318 mt7996_mac_sta_change_links(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
1319 			    struct ieee80211_sta *sta, u16 old_links,
1320 			    u16 new_links)
1321 {
1322 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
1323 	unsigned long add = new_links & ~old_links;
1324 	unsigned long rem = old_links & ~new_links;
1325 	int ret;
1326 
1327 	mutex_lock(&dev->mt76.mutex);
1328 
1329 	mt7996_mac_sta_remove_links(dev, vif, sta, rem, false);
1330 	ret = mt7996_mac_sta_add_links(dev, vif, sta, add);
1331 
1332 	mutex_unlock(&dev->mt76.mutex);
1333 
1334 	return ret;
1335 }
1336 
1337 static int
1338 mt7996_mac_sta_add(struct mt7996_dev *dev, struct ieee80211_vif *vif,
1339 		   struct ieee80211_sta *sta)
1340 {
1341 	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
1342 	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
1343 	unsigned long links = sta->valid_links ? sta->valid_links : BIT(0);
1344 	int err;
1345 
1346 	mutex_lock(&dev->mt76.mutex);
1347 
1348 	msta->deflink_id = IEEE80211_LINK_UNSPECIFIED;
1349 	msta->seclink_id = IEEE80211_LINK_UNSPECIFIED;
1350 	msta->vif = mvif;
1351 	err = mt7996_mac_sta_add_links(dev, vif, sta, links);
1352 
1353 	mutex_unlock(&dev->mt76.mutex);
1354 
1355 	return err;
1356 }
1357 
1358 static int
1359 mt7996_mac_sta_event(struct mt7996_dev *dev, struct ieee80211_vif *vif,
1360 		     struct ieee80211_sta *sta, enum mt76_sta_event ev)
1361 {
1362 	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
1363 	unsigned long links = sta->valid_links;
1364 	struct ieee80211_link_sta *link_sta;
1365 	unsigned int link_id;
1366 	int err = 0;
1367 
1368 	mutex_lock(&dev->mt76.mutex);
1369 
1370 	for_each_sta_active_link(vif, sta, link_sta, link_id) {
1371 		struct ieee80211_bss_conf *link_conf;
1372 		struct mt7996_sta_link *msta_link;
1373 		struct mt7996_vif_link *link;
1374 		int i;
1375 
1376 		link_conf = link_conf_dereference_protected(vif, link_id);
1377 		if (!link_conf)
1378 			continue;
1379 
1380 		link = mt7996_vif_link(dev, vif, link_id);
1381 		if (!link)
1382 			continue;
1383 
1384 		msta_link = mt76_dereference(msta->link[link_id], &dev->mt76);
1385 		if (!msta_link)
1386 			continue;
1387 
1388 		switch (ev) {
1389 		case MT76_STA_EVENT_ASSOC:
1390 			err = mt7996_mcu_add_sta(dev, link_conf, link_sta,
1391 						 link, msta_link,
1392 						 CONN_STATE_CONNECT, true);
1393 			if (err)
1394 				goto unlock;
1395 
1396 			err = mt7996_mcu_add_rate_ctrl(dev, msta_link->sta, vif,
1397 						       link_id, false);
1398 			if (err)
1399 				goto unlock;
1400 
1401 			msta_link->wcid.tx_info |= MT_WCID_TX_INFO_SET;
1402 			break;
1403 		case MT76_STA_EVENT_AUTHORIZE:
1404 			err = mt7996_mcu_add_sta(dev, link_conf, link_sta,
1405 						 link, msta_link,
1406 						 CONN_STATE_PORT_SECURE, false);
1407 			if (err)
1408 				goto unlock;
1409 			break;
1410 		case MT76_STA_EVENT_DISASSOC:
1411 			for (i = 0; i < ARRAY_SIZE(msta_link->twt.flow); i++)
1412 				mt7996_mac_twt_teardown_flow(dev, link,
1413 							     msta_link, i);
1414 
1415 			if (!sta->mlo)
1416 				mt7996_mcu_add_sta(dev, link_conf, link_sta,
1417 						   link, msta_link,
1418 						   CONN_STATE_DISCONNECT, false);
1419 			else if (sta->mlo && links == BIT(link_id)) /* last link */
1420 				mt7996_mcu_teardown_mld_sta(dev, link,
1421 							    msta_link);
1422 			msta_link->wcid.sta_disabled = 1;
1423 			msta_link->wcid.sta = 0;
1424 			links = links & ~BIT(link_id);
1425 			break;
1426 		}
1427 	}
1428 unlock:
1429 	mutex_unlock(&dev->mt76.mutex);
1430 
1431 	return err;
1432 }
1433 
1434 static void
1435 mt7996_mac_sta_remove(struct mt7996_dev *dev, struct ieee80211_vif *vif,
1436 		      struct ieee80211_sta *sta)
1437 {
1438 	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
1439 	int i;
1440 
1441 	mutex_lock(&dev->mt76.mutex);
1442 	for (i = 0; i < ARRAY_SIZE(msta->link); i++)
1443 		mt7996_mac_sta_remove_link(dev, sta, i, true);
1444 	mutex_unlock(&dev->mt76.mutex);
1445 }
1446 
1447 static void
1448 mt7996_set_active_links(struct ieee80211_vif *vif)
1449 {
1450 	u16 active_links;
1451 
1452 	if (vif->type != NL80211_IFTYPE_STATION)
1453 		return;
1454 
1455 	if (!ieee80211_vif_is_mld(vif))
1456 		return;
1457 
1458 	active_links = mt76_select_links(vif, MT7996_MAX_RADIOS);
1459 	if (hweight16(active_links) < 2)
1460 		return;
1461 
1462 	ieee80211_set_active_links_async(vif, active_links);
1463 }
1464 
1465 static int
1466 mt7996_sta_state(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
1467 		 struct ieee80211_sta *sta, enum ieee80211_sta_state old_state,
1468 		 enum ieee80211_sta_state new_state)
1469 {
1470 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
1471 	enum mt76_sta_event ev;
1472 
1473 	if (old_state == IEEE80211_STA_NOTEXIST &&
1474 	    new_state == IEEE80211_STA_NONE)
1475 		return mt7996_mac_sta_add(dev, vif, sta);
1476 
1477 	if (old_state == IEEE80211_STA_NONE &&
1478 	    new_state == IEEE80211_STA_NOTEXIST)
1479 		mt7996_mac_sta_remove(dev, vif, sta);
1480 
1481 	if (old_state == IEEE80211_STA_AUTH &&
1482 	    new_state == IEEE80211_STA_ASSOC) {
1483 		mt7996_set_active_links(vif);
1484 		ev = MT76_STA_EVENT_ASSOC;
1485 	} else if (old_state == IEEE80211_STA_ASSOC &&
1486 		   new_state == IEEE80211_STA_AUTHORIZED) {
1487 		ev = MT76_STA_EVENT_AUTHORIZE;
1488 	} else if (old_state == IEEE80211_STA_ASSOC &&
1489 		   new_state == IEEE80211_STA_AUTH) {
1490 		ev = MT76_STA_EVENT_DISASSOC;
1491 	} else {
1492 		return 0;
1493 	}
1494 
1495 	return mt7996_mac_sta_event(dev, vif, sta, ev);
1496 }
1497 
1498 static void mt7996_tx(struct ieee80211_hw *hw,
1499 		      struct ieee80211_tx_control *control,
1500 		      struct sk_buff *skb)
1501 {
1502 	struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
1503 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
1504 	struct ieee80211_sta *sta = control->sta;
1505 	struct mt7996_sta *msta = sta ? (void *)sta->drv_priv : NULL;
1506 	struct mt76_phy *mphy = hw->priv;
1507 	struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
1508 	struct ieee80211_vif *vif = info->control.vif;
1509 	struct mt7996_vif *mvif = vif ? (void *)vif->drv_priv : NULL;
1510 	struct mt76_wcid *wcid = &dev->mt76.global_wcid;
1511 	u8 link_id = u32_get_bits(info->control.flags,
1512 				  IEEE80211_TX_CTRL_MLO_LINK);
1513 
1514 	rcu_read_lock();
1515 
1516 	/* Use primary link_id if the value from mac80211 is set to
1517 	 * IEEE80211_LINK_UNSPECIFIED.
1518 	 */
1519 	if (link_id == IEEE80211_LINK_UNSPECIFIED) {
1520 		if (msta)
1521 			link_id = msta->deflink_id;
1522 		else if (mvif)
1523 			link_id = mvif->mt76.deflink_id;
1524 	}
1525 
1526 	if (vif && ieee80211_vif_is_mld(vif)) {
1527 		struct ieee80211_bss_conf *link_conf;
1528 
1529 		if (msta) {
1530 			struct ieee80211_link_sta *link_sta;
1531 
1532 			link_sta = rcu_dereference(sta->link[link_id]);
1533 			if (!link_sta)
1534 				link_sta = rcu_dereference(sta->link[msta->deflink_id]);
1535 
1536 			if (link_sta) {
1537 				memcpy(hdr->addr1, link_sta->addr, ETH_ALEN);
1538 				if (ether_addr_equal(sta->addr, hdr->addr3))
1539 					memcpy(hdr->addr3, link_sta->addr, ETH_ALEN);
1540 			}
1541 		}
1542 
1543 		link_conf = rcu_dereference(vif->link_conf[link_id]);
1544 		if (link_conf) {
1545 			memcpy(hdr->addr2, link_conf->addr, ETH_ALEN);
1546 			if (ether_addr_equal(vif->addr, hdr->addr3))
1547 				memcpy(hdr->addr3, link_conf->addr, ETH_ALEN);
1548 		}
1549 	}
1550 
1551 	if (mvif) {
1552 		struct mt76_vif_link *mlink;
1553 
1554 		mlink = rcu_dereference(mvif->mt76.link[link_id]);
1555 		if (mlink && mlink->wcid)
1556 			wcid = mlink->wcid;
1557 
1558 		if (mvif->mt76.roc_phy &&
1559 		    (info->flags & IEEE80211_TX_CTL_TX_OFFCHAN)) {
1560 			mphy = mvif->mt76.roc_phy;
1561 			if (mphy->roc_link)
1562 				wcid = mphy->roc_link->wcid;
1563 		} else if (mlink) {
1564 			mphy = mt76_vif_link_phy(mlink);
1565 		}
1566 	}
1567 
1568 	if (!mphy) {
1569 		ieee80211_free_txskb(hw, skb);
1570 		goto unlock;
1571 	}
1572 
1573 	if (msta) {
1574 		struct mt7996_sta_link *msta_link;
1575 
1576 		msta_link = rcu_dereference(msta->link[link_id]);
1577 		if (msta_link)
1578 			wcid = &msta_link->wcid;
1579 	}
1580 	mt76_tx(mphy, control->sta, wcid, skb);
1581 unlock:
1582 	rcu_read_unlock();
1583 }
1584 
1585 static int mt7996_set_rts_threshold(struct ieee80211_hw *hw, int radio_idx,
1586 				    u32 val)
1587 {
1588 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
1589 	int i, ret = 0;
1590 
1591 	mutex_lock(&dev->mt76.mutex);
1592 
1593 	for (i = 0; i < hw->wiphy->n_radio; i++) {
1594 		struct mt7996_phy *phy = dev->radio_phy[i];
1595 
1596 		ret = mt7996_mcu_set_rts_thresh(phy, val);
1597 		if (ret)
1598 			break;
1599 	}
1600 
1601 	mutex_unlock(&dev->mt76.mutex);
1602 
1603 	return ret;
1604 }
1605 
1606 static int
1607 mt7996_ampdu_action(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
1608 		    struct ieee80211_ampdu_params *params)
1609 {
1610 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
1611 	struct ieee80211_sta *sta = params->sta;
1612 	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
1613 	struct ieee80211_txq *txq = sta->txq[params->tid];
1614 	u16 tid = params->tid;
1615 	u16 ssn = params->ssn;
1616 	struct mt76_txq *mtxq;
1617 	int ret = 0;
1618 
1619 	if (!txq)
1620 		return -EINVAL;
1621 
1622 	mtxq = (struct mt76_txq *)txq->drv_priv;
1623 
1624 	mutex_lock(&dev->mt76.mutex);
1625 
1626 	switch (params->action) {
1627 	case IEEE80211_AMPDU_RX_START:
1628 		/* Since packets belonging to the same TID can be split over
1629 		 * multiple links, store the AMPDU state for reordering in the
1630 		 * primary link
1631 		 */
1632 		mt76_rx_aggr_start(&dev->mt76, &msta->deflink.wcid, tid,
1633 				   ssn, params->buf_size);
1634 		ret = mt7996_mcu_add_rx_ba(dev, params, vif, true);
1635 		break;
1636 	case IEEE80211_AMPDU_RX_STOP:
1637 		mt76_rx_aggr_stop(&dev->mt76, &msta->deflink.wcid, tid);
1638 		ret = mt7996_mcu_add_rx_ba(dev, params, vif, false);
1639 		break;
1640 	case IEEE80211_AMPDU_TX_OPERATIONAL:
1641 		mtxq->aggr = true;
1642 		mtxq->send_bar = false;
1643 		ret = mt7996_mcu_add_tx_ba(dev, params, vif, true);
1644 		break;
1645 	case IEEE80211_AMPDU_TX_STOP_FLUSH:
1646 	case IEEE80211_AMPDU_TX_STOP_FLUSH_CONT:
1647 		mtxq->aggr = false;
1648 		clear_bit(tid, &msta->deflink.wcid.ampdu_state);
1649 		ret = mt7996_mcu_add_tx_ba(dev, params, vif, false);
1650 		break;
1651 	case IEEE80211_AMPDU_TX_START:
1652 		set_bit(tid, &msta->deflink.wcid.ampdu_state);
1653 		ret = IEEE80211_AMPDU_TX_START_IMMEDIATE;
1654 		break;
1655 	case IEEE80211_AMPDU_TX_STOP_CONT:
1656 		mtxq->aggr = false;
1657 		clear_bit(tid, &msta->deflink.wcid.ampdu_state);
1658 		ret = mt7996_mcu_add_tx_ba(dev, params, vif, false);
1659 		ieee80211_stop_tx_ba_cb_irqsafe(vif, sta->addr, tid);
1660 		break;
1661 	}
1662 
1663 	mutex_unlock(&dev->mt76.mutex);
1664 
1665 	return ret;
1666 }
1667 
1668 static int
1669 mt7996_get_stats(struct ieee80211_hw *hw,
1670 		 struct ieee80211_low_level_stats *stats)
1671 {
1672 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
1673 	int i;
1674 
1675 	mutex_lock(&dev->mt76.mutex);
1676 
1677 	memset(stats, 0, sizeof(*stats));
1678 	for (i = 0; i < hw->wiphy->n_radio; i++) {
1679 		struct mt7996_phy *phy = dev->radio_phy[i];
1680 		struct mt76_mib_stats *mib = &phy->mib;
1681 
1682 		stats->dot11RTSSuccessCount += mib->rts_cnt;
1683 		stats->dot11RTSFailureCount += mib->rts_retries_cnt;
1684 		stats->dot11FCSErrorCount += mib->fcs_err_cnt;
1685 		stats->dot11ACKFailureCount += mib->ack_fail_cnt;
1686 	}
1687 
1688 	mutex_unlock(&dev->mt76.mutex);
1689 
1690 	return 0;
1691 }
1692 
1693 u64 __mt7996_get_tsf(struct ieee80211_hw *hw, struct mt7996_vif_link *link)
1694 {
1695 	struct mt7996_phy *phy = mt7996_vif_link_phy(link);
1696 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
1697 	union {
1698 		u64 t64;
1699 		u32 t32[2];
1700 	} tsf;
1701 	u16 n;
1702 
1703 	if (!phy)
1704 		return 0;
1705 
1706 	lockdep_assert_held(&dev->mt76.mutex);
1707 
1708 	n = link->mt76.omac_idx > HW_BSSID_MAX ? HW_BSSID_0
1709 					       : link->mt76.omac_idx;
1710 	/* TSF software read */
1711 	mt76_rmw(dev, MT_LPON_TCR(phy->mt76->band_idx, n), MT_LPON_TCR_SW_MODE,
1712 		 MT_LPON_TCR_SW_READ);
1713 	tsf.t32[0] = mt76_rr(dev, MT_LPON_UTTR0(phy->mt76->band_idx));
1714 	tsf.t32[1] = mt76_rr(dev, MT_LPON_UTTR1(phy->mt76->band_idx));
1715 
1716 	return tsf.t64;
1717 }
1718 
1719 static u64
1720 mt7996_get_tsf(struct ieee80211_hw *hw, struct ieee80211_vif *vif)
1721 {
1722 	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
1723 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
1724 	u64 ret;
1725 
1726 	mutex_lock(&dev->mt76.mutex);
1727 	ret = __mt7996_get_tsf(hw, &mvif->deflink);
1728 	mutex_unlock(&dev->mt76.mutex);
1729 
1730 	return ret;
1731 }
1732 
1733 static void
1734 mt7996_set_tsf(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
1735 	       u64 timestamp)
1736 {
1737 	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
1738 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
1739 	struct mt7996_vif_link *link;
1740 	struct mt7996_phy *phy;
1741 	union {
1742 		u64 t64;
1743 		u32 t32[2];
1744 	} tsf = { .t64 = timestamp, };
1745 	u16 n;
1746 
1747 	mutex_lock(&dev->mt76.mutex);
1748 
1749 	link = mt7996_vif_link(dev, vif, mvif->mt76.deflink_id);
1750 	if (!link)
1751 		goto unlock;
1752 
1753 	n = link->mt76.omac_idx > HW_BSSID_MAX ? HW_BSSID_0
1754 					       : link->mt76.omac_idx;
1755 	phy = mt7996_vif_link_phy(link);
1756 	if (!phy)
1757 		goto unlock;
1758 
1759 	mt76_wr(dev, MT_LPON_UTTR0(phy->mt76->band_idx), tsf.t32[0]);
1760 	mt76_wr(dev, MT_LPON_UTTR1(phy->mt76->band_idx), tsf.t32[1]);
1761 	/* TSF software overwrite */
1762 	mt76_rmw(dev, MT_LPON_TCR(phy->mt76->band_idx, n), MT_LPON_TCR_SW_MODE,
1763 		 MT_LPON_TCR_SW_WRITE);
1764 
1765 unlock:
1766 	mutex_unlock(&dev->mt76.mutex);
1767 }
1768 
1769 static void
1770 mt7996_offset_tsf(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
1771 		  s64 timestamp)
1772 {
1773 	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
1774 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
1775 	struct mt7996_vif_link *link;
1776 	struct mt7996_phy *phy;
1777 	union {
1778 		u64 t64;
1779 		u32 t32[2];
1780 	} tsf = { .t64 = timestamp, };
1781 	u16 n;
1782 
1783 	mutex_lock(&dev->mt76.mutex);
1784 
1785 	link = mt7996_vif_link(dev, vif, mvif->mt76.deflink_id);
1786 	if (!link)
1787 		goto unlock;
1788 
1789 	phy = mt7996_vif_link_phy(link);
1790 	if (!phy)
1791 		goto unlock;
1792 
1793 	n = link->mt76.omac_idx > HW_BSSID_MAX ? HW_BSSID_0
1794 					       : link->mt76.omac_idx;
1795 	mt76_wr(dev, MT_LPON_UTTR0(phy->mt76->band_idx), tsf.t32[0]);
1796 	mt76_wr(dev, MT_LPON_UTTR1(phy->mt76->band_idx), tsf.t32[1]);
1797 	/* TSF software adjust*/
1798 	mt76_rmw(dev, MT_LPON_TCR(phy->mt76->band_idx, n), MT_LPON_TCR_SW_MODE,
1799 		 MT_LPON_TCR_SW_ADJUST);
1800 
1801 unlock:
1802 	mutex_unlock(&dev->mt76.mutex);
1803 }
1804 
1805 static void
1806 mt7996_set_coverage_class(struct ieee80211_hw *hw, int radio_idx,
1807 			  s16 coverage_class)
1808 {
1809 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
1810 	struct mt7996_phy *phy;
1811 
1812 	mutex_lock(&dev->mt76.mutex);
1813 	mt7996_for_each_phy(dev, phy) {
1814 		phy->coverage_class = max_t(s16, coverage_class, 0);
1815 		mt7996_mac_set_coverage_class(phy);
1816 	}
1817 	mutex_unlock(&dev->mt76.mutex);
1818 }
1819 
1820 static int
1821 mt7996_set_antenna(struct ieee80211_hw *hw, int radio_idx,
1822 		   u32 tx_ant, u32 rx_ant)
1823 {
1824 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
1825 	int i;
1826 
1827 	if (tx_ant != rx_ant)
1828 		return -EINVAL;
1829 
1830 	for (i = 0; i < hw->wiphy->n_radio; i++) {
1831 		struct mt7996_phy *phy = dev->radio_phy[i];
1832 
1833 		if (!(tx_ant & phy->orig_chainmask))
1834 			return -EINVAL;
1835 	}
1836 
1837 	mutex_lock(&dev->mt76.mutex);
1838 
1839 	for (i = 0; i < hw->wiphy->n_radio; i++) {
1840 		struct mt7996_phy *phy = dev->radio_phy[i];
1841 		u8 band_idx = phy->mt76->band_idx;
1842 		u8 shift = dev->chainshift[band_idx];
1843 
1844 		phy->mt76->chainmask = tx_ant & phy->orig_chainmask;
1845 		phy->mt76->antenna_mask = (phy->mt76->chainmask >> shift) &
1846 					  phy->orig_antenna_mask;
1847 
1848 		mt76_set_stream_caps(phy->mt76, true);
1849 		mt7996_set_stream_vht_txbf_caps(phy);
1850 		mt7996_set_stream_he_eht_caps(phy);
1851 		mt7996_mcu_set_txpower_sku(phy);
1852 	}
1853 
1854 	mutex_unlock(&dev->mt76.mutex);
1855 
1856 	return 0;
1857 }
1858 
1859 static void mt7996_sta_statistics(struct ieee80211_hw *hw,
1860 				  struct ieee80211_vif *vif,
1861 				  struct ieee80211_sta *sta,
1862 				  struct station_info *sinfo)
1863 {
1864 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
1865 	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
1866 	struct mt7996_sta_link *msta_link = &msta->deflink;
1867 	struct rate_info *txrate = &msta_link->wcid.rate;
1868 
1869 	if (txrate->legacy || txrate->flags) {
1870 		if (txrate->legacy) {
1871 			sinfo->txrate.legacy = txrate->legacy;
1872 		} else {
1873 			sinfo->txrate.mcs = txrate->mcs;
1874 			sinfo->txrate.nss = txrate->nss;
1875 			sinfo->txrate.bw = txrate->bw;
1876 			sinfo->txrate.he_gi = txrate->he_gi;
1877 			sinfo->txrate.he_dcm = txrate->he_dcm;
1878 			sinfo->txrate.he_ru_alloc = txrate->he_ru_alloc;
1879 			sinfo->txrate.eht_gi = txrate->eht_gi;
1880 		}
1881 		sinfo->txrate.flags = txrate->flags;
1882 		sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BITRATE);
1883 	}
1884 	sinfo->txrate.flags = txrate->flags;
1885 	sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BITRATE);
1886 
1887 	sinfo->tx_failed = msta_link->wcid.stats.tx_failed;
1888 	sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_FAILED);
1889 
1890 	sinfo->tx_retries = msta_link->wcid.stats.tx_retries;
1891 	sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_RETRIES);
1892 
1893 	sinfo->ack_signal = (s8)msta_link->ack_signal;
1894 	sinfo->filled |= BIT_ULL(NL80211_STA_INFO_ACK_SIGNAL);
1895 
1896 	sinfo->avg_ack_signal =
1897 		-(s8)ewma_avg_signal_read(&msta_link->avg_ack_signal);
1898 	sinfo->filled |= BIT_ULL(NL80211_STA_INFO_ACK_SIGNAL_AVG);
1899 
1900 	if (mtk_wed_device_active(&dev->mt76.mmio.wed)) {
1901 		sinfo->tx_bytes = msta_link->wcid.stats.tx_bytes;
1902 		sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BYTES64);
1903 
1904 		sinfo->rx_bytes = msta_link->wcid.stats.rx_bytes;
1905 		sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_BYTES64);
1906 
1907 		sinfo->tx_packets = msta_link->wcid.stats.tx_packets;
1908 		sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_PACKETS);
1909 
1910 		sinfo->rx_packets = msta_link->wcid.stats.rx_packets;
1911 		sinfo->filled |= BIT_ULL(NL80211_STA_INFO_RX_PACKETS);
1912 	}
1913 }
1914 
1915 static void mt7996_link_rate_ctrl_update(void *data,
1916 					 struct mt7996_sta_link *msta_link)
1917 {
1918 	struct mt7996_sta *msta = msta_link->sta;
1919 	struct mt7996_phy *phy = mt7996_vif_link_phy(&msta->vif->deflink);
1920 	struct mt7996_dev *dev;
1921 	u32 *changed = data;
1922 
1923 	if (!phy)
1924 		return;
1925 
1926 	dev = phy->dev;
1927 	spin_lock_bh(&dev->mt76.sta_poll_lock);
1928 
1929 	msta_link->changed |= *changed;
1930 	if (list_empty(&msta_link->rc_list))
1931 		list_add_tail(&msta_link->rc_list, &dev->sta_rc_list);
1932 
1933 	spin_unlock_bh(&dev->mt76.sta_poll_lock);
1934 }
1935 
1936 static void mt7996_link_sta_rc_update(struct ieee80211_hw *hw,
1937 				      struct ieee80211_vif *vif,
1938 				      struct ieee80211_link_sta *link_sta,
1939 				      u32 changed)
1940 {
1941 	struct ieee80211_sta *sta = link_sta->sta;
1942 	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
1943 	struct mt7996_sta_link *msta_link;
1944 
1945 	rcu_read_lock();
1946 
1947 	msta_link = rcu_dereference(msta->link[link_sta->link_id]);
1948 	if (msta_link) {
1949 		struct mt7996_dev *dev = mt7996_hw_dev(hw);
1950 
1951 		mt7996_link_rate_ctrl_update(&changed, msta_link);
1952 		ieee80211_queue_work(hw, &dev->rc_work);
1953 	}
1954 
1955 	rcu_read_unlock();
1956 }
1957 
1958 static void mt7996_sta_rate_ctrl_update(void *data, struct ieee80211_sta *sta)
1959 {
1960 	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
1961 	struct mt7996_sta_link *msta_link;
1962 	u32 *changed = data;
1963 
1964 	msta_link = rcu_dereference(msta->link[msta->deflink_id]);
1965 	if (msta_link)
1966 		mt7996_link_rate_ctrl_update(&changed, msta_link);
1967 }
1968 
1969 static int
1970 mt7996_set_bitrate_mask(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
1971 			const struct cfg80211_bitrate_mask *mask)
1972 {
1973 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
1974 	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
1975 	u32 changed = IEEE80211_RC_SUPP_RATES_CHANGED;
1976 
1977 	mvif->deflink.bitrate_mask = *mask;
1978 
1979 	/* if multiple rates across different preambles are given we can
1980 	 * reconfigure this info with all peers using sta_rec command with
1981 	 * the below exception cases.
1982 	 * - single rate : if a rate is passed along with different preambles,
1983 	 * we select the highest one as fixed rate. i.e VHT MCS for VHT peers.
1984 	 * - multiple rates: if it's not in range format i.e 0-{7,8,9} for VHT
1985 	 * then multiple MCS setting (MCS 4,5,6) is not supported.
1986 	 */
1987 	ieee80211_iterate_stations_atomic(hw, mt7996_sta_rate_ctrl_update,
1988 					  &changed);
1989 	ieee80211_queue_work(hw, &dev->rc_work);
1990 
1991 	return 0;
1992 }
1993 
1994 static void mt7996_sta_set_4addr(struct ieee80211_hw *hw,
1995 				 struct ieee80211_vif *vif,
1996 				 struct ieee80211_sta *sta,
1997 				 bool enabled)
1998 {
1999 	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
2000 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
2001 	struct ieee80211_link_sta *link_sta;
2002 	unsigned int link_id;
2003 
2004 	mutex_lock(&dev->mt76.mutex);
2005 
2006 	for_each_sta_active_link(vif, sta, link_sta, link_id) {
2007 		struct mt7996_sta_link *msta_link;
2008 		struct mt7996_vif_link *link;
2009 
2010 		link = mt7996_vif_link(dev, vif, link_id);
2011 		if (!link)
2012 			continue;
2013 
2014 		msta_link = mt76_dereference(msta->link[link_id], &dev->mt76);
2015 		if (!msta_link)
2016 			continue;
2017 
2018 		if (enabled)
2019 			set_bit(MT_WCID_FLAG_4ADDR, &msta_link->wcid.flags);
2020 		else
2021 			clear_bit(MT_WCID_FLAG_4ADDR, &msta_link->wcid.flags);
2022 
2023 		if (!msta_link->wcid.sta)
2024 			continue;
2025 
2026 		mt7996_mcu_wtbl_update_hdr_trans(dev, vif, link, msta_link);
2027 	}
2028 
2029 	mutex_unlock(&dev->mt76.mutex);
2030 }
2031 
2032 static void mt7996_sta_set_decap_offload(struct ieee80211_hw *hw,
2033 					 struct ieee80211_vif *vif,
2034 					 struct ieee80211_sta *sta,
2035 					 bool enabled)
2036 {
2037 	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
2038 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
2039 	struct ieee80211_link_sta *link_sta;
2040 	unsigned int link_id;
2041 
2042 	mutex_lock(&dev->mt76.mutex);
2043 
2044 	for_each_sta_active_link(vif, sta, link_sta, link_id) {
2045 		struct mt7996_sta_link *msta_link;
2046 		struct mt7996_vif_link *link;
2047 
2048 		link = mt7996_vif_link(dev, vif, link_id);
2049 		if (!link)
2050 			continue;
2051 
2052 		msta_link = mt76_dereference(msta->link[link_id], &dev->mt76);
2053 		if (!msta_link)
2054 			continue;
2055 
2056 		if (enabled)
2057 			set_bit(MT_WCID_FLAG_HDR_TRANS,
2058 				&msta_link->wcid.flags);
2059 		else
2060 			clear_bit(MT_WCID_FLAG_HDR_TRANS,
2061 				  &msta_link->wcid.flags);
2062 
2063 		if (!msta_link->wcid.sta)
2064 			continue;
2065 
2066 		mt7996_mcu_wtbl_update_hdr_trans(dev, vif, link, msta_link);
2067 	}
2068 
2069 	mutex_unlock(&dev->mt76.mutex);
2070 }
2071 
2072 static const char mt7996_gstrings_stats[][ETH_GSTRING_LEN] = {
2073 	"tx_ampdu_cnt",
2074 	"tx_stop_q_empty_cnt",
2075 	"tx_mpdu_attempts",
2076 	"tx_mpdu_success",
2077 	"tx_rwp_fail_cnt",
2078 	"tx_rwp_need_cnt",
2079 	"tx_pkt_ebf_cnt",
2080 	"tx_pkt_ibf_cnt",
2081 	"tx_ampdu_len:0-1",
2082 	"tx_ampdu_len:2-10",
2083 	"tx_ampdu_len:11-19",
2084 	"tx_ampdu_len:20-28",
2085 	"tx_ampdu_len:29-37",
2086 	"tx_ampdu_len:38-46",
2087 	"tx_ampdu_len:47-55",
2088 	"tx_ampdu_len:56-79",
2089 	"tx_ampdu_len:80-103",
2090 	"tx_ampdu_len:104-127",
2091 	"tx_ampdu_len:128-151",
2092 	"tx_ampdu_len:152-175",
2093 	"tx_ampdu_len:176-199",
2094 	"tx_ampdu_len:200-223",
2095 	"tx_ampdu_len:224-247",
2096 	"ba_miss_count",
2097 	"tx_beamformer_ppdu_iBF",
2098 	"tx_beamformer_ppdu_eBF",
2099 	"tx_beamformer_rx_feedback_all",
2100 	"tx_beamformer_rx_feedback_he",
2101 	"tx_beamformer_rx_feedback_vht",
2102 	"tx_beamformer_rx_feedback_ht",
2103 	"tx_beamformer_rx_feedback_bw", /* zero based idx: 20, 40, 80, 160 */
2104 	"tx_beamformer_rx_feedback_nc",
2105 	"tx_beamformer_rx_feedback_nr",
2106 	"tx_beamformee_ok_feedback_pkts",
2107 	"tx_beamformee_feedback_trig",
2108 	"tx_mu_beamforming",
2109 	"tx_mu_mpdu",
2110 	"tx_mu_successful_mpdu",
2111 	"tx_su_successful_mpdu",
2112 	"tx_msdu_pack_1",
2113 	"tx_msdu_pack_2",
2114 	"tx_msdu_pack_3",
2115 	"tx_msdu_pack_4",
2116 	"tx_msdu_pack_5",
2117 	"tx_msdu_pack_6",
2118 	"tx_msdu_pack_7",
2119 	"tx_msdu_pack_8",
2120 
2121 	/* rx counters */
2122 	"rx_fifo_full_cnt",
2123 	"rx_mpdu_cnt",
2124 	"channel_idle_cnt",
2125 	"rx_vector_mismatch_cnt",
2126 	"rx_delimiter_fail_cnt",
2127 	"rx_len_mismatch_cnt",
2128 	"rx_ampdu_cnt",
2129 	"rx_ampdu_bytes_cnt",
2130 	"rx_ampdu_valid_subframe_cnt",
2131 	"rx_ampdu_valid_subframe_b_cnt",
2132 	"rx_pfdrop_cnt",
2133 	"rx_vec_queue_overflow_drop_cnt",
2134 	"rx_ba_cnt",
2135 
2136 	/* per vif counters */
2137 	"v_tx_mode_cck",
2138 	"v_tx_mode_ofdm",
2139 	"v_tx_mode_ht",
2140 	"v_tx_mode_ht_gf",
2141 	"v_tx_mode_vht",
2142 	"v_tx_mode_he_su",
2143 	"v_tx_mode_he_ext_su",
2144 	"v_tx_mode_he_tb",
2145 	"v_tx_mode_he_mu",
2146 	"v_tx_mode_eht_su",
2147 	"v_tx_mode_eht_trig",
2148 	"v_tx_mode_eht_mu",
2149 	"v_tx_bw_20",
2150 	"v_tx_bw_40",
2151 	"v_tx_bw_80",
2152 	"v_tx_bw_160",
2153 	"v_tx_bw_320",
2154 	"v_tx_mcs_0",
2155 	"v_tx_mcs_1",
2156 	"v_tx_mcs_2",
2157 	"v_tx_mcs_3",
2158 	"v_tx_mcs_4",
2159 	"v_tx_mcs_5",
2160 	"v_tx_mcs_6",
2161 	"v_tx_mcs_7",
2162 	"v_tx_mcs_8",
2163 	"v_tx_mcs_9",
2164 	"v_tx_mcs_10",
2165 	"v_tx_mcs_11",
2166 	"v_tx_mcs_12",
2167 	"v_tx_mcs_13",
2168 	"v_tx_nss_1",
2169 	"v_tx_nss_2",
2170 	"v_tx_nss_3",
2171 	"v_tx_nss_4",
2172 };
2173 
2174 #define MT7996_SSTATS_LEN ARRAY_SIZE(mt7996_gstrings_stats)
2175 
2176 /* Ethtool related API */
2177 static
2178 void mt7996_get_et_strings(struct ieee80211_hw *hw,
2179 			   struct ieee80211_vif *vif,
2180 			   u32 sset, u8 *data)
2181 {
2182 	if (sset == ETH_SS_STATS)
2183 		memcpy(data, mt7996_gstrings_stats,
2184 		       sizeof(mt7996_gstrings_stats));
2185 }
2186 
2187 static
2188 int mt7996_get_et_sset_count(struct ieee80211_hw *hw,
2189 			     struct ieee80211_vif *vif, int sset)
2190 {
2191 	if (sset == ETH_SS_STATS)
2192 		return MT7996_SSTATS_LEN;
2193 
2194 	return 0;
2195 }
2196 
2197 static void mt7996_ethtool_worker(void *wi_data, struct ieee80211_sta *sta)
2198 {
2199 	struct mt76_ethtool_worker_info *wi = wi_data;
2200 	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
2201 	struct mt7996_sta_link *msta_link = &msta->deflink;
2202 
2203 	if (msta->vif->deflink.mt76.idx != wi->idx)
2204 		return;
2205 
2206 	mt76_ethtool_worker(wi, &msta_link->wcid.stats, true);
2207 }
2208 
2209 static
2210 void mt7996_get_et_stats(struct ieee80211_hw *hw,
2211 			 struct ieee80211_vif *vif,
2212 			 struct ethtool_stats *stats, u64 *data)
2213 {
2214 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
2215 	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
2216 	struct mt7996_phy *phy = mt7996_vif_link_phy(&mvif->deflink);
2217 	struct mt76_mib_stats *mib = &phy->mib;
2218 	struct mt76_ethtool_worker_info wi = {
2219 		.data = data,
2220 		.idx = mvif->deflink.mt76.idx,
2221 	};
2222 	/* See mt7996_ampdu_stat_read_phy, etc */
2223 	int i, ei = 0;
2224 
2225 	if (!phy)
2226 		return;
2227 
2228 	mutex_lock(&dev->mt76.mutex);
2229 
2230 	mt7996_mac_update_stats(phy);
2231 
2232 	data[ei++] = mib->tx_ampdu_cnt;
2233 	data[ei++] = mib->tx_stop_q_empty_cnt;
2234 	data[ei++] = mib->tx_mpdu_attempts_cnt;
2235 	data[ei++] = mib->tx_mpdu_success_cnt;
2236 	data[ei++] = mib->tx_rwp_fail_cnt;
2237 	data[ei++] = mib->tx_rwp_need_cnt;
2238 	data[ei++] = mib->tx_bf_ebf_ppdu_cnt;
2239 	data[ei++] = mib->tx_bf_ibf_ppdu_cnt;
2240 
2241 	/* Tx ampdu stat */
2242 	for (i = 0; i < 15 /*ARRAY_SIZE(bound)*/; i++)
2243 		data[ei++] = phy->mt76->aggr_stats[i];
2244 	data[ei++] = phy->mib.ba_miss_cnt;
2245 
2246 	/* Tx Beamformer monitor */
2247 	data[ei++] = mib->tx_bf_ibf_ppdu_cnt;
2248 	data[ei++] = mib->tx_bf_ebf_ppdu_cnt;
2249 
2250 	/* Tx Beamformer Rx feedback monitor */
2251 	data[ei++] = mib->tx_bf_rx_fb_all_cnt;
2252 	data[ei++] = mib->tx_bf_rx_fb_he_cnt;
2253 	data[ei++] = mib->tx_bf_rx_fb_vht_cnt;
2254 	data[ei++] = mib->tx_bf_rx_fb_ht_cnt;
2255 
2256 	data[ei++] = mib->tx_bf_rx_fb_bw;
2257 	data[ei++] = mib->tx_bf_rx_fb_nc_cnt;
2258 	data[ei++] = mib->tx_bf_rx_fb_nr_cnt;
2259 
2260 	/* Tx Beamformee Rx NDPA & Tx feedback report */
2261 	data[ei++] = mib->tx_bf_fb_cpl_cnt;
2262 	data[ei++] = mib->tx_bf_fb_trig_cnt;
2263 
2264 	/* Tx SU & MU counters */
2265 	data[ei++] = mib->tx_mu_bf_cnt;
2266 	data[ei++] = mib->tx_mu_mpdu_cnt;
2267 	data[ei++] = mib->tx_mu_acked_mpdu_cnt;
2268 	data[ei++] = mib->tx_su_acked_mpdu_cnt;
2269 
2270 	/* Tx amsdu info (pack-count histogram) */
2271 	for (i = 0; i < ARRAY_SIZE(mib->tx_amsdu); i++)
2272 		data[ei++] = mib->tx_amsdu[i];
2273 
2274 	/* rx counters */
2275 	data[ei++] = mib->rx_fifo_full_cnt;
2276 	data[ei++] = mib->rx_mpdu_cnt;
2277 	data[ei++] = mib->channel_idle_cnt;
2278 	data[ei++] = mib->rx_vector_mismatch_cnt;
2279 	data[ei++] = mib->rx_delimiter_fail_cnt;
2280 	data[ei++] = mib->rx_len_mismatch_cnt;
2281 	data[ei++] = mib->rx_ampdu_cnt;
2282 	data[ei++] = mib->rx_ampdu_bytes_cnt;
2283 	data[ei++] = mib->rx_ampdu_valid_subframe_cnt;
2284 	data[ei++] = mib->rx_ampdu_valid_subframe_bytes_cnt;
2285 	data[ei++] = mib->rx_pfdrop_cnt;
2286 	data[ei++] = mib->rx_vec_queue_overflow_drop_cnt;
2287 	data[ei++] = mib->rx_ba_cnt;
2288 
2289 	/* Add values for all stations owned by this vif */
2290 	wi.initial_stat_idx = ei;
2291 	ieee80211_iterate_stations_atomic(hw, mt7996_ethtool_worker, &wi);
2292 
2293 	mutex_unlock(&dev->mt76.mutex);
2294 
2295 	if (wi.sta_count == 0)
2296 		return;
2297 
2298 	ei += wi.worker_stat_count;
2299 	if (ei != MT7996_SSTATS_LEN)
2300 		dev_err(dev->mt76.dev, "ei: %d  MT7996_SSTATS_LEN: %d",
2301 			ei, (int)MT7996_SSTATS_LEN);
2302 }
2303 
2304 static void
2305 mt7996_twt_teardown_request(struct ieee80211_hw *hw,
2306 			    struct ieee80211_sta *sta,
2307 			    u8 flowid)
2308 {
2309 	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
2310 	struct mt7996_sta_link *msta_link = &msta->deflink;
2311 	struct mt7996_vif_link *link = &msta->vif->deflink;
2312 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
2313 
2314 	mutex_lock(&dev->mt76.mutex);
2315 	mt7996_mac_twt_teardown_flow(dev, link, msta_link, flowid);
2316 	mutex_unlock(&dev->mt76.mutex);
2317 }
2318 
2319 static int
2320 mt7996_set_radar_background(struct ieee80211_hw *hw,
2321 			    struct cfg80211_chan_def *chandef)
2322 {
2323 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
2324 	struct mt7996_phy *phy;
2325 	int ret = -EINVAL;
2326 	bool running;
2327 
2328 	if (chandef)
2329 		phy = mt7996_band_phy(dev, chandef->chan->band);
2330 	else
2331 		phy = dev->rdd2_phy;
2332 	if (!phy)
2333 	    return -EINVAL;
2334 
2335 	mutex_lock(&dev->mt76.mutex);
2336 
2337 	if (dev->mt76.region == NL80211_DFS_UNSET)
2338 		goto out;
2339 
2340 	if (dev->rdd2_phy && dev->rdd2_phy != phy) {
2341 		/* rdd2 is already locked */
2342 		ret = -EBUSY;
2343 		goto out;
2344 	}
2345 
2346 	/* rdd2 already configured on a radar channel */
2347 	running = dev->rdd2_phy &&
2348 		  cfg80211_chandef_valid(&dev->rdd2_chandef) &&
2349 		  !!(dev->rdd2_chandef.chan->flags & IEEE80211_CHAN_RADAR);
2350 
2351 	if (!chandef || running ||
2352 	    !(chandef->chan->flags & IEEE80211_CHAN_RADAR)) {
2353 		ret = mt7996_mcu_rdd_background_enable(phy, NULL);
2354 		if (ret)
2355 			goto out;
2356 
2357 		if (!running)
2358 			goto update_phy;
2359 	}
2360 
2361 	ret = mt7996_mcu_rdd_background_enable(phy, chandef);
2362 	if (ret)
2363 		goto out;
2364 
2365 update_phy:
2366 	dev->rdd2_phy = chandef ? phy : NULL;
2367 	if (chandef)
2368 		dev->rdd2_chandef = *chandef;
2369 out:
2370 	mutex_unlock(&dev->mt76.mutex);
2371 
2372 	return ret;
2373 }
2374 
2375 static int
2376 mt7996_net_fill_forward_path(struct ieee80211_hw *hw,
2377 			     struct ieee80211_vif *vif,
2378 			     struct ieee80211_sta *sta,
2379 			     struct net_device_path_ctx *ctx,
2380 			     struct net_device_path *path)
2381 {
2382 	struct mt7996_sta *msta = (struct mt7996_sta *)sta->drv_priv;
2383 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
2384 	struct mtk_wed_device *wed = &dev->mt76.mmio.wed;
2385 	struct mt7996_sta_link *msta_link;
2386 	struct mt7996_vif_link *link;
2387 
2388 	link = mt7996_vif_link(dev, vif, msta->deflink_id);
2389 	if (!link)
2390 		return -EIO;
2391 
2392 	msta_link = rcu_dereference(msta->link[msta->deflink_id]);
2393 	if (!msta_link)
2394 		return -EIO;
2395 
2396 	if (!msta_link->wcid.sta || msta_link->wcid.idx > MT7996_WTBL_STA)
2397 		return -EIO;
2398 
2399 	if (dev->hif2 &&
2400 	    ((is_mt7996(&dev->mt76) && msta_link->wcid.phy_idx == MT_BAND2) ||
2401 	     (is_mt7992(&dev->mt76) && msta_link->wcid.phy_idx == MT_BAND1)))
2402 		wed = &dev->mt76.mmio.wed_hif2;
2403 
2404 	if (!mtk_wed_device_active(wed) &&
2405 	    !mt76_npu_device_active(&dev->mt76))
2406 		return -ENODEV;
2407 
2408 	path->type = DEV_PATH_MTK_WDMA;
2409 	path->dev = ctx->dev;
2410 #ifdef CONFIG_NET_MEDIATEK_SOC_WED
2411 	if (mtk_wed_device_active(wed))
2412 		path->mtk_wdma.wdma_idx = wed->wdma_idx;
2413 	else
2414 #endif
2415 	if (is_mt7996(&dev->mt76) && mt76_npu_device_active(&dev->mt76) &&
2416 	    msta_link->wcid.phy_idx == MT_BAND2)
2417 		path->mtk_wdma.wdma_idx = 1;
2418 	else
2419 		path->mtk_wdma.wdma_idx = link->mt76.band_idx;
2420 	path->mtk_wdma.bss = link->mt76.idx;
2421 	path->mtk_wdma.queue = 0;
2422 	path->mtk_wdma.wcid = msta_link->wcid.idx;
2423 
2424 	if (ieee80211_hw_check(hw, SUPPORTS_AMSDU_IN_AMPDU) &&
2425 	    mtk_wed_is_amsdu_supported(wed))
2426 		path->mtk_wdma.amsdu = msta_link->wcid.amsdu;
2427 	else
2428 		path->mtk_wdma.amsdu = 0;
2429 	ctx->dev = NULL;
2430 
2431 	return 0;
2432 }
2433 
2434 static int
2435 mt7996_change_vif_links(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
2436 			u16 old_links, u16 new_links,
2437 			struct ieee80211_bss_conf *old[IEEE80211_MLD_MAX_NUM_LINKS])
2438 {
2439 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
2440 	struct mt7996_vif *mvif = (struct mt7996_vif *)vif->drv_priv;
2441 	int ret = 0;
2442 
2443 	mutex_lock(&dev->mt76.mutex);
2444 
2445 	if (!old_links) {
2446 		int idx;
2447 
2448 		idx = get_own_mld_idx(dev->mld_idx_mask, true);
2449 		if (idx < 0) {
2450 			ret = -ENOSPC;
2451 			goto out;
2452 		}
2453 		mvif->mld_group_idx = idx;
2454 		dev->mld_idx_mask |= BIT_ULL(mvif->mld_group_idx);
2455 
2456 		idx = get_free_idx(dev->mld_remap_idx_mask, 0, 15) - 1;
2457 		if (idx < 0) {
2458 			ret = -ENOSPC;
2459 			goto out;
2460 		}
2461 		mvif->mld_remap_idx = idx;
2462 		dev->mld_remap_idx_mask |= BIT_ULL(mvif->mld_remap_idx);
2463 	}
2464 
2465 	if (new_links)
2466 		goto out;
2467 
2468 	dev->mld_idx_mask &= ~BIT_ULL(mvif->mld_group_idx);
2469 	dev->mld_remap_idx_mask &= ~BIT_ULL(mvif->mld_remap_idx);
2470 
2471 out:
2472 	mutex_unlock(&dev->mt76.mutex);
2473 
2474 	return ret;
2475 }
2476 
2477 static void
2478 mt7996_reconfig_complete(struct ieee80211_hw *hw,
2479 			 enum ieee80211_reconfig_type reconfig_type)
2480 {
2481 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
2482 	struct mt7996_phy *phy;
2483 
2484 	ieee80211_wake_queues(hw);
2485 	mt7996_for_each_phy(dev, phy)
2486 		ieee80211_queue_delayed_work(hw, &phy->mt76->mac_work,
2487 					     MT7996_WATCHDOG_TIME);
2488 }
2489 
2490 static int
2491 mt7996_set_eml_op_mode(struct ieee80211_hw *hw, struct ieee80211_vif *vif,
2492 		       struct ieee80211_sta *sta,
2493 		       struct ieee80211_eml_params *eml_params)
2494 {
2495 	struct mt7996_dev *dev = mt7996_hw_dev(hw);
2496 	int ret;
2497 
2498 	mutex_lock(&dev->mt76.mutex);
2499 	ret = mt7996_mcu_set_emlsr_mode(dev, vif, sta, eml_params);
2500 	mutex_unlock(&dev->mt76.mutex);
2501 
2502 	return ret;
2503 }
2504 
2505 const struct ieee80211_ops mt7996_ops = {
2506 	.add_chanctx = mt76_add_chanctx,
2507 	.remove_chanctx = mt76_remove_chanctx,
2508 	.change_chanctx = mt76_change_chanctx,
2509 	.assign_vif_chanctx = mt76_assign_vif_chanctx,
2510 	.unassign_vif_chanctx = mt76_unassign_vif_chanctx,
2511 	.switch_vif_chanctx = mt76_switch_vif_chanctx,
2512 	.tx = mt7996_tx,
2513 	.start = mt7996_start,
2514 	.stop = mt7996_stop,
2515 	.add_interface = mt7996_add_interface,
2516 	.remove_interface = mt7996_remove_interface,
2517 	.config = mt7996_config,
2518 	.conf_tx = mt7996_conf_tx,
2519 	.configure_filter = mt7996_configure_filter,
2520 	.vif_cfg_changed = mt7996_vif_cfg_changed,
2521 	.link_info_changed = mt7996_link_info_changed,
2522 	.sta_state = mt7996_sta_state,
2523 	.sta_pre_rcu_remove = mt76_sta_pre_rcu_remove,
2524 	.link_sta_rc_update = mt7996_link_sta_rc_update,
2525 	.set_key = mt7996_set_key,
2526 	.ampdu_action = mt7996_ampdu_action,
2527 	.set_rts_threshold = mt7996_set_rts_threshold,
2528 	.wake_tx_queue = mt76_wake_tx_queue,
2529 	.hw_scan = mt76_hw_scan,
2530 	.cancel_hw_scan = mt76_cancel_hw_scan,
2531 	.remain_on_channel = mt76_remain_on_channel,
2532 	.cancel_remain_on_channel = mt76_cancel_remain_on_channel,
2533 	.release_buffered_frames = mt76_release_buffered_frames,
2534 	.get_txpower = mt7996_get_txpower,
2535 	.channel_switch_beacon = mt7996_channel_switch_beacon,
2536 	.post_channel_switch = mt7996_post_channel_switch,
2537 	.get_stats = mt7996_get_stats,
2538 	.get_et_sset_count = mt7996_get_et_sset_count,
2539 	.get_et_stats = mt7996_get_et_stats,
2540 	.get_et_strings = mt7996_get_et_strings,
2541 	.get_tsf = mt7996_get_tsf,
2542 	.set_tsf = mt7996_set_tsf,
2543 	.offset_tsf = mt7996_offset_tsf,
2544 	.get_survey = mt76_get_survey,
2545 	.get_antenna = mt76_get_antenna,
2546 	.set_antenna = mt7996_set_antenna,
2547 	.set_bitrate_mask = mt7996_set_bitrate_mask,
2548 	.set_coverage_class = mt7996_set_coverage_class,
2549 	.sta_statistics = mt7996_sta_statistics,
2550 	.sta_set_4addr = mt7996_sta_set_4addr,
2551 	.sta_set_decap_offload = mt7996_sta_set_decap_offload,
2552 	.add_twt_setup = mt7996_mac_add_twt_setup,
2553 	.twt_teardown_request = mt7996_twt_teardown_request,
2554 #ifdef CONFIG_MAC80211_DEBUGFS
2555 	.sta_add_debugfs = mt7996_sta_add_debugfs,
2556 	.link_sta_add_debugfs = mt7996_link_sta_add_debugfs,
2557 #endif
2558 	.set_radar_background = mt7996_set_radar_background,
2559 	.net_fill_forward_path = mt7996_net_fill_forward_path,
2560 #ifdef CONFIG_NET_MEDIATEK_SOC_WED
2561 	.net_setup_tc = mt76_wed_net_setup_tc,
2562 #elif defined(CONFIG_MT7996_NPU)
2563 	.net_setup_tc = mt76_npu_net_setup_tc,
2564 #endif
2565 	.change_vif_links = mt7996_change_vif_links,
2566 	.change_sta_links = mt7996_mac_sta_change_links,
2567 	.reconfig_complete = mt7996_reconfig_complete,
2568 	.set_eml_op_mode = mt7996_set_eml_op_mode,
2569 };
2570