1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2 /* Copyright(c) 2020 Realtek Corporation
3 */
4
5 #include <linux/pci.h>
6
7 #include "mac.h"
8 #include "pci.h"
9 #include "reg.h"
10 #include "ser.h"
11
12 static bool rtw89_pci_disable_clkreq;
13 static bool rtw89_pci_disable_aspm_l1;
14 static bool rtw89_pci_disable_l1ss;
15 module_param_named(disable_clkreq, rtw89_pci_disable_clkreq, bool, 0644);
16 module_param_named(disable_aspm_l1, rtw89_pci_disable_aspm_l1, bool, 0644);
17 module_param_named(disable_aspm_l1ss, rtw89_pci_disable_l1ss, bool, 0644);
18 MODULE_PARM_DESC(disable_clkreq, "Set Y to disable PCI clkreq support");
19 MODULE_PARM_DESC(disable_aspm_l1, "Set Y to disable PCI ASPM L1 support");
20 MODULE_PARM_DESC(disable_aspm_l1ss, "Set Y to disable PCI L1SS support");
21
rtw89_pci_rst_bdram_ax(struct rtw89_dev * rtwdev)22 static int rtw89_pci_rst_bdram_ax(struct rtw89_dev *rtwdev)
23 {
24 u32 val;
25 int ret;
26
27 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_RST_BDRAM);
28
29 ret = read_poll_timeout_atomic(rtw89_read32, val, !(val & B_AX_RST_BDRAM),
30 1, RTW89_PCI_POLL_BDRAM_RST_CNT, false,
31 rtwdev, R_AX_PCIE_INIT_CFG1);
32
33 return ret;
34 }
35
rtw89_pci_dma_recalc(struct rtw89_dev * rtwdev,struct rtw89_pci_dma_ring * bd_ring,u32 cur_idx,bool tx)36 static u32 rtw89_pci_dma_recalc(struct rtw89_dev *rtwdev,
37 struct rtw89_pci_dma_ring *bd_ring,
38 u32 cur_idx, bool tx)
39 {
40 const struct rtw89_pci_info *info = rtwdev->pci_info;
41 u32 cnt, cur_rp, wp, rp, len;
42
43 rp = bd_ring->rp;
44 wp = bd_ring->wp;
45 len = bd_ring->len;
46
47 cur_rp = FIELD_GET(TXBD_HW_IDX_MASK, cur_idx);
48 if (tx) {
49 cnt = cur_rp >= rp ? cur_rp - rp : len - (rp - cur_rp);
50 } else {
51 if (info->rx_ring_eq_is_full)
52 wp += 1;
53
54 cnt = cur_rp >= wp ? cur_rp - wp : len - (wp - cur_rp);
55 }
56
57 bd_ring->rp = cur_rp;
58
59 return cnt;
60 }
61
rtw89_pci_txbd_recalc(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring)62 static u32 rtw89_pci_txbd_recalc(struct rtw89_dev *rtwdev,
63 struct rtw89_pci_tx_ring *tx_ring)
64 {
65 struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring;
66 u32 addr_idx = bd_ring->addr.idx;
67 u32 cnt, idx;
68
69 idx = rtw89_read32(rtwdev, addr_idx);
70 cnt = rtw89_pci_dma_recalc(rtwdev, bd_ring, idx, true);
71
72 return cnt;
73 }
74
rtw89_pci_release_fwcmd(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci,u32 cnt,bool release_all)75 static void rtw89_pci_release_fwcmd(struct rtw89_dev *rtwdev,
76 struct rtw89_pci *rtwpci,
77 u32 cnt, bool release_all)
78 {
79 struct rtw89_pci_tx_data *tx_data;
80 struct sk_buff *skb;
81 u32 qlen;
82
83 while (cnt--) {
84 skb = skb_dequeue(&rtwpci->h2c_queue);
85 if (!skb) {
86 rtw89_err(rtwdev, "failed to pre-release fwcmd\n");
87 return;
88 }
89 skb_queue_tail(&rtwpci->h2c_release_queue, skb);
90 }
91
92 qlen = skb_queue_len(&rtwpci->h2c_release_queue);
93 if (!release_all)
94 qlen = qlen > RTW89_PCI_MULTITAG ? qlen - RTW89_PCI_MULTITAG : 0;
95
96 while (qlen--) {
97 skb = skb_dequeue(&rtwpci->h2c_release_queue);
98 if (!skb) {
99 rtw89_err(rtwdev, "failed to release fwcmd\n");
100 return;
101 }
102 tx_data = RTW89_PCI_TX_SKB_CB(skb);
103 dma_unmap_single(&rtwpci->pdev->dev, tx_data->dma, skb->len,
104 DMA_TO_DEVICE);
105 dev_kfree_skb_any(skb);
106 }
107 }
108
rtw89_pci_reclaim_tx_fwcmd(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci)109 static void rtw89_pci_reclaim_tx_fwcmd(struct rtw89_dev *rtwdev,
110 struct rtw89_pci *rtwpci)
111 {
112 struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[RTW89_TXCH_CH12];
113 u32 cnt;
114
115 cnt = rtw89_pci_txbd_recalc(rtwdev, tx_ring);
116 if (!cnt)
117 return;
118 rtw89_pci_release_fwcmd(rtwdev, rtwpci, cnt, false);
119 }
120
rtw89_pci_rxbd_recalc(struct rtw89_dev * rtwdev,struct rtw89_pci_rx_ring * rx_ring)121 static u32 rtw89_pci_rxbd_recalc(struct rtw89_dev *rtwdev,
122 struct rtw89_pci_rx_ring *rx_ring)
123 {
124 struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring;
125 u32 addr_idx = bd_ring->addr.idx;
126 u32 cnt, idx;
127
128 idx = rtw89_read32(rtwdev, addr_idx);
129 cnt = rtw89_pci_dma_recalc(rtwdev, bd_ring, idx, false);
130
131 return cnt;
132 }
133
rtw89_pci_sync_skb_for_cpu(struct rtw89_dev * rtwdev,struct sk_buff * skb)134 static void rtw89_pci_sync_skb_for_cpu(struct rtw89_dev *rtwdev,
135 struct sk_buff *skb)
136 {
137 struct rtw89_pci_rx_info *rx_info;
138 dma_addr_t dma;
139
140 rx_info = RTW89_PCI_RX_SKB_CB(skb);
141 dma = rx_info->dma;
142 dma_sync_single_for_cpu(rtwdev->dev, dma, RTW89_PCI_RX_BUF_SIZE,
143 DMA_FROM_DEVICE);
144 }
145
rtw89_pci_sync_skb_for_device(struct rtw89_dev * rtwdev,struct sk_buff * skb)146 static void rtw89_pci_sync_skb_for_device(struct rtw89_dev *rtwdev,
147 struct sk_buff *skb)
148 {
149 struct rtw89_pci_rx_info *rx_info;
150 dma_addr_t dma;
151
152 rx_info = RTW89_PCI_RX_SKB_CB(skb);
153 dma = rx_info->dma;
154 dma_sync_single_for_device(rtwdev->dev, dma, RTW89_PCI_RX_BUF_SIZE,
155 DMA_FROM_DEVICE);
156 }
157
rtw89_pci_rxbd_info_update(struct rtw89_dev * rtwdev,struct sk_buff * skb)158 static int rtw89_pci_rxbd_info_update(struct rtw89_dev *rtwdev,
159 struct sk_buff *skb)
160 {
161 struct rtw89_pci_rxbd_info *rxbd_info;
162 struct rtw89_pci_rx_info *rx_info = RTW89_PCI_RX_SKB_CB(skb);
163
164 rxbd_info = (struct rtw89_pci_rxbd_info *)skb->data;
165 rx_info->fs = le32_get_bits(rxbd_info->dword, RTW89_PCI_RXBD_FS);
166 rx_info->ls = le32_get_bits(rxbd_info->dword, RTW89_PCI_RXBD_LS);
167 rx_info->len = le32_get_bits(rxbd_info->dword, RTW89_PCI_RXBD_WRITE_SIZE);
168 rx_info->tag = le32_get_bits(rxbd_info->dword, RTW89_PCI_RXBD_TAG);
169
170 return 0;
171 }
172
rtw89_pci_ctrl_txdma_ch_pcie(struct rtw89_dev * rtwdev,bool enable)173 static void rtw89_pci_ctrl_txdma_ch_pcie(struct rtw89_dev *rtwdev, bool enable)
174 {
175 const struct rtw89_pci_info *info = rtwdev->pci_info;
176 const struct rtw89_reg_def *dma_stop1 = &info->dma_stop1;
177 const struct rtw89_reg_def *dma_stop2 = &info->dma_stop2;
178
179 if (enable) {
180 rtw89_write32_clr(rtwdev, dma_stop1->addr, dma_stop1->mask);
181 if (dma_stop2->addr)
182 rtw89_write32_clr(rtwdev, dma_stop2->addr, dma_stop2->mask);
183 } else {
184 rtw89_write32_set(rtwdev, dma_stop1->addr, dma_stop1->mask);
185 if (dma_stop2->addr)
186 rtw89_write32_set(rtwdev, dma_stop2->addr, dma_stop2->mask);
187 }
188 }
189
rtw89_pci_ctrl_txdma_fw_ch_pcie(struct rtw89_dev * rtwdev,bool enable)190 static void rtw89_pci_ctrl_txdma_fw_ch_pcie(struct rtw89_dev *rtwdev, bool enable)
191 {
192 const struct rtw89_pci_info *info = rtwdev->pci_info;
193 const struct rtw89_reg_def *dma_stop1 = &info->dma_stop1;
194
195 if (enable)
196 rtw89_write32_clr(rtwdev, dma_stop1->addr, B_AX_STOP_CH12);
197 else
198 rtw89_write32_set(rtwdev, dma_stop1->addr, B_AX_STOP_CH12);
199 }
200
201 static bool
rtw89_skb_put_rx_data(struct rtw89_dev * rtwdev,bool fs,bool ls,struct sk_buff * new,const struct sk_buff * skb,u32 offset,const struct rtw89_pci_rx_info * rx_info,const struct rtw89_rx_desc_info * desc_info)202 rtw89_skb_put_rx_data(struct rtw89_dev *rtwdev, bool fs, bool ls,
203 struct sk_buff *new,
204 const struct sk_buff *skb, u32 offset,
205 const struct rtw89_pci_rx_info *rx_info,
206 const struct rtw89_rx_desc_info *desc_info)
207 {
208 u32 copy_len = rx_info->len - offset;
209
210 if (unlikely(skb_tailroom(new) < copy_len)) {
211 rtw89_debug(rtwdev, RTW89_DBG_TXRX,
212 "invalid rx data length bd_len=%d desc_len=%d offset=%d (fs=%d ls=%d)\n",
213 rx_info->len, desc_info->pkt_size, offset, fs, ls);
214 rtw89_hex_dump(rtwdev, RTW89_DBG_TXRX, "rx_data: ",
215 skb->data, rx_info->len);
216 /* length of a single segment skb is desc_info->pkt_size */
217 if (fs && ls) {
218 copy_len = desc_info->pkt_size;
219 } else {
220 rtw89_info(rtwdev, "drop rx data due to invalid length\n");
221 return false;
222 }
223 }
224
225 skb_put_data(new, skb->data + offset, copy_len);
226
227 return true;
228 }
229
rtw89_pci_get_rx_skb_idx(struct rtw89_dev * rtwdev,struct rtw89_pci_dma_ring * bd_ring)230 static u32 rtw89_pci_get_rx_skb_idx(struct rtw89_dev *rtwdev,
231 struct rtw89_pci_dma_ring *bd_ring)
232 {
233 const struct rtw89_pci_info *info = rtwdev->pci_info;
234 u32 wp = bd_ring->wp;
235
236 if (!info->rx_ring_eq_is_full)
237 return wp;
238
239 if (++wp >= bd_ring->len)
240 wp = 0;
241
242 return wp;
243 }
244
rtw89_pci_rxbd_deliver_skbs(struct rtw89_dev * rtwdev,struct rtw89_pci_rx_ring * rx_ring)245 static u32 rtw89_pci_rxbd_deliver_skbs(struct rtw89_dev *rtwdev,
246 struct rtw89_pci_rx_ring *rx_ring)
247 {
248 struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring;
249 struct rtw89_pci_rx_info *rx_info;
250 struct rtw89_rx_desc_info *desc_info = &rx_ring->diliver_desc;
251 struct sk_buff *new = rx_ring->diliver_skb;
252 struct sk_buff *skb;
253 u32 rxinfo_size = sizeof(struct rtw89_pci_rxbd_info);
254 u32 skb_idx;
255 u32 offset;
256 u32 cnt = 1;
257 bool fs, ls;
258 int ret;
259
260 skb_idx = rtw89_pci_get_rx_skb_idx(rtwdev, bd_ring);
261 skb = rx_ring->buf[skb_idx];
262 rtw89_pci_sync_skb_for_cpu(rtwdev, skb);
263
264 ret = rtw89_pci_rxbd_info_update(rtwdev, skb);
265 if (ret) {
266 rtw89_err(rtwdev, "failed to update %d RXBD info: %d\n",
267 bd_ring->wp, ret);
268 goto err_sync_device;
269 }
270
271 rx_info = RTW89_PCI_RX_SKB_CB(skb);
272 fs = rx_info->fs;
273 ls = rx_info->ls;
274
275 if (fs) {
276 if (new) {
277 rtw89_debug(rtwdev, RTW89_DBG_UNEXP,
278 "skb should not be ready before first segment start\n");
279 goto err_sync_device;
280 }
281 if (desc_info->ready) {
282 rtw89_warn(rtwdev, "desc info should not be ready before first segment start\n");
283 goto err_sync_device;
284 }
285
286 rtw89_chip_query_rxdesc(rtwdev, desc_info, skb->data, rxinfo_size);
287
288 new = rtw89_alloc_skb_for_rx(rtwdev, desc_info->pkt_size);
289 if (!new)
290 goto err_sync_device;
291
292 rx_ring->diliver_skb = new;
293
294 /* first segment has RX desc */
295 offset = desc_info->offset + desc_info->rxd_len;
296 } else {
297 offset = sizeof(struct rtw89_pci_rxbd_info);
298 if (!new) {
299 rtw89_debug(rtwdev, RTW89_DBG_UNEXP, "no last skb\n");
300 goto err_sync_device;
301 }
302 }
303 if (!rtw89_skb_put_rx_data(rtwdev, fs, ls, new, skb, offset, rx_info, desc_info))
304 goto err_sync_device;
305 rtw89_pci_sync_skb_for_device(rtwdev, skb);
306 rtw89_pci_rxbd_increase(rx_ring, 1);
307
308 if (!desc_info->ready) {
309 rtw89_warn(rtwdev, "no rx desc information\n");
310 goto err_free_resource;
311 }
312 if (ls) {
313 rtw89_core_rx(rtwdev, desc_info, new);
314 rx_ring->diliver_skb = NULL;
315 desc_info->ready = false;
316 }
317
318 return cnt;
319
320 err_sync_device:
321 rtw89_pci_sync_skb_for_device(rtwdev, skb);
322 rtw89_pci_rxbd_increase(rx_ring, 1);
323 err_free_resource:
324 if (new)
325 dev_kfree_skb_any(new);
326 rx_ring->diliver_skb = NULL;
327 desc_info->ready = false;
328
329 return cnt;
330 }
331
rtw89_pci_rxbd_deliver(struct rtw89_dev * rtwdev,struct rtw89_pci_rx_ring * rx_ring,u32 cnt)332 static void rtw89_pci_rxbd_deliver(struct rtw89_dev *rtwdev,
333 struct rtw89_pci_rx_ring *rx_ring,
334 u32 cnt)
335 {
336 struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring;
337 u32 rx_cnt;
338
339 while (cnt && rtwdev->napi_budget_countdown > 0) {
340 rx_cnt = rtw89_pci_rxbd_deliver_skbs(rtwdev, rx_ring);
341 if (!rx_cnt) {
342 rtw89_err(rtwdev, "failed to deliver RXBD skb\n");
343
344 /* skip the rest RXBD bufs */
345 rtw89_pci_rxbd_increase(rx_ring, cnt);
346 break;
347 }
348
349 cnt -= rx_cnt;
350 }
351
352 rtw89_write16(rtwdev, bd_ring->addr.idx, bd_ring->wp);
353 }
354
rtw89_pci_poll_rxq_dma(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci,int budget)355 static int rtw89_pci_poll_rxq_dma(struct rtw89_dev *rtwdev,
356 struct rtw89_pci *rtwpci, int budget)
357 {
358 struct rtw89_pci_rx_ring *rx_ring;
359 int countdown = rtwdev->napi_budget_countdown;
360 u32 cnt;
361
362 rx_ring = &rtwpci->rx_rings[RTW89_RXCH_RXQ];
363
364 cnt = rtw89_pci_rxbd_recalc(rtwdev, rx_ring);
365 if (!cnt)
366 return 0;
367
368 cnt = min_t(u32, budget, cnt);
369
370 rtw89_pci_rxbd_deliver(rtwdev, rx_ring, cnt);
371
372 /* In case of flushing pending SKBs, the countdown may exceed. */
373 if (rtwdev->napi_budget_countdown <= 0)
374 return budget;
375
376 return budget - countdown;
377 }
378
rtw89_pci_tx_status(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring,struct sk_buff * skb,u8 tx_status)379 static void rtw89_pci_tx_status(struct rtw89_dev *rtwdev,
380 struct rtw89_pci_tx_ring *tx_ring,
381 struct sk_buff *skb, u8 tx_status)
382 {
383 struct rtw89_tx_skb_data *skb_data = RTW89_TX_SKB_CB(skb);
384 struct ieee80211_tx_info *info;
385
386 rtw89_core_tx_wait_complete(rtwdev, skb_data, tx_status == RTW89_TX_DONE);
387
388 info = IEEE80211_SKB_CB(skb);
389 ieee80211_tx_info_clear_status(info);
390
391 if (info->flags & IEEE80211_TX_CTL_NO_ACK)
392 info->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED;
393 if (tx_status == RTW89_TX_DONE) {
394 info->flags |= IEEE80211_TX_STAT_ACK;
395 tx_ring->tx_acked++;
396 } else {
397 if (info->flags & IEEE80211_TX_CTL_REQ_TX_STATUS)
398 rtw89_debug(rtwdev, RTW89_DBG_FW,
399 "failed to TX of status %x\n", tx_status);
400 switch (tx_status) {
401 case RTW89_TX_RETRY_LIMIT:
402 tx_ring->tx_retry_lmt++;
403 break;
404 case RTW89_TX_LIFE_TIME:
405 tx_ring->tx_life_time++;
406 break;
407 case RTW89_TX_MACID_DROP:
408 tx_ring->tx_mac_id_drop++;
409 break;
410 default:
411 rtw89_warn(rtwdev, "invalid TX status %x\n", tx_status);
412 break;
413 }
414 }
415
416 ieee80211_tx_status_ni(rtwdev->hw, skb);
417 }
418
rtw89_pci_reclaim_txbd(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring)419 static void rtw89_pci_reclaim_txbd(struct rtw89_dev *rtwdev, struct rtw89_pci_tx_ring *tx_ring)
420 {
421 struct rtw89_pci_tx_wd *txwd;
422 u32 cnt;
423
424 cnt = rtw89_pci_txbd_recalc(rtwdev, tx_ring);
425 while (cnt--) {
426 txwd = list_first_entry_or_null(&tx_ring->busy_pages, struct rtw89_pci_tx_wd, list);
427 if (!txwd) {
428 rtw89_warn(rtwdev, "No busy txwd pages available\n");
429 break;
430 }
431
432 list_del_init(&txwd->list);
433
434 /* this skb has been freed by RPP */
435 if (skb_queue_len(&txwd->queue) == 0)
436 rtw89_pci_enqueue_txwd(tx_ring, txwd);
437 }
438 }
439
rtw89_pci_release_busy_txwd(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring)440 static void rtw89_pci_release_busy_txwd(struct rtw89_dev *rtwdev,
441 struct rtw89_pci_tx_ring *tx_ring)
442 {
443 struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
444 struct rtw89_pci_tx_wd *txwd;
445 int i;
446
447 for (i = 0; i < wd_ring->page_num; i++) {
448 txwd = list_first_entry_or_null(&tx_ring->busy_pages, struct rtw89_pci_tx_wd, list);
449 if (!txwd)
450 break;
451
452 list_del_init(&txwd->list);
453 }
454 }
455
rtw89_pci_release_txwd_skb(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring,struct rtw89_pci_tx_wd * txwd,u16 seq,u8 tx_status)456 static void rtw89_pci_release_txwd_skb(struct rtw89_dev *rtwdev,
457 struct rtw89_pci_tx_ring *tx_ring,
458 struct rtw89_pci_tx_wd *txwd, u16 seq,
459 u8 tx_status)
460 {
461 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
462 struct rtw89_pci_tx_data *tx_data;
463 struct sk_buff *skb, *tmp;
464 u8 txch = tx_ring->txch;
465
466 if (!list_empty(&txwd->list)) {
467 rtw89_pci_reclaim_txbd(rtwdev, tx_ring);
468 /* In low power mode, RPP can receive before updating of TX BD.
469 * In normal mode, it should not happen so give it a warning.
470 */
471 if (!rtwpci->low_power && !list_empty(&txwd->list))
472 rtw89_warn(rtwdev, "queue %d txwd %d is not idle\n",
473 txch, seq);
474 }
475
476 skb_queue_walk_safe(&txwd->queue, skb, tmp) {
477 skb_unlink(skb, &txwd->queue);
478
479 tx_data = RTW89_PCI_TX_SKB_CB(skb);
480 dma_unmap_single(&rtwpci->pdev->dev, tx_data->dma, skb->len,
481 DMA_TO_DEVICE);
482
483 rtw89_pci_tx_status(rtwdev, tx_ring, skb, tx_status);
484 }
485
486 if (list_empty(&txwd->list))
487 rtw89_pci_enqueue_txwd(tx_ring, txwd);
488 }
489
rtw89_pci_release_rpp(struct rtw89_dev * rtwdev,struct rtw89_pci_rpp_fmt * rpp)490 static void rtw89_pci_release_rpp(struct rtw89_dev *rtwdev,
491 struct rtw89_pci_rpp_fmt *rpp)
492 {
493 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
494 struct rtw89_pci_tx_ring *tx_ring;
495 struct rtw89_pci_tx_wd_ring *wd_ring;
496 struct rtw89_pci_tx_wd *txwd;
497 u16 seq;
498 u8 qsel, tx_status, txch;
499
500 seq = le32_get_bits(rpp->dword, RTW89_PCI_RPP_SEQ);
501 qsel = le32_get_bits(rpp->dword, RTW89_PCI_RPP_QSEL);
502 tx_status = le32_get_bits(rpp->dword, RTW89_PCI_RPP_TX_STATUS);
503 txch = rtw89_core_get_ch_dma(rtwdev, qsel);
504
505 if (txch == RTW89_TXCH_CH12) {
506 rtw89_warn(rtwdev, "should no fwcmd release report\n");
507 return;
508 }
509
510 tx_ring = &rtwpci->tx_rings[txch];
511 wd_ring = &tx_ring->wd_ring;
512 txwd = &wd_ring->pages[seq];
513
514 rtw89_pci_release_txwd_skb(rtwdev, tx_ring, txwd, seq, tx_status);
515 }
516
rtw89_pci_release_pending_txwd_skb(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring)517 static void rtw89_pci_release_pending_txwd_skb(struct rtw89_dev *rtwdev,
518 struct rtw89_pci_tx_ring *tx_ring)
519 {
520 struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
521 struct rtw89_pci_tx_wd *txwd;
522 int i;
523
524 for (i = 0; i < wd_ring->page_num; i++) {
525 txwd = &wd_ring->pages[i];
526
527 if (!list_empty(&txwd->list))
528 continue;
529
530 rtw89_pci_release_txwd_skb(rtwdev, tx_ring, txwd, i, RTW89_TX_MACID_DROP);
531 }
532 }
533
rtw89_pci_release_tx_skbs(struct rtw89_dev * rtwdev,struct rtw89_pci_rx_ring * rx_ring,u32 max_cnt)534 static u32 rtw89_pci_release_tx_skbs(struct rtw89_dev *rtwdev,
535 struct rtw89_pci_rx_ring *rx_ring,
536 u32 max_cnt)
537 {
538 struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring;
539 struct rtw89_pci_rx_info *rx_info;
540 struct rtw89_pci_rpp_fmt *rpp;
541 struct rtw89_rx_desc_info desc_info = {};
542 struct sk_buff *skb;
543 u32 cnt = 0;
544 u32 rpp_size = sizeof(struct rtw89_pci_rpp_fmt);
545 u32 rxinfo_size = sizeof(struct rtw89_pci_rxbd_info);
546 u32 skb_idx;
547 u32 offset;
548 int ret;
549
550 skb_idx = rtw89_pci_get_rx_skb_idx(rtwdev, bd_ring);
551 skb = rx_ring->buf[skb_idx];
552 rtw89_pci_sync_skb_for_cpu(rtwdev, skb);
553
554 ret = rtw89_pci_rxbd_info_update(rtwdev, skb);
555 if (ret) {
556 rtw89_err(rtwdev, "failed to update %d RXBD info: %d\n",
557 bd_ring->wp, ret);
558 goto err_sync_device;
559 }
560
561 rx_info = RTW89_PCI_RX_SKB_CB(skb);
562 if (!rx_info->fs || !rx_info->ls) {
563 rtw89_err(rtwdev, "cannot process RP frame not set FS/LS\n");
564 return cnt;
565 }
566
567 rtw89_chip_query_rxdesc(rtwdev, &desc_info, skb->data, rxinfo_size);
568
569 /* first segment has RX desc */
570 offset = desc_info.offset + desc_info.rxd_len;
571 for (; offset + rpp_size <= rx_info->len; offset += rpp_size) {
572 rpp = (struct rtw89_pci_rpp_fmt *)(skb->data + offset);
573 rtw89_pci_release_rpp(rtwdev, rpp);
574 }
575
576 rtw89_pci_sync_skb_for_device(rtwdev, skb);
577 rtw89_pci_rxbd_increase(rx_ring, 1);
578 cnt++;
579
580 return cnt;
581
582 err_sync_device:
583 rtw89_pci_sync_skb_for_device(rtwdev, skb);
584 return 0;
585 }
586
rtw89_pci_release_tx(struct rtw89_dev * rtwdev,struct rtw89_pci_rx_ring * rx_ring,u32 cnt)587 static void rtw89_pci_release_tx(struct rtw89_dev *rtwdev,
588 struct rtw89_pci_rx_ring *rx_ring,
589 u32 cnt)
590 {
591 struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring;
592 u32 release_cnt;
593
594 while (cnt) {
595 release_cnt = rtw89_pci_release_tx_skbs(rtwdev, rx_ring, cnt);
596 if (!release_cnt) {
597 rtw89_err(rtwdev, "failed to release TX skbs\n");
598
599 /* skip the rest RXBD bufs */
600 rtw89_pci_rxbd_increase(rx_ring, cnt);
601 break;
602 }
603
604 cnt -= release_cnt;
605 }
606
607 rtw89_write16(rtwdev, bd_ring->addr.idx, bd_ring->wp);
608 }
609
rtw89_pci_poll_rpq_dma(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci,int budget)610 static int rtw89_pci_poll_rpq_dma(struct rtw89_dev *rtwdev,
611 struct rtw89_pci *rtwpci, int budget)
612 {
613 struct rtw89_pci_rx_ring *rx_ring;
614 u32 cnt;
615 int work_done;
616
617 rx_ring = &rtwpci->rx_rings[RTW89_RXCH_RPQ];
618
619 spin_lock_bh(&rtwpci->trx_lock);
620
621 cnt = rtw89_pci_rxbd_recalc(rtwdev, rx_ring);
622 if (cnt == 0)
623 goto out_unlock;
624
625 rtw89_pci_release_tx(rtwdev, rx_ring, cnt);
626
627 out_unlock:
628 spin_unlock_bh(&rtwpci->trx_lock);
629
630 /* always release all RPQ */
631 work_done = min_t(int, cnt, budget);
632 rtwdev->napi_budget_countdown -= work_done;
633
634 return work_done;
635 }
636
rtw89_pci_isr_rxd_unavail(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci)637 static void rtw89_pci_isr_rxd_unavail(struct rtw89_dev *rtwdev,
638 struct rtw89_pci *rtwpci)
639 {
640 struct rtw89_pci_rx_ring *rx_ring;
641 struct rtw89_pci_dma_ring *bd_ring;
642 u32 reg_idx;
643 u16 hw_idx, hw_idx_next, host_idx;
644 int i;
645
646 for (i = 0; i < RTW89_RXCH_NUM; i++) {
647 rx_ring = &rtwpci->rx_rings[i];
648 bd_ring = &rx_ring->bd_ring;
649
650 reg_idx = rtw89_read32(rtwdev, bd_ring->addr.idx);
651 hw_idx = FIELD_GET(TXBD_HW_IDX_MASK, reg_idx);
652 host_idx = FIELD_GET(TXBD_HOST_IDX_MASK, reg_idx);
653 hw_idx_next = (hw_idx + 1) % bd_ring->len;
654
655 if (hw_idx_next == host_idx)
656 rtw89_debug(rtwdev, RTW89_DBG_UNEXP, "%d RXD unavailable\n", i);
657
658 rtw89_debug(rtwdev, RTW89_DBG_TXRX,
659 "%d RXD unavailable, idx=0x%08x, len=%d\n",
660 i, reg_idx, bd_ring->len);
661 }
662 }
663
rtw89_pci_recognize_intrs(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci,struct rtw89_pci_isrs * isrs)664 void rtw89_pci_recognize_intrs(struct rtw89_dev *rtwdev,
665 struct rtw89_pci *rtwpci,
666 struct rtw89_pci_isrs *isrs)
667 {
668 isrs->halt_c2h_isrs = rtw89_read32(rtwdev, R_AX_HISR0) & rtwpci->halt_c2h_intrs;
669 isrs->isrs[0] = rtw89_read32(rtwdev, R_AX_PCIE_HISR00) & rtwpci->intrs[0];
670 isrs->isrs[1] = rtw89_read32(rtwdev, R_AX_PCIE_HISR10) & rtwpci->intrs[1];
671
672 rtw89_write32(rtwdev, R_AX_HISR0, isrs->halt_c2h_isrs);
673 rtw89_write32(rtwdev, R_AX_PCIE_HISR00, isrs->isrs[0]);
674 rtw89_write32(rtwdev, R_AX_PCIE_HISR10, isrs->isrs[1]);
675 }
676 EXPORT_SYMBOL(rtw89_pci_recognize_intrs);
677
rtw89_pci_recognize_intrs_v1(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci,struct rtw89_pci_isrs * isrs)678 void rtw89_pci_recognize_intrs_v1(struct rtw89_dev *rtwdev,
679 struct rtw89_pci *rtwpci,
680 struct rtw89_pci_isrs *isrs)
681 {
682 isrs->ind_isrs = rtw89_read32(rtwdev, R_AX_PCIE_HISR00_V1) & rtwpci->ind_intrs;
683 isrs->halt_c2h_isrs = isrs->ind_isrs & B_AX_HS0ISR_IND_INT_EN ?
684 rtw89_read32(rtwdev, R_AX_HISR0) & rtwpci->halt_c2h_intrs : 0;
685 isrs->isrs[0] = isrs->ind_isrs & B_AX_HCI_AXIDMA_INT_EN ?
686 rtw89_read32(rtwdev, R_AX_HAXI_HISR00) & rtwpci->intrs[0] : 0;
687 isrs->isrs[1] = isrs->ind_isrs & B_AX_HS1ISR_IND_INT_EN ?
688 rtw89_read32(rtwdev, R_AX_HISR1) & rtwpci->intrs[1] : 0;
689
690 if (isrs->halt_c2h_isrs)
691 rtw89_write32(rtwdev, R_AX_HISR0, isrs->halt_c2h_isrs);
692 if (isrs->isrs[0])
693 rtw89_write32(rtwdev, R_AX_HAXI_HISR00, isrs->isrs[0]);
694 if (isrs->isrs[1])
695 rtw89_write32(rtwdev, R_AX_HISR1, isrs->isrs[1]);
696 }
697 EXPORT_SYMBOL(rtw89_pci_recognize_intrs_v1);
698
rtw89_pci_recognize_intrs_v2(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci,struct rtw89_pci_isrs * isrs)699 void rtw89_pci_recognize_intrs_v2(struct rtw89_dev *rtwdev,
700 struct rtw89_pci *rtwpci,
701 struct rtw89_pci_isrs *isrs)
702 {
703 isrs->ind_isrs = rtw89_read32(rtwdev, R_BE_PCIE_HISR) & rtwpci->ind_intrs;
704 isrs->halt_c2h_isrs = isrs->ind_isrs & B_BE_HS0ISR_IND_INT ?
705 rtw89_read32(rtwdev, R_BE_HISR0) & rtwpci->halt_c2h_intrs : 0;
706 isrs->isrs[0] = isrs->ind_isrs & B_BE_HCI_AXIDMA_INT ?
707 rtw89_read32(rtwdev, R_BE_HAXI_HISR00) & rtwpci->intrs[0] : 0;
708 isrs->isrs[1] = rtw89_read32(rtwdev, R_BE_PCIE_DMA_ISR);
709
710 if (isrs->halt_c2h_isrs)
711 rtw89_write32(rtwdev, R_BE_HISR0, isrs->halt_c2h_isrs);
712 if (isrs->isrs[0])
713 rtw89_write32(rtwdev, R_BE_HAXI_HISR00, isrs->isrs[0]);
714 if (isrs->isrs[1])
715 rtw89_write32(rtwdev, R_BE_PCIE_DMA_ISR, isrs->isrs[1]);
716 rtw89_write32(rtwdev, R_BE_PCIE_HISR, isrs->ind_isrs);
717 }
718 EXPORT_SYMBOL(rtw89_pci_recognize_intrs_v2);
719
rtw89_pci_enable_intr(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci)720 void rtw89_pci_enable_intr(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
721 {
722 rtw89_write32(rtwdev, R_AX_HIMR0, rtwpci->halt_c2h_intrs);
723 rtw89_write32(rtwdev, R_AX_PCIE_HIMR00, rtwpci->intrs[0]);
724 rtw89_write32(rtwdev, R_AX_PCIE_HIMR10, rtwpci->intrs[1]);
725 }
726 EXPORT_SYMBOL(rtw89_pci_enable_intr);
727
rtw89_pci_disable_intr(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci)728 void rtw89_pci_disable_intr(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
729 {
730 rtw89_write32(rtwdev, R_AX_HIMR0, 0);
731 rtw89_write32(rtwdev, R_AX_PCIE_HIMR00, 0);
732 rtw89_write32(rtwdev, R_AX_PCIE_HIMR10, 0);
733 }
734 EXPORT_SYMBOL(rtw89_pci_disable_intr);
735
rtw89_pci_enable_intr_v1(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci)736 void rtw89_pci_enable_intr_v1(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
737 {
738 rtw89_write32(rtwdev, R_AX_PCIE_HIMR00_V1, rtwpci->ind_intrs);
739 rtw89_write32(rtwdev, R_AX_HIMR0, rtwpci->halt_c2h_intrs);
740 rtw89_write32(rtwdev, R_AX_HAXI_HIMR00, rtwpci->intrs[0]);
741 rtw89_write32(rtwdev, R_AX_HIMR1, rtwpci->intrs[1]);
742 }
743 EXPORT_SYMBOL(rtw89_pci_enable_intr_v1);
744
rtw89_pci_disable_intr_v1(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci)745 void rtw89_pci_disable_intr_v1(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
746 {
747 rtw89_write32(rtwdev, R_AX_PCIE_HIMR00_V1, 0);
748 }
749 EXPORT_SYMBOL(rtw89_pci_disable_intr_v1);
750
rtw89_pci_enable_intr_v2(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci)751 void rtw89_pci_enable_intr_v2(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
752 {
753 rtw89_write32(rtwdev, R_BE_HIMR0, rtwpci->halt_c2h_intrs);
754 rtw89_write32(rtwdev, R_BE_HAXI_HIMR00, rtwpci->intrs[0]);
755 rtw89_write32(rtwdev, R_BE_PCIE_DMA_IMR_0_V1, rtwpci->intrs[1]);
756 rtw89_write32(rtwdev, R_BE_PCIE_HIMR0, rtwpci->ind_intrs);
757 }
758 EXPORT_SYMBOL(rtw89_pci_enable_intr_v2);
759
rtw89_pci_disable_intr_v2(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci)760 void rtw89_pci_disable_intr_v2(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci)
761 {
762 rtw89_write32(rtwdev, R_BE_PCIE_HIMR0, 0);
763 rtw89_write32(rtwdev, R_BE_PCIE_DMA_IMR_0_V1, 0);
764 }
765 EXPORT_SYMBOL(rtw89_pci_disable_intr_v2);
766
rtw89_pci_ops_recovery_start(struct rtw89_dev * rtwdev)767 static void rtw89_pci_ops_recovery_start(struct rtw89_dev *rtwdev)
768 {
769 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
770 unsigned long flags;
771
772 spin_lock_irqsave(&rtwpci->irq_lock, flags);
773 rtw89_chip_disable_intr(rtwdev, rtwpci);
774 rtw89_chip_config_intr_mask(rtwdev, RTW89_PCI_INTR_MASK_RECOVERY_START);
775 rtw89_chip_enable_intr(rtwdev, rtwpci);
776 spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
777 }
778
rtw89_pci_ops_recovery_complete(struct rtw89_dev * rtwdev)779 static void rtw89_pci_ops_recovery_complete(struct rtw89_dev *rtwdev)
780 {
781 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
782 unsigned long flags;
783
784 spin_lock_irqsave(&rtwpci->irq_lock, flags);
785 rtw89_chip_disable_intr(rtwdev, rtwpci);
786 rtw89_chip_config_intr_mask(rtwdev, RTW89_PCI_INTR_MASK_RECOVERY_COMPLETE);
787 rtw89_chip_enable_intr(rtwdev, rtwpci);
788 spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
789 }
790
rtw89_pci_low_power_interrupt_handler(struct rtw89_dev * rtwdev)791 static void rtw89_pci_low_power_interrupt_handler(struct rtw89_dev *rtwdev)
792 {
793 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
794 int budget = NAPI_POLL_WEIGHT;
795
796 /* To prevent RXQ get stuck due to run out of budget. */
797 rtwdev->napi_budget_countdown = budget;
798
799 rtw89_pci_poll_rpq_dma(rtwdev, rtwpci, budget);
800 rtw89_pci_poll_rxq_dma(rtwdev, rtwpci, budget);
801 }
802
rtw89_pci_interrupt_threadfn(int irq,void * dev)803 static irqreturn_t rtw89_pci_interrupt_threadfn(int irq, void *dev)
804 {
805 struct rtw89_dev *rtwdev = dev;
806 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
807 const struct rtw89_pci_info *info = rtwdev->pci_info;
808 const struct rtw89_pci_gen_def *gen_def = info->gen_def;
809 struct rtw89_pci_isrs isrs;
810 unsigned long flags;
811
812 spin_lock_irqsave(&rtwpci->irq_lock, flags);
813 rtw89_chip_recognize_intrs(rtwdev, rtwpci, &isrs);
814 spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
815
816 if (unlikely(isrs.isrs[0] & gen_def->isr_rdu))
817 rtw89_pci_isr_rxd_unavail(rtwdev, rtwpci);
818
819 if (unlikely(isrs.halt_c2h_isrs & gen_def->isr_halt_c2h))
820 rtw89_ser_notify(rtwdev, rtw89_mac_get_err_status(rtwdev));
821
822 if (unlikely(isrs.halt_c2h_isrs & gen_def->isr_wdt_timeout))
823 rtw89_ser_notify(rtwdev, MAC_AX_ERR_L2_ERR_WDT_TIMEOUT_INT);
824
825 if (unlikely(rtwpci->under_recovery))
826 goto enable_intr;
827
828 if (unlikely(rtwpci->low_power)) {
829 rtw89_pci_low_power_interrupt_handler(rtwdev);
830 goto enable_intr;
831 }
832
833 if (likely(rtwpci->running)) {
834 local_bh_disable();
835 napi_schedule(&rtwdev->napi);
836 local_bh_enable();
837 }
838
839 return IRQ_HANDLED;
840
841 enable_intr:
842 spin_lock_irqsave(&rtwpci->irq_lock, flags);
843 if (likely(rtwpci->running))
844 rtw89_chip_enable_intr(rtwdev, rtwpci);
845 spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
846 return IRQ_HANDLED;
847 }
848
rtw89_pci_interrupt_handler(int irq,void * dev)849 static irqreturn_t rtw89_pci_interrupt_handler(int irq, void *dev)
850 {
851 struct rtw89_dev *rtwdev = dev;
852 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
853 unsigned long flags;
854 irqreturn_t irqret = IRQ_WAKE_THREAD;
855
856 spin_lock_irqsave(&rtwpci->irq_lock, flags);
857
858 /* If interrupt event is on the road, it is still trigger interrupt
859 * even we have done pci_stop() to turn off IMR.
860 */
861 if (unlikely(!rtwpci->running)) {
862 irqret = IRQ_HANDLED;
863 goto exit;
864 }
865
866 rtw89_chip_disable_intr(rtwdev, rtwpci);
867 exit:
868 spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
869
870 return irqret;
871 }
872
873 #define DEF_TXCHADDRS_TYPE2(gen, ch_idx, txch, v...) \
874 [RTW89_TXCH_##ch_idx] = { \
875 .num = R_##gen##_##txch##_TXBD_NUM ##v, \
876 .idx = R_##gen##_##txch##_TXBD_IDX ##v, \
877 .bdram = 0, \
878 .desa_l = R_##gen##_##txch##_TXBD_DESA_L ##v, \
879 .desa_h = R_##gen##_##txch##_TXBD_DESA_H ##v, \
880 }
881
882 #define DEF_TXCHADDRS_TYPE1(info, txch, v...) \
883 [RTW89_TXCH_##txch] = { \
884 .num = R_AX_##txch##_TXBD_NUM ##v, \
885 .idx = R_AX_##txch##_TXBD_IDX ##v, \
886 .bdram = R_AX_##txch##_BDRAM_CTRL ##v, \
887 .desa_l = R_AX_##txch##_TXBD_DESA_L ##v, \
888 .desa_h = R_AX_##txch##_TXBD_DESA_H ##v, \
889 }
890
891 #define DEF_TXCHADDRS(info, txch, v...) \
892 [RTW89_TXCH_##txch] = { \
893 .num = R_AX_##txch##_TXBD_NUM, \
894 .idx = R_AX_##txch##_TXBD_IDX, \
895 .bdram = R_AX_##txch##_BDRAM_CTRL ##v, \
896 .desa_l = R_AX_##txch##_TXBD_DESA_L ##v, \
897 .desa_h = R_AX_##txch##_TXBD_DESA_H ##v, \
898 }
899
900 #define DEF_RXCHADDRS(gen, ch_idx, rxch, v...) \
901 [RTW89_RXCH_##ch_idx] = { \
902 .num = R_##gen##_##rxch##_RXBD_NUM ##v, \
903 .idx = R_##gen##_##rxch##_RXBD_IDX ##v, \
904 .desa_l = R_##gen##_##rxch##_RXBD_DESA_L ##v, \
905 .desa_h = R_##gen##_##rxch##_RXBD_DESA_H ##v, \
906 }
907
908 const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set = {
909 .tx = {
910 DEF_TXCHADDRS(info, ACH0),
911 DEF_TXCHADDRS(info, ACH1),
912 DEF_TXCHADDRS(info, ACH2),
913 DEF_TXCHADDRS(info, ACH3),
914 DEF_TXCHADDRS(info, ACH4),
915 DEF_TXCHADDRS(info, ACH5),
916 DEF_TXCHADDRS(info, ACH6),
917 DEF_TXCHADDRS(info, ACH7),
918 DEF_TXCHADDRS(info, CH8),
919 DEF_TXCHADDRS(info, CH9),
920 DEF_TXCHADDRS_TYPE1(info, CH10),
921 DEF_TXCHADDRS_TYPE1(info, CH11),
922 DEF_TXCHADDRS(info, CH12),
923 },
924 .rx = {
925 DEF_RXCHADDRS(AX, RXQ, RXQ),
926 DEF_RXCHADDRS(AX, RPQ, RPQ),
927 },
928 };
929 EXPORT_SYMBOL(rtw89_pci_ch_dma_addr_set);
930
931 const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set_v1 = {
932 .tx = {
933 DEF_TXCHADDRS(info, ACH0, _V1),
934 DEF_TXCHADDRS(info, ACH1, _V1),
935 DEF_TXCHADDRS(info, ACH2, _V1),
936 DEF_TXCHADDRS(info, ACH3, _V1),
937 DEF_TXCHADDRS(info, ACH4, _V1),
938 DEF_TXCHADDRS(info, ACH5, _V1),
939 DEF_TXCHADDRS(info, ACH6, _V1),
940 DEF_TXCHADDRS(info, ACH7, _V1),
941 DEF_TXCHADDRS(info, CH8, _V1),
942 DEF_TXCHADDRS(info, CH9, _V1),
943 DEF_TXCHADDRS_TYPE1(info, CH10, _V1),
944 DEF_TXCHADDRS_TYPE1(info, CH11, _V1),
945 DEF_TXCHADDRS(info, CH12, _V1),
946 },
947 .rx = {
948 DEF_RXCHADDRS(AX, RXQ, RXQ, _V1),
949 DEF_RXCHADDRS(AX, RPQ, RPQ, _V1),
950 },
951 };
952 EXPORT_SYMBOL(rtw89_pci_ch_dma_addr_set_v1);
953
954 const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set_be = {
955 .tx = {
956 DEF_TXCHADDRS_TYPE2(BE, ACH0, CH0, _V1),
957 DEF_TXCHADDRS_TYPE2(BE, ACH1, CH1, _V1),
958 DEF_TXCHADDRS_TYPE2(BE, ACH2, CH2, _V1),
959 DEF_TXCHADDRS_TYPE2(BE, ACH3, CH3, _V1),
960 DEF_TXCHADDRS_TYPE2(BE, ACH4, CH4, _V1),
961 DEF_TXCHADDRS_TYPE2(BE, ACH5, CH5, _V1),
962 DEF_TXCHADDRS_TYPE2(BE, ACH6, CH6, _V1),
963 DEF_TXCHADDRS_TYPE2(BE, ACH7, CH7, _V1),
964 DEF_TXCHADDRS_TYPE2(BE, CH8, CH8, _V1),
965 DEF_TXCHADDRS_TYPE2(BE, CH9, CH9, _V1),
966 DEF_TXCHADDRS_TYPE2(BE, CH10, CH10, _V1),
967 DEF_TXCHADDRS_TYPE2(BE, CH11, CH11, _V1),
968 DEF_TXCHADDRS_TYPE2(BE, CH12, CH12, _V1),
969 },
970 .rx = {
971 DEF_RXCHADDRS(BE, RXQ, RXQ0, _V1),
972 DEF_RXCHADDRS(BE, RPQ, RPQ0, _V1),
973 },
974 };
975 EXPORT_SYMBOL(rtw89_pci_ch_dma_addr_set_be);
976
977 #undef DEF_TXCHADDRS_TYPE1
978 #undef DEF_TXCHADDRS
979 #undef DEF_RXCHADDRS
980
rtw89_pci_get_txch_addrs(struct rtw89_dev * rtwdev,enum rtw89_tx_channel txch,const struct rtw89_pci_ch_dma_addr ** addr)981 static int rtw89_pci_get_txch_addrs(struct rtw89_dev *rtwdev,
982 enum rtw89_tx_channel txch,
983 const struct rtw89_pci_ch_dma_addr **addr)
984 {
985 const struct rtw89_pci_info *info = rtwdev->pci_info;
986
987 if (txch >= RTW89_TXCH_NUM)
988 return -EINVAL;
989
990 *addr = &info->dma_addr_set->tx[txch];
991
992 return 0;
993 }
994
rtw89_pci_get_rxch_addrs(struct rtw89_dev * rtwdev,enum rtw89_rx_channel rxch,const struct rtw89_pci_ch_dma_addr ** addr)995 static int rtw89_pci_get_rxch_addrs(struct rtw89_dev *rtwdev,
996 enum rtw89_rx_channel rxch,
997 const struct rtw89_pci_ch_dma_addr **addr)
998 {
999 const struct rtw89_pci_info *info = rtwdev->pci_info;
1000
1001 if (rxch >= RTW89_RXCH_NUM)
1002 return -EINVAL;
1003
1004 *addr = &info->dma_addr_set->rx[rxch];
1005
1006 return 0;
1007 }
1008
rtw89_pci_get_avail_txbd_num(struct rtw89_pci_tx_ring * ring)1009 static u32 rtw89_pci_get_avail_txbd_num(struct rtw89_pci_tx_ring *ring)
1010 {
1011 struct rtw89_pci_dma_ring *bd_ring = &ring->bd_ring;
1012
1013 /* reserved 1 desc check ring is full or not */
1014 if (bd_ring->rp > bd_ring->wp)
1015 return bd_ring->rp - bd_ring->wp - 1;
1016
1017 return bd_ring->len - (bd_ring->wp - bd_ring->rp) - 1;
1018 }
1019
1020 static
__rtw89_pci_check_and_reclaim_tx_fwcmd_resource(struct rtw89_dev * rtwdev)1021 u32 __rtw89_pci_check_and_reclaim_tx_fwcmd_resource(struct rtw89_dev *rtwdev)
1022 {
1023 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1024 struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[RTW89_TXCH_CH12];
1025 u32 cnt;
1026
1027 spin_lock_bh(&rtwpci->trx_lock);
1028 rtw89_pci_reclaim_tx_fwcmd(rtwdev, rtwpci);
1029 cnt = rtw89_pci_get_avail_txbd_num(tx_ring);
1030 spin_unlock_bh(&rtwpci->trx_lock);
1031
1032 return cnt;
1033 }
1034
1035 static
__rtw89_pci_check_and_reclaim_tx_resource_noio(struct rtw89_dev * rtwdev,u8 txch)1036 u32 __rtw89_pci_check_and_reclaim_tx_resource_noio(struct rtw89_dev *rtwdev,
1037 u8 txch)
1038 {
1039 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1040 struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[txch];
1041 struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
1042 u32 cnt;
1043
1044 spin_lock_bh(&rtwpci->trx_lock);
1045 cnt = rtw89_pci_get_avail_txbd_num(tx_ring);
1046 cnt = min(cnt, wd_ring->curr_num);
1047 spin_unlock_bh(&rtwpci->trx_lock);
1048
1049 return cnt;
1050 }
1051
__rtw89_pci_check_and_reclaim_tx_resource(struct rtw89_dev * rtwdev,u8 txch)1052 static u32 __rtw89_pci_check_and_reclaim_tx_resource(struct rtw89_dev *rtwdev,
1053 u8 txch)
1054 {
1055 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1056 struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[txch];
1057 struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
1058 const struct rtw89_chip_info *chip = rtwdev->chip;
1059 u32 bd_cnt, wd_cnt, min_cnt = 0;
1060 struct rtw89_pci_rx_ring *rx_ring;
1061 enum rtw89_debug_mask debug_mask;
1062 u32 cnt;
1063
1064 rx_ring = &rtwpci->rx_rings[RTW89_RXCH_RPQ];
1065
1066 spin_lock_bh(&rtwpci->trx_lock);
1067 bd_cnt = rtw89_pci_get_avail_txbd_num(tx_ring);
1068 wd_cnt = wd_ring->curr_num;
1069
1070 if (wd_cnt == 0 || bd_cnt == 0) {
1071 cnt = rtw89_pci_rxbd_recalc(rtwdev, rx_ring);
1072 if (cnt)
1073 rtw89_pci_release_tx(rtwdev, rx_ring, cnt);
1074 else if (wd_cnt == 0)
1075 goto out_unlock;
1076
1077 bd_cnt = rtw89_pci_get_avail_txbd_num(tx_ring);
1078 if (bd_cnt == 0)
1079 rtw89_pci_reclaim_txbd(rtwdev, tx_ring);
1080 }
1081
1082 bd_cnt = rtw89_pci_get_avail_txbd_num(tx_ring);
1083 wd_cnt = wd_ring->curr_num;
1084 min_cnt = min(bd_cnt, wd_cnt);
1085 if (min_cnt == 0) {
1086 /* This message can be frequently shown in low power mode or
1087 * high traffic with small FIFO chips, and we have recognized it as normal
1088 * behavior, so print with mask RTW89_DBG_TXRX in these situations.
1089 */
1090 if (rtwpci->low_power || chip->small_fifo_size)
1091 debug_mask = RTW89_DBG_TXRX;
1092 else
1093 debug_mask = RTW89_DBG_UNEXP;
1094
1095 rtw89_debug(rtwdev, debug_mask,
1096 "still no tx resource after reclaim: wd_cnt=%d bd_cnt=%d\n",
1097 wd_cnt, bd_cnt);
1098 }
1099
1100 out_unlock:
1101 spin_unlock_bh(&rtwpci->trx_lock);
1102
1103 return min_cnt;
1104 }
1105
rtw89_pci_check_and_reclaim_tx_resource(struct rtw89_dev * rtwdev,u8 txch)1106 static u32 rtw89_pci_check_and_reclaim_tx_resource(struct rtw89_dev *rtwdev,
1107 u8 txch)
1108 {
1109 if (rtwdev->hci.paused)
1110 return __rtw89_pci_check_and_reclaim_tx_resource_noio(rtwdev, txch);
1111
1112 if (txch == RTW89_TXCH_CH12)
1113 return __rtw89_pci_check_and_reclaim_tx_fwcmd_resource(rtwdev);
1114
1115 return __rtw89_pci_check_and_reclaim_tx_resource(rtwdev, txch);
1116 }
1117
__rtw89_pci_tx_kick_off(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring)1118 static void __rtw89_pci_tx_kick_off(struct rtw89_dev *rtwdev, struct rtw89_pci_tx_ring *tx_ring)
1119 {
1120 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1121 struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring;
1122 u32 host_idx, addr;
1123
1124 spin_lock_bh(&rtwpci->trx_lock);
1125
1126 addr = bd_ring->addr.idx;
1127 host_idx = bd_ring->wp;
1128 rtw89_write16(rtwdev, addr, host_idx);
1129
1130 spin_unlock_bh(&rtwpci->trx_lock);
1131 }
1132
rtw89_pci_tx_bd_ring_update(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring,int n_txbd)1133 static void rtw89_pci_tx_bd_ring_update(struct rtw89_dev *rtwdev, struct rtw89_pci_tx_ring *tx_ring,
1134 int n_txbd)
1135 {
1136 struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring;
1137 u32 host_idx, len;
1138
1139 len = bd_ring->len;
1140 host_idx = bd_ring->wp + n_txbd;
1141 host_idx = host_idx < len ? host_idx : host_idx - len;
1142
1143 bd_ring->wp = host_idx;
1144 }
1145
rtw89_pci_ops_tx_kick_off(struct rtw89_dev * rtwdev,u8 txch)1146 static void rtw89_pci_ops_tx_kick_off(struct rtw89_dev *rtwdev, u8 txch)
1147 {
1148 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1149 struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[txch];
1150
1151 if (rtwdev->hci.paused) {
1152 set_bit(txch, rtwpci->kick_map);
1153 return;
1154 }
1155
1156 __rtw89_pci_tx_kick_off(rtwdev, tx_ring);
1157 }
1158
rtw89_pci_tx_kick_off_pending(struct rtw89_dev * rtwdev)1159 static void rtw89_pci_tx_kick_off_pending(struct rtw89_dev *rtwdev)
1160 {
1161 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1162 struct rtw89_pci_tx_ring *tx_ring;
1163 int txch;
1164
1165 for (txch = 0; txch < RTW89_TXCH_NUM; txch++) {
1166 if (!test_and_clear_bit(txch, rtwpci->kick_map))
1167 continue;
1168
1169 tx_ring = &rtwpci->tx_rings[txch];
1170 __rtw89_pci_tx_kick_off(rtwdev, tx_ring);
1171 }
1172 }
1173
__pci_flush_txch(struct rtw89_dev * rtwdev,u8 txch,bool drop)1174 static void __pci_flush_txch(struct rtw89_dev *rtwdev, u8 txch, bool drop)
1175 {
1176 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1177 struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[txch];
1178 struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring;
1179 u32 cur_idx, cur_rp;
1180 u8 i;
1181
1182 /* Because the time taked by the I/O is a bit dynamic, it's hard to
1183 * define a reasonable fixed total timeout to use read_poll_timeout*
1184 * helper. Instead, we can ensure a reasonable polling times, so we
1185 * just use for loop with udelay here.
1186 */
1187 for (i = 0; i < 60; i++) {
1188 cur_idx = rtw89_read32(rtwdev, bd_ring->addr.idx);
1189 cur_rp = FIELD_GET(TXBD_HW_IDX_MASK, cur_idx);
1190 if (cur_rp == bd_ring->wp)
1191 return;
1192
1193 udelay(1);
1194 }
1195
1196 if (!drop)
1197 rtw89_info(rtwdev, "timed out to flush pci txch: %d\n", txch);
1198 }
1199
__rtw89_pci_ops_flush_txchs(struct rtw89_dev * rtwdev,u32 txchs,bool drop)1200 static void __rtw89_pci_ops_flush_txchs(struct rtw89_dev *rtwdev, u32 txchs,
1201 bool drop)
1202 {
1203 const struct rtw89_pci_info *info = rtwdev->pci_info;
1204 u8 i;
1205
1206 for (i = 0; i < RTW89_TXCH_NUM; i++) {
1207 /* It may be unnecessary to flush FWCMD queue. */
1208 if (i == RTW89_TXCH_CH12)
1209 continue;
1210 if (info->tx_dma_ch_mask & BIT(i))
1211 continue;
1212
1213 if (txchs & BIT(i))
1214 __pci_flush_txch(rtwdev, i, drop);
1215 }
1216 }
1217
rtw89_pci_ops_flush_queues(struct rtw89_dev * rtwdev,u32 queues,bool drop)1218 static void rtw89_pci_ops_flush_queues(struct rtw89_dev *rtwdev, u32 queues,
1219 bool drop)
1220 {
1221 __rtw89_pci_ops_flush_txchs(rtwdev, BIT(RTW89_TXCH_NUM) - 1, drop);
1222 }
1223
rtw89_pci_fill_txaddr_info(struct rtw89_dev * rtwdev,void * txaddr_info_addr,u32 total_len,dma_addr_t dma,u8 * add_info_nr)1224 u32 rtw89_pci_fill_txaddr_info(struct rtw89_dev *rtwdev,
1225 void *txaddr_info_addr, u32 total_len,
1226 dma_addr_t dma, u8 *add_info_nr)
1227 {
1228 struct rtw89_pci_tx_addr_info_32 *txaddr_info = txaddr_info_addr;
1229
1230 txaddr_info->length = cpu_to_le16(total_len);
1231 txaddr_info->option = cpu_to_le16(RTW89_PCI_ADDR_MSDU_LS |
1232 RTW89_PCI_ADDR_NUM(1));
1233 txaddr_info->dma = cpu_to_le32(dma);
1234
1235 *add_info_nr = 1;
1236
1237 return sizeof(*txaddr_info);
1238 }
1239 EXPORT_SYMBOL(rtw89_pci_fill_txaddr_info);
1240
rtw89_pci_fill_txaddr_info_v1(struct rtw89_dev * rtwdev,void * txaddr_info_addr,u32 total_len,dma_addr_t dma,u8 * add_info_nr)1241 u32 rtw89_pci_fill_txaddr_info_v1(struct rtw89_dev *rtwdev,
1242 void *txaddr_info_addr, u32 total_len,
1243 dma_addr_t dma, u8 *add_info_nr)
1244 {
1245 struct rtw89_pci_tx_addr_info_32_v1 *txaddr_info = txaddr_info_addr;
1246 u32 remain = total_len;
1247 u32 len;
1248 u16 length_option;
1249 int n;
1250
1251 for (n = 0; n < RTW89_TXADDR_INFO_NR_V1 && remain; n++) {
1252 len = remain >= TXADDR_INFO_LENTHG_V1_MAX ?
1253 TXADDR_INFO_LENTHG_V1_MAX : remain;
1254 remain -= len;
1255
1256 length_option = FIELD_PREP(B_PCIADDR_LEN_V1_MASK, len) |
1257 FIELD_PREP(B_PCIADDR_HIGH_SEL_V1_MASK, 0) |
1258 FIELD_PREP(B_PCIADDR_LS_V1_MASK, remain == 0);
1259 txaddr_info->length_opt = cpu_to_le16(length_option);
1260 txaddr_info->dma_low_lsb = cpu_to_le16(FIELD_GET(GENMASK(15, 0), dma));
1261 txaddr_info->dma_low_msb = cpu_to_le16(FIELD_GET(GENMASK(31, 16), dma));
1262
1263 dma += len;
1264 txaddr_info++;
1265 }
1266
1267 WARN_ONCE(remain, "length overflow remain=%u total_len=%u",
1268 remain, total_len);
1269
1270 *add_info_nr = n;
1271
1272 return n * sizeof(*txaddr_info);
1273 }
1274 EXPORT_SYMBOL(rtw89_pci_fill_txaddr_info_v1);
1275
rtw89_pci_txwd_submit(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring,struct rtw89_pci_tx_wd * txwd,struct rtw89_core_tx_request * tx_req)1276 static int rtw89_pci_txwd_submit(struct rtw89_dev *rtwdev,
1277 struct rtw89_pci_tx_ring *tx_ring,
1278 struct rtw89_pci_tx_wd *txwd,
1279 struct rtw89_core_tx_request *tx_req)
1280 {
1281 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1282 const struct rtw89_chip_info *chip = rtwdev->chip;
1283 struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info;
1284 struct rtw89_pci_tx_wp_info *txwp_info;
1285 void *txaddr_info_addr;
1286 struct pci_dev *pdev = rtwpci->pdev;
1287 struct sk_buff *skb = tx_req->skb;
1288 struct rtw89_pci_tx_data *tx_data = RTW89_PCI_TX_SKB_CB(skb);
1289 struct rtw89_tx_skb_data *skb_data = RTW89_TX_SKB_CB(skb);
1290 bool en_wd_info = desc_info->en_wd_info;
1291 u32 txwd_len;
1292 u32 txwp_len;
1293 u32 txaddr_info_len;
1294 dma_addr_t dma;
1295 int ret;
1296
1297 dma = dma_map_single(&pdev->dev, skb->data, skb->len, DMA_TO_DEVICE);
1298 if (dma_mapping_error(&pdev->dev, dma)) {
1299 rtw89_err(rtwdev, "failed to map skb dma data\n");
1300 ret = -EBUSY;
1301 goto err;
1302 }
1303
1304 tx_data->dma = dma;
1305 rcu_assign_pointer(skb_data->wait, NULL);
1306
1307 txwp_len = sizeof(*txwp_info);
1308 txwd_len = chip->txwd_body_size;
1309 txwd_len += en_wd_info ? chip->txwd_info_size : 0;
1310
1311 txwp_info = txwd->vaddr + txwd_len;
1312 txwp_info->seq0 = cpu_to_le16(txwd->seq | RTW89_PCI_TXWP_VALID);
1313 txwp_info->seq1 = 0;
1314 txwp_info->seq2 = 0;
1315 txwp_info->seq3 = 0;
1316
1317 tx_ring->tx_cnt++;
1318 txaddr_info_addr = txwd->vaddr + txwd_len + txwp_len;
1319 txaddr_info_len =
1320 rtw89_chip_fill_txaddr_info(rtwdev, txaddr_info_addr, skb->len,
1321 dma, &desc_info->addr_info_nr);
1322
1323 txwd->len = txwd_len + txwp_len + txaddr_info_len;
1324
1325 rtw89_chip_fill_txdesc(rtwdev, desc_info, txwd->vaddr);
1326
1327 skb_queue_tail(&txwd->queue, skb);
1328
1329 return 0;
1330
1331 err:
1332 return ret;
1333 }
1334
rtw89_pci_fwcmd_submit(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring,struct rtw89_pci_tx_bd_32 * txbd,struct rtw89_core_tx_request * tx_req)1335 static int rtw89_pci_fwcmd_submit(struct rtw89_dev *rtwdev,
1336 struct rtw89_pci_tx_ring *tx_ring,
1337 struct rtw89_pci_tx_bd_32 *txbd,
1338 struct rtw89_core_tx_request *tx_req)
1339 {
1340 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1341 const struct rtw89_chip_info *chip = rtwdev->chip;
1342 struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info;
1343 void *txdesc;
1344 int txdesc_size = chip->h2c_desc_size;
1345 struct pci_dev *pdev = rtwpci->pdev;
1346 struct sk_buff *skb = tx_req->skb;
1347 struct rtw89_pci_tx_data *tx_data = RTW89_PCI_TX_SKB_CB(skb);
1348 dma_addr_t dma;
1349
1350 txdesc = skb_push(skb, txdesc_size);
1351 memset(txdesc, 0, txdesc_size);
1352 rtw89_chip_fill_txdesc_fwcmd(rtwdev, desc_info, txdesc);
1353
1354 dma = dma_map_single(&pdev->dev, skb->data, skb->len, DMA_TO_DEVICE);
1355 if (dma_mapping_error(&pdev->dev, dma)) {
1356 rtw89_err(rtwdev, "failed to map fwcmd dma data\n");
1357 return -EBUSY;
1358 }
1359
1360 tx_data->dma = dma;
1361 txbd->option = cpu_to_le16(RTW89_PCI_TXBD_OPTION_LS);
1362 txbd->length = cpu_to_le16(skb->len);
1363 txbd->dma = cpu_to_le32(tx_data->dma);
1364 skb_queue_tail(&rtwpci->h2c_queue, skb);
1365
1366 rtw89_pci_tx_bd_ring_update(rtwdev, tx_ring, 1);
1367
1368 return 0;
1369 }
1370
rtw89_pci_txbd_submit(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring,struct rtw89_pci_tx_bd_32 * txbd,struct rtw89_core_tx_request * tx_req)1371 static int rtw89_pci_txbd_submit(struct rtw89_dev *rtwdev,
1372 struct rtw89_pci_tx_ring *tx_ring,
1373 struct rtw89_pci_tx_bd_32 *txbd,
1374 struct rtw89_core_tx_request *tx_req)
1375 {
1376 struct rtw89_pci_tx_wd *txwd;
1377 int ret;
1378
1379 /* FWCMD queue doesn't have wd pages. Instead, it submits the CMD
1380 * buffer with WD BODY only. So here we don't need to check the free
1381 * pages of the wd ring.
1382 */
1383 if (tx_ring->txch == RTW89_TXCH_CH12)
1384 return rtw89_pci_fwcmd_submit(rtwdev, tx_ring, txbd, tx_req);
1385
1386 txwd = rtw89_pci_dequeue_txwd(tx_ring);
1387 if (!txwd) {
1388 rtw89_err(rtwdev, "no available TXWD\n");
1389 ret = -ENOSPC;
1390 goto err;
1391 }
1392
1393 ret = rtw89_pci_txwd_submit(rtwdev, tx_ring, txwd, tx_req);
1394 if (ret) {
1395 rtw89_err(rtwdev, "failed to submit TXWD %d\n", txwd->seq);
1396 goto err_enqueue_wd;
1397 }
1398
1399 list_add_tail(&txwd->list, &tx_ring->busy_pages);
1400
1401 txbd->option = cpu_to_le16(RTW89_PCI_TXBD_OPTION_LS);
1402 txbd->length = cpu_to_le16(txwd->len);
1403 txbd->dma = cpu_to_le32(txwd->paddr);
1404
1405 rtw89_pci_tx_bd_ring_update(rtwdev, tx_ring, 1);
1406
1407 return 0;
1408
1409 err_enqueue_wd:
1410 rtw89_pci_enqueue_txwd(tx_ring, txwd);
1411 err:
1412 return ret;
1413 }
1414
rtw89_pci_tx_write(struct rtw89_dev * rtwdev,struct rtw89_core_tx_request * tx_req,u8 txch)1415 static int rtw89_pci_tx_write(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req,
1416 u8 txch)
1417 {
1418 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1419 struct rtw89_pci_tx_ring *tx_ring;
1420 struct rtw89_pci_tx_bd_32 *txbd;
1421 u32 n_avail_txbd;
1422 int ret = 0;
1423
1424 /* check the tx type and dma channel for fw cmd queue */
1425 if ((txch == RTW89_TXCH_CH12 ||
1426 tx_req->tx_type == RTW89_CORE_TX_TYPE_FWCMD) &&
1427 (txch != RTW89_TXCH_CH12 ||
1428 tx_req->tx_type != RTW89_CORE_TX_TYPE_FWCMD)) {
1429 rtw89_err(rtwdev, "only fw cmd uses dma channel 12\n");
1430 return -EINVAL;
1431 }
1432
1433 tx_ring = &rtwpci->tx_rings[txch];
1434 spin_lock_bh(&rtwpci->trx_lock);
1435
1436 n_avail_txbd = rtw89_pci_get_avail_txbd_num(tx_ring);
1437 if (n_avail_txbd == 0) {
1438 rtw89_err(rtwdev, "no available TXBD\n");
1439 ret = -ENOSPC;
1440 goto err_unlock;
1441 }
1442
1443 txbd = rtw89_pci_get_next_txbd(tx_ring);
1444 ret = rtw89_pci_txbd_submit(rtwdev, tx_ring, txbd, tx_req);
1445 if (ret) {
1446 rtw89_err(rtwdev, "failed to submit TXBD\n");
1447 goto err_unlock;
1448 }
1449
1450 spin_unlock_bh(&rtwpci->trx_lock);
1451 return 0;
1452
1453 err_unlock:
1454 spin_unlock_bh(&rtwpci->trx_lock);
1455 return ret;
1456 }
1457
rtw89_pci_ops_tx_write(struct rtw89_dev * rtwdev,struct rtw89_core_tx_request * tx_req)1458 static int rtw89_pci_ops_tx_write(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req)
1459 {
1460 struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info;
1461 int ret;
1462
1463 ret = rtw89_pci_tx_write(rtwdev, tx_req, desc_info->ch_dma);
1464 if (ret) {
1465 rtw89_err(rtwdev, "failed to TX Queue %d\n", desc_info->ch_dma);
1466 return ret;
1467 }
1468
1469 return 0;
1470 }
1471
1472 const struct rtw89_pci_bd_ram rtw89_bd_ram_table_dual[RTW89_TXCH_NUM] = {
1473 [RTW89_TXCH_ACH0] = {.start_idx = 0, .max_num = 5, .min_num = 2},
1474 [RTW89_TXCH_ACH1] = {.start_idx = 5, .max_num = 5, .min_num = 2},
1475 [RTW89_TXCH_ACH2] = {.start_idx = 10, .max_num = 5, .min_num = 2},
1476 [RTW89_TXCH_ACH3] = {.start_idx = 15, .max_num = 5, .min_num = 2},
1477 [RTW89_TXCH_ACH4] = {.start_idx = 20, .max_num = 5, .min_num = 2},
1478 [RTW89_TXCH_ACH5] = {.start_idx = 25, .max_num = 5, .min_num = 2},
1479 [RTW89_TXCH_ACH6] = {.start_idx = 30, .max_num = 5, .min_num = 2},
1480 [RTW89_TXCH_ACH7] = {.start_idx = 35, .max_num = 5, .min_num = 2},
1481 [RTW89_TXCH_CH8] = {.start_idx = 40, .max_num = 5, .min_num = 1},
1482 [RTW89_TXCH_CH9] = {.start_idx = 45, .max_num = 5, .min_num = 1},
1483 [RTW89_TXCH_CH10] = {.start_idx = 50, .max_num = 5, .min_num = 1},
1484 [RTW89_TXCH_CH11] = {.start_idx = 55, .max_num = 5, .min_num = 1},
1485 [RTW89_TXCH_CH12] = {.start_idx = 60, .max_num = 4, .min_num = 1},
1486 };
1487 EXPORT_SYMBOL(rtw89_bd_ram_table_dual);
1488
1489 const struct rtw89_pci_bd_ram rtw89_bd_ram_table_single[RTW89_TXCH_NUM] = {
1490 [RTW89_TXCH_ACH0] = {.start_idx = 0, .max_num = 5, .min_num = 2},
1491 [RTW89_TXCH_ACH1] = {.start_idx = 5, .max_num = 5, .min_num = 2},
1492 [RTW89_TXCH_ACH2] = {.start_idx = 10, .max_num = 5, .min_num = 2},
1493 [RTW89_TXCH_ACH3] = {.start_idx = 15, .max_num = 5, .min_num = 2},
1494 [RTW89_TXCH_CH8] = {.start_idx = 20, .max_num = 4, .min_num = 1},
1495 [RTW89_TXCH_CH9] = {.start_idx = 24, .max_num = 4, .min_num = 1},
1496 [RTW89_TXCH_CH12] = {.start_idx = 28, .max_num = 4, .min_num = 1},
1497 };
1498 EXPORT_SYMBOL(rtw89_bd_ram_table_single);
1499
rtw89_pci_reset_trx_rings(struct rtw89_dev * rtwdev)1500 static void rtw89_pci_reset_trx_rings(struct rtw89_dev *rtwdev)
1501 {
1502 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1503 const struct rtw89_pci_info *info = rtwdev->pci_info;
1504 const struct rtw89_pci_bd_ram *bd_ram_table = *info->bd_ram_table;
1505 struct rtw89_pci_tx_ring *tx_ring;
1506 struct rtw89_pci_rx_ring *rx_ring;
1507 struct rtw89_pci_dma_ring *bd_ring;
1508 const struct rtw89_pci_bd_ram *bd_ram;
1509 u32 addr_num;
1510 u32 addr_idx;
1511 u32 addr_bdram;
1512 u32 addr_desa_l;
1513 u32 val32;
1514 int i;
1515
1516 for (i = 0; i < RTW89_TXCH_NUM; i++) {
1517 if (info->tx_dma_ch_mask & BIT(i))
1518 continue;
1519
1520 tx_ring = &rtwpci->tx_rings[i];
1521 bd_ring = &tx_ring->bd_ring;
1522 bd_ram = bd_ram_table ? &bd_ram_table[i] : NULL;
1523 addr_num = bd_ring->addr.num;
1524 addr_bdram = bd_ring->addr.bdram;
1525 addr_desa_l = bd_ring->addr.desa_l;
1526 bd_ring->wp = 0;
1527 bd_ring->rp = 0;
1528
1529 rtw89_write16(rtwdev, addr_num, bd_ring->len);
1530 if (addr_bdram && bd_ram) {
1531 val32 = FIELD_PREP(BDRAM_SIDX_MASK, bd_ram->start_idx) |
1532 FIELD_PREP(BDRAM_MAX_MASK, bd_ram->max_num) |
1533 FIELD_PREP(BDRAM_MIN_MASK, bd_ram->min_num);
1534
1535 rtw89_write32(rtwdev, addr_bdram, val32);
1536 }
1537 rtw89_write32(rtwdev, addr_desa_l, bd_ring->dma);
1538 }
1539
1540 for (i = 0; i < RTW89_RXCH_NUM; i++) {
1541 rx_ring = &rtwpci->rx_rings[i];
1542 bd_ring = &rx_ring->bd_ring;
1543 addr_num = bd_ring->addr.num;
1544 addr_idx = bd_ring->addr.idx;
1545 addr_desa_l = bd_ring->addr.desa_l;
1546 if (info->rx_ring_eq_is_full)
1547 bd_ring->wp = bd_ring->len - 1;
1548 else
1549 bd_ring->wp = 0;
1550 bd_ring->rp = 0;
1551 rx_ring->diliver_skb = NULL;
1552 rx_ring->diliver_desc.ready = false;
1553
1554 rtw89_write16(rtwdev, addr_num, bd_ring->len);
1555 rtw89_write32(rtwdev, addr_desa_l, bd_ring->dma);
1556
1557 if (info->rx_ring_eq_is_full)
1558 rtw89_write16(rtwdev, addr_idx, bd_ring->wp);
1559 }
1560 }
1561
rtw89_pci_release_tx_ring(struct rtw89_dev * rtwdev,struct rtw89_pci_tx_ring * tx_ring)1562 static void rtw89_pci_release_tx_ring(struct rtw89_dev *rtwdev,
1563 struct rtw89_pci_tx_ring *tx_ring)
1564 {
1565 rtw89_pci_release_busy_txwd(rtwdev, tx_ring);
1566 rtw89_pci_release_pending_txwd_skb(rtwdev, tx_ring);
1567 }
1568
rtw89_pci_ops_reset(struct rtw89_dev * rtwdev)1569 void rtw89_pci_ops_reset(struct rtw89_dev *rtwdev)
1570 {
1571 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1572 const struct rtw89_pci_info *info = rtwdev->pci_info;
1573 int txch;
1574
1575 rtw89_pci_reset_trx_rings(rtwdev);
1576
1577 spin_lock_bh(&rtwpci->trx_lock);
1578 for (txch = 0; txch < RTW89_TXCH_NUM; txch++) {
1579 if (info->tx_dma_ch_mask & BIT(txch))
1580 continue;
1581 if (txch == RTW89_TXCH_CH12) {
1582 rtw89_pci_release_fwcmd(rtwdev, rtwpci,
1583 skb_queue_len(&rtwpci->h2c_queue), true);
1584 continue;
1585 }
1586 rtw89_pci_release_tx_ring(rtwdev, &rtwpci->tx_rings[txch]);
1587 }
1588 spin_unlock_bh(&rtwpci->trx_lock);
1589 }
1590
rtw89_pci_enable_intr_lock(struct rtw89_dev * rtwdev)1591 static void rtw89_pci_enable_intr_lock(struct rtw89_dev *rtwdev)
1592 {
1593 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1594 unsigned long flags;
1595
1596 spin_lock_irqsave(&rtwpci->irq_lock, flags);
1597 rtwpci->running = true;
1598 rtw89_chip_enable_intr(rtwdev, rtwpci);
1599 spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
1600 }
1601
rtw89_pci_disable_intr_lock(struct rtw89_dev * rtwdev)1602 static void rtw89_pci_disable_intr_lock(struct rtw89_dev *rtwdev)
1603 {
1604 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1605 unsigned long flags;
1606
1607 spin_lock_irqsave(&rtwpci->irq_lock, flags);
1608 rtwpci->running = false;
1609 rtw89_chip_disable_intr(rtwdev, rtwpci);
1610 spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
1611 }
1612
rtw89_pci_ops_start(struct rtw89_dev * rtwdev)1613 static int rtw89_pci_ops_start(struct rtw89_dev *rtwdev)
1614 {
1615 rtw89_core_napi_start(rtwdev);
1616 rtw89_pci_enable_intr_lock(rtwdev);
1617
1618 return 0;
1619 }
1620
rtw89_pci_ops_stop(struct rtw89_dev * rtwdev)1621 static void rtw89_pci_ops_stop(struct rtw89_dev *rtwdev)
1622 {
1623 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1624 struct pci_dev *pdev = rtwpci->pdev;
1625
1626 rtw89_pci_disable_intr_lock(rtwdev);
1627 synchronize_irq(pdev->irq);
1628 rtw89_core_napi_stop(rtwdev);
1629 }
1630
rtw89_pci_ops_pause(struct rtw89_dev * rtwdev,bool pause)1631 static void rtw89_pci_ops_pause(struct rtw89_dev *rtwdev, bool pause)
1632 {
1633 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1634 struct pci_dev *pdev = rtwpci->pdev;
1635
1636 if (pause) {
1637 rtw89_pci_disable_intr_lock(rtwdev);
1638 synchronize_irq(pdev->irq);
1639 if (test_bit(RTW89_FLAG_NAPI_RUNNING, rtwdev->flags))
1640 napi_synchronize(&rtwdev->napi);
1641 } else {
1642 rtw89_pci_enable_intr_lock(rtwdev);
1643 rtw89_pci_tx_kick_off_pending(rtwdev);
1644 }
1645 }
1646
1647 static
rtw89_pci_switch_bd_idx_addr(struct rtw89_dev * rtwdev,bool low_power)1648 void rtw89_pci_switch_bd_idx_addr(struct rtw89_dev *rtwdev, bool low_power)
1649 {
1650 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1651 const struct rtw89_pci_info *info = rtwdev->pci_info;
1652 const struct rtw89_pci_bd_idx_addr *bd_idx_addr = info->bd_idx_addr_low_power;
1653 const struct rtw89_pci_ch_dma_addr_set *dma_addr_set = info->dma_addr_set;
1654 struct rtw89_pci_tx_ring *tx_ring;
1655 struct rtw89_pci_rx_ring *rx_ring;
1656 int i;
1657
1658 if (WARN(!bd_idx_addr, "only HCI with low power mode needs this\n"))
1659 return;
1660
1661 for (i = 0; i < RTW89_TXCH_NUM; i++) {
1662 tx_ring = &rtwpci->tx_rings[i];
1663 tx_ring->bd_ring.addr.idx = low_power ?
1664 bd_idx_addr->tx_bd_addrs[i] :
1665 dma_addr_set->tx[i].idx;
1666 }
1667
1668 for (i = 0; i < RTW89_RXCH_NUM; i++) {
1669 rx_ring = &rtwpci->rx_rings[i];
1670 rx_ring->bd_ring.addr.idx = low_power ?
1671 bd_idx_addr->rx_bd_addrs[i] :
1672 dma_addr_set->rx[i].idx;
1673 }
1674 }
1675
rtw89_pci_ops_switch_mode(struct rtw89_dev * rtwdev,bool low_power)1676 static void rtw89_pci_ops_switch_mode(struct rtw89_dev *rtwdev, bool low_power)
1677 {
1678 enum rtw89_pci_intr_mask_cfg cfg;
1679
1680 WARN(!rtwdev->hci.paused, "HCI isn't paused\n");
1681
1682 cfg = low_power ? RTW89_PCI_INTR_MASK_LOW_POWER : RTW89_PCI_INTR_MASK_NORMAL;
1683 rtw89_chip_config_intr_mask(rtwdev, cfg);
1684 rtw89_pci_switch_bd_idx_addr(rtwdev, low_power);
1685 }
1686
1687 static void rtw89_pci_ops_write32(struct rtw89_dev *rtwdev, u32 addr, u32 data);
1688
rtw89_pci_ops_read32_cmac(struct rtw89_dev * rtwdev,u32 addr)1689 static u32 rtw89_pci_ops_read32_cmac(struct rtw89_dev *rtwdev, u32 addr)
1690 {
1691 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1692 u32 val = readl(rtwpci->mmap + addr);
1693 int count;
1694
1695 for (count = 0; ; count++) {
1696 if (val != RTW89_R32_DEAD)
1697 return val;
1698 if (count >= MAC_REG_POOL_COUNT) {
1699 rtw89_warn(rtwdev, "addr %#x = %#x\n", addr, val);
1700 return RTW89_R32_DEAD;
1701 }
1702 rtw89_pci_ops_write32(rtwdev, R_AX_CK_EN, B_AX_CMAC_ALLCKEN);
1703 val = readl(rtwpci->mmap + addr);
1704 }
1705
1706 return val;
1707 }
1708
rtw89_pci_ops_read8(struct rtw89_dev * rtwdev,u32 addr)1709 static u8 rtw89_pci_ops_read8(struct rtw89_dev *rtwdev, u32 addr)
1710 {
1711 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1712 u32 addr32, val32, shift;
1713
1714 if (!ACCESS_CMAC(addr))
1715 return readb(rtwpci->mmap + addr);
1716
1717 addr32 = addr & ~0x3;
1718 shift = (addr & 0x3) * 8;
1719 val32 = rtw89_pci_ops_read32_cmac(rtwdev, addr32);
1720 return val32 >> shift;
1721 }
1722
rtw89_pci_ops_read16(struct rtw89_dev * rtwdev,u32 addr)1723 static u16 rtw89_pci_ops_read16(struct rtw89_dev *rtwdev, u32 addr)
1724 {
1725 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1726 u32 addr32, val32, shift;
1727
1728 if (!ACCESS_CMAC(addr))
1729 return readw(rtwpci->mmap + addr);
1730
1731 addr32 = addr & ~0x3;
1732 shift = (addr & 0x3) * 8;
1733 val32 = rtw89_pci_ops_read32_cmac(rtwdev, addr32);
1734 return val32 >> shift;
1735 }
1736
rtw89_pci_ops_read32(struct rtw89_dev * rtwdev,u32 addr)1737 static u32 rtw89_pci_ops_read32(struct rtw89_dev *rtwdev, u32 addr)
1738 {
1739 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1740
1741 if (!ACCESS_CMAC(addr))
1742 return readl(rtwpci->mmap + addr);
1743
1744 return rtw89_pci_ops_read32_cmac(rtwdev, addr);
1745 }
1746
rtw89_pci_ops_write8(struct rtw89_dev * rtwdev,u32 addr,u8 data)1747 static void rtw89_pci_ops_write8(struct rtw89_dev *rtwdev, u32 addr, u8 data)
1748 {
1749 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1750
1751 writeb(data, rtwpci->mmap + addr);
1752 }
1753
rtw89_pci_ops_write16(struct rtw89_dev * rtwdev,u32 addr,u16 data)1754 static void rtw89_pci_ops_write16(struct rtw89_dev *rtwdev, u32 addr, u16 data)
1755 {
1756 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1757
1758 writew(data, rtwpci->mmap + addr);
1759 }
1760
rtw89_pci_ops_write32(struct rtw89_dev * rtwdev,u32 addr,u32 data)1761 static void rtw89_pci_ops_write32(struct rtw89_dev *rtwdev, u32 addr, u32 data)
1762 {
1763 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1764
1765 writel(data, rtwpci->mmap + addr);
1766 }
1767
rtw89_pci_ctrl_dma_trx(struct rtw89_dev * rtwdev,bool enable)1768 static void rtw89_pci_ctrl_dma_trx(struct rtw89_dev *rtwdev, bool enable)
1769 {
1770 const struct rtw89_pci_info *info = rtwdev->pci_info;
1771
1772 if (enable)
1773 rtw89_write32_set(rtwdev, info->init_cfg_reg,
1774 info->rxhci_en_bit | info->txhci_en_bit);
1775 else
1776 rtw89_write32_clr(rtwdev, info->init_cfg_reg,
1777 info->rxhci_en_bit | info->txhci_en_bit);
1778 }
1779
rtw89_pci_ctrl_dma_io(struct rtw89_dev * rtwdev,bool enable)1780 static void rtw89_pci_ctrl_dma_io(struct rtw89_dev *rtwdev, bool enable)
1781 {
1782 const struct rtw89_pci_info *info = rtwdev->pci_info;
1783 const struct rtw89_reg_def *reg = &info->dma_io_stop;
1784
1785 if (enable)
1786 rtw89_write32_clr(rtwdev, reg->addr, reg->mask);
1787 else
1788 rtw89_write32_set(rtwdev, reg->addr, reg->mask);
1789 }
1790
rtw89_pci_ctrl_dma_all(struct rtw89_dev * rtwdev,bool enable)1791 void rtw89_pci_ctrl_dma_all(struct rtw89_dev *rtwdev, bool enable)
1792 {
1793 rtw89_pci_ctrl_dma_io(rtwdev, enable);
1794 rtw89_pci_ctrl_dma_trx(rtwdev, enable);
1795 }
1796
rtw89_pci_check_mdio(struct rtw89_dev * rtwdev,u8 addr,u8 speed,u16 rw_bit)1797 static int rtw89_pci_check_mdio(struct rtw89_dev *rtwdev, u8 addr, u8 speed, u16 rw_bit)
1798 {
1799 u16 val;
1800
1801 rtw89_write8(rtwdev, R_AX_MDIO_CFG, addr & 0x1F);
1802
1803 val = rtw89_read16(rtwdev, R_AX_MDIO_CFG);
1804 switch (speed) {
1805 case PCIE_PHY_GEN1:
1806 if (addr < 0x20)
1807 val = u16_replace_bits(val, MDIO_PG0_G1, B_AX_MDIO_PHY_ADDR_MASK);
1808 else
1809 val = u16_replace_bits(val, MDIO_PG1_G1, B_AX_MDIO_PHY_ADDR_MASK);
1810 break;
1811 case PCIE_PHY_GEN2:
1812 if (addr < 0x20)
1813 val = u16_replace_bits(val, MDIO_PG0_G2, B_AX_MDIO_PHY_ADDR_MASK);
1814 else
1815 val = u16_replace_bits(val, MDIO_PG1_G2, B_AX_MDIO_PHY_ADDR_MASK);
1816 break;
1817 default:
1818 rtw89_err(rtwdev, "[ERR]Error Speed %d!\n", speed);
1819 return -EINVAL;
1820 }
1821 rtw89_write16(rtwdev, R_AX_MDIO_CFG, val);
1822 rtw89_write16_set(rtwdev, R_AX_MDIO_CFG, rw_bit);
1823
1824 return read_poll_timeout(rtw89_read16, val, !(val & rw_bit), 10, 2000,
1825 false, rtwdev, R_AX_MDIO_CFG);
1826 }
1827
1828 static int
rtw89_read16_mdio(struct rtw89_dev * rtwdev,u8 addr,u8 speed,u16 * val)1829 rtw89_read16_mdio(struct rtw89_dev *rtwdev, u8 addr, u8 speed, u16 *val)
1830 {
1831 int ret;
1832
1833 ret = rtw89_pci_check_mdio(rtwdev, addr, speed, B_AX_MDIO_RFLAG);
1834 if (ret) {
1835 rtw89_err(rtwdev, "[ERR]MDIO R16 0x%X fail ret=%d!\n", addr, ret);
1836 return ret;
1837 }
1838 *val = rtw89_read16(rtwdev, R_AX_MDIO_RDATA);
1839
1840 return 0;
1841 }
1842
1843 static int
rtw89_write16_mdio(struct rtw89_dev * rtwdev,u8 addr,u16 data,u8 speed)1844 rtw89_write16_mdio(struct rtw89_dev *rtwdev, u8 addr, u16 data, u8 speed)
1845 {
1846 int ret;
1847
1848 rtw89_write16(rtwdev, R_AX_MDIO_WDATA, data);
1849 ret = rtw89_pci_check_mdio(rtwdev, addr, speed, B_AX_MDIO_WFLAG);
1850 if (ret) {
1851 rtw89_err(rtwdev, "[ERR]MDIO W16 0x%X = %x fail ret=%d!\n", addr, data, ret);
1852 return ret;
1853 }
1854
1855 return 0;
1856 }
1857
1858 static int
rtw89_write16_mdio_mask(struct rtw89_dev * rtwdev,u8 addr,u16 mask,u16 data,u8 speed)1859 rtw89_write16_mdio_mask(struct rtw89_dev *rtwdev, u8 addr, u16 mask, u16 data, u8 speed)
1860 {
1861 u32 shift;
1862 int ret;
1863 u16 val;
1864
1865 ret = rtw89_read16_mdio(rtwdev, addr, speed, &val);
1866 if (ret)
1867 return ret;
1868
1869 shift = __ffs(mask);
1870 val &= ~mask;
1871 val |= ((data << shift) & mask);
1872
1873 ret = rtw89_write16_mdio(rtwdev, addr, val, speed);
1874 if (ret)
1875 return ret;
1876
1877 return 0;
1878 }
1879
rtw89_write16_mdio_set(struct rtw89_dev * rtwdev,u8 addr,u16 mask,u8 speed)1880 static int rtw89_write16_mdio_set(struct rtw89_dev *rtwdev, u8 addr, u16 mask, u8 speed)
1881 {
1882 int ret;
1883 u16 val;
1884
1885 ret = rtw89_read16_mdio(rtwdev, addr, speed, &val);
1886 if (ret)
1887 return ret;
1888 ret = rtw89_write16_mdio(rtwdev, addr, val | mask, speed);
1889 if (ret)
1890 return ret;
1891
1892 return 0;
1893 }
1894
rtw89_write16_mdio_clr(struct rtw89_dev * rtwdev,u8 addr,u16 mask,u8 speed)1895 static int rtw89_write16_mdio_clr(struct rtw89_dev *rtwdev, u8 addr, u16 mask, u8 speed)
1896 {
1897 int ret;
1898 u16 val;
1899
1900 ret = rtw89_read16_mdio(rtwdev, addr, speed, &val);
1901 if (ret)
1902 return ret;
1903 ret = rtw89_write16_mdio(rtwdev, addr, val & ~mask, speed);
1904 if (ret)
1905 return ret;
1906
1907 return 0;
1908 }
1909
rtw89_pci_write_config_byte(struct rtw89_dev * rtwdev,u16 addr,u8 data)1910 static int rtw89_pci_write_config_byte(struct rtw89_dev *rtwdev, u16 addr,
1911 u8 data)
1912 {
1913 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1914 struct pci_dev *pdev = rtwpci->pdev;
1915
1916 return pci_write_config_byte(pdev, addr, data);
1917 }
1918
rtw89_pci_read_config_byte(struct rtw89_dev * rtwdev,u16 addr,u8 * value)1919 static int rtw89_pci_read_config_byte(struct rtw89_dev *rtwdev, u16 addr,
1920 u8 *value)
1921 {
1922 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
1923 struct pci_dev *pdev = rtwpci->pdev;
1924
1925 return pci_read_config_byte(pdev, addr, value);
1926 }
1927
rtw89_pci_config_byte_set(struct rtw89_dev * rtwdev,u16 addr,u8 bit)1928 static int rtw89_pci_config_byte_set(struct rtw89_dev *rtwdev, u16 addr,
1929 u8 bit)
1930 {
1931 u8 value;
1932 int ret;
1933
1934 ret = rtw89_pci_read_config_byte(rtwdev, addr, &value);
1935 if (ret)
1936 return ret;
1937
1938 value |= bit;
1939 ret = rtw89_pci_write_config_byte(rtwdev, addr, value);
1940
1941 return ret;
1942 }
1943
rtw89_pci_config_byte_clr(struct rtw89_dev * rtwdev,u16 addr,u8 bit)1944 static int rtw89_pci_config_byte_clr(struct rtw89_dev *rtwdev, u16 addr,
1945 u8 bit)
1946 {
1947 u8 value;
1948 int ret;
1949
1950 ret = rtw89_pci_read_config_byte(rtwdev, addr, &value);
1951 if (ret)
1952 return ret;
1953
1954 value &= ~bit;
1955 ret = rtw89_pci_write_config_byte(rtwdev, addr, value);
1956
1957 return ret;
1958 }
1959
1960 static int
__get_target(struct rtw89_dev * rtwdev,u16 * target,enum rtw89_pcie_phy phy_rate)1961 __get_target(struct rtw89_dev *rtwdev, u16 *target, enum rtw89_pcie_phy phy_rate)
1962 {
1963 u16 val, tar;
1964 int ret;
1965
1966 /* Enable counter */
1967 ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, phy_rate, &val);
1968 if (ret)
1969 return ret;
1970 ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, val & ~B_AX_CLK_CALIB_EN,
1971 phy_rate);
1972 if (ret)
1973 return ret;
1974 ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, val | B_AX_CLK_CALIB_EN,
1975 phy_rate);
1976 if (ret)
1977 return ret;
1978
1979 fsleep(300);
1980
1981 ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, phy_rate, &tar);
1982 if (ret)
1983 return ret;
1984 ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, val & ~B_AX_CLK_CALIB_EN,
1985 phy_rate);
1986 if (ret)
1987 return ret;
1988
1989 tar = tar & 0x0FFF;
1990 if (tar == 0 || tar == 0x0FFF) {
1991 rtw89_err(rtwdev, "[ERR]Get target failed.\n");
1992 return -EINVAL;
1993 }
1994
1995 *target = tar;
1996
1997 return 0;
1998 }
1999
rtw89_pci_autok_x(struct rtw89_dev * rtwdev)2000 static int rtw89_pci_autok_x(struct rtw89_dev *rtwdev)
2001 {
2002 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2003 int ret;
2004
2005 if (chip_id != RTL8852B && chip_id != RTL8851B)
2006 return 0;
2007
2008 ret = rtw89_write16_mdio_mask(rtwdev, RAC_REG_FLD_0, BAC_AUTOK_N_MASK,
2009 PCIE_AUTOK_4, PCIE_PHY_GEN1);
2010 return ret;
2011 }
2012
rtw89_pci_auto_refclk_cal(struct rtw89_dev * rtwdev,bool autook_en)2013 static int rtw89_pci_auto_refclk_cal(struct rtw89_dev *rtwdev, bool autook_en)
2014 {
2015 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2016 enum rtw89_pcie_phy phy_rate;
2017 u16 val16, mgn_set, div_set, tar;
2018 u8 val8, bdr_ori;
2019 bool l1_flag = false;
2020 int ret = 0;
2021
2022 if (chip_id != RTL8852B && chip_id != RTL8851B)
2023 return 0;
2024
2025 ret = rtw89_pci_read_config_byte(rtwdev, RTW89_PCIE_PHY_RATE, &val8);
2026 if (ret) {
2027 rtw89_err(rtwdev, "[ERR]pci config read %X\n",
2028 RTW89_PCIE_PHY_RATE);
2029 return ret;
2030 }
2031
2032 if (FIELD_GET(RTW89_PCIE_PHY_RATE_MASK, val8) == 0x1) {
2033 phy_rate = PCIE_PHY_GEN1;
2034 } else if (FIELD_GET(RTW89_PCIE_PHY_RATE_MASK, val8) == 0x2) {
2035 phy_rate = PCIE_PHY_GEN2;
2036 } else {
2037 rtw89_err(rtwdev, "[ERR]PCIe PHY rate %#x not support\n", val8);
2038 return -EOPNOTSUPP;
2039 }
2040 /* Disable L1BD */
2041 ret = rtw89_pci_read_config_byte(rtwdev, RTW89_PCIE_L1_CTRL, &bdr_ori);
2042 if (ret) {
2043 rtw89_err(rtwdev, "[ERR]pci config read %X\n", RTW89_PCIE_L1_CTRL);
2044 return ret;
2045 }
2046
2047 if (bdr_ori & RTW89_PCIE_BIT_L1) {
2048 ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_L1_CTRL,
2049 bdr_ori & ~RTW89_PCIE_BIT_L1);
2050 if (ret) {
2051 rtw89_err(rtwdev, "[ERR]pci config write %X\n",
2052 RTW89_PCIE_L1_CTRL);
2053 return ret;
2054 }
2055 l1_flag = true;
2056 }
2057
2058 ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, phy_rate, &val16);
2059 if (ret) {
2060 rtw89_err(rtwdev, "[ERR]mdio_r16_pcie %X\n", RAC_CTRL_PPR_V1);
2061 goto end;
2062 }
2063
2064 if (val16 & B_AX_CALIB_EN) {
2065 ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1,
2066 val16 & ~B_AX_CALIB_EN, phy_rate);
2067 if (ret) {
2068 rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1);
2069 goto end;
2070 }
2071 }
2072
2073 if (!autook_en)
2074 goto end;
2075 /* Set div */
2076 ret = rtw89_write16_mdio_clr(rtwdev, RAC_CTRL_PPR_V1, B_AX_DIV, phy_rate);
2077 if (ret) {
2078 rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1);
2079 goto end;
2080 }
2081
2082 /* Obtain div and margin */
2083 ret = __get_target(rtwdev, &tar, phy_rate);
2084 if (ret) {
2085 rtw89_err(rtwdev, "[ERR]1st get target fail %d\n", ret);
2086 goto end;
2087 }
2088
2089 mgn_set = tar * INTF_INTGRA_HOSTREF_V1 / INTF_INTGRA_MINREF_V1 - tar;
2090
2091 if (mgn_set >= 128) {
2092 div_set = 0x0003;
2093 mgn_set = 0x000F;
2094 } else if (mgn_set >= 64) {
2095 div_set = 0x0003;
2096 mgn_set >>= 3;
2097 } else if (mgn_set >= 32) {
2098 div_set = 0x0002;
2099 mgn_set >>= 2;
2100 } else if (mgn_set >= 16) {
2101 div_set = 0x0001;
2102 mgn_set >>= 1;
2103 } else if (mgn_set == 0) {
2104 rtw89_err(rtwdev, "[ERR]cal mgn is 0,tar = %d\n", tar);
2105 goto end;
2106 } else {
2107 div_set = 0x0000;
2108 }
2109
2110 ret = rtw89_read16_mdio(rtwdev, RAC_CTRL_PPR_V1, phy_rate, &val16);
2111 if (ret) {
2112 rtw89_err(rtwdev, "[ERR]mdio_r16_pcie %X\n", RAC_CTRL_PPR_V1);
2113 goto end;
2114 }
2115
2116 val16 |= u16_encode_bits(div_set, B_AX_DIV);
2117
2118 ret = rtw89_write16_mdio(rtwdev, RAC_CTRL_PPR_V1, val16, phy_rate);
2119 if (ret) {
2120 rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1);
2121 goto end;
2122 }
2123
2124 ret = __get_target(rtwdev, &tar, phy_rate);
2125 if (ret) {
2126 rtw89_err(rtwdev, "[ERR]2nd get target fail %d\n", ret);
2127 goto end;
2128 }
2129
2130 rtw89_debug(rtwdev, RTW89_DBG_HCI, "[TRACE]target = 0x%X, div = 0x%X, margin = 0x%X\n",
2131 tar, div_set, mgn_set);
2132 ret = rtw89_write16_mdio(rtwdev, RAC_SET_PPR_V1,
2133 (tar & 0x0FFF) | (mgn_set << 12), phy_rate);
2134 if (ret) {
2135 rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_SET_PPR_V1);
2136 goto end;
2137 }
2138
2139 /* Enable function */
2140 ret = rtw89_write16_mdio_set(rtwdev, RAC_CTRL_PPR_V1, B_AX_CALIB_EN, phy_rate);
2141 if (ret) {
2142 rtw89_err(rtwdev, "[ERR]mdio_w16_pcie %X\n", RAC_CTRL_PPR_V1);
2143 goto end;
2144 }
2145
2146 /* CLK delay = 0 */
2147 ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_CLK_CTRL,
2148 PCIE_CLKDLY_HW_0);
2149
2150 end:
2151 /* Set L1BD to ori */
2152 if (l1_flag) {
2153 ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_L1_CTRL,
2154 bdr_ori);
2155 if (ret) {
2156 rtw89_err(rtwdev, "[ERR]pci config write %X\n",
2157 RTW89_PCIE_L1_CTRL);
2158 return ret;
2159 }
2160 }
2161
2162 return ret;
2163 }
2164
rtw89_pci_deglitch_setting(struct rtw89_dev * rtwdev)2165 static int rtw89_pci_deglitch_setting(struct rtw89_dev *rtwdev)
2166 {
2167 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2168 int ret;
2169
2170 if (chip_id == RTL8852A) {
2171 ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA24, B_AX_DEGLITCH,
2172 PCIE_PHY_GEN1);
2173 if (ret)
2174 return ret;
2175 ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA24, B_AX_DEGLITCH,
2176 PCIE_PHY_GEN2);
2177 if (ret)
2178 return ret;
2179 } else if (chip_id == RTL8852C) {
2180 rtw89_write16_clr(rtwdev, R_RAC_DIRECT_OFFSET_G1 + RAC_ANA24 * 2,
2181 B_AX_DEGLITCH);
2182 rtw89_write16_clr(rtwdev, R_RAC_DIRECT_OFFSET_G2 + RAC_ANA24 * 2,
2183 B_AX_DEGLITCH);
2184 }
2185
2186 return 0;
2187 }
2188
rtw89_pci_rxdma_prefth(struct rtw89_dev * rtwdev)2189 static void rtw89_pci_rxdma_prefth(struct rtw89_dev *rtwdev)
2190 {
2191 if (rtwdev->chip->chip_id != RTL8852A)
2192 return;
2193
2194 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_DIS_RXDMA_PRE);
2195 }
2196
rtw89_pci_l1off_pwroff(struct rtw89_dev * rtwdev)2197 static void rtw89_pci_l1off_pwroff(struct rtw89_dev *rtwdev)
2198 {
2199 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2200
2201 if (chip_id != RTL8852A && chip_id != RTL8852B && chip_id != RTL8851B)
2202 return;
2203
2204 rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL, B_AX_L1OFF_PWR_OFF_EN);
2205 }
2206
rtw89_pci_l2_rxen_lat(struct rtw89_dev * rtwdev)2207 static u32 rtw89_pci_l2_rxen_lat(struct rtw89_dev *rtwdev)
2208 {
2209 int ret;
2210
2211 if (rtwdev->chip->chip_id != RTL8852A)
2212 return 0;
2213
2214 ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA26, B_AX_RXEN,
2215 PCIE_PHY_GEN1);
2216 if (ret)
2217 return ret;
2218
2219 ret = rtw89_write16_mdio_clr(rtwdev, RAC_ANA26, B_AX_RXEN,
2220 PCIE_PHY_GEN2);
2221 if (ret)
2222 return ret;
2223
2224 return 0;
2225 }
2226
rtw89_pci_aphy_pwrcut(struct rtw89_dev * rtwdev)2227 static void rtw89_pci_aphy_pwrcut(struct rtw89_dev *rtwdev)
2228 {
2229 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2230
2231 if (chip_id != RTL8852A && chip_id != RTL8852B && chip_id != RTL8851B)
2232 return;
2233
2234 rtw89_write32_clr(rtwdev, R_AX_SYS_PW_CTRL, B_AX_PSUS_OFF_CAPC_EN);
2235 }
2236
rtw89_pci_hci_ldo(struct rtw89_dev * rtwdev)2237 static void rtw89_pci_hci_ldo(struct rtw89_dev *rtwdev)
2238 {
2239 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2240
2241 if (chip_id == RTL8852A || chip_id == RTL8852B || chip_id == RTL8851B) {
2242 rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL,
2243 B_AX_PCIE_DIS_L2_CTRL_LDO_HCI);
2244 rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL,
2245 B_AX_PCIE_DIS_WLSUS_AFT_PDN);
2246 } else if (rtwdev->chip->chip_id == RTL8852C) {
2247 rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL,
2248 B_AX_PCIE_DIS_L2_CTRL_LDO_HCI);
2249 }
2250 }
2251
rtw89_pci_dphy_delay(struct rtw89_dev * rtwdev)2252 static int rtw89_pci_dphy_delay(struct rtw89_dev *rtwdev)
2253 {
2254 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2255
2256 if (chip_id != RTL8852B && chip_id != RTL8851B)
2257 return 0;
2258
2259 return rtw89_write16_mdio_mask(rtwdev, RAC_REG_REV2, BAC_CMU_EN_DLY_MASK,
2260 PCIE_DPHY_DLY_25US, PCIE_PHY_GEN1);
2261 }
2262
rtw89_pci_power_wake(struct rtw89_dev * rtwdev,bool pwr_up)2263 static void rtw89_pci_power_wake(struct rtw89_dev *rtwdev, bool pwr_up)
2264 {
2265 if (pwr_up)
2266 rtw89_write32_set(rtwdev, R_AX_HCI_OPT_CTRL, BIT_WAKE_CTRL);
2267 else
2268 rtw89_write32_clr(rtwdev, R_AX_HCI_OPT_CTRL, BIT_WAKE_CTRL);
2269 }
2270
rtw89_pci_autoload_hang(struct rtw89_dev * rtwdev)2271 static void rtw89_pci_autoload_hang(struct rtw89_dev *rtwdev)
2272 {
2273 if (rtwdev->chip->chip_id != RTL8852C)
2274 return;
2275
2276 rtw89_write32_set(rtwdev, R_AX_PCIE_BG_CLR, B_AX_BG_CLR_ASYNC_M3);
2277 rtw89_write32_clr(rtwdev, R_AX_PCIE_BG_CLR, B_AX_BG_CLR_ASYNC_M3);
2278 }
2279
rtw89_pci_l12_vmain(struct rtw89_dev * rtwdev)2280 static void rtw89_pci_l12_vmain(struct rtw89_dev *rtwdev)
2281 {
2282 if (!(rtwdev->chip->chip_id == RTL8852C && rtwdev->hal.cv == CHIP_CAV))
2283 return;
2284
2285 rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL, B_AX_PCIE_FORCE_PWR_NGAT);
2286 }
2287
rtw89_pci_gen2_force_ib(struct rtw89_dev * rtwdev)2288 static void rtw89_pci_gen2_force_ib(struct rtw89_dev *rtwdev)
2289 {
2290 if (!(rtwdev->chip->chip_id == RTL8852C && rtwdev->hal.cv == CHIP_CAV))
2291 return;
2292
2293 rtw89_write32_set(rtwdev, R_AX_PMC_DBG_CTRL2,
2294 B_AX_SYSON_DIS_PMCR_AX_WRMSK);
2295 rtw89_write32_set(rtwdev, R_AX_HCI_BG_CTRL, B_AX_BG_CLR_ASYNC_M3);
2296 rtw89_write32_clr(rtwdev, R_AX_PMC_DBG_CTRL2,
2297 B_AX_SYSON_DIS_PMCR_AX_WRMSK);
2298 }
2299
rtw89_pci_l1_ent_lat(struct rtw89_dev * rtwdev)2300 static void rtw89_pci_l1_ent_lat(struct rtw89_dev *rtwdev)
2301 {
2302 if (rtwdev->chip->chip_id != RTL8852C)
2303 return;
2304
2305 rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1, B_AX_SEL_REQ_ENTR_L1);
2306 }
2307
rtw89_pci_wd_exit_l1(struct rtw89_dev * rtwdev)2308 static void rtw89_pci_wd_exit_l1(struct rtw89_dev *rtwdev)
2309 {
2310 if (rtwdev->chip->chip_id != RTL8852C)
2311 return;
2312
2313 rtw89_write32_set(rtwdev, R_AX_PCIE_PS_CTRL_V1, B_AX_DMAC0_EXIT_L1_EN);
2314 }
2315
rtw89_pci_set_sic(struct rtw89_dev * rtwdev)2316 static void rtw89_pci_set_sic(struct rtw89_dev *rtwdev)
2317 {
2318 if (rtwdev->chip->chip_id == RTL8852C)
2319 return;
2320
2321 rtw89_write32_clr(rtwdev, R_AX_PCIE_EXP_CTRL,
2322 B_AX_SIC_EN_FORCE_CLKREQ);
2323 }
2324
rtw89_pci_set_lbc(struct rtw89_dev * rtwdev)2325 static void rtw89_pci_set_lbc(struct rtw89_dev *rtwdev)
2326 {
2327 const struct rtw89_pci_info *info = rtwdev->pci_info;
2328 u32 lbc;
2329
2330 if (rtwdev->chip->chip_id == RTL8852C)
2331 return;
2332
2333 lbc = rtw89_read32(rtwdev, R_AX_LBC_WATCHDOG);
2334 if (info->lbc_en == MAC_AX_PCIE_ENABLE) {
2335 lbc = u32_replace_bits(lbc, info->lbc_tmr, B_AX_LBC_TIMER);
2336 lbc |= B_AX_LBC_FLAG | B_AX_LBC_EN;
2337 rtw89_write32(rtwdev, R_AX_LBC_WATCHDOG, lbc);
2338 } else {
2339 lbc &= ~B_AX_LBC_EN;
2340 }
2341 rtw89_write32_set(rtwdev, R_AX_LBC_WATCHDOG, lbc);
2342 }
2343
rtw89_pci_set_io_rcy(struct rtw89_dev * rtwdev)2344 static void rtw89_pci_set_io_rcy(struct rtw89_dev *rtwdev)
2345 {
2346 const struct rtw89_pci_info *info = rtwdev->pci_info;
2347 u32 val32;
2348
2349 if (rtwdev->chip->chip_id != RTL8852C)
2350 return;
2351
2352 if (info->io_rcy_en == MAC_AX_PCIE_ENABLE) {
2353 val32 = FIELD_PREP(B_AX_PCIE_WDT_TIMER_M1_MASK,
2354 info->io_rcy_tmr);
2355 rtw89_write32(rtwdev, R_AX_PCIE_WDT_TIMER_M1, val32);
2356 rtw89_write32(rtwdev, R_AX_PCIE_WDT_TIMER_M2, val32);
2357 rtw89_write32(rtwdev, R_AX_PCIE_WDT_TIMER_E0, val32);
2358
2359 rtw89_write32_set(rtwdev, R_AX_PCIE_IO_RCY_M1, B_AX_PCIE_IO_RCY_WDT_MODE_M1);
2360 rtw89_write32_set(rtwdev, R_AX_PCIE_IO_RCY_M2, B_AX_PCIE_IO_RCY_WDT_MODE_M2);
2361 rtw89_write32_set(rtwdev, R_AX_PCIE_IO_RCY_E0, B_AX_PCIE_IO_RCY_WDT_MODE_E0);
2362 } else {
2363 rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_M1, B_AX_PCIE_IO_RCY_WDT_MODE_M1);
2364 rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_M2, B_AX_PCIE_IO_RCY_WDT_MODE_M2);
2365 rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_E0, B_AX_PCIE_IO_RCY_WDT_MODE_E0);
2366 }
2367
2368 rtw89_write32_clr(rtwdev, R_AX_PCIE_IO_RCY_S1, B_AX_PCIE_IO_RCY_WDT_MODE_S1);
2369 }
2370
rtw89_pci_set_dbg(struct rtw89_dev * rtwdev)2371 static void rtw89_pci_set_dbg(struct rtw89_dev *rtwdev)
2372 {
2373 if (rtwdev->chip->chip_id == RTL8852C)
2374 return;
2375
2376 rtw89_write32_set(rtwdev, R_AX_PCIE_DBG_CTRL,
2377 B_AX_ASFF_FULL_NO_STK | B_AX_EN_STUCK_DBG);
2378
2379 if (rtwdev->chip->chip_id == RTL8852A)
2380 rtw89_write32_set(rtwdev, R_AX_PCIE_EXP_CTRL,
2381 B_AX_EN_CHKDSC_NO_RX_STUCK);
2382 }
2383
rtw89_pci_set_keep_reg(struct rtw89_dev * rtwdev)2384 static void rtw89_pci_set_keep_reg(struct rtw89_dev *rtwdev)
2385 {
2386 if (rtwdev->chip->chip_id == RTL8852C)
2387 return;
2388
2389 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1,
2390 B_AX_PCIE_TXRST_KEEP_REG | B_AX_PCIE_RXRST_KEEP_REG);
2391 }
2392
rtw89_pci_clr_idx_all_ax(struct rtw89_dev * rtwdev)2393 static void rtw89_pci_clr_idx_all_ax(struct rtw89_dev *rtwdev)
2394 {
2395 const struct rtw89_pci_info *info = rtwdev->pci_info;
2396 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2397 u32 val = B_AX_CLR_ACH0_IDX | B_AX_CLR_ACH1_IDX | B_AX_CLR_ACH2_IDX |
2398 B_AX_CLR_ACH3_IDX | B_AX_CLR_CH8_IDX | B_AX_CLR_CH9_IDX |
2399 B_AX_CLR_CH12_IDX;
2400 u32 rxbd_rwptr_clr = info->rxbd_rwptr_clr_reg;
2401 u32 txbd_rwptr_clr2 = info->txbd_rwptr_clr2_reg;
2402
2403 if (chip_id == RTL8852A || chip_id == RTL8852C)
2404 val |= B_AX_CLR_ACH4_IDX | B_AX_CLR_ACH5_IDX |
2405 B_AX_CLR_ACH6_IDX | B_AX_CLR_ACH7_IDX;
2406 /* clear DMA indexes */
2407 rtw89_write32_set(rtwdev, R_AX_TXBD_RWPTR_CLR1, val);
2408 if (chip_id == RTL8852A || chip_id == RTL8852C)
2409 rtw89_write32_set(rtwdev, txbd_rwptr_clr2,
2410 B_AX_CLR_CH10_IDX | B_AX_CLR_CH11_IDX);
2411 rtw89_write32_set(rtwdev, rxbd_rwptr_clr,
2412 B_AX_CLR_RXQ_IDX | B_AX_CLR_RPQ_IDX);
2413 }
2414
rtw89_poll_txdma_ch_idle_pcie(struct rtw89_dev * rtwdev)2415 static int rtw89_poll_txdma_ch_idle_pcie(struct rtw89_dev *rtwdev)
2416 {
2417 const struct rtw89_pci_info *info = rtwdev->pci_info;
2418 u32 ret, check, dma_busy;
2419 u32 dma_busy1 = info->dma_busy1.addr;
2420 u32 dma_busy2 = info->dma_busy2_reg;
2421
2422 check = info->dma_busy1.mask;
2423
2424 ret = read_poll_timeout(rtw89_read32, dma_busy, (dma_busy & check) == 0,
2425 10, 100, false, rtwdev, dma_busy1);
2426 if (ret)
2427 return ret;
2428
2429 if (!dma_busy2)
2430 return 0;
2431
2432 check = B_AX_CH10_BUSY | B_AX_CH11_BUSY;
2433
2434 ret = read_poll_timeout(rtw89_read32, dma_busy, (dma_busy & check) == 0,
2435 10, 100, false, rtwdev, dma_busy2);
2436 if (ret)
2437 return ret;
2438
2439 return 0;
2440 }
2441
rtw89_poll_rxdma_ch_idle_pcie(struct rtw89_dev * rtwdev)2442 static int rtw89_poll_rxdma_ch_idle_pcie(struct rtw89_dev *rtwdev)
2443 {
2444 const struct rtw89_pci_info *info = rtwdev->pci_info;
2445 u32 ret, check, dma_busy;
2446 u32 dma_busy3 = info->dma_busy3_reg;
2447
2448 check = B_AX_RXQ_BUSY | B_AX_RPQ_BUSY;
2449
2450 ret = read_poll_timeout(rtw89_read32, dma_busy, (dma_busy & check) == 0,
2451 10, 100, false, rtwdev, dma_busy3);
2452 if (ret)
2453 return ret;
2454
2455 return 0;
2456 }
2457
rtw89_pci_poll_dma_all_idle(struct rtw89_dev * rtwdev)2458 static int rtw89_pci_poll_dma_all_idle(struct rtw89_dev *rtwdev)
2459 {
2460 u32 ret;
2461
2462 ret = rtw89_poll_txdma_ch_idle_pcie(rtwdev);
2463 if (ret) {
2464 rtw89_err(rtwdev, "txdma ch busy\n");
2465 return ret;
2466 }
2467
2468 ret = rtw89_poll_rxdma_ch_idle_pcie(rtwdev);
2469 if (ret) {
2470 rtw89_err(rtwdev, "rxdma ch busy\n");
2471 return ret;
2472 }
2473
2474 return 0;
2475 }
2476
rtw89_pci_mode_op(struct rtw89_dev * rtwdev)2477 static int rtw89_pci_mode_op(struct rtw89_dev *rtwdev)
2478 {
2479 const struct rtw89_pci_info *info = rtwdev->pci_info;
2480 enum mac_ax_bd_trunc_mode txbd_trunc_mode = info->txbd_trunc_mode;
2481 enum mac_ax_bd_trunc_mode rxbd_trunc_mode = info->rxbd_trunc_mode;
2482 enum mac_ax_rxbd_mode rxbd_mode = info->rxbd_mode;
2483 enum mac_ax_tag_mode tag_mode = info->tag_mode;
2484 enum mac_ax_wd_dma_intvl wd_dma_idle_intvl = info->wd_dma_idle_intvl;
2485 enum mac_ax_wd_dma_intvl wd_dma_act_intvl = info->wd_dma_act_intvl;
2486 enum mac_ax_tx_burst tx_burst = info->tx_burst;
2487 enum mac_ax_rx_burst rx_burst = info->rx_burst;
2488 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2489 u8 cv = rtwdev->hal.cv;
2490 u32 val32;
2491
2492 if (txbd_trunc_mode == MAC_AX_BD_TRUNC) {
2493 if (chip_id == RTL8852A && cv == CHIP_CBV)
2494 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_TX_TRUNC_MODE);
2495 } else if (txbd_trunc_mode == MAC_AX_BD_NORM) {
2496 if (chip_id == RTL8852A || chip_id == RTL8852B)
2497 rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_TX_TRUNC_MODE);
2498 }
2499
2500 if (rxbd_trunc_mode == MAC_AX_BD_TRUNC) {
2501 if (chip_id == RTL8852A && cv == CHIP_CBV)
2502 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_RX_TRUNC_MODE);
2503 } else if (rxbd_trunc_mode == MAC_AX_BD_NORM) {
2504 if (chip_id == RTL8852A || chip_id == RTL8852B)
2505 rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_RX_TRUNC_MODE);
2506 }
2507
2508 if (rxbd_mode == MAC_AX_RXBD_PKT) {
2509 rtw89_write32_clr(rtwdev, info->init_cfg_reg, info->rxbd_mode_bit);
2510 } else if (rxbd_mode == MAC_AX_RXBD_SEP) {
2511 rtw89_write32_set(rtwdev, info->init_cfg_reg, info->rxbd_mode_bit);
2512
2513 if (chip_id == RTL8852A || chip_id == RTL8852B)
2514 rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG2,
2515 B_AX_PCIE_RX_APPLEN_MASK, 0);
2516 }
2517
2518 if (chip_id == RTL8852A || chip_id == RTL8852B) {
2519 rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_PCIE_MAX_TXDMA_MASK, tx_burst);
2520 rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_PCIE_MAX_RXDMA_MASK, rx_burst);
2521 } else if (chip_id == RTL8852C) {
2522 rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_HAXI_MAX_TXDMA_MASK, tx_burst);
2523 rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_HAXI_MAX_RXDMA_MASK, rx_burst);
2524 }
2525
2526 if (chip_id == RTL8852A || chip_id == RTL8852B) {
2527 if (tag_mode == MAC_AX_TAG_SGL) {
2528 val32 = rtw89_read32(rtwdev, R_AX_PCIE_INIT_CFG1) &
2529 ~B_AX_LATENCY_CONTROL;
2530 rtw89_write32(rtwdev, R_AX_PCIE_INIT_CFG1, val32);
2531 } else if (tag_mode == MAC_AX_TAG_MULTI) {
2532 val32 = rtw89_read32(rtwdev, R_AX_PCIE_INIT_CFG1) |
2533 B_AX_LATENCY_CONTROL;
2534 rtw89_write32(rtwdev, R_AX_PCIE_INIT_CFG1, val32);
2535 }
2536 }
2537
2538 rtw89_write32_mask(rtwdev, info->exp_ctrl_reg, info->max_tag_num_mask,
2539 info->multi_tag_num);
2540
2541 if (chip_id == RTL8852A || chip_id == RTL8852B) {
2542 rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG2, B_AX_WD_ITVL_IDLE,
2543 wd_dma_idle_intvl);
2544 rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG2, B_AX_WD_ITVL_ACT,
2545 wd_dma_act_intvl);
2546 } else if (chip_id == RTL8852C) {
2547 rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_WD_ITVL_IDLE_V1_MASK,
2548 wd_dma_idle_intvl);
2549 rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_WD_ITVL_ACT_V1_MASK,
2550 wd_dma_act_intvl);
2551 }
2552
2553 if (txbd_trunc_mode == MAC_AX_BD_TRUNC) {
2554 rtw89_write32_set(rtwdev, R_AX_TX_ADDRESS_INFO_MODE_SETTING,
2555 B_AX_HOST_ADDR_INFO_8B_SEL);
2556 rtw89_write32_clr(rtwdev, R_AX_PKTIN_SETTING, B_AX_WD_ADDR_INFO_LENGTH);
2557 } else if (txbd_trunc_mode == MAC_AX_BD_NORM) {
2558 rtw89_write32_clr(rtwdev, R_AX_TX_ADDRESS_INFO_MODE_SETTING,
2559 B_AX_HOST_ADDR_INFO_8B_SEL);
2560 rtw89_write32_set(rtwdev, R_AX_PKTIN_SETTING, B_AX_WD_ADDR_INFO_LENGTH);
2561 }
2562
2563 return 0;
2564 }
2565
rtw89_pci_ops_deinit(struct rtw89_dev * rtwdev)2566 static int rtw89_pci_ops_deinit(struct rtw89_dev *rtwdev)
2567 {
2568 const struct rtw89_pci_info *info = rtwdev->pci_info;
2569
2570 if (rtwdev->chip->chip_id == RTL8852A) {
2571 /* ltr sw trigger */
2572 rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, B_AX_APP_LTR_IDLE);
2573 }
2574 info->ltr_set(rtwdev, false);
2575 rtw89_pci_ctrl_dma_all(rtwdev, false);
2576 rtw89_pci_clr_idx_all(rtwdev);
2577
2578 return 0;
2579 }
2580
rtw89_pci_ops_mac_pre_init_ax(struct rtw89_dev * rtwdev)2581 static int rtw89_pci_ops_mac_pre_init_ax(struct rtw89_dev *rtwdev)
2582 {
2583 const struct rtw89_pci_info *info = rtwdev->pci_info;
2584 int ret;
2585
2586 rtw89_pci_rxdma_prefth(rtwdev);
2587 rtw89_pci_l1off_pwroff(rtwdev);
2588 rtw89_pci_deglitch_setting(rtwdev);
2589 ret = rtw89_pci_l2_rxen_lat(rtwdev);
2590 if (ret) {
2591 rtw89_err(rtwdev, "[ERR] pcie l2 rxen lat %d\n", ret);
2592 return ret;
2593 }
2594
2595 rtw89_pci_aphy_pwrcut(rtwdev);
2596 rtw89_pci_hci_ldo(rtwdev);
2597 rtw89_pci_dphy_delay(rtwdev);
2598
2599 ret = rtw89_pci_autok_x(rtwdev);
2600 if (ret) {
2601 rtw89_err(rtwdev, "[ERR] pcie autok_x fail %d\n", ret);
2602 return ret;
2603 }
2604
2605 ret = rtw89_pci_auto_refclk_cal(rtwdev, false);
2606 if (ret) {
2607 rtw89_err(rtwdev, "[ERR] pcie autok fail %d\n", ret);
2608 return ret;
2609 }
2610
2611 rtw89_pci_power_wake(rtwdev, true);
2612 rtw89_pci_autoload_hang(rtwdev);
2613 rtw89_pci_l12_vmain(rtwdev);
2614 rtw89_pci_gen2_force_ib(rtwdev);
2615 rtw89_pci_l1_ent_lat(rtwdev);
2616 rtw89_pci_wd_exit_l1(rtwdev);
2617 rtw89_pci_set_sic(rtwdev);
2618 rtw89_pci_set_lbc(rtwdev);
2619 rtw89_pci_set_io_rcy(rtwdev);
2620 rtw89_pci_set_dbg(rtwdev);
2621 rtw89_pci_set_keep_reg(rtwdev);
2622
2623 rtw89_write32_set(rtwdev, info->dma_stop1.addr, B_AX_STOP_WPDMA);
2624
2625 /* stop DMA activities */
2626 rtw89_pci_ctrl_dma_all(rtwdev, false);
2627
2628 ret = rtw89_pci_poll_dma_all_idle(rtwdev);
2629 if (ret) {
2630 rtw89_err(rtwdev, "[ERR] poll pcie dma all idle\n");
2631 return ret;
2632 }
2633
2634 rtw89_pci_clr_idx_all(rtwdev);
2635 rtw89_pci_mode_op(rtwdev);
2636
2637 /* fill TRX BD indexes */
2638 rtw89_pci_ops_reset(rtwdev);
2639
2640 ret = rtw89_pci_rst_bdram_ax(rtwdev);
2641 if (ret) {
2642 rtw89_warn(rtwdev, "reset bdram busy\n");
2643 return ret;
2644 }
2645
2646 /* disable all channels except to FW CMD channel to download firmware */
2647 rtw89_pci_ctrl_txdma_ch_pcie(rtwdev, false);
2648 rtw89_pci_ctrl_txdma_fw_ch_pcie(rtwdev, true);
2649
2650 /* start DMA activities */
2651 rtw89_pci_ctrl_dma_all(rtwdev, true);
2652
2653 return 0;
2654 }
2655
rtw89_pci_ltr_set(struct rtw89_dev * rtwdev,bool en)2656 int rtw89_pci_ltr_set(struct rtw89_dev *rtwdev, bool en)
2657 {
2658 u32 val;
2659
2660 if (!en)
2661 return 0;
2662
2663 val = rtw89_read32(rtwdev, R_AX_LTR_CTRL_0);
2664 if (rtw89_pci_ltr_is_err_reg_val(val))
2665 return -EINVAL;
2666 val = rtw89_read32(rtwdev, R_AX_LTR_CTRL_1);
2667 if (rtw89_pci_ltr_is_err_reg_val(val))
2668 return -EINVAL;
2669 val = rtw89_read32(rtwdev, R_AX_LTR_IDLE_LATENCY);
2670 if (rtw89_pci_ltr_is_err_reg_val(val))
2671 return -EINVAL;
2672 val = rtw89_read32(rtwdev, R_AX_LTR_ACTIVE_LATENCY);
2673 if (rtw89_pci_ltr_is_err_reg_val(val))
2674 return -EINVAL;
2675
2676 rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_HW_EN | B_AX_LTR_EN |
2677 B_AX_LTR_WD_NOEMP_CHK);
2678 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_SPACE_IDX_MASK,
2679 PCI_LTR_SPC_500US);
2680 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_IDLE_TIMER_IDX_MASK,
2681 PCI_LTR_IDLE_TIMER_3_2MS);
2682 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX0_TH_MASK, 0x28);
2683 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX1_TH_MASK, 0x28);
2684 rtw89_write32(rtwdev, R_AX_LTR_IDLE_LATENCY, 0x90039003);
2685 rtw89_write32(rtwdev, R_AX_LTR_ACTIVE_LATENCY, 0x880b880b);
2686
2687 return 0;
2688 }
2689 EXPORT_SYMBOL(rtw89_pci_ltr_set);
2690
rtw89_pci_ltr_set_v1(struct rtw89_dev * rtwdev,bool en)2691 int rtw89_pci_ltr_set_v1(struct rtw89_dev *rtwdev, bool en)
2692 {
2693 u32 dec_ctrl;
2694 u32 val32;
2695
2696 val32 = rtw89_read32(rtwdev, R_AX_LTR_CTRL_0);
2697 if (rtw89_pci_ltr_is_err_reg_val(val32))
2698 return -EINVAL;
2699 val32 = rtw89_read32(rtwdev, R_AX_LTR_CTRL_1);
2700 if (rtw89_pci_ltr_is_err_reg_val(val32))
2701 return -EINVAL;
2702 dec_ctrl = rtw89_read32(rtwdev, R_AX_LTR_DEC_CTRL);
2703 if (rtw89_pci_ltr_is_err_reg_val(dec_ctrl))
2704 return -EINVAL;
2705 val32 = rtw89_read32(rtwdev, R_AX_LTR_LATENCY_IDX3);
2706 if (rtw89_pci_ltr_is_err_reg_val(val32))
2707 return -EINVAL;
2708 val32 = rtw89_read32(rtwdev, R_AX_LTR_LATENCY_IDX0);
2709 if (rtw89_pci_ltr_is_err_reg_val(val32))
2710 return -EINVAL;
2711
2712 if (!en) {
2713 dec_ctrl &= ~(LTR_EN_BITS | B_AX_LTR_IDX_DRV_MASK | B_AX_LTR_HW_DEC_EN);
2714 dec_ctrl |= FIELD_PREP(B_AX_LTR_IDX_DRV_MASK, PCIE_LTR_IDX_IDLE) |
2715 B_AX_LTR_REQ_DRV;
2716 } else {
2717 dec_ctrl |= B_AX_LTR_HW_DEC_EN;
2718 }
2719
2720 dec_ctrl &= ~B_AX_LTR_SPACE_IDX_V1_MASK;
2721 dec_ctrl |= FIELD_PREP(B_AX_LTR_SPACE_IDX_V1_MASK, PCI_LTR_SPC_500US);
2722
2723 if (en)
2724 rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0,
2725 B_AX_LTR_WD_NOEMP_CHK_V1 | B_AX_LTR_HW_EN);
2726 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_0, B_AX_LTR_IDLE_TIMER_IDX_MASK,
2727 PCI_LTR_IDLE_TIMER_3_2MS);
2728 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX0_TH_MASK, 0x28);
2729 rtw89_write32_mask(rtwdev, R_AX_LTR_CTRL_1, B_AX_LTR_RX1_TH_MASK, 0x28);
2730 rtw89_write32(rtwdev, R_AX_LTR_DEC_CTRL, dec_ctrl);
2731 rtw89_write32(rtwdev, R_AX_LTR_LATENCY_IDX3, 0x90039003);
2732 rtw89_write32(rtwdev, R_AX_LTR_LATENCY_IDX0, 0x880b880b);
2733
2734 return 0;
2735 }
2736 EXPORT_SYMBOL(rtw89_pci_ltr_set_v1);
2737
rtw89_pci_ops_mac_post_init_ax(struct rtw89_dev * rtwdev)2738 static int rtw89_pci_ops_mac_post_init_ax(struct rtw89_dev *rtwdev)
2739 {
2740 const struct rtw89_pci_info *info = rtwdev->pci_info;
2741 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
2742 int ret;
2743
2744 ret = info->ltr_set(rtwdev, true);
2745 if (ret) {
2746 rtw89_err(rtwdev, "pci ltr set fail\n");
2747 return ret;
2748 }
2749 if (chip_id == RTL8852A) {
2750 /* ltr sw trigger */
2751 rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, B_AX_APP_LTR_ACT);
2752 }
2753 if (chip_id == RTL8852A || chip_id == RTL8852B) {
2754 /* ADDR info 8-byte mode */
2755 rtw89_write32_set(rtwdev, R_AX_TX_ADDRESS_INFO_MODE_SETTING,
2756 B_AX_HOST_ADDR_INFO_8B_SEL);
2757 rtw89_write32_clr(rtwdev, R_AX_PKTIN_SETTING, B_AX_WD_ADDR_INFO_LENGTH);
2758 }
2759
2760 /* enable DMA for all queues */
2761 rtw89_pci_ctrl_txdma_ch_pcie(rtwdev, true);
2762
2763 /* Release PCI IO */
2764 rtw89_write32_clr(rtwdev, info->dma_stop1.addr,
2765 B_AX_STOP_WPDMA | B_AX_STOP_PCIEIO);
2766
2767 return 0;
2768 }
2769
rtw89_pci_claim_device(struct rtw89_dev * rtwdev,struct pci_dev * pdev)2770 static int rtw89_pci_claim_device(struct rtw89_dev *rtwdev,
2771 struct pci_dev *pdev)
2772 {
2773 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2774 int ret;
2775
2776 ret = pci_enable_device(pdev);
2777 if (ret) {
2778 rtw89_err(rtwdev, "failed to enable pci device\n");
2779 return ret;
2780 }
2781
2782 pci_set_master(pdev);
2783 pci_set_drvdata(pdev, rtwdev->hw);
2784
2785 rtwpci->pdev = pdev;
2786
2787 return 0;
2788 }
2789
rtw89_pci_declaim_device(struct rtw89_dev * rtwdev,struct pci_dev * pdev)2790 static void rtw89_pci_declaim_device(struct rtw89_dev *rtwdev,
2791 struct pci_dev *pdev)
2792 {
2793 pci_disable_device(pdev);
2794 }
2795
rtw89_pci_setup_mapping(struct rtw89_dev * rtwdev,struct pci_dev * pdev)2796 static int rtw89_pci_setup_mapping(struct rtw89_dev *rtwdev,
2797 struct pci_dev *pdev)
2798 {
2799 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2800 unsigned long resource_len;
2801 u8 bar_id = 2;
2802 int ret;
2803
2804 ret = pci_request_regions(pdev, KBUILD_MODNAME);
2805 if (ret) {
2806 rtw89_err(rtwdev, "failed to request pci regions\n");
2807 goto err;
2808 }
2809
2810 ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
2811 if (ret) {
2812 rtw89_err(rtwdev, "failed to set dma mask to 32-bit\n");
2813 goto err_release_regions;
2814 }
2815
2816 ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
2817 if (ret) {
2818 rtw89_err(rtwdev, "failed to set consistent dma mask to 32-bit\n");
2819 goto err_release_regions;
2820 }
2821
2822 resource_len = pci_resource_len(pdev, bar_id);
2823 rtwpci->mmap = pci_iomap(pdev, bar_id, resource_len);
2824 if (!rtwpci->mmap) {
2825 rtw89_err(rtwdev, "failed to map pci io\n");
2826 ret = -EIO;
2827 goto err_release_regions;
2828 }
2829
2830 return 0;
2831
2832 err_release_regions:
2833 pci_release_regions(pdev);
2834 err:
2835 return ret;
2836 }
2837
rtw89_pci_clear_mapping(struct rtw89_dev * rtwdev,struct pci_dev * pdev)2838 static void rtw89_pci_clear_mapping(struct rtw89_dev *rtwdev,
2839 struct pci_dev *pdev)
2840 {
2841 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2842
2843 if (rtwpci->mmap) {
2844 pci_iounmap(pdev, rtwpci->mmap);
2845 pci_release_regions(pdev);
2846 }
2847 }
2848
rtw89_pci_free_tx_wd_ring(struct rtw89_dev * rtwdev,struct pci_dev * pdev,struct rtw89_pci_tx_ring * tx_ring)2849 static void rtw89_pci_free_tx_wd_ring(struct rtw89_dev *rtwdev,
2850 struct pci_dev *pdev,
2851 struct rtw89_pci_tx_ring *tx_ring)
2852 {
2853 struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
2854 u8 *head = wd_ring->head;
2855 dma_addr_t dma = wd_ring->dma;
2856 u32 page_size = wd_ring->page_size;
2857 u32 page_num = wd_ring->page_num;
2858 u32 ring_sz = page_size * page_num;
2859
2860 dma_free_coherent(&pdev->dev, ring_sz, head, dma);
2861 wd_ring->head = NULL;
2862 }
2863
rtw89_pci_free_tx_ring(struct rtw89_dev * rtwdev,struct pci_dev * pdev,struct rtw89_pci_tx_ring * tx_ring)2864 static void rtw89_pci_free_tx_ring(struct rtw89_dev *rtwdev,
2865 struct pci_dev *pdev,
2866 struct rtw89_pci_tx_ring *tx_ring)
2867 {
2868 int ring_sz;
2869 u8 *head;
2870 dma_addr_t dma;
2871
2872 head = tx_ring->bd_ring.head;
2873 dma = tx_ring->bd_ring.dma;
2874 ring_sz = tx_ring->bd_ring.desc_size * tx_ring->bd_ring.len;
2875 dma_free_coherent(&pdev->dev, ring_sz, head, dma);
2876
2877 tx_ring->bd_ring.head = NULL;
2878 }
2879
rtw89_pci_free_tx_rings(struct rtw89_dev * rtwdev,struct pci_dev * pdev)2880 static void rtw89_pci_free_tx_rings(struct rtw89_dev *rtwdev,
2881 struct pci_dev *pdev)
2882 {
2883 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2884 const struct rtw89_pci_info *info = rtwdev->pci_info;
2885 struct rtw89_pci_tx_ring *tx_ring;
2886 int i;
2887
2888 for (i = 0; i < RTW89_TXCH_NUM; i++) {
2889 if (info->tx_dma_ch_mask & BIT(i))
2890 continue;
2891 tx_ring = &rtwpci->tx_rings[i];
2892 rtw89_pci_free_tx_wd_ring(rtwdev, pdev, tx_ring);
2893 rtw89_pci_free_tx_ring(rtwdev, pdev, tx_ring);
2894 }
2895 }
2896
rtw89_pci_free_rx_ring(struct rtw89_dev * rtwdev,struct pci_dev * pdev,struct rtw89_pci_rx_ring * rx_ring)2897 static void rtw89_pci_free_rx_ring(struct rtw89_dev *rtwdev,
2898 struct pci_dev *pdev,
2899 struct rtw89_pci_rx_ring *rx_ring)
2900 {
2901 struct rtw89_pci_rx_info *rx_info;
2902 struct sk_buff *skb;
2903 dma_addr_t dma;
2904 u32 buf_sz;
2905 u8 *head;
2906 int ring_sz = rx_ring->bd_ring.desc_size * rx_ring->bd_ring.len;
2907 int i;
2908
2909 buf_sz = rx_ring->buf_sz;
2910 for (i = 0; i < rx_ring->bd_ring.len; i++) {
2911 skb = rx_ring->buf[i];
2912 if (!skb)
2913 continue;
2914
2915 rx_info = RTW89_PCI_RX_SKB_CB(skb);
2916 dma = rx_info->dma;
2917 dma_unmap_single(&pdev->dev, dma, buf_sz, DMA_FROM_DEVICE);
2918 dev_kfree_skb(skb);
2919 rx_ring->buf[i] = NULL;
2920 }
2921
2922 head = rx_ring->bd_ring.head;
2923 dma = rx_ring->bd_ring.dma;
2924 dma_free_coherent(&pdev->dev, ring_sz, head, dma);
2925
2926 rx_ring->bd_ring.head = NULL;
2927 }
2928
rtw89_pci_free_rx_rings(struct rtw89_dev * rtwdev,struct pci_dev * pdev)2929 static void rtw89_pci_free_rx_rings(struct rtw89_dev *rtwdev,
2930 struct pci_dev *pdev)
2931 {
2932 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
2933 struct rtw89_pci_rx_ring *rx_ring;
2934 int i;
2935
2936 for (i = 0; i < RTW89_RXCH_NUM; i++) {
2937 rx_ring = &rtwpci->rx_rings[i];
2938 rtw89_pci_free_rx_ring(rtwdev, pdev, rx_ring);
2939 }
2940 }
2941
rtw89_pci_free_trx_rings(struct rtw89_dev * rtwdev,struct pci_dev * pdev)2942 static void rtw89_pci_free_trx_rings(struct rtw89_dev *rtwdev,
2943 struct pci_dev *pdev)
2944 {
2945 rtw89_pci_free_rx_rings(rtwdev, pdev);
2946 rtw89_pci_free_tx_rings(rtwdev, pdev);
2947 }
2948
rtw89_pci_init_rx_bd(struct rtw89_dev * rtwdev,struct pci_dev * pdev,struct rtw89_pci_rx_ring * rx_ring,struct sk_buff * skb,int buf_sz,u32 idx)2949 static int rtw89_pci_init_rx_bd(struct rtw89_dev *rtwdev, struct pci_dev *pdev,
2950 struct rtw89_pci_rx_ring *rx_ring,
2951 struct sk_buff *skb, int buf_sz, u32 idx)
2952 {
2953 struct rtw89_pci_rx_info *rx_info;
2954 struct rtw89_pci_rx_bd_32 *rx_bd;
2955 dma_addr_t dma;
2956
2957 if (!skb)
2958 return -EINVAL;
2959
2960 dma = dma_map_single(&pdev->dev, skb->data, buf_sz, DMA_FROM_DEVICE);
2961 if (dma_mapping_error(&pdev->dev, dma))
2962 return -EBUSY;
2963
2964 rx_info = RTW89_PCI_RX_SKB_CB(skb);
2965 rx_bd = RTW89_PCI_RX_BD(rx_ring, idx);
2966
2967 memset(rx_bd, 0, sizeof(*rx_bd));
2968 rx_bd->buf_size = cpu_to_le16(buf_sz);
2969 rx_bd->dma = cpu_to_le32(dma);
2970 rx_info->dma = dma;
2971
2972 return 0;
2973 }
2974
rtw89_pci_alloc_tx_wd_ring(struct rtw89_dev * rtwdev,struct pci_dev * pdev,struct rtw89_pci_tx_ring * tx_ring,enum rtw89_tx_channel txch)2975 static int rtw89_pci_alloc_tx_wd_ring(struct rtw89_dev *rtwdev,
2976 struct pci_dev *pdev,
2977 struct rtw89_pci_tx_ring *tx_ring,
2978 enum rtw89_tx_channel txch)
2979 {
2980 struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring;
2981 struct rtw89_pci_tx_wd *txwd;
2982 dma_addr_t dma;
2983 dma_addr_t cur_paddr;
2984 u8 *head;
2985 u8 *cur_vaddr;
2986 u32 page_size = RTW89_PCI_TXWD_PAGE_SIZE;
2987 u32 page_num = RTW89_PCI_TXWD_NUM_MAX;
2988 u32 ring_sz = page_size * page_num;
2989 u32 page_offset;
2990 int i;
2991
2992 /* FWCMD queue doesn't use txwd as pages */
2993 if (txch == RTW89_TXCH_CH12)
2994 return 0;
2995
2996 head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL);
2997 if (!head)
2998 return -ENOMEM;
2999
3000 INIT_LIST_HEAD(&wd_ring->free_pages);
3001 wd_ring->head = head;
3002 wd_ring->dma = dma;
3003 wd_ring->page_size = page_size;
3004 wd_ring->page_num = page_num;
3005
3006 page_offset = 0;
3007 for (i = 0; i < page_num; i++) {
3008 txwd = &wd_ring->pages[i];
3009 cur_paddr = dma + page_offset;
3010 cur_vaddr = head + page_offset;
3011
3012 skb_queue_head_init(&txwd->queue);
3013 INIT_LIST_HEAD(&txwd->list);
3014 txwd->paddr = cur_paddr;
3015 txwd->vaddr = cur_vaddr;
3016 txwd->len = page_size;
3017 txwd->seq = i;
3018 rtw89_pci_enqueue_txwd(tx_ring, txwd);
3019
3020 page_offset += page_size;
3021 }
3022
3023 return 0;
3024 }
3025
rtw89_pci_alloc_tx_ring(struct rtw89_dev * rtwdev,struct pci_dev * pdev,struct rtw89_pci_tx_ring * tx_ring,u32 desc_size,u32 len,enum rtw89_tx_channel txch)3026 static int rtw89_pci_alloc_tx_ring(struct rtw89_dev *rtwdev,
3027 struct pci_dev *pdev,
3028 struct rtw89_pci_tx_ring *tx_ring,
3029 u32 desc_size, u32 len,
3030 enum rtw89_tx_channel txch)
3031 {
3032 const struct rtw89_pci_ch_dma_addr *txch_addr;
3033 int ring_sz = desc_size * len;
3034 u8 *head;
3035 dma_addr_t dma;
3036 int ret;
3037
3038 ret = rtw89_pci_alloc_tx_wd_ring(rtwdev, pdev, tx_ring, txch);
3039 if (ret) {
3040 rtw89_err(rtwdev, "failed to alloc txwd ring of txch %d\n", txch);
3041 goto err;
3042 }
3043
3044 ret = rtw89_pci_get_txch_addrs(rtwdev, txch, &txch_addr);
3045 if (ret) {
3046 rtw89_err(rtwdev, "failed to get address of txch %d", txch);
3047 goto err_free_wd_ring;
3048 }
3049
3050 head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL);
3051 if (!head) {
3052 ret = -ENOMEM;
3053 goto err_free_wd_ring;
3054 }
3055
3056 INIT_LIST_HEAD(&tx_ring->busy_pages);
3057 tx_ring->bd_ring.head = head;
3058 tx_ring->bd_ring.dma = dma;
3059 tx_ring->bd_ring.len = len;
3060 tx_ring->bd_ring.desc_size = desc_size;
3061 tx_ring->bd_ring.addr = *txch_addr;
3062 tx_ring->bd_ring.wp = 0;
3063 tx_ring->bd_ring.rp = 0;
3064 tx_ring->txch = txch;
3065
3066 return 0;
3067
3068 err_free_wd_ring:
3069 rtw89_pci_free_tx_wd_ring(rtwdev, pdev, tx_ring);
3070 err:
3071 return ret;
3072 }
3073
rtw89_pci_alloc_tx_rings(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3074 static int rtw89_pci_alloc_tx_rings(struct rtw89_dev *rtwdev,
3075 struct pci_dev *pdev)
3076 {
3077 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3078 const struct rtw89_pci_info *info = rtwdev->pci_info;
3079 struct rtw89_pci_tx_ring *tx_ring;
3080 u32 desc_size;
3081 u32 len;
3082 u32 i, tx_allocated;
3083 int ret;
3084
3085 for (i = 0; i < RTW89_TXCH_NUM; i++) {
3086 if (info->tx_dma_ch_mask & BIT(i))
3087 continue;
3088 tx_ring = &rtwpci->tx_rings[i];
3089 desc_size = sizeof(struct rtw89_pci_tx_bd_32);
3090 len = RTW89_PCI_TXBD_NUM_MAX;
3091 ret = rtw89_pci_alloc_tx_ring(rtwdev, pdev, tx_ring,
3092 desc_size, len, i);
3093 if (ret) {
3094 rtw89_err(rtwdev, "failed to alloc tx ring %d\n", i);
3095 goto err_free;
3096 }
3097 }
3098
3099 return 0;
3100
3101 err_free:
3102 tx_allocated = i;
3103 for (i = 0; i < tx_allocated; i++) {
3104 tx_ring = &rtwpci->tx_rings[i];
3105 rtw89_pci_free_tx_ring(rtwdev, pdev, tx_ring);
3106 }
3107
3108 return ret;
3109 }
3110
rtw89_pci_alloc_rx_ring(struct rtw89_dev * rtwdev,struct pci_dev * pdev,struct rtw89_pci_rx_ring * rx_ring,u32 desc_size,u32 len,u32 rxch)3111 static int rtw89_pci_alloc_rx_ring(struct rtw89_dev *rtwdev,
3112 struct pci_dev *pdev,
3113 struct rtw89_pci_rx_ring *rx_ring,
3114 u32 desc_size, u32 len, u32 rxch)
3115 {
3116 const struct rtw89_pci_info *info = rtwdev->pci_info;
3117 const struct rtw89_pci_ch_dma_addr *rxch_addr;
3118 struct sk_buff *skb;
3119 u8 *head;
3120 dma_addr_t dma;
3121 int ring_sz = desc_size * len;
3122 int buf_sz = RTW89_PCI_RX_BUF_SIZE;
3123 int i, allocated;
3124 int ret;
3125
3126 ret = rtw89_pci_get_rxch_addrs(rtwdev, rxch, &rxch_addr);
3127 if (ret) {
3128 rtw89_err(rtwdev, "failed to get address of rxch %d", rxch);
3129 return ret;
3130 }
3131
3132 head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL);
3133 if (!head) {
3134 ret = -ENOMEM;
3135 goto err;
3136 }
3137
3138 rx_ring->bd_ring.head = head;
3139 rx_ring->bd_ring.dma = dma;
3140 rx_ring->bd_ring.len = len;
3141 rx_ring->bd_ring.desc_size = desc_size;
3142 rx_ring->bd_ring.addr = *rxch_addr;
3143 if (info->rx_ring_eq_is_full)
3144 rx_ring->bd_ring.wp = len - 1;
3145 else
3146 rx_ring->bd_ring.wp = 0;
3147 rx_ring->bd_ring.rp = 0;
3148 rx_ring->buf_sz = buf_sz;
3149 rx_ring->diliver_skb = NULL;
3150 rx_ring->diliver_desc.ready = false;
3151
3152 for (i = 0; i < len; i++) {
3153 skb = dev_alloc_skb(buf_sz);
3154 if (!skb) {
3155 ret = -ENOMEM;
3156 goto err_free;
3157 }
3158
3159 memset(skb->data, 0, buf_sz);
3160 rx_ring->buf[i] = skb;
3161 ret = rtw89_pci_init_rx_bd(rtwdev, pdev, rx_ring, skb,
3162 buf_sz, i);
3163 if (ret) {
3164 rtw89_err(rtwdev, "failed to init rx buf %d\n", i);
3165 dev_kfree_skb_any(skb);
3166 rx_ring->buf[i] = NULL;
3167 goto err_free;
3168 }
3169 }
3170
3171 return 0;
3172
3173 err_free:
3174 allocated = i;
3175 for (i = 0; i < allocated; i++) {
3176 skb = rx_ring->buf[i];
3177 if (!skb)
3178 continue;
3179 dma = *((dma_addr_t *)skb->cb);
3180 dma_unmap_single(&pdev->dev, dma, buf_sz, DMA_FROM_DEVICE);
3181 dev_kfree_skb(skb);
3182 rx_ring->buf[i] = NULL;
3183 }
3184
3185 head = rx_ring->bd_ring.head;
3186 dma = rx_ring->bd_ring.dma;
3187 dma_free_coherent(&pdev->dev, ring_sz, head, dma);
3188
3189 rx_ring->bd_ring.head = NULL;
3190 err:
3191 return ret;
3192 }
3193
rtw89_pci_alloc_rx_rings(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3194 static int rtw89_pci_alloc_rx_rings(struct rtw89_dev *rtwdev,
3195 struct pci_dev *pdev)
3196 {
3197 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3198 struct rtw89_pci_rx_ring *rx_ring;
3199 u32 desc_size;
3200 u32 len;
3201 int i, rx_allocated;
3202 int ret;
3203
3204 for (i = 0; i < RTW89_RXCH_NUM; i++) {
3205 rx_ring = &rtwpci->rx_rings[i];
3206 desc_size = sizeof(struct rtw89_pci_rx_bd_32);
3207 len = RTW89_PCI_RXBD_NUM_MAX;
3208 ret = rtw89_pci_alloc_rx_ring(rtwdev, pdev, rx_ring,
3209 desc_size, len, i);
3210 if (ret) {
3211 rtw89_err(rtwdev, "failed to alloc rx ring %d\n", i);
3212 goto err_free;
3213 }
3214 }
3215
3216 return 0;
3217
3218 err_free:
3219 rx_allocated = i;
3220 for (i = 0; i < rx_allocated; i++) {
3221 rx_ring = &rtwpci->rx_rings[i];
3222 rtw89_pci_free_rx_ring(rtwdev, pdev, rx_ring);
3223 }
3224
3225 return ret;
3226 }
3227
rtw89_pci_alloc_trx_rings(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3228 static int rtw89_pci_alloc_trx_rings(struct rtw89_dev *rtwdev,
3229 struct pci_dev *pdev)
3230 {
3231 int ret;
3232
3233 ret = rtw89_pci_alloc_tx_rings(rtwdev, pdev);
3234 if (ret) {
3235 rtw89_err(rtwdev, "failed to alloc dma tx rings\n");
3236 goto err;
3237 }
3238
3239 ret = rtw89_pci_alloc_rx_rings(rtwdev, pdev);
3240 if (ret) {
3241 rtw89_err(rtwdev, "failed to alloc dma rx rings\n");
3242 goto err_free_tx_rings;
3243 }
3244
3245 return 0;
3246
3247 err_free_tx_rings:
3248 rtw89_pci_free_tx_rings(rtwdev, pdev);
3249 err:
3250 return ret;
3251 }
3252
rtw89_pci_h2c_init(struct rtw89_dev * rtwdev,struct rtw89_pci * rtwpci)3253 static void rtw89_pci_h2c_init(struct rtw89_dev *rtwdev,
3254 struct rtw89_pci *rtwpci)
3255 {
3256 skb_queue_head_init(&rtwpci->h2c_queue);
3257 skb_queue_head_init(&rtwpci->h2c_release_queue);
3258 }
3259
rtw89_pci_setup_resource(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3260 static int rtw89_pci_setup_resource(struct rtw89_dev *rtwdev,
3261 struct pci_dev *pdev)
3262 {
3263 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3264 int ret;
3265
3266 ret = rtw89_pci_setup_mapping(rtwdev, pdev);
3267 if (ret) {
3268 rtw89_err(rtwdev, "failed to setup pci mapping\n");
3269 goto err;
3270 }
3271
3272 ret = rtw89_pci_alloc_trx_rings(rtwdev, pdev);
3273 if (ret) {
3274 rtw89_err(rtwdev, "failed to alloc pci trx rings\n");
3275 goto err_pci_unmap;
3276 }
3277
3278 rtw89_pci_h2c_init(rtwdev, rtwpci);
3279
3280 spin_lock_init(&rtwpci->irq_lock);
3281 spin_lock_init(&rtwpci->trx_lock);
3282
3283 return 0;
3284
3285 err_pci_unmap:
3286 rtw89_pci_clear_mapping(rtwdev, pdev);
3287 err:
3288 return ret;
3289 }
3290
rtw89_pci_clear_resource(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3291 static void rtw89_pci_clear_resource(struct rtw89_dev *rtwdev,
3292 struct pci_dev *pdev)
3293 {
3294 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3295
3296 rtw89_pci_free_trx_rings(rtwdev, pdev);
3297 rtw89_pci_clear_mapping(rtwdev, pdev);
3298 rtw89_pci_release_fwcmd(rtwdev, rtwpci,
3299 skb_queue_len(&rtwpci->h2c_queue), true);
3300 }
3301
rtw89_pci_config_intr_mask(struct rtw89_dev * rtwdev)3302 void rtw89_pci_config_intr_mask(struct rtw89_dev *rtwdev)
3303 {
3304 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3305 const struct rtw89_chip_info *chip = rtwdev->chip;
3306 u32 hs0isr_ind_int_en = B_AX_HS0ISR_IND_INT_EN;
3307
3308 if (chip->chip_id == RTL8851B)
3309 hs0isr_ind_int_en = B_AX_HS0ISR_IND_INT_EN_WKARND;
3310
3311 rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | 0;
3312
3313 if (rtwpci->under_recovery) {
3314 rtwpci->intrs[0] = hs0isr_ind_int_en;
3315 rtwpci->intrs[1] = 0;
3316 } else {
3317 rtwpci->intrs[0] = B_AX_TXDMA_STUCK_INT_EN |
3318 B_AX_RXDMA_INT_EN |
3319 B_AX_RXP1DMA_INT_EN |
3320 B_AX_RPQDMA_INT_EN |
3321 B_AX_RXDMA_STUCK_INT_EN |
3322 B_AX_RDU_INT_EN |
3323 B_AX_RPQBD_FULL_INT_EN |
3324 hs0isr_ind_int_en;
3325
3326 rtwpci->intrs[1] = B_AX_HC10ISR_IND_INT_EN;
3327 }
3328 }
3329 EXPORT_SYMBOL(rtw89_pci_config_intr_mask);
3330
rtw89_pci_recovery_intr_mask_v1(struct rtw89_dev * rtwdev)3331 static void rtw89_pci_recovery_intr_mask_v1(struct rtw89_dev *rtwdev)
3332 {
3333 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3334
3335 rtwpci->ind_intrs = B_AX_HS0ISR_IND_INT_EN;
3336 rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | B_AX_WDT_TIMEOUT_INT_EN;
3337 rtwpci->intrs[0] = 0;
3338 rtwpci->intrs[1] = 0;
3339 }
3340
rtw89_pci_default_intr_mask_v1(struct rtw89_dev * rtwdev)3341 static void rtw89_pci_default_intr_mask_v1(struct rtw89_dev *rtwdev)
3342 {
3343 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3344
3345 rtwpci->ind_intrs = B_AX_HCI_AXIDMA_INT_EN |
3346 B_AX_HS1ISR_IND_INT_EN |
3347 B_AX_HS0ISR_IND_INT_EN;
3348 rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | B_AX_WDT_TIMEOUT_INT_EN;
3349 rtwpci->intrs[0] = B_AX_TXDMA_STUCK_INT_EN |
3350 B_AX_RXDMA_INT_EN |
3351 B_AX_RXP1DMA_INT_EN |
3352 B_AX_RPQDMA_INT_EN |
3353 B_AX_RXDMA_STUCK_INT_EN |
3354 B_AX_RDU_INT_EN |
3355 B_AX_RPQBD_FULL_INT_EN;
3356 rtwpci->intrs[1] = B_AX_GPIO18_INT_EN;
3357 }
3358
rtw89_pci_low_power_intr_mask_v1(struct rtw89_dev * rtwdev)3359 static void rtw89_pci_low_power_intr_mask_v1(struct rtw89_dev *rtwdev)
3360 {
3361 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3362
3363 rtwpci->ind_intrs = B_AX_HS1ISR_IND_INT_EN |
3364 B_AX_HS0ISR_IND_INT_EN;
3365 rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | B_AX_WDT_TIMEOUT_INT_EN;
3366 rtwpci->intrs[0] = 0;
3367 rtwpci->intrs[1] = B_AX_GPIO18_INT_EN;
3368 }
3369
rtw89_pci_config_intr_mask_v1(struct rtw89_dev * rtwdev)3370 void rtw89_pci_config_intr_mask_v1(struct rtw89_dev *rtwdev)
3371 {
3372 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3373
3374 if (rtwpci->under_recovery)
3375 rtw89_pci_recovery_intr_mask_v1(rtwdev);
3376 else if (rtwpci->low_power)
3377 rtw89_pci_low_power_intr_mask_v1(rtwdev);
3378 else
3379 rtw89_pci_default_intr_mask_v1(rtwdev);
3380 }
3381 EXPORT_SYMBOL(rtw89_pci_config_intr_mask_v1);
3382
rtw89_pci_recovery_intr_mask_v2(struct rtw89_dev * rtwdev)3383 static void rtw89_pci_recovery_intr_mask_v2(struct rtw89_dev *rtwdev)
3384 {
3385 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3386
3387 rtwpci->ind_intrs = B_BE_HS0_IND_INT_EN0;
3388 rtwpci->halt_c2h_intrs = B_BE_HALT_C2H_INT_EN | B_BE_WDT_TIMEOUT_INT_EN;
3389 rtwpci->intrs[0] = 0;
3390 rtwpci->intrs[1] = B_BE_PCIE_RX_RX0P2_IMR0_V1 |
3391 B_BE_PCIE_RX_RPQ0_IMR0_V1;
3392 }
3393
rtw89_pci_default_intr_mask_v2(struct rtw89_dev * rtwdev)3394 static void rtw89_pci_default_intr_mask_v2(struct rtw89_dev *rtwdev)
3395 {
3396 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3397
3398 rtwpci->ind_intrs = B_BE_HCI_AXIDMA_INT_EN0 |
3399 B_BE_HS0_IND_INT_EN0;
3400 rtwpci->halt_c2h_intrs = B_BE_HALT_C2H_INT_EN | B_BE_WDT_TIMEOUT_INT_EN;
3401 rtwpci->intrs[0] = B_BE_RDU_CH1_INT_IMR_V1 |
3402 B_BE_RDU_CH0_INT_IMR_V1;
3403 rtwpci->intrs[1] = B_BE_PCIE_RX_RX0P2_IMR0_V1 |
3404 B_BE_PCIE_RX_RPQ0_IMR0_V1;
3405 }
3406
rtw89_pci_low_power_intr_mask_v2(struct rtw89_dev * rtwdev)3407 static void rtw89_pci_low_power_intr_mask_v2(struct rtw89_dev *rtwdev)
3408 {
3409 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3410
3411 rtwpci->ind_intrs = B_BE_HS0_IND_INT_EN0 |
3412 B_BE_HS1_IND_INT_EN0;
3413 rtwpci->halt_c2h_intrs = B_BE_HALT_C2H_INT_EN | B_BE_WDT_TIMEOUT_INT_EN;
3414 rtwpci->intrs[0] = 0;
3415 rtwpci->intrs[1] = B_BE_PCIE_RX_RX0P2_IMR0_V1 |
3416 B_BE_PCIE_RX_RPQ0_IMR0_V1;
3417 }
3418
rtw89_pci_config_intr_mask_v2(struct rtw89_dev * rtwdev)3419 void rtw89_pci_config_intr_mask_v2(struct rtw89_dev *rtwdev)
3420 {
3421 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3422
3423 if (rtwpci->under_recovery)
3424 rtw89_pci_recovery_intr_mask_v2(rtwdev);
3425 else if (rtwpci->low_power)
3426 rtw89_pci_low_power_intr_mask_v2(rtwdev);
3427 else
3428 rtw89_pci_default_intr_mask_v2(rtwdev);
3429 }
3430 EXPORT_SYMBOL(rtw89_pci_config_intr_mask_v2);
3431
rtw89_pci_request_irq(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3432 static int rtw89_pci_request_irq(struct rtw89_dev *rtwdev,
3433 struct pci_dev *pdev)
3434 {
3435 unsigned long flags = 0;
3436 int ret;
3437
3438 flags |= PCI_IRQ_LEGACY | PCI_IRQ_MSI;
3439 ret = pci_alloc_irq_vectors(pdev, 1, 1, flags);
3440 if (ret < 0) {
3441 rtw89_err(rtwdev, "failed to alloc irq vectors, ret %d\n", ret);
3442 goto err;
3443 }
3444
3445 ret = devm_request_threaded_irq(rtwdev->dev, pdev->irq,
3446 rtw89_pci_interrupt_handler,
3447 rtw89_pci_interrupt_threadfn,
3448 IRQF_SHARED, KBUILD_MODNAME, rtwdev);
3449 if (ret) {
3450 rtw89_err(rtwdev, "failed to request threaded irq\n");
3451 goto err_free_vector;
3452 }
3453
3454 rtw89_chip_config_intr_mask(rtwdev, RTW89_PCI_INTR_MASK_RESET);
3455
3456 return 0;
3457
3458 err_free_vector:
3459 pci_free_irq_vectors(pdev);
3460 err:
3461 return ret;
3462 }
3463
rtw89_pci_free_irq(struct rtw89_dev * rtwdev,struct pci_dev * pdev)3464 static void rtw89_pci_free_irq(struct rtw89_dev *rtwdev,
3465 struct pci_dev *pdev)
3466 {
3467 devm_free_irq(rtwdev->dev, pdev->irq, rtwdev);
3468 pci_free_irq_vectors(pdev);
3469 }
3470
gray_code_to_bin(u16 gray_code,u32 bit_num)3471 static u16 gray_code_to_bin(u16 gray_code, u32 bit_num)
3472 {
3473 u16 bin = 0, gray_bit;
3474 u32 bit_idx;
3475
3476 for (bit_idx = 0; bit_idx < bit_num; bit_idx++) {
3477 gray_bit = (gray_code >> bit_idx) & 0x1;
3478 if (bit_num - bit_idx > 1)
3479 gray_bit ^= (gray_code >> (bit_idx + 1)) & 0x1;
3480 bin |= (gray_bit << bit_idx);
3481 }
3482
3483 return bin;
3484 }
3485
rtw89_pci_filter_out(struct rtw89_dev * rtwdev)3486 static int rtw89_pci_filter_out(struct rtw89_dev *rtwdev)
3487 {
3488 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3489 struct pci_dev *pdev = rtwpci->pdev;
3490 u16 val16, filter_out_val;
3491 u32 val, phy_offset;
3492 int ret;
3493
3494 if (rtwdev->chip->chip_id != RTL8852C)
3495 return 0;
3496
3497 val = rtw89_read32_mask(rtwdev, R_AX_PCIE_MIX_CFG_V1, B_AX_ASPM_CTRL_MASK);
3498 if (val == B_AX_ASPM_CTRL_L1)
3499 return 0;
3500
3501 ret = pci_read_config_dword(pdev, RTW89_PCIE_L1_STS_V1, &val);
3502 if (ret)
3503 return ret;
3504
3505 val = FIELD_GET(RTW89_BCFG_LINK_SPEED_MASK, val);
3506 if (val == RTW89_PCIE_GEN1_SPEED) {
3507 phy_offset = R_RAC_DIRECT_OFFSET_G1;
3508 } else if (val == RTW89_PCIE_GEN2_SPEED) {
3509 phy_offset = R_RAC_DIRECT_OFFSET_G2;
3510 val16 = rtw89_read16(rtwdev, phy_offset + RAC_ANA10 * RAC_MULT);
3511 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA10 * RAC_MULT,
3512 val16 | B_PCIE_BIT_PINOUT_DIS);
3513 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA19 * RAC_MULT,
3514 val16 & ~B_PCIE_BIT_RD_SEL);
3515
3516 val16 = rtw89_read16_mask(rtwdev,
3517 phy_offset + RAC_ANA1F * RAC_MULT,
3518 FILTER_OUT_EQ_MASK);
3519 val16 = gray_code_to_bin(val16, hweight16(val16));
3520 filter_out_val = rtw89_read16(rtwdev, phy_offset + RAC_ANA24 *
3521 RAC_MULT);
3522 filter_out_val &= ~REG_FILTER_OUT_MASK;
3523 filter_out_val |= FIELD_PREP(REG_FILTER_OUT_MASK, val16);
3524
3525 rtw89_write16(rtwdev, phy_offset + RAC_ANA24 * RAC_MULT,
3526 filter_out_val);
3527 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA0A * RAC_MULT,
3528 B_BAC_EQ_SEL);
3529 rtw89_write16_set(rtwdev,
3530 R_RAC_DIRECT_OFFSET_G1 + RAC_ANA0C * RAC_MULT,
3531 B_PCIE_BIT_PSAVE);
3532 } else {
3533 return -EOPNOTSUPP;
3534 }
3535 rtw89_write16_set(rtwdev, phy_offset + RAC_ANA0C * RAC_MULT,
3536 B_PCIE_BIT_PSAVE);
3537
3538 return 0;
3539 }
3540
rtw89_pci_clkreq_set(struct rtw89_dev * rtwdev,bool enable)3541 static void rtw89_pci_clkreq_set(struct rtw89_dev *rtwdev, bool enable)
3542 {
3543 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
3544 int ret;
3545
3546 if (rtw89_pci_disable_clkreq)
3547 return;
3548
3549 ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_CLK_CTRL,
3550 PCIE_CLKDLY_HW_30US);
3551 if (ret)
3552 rtw89_err(rtwdev, "failed to set CLKREQ Delay\n");
3553
3554 if (chip_id == RTL8852A || chip_id == RTL8852B || chip_id == RTL8851B) {
3555 if (enable)
3556 ret = rtw89_pci_config_byte_set(rtwdev,
3557 RTW89_PCIE_L1_CTRL,
3558 RTW89_PCIE_BIT_CLK);
3559 else
3560 ret = rtw89_pci_config_byte_clr(rtwdev,
3561 RTW89_PCIE_L1_CTRL,
3562 RTW89_PCIE_BIT_CLK);
3563 if (ret)
3564 rtw89_err(rtwdev, "failed to %s CLKREQ_L1, ret=%d",
3565 enable ? "set" : "unset", ret);
3566 } else if (chip_id == RTL8852C) {
3567 rtw89_write32_set(rtwdev, R_AX_PCIE_LAT_CTRL,
3568 B_AX_CLK_REQ_SEL_OPT | B_AX_CLK_REQ_SEL);
3569 if (enable)
3570 rtw89_write32_set(rtwdev, R_AX_L1_CLK_CTRL,
3571 B_AX_CLK_REQ_N);
3572 else
3573 rtw89_write32_clr(rtwdev, R_AX_L1_CLK_CTRL,
3574 B_AX_CLK_REQ_N);
3575 }
3576 }
3577
rtw89_pci_aspm_set(struct rtw89_dev * rtwdev,bool enable)3578 static void rtw89_pci_aspm_set(struct rtw89_dev *rtwdev, bool enable)
3579 {
3580 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
3581 u8 value = 0;
3582 int ret;
3583
3584 if (rtw89_pci_disable_aspm_l1)
3585 return;
3586
3587 ret = rtw89_pci_read_config_byte(rtwdev, RTW89_PCIE_ASPM_CTRL, &value);
3588 if (ret)
3589 rtw89_err(rtwdev, "failed to read ASPM Delay\n");
3590
3591 value &= ~(RTW89_L1DLY_MASK | RTW89_L0DLY_MASK);
3592 value |= FIELD_PREP(RTW89_L1DLY_MASK, PCIE_L1DLY_16US) |
3593 FIELD_PREP(RTW89_L0DLY_MASK, PCIE_L0SDLY_4US);
3594
3595 ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_ASPM_CTRL, value);
3596 if (ret)
3597 rtw89_err(rtwdev, "failed to read ASPM Delay\n");
3598
3599 if (chip_id == RTL8852A || chip_id == RTL8852B || chip_id == RTL8851B) {
3600 if (enable)
3601 ret = rtw89_pci_config_byte_set(rtwdev,
3602 RTW89_PCIE_L1_CTRL,
3603 RTW89_PCIE_BIT_L1);
3604 else
3605 ret = rtw89_pci_config_byte_clr(rtwdev,
3606 RTW89_PCIE_L1_CTRL,
3607 RTW89_PCIE_BIT_L1);
3608 } else if (chip_id == RTL8852C) {
3609 if (enable)
3610 rtw89_write32_set(rtwdev, R_AX_PCIE_MIX_CFG_V1,
3611 B_AX_ASPM_CTRL_L1);
3612 else
3613 rtw89_write32_clr(rtwdev, R_AX_PCIE_MIX_CFG_V1,
3614 B_AX_ASPM_CTRL_L1);
3615 }
3616 if (ret)
3617 rtw89_err(rtwdev, "failed to %s ASPM L1, ret=%d",
3618 enable ? "set" : "unset", ret);
3619 }
3620
rtw89_pci_recalc_int_mit(struct rtw89_dev * rtwdev)3621 static void rtw89_pci_recalc_int_mit(struct rtw89_dev *rtwdev)
3622 {
3623 enum rtw89_chip_gen chip_gen = rtwdev->chip->chip_gen;
3624 const struct rtw89_pci_info *info = rtwdev->pci_info;
3625 struct rtw89_traffic_stats *stats = &rtwdev->stats;
3626 enum rtw89_tfc_lv tx_tfc_lv = stats->tx_tfc_lv;
3627 enum rtw89_tfc_lv rx_tfc_lv = stats->rx_tfc_lv;
3628 u32 val = 0;
3629
3630 if (rtwdev->scanning ||
3631 (tx_tfc_lv < RTW89_TFC_HIGH && rx_tfc_lv < RTW89_TFC_HIGH))
3632 goto out;
3633
3634 if (chip_gen == RTW89_CHIP_BE)
3635 val = B_BE_PCIE_MIT_RX0P2_EN | B_BE_PCIE_MIT_RX0P1_EN;
3636 else
3637 val = B_AX_RXMIT_RXP2_SEL | B_AX_RXMIT_RXP1_SEL |
3638 FIELD_PREP(B_AX_RXCOUNTER_MATCH_MASK, RTW89_PCI_RXBD_NUM_MAX / 2) |
3639 FIELD_PREP(B_AX_RXTIMER_UNIT_MASK, AX_RXTIMER_UNIT_64US) |
3640 FIELD_PREP(B_AX_RXTIMER_MATCH_MASK, 2048 / 64);
3641
3642 out:
3643 rtw89_write32(rtwdev, info->mit_addr, val);
3644 }
3645
rtw89_pci_link_cfg(struct rtw89_dev * rtwdev)3646 static void rtw89_pci_link_cfg(struct rtw89_dev *rtwdev)
3647 {
3648 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3649 struct pci_dev *pdev = rtwpci->pdev;
3650 u16 link_ctrl;
3651 int ret;
3652
3653 /* Though there is standard PCIE configuration space to set the
3654 * link control register, but by Realtek's design, driver should
3655 * check if host supports CLKREQ/ASPM to enable the HW module.
3656 *
3657 * These functions are implemented by two HW modules associated,
3658 * one is responsible to access PCIE configuration space to
3659 * follow the host settings, and another is in charge of doing
3660 * CLKREQ/ASPM mechanisms, it is default disabled. Because sometimes
3661 * the host does not support it, and due to some reasons or wrong
3662 * settings (ex. CLKREQ# not Bi-Direction), it could lead to device
3663 * loss if HW misbehaves on the link.
3664 *
3665 * Hence it's designed that driver should first check the PCIE
3666 * configuration space is sync'ed and enabled, then driver can turn
3667 * on the other module that is actually working on the mechanism.
3668 */
3669 ret = pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, &link_ctrl);
3670 if (ret) {
3671 rtw89_err(rtwdev, "failed to read PCI cap, ret=%d\n", ret);
3672 return;
3673 }
3674
3675 if (link_ctrl & PCI_EXP_LNKCTL_CLKREQ_EN)
3676 rtw89_pci_clkreq_set(rtwdev, true);
3677
3678 if (link_ctrl & PCI_EXP_LNKCTL_ASPM_L1)
3679 rtw89_pci_aspm_set(rtwdev, true);
3680 }
3681
rtw89_pci_l1ss_set(struct rtw89_dev * rtwdev,bool enable)3682 static void rtw89_pci_l1ss_set(struct rtw89_dev *rtwdev, bool enable)
3683 {
3684 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
3685 int ret;
3686
3687 if (chip_id == RTL8852A || chip_id == RTL8852B || chip_id == RTL8851B) {
3688 if (enable)
3689 ret = rtw89_pci_config_byte_set(rtwdev,
3690 RTW89_PCIE_TIMER_CTRL,
3691 RTW89_PCIE_BIT_L1SUB);
3692 else
3693 ret = rtw89_pci_config_byte_clr(rtwdev,
3694 RTW89_PCIE_TIMER_CTRL,
3695 RTW89_PCIE_BIT_L1SUB);
3696 if (ret)
3697 rtw89_err(rtwdev, "failed to %s L1SS, ret=%d",
3698 enable ? "set" : "unset", ret);
3699 } else if (chip_id == RTL8852C) {
3700 ret = rtw89_pci_config_byte_clr(rtwdev, RTW89_PCIE_L1SS_STS_V1,
3701 RTW89_PCIE_BIT_ASPM_L11 |
3702 RTW89_PCIE_BIT_PCI_L11);
3703 if (ret)
3704 rtw89_warn(rtwdev, "failed to unset ASPM L1.1, ret=%d", ret);
3705 if (enable)
3706 rtw89_write32_clr(rtwdev, R_AX_PCIE_MIX_CFG_V1,
3707 B_AX_L1SUB_DISABLE);
3708 else
3709 rtw89_write32_set(rtwdev, R_AX_PCIE_MIX_CFG_V1,
3710 B_AX_L1SUB_DISABLE);
3711 }
3712 }
3713
rtw89_pci_l1ss_cfg(struct rtw89_dev * rtwdev)3714 static void rtw89_pci_l1ss_cfg(struct rtw89_dev *rtwdev)
3715 {
3716 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3717 struct pci_dev *pdev = rtwpci->pdev;
3718 u32 l1ss_cap_ptr, l1ss_ctrl;
3719
3720 if (rtw89_pci_disable_l1ss)
3721 return;
3722
3723 l1ss_cap_ptr = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_L1SS);
3724 if (!l1ss_cap_ptr)
3725 return;
3726
3727 pci_read_config_dword(pdev, l1ss_cap_ptr + PCI_L1SS_CTL1, &l1ss_ctrl);
3728
3729 if (l1ss_ctrl & PCI_L1SS_CTL1_L1SS_MASK)
3730 rtw89_pci_l1ss_set(rtwdev, true);
3731 }
3732
rtw89_pci_poll_io_idle_ax(struct rtw89_dev * rtwdev)3733 static int rtw89_pci_poll_io_idle_ax(struct rtw89_dev *rtwdev)
3734 {
3735 int ret = 0;
3736 u32 sts;
3737 u32 busy = B_AX_PCIEIO_BUSY | B_AX_PCIEIO_TX_BUSY | B_AX_PCIEIO_RX_BUSY;
3738
3739 ret = read_poll_timeout_atomic(rtw89_read32, sts, (sts & busy) == 0x0,
3740 10, 1000, false, rtwdev,
3741 R_AX_PCIE_DMA_BUSY1);
3742 if (ret) {
3743 rtw89_err(rtwdev, "pci dmach busy1 0x%X\n",
3744 rtw89_read32(rtwdev, R_AX_PCIE_DMA_BUSY1));
3745 return -EINVAL;
3746 }
3747 return ret;
3748 }
3749
rtw89_pci_lv1rst_stop_dma_ax(struct rtw89_dev * rtwdev)3750 static int rtw89_pci_lv1rst_stop_dma_ax(struct rtw89_dev *rtwdev)
3751 {
3752 u32 val;
3753 int ret;
3754
3755 if (rtwdev->chip->chip_id == RTL8852C)
3756 return 0;
3757
3758 rtw89_pci_ctrl_dma_all(rtwdev, false);
3759 ret = rtw89_pci_poll_io_idle_ax(rtwdev);
3760 if (ret) {
3761 val = rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG);
3762 rtw89_debug(rtwdev, RTW89_DBG_HCI,
3763 "[PCIe] poll_io_idle fail, before 0x%08x: 0x%08x\n",
3764 R_AX_DBG_ERR_FLAG, val);
3765 if (val & B_AX_TX_STUCK || val & B_AX_PCIE_TXBD_LEN0)
3766 rtw89_mac_ctrl_hci_dma_tx(rtwdev, false);
3767 if (val & B_AX_RX_STUCK)
3768 rtw89_mac_ctrl_hci_dma_rx(rtwdev, false);
3769 rtw89_mac_ctrl_hci_dma_trx(rtwdev, true);
3770 ret = rtw89_pci_poll_io_idle_ax(rtwdev);
3771 val = rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG);
3772 rtw89_debug(rtwdev, RTW89_DBG_HCI,
3773 "[PCIe] poll_io_idle fail, after 0x%08x: 0x%08x\n",
3774 R_AX_DBG_ERR_FLAG, val);
3775 }
3776
3777 return ret;
3778 }
3779
rtw89_pci_lv1rst_start_dma_ax(struct rtw89_dev * rtwdev)3780 static int rtw89_pci_lv1rst_start_dma_ax(struct rtw89_dev *rtwdev)
3781 {
3782 u32 ret;
3783
3784 if (rtwdev->chip->chip_id == RTL8852C)
3785 return 0;
3786
3787 rtw89_mac_ctrl_hci_dma_trx(rtwdev, false);
3788 rtw89_mac_ctrl_hci_dma_trx(rtwdev, true);
3789 rtw89_pci_clr_idx_all(rtwdev);
3790
3791 ret = rtw89_pci_rst_bdram_ax(rtwdev);
3792 if (ret)
3793 return ret;
3794
3795 rtw89_pci_ctrl_dma_all(rtwdev, true);
3796 return ret;
3797 }
3798
rtw89_pci_ops_mac_lv1_recovery(struct rtw89_dev * rtwdev,enum rtw89_lv1_rcvy_step step)3799 static int rtw89_pci_ops_mac_lv1_recovery(struct rtw89_dev *rtwdev,
3800 enum rtw89_lv1_rcvy_step step)
3801 {
3802 const struct rtw89_pci_info *info = rtwdev->pci_info;
3803 const struct rtw89_pci_gen_def *gen_def = info->gen_def;
3804 int ret;
3805
3806 switch (step) {
3807 case RTW89_LV1_RCVY_STEP_1:
3808 ret = gen_def->lv1rst_stop_dma(rtwdev);
3809 if (ret)
3810 rtw89_err(rtwdev, "lv1 rcvy pci stop dma fail\n");
3811
3812 break;
3813
3814 case RTW89_LV1_RCVY_STEP_2:
3815 ret = gen_def->lv1rst_start_dma(rtwdev);
3816 if (ret)
3817 rtw89_err(rtwdev, "lv1 rcvy pci start dma fail\n");
3818 break;
3819
3820 default:
3821 return -EINVAL;
3822 }
3823
3824 return ret;
3825 }
3826
rtw89_pci_ops_dump_err_status(struct rtw89_dev * rtwdev)3827 static void rtw89_pci_ops_dump_err_status(struct rtw89_dev *rtwdev)
3828 {
3829 if (rtwdev->chip->chip_gen == RTW89_CHIP_BE)
3830 return;
3831
3832 if (rtwdev->chip->chip_id == RTL8852C) {
3833 rtw89_info(rtwdev, "R_AX_DBG_ERR_FLAG=0x%08x\n",
3834 rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG_V1));
3835 rtw89_info(rtwdev, "R_AX_LBC_WATCHDOG=0x%08x\n",
3836 rtw89_read32(rtwdev, R_AX_LBC_WATCHDOG_V1));
3837 } else {
3838 rtw89_info(rtwdev, "R_AX_RPQ_RXBD_IDX =0x%08x\n",
3839 rtw89_read32(rtwdev, R_AX_RPQ_RXBD_IDX));
3840 rtw89_info(rtwdev, "R_AX_DBG_ERR_FLAG=0x%08x\n",
3841 rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG));
3842 rtw89_info(rtwdev, "R_AX_LBC_WATCHDOG=0x%08x\n",
3843 rtw89_read32(rtwdev, R_AX_LBC_WATCHDOG));
3844 }
3845 }
3846
rtw89_pci_napi_poll(struct napi_struct * napi,int budget)3847 static int rtw89_pci_napi_poll(struct napi_struct *napi, int budget)
3848 {
3849 struct rtw89_dev *rtwdev = container_of(napi, struct rtw89_dev, napi);
3850 struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv;
3851 const struct rtw89_pci_info *info = rtwdev->pci_info;
3852 const struct rtw89_pci_gen_def *gen_def = info->gen_def;
3853 unsigned long flags;
3854 int work_done;
3855
3856 rtwdev->napi_budget_countdown = budget;
3857
3858 rtw89_write32(rtwdev, gen_def->isr_clear_rpq.addr, gen_def->isr_clear_rpq.data);
3859 work_done = rtw89_pci_poll_rpq_dma(rtwdev, rtwpci, rtwdev->napi_budget_countdown);
3860 if (work_done == budget)
3861 return budget;
3862
3863 rtw89_write32(rtwdev, gen_def->isr_clear_rxq.addr, gen_def->isr_clear_rxq.data);
3864 work_done += rtw89_pci_poll_rxq_dma(rtwdev, rtwpci, rtwdev->napi_budget_countdown);
3865 if (work_done < budget && napi_complete_done(napi, work_done)) {
3866 spin_lock_irqsave(&rtwpci->irq_lock, flags);
3867 if (likely(rtwpci->running))
3868 rtw89_chip_enable_intr(rtwdev, rtwpci);
3869 spin_unlock_irqrestore(&rtwpci->irq_lock, flags);
3870 }
3871
3872 return work_done;
3873 }
3874
rtw89_pci_suspend(struct device * dev)3875 static int __maybe_unused rtw89_pci_suspend(struct device *dev)
3876 {
3877 struct ieee80211_hw *hw = dev_get_drvdata(dev);
3878 struct rtw89_dev *rtwdev = hw->priv;
3879 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
3880
3881 rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
3882 rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_R_DIS_PRST);
3883 rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
3884 if (chip_id == RTL8852A || chip_id == RTL8852B || chip_id == RTL8851B) {
3885 rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL,
3886 B_AX_PCIE_DIS_L2_CTRL_LDO_HCI);
3887 rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1,
3888 B_AX_PCIE_PERST_KEEP_REG | B_AX_PCIE_TRAIN_KEEP_REG);
3889 } else {
3890 rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1,
3891 B_AX_CMAC_EXIT_L1_EN | B_AX_DMAC0_EXIT_L1_EN);
3892 }
3893
3894 return 0;
3895 }
3896
rtw89_pci_l2_hci_ldo(struct rtw89_dev * rtwdev)3897 static void rtw89_pci_l2_hci_ldo(struct rtw89_dev *rtwdev)
3898 {
3899 if (rtwdev->chip->chip_id == RTL8852C)
3900 return;
3901
3902 /* Hardware need write the reg twice to ensure the setting work */
3903 rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_RST_MSTATE,
3904 RTW89_PCIE_BIT_CFG_RST_MSTATE);
3905 rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_RST_MSTATE,
3906 RTW89_PCIE_BIT_CFG_RST_MSTATE);
3907 }
3908
rtw89_pci_resume(struct device * dev)3909 static int __maybe_unused rtw89_pci_resume(struct device *dev)
3910 {
3911 struct ieee80211_hw *hw = dev_get_drvdata(dev);
3912 struct rtw89_dev *rtwdev = hw->priv;
3913 enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id;
3914
3915 rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
3916 rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_R_DIS_PRST);
3917 rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6);
3918 if (chip_id == RTL8852A || chip_id == RTL8852B || chip_id == RTL8851B) {
3919 rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL,
3920 B_AX_PCIE_DIS_L2_CTRL_LDO_HCI);
3921 rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1,
3922 B_AX_PCIE_PERST_KEEP_REG | B_AX_PCIE_TRAIN_KEEP_REG);
3923 } else {
3924 rtw89_write32_set(rtwdev, R_AX_PCIE_PS_CTRL_V1,
3925 B_AX_CMAC_EXIT_L1_EN | B_AX_DMAC0_EXIT_L1_EN);
3926 rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL_V1,
3927 B_AX_SEL_REQ_ENTR_L1);
3928 }
3929 rtw89_pci_l2_hci_ldo(rtwdev);
3930 rtw89_pci_filter_out(rtwdev);
3931 rtw89_pci_link_cfg(rtwdev);
3932 rtw89_pci_l1ss_cfg(rtwdev);
3933
3934 return 0;
3935 }
3936
3937 SIMPLE_DEV_PM_OPS(rtw89_pm_ops, rtw89_pci_suspend, rtw89_pci_resume);
3938 EXPORT_SYMBOL(rtw89_pm_ops);
3939
3940 const struct rtw89_pci_gen_def rtw89_pci_gen_ax = {
3941 .isr_rdu = B_AX_RDU_INT,
3942 .isr_halt_c2h = B_AX_HALT_C2H_INT_EN,
3943 .isr_wdt_timeout = B_AX_WDT_TIMEOUT_INT_EN,
3944 .isr_clear_rpq = {R_AX_PCIE_HISR00, B_AX_RPQDMA_INT | B_AX_RPQBD_FULL_INT},
3945 .isr_clear_rxq = {R_AX_PCIE_HISR00, B_AX_RXP1DMA_INT | B_AX_RXDMA_INT |
3946 B_AX_RDU_INT},
3947
3948 .mac_pre_init = rtw89_pci_ops_mac_pre_init_ax,
3949 .mac_pre_deinit = NULL,
3950 .mac_post_init = rtw89_pci_ops_mac_post_init_ax,
3951
3952 .clr_idx_all = rtw89_pci_clr_idx_all_ax,
3953 .rst_bdram = rtw89_pci_rst_bdram_ax,
3954
3955 .lv1rst_stop_dma = rtw89_pci_lv1rst_stop_dma_ax,
3956 .lv1rst_start_dma = rtw89_pci_lv1rst_start_dma_ax,
3957 };
3958 EXPORT_SYMBOL(rtw89_pci_gen_ax);
3959
3960 static const struct rtw89_hci_ops rtw89_pci_ops = {
3961 .tx_write = rtw89_pci_ops_tx_write,
3962 .tx_kick_off = rtw89_pci_ops_tx_kick_off,
3963 .flush_queues = rtw89_pci_ops_flush_queues,
3964 .reset = rtw89_pci_ops_reset,
3965 .start = rtw89_pci_ops_start,
3966 .stop = rtw89_pci_ops_stop,
3967 .pause = rtw89_pci_ops_pause,
3968 .switch_mode = rtw89_pci_ops_switch_mode,
3969 .recalc_int_mit = rtw89_pci_recalc_int_mit,
3970
3971 .read8 = rtw89_pci_ops_read8,
3972 .read16 = rtw89_pci_ops_read16,
3973 .read32 = rtw89_pci_ops_read32,
3974 .write8 = rtw89_pci_ops_write8,
3975 .write16 = rtw89_pci_ops_write16,
3976 .write32 = rtw89_pci_ops_write32,
3977
3978 .mac_pre_init = rtw89_pci_ops_mac_pre_init,
3979 .mac_pre_deinit = rtw89_pci_ops_mac_pre_deinit,
3980 .mac_post_init = rtw89_pci_ops_mac_post_init,
3981 .deinit = rtw89_pci_ops_deinit,
3982
3983 .check_and_reclaim_tx_resource = rtw89_pci_check_and_reclaim_tx_resource,
3984 .mac_lv1_rcvy = rtw89_pci_ops_mac_lv1_recovery,
3985 .dump_err_status = rtw89_pci_ops_dump_err_status,
3986 .napi_poll = rtw89_pci_napi_poll,
3987
3988 .recovery_start = rtw89_pci_ops_recovery_start,
3989 .recovery_complete = rtw89_pci_ops_recovery_complete,
3990
3991 .ctrl_txdma_ch = rtw89_pci_ctrl_txdma_ch_pcie,
3992 .ctrl_txdma_fw_ch = rtw89_pci_ctrl_txdma_fw_ch_pcie,
3993 .ctrl_trxhci = rtw89_pci_ctrl_dma_trx,
3994 .poll_txdma_ch = rtw89_poll_txdma_ch_idle_pcie,
3995 .clr_idx_all = rtw89_pci_clr_idx_all,
3996 .clear = rtw89_pci_clear_resource,
3997 .disable_intr = rtw89_pci_disable_intr_lock,
3998 .enable_intr = rtw89_pci_enable_intr_lock,
3999 .rst_bdram = rtw89_pci_reset_bdram,
4000 };
4001
rtw89_pci_probe(struct pci_dev * pdev,const struct pci_device_id * id)4002 int rtw89_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
4003 {
4004 struct rtw89_dev *rtwdev;
4005 const struct rtw89_driver_info *info;
4006 const struct rtw89_pci_info *pci_info;
4007 int ret;
4008
4009 info = (const struct rtw89_driver_info *)id->driver_data;
4010
4011 rtwdev = rtw89_alloc_ieee80211_hw(&pdev->dev,
4012 sizeof(struct rtw89_pci),
4013 info->chip);
4014 if (!rtwdev) {
4015 dev_err(&pdev->dev, "failed to allocate hw\n");
4016 return -ENOMEM;
4017 }
4018
4019 pci_info = info->bus.pci;
4020
4021 rtwdev->pci_info = info->bus.pci;
4022 rtwdev->hci.ops = &rtw89_pci_ops;
4023 rtwdev->hci.type = RTW89_HCI_TYPE_PCIE;
4024 rtwdev->hci.rpwm_addr = pci_info->rpwm_addr;
4025 rtwdev->hci.cpwm_addr = pci_info->cpwm_addr;
4026
4027 SET_IEEE80211_DEV(rtwdev->hw, &pdev->dev);
4028
4029 ret = rtw89_core_init(rtwdev);
4030 if (ret) {
4031 rtw89_err(rtwdev, "failed to initialise core\n");
4032 goto err_release_hw;
4033 }
4034
4035 ret = rtw89_pci_claim_device(rtwdev, pdev);
4036 if (ret) {
4037 rtw89_err(rtwdev, "failed to claim pci device\n");
4038 goto err_core_deinit;
4039 }
4040
4041 ret = rtw89_pci_setup_resource(rtwdev, pdev);
4042 if (ret) {
4043 rtw89_err(rtwdev, "failed to setup pci resource\n");
4044 goto err_declaim_pci;
4045 }
4046
4047 ret = rtw89_chip_info_setup(rtwdev);
4048 if (ret) {
4049 rtw89_err(rtwdev, "failed to setup chip information\n");
4050 goto err_clear_resource;
4051 }
4052
4053 rtw89_pci_filter_out(rtwdev);
4054 rtw89_pci_link_cfg(rtwdev);
4055 rtw89_pci_l1ss_cfg(rtwdev);
4056
4057 rtw89_core_napi_init(rtwdev);
4058
4059 ret = rtw89_pci_request_irq(rtwdev, pdev);
4060 if (ret) {
4061 rtw89_err(rtwdev, "failed to request pci irq\n");
4062 goto err_deinit_napi;
4063 }
4064
4065 ret = rtw89_core_register(rtwdev);
4066 if (ret) {
4067 rtw89_err(rtwdev, "failed to register core\n");
4068 goto err_free_irq;
4069 }
4070
4071 return 0;
4072
4073 err_free_irq:
4074 rtw89_pci_free_irq(rtwdev, pdev);
4075 err_deinit_napi:
4076 rtw89_core_napi_deinit(rtwdev);
4077 err_clear_resource:
4078 rtw89_pci_clear_resource(rtwdev, pdev);
4079 err_declaim_pci:
4080 rtw89_pci_declaim_device(rtwdev, pdev);
4081 err_core_deinit:
4082 rtw89_core_deinit(rtwdev);
4083 err_release_hw:
4084 rtw89_free_ieee80211_hw(rtwdev);
4085
4086 return ret;
4087 }
4088 EXPORT_SYMBOL(rtw89_pci_probe);
4089
rtw89_pci_remove(struct pci_dev * pdev)4090 void rtw89_pci_remove(struct pci_dev *pdev)
4091 {
4092 struct ieee80211_hw *hw = pci_get_drvdata(pdev);
4093 struct rtw89_dev *rtwdev;
4094
4095 rtwdev = hw->priv;
4096
4097 rtw89_pci_free_irq(rtwdev, pdev);
4098 rtw89_core_napi_deinit(rtwdev);
4099 rtw89_core_unregister(rtwdev);
4100 rtw89_pci_clear_resource(rtwdev, pdev);
4101 rtw89_pci_declaim_device(rtwdev, pdev);
4102 rtw89_core_deinit(rtwdev);
4103 rtw89_free_ieee80211_hw(rtwdev);
4104 }
4105 EXPORT_SYMBOL(rtw89_pci_remove);
4106
4107 MODULE_AUTHOR("Realtek Corporation");
4108 MODULE_DESCRIPTION("Realtek PCI 802.11ax wireless driver");
4109 MODULE_LICENSE("Dual BSD/GPL");
4110