1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
2 /* QLogic qed NIC Driver
3 * Copyright (c) 2015-2017 QLogic Corporation
4 * Copyright (c) 2019-2020 Marvell International Ltd.
5 */
6
7 #include <linux/types.h>
8 #include <linux/io.h>
9 #include <linux/delay.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/errno.h>
12 #include <linux/kernel.h>
13 #include <linux/list.h>
14 #include <linux/mutex.h>
15 #include <linux/pci.h>
16 #include <linux/slab.h>
17 #include <linux/spinlock.h>
18 #include <linux/string.h>
19 #include <linux/qed/qed_chain.h>
20 #include "qed.h"
21 #include "qed_hsi.h"
22 #include "qed_hw.h"
23 #include "qed_reg_addr.h"
24 #include "qed_sriov.h"
25
26 #define QED_BAR_ACQUIRE_TIMEOUT_USLEEP_CNT 1000
27 #define QED_BAR_ACQUIRE_TIMEOUT_USLEEP 1000
28 #define QED_BAR_ACQUIRE_TIMEOUT_UDELAY_CNT 100000
29 #define QED_BAR_ACQUIRE_TIMEOUT_UDELAY 10
30
31 /* Invalid values */
32 #define QED_BAR_INVALID_OFFSET (cpu_to_le32(-1))
33
34 struct qed_ptt {
35 struct list_head list_entry;
36 unsigned int idx;
37 struct pxp_ptt_entry pxp;
38 u8 hwfn_id;
39 };
40
41 struct qed_ptt_pool {
42 struct list_head free_list;
43 spinlock_t lock; /* ptt synchronized access */
44 struct qed_ptt ptts[PXP_EXTERNAL_BAR_PF_WINDOW_NUM];
45 };
46
qed_ptt_pool_alloc(struct qed_hwfn * p_hwfn)47 int qed_ptt_pool_alloc(struct qed_hwfn *p_hwfn)
48 {
49 struct qed_ptt_pool *p_pool = kmalloc(sizeof(*p_pool), GFP_KERNEL);
50 int i;
51
52 if (!p_pool)
53 return -ENOMEM;
54
55 INIT_LIST_HEAD(&p_pool->free_list);
56 for (i = 0; i < PXP_EXTERNAL_BAR_PF_WINDOW_NUM; i++) {
57 p_pool->ptts[i].idx = i;
58 p_pool->ptts[i].pxp.offset = QED_BAR_INVALID_OFFSET;
59 p_pool->ptts[i].pxp.pretend.control = 0;
60 p_pool->ptts[i].hwfn_id = p_hwfn->my_id;
61 if (i >= RESERVED_PTT_MAX)
62 list_add(&p_pool->ptts[i].list_entry,
63 &p_pool->free_list);
64 }
65
66 p_hwfn->p_ptt_pool = p_pool;
67 spin_lock_init(&p_pool->lock);
68
69 return 0;
70 }
71
qed_ptt_pool_free(struct qed_hwfn * p_hwfn)72 void qed_ptt_pool_free(struct qed_hwfn *p_hwfn)
73 {
74 kfree(p_hwfn->p_ptt_pool);
75 p_hwfn->p_ptt_pool = NULL;
76 }
77
qed_ptt_acquire(struct qed_hwfn * p_hwfn)78 struct qed_ptt *qed_ptt_acquire(struct qed_hwfn *p_hwfn)
79 {
80 return qed_ptt_acquire_context(p_hwfn, false);
81 }
82
qed_ptt_acquire_context(struct qed_hwfn * p_hwfn,bool is_atomic)83 struct qed_ptt *qed_ptt_acquire_context(struct qed_hwfn *p_hwfn, bool is_atomic)
84 {
85 struct qed_ptt *p_ptt;
86 unsigned int i, count;
87
88 if (is_atomic)
89 count = QED_BAR_ACQUIRE_TIMEOUT_UDELAY_CNT;
90 else
91 count = QED_BAR_ACQUIRE_TIMEOUT_USLEEP_CNT;
92
93 /* Take the free PTT from the list */
94 for (i = 0; i < count; i++) {
95 spin_lock_bh(&p_hwfn->p_ptt_pool->lock);
96
97 if (!list_empty(&p_hwfn->p_ptt_pool->free_list)) {
98 p_ptt = list_first_entry(&p_hwfn->p_ptt_pool->free_list,
99 struct qed_ptt, list_entry);
100 list_del(&p_ptt->list_entry);
101
102 spin_unlock_bh(&p_hwfn->p_ptt_pool->lock);
103
104 DP_VERBOSE(p_hwfn, NETIF_MSG_HW,
105 "allocated ptt %d\n", p_ptt->idx);
106 return p_ptt;
107 }
108
109 spin_unlock_bh(&p_hwfn->p_ptt_pool->lock);
110
111 if (is_atomic)
112 udelay(QED_BAR_ACQUIRE_TIMEOUT_UDELAY);
113 else
114 usleep_range(QED_BAR_ACQUIRE_TIMEOUT_USLEEP,
115 QED_BAR_ACQUIRE_TIMEOUT_USLEEP * 2);
116 }
117
118 DP_NOTICE(p_hwfn, "PTT acquire timeout - failed to allocate PTT\n");
119 return NULL;
120 }
121
qed_ptt_release(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt)122 void qed_ptt_release(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
123 {
124 spin_lock_bh(&p_hwfn->p_ptt_pool->lock);
125 list_add(&p_ptt->list_entry, &p_hwfn->p_ptt_pool->free_list);
126 spin_unlock_bh(&p_hwfn->p_ptt_pool->lock);
127 }
128
qed_ptt_get_hw_addr(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt)129 u32 qed_ptt_get_hw_addr(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
130 {
131 /* The HW is using DWORDS and we need to translate it to Bytes */
132 return le32_to_cpu(p_ptt->pxp.offset) << 2;
133 }
134
qed_ptt_config_addr(struct qed_ptt * p_ptt)135 static u32 qed_ptt_config_addr(struct qed_ptt *p_ptt)
136 {
137 return PXP_PF_WINDOW_ADMIN_PER_PF_START +
138 p_ptt->idx * sizeof(struct pxp_ptt_entry);
139 }
140
qed_ptt_get_bar_addr(struct qed_ptt * p_ptt)141 u32 qed_ptt_get_bar_addr(struct qed_ptt *p_ptt)
142 {
143 return PXP_EXTERNAL_BAR_PF_WINDOW_START +
144 p_ptt->idx * PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE;
145 }
146
qed_ptt_set_win(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt,u32 new_hw_addr)147 void qed_ptt_set_win(struct qed_hwfn *p_hwfn,
148 struct qed_ptt *p_ptt, u32 new_hw_addr)
149 {
150 u32 prev_hw_addr;
151
152 prev_hw_addr = qed_ptt_get_hw_addr(p_hwfn, p_ptt);
153
154 if (new_hw_addr == prev_hw_addr)
155 return;
156
157 /* Update PTT entery in admin window */
158 DP_VERBOSE(p_hwfn, NETIF_MSG_HW,
159 "Updating PTT entry %d to offset 0x%x\n",
160 p_ptt->idx, new_hw_addr);
161
162 /* The HW is using DWORDS and the address is in Bytes */
163 p_ptt->pxp.offset = cpu_to_le32(new_hw_addr >> 2);
164
165 REG_WR(p_hwfn,
166 qed_ptt_config_addr(p_ptt) +
167 offsetof(struct pxp_ptt_entry, offset),
168 le32_to_cpu(p_ptt->pxp.offset));
169 }
170
qed_set_ptt(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt,u32 hw_addr)171 static u32 qed_set_ptt(struct qed_hwfn *p_hwfn,
172 struct qed_ptt *p_ptt, u32 hw_addr)
173 {
174 u32 win_hw_addr = qed_ptt_get_hw_addr(p_hwfn, p_ptt);
175 u32 offset;
176
177 offset = hw_addr - win_hw_addr;
178
179 if (p_ptt->hwfn_id != p_hwfn->my_id)
180 DP_NOTICE(p_hwfn,
181 "ptt[%d] of hwfn[%02x] is used by hwfn[%02x]!\n",
182 p_ptt->idx, p_ptt->hwfn_id, p_hwfn->my_id);
183
184 /* Verify the address is within the window */
185 if (hw_addr < win_hw_addr ||
186 offset >= PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE) {
187 qed_ptt_set_win(p_hwfn, p_ptt, hw_addr);
188 offset = 0;
189 }
190
191 return qed_ptt_get_bar_addr(p_ptt) + offset;
192 }
193
qed_get_reserved_ptt(struct qed_hwfn * p_hwfn,enum reserved_ptts ptt_idx)194 struct qed_ptt *qed_get_reserved_ptt(struct qed_hwfn *p_hwfn,
195 enum reserved_ptts ptt_idx)
196 {
197 if (ptt_idx >= RESERVED_PTT_MAX) {
198 DP_NOTICE(p_hwfn,
199 "Requested PTT %d is out of range\n", ptt_idx);
200 return NULL;
201 }
202
203 return &p_hwfn->p_ptt_pool->ptts[ptt_idx];
204 }
205
qed_wr(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt,u32 hw_addr,u32 val)206 void qed_wr(struct qed_hwfn *p_hwfn,
207 struct qed_ptt *p_ptt,
208 u32 hw_addr, u32 val)
209 {
210 u32 bar_addr = qed_set_ptt(p_hwfn, p_ptt, hw_addr);
211
212 REG_WR(p_hwfn, bar_addr, val);
213 DP_VERBOSE(p_hwfn, NETIF_MSG_HW,
214 "bar_addr 0x%x, hw_addr 0x%x, val 0x%x\n",
215 bar_addr, hw_addr, val);
216 }
217
qed_rd(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt,u32 hw_addr)218 u32 qed_rd(struct qed_hwfn *p_hwfn,
219 struct qed_ptt *p_ptt,
220 u32 hw_addr)
221 {
222 u32 bar_addr = qed_set_ptt(p_hwfn, p_ptt, hw_addr);
223 u32 val = REG_RD(p_hwfn, bar_addr);
224
225 DP_VERBOSE(p_hwfn, NETIF_MSG_HW,
226 "bar_addr 0x%x, hw_addr 0x%x, val 0x%x\n",
227 bar_addr, hw_addr, val);
228
229 return val;
230 }
231
qed_memcpy_hw(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt,void * addr,u32 hw_addr,size_t n,bool to_device)232 static void qed_memcpy_hw(struct qed_hwfn *p_hwfn,
233 struct qed_ptt *p_ptt,
234 void *addr, u32 hw_addr, size_t n, bool to_device)
235 {
236 u32 dw_count, *host_addr, hw_offset;
237 size_t quota, done = 0;
238 u32 __iomem *reg_addr;
239
240 while (done < n) {
241 quota = min_t(size_t, n - done,
242 PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE);
243
244 if (IS_PF(p_hwfn->cdev)) {
245 qed_ptt_set_win(p_hwfn, p_ptt, hw_addr + done);
246 hw_offset = qed_ptt_get_bar_addr(p_ptt);
247 } else {
248 hw_offset = hw_addr + done;
249 }
250
251 dw_count = quota / 4;
252 host_addr = (u32 *)((u8 *)addr + done);
253 reg_addr = (u32 __iomem *)REG_ADDR(p_hwfn, hw_offset);
254 if (to_device)
255 while (dw_count--)
256 DIRECT_REG_WR(reg_addr++, *host_addr++);
257 else
258 while (dw_count--)
259 *host_addr++ = DIRECT_REG_RD(reg_addr++);
260
261 done += quota;
262 }
263 }
264
qed_memcpy_from(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt,void * dest,u32 hw_addr,size_t n)265 void qed_memcpy_from(struct qed_hwfn *p_hwfn,
266 struct qed_ptt *p_ptt, void *dest, u32 hw_addr, size_t n)
267 {
268 DP_VERBOSE(p_hwfn, NETIF_MSG_HW,
269 "hw_addr 0x%x, dest %p hw_addr 0x%x, size %lu\n",
270 hw_addr, dest, hw_addr, (unsigned long)n);
271
272 qed_memcpy_hw(p_hwfn, p_ptt, dest, hw_addr, n, false);
273 }
274
qed_memcpy_to(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt,u32 hw_addr,void * src,size_t n)275 void qed_memcpy_to(struct qed_hwfn *p_hwfn,
276 struct qed_ptt *p_ptt, u32 hw_addr, void *src, size_t n)
277 {
278 DP_VERBOSE(p_hwfn, NETIF_MSG_HW,
279 "hw_addr 0x%x, hw_addr 0x%x, src %p size %lu\n",
280 hw_addr, hw_addr, src, (unsigned long)n);
281
282 qed_memcpy_hw(p_hwfn, p_ptt, src, hw_addr, n, true);
283 }
284
qed_fid_pretend(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt,u16 fid)285 void qed_fid_pretend(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, u16 fid)
286 {
287 u16 control = 0;
288
289 SET_FIELD(control, PXP_PRETEND_CMD_IS_CONCRETE, 1);
290 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_FUNCTION, 1);
291
292 /* Every pretend undos previous pretends, including
293 * previous port pretend.
294 */
295 SET_FIELD(control, PXP_PRETEND_CMD_PORT, 0);
296 SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 0);
297 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1);
298
299 if (!GET_FIELD(fid, PXP_CONCRETE_FID_VFVALID))
300 fid = GET_FIELD(fid, PXP_CONCRETE_FID_PFID);
301
302 p_ptt->pxp.pretend.control = cpu_to_le16(control);
303 p_ptt->pxp.pretend.fid.concrete_fid.fid = cpu_to_le16(fid);
304
305 REG_WR(p_hwfn,
306 qed_ptt_config_addr(p_ptt) +
307 offsetof(struct pxp_ptt_entry, pretend),
308 *(u32 *)&p_ptt->pxp.pretend);
309 }
310
qed_port_pretend(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt,u8 port_id)311 void qed_port_pretend(struct qed_hwfn *p_hwfn,
312 struct qed_ptt *p_ptt, u8 port_id)
313 {
314 u16 control = 0;
315
316 SET_FIELD(control, PXP_PRETEND_CMD_PORT, port_id);
317 SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 1);
318 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1);
319
320 p_ptt->pxp.pretend.control = cpu_to_le16(control);
321
322 REG_WR(p_hwfn,
323 qed_ptt_config_addr(p_ptt) +
324 offsetof(struct pxp_ptt_entry, pretend),
325 *(u32 *)&p_ptt->pxp.pretend);
326 }
327
qed_port_unpretend(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt)328 void qed_port_unpretend(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt)
329 {
330 u16 control = 0;
331
332 SET_FIELD(control, PXP_PRETEND_CMD_PORT, 0);
333 SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 0);
334 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1);
335
336 p_ptt->pxp.pretend.control = cpu_to_le16(control);
337
338 REG_WR(p_hwfn,
339 qed_ptt_config_addr(p_ptt) +
340 offsetof(struct pxp_ptt_entry, pretend),
341 *(u32 *)&p_ptt->pxp.pretend);
342 }
343
qed_port_fid_pretend(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt,u8 port_id,u16 fid)344 void qed_port_fid_pretend(struct qed_hwfn *p_hwfn,
345 struct qed_ptt *p_ptt, u8 port_id, u16 fid)
346 {
347 u16 control = 0;
348
349 SET_FIELD(control, PXP_PRETEND_CMD_PORT, port_id);
350 SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 1);
351 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1);
352 SET_FIELD(control, PXP_PRETEND_CMD_IS_CONCRETE, 1);
353 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_FUNCTION, 1);
354 if (!GET_FIELD(fid, PXP_CONCRETE_FID_VFVALID))
355 fid = GET_FIELD(fid, PXP_CONCRETE_FID_PFID);
356 p_ptt->pxp.pretend.control = cpu_to_le16(control);
357 p_ptt->pxp.pretend.fid.concrete_fid.fid = cpu_to_le16(fid);
358 REG_WR(p_hwfn,
359 qed_ptt_config_addr(p_ptt) +
360 offsetof(struct pxp_ptt_entry, pretend),
361 *(u32 *)&p_ptt->pxp.pretend);
362 }
363
qed_vfid_to_concrete(struct qed_hwfn * p_hwfn,u8 vfid)364 u32 qed_vfid_to_concrete(struct qed_hwfn *p_hwfn, u8 vfid)
365 {
366 u32 concrete_fid = 0;
367
368 SET_FIELD(concrete_fid, PXP_CONCRETE_FID_PFID, p_hwfn->rel_pf_id);
369 SET_FIELD(concrete_fid, PXP_CONCRETE_FID_VFID, vfid);
370 SET_FIELD(concrete_fid, PXP_CONCRETE_FID_VFVALID, 1);
371
372 return concrete_fid;
373 }
374
375 /* DMAE */
376 #define QED_DMAE_FLAGS_IS_SET(params, flag) \
377 ((params) != NULL && GET_FIELD((params)->flags, QED_DMAE_PARAMS_##flag))
378
qed_dmae_opcode(struct qed_hwfn * p_hwfn,const u8 is_src_type_grc,const u8 is_dst_type_grc,struct qed_dmae_params * p_params)379 static void qed_dmae_opcode(struct qed_hwfn *p_hwfn,
380 const u8 is_src_type_grc,
381 const u8 is_dst_type_grc,
382 struct qed_dmae_params *p_params)
383 {
384 u8 src_pfid, dst_pfid, port_id;
385 u16 opcode_b = 0;
386 u32 opcode = 0;
387
388 /* Whether the source is the PCIe or the GRC.
389 * 0- The source is the PCIe
390 * 1- The source is the GRC.
391 */
392 SET_FIELD(opcode, DMAE_CMD_SRC,
393 (is_src_type_grc ? dmae_cmd_src_grc : dmae_cmd_src_pcie));
394 src_pfid = QED_DMAE_FLAGS_IS_SET(p_params, SRC_PF_VALID) ?
395 p_params->src_pfid : p_hwfn->rel_pf_id;
396 SET_FIELD(opcode, DMAE_CMD_SRC_PF_ID, src_pfid);
397
398 /* The destination of the DMA can be: 0-None 1-PCIe 2-GRC 3-None */
399 SET_FIELD(opcode, DMAE_CMD_DST,
400 (is_dst_type_grc ? dmae_cmd_dst_grc : dmae_cmd_dst_pcie));
401 dst_pfid = QED_DMAE_FLAGS_IS_SET(p_params, DST_PF_VALID) ?
402 p_params->dst_pfid : p_hwfn->rel_pf_id;
403 SET_FIELD(opcode, DMAE_CMD_DST_PF_ID, dst_pfid);
404
405
406 /* Whether to write a completion word to the completion destination:
407 * 0-Do not write a completion word
408 * 1-Write the completion word
409 */
410 SET_FIELD(opcode, DMAE_CMD_COMP_WORD_EN, 1);
411 SET_FIELD(opcode, DMAE_CMD_SRC_ADDR_RESET, 1);
412
413 if (QED_DMAE_FLAGS_IS_SET(p_params, COMPLETION_DST))
414 SET_FIELD(opcode, DMAE_CMD_COMP_FUNC, 1);
415
416 /* swapping mode 3 - big endian */
417 SET_FIELD(opcode, DMAE_CMD_ENDIANITY_MODE, DMAE_CMD_ENDIANITY);
418
419 port_id = (QED_DMAE_FLAGS_IS_SET(p_params, PORT_VALID)) ?
420 p_params->port_id : p_hwfn->port_id;
421 SET_FIELD(opcode, DMAE_CMD_PORT_ID, port_id);
422
423 /* reset source address in next go */
424 SET_FIELD(opcode, DMAE_CMD_SRC_ADDR_RESET, 1);
425
426 /* reset dest address in next go */
427 SET_FIELD(opcode, DMAE_CMD_DST_ADDR_RESET, 1);
428
429 /* SRC/DST VFID: all 1's - pf, otherwise VF id */
430 if (QED_DMAE_FLAGS_IS_SET(p_params, SRC_VF_VALID)) {
431 SET_FIELD(opcode, DMAE_CMD_SRC_VF_ID_VALID, 1);
432 SET_FIELD(opcode_b, DMAE_CMD_SRC_VF_ID, p_params->src_vfid);
433 } else {
434 SET_FIELD(opcode_b, DMAE_CMD_SRC_VF_ID, 0xFF);
435 }
436 if (QED_DMAE_FLAGS_IS_SET(p_params, DST_VF_VALID)) {
437 SET_FIELD(opcode, DMAE_CMD_DST_VF_ID_VALID, 1);
438 SET_FIELD(opcode_b, DMAE_CMD_DST_VF_ID, p_params->dst_vfid);
439 } else {
440 SET_FIELD(opcode_b, DMAE_CMD_DST_VF_ID, 0xFF);
441 }
442
443 p_hwfn->dmae_info.p_dmae_cmd->opcode = cpu_to_le32(opcode);
444 p_hwfn->dmae_info.p_dmae_cmd->opcode_b = cpu_to_le16(opcode_b);
445 }
446
qed_dmae_idx_to_go_cmd(u8 idx)447 u32 qed_dmae_idx_to_go_cmd(u8 idx)
448 {
449 /* All the DMAE 'go' registers form an array in internal memory */
450 return DMAE_REG_GO_C0 + (idx << 2);
451 }
452
qed_dmae_post_command(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt)453 static int qed_dmae_post_command(struct qed_hwfn *p_hwfn,
454 struct qed_ptt *p_ptt)
455 {
456 struct dmae_cmd *p_command = p_hwfn->dmae_info.p_dmae_cmd;
457 u8 idx_cmd = p_hwfn->dmae_info.channel, i;
458 int qed_status = 0;
459
460 /* verify address is not NULL */
461 if ((((!p_command->dst_addr_lo) && (!p_command->dst_addr_hi)) ||
462 ((!p_command->src_addr_lo) && (!p_command->src_addr_hi)))) {
463 DP_NOTICE(p_hwfn,
464 "source or destination address 0 idx_cmd=%d\n"
465 "opcode = [0x%08x,0x%04x] len=0x%x src=0x%x:%x dst=0x%x:%x\n",
466 idx_cmd,
467 le32_to_cpu(p_command->opcode),
468 le16_to_cpu(p_command->opcode_b),
469 le16_to_cpu(p_command->length_dw),
470 le32_to_cpu(p_command->src_addr_hi),
471 le32_to_cpu(p_command->src_addr_lo),
472 le32_to_cpu(p_command->dst_addr_hi),
473 le32_to_cpu(p_command->dst_addr_lo));
474
475 return -EINVAL;
476 }
477
478 DP_VERBOSE(p_hwfn,
479 NETIF_MSG_HW,
480 "Posting DMAE command [idx %d]: opcode = [0x%08x,0x%04x] len=0x%x src=0x%x:%x dst=0x%x:%x\n",
481 idx_cmd,
482 le32_to_cpu(p_command->opcode),
483 le16_to_cpu(p_command->opcode_b),
484 le16_to_cpu(p_command->length_dw),
485 le32_to_cpu(p_command->src_addr_hi),
486 le32_to_cpu(p_command->src_addr_lo),
487 le32_to_cpu(p_command->dst_addr_hi),
488 le32_to_cpu(p_command->dst_addr_lo));
489
490 /* Copy the command to DMAE - need to do it before every call
491 * for source/dest address no reset.
492 * The first 9 DWs are the command registers, the 10 DW is the
493 * GO register, and the rest are result registers
494 * (which are read only by the client).
495 */
496 for (i = 0; i < DMAE_CMD_SIZE; i++) {
497 u32 data = (i < DMAE_CMD_SIZE_TO_FILL) ?
498 *(((u32 *)p_command) + i) : 0;
499
500 qed_wr(p_hwfn, p_ptt,
501 DMAE_REG_CMD_MEM +
502 (idx_cmd * DMAE_CMD_SIZE * sizeof(u32)) +
503 (i * sizeof(u32)), data);
504 }
505
506 qed_wr(p_hwfn, p_ptt, qed_dmae_idx_to_go_cmd(idx_cmd), DMAE_GO_VALUE);
507
508 return qed_status;
509 }
510
qed_dmae_info_alloc(struct qed_hwfn * p_hwfn)511 int qed_dmae_info_alloc(struct qed_hwfn *p_hwfn)
512 {
513 dma_addr_t *p_addr = &p_hwfn->dmae_info.completion_word_phys_addr;
514 struct dmae_cmd **p_cmd = &p_hwfn->dmae_info.p_dmae_cmd;
515 u32 **p_buff = &p_hwfn->dmae_info.p_intermediate_buffer;
516 u32 **p_comp = &p_hwfn->dmae_info.p_completion_word;
517
518 *p_comp = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
519 sizeof(u32), p_addr, GFP_KERNEL);
520 if (!*p_comp)
521 goto err;
522
523 p_addr = &p_hwfn->dmae_info.dmae_cmd_phys_addr;
524 *p_cmd = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
525 sizeof(struct dmae_cmd),
526 p_addr, GFP_KERNEL);
527 if (!*p_cmd)
528 goto err;
529
530 p_addr = &p_hwfn->dmae_info.intermediate_buffer_phys_addr;
531 *p_buff = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
532 sizeof(u32) * DMAE_MAX_RW_SIZE,
533 p_addr, GFP_KERNEL);
534 if (!*p_buff)
535 goto err;
536
537 p_hwfn->dmae_info.channel = p_hwfn->rel_pf_id;
538
539 return 0;
540 err:
541 qed_dmae_info_free(p_hwfn);
542 return -ENOMEM;
543 }
544
qed_dmae_info_free(struct qed_hwfn * p_hwfn)545 void qed_dmae_info_free(struct qed_hwfn *p_hwfn)
546 {
547 dma_addr_t p_phys;
548
549 /* Just make sure no one is in the middle */
550 mutex_lock(&p_hwfn->dmae_info.mutex);
551
552 if (p_hwfn->dmae_info.p_completion_word) {
553 p_phys = p_hwfn->dmae_info.completion_word_phys_addr;
554 dma_free_coherent(&p_hwfn->cdev->pdev->dev,
555 sizeof(u32),
556 p_hwfn->dmae_info.p_completion_word, p_phys);
557 p_hwfn->dmae_info.p_completion_word = NULL;
558 }
559
560 if (p_hwfn->dmae_info.p_dmae_cmd) {
561 p_phys = p_hwfn->dmae_info.dmae_cmd_phys_addr;
562 dma_free_coherent(&p_hwfn->cdev->pdev->dev,
563 sizeof(struct dmae_cmd),
564 p_hwfn->dmae_info.p_dmae_cmd, p_phys);
565 p_hwfn->dmae_info.p_dmae_cmd = NULL;
566 }
567
568 if (p_hwfn->dmae_info.p_intermediate_buffer) {
569 p_phys = p_hwfn->dmae_info.intermediate_buffer_phys_addr;
570 dma_free_coherent(&p_hwfn->cdev->pdev->dev,
571 sizeof(u32) * DMAE_MAX_RW_SIZE,
572 p_hwfn->dmae_info.p_intermediate_buffer,
573 p_phys);
574 p_hwfn->dmae_info.p_intermediate_buffer = NULL;
575 }
576
577 mutex_unlock(&p_hwfn->dmae_info.mutex);
578 }
579
qed_dmae_operation_wait(struct qed_hwfn * p_hwfn)580 static int qed_dmae_operation_wait(struct qed_hwfn *p_hwfn)
581 {
582 u32 wait_cnt_limit = 10000, wait_cnt = 0;
583 int qed_status = 0;
584
585 barrier();
586 while (*p_hwfn->dmae_info.p_completion_word != DMAE_COMPLETION_VAL) {
587 udelay(DMAE_MIN_WAIT_TIME);
588 cond_resched();
589 if (++wait_cnt > wait_cnt_limit) {
590 DP_NOTICE(p_hwfn->cdev,
591 "Timed-out waiting for operation to complete. Completion word is 0x%08x expected 0x%08x.\n",
592 *p_hwfn->dmae_info.p_completion_word,
593 DMAE_COMPLETION_VAL);
594 qed_status = -EBUSY;
595 break;
596 }
597
598 /* to sync the completion_word since we are not
599 * using the volatile keyword for p_completion_word
600 */
601 barrier();
602 }
603
604 if (qed_status == 0)
605 *p_hwfn->dmae_info.p_completion_word = 0;
606
607 return qed_status;
608 }
609
qed_dmae_execute_sub_operation(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt,u64 src_addr,u64 dst_addr,u8 src_type,u8 dst_type,u32 length_dw)610 static int qed_dmae_execute_sub_operation(struct qed_hwfn *p_hwfn,
611 struct qed_ptt *p_ptt,
612 u64 src_addr,
613 u64 dst_addr,
614 u8 src_type,
615 u8 dst_type,
616 u32 length_dw)
617 {
618 dma_addr_t phys = p_hwfn->dmae_info.intermediate_buffer_phys_addr;
619 struct dmae_cmd *cmd = p_hwfn->dmae_info.p_dmae_cmd;
620 int qed_status = 0;
621
622 switch (src_type) {
623 case QED_DMAE_ADDRESS_GRC:
624 case QED_DMAE_ADDRESS_HOST_PHYS:
625 cmd->src_addr_hi = cpu_to_le32(upper_32_bits(src_addr));
626 cmd->src_addr_lo = cpu_to_le32(lower_32_bits(src_addr));
627 break;
628 /* for virtual source addresses we use the intermediate buffer. */
629 case QED_DMAE_ADDRESS_HOST_VIRT:
630 cmd->src_addr_hi = cpu_to_le32(upper_32_bits(phys));
631 cmd->src_addr_lo = cpu_to_le32(lower_32_bits(phys));
632 memcpy(&p_hwfn->dmae_info.p_intermediate_buffer[0],
633 (void *)(uintptr_t)src_addr,
634 length_dw * sizeof(u32));
635 break;
636 default:
637 return -EINVAL;
638 }
639
640 switch (dst_type) {
641 case QED_DMAE_ADDRESS_GRC:
642 case QED_DMAE_ADDRESS_HOST_PHYS:
643 cmd->dst_addr_hi = cpu_to_le32(upper_32_bits(dst_addr));
644 cmd->dst_addr_lo = cpu_to_le32(lower_32_bits(dst_addr));
645 break;
646 /* for virtual source addresses we use the intermediate buffer. */
647 case QED_DMAE_ADDRESS_HOST_VIRT:
648 cmd->dst_addr_hi = cpu_to_le32(upper_32_bits(phys));
649 cmd->dst_addr_lo = cpu_to_le32(lower_32_bits(phys));
650 break;
651 default:
652 return -EINVAL;
653 }
654
655 cmd->length_dw = cpu_to_le16((u16)length_dw);
656
657 qed_dmae_post_command(p_hwfn, p_ptt);
658
659 qed_status = qed_dmae_operation_wait(p_hwfn);
660
661 if (qed_status) {
662 DP_NOTICE(p_hwfn,
663 "qed_dmae_host2grc: Wait Failed. source_addr 0x%llx, grc_addr 0x%llx, size_in_dwords 0x%x\n",
664 src_addr, dst_addr, length_dw);
665 return qed_status;
666 }
667
668 if (dst_type == QED_DMAE_ADDRESS_HOST_VIRT)
669 memcpy((void *)(uintptr_t)(dst_addr),
670 &p_hwfn->dmae_info.p_intermediate_buffer[0],
671 length_dw * sizeof(u32));
672
673 return 0;
674 }
675
qed_dmae_execute_command(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt,u64 src_addr,u64 dst_addr,u8 src_type,u8 dst_type,u32 size_in_dwords,struct qed_dmae_params * p_params)676 static int qed_dmae_execute_command(struct qed_hwfn *p_hwfn,
677 struct qed_ptt *p_ptt,
678 u64 src_addr, u64 dst_addr,
679 u8 src_type, u8 dst_type,
680 u32 size_in_dwords,
681 struct qed_dmae_params *p_params)
682 {
683 dma_addr_t phys = p_hwfn->dmae_info.completion_word_phys_addr;
684 u16 length_cur = 0, i = 0, cnt_split = 0, length_mod = 0;
685 struct dmae_cmd *cmd = p_hwfn->dmae_info.p_dmae_cmd;
686 u64 src_addr_split = 0, dst_addr_split = 0;
687 u16 length_limit = DMAE_MAX_RW_SIZE;
688 int qed_status = 0;
689 u32 offset = 0;
690
691 if (p_hwfn->cdev->recov_in_prog) {
692 DP_VERBOSE(p_hwfn,
693 NETIF_MSG_HW,
694 "Recovery is in progress. Avoid DMAE transaction [{src: addr 0x%llx, type %d}, {dst: addr 0x%llx, type %d}, size %d].\n",
695 src_addr, src_type, dst_addr, dst_type,
696 size_in_dwords);
697
698 /* Let the flow complete w/o any error handling */
699 return 0;
700 }
701
702 qed_dmae_opcode(p_hwfn,
703 (src_type == QED_DMAE_ADDRESS_GRC),
704 (dst_type == QED_DMAE_ADDRESS_GRC),
705 p_params);
706
707 cmd->comp_addr_lo = cpu_to_le32(lower_32_bits(phys));
708 cmd->comp_addr_hi = cpu_to_le32(upper_32_bits(phys));
709 cmd->comp_val = cpu_to_le32(DMAE_COMPLETION_VAL);
710
711 /* Check if the grc_addr is valid like < MAX_GRC_OFFSET */
712 cnt_split = size_in_dwords / length_limit;
713 length_mod = size_in_dwords % length_limit;
714
715 src_addr_split = src_addr;
716 dst_addr_split = dst_addr;
717
718 for (i = 0; i <= cnt_split; i++) {
719 offset = length_limit * i;
720
721 if (!QED_DMAE_FLAGS_IS_SET(p_params, RW_REPL_SRC)) {
722 if (src_type == QED_DMAE_ADDRESS_GRC)
723 src_addr_split = src_addr + offset;
724 else
725 src_addr_split = src_addr + (offset * 4);
726 }
727
728 if (dst_type == QED_DMAE_ADDRESS_GRC)
729 dst_addr_split = dst_addr + offset;
730 else
731 dst_addr_split = dst_addr + (offset * 4);
732
733 length_cur = (cnt_split == i) ? length_mod : length_limit;
734
735 /* might be zero on last iteration */
736 if (!length_cur)
737 continue;
738
739 qed_status = qed_dmae_execute_sub_operation(p_hwfn,
740 p_ptt,
741 src_addr_split,
742 dst_addr_split,
743 src_type,
744 dst_type,
745 length_cur);
746 if (qed_status) {
747 qed_hw_err_notify(p_hwfn, p_ptt, QED_HW_ERR_DMAE_FAIL,
748 "qed_dmae_execute_sub_operation Failed with error 0x%x. source_addr 0x%llx, destination addr 0x%llx, size_in_dwords 0x%x\n",
749 qed_status, src_addr,
750 dst_addr, length_cur);
751 break;
752 }
753 }
754
755 return qed_status;
756 }
757
qed_dmae_host2grc(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt,u64 source_addr,u32 grc_addr,u32 size_in_dwords,struct qed_dmae_params * p_params)758 int qed_dmae_host2grc(struct qed_hwfn *p_hwfn,
759 struct qed_ptt *p_ptt,
760 u64 source_addr, u32 grc_addr, u32 size_in_dwords,
761 struct qed_dmae_params *p_params)
762 {
763 u32 grc_addr_in_dw = grc_addr / sizeof(u32);
764 int rc;
765
766
767 mutex_lock(&p_hwfn->dmae_info.mutex);
768
769 rc = qed_dmae_execute_command(p_hwfn, p_ptt, source_addr,
770 grc_addr_in_dw,
771 QED_DMAE_ADDRESS_HOST_VIRT,
772 QED_DMAE_ADDRESS_GRC,
773 size_in_dwords, p_params);
774
775 mutex_unlock(&p_hwfn->dmae_info.mutex);
776
777 return rc;
778 }
779
qed_dmae_grc2host(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt,u32 grc_addr,dma_addr_t dest_addr,u32 size_in_dwords,struct qed_dmae_params * p_params)780 int qed_dmae_grc2host(struct qed_hwfn *p_hwfn,
781 struct qed_ptt *p_ptt,
782 u32 grc_addr,
783 dma_addr_t dest_addr, u32 size_in_dwords,
784 struct qed_dmae_params *p_params)
785 {
786 u32 grc_addr_in_dw = grc_addr / sizeof(u32);
787 int rc;
788
789
790 mutex_lock(&p_hwfn->dmae_info.mutex);
791
792 rc = qed_dmae_execute_command(p_hwfn, p_ptt, grc_addr_in_dw,
793 dest_addr, QED_DMAE_ADDRESS_GRC,
794 QED_DMAE_ADDRESS_HOST_VIRT,
795 size_in_dwords, p_params);
796
797 mutex_unlock(&p_hwfn->dmae_info.mutex);
798
799 return rc;
800 }
801
qed_dmae_host2host(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt,dma_addr_t source_addr,dma_addr_t dest_addr,u32 size_in_dwords,struct qed_dmae_params * p_params)802 int qed_dmae_host2host(struct qed_hwfn *p_hwfn,
803 struct qed_ptt *p_ptt,
804 dma_addr_t source_addr,
805 dma_addr_t dest_addr,
806 u32 size_in_dwords, struct qed_dmae_params *p_params)
807 {
808 int rc;
809
810 mutex_lock(&(p_hwfn->dmae_info.mutex));
811
812 rc = qed_dmae_execute_command(p_hwfn, p_ptt, source_addr,
813 dest_addr,
814 QED_DMAE_ADDRESS_HOST_PHYS,
815 QED_DMAE_ADDRESS_HOST_PHYS,
816 size_in_dwords, p_params);
817
818 mutex_unlock(&(p_hwfn->dmae_info.mutex));
819
820 return rc;
821 }
822
qed_hw_err_notify(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt,enum qed_hw_err_type err_type,const char * fmt,...)823 void qed_hw_err_notify(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt,
824 enum qed_hw_err_type err_type, const char *fmt, ...)
825 {
826 char buf[QED_HW_ERR_MAX_STR_SIZE];
827 va_list vl;
828 int len;
829
830 if (fmt) {
831 va_start(vl, fmt);
832 len = vsnprintf(buf, QED_HW_ERR_MAX_STR_SIZE, fmt, vl);
833 va_end(vl);
834
835 if (len > QED_HW_ERR_MAX_STR_SIZE - 1)
836 len = QED_HW_ERR_MAX_STR_SIZE - 1;
837
838 DP_NOTICE(p_hwfn, "%s", buf);
839 }
840
841 /* Fan failure cannot be masked by handling of another HW error */
842 if (p_hwfn->cdev->recov_in_prog &&
843 err_type != QED_HW_ERR_FAN_FAIL) {
844 DP_VERBOSE(p_hwfn,
845 NETIF_MSG_DRV,
846 "Recovery is in progress. Avoid notifying about HW error %d.\n",
847 err_type);
848 return;
849 }
850
851 qed_hw_error_occurred(p_hwfn, err_type);
852
853 if (fmt)
854 qed_mcp_send_raw_debug_data(p_hwfn, p_ptt, buf, len);
855 }
856
qed_dmae_sanity(struct qed_hwfn * p_hwfn,struct qed_ptt * p_ptt,const char * phase)857 int qed_dmae_sanity(struct qed_hwfn *p_hwfn,
858 struct qed_ptt *p_ptt, const char *phase)
859 {
860 u32 size = PAGE_SIZE / 2, val;
861 int rc = 0;
862 dma_addr_t p_phys;
863 void *p_virt;
864 u32 *p_tmp;
865
866 p_virt = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
867 2 * size, &p_phys, GFP_KERNEL);
868 if (!p_virt) {
869 DP_NOTICE(p_hwfn,
870 "DMAE sanity [%s]: failed to allocate memory\n",
871 phase);
872 return -ENOMEM;
873 }
874
875 /* Fill the bottom half of the allocated memory with a known pattern */
876 for (p_tmp = (u32 *)p_virt;
877 p_tmp < (u32 *)((u8 *)p_virt + size); p_tmp++) {
878 /* Save the address itself as the value */
879 val = (u32)(uintptr_t)p_tmp;
880 *p_tmp = val;
881 }
882
883 /* Zero the top half of the allocated memory */
884 memset((u8 *)p_virt + size, 0, size);
885
886 DP_VERBOSE(p_hwfn,
887 QED_MSG_SP,
888 "DMAE sanity [%s]: src_addr={phys 0x%llx, virt %p}, dst_addr={phys 0x%llx, virt %p}, size 0x%x\n",
889 phase,
890 (u64)p_phys,
891 p_virt, (u64)(p_phys + size), (u8 *)p_virt + size, size);
892
893 rc = qed_dmae_host2host(p_hwfn, p_ptt, p_phys, p_phys + size,
894 size / 4, NULL);
895 if (rc) {
896 DP_NOTICE(p_hwfn,
897 "DMAE sanity [%s]: qed_dmae_host2host() failed. rc = %d.\n",
898 phase, rc);
899 goto out;
900 }
901
902 /* Verify that the top half of the allocated memory has the pattern */
903 for (p_tmp = (u32 *)((u8 *)p_virt + size);
904 p_tmp < (u32 *)((u8 *)p_virt + (2 * size)); p_tmp++) {
905 /* The corresponding address in the bottom half */
906 val = (u32)(uintptr_t)p_tmp - size;
907
908 if (*p_tmp != val) {
909 DP_NOTICE(p_hwfn,
910 "DMAE sanity [%s]: addr={phys 0x%llx, virt %p}, read_val 0x%08x, expected_val 0x%08x\n",
911 phase,
912 (u64)p_phys + ((u8 *)p_tmp - (u8 *)p_virt),
913 p_tmp, *p_tmp, val);
914 rc = -EINVAL;
915 goto out;
916 }
917 }
918
919 out:
920 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 2 * size, p_virt, p_phys);
921 return rc;
922 }
923