Lines Matching +full:row +full:- +full:stride
2 * Broadcom NetXtreme-E RoCE driver.
4 * Copyright (c) 2016 - 2017, Broadcom. All rights reserved. The term
62 qp->sq.condition = false; in bnxt_qplib_cancel_phantom_processing()
63 qp->sq.send_phantom = false; in bnxt_qplib_cancel_phantom_processing()
64 qp->sq.single = false; in bnxt_qplib_cancel_phantom_processing()
72 scq = qp->scq; in __bnxt_qplib_add_flush_qp()
73 rcq = qp->rcq; in __bnxt_qplib_add_flush_qp()
75 if (!qp->sq.flushed) { in __bnxt_qplib_add_flush_qp()
76 dev_dbg(&scq->hwq.pdev->dev, in __bnxt_qplib_add_flush_qp()
79 list_add_tail(&qp->sq_flush, &scq->sqf_head); in __bnxt_qplib_add_flush_qp()
80 qp->sq.flushed = true; in __bnxt_qplib_add_flush_qp()
82 if (!qp->srq) { in __bnxt_qplib_add_flush_qp()
83 if (!qp->rq.flushed) { in __bnxt_qplib_add_flush_qp()
84 dev_dbg(&rcq->hwq.pdev->dev, in __bnxt_qplib_add_flush_qp()
86 list_add_tail(&qp->rq_flush, &rcq->rqf_head); in __bnxt_qplib_add_flush_qp()
87 qp->rq.flushed = true; in __bnxt_qplib_add_flush_qp()
94 __acquires(&qp->scq->flush_lock) __acquires(&qp->rcq->flush_lock) in bnxt_qplib_acquire_cq_flush_locks()
96 spin_lock_irqsave(&qp->scq->flush_lock, *flags); in bnxt_qplib_acquire_cq_flush_locks()
97 if (qp->scq == qp->rcq) in bnxt_qplib_acquire_cq_flush_locks()
98 __acquire(&qp->rcq->flush_lock); in bnxt_qplib_acquire_cq_flush_locks()
100 spin_lock(&qp->rcq->flush_lock); in bnxt_qplib_acquire_cq_flush_locks()
105 __releases(&qp->scq->flush_lock) __releases(&qp->rcq->flush_lock) in bnxt_qplib_release_cq_flush_locks()
107 if (qp->scq == qp->rcq) in bnxt_qplib_release_cq_flush_locks()
108 __release(&qp->rcq->flush_lock); in bnxt_qplib_release_cq_flush_locks()
110 spin_unlock(&qp->rcq->flush_lock); in bnxt_qplib_release_cq_flush_locks()
111 spin_unlock_irqrestore(&qp->scq->flush_lock, *flags); in bnxt_qplib_release_cq_flush_locks()
125 if (qp->sq.flushed) { in __bnxt_qplib_del_flush_qp()
126 qp->sq.flushed = false; in __bnxt_qplib_del_flush_qp()
127 list_del(&qp->sq_flush); in __bnxt_qplib_del_flush_qp()
129 if (!qp->srq) { in __bnxt_qplib_del_flush_qp()
130 if (qp->rq.flushed) { in __bnxt_qplib_del_flush_qp()
131 qp->rq.flushed = false; in __bnxt_qplib_del_flush_qp()
132 list_del(&qp->rq_flush); in __bnxt_qplib_del_flush_qp()
142 __clean_cq(qp->scq, (u64)(unsigned long)qp); in bnxt_qplib_clean_qp()
143 qp->sq.hwq.prod = 0; in bnxt_qplib_clean_qp()
144 qp->sq.hwq.cons = 0; in bnxt_qplib_clean_qp()
145 __clean_cq(qp->rcq, (u64)(unsigned long)qp); in bnxt_qplib_clean_qp()
146 qp->rq.hwq.prod = 0; in bnxt_qplib_clean_qp()
147 qp->rq.hwq.cons = 0; in bnxt_qplib_clean_qp()
158 struct bnxt_qplib_cq *cq = nq_work->cq; in bnxt_qpn_cqn_sched_task()
159 struct bnxt_qplib_nq *nq = nq_work->nq; in bnxt_qpn_cqn_sched_task()
162 spin_lock_bh(&cq->compl_lock); in bnxt_qpn_cqn_sched_task()
163 if (atomic_read(&cq->arm_state) && nq->cqn_handler) { in bnxt_qpn_cqn_sched_task()
164 dev_dbg(&nq->pdev->dev, in bnxt_qpn_cqn_sched_task()
167 nq->cqn_handler(nq, cq); in bnxt_qpn_cqn_sched_task()
169 spin_unlock_bh(&cq->compl_lock); in bnxt_qpn_cqn_sched_task()
177 struct bnxt_qplib_q *rq = &qp->rq; in bnxt_qplib_free_qp_hdr_buf()
178 struct bnxt_qplib_q *sq = &qp->sq; in bnxt_qplib_free_qp_hdr_buf()
180 if (qp->rq_hdr_buf) in bnxt_qplib_free_qp_hdr_buf()
181 dma_free_coherent(&res->pdev->dev, in bnxt_qplib_free_qp_hdr_buf()
182 rq->max_wqe * qp->rq_hdr_buf_size, in bnxt_qplib_free_qp_hdr_buf()
183 qp->rq_hdr_buf, qp->rq_hdr_buf_map); in bnxt_qplib_free_qp_hdr_buf()
184 if (qp->sq_hdr_buf) in bnxt_qplib_free_qp_hdr_buf()
185 dma_free_coherent(&res->pdev->dev, in bnxt_qplib_free_qp_hdr_buf()
186 sq->max_wqe * qp->sq_hdr_buf_size, in bnxt_qplib_free_qp_hdr_buf()
187 qp->sq_hdr_buf, qp->sq_hdr_buf_map); in bnxt_qplib_free_qp_hdr_buf()
188 qp->rq_hdr_buf = NULL; in bnxt_qplib_free_qp_hdr_buf()
189 qp->sq_hdr_buf = NULL; in bnxt_qplib_free_qp_hdr_buf()
190 qp->rq_hdr_buf_map = 0; in bnxt_qplib_free_qp_hdr_buf()
191 qp->sq_hdr_buf_map = 0; in bnxt_qplib_free_qp_hdr_buf()
192 qp->sq_hdr_buf_size = 0; in bnxt_qplib_free_qp_hdr_buf()
193 qp->rq_hdr_buf_size = 0; in bnxt_qplib_free_qp_hdr_buf()
199 struct bnxt_qplib_q *rq = &qp->rq; in bnxt_qplib_alloc_qp_hdr_buf()
200 struct bnxt_qplib_q *sq = &qp->sq; in bnxt_qplib_alloc_qp_hdr_buf()
203 if (qp->sq_hdr_buf_size && sq->max_wqe) { in bnxt_qplib_alloc_qp_hdr_buf()
204 qp->sq_hdr_buf = dma_alloc_coherent(&res->pdev->dev, in bnxt_qplib_alloc_qp_hdr_buf()
205 sq->max_wqe * qp->sq_hdr_buf_size, in bnxt_qplib_alloc_qp_hdr_buf()
206 &qp->sq_hdr_buf_map, GFP_KERNEL); in bnxt_qplib_alloc_qp_hdr_buf()
207 if (!qp->sq_hdr_buf) { in bnxt_qplib_alloc_qp_hdr_buf()
208 rc = -ENOMEM; in bnxt_qplib_alloc_qp_hdr_buf()
209 dev_err(&res->pdev->dev, in bnxt_qplib_alloc_qp_hdr_buf()
215 if (qp->rq_hdr_buf_size && rq->max_wqe) { in bnxt_qplib_alloc_qp_hdr_buf()
216 qp->rq_hdr_buf = dma_alloc_coherent(&res->pdev->dev, in bnxt_qplib_alloc_qp_hdr_buf()
217 rq->max_wqe * in bnxt_qplib_alloc_qp_hdr_buf()
218 qp->rq_hdr_buf_size, in bnxt_qplib_alloc_qp_hdr_buf()
219 &qp->rq_hdr_buf_map, in bnxt_qplib_alloc_qp_hdr_buf()
221 if (!qp->rq_hdr_buf) { in bnxt_qplib_alloc_qp_hdr_buf()
222 rc = -ENOMEM; in bnxt_qplib_alloc_qp_hdr_buf()
223 dev_err(&res->pdev->dev, in bnxt_qplib_alloc_qp_hdr_buf()
237 struct bnxt_qplib_hwq *hwq = &nq->hwq; in clean_nq()
239 int budget = nq->budget; in clean_nq()
243 spin_lock_bh(&hwq->lock); in clean_nq()
245 while (budget--) { in clean_nq()
246 nq_ptr = (struct nq_base **)hwq->pbl_ptr; in clean_nq()
247 nqe = &nq_ptr[NQE_PG(hwq->cons)][NQE_IDX(hwq->cons)]; in clean_nq()
248 if (!NQE_CMP_VALID(nqe, nq->nq_db.dbinfo.flags)) in clean_nq()
257 type = le16_to_cpu(nqe->info10_type) & NQ_BASE_TYPE_MASK; in clean_nq()
263 q_handle = le32_to_cpu(nqcne->cq_handle_low); in clean_nq()
264 q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high) in clean_nq()
267 nqcne->cq_handle_low = 0; in clean_nq()
268 nqcne->cq_handle_high = 0; in clean_nq()
269 cq->cnq_events++; in clean_nq()
276 bnxt_qplib_hwq_incr_cons(hwq->max_elements, &hwq->cons, in clean_nq()
277 1, &nq->nq_db.dbinfo.flags); in clean_nq()
279 spin_unlock_bh(&hwq->lock); in clean_nq()
289 while (retry_cnt--) { in __wait_for_all_nqes()
290 if (cnq_events == cq->cnq_events) in __wait_for_all_nqes()
293 clean_nq(cq->nq, cq); in __wait_for_all_nqes()
300 struct bnxt_qplib_hwq *hwq = &nq->hwq; in bnxt_qplib_service_nq()
302 int budget = nq->budget; in bnxt_qplib_service_nq()
308 spin_lock_bh(&hwq->lock); in bnxt_qplib_service_nq()
310 while (budget--) { in bnxt_qplib_service_nq()
311 nqe = bnxt_qplib_get_qe(hwq, hwq->cons, NULL); in bnxt_qplib_service_nq()
312 if (!NQE_CMP_VALID(nqe, nq->nq_db.dbinfo.flags)) in bnxt_qplib_service_nq()
321 type = le16_to_cpu(nqe->info10_type) & NQ_BASE_TYPE_MASK; in bnxt_qplib_service_nq()
327 q_handle = le32_to_cpu(nqcne->cq_handle_low); in bnxt_qplib_service_nq()
328 q_handle |= (u64)le32_to_cpu(nqcne->cq_handle_high) in bnxt_qplib_service_nq()
333 cq->toggle = (le16_to_cpu(nqe->info10_type) & in bnxt_qplib_service_nq()
335 cq->dbinfo.toggle = cq->toggle; in bnxt_qplib_service_nq()
336 bnxt_qplib_armen_db(&cq->dbinfo, in bnxt_qplib_service_nq()
338 spin_lock_bh(&cq->compl_lock); in bnxt_qplib_service_nq()
339 atomic_set(&cq->arm_state, 0); in bnxt_qplib_service_nq()
340 if (nq->cqn_handler(nq, (cq))) in bnxt_qplib_service_nq()
341 dev_warn(&nq->pdev->dev, in bnxt_qplib_service_nq()
342 "cqn - type 0x%x not handled\n", type); in bnxt_qplib_service_nq()
343 cq->cnq_events++; in bnxt_qplib_service_nq()
344 spin_unlock_bh(&cq->compl_lock); in bnxt_qplib_service_nq()
353 q_handle = le32_to_cpu(nqsrqe->srq_handle_low); in bnxt_qplib_service_nq()
354 q_handle |= (u64)le32_to_cpu(nqsrqe->srq_handle_high) in bnxt_qplib_service_nq()
357 bnxt_qplib_armen_db(&srq->dbinfo, in bnxt_qplib_service_nq()
359 if (nq->srqn_handler(nq, in bnxt_qplib_service_nq()
361 nqsrqe->event)) in bnxt_qplib_service_nq()
362 dev_warn(&nq->pdev->dev, in bnxt_qplib_service_nq()
364 nqsrqe->event); in bnxt_qplib_service_nq()
370 dev_warn(&nq->pdev->dev, in bnxt_qplib_service_nq()
375 bnxt_qplib_hwq_incr_cons(hwq->max_elements, &hwq->cons, in bnxt_qplib_service_nq()
376 1, &nq->nq_db.dbinfo.flags); in bnxt_qplib_service_nq()
379 bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, nq->res->cctx, true); in bnxt_qplib_service_nq()
380 spin_unlock_bh(&hwq->lock); in bnxt_qplib_service_nq()
383 /* bnxt_re_synchronize_nq - self polling notification queue.
384 * @nq - notification queue pointer
394 int budget = nq->budget; in bnxt_re_synchronize_nq()
396 nq->budget = nq->hwq.max_elements; in bnxt_re_synchronize_nq()
397 bnxt_qplib_service_nq(&nq->nq_tasklet); in bnxt_re_synchronize_nq()
398 nq->budget = budget; in bnxt_re_synchronize_nq()
404 struct bnxt_qplib_hwq *hwq = &nq->hwq; in bnxt_qplib_nq_irq()
408 sw_cons = HWQ_CMP(hwq->cons, hwq); in bnxt_qplib_nq_irq()
412 tasklet_schedule(&nq->nq_tasklet); in bnxt_qplib_nq_irq()
419 if (!nq->requested) in bnxt_qplib_nq_stop_irq()
422 nq->requested = false; in bnxt_qplib_nq_stop_irq()
424 bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, nq->res->cctx, false); in bnxt_qplib_nq_stop_irq()
426 synchronize_irq(nq->msix_vec); in bnxt_qplib_nq_stop_irq()
427 irq_set_affinity_hint(nq->msix_vec, NULL); in bnxt_qplib_nq_stop_irq()
428 free_irq(nq->msix_vec, nq); in bnxt_qplib_nq_stop_irq()
429 kfree(nq->name); in bnxt_qplib_nq_stop_irq()
430 nq->name = NULL; in bnxt_qplib_nq_stop_irq()
433 tasklet_kill(&nq->nq_tasklet); in bnxt_qplib_nq_stop_irq()
434 tasklet_disable(&nq->nq_tasklet); in bnxt_qplib_nq_stop_irq()
439 if (nq->cqn_wq) { in bnxt_qplib_disable_nq()
440 destroy_workqueue(nq->cqn_wq); in bnxt_qplib_disable_nq()
441 nq->cqn_wq = NULL; in bnxt_qplib_disable_nq()
447 if (nq->nq_db.reg.bar_reg) { in bnxt_qplib_disable_nq()
448 iounmap(nq->nq_db.reg.bar_reg); in bnxt_qplib_disable_nq()
449 nq->nq_db.reg.bar_reg = NULL; in bnxt_qplib_disable_nq()
452 nq->cqn_handler = NULL; in bnxt_qplib_disable_nq()
453 nq->srqn_handler = NULL; in bnxt_qplib_disable_nq()
454 nq->msix_vec = 0; in bnxt_qplib_disable_nq()
460 struct bnxt_qplib_res *res = nq->res; in bnxt_qplib_nq_start_irq()
463 if (nq->requested) in bnxt_qplib_nq_start_irq()
464 return -EFAULT; in bnxt_qplib_nq_start_irq()
466 nq->msix_vec = msix_vector; in bnxt_qplib_nq_start_irq()
468 tasklet_setup(&nq->nq_tasklet, bnxt_qplib_service_nq); in bnxt_qplib_nq_start_irq()
470 tasklet_enable(&nq->nq_tasklet); in bnxt_qplib_nq_start_irq()
472 nq->name = kasprintf(GFP_KERNEL, "bnxt_re-nq-%d@pci:%s", in bnxt_qplib_nq_start_irq()
473 nq_indx, pci_name(res->pdev)); in bnxt_qplib_nq_start_irq()
474 if (!nq->name) in bnxt_qplib_nq_start_irq()
475 return -ENOMEM; in bnxt_qplib_nq_start_irq()
476 rc = request_irq(nq->msix_vec, bnxt_qplib_nq_irq, 0, nq->name, nq); in bnxt_qplib_nq_start_irq()
478 kfree(nq->name); in bnxt_qplib_nq_start_irq()
479 nq->name = NULL; in bnxt_qplib_nq_start_irq()
480 tasklet_disable(&nq->nq_tasklet); in bnxt_qplib_nq_start_irq()
484 cpumask_clear(&nq->mask); in bnxt_qplib_nq_start_irq()
485 cpumask_set_cpu(nq_indx, &nq->mask); in bnxt_qplib_nq_start_irq()
486 rc = irq_set_affinity_hint(nq->msix_vec, &nq->mask); in bnxt_qplib_nq_start_irq()
488 dev_warn(&nq->pdev->dev, in bnxt_qplib_nq_start_irq()
490 nq->msix_vec, nq_indx); in bnxt_qplib_nq_start_irq()
492 nq->requested = true; in bnxt_qplib_nq_start_irq()
493 bnxt_qplib_ring_nq_db(&nq->nq_db.dbinfo, res->cctx, true); in bnxt_qplib_nq_start_irq()
504 pdev = nq->pdev; in bnxt_qplib_map_nq_db()
505 nq_db = &nq->nq_db; in bnxt_qplib_map_nq_db()
507 nq_db->dbinfo.flags = 0; in bnxt_qplib_map_nq_db()
508 nq_db->reg.bar_id = NQ_CONS_PCI_BAR_REGION; in bnxt_qplib_map_nq_db()
509 nq_db->reg.bar_base = pci_resource_start(pdev, nq_db->reg.bar_id); in bnxt_qplib_map_nq_db()
510 if (!nq_db->reg.bar_base) { in bnxt_qplib_map_nq_db()
511 dev_err(&pdev->dev, "QPLIB: NQ BAR region %d resc start is 0!", in bnxt_qplib_map_nq_db()
512 nq_db->reg.bar_id); in bnxt_qplib_map_nq_db()
513 return -ENOMEM; in bnxt_qplib_map_nq_db()
516 reg_base = nq_db->reg.bar_base + reg_offt; in bnxt_qplib_map_nq_db()
518 nq_db->reg.len = 8; in bnxt_qplib_map_nq_db()
519 nq_db->reg.bar_reg = ioremap(reg_base, nq_db->reg.len); in bnxt_qplib_map_nq_db()
520 if (!nq_db->reg.bar_reg) { in bnxt_qplib_map_nq_db()
521 dev_err(&pdev->dev, "QPLIB: NQ BAR region %d mapping failed", in bnxt_qplib_map_nq_db()
522 nq_db->reg.bar_id); in bnxt_qplib_map_nq_db()
523 return -ENOMEM; in bnxt_qplib_map_nq_db()
526 nq_db->dbinfo.db = nq_db->reg.bar_reg; in bnxt_qplib_map_nq_db()
527 nq_db->dbinfo.hwq = &nq->hwq; in bnxt_qplib_map_nq_db()
528 nq_db->dbinfo.xid = nq->ring_id; in bnxt_qplib_map_nq_db()
540 nq->pdev = pdev; in bnxt_qplib_enable_nq()
541 nq->cqn_handler = cqn_handler; in bnxt_qplib_enable_nq()
542 nq->srqn_handler = srqn_handler; in bnxt_qplib_enable_nq()
545 nq->cqn_wq = create_singlethread_workqueue("bnxt_qplib_nq"); in bnxt_qplib_enable_nq()
546 if (!nq->cqn_wq) in bnxt_qplib_enable_nq()
547 return -ENOMEM; in bnxt_qplib_enable_nq()
555 dev_err(&nq->pdev->dev, in bnxt_qplib_enable_nq()
556 "Failed to request irq for nq-idx %d\n", nq_idx); in bnxt_qplib_enable_nq()
568 if (nq->hwq.max_elements) { in bnxt_qplib_free_nq()
569 bnxt_qplib_free_hwq(nq->res, &nq->hwq); in bnxt_qplib_free_nq()
570 nq->hwq.max_elements = 0; in bnxt_qplib_free_nq()
579 nq->pdev = res->pdev; in bnxt_qplib_alloc_nq()
580 nq->res = res; in bnxt_qplib_alloc_nq()
581 if (!nq->hwq.max_elements || in bnxt_qplib_alloc_nq()
582 nq->hwq.max_elements > BNXT_QPLIB_NQE_MAX_CNT) in bnxt_qplib_alloc_nq()
583 nq->hwq.max_elements = BNXT_QPLIB_NQE_MAX_CNT; in bnxt_qplib_alloc_nq()
589 hwq_attr.depth = nq->hwq.max_elements; in bnxt_qplib_alloc_nq()
590 hwq_attr.stride = sizeof(struct nq_base); in bnxt_qplib_alloc_nq()
591 hwq_attr.type = bnxt_qplib_get_hwq_type(nq->res); in bnxt_qplib_alloc_nq()
592 if (bnxt_qplib_alloc_init_hwq(&nq->hwq, &hwq_attr)) { in bnxt_qplib_alloc_nq()
593 dev_err(&nq->pdev->dev, "FP NQ allocation failed"); in bnxt_qplib_alloc_nq()
594 return -ENOMEM; in bnxt_qplib_alloc_nq()
596 nq->budget = 8; in bnxt_qplib_alloc_nq()
604 struct bnxt_qplib_rcfw *rcfw = res->rcfw; in bnxt_qplib_destroy_srq()
615 req.srq_cid = cpu_to_le32(srq->id); in bnxt_qplib_destroy_srq()
619 kfree(srq->swq); in bnxt_qplib_destroy_srq()
622 bnxt_qplib_free_hwq(res, &srq->hwq); in bnxt_qplib_destroy_srq()
628 struct bnxt_qplib_rcfw *rcfw = res->rcfw; in bnxt_qplib_create_srq()
638 hwq_attr.sginfo = &srq->sg_info; in bnxt_qplib_create_srq()
639 hwq_attr.depth = srq->max_wqe; in bnxt_qplib_create_srq()
640 hwq_attr.stride = srq->wqe_size; in bnxt_qplib_create_srq()
642 rc = bnxt_qplib_alloc_init_hwq(&srq->hwq, &hwq_attr); in bnxt_qplib_create_srq()
646 srq->swq = kcalloc(srq->hwq.max_elements, sizeof(*srq->swq), in bnxt_qplib_create_srq()
648 if (!srq->swq) { in bnxt_qplib_create_srq()
649 rc = -ENOMEM; in bnxt_qplib_create_srq()
652 srq->dbinfo.flags = 0; in bnxt_qplib_create_srq()
658 req.dpi = cpu_to_le32(srq->dpi->dpi); in bnxt_qplib_create_srq()
661 req.srq_size = cpu_to_le16((u16)srq->hwq.max_elements); in bnxt_qplib_create_srq()
662 pbl = &srq->hwq.pbl[PBL_LVL_0]; in bnxt_qplib_create_srq()
663 pg_sz_lvl = ((u16)bnxt_qplib_base_pg_size(&srq->hwq) << in bnxt_qplib_create_srq()
665 pg_sz_lvl |= (srq->hwq.level & CMDQ_CREATE_SRQ_LVL_MASK) << in bnxt_qplib_create_srq()
668 req.pbl = cpu_to_le64(pbl->pg_map_arr[0]); in bnxt_qplib_create_srq()
669 req.pd_id = cpu_to_le32(srq->pd->id); in bnxt_qplib_create_srq()
670 req.eventq_id = cpu_to_le16(srq->eventq_hw_ring_id); in bnxt_qplib_create_srq()
677 spin_lock_init(&srq->lock); in bnxt_qplib_create_srq()
678 srq->start_idx = 0; in bnxt_qplib_create_srq()
679 srq->last_idx = srq->hwq.max_elements - 1; in bnxt_qplib_create_srq()
680 for (idx = 0; idx < srq->hwq.max_elements; idx++) in bnxt_qplib_create_srq()
681 srq->swq[idx].next_idx = idx + 1; in bnxt_qplib_create_srq()
682 srq->swq[srq->last_idx].next_idx = -1; in bnxt_qplib_create_srq()
684 srq->id = le32_to_cpu(resp.xid); in bnxt_qplib_create_srq()
685 srq->dbinfo.hwq = &srq->hwq; in bnxt_qplib_create_srq()
686 srq->dbinfo.xid = srq->id; in bnxt_qplib_create_srq()
687 srq->dbinfo.db = srq->dpi->dbr; in bnxt_qplib_create_srq()
688 srq->dbinfo.max_slot = 1; in bnxt_qplib_create_srq()
689 srq->dbinfo.priv_db = res->dpi_tbl.priv_db; in bnxt_qplib_create_srq()
690 if (srq->threshold) in bnxt_qplib_create_srq()
691 bnxt_qplib_armen_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ_ARMENA); in bnxt_qplib_create_srq()
692 srq->arm_req = false; in bnxt_qplib_create_srq()
696 bnxt_qplib_free_hwq(res, &srq->hwq); in bnxt_qplib_create_srq()
697 kfree(srq->swq); in bnxt_qplib_create_srq()
705 struct bnxt_qplib_hwq *srq_hwq = &srq->hwq; in bnxt_qplib_modify_srq()
709 if (count > srq->threshold) { in bnxt_qplib_modify_srq()
710 srq->arm_req = false; in bnxt_qplib_modify_srq()
711 bnxt_qplib_srq_arm_db(&srq->dbinfo, srq->threshold); in bnxt_qplib_modify_srq()
714 srq->arm_req = true; in bnxt_qplib_modify_srq()
723 struct bnxt_qplib_rcfw *rcfw = res->rcfw; in bnxt_qplib_query_srq()
737 sbuf.sb = dma_alloc_coherent(&rcfw->pdev->dev, sbuf.size, in bnxt_qplib_query_srq()
740 return -ENOMEM; in bnxt_qplib_query_srq()
742 req.srq_cid = cpu_to_le32(srq->id); in bnxt_qplib_query_srq()
748 srq->threshold = le16_to_cpu(sb->srq_limit); in bnxt_qplib_query_srq()
749 dma_free_coherent(&rcfw->pdev->dev, sbuf.size, in bnxt_qplib_query_srq()
758 struct bnxt_qplib_hwq *srq_hwq = &srq->hwq; in bnxt_qplib_post_srq_recv()
764 spin_lock(&srq_hwq->lock); in bnxt_qplib_post_srq_recv()
765 if (srq->start_idx == srq->last_idx) { in bnxt_qplib_post_srq_recv()
766 dev_err(&srq_hwq->pdev->dev, in bnxt_qplib_post_srq_recv()
767 "FP: SRQ (0x%x) is full!\n", srq->id); in bnxt_qplib_post_srq_recv()
768 spin_unlock(&srq_hwq->lock); in bnxt_qplib_post_srq_recv()
769 return -EINVAL; in bnxt_qplib_post_srq_recv()
771 next = srq->start_idx; in bnxt_qplib_post_srq_recv()
772 srq->start_idx = srq->swq[next].next_idx; in bnxt_qplib_post_srq_recv()
773 spin_unlock(&srq_hwq->lock); in bnxt_qplib_post_srq_recv()
775 srqe = bnxt_qplib_get_qe(srq_hwq, srq_hwq->prod, NULL); in bnxt_qplib_post_srq_recv()
776 memset(srqe, 0, srq->wqe_size); in bnxt_qplib_post_srq_recv()
778 for (i = 0, hw_sge = (struct sq_sge *)srqe->data; in bnxt_qplib_post_srq_recv()
779 i < wqe->num_sge; i++, hw_sge++) { in bnxt_qplib_post_srq_recv()
780 hw_sge->va_or_pa = cpu_to_le64(wqe->sg_list[i].addr); in bnxt_qplib_post_srq_recv()
781 hw_sge->l_key = cpu_to_le32(wqe->sg_list[i].lkey); in bnxt_qplib_post_srq_recv()
782 hw_sge->size = cpu_to_le32(wqe->sg_list[i].size); in bnxt_qplib_post_srq_recv()
784 srqe->wqe_type = wqe->type; in bnxt_qplib_post_srq_recv()
785 srqe->flags = wqe->flags; in bnxt_qplib_post_srq_recv()
786 srqe->wqe_size = wqe->num_sge + in bnxt_qplib_post_srq_recv()
788 srqe->wr_id[0] = cpu_to_le32((u32)next); in bnxt_qplib_post_srq_recv()
789 srq->swq[next].wr_id = wqe->wr_id; in bnxt_qplib_post_srq_recv()
791 bnxt_qplib_hwq_incr_prod(&srq->dbinfo, srq_hwq, srq->dbinfo.max_slot); in bnxt_qplib_post_srq_recv()
793 spin_lock(&srq_hwq->lock); in bnxt_qplib_post_srq_recv()
795 spin_unlock(&srq_hwq->lock); in bnxt_qplib_post_srq_recv()
797 bnxt_qplib_ring_prod_db(&srq->dbinfo, DBC_DBC_TYPE_SRQ); in bnxt_qplib_post_srq_recv()
798 if (srq->arm_req == true && count > srq->threshold) { in bnxt_qplib_post_srq_recv()
799 srq->arm_req = false; in bnxt_qplib_post_srq_recv()
800 bnxt_qplib_srq_arm_db(&srq->dbinfo, srq->threshold); in bnxt_qplib_post_srq_recv()
812 que->swq = kcalloc(que->max_wqe, sizeof(*que->swq), GFP_KERNEL); in bnxt_qplib_alloc_init_swq()
813 if (!que->swq) in bnxt_qplib_alloc_init_swq()
814 return -ENOMEM; in bnxt_qplib_alloc_init_swq()
816 que->swq_start = 0; in bnxt_qplib_alloc_init_swq()
817 que->swq_last = que->max_wqe - 1; in bnxt_qplib_alloc_init_swq()
818 for (indx = 0; indx < que->max_wqe; indx++) in bnxt_qplib_alloc_init_swq()
819 que->swq[indx].next_idx = indx + 1; in bnxt_qplib_alloc_init_swq()
820 que->swq[que->swq_last].next_idx = 0; /* Make it circular */ in bnxt_qplib_alloc_init_swq()
821 que->swq_last = 0; in bnxt_qplib_alloc_init_swq()
829 struct bnxt_qplib_rcfw *rcfw = res->rcfw; in bnxt_qplib_create_qp1()
832 struct bnxt_qplib_q *sq = &qp->sq; in bnxt_qplib_create_qp1()
833 struct bnxt_qplib_q *rq = &qp->rq; in bnxt_qplib_create_qp1()
841 sq->dbinfo.flags = 0; in bnxt_qplib_create_qp1()
846 req.type = qp->type; in bnxt_qplib_create_qp1()
847 req.dpi = cpu_to_le32(qp->dpi->dpi); in bnxt_qplib_create_qp1()
848 req.qp_handle = cpu_to_le64(qp->qp_handle); in bnxt_qplib_create_qp1()
852 hwq_attr.sginfo = &sq->sg_info; in bnxt_qplib_create_qp1()
853 hwq_attr.stride = sizeof(struct sq_sge); in bnxt_qplib_create_qp1()
856 rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr); in bnxt_qplib_create_qp1()
864 req.sq_size = cpu_to_le32(bnxt_qplib_set_sq_size(sq, qp->wqe_mode)); in bnxt_qplib_create_qp1()
865 pbl = &sq->hwq.pbl[PBL_LVL_0]; in bnxt_qplib_create_qp1()
866 req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); in bnxt_qplib_create_qp1()
867 pg_sz_lvl = (bnxt_qplib_base_pg_size(&sq->hwq) << in bnxt_qplib_create_qp1()
869 pg_sz_lvl |= (sq->hwq.level & CMDQ_CREATE_QP1_SQ_LVL_MASK); in bnxt_qplib_create_qp1()
872 cpu_to_le16((sq->max_sge & CMDQ_CREATE_QP1_SQ_SGE_MASK) << in bnxt_qplib_create_qp1()
874 req.scq_cid = cpu_to_le32(qp->scq->id); in bnxt_qplib_create_qp1()
877 if (rq->max_wqe) { in bnxt_qplib_create_qp1()
878 rq->dbinfo.flags = 0; in bnxt_qplib_create_qp1()
880 hwq_attr.sginfo = &rq->sg_info; in bnxt_qplib_create_qp1()
881 hwq_attr.stride = sizeof(struct sq_sge); in bnxt_qplib_create_qp1()
884 rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr); in bnxt_qplib_create_qp1()
890 req.rq_size = cpu_to_le32(rq->max_wqe); in bnxt_qplib_create_qp1()
891 pbl = &rq->hwq.pbl[PBL_LVL_0]; in bnxt_qplib_create_qp1()
892 req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); in bnxt_qplib_create_qp1()
893 pg_sz_lvl = (bnxt_qplib_base_pg_size(&rq->hwq) << in bnxt_qplib_create_qp1()
895 pg_sz_lvl |= (rq->hwq.level & CMDQ_CREATE_QP1_RQ_LVL_MASK); in bnxt_qplib_create_qp1()
898 cpu_to_le16((rq->max_sge & in bnxt_qplib_create_qp1()
902 req.rcq_cid = cpu_to_le32(qp->rcq->id); in bnxt_qplib_create_qp1()
903 /* Header buffer - allow hdr_buf pass in */ in bnxt_qplib_create_qp1()
906 rc = -ENOMEM; in bnxt_qplib_create_qp1()
911 req.pd_id = cpu_to_le32(qp->pd->id); in bnxt_qplib_create_qp1()
918 qp->id = le32_to_cpu(resp.xid); in bnxt_qplib_create_qp1()
919 qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET; in bnxt_qplib_create_qp1()
920 qp->cctx = res->cctx; in bnxt_qplib_create_qp1()
921 sq->dbinfo.hwq = &sq->hwq; in bnxt_qplib_create_qp1()
922 sq->dbinfo.xid = qp->id; in bnxt_qplib_create_qp1()
923 sq->dbinfo.db = qp->dpi->dbr; in bnxt_qplib_create_qp1()
924 sq->dbinfo.max_slot = bnxt_qplib_set_sq_max_slot(qp->wqe_mode); in bnxt_qplib_create_qp1()
925 if (rq->max_wqe) { in bnxt_qplib_create_qp1()
926 rq->dbinfo.hwq = &rq->hwq; in bnxt_qplib_create_qp1()
927 rq->dbinfo.xid = qp->id; in bnxt_qplib_create_qp1()
928 rq->dbinfo.db = qp->dpi->dbr; in bnxt_qplib_create_qp1()
929 rq->dbinfo.max_slot = bnxt_qplib_set_rq_max_slot(rq->wqe_size); in bnxt_qplib_create_qp1()
931 tbl_indx = map_qp_id_to_tbl_indx(qp->id, rcfw); in bnxt_qplib_create_qp1()
932 rcfw->qp_tbl[tbl_indx].qp_id = qp->id; in bnxt_qplib_create_qp1()
933 rcfw->qp_tbl[tbl_indx].qp_handle = (void *)qp; in bnxt_qplib_create_qp1()
940 kfree(rq->swq); in bnxt_qplib_create_qp1()
942 bnxt_qplib_free_hwq(res, &rq->hwq); in bnxt_qplib_create_qp1()
944 kfree(sq->swq); in bnxt_qplib_create_qp1()
946 bnxt_qplib_free_hwq(res, &sq->hwq); in bnxt_qplib_create_qp1()
957 sq = &qp->sq; in bnxt_qplib_init_psn_ptr()
958 hwq = &sq->hwq; in bnxt_qplib_init_psn_ptr()
960 fpsne = (u64)bnxt_qplib_get_qe(hwq, hwq->depth, &psn_pg); in bnxt_qplib_init_psn_ptr()
963 hwq->pad_pgofft = indx_pad; in bnxt_qplib_init_psn_ptr()
964 hwq->pad_pg = (u64 *)psn_pg; in bnxt_qplib_init_psn_ptr()
965 hwq->pad_stride = size; in bnxt_qplib_init_psn_ptr()
970 struct bnxt_qplib_rcfw *rcfw = res->rcfw; in bnxt_qplib_create_qp()
975 struct bnxt_qplib_q *sq = &qp->sq; in bnxt_qplib_create_qp()
976 struct bnxt_qplib_q *rq = &qp->rq; in bnxt_qplib_create_qp()
986 if (res->dattr) in bnxt_qplib_create_qp()
987 qp->dev_cap_flags = res->dattr->dev_cap_flags; in bnxt_qplib_create_qp()
989 sq->dbinfo.flags = 0; in bnxt_qplib_create_qp()
995 req.type = qp->type; in bnxt_qplib_create_qp()
996 req.dpi = cpu_to_le32(qp->dpi->dpi); in bnxt_qplib_create_qp()
997 req.qp_handle = cpu_to_le64(qp->qp_handle); in bnxt_qplib_create_qp()
1000 if (qp->type == CMDQ_CREATE_QP_TYPE_RC) { in bnxt_qplib_create_qp()
1001 psn_sz = bnxt_qplib_is_chip_gen_p5_p7(res->cctx) ? in bnxt_qplib_create_qp()
1005 if (BNXT_RE_HW_RETX(qp->dev_cap_flags)) { in bnxt_qplib_create_qp()
1007 qp->msn = 0; in bnxt_qplib_create_qp()
1012 hwq_attr.sginfo = &sq->sg_info; in bnxt_qplib_create_qp()
1013 hwq_attr.stride = sizeof(struct sq_sge); in bnxt_qplib_create_qp()
1016 hwq_attr.aux_depth = bnxt_qplib_set_sq_size(sq, qp->wqe_mode); in bnxt_qplib_create_qp()
1018 if (BNXT_RE_HW_RETX(qp->dev_cap_flags) && psn_sz) { in bnxt_qplib_create_qp()
1019 hwq_attr.aux_depth = roundup_pow_of_two(bnxt_qplib_set_sq_size(sq, qp->wqe_mode)); in bnxt_qplib_create_qp()
1020 qp->msn_tbl_sz = hwq_attr.aux_depth; in bnxt_qplib_create_qp()
1021 qp->msn = 0; in bnxt_qplib_create_qp()
1025 rc = bnxt_qplib_alloc_init_hwq(&sq->hwq, &hwq_attr); in bnxt_qplib_create_qp()
1036 req.sq_size = cpu_to_le32(bnxt_qplib_set_sq_size(sq, qp->wqe_mode)); in bnxt_qplib_create_qp()
1037 pbl = &sq->hwq.pbl[PBL_LVL_0]; in bnxt_qplib_create_qp()
1038 req.sq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); in bnxt_qplib_create_qp()
1039 pg_sz_lvl = (bnxt_qplib_base_pg_size(&sq->hwq) << in bnxt_qplib_create_qp()
1041 pg_sz_lvl |= (sq->hwq.level & CMDQ_CREATE_QP_SQ_LVL_MASK); in bnxt_qplib_create_qp()
1044 cpu_to_le16(((sq->max_sge & CMDQ_CREATE_QP_SQ_SGE_MASK) << in bnxt_qplib_create_qp()
1046 req.scq_cid = cpu_to_le32(qp->scq->id); in bnxt_qplib_create_qp()
1049 if (!qp->srq) { in bnxt_qplib_create_qp()
1050 rq->dbinfo.flags = 0; in bnxt_qplib_create_qp()
1052 hwq_attr.sginfo = &rq->sg_info; in bnxt_qplib_create_qp()
1053 hwq_attr.stride = sizeof(struct sq_sge); in bnxt_qplib_create_qp()
1058 rc = bnxt_qplib_alloc_init_hwq(&rq->hwq, &hwq_attr); in bnxt_qplib_create_qp()
1065 req.rq_size = cpu_to_le32(rq->max_wqe); in bnxt_qplib_create_qp()
1066 pbl = &rq->hwq.pbl[PBL_LVL_0]; in bnxt_qplib_create_qp()
1067 req.rq_pbl = cpu_to_le64(pbl->pg_map_arr[0]); in bnxt_qplib_create_qp()
1068 pg_sz_lvl = (bnxt_qplib_base_pg_size(&rq->hwq) << in bnxt_qplib_create_qp()
1070 pg_sz_lvl |= (rq->hwq.level & CMDQ_CREATE_QP_RQ_LVL_MASK); in bnxt_qplib_create_qp()
1072 nsge = (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_STATIC) ? in bnxt_qplib_create_qp()
1073 6 : rq->max_sge; in bnxt_qplib_create_qp()
1081 req.srq_cid = cpu_to_le32(qp->srq->id); in bnxt_qplib_create_qp()
1083 req.rcq_cid = cpu_to_le32(qp->rcq->id); in bnxt_qplib_create_qp()
1087 if (qp->sig_type) in bnxt_qplib_create_qp()
1089 if (qp->wqe_mode == BNXT_QPLIB_WQE_MODE_VARIABLE) in bnxt_qplib_create_qp()
1091 if (_is_ext_stats_supported(res->dattr->dev_cap_flags) && !res->is_vf) in bnxt_qplib_create_qp()
1098 xrrq = &qp->orrq; in bnxt_qplib_create_qp()
1099 xrrq->max_elements = in bnxt_qplib_create_qp()
1100 ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic); in bnxt_qplib_create_qp()
1101 req_size = xrrq->max_elements * in bnxt_qplib_create_qp()
1102 BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE + PAGE_SIZE - 1; in bnxt_qplib_create_qp()
1103 req_size &= ~(PAGE_SIZE - 1); in bnxt_qplib_create_qp()
1109 hwq_attr.depth = xrrq->max_elements; in bnxt_qplib_create_qp()
1110 hwq_attr.stride = BNXT_QPLIB_MAX_ORRQE_ENTRY_SIZE; in bnxt_qplib_create_qp()
1117 pbl = &xrrq->pbl[PBL_LVL_0]; in bnxt_qplib_create_qp()
1118 req.orrq_addr = cpu_to_le64(pbl->pg_map_arr[0]); in bnxt_qplib_create_qp()
1120 xrrq = &qp->irrq; in bnxt_qplib_create_qp()
1121 xrrq->max_elements = IRD_LIMIT_TO_IRRQ_SLOTS( in bnxt_qplib_create_qp()
1122 qp->max_dest_rd_atomic); in bnxt_qplib_create_qp()
1123 req_size = xrrq->max_elements * in bnxt_qplib_create_qp()
1124 BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE + PAGE_SIZE - 1; in bnxt_qplib_create_qp()
1125 req_size &= ~(PAGE_SIZE - 1); in bnxt_qplib_create_qp()
1127 hwq_attr.depth = xrrq->max_elements; in bnxt_qplib_create_qp()
1128 hwq_attr.stride = BNXT_QPLIB_MAX_IRRQE_ENTRY_SIZE; in bnxt_qplib_create_qp()
1133 pbl = &xrrq->pbl[PBL_LVL_0]; in bnxt_qplib_create_qp()
1134 req.irrq_addr = cpu_to_le64(pbl->pg_map_arr[0]); in bnxt_qplib_create_qp()
1136 req.pd_id = cpu_to_le32(qp->pd->id); in bnxt_qplib_create_qp()
1144 qp->id = le32_to_cpu(resp.xid); in bnxt_qplib_create_qp()
1145 qp->cur_qp_state = CMDQ_MODIFY_QP_NEW_STATE_RESET; in bnxt_qplib_create_qp()
1146 INIT_LIST_HEAD(&qp->sq_flush); in bnxt_qplib_create_qp()
1147 INIT_LIST_HEAD(&qp->rq_flush); in bnxt_qplib_create_qp()
1148 qp->cctx = res->cctx; in bnxt_qplib_create_qp()
1149 sq->dbinfo.hwq = &sq->hwq; in bnxt_qplib_create_qp()
1150 sq->dbinfo.xid = qp->id; in bnxt_qplib_create_qp()
1151 sq->dbinfo.db = qp->dpi->dbr; in bnxt_qplib_create_qp()
1152 sq->dbinfo.max_slot = bnxt_qplib_set_sq_max_slot(qp->wqe_mode); in bnxt_qplib_create_qp()
1153 if (rq->max_wqe) { in bnxt_qplib_create_qp()
1154 rq->dbinfo.hwq = &rq->hwq; in bnxt_qplib_create_qp()
1155 rq->dbinfo.xid = qp->id; in bnxt_qplib_create_qp()
1156 rq->dbinfo.db = qp->dpi->dbr; in bnxt_qplib_create_qp()
1157 rq->dbinfo.max_slot = bnxt_qplib_set_rq_max_slot(rq->wqe_size); in bnxt_qplib_create_qp()
1159 tbl_indx = map_qp_id_to_tbl_indx(qp->id, rcfw); in bnxt_qplib_create_qp()
1160 rcfw->qp_tbl[tbl_indx].qp_id = qp->id; in bnxt_qplib_create_qp()
1161 rcfw->qp_tbl[tbl_indx].qp_handle = (void *)qp; in bnxt_qplib_create_qp()
1165 bnxt_qplib_free_hwq(res, &qp->irrq); in bnxt_qplib_create_qp()
1167 bnxt_qplib_free_hwq(res, &qp->orrq); in bnxt_qplib_create_qp()
1169 kfree(rq->swq); in bnxt_qplib_create_qp()
1171 bnxt_qplib_free_hwq(res, &rq->hwq); in bnxt_qplib_create_qp()
1173 kfree(sq->swq); in bnxt_qplib_create_qp()
1175 bnxt_qplib_free_hwq(res, &sq->hwq); in bnxt_qplib_create_qp()
1181 switch (qp->state) { in __modify_flags_from_init_state()
1183 /* INIT->RTR, configure the path_mtu to the default in __modify_flags_from_init_state()
1186 if (!(qp->modify_flags & in __modify_flags_from_init_state()
1188 qp->modify_flags |= in __modify_flags_from_init_state()
1190 qp->path_mtu = in __modify_flags_from_init_state()
1193 qp->modify_flags &= in __modify_flags_from_init_state()
1196 if (qp->max_dest_rd_atomic < 1) in __modify_flags_from_init_state()
1197 qp->max_dest_rd_atomic = 1; in __modify_flags_from_init_state()
1198 qp->modify_flags &= ~CMDQ_MODIFY_QP_MODIFY_MASK_SRC_MAC; in __modify_flags_from_init_state()
1200 if (!(qp->modify_flags & in __modify_flags_from_init_state()
1202 qp->modify_flags |= in __modify_flags_from_init_state()
1204 qp->ah.sgid_index = 0; in __modify_flags_from_init_state()
1214 switch (qp->state) { in __modify_flags_from_rtr_state()
1217 if (qp->max_rd_atomic < 1) in __modify_flags_from_rtr_state()
1218 qp->max_rd_atomic = 1; in __modify_flags_from_rtr_state()
1225 qp->modify_flags &= in __modify_flags_from_rtr_state()
1246 switch (qp->cur_qp_state) { in __filter_modify_flags()
1270 struct bnxt_qplib_rcfw *rcfw = res->rcfw; in bnxt_qplib_modify_qp()
1282 /* Filter out the qp_attr_mask based on the state->new transition */ in bnxt_qplib_modify_qp()
1284 bmask = qp->modify_flags; in bnxt_qplib_modify_qp()
1285 req.modify_mask = cpu_to_le32(qp->modify_flags); in bnxt_qplib_modify_qp()
1286 req.qp_cid = cpu_to_le32(qp->id); in bnxt_qplib_modify_qp()
1289 (qp->state & CMDQ_MODIFY_QP_NEW_STATE_MASK) | in bnxt_qplib_modify_qp()
1290 (qp->en_sqd_async_notify ? in bnxt_qplib_modify_qp()
1293 req.network_type_en_sqd_async_notify_new_state |= qp->nw_type; in bnxt_qplib_modify_qp()
1296 req.access = qp->access; in bnxt_qplib_modify_qp()
1302 req.qkey = cpu_to_le32(qp->qkey); in bnxt_qplib_modify_qp()
1305 memcpy(temp32, qp->ah.dgid.data, sizeof(struct bnxt_qplib_gid)); in bnxt_qplib_modify_qp()
1312 req.flow_label = cpu_to_le32(qp->ah.flow_label); in bnxt_qplib_modify_qp()
1315 req.sgid_index = cpu_to_le16(res->sgid_tbl.hw_id in bnxt_qplib_modify_qp()
1316 [qp->ah.sgid_index]); in bnxt_qplib_modify_qp()
1319 req.hop_limit = qp->ah.hop_limit; in bnxt_qplib_modify_qp()
1322 req.traffic_class = qp->ah.traffic_class; in bnxt_qplib_modify_qp()
1325 memcpy(req.dest_mac, qp->ah.dmac, 6); in bnxt_qplib_modify_qp()
1328 req.path_mtu_pingpong_push_enable |= qp->path_mtu; in bnxt_qplib_modify_qp()
1331 req.timeout = qp->timeout; in bnxt_qplib_modify_qp()
1334 req.retry_cnt = qp->retry_cnt; in bnxt_qplib_modify_qp()
1337 req.rnr_retry = qp->rnr_retry; in bnxt_qplib_modify_qp()
1340 req.min_rnr_timer = qp->min_rnr_timer; in bnxt_qplib_modify_qp()
1343 req.rq_psn = cpu_to_le32(qp->rq.psn); in bnxt_qplib_modify_qp()
1346 req.sq_psn = cpu_to_le32(qp->sq.psn); in bnxt_qplib_modify_qp()
1350 ORD_LIMIT_TO_ORRQ_SLOTS(qp->max_rd_atomic); in bnxt_qplib_modify_qp()
1354 IRD_LIMIT_TO_IRRQ_SLOTS(qp->max_dest_rd_atomic); in bnxt_qplib_modify_qp()
1356 req.sq_size = cpu_to_le32(qp->sq.hwq.max_elements); in bnxt_qplib_modify_qp()
1357 req.rq_size = cpu_to_le32(qp->rq.hwq.max_elements); in bnxt_qplib_modify_qp()
1358 req.sq_sge = cpu_to_le16(qp->sq.max_sge); in bnxt_qplib_modify_qp()
1359 req.rq_sge = cpu_to_le16(qp->rq.max_sge); in bnxt_qplib_modify_qp()
1360 req.max_inline_data = cpu_to_le32(qp->max_inline_data); in bnxt_qplib_modify_qp()
1362 req.dest_qp_id = cpu_to_le32(qp->dest_qpn); in bnxt_qplib_modify_qp()
1364 req.vlan_pcp_vlan_dei_vlan_id = cpu_to_le16(qp->vlan_id); in bnxt_qplib_modify_qp()
1370 qp->cur_qp_state = qp->state; in bnxt_qplib_modify_qp()
1376 struct bnxt_qplib_rcfw *rcfw = res->rcfw; in bnxt_qplib_query_qp()
1386 sbuf.sb = dma_alloc_coherent(&rcfw->pdev->dev, sbuf.size, in bnxt_qplib_query_qp()
1389 return -ENOMEM; in bnxt_qplib_query_qp()
1396 req.qp_cid = cpu_to_le32(qp->id); in bnxt_qplib_query_qp()
1404 qp->state = sb->en_sqd_async_notify_state & in bnxt_qplib_query_qp()
1406 qp->en_sqd_async_notify = sb->en_sqd_async_notify_state & in bnxt_qplib_query_qp()
1408 qp->access = sb->access; in bnxt_qplib_query_qp()
1409 qp->pkey_index = le16_to_cpu(sb->pkey); in bnxt_qplib_query_qp()
1410 qp->qkey = le32_to_cpu(sb->qkey); in bnxt_qplib_query_qp()
1412 temp32[0] = le32_to_cpu(sb->dgid[0]); in bnxt_qplib_query_qp()
1413 temp32[1] = le32_to_cpu(sb->dgid[1]); in bnxt_qplib_query_qp()
1414 temp32[2] = le32_to_cpu(sb->dgid[2]); in bnxt_qplib_query_qp()
1415 temp32[3] = le32_to_cpu(sb->dgid[3]); in bnxt_qplib_query_qp()
1416 memcpy(qp->ah.dgid.data, temp32, sizeof(qp->ah.dgid.data)); in bnxt_qplib_query_qp()
1418 qp->ah.flow_label = le32_to_cpu(sb->flow_label); in bnxt_qplib_query_qp()
1420 qp->ah.sgid_index = 0; in bnxt_qplib_query_qp()
1421 for (i = 0; i < res->sgid_tbl.max; i++) { in bnxt_qplib_query_qp()
1422 if (res->sgid_tbl.hw_id[i] == le16_to_cpu(sb->sgid_index)) { in bnxt_qplib_query_qp()
1423 qp->ah.sgid_index = i; in bnxt_qplib_query_qp()
1427 if (i == res->sgid_tbl.max) in bnxt_qplib_query_qp()
1428 dev_warn(&res->pdev->dev, "SGID not found??\n"); in bnxt_qplib_query_qp()
1430 qp->ah.hop_limit = sb->hop_limit; in bnxt_qplib_query_qp()
1431 qp->ah.traffic_class = sb->traffic_class; in bnxt_qplib_query_qp()
1432 memcpy(qp->ah.dmac, sb->dest_mac, 6); in bnxt_qplib_query_qp()
1433 qp->ah.vlan_id = (le16_to_cpu(sb->path_mtu_dest_vlan_id) & in bnxt_qplib_query_qp()
1436 qp->path_mtu = (le16_to_cpu(sb->path_mtu_dest_vlan_id) & in bnxt_qplib_query_qp()
1439 qp->timeout = sb->timeout; in bnxt_qplib_query_qp()
1440 qp->retry_cnt = sb->retry_cnt; in bnxt_qplib_query_qp()
1441 qp->rnr_retry = sb->rnr_retry; in bnxt_qplib_query_qp()
1442 qp->min_rnr_timer = sb->min_rnr_timer; in bnxt_qplib_query_qp()
1443 qp->rq.psn = le32_to_cpu(sb->rq_psn); in bnxt_qplib_query_qp()
1444 qp->max_rd_atomic = ORRQ_SLOTS_TO_ORD_LIMIT(sb->max_rd_atomic); in bnxt_qplib_query_qp()
1445 qp->sq.psn = le32_to_cpu(sb->sq_psn); in bnxt_qplib_query_qp()
1446 qp->max_dest_rd_atomic = in bnxt_qplib_query_qp()
1447 IRRQ_SLOTS_TO_IRD_LIMIT(sb->max_dest_rd_atomic); in bnxt_qplib_query_qp()
1448 qp->sq.max_wqe = qp->sq.hwq.max_elements; in bnxt_qplib_query_qp()
1449 qp->rq.max_wqe = qp->rq.hwq.max_elements; in bnxt_qplib_query_qp()
1450 qp->sq.max_sge = le16_to_cpu(sb->sq_sge); in bnxt_qplib_query_qp()
1451 qp->rq.max_sge = le16_to_cpu(sb->rq_sge); in bnxt_qplib_query_qp()
1452 qp->max_inline_data = le32_to_cpu(sb->max_inline_data); in bnxt_qplib_query_qp()
1453 qp->dest_qpn = le32_to_cpu(sb->dest_qp_id); in bnxt_qplib_query_qp()
1454 memcpy(qp->smac, sb->src_mac, 6); in bnxt_qplib_query_qp()
1455 qp->vlan_id = le16_to_cpu(sb->vlan_pcp_vlan_dei_vlan_id); in bnxt_qplib_query_qp()
1457 dma_free_coherent(&rcfw->pdev->dev, sbuf.size, in bnxt_qplib_query_qp()
1464 struct bnxt_qplib_hwq *cq_hwq = &cq->hwq; in __clean_cq()
1469 peek_flags = cq->dbinfo.flags; in __clean_cq()
1470 peek_cons = cq_hwq->cons; in __clean_cq()
1471 for (i = 0; i < cq_hwq->max_elements; i++) { in __clean_cq()
1480 switch (hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK) { in __clean_cq()
1486 if (qp == le64_to_cpu(cqe->qp_handle)) in __clean_cq()
1487 cqe->qp_handle = 0; in __clean_cq()
1496 if (qp == le64_to_cpu(cqe->qp_handle)) in __clean_cq()
1497 cqe->qp_handle = 0; in __clean_cq()
1503 bnxt_qplib_hwq_incr_cons(cq_hwq->max_elements, &peek_cons, in __clean_cq()
1511 struct bnxt_qplib_rcfw *rcfw = res->rcfw; in bnxt_qplib_destroy_qp()
1518 tbl_indx = map_qp_id_to_tbl_indx(qp->id, rcfw); in bnxt_qplib_destroy_qp()
1519 rcfw->qp_tbl[tbl_indx].qp_id = BNXT_QPLIB_QP_ID_INVALID; in bnxt_qplib_destroy_qp()
1520 rcfw->qp_tbl[tbl_indx].qp_handle = NULL; in bnxt_qplib_destroy_qp()
1526 req.qp_cid = cpu_to_le32(qp->id); in bnxt_qplib_destroy_qp()
1531 rcfw->qp_tbl[tbl_indx].qp_id = qp->id; in bnxt_qplib_destroy_qp()
1532 rcfw->qp_tbl[tbl_indx].qp_handle = qp; in bnxt_qplib_destroy_qp()
1543 bnxt_qplib_free_hwq(res, &qp->sq.hwq); in bnxt_qplib_free_qp_res()
1544 kfree(qp->sq.swq); in bnxt_qplib_free_qp_res()
1546 bnxt_qplib_free_hwq(res, &qp->rq.hwq); in bnxt_qplib_free_qp_res()
1547 kfree(qp->rq.swq); in bnxt_qplib_free_qp_res()
1549 if (qp->irrq.max_elements) in bnxt_qplib_free_qp_res()
1550 bnxt_qplib_free_hwq(res, &qp->irrq); in bnxt_qplib_free_qp_res()
1551 if (qp->orrq.max_elements) in bnxt_qplib_free_qp_res()
1552 bnxt_qplib_free_hwq(res, &qp->orrq); in bnxt_qplib_free_qp_res()
1559 struct bnxt_qplib_q *sq = &qp->sq; in bnxt_qplib_get_qp1_sq_buf()
1564 if (qp->sq_hdr_buf) { in bnxt_qplib_get_qp1_sq_buf()
1565 sw_prod = sq->swq_start; in bnxt_qplib_get_qp1_sq_buf()
1566 sge->addr = (dma_addr_t)(qp->sq_hdr_buf_map + in bnxt_qplib_get_qp1_sq_buf()
1567 sw_prod * qp->sq_hdr_buf_size); in bnxt_qplib_get_qp1_sq_buf()
1568 sge->lkey = 0xFFFFFFFF; in bnxt_qplib_get_qp1_sq_buf()
1569 sge->size = qp->sq_hdr_buf_size; in bnxt_qplib_get_qp1_sq_buf()
1570 return qp->sq_hdr_buf + sw_prod * sge->size; in bnxt_qplib_get_qp1_sq_buf()
1577 struct bnxt_qplib_q *rq = &qp->rq; in bnxt_qplib_get_rq_prod_index()
1579 return rq->swq_start; in bnxt_qplib_get_rq_prod_index()
1584 return (qp->rq_hdr_buf_map + index * qp->rq_hdr_buf_size); in bnxt_qplib_get_qp_buf_from_index()
1590 struct bnxt_qplib_q *rq = &qp->rq; in bnxt_qplib_get_qp1_rq_buf()
1595 if (qp->rq_hdr_buf) { in bnxt_qplib_get_qp1_rq_buf()
1596 sw_prod = rq->swq_start; in bnxt_qplib_get_qp1_rq_buf()
1597 sge->addr = (dma_addr_t)(qp->rq_hdr_buf_map + in bnxt_qplib_get_qp1_rq_buf()
1598 sw_prod * qp->rq_hdr_buf_size); in bnxt_qplib_get_qp1_rq_buf()
1599 sge->lkey = 0xFFFFFFFF; in bnxt_qplib_get_qp1_rq_buf()
1600 sge->size = qp->rq_hdr_buf_size; in bnxt_qplib_get_qp1_rq_buf()
1601 return qp->rq_hdr_buf + sw_prod * sge->size; in bnxt_qplib_get_qp1_rq_buf()
1606 /* Fil the MSN table into the next psn row */
1615 msns = (struct sq_msn_search *)swq->psn_search; in bnxt_qplib_fill_msn_search()
1616 msns->start_idx_next_psn_start_psn = 0; in bnxt_qplib_fill_msn_search()
1618 start_psn = swq->start_psn; in bnxt_qplib_fill_msn_search()
1619 next_psn = swq->next_psn; in bnxt_qplib_fill_msn_search()
1620 start_idx = swq->slot_idx; in bnxt_qplib_fill_msn_search()
1621 msns->start_idx_next_psn_start_psn |= in bnxt_qplib_fill_msn_search()
1623 qp->msn++; in bnxt_qplib_fill_msn_search()
1624 qp->msn %= qp->msn_tbl_sz; in bnxt_qplib_fill_msn_search()
1636 if (!swq->psn_search) in bnxt_qplib_fill_psn_search()
1639 if (BNXT_RE_HW_RETX(qp->dev_cap_flags)) { in bnxt_qplib_fill_psn_search()
1643 psns = (struct sq_psn_search *)swq->psn_search; in bnxt_qplib_fill_psn_search()
1644 psns = swq->psn_search; in bnxt_qplib_fill_psn_search()
1645 psns_ext = swq->psn_ext; in bnxt_qplib_fill_psn_search()
1647 op_spsn = ((swq->start_psn << SQ_PSN_SEARCH_START_PSN_SFT) & in bnxt_qplib_fill_psn_search()
1649 op_spsn |= ((wqe->type << SQ_PSN_SEARCH_OPCODE_SFT) & in bnxt_qplib_fill_psn_search()
1651 flg_npsn = ((swq->next_psn << SQ_PSN_SEARCH_NEXT_PSN_SFT) & in bnxt_qplib_fill_psn_search()
1654 if (bnxt_qplib_is_chip_gen_p5_p7(qp->cctx)) { in bnxt_qplib_fill_psn_search()
1655 psns_ext->opcode_start_psn = cpu_to_le32(op_spsn); in bnxt_qplib_fill_psn_search()
1656 psns_ext->flags_next_psn = cpu_to_le32(flg_npsn); in bnxt_qplib_fill_psn_search()
1657 psns_ext->start_slot_idx = cpu_to_le16(swq->slot_idx); in bnxt_qplib_fill_psn_search()
1659 psns->opcode_start_psn = cpu_to_le32(op_spsn); in bnxt_qplib_fill_psn_search()
1660 psns->flags_next_psn = cpu_to_le32(flg_npsn); in bnxt_qplib_fill_psn_search()
1676 hwq = &qp->sq.hwq; in bnxt_qplib_put_inline()
1678 for (indx = 0; indx < wqe->num_sge; indx++) { in bnxt_qplib_put_inline()
1679 len = wqe->sg_list[indx].size; in bnxt_qplib_put_inline()
1680 il_src = (void *)wqe->sg_list[indx].addr; in bnxt_qplib_put_inline()
1682 if (t_len > qp->max_inline_data) in bnxt_qplib_put_inline()
1683 return -ENOMEM; in bnxt_qplib_put_inline()
1694 (sizeof(struct sq_sge) - offt)); in bnxt_qplib_put_inline()
1700 len -= cplen; in bnxt_qplib_put_inline()
1718 dsge->va_or_pa = cpu_to_le64(ssge[indx].addr); in bnxt_qplib_put_sges()
1719 dsge->l_key = cpu_to_le32(ssge[indx].lkey); in bnxt_qplib_put_sges()
1720 dsge->size = cpu_to_le32(ssge[indx].size); in bnxt_qplib_put_sges()
1735 nsge = wqe->num_sge; in bnxt_qplib_required_slots()
1738 if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) { in bnxt_qplib_required_slots()
1739 ilsize = bnxt_qplib_calc_ilsize(wqe, qp->max_inline_data); in bnxt_qplib_required_slots()
1744 *qdf = __xlate_qfd(qp->sq.q_full_delta, bytes); in bnxt_qplib_required_slots()
1760 hwq = &sq->hwq; in bnxt_qplib_pull_psn_buff()
1761 if (!hwq->pad_pg) in bnxt_qplib_pull_psn_buff()
1763 tail = swq->slot_idx / sq->dbinfo.max_slot; in bnxt_qplib_pull_psn_buff()
1766 tail = qp->msn; in bnxt_qplib_pull_psn_buff()
1767 tail %= qp->msn_tbl_sz; in bnxt_qplib_pull_psn_buff()
1769 pg_num = (tail + hwq->pad_pgofft) / (PAGE_SIZE / hwq->pad_stride); in bnxt_qplib_pull_psn_buff()
1770 pg_indx = (tail + hwq->pad_pgofft) % (PAGE_SIZE / hwq->pad_stride); in bnxt_qplib_pull_psn_buff()
1771 buff = (void *)(hwq->pad_pg[pg_num] + pg_indx * hwq->pad_stride); in bnxt_qplib_pull_psn_buff()
1772 swq->psn_ext = buff; in bnxt_qplib_pull_psn_buff()
1773 swq->psn_search = buff; in bnxt_qplib_pull_psn_buff()
1778 struct bnxt_qplib_q *sq = &qp->sq; in bnxt_qplib_post_send_db()
1780 bnxt_qplib_ring_prod_db(&sq->dbinfo, DBC_DBC_TYPE_SQ); in bnxt_qplib_post_send_db()
1788 struct bnxt_qplib_q *sq = &qp->sq; in bnxt_qplib_post_send()
1801 hwq = &sq->hwq; in bnxt_qplib_post_send()
1802 if (qp->state != CMDQ_MODIFY_QP_NEW_STATE_RTS && in bnxt_qplib_post_send()
1803 qp->state != CMDQ_MODIFY_QP_NEW_STATE_ERR) { in bnxt_qplib_post_send()
1804 dev_err(&hwq->pdev->dev, in bnxt_qplib_post_send()
1806 qp->id, qp->state); in bnxt_qplib_post_send()
1807 rc = -EINVAL; in bnxt_qplib_post_send()
1811 slots = bnxt_qplib_required_slots(qp, wqe, &wqe_sz, &qdf, qp->wqe_mode); in bnxt_qplib_post_send()
1813 dev_err(&hwq->pdev->dev, in bnxt_qplib_post_send()
1815 hwq->prod, hwq->cons, hwq->depth, sq->q_full_delta); in bnxt_qplib_post_send()
1816 rc = -ENOMEM; in bnxt_qplib_post_send()
1821 bnxt_qplib_pull_psn_buff(qp, sq, swq, BNXT_RE_HW_RETX(qp->dev_cap_flags)); in bnxt_qplib_post_send()
1824 swq->slot_idx = hwq->prod; in bnxt_qplib_post_send()
1825 swq->slots = slots; in bnxt_qplib_post_send()
1826 swq->wr_id = wqe->wr_id; in bnxt_qplib_post_send()
1827 swq->type = wqe->type; in bnxt_qplib_post_send()
1828 swq->flags = wqe->flags; in bnxt_qplib_post_send()
1829 swq->start_psn = sq->psn & BTH_PSN_MASK; in bnxt_qplib_post_send()
1830 if (qp->sig_type) in bnxt_qplib_post_send()
1831 swq->flags |= SQ_SEND_FLAGS_SIGNAL_COMP; in bnxt_qplib_post_send()
1833 if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { in bnxt_qplib_post_send()
1835 dev_dbg(&hwq->pdev->dev, in bnxt_qplib_post_send()
1845 if (wqe->flags & BNXT_QPLIB_SWQE_FLAGS_INLINE) in bnxt_qplib_post_send()
1849 data_len = bnxt_qplib_put_sges(hwq, wqe->sg_list, wqe->num_sge, in bnxt_qplib_post_send()
1856 switch (wqe->type) { in bnxt_qplib_post_send()
1858 if (qp->type == CMDQ_CREATE_QP1_TYPE_GSI) { in bnxt_qplib_post_send()
1863 sqe->wqe_type = wqe->type; in bnxt_qplib_post_send()
1864 sqe->flags = wqe->flags; in bnxt_qplib_post_send()
1865 sqe->wqe_size = wqe_sz; in bnxt_qplib_post_send()
1866 sqe->cfa_action = cpu_to_le16(wqe->rawqp1.cfa_action); in bnxt_qplib_post_send()
1867 sqe->lflags = cpu_to_le16(wqe->rawqp1.lflags); in bnxt_qplib_post_send()
1868 sqe->length = cpu_to_le32(data_len); in bnxt_qplib_post_send()
1869 ext_sqe->cfa_meta = cpu_to_le32((wqe->rawqp1.cfa_meta & in bnxt_qplib_post_send()
1882 sqe->wqe_type = wqe->type; in bnxt_qplib_post_send()
1883 sqe->flags = wqe->flags; in bnxt_qplib_post_send()
1884 sqe->wqe_size = wqe_sz; in bnxt_qplib_post_send()
1885 sqe->inv_key_or_imm_data = cpu_to_le32(wqe->send.inv_key); in bnxt_qplib_post_send()
1886 if (qp->type == CMDQ_CREATE_QP_TYPE_UD || in bnxt_qplib_post_send()
1887 qp->type == CMDQ_CREATE_QP_TYPE_GSI) { in bnxt_qplib_post_send()
1888 sqe->q_key = cpu_to_le32(wqe->send.q_key); in bnxt_qplib_post_send()
1889 sqe->length = cpu_to_le32(data_len); in bnxt_qplib_post_send()
1890 sq->psn = (sq->psn + 1) & BTH_PSN_MASK; in bnxt_qplib_post_send()
1891 ext_sqe->dst_qp = cpu_to_le32(wqe->send.dst_qp & in bnxt_qplib_post_send()
1893 ext_sqe->avid = cpu_to_le32(wqe->send.avid & in bnxt_qplib_post_send()
1897 sqe->length = cpu_to_le32(data_len); in bnxt_qplib_post_send()
1898 if (qp->mtu) in bnxt_qplib_post_send()
1899 pkt_num = (data_len + qp->mtu - 1) / qp->mtu; in bnxt_qplib_post_send()
1902 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK; in bnxt_qplib_post_send()
1913 sqe->wqe_type = wqe->type; in bnxt_qplib_post_send()
1914 sqe->flags = wqe->flags; in bnxt_qplib_post_send()
1915 sqe->wqe_size = wqe_sz; in bnxt_qplib_post_send()
1916 sqe->imm_data = cpu_to_le32(wqe->rdma.inv_key); in bnxt_qplib_post_send()
1917 sqe->length = cpu_to_le32((u32)data_len); in bnxt_qplib_post_send()
1918 ext_sqe->remote_va = cpu_to_le64(wqe->rdma.remote_va); in bnxt_qplib_post_send()
1919 ext_sqe->remote_key = cpu_to_le32(wqe->rdma.r_key); in bnxt_qplib_post_send()
1920 if (qp->mtu) in bnxt_qplib_post_send()
1921 pkt_num = (data_len + qp->mtu - 1) / qp->mtu; in bnxt_qplib_post_send()
1924 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK; in bnxt_qplib_post_send()
1933 sqe->wqe_type = wqe->type; in bnxt_qplib_post_send()
1934 sqe->flags = wqe->flags; in bnxt_qplib_post_send()
1935 sqe->remote_key = cpu_to_le32(wqe->atomic.r_key); in bnxt_qplib_post_send()
1936 sqe->remote_va = cpu_to_le64(wqe->atomic.remote_va); in bnxt_qplib_post_send()
1937 ext_sqe->swap_data = cpu_to_le64(wqe->atomic.swap_data); in bnxt_qplib_post_send()
1938 ext_sqe->cmp_data = cpu_to_le64(wqe->atomic.cmp_data); in bnxt_qplib_post_send()
1939 if (qp->mtu) in bnxt_qplib_post_send()
1940 pkt_num = (data_len + qp->mtu - 1) / qp->mtu; in bnxt_qplib_post_send()
1943 sq->psn = (sq->psn + pkt_num) & BTH_PSN_MASK; in bnxt_qplib_post_send()
1950 sqe->wqe_type = wqe->type; in bnxt_qplib_post_send()
1951 sqe->flags = wqe->flags; in bnxt_qplib_post_send()
1952 sqe->inv_l_key = cpu_to_le32(wqe->local_inv.inv_l_key); in bnxt_qplib_post_send()
1961 sqe->wqe_type = wqe->type; in bnxt_qplib_post_send()
1962 sqe->flags = wqe->flags; in bnxt_qplib_post_send()
1963 sqe->access_cntl = wqe->frmr.access_cntl | in bnxt_qplib_post_send()
1965 sqe->zero_based_page_size_log = in bnxt_qplib_post_send()
1966 (wqe->frmr.pg_sz_log & SQ_FR_PMR_PAGE_SIZE_LOG_MASK) << in bnxt_qplib_post_send()
1968 (wqe->frmr.zero_based ? SQ_FR_PMR_ZERO_BASED : 0); in bnxt_qplib_post_send()
1969 sqe->l_key = cpu_to_le32(wqe->frmr.l_key); in bnxt_qplib_post_send()
1970 temp32 = cpu_to_le32(wqe->frmr.length); in bnxt_qplib_post_send()
1971 memcpy(sqe->length, &temp32, sizeof(wqe->frmr.length)); in bnxt_qplib_post_send()
1972 sqe->numlevels_pbl_page_size_log = in bnxt_qplib_post_send()
1973 ((wqe->frmr.pbl_pg_sz_log << in bnxt_qplib_post_send()
1976 ((wqe->frmr.levels << SQ_FR_PMR_NUMLEVELS_SFT) & in bnxt_qplib_post_send()
1979 for (i = 0; i < wqe->frmr.page_list_len; i++) in bnxt_qplib_post_send()
1980 wqe->frmr.pbl_ptr[i] = cpu_to_le64( in bnxt_qplib_post_send()
1981 wqe->frmr.page_list[i] | in bnxt_qplib_post_send()
1983 ext_sqe->pblptr = cpu_to_le64(wqe->frmr.pbl_dma_ptr); in bnxt_qplib_post_send()
1984 ext_sqe->va = cpu_to_le64(wqe->frmr.va); in bnxt_qplib_post_send()
1994 sqe->wqe_type = wqe->type; in bnxt_qplib_post_send()
1995 sqe->flags = wqe->flags; in bnxt_qplib_post_send()
1996 sqe->access_cntl = wqe->bind.access_cntl; in bnxt_qplib_post_send()
1997 sqe->mw_type_zero_based = wqe->bind.mw_type | in bnxt_qplib_post_send()
1998 (wqe->bind.zero_based ? SQ_BIND_ZERO_BASED : 0); in bnxt_qplib_post_send()
1999 sqe->parent_l_key = cpu_to_le32(wqe->bind.parent_l_key); in bnxt_qplib_post_send()
2000 sqe->l_key = cpu_to_le32(wqe->bind.r_key); in bnxt_qplib_post_send()
2001 ext_sqe->va = cpu_to_le64(wqe->bind.va); in bnxt_qplib_post_send()
2002 ext_sqe->length_lo = cpu_to_le32(wqe->bind.length); in bnxt_qplib_post_send()
2008 rc = -EINVAL; in bnxt_qplib_post_send()
2011 if (!BNXT_RE_HW_RETX(qp->dev_cap_flags) || msn_update) { in bnxt_qplib_post_send()
2012 swq->next_psn = sq->psn & BTH_PSN_MASK; in bnxt_qplib_post_send()
2017 bnxt_qplib_hwq_incr_prod(&sq->dbinfo, hwq, swq->slots); in bnxt_qplib_post_send()
2018 qp->wqe_cnt++; in bnxt_qplib_post_send()
2023 nq_work->cq = qp->scq; in bnxt_qplib_post_send()
2024 nq_work->nq = qp->scq->nq; in bnxt_qplib_post_send()
2025 INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task); in bnxt_qplib_post_send()
2026 queue_work(qp->scq->nq->cqn_wq, &nq_work->work); in bnxt_qplib_post_send()
2028 dev_err(&hwq->pdev->dev, in bnxt_qplib_post_send()
2030 rc = -ENOMEM; in bnxt_qplib_post_send()
2038 struct bnxt_qplib_q *rq = &qp->rq; in bnxt_qplib_post_recv_db()
2040 bnxt_qplib_ring_prod_db(&rq->dbinfo, DBC_DBC_TYPE_RQ); in bnxt_qplib_post_recv_db()
2047 struct bnxt_qplib_q *rq = &qp->rq; in bnxt_qplib_post_recv()
2057 hwq = &rq->hwq; in bnxt_qplib_post_recv()
2058 if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_RESET) { in bnxt_qplib_post_recv()
2059 dev_err(&hwq->pdev->dev, in bnxt_qplib_post_recv()
2061 qp->id, qp->state); in bnxt_qplib_post_recv()
2062 rc = -EINVAL; in bnxt_qplib_post_recv()
2066 if (bnxt_qplib_queue_full(rq, rq->dbinfo.max_slot)) { in bnxt_qplib_post_recv()
2067 dev_err(&hwq->pdev->dev, in bnxt_qplib_post_recv()
2068 "FP: QP (0x%x) RQ is full!\n", qp->id); in bnxt_qplib_post_recv()
2069 rc = -EINVAL; in bnxt_qplib_post_recv()
2074 swq->wr_id = wqe->wr_id; in bnxt_qplib_post_recv()
2075 swq->slots = rq->dbinfo.max_slot; in bnxt_qplib_post_recv()
2077 if (qp->state == CMDQ_MODIFY_QP_NEW_STATE_ERR) { in bnxt_qplib_post_recv()
2079 dev_dbg(&hwq->pdev->dev, in bnxt_qplib_post_recv()
2090 wqe->num_sge * sizeof(struct sq_sge)) >> 4; in bnxt_qplib_post_recv()
2091 bnxt_qplib_put_sges(hwq, wqe->sg_list, wqe->num_sge, &idx); in bnxt_qplib_post_recv()
2092 if (!wqe->num_sge) { in bnxt_qplib_post_recv()
2096 sge->size = 0; in bnxt_qplib_post_recv()
2099 base_hdr->wqe_type = wqe->type; in bnxt_qplib_post_recv()
2100 base_hdr->flags = wqe->flags; in bnxt_qplib_post_recv()
2101 base_hdr->wqe_size = wqe_sz; in bnxt_qplib_post_recv()
2102 base_hdr->wr_id[0] = cpu_to_le32(wqe_idx); in bnxt_qplib_post_recv()
2105 bnxt_qplib_hwq_incr_prod(&rq->dbinfo, hwq, swq->slots); in bnxt_qplib_post_recv()
2110 nq_work->cq = qp->rcq; in bnxt_qplib_post_recv()
2111 nq_work->nq = qp->rcq->nq; in bnxt_qplib_post_recv()
2112 INIT_WORK(&nq_work->work, bnxt_qpn_cqn_sched_task); in bnxt_qplib_post_recv()
2113 queue_work(qp->rcq->nq->cqn_wq, &nq_work->work); in bnxt_qplib_post_recv()
2115 dev_err(&hwq->pdev->dev, in bnxt_qplib_post_recv()
2117 rc = -ENOMEM; in bnxt_qplib_post_recv()
2127 struct bnxt_qplib_rcfw *rcfw = res->rcfw; in bnxt_qplib_create_cq()
2136 if (!cq->dpi) { in bnxt_qplib_create_cq()
2137 dev_err(&rcfw->pdev->dev, in bnxt_qplib_create_cq()
2139 return -EINVAL; in bnxt_qplib_create_cq()
2142 cq->dbinfo.flags = 0; in bnxt_qplib_create_cq()
2144 hwq_attr.depth = cq->max_wqe; in bnxt_qplib_create_cq()
2145 hwq_attr.stride = sizeof(struct cq_base); in bnxt_qplib_create_cq()
2147 hwq_attr.sginfo = &cq->sg_info; in bnxt_qplib_create_cq()
2148 rc = bnxt_qplib_alloc_init_hwq(&cq->hwq, &hwq_attr); in bnxt_qplib_create_cq()
2156 req.dpi = cpu_to_le32(cq->dpi->dpi); in bnxt_qplib_create_cq()
2157 req.cq_handle = cpu_to_le64(cq->cq_handle); in bnxt_qplib_create_cq()
2158 req.cq_size = cpu_to_le32(cq->max_wqe); in bnxt_qplib_create_cq()
2159 pbl = &cq->hwq.pbl[PBL_LVL_0]; in bnxt_qplib_create_cq()
2160 pg_sz_lvl = (bnxt_qplib_base_pg_size(&cq->hwq) << in bnxt_qplib_create_cq()
2162 pg_sz_lvl |= (cq->hwq.level & CMDQ_CREATE_CQ_LVL_MASK); in bnxt_qplib_create_cq()
2164 req.pbl = cpu_to_le64(pbl->pg_map_arr[0]); in bnxt_qplib_create_cq()
2166 (cq->cnq_hw_ring_id & CMDQ_CREATE_CQ_CNQ_ID_MASK) << in bnxt_qplib_create_cq()
2174 cq->id = le32_to_cpu(resp.xid); in bnxt_qplib_create_cq()
2175 cq->period = BNXT_QPLIB_QUEUE_START_PERIOD; in bnxt_qplib_create_cq()
2176 init_waitqueue_head(&cq->waitq); in bnxt_qplib_create_cq()
2177 INIT_LIST_HEAD(&cq->sqf_head); in bnxt_qplib_create_cq()
2178 INIT_LIST_HEAD(&cq->rqf_head); in bnxt_qplib_create_cq()
2179 spin_lock_init(&cq->compl_lock); in bnxt_qplib_create_cq()
2180 spin_lock_init(&cq->flush_lock); in bnxt_qplib_create_cq()
2182 cq->dbinfo.hwq = &cq->hwq; in bnxt_qplib_create_cq()
2183 cq->dbinfo.xid = cq->id; in bnxt_qplib_create_cq()
2184 cq->dbinfo.db = cq->dpi->dbr; in bnxt_qplib_create_cq()
2185 cq->dbinfo.priv_db = res->dpi_tbl.priv_db; in bnxt_qplib_create_cq()
2186 cq->dbinfo.flags = 0; in bnxt_qplib_create_cq()
2187 cq->dbinfo.toggle = 0; in bnxt_qplib_create_cq()
2189 bnxt_qplib_armen_db(&cq->dbinfo, DBC_DBC_TYPE_CQ_ARMENA); in bnxt_qplib_create_cq()
2194 bnxt_qplib_free_hwq(res, &cq->hwq); in bnxt_qplib_create_cq()
2201 bnxt_qplib_free_hwq(res, &cq->hwq); in bnxt_qplib_resize_cq_complete()
2202 memcpy(&cq->hwq, &cq->resize_hwq, sizeof(cq->hwq)); in bnxt_qplib_resize_cq_complete()
2204 cq->dbinfo.flags &= ~(1UL << BNXT_QPLIB_FLAG_EPOCH_CONS_SHIFT); in bnxt_qplib_resize_cq_complete()
2211 struct bnxt_qplib_rcfw *rcfw = res->rcfw; in bnxt_qplib_resize_cq()
2222 hwq_attr.sginfo = &cq->sg_info; in bnxt_qplib_resize_cq()
2225 hwq_attr.stride = sizeof(struct cq_base); in bnxt_qplib_resize_cq()
2227 rc = bnxt_qplib_alloc_init_hwq(&cq->resize_hwq, &hwq_attr); in bnxt_qplib_resize_cq()
2231 req.cq_cid = cpu_to_le32(cq->id); in bnxt_qplib_resize_cq()
2232 pbl = &cq->resize_hwq.pbl[PBL_LVL_0]; in bnxt_qplib_resize_cq()
2233 pg_sz = bnxt_qplib_base_pg_size(&cq->resize_hwq); in bnxt_qplib_resize_cq()
2234 lvl = (cq->resize_hwq.level << CMDQ_RESIZE_CQ_LVL_SFT) & in bnxt_qplib_resize_cq()
2239 req.new_pbl = cpu_to_le64(pbl->pg_map_arr[0]); in bnxt_qplib_resize_cq()
2249 struct bnxt_qplib_rcfw *rcfw = res->rcfw; in bnxt_qplib_destroy_cq()
2260 req.cq_cid = cpu_to_le32(cq->id); in bnxt_qplib_destroy_cq()
2268 bnxt_qplib_free_hwq(res, &cq->hwq); in bnxt_qplib_destroy_cq()
2280 start = sq->swq_start; in __flush_sq()
2283 last = sq->swq_last; in __flush_sq()
2287 if (sq->swq[last].wr_id == BNXT_QPLIB_FENCE_WRID) { in __flush_sq()
2292 cqe->status = CQ_REQ_STATUS_WORK_REQUEST_FLUSHED_ERR; in __flush_sq()
2293 cqe->opcode = CQ_BASE_CQE_TYPE_REQ; in __flush_sq()
2294 cqe->qp_handle = (u64)(unsigned long)qp; in __flush_sq()
2295 cqe->wr_id = sq->swq[last].wr_id; in __flush_sq()
2296 cqe->src_qp = qp->id; in __flush_sq()
2297 cqe->type = sq->swq[last].type; in __flush_sq()
2299 (*budget)--; in __flush_sq()
2301 bnxt_qplib_hwq_incr_cons(sq->hwq.max_elements, &sq->hwq.cons, in __flush_sq()
2302 sq->swq[last].slots, &sq->dbinfo.flags); in __flush_sq()
2303 sq->swq_last = sq->swq[last].next_idx; in __flush_sq()
2306 if (!(*budget) && sq->swq_last != start) in __flush_sq()
2308 rc = -EAGAIN; in __flush_sq()
2321 switch (qp->type) { in __flush_rq()
2335 start = rq->swq_start; in __flush_rq()
2338 last = rq->swq_last; in __flush_rq()
2342 cqe->status = in __flush_rq()
2344 cqe->opcode = opcode; in __flush_rq()
2345 cqe->qp_handle = (unsigned long)qp; in __flush_rq()
2346 cqe->wr_id = rq->swq[last].wr_id; in __flush_rq()
2348 (*budget)--; in __flush_rq()
2349 bnxt_qplib_hwq_incr_cons(rq->hwq.max_elements, &rq->hwq.cons, in __flush_rq()
2350 rq->swq[last].slots, &rq->dbinfo.flags); in __flush_rq()
2351 rq->swq_last = rq->swq[last].next_idx; in __flush_rq()
2354 if (!*budget && rq->swq_last != start) in __flush_rq()
2356 rc = -EAGAIN; in __flush_rq()
2369 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR; in bnxt_qplib_mark_qp_error()
2380 struct bnxt_qplib_q *sq = &qp->sq; in do_wa9060()
2390 swq = &sq->swq[swq_last]; in do_wa9060()
2391 if (swq->psn_search && in do_wa9060()
2392 le32_to_cpu(swq->psn_search->flags_next_psn) & 0x80000000) { in do_wa9060()
2394 swq->psn_search->flags_next_psn = cpu_to_le32 in do_wa9060()
2395 (le32_to_cpu(swq->psn_search->flags_next_psn) in do_wa9060()
2397 dev_dbg(&cq->hwq.pdev->dev, in do_wa9060()
2399 cq_cons, qp->id, swq_last, cqe_sq_cons); in do_wa9060()
2400 sq->condition = true; in do_wa9060()
2401 sq->send_phantom = true; in do_wa9060()
2404 bnxt_qplib_ring_db(&cq->dbinfo, DBC_DBC_TYPE_CQ_ARMALL); in do_wa9060()
2405 rc = -EAGAIN; in do_wa9060()
2408 if (sq->condition) { in do_wa9060()
2410 peek_flags = cq->dbinfo.flags; in do_wa9060()
2412 i = cq->hwq.max_elements; in do_wa9060()
2413 while (i--) { in do_wa9060()
2414 peek_hwcqe = bnxt_qplib_get_qe(&cq->hwq, in do_wa9060()
2424 if ((peek_hwcqe->cqe_type_toggle & in do_wa9060()
2432 (peek_req_hwcqe->qp_handle)); in do_wa9060()
2433 peek_sq = &peek_qp->sq; in do_wa9060()
2436 peek_req_hwcqe->sq_cons_idx) in do_wa9060()
2437 - 1) % sq->max_wqe); in do_wa9060()
2440 sq->swq[peek_sq_cons_idx].wr_id == in do_wa9060()
2446 dev_dbg(&cq->hwq.pdev->dev, in do_wa9060()
2448 sq->condition = false; in do_wa9060()
2449 sq->single = true; in do_wa9060()
2457 rc = -EINVAL; in do_wa9060()
2460 bnxt_qplib_hwq_incr_cons(cq->hwq.max_elements, in do_wa9060()
2464 dev_err(&cq->hwq.pdev->dev, in do_wa9060()
2466 cq_cons, qp->id, swq_last, cqe_sq_cons); in do_wa9060()
2467 rc = -EINVAL; in do_wa9060()
2486 le64_to_cpu(hwcqe->qp_handle)); in bnxt_qplib_cq_process_req()
2488 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_req()
2490 return -EINVAL; in bnxt_qplib_cq_process_req()
2492 sq = &qp->sq; in bnxt_qplib_cq_process_req()
2494 cqe_sq_cons = le16_to_cpu(hwcqe->sq_cons_idx) % sq->max_wqe; in bnxt_qplib_cq_process_req()
2495 if (qp->sq.flushed) { in bnxt_qplib_cq_process_req()
2496 dev_dbg(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_req()
2506 if (sq->swq_last == cqe_sq_cons) in bnxt_qplib_cq_process_req()
2510 swq = &sq->swq[sq->swq_last]; in bnxt_qplib_cq_process_req()
2512 cqe->opcode = CQ_BASE_CQE_TYPE_REQ; in bnxt_qplib_cq_process_req()
2513 cqe->qp_handle = (u64)(unsigned long)qp; in bnxt_qplib_cq_process_req()
2514 cqe->src_qp = qp->id; in bnxt_qplib_cq_process_req()
2515 cqe->wr_id = swq->wr_id; in bnxt_qplib_cq_process_req()
2516 if (cqe->wr_id == BNXT_QPLIB_FENCE_WRID) in bnxt_qplib_cq_process_req()
2518 cqe->type = swq->type; in bnxt_qplib_cq_process_req()
2524 if (swq->next_idx == cqe_sq_cons && in bnxt_qplib_cq_process_req()
2525 hwcqe->status != CQ_REQ_STATUS_OK) { in bnxt_qplib_cq_process_req()
2526 cqe->status = hwcqe->status; in bnxt_qplib_cq_process_req()
2527 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_req()
2529 sq->swq_last, cqe->wr_id, cqe->status); in bnxt_qplib_cq_process_req()
2531 (*budget)--; in bnxt_qplib_cq_process_req()
2537 if (do_wa9060(qp, cq, cq_cons, sq->swq_last, in bnxt_qplib_cq_process_req()
2542 if (swq->flags & SQ_SEND_FLAGS_SIGNAL_COMP) { in bnxt_qplib_cq_process_req()
2543 cqe->status = CQ_REQ_STATUS_OK; in bnxt_qplib_cq_process_req()
2545 (*budget)--; in bnxt_qplib_cq_process_req()
2549 bnxt_qplib_hwq_incr_cons(sq->hwq.max_elements, &sq->hwq.cons, in bnxt_qplib_cq_process_req()
2550 swq->slots, &sq->dbinfo.flags); in bnxt_qplib_cq_process_req()
2551 sq->swq_last = swq->next_idx; in bnxt_qplib_cq_process_req()
2552 if (sq->single) in bnxt_qplib_cq_process_req()
2557 if (sq->swq_last != cqe_sq_cons) { in bnxt_qplib_cq_process_req()
2559 rc = -EAGAIN; in bnxt_qplib_cq_process_req()
2566 sq->single = false; in bnxt_qplib_cq_process_req()
2573 spin_lock(&srq->hwq.lock); in bnxt_qplib_release_srqe()
2574 srq->swq[srq->last_idx].next_idx = (int)tag; in bnxt_qplib_release_srqe()
2575 srq->last_idx = (int)tag; in bnxt_qplib_release_srqe()
2576 srq->swq[srq->last_idx].next_idx = -1; in bnxt_qplib_release_srqe()
2577 bnxt_qplib_hwq_incr_cons(srq->hwq.max_elements, &srq->hwq.cons, in bnxt_qplib_release_srqe()
2578 srq->dbinfo.max_slot, &srq->dbinfo.flags); in bnxt_qplib_release_srqe()
2579 spin_unlock(&srq->hwq.lock); in bnxt_qplib_release_srqe()
2594 le64_to_cpu(hwcqe->qp_handle)); in bnxt_qplib_cq_process_res_rc()
2596 dev_err(&cq->hwq.pdev->dev, "process_cq RC qp is NULL\n"); in bnxt_qplib_cq_process_res_rc()
2597 return -EINVAL; in bnxt_qplib_cq_process_res_rc()
2599 if (qp->rq.flushed) { in bnxt_qplib_cq_process_res_rc()
2600 dev_dbg(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_res_rc()
2606 cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; in bnxt_qplib_cq_process_res_rc()
2607 cqe->length = le32_to_cpu(hwcqe->length); in bnxt_qplib_cq_process_res_rc()
2608 cqe->invrkey = le32_to_cpu(hwcqe->imm_data_or_inv_r_key); in bnxt_qplib_cq_process_res_rc()
2609 cqe->mr_handle = le64_to_cpu(hwcqe->mr_handle); in bnxt_qplib_cq_process_res_rc()
2610 cqe->flags = le16_to_cpu(hwcqe->flags); in bnxt_qplib_cq_process_res_rc()
2611 cqe->status = hwcqe->status; in bnxt_qplib_cq_process_res_rc()
2612 cqe->qp_handle = (u64)(unsigned long)qp; in bnxt_qplib_cq_process_res_rc()
2614 wr_id_idx = le32_to_cpu(hwcqe->srq_or_rq_wr_id) & in bnxt_qplib_cq_process_res_rc()
2616 if (cqe->flags & CQ_RES_RC_FLAGS_SRQ_SRQ) { in bnxt_qplib_cq_process_res_rc()
2617 srq = qp->srq; in bnxt_qplib_cq_process_res_rc()
2619 return -EINVAL; in bnxt_qplib_cq_process_res_rc()
2620 if (wr_id_idx >= srq->hwq.max_elements) { in bnxt_qplib_cq_process_res_rc()
2621 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_res_rc()
2623 wr_id_idx, srq->hwq.max_elements); in bnxt_qplib_cq_process_res_rc()
2624 return -EINVAL; in bnxt_qplib_cq_process_res_rc()
2626 cqe->wr_id = srq->swq[wr_id_idx].wr_id; in bnxt_qplib_cq_process_res_rc()
2629 (*budget)--; in bnxt_qplib_cq_process_res_rc()
2634 rq = &qp->rq; in bnxt_qplib_cq_process_res_rc()
2635 if (wr_id_idx > (rq->max_wqe - 1)) { in bnxt_qplib_cq_process_res_rc()
2636 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_res_rc()
2638 wr_id_idx, rq->max_wqe); in bnxt_qplib_cq_process_res_rc()
2639 return -EINVAL; in bnxt_qplib_cq_process_res_rc()
2641 if (wr_id_idx != rq->swq_last) in bnxt_qplib_cq_process_res_rc()
2642 return -EINVAL; in bnxt_qplib_cq_process_res_rc()
2643 swq = &rq->swq[rq->swq_last]; in bnxt_qplib_cq_process_res_rc()
2644 cqe->wr_id = swq->wr_id; in bnxt_qplib_cq_process_res_rc()
2646 (*budget)--; in bnxt_qplib_cq_process_res_rc()
2647 bnxt_qplib_hwq_incr_cons(rq->hwq.max_elements, &rq->hwq.cons, in bnxt_qplib_cq_process_res_rc()
2648 swq->slots, &rq->dbinfo.flags); in bnxt_qplib_cq_process_res_rc()
2649 rq->swq_last = swq->next_idx; in bnxt_qplib_cq_process_res_rc()
2652 if (hwcqe->status != CQ_RES_RC_STATUS_OK) { in bnxt_qplib_cq_process_res_rc()
2653 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR; in bnxt_qplib_cq_process_res_rc()
2674 le64_to_cpu(hwcqe->qp_handle)); in bnxt_qplib_cq_process_res_ud()
2676 dev_err(&cq->hwq.pdev->dev, "process_cq UD qp is NULL\n"); in bnxt_qplib_cq_process_res_ud()
2677 return -EINVAL; in bnxt_qplib_cq_process_res_ud()
2679 if (qp->rq.flushed) { in bnxt_qplib_cq_process_res_ud()
2680 dev_dbg(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_res_ud()
2685 cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; in bnxt_qplib_cq_process_res_ud()
2686 cqe->length = le16_to_cpu(hwcqe->length) & CQ_RES_UD_LENGTH_MASK; in bnxt_qplib_cq_process_res_ud()
2687 cqe->cfa_meta = le16_to_cpu(hwcqe->cfa_metadata); in bnxt_qplib_cq_process_res_ud()
2688 cqe->invrkey = le32_to_cpu(hwcqe->imm_data); in bnxt_qplib_cq_process_res_ud()
2689 cqe->flags = le16_to_cpu(hwcqe->flags); in bnxt_qplib_cq_process_res_ud()
2690 cqe->status = hwcqe->status; in bnxt_qplib_cq_process_res_ud()
2691 cqe->qp_handle = (u64)(unsigned long)qp; in bnxt_qplib_cq_process_res_ud()
2693 memcpy(cqe->smac, hwcqe->src_mac, ETH_ALEN); in bnxt_qplib_cq_process_res_ud()
2694 wr_id_idx = le32_to_cpu(hwcqe->src_qp_high_srq_or_rq_wr_id) in bnxt_qplib_cq_process_res_ud()
2696 cqe->src_qp = le16_to_cpu(hwcqe->src_qp_low) | in bnxt_qplib_cq_process_res_ud()
2698 hwcqe->src_qp_high_srq_or_rq_wr_id) & in bnxt_qplib_cq_process_res_ud()
2701 if (cqe->flags & CQ_RES_RC_FLAGS_SRQ_SRQ) { in bnxt_qplib_cq_process_res_ud()
2702 srq = qp->srq; in bnxt_qplib_cq_process_res_ud()
2704 return -EINVAL; in bnxt_qplib_cq_process_res_ud()
2706 if (wr_id_idx >= srq->hwq.max_elements) { in bnxt_qplib_cq_process_res_ud()
2707 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_res_ud()
2709 wr_id_idx, srq->hwq.max_elements); in bnxt_qplib_cq_process_res_ud()
2710 return -EINVAL; in bnxt_qplib_cq_process_res_ud()
2712 cqe->wr_id = srq->swq[wr_id_idx].wr_id; in bnxt_qplib_cq_process_res_ud()
2715 (*budget)--; in bnxt_qplib_cq_process_res_ud()
2720 rq = &qp->rq; in bnxt_qplib_cq_process_res_ud()
2721 if (wr_id_idx > (rq->max_wqe - 1)) { in bnxt_qplib_cq_process_res_ud()
2722 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_res_ud()
2724 wr_id_idx, rq->max_wqe); in bnxt_qplib_cq_process_res_ud()
2725 return -EINVAL; in bnxt_qplib_cq_process_res_ud()
2728 if (rq->swq_last != wr_id_idx) in bnxt_qplib_cq_process_res_ud()
2729 return -EINVAL; in bnxt_qplib_cq_process_res_ud()
2730 swq = &rq->swq[rq->swq_last]; in bnxt_qplib_cq_process_res_ud()
2731 cqe->wr_id = swq->wr_id; in bnxt_qplib_cq_process_res_ud()
2733 (*budget)--; in bnxt_qplib_cq_process_res_ud()
2734 bnxt_qplib_hwq_incr_cons(rq->hwq.max_elements, &rq->hwq.cons, in bnxt_qplib_cq_process_res_ud()
2735 swq->slots, &rq->dbinfo.flags); in bnxt_qplib_cq_process_res_ud()
2736 rq->swq_last = swq->next_idx; in bnxt_qplib_cq_process_res_ud()
2739 if (hwcqe->status != CQ_RES_RC_STATUS_OK) { in bnxt_qplib_cq_process_res_ud()
2740 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR; in bnxt_qplib_cq_process_res_ud()
2754 hw_cqe = bnxt_qplib_get_qe(&cq->hwq, cq->hwq.cons, NULL); in bnxt_qplib_is_cq_empty()
2756 rc = !CQE_CMP_VALID(hw_cqe, cq->dbinfo.flags); in bnxt_qplib_is_cq_empty()
2772 le64_to_cpu(hwcqe->qp_handle)); in bnxt_qplib_cq_process_res_raweth_qp1()
2774 dev_err(&cq->hwq.pdev->dev, "process_cq Raw/QP1 qp is NULL\n"); in bnxt_qplib_cq_process_res_raweth_qp1()
2775 return -EINVAL; in bnxt_qplib_cq_process_res_raweth_qp1()
2777 if (qp->rq.flushed) { in bnxt_qplib_cq_process_res_raweth_qp1()
2778 dev_dbg(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_res_raweth_qp1()
2783 cqe->opcode = hwcqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; in bnxt_qplib_cq_process_res_raweth_qp1()
2784 cqe->flags = le16_to_cpu(hwcqe->flags); in bnxt_qplib_cq_process_res_raweth_qp1()
2785 cqe->qp_handle = (u64)(unsigned long)qp; in bnxt_qplib_cq_process_res_raweth_qp1()
2788 le32_to_cpu(hwcqe->raweth_qp1_payload_offset_srq_or_rq_wr_id) in bnxt_qplib_cq_process_res_raweth_qp1()
2790 cqe->src_qp = qp->id; in bnxt_qplib_cq_process_res_raweth_qp1()
2791 if (qp->id == 1 && !cqe->length) { in bnxt_qplib_cq_process_res_raweth_qp1()
2793 cqe->length = 296; in bnxt_qplib_cq_process_res_raweth_qp1()
2795 cqe->length = le16_to_cpu(hwcqe->length); in bnxt_qplib_cq_process_res_raweth_qp1()
2797 cqe->pkey_index = qp->pkey_index; in bnxt_qplib_cq_process_res_raweth_qp1()
2798 memcpy(cqe->smac, qp->smac, 6); in bnxt_qplib_cq_process_res_raweth_qp1()
2800 cqe->raweth_qp1_flags = le16_to_cpu(hwcqe->raweth_qp1_flags); in bnxt_qplib_cq_process_res_raweth_qp1()
2801 cqe->raweth_qp1_flags2 = le32_to_cpu(hwcqe->raweth_qp1_flags2); in bnxt_qplib_cq_process_res_raweth_qp1()
2802 cqe->raweth_qp1_metadata = le32_to_cpu(hwcqe->raweth_qp1_metadata); in bnxt_qplib_cq_process_res_raweth_qp1()
2804 if (cqe->flags & CQ_RES_RAWETH_QP1_FLAGS_SRQ_SRQ) { in bnxt_qplib_cq_process_res_raweth_qp1()
2805 srq = qp->srq; in bnxt_qplib_cq_process_res_raweth_qp1()
2807 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_res_raweth_qp1()
2809 return -EINVAL; in bnxt_qplib_cq_process_res_raweth_qp1()
2811 if (wr_id_idx >= srq->hwq.max_elements) { in bnxt_qplib_cq_process_res_raweth_qp1()
2812 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_res_raweth_qp1()
2814 wr_id_idx, srq->hwq.max_elements); in bnxt_qplib_cq_process_res_raweth_qp1()
2815 return -EINVAL; in bnxt_qplib_cq_process_res_raweth_qp1()
2817 cqe->wr_id = srq->swq[wr_id_idx].wr_id; in bnxt_qplib_cq_process_res_raweth_qp1()
2820 (*budget)--; in bnxt_qplib_cq_process_res_raweth_qp1()
2825 rq = &qp->rq; in bnxt_qplib_cq_process_res_raweth_qp1()
2826 if (wr_id_idx > (rq->max_wqe - 1)) { in bnxt_qplib_cq_process_res_raweth_qp1()
2827 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_res_raweth_qp1()
2829 wr_id_idx, rq->max_wqe); in bnxt_qplib_cq_process_res_raweth_qp1()
2830 return -EINVAL; in bnxt_qplib_cq_process_res_raweth_qp1()
2832 if (rq->swq_last != wr_id_idx) in bnxt_qplib_cq_process_res_raweth_qp1()
2833 return -EINVAL; in bnxt_qplib_cq_process_res_raweth_qp1()
2834 swq = &rq->swq[rq->swq_last]; in bnxt_qplib_cq_process_res_raweth_qp1()
2835 cqe->wr_id = swq->wr_id; in bnxt_qplib_cq_process_res_raweth_qp1()
2837 (*budget)--; in bnxt_qplib_cq_process_res_raweth_qp1()
2838 bnxt_qplib_hwq_incr_cons(rq->hwq.max_elements, &rq->hwq.cons, in bnxt_qplib_cq_process_res_raweth_qp1()
2839 swq->slots, &rq->dbinfo.flags); in bnxt_qplib_cq_process_res_raweth_qp1()
2840 rq->swq_last = swq->next_idx; in bnxt_qplib_cq_process_res_raweth_qp1()
2843 if (hwcqe->status != CQ_RES_RC_STATUS_OK) { in bnxt_qplib_cq_process_res_raweth_qp1()
2844 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR; in bnxt_qplib_cq_process_res_raweth_qp1()
2865 if (hwcqe->status != CQ_TERMINAL_STATUS_OK) in bnxt_qplib_cq_process_terminal()
2866 dev_warn(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_terminal()
2868 hwcqe->status); in bnxt_qplib_cq_process_terminal()
2871 le64_to_cpu(hwcqe->qp_handle)); in bnxt_qplib_cq_process_terminal()
2873 return -EINVAL; in bnxt_qplib_cq_process_terminal()
2876 qp->state = CMDQ_MODIFY_QP_NEW_STATE_ERR; in bnxt_qplib_cq_process_terminal()
2878 sq = &qp->sq; in bnxt_qplib_cq_process_terminal()
2879 rq = &qp->rq; in bnxt_qplib_cq_process_terminal()
2881 cqe_cons = le16_to_cpu(hwcqe->sq_cons_idx); in bnxt_qplib_cq_process_terminal()
2884 cqe_cons %= sq->max_wqe; in bnxt_qplib_cq_process_terminal()
2886 if (qp->sq.flushed) { in bnxt_qplib_cq_process_terminal()
2887 dev_dbg(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_terminal()
2898 swq_last = sq->swq_last; in bnxt_qplib_cq_process_terminal()
2901 if (sq->swq[swq_last].flags & SQ_SEND_FLAGS_SIGNAL_COMP) { in bnxt_qplib_cq_process_terminal()
2903 cqe->status = CQ_REQ_STATUS_OK; in bnxt_qplib_cq_process_terminal()
2904 cqe->opcode = CQ_BASE_CQE_TYPE_REQ; in bnxt_qplib_cq_process_terminal()
2905 cqe->qp_handle = (u64)(unsigned long)qp; in bnxt_qplib_cq_process_terminal()
2906 cqe->src_qp = qp->id; in bnxt_qplib_cq_process_terminal()
2907 cqe->wr_id = sq->swq[swq_last].wr_id; in bnxt_qplib_cq_process_terminal()
2908 cqe->type = sq->swq[swq_last].type; in bnxt_qplib_cq_process_terminal()
2910 (*budget)--; in bnxt_qplib_cq_process_terminal()
2912 bnxt_qplib_hwq_incr_cons(sq->hwq.max_elements, &sq->hwq.cons, in bnxt_qplib_cq_process_terminal()
2913 sq->swq[swq_last].slots, &sq->dbinfo.flags); in bnxt_qplib_cq_process_terminal()
2914 sq->swq_last = sq->swq[swq_last].next_idx; in bnxt_qplib_cq_process_terminal()
2919 rc = -EAGAIN; in bnxt_qplib_cq_process_terminal()
2926 cqe_cons = le16_to_cpu(hwcqe->rq_cons_idx); in bnxt_qplib_cq_process_terminal()
2929 } else if (cqe_cons > rq->max_wqe - 1) { in bnxt_qplib_cq_process_terminal()
2930 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_terminal()
2932 cqe_cons, rq->max_wqe); in bnxt_qplib_cq_process_terminal()
2933 rc = -EINVAL; in bnxt_qplib_cq_process_terminal()
2937 if (qp->rq.flushed) { in bnxt_qplib_cq_process_terminal()
2938 dev_dbg(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_terminal()
2945 * from the current rq->cons to the rq->prod regardless what the in bnxt_qplib_cq_process_terminal()
2946 * rq->cons the terminal CQE indicates in bnxt_qplib_cq_process_terminal()
2959 if (hwcqe->status != CQ_CUTOFF_STATUS_OK) { in bnxt_qplib_cq_process_cutoff()
2960 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_cq_process_cutoff()
2962 hwcqe->status); in bnxt_qplib_cq_process_cutoff()
2963 return -EINVAL; in bnxt_qplib_cq_process_cutoff()
2965 clear_bit(CQ_FLAGS_RESIZE_IN_PROG, &cq->flags); in bnxt_qplib_cq_process_cutoff()
2966 wake_up_interruptible(&cq->waitq); in bnxt_qplib_cq_process_cutoff()
2979 spin_lock_irqsave(&cq->flush_lock, flags); in bnxt_qplib_process_flush_list()
2980 list_for_each_entry(qp, &cq->sqf_head, sq_flush) { in bnxt_qplib_process_flush_list()
2981 dev_dbg(&cq->hwq.pdev->dev, "FP: Flushing SQ QP= %p\n", qp); in bnxt_qplib_process_flush_list()
2982 __flush_sq(&qp->sq, qp, &cqe, &budget); in bnxt_qplib_process_flush_list()
2985 list_for_each_entry(qp, &cq->rqf_head, rq_flush) { in bnxt_qplib_process_flush_list()
2986 dev_dbg(&cq->hwq.pdev->dev, "FP: Flushing RQ QP= %p\n", qp); in bnxt_qplib_process_flush_list()
2987 __flush_rq(&qp->rq, qp, &cqe, &budget); in bnxt_qplib_process_flush_list()
2989 spin_unlock_irqrestore(&cq->flush_lock, flags); in bnxt_qplib_process_flush_list()
2991 return num_cqes - budget; in bnxt_qplib_process_flush_list()
3005 hw_cqe = bnxt_qplib_get_qe(&cq->hwq, cq->hwq.cons, NULL); in bnxt_qplib_poll_cq()
3008 if (!CQE_CMP_VALID(hw_cqe, cq->dbinfo.flags)) in bnxt_qplib_poll_cq()
3017 type = hw_cqe->cqe_type_toggle & CQ_BASE_CQE_TYPE_MASK; in bnxt_qplib_poll_cq()
3023 cq->hwq.cons, lib_qp); in bnxt_qplib_poll_cq()
3052 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_poll_cq()
3054 hw_cqe->cqe_type_toggle & in bnxt_qplib_poll_cq()
3056 rc = -EINVAL; in bnxt_qplib_poll_cq()
3060 if (rc == -EAGAIN) in bnxt_qplib_poll_cq()
3066 dev_err(&cq->hwq.pdev->dev, in bnxt_qplib_poll_cq()
3070 bnxt_qplib_hwq_incr_cons(cq->hwq.max_elements, &cq->hwq.cons, in bnxt_qplib_poll_cq()
3071 1, &cq->dbinfo.flags); in bnxt_qplib_poll_cq()
3075 bnxt_qplib_ring_db(&cq->dbinfo, DBC_DBC_TYPE_CQ); in bnxt_qplib_poll_cq()
3077 return num_cqes - budget; in bnxt_qplib_poll_cq()
3082 cq->dbinfo.toggle = cq->toggle; in bnxt_qplib_req_notify_cq()
3084 bnxt_qplib_ring_db(&cq->dbinfo, arm_type); in bnxt_qplib_req_notify_cq()
3085 /* Using cq->arm_state variable to track whether to issue cq handler */ in bnxt_qplib_req_notify_cq()
3086 atomic_set(&cq->arm_state, 1); in bnxt_qplib_req_notify_cq()
3091 flush_workqueue(qp->scq->nq->cqn_wq); in bnxt_qplib_flush_cqn_wq()
3092 if (qp->scq != qp->rcq) in bnxt_qplib_flush_cqn_wq()
3093 flush_workqueue(qp->rcq->nq->cqn_wq); in bnxt_qplib_flush_cqn_wq()