Lines Matching +full:post +full:-
2 * Copyright (c) 2015-2016 QLogic Corporation
14 * - Redistributions of source code must retain the above
18 * - Redistributions in binary form must reproduce the above
32 #include <linux/dma-mapping.h>
50 #include <rdma/qedr-abi.h>
55 info->gsi_cons = (info->gsi_cons + 1) % info->max_wr; in qedr_inc_sw_gsi_cons()
61 dev->gsi_qp_created = 1; in qedr_store_gsi_qp_cq()
62 dev->gsi_sqcq = get_qedr_cq(attrs->send_cq); in qedr_store_gsi_qp_cq()
63 dev->gsi_rqcq = get_qedr_cq(attrs->recv_cq); in qedr_store_gsi_qp_cq()
64 dev->gsi_qp = qp; in qedr_store_gsi_qp_cq()
75 struct qedr_cq *cq = dev->gsi_sqcq; in qedr_ll2_complete_tx_packet()
76 struct qedr_qp *qp = dev->gsi_qp; in qedr_ll2_complete_tx_packet()
81 dev->gsi_sqcq, dev->gsi_rqcq, qp->sq.gsi_cons, in qedr_ll2_complete_tx_packet()
82 cq->ibcq.comp_handler ? "Yes" : "No"); in qedr_ll2_complete_tx_packet()
84 dma_free_coherent(&dev->pdev->dev, pkt->header.len, pkt->header.vaddr, in qedr_ll2_complete_tx_packet()
85 pkt->header.baddr); in qedr_ll2_complete_tx_packet()
88 spin_lock_irqsave(&qp->q_lock, flags); in qedr_ll2_complete_tx_packet()
89 qedr_inc_sw_gsi_cons(&qp->sq); in qedr_ll2_complete_tx_packet()
90 spin_unlock_irqrestore(&qp->q_lock, flags); in qedr_ll2_complete_tx_packet()
92 if (cq->ibcq.comp_handler) in qedr_ll2_complete_tx_packet()
93 (*cq->ibcq.comp_handler) (&cq->ibcq, cq->ibcq.cq_context); in qedr_ll2_complete_tx_packet()
100 struct qedr_cq *cq = dev->gsi_rqcq; in qedr_ll2_complete_rx_packet()
101 struct qedr_qp *qp = dev->gsi_qp; in qedr_ll2_complete_rx_packet()
104 spin_lock_irqsave(&qp->q_lock, flags); in qedr_ll2_complete_rx_packet()
106 qp->rqe_wr_id[qp->rq.gsi_cons].rc = data->u.data_length_error ? in qedr_ll2_complete_rx_packet()
107 -EINVAL : 0; in qedr_ll2_complete_rx_packet()
108 qp->rqe_wr_id[qp->rq.gsi_cons].vlan = data->vlan; in qedr_ll2_complete_rx_packet()
110 qp->rqe_wr_id[qp->rq.gsi_cons].sg_list[0].length = in qedr_ll2_complete_rx_packet()
111 data->length.data_length; in qedr_ll2_complete_rx_packet()
112 *((u32 *)&qp->rqe_wr_id[qp->rq.gsi_cons].smac[0]) = in qedr_ll2_complete_rx_packet()
113 ntohl(data->opaque_data_0); in qedr_ll2_complete_rx_packet()
114 *((u16 *)&qp->rqe_wr_id[qp->rq.gsi_cons].smac[4]) = in qedr_ll2_complete_rx_packet()
115 ntohs((u16)data->opaque_data_1); in qedr_ll2_complete_rx_packet()
117 qedr_inc_sw_gsi_cons(&qp->rq); in qedr_ll2_complete_rx_packet()
119 spin_unlock_irqrestore(&qp->q_lock, flags); in qedr_ll2_complete_rx_packet()
121 if (cq->ibcq.comp_handler) in qedr_ll2_complete_rx_packet()
122 (*cq->ibcq.comp_handler) (&cq->ibcq, cq->ibcq.cq_context); in qedr_ll2_complete_rx_packet()
139 cq = get_qedr_cq(attrs->send_cq); in qedr_destroy_gsi_cq()
140 iparams.icid = cq->icid; in qedr_destroy_gsi_cq()
141 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); in qedr_destroy_gsi_cq()
142 dev->ops->common->chain_free(dev->cdev, &cq->pbl); in qedr_destroy_gsi_cq()
144 cq = get_qedr_cq(attrs->recv_cq); in qedr_destroy_gsi_cq()
146 if (iparams.icid != cq->icid) { in qedr_destroy_gsi_cq()
147 iparams.icid = cq->icid; in qedr_destroy_gsi_cq()
148 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); in qedr_destroy_gsi_cq()
149 dev->ops->common->chain_free(dev->cdev, &cq->pbl); in qedr_destroy_gsi_cq()
156 if (attrs->cap.max_recv_sge > QEDR_GSI_MAX_RECV_SGE) { in qedr_check_gsi_qp_attrs()
159 attrs->cap.max_recv_sge, QEDR_GSI_MAX_RECV_SGE); in qedr_check_gsi_qp_attrs()
160 return -EINVAL; in qedr_check_gsi_qp_attrs()
163 if (attrs->cap.max_recv_wr > QEDR_GSI_MAX_RECV_WR) { in qedr_check_gsi_qp_attrs()
166 attrs->cap.max_recv_wr, QEDR_GSI_MAX_RECV_WR); in qedr_check_gsi_qp_attrs()
167 return -EINVAL; in qedr_check_gsi_qp_attrs()
170 if (attrs->cap.max_send_wr > QEDR_GSI_MAX_SEND_WR) { in qedr_check_gsi_qp_attrs()
173 attrs->cap.max_send_wr, QEDR_GSI_MAX_SEND_WR); in qedr_check_gsi_qp_attrs()
174 return -EINVAL; in qedr_check_gsi_qp_attrs()
190 roce_flavor = (pkt->roce_mode == ROCE_V1) ? in qedr_ll2_post_tx()
193 if (pkt->roce_mode == ROCE_V2_IPV4) in qedr_ll2_post_tx()
196 ll2_tx_pkt.num_of_bds = 1 /* hdr */ + pkt->n_seg; in qedr_ll2_post_tx()
198 ll2_tx_pkt.tx_dest = pkt->tx_dest; in qedr_ll2_post_tx()
200 ll2_tx_pkt.first_frag = pkt->header.baddr; in qedr_ll2_post_tx()
201 ll2_tx_pkt.first_frag_len = pkt->header.len; in qedr_ll2_post_tx()
205 rc = dev->ops->ll2_prepare_tx_packet(dev->rdma_ctx, in qedr_ll2_post_tx()
206 dev->gsi_ll2_handle, in qedr_ll2_post_tx()
209 /* TX failed while posting header - release resources */ in qedr_ll2_post_tx()
210 dma_free_coherent(&dev->pdev->dev, pkt->header.len, in qedr_ll2_post_tx()
211 pkt->header.vaddr, pkt->header.baddr); in qedr_ll2_post_tx()
219 for (i = 0; i < pkt->n_seg; i++) { in qedr_ll2_post_tx()
220 rc = dev->ops->ll2_set_fragment_of_tx_packet( in qedr_ll2_post_tx()
221 dev->rdma_ctx, in qedr_ll2_post_tx()
222 dev->gsi_ll2_handle, in qedr_ll2_post_tx()
223 pkt->payload[i].baddr, in qedr_ll2_post_tx()
224 pkt->payload[i].len); in qedr_ll2_post_tx()
243 if (dev->gsi_ll2_handle == QED_LL2_UNUSED_HANDLE) in qedr_ll2_stop()
247 rc = dev->ops->ll2_set_mac_filter(dev->cdev, in qedr_ll2_stop()
248 dev->gsi_ll2_mac_address, NULL); in qedr_ll2_stop()
250 rc = dev->ops->ll2_terminate_connection(dev->rdma_ctx, in qedr_ll2_stop()
251 dev->gsi_ll2_handle); in qedr_ll2_stop()
255 dev->ops->ll2_release_connection(dev->rdma_ctx, dev->gsi_ll2_handle); in qedr_ll2_stop()
257 dev->gsi_ll2_handle = QED_LL2_UNUSED_HANDLE; in qedr_ll2_stop()
278 data.input.mtu = dev->ndev->mtu; in qedr_ll2_start()
279 data.input.rx_num_desc = attrs->cap.max_recv_wr; in qedr_ll2_start()
282 data.input.tx_num_desc = attrs->cap.max_send_wr; in qedr_ll2_start()
288 data.p_connection_handle = &dev->gsi_ll2_handle; in qedr_ll2_start()
291 rc = dev->ops->ll2_acquire_connection(dev->rdma_ctx, &data); in qedr_ll2_start()
299 rc = dev->ops->ll2_establish_connection(dev->rdma_ctx, in qedr_ll2_start()
300 dev->gsi_ll2_handle); in qedr_ll2_start()
308 rc = dev->ops->ll2_set_mac_filter(dev->cdev, NULL, dev->ndev->dev_addr); in qedr_ll2_start()
315 dev->ops->ll2_terminate_connection(dev->rdma_ctx, dev->gsi_ll2_handle); in qedr_ll2_start()
317 dev->ops->ll2_release_connection(dev->rdma_ctx, dev->gsi_ll2_handle); in qedr_ll2_start()
339 qp->ibqp.qp_num = 1; in qedr_create_gsi_qp()
340 qp->rq.max_wr = attrs->cap.max_recv_wr; in qedr_create_gsi_qp()
341 qp->sq.max_wr = attrs->cap.max_send_wr; in qedr_create_gsi_qp()
343 qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id), in qedr_create_gsi_qp()
345 if (!qp->rqe_wr_id) in qedr_create_gsi_qp()
347 qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id), in qedr_create_gsi_qp()
349 if (!qp->wqe_wr_id) in qedr_create_gsi_qp()
353 ether_addr_copy(dev->gsi_ll2_mac_address, dev->ndev->dev_addr); in qedr_create_gsi_qp()
357 dev->gsi_rqcq->cq_type = QEDR_CQ_TYPE_GSI; in qedr_create_gsi_qp()
358 dev->gsi_rqcq->cq_type = QEDR_CQ_TYPE_GSI; in qedr_create_gsi_qp()
362 return &qp->ibqp; in qedr_create_gsi_qp()
365 kfree(qp->rqe_wr_id); in qedr_create_gsi_qp()
371 return ERR_PTR(-ENOMEM); in qedr_create_gsi_qp()
388 struct rdma_ah_attr *ah_attr = &get_qedr_ah(ud_wr(swr)->ah)->attr; in qedr_gsi_build_header()
390 const struct ib_gid_attr *sgid_attr = grh->sgid_attr; in qedr_gsi_build_header()
408 for (i = 0; i < swr->num_sge; ++i) in qedr_gsi_build_header()
409 send_size += swr->sg_list[i].length; in qedr_gsi_build_header()
411 has_udp = (sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP); in qedr_gsi_build_header()
416 } else if (ipv6_addr_v4mapped((struct in6_addr *)&sgid_attr->gid)) { in qedr_gsi_build_header()
432 DP_ERR(dev, "gsi post send: failed to init header\n"); in qedr_gsi_build_header()
437 ether_addr_copy(udh->eth.dmac_h, ah_attr->roce.dmac); in qedr_gsi_build_header()
438 ether_addr_copy(udh->eth.smac_h, dev->ndev->dev_addr); in qedr_gsi_build_header()
440 udh->eth.type = htons(ETH_P_8021Q); in qedr_gsi_build_header()
441 udh->vlan.tag = htons(vlan_id); in qedr_gsi_build_header()
442 udh->vlan.type = htons(ether_type); in qedr_gsi_build_header()
444 udh->eth.type = htons(ether_type); in qedr_gsi_build_header()
448 udh->bth.solicited_event = !!(swr->send_flags & IB_SEND_SOLICITED); in qedr_gsi_build_header()
449 udh->bth.pkey = QEDR_ROCE_PKEY_DEFAULT; in qedr_gsi_build_header()
450 udh->bth.destination_qpn = htonl(ud_wr(swr)->remote_qpn); in qedr_gsi_build_header()
451 udh->bth.psn = htonl((qp->sq_psn++) & ((1 << 24) - 1)); in qedr_gsi_build_header()
452 udh->bth.opcode = IB_OPCODE_UD_SEND_ONLY; in qedr_gsi_build_header()
455 udh->deth.qkey = htonl(0x80010000); in qedr_gsi_build_header()
456 udh->deth.source_qpn = htonl(QEDR_GSI_QPN); in qedr_gsi_build_header()
460 udh->grh.traffic_class = grh->traffic_class; in qedr_gsi_build_header()
461 udh->grh.flow_label = grh->flow_label; in qedr_gsi_build_header()
462 udh->grh.hop_limit = grh->hop_limit; in qedr_gsi_build_header()
463 udh->grh.destination_gid = grh->dgid; in qedr_gsi_build_header()
464 memcpy(&udh->grh.source_gid.raw, sgid_attr->gid.raw, in qedr_gsi_build_header()
465 sizeof(udh->grh.source_gid.raw)); in qedr_gsi_build_header()
470 udh->ip4.protocol = IPPROTO_UDP; in qedr_gsi_build_header()
471 udh->ip4.tos = htonl(grh->flow_label); in qedr_gsi_build_header()
472 udh->ip4.frag_off = htons(IP_DF); in qedr_gsi_build_header()
473 udh->ip4.ttl = grh->hop_limit; in qedr_gsi_build_header()
475 ipv4_addr = qedr_get_ipv4_from_gid(sgid_attr->gid.raw); in qedr_gsi_build_header()
476 udh->ip4.saddr = ipv4_addr; in qedr_gsi_build_header()
477 ipv4_addr = qedr_get_ipv4_from_gid(grh->dgid.raw); in qedr_gsi_build_header()
478 udh->ip4.daddr = ipv4_addr; in qedr_gsi_build_header()
484 udh->udp.sport = htons(QEDR_ROCE_V2_UDP_SPORT); in qedr_gsi_build_header()
485 udh->udp.dport = htons(ROCE_V2_UDP_DPORT); in qedr_gsi_build_header()
486 udh->udp.csum = 0; in qedr_gsi_build_header()
499 struct pci_dev *pdev = dev->pdev; in qedr_gsi_build_packet()
514 return -ENOMEM; in qedr_gsi_build_packet()
516 packet->header.vaddr = dma_alloc_coherent(&pdev->dev, header_size, in qedr_gsi_build_packet()
517 &packet->header.baddr, in qedr_gsi_build_packet()
519 if (!packet->header.vaddr) { in qedr_gsi_build_packet()
521 return -ENOMEM; in qedr_gsi_build_packet()
525 packet->tx_dest = QED_LL2_TX_DEST_LB; in qedr_gsi_build_packet()
527 packet->tx_dest = QED_LL2_TX_DEST_NW; in qedr_gsi_build_packet()
529 packet->roce_mode = roce_mode; in qedr_gsi_build_packet()
530 memcpy(packet->header.vaddr, ud_header_buffer, header_size); in qedr_gsi_build_packet()
531 packet->header.len = header_size; in qedr_gsi_build_packet()
532 packet->n_seg = swr->num_sge; in qedr_gsi_build_packet()
533 for (i = 0; i < packet->n_seg; i++) { in qedr_gsi_build_packet()
534 packet->payload[i].baddr = swr->sg_list[i].addr; in qedr_gsi_build_packet()
535 packet->payload[i].len = swr->sg_list[i].length; in qedr_gsi_build_packet()
548 struct qedr_dev *dev = qp->dev; in qedr_gsi_post_send()
552 if (qp->state != QED_ROCE_QP_STATE_RTS) { in qedr_gsi_post_send()
555 "gsi post recv: failed to post rx buffer. state is %d and not QED_ROCE_QP_STATE_RTS\n", in qedr_gsi_post_send()
556 qp->state); in qedr_gsi_post_send()
557 return -EINVAL; in qedr_gsi_post_send()
560 if (wr->num_sge > RDMA_MAX_SGE_PER_SQ_WQE) { in qedr_gsi_post_send()
561 DP_ERR(dev, "gsi post send: num_sge is too large (%d>%d)\n", in qedr_gsi_post_send()
562 wr->num_sge, RDMA_MAX_SGE_PER_SQ_WQE); in qedr_gsi_post_send()
563 rc = -EINVAL; in qedr_gsi_post_send()
567 if (wr->opcode != IB_WR_SEND) { in qedr_gsi_post_send()
569 "gsi post send: failed due to unsupported opcode %d\n", in qedr_gsi_post_send()
570 wr->opcode); in qedr_gsi_post_send()
571 rc = -EINVAL; in qedr_gsi_post_send()
575 spin_lock_irqsave(&qp->q_lock, flags); in qedr_gsi_post_send()
579 spin_unlock_irqrestore(&qp->q_lock, flags); in qedr_gsi_post_send()
586 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id; in qedr_gsi_post_send()
587 qedr_inc_sw_prod(&qp->sq); in qedr_gsi_post_send()
588 DP_DEBUG(qp->dev, QEDR_MSG_GSI, in qedr_gsi_post_send()
589 "gsi post send: opcode=%d, in_irq=%ld, irqs_disabled=%d, wr_id=%llx\n", in qedr_gsi_post_send()
590 wr->opcode, in_irq(), irqs_disabled(), wr->wr_id); in qedr_gsi_post_send()
592 DP_ERR(dev, "gsi post send: failed to transmit (rc=%d)\n", rc); in qedr_gsi_post_send()
593 rc = -EAGAIN; in qedr_gsi_post_send()
597 spin_unlock_irqrestore(&qp->q_lock, flags); in qedr_gsi_post_send()
599 if (wr->next) { in qedr_gsi_post_send()
601 "gsi post send: failed second WR. Only one WR may be passed at a time\n"); in qedr_gsi_post_send()
602 *bad_wr = wr->next; in qedr_gsi_post_send()
603 rc = -EINVAL; in qedr_gsi_post_send()
616 struct qedr_dev *dev = get_qedr_dev(ibqp->device); in qedr_gsi_post_recv()
621 if ((qp->state != QED_ROCE_QP_STATE_RTR) && in qedr_gsi_post_recv()
622 (qp->state != QED_ROCE_QP_STATE_RTS)) { in qedr_gsi_post_recv()
625 "gsi post recv: failed to post rx buffer. state is %d and not QED_ROCE_QP_STATE_RTR/S\n", in qedr_gsi_post_recv()
626 qp->state); in qedr_gsi_post_recv()
627 return -EINVAL; in qedr_gsi_post_recv()
630 spin_lock_irqsave(&qp->q_lock, flags); in qedr_gsi_post_recv()
633 if (wr->num_sge > QEDR_GSI_MAX_RECV_SGE) { in qedr_gsi_post_recv()
635 "gsi post recv: failed to post rx buffer. too many sges %d>%d\n", in qedr_gsi_post_recv()
636 wr->num_sge, QEDR_GSI_MAX_RECV_SGE); in qedr_gsi_post_recv()
640 rc = dev->ops->ll2_post_rx_buffer(dev->rdma_ctx, in qedr_gsi_post_recv()
641 dev->gsi_ll2_handle, in qedr_gsi_post_recv()
642 wr->sg_list[0].addr, in qedr_gsi_post_recv()
643 wr->sg_list[0].length, in qedr_gsi_post_recv()
648 "gsi post recv: failed to post rx buffer (rc=%d)\n", in qedr_gsi_post_recv()
653 memset(&qp->rqe_wr_id[qp->rq.prod], 0, in qedr_gsi_post_recv()
654 sizeof(qp->rqe_wr_id[qp->rq.prod])); in qedr_gsi_post_recv()
655 qp->rqe_wr_id[qp->rq.prod].sg_list[0] = wr->sg_list[0]; in qedr_gsi_post_recv()
656 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id; in qedr_gsi_post_recv()
658 qedr_inc_sw_prod(&qp->rq); in qedr_gsi_post_recv()
660 wr = wr->next; in qedr_gsi_post_recv()
663 spin_unlock_irqrestore(&qp->q_lock, flags); in qedr_gsi_post_recv()
667 spin_unlock_irqrestore(&qp->q_lock, flags); in qedr_gsi_post_recv()
669 return -ENOMEM; in qedr_gsi_post_recv()
674 struct qedr_dev *dev = get_qedr_dev(ibcq->device); in qedr_gsi_poll_cq()
676 struct qedr_qp *qp = dev->gsi_qp; in qedr_gsi_poll_cq()
681 spin_lock_irqsave(&cq->cq_lock, flags); in qedr_gsi_poll_cq()
683 while (i < num_entries && qp->rq.cons != qp->rq.gsi_cons) { in qedr_gsi_poll_cq()
686 wc[i].qp = &qp->ibqp; in qedr_gsi_poll_cq()
687 wc[i].wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id; in qedr_gsi_poll_cq()
690 wc[i].status = (qp->rqe_wr_id[qp->rq.cons].rc) ? in qedr_gsi_poll_cq()
692 /* 0 - currently only one recv sg is supported */ in qedr_gsi_poll_cq()
693 wc[i].byte_len = qp->rqe_wr_id[qp->rq.cons].sg_list[0].length; in qedr_gsi_poll_cq()
695 ether_addr_copy(wc[i].smac, qp->rqe_wr_id[qp->rq.cons].smac); in qedr_gsi_poll_cq()
698 vlan_id = qp->rqe_wr_id[qp->rq.cons].vlan & in qedr_gsi_poll_cq()
703 wc[i].sl = (qp->rqe_wr_id[qp->rq.cons].vlan & in qedr_gsi_poll_cq()
707 qedr_inc_sw_cons(&qp->rq); in qedr_gsi_poll_cq()
711 while (i < num_entries && qp->sq.cons != qp->sq.gsi_cons) { in qedr_gsi_poll_cq()
714 wc[i].qp = &qp->ibqp; in qedr_gsi_poll_cq()
715 wc[i].wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id; in qedr_gsi_poll_cq()
719 qedr_inc_sw_cons(&qp->sq); in qedr_gsi_poll_cq()
723 spin_unlock_irqrestore(&cq->cq_lock, flags); in qedr_gsi_poll_cq()
726 …"gsi poll_cq: requested entries=%d, actual=%d, qp->rq.cons=%d, qp->rq.gsi_cons=%x, qp->sq.cons=%d,… in qedr_gsi_poll_cq()
727 num_entries, i, qp->rq.cons, qp->rq.gsi_cons, qp->sq.cons, in qedr_gsi_poll_cq()
728 qp->sq.gsi_cons, qp->ibqp.qp_num); in qedr_gsi_poll_cq()