Lines Matching +full:send +full:- +full:flush +full:- +full:out +full:- +full:sequence
1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
4 /* Copyright (c) 2008-2019, IBM Corporation */
30 * per-RDMAP message basis. Please keep order of initializer. All MPA len
36 .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_write) - 2),
44 .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_rreq) - 2),
51 .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_rresp) - 2),
59 .ctrl.mpa_len = htons(sizeof(struct iwarp_send) - 2),
66 .ctrl.mpa_len = htons(sizeof(struct iwarp_send_inv) - 2),
73 .ctrl.mpa_len = htons(sizeof(struct iwarp_send) - 2),
80 .ctrl.mpa_len = htons(sizeof(struct iwarp_send_inv) - 2),
87 .ctrl.mpa_len = htons(sizeof(struct iwarp_terminate) - 2),
100 read_lock(&sk->sk_callback_lock); in siw_qp_llp_data_ready()
102 if (unlikely(!sk->sk_user_data || !sk_to_qp(sk))) in siw_qp_llp_data_ready()
107 if (likely(!qp->rx_stream.rx_suspend && in siw_qp_llp_data_ready()
108 down_read_trylock(&qp->state_lock))) { in siw_qp_llp_data_ready()
111 if (likely(qp->attrs.state == SIW_QP_STATE_RTS)) in siw_qp_llp_data_ready()
120 up_read(&qp->state_lock); in siw_qp_llp_data_ready()
123 qp->rx_stream.rx_suspend); in siw_qp_llp_data_ready()
126 read_unlock(&sk->sk_callback_lock); in siw_qp_llp_data_ready()
132 siw_qp_state_to_string[qp->attrs.state]); in siw_qp_llp_close()
134 down_write(&qp->state_lock); in siw_qp_llp_close()
136 qp->rx_stream.rx_suspend = 1; in siw_qp_llp_close()
137 qp->tx_ctx.tx_suspend = 1; in siw_qp_llp_close()
138 qp->attrs.sk = NULL; in siw_qp_llp_close()
140 switch (qp->attrs.state) { in siw_qp_llp_close()
145 qp->attrs.state = SIW_QP_STATE_ERROR; in siw_qp_llp_close()
154 if (tx_wqe(qp)->wr_status == SIW_WR_IDLE) in siw_qp_llp_close()
155 qp->attrs.state = SIW_QP_STATE_ERROR; in siw_qp_llp_close()
157 qp->attrs.state = SIW_QP_STATE_IDLE; in siw_qp_llp_close()
162 siw_qp_state_to_string[qp->attrs.state]); in siw_qp_llp_close()
171 if (qp->cep) { in siw_qp_llp_close()
172 siw_cep_put(qp->cep); in siw_qp_llp_close()
173 qp->cep = NULL; in siw_qp_llp_close()
176 up_write(&qp->state_lock); in siw_qp_llp_close()
179 siw_qp_state_to_string[qp->attrs.state]); in siw_qp_llp_close()
183 * socket callback routine informing about newly available send space.
190 read_lock(&sk->sk_callback_lock); in siw_qp_llp_write_space()
194 cep->sk_write_space(sk); in siw_qp_llp_write_space()
196 if (!test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) in siw_qp_llp_write_space()
197 (void)siw_sq_start(cep->qp); in siw_qp_llp_write_space()
200 read_unlock(&sk->sk_callback_lock); in siw_qp_llp_write_space()
207 qp->irq = vcalloc(irq_size, sizeof(struct siw_sqe)); in siw_qp_readq_init()
208 if (!qp->irq) { in siw_qp_readq_init()
209 qp->attrs.irq_size = 0; in siw_qp_readq_init()
210 return -ENOMEM; in siw_qp_readq_init()
215 qp->orq = vcalloc(orq_size, sizeof(struct siw_sqe)); in siw_qp_readq_init()
216 if (!qp->orq) { in siw_qp_readq_init()
217 qp->attrs.orq_size = 0; in siw_qp_readq_init()
218 qp->attrs.irq_size = 0; in siw_qp_readq_init()
219 vfree(qp->irq); in siw_qp_readq_init()
220 return -ENOMEM; in siw_qp_readq_init()
223 qp->attrs.irq_size = irq_size; in siw_qp_readq_init()
224 qp->attrs.orq_size = orq_size; in siw_qp_readq_init()
230 * Send a non signalled READ or WRITE to peer side as negotiated
232 * as a current active WR and does not consume Send Queue space.
242 spin_lock_irqsave(&qp->sq_lock, flags); in siw_qp_mpa_rts()
244 if (unlikely(wqe->wr_status != SIW_WR_IDLE)) { in siw_qp_mpa_rts()
245 spin_unlock_irqrestore(&qp->sq_lock, flags); in siw_qp_mpa_rts()
246 return -EIO; in siw_qp_mpa_rts()
248 memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE); in siw_qp_mpa_rts()
250 wqe->wr_status = SIW_WR_QUEUED; in siw_qp_mpa_rts()
251 wqe->sqe.flags = 0; in siw_qp_mpa_rts()
252 wqe->sqe.num_sge = 1; in siw_qp_mpa_rts()
253 wqe->sqe.sge[0].length = 0; in siw_qp_mpa_rts()
254 wqe->sqe.sge[0].laddr = 0; in siw_qp_mpa_rts()
255 wqe->sqe.sge[0].lkey = 0; in siw_qp_mpa_rts()
260 wqe->sqe.rkey = 1; in siw_qp_mpa_rts()
261 wqe->sqe.raddr = 0; in siw_qp_mpa_rts()
262 wqe->processed = 0; in siw_qp_mpa_rts()
265 wqe->sqe.opcode = SIW_OP_WRITE; in siw_qp_mpa_rts()
269 wqe->sqe.opcode = SIW_OP_READ; in siw_qp_mpa_rts()
271 spin_lock(&qp->orq_lock); in siw_qp_mpa_rts()
273 if (qp->attrs.orq_size) in siw_qp_mpa_rts()
276 siw_read_to_orq(rreq, &wqe->sqe); in siw_qp_mpa_rts()
277 qp->orq_put++; in siw_qp_mpa_rts()
279 rv = -EIO; in siw_qp_mpa_rts()
281 spin_unlock(&qp->orq_lock); in siw_qp_mpa_rts()
283 rv = -EINVAL; in siw_qp_mpa_rts()
286 wqe->wr_status = SIW_WR_IDLE; in siw_qp_mpa_rts()
288 spin_unlock_irqrestore(&qp->sq_lock, flags); in siw_qp_mpa_rts()
343 if (!qp->term_info.valid) { in siw_init_terminate()
344 memset(&qp->term_info, 0, sizeof(qp->term_info)); in siw_init_terminate()
345 qp->term_info.layer = layer; in siw_init_terminate()
346 qp->term_info.etype = etype; in siw_init_terminate()
347 qp->term_info.ecode = ecode; in siw_init_terminate()
348 qp->term_info.in_tx = in_tx; in siw_init_terminate()
349 qp->term_info.valid = 1; in siw_init_terminate()
356 * Send a TERMINATE message, as defined in RFC's 5040/5041/5044/6581.
357 * Sending TERMINATE messages is best effort - such messages
358 * can only be send if the QP is still connected and it does
359 * not have another outbound message in-progress, i.e. the
369 struct socket *s = qp->attrs.sk; in siw_send_terminate()
370 struct siw_rx_stream *srx = &qp->rx_stream; in siw_send_terminate()
371 union iwarp_hdr *rx_hdr = &srx->hdr; in siw_send_terminate()
375 if (!qp->term_info.valid) in siw_send_terminate()
378 qp->term_info.valid = 0; in siw_send_terminate()
380 if (tx_wqe(qp)->wr_status == SIW_WR_INPROGRESS) { in siw_send_terminate()
381 siw_dbg_qp(qp, "cannot send TERMINATE: op %d in progress\n", in siw_send_terminate()
385 if (!s && qp->cep) in siw_send_terminate()
387 s = qp->cep->sock; in siw_send_terminate()
390 siw_dbg_qp(qp, "cannot send TERMINATE: not connected\n"); in siw_send_terminate()
398 term->ddp_qn = cpu_to_be32(RDMAP_UNTAGGED_QN_TERMINATE); in siw_send_terminate()
399 term->ddp_mo = 0; in siw_send_terminate()
400 term->ddp_msn = cpu_to_be32(1); in siw_send_terminate()
405 if ((qp->term_info.layer == TERM_ERROR_LAYER_DDP) || in siw_send_terminate()
406 ((qp->term_info.layer == TERM_ERROR_LAYER_RDMAP) && in siw_send_terminate()
407 (qp->term_info.etype != RDMAP_ETYPE_CATASTROPHIC))) { in siw_send_terminate()
414 memcpy(&term->ctrl, &iwarp_pktinfo[RDMAP_TERMINATE].ctrl, in siw_send_terminate()
417 __rdmap_term_set_layer(term, qp->term_info.layer); in siw_send_terminate()
418 __rdmap_term_set_etype(term, qp->term_info.etype); in siw_send_terminate()
419 __rdmap_term_set_ecode(term, qp->term_info.ecode); in siw_send_terminate()
421 switch (qp->term_info.layer) { in siw_send_terminate()
423 if (qp->term_info.etype == RDMAP_ETYPE_CATASTROPHIC) in siw_send_terminate()
427 if (qp->term_info.etype == RDMAP_ETYPE_REMOTE_PROTECTION) { in siw_send_terminate()
432 term->flag_m = 1; in siw_send_terminate()
433 term->flag_d = 1; in siw_send_terminate()
434 term->flag_r = 1; in siw_send_terminate()
436 if (qp->term_info.in_tx) { in siw_send_terminate()
447 memcpy(&rreq->ctrl, in siw_send_terminate()
451 rreq->rsvd = 0; in siw_send_terminate()
452 rreq->ddp_qn = in siw_send_terminate()
456 rreq->ddp_msn = htonl(wqe->sqe.sge[0].length); in siw_send_terminate()
458 rreq->ddp_mo = htonl(wqe->processed); in siw_send_terminate()
459 rreq->sink_stag = htonl(wqe->sqe.rkey); in siw_send_terminate()
460 rreq->sink_to = cpu_to_be64(wqe->sqe.raddr); in siw_send_terminate()
461 rreq->read_size = htonl(wqe->sqe.sge[0].length); in siw_send_terminate()
462 rreq->source_stag = htonl(wqe->sqe.sge[0].lkey); in siw_send_terminate()
463 rreq->source_to = in siw_send_terminate()
464 cpu_to_be64(wqe->sqe.sge[0].laddr); in siw_send_terminate()
476 if (__rdmap_get_opcode(&rx_hdr->ctrl) == in siw_send_terminate()
480 else /* SEND type */ in siw_send_terminate()
488 if ((qp->term_info.ecode == RDMAP_ECODE_VERSION) || in siw_send_terminate()
489 (qp->term_info.ecode == RDMAP_ECODE_OPCODE)) in siw_send_terminate()
495 if (rx_hdr->ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED) in siw_send_terminate()
501 term->flag_m = 1; in siw_send_terminate()
502 term->flag_d = 1; in siw_send_terminate()
504 term->ctrl.mpa_len = cpu_to_be16(iov[1].iov_len); in siw_send_terminate()
516 if (((qp->term_info.etype == DDP_ETYPE_TAGGED_BUF) && in siw_send_terminate()
517 (qp->term_info.ecode == DDP_ECODE_T_VERSION)) || in siw_send_terminate()
518 ((qp->term_info.etype == DDP_ETYPE_UNTAGGED_BUF) && in siw_send_terminate()
519 (qp->term_info.ecode == DDP_ECODE_UT_VERSION))) in siw_send_terminate()
524 if (rx_hdr->ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED) in siw_send_terminate()
529 term->flag_m = 1; in siw_send_terminate()
530 term->flag_d = 1; in siw_send_terminate()
536 if (term->flag_m || term->flag_d || term->flag_r) { in siw_send_terminate()
549 if (term->flag_m) { in siw_send_terminate()
550 u32 real_ddp_len = be16_to_cpu(rx_hdr->ctrl.mpa_len); in siw_send_terminate()
551 enum rdma_opcode op = __rdmap_get_opcode(&rx_hdr->ctrl); in siw_send_terminate()
553 real_ddp_len -= iwarp_pktinfo[op].hdr_len - MPA_HDR_SIZE; in siw_send_terminate()
554 rx_hdr->ctrl.mpa_len = cpu_to_be16(real_ddp_len); in siw_send_terminate()
557 term->ctrl.mpa_len = in siw_send_terminate()
558 cpu_to_be16(len_terminate - (MPA_HDR_SIZE + MPA_CRC_SIZE)); in siw_send_terminate()
559 if (qp->tx_ctx.mpa_crc_enabled) { in siw_send_terminate()
560 siw_crc_init(&qp->tx_ctx.mpa_crc); in siw_send_terminate()
561 siw_crc_update(&qp->tx_ctx.mpa_crc, in siw_send_terminate()
564 siw_crc_update(&qp->tx_ctx.mpa_crc, in siw_send_terminate()
567 siw_crc_final(&qp->tx_ctx.mpa_crc, (u8 *)&crc); in siw_send_terminate()
587 if (attrs->flags & SIW_RDMA_BIND_ENABLED) in siw_qp_modify_nonstate()
588 qp->attrs.flags |= SIW_RDMA_BIND_ENABLED; in siw_qp_modify_nonstate()
590 qp->attrs.flags &= ~SIW_RDMA_BIND_ENABLED; in siw_qp_modify_nonstate()
592 if (attrs->flags & SIW_RDMA_WRITE_ENABLED) in siw_qp_modify_nonstate()
593 qp->attrs.flags |= SIW_RDMA_WRITE_ENABLED; in siw_qp_modify_nonstate()
595 qp->attrs.flags &= ~SIW_RDMA_WRITE_ENABLED; in siw_qp_modify_nonstate()
597 if (attrs->flags & SIW_RDMA_READ_ENABLED) in siw_qp_modify_nonstate()
598 qp->attrs.flags |= SIW_RDMA_READ_ENABLED; in siw_qp_modify_nonstate()
600 qp->attrs.flags &= ~SIW_RDMA_READ_ENABLED; in siw_qp_modify_nonstate()
610 switch (attrs->state) { in siw_qp_nextstate_from_idle()
612 if (attrs->flags & SIW_MPA_CRC) { in siw_qp_nextstate_from_idle()
613 siw_crc_init(&qp->tx_ctx.mpa_crc); in siw_qp_nextstate_from_idle()
614 qp->tx_ctx.mpa_crc_enabled = true; in siw_qp_nextstate_from_idle()
615 siw_crc_init(&qp->rx_stream.mpa_crc); in siw_qp_nextstate_from_idle()
616 qp->rx_stream.mpa_crc_enabled = true; in siw_qp_nextstate_from_idle()
620 rv = -EINVAL; in siw_qp_nextstate_from_idle()
625 rv = -EINVAL; in siw_qp_nextstate_from_idle()
631 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_SEND] = 0; in siw_qp_nextstate_from_idle()
632 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ] = 0; in siw_qp_nextstate_from_idle()
633 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_TERMINATE] = 0; in siw_qp_nextstate_from_idle()
638 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_SEND] = 1; in siw_qp_nextstate_from_idle()
639 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ] = 1; in siw_qp_nextstate_from_idle()
640 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_TERMINATE] = 1; in siw_qp_nextstate_from_idle()
646 rv = siw_qp_readq_init(qp, attrs->irq_size, in siw_qp_nextstate_from_idle()
647 attrs->orq_size); in siw_qp_nextstate_from_idle()
651 qp->attrs.sk = attrs->sk; in siw_qp_nextstate_from_idle()
652 qp->attrs.state = SIW_QP_STATE_RTS; in siw_qp_nextstate_from_idle()
655 attrs->flags & SIW_MPA_CRC ? "y" : "n", in siw_qp_nextstate_from_idle()
656 qp->attrs.orq_size, qp->attrs.irq_size); in siw_qp_nextstate_from_idle()
661 qp->attrs.state = SIW_QP_STATE_ERROR; in siw_qp_nextstate_from_idle()
662 if (qp->cep) { in siw_qp_nextstate_from_idle()
663 siw_cep_put(qp->cep); in siw_qp_nextstate_from_idle()
664 qp->cep = NULL; in siw_qp_nextstate_from_idle()
679 switch (attrs->state) { in siw_qp_nextstate_from_rts()
689 if (tx_wqe(qp)->wr_status == SIW_WR_IDLE) { in siw_qp_nextstate_from_rts()
690 qp->attrs.state = SIW_QP_STATE_CLOSING; in siw_qp_nextstate_from_rts()
692 qp->attrs.state = SIW_QP_STATE_ERROR; in siw_qp_nextstate_from_rts()
701 qp->attrs.state = SIW_QP_STATE_TERMINATE; in siw_qp_nextstate_from_rts()
719 * Esp., how to handle the non-empty IRQ case? in siw_qp_nextstate_from_rts()
725 qp->attrs.state = SIW_QP_STATE_ERROR; in siw_qp_nextstate_from_rts()
738 switch (attrs->state) { in siw_qp_nextstate_from_term()
741 qp->attrs.state = SIW_QP_STATE_ERROR; in siw_qp_nextstate_from_term()
743 if (tx_wqe(qp)->wr_status != SIW_WR_IDLE) in siw_qp_nextstate_from_term()
757 switch (attrs->state) { in siw_qp_nextstate_from_close()
759 WARN_ON(tx_wqe(qp)->wr_status != SIW_WR_IDLE); in siw_qp_nextstate_from_close()
760 qp->attrs.state = SIW_QP_STATE_IDLE; in siw_qp_nextstate_from_close()
775 qp->attrs.state = SIW_QP_STATE_ERROR; in siw_qp_nextstate_from_close()
777 if (tx_wqe(qp)->wr_status != SIW_WR_IDLE) in siw_qp_nextstate_from_close()
785 siw_qp_state_to_string[qp->attrs.state], in siw_qp_nextstate_from_close()
786 siw_qp_state_to_string[attrs->state]); in siw_qp_nextstate_from_close()
788 rv = -ECONNABORTED; in siw_qp_nextstate_from_close()
794 * Caller must hold qp->state_lock
805 siw_qp_state_to_string[qp->attrs.state], in siw_qp_modify()
806 siw_qp_state_to_string[attrs->state]); in siw_qp_modify()
814 switch (qp->attrs.state) { in siw_qp_modify()
842 rreq->id = sqe->id; in siw_read_to_orq()
843 rreq->opcode = sqe->opcode; in siw_read_to_orq()
844 rreq->sge[0].laddr = sqe->sge[0].laddr; in siw_read_to_orq()
845 rreq->sge[0].length = sqe->sge[0].length; in siw_read_to_orq()
846 rreq->sge[0].lkey = sqe->sge[0].lkey; in siw_read_to_orq()
847 rreq->sge[1].lkey = sqe->sge[1].lkey; in siw_read_to_orq()
848 rreq->flags = sqe->flags | SIW_WQE_VALID; in siw_read_to_orq()
849 rreq->num_sge = 1; in siw_read_to_orq()
862 memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE); in siw_activate_tx_from_sq()
863 wqe->wr_status = SIW_WR_QUEUED; in siw_activate_tx_from_sq()
866 memcpy(&wqe->sqe, sqe, sizeof(*sqe)); in siw_activate_tx_from_sq()
868 if (wqe->sqe.opcode >= SIW_NUM_OPCODES) { in siw_activate_tx_from_sq()
869 rv = -EINVAL; in siw_activate_tx_from_sq()
870 goto out; in siw_activate_tx_from_sq()
872 if (wqe->sqe.flags & SIW_WQE_INLINE) { in siw_activate_tx_from_sq()
873 if (wqe->sqe.opcode != SIW_OP_SEND && in siw_activate_tx_from_sq()
874 wqe->sqe.opcode != SIW_OP_WRITE) { in siw_activate_tx_from_sq()
875 rv = -EINVAL; in siw_activate_tx_from_sq()
876 goto out; in siw_activate_tx_from_sq()
878 if (wqe->sqe.sge[0].length > SIW_MAX_INLINE) { in siw_activate_tx_from_sq()
879 rv = -EINVAL; in siw_activate_tx_from_sq()
880 goto out; in siw_activate_tx_from_sq()
882 wqe->sqe.sge[0].laddr = (uintptr_t)&wqe->sqe.sge[1]; in siw_activate_tx_from_sq()
883 wqe->sqe.sge[0].lkey = 0; in siw_activate_tx_from_sq()
884 wqe->sqe.num_sge = 1; in siw_activate_tx_from_sq()
886 if (wqe->sqe.flags & SIW_WQE_READ_FENCE) { in siw_activate_tx_from_sq()
888 if (unlikely(wqe->sqe.opcode == SIW_OP_READ || in siw_activate_tx_from_sq()
889 wqe->sqe.opcode == in siw_activate_tx_from_sq()
892 rv = -EINVAL; in siw_activate_tx_from_sq()
893 goto out; in siw_activate_tx_from_sq()
895 spin_lock(&qp->orq_lock); in siw_activate_tx_from_sq()
897 if (qp->attrs.orq_size && !siw_orq_empty(qp)) { in siw_activate_tx_from_sq()
898 qp->tx_ctx.orq_fence = 1; in siw_activate_tx_from_sq()
901 spin_unlock(&qp->orq_lock); in siw_activate_tx_from_sq()
903 } else if (wqe->sqe.opcode == SIW_OP_READ || in siw_activate_tx_from_sq()
904 wqe->sqe.opcode == SIW_OP_READ_LOCAL_INV) { in siw_activate_tx_from_sq()
907 if (unlikely(!qp->attrs.orq_size)) { in siw_activate_tx_from_sq()
908 /* We negotiated not to send READ req's */ in siw_activate_tx_from_sq()
909 rv = -EINVAL; in siw_activate_tx_from_sq()
910 goto out; in siw_activate_tx_from_sq()
912 wqe->sqe.num_sge = 1; in siw_activate_tx_from_sq()
914 spin_lock(&qp->orq_lock); in siw_activate_tx_from_sq()
922 siw_read_to_orq(rreq, &wqe->sqe); in siw_activate_tx_from_sq()
923 qp->orq_put++; in siw_activate_tx_from_sq()
925 qp->tx_ctx.orq_fence = 1; in siw_activate_tx_from_sq()
928 spin_unlock(&qp->orq_lock); in siw_activate_tx_from_sq()
931 /* Clear SQE, can be re-used by application */ in siw_activate_tx_from_sq()
932 smp_store_mb(sqe->flags, 0); in siw_activate_tx_from_sq()
933 qp->sq_get++; in siw_activate_tx_from_sq()
934 out: in siw_activate_tx_from_sq()
937 wqe->wr_status = SIW_WR_IDLE; in siw_activate_tx_from_sq()
945 * the active IRQ will not be served after qp->irq_burst, if the
953 if (!qp->attrs.irq_size) in siw_activate_tx()
956 irqe = &qp->irq[qp->irq_get % qp->attrs.irq_size]; in siw_activate_tx()
958 if (!(irqe->flags & SIW_WQE_VALID)) in siw_activate_tx()
965 if (sq_get_next(qp) && ++qp->irq_burst >= SIW_IRQ_MAXBURST_SQ_ACTIVE) { in siw_activate_tx()
966 qp->irq_burst = 0; in siw_activate_tx()
969 memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE); in siw_activate_tx()
970 wqe->wr_status = SIW_WR_QUEUED; in siw_activate_tx()
973 wqe->sqe.opcode = SIW_OP_READ_RESPONSE; in siw_activate_tx()
974 wqe->sqe.flags = 0; in siw_activate_tx()
975 if (irqe->num_sge) { in siw_activate_tx()
976 wqe->sqe.num_sge = 1; in siw_activate_tx()
977 wqe->sqe.sge[0].length = irqe->sge[0].length; in siw_activate_tx()
978 wqe->sqe.sge[0].laddr = irqe->sge[0].laddr; in siw_activate_tx()
979 wqe->sqe.sge[0].lkey = irqe->sge[0].lkey; in siw_activate_tx()
981 wqe->sqe.num_sge = 0; in siw_activate_tx()
984 /* Retain original RREQ's message sequence number for in siw_activate_tx()
987 wqe->sqe.sge[1].length = irqe->sge[1].length; in siw_activate_tx()
989 wqe->sqe.rkey = irqe->rkey; in siw_activate_tx()
990 wqe->sqe.raddr = irqe->raddr; in siw_activate_tx()
992 wqe->processed = 0; in siw_activate_tx()
993 qp->irq_get++; in siw_activate_tx()
996 smp_store_mb(irqe->flags, 0); in siw_activate_tx()
1009 if (!cq->base_cq.comp_handler) in siw_cq_notify_now()
1013 cq_notify = READ_ONCE(cq->notify->flags); in siw_cq_notify_now()
1019 * CQ notification is one-shot: Since the in siw_cq_notify_now()
1021 * the CQ gets dis-aremd and must be re-aremd in siw_cq_notify_now()
1024 WRITE_ONCE(cq->notify->flags, SIW_NOTIFY_NOT); in siw_cq_notify_now()
1034 struct siw_cq *cq = qp->scq; in siw_sqe_complete()
1038 u32 sqe_flags = sqe->flags; in siw_sqe_complete()
1043 spin_lock_irqsave(&cq->lock, flags); in siw_sqe_complete()
1045 idx = cq->cq_put % cq->num_cqe; in siw_sqe_complete()
1046 cqe = &cq->queue[idx]; in siw_sqe_complete()
1048 if (!READ_ONCE(cqe->flags)) { in siw_sqe_complete()
1051 cqe->id = sqe->id; in siw_sqe_complete()
1052 cqe->opcode = sqe->opcode; in siw_sqe_complete()
1053 cqe->status = status; in siw_sqe_complete()
1054 cqe->imm_data = 0; in siw_sqe_complete()
1055 cqe->bytes = bytes; in siw_sqe_complete()
1057 if (rdma_is_kernel_res(&cq->base_cq.res)) in siw_sqe_complete()
1058 cqe->base_qp = &qp->base_qp; in siw_sqe_complete()
1060 cqe->qp_id = qp_id(qp); in siw_sqe_complete()
1063 WRITE_ONCE(cqe->flags, SIW_WQE_VALID); in siw_sqe_complete()
1065 smp_store_mb(sqe->flags, 0); in siw_sqe_complete()
1067 cq->cq_put++; in siw_sqe_complete()
1070 spin_unlock_irqrestore(&cq->lock, flags); in siw_sqe_complete()
1074 cq->base_cq.comp_handler(&cq->base_cq, in siw_sqe_complete()
1075 cq->base_cq.cq_context); in siw_sqe_complete()
1078 spin_unlock_irqrestore(&cq->lock, flags); in siw_sqe_complete()
1079 rv = -ENOMEM; in siw_sqe_complete()
1084 smp_store_mb(sqe->flags, 0); in siw_sqe_complete()
1092 struct siw_cq *cq = qp->rcq; in siw_rqe_complete()
1100 spin_lock_irqsave(&cq->lock, flags); in siw_rqe_complete()
1102 idx = cq->cq_put % cq->num_cqe; in siw_rqe_complete()
1103 cqe = &cq->queue[idx]; in siw_rqe_complete()
1105 if (!READ_ONCE(cqe->flags)) { in siw_rqe_complete()
1109 cqe->id = rqe->id; in siw_rqe_complete()
1110 cqe->opcode = SIW_OP_RECEIVE; in siw_rqe_complete()
1111 cqe->status = status; in siw_rqe_complete()
1112 cqe->imm_data = 0; in siw_rqe_complete()
1113 cqe->bytes = bytes; in siw_rqe_complete()
1115 if (rdma_is_kernel_res(&cq->base_cq.res)) { in siw_rqe_complete()
1116 cqe->base_qp = &qp->base_qp; in siw_rqe_complete()
1119 cqe->inval_stag = inval_stag; in siw_rqe_complete()
1122 cqe->qp_id = qp_id(qp); in siw_rqe_complete()
1125 WRITE_ONCE(cqe->flags, cqe_flags); in siw_rqe_complete()
1127 smp_store_mb(rqe->flags, 0); in siw_rqe_complete()
1129 cq->cq_put++; in siw_rqe_complete()
1132 spin_unlock_irqrestore(&cq->lock, flags); in siw_rqe_complete()
1136 cq->base_cq.comp_handler(&cq->base_cq, in siw_rqe_complete()
1137 cq->base_cq.cq_context); in siw_rqe_complete()
1140 spin_unlock_irqrestore(&cq->lock, flags); in siw_rqe_complete()
1141 rv = -ENOMEM; in siw_rqe_complete()
1146 smp_store_mb(rqe->flags, 0); in siw_rqe_complete()
1154 * Flush SQ and ORQ entries to CQ.
1168 while (qp->attrs.orq_size) { in siw_sq_flush()
1169 sqe = &qp->orq[qp->orq_get % qp->attrs.orq_size]; in siw_sq_flush()
1170 if (!READ_ONCE(sqe->flags)) in siw_sq_flush()
1176 WRITE_ONCE(sqe->flags, 0); in siw_sq_flush()
1177 qp->orq_get++; in siw_sq_flush()
1180 * Flush an in-progress WQE if present in siw_sq_flush()
1182 if (wqe->wr_status != SIW_WR_IDLE) { in siw_sq_flush()
1183 siw_dbg_qp(qp, "flush current SQE, type %d, status %d\n", in siw_sq_flush()
1184 tx_type(wqe), wqe->wr_status); in siw_sq_flush()
1191 wqe->wr_status == SIW_WR_QUEUED)) in siw_sq_flush()
1193 * An in-progress Read Request is already in in siw_sq_flush()
1196 siw_sqe_complete(qp, &wqe->sqe, wqe->bytes, in siw_sq_flush()
1199 wqe->wr_status = SIW_WR_IDLE; in siw_sq_flush()
1202 * Flush the Send Queue in siw_sq_flush()
1204 while (qp->attrs.sq_size) { in siw_sq_flush()
1205 sqe = &qp->sendq[qp->sq_get % qp->attrs.sq_size]; in siw_sq_flush()
1206 if (!READ_ONCE(sqe->flags)) in siw_sq_flush()
1217 WRITE_ONCE(sqe->flags, 0); in siw_sq_flush()
1218 qp->sq_get++; in siw_sq_flush()
1227 * Flush recv queue entries to CQ. Also
1237 struct siw_wqe *wqe = &qp->rx_untagged.wqe_active; in siw_rq_flush()
1240 * Flush an in-progress untagged operation if present in siw_rq_flush()
1242 if (wqe->wr_status != SIW_WR_IDLE) { in siw_rq_flush()
1243 siw_dbg_qp(qp, "flush current rqe, type %d, status %d\n", in siw_rq_flush()
1244 rx_type(wqe), wqe->wr_status); in siw_rq_flush()
1249 siw_rqe_complete(qp, &wqe->rqe, wqe->bytes, in siw_rq_flush()
1254 siw_sqe_complete(qp, &wqe->sqe, 0, SIW_WC_WR_FLUSH_ERR); in siw_rq_flush()
1256 wqe->wr_status = SIW_WR_IDLE; in siw_rq_flush()
1258 wqe = &qp->rx_tagged.wqe_active; in siw_rq_flush()
1260 if (wqe->wr_status != SIW_WR_IDLE) { in siw_rq_flush()
1262 wqe->wr_status = SIW_WR_IDLE; in siw_rq_flush()
1265 * Flush the Receive Queue in siw_rq_flush()
1267 while (qp->attrs.rq_size) { in siw_rq_flush()
1269 &qp->recvq[qp->rq_get % qp->attrs.rq_size]; in siw_rq_flush()
1271 if (!READ_ONCE(rqe->flags)) in siw_rq_flush()
1277 WRITE_ONCE(rqe->flags, 0); in siw_rq_flush()
1278 qp->rq_get++; in siw_rq_flush()
1284 int rv = xa_alloc(&sdev->qp_xa, &qp->base_qp.qp_num, qp, xa_limit_32b, in siw_qp_add()
1288 kref_init(&qp->ref); in siw_qp_add()
1289 qp->sdev = sdev; in siw_qp_add()
1298 struct siw_device *sdev = qp->sdev; in siw_free_qp()
1301 if (qp->cep) in siw_free_qp()
1302 siw_cep_put(qp->cep); in siw_free_qp()
1304 found = xa_erase(&sdev->qp_xa, qp_id(qp)); in siw_free_qp()
1306 spin_lock_irqsave(&sdev->lock, flags); in siw_free_qp()
1307 list_del(&qp->devq); in siw_free_qp()
1308 spin_unlock_irqrestore(&sdev->lock, flags); in siw_free_qp()
1310 vfree(qp->sendq); in siw_free_qp()
1311 vfree(qp->recvq); in siw_free_qp()
1312 vfree(qp->irq); in siw_free_qp()
1313 vfree(qp->orq); in siw_free_qp()
1315 siw_put_tx_cpu(qp->tx_cpu); in siw_free_qp()
1316 complete(&qp->qp_free); in siw_free_qp()
1317 atomic_dec(&sdev->num_qp); in siw_free_qp()