1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2
3 /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
4 /* Copyright (c) 2008-2019, IBM Corporation */
5
6 #include <linux/errno.h>
7 #include <linux/types.h>
8 #include <linux/net.h>
9 #include <linux/scatterlist.h>
10 #include <linux/llist.h>
11 #include <asm/barrier.h>
12 #include <net/tcp.h>
13 #include <trace/events/sock.h>
14
15 #include "siw.h"
16 #include "siw_verbs.h"
17 #include "siw_mem.h"
18
19 static char siw_qp_state_to_string[SIW_QP_STATE_COUNT][sizeof "TERMINATE"] = {
20 [SIW_QP_STATE_IDLE] = "IDLE",
21 [SIW_QP_STATE_RTR] = "RTR",
22 [SIW_QP_STATE_RTS] = "RTS",
23 [SIW_QP_STATE_CLOSING] = "CLOSING",
24 [SIW_QP_STATE_TERMINATE] = "TERMINATE",
25 [SIW_QP_STATE_ERROR] = "ERROR"
26 };
27
28 /*
29 * iWARP (RDMAP, DDP and MPA) parameters as well as Softiwarp settings on a
30 * per-RDMAP message basis. Please keep order of initializer. All MPA len
31 * is initialized to minimum packet size.
32 */
33 struct iwarp_msg_info iwarp_pktinfo[RDMAP_TERMINATE + 1] = {
34 { /* RDMAP_RDMA_WRITE */
35 .hdr_len = sizeof(struct iwarp_rdma_write),
36 .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_write) - 2),
37 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_TAGGED | DDP_FLAG_LAST |
38 cpu_to_be16(DDP_VERSION << 8) |
39 cpu_to_be16(RDMAP_VERSION << 6) |
40 cpu_to_be16(RDMAP_RDMA_WRITE),
41 .rx_data = siw_proc_write },
42 { /* RDMAP_RDMA_READ_REQ */
43 .hdr_len = sizeof(struct iwarp_rdma_rreq),
44 .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_rreq) - 2),
45 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) |
46 cpu_to_be16(RDMAP_VERSION << 6) |
47 cpu_to_be16(RDMAP_RDMA_READ_REQ),
48 .rx_data = siw_proc_rreq },
49 { /* RDMAP_RDMA_READ_RESP */
50 .hdr_len = sizeof(struct iwarp_rdma_rresp),
51 .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_rresp) - 2),
52 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_TAGGED | DDP_FLAG_LAST |
53 cpu_to_be16(DDP_VERSION << 8) |
54 cpu_to_be16(RDMAP_VERSION << 6) |
55 cpu_to_be16(RDMAP_RDMA_READ_RESP),
56 .rx_data = siw_proc_rresp },
57 { /* RDMAP_SEND */
58 .hdr_len = sizeof(struct iwarp_send),
59 .ctrl.mpa_len = htons(sizeof(struct iwarp_send) - 2),
60 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) |
61 cpu_to_be16(RDMAP_VERSION << 6) |
62 cpu_to_be16(RDMAP_SEND),
63 .rx_data = siw_proc_send },
64 { /* RDMAP_SEND_INVAL */
65 .hdr_len = sizeof(struct iwarp_send_inv),
66 .ctrl.mpa_len = htons(sizeof(struct iwarp_send_inv) - 2),
67 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) |
68 cpu_to_be16(RDMAP_VERSION << 6) |
69 cpu_to_be16(RDMAP_SEND_INVAL),
70 .rx_data = siw_proc_send },
71 { /* RDMAP_SEND_SE */
72 .hdr_len = sizeof(struct iwarp_send),
73 .ctrl.mpa_len = htons(sizeof(struct iwarp_send) - 2),
74 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) |
75 cpu_to_be16(RDMAP_VERSION << 6) |
76 cpu_to_be16(RDMAP_SEND_SE),
77 .rx_data = siw_proc_send },
78 { /* RDMAP_SEND_SE_INVAL */
79 .hdr_len = sizeof(struct iwarp_send_inv),
80 .ctrl.mpa_len = htons(sizeof(struct iwarp_send_inv) - 2),
81 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) |
82 cpu_to_be16(RDMAP_VERSION << 6) |
83 cpu_to_be16(RDMAP_SEND_SE_INVAL),
84 .rx_data = siw_proc_send },
85 { /* RDMAP_TERMINATE */
86 .hdr_len = sizeof(struct iwarp_terminate),
87 .ctrl.mpa_len = htons(sizeof(struct iwarp_terminate) - 2),
88 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) |
89 cpu_to_be16(RDMAP_VERSION << 6) |
90 cpu_to_be16(RDMAP_TERMINATE),
91 .rx_data = siw_proc_terminate }
92 };
93
siw_qp_llp_data_ready(struct sock * sk)94 void siw_qp_llp_data_ready(struct sock *sk)
95 {
96 struct siw_qp *qp;
97
98 trace_sk_data_ready(sk);
99
100 read_lock(&sk->sk_callback_lock);
101
102 if (unlikely(!sk->sk_user_data || !sk_to_qp(sk)))
103 goto done;
104
105 qp = sk_to_qp(sk);
106
107 if (likely(!qp->rx_stream.rx_suspend &&
108 down_read_trylock(&qp->state_lock))) {
109 read_descriptor_t rd_desc = { .arg.data = qp, .count = 1 };
110
111 if (likely(qp->attrs.state == SIW_QP_STATE_RTS))
112 /*
113 * Implements data receive operation during
114 * socket callback. TCP gracefully catches
115 * the case where there is nothing to receive
116 * (not calling siw_tcp_rx_data() then).
117 */
118 tcp_read_sock(sk, &rd_desc, siw_tcp_rx_data);
119
120 up_read(&qp->state_lock);
121 } else {
122 siw_dbg_qp(qp, "unable to process RX, suspend: %d\n",
123 qp->rx_stream.rx_suspend);
124 }
125 done:
126 read_unlock(&sk->sk_callback_lock);
127 }
128
siw_qp_llp_close(struct siw_qp * qp)129 void siw_qp_llp_close(struct siw_qp *qp)
130 {
131 siw_dbg_qp(qp, "enter llp close, state = %s\n",
132 siw_qp_state_to_string[qp->attrs.state]);
133
134 down_write(&qp->state_lock);
135
136 qp->rx_stream.rx_suspend = 1;
137 qp->tx_ctx.tx_suspend = 1;
138 qp->attrs.sk = NULL;
139
140 switch (qp->attrs.state) {
141 case SIW_QP_STATE_RTS:
142 case SIW_QP_STATE_RTR:
143 case SIW_QP_STATE_IDLE:
144 case SIW_QP_STATE_TERMINATE:
145 qp->attrs.state = SIW_QP_STATE_ERROR;
146 break;
147 /*
148 * SIW_QP_STATE_CLOSING:
149 *
150 * This is a forced close. shall the QP be moved to
151 * ERROR or IDLE ?
152 */
153 case SIW_QP_STATE_CLOSING:
154 if (tx_wqe(qp)->wr_status == SIW_WR_IDLE)
155 qp->attrs.state = SIW_QP_STATE_ERROR;
156 else
157 qp->attrs.state = SIW_QP_STATE_IDLE;
158 break;
159
160 default:
161 siw_dbg_qp(qp, "llp close: no state transition needed: %s\n",
162 siw_qp_state_to_string[qp->attrs.state]);
163 break;
164 }
165 siw_sq_flush(qp);
166 siw_rq_flush(qp);
167
168 /*
169 * Dereference closing CEP
170 */
171 if (qp->cep) {
172 siw_cep_put(qp->cep);
173 qp->cep = NULL;
174 }
175
176 up_write(&qp->state_lock);
177
178 siw_dbg_qp(qp, "llp close exit: state %s\n",
179 siw_qp_state_to_string[qp->attrs.state]);
180 }
181
182 /*
183 * socket callback routine informing about newly available send space.
184 * Function schedules SQ work for processing SQ items.
185 */
siw_qp_llp_write_space(struct sock * sk)186 void siw_qp_llp_write_space(struct sock *sk)
187 {
188 struct siw_cep *cep;
189
190 read_lock(&sk->sk_callback_lock);
191
192 cep = sk_to_cep(sk);
193 if (cep) {
194 cep->sk_write_space(sk);
195
196 if (!test_bit(SOCK_NOSPACE, &sk->sk_socket->flags))
197 (void)siw_sq_start(cep->qp);
198 }
199
200 read_unlock(&sk->sk_callback_lock);
201 }
202
siw_qp_readq_init(struct siw_qp * qp,int irq_size,int orq_size)203 static int siw_qp_readq_init(struct siw_qp *qp, int irq_size, int orq_size)
204 {
205 if (irq_size) {
206 irq_size = roundup_pow_of_two(irq_size);
207 qp->irq = vcalloc(irq_size, sizeof(struct siw_sqe));
208 if (!qp->irq) {
209 qp->attrs.irq_size = 0;
210 return -ENOMEM;
211 }
212 }
213 if (orq_size) {
214 orq_size = roundup_pow_of_two(orq_size);
215 qp->orq = vcalloc(orq_size, sizeof(struct siw_sqe));
216 if (!qp->orq) {
217 qp->attrs.orq_size = 0;
218 qp->attrs.irq_size = 0;
219 vfree(qp->irq);
220 return -ENOMEM;
221 }
222 }
223 qp->attrs.irq_size = irq_size;
224 qp->attrs.orq_size = orq_size;
225 siw_dbg_qp(qp, "ORD %d, IRD %d\n", orq_size, irq_size);
226 return 0;
227 }
228
229 /*
230 * Send a non signalled READ or WRITE to peer side as negotiated
231 * with MPAv2 P2P setup protocol. The work request is only created
232 * as a current active WR and does not consume Send Queue space.
233 *
234 * Caller must hold QP state lock.
235 */
siw_qp_mpa_rts(struct siw_qp * qp,enum mpa_v2_ctrl ctrl)236 int siw_qp_mpa_rts(struct siw_qp *qp, enum mpa_v2_ctrl ctrl)
237 {
238 struct siw_wqe *wqe = tx_wqe(qp);
239 unsigned long flags;
240 int rv = 0;
241
242 spin_lock_irqsave(&qp->sq_lock, flags);
243
244 if (unlikely(wqe->wr_status != SIW_WR_IDLE)) {
245 spin_unlock_irqrestore(&qp->sq_lock, flags);
246 return -EIO;
247 }
248 memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE);
249
250 wqe->wr_status = SIW_WR_QUEUED;
251 wqe->sqe.flags = 0;
252 wqe->sqe.num_sge = 1;
253 wqe->sqe.sge[0].length = 0;
254 wqe->sqe.sge[0].laddr = 0;
255 wqe->sqe.sge[0].lkey = 0;
256 /*
257 * While it must not be checked for inbound zero length
258 * READ/WRITE, some HW may treat STag 0 special.
259 */
260 wqe->sqe.rkey = 1;
261 wqe->sqe.raddr = 0;
262 wqe->processed = 0;
263
264 if (ctrl & MPA_V2_RDMA_WRITE_RTR)
265 wqe->sqe.opcode = SIW_OP_WRITE;
266 else if (ctrl & MPA_V2_RDMA_READ_RTR) {
267 struct siw_sqe *rreq = NULL;
268
269 wqe->sqe.opcode = SIW_OP_READ;
270
271 spin_lock(&qp->orq_lock);
272
273 if (qp->attrs.orq_size)
274 rreq = orq_get_free(qp);
275 if (rreq) {
276 siw_read_to_orq(rreq, &wqe->sqe);
277 qp->orq_put++;
278 } else
279 rv = -EIO;
280
281 spin_unlock(&qp->orq_lock);
282 } else
283 rv = -EINVAL;
284
285 if (rv)
286 wqe->wr_status = SIW_WR_IDLE;
287
288 spin_unlock_irqrestore(&qp->sq_lock, flags);
289
290 if (!rv)
291 rv = siw_sq_start(qp);
292
293 return rv;
294 }
295
296 /*
297 * Map memory access error to DDP tagged error
298 */
siw_tagged_error(enum siw_access_state state)299 enum ddp_ecode siw_tagged_error(enum siw_access_state state)
300 {
301 switch (state) {
302 case E_STAG_INVALID:
303 return DDP_ECODE_T_INVALID_STAG;
304 case E_BASE_BOUNDS:
305 return DDP_ECODE_T_BASE_BOUNDS;
306 case E_PD_MISMATCH:
307 return DDP_ECODE_T_STAG_NOT_ASSOC;
308 case E_ACCESS_PERM:
309 /*
310 * RFC 5041 (DDP) lacks an ecode for insufficient access
311 * permissions. 'Invalid STag' seem to be the closest
312 * match though.
313 */
314 return DDP_ECODE_T_INVALID_STAG;
315 default:
316 WARN_ON(1);
317 return DDP_ECODE_T_INVALID_STAG;
318 }
319 }
320
321 /*
322 * Map memory access error to RDMAP protection error
323 */
siw_rdmap_error(enum siw_access_state state)324 enum rdmap_ecode siw_rdmap_error(enum siw_access_state state)
325 {
326 switch (state) {
327 case E_STAG_INVALID:
328 return RDMAP_ECODE_INVALID_STAG;
329 case E_BASE_BOUNDS:
330 return RDMAP_ECODE_BASE_BOUNDS;
331 case E_PD_MISMATCH:
332 return RDMAP_ECODE_STAG_NOT_ASSOC;
333 case E_ACCESS_PERM:
334 return RDMAP_ECODE_ACCESS_RIGHTS;
335 default:
336 return RDMAP_ECODE_UNSPECIFIED;
337 }
338 }
339
siw_init_terminate(struct siw_qp * qp,enum term_elayer layer,u8 etype,u8 ecode,int in_tx)340 void siw_init_terminate(struct siw_qp *qp, enum term_elayer layer, u8 etype,
341 u8 ecode, int in_tx)
342 {
343 if (!qp->term_info.valid) {
344 memset(&qp->term_info, 0, sizeof(qp->term_info));
345 qp->term_info.layer = layer;
346 qp->term_info.etype = etype;
347 qp->term_info.ecode = ecode;
348 qp->term_info.in_tx = in_tx;
349 qp->term_info.valid = 1;
350 }
351 siw_dbg_qp(qp, "init TERM: layer %d, type %d, code %d, in tx %s\n",
352 layer, etype, ecode, in_tx ? "yes" : "no");
353 }
354
355 /*
356 * Send a TERMINATE message, as defined in RFC's 5040/5041/5044/6581.
357 * Sending TERMINATE messages is best effort - such messages
358 * can only be send if the QP is still connected and it does
359 * not have another outbound message in-progress, i.e. the
360 * TERMINATE message must not interfer with an incomplete current
361 * transmit operation.
362 */
siw_send_terminate(struct siw_qp * qp)363 void siw_send_terminate(struct siw_qp *qp)
364 {
365 struct kvec iov[3];
366 struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_EOR };
367 struct iwarp_terminate *term = NULL;
368 union iwarp_hdr *err_hdr = NULL;
369 struct socket *s = qp->attrs.sk;
370 struct siw_rx_stream *srx = &qp->rx_stream;
371 union iwarp_hdr *rx_hdr = &srx->hdr;
372 u32 crc = 0;
373 int num_frags, len_terminate, rv;
374
375 if (!qp->term_info.valid)
376 return;
377
378 qp->term_info.valid = 0;
379
380 if (tx_wqe(qp)->wr_status == SIW_WR_INPROGRESS) {
381 siw_dbg_qp(qp, "cannot send TERMINATE: op %d in progress\n",
382 tx_type(tx_wqe(qp)));
383 return;
384 }
385 if (!s && qp->cep)
386 /* QP not yet in RTS. Take socket from connection end point */
387 s = qp->cep->sock;
388
389 if (!s) {
390 siw_dbg_qp(qp, "cannot send TERMINATE: not connected\n");
391 return;
392 }
393
394 term = kzalloc(sizeof(*term), GFP_KERNEL);
395 if (!term)
396 return;
397
398 term->ddp_qn = cpu_to_be32(RDMAP_UNTAGGED_QN_TERMINATE);
399 term->ddp_mo = 0;
400 term->ddp_msn = cpu_to_be32(1);
401
402 iov[0].iov_base = term;
403 iov[0].iov_len = sizeof(*term);
404
405 if ((qp->term_info.layer == TERM_ERROR_LAYER_DDP) ||
406 ((qp->term_info.layer == TERM_ERROR_LAYER_RDMAP) &&
407 (qp->term_info.etype != RDMAP_ETYPE_CATASTROPHIC))) {
408 err_hdr = kzalloc(sizeof(*err_hdr), GFP_KERNEL);
409 if (!err_hdr) {
410 kfree(term);
411 return;
412 }
413 }
414 memcpy(&term->ctrl, &iwarp_pktinfo[RDMAP_TERMINATE].ctrl,
415 sizeof(struct iwarp_ctrl));
416
417 __rdmap_term_set_layer(term, qp->term_info.layer);
418 __rdmap_term_set_etype(term, qp->term_info.etype);
419 __rdmap_term_set_ecode(term, qp->term_info.ecode);
420
421 switch (qp->term_info.layer) {
422 case TERM_ERROR_LAYER_RDMAP:
423 if (qp->term_info.etype == RDMAP_ETYPE_CATASTROPHIC)
424 /* No additional DDP/RDMAP header to be included */
425 break;
426
427 if (qp->term_info.etype == RDMAP_ETYPE_REMOTE_PROTECTION) {
428 /*
429 * Complete RDMAP frame will get attached, and
430 * DDP segment length is valid
431 */
432 term->flag_m = 1;
433 term->flag_d = 1;
434 term->flag_r = 1;
435
436 if (qp->term_info.in_tx) {
437 struct iwarp_rdma_rreq *rreq;
438 struct siw_wqe *wqe = tx_wqe(qp);
439
440 /* Inbound RREQ error, detected during
441 * RRESP creation. Take state from
442 * current TX work queue element to
443 * reconstruct peers RREQ.
444 */
445 rreq = (struct iwarp_rdma_rreq *)err_hdr;
446
447 memcpy(&rreq->ctrl,
448 &iwarp_pktinfo[RDMAP_RDMA_READ_REQ].ctrl,
449 sizeof(struct iwarp_ctrl));
450
451 rreq->rsvd = 0;
452 rreq->ddp_qn =
453 htonl(RDMAP_UNTAGGED_QN_RDMA_READ);
454
455 /* Provide RREQ's MSN as kept aside */
456 rreq->ddp_msn = htonl(wqe->sqe.sge[0].length);
457
458 rreq->ddp_mo = htonl(wqe->processed);
459 rreq->sink_stag = htonl(wqe->sqe.rkey);
460 rreq->sink_to = cpu_to_be64(wqe->sqe.raddr);
461 rreq->read_size = htonl(wqe->sqe.sge[0].length);
462 rreq->source_stag = htonl(wqe->sqe.sge[0].lkey);
463 rreq->source_to =
464 cpu_to_be64(wqe->sqe.sge[0].laddr);
465
466 iov[1].iov_base = rreq;
467 iov[1].iov_len = sizeof(*rreq);
468
469 rx_hdr = (union iwarp_hdr *)rreq;
470 } else {
471 /* Take RDMAP/DDP information from
472 * current (failed) inbound frame.
473 */
474 iov[1].iov_base = rx_hdr;
475
476 if (__rdmap_get_opcode(&rx_hdr->ctrl) ==
477 RDMAP_RDMA_READ_REQ)
478 iov[1].iov_len =
479 sizeof(struct iwarp_rdma_rreq);
480 else /* SEND type */
481 iov[1].iov_len =
482 sizeof(struct iwarp_send);
483 }
484 } else {
485 /* Do not report DDP hdr information if packet
486 * layout is unknown
487 */
488 if ((qp->term_info.ecode == RDMAP_ECODE_VERSION) ||
489 (qp->term_info.ecode == RDMAP_ECODE_OPCODE))
490 break;
491
492 iov[1].iov_base = rx_hdr;
493
494 /* Only DDP frame will get attached */
495 if (rx_hdr->ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED)
496 iov[1].iov_len =
497 sizeof(struct iwarp_rdma_write);
498 else
499 iov[1].iov_len = sizeof(struct iwarp_send);
500
501 term->flag_m = 1;
502 term->flag_d = 1;
503 }
504 term->ctrl.mpa_len = cpu_to_be16(iov[1].iov_len);
505 break;
506
507 case TERM_ERROR_LAYER_DDP:
508 /* Report error encountered while DDP processing.
509 * This can only happen as a result of inbound
510 * DDP processing
511 */
512
513 /* Do not report DDP hdr information if packet
514 * layout is unknown
515 */
516 if (((qp->term_info.etype == DDP_ETYPE_TAGGED_BUF) &&
517 (qp->term_info.ecode == DDP_ECODE_T_VERSION)) ||
518 ((qp->term_info.etype == DDP_ETYPE_UNTAGGED_BUF) &&
519 (qp->term_info.ecode == DDP_ECODE_UT_VERSION)))
520 break;
521
522 iov[1].iov_base = rx_hdr;
523
524 if (rx_hdr->ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED)
525 iov[1].iov_len = sizeof(struct iwarp_ctrl_tagged);
526 else
527 iov[1].iov_len = sizeof(struct iwarp_ctrl_untagged);
528
529 term->flag_m = 1;
530 term->flag_d = 1;
531 break;
532
533 default:
534 break;
535 }
536 if (term->flag_m || term->flag_d || term->flag_r) {
537 iov[2].iov_base = &crc;
538 iov[2].iov_len = sizeof(crc);
539 len_terminate = sizeof(*term) + iov[1].iov_len + MPA_CRC_SIZE;
540 num_frags = 3;
541 } else {
542 iov[1].iov_base = &crc;
543 iov[1].iov_len = sizeof(crc);
544 len_terminate = sizeof(*term) + MPA_CRC_SIZE;
545 num_frags = 2;
546 }
547
548 /* Adjust DDP Segment Length parameter, if valid */
549 if (term->flag_m) {
550 u32 real_ddp_len = be16_to_cpu(rx_hdr->ctrl.mpa_len);
551 enum rdma_opcode op = __rdmap_get_opcode(&rx_hdr->ctrl);
552
553 real_ddp_len -= iwarp_pktinfo[op].hdr_len - MPA_HDR_SIZE;
554 rx_hdr->ctrl.mpa_len = cpu_to_be16(real_ddp_len);
555 }
556
557 term->ctrl.mpa_len =
558 cpu_to_be16(len_terminate - (MPA_HDR_SIZE + MPA_CRC_SIZE));
559 if (qp->tx_ctx.mpa_crc_enabled) {
560 siw_crc_init(&qp->tx_ctx.mpa_crc);
561 siw_crc_update(&qp->tx_ctx.mpa_crc,
562 iov[0].iov_base, iov[0].iov_len);
563 if (num_frags == 3) {
564 siw_crc_update(&qp->tx_ctx.mpa_crc,
565 iov[1].iov_base, iov[1].iov_len);
566 }
567 siw_crc_final(&qp->tx_ctx.mpa_crc, (u8 *)&crc);
568 }
569
570 rv = kernel_sendmsg(s, &msg, iov, num_frags, len_terminate);
571 siw_dbg_qp(qp, "sent TERM: %s, layer %d, type %d, code %d (%d bytes)\n",
572 rv == len_terminate ? "success" : "failure",
573 __rdmap_term_layer(term), __rdmap_term_etype(term),
574 __rdmap_term_ecode(term), rv);
575 kfree(term);
576 kfree(err_hdr);
577 }
578
579 /*
580 * Handle all attrs other than state
581 */
siw_qp_modify_nonstate(struct siw_qp * qp,struct siw_qp_attrs * attrs,enum siw_qp_attr_mask mask)582 static void siw_qp_modify_nonstate(struct siw_qp *qp,
583 struct siw_qp_attrs *attrs,
584 enum siw_qp_attr_mask mask)
585 {
586 if (mask & SIW_QP_ATTR_ACCESS_FLAGS) {
587 if (attrs->flags & SIW_RDMA_BIND_ENABLED)
588 qp->attrs.flags |= SIW_RDMA_BIND_ENABLED;
589 else
590 qp->attrs.flags &= ~SIW_RDMA_BIND_ENABLED;
591
592 if (attrs->flags & SIW_RDMA_WRITE_ENABLED)
593 qp->attrs.flags |= SIW_RDMA_WRITE_ENABLED;
594 else
595 qp->attrs.flags &= ~SIW_RDMA_WRITE_ENABLED;
596
597 if (attrs->flags & SIW_RDMA_READ_ENABLED)
598 qp->attrs.flags |= SIW_RDMA_READ_ENABLED;
599 else
600 qp->attrs.flags &= ~SIW_RDMA_READ_ENABLED;
601 }
602 }
603
siw_qp_nextstate_from_idle(struct siw_qp * qp,struct siw_qp_attrs * attrs,enum siw_qp_attr_mask mask)604 static int siw_qp_nextstate_from_idle(struct siw_qp *qp,
605 struct siw_qp_attrs *attrs,
606 enum siw_qp_attr_mask mask)
607 {
608 int rv = 0;
609
610 switch (attrs->state) {
611 case SIW_QP_STATE_RTS:
612 if (attrs->flags & SIW_MPA_CRC) {
613 siw_crc_init(&qp->tx_ctx.mpa_crc);
614 qp->tx_ctx.mpa_crc_enabled = true;
615 siw_crc_init(&qp->rx_stream.mpa_crc);
616 qp->rx_stream.mpa_crc_enabled = true;
617 }
618 if (!(mask & SIW_QP_ATTR_LLP_HANDLE)) {
619 siw_dbg_qp(qp, "no socket\n");
620 rv = -EINVAL;
621 break;
622 }
623 if (!(mask & SIW_QP_ATTR_MPA)) {
624 siw_dbg_qp(qp, "no MPA\n");
625 rv = -EINVAL;
626 break;
627 }
628 /*
629 * Initialize iWARP TX state
630 */
631 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_SEND] = 0;
632 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ] = 0;
633 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_TERMINATE] = 0;
634
635 /*
636 * Initialize iWARP RX state
637 */
638 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_SEND] = 1;
639 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ] = 1;
640 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_TERMINATE] = 1;
641
642 /*
643 * init IRD free queue, caller has already checked
644 * limits.
645 */
646 rv = siw_qp_readq_init(qp, attrs->irq_size,
647 attrs->orq_size);
648 if (rv)
649 break;
650
651 qp->attrs.sk = attrs->sk;
652 qp->attrs.state = SIW_QP_STATE_RTS;
653
654 siw_dbg_qp(qp, "enter RTS: crc=%s, ord=%u, ird=%u\n",
655 attrs->flags & SIW_MPA_CRC ? "y" : "n",
656 qp->attrs.orq_size, qp->attrs.irq_size);
657 break;
658
659 case SIW_QP_STATE_ERROR:
660 siw_rq_flush(qp);
661 qp->attrs.state = SIW_QP_STATE_ERROR;
662 if (qp->cep) {
663 siw_cep_put(qp->cep);
664 qp->cep = NULL;
665 }
666 break;
667
668 default:
669 break;
670 }
671 return rv;
672 }
673
siw_qp_nextstate_from_rts(struct siw_qp * qp,struct siw_qp_attrs * attrs)674 static int siw_qp_nextstate_from_rts(struct siw_qp *qp,
675 struct siw_qp_attrs *attrs)
676 {
677 int drop_conn = 0;
678
679 switch (attrs->state) {
680 case SIW_QP_STATE_CLOSING:
681 /*
682 * Verbs: move to IDLE if SQ and ORQ are empty.
683 * Move to ERROR otherwise. But first of all we must
684 * close the connection. So we keep CLOSING or ERROR
685 * as a transient state, schedule connection drop work
686 * and wait for the socket state change upcall to
687 * come back closed.
688 */
689 if (tx_wqe(qp)->wr_status == SIW_WR_IDLE) {
690 qp->attrs.state = SIW_QP_STATE_CLOSING;
691 } else {
692 qp->attrs.state = SIW_QP_STATE_ERROR;
693 siw_sq_flush(qp);
694 }
695 siw_rq_flush(qp);
696
697 drop_conn = 1;
698 break;
699
700 case SIW_QP_STATE_TERMINATE:
701 qp->attrs.state = SIW_QP_STATE_TERMINATE;
702
703 siw_init_terminate(qp, TERM_ERROR_LAYER_RDMAP,
704 RDMAP_ETYPE_CATASTROPHIC,
705 RDMAP_ECODE_UNSPECIFIED, 1);
706 drop_conn = 1;
707 break;
708
709 case SIW_QP_STATE_ERROR:
710 /*
711 * This is an emergency close.
712 *
713 * Any in progress transmit operation will get
714 * cancelled.
715 * This will likely result in a protocol failure,
716 * if a TX operation is in transit. The caller
717 * could unconditional wait to give the current
718 * operation a chance to complete.
719 * Esp., how to handle the non-empty IRQ case?
720 * The peer was asking for data transfer at a valid
721 * point in time.
722 */
723 siw_sq_flush(qp);
724 siw_rq_flush(qp);
725 qp->attrs.state = SIW_QP_STATE_ERROR;
726 drop_conn = 1;
727 break;
728
729 default:
730 break;
731 }
732 return drop_conn;
733 }
734
siw_qp_nextstate_from_term(struct siw_qp * qp,struct siw_qp_attrs * attrs)735 static void siw_qp_nextstate_from_term(struct siw_qp *qp,
736 struct siw_qp_attrs *attrs)
737 {
738 switch (attrs->state) {
739 case SIW_QP_STATE_ERROR:
740 siw_rq_flush(qp);
741 qp->attrs.state = SIW_QP_STATE_ERROR;
742
743 if (tx_wqe(qp)->wr_status != SIW_WR_IDLE)
744 siw_sq_flush(qp);
745 break;
746
747 default:
748 break;
749 }
750 }
751
siw_qp_nextstate_from_close(struct siw_qp * qp,struct siw_qp_attrs * attrs)752 static int siw_qp_nextstate_from_close(struct siw_qp *qp,
753 struct siw_qp_attrs *attrs)
754 {
755 int rv = 0;
756
757 switch (attrs->state) {
758 case SIW_QP_STATE_IDLE:
759 WARN_ON(tx_wqe(qp)->wr_status != SIW_WR_IDLE);
760 qp->attrs.state = SIW_QP_STATE_IDLE;
761 break;
762
763 case SIW_QP_STATE_CLOSING:
764 /*
765 * The LLP may already moved the QP to closing
766 * due to graceful peer close init
767 */
768 break;
769
770 case SIW_QP_STATE_ERROR:
771 /*
772 * QP was moved to CLOSING by LLP event
773 * not yet seen by user.
774 */
775 qp->attrs.state = SIW_QP_STATE_ERROR;
776
777 if (tx_wqe(qp)->wr_status != SIW_WR_IDLE)
778 siw_sq_flush(qp);
779
780 siw_rq_flush(qp);
781 break;
782
783 default:
784 siw_dbg_qp(qp, "state transition undefined: %s => %s\n",
785 siw_qp_state_to_string[qp->attrs.state],
786 siw_qp_state_to_string[attrs->state]);
787
788 rv = -ECONNABORTED;
789 }
790 return rv;
791 }
792
793 /*
794 * Caller must hold qp->state_lock
795 */
siw_qp_modify(struct siw_qp * qp,struct siw_qp_attrs * attrs,enum siw_qp_attr_mask mask)796 int siw_qp_modify(struct siw_qp *qp, struct siw_qp_attrs *attrs,
797 enum siw_qp_attr_mask mask)
798 {
799 int drop_conn = 0, rv = 0;
800
801 if (!mask)
802 return 0;
803
804 siw_dbg_qp(qp, "state: %s => %s\n",
805 siw_qp_state_to_string[qp->attrs.state],
806 siw_qp_state_to_string[attrs->state]);
807
808 if (mask != SIW_QP_ATTR_STATE)
809 siw_qp_modify_nonstate(qp, attrs, mask);
810
811 if (!(mask & SIW_QP_ATTR_STATE))
812 return 0;
813
814 switch (qp->attrs.state) {
815 case SIW_QP_STATE_IDLE:
816 case SIW_QP_STATE_RTR:
817 rv = siw_qp_nextstate_from_idle(qp, attrs, mask);
818 break;
819
820 case SIW_QP_STATE_RTS:
821 drop_conn = siw_qp_nextstate_from_rts(qp, attrs);
822 break;
823
824 case SIW_QP_STATE_TERMINATE:
825 siw_qp_nextstate_from_term(qp, attrs);
826 break;
827
828 case SIW_QP_STATE_CLOSING:
829 siw_qp_nextstate_from_close(qp, attrs);
830 break;
831 default:
832 break;
833 }
834 if (drop_conn)
835 siw_qp_cm_drop(qp, 0);
836
837 return rv;
838 }
839
siw_read_to_orq(struct siw_sqe * rreq,struct siw_sqe * sqe)840 void siw_read_to_orq(struct siw_sqe *rreq, struct siw_sqe *sqe)
841 {
842 rreq->id = sqe->id;
843 rreq->opcode = sqe->opcode;
844 rreq->sge[0].laddr = sqe->sge[0].laddr;
845 rreq->sge[0].length = sqe->sge[0].length;
846 rreq->sge[0].lkey = sqe->sge[0].lkey;
847 rreq->sge[1].lkey = sqe->sge[1].lkey;
848 rreq->flags = sqe->flags | SIW_WQE_VALID;
849 rreq->num_sge = 1;
850 }
851
siw_activate_tx_from_sq(struct siw_qp * qp)852 static int siw_activate_tx_from_sq(struct siw_qp *qp)
853 {
854 struct siw_sqe *sqe;
855 struct siw_wqe *wqe = tx_wqe(qp);
856 int rv = 1;
857
858 sqe = sq_get_next(qp);
859 if (!sqe)
860 return 0;
861
862 memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE);
863 wqe->wr_status = SIW_WR_QUEUED;
864
865 /* First copy SQE to kernel private memory */
866 memcpy(&wqe->sqe, sqe, sizeof(*sqe));
867
868 if (wqe->sqe.opcode >= SIW_NUM_OPCODES) {
869 rv = -EINVAL;
870 goto out;
871 }
872 if (wqe->sqe.flags & SIW_WQE_INLINE) {
873 if (wqe->sqe.opcode != SIW_OP_SEND &&
874 wqe->sqe.opcode != SIW_OP_WRITE) {
875 rv = -EINVAL;
876 goto out;
877 }
878 if (wqe->sqe.sge[0].length > SIW_MAX_INLINE) {
879 rv = -EINVAL;
880 goto out;
881 }
882 wqe->sqe.sge[0].laddr = (uintptr_t)&wqe->sqe.sge[1];
883 wqe->sqe.sge[0].lkey = 0;
884 wqe->sqe.num_sge = 1;
885 }
886 if (wqe->sqe.flags & SIW_WQE_READ_FENCE) {
887 /* A READ cannot be fenced */
888 if (unlikely(wqe->sqe.opcode == SIW_OP_READ ||
889 wqe->sqe.opcode ==
890 SIW_OP_READ_LOCAL_INV)) {
891 siw_dbg_qp(qp, "cannot fence read\n");
892 rv = -EINVAL;
893 goto out;
894 }
895 spin_lock(&qp->orq_lock);
896
897 if (qp->attrs.orq_size && !siw_orq_empty(qp)) {
898 qp->tx_ctx.orq_fence = 1;
899 rv = 0;
900 }
901 spin_unlock(&qp->orq_lock);
902
903 } else if (wqe->sqe.opcode == SIW_OP_READ ||
904 wqe->sqe.opcode == SIW_OP_READ_LOCAL_INV) {
905 struct siw_sqe *rreq;
906
907 if (unlikely(!qp->attrs.orq_size)) {
908 /* We negotiated not to send READ req's */
909 rv = -EINVAL;
910 goto out;
911 }
912 wqe->sqe.num_sge = 1;
913
914 spin_lock(&qp->orq_lock);
915
916 rreq = orq_get_free(qp);
917 if (rreq) {
918 /*
919 * Make an immediate copy in ORQ to be ready
920 * to process loopback READ reply
921 */
922 siw_read_to_orq(rreq, &wqe->sqe);
923 qp->orq_put++;
924 } else {
925 qp->tx_ctx.orq_fence = 1;
926 rv = 0;
927 }
928 spin_unlock(&qp->orq_lock);
929 }
930
931 /* Clear SQE, can be re-used by application */
932 smp_store_mb(sqe->flags, 0);
933 qp->sq_get++;
934 out:
935 if (unlikely(rv < 0)) {
936 siw_dbg_qp(qp, "error %d\n", rv);
937 wqe->wr_status = SIW_WR_IDLE;
938 }
939 return rv;
940 }
941
942 /*
943 * Must be called with SQ locked.
944 * To avoid complete SQ starvation by constant inbound READ requests,
945 * the active IRQ will not be served after qp->irq_burst, if the
946 * SQ has pending work.
947 */
siw_activate_tx(struct siw_qp * qp)948 int siw_activate_tx(struct siw_qp *qp)
949 {
950 struct siw_sqe *irqe;
951 struct siw_wqe *wqe = tx_wqe(qp);
952
953 if (!qp->attrs.irq_size)
954 return siw_activate_tx_from_sq(qp);
955
956 irqe = &qp->irq[qp->irq_get % qp->attrs.irq_size];
957
958 if (!(irqe->flags & SIW_WQE_VALID))
959 return siw_activate_tx_from_sq(qp);
960
961 /*
962 * Avoid local WQE processing starvation in case
963 * of constant inbound READ request stream
964 */
965 if (sq_get_next(qp) && ++qp->irq_burst >= SIW_IRQ_MAXBURST_SQ_ACTIVE) {
966 qp->irq_burst = 0;
967 return siw_activate_tx_from_sq(qp);
968 }
969 memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE);
970 wqe->wr_status = SIW_WR_QUEUED;
971
972 /* start READ RESPONSE */
973 wqe->sqe.opcode = SIW_OP_READ_RESPONSE;
974 wqe->sqe.flags = 0;
975 if (irqe->num_sge) {
976 wqe->sqe.num_sge = 1;
977 wqe->sqe.sge[0].length = irqe->sge[0].length;
978 wqe->sqe.sge[0].laddr = irqe->sge[0].laddr;
979 wqe->sqe.sge[0].lkey = irqe->sge[0].lkey;
980 } else {
981 wqe->sqe.num_sge = 0;
982 }
983
984 /* Retain original RREQ's message sequence number for
985 * potential error reporting cases.
986 */
987 wqe->sqe.sge[1].length = irqe->sge[1].length;
988
989 wqe->sqe.rkey = irqe->rkey;
990 wqe->sqe.raddr = irqe->raddr;
991
992 wqe->processed = 0;
993 qp->irq_get++;
994
995 /* mark current IRQ entry free */
996 smp_store_mb(irqe->flags, 0);
997
998 return 1;
999 }
1000
1001 /*
1002 * Check if current CQ state qualifies for calling CQ completion
1003 * handler. Must be called with CQ lock held.
1004 */
siw_cq_notify_now(struct siw_cq * cq,u32 flags)1005 static bool siw_cq_notify_now(struct siw_cq *cq, u32 flags)
1006 {
1007 u32 cq_notify;
1008
1009 if (!cq->base_cq.comp_handler)
1010 return false;
1011
1012 /* Read application shared notification state */
1013 cq_notify = READ_ONCE(cq->notify->flags);
1014
1015 if ((cq_notify & SIW_NOTIFY_NEXT_COMPLETION) ||
1016 ((cq_notify & SIW_NOTIFY_SOLICITED) &&
1017 (flags & SIW_WQE_SOLICITED))) {
1018 /*
1019 * CQ notification is one-shot: Since the
1020 * current CQE causes user notification,
1021 * the CQ gets dis-aremd and must be re-aremd
1022 * by the user for a new notification.
1023 */
1024 WRITE_ONCE(cq->notify->flags, SIW_NOTIFY_NOT);
1025
1026 return true;
1027 }
1028 return false;
1029 }
1030
siw_sqe_complete(struct siw_qp * qp,struct siw_sqe * sqe,u32 bytes,enum siw_wc_status status)1031 int siw_sqe_complete(struct siw_qp *qp, struct siw_sqe *sqe, u32 bytes,
1032 enum siw_wc_status status)
1033 {
1034 struct siw_cq *cq = qp->scq;
1035 int rv = 0;
1036
1037 if (cq) {
1038 u32 sqe_flags = sqe->flags;
1039 struct siw_cqe *cqe;
1040 u32 idx;
1041 unsigned long flags;
1042
1043 spin_lock_irqsave(&cq->lock, flags);
1044
1045 idx = cq->cq_put % cq->num_cqe;
1046 cqe = &cq->queue[idx];
1047
1048 if (!READ_ONCE(cqe->flags)) {
1049 bool notify;
1050
1051 cqe->id = sqe->id;
1052 cqe->opcode = sqe->opcode;
1053 cqe->status = status;
1054 cqe->imm_data = 0;
1055 cqe->bytes = bytes;
1056
1057 if (rdma_is_kernel_res(&cq->base_cq.res))
1058 cqe->base_qp = &qp->base_qp;
1059 else
1060 cqe->qp_id = qp_id(qp);
1061
1062 /* mark CQE valid for application */
1063 WRITE_ONCE(cqe->flags, SIW_WQE_VALID);
1064 /* recycle SQE */
1065 smp_store_mb(sqe->flags, 0);
1066
1067 cq->cq_put++;
1068 notify = siw_cq_notify_now(cq, sqe_flags);
1069
1070 spin_unlock_irqrestore(&cq->lock, flags);
1071
1072 if (notify) {
1073 siw_dbg_cq(cq, "Call completion handler\n");
1074 cq->base_cq.comp_handler(&cq->base_cq,
1075 cq->base_cq.cq_context);
1076 }
1077 } else {
1078 spin_unlock_irqrestore(&cq->lock, flags);
1079 rv = -ENOMEM;
1080 siw_cq_event(cq, IB_EVENT_CQ_ERR);
1081 }
1082 } else {
1083 /* recycle SQE */
1084 smp_store_mb(sqe->flags, 0);
1085 }
1086 return rv;
1087 }
1088
siw_rqe_complete(struct siw_qp * qp,struct siw_rqe * rqe,u32 bytes,u32 inval_stag,enum siw_wc_status status)1089 int siw_rqe_complete(struct siw_qp *qp, struct siw_rqe *rqe, u32 bytes,
1090 u32 inval_stag, enum siw_wc_status status)
1091 {
1092 struct siw_cq *cq = qp->rcq;
1093 int rv = 0;
1094
1095 if (cq) {
1096 struct siw_cqe *cqe;
1097 u32 idx;
1098 unsigned long flags;
1099
1100 spin_lock_irqsave(&cq->lock, flags);
1101
1102 idx = cq->cq_put % cq->num_cqe;
1103 cqe = &cq->queue[idx];
1104
1105 if (!READ_ONCE(cqe->flags)) {
1106 bool notify;
1107 u8 cqe_flags = SIW_WQE_VALID;
1108
1109 cqe->id = rqe->id;
1110 cqe->opcode = SIW_OP_RECEIVE;
1111 cqe->status = status;
1112 cqe->imm_data = 0;
1113 cqe->bytes = bytes;
1114
1115 if (rdma_is_kernel_res(&cq->base_cq.res)) {
1116 cqe->base_qp = &qp->base_qp;
1117 if (inval_stag) {
1118 cqe_flags |= SIW_WQE_REM_INVAL;
1119 cqe->inval_stag = inval_stag;
1120 }
1121 } else {
1122 cqe->qp_id = qp_id(qp);
1123 }
1124 /* mark CQE valid for application */
1125 WRITE_ONCE(cqe->flags, cqe_flags);
1126 /* recycle RQE */
1127 smp_store_mb(rqe->flags, 0);
1128
1129 cq->cq_put++;
1130 notify = siw_cq_notify_now(cq, SIW_WQE_SIGNALLED);
1131
1132 spin_unlock_irqrestore(&cq->lock, flags);
1133
1134 if (notify) {
1135 siw_dbg_cq(cq, "Call completion handler\n");
1136 cq->base_cq.comp_handler(&cq->base_cq,
1137 cq->base_cq.cq_context);
1138 }
1139 } else {
1140 spin_unlock_irqrestore(&cq->lock, flags);
1141 rv = -ENOMEM;
1142 siw_cq_event(cq, IB_EVENT_CQ_ERR);
1143 }
1144 } else {
1145 /* recycle RQE */
1146 smp_store_mb(rqe->flags, 0);
1147 }
1148 return rv;
1149 }
1150
1151 /*
1152 * siw_sq_flush()
1153 *
1154 * Flush SQ and ORQ entries to CQ.
1155 *
1156 * Must be called with QP state write lock held.
1157 * Therefore, SQ and ORQ lock must not be taken.
1158 */
siw_sq_flush(struct siw_qp * qp)1159 void siw_sq_flush(struct siw_qp *qp)
1160 {
1161 struct siw_sqe *sqe;
1162 struct siw_wqe *wqe = tx_wqe(qp);
1163 int async_event = 0;
1164
1165 /*
1166 * Start with completing any work currently on the ORQ
1167 */
1168 while (qp->attrs.orq_size) {
1169 sqe = &qp->orq[qp->orq_get % qp->attrs.orq_size];
1170 if (!READ_ONCE(sqe->flags))
1171 break;
1172
1173 if (siw_sqe_complete(qp, sqe, 0, SIW_WC_WR_FLUSH_ERR) != 0)
1174 break;
1175
1176 WRITE_ONCE(sqe->flags, 0);
1177 qp->orq_get++;
1178 }
1179 /*
1180 * Flush an in-progress WQE if present
1181 */
1182 if (wqe->wr_status != SIW_WR_IDLE) {
1183 siw_dbg_qp(qp, "flush current SQE, type %d, status %d\n",
1184 tx_type(wqe), wqe->wr_status);
1185
1186 siw_wqe_put_mem(wqe, tx_type(wqe));
1187
1188 if (tx_type(wqe) != SIW_OP_READ_RESPONSE &&
1189 ((tx_type(wqe) != SIW_OP_READ &&
1190 tx_type(wqe) != SIW_OP_READ_LOCAL_INV) ||
1191 wqe->wr_status == SIW_WR_QUEUED))
1192 /*
1193 * An in-progress Read Request is already in
1194 * the ORQ
1195 */
1196 siw_sqe_complete(qp, &wqe->sqe, wqe->bytes,
1197 SIW_WC_WR_FLUSH_ERR);
1198
1199 wqe->wr_status = SIW_WR_IDLE;
1200 }
1201 /*
1202 * Flush the Send Queue
1203 */
1204 while (qp->attrs.sq_size) {
1205 sqe = &qp->sendq[qp->sq_get % qp->attrs.sq_size];
1206 if (!READ_ONCE(sqe->flags))
1207 break;
1208
1209 async_event = 1;
1210 if (siw_sqe_complete(qp, sqe, 0, SIW_WC_WR_FLUSH_ERR) != 0)
1211 /*
1212 * Shall IB_EVENT_SQ_DRAINED be supressed if work
1213 * completion fails?
1214 */
1215 break;
1216
1217 WRITE_ONCE(sqe->flags, 0);
1218 qp->sq_get++;
1219 }
1220 if (async_event)
1221 siw_qp_event(qp, IB_EVENT_SQ_DRAINED);
1222 }
1223
1224 /*
1225 * siw_rq_flush()
1226 *
1227 * Flush recv queue entries to CQ. Also
1228 * takes care of pending active tagged and untagged
1229 * inbound transfers, which have target memory
1230 * referenced.
1231 *
1232 * Must be called with QP state write lock held.
1233 * Therefore, RQ lock must not be taken.
1234 */
siw_rq_flush(struct siw_qp * qp)1235 void siw_rq_flush(struct siw_qp *qp)
1236 {
1237 struct siw_wqe *wqe = &qp->rx_untagged.wqe_active;
1238
1239 /*
1240 * Flush an in-progress untagged operation if present
1241 */
1242 if (wqe->wr_status != SIW_WR_IDLE) {
1243 siw_dbg_qp(qp, "flush current rqe, type %d, status %d\n",
1244 rx_type(wqe), wqe->wr_status);
1245
1246 siw_wqe_put_mem(wqe, rx_type(wqe));
1247
1248 if (rx_type(wqe) == SIW_OP_RECEIVE) {
1249 siw_rqe_complete(qp, &wqe->rqe, wqe->bytes,
1250 0, SIW_WC_WR_FLUSH_ERR);
1251 } else if (rx_type(wqe) != SIW_OP_READ &&
1252 rx_type(wqe) != SIW_OP_READ_RESPONSE &&
1253 rx_type(wqe) != SIW_OP_WRITE) {
1254 siw_sqe_complete(qp, &wqe->sqe, 0, SIW_WC_WR_FLUSH_ERR);
1255 }
1256 wqe->wr_status = SIW_WR_IDLE;
1257 }
1258 wqe = &qp->rx_tagged.wqe_active;
1259
1260 if (wqe->wr_status != SIW_WR_IDLE) {
1261 siw_wqe_put_mem(wqe, rx_type(wqe));
1262 wqe->wr_status = SIW_WR_IDLE;
1263 }
1264 /*
1265 * Flush the Receive Queue
1266 */
1267 while (qp->attrs.rq_size) {
1268 struct siw_rqe *rqe =
1269 &qp->recvq[qp->rq_get % qp->attrs.rq_size];
1270
1271 if (!READ_ONCE(rqe->flags))
1272 break;
1273
1274 if (siw_rqe_complete(qp, rqe, 0, 0, SIW_WC_WR_FLUSH_ERR) != 0)
1275 break;
1276
1277 WRITE_ONCE(rqe->flags, 0);
1278 qp->rq_get++;
1279 }
1280 }
1281
siw_qp_add(struct siw_device * sdev,struct siw_qp * qp)1282 int siw_qp_add(struct siw_device *sdev, struct siw_qp *qp)
1283 {
1284 int rv = xa_alloc(&sdev->qp_xa, &qp->base_qp.qp_num, qp, xa_limit_32b,
1285 GFP_KERNEL);
1286
1287 if (!rv) {
1288 kref_init(&qp->ref);
1289 qp->sdev = sdev;
1290 siw_dbg_qp(qp, "new QP\n");
1291 }
1292 return rv;
1293 }
1294
siw_free_qp(struct kref * ref)1295 void siw_free_qp(struct kref *ref)
1296 {
1297 struct siw_qp *found, *qp = container_of(ref, struct siw_qp, ref);
1298 struct siw_device *sdev = qp->sdev;
1299 unsigned long flags;
1300
1301 if (qp->cep)
1302 siw_cep_put(qp->cep);
1303
1304 found = xa_erase(&sdev->qp_xa, qp_id(qp));
1305 WARN_ON(found != qp);
1306 spin_lock_irqsave(&sdev->lock, flags);
1307 list_del(&qp->devq);
1308 spin_unlock_irqrestore(&sdev->lock, flags);
1309
1310 vfree(qp->sendq);
1311 vfree(qp->recvq);
1312 vfree(qp->irq);
1313 vfree(qp->orq);
1314
1315 siw_put_tx_cpu(qp->tx_cpu);
1316 complete(&qp->qp_free);
1317 atomic_dec(&sdev->num_qp);
1318 }
1319