Lines Matching +full:send +full:- +full:flush +full:- +full:out +full:- +full:sequence
15 * - Redistributions of source code must retain the above
19 * - Redistributions in binary form must reproduce the above
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
47 len = ((psn - wqe->psn) & QIB_PSN_MASK) * pmtu; in restart_sge()
52 * qib_make_rc_ack - construct a response packet (ACK, NAK, or RDMA read)
71 /* Don't send an ACK if we aren't supposed to. */ in qib_make_rc_ack()
72 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) in qib_make_rc_ack()
75 /* header size in 32-bit words LRH+BTH = (8+12)/4. */ in qib_make_rc_ack()
78 switch (qp->s_ack_state) { in qib_make_rc_ack()
81 e = &qp->s_ack_queue[qp->s_tail_ack_queue]; in qib_make_rc_ack()
82 if (e->rdma_sge.mr) { in qib_make_rc_ack()
83 rvt_put_mr(e->rdma_sge.mr); in qib_make_rc_ack()
84 e->rdma_sge.mr = NULL; in qib_make_rc_ack()
93 if (++qp->s_tail_ack_queue > QIB_MAX_RDMA_ATOMIC) in qib_make_rc_ack()
94 qp->s_tail_ack_queue = 0; in qib_make_rc_ack()
99 if (qp->r_head_ack_queue == qp->s_tail_ack_queue) { in qib_make_rc_ack()
100 if (qp->s_flags & RVT_S_ACK_PENDING) in qib_make_rc_ack()
105 e = &qp->s_ack_queue[qp->s_tail_ack_queue]; in qib_make_rc_ack()
106 if (e->opcode == OP(RDMA_READ_REQUEST)) { in qib_make_rc_ack()
113 len = e->rdma_sge.sge_length; in qib_make_rc_ack()
114 if (len && !e->rdma_sge.mr) { in qib_make_rc_ack()
115 qp->s_tail_ack_queue = qp->r_head_ack_queue; in qib_make_rc_ack()
119 qp->s_rdma_mr = e->rdma_sge.mr; in qib_make_rc_ack()
120 if (qp->s_rdma_mr) in qib_make_rc_ack()
121 rvt_get_mr(qp->s_rdma_mr); in qib_make_rc_ack()
122 qp->s_ack_rdma_sge.sge = e->rdma_sge; in qib_make_rc_ack()
123 qp->s_ack_rdma_sge.num_sge = 1; in qib_make_rc_ack()
124 qp->s_cur_sge = &qp->s_ack_rdma_sge; in qib_make_rc_ack()
127 qp->s_ack_state = OP(RDMA_READ_RESPONSE_FIRST); in qib_make_rc_ack()
129 qp->s_ack_state = OP(RDMA_READ_RESPONSE_ONLY); in qib_make_rc_ack()
130 e->sent = 1; in qib_make_rc_ack()
132 ohdr->u.aeth = rvt_compute_aeth(qp); in qib_make_rc_ack()
134 qp->s_ack_rdma_psn = e->psn; in qib_make_rc_ack()
135 bth2 = qp->s_ack_rdma_psn++ & QIB_PSN_MASK; in qib_make_rc_ack()
138 qp->s_cur_sge = NULL; in qib_make_rc_ack()
140 qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE); in qib_make_rc_ack()
141 ohdr->u.at.aeth = rvt_compute_aeth(qp); in qib_make_rc_ack()
142 ib_u64_put(e->atomic_data, &ohdr->u.at.atomic_ack_eth); in qib_make_rc_ack()
143 hwords += sizeof(ohdr->u.at) / sizeof(u32); in qib_make_rc_ack()
144 bth2 = e->psn & QIB_PSN_MASK; in qib_make_rc_ack()
145 e->sent = 1; in qib_make_rc_ack()
147 bth0 = qp->s_ack_state << 24; in qib_make_rc_ack()
151 qp->s_ack_state = OP(RDMA_READ_RESPONSE_MIDDLE); in qib_make_rc_ack()
154 qp->s_cur_sge = &qp->s_ack_rdma_sge; in qib_make_rc_ack()
155 qp->s_rdma_mr = qp->s_ack_rdma_sge.sge.mr; in qib_make_rc_ack()
156 if (qp->s_rdma_mr) in qib_make_rc_ack()
157 rvt_get_mr(qp->s_rdma_mr); in qib_make_rc_ack()
158 len = qp->s_ack_rdma_sge.sge.sge_length; in qib_make_rc_ack()
162 ohdr->u.aeth = rvt_compute_aeth(qp); in qib_make_rc_ack()
164 qp->s_ack_state = OP(RDMA_READ_RESPONSE_LAST); in qib_make_rc_ack()
165 e = &qp->s_ack_queue[qp->s_tail_ack_queue]; in qib_make_rc_ack()
166 e->sent = 1; in qib_make_rc_ack()
168 bth0 = qp->s_ack_state << 24; in qib_make_rc_ack()
169 bth2 = qp->s_ack_rdma_psn++ & QIB_PSN_MASK; in qib_make_rc_ack()
175 * Send a regular ACK. in qib_make_rc_ack()
180 qp->s_ack_state = OP(SEND_ONLY); in qib_make_rc_ack()
181 qp->s_flags &= ~RVT_S_ACK_PENDING; in qib_make_rc_ack()
182 qp->s_cur_sge = NULL; in qib_make_rc_ack()
183 if (qp->s_nak_state) in qib_make_rc_ack()
184 ohdr->u.aeth = in qib_make_rc_ack()
185 cpu_to_be32((qp->r_msn & IB_MSN_MASK) | in qib_make_rc_ack()
186 (qp->s_nak_state << in qib_make_rc_ack()
189 ohdr->u.aeth = rvt_compute_aeth(qp); in qib_make_rc_ack()
193 bth2 = qp->s_ack_psn & QIB_PSN_MASK; in qib_make_rc_ack()
195 qp->s_rdma_ack_cnt++; in qib_make_rc_ack()
196 qp->s_hdrwords = hwords; in qib_make_rc_ack()
197 qp->s_cur_size = len; in qib_make_rc_ack()
202 qp->s_ack_state = OP(ACKNOWLEDGE); in qib_make_rc_ack()
203 qp->s_flags &= ~(RVT_S_RESP_PENDING | RVT_S_ACK_PENDING); in qib_make_rc_ack()
208 * qib_make_rc_req - construct a request packet (SEND, RDMA r/w, ATOMIC)
218 struct qib_qp_priv *priv = qp->priv; in qib_make_rc_req()
219 struct qib_ibdev *dev = to_idev(qp->ibqp.device); in qib_make_rc_req()
227 u32 pmtu = qp->pmtu; in qib_make_rc_req()
232 ohdr = &priv->s_hdr->u.oth; in qib_make_rc_req()
233 if (rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH) in qib_make_rc_req()
234 ohdr = &priv->s_hdr->u.l.oth; in qib_make_rc_req()
237 if ((qp->s_flags & RVT_S_RESP_PENDING) && in qib_make_rc_req()
241 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_SEND_OK)) { in qib_make_rc_req()
242 if (!(ib_rvt_state_ops[qp->state] & RVT_FLUSH_SEND)) in qib_make_rc_req()
244 /* We are in the error state, flush the work request. */ in qib_make_rc_req()
245 if (qp->s_last == READ_ONCE(qp->s_head)) in qib_make_rc_req()
247 /* If DMAs are in progress, we can't flush immediately. */ in qib_make_rc_req()
248 if (atomic_read(&priv->s_dma_busy)) { in qib_make_rc_req()
249 qp->s_flags |= RVT_S_WAIT_DMA; in qib_make_rc_req()
252 wqe = rvt_get_swqe_ptr(qp, qp->s_last); in qib_make_rc_req()
253 rvt_send_complete(qp, wqe, qp->s_last != qp->s_acked ? in qib_make_rc_req()
259 if (qp->s_flags & (RVT_S_WAIT_RNR | RVT_S_WAIT_ACK)) in qib_make_rc_req()
262 if (qib_cmp24(qp->s_psn, qp->s_sending_hpsn) <= 0) { in qib_make_rc_req()
263 if (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) { in qib_make_rc_req()
264 qp->s_flags |= RVT_S_WAIT_PSN; in qib_make_rc_req()
267 qp->s_sending_psn = qp->s_psn; in qib_make_rc_req()
268 qp->s_sending_hpsn = qp->s_psn - 1; in qib_make_rc_req()
271 /* header size in 32-bit words LRH+BTH = (8+12)/4. */ in qib_make_rc_req()
275 /* Send a request. */ in qib_make_rc_req()
276 wqe = rvt_get_swqe_ptr(qp, qp->s_cur); in qib_make_rc_req()
277 switch (qp->s_state) { in qib_make_rc_req()
279 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_NEXT_SEND_OK)) in qib_make_rc_req()
289 if (qp->s_cur == qp->s_tail) { in qib_make_rc_req()
290 /* Check if send work queue is empty. */ in qib_make_rc_req()
291 if (qp->s_tail == READ_ONCE(qp->s_head)) in qib_make_rc_req()
297 if ((wqe->wr.send_flags & IB_SEND_FENCE) && in qib_make_rc_req()
298 qp->s_num_rd_atomic) { in qib_make_rc_req()
299 qp->s_flags |= RVT_S_WAIT_FENCE; in qib_make_rc_req()
303 qp->s_psn = wqe->psn; in qib_make_rc_req()
310 len = wqe->length; in qib_make_rc_req()
311 ss = &qp->s_sge; in qib_make_rc_req()
312 bth2 = qp->s_psn & QIB_PSN_MASK; in qib_make_rc_req()
313 switch (wqe->wr.opcode) { in qib_make_rc_req()
320 qp->s_state = OP(SEND_FIRST); in qib_make_rc_req()
324 if (wqe->wr.opcode == IB_WR_SEND) in qib_make_rc_req()
325 qp->s_state = OP(SEND_ONLY); in qib_make_rc_req()
327 qp->s_state = OP(SEND_ONLY_WITH_IMMEDIATE); in qib_make_rc_req()
329 ohdr->u.imm_data = wqe->wr.ex.imm_data; in qib_make_rc_req()
332 if (wqe->wr.send_flags & IB_SEND_SOLICITED) in qib_make_rc_req()
335 if (++qp->s_cur == qp->s_size) in qib_make_rc_req()
336 qp->s_cur = 0; in qib_make_rc_req()
340 if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) in qib_make_rc_req()
341 qp->s_lsn++; in qib_make_rc_req()
348 ohdr->u.rc.reth.vaddr = in qib_make_rc_req()
349 cpu_to_be64(wqe->rdma_wr.remote_addr); in qib_make_rc_req()
350 ohdr->u.rc.reth.rkey = in qib_make_rc_req()
351 cpu_to_be32(wqe->rdma_wr.rkey); in qib_make_rc_req()
352 ohdr->u.rc.reth.length = cpu_to_be32(len); in qib_make_rc_req()
355 qp->s_state = OP(RDMA_WRITE_FIRST); in qib_make_rc_req()
359 if (wqe->rdma_wr.wr.opcode == IB_WR_RDMA_WRITE) in qib_make_rc_req()
360 qp->s_state = OP(RDMA_WRITE_ONLY); in qib_make_rc_req()
362 qp->s_state = OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE); in qib_make_rc_req()
364 ohdr->u.rc.imm_data = in qib_make_rc_req()
365 wqe->rdma_wr.wr.ex.imm_data; in qib_make_rc_req()
367 if (wqe->rdma_wr.wr.send_flags & IB_SEND_SOLICITED) in qib_make_rc_req()
371 if (++qp->s_cur == qp->s_size) in qib_make_rc_req()
372 qp->s_cur = 0; in qib_make_rc_req()
381 if (qp->s_num_rd_atomic >= in qib_make_rc_req()
382 qp->s_max_rd_atomic) { in qib_make_rc_req()
383 qp->s_flags |= RVT_S_WAIT_RDMAR; in qib_make_rc_req()
386 qp->s_num_rd_atomic++; in qib_make_rc_req()
387 if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) in qib_make_rc_req()
388 qp->s_lsn++; in qib_make_rc_req()
391 ohdr->u.rc.reth.vaddr = in qib_make_rc_req()
392 cpu_to_be64(wqe->rdma_wr.remote_addr); in qib_make_rc_req()
393 ohdr->u.rc.reth.rkey = in qib_make_rc_req()
394 cpu_to_be32(wqe->rdma_wr.rkey); in qib_make_rc_req()
395 ohdr->u.rc.reth.length = cpu_to_be32(len); in qib_make_rc_req()
396 qp->s_state = OP(RDMA_READ_REQUEST); in qib_make_rc_req()
397 hwords += sizeof(ohdr->u.rc.reth) / sizeof(u32); in qib_make_rc_req()
401 if (++qp->s_cur == qp->s_size) in qib_make_rc_req()
402 qp->s_cur = 0; in qib_make_rc_req()
412 if (qp->s_num_rd_atomic >= in qib_make_rc_req()
413 qp->s_max_rd_atomic) { in qib_make_rc_req()
414 qp->s_flags |= RVT_S_WAIT_RDMAR; in qib_make_rc_req()
417 qp->s_num_rd_atomic++; in qib_make_rc_req()
418 if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) in qib_make_rc_req()
419 qp->s_lsn++; in qib_make_rc_req()
421 if (wqe->atomic_wr.wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP) { in qib_make_rc_req()
422 qp->s_state = OP(COMPARE_SWAP); in qib_make_rc_req()
423 put_ib_ateth_swap(wqe->atomic_wr.swap, in qib_make_rc_req()
424 &ohdr->u.atomic_eth); in qib_make_rc_req()
425 put_ib_ateth_compare(wqe->atomic_wr.compare_add, in qib_make_rc_req()
426 &ohdr->u.atomic_eth); in qib_make_rc_req()
428 qp->s_state = OP(FETCH_ADD); in qib_make_rc_req()
429 put_ib_ateth_swap(wqe->atomic_wr.compare_add, in qib_make_rc_req()
430 &ohdr->u.atomic_eth); in qib_make_rc_req()
431 put_ib_ateth_compare(0, &ohdr->u.atomic_eth); in qib_make_rc_req()
433 put_ib_ateth_vaddr(wqe->atomic_wr.remote_addr, in qib_make_rc_req()
434 &ohdr->u.atomic_eth); in qib_make_rc_req()
435 ohdr->u.atomic_eth.rkey = cpu_to_be32( in qib_make_rc_req()
436 wqe->atomic_wr.rkey); in qib_make_rc_req()
441 if (++qp->s_cur == qp->s_size) in qib_make_rc_req()
442 qp->s_cur = 0; in qib_make_rc_req()
448 qp->s_sge.sge = wqe->sg_list[0]; in qib_make_rc_req()
449 qp->s_sge.sg_list = wqe->sg_list + 1; in qib_make_rc_req()
450 qp->s_sge.num_sge = wqe->wr.num_sge; in qib_make_rc_req()
451 qp->s_sge.total_len = wqe->length; in qib_make_rc_req()
452 qp->s_len = wqe->length; in qib_make_rc_req()
454 qp->s_tail++; in qib_make_rc_req()
455 if (qp->s_tail >= qp->s_size) in qib_make_rc_req()
456 qp->s_tail = 0; in qib_make_rc_req()
458 if (wqe->wr.opcode == IB_WR_RDMA_READ) in qib_make_rc_req()
459 qp->s_psn = wqe->lpsn + 1; in qib_make_rc_req()
461 qp->s_psn++; in qib_make_rc_req()
466 * qp->s_state is normally set to the opcode of the in qib_make_rc_req()
470 * thread to indicate a SEND needs to be restarted from an in qib_make_rc_req()
474 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); in qib_make_rc_req()
477 qp->s_state = OP(SEND_MIDDLE); in qib_make_rc_req()
480 bth2 = qp->s_psn++ & QIB_PSN_MASK; in qib_make_rc_req()
481 ss = &qp->s_sge; in qib_make_rc_req()
482 len = qp->s_len; in qib_make_rc_req()
487 if (wqe->wr.opcode == IB_WR_SEND) in qib_make_rc_req()
488 qp->s_state = OP(SEND_LAST); in qib_make_rc_req()
490 qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE); in qib_make_rc_req()
492 ohdr->u.imm_data = wqe->wr.ex.imm_data; in qib_make_rc_req()
495 if (wqe->wr.send_flags & IB_SEND_SOLICITED) in qib_make_rc_req()
498 qp->s_cur++; in qib_make_rc_req()
499 if (qp->s_cur >= qp->s_size) in qib_make_rc_req()
500 qp->s_cur = 0; in qib_make_rc_req()
505 * qp->s_state is normally set to the opcode of the in qib_make_rc_req()
513 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu); in qib_make_rc_req()
516 qp->s_state = OP(RDMA_WRITE_MIDDLE); in qib_make_rc_req()
519 bth2 = qp->s_psn++ & QIB_PSN_MASK; in qib_make_rc_req()
520 ss = &qp->s_sge; in qib_make_rc_req()
521 len = qp->s_len; in qib_make_rc_req()
526 if (wqe->wr.opcode == IB_WR_RDMA_WRITE) in qib_make_rc_req()
527 qp->s_state = OP(RDMA_WRITE_LAST); in qib_make_rc_req()
529 qp->s_state = OP(RDMA_WRITE_LAST_WITH_IMMEDIATE); in qib_make_rc_req()
531 ohdr->u.imm_data = wqe->wr.ex.imm_data; in qib_make_rc_req()
533 if (wqe->wr.send_flags & IB_SEND_SOLICITED) in qib_make_rc_req()
537 qp->s_cur++; in qib_make_rc_req()
538 if (qp->s_cur >= qp->s_size) in qib_make_rc_req()
539 qp->s_cur = 0; in qib_make_rc_req()
544 * qp->s_state is normally set to the opcode of the in qib_make_rc_req()
552 len = ((qp->s_psn - wqe->psn) & QIB_PSN_MASK) * pmtu; in qib_make_rc_req()
553 ohdr->u.rc.reth.vaddr = in qib_make_rc_req()
554 cpu_to_be64(wqe->rdma_wr.remote_addr + len); in qib_make_rc_req()
555 ohdr->u.rc.reth.rkey = in qib_make_rc_req()
556 cpu_to_be32(wqe->rdma_wr.rkey); in qib_make_rc_req()
557 ohdr->u.rc.reth.length = cpu_to_be32(wqe->length - len); in qib_make_rc_req()
558 qp->s_state = OP(RDMA_READ_REQUEST); in qib_make_rc_req()
559 hwords += sizeof(ohdr->u.rc.reth) / sizeof(u32); in qib_make_rc_req()
560 bth2 = (qp->s_psn & QIB_PSN_MASK) | IB_BTH_REQ_ACK; in qib_make_rc_req()
561 qp->s_psn = wqe->lpsn + 1; in qib_make_rc_req()
564 qp->s_cur++; in qib_make_rc_req()
565 if (qp->s_cur == qp->s_size) in qib_make_rc_req()
566 qp->s_cur = 0; in qib_make_rc_req()
569 qp->s_sending_hpsn = bth2; in qib_make_rc_req()
570 delta = (((int) bth2 - (int) wqe->psn) << 8) >> 8; in qib_make_rc_req()
573 if (qp->s_flags & RVT_S_SEND_ONE) { in qib_make_rc_req()
574 qp->s_flags &= ~RVT_S_SEND_ONE; in qib_make_rc_req()
575 qp->s_flags |= RVT_S_WAIT_ACK; in qib_make_rc_req()
578 qp->s_len -= len; in qib_make_rc_req()
579 qp->s_hdrwords = hwords; in qib_make_rc_req()
580 qp->s_cur_sge = ss; in qib_make_rc_req()
581 qp->s_cur_size = len; in qib_make_rc_req()
582 qib_make_ruc_header(qp, ohdr, bth0 | (qp->s_state << 24), bth2); in qib_make_rc_req()
586 qp->s_flags &= ~RVT_S_BUSY; in qib_make_rc_req()
591 * qib_send_rc_ack - Construct an ACK packet and send it
596 * send side QP state and tasklet.
600 struct qib_devdata *dd = dd_from_ibdev(qp->ibqp.device); in qib_send_rc_ack()
601 struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); in qib_send_rc_ack()
614 spin_lock_irqsave(&qp->s_lock, flags); in qib_send_rc_ack()
616 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) in qib_send_rc_ack()
619 /* Don't send ACK or NAK if a RDMA read or atomic is pending. */ in qib_send_rc_ack()
620 if ((qp->s_flags & RVT_S_RESP_PENDING) || qp->s_rdma_ack_cnt) in qib_send_rc_ack()
626 /* header size in 32-bit words LRH+BTH+AETH = (8+12+4)/4. */ in qib_send_rc_ack()
628 if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) & in qib_send_rc_ack()
631 rdma_ah_read_grh(&qp->remote_ah_attr), in qib_send_rc_ack()
637 bth0 = qib_get_pkey(ibp, qp->s_pkey_index) | (OP(ACKNOWLEDGE) << 24); in qib_send_rc_ack()
638 if (qp->s_mig_state == IB_MIG_MIGRATED) in qib_send_rc_ack()
640 if (qp->r_nak_state) in qib_send_rc_ack()
641 ohdr->u.aeth = cpu_to_be32((qp->r_msn & IB_MSN_MASK) | in qib_send_rc_ack()
642 (qp->r_nak_state << in qib_send_rc_ack()
645 ohdr->u.aeth = rvt_compute_aeth(qp); in qib_send_rc_ack()
646 lrh0 |= ibp->sl_to_vl[rdma_ah_get_sl(&qp->remote_ah_attr)] << 12 | in qib_send_rc_ack()
647 rdma_ah_get_sl(&qp->remote_ah_attr) << 4; in qib_send_rc_ack()
649 hdr.lrh[1] = cpu_to_be16(rdma_ah_get_dlid(&qp->remote_ah_attr)); in qib_send_rc_ack()
651 hdr.lrh[3] = cpu_to_be16(ppd->lid | in qib_send_rc_ack()
652 rdma_ah_get_path_bits(&qp->remote_ah_attr)); in qib_send_rc_ack()
653 ohdr->bth[0] = cpu_to_be32(bth0); in qib_send_rc_ack()
654 ohdr->bth[1] = cpu_to_be32(qp->remote_qpn); in qib_send_rc_ack()
655 ohdr->bth[2] = cpu_to_be32(qp->r_ack_psn & QIB_PSN_MASK); in qib_send_rc_ack()
657 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_send_rc_ack()
659 /* Don't try to send ACKs if the link isn't ACTIVE */ in qib_send_rc_ack()
660 if (!(ppd->lflags & QIBL_LINKACTIVE)) in qib_send_rc_ack()
663 control = dd->f_setpbc_control(ppd, hwords + SIZE_OF_CRC, in qib_send_rc_ack()
664 qp->s_srate, lrh0 >> 12); in qib_send_rc_ack()
668 piobuf = dd->f_getsendbuf(ppd, pbc, &pbufn); in qib_send_rc_ack()
671 * We are out of PIO buffers at the moment. in qib_send_rc_ack()
673 * send tasklet so that when a PIO buffer becomes in qib_send_rc_ack()
677 spin_lock_irqsave(&qp->s_lock, flags); in qib_send_rc_ack()
683 * We have to flush after the PBC for correctness in qib_send_rc_ack()
684 * on some cpus or WC buffer can be written out of order. in qib_send_rc_ack()
688 if (dd->flags & QIB_PIO_FLUSH_WC) { in qib_send_rc_ack()
692 qib_pio_copy(piobuf + 2, hdrp, hwords - 1); in qib_send_rc_ack()
694 __raw_writel(hdrp[hwords - 1], piobuf + hwords + 1); in qib_send_rc_ack()
698 if (dd->flags & QIB_USE_SPCL_TRIG) { in qib_send_rc_ack()
699 u32 spcl_off = (pbufn >= dd->piobcnt2k) ? 2047 : 1023; in qib_send_rc_ack()
708 this_cpu_inc(ibp->pmastats->n_unicast_xmit); in qib_send_rc_ack()
712 if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) { in qib_send_rc_ack()
713 this_cpu_inc(*ibp->rvp.rc_qacks); in qib_send_rc_ack()
714 qp->s_flags |= RVT_S_ACK_PENDING | RVT_S_RESP_PENDING; in qib_send_rc_ack()
715 qp->s_nak_state = qp->r_nak_state; in qib_send_rc_ack()
716 qp->s_ack_psn = qp->r_ack_psn; in qib_send_rc_ack()
718 /* Schedule the send tasklet. */ in qib_send_rc_ack()
722 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_send_rc_ack()
728 * reset_psn - reset the QP state to send starting from PSN
730 * @psn: the packet sequence number to restart at
738 u32 n = qp->s_acked; in reset_psn()
742 qp->s_cur = n; in reset_psn()
746 * let the normal send code handle initialization. in reset_psn()
748 if (qib_cmp24(psn, wqe->psn) <= 0) { in reset_psn()
749 qp->s_state = OP(SEND_LAST); in reset_psn()
754 opcode = wqe->wr.opcode; in reset_psn()
758 if (++n == qp->s_size) in reset_psn()
760 if (n == qp->s_tail) in reset_psn()
763 diff = qib_cmp24(psn, wqe->psn); in reset_psn()
766 qp->s_cur = n; in reset_psn()
769 * let the normal send code handle initialization. in reset_psn()
772 qp->s_state = OP(SEND_LAST); in reset_psn()
775 opcode = wqe->wr.opcode; in reset_psn()
786 qp->s_state = OP(RDMA_READ_RESPONSE_FIRST); in reset_psn()
791 qp->s_state = OP(RDMA_READ_RESPONSE_LAST); in reset_psn()
795 qp->s_state = OP(RDMA_READ_RESPONSE_MIDDLE); in reset_psn()
803 qp->s_state = OP(SEND_LAST); in reset_psn()
806 qp->s_psn = psn; in reset_psn()
809 * asynchronously before the send tasklet can get scheduled. in reset_psn()
812 if ((qib_cmp24(qp->s_psn, qp->s_sending_hpsn) <= 0) && in reset_psn()
813 (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)) in reset_psn()
814 qp->s_flags |= RVT_S_WAIT_PSN; in reset_psn()
818 * Back up requester to resend the last un-ACKed request.
823 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in qib_restart_rc()
826 if (qp->s_retry == 0) { in qib_restart_rc()
827 if (qp->s_mig_state == IB_MIG_ARMED) { in qib_restart_rc()
829 qp->s_retry = qp->s_retry_cnt; in qib_restart_rc()
830 } else if (qp->s_last == qp->s_acked) { in qib_restart_rc()
837 qp->s_retry--; in qib_restart_rc()
839 ibp = to_iport(qp->ibqp.device, qp->port_num); in qib_restart_rc()
840 if (wqe->wr.opcode == IB_WR_RDMA_READ) in qib_restart_rc()
841 ibp->rvp.n_rc_resends++; in qib_restart_rc()
843 ibp->rvp.n_rc_resends += (qp->s_psn - psn) & QIB_PSN_MASK; in qib_restart_rc()
845 qp->s_flags &= ~(RVT_S_WAIT_FENCE | RVT_S_WAIT_RDMAR | in qib_restart_rc()
849 qp->s_flags |= RVT_S_SEND_ONE; in qib_restart_rc()
854 * Set qp->s_sending_psn to the next PSN after the given one.
860 u32 n = qp->s_last; in reset_sending_psn()
865 if (qib_cmp24(psn, wqe->lpsn) <= 0) { in reset_sending_psn()
866 if (wqe->wr.opcode == IB_WR_RDMA_READ) in reset_sending_psn()
867 qp->s_sending_psn = wqe->lpsn + 1; in reset_sending_psn()
869 qp->s_sending_psn = psn + 1; in reset_sending_psn()
872 if (++n == qp->s_size) in reset_sending_psn()
874 if (n == qp->s_tail) in reset_sending_psn()
889 if (!(ib_rvt_state_ops[qp->state] & RVT_SEND_OR_FLUSH_OR_RECV_OK)) in qib_rc_send_complete()
892 /* Find out where the BTH is */ in qib_rc_send_complete()
893 if ((be16_to_cpu(hdr->lrh[0]) & 3) == QIB_LRH_BTH) in qib_rc_send_complete()
894 ohdr = &hdr->u.oth; in qib_rc_send_complete()
896 ohdr = &hdr->u.l.oth; in qib_rc_send_complete()
898 opcode = be32_to_cpu(ohdr->bth[0]) >> 24; in qib_rc_send_complete()
901 WARN_ON(!qp->s_rdma_ack_cnt); in qib_rc_send_complete()
902 qp->s_rdma_ack_cnt--; in qib_rc_send_complete()
906 psn = be32_to_cpu(ohdr->bth[2]); in qib_rc_send_complete()
913 if ((psn & IB_BTH_REQ_ACK) && qp->s_acked != qp->s_tail && in qib_rc_send_complete()
914 !(qp->s_flags & (RVT_S_TIMER | RVT_S_WAIT_RNR | RVT_S_WAIT_PSN)) && in qib_rc_send_complete()
915 (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) in qib_rc_send_complete()
918 while (qp->s_last != qp->s_acked) { in qib_rc_send_complete()
919 wqe = rvt_get_swqe_ptr(qp, qp->s_last); in qib_rc_send_complete()
920 if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) >= 0 && in qib_rc_send_complete()
921 qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) in qib_rc_send_complete()
925 ib_qib_wc_opcode[wqe->wr.opcode], in qib_rc_send_complete()
932 if (qp->s_flags & RVT_S_WAIT_PSN && in qib_rc_send_complete()
933 qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0) { in qib_rc_send_complete()
934 qp->s_flags &= ~RVT_S_WAIT_PSN; in qib_rc_send_complete()
935 qp->s_sending_psn = qp->s_psn; in qib_rc_send_complete()
936 qp->s_sending_hpsn = qp->s_psn - 1; in qib_rc_send_complete()
943 qp->s_last_psn = psn; in update_last_psn()
957 * completion if the SWQE is being resent until the send in do_rc_completion()
960 if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) < 0 || in do_rc_completion()
961 qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0) in do_rc_completion()
964 ib_qib_wc_opcode[wqe->wr.opcode], in do_rc_completion()
967 this_cpu_inc(*ibp->rvp.rc_delayed_comp); in do_rc_completion()
969 qp->s_retry = qp->s_retry_cnt; in do_rc_completion()
970 update_last_psn(qp, wqe->lpsn); in do_rc_completion()
977 if (qp->s_acked == qp->s_cur) { in do_rc_completion()
978 if (++qp->s_cur >= qp->s_size) in do_rc_completion()
979 qp->s_cur = 0; in do_rc_completion()
980 qp->s_acked = qp->s_cur; in do_rc_completion()
981 wqe = rvt_get_swqe_ptr(qp, qp->s_cur); in do_rc_completion()
982 if (qp->s_acked != qp->s_tail) { in do_rc_completion()
983 qp->s_state = OP(SEND_LAST); in do_rc_completion()
984 qp->s_psn = wqe->psn; in do_rc_completion()
987 if (++qp->s_acked >= qp->s_size) in do_rc_completion()
988 qp->s_acked = 0; in do_rc_completion()
989 if (qp->state == IB_QPS_SQD && qp->s_acked == qp->s_cur) in do_rc_completion()
990 qp->s_draining = 0; in do_rc_completion()
991 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in do_rc_completion()
997 * do_rc_ack - process an incoming RC ACK
999 * @psn: the packet sequence number of the ACK
1018 * Note that NAKs implicitly ACK outstanding SEND and RDMA write in do_rc_ack()
1025 ack_psn--; in do_rc_ack()
1026 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in do_rc_ack()
1027 ibp = to_iport(qp->ibqp.device, qp->port_num); in do_rc_ack()
1033 while ((diff = qib_cmp24(ack_psn, wqe->lpsn)) >= 0) { in do_rc_ack()
1040 if (wqe->wr.opcode == IB_WR_RDMA_READ && in do_rc_ack()
1055 if ((wqe->wr.opcode == IB_WR_RDMA_READ && in do_rc_ack()
1057 ((wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || in do_rc_ack()
1058 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) && in do_rc_ack()
1061 if (!(qp->r_flags & RVT_R_RDMAR_SEQ)) { in do_rc_ack()
1062 qp->r_flags |= RVT_R_RDMAR_SEQ; in do_rc_ack()
1063 qib_restart_rc(qp, qp->s_last_psn + 1, 0); in do_rc_ack()
1064 if (list_empty(&qp->rspwait)) { in do_rc_ack()
1065 qp->r_flags |= RVT_R_RSP_SEND; in do_rc_ack()
1067 list_add_tail(&qp->rspwait, in do_rc_ack()
1068 &rcd->qp_wait_list); in do_rc_ack()
1077 if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || in do_rc_ack()
1078 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) { in do_rc_ack()
1079 u64 *vaddr = wqe->sg_list[0].vaddr; in do_rc_ack()
1082 if (qp->s_num_rd_atomic && in do_rc_ack()
1083 (wqe->wr.opcode == IB_WR_RDMA_READ || in do_rc_ack()
1084 wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || in do_rc_ack()
1085 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)) { in do_rc_ack()
1086 qp->s_num_rd_atomic--; in do_rc_ack()
1088 if ((qp->s_flags & RVT_S_WAIT_FENCE) && in do_rc_ack()
1089 !qp->s_num_rd_atomic) { in do_rc_ack()
1090 qp->s_flags &= ~(RVT_S_WAIT_FENCE | in do_rc_ack()
1093 } else if (qp->s_flags & RVT_S_WAIT_RDMAR) { in do_rc_ack()
1094 qp->s_flags &= ~(RVT_S_WAIT_RDMAR | in do_rc_ack()
1100 if (qp->s_acked == qp->s_tail) in do_rc_ack()
1106 this_cpu_inc(*ibp->rvp.rc_acks); in do_rc_ack()
1107 if (qp->s_acked != qp->s_tail) { in do_rc_ack()
1117 if (qib_cmp24(qp->s_psn, psn) <= 0) in do_rc_ack()
1120 /* No more acks - kill all timers */ in do_rc_ack()
1122 if (qib_cmp24(qp->s_psn, psn) <= 0) { in do_rc_ack()
1123 qp->s_state = OP(SEND_LAST); in do_rc_ack()
1124 qp->s_psn = psn + 1; in do_rc_ack()
1127 if (qp->s_flags & RVT_S_WAIT_ACK) { in do_rc_ack()
1128 qp->s_flags &= ~RVT_S_WAIT_ACK; in do_rc_ack()
1132 qp->s_rnr_retry = qp->s_rnr_retry_cnt; in do_rc_ack()
1133 qp->s_retry = qp->s_retry_cnt; in do_rc_ack()
1138 ibp->rvp.n_rnr_naks++; in do_rc_ack()
1139 if (qp->s_acked == qp->s_tail) in do_rc_ack()
1141 if (qp->s_flags & RVT_S_WAIT_RNR) in do_rc_ack()
1143 if (qp->s_rnr_retry == 0) { in do_rc_ack()
1147 if (qp->s_rnr_retry_cnt < 7) in do_rc_ack()
1148 qp->s_rnr_retry--; in do_rc_ack()
1151 update_last_psn(qp, psn - 1); in do_rc_ack()
1153 ibp->rvp.n_rc_resends += (qp->s_psn - psn) & QIB_PSN_MASK; in do_rc_ack()
1157 qp->s_flags &= ~(RVT_S_WAIT_SSN_CREDIT | RVT_S_WAIT_ACK); in do_rc_ack()
1163 if (qp->s_acked == qp->s_tail) in do_rc_ack()
1166 update_last_psn(qp, psn - 1); in do_rc_ack()
1169 case 0: /* PSN sequence error */ in do_rc_ack()
1170 ibp->rvp.n_seq_naks++; in do_rc_ack()
1183 ibp->rvp.n_other_naks++; in do_rc_ack()
1188 ibp->rvp.n_other_naks++; in do_rc_ack()
1193 ibp->rvp.n_other_naks++; in do_rc_ack()
1195 if (qp->s_last == qp->s_acked) { in do_rc_ack()
1205 qp->s_retry = qp->s_retry_cnt; in do_rc_ack()
1206 qp->s_rnr_retry = qp->s_rnr_retry_cnt; in do_rc_ack()
1221 * We have seen an out of sequence RDMA read middle or last packet.
1232 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in rdma_seq_err()
1234 while (qib_cmp24(psn, wqe->lpsn) > 0) { in rdma_seq_err()
1235 if (wqe->wr.opcode == IB_WR_RDMA_READ || in rdma_seq_err()
1236 wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP || in rdma_seq_err()
1237 wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) in rdma_seq_err()
1242 ibp->rvp.n_rdma_seq++; in rdma_seq_err()
1243 qp->r_flags |= RVT_R_RDMAR_SEQ; in rdma_seq_err()
1244 qib_restart_rc(qp, qp->s_last_psn + 1, 0); in rdma_seq_err()
1245 if (list_empty(&qp->rspwait)) { in rdma_seq_err()
1246 qp->r_flags |= RVT_R_RSP_SEND; in rdma_seq_err()
1248 list_add_tail(&qp->rspwait, &rcd->qp_wait_list); in rdma_seq_err()
1253 * qib_rc_rcv_resp - process an incoming RC response packet
1260 * @psn: the packet sequence number for this packet
1291 if ((qib_cmp24(psn, qp->s_sending_psn) >= 0) && in qib_rc_rcv_resp()
1292 (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)) { in qib_rc_rcv_resp()
1295 * If send tasklet not running attempt to progress in qib_rc_rcv_resp()
1298 if (!(qp->s_flags & RVT_S_BUSY)) { in qib_rc_rcv_resp()
1300 spin_lock_irqsave(&ppd->sdma_lock, flags); in qib_rc_rcv_resp()
1304 spin_unlock_irqrestore(&ppd->sdma_lock, flags); in qib_rc_rcv_resp()
1309 spin_lock_irqsave(&qp->s_lock, flags); in qib_rc_rcv_resp()
1310 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK)) in qib_rc_rcv_resp()
1314 if (qib_cmp24(psn, READ_ONCE(qp->s_next_psn)) >= 0) in qib_rc_rcv_resp()
1318 diff = qib_cmp24(psn, qp->s_last_psn); in qib_rc_rcv_resp()
1322 aeth = be32_to_cpu(ohdr->u.aeth); in qib_rc_rcv_resp()
1333 if (qp->r_flags & RVT_R_RDMAR_SEQ) { in qib_rc_rcv_resp()
1334 if (qib_cmp24(psn, qp->s_last_psn + 1) != 0) in qib_rc_rcv_resp()
1336 qp->r_flags &= ~RVT_R_RDMAR_SEQ; in qib_rc_rcv_resp()
1339 if (unlikely(qp->s_acked == qp->s_tail)) in qib_rc_rcv_resp()
1341 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in qib_rc_rcv_resp()
1348 aeth = be32_to_cpu(ohdr->u.aeth); in qib_rc_rcv_resp()
1350 val = ib_u64_get(&ohdr->u.at.atomic_ack_eth); in qib_rc_rcv_resp()
1357 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in qib_rc_rcv_resp()
1358 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) in qib_rc_rcv_resp()
1365 qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge, in qib_rc_rcv_resp()
1371 if (unlikely(qib_cmp24(psn, qp->s_last_psn + 1))) in qib_rc_rcv_resp()
1373 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) in qib_rc_rcv_resp()
1378 if (unlikely(pmtu >= qp->s_rdma_read_len)) in qib_rc_rcv_resp()
1383 * 4.096 usec. * (1 << qp->timeout) in qib_rc_rcv_resp()
1386 if (qp->s_flags & RVT_S_WAIT_ACK) { in qib_rc_rcv_resp()
1387 qp->s_flags &= ~RVT_S_WAIT_ACK; in qib_rc_rcv_resp()
1392 qp->s_retry = qp->s_retry_cnt; in qib_rc_rcv_resp()
1398 qp->s_rdma_read_len -= pmtu; in qib_rc_rcv_resp()
1400 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_rc_rcv_resp()
1401 rvt_copy_sge(qp, &qp->s_rdma_read_sge, in qib_rc_rcv_resp()
1406 aeth = be32_to_cpu(ohdr->u.aeth); in qib_rc_rcv_resp()
1410 pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; in qib_rc_rcv_resp()
1423 wqe = rvt_get_swqe_ptr(qp, qp->s_acked); in qib_rc_rcv_resp()
1424 qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge, in qib_rc_rcv_resp()
1430 if (unlikely(qib_cmp24(psn, qp->s_last_psn + 1))) in qib_rc_rcv_resp()
1432 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ)) in qib_rc_rcv_resp()
1435 pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; in qib_rc_rcv_resp()
1444 tlen -= hdrsize + pad + 8; in qib_rc_rcv_resp()
1445 if (unlikely(tlen != qp->s_rdma_read_len)) in qib_rc_rcv_resp()
1447 aeth = be32_to_cpu(ohdr->u.aeth); in qib_rc_rcv_resp()
1448 rvt_copy_sge(qp, &qp->s_rdma_read_sge, in qib_rc_rcv_resp()
1450 WARN_ON(qp->s_rdma_read_sge.num_sge); in qib_rc_rcv_resp()
1467 if (qp->s_last == qp->s_acked) { in qib_rc_rcv_resp()
1472 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_rc_rcv_resp()
1478 * qib_rc_rcv_error - process an incoming duplicate or error RC packet
1483 * @psn: the packet sequence number for this packet
1501 struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num); in qib_rc_rcv_error()
1509 * Packet sequence error. in qib_rc_rcv_error()
1513 if (!qp->r_nak_state) { in qib_rc_rcv_error()
1514 ibp->rvp.n_rc_seqnak++; in qib_rc_rcv_error()
1515 qp->r_nak_state = IB_NAK_PSN_ERROR; in qib_rc_rcv_error()
1517 qp->r_ack_psn = qp->r_psn; in qib_rc_rcv_error()
1519 * Wait to send the sequence NAK until all packets in qib_rc_rcv_error()
1523 if (list_empty(&qp->rspwait)) { in qib_rc_rcv_error()
1524 qp->r_flags |= RVT_R_RSP_NAK; in qib_rc_rcv_error()
1526 list_add_tail(&qp->rspwait, &rcd->qp_wait_list); in qib_rc_rcv_error()
1533 * Handle a duplicate request. Don't re-execute SEND, RDMA in qib_rc_rcv_error()
1540 * send the earliest so that RDMA reads can be restarted at in qib_rc_rcv_error()
1550 ibp->rvp.n_rc_dupreq++; in qib_rc_rcv_error()
1552 spin_lock_irqsave(&qp->s_lock, flags); in qib_rc_rcv_error()
1554 for (i = qp->r_head_ack_queue; ; i = prev) { in qib_rc_rcv_error()
1555 if (i == qp->s_tail_ack_queue) in qib_rc_rcv_error()
1558 prev = i - 1; in qib_rc_rcv_error()
1561 if (prev == qp->r_head_ack_queue) { in qib_rc_rcv_error()
1565 e = &qp->s_ack_queue[prev]; in qib_rc_rcv_error()
1566 if (!e->opcode) { in qib_rc_rcv_error()
1570 if (qib_cmp24(psn, e->psn) >= 0) { in qib_rc_rcv_error()
1571 if (prev == qp->s_tail_ack_queue && in qib_rc_rcv_error()
1572 qib_cmp24(psn, e->lpsn) <= 0) in qib_rc_rcv_error()
1587 if (!e || e->opcode != OP(RDMA_READ_REQUEST)) in qib_rc_rcv_error()
1590 reth = &ohdr->u.rc.reth; in qib_rc_rcv_error()
1598 offset = ((psn - e->psn) & QIB_PSN_MASK) * in qib_rc_rcv_error()
1599 qp->pmtu; in qib_rc_rcv_error()
1600 len = be32_to_cpu(reth->length); in qib_rc_rcv_error()
1601 if (unlikely(offset + len != e->rdma_sge.sge_length)) in qib_rc_rcv_error()
1603 if (e->rdma_sge.mr) { in qib_rc_rcv_error()
1604 rvt_put_mr(e->rdma_sge.mr); in qib_rc_rcv_error()
1605 e->rdma_sge.mr = NULL; in qib_rc_rcv_error()
1608 u32 rkey = be32_to_cpu(reth->rkey); in qib_rc_rcv_error()
1609 u64 vaddr = be64_to_cpu(reth->vaddr); in qib_rc_rcv_error()
1612 ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr, rkey, in qib_rc_rcv_error()
1617 e->rdma_sge.vaddr = NULL; in qib_rc_rcv_error()
1618 e->rdma_sge.length = 0; in qib_rc_rcv_error()
1619 e->rdma_sge.sge_length = 0; in qib_rc_rcv_error()
1621 e->psn = psn; in qib_rc_rcv_error()
1624 qp->s_tail_ack_queue = prev; in qib_rc_rcv_error()
1632 * or the send tasklet is already backed up to send an in qib_rc_rcv_error()
1635 if (!e || e->opcode != (u8) opcode || old_req) in qib_rc_rcv_error()
1637 qp->s_tail_ack_queue = prev; in qib_rc_rcv_error()
1652 if (i == qp->r_head_ack_queue) { in qib_rc_rcv_error()
1653 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_rc_rcv_error()
1654 qp->r_nak_state = 0; in qib_rc_rcv_error()
1655 qp->r_ack_psn = qp->r_psn - 1; in qib_rc_rcv_error()
1659 * Try to send a simple ACK to work around a Mellanox bug in qib_rc_rcv_error()
1663 if (!(qp->s_flags & RVT_S_RESP_PENDING)) { in qib_rc_rcv_error()
1664 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_rc_rcv_error()
1665 qp->r_nak_state = 0; in qib_rc_rcv_error()
1666 qp->r_ack_psn = qp->s_ack_queue[i].psn - 1; in qib_rc_rcv_error()
1673 qp->s_tail_ack_queue = i; in qib_rc_rcv_error()
1676 qp->s_ack_state = OP(ACKNOWLEDGE); in qib_rc_rcv_error()
1677 qp->s_flags |= RVT_S_RESP_PENDING; in qib_rc_rcv_error()
1678 qp->r_nak_state = 0; in qib_rc_rcv_error()
1682 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_rc_rcv_error()
1697 qp->s_tail_ack_queue = next; in qib_update_ack_queue()
1698 qp->s_ack_state = OP(ACKNOWLEDGE); in qib_update_ack_queue()
1702 * qib_rc_rcv - process an incoming RC packet
1717 struct qib_ibport *ibp = &rcd->ppd->ibport_data; in qib_rc_rcv()
1724 u32 pmtu = qp->pmtu; in qib_rc_rcv()
1732 ohdr = &hdr->u.oth; in qib_rc_rcv()
1735 ohdr = &hdr->u.l.oth; in qib_rc_rcv()
1739 opcode = be32_to_cpu(ohdr->bth[0]); in qib_rc_rcv()
1743 psn = be32_to_cpu(ohdr->bth[2]); in qib_rc_rcv()
1748 * packet sequence number will be for something in the send work in qib_rc_rcv()
1749 * queue rather than the expected receive packet sequence number. in qib_rc_rcv()
1760 diff = qib_cmp24(psn, qp->r_psn); in qib_rc_rcv()
1767 /* Check for opcode sequence errors. */ in qib_rc_rcv()
1768 switch (qp->r_state) { in qib_rc_rcv()
1794 * Note that it is up to the requester to not send a new in qib_rc_rcv()
1801 if (qp->state == IB_QPS_RTR && !(qp->r_flags & RVT_R_COMM_EST)) in qib_rc_rcv()
1812 qp->r_rcv_len = 0; in qib_rc_rcv()
1820 qp->r_rcv_len += pmtu; in qib_rc_rcv()
1821 if (unlikely(qp->r_rcv_len > qp->r_len)) in qib_rc_rcv()
1823 rvt_copy_sge(qp, &qp->r_sge, data, pmtu, true, false); in qib_rc_rcv()
1842 qp->r_rcv_len = 0; in qib_rc_rcv()
1848 wc.ex.imm_data = ohdr->u.imm_data; in qib_rc_rcv()
1859 pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3; in qib_rc_rcv()
1865 tlen -= (hdrsize + pad + 4); in qib_rc_rcv()
1866 wc.byte_len = tlen + qp->r_rcv_len; in qib_rc_rcv()
1867 if (unlikely(wc.byte_len > qp->r_len)) in qib_rc_rcv()
1869 rvt_copy_sge(qp, &qp->r_sge, data, tlen, true, false); in qib_rc_rcv()
1870 rvt_put_ss(&qp->r_sge); in qib_rc_rcv()
1871 qp->r_msn++; in qib_rc_rcv()
1872 if (!test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags)) in qib_rc_rcv()
1874 wc.wr_id = qp->r_wr_id; in qib_rc_rcv()
1881 wc.qp = &qp->ibqp; in qib_rc_rcv()
1882 wc.src_qp = qp->remote_qpn; in qib_rc_rcv()
1883 wc.slid = rdma_ah_get_dlid(&qp->remote_ah_attr); in qib_rc_rcv()
1884 wc.sl = rdma_ah_get_sl(&qp->remote_ah_attr); in qib_rc_rcv()
1897 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE))) in qib_rc_rcv()
1900 reth = &ohdr->u.rc.reth; in qib_rc_rcv()
1902 qp->r_len = be32_to_cpu(reth->length); in qib_rc_rcv()
1903 qp->r_rcv_len = 0; in qib_rc_rcv()
1904 qp->r_sge.sg_list = NULL; in qib_rc_rcv()
1905 if (qp->r_len != 0) { in qib_rc_rcv()
1906 u32 rkey = be32_to_cpu(reth->rkey); in qib_rc_rcv()
1907 u64 vaddr = be64_to_cpu(reth->vaddr); in qib_rc_rcv()
1911 ok = rvt_rkey_ok(qp, &qp->r_sge.sge, qp->r_len, vaddr, in qib_rc_rcv()
1915 qp->r_sge.num_sge = 1; in qib_rc_rcv()
1917 qp->r_sge.num_sge = 0; in qib_rc_rcv()
1918 qp->r_sge.sge.mr = NULL; in qib_rc_rcv()
1919 qp->r_sge.sge.vaddr = NULL; in qib_rc_rcv()
1920 qp->r_sge.sge.length = 0; in qib_rc_rcv()
1921 qp->r_sge.sge.sge_length = 0; in qib_rc_rcv()
1931 rvt_put_ss(&qp->r_sge); in qib_rc_rcv()
1934 wc.ex.imm_data = ohdr->u.rc.imm_data; in qib_rc_rcv()
1944 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ))) in qib_rc_rcv()
1946 next = qp->r_head_ack_queue + 1; in qib_rc_rcv()
1950 spin_lock_irqsave(&qp->s_lock, flags); in qib_rc_rcv()
1951 if (unlikely(next == qp->s_tail_ack_queue)) { in qib_rc_rcv()
1952 if (!qp->s_ack_queue[next].sent) in qib_rc_rcv()
1956 e = &qp->s_ack_queue[qp->r_head_ack_queue]; in qib_rc_rcv()
1957 if (e->opcode == OP(RDMA_READ_REQUEST) && e->rdma_sge.mr) { in qib_rc_rcv()
1958 rvt_put_mr(e->rdma_sge.mr); in qib_rc_rcv()
1959 e->rdma_sge.mr = NULL; in qib_rc_rcv()
1961 reth = &ohdr->u.rc.reth; in qib_rc_rcv()
1962 len = be32_to_cpu(reth->length); in qib_rc_rcv()
1964 u32 rkey = be32_to_cpu(reth->rkey); in qib_rc_rcv()
1965 u64 vaddr = be64_to_cpu(reth->vaddr); in qib_rc_rcv()
1969 ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr, in qib_rc_rcv()
1977 qp->r_psn += rvt_div_mtu(qp, len - 1); in qib_rc_rcv()
1979 e->rdma_sge.mr = NULL; in qib_rc_rcv()
1980 e->rdma_sge.vaddr = NULL; in qib_rc_rcv()
1981 e->rdma_sge.length = 0; in qib_rc_rcv()
1982 e->rdma_sge.sge_length = 0; in qib_rc_rcv()
1984 e->opcode = opcode; in qib_rc_rcv()
1985 e->sent = 0; in qib_rc_rcv()
1986 e->psn = psn; in qib_rc_rcv()
1987 e->lpsn = qp->r_psn; in qib_rc_rcv()
1993 qp->r_msn++; in qib_rc_rcv()
1994 qp->r_psn++; in qib_rc_rcv()
1995 qp->r_state = opcode; in qib_rc_rcv()
1996 qp->r_nak_state = 0; in qib_rc_rcv()
1997 qp->r_head_ack_queue = next; in qib_rc_rcv()
1999 /* Schedule the send tasklet. */ in qib_rc_rcv()
2000 qp->s_flags |= RVT_S_RESP_PENDING; in qib_rc_rcv()
2016 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC))) in qib_rc_rcv()
2018 next = qp->r_head_ack_queue + 1; in qib_rc_rcv()
2021 spin_lock_irqsave(&qp->s_lock, flags); in qib_rc_rcv()
2022 if (unlikely(next == qp->s_tail_ack_queue)) { in qib_rc_rcv()
2023 if (!qp->s_ack_queue[next].sent) in qib_rc_rcv()
2027 e = &qp->s_ack_queue[qp->r_head_ack_queue]; in qib_rc_rcv()
2028 if (e->opcode == OP(RDMA_READ_REQUEST) && e->rdma_sge.mr) { in qib_rc_rcv()
2029 rvt_put_mr(e->rdma_sge.mr); in qib_rc_rcv()
2030 e->rdma_sge.mr = NULL; in qib_rc_rcv()
2032 ateth = &ohdr->u.atomic_eth; in qib_rc_rcv()
2034 if (unlikely(vaddr & (sizeof(u64) - 1))) in qib_rc_rcv()
2036 rkey = be32_to_cpu(ateth->rkey); in qib_rc_rcv()
2038 if (unlikely(!rvt_rkey_ok(qp, &qp->r_sge.sge, sizeof(u64), in qib_rc_rcv()
2043 maddr = (atomic64_t *) qp->r_sge.sge.vaddr; in qib_rc_rcv()
2045 e->atomic_data = (opcode == OP(FETCH_ADD)) ? in qib_rc_rcv()
2046 (u64) atomic64_add_return(sdata, maddr) - sdata : in qib_rc_rcv()
2047 (u64) cmpxchg((u64 *) qp->r_sge.sge.vaddr, in qib_rc_rcv()
2050 rvt_put_mr(qp->r_sge.sge.mr); in qib_rc_rcv()
2051 qp->r_sge.num_sge = 0; in qib_rc_rcv()
2052 e->opcode = opcode; in qib_rc_rcv()
2053 e->sent = 0; in qib_rc_rcv()
2054 e->psn = psn; in qib_rc_rcv()
2055 e->lpsn = psn; in qib_rc_rcv()
2056 qp->r_msn++; in qib_rc_rcv()
2057 qp->r_psn++; in qib_rc_rcv()
2058 qp->r_state = opcode; in qib_rc_rcv()
2059 qp->r_nak_state = 0; in qib_rc_rcv()
2060 qp->r_head_ack_queue = next; in qib_rc_rcv()
2062 /* Schedule the send tasklet. */ in qib_rc_rcv()
2063 qp->s_flags |= RVT_S_RESP_PENDING; in qib_rc_rcv()
2073 qp->r_psn++; in qib_rc_rcv()
2074 qp->r_state = opcode; in qib_rc_rcv()
2075 qp->r_ack_psn = psn; in qib_rc_rcv()
2076 qp->r_nak_state = 0; in qib_rc_rcv()
2077 /* Send an ACK if requested or required. */ in qib_rc_rcv()
2083 qp->r_nak_state = IB_RNR_NAK | qp->r_min_rnr_timer; in qib_rc_rcv()
2084 qp->r_ack_psn = qp->r_psn; in qib_rc_rcv()
2086 if (list_empty(&qp->rspwait)) { in qib_rc_rcv()
2087 qp->r_flags |= RVT_R_RSP_NAK; in qib_rc_rcv()
2089 list_add_tail(&qp->rspwait, &rcd->qp_wait_list); in qib_rc_rcv()
2095 qp->r_nak_state = IB_NAK_REMOTE_OPERATIONAL_ERROR; in qib_rc_rcv()
2096 qp->r_ack_psn = qp->r_psn; in qib_rc_rcv()
2098 if (list_empty(&qp->rspwait)) { in qib_rc_rcv()
2099 qp->r_flags |= RVT_R_RSP_NAK; in qib_rc_rcv()
2101 list_add_tail(&qp->rspwait, &rcd->qp_wait_list); in qib_rc_rcv()
2106 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_rc_rcv()
2109 qp->r_nak_state = IB_NAK_INVALID_REQUEST; in qib_rc_rcv()
2110 qp->r_ack_psn = qp->r_psn; in qib_rc_rcv()
2112 if (list_empty(&qp->rspwait)) { in qib_rc_rcv()
2113 qp->r_flags |= RVT_R_RSP_NAK; in qib_rc_rcv()
2115 list_add_tail(&qp->rspwait, &rcd->qp_wait_list); in qib_rc_rcv()
2120 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_rc_rcv()
2123 qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR; in qib_rc_rcv()
2124 qp->r_ack_psn = qp->r_psn; in qib_rc_rcv()
2130 spin_unlock_irqrestore(&qp->s_lock, flags); in qib_rc_rcv()