Lines Matching +full:rpc +full:- +full:if
1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
3 * Copyright (c) 2014-2017 Oracle. All rights reserved.
4 * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved.
9 * COPYING in the main directory of this source tree, or the BSD-type
39 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45 * This file contains the guts of the RPC RDMA protocol, and
47 * to the Linux RPC framework lives.
57 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
61 /* Returns size of largest RPC-over-RDMA header in a Call message
63 * The largest Call header contains a full-size Read list and a
84 /* Returns size of largest RPC-over-RDMA header in a Reply message
105 * rpcrdma_set_max_header_sizes - Initialize inline payload sizes
108 * The max_inline fields contain the maximum size of an RPC message
110 * for every RPC.
114 unsigned int maxsegs = ep->re_max_rdma_segs; in rpcrdma_set_max_header_sizes()
116 ep->re_max_inline_send = in rpcrdma_set_max_header_sizes()
117 ep->re_inline_send - rpcrdma_max_call_header_size(maxsegs); in rpcrdma_set_max_header_sizes()
118 ep->re_max_inline_recv = in rpcrdma_set_max_header_sizes()
119 ep->re_inline_recv - rpcrdma_max_reply_header_size(maxsegs); in rpcrdma_set_max_header_sizes()
123 * plus the RPC call fit under the transport's inline limit. If the
127 * A Read chunk is also required if sending the RPC call inline would
133 struct xdr_buf *xdr = &rqst->rq_snd_buf; in rpcrdma_args_inline()
134 struct rpcrdma_ep *ep = r_xprt->rx_ep; in rpcrdma_args_inline()
137 if (xdr->len > ep->re_max_inline_send) in rpcrdma_args_inline()
140 if (xdr->page_len) { in rpcrdma_args_inline()
141 remaining = xdr->page_len; in rpcrdma_args_inline()
142 offset = offset_in_page(xdr->page_base); in rpcrdma_args_inline()
145 remaining -= min_t(unsigned int, in rpcrdma_args_inline()
146 PAGE_SIZE - offset, remaining); in rpcrdma_args_inline()
148 if (++count > ep->re_attr.cap.max_send_sge) in rpcrdma_args_inline()
158 * operation. If the maximum combined reply message size exceeds that
165 return rqst->rq_rcv_buf.buflen <= r_xprt->rx_ep->re_max_inline_recv; in rpcrdma_results_inline()
168 /* The client is required to provide a Reply chunk if the maximum
169 * size of the non-payload part of the RPC Reply is larger than
176 const struct xdr_buf *buf = &rqst->rq_rcv_buf; in rpcrdma_nonpayload_inline()
178 return (buf->head[0].iov_len + buf->tail[0].iov_len) < in rpcrdma_nonpayload_inline()
179 r_xprt->rx_ep->re_max_inline_recv; in rpcrdma_nonpayload_inline()
196 base = vec->iov_base; in rpcrdma_convert_kvec()
198 remaining = vec->iov_len; in rpcrdma_convert_kvec()
200 seg->mr_page = NULL; in rpcrdma_convert_kvec()
201 seg->mr_offset = base; in rpcrdma_convert_kvec()
202 seg->mr_len = min_t(u32, PAGE_SIZE - page_offset, remaining); in rpcrdma_convert_kvec()
203 remaining -= seg->mr_len; in rpcrdma_convert_kvec()
204 base += seg->mr_len; in rpcrdma_convert_kvec()
229 if (pos == 0) in rpcrdma_convert_iovs()
230 seg = rpcrdma_convert_kvec(&xdrbuf->head[0], seg, &n); in rpcrdma_convert_iovs()
232 len = xdrbuf->page_len; in rpcrdma_convert_iovs()
233 ppages = xdrbuf->pages + (xdrbuf->page_base >> PAGE_SHIFT); in rpcrdma_convert_iovs()
234 page_base = offset_in_page(xdrbuf->page_base); in rpcrdma_convert_iovs()
236 /* ACL likes to be lazy in allocating pages - ACLs in rpcrdma_convert_iovs()
239 if (unlikely(xdrbuf->flags & XDRBUF_SPARSE_PAGES)) { in rpcrdma_convert_iovs()
240 if (!*ppages) in rpcrdma_convert_iovs()
242 if (!*ppages) in rpcrdma_convert_iovs()
243 return -ENOBUFS; in rpcrdma_convert_iovs()
245 seg->mr_page = *ppages; in rpcrdma_convert_iovs()
246 seg->mr_offset = (char *)page_base; in rpcrdma_convert_iovs()
247 seg->mr_len = min_t(u32, PAGE_SIZE - page_base, len); in rpcrdma_convert_iovs()
248 len -= seg->mr_len; in rpcrdma_convert_iovs()
258 if (type == rpcrdma_readch && r_xprt->rx_ep->re_implicit_roundup) in rpcrdma_convert_iovs()
262 * extra segment for non-XDR-aligned Write chunks. The upper in rpcrdma_convert_iovs()
266 if (type == rpcrdma_writech && r_xprt->rx_ep->re_implicit_roundup) in rpcrdma_convert_iovs()
269 if (xdrbuf->tail[0].iov_len) in rpcrdma_convert_iovs()
270 seg = rpcrdma_convert_kvec(&xdrbuf->tail[0], seg, &n); in rpcrdma_convert_iovs()
273 if (unlikely(n > RPCRDMA_MAX_SEGS)) in rpcrdma_convert_iovs()
274 return -EIO; in rpcrdma_convert_iovs()
284 if (unlikely(!p)) in encode_rdma_segment()
285 return -EMSGSIZE; in encode_rdma_segment()
287 xdr_encode_rdma_segment(p, mr->mr_handle, mr->mr_length, mr->mr_offset); in encode_rdma_segment()
298 if (unlikely(!p)) in encode_read_segment()
299 return -EMSGSIZE; in encode_read_segment()
302 xdr_encode_read_segment(p, position, mr->mr_handle, mr->mr_length, in encode_read_segment()
303 mr->mr_offset); in encode_read_segment()
313 *mr = rpcrdma_mr_pop(&req->rl_free_mrs); in rpcrdma_mr_prepare()
314 if (!*mr) { in rpcrdma_mr_prepare()
316 if (!*mr) in rpcrdma_mr_prepare()
319 (*mr)->mr_req = req; in rpcrdma_mr_prepare()
322 rpcrdma_mr_push(*mr, &req->rl_registered); in rpcrdma_mr_prepare()
323 return frwr_map(r_xprt, seg, nsegs, writing, req->rl_slot.rq_xid, *mr); in rpcrdma_mr_prepare()
327 xprt_wait_for_buffer_space(&r_xprt->rx_xprt); in rpcrdma_mr_prepare()
329 return ERR_PTR(-EAGAIN); in rpcrdma_mr_prepare()
335 * Encoding key for single-list chunks (HLOO = Handle32 Length32 Offset64):
339 * 1 - PHLOO - 1 - PHLOO - ... - 1 - PHLOO - 0
341 * Returns zero on success, or a negative errno if a failure occurred.
351 struct xdr_stream *xdr = &req->rl_stream; in rpcrdma_encode_read_list()
357 if (rtype == rpcrdma_noch_pullup || rtype == rpcrdma_noch_mapped) in rpcrdma_encode_read_list()
360 pos = rqst->rq_snd_buf.head[0].iov_len; in rpcrdma_encode_read_list()
361 if (rtype == rpcrdma_areadch) in rpcrdma_encode_read_list()
363 seg = req->rl_segments; in rpcrdma_encode_read_list()
364 nsegs = rpcrdma_convert_iovs(r_xprt, &rqst->rq_snd_buf, pos, in rpcrdma_encode_read_list()
366 if (nsegs < 0) in rpcrdma_encode_read_list()
371 if (IS_ERR(seg)) in rpcrdma_encode_read_list()
374 if (encode_read_segment(xdr, mr, pos) < 0) in rpcrdma_encode_read_list()
375 return -EMSGSIZE; in rpcrdma_encode_read_list()
377 trace_xprtrdma_chunk_read(rqst->rq_task, pos, mr, nsegs); in rpcrdma_encode_read_list()
378 r_xprt->rx_stats.read_chunk_count++; in rpcrdma_encode_read_list()
379 nsegs -= mr->mr_nents; in rpcrdma_encode_read_list()
383 if (xdr_stream_encode_item_absent(xdr) < 0) in rpcrdma_encode_read_list()
384 return -EMSGSIZE; in rpcrdma_encode_read_list()
392 * Encoding key for single-list chunks (HLOO = Handle32 Length32 Offset64):
396 * 1 - N - HLOO - HLOO - ... - HLOO - 0
398 * Returns zero on success, or a negative errno if a failure occurred.
408 struct xdr_stream *xdr = &req->rl_stream; in rpcrdma_encode_write_list()
414 if (wtype != rpcrdma_writech) in rpcrdma_encode_write_list()
417 seg = req->rl_segments; in rpcrdma_encode_write_list()
418 nsegs = rpcrdma_convert_iovs(r_xprt, &rqst->rq_rcv_buf, in rpcrdma_encode_write_list()
419 rqst->rq_rcv_buf.head[0].iov_len, in rpcrdma_encode_write_list()
421 if (nsegs < 0) in rpcrdma_encode_write_list()
424 if (xdr_stream_encode_item_present(xdr) < 0) in rpcrdma_encode_write_list()
425 return -EMSGSIZE; in rpcrdma_encode_write_list()
427 if (unlikely(!segcount)) in rpcrdma_encode_write_list()
428 return -EMSGSIZE; in rpcrdma_encode_write_list()
434 if (IS_ERR(seg)) in rpcrdma_encode_write_list()
437 if (encode_rdma_segment(xdr, mr) < 0) in rpcrdma_encode_write_list()
438 return -EMSGSIZE; in rpcrdma_encode_write_list()
440 trace_xprtrdma_chunk_write(rqst->rq_task, mr, nsegs); in rpcrdma_encode_write_list()
441 r_xprt->rx_stats.write_chunk_count++; in rpcrdma_encode_write_list()
442 r_xprt->rx_stats.total_rdma_request += mr->mr_length; in rpcrdma_encode_write_list()
444 nsegs -= mr->mr_nents; in rpcrdma_encode_write_list()
451 if (xdr_stream_encode_item_absent(xdr) < 0) in rpcrdma_encode_write_list()
452 return -EMSGSIZE; in rpcrdma_encode_write_list()
459 * Encoding key for single-list chunks (HLOO = Handle32 Length32 Offset64):
463 * 1 - N - HLOO - HLOO - ... - HLOO
465 * Returns zero on success, or a negative errno if a failure occurred.
473 struct xdr_stream *xdr = &req->rl_stream; in rpcrdma_encode_reply_chunk()
479 if (wtype != rpcrdma_replych) { in rpcrdma_encode_reply_chunk()
480 if (xdr_stream_encode_item_absent(xdr) < 0) in rpcrdma_encode_reply_chunk()
481 return -EMSGSIZE; in rpcrdma_encode_reply_chunk()
485 seg = req->rl_segments; in rpcrdma_encode_reply_chunk()
486 nsegs = rpcrdma_convert_iovs(r_xprt, &rqst->rq_rcv_buf, 0, wtype, seg); in rpcrdma_encode_reply_chunk()
487 if (nsegs < 0) in rpcrdma_encode_reply_chunk()
490 if (xdr_stream_encode_item_present(xdr) < 0) in rpcrdma_encode_reply_chunk()
491 return -EMSGSIZE; in rpcrdma_encode_reply_chunk()
493 if (unlikely(!segcount)) in rpcrdma_encode_reply_chunk()
494 return -EMSGSIZE; in rpcrdma_encode_reply_chunk()
500 if (IS_ERR(seg)) in rpcrdma_encode_reply_chunk()
503 if (encode_rdma_segment(xdr, mr) < 0) in rpcrdma_encode_reply_chunk()
504 return -EMSGSIZE; in rpcrdma_encode_reply_chunk()
506 trace_xprtrdma_chunk_reply(rqst->rq_task, mr, nsegs); in rpcrdma_encode_reply_chunk()
507 r_xprt->rx_stats.reply_chunk_count++; in rpcrdma_encode_reply_chunk()
508 r_xprt->rx_stats.total_rdma_request += mr->mr_length; in rpcrdma_encode_reply_chunk()
510 nsegs -= mr->mr_nents; in rpcrdma_encode_reply_chunk()
523 struct rpcrdma_rep *rep = req->rl_reply; in rpcrdma_sendctx_done()
526 rep->rr_rxprt->rx_stats.reply_waits_for_send++; in rpcrdma_sendctx_done()
530 * rpcrdma_sendctx_unmap - DMA-unmap Send buffer
536 struct rpcrdma_regbuf *rb = sc->sc_req->rl_sendbuf; in rpcrdma_sendctx_unmap()
539 if (!sc->sc_unmap_count) in rpcrdma_sendctx_unmap()
544 * they can be cheaply re-used. in rpcrdma_sendctx_unmap()
546 for (sge = &sc->sc_sges[2]; sc->sc_unmap_count; in rpcrdma_sendctx_unmap()
547 ++sge, --sc->sc_unmap_count) in rpcrdma_sendctx_unmap()
548 ib_dma_unmap_page(rdmab_device(rb), sge->addr, sge->length, in rpcrdma_sendctx_unmap()
551 kref_put(&sc->sc_req->rl_kref, rpcrdma_sendctx_done); in rpcrdma_sendctx_unmap()
554 /* Prepare an SGE for the RPC-over-RDMA transport header.
559 struct rpcrdma_sendctx *sc = req->rl_sendctx; in rpcrdma_prepare_hdr_sge()
560 struct rpcrdma_regbuf *rb = req->rl_rdmabuf; in rpcrdma_prepare_hdr_sge()
561 struct ib_sge *sge = &sc->sc_sges[req->rl_wr.num_sge++]; in rpcrdma_prepare_hdr_sge()
563 sge->addr = rdmab_addr(rb); in rpcrdma_prepare_hdr_sge()
564 sge->length = len; in rpcrdma_prepare_hdr_sge()
565 sge->lkey = rdmab_lkey(rb); in rpcrdma_prepare_hdr_sge()
567 ib_dma_sync_single_for_device(rdmab_device(rb), sge->addr, sge->length, in rpcrdma_prepare_hdr_sge()
572 * DMA-mapped. Sync the content that has changed.
577 struct rpcrdma_sendctx *sc = req->rl_sendctx; in rpcrdma_prepare_head_iov()
578 struct ib_sge *sge = &sc->sc_sges[req->rl_wr.num_sge++]; in rpcrdma_prepare_head_iov()
579 struct rpcrdma_regbuf *rb = req->rl_sendbuf; in rpcrdma_prepare_head_iov()
581 if (!rpcrdma_regbuf_dma_map(r_xprt, rb)) in rpcrdma_prepare_head_iov()
584 sge->addr = rdmab_addr(rb); in rpcrdma_prepare_head_iov()
585 sge->length = len; in rpcrdma_prepare_head_iov()
586 sge->lkey = rdmab_lkey(rb); in rpcrdma_prepare_head_iov()
588 ib_dma_sync_single_for_device(rdmab_device(rb), sge->addr, sge->length, in rpcrdma_prepare_head_iov()
593 /* If there is a page list present, DMA map and prepare an
599 struct rpcrdma_sendctx *sc = req->rl_sendctx; in rpcrdma_prepare_pagelist()
600 struct rpcrdma_regbuf *rb = req->rl_sendbuf; in rpcrdma_prepare_pagelist()
605 ppages = xdr->pages + (xdr->page_base >> PAGE_SHIFT); in rpcrdma_prepare_pagelist()
606 page_base = offset_in_page(xdr->page_base); in rpcrdma_prepare_pagelist()
607 remaining = xdr->page_len; in rpcrdma_prepare_pagelist()
609 sge = &sc->sc_sges[req->rl_wr.num_sge++]; in rpcrdma_prepare_pagelist()
610 len = min_t(unsigned int, PAGE_SIZE - page_base, remaining); in rpcrdma_prepare_pagelist()
611 sge->addr = ib_dma_map_page(rdmab_device(rb), *ppages, in rpcrdma_prepare_pagelist()
613 if (ib_dma_mapping_error(rdmab_device(rb), sge->addr)) in rpcrdma_prepare_pagelist()
616 sge->length = len; in rpcrdma_prepare_pagelist()
617 sge->lkey = rdmab_lkey(rb); in rpcrdma_prepare_pagelist()
619 sc->sc_unmap_count++; in rpcrdma_prepare_pagelist()
621 remaining -= len; in rpcrdma_prepare_pagelist()
628 trace_xprtrdma_dma_maperr(sge->addr); in rpcrdma_prepare_pagelist()
640 struct rpcrdma_sendctx *sc = req->rl_sendctx; in rpcrdma_prepare_tail_iov()
641 struct ib_sge *sge = &sc->sc_sges[req->rl_wr.num_sge++]; in rpcrdma_prepare_tail_iov()
642 struct rpcrdma_regbuf *rb = req->rl_sendbuf; in rpcrdma_prepare_tail_iov()
643 struct page *page = virt_to_page(xdr->tail[0].iov_base); in rpcrdma_prepare_tail_iov()
645 sge->addr = ib_dma_map_page(rdmab_device(rb), page, page_base, len, in rpcrdma_prepare_tail_iov()
647 if (ib_dma_mapping_error(rdmab_device(rb), sge->addr)) in rpcrdma_prepare_tail_iov()
650 sge->length = len; in rpcrdma_prepare_tail_iov()
651 sge->lkey = rdmab_lkey(rb); in rpcrdma_prepare_tail_iov()
652 ++sc->sc_unmap_count; in rpcrdma_prepare_tail_iov()
656 trace_xprtrdma_dma_maperr(sge->addr); in rpcrdma_prepare_tail_iov()
668 dst = (unsigned char *)xdr->head[0].iov_base; in rpcrdma_pullup_tail_iov()
669 dst += xdr->head[0].iov_len + xdr->page_len; in rpcrdma_pullup_tail_iov()
670 memmove(dst, xdr->tail[0].iov_base, xdr->tail[0].iov_len); in rpcrdma_pullup_tail_iov()
671 r_xprt->rx_stats.pullup_copy_count += xdr->tail[0].iov_len; in rpcrdma_pullup_tail_iov()
684 dst = (unsigned char *)xdr->head[0].iov_base; in rpcrdma_pullup_pagelist()
685 dst += xdr->head[0].iov_len; in rpcrdma_pullup_pagelist()
686 ppages = xdr->pages + (xdr->page_base >> PAGE_SHIFT); in rpcrdma_pullup_pagelist()
687 page_base = offset_in_page(xdr->page_base); in rpcrdma_pullup_pagelist()
688 remaining = xdr->page_len; in rpcrdma_pullup_pagelist()
692 len = min_t(unsigned int, PAGE_SIZE - page_base, remaining); in rpcrdma_pullup_pagelist()
694 r_xprt->rx_stats.pullup_copy_count += len; in rpcrdma_pullup_pagelist()
698 remaining -= len; in rpcrdma_pullup_pagelist()
704 * When the head, pagelist, and tail are small, a pull-up copy
709 * - the caller has already verified that the total length
710 * of the RPC Call body will fit into @rl_sendbuf.
716 if (unlikely(xdr->tail[0].iov_len)) in rpcrdma_prepare_noch_pullup()
719 if (unlikely(xdr->page_len)) in rpcrdma_prepare_noch_pullup()
722 /* The whole RPC message resides in the head iovec now */ in rpcrdma_prepare_noch_pullup()
723 return rpcrdma_prepare_head_iov(r_xprt, req, xdr->len); in rpcrdma_prepare_noch_pullup()
730 struct kvec *tail = &xdr->tail[0]; in rpcrdma_prepare_noch_mapped()
732 if (!rpcrdma_prepare_head_iov(r_xprt, req, xdr->head[0].iov_len)) in rpcrdma_prepare_noch_mapped()
734 if (xdr->page_len) in rpcrdma_prepare_noch_mapped()
735 if (!rpcrdma_prepare_pagelist(req, xdr)) in rpcrdma_prepare_noch_mapped()
737 if (tail->iov_len) in rpcrdma_prepare_noch_mapped()
738 if (!rpcrdma_prepare_tail_iov(req, xdr, in rpcrdma_prepare_noch_mapped()
739 offset_in_page(tail->iov_base), in rpcrdma_prepare_noch_mapped()
740 tail->iov_len)) in rpcrdma_prepare_noch_mapped()
743 if (req->rl_sendctx->sc_unmap_count) in rpcrdma_prepare_noch_mapped()
744 kref_get(&req->rl_kref); in rpcrdma_prepare_noch_mapped()
752 if (!rpcrdma_prepare_head_iov(r_xprt, req, xdr->head[0].iov_len)) in rpcrdma_prepare_readch()
755 /* If there is a Read chunk, the page list is being handled in rpcrdma_prepare_readch()
759 /* Do not include the tail if it is only an XDR pad */ in rpcrdma_prepare_readch()
760 if (xdr->tail[0].iov_len > 3) { in rpcrdma_prepare_readch()
763 /* If the content in the page list is an odd length, in rpcrdma_prepare_readch()
765 * the tail iovec. Force the tail's non-pad content to in rpcrdma_prepare_readch()
768 page_base = offset_in_page(xdr->tail[0].iov_base); in rpcrdma_prepare_readch()
769 len = xdr->tail[0].iov_len; in rpcrdma_prepare_readch()
771 len -= len & 3; in rpcrdma_prepare_readch()
772 if (!rpcrdma_prepare_tail_iov(req, xdr, page_base, len)) in rpcrdma_prepare_readch()
774 kref_get(&req->rl_kref); in rpcrdma_prepare_readch()
781 * rpcrdma_prepare_send_sges - Construct SGEs for a Send WR
783 * @req: context of RPC Call being marshalled
785 * @xdr: xdr_buf containing RPC Call
797 ret = -EAGAIN; in rpcrdma_prepare_send_sges()
798 req->rl_sendctx = rpcrdma_sendctx_get_locked(r_xprt); in rpcrdma_prepare_send_sges()
799 if (!req->rl_sendctx) in rpcrdma_prepare_send_sges()
801 req->rl_sendctx->sc_unmap_count = 0; in rpcrdma_prepare_send_sges()
802 req->rl_sendctx->sc_req = req; in rpcrdma_prepare_send_sges()
803 kref_init(&req->rl_kref); in rpcrdma_prepare_send_sges()
804 req->rl_wr.wr_cqe = &req->rl_sendctx->sc_cqe; in rpcrdma_prepare_send_sges()
805 req->rl_wr.sg_list = req->rl_sendctx->sc_sges; in rpcrdma_prepare_send_sges()
806 req->rl_wr.num_sge = 0; in rpcrdma_prepare_send_sges()
807 req->rl_wr.opcode = IB_WR_SEND; in rpcrdma_prepare_send_sges()
811 ret = -EIO; in rpcrdma_prepare_send_sges()
814 if (!rpcrdma_prepare_noch_pullup(r_xprt, req, xdr)) in rpcrdma_prepare_send_sges()
818 if (!rpcrdma_prepare_noch_mapped(r_xprt, req, xdr)) in rpcrdma_prepare_send_sges()
822 if (!rpcrdma_prepare_readch(r_xprt, req, xdr)) in rpcrdma_prepare_send_sges()
834 rpcrdma_sendctx_unmap(req->rl_sendctx); in rpcrdma_prepare_send_sges()
836 trace_xprtrdma_prepsend_failed(&req->rl_slot, ret); in rpcrdma_prepare_send_sges()
841 * rpcrdma_marshal_req - Marshal and send one RPC request
843 * @rqst: RPC request to be marshaled
845 * For the RPC in "rqst", this function:
846 * - Chooses the transfer mode (eg., RDMA_MSG or RDMA_NOMSG)
847 * - Registers Read, Write, and Reply chunks
848 * - Constructs the transport header
849 * - Posts a Send WR to send the transport header and request
852 * %0 if the RPC was sent successfully,
853 * %-ENOTCONN if the connection was lost,
854 * %-EAGAIN if the caller should call again with the same arguments,
855 * %-ENOBUFS if the caller should call again after a delay,
856 * %-EMSGSIZE if the transport header is too small,
857 * %-EIO if a permanent problem occurred while marshaling.
863 struct xdr_stream *xdr = &req->rl_stream; in rpcrdma_marshal_req()
865 struct xdr_buf *buf = &rqst->rq_snd_buf; in rpcrdma_marshal_req()
870 rpcrdma_set_xdrlen(&req->rl_hdrbuf, 0); in rpcrdma_marshal_req()
871 xdr_init_encode(xdr, &req->rl_hdrbuf, rdmab_data(req->rl_rdmabuf), in rpcrdma_marshal_req()
875 ret = -EMSGSIZE; in rpcrdma_marshal_req()
877 if (!p) in rpcrdma_marshal_req()
879 *p++ = rqst->rq_xid; in rpcrdma_marshal_req()
881 *p++ = r_xprt->rx_buf.rb_max_requests; in rpcrdma_marshal_req()
888 &rqst->rq_cred->cr_auth->au_flags); in rpcrdma_marshal_req()
893 * o If the expected result is under the inline threshold, all ops in rpcrdma_marshal_req()
897 * o Large non-read ops return as a single reply chunk. in rpcrdma_marshal_req()
899 if (rpcrdma_results_inline(r_xprt, rqst)) in rpcrdma_marshal_req()
901 else if ((ddp_allowed && rqst->rq_rcv_buf.flags & XDRBUF_READ) && in rpcrdma_marshal_req()
910 * o If the total request is under the inline threshold, all ops in rpcrdma_marshal_req()
914 * o Large non-write ops are sent with the entire message as a in rpcrdma_marshal_req()
915 * single read chunk (protocol 0-position special case). in rpcrdma_marshal_req()
918 * that both has a data payload, and whose non-data arguments in rpcrdma_marshal_req()
921 if (rpcrdma_args_inline(r_xprt, rqst)) { in rpcrdma_marshal_req()
923 rtype = buf->len < rdmab_length(req->rl_sendbuf) ? in rpcrdma_marshal_req()
925 } else if (ddp_allowed && buf->flags & XDRBUF_WRITE) { in rpcrdma_marshal_req()
929 r_xprt->rx_stats.nomsg_call_count++; in rpcrdma_marshal_req()
935 * of chunk lists in one RPC-over-RDMA Call message: in rpcrdma_marshal_req()
937 * - Read list in rpcrdma_marshal_req()
938 * - Write list in rpcrdma_marshal_req()
939 * - Reply chunk in rpcrdma_marshal_req()
940 * - Read list + Reply chunk in rpcrdma_marshal_req()
944 * - Read list + Write list in rpcrdma_marshal_req()
948 * - Write list + Reply chunk in rpcrdma_marshal_req()
949 * - Read list + Write list + Reply chunk in rpcrdma_marshal_req()
957 if (ret) in rpcrdma_marshal_req()
960 if (ret) in rpcrdma_marshal_req()
963 if (ret) in rpcrdma_marshal_req()
966 ret = rpcrdma_prepare_send_sges(r_xprt, req, req->rl_hdrbuf.len, in rpcrdma_marshal_req()
968 if (ret) in rpcrdma_marshal_req()
976 r_xprt->rx_stats.failed_marshal_count++; in rpcrdma_marshal_req()
985 buf->rb_credits = grant; in __rpcrdma_update_cwnd_locked()
986 xprt->cwnd = grant << RPC_CWNDSHIFT; in __rpcrdma_update_cwnd_locked()
991 struct rpc_xprt *xprt = &r_xprt->rx_xprt; in rpcrdma_update_cwnd()
993 spin_lock(&xprt->transport_lock); in rpcrdma_update_cwnd()
994 __rpcrdma_update_cwnd_locked(xprt, &r_xprt->rx_buf, grant); in rpcrdma_update_cwnd()
995 spin_unlock(&xprt->transport_lock); in rpcrdma_update_cwnd()
999 * rpcrdma_reset_cwnd - Reset the xprt's congestion window
1007 struct rpc_xprt *xprt = &r_xprt->rx_xprt; in rpcrdma_reset_cwnd()
1009 spin_lock(&xprt->transport_lock); in rpcrdma_reset_cwnd()
1010 xprt->cong = 0; in rpcrdma_reset_cwnd()
1011 __rpcrdma_update_cwnd_locked(xprt, &r_xprt->rx_buf, 1); in rpcrdma_reset_cwnd()
1012 spin_unlock(&xprt->transport_lock); in rpcrdma_reset_cwnd()
1016 * rpcrdma_inline_fixup - Scatter inline received data into rqst's iovecs
1017 * @rqst: controlling RPC request
1018 * @srcp: points to RPC message payload in receive buffer
1042 /* The head iovec is redirected to the RPC reply message in rpcrdma_inline_fixup()
1045 rqst->rq_rcv_buf.head[0].iov_base = srcp; in rpcrdma_inline_fixup()
1046 rqst->rq_private_buf.head[0].iov_base = srcp; in rpcrdma_inline_fixup()
1051 curlen = rqst->rq_rcv_buf.head[0].iov_len; in rpcrdma_inline_fixup()
1052 if (curlen > copy_len) in rpcrdma_inline_fixup()
1055 copy_len -= curlen; in rpcrdma_inline_fixup()
1057 ppages = rqst->rq_rcv_buf.pages + in rpcrdma_inline_fixup()
1058 (rqst->rq_rcv_buf.page_base >> PAGE_SHIFT); in rpcrdma_inline_fixup()
1059 page_base = offset_in_page(rqst->rq_rcv_buf.page_base); in rpcrdma_inline_fixup()
1061 if (copy_len && rqst->rq_rcv_buf.page_len) { in rpcrdma_inline_fixup()
1064 pagelist_len = rqst->rq_rcv_buf.page_len; in rpcrdma_inline_fixup()
1065 if (pagelist_len > copy_len) in rpcrdma_inline_fixup()
1069 curlen = PAGE_SIZE - page_base; in rpcrdma_inline_fixup()
1070 if (curlen > pagelist_len) in rpcrdma_inline_fixup()
1078 copy_len -= curlen; in rpcrdma_inline_fixup()
1080 pagelist_len -= curlen; in rpcrdma_inline_fixup()
1081 if (!pagelist_len) in rpcrdma_inline_fixup()
1092 if (pad) in rpcrdma_inline_fixup()
1093 srcp -= pad; in rpcrdma_inline_fixup()
1099 if (copy_len || pad) { in rpcrdma_inline_fixup()
1100 rqst->rq_rcv_buf.tail[0].iov_base = srcp; in rpcrdma_inline_fixup()
1101 rqst->rq_private_buf.tail[0].iov_base = srcp; in rpcrdma_inline_fixup()
1104 if (fixup_copy_count) in rpcrdma_inline_fixup()
1111 * the RPC/RDMA header small and fixed in size, so it is
1112 * straightforward to check the RPC header's direction field.
1116 #if defined(CONFIG_SUNRPC_BACKCHANNEL) in rpcrdma_is_bcall()
1118 struct xdr_stream *xdr = &rep->rr_stream; in rpcrdma_is_bcall()
1121 if (rep->rr_proc != rdma_msg) in rpcrdma_is_bcall()
1128 if (xdr_item_is_present(p++)) in rpcrdma_is_bcall()
1130 if (xdr_item_is_present(p++)) in rpcrdma_is_bcall()
1132 if (xdr_item_is_present(p++)) in rpcrdma_is_bcall()
1135 /* RPC header */ in rpcrdma_is_bcall()
1136 if (*p++ != rep->rr_xid) in rpcrdma_is_bcall()
1138 if (*p != cpu_to_be32(RPC_CALL)) in rpcrdma_is_bcall()
1142 * advance to the RPC header. in rpcrdma_is_bcall()
1145 if (unlikely(!p)) in rpcrdma_is_bcall()
1152 pr_warn("RPC/RDMA short backward direction call\n"); in rpcrdma_is_bcall()
1168 if (unlikely(!p)) in decode_rdma_segment()
1169 return -EIO; in decode_rdma_segment()
1182 if (unlikely(!p)) in decode_write_chunk()
1183 return -EIO; in decode_write_chunk()
1187 while (segcount--) { in decode_write_chunk()
1188 if (decode_rdma_segment(xdr, &seglength)) in decode_write_chunk()
1189 return -EIO; in decode_write_chunk()
1196 /* In RPC-over-RDMA Version One replies, a Read list is never
1197 * expected. This decoder is a stub that returns an error if
1205 if (unlikely(!p)) in decode_read_list()
1206 return -EIO; in decode_read_list()
1207 if (unlikely(xdr_item_is_present(p))) in decode_read_list()
1208 return -EIO; in decode_read_list()
1224 if (unlikely(!p)) in decode_write_list()
1225 return -EIO; in decode_write_list()
1226 if (xdr_item_is_absent(p)) in decode_write_list()
1228 if (!first) in decode_write_list()
1229 return -EIO; in decode_write_list()
1231 if (decode_write_chunk(xdr, &chunklen)) in decode_write_list()
1232 return -EIO; in decode_write_list()
1244 if (unlikely(!p)) in decode_reply_chunk()
1245 return -EIO; in decode_reply_chunk()
1248 if (xdr_item_is_present(p)) in decode_reply_chunk()
1249 if (decode_write_chunk(xdr, length)) in decode_reply_chunk()
1250 return -EIO; in decode_reply_chunk()
1258 struct xdr_stream *xdr = &rep->rr_stream; in rpcrdma_decode_msg()
1263 if (decode_read_list(xdr)) in rpcrdma_decode_msg()
1264 return -EIO; in rpcrdma_decode_msg()
1265 if (decode_write_list(xdr, &writelist)) in rpcrdma_decode_msg()
1266 return -EIO; in rpcrdma_decode_msg()
1267 if (decode_reply_chunk(xdr, &replychunk)) in rpcrdma_decode_msg()
1268 return -EIO; in rpcrdma_decode_msg()
1271 if (unlikely(replychunk)) in rpcrdma_decode_msg()
1272 return -EIO; in rpcrdma_decode_msg()
1274 /* Build the RPC reply's Payload stream in rqst->rq_rcv_buf */ in rpcrdma_decode_msg()
1277 r_xprt->rx_stats.fixup_copy_count += in rpcrdma_decode_msg()
1280 r_xprt->rx_stats.total_rdma_reply += writelist; in rpcrdma_decode_msg()
1287 struct xdr_stream *xdr = &rep->rr_stream; in rpcrdma_decode_nomsg()
1291 if (decode_read_list(xdr)) in rpcrdma_decode_nomsg()
1292 return -EIO; in rpcrdma_decode_nomsg()
1293 if (decode_write_list(xdr, &writelist)) in rpcrdma_decode_nomsg()
1294 return -EIO; in rpcrdma_decode_nomsg()
1295 if (decode_reply_chunk(xdr, &replychunk)) in rpcrdma_decode_nomsg()
1296 return -EIO; in rpcrdma_decode_nomsg()
1299 if (unlikely(writelist)) in rpcrdma_decode_nomsg()
1300 return -EIO; in rpcrdma_decode_nomsg()
1301 if (unlikely(!replychunk)) in rpcrdma_decode_nomsg()
1302 return -EIO; in rpcrdma_decode_nomsg()
1305 r_xprt->rx_stats.total_rdma_reply += replychunk; in rpcrdma_decode_nomsg()
1313 struct xdr_stream *xdr = &rep->rr_stream; in rpcrdma_decode_error()
1317 if (unlikely(!p)) in rpcrdma_decode_error()
1318 return -EIO; in rpcrdma_decode_error()
1323 if (!p) in rpcrdma_decode_error()
1325 dprintk("RPC: %s: server reports " in rpcrdma_decode_error()
1326 "version error (%u-%u), xid %08x\n", __func__, in rpcrdma_decode_error()
1328 be32_to_cpu(rep->rr_xid)); in rpcrdma_decode_error()
1331 dprintk("RPC: %s: server reports " in rpcrdma_decode_error()
1333 be32_to_cpu(rep->rr_xid)); in rpcrdma_decode_error()
1336 dprintk("RPC: %s: server reports " in rpcrdma_decode_error()
1338 be32_to_cpup(p), be32_to_cpu(rep->rr_xid)); in rpcrdma_decode_error()
1341 return -EIO; in rpcrdma_decode_error()
1344 /* Perform XID lookup, reconstruction of the RPC reply, and
1345 * RPC completion while holding the transport lock to ensure
1350 struct rpcrdma_xprt *r_xprt = rep->rr_rxprt; in rpcrdma_complete_rqst()
1351 struct rpc_xprt *xprt = &r_xprt->rx_xprt; in rpcrdma_complete_rqst()
1352 struct rpc_rqst *rqst = rep->rr_rqst; in rpcrdma_complete_rqst()
1355 switch (rep->rr_proc) { in rpcrdma_complete_rqst()
1366 status = -EIO; in rpcrdma_complete_rqst()
1368 if (status < 0) in rpcrdma_complete_rqst()
1372 spin_lock(&xprt->queue_lock); in rpcrdma_complete_rqst()
1373 xprt_complete_rqst(rqst->rq_task, status); in rpcrdma_complete_rqst()
1375 spin_unlock(&xprt->queue_lock); in rpcrdma_complete_rqst()
1380 r_xprt->rx_stats.bad_reply_count++; in rpcrdma_complete_rqst()
1381 rqst->rq_task->tk_status = status; in rpcrdma_complete_rqst()
1391 rpcrdma_complete_rqst(req->rl_reply); in rpcrdma_reply_done()
1395 * rpcrdma_reply_handler - Process received RPC/RDMA messages
1398 * Errors must result in the RPC task either being awakened, or
1403 struct rpcrdma_xprt *r_xprt = rep->rr_rxprt; in rpcrdma_reply_handler()
1404 struct rpc_xprt *xprt = &r_xprt->rx_xprt; in rpcrdma_reply_handler()
1405 struct rpcrdma_buffer *buf = &r_xprt->rx_buf; in rpcrdma_reply_handler()
1414 if (xprt->reestablish_timeout) in rpcrdma_reply_handler()
1415 xprt->reestablish_timeout = 0; in rpcrdma_reply_handler()
1418 xdr_init_decode(&rep->rr_stream, &rep->rr_hdrbuf, in rpcrdma_reply_handler()
1419 rep->rr_hdrbuf.head[0].iov_base, NULL); in rpcrdma_reply_handler()
1420 p = xdr_inline_decode(&rep->rr_stream, 4 * sizeof(*p)); in rpcrdma_reply_handler()
1421 if (unlikely(!p)) in rpcrdma_reply_handler()
1423 rep->rr_xid = *p++; in rpcrdma_reply_handler()
1424 rep->rr_vers = *p++; in rpcrdma_reply_handler()
1426 rep->rr_proc = *p++; in rpcrdma_reply_handler()
1428 if (rep->rr_vers != rpcrdma_version) in rpcrdma_reply_handler()
1431 if (rpcrdma_is_bcall(r_xprt, rep)) in rpcrdma_reply_handler()
1437 spin_lock(&xprt->queue_lock); in rpcrdma_reply_handler()
1438 rqst = xprt_lookup_rqst(xprt, rep->rr_xid); in rpcrdma_reply_handler()
1439 if (!rqst) in rpcrdma_reply_handler()
1442 spin_unlock(&xprt->queue_lock); in rpcrdma_reply_handler()
1444 if (credits == 0) in rpcrdma_reply_handler()
1446 else if (credits > r_xprt->rx_ep->re_max_requests) in rpcrdma_reply_handler()
1447 credits = r_xprt->rx_ep->re_max_requests; in rpcrdma_reply_handler()
1448 if (buf->rb_credits != credits) in rpcrdma_reply_handler()
1453 if (req->rl_reply) { in rpcrdma_reply_handler()
1454 trace_xprtrdma_leaked_rep(rqst, req->rl_reply); in rpcrdma_reply_handler()
1455 rpcrdma_recv_buffer_put(req->rl_reply); in rpcrdma_reply_handler()
1457 req->rl_reply = rep; in rpcrdma_reply_handler()
1458 rep->rr_rqst = rqst; in rpcrdma_reply_handler()
1460 trace_xprtrdma_reply(rqst->rq_task, rep, req, credits); in rpcrdma_reply_handler()
1462 if (rep->rr_wc_flags & IB_WC_WITH_INVALIDATE) in rpcrdma_reply_handler()
1463 frwr_reminv(rep, &req->rl_registered); in rpcrdma_reply_handler()
1464 if (!list_empty(&req->rl_registered)) in rpcrdma_reply_handler()
1466 /* LocalInv completion will complete the RPC */ in rpcrdma_reply_handler()
1468 kref_put(&req->rl_kref, rpcrdma_reply_done); in rpcrdma_reply_handler()
1476 spin_unlock(&xprt->queue_lock); in rpcrdma_reply_handler()