Lines Matching +full:rpc +full:- +full:if
1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
3 * Copyright (c) 2014-2020, Oracle and/or its affiliates.
4 * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved.
9 * COPYING in the main directory of this source tree, or the BSD-type
39 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45 * This file contains the guts of the RPC RDMA protocol, and
47 * to the Linux RPC framework lives.
57 /* Returns size of largest RPC-over-RDMA header in a Call message
59 * The largest Call header contains a full-size Read list and a
80 /* Returns size of largest RPC-over-RDMA header in a Reply message
101 * rpcrdma_set_max_header_sizes - Initialize inline payload sizes
104 * The max_inline fields contain the maximum size of an RPC message
106 * for every RPC.
110 unsigned int maxsegs = ep->re_max_rdma_segs; in rpcrdma_set_max_header_sizes()
112 ep->re_max_inline_send = in rpcrdma_set_max_header_sizes()
113 ep->re_inline_send - rpcrdma_max_call_header_size(maxsegs); in rpcrdma_set_max_header_sizes()
114 ep->re_max_inline_recv = in rpcrdma_set_max_header_sizes()
115 ep->re_inline_recv - rpcrdma_max_reply_header_size(maxsegs); in rpcrdma_set_max_header_sizes()
119 * plus the RPC call fit under the transport's inline limit. If the
123 * A Read chunk is also required if sending the RPC call inline would
129 struct xdr_buf *xdr = &rqst->rq_snd_buf; in rpcrdma_args_inline()
130 struct rpcrdma_ep *ep = r_xprt->rx_ep; in rpcrdma_args_inline()
133 if (xdr->len > ep->re_max_inline_send) in rpcrdma_args_inline()
136 if (xdr->page_len) { in rpcrdma_args_inline()
137 remaining = xdr->page_len; in rpcrdma_args_inline()
138 offset = offset_in_page(xdr->page_base); in rpcrdma_args_inline()
141 remaining -= min_t(unsigned int, in rpcrdma_args_inline()
142 PAGE_SIZE - offset, remaining); in rpcrdma_args_inline()
144 if (++count > ep->re_attr.cap.max_send_sge) in rpcrdma_args_inline()
154 * operation. If the maximum combined reply message size exceeds that
161 return rqst->rq_rcv_buf.buflen <= r_xprt->rx_ep->re_max_inline_recv; in rpcrdma_results_inline()
164 /* The client is required to provide a Reply chunk if the maximum
165 * size of the non-payload part of the RPC Reply is larger than
172 const struct xdr_buf *buf = &rqst->rq_rcv_buf; in rpcrdma_nonpayload_inline()
174 return (buf->head[0].iov_len + buf->tail[0].iov_len) < in rpcrdma_nonpayload_inline()
175 r_xprt->rx_ep->re_max_inline_recv; in rpcrdma_nonpayload_inline()
189 len = buf->page_len; in rpcrdma_alloc_sparse_pages()
190 ppages = buf->pages + (buf->page_base >> PAGE_SHIFT); in rpcrdma_alloc_sparse_pages()
192 if (!*ppages) in rpcrdma_alloc_sparse_pages()
194 if (!*ppages) in rpcrdma_alloc_sparse_pages()
195 return -ENOBUFS; in rpcrdma_alloc_sparse_pages()
197 len -= PAGE_SIZE; in rpcrdma_alloc_sparse_pages()
212 seg->mr_page = virt_to_page(vec->iov_base); in rpcrdma_convert_kvec()
213 seg->mr_offset = offset_in_page(vec->iov_base); in rpcrdma_convert_kvec()
214 seg->mr_len = vec->iov_len; in rpcrdma_convert_kvec()
237 if (pos == 0) in rpcrdma_convert_iovs()
238 seg = rpcrdma_convert_kvec(&xdrbuf->head[0], seg, &n); in rpcrdma_convert_iovs()
240 len = xdrbuf->page_len; in rpcrdma_convert_iovs()
241 ppages = xdrbuf->pages + (xdrbuf->page_base >> PAGE_SHIFT); in rpcrdma_convert_iovs()
242 page_base = offset_in_page(xdrbuf->page_base); in rpcrdma_convert_iovs()
244 seg->mr_page = *ppages; in rpcrdma_convert_iovs()
245 seg->mr_offset = page_base; in rpcrdma_convert_iovs()
246 seg->mr_len = min_t(u32, PAGE_SIZE - page_base, len); in rpcrdma_convert_iovs()
247 len -= seg->mr_len; in rpcrdma_convert_iovs()
254 if (type == rpcrdma_readch || type == rpcrdma_writech) in rpcrdma_convert_iovs()
257 if (xdrbuf->tail[0].iov_len) in rpcrdma_convert_iovs()
258 rpcrdma_convert_kvec(&xdrbuf->tail[0], seg, &n); in rpcrdma_convert_iovs()
261 if (unlikely(n > RPCRDMA_MAX_SEGS)) in rpcrdma_convert_iovs()
262 return -EIO; in rpcrdma_convert_iovs()
272 if (unlikely(!p)) in encode_rdma_segment()
273 return -EMSGSIZE; in encode_rdma_segment()
275 xdr_encode_rdma_segment(p, mr->mr_handle, mr->mr_length, mr->mr_offset); in encode_rdma_segment()
286 if (unlikely(!p)) in encode_read_segment()
287 return -EMSGSIZE; in encode_read_segment()
290 xdr_encode_read_segment(p, position, mr->mr_handle, mr->mr_length, in encode_read_segment()
291 mr->mr_offset); in encode_read_segment()
301 *mr = rpcrdma_mr_pop(&req->rl_free_mrs); in rpcrdma_mr_prepare()
302 if (!*mr) { in rpcrdma_mr_prepare()
304 if (!*mr) in rpcrdma_mr_prepare()
306 (*mr)->mr_req = req; in rpcrdma_mr_prepare()
309 rpcrdma_mr_push(*mr, &req->rl_registered); in rpcrdma_mr_prepare()
310 return frwr_map(r_xprt, seg, nsegs, writing, req->rl_slot.rq_xid, *mr); in rpcrdma_mr_prepare()
314 xprt_wait_for_buffer_space(&r_xprt->rx_xprt); in rpcrdma_mr_prepare()
316 return ERR_PTR(-EAGAIN); in rpcrdma_mr_prepare()
322 * Encoding key for single-list chunks (HLOO = Handle32 Length32 Offset64):
326 * 1 - PHLOO - 1 - PHLOO - ... - 1 - PHLOO - 0
328 * Returns zero on success, or a negative errno if a failure occurred.
338 struct xdr_stream *xdr = &req->rl_stream; in rpcrdma_encode_read_list()
344 if (rtype == rpcrdma_noch_pullup || rtype == rpcrdma_noch_mapped) in rpcrdma_encode_read_list()
347 pos = rqst->rq_snd_buf.head[0].iov_len; in rpcrdma_encode_read_list()
348 if (rtype == rpcrdma_areadch) in rpcrdma_encode_read_list()
350 seg = req->rl_segments; in rpcrdma_encode_read_list()
351 nsegs = rpcrdma_convert_iovs(r_xprt, &rqst->rq_snd_buf, pos, in rpcrdma_encode_read_list()
353 if (nsegs < 0) in rpcrdma_encode_read_list()
358 if (IS_ERR(seg)) in rpcrdma_encode_read_list()
361 if (encode_read_segment(xdr, mr, pos) < 0) in rpcrdma_encode_read_list()
362 return -EMSGSIZE; in rpcrdma_encode_read_list()
364 trace_xprtrdma_chunk_read(rqst->rq_task, pos, mr, nsegs); in rpcrdma_encode_read_list()
365 r_xprt->rx_stats.read_chunk_count++; in rpcrdma_encode_read_list()
366 nsegs -= mr->mr_nents; in rpcrdma_encode_read_list()
370 if (xdr_stream_encode_item_absent(xdr) < 0) in rpcrdma_encode_read_list()
371 return -EMSGSIZE; in rpcrdma_encode_read_list()
379 * Encoding key for single-list chunks (HLOO = Handle32 Length32 Offset64):
383 * 1 - N - HLOO - HLOO - ... - HLOO - 0
385 * Returns zero on success, or a negative errno if a failure occurred.
395 struct xdr_stream *xdr = &req->rl_stream; in rpcrdma_encode_write_list()
396 struct rpcrdma_ep *ep = r_xprt->rx_ep; in rpcrdma_encode_write_list()
402 if (wtype != rpcrdma_writech) in rpcrdma_encode_write_list()
405 seg = req->rl_segments; in rpcrdma_encode_write_list()
406 nsegs = rpcrdma_convert_iovs(r_xprt, &rqst->rq_rcv_buf, in rpcrdma_encode_write_list()
407 rqst->rq_rcv_buf.head[0].iov_len, in rpcrdma_encode_write_list()
409 if (nsegs < 0) in rpcrdma_encode_write_list()
412 if (xdr_stream_encode_item_present(xdr) < 0) in rpcrdma_encode_write_list()
413 return -EMSGSIZE; in rpcrdma_encode_write_list()
415 if (unlikely(!segcount)) in rpcrdma_encode_write_list()
416 return -EMSGSIZE; in rpcrdma_encode_write_list()
422 if (IS_ERR(seg)) in rpcrdma_encode_write_list()
425 if (encode_rdma_segment(xdr, mr) < 0) in rpcrdma_encode_write_list()
426 return -EMSGSIZE; in rpcrdma_encode_write_list()
428 trace_xprtrdma_chunk_write(rqst->rq_task, mr, nsegs); in rpcrdma_encode_write_list()
429 r_xprt->rx_stats.write_chunk_count++; in rpcrdma_encode_write_list()
430 r_xprt->rx_stats.total_rdma_request += mr->mr_length; in rpcrdma_encode_write_list()
432 nsegs -= mr->mr_nents; in rpcrdma_encode_write_list()
435 if (xdr_pad_size(rqst->rq_rcv_buf.page_len)) { in rpcrdma_encode_write_list()
436 if (encode_rdma_segment(xdr, ep->re_write_pad_mr) < 0) in rpcrdma_encode_write_list()
437 return -EMSGSIZE; in rpcrdma_encode_write_list()
439 trace_xprtrdma_chunk_wp(rqst->rq_task, ep->re_write_pad_mr, in rpcrdma_encode_write_list()
441 r_xprt->rx_stats.write_chunk_count++; in rpcrdma_encode_write_list()
442 r_xprt->rx_stats.total_rdma_request += mr->mr_length; in rpcrdma_encode_write_list()
444 nsegs -= mr->mr_nents; in rpcrdma_encode_write_list()
451 if (xdr_stream_encode_item_absent(xdr) < 0) in rpcrdma_encode_write_list()
452 return -EMSGSIZE; in rpcrdma_encode_write_list()
459 * Encoding key for single-list chunks (HLOO = Handle32 Length32 Offset64):
463 * 1 - N - HLOO - HLOO - ... - HLOO
465 * Returns zero on success, or a negative errno if a failure occurred.
473 struct xdr_stream *xdr = &req->rl_stream; in rpcrdma_encode_reply_chunk()
479 if (wtype != rpcrdma_replych) { in rpcrdma_encode_reply_chunk()
480 if (xdr_stream_encode_item_absent(xdr) < 0) in rpcrdma_encode_reply_chunk()
481 return -EMSGSIZE; in rpcrdma_encode_reply_chunk()
485 seg = req->rl_segments; in rpcrdma_encode_reply_chunk()
486 nsegs = rpcrdma_convert_iovs(r_xprt, &rqst->rq_rcv_buf, 0, wtype, seg); in rpcrdma_encode_reply_chunk()
487 if (nsegs < 0) in rpcrdma_encode_reply_chunk()
490 if (xdr_stream_encode_item_present(xdr) < 0) in rpcrdma_encode_reply_chunk()
491 return -EMSGSIZE; in rpcrdma_encode_reply_chunk()
493 if (unlikely(!segcount)) in rpcrdma_encode_reply_chunk()
494 return -EMSGSIZE; in rpcrdma_encode_reply_chunk()
500 if (IS_ERR(seg)) in rpcrdma_encode_reply_chunk()
503 if (encode_rdma_segment(xdr, mr) < 0) in rpcrdma_encode_reply_chunk()
504 return -EMSGSIZE; in rpcrdma_encode_reply_chunk()
506 trace_xprtrdma_chunk_reply(rqst->rq_task, mr, nsegs); in rpcrdma_encode_reply_chunk()
507 r_xprt->rx_stats.reply_chunk_count++; in rpcrdma_encode_reply_chunk()
508 r_xprt->rx_stats.total_rdma_request += mr->mr_length; in rpcrdma_encode_reply_chunk()
510 nsegs -= mr->mr_nents; in rpcrdma_encode_reply_chunk()
523 struct rpcrdma_rep *rep = req->rl_reply; in rpcrdma_sendctx_done()
526 rep->rr_rxprt->rx_stats.reply_waits_for_send++; in rpcrdma_sendctx_done()
530 * rpcrdma_sendctx_unmap - DMA-unmap Send buffer
536 struct rpcrdma_regbuf *rb = sc->sc_req->rl_sendbuf; in rpcrdma_sendctx_unmap()
539 if (!sc->sc_unmap_count) in rpcrdma_sendctx_unmap()
544 * they can be cheaply re-used. in rpcrdma_sendctx_unmap()
546 for (sge = &sc->sc_sges[2]; sc->sc_unmap_count; in rpcrdma_sendctx_unmap()
547 ++sge, --sc->sc_unmap_count) in rpcrdma_sendctx_unmap()
548 ib_dma_unmap_page(rdmab_device(rb), sge->addr, sge->length, in rpcrdma_sendctx_unmap()
551 kref_put(&sc->sc_req->rl_kref, rpcrdma_sendctx_done); in rpcrdma_sendctx_unmap()
554 /* Prepare an SGE for the RPC-over-RDMA transport header.
559 struct rpcrdma_sendctx *sc = req->rl_sendctx; in rpcrdma_prepare_hdr_sge()
560 struct rpcrdma_regbuf *rb = req->rl_rdmabuf; in rpcrdma_prepare_hdr_sge()
561 struct ib_sge *sge = &sc->sc_sges[req->rl_wr.num_sge++]; in rpcrdma_prepare_hdr_sge()
563 sge->addr = rdmab_addr(rb); in rpcrdma_prepare_hdr_sge()
564 sge->length = len; in rpcrdma_prepare_hdr_sge()
565 sge->lkey = rdmab_lkey(rb); in rpcrdma_prepare_hdr_sge()
567 ib_dma_sync_single_for_device(rdmab_device(rb), sge->addr, sge->length, in rpcrdma_prepare_hdr_sge()
572 * DMA-mapped. Sync the content that has changed.
577 struct rpcrdma_sendctx *sc = req->rl_sendctx; in rpcrdma_prepare_head_iov()
578 struct ib_sge *sge = &sc->sc_sges[req->rl_wr.num_sge++]; in rpcrdma_prepare_head_iov()
579 struct rpcrdma_regbuf *rb = req->rl_sendbuf; in rpcrdma_prepare_head_iov()
581 if (!rpcrdma_regbuf_dma_map(r_xprt, rb)) in rpcrdma_prepare_head_iov()
584 sge->addr = rdmab_addr(rb); in rpcrdma_prepare_head_iov()
585 sge->length = len; in rpcrdma_prepare_head_iov()
586 sge->lkey = rdmab_lkey(rb); in rpcrdma_prepare_head_iov()
588 ib_dma_sync_single_for_device(rdmab_device(rb), sge->addr, sge->length, in rpcrdma_prepare_head_iov()
593 /* If there is a page list present, DMA map and prepare an
599 struct rpcrdma_sendctx *sc = req->rl_sendctx; in rpcrdma_prepare_pagelist()
600 struct rpcrdma_regbuf *rb = req->rl_sendbuf; in rpcrdma_prepare_pagelist()
605 ppages = xdr->pages + (xdr->page_base >> PAGE_SHIFT); in rpcrdma_prepare_pagelist()
606 page_base = offset_in_page(xdr->page_base); in rpcrdma_prepare_pagelist()
607 remaining = xdr->page_len; in rpcrdma_prepare_pagelist()
609 sge = &sc->sc_sges[req->rl_wr.num_sge++]; in rpcrdma_prepare_pagelist()
610 len = min_t(unsigned int, PAGE_SIZE - page_base, remaining); in rpcrdma_prepare_pagelist()
611 sge->addr = ib_dma_map_page(rdmab_device(rb), *ppages, in rpcrdma_prepare_pagelist()
613 if (ib_dma_mapping_error(rdmab_device(rb), sge->addr)) in rpcrdma_prepare_pagelist()
616 sge->length = len; in rpcrdma_prepare_pagelist()
617 sge->lkey = rdmab_lkey(rb); in rpcrdma_prepare_pagelist()
619 sc->sc_unmap_count++; in rpcrdma_prepare_pagelist()
621 remaining -= len; in rpcrdma_prepare_pagelist()
628 trace_xprtrdma_dma_maperr(sge->addr); in rpcrdma_prepare_pagelist()
640 struct rpcrdma_sendctx *sc = req->rl_sendctx; in rpcrdma_prepare_tail_iov()
641 struct ib_sge *sge = &sc->sc_sges[req->rl_wr.num_sge++]; in rpcrdma_prepare_tail_iov()
642 struct rpcrdma_regbuf *rb = req->rl_sendbuf; in rpcrdma_prepare_tail_iov()
643 struct page *page = virt_to_page(xdr->tail[0].iov_base); in rpcrdma_prepare_tail_iov()
645 sge->addr = ib_dma_map_page(rdmab_device(rb), page, page_base, len, in rpcrdma_prepare_tail_iov()
647 if (ib_dma_mapping_error(rdmab_device(rb), sge->addr)) in rpcrdma_prepare_tail_iov()
650 sge->length = len; in rpcrdma_prepare_tail_iov()
651 sge->lkey = rdmab_lkey(rb); in rpcrdma_prepare_tail_iov()
652 ++sc->sc_unmap_count; in rpcrdma_prepare_tail_iov()
656 trace_xprtrdma_dma_maperr(sge->addr); in rpcrdma_prepare_tail_iov()
668 dst = (unsigned char *)xdr->head[0].iov_base; in rpcrdma_pullup_tail_iov()
669 dst += xdr->head[0].iov_len + xdr->page_len; in rpcrdma_pullup_tail_iov()
670 memmove(dst, xdr->tail[0].iov_base, xdr->tail[0].iov_len); in rpcrdma_pullup_tail_iov()
671 r_xprt->rx_stats.pullup_copy_count += xdr->tail[0].iov_len; in rpcrdma_pullup_tail_iov()
684 dst = (unsigned char *)xdr->head[0].iov_base; in rpcrdma_pullup_pagelist()
685 dst += xdr->head[0].iov_len; in rpcrdma_pullup_pagelist()
686 ppages = xdr->pages + (xdr->page_base >> PAGE_SHIFT); in rpcrdma_pullup_pagelist()
687 page_base = offset_in_page(xdr->page_base); in rpcrdma_pullup_pagelist()
688 remaining = xdr->page_len; in rpcrdma_pullup_pagelist()
692 len = min_t(unsigned int, PAGE_SIZE - page_base, remaining); in rpcrdma_pullup_pagelist()
694 r_xprt->rx_stats.pullup_copy_count += len; in rpcrdma_pullup_pagelist()
698 remaining -= len; in rpcrdma_pullup_pagelist()
704 * When the head, pagelist, and tail are small, a pull-up copy
709 * - the caller has already verified that the total length
710 * of the RPC Call body will fit into @rl_sendbuf.
716 if (unlikely(xdr->tail[0].iov_len)) in rpcrdma_prepare_noch_pullup()
719 if (unlikely(xdr->page_len)) in rpcrdma_prepare_noch_pullup()
722 /* The whole RPC message resides in the head iovec now */ in rpcrdma_prepare_noch_pullup()
723 return rpcrdma_prepare_head_iov(r_xprt, req, xdr->len); in rpcrdma_prepare_noch_pullup()
730 struct kvec *tail = &xdr->tail[0]; in rpcrdma_prepare_noch_mapped()
732 if (!rpcrdma_prepare_head_iov(r_xprt, req, xdr->head[0].iov_len)) in rpcrdma_prepare_noch_mapped()
734 if (xdr->page_len) in rpcrdma_prepare_noch_mapped()
735 if (!rpcrdma_prepare_pagelist(req, xdr)) in rpcrdma_prepare_noch_mapped()
737 if (tail->iov_len) in rpcrdma_prepare_noch_mapped()
738 if (!rpcrdma_prepare_tail_iov(req, xdr, in rpcrdma_prepare_noch_mapped()
739 offset_in_page(tail->iov_base), in rpcrdma_prepare_noch_mapped()
740 tail->iov_len)) in rpcrdma_prepare_noch_mapped()
743 if (req->rl_sendctx->sc_unmap_count) in rpcrdma_prepare_noch_mapped()
744 kref_get(&req->rl_kref); in rpcrdma_prepare_noch_mapped()
752 if (!rpcrdma_prepare_head_iov(r_xprt, req, xdr->head[0].iov_len)) in rpcrdma_prepare_readch()
755 /* If there is a Read chunk, the page list is being handled in rpcrdma_prepare_readch()
759 /* Do not include the tail if it is only an XDR pad */ in rpcrdma_prepare_readch()
760 if (xdr->tail[0].iov_len > 3) { in rpcrdma_prepare_readch()
763 /* If the content in the page list is an odd length, in rpcrdma_prepare_readch()
765 * the tail iovec. Force the tail's non-pad content to in rpcrdma_prepare_readch()
768 page_base = offset_in_page(xdr->tail[0].iov_base); in rpcrdma_prepare_readch()
769 len = xdr->tail[0].iov_len; in rpcrdma_prepare_readch()
771 len -= len & 3; in rpcrdma_prepare_readch()
772 if (!rpcrdma_prepare_tail_iov(req, xdr, page_base, len)) in rpcrdma_prepare_readch()
774 kref_get(&req->rl_kref); in rpcrdma_prepare_readch()
781 * rpcrdma_prepare_send_sges - Construct SGEs for a Send WR
783 * @req: context of RPC Call being marshalled
785 * @xdr: xdr_buf containing RPC Call
797 ret = -EAGAIN; in rpcrdma_prepare_send_sges()
798 req->rl_sendctx = rpcrdma_sendctx_get_locked(r_xprt); in rpcrdma_prepare_send_sges()
799 if (!req->rl_sendctx) in rpcrdma_prepare_send_sges()
801 req->rl_sendctx->sc_unmap_count = 0; in rpcrdma_prepare_send_sges()
802 req->rl_sendctx->sc_req = req; in rpcrdma_prepare_send_sges()
803 kref_init(&req->rl_kref); in rpcrdma_prepare_send_sges()
804 req->rl_wr.wr_cqe = &req->rl_sendctx->sc_cqe; in rpcrdma_prepare_send_sges()
805 req->rl_wr.sg_list = req->rl_sendctx->sc_sges; in rpcrdma_prepare_send_sges()
806 req->rl_wr.num_sge = 0; in rpcrdma_prepare_send_sges()
807 req->rl_wr.opcode = IB_WR_SEND; in rpcrdma_prepare_send_sges()
811 ret = -EIO; in rpcrdma_prepare_send_sges()
814 if (!rpcrdma_prepare_noch_pullup(r_xprt, req, xdr)) in rpcrdma_prepare_send_sges()
818 if (!rpcrdma_prepare_noch_mapped(r_xprt, req, xdr)) in rpcrdma_prepare_send_sges()
822 if (!rpcrdma_prepare_readch(r_xprt, req, xdr)) in rpcrdma_prepare_send_sges()
834 rpcrdma_sendctx_unmap(req->rl_sendctx); in rpcrdma_prepare_send_sges()
836 trace_xprtrdma_prepsend_failed(&req->rl_slot, ret); in rpcrdma_prepare_send_sges()
841 * rpcrdma_marshal_req - Marshal and send one RPC request
843 * @rqst: RPC request to be marshaled
845 * For the RPC in "rqst", this function:
846 * - Chooses the transfer mode (eg., RDMA_MSG or RDMA_NOMSG)
847 * - Registers Read, Write, and Reply chunks
848 * - Constructs the transport header
849 * - Posts a Send WR to send the transport header and request
852 * %0 if the RPC was sent successfully,
853 * %-ENOTCONN if the connection was lost,
854 * %-EAGAIN if the caller should call again with the same arguments,
855 * %-ENOBUFS if the caller should call again after a delay,
856 * %-EMSGSIZE if the transport header is too small,
857 * %-EIO if a permanent problem occurred while marshaling.
863 struct xdr_stream *xdr = &req->rl_stream; in rpcrdma_marshal_req()
865 struct xdr_buf *buf = &rqst->rq_snd_buf; in rpcrdma_marshal_req()
870 if (unlikely(rqst->rq_rcv_buf.flags & XDRBUF_SPARSE_PAGES)) { in rpcrdma_marshal_req()
871 ret = rpcrdma_alloc_sparse_pages(&rqst->rq_rcv_buf); in rpcrdma_marshal_req()
872 if (ret) in rpcrdma_marshal_req()
876 rpcrdma_set_xdrlen(&req->rl_hdrbuf, 0); in rpcrdma_marshal_req()
877 xdr_init_encode(xdr, &req->rl_hdrbuf, rdmab_data(req->rl_rdmabuf), in rpcrdma_marshal_req()
881 ret = -EMSGSIZE; in rpcrdma_marshal_req()
883 if (!p) in rpcrdma_marshal_req()
885 *p++ = rqst->rq_xid; in rpcrdma_marshal_req()
887 *p++ = r_xprt->rx_buf.rb_max_requests; in rpcrdma_marshal_req()
894 &rqst->rq_cred->cr_auth->au_flags); in rpcrdma_marshal_req()
899 * o If the expected result is under the inline threshold, all ops in rpcrdma_marshal_req()
903 * o Large non-read ops return as a single reply chunk. in rpcrdma_marshal_req()
905 if (rpcrdma_results_inline(r_xprt, rqst)) in rpcrdma_marshal_req()
907 else if ((ddp_allowed && rqst->rq_rcv_buf.flags & XDRBUF_READ) && in rpcrdma_marshal_req()
916 * o If the total request is under the inline threshold, all ops in rpcrdma_marshal_req()
920 * o Large non-write ops are sent with the entire message as a in rpcrdma_marshal_req()
921 * single read chunk (protocol 0-position special case). in rpcrdma_marshal_req()
924 * that both has a data payload, and whose non-data arguments in rpcrdma_marshal_req()
927 if (rpcrdma_args_inline(r_xprt, rqst)) { in rpcrdma_marshal_req()
929 rtype = buf->len < rdmab_length(req->rl_sendbuf) ? in rpcrdma_marshal_req()
931 } else if (ddp_allowed && buf->flags & XDRBUF_WRITE) { in rpcrdma_marshal_req()
935 r_xprt->rx_stats.nomsg_call_count++; in rpcrdma_marshal_req()
941 * of chunk lists in one RPC-over-RDMA Call message: in rpcrdma_marshal_req()
943 * - Read list in rpcrdma_marshal_req()
944 * - Write list in rpcrdma_marshal_req()
945 * - Reply chunk in rpcrdma_marshal_req()
946 * - Read list + Reply chunk in rpcrdma_marshal_req()
950 * - Read list + Write list in rpcrdma_marshal_req()
954 * - Write list + Reply chunk in rpcrdma_marshal_req()
955 * - Read list + Write list + Reply chunk in rpcrdma_marshal_req()
963 if (ret) in rpcrdma_marshal_req()
966 if (ret) in rpcrdma_marshal_req()
969 if (ret) in rpcrdma_marshal_req()
972 ret = rpcrdma_prepare_send_sges(r_xprt, req, req->rl_hdrbuf.len, in rpcrdma_marshal_req()
974 if (ret) in rpcrdma_marshal_req()
982 r_xprt->rx_stats.failed_marshal_count++; in rpcrdma_marshal_req()
991 buf->rb_credits = grant; in __rpcrdma_update_cwnd_locked()
992 xprt->cwnd = grant << RPC_CWNDSHIFT; in __rpcrdma_update_cwnd_locked()
997 struct rpc_xprt *xprt = &r_xprt->rx_xprt; in rpcrdma_update_cwnd()
999 spin_lock(&xprt->transport_lock); in rpcrdma_update_cwnd()
1000 __rpcrdma_update_cwnd_locked(xprt, &r_xprt->rx_buf, grant); in rpcrdma_update_cwnd()
1001 spin_unlock(&xprt->transport_lock); in rpcrdma_update_cwnd()
1005 * rpcrdma_reset_cwnd - Reset the xprt's congestion window
1013 struct rpc_xprt *xprt = &r_xprt->rx_xprt; in rpcrdma_reset_cwnd()
1015 spin_lock(&xprt->transport_lock); in rpcrdma_reset_cwnd()
1016 xprt->cong = 0; in rpcrdma_reset_cwnd()
1017 __rpcrdma_update_cwnd_locked(xprt, &r_xprt->rx_buf, 1); in rpcrdma_reset_cwnd()
1018 spin_unlock(&xprt->transport_lock); in rpcrdma_reset_cwnd()
1022 * rpcrdma_inline_fixup - Scatter inline received data into rqst's iovecs
1023 * @rqst: controlling RPC request
1024 * @srcp: points to RPC message payload in receive buffer
1048 /* The head iovec is redirected to the RPC reply message in rpcrdma_inline_fixup()
1051 rqst->rq_rcv_buf.head[0].iov_base = srcp; in rpcrdma_inline_fixup()
1052 rqst->rq_private_buf.head[0].iov_base = srcp; in rpcrdma_inline_fixup()
1057 curlen = rqst->rq_rcv_buf.head[0].iov_len; in rpcrdma_inline_fixup()
1058 if (curlen > copy_len) in rpcrdma_inline_fixup()
1061 copy_len -= curlen; in rpcrdma_inline_fixup()
1063 ppages = rqst->rq_rcv_buf.pages + in rpcrdma_inline_fixup()
1064 (rqst->rq_rcv_buf.page_base >> PAGE_SHIFT); in rpcrdma_inline_fixup()
1065 page_base = offset_in_page(rqst->rq_rcv_buf.page_base); in rpcrdma_inline_fixup()
1067 if (copy_len && rqst->rq_rcv_buf.page_len) { in rpcrdma_inline_fixup()
1070 pagelist_len = rqst->rq_rcv_buf.page_len; in rpcrdma_inline_fixup()
1071 if (pagelist_len > copy_len) in rpcrdma_inline_fixup()
1075 curlen = PAGE_SIZE - page_base; in rpcrdma_inline_fixup()
1076 if (curlen > pagelist_len) in rpcrdma_inline_fixup()
1084 copy_len -= curlen; in rpcrdma_inline_fixup()
1086 pagelist_len -= curlen; in rpcrdma_inline_fixup()
1087 if (!pagelist_len) in rpcrdma_inline_fixup()
1098 if (pad) in rpcrdma_inline_fixup()
1099 srcp -= pad; in rpcrdma_inline_fixup()
1105 if (copy_len || pad) { in rpcrdma_inline_fixup()
1106 rqst->rq_rcv_buf.tail[0].iov_base = srcp; in rpcrdma_inline_fixup()
1107 rqst->rq_private_buf.tail[0].iov_base = srcp; in rpcrdma_inline_fixup()
1110 if (fixup_copy_count) in rpcrdma_inline_fixup()
1117 * the RPC/RDMA header small and fixed in size, so it is
1118 * straightforward to check the RPC header's direction field.
1122 #if defined(CONFIG_SUNRPC_BACKCHANNEL) in rpcrdma_is_bcall()
1124 struct rpc_xprt *xprt = &r_xprt->rx_xprt; in rpcrdma_is_bcall()
1125 struct xdr_stream *xdr = &rep->rr_stream; in rpcrdma_is_bcall()
1128 if (rep->rr_proc != rdma_msg) in rpcrdma_is_bcall()
1135 if (xdr_item_is_present(p++)) in rpcrdma_is_bcall()
1137 if (xdr_item_is_present(p++)) in rpcrdma_is_bcall()
1139 if (xdr_item_is_present(p++)) in rpcrdma_is_bcall()
1142 /* RPC header */ in rpcrdma_is_bcall()
1143 if (*p++ != rep->rr_xid) in rpcrdma_is_bcall()
1145 if (*p != cpu_to_be32(RPC_CALL)) in rpcrdma_is_bcall()
1149 if (xprt->bc_serv == NULL) in rpcrdma_is_bcall()
1153 * advance to the RPC header. in rpcrdma_is_bcall()
1156 if (unlikely(!p)) in rpcrdma_is_bcall()
1175 if (unlikely(!p)) in decode_rdma_segment()
1176 return -EIO; in decode_rdma_segment()
1189 if (unlikely(!p)) in decode_write_chunk()
1190 return -EIO; in decode_write_chunk()
1194 while (segcount--) { in decode_write_chunk()
1195 if (decode_rdma_segment(xdr, &seglength)) in decode_write_chunk()
1196 return -EIO; in decode_write_chunk()
1203 /* In RPC-over-RDMA Version One replies, a Read list is never
1204 * expected. This decoder is a stub that returns an error if
1212 if (unlikely(!p)) in decode_read_list()
1213 return -EIO; in decode_read_list()
1214 if (unlikely(xdr_item_is_present(p))) in decode_read_list()
1215 return -EIO; in decode_read_list()
1231 if (unlikely(!p)) in decode_write_list()
1232 return -EIO; in decode_write_list()
1233 if (xdr_item_is_absent(p)) in decode_write_list()
1235 if (!first) in decode_write_list()
1236 return -EIO; in decode_write_list()
1238 if (decode_write_chunk(xdr, &chunklen)) in decode_write_list()
1239 return -EIO; in decode_write_list()
1251 if (unlikely(!p)) in decode_reply_chunk()
1252 return -EIO; in decode_reply_chunk()
1255 if (xdr_item_is_present(p)) in decode_reply_chunk()
1256 if (decode_write_chunk(xdr, length)) in decode_reply_chunk()
1257 return -EIO; in decode_reply_chunk()
1265 struct xdr_stream *xdr = &rep->rr_stream; in rpcrdma_decode_msg()
1270 if (decode_read_list(xdr)) in rpcrdma_decode_msg()
1271 return -EIO; in rpcrdma_decode_msg()
1272 if (decode_write_list(xdr, &writelist)) in rpcrdma_decode_msg()
1273 return -EIO; in rpcrdma_decode_msg()
1274 if (decode_reply_chunk(xdr, &replychunk)) in rpcrdma_decode_msg()
1275 return -EIO; in rpcrdma_decode_msg()
1278 if (unlikely(replychunk)) in rpcrdma_decode_msg()
1279 return -EIO; in rpcrdma_decode_msg()
1281 /* Build the RPC reply's Payload stream in rqst->rq_rcv_buf */ in rpcrdma_decode_msg()
1284 r_xprt->rx_stats.fixup_copy_count += in rpcrdma_decode_msg()
1287 r_xprt->rx_stats.total_rdma_reply += writelist; in rpcrdma_decode_msg()
1294 struct xdr_stream *xdr = &rep->rr_stream; in rpcrdma_decode_nomsg()
1298 if (decode_read_list(xdr)) in rpcrdma_decode_nomsg()
1299 return -EIO; in rpcrdma_decode_nomsg()
1300 if (decode_write_list(xdr, &writelist)) in rpcrdma_decode_nomsg()
1301 return -EIO; in rpcrdma_decode_nomsg()
1302 if (decode_reply_chunk(xdr, &replychunk)) in rpcrdma_decode_nomsg()
1303 return -EIO; in rpcrdma_decode_nomsg()
1306 if (unlikely(writelist)) in rpcrdma_decode_nomsg()
1307 return -EIO; in rpcrdma_decode_nomsg()
1308 if (unlikely(!replychunk)) in rpcrdma_decode_nomsg()
1309 return -EIO; in rpcrdma_decode_nomsg()
1312 r_xprt->rx_stats.total_rdma_reply += replychunk; in rpcrdma_decode_nomsg()
1320 struct xdr_stream *xdr = &rep->rr_stream; in rpcrdma_decode_error()
1324 if (unlikely(!p)) in rpcrdma_decode_error()
1325 return -EIO; in rpcrdma_decode_error()
1330 if (!p) in rpcrdma_decode_error()
1341 return -EIO; in rpcrdma_decode_error()
1345 * rpcrdma_unpin_rqst - Release rqst without completing it
1346 * @rep: RPC/RDMA Receive context
1354 struct rpc_xprt *xprt = &rep->rr_rxprt->rx_xprt; in rpcrdma_unpin_rqst()
1355 struct rpc_rqst *rqst = rep->rr_rqst; in rpcrdma_unpin_rqst()
1358 req->rl_reply = NULL; in rpcrdma_unpin_rqst()
1359 rep->rr_rqst = NULL; in rpcrdma_unpin_rqst()
1361 spin_lock(&xprt->queue_lock); in rpcrdma_unpin_rqst()
1363 spin_unlock(&xprt->queue_lock); in rpcrdma_unpin_rqst()
1367 * rpcrdma_complete_rqst - Pass completed rqst back to RPC
1368 * @rep: RPC/RDMA Receive context
1370 * Reconstruct the RPC reply and complete the transaction
1376 struct rpcrdma_xprt *r_xprt = rep->rr_rxprt; in rpcrdma_complete_rqst()
1377 struct rpc_xprt *xprt = &r_xprt->rx_xprt; in rpcrdma_complete_rqst()
1378 struct rpc_rqst *rqst = rep->rr_rqst; in rpcrdma_complete_rqst()
1381 switch (rep->rr_proc) { in rpcrdma_complete_rqst()
1392 status = -EIO; in rpcrdma_complete_rqst()
1394 if (status < 0) in rpcrdma_complete_rqst()
1398 spin_lock(&xprt->queue_lock); in rpcrdma_complete_rqst()
1399 xprt_complete_rqst(rqst->rq_task, status); in rpcrdma_complete_rqst()
1401 spin_unlock(&xprt->queue_lock); in rpcrdma_complete_rqst()
1406 r_xprt->rx_stats.bad_reply_count++; in rpcrdma_complete_rqst()
1407 rqst->rq_task->tk_status = status; in rpcrdma_complete_rqst()
1417 rpcrdma_complete_rqst(req->rl_reply); in rpcrdma_reply_done()
1421 * rpcrdma_reply_handler - Process received RPC/RDMA messages
1424 * Errors must result in the RPC task either being awakened, or
1429 struct rpcrdma_xprt *r_xprt = rep->rr_rxprt; in rpcrdma_reply_handler()
1430 struct rpc_xprt *xprt = &r_xprt->rx_xprt; in rpcrdma_reply_handler()
1431 struct rpcrdma_buffer *buf = &r_xprt->rx_buf; in rpcrdma_reply_handler()
1440 if (xprt->reestablish_timeout) in rpcrdma_reply_handler()
1441 xprt->reestablish_timeout = 0; in rpcrdma_reply_handler()
1444 xdr_init_decode(&rep->rr_stream, &rep->rr_hdrbuf, in rpcrdma_reply_handler()
1445 rep->rr_hdrbuf.head[0].iov_base, NULL); in rpcrdma_reply_handler()
1446 p = xdr_inline_decode(&rep->rr_stream, 4 * sizeof(*p)); in rpcrdma_reply_handler()
1447 if (unlikely(!p)) in rpcrdma_reply_handler()
1449 rep->rr_xid = *p++; in rpcrdma_reply_handler()
1450 rep->rr_vers = *p++; in rpcrdma_reply_handler()
1452 rep->rr_proc = *p++; in rpcrdma_reply_handler()
1454 if (rep->rr_vers != rpcrdma_version) in rpcrdma_reply_handler()
1457 if (rpcrdma_is_bcall(r_xprt, rep)) in rpcrdma_reply_handler()
1463 spin_lock(&xprt->queue_lock); in rpcrdma_reply_handler()
1464 rqst = xprt_lookup_rqst(xprt, rep->rr_xid); in rpcrdma_reply_handler()
1465 if (!rqst) in rpcrdma_reply_handler()
1468 spin_unlock(&xprt->queue_lock); in rpcrdma_reply_handler()
1470 if (credits == 0) in rpcrdma_reply_handler()
1472 else if (credits > r_xprt->rx_ep->re_max_requests) in rpcrdma_reply_handler()
1473 credits = r_xprt->rx_ep->re_max_requests; in rpcrdma_reply_handler()
1474 rpcrdma_post_recvs(r_xprt, credits + (buf->rb_bc_srv_max_requests << 1), in rpcrdma_reply_handler()
1476 if (buf->rb_credits != credits) in rpcrdma_reply_handler()
1480 if (unlikely(req->rl_reply)) in rpcrdma_reply_handler()
1481 rpcrdma_rep_put(buf, req->rl_reply); in rpcrdma_reply_handler()
1482 req->rl_reply = rep; in rpcrdma_reply_handler()
1483 rep->rr_rqst = rqst; in rpcrdma_reply_handler()
1485 trace_xprtrdma_reply(rqst->rq_task, rep, credits); in rpcrdma_reply_handler()
1487 if (rep->rr_wc_flags & IB_WC_WITH_INVALIDATE) in rpcrdma_reply_handler()
1488 frwr_reminv(rep, &req->rl_registered); in rpcrdma_reply_handler()
1489 if (!list_empty(&req->rl_registered)) in rpcrdma_reply_handler()
1491 /* LocalInv completion will complete the RPC */ in rpcrdma_reply_handler()
1493 kref_put(&req->rl_kref, rpcrdma_reply_done); in rpcrdma_reply_handler()
1501 spin_unlock(&xprt->queue_lock); in rpcrdma_reply_handler()