Lines Matching +full:rpc +full:- +full:if
1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
3 * Copyright (c) 2014-2017 Oracle. All rights reserved.
4 * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved.
9 * COPYING in the main directory of this source tree, or the BSD-type
39 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45 * This file contains the top-level implementation of an RPC RDMA
49 * transport switch. All others are RPC RDMA internal.
63 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
77 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
85 static unsigned int max_memreg = RPCRDMA_LAST - 1;
165 snprintf(buf, sizeof(buf), "%08x", ntohl(sin->sin_addr.s_addr)); in xprt_rdma_format_addresses4()
166 xprt->address_strings[RPC_DISPLAY_HEX_ADDR] = kstrdup(buf, GFP_KERNEL); in xprt_rdma_format_addresses4()
168 xprt->address_strings[RPC_DISPLAY_NETID] = RPCBIND_NETID_RDMA; in xprt_rdma_format_addresses4()
177 snprintf(buf, sizeof(buf), "%pi6", &sin6->sin6_addr); in xprt_rdma_format_addresses6()
178 xprt->address_strings[RPC_DISPLAY_HEX_ADDR] = kstrdup(buf, GFP_KERNEL); in xprt_rdma_format_addresses6()
180 xprt->address_strings[RPC_DISPLAY_NETID] = RPCBIND_NETID_RDMA6; in xprt_rdma_format_addresses6()
188 switch (sap->sa_family) { in xprt_rdma_format_addresses()
201 xprt->address_strings[RPC_DISPLAY_ADDR] = kstrdup(buf, GFP_KERNEL); in xprt_rdma_format_addresses()
204 xprt->address_strings[RPC_DISPLAY_PORT] = kstrdup(buf, GFP_KERNEL); in xprt_rdma_format_addresses()
207 xprt->address_strings[RPC_DISPLAY_HEX_PORT] = kstrdup(buf, GFP_KERNEL); in xprt_rdma_format_addresses()
209 xprt->address_strings[RPC_DISPLAY_PROTO] = "rdma"; in xprt_rdma_format_addresses()
223 kfree(xprt->address_strings[i]); in xprt_rdma_free_addresses()
228 * xprt_rdma_connect_worker - establish connection in the background
232 * transport while a fresh connection is being established. RPC tasks
240 struct rpc_xprt *xprt = &r_xprt->rx_xprt; in xprt_rdma_connect_worker()
245 if (!rc) { in xprt_rdma_connect_worker()
246 xprt->connect_cookie++; in xprt_rdma_connect_worker()
247 xprt->stat.connect_count++; in xprt_rdma_connect_worker()
248 xprt->stat.connect_time += (long)jiffies - in xprt_rdma_connect_worker()
249 xprt->stat.connect_start; in xprt_rdma_connect_worker()
251 rc = -EAGAIN; in xprt_rdma_connect_worker()
254 spin_lock(&xprt->transport_lock); in xprt_rdma_connect_worker()
255 set_bit(XPRT_CLOSE_WAIT, &xprt->state); in xprt_rdma_connect_worker()
256 spin_unlock(&xprt->transport_lock); in xprt_rdma_connect_worker()
262 * xprt_rdma_inject_disconnect - inject a connection fault
265 * If @xprt is connected, disconnect it to simulate spurious connection
274 rdma_disconnect(r_xprt->rx_ep->re_id); in xprt_rdma_inject_disconnect()
278 * xprt_rdma_destroy - Full tear down of transport
289 cancel_delayed_work_sync(&r_xprt->rx_connect_worker); in xprt_rdma_destroy()
292 rpcrdma_buffer_destroy(&r_xprt->rx_buf); in xprt_rdma_destroy()
307 * xprt_setup_rdma - Set up transport to use RDMA
309 * @args: rpc transport arguments
319 if (args->addrlen > sizeof(xprt->addr)) in xprt_setup_rdma()
320 return ERR_PTR(-EBADF); in xprt_setup_rdma()
322 if (!try_module_get(THIS_MODULE)) in xprt_setup_rdma()
323 return ERR_PTR(-EIO); in xprt_setup_rdma()
325 xprt = xprt_alloc(args->net, sizeof(struct rpcrdma_xprt), 0, in xprt_setup_rdma()
327 if (!xprt) { in xprt_setup_rdma()
329 return ERR_PTR(-ENOMEM); in xprt_setup_rdma()
332 xprt->timeout = &xprt_rdma_default_timeout; in xprt_setup_rdma()
333 xprt->connect_timeout = xprt->timeout->to_initval; in xprt_setup_rdma()
334 xprt->max_reconnect_timeout = xprt->timeout->to_maxval; in xprt_setup_rdma()
335 xprt->bind_timeout = RPCRDMA_BIND_TO; in xprt_setup_rdma()
336 xprt->reestablish_timeout = RPCRDMA_INIT_REEST_TO; in xprt_setup_rdma()
337 xprt->idle_timeout = RPCRDMA_IDLE_DISC_TO; in xprt_setup_rdma()
339 xprt->resvport = 0; /* privileged port not needed */ in xprt_setup_rdma()
340 xprt->ops = &xprt_rdma_procs; in xprt_setup_rdma()
343 * Set up RDMA-specific connect data. in xprt_setup_rdma()
345 sap = args->dstaddr; in xprt_setup_rdma()
347 /* Ensure xprt->addr holds valid server TCP (not RDMA) in xprt_setup_rdma()
349 xprt->prot = IPPROTO_TCP; in xprt_setup_rdma()
350 xprt->addrlen = args->addrlen; in xprt_setup_rdma()
351 memcpy(&xprt->addr, sap, xprt->addrlen); in xprt_setup_rdma()
353 if (rpc_get_port(sap)) in xprt_setup_rdma()
359 if (rc) { in xprt_setup_rdma()
366 INIT_DELAYED_WORK(&new_xprt->rx_connect_worker, in xprt_setup_rdma()
369 xprt->max_payload = RPCRDMA_MAX_DATA_SEGS << PAGE_SHIFT; in xprt_setup_rdma()
375 * xprt_rdma_close - close a transport connection
389 xprt->reestablish_timeout = 0; in xprt_rdma_close()
390 ++xprt->connect_cookie; in xprt_rdma_close()
395 * xprt_rdma_set_port - update server port with rpcbind result
396 * @xprt: controlling RPC transport
404 struct sockaddr *sap = (struct sockaddr *)&xprt->addr; in xprt_rdma_set_port()
409 kfree(xprt->address_strings[RPC_DISPLAY_PORT]); in xprt_rdma_set_port()
411 xprt->address_strings[RPC_DISPLAY_PORT] = kstrdup(buf, GFP_KERNEL); in xprt_rdma_set_port()
413 kfree(xprt->address_strings[RPC_DISPLAY_HEX_PORT]); in xprt_rdma_set_port()
415 xprt->address_strings[RPC_DISPLAY_HEX_PORT] = kstrdup(buf, GFP_KERNEL); in xprt_rdma_set_port()
419 * xprt_rdma_timer - invoked when an RPC times out
420 * @xprt: controlling RPC transport
421 * @task: RPC task that timed out
423 * Invoked when the transport is still connected, but an RPC
426 * Since RDMA connections don't have a keep-alive, forcibly
438 * xprt_rdma_set_connect_timeout - set timeouts for establishing a connection
452 spin_lock(&xprt->transport_lock); in xprt_rdma_set_connect_timeout()
454 if (connect_timeout < xprt->connect_timeout) { in xprt_rdma_set_connect_timeout()
458 to = *xprt->timeout; in xprt_rdma_set_connect_timeout()
460 if (initval < RPCRDMA_INIT_REEST_TO << 1) in xprt_rdma_set_connect_timeout()
464 r_xprt->rx_timeout = to; in xprt_rdma_set_connect_timeout()
465 xprt->timeout = &r_xprt->rx_timeout; in xprt_rdma_set_connect_timeout()
466 xprt->connect_timeout = connect_timeout; in xprt_rdma_set_connect_timeout()
469 if (reconnect_timeout < xprt->max_reconnect_timeout) in xprt_rdma_set_connect_timeout()
470 xprt->max_reconnect_timeout = reconnect_timeout; in xprt_rdma_set_connect_timeout()
472 spin_unlock(&xprt->transport_lock); in xprt_rdma_set_connect_timeout()
476 * xprt_rdma_connect - schedule an attempt to reconnect
478 * @task: RPC scheduler context (unused)
485 struct rpcrdma_ep *ep = r_xprt->rx_ep; in xprt_rdma_connect()
489 if (ep && ep->re_connect_status != 0) { in xprt_rdma_connect()
494 queue_delayed_work(xprtiod_workqueue, &r_xprt->rx_connect_worker, in xprt_rdma_connect()
499 * xprt_rdma_alloc_slot - allocate an rpc_rqst
500 * @xprt: controlling RPC transport
501 * @task: RPC task requesting a fresh rpc_rqst
504 * %0 if task->tk_rqstp points to a fresh rpc_rqst
505 * %-EAGAIN if no rpc_rqst is available; queued on backlog
513 req = rpcrdma_buffer_get(&r_xprt->rx_buf); in xprt_rdma_alloc_slot()
514 if (!req) in xprt_rdma_alloc_slot()
516 task->tk_rqstp = &req->rl_slot; in xprt_rdma_alloc_slot()
517 task->tk_status = 0; in xprt_rdma_alloc_slot()
521 set_bit(XPRT_CONGESTED, &xprt->state); in xprt_rdma_alloc_slot()
522 rpc_sleep_on(&xprt->backlog, task, NULL); in xprt_rdma_alloc_slot()
523 task->tk_status = -EAGAIN; in xprt_rdma_alloc_slot()
527 * xprt_rdma_free_slot - release an rpc_rqst
528 * @xprt: controlling RPC transport
539 rpcrdma_buffer_put(&r_xprt->rx_buf, rpcr_to_rdmar(rqst)); in xprt_rdma_free_slot()
540 if (unlikely(!rpc_wake_up_next(&xprt->backlog))) in xprt_rdma_free_slot()
541 clear_bit(XPRT_CONGESTED, &xprt->state); in xprt_rdma_free_slot()
548 if (unlikely(rdmab_length(rb) < size)) { in rpcrdma_check_regbuf()
549 if (!rpcrdma_regbuf_realloc(rb, size, flags)) in rpcrdma_check_regbuf()
551 r_xprt->rx_stats.hardway_register_count += size; in rpcrdma_check_regbuf()
557 * xprt_rdma_allocate - allocate transport resources for an RPC
558 * @task: RPC task
561 * 0: Success; rq_buffer points to RPC buffer to use
568 struct rpc_rqst *rqst = task->tk_rqstp; in xprt_rdma_allocate()
569 struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(rqst->rq_xprt); in xprt_rdma_allocate()
574 if (RPC_IS_SWAPPER(task)) in xprt_rdma_allocate()
577 if (!rpcrdma_check_regbuf(r_xprt, req->rl_sendbuf, rqst->rq_callsize, in xprt_rdma_allocate()
580 if (!rpcrdma_check_regbuf(r_xprt, req->rl_recvbuf, rqst->rq_rcvsize, in xprt_rdma_allocate()
584 rqst->rq_buffer = rdmab_data(req->rl_sendbuf); in xprt_rdma_allocate()
585 rqst->rq_rbuffer = rdmab_data(req->rl_recvbuf); in xprt_rdma_allocate()
589 return -ENOMEM; in xprt_rdma_allocate()
593 * xprt_rdma_free - release resources allocated by xprt_rdma_allocate
594 * @task: RPC task
596 * Caller guarantees rqst->rq_buffer is non-NULL.
601 struct rpc_rqst *rqst = task->tk_rqstp; in xprt_rdma_free()
602 struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(rqst->rq_xprt); in xprt_rdma_free()
605 if (!list_empty(&req->rl_registered)) in xprt_rdma_free()
608 /* XXX: If the RPC is completing because of a signal and in xprt_rdma_free()
616 * xprt_rdma_send_request - marshal and send an RPC request
617 * @rqst: RPC message in rq_snd_buf
622 * %0 if the RPC message has been sent
623 * %-ENOTCONN if the caller should reconnect and call again
624 * %-EAGAIN if the caller should call again
625 * %-ENOBUFS if the caller should call again after a delay
626 * %-EMSGSIZE if encoding ran out of buffer space. The request
628 * %-EIO if an I/O error occurred. The request was not sent.
634 struct rpc_xprt *xprt = rqst->rq_xprt; in xprt_rdma_send_request()
639 #if defined(CONFIG_SUNRPC_BACKCHANNEL) in xprt_rdma_send_request()
640 if (unlikely(!rqst->rq_buffer)) in xprt_rdma_send_request()
644 if (!xprt_connected(xprt)) in xprt_rdma_send_request()
645 return -ENOTCONN; in xprt_rdma_send_request()
647 if (!xprt_request_get_cong(xprt, rqst)) in xprt_rdma_send_request()
648 return -EBADSLT; in xprt_rdma_send_request()
651 if (rc < 0) in xprt_rdma_send_request()
655 if (rqst->rq_connect_cookie == xprt->connect_cookie) in xprt_rdma_send_request()
657 rqst->rq_xtime = ktime_get(); in xprt_rdma_send_request()
659 if (rpcrdma_post_sends(r_xprt, req)) in xprt_rdma_send_request()
662 rqst->rq_xmit_bytes_sent += rqst->rq_snd_buf.len; in xprt_rdma_send_request()
664 /* An RPC with no reply will throw off credit accounting, in xprt_rdma_send_request()
667 if (!rpc_reply_expected(rqst->rq_task)) in xprt_rdma_send_request()
672 if (rc != -ENOTCONN) in xprt_rdma_send_request()
676 return -ENOTCONN; in xprt_rdma_send_request()
684 if (xprt_connected(xprt)) in xprt_rdma_print_stats()
685 idle_time = (long)(jiffies - xprt->last_used) / HZ; in xprt_rdma_print_stats()
690 xprt->stat.bind_count, in xprt_rdma_print_stats()
691 xprt->stat.connect_count, in xprt_rdma_print_stats()
692 xprt->stat.connect_time / HZ, in xprt_rdma_print_stats()
694 xprt->stat.sends, in xprt_rdma_print_stats()
695 xprt->stat.recvs, in xprt_rdma_print_stats()
696 xprt->stat.bad_xids, in xprt_rdma_print_stats()
697 xprt->stat.req_u, in xprt_rdma_print_stats()
698 xprt->stat.bklog_u); in xprt_rdma_print_stats()
700 r_xprt->rx_stats.read_chunk_count, in xprt_rdma_print_stats()
701 r_xprt->rx_stats.write_chunk_count, in xprt_rdma_print_stats()
702 r_xprt->rx_stats.reply_chunk_count, in xprt_rdma_print_stats()
703 r_xprt->rx_stats.total_rdma_request, in xprt_rdma_print_stats()
704 r_xprt->rx_stats.total_rdma_reply, in xprt_rdma_print_stats()
705 r_xprt->rx_stats.pullup_copy_count, in xprt_rdma_print_stats()
706 r_xprt->rx_stats.fixup_copy_count, in xprt_rdma_print_stats()
707 r_xprt->rx_stats.hardway_register_count, in xprt_rdma_print_stats()
708 r_xprt->rx_stats.failed_marshal_count, in xprt_rdma_print_stats()
709 r_xprt->rx_stats.bad_reply_count, in xprt_rdma_print_stats()
710 r_xprt->rx_stats.nomsg_call_count); in xprt_rdma_print_stats()
712 r_xprt->rx_stats.mrs_recycled, in xprt_rdma_print_stats()
713 r_xprt->rx_stats.mrs_orphaned, in xprt_rdma_print_stats()
714 r_xprt->rx_stats.mrs_allocated, in xprt_rdma_print_stats()
715 r_xprt->rx_stats.local_inv_needed, in xprt_rdma_print_stats()
716 r_xprt->rx_stats.empty_sendctx_q, in xprt_rdma_print_stats()
717 r_xprt->rx_stats.reply_waits_for_send); in xprt_rdma_print_stats()
732 * Plumbing for rpc transport switch and kernel module
756 #if defined(CONFIG_SUNRPC_BACKCHANNEL)
775 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) in xprt_rdma_cleanup()
776 if (sunrpc_table_header) { in xprt_rdma_cleanup()
791 if (rc) in xprt_rdma_init()
795 if (rc) { in xprt_rdma_init()
800 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) in xprt_rdma_init()
801 if (!sunrpc_table_header) in xprt_rdma_init()