1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
3  * Copyright (c) 2014-2017 Oracle.  All rights reserved.
4  * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved.
9  * COPYING in the main directory of this source tree, or the BSD-type
39  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45  * This file contains the top-level implementation of an RPC RDMA
49  * transport switch. All others are RPC RDMA internal.
74 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
82 static unsigned int max_memreg = RPCRDMA_LAST - 1;
153 	snprintf(buf, sizeof(buf), "%08x", ntohl(sin->sin_addr.s_addr));  in xprt_rdma_format_addresses4()
154 	xprt->address_strings[RPC_DISPLAY_HEX_ADDR] = kstrdup(buf, GFP_KERNEL);  in xprt_rdma_format_addresses4()
156 	xprt->address_strings[RPC_DISPLAY_NETID] = RPCBIND_NETID_RDMA;  in xprt_rdma_format_addresses4()
165 	snprintf(buf, sizeof(buf), "%pi6", &sin6->sin6_addr);  in xprt_rdma_format_addresses6()
166 	xprt->address_strings[RPC_DISPLAY_HEX_ADDR] = kstrdup(buf, GFP_KERNEL);  in xprt_rdma_format_addresses6()
168 	xprt->address_strings[RPC_DISPLAY_NETID] = RPCBIND_NETID_RDMA6;  in xprt_rdma_format_addresses6()
176 	switch (sap->sa_family) {  in xprt_rdma_format_addresses()
189 	xprt->address_strings[RPC_DISPLAY_ADDR] = kstrdup(buf, GFP_KERNEL);  in xprt_rdma_format_addresses()
192 	xprt->address_strings[RPC_DISPLAY_PORT] = kstrdup(buf, GFP_KERNEL);  in xprt_rdma_format_addresses()
195 	xprt->address_strings[RPC_DISPLAY_HEX_PORT] = kstrdup(buf, GFP_KERNEL);  in xprt_rdma_format_addresses()
197 	xprt->address_strings[RPC_DISPLAY_PROTO] = "rdma";  in xprt_rdma_format_addresses()
211 			kfree(xprt->address_strings[i]);  in xprt_rdma_free_addresses()
216  * xprt_rdma_connect_worker - establish connection in the background
220  * transport while a fresh connection is being established. RPC tasks
228 	struct rpc_xprt *xprt = &r_xprt->rx_xprt;  in xprt_rdma_connect_worker()
229 	unsigned int pflags = current->flags;  in xprt_rdma_connect_worker()
232 	if (atomic_read(&xprt->swapper))  in xprt_rdma_connect_worker()
233 		current->flags |= PF_MEMALLOC;  in xprt_rdma_connect_worker()
236 	if (!rc) {  in xprt_rdma_connect_worker()
237 		xprt->connect_cookie++;  in xprt_rdma_connect_worker()
238 		xprt->stat.connect_count++;  in xprt_rdma_connect_worker()
239 		xprt->stat.connect_time += (long)jiffies -  in xprt_rdma_connect_worker()
240 					   xprt->stat.connect_start;  in xprt_rdma_connect_worker()
242 		rc = -EAGAIN;  in xprt_rdma_connect_worker()
251  * xprt_rdma_inject_disconnect - inject a connection fault
254  * If @xprt is connected, disconnect it to simulate spurious
265 	rdma_disconnect(r_xprt->rx_ep->re_id);  in xprt_rdma_inject_disconnect()
269  * xprt_rdma_destroy - Full tear down of transport
280 	cancel_delayed_work_sync(&r_xprt->rx_connect_worker);  in xprt_rdma_destroy()
283 	rpcrdma_buffer_destroy(&r_xprt->rx_buf);  in xprt_rdma_destroy()
298  * xprt_setup_rdma - Set up transport to use RDMA
300  * @args: rpc transport arguments
310 	if (args->addrlen > sizeof(xprt->addr))  in xprt_setup_rdma()
311 		return ERR_PTR(-EBADF);  in xprt_setup_rdma()
313 	if (!try_module_get(THIS_MODULE))  in xprt_setup_rdma()
314 		return ERR_PTR(-EIO);  in xprt_setup_rdma()
316 	xprt = xprt_alloc(args->net, sizeof(struct rpcrdma_xprt), 0,  in xprt_setup_rdma()
318 	if (!xprt) {  in xprt_setup_rdma()
320 		return ERR_PTR(-ENOMEM);  in xprt_setup_rdma()
323 	xprt->timeout = &xprt_rdma_default_timeout;  in xprt_setup_rdma()
324 	xprt->connect_timeout = xprt->timeout->to_initval;  in xprt_setup_rdma()
325 	xprt->max_reconnect_timeout = xprt->timeout->to_maxval;  in xprt_setup_rdma()
326 	xprt->bind_timeout = RPCRDMA_BIND_TO;  in xprt_setup_rdma()
327 	xprt->reestablish_timeout = RPCRDMA_INIT_REEST_TO;  in xprt_setup_rdma()
328 	xprt->idle_timeout = RPCRDMA_IDLE_DISC_TO;  in xprt_setup_rdma()
330 	xprt->resvport = 0;		/* privileged port not needed */  in xprt_setup_rdma()
331 	xprt->ops = &xprt_rdma_procs;  in xprt_setup_rdma()
334 	 * Set up RDMA-specific connect data.  in xprt_setup_rdma()
336 	sap = args->dstaddr;  in xprt_setup_rdma()
338 	/* Ensure xprt->addr holds valid server TCP (not RDMA)  in xprt_setup_rdma()
340 	xprt->prot = IPPROTO_TCP;  in xprt_setup_rdma()
341 	xprt->xprt_class = &xprt_rdma;  in xprt_setup_rdma()
342 	xprt->addrlen = args->addrlen;  in xprt_setup_rdma()
343 	memcpy(&xprt->addr, sap, xprt->addrlen);  in xprt_setup_rdma()
345 	if (rpc_get_port(sap))  in xprt_setup_rdma()
351 	if (rc) {  in xprt_setup_rdma()
358 	INIT_DELAYED_WORK(&new_xprt->rx_connect_worker,  in xprt_setup_rdma()
361 	xprt->max_payload = RPCRDMA_MAX_DATA_SEGS << PAGE_SHIFT;  in xprt_setup_rdma()
367  * xprt_rdma_close - close a transport connection
381 	xprt->reestablish_timeout = 0;  in xprt_rdma_close()
382 	++xprt->connect_cookie;  in xprt_rdma_close()
387  * xprt_rdma_set_port - update server port with rpcbind result
388  * @xprt: controlling RPC transport
396 	struct sockaddr *sap = (struct sockaddr *)&xprt->addr;  in xprt_rdma_set_port()
401 	kfree(xprt->address_strings[RPC_DISPLAY_PORT]);  in xprt_rdma_set_port()
403 	xprt->address_strings[RPC_DISPLAY_PORT] = kstrdup(buf, GFP_KERNEL);  in xprt_rdma_set_port()
405 	kfree(xprt->address_strings[RPC_DISPLAY_HEX_PORT]);  in xprt_rdma_set_port()
407 	xprt->address_strings[RPC_DISPLAY_HEX_PORT] = kstrdup(buf, GFP_KERNEL);  in xprt_rdma_set_port()
411  * xprt_rdma_timer - invoked when an RPC times out
412  * @xprt: controlling RPC transport
413  * @task: RPC task that timed out
415  * Invoked when the transport is still connected, but an RPC
418  * Since RDMA connections don't have a keep-alive, forcibly
430  * xprt_rdma_set_connect_timeout - set timeouts for establishing a connection
444 	spin_lock(&xprt->transport_lock);  in xprt_rdma_set_connect_timeout()
446 	if (connect_timeout < xprt->connect_timeout) {  in xprt_rdma_set_connect_timeout()
450 		to = *xprt->timeout;  in xprt_rdma_set_connect_timeout()
452 		if (initval < RPCRDMA_INIT_REEST_TO << 1)  in xprt_rdma_set_connect_timeout()
456 		r_xprt->rx_timeout = to;  in xprt_rdma_set_connect_timeout()
457 		xprt->timeout = &r_xprt->rx_timeout;  in xprt_rdma_set_connect_timeout()
458 		xprt->connect_timeout = connect_timeout;  in xprt_rdma_set_connect_timeout()
461 	if (reconnect_timeout < xprt->max_reconnect_timeout)  in xprt_rdma_set_connect_timeout()
462 		xprt->max_reconnect_timeout = reconnect_timeout;  in xprt_rdma_set_connect_timeout()
464 	spin_unlock(&xprt->transport_lock);  in xprt_rdma_set_connect_timeout()
468  * xprt_rdma_connect - schedule an attempt to reconnect
470  * @task: RPC scheduler context (unused)
477 	struct rpcrdma_ep *ep = r_xprt->rx_ep;  in xprt_rdma_connect()
483 	if (ep && ep->re_connect_status != 0) {  in xprt_rdma_connect()
488 	queue_delayed_work(system_long_wq, &r_xprt->rx_connect_worker, delay);  in xprt_rdma_connect()
492  * xprt_rdma_alloc_slot - allocate an rpc_rqst
493  * @xprt: controlling RPC transport
494  * @task: RPC task requesting a fresh rpc_rqst
497  *	%0 if task->tk_rqstp points to a fresh rpc_rqst
498  *	%-EAGAIN if no rpc_rqst is available; queued on backlog
506 	req = rpcrdma_buffer_get(&r_xprt->rx_buf);  in xprt_rdma_alloc_slot()
507 	if (!req)  in xprt_rdma_alloc_slot()
509 	task->tk_rqstp = &req->rl_slot;  in xprt_rdma_alloc_slot()
510 	task->tk_status = 0;  in xprt_rdma_alloc_slot()
514 	task->tk_status = -ENOMEM;  in xprt_rdma_alloc_slot()
519  * xprt_rdma_free_slot - release an rpc_rqst
520  * @xprt: controlling RPC transport
530 	rpcrdma_reply_put(&r_xprt->rx_buf, rpcr_to_rdmar(rqst));  in xprt_rdma_free_slot()
531 	if (!xprt_wake_up_backlog(xprt, rqst)) {  in xprt_rdma_free_slot()
533 		rpcrdma_buffer_put(&r_xprt->rx_buf, rpcr_to_rdmar(rqst));  in xprt_rdma_free_slot()
541 	if (unlikely(rdmab_length(rb) < size)) {  in rpcrdma_check_regbuf()
542 		if (!rpcrdma_regbuf_realloc(rb, size, flags))  in rpcrdma_check_regbuf()
544 		r_xprt->rx_stats.hardway_register_count += size;  in rpcrdma_check_regbuf()
550  * xprt_rdma_allocate - allocate transport resources for an RPC
551  * @task: RPC task
554  *        0:	Success; rq_buffer points to RPC buffer to use
561 	struct rpc_rqst *rqst = task->tk_rqstp;  in xprt_rdma_allocate()
562 	struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(rqst->rq_xprt);  in xprt_rdma_allocate()
566 	if (!rpcrdma_check_regbuf(r_xprt, req->rl_sendbuf, rqst->rq_callsize,  in xprt_rdma_allocate()
569 	if (!rpcrdma_check_regbuf(r_xprt, req->rl_recvbuf, rqst->rq_rcvsize,  in xprt_rdma_allocate()
573 	rqst->rq_buffer = rdmab_data(req->rl_sendbuf);  in xprt_rdma_allocate()
574 	rqst->rq_rbuffer = rdmab_data(req->rl_recvbuf);  in xprt_rdma_allocate()
578 	return -ENOMEM;  in xprt_rdma_allocate()
582  * xprt_rdma_free - release resources allocated by xprt_rdma_allocate
583  * @task: RPC task
585  * Caller guarantees rqst->rq_buffer is non-NULL.
590 	struct rpc_rqst *rqst = task->tk_rqstp;  in xprt_rdma_free()
593 	if (unlikely(!list_empty(&req->rl_registered))) {  in xprt_rdma_free()
595 		frwr_unmap_sync(rpcx_to_rdmax(rqst->rq_xprt), req);  in xprt_rdma_free()
598 	/* XXX: If the RPC is completing because of a signal and  in xprt_rdma_free()
606  * xprt_rdma_send_request - marshal and send an RPC request
607  * @rqst: RPC message in rq_snd_buf
612  *	%0 if the RPC message has been sent
613  *	%-ENOTCONN if the caller should reconnect and call again
614  *	%-EAGAIN if the caller should call again
615  *	%-ENOBUFS if the caller should call again after a delay
616  *	%-EMSGSIZE if encoding ran out of buffer space. The request
618  *	%-EIO if an I/O error occurred. The request was not sent.
624 	struct rpc_xprt *xprt = rqst->rq_xprt;  in xprt_rdma_send_request()
629 #if defined(CONFIG_SUNRPC_BACKCHANNEL)  in xprt_rdma_send_request()
630 	if (unlikely(!rqst->rq_buffer))  in xprt_rdma_send_request()
634 	if (!xprt_connected(xprt))  in xprt_rdma_send_request()
635 		return -ENOTCONN;  in xprt_rdma_send_request()
637 	if (!xprt_request_get_cong(xprt, rqst))  in xprt_rdma_send_request()
638 		return -EBADSLT;  in xprt_rdma_send_request()
641 	if (rc < 0)  in xprt_rdma_send_request()
645 	if (rqst->rq_connect_cookie == xprt->connect_cookie)  in xprt_rdma_send_request()
647 	rqst->rq_xtime = ktime_get();  in xprt_rdma_send_request()
649 	if (frwr_send(r_xprt, req))  in xprt_rdma_send_request()
652 	rqst->rq_xmit_bytes_sent += rqst->rq_snd_buf.len;  in xprt_rdma_send_request()
654 	/* An RPC with no reply will throw off credit accounting,  in xprt_rdma_send_request()
657 	if (!rpc_reply_expected(rqst->rq_task))  in xprt_rdma_send_request()
662 	if (rc != -ENOTCONN)  in xprt_rdma_send_request()
666 	return -ENOTCONN;  in xprt_rdma_send_request()
674 	if (xprt_connected(xprt))  in xprt_rdma_print_stats()
675 		idle_time = (long)(jiffies - xprt->last_used) / HZ;  in xprt_rdma_print_stats()
680 		   xprt->stat.bind_count,  in xprt_rdma_print_stats()
681 		   xprt->stat.connect_count,  in xprt_rdma_print_stats()
682 		   xprt->stat.connect_time / HZ,  in xprt_rdma_print_stats()
684 		   xprt->stat.sends,  in xprt_rdma_print_stats()
685 		   xprt->stat.recvs,  in xprt_rdma_print_stats()
686 		   xprt->stat.bad_xids,  in xprt_rdma_print_stats()
687 		   xprt->stat.req_u,  in xprt_rdma_print_stats()
688 		   xprt->stat.bklog_u);  in xprt_rdma_print_stats()
690 		   r_xprt->rx_stats.read_chunk_count,  in xprt_rdma_print_stats()
691 		   r_xprt->rx_stats.write_chunk_count,  in xprt_rdma_print_stats()
692 		   r_xprt->rx_stats.reply_chunk_count,  in xprt_rdma_print_stats()
693 		   r_xprt->rx_stats.total_rdma_request,  in xprt_rdma_print_stats()
694 		   r_xprt->rx_stats.total_rdma_reply,  in xprt_rdma_print_stats()
695 		   r_xprt->rx_stats.pullup_copy_count,  in xprt_rdma_print_stats()
696 		   r_xprt->rx_stats.fixup_copy_count,  in xprt_rdma_print_stats()
697 		   r_xprt->rx_stats.hardway_register_count,  in xprt_rdma_print_stats()
698 		   r_xprt->rx_stats.failed_marshal_count,  in xprt_rdma_print_stats()
699 		   r_xprt->rx_stats.bad_reply_count,  in xprt_rdma_print_stats()
700 		   r_xprt->rx_stats.nomsg_call_count);  in xprt_rdma_print_stats()
702 		   r_xprt->rx_stats.mrs_recycled,  in xprt_rdma_print_stats()
703 		   r_xprt->rx_stats.mrs_orphaned,  in xprt_rdma_print_stats()
704 		   r_xprt->rx_stats.mrs_allocated,  in xprt_rdma_print_stats()
705 		   r_xprt->rx_stats.local_inv_needed,  in xprt_rdma_print_stats()
706 		   r_xprt->rx_stats.empty_sendctx_q,  in xprt_rdma_print_stats()
707 		   r_xprt->rx_stats.reply_waits_for_send);  in xprt_rdma_print_stats()
722  * Plumbing for rpc transport switch and kernel module
746 #if defined(CONFIG_SUNRPC_BACKCHANNEL)
766 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)  in xprt_rdma_cleanup()
767 	if (sunrpc_table_header) {  in xprt_rdma_cleanup()
782 	if (rc)  in xprt_rdma_init()
786 	if (rc) {  in xprt_rdma_init()
791 #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)  in xprt_rdma_init()
792 	if (!sunrpc_table_header)  in xprt_rdma_init()