Lines Matching full:ic
60 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_set_flow_control() local
64 ic->i_flowctl = 1; in rds_ib_set_flow_control()
67 ic->i_flowctl = 0; in rds_ib_set_flow_control()
86 rds_ib_tune_rnr(struct rds_ib_connection *ic, struct ib_qp_attr *attr) in rds_ib_tune_rnr() argument
91 ret = ib_modify_qp(ic->i_cm_id->qp, attr, IB_QP_MIN_RNR_TIMER); in rds_ib_tune_rnr()
102 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_cm_connect_complete() local
151 ic->i_active_side ? "Active" : "Passive", in rds_ib_cm_connect_complete()
155 ic->i_flowctl ? ", flow control" : ""); in rds_ib_cm_connect_complete()
158 ic->i_sl = ic->i_cm_id->route.path_rec->sl; in rds_ib_cm_connect_complete()
160 atomic_set(&ic->i_cq_quiesce, 0); in rds_ib_cm_connect_complete()
166 rds_ib_send_init_ring(ic); in rds_ib_cm_connect_complete()
167 rds_ib_recv_init_ring(ic); in rds_ib_cm_connect_complete()
173 rds_ib_tune_rnr(ic, &qp_attr); in rds_ib_cm_connect_complete()
176 err = ib_modify_qp(ic->i_cm_id->qp, &qp_attr, IB_QP_STATE); in rds_ib_cm_connect_complete()
181 err = rds_ib_update_ipaddr(ic->rds_ibdev, &conn->c_laddr); in rds_ib_cm_connect_complete()
206 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_cm_fill_conn_param() local
207 struct rds_ib_device *rds_ibdev = ic->rds_ibdev; in rds_ib_cm_fill_conn_param()
230 cpu_to_be64(rds_ib_piggyb_ack(ic)); in rds_ib_cm_fill_conn_param()
245 cpu_to_be64(rds_ib_piggyb_ack(ic)); in rds_ib_cm_fill_conn_param()
253 if (ic->i_flowctl) { in rds_ib_cm_fill_conn_param()
257 (atomic_read(&ic->i_credits)); in rds_ib_cm_fill_conn_param()
263 &ic->i_credits); in rds_ib_cm_fill_conn_param()
285 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_cq_comp_handler_recv() local
291 tasklet_schedule(&ic->i_recv_tasklet); in rds_ib_cq_comp_handler_recv()
294 static void poll_scq(struct rds_ib_connection *ic, struct ib_cq *cq, in poll_scq() argument
307 if (wc->wr_id <= ic->i_send_ring.w_nr || in poll_scq()
309 rds_ib_send_cqe_handler(ic, wc); in poll_scq()
311 rds_ib_mr_cqe_handler(ic, wc); in poll_scq()
319 struct rds_ib_connection *ic = (struct rds_ib_connection *)data; in rds_ib_tasklet_fn_send() local
320 struct rds_connection *conn = ic->conn; in rds_ib_tasklet_fn_send()
325 if (atomic_read(&ic->i_cq_quiesce)) in rds_ib_tasklet_fn_send()
328 poll_scq(ic, ic->i_send_cq, ic->i_send_wc); in rds_ib_tasklet_fn_send()
329 ib_req_notify_cq(ic->i_send_cq, IB_CQ_NEXT_COMP); in rds_ib_tasklet_fn_send()
330 poll_scq(ic, ic->i_send_cq, ic->i_send_wc); in rds_ib_tasklet_fn_send()
335 rds_send_xmit(&ic->conn->c_path[0]); in rds_ib_tasklet_fn_send()
338 static void poll_rcq(struct rds_ib_connection *ic, struct ib_cq *cq, in poll_rcq() argument
352 rds_ib_recv_cqe_handler(ic, wc, ack_state); in poll_rcq()
359 struct rds_ib_connection *ic = (struct rds_ib_connection *)data; in rds_ib_tasklet_fn_recv() local
360 struct rds_connection *conn = ic->conn; in rds_ib_tasklet_fn_recv()
361 struct rds_ib_device *rds_ibdev = ic->rds_ibdev; in rds_ib_tasklet_fn_recv()
370 if (atomic_read(&ic->i_cq_quiesce)) in rds_ib_tasklet_fn_recv()
374 poll_rcq(ic, ic->i_recv_cq, ic->i_recv_wc, &state); in rds_ib_tasklet_fn_recv()
375 ib_req_notify_cq(ic->i_recv_cq, IB_CQ_SOLICITED); in rds_ib_tasklet_fn_recv()
376 poll_rcq(ic, ic->i_recv_cq, ic->i_recv_wc, &state); in rds_ib_tasklet_fn_recv()
379 rds_ib_set_ack(ic, state.ack_next, state.ack_required); in rds_ib_tasklet_fn_recv()
380 if (state.ack_recv_valid && state.ack_recv > ic->i_ack_recv) { in rds_ib_tasklet_fn_recv()
382 ic->i_ack_recv = state.ack_recv; in rds_ib_tasklet_fn_recv()
386 rds_ib_attempt_ack(ic); in rds_ib_tasklet_fn_recv()
392 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_qp_event_handler() local
394 rdsdebug("conn %p ic %p event %u (%s)\n", conn, ic, event->event, in rds_ib_qp_event_handler()
399 rdma_notify(ic->i_cm_id, IB_EVENT_COMM_EST); in rds_ib_qp_event_handler()
413 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_cq_comp_handler_send() local
419 tasklet_schedule(&ic->i_send_tasklet); in rds_ib_cq_comp_handler_send()
512 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_setup_qp() local
513 struct ib_device *dev = ic->i_cm_id->device; in rds_ib_setup_qp()
540 if (ic->i_send_ring.w_nr != max_wrs) in rds_ib_setup_qp()
541 rds_ib_ring_resize(&ic->i_send_ring, max_wrs); in rds_ib_setup_qp()
545 if (ic->i_recv_ring.w_nr != max_wrs) in rds_ib_setup_qp()
546 rds_ib_ring_resize(&ic->i_recv_ring, max_wrs); in rds_ib_setup_qp()
549 ic->i_pd = rds_ibdev->pd; in rds_ib_setup_qp()
551 ic->i_scq_vector = ibdev_get_unused_vector(rds_ibdev); in rds_ib_setup_qp()
552 cq_attr.cqe = ic->i_send_ring.w_nr + fr_queue_space + 1; in rds_ib_setup_qp()
553 cq_attr.comp_vector = ic->i_scq_vector; in rds_ib_setup_qp()
554 ic->i_send_cq = ib_create_cq(dev, rds_ib_cq_comp_handler_send, in rds_ib_setup_qp()
557 if (IS_ERR(ic->i_send_cq)) { in rds_ib_setup_qp()
558 ret = PTR_ERR(ic->i_send_cq); in rds_ib_setup_qp()
559 ic->i_send_cq = NULL; in rds_ib_setup_qp()
560 ibdev_put_vector(rds_ibdev, ic->i_scq_vector); in rds_ib_setup_qp()
565 ic->i_rcq_vector = ibdev_get_unused_vector(rds_ibdev); in rds_ib_setup_qp()
566 cq_attr.cqe = ic->i_recv_ring.w_nr; in rds_ib_setup_qp()
567 cq_attr.comp_vector = ic->i_rcq_vector; in rds_ib_setup_qp()
568 ic->i_recv_cq = ib_create_cq(dev, rds_ib_cq_comp_handler_recv, in rds_ib_setup_qp()
571 if (IS_ERR(ic->i_recv_cq)) { in rds_ib_setup_qp()
572 ret = PTR_ERR(ic->i_recv_cq); in rds_ib_setup_qp()
573 ic->i_recv_cq = NULL; in rds_ib_setup_qp()
574 ibdev_put_vector(rds_ibdev, ic->i_rcq_vector); in rds_ib_setup_qp()
579 ret = ib_req_notify_cq(ic->i_send_cq, IB_CQ_NEXT_COMP); in rds_ib_setup_qp()
585 ret = ib_req_notify_cq(ic->i_recv_cq, IB_CQ_SOLICITED); in rds_ib_setup_qp()
596 attr.cap.max_send_wr = ic->i_send_ring.w_nr + fr_queue_space + 1; in rds_ib_setup_qp()
597 attr.cap.max_recv_wr = ic->i_recv_ring.w_nr + 1; in rds_ib_setup_qp()
602 attr.send_cq = ic->i_send_cq; in rds_ib_setup_qp()
603 attr.recv_cq = ic->i_recv_cq; in rds_ib_setup_qp()
609 ret = rdma_create_qp(ic->i_cm_id, ic->i_pd, &attr); in rds_ib_setup_qp()
616 ic->i_send_hdrs = rds_dma_hdrs_alloc(dev, pool, &ic->i_send_hdrs_dma, in rds_ib_setup_qp()
617 ic->i_send_ring.w_nr); in rds_ib_setup_qp()
618 if (!ic->i_send_hdrs) { in rds_ib_setup_qp()
624 ic->i_recv_hdrs = rds_dma_hdrs_alloc(dev, pool, &ic->i_recv_hdrs_dma, in rds_ib_setup_qp()
625 ic->i_recv_ring.w_nr); in rds_ib_setup_qp()
626 if (!ic->i_recv_hdrs) { in rds_ib_setup_qp()
632 ic->i_ack = dma_pool_zalloc(pool, GFP_KERNEL, in rds_ib_setup_qp()
633 &ic->i_ack_dma); in rds_ib_setup_qp()
634 if (!ic->i_ack) { in rds_ib_setup_qp()
640 ic->i_sends = vzalloc_node(array_size(sizeof(struct rds_ib_send_work), in rds_ib_setup_qp()
641 ic->i_send_ring.w_nr), in rds_ib_setup_qp()
643 if (!ic->i_sends) { in rds_ib_setup_qp()
649 ic->i_recvs = vzalloc_node(array_size(sizeof(struct rds_ib_recv_work), in rds_ib_setup_qp()
650 ic->i_recv_ring.w_nr), in rds_ib_setup_qp()
652 if (!ic->i_recvs) { in rds_ib_setup_qp()
658 rds_ib_recv_init_ack(ic); in rds_ib_setup_qp()
660 rdsdebug("conn %p pd %p cq %p %p\n", conn, ic->i_pd, in rds_ib_setup_qp()
661 ic->i_send_cq, ic->i_recv_cq); in rds_ib_setup_qp()
666 vfree(ic->i_sends); in rds_ib_setup_qp()
669 dma_pool_free(pool, ic->i_ack, ic->i_ack_dma); in rds_ib_setup_qp()
670 ic->i_ack = NULL; in rds_ib_setup_qp()
673 rds_dma_hdrs_free(pool, ic->i_recv_hdrs, ic->i_recv_hdrs_dma, in rds_ib_setup_qp()
674 ic->i_recv_ring.w_nr); in rds_ib_setup_qp()
675 ic->i_recv_hdrs = NULL; in rds_ib_setup_qp()
676 ic->i_recv_hdrs_dma = NULL; in rds_ib_setup_qp()
679 rds_dma_hdrs_free(pool, ic->i_send_hdrs, ic->i_send_hdrs_dma, in rds_ib_setup_qp()
680 ic->i_send_ring.w_nr); in rds_ib_setup_qp()
681 ic->i_send_hdrs = NULL; in rds_ib_setup_qp()
682 ic->i_send_hdrs_dma = NULL; in rds_ib_setup_qp()
685 rdma_destroy_qp(ic->i_cm_id); in rds_ib_setup_qp()
687 ib_destroy_cq(ic->i_recv_cq); in rds_ib_setup_qp()
688 ic->i_recv_cq = NULL; in rds_ib_setup_qp()
690 ib_destroy_cq(ic->i_send_cq); in rds_ib_setup_qp()
691 ic->i_send_cq = NULL; in rds_ib_setup_qp()
796 struct rds_ib_connection *ic = NULL; in rds_ib_cm_handle_connect() local
892 ic = conn->c_transport_data; in rds_ib_cm_handle_connect()
904 BUG_ON(ic->i_cm_id); in rds_ib_cm_handle_connect()
906 ic->i_cm_id = cm_id; in rds_ib_cm_handle_connect()
940 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_cm_initiate_connect() local
948 ic->i_flowctl = rds_ib_sysctl_flow_control; /* advertise flow control */ in rds_ib_cm_initiate_connect()
969 if (ic->i_cm_id == cm_id) in rds_ib_cm_initiate_connect()
972 ic->i_active_side = true; in rds_ib_cm_initiate_connect()
981 struct rds_ib_connection *ic; in rds_ib_conn_path_connect() local
984 ic = conn->c_transport_data; in rds_ib_conn_path_connect()
994 ic->i_cm_id = rdma_create_id(&init_net, handler, conn, in rds_ib_conn_path_connect()
996 if (IS_ERR(ic->i_cm_id)) { in rds_ib_conn_path_connect()
997 ret = PTR_ERR(ic->i_cm_id); in rds_ib_conn_path_connect()
998 ic->i_cm_id = NULL; in rds_ib_conn_path_connect()
1003 rdsdebug("created cm id %p for conn %p\n", ic->i_cm_id, conn); in rds_ib_conn_path_connect()
1033 ret = rdma_resolve_addr(ic->i_cm_id, (struct sockaddr *)&src, in rds_ib_conn_path_connect()
1037 rdsdebug("addr resolve failed for cm id %p: %d\n", ic->i_cm_id, in rds_ib_conn_path_connect()
1039 rdma_destroy_id(ic->i_cm_id); in rds_ib_conn_path_connect()
1040 ic->i_cm_id = NULL; in rds_ib_conn_path_connect()
1055 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_conn_path_shutdown() local
1058 rdsdebug("cm %p pd %p cq %p %p qp %p\n", ic->i_cm_id, in rds_ib_conn_path_shutdown()
1059 ic->i_pd, ic->i_send_cq, ic->i_recv_cq, in rds_ib_conn_path_shutdown()
1060 ic->i_cm_id ? ic->i_cm_id->qp : NULL); in rds_ib_conn_path_shutdown()
1062 if (ic->i_cm_id) { in rds_ib_conn_path_shutdown()
1063 rdsdebug("disconnecting cm %p\n", ic->i_cm_id); in rds_ib_conn_path_shutdown()
1064 err = rdma_disconnect(ic->i_cm_id); in rds_ib_conn_path_shutdown()
1070 ic->i_cm_id, err); in rds_ib_conn_path_shutdown()
1088 rds_ib_ring_empty(&ic->i_recv_ring) && in rds_ib_conn_path_shutdown()
1089 (atomic_read(&ic->i_signaled_sends) == 0) && in rds_ib_conn_path_shutdown()
1090 (atomic_read(&ic->i_fastreg_inuse_count) == 0) && in rds_ib_conn_path_shutdown()
1091 (atomic_read(&ic->i_fastreg_wrs) == RDS_IB_DEFAULT_FR_WR)); in rds_ib_conn_path_shutdown()
1092 tasklet_kill(&ic->i_send_tasklet); in rds_ib_conn_path_shutdown()
1093 tasklet_kill(&ic->i_recv_tasklet); in rds_ib_conn_path_shutdown()
1095 atomic_set(&ic->i_cq_quiesce, 1); in rds_ib_conn_path_shutdown()
1098 if (ic->i_cm_id->qp) in rds_ib_conn_path_shutdown()
1099 rdma_destroy_qp(ic->i_cm_id); in rds_ib_conn_path_shutdown()
1100 if (ic->i_send_cq) { in rds_ib_conn_path_shutdown()
1101 if (ic->rds_ibdev) in rds_ib_conn_path_shutdown()
1102 ibdev_put_vector(ic->rds_ibdev, ic->i_scq_vector); in rds_ib_conn_path_shutdown()
1103 ib_destroy_cq(ic->i_send_cq); in rds_ib_conn_path_shutdown()
1106 if (ic->i_recv_cq) { in rds_ib_conn_path_shutdown()
1107 if (ic->rds_ibdev) in rds_ib_conn_path_shutdown()
1108 ibdev_put_vector(ic->rds_ibdev, ic->i_rcq_vector); in rds_ib_conn_path_shutdown()
1109 ib_destroy_cq(ic->i_recv_cq); in rds_ib_conn_path_shutdown()
1112 if (ic->rds_ibdev) { in rds_ib_conn_path_shutdown()
1115 pool = ic->rds_ibdev->rid_hdrs_pool; in rds_ib_conn_path_shutdown()
1118 if (ic->i_send_hdrs) { in rds_ib_conn_path_shutdown()
1119 rds_dma_hdrs_free(pool, ic->i_send_hdrs, in rds_ib_conn_path_shutdown()
1120 ic->i_send_hdrs_dma, in rds_ib_conn_path_shutdown()
1121 ic->i_send_ring.w_nr); in rds_ib_conn_path_shutdown()
1122 ic->i_send_hdrs = NULL; in rds_ib_conn_path_shutdown()
1123 ic->i_send_hdrs_dma = NULL; in rds_ib_conn_path_shutdown()
1126 if (ic->i_recv_hdrs) { in rds_ib_conn_path_shutdown()
1127 rds_dma_hdrs_free(pool, ic->i_recv_hdrs, in rds_ib_conn_path_shutdown()
1128 ic->i_recv_hdrs_dma, in rds_ib_conn_path_shutdown()
1129 ic->i_recv_ring.w_nr); in rds_ib_conn_path_shutdown()
1130 ic->i_recv_hdrs = NULL; in rds_ib_conn_path_shutdown()
1131 ic->i_recv_hdrs_dma = NULL; in rds_ib_conn_path_shutdown()
1134 if (ic->i_ack) { in rds_ib_conn_path_shutdown()
1135 dma_pool_free(pool, ic->i_ack, ic->i_ack_dma); in rds_ib_conn_path_shutdown()
1136 ic->i_ack = NULL; in rds_ib_conn_path_shutdown()
1139 WARN_ON(ic->i_send_hdrs); in rds_ib_conn_path_shutdown()
1140 WARN_ON(ic->i_send_hdrs_dma); in rds_ib_conn_path_shutdown()
1141 WARN_ON(ic->i_recv_hdrs); in rds_ib_conn_path_shutdown()
1142 WARN_ON(ic->i_recv_hdrs_dma); in rds_ib_conn_path_shutdown()
1143 WARN_ON(ic->i_ack); in rds_ib_conn_path_shutdown()
1146 if (ic->i_sends) in rds_ib_conn_path_shutdown()
1147 rds_ib_send_clear_ring(ic); in rds_ib_conn_path_shutdown()
1148 if (ic->i_recvs) in rds_ib_conn_path_shutdown()
1149 rds_ib_recv_clear_ring(ic); in rds_ib_conn_path_shutdown()
1151 rdma_destroy_id(ic->i_cm_id); in rds_ib_conn_path_shutdown()
1156 if (ic->rds_ibdev) in rds_ib_conn_path_shutdown()
1157 rds_ib_remove_conn(ic->rds_ibdev, conn); in rds_ib_conn_path_shutdown()
1159 ic->i_cm_id = NULL; in rds_ib_conn_path_shutdown()
1160 ic->i_pd = NULL; in rds_ib_conn_path_shutdown()
1161 ic->i_send_cq = NULL; in rds_ib_conn_path_shutdown()
1162 ic->i_recv_cq = NULL; in rds_ib_conn_path_shutdown()
1164 BUG_ON(ic->rds_ibdev); in rds_ib_conn_path_shutdown()
1167 if (ic->i_data_op) { in rds_ib_conn_path_shutdown()
1170 rm = container_of(ic->i_data_op, struct rds_message, data); in rds_ib_conn_path_shutdown()
1172 ic->i_data_op = NULL; in rds_ib_conn_path_shutdown()
1176 clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags); in rds_ib_conn_path_shutdown()
1178 atomic64_set(&ic->i_ack_next, 0); in rds_ib_conn_path_shutdown()
1180 ic->i_ack_next = 0; in rds_ib_conn_path_shutdown()
1182 ic->i_ack_recv = 0; in rds_ib_conn_path_shutdown()
1185 ic->i_flowctl = 0; in rds_ib_conn_path_shutdown()
1186 atomic_set(&ic->i_credits, 0); in rds_ib_conn_path_shutdown()
1189 rds_ib_ring_init(&ic->i_send_ring, ic->i_send_ring.w_nr); in rds_ib_conn_path_shutdown()
1190 rds_ib_ring_init(&ic->i_recv_ring, ic->i_recv_ring.w_nr); in rds_ib_conn_path_shutdown()
1192 if (ic->i_ibinc) { in rds_ib_conn_path_shutdown()
1193 rds_inc_put(&ic->i_ibinc->ii_inc); in rds_ib_conn_path_shutdown()
1194 ic->i_ibinc = NULL; in rds_ib_conn_path_shutdown()
1197 vfree(ic->i_sends); in rds_ib_conn_path_shutdown()
1198 ic->i_sends = NULL; in rds_ib_conn_path_shutdown()
1199 vfree(ic->i_recvs); in rds_ib_conn_path_shutdown()
1200 ic->i_recvs = NULL; in rds_ib_conn_path_shutdown()
1201 ic->i_active_side = false; in rds_ib_conn_path_shutdown()
1206 struct rds_ib_connection *ic; in rds_ib_conn_alloc() local
1211 ic = kzalloc(sizeof(struct rds_ib_connection), gfp); in rds_ib_conn_alloc()
1212 if (!ic) in rds_ib_conn_alloc()
1215 ret = rds_ib_recv_alloc_caches(ic, gfp); in rds_ib_conn_alloc()
1217 kfree(ic); in rds_ib_conn_alloc()
1221 INIT_LIST_HEAD(&ic->ib_node); in rds_ib_conn_alloc()
1222 tasklet_init(&ic->i_send_tasklet, rds_ib_tasklet_fn_send, in rds_ib_conn_alloc()
1223 (unsigned long)ic); in rds_ib_conn_alloc()
1224 tasklet_init(&ic->i_recv_tasklet, rds_ib_tasklet_fn_recv, in rds_ib_conn_alloc()
1225 (unsigned long)ic); in rds_ib_conn_alloc()
1226 mutex_init(&ic->i_recv_mutex); in rds_ib_conn_alloc()
1228 spin_lock_init(&ic->i_ack_lock); in rds_ib_conn_alloc()
1230 atomic_set(&ic->i_signaled_sends, 0); in rds_ib_conn_alloc()
1231 atomic_set(&ic->i_fastreg_wrs, RDS_IB_DEFAULT_FR_WR); in rds_ib_conn_alloc()
1237 rds_ib_ring_init(&ic->i_send_ring, 0); in rds_ib_conn_alloc()
1238 rds_ib_ring_init(&ic->i_recv_ring, 0); in rds_ib_conn_alloc()
1240 ic->conn = conn; in rds_ib_conn_alloc()
1241 conn->c_transport_data = ic; in rds_ib_conn_alloc()
1244 list_add_tail(&ic->ib_node, &ib_nodev_conns); in rds_ib_conn_alloc()
1248 rdsdebug("conn %p conn ic %p\n", conn, conn->c_transport_data); in rds_ib_conn_alloc()
1257 struct rds_ib_connection *ic = arg; in rds_ib_conn_free() local
1260 rdsdebug("ic %p\n", ic); in rds_ib_conn_free()
1267 lock_ptr = ic->rds_ibdev ? &ic->rds_ibdev->spinlock : &ib_nodev_conns_lock; in rds_ib_conn_free()
1270 list_del(&ic->ib_node); in rds_ib_conn_free()
1273 rds_ib_recv_free_caches(ic); in rds_ib_conn_free()
1275 kfree(ic); in rds_ib_conn_free()