Lines Matching full:ic
47 void rds_ib_recv_init_ring(struct rds_ib_connection *ic) in rds_ib_recv_init_ring() argument
52 for (i = 0, recv = ic->i_recvs; i < ic->i_recv_ring.w_nr; i++, recv++) { in rds_ib_recv_init_ring()
64 sge->addr = ic->i_recv_hdrs_dma[i]; in rds_ib_recv_init_ring()
66 sge->lkey = ic->i_pd->local_dma_lkey; in rds_ib_recv_init_ring()
71 sge->lkey = ic->i_pd->local_dma_lkey; in rds_ib_recv_init_ring()
121 int rds_ib_recv_alloc_caches(struct rds_ib_connection *ic, gfp_t gfp) in rds_ib_recv_alloc_caches() argument
125 ret = rds_ib_recv_alloc_cache(&ic->i_cache_incs, gfp); in rds_ib_recv_alloc_caches()
127 ret = rds_ib_recv_alloc_cache(&ic->i_cache_frags, gfp); in rds_ib_recv_alloc_caches()
129 free_percpu(ic->i_cache_incs.percpu); in rds_ib_recv_alloc_caches()
155 void rds_ib_recv_free_caches(struct rds_ib_connection *ic) in rds_ib_recv_free_caches() argument
163 rds_ib_cache_xfer_to_ready(&ic->i_cache_incs); in rds_ib_recv_free_caches()
164 rds_ib_cache_splice_all_lists(&ic->i_cache_incs, &list); in rds_ib_recv_free_caches()
165 free_percpu(ic->i_cache_incs.percpu); in rds_ib_recv_free_caches()
174 rds_ib_cache_xfer_to_ready(&ic->i_cache_frags); in rds_ib_recv_free_caches()
175 rds_ib_cache_splice_all_lists(&ic->i_cache_frags, &list); in rds_ib_recv_free_caches()
176 free_percpu(ic->i_cache_frags.percpu); in rds_ib_recv_free_caches()
192 static void rds_ib_frag_free(struct rds_ib_connection *ic, in rds_ib_frag_free() argument
197 rds_ib_recv_cache_put(&frag->f_cache_entry, &ic->i_cache_frags); in rds_ib_frag_free()
198 atomic_add(RDS_FRAG_SIZE / SZ_1K, &ic->i_cache_allocs); in rds_ib_frag_free()
208 struct rds_ib_connection *ic = inc->i_conn->c_transport_data; in rds_ib_inc_free() local
215 rds_ib_frag_free(ic, frag); in rds_ib_inc_free()
220 rds_ib_recv_cache_put(&ibinc->ii_cache_entry, &ic->i_cache_incs); in rds_ib_inc_free()
223 static void rds_ib_recv_clear_one(struct rds_ib_connection *ic, in rds_ib_recv_clear_one() argument
231 ib_dma_unmap_sg(ic->i_cm_id->device, &recv->r_frag->f_sg, 1, DMA_FROM_DEVICE); in rds_ib_recv_clear_one()
232 rds_ib_frag_free(ic, recv->r_frag); in rds_ib_recv_clear_one()
237 void rds_ib_recv_clear_ring(struct rds_ib_connection *ic) in rds_ib_recv_clear_ring() argument
241 for (i = 0; i < ic->i_recv_ring.w_nr; i++) in rds_ib_recv_clear_ring()
242 rds_ib_recv_clear_one(ic, &ic->i_recvs[i]); in rds_ib_recv_clear_ring()
245 static struct rds_ib_incoming *rds_ib_refill_one_inc(struct rds_ib_connection *ic, in rds_ib_refill_one_inc() argument
252 cache_item = rds_ib_recv_cache_get(&ic->i_cache_incs); in rds_ib_refill_one_inc()
270 rds_inc_init(&ibinc->ii_inc, ic->conn, &ic->conn->c_faddr); in rds_ib_refill_one_inc()
275 static struct rds_page_frag *rds_ib_refill_one_frag(struct rds_ib_connection *ic, in rds_ib_refill_one_frag() argument
282 cache_item = rds_ib_recv_cache_get(&ic->i_cache_frags); in rds_ib_refill_one_frag()
285 atomic_sub(RDS_FRAG_SIZE / SZ_1K, &ic->i_cache_allocs); in rds_ib_refill_one_frag()
310 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_recv_refill_one() local
321 if (!ic->i_cache_incs.ready) in rds_ib_recv_refill_one()
322 rds_ib_cache_xfer_to_ready(&ic->i_cache_incs); in rds_ib_recv_refill_one()
323 if (!ic->i_cache_frags.ready) in rds_ib_recv_refill_one()
324 rds_ib_cache_xfer_to_ready(&ic->i_cache_frags); in rds_ib_recv_refill_one()
331 recv->r_ibinc = rds_ib_refill_one_inc(ic, slab_mask); in rds_ib_recv_refill_one()
337 recv->r_frag = rds_ib_refill_one_frag(ic, slab_mask, page_mask); in rds_ib_recv_refill_one()
341 ret = ib_dma_map_sg(ic->i_cm_id->device, &recv->r_frag->f_sg, in rds_ib_recv_refill_one()
346 sge->addr = ic->i_recv_hdrs_dma[recv - ic->i_recvs]; in rds_ib_recv_refill_one()
383 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_recv_refill() local
399 rds_ib_ring_alloc(&ic->i_recv_ring, 1, &pos)) { in rds_ib_recv_refill()
400 if (pos >= ic->i_recv_ring.w_nr) { in rds_ib_recv_refill()
406 recv = &ic->i_recvs[pos]; in rds_ib_recv_refill()
418 ret = ib_post_recv(ic->i_cm_id->qp, &recv->r_wr, NULL); in rds_ib_recv_refill()
436 if (ic->i_flowctl && posted) in rds_ib_recv_refill()
440 rds_ib_ring_unalloc(&ic->i_recv_ring, 1); in rds_ib_recv_refill()
456 (can_wait && rds_ib_ring_low(&ic->i_recv_ring)) || in rds_ib_recv_refill()
457 rds_ib_ring_empty(&ic->i_recv_ring))) { in rds_ib_recv_refill()
572 /* ic starts out kzalloc()ed */
573 void rds_ib_recv_init_ack(struct rds_ib_connection *ic) in rds_ib_recv_init_ack() argument
575 struct ib_send_wr *wr = &ic->i_ack_wr; in rds_ib_recv_init_ack()
576 struct ib_sge *sge = &ic->i_ack_sge; in rds_ib_recv_init_ack()
578 sge->addr = ic->i_ack_dma; in rds_ib_recv_init_ack()
580 sge->lkey = ic->i_pd->local_dma_lkey; in rds_ib_recv_init_ack()
612 void rds_ib_set_ack(struct rds_ib_connection *ic, u64 seq, int ack_required) in rds_ib_set_ack() argument
616 spin_lock_irqsave(&ic->i_ack_lock, flags); in rds_ib_set_ack()
617 ic->i_ack_next = seq; in rds_ib_set_ack()
619 set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags); in rds_ib_set_ack()
620 spin_unlock_irqrestore(&ic->i_ack_lock, flags); in rds_ib_set_ack()
623 static u64 rds_ib_get_ack(struct rds_ib_connection *ic) in rds_ib_get_ack() argument
628 clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags); in rds_ib_get_ack()
630 spin_lock_irqsave(&ic->i_ack_lock, flags); in rds_ib_get_ack()
631 seq = ic->i_ack_next; in rds_ib_get_ack()
632 spin_unlock_irqrestore(&ic->i_ack_lock, flags); in rds_ib_get_ack()
637 void rds_ib_set_ack(struct rds_ib_connection *ic, u64 seq, int ack_required) in rds_ib_set_ack() argument
639 atomic64_set(&ic->i_ack_next, seq); in rds_ib_set_ack()
642 set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags); in rds_ib_set_ack()
646 static u64 rds_ib_get_ack(struct rds_ib_connection *ic) in rds_ib_get_ack() argument
648 clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags); in rds_ib_get_ack()
651 return atomic64_read(&ic->i_ack_next); in rds_ib_get_ack()
656 static void rds_ib_send_ack(struct rds_ib_connection *ic, unsigned int adv_credits) in rds_ib_send_ack() argument
658 struct rds_header *hdr = ic->i_ack; in rds_ib_send_ack()
662 seq = rds_ib_get_ack(ic); in rds_ib_send_ack()
664 rdsdebug("send_ack: ic %p ack %llu\n", ic, (unsigned long long) seq); in rds_ib_send_ack()
669 ic->i_ack_queued = jiffies; in rds_ib_send_ack()
671 ret = ib_post_send(ic->i_cm_id->qp, &ic->i_ack_wr, NULL); in rds_ib_send_ack()
676 clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags); in rds_ib_send_ack()
677 set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags); in rds_ib_send_ack()
681 rds_ib_conn_error(ic->conn, "sending ack failed\n"); in rds_ib_send_ack()
724 void rds_ib_attempt_ack(struct rds_ib_connection *ic) in rds_ib_attempt_ack() argument
728 if (!test_bit(IB_ACK_REQUESTED, &ic->i_ack_flags)) in rds_ib_attempt_ack()
731 if (test_and_set_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags)) { in rds_ib_attempt_ack()
737 if (!rds_ib_send_grab_credits(ic, 1, &adv_credits, 0, RDS_MAX_ADV_CREDIT)) { in rds_ib_attempt_ack()
739 clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags); in rds_ib_attempt_ack()
743 clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags); in rds_ib_attempt_ack()
744 rds_ib_send_ack(ic, adv_credits); in rds_ib_attempt_ack()
751 void rds_ib_ack_send_complete(struct rds_ib_connection *ic) in rds_ib_ack_send_complete() argument
753 clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags); in rds_ib_ack_send_complete()
754 rds_ib_attempt_ack(ic); in rds_ib_ack_send_complete()
761 u64 rds_ib_piggyb_ack(struct rds_ib_connection *ic) in rds_ib_piggyb_ack() argument
763 if (test_and_clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags)) in rds_ib_piggyb_ack()
765 return rds_ib_get_ack(ic); in rds_ib_piggyb_ack()
845 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_process_recv() local
846 struct rds_ib_incoming *ibinc = ic->i_ibinc; in rds_ib_process_recv()
851 rdsdebug("ic %p ibinc %p recv %p byte len %u\n", ic, ibinc, recv, in rds_ib_process_recv()
864 ihdr = ic->i_recv_hdrs[recv - ic->i_recvs]; in rds_ib_process_recv()
900 rds_ib_frag_free(ic, recv->r_frag); in rds_ib_process_recv()
914 ic->i_ibinc = ibinc; in rds_ib_process_recv()
920 ic->i_recv_data_rem = be32_to_cpu(hdr->h_len); in rds_ib_process_recv()
924 rdsdebug("ic %p ibinc %p rem %u flag 0x%x\n", ic, ibinc, in rds_ib_process_recv()
925 ic->i_recv_data_rem, hdr->h_flags); in rds_ib_process_recv()
943 if (ic->i_recv_data_rem > RDS_FRAG_SIZE) in rds_ib_process_recv()
944 ic->i_recv_data_rem -= RDS_FRAG_SIZE; in rds_ib_process_recv()
946 ic->i_recv_data_rem = 0; in rds_ib_process_recv()
947 ic->i_ibinc = NULL; in rds_ib_process_recv()
970 void rds_ib_recv_cqe_handler(struct rds_ib_connection *ic, in rds_ib_recv_cqe_handler() argument
974 struct rds_connection *conn = ic->conn; in rds_ib_recv_cqe_handler()
983 recv = &ic->i_recvs[rds_ib_ring_oldest(&ic->i_recv_ring)]; in rds_ib_recv_cqe_handler()
984 ib_dma_unmap_sg(ic->i_cm_id->device, &recv->r_frag->f_sg, 1, in rds_ib_recv_cqe_handler()
1011 rds_ib_frag_free(ic, recv->r_frag); in rds_ib_recv_cqe_handler()
1014 rds_ib_ring_free(&ic->i_recv_ring, 1); in rds_ib_recv_cqe_handler()
1019 if (rds_ib_ring_empty(&ic->i_recv_ring)) in rds_ib_recv_cqe_handler()
1022 if (rds_ib_ring_low(&ic->i_recv_ring)) { in rds_ib_recv_cqe_handler()
1031 struct rds_ib_connection *ic = conn->c_transport_data; in rds_ib_recv_path() local
1035 rds_ib_attempt_ack(ic); in rds_ib_recv_path()