Lines Matching full:rm
67 struct rds_message *rm, *tmp; in rds_send_path_reset() local
71 rm = cp->cp_xmit_rm; in rds_send_path_reset()
77 rds_message_unmapped(rm); in rds_send_path_reset()
78 rds_message_put(rm); in rds_send_path_reset()
95 list_for_each_entry_safe(rm, tmp, &cp->cp_retrans, m_conn_item) { in rds_send_path_reset()
96 set_bit(RDS_MSG_ACK_REQUIRED, &rm->m_flags); in rds_send_path_reset()
97 set_bit(RDS_MSG_RETRANSMITTED, &rm->m_flags); in rds_send_path_reset()
140 struct rds_message *rm; in rds_send_xmit() local
202 rm = cp->cp_xmit_rm; in rds_send_xmit()
204 if (!rm) { in rds_send_xmit()
219 if (!rm && test_and_clear_bit(0, &conn->c_map_queued)) { in rds_send_xmit()
220 rm = rds_cong_update_alloc(conn); in rds_send_xmit()
221 if (IS_ERR(rm)) { in rds_send_xmit()
222 ret = PTR_ERR(rm); in rds_send_xmit()
225 rm->data.op_active = 1; in rds_send_xmit()
226 rm->m_inc.i_conn_path = cp; in rds_send_xmit()
227 rm->m_inc.i_conn = cp->cp_conn; in rds_send_xmit()
229 cp->cp_xmit_rm = rm; in rds_send_xmit()
239 if (!rm) { in rds_send_xmit()
255 rm = list_entry(cp->cp_send_queue.next, in rds_send_xmit()
258 rds_message_addref(rm); in rds_send_xmit()
264 list_move_tail(&rm->m_conn_item, in rds_send_xmit()
270 if (!rm) in rds_send_xmit()
280 if (test_bit(RDS_MSG_FLUSH, &rm->m_flags) || in rds_send_xmit()
281 (rm->rdma.op_active && in rds_send_xmit()
282 test_bit(RDS_MSG_RETRANSMITTED, &rm->m_flags))) { in rds_send_xmit()
284 if (test_and_clear_bit(RDS_MSG_ON_CONN, &rm->m_flags)) in rds_send_xmit()
285 list_move(&rm->m_conn_item, &to_be_dropped); in rds_send_xmit()
291 len = ntohl(rm->m_inc.i_hdr.h_len); in rds_send_xmit()
294 set_bit(RDS_MSG_ACK_REQUIRED, &rm->m_flags); in rds_send_xmit()
306 cp->cp_xmit_rm = rm; in rds_send_xmit()
310 if (rm->rdma.op_active && !cp->cp_xmit_rdma_sent) { in rds_send_xmit()
311 rm->m_final_op = &rm->rdma; in rds_send_xmit()
315 set_bit(RDS_MSG_MAPPED, &rm->m_flags); in rds_send_xmit()
316 ret = conn->c_trans->xmit_rdma(conn, &rm->rdma); in rds_send_xmit()
318 clear_bit(RDS_MSG_MAPPED, &rm->m_flags); in rds_send_xmit()
319 wake_up_interruptible(&rm->m_flush_wait); in rds_send_xmit()
326 if (rm->atomic.op_active && !cp->cp_xmit_atomic_sent) { in rds_send_xmit()
327 rm->m_final_op = &rm->atomic; in rds_send_xmit()
331 set_bit(RDS_MSG_MAPPED, &rm->m_flags); in rds_send_xmit()
332 ret = conn->c_trans->xmit_atomic(conn, &rm->atomic); in rds_send_xmit()
334 clear_bit(RDS_MSG_MAPPED, &rm->m_flags); in rds_send_xmit()
335 wake_up_interruptible(&rm->m_flush_wait); in rds_send_xmit()
349 if (rm->data.op_nents == 0) { in rds_send_xmit()
353 ops_present = (rm->atomic.op_active || rm->rdma.op_active); in rds_send_xmit()
354 if (rm->atomic.op_active && !rm->atomic.op_silent) in rds_send_xmit()
356 if (rm->rdma.op_active && !rm->rdma.op_silent) in rds_send_xmit()
360 && !rm->m_rdma_cookie) in rds_send_xmit()
361 rm->data.op_active = 0; in rds_send_xmit()
364 if (rm->data.op_active && !cp->cp_xmit_data_sent) { in rds_send_xmit()
365 rm->m_final_op = &rm->data; in rds_send_xmit()
367 ret = conn->c_trans->xmit(conn, rm, in rds_send_xmit()
382 sg = &rm->data.op_sg[cp->cp_xmit_sg]; in rds_send_xmit()
393 rm->data.op_nents); in rds_send_xmit()
398 (cp->cp_xmit_sg == rm->data.op_nents)) in rds_send_xmit()
403 * A rm will only take multiple times through this loop in rds_send_xmit()
405 * none), then we're done with the rm. in rds_send_xmit()
407 if (!rm->data.op_active || cp->cp_xmit_data_sent) { in rds_send_xmit()
416 rds_message_put(rm); in rds_send_xmit()
428 list_for_each_entry(rm, &to_be_dropped, m_conn_item) in rds_send_xmit()
429 rds_message_put(rm); in rds_send_xmit()
473 static void rds_send_sndbuf_remove(struct rds_sock *rs, struct rds_message *rm) in rds_send_sndbuf_remove() argument
475 u32 len = be32_to_cpu(rm->m_inc.i_hdr.h_len); in rds_send_sndbuf_remove()
486 static inline int rds_send_is_acked(struct rds_message *rm, u64 ack, in rds_send_is_acked() argument
490 return is_acked(rm, ack); in rds_send_is_acked()
491 return be64_to_cpu(rm->m_inc.i_hdr.h_sequence) <= ack; in rds_send_is_acked()
500 void rds_rdma_send_complete(struct rds_message *rm, int status) in rds_rdma_send_complete() argument
507 spin_lock_irqsave(&rm->m_rs_lock, flags); in rds_rdma_send_complete()
509 ro = &rm->rdma; in rds_rdma_send_complete()
510 if (test_bit(RDS_MSG_ON_SOCK, &rm->m_flags) && in rds_rdma_send_complete()
513 rs = rm->m_rs; in rds_rdma_send_complete()
524 spin_unlock_irqrestore(&rm->m_rs_lock, flags); in rds_rdma_send_complete()
536 void rds_atomic_send_complete(struct rds_message *rm, int status) in rds_atomic_send_complete() argument
543 spin_lock_irqsave(&rm->m_rs_lock, flags); in rds_atomic_send_complete()
545 ao = &rm->atomic; in rds_atomic_send_complete()
546 if (test_bit(RDS_MSG_ON_SOCK, &rm->m_flags) in rds_atomic_send_complete()
549 rs = rm->m_rs; in rds_atomic_send_complete()
560 spin_unlock_irqrestore(&rm->m_rs_lock, flags); in rds_atomic_send_complete()
575 __rds_send_complete(struct rds_sock *rs, struct rds_message *rm, int status) in __rds_send_complete() argument
580 ro = &rm->rdma; in __rds_send_complete()
587 ao = &rm->atomic; in __rds_send_complete()
609 struct rds_message *rm; in rds_send_remove_from_sock() local
614 rm = list_entry(messages->next, struct rds_message, in rds_send_remove_from_sock()
616 list_del_init(&rm->m_conn_item); in rds_send_remove_from_sock()
624 * The message spinlock makes sure nobody clears rm->m_rs in rds_send_remove_from_sock()
628 spin_lock_irqsave(&rm->m_rs_lock, flags); in rds_send_remove_from_sock()
629 if (!test_bit(RDS_MSG_ON_SOCK, &rm->m_flags)) in rds_send_remove_from_sock()
632 if (rs != rm->m_rs) { in rds_send_remove_from_sock()
637 rs = rm->m_rs; in rds_send_remove_from_sock()
645 if (test_and_clear_bit(RDS_MSG_ON_SOCK, &rm->m_flags)) { in rds_send_remove_from_sock()
646 struct rm_rdma_op *ro = &rm->rdma; in rds_send_remove_from_sock()
649 list_del_init(&rm->m_sock_item); in rds_send_remove_from_sock()
650 rds_send_sndbuf_remove(rs, rm); in rds_send_remove_from_sock()
659 rm->rdma.op_notifier = NULL; in rds_send_remove_from_sock()
666 spin_unlock_irqrestore(&rm->m_rs_lock, flags); in rds_send_remove_from_sock()
667 rds_message_put(rm); in rds_send_remove_from_sock()
669 rds_message_put(rm); in rds_send_remove_from_sock()
689 struct rds_message *rm, *tmp; in rds_send_path_drop_acked() local
695 list_for_each_entry_safe(rm, tmp, &cp->cp_retrans, m_conn_item) { in rds_send_path_drop_acked()
696 if (!rds_send_is_acked(rm, ack, is_acked)) in rds_send_path_drop_acked()
699 list_move(&rm->m_conn_item, &list); in rds_send_path_drop_acked()
700 clear_bit(RDS_MSG_ON_CONN, &rm->m_flags); in rds_send_path_drop_acked()
724 struct rds_message *rm, *tmp; in rds_send_drop_to() local
733 list_for_each_entry_safe(rm, tmp, &rs->rs_send_queue, m_sock_item) { in rds_send_drop_to()
735 (!ipv6_addr_equal(&dest->sin6_addr, &rm->m_daddr) || in rds_send_drop_to()
736 dest->sin6_port != rm->m_inc.i_hdr.h_dport)) in rds_send_drop_to()
739 list_move(&rm->m_sock_item, &list); in rds_send_drop_to()
740 rds_send_sndbuf_remove(rs, rm); in rds_send_drop_to()
741 clear_bit(RDS_MSG_ON_SOCK, &rm->m_flags); in rds_send_drop_to()
753 list_for_each_entry(rm, &list, m_sock_item) { in rds_send_drop_to()
755 conn = rm->m_inc.i_conn; in rds_send_drop_to()
757 cp = rm->m_inc.i_conn_path; in rds_send_drop_to()
763 * Maybe someone else beat us to removing rm from the conn. in rds_send_drop_to()
767 if (!test_and_clear_bit(RDS_MSG_ON_CONN, &rm->m_flags)) { in rds_send_drop_to()
771 list_del_init(&rm->m_conn_item); in rds_send_drop_to()
778 spin_lock_irqsave(&rm->m_rs_lock, flags); in rds_send_drop_to()
781 __rds_send_complete(rs, rm, RDS_RDMA_CANCELED); in rds_send_drop_to()
784 spin_unlock_irqrestore(&rm->m_rs_lock, flags); in rds_send_drop_to()
786 rds_message_put(rm); in rds_send_drop_to()
792 rm = list_entry(list.next, struct rds_message, m_sock_item); in rds_send_drop_to()
793 list_del_init(&rm->m_sock_item); in rds_send_drop_to()
794 rds_message_wait(rm); in rds_send_drop_to()
801 spin_lock_irqsave(&rm->m_rs_lock, flags); in rds_send_drop_to()
804 __rds_send_complete(rs, rm, RDS_RDMA_CANCELED); in rds_send_drop_to()
807 spin_unlock_irqrestore(&rm->m_rs_lock, flags); in rds_send_drop_to()
809 rds_message_put(rm); in rds_send_drop_to()
820 struct rds_message *rm, __be16 sport, in rds_send_queue_rm() argument
829 len = be32_to_cpu(rm->m_inc.i_hdr.h_len); in rds_send_queue_rm()
852 set_bit(RDS_MSG_ACK_REQUIRED, &rm->m_flags); in rds_send_queue_rm()
854 list_add_tail(&rm->m_sock_item, &rs->rs_send_queue); in rds_send_queue_rm()
855 set_bit(RDS_MSG_ON_SOCK, &rm->m_flags); in rds_send_queue_rm()
856 rds_message_addref(rm); in rds_send_queue_rm()
858 rm->m_rs = rs; in rds_send_queue_rm()
862 rds_message_populate_header(&rm->m_inc.i_hdr, sport, dport, 0); in rds_send_queue_rm()
863 rm->m_inc.i_conn = conn; in rds_send_queue_rm()
864 rm->m_inc.i_conn_path = cp; in rds_send_queue_rm()
865 rds_message_addref(rm); in rds_send_queue_rm()
868 rm->m_inc.i_hdr.h_sequence = cpu_to_be64(cp->cp_next_tx_seq++); in rds_send_queue_rm()
869 list_add_tail(&rm->m_conn_item, &cp->cp_send_queue); in rds_send_queue_rm()
870 set_bit(RDS_MSG_ON_CONN, &rm->m_flags); in rds_send_queue_rm()
874 rm, len, rs, rs->rs_snd_bytes, in rds_send_queue_rm()
875 (unsigned long long)be64_to_cpu(rm->m_inc.i_hdr.h_sequence)); in rds_send_queue_rm()
971 static int rds_cmsg_zcopy(struct rds_sock *rs, struct rds_message *rm, in rds_cmsg_zcopy() argument
977 !rm->data.op_mmp_znotifier) in rds_cmsg_zcopy()
980 rm->data.op_mmp_znotifier->z_cookie = *cookie; in rds_cmsg_zcopy()
984 static int rds_cmsg_send(struct rds_sock *rs, struct rds_message *rm, in rds_cmsg_send() argument
999 * rm->rdma.m_rdma_cookie and rm->rdma.m_rdma_mr. in rds_cmsg_send()
1005 ret = rds_cmsg_rdma_args(rs, rm, cmsg, &vct->vec[ind]); in rds_cmsg_send()
1010 ret = rds_cmsg_rdma_dest(rs, rm, cmsg); in rds_cmsg_send()
1014 ret = rds_cmsg_rdma_map(rs, rm, cmsg); in rds_cmsg_send()
1027 ret = rds_cmsg_atomic(rs, rm, cmsg); in rds_cmsg_send()
1031 ret = rds_cmsg_zcopy(rs, rm, cmsg); in rds_cmsg_send()
1108 struct rds_message *rm = NULL; in rds_sendmsg() local
1263 /* size of rm including all sgs */ in rds_sendmsg()
1268 rm = rds_message_alloc(ret, GFP_KERNEL); in rds_sendmsg()
1269 if (!rm) { in rds_sendmsg()
1274 /* Attach data to the rm */ in rds_sendmsg()
1276 rm->data.op_sg = rds_message_alloc_sgs(rm, num_sgs); in rds_sendmsg()
1277 if (IS_ERR(rm->data.op_sg)) { in rds_sendmsg()
1278 ret = PTR_ERR(rm->data.op_sg); in rds_sendmsg()
1281 ret = rds_message_copy_from_user(rm, &msg->msg_iter, zcopy); in rds_sendmsg()
1285 rm->data.op_active = 1; in rds_sendmsg()
1287 rm->m_daddr = daddr; in rds_sendmsg()
1312 rm->m_conn_path = cpath; in rds_sendmsg()
1315 ret = rds_cmsg_send(rs, rm, msg, &allocated_mr, &vct); in rds_sendmsg()
1319 if (rm->rdma.op_active && !conn->c_trans->xmit_rdma) { in rds_sendmsg()
1321 &rm->rdma, conn->c_trans->xmit_rdma); in rds_sendmsg()
1326 if (rm->atomic.op_active && !conn->c_trans->xmit_atomic) { in rds_sendmsg()
1328 &rm->atomic, conn->c_trans->xmit_atomic); in rds_sendmsg()
1346 while (!rds_send_queue_rm(rs, conn, cpath, rm, rs->rs_bound_port, in rds_sendmsg()
1356 rds_send_queue_rm(rs, conn, cpath, rm, in rds_sendmsg()
1389 rds_message_put(rm); in rds_sendmsg()
1406 rds_rdma_unuse(rs, rds_rdma_cookie_key(rm->m_rdma_cookie), 1); in rds_sendmsg()
1408 if (rm) in rds_sendmsg()
1409 rds_message_put(rm); in rds_sendmsg()
1425 struct rds_message *rm; in rds_send_probe() local
1429 rm = rds_message_alloc(0, GFP_ATOMIC); in rds_send_probe()
1430 if (!rm) { in rds_send_probe()
1435 rm->m_daddr = cp->cp_conn->c_faddr; in rds_send_probe()
1436 rm->data.op_active = 1; in rds_send_probe()
1445 list_add_tail(&rm->m_conn_item, &cp->cp_send_queue); in rds_send_probe()
1446 set_bit(RDS_MSG_ON_CONN, &rm->m_flags); in rds_send_probe()
1447 rds_message_addref(rm); in rds_send_probe()
1448 rm->m_inc.i_conn = cp->cp_conn; in rds_send_probe()
1449 rm->m_inc.i_conn_path = cp; in rds_send_probe()
1451 rds_message_populate_header(&rm->m_inc.i_hdr, sport, dport, in rds_send_probe()
1453 rm->m_inc.i_hdr.h_flags |= h_flags; in rds_send_probe()
1461 rds_message_add_extension(&rm->m_inc.i_hdr, in rds_send_probe()
1464 rds_message_add_extension(&rm->m_inc.i_hdr, in rds_send_probe()
1480 rds_message_put(rm); in rds_send_probe()
1484 if (rm) in rds_send_probe()
1485 rds_message_put(rm); in rds_send_probe()