Lines Matching +full:- +full:t
1 // SPDX-License-Identifier: GPL-2.0-or-later
51 * as defined in [MS-SMBD] 3.1.1.1
67 /* The maximum fragmented upper-layer payload receive size supported */
70 /* The maximum single-message size which can be received */
160 #define KSMBD_TRANS(t) ((struct ksmbd_transport *)&((t)->transport)) argument
197 struct smb_direct_transport *t; member
220 return DIV_ROUND_UP((uintptr_t)buf + size, PAGE_SIZE) - in get_buf_page_count()
226 static int smb_direct_post_send_data(struct smb_direct_transport *t,
232 smb_trans_direct_transfort(struct ksmbd_transport *t) in smb_trans_direct_transfort() argument
234 return container_of(t, struct smb_direct_transport, transport); in smb_trans_direct_transfort()
240 return (void *)recvmsg->packet; in smb_direct_recvmsg_payload()
251 smb_direct_recvmsg *get_free_recvmsg(struct smb_direct_transport *t) in get_free_recvmsg() argument
255 spin_lock(&t->recvmsg_queue_lock); in get_free_recvmsg()
256 if (!list_empty(&t->recvmsg_queue)) { in get_free_recvmsg()
257 recvmsg = list_first_entry(&t->recvmsg_queue, in get_free_recvmsg()
260 list_del(&recvmsg->list); in get_free_recvmsg()
262 spin_unlock(&t->recvmsg_queue_lock); in get_free_recvmsg()
266 static void put_recvmsg(struct smb_direct_transport *t, in put_recvmsg() argument
269 ib_dma_unmap_single(t->cm_id->device, recvmsg->sge.addr, in put_recvmsg()
270 recvmsg->sge.length, DMA_FROM_DEVICE); in put_recvmsg()
272 spin_lock(&t->recvmsg_queue_lock); in put_recvmsg()
273 list_add(&recvmsg->list, &t->recvmsg_queue); in put_recvmsg()
274 spin_unlock(&t->recvmsg_queue_lock); in put_recvmsg()
278 smb_direct_recvmsg *get_empty_recvmsg(struct smb_direct_transport *t) in get_empty_recvmsg() argument
282 spin_lock(&t->empty_recvmsg_queue_lock); in get_empty_recvmsg()
283 if (!list_empty(&t->empty_recvmsg_queue)) { in get_empty_recvmsg()
284 recvmsg = list_first_entry(&t->empty_recvmsg_queue, in get_empty_recvmsg()
286 list_del(&recvmsg->list); in get_empty_recvmsg()
288 spin_unlock(&t->empty_recvmsg_queue_lock); in get_empty_recvmsg()
292 static void put_empty_recvmsg(struct smb_direct_transport *t, in put_empty_recvmsg() argument
295 ib_dma_unmap_single(t->cm_id->device, recvmsg->sge.addr, in put_empty_recvmsg()
296 recvmsg->sge.length, DMA_FROM_DEVICE); in put_empty_recvmsg()
298 spin_lock(&t->empty_recvmsg_queue_lock); in put_empty_recvmsg()
299 list_add_tail(&recvmsg->list, &t->empty_recvmsg_queue); in put_empty_recvmsg()
300 spin_unlock(&t->empty_recvmsg_queue_lock); in put_empty_recvmsg()
303 static void enqueue_reassembly(struct smb_direct_transport *t, in enqueue_reassembly() argument
307 spin_lock(&t->reassembly_queue_lock); in enqueue_reassembly()
308 list_add_tail(&recvmsg->list, &t->reassembly_queue); in enqueue_reassembly()
309 t->reassembly_queue_length++; in enqueue_reassembly()
317 t->reassembly_data_length += data_length; in enqueue_reassembly()
318 spin_unlock(&t->reassembly_queue_lock); in enqueue_reassembly()
321 static struct smb_direct_recvmsg *get_first_reassembly(struct smb_direct_transport *t) in get_first_reassembly() argument
323 if (!list_empty(&t->reassembly_queue)) in get_first_reassembly()
324 return list_first_entry(&t->reassembly_queue, in get_first_reassembly()
332 struct smb_direct_transport *t = in smb_direct_disconnect_rdma_work() local
336 if (t->status == SMB_DIRECT_CS_CONNECTED) { in smb_direct_disconnect_rdma_work()
337 t->status = SMB_DIRECT_CS_DISCONNECTING; in smb_direct_disconnect_rdma_work()
338 rdma_disconnect(t->cm_id); in smb_direct_disconnect_rdma_work()
343 smb_direct_disconnect_rdma_connection(struct smb_direct_transport *t) in smb_direct_disconnect_rdma_connection() argument
345 if (t->status == SMB_DIRECT_CS_CONNECTED) in smb_direct_disconnect_rdma_connection()
346 queue_work(smb_direct_wq, &t->disconnect_work); in smb_direct_disconnect_rdma_connection()
351 struct smb_direct_transport *t = container_of(work, in smb_direct_send_immediate_work() local
354 if (t->status != SMB_DIRECT_CS_CONNECTED) in smb_direct_send_immediate_work()
357 smb_direct_post_send_data(t, NULL, NULL, 0, 0); in smb_direct_send_immediate_work()
362 struct smb_direct_transport *t; in alloc_transport() local
365 t = kzalloc(sizeof(*t), GFP_KERNEL); in alloc_transport()
366 if (!t) in alloc_transport()
369 t->cm_id = cm_id; in alloc_transport()
370 cm_id->context = t; in alloc_transport()
372 t->status = SMB_DIRECT_CS_NEW; in alloc_transport()
373 init_waitqueue_head(&t->wait_status); in alloc_transport()
375 spin_lock_init(&t->reassembly_queue_lock); in alloc_transport()
376 INIT_LIST_HEAD(&t->reassembly_queue); in alloc_transport()
377 t->reassembly_data_length = 0; in alloc_transport()
378 t->reassembly_queue_length = 0; in alloc_transport()
379 init_waitqueue_head(&t->wait_reassembly_queue); in alloc_transport()
380 init_waitqueue_head(&t->wait_send_credits); in alloc_transport()
381 init_waitqueue_head(&t->wait_rw_credits); in alloc_transport()
383 spin_lock_init(&t->receive_credit_lock); in alloc_transport()
384 spin_lock_init(&t->recvmsg_queue_lock); in alloc_transport()
385 INIT_LIST_HEAD(&t->recvmsg_queue); in alloc_transport()
387 spin_lock_init(&t->empty_recvmsg_queue_lock); in alloc_transport()
388 INIT_LIST_HEAD(&t->empty_recvmsg_queue); in alloc_transport()
390 init_waitqueue_head(&t->wait_send_pending); in alloc_transport()
391 atomic_set(&t->send_pending, 0); in alloc_transport()
393 spin_lock_init(&t->lock_new_recv_credits); in alloc_transport()
395 INIT_DELAYED_WORK(&t->post_recv_credits_work, in alloc_transport()
397 INIT_WORK(&t->send_immediate_work, smb_direct_send_immediate_work); in alloc_transport()
398 INIT_WORK(&t->disconnect_work, smb_direct_disconnect_rdma_work); in alloc_transport()
403 conn->transport = KSMBD_TRANS(t); in alloc_transport()
404 KSMBD_TRANS(t)->conn = conn; in alloc_transport()
405 KSMBD_TRANS(t)->ops = &ksmbd_smb_direct_transport_ops; in alloc_transport()
406 return t; in alloc_transport()
408 kfree(t); in alloc_transport()
412 static void free_transport(struct smb_direct_transport *t) in free_transport() argument
416 wake_up_interruptible(&t->wait_send_credits); in free_transport()
419 wait_event(t->wait_send_pending, in free_transport()
420 atomic_read(&t->send_pending) == 0); in free_transport()
422 cancel_work_sync(&t->disconnect_work); in free_transport()
423 cancel_delayed_work_sync(&t->post_recv_credits_work); in free_transport()
424 cancel_work_sync(&t->send_immediate_work); in free_transport()
426 if (t->qp) { in free_transport()
427 ib_drain_qp(t->qp); in free_transport()
428 ib_mr_pool_destroy(t->qp, &t->qp->rdma_mrs); in free_transport()
429 ib_destroy_qp(t->qp); in free_transport()
434 spin_lock(&t->reassembly_queue_lock); in free_transport()
435 recvmsg = get_first_reassembly(t); in free_transport()
437 list_del(&recvmsg->list); in free_transport()
438 spin_unlock(&t->reassembly_queue_lock); in free_transport()
439 put_recvmsg(t, recvmsg); in free_transport()
441 spin_unlock(&t->reassembly_queue_lock); in free_transport()
444 t->reassembly_data_length = 0; in free_transport()
446 if (t->send_cq) in free_transport()
447 ib_free_cq(t->send_cq); in free_transport()
448 if (t->recv_cq) in free_transport()
449 ib_free_cq(t->recv_cq); in free_transport()
450 if (t->pd) in free_transport()
451 ib_dealloc_pd(t->pd); in free_transport()
452 if (t->cm_id) in free_transport()
453 rdma_destroy_id(t->cm_id); in free_transport()
455 smb_direct_destroy_pools(t); in free_transport()
456 ksmbd_conn_free(KSMBD_TRANS(t)->conn); in free_transport()
457 kfree(t); in free_transport()
461 *smb_direct_alloc_sendmsg(struct smb_direct_transport *t) in smb_direct_alloc_sendmsg() argument
465 msg = mempool_alloc(t->sendmsg_mempool, GFP_KERNEL); in smb_direct_alloc_sendmsg()
467 return ERR_PTR(-ENOMEM); in smb_direct_alloc_sendmsg()
468 msg->transport = t; in smb_direct_alloc_sendmsg()
469 INIT_LIST_HEAD(&msg->list); in smb_direct_alloc_sendmsg()
470 msg->num_sge = 0; in smb_direct_alloc_sendmsg()
474 static void smb_direct_free_sendmsg(struct smb_direct_transport *t, in smb_direct_free_sendmsg() argument
479 if (msg->num_sge > 0) { in smb_direct_free_sendmsg()
480 ib_dma_unmap_single(t->cm_id->device, in smb_direct_free_sendmsg()
481 msg->sge[0].addr, msg->sge[0].length, in smb_direct_free_sendmsg()
483 for (i = 1; i < msg->num_sge; i++) in smb_direct_free_sendmsg()
484 ib_dma_unmap_page(t->cm_id->device, in smb_direct_free_sendmsg()
485 msg->sge[i].addr, msg->sge[i].length, in smb_direct_free_sendmsg()
488 mempool_free(msg, t->sendmsg_mempool); in smb_direct_free_sendmsg()
493 switch (recvmsg->type) { in smb_direct_check_recvmsg()
496 (struct smb_direct_data_transfer *)recvmsg->packet; in smb_direct_check_recvmsg()
497 struct smb2_hdr *hdr = (struct smb2_hdr *)(recvmsg->packet in smb_direct_check_recvmsg()
498 + le32_to_cpu(req->data_offset)); in smb_direct_check_recvmsg()
501 le16_to_cpu(req->credits_granted), in smb_direct_check_recvmsg()
502 le16_to_cpu(req->credits_requested), in smb_direct_check_recvmsg()
503 req->data_length, req->remaining_data_length, in smb_direct_check_recvmsg()
504 hdr->ProtocolId, hdr->Command); in smb_direct_check_recvmsg()
509 (struct smb_direct_negotiate_req *)recvmsg->packet; in smb_direct_check_recvmsg()
512 le16_to_cpu(req->min_version), in smb_direct_check_recvmsg()
513 le16_to_cpu(req->max_version), in smb_direct_check_recvmsg()
514 le16_to_cpu(req->credits_requested), in smb_direct_check_recvmsg()
515 le32_to_cpu(req->preferred_send_size), in smb_direct_check_recvmsg()
516 le32_to_cpu(req->max_receive_size), in smb_direct_check_recvmsg()
517 le32_to_cpu(req->max_fragmented_size)); in smb_direct_check_recvmsg()
518 if (le16_to_cpu(req->min_version) > 0x0100 || in smb_direct_check_recvmsg()
519 le16_to_cpu(req->max_version) < 0x0100) in smb_direct_check_recvmsg()
520 return -EOPNOTSUPP; in smb_direct_check_recvmsg()
521 if (le16_to_cpu(req->credits_requested) <= 0 || in smb_direct_check_recvmsg()
522 le32_to_cpu(req->max_receive_size) <= 128 || in smb_direct_check_recvmsg()
523 le32_to_cpu(req->max_fragmented_size) <= in smb_direct_check_recvmsg()
525 return -ECONNABORTED; in smb_direct_check_recvmsg()
530 return -EINVAL; in smb_direct_check_recvmsg()
538 struct smb_direct_transport *t; in recv_done() local
540 recvmsg = container_of(wc->wr_cqe, struct smb_direct_recvmsg, cqe); in recv_done()
541 t = recvmsg->transport; in recv_done()
543 if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_RECV) { in recv_done()
544 if (wc->status != IB_WC_WR_FLUSH_ERR) { in recv_done()
546 ib_wc_status_msg(wc->status), wc->status, in recv_done()
547 wc->opcode); in recv_done()
548 smb_direct_disconnect_rdma_connection(t); in recv_done()
550 put_empty_recvmsg(t, recvmsg); in recv_done()
555 ib_wc_status_msg(wc->status), wc->status, in recv_done()
556 wc->opcode); in recv_done()
558 ib_dma_sync_single_for_cpu(wc->qp->device, recvmsg->sge.addr, in recv_done()
559 recvmsg->sge.length, DMA_FROM_DEVICE); in recv_done()
561 switch (recvmsg->type) { in recv_done()
563 if (wc->byte_len < sizeof(struct smb_direct_negotiate_req)) { in recv_done()
564 put_empty_recvmsg(t, recvmsg); in recv_done()
567 t->negotiation_requested = true; in recv_done()
568 t->full_packet_received = true; in recv_done()
569 t->status = SMB_DIRECT_CS_CONNECTED; in recv_done()
570 enqueue_reassembly(t, recvmsg, 0); in recv_done()
571 wake_up_interruptible(&t->wait_status); in recv_done()
575 (struct smb_direct_data_transfer *)recvmsg->packet; in recv_done()
579 if (wc->byte_len < in recv_done()
581 put_empty_recvmsg(t, recvmsg); in recv_done()
585 data_length = le32_to_cpu(data_transfer->data_length); in recv_done()
587 if (wc->byte_len < sizeof(struct smb_direct_data_transfer) + in recv_done()
589 put_empty_recvmsg(t, recvmsg); in recv_done()
593 if (t->full_packet_received) in recv_done()
594 recvmsg->first_segment = true; in recv_done()
596 if (le32_to_cpu(data_transfer->remaining_data_length)) in recv_done()
597 t->full_packet_received = false; in recv_done()
599 t->full_packet_received = true; in recv_done()
601 enqueue_reassembly(t, recvmsg, (int)data_length); in recv_done()
602 wake_up_interruptible(&t->wait_reassembly_queue); in recv_done()
604 spin_lock(&t->receive_credit_lock); in recv_done()
605 receive_credits = --(t->recv_credits); in recv_done()
606 avail_recvmsg_count = t->count_avail_recvmsg; in recv_done()
607 spin_unlock(&t->receive_credit_lock); in recv_done()
609 put_empty_recvmsg(t, recvmsg); in recv_done()
611 spin_lock(&t->receive_credit_lock); in recv_done()
612 receive_credits = --(t->recv_credits); in recv_done()
613 avail_recvmsg_count = ++(t->count_avail_recvmsg); in recv_done()
614 spin_unlock(&t->receive_credit_lock); in recv_done()
617 t->recv_credit_target = in recv_done()
618 le16_to_cpu(data_transfer->credits_requested); in recv_done()
619 atomic_add(le16_to_cpu(data_transfer->credits_granted), in recv_done()
620 &t->send_credits); in recv_done()
622 if (le16_to_cpu(data_transfer->flags) & in recv_done()
624 queue_work(smb_direct_wq, &t->send_immediate_work); in recv_done()
626 if (atomic_read(&t->send_credits) > 0) in recv_done()
627 wake_up_interruptible(&t->wait_send_credits); in recv_done()
631 &t->post_recv_credits_work, 0); in recv_done()
639 static int smb_direct_post_recv(struct smb_direct_transport *t, in smb_direct_post_recv() argument
645 recvmsg->sge.addr = ib_dma_map_single(t->cm_id->device, in smb_direct_post_recv()
646 recvmsg->packet, t->max_recv_size, in smb_direct_post_recv()
648 ret = ib_dma_mapping_error(t->cm_id->device, recvmsg->sge.addr); in smb_direct_post_recv()
651 recvmsg->sge.length = t->max_recv_size; in smb_direct_post_recv()
652 recvmsg->sge.lkey = t->pd->local_dma_lkey; in smb_direct_post_recv()
653 recvmsg->cqe.done = recv_done; in smb_direct_post_recv()
655 wr.wr_cqe = &recvmsg->cqe; in smb_direct_post_recv()
657 wr.sg_list = &recvmsg->sge; in smb_direct_post_recv()
660 ret = ib_post_recv(t->qp, &wr, NULL); in smb_direct_post_recv()
662 pr_err("Can't post recv: %d\n", ret); in smb_direct_post_recv()
663 ib_dma_unmap_single(t->cm_id->device, in smb_direct_post_recv()
664 recvmsg->sge.addr, recvmsg->sge.length, in smb_direct_post_recv()
666 smb_direct_disconnect_rdma_connection(t); in smb_direct_post_recv()
672 static int smb_direct_read(struct ksmbd_transport *t, char *buf, in smb_direct_read() argument
680 struct smb_direct_transport *st = smb_trans_direct_transfort(t); in smb_direct_read()
683 if (st->status != SMB_DIRECT_CS_CONNECTED) { in smb_direct_read()
685 return -ENOTCONN; in smb_direct_read()
693 if (st->reassembly_data_length >= size) { in smb_direct_read()
705 queue_length = st->reassembly_queue_length; in smb_direct_read()
708 offset = st->first_entry_offset; in smb_direct_read()
712 data_length = le32_to_cpu(data_transfer->data_length); in smb_direct_read()
714 le32_to_cpu(data_transfer->remaining_data_length); in smb_direct_read()
715 data_offset = le32_to_cpu(data_transfer->data_offset); in smb_direct_read()
725 if (recvmsg->first_segment && size == 4) { in smb_direct_read()
730 recvmsg->first_segment = false; in smb_direct_read()
737 to_copy = min_t(int, data_length - offset, to_read); in smb_direct_read()
742 if (to_copy == data_length - offset) { in smb_direct_read()
743 queue_length--; in smb_direct_read()
749 list_del(&recvmsg->list); in smb_direct_read()
751 spin_lock_irq(&st->reassembly_queue_lock); in smb_direct_read()
752 list_del(&recvmsg->list); in smb_direct_read()
753 spin_unlock_irq(&st->reassembly_queue_lock); in smb_direct_read()
762 to_read -= to_copy; in smb_direct_read()
766 spin_lock_irq(&st->reassembly_queue_lock); in smb_direct_read()
767 st->reassembly_data_length -= data_read; in smb_direct_read()
768 st->reassembly_queue_length -= queue_removed; in smb_direct_read()
769 spin_unlock_irq(&st->reassembly_queue_lock); in smb_direct_read()
771 spin_lock(&st->receive_credit_lock); in smb_direct_read()
772 st->count_avail_recvmsg += queue_removed; in smb_direct_read()
773 if (is_receive_credit_post_required(st->recv_credits, st->count_avail_recvmsg)) { in smb_direct_read()
774 spin_unlock(&st->receive_credit_lock); in smb_direct_read()
776 &st->post_recv_credits_work, 0); in smb_direct_read()
778 spin_unlock(&st->receive_credit_lock); in smb_direct_read()
781 st->first_entry_offset = offset; in smb_direct_read()
784 data_read, st->reassembly_data_length, in smb_direct_read()
785 st->first_entry_offset); in smb_direct_read()
791 rc = wait_event_interruptible(st->wait_reassembly_queue, in smb_direct_read()
792 st->reassembly_data_length >= size || in smb_direct_read()
793 st->status != SMB_DIRECT_CS_CONNECTED); in smb_direct_read()
795 return -EINTR; in smb_direct_read()
802 struct smb_direct_transport *t = container_of(work, in smb_direct_post_recv_credits() local
809 spin_lock(&t->receive_credit_lock); in smb_direct_post_recv_credits()
810 receive_credits = t->recv_credits; in smb_direct_post_recv_credits()
811 spin_unlock(&t->receive_credit_lock); in smb_direct_post_recv_credits()
813 if (receive_credits < t->recv_credit_target) { in smb_direct_post_recv_credits()
816 recvmsg = get_free_recvmsg(t); in smb_direct_post_recv_credits()
818 recvmsg = get_empty_recvmsg(t); in smb_direct_post_recv_credits()
828 recvmsg->type = SMB_DIRECT_MSG_DATA_TRANSFER; in smb_direct_post_recv_credits()
829 recvmsg->first_segment = false; in smb_direct_post_recv_credits()
831 ret = smb_direct_post_recv(t, recvmsg); in smb_direct_post_recv_credits()
833 pr_err("Can't post recv: %d\n", ret); in smb_direct_post_recv_credits()
834 put_recvmsg(t, recvmsg); in smb_direct_post_recv_credits()
841 spin_lock(&t->receive_credit_lock); in smb_direct_post_recv_credits()
842 t->recv_credits += credits; in smb_direct_post_recv_credits()
843 t->count_avail_recvmsg -= credits; in smb_direct_post_recv_credits()
844 spin_unlock(&t->receive_credit_lock); in smb_direct_post_recv_credits()
846 spin_lock(&t->lock_new_recv_credits); in smb_direct_post_recv_credits()
847 t->new_recv_credits += credits; in smb_direct_post_recv_credits()
848 spin_unlock(&t->lock_new_recv_credits); in smb_direct_post_recv_credits()
851 queue_work(smb_direct_wq, &t->send_immediate_work); in smb_direct_post_recv_credits()
857 struct smb_direct_transport *t; in send_done() local
860 sendmsg = container_of(wc->wr_cqe, struct smb_direct_sendmsg, cqe); in send_done()
861 t = sendmsg->transport; in send_done()
864 ib_wc_status_msg(wc->status), wc->status, in send_done()
865 wc->opcode); in send_done()
867 if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_SEND) { in send_done()
869 ib_wc_status_msg(wc->status), wc->status, in send_done()
870 wc->opcode); in send_done()
871 smb_direct_disconnect_rdma_connection(t); in send_done()
874 if (atomic_dec_and_test(&t->send_pending)) in send_done()
875 wake_up(&t->wait_send_pending); in send_done()
880 for (pos = &sendmsg->list, prev = pos->prev, end = sendmsg->list.next; in send_done()
881 prev != end; pos = prev, prev = prev->prev) { in send_done()
883 smb_direct_free_sendmsg(t, sibling); in send_done()
887 smb_direct_free_sendmsg(t, sibling); in send_done()
890 static int manage_credits_prior_sending(struct smb_direct_transport *t) in manage_credits_prior_sending() argument
894 spin_lock(&t->lock_new_recv_credits); in manage_credits_prior_sending()
895 new_credits = t->new_recv_credits; in manage_credits_prior_sending()
896 t->new_recv_credits = 0; in manage_credits_prior_sending()
897 spin_unlock(&t->lock_new_recv_credits); in manage_credits_prior_sending()
902 static int smb_direct_post_send(struct smb_direct_transport *t, in smb_direct_post_send() argument
907 atomic_inc(&t->send_pending); in smb_direct_post_send()
908 ret = ib_post_send(t->qp, wr, NULL); in smb_direct_post_send()
911 if (atomic_dec_and_test(&t->send_pending)) in smb_direct_post_send()
912 wake_up(&t->wait_send_pending); in smb_direct_post_send()
913 smb_direct_disconnect_rdma_connection(t); in smb_direct_post_send()
918 static void smb_direct_send_ctx_init(struct smb_direct_transport *t, in smb_direct_send_ctx_init() argument
923 INIT_LIST_HEAD(&send_ctx->msg_list); in smb_direct_send_ctx_init()
924 send_ctx->wr_cnt = 0; in smb_direct_send_ctx_init()
925 send_ctx->need_invalidate_rkey = need_invalidate_rkey; in smb_direct_send_ctx_init()
926 send_ctx->remote_key = remote_key; in smb_direct_send_ctx_init()
929 static int smb_direct_flush_send_list(struct smb_direct_transport *t, in smb_direct_flush_send_list() argument
936 if (list_empty(&send_ctx->msg_list)) in smb_direct_flush_send_list()
939 first = list_first_entry(&send_ctx->msg_list, in smb_direct_flush_send_list()
942 last = list_last_entry(&send_ctx->msg_list, in smb_direct_flush_send_list()
946 last->wr.send_flags = IB_SEND_SIGNALED; in smb_direct_flush_send_list()
947 last->wr.wr_cqe = &last->cqe; in smb_direct_flush_send_list()
948 if (is_last && send_ctx->need_invalidate_rkey) { in smb_direct_flush_send_list()
949 last->wr.opcode = IB_WR_SEND_WITH_INV; in smb_direct_flush_send_list()
950 last->wr.ex.invalidate_rkey = send_ctx->remote_key; in smb_direct_flush_send_list()
953 ret = smb_direct_post_send(t, &first->wr); in smb_direct_flush_send_list()
955 smb_direct_send_ctx_init(t, send_ctx, in smb_direct_flush_send_list()
956 send_ctx->need_invalidate_rkey, in smb_direct_flush_send_list()
957 send_ctx->remote_key); in smb_direct_flush_send_list()
959 atomic_add(send_ctx->wr_cnt, &t->send_credits); in smb_direct_flush_send_list()
960 wake_up(&t->wait_send_credits); in smb_direct_flush_send_list()
961 list_for_each_entry_safe(first, last, &send_ctx->msg_list, in smb_direct_flush_send_list()
963 smb_direct_free_sendmsg(t, first); in smb_direct_flush_send_list()
969 static int wait_for_credits(struct smb_direct_transport *t, in wait_for_credits() argument
982 t->status != SMB_DIRECT_CS_CONNECTED); in wait_for_credits()
984 if (t->status != SMB_DIRECT_CS_CONNECTED) in wait_for_credits()
985 return -ENOTCONN; in wait_for_credits()
991 static int wait_for_send_credits(struct smb_direct_transport *t, in wait_for_send_credits() argument
997 (send_ctx->wr_cnt >= 16 || atomic_read(&t->send_credits) <= 1)) { in wait_for_send_credits()
998 ret = smb_direct_flush_send_list(t, send_ctx, false); in wait_for_send_credits()
1003 return wait_for_credits(t, &t->wait_send_credits, &t->send_credits, 1); in wait_for_send_credits()
1006 static int wait_for_rw_credits(struct smb_direct_transport *t, int credits) in wait_for_rw_credits() argument
1008 return wait_for_credits(t, &t->wait_rw_credits, &t->rw_credits, credits); in wait_for_rw_credits()
1011 static int calc_rw_credits(struct smb_direct_transport *t, in calc_rw_credits() argument
1015 t->pages_per_rw_credit); in calc_rw_credits()
1018 static int smb_direct_create_header(struct smb_direct_transport *t, in smb_direct_create_header() argument
1027 sendmsg = smb_direct_alloc_sendmsg(t); in smb_direct_create_header()
1032 packet = (struct smb_direct_data_transfer *)sendmsg->packet; in smb_direct_create_header()
1033 packet->credits_requested = cpu_to_le16(t->send_credit_target); in smb_direct_create_header()
1034 packet->credits_granted = cpu_to_le16(manage_credits_prior_sending(t)); in smb_direct_create_header()
1036 packet->flags = 0; in smb_direct_create_header()
1037 packet->reserved = 0; in smb_direct_create_header()
1039 packet->data_offset = 0; in smb_direct_create_header()
1041 packet->data_offset = cpu_to_le32(24); in smb_direct_create_header()
1042 packet->data_length = cpu_to_le32(size); in smb_direct_create_header()
1043 packet->remaining_data_length = cpu_to_le32(remaining_data_length); in smb_direct_create_header()
1044 packet->padding = 0; in smb_direct_create_header()
1048 le16_to_cpu(packet->credits_requested), in smb_direct_create_header()
1049 le16_to_cpu(packet->credits_granted), in smb_direct_create_header()
1050 le32_to_cpu(packet->data_offset), in smb_direct_create_header()
1051 le32_to_cpu(packet->data_length), in smb_direct_create_header()
1052 le32_to_cpu(packet->remaining_data_length)); in smb_direct_create_header()
1056 /* If this is a packet without payload, don't send padding */ in smb_direct_create_header()
1061 sendmsg->sge[0].addr = ib_dma_map_single(t->cm_id->device, in smb_direct_create_header()
1065 ret = ib_dma_mapping_error(t->cm_id->device, sendmsg->sge[0].addr); in smb_direct_create_header()
1067 smb_direct_free_sendmsg(t, sendmsg); in smb_direct_create_header()
1071 sendmsg->num_sge = 1; in smb_direct_create_header()
1072 sendmsg->sge[0].length = header_length; in smb_direct_create_header()
1073 sendmsg->sge[0].lkey = t->pd->local_dma_lkey; in smb_direct_create_header()
1087 return -EINVAL; in get_sg_list()
1090 buf -= offset; in get_sg_list()
1092 len = min_t(int, PAGE_SIZE - offset, size); in get_sg_list()
1099 return -EINVAL; in get_sg_list()
1104 size -= len; in get_sg_list()
1119 return -EINVAL; in get_mapped_sg_list()
1123 static int post_sendmsg(struct smb_direct_transport *t, in post_sendmsg() argument
1129 for (i = 0; i < msg->num_sge; i++) in post_sendmsg()
1130 ib_dma_sync_single_for_device(t->cm_id->device, in post_sendmsg()
1131 msg->sge[i].addr, msg->sge[i].length, in post_sendmsg()
1134 msg->cqe.done = send_done; in post_sendmsg()
1135 msg->wr.opcode = IB_WR_SEND; in post_sendmsg()
1136 msg->wr.sg_list = &msg->sge[0]; in post_sendmsg()
1137 msg->wr.num_sge = msg->num_sge; in post_sendmsg()
1138 msg->wr.next = NULL; in post_sendmsg()
1141 msg->wr.wr_cqe = NULL; in post_sendmsg()
1142 msg->wr.send_flags = 0; in post_sendmsg()
1143 if (!list_empty(&send_ctx->msg_list)) { in post_sendmsg()
1146 last = list_last_entry(&send_ctx->msg_list, in post_sendmsg()
1149 last->wr.next = &msg->wr; in post_sendmsg()
1151 list_add_tail(&msg->list, &send_ctx->msg_list); in post_sendmsg()
1152 send_ctx->wr_cnt++; in post_sendmsg()
1156 msg->wr.wr_cqe = &msg->cqe; in post_sendmsg()
1157 msg->wr.send_flags = IB_SEND_SIGNALED; in post_sendmsg()
1158 return smb_direct_post_send(t, &msg->wr); in post_sendmsg()
1161 static int smb_direct_post_send_data(struct smb_direct_transport *t, in smb_direct_post_send_data() argument
1169 struct scatterlist sg[SMB_DIRECT_MAX_SEND_SGES - 1]; in smb_direct_post_send_data()
1171 ret = wait_for_send_credits(t, send_ctx); in smb_direct_post_send_data()
1179 ret = smb_direct_create_header(t, data_length, remaining_data_length, in smb_direct_post_send_data()
1182 atomic_inc(&t->send_credits); in smb_direct_post_send_data()
1190 sg_init_table(sg, SMB_DIRECT_MAX_SEND_SGES - 1); in smb_direct_post_send_data()
1191 sg_cnt = get_mapped_sg_list(t->cm_id->device, in smb_direct_post_send_data()
1193 sg, SMB_DIRECT_MAX_SEND_SGES - 1, in smb_direct_post_send_data()
1197 ret = -ENOMEM; in smb_direct_post_send_data()
1199 } else if (sg_cnt + msg->num_sge > SMB_DIRECT_MAX_SEND_SGES) { in smb_direct_post_send_data()
1201 ret = -E2BIG; in smb_direct_post_send_data()
1202 ib_dma_unmap_sg(t->cm_id->device, sg, sg_cnt, in smb_direct_post_send_data()
1208 sge = &msg->sge[msg->num_sge]; in smb_direct_post_send_data()
1209 sge->addr = sg_dma_address(&sg[j]); in smb_direct_post_send_data()
1210 sge->length = sg_dma_len(&sg[j]); in smb_direct_post_send_data()
1211 sge->lkey = t->pd->local_dma_lkey; in smb_direct_post_send_data()
1212 msg->num_sge++; in smb_direct_post_send_data()
1216 ret = post_sendmsg(t, send_ctx, msg); in smb_direct_post_send_data()
1221 smb_direct_free_sendmsg(t, msg); in smb_direct_post_send_data()
1222 atomic_inc(&t->send_credits); in smb_direct_post_send_data()
1226 static int smb_direct_writev(struct ksmbd_transport *t, in smb_direct_writev() argument
1230 struct smb_direct_transport *st = smb_trans_direct_transfort(t); in smb_direct_writev()
1233 int max_iov_size = st->max_send_size - in smb_direct_writev()
1239 if (st->status != SMB_DIRECT_CS_CONNECTED) in smb_direct_writev()
1240 return -ENOTCONN; in smb_direct_writev()
1243 buflen -= 4; in smb_direct_writev()
1255 remaining_data_length -= in smb_direct_writev()
1256 (buflen - iov[i].iov_len); in smb_direct_writev()
1258 &iov[start], i - start, in smb_direct_writev()
1264 int nvec = (buflen + max_iov_size - 1) / in smb_direct_writev()
1273 buflen - max_iov_size * j); in smb_direct_writev()
1274 remaining_data_length -= vec.iov_len; in smb_direct_writev()
1290 remaining_data_length -= buflen; in smb_direct_writev()
1292 &iov[start], i - start, in smb_direct_writev()
1305 * As an optimization, we don't wait for individual I/O to finish in smb_direct_writev()
1311 wait_event(st->wait_send_pending, in smb_direct_writev()
1312 atomic_read(&st->send_pending) == 0); in smb_direct_writev()
1316 static void smb_direct_free_rdma_rw_msg(struct smb_direct_transport *t, in smb_direct_free_rdma_rw_msg() argument
1320 rdma_rw_ctx_destroy(&msg->rw_ctx, t->qp, t->qp->port, in smb_direct_free_rdma_rw_msg()
1321 msg->sgt.sgl, msg->sgt.nents, dir); in smb_direct_free_rdma_rw_msg()
1322 sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE); in smb_direct_free_rdma_rw_msg()
1329 struct smb_direct_rdma_rw_msg *msg = container_of(wc->wr_cqe, in read_write_done()
1331 struct smb_direct_transport *t = msg->t; in read_write_done() local
1333 if (wc->status != IB_WC_SUCCESS) { in read_write_done()
1334 msg->status = -EIO; in read_write_done()
1336 wc->opcode, ib_wc_status_msg(wc->status), wc->status); in read_write_done()
1337 if (wc->status != IB_WC_WR_FLUSH_ERR) in read_write_done()
1338 smb_direct_disconnect_rdma_connection(t); in read_write_done()
1341 complete(msg->completion); in read_write_done()
1354 static int smb_direct_rdma_xmit(struct smb_direct_transport *t, in smb_direct_rdma_xmit() argument
1369 if (t->status != SMB_DIRECT_CS_CONNECTED) in smb_direct_rdma_xmit()
1370 return -ENOTCONN; in smb_direct_rdma_xmit()
1372 if (buf_len > t->max_rdma_rw_size) in smb_direct_rdma_xmit()
1373 return -EINVAL; in smb_direct_rdma_xmit()
1384 return -EINVAL; in smb_direct_rdma_xmit()
1392 credits_needed += calc_rw_credits(t, desc_buf, desc_buf_len); in smb_direct_rdma_xmit()
1394 buf_len -= desc_buf_len; in smb_direct_rdma_xmit()
1401 ret = wait_for_rw_credits(t, credits_needed); in smb_direct_rdma_xmit()
1411 ret = -ENOMEM; in smb_direct_rdma_xmit()
1417 msg->t = t; in smb_direct_rdma_xmit()
1418 msg->cqe.done = is_read ? read_done : write_done; in smb_direct_rdma_xmit()
1419 msg->completion = &completion; in smb_direct_rdma_xmit()
1421 msg->sgt.sgl = &msg->sg_list[0]; in smb_direct_rdma_xmit()
1422 ret = sg_alloc_table_chained(&msg->sgt, in smb_direct_rdma_xmit()
1424 msg->sg_list, SG_CHUNK_SIZE); in smb_direct_rdma_xmit()
1427 ret = -ENOMEM; in smb_direct_rdma_xmit()
1432 msg->sgt.sgl, msg->sgt.orig_nents); in smb_direct_rdma_xmit()
1434 sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE); in smb_direct_rdma_xmit()
1439 ret = rdma_rw_ctx_init(&msg->rw_ctx, t->qp, t->qp->port, in smb_direct_rdma_xmit()
1440 msg->sgt.sgl, in smb_direct_rdma_xmit()
1448 sg_free_table_chained(&msg->sgt, SG_CHUNK_SIZE); in smb_direct_rdma_xmit()
1453 list_add_tail(&msg->list, &msg_list); in smb_direct_rdma_xmit()
1460 first_wr = rdma_rw_ctx_wrs(&msg->rw_ctx, t->qp, t->qp->port, in smb_direct_rdma_xmit()
1461 &msg->cqe, first_wr); in smb_direct_rdma_xmit()
1464 ret = ib_post_send(t->qp, first_wr, NULL); in smb_direct_rdma_xmit()
1472 ret = msg->status; in smb_direct_rdma_xmit()
1475 list_del(&msg->list); in smb_direct_rdma_xmit()
1476 smb_direct_free_rdma_rw_msg(t, msg, in smb_direct_rdma_xmit()
1479 atomic_add(credits_needed, &t->rw_credits); in smb_direct_rdma_xmit()
1480 wake_up(&t->wait_rw_credits); in smb_direct_rdma_xmit()
1484 static int smb_direct_rdma_write(struct ksmbd_transport *t, in smb_direct_rdma_write() argument
1489 return smb_direct_rdma_xmit(smb_trans_direct_transfort(t), buf, buflen, in smb_direct_rdma_write()
1493 static int smb_direct_rdma_read(struct ksmbd_transport *t, in smb_direct_rdma_read() argument
1498 return smb_direct_rdma_xmit(smb_trans_direct_transfort(t), buf, buflen, in smb_direct_rdma_read()
1502 static void smb_direct_disconnect(struct ksmbd_transport *t) in smb_direct_disconnect() argument
1504 struct smb_direct_transport *st = smb_trans_direct_transfort(t); in smb_direct_disconnect()
1506 ksmbd_debug(RDMA, "Disconnecting cm_id=%p\n", st->cm_id); in smb_direct_disconnect()
1508 smb_direct_disconnect_rdma_work(&st->disconnect_work); in smb_direct_disconnect()
1509 wait_event_interruptible(st->wait_status, in smb_direct_disconnect()
1510 st->status == SMB_DIRECT_CS_DISCONNECTED); in smb_direct_disconnect()
1514 static void smb_direct_shutdown(struct ksmbd_transport *t) in smb_direct_shutdown() argument
1516 struct smb_direct_transport *st = smb_trans_direct_transfort(t); in smb_direct_shutdown()
1518 ksmbd_debug(RDMA, "smb-direct shutdown cm_id=%p\n", st->cm_id); in smb_direct_shutdown()
1520 smb_direct_disconnect_rdma_work(&st->disconnect_work); in smb_direct_shutdown()
1526 struct smb_direct_transport *t = cm_id->context; in smb_direct_cm_handler() local
1529 cm_id, rdma_event_msg(event->event), event->event); in smb_direct_cm_handler()
1531 switch (event->event) { in smb_direct_cm_handler()
1533 t->status = SMB_DIRECT_CS_CONNECTED; in smb_direct_cm_handler()
1534 wake_up_interruptible(&t->wait_status); in smb_direct_cm_handler()
1539 ib_drain_qp(t->qp); in smb_direct_cm_handler()
1541 t->status = SMB_DIRECT_CS_DISCONNECTED; in smb_direct_cm_handler()
1542 wake_up_interruptible(&t->wait_status); in smb_direct_cm_handler()
1543 wake_up_interruptible(&t->wait_reassembly_queue); in smb_direct_cm_handler()
1544 wake_up(&t->wait_send_credits); in smb_direct_cm_handler()
1548 t->status = SMB_DIRECT_CS_DISCONNECTED; in smb_direct_cm_handler()
1549 wake_up_interruptible(&t->wait_status); in smb_direct_cm_handler()
1554 cm_id, rdma_event_msg(event->event), in smb_direct_cm_handler()
1555 event->event); in smb_direct_cm_handler()
1563 struct smb_direct_transport *t = context; in smb_direct_qpair_handler() local
1566 t->cm_id, ib_event_msg(event->event), event->event); in smb_direct_qpair_handler()
1568 switch (event->event) { in smb_direct_qpair_handler()
1571 smb_direct_disconnect_rdma_connection(t); in smb_direct_qpair_handler()
1578 static int smb_direct_send_negotiate_response(struct smb_direct_transport *t, in smb_direct_send_negotiate_response() argument
1585 sendmsg = smb_direct_alloc_sendmsg(t); in smb_direct_send_negotiate_response()
1587 return -ENOMEM; in smb_direct_send_negotiate_response()
1589 resp = (struct smb_direct_negotiate_resp *)sendmsg->packet; in smb_direct_send_negotiate_response()
1592 resp->min_version = cpu_to_le16(0x0100); in smb_direct_send_negotiate_response()
1593 resp->max_version = cpu_to_le16(0x0100); in smb_direct_send_negotiate_response()
1594 resp->status = STATUS_NOT_SUPPORTED; in smb_direct_send_negotiate_response()
1596 resp->status = STATUS_SUCCESS; in smb_direct_send_negotiate_response()
1597 resp->min_version = SMB_DIRECT_VERSION_LE; in smb_direct_send_negotiate_response()
1598 resp->max_version = SMB_DIRECT_VERSION_LE; in smb_direct_send_negotiate_response()
1599 resp->negotiated_version = SMB_DIRECT_VERSION_LE; in smb_direct_send_negotiate_response()
1600 resp->reserved = 0; in smb_direct_send_negotiate_response()
1601 resp->credits_requested = in smb_direct_send_negotiate_response()
1602 cpu_to_le16(t->send_credit_target); in smb_direct_send_negotiate_response()
1603 resp->credits_granted = cpu_to_le16(manage_credits_prior_sending(t)); in smb_direct_send_negotiate_response()
1604 resp->max_readwrite_size = cpu_to_le32(t->max_rdma_rw_size); in smb_direct_send_negotiate_response()
1605 resp->preferred_send_size = cpu_to_le32(t->max_send_size); in smb_direct_send_negotiate_response()
1606 resp->max_receive_size = cpu_to_le32(t->max_recv_size); in smb_direct_send_negotiate_response()
1607 resp->max_fragmented_size = in smb_direct_send_negotiate_response()
1608 cpu_to_le32(t->max_fragmented_recv_size); in smb_direct_send_negotiate_response()
1611 sendmsg->sge[0].addr = ib_dma_map_single(t->cm_id->device, in smb_direct_send_negotiate_response()
1614 ret = ib_dma_mapping_error(t->cm_id->device, sendmsg->sge[0].addr); in smb_direct_send_negotiate_response()
1616 smb_direct_free_sendmsg(t, sendmsg); in smb_direct_send_negotiate_response()
1620 sendmsg->num_sge = 1; in smb_direct_send_negotiate_response()
1621 sendmsg->sge[0].length = sizeof(*resp); in smb_direct_send_negotiate_response()
1622 sendmsg->sge[0].lkey = t->pd->local_dma_lkey; in smb_direct_send_negotiate_response()
1624 ret = post_sendmsg(t, NULL, sendmsg); in smb_direct_send_negotiate_response()
1626 smb_direct_free_sendmsg(t, sendmsg); in smb_direct_send_negotiate_response()
1630 wait_event(t->wait_send_pending, in smb_direct_send_negotiate_response()
1631 atomic_read(&t->send_pending) == 0); in smb_direct_send_negotiate_response()
1635 static int smb_direct_accept_client(struct smb_direct_transport *t) in smb_direct_accept_client() argument
1643 conn_param.initiator_depth = min_t(u8, t->cm_id->device->attrs.max_qp_rd_atom, in smb_direct_accept_client()
1647 t->cm_id->device->ops.get_port_immutable(t->cm_id->device, in smb_direct_accept_client()
1648 t->cm_id->port_num, in smb_direct_accept_client()
1663 ret = rdma_accept(t->cm_id, &conn_param); in smb_direct_accept_client()
1671 static int smb_direct_prepare_negotiation(struct smb_direct_transport *t) in smb_direct_prepare_negotiation() argument
1676 recvmsg = get_free_recvmsg(t); in smb_direct_prepare_negotiation()
1678 return -ENOMEM; in smb_direct_prepare_negotiation()
1679 recvmsg->type = SMB_DIRECT_MSG_NEGOTIATE_REQ; in smb_direct_prepare_negotiation()
1681 ret = smb_direct_post_recv(t, recvmsg); in smb_direct_prepare_negotiation()
1683 pr_err("Can't post recv: %d\n", ret); in smb_direct_prepare_negotiation()
1687 t->negotiation_requested = false; in smb_direct_prepare_negotiation()
1688 ret = smb_direct_accept_client(t); in smb_direct_prepare_negotiation()
1690 pr_err("Can't accept client\n"); in smb_direct_prepare_negotiation()
1694 smb_direct_post_recv_credits(&t->post_recv_credits_work.work); in smb_direct_prepare_negotiation()
1697 put_recvmsg(t, recvmsg); in smb_direct_prepare_negotiation()
1701 static unsigned int smb_direct_get_max_fr_pages(struct smb_direct_transport *t) in smb_direct_get_max_fr_pages() argument
1704 t->cm_id->device->attrs.max_fast_reg_page_list_len, in smb_direct_get_max_fr_pages()
1708 static int smb_direct_init_params(struct smb_direct_transport *t, in smb_direct_init_params() argument
1711 struct ib_device *device = t->cm_id->device; in smb_direct_init_params()
1718 t->max_send_size = smb_direct_max_send_size; in smb_direct_init_params()
1719 max_send_sges = DIV_ROUND_UP(t->max_send_size, PAGE_SIZE) + 3; in smb_direct_init_params()
1721 pr_err("max_send_size %d is too large\n", t->max_send_size); in smb_direct_init_params()
1722 return -EINVAL; in smb_direct_init_params()
1732 t->max_rdma_rw_size = smb_direct_max_read_write_size; in smb_direct_init_params()
1733 t->pages_per_rw_credit = smb_direct_get_max_fr_pages(t); in smb_direct_init_params()
1734 t->max_rw_credits = DIV_ROUND_UP(t->max_rdma_rw_size, in smb_direct_init_params()
1735 (t->pages_per_rw_credit - 1) * in smb_direct_init_params()
1738 max_sge_per_wr = min_t(unsigned int, device->attrs.max_send_sge, in smb_direct_init_params()
1739 device->attrs.max_sge_rd); in smb_direct_init_params()
1743 DIV_ROUND_UP(t->pages_per_rw_credit, in smb_direct_init_params()
1745 max_rw_wrs = t->max_rw_credits * wrs_per_credit; in smb_direct_init_params()
1748 if (max_send_wrs > device->attrs.max_cqe || in smb_direct_init_params()
1749 max_send_wrs > device->attrs.max_qp_wr) { in smb_direct_init_params()
1753 device->attrs.max_cqe, device->attrs.max_qp_wr); in smb_direct_init_params()
1754 return -EINVAL; in smb_direct_init_params()
1757 if (smb_direct_receive_credit_max > device->attrs.max_cqe || in smb_direct_init_params()
1758 smb_direct_receive_credit_max > device->attrs.max_qp_wr) { in smb_direct_init_params()
1762 device->attrs.max_cqe, device->attrs.max_qp_wr); in smb_direct_init_params()
1763 return -EINVAL; in smb_direct_init_params()
1766 if (device->attrs.max_recv_sge < SMB_DIRECT_MAX_RECV_SGES) { in smb_direct_init_params()
1768 device->attrs.max_recv_sge); in smb_direct_init_params()
1769 return -EINVAL; in smb_direct_init_params()
1772 t->recv_credits = 0; in smb_direct_init_params()
1773 t->count_avail_recvmsg = 0; in smb_direct_init_params()
1775 t->recv_credit_max = smb_direct_receive_credit_max; in smb_direct_init_params()
1776 t->recv_credit_target = 10; in smb_direct_init_params()
1777 t->new_recv_credits = 0; in smb_direct_init_params()
1779 t->send_credit_target = smb_direct_send_credit_target; in smb_direct_init_params()
1780 atomic_set(&t->send_credits, 0); in smb_direct_init_params()
1781 atomic_set(&t->rw_credits, t->max_rw_credits); in smb_direct_init_params()
1783 t->max_send_size = smb_direct_max_send_size; in smb_direct_init_params()
1784 t->max_recv_size = smb_direct_max_receive_size; in smb_direct_init_params()
1785 t->max_fragmented_recv_size = smb_direct_max_fragmented_recv_size; in smb_direct_init_params()
1787 cap->max_send_wr = max_send_wrs; in smb_direct_init_params()
1788 cap->max_recv_wr = t->recv_credit_max; in smb_direct_init_params()
1789 cap->max_send_sge = max_sge_per_wr; in smb_direct_init_params()
1790 cap->max_recv_sge = SMB_DIRECT_MAX_RECV_SGES; in smb_direct_init_params()
1791 cap->max_inline_data = 0; in smb_direct_init_params()
1792 cap->max_rdma_ctxs = t->max_rw_credits; in smb_direct_init_params()
1796 static void smb_direct_destroy_pools(struct smb_direct_transport *t) in smb_direct_destroy_pools() argument
1800 while ((recvmsg = get_free_recvmsg(t))) in smb_direct_destroy_pools()
1801 mempool_free(recvmsg, t->recvmsg_mempool); in smb_direct_destroy_pools()
1802 while ((recvmsg = get_empty_recvmsg(t))) in smb_direct_destroy_pools()
1803 mempool_free(recvmsg, t->recvmsg_mempool); in smb_direct_destroy_pools()
1805 mempool_destroy(t->recvmsg_mempool); in smb_direct_destroy_pools()
1806 t->recvmsg_mempool = NULL; in smb_direct_destroy_pools()
1808 kmem_cache_destroy(t->recvmsg_cache); in smb_direct_destroy_pools()
1809 t->recvmsg_cache = NULL; in smb_direct_destroy_pools()
1811 mempool_destroy(t->sendmsg_mempool); in smb_direct_destroy_pools()
1812 t->sendmsg_mempool = NULL; in smb_direct_destroy_pools()
1814 kmem_cache_destroy(t->sendmsg_cache); in smb_direct_destroy_pools()
1815 t->sendmsg_cache = NULL; in smb_direct_destroy_pools()
1818 static int smb_direct_create_pools(struct smb_direct_transport *t) in smb_direct_create_pools() argument
1824 snprintf(name, sizeof(name), "smb_direct_rqst_pool_%p", t); in smb_direct_create_pools()
1825 t->sendmsg_cache = kmem_cache_create(name, in smb_direct_create_pools()
1829 if (!t->sendmsg_cache) in smb_direct_create_pools()
1830 return -ENOMEM; in smb_direct_create_pools()
1832 t->sendmsg_mempool = mempool_create(t->send_credit_target, in smb_direct_create_pools()
1834 t->sendmsg_cache); in smb_direct_create_pools()
1835 if (!t->sendmsg_mempool) in smb_direct_create_pools()
1838 snprintf(name, sizeof(name), "smb_direct_resp_%p", t); in smb_direct_create_pools()
1839 t->recvmsg_cache = kmem_cache_create(name, in smb_direct_create_pools()
1841 t->max_recv_size, in smb_direct_create_pools()
1843 if (!t->recvmsg_cache) in smb_direct_create_pools()
1846 t->recvmsg_mempool = in smb_direct_create_pools()
1847 mempool_create(t->recv_credit_max, mempool_alloc_slab, in smb_direct_create_pools()
1848 mempool_free_slab, t->recvmsg_cache); in smb_direct_create_pools()
1849 if (!t->recvmsg_mempool) in smb_direct_create_pools()
1852 INIT_LIST_HEAD(&t->recvmsg_queue); in smb_direct_create_pools()
1854 for (i = 0; i < t->recv_credit_max; i++) { in smb_direct_create_pools()
1855 recvmsg = mempool_alloc(t->recvmsg_mempool, GFP_KERNEL); in smb_direct_create_pools()
1858 recvmsg->transport = t; in smb_direct_create_pools()
1859 list_add(&recvmsg->list, &t->recvmsg_queue); in smb_direct_create_pools()
1861 t->count_avail_recvmsg = t->recv_credit_max; in smb_direct_create_pools()
1865 smb_direct_destroy_pools(t); in smb_direct_create_pools()
1866 return -ENOMEM; in smb_direct_create_pools()
1869 static int smb_direct_create_qpair(struct smb_direct_transport *t, in smb_direct_create_qpair() argument
1876 t->pd = ib_alloc_pd(t->cm_id->device, 0); in smb_direct_create_qpair()
1877 if (IS_ERR(t->pd)) { in smb_direct_create_qpair()
1878 pr_err("Can't create RDMA PD\n"); in smb_direct_create_qpair()
1879 ret = PTR_ERR(t->pd); in smb_direct_create_qpair()
1880 t->pd = NULL; in smb_direct_create_qpair()
1884 t->send_cq = ib_alloc_cq(t->cm_id->device, t, in smb_direct_create_qpair()
1885 smb_direct_send_credit_target + cap->max_rdma_ctxs, in smb_direct_create_qpair()
1887 if (IS_ERR(t->send_cq)) { in smb_direct_create_qpair()
1888 pr_err("Can't create RDMA send CQ\n"); in smb_direct_create_qpair()
1889 ret = PTR_ERR(t->send_cq); in smb_direct_create_qpair()
1890 t->send_cq = NULL; in smb_direct_create_qpair()
1894 t->recv_cq = ib_alloc_cq(t->cm_id->device, t, in smb_direct_create_qpair()
1895 t->recv_credit_max, 0, IB_POLL_WORKQUEUE); in smb_direct_create_qpair()
1896 if (IS_ERR(t->recv_cq)) { in smb_direct_create_qpair()
1897 pr_err("Can't create RDMA recv CQ\n"); in smb_direct_create_qpair()
1898 ret = PTR_ERR(t->recv_cq); in smb_direct_create_qpair()
1899 t->recv_cq = NULL; in smb_direct_create_qpair()
1905 qp_attr.qp_context = t; in smb_direct_create_qpair()
1909 qp_attr.send_cq = t->send_cq; in smb_direct_create_qpair()
1910 qp_attr.recv_cq = t->recv_cq; in smb_direct_create_qpair()
1913 ret = rdma_create_qp(t->cm_id, t->pd, &qp_attr); in smb_direct_create_qpair()
1915 pr_err("Can't create RDMA QP: %d\n", ret); in smb_direct_create_qpair()
1919 t->qp = t->cm_id->qp; in smb_direct_create_qpair()
1920 t->cm_id->event_handler = smb_direct_cm_handler; in smb_direct_create_qpair()
1922 pages_per_rw = DIV_ROUND_UP(t->max_rdma_rw_size, PAGE_SIZE) + 1; in smb_direct_create_qpair()
1923 if (pages_per_rw > t->cm_id->device->attrs.max_sgl_rd) { in smb_direct_create_qpair()
1924 ret = ib_mr_pool_init(t->qp, &t->qp->rdma_mrs, in smb_direct_create_qpair()
1925 t->max_rw_credits, IB_MR_TYPE_MEM_REG, in smb_direct_create_qpair()
1926 t->pages_per_rw_credit, 0); in smb_direct_create_qpair()
1929 t->max_rw_credits, t->pages_per_rw_credit); in smb_direct_create_qpair()
1936 if (t->qp) { in smb_direct_create_qpair()
1937 ib_destroy_qp(t->qp); in smb_direct_create_qpair()
1938 t->qp = NULL; in smb_direct_create_qpair()
1940 if (t->recv_cq) { in smb_direct_create_qpair()
1941 ib_destroy_cq(t->recv_cq); in smb_direct_create_qpair()
1942 t->recv_cq = NULL; in smb_direct_create_qpair()
1944 if (t->send_cq) { in smb_direct_create_qpair()
1945 ib_destroy_cq(t->send_cq); in smb_direct_create_qpair()
1946 t->send_cq = NULL; in smb_direct_create_qpair()
1948 if (t->pd) { in smb_direct_create_qpair()
1949 ib_dealloc_pd(t->pd); in smb_direct_create_qpair()
1950 t->pd = NULL; in smb_direct_create_qpair()
1955 static int smb_direct_prepare(struct ksmbd_transport *t) in smb_direct_prepare() argument
1957 struct smb_direct_transport *st = smb_trans_direct_transfort(t); in smb_direct_prepare()
1963 ret = wait_event_interruptible_timeout(st->wait_status, in smb_direct_prepare()
1964 st->negotiation_requested || in smb_direct_prepare()
1965 st->status == SMB_DIRECT_CS_DISCONNECTED, in smb_direct_prepare()
1967 if (ret <= 0 || st->status == SMB_DIRECT_CS_DISCONNECTED) in smb_direct_prepare()
1968 return ret < 0 ? ret : -ETIMEDOUT; in smb_direct_prepare()
1972 return -ECONNABORTED; in smb_direct_prepare()
1975 if (ret == -ECONNABORTED) in smb_direct_prepare()
1978 req = (struct smb_direct_negotiate_req *)recvmsg->packet; in smb_direct_prepare()
1979 st->max_recv_size = min_t(int, st->max_recv_size, in smb_direct_prepare()
1980 le32_to_cpu(req->preferred_send_size)); in smb_direct_prepare()
1981 st->max_send_size = min_t(int, st->max_send_size, in smb_direct_prepare()
1982 le32_to_cpu(req->max_receive_size)); in smb_direct_prepare()
1983 st->max_fragmented_send_size = in smb_direct_prepare()
1984 le32_to_cpu(req->max_fragmented_size); in smb_direct_prepare()
1985 st->max_fragmented_recv_size = in smb_direct_prepare()
1986 (st->recv_credit_max * st->max_recv_size) / 2; in smb_direct_prepare()
1990 spin_lock_irq(&st->reassembly_queue_lock); in smb_direct_prepare()
1991 st->reassembly_queue_length--; in smb_direct_prepare()
1992 list_del(&recvmsg->list); in smb_direct_prepare()
1993 spin_unlock_irq(&st->reassembly_queue_lock); in smb_direct_prepare()
2006 pr_err("Can't configure RDMA parameters\n"); in smb_direct_connect()
2012 pr_err("Can't init RDMA pool: %d\n", ret); in smb_direct_connect()
2018 pr_err("Can't accept RDMA client: %d\n", ret); in smb_direct_connect()
2024 pr_err("Can't negotiate: %d\n", ret); in smb_direct_connect()
2032 if (!(attrs->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS)) in rdma_frwr_is_supported()
2034 if (attrs->max_fast_reg_page_list_len == 0) in rdma_frwr_is_supported()
2041 struct smb_direct_transport *t; in smb_direct_handle_connect_request() local
2045 if (!rdma_frwr_is_supported(&new_cm_id->device->attrs)) { in smb_direct_handle_connect_request()
2048 new_cm_id->device->attrs.device_cap_flags); in smb_direct_handle_connect_request()
2049 return -EPROTONOSUPPORT; in smb_direct_handle_connect_request()
2052 t = alloc_transport(new_cm_id); in smb_direct_handle_connect_request()
2053 if (!t) in smb_direct_handle_connect_request()
2054 return -ENOMEM; in smb_direct_handle_connect_request()
2056 ret = smb_direct_connect(t); in smb_direct_handle_connect_request()
2061 KSMBD_TRANS(t)->conn, "ksmbd:r%u", in smb_direct_handle_connect_request()
2065 pr_err("Can't start thread\n"); in smb_direct_handle_connect_request()
2071 free_transport(t); in smb_direct_handle_connect_request()
2078 switch (event->event) { in smb_direct_listen_handler()
2083 pr_err("Can't create transport: %d\n", ret); in smb_direct_listen_handler()
2093 cm_id, rdma_event_msg(event->event), event->event); in smb_direct_listen_handler()
2112 pr_err("Can't create cm id: %ld\n", PTR_ERR(cm_id)); in smb_direct_listen()
2118 pr_err("Can't bind: %d\n", ret); in smb_direct_listen()
2126 pr_err("Can't listen: %d\n", ret); in smb_direct_listen()
2141 if (ib_dev->node_type != RDMA_NODE_IB_CA) in smb_direct_ib_client_add()
2144 if (!rdma_frwr_is_supported(&ib_dev->attrs)) in smb_direct_ib_client_add()
2149 return -ENOMEM; in smb_direct_ib_client_add()
2150 smb_dev->ib_dev = ib_dev; in smb_direct_ib_client_add()
2153 list_add(&smb_dev->list, &smb_direct_device_list); in smb_direct_ib_client_add()
2156 ksmbd_debug(RDMA, "ib device added: name %s\n", ib_dev->name); in smb_direct_ib_client_add()
2167 if (smb_dev->ib_dev == ib_dev) { in smb_direct_ib_client_remove()
2168 list_del(&smb_dev->list); in smb_direct_ib_client_remove()
2199 smb_direct_wq = alloc_workqueue("ksmbd-smb_direct-wq", in ksmbd_rdma_init()
2202 return -ENOMEM; in ksmbd_rdma_init()
2208 pr_err("Can't listen: %d\n", ret); in ksmbd_rdma_init()
2241 for (i = 0; i < smb_dev->ib_dev->phys_port_cnt; i++) { in ksmbd_rdma_capable_netdev()
2244 if (smb_dev->ib_dev->ops.get_netdev) { in ksmbd_rdma_capable_netdev()
2245 ndev = smb_dev->ib_dev->ops.get_netdev( in ksmbd_rdma_capable_netdev()
2246 smb_dev->ib_dev, i + 1); in ksmbd_rdma_capable_netdev()
2259 } else if (netdev->type == ARPHRD_INFINIBAND) { in ksmbd_rdma_capable_netdev()
2266 ha, &netdev->dev_addrs) { in ksmbd_rdma_capable_netdev()
2267 memcpy(&gid, ha->addr + 4, sizeof(gid)); in ksmbd_rdma_capable_netdev()
2268 ret = ib_find_gid(smb_dev->ib_dev, &gid, in ksmbd_rdma_capable_netdev()
2286 if (rdma_frwr_is_supported(&ibdev->attrs)) in ksmbd_rdma_capable_netdev()