1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2 3 /* Authors: Cheng Xu <chengyou@linux.alibaba.com> */ 4 /* Kai Shen <kaishen@linux.alibaba.com> */ 5 /* Copyright (c) 2020-2022, Alibaba Group. */ 6 7 /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */ 8 /* Fredy Neeser */ 9 /* Greg Joyce <greg@opengridcomputing.com> */ 10 /* Copyright (c) 2008-2019, IBM Corporation */ 11 /* Copyright (c) 2017, Open Grid Computing, Inc. */ 12 13 #include <linux/workqueue.h> 14 #include <trace/events/sock.h> 15 16 #include "erdma.h" 17 #include "erdma_cm.h" 18 #include "erdma_verbs.h" 19 20 static struct workqueue_struct *erdma_cm_wq; 21 22 static void erdma_cm_llp_state_change(struct sock *sk); 23 static void erdma_cm_llp_data_ready(struct sock *sk); 24 static void erdma_cm_llp_error_report(struct sock *sk); 25 26 static void erdma_sk_assign_cm_upcalls(struct sock *sk) 27 { 28 write_lock_bh(&sk->sk_callback_lock); 29 sk->sk_state_change = erdma_cm_llp_state_change; 30 sk->sk_data_ready = erdma_cm_llp_data_ready; 31 sk->sk_error_report = erdma_cm_llp_error_report; 32 write_unlock_bh(&sk->sk_callback_lock); 33 } 34 35 static void erdma_sk_save_upcalls(struct sock *sk) 36 { 37 struct erdma_cep *cep = sk_to_cep(sk); 38 39 write_lock_bh(&sk->sk_callback_lock); 40 cep->sk_state_change = sk->sk_state_change; 41 cep->sk_data_ready = sk->sk_data_ready; 42 cep->sk_error_report = sk->sk_error_report; 43 write_unlock_bh(&sk->sk_callback_lock); 44 } 45 46 static void erdma_sk_restore_upcalls(struct sock *sk, struct erdma_cep *cep) 47 { 48 sk->sk_state_change = cep->sk_state_change; 49 sk->sk_data_ready = cep->sk_data_ready; 50 sk->sk_error_report = cep->sk_error_report; 51 sk->sk_user_data = NULL; 52 } 53 54 static void erdma_socket_disassoc(struct socket *s) 55 { 56 struct sock *sk = s->sk; 57 struct erdma_cep *cep; 58 59 if (sk) { 60 write_lock_bh(&sk->sk_callback_lock); 61 cep = sk_to_cep(sk); 62 if (cep) { 63 erdma_sk_restore_upcalls(sk, cep); 64 erdma_cep_put(cep); 65 } else { 66 WARN_ON_ONCE(1); 67 } 68 write_unlock_bh(&sk->sk_callback_lock); 69 } else { 70 WARN_ON_ONCE(1); 71 } 72 } 73 74 static void erdma_cep_socket_assoc(struct erdma_cep *cep, struct socket *s) 75 { 76 cep->sock = s; 77 erdma_cep_get(cep); 78 s->sk->sk_user_data = cep; 79 80 erdma_sk_save_upcalls(s->sk); 81 erdma_sk_assign_cm_upcalls(s->sk); 82 } 83 84 static void erdma_disassoc_listen_cep(struct erdma_cep *cep) 85 { 86 if (cep->listen_cep) { 87 erdma_cep_put(cep->listen_cep); 88 cep->listen_cep = NULL; 89 } 90 } 91 92 static struct erdma_cep *erdma_cep_alloc(struct erdma_dev *dev) 93 { 94 struct erdma_cep *cep = kzalloc(sizeof(*cep), GFP_KERNEL); 95 unsigned long flags; 96 97 if (!cep) 98 return NULL; 99 100 INIT_LIST_HEAD(&cep->listenq); 101 INIT_LIST_HEAD(&cep->devq); 102 INIT_LIST_HEAD(&cep->work_freelist); 103 104 kref_init(&cep->ref); 105 cep->state = ERDMA_EPSTATE_IDLE; 106 init_waitqueue_head(&cep->waitq); 107 spin_lock_init(&cep->lock); 108 cep->dev = dev; 109 110 spin_lock_irqsave(&dev->lock, flags); 111 list_add_tail(&cep->devq, &dev->cep_list); 112 spin_unlock_irqrestore(&dev->lock, flags); 113 114 return cep; 115 } 116 117 static void erdma_cm_free_work(struct erdma_cep *cep) 118 { 119 struct list_head *w, *tmp; 120 struct erdma_cm_work *work; 121 122 list_for_each_safe(w, tmp, &cep->work_freelist) { 123 work = list_entry(w, struct erdma_cm_work, list); 124 list_del(&work->list); 125 kfree(work); 126 } 127 } 128 129 static void erdma_cancel_mpatimer(struct erdma_cep *cep) 130 { 131 spin_lock_bh(&cep->lock); 132 if (cep->mpa_timer) { 133 if (cancel_delayed_work(&cep->mpa_timer->work)) { 134 erdma_cep_put(cep); 135 kfree(cep->mpa_timer); 136 } 137 cep->mpa_timer = NULL; 138 } 139 spin_unlock_bh(&cep->lock); 140 } 141 142 static void erdma_put_work(struct erdma_cm_work *work) 143 { 144 INIT_LIST_HEAD(&work->list); 145 spin_lock_bh(&work->cep->lock); 146 list_add(&work->list, &work->cep->work_freelist); 147 spin_unlock_bh(&work->cep->lock); 148 } 149 150 static void erdma_cep_set_inuse(struct erdma_cep *cep) 151 { 152 unsigned long flags; 153 154 spin_lock_irqsave(&cep->lock, flags); 155 while (cep->in_use) { 156 spin_unlock_irqrestore(&cep->lock, flags); 157 wait_event_interruptible(cep->waitq, !cep->in_use); 158 if (signal_pending(current)) 159 flush_signals(current); 160 161 spin_lock_irqsave(&cep->lock, flags); 162 } 163 164 cep->in_use = 1; 165 spin_unlock_irqrestore(&cep->lock, flags); 166 } 167 168 static void erdma_cep_set_free(struct erdma_cep *cep) 169 { 170 unsigned long flags; 171 172 spin_lock_irqsave(&cep->lock, flags); 173 cep->in_use = 0; 174 spin_unlock_irqrestore(&cep->lock, flags); 175 176 wake_up(&cep->waitq); 177 } 178 179 static void __erdma_cep_dealloc(struct kref *ref) 180 { 181 struct erdma_cep *cep = container_of(ref, struct erdma_cep, ref); 182 struct erdma_dev *dev = cep->dev; 183 unsigned long flags; 184 185 WARN_ON(cep->listen_cep); 186 187 kfree(cep->private_data); 188 kfree(cep->mpa.pdata); 189 spin_lock_bh(&cep->lock); 190 if (!list_empty(&cep->work_freelist)) 191 erdma_cm_free_work(cep); 192 spin_unlock_bh(&cep->lock); 193 194 spin_lock_irqsave(&dev->lock, flags); 195 list_del(&cep->devq); 196 spin_unlock_irqrestore(&dev->lock, flags); 197 kfree(cep); 198 } 199 200 static struct erdma_cm_work *erdma_get_work(struct erdma_cep *cep) 201 { 202 struct erdma_cm_work *work = NULL; 203 204 spin_lock_bh(&cep->lock); 205 if (!list_empty(&cep->work_freelist)) { 206 work = list_entry(cep->work_freelist.next, struct erdma_cm_work, 207 list); 208 list_del_init(&work->list); 209 } 210 211 spin_unlock_bh(&cep->lock); 212 return work; 213 } 214 215 static int erdma_cm_alloc_work(struct erdma_cep *cep, int num) 216 { 217 struct erdma_cm_work *work; 218 219 while (num--) { 220 work = kmalloc(sizeof(*work), GFP_KERNEL); 221 if (!work) { 222 if (!(list_empty(&cep->work_freelist))) 223 erdma_cm_free_work(cep); 224 return -ENOMEM; 225 } 226 work->cep = cep; 227 INIT_LIST_HEAD(&work->list); 228 list_add(&work->list, &cep->work_freelist); 229 } 230 231 return 0; 232 } 233 234 static int erdma_cm_upcall(struct erdma_cep *cep, enum iw_cm_event_type reason, 235 int status) 236 { 237 struct iw_cm_event event; 238 struct iw_cm_id *cm_id; 239 240 memset(&event, 0, sizeof(event)); 241 event.status = status; 242 event.event = reason; 243 244 if (reason == IW_CM_EVENT_CONNECT_REQUEST) { 245 event.provider_data = cep; 246 cm_id = cep->listen_cep->cm_id; 247 248 event.ird = cep->dev->attrs.max_ird; 249 event.ord = cep->dev->attrs.max_ord; 250 } else { 251 cm_id = cep->cm_id; 252 } 253 254 if (reason == IW_CM_EVENT_CONNECT_REQUEST || 255 reason == IW_CM_EVENT_CONNECT_REPLY) { 256 u16 pd_len = be16_to_cpu(cep->mpa.hdr.params.pd_len); 257 258 if (pd_len && cep->mpa.pdata) { 259 event.private_data_len = pd_len; 260 event.private_data = cep->mpa.pdata; 261 } 262 263 getname_local(cep->sock, &event.local_addr); 264 getname_peer(cep->sock, &event.remote_addr); 265 } 266 267 return cm_id->event_handler(cm_id, &event); 268 } 269 270 void erdma_qp_cm_drop(struct erdma_qp *qp) 271 { 272 struct erdma_cep *cep = qp->cep; 273 274 if (!qp->cep) 275 return; 276 277 erdma_cep_set_inuse(cep); 278 279 /* already closed. */ 280 if (cep->state == ERDMA_EPSTATE_CLOSED) 281 goto out; 282 283 if (cep->cm_id) { 284 switch (cep->state) { 285 case ERDMA_EPSTATE_AWAIT_MPAREP: 286 erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, 287 -EINVAL); 288 break; 289 case ERDMA_EPSTATE_RDMA_MODE: 290 erdma_cm_upcall(cep, IW_CM_EVENT_CLOSE, 0); 291 break; 292 case ERDMA_EPSTATE_IDLE: 293 case ERDMA_EPSTATE_LISTENING: 294 case ERDMA_EPSTATE_CONNECTING: 295 case ERDMA_EPSTATE_AWAIT_MPAREQ: 296 case ERDMA_EPSTATE_RECVD_MPAREQ: 297 case ERDMA_EPSTATE_CLOSED: 298 default: 299 break; 300 } 301 cep->cm_id->rem_ref(cep->cm_id); 302 cep->cm_id = NULL; 303 erdma_cep_put(cep); 304 } 305 cep->state = ERDMA_EPSTATE_CLOSED; 306 307 if (cep->sock) { 308 erdma_socket_disassoc(cep->sock); 309 sock_release(cep->sock); 310 cep->sock = NULL; 311 } 312 313 if (cep->qp) { 314 cep->qp = NULL; 315 erdma_qp_put(qp); 316 } 317 out: 318 erdma_cep_set_free(cep); 319 } 320 321 void erdma_cep_put(struct erdma_cep *cep) 322 { 323 WARN_ON(kref_read(&cep->ref) < 1); 324 kref_put(&cep->ref, __erdma_cep_dealloc); 325 } 326 327 void erdma_cep_get(struct erdma_cep *cep) 328 { 329 kref_get(&cep->ref); 330 } 331 332 static int erdma_send_mpareqrep(struct erdma_cep *cep, const void *pdata, 333 u8 pd_len) 334 { 335 struct socket *s = cep->sock; 336 struct mpa_rr *rr = &cep->mpa.hdr; 337 struct kvec iov[3]; 338 struct msghdr msg; 339 int iovec_num = 0; 340 int ret; 341 int mpa_len; 342 343 memset(&msg, 0, sizeof(msg)); 344 345 rr->params.pd_len = cpu_to_be16(pd_len); 346 347 iov[iovec_num].iov_base = rr; 348 iov[iovec_num].iov_len = sizeof(*rr); 349 iovec_num++; 350 mpa_len = sizeof(*rr); 351 352 iov[iovec_num].iov_base = &cep->mpa.ext_data; 353 iov[iovec_num].iov_len = sizeof(cep->mpa.ext_data); 354 iovec_num++; 355 mpa_len += sizeof(cep->mpa.ext_data); 356 357 if (pd_len) { 358 iov[iovec_num].iov_base = (char *)pdata; 359 iov[iovec_num].iov_len = pd_len; 360 mpa_len += pd_len; 361 iovec_num++; 362 } 363 364 ret = kernel_sendmsg(s, &msg, iov, iovec_num, mpa_len); 365 366 return ret < 0 ? ret : 0; 367 } 368 369 static inline int ksock_recv(struct socket *sock, char *buf, size_t size, 370 int flags) 371 { 372 struct kvec iov = { buf, size }; 373 struct msghdr msg = { .msg_name = NULL, .msg_flags = flags }; 374 375 return kernel_recvmsg(sock, &msg, &iov, 1, size, flags); 376 } 377 378 static int __recv_mpa_hdr(struct erdma_cep *cep, int hdr_rcvd, char *hdr, 379 int hdr_size, int *rcvd_out) 380 { 381 struct socket *s = cep->sock; 382 int rcvd; 383 384 *rcvd_out = 0; 385 if (hdr_rcvd < hdr_size) { 386 rcvd = ksock_recv(s, hdr + hdr_rcvd, hdr_size - hdr_rcvd, 387 MSG_DONTWAIT); 388 if (rcvd == -EAGAIN) 389 return -EAGAIN; 390 391 if (rcvd <= 0) 392 return -ECONNABORTED; 393 394 hdr_rcvd += rcvd; 395 *rcvd_out = rcvd; 396 397 if (hdr_rcvd < hdr_size) 398 return -EAGAIN; 399 } 400 401 return 0; 402 } 403 404 static void __mpa_rr_set_revision(__be16 *bits, u8 rev) 405 { 406 *bits = (*bits & ~MPA_RR_MASK_REVISION) | 407 (cpu_to_be16(rev) & MPA_RR_MASK_REVISION); 408 } 409 410 static u8 __mpa_rr_revision(__be16 mpa_rr_bits) 411 { 412 __be16 rev = mpa_rr_bits & MPA_RR_MASK_REVISION; 413 414 return (u8)be16_to_cpu(rev); 415 } 416 417 static void __mpa_ext_set_cc(__be32 *bits, u32 cc) 418 { 419 *bits = (*bits & ~MPA_EXT_FLAG_CC) | 420 (cpu_to_be32(cc) & MPA_EXT_FLAG_CC); 421 } 422 423 static u8 __mpa_ext_cc(__be32 mpa_ext_bits) 424 { 425 __be32 cc = mpa_ext_bits & MPA_EXT_FLAG_CC; 426 427 return (u8)be32_to_cpu(cc); 428 } 429 430 /* 431 * Receive MPA Request/Reply header. 432 * 433 * Returns 0 if complete MPA Request/Reply haeder including 434 * eventual private data was received. Returns -EAGAIN if 435 * header was partially received or negative error code otherwise. 436 * 437 * Context: May be called in process context only 438 */ 439 static int erdma_recv_mpa_rr(struct erdma_cep *cep) 440 { 441 struct mpa_rr *hdr = &cep->mpa.hdr; 442 struct socket *s = cep->sock; 443 u16 pd_len; 444 int rcvd, to_rcv, ret, pd_rcvd; 445 446 if (cep->mpa.bytes_rcvd < sizeof(struct mpa_rr)) { 447 ret = __recv_mpa_hdr(cep, cep->mpa.bytes_rcvd, 448 (char *)&cep->mpa.hdr, 449 sizeof(struct mpa_rr), &rcvd); 450 cep->mpa.bytes_rcvd += rcvd; 451 if (ret) 452 return ret; 453 } 454 455 if (be16_to_cpu(hdr->params.pd_len) > MPA_MAX_PRIVDATA || 456 __mpa_rr_revision(hdr->params.bits) != MPA_REVISION_EXT_1) 457 return -EPROTO; 458 459 if (cep->mpa.bytes_rcvd - sizeof(struct mpa_rr) < 460 sizeof(struct erdma_mpa_ext)) { 461 ret = __recv_mpa_hdr( 462 cep, cep->mpa.bytes_rcvd - sizeof(struct mpa_rr), 463 (char *)&cep->mpa.ext_data, 464 sizeof(struct erdma_mpa_ext), &rcvd); 465 cep->mpa.bytes_rcvd += rcvd; 466 if (ret) 467 return ret; 468 } 469 470 pd_len = be16_to_cpu(hdr->params.pd_len); 471 pd_rcvd = cep->mpa.bytes_rcvd - sizeof(struct mpa_rr) - 472 sizeof(struct erdma_mpa_ext); 473 to_rcv = pd_len - pd_rcvd; 474 475 if (!to_rcv) { 476 /* 477 * We have received the whole MPA Request/Reply message. 478 * Check against peer protocol violation. 479 */ 480 u32 word; 481 482 ret = __recv_mpa_hdr(cep, 0, (char *)&word, sizeof(word), 483 &rcvd); 484 if (ret == -EAGAIN && rcvd == 0) 485 return 0; 486 487 if (ret) 488 return ret; 489 490 return -EPROTO; 491 } 492 493 /* 494 * At this point, MPA header has been fully received, and pd_len != 0. 495 * So, begin to receive private data. 496 */ 497 if (!cep->mpa.pdata) { 498 cep->mpa.pdata = kmalloc(pd_len + 4, GFP_KERNEL); 499 if (!cep->mpa.pdata) 500 return -ENOMEM; 501 } 502 503 rcvd = ksock_recv(s, cep->mpa.pdata + pd_rcvd, to_rcv + 4, 504 MSG_DONTWAIT); 505 if (rcvd < 0) 506 return rcvd; 507 508 if (rcvd > to_rcv) 509 return -EPROTO; 510 511 cep->mpa.bytes_rcvd += rcvd; 512 513 if (to_rcv == rcvd) 514 return 0; 515 516 return -EAGAIN; 517 } 518 519 /* 520 * erdma_proc_mpareq() 521 * 522 * Read MPA Request from socket and signal new connection to IWCM 523 * if success. Caller must hold lock on corresponding listening CEP. 524 */ 525 static int erdma_proc_mpareq(struct erdma_cep *cep) 526 { 527 struct mpa_rr *req; 528 int ret; 529 530 ret = erdma_recv_mpa_rr(cep); 531 if (ret) 532 return ret; 533 534 req = &cep->mpa.hdr; 535 536 if (memcmp(req->key, MPA_KEY_REQ, MPA_KEY_SIZE)) 537 return -EPROTO; 538 539 memcpy(req->key, MPA_KEY_REP, MPA_KEY_SIZE); 540 541 /* Currently does not support marker and crc. */ 542 if (req->params.bits & MPA_RR_FLAG_MARKERS || 543 req->params.bits & MPA_RR_FLAG_CRC) 544 goto reject_conn; 545 546 cep->state = ERDMA_EPSTATE_RECVD_MPAREQ; 547 548 /* Keep reference until IWCM accepts/rejects */ 549 erdma_cep_get(cep); 550 ret = erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REQUEST, 0); 551 if (ret) 552 erdma_cep_put(cep); 553 554 return ret; 555 556 reject_conn: 557 req->params.bits &= ~MPA_RR_FLAG_MARKERS; 558 req->params.bits |= MPA_RR_FLAG_REJECT; 559 req->params.bits &= ~MPA_RR_FLAG_CRC; 560 561 kfree(cep->mpa.pdata); 562 cep->mpa.pdata = NULL; 563 erdma_send_mpareqrep(cep, NULL, 0); 564 565 return -EOPNOTSUPP; 566 } 567 568 static int erdma_proc_mpareply(struct erdma_cep *cep) 569 { 570 enum erdma_qpa_mask_iwarp to_modify_attrs = 0; 571 struct erdma_mod_qp_params_iwarp params; 572 struct erdma_qp *qp = cep->qp; 573 struct mpa_rr *rep; 574 int ret; 575 576 ret = erdma_recv_mpa_rr(cep); 577 if (ret) 578 goto out_err; 579 580 erdma_cancel_mpatimer(cep); 581 582 rep = &cep->mpa.hdr; 583 584 if (memcmp(rep->key, MPA_KEY_REP, MPA_KEY_SIZE)) { 585 ret = -EPROTO; 586 goto out_err; 587 } 588 589 if (rep->params.bits & MPA_RR_FLAG_REJECT) { 590 erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, -ECONNRESET); 591 return -ECONNRESET; 592 } 593 594 /* Currently does not support marker and crc. */ 595 if ((rep->params.bits & MPA_RR_FLAG_MARKERS) || 596 (rep->params.bits & MPA_RR_FLAG_CRC)) { 597 erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, -ECONNREFUSED); 598 return -EINVAL; 599 } 600 601 memset(¶ms, 0, sizeof(params)); 602 params.state = ERDMA_QPS_IWARP_RTS; 603 params.irq_size = cep->ird; 604 params.orq_size = cep->ord; 605 606 down_write(&qp->state_lock); 607 if (qp->attrs.iwarp.state > ERDMA_QPS_IWARP_RTR) { 608 ret = -EINVAL; 609 up_write(&qp->state_lock); 610 goto out_err; 611 } 612 613 to_modify_attrs = ERDMA_QPA_IWARP_STATE | ERDMA_QPA_IWARP_LLP_HANDLE | 614 ERDMA_QPA_IWARP_MPA | ERDMA_QPA_IWARP_IRD | 615 ERDMA_QPA_IWARP_ORD; 616 617 params.qp_type = ERDMA_QP_ACTIVE; 618 if (__mpa_ext_cc(cep->mpa.ext_data.bits) != qp->attrs.cc) { 619 to_modify_attrs |= ERDMA_QPA_IWARP_CC; 620 params.cc = COMPROMISE_CC; 621 } 622 623 ret = erdma_modify_qp_state_iwarp(qp, ¶ms, to_modify_attrs); 624 625 up_write(&qp->state_lock); 626 627 if (!ret) { 628 ret = erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, 0); 629 if (!ret) 630 cep->state = ERDMA_EPSTATE_RDMA_MODE; 631 632 return 0; 633 } 634 635 out_err: 636 if (ret != -EAGAIN) 637 erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, -EINVAL); 638 639 return ret; 640 } 641 642 static void erdma_accept_newconn(struct erdma_cep *cep) 643 { 644 struct socket *s = cep->sock; 645 struct socket *new_s = NULL; 646 struct erdma_cep *new_cep = NULL; 647 int ret = 0; 648 649 if (cep->state != ERDMA_EPSTATE_LISTENING) 650 goto error; 651 652 new_cep = erdma_cep_alloc(cep->dev); 653 if (!new_cep) 654 goto error; 655 656 /* 657 * 4: Allocate a sufficient number of work elements 658 * to allow concurrent handling of local + peer close 659 * events, MPA header processing + MPA timeout. 660 */ 661 if (erdma_cm_alloc_work(new_cep, 4) != 0) 662 goto error; 663 664 /* 665 * Copy saved socket callbacks from listening CEP 666 * and assign new socket with new CEP 667 */ 668 new_cep->sk_state_change = cep->sk_state_change; 669 new_cep->sk_data_ready = cep->sk_data_ready; 670 new_cep->sk_error_report = cep->sk_error_report; 671 672 ret = kernel_accept(s, &new_s, O_NONBLOCK); 673 if (ret != 0) 674 goto error; 675 676 new_cep->sock = new_s; 677 erdma_cep_get(new_cep); 678 new_s->sk->sk_user_data = new_cep; 679 680 tcp_sock_set_nodelay(new_s->sk); 681 new_cep->state = ERDMA_EPSTATE_AWAIT_MPAREQ; 682 683 ret = erdma_cm_queue_work(new_cep, ERDMA_CM_WORK_MPATIMEOUT); 684 if (ret) 685 goto error; 686 687 new_cep->listen_cep = cep; 688 erdma_cep_get(cep); 689 690 if (atomic_read(&new_s->sk->sk_rmem_alloc)) { 691 /* MPA REQ already queued */ 692 erdma_cep_set_inuse(new_cep); 693 ret = erdma_proc_mpareq(new_cep); 694 if (ret != -EAGAIN) { 695 erdma_cep_put(cep); 696 new_cep->listen_cep = NULL; 697 if (ret) { 698 erdma_cep_set_free(new_cep); 699 goto error; 700 } 701 } 702 erdma_cep_set_free(new_cep); 703 } 704 return; 705 706 error: 707 if (new_cep) { 708 new_cep->state = ERDMA_EPSTATE_CLOSED; 709 erdma_cancel_mpatimer(new_cep); 710 711 erdma_cep_put(new_cep); 712 new_cep->sock = NULL; 713 } 714 715 if (new_s) { 716 erdma_socket_disassoc(new_s); 717 sock_release(new_s); 718 } 719 } 720 721 static int erdma_newconn_connected(struct erdma_cep *cep) 722 { 723 int ret = 0; 724 725 cep->mpa.hdr.params.bits = 0; 726 __mpa_rr_set_revision(&cep->mpa.hdr.params.bits, MPA_REVISION_EXT_1); 727 728 memcpy(cep->mpa.hdr.key, MPA_KEY_REQ, MPA_KEY_SIZE); 729 cep->mpa.ext_data.cookie = cpu_to_be32(cep->qp->attrs.iwarp.cookie); 730 __mpa_ext_set_cc(&cep->mpa.ext_data.bits, cep->qp->attrs.cc); 731 732 ret = erdma_send_mpareqrep(cep, cep->private_data, cep->pd_len); 733 cep->state = ERDMA_EPSTATE_AWAIT_MPAREP; 734 cep->mpa.hdr.params.pd_len = 0; 735 736 if (ret >= 0) 737 ret = erdma_cm_queue_work(cep, ERDMA_CM_WORK_MPATIMEOUT); 738 739 return ret; 740 } 741 742 static void erdma_cm_work_handler(struct work_struct *w) 743 { 744 struct erdma_cm_work *work; 745 struct erdma_cep *cep; 746 int release_cep = 0, ret = 0; 747 748 work = container_of(w, struct erdma_cm_work, work.work); 749 cep = work->cep; 750 751 erdma_cep_set_inuse(cep); 752 753 switch (work->type) { 754 case ERDMA_CM_WORK_CONNECTED: 755 erdma_cancel_mpatimer(cep); 756 if (cep->state == ERDMA_EPSTATE_CONNECTING) { 757 ret = erdma_newconn_connected(cep); 758 if (ret) { 759 erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, 760 -EIO); 761 release_cep = 1; 762 } 763 } 764 break; 765 case ERDMA_CM_WORK_CONNECTTIMEOUT: 766 if (cep->state == ERDMA_EPSTATE_CONNECTING) { 767 cep->mpa_timer = NULL; 768 erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, 769 -ETIMEDOUT); 770 release_cep = 1; 771 } 772 break; 773 case ERDMA_CM_WORK_ACCEPT: 774 erdma_accept_newconn(cep); 775 break; 776 case ERDMA_CM_WORK_READ_MPAHDR: 777 if (cep->state == ERDMA_EPSTATE_AWAIT_MPAREQ) { 778 if (cep->listen_cep) { 779 erdma_cep_set_inuse(cep->listen_cep); 780 781 if (cep->listen_cep->state == 782 ERDMA_EPSTATE_LISTENING) 783 ret = erdma_proc_mpareq(cep); 784 else 785 ret = -EFAULT; 786 787 erdma_cep_set_free(cep->listen_cep); 788 789 if (ret != -EAGAIN) { 790 erdma_cep_put(cep->listen_cep); 791 cep->listen_cep = NULL; 792 if (ret) 793 erdma_cep_put(cep); 794 } 795 } 796 } else if (cep->state == ERDMA_EPSTATE_AWAIT_MPAREP) { 797 ret = erdma_proc_mpareply(cep); 798 } 799 800 if (ret && ret != -EAGAIN) 801 release_cep = 1; 802 break; 803 case ERDMA_CM_WORK_CLOSE_LLP: 804 if (cep->cm_id) 805 erdma_cm_upcall(cep, IW_CM_EVENT_CLOSE, 0); 806 release_cep = 1; 807 break; 808 case ERDMA_CM_WORK_PEER_CLOSE: 809 if (cep->cm_id) { 810 if (cep->state == ERDMA_EPSTATE_CONNECTING || 811 cep->state == ERDMA_EPSTATE_AWAIT_MPAREP) { 812 /* 813 * MPA reply not received, but connection drop 814 */ 815 erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, 816 -ECONNRESET); 817 } else if (cep->state == ERDMA_EPSTATE_RDMA_MODE) { 818 /* 819 * NOTE: IW_CM_EVENT_DISCONNECT is given just 820 * to transition IWCM into CLOSING. 821 */ 822 erdma_cm_upcall(cep, IW_CM_EVENT_DISCONNECT, 0); 823 erdma_cm_upcall(cep, IW_CM_EVENT_CLOSE, 0); 824 } 825 } else if (cep->state == ERDMA_EPSTATE_AWAIT_MPAREQ) { 826 /* Socket close before MPA request received. */ 827 erdma_disassoc_listen_cep(cep); 828 erdma_cep_put(cep); 829 } 830 release_cep = 1; 831 break; 832 case ERDMA_CM_WORK_MPATIMEOUT: 833 cep->mpa_timer = NULL; 834 if (cep->state == ERDMA_EPSTATE_AWAIT_MPAREP) { 835 /* 836 * MPA request timed out: 837 * Hide any partially received private data and signal 838 * timeout 839 */ 840 cep->mpa.hdr.params.pd_len = 0; 841 842 if (cep->cm_id) 843 erdma_cm_upcall(cep, IW_CM_EVENT_CONNECT_REPLY, 844 -ETIMEDOUT); 845 release_cep = 1; 846 } else if (cep->state == ERDMA_EPSTATE_AWAIT_MPAREQ) { 847 /* No MPA req received after peer TCP stream setup. */ 848 erdma_disassoc_listen_cep(cep); 849 850 erdma_cep_put(cep); 851 release_cep = 1; 852 } 853 break; 854 default: 855 WARN(1, "Undefined CM work type: %d\n", work->type); 856 } 857 858 if (release_cep) { 859 erdma_cancel_mpatimer(cep); 860 cep->state = ERDMA_EPSTATE_CLOSED; 861 if (cep->qp) { 862 struct erdma_qp *qp = cep->qp; 863 /* 864 * Serialize a potential race with application 865 * closing the QP and calling erdma_qp_cm_drop() 866 */ 867 erdma_qp_get(qp); 868 erdma_cep_set_free(cep); 869 870 erdma_qp_llp_close(qp); 871 erdma_qp_put(qp); 872 873 erdma_cep_set_inuse(cep); 874 cep->qp = NULL; 875 erdma_qp_put(qp); 876 } 877 878 if (cep->sock) { 879 erdma_socket_disassoc(cep->sock); 880 sock_release(cep->sock); 881 cep->sock = NULL; 882 } 883 884 if (cep->cm_id) { 885 cep->cm_id->rem_ref(cep->cm_id); 886 cep->cm_id = NULL; 887 if (cep->state != ERDMA_EPSTATE_LISTENING) 888 erdma_cep_put(cep); 889 } 890 } 891 erdma_cep_set_free(cep); 892 erdma_put_work(work); 893 erdma_cep_put(cep); 894 } 895 896 int erdma_cm_queue_work(struct erdma_cep *cep, enum erdma_work_type type) 897 { 898 struct erdma_cm_work *work = erdma_get_work(cep); 899 unsigned long delay = 0; 900 901 if (!work) 902 return -ENOMEM; 903 904 work->type = type; 905 work->cep = cep; 906 907 erdma_cep_get(cep); 908 909 INIT_DELAYED_WORK(&work->work, erdma_cm_work_handler); 910 911 if (type == ERDMA_CM_WORK_MPATIMEOUT) { 912 cep->mpa_timer = work; 913 914 if (cep->state == ERDMA_EPSTATE_AWAIT_MPAREP) 915 delay = MPAREP_TIMEOUT; 916 else 917 delay = MPAREQ_TIMEOUT; 918 } else if (type == ERDMA_CM_WORK_CONNECTTIMEOUT) { 919 cep->mpa_timer = work; 920 921 delay = CONNECT_TIMEOUT; 922 } 923 924 queue_delayed_work(erdma_cm_wq, &work->work, delay); 925 926 return 0; 927 } 928 929 static void erdma_cm_llp_data_ready(struct sock *sk) 930 { 931 struct erdma_cep *cep; 932 933 trace_sk_data_ready(sk); 934 935 read_lock(&sk->sk_callback_lock); 936 937 cep = sk_to_cep(sk); 938 if (!cep) 939 goto out; 940 941 if (cep->state == ERDMA_EPSTATE_AWAIT_MPAREQ || 942 cep->state == ERDMA_EPSTATE_AWAIT_MPAREP) 943 erdma_cm_queue_work(cep, ERDMA_CM_WORK_READ_MPAHDR); 944 945 out: 946 read_unlock(&sk->sk_callback_lock); 947 } 948 949 static void erdma_cm_llp_error_report(struct sock *sk) 950 { 951 struct erdma_cep *cep = sk_to_cep(sk); 952 953 if (cep) 954 cep->sk_error_report(sk); 955 } 956 957 static void erdma_cm_llp_state_change(struct sock *sk) 958 { 959 struct erdma_cep *cep; 960 void (*orig_state_change)(struct sock *sk); 961 962 read_lock(&sk->sk_callback_lock); 963 964 cep = sk_to_cep(sk); 965 if (!cep) { 966 read_unlock(&sk->sk_callback_lock); 967 return; 968 } 969 orig_state_change = cep->sk_state_change; 970 971 switch (sk->sk_state) { 972 case TCP_ESTABLISHED: 973 if (cep->state == ERDMA_EPSTATE_CONNECTING) 974 erdma_cm_queue_work(cep, ERDMA_CM_WORK_CONNECTED); 975 else 976 erdma_cm_queue_work(cep, ERDMA_CM_WORK_ACCEPT); 977 break; 978 case TCP_CLOSE: 979 case TCP_CLOSE_WAIT: 980 if (cep->state != ERDMA_EPSTATE_LISTENING) 981 erdma_cm_queue_work(cep, ERDMA_CM_WORK_PEER_CLOSE); 982 break; 983 default: 984 break; 985 } 986 read_unlock(&sk->sk_callback_lock); 987 orig_state_change(sk); 988 } 989 990 static int kernel_bindconnect(struct socket *s, struct sockaddr *laddr, 991 int laddrlen, struct sockaddr *raddr, 992 int raddrlen, int flags) 993 { 994 int ret; 995 996 sock_set_reuseaddr(s->sk); 997 ret = s->ops->bind(s, laddr, laddrlen); 998 if (ret) 999 return ret; 1000 ret = s->ops->connect(s, raddr, raddrlen, flags); 1001 return ret < 0 ? ret : 0; 1002 } 1003 1004 int erdma_connect(struct iw_cm_id *id, struct iw_cm_conn_param *params) 1005 { 1006 struct erdma_dev *dev = to_edev(id->device); 1007 struct erdma_qp *qp; 1008 struct erdma_cep *cep = NULL; 1009 struct socket *s = NULL; 1010 struct sockaddr *laddr = (struct sockaddr *)&id->m_local_addr; 1011 struct sockaddr *raddr = (struct sockaddr *)&id->m_remote_addr; 1012 u16 pd_len = params->private_data_len; 1013 int ret; 1014 1015 if (pd_len > MPA_MAX_PRIVDATA) 1016 return -EINVAL; 1017 1018 if (params->ird > dev->attrs.max_ird || 1019 params->ord > dev->attrs.max_ord) 1020 return -EINVAL; 1021 1022 if (laddr->sa_family != AF_INET || raddr->sa_family != AF_INET) 1023 return -EAFNOSUPPORT; 1024 1025 qp = find_qp_by_qpn(dev, params->qpn); 1026 if (!qp) 1027 return -ENOENT; 1028 erdma_qp_get(qp); 1029 1030 ret = sock_create(AF_INET, SOCK_STREAM, IPPROTO_TCP, &s); 1031 if (ret < 0) 1032 goto error_put_qp; 1033 1034 cep = erdma_cep_alloc(dev); 1035 if (!cep) { 1036 ret = -ENOMEM; 1037 goto error_release_sock; 1038 } 1039 1040 erdma_cep_set_inuse(cep); 1041 1042 /* Associate QP with CEP */ 1043 erdma_cep_get(cep); 1044 qp->cep = cep; 1045 cep->qp = qp; 1046 1047 /* Associate cm_id with CEP */ 1048 id->add_ref(id); 1049 cep->cm_id = id; 1050 1051 /* 1052 * 6: Allocate a sufficient number of work elements 1053 * to allow concurrent handling of local + peer close 1054 * events, MPA header processing + MPA timeout, connected event 1055 * and connect timeout. 1056 */ 1057 ret = erdma_cm_alloc_work(cep, 6); 1058 if (ret != 0) { 1059 ret = -ENOMEM; 1060 goto error_release_cep; 1061 } 1062 1063 cep->ird = params->ird; 1064 cep->ord = params->ord; 1065 cep->state = ERDMA_EPSTATE_CONNECTING; 1066 1067 erdma_cep_socket_assoc(cep, s); 1068 1069 if (pd_len) { 1070 cep->pd_len = pd_len; 1071 cep->private_data = kmalloc(pd_len, GFP_KERNEL); 1072 if (!cep->private_data) { 1073 ret = -ENOMEM; 1074 goto error_disassoc; 1075 } 1076 1077 memcpy(cep->private_data, params->private_data, 1078 params->private_data_len); 1079 } 1080 1081 ret = kernel_bindconnect(s, laddr, sizeof(*laddr), raddr, 1082 sizeof(*raddr), O_NONBLOCK); 1083 if (ret != -EINPROGRESS && ret != 0) { 1084 goto error_disassoc; 1085 } else if (ret == 0) { 1086 ret = erdma_cm_queue_work(cep, ERDMA_CM_WORK_CONNECTED); 1087 if (ret) 1088 goto error_disassoc; 1089 } else { 1090 ret = erdma_cm_queue_work(cep, ERDMA_CM_WORK_CONNECTTIMEOUT); 1091 if (ret) 1092 goto error_disassoc; 1093 } 1094 1095 erdma_cep_set_free(cep); 1096 return 0; 1097 1098 error_disassoc: 1099 kfree(cep->private_data); 1100 cep->private_data = NULL; 1101 cep->pd_len = 0; 1102 1103 erdma_socket_disassoc(s); 1104 1105 error_release_cep: 1106 /* disassoc with cm_id */ 1107 cep->cm_id = NULL; 1108 id->rem_ref(id); 1109 1110 /* disassoc with qp */ 1111 qp->cep = NULL; 1112 erdma_cep_put(cep); 1113 cep->qp = NULL; 1114 1115 cep->state = ERDMA_EPSTATE_CLOSED; 1116 1117 erdma_cep_set_free(cep); 1118 1119 /* release the cep. */ 1120 erdma_cep_put(cep); 1121 1122 error_release_sock: 1123 if (s) 1124 sock_release(s); 1125 error_put_qp: 1126 erdma_qp_put(qp); 1127 1128 return ret; 1129 } 1130 1131 int erdma_accept(struct iw_cm_id *id, struct iw_cm_conn_param *params) 1132 { 1133 struct erdma_cep *cep = (struct erdma_cep *)id->provider_data; 1134 struct erdma_mod_qp_params_iwarp mod_qp_params; 1135 enum erdma_qpa_mask_iwarp to_modify_attrs = 0; 1136 struct erdma_dev *dev = to_edev(id->device); 1137 struct erdma_qp *qp; 1138 int ret; 1139 1140 erdma_cep_set_inuse(cep); 1141 erdma_cep_put(cep); 1142 1143 /* Free lingering inbound private data */ 1144 if (cep->mpa.hdr.params.pd_len) { 1145 cep->mpa.hdr.params.pd_len = 0; 1146 kfree(cep->mpa.pdata); 1147 cep->mpa.pdata = NULL; 1148 } 1149 erdma_cancel_mpatimer(cep); 1150 1151 if (cep->state != ERDMA_EPSTATE_RECVD_MPAREQ) { 1152 erdma_cep_set_free(cep); 1153 erdma_cep_put(cep); 1154 1155 return -ECONNRESET; 1156 } 1157 1158 qp = find_qp_by_qpn(dev, params->qpn); 1159 if (!qp) 1160 return -ENOENT; 1161 erdma_qp_get(qp); 1162 1163 down_write(&qp->state_lock); 1164 if (qp->attrs.iwarp.state > ERDMA_QPS_IWARP_RTR) { 1165 ret = -EINVAL; 1166 up_write(&qp->state_lock); 1167 goto error; 1168 } 1169 1170 if (params->ord > dev->attrs.max_ord || 1171 params->ird > dev->attrs.max_ord) { 1172 ret = -EINVAL; 1173 up_write(&qp->state_lock); 1174 goto error; 1175 } 1176 1177 if (params->private_data_len > MPA_MAX_PRIVDATA) { 1178 ret = -EINVAL; 1179 up_write(&qp->state_lock); 1180 goto error; 1181 } 1182 1183 cep->ird = params->ird; 1184 cep->ord = params->ord; 1185 1186 cep->cm_id = id; 1187 id->add_ref(id); 1188 1189 memset(&mod_qp_params, 0, sizeof(mod_qp_params)); 1190 1191 mod_qp_params.irq_size = params->ird; 1192 mod_qp_params.orq_size = params->ord; 1193 mod_qp_params.state = ERDMA_QPS_IWARP_RTS; 1194 1195 /* Associate QP with CEP */ 1196 erdma_cep_get(cep); 1197 qp->cep = cep; 1198 cep->qp = qp; 1199 1200 cep->state = ERDMA_EPSTATE_RDMA_MODE; 1201 1202 mod_qp_params.qp_type = ERDMA_QP_PASSIVE; 1203 mod_qp_params.pd_len = params->private_data_len; 1204 1205 to_modify_attrs = ERDMA_QPA_IWARP_STATE | ERDMA_QPA_IWARP_ORD | 1206 ERDMA_QPA_IWARP_LLP_HANDLE | ERDMA_QPA_IWARP_IRD | 1207 ERDMA_QPA_IWARP_MPA; 1208 1209 if (qp->attrs.cc != __mpa_ext_cc(cep->mpa.ext_data.bits)) { 1210 to_modify_attrs |= ERDMA_QPA_IWARP_CC; 1211 mod_qp_params.cc = COMPROMISE_CC; 1212 } 1213 1214 /* move to rts */ 1215 ret = erdma_modify_qp_state_iwarp(qp, &mod_qp_params, to_modify_attrs); 1216 1217 up_write(&qp->state_lock); 1218 1219 if (ret) 1220 goto error; 1221 1222 cep->mpa.ext_data.bits = 0; 1223 __mpa_ext_set_cc(&cep->mpa.ext_data.bits, qp->attrs.cc); 1224 cep->mpa.ext_data.cookie = cpu_to_be32(cep->qp->attrs.iwarp.cookie); 1225 1226 ret = erdma_send_mpareqrep(cep, params->private_data, 1227 params->private_data_len); 1228 if (!ret) { 1229 ret = erdma_cm_upcall(cep, IW_CM_EVENT_ESTABLISHED, 0); 1230 if (ret) 1231 goto error; 1232 1233 erdma_cep_set_free(cep); 1234 1235 return 0; 1236 } 1237 1238 error: 1239 erdma_socket_disassoc(cep->sock); 1240 sock_release(cep->sock); 1241 cep->sock = NULL; 1242 1243 cep->state = ERDMA_EPSTATE_CLOSED; 1244 1245 if (cep->cm_id) { 1246 cep->cm_id->rem_ref(id); 1247 cep->cm_id = NULL; 1248 } 1249 1250 if (qp->cep) { 1251 erdma_cep_put(cep); 1252 qp->cep = NULL; 1253 } 1254 1255 cep->qp = NULL; 1256 erdma_qp_put(qp); 1257 1258 erdma_cep_set_free(cep); 1259 erdma_cep_put(cep); 1260 1261 return ret; 1262 } 1263 1264 int erdma_reject(struct iw_cm_id *id, const void *pdata, u8 plen) 1265 { 1266 struct erdma_cep *cep = (struct erdma_cep *)id->provider_data; 1267 1268 erdma_cep_set_inuse(cep); 1269 erdma_cep_put(cep); 1270 1271 erdma_cancel_mpatimer(cep); 1272 1273 if (cep->state != ERDMA_EPSTATE_RECVD_MPAREQ) { 1274 erdma_cep_set_free(cep); 1275 erdma_cep_put(cep); 1276 1277 return -ECONNRESET; 1278 } 1279 1280 if (__mpa_rr_revision(cep->mpa.hdr.params.bits) == MPA_REVISION_EXT_1) { 1281 cep->mpa.hdr.params.bits |= MPA_RR_FLAG_REJECT; /* reject */ 1282 erdma_send_mpareqrep(cep, pdata, plen); 1283 } 1284 1285 erdma_socket_disassoc(cep->sock); 1286 sock_release(cep->sock); 1287 cep->sock = NULL; 1288 1289 cep->state = ERDMA_EPSTATE_CLOSED; 1290 1291 erdma_cep_set_free(cep); 1292 erdma_cep_put(cep); 1293 1294 return 0; 1295 } 1296 1297 int erdma_create_listen(struct iw_cm_id *id, int backlog) 1298 { 1299 struct socket *s; 1300 struct erdma_cep *cep = NULL; 1301 int ret = 0; 1302 struct erdma_dev *dev = to_edev(id->device); 1303 int addr_family = id->local_addr.ss_family; 1304 struct sockaddr_in *laddr = &to_sockaddr_in(id->local_addr); 1305 1306 if (addr_family != AF_INET) 1307 return -EAFNOSUPPORT; 1308 1309 ret = sock_create(addr_family, SOCK_STREAM, IPPROTO_TCP, &s); 1310 if (ret < 0) 1311 return ret; 1312 1313 sock_set_reuseaddr(s->sk); 1314 1315 /* For wildcard addr, limit binding to current device only */ 1316 if (ipv4_is_zeronet(laddr->sin_addr.s_addr)) 1317 s->sk->sk_bound_dev_if = dev->netdev->ifindex; 1318 1319 ret = s->ops->bind(s, (struct sockaddr *)laddr, 1320 sizeof(struct sockaddr_in)); 1321 if (ret) 1322 goto error; 1323 1324 cep = erdma_cep_alloc(dev); 1325 if (!cep) { 1326 ret = -ENOMEM; 1327 goto error; 1328 } 1329 erdma_cep_socket_assoc(cep, s); 1330 1331 ret = erdma_cm_alloc_work(cep, backlog); 1332 if (ret) 1333 goto error; 1334 1335 ret = s->ops->listen(s, backlog); 1336 if (ret) 1337 goto error; 1338 1339 cep->cm_id = id; 1340 id->add_ref(id); 1341 1342 if (!id->provider_data) { 1343 id->provider_data = 1344 kmalloc(sizeof(struct list_head), GFP_KERNEL); 1345 if (!id->provider_data) { 1346 ret = -ENOMEM; 1347 goto error; 1348 } 1349 INIT_LIST_HEAD((struct list_head *)id->provider_data); 1350 } 1351 1352 list_add_tail(&cep->listenq, (struct list_head *)id->provider_data); 1353 cep->state = ERDMA_EPSTATE_LISTENING; 1354 1355 return 0; 1356 1357 error: 1358 if (cep) { 1359 erdma_cep_set_inuse(cep); 1360 1361 if (cep->cm_id) { 1362 cep->cm_id->rem_ref(cep->cm_id); 1363 cep->cm_id = NULL; 1364 } 1365 cep->sock = NULL; 1366 erdma_socket_disassoc(s); 1367 cep->state = ERDMA_EPSTATE_CLOSED; 1368 1369 erdma_cep_set_free(cep); 1370 erdma_cep_put(cep); 1371 } 1372 sock_release(s); 1373 1374 return ret; 1375 } 1376 1377 static void erdma_drop_listeners(struct iw_cm_id *id) 1378 { 1379 struct list_head *p, *tmp; 1380 /* 1381 * In case of a wildcard rdma_listen on a multi-homed device, 1382 * a listener's IWCM id is associated with more than one listening CEP. 1383 */ 1384 list_for_each_safe(p, tmp, (struct list_head *)id->provider_data) { 1385 struct erdma_cep *cep = 1386 list_entry(p, struct erdma_cep, listenq); 1387 1388 list_del(p); 1389 1390 erdma_cep_set_inuse(cep); 1391 1392 if (cep->cm_id) { 1393 cep->cm_id->rem_ref(cep->cm_id); 1394 cep->cm_id = NULL; 1395 } 1396 if (cep->sock) { 1397 erdma_socket_disassoc(cep->sock); 1398 sock_release(cep->sock); 1399 cep->sock = NULL; 1400 } 1401 cep->state = ERDMA_EPSTATE_CLOSED; 1402 erdma_cep_set_free(cep); 1403 erdma_cep_put(cep); 1404 } 1405 } 1406 1407 int erdma_destroy_listen(struct iw_cm_id *id) 1408 { 1409 if (!id->provider_data) 1410 return 0; 1411 1412 erdma_drop_listeners(id); 1413 kfree(id->provider_data); 1414 id->provider_data = NULL; 1415 1416 return 0; 1417 } 1418 1419 int erdma_cm_init(void) 1420 { 1421 erdma_cm_wq = create_singlethread_workqueue("erdma_cm_wq"); 1422 if (!erdma_cm_wq) 1423 return -ENOMEM; 1424 1425 return 0; 1426 } 1427 1428 void erdma_cm_exit(void) 1429 { 1430 if (erdma_cm_wq) 1431 destroy_workqueue(erdma_cm_wq); 1432 } 1433