1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. 4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. 5 */ 6 7 #include <linux/dma-mapping.h> 8 #include <net/addrconf.h> 9 #include <rdma/uverbs_ioctl.h> 10 11 #include "rxe.h" 12 #include "rxe_queue.h" 13 #include "rxe_hw_counters.h" 14 15 static int post_one_recv(struct rxe_rq *rq, const struct ib_recv_wr *ibwr); 16 17 /* dev */ 18 static int rxe_query_device(struct ib_device *ibdev, 19 struct ib_device_attr *attr, 20 struct ib_udata *udata) 21 { 22 struct rxe_dev *rxe = to_rdev(ibdev); 23 int err; 24 25 if (udata->inlen || udata->outlen) { 26 rxe_dbg_dev(rxe, "malformed udata\n"); 27 err = -EINVAL; 28 goto err_out; 29 } 30 31 memcpy(attr, &rxe->attr, sizeof(*attr)); 32 33 return 0; 34 35 err_out: 36 rxe_err_dev(rxe, "returned err = %d\n", err); 37 return err; 38 } 39 40 static int rxe_query_port(struct ib_device *ibdev, 41 u32 port_num, struct ib_port_attr *attr) 42 { 43 struct rxe_dev *rxe = to_rdev(ibdev); 44 struct net_device *ndev; 45 int err, ret; 46 47 if (port_num != 1) { 48 err = -EINVAL; 49 rxe_dbg_dev(rxe, "bad port_num = %d\n", port_num); 50 goto err_out; 51 } 52 53 ndev = rxe_ib_device_get_netdev(ibdev); 54 if (!ndev) { 55 err = -ENODEV; 56 goto err_out; 57 } 58 59 memcpy(attr, &rxe->port.attr, sizeof(*attr)); 60 61 mutex_lock(&rxe->usdev_lock); 62 ret = ib_get_eth_speed(ibdev, port_num, &attr->active_speed, 63 &attr->active_width); 64 65 attr->state = ib_get_curr_port_state(ndev); 66 if (attr->state == IB_PORT_ACTIVE) 67 attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP; 68 else if (dev_get_flags(ndev) & IFF_UP) 69 attr->phys_state = IB_PORT_PHYS_STATE_POLLING; 70 else 71 attr->phys_state = IB_PORT_PHYS_STATE_DISABLED; 72 73 mutex_unlock(&rxe->usdev_lock); 74 75 dev_put(ndev); 76 return ret; 77 78 err_out: 79 rxe_err_dev(rxe, "returned err = %d\n", err); 80 return err; 81 } 82 83 static int rxe_query_gid(struct ib_device *ibdev, u32 port, int idx, 84 union ib_gid *gid) 85 { 86 struct rxe_dev *rxe = to_rdev(ibdev); 87 88 /* subnet_prefix == interface_id == 0; */ 89 memset(gid, 0, sizeof(*gid)); 90 memcpy(gid->raw, rxe->raw_gid, ETH_ALEN); 91 92 return 0; 93 } 94 95 static int rxe_query_pkey(struct ib_device *ibdev, 96 u32 port_num, u16 index, u16 *pkey) 97 { 98 struct rxe_dev *rxe = to_rdev(ibdev); 99 int err; 100 101 if (index != 0) { 102 err = -EINVAL; 103 rxe_dbg_dev(rxe, "bad pkey index = %d\n", index); 104 goto err_out; 105 } 106 107 *pkey = IB_DEFAULT_PKEY_FULL; 108 return 0; 109 110 err_out: 111 rxe_err_dev(rxe, "returned err = %d\n", err); 112 return err; 113 } 114 115 static int rxe_modify_device(struct ib_device *ibdev, 116 int mask, struct ib_device_modify *attr) 117 { 118 struct rxe_dev *rxe = to_rdev(ibdev); 119 int err; 120 121 if (mask & ~(IB_DEVICE_MODIFY_SYS_IMAGE_GUID | 122 IB_DEVICE_MODIFY_NODE_DESC)) { 123 err = -EOPNOTSUPP; 124 rxe_dbg_dev(rxe, "unsupported mask = 0x%x\n", mask); 125 goto err_out; 126 } 127 128 if (mask & IB_DEVICE_MODIFY_SYS_IMAGE_GUID) 129 rxe->attr.sys_image_guid = cpu_to_be64(attr->sys_image_guid); 130 131 if (mask & IB_DEVICE_MODIFY_NODE_DESC) { 132 memcpy(rxe->ib_dev.node_desc, 133 attr->node_desc, sizeof(rxe->ib_dev.node_desc)); 134 } 135 136 return 0; 137 138 err_out: 139 rxe_err_dev(rxe, "returned err = %d\n", err); 140 return err; 141 } 142 143 static int rxe_modify_port(struct ib_device *ibdev, u32 port_num, 144 int mask, struct ib_port_modify *attr) 145 { 146 struct rxe_dev *rxe = to_rdev(ibdev); 147 struct rxe_port *port; 148 int err; 149 150 if (port_num != 1) { 151 err = -EINVAL; 152 rxe_dbg_dev(rxe, "bad port_num = %d\n", port_num); 153 goto err_out; 154 } 155 156 //TODO is shutdown useful 157 if (mask & ~(IB_PORT_RESET_QKEY_CNTR)) { 158 err = -EOPNOTSUPP; 159 rxe_dbg_dev(rxe, "unsupported mask = 0x%x\n", mask); 160 goto err_out; 161 } 162 163 port = &rxe->port; 164 port->attr.port_cap_flags |= attr->set_port_cap_mask; 165 port->attr.port_cap_flags &= ~attr->clr_port_cap_mask; 166 167 if (mask & IB_PORT_RESET_QKEY_CNTR) 168 port->attr.qkey_viol_cntr = 0; 169 170 return 0; 171 172 err_out: 173 rxe_err_dev(rxe, "returned err = %d\n", err); 174 return err; 175 } 176 177 static enum rdma_link_layer rxe_get_link_layer(struct ib_device *ibdev, 178 u32 port_num) 179 { 180 struct rxe_dev *rxe = to_rdev(ibdev); 181 int err; 182 183 if (port_num != 1) { 184 err = -EINVAL; 185 rxe_dbg_dev(rxe, "bad port_num = %d\n", port_num); 186 goto err_out; 187 } 188 189 return IB_LINK_LAYER_ETHERNET; 190 191 err_out: 192 rxe_err_dev(rxe, "returned err = %d\n", err); 193 return err; 194 } 195 196 static int rxe_port_immutable(struct ib_device *ibdev, u32 port_num, 197 struct ib_port_immutable *immutable) 198 { 199 struct rxe_dev *rxe = to_rdev(ibdev); 200 struct ib_port_attr attr = {}; 201 int err; 202 203 if (port_num != 1) { 204 err = -EINVAL; 205 rxe_dbg_dev(rxe, "bad port_num = %d\n", port_num); 206 goto err_out; 207 } 208 209 err = ib_query_port(ibdev, port_num, &attr); 210 if (err) 211 goto err_out; 212 213 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; 214 immutable->pkey_tbl_len = attr.pkey_tbl_len; 215 immutable->gid_tbl_len = attr.gid_tbl_len; 216 immutable->max_mad_size = IB_MGMT_MAD_SIZE; 217 218 return 0; 219 220 err_out: 221 rxe_err_dev(rxe, "returned err = %d\n", err); 222 return err; 223 } 224 225 /* uc */ 226 static int rxe_alloc_ucontext(struct ib_ucontext *ibuc, struct ib_udata *udata) 227 { 228 struct rxe_dev *rxe = to_rdev(ibuc->device); 229 struct rxe_ucontext *uc = to_ruc(ibuc); 230 int err; 231 232 err = rxe_add_to_pool(&rxe->uc_pool, uc); 233 if (err) 234 rxe_err_dev(rxe, "unable to create uc\n"); 235 236 return err; 237 } 238 239 static void rxe_dealloc_ucontext(struct ib_ucontext *ibuc) 240 { 241 struct rxe_ucontext *uc = to_ruc(ibuc); 242 int err; 243 244 err = rxe_cleanup(uc); 245 if (err) 246 rxe_err_uc(uc, "cleanup failed, err = %d\n", err); 247 } 248 249 /* pd */ 250 static int rxe_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 251 { 252 struct rxe_dev *rxe = to_rdev(ibpd->device); 253 struct rxe_pd *pd = to_rpd(ibpd); 254 int err; 255 256 err = rxe_add_to_pool(&rxe->pd_pool, pd); 257 if (err) { 258 rxe_dbg_dev(rxe, "unable to alloc pd\n"); 259 goto err_out; 260 } 261 262 return 0; 263 264 err_out: 265 rxe_err_dev(rxe, "returned err = %d\n", err); 266 return err; 267 } 268 269 static int rxe_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) 270 { 271 struct rxe_pd *pd = to_rpd(ibpd); 272 int err; 273 274 err = rxe_cleanup(pd); 275 if (err) 276 rxe_err_pd(pd, "cleanup failed, err = %d\n", err); 277 278 return 0; 279 } 280 281 /* ah */ 282 static int rxe_create_ah(struct ib_ah *ibah, 283 struct rdma_ah_init_attr *init_attr, 284 struct ib_udata *udata) 285 { 286 struct rxe_dev *rxe = to_rdev(ibah->device); 287 struct rxe_ah *ah = to_rah(ibah); 288 struct rxe_create_ah_resp __user *uresp = NULL; 289 int err, cleanup_err; 290 291 if (udata) { 292 /* test if new user provider */ 293 if (udata->outlen >= sizeof(*uresp)) 294 uresp = udata->outbuf; 295 ah->is_user = true; 296 } else { 297 ah->is_user = false; 298 } 299 300 err = rxe_add_to_pool_ah(&rxe->ah_pool, ah, 301 init_attr->flags & RDMA_CREATE_AH_SLEEPABLE); 302 if (err) { 303 rxe_dbg_dev(rxe, "unable to create ah\n"); 304 goto err_out; 305 } 306 307 /* create index > 0 */ 308 ah->ah_num = ah->elem.index; 309 310 err = rxe_ah_chk_attr(ah, init_attr->ah_attr); 311 if (err) { 312 rxe_dbg_ah(ah, "bad attr\n"); 313 goto err_cleanup; 314 } 315 316 if (uresp) { 317 /* only if new user provider */ 318 err = copy_to_user(&uresp->ah_num, &ah->ah_num, 319 sizeof(uresp->ah_num)); 320 if (err) { 321 err = -EFAULT; 322 rxe_dbg_ah(ah, "unable to copy to user\n"); 323 goto err_cleanup; 324 } 325 } else if (ah->is_user) { 326 /* only if old user provider */ 327 ah->ah_num = 0; 328 } 329 330 rxe_init_av(init_attr->ah_attr, &ah->av); 331 rxe_finalize(ah); 332 333 return 0; 334 335 err_cleanup: 336 cleanup_err = rxe_cleanup(ah); 337 if (cleanup_err) 338 rxe_err_ah(ah, "cleanup failed, err = %d\n", cleanup_err); 339 err_out: 340 rxe_err_ah(ah, "returned err = %d\n", err); 341 return err; 342 } 343 344 static int rxe_modify_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr) 345 { 346 struct rxe_ah *ah = to_rah(ibah); 347 int err; 348 349 err = rxe_ah_chk_attr(ah, attr); 350 if (err) { 351 rxe_dbg_ah(ah, "bad attr\n"); 352 goto err_out; 353 } 354 355 rxe_init_av(attr, &ah->av); 356 357 return 0; 358 359 err_out: 360 rxe_err_ah(ah, "returned err = %d\n", err); 361 return err; 362 } 363 364 static int rxe_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr) 365 { 366 struct rxe_ah *ah = to_rah(ibah); 367 368 memset(attr, 0, sizeof(*attr)); 369 attr->type = ibah->type; 370 rxe_av_to_attr(&ah->av, attr); 371 372 return 0; 373 } 374 375 static int rxe_destroy_ah(struct ib_ah *ibah, u32 flags) 376 { 377 struct rxe_ah *ah = to_rah(ibah); 378 int err; 379 380 err = rxe_cleanup_ah(ah, flags & RDMA_DESTROY_AH_SLEEPABLE); 381 if (err) 382 rxe_err_ah(ah, "cleanup failed, err = %d\n", err); 383 384 return 0; 385 } 386 387 /* srq */ 388 static int rxe_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init, 389 struct ib_udata *udata) 390 { 391 struct rxe_dev *rxe = to_rdev(ibsrq->device); 392 struct rxe_pd *pd = to_rpd(ibsrq->pd); 393 struct rxe_srq *srq = to_rsrq(ibsrq); 394 struct rxe_create_srq_resp __user *uresp = NULL; 395 int err, cleanup_err; 396 397 if (udata) { 398 if (udata->outlen < sizeof(*uresp)) { 399 err = -EINVAL; 400 rxe_err_dev(rxe, "malformed udata\n"); 401 goto err_out; 402 } 403 uresp = udata->outbuf; 404 } 405 406 if (init->srq_type != IB_SRQT_BASIC) { 407 err = -EOPNOTSUPP; 408 rxe_dbg_dev(rxe, "srq type = %d, not supported\n", 409 init->srq_type); 410 goto err_out; 411 } 412 413 err = rxe_srq_chk_init(rxe, init); 414 if (err) { 415 rxe_dbg_dev(rxe, "invalid init attributes\n"); 416 goto err_out; 417 } 418 419 err = rxe_add_to_pool(&rxe->srq_pool, srq); 420 if (err) { 421 rxe_dbg_dev(rxe, "unable to create srq, err = %d\n", err); 422 goto err_out; 423 } 424 425 rxe_get(pd); 426 srq->pd = pd; 427 428 err = rxe_srq_from_init(rxe, srq, init, udata, uresp); 429 if (err) { 430 rxe_dbg_srq(srq, "create srq failed, err = %d\n", err); 431 goto err_cleanup; 432 } 433 434 return 0; 435 436 err_cleanup: 437 cleanup_err = rxe_cleanup(srq); 438 if (cleanup_err) 439 rxe_err_srq(srq, "cleanup failed, err = %d\n", cleanup_err); 440 err_out: 441 rxe_err_dev(rxe, "returned err = %d\n", err); 442 return err; 443 } 444 445 static int rxe_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, 446 enum ib_srq_attr_mask mask, 447 struct ib_udata *udata) 448 { 449 struct rxe_srq *srq = to_rsrq(ibsrq); 450 struct rxe_dev *rxe = to_rdev(ibsrq->device); 451 struct rxe_modify_srq_cmd cmd = {}; 452 int err; 453 454 if (udata) { 455 if (udata->inlen < sizeof(cmd)) { 456 err = -EINVAL; 457 rxe_dbg_srq(srq, "malformed udata\n"); 458 goto err_out; 459 } 460 461 err = ib_copy_from_udata(&cmd, udata, sizeof(cmd)); 462 if (err) { 463 err = -EFAULT; 464 rxe_dbg_srq(srq, "unable to read udata\n"); 465 goto err_out; 466 } 467 } 468 469 err = rxe_srq_chk_attr(rxe, srq, attr, mask); 470 if (err) { 471 rxe_dbg_srq(srq, "bad init attributes\n"); 472 goto err_out; 473 } 474 475 err = rxe_srq_from_attr(rxe, srq, attr, mask, &cmd, udata); 476 if (err) { 477 rxe_dbg_srq(srq, "bad attr\n"); 478 goto err_out; 479 } 480 481 return 0; 482 483 err_out: 484 rxe_err_srq(srq, "returned err = %d\n", err); 485 return err; 486 } 487 488 static int rxe_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr) 489 { 490 struct rxe_srq *srq = to_rsrq(ibsrq); 491 int err; 492 493 if (srq->error) { 494 err = -EINVAL; 495 rxe_dbg_srq(srq, "srq in error state\n"); 496 goto err_out; 497 } 498 499 attr->max_wr = srq->rq.queue->buf->index_mask; 500 attr->max_sge = srq->rq.max_sge; 501 attr->srq_limit = srq->limit; 502 return 0; 503 504 err_out: 505 rxe_err_srq(srq, "returned err = %d\n", err); 506 return err; 507 } 508 509 static int rxe_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr, 510 const struct ib_recv_wr **bad_wr) 511 { 512 int err = 0; 513 struct rxe_srq *srq = to_rsrq(ibsrq); 514 unsigned long flags; 515 516 spin_lock_irqsave(&srq->rq.producer_lock, flags); 517 518 while (wr) { 519 err = post_one_recv(&srq->rq, wr); 520 if (unlikely(err)) 521 break; 522 wr = wr->next; 523 } 524 525 spin_unlock_irqrestore(&srq->rq.producer_lock, flags); 526 527 if (err) { 528 *bad_wr = wr; 529 rxe_err_srq(srq, "returned err = %d\n", err); 530 } 531 532 return err; 533 } 534 535 static int rxe_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) 536 { 537 struct rxe_srq *srq = to_rsrq(ibsrq); 538 int err; 539 540 err = rxe_cleanup(srq); 541 if (err) 542 rxe_err_srq(srq, "cleanup failed, err = %d\n", err); 543 544 return 0; 545 } 546 547 /* qp */ 548 static int rxe_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *init, 549 struct ib_udata *udata) 550 { 551 struct rxe_dev *rxe = to_rdev(ibqp->device); 552 struct rxe_pd *pd = to_rpd(ibqp->pd); 553 struct rxe_qp *qp = to_rqp(ibqp); 554 struct rxe_create_qp_resp __user *uresp = NULL; 555 int err, cleanup_err; 556 557 if (udata) { 558 if (udata->inlen) { 559 err = -EINVAL; 560 rxe_dbg_dev(rxe, "malformed udata, err = %d\n", err); 561 goto err_out; 562 } 563 564 if (udata->outlen < sizeof(*uresp)) { 565 err = -EINVAL; 566 rxe_dbg_dev(rxe, "malformed udata, err = %d\n", err); 567 goto err_out; 568 } 569 570 qp->is_user = true; 571 uresp = udata->outbuf; 572 } else { 573 qp->is_user = false; 574 } 575 576 if (init->create_flags) { 577 err = -EOPNOTSUPP; 578 rxe_dbg_dev(rxe, "unsupported create_flags, err = %d\n", err); 579 goto err_out; 580 } 581 582 err = rxe_qp_chk_init(rxe, init); 583 if (err) { 584 rxe_dbg_dev(rxe, "bad init attr, err = %d\n", err); 585 goto err_out; 586 } 587 588 err = rxe_add_to_pool(&rxe->qp_pool, qp); 589 if (err) { 590 rxe_dbg_dev(rxe, "unable to create qp, err = %d\n", err); 591 goto err_out; 592 } 593 594 err = rxe_qp_from_init(rxe, qp, pd, init, uresp, ibqp->pd, udata); 595 if (err) { 596 rxe_dbg_qp(qp, "create qp failed, err = %d\n", err); 597 goto err_cleanup; 598 } 599 600 rxe_finalize(qp); 601 return 0; 602 603 err_cleanup: 604 cleanup_err = rxe_cleanup(qp); 605 if (cleanup_err) 606 rxe_err_qp(qp, "cleanup failed, err = %d\n", cleanup_err); 607 err_out: 608 rxe_err_dev(rxe, "returned err = %d\n", err); 609 return err; 610 } 611 612 static int rxe_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 613 int mask, struct ib_udata *udata) 614 { 615 struct rxe_dev *rxe = to_rdev(ibqp->device); 616 struct rxe_qp *qp = to_rqp(ibqp); 617 int err; 618 619 if (mask & ~IB_QP_ATTR_STANDARD_BITS) { 620 err = -EOPNOTSUPP; 621 rxe_dbg_qp(qp, "unsupported mask = 0x%x, err = %d\n", 622 mask, err); 623 goto err_out; 624 } 625 626 err = rxe_qp_chk_attr(rxe, qp, attr, mask); 627 if (err) { 628 rxe_dbg_qp(qp, "bad mask/attr, err = %d\n", err); 629 goto err_out; 630 } 631 632 err = rxe_qp_from_attr(qp, attr, mask, udata); 633 if (err) { 634 rxe_dbg_qp(qp, "modify qp failed, err = %d\n", err); 635 goto err_out; 636 } 637 638 if ((mask & IB_QP_AV) && (attr->ah_attr.ah_flags & IB_AH_GRH)) 639 qp->src_port = rdma_get_udp_sport(attr->ah_attr.grh.flow_label, 640 qp->ibqp.qp_num, 641 qp->attr.dest_qp_num); 642 643 return 0; 644 645 err_out: 646 rxe_err_qp(qp, "returned err = %d\n", err); 647 return err; 648 } 649 650 static int rxe_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, 651 int mask, struct ib_qp_init_attr *init) 652 { 653 struct rxe_qp *qp = to_rqp(ibqp); 654 655 rxe_qp_to_init(qp, init); 656 rxe_qp_to_attr(qp, attr, mask); 657 658 return 0; 659 } 660 661 static int rxe_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) 662 { 663 struct rxe_qp *qp = to_rqp(ibqp); 664 int err; 665 666 err = rxe_qp_chk_destroy(qp); 667 if (err) { 668 rxe_dbg_qp(qp, "unable to destroy qp, err = %d\n", err); 669 goto err_out; 670 } 671 672 err = rxe_cleanup(qp); 673 if (err) 674 rxe_err_qp(qp, "cleanup failed, err = %d\n", err); 675 676 return 0; 677 678 err_out: 679 rxe_err_qp(qp, "returned err = %d\n", err); 680 return err; 681 } 682 683 /* send wr */ 684 685 /* sanity check incoming send work request */ 686 static int validate_send_wr(struct rxe_qp *qp, const struct ib_send_wr *ibwr, 687 unsigned int *maskp, unsigned int *lengthp) 688 { 689 int num_sge = ibwr->num_sge; 690 struct rxe_sq *sq = &qp->sq; 691 unsigned int mask = 0; 692 unsigned long length = 0; 693 int err = -EINVAL; 694 int i; 695 696 do { 697 mask = wr_opcode_mask(ibwr->opcode, qp); 698 if (!mask) { 699 rxe_err_qp(qp, "bad wr opcode for qp type\n"); 700 break; 701 } 702 703 if (num_sge > sq->max_sge) { 704 rxe_err_qp(qp, "num_sge > max_sge\n"); 705 break; 706 } 707 708 length = 0; 709 for (i = 0; i < ibwr->num_sge; i++) 710 length += ibwr->sg_list[i].length; 711 712 if (length > RXE_PORT_MAX_MSG_SZ) { 713 rxe_err_qp(qp, "message length too long\n"); 714 break; 715 } 716 717 if (mask & WR_ATOMIC_MASK) { 718 if (length != 8) { 719 rxe_err_qp(qp, "atomic length != 8\n"); 720 break; 721 } 722 if (atomic_wr(ibwr)->remote_addr & 0x7) { 723 rxe_err_qp(qp, "misaligned atomic address\n"); 724 break; 725 } 726 } 727 if (ibwr->send_flags & IB_SEND_INLINE) { 728 if (!(mask & WR_INLINE_MASK)) { 729 rxe_err_qp(qp, "opcode doesn't support inline data\n"); 730 break; 731 } 732 if (length > sq->max_inline) { 733 rxe_err_qp(qp, "inline length too big\n"); 734 break; 735 } 736 } 737 738 err = 0; 739 } while (0); 740 741 *maskp = mask; 742 *lengthp = (int)length; 743 744 return err; 745 } 746 747 static int init_send_wr(struct rxe_qp *qp, struct rxe_send_wr *wr, 748 const struct ib_send_wr *ibwr) 749 { 750 wr->wr_id = ibwr->wr_id; 751 wr->opcode = ibwr->opcode; 752 wr->send_flags = ibwr->send_flags; 753 754 if (qp_type(qp) == IB_QPT_UD || 755 qp_type(qp) == IB_QPT_GSI) { 756 struct ib_ah *ibah = ud_wr(ibwr)->ah; 757 758 wr->wr.ud.remote_qpn = ud_wr(ibwr)->remote_qpn; 759 wr->wr.ud.remote_qkey = ud_wr(ibwr)->remote_qkey; 760 wr->wr.ud.ah_num = to_rah(ibah)->ah_num; 761 if (qp_type(qp) == IB_QPT_GSI) 762 wr->wr.ud.pkey_index = ud_wr(ibwr)->pkey_index; 763 764 switch (wr->opcode) { 765 case IB_WR_SEND_WITH_IMM: 766 wr->ex.imm_data = ibwr->ex.imm_data; 767 break; 768 case IB_WR_SEND: 769 break; 770 default: 771 rxe_err_qp(qp, "bad wr opcode %d for UD/GSI QP\n", 772 wr->opcode); 773 return -EINVAL; 774 } 775 } else { 776 switch (wr->opcode) { 777 case IB_WR_RDMA_WRITE_WITH_IMM: 778 wr->ex.imm_data = ibwr->ex.imm_data; 779 fallthrough; 780 case IB_WR_RDMA_READ: 781 case IB_WR_RDMA_WRITE: 782 wr->wr.rdma.remote_addr = rdma_wr(ibwr)->remote_addr; 783 wr->wr.rdma.rkey = rdma_wr(ibwr)->rkey; 784 break; 785 case IB_WR_SEND_WITH_IMM: 786 wr->ex.imm_data = ibwr->ex.imm_data; 787 break; 788 case IB_WR_SEND_WITH_INV: 789 wr->ex.invalidate_rkey = ibwr->ex.invalidate_rkey; 790 break; 791 case IB_WR_RDMA_READ_WITH_INV: 792 wr->ex.invalidate_rkey = ibwr->ex.invalidate_rkey; 793 wr->wr.rdma.remote_addr = rdma_wr(ibwr)->remote_addr; 794 wr->wr.rdma.rkey = rdma_wr(ibwr)->rkey; 795 break; 796 case IB_WR_ATOMIC_CMP_AND_SWP: 797 case IB_WR_ATOMIC_FETCH_AND_ADD: 798 wr->wr.atomic.remote_addr = 799 atomic_wr(ibwr)->remote_addr; 800 wr->wr.atomic.compare_add = 801 atomic_wr(ibwr)->compare_add; 802 wr->wr.atomic.swap = atomic_wr(ibwr)->swap; 803 wr->wr.atomic.rkey = atomic_wr(ibwr)->rkey; 804 break; 805 case IB_WR_LOCAL_INV: 806 wr->ex.invalidate_rkey = ibwr->ex.invalidate_rkey; 807 break; 808 case IB_WR_REG_MR: 809 wr->wr.reg.mr = reg_wr(ibwr)->mr; 810 wr->wr.reg.key = reg_wr(ibwr)->key; 811 wr->wr.reg.access = reg_wr(ibwr)->access; 812 break; 813 case IB_WR_SEND: 814 case IB_WR_BIND_MW: 815 case IB_WR_FLUSH: 816 case IB_WR_ATOMIC_WRITE: 817 break; 818 default: 819 rxe_err_qp(qp, "unsupported wr opcode %d\n", 820 wr->opcode); 821 return -EINVAL; 822 } 823 } 824 825 return 0; 826 } 827 828 static void copy_inline_data_to_wqe(struct rxe_send_wqe *wqe, 829 const struct ib_send_wr *ibwr) 830 { 831 struct ib_sge *sge = ibwr->sg_list; 832 u8 *p = wqe->dma.inline_data; 833 int i; 834 835 for (i = 0; i < ibwr->num_sge; i++, sge++) { 836 memcpy(p, ib_virt_dma_to_ptr(sge->addr), sge->length); 837 p += sge->length; 838 } 839 } 840 841 static int init_send_wqe(struct rxe_qp *qp, const struct ib_send_wr *ibwr, 842 unsigned int mask, unsigned int length, 843 struct rxe_send_wqe *wqe) 844 { 845 int num_sge = ibwr->num_sge; 846 int err; 847 848 err = init_send_wr(qp, &wqe->wr, ibwr); 849 if (err) 850 return err; 851 852 /* local operation */ 853 if (unlikely(mask & WR_LOCAL_OP_MASK)) { 854 wqe->mask = mask; 855 wqe->state = wqe_state_posted; 856 return 0; 857 } 858 859 if (unlikely(ibwr->send_flags & IB_SEND_INLINE)) 860 copy_inline_data_to_wqe(wqe, ibwr); 861 else 862 memcpy(wqe->dma.sge, ibwr->sg_list, 863 num_sge * sizeof(struct ib_sge)); 864 865 wqe->iova = mask & WR_ATOMIC_MASK ? atomic_wr(ibwr)->remote_addr : 866 mask & WR_READ_OR_WRITE_MASK ? rdma_wr(ibwr)->remote_addr : 0; 867 wqe->mask = mask; 868 wqe->dma.length = length; 869 wqe->dma.resid = length; 870 wqe->dma.num_sge = num_sge; 871 wqe->dma.cur_sge = 0; 872 wqe->dma.sge_offset = 0; 873 wqe->state = wqe_state_posted; 874 wqe->ssn = atomic_add_return(1, &qp->ssn); 875 876 return 0; 877 } 878 879 static int post_one_send(struct rxe_qp *qp, const struct ib_send_wr *ibwr) 880 { 881 int err; 882 struct rxe_sq *sq = &qp->sq; 883 struct rxe_send_wqe *send_wqe; 884 unsigned int mask; 885 unsigned int length; 886 int full; 887 888 err = validate_send_wr(qp, ibwr, &mask, &length); 889 if (err) 890 return err; 891 892 full = queue_full(sq->queue, QUEUE_TYPE_FROM_ULP); 893 if (unlikely(full)) { 894 rxe_err_qp(qp, "send queue full\n"); 895 return -ENOMEM; 896 } 897 898 send_wqe = queue_producer_addr(sq->queue, QUEUE_TYPE_FROM_ULP); 899 err = init_send_wqe(qp, ibwr, mask, length, send_wqe); 900 if (!err) 901 queue_advance_producer(sq->queue, QUEUE_TYPE_FROM_ULP); 902 903 return err; 904 } 905 906 static int rxe_post_send_kernel(struct rxe_qp *qp, 907 const struct ib_send_wr *ibwr, 908 const struct ib_send_wr **bad_wr) 909 { 910 int err = 0; 911 unsigned long flags; 912 int good = 0; 913 914 spin_lock_irqsave(&qp->sq.sq_lock, flags); 915 while (ibwr) { 916 err = post_one_send(qp, ibwr); 917 if (err) { 918 *bad_wr = ibwr; 919 break; 920 } else { 921 good++; 922 } 923 ibwr = ibwr->next; 924 } 925 spin_unlock_irqrestore(&qp->sq.sq_lock, flags); 926 927 /* kickoff processing of any posted wqes */ 928 if (good) 929 rxe_sched_task(&qp->send_task); 930 931 return err; 932 } 933 934 static int rxe_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, 935 const struct ib_send_wr **bad_wr) 936 { 937 struct rxe_qp *qp = to_rqp(ibqp); 938 int err; 939 unsigned long flags; 940 941 spin_lock_irqsave(&qp->state_lock, flags); 942 /* caller has already called destroy_qp */ 943 if (WARN_ON_ONCE(!qp->valid)) { 944 spin_unlock_irqrestore(&qp->state_lock, flags); 945 rxe_err_qp(qp, "qp has been destroyed\n"); 946 return -EINVAL; 947 } 948 949 if (unlikely(qp_state(qp) < IB_QPS_RTS)) { 950 spin_unlock_irqrestore(&qp->state_lock, flags); 951 *bad_wr = wr; 952 rxe_err_qp(qp, "qp not ready to send\n"); 953 return -EINVAL; 954 } 955 spin_unlock_irqrestore(&qp->state_lock, flags); 956 957 if (qp->is_user) { 958 /* Utilize process context to do protocol processing */ 959 rxe_sched_task(&qp->send_task); 960 } else { 961 err = rxe_post_send_kernel(qp, wr, bad_wr); 962 if (err) 963 return err; 964 } 965 966 return 0; 967 } 968 969 /* recv wr */ 970 static int post_one_recv(struct rxe_rq *rq, const struct ib_recv_wr *ibwr) 971 { 972 int i; 973 unsigned long length; 974 struct rxe_recv_wqe *recv_wqe; 975 int num_sge = ibwr->num_sge; 976 int full; 977 int err; 978 979 full = queue_full(rq->queue, QUEUE_TYPE_FROM_ULP); 980 if (unlikely(full)) { 981 err = -ENOMEM; 982 rxe_dbg("queue full\n"); 983 goto err_out; 984 } 985 986 if (unlikely(num_sge > rq->max_sge)) { 987 err = -EINVAL; 988 rxe_dbg("bad num_sge > max_sge\n"); 989 goto err_out; 990 } 991 992 length = 0; 993 for (i = 0; i < num_sge; i++) 994 length += ibwr->sg_list[i].length; 995 996 if (length > RXE_PORT_MAX_MSG_SZ) { 997 err = -EINVAL; 998 rxe_dbg("message length too long\n"); 999 goto err_out; 1000 } 1001 1002 recv_wqe = queue_producer_addr(rq->queue, QUEUE_TYPE_FROM_ULP); 1003 1004 recv_wqe->wr_id = ibwr->wr_id; 1005 recv_wqe->dma.length = length; 1006 recv_wqe->dma.resid = length; 1007 recv_wqe->dma.num_sge = num_sge; 1008 recv_wqe->dma.cur_sge = 0; 1009 recv_wqe->dma.sge_offset = 0; 1010 memcpy(recv_wqe->dma.sge, ibwr->sg_list, 1011 num_sge * sizeof(struct ib_sge)); 1012 1013 queue_advance_producer(rq->queue, QUEUE_TYPE_FROM_ULP); 1014 1015 return 0; 1016 1017 err_out: 1018 rxe_dbg("returned err = %d\n", err); 1019 return err; 1020 } 1021 1022 static int rxe_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr, 1023 const struct ib_recv_wr **bad_wr) 1024 { 1025 int err = 0; 1026 struct rxe_qp *qp = to_rqp(ibqp); 1027 struct rxe_rq *rq = &qp->rq; 1028 unsigned long flags; 1029 1030 spin_lock_irqsave(&qp->state_lock, flags); 1031 /* caller has already called destroy_qp */ 1032 if (WARN_ON_ONCE(!qp->valid)) { 1033 spin_unlock_irqrestore(&qp->state_lock, flags); 1034 rxe_err_qp(qp, "qp has been destroyed\n"); 1035 return -EINVAL; 1036 } 1037 1038 /* see C10-97.2.1 */ 1039 if (unlikely((qp_state(qp) < IB_QPS_INIT))) { 1040 spin_unlock_irqrestore(&qp->state_lock, flags); 1041 *bad_wr = wr; 1042 rxe_dbg_qp(qp, "qp not ready to post recv\n"); 1043 return -EINVAL; 1044 } 1045 spin_unlock_irqrestore(&qp->state_lock, flags); 1046 1047 if (unlikely(qp->srq)) { 1048 *bad_wr = wr; 1049 rxe_dbg_qp(qp, "qp has srq, use post_srq_recv instead\n"); 1050 return -EINVAL; 1051 } 1052 1053 spin_lock_irqsave(&rq->producer_lock, flags); 1054 1055 while (wr) { 1056 err = post_one_recv(rq, wr); 1057 if (unlikely(err)) { 1058 *bad_wr = wr; 1059 break; 1060 } 1061 wr = wr->next; 1062 } 1063 1064 spin_unlock_irqrestore(&rq->producer_lock, flags); 1065 1066 spin_lock_irqsave(&qp->state_lock, flags); 1067 if (qp_state(qp) == IB_QPS_ERR) 1068 rxe_sched_task(&qp->recv_task); 1069 spin_unlock_irqrestore(&qp->state_lock, flags); 1070 1071 return err; 1072 } 1073 1074 /* cq */ 1075 static int rxe_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, 1076 struct uverbs_attr_bundle *attrs) 1077 { 1078 struct ib_udata *udata = &attrs->driver_udata; 1079 struct ib_device *dev = ibcq->device; 1080 struct rxe_dev *rxe = to_rdev(dev); 1081 struct rxe_cq *cq = to_rcq(ibcq); 1082 struct rxe_create_cq_resp __user *uresp = NULL; 1083 int err, cleanup_err; 1084 1085 if (udata) { 1086 if (udata->outlen < sizeof(*uresp)) { 1087 err = -EINVAL; 1088 rxe_dbg_dev(rxe, "malformed udata, err = %d\n", err); 1089 goto err_out; 1090 } 1091 uresp = udata->outbuf; 1092 } 1093 1094 if (attr->flags) { 1095 err = -EOPNOTSUPP; 1096 rxe_dbg_dev(rxe, "bad attr->flags, err = %d\n", err); 1097 goto err_out; 1098 } 1099 1100 err = rxe_cq_chk_attr(rxe, NULL, attr->cqe, attr->comp_vector); 1101 if (err) { 1102 rxe_dbg_dev(rxe, "bad init attributes, err = %d\n", err); 1103 goto err_out; 1104 } 1105 1106 err = rxe_add_to_pool(&rxe->cq_pool, cq); 1107 if (err) { 1108 rxe_dbg_dev(rxe, "unable to create cq, err = %d\n", err); 1109 goto err_out; 1110 } 1111 1112 err = rxe_cq_from_init(rxe, cq, attr->cqe, attr->comp_vector, udata, 1113 uresp); 1114 if (err) { 1115 rxe_dbg_cq(cq, "create cq failed, err = %d\n", err); 1116 goto err_cleanup; 1117 } 1118 1119 return 0; 1120 1121 err_cleanup: 1122 cleanup_err = rxe_cleanup(cq); 1123 if (cleanup_err) 1124 rxe_err_cq(cq, "cleanup failed, err = %d\n", cleanup_err); 1125 err_out: 1126 rxe_err_dev(rxe, "returned err = %d\n", err); 1127 return err; 1128 } 1129 1130 static int rxe_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata) 1131 { 1132 struct rxe_cq *cq = to_rcq(ibcq); 1133 struct rxe_dev *rxe = to_rdev(ibcq->device); 1134 struct rxe_resize_cq_resp __user *uresp = NULL; 1135 int err; 1136 1137 if (udata) { 1138 if (udata->outlen < sizeof(*uresp)) { 1139 err = -EINVAL; 1140 rxe_dbg_cq(cq, "malformed udata\n"); 1141 goto err_out; 1142 } 1143 uresp = udata->outbuf; 1144 } 1145 1146 err = rxe_cq_chk_attr(rxe, cq, cqe, 0); 1147 if (err) { 1148 rxe_dbg_cq(cq, "bad attr, err = %d\n", err); 1149 goto err_out; 1150 } 1151 1152 err = rxe_cq_resize_queue(cq, cqe, uresp, udata); 1153 if (err) { 1154 rxe_dbg_cq(cq, "resize cq failed, err = %d\n", err); 1155 goto err_out; 1156 } 1157 1158 return 0; 1159 1160 err_out: 1161 rxe_err_cq(cq, "returned err = %d\n", err); 1162 return err; 1163 } 1164 1165 static int rxe_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) 1166 { 1167 int i; 1168 struct rxe_cq *cq = to_rcq(ibcq); 1169 struct rxe_cqe *cqe; 1170 unsigned long flags; 1171 1172 spin_lock_irqsave(&cq->cq_lock, flags); 1173 for (i = 0; i < num_entries; i++) { 1174 cqe = queue_head(cq->queue, QUEUE_TYPE_TO_ULP); 1175 if (!cqe) 1176 break; /* queue empty */ 1177 1178 memcpy(wc++, &cqe->ibwc, sizeof(*wc)); 1179 queue_advance_consumer(cq->queue, QUEUE_TYPE_TO_ULP); 1180 } 1181 spin_unlock_irqrestore(&cq->cq_lock, flags); 1182 1183 return i; 1184 } 1185 1186 static int rxe_peek_cq(struct ib_cq *ibcq, int wc_cnt) 1187 { 1188 struct rxe_cq *cq = to_rcq(ibcq); 1189 int count; 1190 1191 count = queue_count(cq->queue, QUEUE_TYPE_TO_ULP); 1192 1193 return (count > wc_cnt) ? wc_cnt : count; 1194 } 1195 1196 static int rxe_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) 1197 { 1198 struct rxe_cq *cq = to_rcq(ibcq); 1199 int ret = 0; 1200 int empty; 1201 unsigned long irq_flags; 1202 1203 spin_lock_irqsave(&cq->cq_lock, irq_flags); 1204 cq->notify |= flags & IB_CQ_SOLICITED_MASK; 1205 empty = queue_empty(cq->queue, QUEUE_TYPE_TO_ULP); 1206 1207 if ((flags & IB_CQ_REPORT_MISSED_EVENTS) && !empty) 1208 ret = 1; 1209 1210 spin_unlock_irqrestore(&cq->cq_lock, irq_flags); 1211 1212 return ret; 1213 } 1214 1215 static int rxe_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) 1216 { 1217 struct rxe_cq *cq = to_rcq(ibcq); 1218 int err; 1219 1220 /* See IBA C11-17: The CI shall return an error if this Verb is 1221 * invoked while a Work Queue is still associated with the CQ. 1222 */ 1223 if (atomic_read(&cq->num_wq)) { 1224 err = -EINVAL; 1225 rxe_dbg_cq(cq, "still in use\n"); 1226 goto err_out; 1227 } 1228 1229 err = rxe_cleanup(cq); 1230 if (err) 1231 rxe_err_cq(cq, "cleanup failed, err = %d\n", err); 1232 1233 return 0; 1234 1235 err_out: 1236 rxe_err_cq(cq, "returned err = %d\n", err); 1237 return err; 1238 } 1239 1240 /* mr */ 1241 static struct ib_mr *rxe_get_dma_mr(struct ib_pd *ibpd, int access) 1242 { 1243 struct rxe_dev *rxe = to_rdev(ibpd->device); 1244 struct rxe_pd *pd = to_rpd(ibpd); 1245 struct rxe_mr *mr; 1246 int err; 1247 1248 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 1249 if (!mr) 1250 return ERR_PTR(-ENOMEM); 1251 1252 err = rxe_add_to_pool(&rxe->mr_pool, mr); 1253 if (err) { 1254 rxe_dbg_dev(rxe, "unable to create mr\n"); 1255 goto err_free; 1256 } 1257 1258 rxe_get(pd); 1259 mr->ibmr.pd = ibpd; 1260 mr->ibmr.device = ibpd->device; 1261 1262 rxe_mr_init_dma(access, mr); 1263 rxe_finalize(mr); 1264 return &mr->ibmr; 1265 1266 err_free: 1267 kfree(mr); 1268 rxe_err_pd(pd, "returned err = %d\n", err); 1269 return ERR_PTR(err); 1270 } 1271 1272 static struct ib_mr *rxe_reg_user_mr(struct ib_pd *ibpd, u64 start, 1273 u64 length, u64 iova, int access, 1274 struct ib_udata *udata) 1275 { 1276 struct rxe_dev *rxe = to_rdev(ibpd->device); 1277 struct rxe_pd *pd = to_rpd(ibpd); 1278 struct rxe_mr *mr; 1279 int err, cleanup_err; 1280 1281 if (access & ~RXE_ACCESS_SUPPORTED_MR) { 1282 rxe_err_pd(pd, "access = %#x not supported (%#x)\n", access, 1283 RXE_ACCESS_SUPPORTED_MR); 1284 return ERR_PTR(-EOPNOTSUPP); 1285 } 1286 1287 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 1288 if (!mr) 1289 return ERR_PTR(-ENOMEM); 1290 1291 err = rxe_add_to_pool(&rxe->mr_pool, mr); 1292 if (err) { 1293 rxe_dbg_pd(pd, "unable to create mr\n"); 1294 goto err_free; 1295 } 1296 1297 rxe_get(pd); 1298 mr->ibmr.pd = ibpd; 1299 mr->ibmr.device = ibpd->device; 1300 1301 if (access & IB_ACCESS_ON_DEMAND) 1302 err = rxe_odp_mr_init_user(rxe, start, length, iova, access, mr); 1303 else 1304 err = rxe_mr_init_user(rxe, start, length, access, mr); 1305 if (err) { 1306 rxe_dbg_mr(mr, "reg_user_mr failed, err = %d\n", err); 1307 goto err_cleanup; 1308 } 1309 1310 rxe_finalize(mr); 1311 return &mr->ibmr; 1312 1313 err_cleanup: 1314 cleanup_err = rxe_cleanup(mr); 1315 if (cleanup_err) 1316 rxe_err_mr(mr, "cleanup failed, err = %d\n", cleanup_err); 1317 err_free: 1318 kfree(mr); 1319 rxe_err_pd(pd, "returned err = %d\n", err); 1320 return ERR_PTR(err); 1321 } 1322 1323 static struct ib_mr *rxe_rereg_user_mr(struct ib_mr *ibmr, int flags, 1324 u64 start, u64 length, u64 iova, 1325 int access, struct ib_pd *ibpd, 1326 struct ib_udata *udata) 1327 { 1328 struct rxe_mr *mr = to_rmr(ibmr); 1329 struct rxe_pd *old_pd = to_rpd(ibmr->pd); 1330 struct rxe_pd *pd = to_rpd(ibpd); 1331 1332 /* for now only support the two easy cases: 1333 * rereg_pd and rereg_access 1334 */ 1335 if (flags & ~RXE_MR_REREG_SUPPORTED) { 1336 rxe_err_mr(mr, "flags = %#x not supported\n", flags); 1337 return ERR_PTR(-EOPNOTSUPP); 1338 } 1339 1340 if (flags & IB_MR_REREG_PD) { 1341 rxe_put(old_pd); 1342 rxe_get(pd); 1343 mr->ibmr.pd = ibpd; 1344 } 1345 1346 if (flags & IB_MR_REREG_ACCESS) { 1347 if (access & ~RXE_ACCESS_SUPPORTED_MR) { 1348 rxe_err_mr(mr, "access = %#x not supported\n", access); 1349 return ERR_PTR(-EOPNOTSUPP); 1350 } 1351 mr->access = access; 1352 } 1353 1354 return NULL; 1355 } 1356 1357 static struct ib_mr *rxe_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, 1358 u32 max_num_sg) 1359 { 1360 struct rxe_dev *rxe = to_rdev(ibpd->device); 1361 struct rxe_pd *pd = to_rpd(ibpd); 1362 struct rxe_mr *mr; 1363 int err, cleanup_err; 1364 1365 if (mr_type != IB_MR_TYPE_MEM_REG) { 1366 err = -EINVAL; 1367 rxe_dbg_pd(pd, "mr type %d not supported, err = %d\n", 1368 mr_type, err); 1369 goto err_out; 1370 } 1371 1372 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 1373 if (!mr) 1374 return ERR_PTR(-ENOMEM); 1375 1376 err = rxe_add_to_pool(&rxe->mr_pool, mr); 1377 if (err) 1378 goto err_free; 1379 1380 rxe_get(pd); 1381 mr->ibmr.pd = ibpd; 1382 mr->ibmr.device = ibpd->device; 1383 1384 err = rxe_mr_init_fast(max_num_sg, mr); 1385 if (err) { 1386 rxe_dbg_mr(mr, "alloc_mr failed, err = %d\n", err); 1387 goto err_cleanup; 1388 } 1389 1390 rxe_finalize(mr); 1391 return &mr->ibmr; 1392 1393 err_cleanup: 1394 cleanup_err = rxe_cleanup(mr); 1395 if (cleanup_err) 1396 rxe_err_mr(mr, "cleanup failed, err = %d\n", err); 1397 err_free: 1398 kfree(mr); 1399 err_out: 1400 rxe_err_pd(pd, "returned err = %d\n", err); 1401 return ERR_PTR(err); 1402 } 1403 1404 static int rxe_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) 1405 { 1406 struct rxe_mr *mr = to_rmr(ibmr); 1407 int err, cleanup_err; 1408 1409 /* See IBA 10.6.7.2.6 */ 1410 if (atomic_read(&mr->num_mw) > 0) { 1411 err = -EINVAL; 1412 rxe_dbg_mr(mr, "mr has mw's bound\n"); 1413 goto err_out; 1414 } 1415 1416 cleanup_err = rxe_cleanup(mr); 1417 if (cleanup_err) 1418 rxe_err_mr(mr, "cleanup failed, err = %d\n", cleanup_err); 1419 1420 kfree_rcu_mightsleep(mr); 1421 return 0; 1422 1423 err_out: 1424 rxe_err_mr(mr, "returned err = %d\n", err); 1425 return err; 1426 } 1427 1428 static ssize_t parent_show(struct device *device, 1429 struct device_attribute *attr, char *buf) 1430 { 1431 struct rxe_dev *rxe = 1432 rdma_device_to_drv_device(device, struct rxe_dev, ib_dev); 1433 1434 return sysfs_emit(buf, "%s\n", rxe_parent_name(rxe, 1)); 1435 } 1436 1437 static DEVICE_ATTR_RO(parent); 1438 1439 static struct attribute *rxe_dev_attributes[] = { 1440 &dev_attr_parent.attr, 1441 NULL 1442 }; 1443 1444 static const struct attribute_group rxe_attr_group = { 1445 .attrs = rxe_dev_attributes, 1446 }; 1447 1448 static int rxe_enable_driver(struct ib_device *ib_dev) 1449 { 1450 struct rxe_dev *rxe = container_of(ib_dev, struct rxe_dev, ib_dev); 1451 struct net_device *ndev; 1452 1453 ndev = rxe_ib_device_get_netdev(ib_dev); 1454 if (!ndev) 1455 return -ENODEV; 1456 1457 rxe_set_port_state(rxe); 1458 dev_info(&rxe->ib_dev.dev, "added %s\n", netdev_name(ndev)); 1459 1460 dev_put(ndev); 1461 return 0; 1462 } 1463 1464 static const struct ib_device_ops rxe_dev_ops = { 1465 .owner = THIS_MODULE, 1466 .driver_id = RDMA_DRIVER_RXE, 1467 .uverbs_abi_ver = RXE_UVERBS_ABI_VERSION, 1468 1469 .alloc_hw_port_stats = rxe_ib_alloc_hw_port_stats, 1470 .alloc_mr = rxe_alloc_mr, 1471 .alloc_mw = rxe_alloc_mw, 1472 .alloc_pd = rxe_alloc_pd, 1473 .alloc_ucontext = rxe_alloc_ucontext, 1474 .attach_mcast = rxe_attach_mcast, 1475 .create_ah = rxe_create_ah, 1476 .create_cq = rxe_create_cq, 1477 .create_qp = rxe_create_qp, 1478 .create_srq = rxe_create_srq, 1479 .create_user_ah = rxe_create_ah, 1480 .dealloc_driver = rxe_dealloc, 1481 .dealloc_mw = rxe_dealloc_mw, 1482 .dealloc_pd = rxe_dealloc_pd, 1483 .dealloc_ucontext = rxe_dealloc_ucontext, 1484 .dereg_mr = rxe_dereg_mr, 1485 .destroy_ah = rxe_destroy_ah, 1486 .destroy_cq = rxe_destroy_cq, 1487 .destroy_qp = rxe_destroy_qp, 1488 .destroy_srq = rxe_destroy_srq, 1489 .detach_mcast = rxe_detach_mcast, 1490 .device_group = &rxe_attr_group, 1491 .enable_driver = rxe_enable_driver, 1492 .get_dma_mr = rxe_get_dma_mr, 1493 .get_hw_stats = rxe_ib_get_hw_stats, 1494 .get_link_layer = rxe_get_link_layer, 1495 .get_port_immutable = rxe_port_immutable, 1496 .map_mr_sg = rxe_map_mr_sg, 1497 .mmap = rxe_mmap, 1498 .modify_ah = rxe_modify_ah, 1499 .modify_device = rxe_modify_device, 1500 .modify_port = rxe_modify_port, 1501 .modify_qp = rxe_modify_qp, 1502 .modify_srq = rxe_modify_srq, 1503 .peek_cq = rxe_peek_cq, 1504 .poll_cq = rxe_poll_cq, 1505 .post_recv = rxe_post_recv, 1506 .post_send = rxe_post_send, 1507 .post_srq_recv = rxe_post_srq_recv, 1508 .query_ah = rxe_query_ah, 1509 .query_device = rxe_query_device, 1510 .query_pkey = rxe_query_pkey, 1511 .query_gid = rxe_query_gid, 1512 .query_port = rxe_query_port, 1513 .query_qp = rxe_query_qp, 1514 .query_srq = rxe_query_srq, 1515 .reg_user_mr = rxe_reg_user_mr, 1516 .req_notify_cq = rxe_req_notify_cq, 1517 .rereg_user_mr = rxe_rereg_user_mr, 1518 .resize_cq = rxe_resize_cq, 1519 1520 INIT_RDMA_OBJ_SIZE(ib_ah, rxe_ah, ibah), 1521 INIT_RDMA_OBJ_SIZE(ib_cq, rxe_cq, ibcq), 1522 INIT_RDMA_OBJ_SIZE(ib_pd, rxe_pd, ibpd), 1523 INIT_RDMA_OBJ_SIZE(ib_qp, rxe_qp, ibqp), 1524 INIT_RDMA_OBJ_SIZE(ib_srq, rxe_srq, ibsrq), 1525 INIT_RDMA_OBJ_SIZE(ib_ucontext, rxe_ucontext, ibuc), 1526 INIT_RDMA_OBJ_SIZE(ib_mw, rxe_mw, ibmw), 1527 }; 1528 1529 int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name, 1530 struct net_device *ndev) 1531 { 1532 int err; 1533 struct ib_device *dev = &rxe->ib_dev; 1534 1535 strscpy(dev->node_desc, "rxe", sizeof(dev->node_desc)); 1536 1537 dev->node_type = RDMA_NODE_IB_CA; 1538 dev->phys_port_cnt = 1; 1539 dev->num_comp_vectors = num_possible_cpus(); 1540 dev->local_dma_lkey = 0; 1541 addrconf_addr_eui48((unsigned char *)&dev->node_guid, 1542 rxe->raw_gid); 1543 1544 dev->uverbs_cmd_mask |= BIT_ULL(IB_USER_VERBS_CMD_POST_SEND) | 1545 BIT_ULL(IB_USER_VERBS_CMD_REQ_NOTIFY_CQ); 1546 1547 ib_set_device_ops(dev, &rxe_dev_ops); 1548 err = ib_device_set_netdev(&rxe->ib_dev, ndev, 1); 1549 if (err) 1550 return err; 1551 1552 err = ib_register_device(dev, ibdev_name, NULL); 1553 if (err) 1554 rxe_dbg_dev(rxe, "failed with error %d\n", err); 1555 1556 /* 1557 * Note that rxe may be invalid at this point if another thread 1558 * unregistered it. 1559 */ 1560 return err; 1561 } 1562