1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3 * Copyright (c) 2004-2007 Intel Corporation. All rights reserved.
4 * Copyright (c) 2004 Topspin Corporation. All rights reserved.
5 * Copyright (c) 2004, 2005 Voltaire Corporation. All rights reserved.
6 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
7 * Copyright (c) 2019, Mellanox Technologies inc. All rights reserved.
8 */
9
10 #include <linux/completion.h>
11 #include <linux/dma-mapping.h>
12 #include <linux/device.h>
13 #include <linux/module.h>
14 #include <linux/err.h>
15 #include <linux/idr.h>
16 #include <linux/interrupt.h>
17 #include <linux/random.h>
18 #include <linux/rbtree.h>
19 #include <linux/spinlock.h>
20 #include <linux/slab.h>
21 #include <linux/sysfs.h>
22 #include <linux/workqueue.h>
23 #include <linux/kdev_t.h>
24 #include <linux/etherdevice.h>
25
26 #include <rdma/ib_cache.h>
27 #include <rdma/ib_cm.h>
28 #include <rdma/ib_sysfs.h>
29 #include "cm_msgs.h"
30 #include "core_priv.h"
31 #include "cm_trace.h"
32
33 MODULE_AUTHOR("Sean Hefty");
34 MODULE_DESCRIPTION("InfiniBand CM");
35 MODULE_LICENSE("Dual BSD/GPL");
36
37 #define CM_DESTROY_ID_WAIT_TIMEOUT 10000 /* msecs */
38 #define CM_DIRECT_RETRY_CTX ((void *) 1UL)
39 #define CM_MRA_SETTING 24 /* 4.096us * 2^24 = ~68.7 seconds */
40
41 static const char * const ibcm_rej_reason_strs[] = {
42 [IB_CM_REJ_NO_QP] = "no QP",
43 [IB_CM_REJ_NO_EEC] = "no EEC",
44 [IB_CM_REJ_NO_RESOURCES] = "no resources",
45 [IB_CM_REJ_TIMEOUT] = "timeout",
46 [IB_CM_REJ_UNSUPPORTED] = "unsupported",
47 [IB_CM_REJ_INVALID_COMM_ID] = "invalid comm ID",
48 [IB_CM_REJ_INVALID_COMM_INSTANCE] = "invalid comm instance",
49 [IB_CM_REJ_INVALID_SERVICE_ID] = "invalid service ID",
50 [IB_CM_REJ_INVALID_TRANSPORT_TYPE] = "invalid transport type",
51 [IB_CM_REJ_STALE_CONN] = "stale conn",
52 [IB_CM_REJ_RDC_NOT_EXIST] = "RDC not exist",
53 [IB_CM_REJ_INVALID_GID] = "invalid GID",
54 [IB_CM_REJ_INVALID_LID] = "invalid LID",
55 [IB_CM_REJ_INVALID_SL] = "invalid SL",
56 [IB_CM_REJ_INVALID_TRAFFIC_CLASS] = "invalid traffic class",
57 [IB_CM_REJ_INVALID_HOP_LIMIT] = "invalid hop limit",
58 [IB_CM_REJ_INVALID_PACKET_RATE] = "invalid packet rate",
59 [IB_CM_REJ_INVALID_ALT_GID] = "invalid alt GID",
60 [IB_CM_REJ_INVALID_ALT_LID] = "invalid alt LID",
61 [IB_CM_REJ_INVALID_ALT_SL] = "invalid alt SL",
62 [IB_CM_REJ_INVALID_ALT_TRAFFIC_CLASS] = "invalid alt traffic class",
63 [IB_CM_REJ_INVALID_ALT_HOP_LIMIT] = "invalid alt hop limit",
64 [IB_CM_REJ_INVALID_ALT_PACKET_RATE] = "invalid alt packet rate",
65 [IB_CM_REJ_PORT_CM_REDIRECT] = "port CM redirect",
66 [IB_CM_REJ_PORT_REDIRECT] = "port redirect",
67 [IB_CM_REJ_INVALID_MTU] = "invalid MTU",
68 [IB_CM_REJ_INSUFFICIENT_RESP_RESOURCES] = "insufficient resp resources",
69 [IB_CM_REJ_CONSUMER_DEFINED] = "consumer defined",
70 [IB_CM_REJ_INVALID_RNR_RETRY] = "invalid RNR retry",
71 [IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID] = "duplicate local comm ID",
72 [IB_CM_REJ_INVALID_CLASS_VERSION] = "invalid class version",
73 [IB_CM_REJ_INVALID_FLOW_LABEL] = "invalid flow label",
74 [IB_CM_REJ_INVALID_ALT_FLOW_LABEL] = "invalid alt flow label",
75 [IB_CM_REJ_VENDOR_OPTION_NOT_SUPPORTED] =
76 "vendor option is not supported",
77 };
78
ibcm_reject_msg(int reason)79 const char *__attribute_const__ ibcm_reject_msg(int reason)
80 {
81 size_t index = reason;
82
83 if (index < ARRAY_SIZE(ibcm_rej_reason_strs) &&
84 ibcm_rej_reason_strs[index])
85 return ibcm_rej_reason_strs[index];
86 else
87 return "unrecognized reason";
88 }
89 EXPORT_SYMBOL(ibcm_reject_msg);
90
91 struct cm_id_private;
92 struct cm_work;
93 static int cm_add_one(struct ib_device *device);
94 static void cm_remove_one(struct ib_device *device, void *client_data);
95 static void cm_process_work(struct cm_id_private *cm_id_priv,
96 struct cm_work *work);
97 static int cm_send_sidr_rep_locked(struct cm_id_private *cm_id_priv,
98 struct ib_cm_sidr_rep_param *param);
99 static void cm_issue_dreq(struct cm_id_private *cm_id_priv);
100 static int cm_send_drep_locked(struct cm_id_private *cm_id_priv,
101 void *private_data, u8 private_data_len);
102 static int cm_send_rej_locked(struct cm_id_private *cm_id_priv,
103 enum ib_cm_rej_reason reason, void *ari,
104 u8 ari_length, const void *private_data,
105 u8 private_data_len);
106
107 static struct ib_client cm_client = {
108 .name = "cm",
109 .add = cm_add_one,
110 .remove = cm_remove_one
111 };
112
113 static struct ib_cm {
114 spinlock_t lock;
115 struct list_head device_list;
116 rwlock_t device_lock;
117 struct rb_root listen_service_table;
118 u64 listen_service_id;
119 /* struct rb_root peer_service_table; todo: fix peer to peer */
120 struct rb_root remote_qp_table;
121 struct rb_root remote_id_table;
122 struct rb_root remote_sidr_table;
123 struct xarray local_id_table;
124 u32 local_id_next;
125 __be32 random_id_operand;
126 struct list_head timewait_list;
127 struct workqueue_struct *wq;
128 } cm;
129
130 /* Counter indexes ordered by attribute ID */
131 enum {
132 CM_REQ_COUNTER,
133 CM_MRA_COUNTER,
134 CM_REJ_COUNTER,
135 CM_REP_COUNTER,
136 CM_RTU_COUNTER,
137 CM_DREQ_COUNTER,
138 CM_DREP_COUNTER,
139 CM_SIDR_REQ_COUNTER,
140 CM_SIDR_REP_COUNTER,
141 CM_LAP_COUNTER,
142 CM_APR_COUNTER,
143 CM_ATTR_COUNT,
144 CM_ATTR_ID_OFFSET = 0x0010,
145 };
146
147 enum {
148 CM_XMIT,
149 CM_XMIT_RETRIES,
150 CM_RECV,
151 CM_RECV_DUPLICATES,
152 CM_COUNTER_GROUPS
153 };
154
155 struct cm_counter_attribute {
156 struct ib_port_attribute attr;
157 unsigned short group;
158 unsigned short index;
159 };
160
161 struct cm_port {
162 struct cm_device *cm_dev;
163 struct ib_mad_agent *mad_agent;
164 struct ib_mad_agent *rep_agent;
165 u32 port_num;
166 atomic_long_t counters[CM_COUNTER_GROUPS][CM_ATTR_COUNT];
167 };
168
169 struct cm_device {
170 struct kref kref;
171 struct list_head list;
172 rwlock_t mad_agent_lock;
173 struct ib_device *ib_device;
174 u8 ack_delay;
175 int going_down;
176 struct cm_port *port[];
177 };
178
179 struct cm_av {
180 struct cm_port *port;
181 struct rdma_ah_attr ah_attr;
182 u16 dlid_datapath;
183 u16 pkey_index;
184 u8 timeout;
185 };
186
187 struct cm_work {
188 struct delayed_work work;
189 struct list_head list;
190 struct cm_port *port;
191 struct ib_mad_recv_wc *mad_recv_wc; /* Received MADs */
192 __be32 local_id; /* Established / timewait */
193 __be32 remote_id;
194 struct ib_cm_event cm_event;
195 struct sa_path_rec path[];
196 };
197
198 struct cm_timewait_info {
199 struct cm_work work;
200 struct list_head list;
201 struct rb_node remote_qp_node;
202 struct rb_node remote_id_node;
203 __be64 remote_ca_guid;
204 __be32 remote_qpn;
205 u8 inserted_remote_qp;
206 u8 inserted_remote_id;
207 };
208
209 struct cm_id_private {
210 struct ib_cm_id id;
211
212 struct rb_node service_node;
213 struct rb_node sidr_id_node;
214 u32 sidr_slid;
215 spinlock_t lock; /* Do not acquire inside cm.lock */
216 struct completion comp;
217 refcount_t refcount;
218 /* Number of clients sharing this ib_cm_id. Only valid for listeners.
219 * Protected by the cm.lock spinlock.
220 */
221 int listen_sharecount;
222 struct rcu_head rcu;
223
224 struct ib_mad_send_buf *msg;
225 struct cm_timewait_info *timewait_info;
226 /* todo: use alternate port on send failure */
227 struct cm_av av;
228 struct cm_av alt_av;
229
230 void *private_data;
231 __be64 tid;
232 __be32 local_qpn;
233 __be32 remote_qpn;
234 enum ib_qp_type qp_type;
235 __be32 sq_psn;
236 __be32 rq_psn;
237 int timeout_ms;
238 enum ib_mtu path_mtu;
239 __be16 pkey;
240 u8 private_data_len;
241 u8 max_cm_retries;
242 u8 responder_resources;
243 u8 initiator_depth;
244 u8 retry_count;
245 u8 rnr_retry_count;
246 u8 target_ack_delay;
247
248 struct list_head work_list;
249 atomic_t work_count;
250
251 struct rdma_ucm_ece ece;
252 };
253
cm_dev_release(struct kref * kref)254 static void cm_dev_release(struct kref *kref)
255 {
256 struct cm_device *cm_dev = container_of(kref, struct cm_device, kref);
257 u32 i;
258
259 rdma_for_each_port(cm_dev->ib_device, i)
260 kfree(cm_dev->port[i - 1]);
261
262 kfree(cm_dev);
263 }
264
cm_device_put(struct cm_device * cm_dev)265 static void cm_device_put(struct cm_device *cm_dev)
266 {
267 kref_put(&cm_dev->kref, cm_dev_release);
268 }
269
270 static void cm_work_handler(struct work_struct *work);
271
cm_deref_id(struct cm_id_private * cm_id_priv)272 static inline void cm_deref_id(struct cm_id_private *cm_id_priv)
273 {
274 if (refcount_dec_and_test(&cm_id_priv->refcount))
275 complete(&cm_id_priv->comp);
276 }
277
278 static struct ib_mad_send_buf *
cm_alloc_msg_agent(struct cm_id_private * cm_id_priv,bool rep_agent)279 cm_alloc_msg_agent(struct cm_id_private *cm_id_priv, bool rep_agent)
280 {
281 struct ib_mad_agent *mad_agent;
282 struct ib_mad_send_buf *m;
283 struct ib_ah *ah;
284
285 lockdep_assert_held(&cm_id_priv->lock);
286
287 if (!cm_id_priv->av.port)
288 return ERR_PTR(-EINVAL);
289
290 read_lock(&cm_id_priv->av.port->cm_dev->mad_agent_lock);
291 mad_agent = rep_agent ? cm_id_priv->av.port->rep_agent :
292 cm_id_priv->av.port->mad_agent;
293 if (!mad_agent) {
294 m = ERR_PTR(-EINVAL);
295 goto out;
296 }
297
298 ah = rdma_create_ah(mad_agent->qp->pd, &cm_id_priv->av.ah_attr, 0);
299 if (IS_ERR(ah)) {
300 m = ERR_CAST(ah);
301 goto out;
302 }
303
304 m = ib_create_send_mad(mad_agent, cm_id_priv->id.remote_cm_qpn,
305 cm_id_priv->av.pkey_index,
306 0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
307 GFP_ATOMIC,
308 IB_MGMT_BASE_VERSION);
309 if (IS_ERR(m)) {
310 rdma_destroy_ah(ah, 0);
311 goto out;
312 }
313
314 m->ah = ah;
315
316 out:
317 read_unlock(&cm_id_priv->av.port->cm_dev->mad_agent_lock);
318 return m;
319 }
320
cm_alloc_msg(struct cm_id_private * cm_id_priv)321 static struct ib_mad_send_buf *cm_alloc_msg(struct cm_id_private *cm_id_priv)
322 {
323 return cm_alloc_msg_agent(cm_id_priv, false);
324 }
325
cm_free_msg(struct ib_mad_send_buf * msg)326 static void cm_free_msg(struct ib_mad_send_buf *msg)
327 {
328 if (msg->ah)
329 rdma_destroy_ah(msg->ah, 0);
330 ib_free_send_mad(msg);
331 }
332
333 static struct ib_mad_send_buf *
cm_alloc_priv_msg_rep(struct cm_id_private * cm_id_priv,enum ib_cm_state state,bool rep_agent)334 cm_alloc_priv_msg_rep(struct cm_id_private *cm_id_priv, enum ib_cm_state state,
335 bool rep_agent)
336 {
337 struct ib_mad_send_buf *msg;
338
339 lockdep_assert_held(&cm_id_priv->lock);
340
341 msg = cm_alloc_msg_agent(cm_id_priv, rep_agent);
342 if (IS_ERR(msg))
343 return msg;
344
345 cm_id_priv->msg = msg;
346 refcount_inc(&cm_id_priv->refcount);
347 msg->context[0] = cm_id_priv;
348 msg->context[1] = (void *) (unsigned long) state;
349
350 msg->retries = cm_id_priv->max_cm_retries;
351 msg->timeout_ms = cm_id_priv->timeout_ms;
352
353 return msg;
354 }
355
356 static struct ib_mad_send_buf *
cm_alloc_priv_msg(struct cm_id_private * cm_id_priv,enum ib_cm_state state)357 cm_alloc_priv_msg(struct cm_id_private *cm_id_priv, enum ib_cm_state state)
358 {
359 return cm_alloc_priv_msg_rep(cm_id_priv, state, false);
360 }
361
cm_free_priv_msg(struct ib_mad_send_buf * msg)362 static void cm_free_priv_msg(struct ib_mad_send_buf *msg)
363 {
364 struct cm_id_private *cm_id_priv = msg->context[0];
365
366 lockdep_assert_held(&cm_id_priv->lock);
367
368 if (!WARN_ON(cm_id_priv->msg != msg))
369 cm_id_priv->msg = NULL;
370
371 if (msg->ah)
372 rdma_destroy_ah(msg->ah, 0);
373 cm_deref_id(cm_id_priv);
374 ib_free_send_mad(msg);
375 }
376
377 static struct ib_mad_send_buf *
cm_alloc_response_msg_no_ah(struct cm_port * port,struct ib_mad_recv_wc * mad_recv_wc,bool direct_retry)378 cm_alloc_response_msg_no_ah(struct cm_port *port,
379 struct ib_mad_recv_wc *mad_recv_wc,
380 bool direct_retry)
381 {
382 struct ib_mad_send_buf *m;
383
384 m = ib_create_send_mad(port->mad_agent, 1, mad_recv_wc->wc->pkey_index,
385 0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
386 GFP_ATOMIC, IB_MGMT_BASE_VERSION);
387 if (!IS_ERR(m))
388 m->context[0] = direct_retry ? CM_DIRECT_RETRY_CTX : NULL;
389
390 return m;
391 }
392
cm_create_response_msg_ah(struct cm_port * port,struct ib_mad_recv_wc * mad_recv_wc,struct ib_mad_send_buf * msg)393 static int cm_create_response_msg_ah(struct cm_port *port,
394 struct ib_mad_recv_wc *mad_recv_wc,
395 struct ib_mad_send_buf *msg)
396 {
397 struct ib_ah *ah;
398
399 ah = ib_create_ah_from_wc(port->mad_agent->qp->pd, mad_recv_wc->wc,
400 mad_recv_wc->recv_buf.grh, port->port_num);
401 if (IS_ERR(ah))
402 return PTR_ERR(ah);
403
404 msg->ah = ah;
405 return 0;
406 }
407
cm_alloc_response_msg(struct cm_port * port,struct ib_mad_recv_wc * mad_recv_wc,bool direct_retry,struct ib_mad_send_buf ** msg)408 static int cm_alloc_response_msg(struct cm_port *port,
409 struct ib_mad_recv_wc *mad_recv_wc,
410 bool direct_retry,
411 struct ib_mad_send_buf **msg)
412 {
413 struct ib_mad_send_buf *m;
414 int ret;
415
416 m = cm_alloc_response_msg_no_ah(port, mad_recv_wc, direct_retry);
417 if (IS_ERR(m))
418 return PTR_ERR(m);
419
420 ret = cm_create_response_msg_ah(port, mad_recv_wc, m);
421 if (ret) {
422 ib_free_send_mad(m);
423 return ret;
424 }
425
426 *msg = m;
427 return 0;
428 }
429
cm_copy_private_data(const void * private_data,u8 private_data_len)430 static void *cm_copy_private_data(const void *private_data, u8 private_data_len)
431 {
432 void *data;
433
434 if (!private_data || !private_data_len)
435 return NULL;
436
437 data = kmemdup(private_data, private_data_len, GFP_KERNEL);
438 if (!data)
439 return ERR_PTR(-ENOMEM);
440
441 return data;
442 }
443
cm_set_private_data(struct cm_id_private * cm_id_priv,void * private_data,u8 private_data_len)444 static void cm_set_private_data(struct cm_id_private *cm_id_priv,
445 void *private_data, u8 private_data_len)
446 {
447 if (cm_id_priv->private_data && cm_id_priv->private_data_len)
448 kfree(cm_id_priv->private_data);
449
450 cm_id_priv->private_data = private_data;
451 cm_id_priv->private_data_len = private_data_len;
452 }
453
cm_set_av_port(struct cm_av * av,struct cm_port * port)454 static void cm_set_av_port(struct cm_av *av, struct cm_port *port)
455 {
456 struct cm_port *old_port = av->port;
457
458 if (old_port == port)
459 return;
460
461 av->port = port;
462 if (old_port)
463 cm_device_put(old_port->cm_dev);
464 if (port)
465 kref_get(&port->cm_dev->kref);
466 }
467
cm_init_av_for_lap(struct cm_port * port,struct ib_wc * wc,struct rdma_ah_attr * ah_attr,struct cm_av * av)468 static void cm_init_av_for_lap(struct cm_port *port, struct ib_wc *wc,
469 struct rdma_ah_attr *ah_attr, struct cm_av *av)
470 {
471 cm_set_av_port(av, port);
472 av->pkey_index = wc->pkey_index;
473 rdma_move_ah_attr(&av->ah_attr, ah_attr);
474 }
475
cm_init_av_for_response(struct cm_port * port,struct ib_wc * wc,struct ib_grh * grh,struct cm_av * av)476 static int cm_init_av_for_response(struct cm_port *port, struct ib_wc *wc,
477 struct ib_grh *grh, struct cm_av *av)
478 {
479 cm_set_av_port(av, port);
480 av->pkey_index = wc->pkey_index;
481 return ib_init_ah_attr_from_wc(port->cm_dev->ib_device,
482 port->port_num, wc,
483 grh, &av->ah_attr);
484 }
485
486 static struct cm_port *
get_cm_port_from_path(struct sa_path_rec * path,const struct ib_gid_attr * attr)487 get_cm_port_from_path(struct sa_path_rec *path, const struct ib_gid_attr *attr)
488 {
489 struct cm_device *cm_dev;
490 struct cm_port *port = NULL;
491 unsigned long flags;
492
493 if (attr) {
494 read_lock_irqsave(&cm.device_lock, flags);
495 list_for_each_entry(cm_dev, &cm.device_list, list) {
496 if (cm_dev->ib_device == attr->device) {
497 port = cm_dev->port[attr->port_num - 1];
498 break;
499 }
500 }
501 read_unlock_irqrestore(&cm.device_lock, flags);
502 } else {
503 /* SGID attribute can be NULL in following
504 * conditions.
505 * (a) Alternative path
506 * (b) IB link layer without GRH
507 * (c) LAP send messages
508 */
509 read_lock_irqsave(&cm.device_lock, flags);
510 list_for_each_entry(cm_dev, &cm.device_list, list) {
511 attr = rdma_find_gid(cm_dev->ib_device,
512 &path->sgid,
513 sa_conv_pathrec_to_gid_type(path),
514 NULL);
515 if (!IS_ERR(attr)) {
516 port = cm_dev->port[attr->port_num - 1];
517 break;
518 }
519 }
520 read_unlock_irqrestore(&cm.device_lock, flags);
521 if (port)
522 rdma_put_gid_attr(attr);
523 }
524 return port;
525 }
526
cm_init_av_by_path(struct sa_path_rec * path,const struct ib_gid_attr * sgid_attr,struct cm_av * av)527 static int cm_init_av_by_path(struct sa_path_rec *path,
528 const struct ib_gid_attr *sgid_attr,
529 struct cm_av *av)
530 {
531 struct rdma_ah_attr new_ah_attr;
532 struct cm_device *cm_dev;
533 struct cm_port *port;
534 int ret;
535
536 port = get_cm_port_from_path(path, sgid_attr);
537 if (!port)
538 return -EINVAL;
539 cm_dev = port->cm_dev;
540
541 ret = ib_find_cached_pkey(cm_dev->ib_device, port->port_num,
542 be16_to_cpu(path->pkey), &av->pkey_index);
543 if (ret)
544 return ret;
545
546 cm_set_av_port(av, port);
547
548 /*
549 * av->ah_attr might be initialized based on wc or during
550 * request processing time which might have reference to sgid_attr.
551 * So initialize a new ah_attr on stack.
552 * If initialization fails, old ah_attr is used for sending any
553 * responses. If initialization is successful, than new ah_attr
554 * is used by overwriting the old one. So that right ah_attr
555 * can be used to return an error response.
556 */
557 ret = ib_init_ah_attr_from_path(cm_dev->ib_device, port->port_num, path,
558 &new_ah_attr, sgid_attr);
559 if (ret)
560 return ret;
561
562 av->timeout = path->packet_life_time + 1;
563 rdma_move_ah_attr(&av->ah_attr, &new_ah_attr);
564 return 0;
565 }
566
567 /* Move av created by cm_init_av_by_path(), so av.dgid is not moved */
cm_move_av_from_path(struct cm_av * dest,struct cm_av * src)568 static void cm_move_av_from_path(struct cm_av *dest, struct cm_av *src)
569 {
570 cm_set_av_port(dest, src->port);
571 cm_set_av_port(src, NULL);
572 dest->pkey_index = src->pkey_index;
573 rdma_move_ah_attr(&dest->ah_attr, &src->ah_attr);
574 dest->timeout = src->timeout;
575 }
576
cm_destroy_av(struct cm_av * av)577 static void cm_destroy_av(struct cm_av *av)
578 {
579 rdma_destroy_ah_attr(&av->ah_attr);
580 cm_set_av_port(av, NULL);
581 }
582
cm_local_id(__be32 local_id)583 static u32 cm_local_id(__be32 local_id)
584 {
585 return (__force u32) (local_id ^ cm.random_id_operand);
586 }
587
cm_acquire_id(__be32 local_id,__be32 remote_id)588 static struct cm_id_private *cm_acquire_id(__be32 local_id, __be32 remote_id)
589 {
590 struct cm_id_private *cm_id_priv;
591
592 rcu_read_lock();
593 cm_id_priv = xa_load(&cm.local_id_table, cm_local_id(local_id));
594 if (!cm_id_priv || cm_id_priv->id.remote_id != remote_id ||
595 !refcount_inc_not_zero(&cm_id_priv->refcount))
596 cm_id_priv = NULL;
597 rcu_read_unlock();
598
599 return cm_id_priv;
600 }
601
602 /*
603 * Trivial helpers to strip endian annotation and compare; the
604 * endianness doesn't actually matter since we just need a stable
605 * order for the RB tree.
606 */
be32_lt(__be32 a,__be32 b)607 static int be32_lt(__be32 a, __be32 b)
608 {
609 return (__force u32) a < (__force u32) b;
610 }
611
be32_gt(__be32 a,__be32 b)612 static int be32_gt(__be32 a, __be32 b)
613 {
614 return (__force u32) a > (__force u32) b;
615 }
616
be64_lt(__be64 a,__be64 b)617 static int be64_lt(__be64 a, __be64 b)
618 {
619 return (__force u64) a < (__force u64) b;
620 }
621
be64_gt(__be64 a,__be64 b)622 static int be64_gt(__be64 a, __be64 b)
623 {
624 return (__force u64) a > (__force u64) b;
625 }
626
627 /*
628 * Inserts a new cm_id_priv into the listen_service_table. Returns cm_id_priv
629 * if the new ID was inserted, NULL if it could not be inserted due to a
630 * collision, or the existing cm_id_priv ready for shared usage.
631 */
cm_insert_listen(struct cm_id_private * cm_id_priv,ib_cm_handler shared_handler)632 static struct cm_id_private *cm_insert_listen(struct cm_id_private *cm_id_priv,
633 ib_cm_handler shared_handler)
634 {
635 struct rb_node **link = &cm.listen_service_table.rb_node;
636 struct rb_node *parent = NULL;
637 struct cm_id_private *cur_cm_id_priv;
638 __be64 service_id = cm_id_priv->id.service_id;
639 unsigned long flags;
640
641 spin_lock_irqsave(&cm.lock, flags);
642 while (*link) {
643 parent = *link;
644 cur_cm_id_priv = rb_entry(parent, struct cm_id_private,
645 service_node);
646
647 if (cm_id_priv->id.device < cur_cm_id_priv->id.device)
648 link = &(*link)->rb_left;
649 else if (cm_id_priv->id.device > cur_cm_id_priv->id.device)
650 link = &(*link)->rb_right;
651 else if (be64_lt(service_id, cur_cm_id_priv->id.service_id))
652 link = &(*link)->rb_left;
653 else if (be64_gt(service_id, cur_cm_id_priv->id.service_id))
654 link = &(*link)->rb_right;
655 else {
656 /*
657 * Sharing an ib_cm_id with different handlers is not
658 * supported
659 */
660 if (cur_cm_id_priv->id.cm_handler != shared_handler ||
661 cur_cm_id_priv->id.context ||
662 WARN_ON(!cur_cm_id_priv->id.cm_handler)) {
663 spin_unlock_irqrestore(&cm.lock, flags);
664 return NULL;
665 }
666 refcount_inc(&cur_cm_id_priv->refcount);
667 cur_cm_id_priv->listen_sharecount++;
668 spin_unlock_irqrestore(&cm.lock, flags);
669 return cur_cm_id_priv;
670 }
671 }
672 cm_id_priv->listen_sharecount++;
673 rb_link_node(&cm_id_priv->service_node, parent, link);
674 rb_insert_color(&cm_id_priv->service_node, &cm.listen_service_table);
675 spin_unlock_irqrestore(&cm.lock, flags);
676 return cm_id_priv;
677 }
678
cm_find_listen(struct ib_device * device,__be64 service_id)679 static struct cm_id_private *cm_find_listen(struct ib_device *device,
680 __be64 service_id)
681 {
682 struct rb_node *node = cm.listen_service_table.rb_node;
683 struct cm_id_private *cm_id_priv;
684
685 while (node) {
686 cm_id_priv = rb_entry(node, struct cm_id_private, service_node);
687
688 if (device < cm_id_priv->id.device)
689 node = node->rb_left;
690 else if (device > cm_id_priv->id.device)
691 node = node->rb_right;
692 else if (be64_lt(service_id, cm_id_priv->id.service_id))
693 node = node->rb_left;
694 else if (be64_gt(service_id, cm_id_priv->id.service_id))
695 node = node->rb_right;
696 else {
697 refcount_inc(&cm_id_priv->refcount);
698 return cm_id_priv;
699 }
700 }
701 return NULL;
702 }
703
704 static struct cm_timewait_info *
cm_insert_remote_id(struct cm_timewait_info * timewait_info)705 cm_insert_remote_id(struct cm_timewait_info *timewait_info)
706 {
707 struct rb_node **link = &cm.remote_id_table.rb_node;
708 struct rb_node *parent = NULL;
709 struct cm_timewait_info *cur_timewait_info;
710 __be64 remote_ca_guid = timewait_info->remote_ca_guid;
711 __be32 remote_id = timewait_info->work.remote_id;
712
713 while (*link) {
714 parent = *link;
715 cur_timewait_info = rb_entry(parent, struct cm_timewait_info,
716 remote_id_node);
717 if (be32_lt(remote_id, cur_timewait_info->work.remote_id))
718 link = &(*link)->rb_left;
719 else if (be32_gt(remote_id, cur_timewait_info->work.remote_id))
720 link = &(*link)->rb_right;
721 else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
722 link = &(*link)->rb_left;
723 else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
724 link = &(*link)->rb_right;
725 else
726 return cur_timewait_info;
727 }
728 timewait_info->inserted_remote_id = 1;
729 rb_link_node(&timewait_info->remote_id_node, parent, link);
730 rb_insert_color(&timewait_info->remote_id_node, &cm.remote_id_table);
731 return NULL;
732 }
733
cm_find_remote_id(__be64 remote_ca_guid,__be32 remote_id)734 static struct cm_id_private *cm_find_remote_id(__be64 remote_ca_guid,
735 __be32 remote_id)
736 {
737 struct rb_node *node = cm.remote_id_table.rb_node;
738 struct cm_timewait_info *timewait_info;
739 struct cm_id_private *res = NULL;
740
741 spin_lock_irq(&cm.lock);
742 while (node) {
743 timewait_info = rb_entry(node, struct cm_timewait_info,
744 remote_id_node);
745 if (be32_lt(remote_id, timewait_info->work.remote_id))
746 node = node->rb_left;
747 else if (be32_gt(remote_id, timewait_info->work.remote_id))
748 node = node->rb_right;
749 else if (be64_lt(remote_ca_guid, timewait_info->remote_ca_guid))
750 node = node->rb_left;
751 else if (be64_gt(remote_ca_guid, timewait_info->remote_ca_guid))
752 node = node->rb_right;
753 else {
754 res = cm_acquire_id(timewait_info->work.local_id,
755 timewait_info->work.remote_id);
756 break;
757 }
758 }
759 spin_unlock_irq(&cm.lock);
760 return res;
761 }
762
763 static struct cm_timewait_info *
cm_insert_remote_qpn(struct cm_timewait_info * timewait_info)764 cm_insert_remote_qpn(struct cm_timewait_info *timewait_info)
765 {
766 struct rb_node **link = &cm.remote_qp_table.rb_node;
767 struct rb_node *parent = NULL;
768 struct cm_timewait_info *cur_timewait_info;
769 __be64 remote_ca_guid = timewait_info->remote_ca_guid;
770 __be32 remote_qpn = timewait_info->remote_qpn;
771
772 while (*link) {
773 parent = *link;
774 cur_timewait_info = rb_entry(parent, struct cm_timewait_info,
775 remote_qp_node);
776 if (be32_lt(remote_qpn, cur_timewait_info->remote_qpn))
777 link = &(*link)->rb_left;
778 else if (be32_gt(remote_qpn, cur_timewait_info->remote_qpn))
779 link = &(*link)->rb_right;
780 else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
781 link = &(*link)->rb_left;
782 else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
783 link = &(*link)->rb_right;
784 else
785 return cur_timewait_info;
786 }
787 timewait_info->inserted_remote_qp = 1;
788 rb_link_node(&timewait_info->remote_qp_node, parent, link);
789 rb_insert_color(&timewait_info->remote_qp_node, &cm.remote_qp_table);
790 return NULL;
791 }
792
793 static struct cm_id_private *
cm_insert_remote_sidr(struct cm_id_private * cm_id_priv)794 cm_insert_remote_sidr(struct cm_id_private *cm_id_priv)
795 {
796 struct rb_node **link = &cm.remote_sidr_table.rb_node;
797 struct rb_node *parent = NULL;
798 struct cm_id_private *cur_cm_id_priv;
799 __be32 remote_id = cm_id_priv->id.remote_id;
800
801 while (*link) {
802 parent = *link;
803 cur_cm_id_priv = rb_entry(parent, struct cm_id_private,
804 sidr_id_node);
805 if (be32_lt(remote_id, cur_cm_id_priv->id.remote_id))
806 link = &(*link)->rb_left;
807 else if (be32_gt(remote_id, cur_cm_id_priv->id.remote_id))
808 link = &(*link)->rb_right;
809 else {
810 if (cur_cm_id_priv->sidr_slid < cm_id_priv->sidr_slid)
811 link = &(*link)->rb_left;
812 else if (cur_cm_id_priv->sidr_slid > cm_id_priv->sidr_slid)
813 link = &(*link)->rb_right;
814 else
815 return cur_cm_id_priv;
816 }
817 }
818 rb_link_node(&cm_id_priv->sidr_id_node, parent, link);
819 rb_insert_color(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
820 return NULL;
821 }
822
cm_alloc_id_priv(struct ib_device * device,ib_cm_handler cm_handler,void * context)823 static struct cm_id_private *cm_alloc_id_priv(struct ib_device *device,
824 ib_cm_handler cm_handler,
825 void *context)
826 {
827 struct cm_id_private *cm_id_priv;
828 u32 id;
829 int ret;
830
831 cm_id_priv = kzalloc(sizeof *cm_id_priv, GFP_KERNEL);
832 if (!cm_id_priv)
833 return ERR_PTR(-ENOMEM);
834
835 cm_id_priv->id.state = IB_CM_IDLE;
836 cm_id_priv->id.device = device;
837 cm_id_priv->id.cm_handler = cm_handler;
838 cm_id_priv->id.context = context;
839 cm_id_priv->id.remote_cm_qpn = 1;
840
841 RB_CLEAR_NODE(&cm_id_priv->service_node);
842 RB_CLEAR_NODE(&cm_id_priv->sidr_id_node);
843 spin_lock_init(&cm_id_priv->lock);
844 init_completion(&cm_id_priv->comp);
845 INIT_LIST_HEAD(&cm_id_priv->work_list);
846 atomic_set(&cm_id_priv->work_count, -1);
847 refcount_set(&cm_id_priv->refcount, 1);
848
849 ret = xa_alloc_cyclic(&cm.local_id_table, &id, NULL, xa_limit_32b,
850 &cm.local_id_next, GFP_KERNEL);
851 if (ret < 0)
852 goto error;
853 cm_id_priv->id.local_id = (__force __be32)id ^ cm.random_id_operand;
854
855 return cm_id_priv;
856
857 error:
858 kfree(cm_id_priv);
859 return ERR_PTR(ret);
860 }
861
862 /*
863 * Make the ID visible to the MAD handlers and other threads that use the
864 * xarray.
865 */
cm_finalize_id(struct cm_id_private * cm_id_priv)866 static void cm_finalize_id(struct cm_id_private *cm_id_priv)
867 {
868 xa_store(&cm.local_id_table, cm_local_id(cm_id_priv->id.local_id),
869 cm_id_priv, GFP_ATOMIC);
870 }
871
ib_create_cm_id(struct ib_device * device,ib_cm_handler cm_handler,void * context)872 struct ib_cm_id *ib_create_cm_id(struct ib_device *device,
873 ib_cm_handler cm_handler,
874 void *context)
875 {
876 struct cm_id_private *cm_id_priv;
877
878 cm_id_priv = cm_alloc_id_priv(device, cm_handler, context);
879 if (IS_ERR(cm_id_priv))
880 return ERR_CAST(cm_id_priv);
881
882 cm_finalize_id(cm_id_priv);
883 return &cm_id_priv->id;
884 }
885 EXPORT_SYMBOL(ib_create_cm_id);
886
cm_dequeue_work(struct cm_id_private * cm_id_priv)887 static struct cm_work *cm_dequeue_work(struct cm_id_private *cm_id_priv)
888 {
889 struct cm_work *work;
890
891 if (list_empty(&cm_id_priv->work_list))
892 return NULL;
893
894 work = list_entry(cm_id_priv->work_list.next, struct cm_work, list);
895 list_del(&work->list);
896 return work;
897 }
898
cm_free_work(struct cm_work * work)899 static void cm_free_work(struct cm_work *work)
900 {
901 if (work->mad_recv_wc)
902 ib_free_recv_mad(work->mad_recv_wc);
903 kfree(work);
904 }
905
cm_queue_work_unlock(struct cm_id_private * cm_id_priv,struct cm_work * work)906 static void cm_queue_work_unlock(struct cm_id_private *cm_id_priv,
907 struct cm_work *work)
908 __releases(&cm_id_priv->lock)
909 {
910 bool immediate;
911
912 /*
913 * To deliver the event to the user callback we have the drop the
914 * spinlock, however, we need to ensure that the user callback is single
915 * threaded and receives events in the temporal order. If there are
916 * already events being processed then thread new events onto a list,
917 * the thread currently processing will pick them up.
918 */
919 immediate = atomic_inc_and_test(&cm_id_priv->work_count);
920 if (!immediate) {
921 list_add_tail(&work->list, &cm_id_priv->work_list);
922 /*
923 * This routine always consumes incoming reference. Once queued
924 * to the work_list then a reference is held by the thread
925 * currently running cm_process_work() and this reference is not
926 * needed.
927 */
928 cm_deref_id(cm_id_priv);
929 }
930 spin_unlock_irq(&cm_id_priv->lock);
931
932 if (immediate)
933 cm_process_work(cm_id_priv, work);
934 }
935
cm_convert_to_ms(int iba_time)936 static inline int cm_convert_to_ms(int iba_time)
937 {
938 /* approximate conversion to ms from 4.096us x 2^iba_time */
939 return 1 << max(iba_time - 8, 0);
940 }
941
942 /*
943 * calculate: 4.096x2^ack_timeout = 4.096x2^ack_delay + 2x4.096x2^life_time
944 * Because of how ack_timeout is stored, adding one doubles the timeout.
945 * To avoid large timeouts, select the max(ack_delay, life_time + 1), and
946 * increment it (round up) only if the other is within 50%.
947 */
cm_ack_timeout(u8 ca_ack_delay,u8 packet_life_time)948 static u8 cm_ack_timeout(u8 ca_ack_delay, u8 packet_life_time)
949 {
950 int ack_timeout = packet_life_time + 1;
951
952 if (ack_timeout >= ca_ack_delay)
953 ack_timeout += (ca_ack_delay >= (ack_timeout - 1));
954 else
955 ack_timeout = ca_ack_delay +
956 (ack_timeout >= (ca_ack_delay - 1));
957
958 return min(31, ack_timeout);
959 }
960
cm_remove_remote(struct cm_id_private * cm_id_priv)961 static void cm_remove_remote(struct cm_id_private *cm_id_priv)
962 {
963 struct cm_timewait_info *timewait_info = cm_id_priv->timewait_info;
964
965 if (timewait_info->inserted_remote_id) {
966 rb_erase(&timewait_info->remote_id_node, &cm.remote_id_table);
967 timewait_info->inserted_remote_id = 0;
968 }
969
970 if (timewait_info->inserted_remote_qp) {
971 rb_erase(&timewait_info->remote_qp_node, &cm.remote_qp_table);
972 timewait_info->inserted_remote_qp = 0;
973 }
974 }
975
cm_create_timewait_info(__be32 local_id)976 static struct cm_timewait_info *cm_create_timewait_info(__be32 local_id)
977 {
978 struct cm_timewait_info *timewait_info;
979
980 timewait_info = kzalloc(sizeof *timewait_info, GFP_KERNEL);
981 if (!timewait_info)
982 return ERR_PTR(-ENOMEM);
983
984 timewait_info->work.local_id = local_id;
985 INIT_DELAYED_WORK(&timewait_info->work.work, cm_work_handler);
986 timewait_info->work.cm_event.event = IB_CM_TIMEWAIT_EXIT;
987 return timewait_info;
988 }
989
cm_enter_timewait(struct cm_id_private * cm_id_priv)990 static void cm_enter_timewait(struct cm_id_private *cm_id_priv)
991 {
992 int wait_time;
993 unsigned long flags;
994 struct cm_device *cm_dev;
995
996 lockdep_assert_held(&cm_id_priv->lock);
997
998 cm_dev = ib_get_client_data(cm_id_priv->id.device, &cm_client);
999 if (!cm_dev)
1000 return;
1001
1002 spin_lock_irqsave(&cm.lock, flags);
1003 cm_remove_remote(cm_id_priv);
1004 list_add_tail(&cm_id_priv->timewait_info->list, &cm.timewait_list);
1005 spin_unlock_irqrestore(&cm.lock, flags);
1006
1007 /*
1008 * The cm_id could be destroyed by the user before we exit timewait.
1009 * To protect against this, we search for the cm_id after exiting
1010 * timewait before notifying the user that we've exited timewait.
1011 */
1012 cm_id_priv->id.state = IB_CM_TIMEWAIT;
1013 wait_time = cm_convert_to_ms(cm_id_priv->av.timeout);
1014
1015 /* Check if the device started its remove_one */
1016 spin_lock_irqsave(&cm.lock, flags);
1017 if (!cm_dev->going_down)
1018 queue_delayed_work(cm.wq, &cm_id_priv->timewait_info->work.work,
1019 msecs_to_jiffies(wait_time));
1020 spin_unlock_irqrestore(&cm.lock, flags);
1021
1022 /*
1023 * The timewait_info is converted into a work and gets freed during
1024 * cm_free_work() in cm_timewait_handler().
1025 */
1026 BUILD_BUG_ON(offsetof(struct cm_timewait_info, work) != 0);
1027 cm_id_priv->timewait_info = NULL;
1028 }
1029
cm_reset_to_idle(struct cm_id_private * cm_id_priv)1030 static void cm_reset_to_idle(struct cm_id_private *cm_id_priv)
1031 {
1032 unsigned long flags;
1033
1034 lockdep_assert_held(&cm_id_priv->lock);
1035
1036 cm_id_priv->id.state = IB_CM_IDLE;
1037 if (cm_id_priv->timewait_info) {
1038 spin_lock_irqsave(&cm.lock, flags);
1039 cm_remove_remote(cm_id_priv);
1040 spin_unlock_irqrestore(&cm.lock, flags);
1041 kfree(cm_id_priv->timewait_info);
1042 cm_id_priv->timewait_info = NULL;
1043 }
1044 }
1045
cm_destroy_id_wait_timeout(struct ib_cm_id * cm_id,enum ib_cm_state old_state)1046 static noinline void cm_destroy_id_wait_timeout(struct ib_cm_id *cm_id,
1047 enum ib_cm_state old_state)
1048 {
1049 struct cm_id_private *cm_id_priv;
1050
1051 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1052 pr_err("%s: cm_id=%p timed out. state %d -> %d, refcnt=%d\n", __func__,
1053 cm_id, old_state, cm_id->state, refcount_read(&cm_id_priv->refcount));
1054 }
1055
cm_destroy_id(struct ib_cm_id * cm_id,int err)1056 static void cm_destroy_id(struct ib_cm_id *cm_id, int err)
1057 {
1058 struct cm_id_private *cm_id_priv;
1059 enum ib_cm_state old_state;
1060 struct cm_work *work;
1061 int ret;
1062
1063 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1064 spin_lock_irq(&cm_id_priv->lock);
1065 old_state = cm_id->state;
1066 retest:
1067 switch (cm_id->state) {
1068 case IB_CM_LISTEN:
1069 spin_lock(&cm.lock);
1070 if (--cm_id_priv->listen_sharecount > 0) {
1071 /* The id is still shared. */
1072 WARN_ON(refcount_read(&cm_id_priv->refcount) == 1);
1073 spin_unlock(&cm.lock);
1074 spin_unlock_irq(&cm_id_priv->lock);
1075 cm_deref_id(cm_id_priv);
1076 return;
1077 }
1078 cm_id->state = IB_CM_IDLE;
1079 rb_erase(&cm_id_priv->service_node, &cm.listen_service_table);
1080 RB_CLEAR_NODE(&cm_id_priv->service_node);
1081 spin_unlock(&cm.lock);
1082 break;
1083 case IB_CM_SIDR_REQ_SENT:
1084 cm_id->state = IB_CM_IDLE;
1085 ib_cancel_mad(cm_id_priv->msg);
1086 break;
1087 case IB_CM_SIDR_REQ_RCVD:
1088 cm_send_sidr_rep_locked(cm_id_priv,
1089 &(struct ib_cm_sidr_rep_param){
1090 .status = IB_SIDR_REJECT });
1091 /* cm_send_sidr_rep_locked will not move to IDLE if it fails */
1092 cm_id->state = IB_CM_IDLE;
1093 break;
1094 case IB_CM_REQ_SENT:
1095 case IB_CM_MRA_REQ_RCVD:
1096 ib_cancel_mad(cm_id_priv->msg);
1097 cm_send_rej_locked(cm_id_priv, IB_CM_REJ_TIMEOUT,
1098 &cm_id_priv->id.device->node_guid,
1099 sizeof(cm_id_priv->id.device->node_guid),
1100 NULL, 0);
1101 break;
1102 case IB_CM_REQ_RCVD:
1103 if (err == -ENOMEM) {
1104 /* Do not reject to allow future retries. */
1105 cm_reset_to_idle(cm_id_priv);
1106 } else {
1107 cm_send_rej_locked(cm_id_priv,
1108 IB_CM_REJ_CONSUMER_DEFINED, NULL, 0,
1109 NULL, 0);
1110 }
1111 break;
1112 case IB_CM_REP_SENT:
1113 case IB_CM_MRA_REP_RCVD:
1114 ib_cancel_mad(cm_id_priv->msg);
1115 cm_send_rej_locked(cm_id_priv, IB_CM_REJ_CONSUMER_DEFINED, NULL,
1116 0, NULL, 0);
1117 goto retest;
1118 case IB_CM_MRA_REQ_SENT:
1119 case IB_CM_REP_RCVD:
1120 case IB_CM_MRA_REP_SENT:
1121 cm_send_rej_locked(cm_id_priv, IB_CM_REJ_CONSUMER_DEFINED, NULL,
1122 0, NULL, 0);
1123 break;
1124 case IB_CM_ESTABLISHED:
1125 if (cm_id_priv->qp_type == IB_QPT_XRC_TGT) {
1126 cm_id->state = IB_CM_IDLE;
1127 break;
1128 }
1129 cm_issue_dreq(cm_id_priv);
1130 cm_enter_timewait(cm_id_priv);
1131 goto retest;
1132 case IB_CM_DREQ_SENT:
1133 ib_cancel_mad(cm_id_priv->msg);
1134 cm_enter_timewait(cm_id_priv);
1135 goto retest;
1136 case IB_CM_DREQ_RCVD:
1137 cm_send_drep_locked(cm_id_priv, NULL, 0);
1138 WARN_ON(cm_id->state != IB_CM_TIMEWAIT);
1139 goto retest;
1140 case IB_CM_TIMEWAIT:
1141 /*
1142 * The cm_acquire_id in cm_timewait_handler will stop working
1143 * once we do xa_erase below, so just move to idle here for
1144 * consistency.
1145 */
1146 cm_id->state = IB_CM_IDLE;
1147 break;
1148 case IB_CM_IDLE:
1149 break;
1150 }
1151 WARN_ON(cm_id->state != IB_CM_IDLE);
1152
1153 spin_lock(&cm.lock);
1154 /* Required for cleanup paths related cm_req_handler() */
1155 if (cm_id_priv->timewait_info) {
1156 cm_remove_remote(cm_id_priv);
1157 kfree(cm_id_priv->timewait_info);
1158 cm_id_priv->timewait_info = NULL;
1159 }
1160
1161 WARN_ON(cm_id_priv->listen_sharecount);
1162 WARN_ON(!RB_EMPTY_NODE(&cm_id_priv->service_node));
1163 if (!RB_EMPTY_NODE(&cm_id_priv->sidr_id_node))
1164 rb_erase(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
1165 spin_unlock(&cm.lock);
1166 spin_unlock_irq(&cm_id_priv->lock);
1167
1168 xa_erase(&cm.local_id_table, cm_local_id(cm_id->local_id));
1169 cm_deref_id(cm_id_priv);
1170 do {
1171 ret = wait_for_completion_timeout(&cm_id_priv->comp,
1172 msecs_to_jiffies(
1173 CM_DESTROY_ID_WAIT_TIMEOUT));
1174 if (!ret) /* timeout happened */
1175 cm_destroy_id_wait_timeout(cm_id, old_state);
1176 } while (!ret);
1177
1178 while ((work = cm_dequeue_work(cm_id_priv)) != NULL)
1179 cm_free_work(work);
1180
1181 cm_destroy_av(&cm_id_priv->av);
1182 cm_destroy_av(&cm_id_priv->alt_av);
1183 kfree(cm_id_priv->private_data);
1184 kfree_rcu(cm_id_priv, rcu);
1185 }
1186
ib_destroy_cm_id(struct ib_cm_id * cm_id)1187 void ib_destroy_cm_id(struct ib_cm_id *cm_id)
1188 {
1189 cm_destroy_id(cm_id, 0);
1190 }
1191 EXPORT_SYMBOL(ib_destroy_cm_id);
1192
cm_init_listen(struct cm_id_private * cm_id_priv,__be64 service_id)1193 static int cm_init_listen(struct cm_id_private *cm_id_priv, __be64 service_id)
1194 {
1195 if ((service_id & IB_SERVICE_ID_AGN_MASK) == IB_CM_ASSIGN_SERVICE_ID &&
1196 (service_id != IB_CM_ASSIGN_SERVICE_ID))
1197 return -EINVAL;
1198
1199 if (service_id == IB_CM_ASSIGN_SERVICE_ID)
1200 cm_id_priv->id.service_id = cpu_to_be64(cm.listen_service_id++);
1201 else
1202 cm_id_priv->id.service_id = service_id;
1203
1204 return 0;
1205 }
1206
1207 /**
1208 * ib_cm_listen - Initiates listening on the specified service ID for
1209 * connection and service ID resolution requests.
1210 * @cm_id: Connection identifier associated with the listen request.
1211 * @service_id: Service identifier matched against incoming connection
1212 * and service ID resolution requests. The service ID should be specified
1213 * network-byte order. If set to IB_CM_ASSIGN_SERVICE_ID, the CM will
1214 * assign a service ID to the caller.
1215 */
ib_cm_listen(struct ib_cm_id * cm_id,__be64 service_id)1216 int ib_cm_listen(struct ib_cm_id *cm_id, __be64 service_id)
1217 {
1218 struct cm_id_private *cm_id_priv =
1219 container_of(cm_id, struct cm_id_private, id);
1220 unsigned long flags;
1221 int ret;
1222
1223 spin_lock_irqsave(&cm_id_priv->lock, flags);
1224 if (cm_id_priv->id.state != IB_CM_IDLE) {
1225 ret = -EINVAL;
1226 goto out;
1227 }
1228
1229 ret = cm_init_listen(cm_id_priv, service_id);
1230 if (ret)
1231 goto out;
1232
1233 if (!cm_insert_listen(cm_id_priv, NULL)) {
1234 ret = -EBUSY;
1235 goto out;
1236 }
1237
1238 cm_id_priv->id.state = IB_CM_LISTEN;
1239 ret = 0;
1240
1241 out:
1242 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1243 return ret;
1244 }
1245 EXPORT_SYMBOL(ib_cm_listen);
1246
1247 /**
1248 * ib_cm_insert_listen - Create a new listening ib_cm_id and listen on
1249 * the given service ID.
1250 *
1251 * If there's an existing ID listening on that same device and service ID,
1252 * return it.
1253 *
1254 * @device: Device associated with the cm_id. All related communication will
1255 * be associated with the specified device.
1256 * @cm_handler: Callback invoked to notify the user of CM events.
1257 * @service_id: Service identifier matched against incoming connection
1258 * and service ID resolution requests. The service ID should be specified
1259 * network-byte order. If set to IB_CM_ASSIGN_SERVICE_ID, the CM will
1260 * assign a service ID to the caller.
1261 *
1262 * Callers should call ib_destroy_cm_id when done with the listener ID.
1263 */
ib_cm_insert_listen(struct ib_device * device,ib_cm_handler cm_handler,__be64 service_id)1264 struct ib_cm_id *ib_cm_insert_listen(struct ib_device *device,
1265 ib_cm_handler cm_handler,
1266 __be64 service_id)
1267 {
1268 struct cm_id_private *listen_id_priv;
1269 struct cm_id_private *cm_id_priv;
1270 int err = 0;
1271
1272 /* Create an ID in advance, since the creation may sleep */
1273 cm_id_priv = cm_alloc_id_priv(device, cm_handler, NULL);
1274 if (IS_ERR(cm_id_priv))
1275 return ERR_CAST(cm_id_priv);
1276
1277 err = cm_init_listen(cm_id_priv, service_id);
1278 if (err) {
1279 ib_destroy_cm_id(&cm_id_priv->id);
1280 return ERR_PTR(err);
1281 }
1282
1283 spin_lock_irq(&cm_id_priv->lock);
1284 listen_id_priv = cm_insert_listen(cm_id_priv, cm_handler);
1285 if (listen_id_priv != cm_id_priv) {
1286 spin_unlock_irq(&cm_id_priv->lock);
1287 ib_destroy_cm_id(&cm_id_priv->id);
1288 if (!listen_id_priv)
1289 return ERR_PTR(-EINVAL);
1290 return &listen_id_priv->id;
1291 }
1292 cm_id_priv->id.state = IB_CM_LISTEN;
1293 spin_unlock_irq(&cm_id_priv->lock);
1294
1295 /*
1296 * A listen ID does not need to be in the xarray since it does not
1297 * receive mads, is not placed in the remote_id or remote_qpn rbtree,
1298 * and does not enter timewait.
1299 */
1300
1301 return &cm_id_priv->id;
1302 }
1303 EXPORT_SYMBOL(ib_cm_insert_listen);
1304
cm_form_tid(struct cm_id_private * cm_id_priv)1305 static __be64 cm_form_tid(struct cm_id_private *cm_id_priv)
1306 {
1307 u64 hi_tid = 0, low_tid;
1308
1309 lockdep_assert_held(&cm_id_priv->lock);
1310
1311 low_tid = (u64)cm_id_priv->id.local_id;
1312 if (!cm_id_priv->av.port)
1313 return cpu_to_be64(low_tid);
1314
1315 read_lock(&cm_id_priv->av.port->cm_dev->mad_agent_lock);
1316 if (cm_id_priv->av.port->mad_agent)
1317 hi_tid = ((u64)cm_id_priv->av.port->mad_agent->hi_tid) << 32;
1318 read_unlock(&cm_id_priv->av.port->cm_dev->mad_agent_lock);
1319 return cpu_to_be64(hi_tid | low_tid);
1320 }
1321
cm_format_mad_hdr(struct ib_mad_hdr * hdr,__be16 attr_id,__be64 tid)1322 static void cm_format_mad_hdr(struct ib_mad_hdr *hdr,
1323 __be16 attr_id, __be64 tid)
1324 {
1325 hdr->base_version = IB_MGMT_BASE_VERSION;
1326 hdr->mgmt_class = IB_MGMT_CLASS_CM;
1327 hdr->class_version = IB_CM_CLASS_VERSION;
1328 hdr->method = IB_MGMT_METHOD_SEND;
1329 hdr->attr_id = attr_id;
1330 hdr->tid = tid;
1331 }
1332
cm_format_mad_ece_hdr(struct ib_mad_hdr * hdr,__be16 attr_id,__be64 tid,u32 attr_mod)1333 static void cm_format_mad_ece_hdr(struct ib_mad_hdr *hdr, __be16 attr_id,
1334 __be64 tid, u32 attr_mod)
1335 {
1336 cm_format_mad_hdr(hdr, attr_id, tid);
1337 hdr->attr_mod = cpu_to_be32(attr_mod);
1338 }
1339
cm_format_req(struct cm_req_msg * req_msg,struct cm_id_private * cm_id_priv,struct ib_cm_req_param * param)1340 static void cm_format_req(struct cm_req_msg *req_msg,
1341 struct cm_id_private *cm_id_priv,
1342 struct ib_cm_req_param *param)
1343 {
1344 struct sa_path_rec *pri_path = param->primary_path;
1345 struct sa_path_rec *alt_path = param->alternate_path;
1346 bool pri_ext = false;
1347 __be16 lid;
1348
1349 if (pri_path->rec_type == SA_PATH_REC_TYPE_OPA)
1350 pri_ext = opa_is_extended_lid(pri_path->opa.dlid,
1351 pri_path->opa.slid);
1352
1353 cm_format_mad_ece_hdr(&req_msg->hdr, CM_REQ_ATTR_ID,
1354 cm_form_tid(cm_id_priv), param->ece.attr_mod);
1355
1356 IBA_SET(CM_REQ_LOCAL_COMM_ID, req_msg,
1357 be32_to_cpu(cm_id_priv->id.local_id));
1358 IBA_SET(CM_REQ_SERVICE_ID, req_msg, be64_to_cpu(param->service_id));
1359 IBA_SET(CM_REQ_LOCAL_CA_GUID, req_msg,
1360 be64_to_cpu(cm_id_priv->id.device->node_guid));
1361 IBA_SET(CM_REQ_LOCAL_QPN, req_msg, param->qp_num);
1362 IBA_SET(CM_REQ_INITIATOR_DEPTH, req_msg, param->initiator_depth);
1363 IBA_SET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, req_msg,
1364 param->remote_cm_response_timeout);
1365 cm_req_set_qp_type(req_msg, param->qp_type);
1366 IBA_SET(CM_REQ_END_TO_END_FLOW_CONTROL, req_msg, param->flow_control);
1367 IBA_SET(CM_REQ_STARTING_PSN, req_msg, param->starting_psn);
1368 IBA_SET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg,
1369 param->local_cm_response_timeout);
1370 IBA_SET(CM_REQ_PARTITION_KEY, req_msg,
1371 be16_to_cpu(param->primary_path->pkey));
1372 IBA_SET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg,
1373 param->primary_path->mtu);
1374 IBA_SET(CM_REQ_MAX_CM_RETRIES, req_msg, param->max_cm_retries);
1375
1376 if (param->qp_type != IB_QPT_XRC_INI) {
1377 IBA_SET(CM_REQ_RESPONDER_RESOURCES, req_msg,
1378 param->responder_resources);
1379 IBA_SET(CM_REQ_RETRY_COUNT, req_msg, param->retry_count);
1380 IBA_SET(CM_REQ_RNR_RETRY_COUNT, req_msg,
1381 param->rnr_retry_count);
1382 IBA_SET(CM_REQ_SRQ, req_msg, param->srq);
1383 }
1384
1385 *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg) =
1386 pri_path->sgid;
1387 *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg) =
1388 pri_path->dgid;
1389 if (pri_ext) {
1390 IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg)
1391 ->global.interface_id =
1392 OPA_MAKE_ID(be32_to_cpu(pri_path->opa.slid));
1393 IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg)
1394 ->global.interface_id =
1395 OPA_MAKE_ID(be32_to_cpu(pri_path->opa.dlid));
1396 }
1397 if (pri_path->hop_limit <= 1) {
1398 IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
1399 be16_to_cpu(pri_ext ? 0 :
1400 htons(ntohl(sa_path_get_slid(
1401 pri_path)))));
1402 IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg,
1403 be16_to_cpu(pri_ext ? 0 :
1404 htons(ntohl(sa_path_get_dlid(
1405 pri_path)))));
1406 } else {
1407
1408 if (param->primary_path_inbound) {
1409 lid = param->primary_path_inbound->ib.dlid;
1410 IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
1411 be16_to_cpu(lid));
1412 } else
1413 IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
1414 be16_to_cpu(IB_LID_PERMISSIVE));
1415
1416 /* Work-around until there's a way to obtain remote LID info */
1417 IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg,
1418 be16_to_cpu(IB_LID_PERMISSIVE));
1419 }
1420 IBA_SET(CM_REQ_PRIMARY_FLOW_LABEL, req_msg,
1421 be32_to_cpu(pri_path->flow_label));
1422 IBA_SET(CM_REQ_PRIMARY_PACKET_RATE, req_msg, pri_path->rate);
1423 IBA_SET(CM_REQ_PRIMARY_TRAFFIC_CLASS, req_msg, pri_path->traffic_class);
1424 IBA_SET(CM_REQ_PRIMARY_HOP_LIMIT, req_msg, pri_path->hop_limit);
1425 IBA_SET(CM_REQ_PRIMARY_SL, req_msg, pri_path->sl);
1426 IBA_SET(CM_REQ_PRIMARY_SUBNET_LOCAL, req_msg,
1427 (pri_path->hop_limit <= 1));
1428 IBA_SET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, req_msg,
1429 cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
1430 pri_path->packet_life_time));
1431
1432 if (alt_path) {
1433 bool alt_ext = false;
1434
1435 if (alt_path->rec_type == SA_PATH_REC_TYPE_OPA)
1436 alt_ext = opa_is_extended_lid(alt_path->opa.dlid,
1437 alt_path->opa.slid);
1438
1439 *IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg) =
1440 alt_path->sgid;
1441 *IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg) =
1442 alt_path->dgid;
1443 if (alt_ext) {
1444 IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID,
1445 req_msg)
1446 ->global.interface_id =
1447 OPA_MAKE_ID(be32_to_cpu(alt_path->opa.slid));
1448 IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_REMOTE_PORT_GID,
1449 req_msg)
1450 ->global.interface_id =
1451 OPA_MAKE_ID(be32_to_cpu(alt_path->opa.dlid));
1452 }
1453 if (alt_path->hop_limit <= 1) {
1454 IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg,
1455 be16_to_cpu(
1456 alt_ext ? 0 :
1457 htons(ntohl(sa_path_get_slid(
1458 alt_path)))));
1459 IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg,
1460 be16_to_cpu(
1461 alt_ext ? 0 :
1462 htons(ntohl(sa_path_get_dlid(
1463 alt_path)))));
1464 } else {
1465 IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg,
1466 be16_to_cpu(IB_LID_PERMISSIVE));
1467 IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg,
1468 be16_to_cpu(IB_LID_PERMISSIVE));
1469 }
1470 IBA_SET(CM_REQ_ALTERNATE_FLOW_LABEL, req_msg,
1471 be32_to_cpu(alt_path->flow_label));
1472 IBA_SET(CM_REQ_ALTERNATE_PACKET_RATE, req_msg, alt_path->rate);
1473 IBA_SET(CM_REQ_ALTERNATE_TRAFFIC_CLASS, req_msg,
1474 alt_path->traffic_class);
1475 IBA_SET(CM_REQ_ALTERNATE_HOP_LIMIT, req_msg,
1476 alt_path->hop_limit);
1477 IBA_SET(CM_REQ_ALTERNATE_SL, req_msg, alt_path->sl);
1478 IBA_SET(CM_REQ_ALTERNATE_SUBNET_LOCAL, req_msg,
1479 (alt_path->hop_limit <= 1));
1480 IBA_SET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, req_msg,
1481 cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
1482 alt_path->packet_life_time));
1483 }
1484 IBA_SET(CM_REQ_VENDOR_ID, req_msg, param->ece.vendor_id);
1485
1486 if (param->private_data && param->private_data_len)
1487 IBA_SET_MEM(CM_REQ_PRIVATE_DATA, req_msg, param->private_data,
1488 param->private_data_len);
1489 }
1490
cm_validate_req_param(struct ib_cm_req_param * param)1491 static int cm_validate_req_param(struct ib_cm_req_param *param)
1492 {
1493 if (!param->primary_path)
1494 return -EINVAL;
1495
1496 if (param->qp_type != IB_QPT_RC && param->qp_type != IB_QPT_UC &&
1497 param->qp_type != IB_QPT_XRC_INI)
1498 return -EINVAL;
1499
1500 if (param->private_data &&
1501 param->private_data_len > IB_CM_REQ_PRIVATE_DATA_SIZE)
1502 return -EINVAL;
1503
1504 if (param->alternate_path &&
1505 (param->alternate_path->pkey != param->primary_path->pkey ||
1506 param->alternate_path->mtu != param->primary_path->mtu))
1507 return -EINVAL;
1508
1509 return 0;
1510 }
1511
ib_send_cm_req(struct ib_cm_id * cm_id,struct ib_cm_req_param * param)1512 int ib_send_cm_req(struct ib_cm_id *cm_id,
1513 struct ib_cm_req_param *param)
1514 {
1515 struct cm_av av = {}, alt_av = {};
1516 struct cm_id_private *cm_id_priv;
1517 struct ib_mad_send_buf *msg;
1518 struct cm_req_msg *req_msg;
1519 unsigned long flags;
1520 int ret;
1521
1522 ret = cm_validate_req_param(param);
1523 if (ret)
1524 return ret;
1525
1526 /* Verify that we're not in timewait. */
1527 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1528 spin_lock_irqsave(&cm_id_priv->lock, flags);
1529 if (cm_id->state != IB_CM_IDLE || WARN_ON(cm_id_priv->timewait_info)) {
1530 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1531 return -EINVAL;
1532 }
1533 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1534
1535 cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv->
1536 id.local_id);
1537 if (IS_ERR(cm_id_priv->timewait_info)) {
1538 ret = PTR_ERR(cm_id_priv->timewait_info);
1539 cm_id_priv->timewait_info = NULL;
1540 return ret;
1541 }
1542
1543 ret = cm_init_av_by_path(param->primary_path,
1544 param->ppath_sgid_attr, &av);
1545 if (ret)
1546 return ret;
1547 if (param->alternate_path) {
1548 ret = cm_init_av_by_path(param->alternate_path, NULL,
1549 &alt_av);
1550 if (ret) {
1551 cm_destroy_av(&av);
1552 return ret;
1553 }
1554 }
1555 cm_id->service_id = param->service_id;
1556 cm_id_priv->timeout_ms = cm_convert_to_ms(
1557 param->primary_path->packet_life_time) * 2 +
1558 cm_convert_to_ms(
1559 param->remote_cm_response_timeout);
1560 cm_id_priv->max_cm_retries = param->max_cm_retries;
1561 cm_id_priv->initiator_depth = param->initiator_depth;
1562 cm_id_priv->responder_resources = param->responder_resources;
1563 cm_id_priv->retry_count = param->retry_count;
1564 cm_id_priv->path_mtu = param->primary_path->mtu;
1565 cm_id_priv->pkey = param->primary_path->pkey;
1566 cm_id_priv->qp_type = param->qp_type;
1567
1568 spin_lock_irqsave(&cm_id_priv->lock, flags);
1569
1570 cm_move_av_from_path(&cm_id_priv->av, &av);
1571 if (param->primary_path_outbound)
1572 cm_id_priv->av.dlid_datapath =
1573 be16_to_cpu(param->primary_path_outbound->ib.dlid);
1574
1575 if (param->alternate_path)
1576 cm_move_av_from_path(&cm_id_priv->alt_av, &alt_av);
1577
1578 msg = cm_alloc_priv_msg(cm_id_priv, IB_CM_REQ_SENT);
1579 if (IS_ERR(msg)) {
1580 ret = PTR_ERR(msg);
1581 goto out_unlock;
1582 }
1583
1584 req_msg = (struct cm_req_msg *)msg->mad;
1585 cm_format_req(req_msg, cm_id_priv, param);
1586 cm_id_priv->tid = req_msg->hdr.tid;
1587
1588 cm_id_priv->local_qpn = cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg));
1589 cm_id_priv->rq_psn = cpu_to_be32(IBA_GET(CM_REQ_STARTING_PSN, req_msg));
1590
1591 trace_icm_send_req(&cm_id_priv->id);
1592 ret = ib_post_send_mad(msg, NULL);
1593 if (ret)
1594 goto out_free;
1595 BUG_ON(cm_id->state != IB_CM_IDLE);
1596 cm_id->state = IB_CM_REQ_SENT;
1597 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1598 return 0;
1599 out_free:
1600 cm_free_priv_msg(msg);
1601 out_unlock:
1602 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1603 return ret;
1604 }
1605 EXPORT_SYMBOL(ib_send_cm_req);
1606
cm_issue_rej(struct cm_port * port,struct ib_mad_recv_wc * mad_recv_wc,enum ib_cm_rej_reason reason,enum cm_msg_response msg_rejected,void * ari,u8 ari_length)1607 static int cm_issue_rej(struct cm_port *port,
1608 struct ib_mad_recv_wc *mad_recv_wc,
1609 enum ib_cm_rej_reason reason,
1610 enum cm_msg_response msg_rejected,
1611 void *ari, u8 ari_length)
1612 {
1613 struct ib_mad_send_buf *msg = NULL;
1614 struct cm_rej_msg *rej_msg, *rcv_msg;
1615 int ret;
1616
1617 ret = cm_alloc_response_msg(port, mad_recv_wc, false, &msg);
1618 if (ret)
1619 return ret;
1620
1621 /* We just need common CM header information. Cast to any message. */
1622 rcv_msg = (struct cm_rej_msg *) mad_recv_wc->recv_buf.mad;
1623 rej_msg = (struct cm_rej_msg *) msg->mad;
1624
1625 cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, rcv_msg->hdr.tid);
1626 IBA_SET(CM_REJ_REMOTE_COMM_ID, rej_msg,
1627 IBA_GET(CM_REJ_LOCAL_COMM_ID, rcv_msg));
1628 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1629 IBA_GET(CM_REJ_REMOTE_COMM_ID, rcv_msg));
1630 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, msg_rejected);
1631 IBA_SET(CM_REJ_REASON, rej_msg, reason);
1632
1633 if (ari && ari_length) {
1634 IBA_SET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg, ari_length);
1635 IBA_SET_MEM(CM_REJ_ARI, rej_msg, ari, ari_length);
1636 }
1637
1638 trace_icm_issue_rej(
1639 IBA_GET(CM_REJ_LOCAL_COMM_ID, rcv_msg),
1640 IBA_GET(CM_REJ_REMOTE_COMM_ID, rcv_msg));
1641 ret = ib_post_send_mad(msg, NULL);
1642 if (ret)
1643 cm_free_msg(msg);
1644
1645 return ret;
1646 }
1647
cm_req_has_alt_path(struct cm_req_msg * req_msg)1648 static bool cm_req_has_alt_path(struct cm_req_msg *req_msg)
1649 {
1650 return ((cpu_to_be16(
1651 IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg))) ||
1652 (ib_is_opa_gid(IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID,
1653 req_msg))));
1654 }
1655
cm_path_set_rec_type(struct ib_device * ib_device,u32 port_num,struct sa_path_rec * path,union ib_gid * gid)1656 static void cm_path_set_rec_type(struct ib_device *ib_device, u32 port_num,
1657 struct sa_path_rec *path, union ib_gid *gid)
1658 {
1659 if (ib_is_opa_gid(gid) && rdma_cap_opa_ah(ib_device, port_num))
1660 path->rec_type = SA_PATH_REC_TYPE_OPA;
1661 else
1662 path->rec_type = SA_PATH_REC_TYPE_IB;
1663 }
1664
cm_format_path_lid_from_req(struct cm_req_msg * req_msg,struct sa_path_rec * primary_path,struct sa_path_rec * alt_path,struct ib_wc * wc)1665 static void cm_format_path_lid_from_req(struct cm_req_msg *req_msg,
1666 struct sa_path_rec *primary_path,
1667 struct sa_path_rec *alt_path,
1668 struct ib_wc *wc)
1669 {
1670 u32 lid;
1671
1672 if (primary_path->rec_type != SA_PATH_REC_TYPE_OPA) {
1673 sa_path_set_dlid(primary_path, wc->slid);
1674 sa_path_set_slid(primary_path,
1675 IBA_GET(CM_REQ_PRIMARY_REMOTE_PORT_LID,
1676 req_msg));
1677 } else {
1678 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1679 CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg));
1680 sa_path_set_dlid(primary_path, lid);
1681
1682 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1683 CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg));
1684 sa_path_set_slid(primary_path, lid);
1685 }
1686
1687 if (!cm_req_has_alt_path(req_msg))
1688 return;
1689
1690 if (alt_path->rec_type != SA_PATH_REC_TYPE_OPA) {
1691 sa_path_set_dlid(alt_path,
1692 IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID,
1693 req_msg));
1694 sa_path_set_slid(alt_path,
1695 IBA_GET(CM_REQ_ALTERNATE_REMOTE_PORT_LID,
1696 req_msg));
1697 } else {
1698 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1699 CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg));
1700 sa_path_set_dlid(alt_path, lid);
1701
1702 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1703 CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg));
1704 sa_path_set_slid(alt_path, lid);
1705 }
1706 }
1707
cm_format_paths_from_req(struct cm_req_msg * req_msg,struct sa_path_rec * primary_path,struct sa_path_rec * alt_path,struct ib_wc * wc)1708 static void cm_format_paths_from_req(struct cm_req_msg *req_msg,
1709 struct sa_path_rec *primary_path,
1710 struct sa_path_rec *alt_path,
1711 struct ib_wc *wc)
1712 {
1713 primary_path->dgid =
1714 *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg);
1715 primary_path->sgid =
1716 *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg);
1717 primary_path->flow_label =
1718 cpu_to_be32(IBA_GET(CM_REQ_PRIMARY_FLOW_LABEL, req_msg));
1719 primary_path->hop_limit = IBA_GET(CM_REQ_PRIMARY_HOP_LIMIT, req_msg);
1720 primary_path->traffic_class =
1721 IBA_GET(CM_REQ_PRIMARY_TRAFFIC_CLASS, req_msg);
1722 primary_path->reversible = 1;
1723 primary_path->pkey =
1724 cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg));
1725 primary_path->sl = IBA_GET(CM_REQ_PRIMARY_SL, req_msg);
1726 primary_path->mtu_selector = IB_SA_EQ;
1727 primary_path->mtu = IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg);
1728 primary_path->rate_selector = IB_SA_EQ;
1729 primary_path->rate = IBA_GET(CM_REQ_PRIMARY_PACKET_RATE, req_msg);
1730 primary_path->packet_life_time_selector = IB_SA_EQ;
1731 primary_path->packet_life_time =
1732 IBA_GET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, req_msg);
1733 primary_path->packet_life_time -= (primary_path->packet_life_time > 0);
1734 primary_path->service_id =
1735 cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg));
1736 if (sa_path_is_roce(primary_path))
1737 primary_path->roce.route_resolved = false;
1738
1739 if (cm_req_has_alt_path(req_msg)) {
1740 alt_path->dgid = *IBA_GET_MEM_PTR(
1741 CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg);
1742 alt_path->sgid = *IBA_GET_MEM_PTR(
1743 CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg);
1744 alt_path->flow_label = cpu_to_be32(
1745 IBA_GET(CM_REQ_ALTERNATE_FLOW_LABEL, req_msg));
1746 alt_path->hop_limit =
1747 IBA_GET(CM_REQ_ALTERNATE_HOP_LIMIT, req_msg);
1748 alt_path->traffic_class =
1749 IBA_GET(CM_REQ_ALTERNATE_TRAFFIC_CLASS, req_msg);
1750 alt_path->reversible = 1;
1751 alt_path->pkey =
1752 cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg));
1753 alt_path->sl = IBA_GET(CM_REQ_ALTERNATE_SL, req_msg);
1754 alt_path->mtu_selector = IB_SA_EQ;
1755 alt_path->mtu =
1756 IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg);
1757 alt_path->rate_selector = IB_SA_EQ;
1758 alt_path->rate = IBA_GET(CM_REQ_ALTERNATE_PACKET_RATE, req_msg);
1759 alt_path->packet_life_time_selector = IB_SA_EQ;
1760 alt_path->packet_life_time =
1761 IBA_GET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, req_msg);
1762 alt_path->packet_life_time -= (alt_path->packet_life_time > 0);
1763 alt_path->service_id =
1764 cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg));
1765
1766 if (sa_path_is_roce(alt_path))
1767 alt_path->roce.route_resolved = false;
1768 }
1769 cm_format_path_lid_from_req(req_msg, primary_path, alt_path, wc);
1770 }
1771
cm_get_bth_pkey(struct cm_work * work)1772 static u16 cm_get_bth_pkey(struct cm_work *work)
1773 {
1774 struct ib_device *ib_dev = work->port->cm_dev->ib_device;
1775 u32 port_num = work->port->port_num;
1776 u16 pkey_index = work->mad_recv_wc->wc->pkey_index;
1777 u16 pkey;
1778 int ret;
1779
1780 ret = ib_get_cached_pkey(ib_dev, port_num, pkey_index, &pkey);
1781 if (ret) {
1782 dev_warn_ratelimited(&ib_dev->dev, "ib_cm: Couldn't retrieve pkey for incoming request (port %u, pkey index %u). %d\n",
1783 port_num, pkey_index, ret);
1784 return 0;
1785 }
1786
1787 return pkey;
1788 }
1789
1790 /**
1791 * cm_opa_to_ib_sgid - Convert OPA SGID to IB SGID
1792 * ULPs (such as IPoIB) do not understand OPA GIDs and will
1793 * reject them as the local_gid will not match the sgid. Therefore,
1794 * change the pathrec's SGID to an IB SGID.
1795 *
1796 * @work: Work completion
1797 * @path: Path record
1798 */
cm_opa_to_ib_sgid(struct cm_work * work,struct sa_path_rec * path)1799 static void cm_opa_to_ib_sgid(struct cm_work *work,
1800 struct sa_path_rec *path)
1801 {
1802 struct ib_device *dev = work->port->cm_dev->ib_device;
1803 u32 port_num = work->port->port_num;
1804
1805 if (rdma_cap_opa_ah(dev, port_num) &&
1806 (ib_is_opa_gid(&path->sgid))) {
1807 union ib_gid sgid;
1808
1809 if (rdma_query_gid(dev, port_num, 0, &sgid)) {
1810 dev_warn(&dev->dev,
1811 "Error updating sgid in CM request\n");
1812 return;
1813 }
1814
1815 path->sgid = sgid;
1816 }
1817 }
1818
cm_format_req_event(struct cm_work * work,struct cm_id_private * cm_id_priv,struct ib_cm_id * listen_id)1819 static void cm_format_req_event(struct cm_work *work,
1820 struct cm_id_private *cm_id_priv,
1821 struct ib_cm_id *listen_id)
1822 {
1823 struct cm_req_msg *req_msg;
1824 struct ib_cm_req_event_param *param;
1825
1826 req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
1827 param = &work->cm_event.param.req_rcvd;
1828 param->listen_id = listen_id;
1829 param->bth_pkey = cm_get_bth_pkey(work);
1830 param->port = cm_id_priv->av.port->port_num;
1831 param->primary_path = &work->path[0];
1832 cm_opa_to_ib_sgid(work, param->primary_path);
1833 if (cm_req_has_alt_path(req_msg)) {
1834 param->alternate_path = &work->path[1];
1835 cm_opa_to_ib_sgid(work, param->alternate_path);
1836 } else {
1837 param->alternate_path = NULL;
1838 }
1839 param->remote_ca_guid =
1840 cpu_to_be64(IBA_GET(CM_REQ_LOCAL_CA_GUID, req_msg));
1841 param->remote_qkey = IBA_GET(CM_REQ_LOCAL_Q_KEY, req_msg);
1842 param->remote_qpn = IBA_GET(CM_REQ_LOCAL_QPN, req_msg);
1843 param->qp_type = cm_req_get_qp_type(req_msg);
1844 param->starting_psn = IBA_GET(CM_REQ_STARTING_PSN, req_msg);
1845 param->responder_resources = IBA_GET(CM_REQ_INITIATOR_DEPTH, req_msg);
1846 param->initiator_depth = IBA_GET(CM_REQ_RESPONDER_RESOURCES, req_msg);
1847 param->local_cm_response_timeout =
1848 IBA_GET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, req_msg);
1849 param->flow_control = IBA_GET(CM_REQ_END_TO_END_FLOW_CONTROL, req_msg);
1850 param->remote_cm_response_timeout =
1851 IBA_GET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg);
1852 param->retry_count = IBA_GET(CM_REQ_RETRY_COUNT, req_msg);
1853 param->rnr_retry_count = IBA_GET(CM_REQ_RNR_RETRY_COUNT, req_msg);
1854 param->srq = IBA_GET(CM_REQ_SRQ, req_msg);
1855 param->ppath_sgid_attr = cm_id_priv->av.ah_attr.grh.sgid_attr;
1856 param->ece.vendor_id = IBA_GET(CM_REQ_VENDOR_ID, req_msg);
1857 param->ece.attr_mod = be32_to_cpu(req_msg->hdr.attr_mod);
1858
1859 work->cm_event.private_data =
1860 IBA_GET_MEM_PTR(CM_REQ_PRIVATE_DATA, req_msg);
1861 }
1862
cm_process_work(struct cm_id_private * cm_id_priv,struct cm_work * work)1863 static void cm_process_work(struct cm_id_private *cm_id_priv,
1864 struct cm_work *work)
1865 {
1866 int ret;
1867
1868 /* We will typically only have the current event to report. */
1869 ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &work->cm_event);
1870 cm_free_work(work);
1871
1872 while (!ret && !atomic_add_negative(-1, &cm_id_priv->work_count)) {
1873 spin_lock_irq(&cm_id_priv->lock);
1874 work = cm_dequeue_work(cm_id_priv);
1875 spin_unlock_irq(&cm_id_priv->lock);
1876 if (!work)
1877 return;
1878
1879 ret = cm_id_priv->id.cm_handler(&cm_id_priv->id,
1880 &work->cm_event);
1881 cm_free_work(work);
1882 }
1883 cm_deref_id(cm_id_priv);
1884 if (ret)
1885 cm_destroy_id(&cm_id_priv->id, ret);
1886 }
1887
cm_format_mra(struct cm_mra_msg * mra_msg,struct cm_id_private * cm_id_priv,enum cm_msg_response msg_mraed,const void * private_data,u8 private_data_len)1888 static void cm_format_mra(struct cm_mra_msg *mra_msg,
1889 struct cm_id_private *cm_id_priv,
1890 enum cm_msg_response msg_mraed,
1891 const void *private_data, u8 private_data_len)
1892 {
1893 cm_format_mad_hdr(&mra_msg->hdr, CM_MRA_ATTR_ID, cm_id_priv->tid);
1894 IBA_SET(CM_MRA_MESSAGE_MRAED, mra_msg, msg_mraed);
1895 IBA_SET(CM_MRA_LOCAL_COMM_ID, mra_msg,
1896 be32_to_cpu(cm_id_priv->id.local_id));
1897 IBA_SET(CM_MRA_REMOTE_COMM_ID, mra_msg,
1898 be32_to_cpu(cm_id_priv->id.remote_id));
1899 IBA_SET(CM_MRA_SERVICE_TIMEOUT, mra_msg, CM_MRA_SETTING);
1900
1901 if (private_data && private_data_len)
1902 IBA_SET_MEM(CM_MRA_PRIVATE_DATA, mra_msg, private_data,
1903 private_data_len);
1904 }
1905
cm_format_rej(struct cm_rej_msg * rej_msg,struct cm_id_private * cm_id_priv,enum ib_cm_rej_reason reason,void * ari,u8 ari_length,const void * private_data,u8 private_data_len,enum ib_cm_state state)1906 static void cm_format_rej(struct cm_rej_msg *rej_msg,
1907 struct cm_id_private *cm_id_priv,
1908 enum ib_cm_rej_reason reason, void *ari,
1909 u8 ari_length, const void *private_data,
1910 u8 private_data_len, enum ib_cm_state state)
1911 {
1912 lockdep_assert_held(&cm_id_priv->lock);
1913
1914 cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, cm_id_priv->tid);
1915 IBA_SET(CM_REJ_REMOTE_COMM_ID, rej_msg,
1916 be32_to_cpu(cm_id_priv->id.remote_id));
1917
1918 switch (state) {
1919 case IB_CM_REQ_RCVD:
1920 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg, be32_to_cpu(0));
1921 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REQ);
1922 break;
1923 case IB_CM_MRA_REQ_SENT:
1924 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1925 be32_to_cpu(cm_id_priv->id.local_id));
1926 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REQ);
1927 break;
1928 case IB_CM_REP_RCVD:
1929 case IB_CM_MRA_REP_SENT:
1930 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1931 be32_to_cpu(cm_id_priv->id.local_id));
1932 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REP);
1933 break;
1934 default:
1935 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1936 be32_to_cpu(cm_id_priv->id.local_id));
1937 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg,
1938 CM_MSG_RESPONSE_OTHER);
1939 break;
1940 }
1941
1942 IBA_SET(CM_REJ_REASON, rej_msg, reason);
1943 if (ari && ari_length) {
1944 IBA_SET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg, ari_length);
1945 IBA_SET_MEM(CM_REJ_ARI, rej_msg, ari, ari_length);
1946 }
1947
1948 if (private_data && private_data_len)
1949 IBA_SET_MEM(CM_REJ_PRIVATE_DATA, rej_msg, private_data,
1950 private_data_len);
1951 }
1952
cm_dup_req_handler(struct cm_work * work,struct cm_id_private * cm_id_priv)1953 static void cm_dup_req_handler(struct cm_work *work,
1954 struct cm_id_private *cm_id_priv)
1955 {
1956 struct ib_mad_send_buf *msg = NULL;
1957 int ret;
1958
1959 atomic_long_inc(
1960 &work->port->counters[CM_RECV_DUPLICATES][CM_REQ_COUNTER]);
1961
1962 /* Quick state check to discard duplicate REQs. */
1963 spin_lock_irq(&cm_id_priv->lock);
1964 if (cm_id_priv->id.state == IB_CM_REQ_RCVD) {
1965 spin_unlock_irq(&cm_id_priv->lock);
1966 return;
1967 }
1968 spin_unlock_irq(&cm_id_priv->lock);
1969
1970 ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, true, &msg);
1971 if (ret)
1972 return;
1973
1974 spin_lock_irq(&cm_id_priv->lock);
1975 switch (cm_id_priv->id.state) {
1976 case IB_CM_MRA_REQ_SENT:
1977 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
1978 CM_MSG_RESPONSE_REQ,
1979 cm_id_priv->private_data,
1980 cm_id_priv->private_data_len);
1981 break;
1982 case IB_CM_TIMEWAIT:
1983 cm_format_rej((struct cm_rej_msg *)msg->mad, cm_id_priv,
1984 IB_CM_REJ_STALE_CONN, NULL, 0, NULL, 0,
1985 IB_CM_TIMEWAIT);
1986 break;
1987 default:
1988 goto unlock;
1989 }
1990 spin_unlock_irq(&cm_id_priv->lock);
1991
1992 trace_icm_send_dup_req(&cm_id_priv->id);
1993 ret = ib_post_send_mad(msg, NULL);
1994 if (ret)
1995 goto free;
1996 return;
1997
1998 unlock: spin_unlock_irq(&cm_id_priv->lock);
1999 free: cm_free_msg(msg);
2000 }
2001
cm_match_req(struct cm_work * work,struct cm_id_private * cm_id_priv)2002 static struct cm_id_private *cm_match_req(struct cm_work *work,
2003 struct cm_id_private *cm_id_priv)
2004 {
2005 struct cm_id_private *listen_cm_id_priv, *cur_cm_id_priv;
2006 struct cm_timewait_info *timewait_info;
2007 struct cm_req_msg *req_msg;
2008
2009 req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
2010
2011 /* Check for possible duplicate REQ. */
2012 spin_lock_irq(&cm.lock);
2013 timewait_info = cm_insert_remote_id(cm_id_priv->timewait_info);
2014 if (timewait_info) {
2015 cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
2016 timewait_info->work.remote_id);
2017 spin_unlock_irq(&cm.lock);
2018 if (cur_cm_id_priv) {
2019 cm_dup_req_handler(work, cur_cm_id_priv);
2020 cm_deref_id(cur_cm_id_priv);
2021 }
2022 return NULL;
2023 }
2024
2025 /* Check for stale connections. */
2026 timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info);
2027 if (timewait_info) {
2028 cm_remove_remote(cm_id_priv);
2029 cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
2030 timewait_info->work.remote_id);
2031
2032 spin_unlock_irq(&cm.lock);
2033 cm_issue_rej(work->port, work->mad_recv_wc,
2034 IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REQ,
2035 NULL, 0);
2036 if (cur_cm_id_priv) {
2037 ib_send_cm_dreq(&cur_cm_id_priv->id, NULL, 0);
2038 cm_deref_id(cur_cm_id_priv);
2039 }
2040 return NULL;
2041 }
2042
2043 /* Find matching listen request. */
2044 listen_cm_id_priv = cm_find_listen(
2045 cm_id_priv->id.device,
2046 cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg)));
2047 if (!listen_cm_id_priv) {
2048 cm_remove_remote(cm_id_priv);
2049 spin_unlock_irq(&cm.lock);
2050 cm_issue_rej(work->port, work->mad_recv_wc,
2051 IB_CM_REJ_INVALID_SERVICE_ID, CM_MSG_RESPONSE_REQ,
2052 NULL, 0);
2053 return NULL;
2054 }
2055 spin_unlock_irq(&cm.lock);
2056 return listen_cm_id_priv;
2057 }
2058
2059 /*
2060 * Work-around for inter-subnet connections. If the LIDs are permissive,
2061 * we need to override the LID/SL data in the REQ with the LID information
2062 * in the work completion.
2063 */
cm_process_routed_req(struct cm_req_msg * req_msg,struct ib_wc * wc)2064 static void cm_process_routed_req(struct cm_req_msg *req_msg, struct ib_wc *wc)
2065 {
2066 if (!IBA_GET(CM_REQ_PRIMARY_SUBNET_LOCAL, req_msg)) {
2067 if (cpu_to_be16(IBA_GET(CM_REQ_PRIMARY_LOCAL_PORT_LID,
2068 req_msg)) == IB_LID_PERMISSIVE) {
2069 IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
2070 be16_to_cpu(ib_lid_be16(wc->slid)));
2071 IBA_SET(CM_REQ_PRIMARY_SL, req_msg, wc->sl);
2072 }
2073
2074 if (cpu_to_be16(IBA_GET(CM_REQ_PRIMARY_REMOTE_PORT_LID,
2075 req_msg)) == IB_LID_PERMISSIVE)
2076 IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg,
2077 wc->dlid_path_bits);
2078 }
2079
2080 if (!IBA_GET(CM_REQ_ALTERNATE_SUBNET_LOCAL, req_msg)) {
2081 if (cpu_to_be16(IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID,
2082 req_msg)) == IB_LID_PERMISSIVE) {
2083 IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg,
2084 be16_to_cpu(ib_lid_be16(wc->slid)));
2085 IBA_SET(CM_REQ_ALTERNATE_SL, req_msg, wc->sl);
2086 }
2087
2088 if (cpu_to_be16(IBA_GET(CM_REQ_ALTERNATE_REMOTE_PORT_LID,
2089 req_msg)) == IB_LID_PERMISSIVE)
2090 IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg,
2091 wc->dlid_path_bits);
2092 }
2093 }
2094
cm_req_handler(struct cm_work * work)2095 static int cm_req_handler(struct cm_work *work)
2096 {
2097 struct cm_id_private *cm_id_priv, *listen_cm_id_priv;
2098 struct cm_req_msg *req_msg;
2099 const struct ib_global_route *grh;
2100 const struct ib_gid_attr *gid_attr;
2101 int ret;
2102
2103 req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
2104
2105 cm_id_priv =
2106 cm_alloc_id_priv(work->port->cm_dev->ib_device, NULL, NULL);
2107 if (IS_ERR(cm_id_priv))
2108 return PTR_ERR(cm_id_priv);
2109
2110 cm_id_priv->id.remote_id =
2111 cpu_to_be32(IBA_GET(CM_REQ_LOCAL_COMM_ID, req_msg));
2112 cm_id_priv->id.service_id =
2113 cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg));
2114 cm_id_priv->tid = req_msg->hdr.tid;
2115 cm_id_priv->timeout_ms = cm_convert_to_ms(
2116 IBA_GET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg));
2117 cm_id_priv->max_cm_retries = IBA_GET(CM_REQ_MAX_CM_RETRIES, req_msg);
2118 cm_id_priv->remote_qpn =
2119 cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg));
2120 cm_id_priv->initiator_depth =
2121 IBA_GET(CM_REQ_RESPONDER_RESOURCES, req_msg);
2122 cm_id_priv->responder_resources =
2123 IBA_GET(CM_REQ_INITIATOR_DEPTH, req_msg);
2124 cm_id_priv->path_mtu = IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg);
2125 cm_id_priv->pkey = cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg));
2126 cm_id_priv->sq_psn = cpu_to_be32(IBA_GET(CM_REQ_STARTING_PSN, req_msg));
2127 cm_id_priv->retry_count = IBA_GET(CM_REQ_RETRY_COUNT, req_msg);
2128 cm_id_priv->rnr_retry_count = IBA_GET(CM_REQ_RNR_RETRY_COUNT, req_msg);
2129 cm_id_priv->qp_type = cm_req_get_qp_type(req_msg);
2130
2131 ret = cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
2132 work->mad_recv_wc->recv_buf.grh,
2133 &cm_id_priv->av);
2134 if (ret)
2135 goto destroy;
2136 cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv->
2137 id.local_id);
2138 if (IS_ERR(cm_id_priv->timewait_info)) {
2139 ret = PTR_ERR(cm_id_priv->timewait_info);
2140 cm_id_priv->timewait_info = NULL;
2141 goto destroy;
2142 }
2143 cm_id_priv->timewait_info->work.remote_id = cm_id_priv->id.remote_id;
2144 cm_id_priv->timewait_info->remote_ca_guid =
2145 cpu_to_be64(IBA_GET(CM_REQ_LOCAL_CA_GUID, req_msg));
2146 cm_id_priv->timewait_info->remote_qpn = cm_id_priv->remote_qpn;
2147
2148 /*
2149 * Note that the ID pointer is not in the xarray at this point,
2150 * so this set is only visible to the local thread.
2151 */
2152 cm_id_priv->id.state = IB_CM_REQ_RCVD;
2153
2154 listen_cm_id_priv = cm_match_req(work, cm_id_priv);
2155 if (!listen_cm_id_priv) {
2156 trace_icm_no_listener_err(&cm_id_priv->id);
2157 cm_id_priv->id.state = IB_CM_IDLE;
2158 ret = -EINVAL;
2159 goto destroy;
2160 }
2161
2162 memset(&work->path[0], 0, sizeof(work->path[0]));
2163 if (cm_req_has_alt_path(req_msg))
2164 memset(&work->path[1], 0, sizeof(work->path[1]));
2165 grh = rdma_ah_read_grh(&cm_id_priv->av.ah_attr);
2166 gid_attr = grh->sgid_attr;
2167
2168 if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE) {
2169 work->path[0].rec_type =
2170 sa_conv_gid_to_pathrec_type(gid_attr->gid_type);
2171 } else {
2172 cm_process_routed_req(req_msg, work->mad_recv_wc->wc);
2173 cm_path_set_rec_type(
2174 work->port->cm_dev->ib_device, work->port->port_num,
2175 &work->path[0],
2176 IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID,
2177 req_msg));
2178 }
2179 if (cm_req_has_alt_path(req_msg))
2180 work->path[1].rec_type = work->path[0].rec_type;
2181 cm_format_paths_from_req(req_msg, &work->path[0],
2182 &work->path[1], work->mad_recv_wc->wc);
2183 if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE)
2184 sa_path_set_dmac(&work->path[0],
2185 cm_id_priv->av.ah_attr.roce.dmac);
2186 work->path[0].hop_limit = grh->hop_limit;
2187
2188 /* This destroy call is needed to pair with cm_init_av_for_response */
2189 cm_destroy_av(&cm_id_priv->av);
2190 ret = cm_init_av_by_path(&work->path[0], gid_attr, &cm_id_priv->av);
2191 if (ret) {
2192 int err;
2193
2194 err = rdma_query_gid(work->port->cm_dev->ib_device,
2195 work->port->port_num, 0,
2196 &work->path[0].sgid);
2197 if (err)
2198 ib_send_cm_rej(&cm_id_priv->id, IB_CM_REJ_INVALID_GID,
2199 NULL, 0, NULL, 0);
2200 else
2201 ib_send_cm_rej(&cm_id_priv->id, IB_CM_REJ_INVALID_GID,
2202 &work->path[0].sgid,
2203 sizeof(work->path[0].sgid),
2204 NULL, 0);
2205 goto rejected;
2206 }
2207 if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_IB)
2208 cm_id_priv->av.dlid_datapath =
2209 IBA_GET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg);
2210
2211 if (cm_req_has_alt_path(req_msg)) {
2212 ret = cm_init_av_by_path(&work->path[1], NULL,
2213 &cm_id_priv->alt_av);
2214 if (ret) {
2215 ib_send_cm_rej(&cm_id_priv->id,
2216 IB_CM_REJ_INVALID_ALT_GID,
2217 &work->path[0].sgid,
2218 sizeof(work->path[0].sgid), NULL, 0);
2219 goto rejected;
2220 }
2221 }
2222
2223 cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler;
2224 cm_id_priv->id.context = listen_cm_id_priv->id.context;
2225 cm_format_req_event(work, cm_id_priv, &listen_cm_id_priv->id);
2226
2227 /* Now MAD handlers can see the new ID */
2228 spin_lock_irq(&cm_id_priv->lock);
2229 cm_finalize_id(cm_id_priv);
2230
2231 /* Refcount belongs to the event, pairs with cm_process_work() */
2232 refcount_inc(&cm_id_priv->refcount);
2233 cm_queue_work_unlock(cm_id_priv, work);
2234 /*
2235 * Since this ID was just created and was not made visible to other MAD
2236 * handlers until the cm_finalize_id() above we know that the
2237 * cm_process_work() will deliver the event and the listen_cm_id
2238 * embedded in the event can be derefed here.
2239 */
2240 cm_deref_id(listen_cm_id_priv);
2241 return 0;
2242
2243 rejected:
2244 cm_deref_id(listen_cm_id_priv);
2245 destroy:
2246 ib_destroy_cm_id(&cm_id_priv->id);
2247 return ret;
2248 }
2249
cm_format_rep(struct cm_rep_msg * rep_msg,struct cm_id_private * cm_id_priv,struct ib_cm_rep_param * param)2250 static void cm_format_rep(struct cm_rep_msg *rep_msg,
2251 struct cm_id_private *cm_id_priv,
2252 struct ib_cm_rep_param *param)
2253 {
2254 cm_format_mad_ece_hdr(&rep_msg->hdr, CM_REP_ATTR_ID, cm_id_priv->tid,
2255 param->ece.attr_mod);
2256 IBA_SET(CM_REP_LOCAL_COMM_ID, rep_msg,
2257 be32_to_cpu(cm_id_priv->id.local_id));
2258 IBA_SET(CM_REP_REMOTE_COMM_ID, rep_msg,
2259 be32_to_cpu(cm_id_priv->id.remote_id));
2260 IBA_SET(CM_REP_STARTING_PSN, rep_msg, param->starting_psn);
2261 IBA_SET(CM_REP_RESPONDER_RESOURCES, rep_msg,
2262 param->responder_resources);
2263 IBA_SET(CM_REP_TARGET_ACK_DELAY, rep_msg,
2264 cm_id_priv->av.port->cm_dev->ack_delay);
2265 IBA_SET(CM_REP_FAILOVER_ACCEPTED, rep_msg, param->failover_accepted);
2266 IBA_SET(CM_REP_RNR_RETRY_COUNT, rep_msg, param->rnr_retry_count);
2267 IBA_SET(CM_REP_LOCAL_CA_GUID, rep_msg,
2268 be64_to_cpu(cm_id_priv->id.device->node_guid));
2269
2270 if (cm_id_priv->qp_type != IB_QPT_XRC_TGT) {
2271 IBA_SET(CM_REP_INITIATOR_DEPTH, rep_msg,
2272 param->initiator_depth);
2273 IBA_SET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg,
2274 param->flow_control);
2275 IBA_SET(CM_REP_SRQ, rep_msg, param->srq);
2276 IBA_SET(CM_REP_LOCAL_QPN, rep_msg, param->qp_num);
2277 } else {
2278 IBA_SET(CM_REP_SRQ, rep_msg, 1);
2279 IBA_SET(CM_REP_LOCAL_EE_CONTEXT_NUMBER, rep_msg, param->qp_num);
2280 }
2281
2282 IBA_SET(CM_REP_VENDOR_ID_L, rep_msg, param->ece.vendor_id);
2283 IBA_SET(CM_REP_VENDOR_ID_M, rep_msg, param->ece.vendor_id >> 8);
2284 IBA_SET(CM_REP_VENDOR_ID_H, rep_msg, param->ece.vendor_id >> 16);
2285
2286 if (param->private_data && param->private_data_len)
2287 IBA_SET_MEM(CM_REP_PRIVATE_DATA, rep_msg, param->private_data,
2288 param->private_data_len);
2289 }
2290
ib_send_cm_rep(struct ib_cm_id * cm_id,struct ib_cm_rep_param * param)2291 int ib_send_cm_rep(struct ib_cm_id *cm_id,
2292 struct ib_cm_rep_param *param)
2293 {
2294 struct cm_id_private *cm_id_priv;
2295 struct ib_mad_send_buf *msg;
2296 struct cm_rep_msg *rep_msg;
2297 unsigned long flags;
2298 int ret;
2299
2300 if (param->private_data &&
2301 param->private_data_len > IB_CM_REP_PRIVATE_DATA_SIZE)
2302 return -EINVAL;
2303
2304 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2305 spin_lock_irqsave(&cm_id_priv->lock, flags);
2306 if (cm_id->state != IB_CM_REQ_RCVD &&
2307 cm_id->state != IB_CM_MRA_REQ_SENT) {
2308 trace_icm_send_rep_err(cm_id_priv->id.local_id, cm_id->state);
2309 ret = -EINVAL;
2310 goto out;
2311 }
2312
2313 msg = cm_alloc_priv_msg_rep(cm_id_priv, IB_CM_REP_SENT, true);
2314 if (IS_ERR(msg)) {
2315 ret = PTR_ERR(msg);
2316 goto out;
2317 }
2318
2319 rep_msg = (struct cm_rep_msg *) msg->mad;
2320 cm_format_rep(rep_msg, cm_id_priv, param);
2321
2322 trace_icm_send_rep(cm_id);
2323 ret = ib_post_send_mad(msg, NULL);
2324 if (ret)
2325 goto out_free;
2326
2327 cm_id->state = IB_CM_REP_SENT;
2328 cm_id_priv->initiator_depth = param->initiator_depth;
2329 cm_id_priv->responder_resources = param->responder_resources;
2330 cm_id_priv->rq_psn = cpu_to_be32(IBA_GET(CM_REP_STARTING_PSN, rep_msg));
2331 WARN_ONCE(param->qp_num & 0xFF000000,
2332 "IBTA declares QPN to be 24 bits, but it is 0x%X\n",
2333 param->qp_num);
2334 cm_id_priv->local_qpn = cpu_to_be32(param->qp_num & 0xFFFFFF);
2335 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2336 return 0;
2337
2338 out_free:
2339 cm_free_priv_msg(msg);
2340 out:
2341 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2342 return ret;
2343 }
2344 EXPORT_SYMBOL(ib_send_cm_rep);
2345
cm_format_rtu(struct cm_rtu_msg * rtu_msg,struct cm_id_private * cm_id_priv,const void * private_data,u8 private_data_len)2346 static void cm_format_rtu(struct cm_rtu_msg *rtu_msg,
2347 struct cm_id_private *cm_id_priv,
2348 const void *private_data,
2349 u8 private_data_len)
2350 {
2351 cm_format_mad_hdr(&rtu_msg->hdr, CM_RTU_ATTR_ID, cm_id_priv->tid);
2352 IBA_SET(CM_RTU_LOCAL_COMM_ID, rtu_msg,
2353 be32_to_cpu(cm_id_priv->id.local_id));
2354 IBA_SET(CM_RTU_REMOTE_COMM_ID, rtu_msg,
2355 be32_to_cpu(cm_id_priv->id.remote_id));
2356
2357 if (private_data && private_data_len)
2358 IBA_SET_MEM(CM_RTU_PRIVATE_DATA, rtu_msg, private_data,
2359 private_data_len);
2360 }
2361
ib_send_cm_rtu(struct ib_cm_id * cm_id,const void * private_data,u8 private_data_len)2362 int ib_send_cm_rtu(struct ib_cm_id *cm_id,
2363 const void *private_data,
2364 u8 private_data_len)
2365 {
2366 struct cm_id_private *cm_id_priv;
2367 struct ib_mad_send_buf *msg;
2368 unsigned long flags;
2369 void *data;
2370 int ret;
2371
2372 if (private_data && private_data_len > IB_CM_RTU_PRIVATE_DATA_SIZE)
2373 return -EINVAL;
2374
2375 data = cm_copy_private_data(private_data, private_data_len);
2376 if (IS_ERR(data))
2377 return PTR_ERR(data);
2378
2379 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2380 spin_lock_irqsave(&cm_id_priv->lock, flags);
2381 if (cm_id->state != IB_CM_REP_RCVD &&
2382 cm_id->state != IB_CM_MRA_REP_SENT) {
2383 trace_icm_send_cm_rtu_err(cm_id);
2384 ret = -EINVAL;
2385 goto error;
2386 }
2387
2388 msg = cm_alloc_msg(cm_id_priv);
2389 if (IS_ERR(msg)) {
2390 ret = PTR_ERR(msg);
2391 goto error;
2392 }
2393
2394 cm_format_rtu((struct cm_rtu_msg *) msg->mad, cm_id_priv,
2395 private_data, private_data_len);
2396
2397 trace_icm_send_rtu(cm_id);
2398 ret = ib_post_send_mad(msg, NULL);
2399 if (ret) {
2400 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2401 cm_free_msg(msg);
2402 kfree(data);
2403 return ret;
2404 }
2405
2406 cm_id->state = IB_CM_ESTABLISHED;
2407 cm_set_private_data(cm_id_priv, data, private_data_len);
2408 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2409 return 0;
2410
2411 error: spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2412 kfree(data);
2413 return ret;
2414 }
2415 EXPORT_SYMBOL(ib_send_cm_rtu);
2416
cm_format_rep_event(struct cm_work * work,enum ib_qp_type qp_type)2417 static void cm_format_rep_event(struct cm_work *work, enum ib_qp_type qp_type)
2418 {
2419 struct cm_rep_msg *rep_msg;
2420 struct ib_cm_rep_event_param *param;
2421
2422 rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad;
2423 param = &work->cm_event.param.rep_rcvd;
2424 param->remote_ca_guid =
2425 cpu_to_be64(IBA_GET(CM_REP_LOCAL_CA_GUID, rep_msg));
2426 param->remote_qkey = IBA_GET(CM_REP_LOCAL_Q_KEY, rep_msg);
2427 param->remote_qpn = be32_to_cpu(cm_rep_get_qpn(rep_msg, qp_type));
2428 param->starting_psn = IBA_GET(CM_REP_STARTING_PSN, rep_msg);
2429 param->responder_resources = IBA_GET(CM_REP_INITIATOR_DEPTH, rep_msg);
2430 param->initiator_depth = IBA_GET(CM_REP_RESPONDER_RESOURCES, rep_msg);
2431 param->target_ack_delay = IBA_GET(CM_REP_TARGET_ACK_DELAY, rep_msg);
2432 param->failover_accepted = IBA_GET(CM_REP_FAILOVER_ACCEPTED, rep_msg);
2433 param->flow_control = IBA_GET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg);
2434 param->rnr_retry_count = IBA_GET(CM_REP_RNR_RETRY_COUNT, rep_msg);
2435 param->srq = IBA_GET(CM_REP_SRQ, rep_msg);
2436 param->ece.vendor_id = IBA_GET(CM_REP_VENDOR_ID_H, rep_msg) << 16;
2437 param->ece.vendor_id |= IBA_GET(CM_REP_VENDOR_ID_M, rep_msg) << 8;
2438 param->ece.vendor_id |= IBA_GET(CM_REP_VENDOR_ID_L, rep_msg);
2439 param->ece.attr_mod = be32_to_cpu(rep_msg->hdr.attr_mod);
2440
2441 work->cm_event.private_data =
2442 IBA_GET_MEM_PTR(CM_REP_PRIVATE_DATA, rep_msg);
2443 }
2444
cm_dup_rep_handler(struct cm_work * work)2445 static void cm_dup_rep_handler(struct cm_work *work)
2446 {
2447 struct cm_id_private *cm_id_priv;
2448 struct cm_rep_msg *rep_msg;
2449 struct ib_mad_send_buf *msg = NULL;
2450 int ret;
2451
2452 rep_msg = (struct cm_rep_msg *) work->mad_recv_wc->recv_buf.mad;
2453 cm_id_priv = cm_acquire_id(
2454 cpu_to_be32(IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)),
2455 cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg)));
2456 if (!cm_id_priv)
2457 return;
2458
2459 atomic_long_inc(
2460 &work->port->counters[CM_RECV_DUPLICATES][CM_REP_COUNTER]);
2461 ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, true, &msg);
2462 if (ret)
2463 goto deref;
2464
2465 spin_lock_irq(&cm_id_priv->lock);
2466 if (cm_id_priv->id.state == IB_CM_ESTABLISHED)
2467 cm_format_rtu((struct cm_rtu_msg *) msg->mad, cm_id_priv,
2468 cm_id_priv->private_data,
2469 cm_id_priv->private_data_len);
2470 else if (cm_id_priv->id.state == IB_CM_MRA_REP_SENT)
2471 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
2472 CM_MSG_RESPONSE_REP,
2473 cm_id_priv->private_data,
2474 cm_id_priv->private_data_len);
2475 else
2476 goto unlock;
2477 spin_unlock_irq(&cm_id_priv->lock);
2478
2479 trace_icm_send_dup_rep(&cm_id_priv->id);
2480 ret = ib_post_send_mad(msg, NULL);
2481 if (ret)
2482 goto free;
2483 goto deref;
2484
2485 unlock: spin_unlock_irq(&cm_id_priv->lock);
2486 free: cm_free_msg(msg);
2487 deref: cm_deref_id(cm_id_priv);
2488 }
2489
cm_rep_handler(struct cm_work * work)2490 static int cm_rep_handler(struct cm_work *work)
2491 {
2492 struct cm_id_private *cm_id_priv;
2493 struct cm_rep_msg *rep_msg;
2494 int ret;
2495 struct cm_id_private *cur_cm_id_priv;
2496 struct cm_timewait_info *timewait_info;
2497
2498 rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad;
2499 cm_id_priv = cm_acquire_id(
2500 cpu_to_be32(IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)), 0);
2501 if (!cm_id_priv) {
2502 cm_dup_rep_handler(work);
2503 trace_icm_remote_no_priv_err(
2504 IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
2505 return -EINVAL;
2506 }
2507
2508 cm_format_rep_event(work, cm_id_priv->qp_type);
2509
2510 spin_lock_irq(&cm_id_priv->lock);
2511 switch (cm_id_priv->id.state) {
2512 case IB_CM_REQ_SENT:
2513 case IB_CM_MRA_REQ_RCVD:
2514 break;
2515 default:
2516 ret = -EINVAL;
2517 trace_icm_rep_unknown_err(
2518 IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg),
2519 IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg),
2520 cm_id_priv->id.state);
2521 spin_unlock_irq(&cm_id_priv->lock);
2522 goto error;
2523 }
2524
2525 cm_id_priv->timewait_info->work.remote_id =
2526 cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg));
2527 cm_id_priv->timewait_info->remote_ca_guid =
2528 cpu_to_be64(IBA_GET(CM_REP_LOCAL_CA_GUID, rep_msg));
2529 cm_id_priv->timewait_info->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type);
2530
2531 spin_lock(&cm.lock);
2532 /* Check for duplicate REP. */
2533 if (cm_insert_remote_id(cm_id_priv->timewait_info)) {
2534 spin_unlock(&cm.lock);
2535 spin_unlock_irq(&cm_id_priv->lock);
2536 ret = -EINVAL;
2537 trace_icm_insert_failed_err(
2538 IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
2539 goto error;
2540 }
2541 /* Check for a stale connection. */
2542 timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info);
2543 if (timewait_info) {
2544 cm_remove_remote(cm_id_priv);
2545 cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
2546 timewait_info->work.remote_id);
2547
2548 spin_unlock(&cm.lock);
2549 spin_unlock_irq(&cm_id_priv->lock);
2550 cm_issue_rej(work->port, work->mad_recv_wc,
2551 IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REP,
2552 NULL, 0);
2553 ret = -EINVAL;
2554 trace_icm_staleconn_err(
2555 IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg),
2556 IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
2557
2558 if (cur_cm_id_priv) {
2559 ib_send_cm_dreq(&cur_cm_id_priv->id, NULL, 0);
2560 cm_deref_id(cur_cm_id_priv);
2561 }
2562
2563 goto error;
2564 }
2565 spin_unlock(&cm.lock);
2566
2567 cm_id_priv->id.state = IB_CM_REP_RCVD;
2568 cm_id_priv->id.remote_id =
2569 cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg));
2570 cm_id_priv->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type);
2571 cm_id_priv->initiator_depth =
2572 IBA_GET(CM_REP_RESPONDER_RESOURCES, rep_msg);
2573 cm_id_priv->responder_resources =
2574 IBA_GET(CM_REP_INITIATOR_DEPTH, rep_msg);
2575 cm_id_priv->sq_psn = cpu_to_be32(IBA_GET(CM_REP_STARTING_PSN, rep_msg));
2576 cm_id_priv->rnr_retry_count = IBA_GET(CM_REP_RNR_RETRY_COUNT, rep_msg);
2577 cm_id_priv->target_ack_delay =
2578 IBA_GET(CM_REP_TARGET_ACK_DELAY, rep_msg);
2579 cm_id_priv->av.timeout =
2580 cm_ack_timeout(cm_id_priv->target_ack_delay,
2581 cm_id_priv->av.timeout - 1);
2582 cm_id_priv->alt_av.timeout =
2583 cm_ack_timeout(cm_id_priv->target_ack_delay,
2584 cm_id_priv->alt_av.timeout - 1);
2585
2586 ib_cancel_mad(cm_id_priv->msg);
2587 cm_queue_work_unlock(cm_id_priv, work);
2588 return 0;
2589
2590 error:
2591 cm_deref_id(cm_id_priv);
2592 return ret;
2593 }
2594
cm_establish_handler(struct cm_work * work)2595 static int cm_establish_handler(struct cm_work *work)
2596 {
2597 struct cm_id_private *cm_id_priv;
2598
2599 /* See comment in cm_establish about lookup. */
2600 cm_id_priv = cm_acquire_id(work->local_id, work->remote_id);
2601 if (!cm_id_priv)
2602 return -EINVAL;
2603
2604 spin_lock_irq(&cm_id_priv->lock);
2605 if (cm_id_priv->id.state != IB_CM_ESTABLISHED) {
2606 spin_unlock_irq(&cm_id_priv->lock);
2607 goto out;
2608 }
2609
2610 ib_cancel_mad(cm_id_priv->msg);
2611 cm_queue_work_unlock(cm_id_priv, work);
2612 return 0;
2613 out:
2614 cm_deref_id(cm_id_priv);
2615 return -EINVAL;
2616 }
2617
cm_rtu_handler(struct cm_work * work)2618 static int cm_rtu_handler(struct cm_work *work)
2619 {
2620 struct cm_id_private *cm_id_priv;
2621 struct cm_rtu_msg *rtu_msg;
2622
2623 rtu_msg = (struct cm_rtu_msg *)work->mad_recv_wc->recv_buf.mad;
2624 cm_id_priv = cm_acquire_id(
2625 cpu_to_be32(IBA_GET(CM_RTU_REMOTE_COMM_ID, rtu_msg)),
2626 cpu_to_be32(IBA_GET(CM_RTU_LOCAL_COMM_ID, rtu_msg)));
2627 if (!cm_id_priv)
2628 return -EINVAL;
2629
2630 work->cm_event.private_data =
2631 IBA_GET_MEM_PTR(CM_RTU_PRIVATE_DATA, rtu_msg);
2632
2633 spin_lock_irq(&cm_id_priv->lock);
2634 if (cm_id_priv->id.state != IB_CM_REP_SENT &&
2635 cm_id_priv->id.state != IB_CM_MRA_REP_RCVD) {
2636 spin_unlock_irq(&cm_id_priv->lock);
2637 atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
2638 [CM_RTU_COUNTER]);
2639 goto out;
2640 }
2641 cm_id_priv->id.state = IB_CM_ESTABLISHED;
2642
2643 ib_cancel_mad(cm_id_priv->msg);
2644 cm_queue_work_unlock(cm_id_priv, work);
2645 return 0;
2646 out:
2647 cm_deref_id(cm_id_priv);
2648 return -EINVAL;
2649 }
2650
cm_format_dreq(struct cm_dreq_msg * dreq_msg,struct cm_id_private * cm_id_priv,const void * private_data,u8 private_data_len)2651 static void cm_format_dreq(struct cm_dreq_msg *dreq_msg,
2652 struct cm_id_private *cm_id_priv,
2653 const void *private_data,
2654 u8 private_data_len)
2655 {
2656 cm_format_mad_hdr(&dreq_msg->hdr, CM_DREQ_ATTR_ID,
2657 cm_form_tid(cm_id_priv));
2658 IBA_SET(CM_DREQ_LOCAL_COMM_ID, dreq_msg,
2659 be32_to_cpu(cm_id_priv->id.local_id));
2660 IBA_SET(CM_DREQ_REMOTE_COMM_ID, dreq_msg,
2661 be32_to_cpu(cm_id_priv->id.remote_id));
2662 IBA_SET(CM_DREQ_REMOTE_QPN_EECN, dreq_msg,
2663 be32_to_cpu(cm_id_priv->remote_qpn));
2664
2665 if (private_data && private_data_len)
2666 IBA_SET_MEM(CM_DREQ_PRIVATE_DATA, dreq_msg, private_data,
2667 private_data_len);
2668 }
2669
cm_issue_dreq(struct cm_id_private * cm_id_priv)2670 static void cm_issue_dreq(struct cm_id_private *cm_id_priv)
2671 {
2672 struct ib_mad_send_buf *msg;
2673 int ret;
2674
2675 lockdep_assert_held(&cm_id_priv->lock);
2676
2677 msg = cm_alloc_msg(cm_id_priv);
2678 if (IS_ERR(msg))
2679 return;
2680
2681 cm_format_dreq((struct cm_dreq_msg *) msg->mad, cm_id_priv, NULL, 0);
2682
2683 trace_icm_send_dreq(&cm_id_priv->id);
2684 ret = ib_post_send_mad(msg, NULL);
2685 if (ret)
2686 cm_free_msg(msg);
2687 }
2688
ib_send_cm_dreq(struct ib_cm_id * cm_id,const void * private_data,u8 private_data_len)2689 int ib_send_cm_dreq(struct ib_cm_id *cm_id, const void *private_data,
2690 u8 private_data_len)
2691 {
2692 struct cm_id_private *cm_id_priv =
2693 container_of(cm_id, struct cm_id_private, id);
2694 struct ib_mad_send_buf *msg;
2695 unsigned long flags;
2696 int ret;
2697
2698 if (private_data && private_data_len > IB_CM_DREQ_PRIVATE_DATA_SIZE)
2699 return -EINVAL;
2700
2701 spin_lock_irqsave(&cm_id_priv->lock, flags);
2702 if (cm_id_priv->id.state != IB_CM_ESTABLISHED) {
2703 trace_icm_dreq_skipped(&cm_id_priv->id);
2704 ret = -EINVAL;
2705 goto unlock;
2706 }
2707
2708 if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT ||
2709 cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
2710 ib_cancel_mad(cm_id_priv->msg);
2711
2712 msg = cm_alloc_priv_msg(cm_id_priv, IB_CM_DREQ_SENT);
2713 if (IS_ERR(msg)) {
2714 cm_enter_timewait(cm_id_priv);
2715 ret = PTR_ERR(msg);
2716 goto unlock;
2717 }
2718
2719 cm_format_dreq((struct cm_dreq_msg *) msg->mad, cm_id_priv,
2720 private_data, private_data_len);
2721
2722 trace_icm_send_dreq(&cm_id_priv->id);
2723 ret = ib_post_send_mad(msg, NULL);
2724 if (ret) {
2725 cm_enter_timewait(cm_id_priv);
2726 cm_free_priv_msg(msg);
2727 goto unlock;
2728 }
2729
2730 cm_id_priv->id.state = IB_CM_DREQ_SENT;
2731 unlock:
2732 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2733 return ret;
2734 }
2735 EXPORT_SYMBOL(ib_send_cm_dreq);
2736
cm_format_drep(struct cm_drep_msg * drep_msg,struct cm_id_private * cm_id_priv,const void * private_data,u8 private_data_len)2737 static void cm_format_drep(struct cm_drep_msg *drep_msg,
2738 struct cm_id_private *cm_id_priv,
2739 const void *private_data,
2740 u8 private_data_len)
2741 {
2742 cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, cm_id_priv->tid);
2743 IBA_SET(CM_DREP_LOCAL_COMM_ID, drep_msg,
2744 be32_to_cpu(cm_id_priv->id.local_id));
2745 IBA_SET(CM_DREP_REMOTE_COMM_ID, drep_msg,
2746 be32_to_cpu(cm_id_priv->id.remote_id));
2747
2748 if (private_data && private_data_len)
2749 IBA_SET_MEM(CM_DREP_PRIVATE_DATA, drep_msg, private_data,
2750 private_data_len);
2751 }
2752
cm_send_drep_locked(struct cm_id_private * cm_id_priv,void * private_data,u8 private_data_len)2753 static int cm_send_drep_locked(struct cm_id_private *cm_id_priv,
2754 void *private_data, u8 private_data_len)
2755 {
2756 struct ib_mad_send_buf *msg;
2757 int ret;
2758
2759 lockdep_assert_held(&cm_id_priv->lock);
2760
2761 if (private_data && private_data_len > IB_CM_DREP_PRIVATE_DATA_SIZE)
2762 return -EINVAL;
2763
2764 if (cm_id_priv->id.state != IB_CM_DREQ_RCVD) {
2765 trace_icm_send_drep_err(&cm_id_priv->id);
2766 kfree(private_data);
2767 return -EINVAL;
2768 }
2769
2770 cm_set_private_data(cm_id_priv, private_data, private_data_len);
2771 cm_enter_timewait(cm_id_priv);
2772
2773 msg = cm_alloc_msg(cm_id_priv);
2774 if (IS_ERR(msg))
2775 return PTR_ERR(msg);
2776
2777 cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv,
2778 private_data, private_data_len);
2779
2780 trace_icm_send_drep(&cm_id_priv->id);
2781 ret = ib_post_send_mad(msg, NULL);
2782 if (ret) {
2783 cm_free_msg(msg);
2784 return ret;
2785 }
2786 return 0;
2787 }
2788
ib_send_cm_drep(struct ib_cm_id * cm_id,const void * private_data,u8 private_data_len)2789 int ib_send_cm_drep(struct ib_cm_id *cm_id, const void *private_data,
2790 u8 private_data_len)
2791 {
2792 struct cm_id_private *cm_id_priv =
2793 container_of(cm_id, struct cm_id_private, id);
2794 unsigned long flags;
2795 void *data;
2796 int ret;
2797
2798 data = cm_copy_private_data(private_data, private_data_len);
2799 if (IS_ERR(data))
2800 return PTR_ERR(data);
2801
2802 spin_lock_irqsave(&cm_id_priv->lock, flags);
2803 ret = cm_send_drep_locked(cm_id_priv, data, private_data_len);
2804 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2805 return ret;
2806 }
2807 EXPORT_SYMBOL(ib_send_cm_drep);
2808
cm_issue_drep(struct cm_port * port,struct ib_mad_recv_wc * mad_recv_wc)2809 static int cm_issue_drep(struct cm_port *port,
2810 struct ib_mad_recv_wc *mad_recv_wc)
2811 {
2812 struct ib_mad_send_buf *msg = NULL;
2813 struct cm_dreq_msg *dreq_msg;
2814 struct cm_drep_msg *drep_msg;
2815 int ret;
2816
2817 ret = cm_alloc_response_msg(port, mad_recv_wc, true, &msg);
2818 if (ret)
2819 return ret;
2820
2821 dreq_msg = (struct cm_dreq_msg *) mad_recv_wc->recv_buf.mad;
2822 drep_msg = (struct cm_drep_msg *) msg->mad;
2823
2824 cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, dreq_msg->hdr.tid);
2825 IBA_SET(CM_DREP_REMOTE_COMM_ID, drep_msg,
2826 IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg));
2827 IBA_SET(CM_DREP_LOCAL_COMM_ID, drep_msg,
2828 IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg));
2829
2830 trace_icm_issue_drep(
2831 IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg),
2832 IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg));
2833 ret = ib_post_send_mad(msg, NULL);
2834 if (ret)
2835 cm_free_msg(msg);
2836
2837 return ret;
2838 }
2839
cm_dreq_handler(struct cm_work * work)2840 static int cm_dreq_handler(struct cm_work *work)
2841 {
2842 struct cm_id_private *cm_id_priv;
2843 struct cm_dreq_msg *dreq_msg;
2844 struct ib_mad_send_buf *msg = NULL;
2845
2846 dreq_msg = (struct cm_dreq_msg *)work->mad_recv_wc->recv_buf.mad;
2847 cm_id_priv = cm_acquire_id(
2848 cpu_to_be32(IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg)),
2849 cpu_to_be32(IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg)));
2850 if (!cm_id_priv) {
2851 atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
2852 [CM_DREQ_COUNTER]);
2853 cm_issue_drep(work->port, work->mad_recv_wc);
2854 trace_icm_no_priv_err(
2855 IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg),
2856 IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg));
2857 return -EINVAL;
2858 }
2859
2860 work->cm_event.private_data =
2861 IBA_GET_MEM_PTR(CM_DREQ_PRIVATE_DATA, dreq_msg);
2862
2863 spin_lock_irq(&cm_id_priv->lock);
2864 if (cm_id_priv->local_qpn !=
2865 cpu_to_be32(IBA_GET(CM_DREQ_REMOTE_QPN_EECN, dreq_msg)))
2866 goto unlock;
2867
2868 switch (cm_id_priv->id.state) {
2869 case IB_CM_REP_SENT:
2870 case IB_CM_DREQ_SENT:
2871 case IB_CM_MRA_REP_RCVD:
2872 ib_cancel_mad(cm_id_priv->msg);
2873 break;
2874 case IB_CM_ESTABLISHED:
2875 if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT ||
2876 cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
2877 ib_cancel_mad(cm_id_priv->msg);
2878 break;
2879 case IB_CM_TIMEWAIT:
2880 atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
2881 [CM_DREQ_COUNTER]);
2882 msg = cm_alloc_response_msg_no_ah(work->port, work->mad_recv_wc,
2883 true);
2884 if (IS_ERR(msg))
2885 goto unlock;
2886
2887 cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv,
2888 cm_id_priv->private_data,
2889 cm_id_priv->private_data_len);
2890 spin_unlock_irq(&cm_id_priv->lock);
2891
2892 if (cm_create_response_msg_ah(work->port, work->mad_recv_wc, msg) ||
2893 ib_post_send_mad(msg, NULL))
2894 cm_free_msg(msg);
2895 goto deref;
2896 case IB_CM_DREQ_RCVD:
2897 atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
2898 [CM_DREQ_COUNTER]);
2899 goto unlock;
2900 default:
2901 trace_icm_dreq_unknown_err(&cm_id_priv->id);
2902 goto unlock;
2903 }
2904 cm_id_priv->id.state = IB_CM_DREQ_RCVD;
2905 cm_id_priv->tid = dreq_msg->hdr.tid;
2906 cm_queue_work_unlock(cm_id_priv, work);
2907 return 0;
2908
2909 unlock: spin_unlock_irq(&cm_id_priv->lock);
2910 deref: cm_deref_id(cm_id_priv);
2911 return -EINVAL;
2912 }
2913
cm_drep_handler(struct cm_work * work)2914 static int cm_drep_handler(struct cm_work *work)
2915 {
2916 struct cm_id_private *cm_id_priv;
2917 struct cm_drep_msg *drep_msg;
2918
2919 drep_msg = (struct cm_drep_msg *)work->mad_recv_wc->recv_buf.mad;
2920 cm_id_priv = cm_acquire_id(
2921 cpu_to_be32(IBA_GET(CM_DREP_REMOTE_COMM_ID, drep_msg)),
2922 cpu_to_be32(IBA_GET(CM_DREP_LOCAL_COMM_ID, drep_msg)));
2923 if (!cm_id_priv)
2924 return -EINVAL;
2925
2926 work->cm_event.private_data =
2927 IBA_GET_MEM_PTR(CM_DREP_PRIVATE_DATA, drep_msg);
2928
2929 spin_lock_irq(&cm_id_priv->lock);
2930 if (cm_id_priv->id.state != IB_CM_DREQ_SENT &&
2931 cm_id_priv->id.state != IB_CM_DREQ_RCVD) {
2932 spin_unlock_irq(&cm_id_priv->lock);
2933 goto out;
2934 }
2935 cm_enter_timewait(cm_id_priv);
2936
2937 ib_cancel_mad(cm_id_priv->msg);
2938 cm_queue_work_unlock(cm_id_priv, work);
2939 return 0;
2940 out:
2941 cm_deref_id(cm_id_priv);
2942 return -EINVAL;
2943 }
2944
cm_send_rej_locked(struct cm_id_private * cm_id_priv,enum ib_cm_rej_reason reason,void * ari,u8 ari_length,const void * private_data,u8 private_data_len)2945 static int cm_send_rej_locked(struct cm_id_private *cm_id_priv,
2946 enum ib_cm_rej_reason reason, void *ari,
2947 u8 ari_length, const void *private_data,
2948 u8 private_data_len)
2949 {
2950 enum ib_cm_state state = cm_id_priv->id.state;
2951 struct ib_mad_send_buf *msg;
2952 int ret;
2953
2954 lockdep_assert_held(&cm_id_priv->lock);
2955
2956 if ((private_data && private_data_len > IB_CM_REJ_PRIVATE_DATA_SIZE) ||
2957 (ari && ari_length > IB_CM_REJ_ARI_LENGTH))
2958 return -EINVAL;
2959
2960 trace_icm_send_rej(&cm_id_priv->id, reason);
2961
2962 switch (state) {
2963 case IB_CM_REQ_SENT:
2964 case IB_CM_MRA_REQ_RCVD:
2965 case IB_CM_REQ_RCVD:
2966 case IB_CM_MRA_REQ_SENT:
2967 case IB_CM_REP_RCVD:
2968 case IB_CM_MRA_REP_SENT:
2969 cm_reset_to_idle(cm_id_priv);
2970 msg = cm_alloc_msg(cm_id_priv);
2971 if (IS_ERR(msg))
2972 return PTR_ERR(msg);
2973 cm_format_rej((struct cm_rej_msg *)msg->mad, cm_id_priv, reason,
2974 ari, ari_length, private_data, private_data_len,
2975 state);
2976 break;
2977 case IB_CM_REP_SENT:
2978 case IB_CM_MRA_REP_RCVD:
2979 cm_enter_timewait(cm_id_priv);
2980 msg = cm_alloc_msg(cm_id_priv);
2981 if (IS_ERR(msg))
2982 return PTR_ERR(msg);
2983 cm_format_rej((struct cm_rej_msg *)msg->mad, cm_id_priv, reason,
2984 ari, ari_length, private_data, private_data_len,
2985 state);
2986 break;
2987 default:
2988 trace_icm_send_unknown_rej_err(&cm_id_priv->id);
2989 return -EINVAL;
2990 }
2991
2992 ret = ib_post_send_mad(msg, NULL);
2993 if (ret) {
2994 cm_free_msg(msg);
2995 return ret;
2996 }
2997
2998 return 0;
2999 }
3000
ib_send_cm_rej(struct ib_cm_id * cm_id,enum ib_cm_rej_reason reason,void * ari,u8 ari_length,const void * private_data,u8 private_data_len)3001 int ib_send_cm_rej(struct ib_cm_id *cm_id, enum ib_cm_rej_reason reason,
3002 void *ari, u8 ari_length, const void *private_data,
3003 u8 private_data_len)
3004 {
3005 struct cm_id_private *cm_id_priv =
3006 container_of(cm_id, struct cm_id_private, id);
3007 unsigned long flags;
3008 int ret;
3009
3010 spin_lock_irqsave(&cm_id_priv->lock, flags);
3011 ret = cm_send_rej_locked(cm_id_priv, reason, ari, ari_length,
3012 private_data, private_data_len);
3013 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3014 return ret;
3015 }
3016 EXPORT_SYMBOL(ib_send_cm_rej);
3017
cm_format_rej_event(struct cm_work * work)3018 static void cm_format_rej_event(struct cm_work *work)
3019 {
3020 struct cm_rej_msg *rej_msg;
3021 struct ib_cm_rej_event_param *param;
3022
3023 rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad;
3024 param = &work->cm_event.param.rej_rcvd;
3025 param->ari = IBA_GET_MEM_PTR(CM_REJ_ARI, rej_msg);
3026 param->ari_length = IBA_GET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg);
3027 param->reason = IBA_GET(CM_REJ_REASON, rej_msg);
3028 work->cm_event.private_data =
3029 IBA_GET_MEM_PTR(CM_REJ_PRIVATE_DATA, rej_msg);
3030 }
3031
cm_acquire_rejected_id(struct cm_rej_msg * rej_msg)3032 static struct cm_id_private *cm_acquire_rejected_id(struct cm_rej_msg *rej_msg)
3033 {
3034 struct cm_id_private *cm_id_priv;
3035 __be32 remote_id;
3036
3037 remote_id = cpu_to_be32(IBA_GET(CM_REJ_LOCAL_COMM_ID, rej_msg));
3038
3039 if (IBA_GET(CM_REJ_REASON, rej_msg) == IB_CM_REJ_TIMEOUT) {
3040 cm_id_priv = cm_find_remote_id(
3041 *((__be64 *)IBA_GET_MEM_PTR(CM_REJ_ARI, rej_msg)),
3042 remote_id);
3043 } else if (IBA_GET(CM_REJ_MESSAGE_REJECTED, rej_msg) ==
3044 CM_MSG_RESPONSE_REQ)
3045 cm_id_priv = cm_acquire_id(
3046 cpu_to_be32(IBA_GET(CM_REJ_REMOTE_COMM_ID, rej_msg)),
3047 0);
3048 else
3049 cm_id_priv = cm_acquire_id(
3050 cpu_to_be32(IBA_GET(CM_REJ_REMOTE_COMM_ID, rej_msg)),
3051 remote_id);
3052
3053 return cm_id_priv;
3054 }
3055
cm_rej_handler(struct cm_work * work)3056 static int cm_rej_handler(struct cm_work *work)
3057 {
3058 struct cm_id_private *cm_id_priv;
3059 struct cm_rej_msg *rej_msg;
3060
3061 rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad;
3062 cm_id_priv = cm_acquire_rejected_id(rej_msg);
3063 if (!cm_id_priv)
3064 return -EINVAL;
3065
3066 cm_format_rej_event(work);
3067
3068 spin_lock_irq(&cm_id_priv->lock);
3069 switch (cm_id_priv->id.state) {
3070 case IB_CM_REQ_SENT:
3071 case IB_CM_MRA_REQ_RCVD:
3072 case IB_CM_REP_SENT:
3073 case IB_CM_MRA_REP_RCVD:
3074 ib_cancel_mad(cm_id_priv->msg);
3075 fallthrough;
3076 case IB_CM_REQ_RCVD:
3077 case IB_CM_MRA_REQ_SENT:
3078 if (IBA_GET(CM_REJ_REASON, rej_msg) == IB_CM_REJ_STALE_CONN)
3079 cm_enter_timewait(cm_id_priv);
3080 else
3081 cm_reset_to_idle(cm_id_priv);
3082 break;
3083 case IB_CM_DREQ_SENT:
3084 ib_cancel_mad(cm_id_priv->msg);
3085 fallthrough;
3086 case IB_CM_REP_RCVD:
3087 case IB_CM_MRA_REP_SENT:
3088 cm_enter_timewait(cm_id_priv);
3089 break;
3090 case IB_CM_ESTABLISHED:
3091 if (cm_id_priv->id.lap_state == IB_CM_LAP_UNINIT ||
3092 cm_id_priv->id.lap_state == IB_CM_LAP_SENT) {
3093 if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT)
3094 ib_cancel_mad(cm_id_priv->msg);
3095 cm_enter_timewait(cm_id_priv);
3096 break;
3097 }
3098 fallthrough;
3099 default:
3100 trace_icm_rej_unknown_err(&cm_id_priv->id);
3101 spin_unlock_irq(&cm_id_priv->lock);
3102 goto out;
3103 }
3104
3105 cm_queue_work_unlock(cm_id_priv, work);
3106 return 0;
3107 out:
3108 cm_deref_id(cm_id_priv);
3109 return -EINVAL;
3110 }
3111
ib_prepare_cm_mra(struct ib_cm_id * cm_id)3112 int ib_prepare_cm_mra(struct ib_cm_id *cm_id)
3113 {
3114 struct cm_id_private *cm_id_priv;
3115 enum ib_cm_state cm_state;
3116 enum ib_cm_lap_state lap_state;
3117 unsigned long flags;
3118 int ret = 0;
3119
3120 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3121
3122 spin_lock_irqsave(&cm_id_priv->lock, flags);
3123 switch (cm_id_priv->id.state) {
3124 case IB_CM_REQ_RCVD:
3125 cm_state = IB_CM_MRA_REQ_SENT;
3126 lap_state = cm_id->lap_state;
3127 break;
3128 case IB_CM_REP_RCVD:
3129 cm_state = IB_CM_MRA_REP_SENT;
3130 lap_state = cm_id->lap_state;
3131 break;
3132 case IB_CM_ESTABLISHED:
3133 if (cm_id->lap_state == IB_CM_LAP_RCVD) {
3134 cm_state = cm_id->state;
3135 lap_state = IB_CM_MRA_LAP_SENT;
3136 break;
3137 }
3138 fallthrough;
3139 default:
3140 trace_icm_prepare_mra_unknown_err(&cm_id_priv->id);
3141 ret = -EINVAL;
3142 goto error_unlock;
3143 }
3144
3145 cm_id->state = cm_state;
3146 cm_id->lap_state = lap_state;
3147 cm_set_private_data(cm_id_priv, NULL, 0);
3148
3149 error_unlock:
3150 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3151 return ret;
3152 }
3153 EXPORT_SYMBOL(ib_prepare_cm_mra);
3154
cm_acquire_mraed_id(struct cm_mra_msg * mra_msg)3155 static struct cm_id_private *cm_acquire_mraed_id(struct cm_mra_msg *mra_msg)
3156 {
3157 switch (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg)) {
3158 case CM_MSG_RESPONSE_REQ:
3159 return cm_acquire_id(
3160 cpu_to_be32(IBA_GET(CM_MRA_REMOTE_COMM_ID, mra_msg)),
3161 0);
3162 case CM_MSG_RESPONSE_REP:
3163 case CM_MSG_RESPONSE_OTHER:
3164 return cm_acquire_id(
3165 cpu_to_be32(IBA_GET(CM_MRA_REMOTE_COMM_ID, mra_msg)),
3166 cpu_to_be32(IBA_GET(CM_MRA_LOCAL_COMM_ID, mra_msg)));
3167 default:
3168 return NULL;
3169 }
3170 }
3171
cm_mra_handler(struct cm_work * work)3172 static int cm_mra_handler(struct cm_work *work)
3173 {
3174 struct cm_id_private *cm_id_priv;
3175 struct cm_mra_msg *mra_msg;
3176 int timeout;
3177
3178 mra_msg = (struct cm_mra_msg *)work->mad_recv_wc->recv_buf.mad;
3179 cm_id_priv = cm_acquire_mraed_id(mra_msg);
3180 if (!cm_id_priv)
3181 return -EINVAL;
3182
3183 work->cm_event.private_data =
3184 IBA_GET_MEM_PTR(CM_MRA_PRIVATE_DATA, mra_msg);
3185 work->cm_event.param.mra_rcvd.service_timeout =
3186 IBA_GET(CM_MRA_SERVICE_TIMEOUT, mra_msg);
3187 timeout = cm_convert_to_ms(IBA_GET(CM_MRA_SERVICE_TIMEOUT, mra_msg)) +
3188 cm_convert_to_ms(cm_id_priv->av.timeout);
3189
3190 spin_lock_irq(&cm_id_priv->lock);
3191 switch (cm_id_priv->id.state) {
3192 case IB_CM_REQ_SENT:
3193 if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) !=
3194 CM_MSG_RESPONSE_REQ ||
3195 ib_modify_mad(cm_id_priv->msg, timeout))
3196 goto out;
3197 cm_id_priv->id.state = IB_CM_MRA_REQ_RCVD;
3198 break;
3199 case IB_CM_REP_SENT:
3200 if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) !=
3201 CM_MSG_RESPONSE_REP ||
3202 ib_modify_mad(cm_id_priv->msg, timeout))
3203 goto out;
3204 cm_id_priv->id.state = IB_CM_MRA_REP_RCVD;
3205 break;
3206 case IB_CM_ESTABLISHED:
3207 if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) !=
3208 CM_MSG_RESPONSE_OTHER ||
3209 cm_id_priv->id.lap_state != IB_CM_LAP_SENT ||
3210 ib_modify_mad(cm_id_priv->msg, timeout)) {
3211 if (cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
3212 atomic_long_inc(
3213 &work->port->counters[CM_RECV_DUPLICATES]
3214 [CM_MRA_COUNTER]);
3215 goto out;
3216 }
3217 cm_id_priv->id.lap_state = IB_CM_MRA_LAP_RCVD;
3218 break;
3219 case IB_CM_MRA_REQ_RCVD:
3220 case IB_CM_MRA_REP_RCVD:
3221 atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
3222 [CM_MRA_COUNTER]);
3223 fallthrough;
3224 default:
3225 trace_icm_mra_unknown_err(&cm_id_priv->id);
3226 goto out;
3227 }
3228
3229 cm_id_priv->msg->context[1] = (void *) (unsigned long)
3230 cm_id_priv->id.state;
3231 cm_queue_work_unlock(cm_id_priv, work);
3232 return 0;
3233 out:
3234 spin_unlock_irq(&cm_id_priv->lock);
3235 cm_deref_id(cm_id_priv);
3236 return -EINVAL;
3237 }
3238
cm_format_path_lid_from_lap(struct cm_lap_msg * lap_msg,struct sa_path_rec * path)3239 static void cm_format_path_lid_from_lap(struct cm_lap_msg *lap_msg,
3240 struct sa_path_rec *path)
3241 {
3242 u32 lid;
3243
3244 if (path->rec_type != SA_PATH_REC_TYPE_OPA) {
3245 sa_path_set_dlid(path, IBA_GET(CM_LAP_ALTERNATE_LOCAL_PORT_LID,
3246 lap_msg));
3247 sa_path_set_slid(path, IBA_GET(CM_LAP_ALTERNATE_REMOTE_PORT_LID,
3248 lap_msg));
3249 } else {
3250 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
3251 CM_LAP_ALTERNATE_LOCAL_PORT_GID, lap_msg));
3252 sa_path_set_dlid(path, lid);
3253
3254 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
3255 CM_LAP_ALTERNATE_REMOTE_PORT_GID, lap_msg));
3256 sa_path_set_slid(path, lid);
3257 }
3258 }
3259
cm_format_path_from_lap(struct cm_id_private * cm_id_priv,struct sa_path_rec * path,struct cm_lap_msg * lap_msg)3260 static void cm_format_path_from_lap(struct cm_id_private *cm_id_priv,
3261 struct sa_path_rec *path,
3262 struct cm_lap_msg *lap_msg)
3263 {
3264 path->dgid = *IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_LOCAL_PORT_GID, lap_msg);
3265 path->sgid =
3266 *IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_REMOTE_PORT_GID, lap_msg);
3267 path->flow_label =
3268 cpu_to_be32(IBA_GET(CM_LAP_ALTERNATE_FLOW_LABEL, lap_msg));
3269 path->hop_limit = IBA_GET(CM_LAP_ALTERNATE_HOP_LIMIT, lap_msg);
3270 path->traffic_class = IBA_GET(CM_LAP_ALTERNATE_TRAFFIC_CLASS, lap_msg);
3271 path->reversible = 1;
3272 path->pkey = cm_id_priv->pkey;
3273 path->sl = IBA_GET(CM_LAP_ALTERNATE_SL, lap_msg);
3274 path->mtu_selector = IB_SA_EQ;
3275 path->mtu = cm_id_priv->path_mtu;
3276 path->rate_selector = IB_SA_EQ;
3277 path->rate = IBA_GET(CM_LAP_ALTERNATE_PACKET_RATE, lap_msg);
3278 path->packet_life_time_selector = IB_SA_EQ;
3279 path->packet_life_time =
3280 IBA_GET(CM_LAP_ALTERNATE_LOCAL_ACK_TIMEOUT, lap_msg);
3281 path->packet_life_time -= (path->packet_life_time > 0);
3282 cm_format_path_lid_from_lap(lap_msg, path);
3283 }
3284
cm_lap_handler(struct cm_work * work)3285 static int cm_lap_handler(struct cm_work *work)
3286 {
3287 struct cm_id_private *cm_id_priv;
3288 struct cm_lap_msg *lap_msg;
3289 struct ib_cm_lap_event_param *param;
3290 struct ib_mad_send_buf *msg = NULL;
3291 struct rdma_ah_attr ah_attr;
3292 struct cm_av alt_av = {};
3293 int ret;
3294
3295 /* Currently Alternate path messages are not supported for
3296 * RoCE link layer.
3297 */
3298 if (rdma_protocol_roce(work->port->cm_dev->ib_device,
3299 work->port->port_num))
3300 return -EINVAL;
3301
3302 /* todo: verify LAP request and send reject APR if invalid. */
3303 lap_msg = (struct cm_lap_msg *)work->mad_recv_wc->recv_buf.mad;
3304 cm_id_priv = cm_acquire_id(
3305 cpu_to_be32(IBA_GET(CM_LAP_REMOTE_COMM_ID, lap_msg)),
3306 cpu_to_be32(IBA_GET(CM_LAP_LOCAL_COMM_ID, lap_msg)));
3307 if (!cm_id_priv)
3308 return -EINVAL;
3309
3310 param = &work->cm_event.param.lap_rcvd;
3311 memset(&work->path[0], 0, sizeof(work->path[1]));
3312 cm_path_set_rec_type(work->port->cm_dev->ib_device,
3313 work->port->port_num, &work->path[0],
3314 IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_LOCAL_PORT_GID,
3315 lap_msg));
3316 param->alternate_path = &work->path[0];
3317 cm_format_path_from_lap(cm_id_priv, param->alternate_path, lap_msg);
3318 work->cm_event.private_data =
3319 IBA_GET_MEM_PTR(CM_LAP_PRIVATE_DATA, lap_msg);
3320
3321 ret = ib_init_ah_attr_from_wc(work->port->cm_dev->ib_device,
3322 work->port->port_num,
3323 work->mad_recv_wc->wc,
3324 work->mad_recv_wc->recv_buf.grh,
3325 &ah_attr);
3326 if (ret)
3327 goto deref;
3328
3329 ret = cm_init_av_by_path(param->alternate_path, NULL, &alt_av);
3330 if (ret) {
3331 rdma_destroy_ah_attr(&ah_attr);
3332 goto deref;
3333 }
3334
3335 spin_lock_irq(&cm_id_priv->lock);
3336 cm_init_av_for_lap(work->port, work->mad_recv_wc->wc,
3337 &ah_attr, &cm_id_priv->av);
3338 cm_move_av_from_path(&cm_id_priv->alt_av, &alt_av);
3339
3340 if (cm_id_priv->id.state != IB_CM_ESTABLISHED)
3341 goto unlock;
3342
3343 switch (cm_id_priv->id.lap_state) {
3344 case IB_CM_LAP_UNINIT:
3345 case IB_CM_LAP_IDLE:
3346 break;
3347 case IB_CM_MRA_LAP_SENT:
3348 atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
3349 [CM_LAP_COUNTER]);
3350 msg = cm_alloc_response_msg_no_ah(work->port, work->mad_recv_wc,
3351 true);
3352 if (IS_ERR(msg))
3353 goto unlock;
3354
3355 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
3356 CM_MSG_RESPONSE_OTHER,
3357 cm_id_priv->private_data,
3358 cm_id_priv->private_data_len);
3359 spin_unlock_irq(&cm_id_priv->lock);
3360
3361 if (cm_create_response_msg_ah(work->port, work->mad_recv_wc, msg) ||
3362 ib_post_send_mad(msg, NULL))
3363 cm_free_msg(msg);
3364 goto deref;
3365 case IB_CM_LAP_RCVD:
3366 atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
3367 [CM_LAP_COUNTER]);
3368 goto unlock;
3369 default:
3370 goto unlock;
3371 }
3372
3373 cm_id_priv->id.lap_state = IB_CM_LAP_RCVD;
3374 cm_id_priv->tid = lap_msg->hdr.tid;
3375 cm_queue_work_unlock(cm_id_priv, work);
3376 return 0;
3377
3378 unlock: spin_unlock_irq(&cm_id_priv->lock);
3379 deref: cm_deref_id(cm_id_priv);
3380 return -EINVAL;
3381 }
3382
cm_apr_handler(struct cm_work * work)3383 static int cm_apr_handler(struct cm_work *work)
3384 {
3385 struct cm_id_private *cm_id_priv;
3386 struct cm_apr_msg *apr_msg;
3387
3388 /* Currently Alternate path messages are not supported for
3389 * RoCE link layer.
3390 */
3391 if (rdma_protocol_roce(work->port->cm_dev->ib_device,
3392 work->port->port_num))
3393 return -EINVAL;
3394
3395 apr_msg = (struct cm_apr_msg *)work->mad_recv_wc->recv_buf.mad;
3396 cm_id_priv = cm_acquire_id(
3397 cpu_to_be32(IBA_GET(CM_APR_REMOTE_COMM_ID, apr_msg)),
3398 cpu_to_be32(IBA_GET(CM_APR_LOCAL_COMM_ID, apr_msg)));
3399 if (!cm_id_priv)
3400 return -EINVAL; /* Unmatched reply. */
3401
3402 work->cm_event.param.apr_rcvd.ap_status =
3403 IBA_GET(CM_APR_AR_STATUS, apr_msg);
3404 work->cm_event.param.apr_rcvd.apr_info =
3405 IBA_GET_MEM_PTR(CM_APR_ADDITIONAL_INFORMATION, apr_msg);
3406 work->cm_event.param.apr_rcvd.info_len =
3407 IBA_GET(CM_APR_ADDITIONAL_INFORMATION_LENGTH, apr_msg);
3408 work->cm_event.private_data =
3409 IBA_GET_MEM_PTR(CM_APR_PRIVATE_DATA, apr_msg);
3410
3411 spin_lock_irq(&cm_id_priv->lock);
3412 if (cm_id_priv->id.state != IB_CM_ESTABLISHED ||
3413 (cm_id_priv->id.lap_state != IB_CM_LAP_SENT &&
3414 cm_id_priv->id.lap_state != IB_CM_MRA_LAP_RCVD)) {
3415 spin_unlock_irq(&cm_id_priv->lock);
3416 goto out;
3417 }
3418 cm_id_priv->id.lap_state = IB_CM_LAP_IDLE;
3419 ib_cancel_mad(cm_id_priv->msg);
3420 cm_queue_work_unlock(cm_id_priv, work);
3421 return 0;
3422 out:
3423 cm_deref_id(cm_id_priv);
3424 return -EINVAL;
3425 }
3426
cm_timewait_handler(struct cm_work * work)3427 static int cm_timewait_handler(struct cm_work *work)
3428 {
3429 struct cm_timewait_info *timewait_info;
3430 struct cm_id_private *cm_id_priv;
3431
3432 timewait_info = container_of(work, struct cm_timewait_info, work);
3433 spin_lock_irq(&cm.lock);
3434 list_del(&timewait_info->list);
3435 spin_unlock_irq(&cm.lock);
3436
3437 cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
3438 timewait_info->work.remote_id);
3439 if (!cm_id_priv)
3440 return -EINVAL;
3441
3442 spin_lock_irq(&cm_id_priv->lock);
3443 if (cm_id_priv->id.state != IB_CM_TIMEWAIT ||
3444 cm_id_priv->remote_qpn != timewait_info->remote_qpn) {
3445 spin_unlock_irq(&cm_id_priv->lock);
3446 goto out;
3447 }
3448 cm_id_priv->id.state = IB_CM_IDLE;
3449 cm_queue_work_unlock(cm_id_priv, work);
3450 return 0;
3451 out:
3452 cm_deref_id(cm_id_priv);
3453 return -EINVAL;
3454 }
3455
cm_format_sidr_req(struct cm_sidr_req_msg * sidr_req_msg,struct cm_id_private * cm_id_priv,struct ib_cm_sidr_req_param * param)3456 static void cm_format_sidr_req(struct cm_sidr_req_msg *sidr_req_msg,
3457 struct cm_id_private *cm_id_priv,
3458 struct ib_cm_sidr_req_param *param)
3459 {
3460 cm_format_mad_hdr(&sidr_req_msg->hdr, CM_SIDR_REQ_ATTR_ID,
3461 cm_form_tid(cm_id_priv));
3462 IBA_SET(CM_SIDR_REQ_REQUESTID, sidr_req_msg,
3463 be32_to_cpu(cm_id_priv->id.local_id));
3464 IBA_SET(CM_SIDR_REQ_PARTITION_KEY, sidr_req_msg,
3465 be16_to_cpu(param->path->pkey));
3466 IBA_SET(CM_SIDR_REQ_SERVICEID, sidr_req_msg,
3467 be64_to_cpu(param->service_id));
3468
3469 if (param->private_data && param->private_data_len)
3470 IBA_SET_MEM(CM_SIDR_REQ_PRIVATE_DATA, sidr_req_msg,
3471 param->private_data, param->private_data_len);
3472 }
3473
ib_send_cm_sidr_req(struct ib_cm_id * cm_id,struct ib_cm_sidr_req_param * param)3474 int ib_send_cm_sidr_req(struct ib_cm_id *cm_id,
3475 struct ib_cm_sidr_req_param *param)
3476 {
3477 struct cm_id_private *cm_id_priv;
3478 struct ib_mad_send_buf *msg;
3479 struct cm_av av = {};
3480 unsigned long flags;
3481 int ret;
3482
3483 if (!param->path || (param->private_data &&
3484 param->private_data_len > IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE))
3485 return -EINVAL;
3486
3487 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3488 ret = cm_init_av_by_path(param->path, param->sgid_attr, &av);
3489 if (ret)
3490 return ret;
3491
3492 spin_lock_irqsave(&cm_id_priv->lock, flags);
3493 cm_move_av_from_path(&cm_id_priv->av, &av);
3494 cm_id->service_id = param->service_id;
3495 cm_id_priv->timeout_ms = param->timeout_ms;
3496 cm_id_priv->max_cm_retries = param->max_cm_retries;
3497 if (cm_id->state != IB_CM_IDLE) {
3498 ret = -EINVAL;
3499 goto out_unlock;
3500 }
3501
3502 msg = cm_alloc_priv_msg(cm_id_priv, IB_CM_SIDR_REQ_SENT);
3503 if (IS_ERR(msg)) {
3504 ret = PTR_ERR(msg);
3505 goto out_unlock;
3506 }
3507
3508 cm_format_sidr_req((struct cm_sidr_req_msg *)msg->mad, cm_id_priv,
3509 param);
3510
3511 trace_icm_send_sidr_req(&cm_id_priv->id);
3512 ret = ib_post_send_mad(msg, NULL);
3513 if (ret)
3514 goto out_free;
3515 cm_id->state = IB_CM_SIDR_REQ_SENT;
3516 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3517 return 0;
3518 out_free:
3519 cm_free_priv_msg(msg);
3520 out_unlock:
3521 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3522 return ret;
3523 }
3524 EXPORT_SYMBOL(ib_send_cm_sidr_req);
3525
cm_format_sidr_req_event(struct cm_work * work,const struct cm_id_private * rx_cm_id,struct ib_cm_id * listen_id)3526 static void cm_format_sidr_req_event(struct cm_work *work,
3527 const struct cm_id_private *rx_cm_id,
3528 struct ib_cm_id *listen_id)
3529 {
3530 struct cm_sidr_req_msg *sidr_req_msg;
3531 struct ib_cm_sidr_req_event_param *param;
3532
3533 sidr_req_msg = (struct cm_sidr_req_msg *)
3534 work->mad_recv_wc->recv_buf.mad;
3535 param = &work->cm_event.param.sidr_req_rcvd;
3536 param->pkey = IBA_GET(CM_SIDR_REQ_PARTITION_KEY, sidr_req_msg);
3537 param->listen_id = listen_id;
3538 param->service_id =
3539 cpu_to_be64(IBA_GET(CM_SIDR_REQ_SERVICEID, sidr_req_msg));
3540 param->bth_pkey = cm_get_bth_pkey(work);
3541 param->port = work->port->port_num;
3542 param->sgid_attr = rx_cm_id->av.ah_attr.grh.sgid_attr;
3543 work->cm_event.private_data =
3544 IBA_GET_MEM_PTR(CM_SIDR_REQ_PRIVATE_DATA, sidr_req_msg);
3545 }
3546
cm_sidr_req_handler(struct cm_work * work)3547 static int cm_sidr_req_handler(struct cm_work *work)
3548 {
3549 struct cm_id_private *cm_id_priv, *listen_cm_id_priv;
3550 struct cm_sidr_req_msg *sidr_req_msg;
3551 struct ib_wc *wc;
3552 int ret;
3553
3554 cm_id_priv =
3555 cm_alloc_id_priv(work->port->cm_dev->ib_device, NULL, NULL);
3556 if (IS_ERR(cm_id_priv))
3557 return PTR_ERR(cm_id_priv);
3558
3559 /* Record SGID/SLID and request ID for lookup. */
3560 sidr_req_msg = (struct cm_sidr_req_msg *)
3561 work->mad_recv_wc->recv_buf.mad;
3562
3563 cm_id_priv->id.remote_id =
3564 cpu_to_be32(IBA_GET(CM_SIDR_REQ_REQUESTID, sidr_req_msg));
3565 cm_id_priv->id.service_id =
3566 cpu_to_be64(IBA_GET(CM_SIDR_REQ_SERVICEID, sidr_req_msg));
3567 cm_id_priv->tid = sidr_req_msg->hdr.tid;
3568
3569 wc = work->mad_recv_wc->wc;
3570 cm_id_priv->sidr_slid = wc->slid;
3571 ret = cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
3572 work->mad_recv_wc->recv_buf.grh,
3573 &cm_id_priv->av);
3574 if (ret)
3575 goto out;
3576
3577 spin_lock_irq(&cm.lock);
3578 listen_cm_id_priv = cm_insert_remote_sidr(cm_id_priv);
3579 if (listen_cm_id_priv) {
3580 spin_unlock_irq(&cm.lock);
3581 atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
3582 [CM_SIDR_REQ_COUNTER]);
3583 goto out; /* Duplicate message. */
3584 }
3585 cm_id_priv->id.state = IB_CM_SIDR_REQ_RCVD;
3586 listen_cm_id_priv = cm_find_listen(cm_id_priv->id.device,
3587 cm_id_priv->id.service_id);
3588 if (!listen_cm_id_priv) {
3589 spin_unlock_irq(&cm.lock);
3590 ib_send_cm_sidr_rep(&cm_id_priv->id,
3591 &(struct ib_cm_sidr_rep_param){
3592 .status = IB_SIDR_UNSUPPORTED });
3593 goto out; /* No match. */
3594 }
3595 spin_unlock_irq(&cm.lock);
3596
3597 cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler;
3598 cm_id_priv->id.context = listen_cm_id_priv->id.context;
3599
3600 /*
3601 * A SIDR ID does not need to be in the xarray since it does not receive
3602 * mads, is not placed in the remote_id or remote_qpn rbtree, and does
3603 * not enter timewait.
3604 */
3605
3606 cm_format_sidr_req_event(work, cm_id_priv, &listen_cm_id_priv->id);
3607 ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &work->cm_event);
3608 cm_free_work(work);
3609 /*
3610 * A pointer to the listen_cm_id is held in the event, so this deref
3611 * must be after the event is delivered above.
3612 */
3613 cm_deref_id(listen_cm_id_priv);
3614 if (ret)
3615 cm_destroy_id(&cm_id_priv->id, ret);
3616 return 0;
3617 out:
3618 ib_destroy_cm_id(&cm_id_priv->id);
3619 return -EINVAL;
3620 }
3621
cm_format_sidr_rep(struct cm_sidr_rep_msg * sidr_rep_msg,struct cm_id_private * cm_id_priv,struct ib_cm_sidr_rep_param * param)3622 static void cm_format_sidr_rep(struct cm_sidr_rep_msg *sidr_rep_msg,
3623 struct cm_id_private *cm_id_priv,
3624 struct ib_cm_sidr_rep_param *param)
3625 {
3626 cm_format_mad_ece_hdr(&sidr_rep_msg->hdr, CM_SIDR_REP_ATTR_ID,
3627 cm_id_priv->tid, param->ece.attr_mod);
3628 IBA_SET(CM_SIDR_REP_REQUESTID, sidr_rep_msg,
3629 be32_to_cpu(cm_id_priv->id.remote_id));
3630 IBA_SET(CM_SIDR_REP_STATUS, sidr_rep_msg, param->status);
3631 IBA_SET(CM_SIDR_REP_QPN, sidr_rep_msg, param->qp_num);
3632 IBA_SET(CM_SIDR_REP_SERVICEID, sidr_rep_msg,
3633 be64_to_cpu(cm_id_priv->id.service_id));
3634 IBA_SET(CM_SIDR_REP_Q_KEY, sidr_rep_msg, param->qkey);
3635 IBA_SET(CM_SIDR_REP_VENDOR_ID_L, sidr_rep_msg,
3636 param->ece.vendor_id & 0xFF);
3637 IBA_SET(CM_SIDR_REP_VENDOR_ID_H, sidr_rep_msg,
3638 (param->ece.vendor_id >> 8) & 0xFF);
3639
3640 if (param->info && param->info_length)
3641 IBA_SET_MEM(CM_SIDR_REP_ADDITIONAL_INFORMATION, sidr_rep_msg,
3642 param->info, param->info_length);
3643
3644 if (param->private_data && param->private_data_len)
3645 IBA_SET_MEM(CM_SIDR_REP_PRIVATE_DATA, sidr_rep_msg,
3646 param->private_data, param->private_data_len);
3647 }
3648
cm_send_sidr_rep_locked(struct cm_id_private * cm_id_priv,struct ib_cm_sidr_rep_param * param)3649 static int cm_send_sidr_rep_locked(struct cm_id_private *cm_id_priv,
3650 struct ib_cm_sidr_rep_param *param)
3651 {
3652 struct ib_mad_send_buf *msg;
3653 unsigned long flags;
3654 int ret;
3655
3656 lockdep_assert_held(&cm_id_priv->lock);
3657
3658 if ((param->info && param->info_length > IB_CM_SIDR_REP_INFO_LENGTH) ||
3659 (param->private_data &&
3660 param->private_data_len > IB_CM_SIDR_REP_PRIVATE_DATA_SIZE))
3661 return -EINVAL;
3662
3663 if (cm_id_priv->id.state != IB_CM_SIDR_REQ_RCVD)
3664 return -EINVAL;
3665
3666 msg = cm_alloc_msg(cm_id_priv);
3667 if (IS_ERR(msg))
3668 return PTR_ERR(msg);
3669
3670 cm_format_sidr_rep((struct cm_sidr_rep_msg *) msg->mad, cm_id_priv,
3671 param);
3672 trace_icm_send_sidr_rep(&cm_id_priv->id);
3673 ret = ib_post_send_mad(msg, NULL);
3674 if (ret) {
3675 cm_free_msg(msg);
3676 return ret;
3677 }
3678 cm_id_priv->id.state = IB_CM_IDLE;
3679 spin_lock_irqsave(&cm.lock, flags);
3680 if (!RB_EMPTY_NODE(&cm_id_priv->sidr_id_node)) {
3681 rb_erase(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
3682 RB_CLEAR_NODE(&cm_id_priv->sidr_id_node);
3683 }
3684 spin_unlock_irqrestore(&cm.lock, flags);
3685 return 0;
3686 }
3687
ib_send_cm_sidr_rep(struct ib_cm_id * cm_id,struct ib_cm_sidr_rep_param * param)3688 int ib_send_cm_sidr_rep(struct ib_cm_id *cm_id,
3689 struct ib_cm_sidr_rep_param *param)
3690 {
3691 struct cm_id_private *cm_id_priv =
3692 container_of(cm_id, struct cm_id_private, id);
3693 unsigned long flags;
3694 int ret;
3695
3696 spin_lock_irqsave(&cm_id_priv->lock, flags);
3697 ret = cm_send_sidr_rep_locked(cm_id_priv, param);
3698 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3699 return ret;
3700 }
3701 EXPORT_SYMBOL(ib_send_cm_sidr_rep);
3702
cm_format_sidr_rep_event(struct cm_work * work,const struct cm_id_private * cm_id_priv)3703 static void cm_format_sidr_rep_event(struct cm_work *work,
3704 const struct cm_id_private *cm_id_priv)
3705 {
3706 struct cm_sidr_rep_msg *sidr_rep_msg;
3707 struct ib_cm_sidr_rep_event_param *param;
3708
3709 sidr_rep_msg = (struct cm_sidr_rep_msg *)
3710 work->mad_recv_wc->recv_buf.mad;
3711 param = &work->cm_event.param.sidr_rep_rcvd;
3712 param->status = IBA_GET(CM_SIDR_REP_STATUS, sidr_rep_msg);
3713 param->qkey = IBA_GET(CM_SIDR_REP_Q_KEY, sidr_rep_msg);
3714 param->qpn = IBA_GET(CM_SIDR_REP_QPN, sidr_rep_msg);
3715 param->info = IBA_GET_MEM_PTR(CM_SIDR_REP_ADDITIONAL_INFORMATION,
3716 sidr_rep_msg);
3717 param->info_len = IBA_GET(CM_SIDR_REP_ADDITIONAL_INFORMATION_LENGTH,
3718 sidr_rep_msg);
3719 param->sgid_attr = cm_id_priv->av.ah_attr.grh.sgid_attr;
3720 work->cm_event.private_data =
3721 IBA_GET_MEM_PTR(CM_SIDR_REP_PRIVATE_DATA, sidr_rep_msg);
3722 }
3723
cm_sidr_rep_handler(struct cm_work * work)3724 static int cm_sidr_rep_handler(struct cm_work *work)
3725 {
3726 struct cm_sidr_rep_msg *sidr_rep_msg;
3727 struct cm_id_private *cm_id_priv;
3728
3729 sidr_rep_msg = (struct cm_sidr_rep_msg *)
3730 work->mad_recv_wc->recv_buf.mad;
3731 cm_id_priv = cm_acquire_id(
3732 cpu_to_be32(IBA_GET(CM_SIDR_REP_REQUESTID, sidr_rep_msg)), 0);
3733 if (!cm_id_priv)
3734 return -EINVAL; /* Unmatched reply. */
3735
3736 spin_lock_irq(&cm_id_priv->lock);
3737 if (cm_id_priv->id.state != IB_CM_SIDR_REQ_SENT) {
3738 spin_unlock_irq(&cm_id_priv->lock);
3739 goto out;
3740 }
3741 cm_id_priv->id.state = IB_CM_IDLE;
3742 ib_cancel_mad(cm_id_priv->msg);
3743 spin_unlock_irq(&cm_id_priv->lock);
3744
3745 cm_format_sidr_rep_event(work, cm_id_priv);
3746 cm_process_work(cm_id_priv, work);
3747 return 0;
3748 out:
3749 cm_deref_id(cm_id_priv);
3750 return -EINVAL;
3751 }
3752
cm_process_send_error(struct cm_id_private * cm_id_priv,struct ib_mad_send_buf * msg,enum ib_wc_status wc_status)3753 static void cm_process_send_error(struct cm_id_private *cm_id_priv,
3754 struct ib_mad_send_buf *msg,
3755 enum ib_wc_status wc_status)
3756 {
3757 enum ib_cm_state state = (unsigned long) msg->context[1];
3758 struct ib_cm_event cm_event = {};
3759 int ret;
3760
3761 /* Discard old sends. */
3762 spin_lock_irq(&cm_id_priv->lock);
3763 if (msg != cm_id_priv->msg) {
3764 spin_unlock_irq(&cm_id_priv->lock);
3765 cm_free_msg(msg);
3766 cm_deref_id(cm_id_priv);
3767 return;
3768 }
3769 cm_free_priv_msg(msg);
3770
3771 if (state != cm_id_priv->id.state || wc_status == IB_WC_SUCCESS ||
3772 wc_status == IB_WC_WR_FLUSH_ERR)
3773 goto out_unlock;
3774
3775 trace_icm_mad_send_err(state, wc_status);
3776 switch (state) {
3777 case IB_CM_REQ_SENT:
3778 case IB_CM_MRA_REQ_RCVD:
3779 cm_reset_to_idle(cm_id_priv);
3780 cm_event.event = IB_CM_REQ_ERROR;
3781 break;
3782 case IB_CM_REP_SENT:
3783 case IB_CM_MRA_REP_RCVD:
3784 cm_reset_to_idle(cm_id_priv);
3785 cm_event.event = IB_CM_REP_ERROR;
3786 break;
3787 case IB_CM_DREQ_SENT:
3788 cm_enter_timewait(cm_id_priv);
3789 cm_event.event = IB_CM_DREQ_ERROR;
3790 break;
3791 case IB_CM_SIDR_REQ_SENT:
3792 cm_id_priv->id.state = IB_CM_IDLE;
3793 cm_event.event = IB_CM_SIDR_REQ_ERROR;
3794 break;
3795 default:
3796 goto out_unlock;
3797 }
3798 spin_unlock_irq(&cm_id_priv->lock);
3799 cm_event.param.send_status = wc_status;
3800
3801 /* No other events can occur on the cm_id at this point. */
3802 ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &cm_event);
3803 if (ret)
3804 ib_destroy_cm_id(&cm_id_priv->id);
3805 return;
3806 out_unlock:
3807 spin_unlock_irq(&cm_id_priv->lock);
3808 }
3809
cm_send_handler(struct ib_mad_agent * mad_agent,struct ib_mad_send_wc * mad_send_wc)3810 static void cm_send_handler(struct ib_mad_agent *mad_agent,
3811 struct ib_mad_send_wc *mad_send_wc)
3812 {
3813 struct ib_mad_send_buf *msg = mad_send_wc->send_buf;
3814 struct cm_id_private *cm_id_priv;
3815 struct cm_port *port;
3816 u16 attr_index;
3817
3818 port = mad_agent->context;
3819 attr_index = be16_to_cpu(((struct ib_mad_hdr *)
3820 msg->mad)->attr_id) - CM_ATTR_ID_OFFSET;
3821
3822 if (msg->context[0] == CM_DIRECT_RETRY_CTX) {
3823 msg->retries = 1;
3824 cm_id_priv = NULL;
3825 } else {
3826 cm_id_priv = msg->context[0];
3827 }
3828
3829 atomic_long_add(1 + msg->retries, &port->counters[CM_XMIT][attr_index]);
3830 if (msg->retries)
3831 atomic_long_add(msg->retries,
3832 &port->counters[CM_XMIT_RETRIES][attr_index]);
3833
3834 if (cm_id_priv)
3835 cm_process_send_error(cm_id_priv, msg, mad_send_wc->status);
3836 else
3837 cm_free_msg(msg);
3838 }
3839
cm_work_handler(struct work_struct * _work)3840 static void cm_work_handler(struct work_struct *_work)
3841 {
3842 struct cm_work *work = container_of(_work, struct cm_work, work.work);
3843 int ret;
3844
3845 switch (work->cm_event.event) {
3846 case IB_CM_REQ_RECEIVED:
3847 ret = cm_req_handler(work);
3848 break;
3849 case IB_CM_MRA_RECEIVED:
3850 ret = cm_mra_handler(work);
3851 break;
3852 case IB_CM_REJ_RECEIVED:
3853 ret = cm_rej_handler(work);
3854 break;
3855 case IB_CM_REP_RECEIVED:
3856 ret = cm_rep_handler(work);
3857 break;
3858 case IB_CM_RTU_RECEIVED:
3859 ret = cm_rtu_handler(work);
3860 break;
3861 case IB_CM_USER_ESTABLISHED:
3862 ret = cm_establish_handler(work);
3863 break;
3864 case IB_CM_DREQ_RECEIVED:
3865 ret = cm_dreq_handler(work);
3866 break;
3867 case IB_CM_DREP_RECEIVED:
3868 ret = cm_drep_handler(work);
3869 break;
3870 case IB_CM_SIDR_REQ_RECEIVED:
3871 ret = cm_sidr_req_handler(work);
3872 break;
3873 case IB_CM_SIDR_REP_RECEIVED:
3874 ret = cm_sidr_rep_handler(work);
3875 break;
3876 case IB_CM_LAP_RECEIVED:
3877 ret = cm_lap_handler(work);
3878 break;
3879 case IB_CM_APR_RECEIVED:
3880 ret = cm_apr_handler(work);
3881 break;
3882 case IB_CM_TIMEWAIT_EXIT:
3883 ret = cm_timewait_handler(work);
3884 break;
3885 default:
3886 trace_icm_handler_err(work->cm_event.event);
3887 ret = -EINVAL;
3888 break;
3889 }
3890 if (ret)
3891 cm_free_work(work);
3892 }
3893
cm_establish(struct ib_cm_id * cm_id)3894 static int cm_establish(struct ib_cm_id *cm_id)
3895 {
3896 struct cm_id_private *cm_id_priv;
3897 struct cm_work *work;
3898 unsigned long flags;
3899 int ret = 0;
3900 struct cm_device *cm_dev;
3901
3902 cm_dev = ib_get_client_data(cm_id->device, &cm_client);
3903 if (!cm_dev)
3904 return -ENODEV;
3905
3906 work = kmalloc(sizeof *work, GFP_ATOMIC);
3907 if (!work)
3908 return -ENOMEM;
3909
3910 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3911 spin_lock_irqsave(&cm_id_priv->lock, flags);
3912 switch (cm_id->state) {
3913 case IB_CM_REP_SENT:
3914 case IB_CM_MRA_REP_RCVD:
3915 cm_id->state = IB_CM_ESTABLISHED;
3916 break;
3917 case IB_CM_ESTABLISHED:
3918 ret = -EISCONN;
3919 break;
3920 default:
3921 trace_icm_establish_err(cm_id);
3922 ret = -EINVAL;
3923 break;
3924 }
3925 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3926
3927 if (ret) {
3928 kfree(work);
3929 goto out;
3930 }
3931
3932 /*
3933 * The CM worker thread may try to destroy the cm_id before it
3934 * can execute this work item. To prevent potential deadlock,
3935 * we need to find the cm_id once we're in the context of the
3936 * worker thread, rather than holding a reference on it.
3937 */
3938 INIT_DELAYED_WORK(&work->work, cm_work_handler);
3939 work->local_id = cm_id->local_id;
3940 work->remote_id = cm_id->remote_id;
3941 work->mad_recv_wc = NULL;
3942 work->cm_event.event = IB_CM_USER_ESTABLISHED;
3943
3944 /* Check if the device started its remove_one */
3945 spin_lock_irqsave(&cm.lock, flags);
3946 if (!cm_dev->going_down) {
3947 queue_delayed_work(cm.wq, &work->work, 0);
3948 } else {
3949 kfree(work);
3950 ret = -ENODEV;
3951 }
3952 spin_unlock_irqrestore(&cm.lock, flags);
3953
3954 out:
3955 return ret;
3956 }
3957
cm_migrate(struct ib_cm_id * cm_id)3958 static int cm_migrate(struct ib_cm_id *cm_id)
3959 {
3960 struct cm_id_private *cm_id_priv;
3961 unsigned long flags;
3962 int ret = 0;
3963
3964 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3965 spin_lock_irqsave(&cm_id_priv->lock, flags);
3966 if (cm_id->state == IB_CM_ESTABLISHED &&
3967 (cm_id->lap_state == IB_CM_LAP_UNINIT ||
3968 cm_id->lap_state == IB_CM_LAP_IDLE)) {
3969 cm_id->lap_state = IB_CM_LAP_IDLE;
3970 cm_id_priv->av = cm_id_priv->alt_av;
3971 } else
3972 ret = -EINVAL;
3973 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3974
3975 return ret;
3976 }
3977
ib_cm_notify(struct ib_cm_id * cm_id,enum ib_event_type event)3978 int ib_cm_notify(struct ib_cm_id *cm_id, enum ib_event_type event)
3979 {
3980 int ret;
3981
3982 switch (event) {
3983 case IB_EVENT_COMM_EST:
3984 ret = cm_establish(cm_id);
3985 break;
3986 case IB_EVENT_PATH_MIG:
3987 ret = cm_migrate(cm_id);
3988 break;
3989 default:
3990 ret = -EINVAL;
3991 }
3992 return ret;
3993 }
3994 EXPORT_SYMBOL(ib_cm_notify);
3995
cm_recv_handler(struct ib_mad_agent * mad_agent,struct ib_mad_send_buf * send_buf,struct ib_mad_recv_wc * mad_recv_wc)3996 static void cm_recv_handler(struct ib_mad_agent *mad_agent,
3997 struct ib_mad_send_buf *send_buf,
3998 struct ib_mad_recv_wc *mad_recv_wc)
3999 {
4000 struct cm_port *port = mad_agent->context;
4001 struct cm_work *work;
4002 enum ib_cm_event_type event;
4003 bool alt_path = false;
4004 u16 attr_id;
4005 int paths = 0;
4006 int going_down = 0;
4007
4008 switch (mad_recv_wc->recv_buf.mad->mad_hdr.attr_id) {
4009 case CM_REQ_ATTR_ID:
4010 alt_path = cm_req_has_alt_path((struct cm_req_msg *)
4011 mad_recv_wc->recv_buf.mad);
4012 paths = 1 + (alt_path != 0);
4013 event = IB_CM_REQ_RECEIVED;
4014 break;
4015 case CM_MRA_ATTR_ID:
4016 event = IB_CM_MRA_RECEIVED;
4017 break;
4018 case CM_REJ_ATTR_ID:
4019 event = IB_CM_REJ_RECEIVED;
4020 break;
4021 case CM_REP_ATTR_ID:
4022 event = IB_CM_REP_RECEIVED;
4023 break;
4024 case CM_RTU_ATTR_ID:
4025 event = IB_CM_RTU_RECEIVED;
4026 break;
4027 case CM_DREQ_ATTR_ID:
4028 event = IB_CM_DREQ_RECEIVED;
4029 break;
4030 case CM_DREP_ATTR_ID:
4031 event = IB_CM_DREP_RECEIVED;
4032 break;
4033 case CM_SIDR_REQ_ATTR_ID:
4034 event = IB_CM_SIDR_REQ_RECEIVED;
4035 break;
4036 case CM_SIDR_REP_ATTR_ID:
4037 event = IB_CM_SIDR_REP_RECEIVED;
4038 break;
4039 case CM_LAP_ATTR_ID:
4040 paths = 1;
4041 event = IB_CM_LAP_RECEIVED;
4042 break;
4043 case CM_APR_ATTR_ID:
4044 event = IB_CM_APR_RECEIVED;
4045 break;
4046 default:
4047 ib_free_recv_mad(mad_recv_wc);
4048 return;
4049 }
4050
4051 attr_id = be16_to_cpu(mad_recv_wc->recv_buf.mad->mad_hdr.attr_id);
4052 atomic_long_inc(&port->counters[CM_RECV][attr_id - CM_ATTR_ID_OFFSET]);
4053
4054 work = kmalloc(struct_size(work, path, paths), GFP_KERNEL);
4055 if (!work) {
4056 ib_free_recv_mad(mad_recv_wc);
4057 return;
4058 }
4059
4060 INIT_DELAYED_WORK(&work->work, cm_work_handler);
4061 work->cm_event.event = event;
4062 work->mad_recv_wc = mad_recv_wc;
4063 work->port = port;
4064
4065 /* Check if the device started its remove_one */
4066 spin_lock_irq(&cm.lock);
4067 if (!port->cm_dev->going_down)
4068 queue_delayed_work(cm.wq, &work->work, 0);
4069 else
4070 going_down = 1;
4071 spin_unlock_irq(&cm.lock);
4072
4073 if (going_down) {
4074 kfree(work);
4075 ib_free_recv_mad(mad_recv_wc);
4076 }
4077 }
4078
cm_init_qp_init_attr(struct cm_id_private * cm_id_priv,struct ib_qp_attr * qp_attr,int * qp_attr_mask)4079 static int cm_init_qp_init_attr(struct cm_id_private *cm_id_priv,
4080 struct ib_qp_attr *qp_attr,
4081 int *qp_attr_mask)
4082 {
4083 unsigned long flags;
4084 int ret;
4085
4086 spin_lock_irqsave(&cm_id_priv->lock, flags);
4087 switch (cm_id_priv->id.state) {
4088 case IB_CM_REQ_SENT:
4089 case IB_CM_MRA_REQ_RCVD:
4090 case IB_CM_REQ_RCVD:
4091 case IB_CM_MRA_REQ_SENT:
4092 case IB_CM_REP_RCVD:
4093 case IB_CM_MRA_REP_SENT:
4094 case IB_CM_REP_SENT:
4095 case IB_CM_MRA_REP_RCVD:
4096 case IB_CM_ESTABLISHED:
4097 *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS |
4098 IB_QP_PKEY_INDEX | IB_QP_PORT;
4099 qp_attr->qp_access_flags = IB_ACCESS_REMOTE_WRITE;
4100 if (cm_id_priv->responder_resources) {
4101 struct ib_device *ib_dev = cm_id_priv->id.device;
4102 u64 support_flush = ib_dev->attrs.device_cap_flags &
4103 (IB_DEVICE_FLUSH_GLOBAL | IB_DEVICE_FLUSH_PERSISTENT);
4104 u32 flushable = support_flush ?
4105 (IB_ACCESS_FLUSH_GLOBAL |
4106 IB_ACCESS_FLUSH_PERSISTENT) : 0;
4107
4108 qp_attr->qp_access_flags |= IB_ACCESS_REMOTE_READ |
4109 IB_ACCESS_REMOTE_ATOMIC |
4110 flushable;
4111 }
4112 qp_attr->pkey_index = cm_id_priv->av.pkey_index;
4113 if (cm_id_priv->av.port)
4114 qp_attr->port_num = cm_id_priv->av.port->port_num;
4115 ret = 0;
4116 break;
4117 default:
4118 trace_icm_qp_init_err(&cm_id_priv->id);
4119 ret = -EINVAL;
4120 break;
4121 }
4122 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4123 return ret;
4124 }
4125
cm_init_qp_rtr_attr(struct cm_id_private * cm_id_priv,struct ib_qp_attr * qp_attr,int * qp_attr_mask)4126 static int cm_init_qp_rtr_attr(struct cm_id_private *cm_id_priv,
4127 struct ib_qp_attr *qp_attr,
4128 int *qp_attr_mask)
4129 {
4130 unsigned long flags;
4131 int ret;
4132
4133 spin_lock_irqsave(&cm_id_priv->lock, flags);
4134 switch (cm_id_priv->id.state) {
4135 case IB_CM_REQ_RCVD:
4136 case IB_CM_MRA_REQ_SENT:
4137 case IB_CM_REP_RCVD:
4138 case IB_CM_MRA_REP_SENT:
4139 case IB_CM_REP_SENT:
4140 case IB_CM_MRA_REP_RCVD:
4141 case IB_CM_ESTABLISHED:
4142 *qp_attr_mask = IB_QP_STATE | IB_QP_AV | IB_QP_PATH_MTU |
4143 IB_QP_DEST_QPN | IB_QP_RQ_PSN;
4144 qp_attr->ah_attr = cm_id_priv->av.ah_attr;
4145 if ((qp_attr->ah_attr.type == RDMA_AH_ATTR_TYPE_IB) &&
4146 cm_id_priv->av.dlid_datapath &&
4147 (cm_id_priv->av.dlid_datapath != 0xffff))
4148 qp_attr->ah_attr.ib.dlid = cm_id_priv->av.dlid_datapath;
4149 qp_attr->path_mtu = cm_id_priv->path_mtu;
4150 qp_attr->dest_qp_num = be32_to_cpu(cm_id_priv->remote_qpn);
4151 qp_attr->rq_psn = be32_to_cpu(cm_id_priv->rq_psn);
4152 if (cm_id_priv->qp_type == IB_QPT_RC ||
4153 cm_id_priv->qp_type == IB_QPT_XRC_TGT) {
4154 *qp_attr_mask |= IB_QP_MAX_DEST_RD_ATOMIC |
4155 IB_QP_MIN_RNR_TIMER;
4156 qp_attr->max_dest_rd_atomic =
4157 cm_id_priv->responder_resources;
4158 qp_attr->min_rnr_timer = 0;
4159 }
4160 if (rdma_ah_get_dlid(&cm_id_priv->alt_av.ah_attr) &&
4161 cm_id_priv->alt_av.port) {
4162 *qp_attr_mask |= IB_QP_ALT_PATH;
4163 qp_attr->alt_port_num = cm_id_priv->alt_av.port->port_num;
4164 qp_attr->alt_pkey_index = cm_id_priv->alt_av.pkey_index;
4165 qp_attr->alt_timeout = cm_id_priv->alt_av.timeout;
4166 qp_attr->alt_ah_attr = cm_id_priv->alt_av.ah_attr;
4167 }
4168 ret = 0;
4169 break;
4170 default:
4171 trace_icm_qp_rtr_err(&cm_id_priv->id);
4172 ret = -EINVAL;
4173 break;
4174 }
4175 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4176 return ret;
4177 }
4178
cm_init_qp_rts_attr(struct cm_id_private * cm_id_priv,struct ib_qp_attr * qp_attr,int * qp_attr_mask)4179 static int cm_init_qp_rts_attr(struct cm_id_private *cm_id_priv,
4180 struct ib_qp_attr *qp_attr,
4181 int *qp_attr_mask)
4182 {
4183 unsigned long flags;
4184 int ret;
4185
4186 spin_lock_irqsave(&cm_id_priv->lock, flags);
4187 switch (cm_id_priv->id.state) {
4188 /* Allow transition to RTS before sending REP */
4189 case IB_CM_REQ_RCVD:
4190 case IB_CM_MRA_REQ_SENT:
4191
4192 case IB_CM_REP_RCVD:
4193 case IB_CM_MRA_REP_SENT:
4194 case IB_CM_REP_SENT:
4195 case IB_CM_MRA_REP_RCVD:
4196 case IB_CM_ESTABLISHED:
4197 if (cm_id_priv->id.lap_state == IB_CM_LAP_UNINIT) {
4198 *qp_attr_mask = IB_QP_STATE | IB_QP_SQ_PSN;
4199 qp_attr->sq_psn = be32_to_cpu(cm_id_priv->sq_psn);
4200 switch (cm_id_priv->qp_type) {
4201 case IB_QPT_RC:
4202 case IB_QPT_XRC_INI:
4203 *qp_attr_mask |= IB_QP_RETRY_CNT | IB_QP_RNR_RETRY |
4204 IB_QP_MAX_QP_RD_ATOMIC;
4205 qp_attr->retry_cnt = cm_id_priv->retry_count;
4206 qp_attr->rnr_retry = cm_id_priv->rnr_retry_count;
4207 qp_attr->max_rd_atomic = cm_id_priv->initiator_depth;
4208 fallthrough;
4209 case IB_QPT_XRC_TGT:
4210 *qp_attr_mask |= IB_QP_TIMEOUT;
4211 qp_attr->timeout = cm_id_priv->av.timeout;
4212 break;
4213 default:
4214 break;
4215 }
4216 if (rdma_ah_get_dlid(&cm_id_priv->alt_av.ah_attr)) {
4217 *qp_attr_mask |= IB_QP_PATH_MIG_STATE;
4218 qp_attr->path_mig_state = IB_MIG_REARM;
4219 }
4220 } else {
4221 *qp_attr_mask = IB_QP_ALT_PATH | IB_QP_PATH_MIG_STATE;
4222 if (cm_id_priv->alt_av.port)
4223 qp_attr->alt_port_num =
4224 cm_id_priv->alt_av.port->port_num;
4225 qp_attr->alt_pkey_index = cm_id_priv->alt_av.pkey_index;
4226 qp_attr->alt_timeout = cm_id_priv->alt_av.timeout;
4227 qp_attr->alt_ah_attr = cm_id_priv->alt_av.ah_attr;
4228 qp_attr->path_mig_state = IB_MIG_REARM;
4229 }
4230 ret = 0;
4231 break;
4232 default:
4233 trace_icm_qp_rts_err(&cm_id_priv->id);
4234 ret = -EINVAL;
4235 break;
4236 }
4237 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4238 return ret;
4239 }
4240
ib_cm_init_qp_attr(struct ib_cm_id * cm_id,struct ib_qp_attr * qp_attr,int * qp_attr_mask)4241 int ib_cm_init_qp_attr(struct ib_cm_id *cm_id,
4242 struct ib_qp_attr *qp_attr,
4243 int *qp_attr_mask)
4244 {
4245 struct cm_id_private *cm_id_priv;
4246 int ret;
4247
4248 cm_id_priv = container_of(cm_id, struct cm_id_private, id);
4249 switch (qp_attr->qp_state) {
4250 case IB_QPS_INIT:
4251 ret = cm_init_qp_init_attr(cm_id_priv, qp_attr, qp_attr_mask);
4252 break;
4253 case IB_QPS_RTR:
4254 ret = cm_init_qp_rtr_attr(cm_id_priv, qp_attr, qp_attr_mask);
4255 break;
4256 case IB_QPS_RTS:
4257 ret = cm_init_qp_rts_attr(cm_id_priv, qp_attr, qp_attr_mask);
4258 break;
4259 default:
4260 ret = -EINVAL;
4261 break;
4262 }
4263 return ret;
4264 }
4265 EXPORT_SYMBOL(ib_cm_init_qp_attr);
4266
cm_show_counter(struct ib_device * ibdev,u32 port_num,struct ib_port_attribute * attr,char * buf)4267 static ssize_t cm_show_counter(struct ib_device *ibdev, u32 port_num,
4268 struct ib_port_attribute *attr, char *buf)
4269 {
4270 struct cm_counter_attribute *cm_attr =
4271 container_of(attr, struct cm_counter_attribute, attr);
4272 struct cm_device *cm_dev = ib_get_client_data(ibdev, &cm_client);
4273
4274 if (WARN_ON(!cm_dev))
4275 return -EINVAL;
4276
4277 return sysfs_emit(
4278 buf, "%ld\n",
4279 atomic_long_read(
4280 &cm_dev->port[port_num - 1]
4281 ->counters[cm_attr->group][cm_attr->index]));
4282 }
4283
4284 #define CM_COUNTER_ATTR(_name, _group, _index) \
4285 { \
4286 .attr = __ATTR(_name, 0444, cm_show_counter, NULL), \
4287 .group = _group, .index = _index \
4288 }
4289
4290 #define CM_COUNTER_GROUP(_group, _name) \
4291 static struct cm_counter_attribute cm_counter_attr_##_group[] = { \
4292 CM_COUNTER_ATTR(req, _group, CM_REQ_COUNTER), \
4293 CM_COUNTER_ATTR(mra, _group, CM_MRA_COUNTER), \
4294 CM_COUNTER_ATTR(rej, _group, CM_REJ_COUNTER), \
4295 CM_COUNTER_ATTR(rep, _group, CM_REP_COUNTER), \
4296 CM_COUNTER_ATTR(rtu, _group, CM_RTU_COUNTER), \
4297 CM_COUNTER_ATTR(dreq, _group, CM_DREQ_COUNTER), \
4298 CM_COUNTER_ATTR(drep, _group, CM_DREP_COUNTER), \
4299 CM_COUNTER_ATTR(sidr_req, _group, CM_SIDR_REQ_COUNTER), \
4300 CM_COUNTER_ATTR(sidr_rep, _group, CM_SIDR_REP_COUNTER), \
4301 CM_COUNTER_ATTR(lap, _group, CM_LAP_COUNTER), \
4302 CM_COUNTER_ATTR(apr, _group, CM_APR_COUNTER), \
4303 }; \
4304 static struct attribute *cm_counter_attrs_##_group[] = { \
4305 &cm_counter_attr_##_group[0].attr.attr, \
4306 &cm_counter_attr_##_group[1].attr.attr, \
4307 &cm_counter_attr_##_group[2].attr.attr, \
4308 &cm_counter_attr_##_group[3].attr.attr, \
4309 &cm_counter_attr_##_group[4].attr.attr, \
4310 &cm_counter_attr_##_group[5].attr.attr, \
4311 &cm_counter_attr_##_group[6].attr.attr, \
4312 &cm_counter_attr_##_group[7].attr.attr, \
4313 &cm_counter_attr_##_group[8].attr.attr, \
4314 &cm_counter_attr_##_group[9].attr.attr, \
4315 &cm_counter_attr_##_group[10].attr.attr, \
4316 NULL, \
4317 }; \
4318 static const struct attribute_group cm_counter_group_##_group = { \
4319 .name = _name, \
4320 .attrs = cm_counter_attrs_##_group, \
4321 };
4322
4323 CM_COUNTER_GROUP(CM_XMIT, "cm_tx_msgs")
4324 CM_COUNTER_GROUP(CM_XMIT_RETRIES, "cm_tx_retries")
4325 CM_COUNTER_GROUP(CM_RECV, "cm_rx_msgs")
4326 CM_COUNTER_GROUP(CM_RECV_DUPLICATES, "cm_rx_duplicates")
4327
4328 static const struct attribute_group *cm_counter_groups[] = {
4329 &cm_counter_group_CM_XMIT,
4330 &cm_counter_group_CM_XMIT_RETRIES,
4331 &cm_counter_group_CM_RECV,
4332 &cm_counter_group_CM_RECV_DUPLICATES,
4333 NULL,
4334 };
4335
cm_add_one(struct ib_device * ib_device)4336 static int cm_add_one(struct ib_device *ib_device)
4337 {
4338 struct cm_device *cm_dev;
4339 struct cm_port *port;
4340 struct ib_mad_reg_req reg_req = {
4341 .mgmt_class = IB_MGMT_CLASS_CM,
4342 .mgmt_class_version = IB_CM_CLASS_VERSION,
4343 };
4344 struct ib_port_modify port_modify = {
4345 .set_port_cap_mask = IB_PORT_CM_SUP
4346 };
4347 unsigned long flags;
4348 int ret;
4349 int count = 0;
4350 u32 i;
4351
4352 cm_dev = kzalloc(struct_size(cm_dev, port, ib_device->phys_port_cnt),
4353 GFP_KERNEL);
4354 if (!cm_dev)
4355 return -ENOMEM;
4356
4357 kref_init(&cm_dev->kref);
4358 rwlock_init(&cm_dev->mad_agent_lock);
4359 cm_dev->ib_device = ib_device;
4360 cm_dev->ack_delay = ib_device->attrs.local_ca_ack_delay;
4361 cm_dev->going_down = 0;
4362
4363 ib_set_client_data(ib_device, &cm_client, cm_dev);
4364
4365 set_bit(IB_MGMT_METHOD_SEND, reg_req.method_mask);
4366 rdma_for_each_port (ib_device, i) {
4367 if (!rdma_cap_ib_cm(ib_device, i))
4368 continue;
4369
4370 port = kzalloc(sizeof *port, GFP_KERNEL);
4371 if (!port) {
4372 ret = -ENOMEM;
4373 goto error1;
4374 }
4375
4376 cm_dev->port[i-1] = port;
4377 port->cm_dev = cm_dev;
4378 port->port_num = i;
4379
4380 ret = ib_port_register_client_groups(ib_device, i,
4381 cm_counter_groups);
4382 if (ret)
4383 goto error1;
4384
4385 port->mad_agent = ib_register_mad_agent(ib_device, i,
4386 IB_QPT_GSI,
4387 ®_req,
4388 0,
4389 cm_send_handler,
4390 cm_recv_handler,
4391 port,
4392 0);
4393 if (IS_ERR(port->mad_agent)) {
4394 ret = PTR_ERR(port->mad_agent);
4395 goto error2;
4396 }
4397
4398 port->rep_agent = ib_register_mad_agent(ib_device, i,
4399 IB_QPT_GSI,
4400 NULL,
4401 0,
4402 cm_send_handler,
4403 NULL,
4404 port,
4405 0);
4406 if (IS_ERR(port->rep_agent)) {
4407 ret = PTR_ERR(port->rep_agent);
4408 goto error3;
4409 }
4410
4411 ret = ib_modify_port(ib_device, i, 0, &port_modify);
4412 if (ret)
4413 goto error4;
4414
4415 count++;
4416 }
4417
4418 if (!count) {
4419 ret = -EOPNOTSUPP;
4420 goto free;
4421 }
4422
4423 write_lock_irqsave(&cm.device_lock, flags);
4424 list_add_tail(&cm_dev->list, &cm.device_list);
4425 write_unlock_irqrestore(&cm.device_lock, flags);
4426 return 0;
4427
4428 error4:
4429 ib_unregister_mad_agent(port->rep_agent);
4430 error3:
4431 ib_unregister_mad_agent(port->mad_agent);
4432 error2:
4433 ib_port_unregister_client_groups(ib_device, i, cm_counter_groups);
4434 error1:
4435 port_modify.set_port_cap_mask = 0;
4436 port_modify.clr_port_cap_mask = IB_PORT_CM_SUP;
4437 while (--i) {
4438 if (!rdma_cap_ib_cm(ib_device, i))
4439 continue;
4440
4441 port = cm_dev->port[i-1];
4442 ib_modify_port(ib_device, port->port_num, 0, &port_modify);
4443 ib_unregister_mad_agent(port->rep_agent);
4444 ib_unregister_mad_agent(port->mad_agent);
4445 ib_port_unregister_client_groups(ib_device, i,
4446 cm_counter_groups);
4447 }
4448 free:
4449 cm_device_put(cm_dev);
4450 return ret;
4451 }
4452
cm_remove_one(struct ib_device * ib_device,void * client_data)4453 static void cm_remove_one(struct ib_device *ib_device, void *client_data)
4454 {
4455 struct cm_device *cm_dev = client_data;
4456 struct cm_port *port;
4457 struct ib_port_modify port_modify = {
4458 .clr_port_cap_mask = IB_PORT_CM_SUP
4459 };
4460 unsigned long flags;
4461 u32 i;
4462
4463 write_lock_irqsave(&cm.device_lock, flags);
4464 list_del(&cm_dev->list);
4465 write_unlock_irqrestore(&cm.device_lock, flags);
4466
4467 spin_lock_irq(&cm.lock);
4468 cm_dev->going_down = 1;
4469 spin_unlock_irq(&cm.lock);
4470
4471 rdma_for_each_port (ib_device, i) {
4472 struct ib_mad_agent *mad_agent;
4473 struct ib_mad_agent *rep_agent;
4474
4475 if (!rdma_cap_ib_cm(ib_device, i))
4476 continue;
4477
4478 port = cm_dev->port[i-1];
4479 mad_agent = port->mad_agent;
4480 rep_agent = port->rep_agent;
4481 ib_modify_port(ib_device, port->port_num, 0, &port_modify);
4482 /*
4483 * We flush the queue here after the going_down set, this
4484 * verify that no new works will be queued in the recv handler,
4485 * after that we can call the unregister_mad_agent
4486 */
4487 flush_workqueue(cm.wq);
4488 /*
4489 * The above ensures no call paths from the work are running,
4490 * the remaining paths all take the mad_agent_lock.
4491 */
4492 write_lock(&cm_dev->mad_agent_lock);
4493 port->mad_agent = NULL;
4494 port->rep_agent = NULL;
4495 write_unlock(&cm_dev->mad_agent_lock);
4496 ib_unregister_mad_agent(mad_agent);
4497 ib_unregister_mad_agent(rep_agent);
4498 ib_port_unregister_client_groups(ib_device, i,
4499 cm_counter_groups);
4500 }
4501
4502 cm_device_put(cm_dev);
4503 }
4504
ib_cm_init(void)4505 static int __init ib_cm_init(void)
4506 {
4507 int ret;
4508
4509 INIT_LIST_HEAD(&cm.device_list);
4510 rwlock_init(&cm.device_lock);
4511 spin_lock_init(&cm.lock);
4512 cm.listen_service_table = RB_ROOT;
4513 cm.listen_service_id = be64_to_cpu(IB_CM_ASSIGN_SERVICE_ID);
4514 cm.remote_id_table = RB_ROOT;
4515 cm.remote_qp_table = RB_ROOT;
4516 cm.remote_sidr_table = RB_ROOT;
4517 xa_init_flags(&cm.local_id_table, XA_FLAGS_ALLOC);
4518 get_random_bytes(&cm.random_id_operand, sizeof cm.random_id_operand);
4519 INIT_LIST_HEAD(&cm.timewait_list);
4520
4521 cm.wq = alloc_workqueue("ib_cm", 0, 1);
4522 if (!cm.wq) {
4523 ret = -ENOMEM;
4524 goto error2;
4525 }
4526
4527 ret = ib_register_client(&cm_client);
4528 if (ret)
4529 goto error3;
4530
4531 return 0;
4532 error3:
4533 destroy_workqueue(cm.wq);
4534 error2:
4535 return ret;
4536 }
4537
ib_cm_cleanup(void)4538 static void __exit ib_cm_cleanup(void)
4539 {
4540 struct cm_timewait_info *timewait_info, *tmp;
4541
4542 spin_lock_irq(&cm.lock);
4543 list_for_each_entry(timewait_info, &cm.timewait_list, list)
4544 cancel_delayed_work(&timewait_info->work.work);
4545 spin_unlock_irq(&cm.lock);
4546
4547 ib_unregister_client(&cm_client);
4548 destroy_workqueue(cm.wq);
4549
4550 list_for_each_entry_safe(timewait_info, tmp, &cm.timewait_list, list) {
4551 list_del(&timewait_info->list);
4552 kfree(timewait_info);
4553 }
4554
4555 WARN_ON(!xa_empty(&cm.local_id_table));
4556 }
4557
4558 module_init(ib_cm_init);
4559 module_exit(ib_cm_cleanup);
4560