1 /*
2 * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved.
3 * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33 #define _GNU_SOURCE
34 #include <config.h>
35
36 #include <infiniband/endian.h>
37 #include <stdio.h>
38 #include <sys/types.h>
39 #include <sys/stat.h>
40 #include <fcntl.h>
41 #include <unistd.h>
42 #include <stdlib.h>
43 #include <errno.h>
44
45 #include "ibverbs.h"
46
47 /* Hack to avoid GCC's -Wmissing-prototypes and the similar error from sparse
48 with these prototypes. Symbol versionining requires the goofy names, the
49 prototype must match the version in verbs.h.
50 */
51 struct ibv_device **__ibv_get_device_list(int *num_devices);
52 void __ibv_free_device_list(struct ibv_device **list);
53 const char *__ibv_get_device_name(struct ibv_device *device);
54 __be64 __ibv_get_device_guid(struct ibv_device *device);
55 struct ibv_context *__ibv_open_device(struct ibv_device *device);
56 int __ibv_close_device(struct ibv_context *context);
57 int __ibv_get_async_event(struct ibv_context *context,
58 struct ibv_async_event *event);
59 void __ibv_ack_async_event(struct ibv_async_event *event);
60
61 static pthread_once_t device_list_once = PTHREAD_ONCE_INIT;
62 static int num_devices;
63 static struct ibv_device **device_list;
64
count_devices(void)65 static void count_devices(void)
66 {
67 num_devices = ibverbs_init(&device_list);
68 }
69
__ibv_get_device_list(int * num)70 struct ibv_device **__ibv_get_device_list(int *num)
71 {
72 struct ibv_device **l;
73 int i;
74
75 if (num)
76 *num = 0;
77
78 pthread_once(&device_list_once, count_devices);
79
80 if (num_devices < 0) {
81 errno = -num_devices;
82 return NULL;
83 }
84
85 l = calloc(num_devices + 1, sizeof (struct ibv_device *));
86 if (!l) {
87 errno = ENOMEM;
88 return NULL;
89 }
90
91 for (i = 0; i < num_devices; ++i)
92 l[i] = device_list[i];
93 if (num)
94 *num = num_devices;
95
96 return l;
97 }
98 default_symver(__ibv_get_device_list, ibv_get_device_list);
99
__ibv_free_device_list(struct ibv_device ** list)100 void __ibv_free_device_list(struct ibv_device **list)
101 {
102 free(list);
103 }
104 default_symver(__ibv_free_device_list, ibv_free_device_list);
105
__ibv_get_device_name(struct ibv_device * device)106 const char *__ibv_get_device_name(struct ibv_device *device)
107 {
108 return device->name;
109 }
110 default_symver(__ibv_get_device_name, ibv_get_device_name);
111
__ibv_get_device_guid(struct ibv_device * device)112 __be64 __ibv_get_device_guid(struct ibv_device *device)
113 {
114 char attr[24];
115 uint64_t guid = 0;
116 uint16_t parts[4];
117 int i;
118
119 if (ibv_read_sysfs_file(device->ibdev_path, "node_guid",
120 attr, sizeof attr) < 0)
121 return 0;
122
123 if (sscanf(attr, "%hx:%hx:%hx:%hx",
124 parts, parts + 1, parts + 2, parts + 3) != 4)
125 return 0;
126
127 for (i = 0; i < 4; ++i)
128 guid = (guid << 16) | parts[i];
129
130 return htobe64(guid);
131 }
132 default_symver(__ibv_get_device_guid, ibv_get_device_guid);
133
verbs_init_cq(struct ibv_cq * cq,struct ibv_context * context,struct ibv_comp_channel * channel,void * cq_context)134 int verbs_init_cq(struct ibv_cq *cq, struct ibv_context *context,
135 struct ibv_comp_channel *channel,
136 void *cq_context)
137 {
138 int err = 0;
139
140 cq->context = context;
141 cq->channel = channel;
142
143 err = pthread_mutex_init(&cq->mutex, NULL);
144 if (err)
145 return err;
146 err = pthread_cond_init(&cq->cond, NULL);
147 if (err)
148 goto err;
149
150 if (cq->channel) {
151 pthread_mutex_lock(&context->mutex);
152 ++cq->channel->refcnt;
153 pthread_mutex_unlock(&context->mutex);
154 }
155
156 cq->cq_context = cq_context;
157 cq->comp_events_completed = 0;
158 cq->async_events_completed = 0;
159
160 return err;
161
162 err:
163 pthread_mutex_destroy(&cq->mutex);
164
165 return err;
166 }
167
verbs_cleanup_cq(struct ibv_cq * cq)168 void verbs_cleanup_cq(struct ibv_cq *cq)
169 {
170 pthread_cond_destroy(&cq->cond);
171 pthread_mutex_destroy(&cq->mutex);
172 }
173
174 static struct ibv_cq_ex *
__lib_ibv_create_cq_ex(struct ibv_context * context,struct ibv_cq_init_attr_ex * cq_attr)175 __lib_ibv_create_cq_ex(struct ibv_context *context,
176 struct ibv_cq_init_attr_ex *cq_attr)
177 {
178 struct verbs_context *vctx = verbs_get_ctx(context);
179 struct ibv_cq_ex *cq;
180 int err = 0;
181
182 if (cq_attr->wc_flags & ~IBV_CREATE_CQ_SUP_WC_FLAGS) {
183 errno = EOPNOTSUPP;
184 return NULL;
185 }
186
187 cq = vctx->priv->create_cq_ex(context, cq_attr);
188 if (!cq)
189 return NULL;
190
191 err = verbs_init_cq(ibv_cq_ex_to_cq(cq), context,
192 cq_attr->channel, cq_attr->cq_context);
193 if (err)
194 goto err;
195
196 return cq;
197
198 err:
199 context->ops.destroy_cq(ibv_cq_ex_to_cq(cq));
200
201 return NULL;
202 }
203
__ibv_open_device(struct ibv_device * device)204 struct ibv_context *__ibv_open_device(struct ibv_device *device)
205 {
206 struct verbs_device *verbs_device = verbs_get_device(device);
207 char *devpath;
208 int cmd_fd, ret;
209 struct ibv_context *context;
210 struct verbs_context *context_ex;
211
212 if (asprintf(&devpath, "/dev/%s", device->dev_name) < 0)
213 return NULL;
214
215 /*
216 * We'll only be doing writes, but we need O_RDWR in case the
217 * provider needs to mmap() the file.
218 */
219 cmd_fd = open(devpath, O_RDWR | O_CLOEXEC);
220 free(devpath);
221
222 if (cmd_fd < 0)
223 return NULL;
224
225 if (!verbs_device->ops->init_context) {
226 context = verbs_device->ops->alloc_context(device, cmd_fd);
227 if (!context)
228 goto err;
229
230 if (pthread_mutex_init(&context->mutex, NULL)) {
231 verbs_device->ops->free_context(context);
232 goto err;
233 }
234 } else {
235 struct verbs_ex_private *priv;
236
237 /* Library now allocates the context */
238 context_ex = calloc(1, sizeof(*context_ex) +
239 verbs_device->size_of_context);
240 if (!context_ex) {
241 errno = ENOMEM;
242 goto err;
243 }
244
245 priv = calloc(1, sizeof(*priv));
246 if (!priv) {
247 errno = ENOMEM;
248 goto err_context;
249 }
250
251 context_ex->priv = priv;
252 context_ex->context.abi_compat = __VERBS_ABI_IS_EXTENDED;
253 context_ex->sz = sizeof(*context_ex);
254
255 context = &context_ex->context;
256 if (pthread_mutex_init(&context->mutex, NULL))
257 goto verbs_err;
258
259 ret = verbs_device->ops->init_context(verbs_device, context, cmd_fd);
260 if (ret)
261 goto err_mutex;
262 /*
263 * In order to maintain backward/forward binary compatibility
264 * with apps compiled against libibverbs-1.1.8 that use the
265 * flow steering addition, we need to set the two
266 * ABI_placeholder entries to match the driver set flow
267 * entries. This is because apps compiled against
268 * libibverbs-1.1.8 use an inline ibv_create_flow and
269 * ibv_destroy_flow function that looks in the placeholder
270 * spots for the proper entry points. For apps compiled
271 * against libibverbs-1.1.9 and later, the inline functions
272 * will be looking in the right place.
273 */
274 context_ex->ABI_placeholder1 = (void (*)(void)) context_ex->ibv_create_flow;
275 context_ex->ABI_placeholder2 = (void (*)(void)) context_ex->ibv_destroy_flow;
276
277 if (context_ex->create_cq_ex) {
278 priv->create_cq_ex = context_ex->create_cq_ex;
279 context_ex->create_cq_ex = __lib_ibv_create_cq_ex;
280 }
281 }
282
283 context->device = device;
284 context->cmd_fd = cmd_fd;
285
286 return context;
287
288 err_mutex:
289 pthread_mutex_destroy(&context->mutex);
290 verbs_err:
291 free(context_ex->priv);
292 err_context:
293 free(context_ex);
294 err:
295 close(cmd_fd);
296 return NULL;
297 }
298 default_symver(__ibv_open_device, ibv_open_device);
299
__ibv_close_device(struct ibv_context * context)300 int __ibv_close_device(struct ibv_context *context)
301 {
302 int async_fd = context->async_fd;
303 int cmd_fd = context->cmd_fd;
304 struct verbs_context *context_ex;
305 struct verbs_device *verbs_device = verbs_get_device(context->device);
306
307 pthread_mutex_destroy(&context->mutex);
308 context_ex = verbs_get_ctx(context);
309 if (context_ex) {
310 verbs_device->ops->uninit_context(verbs_device, context);
311 free(context_ex->priv);
312 free(context_ex);
313 } else {
314 verbs_device->ops->free_context(context);
315 }
316
317 close(async_fd);
318 close(cmd_fd);
319
320 return 0;
321 }
322 default_symver(__ibv_close_device, ibv_close_device);
323
__ibv_get_async_event(struct ibv_context * context,struct ibv_async_event * event)324 int __ibv_get_async_event(struct ibv_context *context,
325 struct ibv_async_event *event)
326 {
327 struct ibv_kern_async_event ev;
328
329 if (read(context->async_fd, &ev, sizeof ev) != sizeof ev)
330 return -1;
331
332 event->event_type = ev.event_type;
333
334 switch (event->event_type) {
335 case IBV_EVENT_CQ_ERR:
336 event->element.cq = (void *) (uintptr_t) ev.element;
337 break;
338
339 case IBV_EVENT_QP_FATAL:
340 case IBV_EVENT_QP_REQ_ERR:
341 case IBV_EVENT_QP_ACCESS_ERR:
342 case IBV_EVENT_COMM_EST:
343 case IBV_EVENT_SQ_DRAINED:
344 case IBV_EVENT_PATH_MIG:
345 case IBV_EVENT_PATH_MIG_ERR:
346 case IBV_EVENT_QP_LAST_WQE_REACHED:
347 event->element.qp = (void *) (uintptr_t) ev.element;
348 break;
349
350 case IBV_EVENT_SRQ_ERR:
351 case IBV_EVENT_SRQ_LIMIT_REACHED:
352 event->element.srq = (void *) (uintptr_t) ev.element;
353 break;
354
355 case IBV_EVENT_WQ_FATAL:
356 event->element.wq = (void *) (uintptr_t) ev.element;
357 break;
358 default:
359 event->element.port_num = ev.element;
360 break;
361 }
362
363 if (context->ops.async_event)
364 context->ops.async_event(event);
365
366 return 0;
367 }
368 default_symver(__ibv_get_async_event, ibv_get_async_event);
369
__ibv_ack_async_event(struct ibv_async_event * event)370 void __ibv_ack_async_event(struct ibv_async_event *event)
371 {
372 switch (event->event_type) {
373 case IBV_EVENT_CQ_ERR:
374 {
375 struct ibv_cq *cq = event->element.cq;
376
377 pthread_mutex_lock(&cq->mutex);
378 ++cq->async_events_completed;
379 pthread_cond_signal(&cq->cond);
380 pthread_mutex_unlock(&cq->mutex);
381
382 return;
383 }
384
385 case IBV_EVENT_QP_FATAL:
386 case IBV_EVENT_QP_REQ_ERR:
387 case IBV_EVENT_QP_ACCESS_ERR:
388 case IBV_EVENT_COMM_EST:
389 case IBV_EVENT_SQ_DRAINED:
390 case IBV_EVENT_PATH_MIG:
391 case IBV_EVENT_PATH_MIG_ERR:
392 case IBV_EVENT_QP_LAST_WQE_REACHED:
393 {
394 struct ibv_qp *qp = event->element.qp;
395
396 pthread_mutex_lock(&qp->mutex);
397 ++qp->events_completed;
398 pthread_cond_signal(&qp->cond);
399 pthread_mutex_unlock(&qp->mutex);
400
401 return;
402 }
403
404 case IBV_EVENT_SRQ_ERR:
405 case IBV_EVENT_SRQ_LIMIT_REACHED:
406 {
407 struct ibv_srq *srq = event->element.srq;
408
409 pthread_mutex_lock(&srq->mutex);
410 ++srq->events_completed;
411 pthread_cond_signal(&srq->cond);
412 pthread_mutex_unlock(&srq->mutex);
413
414 return;
415 }
416
417 case IBV_EVENT_WQ_FATAL:
418 {
419 struct ibv_wq *wq = event->element.wq;
420
421 pthread_mutex_lock(&wq->mutex);
422 ++wq->events_completed;
423 pthread_cond_signal(&wq->cond);
424 pthread_mutex_unlock(&wq->mutex);
425
426 return;
427 }
428
429 default:
430 return;
431 }
432 }
433 default_symver(__ibv_ack_async_event, ibv_ack_async_event);
434
__ibv_init_wq(struct ibv_wq * wq)435 int __ibv_init_wq(struct ibv_wq *wq)
436 {
437 int err = 0;
438 wq->events_completed = 0;
439 err = pthread_mutex_init(&wq->mutex, NULL);
440 if (err)
441 return err;
442
443 err = pthread_cond_init(&wq->cond, NULL);
444 if (err)
445 goto err;
446
447 return err;
448
449 err:
450 pthread_mutex_destroy(&wq->mutex);
451
452 return err;
453 }
454 default_symver(__ibv_init_wq, ibv_init_wq);
455
__ibv_cleanup_wq(struct ibv_wq * wq)456 void __ibv_cleanup_wq(struct ibv_wq *wq)
457 {
458 pthread_cond_destroy(&wq->cond);
459 pthread_mutex_destroy(&wq->mutex);
460 }
461 default_symver(__ibv_cleanup_wq, ibv_cleanup_wq);
462