1 /*
2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #include <linux/highmem.h>
34 #include <linux/module.h>
35 #include <linux/init.h>
36 #include <linux/errno.h>
37 #include <linux/pci.h>
38 #include <linux/dma-mapping.h>
39 #include <linux/slab.h>
40 #include <linux/interrupt.h>
41 #include <linux/delay.h>
42 #include <linux/mlx5/driver.h>
43 #include <linux/mlx5/cq.h>
44 #include <linux/mlx5/qp.h>
45 #include <linux/debugfs.h>
46 #include <linux/kmod.h>
47 #include <linux/mlx5/mlx5_ifc.h>
48 #include <linux/mlx5/vport.h>
49 #include <linux/version.h>
50 #include <net/devlink.h>
51 #include "mlx5_core.h"
52 #include "lib/eq.h"
53 #include "fs_core.h"
54 #include "lib/mpfs.h"
55 #include "eswitch.h"
56 #include "devlink.h"
57 #include "fw_reset.h"
58 #include "lib/mlx5.h"
59 #include "lib/tout.h"
60 #include "fpga/core.h"
61 #include "en_accel/ipsec.h"
62 #include "lib/clock.h"
63 #include "lib/vxlan.h"
64 #include "lib/geneve.h"
65 #include "lib/devcom.h"
66 #include "lib/pci_vsc.h"
67 #include "diag/fw_tracer.h"
68 #include "ecpf.h"
69 #include "lib/hv_vhca.h"
70 #include "diag/rsc_dump.h"
71 #include "sf/vhca_event.h"
72 #include "sf/dev/dev.h"
73 #include "sf/sf.h"
74 #include "mlx5_irq.h"
75 #include "hwmon.h"
76 #include "lag/lag.h"
77
78 MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
79 MODULE_DESCRIPTION("Mellanox 5th generation network adapters (ConnectX series) core driver");
80 MODULE_LICENSE("Dual BSD/GPL");
81
82 unsigned int mlx5_core_debug_mask;
83 module_param_named(debug_mask, mlx5_core_debug_mask, uint, 0644);
84 MODULE_PARM_DESC(debug_mask, "debug mask: 1 = dump cmd data, 2 = dump cmd exec time, 3 = both. Default=0");
85
86 static unsigned int prof_sel = MLX5_DEFAULT_PROF;
87 module_param_named(prof_sel, prof_sel, uint, 0444);
88 MODULE_PARM_DESC(prof_sel, "profile selector. Valid range 0 - 2");
89
90 static u32 sw_owner_id[4];
91 #define MAX_SW_VHCA_ID (BIT(__mlx5_bit_sz(cmd_hca_cap_2, sw_vhca_id)) - 1)
92 static DEFINE_IDA(sw_vhca_ida);
93
94 enum {
95 MLX5_ATOMIC_REQ_MODE_BE = 0x0,
96 MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS = 0x1,
97 };
98
99 #define LOG_MAX_SUPPORTED_QPS 0xff
100
101 static struct mlx5_profile profile[] = {
102 [0] = {
103 .mask = 0,
104 .num_cmd_caches = MLX5_NUM_COMMAND_CACHES,
105 },
106 [1] = {
107 .mask = MLX5_PROF_MASK_QP_SIZE,
108 .log_max_qp = 12,
109 .num_cmd_caches = MLX5_NUM_COMMAND_CACHES,
110
111 },
112 [2] = {
113 .mask = MLX5_PROF_MASK_QP_SIZE |
114 MLX5_PROF_MASK_MR_CACHE,
115 .log_max_qp = LOG_MAX_SUPPORTED_QPS,
116 .num_cmd_caches = MLX5_NUM_COMMAND_CACHES,
117 .mr_cache[0] = {
118 .size = 500,
119 .limit = 250
120 },
121 .mr_cache[1] = {
122 .size = 500,
123 .limit = 250
124 },
125 .mr_cache[2] = {
126 .size = 500,
127 .limit = 250
128 },
129 .mr_cache[3] = {
130 .size = 500,
131 .limit = 250
132 },
133 .mr_cache[4] = {
134 .size = 500,
135 .limit = 250
136 },
137 .mr_cache[5] = {
138 .size = 500,
139 .limit = 250
140 },
141 .mr_cache[6] = {
142 .size = 500,
143 .limit = 250
144 },
145 .mr_cache[7] = {
146 .size = 500,
147 .limit = 250
148 },
149 .mr_cache[8] = {
150 .size = 500,
151 .limit = 250
152 },
153 .mr_cache[9] = {
154 .size = 500,
155 .limit = 250
156 },
157 .mr_cache[10] = {
158 .size = 500,
159 .limit = 250
160 },
161 .mr_cache[11] = {
162 .size = 500,
163 .limit = 250
164 },
165 .mr_cache[12] = {
166 .size = 64,
167 .limit = 32
168 },
169 .mr_cache[13] = {
170 .size = 32,
171 .limit = 16
172 },
173 .mr_cache[14] = {
174 .size = 16,
175 .limit = 8
176 },
177 .mr_cache[15] = {
178 .size = 8,
179 .limit = 4
180 },
181 },
182 [3] = {
183 .mask = MLX5_PROF_MASK_QP_SIZE,
184 .log_max_qp = LOG_MAX_SUPPORTED_QPS,
185 .num_cmd_caches = 0,
186 },
187 };
188
wait_fw_init(struct mlx5_core_dev * dev,u32 max_wait_mili,u32 warn_time_mili)189 static int wait_fw_init(struct mlx5_core_dev *dev, u32 max_wait_mili,
190 u32 warn_time_mili)
191 {
192 unsigned long warn = jiffies + msecs_to_jiffies(warn_time_mili);
193 unsigned long end = jiffies + msecs_to_jiffies(max_wait_mili);
194 u32 fw_initializing;
195 int err = 0;
196
197 do {
198 fw_initializing = ioread32be(&dev->iseg->initializing);
199 if (!(fw_initializing >> 31))
200 break;
201 if (time_after(jiffies, end) ||
202 test_bit(MLX5_BREAK_FW_WAIT, &dev->intf_state)) {
203 err = -EBUSY;
204 break;
205 }
206 if (warn_time_mili && time_after(jiffies, warn)) {
207 mlx5_core_warn(dev, "Waiting for FW initialization, timeout abort in %ds (0x%x)\n",
208 jiffies_to_msecs(end - warn) / 1000, fw_initializing);
209 warn = jiffies + msecs_to_jiffies(warn_time_mili);
210 }
211 msleep(mlx5_tout_ms(dev, FW_PRE_INIT_WAIT));
212 } while (true);
213
214 return err;
215 }
216
mlx5_set_driver_version(struct mlx5_core_dev * dev)217 static void mlx5_set_driver_version(struct mlx5_core_dev *dev)
218 {
219 int driver_ver_sz = MLX5_FLD_SZ_BYTES(set_driver_version_in,
220 driver_version);
221 u8 in[MLX5_ST_SZ_BYTES(set_driver_version_in)] = {};
222 char *string;
223
224 if (!MLX5_CAP_GEN(dev, driver_version))
225 return;
226
227 string = MLX5_ADDR_OF(set_driver_version_in, in, driver_version);
228
229 snprintf(string, driver_ver_sz, "Linux,%s,%u.%u.%u",
230 KBUILD_MODNAME, LINUX_VERSION_MAJOR,
231 LINUX_VERSION_PATCHLEVEL, LINUX_VERSION_SUBLEVEL);
232
233 /*Send the command*/
234 MLX5_SET(set_driver_version_in, in, opcode,
235 MLX5_CMD_OP_SET_DRIVER_VERSION);
236
237 mlx5_cmd_exec_in(dev, set_driver_version, in);
238 }
239
set_dma_caps(struct pci_dev * pdev)240 static int set_dma_caps(struct pci_dev *pdev)
241 {
242 int err;
243
244 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
245 if (err) {
246 dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask\n");
247 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
248 if (err) {
249 dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting\n");
250 return err;
251 }
252 }
253
254 dma_set_max_seg_size(&pdev->dev, 2u * 1024 * 1024 * 1024);
255 return err;
256 }
257
mlx5_pci_enable_device(struct mlx5_core_dev * dev)258 static int mlx5_pci_enable_device(struct mlx5_core_dev *dev)
259 {
260 struct pci_dev *pdev = dev->pdev;
261 int err = 0;
262
263 mutex_lock(&dev->pci_status_mutex);
264 if (dev->pci_status == MLX5_PCI_STATUS_DISABLED) {
265 err = pci_enable_device(pdev);
266 if (!err)
267 dev->pci_status = MLX5_PCI_STATUS_ENABLED;
268 }
269 mutex_unlock(&dev->pci_status_mutex);
270
271 return err;
272 }
273
mlx5_pci_disable_device(struct mlx5_core_dev * dev)274 static void mlx5_pci_disable_device(struct mlx5_core_dev *dev)
275 {
276 struct pci_dev *pdev = dev->pdev;
277
278 mutex_lock(&dev->pci_status_mutex);
279 if (dev->pci_status == MLX5_PCI_STATUS_ENABLED) {
280 pci_disable_device(pdev);
281 dev->pci_status = MLX5_PCI_STATUS_DISABLED;
282 }
283 mutex_unlock(&dev->pci_status_mutex);
284 }
285
request_bar(struct pci_dev * pdev)286 static int request_bar(struct pci_dev *pdev)
287 {
288 int err = 0;
289
290 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
291 dev_err(&pdev->dev, "Missing registers BAR, aborting\n");
292 return -ENODEV;
293 }
294
295 err = pci_request_regions(pdev, KBUILD_MODNAME);
296 if (err)
297 dev_err(&pdev->dev, "Couldn't get PCI resources, aborting\n");
298
299 return err;
300 }
301
release_bar(struct pci_dev * pdev)302 static void release_bar(struct pci_dev *pdev)
303 {
304 pci_release_regions(pdev);
305 }
306
307 struct mlx5_reg_host_endianness {
308 u8 he;
309 u8 rsvd[15];
310 };
311
to_fw_pkey_sz(struct mlx5_core_dev * dev,u32 size)312 static u16 to_fw_pkey_sz(struct mlx5_core_dev *dev, u32 size)
313 {
314 switch (size) {
315 case 128:
316 return 0;
317 case 256:
318 return 1;
319 case 512:
320 return 2;
321 case 1024:
322 return 3;
323 case 2048:
324 return 4;
325 case 4096:
326 return 5;
327 default:
328 mlx5_core_warn(dev, "invalid pkey table size %d\n", size);
329 return 0;
330 }
331 }
332
mlx5_core_uplink_netdev_set(struct mlx5_core_dev * dev,struct net_device * netdev)333 void mlx5_core_uplink_netdev_set(struct mlx5_core_dev *dev, struct net_device *netdev)
334 {
335 mutex_lock(&dev->mlx5e_res.uplink_netdev_lock);
336 dev->mlx5e_res.uplink_netdev = netdev;
337 mlx5_blocking_notifier_call_chain(dev, MLX5_DRIVER_EVENT_UPLINK_NETDEV,
338 netdev);
339 mutex_unlock(&dev->mlx5e_res.uplink_netdev_lock);
340 }
341
mlx5_core_uplink_netdev_event_replay(struct mlx5_core_dev * dev)342 void mlx5_core_uplink_netdev_event_replay(struct mlx5_core_dev *dev)
343 {
344 mutex_lock(&dev->mlx5e_res.uplink_netdev_lock);
345 mlx5_blocking_notifier_call_chain(dev, MLX5_DRIVER_EVENT_UPLINK_NETDEV,
346 dev->mlx5e_res.uplink_netdev);
347 mutex_unlock(&dev->mlx5e_res.uplink_netdev_lock);
348 }
349 EXPORT_SYMBOL(mlx5_core_uplink_netdev_event_replay);
350
mlx5_core_mp_event_replay(struct mlx5_core_dev * dev,u32 event,void * data)351 void mlx5_core_mp_event_replay(struct mlx5_core_dev *dev, u32 event, void *data)
352 {
353 mlx5_blocking_notifier_call_chain(dev, event, data);
354 }
355 EXPORT_SYMBOL(mlx5_core_mp_event_replay);
356
mlx5_core_get_caps_mode(struct mlx5_core_dev * dev,enum mlx5_cap_type cap_type,enum mlx5_cap_mode cap_mode)357 int mlx5_core_get_caps_mode(struct mlx5_core_dev *dev, enum mlx5_cap_type cap_type,
358 enum mlx5_cap_mode cap_mode)
359 {
360 u8 in[MLX5_ST_SZ_BYTES(query_hca_cap_in)];
361 int out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
362 void *out, *hca_caps;
363 u16 opmod = (cap_type << 1) | (cap_mode & 0x01);
364 int err;
365
366 memset(in, 0, sizeof(in));
367 out = kzalloc(out_sz, GFP_KERNEL);
368 if (!out)
369 return -ENOMEM;
370
371 MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP);
372 MLX5_SET(query_hca_cap_in, in, op_mod, opmod);
373 err = mlx5_cmd_exec_inout(dev, query_hca_cap, in, out);
374 if (err) {
375 mlx5_core_warn(dev,
376 "QUERY_HCA_CAP : type(%x) opmode(%x) Failed(%d)\n",
377 cap_type, cap_mode, err);
378 goto query_ex;
379 }
380
381 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, out, capability);
382
383 switch (cap_mode) {
384 case HCA_CAP_OPMOD_GET_MAX:
385 memcpy(dev->caps.hca[cap_type]->max, hca_caps,
386 MLX5_UN_SZ_BYTES(hca_cap_union));
387 break;
388 case HCA_CAP_OPMOD_GET_CUR:
389 memcpy(dev->caps.hca[cap_type]->cur, hca_caps,
390 MLX5_UN_SZ_BYTES(hca_cap_union));
391 break;
392 default:
393 mlx5_core_warn(dev,
394 "Tried to query dev cap type(%x) with wrong opmode(%x)\n",
395 cap_type, cap_mode);
396 err = -EINVAL;
397 break;
398 }
399 query_ex:
400 kfree(out);
401 return err;
402 }
403
mlx5_core_get_caps(struct mlx5_core_dev * dev,enum mlx5_cap_type cap_type)404 int mlx5_core_get_caps(struct mlx5_core_dev *dev, enum mlx5_cap_type cap_type)
405 {
406 int ret;
407
408 ret = mlx5_core_get_caps_mode(dev, cap_type, HCA_CAP_OPMOD_GET_CUR);
409 if (ret)
410 return ret;
411 return mlx5_core_get_caps_mode(dev, cap_type, HCA_CAP_OPMOD_GET_MAX);
412 }
413
set_caps(struct mlx5_core_dev * dev,void * in,int opmod)414 static int set_caps(struct mlx5_core_dev *dev, void *in, int opmod)
415 {
416 MLX5_SET(set_hca_cap_in, in, opcode, MLX5_CMD_OP_SET_HCA_CAP);
417 MLX5_SET(set_hca_cap_in, in, op_mod, opmod << 1);
418 return mlx5_cmd_exec_in(dev, set_hca_cap, in);
419 }
420
handle_hca_cap_atomic(struct mlx5_core_dev * dev,void * set_ctx)421 static int handle_hca_cap_atomic(struct mlx5_core_dev *dev, void *set_ctx)
422 {
423 void *set_hca_cap;
424 int req_endianness;
425 int err;
426
427 if (!MLX5_CAP_GEN(dev, atomic))
428 return 0;
429
430 err = mlx5_core_get_caps(dev, MLX5_CAP_ATOMIC);
431 if (err)
432 return err;
433
434 req_endianness =
435 MLX5_CAP_ATOMIC(dev,
436 supported_atomic_req_8B_endianness_mode_1);
437
438 if (req_endianness != MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS)
439 return 0;
440
441 set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, capability);
442
443 /* Set requestor to host endianness */
444 MLX5_SET(atomic_caps, set_hca_cap, atomic_req_8B_endianness_mode,
445 MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS);
446
447 return set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_ATOMIC);
448 }
449
handle_hca_cap_odp(struct mlx5_core_dev * dev,void * set_ctx)450 static int handle_hca_cap_odp(struct mlx5_core_dev *dev, void *set_ctx)
451 {
452 void *set_hca_cap;
453 bool do_set = false;
454 int err;
455
456 if (!IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING) ||
457 !MLX5_CAP_GEN(dev, pg))
458 return 0;
459
460 err = mlx5_core_get_caps(dev, MLX5_CAP_ODP);
461 if (err)
462 return err;
463
464 set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, capability);
465 memcpy(set_hca_cap, dev->caps.hca[MLX5_CAP_ODP]->cur,
466 MLX5_ST_SZ_BYTES(odp_cap));
467
468 #define ODP_CAP_SET_MAX(dev, field) \
469 do { \
470 u32 _res = MLX5_CAP_ODP_MAX(dev, field); \
471 if (_res) { \
472 do_set = true; \
473 MLX5_SET(odp_cap, set_hca_cap, field, _res); \
474 } \
475 } while (0)
476
477 ODP_CAP_SET_MAX(dev, ud_odp_caps.srq_receive);
478 ODP_CAP_SET_MAX(dev, rc_odp_caps.srq_receive);
479 ODP_CAP_SET_MAX(dev, xrc_odp_caps.srq_receive);
480 ODP_CAP_SET_MAX(dev, xrc_odp_caps.send);
481 ODP_CAP_SET_MAX(dev, xrc_odp_caps.receive);
482 ODP_CAP_SET_MAX(dev, xrc_odp_caps.write);
483 ODP_CAP_SET_MAX(dev, xrc_odp_caps.read);
484 ODP_CAP_SET_MAX(dev, xrc_odp_caps.atomic);
485 ODP_CAP_SET_MAX(dev, dc_odp_caps.srq_receive);
486 ODP_CAP_SET_MAX(dev, dc_odp_caps.send);
487 ODP_CAP_SET_MAX(dev, dc_odp_caps.receive);
488 ODP_CAP_SET_MAX(dev, dc_odp_caps.write);
489 ODP_CAP_SET_MAX(dev, dc_odp_caps.read);
490 ODP_CAP_SET_MAX(dev, dc_odp_caps.atomic);
491
492 if (!do_set)
493 return 0;
494
495 return set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_ODP);
496 }
497
max_uc_list_get_devlink_param(struct mlx5_core_dev * dev)498 static int max_uc_list_get_devlink_param(struct mlx5_core_dev *dev)
499 {
500 struct devlink *devlink = priv_to_devlink(dev);
501 union devlink_param_value val;
502 int err;
503
504 err = devl_param_driverinit_value_get(devlink,
505 DEVLINK_PARAM_GENERIC_ID_MAX_MACS,
506 &val);
507 if (!err)
508 return val.vu32;
509 mlx5_core_dbg(dev, "Failed to get param. err = %d\n", err);
510 return err;
511 }
512
mlx5_is_roce_on(struct mlx5_core_dev * dev)513 bool mlx5_is_roce_on(struct mlx5_core_dev *dev)
514 {
515 struct devlink *devlink = priv_to_devlink(dev);
516 union devlink_param_value val;
517 int err;
518
519 err = devl_param_driverinit_value_get(devlink,
520 DEVLINK_PARAM_GENERIC_ID_ENABLE_ROCE,
521 &val);
522
523 if (!err)
524 return val.vbool;
525
526 mlx5_core_dbg(dev, "Failed to get param. err = %d\n", err);
527 return MLX5_CAP_GEN(dev, roce);
528 }
529 EXPORT_SYMBOL(mlx5_is_roce_on);
530
handle_hca_cap_2(struct mlx5_core_dev * dev,void * set_ctx)531 static int handle_hca_cap_2(struct mlx5_core_dev *dev, void *set_ctx)
532 {
533 void *set_hca_cap;
534 int err;
535
536 if (!MLX5_CAP_GEN_MAX(dev, hca_cap_2))
537 return 0;
538
539 err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL_2);
540 if (err)
541 return err;
542
543 if (!MLX5_CAP_GEN_2_MAX(dev, sw_vhca_id_valid) ||
544 !(dev->priv.sw_vhca_id > 0))
545 return 0;
546
547 set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx,
548 capability);
549 memcpy(set_hca_cap, dev->caps.hca[MLX5_CAP_GENERAL_2]->cur,
550 MLX5_ST_SZ_BYTES(cmd_hca_cap_2));
551 MLX5_SET(cmd_hca_cap_2, set_hca_cap, sw_vhca_id_valid, 1);
552
553 return set_caps(dev, set_ctx, MLX5_CAP_GENERAL_2);
554 }
555
handle_hca_cap(struct mlx5_core_dev * dev,void * set_ctx)556 static int handle_hca_cap(struct mlx5_core_dev *dev, void *set_ctx)
557 {
558 struct mlx5_profile *prof = &dev->profile;
559 void *set_hca_cap;
560 int max_uc_list;
561 int err;
562
563 err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL);
564 if (err)
565 return err;
566
567 set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx,
568 capability);
569 memcpy(set_hca_cap, dev->caps.hca[MLX5_CAP_GENERAL]->cur,
570 MLX5_ST_SZ_BYTES(cmd_hca_cap));
571
572 mlx5_core_dbg(dev, "Current Pkey table size %d Setting new size %d\n",
573 mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(dev, pkey_table_size)),
574 128);
575 /* we limit the size of the pkey table to 128 entries for now */
576 MLX5_SET(cmd_hca_cap, set_hca_cap, pkey_table_size,
577 to_fw_pkey_sz(dev, 128));
578
579 /* Check log_max_qp from HCA caps to set in current profile */
580 if (prof->log_max_qp == LOG_MAX_SUPPORTED_QPS) {
581 prof->log_max_qp = min_t(u8, 18, MLX5_CAP_GEN_MAX(dev, log_max_qp));
582 } else if (MLX5_CAP_GEN_MAX(dev, log_max_qp) < prof->log_max_qp) {
583 mlx5_core_warn(dev, "log_max_qp value in current profile is %d, changing it to HCA capability limit (%d)\n",
584 prof->log_max_qp,
585 MLX5_CAP_GEN_MAX(dev, log_max_qp));
586 prof->log_max_qp = MLX5_CAP_GEN_MAX(dev, log_max_qp);
587 }
588 if (prof->mask & MLX5_PROF_MASK_QP_SIZE)
589 MLX5_SET(cmd_hca_cap, set_hca_cap, log_max_qp,
590 prof->log_max_qp);
591
592 /* disable cmdif checksum */
593 MLX5_SET(cmd_hca_cap, set_hca_cap, cmdif_checksum, 0);
594
595 /* Enable 4K UAR only when HCA supports it and page size is bigger
596 * than 4K.
597 */
598 if (MLX5_CAP_GEN_MAX(dev, uar_4k) && PAGE_SIZE > 4096)
599 MLX5_SET(cmd_hca_cap, set_hca_cap, uar_4k, 1);
600
601 MLX5_SET(cmd_hca_cap, set_hca_cap, log_uar_page_sz, PAGE_SHIFT - 12);
602
603 if (MLX5_CAP_GEN_MAX(dev, cache_line_128byte))
604 MLX5_SET(cmd_hca_cap,
605 set_hca_cap,
606 cache_line_128byte,
607 cache_line_size() >= 128 ? 1 : 0);
608
609 if (MLX5_CAP_GEN_MAX(dev, dct))
610 MLX5_SET(cmd_hca_cap, set_hca_cap, dct, 1);
611
612 if (MLX5_CAP_GEN_MAX(dev, pci_sync_for_fw_update_event))
613 MLX5_SET(cmd_hca_cap, set_hca_cap, pci_sync_for_fw_update_event, 1);
614 if (MLX5_CAP_GEN_MAX(dev, pci_sync_for_fw_update_with_driver_unload))
615 MLX5_SET(cmd_hca_cap, set_hca_cap,
616 pci_sync_for_fw_update_with_driver_unload, 1);
617
618 if (MLX5_CAP_GEN_MAX(dev, num_vhca_ports))
619 MLX5_SET(cmd_hca_cap,
620 set_hca_cap,
621 num_vhca_ports,
622 MLX5_CAP_GEN_MAX(dev, num_vhca_ports));
623
624 if (MLX5_CAP_GEN_MAX(dev, release_all_pages))
625 MLX5_SET(cmd_hca_cap, set_hca_cap, release_all_pages, 1);
626
627 if (MLX5_CAP_GEN_MAX(dev, mkey_by_name))
628 MLX5_SET(cmd_hca_cap, set_hca_cap, mkey_by_name, 1);
629
630 mlx5_vhca_state_cap_handle(dev, set_hca_cap);
631
632 if (MLX5_CAP_GEN_MAX(dev, num_total_dynamic_vf_msix))
633 MLX5_SET(cmd_hca_cap, set_hca_cap, num_total_dynamic_vf_msix,
634 MLX5_CAP_GEN_MAX(dev, num_total_dynamic_vf_msix));
635
636 if (MLX5_CAP_GEN(dev, roce_rw_supported) && MLX5_CAP_GEN_MAX(dev, roce))
637 MLX5_SET(cmd_hca_cap, set_hca_cap, roce,
638 mlx5_is_roce_on(dev));
639
640 max_uc_list = max_uc_list_get_devlink_param(dev);
641 if (max_uc_list > 0)
642 MLX5_SET(cmd_hca_cap, set_hca_cap, log_max_current_uc_list,
643 ilog2(max_uc_list));
644
645 return set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE);
646 }
647
648 /* Cached MLX5_CAP_GEN(dev, roce) can be out of sync this early in the
649 * boot process.
650 * In case RoCE cap is writable in FW and user/devlink requested to change the
651 * cap, we are yet to query the final state of the above cap.
652 * Hence, the need for this function.
653 *
654 * Returns
655 * True:
656 * 1) RoCE cap is read only in FW and already disabled
657 * OR:
658 * 2) RoCE cap is writable in FW and user/devlink requested it off.
659 *
660 * In any other case, return False.
661 */
is_roce_fw_disabled(struct mlx5_core_dev * dev)662 static bool is_roce_fw_disabled(struct mlx5_core_dev *dev)
663 {
664 return (MLX5_CAP_GEN(dev, roce_rw_supported) && !mlx5_is_roce_on(dev)) ||
665 (!MLX5_CAP_GEN(dev, roce_rw_supported) && !MLX5_CAP_GEN(dev, roce));
666 }
667
handle_hca_cap_roce(struct mlx5_core_dev * dev,void * set_ctx)668 static int handle_hca_cap_roce(struct mlx5_core_dev *dev, void *set_ctx)
669 {
670 void *set_hca_cap;
671 int err;
672
673 if (is_roce_fw_disabled(dev))
674 return 0;
675
676 err = mlx5_core_get_caps(dev, MLX5_CAP_ROCE);
677 if (err)
678 return err;
679
680 if (MLX5_CAP_ROCE(dev, sw_r_roce_src_udp_port) ||
681 !MLX5_CAP_ROCE_MAX(dev, sw_r_roce_src_udp_port))
682 return 0;
683
684 set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, capability);
685 memcpy(set_hca_cap, dev->caps.hca[MLX5_CAP_ROCE]->cur,
686 MLX5_ST_SZ_BYTES(roce_cap));
687 MLX5_SET(roce_cap, set_hca_cap, sw_r_roce_src_udp_port, 1);
688
689 if (MLX5_CAP_ROCE_MAX(dev, qp_ooo_transmit_default))
690 MLX5_SET(roce_cap, set_hca_cap, qp_ooo_transmit_default, 1);
691
692 err = set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_ROCE);
693 return err;
694 }
695
handle_hca_cap_port_selection(struct mlx5_core_dev * dev,void * set_ctx)696 static int handle_hca_cap_port_selection(struct mlx5_core_dev *dev,
697 void *set_ctx)
698 {
699 void *set_hca_cap;
700 int err;
701
702 if (!MLX5_CAP_GEN(dev, port_selection_cap))
703 return 0;
704
705 err = mlx5_core_get_caps(dev, MLX5_CAP_PORT_SELECTION);
706 if (err)
707 return err;
708
709 if (MLX5_CAP_PORT_SELECTION(dev, port_select_flow_table_bypass) ||
710 !MLX5_CAP_PORT_SELECTION_MAX(dev, port_select_flow_table_bypass))
711 return 0;
712
713 set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, capability);
714 memcpy(set_hca_cap, dev->caps.hca[MLX5_CAP_PORT_SELECTION]->cur,
715 MLX5_ST_SZ_BYTES(port_selection_cap));
716 MLX5_SET(port_selection_cap, set_hca_cap, port_select_flow_table_bypass, 1);
717
718 err = set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_PORT_SELECTION);
719
720 return err;
721 }
722
set_hca_cap(struct mlx5_core_dev * dev)723 static int set_hca_cap(struct mlx5_core_dev *dev)
724 {
725 int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in);
726 void *set_ctx;
727 int err;
728
729 set_ctx = kzalloc(set_sz, GFP_KERNEL);
730 if (!set_ctx)
731 return -ENOMEM;
732
733 err = handle_hca_cap(dev, set_ctx);
734 if (err) {
735 mlx5_core_err(dev, "handle_hca_cap failed\n");
736 goto out;
737 }
738
739 memset(set_ctx, 0, set_sz);
740 err = handle_hca_cap_atomic(dev, set_ctx);
741 if (err) {
742 mlx5_core_err(dev, "handle_hca_cap_atomic failed\n");
743 goto out;
744 }
745
746 memset(set_ctx, 0, set_sz);
747 err = handle_hca_cap_odp(dev, set_ctx);
748 if (err) {
749 mlx5_core_err(dev, "handle_hca_cap_odp failed\n");
750 goto out;
751 }
752
753 memset(set_ctx, 0, set_sz);
754 err = handle_hca_cap_roce(dev, set_ctx);
755 if (err) {
756 mlx5_core_err(dev, "handle_hca_cap_roce failed\n");
757 goto out;
758 }
759
760 memset(set_ctx, 0, set_sz);
761 err = handle_hca_cap_2(dev, set_ctx);
762 if (err) {
763 mlx5_core_err(dev, "handle_hca_cap_2 failed\n");
764 goto out;
765 }
766
767 memset(set_ctx, 0, set_sz);
768 err = handle_hca_cap_port_selection(dev, set_ctx);
769 if (err) {
770 mlx5_core_err(dev, "handle_hca_cap_port_selection failed\n");
771 goto out;
772 }
773
774 out:
775 kfree(set_ctx);
776 return err;
777 }
778
set_hca_ctrl(struct mlx5_core_dev * dev)779 static int set_hca_ctrl(struct mlx5_core_dev *dev)
780 {
781 struct mlx5_reg_host_endianness he_in;
782 struct mlx5_reg_host_endianness he_out;
783 int err;
784
785 if (!mlx5_core_is_pf(dev))
786 return 0;
787
788 memset(&he_in, 0, sizeof(he_in));
789 he_in.he = MLX5_SET_HOST_ENDIANNESS;
790 err = mlx5_core_access_reg(dev, &he_in, sizeof(he_in),
791 &he_out, sizeof(he_out),
792 MLX5_REG_HOST_ENDIANNESS, 0, 1);
793 return err;
794 }
795
mlx5_core_set_hca_defaults(struct mlx5_core_dev * dev)796 static int mlx5_core_set_hca_defaults(struct mlx5_core_dev *dev)
797 {
798 int ret = 0;
799
800 /* Disable local_lb by default */
801 if (MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_ETH)
802 ret = mlx5_nic_vport_update_local_lb(dev, false);
803
804 return ret;
805 }
806
mlx5_core_enable_hca(struct mlx5_core_dev * dev,u16 func_id)807 int mlx5_core_enable_hca(struct mlx5_core_dev *dev, u16 func_id)
808 {
809 u32 in[MLX5_ST_SZ_DW(enable_hca_in)] = {};
810
811 MLX5_SET(enable_hca_in, in, opcode, MLX5_CMD_OP_ENABLE_HCA);
812 MLX5_SET(enable_hca_in, in, function_id, func_id);
813 MLX5_SET(enable_hca_in, in, embedded_cpu_function,
814 dev->caps.embedded_cpu);
815 return mlx5_cmd_exec_in(dev, enable_hca, in);
816 }
817
mlx5_core_disable_hca(struct mlx5_core_dev * dev,u16 func_id)818 int mlx5_core_disable_hca(struct mlx5_core_dev *dev, u16 func_id)
819 {
820 u32 in[MLX5_ST_SZ_DW(disable_hca_in)] = {};
821
822 MLX5_SET(disable_hca_in, in, opcode, MLX5_CMD_OP_DISABLE_HCA);
823 MLX5_SET(disable_hca_in, in, function_id, func_id);
824 MLX5_SET(enable_hca_in, in, embedded_cpu_function,
825 dev->caps.embedded_cpu);
826 return mlx5_cmd_exec_in(dev, disable_hca, in);
827 }
828
mlx5_core_set_issi(struct mlx5_core_dev * dev)829 static int mlx5_core_set_issi(struct mlx5_core_dev *dev)
830 {
831 u32 query_out[MLX5_ST_SZ_DW(query_issi_out)] = {};
832 u32 query_in[MLX5_ST_SZ_DW(query_issi_in)] = {};
833 u32 sup_issi;
834 int err;
835
836 MLX5_SET(query_issi_in, query_in, opcode, MLX5_CMD_OP_QUERY_ISSI);
837 err = mlx5_cmd_exec_inout(dev, query_issi, query_in, query_out);
838 if (err) {
839 u32 syndrome = MLX5_GET(query_issi_out, query_out, syndrome);
840 u8 status = MLX5_GET(query_issi_out, query_out, status);
841
842 if (!status || syndrome == MLX5_DRIVER_SYND) {
843 mlx5_core_err(dev, "Failed to query ISSI err(%d) status(%d) synd(%d)\n",
844 err, status, syndrome);
845 return err;
846 }
847
848 mlx5_core_warn(dev, "Query ISSI is not supported by FW, ISSI is 0\n");
849 dev->issi = 0;
850 return 0;
851 }
852
853 sup_issi = MLX5_GET(query_issi_out, query_out, supported_issi_dw0);
854
855 if (sup_issi & (1 << 1)) {
856 u32 set_in[MLX5_ST_SZ_DW(set_issi_in)] = {};
857
858 MLX5_SET(set_issi_in, set_in, opcode, MLX5_CMD_OP_SET_ISSI);
859 MLX5_SET(set_issi_in, set_in, current_issi, 1);
860 err = mlx5_cmd_exec_in(dev, set_issi, set_in);
861 if (err) {
862 mlx5_core_err(dev, "Failed to set ISSI to 1 err(%d)\n",
863 err);
864 return err;
865 }
866
867 dev->issi = 1;
868
869 return 0;
870 } else if (sup_issi & (1 << 0) || !sup_issi) {
871 return 0;
872 }
873
874 return -EOPNOTSUPP;
875 }
876
mlx5_pci_init(struct mlx5_core_dev * dev,struct pci_dev * pdev,const struct pci_device_id * id)877 static int mlx5_pci_init(struct mlx5_core_dev *dev, struct pci_dev *pdev,
878 const struct pci_device_id *id)
879 {
880 int err = 0;
881
882 mutex_init(&dev->pci_status_mutex);
883 pci_set_drvdata(dev->pdev, dev);
884
885 dev->bar_addr = pci_resource_start(pdev, 0);
886
887 err = mlx5_pci_enable_device(dev);
888 if (err) {
889 mlx5_core_err(dev, "Cannot enable PCI device, aborting\n");
890 return err;
891 }
892
893 err = request_bar(pdev);
894 if (err) {
895 mlx5_core_err(dev, "error requesting BARs, aborting\n");
896 goto err_disable;
897 }
898
899 pci_set_master(pdev);
900
901 err = set_dma_caps(pdev);
902 if (err) {
903 mlx5_core_err(dev, "Failed setting DMA capabilities mask, aborting\n");
904 goto err_clr_master;
905 }
906
907 if (pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP32) &&
908 pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP64) &&
909 pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP128))
910 mlx5_core_dbg(dev, "Enabling pci atomics failed\n");
911
912 dev->iseg_base = dev->bar_addr;
913 dev->iseg = ioremap(dev->iseg_base, sizeof(*dev->iseg));
914 if (!dev->iseg) {
915 err = -ENOMEM;
916 mlx5_core_err(dev, "Failed mapping initialization segment, aborting\n");
917 goto err_clr_master;
918 }
919
920 mlx5_pci_vsc_init(dev);
921 return 0;
922
923 err_clr_master:
924 release_bar(dev->pdev);
925 err_disable:
926 mlx5_pci_disable_device(dev);
927 return err;
928 }
929
mlx5_pci_close(struct mlx5_core_dev * dev)930 static void mlx5_pci_close(struct mlx5_core_dev *dev)
931 {
932 /* health work might still be active, and it needs pci bar in
933 * order to know the NIC state. Therefore, drain the health WQ
934 * before removing the pci bars
935 */
936 mlx5_drain_health_wq(dev);
937 iounmap(dev->iseg);
938 release_bar(dev->pdev);
939 mlx5_pci_disable_device(dev);
940 }
941
mlx5_register_hca_devcom_comp(struct mlx5_core_dev * dev)942 static void mlx5_register_hca_devcom_comp(struct mlx5_core_dev *dev)
943 {
944 /* This component is use to sync adding core_dev to lag_dev and to sync
945 * changes of mlx5_adev_devices between LAG layer and other layers.
946 */
947 if (!mlx5_lag_is_supported(dev))
948 return;
949
950 dev->priv.hca_devcom_comp =
951 mlx5_devcom_register_component(dev->priv.devc, MLX5_DEVCOM_HCA_PORTS,
952 mlx5_query_nic_system_image_guid(dev),
953 NULL, dev);
954 if (IS_ERR_OR_NULL(dev->priv.hca_devcom_comp))
955 mlx5_core_err(dev, "Failed to register devcom HCA component\n");
956 }
957
mlx5_unregister_hca_devcom_comp(struct mlx5_core_dev * dev)958 static void mlx5_unregister_hca_devcom_comp(struct mlx5_core_dev *dev)
959 {
960 mlx5_devcom_unregister_component(dev->priv.hca_devcom_comp);
961 }
962
mlx5_init_once(struct mlx5_core_dev * dev)963 static int mlx5_init_once(struct mlx5_core_dev *dev)
964 {
965 int err;
966
967 dev->priv.devc = mlx5_devcom_register_device(dev);
968 if (IS_ERR(dev->priv.devc))
969 mlx5_core_warn(dev, "failed to register devcom device %ld\n",
970 PTR_ERR(dev->priv.devc));
971 mlx5_register_hca_devcom_comp(dev);
972
973 err = mlx5_query_board_id(dev);
974 if (err) {
975 mlx5_core_err(dev, "query board id failed\n");
976 goto err_devcom;
977 }
978
979 err = mlx5_irq_table_init(dev);
980 if (err) {
981 mlx5_core_err(dev, "failed to initialize irq table\n");
982 goto err_devcom;
983 }
984
985 err = mlx5_eq_table_init(dev);
986 if (err) {
987 mlx5_core_err(dev, "failed to initialize eq\n");
988 goto err_irq_cleanup;
989 }
990
991 err = mlx5_events_init(dev);
992 if (err) {
993 mlx5_core_err(dev, "failed to initialize events\n");
994 goto err_eq_cleanup;
995 }
996
997 err = mlx5_fw_reset_init(dev);
998 if (err) {
999 mlx5_core_err(dev, "failed to initialize fw reset events\n");
1000 goto err_events_cleanup;
1001 }
1002
1003 mlx5_cq_debugfs_init(dev);
1004
1005 mlx5_init_reserved_gids(dev);
1006
1007 mlx5_init_clock(dev);
1008
1009 dev->vxlan = mlx5_vxlan_create(dev);
1010 dev->geneve = mlx5_geneve_create(dev);
1011
1012 err = mlx5_init_rl_table(dev);
1013 if (err) {
1014 mlx5_core_err(dev, "Failed to init rate limiting\n");
1015 goto err_tables_cleanup;
1016 }
1017
1018 err = mlx5_mpfs_init(dev);
1019 if (err) {
1020 mlx5_core_err(dev, "Failed to init l2 table %d\n", err);
1021 goto err_rl_cleanup;
1022 }
1023
1024 err = mlx5_sriov_init(dev);
1025 if (err) {
1026 mlx5_core_err(dev, "Failed to init sriov %d\n", err);
1027 goto err_mpfs_cleanup;
1028 }
1029
1030 err = mlx5_eswitch_init(dev);
1031 if (err) {
1032 mlx5_core_err(dev, "Failed to init eswitch %d\n", err);
1033 goto err_sriov_cleanup;
1034 }
1035
1036 err = mlx5_fpga_init(dev);
1037 if (err) {
1038 mlx5_core_err(dev, "Failed to init fpga device %d\n", err);
1039 goto err_eswitch_cleanup;
1040 }
1041
1042 err = mlx5_vhca_event_init(dev);
1043 if (err) {
1044 mlx5_core_err(dev, "Failed to init vhca event notifier %d\n", err);
1045 goto err_fpga_cleanup;
1046 }
1047
1048 err = mlx5_sf_hw_table_init(dev);
1049 if (err) {
1050 mlx5_core_err(dev, "Failed to init SF HW table %d\n", err);
1051 goto err_sf_hw_table_cleanup;
1052 }
1053
1054 err = mlx5_sf_table_init(dev);
1055 if (err) {
1056 mlx5_core_err(dev, "Failed to init SF table %d\n", err);
1057 goto err_sf_table_cleanup;
1058 }
1059
1060 err = mlx5_fs_core_alloc(dev);
1061 if (err) {
1062 mlx5_core_err(dev, "Failed to alloc flow steering\n");
1063 goto err_fs;
1064 }
1065
1066 dev->dm = mlx5_dm_create(dev);
1067 if (IS_ERR(dev->dm))
1068 mlx5_core_warn(dev, "Failed to init device memory %ld\n", PTR_ERR(dev->dm));
1069
1070 dev->tracer = mlx5_fw_tracer_create(dev);
1071 dev->hv_vhca = mlx5_hv_vhca_create(dev);
1072 dev->rsc_dump = mlx5_rsc_dump_create(dev);
1073
1074 return 0;
1075
1076 err_fs:
1077 mlx5_sf_table_cleanup(dev);
1078 err_sf_table_cleanup:
1079 mlx5_sf_hw_table_cleanup(dev);
1080 err_sf_hw_table_cleanup:
1081 mlx5_vhca_event_cleanup(dev);
1082 err_fpga_cleanup:
1083 mlx5_fpga_cleanup(dev);
1084 err_eswitch_cleanup:
1085 mlx5_eswitch_cleanup(dev->priv.eswitch);
1086 err_sriov_cleanup:
1087 mlx5_sriov_cleanup(dev);
1088 err_mpfs_cleanup:
1089 mlx5_mpfs_cleanup(dev);
1090 err_rl_cleanup:
1091 mlx5_cleanup_rl_table(dev);
1092 err_tables_cleanup:
1093 mlx5_geneve_destroy(dev->geneve);
1094 mlx5_vxlan_destroy(dev->vxlan);
1095 mlx5_cleanup_clock(dev);
1096 mlx5_cleanup_reserved_gids(dev);
1097 mlx5_cq_debugfs_cleanup(dev);
1098 mlx5_fw_reset_cleanup(dev);
1099 err_events_cleanup:
1100 mlx5_events_cleanup(dev);
1101 err_eq_cleanup:
1102 mlx5_eq_table_cleanup(dev);
1103 err_irq_cleanup:
1104 mlx5_irq_table_cleanup(dev);
1105 err_devcom:
1106 mlx5_unregister_hca_devcom_comp(dev);
1107 mlx5_devcom_unregister_device(dev->priv.devc);
1108
1109 return err;
1110 }
1111
mlx5_cleanup_once(struct mlx5_core_dev * dev)1112 static void mlx5_cleanup_once(struct mlx5_core_dev *dev)
1113 {
1114 mlx5_rsc_dump_destroy(dev);
1115 mlx5_hv_vhca_destroy(dev->hv_vhca);
1116 mlx5_fw_tracer_destroy(dev->tracer);
1117 mlx5_dm_cleanup(dev);
1118 mlx5_fs_core_free(dev);
1119 mlx5_sf_table_cleanup(dev);
1120 mlx5_sf_hw_table_cleanup(dev);
1121 mlx5_vhca_event_cleanup(dev);
1122 mlx5_fpga_cleanup(dev);
1123 mlx5_eswitch_cleanup(dev->priv.eswitch);
1124 mlx5_sriov_cleanup(dev);
1125 mlx5_mpfs_cleanup(dev);
1126 mlx5_cleanup_rl_table(dev);
1127 mlx5_geneve_destroy(dev->geneve);
1128 mlx5_vxlan_destroy(dev->vxlan);
1129 mlx5_cleanup_clock(dev);
1130 mlx5_cleanup_reserved_gids(dev);
1131 mlx5_cq_debugfs_cleanup(dev);
1132 mlx5_fw_reset_cleanup(dev);
1133 mlx5_events_cleanup(dev);
1134 mlx5_eq_table_cleanup(dev);
1135 mlx5_irq_table_cleanup(dev);
1136 mlx5_unregister_hca_devcom_comp(dev);
1137 mlx5_devcom_unregister_device(dev->priv.devc);
1138 }
1139
mlx5_function_enable(struct mlx5_core_dev * dev,bool boot,u64 timeout)1140 static int mlx5_function_enable(struct mlx5_core_dev *dev, bool boot, u64 timeout)
1141 {
1142 int err;
1143
1144 mlx5_core_info(dev, "firmware version: %d.%d.%d\n", fw_rev_maj(dev),
1145 fw_rev_min(dev), fw_rev_sub(dev));
1146
1147 /* Only PFs hold the relevant PCIe information for this query */
1148 if (mlx5_core_is_pf(dev))
1149 pcie_print_link_status(dev->pdev);
1150
1151 /* wait for firmware to accept initialization segments configurations
1152 */
1153 err = wait_fw_init(dev, timeout,
1154 mlx5_tout_ms(dev, FW_PRE_INIT_WARN_MESSAGE_INTERVAL));
1155 if (err) {
1156 mlx5_core_err(dev, "Firmware over %llu MS in pre-initializing state, aborting\n",
1157 timeout);
1158 return err;
1159 }
1160
1161 err = mlx5_cmd_enable(dev);
1162 if (err) {
1163 mlx5_core_err(dev, "Failed initializing command interface, aborting\n");
1164 return err;
1165 }
1166
1167 mlx5_tout_query_iseg(dev);
1168
1169 err = wait_fw_init(dev, mlx5_tout_ms(dev, FW_INIT), 0);
1170 if (err) {
1171 mlx5_core_err(dev, "Firmware over %llu MS in initializing state, aborting\n",
1172 mlx5_tout_ms(dev, FW_INIT));
1173 goto err_cmd_cleanup;
1174 }
1175
1176 dev->caps.embedded_cpu = mlx5_read_embedded_cpu(dev);
1177 mlx5_cmd_set_state(dev, MLX5_CMDIF_STATE_UP);
1178
1179 mlx5_start_health_poll(dev);
1180
1181 err = mlx5_core_enable_hca(dev, 0);
1182 if (err) {
1183 mlx5_core_err(dev, "enable hca failed\n");
1184 goto stop_health_poll;
1185 }
1186
1187 err = mlx5_core_set_issi(dev);
1188 if (err) {
1189 mlx5_core_err(dev, "failed to set issi\n");
1190 goto err_disable_hca;
1191 }
1192
1193 err = mlx5_satisfy_startup_pages(dev, 1);
1194 if (err) {
1195 mlx5_core_err(dev, "failed to allocate boot pages\n");
1196 goto err_disable_hca;
1197 }
1198
1199 err = mlx5_tout_query_dtor(dev);
1200 if (err) {
1201 mlx5_core_err(dev, "failed to read dtor\n");
1202 goto reclaim_boot_pages;
1203 }
1204
1205 return 0;
1206
1207 reclaim_boot_pages:
1208 mlx5_reclaim_startup_pages(dev);
1209 err_disable_hca:
1210 mlx5_core_disable_hca(dev, 0);
1211 stop_health_poll:
1212 mlx5_stop_health_poll(dev, boot);
1213 err_cmd_cleanup:
1214 mlx5_cmd_set_state(dev, MLX5_CMDIF_STATE_DOWN);
1215 mlx5_cmd_disable(dev);
1216
1217 return err;
1218 }
1219
mlx5_function_disable(struct mlx5_core_dev * dev,bool boot)1220 static void mlx5_function_disable(struct mlx5_core_dev *dev, bool boot)
1221 {
1222 mlx5_reclaim_startup_pages(dev);
1223 mlx5_core_disable_hca(dev, 0);
1224 mlx5_stop_health_poll(dev, boot);
1225 mlx5_cmd_set_state(dev, MLX5_CMDIF_STATE_DOWN);
1226 mlx5_cmd_disable(dev);
1227 }
1228
mlx5_function_open(struct mlx5_core_dev * dev)1229 static int mlx5_function_open(struct mlx5_core_dev *dev)
1230 {
1231 int err;
1232
1233 err = set_hca_ctrl(dev);
1234 if (err) {
1235 mlx5_core_err(dev, "set_hca_ctrl failed\n");
1236 return err;
1237 }
1238
1239 err = set_hca_cap(dev);
1240 if (err) {
1241 mlx5_core_err(dev, "set_hca_cap failed\n");
1242 return err;
1243 }
1244
1245 err = mlx5_satisfy_startup_pages(dev, 0);
1246 if (err) {
1247 mlx5_core_err(dev, "failed to allocate init pages\n");
1248 return err;
1249 }
1250
1251 err = mlx5_cmd_init_hca(dev, sw_owner_id);
1252 if (err) {
1253 mlx5_core_err(dev, "init hca failed\n");
1254 return err;
1255 }
1256
1257 mlx5_set_driver_version(dev);
1258
1259 err = mlx5_query_hca_caps(dev);
1260 if (err) {
1261 mlx5_core_err(dev, "query hca failed\n");
1262 return err;
1263 }
1264 mlx5_start_health_fw_log_up(dev);
1265 return 0;
1266 }
1267
mlx5_function_close(struct mlx5_core_dev * dev)1268 static int mlx5_function_close(struct mlx5_core_dev *dev)
1269 {
1270 int err;
1271
1272 err = mlx5_cmd_teardown_hca(dev);
1273 if (err) {
1274 mlx5_core_err(dev, "tear_down_hca failed, skip cleanup\n");
1275 return err;
1276 }
1277
1278 return 0;
1279 }
1280
mlx5_function_setup(struct mlx5_core_dev * dev,bool boot,u64 timeout)1281 static int mlx5_function_setup(struct mlx5_core_dev *dev, bool boot, u64 timeout)
1282 {
1283 int err;
1284
1285 err = mlx5_function_enable(dev, boot, timeout);
1286 if (err)
1287 return err;
1288
1289 err = mlx5_function_open(dev);
1290 if (err)
1291 mlx5_function_disable(dev, boot);
1292 return err;
1293 }
1294
mlx5_function_teardown(struct mlx5_core_dev * dev,bool boot)1295 static int mlx5_function_teardown(struct mlx5_core_dev *dev, bool boot)
1296 {
1297 int err = mlx5_function_close(dev);
1298
1299 if (!err)
1300 mlx5_function_disable(dev, boot);
1301 return err;
1302 }
1303
mlx5_load(struct mlx5_core_dev * dev)1304 static int mlx5_load(struct mlx5_core_dev *dev)
1305 {
1306 int err;
1307
1308 dev->priv.uar = mlx5_get_uars_page(dev);
1309 if (IS_ERR(dev->priv.uar)) {
1310 mlx5_core_err(dev, "Failed allocating uar, aborting\n");
1311 err = PTR_ERR(dev->priv.uar);
1312 return err;
1313 }
1314
1315 mlx5_events_start(dev);
1316 mlx5_pagealloc_start(dev);
1317
1318 err = mlx5_irq_table_create(dev);
1319 if (err) {
1320 mlx5_core_err(dev, "Failed to alloc IRQs\n");
1321 goto err_irq_table;
1322 }
1323
1324 err = mlx5_eq_table_create(dev);
1325 if (err) {
1326 mlx5_core_err(dev, "Failed to create EQs\n");
1327 goto err_eq_table;
1328 }
1329
1330 err = mlx5_fw_tracer_init(dev->tracer);
1331 if (err) {
1332 mlx5_core_err(dev, "Failed to init FW tracer %d\n", err);
1333 mlx5_fw_tracer_destroy(dev->tracer);
1334 dev->tracer = NULL;
1335 }
1336
1337 mlx5_fw_reset_events_start(dev);
1338 mlx5_hv_vhca_init(dev->hv_vhca);
1339
1340 err = mlx5_rsc_dump_init(dev);
1341 if (err) {
1342 mlx5_core_err(dev, "Failed to init Resource dump %d\n", err);
1343 mlx5_rsc_dump_destroy(dev);
1344 dev->rsc_dump = NULL;
1345 }
1346
1347 err = mlx5_fpga_device_start(dev);
1348 if (err) {
1349 mlx5_core_err(dev, "fpga device start failed %d\n", err);
1350 goto err_fpga_start;
1351 }
1352
1353 err = mlx5_fs_core_init(dev);
1354 if (err) {
1355 mlx5_core_err(dev, "Failed to init flow steering\n");
1356 goto err_fs;
1357 }
1358
1359 err = mlx5_core_set_hca_defaults(dev);
1360 if (err) {
1361 mlx5_core_err(dev, "Failed to set hca defaults\n");
1362 goto err_set_hca;
1363 }
1364
1365 mlx5_vhca_event_start(dev);
1366
1367 err = mlx5_sf_hw_table_create(dev);
1368 if (err) {
1369 mlx5_core_err(dev, "sf table create failed %d\n", err);
1370 goto err_vhca;
1371 }
1372
1373 err = mlx5_ec_init(dev);
1374 if (err) {
1375 mlx5_core_err(dev, "Failed to init embedded CPU\n");
1376 goto err_ec;
1377 }
1378
1379 mlx5_lag_add_mdev(dev);
1380 err = mlx5_sriov_attach(dev);
1381 if (err) {
1382 mlx5_core_err(dev, "sriov init failed %d\n", err);
1383 goto err_sriov;
1384 }
1385
1386 mlx5_sf_dev_table_create(dev);
1387
1388 err = mlx5_devlink_traps_register(priv_to_devlink(dev));
1389 if (err)
1390 goto err_traps_reg;
1391
1392 return 0;
1393
1394 err_traps_reg:
1395 mlx5_sf_dev_table_destroy(dev);
1396 mlx5_sriov_detach(dev);
1397 err_sriov:
1398 mlx5_lag_remove_mdev(dev);
1399 mlx5_ec_cleanup(dev);
1400 err_ec:
1401 mlx5_sf_hw_table_destroy(dev);
1402 err_vhca:
1403 mlx5_vhca_event_stop(dev);
1404 err_set_hca:
1405 mlx5_fs_core_cleanup(dev);
1406 err_fs:
1407 mlx5_fpga_device_stop(dev);
1408 err_fpga_start:
1409 mlx5_rsc_dump_cleanup(dev);
1410 mlx5_hv_vhca_cleanup(dev->hv_vhca);
1411 mlx5_fw_reset_events_stop(dev);
1412 mlx5_fw_tracer_cleanup(dev->tracer);
1413 mlx5_eq_table_destroy(dev);
1414 err_eq_table:
1415 mlx5_irq_table_destroy(dev);
1416 err_irq_table:
1417 mlx5_pagealloc_stop(dev);
1418 mlx5_events_stop(dev);
1419 mlx5_put_uars_page(dev, dev->priv.uar);
1420 return err;
1421 }
1422
mlx5_unload(struct mlx5_core_dev * dev)1423 static void mlx5_unload(struct mlx5_core_dev *dev)
1424 {
1425 mlx5_eswitch_disable(dev->priv.eswitch);
1426 mlx5_devlink_traps_unregister(priv_to_devlink(dev));
1427 mlx5_sf_dev_table_destroy(dev);
1428 mlx5_sriov_detach(dev);
1429 mlx5_lag_remove_mdev(dev);
1430 mlx5_ec_cleanup(dev);
1431 mlx5_sf_hw_table_destroy(dev);
1432 mlx5_vhca_event_stop(dev);
1433 mlx5_fs_core_cleanup(dev);
1434 mlx5_fpga_device_stop(dev);
1435 mlx5_rsc_dump_cleanup(dev);
1436 mlx5_hv_vhca_cleanup(dev->hv_vhca);
1437 mlx5_fw_reset_events_stop(dev);
1438 mlx5_fw_tracer_cleanup(dev->tracer);
1439 mlx5_eq_table_destroy(dev);
1440 mlx5_irq_table_destroy(dev);
1441 mlx5_pagealloc_stop(dev);
1442 mlx5_events_stop(dev);
1443 mlx5_put_uars_page(dev, dev->priv.uar);
1444 }
1445
mlx5_init_one_devl_locked(struct mlx5_core_dev * dev)1446 int mlx5_init_one_devl_locked(struct mlx5_core_dev *dev)
1447 {
1448 bool light_probe = mlx5_dev_is_lightweight(dev);
1449 int err = 0;
1450
1451 mutex_lock(&dev->intf_state_mutex);
1452 dev->state = MLX5_DEVICE_STATE_UP;
1453
1454 err = mlx5_function_setup(dev, true, mlx5_tout_ms(dev, FW_PRE_INIT_TIMEOUT));
1455 if (err)
1456 goto err_function;
1457
1458 err = mlx5_init_once(dev);
1459 if (err) {
1460 mlx5_core_err(dev, "sw objs init failed\n");
1461 goto function_teardown;
1462 }
1463
1464 /* In case of light_probe, mlx5_devlink is already registered.
1465 * Hence, don't register devlink again.
1466 */
1467 if (!light_probe) {
1468 err = mlx5_devlink_params_register(priv_to_devlink(dev));
1469 if (err)
1470 goto err_devlink_params_reg;
1471 }
1472
1473 err = mlx5_load(dev);
1474 if (err)
1475 goto err_load;
1476
1477 set_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
1478
1479 err = mlx5_register_device(dev);
1480 if (err)
1481 goto err_register;
1482
1483 mutex_unlock(&dev->intf_state_mutex);
1484 return 0;
1485
1486 err_register:
1487 clear_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
1488 mlx5_unload(dev);
1489 err_load:
1490 if (!light_probe)
1491 mlx5_devlink_params_unregister(priv_to_devlink(dev));
1492 err_devlink_params_reg:
1493 mlx5_cleanup_once(dev);
1494 function_teardown:
1495 mlx5_function_teardown(dev, true);
1496 err_function:
1497 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
1498 mutex_unlock(&dev->intf_state_mutex);
1499 return err;
1500 }
1501
mlx5_init_one(struct mlx5_core_dev * dev)1502 int mlx5_init_one(struct mlx5_core_dev *dev)
1503 {
1504 struct devlink *devlink = priv_to_devlink(dev);
1505 int err;
1506
1507 devl_lock(devlink);
1508 err = mlx5_init_one_devl_locked(dev);
1509 devl_unlock(devlink);
1510 return err;
1511 }
1512
mlx5_uninit_one(struct mlx5_core_dev * dev)1513 void mlx5_uninit_one(struct mlx5_core_dev *dev)
1514 {
1515 struct devlink *devlink = priv_to_devlink(dev);
1516
1517 devl_lock(devlink);
1518 mutex_lock(&dev->intf_state_mutex);
1519
1520 mlx5_unregister_device(dev);
1521
1522 if (!test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) {
1523 mlx5_core_warn(dev, "%s: interface is down, NOP\n",
1524 __func__);
1525 mlx5_devlink_params_unregister(priv_to_devlink(dev));
1526 mlx5_cleanup_once(dev);
1527 goto out;
1528 }
1529
1530 clear_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
1531 mlx5_unload(dev);
1532 mlx5_devlink_params_unregister(priv_to_devlink(dev));
1533 mlx5_cleanup_once(dev);
1534 mlx5_function_teardown(dev, true);
1535 out:
1536 mutex_unlock(&dev->intf_state_mutex);
1537 devl_unlock(devlink);
1538 }
1539
mlx5_load_one_devl_locked(struct mlx5_core_dev * dev,bool recovery)1540 int mlx5_load_one_devl_locked(struct mlx5_core_dev *dev, bool recovery)
1541 {
1542 int err = 0;
1543 u64 timeout;
1544
1545 devl_assert_locked(priv_to_devlink(dev));
1546 mutex_lock(&dev->intf_state_mutex);
1547 if (test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) {
1548 mlx5_core_warn(dev, "interface is up, NOP\n");
1549 goto out;
1550 }
1551 /* remove any previous indication of internal error */
1552 dev->state = MLX5_DEVICE_STATE_UP;
1553
1554 if (recovery)
1555 timeout = mlx5_tout_ms(dev, FW_PRE_INIT_ON_RECOVERY_TIMEOUT);
1556 else
1557 timeout = mlx5_tout_ms(dev, FW_PRE_INIT_TIMEOUT);
1558 err = mlx5_function_setup(dev, false, timeout);
1559 if (err)
1560 goto err_function;
1561
1562 err = mlx5_load(dev);
1563 if (err)
1564 goto err_load;
1565
1566 set_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
1567
1568 err = mlx5_attach_device(dev);
1569 if (err)
1570 goto err_attach;
1571
1572 mutex_unlock(&dev->intf_state_mutex);
1573 return 0;
1574
1575 err_attach:
1576 clear_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
1577 mlx5_unload(dev);
1578 err_load:
1579 mlx5_function_teardown(dev, false);
1580 err_function:
1581 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
1582 out:
1583 mutex_unlock(&dev->intf_state_mutex);
1584 return err;
1585 }
1586
mlx5_load_one(struct mlx5_core_dev * dev,bool recovery)1587 int mlx5_load_one(struct mlx5_core_dev *dev, bool recovery)
1588 {
1589 struct devlink *devlink = priv_to_devlink(dev);
1590 int ret;
1591
1592 devl_lock(devlink);
1593 ret = mlx5_load_one_devl_locked(dev, recovery);
1594 devl_unlock(devlink);
1595 return ret;
1596 }
1597
mlx5_unload_one_devl_locked(struct mlx5_core_dev * dev,bool suspend)1598 void mlx5_unload_one_devl_locked(struct mlx5_core_dev *dev, bool suspend)
1599 {
1600 devl_assert_locked(priv_to_devlink(dev));
1601 mutex_lock(&dev->intf_state_mutex);
1602
1603 mlx5_detach_device(dev, suspend);
1604
1605 if (!test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) {
1606 mlx5_core_warn(dev, "%s: interface is down, NOP\n",
1607 __func__);
1608 goto out;
1609 }
1610
1611 clear_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
1612 mlx5_unload(dev);
1613 mlx5_function_teardown(dev, false);
1614 out:
1615 mutex_unlock(&dev->intf_state_mutex);
1616 }
1617
mlx5_unload_one(struct mlx5_core_dev * dev,bool suspend)1618 void mlx5_unload_one(struct mlx5_core_dev *dev, bool suspend)
1619 {
1620 struct devlink *devlink = priv_to_devlink(dev);
1621
1622 devl_lock(devlink);
1623 mlx5_unload_one_devl_locked(dev, suspend);
1624 devl_unlock(devlink);
1625 }
1626
1627 /* In case of light probe, we don't need a full query of hca_caps, but only the bellow caps.
1628 * A full query of hca_caps will be done when the device will reload.
1629 */
mlx5_query_hca_caps_light(struct mlx5_core_dev * dev)1630 static int mlx5_query_hca_caps_light(struct mlx5_core_dev *dev)
1631 {
1632 int err;
1633
1634 err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL);
1635 if (err)
1636 return err;
1637
1638 if (MLX5_CAP_GEN(dev, eth_net_offloads)) {
1639 err = mlx5_core_get_caps_mode(dev, MLX5_CAP_ETHERNET_OFFLOADS,
1640 HCA_CAP_OPMOD_GET_CUR);
1641 if (err)
1642 return err;
1643 }
1644
1645 if (MLX5_CAP_GEN(dev, nic_flow_table) ||
1646 MLX5_CAP_GEN(dev, ipoib_enhanced_offloads)) {
1647 err = mlx5_core_get_caps_mode(dev, MLX5_CAP_FLOW_TABLE,
1648 HCA_CAP_OPMOD_GET_CUR);
1649 if (err)
1650 return err;
1651 }
1652
1653 if (MLX5_CAP_GEN_64(dev, general_obj_types) &
1654 MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_NET_Q) {
1655 err = mlx5_core_get_caps_mode(dev, MLX5_CAP_VDPA_EMULATION,
1656 HCA_CAP_OPMOD_GET_CUR);
1657 if (err)
1658 return err;
1659 }
1660
1661 return 0;
1662 }
1663
mlx5_init_one_light(struct mlx5_core_dev * dev)1664 int mlx5_init_one_light(struct mlx5_core_dev *dev)
1665 {
1666 struct devlink *devlink = priv_to_devlink(dev);
1667 int err;
1668
1669 dev->state = MLX5_DEVICE_STATE_UP;
1670 err = mlx5_function_enable(dev, true, mlx5_tout_ms(dev, FW_PRE_INIT_TIMEOUT));
1671 if (err) {
1672 mlx5_core_warn(dev, "mlx5_function_enable err=%d\n", err);
1673 goto out;
1674 }
1675
1676 err = mlx5_query_hca_caps_light(dev);
1677 if (err) {
1678 mlx5_core_warn(dev, "mlx5_query_hca_caps_light err=%d\n", err);
1679 goto query_hca_caps_err;
1680 }
1681
1682 devl_lock(devlink);
1683 err = mlx5_devlink_params_register(priv_to_devlink(dev));
1684 devl_unlock(devlink);
1685 if (err) {
1686 mlx5_core_warn(dev, "mlx5_devlink_param_reg err = %d\n", err);
1687 goto query_hca_caps_err;
1688 }
1689
1690 return 0;
1691
1692 query_hca_caps_err:
1693 mlx5_function_disable(dev, true);
1694 out:
1695 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
1696 return err;
1697 }
1698
mlx5_uninit_one_light(struct mlx5_core_dev * dev)1699 void mlx5_uninit_one_light(struct mlx5_core_dev *dev)
1700 {
1701 struct devlink *devlink = priv_to_devlink(dev);
1702
1703 devl_lock(devlink);
1704 mlx5_devlink_params_unregister(priv_to_devlink(dev));
1705 devl_unlock(devlink);
1706 if (dev->state != MLX5_DEVICE_STATE_UP)
1707 return;
1708 mlx5_function_disable(dev, true);
1709 }
1710
1711 /* xxx_light() function are used in order to configure the device without full
1712 * init (light init). e.g.: There isn't a point in reload a device to light state.
1713 * Hence, mlx5_load_one_light() isn't needed.
1714 */
1715
mlx5_unload_one_light(struct mlx5_core_dev * dev)1716 void mlx5_unload_one_light(struct mlx5_core_dev *dev)
1717 {
1718 if (dev->state != MLX5_DEVICE_STATE_UP)
1719 return;
1720 mlx5_function_disable(dev, false);
1721 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
1722 }
1723
1724 static const int types[] = {
1725 MLX5_CAP_GENERAL,
1726 MLX5_CAP_GENERAL_2,
1727 MLX5_CAP_ETHERNET_OFFLOADS,
1728 MLX5_CAP_IPOIB_ENHANCED_OFFLOADS,
1729 MLX5_CAP_ODP,
1730 MLX5_CAP_ATOMIC,
1731 MLX5_CAP_ROCE,
1732 MLX5_CAP_IPOIB_OFFLOADS,
1733 MLX5_CAP_FLOW_TABLE,
1734 MLX5_CAP_ESWITCH_FLOW_TABLE,
1735 MLX5_CAP_ESWITCH,
1736 MLX5_CAP_QOS,
1737 MLX5_CAP_DEBUG,
1738 MLX5_CAP_DEV_MEM,
1739 MLX5_CAP_DEV_EVENT,
1740 MLX5_CAP_TLS,
1741 MLX5_CAP_VDPA_EMULATION,
1742 MLX5_CAP_IPSEC,
1743 MLX5_CAP_PORT_SELECTION,
1744 MLX5_CAP_MACSEC,
1745 MLX5_CAP_ADV_VIRTUALIZATION,
1746 MLX5_CAP_CRYPTO,
1747 };
1748
mlx5_hca_caps_free(struct mlx5_core_dev * dev)1749 static void mlx5_hca_caps_free(struct mlx5_core_dev *dev)
1750 {
1751 int type;
1752 int i;
1753
1754 for (i = 0; i < ARRAY_SIZE(types); i++) {
1755 type = types[i];
1756 kfree(dev->caps.hca[type]);
1757 }
1758 }
1759
mlx5_hca_caps_alloc(struct mlx5_core_dev * dev)1760 static int mlx5_hca_caps_alloc(struct mlx5_core_dev *dev)
1761 {
1762 struct mlx5_hca_cap *cap;
1763 int type;
1764 int i;
1765
1766 for (i = 0; i < ARRAY_SIZE(types); i++) {
1767 cap = kzalloc(sizeof(*cap), GFP_KERNEL);
1768 if (!cap)
1769 goto err;
1770 type = types[i];
1771 dev->caps.hca[type] = cap;
1772 }
1773
1774 return 0;
1775
1776 err:
1777 mlx5_hca_caps_free(dev);
1778 return -ENOMEM;
1779 }
1780
vhca_id_show(struct seq_file * file,void * priv)1781 static int vhca_id_show(struct seq_file *file, void *priv)
1782 {
1783 struct mlx5_core_dev *dev = file->private;
1784
1785 seq_printf(file, "0x%x\n", MLX5_CAP_GEN(dev, vhca_id));
1786 return 0;
1787 }
1788
1789 DEFINE_SHOW_ATTRIBUTE(vhca_id);
1790
mlx5_mdev_init(struct mlx5_core_dev * dev,int profile_idx)1791 int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx)
1792 {
1793 struct mlx5_priv *priv = &dev->priv;
1794 int err;
1795
1796 memcpy(&dev->profile, &profile[profile_idx], sizeof(dev->profile));
1797 lockdep_register_key(&dev->lock_key);
1798 mutex_init(&dev->intf_state_mutex);
1799 lockdep_set_class(&dev->intf_state_mutex, &dev->lock_key);
1800 mutex_init(&dev->mlx5e_res.uplink_netdev_lock);
1801
1802 mutex_init(&priv->bfregs.reg_head.lock);
1803 mutex_init(&priv->bfregs.wc_head.lock);
1804 INIT_LIST_HEAD(&priv->bfregs.reg_head.list);
1805 INIT_LIST_HEAD(&priv->bfregs.wc_head.list);
1806
1807 mutex_init(&priv->alloc_mutex);
1808 mutex_init(&priv->pgdir_mutex);
1809 INIT_LIST_HEAD(&priv->pgdir_list);
1810
1811 priv->numa_node = dev_to_node(mlx5_core_dma_dev(dev));
1812 priv->dbg.dbg_root = debugfs_create_dir(dev_name(dev->device),
1813 mlx5_debugfs_root);
1814 debugfs_create_file("vhca_id", 0400, priv->dbg.dbg_root, dev, &vhca_id_fops);
1815 INIT_LIST_HEAD(&priv->traps);
1816
1817 err = mlx5_cmd_init(dev);
1818 if (err) {
1819 mlx5_core_err(dev, "Failed initializing cmdif SW structs, aborting\n");
1820 goto err_cmd_init;
1821 }
1822
1823 err = mlx5_tout_init(dev);
1824 if (err) {
1825 mlx5_core_err(dev, "Failed initializing timeouts, aborting\n");
1826 goto err_timeout_init;
1827 }
1828
1829 err = mlx5_health_init(dev);
1830 if (err)
1831 goto err_health_init;
1832
1833 err = mlx5_pagealloc_init(dev);
1834 if (err)
1835 goto err_pagealloc_init;
1836
1837 err = mlx5_adev_init(dev);
1838 if (err)
1839 goto err_adev_init;
1840
1841 err = mlx5_hca_caps_alloc(dev);
1842 if (err)
1843 goto err_hca_caps;
1844
1845 /* The conjunction of sw_vhca_id with sw_owner_id will be a global
1846 * unique id per function which uses mlx5_core.
1847 * Those values are supplied to FW as part of the init HCA command to
1848 * be used by both driver and FW when it's applicable.
1849 */
1850 dev->priv.sw_vhca_id = ida_alloc_range(&sw_vhca_ida, 1,
1851 MAX_SW_VHCA_ID,
1852 GFP_KERNEL);
1853 if (dev->priv.sw_vhca_id < 0)
1854 mlx5_core_err(dev, "failed to allocate sw_vhca_id, err=%d\n",
1855 dev->priv.sw_vhca_id);
1856
1857 return 0;
1858
1859 err_hca_caps:
1860 mlx5_adev_cleanup(dev);
1861 err_adev_init:
1862 mlx5_pagealloc_cleanup(dev);
1863 err_pagealloc_init:
1864 mlx5_health_cleanup(dev);
1865 err_health_init:
1866 mlx5_tout_cleanup(dev);
1867 err_timeout_init:
1868 mlx5_cmd_cleanup(dev);
1869 err_cmd_init:
1870 debugfs_remove(dev->priv.dbg.dbg_root);
1871 mutex_destroy(&priv->pgdir_mutex);
1872 mutex_destroy(&priv->alloc_mutex);
1873 mutex_destroy(&priv->bfregs.wc_head.lock);
1874 mutex_destroy(&priv->bfregs.reg_head.lock);
1875 mutex_destroy(&dev->intf_state_mutex);
1876 lockdep_unregister_key(&dev->lock_key);
1877 return err;
1878 }
1879
mlx5_mdev_uninit(struct mlx5_core_dev * dev)1880 void mlx5_mdev_uninit(struct mlx5_core_dev *dev)
1881 {
1882 struct mlx5_priv *priv = &dev->priv;
1883
1884 if (priv->sw_vhca_id > 0)
1885 ida_free(&sw_vhca_ida, dev->priv.sw_vhca_id);
1886
1887 mlx5_hca_caps_free(dev);
1888 mlx5_adev_cleanup(dev);
1889 mlx5_pagealloc_cleanup(dev);
1890 mlx5_health_cleanup(dev);
1891 mlx5_tout_cleanup(dev);
1892 mlx5_cmd_cleanup(dev);
1893 debugfs_remove_recursive(dev->priv.dbg.dbg_root);
1894 mutex_destroy(&priv->pgdir_mutex);
1895 mutex_destroy(&priv->alloc_mutex);
1896 mutex_destroy(&priv->bfregs.wc_head.lock);
1897 mutex_destroy(&priv->bfregs.reg_head.lock);
1898 mutex_destroy(&dev->mlx5e_res.uplink_netdev_lock);
1899 mutex_destroy(&dev->intf_state_mutex);
1900 lockdep_unregister_key(&dev->lock_key);
1901 }
1902
probe_one(struct pci_dev * pdev,const struct pci_device_id * id)1903 static int probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
1904 {
1905 struct mlx5_core_dev *dev;
1906 struct devlink *devlink;
1907 int err;
1908
1909 devlink = mlx5_devlink_alloc(&pdev->dev);
1910 if (!devlink) {
1911 dev_err(&pdev->dev, "devlink alloc failed\n");
1912 return -ENOMEM;
1913 }
1914
1915 dev = devlink_priv(devlink);
1916 dev->device = &pdev->dev;
1917 dev->pdev = pdev;
1918
1919 dev->coredev_type = id->driver_data & MLX5_PCI_DEV_IS_VF ?
1920 MLX5_COREDEV_VF : MLX5_COREDEV_PF;
1921
1922 dev->priv.adev_idx = mlx5_adev_idx_alloc();
1923 if (dev->priv.adev_idx < 0) {
1924 err = dev->priv.adev_idx;
1925 goto adev_init_err;
1926 }
1927
1928 err = mlx5_mdev_init(dev, prof_sel);
1929 if (err)
1930 goto mdev_init_err;
1931
1932 err = mlx5_pci_init(dev, pdev, id);
1933 if (err) {
1934 mlx5_core_err(dev, "mlx5_pci_init failed with error code %d\n",
1935 err);
1936 goto pci_init_err;
1937 }
1938
1939 err = mlx5_init_one(dev);
1940 if (err) {
1941 mlx5_core_err(dev, "mlx5_init_one failed with error code %d\n",
1942 err);
1943 goto err_init_one;
1944 }
1945
1946 err = mlx5_crdump_enable(dev);
1947 if (err)
1948 dev_err(&pdev->dev, "mlx5_crdump_enable failed with error code %d\n", err);
1949
1950 err = mlx5_hwmon_dev_register(dev);
1951 if (err)
1952 mlx5_core_err(dev, "mlx5_hwmon_dev_register failed with error code %d\n", err);
1953
1954 pci_save_state(pdev);
1955 devlink_register(devlink);
1956 return 0;
1957
1958 err_init_one:
1959 mlx5_pci_close(dev);
1960 pci_init_err:
1961 mlx5_mdev_uninit(dev);
1962 mdev_init_err:
1963 mlx5_adev_idx_free(dev->priv.adev_idx);
1964 adev_init_err:
1965 mlx5_devlink_free(devlink);
1966
1967 return err;
1968 }
1969
remove_one(struct pci_dev * pdev)1970 static void remove_one(struct pci_dev *pdev)
1971 {
1972 struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
1973 struct devlink *devlink = priv_to_devlink(dev);
1974
1975 set_bit(MLX5_BREAK_FW_WAIT, &dev->intf_state);
1976 /* mlx5_drain_fw_reset() and mlx5_drain_health_wq() are using
1977 * devlink notify APIs.
1978 * Hence, we must drain them before unregistering the devlink.
1979 */
1980 mlx5_drain_fw_reset(dev);
1981 mlx5_drain_health_wq(dev);
1982 devlink_unregister(devlink);
1983 mlx5_sriov_disable(pdev, false);
1984 mlx5_hwmon_dev_unregister(dev);
1985 mlx5_crdump_disable(dev);
1986 mlx5_uninit_one(dev);
1987 mlx5_pci_close(dev);
1988 mlx5_mdev_uninit(dev);
1989 mlx5_adev_idx_free(dev->priv.adev_idx);
1990 mlx5_devlink_free(devlink);
1991 }
1992
1993 #define mlx5_pci_trace(dev, fmt, ...) ({ \
1994 struct mlx5_core_dev *__dev = (dev); \
1995 mlx5_core_info(__dev, "%s Device state = %d health sensors: %d pci_status: %d. " fmt, \
1996 __func__, __dev->state, mlx5_health_check_fatal_sensors(__dev), \
1997 __dev->pci_status, ##__VA_ARGS__); \
1998 })
1999
result2str(enum pci_ers_result result)2000 static const char *result2str(enum pci_ers_result result)
2001 {
2002 return result == PCI_ERS_RESULT_NEED_RESET ? "need reset" :
2003 result == PCI_ERS_RESULT_DISCONNECT ? "disconnect" :
2004 result == PCI_ERS_RESULT_RECOVERED ? "recovered" :
2005 "unknown";
2006 }
2007
mlx5_pci_err_detected(struct pci_dev * pdev,pci_channel_state_t state)2008 static pci_ers_result_t mlx5_pci_err_detected(struct pci_dev *pdev,
2009 pci_channel_state_t state)
2010 {
2011 struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
2012 enum pci_ers_result res;
2013
2014 mlx5_pci_trace(dev, "Enter, pci channel state = %d\n", state);
2015
2016 mlx5_enter_error_state(dev, false);
2017 mlx5_error_sw_reset(dev);
2018 mlx5_unload_one(dev, false);
2019 mlx5_drain_health_wq(dev);
2020 mlx5_pci_disable_device(dev);
2021
2022 res = state == pci_channel_io_perm_failure ?
2023 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
2024
2025 mlx5_core_info(dev, "%s Device state = %d pci_status: %d. Exit, result = %d, %s\n",
2026 __func__, dev->state, dev->pci_status, res, result2str(res));
2027 return res;
2028 }
2029
2030 /* wait for the device to show vital signs by waiting
2031 * for the health counter to start counting.
2032 */
wait_vital(struct pci_dev * pdev)2033 static int wait_vital(struct pci_dev *pdev)
2034 {
2035 struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
2036 struct mlx5_core_health *health = &dev->priv.health;
2037 const int niter = 100;
2038 u32 last_count = 0;
2039 u32 count;
2040 int i;
2041
2042 for (i = 0; i < niter; i++) {
2043 count = ioread32be(health->health_counter);
2044 if (count && count != 0xffffffff) {
2045 if (last_count && last_count != count) {
2046 mlx5_core_info(dev,
2047 "wait vital counter value 0x%x after %d iterations\n",
2048 count, i);
2049 return 0;
2050 }
2051 last_count = count;
2052 }
2053 msleep(50);
2054 }
2055
2056 return -ETIMEDOUT;
2057 }
2058
mlx5_pci_slot_reset(struct pci_dev * pdev)2059 static pci_ers_result_t mlx5_pci_slot_reset(struct pci_dev *pdev)
2060 {
2061 enum pci_ers_result res = PCI_ERS_RESULT_DISCONNECT;
2062 struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
2063 int err;
2064
2065 mlx5_core_info(dev, "%s Device state = %d pci_status: %d. Enter\n",
2066 __func__, dev->state, dev->pci_status);
2067
2068 err = mlx5_pci_enable_device(dev);
2069 if (err) {
2070 mlx5_core_err(dev, "%s: mlx5_pci_enable_device failed with error code: %d\n",
2071 __func__, err);
2072 goto out;
2073 }
2074
2075 pci_set_master(pdev);
2076 pci_restore_state(pdev);
2077 pci_save_state(pdev);
2078
2079 err = wait_vital(pdev);
2080 if (err) {
2081 mlx5_core_err(dev, "%s: wait vital failed with error code: %d\n",
2082 __func__, err);
2083 goto out;
2084 }
2085
2086 res = PCI_ERS_RESULT_RECOVERED;
2087 out:
2088 mlx5_core_info(dev, "%s Device state = %d pci_status: %d. Exit, err = %d, result = %d, %s\n",
2089 __func__, dev->state, dev->pci_status, err, res, result2str(res));
2090 return res;
2091 }
2092
mlx5_pci_resume(struct pci_dev * pdev)2093 static void mlx5_pci_resume(struct pci_dev *pdev)
2094 {
2095 struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
2096 int err;
2097
2098 mlx5_pci_trace(dev, "Enter, loading driver..\n");
2099
2100 err = mlx5_load_one(dev, false);
2101
2102 if (!err)
2103 devlink_health_reporter_state_update(dev->priv.health.fw_fatal_reporter,
2104 DEVLINK_HEALTH_REPORTER_STATE_HEALTHY);
2105
2106 mlx5_pci_trace(dev, "Done, err = %d, device %s\n", err,
2107 !err ? "recovered" : "Failed");
2108 }
2109
2110 static const struct pci_error_handlers mlx5_err_handler = {
2111 .error_detected = mlx5_pci_err_detected,
2112 .slot_reset = mlx5_pci_slot_reset,
2113 .resume = mlx5_pci_resume
2114 };
2115
mlx5_try_fast_unload(struct mlx5_core_dev * dev)2116 static int mlx5_try_fast_unload(struct mlx5_core_dev *dev)
2117 {
2118 bool fast_teardown = false, force_teardown = false;
2119 int ret = 1;
2120
2121 fast_teardown = MLX5_CAP_GEN(dev, fast_teardown);
2122 force_teardown = MLX5_CAP_GEN(dev, force_teardown);
2123
2124 mlx5_core_dbg(dev, "force teardown firmware support=%d\n", force_teardown);
2125 mlx5_core_dbg(dev, "fast teardown firmware support=%d\n", fast_teardown);
2126
2127 if (!fast_teardown && !force_teardown)
2128 return -EOPNOTSUPP;
2129
2130 if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
2131 mlx5_core_dbg(dev, "Device in internal error state, giving up\n");
2132 return -EAGAIN;
2133 }
2134
2135 /* Panic tear down fw command will stop the PCI bus communication
2136 * with the HCA, so the health poll is no longer needed.
2137 */
2138 mlx5_drain_health_wq(dev);
2139 mlx5_stop_health_poll(dev, false);
2140
2141 ret = mlx5_cmd_fast_teardown_hca(dev);
2142 if (!ret)
2143 goto succeed;
2144
2145 ret = mlx5_cmd_force_teardown_hca(dev);
2146 if (!ret)
2147 goto succeed;
2148
2149 mlx5_core_dbg(dev, "Firmware couldn't do fast unload error: %d\n", ret);
2150 mlx5_start_health_poll(dev);
2151 return ret;
2152
2153 succeed:
2154 mlx5_enter_error_state(dev, true);
2155
2156 /* Some platforms requiring freeing the IRQ's in the shutdown
2157 * flow. If they aren't freed they can't be allocated after
2158 * kexec. There is no need to cleanup the mlx5_core software
2159 * contexts.
2160 */
2161 mlx5_core_eq_free_irqs(dev);
2162
2163 return 0;
2164 }
2165
shutdown(struct pci_dev * pdev)2166 static void shutdown(struct pci_dev *pdev)
2167 {
2168 struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
2169 int err;
2170
2171 mlx5_core_info(dev, "Shutdown was called\n");
2172 set_bit(MLX5_BREAK_FW_WAIT, &dev->intf_state);
2173 err = mlx5_try_fast_unload(dev);
2174 if (err)
2175 mlx5_unload_one(dev, false);
2176 mlx5_pci_disable_device(dev);
2177 }
2178
mlx5_suspend(struct pci_dev * pdev,pm_message_t state)2179 static int mlx5_suspend(struct pci_dev *pdev, pm_message_t state)
2180 {
2181 struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
2182
2183 mlx5_unload_one(dev, true);
2184
2185 return 0;
2186 }
2187
mlx5_resume(struct pci_dev * pdev)2188 static int mlx5_resume(struct pci_dev *pdev)
2189 {
2190 struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
2191
2192 return mlx5_load_one(dev, false);
2193 }
2194
2195 static const struct pci_device_id mlx5_core_pci_table[] = {
2196 { PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_CONNECTIB) },
2197 { PCI_VDEVICE(MELLANOX, 0x1012), MLX5_PCI_DEV_IS_VF}, /* Connect-IB VF */
2198 { PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_CONNECTX4) },
2199 { PCI_VDEVICE(MELLANOX, 0x1014), MLX5_PCI_DEV_IS_VF}, /* ConnectX-4 VF */
2200 { PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_CONNECTX4_LX) },
2201 { PCI_VDEVICE(MELLANOX, 0x1016), MLX5_PCI_DEV_IS_VF}, /* ConnectX-4LX VF */
2202 { PCI_VDEVICE(MELLANOX, 0x1017) }, /* ConnectX-5, PCIe 3.0 */
2203 { PCI_VDEVICE(MELLANOX, 0x1018), MLX5_PCI_DEV_IS_VF}, /* ConnectX-5 VF */
2204 { PCI_VDEVICE(MELLANOX, 0x1019) }, /* ConnectX-5 Ex */
2205 { PCI_VDEVICE(MELLANOX, 0x101a), MLX5_PCI_DEV_IS_VF}, /* ConnectX-5 Ex VF */
2206 { PCI_VDEVICE(MELLANOX, 0x101b) }, /* ConnectX-6 */
2207 { PCI_VDEVICE(MELLANOX, 0x101c), MLX5_PCI_DEV_IS_VF}, /* ConnectX-6 VF */
2208 { PCI_VDEVICE(MELLANOX, 0x101d) }, /* ConnectX-6 Dx */
2209 { PCI_VDEVICE(MELLANOX, 0x101e), MLX5_PCI_DEV_IS_VF}, /* ConnectX Family mlx5Gen Virtual Function */
2210 { PCI_VDEVICE(MELLANOX, 0x101f) }, /* ConnectX-6 LX */
2211 { PCI_VDEVICE(MELLANOX, 0x1021) }, /* ConnectX-7 */
2212 { PCI_VDEVICE(MELLANOX, 0x1023) }, /* ConnectX-8 */
2213 { PCI_VDEVICE(MELLANOX, 0xa2d2) }, /* BlueField integrated ConnectX-5 network controller */
2214 { PCI_VDEVICE(MELLANOX, 0xa2d3), MLX5_PCI_DEV_IS_VF}, /* BlueField integrated ConnectX-5 network controller VF */
2215 { PCI_VDEVICE(MELLANOX, 0xa2d6) }, /* BlueField-2 integrated ConnectX-6 Dx network controller */
2216 { PCI_VDEVICE(MELLANOX, 0xa2dc) }, /* BlueField-3 integrated ConnectX-7 network controller */
2217 { PCI_VDEVICE(MELLANOX, 0xa2df) }, /* BlueField-4 integrated ConnectX-8 network controller */
2218 { 0, }
2219 };
2220
2221 MODULE_DEVICE_TABLE(pci, mlx5_core_pci_table);
2222
mlx5_disable_device(struct mlx5_core_dev * dev)2223 void mlx5_disable_device(struct mlx5_core_dev *dev)
2224 {
2225 mlx5_error_sw_reset(dev);
2226 mlx5_unload_one_devl_locked(dev, false);
2227 }
2228
mlx5_recover_device(struct mlx5_core_dev * dev)2229 int mlx5_recover_device(struct mlx5_core_dev *dev)
2230 {
2231 if (!mlx5_core_is_sf(dev)) {
2232 mlx5_pci_disable_device(dev);
2233 if (mlx5_pci_slot_reset(dev->pdev) != PCI_ERS_RESULT_RECOVERED)
2234 return -EIO;
2235 }
2236
2237 return mlx5_load_one_devl_locked(dev, true);
2238 }
2239
2240 static struct pci_driver mlx5_core_driver = {
2241 .name = KBUILD_MODNAME,
2242 .id_table = mlx5_core_pci_table,
2243 .probe = probe_one,
2244 .remove = remove_one,
2245 .suspend = mlx5_suspend,
2246 .resume = mlx5_resume,
2247 .shutdown = shutdown,
2248 .err_handler = &mlx5_err_handler,
2249 .sriov_configure = mlx5_core_sriov_configure,
2250 .sriov_get_vf_total_msix = mlx5_sriov_get_vf_total_msix,
2251 .sriov_set_msix_vec_count = mlx5_core_sriov_set_msix_vec_count,
2252 };
2253
2254 /**
2255 * mlx5_vf_get_core_dev - Get the mlx5 core device from a given VF PCI device if
2256 * mlx5_core is its driver.
2257 * @pdev: The associated PCI device.
2258 *
2259 * Upon return the interface state lock stay held to let caller uses it safely.
2260 * Caller must ensure to use the returned mlx5 device for a narrow window
2261 * and put it back with mlx5_vf_put_core_dev() immediately once usage was over.
2262 *
2263 * Return: Pointer to the associated mlx5_core_dev or NULL.
2264 */
mlx5_vf_get_core_dev(struct pci_dev * pdev)2265 struct mlx5_core_dev *mlx5_vf_get_core_dev(struct pci_dev *pdev)
2266 {
2267 struct mlx5_core_dev *mdev;
2268
2269 mdev = pci_iov_get_pf_drvdata(pdev, &mlx5_core_driver);
2270 if (IS_ERR(mdev))
2271 return NULL;
2272
2273 mutex_lock(&mdev->intf_state_mutex);
2274 if (!test_bit(MLX5_INTERFACE_STATE_UP, &mdev->intf_state)) {
2275 mutex_unlock(&mdev->intf_state_mutex);
2276 return NULL;
2277 }
2278
2279 return mdev;
2280 }
2281 EXPORT_SYMBOL(mlx5_vf_get_core_dev);
2282
2283 /**
2284 * mlx5_vf_put_core_dev - Put the mlx5 core device back.
2285 * @mdev: The mlx5 core device.
2286 *
2287 * Upon return the interface state lock is unlocked and caller should not
2288 * access the mdev any more.
2289 */
mlx5_vf_put_core_dev(struct mlx5_core_dev * mdev)2290 void mlx5_vf_put_core_dev(struct mlx5_core_dev *mdev)
2291 {
2292 mutex_unlock(&mdev->intf_state_mutex);
2293 }
2294 EXPORT_SYMBOL(mlx5_vf_put_core_dev);
2295
mlx5_core_verify_params(void)2296 static void mlx5_core_verify_params(void)
2297 {
2298 if (prof_sel >= ARRAY_SIZE(profile)) {
2299 pr_warn("mlx5_core: WARNING: Invalid module parameter prof_sel %d, valid range 0-%zu, changing back to default(%d)\n",
2300 prof_sel,
2301 ARRAY_SIZE(profile) - 1,
2302 MLX5_DEFAULT_PROF);
2303 prof_sel = MLX5_DEFAULT_PROF;
2304 }
2305 }
2306
mlx5_init(void)2307 static int __init mlx5_init(void)
2308 {
2309 int err;
2310
2311 WARN_ONCE(strcmp(MLX5_ADEV_NAME, KBUILD_MODNAME),
2312 "mlx5_core name not in sync with kernel module name");
2313
2314 get_random_bytes(&sw_owner_id, sizeof(sw_owner_id));
2315
2316 mlx5_core_verify_params();
2317 mlx5_register_debugfs();
2318
2319 err = mlx5e_init();
2320 if (err)
2321 goto err_debug;
2322
2323 err = mlx5_sf_driver_register();
2324 if (err)
2325 goto err_sf;
2326
2327 err = pci_register_driver(&mlx5_core_driver);
2328 if (err)
2329 goto err_pci;
2330
2331 return 0;
2332
2333 err_pci:
2334 mlx5_sf_driver_unregister();
2335 err_sf:
2336 mlx5e_cleanup();
2337 err_debug:
2338 mlx5_unregister_debugfs();
2339 return err;
2340 }
2341
mlx5_cleanup(void)2342 static void __exit mlx5_cleanup(void)
2343 {
2344 pci_unregister_driver(&mlx5_core_driver);
2345 mlx5_sf_driver_unregister();
2346 mlx5e_cleanup();
2347 mlx5_unregister_debugfs();
2348 }
2349
2350 module_init(mlx5_init);
2351 module_exit(mlx5_cleanup);
2352