1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2020, Mellanox Technologies inc.  All rights reserved. */
3 
4 #include <devlink.h>
5 
6 #include "fw_reset.h"
7 #include "diag/fw_tracer.h"
8 #include "lib/tout.h"
9 
10 enum {
11 	MLX5_FW_RESET_FLAGS_RESET_REQUESTED,
12 	MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST,
13 	MLX5_FW_RESET_FLAGS_PENDING_COMP,
14 	MLX5_FW_RESET_FLAGS_DROP_NEW_REQUESTS,
15 	MLX5_FW_RESET_FLAGS_RELOAD_REQUIRED
16 };
17 
18 struct mlx5_fw_reset {
19 	struct mlx5_core_dev *dev;
20 	struct mlx5_nb nb;
21 	struct workqueue_struct *wq;
22 	struct work_struct fw_live_patch_work;
23 	struct work_struct reset_request_work;
24 	struct work_struct reset_unload_work;
25 	struct work_struct reset_reload_work;
26 	struct work_struct reset_now_work;
27 	struct work_struct reset_abort_work;
28 	unsigned long reset_flags;
29 	u8 reset_method;
30 	struct timer_list timer;
31 	struct completion done;
32 	int ret;
33 };
34 
35 enum {
36 	MLX5_FW_RST_STATE_IDLE = 0,
37 	MLX5_FW_RST_STATE_TOGGLE_REQ = 4,
38 	MLX5_FW_RST_STATE_DROP_MODE = 5,
39 };
40 
41 enum {
42 	MLX5_RST_STATE_BIT_NUM = 12,
43 	MLX5_RST_ACK_BIT_NUM = 22,
44 };
45 
mlx5_get_fw_rst_state(struct mlx5_core_dev * dev)46 static u8 mlx5_get_fw_rst_state(struct mlx5_core_dev *dev)
47 {
48 	return (ioread32be(&dev->iseg->initializing) >> MLX5_RST_STATE_BIT_NUM) & 0xF;
49 }
50 
mlx5_set_fw_rst_ack(struct mlx5_core_dev * dev)51 static void mlx5_set_fw_rst_ack(struct mlx5_core_dev *dev)
52 {
53 	iowrite32be(BIT(MLX5_RST_ACK_BIT_NUM), &dev->iseg->initializing);
54 }
55 
mlx5_fw_reset_enable_remote_dev_reset_set(struct devlink * devlink,u32 id,struct devlink_param_gset_ctx * ctx,struct netlink_ext_ack * extack)56 static int mlx5_fw_reset_enable_remote_dev_reset_set(struct devlink *devlink, u32 id,
57 						     struct devlink_param_gset_ctx *ctx,
58 						     struct netlink_ext_ack *extack)
59 {
60 	struct mlx5_core_dev *dev = devlink_priv(devlink);
61 	struct mlx5_fw_reset *fw_reset;
62 
63 	fw_reset = dev->priv.fw_reset;
64 
65 	if (ctx->val.vbool)
66 		clear_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags);
67 	else
68 		set_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags);
69 	return 0;
70 }
71 
mlx5_fw_reset_enable_remote_dev_reset_get(struct devlink * devlink,u32 id,struct devlink_param_gset_ctx * ctx)72 static int mlx5_fw_reset_enable_remote_dev_reset_get(struct devlink *devlink, u32 id,
73 						     struct devlink_param_gset_ctx *ctx)
74 {
75 	struct mlx5_core_dev *dev = devlink_priv(devlink);
76 	struct mlx5_fw_reset *fw_reset;
77 
78 	fw_reset = dev->priv.fw_reset;
79 
80 	ctx->val.vbool = !test_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST,
81 				   &fw_reset->reset_flags);
82 	return 0;
83 }
84 
mlx5_reg_mfrl_set(struct mlx5_core_dev * dev,u8 reset_level,u8 reset_type_sel,u8 sync_resp,bool sync_start)85 static int mlx5_reg_mfrl_set(struct mlx5_core_dev *dev, u8 reset_level,
86 			     u8 reset_type_sel, u8 sync_resp, bool sync_start)
87 {
88 	u32 out[MLX5_ST_SZ_DW(mfrl_reg)] = {};
89 	u32 in[MLX5_ST_SZ_DW(mfrl_reg)] = {};
90 
91 	MLX5_SET(mfrl_reg, in, reset_level, reset_level);
92 	MLX5_SET(mfrl_reg, in, rst_type_sel, reset_type_sel);
93 	MLX5_SET(mfrl_reg, in, pci_sync_for_fw_update_resp, sync_resp);
94 	MLX5_SET(mfrl_reg, in, pci_sync_for_fw_update_start, sync_start);
95 
96 	return mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), MLX5_REG_MFRL, 0, 1);
97 }
98 
mlx5_reg_mfrl_query(struct mlx5_core_dev * dev,u8 * reset_level,u8 * reset_type,u8 * reset_state,u8 * reset_method)99 static int mlx5_reg_mfrl_query(struct mlx5_core_dev *dev, u8 *reset_level,
100 			       u8 *reset_type, u8 *reset_state, u8 *reset_method)
101 {
102 	u32 out[MLX5_ST_SZ_DW(mfrl_reg)] = {};
103 	u32 in[MLX5_ST_SZ_DW(mfrl_reg)] = {};
104 	int err;
105 
106 	err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), MLX5_REG_MFRL, 0, 0);
107 	if (err)
108 		return err;
109 
110 	if (reset_level)
111 		*reset_level = MLX5_GET(mfrl_reg, out, reset_level);
112 	if (reset_type)
113 		*reset_type = MLX5_GET(mfrl_reg, out, reset_type);
114 	if (reset_state)
115 		*reset_state = MLX5_GET(mfrl_reg, out, reset_state);
116 	if (reset_method)
117 		*reset_method = MLX5_GET(mfrl_reg, out, pci_reset_req_method);
118 
119 	return 0;
120 }
121 
mlx5_fw_reset_query(struct mlx5_core_dev * dev,u8 * reset_level,u8 * reset_type)122 int mlx5_fw_reset_query(struct mlx5_core_dev *dev, u8 *reset_level, u8 *reset_type)
123 {
124 	return mlx5_reg_mfrl_query(dev, reset_level, reset_type, NULL, NULL);
125 }
126 
mlx5_fw_reset_get_reset_method(struct mlx5_core_dev * dev,u8 * reset_method)127 static int mlx5_fw_reset_get_reset_method(struct mlx5_core_dev *dev,
128 					  u8 *reset_method)
129 {
130 	if (!MLX5_CAP_GEN(dev, pcie_reset_using_hotreset_method)) {
131 		*reset_method = MLX5_MFRL_REG_PCI_RESET_METHOD_LINK_TOGGLE;
132 		return 0;
133 	}
134 
135 	return mlx5_reg_mfrl_query(dev, NULL, NULL, NULL, reset_method);
136 }
137 
mlx5_fw_reset_get_reset_state_err(struct mlx5_core_dev * dev,struct netlink_ext_ack * extack)138 static int mlx5_fw_reset_get_reset_state_err(struct mlx5_core_dev *dev,
139 					     struct netlink_ext_ack *extack)
140 {
141 	u8 reset_state;
142 
143 	if (mlx5_reg_mfrl_query(dev, NULL, NULL, &reset_state, NULL))
144 		goto out;
145 
146 	if (!reset_state)
147 		return 0;
148 
149 	switch (reset_state) {
150 	case MLX5_MFRL_REG_RESET_STATE_IN_NEGOTIATION:
151 	case MLX5_MFRL_REG_RESET_STATE_RESET_IN_PROGRESS:
152 		NL_SET_ERR_MSG_MOD(extack, "Sync reset still in progress");
153 		return -EBUSY;
154 	case MLX5_MFRL_REG_RESET_STATE_NEG_TIMEOUT:
155 		NL_SET_ERR_MSG_MOD(extack, "Sync reset negotiation timeout");
156 		return -ETIMEDOUT;
157 	case MLX5_MFRL_REG_RESET_STATE_NACK:
158 		NL_SET_ERR_MSG_MOD(extack, "One of the hosts disabled reset");
159 		return -EPERM;
160 	case MLX5_MFRL_REG_RESET_STATE_UNLOAD_TIMEOUT:
161 		NL_SET_ERR_MSG_MOD(extack, "Sync reset unload timeout");
162 		return -ETIMEDOUT;
163 	}
164 
165 out:
166 	NL_SET_ERR_MSG_MOD(extack, "Sync reset failed");
167 	return -EIO;
168 }
169 
mlx5_fw_reset_set_reset_sync(struct mlx5_core_dev * dev,u8 reset_type_sel,struct netlink_ext_ack * extack)170 int mlx5_fw_reset_set_reset_sync(struct mlx5_core_dev *dev, u8 reset_type_sel,
171 				 struct netlink_ext_ack *extack)
172 {
173 	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
174 	u32 out[MLX5_ST_SZ_DW(mfrl_reg)] = {};
175 	u32 in[MLX5_ST_SZ_DW(mfrl_reg)] = {};
176 	int err, rst_res;
177 
178 	set_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags);
179 
180 	MLX5_SET(mfrl_reg, in, reset_level, MLX5_MFRL_REG_RESET_LEVEL3);
181 	MLX5_SET(mfrl_reg, in, rst_type_sel, reset_type_sel);
182 	MLX5_SET(mfrl_reg, in, pci_sync_for_fw_update_start, 1);
183 	err = mlx5_access_reg(dev, in, sizeof(in), out, sizeof(out),
184 			      MLX5_REG_MFRL, 0, 1, false);
185 	if (!err)
186 		return 0;
187 
188 	clear_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags);
189 	if (err == -EREMOTEIO && MLX5_CAP_MCAM_FEATURE(dev, reset_state)) {
190 		rst_res = mlx5_fw_reset_get_reset_state_err(dev, extack);
191 		return rst_res ? rst_res : err;
192 	}
193 
194 	NL_SET_ERR_MSG_MOD(extack, "Sync reset command failed");
195 	return mlx5_cmd_check(dev, err, in, out);
196 }
197 
mlx5_fw_reset_verify_fw_complete(struct mlx5_core_dev * dev,struct netlink_ext_ack * extack)198 int mlx5_fw_reset_verify_fw_complete(struct mlx5_core_dev *dev,
199 				     struct netlink_ext_ack *extack)
200 {
201 	u8 rst_state;
202 	int err;
203 
204 	err = mlx5_fw_reset_get_reset_state_err(dev, extack);
205 	if (err)
206 		return err;
207 
208 	rst_state = mlx5_get_fw_rst_state(dev);
209 	if (!rst_state)
210 		return 0;
211 
212 	mlx5_core_err(dev, "Sync reset did not complete, state=%d\n", rst_state);
213 	NL_SET_ERR_MSG_MOD(extack, "Sync reset did not complete successfully");
214 	return rst_state;
215 }
216 
mlx5_fw_reset_set_live_patch(struct mlx5_core_dev * dev)217 int mlx5_fw_reset_set_live_patch(struct mlx5_core_dev *dev)
218 {
219 	return mlx5_reg_mfrl_set(dev, MLX5_MFRL_REG_RESET_LEVEL0, 0, 0, false);
220 }
221 
mlx5_fw_reset_complete_reload(struct mlx5_core_dev * dev,bool unloaded)222 static void mlx5_fw_reset_complete_reload(struct mlx5_core_dev *dev, bool unloaded)
223 {
224 	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
225 	struct devlink *devlink = priv_to_devlink(dev);
226 
227 	/* if this is the driver that initiated the fw reset, devlink completed the reload */
228 	if (test_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags)) {
229 		complete(&fw_reset->done);
230 	} else {
231 		if (!unloaded)
232 			mlx5_unload_one(dev, false);
233 		if (mlx5_health_wait_pci_up(dev))
234 			mlx5_core_err(dev, "reset reload flow aborted, PCI reads still not working\n");
235 		else
236 			mlx5_load_one(dev, true);
237 		devl_lock(devlink);
238 		devlink_remote_reload_actions_performed(devlink, 0,
239 							BIT(DEVLINK_RELOAD_ACTION_DRIVER_REINIT) |
240 							BIT(DEVLINK_RELOAD_ACTION_FW_ACTIVATE));
241 		devl_unlock(devlink);
242 	}
243 }
244 
mlx5_stop_sync_reset_poll(struct mlx5_core_dev * dev)245 static void mlx5_stop_sync_reset_poll(struct mlx5_core_dev *dev)
246 {
247 	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
248 
249 	timer_delete_sync(&fw_reset->timer);
250 }
251 
mlx5_sync_reset_clear_reset_requested(struct mlx5_core_dev * dev,bool poll_health)252 static int mlx5_sync_reset_clear_reset_requested(struct mlx5_core_dev *dev, bool poll_health)
253 {
254 	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
255 
256 	if (!test_and_clear_bit(MLX5_FW_RESET_FLAGS_RESET_REQUESTED, &fw_reset->reset_flags)) {
257 		mlx5_core_warn(dev, "Reset request was already cleared\n");
258 		return -EALREADY;
259 	}
260 
261 	mlx5_stop_sync_reset_poll(dev);
262 	if (poll_health)
263 		mlx5_start_health_poll(dev);
264 	return 0;
265 }
266 
mlx5_sync_reset_reload_work(struct work_struct * work)267 static void mlx5_sync_reset_reload_work(struct work_struct *work)
268 {
269 	struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
270 						      reset_reload_work);
271 	struct mlx5_core_dev *dev = fw_reset->dev;
272 
273 	mlx5_sync_reset_clear_reset_requested(dev, false);
274 	mlx5_enter_error_state(dev, true);
275 	mlx5_fw_reset_complete_reload(dev, false);
276 }
277 
278 #define MLX5_RESET_POLL_INTERVAL	(HZ / 10)
poll_sync_reset(struct timer_list * t)279 static void poll_sync_reset(struct timer_list *t)
280 {
281 	struct mlx5_fw_reset *fw_reset = from_timer(fw_reset, t, timer);
282 	struct mlx5_core_dev *dev = fw_reset->dev;
283 	u32 fatal_error;
284 
285 	if (!test_bit(MLX5_FW_RESET_FLAGS_RESET_REQUESTED, &fw_reset->reset_flags))
286 		return;
287 
288 	fatal_error = mlx5_health_check_fatal_sensors(dev);
289 
290 	if (fatal_error) {
291 		mlx5_core_warn(dev, "Got Device Reset\n");
292 		if (!test_bit(MLX5_FW_RESET_FLAGS_DROP_NEW_REQUESTS, &fw_reset->reset_flags))
293 			queue_work(fw_reset->wq, &fw_reset->reset_reload_work);
294 		else
295 			mlx5_core_err(dev, "Device is being removed, Drop new reset work\n");
296 		return;
297 	}
298 
299 	mod_timer(&fw_reset->timer, round_jiffies(jiffies + MLX5_RESET_POLL_INTERVAL));
300 }
301 
mlx5_start_sync_reset_poll(struct mlx5_core_dev * dev)302 static void mlx5_start_sync_reset_poll(struct mlx5_core_dev *dev)
303 {
304 	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
305 
306 	timer_setup(&fw_reset->timer, poll_sync_reset, 0);
307 	fw_reset->timer.expires = round_jiffies(jiffies + MLX5_RESET_POLL_INTERVAL);
308 	add_timer(&fw_reset->timer);
309 }
310 
mlx5_fw_reset_set_reset_sync_ack(struct mlx5_core_dev * dev)311 static int mlx5_fw_reset_set_reset_sync_ack(struct mlx5_core_dev *dev)
312 {
313 	return mlx5_reg_mfrl_set(dev, MLX5_MFRL_REG_RESET_LEVEL3, 0, 1, false);
314 }
315 
mlx5_fw_reset_set_reset_sync_nack(struct mlx5_core_dev * dev)316 static int mlx5_fw_reset_set_reset_sync_nack(struct mlx5_core_dev *dev)
317 {
318 	return mlx5_reg_mfrl_set(dev, MLX5_MFRL_REG_RESET_LEVEL3, 0, 2, false);
319 }
320 
mlx5_sync_reset_set_reset_requested(struct mlx5_core_dev * dev)321 static int mlx5_sync_reset_set_reset_requested(struct mlx5_core_dev *dev)
322 {
323 	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
324 
325 	if (test_and_set_bit(MLX5_FW_RESET_FLAGS_RESET_REQUESTED, &fw_reset->reset_flags)) {
326 		mlx5_core_warn(dev, "Reset request was already set\n");
327 		return -EALREADY;
328 	}
329 	mlx5_stop_health_poll(dev, true);
330 	mlx5_start_sync_reset_poll(dev);
331 	return 0;
332 }
333 
mlx5_fw_live_patch_event(struct work_struct * work)334 static void mlx5_fw_live_patch_event(struct work_struct *work)
335 {
336 	struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
337 						      fw_live_patch_work);
338 	struct mlx5_core_dev *dev = fw_reset->dev;
339 
340 	mlx5_core_info(dev, "Live patch updated firmware version: %d.%d.%d\n", fw_rev_maj(dev),
341 		       fw_rev_min(dev), fw_rev_sub(dev));
342 
343 	if (mlx5_fw_tracer_reload(dev->tracer))
344 		mlx5_core_err(dev, "Failed to reload FW tracer\n");
345 }
346 
347 #if IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE)
mlx5_check_hotplug_interrupt(struct mlx5_core_dev * dev,struct pci_dev * bridge)348 static int mlx5_check_hotplug_interrupt(struct mlx5_core_dev *dev,
349 					struct pci_dev *bridge)
350 {
351 	u16 reg16;
352 	int err;
353 
354 	err = pcie_capability_read_word(bridge, PCI_EXP_SLTCTL, &reg16);
355 	if (err)
356 		return err;
357 
358 	if ((reg16 & PCI_EXP_SLTCTL_HPIE) && (reg16 & PCI_EXP_SLTCTL_DLLSCE)) {
359 		mlx5_core_warn(dev, "FW reset is not supported as HotPlug is enabled\n");
360 		return -EOPNOTSUPP;
361 	}
362 
363 	return 0;
364 }
365 #endif
366 
367 static const struct pci_device_id mgt_ifc_device_ids[] = {
368 	{ PCI_VDEVICE(MELLANOX, 0xc2d2) }, /* BlueField1 MGT interface device ID */
369 	{ PCI_VDEVICE(MELLANOX, 0xc2d3) }, /* BlueField2 MGT interface device ID */
370 	{ PCI_VDEVICE(MELLANOX, 0xc2d4) }, /* BlueField3-Lx MGT interface device ID */
371 	{ PCI_VDEVICE(MELLANOX, 0xc2d5) }, /* BlueField3 MGT interface device ID */
372 	{ PCI_VDEVICE(MELLANOX, 0xc2d6) }, /* BlueField4 MGT interface device ID */
373 };
374 
mlx5_is_mgt_ifc_pci_device(struct mlx5_core_dev * dev,u16 dev_id)375 static bool mlx5_is_mgt_ifc_pci_device(struct mlx5_core_dev *dev, u16 dev_id)
376 {
377 	int i;
378 
379 	for (i = 0; i < ARRAY_SIZE(mgt_ifc_device_ids); ++i)
380 		if (mgt_ifc_device_ids[i].device == dev_id)
381 			return true;
382 
383 	return false;
384 }
385 
mlx5_check_dev_ids(struct mlx5_core_dev * dev,u16 dev_id)386 static int mlx5_check_dev_ids(struct mlx5_core_dev *dev, u16 dev_id)
387 {
388 	struct pci_bus *bridge_bus = dev->pdev->bus;
389 	struct pci_dev *sdev;
390 	u16 sdev_id;
391 	int err;
392 
393 	/* Check that all functions under the pci bridge are PFs of
394 	 * this device otherwise fail this function.
395 	 */
396 	list_for_each_entry(sdev, &bridge_bus->devices, bus_list) {
397 		err = pci_read_config_word(sdev, PCI_DEVICE_ID, &sdev_id);
398 		if (err)
399 			return pcibios_err_to_errno(err);
400 
401 		if (sdev_id == dev_id)
402 			continue;
403 
404 		if (mlx5_is_mgt_ifc_pci_device(dev, sdev_id))
405 			continue;
406 
407 		mlx5_core_warn(dev, "unrecognized dev_id (0x%x)\n", sdev_id);
408 		return -EPERM;
409 	}
410 	return 0;
411 }
412 
mlx5_is_reset_now_capable(struct mlx5_core_dev * dev,u8 reset_method)413 static bool mlx5_is_reset_now_capable(struct mlx5_core_dev *dev,
414 				      u8 reset_method)
415 {
416 	struct pci_dev *bridge = dev->pdev->bus->self;
417 	u16 dev_id;
418 	int err;
419 
420 	if (!bridge) {
421 		mlx5_core_warn(dev, "PCI bus bridge is not accessible\n");
422 		return false;
423 	}
424 
425 	if (!MLX5_CAP_GEN(dev, fast_teardown)) {
426 		mlx5_core_warn(dev, "fast teardown is not supported by firmware\n");
427 		return false;
428 	}
429 
430 #if IS_ENABLED(CONFIG_HOTPLUG_PCI_PCIE)
431 	if (reset_method != MLX5_MFRL_REG_PCI_RESET_METHOD_HOT_RESET) {
432 		err = mlx5_check_hotplug_interrupt(dev, bridge);
433 		if (err)
434 			return false;
435 	}
436 #endif
437 
438 	err = pci_read_config_word(dev->pdev, PCI_DEVICE_ID, &dev_id);
439 	if (err)
440 		return false;
441 	return (!mlx5_check_dev_ids(dev, dev_id));
442 }
443 
mlx5_sync_reset_request_event(struct work_struct * work)444 static void mlx5_sync_reset_request_event(struct work_struct *work)
445 {
446 	struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
447 						      reset_request_work);
448 	struct mlx5_core_dev *dev = fw_reset->dev;
449 	int err;
450 
451 	err = mlx5_fw_reset_get_reset_method(dev, &fw_reset->reset_method);
452 	if (err)
453 		mlx5_core_warn(dev, "Failed reading MFRL, err %d\n", err);
454 
455 	if (err || test_bit(MLX5_FW_RESET_FLAGS_NACK_RESET_REQUEST, &fw_reset->reset_flags) ||
456 	    !mlx5_is_reset_now_capable(dev, fw_reset->reset_method)) {
457 		err = mlx5_fw_reset_set_reset_sync_nack(dev);
458 		mlx5_core_warn(dev, "PCI Sync FW Update Reset Nack %s",
459 			       err ? "Failed" : "Sent");
460 		return;
461 	}
462 	if (mlx5_sync_reset_set_reset_requested(dev))
463 		return;
464 
465 	err = mlx5_fw_reset_set_reset_sync_ack(dev);
466 	if (err)
467 		mlx5_core_warn(dev, "PCI Sync FW Update Reset Ack Failed. Error code: %d\n", err);
468 	else
469 		mlx5_core_warn(dev, "PCI Sync FW Update Reset Ack. Device reset is expected.\n");
470 }
471 
mlx5_pci_link_toggle(struct mlx5_core_dev * dev,u16 dev_id)472 static int mlx5_pci_link_toggle(struct mlx5_core_dev *dev, u16 dev_id)
473 {
474 	struct pci_bus *bridge_bus = dev->pdev->bus;
475 	struct pci_dev *bridge = bridge_bus->self;
476 	unsigned long timeout;
477 	struct pci_dev *sdev;
478 	int cap, err;
479 	u16 reg16;
480 
481 	cap = pci_find_capability(bridge, PCI_CAP_ID_EXP);
482 	if (!cap)
483 		return -EOPNOTSUPP;
484 
485 	list_for_each_entry(sdev, &bridge_bus->devices, bus_list) {
486 		pci_save_state(sdev);
487 		pci_cfg_access_lock(sdev);
488 	}
489 	/* PCI link toggle */
490 	err = pcie_capability_set_word(bridge, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_LD);
491 	if (err)
492 		return pcibios_err_to_errno(err);
493 	msleep(500);
494 	err = pcie_capability_clear_word(bridge, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_LD);
495 	if (err)
496 		return pcibios_err_to_errno(err);
497 
498 	/* Check link */
499 	if (!bridge->link_active_reporting) {
500 		mlx5_core_warn(dev, "No PCI link reporting capability\n");
501 		msleep(1000);
502 		goto restore;
503 	}
504 
505 	timeout = jiffies + msecs_to_jiffies(mlx5_tout_ms(dev, PCI_TOGGLE));
506 	do {
507 		err = pci_read_config_word(bridge, cap + PCI_EXP_LNKSTA, &reg16);
508 		if (err)
509 			return pcibios_err_to_errno(err);
510 		if (reg16 & PCI_EXP_LNKSTA_DLLLA)
511 			break;
512 		msleep(20);
513 	} while (!time_after(jiffies, timeout));
514 
515 	if (reg16 & PCI_EXP_LNKSTA_DLLLA) {
516 		mlx5_core_info(dev, "PCI Link up\n");
517 	} else {
518 		mlx5_core_err(dev, "PCI link not ready (0x%04x) after %llu ms\n",
519 			      reg16, mlx5_tout_ms(dev, PCI_TOGGLE));
520 		err = -ETIMEDOUT;
521 		goto restore;
522 	}
523 
524 	do {
525 		err = pci_read_config_word(dev->pdev, PCI_DEVICE_ID, &reg16);
526 		if (err)
527 			return pcibios_err_to_errno(err);
528 		if (reg16 == dev_id)
529 			break;
530 		msleep(20);
531 	} while (!time_after(jiffies, timeout));
532 
533 	if (reg16 == dev_id) {
534 		mlx5_core_info(dev, "Firmware responds to PCI config cycles again\n");
535 	} else {
536 		mlx5_core_err(dev, "Firmware is not responsive (0x%04x) after %llu ms\n",
537 			      reg16, mlx5_tout_ms(dev, PCI_TOGGLE));
538 		err = -ETIMEDOUT;
539 	}
540 
541 restore:
542 	list_for_each_entry(sdev, &bridge_bus->devices, bus_list) {
543 		pci_cfg_access_unlock(sdev);
544 		pci_restore_state(sdev);
545 	}
546 
547 	return err;
548 }
549 
mlx5_pci_reset_bus(struct mlx5_core_dev * dev)550 static int mlx5_pci_reset_bus(struct mlx5_core_dev *dev)
551 {
552 	if (!MLX5_CAP_GEN(dev, pcie_reset_using_hotreset_method))
553 		return -EOPNOTSUPP;
554 
555 	return pci_reset_bus(dev->pdev);
556 }
557 
mlx5_sync_pci_reset(struct mlx5_core_dev * dev,u8 reset_method)558 static int mlx5_sync_pci_reset(struct mlx5_core_dev *dev, u8 reset_method)
559 {
560 	u16 dev_id;
561 	int err;
562 
563 	err = pci_read_config_word(dev->pdev, PCI_DEVICE_ID, &dev_id);
564 	if (err)
565 		return pcibios_err_to_errno(err);
566 	err = mlx5_check_dev_ids(dev, dev_id);
567 	if (err)
568 		return err;
569 
570 	switch (reset_method) {
571 	case MLX5_MFRL_REG_PCI_RESET_METHOD_LINK_TOGGLE:
572 		err = mlx5_pci_link_toggle(dev, dev_id);
573 		if (err)
574 			mlx5_core_warn(dev, "mlx5_pci_link_toggle failed\n");
575 		break;
576 	case MLX5_MFRL_REG_PCI_RESET_METHOD_HOT_RESET:
577 		err = mlx5_pci_reset_bus(dev);
578 		if (err)
579 			mlx5_core_warn(dev, "mlx5_pci_reset_bus failed\n");
580 		break;
581 	default:
582 		return -EOPNOTSUPP;
583 	}
584 
585 	return err;
586 }
587 
mlx5_sync_reset_now_event(struct work_struct * work)588 static void mlx5_sync_reset_now_event(struct work_struct *work)
589 {
590 	struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
591 						      reset_now_work);
592 	struct mlx5_core_dev *dev = fw_reset->dev;
593 	int err;
594 
595 	if (mlx5_sync_reset_clear_reset_requested(dev, false))
596 		return;
597 
598 	mlx5_core_warn(dev, "Sync Reset now. Device is going to reset.\n");
599 
600 	err = mlx5_cmd_fast_teardown_hca(dev);
601 	if (err) {
602 		mlx5_core_warn(dev, "Fast teardown failed, no reset done, err %d\n", err);
603 		goto done;
604 	}
605 
606 	err = mlx5_sync_pci_reset(dev, fw_reset->reset_method);
607 	if (err) {
608 		mlx5_core_warn(dev, "mlx5_sync_pci_reset failed, no reset done, err %d\n", err);
609 		set_bit(MLX5_FW_RESET_FLAGS_RELOAD_REQUIRED, &fw_reset->reset_flags);
610 	}
611 
612 	mlx5_enter_error_state(dev, true);
613 done:
614 	fw_reset->ret = err;
615 	mlx5_fw_reset_complete_reload(dev, false);
616 }
617 
mlx5_sync_reset_unload_event(struct work_struct * work)618 static void mlx5_sync_reset_unload_event(struct work_struct *work)
619 {
620 	struct mlx5_fw_reset *fw_reset;
621 	struct mlx5_core_dev *dev;
622 	unsigned long timeout;
623 	int poll_freq = 20;
624 	bool reset_action;
625 	u8 rst_state;
626 	int err;
627 
628 	fw_reset = container_of(work, struct mlx5_fw_reset, reset_unload_work);
629 	dev = fw_reset->dev;
630 
631 	if (mlx5_sync_reset_clear_reset_requested(dev, false))
632 		return;
633 
634 	mlx5_core_warn(dev, "Sync Reset Unload. Function is forced down.\n");
635 
636 	err = mlx5_cmd_fast_teardown_hca(dev);
637 	if (err)
638 		mlx5_core_warn(dev, "Fast teardown failed, unloading, err %d\n", err);
639 	else
640 		mlx5_enter_error_state(dev, true);
641 
642 	if (test_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags))
643 		mlx5_unload_one_devl_locked(dev, false);
644 	else
645 		mlx5_unload_one(dev, false);
646 
647 	mlx5_set_fw_rst_ack(dev);
648 	mlx5_core_warn(dev, "Sync Reset Unload done, device reset expected\n");
649 
650 	reset_action = false;
651 	timeout = jiffies + msecs_to_jiffies(mlx5_tout_ms(dev, RESET_UNLOAD));
652 	do {
653 		rst_state = mlx5_get_fw_rst_state(dev);
654 		if (rst_state == MLX5_FW_RST_STATE_TOGGLE_REQ ||
655 		    rst_state == MLX5_FW_RST_STATE_IDLE) {
656 			reset_action = true;
657 			break;
658 		}
659 		if (rst_state == MLX5_FW_RST_STATE_DROP_MODE) {
660 			mlx5_core_info(dev, "Sync Reset Drop mode ack\n");
661 			mlx5_set_fw_rst_ack(dev);
662 			poll_freq = 1000;
663 		}
664 		msleep(poll_freq);
665 	} while (!time_after(jiffies, timeout));
666 
667 	if (!reset_action) {
668 		mlx5_core_err(dev, "Got timeout waiting for sync reset action, state = %u\n",
669 			      rst_state);
670 		fw_reset->ret = -ETIMEDOUT;
671 		goto done;
672 	}
673 
674 	mlx5_core_warn(dev, "Sync Reset, got reset action. rst_state = %u\n", rst_state);
675 	if (rst_state == MLX5_FW_RST_STATE_TOGGLE_REQ) {
676 		err = mlx5_sync_pci_reset(dev, fw_reset->reset_method);
677 		if (err) {
678 			mlx5_core_warn(dev, "mlx5_sync_pci_reset failed, err %d\n", err);
679 			fw_reset->ret = err;
680 		}
681 	}
682 
683 done:
684 	mlx5_fw_reset_complete_reload(dev, true);
685 }
686 
mlx5_sync_reset_abort_event(struct work_struct * work)687 static void mlx5_sync_reset_abort_event(struct work_struct *work)
688 {
689 	struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset,
690 						      reset_abort_work);
691 	struct mlx5_core_dev *dev = fw_reset->dev;
692 
693 	if (mlx5_sync_reset_clear_reset_requested(dev, true))
694 		return;
695 	mlx5_core_warn(dev, "PCI Sync FW Update Reset Aborted.\n");
696 }
697 
mlx5_sync_reset_events_handle(struct mlx5_fw_reset * fw_reset,struct mlx5_eqe * eqe)698 static void mlx5_sync_reset_events_handle(struct mlx5_fw_reset *fw_reset, struct mlx5_eqe *eqe)
699 {
700 	struct mlx5_eqe_sync_fw_update *sync_fw_update_eqe;
701 	u8 sync_event_rst_type;
702 
703 	sync_fw_update_eqe = &eqe->data.sync_fw_update;
704 	sync_event_rst_type = sync_fw_update_eqe->sync_rst_state & SYNC_RST_STATE_MASK;
705 	switch (sync_event_rst_type) {
706 	case MLX5_SYNC_RST_STATE_RESET_REQUEST:
707 		queue_work(fw_reset->wq, &fw_reset->reset_request_work);
708 		break;
709 	case MLX5_SYNC_RST_STATE_RESET_UNLOAD:
710 		queue_work(fw_reset->wq, &fw_reset->reset_unload_work);
711 		break;
712 	case MLX5_SYNC_RST_STATE_RESET_NOW:
713 		queue_work(fw_reset->wq, &fw_reset->reset_now_work);
714 		break;
715 	case MLX5_SYNC_RST_STATE_RESET_ABORT:
716 		queue_work(fw_reset->wq, &fw_reset->reset_abort_work);
717 		break;
718 	}
719 }
720 
fw_reset_event_notifier(struct notifier_block * nb,unsigned long action,void * data)721 static int fw_reset_event_notifier(struct notifier_block *nb, unsigned long action, void *data)
722 {
723 	struct mlx5_fw_reset *fw_reset = mlx5_nb_cof(nb, struct mlx5_fw_reset, nb);
724 	struct mlx5_eqe *eqe = data;
725 
726 	if (test_bit(MLX5_FW_RESET_FLAGS_DROP_NEW_REQUESTS, &fw_reset->reset_flags))
727 		return NOTIFY_DONE;
728 
729 	switch (eqe->sub_type) {
730 	case MLX5_GENERAL_SUBTYPE_FW_LIVE_PATCH_EVENT:
731 		queue_work(fw_reset->wq, &fw_reset->fw_live_patch_work);
732 		break;
733 	case MLX5_GENERAL_SUBTYPE_PCI_SYNC_FOR_FW_UPDATE_EVENT:
734 		mlx5_sync_reset_events_handle(fw_reset, eqe);
735 		break;
736 	default:
737 		return NOTIFY_DONE;
738 	}
739 
740 	return NOTIFY_OK;
741 }
742 
mlx5_fw_reset_wait_reset_done(struct mlx5_core_dev * dev)743 int mlx5_fw_reset_wait_reset_done(struct mlx5_core_dev *dev)
744 {
745 	unsigned long pci_sync_update_timeout = mlx5_tout_ms(dev, PCI_SYNC_UPDATE);
746 	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
747 	unsigned long timeout;
748 	int err;
749 
750 	if (MLX5_CAP_GEN(dev, pci_sync_for_fw_update_with_driver_unload))
751 		pci_sync_update_timeout += mlx5_tout_ms(dev, RESET_UNLOAD);
752 	timeout = msecs_to_jiffies(pci_sync_update_timeout);
753 	if (!wait_for_completion_timeout(&fw_reset->done, timeout)) {
754 		mlx5_core_warn(dev, "FW sync reset timeout after %lu seconds\n",
755 			       pci_sync_update_timeout / 1000);
756 		err = -ETIMEDOUT;
757 		goto out;
758 	}
759 	err = fw_reset->ret;
760 	if (test_and_clear_bit(MLX5_FW_RESET_FLAGS_RELOAD_REQUIRED, &fw_reset->reset_flags)) {
761 		mlx5_unload_one_devl_locked(dev, false);
762 		mlx5_load_one_devl_locked(dev, true);
763 	}
764 out:
765 	clear_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags);
766 	return err;
767 }
768 
mlx5_fw_reset_events_start(struct mlx5_core_dev * dev)769 void mlx5_fw_reset_events_start(struct mlx5_core_dev *dev)
770 {
771 	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
772 
773 	if (!fw_reset)
774 		return;
775 
776 	MLX5_NB_INIT(&fw_reset->nb, fw_reset_event_notifier, GENERAL_EVENT);
777 	mlx5_eq_notifier_register(dev, &fw_reset->nb);
778 }
779 
mlx5_fw_reset_events_stop(struct mlx5_core_dev * dev)780 void mlx5_fw_reset_events_stop(struct mlx5_core_dev *dev)
781 {
782 	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
783 
784 	if (!fw_reset)
785 		return;
786 
787 	mlx5_eq_notifier_unregister(dev, &fw_reset->nb);
788 }
789 
mlx5_drain_fw_reset(struct mlx5_core_dev * dev)790 void mlx5_drain_fw_reset(struct mlx5_core_dev *dev)
791 {
792 	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
793 
794 	if (!fw_reset)
795 		return;
796 
797 	set_bit(MLX5_FW_RESET_FLAGS_DROP_NEW_REQUESTS, &fw_reset->reset_flags);
798 	cancel_work_sync(&fw_reset->fw_live_patch_work);
799 	cancel_work_sync(&fw_reset->reset_request_work);
800 	cancel_work_sync(&fw_reset->reset_unload_work);
801 	cancel_work_sync(&fw_reset->reset_reload_work);
802 	cancel_work_sync(&fw_reset->reset_now_work);
803 	cancel_work_sync(&fw_reset->reset_abort_work);
804 }
805 
806 static const struct devlink_param mlx5_fw_reset_devlink_params[] = {
807 	DEVLINK_PARAM_GENERIC(ENABLE_REMOTE_DEV_RESET, BIT(DEVLINK_PARAM_CMODE_RUNTIME),
808 			      mlx5_fw_reset_enable_remote_dev_reset_get,
809 			      mlx5_fw_reset_enable_remote_dev_reset_set, NULL),
810 };
811 
mlx5_fw_reset_init(struct mlx5_core_dev * dev)812 int mlx5_fw_reset_init(struct mlx5_core_dev *dev)
813 {
814 	struct mlx5_fw_reset *fw_reset;
815 	int err;
816 
817 	if (!MLX5_CAP_MCAM_REG(dev, mfrl))
818 		return 0;
819 
820 	fw_reset = kzalloc(sizeof(*fw_reset), GFP_KERNEL);
821 	if (!fw_reset)
822 		return -ENOMEM;
823 	fw_reset->wq = create_singlethread_workqueue("mlx5_fw_reset_events");
824 	if (!fw_reset->wq) {
825 		kfree(fw_reset);
826 		return -ENOMEM;
827 	}
828 
829 	fw_reset->dev = dev;
830 	dev->priv.fw_reset = fw_reset;
831 
832 	err = devl_params_register(priv_to_devlink(dev),
833 				   mlx5_fw_reset_devlink_params,
834 				   ARRAY_SIZE(mlx5_fw_reset_devlink_params));
835 	if (err) {
836 		destroy_workqueue(fw_reset->wq);
837 		kfree(fw_reset);
838 		return err;
839 	}
840 
841 	INIT_WORK(&fw_reset->fw_live_patch_work, mlx5_fw_live_patch_event);
842 	INIT_WORK(&fw_reset->reset_request_work, mlx5_sync_reset_request_event);
843 	INIT_WORK(&fw_reset->reset_unload_work, mlx5_sync_reset_unload_event);
844 	INIT_WORK(&fw_reset->reset_reload_work, mlx5_sync_reset_reload_work);
845 	INIT_WORK(&fw_reset->reset_now_work, mlx5_sync_reset_now_event);
846 	INIT_WORK(&fw_reset->reset_abort_work, mlx5_sync_reset_abort_event);
847 
848 	init_completion(&fw_reset->done);
849 	return 0;
850 }
851 
mlx5_fw_reset_cleanup(struct mlx5_core_dev * dev)852 void mlx5_fw_reset_cleanup(struct mlx5_core_dev *dev)
853 {
854 	struct mlx5_fw_reset *fw_reset = dev->priv.fw_reset;
855 
856 	if (!fw_reset)
857 		return;
858 
859 	devl_params_unregister(priv_to_devlink(dev),
860 			       mlx5_fw_reset_devlink_params,
861 			       ARRAY_SIZE(mlx5_fw_reset_devlink_params));
862 	destroy_workqueue(fw_reset->wq);
863 	kfree(dev->priv.fw_reset);
864 }
865