1 // SPDX-License-Identifier: GPL-2.0-only
2 
3 /* Copyright (c) 2019-2021, The Linux Foundation. All rights reserved. */
4 /* Copyright (c) 2021-2023 Qualcomm Innovation Center, Inc. All rights reserved. */
5 
6 #include <linux/delay.h>
7 #include <linux/dma-mapping.h>
8 #include <linux/idr.h>
9 #include <linux/interrupt.h>
10 #include <linux/list.h>
11 #include <linux/kobject.h>
12 #include <linux/kref.h>
13 #include <linux/mhi.h>
14 #include <linux/module.h>
15 #include <linux/msi.h>
16 #include <linux/mutex.h>
17 #include <linux/pci.h>
18 #include <linux/spinlock.h>
19 #include <linux/workqueue.h>
20 #include <linux/wait.h>
21 #include <drm/drm_accel.h>
22 #include <drm/drm_drv.h>
23 #include <drm/drm_file.h>
24 #include <drm/drm_gem.h>
25 #include <drm/drm_ioctl.h>
26 #include <drm/drm_managed.h>
27 #include <uapi/drm/qaic_accel.h>
28 
29 #include "mhi_controller.h"
30 #include "qaic.h"
31 #include "qaic_debugfs.h"
32 #include "qaic_timesync.h"
33 #include "sahara.h"
34 
35 MODULE_IMPORT_NS("DMA_BUF");
36 
37 #define PCI_DEVICE_ID_QCOM_AIC080	0xa080
38 #define PCI_DEVICE_ID_QCOM_AIC100	0xa100
39 #define PCI_DEVICE_ID_QCOM_AIC200	0xa110
40 #define QAIC_NAME			"qaic"
41 #define QAIC_DESC			"Qualcomm Cloud AI Accelerators"
42 #define CNTL_MAJOR			5
43 #define CNTL_MINOR			0
44 
45 struct qaic_device_config {
46 	/* Indicates the AIC family the device belongs to */
47 	int family;
48 	/* A bitmask representing the available BARs */
49 	int bar_mask;
50 	/* An index value used to identify the MHI controller BAR */
51 	unsigned int mhi_bar_idx;
52 	/* An index value used to identify the DBCs BAR */
53 	unsigned int dbc_bar_idx;
54 };
55 
56 static const struct qaic_device_config aic080_config = {
57 	.family = FAMILY_AIC100,
58 	.bar_mask = BIT(0) | BIT(2) | BIT(4),
59 	.mhi_bar_idx = 0,
60 	.dbc_bar_idx = 2,
61 };
62 
63 static const struct qaic_device_config aic100_config = {
64 	.family = FAMILY_AIC100,
65 	.bar_mask = BIT(0) | BIT(2) | BIT(4),
66 	.mhi_bar_idx = 0,
67 	.dbc_bar_idx = 2,
68 };
69 
70 static const struct qaic_device_config aic200_config = {
71 	.family = FAMILY_AIC200,
72 	.bar_mask = BIT(0) | BIT(1) | BIT(2) | BIT(4),
73 	.mhi_bar_idx = 1,
74 	.dbc_bar_idx = 2,
75 };
76 
77 bool datapath_polling;
78 module_param(datapath_polling, bool, 0400);
79 MODULE_PARM_DESC(datapath_polling, "Operate the datapath in polling mode");
80 static bool link_up;
81 static DEFINE_IDA(qaic_usrs);
82 
qaicm_wq_release(struct drm_device * dev,void * res)83 static void qaicm_wq_release(struct drm_device *dev, void *res)
84 {
85 	struct workqueue_struct *wq = res;
86 
87 	destroy_workqueue(wq);
88 }
89 
qaicm_wq_init(struct drm_device * dev,const char * name)90 static struct workqueue_struct *qaicm_wq_init(struct drm_device *dev, const char *name)
91 {
92 	struct workqueue_struct *wq;
93 	int ret;
94 
95 	wq = alloc_workqueue("%s", WQ_UNBOUND, 0, name);
96 	if (!wq)
97 		return ERR_PTR(-ENOMEM);
98 	ret = drmm_add_action_or_reset(dev, qaicm_wq_release, wq);
99 	if (ret)
100 		return ERR_PTR(ret);
101 
102 	return wq;
103 }
104 
qaicm_srcu_release(struct drm_device * dev,void * res)105 static void qaicm_srcu_release(struct drm_device *dev, void *res)
106 {
107 	struct srcu_struct *lock = res;
108 
109 	cleanup_srcu_struct(lock);
110 }
111 
qaicm_srcu_init(struct drm_device * dev,struct srcu_struct * lock)112 static int qaicm_srcu_init(struct drm_device *dev, struct srcu_struct *lock)
113 {
114 	int ret;
115 
116 	ret = init_srcu_struct(lock);
117 	if (ret)
118 		return ret;
119 
120 	return drmm_add_action_or_reset(dev, qaicm_srcu_release, lock);
121 }
122 
qaicm_pci_release(struct drm_device * dev,void * res)123 static void qaicm_pci_release(struct drm_device *dev, void *res)
124 {
125 	struct qaic_device *qdev = to_qaic_device(dev);
126 
127 	pci_set_drvdata(qdev->pdev, NULL);
128 }
129 
free_usr(struct kref * kref)130 static void free_usr(struct kref *kref)
131 {
132 	struct qaic_user *usr = container_of(kref, struct qaic_user, ref_count);
133 
134 	cleanup_srcu_struct(&usr->qddev_lock);
135 	ida_free(&qaic_usrs, usr->handle);
136 	kfree(usr);
137 }
138 
qaic_open(struct drm_device * dev,struct drm_file * file)139 static int qaic_open(struct drm_device *dev, struct drm_file *file)
140 {
141 	struct qaic_drm_device *qddev = to_qaic_drm_device(dev);
142 	struct qaic_device *qdev = qddev->qdev;
143 	struct qaic_user *usr;
144 	int rcu_id;
145 	int ret;
146 
147 	rcu_id = srcu_read_lock(&qdev->dev_lock);
148 	if (qdev->dev_state != QAIC_ONLINE) {
149 		ret = -ENODEV;
150 		goto dev_unlock;
151 	}
152 
153 	usr = kmalloc(sizeof(*usr), GFP_KERNEL);
154 	if (!usr) {
155 		ret = -ENOMEM;
156 		goto dev_unlock;
157 	}
158 
159 	usr->handle = ida_alloc(&qaic_usrs, GFP_KERNEL);
160 	if (usr->handle < 0) {
161 		ret = usr->handle;
162 		goto free_usr;
163 	}
164 	usr->qddev = qddev;
165 	atomic_set(&usr->chunk_id, 0);
166 	init_srcu_struct(&usr->qddev_lock);
167 	kref_init(&usr->ref_count);
168 
169 	ret = mutex_lock_interruptible(&qddev->users_mutex);
170 	if (ret)
171 		goto cleanup_usr;
172 
173 	list_add(&usr->node, &qddev->users);
174 	mutex_unlock(&qddev->users_mutex);
175 
176 	file->driver_priv = usr;
177 
178 	srcu_read_unlock(&qdev->dev_lock, rcu_id);
179 	return 0;
180 
181 cleanup_usr:
182 	cleanup_srcu_struct(&usr->qddev_lock);
183 	ida_free(&qaic_usrs, usr->handle);
184 free_usr:
185 	kfree(usr);
186 dev_unlock:
187 	srcu_read_unlock(&qdev->dev_lock, rcu_id);
188 	return ret;
189 }
190 
qaic_postclose(struct drm_device * dev,struct drm_file * file)191 static void qaic_postclose(struct drm_device *dev, struct drm_file *file)
192 {
193 	struct qaic_user *usr = file->driver_priv;
194 	struct qaic_drm_device *qddev;
195 	struct qaic_device *qdev;
196 	int qdev_rcu_id;
197 	int usr_rcu_id;
198 	int i;
199 
200 	qddev = usr->qddev;
201 	usr_rcu_id = srcu_read_lock(&usr->qddev_lock);
202 	if (qddev) {
203 		qdev = qddev->qdev;
204 		qdev_rcu_id = srcu_read_lock(&qdev->dev_lock);
205 		if (qdev->dev_state == QAIC_ONLINE) {
206 			qaic_release_usr(qdev, usr);
207 			for (i = 0; i < qdev->num_dbc; ++i)
208 				if (qdev->dbc[i].usr && qdev->dbc[i].usr->handle == usr->handle)
209 					release_dbc(qdev, i);
210 		}
211 		srcu_read_unlock(&qdev->dev_lock, qdev_rcu_id);
212 
213 		mutex_lock(&qddev->users_mutex);
214 		if (!list_empty(&usr->node))
215 			list_del_init(&usr->node);
216 		mutex_unlock(&qddev->users_mutex);
217 	}
218 
219 	srcu_read_unlock(&usr->qddev_lock, usr_rcu_id);
220 	kref_put(&usr->ref_count, free_usr);
221 
222 	file->driver_priv = NULL;
223 }
224 
225 DEFINE_DRM_ACCEL_FOPS(qaic_accel_fops);
226 
227 static const struct drm_ioctl_desc qaic_drm_ioctls[] = {
228 	DRM_IOCTL_DEF_DRV(QAIC_MANAGE, qaic_manage_ioctl, 0),
229 	DRM_IOCTL_DEF_DRV(QAIC_CREATE_BO, qaic_create_bo_ioctl, 0),
230 	DRM_IOCTL_DEF_DRV(QAIC_MMAP_BO, qaic_mmap_bo_ioctl, 0),
231 	DRM_IOCTL_DEF_DRV(QAIC_ATTACH_SLICE_BO, qaic_attach_slice_bo_ioctl, 0),
232 	DRM_IOCTL_DEF_DRV(QAIC_EXECUTE_BO, qaic_execute_bo_ioctl, 0),
233 	DRM_IOCTL_DEF_DRV(QAIC_PARTIAL_EXECUTE_BO, qaic_partial_execute_bo_ioctl, 0),
234 	DRM_IOCTL_DEF_DRV(QAIC_WAIT_BO, qaic_wait_bo_ioctl, 0),
235 	DRM_IOCTL_DEF_DRV(QAIC_PERF_STATS_BO, qaic_perf_stats_bo_ioctl, 0),
236 	DRM_IOCTL_DEF_DRV(QAIC_DETACH_SLICE_BO, qaic_detach_slice_bo_ioctl, 0),
237 };
238 
239 static const struct drm_driver qaic_accel_driver = {
240 	.driver_features	= DRIVER_GEM | DRIVER_COMPUTE_ACCEL,
241 
242 	.name			= QAIC_NAME,
243 	.desc			= QAIC_DESC,
244 
245 	.fops			= &qaic_accel_fops,
246 	.open			= qaic_open,
247 	.postclose		= qaic_postclose,
248 
249 	.ioctls			= qaic_drm_ioctls,
250 	.num_ioctls		= ARRAY_SIZE(qaic_drm_ioctls),
251 	.gem_prime_import	= qaic_gem_prime_import,
252 };
253 
qaic_create_drm_device(struct qaic_device * qdev,s32 partition_id)254 static int qaic_create_drm_device(struct qaic_device *qdev, s32 partition_id)
255 {
256 	struct qaic_drm_device *qddev = qdev->qddev;
257 	struct drm_device *drm = to_drm(qddev);
258 	int ret;
259 
260 	/* Hold off implementing partitions until the uapi is determined */
261 	if (partition_id != QAIC_NO_PARTITION)
262 		return -EINVAL;
263 
264 	qddev->partition_id = partition_id;
265 
266 	ret = drm_dev_register(drm, 0);
267 	if (ret) {
268 		pci_dbg(qdev->pdev, "drm_dev_register failed %d\n", ret);
269 		return ret;
270 	}
271 
272 	qaic_debugfs_init(qddev);
273 
274 	return ret;
275 }
276 
qaic_destroy_drm_device(struct qaic_device * qdev,s32 partition_id)277 static void qaic_destroy_drm_device(struct qaic_device *qdev, s32 partition_id)
278 {
279 	struct qaic_drm_device *qddev = qdev->qddev;
280 	struct drm_device *drm = to_drm(qddev);
281 	struct qaic_user *usr;
282 
283 	drm_dev_unregister(drm);
284 	qddev->partition_id = 0;
285 	/*
286 	 * Existing users get unresolvable errors till they close FDs.
287 	 * Need to sync carefully with users calling close(). The
288 	 * list of users can be modified elsewhere when the lock isn't
289 	 * held here, but the sync'ing the srcu with the mutex held
290 	 * could deadlock. Grab the mutex so that the list will be
291 	 * unmodified. The user we get will exist as long as the
292 	 * lock is held. Signal that the qcdev is going away, and
293 	 * grab a reference to the user so they don't go away for
294 	 * synchronize_srcu(). Then release the mutex to avoid
295 	 * deadlock and make sure the user has observed the signal.
296 	 * With the lock released, we cannot maintain any state of the
297 	 * user list.
298 	 */
299 	mutex_lock(&qddev->users_mutex);
300 	while (!list_empty(&qddev->users)) {
301 		usr = list_first_entry(&qddev->users, struct qaic_user, node);
302 		list_del_init(&usr->node);
303 		kref_get(&usr->ref_count);
304 		usr->qddev = NULL;
305 		mutex_unlock(&qddev->users_mutex);
306 		synchronize_srcu(&usr->qddev_lock);
307 		kref_put(&usr->ref_count, free_usr);
308 		mutex_lock(&qddev->users_mutex);
309 	}
310 	mutex_unlock(&qddev->users_mutex);
311 }
312 
qaic_mhi_probe(struct mhi_device * mhi_dev,const struct mhi_device_id * id)313 static int qaic_mhi_probe(struct mhi_device *mhi_dev, const struct mhi_device_id *id)
314 {
315 	u16 major = -1, minor = -1;
316 	struct qaic_device *qdev;
317 	int ret;
318 
319 	/*
320 	 * Invoking this function indicates that the control channel to the
321 	 * device is available. We use that as a signal to indicate that
322 	 * the device side firmware has booted. The device side firmware
323 	 * manages the device resources, so we need to communicate with it
324 	 * via the control channel in order to utilize the device. Therefore
325 	 * we wait until this signal to create the drm dev that userspace will
326 	 * use to control the device, because without the device side firmware,
327 	 * userspace can't do anything useful.
328 	 */
329 
330 	qdev = pci_get_drvdata(to_pci_dev(mhi_dev->mhi_cntrl->cntrl_dev));
331 
332 	dev_set_drvdata(&mhi_dev->dev, qdev);
333 	qdev->cntl_ch = mhi_dev;
334 
335 	ret = qaic_control_open(qdev);
336 	if (ret) {
337 		pci_dbg(qdev->pdev, "%s: control_open failed %d\n", __func__, ret);
338 		return ret;
339 	}
340 
341 	qdev->dev_state = QAIC_BOOT;
342 	ret = get_cntl_version(qdev, NULL, &major, &minor);
343 	if (ret || major != CNTL_MAJOR || minor > CNTL_MINOR) {
344 		pci_err(qdev->pdev, "%s: Control protocol version (%d.%d) not supported. Supported version is (%d.%d). Ret: %d\n",
345 			__func__, major, minor, CNTL_MAJOR, CNTL_MINOR, ret);
346 		ret = -EINVAL;
347 		goto close_control;
348 	}
349 	qdev->dev_state = QAIC_ONLINE;
350 	kobject_uevent(&(to_accel_kdev(qdev->qddev))->kobj, KOBJ_ONLINE);
351 
352 	return ret;
353 
354 close_control:
355 	qaic_control_close(qdev);
356 	return ret;
357 }
358 
qaic_mhi_remove(struct mhi_device * mhi_dev)359 static void qaic_mhi_remove(struct mhi_device *mhi_dev)
360 {
361 /* This is redundant since we have already observed the device crash */
362 }
363 
qaic_notify_reset(struct qaic_device * qdev)364 static void qaic_notify_reset(struct qaic_device *qdev)
365 {
366 	int i;
367 
368 	kobject_uevent(&(to_accel_kdev(qdev->qddev))->kobj, KOBJ_OFFLINE);
369 	qdev->dev_state = QAIC_OFFLINE;
370 	/* wake up any waiters to avoid waiting for timeouts at sync */
371 	wake_all_cntl(qdev);
372 	for (i = 0; i < qdev->num_dbc; ++i)
373 		wakeup_dbc(qdev, i);
374 	synchronize_srcu(&qdev->dev_lock);
375 }
376 
qaic_dev_reset_clean_local_state(struct qaic_device * qdev)377 void qaic_dev_reset_clean_local_state(struct qaic_device *qdev)
378 {
379 	int i;
380 
381 	qaic_notify_reset(qdev);
382 
383 	/* start tearing things down */
384 	for (i = 0; i < qdev->num_dbc; ++i)
385 		release_dbc(qdev, i);
386 }
387 
create_qdev(struct pci_dev * pdev,const struct qaic_device_config * config)388 static struct qaic_device *create_qdev(struct pci_dev *pdev,
389 				       const struct qaic_device_config *config)
390 {
391 	struct device *dev = &pdev->dev;
392 	struct qaic_drm_device *qddev;
393 	struct qaic_device *qdev;
394 	struct drm_device *drm;
395 	int i, ret;
396 
397 	qdev = devm_kzalloc(dev, sizeof(*qdev), GFP_KERNEL);
398 	if (!qdev)
399 		return NULL;
400 
401 	qdev->dev_state = QAIC_OFFLINE;
402 	qdev->num_dbc = 16;
403 	qdev->dbc = devm_kcalloc(dev, qdev->num_dbc, sizeof(*qdev->dbc), GFP_KERNEL);
404 	if (!qdev->dbc)
405 		return NULL;
406 
407 	qddev = devm_drm_dev_alloc(&pdev->dev, &qaic_accel_driver, struct qaic_drm_device, drm);
408 	if (IS_ERR(qddev))
409 		return NULL;
410 
411 	drm = to_drm(qddev);
412 	pci_set_drvdata(pdev, qdev);
413 
414 	ret = drmm_mutex_init(drm, &qddev->users_mutex);
415 	if (ret)
416 		return NULL;
417 	ret = drmm_add_action_or_reset(drm, qaicm_pci_release, NULL);
418 	if (ret)
419 		return NULL;
420 	ret = drmm_mutex_init(drm, &qdev->cntl_mutex);
421 	if (ret)
422 		return NULL;
423 	ret = drmm_mutex_init(drm, &qdev->bootlog_mutex);
424 	if (ret)
425 		return NULL;
426 
427 	qdev->cntl_wq = qaicm_wq_init(drm, "qaic_cntl");
428 	if (IS_ERR(qdev->cntl_wq))
429 		return NULL;
430 	qdev->qts_wq = qaicm_wq_init(drm, "qaic_ts");
431 	if (IS_ERR(qdev->qts_wq))
432 		return NULL;
433 
434 	ret = qaicm_srcu_init(drm, &qdev->dev_lock);
435 	if (ret)
436 		return NULL;
437 
438 	qdev->qddev = qddev;
439 	qdev->pdev = pdev;
440 	qddev->qdev = qdev;
441 
442 	INIT_LIST_HEAD(&qdev->cntl_xfer_list);
443 	INIT_LIST_HEAD(&qdev->bootlog);
444 	INIT_LIST_HEAD(&qddev->users);
445 
446 	for (i = 0; i < qdev->num_dbc; ++i) {
447 		spin_lock_init(&qdev->dbc[i].xfer_lock);
448 		qdev->dbc[i].qdev = qdev;
449 		qdev->dbc[i].id = i;
450 		INIT_LIST_HEAD(&qdev->dbc[i].xfer_list);
451 		ret = qaicm_srcu_init(drm, &qdev->dbc[i].ch_lock);
452 		if (ret)
453 			return NULL;
454 		init_waitqueue_head(&qdev->dbc[i].dbc_release);
455 		INIT_LIST_HEAD(&qdev->dbc[i].bo_lists);
456 	}
457 
458 	return qdev;
459 }
460 
init_pci(struct qaic_device * qdev,struct pci_dev * pdev,const struct qaic_device_config * config)461 static int init_pci(struct qaic_device *qdev, struct pci_dev *pdev,
462 		    const struct qaic_device_config *config)
463 {
464 	int bars;
465 	int ret;
466 
467 	bars = pci_select_bars(pdev, IORESOURCE_MEM) & 0x3f;
468 
469 	/* make sure the device has the expected BARs */
470 	if (bars != config->bar_mask) {
471 		pci_dbg(pdev, "%s: expected BARs %#x not found in device. Found %#x\n",
472 			__func__, config->bar_mask, bars);
473 		return -EINVAL;
474 	}
475 
476 	ret = pcim_enable_device(pdev);
477 	if (ret)
478 		return ret;
479 
480 	ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
481 	if (ret)
482 		return ret;
483 	dma_set_max_seg_size(&pdev->dev, UINT_MAX);
484 
485 	qdev->bar_mhi = devm_ioremap_resource(&pdev->dev, &pdev->resource[config->mhi_bar_idx]);
486 	if (IS_ERR(qdev->bar_mhi))
487 		return PTR_ERR(qdev->bar_mhi);
488 
489 	qdev->bar_dbc = devm_ioremap_resource(&pdev->dev, &pdev->resource[config->dbc_bar_idx]);
490 	if (IS_ERR(qdev->bar_dbc))
491 		return PTR_ERR(qdev->bar_dbc);
492 
493 	/* Managed release since we use pcim_enable_device above */
494 	pci_set_master(pdev);
495 
496 	return 0;
497 }
498 
init_msi(struct qaic_device * qdev,struct pci_dev * pdev)499 static int init_msi(struct qaic_device *qdev, struct pci_dev *pdev)
500 {
501 	int irq_count = qdev->num_dbc + 1;
502 	int mhi_irq;
503 	int ret;
504 	int i;
505 
506 	/* Managed release since we use pcim_enable_device */
507 	ret = pci_alloc_irq_vectors(pdev, irq_count, irq_count, PCI_IRQ_MSI | PCI_IRQ_MSIX);
508 	if (ret == -ENOSPC) {
509 		ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI | PCI_IRQ_MSIX);
510 		if (ret < 0)
511 			return ret;
512 
513 		/*
514 		 * Operate in one MSI mode. All interrupts will be directed to
515 		 * MSI0; every interrupt will wake up all the interrupt handlers
516 		 * (MHI and DBC[0-15]). Since the interrupt is now shared, it is
517 		 * not disabled during DBC threaded handler, but only one thread
518 		 * will be allowed to run per DBC, so while it can be
519 		 * interrupted, it shouldn't race with itself.
520 		 */
521 		qdev->single_msi = true;
522 		pci_info(pdev, "Allocating %d MSIs failed, operating in 1 MSI mode. Performance may be impacted.\n",
523 			 irq_count);
524 	} else if (ret < 0) {
525 		return ret;
526 	}
527 
528 	mhi_irq = pci_irq_vector(pdev, 0);
529 	if (mhi_irq < 0)
530 		return mhi_irq;
531 
532 	for (i = 0; i < qdev->num_dbc; ++i) {
533 		ret = devm_request_threaded_irq(&pdev->dev,
534 						pci_irq_vector(pdev, qdev->single_msi ? 0 : i + 1),
535 						dbc_irq_handler, dbc_irq_threaded_fn, IRQF_SHARED,
536 						"qaic_dbc", &qdev->dbc[i]);
537 		if (ret)
538 			return ret;
539 
540 		if (datapath_polling) {
541 			qdev->dbc[i].irq = pci_irq_vector(pdev, qdev->single_msi ? 0 : i + 1);
542 			if (!qdev->single_msi)
543 				disable_irq_nosync(qdev->dbc[i].irq);
544 			INIT_WORK(&qdev->dbc[i].poll_work, irq_polling_work);
545 		}
546 	}
547 
548 	return mhi_irq;
549 }
550 
qaic_pci_probe(struct pci_dev * pdev,const struct pci_device_id * id)551 static int qaic_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
552 {
553 	struct qaic_device_config *config = (struct qaic_device_config *)id->driver_data;
554 	struct qaic_device *qdev;
555 	int mhi_irq;
556 	int ret;
557 	int i;
558 
559 	qdev = create_qdev(pdev, config);
560 	if (!qdev)
561 		return -ENOMEM;
562 
563 	ret = init_pci(qdev, pdev, config);
564 	if (ret)
565 		return ret;
566 
567 	for (i = 0; i < qdev->num_dbc; ++i)
568 		qdev->dbc[i].dbc_base = qdev->bar_dbc + QAIC_DBC_OFF(i);
569 
570 	mhi_irq = init_msi(qdev, pdev);
571 	if (mhi_irq < 0)
572 		return mhi_irq;
573 
574 	ret = qaic_create_drm_device(qdev, QAIC_NO_PARTITION);
575 	if (ret)
576 		return ret;
577 
578 	qdev->mhi_cntrl = qaic_mhi_register_controller(pdev, qdev->bar_mhi, mhi_irq,
579 						       qdev->single_msi, config->family);
580 	if (IS_ERR(qdev->mhi_cntrl)) {
581 		ret = PTR_ERR(qdev->mhi_cntrl);
582 		qaic_destroy_drm_device(qdev, QAIC_NO_PARTITION);
583 		return ret;
584 	}
585 
586 	return 0;
587 }
588 
qaic_pci_remove(struct pci_dev * pdev)589 static void qaic_pci_remove(struct pci_dev *pdev)
590 {
591 	struct qaic_device *qdev = pci_get_drvdata(pdev);
592 
593 	if (!qdev)
594 		return;
595 
596 	qaic_dev_reset_clean_local_state(qdev);
597 	qaic_mhi_free_controller(qdev->mhi_cntrl, link_up);
598 	qaic_destroy_drm_device(qdev, QAIC_NO_PARTITION);
599 }
600 
qaic_pci_shutdown(struct pci_dev * pdev)601 static void qaic_pci_shutdown(struct pci_dev *pdev)
602 {
603 	/* see qaic_exit for what link_up is doing */
604 	link_up = true;
605 	qaic_pci_remove(pdev);
606 }
607 
qaic_pci_error_detected(struct pci_dev * pdev,pci_channel_state_t error)608 static pci_ers_result_t qaic_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t error)
609 {
610 	return PCI_ERS_RESULT_NEED_RESET;
611 }
612 
qaic_pci_reset_prepare(struct pci_dev * pdev)613 static void qaic_pci_reset_prepare(struct pci_dev *pdev)
614 {
615 	struct qaic_device *qdev = pci_get_drvdata(pdev);
616 
617 	qaic_notify_reset(qdev);
618 	qaic_mhi_start_reset(qdev->mhi_cntrl);
619 	qaic_dev_reset_clean_local_state(qdev);
620 }
621 
qaic_pci_reset_done(struct pci_dev * pdev)622 static void qaic_pci_reset_done(struct pci_dev *pdev)
623 {
624 	struct qaic_device *qdev = pci_get_drvdata(pdev);
625 
626 	qaic_mhi_reset_done(qdev->mhi_cntrl);
627 }
628 
629 static const struct mhi_device_id qaic_mhi_match_table[] = {
630 	{ .chan = "QAIC_CONTROL", },
631 	{},
632 };
633 
634 static struct mhi_driver qaic_mhi_driver = {
635 	.id_table = qaic_mhi_match_table,
636 	.remove = qaic_mhi_remove,
637 	.probe = qaic_mhi_probe,
638 	.ul_xfer_cb = qaic_mhi_ul_xfer_cb,
639 	.dl_xfer_cb = qaic_mhi_dl_xfer_cb,
640 	.driver = {
641 		.name = "qaic_mhi",
642 	},
643 };
644 
645 static const struct pci_device_id qaic_ids[] = {
646 	{ PCI_DEVICE_DATA(QCOM, AIC080, (kernel_ulong_t)&aic080_config), },
647 	{ PCI_DEVICE_DATA(QCOM, AIC100, (kernel_ulong_t)&aic100_config), },
648 	{ PCI_DEVICE_DATA(QCOM, AIC200, (kernel_ulong_t)&aic200_config), },
649 	{ }
650 };
651 MODULE_DEVICE_TABLE(pci, qaic_ids);
652 
653 static const struct pci_error_handlers qaic_pci_err_handler = {
654 	.error_detected = qaic_pci_error_detected,
655 	.reset_prepare = qaic_pci_reset_prepare,
656 	.reset_done = qaic_pci_reset_done,
657 };
658 
659 static struct pci_driver qaic_pci_driver = {
660 	.name = QAIC_NAME,
661 	.id_table = qaic_ids,
662 	.probe = qaic_pci_probe,
663 	.remove = qaic_pci_remove,
664 	.shutdown = qaic_pci_shutdown,
665 	.err_handler = &qaic_pci_err_handler,
666 };
667 
qaic_init(void)668 static int __init qaic_init(void)
669 {
670 	int ret;
671 
672 	ret = pci_register_driver(&qaic_pci_driver);
673 	if (ret) {
674 		pr_debug("qaic: pci_register_driver failed %d\n", ret);
675 		return ret;
676 	}
677 
678 	ret = mhi_driver_register(&qaic_mhi_driver);
679 	if (ret) {
680 		pr_debug("qaic: mhi_driver_register failed %d\n", ret);
681 		goto free_pci;
682 	}
683 
684 	ret = sahara_register();
685 	if (ret) {
686 		pr_debug("qaic: sahara_register failed %d\n", ret);
687 		goto free_mhi;
688 	}
689 
690 	ret = qaic_timesync_init();
691 	if (ret)
692 		pr_debug("qaic: qaic_timesync_init failed %d\n", ret);
693 
694 	ret = qaic_bootlog_register();
695 	if (ret)
696 		pr_debug("qaic: qaic_bootlog_register failed %d\n", ret);
697 
698 	return 0;
699 
700 free_mhi:
701 	mhi_driver_unregister(&qaic_mhi_driver);
702 free_pci:
703 	pci_unregister_driver(&qaic_pci_driver);
704 	return ret;
705 }
706 
qaic_exit(void)707 static void __exit qaic_exit(void)
708 {
709 	/*
710 	 * We assume that qaic_pci_remove() is called due to a hotplug event
711 	 * which would mean that the link is down, and thus
712 	 * qaic_mhi_free_controller() should not try to access the device during
713 	 * cleanup.
714 	 * We call pci_unregister_driver() below, which also triggers
715 	 * qaic_pci_remove(), but since this is module exit, we expect the link
716 	 * to the device to be up, in which case qaic_mhi_free_controller()
717 	 * should try to access the device during cleanup to put the device in
718 	 * a sane state.
719 	 * For that reason, we set link_up here to let qaic_mhi_free_controller
720 	 * know the expected link state. Since the module is going to be
721 	 * removed at the end of this, we don't need to worry about
722 	 * reinitializing the link_up state after the cleanup is done.
723 	 */
724 	link_up = true;
725 	qaic_bootlog_unregister();
726 	qaic_timesync_deinit();
727 	sahara_unregister();
728 	mhi_driver_unregister(&qaic_mhi_driver);
729 	pci_unregister_driver(&qaic_pci_driver);
730 }
731 
732 module_init(qaic_init);
733 module_exit(qaic_exit);
734 
735 MODULE_AUTHOR(QAIC_DESC " Kernel Driver Team");
736 MODULE_DESCRIPTION(QAIC_DESC " Accel Driver");
737 MODULE_LICENSE("GPL");
738