1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2021 Intel Corporation
4  */
5 
6 #include "xe_device.h"
7 
8 #include <linux/aperture.h>
9 #include <linux/delay.h>
10 #include <linux/fault-inject.h>
11 #include <linux/units.h>
12 
13 #include <drm/drm_atomic_helper.h>
14 #include <drm/drm_client.h>
15 #include <drm/drm_gem_ttm_helper.h>
16 #include <drm/drm_ioctl.h>
17 #include <drm/drm_managed.h>
18 #include <drm/drm_print.h>
19 #include <uapi/drm/xe_drm.h>
20 
21 #include "display/xe_display.h"
22 #include "instructions/xe_gpu_commands.h"
23 #include "regs/xe_gt_regs.h"
24 #include "regs/xe_regs.h"
25 #include "xe_bo.h"
26 #include "xe_debugfs.h"
27 #include "xe_devcoredump.h"
28 #include "xe_dma_buf.h"
29 #include "xe_drm_client.h"
30 #include "xe_drv.h"
31 #include "xe_exec.h"
32 #include "xe_exec_queue.h"
33 #include "xe_force_wake.h"
34 #include "xe_ggtt.h"
35 #include "xe_gsc_proxy.h"
36 #include "xe_gt.h"
37 #include "xe_gt_mcr.h"
38 #include "xe_gt_printk.h"
39 #include "xe_gt_sriov_vf.h"
40 #include "xe_guc.h"
41 #include "xe_hw_engine_group.h"
42 #include "xe_hwmon.h"
43 #include "xe_irq.h"
44 #include "xe_memirq.h"
45 #include "xe_mmio.h"
46 #include "xe_module.h"
47 #include "xe_oa.h"
48 #include "xe_observation.h"
49 #include "xe_pat.h"
50 #include "xe_pcode.h"
51 #include "xe_pm.h"
52 #include "xe_pmu.h"
53 #include "xe_pxp.h"
54 #include "xe_query.h"
55 #include "xe_shrinker.h"
56 #include "xe_survivability_mode.h"
57 #include "xe_sriov.h"
58 #include "xe_tile.h"
59 #include "xe_ttm_stolen_mgr.h"
60 #include "xe_ttm_sys_mgr.h"
61 #include "xe_vm.h"
62 #include "xe_vram.h"
63 #include "xe_vsec.h"
64 #include "xe_wait_user_fence.h"
65 #include "xe_wa.h"
66 
67 #include <generated/xe_wa_oob.h>
68 
xe_file_open(struct drm_device * dev,struct drm_file * file)69 static int xe_file_open(struct drm_device *dev, struct drm_file *file)
70 {
71 	struct xe_device *xe = to_xe_device(dev);
72 	struct xe_drm_client *client;
73 	struct xe_file *xef;
74 	int ret = -ENOMEM;
75 	struct task_struct *task = NULL;
76 
77 	xef = kzalloc(sizeof(*xef), GFP_KERNEL);
78 	if (!xef)
79 		return ret;
80 
81 	client = xe_drm_client_alloc();
82 	if (!client) {
83 		kfree(xef);
84 		return ret;
85 	}
86 
87 	xef->drm = file;
88 	xef->client = client;
89 	xef->xe = xe;
90 
91 	mutex_init(&xef->vm.lock);
92 	xa_init_flags(&xef->vm.xa, XA_FLAGS_ALLOC1);
93 
94 	mutex_init(&xef->exec_queue.lock);
95 	xa_init_flags(&xef->exec_queue.xa, XA_FLAGS_ALLOC1);
96 
97 	file->driver_priv = xef;
98 	kref_init(&xef->refcount);
99 
100 	task = get_pid_task(rcu_access_pointer(file->pid), PIDTYPE_PID);
101 	if (task) {
102 		xef->process_name = kstrdup(task->comm, GFP_KERNEL);
103 		xef->pid = task->pid;
104 		put_task_struct(task);
105 	}
106 
107 	return 0;
108 }
109 
xe_file_destroy(struct kref * ref)110 static void xe_file_destroy(struct kref *ref)
111 {
112 	struct xe_file *xef = container_of(ref, struct xe_file, refcount);
113 
114 	xa_destroy(&xef->exec_queue.xa);
115 	mutex_destroy(&xef->exec_queue.lock);
116 	xa_destroy(&xef->vm.xa);
117 	mutex_destroy(&xef->vm.lock);
118 
119 	xe_drm_client_put(xef->client);
120 	kfree(xef->process_name);
121 	kfree(xef);
122 }
123 
124 /**
125  * xe_file_get() - Take a reference to the xe file object
126  * @xef: Pointer to the xe file
127  *
128  * Anyone with a pointer to xef must take a reference to the xe file
129  * object using this call.
130  *
131  * Return: xe file pointer
132  */
xe_file_get(struct xe_file * xef)133 struct xe_file *xe_file_get(struct xe_file *xef)
134 {
135 	kref_get(&xef->refcount);
136 	return xef;
137 }
138 
139 /**
140  * xe_file_put() - Drop a reference to the xe file object
141  * @xef: Pointer to the xe file
142  *
143  * Used to drop reference to the xef object
144  */
xe_file_put(struct xe_file * xef)145 void xe_file_put(struct xe_file *xef)
146 {
147 	kref_put(&xef->refcount, xe_file_destroy);
148 }
149 
xe_file_close(struct drm_device * dev,struct drm_file * file)150 static void xe_file_close(struct drm_device *dev, struct drm_file *file)
151 {
152 	struct xe_device *xe = to_xe_device(dev);
153 	struct xe_file *xef = file->driver_priv;
154 	struct xe_vm *vm;
155 	struct xe_exec_queue *q;
156 	unsigned long idx;
157 
158 	xe_pm_runtime_get(xe);
159 
160 	/*
161 	 * No need for exec_queue.lock here as there is no contention for it
162 	 * when FD is closing as IOCTLs presumably can't be modifying the
163 	 * xarray. Taking exec_queue.lock here causes undue dependency on
164 	 * vm->lock taken during xe_exec_queue_kill().
165 	 */
166 	xa_for_each(&xef->exec_queue.xa, idx, q) {
167 		if (q->vm && q->hwe->hw_engine_group)
168 			xe_hw_engine_group_del_exec_queue(q->hwe->hw_engine_group, q);
169 		xe_exec_queue_kill(q);
170 		xe_exec_queue_put(q);
171 	}
172 	xa_for_each(&xef->vm.xa, idx, vm)
173 		xe_vm_close_and_put(vm);
174 
175 	xe_file_put(xef);
176 
177 	xe_pm_runtime_put(xe);
178 }
179 
180 static const struct drm_ioctl_desc xe_ioctls[] = {
181 	DRM_IOCTL_DEF_DRV(XE_DEVICE_QUERY, xe_query_ioctl, DRM_RENDER_ALLOW),
182 	DRM_IOCTL_DEF_DRV(XE_GEM_CREATE, xe_gem_create_ioctl, DRM_RENDER_ALLOW),
183 	DRM_IOCTL_DEF_DRV(XE_GEM_MMAP_OFFSET, xe_gem_mmap_offset_ioctl,
184 			  DRM_RENDER_ALLOW),
185 	DRM_IOCTL_DEF_DRV(XE_VM_CREATE, xe_vm_create_ioctl, DRM_RENDER_ALLOW),
186 	DRM_IOCTL_DEF_DRV(XE_VM_DESTROY, xe_vm_destroy_ioctl, DRM_RENDER_ALLOW),
187 	DRM_IOCTL_DEF_DRV(XE_VM_BIND, xe_vm_bind_ioctl, DRM_RENDER_ALLOW),
188 	DRM_IOCTL_DEF_DRV(XE_EXEC, xe_exec_ioctl, DRM_RENDER_ALLOW),
189 	DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_CREATE, xe_exec_queue_create_ioctl,
190 			  DRM_RENDER_ALLOW),
191 	DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_DESTROY, xe_exec_queue_destroy_ioctl,
192 			  DRM_RENDER_ALLOW),
193 	DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_GET_PROPERTY, xe_exec_queue_get_property_ioctl,
194 			  DRM_RENDER_ALLOW),
195 	DRM_IOCTL_DEF_DRV(XE_WAIT_USER_FENCE, xe_wait_user_fence_ioctl,
196 			  DRM_RENDER_ALLOW),
197 	DRM_IOCTL_DEF_DRV(XE_OBSERVATION, xe_observation_ioctl, DRM_RENDER_ALLOW),
198 };
199 
xe_drm_ioctl(struct file * file,unsigned int cmd,unsigned long arg)200 static long xe_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
201 {
202 	struct drm_file *file_priv = file->private_data;
203 	struct xe_device *xe = to_xe_device(file_priv->minor->dev);
204 	long ret;
205 
206 	if (xe_device_wedged(xe))
207 		return -ECANCELED;
208 
209 	ret = xe_pm_runtime_get_ioctl(xe);
210 	if (ret >= 0)
211 		ret = drm_ioctl(file, cmd, arg);
212 	xe_pm_runtime_put(xe);
213 
214 	return ret;
215 }
216 
217 #ifdef CONFIG_COMPAT
xe_drm_compat_ioctl(struct file * file,unsigned int cmd,unsigned long arg)218 static long xe_drm_compat_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
219 {
220 	struct drm_file *file_priv = file->private_data;
221 	struct xe_device *xe = to_xe_device(file_priv->minor->dev);
222 	long ret;
223 
224 	if (xe_device_wedged(xe))
225 		return -ECANCELED;
226 
227 	ret = xe_pm_runtime_get_ioctl(xe);
228 	if (ret >= 0)
229 		ret = drm_compat_ioctl(file, cmd, arg);
230 	xe_pm_runtime_put(xe);
231 
232 	return ret;
233 }
234 #else
235 /* similarly to drm_compat_ioctl, let's it be assigned to .compat_ioct unconditionally */
236 #define xe_drm_compat_ioctl NULL
237 #endif
238 
barrier_open(struct vm_area_struct * vma)239 static void barrier_open(struct vm_area_struct *vma)
240 {
241 	drm_dev_get(vma->vm_private_data);
242 }
243 
barrier_close(struct vm_area_struct * vma)244 static void barrier_close(struct vm_area_struct *vma)
245 {
246 	drm_dev_put(vma->vm_private_data);
247 }
248 
barrier_release_dummy_page(struct drm_device * dev,void * res)249 static void barrier_release_dummy_page(struct drm_device *dev, void *res)
250 {
251 	struct page *dummy_page = (struct page *)res;
252 
253 	__free_page(dummy_page);
254 }
255 
barrier_fault(struct vm_fault * vmf)256 static vm_fault_t barrier_fault(struct vm_fault *vmf)
257 {
258 	struct drm_device *dev = vmf->vma->vm_private_data;
259 	struct vm_area_struct *vma = vmf->vma;
260 	vm_fault_t ret = VM_FAULT_NOPAGE;
261 	pgprot_t prot;
262 	int idx;
263 
264 	prot = vm_get_page_prot(vma->vm_flags);
265 
266 	if (drm_dev_enter(dev, &idx)) {
267 		unsigned long pfn;
268 
269 #define LAST_DB_PAGE_OFFSET 0x7ff001
270 		pfn = PHYS_PFN(pci_resource_start(to_pci_dev(dev->dev), 0) +
271 				LAST_DB_PAGE_OFFSET);
272 		ret = vmf_insert_pfn_prot(vma, vma->vm_start, pfn,
273 					  pgprot_noncached(prot));
274 		drm_dev_exit(idx);
275 	} else {
276 		struct page *page;
277 
278 		/* Allocate new dummy page to map all the VA range in this VMA to it*/
279 		page = alloc_page(GFP_KERNEL | __GFP_ZERO);
280 		if (!page)
281 			return VM_FAULT_OOM;
282 
283 		/* Set the page to be freed using drmm release action */
284 		if (drmm_add_action_or_reset(dev, barrier_release_dummy_page, page))
285 			return VM_FAULT_OOM;
286 
287 		ret = vmf_insert_pfn_prot(vma, vma->vm_start, page_to_pfn(page),
288 					  prot);
289 	}
290 
291 	return ret;
292 }
293 
294 static const struct vm_operations_struct vm_ops_barrier = {
295 	.open = barrier_open,
296 	.close = barrier_close,
297 	.fault = barrier_fault,
298 };
299 
xe_pci_barrier_mmap(struct file * filp,struct vm_area_struct * vma)300 static int xe_pci_barrier_mmap(struct file *filp,
301 			       struct vm_area_struct *vma)
302 {
303 	struct drm_file *priv = filp->private_data;
304 	struct drm_device *dev = priv->minor->dev;
305 	struct xe_device *xe = to_xe_device(dev);
306 
307 	if (!IS_DGFX(xe))
308 		return -EINVAL;
309 
310 	if (vma->vm_end - vma->vm_start > SZ_4K)
311 		return -EINVAL;
312 
313 	if (is_cow_mapping(vma->vm_flags))
314 		return -EINVAL;
315 
316 	if (vma->vm_flags & (VM_READ | VM_EXEC))
317 		return -EINVAL;
318 
319 	vm_flags_clear(vma, VM_MAYREAD | VM_MAYEXEC);
320 	vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_IO);
321 	vma->vm_ops = &vm_ops_barrier;
322 	vma->vm_private_data = dev;
323 	drm_dev_get(vma->vm_private_data);
324 
325 	return 0;
326 }
327 
xe_mmap(struct file * filp,struct vm_area_struct * vma)328 static int xe_mmap(struct file *filp, struct vm_area_struct *vma)
329 {
330 	struct drm_file *priv = filp->private_data;
331 	struct drm_device *dev = priv->minor->dev;
332 
333 	if (drm_dev_is_unplugged(dev))
334 		return -ENODEV;
335 
336 	switch (vma->vm_pgoff) {
337 	case XE_PCI_BARRIER_MMAP_OFFSET >> XE_PTE_SHIFT:
338 		return xe_pci_barrier_mmap(filp, vma);
339 	}
340 
341 	return drm_gem_mmap(filp, vma);
342 }
343 
344 static const struct file_operations xe_driver_fops = {
345 	.owner = THIS_MODULE,
346 	.open = drm_open,
347 	.release = drm_release_noglobal,
348 	.unlocked_ioctl = xe_drm_ioctl,
349 	.mmap = xe_mmap,
350 	.poll = drm_poll,
351 	.read = drm_read,
352 	.compat_ioctl = xe_drm_compat_ioctl,
353 	.llseek = noop_llseek,
354 #ifdef CONFIG_PROC_FS
355 	.show_fdinfo = drm_show_fdinfo,
356 #endif
357 	.fop_flags = FOP_UNSIGNED_OFFSET,
358 };
359 
360 static struct drm_driver driver = {
361 	/* Don't use MTRRs here; the Xserver or userspace app should
362 	 * deal with them for Intel hardware.
363 	 */
364 	.driver_features =
365 	    DRIVER_GEM |
366 	    DRIVER_RENDER | DRIVER_SYNCOBJ |
367 	    DRIVER_SYNCOBJ_TIMELINE | DRIVER_GEM_GPUVA,
368 	.open = xe_file_open,
369 	.postclose = xe_file_close,
370 
371 	.gem_prime_import = xe_gem_prime_import,
372 
373 	.dumb_create = xe_bo_dumb_create,
374 	.dumb_map_offset = drm_gem_ttm_dumb_map_offset,
375 #ifdef CONFIG_PROC_FS
376 	.show_fdinfo = xe_drm_client_fdinfo,
377 #endif
378 	.ioctls = xe_ioctls,
379 	.num_ioctls = ARRAY_SIZE(xe_ioctls),
380 	.fops = &xe_driver_fops,
381 	.name = DRIVER_NAME,
382 	.desc = DRIVER_DESC,
383 	.major = DRIVER_MAJOR,
384 	.minor = DRIVER_MINOR,
385 	.patchlevel = DRIVER_PATCHLEVEL,
386 };
387 
xe_device_destroy(struct drm_device * dev,void * dummy)388 static void xe_device_destroy(struct drm_device *dev, void *dummy)
389 {
390 	struct xe_device *xe = to_xe_device(dev);
391 
392 	xe_bo_dev_fini(&xe->bo_device);
393 
394 	if (xe->preempt_fence_wq)
395 		destroy_workqueue(xe->preempt_fence_wq);
396 
397 	if (xe->ordered_wq)
398 		destroy_workqueue(xe->ordered_wq);
399 
400 	if (xe->unordered_wq)
401 		destroy_workqueue(xe->unordered_wq);
402 
403 	if (!IS_ERR_OR_NULL(xe->mem.shrinker))
404 		xe_shrinker_destroy(xe->mem.shrinker);
405 
406 	if (xe->destroy_wq)
407 		destroy_workqueue(xe->destroy_wq);
408 
409 	ttm_device_fini(&xe->ttm);
410 }
411 
xe_device_create(struct pci_dev * pdev,const struct pci_device_id * ent)412 struct xe_device *xe_device_create(struct pci_dev *pdev,
413 				   const struct pci_device_id *ent)
414 {
415 	struct xe_device *xe;
416 	int err;
417 
418 	xe_display_driver_set_hooks(&driver);
419 
420 	err = aperture_remove_conflicting_pci_devices(pdev, driver.name);
421 	if (err)
422 		return ERR_PTR(err);
423 
424 	xe = devm_drm_dev_alloc(&pdev->dev, &driver, struct xe_device, drm);
425 	if (IS_ERR(xe))
426 		return xe;
427 
428 	err = ttm_device_init(&xe->ttm, &xe_ttm_funcs, xe->drm.dev,
429 			      xe->drm.anon_inode->i_mapping,
430 			      xe->drm.vma_offset_manager, false, false);
431 	if (WARN_ON(err))
432 		goto err;
433 
434 	xe_bo_dev_init(&xe->bo_device);
435 	err = drmm_add_action_or_reset(&xe->drm, xe_device_destroy, NULL);
436 	if (err)
437 		goto err;
438 
439 	xe->mem.shrinker = xe_shrinker_create(xe);
440 	if (IS_ERR(xe->mem.shrinker))
441 		return ERR_CAST(xe->mem.shrinker);
442 
443 	xe->info.devid = pdev->device;
444 	xe->info.revid = pdev->revision;
445 	xe->info.force_execlist = xe_modparam.force_execlist;
446 
447 	err = xe_irq_init(xe);
448 	if (err)
449 		goto err;
450 
451 	init_waitqueue_head(&xe->ufence_wq);
452 
453 	init_rwsem(&xe->usm.lock);
454 
455 	xa_init_flags(&xe->usm.asid_to_vm, XA_FLAGS_ALLOC);
456 
457 	if (IS_ENABLED(CONFIG_DRM_XE_DEBUG)) {
458 		/* Trigger a large asid and an early asid wrap. */
459 		u32 asid;
460 
461 		BUILD_BUG_ON(XE_MAX_ASID < 2);
462 		err = xa_alloc_cyclic(&xe->usm.asid_to_vm, &asid, NULL,
463 				      XA_LIMIT(XE_MAX_ASID - 2, XE_MAX_ASID - 1),
464 				      &xe->usm.next_asid, GFP_KERNEL);
465 		drm_WARN_ON(&xe->drm, err);
466 		if (err >= 0)
467 			xa_erase(&xe->usm.asid_to_vm, asid);
468 	}
469 
470 	spin_lock_init(&xe->pinned.lock);
471 	INIT_LIST_HEAD(&xe->pinned.kernel_bo_present);
472 	INIT_LIST_HEAD(&xe->pinned.external_vram);
473 	INIT_LIST_HEAD(&xe->pinned.evicted);
474 
475 	xe->preempt_fence_wq = alloc_ordered_workqueue("xe-preempt-fence-wq",
476 						       WQ_MEM_RECLAIM);
477 	xe->ordered_wq = alloc_ordered_workqueue("xe-ordered-wq", 0);
478 	xe->unordered_wq = alloc_workqueue("xe-unordered-wq", 0, 0);
479 	xe->destroy_wq = alloc_workqueue("xe-destroy-wq", 0, 0);
480 	if (!xe->ordered_wq || !xe->unordered_wq ||
481 	    !xe->preempt_fence_wq || !xe->destroy_wq) {
482 		/*
483 		 * Cleanup done in xe_device_destroy via
484 		 * drmm_add_action_or_reset register above
485 		 */
486 		drm_err(&xe->drm, "Failed to allocate xe workqueues\n");
487 		err = -ENOMEM;
488 		goto err;
489 	}
490 
491 	err = drmm_mutex_init(&xe->drm, &xe->pmt.lock);
492 	if (err)
493 		goto err;
494 
495 	err = xe_display_create(xe);
496 	if (WARN_ON(err))
497 		goto err;
498 
499 	return xe;
500 
501 err:
502 	return ERR_PTR(err);
503 }
504 ALLOW_ERROR_INJECTION(xe_device_create, ERRNO); /* See xe_pci_probe() */
505 
xe_driver_flr_disabled(struct xe_device * xe)506 static bool xe_driver_flr_disabled(struct xe_device *xe)
507 {
508 	return xe_mmio_read32(xe_root_tile_mmio(xe), GU_CNTL_PROTECTED) & DRIVERINT_FLR_DIS;
509 }
510 
511 /*
512  * The driver-initiated FLR is the highest level of reset that we can trigger
513  * from within the driver. It is different from the PCI FLR in that it doesn't
514  * fully reset the SGUnit and doesn't modify the PCI config space and therefore
515  * it doesn't require a re-enumeration of the PCI BARs. However, the
516  * driver-initiated FLR does still cause a reset of both GT and display and a
517  * memory wipe of local and stolen memory, so recovery would require a full HW
518  * re-init and saving/restoring (or re-populating) the wiped memory. Since we
519  * perform the FLR as the very last action before releasing access to the HW
520  * during the driver release flow, we don't attempt recovery at all, because
521  * if/when a new instance of i915 is bound to the device it will do a full
522  * re-init anyway.
523  */
__xe_driver_flr(struct xe_device * xe)524 static void __xe_driver_flr(struct xe_device *xe)
525 {
526 	const unsigned int flr_timeout = 3 * MICRO; /* specs recommend a 3s wait */
527 	struct xe_mmio *mmio = xe_root_tile_mmio(xe);
528 	int ret;
529 
530 	drm_dbg(&xe->drm, "Triggering Driver-FLR\n");
531 
532 	/*
533 	 * Make sure any pending FLR requests have cleared by waiting for the
534 	 * FLR trigger bit to go to zero. Also clear GU_DEBUG's DRIVERFLR_STATUS
535 	 * to make sure it's not still set from a prior attempt (it's a write to
536 	 * clear bit).
537 	 * Note that we should never be in a situation where a previous attempt
538 	 * is still pending (unless the HW is totally dead), but better to be
539 	 * safe in case something unexpected happens
540 	 */
541 	ret = xe_mmio_wait32(mmio, GU_CNTL, DRIVERFLR, 0, flr_timeout, NULL, false);
542 	if (ret) {
543 		drm_err(&xe->drm, "Driver-FLR-prepare wait for ready failed! %d\n", ret);
544 		return;
545 	}
546 	xe_mmio_write32(mmio, GU_DEBUG, DRIVERFLR_STATUS);
547 
548 	/* Trigger the actual Driver-FLR */
549 	xe_mmio_rmw32(mmio, GU_CNTL, 0, DRIVERFLR);
550 
551 	/* Wait for hardware teardown to complete */
552 	ret = xe_mmio_wait32(mmio, GU_CNTL, DRIVERFLR, 0, flr_timeout, NULL, false);
553 	if (ret) {
554 		drm_err(&xe->drm, "Driver-FLR-teardown wait completion failed! %d\n", ret);
555 		return;
556 	}
557 
558 	/* Wait for hardware/firmware re-init to complete */
559 	ret = xe_mmio_wait32(mmio, GU_DEBUG, DRIVERFLR_STATUS, DRIVERFLR_STATUS,
560 			     flr_timeout, NULL, false);
561 	if (ret) {
562 		drm_err(&xe->drm, "Driver-FLR-reinit wait completion failed! %d\n", ret);
563 		return;
564 	}
565 
566 	/* Clear sticky completion status */
567 	xe_mmio_write32(mmio, GU_DEBUG, DRIVERFLR_STATUS);
568 }
569 
xe_driver_flr(struct xe_device * xe)570 static void xe_driver_flr(struct xe_device *xe)
571 {
572 	if (xe_driver_flr_disabled(xe)) {
573 		drm_info_once(&xe->drm, "BIOS Disabled Driver-FLR\n");
574 		return;
575 	}
576 
577 	__xe_driver_flr(xe);
578 }
579 
xe_driver_flr_fini(void * arg)580 static void xe_driver_flr_fini(void *arg)
581 {
582 	struct xe_device *xe = arg;
583 
584 	if (xe->needs_flr_on_fini)
585 		xe_driver_flr(xe);
586 }
587 
xe_device_sanitize(void * arg)588 static void xe_device_sanitize(void *arg)
589 {
590 	struct xe_device *xe = arg;
591 	struct xe_gt *gt;
592 	u8 id;
593 
594 	for_each_gt(gt, xe, id)
595 		xe_gt_sanitize(gt);
596 }
597 
xe_set_dma_info(struct xe_device * xe)598 static int xe_set_dma_info(struct xe_device *xe)
599 {
600 	unsigned int mask_size = xe->info.dma_mask_size;
601 	int err;
602 
603 	dma_set_max_seg_size(xe->drm.dev, xe_sg_segment_size(xe->drm.dev));
604 
605 	err = dma_set_mask(xe->drm.dev, DMA_BIT_MASK(mask_size));
606 	if (err)
607 		goto mask_err;
608 
609 	err = dma_set_coherent_mask(xe->drm.dev, DMA_BIT_MASK(mask_size));
610 	if (err)
611 		goto mask_err;
612 
613 	return 0;
614 
615 mask_err:
616 	drm_err(&xe->drm, "Can't set DMA mask/consistent mask (%d)\n", err);
617 	return err;
618 }
619 
verify_lmem_ready(struct xe_device * xe)620 static bool verify_lmem_ready(struct xe_device *xe)
621 {
622 	u32 val = xe_mmio_read32(xe_root_tile_mmio(xe), GU_CNTL) & LMEM_INIT;
623 
624 	return !!val;
625 }
626 
wait_for_lmem_ready(struct xe_device * xe)627 static int wait_for_lmem_ready(struct xe_device *xe)
628 {
629 	unsigned long timeout, start;
630 
631 	if (!IS_DGFX(xe))
632 		return 0;
633 
634 	if (IS_SRIOV_VF(xe))
635 		return 0;
636 
637 	if (verify_lmem_ready(xe))
638 		return 0;
639 
640 	drm_dbg(&xe->drm, "Waiting for lmem initialization\n");
641 
642 	start = jiffies;
643 	timeout = start + secs_to_jiffies(60); /* 60 sec! */
644 
645 	do {
646 		if (signal_pending(current))
647 			return -EINTR;
648 
649 		/*
650 		 * The boot firmware initializes local memory and
651 		 * assesses its health. If memory training fails,
652 		 * the punit will have been instructed to keep the GT powered
653 		 * down.we won't be able to communicate with it
654 		 *
655 		 * If the status check is done before punit updates the register,
656 		 * it can lead to the system being unusable.
657 		 * use a timeout and defer the probe to prevent this.
658 		 */
659 		if (time_after(jiffies, timeout)) {
660 			drm_dbg(&xe->drm, "lmem not initialized by firmware\n");
661 			return -EPROBE_DEFER;
662 		}
663 
664 		msleep(20);
665 
666 	} while (!verify_lmem_ready(xe));
667 
668 	drm_dbg(&xe->drm, "lmem ready after %ums",
669 		jiffies_to_msecs(jiffies - start));
670 
671 	return 0;
672 }
673 ALLOW_ERROR_INJECTION(wait_for_lmem_ready, ERRNO); /* See xe_pci_probe() */
674 
sriov_update_device_info(struct xe_device * xe)675 static void sriov_update_device_info(struct xe_device *xe)
676 {
677 	/* disable features that are not available/applicable to VFs */
678 	if (IS_SRIOV_VF(xe)) {
679 		xe->info.probe_display = 0;
680 		xe->info.has_heci_gscfi = 0;
681 		xe->info.skip_guc_pc = 1;
682 		xe->info.skip_pcode = 1;
683 	}
684 }
685 
686 /**
687  * xe_device_probe_early: Device early probe
688  * @xe: xe device instance
689  *
690  * Initialize MMIO resources that don't require any
691  * knowledge about tile count. Also initialize pcode and
692  * check vram initialization on root tile.
693  *
694  * Return: 0 on success, error code on failure
695  */
xe_device_probe_early(struct xe_device * xe)696 int xe_device_probe_early(struct xe_device *xe)
697 {
698 	int err;
699 
700 	err = xe_mmio_probe_early(xe);
701 	if (err)
702 		return err;
703 
704 	xe_sriov_probe_early(xe);
705 
706 	sriov_update_device_info(xe);
707 
708 	err = xe_pcode_probe_early(xe);
709 	if (err) {
710 		int save_err = err;
711 
712 		/*
713 		 * Try to leave device in survivability mode if device is
714 		 * possible, but still return the previous error for error
715 		 * propagation
716 		 */
717 		err = xe_survivability_mode_enable(xe);
718 		if (err)
719 			return err;
720 
721 		return save_err;
722 	}
723 
724 	err = wait_for_lmem_ready(xe);
725 	if (err)
726 		return err;
727 
728 	xe->wedged.mode = xe_modparam.wedged_mode;
729 
730 	return 0;
731 }
732 
probe_has_flat_ccs(struct xe_device * xe)733 static int probe_has_flat_ccs(struct xe_device *xe)
734 {
735 	struct xe_gt *gt;
736 	unsigned int fw_ref;
737 	u32 reg;
738 
739 	/* Always enabled/disabled, no runtime check to do */
740 	if (GRAPHICS_VER(xe) < 20 || !xe->info.has_flat_ccs || IS_SRIOV_VF(xe))
741 		return 0;
742 
743 	gt = xe_root_mmio_gt(xe);
744 
745 	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
746 	if (!fw_ref)
747 		return -ETIMEDOUT;
748 
749 	reg = xe_gt_mcr_unicast_read_any(gt, XE2_FLAT_CCS_BASE_RANGE_LOWER);
750 	xe->info.has_flat_ccs = (reg & XE2_FLAT_CCS_ENABLE);
751 
752 	if (!xe->info.has_flat_ccs)
753 		drm_dbg(&xe->drm,
754 			"Flat CCS has been disabled in bios, May lead to performance impact");
755 
756 	xe_force_wake_put(gt_to_fw(gt), fw_ref);
757 
758 	return 0;
759 }
760 
xe_device_probe(struct xe_device * xe)761 int xe_device_probe(struct xe_device *xe)
762 {
763 	struct xe_tile *tile;
764 	struct xe_gt *gt;
765 	int err;
766 	u8 id;
767 
768 	xe_pat_init_early(xe);
769 
770 	err = xe_sriov_init(xe);
771 	if (err)
772 		return err;
773 
774 	xe->info.mem_region_mask = 1;
775 
776 	err = xe_set_dma_info(xe);
777 	if (err)
778 		return err;
779 
780 	err = xe_mmio_probe_tiles(xe);
781 	if (err)
782 		return err;
783 
784 	err = xe_ttm_sys_mgr_init(xe);
785 	if (err)
786 		return err;
787 
788 	for_each_gt(gt, xe, id) {
789 		err = xe_gt_init_early(gt);
790 		if (err)
791 			return err;
792 
793 		/*
794 		 * Only after this point can GT-specific MMIO operations
795 		 * (including things like communication with the GuC)
796 		 * be performed.
797 		 */
798 		xe_gt_mmio_init(gt);
799 	}
800 
801 	for_each_tile(tile, xe, id) {
802 		if (IS_SRIOV_VF(xe)) {
803 			xe_guc_comm_init_early(&tile->primary_gt->uc.guc);
804 			err = xe_gt_sriov_vf_bootstrap(tile->primary_gt);
805 			if (err)
806 				return err;
807 			err = xe_gt_sriov_vf_query_config(tile->primary_gt);
808 			if (err)
809 				return err;
810 		}
811 		err = xe_ggtt_init_early(tile->mem.ggtt);
812 		if (err)
813 			return err;
814 		err = xe_memirq_init(&tile->memirq);
815 		if (err)
816 			return err;
817 	}
818 
819 	for_each_gt(gt, xe, id) {
820 		err = xe_gt_init_hwconfig(gt);
821 		if (err)
822 			return err;
823 	}
824 
825 	err = xe_devcoredump_init(xe);
826 	if (err)
827 		return err;
828 
829 	/*
830 	 * From here on, if a step fails, make sure a Driver-FLR is triggereed
831 	 */
832 	err = devm_add_action_or_reset(xe->drm.dev, xe_driver_flr_fini, xe);
833 	if (err)
834 		return err;
835 
836 	err = probe_has_flat_ccs(xe);
837 	if (err)
838 		return err;
839 
840 	err = xe_vram_probe(xe);
841 	if (err)
842 		return err;
843 
844 	for_each_tile(tile, xe, id) {
845 		err = xe_tile_init_noalloc(tile);
846 		if (err)
847 			return err;
848 	}
849 
850 	/* Allocate and map stolen after potential VRAM resize */
851 	err = xe_ttm_stolen_mgr_init(xe);
852 	if (err)
853 		return err;
854 
855 	/*
856 	 * Now that GT is initialized (TTM in particular),
857 	 * we can try to init display, and inherit the initial fb.
858 	 * This is the reason the first allocation needs to be done
859 	 * inside display.
860 	 */
861 	err = xe_display_init_early(xe);
862 	if (err)
863 		return err;
864 
865 	for_each_tile(tile, xe, id) {
866 		err = xe_tile_init(tile);
867 		if (err)
868 			return err;
869 	}
870 
871 	err = xe_irq_install(xe);
872 	if (err)
873 		return err;
874 
875 	for_each_gt(gt, xe, id) {
876 		err = xe_gt_init(gt);
877 		if (err)
878 			return err;
879 	}
880 
881 	err = xe_heci_gsc_init(xe);
882 	if (err)
883 		return err;
884 
885 	err = xe_oa_init(xe);
886 	if (err)
887 		return err;
888 
889 	err = xe_display_init(xe);
890 	if (err)
891 		return err;
892 
893 	err = xe_pxp_init(xe);
894 	if (err)
895 		return err;
896 
897 	err = drm_dev_register(&xe->drm, 0);
898 	if (err)
899 		return err;
900 
901 	xe_display_register(xe);
902 
903 	err = xe_oa_register(xe);
904 	if (err)
905 		goto err_unregister_display;
906 
907 	err = xe_pmu_register(&xe->pmu);
908 	if (err)
909 		goto err_unregister_display;
910 
911 	xe_debugfs_register(xe);
912 
913 	err = xe_hwmon_register(xe);
914 	if (err)
915 		goto err_unregister_display;
916 
917 	for_each_gt(gt, xe, id)
918 		xe_gt_sanitize_freq(gt);
919 
920 	xe_vsec_init(xe);
921 
922 	return devm_add_action_or_reset(xe->drm.dev, xe_device_sanitize, xe);
923 
924 err_unregister_display:
925 	xe_display_unregister(xe);
926 
927 	return err;
928 }
929 
xe_device_remove(struct xe_device * xe)930 void xe_device_remove(struct xe_device *xe)
931 {
932 	xe_display_unregister(xe);
933 
934 	drm_dev_unplug(&xe->drm);
935 }
936 
xe_device_shutdown(struct xe_device * xe)937 void xe_device_shutdown(struct xe_device *xe)
938 {
939 	struct xe_gt *gt;
940 	u8 id;
941 
942 	drm_dbg(&xe->drm, "Shutting down device\n");
943 
944 	if (xe_driver_flr_disabled(xe)) {
945 		xe_display_pm_shutdown(xe);
946 
947 		xe_irq_suspend(xe);
948 
949 		for_each_gt(gt, xe, id)
950 			xe_gt_shutdown(gt);
951 
952 		xe_display_pm_shutdown_late(xe);
953 	} else {
954 		/* BOOM! */
955 		__xe_driver_flr(xe);
956 	}
957 }
958 
959 /**
960  * xe_device_wmb() - Device specific write memory barrier
961  * @xe: the &xe_device
962  *
963  * While wmb() is sufficient for a barrier if we use system memory, on discrete
964  * platforms with device memory we additionally need to issue a register write.
965  * Since it doesn't matter which register we write to, use the read-only VF_CAP
966  * register that is also marked as accessible by the VFs.
967  */
xe_device_wmb(struct xe_device * xe)968 void xe_device_wmb(struct xe_device *xe)
969 {
970 	wmb();
971 	if (IS_DGFX(xe))
972 		xe_mmio_write32(xe_root_tile_mmio(xe), VF_CAP_REG, 0);
973 }
974 
975 /**
976  * xe_device_td_flush() - Flush transient L3 cache entries
977  * @xe: The device
978  *
979  * Display engine has direct access to memory and is never coherent with L3/L4
980  * caches (or CPU caches), however KMD is responsible for specifically flushing
981  * transient L3 GPU cache entries prior to the flip sequence to ensure scanout
982  * can happen from such a surface without seeing corruption.
983  *
984  * Display surfaces can be tagged as transient by mapping it using one of the
985  * various L3:XD PAT index modes on Xe2.
986  *
987  * Note: On non-discrete xe2 platforms, like LNL, the entire L3 cache is flushed
988  * at the end of each submission via PIPE_CONTROL for compute/render, since SA
989  * Media is not coherent with L3 and we want to support render-vs-media
990  * usescases. For other engines like copy/blt the HW internally forces uncached
991  * behaviour, hence why we can skip the TDF on such platforms.
992  */
xe_device_td_flush(struct xe_device * xe)993 void xe_device_td_flush(struct xe_device *xe)
994 {
995 	struct xe_gt *gt;
996 	unsigned int fw_ref;
997 	u8 id;
998 
999 	if (!IS_DGFX(xe) || GRAPHICS_VER(xe) < 20)
1000 		return;
1001 
1002 	if (XE_WA(xe_root_mmio_gt(xe), 16023588340)) {
1003 		xe_device_l2_flush(xe);
1004 		return;
1005 	}
1006 
1007 	for_each_gt(gt, xe, id) {
1008 		if (xe_gt_is_media_type(gt))
1009 			continue;
1010 
1011 		fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
1012 		if (!fw_ref)
1013 			return;
1014 
1015 		xe_mmio_write32(&gt->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST);
1016 		/*
1017 		 * FIXME: We can likely do better here with our choice of
1018 		 * timeout. Currently we just assume the worst case, i.e. 150us,
1019 		 * which is believed to be sufficient to cover the worst case
1020 		 * scenario on current platforms if all cache entries are
1021 		 * transient and need to be flushed..
1022 		 */
1023 		if (xe_mmio_wait32(&gt->mmio, XE2_TDF_CTRL, TRANSIENT_FLUSH_REQUEST, 0,
1024 				   150, NULL, false))
1025 			xe_gt_err_once(gt, "TD flush timeout\n");
1026 
1027 		xe_force_wake_put(gt_to_fw(gt), fw_ref);
1028 	}
1029 }
1030 
xe_device_l2_flush(struct xe_device * xe)1031 void xe_device_l2_flush(struct xe_device *xe)
1032 {
1033 	struct xe_gt *gt;
1034 	unsigned int fw_ref;
1035 
1036 	gt = xe_root_mmio_gt(xe);
1037 
1038 	if (!XE_WA(gt, 16023588340))
1039 		return;
1040 
1041 	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
1042 	if (!fw_ref)
1043 		return;
1044 
1045 	spin_lock(&gt->global_invl_lock);
1046 	xe_mmio_write32(&gt->mmio, XE2_GLOBAL_INVAL, 0x1);
1047 
1048 	if (xe_mmio_wait32(&gt->mmio, XE2_GLOBAL_INVAL, 0x1, 0x0, 500, NULL, true))
1049 		xe_gt_err_once(gt, "Global invalidation timeout\n");
1050 	spin_unlock(&gt->global_invl_lock);
1051 
1052 	xe_force_wake_put(gt_to_fw(gt), fw_ref);
1053 }
1054 
xe_device_ccs_bytes(struct xe_device * xe,u64 size)1055 u32 xe_device_ccs_bytes(struct xe_device *xe, u64 size)
1056 {
1057 	return xe_device_has_flat_ccs(xe) ?
1058 		DIV_ROUND_UP_ULL(size, NUM_BYTES_PER_CCS_BYTE(xe)) : 0;
1059 }
1060 
1061 /**
1062  * xe_device_assert_mem_access - Inspect the current runtime_pm state.
1063  * @xe: xe device instance
1064  *
1065  * To be used before any kind of memory access. It will splat a debug warning
1066  * if the device is currently sleeping. But it doesn't guarantee in any way
1067  * that the device is going to remain awake. Xe PM runtime get and put
1068  * functions might be added to the outer bound of the memory access, while
1069  * this check is intended for inner usage to splat some warning if the worst
1070  * case has just happened.
1071  */
xe_device_assert_mem_access(struct xe_device * xe)1072 void xe_device_assert_mem_access(struct xe_device *xe)
1073 {
1074 	xe_assert(xe, !xe_pm_runtime_suspended(xe));
1075 }
1076 
xe_device_snapshot_print(struct xe_device * xe,struct drm_printer * p)1077 void xe_device_snapshot_print(struct xe_device *xe, struct drm_printer *p)
1078 {
1079 	struct xe_gt *gt;
1080 	u8 id;
1081 
1082 	drm_printf(p, "PCI ID: 0x%04x\n", xe->info.devid);
1083 	drm_printf(p, "PCI revision: 0x%02x\n", xe->info.revid);
1084 
1085 	for_each_gt(gt, xe, id) {
1086 		drm_printf(p, "GT id: %u\n", id);
1087 		drm_printf(p, "\tTile: %u\n", gt->tile->id);
1088 		drm_printf(p, "\tType: %s\n",
1089 			   gt->info.type == XE_GT_TYPE_MAIN ? "main" : "media");
1090 		drm_printf(p, "\tIP ver: %u.%u.%u\n",
1091 			   REG_FIELD_GET(GMD_ID_ARCH_MASK, gt->info.gmdid),
1092 			   REG_FIELD_GET(GMD_ID_RELEASE_MASK, gt->info.gmdid),
1093 			   REG_FIELD_GET(GMD_ID_REVID, gt->info.gmdid));
1094 		drm_printf(p, "\tCS reference clock: %u\n", gt->info.reference_clock);
1095 	}
1096 }
1097 
xe_device_canonicalize_addr(struct xe_device * xe,u64 address)1098 u64 xe_device_canonicalize_addr(struct xe_device *xe, u64 address)
1099 {
1100 	return sign_extend64(address, xe->info.va_bits - 1);
1101 }
1102 
xe_device_uncanonicalize_addr(struct xe_device * xe,u64 address)1103 u64 xe_device_uncanonicalize_addr(struct xe_device *xe, u64 address)
1104 {
1105 	return address & GENMASK_ULL(xe->info.va_bits - 1, 0);
1106 }
1107 
xe_device_wedged_fini(struct drm_device * drm,void * arg)1108 static void xe_device_wedged_fini(struct drm_device *drm, void *arg)
1109 {
1110 	struct xe_device *xe = arg;
1111 
1112 	xe_pm_runtime_put(xe);
1113 }
1114 
1115 /**
1116  * xe_device_declare_wedged - Declare device wedged
1117  * @xe: xe device instance
1118  *
1119  * This is a final state that can only be cleared with a module
1120  * re-probe (unbind + bind).
1121  * In this state every IOCTL will be blocked so the GT cannot be used.
1122  * In general it will be called upon any critical error such as gt reset
1123  * failure or guc loading failure. Userspace will be notified of this state
1124  * through device wedged uevent.
1125  * If xe.wedged module parameter is set to 2, this function will be called
1126  * on every single execution timeout (a.k.a. GPU hang) right after devcoredump
1127  * snapshot capture. In this mode, GT reset won't be attempted so the state of
1128  * the issue is preserved for further debugging.
1129  */
xe_device_declare_wedged(struct xe_device * xe)1130 void xe_device_declare_wedged(struct xe_device *xe)
1131 {
1132 	struct xe_gt *gt;
1133 	u8 id;
1134 
1135 	if (xe->wedged.mode == 0) {
1136 		drm_dbg(&xe->drm, "Wedged mode is forcibly disabled\n");
1137 		return;
1138 	}
1139 
1140 	xe_pm_runtime_get_noresume(xe);
1141 
1142 	if (drmm_add_action_or_reset(&xe->drm, xe_device_wedged_fini, xe)) {
1143 		drm_err(&xe->drm, "Failed to register xe_device_wedged_fini clean-up. Although device is wedged.\n");
1144 		return;
1145 	}
1146 
1147 	if (!atomic_xchg(&xe->wedged.flag, 1)) {
1148 		xe->needs_flr_on_fini = true;
1149 		drm_err(&xe->drm,
1150 			"CRITICAL: Xe has declared device %s as wedged.\n"
1151 			"IOCTLs and executions are blocked. Only a rebind may clear the failure\n"
1152 			"Please file a _new_ bug report at https://gitlab.freedesktop.org/drm/xe/kernel/issues/new\n",
1153 			dev_name(xe->drm.dev));
1154 
1155 		/* Notify userspace of wedged device */
1156 		drm_dev_wedged_event(&xe->drm,
1157 				     DRM_WEDGE_RECOVERY_REBIND | DRM_WEDGE_RECOVERY_BUS_RESET);
1158 	}
1159 
1160 	for_each_gt(gt, xe, id)
1161 		xe_gt_declare_wedged(gt);
1162 }
1163