drm/i915/i915_driver.c

118 static int i915_workqueues_init(struct drm_i915_private *dev_priv)
134 	dev_priv->wq = alloc_ordered_workqueue("i915", 0);
135 	if (dev_priv->wq == NULL)
144 	dev_priv->unordered_wq = alloc_workqueue("i915-unordered", 0, 0);
145 	if (dev_priv->unordered_wq == NULL)
151 	destroy_workqueue(dev_priv->wq);
153 	drm_err(&dev_priv->drm, "Failed to allocate workqueues.\n");
158 static void i915_workqueues_cleanup(struct drm_i915_private *dev_priv)
160 	destroy_workqueue(dev_priv->unordered_wq);
161 	destroy_workqueue(dev_priv->wq);
174 static void intel_detect_preproduction_hw(struct drm_i915_private *dev_priv)
178 	pre |= IS_HASWELL_EARLY_SDV(dev_priv);
179 	pre |= IS_SKYLAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x6;
180 	pre |= IS_BROXTON(dev_priv) && INTEL_REVID(dev_priv) < 0xA;
181 	pre |= IS_KABYLAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x1;
182 	pre |= IS_GEMINILAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x3;
183 	pre |= IS_ICELAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x7;
184 	pre |= IS_TIGERLAKE(dev_priv) && INTEL_REVID(dev_priv) < 0x1;
185 	pre |= IS_DG1(dev_priv) && INTEL_REVID(dev_priv) < 0x1;
186 	pre |= IS_DG2_G10(dev_priv) && INTEL_REVID(dev_priv) < 0x8;
187 	pre |= IS_DG2_G11(dev_priv) && INTEL_REVID(dev_priv) < 0x5;
188 	pre |= IS_DG2_G12(dev_priv) && INTEL_REVID(dev_priv) < 0x1;
191 		drm_err(&dev_priv->drm, "This is a pre-production stepping. "
210  * @dev_priv: device private
218 static int i915_driver_early_probe(struct drm_i915_private *dev_priv)
220 	struct intel_display *display = dev_priv->display;
223 	if (i915_inject_probe_failure(dev_priv))
226 	intel_device_info_runtime_init_early(dev_priv);
228 	intel_step_init(dev_priv);
230 	intel_uncore_mmio_debug_init_early(dev_priv);
232 	spin_lock_init(&dev_priv->gpu_error.lock);
235 	vlv_iosf_sb_init(dev_priv);
236 	mutex_init(&dev_priv->sb_lock);
238 	i915_memcpy_init_early(dev_priv);
239 	intel_runtime_pm_init_early(&dev_priv->runtime_pm);
241 	ret = i915_workqueues_init(dev_priv);
245 	ret = vlv_suspend_init(dev_priv);
249 	ret = intel_region_ttm_device_init(dev_priv);
253 	ret = intel_root_gt_init_early(dev_priv);
257 	i915_gem_init_early(dev_priv);
259 	intel_irq_init(dev_priv);
261 	intel_clock_gating_hooks_init(dev_priv);
263 	intel_detect_preproduction_hw(dev_priv);
268 	intel_region_ttm_device_fini(dev_priv);
270 	vlv_suspend_cleanup(dev_priv);
272 	i915_workqueues_cleanup(dev_priv);
279  * @dev_priv: device private
281 static void i915_driver_late_release(struct drm_i915_private *dev_priv)
283 	struct intel_display *display = dev_priv->display;
285 	intel_irq_fini(dev_priv);
287 	i915_gem_cleanup_early(dev_priv);
288 	intel_gt_driver_late_release_all(dev_priv);
289 	intel_region_ttm_device_fini(dev_priv);
290 	vlv_suspend_cleanup(dev_priv);
291 	i915_workqueues_cleanup(dev_priv);
293 	mutex_destroy(&dev_priv->sb_lock);
294 	vlv_iosf_sb_fini(dev_priv);
297 	i915_params_free(&dev_priv->params);
304  * @dev_priv: device private
311 static int i915_driver_mmio_probe(struct drm_i915_private *dev_priv)
313 	struct intel_display *display = dev_priv->display;
317 	if (i915_inject_probe_failure(dev_priv))
320 	ret = intel_gmch_bridge_setup(dev_priv);
324 	for_each_gt(gt, dev_priv, i) {
329 		ret = drmm_add_action_or_reset(&dev_priv->drm,
337 	intel_gmch_bar_setup(dev_priv);
338 	intel_device_info_runtime_init(dev_priv);
341 	for_each_gt(gt, dev_priv, i) {
348 	sanitize_gpu(dev_priv);
353 	intel_gmch_bar_teardown(dev_priv);
360  * @dev_priv: device private
362 static void i915_driver_mmio_release(struct drm_i915_private *dev_priv)
364 	intel_gmch_bar_teardown(dev_priv);
453  * @dev_priv: device private
458 static int i915_driver_hw_probe(struct drm_i915_private *dev_priv)
460 	struct intel_display *display = dev_priv->display;
461 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
464 	if (i915_inject_probe_failure(dev_priv))
467 	if (HAS_PPGTT(dev_priv)) {
468 		if (intel_vgpu_active(dev_priv) &&
469 		    !intel_vgpu_has_full_ppgtt(dev_priv)) {
470 			drm_err(&dev_priv->drm,
476 	if (HAS_EXECLISTS(dev_priv)) {
482 		if (intel_vgpu_active(dev_priv) &&
483 		    !intel_vgpu_has_hwsp_emulation(dev_priv)) {
484 			drm_err(&dev_priv->drm,
491 	intel_dram_edram_detect(dev_priv);
493 	ret = i915_set_dma_info(dev_priv);
497 	ret = i915_perf_init(dev_priv);
501 	ret = i915_ggtt_probe_hw(dev_priv);
505 	ret = aperture_remove_conflicting_pci_devices(pdev, dev_priv->drm.driver->name);
509 	ret = i915_ggtt_init_hw(dev_priv);
517 	ret = intel_gt_tiles_init(dev_priv);
521 	ret = intel_memory_regions_hw_probe(dev_priv);
525 	ret = i915_ggtt_enable_hw(dev_priv);
527 		drm_err(&dev_priv->drm, "failed to enable GGTT\n");
552 	if (GRAPHICS_VER(dev_priv) >= 5) {
554 			drm_dbg(&dev_priv->drm, "can't enable MSI");
557 	ret = intel_gvt_init(dev_priv);
563 	ret = i915_pcode_init(dev_priv);
571 	ret = intel_dram_detect(dev_priv);
585 	intel_memory_regions_driver_release(dev_priv);
587 	i915_ggtt_driver_release(dev_priv);
588 	i915_gem_drain_freed_objects(dev_priv);
589 	i915_ggtt_driver_late_release(dev_priv);
591 	i915_perf_fini(dev_priv);
597  * @dev_priv: device private
599 static void i915_driver_hw_remove(struct drm_i915_private *dev_priv)
601 	struct intel_display *display = dev_priv->display;
602 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
604 	i915_perf_fini(dev_priv);
614  * @dev_priv: device private
619 static int i915_driver_register(struct drm_i915_private *dev_priv)
621 	struct intel_display *display = dev_priv->display;
626 	i915_gem_driver_register(dev_priv);
627 	i915_pmu_register(dev_priv);
629 	intel_vgpu_register(dev_priv);
632 	ret = drm_dev_register(&dev_priv->drm, 0);
634 		i915_probe_error(dev_priv,
636 		drm_dev_unregister(&dev_priv->drm);
637 		i915_pmu_unregister(dev_priv);
638 		i915_gem_driver_unregister(dev_priv);
642 	i915_debugfs_register(dev_priv);
643 	i915_setup_sysfs(dev_priv);
646 	i915_perf_register(dev_priv);
648 	for_each_gt(gt, dev_priv, i)
651 	intel_pxp_debugfs_register(dev_priv->pxp);
653 	i915_hwmon_register(dev_priv);
658 	intel_runtime_pm_enable(&dev_priv->runtime_pm);
660 	if (i915_switcheroo_register(dev_priv))
661 		drm_err(&dev_priv->drm, "Failed to register vga switcheroo!\n");
668  * @dev_priv: device private
670 static void i915_driver_unregister(struct drm_i915_private *dev_priv)
672 	struct intel_display *display = dev_priv->display;
676 	i915_switcheroo_unregister(dev_priv);
678 	intel_runtime_pm_disable(&dev_priv->runtime_pm);
683 	intel_pxp_fini(dev_priv);
685 	for_each_gt(gt, dev_priv, i)
688 	i915_hwmon_unregister(dev_priv);
690 	i915_perf_unregister(dev_priv);
691 	i915_pmu_unregister(dev_priv);
693 	i915_teardown_sysfs(dev_priv);
694 	drm_dev_unplug(&dev_priv->drm);
696 	i915_gem_driver_unregister(dev_priv);
706 static void i915_welcome_messages(struct drm_i915_private *dev_priv)
709 		struct drm_printer p = drm_dbg_printer(&dev_priv->drm, DRM_UT_DRIVER,
715 			   INTEL_DEVID(dev_priv),
716 			   INTEL_REVID(dev_priv),
717 			   intel_platform_name(INTEL_INFO(dev_priv)->platform),
718 			   intel_subplatform(RUNTIME_INFO(dev_priv),
719 					     INTEL_INFO(dev_priv)->platform),
720 			   GRAPHICS_VER(dev_priv));
722 		intel_device_info_print(INTEL_INFO(dev_priv),
723 					RUNTIME_INFO(dev_priv), &p);
724 		i915_print_iommu_status(dev_priv, &p);
725 		for_each_gt(gt, dev_priv, i)
730 		drm_info(&dev_priv->drm, "DRM_I915_DEBUG enabled\n");
732 		drm_info(&dev_priv->drm, "DRM_I915_DEBUG_GEM enabled\n");
734 		drm_info(&dev_priv->drm,
921 	struct drm_i915_private *dev_priv = to_i915(dev);
922 	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
925 	if (!dev_priv->do_release)
930 	i915_gem_driver_release(dev_priv);
932 	intel_memory_regions_driver_release(dev_priv);
933 	i915_ggtt_driver_release(dev_priv);
934 	i915_gem_drain_freed_objects(dev_priv);
935 	i915_ggtt_driver_late_release(dev_priv);
937 	i915_driver_mmio_release(dev_priv);
943 	i915_driver_late_release(dev_priv);
1019 static bool suspend_to_idle(struct drm_i915_private *dev_priv)
1052 	struct drm_i915_private *dev_priv = to_i915(dev);
1053 	struct intel_display *display = dev_priv->display;
1054 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
1057 	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1063 	if (HAS_DISPLAY(dev_priv)) {
1072 	intel_irq_suspend(dev_priv);
1075 	if (HAS_DISPLAY(dev_priv))
1082 	i915_ggtt_suspend(to_gt(dev_priv)->ggtt);
1086 	opregion_target_state = suspend_to_idle(dev_priv) ? PCI_D1 : PCI_D3cold;
1089 	dev_priv->suspend_count++;
1093 	enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1095 	i915_gem_drain_freed_objects(dev_priv);
1102 	struct drm_i915_private *dev_priv = to_i915(dev);
1103 	struct intel_display *display = dev_priv->display;
1104 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
1105 	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
1108 	bool s2idle = !hibernation && suspend_to_idle(dev_priv);
1112 	intel_pxp_suspend(dev_priv->pxp);
1114 	i915_gem_suspend_late(dev_priv);
1116 	for_each_gt(gt, dev_priv, i)
1121 	ret = vlv_suspend_complete(dev_priv);
1123 		drm_err(&dev_priv->drm, "Suspend complete failed: %d\n", ret);
1142 	if (!(hibernation && GRAPHICS_VER(dev_priv) < 6))
1147 	if (!dev_priv->uncore.user_forcewake_count)
1174 	struct drm_i915_private *dev_priv = to_i915(dev);
1175 	struct intel_display *display = dev_priv->display;
1179 	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1181 	ret = i915_pcode_init(dev_priv);
1185 	sanitize_gpu(dev_priv);
1187 	ret = i915_ggtt_enable_hw(dev_priv);
1189 		drm_err(&dev_priv->drm, "failed to re-enable GGTT\n");
1191 	i915_ggtt_resume(to_gt(dev_priv)->ggtt);
1193 	for_each_gt(gt, dev_priv, i)
1220 	intel_irq_resume(dev_priv);
1222 	if (HAS_DISPLAY(dev_priv))
1225 	i915_gem_resume(dev_priv);
1229 	intel_clock_gating_init(dev_priv);
1231 	if (HAS_DISPLAY(dev_priv))
1238 	if (HAS_DISPLAY(dev_priv)) {
1250 	intel_gvt_resume(dev_priv);
1252 	enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1259 	struct drm_i915_private *dev_priv = to_i915(dev);
1260 	struct intel_display *display = dev_priv->display;
1261 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
1287 		drm_err(&dev_priv->drm,
1310 	disable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1312 	ret = vlv_resume_prepare(dev_priv, false);
1314 		drm_err(&dev_priv->drm,
1317 	for_each_gt(gt, dev_priv, i)
1322 	enable_rpm_wakeref_asserts(&dev_priv->runtime_pm);
1491 	struct drm_i915_private *dev_priv = kdev_to_i915(kdev);
1492 	struct intel_display *display = dev_priv->display;
1493 	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
1494 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
1499 	if (drm_WARN_ON_ONCE(&dev_priv->drm, !HAS_RUNTIME_PM(dev_priv)))
1502 	drm_dbg(&dev_priv->drm, "Suspending device\n");
1510 	i915_gem_runtime_suspend(dev_priv);
1512 	intel_pxp_runtime_suspend(dev_priv->pxp);
1514 	for_each_gt(gt, dev_priv, i)
1517 	intel_irq_suspend(dev_priv);
1519 	for_each_gt(gt, dev_priv, i)
1524 	ret = vlv_suspend_complete(dev_priv);
1526 		drm_err(&dev_priv->drm,
1528 		intel_uncore_runtime_resume(&dev_priv->uncore);
1530 		intel_irq_resume(dev_priv);
1532 		for_each_gt(gt, dev_priv, i)
1543 	if (intel_uncore_arm_unclaimed_mmio_detection(&dev_priv->uncore))
1544 		drm_err(&dev_priv->drm,
1560 	if (IS_BROADWELL(dev_priv)) {
1579 	assert_forcewakes_inactive(&dev_priv->uncore);
1581 	if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
1584 	drm_dbg(&dev_priv->drm, "Device suspended\n");
1590 	struct drm_i915_private *dev_priv = kdev_to_i915(kdev);
1591 	struct intel_display *display = dev_priv->display;
1592 	struct intel_runtime_pm *rpm = &dev_priv->runtime_pm;
1593 	struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev);
1598 	if (drm_WARN_ON_ONCE(&dev_priv->drm, !HAS_RUNTIME_PM(dev_priv)))
1601 	drm_dbg(&dev_priv->drm, "Resuming device\n");
1603 	drm_WARN_ON_ONCE(&dev_priv->drm, atomic_read(&rpm->wakeref_count));
1612 	if (intel_uncore_unclaimed_mmio(&dev_priv->uncore))
1613 		drm_dbg(&dev_priv->drm,
1618 	ret = vlv_resume_prepare(dev_priv, true);
1620 	for_each_gt(gt, dev_priv, i)
1623 	intel_irq_resume(dev_priv);
1629 	for_each_gt(gt, dev_priv, i)
1632 	intel_pxp_runtime_resume(dev_priv->pxp);
1639 	if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv)) {
1649 		drm_err(&dev_priv->drm,
1652 		drm_dbg(&dev_priv->drm, "Device resumed\n");