1 // SPDX-License-Identifier: MIT
30 	int count = atomic_read(&gt->user_wakeref);  in user_forcewake()
39 		GEM_BUG_ON(count > atomic_read(&gt->wakeref.count));  in user_forcewake()
40 		atomic_sub(count, &gt->wakeref.count);  in user_forcewake()
42 		atomic_add(count, &gt->wakeref.count);  in user_forcewake()
50 	write_seqcount_begin(&gt->stats.lock);  in runtime_begin()
51 	gt->stats.start = ktime_get();  in runtime_begin()
52 	gt->stats.active = true;  in runtime_begin()
53 	write_seqcount_end(&gt->stats.lock);  in runtime_begin()
60 	write_seqcount_begin(&gt->stats.lock);  in runtime_end()
61 	gt->stats.active = false;  in runtime_end()
62 	gt->stats.total =  in runtime_end()
63 		ktime_add(gt->stats.total,  in runtime_end()
64 			  ktime_sub(ktime_get(), gt->stats.start));  in runtime_end()
65 	write_seqcount_end(&gt->stats.lock);  in runtime_end()
72 	struct drm_i915_private *i915 = gt->i915;  in __gt_unpark()
87 	gt->awake = intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);  in __gt_unpark()
88 	GEM_BUG_ON(!gt->awake);  in __gt_unpark()
90 	intel_rc6_unpark(&gt->rc6);  in __gt_unpark()
91 	intel_rps_unpark(&gt->rps);  in __gt_unpark()
104 	intel_wakeref_t wakeref = fetch_and_zero(&gt->awake);  in __gt_park()
105 	struct drm_i915_private *i915 = gt->i915;  in __gt_park()
115 	intel_rps_park(&gt->rps);  in __gt_park()
116 	intel_rc6_park(&gt->rc6);  in __gt_park()
136 	 * We access the runtime_pm structure via gt->i915 here rather than  in intel_gt_pm_init_early()
137 	 * gt->uncore as we do elsewhere in the file because gt->uncore is not  in intel_gt_pm_init_early()
139 	 * runtime_pm is per-device rather than per-tile, so this is still the  in intel_gt_pm_init_early()
142 	intel_wakeref_init(&gt->wakeref, gt->i915, &wf_ops, "GT");  in intel_gt_pm_init_early()
143 	seqcount_mutex_init(&gt->stats.lock, &gt->wakeref.mutex);  in intel_gt_pm_init_early()
149 	 * Enabling power-management should be "self-healing". If we cannot  in intel_gt_pm_init()
153 	intel_rc6_init(&gt->rc6);  in intel_gt_pm_init()
154 	intel_rps_init(&gt->rps);  in intel_gt_pm_init()
159 	if (INTEL_INFO(gt->i915)->gpu_reset_clobbers_display)  in reset_engines()
174 	wakeref = intel_runtime_pm_get(gt->uncore->rpm);  in gt_sanitize()
175 	intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);  in gt_sanitize()
189 	intel_uc_reset_prepare(&gt->uc);  in gt_sanitize()
192 		if (engine->reset.prepare)  in gt_sanitize()
193 			engine->reset.prepare(engine);  in gt_sanitize()
195 		if (engine->sanitize)  in gt_sanitize()
196 			engine->sanitize(engine);  in gt_sanitize()
204 	intel_uc_reset(&gt->uc, false);  in gt_sanitize()
207 		if (engine->reset.finish)  in gt_sanitize()
208 			engine->reset.finish(engine);  in gt_sanitize()
210 	intel_rps_sanitize(&gt->rps);  in gt_sanitize()
212 	intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);  in gt_sanitize()
213 	intel_runtime_pm_put(gt->uncore->rpm, wakeref);  in gt_sanitize()
218 	intel_rc6_fini(&gt->rc6);  in intel_gt_pm_fini()
232 	intel_uncore_resume_early(gt->uncore);  in intel_gt_resume_early()
259 	intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);  in intel_gt_resume()
260 	intel_rc6_sanitize(&gt->rc6);  in intel_gt_resume()
262 		err = -EIO;  in intel_gt_resume()
266 	/* Only when the HW is re-initialised, can we replay the requests */  in intel_gt_resume()
273 	intel_uc_reset_finish(&gt->uc);  in intel_gt_resume()
275 	intel_rps_enable(&gt->rps);  in intel_gt_resume()
276 	intel_llc_enable(&gt->llc);  in intel_gt_resume()
281 		engine->serial++; /* kernel context lost */  in intel_gt_resume()
287 			       engine->name, err);  in intel_gt_resume()
292 	intel_rc6_enable(&gt->rc6);  in intel_gt_resume()
294 	intel_uc_resume(&gt->uc);  in intel_gt_resume()
299 	intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);  in intel_gt_resume()
314 	if (intel_gt_wait_for_idle(gt, I915_GT_SUSPEND_IDLE_TIMEOUT) == -ETIME) {  in wait_for_suspend()
352 	GEM_BUG_ON(gt->awake);  in intel_gt_suspend_late()
354 	intel_uc_suspend(&gt->uc);  in intel_gt_suspend_late()
360 	 * However, not all suspend-states disable the device. S0 (s2idle)  in intel_gt_suspend_late()
361 	 * is effectively runtime-suspend, the device is left powered on  in intel_gt_suspend_late()
369 	with_intel_runtime_pm(gt->uncore->rpm, wakeref) {  in intel_gt_suspend_late()
370 		intel_rps_disable(&gt->rps);  in intel_gt_suspend_late()
371 		intel_rc6_disable(&gt->rc6);  in intel_gt_suspend_late()
372 		intel_llc_disable(&gt->llc);  in intel_gt_suspend_late()
383 	intel_uc_runtime_suspend(&gt->uc);  in intel_gt_runtime_suspend()
394 	intel_ggtt_restore_fences(gt->ggtt);  in intel_gt_runtime_resume()
396 	ret = intel_uc_runtime_resume(&gt->uc);  in intel_gt_runtime_resume()
406 	ktime_t total = gt->stats.total;  in __intel_gt_get_awake_time()
408 	if (gt->stats.active)  in __intel_gt_get_awake_time()
410 				  ktime_sub(ktime_get(), gt->stats.start));  in __intel_gt_get_awake_time()
421 		seq = read_seqcount_begin(&gt->stats.lock);  in intel_gt_get_awake_time()
423 	} while (read_seqcount_retry(&gt->stats.lock, seq));  in intel_gt_get_awake_time()