1 // SPDX-License-Identifier: MIT
30 	int count = atomic_read(&gt->user_wakeref);  in user_forcewake()
39 		GEM_BUG_ON(count > atomic_read(&gt->wakeref.count));  in user_forcewake()
40 		atomic_sub(count, &gt->wakeref.count);  in user_forcewake()
42 		atomic_add(count, &gt->wakeref.count);  in user_forcewake()
50 	write_seqcount_begin(&gt->stats.lock);  in runtime_begin()
51 	gt->stats.start = ktime_get();  in runtime_begin()
52 	gt->stats.active = true;  in runtime_begin()
53 	write_seqcount_end(&gt->stats.lock);  in runtime_begin()
60 	write_seqcount_begin(&gt->stats.lock);  in runtime_end()
61 	gt->stats.active = false;  in runtime_end()
62 	gt->stats.total =  in runtime_end()
63 		ktime_add(gt->stats.total,  in runtime_end()
64 			  ktime_sub(ktime_get(), gt->stats.start));  in runtime_end()
65 	write_seqcount_end(&gt->stats.lock);  in runtime_end()
72 	struct drm_i915_private *i915 = gt->i915;  in __gt_unpark()
73 	struct intel_display *display = &i915->display;  in __gt_unpark()
88 	gt->awake = intel_display_power_get(display, POWER_DOMAIN_GT_IRQ);  in __gt_unpark()
89 	GEM_BUG_ON(!gt->awake);  in __gt_unpark()
91 	intel_rc6_unpark(&gt->rc6);  in __gt_unpark()
92 	intel_rps_unpark(&gt->rps);  in __gt_unpark()
105 	intel_wakeref_t wakeref = fetch_and_zero(&gt->awake);  in __gt_park()
106 	struct drm_i915_private *i915 = gt->i915;  in __gt_park()
107 	struct intel_display *display = &i915->display;  in __gt_park()
117 	intel_rps_park(&gt->rps);  in __gt_park()
118 	intel_rc6_park(&gt->rc6);  in __gt_park()
138 	 * We access the runtime_pm structure via gt->i915 here rather than  in intel_gt_pm_init_early()
139 	 * gt->uncore as we do elsewhere in the file because gt->uncore is not  in intel_gt_pm_init_early()
141 	 * runtime_pm is per-device rather than per-tile, so this is still the  in intel_gt_pm_init_early()
144 	intel_wakeref_init(&gt->wakeref, gt->i915, &wf_ops, "GT");  in intel_gt_pm_init_early()
145 	seqcount_mutex_init(&gt->stats.lock, &gt->wakeref.mutex);  in intel_gt_pm_init_early()
151 	 * Enabling power-management should be "self-healing". If we cannot  in intel_gt_pm_init()
155 	intel_rc6_init(&gt->rc6);  in intel_gt_pm_init()
156 	intel_rps_init(&gt->rps);  in intel_gt_pm_init()
176 	wakeref = intel_runtime_pm_get(gt->uncore->rpm);  in gt_sanitize()
177 	intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);  in gt_sanitize()
191 	intel_uc_reset_prepare(&gt->uc);  in gt_sanitize()
194 		if (engine->reset.prepare)  in gt_sanitize()
195 			engine->reset.prepare(engine);  in gt_sanitize()
197 		if (engine->sanitize)  in gt_sanitize()
198 			engine->sanitize(engine);  in gt_sanitize()
206 	intel_uc_reset(&gt->uc, false);  in gt_sanitize()
209 		if (engine->reset.finish)  in gt_sanitize()
210 			engine->reset.finish(engine);  in gt_sanitize()
212 	intel_rps_sanitize(&gt->rps);  in gt_sanitize()
214 	intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);  in gt_sanitize()
215 	intel_runtime_pm_put(gt->uncore->rpm, wakeref);  in gt_sanitize()
220 	intel_rc6_fini(&gt->rc6);  in intel_gt_pm_fini()
234 	intel_uncore_resume_early(gt->uncore);  in intel_gt_resume_early()
261 	intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);  in intel_gt_resume()
262 	intel_rc6_sanitize(&gt->rc6);  in intel_gt_resume()
264 		err = -EIO;  in intel_gt_resume()
268 	/* Only when the HW is re-initialised, can we replay the requests */  in intel_gt_resume()
275 	intel_uc_reset_finish(&gt->uc);  in intel_gt_resume()
277 	intel_rps_enable(&gt->rps);  in intel_gt_resume()
278 	intel_llc_enable(&gt->llc);  in intel_gt_resume()
283 		engine->serial++; /* kernel context lost */  in intel_gt_resume()
289 			       engine->name, err);  in intel_gt_resume()
294 	intel_rc6_enable(&gt->rc6);  in intel_gt_resume()
296 	intel_uc_resume(&gt->uc);  in intel_gt_resume()
301 	intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);  in intel_gt_resume()
316 	if (intel_gt_wait_for_idle(gt, I915_GT_SUSPEND_IDLE_TIMEOUT) == -ETIME) {  in wait_for_suspend()
354 	GEM_BUG_ON(gt->awake);  in intel_gt_suspend_late()
356 	intel_uc_suspend(&gt->uc);  in intel_gt_suspend_late()
362 	 * However, not all suspend-states disable the device. S0 (s2idle)  in intel_gt_suspend_late()
363 	 * is effectively runtime-suspend, the device is left powered on  in intel_gt_suspend_late()
371 	with_intel_runtime_pm(gt->uncore->rpm, wakeref) {  in intel_gt_suspend_late()
372 		intel_rps_disable(&gt->rps);  in intel_gt_suspend_late()
373 		intel_rc6_disable(&gt->rc6);  in intel_gt_suspend_late()
374 		intel_llc_disable(&gt->llc);  in intel_gt_suspend_late()
385 	intel_uc_runtime_suspend(&gt->uc);  in intel_gt_runtime_suspend()
396 	intel_ggtt_restore_fences(gt->ggtt);  in intel_gt_runtime_resume()
398 	ret = intel_uc_runtime_resume(&gt->uc);  in intel_gt_runtime_resume()
408 	ktime_t total = gt->stats.total;  in __intel_gt_get_awake_time()
410 	if (gt->stats.active)  in __intel_gt_get_awake_time()
412 				  ktime_sub(ktime_get(), gt->stats.start));  in __intel_gt_get_awake_time()
423 		seq = read_seqcount_begin(&gt->stats.lock);  in intel_gt_get_awake_time()
425 	} while (read_seqcount_retry(&gt->stats.lock, seq));  in intel_gt_get_awake_time()