53 static struct drm_i915_private *rpm_to_i915(struct intel_runtime_pm *rpm)  in rpm_to_i915()  argument
55 	return container_of(rpm, struct drm_i915_private, runtime_pm);  in rpm_to_i915()
60 static void init_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm)  in init_intel_runtime_pm_wakeref()  argument
62 	ref_tracker_dir_init(&rpm->debug, INTEL_REFTRACK_DEAD_COUNT, dev_name(rpm->kdev));  in init_intel_runtime_pm_wakeref()
66 track_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm)  in track_intel_runtime_pm_wakeref()  argument
68 	if (!rpm->available || rpm->no_wakeref_tracking)  in track_intel_runtime_pm_wakeref()
71 	return intel_ref_tracker_alloc(&rpm->debug);  in track_intel_runtime_pm_wakeref()
74 static void untrack_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm,  in untrack_intel_runtime_pm_wakeref()  argument
77 	if (!rpm->available || rpm->no_wakeref_tracking)  in untrack_intel_runtime_pm_wakeref()
80 	intel_ref_tracker_free(&rpm->debug, wakeref);  in untrack_intel_runtime_pm_wakeref()
83 static void untrack_all_intel_runtime_pm_wakerefs(struct intel_runtime_pm *rpm)  in untrack_all_intel_runtime_pm_wakerefs()  argument
85 	ref_tracker_dir_exit(&rpm->debug);  in untrack_all_intel_runtime_pm_wakerefs()
89 __intel_wakeref_dec_and_check_tracking(struct intel_runtime_pm *rpm)  in __intel_wakeref_dec_and_check_tracking()  argument
93 	if (!atomic_dec_and_lock_irqsave(&rpm->wakeref_count,  in __intel_wakeref_dec_and_check_tracking()
94 					 &rpm->debug.lock,  in __intel_wakeref_dec_and_check_tracking()
98 	ref_tracker_dir_print_locked(&rpm->debug, INTEL_REFTRACK_PRINT_LIMIT);  in __intel_wakeref_dec_and_check_tracking()
99 	spin_unlock_irqrestore(&rpm->debug.lock, flags);  in __intel_wakeref_dec_and_check_tracking()
102 void print_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm,  in print_intel_runtime_pm_wakeref()  argument
105 	intel_ref_tracker_show(&rpm->debug, p);  in print_intel_runtime_pm_wakeref()
110 static void init_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm)  in init_intel_runtime_pm_wakeref()  argument
115 track_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm)  in track_intel_runtime_pm_wakeref()  argument
120 static void untrack_intel_runtime_pm_wakeref(struct intel_runtime_pm *rpm,  in untrack_intel_runtime_pm_wakeref()  argument
126 __intel_wakeref_dec_and_check_tracking(struct intel_runtime_pm *rpm)  in __intel_wakeref_dec_and_check_tracking()  argument
128 	atomic_dec(&rpm->wakeref_count);  in __intel_wakeref_dec_and_check_tracking()
132 untrack_all_intel_runtime_pm_wakerefs(struct intel_runtime_pm *rpm)  in untrack_all_intel_runtime_pm_wakerefs()  argument
139 intel_runtime_pm_acquire(struct intel_runtime_pm *rpm, bool wakelock)  in intel_runtime_pm_acquire()  argument
142 		atomic_add(1 + INTEL_RPM_WAKELOCK_BIAS, &rpm->wakeref_count);  in intel_runtime_pm_acquire()
143 		assert_rpm_wakelock_held(rpm);  in intel_runtime_pm_acquire()
145 		atomic_inc(&rpm->wakeref_count);  in intel_runtime_pm_acquire()
146 		assert_rpm_raw_wakeref_held(rpm);  in intel_runtime_pm_acquire()
151 intel_runtime_pm_release(struct intel_runtime_pm *rpm, int wakelock)  in intel_runtime_pm_release()  argument
154 		assert_rpm_wakelock_held(rpm);  in intel_runtime_pm_release()
155 		atomic_sub(INTEL_RPM_WAKELOCK_BIAS, &rpm->wakeref_count);  in intel_runtime_pm_release()
157 		assert_rpm_raw_wakeref_held(rpm);  in intel_runtime_pm_release()
160 	__intel_wakeref_dec_and_check_tracking(rpm);  in intel_runtime_pm_release()
163 static intel_wakeref_t __intel_runtime_pm_get(struct intel_runtime_pm *rpm,  in __intel_runtime_pm_get()  argument
166 	struct drm_i915_private *i915 = rpm_to_i915(rpm);  in __intel_runtime_pm_get()
169 	ret = pm_runtime_get_sync(rpm->kdev);  in __intel_runtime_pm_get()
173 	intel_runtime_pm_acquire(rpm, wakelock);  in __intel_runtime_pm_get()
175 	return track_intel_runtime_pm_wakeref(rpm);  in __intel_runtime_pm_get()
180  * @rpm: the intel_runtime_pm structure
195 intel_wakeref_t intel_runtime_pm_get_raw(struct intel_runtime_pm *rpm)  in intel_runtime_pm_get_raw()  argument
197 	return __intel_runtime_pm_get(rpm, false);  in intel_runtime_pm_get_raw()
202  * @rpm: the intel_runtime_pm structure
212 intel_wakeref_t intel_runtime_pm_get(struct intel_runtime_pm *rpm)  in intel_runtime_pm_get()  argument
214 	return __intel_runtime_pm_get(rpm, true);  in intel_runtime_pm_get()
219  * @rpm: the intel_runtime_pm structure
239 static intel_wakeref_t __intel_runtime_pm_get_if_active(struct intel_runtime_pm *rpm,  in __intel_runtime_pm_get_if_active()  argument
244 		 * In cases runtime PM is disabled by the RPM core and we get  in __intel_runtime_pm_get_if_active()
249 		if (pm_runtime_get_if_active(rpm->kdev, ignore_usecount) <= 0)  in __intel_runtime_pm_get_if_active()
253 	intel_runtime_pm_acquire(rpm, true);  in __intel_runtime_pm_get_if_active()
255 	return track_intel_runtime_pm_wakeref(rpm);  in __intel_runtime_pm_get_if_active()
258 intel_wakeref_t intel_runtime_pm_get_if_in_use(struct intel_runtime_pm *rpm)  in intel_runtime_pm_get_if_in_use()  argument
260 	return __intel_runtime_pm_get_if_active(rpm, false);  in intel_runtime_pm_get_if_in_use()
263 intel_wakeref_t intel_runtime_pm_get_if_active(struct intel_runtime_pm *rpm)  in intel_runtime_pm_get_if_active()  argument
265 	return __intel_runtime_pm_get_if_active(rpm, true);  in intel_runtime_pm_get_if_active()
270  * @rpm: the intel_runtime_pm structure
287 intel_wakeref_t intel_runtime_pm_get_noresume(struct intel_runtime_pm *rpm)  in intel_runtime_pm_get_noresume()  argument
289 	assert_rpm_wakelock_held(rpm);  in intel_runtime_pm_get_noresume()
290 	pm_runtime_get_noresume(rpm->kdev);  in intel_runtime_pm_get_noresume()
292 	intel_runtime_pm_acquire(rpm, true);  in intel_runtime_pm_get_noresume()
294 	return track_intel_runtime_pm_wakeref(rpm);  in intel_runtime_pm_get_noresume()
297 static void __intel_runtime_pm_put(struct intel_runtime_pm *rpm,  in __intel_runtime_pm_put()  argument
301 	struct device *kdev = rpm->kdev;  in __intel_runtime_pm_put()
303 	untrack_intel_runtime_pm_wakeref(rpm, wref);  in __intel_runtime_pm_put()
305 	intel_runtime_pm_release(rpm, wakelock);  in __intel_runtime_pm_put()
313  * @rpm: the intel_runtime_pm structure
321 intel_runtime_pm_put_raw(struct intel_runtime_pm *rpm, intel_wakeref_t wref)  in intel_runtime_pm_put_raw()  argument
323 	__intel_runtime_pm_put(rpm, wref, false);  in intel_runtime_pm_put_raw()
328  * @rpm: the intel_runtime_pm structure
338 void intel_runtime_pm_put_unchecked(struct intel_runtime_pm *rpm)  in intel_runtime_pm_put_unchecked()  argument
340 	__intel_runtime_pm_put(rpm, -1, true);  in intel_runtime_pm_put_unchecked()
346  * @rpm: the intel_runtime_pm structure
353 void intel_runtime_pm_put(struct intel_runtime_pm *rpm, intel_wakeref_t wref)  in intel_runtime_pm_put()  argument
355 	__intel_runtime_pm_put(rpm, wref, true);  in intel_runtime_pm_put()
361  * @rpm: the intel_runtime_pm structure
369 void intel_runtime_pm_enable(struct intel_runtime_pm *rpm)  in intel_runtime_pm_enable()  argument
371 	struct drm_i915_private *i915 = rpm_to_i915(rpm);  in intel_runtime_pm_enable()
372 	struct device *kdev = rpm->kdev;  in intel_runtime_pm_enable()
388 	 * Take a permanent reference to disable the RPM functionality and drop  in intel_runtime_pm_enable()
390 	 * so the driver's own RPM reference tracking asserts also work on  in intel_runtime_pm_enable()
391 	 * platforms without RPM support.  in intel_runtime_pm_enable()
393 	if (!rpm->available) {  in intel_runtime_pm_enable()
415 	 * The core calls the driver load handler with an RPM reference held.  in intel_runtime_pm_enable()
422 void intel_runtime_pm_disable(struct intel_runtime_pm *rpm)  in intel_runtime_pm_disable()  argument
424 	struct drm_i915_private *i915 = rpm_to_i915(rpm);  in intel_runtime_pm_disable()
425 	struct device *kdev = rpm->kdev;  in intel_runtime_pm_disable()
427 	/* Transfer rpm ownership back to core */  in intel_runtime_pm_disable()
429 		 "Failed to pass rpm ownership back to core\n");  in intel_runtime_pm_disable()
433 	if (!rpm->available)  in intel_runtime_pm_disable()
437 void intel_runtime_pm_driver_release(struct intel_runtime_pm *rpm)  in intel_runtime_pm_driver_release()  argument
439 	struct drm_i915_private *i915 = rpm_to_i915(rpm);  in intel_runtime_pm_driver_release()
440 	int count = atomic_read(&rpm->wakeref_count);  in intel_runtime_pm_driver_release()
442 	intel_wakeref_auto_fini(&rpm->userfault_wakeref);  in intel_runtime_pm_driver_release()
450 void intel_runtime_pm_driver_last_release(struct intel_runtime_pm *rpm)  in intel_runtime_pm_driver_last_release()  argument
452 	intel_runtime_pm_driver_release(rpm);  in intel_runtime_pm_driver_last_release()
453 	untrack_all_intel_runtime_pm_wakerefs(rpm);  in intel_runtime_pm_driver_last_release()
456 void intel_runtime_pm_init_early(struct intel_runtime_pm *rpm)  in intel_runtime_pm_init_early()  argument
458 	struct drm_i915_private *i915 = rpm_to_i915(rpm);  in intel_runtime_pm_init_early()
462 	rpm->kdev = kdev;  in intel_runtime_pm_init_early()
463 	rpm->available = HAS_RUNTIME_PM(i915);  in intel_runtime_pm_init_early()
464 	atomic_set(&rpm->wakeref_count, 0);  in intel_runtime_pm_init_early()
466 	init_intel_runtime_pm_wakeref(rpm);  in intel_runtime_pm_init_early()
467 	INIT_LIST_HEAD(&rpm->lmem_userfault_list);  in intel_runtime_pm_init_early()
468 	spin_lock_init(&rpm->lmem_userfault_lock);  in intel_runtime_pm_init_early()
469 	intel_wakeref_auto_init(&rpm->userfault_wakeref, i915);  in intel_runtime_pm_init_early()