2  * SPDX-License-Identifier: MIT
20  * low-voltage mode when idle, using down to 0V while at this stage.  This
26  * among each other with the latency required to enter and leave RC6 and
34  * require higher latency to switch to and wake up.
44 	return rc6_to_gt(rc)->uncore;  in rc6_to_uncore()
49 	return rc6_to_gt(rc)->i915;  in rc6_to_i915()
60 	struct intel_uncore *uncore = gt->uncore;  in gen11_rc6_enable()
73 		set(uncore, RING_MAX_IDLE(engine->mmio_base), 10);  in gen11_rc6_enable()
84 	 * Bspec's guidance is to use 25us (really 25 * 1280ns) here. What we  in gen11_rc6_enable()
86 	 * it takes us to service a CS interrupt and submit a new ELSP - that  in gen11_rc6_enable()
89 	 * interrupt service latency, the hardware will automatically gate  in gen11_rc6_enable()
91 	 * the service latency. A similar guide from plane_state is that we  in gen11_rc6_enable()
92 	 * do not want the enable hysteresis to less than the wakeup latency.  in gen11_rc6_enable()
95 	 * service latency, and puts it under 10us for Icelake, similar to  in gen11_rc6_enable()
103 	rc6->ctl_enable =  in gen11_rc6_enable()
113 	if (INTEL_GEN(gt->i915) >= 12) {  in gen11_rc6_enable()
146 		set(uncore, RING_MAX_IDLE(engine->mmio_base), 10);  in gen9_rc6_enable()
155 	 * Bspec's guidance is to use 25us (really 25 * 1280ns) here. What we  in gen9_rc6_enable()
157 	 * it takes us to service a CS interrupt and submit a new ELSP - that  in gen9_rc6_enable()
160 	 * interrupt service latency, the hardware will automatically gate  in gen9_rc6_enable()
162 	 * the service latency. A similar guide from plane_state is that we  in gen9_rc6_enable()
163 	 * do not want the enable hysteresis to less than the wakeup latency.  in gen9_rc6_enable()
166 	 * service latency, and puts it around 10us for Broadwell (and other  in gen9_rc6_enable()
167 	 * big core) and around 40us for Broxton (and other low power cores).  in gen9_rc6_enable()
168 	 * [Note that for legacy ringbuffer submission, this is less than 1us!]  in gen9_rc6_enable()
169 	 * However, the wakeup latency on Broxton is closer to 100us. To be  in gen9_rc6_enable()
180 	rc6->ctl_enable =  in gen9_rc6_enable()
187 	 *   - Render/Media PG need to be disabled with RC6.  in gen9_rc6_enable()
205 		set(uncore, RING_MAX_IDLE(engine->mmio_base), 10);  in gen8_rc6_enable()
207 	set(uncore, GEN6_RC6_THRESHOLD, 625); /* 800us/1.28 for TO */  in gen8_rc6_enable()
210 	rc6->ctl_enable =  in gen8_rc6_enable()
232 		set(uncore, RING_MAX_IDLE(engine->mmio_base), 10);  in gen6_rc6_enable()
246 	rc6->ctl_enable =  in gen6_rc6_enable()
255 		drm_dbg(&i915->drm, "Couldn't check for BIOS workaround\n");  in gen6_rc6_enable()
258 		drm_dbg(&i915->drm,  in gen6_rc6_enable()
259 			"You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",  in gen6_rc6_enable()
265 			drm_err(&i915->drm,  in gen6_rc6_enable()
281 		drm_dbg(&i915->drm, "BIOS didn't set up PCBR, fixing up\n");  in chv_rc6_init()
282 		paddr = i915->dsm.end + 1 - pctx_size;  in chv_rc6_init()
303 		/* BIOS set it up already, grab the pre-alloc'd space */  in vlv_rc6_init()
306 		pcbr_offset = (pcbr & ~4095) - i915->dsm.start;  in vlv_rc6_init()
316 	drm_dbg(&i915->drm, "BIOS didn't set up PCBR, fixing up\n");  in vlv_rc6_init()
328 		drm_dbg(&i915->drm,  in vlv_rc6_init()
334 					 i915->dsm.start,  in vlv_rc6_init()
335 					 pctx->stolen->start,  in vlv_rc6_init()
337 	pctx_paddr = i915->dsm.start + pctx->stolen->start;  in vlv_rc6_init()
341 	rc6->pctx = pctx;  in vlv_rc6_init()
357 		set(uncore, RING_MAX_IDLE(engine->mmio_base), 10);  in chv_rc6_enable()
360 	/* TO threshold set to 500 us (0x186 * 1.28 us) */  in chv_rc6_enable()
370 	rc6->ctl_enable = GEN7_RC_CTL_TO_MODE;  in chv_rc6_enable()
384 		set(uncore, RING_MAX_IDLE(engine->mmio_base), 10);  in vlv_rc6_enable()
396 	rc6->ctl_enable =  in vlv_rc6_enable()
411 	drm_dbg(&i915->drm, "BIOS enabled RC states: "  in bxt_check_bios_rc6_setup()
418 		drm_dbg(&i915->drm, "RC6 Base location not set properly.\n");  in bxt_check_bios_rc6_setup()
428 	if (!(rc6_ctx_base >= i915->dsm_reserved.start &&  in bxt_check_bios_rc6_setup()
429 	      rc6_ctx_base + PAGE_SIZE < i915->dsm_reserved.end)) {  in bxt_check_bios_rc6_setup()
430 		drm_dbg(&i915->drm, "RC6 Base address not as expected.\n");  in bxt_check_bios_rc6_setup()
438 		drm_dbg(&i915->drm,  in bxt_check_bios_rc6_setup()
446 		drm_dbg(&i915->drm, "Pushbus not setup properly.\n");  in bxt_check_bios_rc6_setup()
451 		drm_dbg(&i915->drm, "GFX pause not setup properly.\n");  in bxt_check_bios_rc6_setup()
456 		drm_dbg(&i915->drm, "GPM control not setup properly.\n");  in bxt_check_bios_rc6_setup()
477 		drm_notice(&i915->drm,  in rc6_supported()
487 	GEM_BUG_ON(rc6->wakeref);  in rpm_get()
488 	pm_runtime_get_sync(&rc6_to_i915(rc6)->drm.pdev->dev);  in rpm_get()
489 	rc6->wakeref = true;  in rpm_get()
494 	GEM_BUG_ON(!rc6->wakeref);  in rpm_put()
495 	pm_runtime_put(&rc6_to_i915(rc6)->drm.pdev->dev);  in rpm_put()
496 	rc6->wakeref = false;  in rpm_put()
509 	drm_notice(&i915->drm,  in pctx_corrupted()
532 	/* Disable runtime-pm until we can save the GPU state with rc6 pctx */  in intel_rc6_init()
548 	rc6->supported = err == 0;  in intel_rc6_init()
553 	memset(rc6->prev_hw_residency, 0, sizeof(rc6->prev_hw_residency));  in intel_rc6_sanitize()
555 	if (rc6->enabled) { /* unbalanced suspend/resume */  in intel_rc6_sanitize()
557 		rc6->enabled = false;  in intel_rc6_sanitize()
560 	if (rc6->supported)  in intel_rc6_sanitize()
569 	if (!rc6->supported)  in intel_rc6_enable()
572 	GEM_BUG_ON(rc6->enabled);  in intel_rc6_enable()
589 	rc6->manual = rc6->ctl_enable & GEN6_RC_CTL_RC6_ENABLE;  in intel_rc6_enable()
591 		rc6->ctl_enable = 0;  in intel_rc6_enable()
598 	/* rc6 is ready, runtime-pm is go! */  in intel_rc6_enable()
600 	rc6->enabled = true;  in intel_rc6_enable()
607 	if (!rc6->enabled)  in intel_rc6_unpark()
610 	/* Restore HW timers for automatic RC6 entry while busy */  in intel_rc6_unpark()
611 	set(uncore, GEN6_RC_CONTROL, rc6->ctl_enable);  in intel_rc6_unpark()
619 	if (!rc6->enabled)  in intel_rc6_park()
627 	if (!rc6->manual)  in intel_rc6_park()
644 	if (!rc6->enabled)  in intel_rc6_disable()
648 	rc6->enabled = false;  in intel_rc6_disable()
659 	pctx = fetch_and_zero(&rc6->pctx);  in intel_rc6_fini()
663 	if (rc6->wakeref)  in intel_rc6_fini()
676 	lockdep_assert_held(&uncore->lock);  in vlv_residency_raw()
683 	 * Although we always use the counter in high-range mode elsewhere,  in vlv_residency_raw()
701 	} while (upper != tmp && --loop);  in vlv_residency_raw()
705 	 * VLV_COUNT_RANGE_HIGH bit set - so it is safe to leave it set  in vlv_residency_raw()
722 	if (!rc6->supported)  in intel_rc6_residency_ns()
726 	 * Store previous hw counter values for counter wrap-around handling.  in intel_rc6_residency_ns()
732 	i = (i915_mmio_reg_offset(reg) -  in intel_rc6_residency_ns()
734 	if (drm_WARN_ON_ONCE(&i915->drm, i >= ARRAY_SIZE(rc6->cur_residency)))  in intel_rc6_residency_ns()
739 	spin_lock_irqsave(&uncore->lock, flags);  in intel_rc6_residency_ns()
742 	/* On VLV and CHV, residency time is in CZ units rather than 1.28us */  in intel_rc6_residency_ns()
745 		div = i915->czclk_freq;  in intel_rc6_residency_ns()
749 		/* 833.33ns units on Gen9LP, 1.28us elsewhere. */  in intel_rc6_residency_ns()
768 	prev_hw = rc6->prev_hw_residency[i];  in intel_rc6_residency_ns()
769 	rc6->prev_hw_residency[i] = time_hw;  in intel_rc6_residency_ns()
773 		time_hw -= prev_hw;  in intel_rc6_residency_ns()
775 		time_hw += overflow_hw - prev_hw;  in intel_rc6_residency_ns()
778 	time_hw += rc6->cur_residency[i];  in intel_rc6_residency_ns()
779 	rc6->cur_residency[i] = time_hw;  in intel_rc6_residency_ns()
782 	spin_unlock_irqrestore(&uncore->lock, flags);  in intel_rc6_residency_ns()