1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2019 Intel Corporation
4 */
5
6 #include <linux/pm_runtime.h>
7 #include <linux/string_helpers.h>
8
9 #include <drm/drm_print.h>
10
11 #include "display/vlv_clock.h"
12 #include "gem/i915_gem_region.h"
13 #include "i915_drv.h"
14 #include "i915_reg.h"
15 #include "i915_vgpu.h"
16 #include "intel_engine_regs.h"
17 #include "intel_gt.h"
18 #include "intel_gt_pm.h"
19 #include "intel_gt_regs.h"
20 #include "intel_pcode.h"
21 #include "intel_rc6.h"
22
23 /**
24 * DOC: RC6
25 *
26 * RC6 is a special power stage which allows the GPU to enter an very
27 * low-voltage mode when idle, using down to 0V while at this stage. This
28 * stage is entered automatically when the GPU is idle when RC6 support is
29 * enabled, and as soon as new workload arises GPU wakes up automatically as
30 * well.
31 *
32 * There are different RC6 modes available in Intel GPU, which differentiate
33 * among each other with the latency required to enter and leave RC6 and
34 * voltage consumed by the GPU in different states.
35 *
36 * The combination of the following flags define which states GPU is allowed
37 * to enter, while RC6 is the normal RC6 state, RC6p is the deep RC6, and
38 * RC6pp is deepest RC6. Their support by hardware varies according to the
39 * GPU, BIOS, chipset and platform. RC6 is usually the safest one and the one
40 * which brings the most power savings; deeper states save more power, but
41 * require higher latency to switch to and wake up.
42 */
43
rc6_to_gt(struct intel_rc6 * rc6)44 static struct intel_gt *rc6_to_gt(struct intel_rc6 *rc6)
45 {
46 return container_of(rc6, struct intel_gt, rc6);
47 }
48
rc6_to_uncore(struct intel_rc6 * rc)49 static struct intel_uncore *rc6_to_uncore(struct intel_rc6 *rc)
50 {
51 return rc6_to_gt(rc)->uncore;
52 }
53
rc6_to_i915(struct intel_rc6 * rc)54 static struct drm_i915_private *rc6_to_i915(struct intel_rc6 *rc)
55 {
56 return rc6_to_gt(rc)->i915;
57 }
58
gen11_rc6_enable(struct intel_rc6 * rc6)59 static void gen11_rc6_enable(struct intel_rc6 *rc6)
60 {
61 struct intel_gt *gt = rc6_to_gt(rc6);
62 struct intel_uncore *uncore = gt->uncore;
63 struct intel_engine_cs *engine;
64 enum intel_engine_id id;
65 u32 pg_enable;
66 int i;
67
68 /*
69 * With GuCRC, these parameters are set by GuC
70 */
71 if (!intel_uc_uses_guc_rc(>->uc)) {
72 /* 2b: Program RC6 thresholds.*/
73 intel_uncore_write_fw(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16 | 85);
74 intel_uncore_write_fw(uncore, GEN10_MEDIA_WAKE_RATE_LIMIT, 150);
75
76 intel_uncore_write_fw(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
77 intel_uncore_write_fw(uncore, GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
78 for_each_engine(engine, rc6_to_gt(rc6), id)
79 intel_uncore_write_fw(uncore, RING_MAX_IDLE(engine->mmio_base), 10);
80
81 intel_uncore_write_fw(uncore, GUC_MAX_IDLE_COUNT, 0xA);
82
83 intel_uncore_write_fw(uncore, GEN6_RC_SLEEP, 0);
84
85 intel_uncore_write_fw(uncore, GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */
86 }
87
88 /*
89 * 2c: Program Coarse Power Gating Policies.
90 *
91 * Bspec's guidance is to use 25us (really 25 * 1280ns) here. What we
92 * use instead is a more conservative estimate for the maximum time
93 * it takes us to service a CS interrupt and submit a new ELSP - that
94 * is the time which the GPU is idle waiting for the CPU to select the
95 * next request to execute. If the idle hysteresis is less than that
96 * interrupt service latency, the hardware will automatically gate
97 * the power well and we will then incur the wake up cost on top of
98 * the service latency. A similar guide from plane_state is that we
99 * do not want the enable hysteresis to less than the wakeup latency.
100 *
101 * igt/gem_exec_nop/sequential provides a rough estimate for the
102 * service latency, and puts it under 10us for Icelake, similar to
103 * Broadwell+, To be conservative, we want to factor in a context
104 * switch on top (due to ksoftirqd).
105 */
106 intel_uncore_write_fw(uncore, GEN9_MEDIA_PG_IDLE_HYSTERESIS, 60);
107 intel_uncore_write_fw(uncore, GEN9_RENDER_PG_IDLE_HYSTERESIS, 60);
108
109 /* 3a: Enable RC6
110 *
111 * With GuCRC, we do not enable bit 31 of RC_CTL,
112 * thus allowing GuC to control RC6 entry/exit fully instead.
113 * We will not set the HW ENABLE and EI bits
114 */
115 if (!intel_guc_rc_enable(gt_to_guc(gt)))
116 rc6->ctl_enable = GEN6_RC_CTL_RC6_ENABLE;
117 else
118 rc6->ctl_enable =
119 GEN6_RC_CTL_HW_ENABLE |
120 GEN6_RC_CTL_RC6_ENABLE |
121 GEN6_RC_CTL_EI_MODE(1);
122
123 pg_enable =
124 GEN9_RENDER_PG_ENABLE |
125 GEN9_MEDIA_PG_ENABLE |
126 GEN11_MEDIA_SAMPLER_PG_ENABLE;
127
128 if (GRAPHICS_VER(gt->i915) >= 12 && !IS_DG1(gt->i915)) {
129 for (i = 0; i < I915_MAX_VCS; i++)
130 if (HAS_ENGINE(gt, _VCS(i)))
131 pg_enable |= (VDN_HCP_POWERGATE_ENABLE(i) |
132 VDN_MFX_POWERGATE_ENABLE(i));
133 }
134
135 intel_uncore_write_fw(uncore, GEN9_PG_ENABLE, pg_enable);
136 }
137
gen9_rc6_enable(struct intel_rc6 * rc6)138 static void gen9_rc6_enable(struct intel_rc6 *rc6)
139 {
140 struct intel_uncore *uncore = rc6_to_uncore(rc6);
141 struct intel_engine_cs *engine;
142 enum intel_engine_id id;
143
144 /* 2b: Program RC6 thresholds.*/
145 if (GRAPHICS_VER(rc6_to_i915(rc6)) >= 11) {
146 intel_uncore_write_fw(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16 | 85);
147 intel_uncore_write_fw(uncore, GEN10_MEDIA_WAKE_RATE_LIMIT, 150);
148 } else if (IS_SKYLAKE(rc6_to_i915(rc6))) {
149 /*
150 * WaRsDoubleRc6WrlWithCoarsePowerGating:skl Doubling WRL only
151 * when CPG is enabled
152 */
153 intel_uncore_write_fw(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 108 << 16);
154 } else {
155 intel_uncore_write_fw(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16);
156 }
157
158 intel_uncore_write_fw(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
159 intel_uncore_write_fw(uncore, GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
160 for_each_engine(engine, rc6_to_gt(rc6), id)
161 intel_uncore_write_fw(uncore, RING_MAX_IDLE(engine->mmio_base), 10);
162
163 intel_uncore_write_fw(uncore, GUC_MAX_IDLE_COUNT, 0xA);
164
165 intel_uncore_write_fw(uncore, GEN6_RC_SLEEP, 0);
166
167 /*
168 * 2c: Program Coarse Power Gating Policies.
169 *
170 * Bspec's guidance is to use 25us (really 25 * 1280ns) here. What we
171 * use instead is a more conservative estimate for the maximum time
172 * it takes us to service a CS interrupt and submit a new ELSP - that
173 * is the time which the GPU is idle waiting for the CPU to select the
174 * next request to execute. If the idle hysteresis is less than that
175 * interrupt service latency, the hardware will automatically gate
176 * the power well and we will then incur the wake up cost on top of
177 * the service latency. A similar guide from plane_state is that we
178 * do not want the enable hysteresis to less than the wakeup latency.
179 *
180 * igt/gem_exec_nop/sequential provides a rough estimate for the
181 * service latency, and puts it around 10us for Broadwell (and other
182 * big core) and around 40us for Broxton (and other low power cores).
183 * [Note that for legacy ringbuffer submission, this is less than 1us!]
184 * However, the wakeup latency on Broxton is closer to 100us. To be
185 * conservative, we have to factor in a context switch on top (due
186 * to ksoftirqd).
187 */
188 intel_uncore_write_fw(uncore, GEN9_MEDIA_PG_IDLE_HYSTERESIS, 250);
189 intel_uncore_write_fw(uncore, GEN9_RENDER_PG_IDLE_HYSTERESIS, 250);
190
191 /* 3a: Enable RC6 */
192 intel_uncore_write_fw(uncore, GEN6_RC6_THRESHOLD, 37500); /* 37.5/125ms per EI */
193
194 rc6->ctl_enable =
195 GEN6_RC_CTL_HW_ENABLE |
196 GEN6_RC_CTL_RC6_ENABLE |
197 GEN6_RC_CTL_EI_MODE(1);
198
199 /*
200 * WaRsDisableCoarsePowerGating:skl,cnl
201 * - Render/Media PG need to be disabled with RC6.
202 */
203 if (!NEEDS_WaRsDisableCoarsePowerGating(rc6_to_i915(rc6)))
204 intel_uncore_write_fw(uncore, GEN9_PG_ENABLE,
205 GEN9_RENDER_PG_ENABLE | GEN9_MEDIA_PG_ENABLE);
206 }
207
gen8_rc6_enable(struct intel_rc6 * rc6)208 static void gen8_rc6_enable(struct intel_rc6 *rc6)
209 {
210 struct intel_uncore *uncore = rc6_to_uncore(rc6);
211 struct intel_engine_cs *engine;
212 enum intel_engine_id id;
213
214 /* 2b: Program RC6 thresholds.*/
215 intel_uncore_write_fw(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
216 intel_uncore_write_fw(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
217 intel_uncore_write_fw(uncore, GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
218 for_each_engine(engine, rc6_to_gt(rc6), id)
219 intel_uncore_write_fw(uncore, RING_MAX_IDLE(engine->mmio_base), 10);
220 intel_uncore_write_fw(uncore, GEN6_RC_SLEEP, 0);
221 intel_uncore_write_fw(uncore, GEN6_RC6_THRESHOLD, 625); /* 800us/1.28 for TO */
222
223 /* 3: Enable RC6 */
224 rc6->ctl_enable =
225 GEN6_RC_CTL_HW_ENABLE |
226 GEN7_RC_CTL_TO_MODE |
227 GEN6_RC_CTL_RC6_ENABLE;
228 }
229
gen6_rc6_enable(struct intel_rc6 * rc6)230 static void gen6_rc6_enable(struct intel_rc6 *rc6)
231 {
232 struct intel_uncore *uncore = rc6_to_uncore(rc6);
233 struct drm_i915_private *i915 = rc6_to_i915(rc6);
234 struct intel_engine_cs *engine;
235 enum intel_engine_id id;
236 u32 rc6vids, rc6_mask;
237 int ret;
238
239 intel_uncore_write_fw(uncore, GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
240 intel_uncore_write_fw(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 | 30);
241 intel_uncore_write_fw(uncore, GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
242 intel_uncore_write_fw(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000);
243 intel_uncore_write_fw(uncore, GEN6_RC_IDLE_HYSTERSIS, 25);
244
245 for_each_engine(engine, rc6_to_gt(rc6), id)
246 intel_uncore_write_fw(uncore, RING_MAX_IDLE(engine->mmio_base), 10);
247
248 intel_uncore_write_fw(uncore, GEN6_RC_SLEEP, 0);
249 intel_uncore_write_fw(uncore, GEN6_RC1e_THRESHOLD, 1000);
250 intel_uncore_write_fw(uncore, GEN6_RC6_THRESHOLD, 50000);
251 intel_uncore_write_fw(uncore, GEN6_RC6p_THRESHOLD, 150000);
252 intel_uncore_write_fw(uncore, GEN6_RC6pp_THRESHOLD, 64000); /* unused */
253
254 /* We don't use those on Haswell */
255 rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
256 if (HAS_RC6p(i915))
257 rc6_mask |= GEN6_RC_CTL_RC6p_ENABLE;
258 if (HAS_RC6pp(i915))
259 rc6_mask |= GEN6_RC_CTL_RC6pp_ENABLE;
260 rc6->ctl_enable =
261 rc6_mask |
262 GEN6_RC_CTL_EI_MODE(1) |
263 GEN6_RC_CTL_HW_ENABLE;
264
265 rc6vids = 0;
266 ret = snb_pcode_read(rc6_to_gt(rc6)->uncore, GEN6_PCODE_READ_RC6VIDS, &rc6vids, NULL);
267 if (GRAPHICS_VER(i915) == 6 && ret) {
268 drm_dbg(&i915->drm, "Couldn't check for BIOS workaround\n");
269 } else if (GRAPHICS_VER(i915) == 6 &&
270 (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
271 drm_dbg(&i915->drm,
272 "You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
273 GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
274 rc6vids &= 0xffff00;
275 rc6vids |= GEN6_ENCODE_RC6_VID(450);
276 ret = snb_pcode_write(rc6_to_gt(rc6)->uncore, GEN6_PCODE_WRITE_RC6VIDS, rc6vids);
277 if (ret)
278 drm_err(&i915->drm,
279 "Couldn't fix incorrect rc6 voltage\n");
280 }
281 }
282
283 /* Check that the pcbr address is not empty. */
chv_rc6_init(struct intel_rc6 * rc6)284 static int chv_rc6_init(struct intel_rc6 *rc6)
285 {
286 struct intel_uncore *uncore = rc6_to_uncore(rc6);
287 struct drm_i915_private *i915 = rc6_to_i915(rc6);
288 resource_size_t pctx_paddr, paddr;
289 resource_size_t pctx_size = 32 * SZ_1K;
290 u32 pcbr;
291
292 pcbr = intel_uncore_read(uncore, VLV_PCBR);
293 if ((pcbr >> VLV_PCBR_ADDR_SHIFT) == 0) {
294 drm_dbg(&i915->drm, "BIOS didn't set up PCBR, fixing up\n");
295 paddr = i915->dsm.stolen.end + 1 - pctx_size;
296 GEM_BUG_ON(paddr > U32_MAX);
297
298 pctx_paddr = (paddr & ~4095);
299 intel_uncore_write(uncore, VLV_PCBR, pctx_paddr);
300 }
301
302 return 0;
303 }
304
vlv_rc6_init(struct intel_rc6 * rc6)305 static int vlv_rc6_init(struct intel_rc6 *rc6)
306 {
307 struct drm_i915_private *i915 = rc6_to_i915(rc6);
308 struct intel_uncore *uncore = rc6_to_uncore(rc6);
309 struct drm_i915_gem_object *pctx;
310 resource_size_t pctx_paddr;
311 resource_size_t pctx_size = 24 * SZ_1K;
312 u32 pcbr;
313
314 pcbr = intel_uncore_read(uncore, VLV_PCBR);
315 if (pcbr) {
316 /* BIOS set it up already, grab the pre-alloc'd space */
317 resource_size_t pcbr_offset;
318
319 pcbr_offset = (pcbr & ~4095) - i915->dsm.stolen.start;
320 pctx = i915_gem_object_create_region_at(i915->mm.stolen_region,
321 pcbr_offset,
322 pctx_size,
323 0);
324 if (IS_ERR(pctx))
325 return PTR_ERR(pctx);
326
327 goto out;
328 }
329
330 drm_dbg(&i915->drm, "BIOS didn't set up PCBR, fixing up\n");
331
332 /*
333 * From the Gunit register HAS:
334 * The Gfx driver is expected to program this register and ensure
335 * proper allocation within Gfx stolen memory. For example, this
336 * register should be programmed such than the PCBR range does not
337 * overlap with other ranges, such as the frame buffer, protected
338 * memory, or any other relevant ranges.
339 */
340 pctx = i915_gem_object_create_stolen(i915, pctx_size);
341 if (IS_ERR(pctx)) {
342 drm_dbg(&i915->drm,
343 "not enough stolen space for PCTX, disabling\n");
344 return PTR_ERR(pctx);
345 }
346
347 GEM_BUG_ON(range_end_overflows_t(u64,
348 i915->dsm.stolen.start,
349 pctx->stolen->start,
350 U32_MAX));
351 pctx_paddr = i915->dsm.stolen.start + pctx->stolen->start;
352 intel_uncore_write(uncore, VLV_PCBR, pctx_paddr);
353
354 out:
355 rc6->pctx = pctx;
356 return 0;
357 }
358
chv_rc6_enable(struct intel_rc6 * rc6)359 static void chv_rc6_enable(struct intel_rc6 *rc6)
360 {
361 struct intel_uncore *uncore = rc6_to_uncore(rc6);
362 struct intel_engine_cs *engine;
363 enum intel_engine_id id;
364
365 /* 2a: Program RC6 thresholds.*/
366 intel_uncore_write_fw(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
367 intel_uncore_write_fw(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
368 intel_uncore_write_fw(uncore, GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
369
370 for_each_engine(engine, rc6_to_gt(rc6), id)
371 intel_uncore_write_fw(uncore, RING_MAX_IDLE(engine->mmio_base), 10);
372 intel_uncore_write_fw(uncore, GEN6_RC_SLEEP, 0);
373
374 /* TO threshold set to 500 us (0x186 * 1.28 us) */
375 intel_uncore_write_fw(uncore, GEN6_RC6_THRESHOLD, 0x186);
376
377 /* Allows RC6 residency counter to work */
378 intel_uncore_write_fw(uncore, VLV_COUNTER_CONTROL,
379 _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH |
380 VLV_MEDIA_RC6_COUNT_EN |
381 VLV_RENDER_RC6_COUNT_EN));
382
383 /* 3: Enable RC6 */
384 rc6->ctl_enable = GEN7_RC_CTL_TO_MODE;
385 }
386
vlv_rc6_enable(struct intel_rc6 * rc6)387 static void vlv_rc6_enable(struct intel_rc6 *rc6)
388 {
389 struct intel_uncore *uncore = rc6_to_uncore(rc6);
390 struct intel_engine_cs *engine;
391 enum intel_engine_id id;
392
393 intel_uncore_write_fw(uncore, GEN6_RC6_WAKE_RATE_LIMIT, 0x00280000);
394 intel_uncore_write_fw(uncore, GEN6_RC_EVALUATION_INTERVAL, 125000);
395 intel_uncore_write_fw(uncore, GEN6_RC_IDLE_HYSTERSIS, 25);
396
397 for_each_engine(engine, rc6_to_gt(rc6), id)
398 intel_uncore_write_fw(uncore, RING_MAX_IDLE(engine->mmio_base), 10);
399
400 intel_uncore_write_fw(uncore, GEN6_RC6_THRESHOLD, 0x557);
401
402 /* Allows RC6 residency counter to work */
403 intel_uncore_write_fw(uncore, VLV_COUNTER_CONTROL,
404 _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH |
405 VLV_MEDIA_RC0_COUNT_EN |
406 VLV_RENDER_RC0_COUNT_EN |
407 VLV_MEDIA_RC6_COUNT_EN |
408 VLV_RENDER_RC6_COUNT_EN));
409
410 rc6->ctl_enable =
411 GEN7_RC_CTL_TO_MODE | VLV_RC_CTL_CTX_RST_PARALLEL;
412 }
413
intel_check_bios_c6_setup(struct intel_rc6 * rc6)414 bool intel_check_bios_c6_setup(struct intel_rc6 *rc6)
415 {
416 if (!rc6->bios_state_captured) {
417 struct intel_uncore *uncore = rc6_to_uncore(rc6);
418 intel_wakeref_t wakeref;
419
420 with_intel_runtime_pm(uncore->rpm, wakeref)
421 rc6->bios_rc_state = intel_uncore_read(uncore, GEN6_RC_STATE);
422
423 rc6->bios_state_captured = true;
424 }
425
426 return rc6->bios_rc_state & RC_SW_TARGET_STATE_MASK;
427 }
428
bxt_check_bios_rc6_setup(struct intel_rc6 * rc6)429 static bool bxt_check_bios_rc6_setup(struct intel_rc6 *rc6)
430 {
431 struct intel_uncore *uncore = rc6_to_uncore(rc6);
432 struct drm_i915_private *i915 = rc6_to_i915(rc6);
433 u32 rc6_ctx_base, rc_ctl, rc_sw_target;
434 bool enable_rc6 = true;
435
436 rc_ctl = intel_uncore_read(uncore, GEN6_RC_CONTROL);
437 rc_sw_target = intel_uncore_read(uncore, GEN6_RC_STATE);
438 rc_sw_target &= RC_SW_TARGET_STATE_MASK;
439 rc_sw_target >>= RC_SW_TARGET_STATE_SHIFT;
440 drm_dbg(&i915->drm, "BIOS enabled RC states: "
441 "HW_CTRL %s HW_RC6 %s SW_TARGET_STATE %x\n",
442 str_on_off(rc_ctl & GEN6_RC_CTL_HW_ENABLE),
443 str_on_off(rc_ctl & GEN6_RC_CTL_RC6_ENABLE),
444 rc_sw_target);
445
446 if (!(intel_uncore_read(uncore, RC6_LOCATION) & RC6_CTX_IN_DRAM)) {
447 drm_dbg(&i915->drm, "RC6 Base location not set properly.\n");
448 enable_rc6 = false;
449 }
450
451 /*
452 * The exact context size is not known for BXT, so assume a page size
453 * for this check.
454 */
455 rc6_ctx_base =
456 intel_uncore_read(uncore, RC6_CTX_BASE) & RC6_CTX_BASE_MASK;
457 if (!(rc6_ctx_base >= i915->dsm.reserved.start &&
458 rc6_ctx_base + PAGE_SIZE < i915->dsm.reserved.end)) {
459 drm_dbg(&i915->drm, "RC6 Base address not as expected.\n");
460 enable_rc6 = false;
461 }
462
463 if (!((intel_uncore_read(uncore, PWRCTX_MAXCNT(RENDER_RING_BASE)) & IDLE_TIME_MASK) > 1 &&
464 (intel_uncore_read(uncore, PWRCTX_MAXCNT(GEN6_BSD_RING_BASE)) & IDLE_TIME_MASK) > 1 &&
465 (intel_uncore_read(uncore, PWRCTX_MAXCNT(BLT_RING_BASE)) & IDLE_TIME_MASK) > 1 &&
466 (intel_uncore_read(uncore, PWRCTX_MAXCNT(VEBOX_RING_BASE)) & IDLE_TIME_MASK) > 1)) {
467 drm_dbg(&i915->drm,
468 "Engine Idle wait time not set properly.\n");
469 enable_rc6 = false;
470 }
471
472 if (!intel_uncore_read(uncore, GEN8_PUSHBUS_CONTROL) ||
473 !intel_uncore_read(uncore, GEN8_PUSHBUS_ENABLE) ||
474 !intel_uncore_read(uncore, GEN8_PUSHBUS_SHIFT)) {
475 drm_dbg(&i915->drm, "Pushbus not setup properly.\n");
476 enable_rc6 = false;
477 }
478
479 if (!intel_uncore_read(uncore, GEN6_GFXPAUSE)) {
480 drm_dbg(&i915->drm, "GFX pause not setup properly.\n");
481 enable_rc6 = false;
482 }
483
484 if (!intel_uncore_read(uncore, GEN8_MISC_CTRL0)) {
485 drm_dbg(&i915->drm, "GPM control not setup properly.\n");
486 enable_rc6 = false;
487 }
488
489 return enable_rc6;
490 }
491
rc6_supported(struct intel_rc6 * rc6)492 static bool rc6_supported(struct intel_rc6 *rc6)
493 {
494 struct drm_i915_private *i915 = rc6_to_i915(rc6);
495 struct intel_gt *gt = rc6_to_gt(rc6);
496
497 if (!HAS_RC6(i915))
498 return false;
499
500 if (intel_vgpu_active(i915))
501 return false;
502
503 if (is_mock_gt(rc6_to_gt(rc6)))
504 return false;
505
506 if (IS_GEN9_LP(i915) && !bxt_check_bios_rc6_setup(rc6)) {
507 drm_notice(&i915->drm,
508 "RC6 and powersaving disabled by BIOS\n");
509 return false;
510 }
511
512 if (IS_METEORLAKE(gt->i915) &&
513 !intel_check_bios_c6_setup(rc6)) {
514 drm_notice(&i915->drm,
515 "C6 disabled by BIOS\n");
516 return false;
517 }
518
519 if (IS_MEDIA_GT_IP_STEP(gt, IP_VER(13, 0), STEP_A0, STEP_B0)) {
520 drm_notice(&i915->drm,
521 "Media RC6 disabled on A step\n");
522 return false;
523 }
524
525 return true;
526 }
527
rpm_get(struct intel_rc6 * rc6)528 static void rpm_get(struct intel_rc6 *rc6)
529 {
530 GEM_BUG_ON(rc6->wakeref);
531 pm_runtime_get_sync(rc6_to_i915(rc6)->drm.dev);
532 rc6->wakeref = true;
533 }
534
rpm_put(struct intel_rc6 * rc6)535 static void rpm_put(struct intel_rc6 *rc6)
536 {
537 GEM_BUG_ON(!rc6->wakeref);
538 pm_runtime_put(rc6_to_i915(rc6)->drm.dev);
539 rc6->wakeref = false;
540 }
541
pctx_corrupted(struct intel_rc6 * rc6)542 static bool pctx_corrupted(struct intel_rc6 *rc6)
543 {
544 struct drm_i915_private *i915 = rc6_to_i915(rc6);
545
546 if (!NEEDS_RC6_CTX_CORRUPTION_WA(i915))
547 return false;
548
549 if (intel_uncore_read(rc6_to_uncore(rc6), GEN8_RC6_CTX_INFO))
550 return false;
551
552 drm_notice(&i915->drm,
553 "RC6 context corruption, disabling runtime power management\n");
554 return true;
555 }
556
__intel_rc6_disable(struct intel_rc6 * rc6)557 static void __intel_rc6_disable(struct intel_rc6 *rc6)
558 {
559 struct drm_i915_private *i915 = rc6_to_i915(rc6);
560 struct intel_uncore *uncore = rc6_to_uncore(rc6);
561 struct intel_gt *gt = rc6_to_gt(rc6);
562
563 /* Take control of RC6 back from GuC */
564 intel_guc_rc_disable(gt_to_guc(gt));
565
566 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
567 if (GRAPHICS_VER(i915) >= 9)
568 intel_uncore_write_fw(uncore, GEN9_PG_ENABLE, 0);
569 intel_uncore_write_fw(uncore, GEN6_RC_CONTROL, 0);
570 intel_uncore_write_fw(uncore, GEN6_RC_STATE, 0);
571 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
572 }
573
rc6_res_reg_init(struct intel_rc6 * rc6)574 static void rc6_res_reg_init(struct intel_rc6 *rc6)
575 {
576 i915_reg_t res_reg[INTEL_RC6_RES_MAX] = {
577 [0 ... INTEL_RC6_RES_MAX - 1] = INVALID_MMIO_REG,
578 };
579
580 switch (rc6_to_gt(rc6)->type) {
581 case GT_MEDIA:
582 res_reg[INTEL_RC6_RES_RC6] = MTL_MEDIA_MC6;
583 break;
584 default:
585 res_reg[INTEL_RC6_RES_RC6_LOCKED] = GEN6_GT_GFX_RC6_LOCKED;
586 res_reg[INTEL_RC6_RES_RC6] = GEN6_GT_GFX_RC6;
587 res_reg[INTEL_RC6_RES_RC6p] = GEN6_GT_GFX_RC6p;
588 res_reg[INTEL_RC6_RES_RC6pp] = GEN6_GT_GFX_RC6pp;
589 break;
590 }
591
592 memcpy(rc6->res_reg, res_reg, sizeof(res_reg));
593 }
594
intel_rc6_init(struct intel_rc6 * rc6)595 void intel_rc6_init(struct intel_rc6 *rc6)
596 {
597 struct drm_i915_private *i915 = rc6_to_i915(rc6);
598 int err;
599
600 /* Disable runtime-pm until we can save the GPU state with rc6 pctx */
601 rpm_get(rc6);
602
603 if (!rc6_supported(rc6))
604 return;
605
606 rc6_res_reg_init(rc6);
607
608 if (IS_CHERRYVIEW(i915))
609 err = chv_rc6_init(rc6);
610 else if (IS_VALLEYVIEW(i915))
611 err = vlv_rc6_init(rc6);
612 else
613 err = 0;
614
615 /* Sanitize rc6, ensure it is disabled before we are ready. */
616 __intel_rc6_disable(rc6);
617
618 rc6->supported = err == 0;
619 }
620
intel_rc6_sanitize(struct intel_rc6 * rc6)621 void intel_rc6_sanitize(struct intel_rc6 *rc6)
622 {
623 memset(rc6->prev_hw_residency, 0, sizeof(rc6->prev_hw_residency));
624
625 if (rc6->enabled) { /* unbalanced suspend/resume */
626 rpm_get(rc6);
627 rc6->enabled = false;
628 }
629
630 if (rc6->supported)
631 __intel_rc6_disable(rc6);
632 }
633
intel_rc6_enable(struct intel_rc6 * rc6)634 void intel_rc6_enable(struct intel_rc6 *rc6)
635 {
636 struct drm_i915_private *i915 = rc6_to_i915(rc6);
637 struct intel_uncore *uncore = rc6_to_uncore(rc6);
638
639 if (!rc6->supported)
640 return;
641
642 GEM_BUG_ON(rc6->enabled);
643
644 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
645
646 if (IS_CHERRYVIEW(i915))
647 chv_rc6_enable(rc6);
648 else if (IS_VALLEYVIEW(i915))
649 vlv_rc6_enable(rc6);
650 else if (GRAPHICS_VER(i915) >= 11)
651 gen11_rc6_enable(rc6);
652 else if (GRAPHICS_VER(i915) >= 9)
653 gen9_rc6_enable(rc6);
654 else if (IS_BROADWELL(i915))
655 gen8_rc6_enable(rc6);
656 else if (GRAPHICS_VER(i915) >= 6)
657 gen6_rc6_enable(rc6);
658
659 rc6->manual = rc6->ctl_enable & GEN6_RC_CTL_RC6_ENABLE;
660 if (NEEDS_RC6_CTX_CORRUPTION_WA(i915))
661 rc6->ctl_enable = 0;
662
663 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
664
665 if (unlikely(pctx_corrupted(rc6)))
666 return;
667
668 /* rc6 is ready, runtime-pm is go! */
669 rpm_put(rc6);
670 rc6->enabled = true;
671 }
672
intel_rc6_unpark(struct intel_rc6 * rc6)673 void intel_rc6_unpark(struct intel_rc6 *rc6)
674 {
675 struct intel_uncore *uncore = rc6_to_uncore(rc6);
676
677 if (!rc6->enabled)
678 return;
679
680 /* Restore HW timers for automatic RC6 entry while busy */
681 intel_uncore_write_fw(uncore, GEN6_RC_CONTROL, rc6->ctl_enable);
682 }
683
intel_rc6_park(struct intel_rc6 * rc6)684 void intel_rc6_park(struct intel_rc6 *rc6)
685 {
686 struct intel_uncore *uncore = rc6_to_uncore(rc6);
687 unsigned int target;
688
689 if (!rc6->enabled)
690 return;
691
692 if (unlikely(pctx_corrupted(rc6))) {
693 intel_rc6_disable(rc6);
694 return;
695 }
696
697 if (!rc6->manual)
698 return;
699
700 /* Turn off the HW timers and go directly to rc6 */
701 intel_uncore_write_fw(uncore, GEN6_RC_CONTROL, GEN6_RC_CTL_RC6_ENABLE);
702
703 if (HAS_RC6pp(rc6_to_i915(rc6)))
704 target = 0x6; /* deepest rc6 */
705 else if (HAS_RC6p(rc6_to_i915(rc6)))
706 target = 0x5; /* deep rc6 */
707 else
708 target = 0x4; /* normal rc6 */
709 intel_uncore_write_fw(uncore, GEN6_RC_STATE, target << RC_SW_TARGET_STATE_SHIFT);
710 }
711
intel_rc6_disable(struct intel_rc6 * rc6)712 void intel_rc6_disable(struct intel_rc6 *rc6)
713 {
714 if (!rc6->enabled)
715 return;
716
717 rpm_get(rc6);
718 rc6->enabled = false;
719
720 __intel_rc6_disable(rc6);
721 }
722
intel_rc6_fini(struct intel_rc6 * rc6)723 void intel_rc6_fini(struct intel_rc6 *rc6)
724 {
725 struct drm_i915_gem_object *pctx;
726 struct intel_uncore *uncore = rc6_to_uncore(rc6);
727
728 intel_rc6_disable(rc6);
729
730 /* We want the BIOS C6 state preserved across loads for MTL */
731 if (IS_METEORLAKE(rc6_to_i915(rc6)) && rc6->bios_state_captured)
732 intel_uncore_write_fw(uncore, GEN6_RC_STATE, rc6->bios_rc_state);
733
734 pctx = fetch_and_zero(&rc6->pctx);
735 if (pctx)
736 i915_gem_object_put(pctx);
737
738 if (rc6->wakeref)
739 rpm_put(rc6);
740 }
741
vlv_residency_raw(struct intel_uncore * uncore,const i915_reg_t reg)742 static u64 vlv_residency_raw(struct intel_uncore *uncore, const i915_reg_t reg)
743 {
744 u32 lower, upper, tmp;
745 int loop = 2;
746
747 /*
748 * The register accessed do not need forcewake. We borrow
749 * uncore lock to prevent concurrent access to range reg.
750 */
751 lockdep_assert_held(&uncore->lock);
752
753 /*
754 * vlv and chv residency counters are 40 bits in width.
755 * With a control bit, we can choose between upper or lower
756 * 32bit window into this counter.
757 *
758 * Although we always use the counter in high-range mode elsewhere,
759 * userspace may attempt to read the value before rc6 is initialised,
760 * before we have set the default VLV_COUNTER_CONTROL value. So always
761 * set the high bit to be safe.
762 */
763 intel_uncore_write_fw(uncore, VLV_COUNTER_CONTROL,
764 _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH));
765 upper = intel_uncore_read_fw(uncore, reg);
766 do {
767 tmp = upper;
768
769 intel_uncore_write_fw(uncore, VLV_COUNTER_CONTROL,
770 _MASKED_BIT_DISABLE(VLV_COUNT_RANGE_HIGH));
771 lower = intel_uncore_read_fw(uncore, reg);
772
773 intel_uncore_write_fw(uncore, VLV_COUNTER_CONTROL,
774 _MASKED_BIT_ENABLE(VLV_COUNT_RANGE_HIGH));
775 upper = intel_uncore_read_fw(uncore, reg);
776 } while (upper != tmp && --loop);
777
778 /*
779 * Everywhere else we always use VLV_COUNTER_CONTROL with the
780 * VLV_COUNT_RANGE_HIGH bit set - so it is safe to leave it set
781 * now.
782 */
783
784 return lower | (u64)upper << 8;
785 }
786
intel_rc6_residency_ns(struct intel_rc6 * rc6,enum intel_rc6_res_type id)787 u64 intel_rc6_residency_ns(struct intel_rc6 *rc6, enum intel_rc6_res_type id)
788 {
789 struct drm_i915_private *i915 = rc6_to_i915(rc6);
790 struct intel_uncore *uncore = rc6_to_uncore(rc6);
791 u64 time_hw, prev_hw, overflow_hw;
792 i915_reg_t reg = rc6->res_reg[id];
793 unsigned int fw_domains;
794 unsigned long flags;
795 u32 mul, div;
796
797 if (!rc6->supported)
798 return 0;
799
800 fw_domains = intel_uncore_forcewake_for_reg(uncore, reg, FW_REG_READ);
801
802 spin_lock_irqsave(&uncore->lock, flags);
803 intel_uncore_forcewake_get__locked(uncore, fw_domains);
804
805 /* On VLV and CHV, residency time is in CZ units rather than 1.28us */
806 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) {
807 mul = 1000000;
808 div = vlv_clock_get_czclk(&i915->drm);
809 overflow_hw = BIT_ULL(40);
810 time_hw = vlv_residency_raw(uncore, reg);
811 } else {
812 /* 833.33ns units on Gen9LP, 1.28us elsewhere. */
813 if (IS_GEN9_LP(i915)) {
814 mul = 10000;
815 div = 12;
816 } else {
817 mul = 1280;
818 div = 1;
819 }
820
821 overflow_hw = BIT_ULL(32);
822 time_hw = intel_uncore_read_fw(uncore, reg);
823 }
824
825 /*
826 * Counter wrap handling.
827 *
828 * Store previous hw counter values for counter wrap-around handling. But
829 * relying on a sufficient frequency of queries otherwise counters can still wrap.
830 */
831 prev_hw = rc6->prev_hw_residency[id];
832 rc6->prev_hw_residency[id] = time_hw;
833
834 /* RC6 delta from last sample. */
835 if (time_hw >= prev_hw)
836 time_hw -= prev_hw;
837 else
838 time_hw += overflow_hw - prev_hw;
839
840 /* Add delta to RC6 extended raw driver copy. */
841 time_hw += rc6->cur_residency[id];
842 rc6->cur_residency[id] = time_hw;
843
844 intel_uncore_forcewake_put__locked(uncore, fw_domains);
845 spin_unlock_irqrestore(&uncore->lock, flags);
846
847 return mul_u64_u32_div(time_hw, mul, div);
848 }
849
intel_rc6_residency_us(struct intel_rc6 * rc6,enum intel_rc6_res_type id)850 u64 intel_rc6_residency_us(struct intel_rc6 *rc6, enum intel_rc6_res_type id)
851 {
852 return DIV_ROUND_UP_ULL(intel_rc6_residency_ns(rc6, id), 1000);
853 }
854
intel_rc6_print_residency(struct seq_file * m,const char * title,enum intel_rc6_res_type id)855 void intel_rc6_print_residency(struct seq_file *m, const char *title,
856 enum intel_rc6_res_type id)
857 {
858 struct intel_gt *gt = m->private;
859 i915_reg_t reg = gt->rc6.res_reg[id];
860 intel_wakeref_t wakeref;
861
862 with_intel_runtime_pm(gt->uncore->rpm, wakeref)
863 seq_printf(m, "%s %u (%llu us)\n", title,
864 intel_uncore_read(gt->uncore, reg),
865 intel_rc6_residency_us(>->rc6, id));
866 }
867
868 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
869 #include "selftest_rc6.c"
870 #endif
871