1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2023 Intel Corporation
4 */
5
6 #include "i915_drv.h"
7 #include "i915_reg.h"
8 #include "i9xx_wm.h"
9 #include "intel_atomic.h"
10 #include "intel_display.h"
11 #include "intel_display_trace.h"
12 #include "intel_mchbar_regs.h"
13 #include "intel_wm.h"
14 #include "skl_watermark.h"
15 #include "vlv_sideband.h"
16
17 /* used in computing the new watermarks state */
18 struct intel_wm_config {
19 unsigned int num_pipes_active;
20 bool sprites_enabled;
21 bool sprites_scaled;
22 };
23
24 struct cxsr_latency {
25 bool is_desktop : 1;
26 bool is_ddr3 : 1;
27 u16 fsb_freq;
28 u16 mem_freq;
29 u16 display_sr;
30 u16 display_hpll_disable;
31 u16 cursor_sr;
32 u16 cursor_hpll_disable;
33 };
34
35 static const struct cxsr_latency cxsr_latency_table[] = {
36 {1, 0, 800, 400, 3382, 33382, 3983, 33983}, /* DDR2-400 SC */
37 {1, 0, 800, 667, 3354, 33354, 3807, 33807}, /* DDR2-667 SC */
38 {1, 0, 800, 800, 3347, 33347, 3763, 33763}, /* DDR2-800 SC */
39 {1, 1, 800, 667, 6420, 36420, 6873, 36873}, /* DDR3-667 SC */
40 {1, 1, 800, 800, 5902, 35902, 6318, 36318}, /* DDR3-800 SC */
41
42 {1, 0, 667, 400, 3400, 33400, 4021, 34021}, /* DDR2-400 SC */
43 {1, 0, 667, 667, 3372, 33372, 3845, 33845}, /* DDR2-667 SC */
44 {1, 0, 667, 800, 3386, 33386, 3822, 33822}, /* DDR2-800 SC */
45 {1, 1, 667, 667, 6438, 36438, 6911, 36911}, /* DDR3-667 SC */
46 {1, 1, 667, 800, 5941, 35941, 6377, 36377}, /* DDR3-800 SC */
47
48 {1, 0, 400, 400, 3472, 33472, 4173, 34173}, /* DDR2-400 SC */
49 {1, 0, 400, 667, 3443, 33443, 3996, 33996}, /* DDR2-667 SC */
50 {1, 0, 400, 800, 3430, 33430, 3946, 33946}, /* DDR2-800 SC */
51 {1, 1, 400, 667, 6509, 36509, 7062, 37062}, /* DDR3-667 SC */
52 {1, 1, 400, 800, 5985, 35985, 6501, 36501}, /* DDR3-800 SC */
53
54 {0, 0, 800, 400, 3438, 33438, 4065, 34065}, /* DDR2-400 SC */
55 {0, 0, 800, 667, 3410, 33410, 3889, 33889}, /* DDR2-667 SC */
56 {0, 0, 800, 800, 3403, 33403, 3845, 33845}, /* DDR2-800 SC */
57 {0, 1, 800, 667, 6476, 36476, 6955, 36955}, /* DDR3-667 SC */
58 {0, 1, 800, 800, 5958, 35958, 6400, 36400}, /* DDR3-800 SC */
59
60 {0, 0, 667, 400, 3456, 33456, 4103, 34106}, /* DDR2-400 SC */
61 {0, 0, 667, 667, 3428, 33428, 3927, 33927}, /* DDR2-667 SC */
62 {0, 0, 667, 800, 3443, 33443, 3905, 33905}, /* DDR2-800 SC */
63 {0, 1, 667, 667, 6494, 36494, 6993, 36993}, /* DDR3-667 SC */
64 {0, 1, 667, 800, 5998, 35998, 6460, 36460}, /* DDR3-800 SC */
65
66 {0, 0, 400, 400, 3528, 33528, 4255, 34255}, /* DDR2-400 SC */
67 {0, 0, 400, 667, 3500, 33500, 4079, 34079}, /* DDR2-667 SC */
68 {0, 0, 400, 800, 3487, 33487, 4029, 34029}, /* DDR2-800 SC */
69 {0, 1, 400, 667, 6566, 36566, 7145, 37145}, /* DDR3-667 SC */
70 {0, 1, 400, 800, 6042, 36042, 6584, 36584}, /* DDR3-800 SC */
71 };
72
pnv_get_cxsr_latency(struct drm_i915_private * i915)73 static const struct cxsr_latency *pnv_get_cxsr_latency(struct drm_i915_private *i915)
74 {
75 int i;
76
77 for (i = 0; i < ARRAY_SIZE(cxsr_latency_table); i++) {
78 const struct cxsr_latency *latency = &cxsr_latency_table[i];
79 bool is_desktop = !IS_MOBILE(i915);
80
81 if (is_desktop == latency->is_desktop &&
82 i915->is_ddr3 == latency->is_ddr3 &&
83 DIV_ROUND_CLOSEST(i915->fsb_freq, 1000) == latency->fsb_freq &&
84 DIV_ROUND_CLOSEST(i915->mem_freq, 1000) == latency->mem_freq)
85 return latency;
86 }
87
88 drm_dbg_kms(&i915->drm,
89 "Could not find CxSR latency for DDR%s, FSB %u kHz, MEM %u kHz\n",
90 i915->is_ddr3 ? "3" : "2", i915->fsb_freq, i915->mem_freq);
91
92 return NULL;
93 }
94
chv_set_memory_dvfs(struct drm_i915_private * dev_priv,bool enable)95 static void chv_set_memory_dvfs(struct drm_i915_private *dev_priv, bool enable)
96 {
97 u32 val;
98
99 vlv_punit_get(dev_priv);
100
101 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
102 if (enable)
103 val &= ~FORCE_DDR_HIGH_FREQ;
104 else
105 val |= FORCE_DDR_HIGH_FREQ;
106 val &= ~FORCE_DDR_LOW_FREQ;
107 val |= FORCE_DDR_FREQ_REQ_ACK;
108 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
109
110 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
111 FORCE_DDR_FREQ_REQ_ACK) == 0, 3))
112 drm_err(&dev_priv->drm,
113 "timed out waiting for Punit DDR DVFS request\n");
114
115 vlv_punit_put(dev_priv);
116 }
117
chv_set_memory_pm5(struct drm_i915_private * dev_priv,bool enable)118 static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
119 {
120 u32 val;
121
122 vlv_punit_get(dev_priv);
123
124 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
125 if (enable)
126 val |= DSP_MAXFIFO_PM5_ENABLE;
127 else
128 val &= ~DSP_MAXFIFO_PM5_ENABLE;
129 vlv_punit_write(dev_priv, PUNIT_REG_DSPSSPM, val);
130
131 vlv_punit_put(dev_priv);
132 }
133
134 #define FW_WM(value, plane) \
135 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK)
136
_intel_set_memory_cxsr(struct drm_i915_private * dev_priv,bool enable)137 static bool _intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
138 {
139 bool was_enabled;
140 u32 val;
141
142 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
143 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
144 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);
145 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF_VLV);
146 } else if (IS_G4X(dev_priv) || IS_I965GM(dev_priv)) {
147 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
148 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);
149 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
150 } else if (IS_PINEVIEW(dev_priv)) {
151 val = intel_uncore_read(&dev_priv->uncore, DSPFW3(dev_priv));
152 was_enabled = val & PINEVIEW_SELF_REFRESH_EN;
153 if (enable)
154 val |= PINEVIEW_SELF_REFRESH_EN;
155 else
156 val &= ~PINEVIEW_SELF_REFRESH_EN;
157 intel_uncore_write(&dev_priv->uncore, DSPFW3(dev_priv), val);
158 intel_uncore_posting_read(&dev_priv->uncore, DSPFW3(dev_priv));
159 } else if (IS_I945G(dev_priv) || IS_I945GM(dev_priv)) {
160 was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
161 val = enable ? _MASKED_BIT_ENABLE(FW_BLC_SELF_EN) :
162 _MASKED_BIT_DISABLE(FW_BLC_SELF_EN);
163 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, val);
164 intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);
165 } else if (IS_I915GM(dev_priv)) {
166 /*
167 * FIXME can't find a bit like this for 915G, and
168 * yet it does have the related watermark in
169 * FW_BLC_SELF. What's going on?
170 */
171 was_enabled = intel_uncore_read(&dev_priv->uncore, INSTPM) & INSTPM_SELF_EN;
172 val = enable ? _MASKED_BIT_ENABLE(INSTPM_SELF_EN) :
173 _MASKED_BIT_DISABLE(INSTPM_SELF_EN);
174 intel_uncore_write(&dev_priv->uncore, INSTPM, val);
175 intel_uncore_posting_read(&dev_priv->uncore, INSTPM);
176 } else {
177 return false;
178 }
179
180 trace_intel_memory_cxsr(dev_priv, was_enabled, enable);
181
182 drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n",
183 str_enabled_disabled(enable),
184 str_enabled_disabled(was_enabled));
185
186 return was_enabled;
187 }
188
189 /**
190 * intel_set_memory_cxsr - Configure CxSR state
191 * @dev_priv: i915 device
192 * @enable: Allow vs. disallow CxSR
193 *
194 * Allow or disallow the system to enter a special CxSR
195 * (C-state self refresh) state. What typically happens in CxSR mode
196 * is that several display FIFOs may get combined into a single larger
197 * FIFO for a particular plane (so called max FIFO mode) to allow the
198 * system to defer memory fetches longer, and the memory will enter
199 * self refresh.
200 *
201 * Note that enabling CxSR does not guarantee that the system enter
202 * this special mode, nor does it guarantee that the system stays
203 * in that mode once entered. So this just allows/disallows the system
204 * to autonomously utilize the CxSR mode. Other factors such as core
205 * C-states will affect when/if the system actually enters/exits the
206 * CxSR mode.
207 *
208 * Note that on VLV/CHV this actually only controls the max FIFO mode,
209 * and the system is free to enter/exit memory self refresh at any time
210 * even when the use of CxSR has been disallowed.
211 *
212 * While the system is actually in the CxSR/max FIFO mode, some plane
213 * control registers will not get latched on vblank. Thus in order to
214 * guarantee the system will respond to changes in the plane registers
215 * we must always disallow CxSR prior to making changes to those registers.
216 * Unfortunately the system will re-evaluate the CxSR conditions at
217 * frame start which happens after vblank start (which is when the plane
218 * registers would get latched), so we can't proceed with the plane update
219 * during the same frame where we disallowed CxSR.
220 *
221 * Certain platforms also have a deeper HPLL SR mode. Fortunately the
222 * HPLL SR mode depends on CxSR itself, so we don't have to hand hold
223 * the hardware w.r.t. HPLL SR when writing to plane registers.
224 * Disallowing just CxSR is sufficient.
225 */
intel_set_memory_cxsr(struct drm_i915_private * dev_priv,bool enable)226 bool intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
227 {
228 bool ret;
229
230 mutex_lock(&dev_priv->display.wm.wm_mutex);
231 ret = _intel_set_memory_cxsr(dev_priv, enable);
232 if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
233 dev_priv->display.wm.vlv.cxsr = enable;
234 else if (IS_G4X(dev_priv))
235 dev_priv->display.wm.g4x.cxsr = enable;
236 mutex_unlock(&dev_priv->display.wm.wm_mutex);
237
238 return ret;
239 }
240
241 /*
242 * Latency for FIFO fetches is dependent on several factors:
243 * - memory configuration (speed, channels)
244 * - chipset
245 * - current MCH state
246 * It can be fairly high in some situations, so here we assume a fairly
247 * pessimal value. It's a tradeoff between extra memory fetches (if we
248 * set this value too high, the FIFO will fetch frequently to stay full)
249 * and power consumption (set it too low to save power and we might see
250 * FIFO underruns and display "flicker").
251 *
252 * A value of 5us seems to be a good balance; safe for very low end
253 * platforms but not overly aggressive on lower latency configs.
254 */
255 static const int pessimal_latency_ns = 5000;
256
257 #define VLV_FIFO_START(dsparb, dsparb2, lo_shift, hi_shift) \
258 ((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
259
vlv_get_fifo_size(struct intel_crtc_state * crtc_state)260 static void vlv_get_fifo_size(struct intel_crtc_state *crtc_state)
261 {
262 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
263 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
264 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
265 enum pipe pipe = crtc->pipe;
266 int sprite0_start, sprite1_start;
267 u32 dsparb, dsparb2, dsparb3;
268
269 switch (pipe) {
270 case PIPE_A:
271 dsparb = intel_uncore_read(&dev_priv->uncore,
272 DSPARB(dev_priv));
273 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
274 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 0, 0);
275 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);
276 break;
277 case PIPE_B:
278 dsparb = intel_uncore_read(&dev_priv->uncore,
279 DSPARB(dev_priv));
280 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
281 sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);
282 sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 24, 12);
283 break;
284 case PIPE_C:
285 dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);
286 dsparb3 = intel_uncore_read(&dev_priv->uncore, DSPARB3);
287 sprite0_start = VLV_FIFO_START(dsparb3, dsparb2, 0, 16);
288 sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);
289 break;
290 default:
291 MISSING_CASE(pipe);
292 return;
293 }
294
295 fifo_state->plane[PLANE_PRIMARY] = sprite0_start;
296 fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;
297 fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;
298 fifo_state->plane[PLANE_CURSOR] = 63;
299 }
300
i9xx_get_fifo_size(struct drm_i915_private * dev_priv,enum i9xx_plane_id i9xx_plane)301 static int i9xx_get_fifo_size(struct drm_i915_private *dev_priv,
302 enum i9xx_plane_id i9xx_plane)
303 {
304 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB(dev_priv));
305 int size;
306
307 size = dsparb & 0x7f;
308 if (i9xx_plane == PLANE_B)
309 size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
310
311 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
312 dsparb, plane_name(i9xx_plane), size);
313
314 return size;
315 }
316
i830_get_fifo_size(struct drm_i915_private * dev_priv,enum i9xx_plane_id i9xx_plane)317 static int i830_get_fifo_size(struct drm_i915_private *dev_priv,
318 enum i9xx_plane_id i9xx_plane)
319 {
320 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB(dev_priv));
321 int size;
322
323 size = dsparb & 0x1ff;
324 if (i9xx_plane == PLANE_B)
325 size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
326 size >>= 1; /* Convert to cachelines */
327
328 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
329 dsparb, plane_name(i9xx_plane), size);
330
331 return size;
332 }
333
i845_get_fifo_size(struct drm_i915_private * dev_priv,enum i9xx_plane_id i9xx_plane)334 static int i845_get_fifo_size(struct drm_i915_private *dev_priv,
335 enum i9xx_plane_id i9xx_plane)
336 {
337 u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB(dev_priv));
338 int size;
339
340 size = dsparb & 0x7f;
341 size >>= 2; /* Convert to cachelines */
342
343 drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",
344 dsparb, plane_name(i9xx_plane), size);
345
346 return size;
347 }
348
349 /* Pineview has different values for various configs */
350 static const struct intel_watermark_params pnv_display_wm = {
351 .fifo_size = PINEVIEW_DISPLAY_FIFO,
352 .max_wm = PINEVIEW_MAX_WM,
353 .default_wm = PINEVIEW_DFT_WM,
354 .guard_size = PINEVIEW_GUARD_WM,
355 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
356 };
357
358 static const struct intel_watermark_params pnv_display_hplloff_wm = {
359 .fifo_size = PINEVIEW_DISPLAY_FIFO,
360 .max_wm = PINEVIEW_MAX_WM,
361 .default_wm = PINEVIEW_DFT_HPLLOFF_WM,
362 .guard_size = PINEVIEW_GUARD_WM,
363 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
364 };
365
366 static const struct intel_watermark_params pnv_cursor_wm = {
367 .fifo_size = PINEVIEW_CURSOR_FIFO,
368 .max_wm = PINEVIEW_CURSOR_MAX_WM,
369 .default_wm = PINEVIEW_CURSOR_DFT_WM,
370 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
371 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
372 };
373
374 static const struct intel_watermark_params pnv_cursor_hplloff_wm = {
375 .fifo_size = PINEVIEW_CURSOR_FIFO,
376 .max_wm = PINEVIEW_CURSOR_MAX_WM,
377 .default_wm = PINEVIEW_CURSOR_DFT_WM,
378 .guard_size = PINEVIEW_CURSOR_GUARD_WM,
379 .cacheline_size = PINEVIEW_FIFO_LINE_SIZE,
380 };
381
382 static const struct intel_watermark_params i965_cursor_wm_info = {
383 .fifo_size = I965_CURSOR_FIFO,
384 .max_wm = I965_CURSOR_MAX_WM,
385 .default_wm = I965_CURSOR_DFT_WM,
386 .guard_size = 2,
387 .cacheline_size = I915_FIFO_LINE_SIZE,
388 };
389
390 static const struct intel_watermark_params i945_wm_info = {
391 .fifo_size = I945_FIFO_SIZE,
392 .max_wm = I915_MAX_WM,
393 .default_wm = 1,
394 .guard_size = 2,
395 .cacheline_size = I915_FIFO_LINE_SIZE,
396 };
397
398 static const struct intel_watermark_params i915_wm_info = {
399 .fifo_size = I915_FIFO_SIZE,
400 .max_wm = I915_MAX_WM,
401 .default_wm = 1,
402 .guard_size = 2,
403 .cacheline_size = I915_FIFO_LINE_SIZE,
404 };
405
406 static const struct intel_watermark_params i830_a_wm_info = {
407 .fifo_size = I855GM_FIFO_SIZE,
408 .max_wm = I915_MAX_WM,
409 .default_wm = 1,
410 .guard_size = 2,
411 .cacheline_size = I830_FIFO_LINE_SIZE,
412 };
413
414 static const struct intel_watermark_params i830_bc_wm_info = {
415 .fifo_size = I855GM_FIFO_SIZE,
416 .max_wm = I915_MAX_WM / 2,
417 .default_wm = 1,
418 .guard_size = 2,
419 .cacheline_size = I830_FIFO_LINE_SIZE,
420 };
421
422 static const struct intel_watermark_params i845_wm_info = {
423 .fifo_size = I830_FIFO_SIZE,
424 .max_wm = I915_MAX_WM,
425 .default_wm = 1,
426 .guard_size = 2,
427 .cacheline_size = I830_FIFO_LINE_SIZE,
428 };
429
430 /**
431 * intel_wm_method1 - Method 1 / "small buffer" watermark formula
432 * @pixel_rate: Pipe pixel rate in kHz
433 * @cpp: Plane bytes per pixel
434 * @latency: Memory wakeup latency in 0.1us units
435 *
436 * Compute the watermark using the method 1 or "small buffer"
437 * formula. The caller may additonally add extra cachelines
438 * to account for TLB misses and clock crossings.
439 *
440 * This method is concerned with the short term drain rate
441 * of the FIFO, ie. it does not account for blanking periods
442 * which would effectively reduce the average drain rate across
443 * a longer period. The name "small" refers to the fact the
444 * FIFO is relatively small compared to the amount of data
445 * fetched.
446 *
447 * The FIFO level vs. time graph might look something like:
448 *
449 * |\ |\
450 * | \ | \
451 * __---__---__ (- plane active, _ blanking)
452 * -> time
453 *
454 * or perhaps like this:
455 *
456 * |\|\ |\|\
457 * __----__----__ (- plane active, _ blanking)
458 * -> time
459 *
460 * Returns:
461 * The watermark in bytes
462 */
intel_wm_method1(unsigned int pixel_rate,unsigned int cpp,unsigned int latency)463 static unsigned int intel_wm_method1(unsigned int pixel_rate,
464 unsigned int cpp,
465 unsigned int latency)
466 {
467 u64 ret;
468
469 ret = mul_u32_u32(pixel_rate, cpp * latency);
470 ret = DIV_ROUND_UP_ULL(ret, 10000);
471
472 return ret;
473 }
474
475 /**
476 * intel_wm_method2 - Method 2 / "large buffer" watermark formula
477 * @pixel_rate: Pipe pixel rate in kHz
478 * @htotal: Pipe horizontal total
479 * @width: Plane width in pixels
480 * @cpp: Plane bytes per pixel
481 * @latency: Memory wakeup latency in 0.1us units
482 *
483 * Compute the watermark using the method 2 or "large buffer"
484 * formula. The caller may additonally add extra cachelines
485 * to account for TLB misses and clock crossings.
486 *
487 * This method is concerned with the long term drain rate
488 * of the FIFO, ie. it does account for blanking periods
489 * which effectively reduce the average drain rate across
490 * a longer period. The name "large" refers to the fact the
491 * FIFO is relatively large compared to the amount of data
492 * fetched.
493 *
494 * The FIFO level vs. time graph might look something like:
495 *
496 * |\___ |\___
497 * | \___ | \___
498 * | \ | \
499 * __ --__--__--__--__--__--__ (- plane active, _ blanking)
500 * -> time
501 *
502 * Returns:
503 * The watermark in bytes
504 */
intel_wm_method2(unsigned int pixel_rate,unsigned int htotal,unsigned int width,unsigned int cpp,unsigned int latency)505 static unsigned int intel_wm_method2(unsigned int pixel_rate,
506 unsigned int htotal,
507 unsigned int width,
508 unsigned int cpp,
509 unsigned int latency)
510 {
511 unsigned int ret;
512
513 /*
514 * FIXME remove once all users are computing
515 * watermarks in the correct place.
516 */
517 if (WARN_ON_ONCE(htotal == 0))
518 htotal = 1;
519
520 ret = (latency * pixel_rate) / (htotal * 10000);
521 ret = (ret + 1) * width * cpp;
522
523 return ret;
524 }
525
526 /**
527 * intel_calculate_wm - calculate watermark level
528 * @i915: the device
529 * @pixel_rate: pixel clock
530 * @wm: chip FIFO params
531 * @fifo_size: size of the FIFO buffer
532 * @cpp: bytes per pixel
533 * @latency_ns: memory latency for the platform
534 *
535 * Calculate the watermark level (the level at which the display plane will
536 * start fetching from memory again). Each chip has a different display
537 * FIFO size and allocation, so the caller needs to figure that out and pass
538 * in the correct intel_watermark_params structure.
539 *
540 * As the pixel clock runs, the FIFO will be drained at a rate that depends
541 * on the pixel size. When it reaches the watermark level, it'll start
542 * fetching FIFO line sized based chunks from memory until the FIFO fills
543 * past the watermark point. If the FIFO drains completely, a FIFO underrun
544 * will occur, and a display engine hang could result.
545 */
intel_calculate_wm(struct drm_i915_private * i915,int pixel_rate,const struct intel_watermark_params * wm,int fifo_size,int cpp,unsigned int latency_ns)546 static unsigned int intel_calculate_wm(struct drm_i915_private *i915,
547 int pixel_rate,
548 const struct intel_watermark_params *wm,
549 int fifo_size, int cpp,
550 unsigned int latency_ns)
551 {
552 int entries, wm_size;
553
554 /*
555 * Note: we need to make sure we don't overflow for various clock &
556 * latency values.
557 * clocks go from a few thousand to several hundred thousand.
558 * latency is usually a few thousand
559 */
560 entries = intel_wm_method1(pixel_rate, cpp,
561 latency_ns / 100);
562 entries = DIV_ROUND_UP(entries, wm->cacheline_size) +
563 wm->guard_size;
564 drm_dbg_kms(&i915->drm, "FIFO entries required for mode: %d\n", entries);
565
566 wm_size = fifo_size - entries;
567 drm_dbg_kms(&i915->drm, "FIFO watermark level: %d\n", wm_size);
568
569 /* Don't promote wm_size to unsigned... */
570 if (wm_size > wm->max_wm)
571 wm_size = wm->max_wm;
572 if (wm_size <= 0)
573 wm_size = wm->default_wm;
574
575 /*
576 * Bspec seems to indicate that the value shouldn't be lower than
577 * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
578 * Lets go for 8 which is the burst size since certain platforms
579 * already use a hardcoded 8 (which is what the spec says should be
580 * done).
581 */
582 if (wm_size <= 8)
583 wm_size = 8;
584
585 return wm_size;
586 }
587
is_disabling(int old,int new,int threshold)588 static bool is_disabling(int old, int new, int threshold)
589 {
590 return old >= threshold && new < threshold;
591 }
592
is_enabling(int old,int new,int threshold)593 static bool is_enabling(int old, int new, int threshold)
594 {
595 return old < threshold && new >= threshold;
596 }
597
intel_crtc_active(struct intel_crtc * crtc)598 static bool intel_crtc_active(struct intel_crtc *crtc)
599 {
600 /* Be paranoid as we can arrive here with only partial
601 * state retrieved from the hardware during setup.
602 *
603 * We can ditch the adjusted_mode.crtc_clock check as soon
604 * as Haswell has gained clock readout/fastboot support.
605 *
606 * We can ditch the crtc->primary->state->fb check as soon as we can
607 * properly reconstruct framebuffers.
608 *
609 * FIXME: The intel_crtc->active here should be switched to
610 * crtc->state->active once we have proper CRTC states wired up
611 * for atomic.
612 */
613 return crtc->active && crtc->base.primary->state->fb &&
614 crtc->config->hw.adjusted_mode.crtc_clock;
615 }
616
single_enabled_crtc(struct drm_i915_private * dev_priv)617 static struct intel_crtc *single_enabled_crtc(struct drm_i915_private *dev_priv)
618 {
619 struct intel_crtc *crtc, *enabled = NULL;
620
621 for_each_intel_crtc(&dev_priv->drm, crtc) {
622 if (intel_crtc_active(crtc)) {
623 if (enabled)
624 return NULL;
625 enabled = crtc;
626 }
627 }
628
629 return enabled;
630 }
631
pnv_update_wm(struct drm_i915_private * dev_priv)632 static void pnv_update_wm(struct drm_i915_private *dev_priv)
633 {
634 struct intel_crtc *crtc;
635 const struct cxsr_latency *latency;
636 u32 reg;
637 unsigned int wm;
638
639 latency = pnv_get_cxsr_latency(dev_priv);
640 if (!latency) {
641 drm_dbg_kms(&dev_priv->drm, "Unknown FSB/MEM, disabling CxSR\n");
642 intel_set_memory_cxsr(dev_priv, false);
643 return;
644 }
645
646 crtc = single_enabled_crtc(dev_priv);
647 if (crtc) {
648 const struct drm_framebuffer *fb =
649 crtc->base.primary->state->fb;
650 int pixel_rate = crtc->config->pixel_rate;
651 int cpp = fb->format->cpp[0];
652
653 /* Display SR */
654 wm = intel_calculate_wm(dev_priv, pixel_rate,
655 &pnv_display_wm,
656 pnv_display_wm.fifo_size,
657 cpp, latency->display_sr);
658 reg = intel_uncore_read(&dev_priv->uncore, DSPFW1(dev_priv));
659 reg &= ~DSPFW_SR_MASK;
660 reg |= FW_WM(wm, SR);
661 intel_uncore_write(&dev_priv->uncore, DSPFW1(dev_priv), reg);
662 drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg);
663
664 /* cursor SR */
665 wm = intel_calculate_wm(dev_priv, pixel_rate,
666 &pnv_cursor_wm,
667 pnv_display_wm.fifo_size,
668 4, latency->cursor_sr);
669 intel_uncore_rmw(&dev_priv->uncore, DSPFW3(dev_priv),
670 DSPFW_CURSOR_SR_MASK,
671 FW_WM(wm, CURSOR_SR));
672
673 /* Display HPLL off SR */
674 wm = intel_calculate_wm(dev_priv, pixel_rate,
675 &pnv_display_hplloff_wm,
676 pnv_display_hplloff_wm.fifo_size,
677 cpp, latency->display_hpll_disable);
678 intel_uncore_rmw(&dev_priv->uncore, DSPFW3(dev_priv),
679 DSPFW_HPLL_SR_MASK, FW_WM(wm, HPLL_SR));
680
681 /* cursor HPLL off SR */
682 wm = intel_calculate_wm(dev_priv, pixel_rate,
683 &pnv_cursor_hplloff_wm,
684 pnv_display_hplloff_wm.fifo_size,
685 4, latency->cursor_hpll_disable);
686 reg = intel_uncore_read(&dev_priv->uncore, DSPFW3(dev_priv));
687 reg &= ~DSPFW_HPLL_CURSOR_MASK;
688 reg |= FW_WM(wm, HPLL_CURSOR);
689 intel_uncore_write(&dev_priv->uncore, DSPFW3(dev_priv), reg);
690 drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg);
691
692 intel_set_memory_cxsr(dev_priv, true);
693 } else {
694 intel_set_memory_cxsr(dev_priv, false);
695 }
696 }
697
698 /*
699 * Documentation says:
700 * "If the line size is small, the TLB fetches can get in the way of the
701 * data fetches, causing some lag in the pixel data return which is not
702 * accounted for in the above formulas. The following adjustment only
703 * needs to be applied if eight whole lines fit in the buffer at once.
704 * The WM is adjusted upwards by the difference between the FIFO size
705 * and the size of 8 whole lines. This adjustment is always performed
706 * in the actual pixel depth regardless of whether FBC is enabled or not."
707 */
g4x_tlb_miss_wa(int fifo_size,int width,int cpp)708 static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
709 {
710 int tlb_miss = fifo_size * 64 - width * cpp * 8;
711
712 return max(0, tlb_miss);
713 }
714
g4x_write_wm_values(struct drm_i915_private * dev_priv,const struct g4x_wm_values * wm)715 static void g4x_write_wm_values(struct drm_i915_private *dev_priv,
716 const struct g4x_wm_values *wm)
717 {
718 enum pipe pipe;
719
720 for_each_pipe(dev_priv, pipe)
721 trace_g4x_wm(intel_crtc_for_pipe(dev_priv, pipe), wm);
722
723 intel_uncore_write(&dev_priv->uncore, DSPFW1(dev_priv),
724 FW_WM(wm->sr.plane, SR) |
725 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
726 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
727 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
728 intel_uncore_write(&dev_priv->uncore, DSPFW2(dev_priv),
729 (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |
730 FW_WM(wm->sr.fbc, FBC_SR) |
731 FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |
732 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |
733 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
734 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
735 intel_uncore_write(&dev_priv->uncore, DSPFW3(dev_priv),
736 (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |
737 FW_WM(wm->sr.cursor, CURSOR_SR) |
738 FW_WM(wm->hpll.cursor, HPLL_CURSOR) |
739 FW_WM(wm->hpll.plane, HPLL_SR));
740
741 intel_uncore_posting_read(&dev_priv->uncore, DSPFW1(dev_priv));
742 }
743
744 #define FW_WM_VLV(value, plane) \
745 (((value) << DSPFW_ ## plane ## _SHIFT) & DSPFW_ ## plane ## _MASK_VLV)
746
vlv_write_wm_values(struct drm_i915_private * dev_priv,const struct vlv_wm_values * wm)747 static void vlv_write_wm_values(struct drm_i915_private *dev_priv,
748 const struct vlv_wm_values *wm)
749 {
750 enum pipe pipe;
751
752 for_each_pipe(dev_priv, pipe) {
753 trace_vlv_wm(intel_crtc_for_pipe(dev_priv, pipe), wm);
754
755 intel_uncore_write(&dev_priv->uncore, VLV_DDL(pipe),
756 (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |
757 (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |
758 (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |
759 (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));
760 }
761
762 /*
763 * Zero the (unused) WM1 watermarks, and also clear all the
764 * high order bits so that there are no out of bounds values
765 * present in the registers during the reprogramming.
766 */
767 intel_uncore_write(&dev_priv->uncore, DSPHOWM, 0);
768 intel_uncore_write(&dev_priv->uncore, DSPHOWM1, 0);
769 intel_uncore_write(&dev_priv->uncore, DSPFW4, 0);
770 intel_uncore_write(&dev_priv->uncore, DSPFW5, 0);
771 intel_uncore_write(&dev_priv->uncore, DSPFW6, 0);
772
773 intel_uncore_write(&dev_priv->uncore, DSPFW1(dev_priv),
774 FW_WM(wm->sr.plane, SR) |
775 FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |
776 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |
777 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));
778 intel_uncore_write(&dev_priv->uncore, DSPFW2(dev_priv),
779 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |
780 FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |
781 FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));
782 intel_uncore_write(&dev_priv->uncore, DSPFW3(dev_priv),
783 FW_WM(wm->sr.cursor, CURSOR_SR));
784
785 if (IS_CHERRYVIEW(dev_priv)) {
786 intel_uncore_write(&dev_priv->uncore, DSPFW7_CHV,
787 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
788 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
789 intel_uncore_write(&dev_priv->uncore, DSPFW8_CHV,
790 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |
791 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));
792 intel_uncore_write(&dev_priv->uncore, DSPFW9_CHV,
793 FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |
794 FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));
795 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
796 FW_WM(wm->sr.plane >> 9, SR_HI) |
797 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |
798 FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |
799 FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |
800 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
801 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
802 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
803 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
804 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
805 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
806 } else {
807 intel_uncore_write(&dev_priv->uncore, DSPFW7,
808 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |
809 FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));
810 intel_uncore_write(&dev_priv->uncore, DSPHOWM,
811 FW_WM(wm->sr.plane >> 9, SR_HI) |
812 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |
813 FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |
814 FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |
815 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |
816 FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |
817 FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));
818 }
819
820 intel_uncore_posting_read(&dev_priv->uncore, DSPFW1(dev_priv));
821 }
822
823 #undef FW_WM_VLV
824
g4x_setup_wm_latency(struct drm_i915_private * dev_priv)825 static void g4x_setup_wm_latency(struct drm_i915_private *dev_priv)
826 {
827 /* all latencies in usec */
828 dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;
829 dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_SR] = 12;
830 dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;
831
832 dev_priv->display.wm.num_levels = G4X_WM_LEVEL_HPLL + 1;
833 }
834
g4x_plane_fifo_size(enum plane_id plane_id,int level)835 static int g4x_plane_fifo_size(enum plane_id plane_id, int level)
836 {
837 /*
838 * DSPCNTR[13] supposedly controls whether the
839 * primary plane can use the FIFO space otherwise
840 * reserved for the sprite plane. It's not 100% clear
841 * what the actual FIFO size is, but it looks like we
842 * can happily set both primary and sprite watermarks
843 * up to 127 cachelines. So that would seem to mean
844 * that either DSPCNTR[13] doesn't do anything, or that
845 * the total FIFO is >= 256 cachelines in size. Either
846 * way, we don't seem to have to worry about this
847 * repartitioning as the maximum watermark value the
848 * register can hold for each plane is lower than the
849 * minimum FIFO size.
850 */
851 switch (plane_id) {
852 case PLANE_CURSOR:
853 return 63;
854 case PLANE_PRIMARY:
855 return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;
856 case PLANE_SPRITE0:
857 return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;
858 default:
859 MISSING_CASE(plane_id);
860 return 0;
861 }
862 }
863
g4x_fbc_fifo_size(int level)864 static int g4x_fbc_fifo_size(int level)
865 {
866 switch (level) {
867 case G4X_WM_LEVEL_SR:
868 return 7;
869 case G4X_WM_LEVEL_HPLL:
870 return 15;
871 default:
872 MISSING_CASE(level);
873 return 0;
874 }
875 }
876
g4x_compute_wm(const struct intel_crtc_state * crtc_state,const struct intel_plane_state * plane_state,int level)877 static u16 g4x_compute_wm(const struct intel_crtc_state *crtc_state,
878 const struct intel_plane_state *plane_state,
879 int level)
880 {
881 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
882 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
883 const struct drm_display_mode *pipe_mode =
884 &crtc_state->hw.pipe_mode;
885 unsigned int latency = dev_priv->display.wm.pri_latency[level] * 10;
886 unsigned int pixel_rate, htotal, cpp, width, wm;
887
888 if (latency == 0)
889 return USHRT_MAX;
890
891 if (!intel_wm_plane_visible(crtc_state, plane_state))
892 return 0;
893
894 cpp = plane_state->hw.fb->format->cpp[0];
895
896 /*
897 * WaUse32BppForSRWM:ctg,elk
898 *
899 * The spec fails to list this restriction for the
900 * HPLL watermark, which seems a little strange.
901 * Let's use 32bpp for the HPLL watermark as well.
902 */
903 if (plane->id == PLANE_PRIMARY &&
904 level != G4X_WM_LEVEL_NORMAL)
905 cpp = max(cpp, 4u);
906
907 pixel_rate = crtc_state->pixel_rate;
908 htotal = pipe_mode->crtc_htotal;
909 width = drm_rect_width(&plane_state->uapi.src) >> 16;
910
911 if (plane->id == PLANE_CURSOR) {
912 wm = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
913 } else if (plane->id == PLANE_PRIMARY &&
914 level == G4X_WM_LEVEL_NORMAL) {
915 wm = intel_wm_method1(pixel_rate, cpp, latency);
916 } else {
917 unsigned int small, large;
918
919 small = intel_wm_method1(pixel_rate, cpp, latency);
920 large = intel_wm_method2(pixel_rate, htotal, width, cpp, latency);
921
922 wm = min(small, large);
923 }
924
925 wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),
926 width, cpp);
927
928 wm = DIV_ROUND_UP(wm, 64) + 2;
929
930 return min_t(unsigned int, wm, USHRT_MAX);
931 }
932
g4x_raw_plane_wm_set(struct intel_crtc_state * crtc_state,int level,enum plane_id plane_id,u16 value)933 static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
934 int level, enum plane_id plane_id, u16 value)
935 {
936 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
937 bool dirty = false;
938
939 for (; level < dev_priv->display.wm.num_levels; level++) {
940 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
941
942 dirty |= raw->plane[plane_id] != value;
943 raw->plane[plane_id] = value;
944 }
945
946 return dirty;
947 }
948
g4x_raw_fbc_wm_set(struct intel_crtc_state * crtc_state,int level,u16 value)949 static bool g4x_raw_fbc_wm_set(struct intel_crtc_state *crtc_state,
950 int level, u16 value)
951 {
952 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
953 bool dirty = false;
954
955 /* NORMAL level doesn't have an FBC watermark */
956 level = max(level, G4X_WM_LEVEL_SR);
957
958 for (; level < dev_priv->display.wm.num_levels; level++) {
959 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
960
961 dirty |= raw->fbc != value;
962 raw->fbc = value;
963 }
964
965 return dirty;
966 }
967
968 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
969 const struct intel_plane_state *plane_state,
970 u32 pri_val);
971
g4x_raw_plane_wm_compute(struct intel_crtc_state * crtc_state,const struct intel_plane_state * plane_state)972 static bool g4x_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
973 const struct intel_plane_state *plane_state)
974 {
975 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
976 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
977 enum plane_id plane_id = plane->id;
978 bool dirty = false;
979 int level;
980
981 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
982 dirty |= g4x_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
983 if (plane_id == PLANE_PRIMARY)
984 dirty |= g4x_raw_fbc_wm_set(crtc_state, 0, 0);
985 goto out;
986 }
987
988 for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
989 struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
990 int wm, max_wm;
991
992 wm = g4x_compute_wm(crtc_state, plane_state, level);
993 max_wm = g4x_plane_fifo_size(plane_id, level);
994
995 if (wm > max_wm)
996 break;
997
998 dirty |= raw->plane[plane_id] != wm;
999 raw->plane[plane_id] = wm;
1000
1001 if (plane_id != PLANE_PRIMARY ||
1002 level == G4X_WM_LEVEL_NORMAL)
1003 continue;
1004
1005 wm = ilk_compute_fbc_wm(crtc_state, plane_state,
1006 raw->plane[plane_id]);
1007 max_wm = g4x_fbc_fifo_size(level);
1008
1009 /*
1010 * FBC wm is not mandatory as we
1011 * can always just disable its use.
1012 */
1013 if (wm > max_wm)
1014 wm = USHRT_MAX;
1015
1016 dirty |= raw->fbc != wm;
1017 raw->fbc = wm;
1018 }
1019
1020 /* mark watermarks as invalid */
1021 dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1022
1023 if (plane_id == PLANE_PRIMARY)
1024 dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
1025
1026 out:
1027 if (dirty) {
1028 drm_dbg_kms(&dev_priv->drm,
1029 "%s watermarks: normal=%d, SR=%d, HPLL=%d\n",
1030 plane->base.name,
1031 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],
1032 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],
1033 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);
1034
1035 if (plane_id == PLANE_PRIMARY)
1036 drm_dbg_kms(&dev_priv->drm,
1037 "FBC watermarks: SR=%d, HPLL=%d\n",
1038 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,
1039 crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);
1040 }
1041
1042 return dirty;
1043 }
1044
g4x_raw_plane_wm_is_valid(const struct intel_crtc_state * crtc_state,enum plane_id plane_id,int level)1045 static bool g4x_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1046 enum plane_id plane_id, int level)
1047 {
1048 const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];
1049
1050 return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);
1051 }
1052
g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state * crtc_state,int level)1053 static bool g4x_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state,
1054 int level)
1055 {
1056 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1057
1058 if (level >= dev_priv->display.wm.num_levels)
1059 return false;
1060
1061 return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1062 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1063 g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1064 }
1065
1066 /* mark all levels starting from 'level' as invalid */
g4x_invalidate_wms(struct intel_crtc * crtc,struct g4x_wm_state * wm_state,int level)1067 static void g4x_invalidate_wms(struct intel_crtc *crtc,
1068 struct g4x_wm_state *wm_state, int level)
1069 {
1070 if (level <= G4X_WM_LEVEL_NORMAL) {
1071 enum plane_id plane_id;
1072
1073 for_each_plane_id_on_crtc(crtc, plane_id)
1074 wm_state->wm.plane[plane_id] = USHRT_MAX;
1075 }
1076
1077 if (level <= G4X_WM_LEVEL_SR) {
1078 wm_state->cxsr = false;
1079 wm_state->sr.cursor = USHRT_MAX;
1080 wm_state->sr.plane = USHRT_MAX;
1081 wm_state->sr.fbc = USHRT_MAX;
1082 }
1083
1084 if (level <= G4X_WM_LEVEL_HPLL) {
1085 wm_state->hpll_en = false;
1086 wm_state->hpll.cursor = USHRT_MAX;
1087 wm_state->hpll.plane = USHRT_MAX;
1088 wm_state->hpll.fbc = USHRT_MAX;
1089 }
1090 }
1091
g4x_compute_fbc_en(const struct g4x_wm_state * wm_state,int level)1092 static bool g4x_compute_fbc_en(const struct g4x_wm_state *wm_state,
1093 int level)
1094 {
1095 if (level < G4X_WM_LEVEL_SR)
1096 return false;
1097
1098 if (level >= G4X_WM_LEVEL_SR &&
1099 wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))
1100 return false;
1101
1102 if (level >= G4X_WM_LEVEL_HPLL &&
1103 wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))
1104 return false;
1105
1106 return true;
1107 }
1108
_g4x_compute_pipe_wm(struct intel_crtc_state * crtc_state)1109 static int _g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1110 {
1111 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1112 struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;
1113 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1114 const struct g4x_pipe_wm *raw;
1115 enum plane_id plane_id;
1116 int level;
1117
1118 level = G4X_WM_LEVEL_NORMAL;
1119 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1120 goto out;
1121
1122 raw = &crtc_state->wm.g4x.raw[level];
1123 for_each_plane_id_on_crtc(crtc, plane_id)
1124 wm_state->wm.plane[plane_id] = raw->plane[plane_id];
1125
1126 level = G4X_WM_LEVEL_SR;
1127 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1128 goto out;
1129
1130 raw = &crtc_state->wm.g4x.raw[level];
1131 wm_state->sr.plane = raw->plane[PLANE_PRIMARY];
1132 wm_state->sr.cursor = raw->plane[PLANE_CURSOR];
1133 wm_state->sr.fbc = raw->fbc;
1134
1135 wm_state->cxsr = active_planes == BIT(PLANE_PRIMARY);
1136
1137 level = G4X_WM_LEVEL_HPLL;
1138 if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))
1139 goto out;
1140
1141 raw = &crtc_state->wm.g4x.raw[level];
1142 wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];
1143 wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];
1144 wm_state->hpll.fbc = raw->fbc;
1145
1146 wm_state->hpll_en = wm_state->cxsr;
1147
1148 level++;
1149
1150 out:
1151 if (level == G4X_WM_LEVEL_NORMAL)
1152 return -EINVAL;
1153
1154 /* invalidate the higher levels */
1155 g4x_invalidate_wms(crtc, wm_state, level);
1156
1157 /*
1158 * Determine if the FBC watermark(s) can be used. IF
1159 * this isn't the case we prefer to disable the FBC
1160 * watermark(s) rather than disable the SR/HPLL
1161 * level(s) entirely. 'level-1' is the highest valid
1162 * level here.
1163 */
1164 wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level - 1);
1165
1166 return 0;
1167 }
1168
g4x_compute_pipe_wm(struct intel_atomic_state * state,struct intel_crtc * crtc)1169 static int g4x_compute_pipe_wm(struct intel_atomic_state *state,
1170 struct intel_crtc *crtc)
1171 {
1172 struct intel_crtc_state *crtc_state =
1173 intel_atomic_get_new_crtc_state(state, crtc);
1174 const struct intel_plane_state *old_plane_state;
1175 const struct intel_plane_state *new_plane_state;
1176 struct intel_plane *plane;
1177 unsigned int dirty = 0;
1178 int i;
1179
1180 for_each_oldnew_intel_plane_in_state(state, plane,
1181 old_plane_state,
1182 new_plane_state, i) {
1183 if (new_plane_state->hw.crtc != &crtc->base &&
1184 old_plane_state->hw.crtc != &crtc->base)
1185 continue;
1186
1187 if (g4x_raw_plane_wm_compute(crtc_state, new_plane_state))
1188 dirty |= BIT(plane->id);
1189 }
1190
1191 if (!dirty)
1192 return 0;
1193
1194 return _g4x_compute_pipe_wm(crtc_state);
1195 }
1196
g4x_compute_intermediate_wm(struct intel_atomic_state * state,struct intel_crtc * crtc)1197 static int g4x_compute_intermediate_wm(struct intel_atomic_state *state,
1198 struct intel_crtc *crtc)
1199 {
1200 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1201 struct intel_crtc_state *new_crtc_state =
1202 intel_atomic_get_new_crtc_state(state, crtc);
1203 const struct intel_crtc_state *old_crtc_state =
1204 intel_atomic_get_old_crtc_state(state, crtc);
1205 struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
1206 const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
1207 const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
1208 enum plane_id plane_id;
1209
1210 if (!new_crtc_state->hw.active ||
1211 intel_crtc_needs_modeset(new_crtc_state)) {
1212 *intermediate = *optimal;
1213
1214 intermediate->cxsr = false;
1215 intermediate->hpll_en = false;
1216 goto out;
1217 }
1218
1219 intermediate->cxsr = optimal->cxsr && active->cxsr &&
1220 !new_crtc_state->disable_cxsr;
1221 intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
1222 !new_crtc_state->disable_cxsr;
1223 intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
1224
1225 for_each_plane_id_on_crtc(crtc, plane_id) {
1226 intermediate->wm.plane[plane_id] =
1227 max(optimal->wm.plane[plane_id],
1228 active->wm.plane[plane_id]);
1229
1230 drm_WARN_ON(&dev_priv->drm, intermediate->wm.plane[plane_id] >
1231 g4x_plane_fifo_size(plane_id, G4X_WM_LEVEL_NORMAL));
1232 }
1233
1234 intermediate->sr.plane = max(optimal->sr.plane,
1235 active->sr.plane);
1236 intermediate->sr.cursor = max(optimal->sr.cursor,
1237 active->sr.cursor);
1238 intermediate->sr.fbc = max(optimal->sr.fbc,
1239 active->sr.fbc);
1240
1241 intermediate->hpll.plane = max(optimal->hpll.plane,
1242 active->hpll.plane);
1243 intermediate->hpll.cursor = max(optimal->hpll.cursor,
1244 active->hpll.cursor);
1245 intermediate->hpll.fbc = max(optimal->hpll.fbc,
1246 active->hpll.fbc);
1247
1248 drm_WARN_ON(&dev_priv->drm,
1249 (intermediate->sr.plane >
1250 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_SR) ||
1251 intermediate->sr.cursor >
1252 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_SR)) &&
1253 intermediate->cxsr);
1254 drm_WARN_ON(&dev_priv->drm,
1255 (intermediate->sr.plane >
1256 g4x_plane_fifo_size(PLANE_PRIMARY, G4X_WM_LEVEL_HPLL) ||
1257 intermediate->sr.cursor >
1258 g4x_plane_fifo_size(PLANE_CURSOR, G4X_WM_LEVEL_HPLL)) &&
1259 intermediate->hpll_en);
1260
1261 drm_WARN_ON(&dev_priv->drm,
1262 intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&
1263 intermediate->fbc_en && intermediate->cxsr);
1264 drm_WARN_ON(&dev_priv->drm,
1265 intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
1266 intermediate->fbc_en && intermediate->hpll_en);
1267
1268 out:
1269 /*
1270 * If our intermediate WM are identical to the final WM, then we can
1271 * omit the post-vblank programming; only update if it's different.
1272 */
1273 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1274 new_crtc_state->wm.need_postvbl_update = true;
1275
1276 return 0;
1277 }
1278
g4x_merge_wm(struct drm_i915_private * dev_priv,struct g4x_wm_values * wm)1279 static void g4x_merge_wm(struct drm_i915_private *dev_priv,
1280 struct g4x_wm_values *wm)
1281 {
1282 struct intel_crtc *crtc;
1283 int num_active_pipes = 0;
1284
1285 wm->cxsr = true;
1286 wm->hpll_en = true;
1287 wm->fbc_en = true;
1288
1289 for_each_intel_crtc(&dev_priv->drm, crtc) {
1290 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1291
1292 if (!crtc->active)
1293 continue;
1294
1295 if (!wm_state->cxsr)
1296 wm->cxsr = false;
1297 if (!wm_state->hpll_en)
1298 wm->hpll_en = false;
1299 if (!wm_state->fbc_en)
1300 wm->fbc_en = false;
1301
1302 num_active_pipes++;
1303 }
1304
1305 if (num_active_pipes != 1) {
1306 wm->cxsr = false;
1307 wm->hpll_en = false;
1308 wm->fbc_en = false;
1309 }
1310
1311 for_each_intel_crtc(&dev_priv->drm, crtc) {
1312 const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;
1313 enum pipe pipe = crtc->pipe;
1314
1315 wm->pipe[pipe] = wm_state->wm;
1316 if (crtc->active && wm->cxsr)
1317 wm->sr = wm_state->sr;
1318 if (crtc->active && wm->hpll_en)
1319 wm->hpll = wm_state->hpll;
1320 }
1321 }
1322
g4x_program_watermarks(struct drm_i915_private * dev_priv)1323 static void g4x_program_watermarks(struct drm_i915_private *dev_priv)
1324 {
1325 struct g4x_wm_values *old_wm = &dev_priv->display.wm.g4x;
1326 struct g4x_wm_values new_wm = {};
1327
1328 g4x_merge_wm(dev_priv, &new_wm);
1329
1330 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1331 return;
1332
1333 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1334 _intel_set_memory_cxsr(dev_priv, false);
1335
1336 g4x_write_wm_values(dev_priv, &new_wm);
1337
1338 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1339 _intel_set_memory_cxsr(dev_priv, true);
1340
1341 *old_wm = new_wm;
1342 }
1343
g4x_initial_watermarks(struct intel_atomic_state * state,struct intel_crtc * crtc)1344 static void g4x_initial_watermarks(struct intel_atomic_state *state,
1345 struct intel_crtc *crtc)
1346 {
1347 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1348 const struct intel_crtc_state *crtc_state =
1349 intel_atomic_get_new_crtc_state(state, crtc);
1350
1351 mutex_lock(&dev_priv->display.wm.wm_mutex);
1352 crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;
1353 g4x_program_watermarks(dev_priv);
1354 mutex_unlock(&dev_priv->display.wm.wm_mutex);
1355 }
1356
g4x_optimize_watermarks(struct intel_atomic_state * state,struct intel_crtc * crtc)1357 static void g4x_optimize_watermarks(struct intel_atomic_state *state,
1358 struct intel_crtc *crtc)
1359 {
1360 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1361 const struct intel_crtc_state *crtc_state =
1362 intel_atomic_get_new_crtc_state(state, crtc);
1363
1364 if (!crtc_state->wm.need_postvbl_update)
1365 return;
1366
1367 mutex_lock(&dev_priv->display.wm.wm_mutex);
1368 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
1369 g4x_program_watermarks(dev_priv);
1370 mutex_unlock(&dev_priv->display.wm.wm_mutex);
1371 }
1372
1373 /* latency must be in 0.1us units. */
vlv_wm_method2(unsigned int pixel_rate,unsigned int htotal,unsigned int width,unsigned int cpp,unsigned int latency)1374 static unsigned int vlv_wm_method2(unsigned int pixel_rate,
1375 unsigned int htotal,
1376 unsigned int width,
1377 unsigned int cpp,
1378 unsigned int latency)
1379 {
1380 unsigned int ret;
1381
1382 ret = intel_wm_method2(pixel_rate, htotal,
1383 width, cpp, latency);
1384 ret = DIV_ROUND_UP(ret, 64);
1385
1386 return ret;
1387 }
1388
vlv_setup_wm_latency(struct drm_i915_private * dev_priv)1389 static void vlv_setup_wm_latency(struct drm_i915_private *dev_priv)
1390 {
1391 /* all latencies in usec */
1392 dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
1393
1394 dev_priv->display.wm.num_levels = VLV_WM_LEVEL_PM2 + 1;
1395
1396 if (IS_CHERRYVIEW(dev_priv)) {
1397 dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;
1398 dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;
1399
1400 dev_priv->display.wm.num_levels = VLV_WM_LEVEL_DDR_DVFS + 1;
1401 }
1402 }
1403
vlv_compute_wm_level(const struct intel_crtc_state * crtc_state,const struct intel_plane_state * plane_state,int level)1404 static u16 vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
1405 const struct intel_plane_state *plane_state,
1406 int level)
1407 {
1408 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1409 struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
1410 const struct drm_display_mode *pipe_mode =
1411 &crtc_state->hw.pipe_mode;
1412 unsigned int pixel_rate, htotal, cpp, width, wm;
1413
1414 if (dev_priv->display.wm.pri_latency[level] == 0)
1415 return USHRT_MAX;
1416
1417 if (!intel_wm_plane_visible(crtc_state, plane_state))
1418 return 0;
1419
1420 cpp = plane_state->hw.fb->format->cpp[0];
1421 pixel_rate = crtc_state->pixel_rate;
1422 htotal = pipe_mode->crtc_htotal;
1423 width = drm_rect_width(&plane_state->uapi.src) >> 16;
1424
1425 if (plane->id == PLANE_CURSOR) {
1426 /*
1427 * FIXME the formula gives values that are
1428 * too big for the cursor FIFO, and hence we
1429 * would never be able to use cursors. For
1430 * now just hardcode the watermark.
1431 */
1432 wm = 63;
1433 } else {
1434 wm = vlv_wm_method2(pixel_rate, htotal, width, cpp,
1435 dev_priv->display.wm.pri_latency[level] * 10);
1436 }
1437
1438 return min_t(unsigned int, wm, USHRT_MAX);
1439 }
1440
vlv_need_sprite0_fifo_workaround(unsigned int active_planes)1441 static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
1442 {
1443 return (active_planes & (BIT(PLANE_SPRITE0) |
1444 BIT(PLANE_SPRITE1))) == BIT(PLANE_SPRITE1);
1445 }
1446
vlv_compute_fifo(struct intel_crtc_state * crtc_state)1447 static int vlv_compute_fifo(struct intel_crtc_state *crtc_state)
1448 {
1449 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1450 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1451 const struct g4x_pipe_wm *raw =
1452 &crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];
1453 struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;
1454 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1455 int num_active_planes = hweight8(active_planes);
1456 const int fifo_size = 511;
1457 int fifo_extra, fifo_left = fifo_size;
1458 int sprite0_fifo_extra = 0;
1459 unsigned int total_rate;
1460 enum plane_id plane_id;
1461
1462 /*
1463 * When enabling sprite0 after sprite1 has already been enabled
1464 * we tend to get an underrun unless sprite0 already has some
1465 * FIFO space allcoated. Hence we always allocate at least one
1466 * cacheline for sprite0 whenever sprite1 is enabled.
1467 *
1468 * All other plane enable sequences appear immune to this problem.
1469 */
1470 if (vlv_need_sprite0_fifo_workaround(active_planes))
1471 sprite0_fifo_extra = 1;
1472
1473 total_rate = raw->plane[PLANE_PRIMARY] +
1474 raw->plane[PLANE_SPRITE0] +
1475 raw->plane[PLANE_SPRITE1] +
1476 sprite0_fifo_extra;
1477
1478 if (total_rate > fifo_size)
1479 return -EINVAL;
1480
1481 if (total_rate == 0)
1482 total_rate = 1;
1483
1484 for_each_plane_id_on_crtc(crtc, plane_id) {
1485 unsigned int rate;
1486
1487 if ((active_planes & BIT(plane_id)) == 0) {
1488 fifo_state->plane[plane_id] = 0;
1489 continue;
1490 }
1491
1492 rate = raw->plane[plane_id];
1493 fifo_state->plane[plane_id] = fifo_size * rate / total_rate;
1494 fifo_left -= fifo_state->plane[plane_id];
1495 }
1496
1497 fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;
1498 fifo_left -= sprite0_fifo_extra;
1499
1500 fifo_state->plane[PLANE_CURSOR] = 63;
1501
1502 fifo_extra = DIV_ROUND_UP(fifo_left, num_active_planes ?: 1);
1503
1504 /* spread the remainder evenly */
1505 for_each_plane_id_on_crtc(crtc, plane_id) {
1506 int plane_extra;
1507
1508 if (fifo_left == 0)
1509 break;
1510
1511 if ((active_planes & BIT(plane_id)) == 0)
1512 continue;
1513
1514 plane_extra = min(fifo_extra, fifo_left);
1515 fifo_state->plane[plane_id] += plane_extra;
1516 fifo_left -= plane_extra;
1517 }
1518
1519 drm_WARN_ON(&dev_priv->drm, active_planes != 0 && fifo_left != 0);
1520
1521 /* give it all to the first plane if none are active */
1522 if (active_planes == 0) {
1523 drm_WARN_ON(&dev_priv->drm, fifo_left != fifo_size);
1524 fifo_state->plane[PLANE_PRIMARY] = fifo_left;
1525 }
1526
1527 return 0;
1528 }
1529
1530 /* mark all levels starting from 'level' as invalid */
vlv_invalidate_wms(struct intel_crtc * crtc,struct vlv_wm_state * wm_state,int level)1531 static void vlv_invalidate_wms(struct intel_crtc *crtc,
1532 struct vlv_wm_state *wm_state, int level)
1533 {
1534 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1535
1536 for (; level < dev_priv->display.wm.num_levels; level++) {
1537 enum plane_id plane_id;
1538
1539 for_each_plane_id_on_crtc(crtc, plane_id)
1540 wm_state->wm[level].plane[plane_id] = USHRT_MAX;
1541
1542 wm_state->sr[level].cursor = USHRT_MAX;
1543 wm_state->sr[level].plane = USHRT_MAX;
1544 }
1545 }
1546
vlv_invert_wm_value(u16 wm,u16 fifo_size)1547 static u16 vlv_invert_wm_value(u16 wm, u16 fifo_size)
1548 {
1549 if (wm > fifo_size)
1550 return USHRT_MAX;
1551 else
1552 return fifo_size - wm;
1553 }
1554
1555 /*
1556 * Starting from 'level' set all higher
1557 * levels to 'value' in the "raw" watermarks.
1558 */
vlv_raw_plane_wm_set(struct intel_crtc_state * crtc_state,int level,enum plane_id plane_id,u16 value)1559 static bool vlv_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
1560 int level, enum plane_id plane_id, u16 value)
1561 {
1562 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1563 bool dirty = false;
1564
1565 for (; level < dev_priv->display.wm.num_levels; level++) {
1566 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1567
1568 dirty |= raw->plane[plane_id] != value;
1569 raw->plane[plane_id] = value;
1570 }
1571
1572 return dirty;
1573 }
1574
vlv_raw_plane_wm_compute(struct intel_crtc_state * crtc_state,const struct intel_plane_state * plane_state)1575 static bool vlv_raw_plane_wm_compute(struct intel_crtc_state *crtc_state,
1576 const struct intel_plane_state *plane_state)
1577 {
1578 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
1579 struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);
1580 enum plane_id plane_id = plane->id;
1581 int level;
1582 bool dirty = false;
1583
1584 if (!intel_wm_plane_visible(crtc_state, plane_state)) {
1585 dirty |= vlv_raw_plane_wm_set(crtc_state, 0, plane_id, 0);
1586 goto out;
1587 }
1588
1589 for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
1590 struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1591 int wm = vlv_compute_wm_level(crtc_state, plane_state, level);
1592 int max_wm = plane_id == PLANE_CURSOR ? 63 : 511;
1593
1594 if (wm > max_wm)
1595 break;
1596
1597 dirty |= raw->plane[plane_id] != wm;
1598 raw->plane[plane_id] = wm;
1599 }
1600
1601 /* mark all higher levels as invalid */
1602 dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);
1603
1604 out:
1605 if (dirty)
1606 drm_dbg_kms(&dev_priv->drm,
1607 "%s watermarks: PM2=%d, PM5=%d, DDR DVFS=%d\n",
1608 plane->base.name,
1609 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],
1610 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],
1611 crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);
1612
1613 return dirty;
1614 }
1615
vlv_raw_plane_wm_is_valid(const struct intel_crtc_state * crtc_state,enum plane_id plane_id,int level)1616 static bool vlv_raw_plane_wm_is_valid(const struct intel_crtc_state *crtc_state,
1617 enum plane_id plane_id, int level)
1618 {
1619 const struct g4x_pipe_wm *raw =
1620 &crtc_state->wm.vlv.raw[level];
1621 const struct vlv_fifo_state *fifo_state =
1622 &crtc_state->wm.vlv.fifo_state;
1623
1624 return raw->plane[plane_id] <= fifo_state->plane[plane_id];
1625 }
1626
vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state * crtc_state,int level)1627 static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)
1628 {
1629 return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&
1630 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&
1631 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&
1632 vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);
1633 }
1634
_vlv_compute_pipe_wm(struct intel_crtc_state * crtc_state)1635 static int _vlv_compute_pipe_wm(struct intel_crtc_state *crtc_state)
1636 {
1637 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1638 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1639 struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;
1640 const struct vlv_fifo_state *fifo_state =
1641 &crtc_state->wm.vlv.fifo_state;
1642 u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);
1643 int num_active_planes = hweight8(active_planes);
1644 enum plane_id plane_id;
1645 int level;
1646
1647 /* initially allow all levels */
1648 wm_state->num_levels = dev_priv->display.wm.num_levels;
1649 /*
1650 * Note that enabling cxsr with no primary/sprite planes
1651 * enabled can wedge the pipe. Hence we only allow cxsr
1652 * with exactly one enabled primary/sprite plane.
1653 */
1654 wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;
1655
1656 for (level = 0; level < wm_state->num_levels; level++) {
1657 const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];
1658 const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;
1659
1660 if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))
1661 break;
1662
1663 for_each_plane_id_on_crtc(crtc, plane_id) {
1664 wm_state->wm[level].plane[plane_id] =
1665 vlv_invert_wm_value(raw->plane[plane_id],
1666 fifo_state->plane[plane_id]);
1667 }
1668
1669 wm_state->sr[level].plane =
1670 vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],
1671 raw->plane[PLANE_SPRITE0],
1672 raw->plane[PLANE_SPRITE1]),
1673 sr_fifo_size);
1674
1675 wm_state->sr[level].cursor =
1676 vlv_invert_wm_value(raw->plane[PLANE_CURSOR],
1677 63);
1678 }
1679
1680 if (level == 0)
1681 return -EINVAL;
1682
1683 /* limit to only levels we can actually handle */
1684 wm_state->num_levels = level;
1685
1686 /* invalidate the higher levels */
1687 vlv_invalidate_wms(crtc, wm_state, level);
1688
1689 return 0;
1690 }
1691
vlv_compute_pipe_wm(struct intel_atomic_state * state,struct intel_crtc * crtc)1692 static int vlv_compute_pipe_wm(struct intel_atomic_state *state,
1693 struct intel_crtc *crtc)
1694 {
1695 struct intel_crtc_state *crtc_state =
1696 intel_atomic_get_new_crtc_state(state, crtc);
1697 const struct intel_plane_state *old_plane_state;
1698 const struct intel_plane_state *new_plane_state;
1699 struct intel_plane *plane;
1700 unsigned int dirty = 0;
1701 int i;
1702
1703 for_each_oldnew_intel_plane_in_state(state, plane,
1704 old_plane_state,
1705 new_plane_state, i) {
1706 if (new_plane_state->hw.crtc != &crtc->base &&
1707 old_plane_state->hw.crtc != &crtc->base)
1708 continue;
1709
1710 if (vlv_raw_plane_wm_compute(crtc_state, new_plane_state))
1711 dirty |= BIT(plane->id);
1712 }
1713
1714 /*
1715 * DSPARB registers may have been reset due to the
1716 * power well being turned off. Make sure we restore
1717 * them to a consistent state even if no primary/sprite
1718 * planes are initially active. We also force a FIFO
1719 * recomputation so that we are sure to sanitize the
1720 * FIFO setting we took over from the BIOS even if there
1721 * are no active planes on the crtc.
1722 */
1723 if (intel_crtc_needs_modeset(crtc_state))
1724 dirty = ~0;
1725
1726 if (!dirty)
1727 return 0;
1728
1729 /* cursor changes don't warrant a FIFO recompute */
1730 if (dirty & ~BIT(PLANE_CURSOR)) {
1731 const struct intel_crtc_state *old_crtc_state =
1732 intel_atomic_get_old_crtc_state(state, crtc);
1733 const struct vlv_fifo_state *old_fifo_state =
1734 &old_crtc_state->wm.vlv.fifo_state;
1735 const struct vlv_fifo_state *new_fifo_state =
1736 &crtc_state->wm.vlv.fifo_state;
1737 int ret;
1738
1739 ret = vlv_compute_fifo(crtc_state);
1740 if (ret)
1741 return ret;
1742
1743 if (intel_crtc_needs_modeset(crtc_state) ||
1744 memcmp(old_fifo_state, new_fifo_state,
1745 sizeof(*new_fifo_state)) != 0)
1746 crtc_state->fifo_changed = true;
1747 }
1748
1749 return _vlv_compute_pipe_wm(crtc_state);
1750 }
1751
1752 #define VLV_FIFO(plane, value) \
1753 (((value) << DSPARB_ ## plane ## _SHIFT_VLV) & DSPARB_ ## plane ## _MASK_VLV)
1754
vlv_atomic_update_fifo(struct intel_atomic_state * state,struct intel_crtc * crtc)1755 static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
1756 struct intel_crtc *crtc)
1757 {
1758 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1759 struct intel_uncore *uncore = &dev_priv->uncore;
1760 const struct intel_crtc_state *crtc_state =
1761 intel_atomic_get_new_crtc_state(state, crtc);
1762 const struct vlv_fifo_state *fifo_state =
1763 &crtc_state->wm.vlv.fifo_state;
1764 int sprite0_start, sprite1_start, fifo_size;
1765 u32 dsparb, dsparb2, dsparb3;
1766
1767 if (!crtc_state->fifo_changed)
1768 return;
1769
1770 sprite0_start = fifo_state->plane[PLANE_PRIMARY];
1771 sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;
1772 fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;
1773
1774 drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63);
1775 drm_WARN_ON(&dev_priv->drm, fifo_size != 511);
1776
1777 trace_vlv_fifo_size(crtc, sprite0_start, sprite1_start, fifo_size);
1778
1779 /*
1780 * uncore.lock serves a double purpose here. It allows us to
1781 * use the less expensive I915_{READ,WRITE}_FW() functions, and
1782 * it protects the DSPARB registers from getting clobbered by
1783 * parallel updates from multiple pipes.
1784 *
1785 * intel_pipe_update_start() has already disabled interrupts
1786 * for us, so a plain spin_lock() is sufficient here.
1787 */
1788 spin_lock(&uncore->lock);
1789
1790 switch (crtc->pipe) {
1791 case PIPE_A:
1792 dsparb = intel_uncore_read_fw(uncore, DSPARB(dev_priv));
1793 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
1794
1795 dsparb &= ~(VLV_FIFO(SPRITEA, 0xff) |
1796 VLV_FIFO(SPRITEB, 0xff));
1797 dsparb |= (VLV_FIFO(SPRITEA, sprite0_start) |
1798 VLV_FIFO(SPRITEB, sprite1_start));
1799
1800 dsparb2 &= ~(VLV_FIFO(SPRITEA_HI, 0x1) |
1801 VLV_FIFO(SPRITEB_HI, 0x1));
1802 dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |
1803 VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));
1804
1805 intel_uncore_write_fw(uncore, DSPARB(dev_priv), dsparb);
1806 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
1807 break;
1808 case PIPE_B:
1809 dsparb = intel_uncore_read_fw(uncore, DSPARB(dev_priv));
1810 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
1811
1812 dsparb &= ~(VLV_FIFO(SPRITEC, 0xff) |
1813 VLV_FIFO(SPRITED, 0xff));
1814 dsparb |= (VLV_FIFO(SPRITEC, sprite0_start) |
1815 VLV_FIFO(SPRITED, sprite1_start));
1816
1817 dsparb2 &= ~(VLV_FIFO(SPRITEC_HI, 0xff) |
1818 VLV_FIFO(SPRITED_HI, 0xff));
1819 dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |
1820 VLV_FIFO(SPRITED_HI, sprite1_start >> 8));
1821
1822 intel_uncore_write_fw(uncore, DSPARB(dev_priv), dsparb);
1823 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
1824 break;
1825 case PIPE_C:
1826 dsparb3 = intel_uncore_read_fw(uncore, DSPARB3);
1827 dsparb2 = intel_uncore_read_fw(uncore, DSPARB2);
1828
1829 dsparb3 &= ~(VLV_FIFO(SPRITEE, 0xff) |
1830 VLV_FIFO(SPRITEF, 0xff));
1831 dsparb3 |= (VLV_FIFO(SPRITEE, sprite0_start) |
1832 VLV_FIFO(SPRITEF, sprite1_start));
1833
1834 dsparb2 &= ~(VLV_FIFO(SPRITEE_HI, 0xff) |
1835 VLV_FIFO(SPRITEF_HI, 0xff));
1836 dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |
1837 VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));
1838
1839 intel_uncore_write_fw(uncore, DSPARB3, dsparb3);
1840 intel_uncore_write_fw(uncore, DSPARB2, dsparb2);
1841 break;
1842 default:
1843 break;
1844 }
1845
1846 intel_uncore_posting_read_fw(uncore, DSPARB(dev_priv));
1847
1848 spin_unlock(&uncore->lock);
1849 }
1850
1851 #undef VLV_FIFO
1852
vlv_compute_intermediate_wm(struct intel_atomic_state * state,struct intel_crtc * crtc)1853 static int vlv_compute_intermediate_wm(struct intel_atomic_state *state,
1854 struct intel_crtc *crtc)
1855 {
1856 struct intel_crtc_state *new_crtc_state =
1857 intel_atomic_get_new_crtc_state(state, crtc);
1858 const struct intel_crtc_state *old_crtc_state =
1859 intel_atomic_get_old_crtc_state(state, crtc);
1860 struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
1861 const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
1862 const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
1863 int level;
1864
1865 if (!new_crtc_state->hw.active ||
1866 intel_crtc_needs_modeset(new_crtc_state)) {
1867 *intermediate = *optimal;
1868
1869 intermediate->cxsr = false;
1870 goto out;
1871 }
1872
1873 intermediate->num_levels = min(optimal->num_levels, active->num_levels);
1874 intermediate->cxsr = optimal->cxsr && active->cxsr &&
1875 !new_crtc_state->disable_cxsr;
1876
1877 for (level = 0; level < intermediate->num_levels; level++) {
1878 enum plane_id plane_id;
1879
1880 for_each_plane_id_on_crtc(crtc, plane_id) {
1881 intermediate->wm[level].plane[plane_id] =
1882 min(optimal->wm[level].plane[plane_id],
1883 active->wm[level].plane[plane_id]);
1884 }
1885
1886 intermediate->sr[level].plane = min(optimal->sr[level].plane,
1887 active->sr[level].plane);
1888 intermediate->sr[level].cursor = min(optimal->sr[level].cursor,
1889 active->sr[level].cursor);
1890 }
1891
1892 vlv_invalidate_wms(crtc, intermediate, level);
1893
1894 out:
1895 /*
1896 * If our intermediate WM are identical to the final WM, then we can
1897 * omit the post-vblank programming; only update if it's different.
1898 */
1899 if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
1900 new_crtc_state->wm.need_postvbl_update = true;
1901
1902 return 0;
1903 }
1904
vlv_merge_wm(struct drm_i915_private * dev_priv,struct vlv_wm_values * wm)1905 static void vlv_merge_wm(struct drm_i915_private *dev_priv,
1906 struct vlv_wm_values *wm)
1907 {
1908 struct intel_crtc *crtc;
1909 int num_active_pipes = 0;
1910
1911 wm->level = dev_priv->display.wm.num_levels - 1;
1912 wm->cxsr = true;
1913
1914 for_each_intel_crtc(&dev_priv->drm, crtc) {
1915 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
1916
1917 if (!crtc->active)
1918 continue;
1919
1920 if (!wm_state->cxsr)
1921 wm->cxsr = false;
1922
1923 num_active_pipes++;
1924 wm->level = min_t(int, wm->level, wm_state->num_levels - 1);
1925 }
1926
1927 if (num_active_pipes != 1)
1928 wm->cxsr = false;
1929
1930 if (num_active_pipes > 1)
1931 wm->level = VLV_WM_LEVEL_PM2;
1932
1933 for_each_intel_crtc(&dev_priv->drm, crtc) {
1934 const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;
1935 enum pipe pipe = crtc->pipe;
1936
1937 wm->pipe[pipe] = wm_state->wm[wm->level];
1938 if (crtc->active && wm->cxsr)
1939 wm->sr = wm_state->sr[wm->level];
1940
1941 wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;
1942 wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;
1943 wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;
1944 wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;
1945 }
1946 }
1947
vlv_program_watermarks(struct drm_i915_private * dev_priv)1948 static void vlv_program_watermarks(struct drm_i915_private *dev_priv)
1949 {
1950 struct vlv_wm_values *old_wm = &dev_priv->display.wm.vlv;
1951 struct vlv_wm_values new_wm = {};
1952
1953 vlv_merge_wm(dev_priv, &new_wm);
1954
1955 if (memcmp(old_wm, &new_wm, sizeof(new_wm)) == 0)
1956 return;
1957
1958 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
1959 chv_set_memory_dvfs(dev_priv, false);
1960
1961 if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
1962 chv_set_memory_pm5(dev_priv, false);
1963
1964 if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))
1965 _intel_set_memory_cxsr(dev_priv, false);
1966
1967 vlv_write_wm_values(dev_priv, &new_wm);
1968
1969 if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))
1970 _intel_set_memory_cxsr(dev_priv, true);
1971
1972 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))
1973 chv_set_memory_pm5(dev_priv, true);
1974
1975 if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))
1976 chv_set_memory_dvfs(dev_priv, true);
1977
1978 *old_wm = new_wm;
1979 }
1980
vlv_initial_watermarks(struct intel_atomic_state * state,struct intel_crtc * crtc)1981 static void vlv_initial_watermarks(struct intel_atomic_state *state,
1982 struct intel_crtc *crtc)
1983 {
1984 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1985 const struct intel_crtc_state *crtc_state =
1986 intel_atomic_get_new_crtc_state(state, crtc);
1987
1988 mutex_lock(&dev_priv->display.wm.wm_mutex);
1989 crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;
1990 vlv_program_watermarks(dev_priv);
1991 mutex_unlock(&dev_priv->display.wm.wm_mutex);
1992 }
1993
vlv_optimize_watermarks(struct intel_atomic_state * state,struct intel_crtc * crtc)1994 static void vlv_optimize_watermarks(struct intel_atomic_state *state,
1995 struct intel_crtc *crtc)
1996 {
1997 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
1998 const struct intel_crtc_state *crtc_state =
1999 intel_atomic_get_new_crtc_state(state, crtc);
2000
2001 if (!crtc_state->wm.need_postvbl_update)
2002 return;
2003
2004 mutex_lock(&dev_priv->display.wm.wm_mutex);
2005 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
2006 vlv_program_watermarks(dev_priv);
2007 mutex_unlock(&dev_priv->display.wm.wm_mutex);
2008 }
2009
i965_update_wm(struct drm_i915_private * dev_priv)2010 static void i965_update_wm(struct drm_i915_private *dev_priv)
2011 {
2012 struct intel_crtc *crtc;
2013 int srwm = 1;
2014 int cursor_sr = 16;
2015 bool cxsr_enabled;
2016
2017 /* Calc sr entries for one plane configs */
2018 crtc = single_enabled_crtc(dev_priv);
2019 if (crtc) {
2020 /* self-refresh has much higher latency */
2021 static const int sr_latency_ns = 12000;
2022 const struct drm_display_mode *pipe_mode =
2023 &crtc->config->hw.pipe_mode;
2024 const struct drm_framebuffer *fb =
2025 crtc->base.primary->state->fb;
2026 int pixel_rate = crtc->config->pixel_rate;
2027 int htotal = pipe_mode->crtc_htotal;
2028 int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;
2029 int cpp = fb->format->cpp[0];
2030 int entries;
2031
2032 entries = intel_wm_method2(pixel_rate, htotal,
2033 width, cpp, sr_latency_ns / 100);
2034 entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
2035 srwm = I965_FIFO_SIZE - entries;
2036 if (srwm < 0)
2037 srwm = 1;
2038 srwm &= 0x1ff;
2039 drm_dbg_kms(&dev_priv->drm,
2040 "self-refresh entries: %d, wm: %d\n",
2041 entries, srwm);
2042
2043 entries = intel_wm_method2(pixel_rate, htotal,
2044 crtc->base.cursor->state->crtc_w, 4,
2045 sr_latency_ns / 100);
2046 entries = DIV_ROUND_UP(entries,
2047 i965_cursor_wm_info.cacheline_size) +
2048 i965_cursor_wm_info.guard_size;
2049
2050 cursor_sr = i965_cursor_wm_info.fifo_size - entries;
2051 if (cursor_sr > i965_cursor_wm_info.max_wm)
2052 cursor_sr = i965_cursor_wm_info.max_wm;
2053
2054 drm_dbg_kms(&dev_priv->drm,
2055 "self-refresh watermark: display plane %d "
2056 "cursor %d\n", srwm, cursor_sr);
2057
2058 cxsr_enabled = true;
2059 } else {
2060 cxsr_enabled = false;
2061 /* Turn off self refresh if both pipes are enabled */
2062 intel_set_memory_cxsr(dev_priv, false);
2063 }
2064
2065 drm_dbg_kms(&dev_priv->drm,
2066 "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
2067 srwm);
2068
2069 /* 965 has limitations... */
2070 intel_uncore_write(&dev_priv->uncore, DSPFW1(dev_priv),
2071 FW_WM(srwm, SR) |
2072 FW_WM(8, CURSORB) |
2073 FW_WM(8, PLANEB) |
2074 FW_WM(8, PLANEA));
2075 intel_uncore_write(&dev_priv->uncore, DSPFW2(dev_priv),
2076 FW_WM(8, CURSORA) |
2077 FW_WM(8, PLANEC_OLD));
2078 /* update cursor SR watermark */
2079 intel_uncore_write(&dev_priv->uncore, DSPFW3(dev_priv),
2080 FW_WM(cursor_sr, CURSOR_SR));
2081
2082 if (cxsr_enabled)
2083 intel_set_memory_cxsr(dev_priv, true);
2084 }
2085
2086 #undef FW_WM
2087
intel_crtc_for_plane(struct drm_i915_private * i915,enum i9xx_plane_id i9xx_plane)2088 static struct intel_crtc *intel_crtc_for_plane(struct drm_i915_private *i915,
2089 enum i9xx_plane_id i9xx_plane)
2090 {
2091 struct intel_plane *plane;
2092
2093 for_each_intel_plane(&i915->drm, plane) {
2094 if (plane->id == PLANE_PRIMARY &&
2095 plane->i9xx_plane == i9xx_plane)
2096 return intel_crtc_for_pipe(i915, plane->pipe);
2097 }
2098
2099 return NULL;
2100 }
2101
i9xx_update_wm(struct drm_i915_private * dev_priv)2102 static void i9xx_update_wm(struct drm_i915_private *dev_priv)
2103 {
2104 const struct intel_watermark_params *wm_info;
2105 u32 fwater_lo;
2106 u32 fwater_hi;
2107 int cwm, srwm = 1;
2108 int fifo_size;
2109 int planea_wm, planeb_wm;
2110 struct intel_crtc *crtc;
2111
2112 if (IS_I945GM(dev_priv))
2113 wm_info = &i945_wm_info;
2114 else if (DISPLAY_VER(dev_priv) != 2)
2115 wm_info = &i915_wm_info;
2116 else
2117 wm_info = &i830_a_wm_info;
2118
2119 if (DISPLAY_VER(dev_priv) == 2)
2120 fifo_size = i830_get_fifo_size(dev_priv, PLANE_A);
2121 else
2122 fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_A);
2123 crtc = intel_crtc_for_plane(dev_priv, PLANE_A);
2124 if (intel_crtc_active(crtc)) {
2125 const struct drm_framebuffer *fb =
2126 crtc->base.primary->state->fb;
2127 int cpp;
2128
2129 if (DISPLAY_VER(dev_priv) == 2)
2130 cpp = 4;
2131 else
2132 cpp = fb->format->cpp[0];
2133
2134 planea_wm = intel_calculate_wm(dev_priv, crtc->config->pixel_rate,
2135 wm_info, fifo_size, cpp,
2136 pessimal_latency_ns);
2137 } else {
2138 planea_wm = fifo_size - wm_info->guard_size;
2139 if (planea_wm > (long)wm_info->max_wm)
2140 planea_wm = wm_info->max_wm;
2141 }
2142
2143 if (DISPLAY_VER(dev_priv) == 2)
2144 wm_info = &i830_bc_wm_info;
2145
2146 if (DISPLAY_VER(dev_priv) == 2)
2147 fifo_size = i830_get_fifo_size(dev_priv, PLANE_B);
2148 else
2149 fifo_size = i9xx_get_fifo_size(dev_priv, PLANE_B);
2150 crtc = intel_crtc_for_plane(dev_priv, PLANE_B);
2151 if (intel_crtc_active(crtc)) {
2152 const struct drm_framebuffer *fb =
2153 crtc->base.primary->state->fb;
2154 int cpp;
2155
2156 if (DISPLAY_VER(dev_priv) == 2)
2157 cpp = 4;
2158 else
2159 cpp = fb->format->cpp[0];
2160
2161 planeb_wm = intel_calculate_wm(dev_priv, crtc->config->pixel_rate,
2162 wm_info, fifo_size, cpp,
2163 pessimal_latency_ns);
2164 } else {
2165 planeb_wm = fifo_size - wm_info->guard_size;
2166 if (planeb_wm > (long)wm_info->max_wm)
2167 planeb_wm = wm_info->max_wm;
2168 }
2169
2170 drm_dbg_kms(&dev_priv->drm,
2171 "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
2172
2173 crtc = single_enabled_crtc(dev_priv);
2174 if (IS_I915GM(dev_priv) && crtc) {
2175 struct drm_i915_gem_object *obj;
2176
2177 obj = intel_fb_obj(crtc->base.primary->state->fb);
2178
2179 /* self-refresh seems busted with untiled */
2180 if (!i915_gem_object_is_tiled(obj))
2181 crtc = NULL;
2182 }
2183
2184 /*
2185 * Overlay gets an aggressive default since video jitter is bad.
2186 */
2187 cwm = 2;
2188
2189 /* Play safe and disable self-refresh before adjusting watermarks. */
2190 intel_set_memory_cxsr(dev_priv, false);
2191
2192 /* Calc sr entries for one plane configs */
2193 if (HAS_FW_BLC(dev_priv) && crtc) {
2194 /* self-refresh has much higher latency */
2195 static const int sr_latency_ns = 6000;
2196 const struct drm_display_mode *pipe_mode =
2197 &crtc->config->hw.pipe_mode;
2198 const struct drm_framebuffer *fb =
2199 crtc->base.primary->state->fb;
2200 int pixel_rate = crtc->config->pixel_rate;
2201 int htotal = pipe_mode->crtc_htotal;
2202 int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;
2203 int cpp;
2204 int entries;
2205
2206 if (IS_I915GM(dev_priv) || IS_I945GM(dev_priv))
2207 cpp = 4;
2208 else
2209 cpp = fb->format->cpp[0];
2210
2211 entries = intel_wm_method2(pixel_rate, htotal, width, cpp,
2212 sr_latency_ns / 100);
2213 entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
2214 drm_dbg_kms(&dev_priv->drm,
2215 "self-refresh entries: %d\n", entries);
2216 srwm = wm_info->fifo_size - entries;
2217 if (srwm < 0)
2218 srwm = 1;
2219
2220 if (IS_I945G(dev_priv) || IS_I945GM(dev_priv))
2221 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF,
2222 FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
2223 else
2224 intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, srwm & 0x3f);
2225 }
2226
2227 drm_dbg_kms(&dev_priv->drm,
2228 "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
2229 planea_wm, planeb_wm, cwm, srwm);
2230
2231 fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
2232 fwater_hi = (cwm & 0x1f);
2233
2234 /* Set request length to 8 cachelines per fetch */
2235 fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
2236 fwater_hi = fwater_hi | (1 << 8);
2237
2238 intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2239 intel_uncore_write(&dev_priv->uncore, FW_BLC2, fwater_hi);
2240
2241 if (crtc)
2242 intel_set_memory_cxsr(dev_priv, true);
2243 }
2244
i845_update_wm(struct drm_i915_private * dev_priv)2245 static void i845_update_wm(struct drm_i915_private *dev_priv)
2246 {
2247 struct intel_crtc *crtc;
2248 u32 fwater_lo;
2249 int planea_wm;
2250
2251 crtc = single_enabled_crtc(dev_priv);
2252 if (crtc == NULL)
2253 return;
2254
2255 planea_wm = intel_calculate_wm(dev_priv, crtc->config->pixel_rate,
2256 &i845_wm_info,
2257 i845_get_fifo_size(dev_priv, PLANE_A),
2258 4, pessimal_latency_ns);
2259 fwater_lo = intel_uncore_read(&dev_priv->uncore, FW_BLC) & ~0xfff;
2260 fwater_lo |= (3<<8) | planea_wm;
2261
2262 drm_dbg_kms(&dev_priv->drm,
2263 "Setting FIFO watermarks - A: %d\n", planea_wm);
2264
2265 intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);
2266 }
2267
2268 /* latency must be in 0.1us units. */
ilk_wm_method1(unsigned int pixel_rate,unsigned int cpp,unsigned int latency)2269 static unsigned int ilk_wm_method1(unsigned int pixel_rate,
2270 unsigned int cpp,
2271 unsigned int latency)
2272 {
2273 unsigned int ret;
2274
2275 ret = intel_wm_method1(pixel_rate, cpp, latency);
2276 ret = DIV_ROUND_UP(ret, 64) + 2;
2277
2278 return ret;
2279 }
2280
2281 /* latency must be in 0.1us units. */
ilk_wm_method2(unsigned int pixel_rate,unsigned int htotal,unsigned int width,unsigned int cpp,unsigned int latency)2282 static unsigned int ilk_wm_method2(unsigned int pixel_rate,
2283 unsigned int htotal,
2284 unsigned int width,
2285 unsigned int cpp,
2286 unsigned int latency)
2287 {
2288 unsigned int ret;
2289
2290 ret = intel_wm_method2(pixel_rate, htotal,
2291 width, cpp, latency);
2292 ret = DIV_ROUND_UP(ret, 64) + 2;
2293
2294 return ret;
2295 }
2296
ilk_wm_fbc(u32 pri_val,u32 horiz_pixels,u8 cpp)2297 static u32 ilk_wm_fbc(u32 pri_val, u32 horiz_pixels, u8 cpp)
2298 {
2299 /*
2300 * Neither of these should be possible since this function shouldn't be
2301 * called if the CRTC is off or the plane is invisible. But let's be
2302 * extra paranoid to avoid a potential divide-by-zero if we screw up
2303 * elsewhere in the driver.
2304 */
2305 if (WARN_ON(!cpp))
2306 return 0;
2307 if (WARN_ON(!horiz_pixels))
2308 return 0;
2309
2310 return DIV_ROUND_UP(pri_val * 64, horiz_pixels * cpp) + 2;
2311 }
2312
2313 struct ilk_wm_maximums {
2314 u16 pri;
2315 u16 spr;
2316 u16 cur;
2317 u16 fbc;
2318 };
2319
2320 /*
2321 * For both WM_PIPE and WM_LP.
2322 * mem_value must be in 0.1us units.
2323 */
ilk_compute_pri_wm(const struct intel_crtc_state * crtc_state,const struct intel_plane_state * plane_state,u32 mem_value,bool is_lp)2324 static u32 ilk_compute_pri_wm(const struct intel_crtc_state *crtc_state,
2325 const struct intel_plane_state *plane_state,
2326 u32 mem_value, bool is_lp)
2327 {
2328 u32 method1, method2;
2329 int cpp;
2330
2331 if (mem_value == 0)
2332 return U32_MAX;
2333
2334 if (!intel_wm_plane_visible(crtc_state, plane_state))
2335 return 0;
2336
2337 cpp = plane_state->hw.fb->format->cpp[0];
2338
2339 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2340
2341 if (!is_lp)
2342 return method1;
2343
2344 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2345 crtc_state->hw.pipe_mode.crtc_htotal,
2346 drm_rect_width(&plane_state->uapi.src) >> 16,
2347 cpp, mem_value);
2348
2349 return min(method1, method2);
2350 }
2351
2352 /*
2353 * For both WM_PIPE and WM_LP.
2354 * mem_value must be in 0.1us units.
2355 */
ilk_compute_spr_wm(const struct intel_crtc_state * crtc_state,const struct intel_plane_state * plane_state,u32 mem_value)2356 static u32 ilk_compute_spr_wm(const struct intel_crtc_state *crtc_state,
2357 const struct intel_plane_state *plane_state,
2358 u32 mem_value)
2359 {
2360 u32 method1, method2;
2361 int cpp;
2362
2363 if (mem_value == 0)
2364 return U32_MAX;
2365
2366 if (!intel_wm_plane_visible(crtc_state, plane_state))
2367 return 0;
2368
2369 cpp = plane_state->hw.fb->format->cpp[0];
2370
2371 method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);
2372 method2 = ilk_wm_method2(crtc_state->pixel_rate,
2373 crtc_state->hw.pipe_mode.crtc_htotal,
2374 drm_rect_width(&plane_state->uapi.src) >> 16,
2375 cpp, mem_value);
2376 return min(method1, method2);
2377 }
2378
2379 /*
2380 * For both WM_PIPE and WM_LP.
2381 * mem_value must be in 0.1us units.
2382 */
ilk_compute_cur_wm(const struct intel_crtc_state * crtc_state,const struct intel_plane_state * plane_state,u32 mem_value)2383 static u32 ilk_compute_cur_wm(const struct intel_crtc_state *crtc_state,
2384 const struct intel_plane_state *plane_state,
2385 u32 mem_value)
2386 {
2387 int cpp;
2388
2389 if (mem_value == 0)
2390 return U32_MAX;
2391
2392 if (!intel_wm_plane_visible(crtc_state, plane_state))
2393 return 0;
2394
2395 cpp = plane_state->hw.fb->format->cpp[0];
2396
2397 return ilk_wm_method2(crtc_state->pixel_rate,
2398 crtc_state->hw.pipe_mode.crtc_htotal,
2399 drm_rect_width(&plane_state->uapi.src) >> 16,
2400 cpp, mem_value);
2401 }
2402
2403 /* Only for WM_LP. */
ilk_compute_fbc_wm(const struct intel_crtc_state * crtc_state,const struct intel_plane_state * plane_state,u32 pri_val)2404 static u32 ilk_compute_fbc_wm(const struct intel_crtc_state *crtc_state,
2405 const struct intel_plane_state *plane_state,
2406 u32 pri_val)
2407 {
2408 int cpp;
2409
2410 if (!intel_wm_plane_visible(crtc_state, plane_state))
2411 return 0;
2412
2413 cpp = plane_state->hw.fb->format->cpp[0];
2414
2415 return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.src) >> 16,
2416 cpp);
2417 }
2418
2419 static unsigned int
ilk_display_fifo_size(const struct drm_i915_private * dev_priv)2420 ilk_display_fifo_size(const struct drm_i915_private *dev_priv)
2421 {
2422 if (DISPLAY_VER(dev_priv) >= 8)
2423 return 3072;
2424 else if (DISPLAY_VER(dev_priv) >= 7)
2425 return 768;
2426 else
2427 return 512;
2428 }
2429
2430 static unsigned int
ilk_plane_wm_reg_max(const struct drm_i915_private * dev_priv,int level,bool is_sprite)2431 ilk_plane_wm_reg_max(const struct drm_i915_private *dev_priv,
2432 int level, bool is_sprite)
2433 {
2434 if (DISPLAY_VER(dev_priv) >= 8)
2435 /* BDW primary/sprite plane watermarks */
2436 return level == 0 ? 255 : 2047;
2437 else if (DISPLAY_VER(dev_priv) >= 7)
2438 /* IVB/HSW primary/sprite plane watermarks */
2439 return level == 0 ? 127 : 1023;
2440 else if (!is_sprite)
2441 /* ILK/SNB primary plane watermarks */
2442 return level == 0 ? 127 : 511;
2443 else
2444 /* ILK/SNB sprite plane watermarks */
2445 return level == 0 ? 63 : 255;
2446 }
2447
2448 static unsigned int
ilk_cursor_wm_reg_max(const struct drm_i915_private * dev_priv,int level)2449 ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)
2450 {
2451 if (DISPLAY_VER(dev_priv) >= 7)
2452 return level == 0 ? 63 : 255;
2453 else
2454 return level == 0 ? 31 : 63;
2455 }
2456
ilk_fbc_wm_reg_max(const struct drm_i915_private * dev_priv)2457 static unsigned int ilk_fbc_wm_reg_max(const struct drm_i915_private *dev_priv)
2458 {
2459 if (DISPLAY_VER(dev_priv) >= 8)
2460 return 31;
2461 else
2462 return 15;
2463 }
2464
2465 /* Calculate the maximum primary/sprite plane watermark */
ilk_plane_wm_max(const struct drm_i915_private * dev_priv,int level,const struct intel_wm_config * config,enum intel_ddb_partitioning ddb_partitioning,bool is_sprite)2466 static unsigned int ilk_plane_wm_max(const struct drm_i915_private *dev_priv,
2467 int level,
2468 const struct intel_wm_config *config,
2469 enum intel_ddb_partitioning ddb_partitioning,
2470 bool is_sprite)
2471 {
2472 unsigned int fifo_size = ilk_display_fifo_size(dev_priv);
2473
2474 /* if sprites aren't enabled, sprites get nothing */
2475 if (is_sprite && !config->sprites_enabled)
2476 return 0;
2477
2478 /* HSW allows LP1+ watermarks even with multiple pipes */
2479 if (level == 0 || config->num_pipes_active > 1) {
2480 fifo_size /= INTEL_NUM_PIPES(dev_priv);
2481
2482 /*
2483 * For some reason the non self refresh
2484 * FIFO size is only half of the self
2485 * refresh FIFO size on ILK/SNB.
2486 */
2487 if (DISPLAY_VER(dev_priv) < 7)
2488 fifo_size /= 2;
2489 }
2490
2491 if (config->sprites_enabled) {
2492 /* level 0 is always calculated with 1:1 split */
2493 if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {
2494 if (is_sprite)
2495 fifo_size *= 5;
2496 fifo_size /= 6;
2497 } else {
2498 fifo_size /= 2;
2499 }
2500 }
2501
2502 /* clamp to max that the registers can hold */
2503 return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));
2504 }
2505
2506 /* Calculate the maximum cursor plane watermark */
ilk_cursor_wm_max(const struct drm_i915_private * dev_priv,int level,const struct intel_wm_config * config)2507 static unsigned int ilk_cursor_wm_max(const struct drm_i915_private *dev_priv,
2508 int level,
2509 const struct intel_wm_config *config)
2510 {
2511 /* HSW LP1+ watermarks w/ multiple pipes */
2512 if (level > 0 && config->num_pipes_active > 1)
2513 return 64;
2514
2515 /* otherwise just report max that registers can hold */
2516 return ilk_cursor_wm_reg_max(dev_priv, level);
2517 }
2518
ilk_compute_wm_maximums(const struct drm_i915_private * dev_priv,int level,const struct intel_wm_config * config,enum intel_ddb_partitioning ddb_partitioning,struct ilk_wm_maximums * max)2519 static void ilk_compute_wm_maximums(const struct drm_i915_private *dev_priv,
2520 int level,
2521 const struct intel_wm_config *config,
2522 enum intel_ddb_partitioning ddb_partitioning,
2523 struct ilk_wm_maximums *max)
2524 {
2525 max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);
2526 max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);
2527 max->cur = ilk_cursor_wm_max(dev_priv, level, config);
2528 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2529 }
2530
ilk_compute_wm_reg_maximums(const struct drm_i915_private * dev_priv,int level,struct ilk_wm_maximums * max)2531 static void ilk_compute_wm_reg_maximums(const struct drm_i915_private *dev_priv,
2532 int level,
2533 struct ilk_wm_maximums *max)
2534 {
2535 max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);
2536 max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);
2537 max->cur = ilk_cursor_wm_reg_max(dev_priv, level);
2538 max->fbc = ilk_fbc_wm_reg_max(dev_priv);
2539 }
2540
ilk_validate_wm_level(struct drm_i915_private * i915,int level,const struct ilk_wm_maximums * max,struct intel_wm_level * result)2541 static bool ilk_validate_wm_level(struct drm_i915_private *i915,
2542 int level,
2543 const struct ilk_wm_maximums *max,
2544 struct intel_wm_level *result)
2545 {
2546 bool ret;
2547
2548 /* already determined to be invalid? */
2549 if (!result->enable)
2550 return false;
2551
2552 result->enable = result->pri_val <= max->pri &&
2553 result->spr_val <= max->spr &&
2554 result->cur_val <= max->cur;
2555
2556 ret = result->enable;
2557
2558 /*
2559 * HACK until we can pre-compute everything,
2560 * and thus fail gracefully if LP0 watermarks
2561 * are exceeded...
2562 */
2563 if (level == 0 && !result->enable) {
2564 if (result->pri_val > max->pri)
2565 drm_dbg_kms(&i915->drm,
2566 "Primary WM%d too large %u (max %u)\n",
2567 level, result->pri_val, max->pri);
2568 if (result->spr_val > max->spr)
2569 drm_dbg_kms(&i915->drm,
2570 "Sprite WM%d too large %u (max %u)\n",
2571 level, result->spr_val, max->spr);
2572 if (result->cur_val > max->cur)
2573 drm_dbg_kms(&i915->drm,
2574 "Cursor WM%d too large %u (max %u)\n",
2575 level, result->cur_val, max->cur);
2576
2577 result->pri_val = min_t(u32, result->pri_val, max->pri);
2578 result->spr_val = min_t(u32, result->spr_val, max->spr);
2579 result->cur_val = min_t(u32, result->cur_val, max->cur);
2580 result->enable = true;
2581 }
2582
2583 return ret;
2584 }
2585
ilk_compute_wm_level(const struct drm_i915_private * dev_priv,const struct intel_crtc * crtc,int level,struct intel_crtc_state * crtc_state,const struct intel_plane_state * pristate,const struct intel_plane_state * sprstate,const struct intel_plane_state * curstate,struct intel_wm_level * result)2586 static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
2587 const struct intel_crtc *crtc,
2588 int level,
2589 struct intel_crtc_state *crtc_state,
2590 const struct intel_plane_state *pristate,
2591 const struct intel_plane_state *sprstate,
2592 const struct intel_plane_state *curstate,
2593 struct intel_wm_level *result)
2594 {
2595 u16 pri_latency = dev_priv->display.wm.pri_latency[level];
2596 u16 spr_latency = dev_priv->display.wm.spr_latency[level];
2597 u16 cur_latency = dev_priv->display.wm.cur_latency[level];
2598
2599 /* WM1+ latency values stored in 0.5us units */
2600 if (level > 0) {
2601 pri_latency *= 5;
2602 spr_latency *= 5;
2603 cur_latency *= 5;
2604 }
2605
2606 if (pristate) {
2607 result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,
2608 pri_latency, level);
2609 result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);
2610 }
2611
2612 if (sprstate)
2613 result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);
2614
2615 if (curstate)
2616 result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);
2617
2618 result->enable = true;
2619 }
2620
hsw_read_wm_latency(struct drm_i915_private * i915,u16 wm[])2621 static void hsw_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2622 {
2623 u64 sskpd;
2624
2625 i915->display.wm.num_levels = 5;
2626
2627 sskpd = intel_uncore_read64(&i915->uncore, MCH_SSKPD);
2628
2629 wm[0] = REG_FIELD_GET64(SSKPD_NEW_WM0_MASK_HSW, sskpd);
2630 if (wm[0] == 0)
2631 wm[0] = REG_FIELD_GET64(SSKPD_OLD_WM0_MASK_HSW, sskpd);
2632 wm[1] = REG_FIELD_GET64(SSKPD_WM1_MASK_HSW, sskpd);
2633 wm[2] = REG_FIELD_GET64(SSKPD_WM2_MASK_HSW, sskpd);
2634 wm[3] = REG_FIELD_GET64(SSKPD_WM3_MASK_HSW, sskpd);
2635 wm[4] = REG_FIELD_GET64(SSKPD_WM4_MASK_HSW, sskpd);
2636 }
2637
snb_read_wm_latency(struct drm_i915_private * i915,u16 wm[])2638 static void snb_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2639 {
2640 u32 sskpd;
2641
2642 i915->display.wm.num_levels = 4;
2643
2644 sskpd = intel_uncore_read(&i915->uncore, MCH_SSKPD);
2645
2646 wm[0] = REG_FIELD_GET(SSKPD_WM0_MASK_SNB, sskpd);
2647 wm[1] = REG_FIELD_GET(SSKPD_WM1_MASK_SNB, sskpd);
2648 wm[2] = REG_FIELD_GET(SSKPD_WM2_MASK_SNB, sskpd);
2649 wm[3] = REG_FIELD_GET(SSKPD_WM3_MASK_SNB, sskpd);
2650 }
2651
ilk_read_wm_latency(struct drm_i915_private * i915,u16 wm[])2652 static void ilk_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
2653 {
2654 u32 mltr;
2655
2656 i915->display.wm.num_levels = 3;
2657
2658 mltr = intel_uncore_read(&i915->uncore, MLTR_ILK);
2659
2660 /* ILK primary LP0 latency is 700 ns */
2661 wm[0] = 7;
2662 wm[1] = REG_FIELD_GET(MLTR_WM1_MASK, mltr);
2663 wm[2] = REG_FIELD_GET(MLTR_WM2_MASK, mltr);
2664 }
2665
intel_fixup_spr_wm_latency(struct drm_i915_private * dev_priv,u16 wm[5])2666 static void intel_fixup_spr_wm_latency(struct drm_i915_private *dev_priv,
2667 u16 wm[5])
2668 {
2669 /* ILK sprite LP0 latency is 1300 ns */
2670 if (DISPLAY_VER(dev_priv) == 5)
2671 wm[0] = 13;
2672 }
2673
intel_fixup_cur_wm_latency(struct drm_i915_private * dev_priv,u16 wm[5])2674 static void intel_fixup_cur_wm_latency(struct drm_i915_private *dev_priv,
2675 u16 wm[5])
2676 {
2677 /* ILK cursor LP0 latency is 1300 ns */
2678 if (DISPLAY_VER(dev_priv) == 5)
2679 wm[0] = 13;
2680 }
2681
ilk_increase_wm_latency(struct drm_i915_private * dev_priv,u16 wm[5],u16 min)2682 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
2683 u16 wm[5], u16 min)
2684 {
2685 int level;
2686
2687 if (wm[0] >= min)
2688 return false;
2689
2690 wm[0] = max(wm[0], min);
2691 for (level = 1; level < dev_priv->display.wm.num_levels; level++)
2692 wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));
2693
2694 return true;
2695 }
2696
snb_wm_latency_quirk(struct drm_i915_private * dev_priv)2697 static void snb_wm_latency_quirk(struct drm_i915_private *dev_priv)
2698 {
2699 bool changed;
2700
2701 /*
2702 * The BIOS provided WM memory latency values are often
2703 * inadequate for high resolution displays. Adjust them.
2704 */
2705 changed = ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.pri_latency, 12);
2706 changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.spr_latency, 12);
2707 changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.cur_latency, 12);
2708
2709 if (!changed)
2710 return;
2711
2712 drm_dbg_kms(&dev_priv->drm,
2713 "WM latency values increased to avoid potential underruns\n");
2714 intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
2715 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
2716 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
2717 }
2718
snb_wm_lp3_irq_quirk(struct drm_i915_private * dev_priv)2719 static void snb_wm_lp3_irq_quirk(struct drm_i915_private *dev_priv)
2720 {
2721 /*
2722 * On some SNB machines (Thinkpad X220 Tablet at least)
2723 * LP3 usage can cause vblank interrupts to be lost.
2724 * The DEIIR bit will go high but it looks like the CPU
2725 * never gets interrupted.
2726 *
2727 * It's not clear whether other interrupt source could
2728 * be affected or if this is somehow limited to vblank
2729 * interrupts only. To play it safe we disable LP3
2730 * watermarks entirely.
2731 */
2732 if (dev_priv->display.wm.pri_latency[3] == 0 &&
2733 dev_priv->display.wm.spr_latency[3] == 0 &&
2734 dev_priv->display.wm.cur_latency[3] == 0)
2735 return;
2736
2737 dev_priv->display.wm.pri_latency[3] = 0;
2738 dev_priv->display.wm.spr_latency[3] = 0;
2739 dev_priv->display.wm.cur_latency[3] = 0;
2740
2741 drm_dbg_kms(&dev_priv->drm,
2742 "LP3 watermarks disabled due to potential for lost interrupts\n");
2743 intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
2744 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
2745 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
2746 }
2747
ilk_setup_wm_latency(struct drm_i915_private * dev_priv)2748 static void ilk_setup_wm_latency(struct drm_i915_private *dev_priv)
2749 {
2750 if (IS_BROADWELL(dev_priv) || IS_HASWELL(dev_priv))
2751 hsw_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
2752 else if (DISPLAY_VER(dev_priv) >= 6)
2753 snb_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
2754 else
2755 ilk_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);
2756
2757 memcpy(dev_priv->display.wm.spr_latency, dev_priv->display.wm.pri_latency,
2758 sizeof(dev_priv->display.wm.pri_latency));
2759 memcpy(dev_priv->display.wm.cur_latency, dev_priv->display.wm.pri_latency,
2760 sizeof(dev_priv->display.wm.pri_latency));
2761
2762 intel_fixup_spr_wm_latency(dev_priv, dev_priv->display.wm.spr_latency);
2763 intel_fixup_cur_wm_latency(dev_priv, dev_priv->display.wm.cur_latency);
2764
2765 intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);
2766 intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);
2767 intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);
2768
2769 if (DISPLAY_VER(dev_priv) == 6) {
2770 snb_wm_latency_quirk(dev_priv);
2771 snb_wm_lp3_irq_quirk(dev_priv);
2772 }
2773 }
2774
ilk_validate_pipe_wm(struct drm_i915_private * dev_priv,struct intel_pipe_wm * pipe_wm)2775 static bool ilk_validate_pipe_wm(struct drm_i915_private *dev_priv,
2776 struct intel_pipe_wm *pipe_wm)
2777 {
2778 /* LP0 watermark maximums depend on this pipe alone */
2779 const struct intel_wm_config config = {
2780 .num_pipes_active = 1,
2781 .sprites_enabled = pipe_wm->sprites_enabled,
2782 .sprites_scaled = pipe_wm->sprites_scaled,
2783 };
2784 struct ilk_wm_maximums max;
2785
2786 /* LP0 watermarks always use 1/2 DDB partitioning */
2787 ilk_compute_wm_maximums(dev_priv, 0, &config, INTEL_DDB_PART_1_2, &max);
2788
2789 /* At least LP0 must be valid */
2790 if (!ilk_validate_wm_level(dev_priv, 0, &max, &pipe_wm->wm[0])) {
2791 drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n");
2792 return false;
2793 }
2794
2795 return true;
2796 }
2797
2798 /* Compute new watermarks for the pipe */
ilk_compute_pipe_wm(struct intel_atomic_state * state,struct intel_crtc * crtc)2799 static int ilk_compute_pipe_wm(struct intel_atomic_state *state,
2800 struct intel_crtc *crtc)
2801 {
2802 struct drm_i915_private *dev_priv = to_i915(state->base.dev);
2803 struct intel_crtc_state *crtc_state =
2804 intel_atomic_get_new_crtc_state(state, crtc);
2805 struct intel_pipe_wm *pipe_wm;
2806 struct intel_plane *plane;
2807 const struct intel_plane_state *plane_state;
2808 const struct intel_plane_state *pristate = NULL;
2809 const struct intel_plane_state *sprstate = NULL;
2810 const struct intel_plane_state *curstate = NULL;
2811 struct ilk_wm_maximums max;
2812 int level, usable_level;
2813
2814 pipe_wm = &crtc_state->wm.ilk.optimal;
2815
2816 intel_atomic_crtc_state_for_each_plane_state(plane, plane_state, crtc_state) {
2817 if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)
2818 pristate = plane_state;
2819 else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)
2820 sprstate = plane_state;
2821 else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)
2822 curstate = plane_state;
2823 }
2824
2825 pipe_wm->pipe_enabled = crtc_state->hw.active;
2826 pipe_wm->sprites_enabled = crtc_state->active_planes & BIT(PLANE_SPRITE0);
2827 pipe_wm->sprites_scaled = crtc_state->scaled_planes & BIT(PLANE_SPRITE0);
2828
2829 usable_level = dev_priv->display.wm.num_levels - 1;
2830
2831 /* ILK/SNB: LP2+ watermarks only w/o sprites */
2832 if (DISPLAY_VER(dev_priv) < 7 && pipe_wm->sprites_enabled)
2833 usable_level = 1;
2834
2835 /* ILK/SNB/IVB: LP1+ watermarks only w/o scaling */
2836 if (pipe_wm->sprites_scaled)
2837 usable_level = 0;
2838
2839 memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));
2840 ilk_compute_wm_level(dev_priv, crtc, 0, crtc_state,
2841 pristate, sprstate, curstate, &pipe_wm->wm[0]);
2842
2843 if (!ilk_validate_pipe_wm(dev_priv, pipe_wm))
2844 return -EINVAL;
2845
2846 ilk_compute_wm_reg_maximums(dev_priv, 1, &max);
2847
2848 for (level = 1; level <= usable_level; level++) {
2849 struct intel_wm_level *wm = &pipe_wm->wm[level];
2850
2851 ilk_compute_wm_level(dev_priv, crtc, level, crtc_state,
2852 pristate, sprstate, curstate, wm);
2853
2854 /*
2855 * Disable any watermark level that exceeds the
2856 * register maximums since such watermarks are
2857 * always invalid.
2858 */
2859 if (!ilk_validate_wm_level(dev_priv, level, &max, wm)) {
2860 memset(wm, 0, sizeof(*wm));
2861 break;
2862 }
2863 }
2864
2865 return 0;
2866 }
2867
2868 /*
2869 * Build a set of 'intermediate' watermark values that satisfy both the old
2870 * state and the new state. These can be programmed to the hardware
2871 * immediately.
2872 */
ilk_compute_intermediate_wm(struct intel_atomic_state * state,struct intel_crtc * crtc)2873 static int ilk_compute_intermediate_wm(struct intel_atomic_state *state,
2874 struct intel_crtc *crtc)
2875 {
2876 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
2877 struct intel_crtc_state *new_crtc_state =
2878 intel_atomic_get_new_crtc_state(state, crtc);
2879 const struct intel_crtc_state *old_crtc_state =
2880 intel_atomic_get_old_crtc_state(state, crtc);
2881 struct intel_pipe_wm *a = &new_crtc_state->wm.ilk.intermediate;
2882 const struct intel_pipe_wm *b = &old_crtc_state->wm.ilk.optimal;
2883 int level;
2884
2885 /*
2886 * Start with the final, target watermarks, then combine with the
2887 * currently active watermarks to get values that are safe both before
2888 * and after the vblank.
2889 */
2890 *a = new_crtc_state->wm.ilk.optimal;
2891 if (!new_crtc_state->hw.active ||
2892 intel_crtc_needs_modeset(new_crtc_state) ||
2893 state->skip_intermediate_wm)
2894 return 0;
2895
2896 a->pipe_enabled |= b->pipe_enabled;
2897 a->sprites_enabled |= b->sprites_enabled;
2898 a->sprites_scaled |= b->sprites_scaled;
2899
2900 for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
2901 struct intel_wm_level *a_wm = &a->wm[level];
2902 const struct intel_wm_level *b_wm = &b->wm[level];
2903
2904 a_wm->enable &= b_wm->enable;
2905 a_wm->pri_val = max(a_wm->pri_val, b_wm->pri_val);
2906 a_wm->spr_val = max(a_wm->spr_val, b_wm->spr_val);
2907 a_wm->cur_val = max(a_wm->cur_val, b_wm->cur_val);
2908 a_wm->fbc_val = max(a_wm->fbc_val, b_wm->fbc_val);
2909 }
2910
2911 /*
2912 * We need to make sure that these merged watermark values are
2913 * actually a valid configuration themselves. If they're not,
2914 * there's no safe way to transition from the old state to
2915 * the new state, so we need to fail the atomic transaction.
2916 */
2917 if (!ilk_validate_pipe_wm(dev_priv, a))
2918 return -EINVAL;
2919
2920 /*
2921 * If our intermediate WM are identical to the final WM, then we can
2922 * omit the post-vblank programming; only update if it's different.
2923 */
2924 if (memcmp(a, &new_crtc_state->wm.ilk.optimal, sizeof(*a)) != 0)
2925 new_crtc_state->wm.need_postvbl_update = true;
2926
2927 return 0;
2928 }
2929
2930 /*
2931 * Merge the watermarks from all active pipes for a specific level.
2932 */
ilk_merge_wm_level(struct drm_i915_private * dev_priv,int level,struct intel_wm_level * ret_wm)2933 static void ilk_merge_wm_level(struct drm_i915_private *dev_priv,
2934 int level,
2935 struct intel_wm_level *ret_wm)
2936 {
2937 const struct intel_crtc *crtc;
2938
2939 ret_wm->enable = true;
2940
2941 for_each_intel_crtc(&dev_priv->drm, crtc) {
2942 const struct intel_pipe_wm *active = &crtc->wm.active.ilk;
2943 const struct intel_wm_level *wm = &active->wm[level];
2944
2945 if (!active->pipe_enabled)
2946 continue;
2947
2948 /*
2949 * The watermark values may have been used in the past,
2950 * so we must maintain them in the registers for some
2951 * time even if the level is now disabled.
2952 */
2953 if (!wm->enable)
2954 ret_wm->enable = false;
2955
2956 ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);
2957 ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);
2958 ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);
2959 ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);
2960 }
2961 }
2962
2963 /*
2964 * Merge all low power watermarks for all active pipes.
2965 */
ilk_wm_merge(struct drm_i915_private * dev_priv,const struct intel_wm_config * config,const struct ilk_wm_maximums * max,struct intel_pipe_wm * merged)2966 static void ilk_wm_merge(struct drm_i915_private *dev_priv,
2967 const struct intel_wm_config *config,
2968 const struct ilk_wm_maximums *max,
2969 struct intel_pipe_wm *merged)
2970 {
2971 int level, num_levels = dev_priv->display.wm.num_levels;
2972 int last_enabled_level = num_levels - 1;
2973
2974 /* ILK/SNB/IVB: LP1+ watermarks only w/ single pipe */
2975 if ((DISPLAY_VER(dev_priv) < 7 || IS_IVYBRIDGE(dev_priv)) &&
2976 config->num_pipes_active > 1)
2977 last_enabled_level = 0;
2978
2979 /* ILK: FBC WM must be disabled always */
2980 merged->fbc_wm_enabled = DISPLAY_VER(dev_priv) >= 6;
2981
2982 /* merge each WM1+ level */
2983 for (level = 1; level < num_levels; level++) {
2984 struct intel_wm_level *wm = &merged->wm[level];
2985
2986 ilk_merge_wm_level(dev_priv, level, wm);
2987
2988 if (level > last_enabled_level)
2989 wm->enable = false;
2990 else if (!ilk_validate_wm_level(dev_priv, level, max, wm))
2991 /* make sure all following levels get disabled */
2992 last_enabled_level = level - 1;
2993
2994 /*
2995 * The spec says it is preferred to disable
2996 * FBC WMs instead of disabling a WM level.
2997 */
2998 if (wm->fbc_val > max->fbc) {
2999 if (wm->enable)
3000 merged->fbc_wm_enabled = false;
3001 wm->fbc_val = 0;
3002 }
3003 }
3004
3005 /* ILK: LP2+ must be disabled when FBC WM is disabled but FBC enabled */
3006 if (DISPLAY_VER(dev_priv) == 5 && HAS_FBC(dev_priv) &&
3007 dev_priv->display.params.enable_fbc && !merged->fbc_wm_enabled) {
3008 for (level = 2; level < num_levels; level++) {
3009 struct intel_wm_level *wm = &merged->wm[level];
3010
3011 wm->enable = false;
3012 }
3013 }
3014 }
3015
ilk_wm_lp_to_level(int wm_lp,const struct intel_pipe_wm * pipe_wm)3016 static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
3017 {
3018 /* LP1,LP2,LP3 levels are either 1,2,3 or 1,3,4 */
3019 return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);
3020 }
3021
3022 /* The value we need to program into the WM_LPx latency field */
ilk_wm_lp_latency(struct drm_i915_private * dev_priv,int level)3023 static unsigned int ilk_wm_lp_latency(struct drm_i915_private *dev_priv,
3024 int level)
3025 {
3026 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3027 return 2 * level;
3028 else
3029 return dev_priv->display.wm.pri_latency[level];
3030 }
3031
ilk_compute_wm_results(struct drm_i915_private * dev_priv,const struct intel_pipe_wm * merged,enum intel_ddb_partitioning partitioning,struct ilk_wm_values * results)3032 static void ilk_compute_wm_results(struct drm_i915_private *dev_priv,
3033 const struct intel_pipe_wm *merged,
3034 enum intel_ddb_partitioning partitioning,
3035 struct ilk_wm_values *results)
3036 {
3037 struct intel_crtc *crtc;
3038 int level, wm_lp;
3039
3040 results->enable_fbc_wm = merged->fbc_wm_enabled;
3041 results->partitioning = partitioning;
3042
3043 /* LP1+ register values */
3044 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3045 const struct intel_wm_level *r;
3046
3047 level = ilk_wm_lp_to_level(wm_lp, merged);
3048
3049 r = &merged->wm[level];
3050
3051 /*
3052 * Maintain the watermark values even if the level is
3053 * disabled. Doing otherwise could cause underruns.
3054 */
3055 results->wm_lp[wm_lp - 1] =
3056 WM_LP_LATENCY(ilk_wm_lp_latency(dev_priv, level)) |
3057 WM_LP_PRIMARY(r->pri_val) |
3058 WM_LP_CURSOR(r->cur_val);
3059
3060 if (r->enable)
3061 results->wm_lp[wm_lp - 1] |= WM_LP_ENABLE;
3062
3063 if (DISPLAY_VER(dev_priv) >= 8)
3064 results->wm_lp[wm_lp - 1] |= WM_LP_FBC_BDW(r->fbc_val);
3065 else
3066 results->wm_lp[wm_lp - 1] |= WM_LP_FBC_ILK(r->fbc_val);
3067
3068 results->wm_lp_spr[wm_lp - 1] = WM_LP_SPRITE(r->spr_val);
3069
3070 /*
3071 * Always set WM_LP_SPRITE_EN when spr_val != 0, even if the
3072 * level is disabled. Doing otherwise could cause underruns.
3073 */
3074 if (DISPLAY_VER(dev_priv) < 7 && r->spr_val) {
3075 drm_WARN_ON(&dev_priv->drm, wm_lp != 1);
3076 results->wm_lp_spr[wm_lp - 1] |= WM_LP_SPRITE_ENABLE;
3077 }
3078 }
3079
3080 /* LP0 register values */
3081 for_each_intel_crtc(&dev_priv->drm, crtc) {
3082 enum pipe pipe = crtc->pipe;
3083 const struct intel_pipe_wm *pipe_wm = &crtc->wm.active.ilk;
3084 const struct intel_wm_level *r = &pipe_wm->wm[0];
3085
3086 if (drm_WARN_ON(&dev_priv->drm, !r->enable))
3087 continue;
3088
3089 results->wm_pipe[pipe] =
3090 WM0_PIPE_PRIMARY(r->pri_val) |
3091 WM0_PIPE_SPRITE(r->spr_val) |
3092 WM0_PIPE_CURSOR(r->cur_val);
3093 }
3094 }
3095
3096 /*
3097 * Find the result with the highest level enabled. Check for enable_fbc_wm in
3098 * case both are at the same level. Prefer r1 in case they're the same.
3099 */
3100 static struct intel_pipe_wm *
ilk_find_best_result(struct drm_i915_private * dev_priv,struct intel_pipe_wm * r1,struct intel_pipe_wm * r2)3101 ilk_find_best_result(struct drm_i915_private *dev_priv,
3102 struct intel_pipe_wm *r1,
3103 struct intel_pipe_wm *r2)
3104 {
3105 int level, level1 = 0, level2 = 0;
3106
3107 for (level = 1; level < dev_priv->display.wm.num_levels; level++) {
3108 if (r1->wm[level].enable)
3109 level1 = level;
3110 if (r2->wm[level].enable)
3111 level2 = level;
3112 }
3113
3114 if (level1 == level2) {
3115 if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)
3116 return r2;
3117 else
3118 return r1;
3119 } else if (level1 > level2) {
3120 return r1;
3121 } else {
3122 return r2;
3123 }
3124 }
3125
3126 /* dirty bits used to track which watermarks need changes */
3127 #define WM_DIRTY_PIPE(pipe) (1 << (pipe))
3128 #define WM_DIRTY_LP(wm_lp) (1 << (15 + (wm_lp)))
3129 #define WM_DIRTY_LP_ALL (WM_DIRTY_LP(1) | WM_DIRTY_LP(2) | WM_DIRTY_LP(3))
3130 #define WM_DIRTY_FBC (1 << 24)
3131 #define WM_DIRTY_DDB (1 << 25)
3132
ilk_compute_wm_dirty(struct drm_i915_private * dev_priv,const struct ilk_wm_values * old,const struct ilk_wm_values * new)3133 static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
3134 const struct ilk_wm_values *old,
3135 const struct ilk_wm_values *new)
3136 {
3137 unsigned int dirty = 0;
3138 enum pipe pipe;
3139 int wm_lp;
3140
3141 for_each_pipe(dev_priv, pipe) {
3142 if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {
3143 dirty |= WM_DIRTY_PIPE(pipe);
3144 /* Must disable LP1+ watermarks too */
3145 dirty |= WM_DIRTY_LP_ALL;
3146 }
3147 }
3148
3149 if (old->enable_fbc_wm != new->enable_fbc_wm) {
3150 dirty |= WM_DIRTY_FBC;
3151 /* Must disable LP1+ watermarks too */
3152 dirty |= WM_DIRTY_LP_ALL;
3153 }
3154
3155 if (old->partitioning != new->partitioning) {
3156 dirty |= WM_DIRTY_DDB;
3157 /* Must disable LP1+ watermarks too */
3158 dirty |= WM_DIRTY_LP_ALL;
3159 }
3160
3161 /* LP1+ watermarks already deemed dirty, no need to continue */
3162 if (dirty & WM_DIRTY_LP_ALL)
3163 return dirty;
3164
3165 /* Find the lowest numbered LP1+ watermark in need of an update... */
3166 for (wm_lp = 1; wm_lp <= 3; wm_lp++) {
3167 if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||
3168 old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])
3169 break;
3170 }
3171
3172 /* ...and mark it and all higher numbered LP1+ watermarks as dirty */
3173 for (; wm_lp <= 3; wm_lp++)
3174 dirty |= WM_DIRTY_LP(wm_lp);
3175
3176 return dirty;
3177 }
3178
_ilk_disable_lp_wm(struct drm_i915_private * dev_priv,unsigned int dirty)3179 static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
3180 unsigned int dirty)
3181 {
3182 struct ilk_wm_values *previous = &dev_priv->display.wm.hw;
3183 bool changed = false;
3184
3185 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM_LP_ENABLE) {
3186 previous->wm_lp[2] &= ~WM_LP_ENABLE;
3187 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, previous->wm_lp[2]);
3188 changed = true;
3189 }
3190 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM_LP_ENABLE) {
3191 previous->wm_lp[1] &= ~WM_LP_ENABLE;
3192 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, previous->wm_lp[1]);
3193 changed = true;
3194 }
3195 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM_LP_ENABLE) {
3196 previous->wm_lp[0] &= ~WM_LP_ENABLE;
3197 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, previous->wm_lp[0]);
3198 changed = true;
3199 }
3200
3201 /*
3202 * Don't touch WM_LP_SPRITE_ENABLE here.
3203 * Doing so could cause underruns.
3204 */
3205
3206 return changed;
3207 }
3208
3209 /*
3210 * The spec says we shouldn't write when we don't need, because every write
3211 * causes WMs to be re-evaluated, expending some power.
3212 */
ilk_write_wm_values(struct drm_i915_private * dev_priv,struct ilk_wm_values * results)3213 static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
3214 struct ilk_wm_values *results)
3215 {
3216 struct ilk_wm_values *previous = &dev_priv->display.wm.hw;
3217 unsigned int dirty;
3218
3219 dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
3220 if (!dirty)
3221 return;
3222
3223 _ilk_disable_lp_wm(dev_priv, dirty);
3224
3225 if (dirty & WM_DIRTY_PIPE(PIPE_A))
3226 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_A), results->wm_pipe[0]);
3227 if (dirty & WM_DIRTY_PIPE(PIPE_B))
3228 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_B), results->wm_pipe[1]);
3229 if (dirty & WM_DIRTY_PIPE(PIPE_C))
3230 intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_C), results->wm_pipe[2]);
3231
3232 if (dirty & WM_DIRTY_DDB) {
3233 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3234 intel_uncore_rmw(&dev_priv->uncore, WM_MISC, WM_MISC_DATA_PARTITION_5_6,
3235 results->partitioning == INTEL_DDB_PART_1_2 ? 0 :
3236 WM_MISC_DATA_PARTITION_5_6);
3237 else
3238 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL2, DISP_DATA_PARTITION_5_6,
3239 results->partitioning == INTEL_DDB_PART_1_2 ? 0 :
3240 DISP_DATA_PARTITION_5_6);
3241 }
3242
3243 if (dirty & WM_DIRTY_FBC)
3244 intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, DISP_FBC_WM_DIS,
3245 results->enable_fbc_wm ? 0 : DISP_FBC_WM_DIS);
3246
3247 if (dirty & WM_DIRTY_LP(1) &&
3248 previous->wm_lp_spr[0] != results->wm_lp_spr[0])
3249 intel_uncore_write(&dev_priv->uncore, WM1S_LP_ILK, results->wm_lp_spr[0]);
3250
3251 if (DISPLAY_VER(dev_priv) >= 7) {
3252 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])
3253 intel_uncore_write(&dev_priv->uncore, WM2S_LP_IVB, results->wm_lp_spr[1]);
3254 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])
3255 intel_uncore_write(&dev_priv->uncore, WM3S_LP_IVB, results->wm_lp_spr[2]);
3256 }
3257
3258 if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])
3259 intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, results->wm_lp[0]);
3260 if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])
3261 intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, results->wm_lp[1]);
3262 if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])
3263 intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, results->wm_lp[2]);
3264
3265 dev_priv->display.wm.hw = *results;
3266 }
3267
ilk_disable_lp_wm(struct drm_i915_private * dev_priv)3268 bool ilk_disable_lp_wm(struct drm_i915_private *dev_priv)
3269 {
3270 return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
3271 }
3272
ilk_compute_wm_config(struct drm_i915_private * dev_priv,struct intel_wm_config * config)3273 static void ilk_compute_wm_config(struct drm_i915_private *dev_priv,
3274 struct intel_wm_config *config)
3275 {
3276 struct intel_crtc *crtc;
3277
3278 /* Compute the currently _active_ config */
3279 for_each_intel_crtc(&dev_priv->drm, crtc) {
3280 const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;
3281
3282 if (!wm->pipe_enabled)
3283 continue;
3284
3285 config->sprites_enabled |= wm->sprites_enabled;
3286 config->sprites_scaled |= wm->sprites_scaled;
3287 config->num_pipes_active++;
3288 }
3289 }
3290
ilk_program_watermarks(struct drm_i915_private * dev_priv)3291 static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
3292 {
3293 struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
3294 struct ilk_wm_maximums max;
3295 struct intel_wm_config config = {};
3296 struct ilk_wm_values results = {};
3297 enum intel_ddb_partitioning partitioning;
3298
3299 ilk_compute_wm_config(dev_priv, &config);
3300
3301 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_1_2, &max);
3302 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_1_2);
3303
3304 /* 5/6 split only in single pipe config on IVB+ */
3305 if (DISPLAY_VER(dev_priv) >= 7 &&
3306 config.num_pipes_active == 1 && config.sprites_enabled) {
3307 ilk_compute_wm_maximums(dev_priv, 1, &config, INTEL_DDB_PART_5_6, &max);
3308 ilk_wm_merge(dev_priv, &config, &max, &lp_wm_5_6);
3309
3310 best_lp_wm = ilk_find_best_result(dev_priv, &lp_wm_1_2, &lp_wm_5_6);
3311 } else {
3312 best_lp_wm = &lp_wm_1_2;
3313 }
3314
3315 partitioning = (best_lp_wm == &lp_wm_1_2) ?
3316 INTEL_DDB_PART_1_2 : INTEL_DDB_PART_5_6;
3317
3318 ilk_compute_wm_results(dev_priv, best_lp_wm, partitioning, &results);
3319
3320 ilk_write_wm_values(dev_priv, &results);
3321 }
3322
ilk_initial_watermarks(struct intel_atomic_state * state,struct intel_crtc * crtc)3323 static void ilk_initial_watermarks(struct intel_atomic_state *state,
3324 struct intel_crtc *crtc)
3325 {
3326 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3327 const struct intel_crtc_state *crtc_state =
3328 intel_atomic_get_new_crtc_state(state, crtc);
3329
3330 mutex_lock(&dev_priv->display.wm.wm_mutex);
3331 crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;
3332 ilk_program_watermarks(dev_priv);
3333 mutex_unlock(&dev_priv->display.wm.wm_mutex);
3334 }
3335
ilk_optimize_watermarks(struct intel_atomic_state * state,struct intel_crtc * crtc)3336 static void ilk_optimize_watermarks(struct intel_atomic_state *state,
3337 struct intel_crtc *crtc)
3338 {
3339 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
3340 const struct intel_crtc_state *crtc_state =
3341 intel_atomic_get_new_crtc_state(state, crtc);
3342
3343 if (!crtc_state->wm.need_postvbl_update)
3344 return;
3345
3346 mutex_lock(&dev_priv->display.wm.wm_mutex);
3347 crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;
3348 ilk_program_watermarks(dev_priv);
3349 mutex_unlock(&dev_priv->display.wm.wm_mutex);
3350 }
3351
ilk_pipe_wm_get_hw_state(struct intel_crtc * crtc)3352 static void ilk_pipe_wm_get_hw_state(struct intel_crtc *crtc)
3353 {
3354 struct drm_device *dev = crtc->base.dev;
3355 struct drm_i915_private *dev_priv = to_i915(dev);
3356 struct ilk_wm_values *hw = &dev_priv->display.wm.hw;
3357 struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);
3358 struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;
3359 enum pipe pipe = crtc->pipe;
3360
3361 hw->wm_pipe[pipe] = intel_uncore_read(&dev_priv->uncore, WM0_PIPE_ILK(pipe));
3362
3363 memset(active, 0, sizeof(*active));
3364
3365 active->pipe_enabled = crtc->active;
3366
3367 if (active->pipe_enabled) {
3368 u32 tmp = hw->wm_pipe[pipe];
3369
3370 /*
3371 * For active pipes LP0 watermark is marked as
3372 * enabled, and LP1+ watermaks as disabled since
3373 * we can't really reverse compute them in case
3374 * multiple pipes are active.
3375 */
3376 active->wm[0].enable = true;
3377 active->wm[0].pri_val = REG_FIELD_GET(WM0_PIPE_PRIMARY_MASK, tmp);
3378 active->wm[0].spr_val = REG_FIELD_GET(WM0_PIPE_SPRITE_MASK, tmp);
3379 active->wm[0].cur_val = REG_FIELD_GET(WM0_PIPE_CURSOR_MASK, tmp);
3380 } else {
3381 int level;
3382
3383 /*
3384 * For inactive pipes, all watermark levels
3385 * should be marked as enabled but zeroed,
3386 * which is what we'd compute them to.
3387 */
3388 for (level = 0; level < dev_priv->display.wm.num_levels; level++)
3389 active->wm[level].enable = true;
3390 }
3391
3392 crtc->wm.active.ilk = *active;
3393 }
3394
ilk_sanitize_watermarks_add_affected(struct drm_atomic_state * state)3395 static int ilk_sanitize_watermarks_add_affected(struct drm_atomic_state *state)
3396 {
3397 struct drm_plane *plane;
3398 struct intel_crtc *crtc;
3399
3400 for_each_intel_crtc(state->dev, crtc) {
3401 struct intel_crtc_state *crtc_state;
3402
3403 crtc_state = intel_atomic_get_crtc_state(state, crtc);
3404 if (IS_ERR(crtc_state))
3405 return PTR_ERR(crtc_state);
3406
3407 if (crtc_state->hw.active) {
3408 /*
3409 * Preserve the inherited flag to avoid
3410 * taking the full modeset path.
3411 */
3412 crtc_state->inherited = true;
3413 }
3414 }
3415
3416 drm_for_each_plane(plane, state->dev) {
3417 struct drm_plane_state *plane_state;
3418
3419 plane_state = drm_atomic_get_plane_state(state, plane);
3420 if (IS_ERR(plane_state))
3421 return PTR_ERR(plane_state);
3422 }
3423
3424 return 0;
3425 }
3426
3427 /*
3428 * Calculate what we think the watermarks should be for the state we've read
3429 * out of the hardware and then immediately program those watermarks so that
3430 * we ensure the hardware settings match our internal state.
3431 *
3432 * We can calculate what we think WM's should be by creating a duplicate of the
3433 * current state (which was constructed during hardware readout) and running it
3434 * through the atomic check code to calculate new watermark values in the
3435 * state object.
3436 */
ilk_wm_sanitize(struct drm_i915_private * dev_priv)3437 void ilk_wm_sanitize(struct drm_i915_private *dev_priv)
3438 {
3439 struct drm_atomic_state *state;
3440 struct intel_atomic_state *intel_state;
3441 struct intel_crtc *crtc;
3442 struct intel_crtc_state *crtc_state;
3443 struct drm_modeset_acquire_ctx ctx;
3444 int ret;
3445 int i;
3446
3447 /* Only supported on platforms that use atomic watermark design */
3448 if (!dev_priv->display.funcs.wm->optimize_watermarks)
3449 return;
3450
3451 if (drm_WARN_ON(&dev_priv->drm, DISPLAY_VER(dev_priv) >= 9))
3452 return;
3453
3454 state = drm_atomic_state_alloc(&dev_priv->drm);
3455 if (drm_WARN_ON(&dev_priv->drm, !state))
3456 return;
3457
3458 intel_state = to_intel_atomic_state(state);
3459
3460 drm_modeset_acquire_init(&ctx, 0);
3461
3462 state->acquire_ctx = &ctx;
3463 to_intel_atomic_state(state)->internal = true;
3464
3465 retry:
3466 /*
3467 * Hardware readout is the only time we don't want to calculate
3468 * intermediate watermarks (since we don't trust the current
3469 * watermarks).
3470 */
3471 if (!HAS_GMCH(dev_priv))
3472 intel_state->skip_intermediate_wm = true;
3473
3474 ret = ilk_sanitize_watermarks_add_affected(state);
3475 if (ret)
3476 goto fail;
3477
3478 ret = intel_atomic_check(&dev_priv->drm, state);
3479 if (ret)
3480 goto fail;
3481
3482 /* Write calculated watermark values back */
3483 for_each_new_intel_crtc_in_state(intel_state, crtc, crtc_state, i) {
3484 crtc_state->wm.need_postvbl_update = true;
3485 intel_optimize_watermarks(intel_state, crtc);
3486
3487 to_intel_crtc_state(crtc->base.state)->wm = crtc_state->wm;
3488 }
3489
3490 fail:
3491 if (ret == -EDEADLK) {
3492 drm_atomic_state_clear(state);
3493 drm_modeset_backoff(&ctx);
3494 goto retry;
3495 }
3496
3497 /*
3498 * If we fail here, it means that the hardware appears to be
3499 * programmed in a way that shouldn't be possible, given our
3500 * understanding of watermark requirements. This might mean a
3501 * mistake in the hardware readout code or a mistake in the
3502 * watermark calculations for a given platform. Raise a WARN
3503 * so that this is noticeable.
3504 *
3505 * If this actually happens, we'll have to just leave the
3506 * BIOS-programmed watermarks untouched and hope for the best.
3507 */
3508 drm_WARN(&dev_priv->drm, ret,
3509 "Could not determine valid watermarks for inherited state\n");
3510
3511 drm_atomic_state_put(state);
3512
3513 drm_modeset_drop_locks(&ctx);
3514 drm_modeset_acquire_fini(&ctx);
3515 }
3516
3517 #define _FW_WM(value, plane) \
3518 (((value) & DSPFW_ ## plane ## _MASK) >> DSPFW_ ## plane ## _SHIFT)
3519 #define _FW_WM_VLV(value, plane) \
3520 (((value) & DSPFW_ ## plane ## _MASK_VLV) >> DSPFW_ ## plane ## _SHIFT)
3521
g4x_read_wm_values(struct drm_i915_private * dev_priv,struct g4x_wm_values * wm)3522 static void g4x_read_wm_values(struct drm_i915_private *dev_priv,
3523 struct g4x_wm_values *wm)
3524 {
3525 u32 tmp;
3526
3527 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1(dev_priv));
3528 wm->sr.plane = _FW_WM(tmp, SR);
3529 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
3530 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);
3531 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);
3532
3533 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2(dev_priv));
3534 wm->fbc_en = tmp & DSPFW_FBC_SR_EN;
3535 wm->sr.fbc = _FW_WM(tmp, FBC_SR);
3536 wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);
3537 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);
3538 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
3539 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);
3540
3541 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3(dev_priv));
3542 wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;
3543 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
3544 wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);
3545 wm->hpll.plane = _FW_WM(tmp, HPLL_SR);
3546 }
3547
vlv_read_wm_values(struct drm_i915_private * dev_priv,struct vlv_wm_values * wm)3548 static void vlv_read_wm_values(struct drm_i915_private *dev_priv,
3549 struct vlv_wm_values *wm)
3550 {
3551 enum pipe pipe;
3552 u32 tmp;
3553
3554 for_each_pipe(dev_priv, pipe) {
3555 tmp = intel_uncore_read(&dev_priv->uncore, VLV_DDL(pipe));
3556
3557 wm->ddl[pipe].plane[PLANE_PRIMARY] =
3558 (tmp >> DDL_PLANE_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3559 wm->ddl[pipe].plane[PLANE_CURSOR] =
3560 (tmp >> DDL_CURSOR_SHIFT) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3561 wm->ddl[pipe].plane[PLANE_SPRITE0] =
3562 (tmp >> DDL_SPRITE_SHIFT(0)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3563 wm->ddl[pipe].plane[PLANE_SPRITE1] =
3564 (tmp >> DDL_SPRITE_SHIFT(1)) & (DDL_PRECISION_HIGH | DRAIN_LATENCY_MASK);
3565 }
3566
3567 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1(dev_priv));
3568 wm->sr.plane = _FW_WM(tmp, SR);
3569 wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);
3570 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);
3571 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);
3572
3573 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2(dev_priv));
3574 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);
3575 wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);
3576 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);
3577
3578 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3(dev_priv));
3579 wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);
3580
3581 if (IS_CHERRYVIEW(dev_priv)) {
3582 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7_CHV);
3583 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
3584 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
3585
3586 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW8_CHV);
3587 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);
3588 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);
3589
3590 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW9_CHV);
3591 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);
3592 wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);
3593
3594 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
3595 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
3596 wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;
3597 wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;
3598 wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;
3599 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
3600 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
3601 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
3602 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
3603 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
3604 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
3605 } else {
3606 tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7);
3607 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);
3608 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);
3609
3610 tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);
3611 wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;
3612 wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;
3613 wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;
3614 wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;
3615 wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;
3616 wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;
3617 wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;
3618 }
3619 }
3620
3621 #undef _FW_WM
3622 #undef _FW_WM_VLV
3623
g4x_wm_get_hw_state(struct drm_i915_private * dev_priv)3624 static void g4x_wm_get_hw_state(struct drm_i915_private *dev_priv)
3625 {
3626 struct g4x_wm_values *wm = &dev_priv->display.wm.g4x;
3627 struct intel_crtc *crtc;
3628
3629 g4x_read_wm_values(dev_priv, wm);
3630
3631 wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;
3632
3633 for_each_intel_crtc(&dev_priv->drm, crtc) {
3634 struct intel_crtc_state *crtc_state =
3635 to_intel_crtc_state(crtc->base.state);
3636 struct g4x_wm_state *active = &crtc->wm.active.g4x;
3637 struct g4x_pipe_wm *raw;
3638 enum pipe pipe = crtc->pipe;
3639 enum plane_id plane_id;
3640 int level, max_level;
3641
3642 active->cxsr = wm->cxsr;
3643 active->hpll_en = wm->hpll_en;
3644 active->fbc_en = wm->fbc_en;
3645
3646 active->sr = wm->sr;
3647 active->hpll = wm->hpll;
3648
3649 for_each_plane_id_on_crtc(crtc, plane_id) {
3650 active->wm.plane[plane_id] =
3651 wm->pipe[pipe].plane[plane_id];
3652 }
3653
3654 if (wm->cxsr && wm->hpll_en)
3655 max_level = G4X_WM_LEVEL_HPLL;
3656 else if (wm->cxsr)
3657 max_level = G4X_WM_LEVEL_SR;
3658 else
3659 max_level = G4X_WM_LEVEL_NORMAL;
3660
3661 level = G4X_WM_LEVEL_NORMAL;
3662 raw = &crtc_state->wm.g4x.raw[level];
3663 for_each_plane_id_on_crtc(crtc, plane_id)
3664 raw->plane[plane_id] = active->wm.plane[plane_id];
3665
3666 level = G4X_WM_LEVEL_SR;
3667 if (level > max_level)
3668 goto out;
3669
3670 raw = &crtc_state->wm.g4x.raw[level];
3671 raw->plane[PLANE_PRIMARY] = active->sr.plane;
3672 raw->plane[PLANE_CURSOR] = active->sr.cursor;
3673 raw->plane[PLANE_SPRITE0] = 0;
3674 raw->fbc = active->sr.fbc;
3675
3676 level = G4X_WM_LEVEL_HPLL;
3677 if (level > max_level)
3678 goto out;
3679
3680 raw = &crtc_state->wm.g4x.raw[level];
3681 raw->plane[PLANE_PRIMARY] = active->hpll.plane;
3682 raw->plane[PLANE_CURSOR] = active->hpll.cursor;
3683 raw->plane[PLANE_SPRITE0] = 0;
3684 raw->fbc = active->hpll.fbc;
3685
3686 level++;
3687 out:
3688 for_each_plane_id_on_crtc(crtc, plane_id)
3689 g4x_raw_plane_wm_set(crtc_state, level,
3690 plane_id, USHRT_MAX);
3691 g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);
3692
3693 g4x_invalidate_wms(crtc, active, level);
3694
3695 crtc_state->wm.g4x.optimal = *active;
3696 crtc_state->wm.g4x.intermediate = *active;
3697
3698 drm_dbg_kms(&dev_priv->drm,
3699 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite=%d\n",
3700 pipe_name(pipe),
3701 wm->pipe[pipe].plane[PLANE_PRIMARY],
3702 wm->pipe[pipe].plane[PLANE_CURSOR],
3703 wm->pipe[pipe].plane[PLANE_SPRITE0]);
3704 }
3705
3706 drm_dbg_kms(&dev_priv->drm,
3707 "Initial SR watermarks: plane=%d, cursor=%d fbc=%d\n",
3708 wm->sr.plane, wm->sr.cursor, wm->sr.fbc);
3709 drm_dbg_kms(&dev_priv->drm,
3710 "Initial HPLL watermarks: plane=%d, SR cursor=%d fbc=%d\n",
3711 wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);
3712 drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n",
3713 str_yes_no(wm->cxsr), str_yes_no(wm->hpll_en),
3714 str_yes_no(wm->fbc_en));
3715 }
3716
g4x_wm_sanitize(struct drm_i915_private * dev_priv)3717 static void g4x_wm_sanitize(struct drm_i915_private *dev_priv)
3718 {
3719 struct intel_plane *plane;
3720 struct intel_crtc *crtc;
3721
3722 mutex_lock(&dev_priv->display.wm.wm_mutex);
3723
3724 for_each_intel_plane(&dev_priv->drm, plane) {
3725 struct intel_crtc *crtc =
3726 intel_crtc_for_pipe(dev_priv, plane->pipe);
3727 struct intel_crtc_state *crtc_state =
3728 to_intel_crtc_state(crtc->base.state);
3729 struct intel_plane_state *plane_state =
3730 to_intel_plane_state(plane->base.state);
3731 enum plane_id plane_id = plane->id;
3732 int level;
3733
3734 if (plane_state->uapi.visible)
3735 continue;
3736
3737 for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
3738 struct g4x_pipe_wm *raw =
3739 &crtc_state->wm.g4x.raw[level];
3740
3741 raw->plane[plane_id] = 0;
3742
3743 if (plane_id == PLANE_PRIMARY)
3744 raw->fbc = 0;
3745 }
3746 }
3747
3748 for_each_intel_crtc(&dev_priv->drm, crtc) {
3749 struct intel_crtc_state *crtc_state =
3750 to_intel_crtc_state(crtc->base.state);
3751 int ret;
3752
3753 ret = _g4x_compute_pipe_wm(crtc_state);
3754 drm_WARN_ON(&dev_priv->drm, ret);
3755
3756 crtc_state->wm.g4x.intermediate =
3757 crtc_state->wm.g4x.optimal;
3758 crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;
3759 }
3760
3761 g4x_program_watermarks(dev_priv);
3762
3763 mutex_unlock(&dev_priv->display.wm.wm_mutex);
3764 }
3765
g4x_wm_get_hw_state_and_sanitize(struct drm_i915_private * i915)3766 static void g4x_wm_get_hw_state_and_sanitize(struct drm_i915_private *i915)
3767 {
3768 g4x_wm_get_hw_state(i915);
3769 g4x_wm_sanitize(i915);
3770 }
3771
vlv_wm_get_hw_state(struct drm_i915_private * dev_priv)3772 static void vlv_wm_get_hw_state(struct drm_i915_private *dev_priv)
3773 {
3774 struct vlv_wm_values *wm = &dev_priv->display.wm.vlv;
3775 struct intel_crtc *crtc;
3776 u32 val;
3777
3778 vlv_read_wm_values(dev_priv, wm);
3779
3780 wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;
3781 wm->level = VLV_WM_LEVEL_PM2;
3782
3783 if (IS_CHERRYVIEW(dev_priv)) {
3784 vlv_punit_get(dev_priv);
3785
3786 val = vlv_punit_read(dev_priv, PUNIT_REG_DSPSSPM);
3787 if (val & DSP_MAXFIFO_PM5_ENABLE)
3788 wm->level = VLV_WM_LEVEL_PM5;
3789
3790 /*
3791 * If DDR DVFS is disabled in the BIOS, Punit
3792 * will never ack the request. So if that happens
3793 * assume we don't have to enable/disable DDR DVFS
3794 * dynamically. To test that just set the REQ_ACK
3795 * bit to poke the Punit, but don't change the
3796 * HIGH/LOW bits so that we don't actually change
3797 * the current state.
3798 */
3799 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
3800 val |= FORCE_DDR_FREQ_REQ_ACK;
3801 vlv_punit_write(dev_priv, PUNIT_REG_DDR_SETUP2, val);
3802
3803 if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2) &
3804 FORCE_DDR_FREQ_REQ_ACK) == 0, 3)) {
3805 drm_dbg_kms(&dev_priv->drm,
3806 "Punit not acking DDR DVFS request, "
3807 "assuming DDR DVFS is disabled\n");
3808 dev_priv->display.wm.num_levels = VLV_WM_LEVEL_PM5 + 1;
3809 } else {
3810 val = vlv_punit_read(dev_priv, PUNIT_REG_DDR_SETUP2);
3811 if ((val & FORCE_DDR_HIGH_FREQ) == 0)
3812 wm->level = VLV_WM_LEVEL_DDR_DVFS;
3813 }
3814
3815 vlv_punit_put(dev_priv);
3816 }
3817
3818 for_each_intel_crtc(&dev_priv->drm, crtc) {
3819 struct intel_crtc_state *crtc_state =
3820 to_intel_crtc_state(crtc->base.state);
3821 struct vlv_wm_state *active = &crtc->wm.active.vlv;
3822 const struct vlv_fifo_state *fifo_state =
3823 &crtc_state->wm.vlv.fifo_state;
3824 enum pipe pipe = crtc->pipe;
3825 enum plane_id plane_id;
3826 int level;
3827
3828 vlv_get_fifo_size(crtc_state);
3829
3830 active->num_levels = wm->level + 1;
3831 active->cxsr = wm->cxsr;
3832
3833 for (level = 0; level < active->num_levels; level++) {
3834 struct g4x_pipe_wm *raw =
3835 &crtc_state->wm.vlv.raw[level];
3836
3837 active->sr[level].plane = wm->sr.plane;
3838 active->sr[level].cursor = wm->sr.cursor;
3839
3840 for_each_plane_id_on_crtc(crtc, plane_id) {
3841 active->wm[level].plane[plane_id] =
3842 wm->pipe[pipe].plane[plane_id];
3843
3844 raw->plane[plane_id] =
3845 vlv_invert_wm_value(active->wm[level].plane[plane_id],
3846 fifo_state->plane[plane_id]);
3847 }
3848 }
3849
3850 for_each_plane_id_on_crtc(crtc, plane_id)
3851 vlv_raw_plane_wm_set(crtc_state, level,
3852 plane_id, USHRT_MAX);
3853 vlv_invalidate_wms(crtc, active, level);
3854
3855 crtc_state->wm.vlv.optimal = *active;
3856 crtc_state->wm.vlv.intermediate = *active;
3857
3858 drm_dbg_kms(&dev_priv->drm,
3859 "Initial watermarks: pipe %c, plane=%d, cursor=%d, sprite0=%d, sprite1=%d\n",
3860 pipe_name(pipe),
3861 wm->pipe[pipe].plane[PLANE_PRIMARY],
3862 wm->pipe[pipe].plane[PLANE_CURSOR],
3863 wm->pipe[pipe].plane[PLANE_SPRITE0],
3864 wm->pipe[pipe].plane[PLANE_SPRITE1]);
3865 }
3866
3867 drm_dbg_kms(&dev_priv->drm,
3868 "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",
3869 wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);
3870 }
3871
vlv_wm_sanitize(struct drm_i915_private * dev_priv)3872 static void vlv_wm_sanitize(struct drm_i915_private *dev_priv)
3873 {
3874 struct intel_plane *plane;
3875 struct intel_crtc *crtc;
3876
3877 mutex_lock(&dev_priv->display.wm.wm_mutex);
3878
3879 for_each_intel_plane(&dev_priv->drm, plane) {
3880 struct intel_crtc *crtc =
3881 intel_crtc_for_pipe(dev_priv, plane->pipe);
3882 struct intel_crtc_state *crtc_state =
3883 to_intel_crtc_state(crtc->base.state);
3884 struct intel_plane_state *plane_state =
3885 to_intel_plane_state(plane->base.state);
3886 enum plane_id plane_id = plane->id;
3887 int level;
3888
3889 if (plane_state->uapi.visible)
3890 continue;
3891
3892 for (level = 0; level < dev_priv->display.wm.num_levels; level++) {
3893 struct g4x_pipe_wm *raw =
3894 &crtc_state->wm.vlv.raw[level];
3895
3896 raw->plane[plane_id] = 0;
3897 }
3898 }
3899
3900 for_each_intel_crtc(&dev_priv->drm, crtc) {
3901 struct intel_crtc_state *crtc_state =
3902 to_intel_crtc_state(crtc->base.state);
3903 int ret;
3904
3905 ret = _vlv_compute_pipe_wm(crtc_state);
3906 drm_WARN_ON(&dev_priv->drm, ret);
3907
3908 crtc_state->wm.vlv.intermediate =
3909 crtc_state->wm.vlv.optimal;
3910 crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;
3911 }
3912
3913 vlv_program_watermarks(dev_priv);
3914
3915 mutex_unlock(&dev_priv->display.wm.wm_mutex);
3916 }
3917
vlv_wm_get_hw_state_and_sanitize(struct drm_i915_private * i915)3918 static void vlv_wm_get_hw_state_and_sanitize(struct drm_i915_private *i915)
3919 {
3920 vlv_wm_get_hw_state(i915);
3921 vlv_wm_sanitize(i915);
3922 }
3923
3924 /*
3925 * FIXME should probably kill this and improve
3926 * the real watermark readout/sanitation instead
3927 */
ilk_init_lp_watermarks(struct drm_i915_private * dev_priv)3928 static void ilk_init_lp_watermarks(struct drm_i915_private *dev_priv)
3929 {
3930 intel_uncore_rmw(&dev_priv->uncore, WM3_LP_ILK, WM_LP_ENABLE, 0);
3931 intel_uncore_rmw(&dev_priv->uncore, WM2_LP_ILK, WM_LP_ENABLE, 0);
3932 intel_uncore_rmw(&dev_priv->uncore, WM1_LP_ILK, WM_LP_ENABLE, 0);
3933
3934 /*
3935 * Don't touch WM_LP_SPRITE_ENABLE here.
3936 * Doing so could cause underruns.
3937 */
3938 }
3939
ilk_wm_get_hw_state(struct drm_i915_private * dev_priv)3940 static void ilk_wm_get_hw_state(struct drm_i915_private *dev_priv)
3941 {
3942 struct ilk_wm_values *hw = &dev_priv->display.wm.hw;
3943 struct intel_crtc *crtc;
3944
3945 ilk_init_lp_watermarks(dev_priv);
3946
3947 for_each_intel_crtc(&dev_priv->drm, crtc)
3948 ilk_pipe_wm_get_hw_state(crtc);
3949
3950 hw->wm_lp[0] = intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK);
3951 hw->wm_lp[1] = intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK);
3952 hw->wm_lp[2] = intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK);
3953
3954 hw->wm_lp_spr[0] = intel_uncore_read(&dev_priv->uncore, WM1S_LP_ILK);
3955 if (DISPLAY_VER(dev_priv) >= 7) {
3956 hw->wm_lp_spr[1] = intel_uncore_read(&dev_priv->uncore, WM2S_LP_IVB);
3957 hw->wm_lp_spr[2] = intel_uncore_read(&dev_priv->uncore, WM3S_LP_IVB);
3958 }
3959
3960 if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
3961 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, WM_MISC) &
3962 WM_MISC_DATA_PARTITION_5_6) ?
3963 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
3964 else if (IS_IVYBRIDGE(dev_priv))
3965 hw->partitioning = (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2) &
3966 DISP_DATA_PARTITION_5_6) ?
3967 INTEL_DDB_PART_5_6 : INTEL_DDB_PART_1_2;
3968
3969 hw->enable_fbc_wm =
3970 !(intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) & DISP_FBC_WM_DIS);
3971 }
3972
3973 static const struct intel_wm_funcs ilk_wm_funcs = {
3974 .compute_pipe_wm = ilk_compute_pipe_wm,
3975 .compute_intermediate_wm = ilk_compute_intermediate_wm,
3976 .initial_watermarks = ilk_initial_watermarks,
3977 .optimize_watermarks = ilk_optimize_watermarks,
3978 .get_hw_state = ilk_wm_get_hw_state,
3979 };
3980
3981 static const struct intel_wm_funcs vlv_wm_funcs = {
3982 .compute_pipe_wm = vlv_compute_pipe_wm,
3983 .compute_intermediate_wm = vlv_compute_intermediate_wm,
3984 .initial_watermarks = vlv_initial_watermarks,
3985 .optimize_watermarks = vlv_optimize_watermarks,
3986 .atomic_update_watermarks = vlv_atomic_update_fifo,
3987 .get_hw_state = vlv_wm_get_hw_state_and_sanitize,
3988 };
3989
3990 static const struct intel_wm_funcs g4x_wm_funcs = {
3991 .compute_pipe_wm = g4x_compute_pipe_wm,
3992 .compute_intermediate_wm = g4x_compute_intermediate_wm,
3993 .initial_watermarks = g4x_initial_watermarks,
3994 .optimize_watermarks = g4x_optimize_watermarks,
3995 .get_hw_state = g4x_wm_get_hw_state_and_sanitize,
3996 };
3997
3998 static const struct intel_wm_funcs pnv_wm_funcs = {
3999 .update_wm = pnv_update_wm,
4000 };
4001
4002 static const struct intel_wm_funcs i965_wm_funcs = {
4003 .update_wm = i965_update_wm,
4004 };
4005
4006 static const struct intel_wm_funcs i9xx_wm_funcs = {
4007 .update_wm = i9xx_update_wm,
4008 };
4009
4010 static const struct intel_wm_funcs i845_wm_funcs = {
4011 .update_wm = i845_update_wm,
4012 };
4013
4014 static const struct intel_wm_funcs nop_funcs = {
4015 };
4016
i9xx_wm_init(struct drm_i915_private * dev_priv)4017 void i9xx_wm_init(struct drm_i915_private *dev_priv)
4018 {
4019 /* For FIFO watermark updates */
4020 if (HAS_PCH_SPLIT(dev_priv)) {
4021 ilk_setup_wm_latency(dev_priv);
4022 dev_priv->display.funcs.wm = &ilk_wm_funcs;
4023 } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
4024 vlv_setup_wm_latency(dev_priv);
4025 dev_priv->display.funcs.wm = &vlv_wm_funcs;
4026 } else if (IS_G4X(dev_priv)) {
4027 g4x_setup_wm_latency(dev_priv);
4028 dev_priv->display.funcs.wm = &g4x_wm_funcs;
4029 } else if (IS_PINEVIEW(dev_priv)) {
4030 if (!pnv_get_cxsr_latency(dev_priv)) {
4031 drm_info(&dev_priv->drm, "Unknown FSB/MEM, disabling CxSR\n");
4032 /* Disable CxSR and never update its watermark again */
4033 intel_set_memory_cxsr(dev_priv, false);
4034 dev_priv->display.funcs.wm = &nop_funcs;
4035 } else {
4036 dev_priv->display.funcs.wm = &pnv_wm_funcs;
4037 }
4038 } else if (DISPLAY_VER(dev_priv) == 4) {
4039 dev_priv->display.funcs.wm = &i965_wm_funcs;
4040 } else if (DISPLAY_VER(dev_priv) == 3) {
4041 dev_priv->display.funcs.wm = &i9xx_wm_funcs;
4042 } else if (DISPLAY_VER(dev_priv) == 2) {
4043 if (INTEL_NUM_PIPES(dev_priv) == 1)
4044 dev_priv->display.funcs.wm = &i845_wm_funcs;
4045 else
4046 dev_priv->display.funcs.wm = &i9xx_wm_funcs;
4047 } else {
4048 drm_err(&dev_priv->drm,
4049 "unexpected fall-through in %s\n", __func__);
4050 dev_priv->display.funcs.wm = &nop_funcs;
4051 }
4052 }
4053