1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2022 Intel Corporation
4 */
5
6 #include <linux/debugfs.h>
7
8 #include <drm/drm_blend.h>
9
10 #include "i915_drv.h"
11 #include "i915_reg.h"
12 #include "i9xx_wm.h"
13 #include "intel_atomic.h"
14 #include "intel_atomic_plane.h"
15 #include "intel_bw.h"
16 #include "intel_cdclk.h"
17 #include "intel_crtc.h"
18 #include "intel_cursor_regs.h"
19 #include "intel_de.h"
20 #include "intel_display.h"
21 #include "intel_display_power.h"
22 #include "intel_display_types.h"
23 #include "intel_fb.h"
24 #include "intel_fixed.h"
25 #include "intel_pcode.h"
26 #include "intel_wm.h"
27 #include "skl_universal_plane_regs.h"
28 #include "skl_watermark.h"
29 #include "skl_watermark_regs.h"
30
31 /*It is expected that DSB can do posted writes to every register in
32 * the pipe and planes within 100us. For flip queue use case, the
33 * recommended DSB execution time is 100us + one SAGV block time.
34 */
35 #define DSB_EXE_TIME 100
36
37 static void skl_sagv_disable(struct drm_i915_private *i915);
38
39 /* Stores plane specific WM parameters */
40 struct skl_wm_params {
41 bool x_tiled, y_tiled;
42 bool rc_surface;
43 bool is_planar;
44 u32 width;
45 u8 cpp;
46 u32 plane_pixel_rate;
47 u32 y_min_scanlines;
48 u32 plane_bytes_per_line;
49 uint_fixed_16_16_t plane_blocks_per_line;
50 uint_fixed_16_16_t y_tile_minimum;
51 u32 linetime_us;
52 u32 dbuf_block_size;
53 };
54
intel_enabled_dbuf_slices_mask(struct intel_display * display)55 u8 intel_enabled_dbuf_slices_mask(struct intel_display *display)
56 {
57 u8 enabled_slices = 0;
58 enum dbuf_slice slice;
59
60 for_each_dbuf_slice(display, slice) {
61 if (intel_de_read(display, DBUF_CTL_S(slice)) & DBUF_POWER_STATE)
62 enabled_slices |= BIT(slice);
63 }
64
65 return enabled_slices;
66 }
67
68 /*
69 * FIXME: We still don't have the proper code detect if we need to apply the WA,
70 * so assume we'll always need it in order to avoid underruns.
71 */
skl_needs_memory_bw_wa(struct drm_i915_private * i915)72 static bool skl_needs_memory_bw_wa(struct drm_i915_private *i915)
73 {
74 return DISPLAY_VER(i915) == 9;
75 }
76
77 bool
intel_has_sagv(struct drm_i915_private * i915)78 intel_has_sagv(struct drm_i915_private *i915)
79 {
80 struct intel_display *display = &i915->display;
81
82 return HAS_SAGV(display) && display->sagv.status != I915_SAGV_NOT_CONTROLLED;
83 }
84
85 static u32
intel_sagv_block_time(struct drm_i915_private * i915)86 intel_sagv_block_time(struct drm_i915_private *i915)
87 {
88 struct intel_display *display = &i915->display;
89
90 if (DISPLAY_VER(display) >= 14) {
91 u32 val;
92
93 val = intel_de_read(display, MTL_LATENCY_SAGV);
94
95 return REG_FIELD_GET(MTL_LATENCY_QCLK_SAGV, val);
96 } else if (DISPLAY_VER(display) >= 12) {
97 u32 val = 0;
98 int ret;
99
100 ret = snb_pcode_read(&i915->uncore,
101 GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
102 &val, NULL);
103 if (ret) {
104 drm_dbg_kms(display->drm, "Couldn't read SAGV block time!\n");
105 return 0;
106 }
107
108 return val;
109 } else if (DISPLAY_VER(display) == 11) {
110 return 10;
111 } else if (HAS_SAGV(display)) {
112 return 30;
113 } else {
114 return 0;
115 }
116 }
117
intel_sagv_init(struct drm_i915_private * i915)118 static void intel_sagv_init(struct drm_i915_private *i915)
119 {
120 struct intel_display *display = &i915->display;
121
122 if (!HAS_SAGV(display))
123 display->sagv.status = I915_SAGV_NOT_CONTROLLED;
124
125 /*
126 * Probe to see if we have working SAGV control.
127 * For icl+ this was already determined by intel_bw_init_hw().
128 */
129 if (DISPLAY_VER(display) < 11)
130 skl_sagv_disable(i915);
131
132 drm_WARN_ON(display->drm, display->sagv.status == I915_SAGV_UNKNOWN);
133
134 display->sagv.block_time_us = intel_sagv_block_time(i915);
135
136 drm_dbg_kms(display->drm, "SAGV supported: %s, original SAGV block time: %u us\n",
137 str_yes_no(intel_has_sagv(i915)), display->sagv.block_time_us);
138
139 /* avoid overflow when adding with wm0 latency/etc. */
140 if (drm_WARN(display->drm, display->sagv.block_time_us > U16_MAX,
141 "Excessive SAGV block time %u, ignoring\n",
142 display->sagv.block_time_us))
143 display->sagv.block_time_us = 0;
144
145 if (!intel_has_sagv(i915))
146 display->sagv.block_time_us = 0;
147 }
148
149 /*
150 * SAGV dynamically adjusts the system agent voltage and clock frequencies
151 * depending on power and performance requirements. The display engine access
152 * to system memory is blocked during the adjustment time. Because of the
153 * blocking time, having this enabled can cause full system hangs and/or pipe
154 * underruns if we don't meet all of the following requirements:
155 *
156 * - <= 1 pipe enabled
157 * - All planes can enable watermarks for latencies >= SAGV engine block time
158 * - We're not using an interlaced display configuration
159 */
skl_sagv_enable(struct drm_i915_private * i915)160 static void skl_sagv_enable(struct drm_i915_private *i915)
161 {
162 int ret;
163
164 if (!intel_has_sagv(i915))
165 return;
166
167 if (i915->display.sagv.status == I915_SAGV_ENABLED)
168 return;
169
170 drm_dbg_kms(&i915->drm, "Enabling SAGV\n");
171 ret = snb_pcode_write(&i915->uncore, GEN9_PCODE_SAGV_CONTROL,
172 GEN9_SAGV_ENABLE);
173
174 /* We don't need to wait for SAGV when enabling */
175
176 /*
177 * Some skl systems, pre-release machines in particular,
178 * don't actually have SAGV.
179 */
180 if (IS_SKYLAKE(i915) && ret == -ENXIO) {
181 drm_dbg(&i915->drm, "No SAGV found on system, ignoring\n");
182 i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
183 return;
184 } else if (ret < 0) {
185 drm_err(&i915->drm, "Failed to enable SAGV\n");
186 return;
187 }
188
189 i915->display.sagv.status = I915_SAGV_ENABLED;
190 }
191
skl_sagv_disable(struct drm_i915_private * i915)192 static void skl_sagv_disable(struct drm_i915_private *i915)
193 {
194 int ret;
195
196 if (!intel_has_sagv(i915))
197 return;
198
199 if (i915->display.sagv.status == I915_SAGV_DISABLED)
200 return;
201
202 drm_dbg_kms(&i915->drm, "Disabling SAGV\n");
203 /* bspec says to keep retrying for at least 1 ms */
204 ret = skl_pcode_request(&i915->uncore, GEN9_PCODE_SAGV_CONTROL,
205 GEN9_SAGV_DISABLE,
206 GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
207 1);
208 /*
209 * Some skl systems, pre-release machines in particular,
210 * don't actually have SAGV.
211 */
212 if (IS_SKYLAKE(i915) && ret == -ENXIO) {
213 drm_dbg(&i915->drm, "No SAGV found on system, ignoring\n");
214 i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
215 return;
216 } else if (ret < 0) {
217 drm_err(&i915->drm, "Failed to disable SAGV (%d)\n", ret);
218 return;
219 }
220
221 i915->display.sagv.status = I915_SAGV_DISABLED;
222 }
223
skl_sagv_pre_plane_update(struct intel_atomic_state * state)224 static void skl_sagv_pre_plane_update(struct intel_atomic_state *state)
225 {
226 struct drm_i915_private *i915 = to_i915(state->base.dev);
227 const struct intel_bw_state *new_bw_state =
228 intel_atomic_get_new_bw_state(state);
229
230 if (!new_bw_state)
231 return;
232
233 if (!intel_can_enable_sagv(i915, new_bw_state))
234 skl_sagv_disable(i915);
235 }
236
skl_sagv_post_plane_update(struct intel_atomic_state * state)237 static void skl_sagv_post_plane_update(struct intel_atomic_state *state)
238 {
239 struct drm_i915_private *i915 = to_i915(state->base.dev);
240 const struct intel_bw_state *new_bw_state =
241 intel_atomic_get_new_bw_state(state);
242
243 if (!new_bw_state)
244 return;
245
246 if (intel_can_enable_sagv(i915, new_bw_state))
247 skl_sagv_enable(i915);
248 }
249
icl_sagv_pre_plane_update(struct intel_atomic_state * state)250 static void icl_sagv_pre_plane_update(struct intel_atomic_state *state)
251 {
252 struct drm_i915_private *i915 = to_i915(state->base.dev);
253 const struct intel_bw_state *old_bw_state =
254 intel_atomic_get_old_bw_state(state);
255 const struct intel_bw_state *new_bw_state =
256 intel_atomic_get_new_bw_state(state);
257 u16 old_mask, new_mask;
258
259 if (!new_bw_state)
260 return;
261
262 old_mask = old_bw_state->qgv_points_mask;
263 new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
264
265 if (old_mask == new_mask)
266 return;
267
268 WARN_ON(!new_bw_state->base.changed);
269
270 drm_dbg_kms(&i915->drm, "Restricting QGV points: 0x%x -> 0x%x\n",
271 old_mask, new_mask);
272
273 /*
274 * Restrict required qgv points before updating the configuration.
275 * According to BSpec we can't mask and unmask qgv points at the same
276 * time. Also masking should be done before updating the configuration
277 * and unmasking afterwards.
278 */
279 icl_pcode_restrict_qgv_points(i915, new_mask);
280 }
281
icl_sagv_post_plane_update(struct intel_atomic_state * state)282 static void icl_sagv_post_plane_update(struct intel_atomic_state *state)
283 {
284 struct drm_i915_private *i915 = to_i915(state->base.dev);
285 const struct intel_bw_state *old_bw_state =
286 intel_atomic_get_old_bw_state(state);
287 const struct intel_bw_state *new_bw_state =
288 intel_atomic_get_new_bw_state(state);
289 u16 old_mask, new_mask;
290
291 if (!new_bw_state)
292 return;
293
294 old_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
295 new_mask = new_bw_state->qgv_points_mask;
296
297 if (old_mask == new_mask)
298 return;
299
300 WARN_ON(!new_bw_state->base.changed);
301
302 drm_dbg_kms(&i915->drm, "Relaxing QGV points: 0x%x -> 0x%x\n",
303 old_mask, new_mask);
304
305 /*
306 * Allow required qgv points after updating the configuration.
307 * According to BSpec we can't mask and unmask qgv points at the same
308 * time. Also masking should be done before updating the configuration
309 * and unmasking afterwards.
310 */
311 icl_pcode_restrict_qgv_points(i915, new_mask);
312 }
313
intel_sagv_pre_plane_update(struct intel_atomic_state * state)314 void intel_sagv_pre_plane_update(struct intel_atomic_state *state)
315 {
316 struct drm_i915_private *i915 = to_i915(state->base.dev);
317
318 /*
319 * Just return if we can't control SAGV or don't have it.
320 * This is different from situation when we have SAGV but just can't
321 * afford it due to DBuf limitation - in case if SAGV is completely
322 * disabled in a BIOS, we are not even allowed to send a PCode request,
323 * as it will throw an error. So have to check it here.
324 */
325 if (!intel_has_sagv(i915))
326 return;
327
328 if (DISPLAY_VER(i915) >= 11)
329 icl_sagv_pre_plane_update(state);
330 else
331 skl_sagv_pre_plane_update(state);
332 }
333
intel_sagv_post_plane_update(struct intel_atomic_state * state)334 void intel_sagv_post_plane_update(struct intel_atomic_state *state)
335 {
336 struct drm_i915_private *i915 = to_i915(state->base.dev);
337
338 /*
339 * Just return if we can't control SAGV or don't have it.
340 * This is different from situation when we have SAGV but just can't
341 * afford it due to DBuf limitation - in case if SAGV is completely
342 * disabled in a BIOS, we are not even allowed to send a PCode request,
343 * as it will throw an error. So have to check it here.
344 */
345 if (!intel_has_sagv(i915))
346 return;
347
348 if (DISPLAY_VER(i915) >= 11)
349 icl_sagv_post_plane_update(state);
350 else
351 skl_sagv_post_plane_update(state);
352 }
353
skl_crtc_can_enable_sagv(const struct intel_crtc_state * crtc_state)354 static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
355 {
356 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
357 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
358 enum plane_id plane_id;
359 int max_level = INT_MAX;
360
361 if (!intel_has_sagv(i915))
362 return false;
363
364 if (!crtc_state->hw.active)
365 return true;
366
367 if (crtc_state->hw.pipe_mode.flags & DRM_MODE_FLAG_INTERLACE)
368 return false;
369
370 for_each_plane_id_on_crtc(crtc, plane_id) {
371 const struct skl_plane_wm *wm =
372 &crtc_state->wm.skl.optimal.planes[plane_id];
373 int level;
374
375 /* Skip this plane if it's not enabled */
376 if (!wm->wm[0].enable)
377 continue;
378
379 /* Find the highest enabled wm level for this plane */
380 for (level = i915->display.wm.num_levels - 1;
381 !wm->wm[level].enable; --level)
382 { }
383
384 /* Highest common enabled wm level for all planes */
385 max_level = min(level, max_level);
386 }
387
388 /* No enabled planes? */
389 if (max_level == INT_MAX)
390 return true;
391
392 for_each_plane_id_on_crtc(crtc, plane_id) {
393 const struct skl_plane_wm *wm =
394 &crtc_state->wm.skl.optimal.planes[plane_id];
395
396 /*
397 * All enabled planes must have enabled a common wm level that
398 * can tolerate memory latencies higher than sagv_block_time_us
399 */
400 if (wm->wm[0].enable && !wm->wm[max_level].can_sagv)
401 return false;
402 }
403
404 return true;
405 }
406
tgl_crtc_can_enable_sagv(const struct intel_crtc_state * crtc_state)407 static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
408 {
409 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
410 enum plane_id plane_id;
411
412 if (!crtc_state->hw.active)
413 return true;
414
415 for_each_plane_id_on_crtc(crtc, plane_id) {
416 const struct skl_plane_wm *wm =
417 &crtc_state->wm.skl.optimal.planes[plane_id];
418
419 if (wm->wm[0].enable && !wm->sagv.wm0.enable)
420 return false;
421 }
422
423 return true;
424 }
425
intel_crtc_can_enable_sagv(const struct intel_crtc_state * crtc_state)426 static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
427 {
428 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
429 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
430
431 if (!i915->display.params.enable_sagv)
432 return false;
433
434 if (DISPLAY_VER(i915) >= 12)
435 return tgl_crtc_can_enable_sagv(crtc_state);
436 else
437 return skl_crtc_can_enable_sagv(crtc_state);
438 }
439
intel_can_enable_sagv(struct drm_i915_private * i915,const struct intel_bw_state * bw_state)440 bool intel_can_enable_sagv(struct drm_i915_private *i915,
441 const struct intel_bw_state *bw_state)
442 {
443 if (DISPLAY_VER(i915) < 11 &&
444 bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes))
445 return false;
446
447 return bw_state->pipe_sagv_reject == 0;
448 }
449
intel_compute_sagv_mask(struct intel_atomic_state * state)450 static int intel_compute_sagv_mask(struct intel_atomic_state *state)
451 {
452 struct intel_display *display = to_intel_display(state);
453 struct drm_i915_private *i915 = to_i915(state->base.dev);
454 int ret;
455 struct intel_crtc *crtc;
456 struct intel_crtc_state *new_crtc_state;
457 struct intel_bw_state *new_bw_state = NULL;
458 const struct intel_bw_state *old_bw_state = NULL;
459 int i;
460
461 for_each_new_intel_crtc_in_state(state, crtc,
462 new_crtc_state, i) {
463 struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal;
464
465 new_bw_state = intel_atomic_get_bw_state(state);
466 if (IS_ERR(new_bw_state))
467 return PTR_ERR(new_bw_state);
468
469 old_bw_state = intel_atomic_get_old_bw_state(state);
470
471 /*
472 * We store use_sagv_wm in the crtc state rather than relying on
473 * that bw state since we have no convenient way to get at the
474 * latter from the plane commit hooks (especially in the legacy
475 * cursor case).
476 *
477 * drm_atomic_check_only() gets upset if we pull more crtcs
478 * into the state, so we have to calculate this based on the
479 * individual intel_crtc_can_enable_sagv() rather than
480 * the overall intel_can_enable_sagv(). Otherwise the
481 * crtcs not included in the commit would not switch to the
482 * SAGV watermarks when we are about to enable SAGV, and that
483 * would lead to underruns. This does mean extra power draw
484 * when only a subset of the crtcs are blocking SAGV as the
485 * other crtcs can't be allowed to use the more optimal
486 * normal (ie. non-SAGV) watermarks.
487 */
488 pipe_wm->use_sagv_wm = !HAS_HW_SAGV_WM(display) &&
489 DISPLAY_VER(i915) >= 12 &&
490 intel_crtc_can_enable_sagv(new_crtc_state);
491
492 if (intel_crtc_can_enable_sagv(new_crtc_state))
493 new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe);
494 else
495 new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe);
496 }
497
498 if (!new_bw_state)
499 return 0;
500
501 new_bw_state->active_pipes =
502 intel_calc_active_pipes(state, old_bw_state->active_pipes);
503
504 if (new_bw_state->active_pipes != old_bw_state->active_pipes) {
505 ret = intel_atomic_lock_global_state(&new_bw_state->base);
506 if (ret)
507 return ret;
508 }
509
510 if (intel_can_enable_sagv(i915, new_bw_state) !=
511 intel_can_enable_sagv(i915, old_bw_state)) {
512 ret = intel_atomic_serialize_global_state(&new_bw_state->base);
513 if (ret)
514 return ret;
515 } else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
516 ret = intel_atomic_lock_global_state(&new_bw_state->base);
517 if (ret)
518 return ret;
519 }
520
521 return 0;
522 }
523
skl_ddb_entry_init(struct skl_ddb_entry * entry,u16 start,u16 end)524 static u16 skl_ddb_entry_init(struct skl_ddb_entry *entry,
525 u16 start, u16 end)
526 {
527 entry->start = start;
528 entry->end = end;
529
530 return end;
531 }
532
intel_dbuf_slice_size(struct drm_i915_private * i915)533 static int intel_dbuf_slice_size(struct drm_i915_private *i915)
534 {
535 return DISPLAY_INFO(i915)->dbuf.size /
536 hweight8(DISPLAY_INFO(i915)->dbuf.slice_mask);
537 }
538
539 static void
skl_ddb_entry_for_slices(struct drm_i915_private * i915,u8 slice_mask,struct skl_ddb_entry * ddb)540 skl_ddb_entry_for_slices(struct drm_i915_private *i915, u8 slice_mask,
541 struct skl_ddb_entry *ddb)
542 {
543 int slice_size = intel_dbuf_slice_size(i915);
544
545 if (!slice_mask) {
546 ddb->start = 0;
547 ddb->end = 0;
548 return;
549 }
550
551 ddb->start = (ffs(slice_mask) - 1) * slice_size;
552 ddb->end = fls(slice_mask) * slice_size;
553
554 WARN_ON(ddb->start >= ddb->end);
555 WARN_ON(ddb->end > DISPLAY_INFO(i915)->dbuf.size);
556 }
557
mbus_ddb_offset(struct drm_i915_private * i915,u8 slice_mask)558 static unsigned int mbus_ddb_offset(struct drm_i915_private *i915, u8 slice_mask)
559 {
560 struct skl_ddb_entry ddb;
561
562 if (slice_mask & (BIT(DBUF_S1) | BIT(DBUF_S2)))
563 slice_mask = BIT(DBUF_S1);
564 else if (slice_mask & (BIT(DBUF_S3) | BIT(DBUF_S4)))
565 slice_mask = BIT(DBUF_S3);
566
567 skl_ddb_entry_for_slices(i915, slice_mask, &ddb);
568
569 return ddb.start;
570 }
571
skl_ddb_dbuf_slice_mask(struct drm_i915_private * i915,const struct skl_ddb_entry * entry)572 u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *i915,
573 const struct skl_ddb_entry *entry)
574 {
575 int slice_size = intel_dbuf_slice_size(i915);
576 enum dbuf_slice start_slice, end_slice;
577 u8 slice_mask = 0;
578
579 if (!skl_ddb_entry_size(entry))
580 return 0;
581
582 start_slice = entry->start / slice_size;
583 end_slice = (entry->end - 1) / slice_size;
584
585 /*
586 * Per plane DDB entry can in a really worst case be on multiple slices
587 * but single entry is anyway contiguous.
588 */
589 while (start_slice <= end_slice) {
590 slice_mask |= BIT(start_slice);
591 start_slice++;
592 }
593
594 return slice_mask;
595 }
596
intel_crtc_ddb_weight(const struct intel_crtc_state * crtc_state)597 static unsigned int intel_crtc_ddb_weight(const struct intel_crtc_state *crtc_state)
598 {
599 const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode;
600 int hdisplay, vdisplay;
601
602 if (!crtc_state->hw.active)
603 return 0;
604
605 /*
606 * Watermark/ddb requirement highly depends upon width of the
607 * framebuffer, So instead of allocating DDB equally among pipes
608 * distribute DDB based on resolution/width of the display.
609 */
610 drm_mode_get_hv_timing(pipe_mode, &hdisplay, &vdisplay);
611
612 return hdisplay;
613 }
614
intel_crtc_dbuf_weights(const struct intel_dbuf_state * dbuf_state,enum pipe for_pipe,unsigned int * weight_start,unsigned int * weight_end,unsigned int * weight_total)615 static void intel_crtc_dbuf_weights(const struct intel_dbuf_state *dbuf_state,
616 enum pipe for_pipe,
617 unsigned int *weight_start,
618 unsigned int *weight_end,
619 unsigned int *weight_total)
620 {
621 struct drm_i915_private *i915 =
622 to_i915(dbuf_state->base.state->base.dev);
623 enum pipe pipe;
624
625 *weight_start = 0;
626 *weight_end = 0;
627 *weight_total = 0;
628
629 for_each_pipe(i915, pipe) {
630 int weight = dbuf_state->weight[pipe];
631
632 /*
633 * Do not account pipes using other slice sets
634 * luckily as of current BSpec slice sets do not partially
635 * intersect(pipes share either same one slice or same slice set
636 * i.e no partial intersection), so it is enough to check for
637 * equality for now.
638 */
639 if (dbuf_state->slices[pipe] != dbuf_state->slices[for_pipe])
640 continue;
641
642 *weight_total += weight;
643 if (pipe < for_pipe) {
644 *weight_start += weight;
645 *weight_end += weight;
646 } else if (pipe == for_pipe) {
647 *weight_end += weight;
648 }
649 }
650 }
651
652 static int
skl_crtc_allocate_ddb(struct intel_atomic_state * state,struct intel_crtc * crtc)653 skl_crtc_allocate_ddb(struct intel_atomic_state *state, struct intel_crtc *crtc)
654 {
655 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
656 unsigned int weight_total, weight_start, weight_end;
657 const struct intel_dbuf_state *old_dbuf_state =
658 intel_atomic_get_old_dbuf_state(state);
659 struct intel_dbuf_state *new_dbuf_state =
660 intel_atomic_get_new_dbuf_state(state);
661 struct intel_crtc_state *crtc_state;
662 struct skl_ddb_entry ddb_slices;
663 enum pipe pipe = crtc->pipe;
664 unsigned int mbus_offset = 0;
665 u32 ddb_range_size;
666 u32 dbuf_slice_mask;
667 u32 start, end;
668 int ret;
669
670 if (new_dbuf_state->weight[pipe] == 0) {
671 skl_ddb_entry_init(&new_dbuf_state->ddb[pipe], 0, 0);
672 goto out;
673 }
674
675 dbuf_slice_mask = new_dbuf_state->slices[pipe];
676
677 skl_ddb_entry_for_slices(i915, dbuf_slice_mask, &ddb_slices);
678 mbus_offset = mbus_ddb_offset(i915, dbuf_slice_mask);
679 ddb_range_size = skl_ddb_entry_size(&ddb_slices);
680
681 intel_crtc_dbuf_weights(new_dbuf_state, pipe,
682 &weight_start, &weight_end, &weight_total);
683
684 start = ddb_range_size * weight_start / weight_total;
685 end = ddb_range_size * weight_end / weight_total;
686
687 skl_ddb_entry_init(&new_dbuf_state->ddb[pipe],
688 ddb_slices.start - mbus_offset + start,
689 ddb_slices.start - mbus_offset + end);
690
691 out:
692 if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe] &&
693 skl_ddb_entry_equal(&old_dbuf_state->ddb[pipe],
694 &new_dbuf_state->ddb[pipe]))
695 return 0;
696
697 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
698 if (ret)
699 return ret;
700
701 crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
702 if (IS_ERR(crtc_state))
703 return PTR_ERR(crtc_state);
704
705 /*
706 * Used for checking overlaps, so we need absolute
707 * offsets instead of MBUS relative offsets.
708 */
709 crtc_state->wm.skl.ddb.start = mbus_offset + new_dbuf_state->ddb[pipe].start;
710 crtc_state->wm.skl.ddb.end = mbus_offset + new_dbuf_state->ddb[pipe].end;
711
712 drm_dbg_kms(&i915->drm,
713 "[CRTC:%d:%s] dbuf slices 0x%x -> 0x%x, ddb (%d - %d) -> (%d - %d), active pipes 0x%x -> 0x%x\n",
714 crtc->base.base.id, crtc->base.name,
715 old_dbuf_state->slices[pipe], new_dbuf_state->slices[pipe],
716 old_dbuf_state->ddb[pipe].start, old_dbuf_state->ddb[pipe].end,
717 new_dbuf_state->ddb[pipe].start, new_dbuf_state->ddb[pipe].end,
718 old_dbuf_state->active_pipes, new_dbuf_state->active_pipes);
719
720 return 0;
721 }
722
723 static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
724 int width, const struct drm_format_info *format,
725 u64 modifier, unsigned int rotation,
726 u32 plane_pixel_rate, struct skl_wm_params *wp,
727 int color_plane, unsigned int pan_x);
728
729 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
730 struct intel_plane *plane,
731 int level,
732 unsigned int latency,
733 const struct skl_wm_params *wp,
734 const struct skl_wm_level *result_prev,
735 struct skl_wm_level *result /* out */);
736
skl_wm_latency(struct drm_i915_private * i915,int level,const struct skl_wm_params * wp)737 static unsigned int skl_wm_latency(struct drm_i915_private *i915, int level,
738 const struct skl_wm_params *wp)
739 {
740 unsigned int latency = i915->display.wm.skl_latency[level];
741
742 if (latency == 0)
743 return 0;
744
745 /*
746 * WaIncreaseLatencyIPCEnabled: kbl,cfl
747 * Display WA #1141: kbl,cfl
748 */
749 if ((IS_KABYLAKE(i915) || IS_COFFEELAKE(i915) || IS_COMETLAKE(i915)) &&
750 skl_watermark_ipc_enabled(i915))
751 latency += 4;
752
753 if (skl_needs_memory_bw_wa(i915) && wp && wp->x_tiled)
754 latency += 15;
755
756 return latency;
757 }
758
759 static unsigned int
skl_cursor_allocation(const struct intel_crtc_state * crtc_state,int num_active)760 skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
761 int num_active)
762 {
763 struct intel_plane *plane = to_intel_plane(crtc_state->uapi.crtc->cursor);
764 struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
765 struct skl_wm_level wm = {};
766 int ret, min_ddb_alloc = 0;
767 struct skl_wm_params wp;
768 int level;
769
770 ret = skl_compute_wm_params(crtc_state, 256,
771 drm_format_info(DRM_FORMAT_ARGB8888),
772 DRM_FORMAT_MOD_LINEAR,
773 DRM_MODE_ROTATE_0,
774 crtc_state->pixel_rate, &wp, 0, 0);
775 drm_WARN_ON(&i915->drm, ret);
776
777 for (level = 0; level < i915->display.wm.num_levels; level++) {
778 unsigned int latency = skl_wm_latency(i915, level, &wp);
779
780 skl_compute_plane_wm(crtc_state, plane, level, latency, &wp, &wm, &wm);
781 if (wm.min_ddb_alloc == U16_MAX)
782 break;
783
784 min_ddb_alloc = wm.min_ddb_alloc;
785 }
786
787 return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
788 }
789
skl_ddb_entry_init_from_hw(struct skl_ddb_entry * entry,u32 reg)790 static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg)
791 {
792 skl_ddb_entry_init(entry,
793 REG_FIELD_GET(PLANE_BUF_START_MASK, reg),
794 REG_FIELD_GET(PLANE_BUF_END_MASK, reg));
795 if (entry->end)
796 entry->end++;
797 }
798
799 static void
skl_ddb_get_hw_plane_state(struct drm_i915_private * i915,const enum pipe pipe,const enum plane_id plane_id,struct skl_ddb_entry * ddb,struct skl_ddb_entry * ddb_y,u16 * min_ddb,u16 * interim_ddb)800 skl_ddb_get_hw_plane_state(struct drm_i915_private *i915,
801 const enum pipe pipe,
802 const enum plane_id plane_id,
803 struct skl_ddb_entry *ddb,
804 struct skl_ddb_entry *ddb_y,
805 u16 *min_ddb, u16 *interim_ddb)
806 {
807 struct intel_display *display = &i915->display;
808 u32 val;
809
810 /* Cursor doesn't support NV12/planar, so no extra calculation needed */
811 if (plane_id == PLANE_CURSOR) {
812 val = intel_de_read(display, CUR_BUF_CFG(pipe));
813 skl_ddb_entry_init_from_hw(ddb, val);
814 return;
815 }
816
817 val = intel_de_read(display, PLANE_BUF_CFG(pipe, plane_id));
818 skl_ddb_entry_init_from_hw(ddb, val);
819
820 if (DISPLAY_VER(display) >= 30) {
821 val = intel_de_read(display, PLANE_MIN_BUF_CFG(pipe, plane_id));
822
823 *min_ddb = REG_FIELD_GET(PLANE_MIN_DBUF_BLOCKS_MASK, val);
824 *interim_ddb = REG_FIELD_GET(PLANE_INTERIM_DBUF_BLOCKS_MASK, val);
825 }
826
827 if (DISPLAY_VER(display) >= 11)
828 return;
829
830 val = intel_de_read(display, PLANE_NV12_BUF_CFG(pipe, plane_id));
831 skl_ddb_entry_init_from_hw(ddb_y, val);
832 }
833
skl_pipe_ddb_get_hw_state(struct intel_crtc * crtc,struct skl_ddb_entry * ddb,struct skl_ddb_entry * ddb_y,u16 * min_ddb,u16 * interim_ddb)834 static void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
835 struct skl_ddb_entry *ddb,
836 struct skl_ddb_entry *ddb_y,
837 u16 *min_ddb, u16 *interim_ddb)
838 {
839 struct intel_display *display = to_intel_display(crtc);
840 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
841 enum intel_display_power_domain power_domain;
842 enum pipe pipe = crtc->pipe;
843 intel_wakeref_t wakeref;
844 enum plane_id plane_id;
845
846 power_domain = POWER_DOMAIN_PIPE(pipe);
847 wakeref = intel_display_power_get_if_enabled(display, power_domain);
848 if (!wakeref)
849 return;
850
851 for_each_plane_id_on_crtc(crtc, plane_id)
852 skl_ddb_get_hw_plane_state(i915, pipe,
853 plane_id,
854 &ddb[plane_id],
855 &ddb_y[plane_id],
856 &min_ddb[plane_id],
857 &interim_ddb[plane_id]);
858
859 intel_display_power_put(display, power_domain, wakeref);
860 }
861
862 struct dbuf_slice_conf_entry {
863 u8 active_pipes;
864 u8 dbuf_mask[I915_MAX_PIPES];
865 bool join_mbus;
866 };
867
868 /*
869 * Table taken from Bspec 12716
870 * Pipes do have some preferred DBuf slice affinity,
871 * plus there are some hardcoded requirements on how
872 * those should be distributed for multipipe scenarios.
873 * For more DBuf slices algorithm can get even more messy
874 * and less readable, so decided to use a table almost
875 * as is from BSpec itself - that way it is at least easier
876 * to compare, change and check.
877 */
878 static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] =
879 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
880 {
881 {
882 .active_pipes = BIT(PIPE_A),
883 .dbuf_mask = {
884 [PIPE_A] = BIT(DBUF_S1),
885 },
886 },
887 {
888 .active_pipes = BIT(PIPE_B),
889 .dbuf_mask = {
890 [PIPE_B] = BIT(DBUF_S1),
891 },
892 },
893 {
894 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
895 .dbuf_mask = {
896 [PIPE_A] = BIT(DBUF_S1),
897 [PIPE_B] = BIT(DBUF_S2),
898 },
899 },
900 {
901 .active_pipes = BIT(PIPE_C),
902 .dbuf_mask = {
903 [PIPE_C] = BIT(DBUF_S2),
904 },
905 },
906 {
907 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
908 .dbuf_mask = {
909 [PIPE_A] = BIT(DBUF_S1),
910 [PIPE_C] = BIT(DBUF_S2),
911 },
912 },
913 {
914 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
915 .dbuf_mask = {
916 [PIPE_B] = BIT(DBUF_S1),
917 [PIPE_C] = BIT(DBUF_S2),
918 },
919 },
920 {
921 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
922 .dbuf_mask = {
923 [PIPE_A] = BIT(DBUF_S1),
924 [PIPE_B] = BIT(DBUF_S1),
925 [PIPE_C] = BIT(DBUF_S2),
926 },
927 },
928 {}
929 };
930
931 /*
932 * Table taken from Bspec 49255
933 * Pipes do have some preferred DBuf slice affinity,
934 * plus there are some hardcoded requirements on how
935 * those should be distributed for multipipe scenarios.
936 * For more DBuf slices algorithm can get even more messy
937 * and less readable, so decided to use a table almost
938 * as is from BSpec itself - that way it is at least easier
939 * to compare, change and check.
940 */
941 static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] =
942 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
943 {
944 {
945 .active_pipes = BIT(PIPE_A),
946 .dbuf_mask = {
947 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
948 },
949 },
950 {
951 .active_pipes = BIT(PIPE_B),
952 .dbuf_mask = {
953 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
954 },
955 },
956 {
957 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
958 .dbuf_mask = {
959 [PIPE_A] = BIT(DBUF_S2),
960 [PIPE_B] = BIT(DBUF_S1),
961 },
962 },
963 {
964 .active_pipes = BIT(PIPE_C),
965 .dbuf_mask = {
966 [PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1),
967 },
968 },
969 {
970 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
971 .dbuf_mask = {
972 [PIPE_A] = BIT(DBUF_S1),
973 [PIPE_C] = BIT(DBUF_S2),
974 },
975 },
976 {
977 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
978 .dbuf_mask = {
979 [PIPE_B] = BIT(DBUF_S1),
980 [PIPE_C] = BIT(DBUF_S2),
981 },
982 },
983 {
984 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
985 .dbuf_mask = {
986 [PIPE_A] = BIT(DBUF_S1),
987 [PIPE_B] = BIT(DBUF_S1),
988 [PIPE_C] = BIT(DBUF_S2),
989 },
990 },
991 {
992 .active_pipes = BIT(PIPE_D),
993 .dbuf_mask = {
994 [PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1),
995 },
996 },
997 {
998 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
999 .dbuf_mask = {
1000 [PIPE_A] = BIT(DBUF_S1),
1001 [PIPE_D] = BIT(DBUF_S2),
1002 },
1003 },
1004 {
1005 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
1006 .dbuf_mask = {
1007 [PIPE_B] = BIT(DBUF_S1),
1008 [PIPE_D] = BIT(DBUF_S2),
1009 },
1010 },
1011 {
1012 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
1013 .dbuf_mask = {
1014 [PIPE_A] = BIT(DBUF_S1),
1015 [PIPE_B] = BIT(DBUF_S1),
1016 [PIPE_D] = BIT(DBUF_S2),
1017 },
1018 },
1019 {
1020 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
1021 .dbuf_mask = {
1022 [PIPE_C] = BIT(DBUF_S1),
1023 [PIPE_D] = BIT(DBUF_S2),
1024 },
1025 },
1026 {
1027 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
1028 .dbuf_mask = {
1029 [PIPE_A] = BIT(DBUF_S1),
1030 [PIPE_C] = BIT(DBUF_S2),
1031 [PIPE_D] = BIT(DBUF_S2),
1032 },
1033 },
1034 {
1035 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1036 .dbuf_mask = {
1037 [PIPE_B] = BIT(DBUF_S1),
1038 [PIPE_C] = BIT(DBUF_S2),
1039 [PIPE_D] = BIT(DBUF_S2),
1040 },
1041 },
1042 {
1043 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1044 .dbuf_mask = {
1045 [PIPE_A] = BIT(DBUF_S1),
1046 [PIPE_B] = BIT(DBUF_S1),
1047 [PIPE_C] = BIT(DBUF_S2),
1048 [PIPE_D] = BIT(DBUF_S2),
1049 },
1050 },
1051 {}
1052 };
1053
1054 static const struct dbuf_slice_conf_entry dg2_allowed_dbufs[] = {
1055 {
1056 .active_pipes = BIT(PIPE_A),
1057 .dbuf_mask = {
1058 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1059 },
1060 },
1061 {
1062 .active_pipes = BIT(PIPE_B),
1063 .dbuf_mask = {
1064 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1065 },
1066 },
1067 {
1068 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
1069 .dbuf_mask = {
1070 [PIPE_A] = BIT(DBUF_S1),
1071 [PIPE_B] = BIT(DBUF_S2),
1072 },
1073 },
1074 {
1075 .active_pipes = BIT(PIPE_C),
1076 .dbuf_mask = {
1077 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1078 },
1079 },
1080 {
1081 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
1082 .dbuf_mask = {
1083 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1084 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1085 },
1086 },
1087 {
1088 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
1089 .dbuf_mask = {
1090 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1091 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1092 },
1093 },
1094 {
1095 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
1096 .dbuf_mask = {
1097 [PIPE_A] = BIT(DBUF_S1),
1098 [PIPE_B] = BIT(DBUF_S2),
1099 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1100 },
1101 },
1102 {
1103 .active_pipes = BIT(PIPE_D),
1104 .dbuf_mask = {
1105 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1106 },
1107 },
1108 {
1109 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
1110 .dbuf_mask = {
1111 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1112 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1113 },
1114 },
1115 {
1116 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
1117 .dbuf_mask = {
1118 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1119 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1120 },
1121 },
1122 {
1123 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
1124 .dbuf_mask = {
1125 [PIPE_A] = BIT(DBUF_S1),
1126 [PIPE_B] = BIT(DBUF_S2),
1127 [PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1128 },
1129 },
1130 {
1131 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
1132 .dbuf_mask = {
1133 [PIPE_C] = BIT(DBUF_S3),
1134 [PIPE_D] = BIT(DBUF_S4),
1135 },
1136 },
1137 {
1138 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
1139 .dbuf_mask = {
1140 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1141 [PIPE_C] = BIT(DBUF_S3),
1142 [PIPE_D] = BIT(DBUF_S4),
1143 },
1144 },
1145 {
1146 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1147 .dbuf_mask = {
1148 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1149 [PIPE_C] = BIT(DBUF_S3),
1150 [PIPE_D] = BIT(DBUF_S4),
1151 },
1152 },
1153 {
1154 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1155 .dbuf_mask = {
1156 [PIPE_A] = BIT(DBUF_S1),
1157 [PIPE_B] = BIT(DBUF_S2),
1158 [PIPE_C] = BIT(DBUF_S3),
1159 [PIPE_D] = BIT(DBUF_S4),
1160 },
1161 },
1162 {}
1163 };
1164
1165 static const struct dbuf_slice_conf_entry adlp_allowed_dbufs[] = {
1166 /*
1167 * Keep the join_mbus cases first so check_mbus_joined()
1168 * will prefer them over the !join_mbus cases.
1169 */
1170 {
1171 .active_pipes = BIT(PIPE_A),
1172 .dbuf_mask = {
1173 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
1174 },
1175 .join_mbus = true,
1176 },
1177 {
1178 .active_pipes = BIT(PIPE_B),
1179 .dbuf_mask = {
1180 [PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
1181 },
1182 .join_mbus = true,
1183 },
1184 {
1185 .active_pipes = BIT(PIPE_A),
1186 .dbuf_mask = {
1187 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1188 },
1189 .join_mbus = false,
1190 },
1191 {
1192 .active_pipes = BIT(PIPE_B),
1193 .dbuf_mask = {
1194 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1195 },
1196 .join_mbus = false,
1197 },
1198 {
1199 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
1200 .dbuf_mask = {
1201 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1202 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1203 },
1204 },
1205 {
1206 .active_pipes = BIT(PIPE_C),
1207 .dbuf_mask = {
1208 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1209 },
1210 },
1211 {
1212 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
1213 .dbuf_mask = {
1214 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1215 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1216 },
1217 },
1218 {
1219 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
1220 .dbuf_mask = {
1221 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1222 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1223 },
1224 },
1225 {
1226 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
1227 .dbuf_mask = {
1228 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1229 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1230 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1231 },
1232 },
1233 {
1234 .active_pipes = BIT(PIPE_D),
1235 .dbuf_mask = {
1236 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1237 },
1238 },
1239 {
1240 .active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
1241 .dbuf_mask = {
1242 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1243 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1244 },
1245 },
1246 {
1247 .active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
1248 .dbuf_mask = {
1249 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1250 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1251 },
1252 },
1253 {
1254 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
1255 .dbuf_mask = {
1256 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1257 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1258 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1259 },
1260 },
1261 {
1262 .active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
1263 .dbuf_mask = {
1264 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1265 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1266 },
1267 },
1268 {
1269 .active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
1270 .dbuf_mask = {
1271 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1272 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1273 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1274 },
1275 },
1276 {
1277 .active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1278 .dbuf_mask = {
1279 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1280 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1281 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1282 },
1283 },
1284 {
1285 .active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1286 .dbuf_mask = {
1287 [PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1288 [PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1289 [PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1290 [PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1291 },
1292 },
1293 {}
1294
1295 };
1296
check_mbus_joined(u8 active_pipes,const struct dbuf_slice_conf_entry * dbuf_slices)1297 static bool check_mbus_joined(u8 active_pipes,
1298 const struct dbuf_slice_conf_entry *dbuf_slices)
1299 {
1300 int i;
1301
1302 for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
1303 if (dbuf_slices[i].active_pipes == active_pipes)
1304 return dbuf_slices[i].join_mbus;
1305 }
1306 return false;
1307 }
1308
adlp_check_mbus_joined(u8 active_pipes)1309 static bool adlp_check_mbus_joined(u8 active_pipes)
1310 {
1311 return check_mbus_joined(active_pipes, adlp_allowed_dbufs);
1312 }
1313
compute_dbuf_slices(enum pipe pipe,u8 active_pipes,bool join_mbus,const struct dbuf_slice_conf_entry * dbuf_slices)1314 static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus,
1315 const struct dbuf_slice_conf_entry *dbuf_slices)
1316 {
1317 int i;
1318
1319 for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
1320 if (dbuf_slices[i].active_pipes == active_pipes &&
1321 dbuf_slices[i].join_mbus == join_mbus)
1322 return dbuf_slices[i].dbuf_mask[pipe];
1323 }
1324 return 0;
1325 }
1326
1327 /*
1328 * This function finds an entry with same enabled pipe configuration and
1329 * returns correspondent DBuf slice mask as stated in BSpec for particular
1330 * platform.
1331 */
icl_compute_dbuf_slices(enum pipe pipe,u8 active_pipes,bool join_mbus)1332 static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1333 {
1334 /*
1335 * FIXME: For ICL this is still a bit unclear as prev BSpec revision
1336 * required calculating "pipe ratio" in order to determine
1337 * if one or two slices can be used for single pipe configurations
1338 * as additional constraint to the existing table.
1339 * However based on recent info, it should be not "pipe ratio"
1340 * but rather ratio between pixel_rate and cdclk with additional
1341 * constants, so for now we are using only table until this is
1342 * clarified. Also this is the reason why crtc_state param is
1343 * still here - we will need it once those additional constraints
1344 * pop up.
1345 */
1346 return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1347 icl_allowed_dbufs);
1348 }
1349
tgl_compute_dbuf_slices(enum pipe pipe,u8 active_pipes,bool join_mbus)1350 static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1351 {
1352 return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1353 tgl_allowed_dbufs);
1354 }
1355
adlp_compute_dbuf_slices(enum pipe pipe,u8 active_pipes,bool join_mbus)1356 static u8 adlp_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1357 {
1358 return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1359 adlp_allowed_dbufs);
1360 }
1361
dg2_compute_dbuf_slices(enum pipe pipe,u8 active_pipes,bool join_mbus)1362 static u8 dg2_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1363 {
1364 return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1365 dg2_allowed_dbufs);
1366 }
1367
skl_compute_dbuf_slices(struct intel_crtc * crtc,u8 active_pipes,bool join_mbus)1368 static u8 skl_compute_dbuf_slices(struct intel_crtc *crtc, u8 active_pipes, bool join_mbus)
1369 {
1370 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1371 enum pipe pipe = crtc->pipe;
1372
1373 if (IS_DG2(i915))
1374 return dg2_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1375 else if (DISPLAY_VER(i915) >= 13)
1376 return adlp_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1377 else if (DISPLAY_VER(i915) == 12)
1378 return tgl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1379 else if (DISPLAY_VER(i915) == 11)
1380 return icl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1381 /*
1382 * For anything else just return one slice yet.
1383 * Should be extended for other platforms.
1384 */
1385 return active_pipes & BIT(pipe) ? BIT(DBUF_S1) : 0;
1386 }
1387
1388 static bool
use_minimal_wm0_only(const struct intel_crtc_state * crtc_state,struct intel_plane * plane)1389 use_minimal_wm0_only(const struct intel_crtc_state *crtc_state,
1390 struct intel_plane *plane)
1391 {
1392 struct intel_display *display = to_intel_display(plane);
1393
1394 /* Xe3+ are auto minimum DDB capble. So don't force minimal wm0 */
1395 return IS_DISPLAY_VER(display, 13, 20) &&
1396 crtc_state->uapi.async_flip &&
1397 plane->async_flip;
1398 }
1399
1400 unsigned int
skl_plane_relative_data_rate(const struct intel_crtc_state * crtc_state,struct intel_plane * plane,int width,int height,int cpp)1401 skl_plane_relative_data_rate(const struct intel_crtc_state *crtc_state,
1402 struct intel_plane *plane, int width, int height,
1403 int cpp)
1404 {
1405 /*
1406 * We calculate extra ddb based on ratio plane rate/total data rate
1407 * in case, in some cases we should not allocate extra ddb for the plane,
1408 * so do not count its data rate, if this is the case.
1409 */
1410 if (use_minimal_wm0_only(crtc_state, plane))
1411 return 0;
1412
1413 return width * height * cpp;
1414 }
1415
1416 static u64
skl_total_relative_data_rate(const struct intel_crtc_state * crtc_state)1417 skl_total_relative_data_rate(const struct intel_crtc_state *crtc_state)
1418 {
1419 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1420 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1421 enum plane_id plane_id;
1422 u64 data_rate = 0;
1423
1424 for_each_plane_id_on_crtc(crtc, plane_id) {
1425 if (plane_id == PLANE_CURSOR)
1426 continue;
1427
1428 data_rate += crtc_state->rel_data_rate[plane_id];
1429
1430 if (DISPLAY_VER(i915) < 11)
1431 data_rate += crtc_state->rel_data_rate_y[plane_id];
1432 }
1433
1434 return data_rate;
1435 }
1436
1437 const struct skl_wm_level *
skl_plane_wm_level(const struct skl_pipe_wm * pipe_wm,enum plane_id plane_id,int level)1438 skl_plane_wm_level(const struct skl_pipe_wm *pipe_wm,
1439 enum plane_id plane_id,
1440 int level)
1441 {
1442 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
1443
1444 if (level == 0 && pipe_wm->use_sagv_wm)
1445 return &wm->sagv.wm0;
1446
1447 return &wm->wm[level];
1448 }
1449
1450 const struct skl_wm_level *
skl_plane_trans_wm(const struct skl_pipe_wm * pipe_wm,enum plane_id plane_id)1451 skl_plane_trans_wm(const struct skl_pipe_wm *pipe_wm,
1452 enum plane_id plane_id)
1453 {
1454 const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
1455
1456 if (pipe_wm->use_sagv_wm)
1457 return &wm->sagv.trans_wm;
1458
1459 return &wm->trans_wm;
1460 }
1461
1462 /*
1463 * We only disable the watermarks for each plane if
1464 * they exceed the ddb allocation of said plane. This
1465 * is done so that we don't end up touching cursor
1466 * watermarks needlessly when some other plane reduces
1467 * our max possible watermark level.
1468 *
1469 * Bspec has this to say about the PLANE_WM enable bit:
1470 * "All the watermarks at this level for all enabled
1471 * planes must be enabled before the level will be used."
1472 * So this is actually safe to do.
1473 */
1474 static void
skl_check_wm_level(struct skl_wm_level * wm,const struct skl_ddb_entry * ddb)1475 skl_check_wm_level(struct skl_wm_level *wm, const struct skl_ddb_entry *ddb)
1476 {
1477 if (wm->min_ddb_alloc > skl_ddb_entry_size(ddb))
1478 memset(wm, 0, sizeof(*wm));
1479 }
1480
1481 static void
skl_check_nv12_wm_level(struct skl_wm_level * wm,struct skl_wm_level * uv_wm,const struct skl_ddb_entry * ddb_y,const struct skl_ddb_entry * ddb)1482 skl_check_nv12_wm_level(struct skl_wm_level *wm, struct skl_wm_level *uv_wm,
1483 const struct skl_ddb_entry *ddb_y, const struct skl_ddb_entry *ddb)
1484 {
1485 if (wm->min_ddb_alloc > skl_ddb_entry_size(ddb_y) ||
1486 uv_wm->min_ddb_alloc > skl_ddb_entry_size(ddb)) {
1487 memset(wm, 0, sizeof(*wm));
1488 memset(uv_wm, 0, sizeof(*uv_wm));
1489 }
1490 }
1491
skl_need_wm_copy_wa(struct drm_i915_private * i915,int level,const struct skl_plane_wm * wm)1492 static bool skl_need_wm_copy_wa(struct drm_i915_private *i915, int level,
1493 const struct skl_plane_wm *wm)
1494 {
1495 /*
1496 * Wa_1408961008:icl, ehl
1497 * Wa_14012656716:tgl, adl
1498 * Wa_14017887344:icl
1499 * Wa_14017868169:adl, tgl
1500 * Due to some power saving optimizations, different subsystems
1501 * like PSR, might still use even disabled wm level registers,
1502 * for "reference", so lets keep at least the values sane.
1503 * Considering amount of WA requiring us to do similar things, was
1504 * decided to simply do it for all of the platforms, as those wm
1505 * levels are disabled, this isn't going to do harm anyway.
1506 */
1507 return level > 0 && !wm->wm[level].enable;
1508 }
1509
1510 struct skl_plane_ddb_iter {
1511 u64 data_rate;
1512 u16 start, size;
1513 };
1514
1515 static void
skl_allocate_plane_ddb(struct skl_plane_ddb_iter * iter,struct skl_ddb_entry * ddb,const struct skl_wm_level * wm,u64 data_rate)1516 skl_allocate_plane_ddb(struct skl_plane_ddb_iter *iter,
1517 struct skl_ddb_entry *ddb,
1518 const struct skl_wm_level *wm,
1519 u64 data_rate)
1520 {
1521 u16 size, extra = 0;
1522
1523 if (data_rate) {
1524 extra = min_t(u16, iter->size,
1525 DIV64_U64_ROUND_UP(iter->size * data_rate,
1526 iter->data_rate));
1527 iter->size -= extra;
1528 iter->data_rate -= data_rate;
1529 }
1530
1531 /*
1532 * Keep ddb entry of all disabled planes explicitly zeroed
1533 * to avoid skl_ddb_add_affected_planes() adding them to
1534 * the state when other planes change their allocations.
1535 */
1536 size = wm->min_ddb_alloc + extra;
1537 if (size)
1538 iter->start = skl_ddb_entry_init(ddb, iter->start,
1539 iter->start + size);
1540 }
1541
1542 static int
skl_crtc_allocate_plane_ddb(struct intel_atomic_state * state,struct intel_crtc * crtc)1543 skl_crtc_allocate_plane_ddb(struct intel_atomic_state *state,
1544 struct intel_crtc *crtc)
1545 {
1546 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1547 struct intel_crtc_state *crtc_state =
1548 intel_atomic_get_new_crtc_state(state, crtc);
1549 const struct intel_dbuf_state *dbuf_state =
1550 intel_atomic_get_new_dbuf_state(state);
1551 const struct skl_ddb_entry *alloc = &dbuf_state->ddb[crtc->pipe];
1552 struct intel_display *display = to_intel_display(state);
1553 int num_active = hweight8(dbuf_state->active_pipes);
1554 struct skl_plane_ddb_iter iter;
1555 enum plane_id plane_id;
1556 u16 cursor_size;
1557 u32 blocks;
1558 int level;
1559
1560 /* Clear the partitioning for disabled planes. */
1561 memset(crtc_state->wm.skl.plane_ddb, 0, sizeof(crtc_state->wm.skl.plane_ddb));
1562 memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y));
1563 memset(crtc_state->wm.skl.plane_min_ddb, 0,
1564 sizeof(crtc_state->wm.skl.plane_min_ddb));
1565 memset(crtc_state->wm.skl.plane_interim_ddb, 0,
1566 sizeof(crtc_state->wm.skl.plane_interim_ddb));
1567
1568 if (!crtc_state->hw.active)
1569 return 0;
1570
1571 iter.start = alloc->start;
1572 iter.size = skl_ddb_entry_size(alloc);
1573 if (iter.size == 0)
1574 return 0;
1575
1576 /* Allocate fixed number of blocks for cursor. */
1577 cursor_size = skl_cursor_allocation(crtc_state, num_active);
1578 iter.size -= cursor_size;
1579 skl_ddb_entry_init(&crtc_state->wm.skl.plane_ddb[PLANE_CURSOR],
1580 alloc->end - cursor_size, alloc->end);
1581
1582 iter.data_rate = skl_total_relative_data_rate(crtc_state);
1583
1584 /*
1585 * Find the highest watermark level for which we can satisfy the block
1586 * requirement of active planes.
1587 */
1588 for (level = i915->display.wm.num_levels - 1; level >= 0; level--) {
1589 blocks = 0;
1590 for_each_plane_id_on_crtc(crtc, plane_id) {
1591 const struct skl_plane_wm *wm =
1592 &crtc_state->wm.skl.optimal.planes[plane_id];
1593
1594 if (plane_id == PLANE_CURSOR) {
1595 const struct skl_ddb_entry *ddb =
1596 &crtc_state->wm.skl.plane_ddb[plane_id];
1597
1598 if (wm->wm[level].min_ddb_alloc > skl_ddb_entry_size(ddb)) {
1599 drm_WARN_ON(&i915->drm,
1600 wm->wm[level].min_ddb_alloc != U16_MAX);
1601 blocks = U32_MAX;
1602 break;
1603 }
1604 continue;
1605 }
1606
1607 blocks += wm->wm[level].min_ddb_alloc;
1608 blocks += wm->uv_wm[level].min_ddb_alloc;
1609 }
1610
1611 if (blocks <= iter.size) {
1612 iter.size -= blocks;
1613 break;
1614 }
1615 }
1616
1617 if (level < 0) {
1618 drm_dbg_kms(&i915->drm,
1619 "Requested display configuration exceeds system DDB limitations");
1620 drm_dbg_kms(&i915->drm, "minimum required %d/%d\n",
1621 blocks, iter.size);
1622 return -EINVAL;
1623 }
1624
1625 /* avoid the WARN later when we don't allocate any extra DDB */
1626 if (iter.data_rate == 0)
1627 iter.size = 0;
1628
1629 /*
1630 * Grant each plane the blocks it requires at the highest achievable
1631 * watermark level, plus an extra share of the leftover blocks
1632 * proportional to its relative data rate.
1633 */
1634 for_each_plane_id_on_crtc(crtc, plane_id) {
1635 struct skl_ddb_entry *ddb =
1636 &crtc_state->wm.skl.plane_ddb[plane_id];
1637 struct skl_ddb_entry *ddb_y =
1638 &crtc_state->wm.skl.plane_ddb_y[plane_id];
1639 u16 *min_ddb = &crtc_state->wm.skl.plane_min_ddb[plane_id];
1640 u16 *interim_ddb =
1641 &crtc_state->wm.skl.plane_interim_ddb[plane_id];
1642 const struct skl_plane_wm *wm =
1643 &crtc_state->wm.skl.optimal.planes[plane_id];
1644
1645 if (plane_id == PLANE_CURSOR)
1646 continue;
1647
1648 if (DISPLAY_VER(i915) < 11 &&
1649 crtc_state->nv12_planes & BIT(plane_id)) {
1650 skl_allocate_plane_ddb(&iter, ddb_y, &wm->wm[level],
1651 crtc_state->rel_data_rate_y[plane_id]);
1652 skl_allocate_plane_ddb(&iter, ddb, &wm->uv_wm[level],
1653 crtc_state->rel_data_rate[plane_id]);
1654 } else {
1655 skl_allocate_plane_ddb(&iter, ddb, &wm->wm[level],
1656 crtc_state->rel_data_rate[plane_id]);
1657 }
1658
1659 if (DISPLAY_VER(display) >= 30) {
1660 *min_ddb = wm->wm[0].min_ddb_alloc;
1661 *interim_ddb = wm->sagv.wm0.min_ddb_alloc;
1662 }
1663 }
1664 drm_WARN_ON(&i915->drm, iter.size != 0 || iter.data_rate != 0);
1665
1666 /*
1667 * When we calculated watermark values we didn't know how high
1668 * of a level we'd actually be able to hit, so we just marked
1669 * all levels as "enabled." Go back now and disable the ones
1670 * that aren't actually possible.
1671 */
1672 for (level++; level < i915->display.wm.num_levels; level++) {
1673 for_each_plane_id_on_crtc(crtc, plane_id) {
1674 const struct skl_ddb_entry *ddb =
1675 &crtc_state->wm.skl.plane_ddb[plane_id];
1676 const struct skl_ddb_entry *ddb_y =
1677 &crtc_state->wm.skl.plane_ddb_y[plane_id];
1678 struct skl_plane_wm *wm =
1679 &crtc_state->wm.skl.optimal.planes[plane_id];
1680
1681 if (DISPLAY_VER(i915) < 11 &&
1682 crtc_state->nv12_planes & BIT(plane_id))
1683 skl_check_nv12_wm_level(&wm->wm[level],
1684 &wm->uv_wm[level],
1685 ddb_y, ddb);
1686 else
1687 skl_check_wm_level(&wm->wm[level], ddb);
1688
1689 if (skl_need_wm_copy_wa(i915, level, wm)) {
1690 wm->wm[level].blocks = wm->wm[level - 1].blocks;
1691 wm->wm[level].lines = wm->wm[level - 1].lines;
1692 wm->wm[level].ignore_lines = wm->wm[level - 1].ignore_lines;
1693 }
1694 }
1695 }
1696
1697 /*
1698 * Go back and disable the transition and SAGV watermarks
1699 * if it turns out we don't have enough DDB blocks for them.
1700 */
1701 for_each_plane_id_on_crtc(crtc, plane_id) {
1702 const struct skl_ddb_entry *ddb =
1703 &crtc_state->wm.skl.plane_ddb[plane_id];
1704 const struct skl_ddb_entry *ddb_y =
1705 &crtc_state->wm.skl.plane_ddb_y[plane_id];
1706 u16 *interim_ddb =
1707 &crtc_state->wm.skl.plane_interim_ddb[plane_id];
1708 struct skl_plane_wm *wm =
1709 &crtc_state->wm.skl.optimal.planes[plane_id];
1710
1711 if (DISPLAY_VER(i915) < 11 &&
1712 crtc_state->nv12_planes & BIT(plane_id)) {
1713 skl_check_wm_level(&wm->trans_wm, ddb_y);
1714 } else {
1715 WARN_ON(skl_ddb_entry_size(ddb_y));
1716
1717 skl_check_wm_level(&wm->trans_wm, ddb);
1718 }
1719
1720 skl_check_wm_level(&wm->sagv.wm0, ddb);
1721 if (DISPLAY_VER(display) >= 30)
1722 *interim_ddb = wm->sagv.wm0.min_ddb_alloc;
1723
1724 skl_check_wm_level(&wm->sagv.trans_wm, ddb);
1725 }
1726
1727 return 0;
1728 }
1729
1730 /*
1731 * The max latency should be 257 (max the punit can code is 255 and we add 2us
1732 * for the read latency) and cpp should always be <= 8, so that
1733 * should allow pixel_rate up to ~2 GHz which seems sufficient since max
1734 * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
1735 */
1736 static uint_fixed_16_16_t
skl_wm_method1(const struct drm_i915_private * i915,u32 pixel_rate,u8 cpp,u32 latency,u32 dbuf_block_size)1737 skl_wm_method1(const struct drm_i915_private *i915, u32 pixel_rate,
1738 u8 cpp, u32 latency, u32 dbuf_block_size)
1739 {
1740 u32 wm_intermediate_val;
1741 uint_fixed_16_16_t ret;
1742
1743 if (latency == 0)
1744 return FP_16_16_MAX;
1745
1746 wm_intermediate_val = latency * pixel_rate * cpp;
1747 ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
1748
1749 if (DISPLAY_VER(i915) >= 10)
1750 ret = add_fixed16_u32(ret, 1);
1751
1752 return ret;
1753 }
1754
1755 static uint_fixed_16_16_t
skl_wm_method2(u32 pixel_rate,u32 pipe_htotal,u32 latency,uint_fixed_16_16_t plane_blocks_per_line)1756 skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
1757 uint_fixed_16_16_t plane_blocks_per_line)
1758 {
1759 u32 wm_intermediate_val;
1760 uint_fixed_16_16_t ret;
1761
1762 if (latency == 0)
1763 return FP_16_16_MAX;
1764
1765 wm_intermediate_val = latency * pixel_rate;
1766 wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
1767 pipe_htotal * 1000);
1768 ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
1769 return ret;
1770 }
1771
1772 static uint_fixed_16_16_t
intel_get_linetime_us(const struct intel_crtc_state * crtc_state)1773 intel_get_linetime_us(const struct intel_crtc_state *crtc_state)
1774 {
1775 struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
1776 u32 pixel_rate;
1777 u32 crtc_htotal;
1778 uint_fixed_16_16_t linetime_us;
1779
1780 if (!crtc_state->hw.active)
1781 return u32_to_fixed16(0);
1782
1783 pixel_rate = crtc_state->pixel_rate;
1784
1785 if (drm_WARN_ON(&i915->drm, pixel_rate == 0))
1786 return u32_to_fixed16(0);
1787
1788 crtc_htotal = crtc_state->hw.pipe_mode.crtc_htotal;
1789 linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
1790
1791 return linetime_us;
1792 }
1793
1794 static int
skl_compute_wm_params(const struct intel_crtc_state * crtc_state,int width,const struct drm_format_info * format,u64 modifier,unsigned int rotation,u32 plane_pixel_rate,struct skl_wm_params * wp,int color_plane,unsigned int pan_x)1795 skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
1796 int width, const struct drm_format_info *format,
1797 u64 modifier, unsigned int rotation,
1798 u32 plane_pixel_rate, struct skl_wm_params *wp,
1799 int color_plane, unsigned int pan_x)
1800 {
1801 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1802 struct intel_display *display = to_intel_display(crtc_state);
1803 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1804 u32 interm_pbpl;
1805
1806 /* only planar format has two planes */
1807 if (color_plane == 1 &&
1808 !intel_format_info_is_yuv_semiplanar(format, modifier)) {
1809 drm_dbg_kms(&i915->drm,
1810 "Non planar format have single plane\n");
1811 return -EINVAL;
1812 }
1813
1814 wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
1815 wp->y_tiled = modifier != I915_FORMAT_MOD_X_TILED &&
1816 intel_fb_is_tiled_modifier(modifier);
1817 wp->rc_surface = intel_fb_is_ccs_modifier(modifier);
1818 wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier);
1819
1820 wp->width = width;
1821 if (color_plane == 1 && wp->is_planar)
1822 wp->width /= 2;
1823
1824 wp->cpp = format->cpp[color_plane];
1825 wp->plane_pixel_rate = plane_pixel_rate;
1826
1827 if (DISPLAY_VER(i915) >= 11 &&
1828 modifier == I915_FORMAT_MOD_Yf_TILED && wp->cpp == 1)
1829 wp->dbuf_block_size = 256;
1830 else
1831 wp->dbuf_block_size = 512;
1832
1833 if (drm_rotation_90_or_270(rotation)) {
1834 switch (wp->cpp) {
1835 case 1:
1836 wp->y_min_scanlines = 16;
1837 break;
1838 case 2:
1839 wp->y_min_scanlines = 8;
1840 break;
1841 case 4:
1842 wp->y_min_scanlines = 4;
1843 break;
1844 default:
1845 MISSING_CASE(wp->cpp);
1846 return -EINVAL;
1847 }
1848 } else {
1849 wp->y_min_scanlines = 4;
1850 }
1851
1852 if (skl_needs_memory_bw_wa(i915))
1853 wp->y_min_scanlines *= 2;
1854
1855 wp->plane_bytes_per_line = wp->width * wp->cpp;
1856 if (wp->y_tiled) {
1857 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
1858 wp->y_min_scanlines,
1859 wp->dbuf_block_size);
1860
1861 if (DISPLAY_VER(display) >= 30)
1862 interm_pbpl += (pan_x != 0);
1863 else if (DISPLAY_VER(i915) >= 10)
1864 interm_pbpl++;
1865
1866 wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
1867 wp->y_min_scanlines);
1868 } else {
1869 interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
1870 wp->dbuf_block_size);
1871
1872 if (!wp->x_tiled || DISPLAY_VER(i915) >= 10)
1873 interm_pbpl++;
1874
1875 wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
1876 }
1877
1878 wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
1879 wp->plane_blocks_per_line);
1880
1881 wp->linetime_us = fixed16_to_u32_round_up(intel_get_linetime_us(crtc_state));
1882
1883 return 0;
1884 }
1885
1886 static int
skl_compute_plane_wm_params(const struct intel_crtc_state * crtc_state,const struct intel_plane_state * plane_state,struct skl_wm_params * wp,int color_plane)1887 skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
1888 const struct intel_plane_state *plane_state,
1889 struct skl_wm_params *wp, int color_plane)
1890 {
1891 const struct drm_framebuffer *fb = plane_state->hw.fb;
1892 int width;
1893
1894 /*
1895 * Src coordinates are already rotated by 270 degrees for
1896 * the 90/270 degree plane rotation cases (to match the
1897 * GTT mapping), hence no need to account for rotation here.
1898 */
1899 width = drm_rect_width(&plane_state->uapi.src) >> 16;
1900
1901 return skl_compute_wm_params(crtc_state, width,
1902 fb->format, fb->modifier,
1903 plane_state->hw.rotation,
1904 intel_plane_pixel_rate(crtc_state, plane_state),
1905 wp, color_plane,
1906 plane_state->uapi.src.x1);
1907 }
1908
skl_wm_has_lines(struct drm_i915_private * i915,int level)1909 static bool skl_wm_has_lines(struct drm_i915_private *i915, int level)
1910 {
1911 if (DISPLAY_VER(i915) >= 10)
1912 return true;
1913
1914 /* The number of lines are ignored for the level 0 watermark. */
1915 return level > 0;
1916 }
1917
skl_wm_max_lines(struct drm_i915_private * i915)1918 static int skl_wm_max_lines(struct drm_i915_private *i915)
1919 {
1920 if (DISPLAY_VER(i915) >= 13)
1921 return 255;
1922 else
1923 return 31;
1924 }
1925
xe3_auto_min_alloc_capable(struct intel_plane * plane,int level)1926 static bool xe3_auto_min_alloc_capable(struct intel_plane *plane, int level)
1927 {
1928 struct intel_display *display = to_intel_display(plane);
1929
1930 return DISPLAY_VER(display) >= 30 && level == 0 && plane->id != PLANE_CURSOR;
1931 }
1932
skl_compute_plane_wm(const struct intel_crtc_state * crtc_state,struct intel_plane * plane,int level,unsigned int latency,const struct skl_wm_params * wp,const struct skl_wm_level * result_prev,struct skl_wm_level * result)1933 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
1934 struct intel_plane *plane,
1935 int level,
1936 unsigned int latency,
1937 const struct skl_wm_params *wp,
1938 const struct skl_wm_level *result_prev,
1939 struct skl_wm_level *result /* out */)
1940 {
1941 struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
1942 uint_fixed_16_16_t method1, method2;
1943 uint_fixed_16_16_t selected_result;
1944 u32 blocks, lines, min_ddb_alloc = 0;
1945
1946 if (latency == 0 ||
1947 (use_minimal_wm0_only(crtc_state, plane) && level > 0)) {
1948 /* reject it */
1949 result->min_ddb_alloc = U16_MAX;
1950 return;
1951 }
1952
1953 method1 = skl_wm_method1(i915, wp->plane_pixel_rate,
1954 wp->cpp, latency, wp->dbuf_block_size);
1955 method2 = skl_wm_method2(wp->plane_pixel_rate,
1956 crtc_state->hw.pipe_mode.crtc_htotal,
1957 latency,
1958 wp->plane_blocks_per_line);
1959
1960 if (wp->y_tiled) {
1961 selected_result = max_fixed16(method2, wp->y_tile_minimum);
1962 } else {
1963 if ((wp->cpp * crtc_state->hw.pipe_mode.crtc_htotal /
1964 wp->dbuf_block_size < 1) &&
1965 (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
1966 selected_result = method2;
1967 } else if (latency >= wp->linetime_us) {
1968 if (DISPLAY_VER(i915) == 9)
1969 selected_result = min_fixed16(method1, method2);
1970 else
1971 selected_result = method2;
1972 } else {
1973 selected_result = method1;
1974 }
1975 }
1976
1977 blocks = fixed16_to_u32_round_up(selected_result);
1978 if (DISPLAY_VER(i915) < 30)
1979 blocks++;
1980
1981 /*
1982 * Lets have blocks at minimum equivalent to plane_blocks_per_line
1983 * as there will be at minimum one line for lines configuration. This
1984 * is a work around for FIFO underruns observed with resolutions like
1985 * 4k 60 Hz in single channel DRAM configurations.
1986 *
1987 * As per the Bspec 49325, if the ddb allocation can hold at least
1988 * one plane_blocks_per_line, we should have selected method2 in
1989 * the above logic. Assuming that modern versions have enough dbuf
1990 * and method2 guarantees blocks equivalent to at least 1 line,
1991 * select the blocks as plane_blocks_per_line.
1992 *
1993 * TODO: Revisit the logic when we have better understanding on DRAM
1994 * channels' impact on the level 0 memory latency and the relevant
1995 * wm calculations.
1996 */
1997 if (skl_wm_has_lines(i915, level))
1998 blocks = max(blocks,
1999 fixed16_to_u32_round_up(wp->plane_blocks_per_line));
2000 lines = div_round_up_fixed16(selected_result,
2001 wp->plane_blocks_per_line);
2002
2003 if (DISPLAY_VER(i915) == 9) {
2004 /* Display WA #1125: skl,bxt,kbl */
2005 if (level == 0 && wp->rc_surface)
2006 blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
2007
2008 /* Display WA #1126: skl,bxt,kbl */
2009 if (level >= 1 && level <= 7) {
2010 if (wp->y_tiled) {
2011 blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
2012 lines += wp->y_min_scanlines;
2013 } else {
2014 blocks++;
2015 }
2016
2017 /*
2018 * Make sure result blocks for higher latency levels are
2019 * at least as high as level below the current level.
2020 * Assumption in DDB algorithm optimization for special
2021 * cases. Also covers Display WA #1125 for RC.
2022 */
2023 if (result_prev->blocks > blocks)
2024 blocks = result_prev->blocks;
2025 }
2026 }
2027
2028 if (DISPLAY_VER(i915) >= 11) {
2029 if (wp->y_tiled) {
2030 int extra_lines;
2031
2032 if (lines % wp->y_min_scanlines == 0)
2033 extra_lines = wp->y_min_scanlines;
2034 else
2035 extra_lines = wp->y_min_scanlines * 2 -
2036 lines % wp->y_min_scanlines;
2037
2038 min_ddb_alloc = mul_round_up_u32_fixed16(lines + extra_lines,
2039 wp->plane_blocks_per_line);
2040 } else {
2041 min_ddb_alloc = blocks + DIV_ROUND_UP(blocks, 10);
2042 }
2043 }
2044
2045 if (!skl_wm_has_lines(i915, level))
2046 lines = 0;
2047
2048 if (lines > skl_wm_max_lines(i915)) {
2049 /* reject it */
2050 result->min_ddb_alloc = U16_MAX;
2051 return;
2052 }
2053
2054 /*
2055 * If lines is valid, assume we can use this watermark level
2056 * for now. We'll come back and disable it after we calculate the
2057 * DDB allocation if it turns out we don't actually have enough
2058 * blocks to satisfy it.
2059 */
2060 result->blocks = blocks;
2061 result->lines = lines;
2062 /* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */
2063 result->min_ddb_alloc = max(min_ddb_alloc, blocks) + 1;
2064 result->enable = true;
2065 result->auto_min_alloc_wm_enable = xe3_auto_min_alloc_capable(plane, level);
2066
2067 if (DISPLAY_VER(i915) < 12 && i915->display.sagv.block_time_us)
2068 result->can_sagv = latency >= i915->display.sagv.block_time_us;
2069 }
2070
2071 static void
skl_compute_wm_levels(const struct intel_crtc_state * crtc_state,struct intel_plane * plane,const struct skl_wm_params * wm_params,struct skl_wm_level * levels)2072 skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
2073 struct intel_plane *plane,
2074 const struct skl_wm_params *wm_params,
2075 struct skl_wm_level *levels)
2076 {
2077 struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
2078 struct skl_wm_level *result_prev = &levels[0];
2079 int level;
2080
2081 for (level = 0; level < i915->display.wm.num_levels; level++) {
2082 struct skl_wm_level *result = &levels[level];
2083 unsigned int latency = skl_wm_latency(i915, level, wm_params);
2084
2085 skl_compute_plane_wm(crtc_state, plane, level, latency,
2086 wm_params, result_prev, result);
2087
2088 result_prev = result;
2089 }
2090 }
2091
tgl_compute_sagv_wm(const struct intel_crtc_state * crtc_state,struct intel_plane * plane,const struct skl_wm_params * wm_params,struct skl_plane_wm * plane_wm)2092 static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state,
2093 struct intel_plane *plane,
2094 const struct skl_wm_params *wm_params,
2095 struct skl_plane_wm *plane_wm)
2096 {
2097 struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
2098 struct skl_wm_level *sagv_wm = &plane_wm->sagv.wm0;
2099 struct skl_wm_level *levels = plane_wm->wm;
2100 unsigned int latency = 0;
2101
2102 if (i915->display.sagv.block_time_us)
2103 latency = i915->display.sagv.block_time_us +
2104 skl_wm_latency(i915, 0, wm_params);
2105
2106 skl_compute_plane_wm(crtc_state, plane, 0, latency,
2107 wm_params, &levels[0],
2108 sagv_wm);
2109 }
2110
skl_compute_transition_wm(struct drm_i915_private * i915,struct skl_wm_level * trans_wm,const struct skl_wm_level * wm0,const struct skl_wm_params * wp)2111 static void skl_compute_transition_wm(struct drm_i915_private *i915,
2112 struct skl_wm_level *trans_wm,
2113 const struct skl_wm_level *wm0,
2114 const struct skl_wm_params *wp)
2115 {
2116 u16 trans_min, trans_amount, trans_y_tile_min;
2117 u16 wm0_blocks, trans_offset, blocks;
2118
2119 /* Transition WM don't make any sense if ipc is disabled */
2120 if (!skl_watermark_ipc_enabled(i915))
2121 return;
2122
2123 /*
2124 * WaDisableTWM:skl,kbl,cfl,bxt
2125 * Transition WM are not recommended by HW team for GEN9
2126 */
2127 if (DISPLAY_VER(i915) == 9)
2128 return;
2129
2130 if (DISPLAY_VER(i915) >= 11)
2131 trans_min = 4;
2132 else
2133 trans_min = 14;
2134
2135 /* Display WA #1140: glk,cnl */
2136 if (DISPLAY_VER(i915) == 10)
2137 trans_amount = 0;
2138 else
2139 trans_amount = 10; /* This is configurable amount */
2140
2141 trans_offset = trans_min + trans_amount;
2142
2143 /*
2144 * The spec asks for Selected Result Blocks for wm0 (the real value),
2145 * not Result Blocks (the integer value). Pay attention to the capital
2146 * letters. The value wm_l0->blocks is actually Result Blocks, but
2147 * since Result Blocks is the ceiling of Selected Result Blocks plus 1,
2148 * and since we later will have to get the ceiling of the sum in the
2149 * transition watermarks calculation, we can just pretend Selected
2150 * Result Blocks is Result Blocks minus 1 and it should work for the
2151 * current platforms.
2152 */
2153 wm0_blocks = wm0->blocks - 1;
2154
2155 if (wp->y_tiled) {
2156 trans_y_tile_min =
2157 (u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum);
2158 blocks = max(wm0_blocks, trans_y_tile_min) + trans_offset;
2159 } else {
2160 blocks = wm0_blocks + trans_offset;
2161 }
2162 blocks++;
2163
2164 /*
2165 * Just assume we can enable the transition watermark. After
2166 * computing the DDB we'll come back and disable it if that
2167 * assumption turns out to be false.
2168 */
2169 trans_wm->blocks = blocks;
2170 trans_wm->min_ddb_alloc = max_t(u16, wm0->min_ddb_alloc, blocks + 1);
2171 trans_wm->enable = true;
2172 }
2173
skl_build_plane_wm_single(struct intel_crtc_state * crtc_state,const struct intel_plane_state * plane_state,struct intel_plane * plane,int color_plane)2174 static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
2175 const struct intel_plane_state *plane_state,
2176 struct intel_plane *plane, int color_plane)
2177 {
2178 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2179 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2180 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id];
2181 struct skl_wm_params wm_params;
2182 int ret;
2183
2184 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
2185 &wm_params, color_plane);
2186 if (ret)
2187 return ret;
2188
2189 skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->wm);
2190
2191 skl_compute_transition_wm(i915, &wm->trans_wm,
2192 &wm->wm[0], &wm_params);
2193
2194 if (DISPLAY_VER(i915) >= 12) {
2195 tgl_compute_sagv_wm(crtc_state, plane, &wm_params, wm);
2196
2197 skl_compute_transition_wm(i915, &wm->sagv.trans_wm,
2198 &wm->sagv.wm0, &wm_params);
2199 }
2200
2201 return 0;
2202 }
2203
skl_build_plane_wm_uv(struct intel_crtc_state * crtc_state,const struct intel_plane_state * plane_state,struct intel_plane * plane)2204 static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
2205 const struct intel_plane_state *plane_state,
2206 struct intel_plane *plane)
2207 {
2208 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id];
2209 struct skl_wm_params wm_params;
2210 int ret;
2211
2212 wm->is_planar = true;
2213
2214 /* uv plane watermarks must also be validated for NV12/Planar */
2215 ret = skl_compute_plane_wm_params(crtc_state, plane_state,
2216 &wm_params, 1);
2217 if (ret)
2218 return ret;
2219
2220 skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->uv_wm);
2221
2222 return 0;
2223 }
2224
skl_build_plane_wm(struct intel_crtc_state * crtc_state,const struct intel_plane_state * plane_state)2225 static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
2226 const struct intel_plane_state *plane_state)
2227 {
2228 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2229 enum plane_id plane_id = plane->id;
2230 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
2231 const struct drm_framebuffer *fb = plane_state->hw.fb;
2232 int ret;
2233
2234 memset(wm, 0, sizeof(*wm));
2235
2236 if (!intel_wm_plane_visible(crtc_state, plane_state))
2237 return 0;
2238
2239 ret = skl_build_plane_wm_single(crtc_state, plane_state,
2240 plane, 0);
2241 if (ret)
2242 return ret;
2243
2244 if (fb->format->is_yuv && fb->format->num_planes > 1) {
2245 ret = skl_build_plane_wm_uv(crtc_state, plane_state,
2246 plane);
2247 if (ret)
2248 return ret;
2249 }
2250
2251 return 0;
2252 }
2253
icl_build_plane_wm(struct intel_crtc_state * crtc_state,const struct intel_plane_state * plane_state)2254 static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
2255 const struct intel_plane_state *plane_state)
2256 {
2257 struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2258 struct drm_i915_private *i915 = to_i915(plane->base.dev);
2259 enum plane_id plane_id = plane->id;
2260 struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
2261 int ret;
2262
2263 /* Watermarks calculated on UV plane */
2264 if (plane_state->is_y_plane)
2265 return 0;
2266
2267 memset(wm, 0, sizeof(*wm));
2268
2269 if (plane_state->planar_linked_plane) {
2270 const struct drm_framebuffer *fb = plane_state->hw.fb;
2271
2272 drm_WARN_ON(&i915->drm,
2273 !intel_wm_plane_visible(crtc_state, plane_state));
2274 drm_WARN_ON(&i915->drm, !fb->format->is_yuv ||
2275 fb->format->num_planes == 1);
2276
2277 ret = skl_build_plane_wm_single(crtc_state, plane_state,
2278 plane_state->planar_linked_plane, 0);
2279 if (ret)
2280 return ret;
2281
2282 ret = skl_build_plane_wm_single(crtc_state, plane_state,
2283 plane, 1);
2284 if (ret)
2285 return ret;
2286 } else if (intel_wm_plane_visible(crtc_state, plane_state)) {
2287 ret = skl_build_plane_wm_single(crtc_state, plane_state,
2288 plane, 0);
2289 if (ret)
2290 return ret;
2291 }
2292
2293 return 0;
2294 }
2295
2296 static int
cdclk_prefill_adjustment(const struct intel_crtc_state * crtc_state)2297 cdclk_prefill_adjustment(const struct intel_crtc_state *crtc_state)
2298 {
2299 struct intel_display *display = to_intel_display(crtc_state);
2300 struct intel_atomic_state *state =
2301 to_intel_atomic_state(crtc_state->uapi.state);
2302 const struct intel_cdclk_state *cdclk_state;
2303
2304 cdclk_state = intel_atomic_get_cdclk_state(state);
2305 if (IS_ERR(cdclk_state)) {
2306 drm_WARN_ON(display->drm, PTR_ERR(cdclk_state));
2307 return 1;
2308 }
2309
2310 return min(1, DIV_ROUND_UP(crtc_state->pixel_rate,
2311 2 * cdclk_state->logical.cdclk));
2312 }
2313
2314 static int
dsc_prefill_latency(const struct intel_crtc_state * crtc_state)2315 dsc_prefill_latency(const struct intel_crtc_state *crtc_state)
2316 {
2317 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2318 const struct intel_crtc_scaler_state *scaler_state =
2319 &crtc_state->scaler_state;
2320 int linetime = DIV_ROUND_UP(1000 * crtc_state->hw.adjusted_mode.htotal,
2321 crtc_state->hw.adjusted_mode.clock);
2322 int num_scaler_users = hweight32(scaler_state->scaler_users);
2323 int chroma_downscaling_factor =
2324 crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 ? 2 : 1;
2325 u32 dsc_prefill_latency = 0;
2326
2327 if (!crtc_state->dsc.compression_enable ||
2328 !num_scaler_users ||
2329 num_scaler_users > crtc->num_scalers)
2330 return dsc_prefill_latency;
2331
2332 dsc_prefill_latency = DIV_ROUND_UP(15 * linetime * chroma_downscaling_factor, 10);
2333
2334 for (int i = 0; i < num_scaler_users; i++) {
2335 u64 hscale_k, vscale_k;
2336
2337 hscale_k = max(1000, mul_u32_u32(scaler_state->scalers[i].hscale, 1000) >> 16);
2338 vscale_k = max(1000, mul_u32_u32(scaler_state->scalers[i].vscale, 1000) >> 16);
2339 dsc_prefill_latency = DIV_ROUND_UP_ULL(dsc_prefill_latency * hscale_k * vscale_k,
2340 1000000);
2341 }
2342
2343 dsc_prefill_latency *= cdclk_prefill_adjustment(crtc_state);
2344
2345 return intel_usecs_to_scanlines(&crtc_state->hw.adjusted_mode, dsc_prefill_latency);
2346 }
2347
2348 static int
scaler_prefill_latency(const struct intel_crtc_state * crtc_state)2349 scaler_prefill_latency(const struct intel_crtc_state *crtc_state)
2350 {
2351 const struct intel_crtc_scaler_state *scaler_state =
2352 &crtc_state->scaler_state;
2353 int num_scaler_users = hweight32(scaler_state->scaler_users);
2354 int scaler_prefill_latency = 0;
2355 int linetime = DIV_ROUND_UP(1000 * crtc_state->hw.adjusted_mode.htotal,
2356 crtc_state->hw.adjusted_mode.clock);
2357
2358 if (!num_scaler_users)
2359 return scaler_prefill_latency;
2360
2361 scaler_prefill_latency = 4 * linetime;
2362
2363 if (num_scaler_users > 1) {
2364 u64 hscale_k = max(1000, mul_u32_u32(scaler_state->scalers[0].hscale, 1000) >> 16);
2365 u64 vscale_k = max(1000, mul_u32_u32(scaler_state->scalers[0].vscale, 1000) >> 16);
2366 int chroma_downscaling_factor =
2367 crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 ? 2 : 1;
2368 int latency;
2369
2370 latency = DIV_ROUND_UP_ULL((4 * linetime * hscale_k * vscale_k *
2371 chroma_downscaling_factor), 1000000);
2372 scaler_prefill_latency += latency;
2373 }
2374
2375 scaler_prefill_latency *= cdclk_prefill_adjustment(crtc_state);
2376
2377 return intel_usecs_to_scanlines(&crtc_state->hw.adjusted_mode, scaler_prefill_latency);
2378 }
2379
2380 static bool
skl_is_vblank_too_short(const struct intel_crtc_state * crtc_state,int wm0_lines,int latency)2381 skl_is_vblank_too_short(const struct intel_crtc_state *crtc_state,
2382 int wm0_lines, int latency)
2383 {
2384 const struct drm_display_mode *adjusted_mode =
2385 &crtc_state->hw.adjusted_mode;
2386
2387 return crtc_state->framestart_delay +
2388 intel_usecs_to_scanlines(adjusted_mode, latency) +
2389 scaler_prefill_latency(crtc_state) +
2390 dsc_prefill_latency(crtc_state) +
2391 wm0_lines >
2392 adjusted_mode->crtc_vtotal - adjusted_mode->crtc_vblank_start;
2393 }
2394
skl_max_wm0_lines(const struct intel_crtc_state * crtc_state)2395 static int skl_max_wm0_lines(const struct intel_crtc_state *crtc_state)
2396 {
2397 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2398 enum plane_id plane_id;
2399 int wm0_lines = 0;
2400
2401 for_each_plane_id_on_crtc(crtc, plane_id) {
2402 const struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id];
2403
2404 /* FIXME what about !skl_wm_has_lines() platforms? */
2405 wm0_lines = max_t(int, wm0_lines, wm->wm[0].lines);
2406 }
2407
2408 return wm0_lines;
2409 }
2410
skl_max_wm_level_for_vblank(struct intel_crtc_state * crtc_state,int wm0_lines)2411 static int skl_max_wm_level_for_vblank(struct intel_crtc_state *crtc_state,
2412 int wm0_lines)
2413 {
2414 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2415 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2416 int level;
2417
2418 for (level = i915->display.wm.num_levels - 1; level >= 0; level--) {
2419 int latency;
2420
2421 /* FIXME should we care about the latency w/a's? */
2422 latency = skl_wm_latency(i915, level, NULL);
2423 if (latency == 0)
2424 continue;
2425
2426 /* FIXME is it correct to use 0 latency for wm0 here? */
2427 if (level == 0)
2428 latency = 0;
2429
2430 if (!skl_is_vblank_too_short(crtc_state, wm0_lines, latency))
2431 return level;
2432 }
2433
2434 return -EINVAL;
2435 }
2436
skl_wm_check_vblank(struct intel_crtc_state * crtc_state)2437 static int skl_wm_check_vblank(struct intel_crtc_state *crtc_state)
2438 {
2439 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2440 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2441 int wm0_lines, level;
2442
2443 if (!crtc_state->hw.active)
2444 return 0;
2445
2446 wm0_lines = skl_max_wm0_lines(crtc_state);
2447
2448 level = skl_max_wm_level_for_vblank(crtc_state, wm0_lines);
2449 if (level < 0)
2450 return level;
2451
2452 /*
2453 * PSR needs to toggle LATENCY_REPORTING_REMOVED_PIPE_*
2454 * based on whether we're limited by the vblank duration.
2455 */
2456 crtc_state->wm_level_disabled = level < i915->display.wm.num_levels - 1;
2457
2458 for (level++; level < i915->display.wm.num_levels; level++) {
2459 enum plane_id plane_id;
2460
2461 for_each_plane_id_on_crtc(crtc, plane_id) {
2462 struct skl_plane_wm *wm =
2463 &crtc_state->wm.skl.optimal.planes[plane_id];
2464
2465 /*
2466 * FIXME just clear enable or flag the entire
2467 * thing as bad via min_ddb_alloc=U16_MAX?
2468 */
2469 wm->wm[level].enable = false;
2470 wm->uv_wm[level].enable = false;
2471 }
2472 }
2473
2474 if (DISPLAY_VER(i915) >= 12 &&
2475 i915->display.sagv.block_time_us &&
2476 skl_is_vblank_too_short(crtc_state, wm0_lines,
2477 i915->display.sagv.block_time_us)) {
2478 enum plane_id plane_id;
2479
2480 for_each_plane_id_on_crtc(crtc, plane_id) {
2481 struct skl_plane_wm *wm =
2482 &crtc_state->wm.skl.optimal.planes[plane_id];
2483
2484 wm->sagv.wm0.enable = false;
2485 wm->sagv.trans_wm.enable = false;
2486 }
2487 }
2488
2489 return 0;
2490 }
2491
skl_build_pipe_wm(struct intel_atomic_state * state,struct intel_crtc * crtc)2492 static int skl_build_pipe_wm(struct intel_atomic_state *state,
2493 struct intel_crtc *crtc)
2494 {
2495 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2496 struct intel_crtc_state *crtc_state =
2497 intel_atomic_get_new_crtc_state(state, crtc);
2498 const struct intel_plane_state *plane_state;
2499 struct intel_plane *plane;
2500 int ret, i;
2501
2502 for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
2503 /*
2504 * FIXME should perhaps check {old,new}_plane_crtc->hw.crtc
2505 * instead but we don't populate that correctly for NV12 Y
2506 * planes so for now hack this.
2507 */
2508 if (plane->pipe != crtc->pipe)
2509 continue;
2510
2511 if (DISPLAY_VER(i915) >= 11)
2512 ret = icl_build_plane_wm(crtc_state, plane_state);
2513 else
2514 ret = skl_build_plane_wm(crtc_state, plane_state);
2515 if (ret)
2516 return ret;
2517 }
2518
2519 crtc_state->wm.skl.optimal = crtc_state->wm.skl.raw;
2520
2521 return skl_wm_check_vblank(crtc_state);
2522 }
2523
skl_wm_level_equals(const struct skl_wm_level * l1,const struct skl_wm_level * l2)2524 static bool skl_wm_level_equals(const struct skl_wm_level *l1,
2525 const struct skl_wm_level *l2)
2526 {
2527 return l1->enable == l2->enable &&
2528 l1->ignore_lines == l2->ignore_lines &&
2529 l1->lines == l2->lines &&
2530 l1->blocks == l2->blocks &&
2531 l1->auto_min_alloc_wm_enable == l2->auto_min_alloc_wm_enable;
2532 }
2533
skl_plane_wm_equals(struct drm_i915_private * i915,const struct skl_plane_wm * wm1,const struct skl_plane_wm * wm2)2534 static bool skl_plane_wm_equals(struct drm_i915_private *i915,
2535 const struct skl_plane_wm *wm1,
2536 const struct skl_plane_wm *wm2)
2537 {
2538 struct intel_display *display = &i915->display;
2539 int level;
2540
2541 for (level = 0; level < display->wm.num_levels; level++) {
2542 /*
2543 * We don't check uv_wm as the hardware doesn't actually
2544 * use it. It only gets used for calculating the required
2545 * ddb allocation.
2546 */
2547 if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]))
2548 return false;
2549 }
2550
2551 return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm) &&
2552 skl_wm_level_equals(&wm1->sagv.wm0, &wm2->sagv.wm0) &&
2553 skl_wm_level_equals(&wm1->sagv.trans_wm, &wm2->sagv.trans_wm);
2554 }
2555
skl_ddb_entries_overlap(const struct skl_ddb_entry * a,const struct skl_ddb_entry * b)2556 static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
2557 const struct skl_ddb_entry *b)
2558 {
2559 return a->start < b->end && b->start < a->end;
2560 }
2561
skl_ddb_entry_union(struct skl_ddb_entry * a,const struct skl_ddb_entry * b)2562 static void skl_ddb_entry_union(struct skl_ddb_entry *a,
2563 const struct skl_ddb_entry *b)
2564 {
2565 if (a->end && b->end) {
2566 a->start = min(a->start, b->start);
2567 a->end = max(a->end, b->end);
2568 } else if (b->end) {
2569 a->start = b->start;
2570 a->end = b->end;
2571 }
2572 }
2573
skl_ddb_allocation_overlaps(const struct skl_ddb_entry * ddb,const struct skl_ddb_entry * entries,int num_entries,int ignore_idx)2574 bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
2575 const struct skl_ddb_entry *entries,
2576 int num_entries, int ignore_idx)
2577 {
2578 int i;
2579
2580 for (i = 0; i < num_entries; i++) {
2581 if (i != ignore_idx &&
2582 skl_ddb_entries_overlap(ddb, &entries[i]))
2583 return true;
2584 }
2585
2586 return false;
2587 }
2588
2589 static int
skl_ddb_add_affected_planes(struct intel_atomic_state * state,struct intel_crtc * crtc)2590 skl_ddb_add_affected_planes(struct intel_atomic_state *state,
2591 struct intel_crtc *crtc)
2592 {
2593 struct drm_i915_private *i915 = to_i915(state->base.dev);
2594 const struct intel_crtc_state *old_crtc_state =
2595 intel_atomic_get_old_crtc_state(state, crtc);
2596 struct intel_crtc_state *new_crtc_state =
2597 intel_atomic_get_new_crtc_state(state, crtc);
2598 struct intel_plane *plane;
2599
2600 for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2601 struct intel_plane_state *plane_state;
2602 enum plane_id plane_id = plane->id;
2603
2604 if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb[plane_id],
2605 &new_crtc_state->wm.skl.plane_ddb[plane_id]) &&
2606 skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id],
2607 &new_crtc_state->wm.skl.plane_ddb_y[plane_id]))
2608 continue;
2609
2610 if (new_crtc_state->do_async_flip) {
2611 drm_dbg_kms(&i915->drm, "[PLANE:%d:%s] Can't change DDB during async flip\n",
2612 plane->base.base.id, plane->base.name);
2613 return -EINVAL;
2614 }
2615
2616 plane_state = intel_atomic_get_plane_state(state, plane);
2617 if (IS_ERR(plane_state))
2618 return PTR_ERR(plane_state);
2619
2620 new_crtc_state->update_planes |= BIT(plane_id);
2621 new_crtc_state->async_flip_planes = 0;
2622 new_crtc_state->do_async_flip = false;
2623 }
2624
2625 return 0;
2626 }
2627
intel_dbuf_enabled_slices(const struct intel_dbuf_state * dbuf_state)2628 static u8 intel_dbuf_enabled_slices(const struct intel_dbuf_state *dbuf_state)
2629 {
2630 struct drm_i915_private *i915 = to_i915(dbuf_state->base.state->base.dev);
2631 u8 enabled_slices;
2632 enum pipe pipe;
2633
2634 /*
2635 * FIXME: For now we always enable slice S1 as per
2636 * the Bspec display initialization sequence.
2637 */
2638 enabled_slices = BIT(DBUF_S1);
2639
2640 for_each_pipe(i915, pipe)
2641 enabled_slices |= dbuf_state->slices[pipe];
2642
2643 return enabled_slices;
2644 }
2645
2646 static int
skl_compute_ddb(struct intel_atomic_state * state)2647 skl_compute_ddb(struct intel_atomic_state *state)
2648 {
2649 struct intel_display *display = to_intel_display(state);
2650 struct drm_i915_private *i915 = to_i915(state->base.dev);
2651 const struct intel_dbuf_state *old_dbuf_state;
2652 struct intel_dbuf_state *new_dbuf_state = NULL;
2653 struct intel_crtc_state *new_crtc_state;
2654 struct intel_crtc *crtc;
2655 int ret, i;
2656
2657 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2658 new_dbuf_state = intel_atomic_get_dbuf_state(state);
2659 if (IS_ERR(new_dbuf_state))
2660 return PTR_ERR(new_dbuf_state);
2661
2662 old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
2663 break;
2664 }
2665
2666 if (!new_dbuf_state)
2667 return 0;
2668
2669 new_dbuf_state->active_pipes =
2670 intel_calc_active_pipes(state, old_dbuf_state->active_pipes);
2671
2672 if (old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) {
2673 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
2674 if (ret)
2675 return ret;
2676 }
2677
2678 if (HAS_MBUS_JOINING(display)) {
2679 new_dbuf_state->joined_mbus =
2680 adlp_check_mbus_joined(new_dbuf_state->active_pipes);
2681
2682 if (old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
2683 ret = intel_cdclk_state_set_joined_mbus(state, new_dbuf_state->joined_mbus);
2684 if (ret)
2685 return ret;
2686 }
2687 }
2688
2689 for_each_intel_crtc(&i915->drm, crtc) {
2690 enum pipe pipe = crtc->pipe;
2691
2692 new_dbuf_state->slices[pipe] =
2693 skl_compute_dbuf_slices(crtc, new_dbuf_state->active_pipes,
2694 new_dbuf_state->joined_mbus);
2695
2696 if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe])
2697 continue;
2698
2699 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
2700 if (ret)
2701 return ret;
2702 }
2703
2704 new_dbuf_state->enabled_slices = intel_dbuf_enabled_slices(new_dbuf_state);
2705
2706 if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices ||
2707 old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
2708 ret = intel_atomic_serialize_global_state(&new_dbuf_state->base);
2709 if (ret)
2710 return ret;
2711
2712 drm_dbg_kms(&i915->drm,
2713 "Enabled dbuf slices 0x%x -> 0x%x (total dbuf slices 0x%x), mbus joined? %s->%s\n",
2714 old_dbuf_state->enabled_slices,
2715 new_dbuf_state->enabled_slices,
2716 DISPLAY_INFO(i915)->dbuf.slice_mask,
2717 str_yes_no(old_dbuf_state->joined_mbus),
2718 str_yes_no(new_dbuf_state->joined_mbus));
2719 }
2720
2721 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2722 enum pipe pipe = crtc->pipe;
2723
2724 new_dbuf_state->weight[pipe] = intel_crtc_ddb_weight(new_crtc_state);
2725
2726 if (old_dbuf_state->weight[pipe] == new_dbuf_state->weight[pipe])
2727 continue;
2728
2729 ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
2730 if (ret)
2731 return ret;
2732 }
2733
2734 for_each_intel_crtc(&i915->drm, crtc) {
2735 ret = skl_crtc_allocate_ddb(state, crtc);
2736 if (ret)
2737 return ret;
2738 }
2739
2740 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2741 ret = skl_crtc_allocate_plane_ddb(state, crtc);
2742 if (ret)
2743 return ret;
2744
2745 ret = skl_ddb_add_affected_planes(state, crtc);
2746 if (ret)
2747 return ret;
2748 }
2749
2750 return 0;
2751 }
2752
enast(bool enable)2753 static char enast(bool enable)
2754 {
2755 return enable ? '*' : ' ';
2756 }
2757
2758 static void
skl_print_wm_changes(struct intel_atomic_state * state)2759 skl_print_wm_changes(struct intel_atomic_state *state)
2760 {
2761 struct drm_i915_private *i915 = to_i915(state->base.dev);
2762 const struct intel_crtc_state *old_crtc_state;
2763 const struct intel_crtc_state *new_crtc_state;
2764 struct intel_plane *plane;
2765 struct intel_crtc *crtc;
2766 int i;
2767
2768 if (!drm_debug_enabled(DRM_UT_KMS))
2769 return;
2770
2771 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
2772 new_crtc_state, i) {
2773 const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
2774
2775 old_pipe_wm = &old_crtc_state->wm.skl.optimal;
2776 new_pipe_wm = &new_crtc_state->wm.skl.optimal;
2777
2778 for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2779 enum plane_id plane_id = plane->id;
2780 const struct skl_ddb_entry *old, *new;
2781
2782 old = &old_crtc_state->wm.skl.plane_ddb[plane_id];
2783 new = &new_crtc_state->wm.skl.plane_ddb[plane_id];
2784
2785 if (skl_ddb_entry_equal(old, new))
2786 continue;
2787
2788 drm_dbg_kms(&i915->drm,
2789 "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
2790 plane->base.base.id, plane->base.name,
2791 old->start, old->end, new->start, new->end,
2792 skl_ddb_entry_size(old), skl_ddb_entry_size(new));
2793 }
2794
2795 for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2796 enum plane_id plane_id = plane->id;
2797 const struct skl_plane_wm *old_wm, *new_wm;
2798
2799 old_wm = &old_pipe_wm->planes[plane_id];
2800 new_wm = &new_pipe_wm->planes[plane_id];
2801
2802 if (skl_plane_wm_equals(i915, old_wm, new_wm))
2803 continue;
2804
2805 drm_dbg_kms(&i915->drm,
2806 "[PLANE:%d:%s] level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm"
2807 " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm\n",
2808 plane->base.base.id, plane->base.name,
2809 enast(old_wm->wm[0].enable), enast(old_wm->wm[1].enable),
2810 enast(old_wm->wm[2].enable), enast(old_wm->wm[3].enable),
2811 enast(old_wm->wm[4].enable), enast(old_wm->wm[5].enable),
2812 enast(old_wm->wm[6].enable), enast(old_wm->wm[7].enable),
2813 enast(old_wm->trans_wm.enable),
2814 enast(old_wm->sagv.wm0.enable),
2815 enast(old_wm->sagv.trans_wm.enable),
2816 enast(new_wm->wm[0].enable), enast(new_wm->wm[1].enable),
2817 enast(new_wm->wm[2].enable), enast(new_wm->wm[3].enable),
2818 enast(new_wm->wm[4].enable), enast(new_wm->wm[5].enable),
2819 enast(new_wm->wm[6].enable), enast(new_wm->wm[7].enable),
2820 enast(new_wm->trans_wm.enable),
2821 enast(new_wm->sagv.wm0.enable),
2822 enast(new_wm->sagv.trans_wm.enable));
2823
2824 drm_dbg_kms(&i915->drm,
2825 "[PLANE:%d:%s] lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d"
2826 " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d\n",
2827 plane->base.base.id, plane->base.name,
2828 enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].lines,
2829 enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].lines,
2830 enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].lines,
2831 enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].lines,
2832 enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].lines,
2833 enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].lines,
2834 enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].lines,
2835 enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].lines,
2836 enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.lines,
2837 enast(old_wm->sagv.wm0.ignore_lines), old_wm->sagv.wm0.lines,
2838 enast(old_wm->sagv.trans_wm.ignore_lines), old_wm->sagv.trans_wm.lines,
2839 enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].lines,
2840 enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].lines,
2841 enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].lines,
2842 enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].lines,
2843 enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].lines,
2844 enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].lines,
2845 enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].lines,
2846 enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].lines,
2847 enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.lines,
2848 enast(new_wm->sagv.wm0.ignore_lines), new_wm->sagv.wm0.lines,
2849 enast(new_wm->sagv.trans_wm.ignore_lines), new_wm->sagv.trans_wm.lines);
2850
2851 drm_dbg_kms(&i915->drm,
2852 "[PLANE:%d:%s] blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
2853 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
2854 plane->base.base.id, plane->base.name,
2855 old_wm->wm[0].blocks, old_wm->wm[1].blocks,
2856 old_wm->wm[2].blocks, old_wm->wm[3].blocks,
2857 old_wm->wm[4].blocks, old_wm->wm[5].blocks,
2858 old_wm->wm[6].blocks, old_wm->wm[7].blocks,
2859 old_wm->trans_wm.blocks,
2860 old_wm->sagv.wm0.blocks,
2861 old_wm->sagv.trans_wm.blocks,
2862 new_wm->wm[0].blocks, new_wm->wm[1].blocks,
2863 new_wm->wm[2].blocks, new_wm->wm[3].blocks,
2864 new_wm->wm[4].blocks, new_wm->wm[5].blocks,
2865 new_wm->wm[6].blocks, new_wm->wm[7].blocks,
2866 new_wm->trans_wm.blocks,
2867 new_wm->sagv.wm0.blocks,
2868 new_wm->sagv.trans_wm.blocks);
2869
2870 drm_dbg_kms(&i915->drm,
2871 "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
2872 " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
2873 plane->base.base.id, plane->base.name,
2874 old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
2875 old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
2876 old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
2877 old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
2878 old_wm->trans_wm.min_ddb_alloc,
2879 old_wm->sagv.wm0.min_ddb_alloc,
2880 old_wm->sagv.trans_wm.min_ddb_alloc,
2881 new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
2882 new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
2883 new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
2884 new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
2885 new_wm->trans_wm.min_ddb_alloc,
2886 new_wm->sagv.wm0.min_ddb_alloc,
2887 new_wm->sagv.trans_wm.min_ddb_alloc);
2888 }
2889 }
2890 }
2891
skl_plane_selected_wm_equals(struct intel_plane * plane,const struct skl_pipe_wm * old_pipe_wm,const struct skl_pipe_wm * new_pipe_wm)2892 static bool skl_plane_selected_wm_equals(struct intel_plane *plane,
2893 const struct skl_pipe_wm *old_pipe_wm,
2894 const struct skl_pipe_wm *new_pipe_wm)
2895 {
2896 struct intel_display *display = to_intel_display(plane);
2897 int level;
2898
2899 for (level = 0; level < display->wm.num_levels; level++) {
2900 /*
2901 * We don't check uv_wm as the hardware doesn't actually
2902 * use it. It only gets used for calculating the required
2903 * ddb allocation.
2904 */
2905 if (!skl_wm_level_equals(skl_plane_wm_level(old_pipe_wm, plane->id, level),
2906 skl_plane_wm_level(new_pipe_wm, plane->id, level)))
2907 return false;
2908 }
2909
2910 if (HAS_HW_SAGV_WM(display)) {
2911 const struct skl_plane_wm *old_wm = &old_pipe_wm->planes[plane->id];
2912 const struct skl_plane_wm *new_wm = &new_pipe_wm->planes[plane->id];
2913
2914 if (!skl_wm_level_equals(&old_wm->sagv.wm0, &new_wm->sagv.wm0) ||
2915 !skl_wm_level_equals(&old_wm->sagv.trans_wm, &new_wm->sagv.trans_wm))
2916 return false;
2917 }
2918
2919 return skl_wm_level_equals(skl_plane_trans_wm(old_pipe_wm, plane->id),
2920 skl_plane_trans_wm(new_pipe_wm, plane->id));
2921 }
2922
2923 /*
2924 * To make sure the cursor watermark registers are always consistent
2925 * with our computed state the following scenario needs special
2926 * treatment:
2927 *
2928 * 1. enable cursor
2929 * 2. move cursor entirely offscreen
2930 * 3. disable cursor
2931 *
2932 * Step 2. does call .disable_plane() but does not zero the watermarks
2933 * (since we consider an offscreen cursor still active for the purposes
2934 * of watermarks). Step 3. would not normally call .disable_plane()
2935 * because the actual plane visibility isn't changing, and we don't
2936 * deallocate the cursor ddb until the pipe gets disabled. So we must
2937 * force step 3. to call .disable_plane() to update the watermark
2938 * registers properly.
2939 *
2940 * Other planes do not suffer from this issues as their watermarks are
2941 * calculated based on the actual plane visibility. The only time this
2942 * can trigger for the other planes is during the initial readout as the
2943 * default value of the watermarks registers is not zero.
2944 */
skl_wm_add_affected_planes(struct intel_atomic_state * state,struct intel_crtc * crtc)2945 static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
2946 struct intel_crtc *crtc)
2947 {
2948 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2949 const struct intel_crtc_state *old_crtc_state =
2950 intel_atomic_get_old_crtc_state(state, crtc);
2951 struct intel_crtc_state *new_crtc_state =
2952 intel_atomic_get_new_crtc_state(state, crtc);
2953 struct intel_plane *plane;
2954
2955 for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2956 struct intel_plane_state *plane_state;
2957 enum plane_id plane_id = plane->id;
2958
2959 /*
2960 * Force a full wm update for every plane on modeset.
2961 * Required because the reset value of the wm registers
2962 * is non-zero, whereas we want all disabled planes to
2963 * have zero watermarks. So if we turn off the relevant
2964 * power well the hardware state will go out of sync
2965 * with the software state.
2966 */
2967 if (!intel_crtc_needs_modeset(new_crtc_state) &&
2968 skl_plane_selected_wm_equals(plane,
2969 &old_crtc_state->wm.skl.optimal,
2970 &new_crtc_state->wm.skl.optimal))
2971 continue;
2972
2973 if (new_crtc_state->do_async_flip) {
2974 drm_dbg_kms(&i915->drm, "[PLANE:%d:%s] Can't change watermarks during async flip\n",
2975 plane->base.base.id, plane->base.name);
2976 return -EINVAL;
2977 }
2978
2979 plane_state = intel_atomic_get_plane_state(state, plane);
2980 if (IS_ERR(plane_state))
2981 return PTR_ERR(plane_state);
2982
2983 new_crtc_state->update_planes |= BIT(plane_id);
2984 new_crtc_state->async_flip_planes = 0;
2985 new_crtc_state->do_async_flip = false;
2986 }
2987
2988 return 0;
2989 }
2990
2991 /*
2992 * If Fixed Refresh Rate or For VRR case Vmin = Vmax = Flipline:
2993 * Program DEEP PKG_C_LATENCY Pkg C with highest valid latency from
2994 * watermark level1 and up and above. If watermark level 1 is
2995 * invalid program it with all 1's.
2996 * Program PKG_C_LATENCY Added Wake Time = DSB execution time
2997 * If Variable Refresh Rate where Vmin != Vmax != Flipline:
2998 * Program DEEP PKG_C_LATENCY Pkg C with all 1's.
2999 * Program PKG_C_LATENCY Added Wake Time = 0
3000 */
3001 void
intel_program_dpkgc_latency(struct intel_atomic_state * state)3002 intel_program_dpkgc_latency(struct intel_atomic_state *state)
3003 {
3004 struct intel_display *display = to_intel_display(state);
3005 struct drm_i915_private *i915 = to_i915(display->drm);
3006 struct intel_crtc *crtc;
3007 struct intel_crtc_state *new_crtc_state;
3008 u32 latency = LNL_PKG_C_LATENCY_MASK;
3009 u32 added_wake_time = 0;
3010 u32 max_linetime = 0;
3011 u32 clear, val;
3012 bool fixed_refresh_rate = false;
3013 int i;
3014
3015 if (DISPLAY_VER(display) < 20)
3016 return;
3017
3018 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
3019 if (!new_crtc_state->vrr.enable ||
3020 (new_crtc_state->vrr.vmin == new_crtc_state->vrr.vmax &&
3021 new_crtc_state->vrr.vmin == new_crtc_state->vrr.flipline))
3022 fixed_refresh_rate = true;
3023
3024 max_linetime = max(new_crtc_state->linetime, max_linetime);
3025 }
3026
3027 if (fixed_refresh_rate) {
3028 added_wake_time = DSB_EXE_TIME +
3029 display->sagv.block_time_us;
3030
3031 latency = skl_watermark_max_latency(i915, 1);
3032
3033 /* Wa_22020432604 */
3034 if ((DISPLAY_VER(display) == 20 || DISPLAY_VER(display) == 30) && !latency) {
3035 latency += added_wake_time;
3036 added_wake_time = 0;
3037 }
3038
3039 /* Wa_22020299601 */
3040 if ((latency && max_linetime) &&
3041 (DISPLAY_VER(display) == 20 || DISPLAY_VER(display) == 30)) {
3042 latency = max_linetime * DIV_ROUND_UP(latency, max_linetime);
3043 } else if (!latency) {
3044 latency = LNL_PKG_C_LATENCY_MASK;
3045 }
3046 }
3047
3048 clear = LNL_ADDED_WAKE_TIME_MASK | LNL_PKG_C_LATENCY_MASK;
3049 val = REG_FIELD_PREP(LNL_PKG_C_LATENCY_MASK, latency) |
3050 REG_FIELD_PREP(LNL_ADDED_WAKE_TIME_MASK, added_wake_time);
3051
3052 intel_de_rmw(display, LNL_PKG_C_LATENCY, clear, val);
3053 }
3054
3055 static int
skl_compute_wm(struct intel_atomic_state * state)3056 skl_compute_wm(struct intel_atomic_state *state)
3057 {
3058 struct intel_crtc *crtc;
3059 struct intel_crtc_state __maybe_unused *new_crtc_state;
3060 int ret, i;
3061
3062 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
3063 ret = skl_build_pipe_wm(state, crtc);
3064 if (ret)
3065 return ret;
3066 }
3067
3068 ret = skl_compute_ddb(state);
3069 if (ret)
3070 return ret;
3071
3072 ret = intel_compute_sagv_mask(state);
3073 if (ret)
3074 return ret;
3075
3076 /*
3077 * skl_compute_ddb() will have adjusted the final watermarks
3078 * based on how much ddb is available. Now we can actually
3079 * check if the final watermarks changed.
3080 */
3081 for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
3082 ret = skl_wm_add_affected_planes(state, crtc);
3083 if (ret)
3084 return ret;
3085 }
3086
3087 skl_print_wm_changes(state);
3088
3089 return 0;
3090 }
3091
skl_wm_level_from_reg_val(struct intel_display * display,u32 val,struct skl_wm_level * level)3092 static void skl_wm_level_from_reg_val(struct intel_display *display,
3093 u32 val, struct skl_wm_level *level)
3094 {
3095 level->enable = val & PLANE_WM_EN;
3096 level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
3097 level->blocks = REG_FIELD_GET(PLANE_WM_BLOCKS_MASK, val);
3098 level->lines = REG_FIELD_GET(PLANE_WM_LINES_MASK, val);
3099 level->auto_min_alloc_wm_enable = DISPLAY_VER(display) >= 30 ?
3100 val & PLANE_WM_AUTO_MIN_ALLOC_EN : 0;
3101 }
3102
skl_pipe_wm_get_hw_state(struct intel_crtc * crtc,struct skl_pipe_wm * out)3103 static void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
3104 struct skl_pipe_wm *out)
3105 {
3106 struct intel_display *display = to_intel_display(crtc);
3107 enum pipe pipe = crtc->pipe;
3108 enum plane_id plane_id;
3109 int level;
3110 u32 val;
3111
3112 for_each_plane_id_on_crtc(crtc, plane_id) {
3113 struct skl_plane_wm *wm = &out->planes[plane_id];
3114
3115 for (level = 0; level < display->wm.num_levels; level++) {
3116 if (plane_id != PLANE_CURSOR)
3117 val = intel_de_read(display, PLANE_WM(pipe, plane_id, level));
3118 else
3119 val = intel_de_read(display, CUR_WM(pipe, level));
3120
3121 skl_wm_level_from_reg_val(display, val, &wm->wm[level]);
3122 }
3123
3124 if (plane_id != PLANE_CURSOR)
3125 val = intel_de_read(display, PLANE_WM_TRANS(pipe, plane_id));
3126 else
3127 val = intel_de_read(display, CUR_WM_TRANS(pipe));
3128
3129 skl_wm_level_from_reg_val(display, val, &wm->trans_wm);
3130
3131 if (HAS_HW_SAGV_WM(display)) {
3132 if (plane_id != PLANE_CURSOR)
3133 val = intel_de_read(display, PLANE_WM_SAGV(pipe, plane_id));
3134 else
3135 val = intel_de_read(display, CUR_WM_SAGV(pipe));
3136
3137 skl_wm_level_from_reg_val(display, val, &wm->sagv.wm0);
3138
3139 if (plane_id != PLANE_CURSOR)
3140 val = intel_de_read(display, PLANE_WM_SAGV_TRANS(pipe, plane_id));
3141 else
3142 val = intel_de_read(display, CUR_WM_SAGV_TRANS(pipe));
3143
3144 skl_wm_level_from_reg_val(display, val, &wm->sagv.trans_wm);
3145 } else if (DISPLAY_VER(display) >= 12) {
3146 wm->sagv.wm0 = wm->wm[0];
3147 wm->sagv.trans_wm = wm->trans_wm;
3148 }
3149 }
3150 }
3151
skl_wm_get_hw_state(struct drm_i915_private * i915)3152 static void skl_wm_get_hw_state(struct drm_i915_private *i915)
3153 {
3154 struct intel_display *display = &i915->display;
3155 struct intel_dbuf_state *dbuf_state =
3156 to_intel_dbuf_state(i915->display.dbuf.obj.state);
3157 struct intel_crtc *crtc;
3158
3159 if (HAS_MBUS_JOINING(display))
3160 dbuf_state->joined_mbus = intel_de_read(display, MBUS_CTL) & MBUS_JOIN;
3161
3162 dbuf_state->mdclk_cdclk_ratio = intel_mdclk_cdclk_ratio(display, &display->cdclk.hw);
3163 dbuf_state->active_pipes = 0;
3164
3165 for_each_intel_crtc(display->drm, crtc) {
3166 struct intel_crtc_state *crtc_state =
3167 to_intel_crtc_state(crtc->base.state);
3168 enum pipe pipe = crtc->pipe;
3169 unsigned int mbus_offset;
3170 enum plane_id plane_id;
3171 u8 slices;
3172
3173 memset(&crtc_state->wm.skl.optimal, 0,
3174 sizeof(crtc_state->wm.skl.optimal));
3175 if (crtc_state->hw.active) {
3176 skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal);
3177 dbuf_state->active_pipes |= BIT(pipe);
3178 }
3179 crtc_state->wm.skl.raw = crtc_state->wm.skl.optimal;
3180
3181 memset(&dbuf_state->ddb[pipe], 0, sizeof(dbuf_state->ddb[pipe]));
3182
3183 for_each_plane_id_on_crtc(crtc, plane_id) {
3184 struct skl_ddb_entry *ddb =
3185 &crtc_state->wm.skl.plane_ddb[plane_id];
3186 struct skl_ddb_entry *ddb_y =
3187 &crtc_state->wm.skl.plane_ddb_y[plane_id];
3188 u16 *min_ddb =
3189 &crtc_state->wm.skl.plane_min_ddb[plane_id];
3190 u16 *interim_ddb =
3191 &crtc_state->wm.skl.plane_interim_ddb[plane_id];
3192
3193 if (!crtc_state->hw.active)
3194 continue;
3195
3196 skl_ddb_get_hw_plane_state(i915, crtc->pipe,
3197 plane_id, ddb, ddb_y,
3198 min_ddb, interim_ddb);
3199
3200 skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb);
3201 skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_y);
3202 }
3203
3204 dbuf_state->weight[pipe] = intel_crtc_ddb_weight(crtc_state);
3205
3206 /*
3207 * Used for checking overlaps, so we need absolute
3208 * offsets instead of MBUS relative offsets.
3209 */
3210 slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes,
3211 dbuf_state->joined_mbus);
3212 mbus_offset = mbus_ddb_offset(i915, slices);
3213 crtc_state->wm.skl.ddb.start = mbus_offset + dbuf_state->ddb[pipe].start;
3214 crtc_state->wm.skl.ddb.end = mbus_offset + dbuf_state->ddb[pipe].end;
3215
3216 /* The slices actually used by the planes on the pipe */
3217 dbuf_state->slices[pipe] =
3218 skl_ddb_dbuf_slice_mask(i915, &crtc_state->wm.skl.ddb);
3219
3220 drm_dbg_kms(display->drm,
3221 "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x, mbus joined: %s\n",
3222 crtc->base.base.id, crtc->base.name,
3223 dbuf_state->slices[pipe], dbuf_state->ddb[pipe].start,
3224 dbuf_state->ddb[pipe].end, dbuf_state->active_pipes,
3225 str_yes_no(dbuf_state->joined_mbus));
3226 }
3227
3228 dbuf_state->enabled_slices = display->dbuf.enabled_slices;
3229 }
3230
skl_watermark_ipc_enabled(struct drm_i915_private * i915)3231 bool skl_watermark_ipc_enabled(struct drm_i915_private *i915)
3232 {
3233 return i915->display.wm.ipc_enabled;
3234 }
3235
skl_watermark_ipc_update(struct drm_i915_private * i915)3236 void skl_watermark_ipc_update(struct drm_i915_private *i915)
3237 {
3238 if (!HAS_IPC(i915))
3239 return;
3240
3241 intel_de_rmw(i915, DISP_ARB_CTL2, DISP_IPC_ENABLE,
3242 skl_watermark_ipc_enabled(i915) ? DISP_IPC_ENABLE : 0);
3243 }
3244
skl_watermark_ipc_can_enable(struct drm_i915_private * i915)3245 static bool skl_watermark_ipc_can_enable(struct drm_i915_private *i915)
3246 {
3247 /* Display WA #0477 WaDisableIPC: skl */
3248 if (IS_SKYLAKE(i915))
3249 return false;
3250
3251 /* Display WA #1141: SKL:all KBL:all CFL */
3252 if (IS_KABYLAKE(i915) ||
3253 IS_COFFEELAKE(i915) ||
3254 IS_COMETLAKE(i915))
3255 return i915->dram_info.symmetric_memory;
3256
3257 return true;
3258 }
3259
skl_watermark_ipc_init(struct drm_i915_private * i915)3260 void skl_watermark_ipc_init(struct drm_i915_private *i915)
3261 {
3262 if (!HAS_IPC(i915))
3263 return;
3264
3265 i915->display.wm.ipc_enabled = skl_watermark_ipc_can_enable(i915);
3266
3267 skl_watermark_ipc_update(i915);
3268 }
3269
3270 static void
adjust_wm_latency(struct drm_i915_private * i915,u16 wm[],int num_levels,int read_latency)3271 adjust_wm_latency(struct drm_i915_private *i915,
3272 u16 wm[], int num_levels, int read_latency)
3273 {
3274 bool wm_lv_0_adjust_needed = i915->dram_info.wm_lv_0_adjust_needed;
3275 int i, level;
3276
3277 /*
3278 * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
3279 * need to be disabled. We make sure to sanitize the values out
3280 * of the punit to satisfy this requirement.
3281 */
3282 for (level = 1; level < num_levels; level++) {
3283 if (wm[level] == 0) {
3284 for (i = level + 1; i < num_levels; i++)
3285 wm[i] = 0;
3286
3287 num_levels = level;
3288 break;
3289 }
3290 }
3291
3292 /*
3293 * WaWmMemoryReadLatency
3294 *
3295 * punit doesn't take into account the read latency so we need
3296 * to add proper adjustment to each valid level we retrieve
3297 * from the punit when level 0 response data is 0us.
3298 */
3299 if (wm[0] == 0) {
3300 for (level = 0; level < num_levels; level++)
3301 wm[level] += read_latency;
3302 }
3303
3304 /*
3305 * WA Level-0 adjustment for 16GB DIMMs: SKL+
3306 * If we could not get dimm info enable this WA to prevent from
3307 * any underrun. If not able to get Dimm info assume 16GB dimm
3308 * to avoid any underrun.
3309 */
3310 if (wm_lv_0_adjust_needed)
3311 wm[0] += 1;
3312 }
3313
mtl_read_wm_latency(struct drm_i915_private * i915,u16 wm[])3314 static void mtl_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
3315 {
3316 int num_levels = i915->display.wm.num_levels;
3317 u32 val;
3318
3319 val = intel_de_read(i915, MTL_LATENCY_LP0_LP1);
3320 wm[0] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
3321 wm[1] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
3322
3323 val = intel_de_read(i915, MTL_LATENCY_LP2_LP3);
3324 wm[2] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
3325 wm[3] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
3326
3327 val = intel_de_read(i915, MTL_LATENCY_LP4_LP5);
3328 wm[4] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
3329 wm[5] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
3330
3331 adjust_wm_latency(i915, wm, num_levels, 6);
3332 }
3333
skl_read_wm_latency(struct drm_i915_private * i915,u16 wm[])3334 static void skl_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
3335 {
3336 int num_levels = i915->display.wm.num_levels;
3337 int read_latency = DISPLAY_VER(i915) >= 12 ? 3 : 2;
3338 int mult = IS_DG2(i915) ? 2 : 1;
3339 u32 val;
3340 int ret;
3341
3342 /* read the first set of memory latencies[0:3] */
3343 val = 0; /* data0 to be programmed to 0 for first set */
3344 ret = snb_pcode_read(&i915->uncore, GEN9_PCODE_READ_MEM_LATENCY, &val, NULL);
3345 if (ret) {
3346 drm_err(&i915->drm, "SKL Mailbox read error = %d\n", ret);
3347 return;
3348 }
3349
3350 wm[0] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_0_4_MASK, val) * mult;
3351 wm[1] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_1_5_MASK, val) * mult;
3352 wm[2] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_2_6_MASK, val) * mult;
3353 wm[3] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_3_7_MASK, val) * mult;
3354
3355 /* read the second set of memory latencies[4:7] */
3356 val = 1; /* data0 to be programmed to 1 for second set */
3357 ret = snb_pcode_read(&i915->uncore, GEN9_PCODE_READ_MEM_LATENCY, &val, NULL);
3358 if (ret) {
3359 drm_err(&i915->drm, "SKL Mailbox read error = %d\n", ret);
3360 return;
3361 }
3362
3363 wm[4] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_0_4_MASK, val) * mult;
3364 wm[5] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_1_5_MASK, val) * mult;
3365 wm[6] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_2_6_MASK, val) * mult;
3366 wm[7] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_3_7_MASK, val) * mult;
3367
3368 adjust_wm_latency(i915, wm, num_levels, read_latency);
3369 }
3370
skl_setup_wm_latency(struct drm_i915_private * i915)3371 static void skl_setup_wm_latency(struct drm_i915_private *i915)
3372 {
3373 struct intel_display *display = &i915->display;
3374
3375 if (HAS_HW_SAGV_WM(display))
3376 display->wm.num_levels = 6;
3377 else
3378 display->wm.num_levels = 8;
3379
3380 if (DISPLAY_VER(display) >= 14)
3381 mtl_read_wm_latency(i915, display->wm.skl_latency);
3382 else
3383 skl_read_wm_latency(i915, display->wm.skl_latency);
3384
3385 intel_print_wm_latency(i915, "Gen9 Plane", display->wm.skl_latency);
3386 }
3387
intel_dbuf_duplicate_state(struct intel_global_obj * obj)3388 static struct intel_global_state *intel_dbuf_duplicate_state(struct intel_global_obj *obj)
3389 {
3390 struct intel_dbuf_state *dbuf_state;
3391
3392 dbuf_state = kmemdup(obj->state, sizeof(*dbuf_state), GFP_KERNEL);
3393 if (!dbuf_state)
3394 return NULL;
3395
3396 return &dbuf_state->base;
3397 }
3398
intel_dbuf_destroy_state(struct intel_global_obj * obj,struct intel_global_state * state)3399 static void intel_dbuf_destroy_state(struct intel_global_obj *obj,
3400 struct intel_global_state *state)
3401 {
3402 kfree(state);
3403 }
3404
3405 static const struct intel_global_state_funcs intel_dbuf_funcs = {
3406 .atomic_duplicate_state = intel_dbuf_duplicate_state,
3407 .atomic_destroy_state = intel_dbuf_destroy_state,
3408 };
3409
3410 struct intel_dbuf_state *
intel_atomic_get_dbuf_state(struct intel_atomic_state * state)3411 intel_atomic_get_dbuf_state(struct intel_atomic_state *state)
3412 {
3413 struct drm_i915_private *i915 = to_i915(state->base.dev);
3414 struct intel_global_state *dbuf_state;
3415
3416 dbuf_state = intel_atomic_get_global_obj_state(state, &i915->display.dbuf.obj);
3417 if (IS_ERR(dbuf_state))
3418 return ERR_CAST(dbuf_state);
3419
3420 return to_intel_dbuf_state(dbuf_state);
3421 }
3422
intel_dbuf_init(struct drm_i915_private * i915)3423 int intel_dbuf_init(struct drm_i915_private *i915)
3424 {
3425 struct intel_display *display = &i915->display;
3426 struct intel_dbuf_state *dbuf_state;
3427
3428 dbuf_state = kzalloc(sizeof(*dbuf_state), GFP_KERNEL);
3429 if (!dbuf_state)
3430 return -ENOMEM;
3431
3432 intel_atomic_global_obj_init(display, &display->dbuf.obj,
3433 &dbuf_state->base, &intel_dbuf_funcs);
3434
3435 return 0;
3436 }
3437
xelpdp_is_only_pipe_per_dbuf_bank(enum pipe pipe,u8 active_pipes)3438 static bool xelpdp_is_only_pipe_per_dbuf_bank(enum pipe pipe, u8 active_pipes)
3439 {
3440 switch (pipe) {
3441 case PIPE_A:
3442 case PIPE_D:
3443 active_pipes &= BIT(PIPE_A) | BIT(PIPE_D);
3444 break;
3445 case PIPE_B:
3446 case PIPE_C:
3447 active_pipes &= BIT(PIPE_B) | BIT(PIPE_C);
3448 break;
3449 default: /* to suppress compiler warning */
3450 MISSING_CASE(pipe);
3451 return false;
3452 }
3453
3454 return is_power_of_2(active_pipes);
3455 }
3456
pipe_mbus_dbox_ctl(const struct intel_crtc * crtc,const struct intel_dbuf_state * dbuf_state)3457 static u32 pipe_mbus_dbox_ctl(const struct intel_crtc *crtc,
3458 const struct intel_dbuf_state *dbuf_state)
3459 {
3460 struct drm_i915_private *i915 = to_i915(crtc->base.dev);
3461 u32 val = 0;
3462
3463 if (DISPLAY_VER(i915) >= 14)
3464 val |= MBUS_DBOX_I_CREDIT(2);
3465
3466 if (DISPLAY_VER(i915) >= 12) {
3467 val |= MBUS_DBOX_B2B_TRANSACTIONS_MAX(16);
3468 val |= MBUS_DBOX_B2B_TRANSACTIONS_DELAY(1);
3469 val |= MBUS_DBOX_REGULATE_B2B_TRANSACTIONS_EN;
3470 }
3471
3472 if (DISPLAY_VER(i915) >= 14)
3473 val |= dbuf_state->joined_mbus ?
3474 MBUS_DBOX_A_CREDIT(12) : MBUS_DBOX_A_CREDIT(8);
3475 else if (IS_ALDERLAKE_P(i915))
3476 /* Wa_22010947358:adl-p */
3477 val |= dbuf_state->joined_mbus ?
3478 MBUS_DBOX_A_CREDIT(6) : MBUS_DBOX_A_CREDIT(4);
3479 else
3480 val |= MBUS_DBOX_A_CREDIT(2);
3481
3482 if (DISPLAY_VER(i915) >= 14) {
3483 val |= MBUS_DBOX_B_CREDIT(0xA);
3484 } else if (IS_ALDERLAKE_P(i915)) {
3485 val |= MBUS_DBOX_BW_CREDIT(2);
3486 val |= MBUS_DBOX_B_CREDIT(8);
3487 } else if (DISPLAY_VER(i915) >= 12) {
3488 val |= MBUS_DBOX_BW_CREDIT(2);
3489 val |= MBUS_DBOX_B_CREDIT(12);
3490 } else {
3491 val |= MBUS_DBOX_BW_CREDIT(1);
3492 val |= MBUS_DBOX_B_CREDIT(8);
3493 }
3494
3495 if (DISPLAY_VERx100(i915) == 1400) {
3496 if (xelpdp_is_only_pipe_per_dbuf_bank(crtc->pipe, dbuf_state->active_pipes))
3497 val |= MBUS_DBOX_BW_8CREDITS_MTL;
3498 else
3499 val |= MBUS_DBOX_BW_4CREDITS_MTL;
3500 }
3501
3502 return val;
3503 }
3504
pipe_mbus_dbox_ctl_update(struct drm_i915_private * i915,const struct intel_dbuf_state * dbuf_state)3505 static void pipe_mbus_dbox_ctl_update(struct drm_i915_private *i915,
3506 const struct intel_dbuf_state *dbuf_state)
3507 {
3508 struct intel_crtc *crtc;
3509
3510 for_each_intel_crtc_in_pipe_mask(&i915->drm, crtc, dbuf_state->active_pipes)
3511 intel_de_write(i915, PIPE_MBUS_DBOX_CTL(crtc->pipe),
3512 pipe_mbus_dbox_ctl(crtc, dbuf_state));
3513 }
3514
intel_mbus_dbox_update(struct intel_atomic_state * state)3515 static void intel_mbus_dbox_update(struct intel_atomic_state *state)
3516 {
3517 struct drm_i915_private *i915 = to_i915(state->base.dev);
3518 const struct intel_dbuf_state *new_dbuf_state, *old_dbuf_state;
3519
3520 if (DISPLAY_VER(i915) < 11)
3521 return;
3522
3523 new_dbuf_state = intel_atomic_get_new_dbuf_state(state);
3524 old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
3525 if (!new_dbuf_state ||
3526 (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus &&
3527 new_dbuf_state->active_pipes == old_dbuf_state->active_pipes))
3528 return;
3529
3530 pipe_mbus_dbox_ctl_update(i915, new_dbuf_state);
3531 }
3532
intel_dbuf_state_set_mdclk_cdclk_ratio(struct intel_atomic_state * state,int ratio)3533 int intel_dbuf_state_set_mdclk_cdclk_ratio(struct intel_atomic_state *state,
3534 int ratio)
3535 {
3536 struct intel_dbuf_state *dbuf_state;
3537
3538 dbuf_state = intel_atomic_get_dbuf_state(state);
3539 if (IS_ERR(dbuf_state))
3540 return PTR_ERR(dbuf_state);
3541
3542 dbuf_state->mdclk_cdclk_ratio = ratio;
3543
3544 return intel_atomic_lock_global_state(&dbuf_state->base);
3545 }
3546
intel_dbuf_mdclk_cdclk_ratio_update(struct drm_i915_private * i915,int ratio,bool joined_mbus)3547 void intel_dbuf_mdclk_cdclk_ratio_update(struct drm_i915_private *i915,
3548 int ratio, bool joined_mbus)
3549 {
3550 struct intel_display *display = &i915->display;
3551 enum dbuf_slice slice;
3552
3553 if (!HAS_MBUS_JOINING(display))
3554 return;
3555
3556 if (DISPLAY_VER(display) >= 20)
3557 intel_de_rmw(display, MBUS_CTL, MBUS_TRANSLATION_THROTTLE_MIN_MASK,
3558 MBUS_TRANSLATION_THROTTLE_MIN(ratio - 1));
3559
3560 if (joined_mbus)
3561 ratio *= 2;
3562
3563 drm_dbg_kms(display->drm, "Updating dbuf ratio to %d (mbus joined: %s)\n",
3564 ratio, str_yes_no(joined_mbus));
3565
3566 for_each_dbuf_slice(display, slice)
3567 intel_de_rmw(display, DBUF_CTL_S(slice),
3568 DBUF_MIN_TRACKER_STATE_SERVICE_MASK,
3569 DBUF_MIN_TRACKER_STATE_SERVICE(ratio - 1));
3570 }
3571
intel_dbuf_mdclk_min_tracker_update(struct intel_atomic_state * state)3572 static void intel_dbuf_mdclk_min_tracker_update(struct intel_atomic_state *state)
3573 {
3574 struct drm_i915_private *i915 = to_i915(state->base.dev);
3575 const struct intel_dbuf_state *old_dbuf_state =
3576 intel_atomic_get_old_dbuf_state(state);
3577 const struct intel_dbuf_state *new_dbuf_state =
3578 intel_atomic_get_new_dbuf_state(state);
3579 int mdclk_cdclk_ratio;
3580
3581 if (intel_cdclk_is_decreasing_later(state)) {
3582 /* cdclk/mdclk will be changed later by intel_set_cdclk_post_plane_update() */
3583 mdclk_cdclk_ratio = old_dbuf_state->mdclk_cdclk_ratio;
3584 } else {
3585 /* cdclk/mdclk already changed by intel_set_cdclk_pre_plane_update() */
3586 mdclk_cdclk_ratio = new_dbuf_state->mdclk_cdclk_ratio;
3587 }
3588
3589 intel_dbuf_mdclk_cdclk_ratio_update(i915, mdclk_cdclk_ratio,
3590 new_dbuf_state->joined_mbus);
3591 }
3592
intel_mbus_joined_pipe(struct intel_atomic_state * state,const struct intel_dbuf_state * dbuf_state)3593 static enum pipe intel_mbus_joined_pipe(struct intel_atomic_state *state,
3594 const struct intel_dbuf_state *dbuf_state)
3595 {
3596 struct intel_display *display = to_intel_display(state);
3597 struct drm_i915_private *i915 = to_i915(state->base.dev);
3598 enum pipe pipe = ffs(dbuf_state->active_pipes) - 1;
3599 const struct intel_crtc_state *new_crtc_state;
3600 struct intel_crtc *crtc;
3601
3602 drm_WARN_ON(&i915->drm, !dbuf_state->joined_mbus);
3603 drm_WARN_ON(&i915->drm, !is_power_of_2(dbuf_state->active_pipes));
3604
3605 crtc = intel_crtc_for_pipe(display, pipe);
3606 new_crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
3607
3608 if (new_crtc_state && !intel_crtc_needs_modeset(new_crtc_state))
3609 return pipe;
3610 else
3611 return INVALID_PIPE;
3612 }
3613
mbus_ctl_join_update(struct drm_i915_private * i915,const struct intel_dbuf_state * dbuf_state,enum pipe pipe)3614 static void mbus_ctl_join_update(struct drm_i915_private *i915,
3615 const struct intel_dbuf_state *dbuf_state,
3616 enum pipe pipe)
3617 {
3618 u32 mbus_ctl;
3619
3620 if (dbuf_state->joined_mbus)
3621 mbus_ctl = MBUS_HASHING_MODE_1x4 | MBUS_JOIN;
3622 else
3623 mbus_ctl = MBUS_HASHING_MODE_2x2;
3624
3625 if (pipe != INVALID_PIPE)
3626 mbus_ctl |= MBUS_JOIN_PIPE_SELECT(pipe);
3627 else
3628 mbus_ctl |= MBUS_JOIN_PIPE_SELECT_NONE;
3629
3630 intel_de_rmw(i915, MBUS_CTL,
3631 MBUS_HASHING_MODE_MASK | MBUS_JOIN |
3632 MBUS_JOIN_PIPE_SELECT_MASK, mbus_ctl);
3633 }
3634
intel_dbuf_mbus_join_update(struct intel_atomic_state * state,enum pipe pipe)3635 static void intel_dbuf_mbus_join_update(struct intel_atomic_state *state,
3636 enum pipe pipe)
3637 {
3638 struct drm_i915_private *i915 = to_i915(state->base.dev);
3639 const struct intel_dbuf_state *old_dbuf_state =
3640 intel_atomic_get_old_dbuf_state(state);
3641 const struct intel_dbuf_state *new_dbuf_state =
3642 intel_atomic_get_new_dbuf_state(state);
3643
3644 drm_dbg_kms(&i915->drm, "Changing mbus joined: %s -> %s (pipe: %c)\n",
3645 str_yes_no(old_dbuf_state->joined_mbus),
3646 str_yes_no(new_dbuf_state->joined_mbus),
3647 pipe != INVALID_PIPE ? pipe_name(pipe) : '*');
3648
3649 mbus_ctl_join_update(i915, new_dbuf_state, pipe);
3650 }
3651
intel_dbuf_mbus_pre_ddb_update(struct intel_atomic_state * state)3652 void intel_dbuf_mbus_pre_ddb_update(struct intel_atomic_state *state)
3653 {
3654 const struct intel_dbuf_state *new_dbuf_state =
3655 intel_atomic_get_new_dbuf_state(state);
3656 const struct intel_dbuf_state *old_dbuf_state =
3657 intel_atomic_get_old_dbuf_state(state);
3658
3659 if (!new_dbuf_state)
3660 return;
3661
3662 if (!old_dbuf_state->joined_mbus && new_dbuf_state->joined_mbus) {
3663 enum pipe pipe = intel_mbus_joined_pipe(state, new_dbuf_state);
3664
3665 WARN_ON(!new_dbuf_state->base.changed);
3666
3667 intel_dbuf_mbus_join_update(state, pipe);
3668 intel_mbus_dbox_update(state);
3669 intel_dbuf_mdclk_min_tracker_update(state);
3670 }
3671 }
3672
intel_dbuf_mbus_post_ddb_update(struct intel_atomic_state * state)3673 void intel_dbuf_mbus_post_ddb_update(struct intel_atomic_state *state)
3674 {
3675 struct intel_display *display = to_intel_display(state);
3676 const struct intel_dbuf_state *new_dbuf_state =
3677 intel_atomic_get_new_dbuf_state(state);
3678 const struct intel_dbuf_state *old_dbuf_state =
3679 intel_atomic_get_old_dbuf_state(state);
3680
3681 if (!new_dbuf_state)
3682 return;
3683
3684 if (old_dbuf_state->joined_mbus && !new_dbuf_state->joined_mbus) {
3685 enum pipe pipe = intel_mbus_joined_pipe(state, old_dbuf_state);
3686
3687 WARN_ON(!new_dbuf_state->base.changed);
3688
3689 intel_dbuf_mdclk_min_tracker_update(state);
3690 intel_mbus_dbox_update(state);
3691 intel_dbuf_mbus_join_update(state, pipe);
3692
3693 if (pipe != INVALID_PIPE) {
3694 struct intel_crtc *crtc = intel_crtc_for_pipe(display, pipe);
3695
3696 intel_crtc_wait_for_next_vblank(crtc);
3697 }
3698 } else if (old_dbuf_state->joined_mbus == new_dbuf_state->joined_mbus &&
3699 old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) {
3700 WARN_ON(!new_dbuf_state->base.changed);
3701
3702 intel_dbuf_mdclk_min_tracker_update(state);
3703 intel_mbus_dbox_update(state);
3704 }
3705
3706 }
3707
intel_dbuf_pre_plane_update(struct intel_atomic_state * state)3708 void intel_dbuf_pre_plane_update(struct intel_atomic_state *state)
3709 {
3710 struct intel_display *display = to_intel_display(state);
3711 const struct intel_dbuf_state *new_dbuf_state =
3712 intel_atomic_get_new_dbuf_state(state);
3713 const struct intel_dbuf_state *old_dbuf_state =
3714 intel_atomic_get_old_dbuf_state(state);
3715 u8 old_slices, new_slices;
3716
3717 if (!new_dbuf_state)
3718 return;
3719
3720 old_slices = old_dbuf_state->enabled_slices;
3721 new_slices = old_dbuf_state->enabled_slices | new_dbuf_state->enabled_slices;
3722
3723 if (old_slices == new_slices)
3724 return;
3725
3726 WARN_ON(!new_dbuf_state->base.changed);
3727
3728 gen9_dbuf_slices_update(display, new_slices);
3729 }
3730
intel_dbuf_post_plane_update(struct intel_atomic_state * state)3731 void intel_dbuf_post_plane_update(struct intel_atomic_state *state)
3732 {
3733 struct intel_display *display = to_intel_display(state);
3734 const struct intel_dbuf_state *new_dbuf_state =
3735 intel_atomic_get_new_dbuf_state(state);
3736 const struct intel_dbuf_state *old_dbuf_state =
3737 intel_atomic_get_old_dbuf_state(state);
3738 u8 old_slices, new_slices;
3739
3740 if (!new_dbuf_state)
3741 return;
3742
3743 old_slices = old_dbuf_state->enabled_slices | new_dbuf_state->enabled_slices;
3744 new_slices = new_dbuf_state->enabled_slices;
3745
3746 if (old_slices == new_slices)
3747 return;
3748
3749 WARN_ON(!new_dbuf_state->base.changed);
3750
3751 gen9_dbuf_slices_update(display, new_slices);
3752 }
3753
skl_mbus_sanitize(struct drm_i915_private * i915)3754 static void skl_mbus_sanitize(struct drm_i915_private *i915)
3755 {
3756 struct intel_display *display = &i915->display;
3757 struct intel_dbuf_state *dbuf_state =
3758 to_intel_dbuf_state(display->dbuf.obj.state);
3759
3760 if (!HAS_MBUS_JOINING(display))
3761 return;
3762
3763 if (!dbuf_state->joined_mbus ||
3764 adlp_check_mbus_joined(dbuf_state->active_pipes))
3765 return;
3766
3767 drm_dbg_kms(display->drm, "Disabling redundant MBUS joining (active pipes 0x%x)\n",
3768 dbuf_state->active_pipes);
3769
3770 dbuf_state->joined_mbus = false;
3771 intel_dbuf_mdclk_cdclk_ratio_update(i915,
3772 dbuf_state->mdclk_cdclk_ratio,
3773 dbuf_state->joined_mbus);
3774 pipe_mbus_dbox_ctl_update(i915, dbuf_state);
3775 mbus_ctl_join_update(i915, dbuf_state, INVALID_PIPE);
3776 }
3777
skl_dbuf_is_misconfigured(struct drm_i915_private * i915)3778 static bool skl_dbuf_is_misconfigured(struct drm_i915_private *i915)
3779 {
3780 const struct intel_dbuf_state *dbuf_state =
3781 to_intel_dbuf_state(i915->display.dbuf.obj.state);
3782 struct skl_ddb_entry entries[I915_MAX_PIPES] = {};
3783 struct intel_crtc *crtc;
3784
3785 for_each_intel_crtc(&i915->drm, crtc) {
3786 const struct intel_crtc_state *crtc_state =
3787 to_intel_crtc_state(crtc->base.state);
3788
3789 entries[crtc->pipe] = crtc_state->wm.skl.ddb;
3790 }
3791
3792 for_each_intel_crtc(&i915->drm, crtc) {
3793 const struct intel_crtc_state *crtc_state =
3794 to_intel_crtc_state(crtc->base.state);
3795 u8 slices;
3796
3797 slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes,
3798 dbuf_state->joined_mbus);
3799 if (dbuf_state->slices[crtc->pipe] & ~slices)
3800 return true;
3801
3802 if (skl_ddb_allocation_overlaps(&crtc_state->wm.skl.ddb, entries,
3803 I915_MAX_PIPES, crtc->pipe))
3804 return true;
3805 }
3806
3807 return false;
3808 }
3809
skl_dbuf_sanitize(struct drm_i915_private * i915)3810 static void skl_dbuf_sanitize(struct drm_i915_private *i915)
3811 {
3812 struct intel_crtc *crtc;
3813
3814 /*
3815 * On TGL/RKL (at least) the BIOS likes to assign the planes
3816 * to the wrong DBUF slices. This will cause an infinite loop
3817 * in skl_commit_modeset_enables() as it can't find a way to
3818 * transition between the old bogus DBUF layout to the new
3819 * proper DBUF layout without DBUF allocation overlaps between
3820 * the planes (which cannot be allowed or else the hardware
3821 * may hang). If we detect a bogus DBUF layout just turn off
3822 * all the planes so that skl_commit_modeset_enables() can
3823 * simply ignore them.
3824 */
3825 if (!skl_dbuf_is_misconfigured(i915))
3826 return;
3827
3828 drm_dbg_kms(&i915->drm, "BIOS has misprogrammed the DBUF, disabling all planes\n");
3829
3830 for_each_intel_crtc(&i915->drm, crtc) {
3831 struct intel_plane *plane = to_intel_plane(crtc->base.primary);
3832 const struct intel_plane_state *plane_state =
3833 to_intel_plane_state(plane->base.state);
3834 struct intel_crtc_state *crtc_state =
3835 to_intel_crtc_state(crtc->base.state);
3836
3837 if (plane_state->uapi.visible)
3838 intel_plane_disable_noatomic(crtc, plane);
3839
3840 drm_WARN_ON(&i915->drm, crtc_state->active_planes != 0);
3841
3842 memset(&crtc_state->wm.skl.ddb, 0, sizeof(crtc_state->wm.skl.ddb));
3843 }
3844 }
3845
skl_wm_sanitize(struct drm_i915_private * i915)3846 static void skl_wm_sanitize(struct drm_i915_private *i915)
3847 {
3848 skl_mbus_sanitize(i915);
3849 skl_dbuf_sanitize(i915);
3850 }
3851
skl_wm_crtc_disable_noatomic(struct intel_crtc * crtc)3852 void skl_wm_crtc_disable_noatomic(struct intel_crtc *crtc)
3853 {
3854 struct intel_display *display = to_intel_display(crtc);
3855 struct intel_crtc_state *crtc_state =
3856 to_intel_crtc_state(crtc->base.state);
3857 struct intel_dbuf_state *dbuf_state =
3858 to_intel_dbuf_state(display->dbuf.obj.state);
3859 enum pipe pipe = crtc->pipe;
3860
3861 if (DISPLAY_VER(display) < 9)
3862 return;
3863
3864 dbuf_state->active_pipes &= ~BIT(pipe);
3865
3866 dbuf_state->weight[pipe] = 0;
3867 dbuf_state->slices[pipe] = 0;
3868
3869 memset(&dbuf_state->ddb[pipe], 0, sizeof(dbuf_state->ddb[pipe]));
3870
3871 memset(&crtc_state->wm.skl.ddb, 0, sizeof(crtc_state->wm.skl.ddb));
3872 }
3873
skl_wm_plane_disable_noatomic(struct intel_crtc * crtc,struct intel_plane * plane)3874 void skl_wm_plane_disable_noatomic(struct intel_crtc *crtc,
3875 struct intel_plane *plane)
3876 {
3877 struct intel_display *display = to_intel_display(crtc);
3878 struct intel_crtc_state *crtc_state =
3879 to_intel_crtc_state(crtc->base.state);
3880
3881 if (DISPLAY_VER(display) < 9)
3882 return;
3883
3884 skl_ddb_entry_init(&crtc_state->wm.skl.plane_ddb[plane->id], 0, 0);
3885 skl_ddb_entry_init(&crtc_state->wm.skl.plane_ddb[plane->id], 0, 0);
3886
3887 crtc_state->wm.skl.plane_min_ddb[plane->id] = 0;
3888 crtc_state->wm.skl.plane_interim_ddb[plane->id] = 0;
3889
3890 memset(&crtc_state->wm.skl.raw.planes[plane->id], 0,
3891 sizeof(crtc_state->wm.skl.raw.planes[plane->id]));
3892 memset(&crtc_state->wm.skl.optimal.planes[plane->id], 0,
3893 sizeof(crtc_state->wm.skl.optimal.planes[plane->id]));
3894 }
3895
intel_wm_state_verify(struct intel_atomic_state * state,struct intel_crtc * crtc)3896 void intel_wm_state_verify(struct intel_atomic_state *state,
3897 struct intel_crtc *crtc)
3898 {
3899 struct intel_display *display = to_intel_display(state);
3900 struct drm_i915_private *i915 = to_i915(state->base.dev);
3901 const struct intel_crtc_state *new_crtc_state =
3902 intel_atomic_get_new_crtc_state(state, crtc);
3903 struct skl_hw_state {
3904 struct skl_ddb_entry ddb[I915_MAX_PLANES];
3905 struct skl_ddb_entry ddb_y[I915_MAX_PLANES];
3906 u16 min_ddb[I915_MAX_PLANES];
3907 u16 interim_ddb[I915_MAX_PLANES];
3908 struct skl_pipe_wm wm;
3909 } *hw;
3910 const struct skl_pipe_wm *sw_wm = &new_crtc_state->wm.skl.optimal;
3911 struct intel_plane *plane;
3912 u8 hw_enabled_slices;
3913 int level;
3914
3915 if (DISPLAY_VER(i915) < 9 || !new_crtc_state->hw.active)
3916 return;
3917
3918 hw = kzalloc(sizeof(*hw), GFP_KERNEL);
3919 if (!hw)
3920 return;
3921
3922 skl_pipe_wm_get_hw_state(crtc, &hw->wm);
3923
3924 skl_pipe_ddb_get_hw_state(crtc, hw->ddb, hw->ddb_y, hw->min_ddb, hw->interim_ddb);
3925
3926 hw_enabled_slices = intel_enabled_dbuf_slices_mask(display);
3927
3928 if (DISPLAY_VER(i915) >= 11 &&
3929 hw_enabled_slices != i915->display.dbuf.enabled_slices)
3930 drm_err(&i915->drm,
3931 "mismatch in DBUF Slices (expected 0x%x, got 0x%x)\n",
3932 i915->display.dbuf.enabled_slices,
3933 hw_enabled_slices);
3934
3935 for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
3936 const struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry;
3937 const struct skl_wm_level *hw_wm_level, *sw_wm_level;
3938
3939 /* Watermarks */
3940 for (level = 0; level < i915->display.wm.num_levels; level++) {
3941 hw_wm_level = &hw->wm.planes[plane->id].wm[level];
3942 sw_wm_level = skl_plane_wm_level(sw_wm, plane->id, level);
3943
3944 if (skl_wm_level_equals(hw_wm_level, sw_wm_level))
3945 continue;
3946
3947 drm_err(&i915->drm,
3948 "[PLANE:%d:%s] mismatch in WM%d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3949 plane->base.base.id, plane->base.name, level,
3950 sw_wm_level->enable,
3951 sw_wm_level->blocks,
3952 sw_wm_level->lines,
3953 hw_wm_level->enable,
3954 hw_wm_level->blocks,
3955 hw_wm_level->lines);
3956 }
3957
3958 hw_wm_level = &hw->wm.planes[plane->id].trans_wm;
3959 sw_wm_level = skl_plane_trans_wm(sw_wm, plane->id);
3960
3961 if (!skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
3962 drm_err(&i915->drm,
3963 "[PLANE:%d:%s] mismatch in trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3964 plane->base.base.id, plane->base.name,
3965 sw_wm_level->enable,
3966 sw_wm_level->blocks,
3967 sw_wm_level->lines,
3968 hw_wm_level->enable,
3969 hw_wm_level->blocks,
3970 hw_wm_level->lines);
3971 }
3972
3973 hw_wm_level = &hw->wm.planes[plane->id].sagv.wm0;
3974 sw_wm_level = &sw_wm->planes[plane->id].sagv.wm0;
3975
3976 if (HAS_HW_SAGV_WM(display) &&
3977 !skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
3978 drm_err(&i915->drm,
3979 "[PLANE:%d:%s] mismatch in SAGV WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3980 plane->base.base.id, plane->base.name,
3981 sw_wm_level->enable,
3982 sw_wm_level->blocks,
3983 sw_wm_level->lines,
3984 hw_wm_level->enable,
3985 hw_wm_level->blocks,
3986 hw_wm_level->lines);
3987 }
3988
3989 hw_wm_level = &hw->wm.planes[plane->id].sagv.trans_wm;
3990 sw_wm_level = &sw_wm->planes[plane->id].sagv.trans_wm;
3991
3992 if (HAS_HW_SAGV_WM(display) &&
3993 !skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
3994 drm_err(&i915->drm,
3995 "[PLANE:%d:%s] mismatch in SAGV trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3996 plane->base.base.id, plane->base.name,
3997 sw_wm_level->enable,
3998 sw_wm_level->blocks,
3999 sw_wm_level->lines,
4000 hw_wm_level->enable,
4001 hw_wm_level->blocks,
4002 hw_wm_level->lines);
4003 }
4004
4005 /* DDB */
4006 hw_ddb_entry = &hw->ddb[PLANE_CURSOR];
4007 sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb[PLANE_CURSOR];
4008
4009 if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
4010 drm_err(&i915->drm,
4011 "[PLANE:%d:%s] mismatch in DDB (expected (%u,%u), found (%u,%u))\n",
4012 plane->base.base.id, plane->base.name,
4013 sw_ddb_entry->start, sw_ddb_entry->end,
4014 hw_ddb_entry->start, hw_ddb_entry->end);
4015 }
4016 }
4017
4018 kfree(hw);
4019 }
4020
4021 static const struct intel_wm_funcs skl_wm_funcs = {
4022 .compute_global_watermarks = skl_compute_wm,
4023 .get_hw_state = skl_wm_get_hw_state,
4024 .sanitize = skl_wm_sanitize,
4025 };
4026
skl_wm_init(struct drm_i915_private * i915)4027 void skl_wm_init(struct drm_i915_private *i915)
4028 {
4029 intel_sagv_init(i915);
4030
4031 skl_setup_wm_latency(i915);
4032
4033 i915->display.funcs.wm = &skl_wm_funcs;
4034 }
4035
skl_watermark_ipc_status_show(struct seq_file * m,void * data)4036 static int skl_watermark_ipc_status_show(struct seq_file *m, void *data)
4037 {
4038 struct drm_i915_private *i915 = m->private;
4039
4040 seq_printf(m, "Isochronous Priority Control: %s\n",
4041 str_yes_no(skl_watermark_ipc_enabled(i915)));
4042 return 0;
4043 }
4044
skl_watermark_ipc_status_open(struct inode * inode,struct file * file)4045 static int skl_watermark_ipc_status_open(struct inode *inode, struct file *file)
4046 {
4047 struct drm_i915_private *i915 = inode->i_private;
4048
4049 return single_open(file, skl_watermark_ipc_status_show, i915);
4050 }
4051
skl_watermark_ipc_status_write(struct file * file,const char __user * ubuf,size_t len,loff_t * offp)4052 static ssize_t skl_watermark_ipc_status_write(struct file *file,
4053 const char __user *ubuf,
4054 size_t len, loff_t *offp)
4055 {
4056 struct seq_file *m = file->private_data;
4057 struct drm_i915_private *i915 = m->private;
4058 intel_wakeref_t wakeref;
4059 bool enable;
4060 int ret;
4061
4062 ret = kstrtobool_from_user(ubuf, len, &enable);
4063 if (ret < 0)
4064 return ret;
4065
4066 with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
4067 if (!skl_watermark_ipc_enabled(i915) && enable)
4068 drm_info(&i915->drm,
4069 "Enabling IPC: WM will be proper only after next commit\n");
4070 i915->display.wm.ipc_enabled = enable;
4071 skl_watermark_ipc_update(i915);
4072 }
4073
4074 return len;
4075 }
4076
4077 static const struct file_operations skl_watermark_ipc_status_fops = {
4078 .owner = THIS_MODULE,
4079 .open = skl_watermark_ipc_status_open,
4080 .read = seq_read,
4081 .llseek = seq_lseek,
4082 .release = single_release,
4083 .write = skl_watermark_ipc_status_write
4084 };
4085
intel_sagv_status_show(struct seq_file * m,void * unused)4086 static int intel_sagv_status_show(struct seq_file *m, void *unused)
4087 {
4088 struct drm_i915_private *i915 = m->private;
4089 static const char * const sagv_status[] = {
4090 [I915_SAGV_UNKNOWN] = "unknown",
4091 [I915_SAGV_DISABLED] = "disabled",
4092 [I915_SAGV_ENABLED] = "enabled",
4093 [I915_SAGV_NOT_CONTROLLED] = "not controlled",
4094 };
4095
4096 seq_printf(m, "SAGV available: %s\n", str_yes_no(intel_has_sagv(i915)));
4097 seq_printf(m, "SAGV modparam: %s\n",
4098 str_enabled_disabled(i915->display.params.enable_sagv));
4099 seq_printf(m, "SAGV status: %s\n", sagv_status[i915->display.sagv.status]);
4100 seq_printf(m, "SAGV block time: %d usec\n", i915->display.sagv.block_time_us);
4101
4102 return 0;
4103 }
4104
4105 DEFINE_SHOW_ATTRIBUTE(intel_sagv_status);
4106
skl_watermark_debugfs_register(struct drm_i915_private * i915)4107 void skl_watermark_debugfs_register(struct drm_i915_private *i915)
4108 {
4109 struct intel_display *display = &i915->display;
4110 struct drm_minor *minor = display->drm->primary;
4111
4112 if (HAS_IPC(display))
4113 debugfs_create_file("i915_ipc_status", 0644, minor->debugfs_root, i915,
4114 &skl_watermark_ipc_status_fops);
4115
4116 if (HAS_SAGV(display))
4117 debugfs_create_file("i915_sagv_status", 0444, minor->debugfs_root, i915,
4118 &intel_sagv_status_fops);
4119 }
4120
skl_watermark_max_latency(struct drm_i915_private * i915,int initial_wm_level)4121 unsigned int skl_watermark_max_latency(struct drm_i915_private *i915, int initial_wm_level)
4122 {
4123 int level;
4124
4125 for (level = i915->display.wm.num_levels - 1; level >= initial_wm_level; level--) {
4126 unsigned int latency = skl_wm_latency(i915, level, NULL);
4127
4128 if (latency)
4129 return latency;
4130 }
4131
4132 return 0;
4133 }
4134