1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2022 Intel Corporation
4  */
5 
6 #include <linux/debugfs.h>
7 
8 #include <drm/drm_blend.h>
9 
10 #include "i915_drv.h"
11 #include "i915_reg.h"
12 #include "i9xx_wm.h"
13 #include "intel_atomic.h"
14 #include "intel_atomic_plane.h"
15 #include "intel_bw.h"
16 #include "intel_cdclk.h"
17 #include "intel_crtc.h"
18 #include "intel_cursor_regs.h"
19 #include "intel_de.h"
20 #include "intel_display.h"
21 #include "intel_display_power.h"
22 #include "intel_display_types.h"
23 #include "intel_fb.h"
24 #include "intel_fixed.h"
25 #include "intel_pcode.h"
26 #include "intel_wm.h"
27 #include "skl_universal_plane_regs.h"
28 #include "skl_watermark.h"
29 #include "skl_watermark_regs.h"
30 
31 /*It is expected that DSB can do posted writes to every register in
32  * the pipe and planes within 100us. For flip queue use case, the
33  * recommended DSB execution time is 100us + one SAGV block time.
34  */
35 #define DSB_EXE_TIME 100
36 
37 static void skl_sagv_disable(struct drm_i915_private *i915);
38 
39 /* Stores plane specific WM parameters */
40 struct skl_wm_params {
41 	bool x_tiled, y_tiled;
42 	bool rc_surface;
43 	bool is_planar;
44 	u32 width;
45 	u8 cpp;
46 	u32 plane_pixel_rate;
47 	u32 y_min_scanlines;
48 	u32 plane_bytes_per_line;
49 	uint_fixed_16_16_t plane_blocks_per_line;
50 	uint_fixed_16_16_t y_tile_minimum;
51 	u32 linetime_us;
52 	u32 dbuf_block_size;
53 };
54 
intel_enabled_dbuf_slices_mask(struct intel_display * display)55 u8 intel_enabled_dbuf_slices_mask(struct intel_display *display)
56 {
57 	u8 enabled_slices = 0;
58 	enum dbuf_slice slice;
59 
60 	for_each_dbuf_slice(display, slice) {
61 		if (intel_de_read(display, DBUF_CTL_S(slice)) & DBUF_POWER_STATE)
62 			enabled_slices |= BIT(slice);
63 	}
64 
65 	return enabled_slices;
66 }
67 
68 /*
69  * FIXME: We still don't have the proper code detect if we need to apply the WA,
70  * so assume we'll always need it in order to avoid underruns.
71  */
skl_needs_memory_bw_wa(struct drm_i915_private * i915)72 static bool skl_needs_memory_bw_wa(struct drm_i915_private *i915)
73 {
74 	return DISPLAY_VER(i915) == 9;
75 }
76 
77 bool
intel_has_sagv(struct drm_i915_private * i915)78 intel_has_sagv(struct drm_i915_private *i915)
79 {
80 	struct intel_display *display = &i915->display;
81 
82 	return HAS_SAGV(display) && display->sagv.status != I915_SAGV_NOT_CONTROLLED;
83 }
84 
85 static u32
intel_sagv_block_time(struct drm_i915_private * i915)86 intel_sagv_block_time(struct drm_i915_private *i915)
87 {
88 	struct intel_display *display = &i915->display;
89 
90 	if (DISPLAY_VER(display) >= 14) {
91 		u32 val;
92 
93 		val = intel_de_read(display, MTL_LATENCY_SAGV);
94 
95 		return REG_FIELD_GET(MTL_LATENCY_QCLK_SAGV, val);
96 	} else if (DISPLAY_VER(display) >= 12) {
97 		u32 val = 0;
98 		int ret;
99 
100 		ret = snb_pcode_read(&i915->uncore,
101 				     GEN12_PCODE_READ_SAGV_BLOCK_TIME_US,
102 				     &val, NULL);
103 		if (ret) {
104 			drm_dbg_kms(display->drm, "Couldn't read SAGV block time!\n");
105 			return 0;
106 		}
107 
108 		return val;
109 	} else if (DISPLAY_VER(display) == 11) {
110 		return 10;
111 	} else if (HAS_SAGV(display)) {
112 		return 30;
113 	} else {
114 		return 0;
115 	}
116 }
117 
intel_sagv_init(struct drm_i915_private * i915)118 static void intel_sagv_init(struct drm_i915_private *i915)
119 {
120 	struct intel_display *display = &i915->display;
121 
122 	if (!HAS_SAGV(display))
123 		display->sagv.status = I915_SAGV_NOT_CONTROLLED;
124 
125 	/*
126 	 * Probe to see if we have working SAGV control.
127 	 * For icl+ this was already determined by intel_bw_init_hw().
128 	 */
129 	if (DISPLAY_VER(display) < 11)
130 		skl_sagv_disable(i915);
131 
132 	drm_WARN_ON(display->drm, display->sagv.status == I915_SAGV_UNKNOWN);
133 
134 	display->sagv.block_time_us = intel_sagv_block_time(i915);
135 
136 	drm_dbg_kms(display->drm, "SAGV supported: %s, original SAGV block time: %u us\n",
137 		    str_yes_no(intel_has_sagv(i915)), display->sagv.block_time_us);
138 
139 	/* avoid overflow when adding with wm0 latency/etc. */
140 	if (drm_WARN(display->drm, display->sagv.block_time_us > U16_MAX,
141 		     "Excessive SAGV block time %u, ignoring\n",
142 		     display->sagv.block_time_us))
143 		display->sagv.block_time_us = 0;
144 
145 	if (!intel_has_sagv(i915))
146 		display->sagv.block_time_us = 0;
147 }
148 
149 /*
150  * SAGV dynamically adjusts the system agent voltage and clock frequencies
151  * depending on power and performance requirements. The display engine access
152  * to system memory is blocked during the adjustment time. Because of the
153  * blocking time, having this enabled can cause full system hangs and/or pipe
154  * underruns if we don't meet all of the following requirements:
155  *
156  *  - <= 1 pipe enabled
157  *  - All planes can enable watermarks for latencies >= SAGV engine block time
158  *  - We're not using an interlaced display configuration
159  */
skl_sagv_enable(struct drm_i915_private * i915)160 static void skl_sagv_enable(struct drm_i915_private *i915)
161 {
162 	int ret;
163 
164 	if (!intel_has_sagv(i915))
165 		return;
166 
167 	if (i915->display.sagv.status == I915_SAGV_ENABLED)
168 		return;
169 
170 	drm_dbg_kms(&i915->drm, "Enabling SAGV\n");
171 	ret = snb_pcode_write(&i915->uncore, GEN9_PCODE_SAGV_CONTROL,
172 			      GEN9_SAGV_ENABLE);
173 
174 	/* We don't need to wait for SAGV when enabling */
175 
176 	/*
177 	 * Some skl systems, pre-release machines in particular,
178 	 * don't actually have SAGV.
179 	 */
180 	if (IS_SKYLAKE(i915) && ret == -ENXIO) {
181 		drm_dbg(&i915->drm, "No SAGV found on system, ignoring\n");
182 		i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
183 		return;
184 	} else if (ret < 0) {
185 		drm_err(&i915->drm, "Failed to enable SAGV\n");
186 		return;
187 	}
188 
189 	i915->display.sagv.status = I915_SAGV_ENABLED;
190 }
191 
skl_sagv_disable(struct drm_i915_private * i915)192 static void skl_sagv_disable(struct drm_i915_private *i915)
193 {
194 	int ret;
195 
196 	if (!intel_has_sagv(i915))
197 		return;
198 
199 	if (i915->display.sagv.status == I915_SAGV_DISABLED)
200 		return;
201 
202 	drm_dbg_kms(&i915->drm, "Disabling SAGV\n");
203 	/* bspec says to keep retrying for at least 1 ms */
204 	ret = skl_pcode_request(&i915->uncore, GEN9_PCODE_SAGV_CONTROL,
205 				GEN9_SAGV_DISABLE,
206 				GEN9_SAGV_IS_DISABLED, GEN9_SAGV_IS_DISABLED,
207 				1);
208 	/*
209 	 * Some skl systems, pre-release machines in particular,
210 	 * don't actually have SAGV.
211 	 */
212 	if (IS_SKYLAKE(i915) && ret == -ENXIO) {
213 		drm_dbg(&i915->drm, "No SAGV found on system, ignoring\n");
214 		i915->display.sagv.status = I915_SAGV_NOT_CONTROLLED;
215 		return;
216 	} else if (ret < 0) {
217 		drm_err(&i915->drm, "Failed to disable SAGV (%d)\n", ret);
218 		return;
219 	}
220 
221 	i915->display.sagv.status = I915_SAGV_DISABLED;
222 }
223 
skl_sagv_pre_plane_update(struct intel_atomic_state * state)224 static void skl_sagv_pre_plane_update(struct intel_atomic_state *state)
225 {
226 	struct drm_i915_private *i915 = to_i915(state->base.dev);
227 	const struct intel_bw_state *new_bw_state =
228 		intel_atomic_get_new_bw_state(state);
229 
230 	if (!new_bw_state)
231 		return;
232 
233 	if (!intel_can_enable_sagv(i915, new_bw_state))
234 		skl_sagv_disable(i915);
235 }
236 
skl_sagv_post_plane_update(struct intel_atomic_state * state)237 static void skl_sagv_post_plane_update(struct intel_atomic_state *state)
238 {
239 	struct drm_i915_private *i915 = to_i915(state->base.dev);
240 	const struct intel_bw_state *new_bw_state =
241 		intel_atomic_get_new_bw_state(state);
242 
243 	if (!new_bw_state)
244 		return;
245 
246 	if (intel_can_enable_sagv(i915, new_bw_state))
247 		skl_sagv_enable(i915);
248 }
249 
icl_sagv_pre_plane_update(struct intel_atomic_state * state)250 static void icl_sagv_pre_plane_update(struct intel_atomic_state *state)
251 {
252 	struct drm_i915_private *i915 = to_i915(state->base.dev);
253 	const struct intel_bw_state *old_bw_state =
254 		intel_atomic_get_old_bw_state(state);
255 	const struct intel_bw_state *new_bw_state =
256 		intel_atomic_get_new_bw_state(state);
257 	u16 old_mask, new_mask;
258 
259 	if (!new_bw_state)
260 		return;
261 
262 	old_mask = old_bw_state->qgv_points_mask;
263 	new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
264 
265 	if (old_mask == new_mask)
266 		return;
267 
268 	WARN_ON(!new_bw_state->base.changed);
269 
270 	drm_dbg_kms(&i915->drm, "Restricting QGV points: 0x%x -> 0x%x\n",
271 		    old_mask, new_mask);
272 
273 	/*
274 	 * Restrict required qgv points before updating the configuration.
275 	 * According to BSpec we can't mask and unmask qgv points at the same
276 	 * time. Also masking should be done before updating the configuration
277 	 * and unmasking afterwards.
278 	 */
279 	icl_pcode_restrict_qgv_points(i915, new_mask);
280 }
281 
icl_sagv_post_plane_update(struct intel_atomic_state * state)282 static void icl_sagv_post_plane_update(struct intel_atomic_state *state)
283 {
284 	struct drm_i915_private *i915 = to_i915(state->base.dev);
285 	const struct intel_bw_state *old_bw_state =
286 		intel_atomic_get_old_bw_state(state);
287 	const struct intel_bw_state *new_bw_state =
288 		intel_atomic_get_new_bw_state(state);
289 	u16 old_mask, new_mask;
290 
291 	if (!new_bw_state)
292 		return;
293 
294 	old_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
295 	new_mask = new_bw_state->qgv_points_mask;
296 
297 	if (old_mask == new_mask)
298 		return;
299 
300 	WARN_ON(!new_bw_state->base.changed);
301 
302 	drm_dbg_kms(&i915->drm, "Relaxing QGV points: 0x%x -> 0x%x\n",
303 		    old_mask, new_mask);
304 
305 	/*
306 	 * Allow required qgv points after updating the configuration.
307 	 * According to BSpec we can't mask and unmask qgv points at the same
308 	 * time. Also masking should be done before updating the configuration
309 	 * and unmasking afterwards.
310 	 */
311 	icl_pcode_restrict_qgv_points(i915, new_mask);
312 }
313 
intel_sagv_pre_plane_update(struct intel_atomic_state * state)314 void intel_sagv_pre_plane_update(struct intel_atomic_state *state)
315 {
316 	struct drm_i915_private *i915 = to_i915(state->base.dev);
317 
318 	/*
319 	 * Just return if we can't control SAGV or don't have it.
320 	 * This is different from situation when we have SAGV but just can't
321 	 * afford it due to DBuf limitation - in case if SAGV is completely
322 	 * disabled in a BIOS, we are not even allowed to send a PCode request,
323 	 * as it will throw an error. So have to check it here.
324 	 */
325 	if (!intel_has_sagv(i915))
326 		return;
327 
328 	if (DISPLAY_VER(i915) >= 11)
329 		icl_sagv_pre_plane_update(state);
330 	else
331 		skl_sagv_pre_plane_update(state);
332 }
333 
intel_sagv_post_plane_update(struct intel_atomic_state * state)334 void intel_sagv_post_plane_update(struct intel_atomic_state *state)
335 {
336 	struct drm_i915_private *i915 = to_i915(state->base.dev);
337 
338 	/*
339 	 * Just return if we can't control SAGV or don't have it.
340 	 * This is different from situation when we have SAGV but just can't
341 	 * afford it due to DBuf limitation - in case if SAGV is completely
342 	 * disabled in a BIOS, we are not even allowed to send a PCode request,
343 	 * as it will throw an error. So have to check it here.
344 	 */
345 	if (!intel_has_sagv(i915))
346 		return;
347 
348 	if (DISPLAY_VER(i915) >= 11)
349 		icl_sagv_post_plane_update(state);
350 	else
351 		skl_sagv_post_plane_update(state);
352 }
353 
skl_crtc_can_enable_sagv(const struct intel_crtc_state * crtc_state)354 static bool skl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
355 {
356 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
357 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
358 	enum plane_id plane_id;
359 	int max_level = INT_MAX;
360 
361 	if (!intel_has_sagv(i915))
362 		return false;
363 
364 	if (!crtc_state->hw.active)
365 		return true;
366 
367 	if (crtc_state->hw.pipe_mode.flags & DRM_MODE_FLAG_INTERLACE)
368 		return false;
369 
370 	for_each_plane_id_on_crtc(crtc, plane_id) {
371 		const struct skl_plane_wm *wm =
372 			&crtc_state->wm.skl.optimal.planes[plane_id];
373 		int level;
374 
375 		/* Skip this plane if it's not enabled */
376 		if (!wm->wm[0].enable)
377 			continue;
378 
379 		/* Find the highest enabled wm level for this plane */
380 		for (level = i915->display.wm.num_levels - 1;
381 		     !wm->wm[level].enable; --level)
382 		     { }
383 
384 		/* Highest common enabled wm level for all planes */
385 		max_level = min(level, max_level);
386 	}
387 
388 	/* No enabled planes? */
389 	if (max_level == INT_MAX)
390 		return true;
391 
392 	for_each_plane_id_on_crtc(crtc, plane_id) {
393 		const struct skl_plane_wm *wm =
394 			&crtc_state->wm.skl.optimal.planes[plane_id];
395 
396 		/*
397 		 * All enabled planes must have enabled a common wm level that
398 		 * can tolerate memory latencies higher than sagv_block_time_us
399 		 */
400 		if (wm->wm[0].enable && !wm->wm[max_level].can_sagv)
401 			return false;
402 	}
403 
404 	return true;
405 }
406 
tgl_crtc_can_enable_sagv(const struct intel_crtc_state * crtc_state)407 static bool tgl_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
408 {
409 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
410 	enum plane_id plane_id;
411 
412 	if (!crtc_state->hw.active)
413 		return true;
414 
415 	for_each_plane_id_on_crtc(crtc, plane_id) {
416 		const struct skl_plane_wm *wm =
417 			&crtc_state->wm.skl.optimal.planes[plane_id];
418 
419 		if (wm->wm[0].enable && !wm->sagv.wm0.enable)
420 			return false;
421 	}
422 
423 	return true;
424 }
425 
intel_crtc_can_enable_sagv(const struct intel_crtc_state * crtc_state)426 static bool intel_crtc_can_enable_sagv(const struct intel_crtc_state *crtc_state)
427 {
428 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
429 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
430 
431 	if (!i915->display.params.enable_sagv)
432 		return false;
433 
434 	if (DISPLAY_VER(i915) >= 12)
435 		return tgl_crtc_can_enable_sagv(crtc_state);
436 	else
437 		return skl_crtc_can_enable_sagv(crtc_state);
438 }
439 
intel_can_enable_sagv(struct drm_i915_private * i915,const struct intel_bw_state * bw_state)440 bool intel_can_enable_sagv(struct drm_i915_private *i915,
441 			   const struct intel_bw_state *bw_state)
442 {
443 	if (DISPLAY_VER(i915) < 11 &&
444 	    bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes))
445 		return false;
446 
447 	return bw_state->pipe_sagv_reject == 0;
448 }
449 
intel_compute_sagv_mask(struct intel_atomic_state * state)450 static int intel_compute_sagv_mask(struct intel_atomic_state *state)
451 {
452 	struct intel_display *display = to_intel_display(state);
453 	struct drm_i915_private *i915 = to_i915(state->base.dev);
454 	int ret;
455 	struct intel_crtc *crtc;
456 	struct intel_crtc_state *new_crtc_state;
457 	struct intel_bw_state *new_bw_state = NULL;
458 	const struct intel_bw_state *old_bw_state = NULL;
459 	int i;
460 
461 	for_each_new_intel_crtc_in_state(state, crtc,
462 					 new_crtc_state, i) {
463 		struct skl_pipe_wm *pipe_wm = &new_crtc_state->wm.skl.optimal;
464 
465 		new_bw_state = intel_atomic_get_bw_state(state);
466 		if (IS_ERR(new_bw_state))
467 			return PTR_ERR(new_bw_state);
468 
469 		old_bw_state = intel_atomic_get_old_bw_state(state);
470 
471 		/*
472 		 * We store use_sagv_wm in the crtc state rather than relying on
473 		 * that bw state since we have no convenient way to get at the
474 		 * latter from the plane commit hooks (especially in the legacy
475 		 * cursor case).
476 		 *
477 		 * drm_atomic_check_only() gets upset if we pull more crtcs
478 		 * into the state, so we have to calculate this based on the
479 		 * individual intel_crtc_can_enable_sagv() rather than
480 		 * the overall intel_can_enable_sagv(). Otherwise the
481 		 * crtcs not included in the commit would not switch to the
482 		 * SAGV watermarks when we are about to enable SAGV, and that
483 		 * would lead to underruns. This does mean extra power draw
484 		 * when only a subset of the crtcs are blocking SAGV as the
485 		 * other crtcs can't be allowed to use the more optimal
486 		 * normal (ie. non-SAGV) watermarks.
487 		 */
488 		pipe_wm->use_sagv_wm = !HAS_HW_SAGV_WM(display) &&
489 			DISPLAY_VER(i915) >= 12 &&
490 			intel_crtc_can_enable_sagv(new_crtc_state);
491 
492 		if (intel_crtc_can_enable_sagv(new_crtc_state))
493 			new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe);
494 		else
495 			new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe);
496 	}
497 
498 	if (!new_bw_state)
499 		return 0;
500 
501 	new_bw_state->active_pipes =
502 		intel_calc_active_pipes(state, old_bw_state->active_pipes);
503 
504 	if (new_bw_state->active_pipes != old_bw_state->active_pipes) {
505 		ret = intel_atomic_lock_global_state(&new_bw_state->base);
506 		if (ret)
507 			return ret;
508 	}
509 
510 	if (intel_can_enable_sagv(i915, new_bw_state) !=
511 	    intel_can_enable_sagv(i915, old_bw_state)) {
512 		ret = intel_atomic_serialize_global_state(&new_bw_state->base);
513 		if (ret)
514 			return ret;
515 	} else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
516 		ret = intel_atomic_lock_global_state(&new_bw_state->base);
517 		if (ret)
518 			return ret;
519 	}
520 
521 	return 0;
522 }
523 
skl_ddb_entry_init(struct skl_ddb_entry * entry,u16 start,u16 end)524 static u16 skl_ddb_entry_init(struct skl_ddb_entry *entry,
525 			      u16 start, u16 end)
526 {
527 	entry->start = start;
528 	entry->end = end;
529 
530 	return end;
531 }
532 
intel_dbuf_slice_size(struct drm_i915_private * i915)533 static int intel_dbuf_slice_size(struct drm_i915_private *i915)
534 {
535 	return DISPLAY_INFO(i915)->dbuf.size /
536 		hweight8(DISPLAY_INFO(i915)->dbuf.slice_mask);
537 }
538 
539 static void
skl_ddb_entry_for_slices(struct drm_i915_private * i915,u8 slice_mask,struct skl_ddb_entry * ddb)540 skl_ddb_entry_for_slices(struct drm_i915_private *i915, u8 slice_mask,
541 			 struct skl_ddb_entry *ddb)
542 {
543 	int slice_size = intel_dbuf_slice_size(i915);
544 
545 	if (!slice_mask) {
546 		ddb->start = 0;
547 		ddb->end = 0;
548 		return;
549 	}
550 
551 	ddb->start = (ffs(slice_mask) - 1) * slice_size;
552 	ddb->end = fls(slice_mask) * slice_size;
553 
554 	WARN_ON(ddb->start >= ddb->end);
555 	WARN_ON(ddb->end > DISPLAY_INFO(i915)->dbuf.size);
556 }
557 
mbus_ddb_offset(struct drm_i915_private * i915,u8 slice_mask)558 static unsigned int mbus_ddb_offset(struct drm_i915_private *i915, u8 slice_mask)
559 {
560 	struct skl_ddb_entry ddb;
561 
562 	if (slice_mask & (BIT(DBUF_S1) | BIT(DBUF_S2)))
563 		slice_mask = BIT(DBUF_S1);
564 	else if (slice_mask & (BIT(DBUF_S3) | BIT(DBUF_S4)))
565 		slice_mask = BIT(DBUF_S3);
566 
567 	skl_ddb_entry_for_slices(i915, slice_mask, &ddb);
568 
569 	return ddb.start;
570 }
571 
skl_ddb_dbuf_slice_mask(struct drm_i915_private * i915,const struct skl_ddb_entry * entry)572 u32 skl_ddb_dbuf_slice_mask(struct drm_i915_private *i915,
573 			    const struct skl_ddb_entry *entry)
574 {
575 	int slice_size = intel_dbuf_slice_size(i915);
576 	enum dbuf_slice start_slice, end_slice;
577 	u8 slice_mask = 0;
578 
579 	if (!skl_ddb_entry_size(entry))
580 		return 0;
581 
582 	start_slice = entry->start / slice_size;
583 	end_slice = (entry->end - 1) / slice_size;
584 
585 	/*
586 	 * Per plane DDB entry can in a really worst case be on multiple slices
587 	 * but single entry is anyway contiguous.
588 	 */
589 	while (start_slice <= end_slice) {
590 		slice_mask |= BIT(start_slice);
591 		start_slice++;
592 	}
593 
594 	return slice_mask;
595 }
596 
intel_crtc_ddb_weight(const struct intel_crtc_state * crtc_state)597 static unsigned int intel_crtc_ddb_weight(const struct intel_crtc_state *crtc_state)
598 {
599 	const struct drm_display_mode *pipe_mode = &crtc_state->hw.pipe_mode;
600 	int hdisplay, vdisplay;
601 
602 	if (!crtc_state->hw.active)
603 		return 0;
604 
605 	/*
606 	 * Watermark/ddb requirement highly depends upon width of the
607 	 * framebuffer, So instead of allocating DDB equally among pipes
608 	 * distribute DDB based on resolution/width of the display.
609 	 */
610 	drm_mode_get_hv_timing(pipe_mode, &hdisplay, &vdisplay);
611 
612 	return hdisplay;
613 }
614 
intel_crtc_dbuf_weights(const struct intel_dbuf_state * dbuf_state,enum pipe for_pipe,unsigned int * weight_start,unsigned int * weight_end,unsigned int * weight_total)615 static void intel_crtc_dbuf_weights(const struct intel_dbuf_state *dbuf_state,
616 				    enum pipe for_pipe,
617 				    unsigned int *weight_start,
618 				    unsigned int *weight_end,
619 				    unsigned int *weight_total)
620 {
621 	struct drm_i915_private *i915 =
622 		to_i915(dbuf_state->base.state->base.dev);
623 	enum pipe pipe;
624 
625 	*weight_start = 0;
626 	*weight_end = 0;
627 	*weight_total = 0;
628 
629 	for_each_pipe(i915, pipe) {
630 		int weight = dbuf_state->weight[pipe];
631 
632 		/*
633 		 * Do not account pipes using other slice sets
634 		 * luckily as of current BSpec slice sets do not partially
635 		 * intersect(pipes share either same one slice or same slice set
636 		 * i.e no partial intersection), so it is enough to check for
637 		 * equality for now.
638 		 */
639 		if (dbuf_state->slices[pipe] != dbuf_state->slices[for_pipe])
640 			continue;
641 
642 		*weight_total += weight;
643 		if (pipe < for_pipe) {
644 			*weight_start += weight;
645 			*weight_end += weight;
646 		} else if (pipe == for_pipe) {
647 			*weight_end += weight;
648 		}
649 	}
650 }
651 
652 static int
skl_crtc_allocate_ddb(struct intel_atomic_state * state,struct intel_crtc * crtc)653 skl_crtc_allocate_ddb(struct intel_atomic_state *state, struct intel_crtc *crtc)
654 {
655 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
656 	unsigned int weight_total, weight_start, weight_end;
657 	const struct intel_dbuf_state *old_dbuf_state =
658 		intel_atomic_get_old_dbuf_state(state);
659 	struct intel_dbuf_state *new_dbuf_state =
660 		intel_atomic_get_new_dbuf_state(state);
661 	struct intel_crtc_state *crtc_state;
662 	struct skl_ddb_entry ddb_slices;
663 	enum pipe pipe = crtc->pipe;
664 	unsigned int mbus_offset = 0;
665 	u32 ddb_range_size;
666 	u32 dbuf_slice_mask;
667 	u32 start, end;
668 	int ret;
669 
670 	if (new_dbuf_state->weight[pipe] == 0) {
671 		skl_ddb_entry_init(&new_dbuf_state->ddb[pipe], 0, 0);
672 		goto out;
673 	}
674 
675 	dbuf_slice_mask = new_dbuf_state->slices[pipe];
676 
677 	skl_ddb_entry_for_slices(i915, dbuf_slice_mask, &ddb_slices);
678 	mbus_offset = mbus_ddb_offset(i915, dbuf_slice_mask);
679 	ddb_range_size = skl_ddb_entry_size(&ddb_slices);
680 
681 	intel_crtc_dbuf_weights(new_dbuf_state, pipe,
682 				&weight_start, &weight_end, &weight_total);
683 
684 	start = ddb_range_size * weight_start / weight_total;
685 	end = ddb_range_size * weight_end / weight_total;
686 
687 	skl_ddb_entry_init(&new_dbuf_state->ddb[pipe],
688 			   ddb_slices.start - mbus_offset + start,
689 			   ddb_slices.start - mbus_offset + end);
690 
691 out:
692 	if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe] &&
693 	    skl_ddb_entry_equal(&old_dbuf_state->ddb[pipe],
694 				&new_dbuf_state->ddb[pipe]))
695 		return 0;
696 
697 	ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
698 	if (ret)
699 		return ret;
700 
701 	crtc_state = intel_atomic_get_crtc_state(&state->base, crtc);
702 	if (IS_ERR(crtc_state))
703 		return PTR_ERR(crtc_state);
704 
705 	/*
706 	 * Used for checking overlaps, so we need absolute
707 	 * offsets instead of MBUS relative offsets.
708 	 */
709 	crtc_state->wm.skl.ddb.start = mbus_offset + new_dbuf_state->ddb[pipe].start;
710 	crtc_state->wm.skl.ddb.end = mbus_offset + new_dbuf_state->ddb[pipe].end;
711 
712 	drm_dbg_kms(&i915->drm,
713 		    "[CRTC:%d:%s] dbuf slices 0x%x -> 0x%x, ddb (%d - %d) -> (%d - %d), active pipes 0x%x -> 0x%x\n",
714 		    crtc->base.base.id, crtc->base.name,
715 		    old_dbuf_state->slices[pipe], new_dbuf_state->slices[pipe],
716 		    old_dbuf_state->ddb[pipe].start, old_dbuf_state->ddb[pipe].end,
717 		    new_dbuf_state->ddb[pipe].start, new_dbuf_state->ddb[pipe].end,
718 		    old_dbuf_state->active_pipes, new_dbuf_state->active_pipes);
719 
720 	return 0;
721 }
722 
723 static int skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
724 				 int width, const struct drm_format_info *format,
725 				 u64 modifier, unsigned int rotation,
726 				 u32 plane_pixel_rate, struct skl_wm_params *wp,
727 				 int color_plane, unsigned int pan_x);
728 
729 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
730 				 struct intel_plane *plane,
731 				 int level,
732 				 unsigned int latency,
733 				 const struct skl_wm_params *wp,
734 				 const struct skl_wm_level *result_prev,
735 				 struct skl_wm_level *result /* out */);
736 
skl_wm_latency(struct drm_i915_private * i915,int level,const struct skl_wm_params * wp)737 static unsigned int skl_wm_latency(struct drm_i915_private *i915, int level,
738 				   const struct skl_wm_params *wp)
739 {
740 	unsigned int latency = i915->display.wm.skl_latency[level];
741 
742 	if (latency == 0)
743 		return 0;
744 
745 	/*
746 	 * WaIncreaseLatencyIPCEnabled: kbl,cfl
747 	 * Display WA #1141: kbl,cfl
748 	 */
749 	if ((IS_KABYLAKE(i915) || IS_COFFEELAKE(i915) || IS_COMETLAKE(i915)) &&
750 	    skl_watermark_ipc_enabled(i915))
751 		latency += 4;
752 
753 	if (skl_needs_memory_bw_wa(i915) && wp && wp->x_tiled)
754 		latency += 15;
755 
756 	return latency;
757 }
758 
759 static unsigned int
skl_cursor_allocation(const struct intel_crtc_state * crtc_state,int num_active)760 skl_cursor_allocation(const struct intel_crtc_state *crtc_state,
761 		      int num_active)
762 {
763 	struct intel_plane *plane = to_intel_plane(crtc_state->uapi.crtc->cursor);
764 	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
765 	struct skl_wm_level wm = {};
766 	int ret, min_ddb_alloc = 0;
767 	struct skl_wm_params wp;
768 	int level;
769 
770 	ret = skl_compute_wm_params(crtc_state, 256,
771 				    drm_format_info(DRM_FORMAT_ARGB8888),
772 				    DRM_FORMAT_MOD_LINEAR,
773 				    DRM_MODE_ROTATE_0,
774 				    crtc_state->pixel_rate, &wp, 0, 0);
775 	drm_WARN_ON(&i915->drm, ret);
776 
777 	for (level = 0; level < i915->display.wm.num_levels; level++) {
778 		unsigned int latency = skl_wm_latency(i915, level, &wp);
779 
780 		skl_compute_plane_wm(crtc_state, plane, level, latency, &wp, &wm, &wm);
781 		if (wm.min_ddb_alloc == U16_MAX)
782 			break;
783 
784 		min_ddb_alloc = wm.min_ddb_alloc;
785 	}
786 
787 	return max(num_active == 1 ? 32 : 8, min_ddb_alloc);
788 }
789 
skl_ddb_entry_init_from_hw(struct skl_ddb_entry * entry,u32 reg)790 static void skl_ddb_entry_init_from_hw(struct skl_ddb_entry *entry, u32 reg)
791 {
792 	skl_ddb_entry_init(entry,
793 			   REG_FIELD_GET(PLANE_BUF_START_MASK, reg),
794 			   REG_FIELD_GET(PLANE_BUF_END_MASK, reg));
795 	if (entry->end)
796 		entry->end++;
797 }
798 
799 static void
skl_ddb_get_hw_plane_state(struct drm_i915_private * i915,const enum pipe pipe,const enum plane_id plane_id,struct skl_ddb_entry * ddb,struct skl_ddb_entry * ddb_y,u16 * min_ddb,u16 * interim_ddb)800 skl_ddb_get_hw_plane_state(struct drm_i915_private *i915,
801 			   const enum pipe pipe,
802 			   const enum plane_id plane_id,
803 			   struct skl_ddb_entry *ddb,
804 			   struct skl_ddb_entry *ddb_y,
805 			   u16 *min_ddb, u16 *interim_ddb)
806 {
807 	struct intel_display *display = &i915->display;
808 	u32 val;
809 
810 	/* Cursor doesn't support NV12/planar, so no extra calculation needed */
811 	if (plane_id == PLANE_CURSOR) {
812 		val = intel_de_read(display, CUR_BUF_CFG(pipe));
813 		skl_ddb_entry_init_from_hw(ddb, val);
814 		return;
815 	}
816 
817 	val = intel_de_read(display, PLANE_BUF_CFG(pipe, plane_id));
818 	skl_ddb_entry_init_from_hw(ddb, val);
819 
820 	if (DISPLAY_VER(display) >= 30) {
821 		val = intel_de_read(display, PLANE_MIN_BUF_CFG(pipe, plane_id));
822 
823 		*min_ddb = REG_FIELD_GET(PLANE_MIN_DBUF_BLOCKS_MASK, val);
824 		*interim_ddb = REG_FIELD_GET(PLANE_INTERIM_DBUF_BLOCKS_MASK, val);
825 	}
826 
827 	if (DISPLAY_VER(display) >= 11)
828 		return;
829 
830 	val = intel_de_read(display, PLANE_NV12_BUF_CFG(pipe, plane_id));
831 	skl_ddb_entry_init_from_hw(ddb_y, val);
832 }
833 
skl_pipe_ddb_get_hw_state(struct intel_crtc * crtc,struct skl_ddb_entry * ddb,struct skl_ddb_entry * ddb_y,u16 * min_ddb,u16 * interim_ddb)834 static void skl_pipe_ddb_get_hw_state(struct intel_crtc *crtc,
835 				      struct skl_ddb_entry *ddb,
836 				      struct skl_ddb_entry *ddb_y,
837 				      u16 *min_ddb, u16 *interim_ddb)
838 {
839 	struct intel_display *display = to_intel_display(crtc);
840 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
841 	enum intel_display_power_domain power_domain;
842 	enum pipe pipe = crtc->pipe;
843 	intel_wakeref_t wakeref;
844 	enum plane_id plane_id;
845 
846 	power_domain = POWER_DOMAIN_PIPE(pipe);
847 	wakeref = intel_display_power_get_if_enabled(display, power_domain);
848 	if (!wakeref)
849 		return;
850 
851 	for_each_plane_id_on_crtc(crtc, plane_id)
852 		skl_ddb_get_hw_plane_state(i915, pipe,
853 					   plane_id,
854 					   &ddb[plane_id],
855 					   &ddb_y[plane_id],
856 					   &min_ddb[plane_id],
857 					   &interim_ddb[plane_id]);
858 
859 	intel_display_power_put(display, power_domain, wakeref);
860 }
861 
862 struct dbuf_slice_conf_entry {
863 	u8 active_pipes;
864 	u8 dbuf_mask[I915_MAX_PIPES];
865 	bool join_mbus;
866 };
867 
868 /*
869  * Table taken from Bspec 12716
870  * Pipes do have some preferred DBuf slice affinity,
871  * plus there are some hardcoded requirements on how
872  * those should be distributed for multipipe scenarios.
873  * For more DBuf slices algorithm can get even more messy
874  * and less readable, so decided to use a table almost
875  * as is from BSpec itself - that way it is at least easier
876  * to compare, change and check.
877  */
878 static const struct dbuf_slice_conf_entry icl_allowed_dbufs[] =
879 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
880 {
881 	{
882 		.active_pipes = BIT(PIPE_A),
883 		.dbuf_mask = {
884 			[PIPE_A] = BIT(DBUF_S1),
885 		},
886 	},
887 	{
888 		.active_pipes = BIT(PIPE_B),
889 		.dbuf_mask = {
890 			[PIPE_B] = BIT(DBUF_S1),
891 		},
892 	},
893 	{
894 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
895 		.dbuf_mask = {
896 			[PIPE_A] = BIT(DBUF_S1),
897 			[PIPE_B] = BIT(DBUF_S2),
898 		},
899 	},
900 	{
901 		.active_pipes = BIT(PIPE_C),
902 		.dbuf_mask = {
903 			[PIPE_C] = BIT(DBUF_S2),
904 		},
905 	},
906 	{
907 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
908 		.dbuf_mask = {
909 			[PIPE_A] = BIT(DBUF_S1),
910 			[PIPE_C] = BIT(DBUF_S2),
911 		},
912 	},
913 	{
914 		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
915 		.dbuf_mask = {
916 			[PIPE_B] = BIT(DBUF_S1),
917 			[PIPE_C] = BIT(DBUF_S2),
918 		},
919 	},
920 	{
921 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
922 		.dbuf_mask = {
923 			[PIPE_A] = BIT(DBUF_S1),
924 			[PIPE_B] = BIT(DBUF_S1),
925 			[PIPE_C] = BIT(DBUF_S2),
926 		},
927 	},
928 	{}
929 };
930 
931 /*
932  * Table taken from Bspec 49255
933  * Pipes do have some preferred DBuf slice affinity,
934  * plus there are some hardcoded requirements on how
935  * those should be distributed for multipipe scenarios.
936  * For more DBuf slices algorithm can get even more messy
937  * and less readable, so decided to use a table almost
938  * as is from BSpec itself - that way it is at least easier
939  * to compare, change and check.
940  */
941 static const struct dbuf_slice_conf_entry tgl_allowed_dbufs[] =
942 /* Autogenerated with igt/tools/intel_dbuf_map tool: */
943 {
944 	{
945 		.active_pipes = BIT(PIPE_A),
946 		.dbuf_mask = {
947 			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
948 		},
949 	},
950 	{
951 		.active_pipes = BIT(PIPE_B),
952 		.dbuf_mask = {
953 			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
954 		},
955 	},
956 	{
957 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
958 		.dbuf_mask = {
959 			[PIPE_A] = BIT(DBUF_S2),
960 			[PIPE_B] = BIT(DBUF_S1),
961 		},
962 	},
963 	{
964 		.active_pipes = BIT(PIPE_C),
965 		.dbuf_mask = {
966 			[PIPE_C] = BIT(DBUF_S2) | BIT(DBUF_S1),
967 		},
968 	},
969 	{
970 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
971 		.dbuf_mask = {
972 			[PIPE_A] = BIT(DBUF_S1),
973 			[PIPE_C] = BIT(DBUF_S2),
974 		},
975 	},
976 	{
977 		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
978 		.dbuf_mask = {
979 			[PIPE_B] = BIT(DBUF_S1),
980 			[PIPE_C] = BIT(DBUF_S2),
981 		},
982 	},
983 	{
984 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
985 		.dbuf_mask = {
986 			[PIPE_A] = BIT(DBUF_S1),
987 			[PIPE_B] = BIT(DBUF_S1),
988 			[PIPE_C] = BIT(DBUF_S2),
989 		},
990 	},
991 	{
992 		.active_pipes = BIT(PIPE_D),
993 		.dbuf_mask = {
994 			[PIPE_D] = BIT(DBUF_S2) | BIT(DBUF_S1),
995 		},
996 	},
997 	{
998 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
999 		.dbuf_mask = {
1000 			[PIPE_A] = BIT(DBUF_S1),
1001 			[PIPE_D] = BIT(DBUF_S2),
1002 		},
1003 	},
1004 	{
1005 		.active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
1006 		.dbuf_mask = {
1007 			[PIPE_B] = BIT(DBUF_S1),
1008 			[PIPE_D] = BIT(DBUF_S2),
1009 		},
1010 	},
1011 	{
1012 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
1013 		.dbuf_mask = {
1014 			[PIPE_A] = BIT(DBUF_S1),
1015 			[PIPE_B] = BIT(DBUF_S1),
1016 			[PIPE_D] = BIT(DBUF_S2),
1017 		},
1018 	},
1019 	{
1020 		.active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
1021 		.dbuf_mask = {
1022 			[PIPE_C] = BIT(DBUF_S1),
1023 			[PIPE_D] = BIT(DBUF_S2),
1024 		},
1025 	},
1026 	{
1027 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
1028 		.dbuf_mask = {
1029 			[PIPE_A] = BIT(DBUF_S1),
1030 			[PIPE_C] = BIT(DBUF_S2),
1031 			[PIPE_D] = BIT(DBUF_S2),
1032 		},
1033 	},
1034 	{
1035 		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1036 		.dbuf_mask = {
1037 			[PIPE_B] = BIT(DBUF_S1),
1038 			[PIPE_C] = BIT(DBUF_S2),
1039 			[PIPE_D] = BIT(DBUF_S2),
1040 		},
1041 	},
1042 	{
1043 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1044 		.dbuf_mask = {
1045 			[PIPE_A] = BIT(DBUF_S1),
1046 			[PIPE_B] = BIT(DBUF_S1),
1047 			[PIPE_C] = BIT(DBUF_S2),
1048 			[PIPE_D] = BIT(DBUF_S2),
1049 		},
1050 	},
1051 	{}
1052 };
1053 
1054 static const struct dbuf_slice_conf_entry dg2_allowed_dbufs[] = {
1055 	{
1056 		.active_pipes = BIT(PIPE_A),
1057 		.dbuf_mask = {
1058 			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1059 		},
1060 	},
1061 	{
1062 		.active_pipes = BIT(PIPE_B),
1063 		.dbuf_mask = {
1064 			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1065 		},
1066 	},
1067 	{
1068 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
1069 		.dbuf_mask = {
1070 			[PIPE_A] = BIT(DBUF_S1),
1071 			[PIPE_B] = BIT(DBUF_S2),
1072 		},
1073 	},
1074 	{
1075 		.active_pipes = BIT(PIPE_C),
1076 		.dbuf_mask = {
1077 			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1078 		},
1079 	},
1080 	{
1081 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
1082 		.dbuf_mask = {
1083 			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1084 			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1085 		},
1086 	},
1087 	{
1088 		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
1089 		.dbuf_mask = {
1090 			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1091 			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1092 		},
1093 	},
1094 	{
1095 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
1096 		.dbuf_mask = {
1097 			[PIPE_A] = BIT(DBUF_S1),
1098 			[PIPE_B] = BIT(DBUF_S2),
1099 			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1100 		},
1101 	},
1102 	{
1103 		.active_pipes = BIT(PIPE_D),
1104 		.dbuf_mask = {
1105 			[PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1106 		},
1107 	},
1108 	{
1109 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
1110 		.dbuf_mask = {
1111 			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1112 			[PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1113 		},
1114 	},
1115 	{
1116 		.active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
1117 		.dbuf_mask = {
1118 			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1119 			[PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1120 		},
1121 	},
1122 	{
1123 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
1124 		.dbuf_mask = {
1125 			[PIPE_A] = BIT(DBUF_S1),
1126 			[PIPE_B] = BIT(DBUF_S2),
1127 			[PIPE_D] = BIT(DBUF_S3) | BIT(DBUF_S4),
1128 		},
1129 	},
1130 	{
1131 		.active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
1132 		.dbuf_mask = {
1133 			[PIPE_C] = BIT(DBUF_S3),
1134 			[PIPE_D] = BIT(DBUF_S4),
1135 		},
1136 	},
1137 	{
1138 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
1139 		.dbuf_mask = {
1140 			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1141 			[PIPE_C] = BIT(DBUF_S3),
1142 			[PIPE_D] = BIT(DBUF_S4),
1143 		},
1144 	},
1145 	{
1146 		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1147 		.dbuf_mask = {
1148 			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2),
1149 			[PIPE_C] = BIT(DBUF_S3),
1150 			[PIPE_D] = BIT(DBUF_S4),
1151 		},
1152 	},
1153 	{
1154 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1155 		.dbuf_mask = {
1156 			[PIPE_A] = BIT(DBUF_S1),
1157 			[PIPE_B] = BIT(DBUF_S2),
1158 			[PIPE_C] = BIT(DBUF_S3),
1159 			[PIPE_D] = BIT(DBUF_S4),
1160 		},
1161 	},
1162 	{}
1163 };
1164 
1165 static const struct dbuf_slice_conf_entry adlp_allowed_dbufs[] = {
1166 	/*
1167 	 * Keep the join_mbus cases first so check_mbus_joined()
1168 	 * will prefer them over the !join_mbus cases.
1169 	 */
1170 	{
1171 		.active_pipes = BIT(PIPE_A),
1172 		.dbuf_mask = {
1173 			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
1174 		},
1175 		.join_mbus = true,
1176 	},
1177 	{
1178 		.active_pipes = BIT(PIPE_B),
1179 		.dbuf_mask = {
1180 			[PIPE_B] = BIT(DBUF_S1) | BIT(DBUF_S2) | BIT(DBUF_S3) | BIT(DBUF_S4),
1181 		},
1182 		.join_mbus = true,
1183 	},
1184 	{
1185 		.active_pipes = BIT(PIPE_A),
1186 		.dbuf_mask = {
1187 			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1188 		},
1189 		.join_mbus = false,
1190 	},
1191 	{
1192 		.active_pipes = BIT(PIPE_B),
1193 		.dbuf_mask = {
1194 			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1195 		},
1196 		.join_mbus = false,
1197 	},
1198 	{
1199 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B),
1200 		.dbuf_mask = {
1201 			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1202 			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1203 		},
1204 	},
1205 	{
1206 		.active_pipes = BIT(PIPE_C),
1207 		.dbuf_mask = {
1208 			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1209 		},
1210 	},
1211 	{
1212 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C),
1213 		.dbuf_mask = {
1214 			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1215 			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1216 		},
1217 	},
1218 	{
1219 		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C),
1220 		.dbuf_mask = {
1221 			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1222 			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1223 		},
1224 	},
1225 	{
1226 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C),
1227 		.dbuf_mask = {
1228 			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1229 			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1230 			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1231 		},
1232 	},
1233 	{
1234 		.active_pipes = BIT(PIPE_D),
1235 		.dbuf_mask = {
1236 			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1237 		},
1238 	},
1239 	{
1240 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_D),
1241 		.dbuf_mask = {
1242 			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1243 			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1244 		},
1245 	},
1246 	{
1247 		.active_pipes = BIT(PIPE_B) | BIT(PIPE_D),
1248 		.dbuf_mask = {
1249 			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1250 			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1251 		},
1252 	},
1253 	{
1254 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_D),
1255 		.dbuf_mask = {
1256 			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1257 			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1258 			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1259 		},
1260 	},
1261 	{
1262 		.active_pipes = BIT(PIPE_C) | BIT(PIPE_D),
1263 		.dbuf_mask = {
1264 			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1265 			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1266 		},
1267 	},
1268 	{
1269 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_C) | BIT(PIPE_D),
1270 		.dbuf_mask = {
1271 			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1272 			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1273 			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1274 		},
1275 	},
1276 	{
1277 		.active_pipes = BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1278 		.dbuf_mask = {
1279 			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1280 			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1281 			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1282 		},
1283 	},
1284 	{
1285 		.active_pipes = BIT(PIPE_A) | BIT(PIPE_B) | BIT(PIPE_C) | BIT(PIPE_D),
1286 		.dbuf_mask = {
1287 			[PIPE_A] = BIT(DBUF_S1) | BIT(DBUF_S2),
1288 			[PIPE_B] = BIT(DBUF_S3) | BIT(DBUF_S4),
1289 			[PIPE_C] = BIT(DBUF_S3) | BIT(DBUF_S4),
1290 			[PIPE_D] = BIT(DBUF_S1) | BIT(DBUF_S2),
1291 		},
1292 	},
1293 	{}
1294 
1295 };
1296 
check_mbus_joined(u8 active_pipes,const struct dbuf_slice_conf_entry * dbuf_slices)1297 static bool check_mbus_joined(u8 active_pipes,
1298 			      const struct dbuf_slice_conf_entry *dbuf_slices)
1299 {
1300 	int i;
1301 
1302 	for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
1303 		if (dbuf_slices[i].active_pipes == active_pipes)
1304 			return dbuf_slices[i].join_mbus;
1305 	}
1306 	return false;
1307 }
1308 
adlp_check_mbus_joined(u8 active_pipes)1309 static bool adlp_check_mbus_joined(u8 active_pipes)
1310 {
1311 	return check_mbus_joined(active_pipes, adlp_allowed_dbufs);
1312 }
1313 
compute_dbuf_slices(enum pipe pipe,u8 active_pipes,bool join_mbus,const struct dbuf_slice_conf_entry * dbuf_slices)1314 static u8 compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus,
1315 			      const struct dbuf_slice_conf_entry *dbuf_slices)
1316 {
1317 	int i;
1318 
1319 	for (i = 0; dbuf_slices[i].active_pipes != 0; i++) {
1320 		if (dbuf_slices[i].active_pipes == active_pipes &&
1321 		    dbuf_slices[i].join_mbus == join_mbus)
1322 			return dbuf_slices[i].dbuf_mask[pipe];
1323 	}
1324 	return 0;
1325 }
1326 
1327 /*
1328  * This function finds an entry with same enabled pipe configuration and
1329  * returns correspondent DBuf slice mask as stated in BSpec for particular
1330  * platform.
1331  */
icl_compute_dbuf_slices(enum pipe pipe,u8 active_pipes,bool join_mbus)1332 static u8 icl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1333 {
1334 	/*
1335 	 * FIXME: For ICL this is still a bit unclear as prev BSpec revision
1336 	 * required calculating "pipe ratio" in order to determine
1337 	 * if one or two slices can be used for single pipe configurations
1338 	 * as additional constraint to the existing table.
1339 	 * However based on recent info, it should be not "pipe ratio"
1340 	 * but rather ratio between pixel_rate and cdclk with additional
1341 	 * constants, so for now we are using only table until this is
1342 	 * clarified. Also this is the reason why crtc_state param is
1343 	 * still here - we will need it once those additional constraints
1344 	 * pop up.
1345 	 */
1346 	return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1347 				   icl_allowed_dbufs);
1348 }
1349 
tgl_compute_dbuf_slices(enum pipe pipe,u8 active_pipes,bool join_mbus)1350 static u8 tgl_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1351 {
1352 	return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1353 				   tgl_allowed_dbufs);
1354 }
1355 
adlp_compute_dbuf_slices(enum pipe pipe,u8 active_pipes,bool join_mbus)1356 static u8 adlp_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1357 {
1358 	return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1359 				   adlp_allowed_dbufs);
1360 }
1361 
dg2_compute_dbuf_slices(enum pipe pipe,u8 active_pipes,bool join_mbus)1362 static u8 dg2_compute_dbuf_slices(enum pipe pipe, u8 active_pipes, bool join_mbus)
1363 {
1364 	return compute_dbuf_slices(pipe, active_pipes, join_mbus,
1365 				   dg2_allowed_dbufs);
1366 }
1367 
skl_compute_dbuf_slices(struct intel_crtc * crtc,u8 active_pipes,bool join_mbus)1368 static u8 skl_compute_dbuf_slices(struct intel_crtc *crtc, u8 active_pipes, bool join_mbus)
1369 {
1370 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1371 	enum pipe pipe = crtc->pipe;
1372 
1373 	if (IS_DG2(i915))
1374 		return dg2_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1375 	else if (DISPLAY_VER(i915) >= 13)
1376 		return adlp_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1377 	else if (DISPLAY_VER(i915) == 12)
1378 		return tgl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1379 	else if (DISPLAY_VER(i915) == 11)
1380 		return icl_compute_dbuf_slices(pipe, active_pipes, join_mbus);
1381 	/*
1382 	 * For anything else just return one slice yet.
1383 	 * Should be extended for other platforms.
1384 	 */
1385 	return active_pipes & BIT(pipe) ? BIT(DBUF_S1) : 0;
1386 }
1387 
1388 static bool
use_minimal_wm0_only(const struct intel_crtc_state * crtc_state,struct intel_plane * plane)1389 use_minimal_wm0_only(const struct intel_crtc_state *crtc_state,
1390 		     struct intel_plane *plane)
1391 {
1392 	struct intel_display *display = to_intel_display(plane);
1393 
1394 	/* Xe3+ are auto minimum DDB capble. So don't force minimal wm0 */
1395 	return IS_DISPLAY_VER(display, 13, 20) &&
1396 	       crtc_state->uapi.async_flip &&
1397 	       plane->async_flip;
1398 }
1399 
1400 unsigned int
skl_plane_relative_data_rate(const struct intel_crtc_state * crtc_state,struct intel_plane * plane,int width,int height,int cpp)1401 skl_plane_relative_data_rate(const struct intel_crtc_state *crtc_state,
1402 			     struct intel_plane *plane, int width, int height,
1403 			     int cpp)
1404 {
1405 	/*
1406 	 * We calculate extra ddb based on ratio plane rate/total data rate
1407 	 * in case, in some cases we should not allocate extra ddb for the plane,
1408 	 * so do not count its data rate, if this is the case.
1409 	 */
1410 	if (use_minimal_wm0_only(crtc_state, plane))
1411 		return 0;
1412 
1413 	return width * height * cpp;
1414 }
1415 
1416 static u64
skl_total_relative_data_rate(const struct intel_crtc_state * crtc_state)1417 skl_total_relative_data_rate(const struct intel_crtc_state *crtc_state)
1418 {
1419 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1420 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1421 	enum plane_id plane_id;
1422 	u64 data_rate = 0;
1423 
1424 	for_each_plane_id_on_crtc(crtc, plane_id) {
1425 		if (plane_id == PLANE_CURSOR)
1426 			continue;
1427 
1428 		data_rate += crtc_state->rel_data_rate[plane_id];
1429 
1430 		if (DISPLAY_VER(i915) < 11)
1431 			data_rate += crtc_state->rel_data_rate_y[plane_id];
1432 	}
1433 
1434 	return data_rate;
1435 }
1436 
1437 const struct skl_wm_level *
skl_plane_wm_level(const struct skl_pipe_wm * pipe_wm,enum plane_id plane_id,int level)1438 skl_plane_wm_level(const struct skl_pipe_wm *pipe_wm,
1439 		   enum plane_id plane_id,
1440 		   int level)
1441 {
1442 	const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
1443 
1444 	if (level == 0 && pipe_wm->use_sagv_wm)
1445 		return &wm->sagv.wm0;
1446 
1447 	return &wm->wm[level];
1448 }
1449 
1450 const struct skl_wm_level *
skl_plane_trans_wm(const struct skl_pipe_wm * pipe_wm,enum plane_id plane_id)1451 skl_plane_trans_wm(const struct skl_pipe_wm *pipe_wm,
1452 		   enum plane_id plane_id)
1453 {
1454 	const struct skl_plane_wm *wm = &pipe_wm->planes[plane_id];
1455 
1456 	if (pipe_wm->use_sagv_wm)
1457 		return &wm->sagv.trans_wm;
1458 
1459 	return &wm->trans_wm;
1460 }
1461 
1462 /*
1463  * We only disable the watermarks for each plane if
1464  * they exceed the ddb allocation of said plane. This
1465  * is done so that we don't end up touching cursor
1466  * watermarks needlessly when some other plane reduces
1467  * our max possible watermark level.
1468  *
1469  * Bspec has this to say about the PLANE_WM enable bit:
1470  * "All the watermarks at this level for all enabled
1471  *  planes must be enabled before the level will be used."
1472  * So this is actually safe to do.
1473  */
1474 static void
skl_check_wm_level(struct skl_wm_level * wm,const struct skl_ddb_entry * ddb)1475 skl_check_wm_level(struct skl_wm_level *wm, const struct skl_ddb_entry *ddb)
1476 {
1477 	if (wm->min_ddb_alloc > skl_ddb_entry_size(ddb))
1478 		memset(wm, 0, sizeof(*wm));
1479 }
1480 
1481 static void
skl_check_nv12_wm_level(struct skl_wm_level * wm,struct skl_wm_level * uv_wm,const struct skl_ddb_entry * ddb_y,const struct skl_ddb_entry * ddb)1482 skl_check_nv12_wm_level(struct skl_wm_level *wm, struct skl_wm_level *uv_wm,
1483 			const struct skl_ddb_entry *ddb_y, const struct skl_ddb_entry *ddb)
1484 {
1485 	if (wm->min_ddb_alloc > skl_ddb_entry_size(ddb_y) ||
1486 	    uv_wm->min_ddb_alloc > skl_ddb_entry_size(ddb)) {
1487 		memset(wm, 0, sizeof(*wm));
1488 		memset(uv_wm, 0, sizeof(*uv_wm));
1489 	}
1490 }
1491 
skl_need_wm_copy_wa(struct drm_i915_private * i915,int level,const struct skl_plane_wm * wm)1492 static bool skl_need_wm_copy_wa(struct drm_i915_private *i915, int level,
1493 				const struct skl_plane_wm *wm)
1494 {
1495 	/*
1496 	 * Wa_1408961008:icl, ehl
1497 	 * Wa_14012656716:tgl, adl
1498 	 * Wa_14017887344:icl
1499 	 * Wa_14017868169:adl, tgl
1500 	 * Due to some power saving optimizations, different subsystems
1501 	 * like PSR, might still use even disabled wm level registers,
1502 	 * for "reference", so lets keep at least the values sane.
1503 	 * Considering amount of WA requiring us to do similar things, was
1504 	 * decided to simply do it for all of the platforms, as those wm
1505 	 * levels are disabled, this isn't going to do harm anyway.
1506 	 */
1507 	return level > 0 && !wm->wm[level].enable;
1508 }
1509 
1510 struct skl_plane_ddb_iter {
1511 	u64 data_rate;
1512 	u16 start, size;
1513 };
1514 
1515 static void
skl_allocate_plane_ddb(struct skl_plane_ddb_iter * iter,struct skl_ddb_entry * ddb,const struct skl_wm_level * wm,u64 data_rate)1516 skl_allocate_plane_ddb(struct skl_plane_ddb_iter *iter,
1517 		       struct skl_ddb_entry *ddb,
1518 		       const struct skl_wm_level *wm,
1519 		       u64 data_rate)
1520 {
1521 	u16 size, extra = 0;
1522 
1523 	if (data_rate) {
1524 		extra = min_t(u16, iter->size,
1525 			      DIV64_U64_ROUND_UP(iter->size * data_rate,
1526 						 iter->data_rate));
1527 		iter->size -= extra;
1528 		iter->data_rate -= data_rate;
1529 	}
1530 
1531 	/*
1532 	 * Keep ddb entry of all disabled planes explicitly zeroed
1533 	 * to avoid skl_ddb_add_affected_planes() adding them to
1534 	 * the state when other planes change their allocations.
1535 	 */
1536 	size = wm->min_ddb_alloc + extra;
1537 	if (size)
1538 		iter->start = skl_ddb_entry_init(ddb, iter->start,
1539 						 iter->start + size);
1540 }
1541 
1542 static int
skl_crtc_allocate_plane_ddb(struct intel_atomic_state * state,struct intel_crtc * crtc)1543 skl_crtc_allocate_plane_ddb(struct intel_atomic_state *state,
1544 			    struct intel_crtc *crtc)
1545 {
1546 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1547 	struct intel_crtc_state *crtc_state =
1548 		intel_atomic_get_new_crtc_state(state, crtc);
1549 	const struct intel_dbuf_state *dbuf_state =
1550 		intel_atomic_get_new_dbuf_state(state);
1551 	const struct skl_ddb_entry *alloc = &dbuf_state->ddb[crtc->pipe];
1552 	struct intel_display *display = to_intel_display(state);
1553 	int num_active = hweight8(dbuf_state->active_pipes);
1554 	struct skl_plane_ddb_iter iter;
1555 	enum plane_id plane_id;
1556 	u16 cursor_size;
1557 	u32 blocks;
1558 	int level;
1559 
1560 	/* Clear the partitioning for disabled planes. */
1561 	memset(crtc_state->wm.skl.plane_ddb, 0, sizeof(crtc_state->wm.skl.plane_ddb));
1562 	memset(crtc_state->wm.skl.plane_ddb_y, 0, sizeof(crtc_state->wm.skl.plane_ddb_y));
1563 	memset(crtc_state->wm.skl.plane_min_ddb, 0,
1564 	       sizeof(crtc_state->wm.skl.plane_min_ddb));
1565 	memset(crtc_state->wm.skl.plane_interim_ddb, 0,
1566 	       sizeof(crtc_state->wm.skl.plane_interim_ddb));
1567 
1568 	if (!crtc_state->hw.active)
1569 		return 0;
1570 
1571 	iter.start = alloc->start;
1572 	iter.size = skl_ddb_entry_size(alloc);
1573 	if (iter.size == 0)
1574 		return 0;
1575 
1576 	/* Allocate fixed number of blocks for cursor. */
1577 	cursor_size = skl_cursor_allocation(crtc_state, num_active);
1578 	iter.size -= cursor_size;
1579 	skl_ddb_entry_init(&crtc_state->wm.skl.plane_ddb[PLANE_CURSOR],
1580 			   alloc->end - cursor_size, alloc->end);
1581 
1582 	iter.data_rate = skl_total_relative_data_rate(crtc_state);
1583 
1584 	/*
1585 	 * Find the highest watermark level for which we can satisfy the block
1586 	 * requirement of active planes.
1587 	 */
1588 	for (level = i915->display.wm.num_levels - 1; level >= 0; level--) {
1589 		blocks = 0;
1590 		for_each_plane_id_on_crtc(crtc, plane_id) {
1591 			const struct skl_plane_wm *wm =
1592 				&crtc_state->wm.skl.optimal.planes[plane_id];
1593 
1594 			if (plane_id == PLANE_CURSOR) {
1595 				const struct skl_ddb_entry *ddb =
1596 					&crtc_state->wm.skl.plane_ddb[plane_id];
1597 
1598 				if (wm->wm[level].min_ddb_alloc > skl_ddb_entry_size(ddb)) {
1599 					drm_WARN_ON(&i915->drm,
1600 						    wm->wm[level].min_ddb_alloc != U16_MAX);
1601 					blocks = U32_MAX;
1602 					break;
1603 				}
1604 				continue;
1605 			}
1606 
1607 			blocks += wm->wm[level].min_ddb_alloc;
1608 			blocks += wm->uv_wm[level].min_ddb_alloc;
1609 		}
1610 
1611 		if (blocks <= iter.size) {
1612 			iter.size -= blocks;
1613 			break;
1614 		}
1615 	}
1616 
1617 	if (level < 0) {
1618 		drm_dbg_kms(&i915->drm,
1619 			    "Requested display configuration exceeds system DDB limitations");
1620 		drm_dbg_kms(&i915->drm, "minimum required %d/%d\n",
1621 			    blocks, iter.size);
1622 		return -EINVAL;
1623 	}
1624 
1625 	/* avoid the WARN later when we don't allocate any extra DDB */
1626 	if (iter.data_rate == 0)
1627 		iter.size = 0;
1628 
1629 	/*
1630 	 * Grant each plane the blocks it requires at the highest achievable
1631 	 * watermark level, plus an extra share of the leftover blocks
1632 	 * proportional to its relative data rate.
1633 	 */
1634 	for_each_plane_id_on_crtc(crtc, plane_id) {
1635 		struct skl_ddb_entry *ddb =
1636 			&crtc_state->wm.skl.plane_ddb[plane_id];
1637 		struct skl_ddb_entry *ddb_y =
1638 			&crtc_state->wm.skl.plane_ddb_y[plane_id];
1639 		u16 *min_ddb = &crtc_state->wm.skl.plane_min_ddb[plane_id];
1640 		u16 *interim_ddb =
1641 			&crtc_state->wm.skl.plane_interim_ddb[plane_id];
1642 		const struct skl_plane_wm *wm =
1643 			&crtc_state->wm.skl.optimal.planes[plane_id];
1644 
1645 		if (plane_id == PLANE_CURSOR)
1646 			continue;
1647 
1648 		if (DISPLAY_VER(i915) < 11 &&
1649 		    crtc_state->nv12_planes & BIT(plane_id)) {
1650 			skl_allocate_plane_ddb(&iter, ddb_y, &wm->wm[level],
1651 					       crtc_state->rel_data_rate_y[plane_id]);
1652 			skl_allocate_plane_ddb(&iter, ddb, &wm->uv_wm[level],
1653 					       crtc_state->rel_data_rate[plane_id]);
1654 		} else {
1655 			skl_allocate_plane_ddb(&iter, ddb, &wm->wm[level],
1656 					       crtc_state->rel_data_rate[plane_id]);
1657 		}
1658 
1659 		if (DISPLAY_VER(display) >= 30) {
1660 			*min_ddb = wm->wm[0].min_ddb_alloc;
1661 			*interim_ddb = wm->sagv.wm0.min_ddb_alloc;
1662 		}
1663 	}
1664 	drm_WARN_ON(&i915->drm, iter.size != 0 || iter.data_rate != 0);
1665 
1666 	/*
1667 	 * When we calculated watermark values we didn't know how high
1668 	 * of a level we'd actually be able to hit, so we just marked
1669 	 * all levels as "enabled."  Go back now and disable the ones
1670 	 * that aren't actually possible.
1671 	 */
1672 	for (level++; level < i915->display.wm.num_levels; level++) {
1673 		for_each_plane_id_on_crtc(crtc, plane_id) {
1674 			const struct skl_ddb_entry *ddb =
1675 				&crtc_state->wm.skl.plane_ddb[plane_id];
1676 			const struct skl_ddb_entry *ddb_y =
1677 				&crtc_state->wm.skl.plane_ddb_y[plane_id];
1678 			struct skl_plane_wm *wm =
1679 				&crtc_state->wm.skl.optimal.planes[plane_id];
1680 
1681 			if (DISPLAY_VER(i915) < 11 &&
1682 			    crtc_state->nv12_planes & BIT(plane_id))
1683 				skl_check_nv12_wm_level(&wm->wm[level],
1684 							&wm->uv_wm[level],
1685 							ddb_y, ddb);
1686 			else
1687 				skl_check_wm_level(&wm->wm[level], ddb);
1688 
1689 			if (skl_need_wm_copy_wa(i915, level, wm)) {
1690 				wm->wm[level].blocks = wm->wm[level - 1].blocks;
1691 				wm->wm[level].lines = wm->wm[level - 1].lines;
1692 				wm->wm[level].ignore_lines = wm->wm[level - 1].ignore_lines;
1693 			}
1694 		}
1695 	}
1696 
1697 	/*
1698 	 * Go back and disable the transition and SAGV watermarks
1699 	 * if it turns out we don't have enough DDB blocks for them.
1700 	 */
1701 	for_each_plane_id_on_crtc(crtc, plane_id) {
1702 		const struct skl_ddb_entry *ddb =
1703 			&crtc_state->wm.skl.plane_ddb[plane_id];
1704 		const struct skl_ddb_entry *ddb_y =
1705 			&crtc_state->wm.skl.plane_ddb_y[plane_id];
1706 		u16 *interim_ddb =
1707 			&crtc_state->wm.skl.plane_interim_ddb[plane_id];
1708 		struct skl_plane_wm *wm =
1709 			&crtc_state->wm.skl.optimal.planes[plane_id];
1710 
1711 		if (DISPLAY_VER(i915) < 11 &&
1712 		    crtc_state->nv12_planes & BIT(plane_id)) {
1713 			skl_check_wm_level(&wm->trans_wm, ddb_y);
1714 		} else {
1715 			WARN_ON(skl_ddb_entry_size(ddb_y));
1716 
1717 			skl_check_wm_level(&wm->trans_wm, ddb);
1718 		}
1719 
1720 		skl_check_wm_level(&wm->sagv.wm0, ddb);
1721 		if (DISPLAY_VER(display) >= 30)
1722 			*interim_ddb = wm->sagv.wm0.min_ddb_alloc;
1723 
1724 		skl_check_wm_level(&wm->sagv.trans_wm, ddb);
1725 	}
1726 
1727 	return 0;
1728 }
1729 
1730 /*
1731  * The max latency should be 257 (max the punit can code is 255 and we add 2us
1732  * for the read latency) and cpp should always be <= 8, so that
1733  * should allow pixel_rate up to ~2 GHz which seems sufficient since max
1734  * 2xcdclk is 1350 MHz and the pixel rate should never exceed that.
1735  */
1736 static uint_fixed_16_16_t
skl_wm_method1(const struct drm_i915_private * i915,u32 pixel_rate,u8 cpp,u32 latency,u32 dbuf_block_size)1737 skl_wm_method1(const struct drm_i915_private *i915, u32 pixel_rate,
1738 	       u8 cpp, u32 latency, u32 dbuf_block_size)
1739 {
1740 	u32 wm_intermediate_val;
1741 	uint_fixed_16_16_t ret;
1742 
1743 	if (latency == 0)
1744 		return FP_16_16_MAX;
1745 
1746 	wm_intermediate_val = latency * pixel_rate * cpp;
1747 	ret = div_fixed16(wm_intermediate_val, 1000 * dbuf_block_size);
1748 
1749 	if (DISPLAY_VER(i915) >= 10)
1750 		ret = add_fixed16_u32(ret, 1);
1751 
1752 	return ret;
1753 }
1754 
1755 static uint_fixed_16_16_t
skl_wm_method2(u32 pixel_rate,u32 pipe_htotal,u32 latency,uint_fixed_16_16_t plane_blocks_per_line)1756 skl_wm_method2(u32 pixel_rate, u32 pipe_htotal, u32 latency,
1757 	       uint_fixed_16_16_t plane_blocks_per_line)
1758 {
1759 	u32 wm_intermediate_val;
1760 	uint_fixed_16_16_t ret;
1761 
1762 	if (latency == 0)
1763 		return FP_16_16_MAX;
1764 
1765 	wm_intermediate_val = latency * pixel_rate;
1766 	wm_intermediate_val = DIV_ROUND_UP(wm_intermediate_val,
1767 					   pipe_htotal * 1000);
1768 	ret = mul_u32_fixed16(wm_intermediate_val, plane_blocks_per_line);
1769 	return ret;
1770 }
1771 
1772 static uint_fixed_16_16_t
intel_get_linetime_us(const struct intel_crtc_state * crtc_state)1773 intel_get_linetime_us(const struct intel_crtc_state *crtc_state)
1774 {
1775 	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
1776 	u32 pixel_rate;
1777 	u32 crtc_htotal;
1778 	uint_fixed_16_16_t linetime_us;
1779 
1780 	if (!crtc_state->hw.active)
1781 		return u32_to_fixed16(0);
1782 
1783 	pixel_rate = crtc_state->pixel_rate;
1784 
1785 	if (drm_WARN_ON(&i915->drm, pixel_rate == 0))
1786 		return u32_to_fixed16(0);
1787 
1788 	crtc_htotal = crtc_state->hw.pipe_mode.crtc_htotal;
1789 	linetime_us = div_fixed16(crtc_htotal * 1000, pixel_rate);
1790 
1791 	return linetime_us;
1792 }
1793 
1794 static int
skl_compute_wm_params(const struct intel_crtc_state * crtc_state,int width,const struct drm_format_info * format,u64 modifier,unsigned int rotation,u32 plane_pixel_rate,struct skl_wm_params * wp,int color_plane,unsigned int pan_x)1795 skl_compute_wm_params(const struct intel_crtc_state *crtc_state,
1796 		      int width, const struct drm_format_info *format,
1797 		      u64 modifier, unsigned int rotation,
1798 		      u32 plane_pixel_rate, struct skl_wm_params *wp,
1799 		      int color_plane, unsigned int pan_x)
1800 {
1801 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1802 	struct intel_display *display = to_intel_display(crtc_state);
1803 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
1804 	u32 interm_pbpl;
1805 
1806 	/* only planar format has two planes */
1807 	if (color_plane == 1 &&
1808 	    !intel_format_info_is_yuv_semiplanar(format, modifier)) {
1809 		drm_dbg_kms(&i915->drm,
1810 			    "Non planar format have single plane\n");
1811 		return -EINVAL;
1812 	}
1813 
1814 	wp->x_tiled = modifier == I915_FORMAT_MOD_X_TILED;
1815 	wp->y_tiled = modifier != I915_FORMAT_MOD_X_TILED &&
1816 		intel_fb_is_tiled_modifier(modifier);
1817 	wp->rc_surface = intel_fb_is_ccs_modifier(modifier);
1818 	wp->is_planar = intel_format_info_is_yuv_semiplanar(format, modifier);
1819 
1820 	wp->width = width;
1821 	if (color_plane == 1 && wp->is_planar)
1822 		wp->width /= 2;
1823 
1824 	wp->cpp = format->cpp[color_plane];
1825 	wp->plane_pixel_rate = plane_pixel_rate;
1826 
1827 	if (DISPLAY_VER(i915) >= 11 &&
1828 	    modifier == I915_FORMAT_MOD_Yf_TILED  && wp->cpp == 1)
1829 		wp->dbuf_block_size = 256;
1830 	else
1831 		wp->dbuf_block_size = 512;
1832 
1833 	if (drm_rotation_90_or_270(rotation)) {
1834 		switch (wp->cpp) {
1835 		case 1:
1836 			wp->y_min_scanlines = 16;
1837 			break;
1838 		case 2:
1839 			wp->y_min_scanlines = 8;
1840 			break;
1841 		case 4:
1842 			wp->y_min_scanlines = 4;
1843 			break;
1844 		default:
1845 			MISSING_CASE(wp->cpp);
1846 			return -EINVAL;
1847 		}
1848 	} else {
1849 		wp->y_min_scanlines = 4;
1850 	}
1851 
1852 	if (skl_needs_memory_bw_wa(i915))
1853 		wp->y_min_scanlines *= 2;
1854 
1855 	wp->plane_bytes_per_line = wp->width * wp->cpp;
1856 	if (wp->y_tiled) {
1857 		interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line *
1858 					   wp->y_min_scanlines,
1859 					   wp->dbuf_block_size);
1860 
1861 		if (DISPLAY_VER(display) >= 30)
1862 			interm_pbpl += (pan_x != 0);
1863 		else if (DISPLAY_VER(i915) >= 10)
1864 			interm_pbpl++;
1865 
1866 		wp->plane_blocks_per_line = div_fixed16(interm_pbpl,
1867 							wp->y_min_scanlines);
1868 	} else {
1869 		interm_pbpl = DIV_ROUND_UP(wp->plane_bytes_per_line,
1870 					   wp->dbuf_block_size);
1871 
1872 		if (!wp->x_tiled || DISPLAY_VER(i915) >= 10)
1873 			interm_pbpl++;
1874 
1875 		wp->plane_blocks_per_line = u32_to_fixed16(interm_pbpl);
1876 	}
1877 
1878 	wp->y_tile_minimum = mul_u32_fixed16(wp->y_min_scanlines,
1879 					     wp->plane_blocks_per_line);
1880 
1881 	wp->linetime_us = fixed16_to_u32_round_up(intel_get_linetime_us(crtc_state));
1882 
1883 	return 0;
1884 }
1885 
1886 static int
skl_compute_plane_wm_params(const struct intel_crtc_state * crtc_state,const struct intel_plane_state * plane_state,struct skl_wm_params * wp,int color_plane)1887 skl_compute_plane_wm_params(const struct intel_crtc_state *crtc_state,
1888 			    const struct intel_plane_state *plane_state,
1889 			    struct skl_wm_params *wp, int color_plane)
1890 {
1891 	const struct drm_framebuffer *fb = plane_state->hw.fb;
1892 	int width;
1893 
1894 	/*
1895 	 * Src coordinates are already rotated by 270 degrees for
1896 	 * the 90/270 degree plane rotation cases (to match the
1897 	 * GTT mapping), hence no need to account for rotation here.
1898 	 */
1899 	width = drm_rect_width(&plane_state->uapi.src) >> 16;
1900 
1901 	return skl_compute_wm_params(crtc_state, width,
1902 				     fb->format, fb->modifier,
1903 				     plane_state->hw.rotation,
1904 				     intel_plane_pixel_rate(crtc_state, plane_state),
1905 				     wp, color_plane,
1906 				     plane_state->uapi.src.x1);
1907 }
1908 
skl_wm_has_lines(struct drm_i915_private * i915,int level)1909 static bool skl_wm_has_lines(struct drm_i915_private *i915, int level)
1910 {
1911 	if (DISPLAY_VER(i915) >= 10)
1912 		return true;
1913 
1914 	/* The number of lines are ignored for the level 0 watermark. */
1915 	return level > 0;
1916 }
1917 
skl_wm_max_lines(struct drm_i915_private * i915)1918 static int skl_wm_max_lines(struct drm_i915_private *i915)
1919 {
1920 	if (DISPLAY_VER(i915) >= 13)
1921 		return 255;
1922 	else
1923 		return 31;
1924 }
1925 
xe3_auto_min_alloc_capable(struct intel_plane * plane,int level)1926 static bool xe3_auto_min_alloc_capable(struct intel_plane *plane, int level)
1927 {
1928 	struct intel_display *display = to_intel_display(plane);
1929 
1930 	return DISPLAY_VER(display) >= 30 && level == 0 && plane->id != PLANE_CURSOR;
1931 }
1932 
skl_compute_plane_wm(const struct intel_crtc_state * crtc_state,struct intel_plane * plane,int level,unsigned int latency,const struct skl_wm_params * wp,const struct skl_wm_level * result_prev,struct skl_wm_level * result)1933 static void skl_compute_plane_wm(const struct intel_crtc_state *crtc_state,
1934 				 struct intel_plane *plane,
1935 				 int level,
1936 				 unsigned int latency,
1937 				 const struct skl_wm_params *wp,
1938 				 const struct skl_wm_level *result_prev,
1939 				 struct skl_wm_level *result /* out */)
1940 {
1941 	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
1942 	uint_fixed_16_16_t method1, method2;
1943 	uint_fixed_16_16_t selected_result;
1944 	u32 blocks, lines, min_ddb_alloc = 0;
1945 
1946 	if (latency == 0 ||
1947 	    (use_minimal_wm0_only(crtc_state, plane) && level > 0)) {
1948 		/* reject it */
1949 		result->min_ddb_alloc = U16_MAX;
1950 		return;
1951 	}
1952 
1953 	method1 = skl_wm_method1(i915, wp->plane_pixel_rate,
1954 				 wp->cpp, latency, wp->dbuf_block_size);
1955 	method2 = skl_wm_method2(wp->plane_pixel_rate,
1956 				 crtc_state->hw.pipe_mode.crtc_htotal,
1957 				 latency,
1958 				 wp->plane_blocks_per_line);
1959 
1960 	if (wp->y_tiled) {
1961 		selected_result = max_fixed16(method2, wp->y_tile_minimum);
1962 	} else {
1963 		if ((wp->cpp * crtc_state->hw.pipe_mode.crtc_htotal /
1964 		     wp->dbuf_block_size < 1) &&
1965 		     (wp->plane_bytes_per_line / wp->dbuf_block_size < 1)) {
1966 			selected_result = method2;
1967 		} else if (latency >= wp->linetime_us) {
1968 			if (DISPLAY_VER(i915) == 9)
1969 				selected_result = min_fixed16(method1, method2);
1970 			else
1971 				selected_result = method2;
1972 		} else {
1973 			selected_result = method1;
1974 		}
1975 	}
1976 
1977 	blocks = fixed16_to_u32_round_up(selected_result);
1978 	if (DISPLAY_VER(i915) < 30)
1979 		blocks++;
1980 
1981 	/*
1982 	 * Lets have blocks at minimum equivalent to plane_blocks_per_line
1983 	 * as there will be at minimum one line for lines configuration. This
1984 	 * is a work around for FIFO underruns observed with resolutions like
1985 	 * 4k 60 Hz in single channel DRAM configurations.
1986 	 *
1987 	 * As per the Bspec 49325, if the ddb allocation can hold at least
1988 	 * one plane_blocks_per_line, we should have selected method2 in
1989 	 * the above logic. Assuming that modern versions have enough dbuf
1990 	 * and method2 guarantees blocks equivalent to at least 1 line,
1991 	 * select the blocks as plane_blocks_per_line.
1992 	 *
1993 	 * TODO: Revisit the logic when we have better understanding on DRAM
1994 	 * channels' impact on the level 0 memory latency and the relevant
1995 	 * wm calculations.
1996 	 */
1997 	if (skl_wm_has_lines(i915, level))
1998 		blocks = max(blocks,
1999 			     fixed16_to_u32_round_up(wp->plane_blocks_per_line));
2000 	lines = div_round_up_fixed16(selected_result,
2001 				     wp->plane_blocks_per_line);
2002 
2003 	if (DISPLAY_VER(i915) == 9) {
2004 		/* Display WA #1125: skl,bxt,kbl */
2005 		if (level == 0 && wp->rc_surface)
2006 			blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
2007 
2008 		/* Display WA #1126: skl,bxt,kbl */
2009 		if (level >= 1 && level <= 7) {
2010 			if (wp->y_tiled) {
2011 				blocks += fixed16_to_u32_round_up(wp->y_tile_minimum);
2012 				lines += wp->y_min_scanlines;
2013 			} else {
2014 				blocks++;
2015 			}
2016 
2017 			/*
2018 			 * Make sure result blocks for higher latency levels are
2019 			 * at least as high as level below the current level.
2020 			 * Assumption in DDB algorithm optimization for special
2021 			 * cases. Also covers Display WA #1125 for RC.
2022 			 */
2023 			if (result_prev->blocks > blocks)
2024 				blocks = result_prev->blocks;
2025 		}
2026 	}
2027 
2028 	if (DISPLAY_VER(i915) >= 11) {
2029 		if (wp->y_tiled) {
2030 			int extra_lines;
2031 
2032 			if (lines % wp->y_min_scanlines == 0)
2033 				extra_lines = wp->y_min_scanlines;
2034 			else
2035 				extra_lines = wp->y_min_scanlines * 2 -
2036 					lines % wp->y_min_scanlines;
2037 
2038 			min_ddb_alloc = mul_round_up_u32_fixed16(lines + extra_lines,
2039 								 wp->plane_blocks_per_line);
2040 		} else {
2041 			min_ddb_alloc = blocks + DIV_ROUND_UP(blocks, 10);
2042 		}
2043 	}
2044 
2045 	if (!skl_wm_has_lines(i915, level))
2046 		lines = 0;
2047 
2048 	if (lines > skl_wm_max_lines(i915)) {
2049 		/* reject it */
2050 		result->min_ddb_alloc = U16_MAX;
2051 		return;
2052 	}
2053 
2054 	/*
2055 	 * If lines is valid, assume we can use this watermark level
2056 	 * for now.  We'll come back and disable it after we calculate the
2057 	 * DDB allocation if it turns out we don't actually have enough
2058 	 * blocks to satisfy it.
2059 	 */
2060 	result->blocks = blocks;
2061 	result->lines = lines;
2062 	/* Bspec says: value >= plane ddb allocation -> invalid, hence the +1 here */
2063 	result->min_ddb_alloc = max(min_ddb_alloc, blocks) + 1;
2064 	result->enable = true;
2065 	result->auto_min_alloc_wm_enable = xe3_auto_min_alloc_capable(plane, level);
2066 
2067 	if (DISPLAY_VER(i915) < 12 && i915->display.sagv.block_time_us)
2068 		result->can_sagv = latency >= i915->display.sagv.block_time_us;
2069 }
2070 
2071 static void
skl_compute_wm_levels(const struct intel_crtc_state * crtc_state,struct intel_plane * plane,const struct skl_wm_params * wm_params,struct skl_wm_level * levels)2072 skl_compute_wm_levels(const struct intel_crtc_state *crtc_state,
2073 		      struct intel_plane *plane,
2074 		      const struct skl_wm_params *wm_params,
2075 		      struct skl_wm_level *levels)
2076 {
2077 	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
2078 	struct skl_wm_level *result_prev = &levels[0];
2079 	int level;
2080 
2081 	for (level = 0; level < i915->display.wm.num_levels; level++) {
2082 		struct skl_wm_level *result = &levels[level];
2083 		unsigned int latency = skl_wm_latency(i915, level, wm_params);
2084 
2085 		skl_compute_plane_wm(crtc_state, plane, level, latency,
2086 				     wm_params, result_prev, result);
2087 
2088 		result_prev = result;
2089 	}
2090 }
2091 
tgl_compute_sagv_wm(const struct intel_crtc_state * crtc_state,struct intel_plane * plane,const struct skl_wm_params * wm_params,struct skl_plane_wm * plane_wm)2092 static void tgl_compute_sagv_wm(const struct intel_crtc_state *crtc_state,
2093 				struct intel_plane *plane,
2094 				const struct skl_wm_params *wm_params,
2095 				struct skl_plane_wm *plane_wm)
2096 {
2097 	struct drm_i915_private *i915 = to_i915(crtc_state->uapi.crtc->dev);
2098 	struct skl_wm_level *sagv_wm = &plane_wm->sagv.wm0;
2099 	struct skl_wm_level *levels = plane_wm->wm;
2100 	unsigned int latency = 0;
2101 
2102 	if (i915->display.sagv.block_time_us)
2103 		latency = i915->display.sagv.block_time_us +
2104 			skl_wm_latency(i915, 0, wm_params);
2105 
2106 	skl_compute_plane_wm(crtc_state, plane, 0, latency,
2107 			     wm_params, &levels[0],
2108 			     sagv_wm);
2109 }
2110 
skl_compute_transition_wm(struct drm_i915_private * i915,struct skl_wm_level * trans_wm,const struct skl_wm_level * wm0,const struct skl_wm_params * wp)2111 static void skl_compute_transition_wm(struct drm_i915_private *i915,
2112 				      struct skl_wm_level *trans_wm,
2113 				      const struct skl_wm_level *wm0,
2114 				      const struct skl_wm_params *wp)
2115 {
2116 	u16 trans_min, trans_amount, trans_y_tile_min;
2117 	u16 wm0_blocks, trans_offset, blocks;
2118 
2119 	/* Transition WM don't make any sense if ipc is disabled */
2120 	if (!skl_watermark_ipc_enabled(i915))
2121 		return;
2122 
2123 	/*
2124 	 * WaDisableTWM:skl,kbl,cfl,bxt
2125 	 * Transition WM are not recommended by HW team for GEN9
2126 	 */
2127 	if (DISPLAY_VER(i915) == 9)
2128 		return;
2129 
2130 	if (DISPLAY_VER(i915) >= 11)
2131 		trans_min = 4;
2132 	else
2133 		trans_min = 14;
2134 
2135 	/* Display WA #1140: glk,cnl */
2136 	if (DISPLAY_VER(i915) == 10)
2137 		trans_amount = 0;
2138 	else
2139 		trans_amount = 10; /* This is configurable amount */
2140 
2141 	trans_offset = trans_min + trans_amount;
2142 
2143 	/*
2144 	 * The spec asks for Selected Result Blocks for wm0 (the real value),
2145 	 * not Result Blocks (the integer value). Pay attention to the capital
2146 	 * letters. The value wm_l0->blocks is actually Result Blocks, but
2147 	 * since Result Blocks is the ceiling of Selected Result Blocks plus 1,
2148 	 * and since we later will have to get the ceiling of the sum in the
2149 	 * transition watermarks calculation, we can just pretend Selected
2150 	 * Result Blocks is Result Blocks minus 1 and it should work for the
2151 	 * current platforms.
2152 	 */
2153 	wm0_blocks = wm0->blocks - 1;
2154 
2155 	if (wp->y_tiled) {
2156 		trans_y_tile_min =
2157 			(u16)mul_round_up_u32_fixed16(2, wp->y_tile_minimum);
2158 		blocks = max(wm0_blocks, trans_y_tile_min) + trans_offset;
2159 	} else {
2160 		blocks = wm0_blocks + trans_offset;
2161 	}
2162 	blocks++;
2163 
2164 	/*
2165 	 * Just assume we can enable the transition watermark.  After
2166 	 * computing the DDB we'll come back and disable it if that
2167 	 * assumption turns out to be false.
2168 	 */
2169 	trans_wm->blocks = blocks;
2170 	trans_wm->min_ddb_alloc = max_t(u16, wm0->min_ddb_alloc, blocks + 1);
2171 	trans_wm->enable = true;
2172 }
2173 
skl_build_plane_wm_single(struct intel_crtc_state * crtc_state,const struct intel_plane_state * plane_state,struct intel_plane * plane,int color_plane)2174 static int skl_build_plane_wm_single(struct intel_crtc_state *crtc_state,
2175 				     const struct intel_plane_state *plane_state,
2176 				     struct intel_plane *plane, int color_plane)
2177 {
2178 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2179 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2180 	struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id];
2181 	struct skl_wm_params wm_params;
2182 	int ret;
2183 
2184 	ret = skl_compute_plane_wm_params(crtc_state, plane_state,
2185 					  &wm_params, color_plane);
2186 	if (ret)
2187 		return ret;
2188 
2189 	skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->wm);
2190 
2191 	skl_compute_transition_wm(i915, &wm->trans_wm,
2192 				  &wm->wm[0], &wm_params);
2193 
2194 	if (DISPLAY_VER(i915) >= 12) {
2195 		tgl_compute_sagv_wm(crtc_state, plane, &wm_params, wm);
2196 
2197 		skl_compute_transition_wm(i915, &wm->sagv.trans_wm,
2198 					  &wm->sagv.wm0, &wm_params);
2199 	}
2200 
2201 	return 0;
2202 }
2203 
skl_build_plane_wm_uv(struct intel_crtc_state * crtc_state,const struct intel_plane_state * plane_state,struct intel_plane * plane)2204 static int skl_build_plane_wm_uv(struct intel_crtc_state *crtc_state,
2205 				 const struct intel_plane_state *plane_state,
2206 				 struct intel_plane *plane)
2207 {
2208 	struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane->id];
2209 	struct skl_wm_params wm_params;
2210 	int ret;
2211 
2212 	wm->is_planar = true;
2213 
2214 	/* uv plane watermarks must also be validated for NV12/Planar */
2215 	ret = skl_compute_plane_wm_params(crtc_state, plane_state,
2216 					  &wm_params, 1);
2217 	if (ret)
2218 		return ret;
2219 
2220 	skl_compute_wm_levels(crtc_state, plane, &wm_params, wm->uv_wm);
2221 
2222 	return 0;
2223 }
2224 
skl_build_plane_wm(struct intel_crtc_state * crtc_state,const struct intel_plane_state * plane_state)2225 static int skl_build_plane_wm(struct intel_crtc_state *crtc_state,
2226 			      const struct intel_plane_state *plane_state)
2227 {
2228 	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2229 	enum plane_id plane_id = plane->id;
2230 	struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
2231 	const struct drm_framebuffer *fb = plane_state->hw.fb;
2232 	int ret;
2233 
2234 	memset(wm, 0, sizeof(*wm));
2235 
2236 	if (!intel_wm_plane_visible(crtc_state, plane_state))
2237 		return 0;
2238 
2239 	ret = skl_build_plane_wm_single(crtc_state, plane_state,
2240 					plane, 0);
2241 	if (ret)
2242 		return ret;
2243 
2244 	if (fb->format->is_yuv && fb->format->num_planes > 1) {
2245 		ret = skl_build_plane_wm_uv(crtc_state, plane_state,
2246 					    plane);
2247 		if (ret)
2248 			return ret;
2249 	}
2250 
2251 	return 0;
2252 }
2253 
icl_build_plane_wm(struct intel_crtc_state * crtc_state,const struct intel_plane_state * plane_state)2254 static int icl_build_plane_wm(struct intel_crtc_state *crtc_state,
2255 			      const struct intel_plane_state *plane_state)
2256 {
2257 	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);
2258 	struct drm_i915_private *i915 = to_i915(plane->base.dev);
2259 	enum plane_id plane_id = plane->id;
2260 	struct skl_plane_wm *wm = &crtc_state->wm.skl.raw.planes[plane_id];
2261 	int ret;
2262 
2263 	/* Watermarks calculated on UV plane */
2264 	if (plane_state->is_y_plane)
2265 		return 0;
2266 
2267 	memset(wm, 0, sizeof(*wm));
2268 
2269 	if (plane_state->planar_linked_plane) {
2270 		const struct drm_framebuffer *fb = plane_state->hw.fb;
2271 
2272 		drm_WARN_ON(&i915->drm,
2273 			    !intel_wm_plane_visible(crtc_state, plane_state));
2274 		drm_WARN_ON(&i915->drm, !fb->format->is_yuv ||
2275 			    fb->format->num_planes == 1);
2276 
2277 		ret = skl_build_plane_wm_single(crtc_state, plane_state,
2278 						plane_state->planar_linked_plane, 0);
2279 		if (ret)
2280 			return ret;
2281 
2282 		ret = skl_build_plane_wm_single(crtc_state, plane_state,
2283 						plane, 1);
2284 		if (ret)
2285 			return ret;
2286 	} else if (intel_wm_plane_visible(crtc_state, plane_state)) {
2287 		ret = skl_build_plane_wm_single(crtc_state, plane_state,
2288 						plane, 0);
2289 		if (ret)
2290 			return ret;
2291 	}
2292 
2293 	return 0;
2294 }
2295 
2296 static int
cdclk_prefill_adjustment(const struct intel_crtc_state * crtc_state)2297 cdclk_prefill_adjustment(const struct intel_crtc_state *crtc_state)
2298 {
2299 	struct intel_display *display = to_intel_display(crtc_state);
2300 	struct intel_atomic_state *state =
2301 		to_intel_atomic_state(crtc_state->uapi.state);
2302 	const struct intel_cdclk_state *cdclk_state;
2303 
2304 	cdclk_state = intel_atomic_get_cdclk_state(state);
2305 	if (IS_ERR(cdclk_state)) {
2306 		drm_WARN_ON(display->drm, PTR_ERR(cdclk_state));
2307 		return 1;
2308 	}
2309 
2310 	return min(1, DIV_ROUND_UP(crtc_state->pixel_rate,
2311 				   2 * cdclk_state->logical.cdclk));
2312 }
2313 
2314 static int
dsc_prefill_latency(const struct intel_crtc_state * crtc_state)2315 dsc_prefill_latency(const struct intel_crtc_state *crtc_state)
2316 {
2317 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2318 	const struct intel_crtc_scaler_state *scaler_state =
2319 					&crtc_state->scaler_state;
2320 	int linetime = DIV_ROUND_UP(1000 * crtc_state->hw.adjusted_mode.htotal,
2321 				    crtc_state->hw.adjusted_mode.clock);
2322 	int num_scaler_users = hweight32(scaler_state->scaler_users);
2323 	int chroma_downscaling_factor =
2324 		crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 ? 2 : 1;
2325 	u32 dsc_prefill_latency = 0;
2326 
2327 	if (!crtc_state->dsc.compression_enable ||
2328 	    !num_scaler_users ||
2329 	    num_scaler_users > crtc->num_scalers)
2330 		return dsc_prefill_latency;
2331 
2332 	dsc_prefill_latency = DIV_ROUND_UP(15 * linetime * chroma_downscaling_factor, 10);
2333 
2334 	for (int i = 0; i < num_scaler_users; i++) {
2335 		u64 hscale_k, vscale_k;
2336 
2337 		hscale_k = max(1000, mul_u32_u32(scaler_state->scalers[i].hscale, 1000) >> 16);
2338 		vscale_k = max(1000, mul_u32_u32(scaler_state->scalers[i].vscale, 1000) >> 16);
2339 		dsc_prefill_latency = DIV_ROUND_UP_ULL(dsc_prefill_latency * hscale_k * vscale_k,
2340 						       1000000);
2341 	}
2342 
2343 	dsc_prefill_latency *= cdclk_prefill_adjustment(crtc_state);
2344 
2345 	return intel_usecs_to_scanlines(&crtc_state->hw.adjusted_mode, dsc_prefill_latency);
2346 }
2347 
2348 static int
scaler_prefill_latency(const struct intel_crtc_state * crtc_state)2349 scaler_prefill_latency(const struct intel_crtc_state *crtc_state)
2350 {
2351 	const struct intel_crtc_scaler_state *scaler_state =
2352 					&crtc_state->scaler_state;
2353 	int num_scaler_users = hweight32(scaler_state->scaler_users);
2354 	int scaler_prefill_latency = 0;
2355 	int linetime = DIV_ROUND_UP(1000 * crtc_state->hw.adjusted_mode.htotal,
2356 				    crtc_state->hw.adjusted_mode.clock);
2357 
2358 	if (!num_scaler_users)
2359 		return scaler_prefill_latency;
2360 
2361 	scaler_prefill_latency = 4 * linetime;
2362 
2363 	if (num_scaler_users > 1) {
2364 		u64 hscale_k = max(1000, mul_u32_u32(scaler_state->scalers[0].hscale, 1000) >> 16);
2365 		u64 vscale_k = max(1000, mul_u32_u32(scaler_state->scalers[0].vscale, 1000) >> 16);
2366 		int chroma_downscaling_factor =
2367 			crtc_state->output_format == INTEL_OUTPUT_FORMAT_YCBCR420 ? 2 : 1;
2368 		int latency;
2369 
2370 		latency = DIV_ROUND_UP_ULL((4 * linetime * hscale_k * vscale_k *
2371 					    chroma_downscaling_factor), 1000000);
2372 		scaler_prefill_latency += latency;
2373 	}
2374 
2375 	scaler_prefill_latency *= cdclk_prefill_adjustment(crtc_state);
2376 
2377 	return intel_usecs_to_scanlines(&crtc_state->hw.adjusted_mode, scaler_prefill_latency);
2378 }
2379 
2380 static bool
skl_is_vblank_too_short(const struct intel_crtc_state * crtc_state,int wm0_lines,int latency)2381 skl_is_vblank_too_short(const struct intel_crtc_state *crtc_state,
2382 			int wm0_lines, int latency)
2383 {
2384 	const struct drm_display_mode *adjusted_mode =
2385 		&crtc_state->hw.adjusted_mode;
2386 
2387 	return crtc_state->framestart_delay +
2388 		intel_usecs_to_scanlines(adjusted_mode, latency) +
2389 		scaler_prefill_latency(crtc_state) +
2390 		dsc_prefill_latency(crtc_state) +
2391 		wm0_lines >
2392 		adjusted_mode->crtc_vtotal - adjusted_mode->crtc_vblank_start;
2393 }
2394 
skl_max_wm0_lines(const struct intel_crtc_state * crtc_state)2395 static int skl_max_wm0_lines(const struct intel_crtc_state *crtc_state)
2396 {
2397 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2398 	enum plane_id plane_id;
2399 	int wm0_lines = 0;
2400 
2401 	for_each_plane_id_on_crtc(crtc, plane_id) {
2402 		const struct skl_plane_wm *wm = &crtc_state->wm.skl.optimal.planes[plane_id];
2403 
2404 		/* FIXME what about !skl_wm_has_lines() platforms? */
2405 		wm0_lines = max_t(int, wm0_lines, wm->wm[0].lines);
2406 	}
2407 
2408 	return wm0_lines;
2409 }
2410 
skl_max_wm_level_for_vblank(struct intel_crtc_state * crtc_state,int wm0_lines)2411 static int skl_max_wm_level_for_vblank(struct intel_crtc_state *crtc_state,
2412 				       int wm0_lines)
2413 {
2414 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2415 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2416 	int level;
2417 
2418 	for (level = i915->display.wm.num_levels - 1; level >= 0; level--) {
2419 		int latency;
2420 
2421 		/* FIXME should we care about the latency w/a's? */
2422 		latency = skl_wm_latency(i915, level, NULL);
2423 		if (latency == 0)
2424 			continue;
2425 
2426 		/* FIXME is it correct to use 0 latency for wm0 here? */
2427 		if (level == 0)
2428 			latency = 0;
2429 
2430 		if (!skl_is_vblank_too_short(crtc_state, wm0_lines, latency))
2431 			return level;
2432 	}
2433 
2434 	return -EINVAL;
2435 }
2436 
skl_wm_check_vblank(struct intel_crtc_state * crtc_state)2437 static int skl_wm_check_vblank(struct intel_crtc_state *crtc_state)
2438 {
2439 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
2440 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2441 	int wm0_lines, level;
2442 
2443 	if (!crtc_state->hw.active)
2444 		return 0;
2445 
2446 	wm0_lines = skl_max_wm0_lines(crtc_state);
2447 
2448 	level = skl_max_wm_level_for_vblank(crtc_state, wm0_lines);
2449 	if (level < 0)
2450 		return level;
2451 
2452 	/*
2453 	 * PSR needs to toggle LATENCY_REPORTING_REMOVED_PIPE_*
2454 	 * based on whether we're limited by the vblank duration.
2455 	 */
2456 	crtc_state->wm_level_disabled = level < i915->display.wm.num_levels - 1;
2457 
2458 	for (level++; level < i915->display.wm.num_levels; level++) {
2459 		enum plane_id plane_id;
2460 
2461 		for_each_plane_id_on_crtc(crtc, plane_id) {
2462 			struct skl_plane_wm *wm =
2463 				&crtc_state->wm.skl.optimal.planes[plane_id];
2464 
2465 			/*
2466 			 * FIXME just clear enable or flag the entire
2467 			 * thing as bad via min_ddb_alloc=U16_MAX?
2468 			 */
2469 			wm->wm[level].enable = false;
2470 			wm->uv_wm[level].enable = false;
2471 		}
2472 	}
2473 
2474 	if (DISPLAY_VER(i915) >= 12 &&
2475 	    i915->display.sagv.block_time_us &&
2476 	    skl_is_vblank_too_short(crtc_state, wm0_lines,
2477 				    i915->display.sagv.block_time_us)) {
2478 		enum plane_id plane_id;
2479 
2480 		for_each_plane_id_on_crtc(crtc, plane_id) {
2481 			struct skl_plane_wm *wm =
2482 				&crtc_state->wm.skl.optimal.planes[plane_id];
2483 
2484 			wm->sagv.wm0.enable = false;
2485 			wm->sagv.trans_wm.enable = false;
2486 		}
2487 	}
2488 
2489 	return 0;
2490 }
2491 
skl_build_pipe_wm(struct intel_atomic_state * state,struct intel_crtc * crtc)2492 static int skl_build_pipe_wm(struct intel_atomic_state *state,
2493 			     struct intel_crtc *crtc)
2494 {
2495 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2496 	struct intel_crtc_state *crtc_state =
2497 		intel_atomic_get_new_crtc_state(state, crtc);
2498 	const struct intel_plane_state *plane_state;
2499 	struct intel_plane *plane;
2500 	int ret, i;
2501 
2502 	for_each_new_intel_plane_in_state(state, plane, plane_state, i) {
2503 		/*
2504 		 * FIXME should perhaps check {old,new}_plane_crtc->hw.crtc
2505 		 * instead but we don't populate that correctly for NV12 Y
2506 		 * planes so for now hack this.
2507 		 */
2508 		if (plane->pipe != crtc->pipe)
2509 			continue;
2510 
2511 		if (DISPLAY_VER(i915) >= 11)
2512 			ret = icl_build_plane_wm(crtc_state, plane_state);
2513 		else
2514 			ret = skl_build_plane_wm(crtc_state, plane_state);
2515 		if (ret)
2516 			return ret;
2517 	}
2518 
2519 	crtc_state->wm.skl.optimal = crtc_state->wm.skl.raw;
2520 
2521 	return skl_wm_check_vblank(crtc_state);
2522 }
2523 
skl_wm_level_equals(const struct skl_wm_level * l1,const struct skl_wm_level * l2)2524 static bool skl_wm_level_equals(const struct skl_wm_level *l1,
2525 				const struct skl_wm_level *l2)
2526 {
2527 	return l1->enable == l2->enable &&
2528 		l1->ignore_lines == l2->ignore_lines &&
2529 		l1->lines == l2->lines &&
2530 		l1->blocks == l2->blocks &&
2531 		l1->auto_min_alloc_wm_enable == l2->auto_min_alloc_wm_enable;
2532 }
2533 
skl_plane_wm_equals(struct drm_i915_private * i915,const struct skl_plane_wm * wm1,const struct skl_plane_wm * wm2)2534 static bool skl_plane_wm_equals(struct drm_i915_private *i915,
2535 				const struct skl_plane_wm *wm1,
2536 				const struct skl_plane_wm *wm2)
2537 {
2538 	struct intel_display *display = &i915->display;
2539 	int level;
2540 
2541 	for (level = 0; level < display->wm.num_levels; level++) {
2542 		/*
2543 		 * We don't check uv_wm as the hardware doesn't actually
2544 		 * use it. It only gets used for calculating the required
2545 		 * ddb allocation.
2546 		 */
2547 		if (!skl_wm_level_equals(&wm1->wm[level], &wm2->wm[level]))
2548 			return false;
2549 	}
2550 
2551 	return skl_wm_level_equals(&wm1->trans_wm, &wm2->trans_wm) &&
2552 		skl_wm_level_equals(&wm1->sagv.wm0, &wm2->sagv.wm0) &&
2553 		skl_wm_level_equals(&wm1->sagv.trans_wm, &wm2->sagv.trans_wm);
2554 }
2555 
skl_ddb_entries_overlap(const struct skl_ddb_entry * a,const struct skl_ddb_entry * b)2556 static bool skl_ddb_entries_overlap(const struct skl_ddb_entry *a,
2557 				    const struct skl_ddb_entry *b)
2558 {
2559 	return a->start < b->end && b->start < a->end;
2560 }
2561 
skl_ddb_entry_union(struct skl_ddb_entry * a,const struct skl_ddb_entry * b)2562 static void skl_ddb_entry_union(struct skl_ddb_entry *a,
2563 				const struct skl_ddb_entry *b)
2564 {
2565 	if (a->end && b->end) {
2566 		a->start = min(a->start, b->start);
2567 		a->end = max(a->end, b->end);
2568 	} else if (b->end) {
2569 		a->start = b->start;
2570 		a->end = b->end;
2571 	}
2572 }
2573 
skl_ddb_allocation_overlaps(const struct skl_ddb_entry * ddb,const struct skl_ddb_entry * entries,int num_entries,int ignore_idx)2574 bool skl_ddb_allocation_overlaps(const struct skl_ddb_entry *ddb,
2575 				 const struct skl_ddb_entry *entries,
2576 				 int num_entries, int ignore_idx)
2577 {
2578 	int i;
2579 
2580 	for (i = 0; i < num_entries; i++) {
2581 		if (i != ignore_idx &&
2582 		    skl_ddb_entries_overlap(ddb, &entries[i]))
2583 			return true;
2584 	}
2585 
2586 	return false;
2587 }
2588 
2589 static int
skl_ddb_add_affected_planes(struct intel_atomic_state * state,struct intel_crtc * crtc)2590 skl_ddb_add_affected_planes(struct intel_atomic_state *state,
2591 			    struct intel_crtc *crtc)
2592 {
2593 	struct drm_i915_private *i915 = to_i915(state->base.dev);
2594 	const struct intel_crtc_state *old_crtc_state =
2595 		intel_atomic_get_old_crtc_state(state, crtc);
2596 	struct intel_crtc_state *new_crtc_state =
2597 		intel_atomic_get_new_crtc_state(state, crtc);
2598 	struct intel_plane *plane;
2599 
2600 	for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2601 		struct intel_plane_state *plane_state;
2602 		enum plane_id plane_id = plane->id;
2603 
2604 		if (skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb[plane_id],
2605 					&new_crtc_state->wm.skl.plane_ddb[plane_id]) &&
2606 		    skl_ddb_entry_equal(&old_crtc_state->wm.skl.plane_ddb_y[plane_id],
2607 					&new_crtc_state->wm.skl.plane_ddb_y[plane_id]))
2608 			continue;
2609 
2610 		if (new_crtc_state->do_async_flip) {
2611 			drm_dbg_kms(&i915->drm, "[PLANE:%d:%s] Can't change DDB during async flip\n",
2612 				    plane->base.base.id, plane->base.name);
2613 			return -EINVAL;
2614 		}
2615 
2616 		plane_state = intel_atomic_get_plane_state(state, plane);
2617 		if (IS_ERR(plane_state))
2618 			return PTR_ERR(plane_state);
2619 
2620 		new_crtc_state->update_planes |= BIT(plane_id);
2621 		new_crtc_state->async_flip_planes = 0;
2622 		new_crtc_state->do_async_flip = false;
2623 	}
2624 
2625 	return 0;
2626 }
2627 
intel_dbuf_enabled_slices(const struct intel_dbuf_state * dbuf_state)2628 static u8 intel_dbuf_enabled_slices(const struct intel_dbuf_state *dbuf_state)
2629 {
2630 	struct drm_i915_private *i915 = to_i915(dbuf_state->base.state->base.dev);
2631 	u8 enabled_slices;
2632 	enum pipe pipe;
2633 
2634 	/*
2635 	 * FIXME: For now we always enable slice S1 as per
2636 	 * the Bspec display initialization sequence.
2637 	 */
2638 	enabled_slices = BIT(DBUF_S1);
2639 
2640 	for_each_pipe(i915, pipe)
2641 		enabled_slices |= dbuf_state->slices[pipe];
2642 
2643 	return enabled_slices;
2644 }
2645 
2646 static int
skl_compute_ddb(struct intel_atomic_state * state)2647 skl_compute_ddb(struct intel_atomic_state *state)
2648 {
2649 	struct intel_display *display = to_intel_display(state);
2650 	struct drm_i915_private *i915 = to_i915(state->base.dev);
2651 	const struct intel_dbuf_state *old_dbuf_state;
2652 	struct intel_dbuf_state *new_dbuf_state = NULL;
2653 	struct intel_crtc_state *new_crtc_state;
2654 	struct intel_crtc *crtc;
2655 	int ret, i;
2656 
2657 	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2658 		new_dbuf_state = intel_atomic_get_dbuf_state(state);
2659 		if (IS_ERR(new_dbuf_state))
2660 			return PTR_ERR(new_dbuf_state);
2661 
2662 		old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
2663 		break;
2664 	}
2665 
2666 	if (!new_dbuf_state)
2667 		return 0;
2668 
2669 	new_dbuf_state->active_pipes =
2670 		intel_calc_active_pipes(state, old_dbuf_state->active_pipes);
2671 
2672 	if (old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) {
2673 		ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
2674 		if (ret)
2675 			return ret;
2676 	}
2677 
2678 	if (HAS_MBUS_JOINING(display)) {
2679 		new_dbuf_state->joined_mbus =
2680 			adlp_check_mbus_joined(new_dbuf_state->active_pipes);
2681 
2682 		if (old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
2683 			ret = intel_cdclk_state_set_joined_mbus(state, new_dbuf_state->joined_mbus);
2684 			if (ret)
2685 				return ret;
2686 		}
2687 	}
2688 
2689 	for_each_intel_crtc(&i915->drm, crtc) {
2690 		enum pipe pipe = crtc->pipe;
2691 
2692 		new_dbuf_state->slices[pipe] =
2693 			skl_compute_dbuf_slices(crtc, new_dbuf_state->active_pipes,
2694 						new_dbuf_state->joined_mbus);
2695 
2696 		if (old_dbuf_state->slices[pipe] == new_dbuf_state->slices[pipe])
2697 			continue;
2698 
2699 		ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
2700 		if (ret)
2701 			return ret;
2702 	}
2703 
2704 	new_dbuf_state->enabled_slices = intel_dbuf_enabled_slices(new_dbuf_state);
2705 
2706 	if (old_dbuf_state->enabled_slices != new_dbuf_state->enabled_slices ||
2707 	    old_dbuf_state->joined_mbus != new_dbuf_state->joined_mbus) {
2708 		ret = intel_atomic_serialize_global_state(&new_dbuf_state->base);
2709 		if (ret)
2710 			return ret;
2711 
2712 		drm_dbg_kms(&i915->drm,
2713 			    "Enabled dbuf slices 0x%x -> 0x%x (total dbuf slices 0x%x), mbus joined? %s->%s\n",
2714 			    old_dbuf_state->enabled_slices,
2715 			    new_dbuf_state->enabled_slices,
2716 			    DISPLAY_INFO(i915)->dbuf.slice_mask,
2717 			    str_yes_no(old_dbuf_state->joined_mbus),
2718 			    str_yes_no(new_dbuf_state->joined_mbus));
2719 	}
2720 
2721 	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2722 		enum pipe pipe = crtc->pipe;
2723 
2724 		new_dbuf_state->weight[pipe] = intel_crtc_ddb_weight(new_crtc_state);
2725 
2726 		if (old_dbuf_state->weight[pipe] == new_dbuf_state->weight[pipe])
2727 			continue;
2728 
2729 		ret = intel_atomic_lock_global_state(&new_dbuf_state->base);
2730 		if (ret)
2731 			return ret;
2732 	}
2733 
2734 	for_each_intel_crtc(&i915->drm, crtc) {
2735 		ret = skl_crtc_allocate_ddb(state, crtc);
2736 		if (ret)
2737 			return ret;
2738 	}
2739 
2740 	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
2741 		ret = skl_crtc_allocate_plane_ddb(state, crtc);
2742 		if (ret)
2743 			return ret;
2744 
2745 		ret = skl_ddb_add_affected_planes(state, crtc);
2746 		if (ret)
2747 			return ret;
2748 	}
2749 
2750 	return 0;
2751 }
2752 
enast(bool enable)2753 static char enast(bool enable)
2754 {
2755 	return enable ? '*' : ' ';
2756 }
2757 
2758 static void
skl_print_wm_changes(struct intel_atomic_state * state)2759 skl_print_wm_changes(struct intel_atomic_state *state)
2760 {
2761 	struct drm_i915_private *i915 = to_i915(state->base.dev);
2762 	const struct intel_crtc_state *old_crtc_state;
2763 	const struct intel_crtc_state *new_crtc_state;
2764 	struct intel_plane *plane;
2765 	struct intel_crtc *crtc;
2766 	int i;
2767 
2768 	if (!drm_debug_enabled(DRM_UT_KMS))
2769 		return;
2770 
2771 	for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
2772 					    new_crtc_state, i) {
2773 		const struct skl_pipe_wm *old_pipe_wm, *new_pipe_wm;
2774 
2775 		old_pipe_wm = &old_crtc_state->wm.skl.optimal;
2776 		new_pipe_wm = &new_crtc_state->wm.skl.optimal;
2777 
2778 		for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2779 			enum plane_id plane_id = plane->id;
2780 			const struct skl_ddb_entry *old, *new;
2781 
2782 			old = &old_crtc_state->wm.skl.plane_ddb[plane_id];
2783 			new = &new_crtc_state->wm.skl.plane_ddb[plane_id];
2784 
2785 			if (skl_ddb_entry_equal(old, new))
2786 				continue;
2787 
2788 			drm_dbg_kms(&i915->drm,
2789 				    "[PLANE:%d:%s] ddb (%4d - %4d) -> (%4d - %4d), size %4d -> %4d\n",
2790 				    plane->base.base.id, plane->base.name,
2791 				    old->start, old->end, new->start, new->end,
2792 				    skl_ddb_entry_size(old), skl_ddb_entry_size(new));
2793 		}
2794 
2795 		for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2796 			enum plane_id plane_id = plane->id;
2797 			const struct skl_plane_wm *old_wm, *new_wm;
2798 
2799 			old_wm = &old_pipe_wm->planes[plane_id];
2800 			new_wm = &new_pipe_wm->planes[plane_id];
2801 
2802 			if (skl_plane_wm_equals(i915, old_wm, new_wm))
2803 				continue;
2804 
2805 			drm_dbg_kms(&i915->drm,
2806 				    "[PLANE:%d:%s]   level %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm"
2807 				    " -> %cwm0,%cwm1,%cwm2,%cwm3,%cwm4,%cwm5,%cwm6,%cwm7,%ctwm,%cswm,%cstwm\n",
2808 				    plane->base.base.id, plane->base.name,
2809 				    enast(old_wm->wm[0].enable), enast(old_wm->wm[1].enable),
2810 				    enast(old_wm->wm[2].enable), enast(old_wm->wm[3].enable),
2811 				    enast(old_wm->wm[4].enable), enast(old_wm->wm[5].enable),
2812 				    enast(old_wm->wm[6].enable), enast(old_wm->wm[7].enable),
2813 				    enast(old_wm->trans_wm.enable),
2814 				    enast(old_wm->sagv.wm0.enable),
2815 				    enast(old_wm->sagv.trans_wm.enable),
2816 				    enast(new_wm->wm[0].enable), enast(new_wm->wm[1].enable),
2817 				    enast(new_wm->wm[2].enable), enast(new_wm->wm[3].enable),
2818 				    enast(new_wm->wm[4].enable), enast(new_wm->wm[5].enable),
2819 				    enast(new_wm->wm[6].enable), enast(new_wm->wm[7].enable),
2820 				    enast(new_wm->trans_wm.enable),
2821 				    enast(new_wm->sagv.wm0.enable),
2822 				    enast(new_wm->sagv.trans_wm.enable));
2823 
2824 			drm_dbg_kms(&i915->drm,
2825 				    "[PLANE:%d:%s]   lines %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d"
2826 				      " -> %c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%3d,%c%4d\n",
2827 				    plane->base.base.id, plane->base.name,
2828 				    enast(old_wm->wm[0].ignore_lines), old_wm->wm[0].lines,
2829 				    enast(old_wm->wm[1].ignore_lines), old_wm->wm[1].lines,
2830 				    enast(old_wm->wm[2].ignore_lines), old_wm->wm[2].lines,
2831 				    enast(old_wm->wm[3].ignore_lines), old_wm->wm[3].lines,
2832 				    enast(old_wm->wm[4].ignore_lines), old_wm->wm[4].lines,
2833 				    enast(old_wm->wm[5].ignore_lines), old_wm->wm[5].lines,
2834 				    enast(old_wm->wm[6].ignore_lines), old_wm->wm[6].lines,
2835 				    enast(old_wm->wm[7].ignore_lines), old_wm->wm[7].lines,
2836 				    enast(old_wm->trans_wm.ignore_lines), old_wm->trans_wm.lines,
2837 				    enast(old_wm->sagv.wm0.ignore_lines), old_wm->sagv.wm0.lines,
2838 				    enast(old_wm->sagv.trans_wm.ignore_lines), old_wm->sagv.trans_wm.lines,
2839 				    enast(new_wm->wm[0].ignore_lines), new_wm->wm[0].lines,
2840 				    enast(new_wm->wm[1].ignore_lines), new_wm->wm[1].lines,
2841 				    enast(new_wm->wm[2].ignore_lines), new_wm->wm[2].lines,
2842 				    enast(new_wm->wm[3].ignore_lines), new_wm->wm[3].lines,
2843 				    enast(new_wm->wm[4].ignore_lines), new_wm->wm[4].lines,
2844 				    enast(new_wm->wm[5].ignore_lines), new_wm->wm[5].lines,
2845 				    enast(new_wm->wm[6].ignore_lines), new_wm->wm[6].lines,
2846 				    enast(new_wm->wm[7].ignore_lines), new_wm->wm[7].lines,
2847 				    enast(new_wm->trans_wm.ignore_lines), new_wm->trans_wm.lines,
2848 				    enast(new_wm->sagv.wm0.ignore_lines), new_wm->sagv.wm0.lines,
2849 				    enast(new_wm->sagv.trans_wm.ignore_lines), new_wm->sagv.trans_wm.lines);
2850 
2851 			drm_dbg_kms(&i915->drm,
2852 				    "[PLANE:%d:%s]  blocks %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
2853 				    " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
2854 				    plane->base.base.id, plane->base.name,
2855 				    old_wm->wm[0].blocks, old_wm->wm[1].blocks,
2856 				    old_wm->wm[2].blocks, old_wm->wm[3].blocks,
2857 				    old_wm->wm[4].blocks, old_wm->wm[5].blocks,
2858 				    old_wm->wm[6].blocks, old_wm->wm[7].blocks,
2859 				    old_wm->trans_wm.blocks,
2860 				    old_wm->sagv.wm0.blocks,
2861 				    old_wm->sagv.trans_wm.blocks,
2862 				    new_wm->wm[0].blocks, new_wm->wm[1].blocks,
2863 				    new_wm->wm[2].blocks, new_wm->wm[3].blocks,
2864 				    new_wm->wm[4].blocks, new_wm->wm[5].blocks,
2865 				    new_wm->wm[6].blocks, new_wm->wm[7].blocks,
2866 				    new_wm->trans_wm.blocks,
2867 				    new_wm->sagv.wm0.blocks,
2868 				    new_wm->sagv.trans_wm.blocks);
2869 
2870 			drm_dbg_kms(&i915->drm,
2871 				    "[PLANE:%d:%s] min_ddb %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d"
2872 				    " -> %4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%4d,%5d\n",
2873 				    plane->base.base.id, plane->base.name,
2874 				    old_wm->wm[0].min_ddb_alloc, old_wm->wm[1].min_ddb_alloc,
2875 				    old_wm->wm[2].min_ddb_alloc, old_wm->wm[3].min_ddb_alloc,
2876 				    old_wm->wm[4].min_ddb_alloc, old_wm->wm[5].min_ddb_alloc,
2877 				    old_wm->wm[6].min_ddb_alloc, old_wm->wm[7].min_ddb_alloc,
2878 				    old_wm->trans_wm.min_ddb_alloc,
2879 				    old_wm->sagv.wm0.min_ddb_alloc,
2880 				    old_wm->sagv.trans_wm.min_ddb_alloc,
2881 				    new_wm->wm[0].min_ddb_alloc, new_wm->wm[1].min_ddb_alloc,
2882 				    new_wm->wm[2].min_ddb_alloc, new_wm->wm[3].min_ddb_alloc,
2883 				    new_wm->wm[4].min_ddb_alloc, new_wm->wm[5].min_ddb_alloc,
2884 				    new_wm->wm[6].min_ddb_alloc, new_wm->wm[7].min_ddb_alloc,
2885 				    new_wm->trans_wm.min_ddb_alloc,
2886 				    new_wm->sagv.wm0.min_ddb_alloc,
2887 				    new_wm->sagv.trans_wm.min_ddb_alloc);
2888 		}
2889 	}
2890 }
2891 
skl_plane_selected_wm_equals(struct intel_plane * plane,const struct skl_pipe_wm * old_pipe_wm,const struct skl_pipe_wm * new_pipe_wm)2892 static bool skl_plane_selected_wm_equals(struct intel_plane *plane,
2893 					 const struct skl_pipe_wm *old_pipe_wm,
2894 					 const struct skl_pipe_wm *new_pipe_wm)
2895 {
2896 	struct intel_display *display = to_intel_display(plane);
2897 	int level;
2898 
2899 	for (level = 0; level < display->wm.num_levels; level++) {
2900 		/*
2901 		 * We don't check uv_wm as the hardware doesn't actually
2902 		 * use it. It only gets used for calculating the required
2903 		 * ddb allocation.
2904 		 */
2905 		if (!skl_wm_level_equals(skl_plane_wm_level(old_pipe_wm, plane->id, level),
2906 					 skl_plane_wm_level(new_pipe_wm, plane->id, level)))
2907 			return false;
2908 	}
2909 
2910 	if (HAS_HW_SAGV_WM(display)) {
2911 		const struct skl_plane_wm *old_wm = &old_pipe_wm->planes[plane->id];
2912 		const struct skl_plane_wm *new_wm = &new_pipe_wm->planes[plane->id];
2913 
2914 		if (!skl_wm_level_equals(&old_wm->sagv.wm0, &new_wm->sagv.wm0) ||
2915 		    !skl_wm_level_equals(&old_wm->sagv.trans_wm, &new_wm->sagv.trans_wm))
2916 			return false;
2917 	}
2918 
2919 	return skl_wm_level_equals(skl_plane_trans_wm(old_pipe_wm, plane->id),
2920 				   skl_plane_trans_wm(new_pipe_wm, plane->id));
2921 }
2922 
2923 /*
2924  * To make sure the cursor watermark registers are always consistent
2925  * with our computed state the following scenario needs special
2926  * treatment:
2927  *
2928  * 1. enable cursor
2929  * 2. move cursor entirely offscreen
2930  * 3. disable cursor
2931  *
2932  * Step 2. does call .disable_plane() but does not zero the watermarks
2933  * (since we consider an offscreen cursor still active for the purposes
2934  * of watermarks). Step 3. would not normally call .disable_plane()
2935  * because the actual plane visibility isn't changing, and we don't
2936  * deallocate the cursor ddb until the pipe gets disabled. So we must
2937  * force step 3. to call .disable_plane() to update the watermark
2938  * registers properly.
2939  *
2940  * Other planes do not suffer from this issues as their watermarks are
2941  * calculated based on the actual plane visibility. The only time this
2942  * can trigger for the other planes is during the initial readout as the
2943  * default value of the watermarks registers is not zero.
2944  */
skl_wm_add_affected_planes(struct intel_atomic_state * state,struct intel_crtc * crtc)2945 static int skl_wm_add_affected_planes(struct intel_atomic_state *state,
2946 				      struct intel_crtc *crtc)
2947 {
2948 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
2949 	const struct intel_crtc_state *old_crtc_state =
2950 		intel_atomic_get_old_crtc_state(state, crtc);
2951 	struct intel_crtc_state *new_crtc_state =
2952 		intel_atomic_get_new_crtc_state(state, crtc);
2953 	struct intel_plane *plane;
2954 
2955 	for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
2956 		struct intel_plane_state *plane_state;
2957 		enum plane_id plane_id = plane->id;
2958 
2959 		/*
2960 		 * Force a full wm update for every plane on modeset.
2961 		 * Required because the reset value of the wm registers
2962 		 * is non-zero, whereas we want all disabled planes to
2963 		 * have zero watermarks. So if we turn off the relevant
2964 		 * power well the hardware state will go out of sync
2965 		 * with the software state.
2966 		 */
2967 		if (!intel_crtc_needs_modeset(new_crtc_state) &&
2968 		    skl_plane_selected_wm_equals(plane,
2969 						 &old_crtc_state->wm.skl.optimal,
2970 						 &new_crtc_state->wm.skl.optimal))
2971 			continue;
2972 
2973 		if (new_crtc_state->do_async_flip) {
2974 			drm_dbg_kms(&i915->drm, "[PLANE:%d:%s] Can't change watermarks during async flip\n",
2975 				    plane->base.base.id, plane->base.name);
2976 			return -EINVAL;
2977 		}
2978 
2979 		plane_state = intel_atomic_get_plane_state(state, plane);
2980 		if (IS_ERR(plane_state))
2981 			return PTR_ERR(plane_state);
2982 
2983 		new_crtc_state->update_planes |= BIT(plane_id);
2984 		new_crtc_state->async_flip_planes = 0;
2985 		new_crtc_state->do_async_flip = false;
2986 	}
2987 
2988 	return 0;
2989 }
2990 
2991 /*
2992  * If Fixed Refresh Rate or For VRR case Vmin = Vmax = Flipline:
2993  * Program DEEP PKG_C_LATENCY Pkg C with highest valid latency from
2994  * watermark level1 and up and above. If watermark level 1 is
2995  * invalid program it with all 1's.
2996  * Program PKG_C_LATENCY Added Wake Time = DSB execution time
2997  * If Variable Refresh Rate where Vmin != Vmax != Flipline:
2998  * Program DEEP PKG_C_LATENCY Pkg C with all 1's.
2999  * Program PKG_C_LATENCY Added Wake Time = 0
3000  */
3001 void
intel_program_dpkgc_latency(struct intel_atomic_state * state)3002 intel_program_dpkgc_latency(struct intel_atomic_state *state)
3003 {
3004 	struct intel_display *display = to_intel_display(state);
3005 	struct drm_i915_private *i915 = to_i915(display->drm);
3006 	struct intel_crtc *crtc;
3007 	struct intel_crtc_state *new_crtc_state;
3008 	u32 latency = LNL_PKG_C_LATENCY_MASK;
3009 	u32 added_wake_time = 0;
3010 	u32 max_linetime = 0;
3011 	u32 clear, val;
3012 	bool fixed_refresh_rate = false;
3013 	int i;
3014 
3015 	if (DISPLAY_VER(display) < 20)
3016 		return;
3017 
3018 	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
3019 		if (!new_crtc_state->vrr.enable ||
3020 		    (new_crtc_state->vrr.vmin == new_crtc_state->vrr.vmax &&
3021 		     new_crtc_state->vrr.vmin == new_crtc_state->vrr.flipline))
3022 			fixed_refresh_rate = true;
3023 
3024 		max_linetime = max(new_crtc_state->linetime, max_linetime);
3025 	}
3026 
3027 	if (fixed_refresh_rate) {
3028 		added_wake_time = DSB_EXE_TIME +
3029 			display->sagv.block_time_us;
3030 
3031 		latency = skl_watermark_max_latency(i915, 1);
3032 
3033 		/* Wa_22020432604 */
3034 		if ((DISPLAY_VER(display) == 20 || DISPLAY_VER(display) == 30) && !latency) {
3035 			latency += added_wake_time;
3036 			added_wake_time = 0;
3037 		}
3038 
3039 		/* Wa_22020299601 */
3040 		if ((latency && max_linetime) &&
3041 		    (DISPLAY_VER(display) == 20 || DISPLAY_VER(display) == 30)) {
3042 			latency = max_linetime * DIV_ROUND_UP(latency, max_linetime);
3043 		} else if (!latency) {
3044 			latency = LNL_PKG_C_LATENCY_MASK;
3045 		}
3046 	}
3047 
3048 	clear = LNL_ADDED_WAKE_TIME_MASK | LNL_PKG_C_LATENCY_MASK;
3049 	val = REG_FIELD_PREP(LNL_PKG_C_LATENCY_MASK, latency) |
3050 		REG_FIELD_PREP(LNL_ADDED_WAKE_TIME_MASK, added_wake_time);
3051 
3052 	intel_de_rmw(display, LNL_PKG_C_LATENCY, clear, val);
3053 }
3054 
3055 static int
skl_compute_wm(struct intel_atomic_state * state)3056 skl_compute_wm(struct intel_atomic_state *state)
3057 {
3058 	struct intel_crtc *crtc;
3059 	struct intel_crtc_state __maybe_unused *new_crtc_state;
3060 	int ret, i;
3061 
3062 	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
3063 		ret = skl_build_pipe_wm(state, crtc);
3064 		if (ret)
3065 			return ret;
3066 	}
3067 
3068 	ret = skl_compute_ddb(state);
3069 	if (ret)
3070 		return ret;
3071 
3072 	ret = intel_compute_sagv_mask(state);
3073 	if (ret)
3074 		return ret;
3075 
3076 	/*
3077 	 * skl_compute_ddb() will have adjusted the final watermarks
3078 	 * based on how much ddb is available. Now we can actually
3079 	 * check if the final watermarks changed.
3080 	 */
3081 	for_each_new_intel_crtc_in_state(state, crtc, new_crtc_state, i) {
3082 		ret = skl_wm_add_affected_planes(state, crtc);
3083 		if (ret)
3084 			return ret;
3085 	}
3086 
3087 	skl_print_wm_changes(state);
3088 
3089 	return 0;
3090 }
3091 
skl_wm_level_from_reg_val(struct intel_display * display,u32 val,struct skl_wm_level * level)3092 static void skl_wm_level_from_reg_val(struct intel_display *display,
3093 				      u32 val, struct skl_wm_level *level)
3094 {
3095 	level->enable = val & PLANE_WM_EN;
3096 	level->ignore_lines = val & PLANE_WM_IGNORE_LINES;
3097 	level->blocks = REG_FIELD_GET(PLANE_WM_BLOCKS_MASK, val);
3098 	level->lines = REG_FIELD_GET(PLANE_WM_LINES_MASK, val);
3099 	level->auto_min_alloc_wm_enable = DISPLAY_VER(display) >= 30 ?
3100 					   val & PLANE_WM_AUTO_MIN_ALLOC_EN : 0;
3101 }
3102 
skl_pipe_wm_get_hw_state(struct intel_crtc * crtc,struct skl_pipe_wm * out)3103 static void skl_pipe_wm_get_hw_state(struct intel_crtc *crtc,
3104 				     struct skl_pipe_wm *out)
3105 {
3106 	struct intel_display *display = to_intel_display(crtc);
3107 	enum pipe pipe = crtc->pipe;
3108 	enum plane_id plane_id;
3109 	int level;
3110 	u32 val;
3111 
3112 	for_each_plane_id_on_crtc(crtc, plane_id) {
3113 		struct skl_plane_wm *wm = &out->planes[plane_id];
3114 
3115 		for (level = 0; level < display->wm.num_levels; level++) {
3116 			if (plane_id != PLANE_CURSOR)
3117 				val = intel_de_read(display, PLANE_WM(pipe, plane_id, level));
3118 			else
3119 				val = intel_de_read(display, CUR_WM(pipe, level));
3120 
3121 			skl_wm_level_from_reg_val(display, val, &wm->wm[level]);
3122 		}
3123 
3124 		if (plane_id != PLANE_CURSOR)
3125 			val = intel_de_read(display, PLANE_WM_TRANS(pipe, plane_id));
3126 		else
3127 			val = intel_de_read(display, CUR_WM_TRANS(pipe));
3128 
3129 		skl_wm_level_from_reg_val(display, val, &wm->trans_wm);
3130 
3131 		if (HAS_HW_SAGV_WM(display)) {
3132 			if (plane_id != PLANE_CURSOR)
3133 				val = intel_de_read(display, PLANE_WM_SAGV(pipe, plane_id));
3134 			else
3135 				val = intel_de_read(display, CUR_WM_SAGV(pipe));
3136 
3137 			skl_wm_level_from_reg_val(display, val, &wm->sagv.wm0);
3138 
3139 			if (plane_id != PLANE_CURSOR)
3140 				val = intel_de_read(display, PLANE_WM_SAGV_TRANS(pipe, plane_id));
3141 			else
3142 				val = intel_de_read(display, CUR_WM_SAGV_TRANS(pipe));
3143 
3144 			skl_wm_level_from_reg_val(display, val, &wm->sagv.trans_wm);
3145 		} else if (DISPLAY_VER(display) >= 12) {
3146 			wm->sagv.wm0 = wm->wm[0];
3147 			wm->sagv.trans_wm = wm->trans_wm;
3148 		}
3149 	}
3150 }
3151 
skl_wm_get_hw_state(struct drm_i915_private * i915)3152 static void skl_wm_get_hw_state(struct drm_i915_private *i915)
3153 {
3154 	struct intel_display *display = &i915->display;
3155 	struct intel_dbuf_state *dbuf_state =
3156 		to_intel_dbuf_state(i915->display.dbuf.obj.state);
3157 	struct intel_crtc *crtc;
3158 
3159 	if (HAS_MBUS_JOINING(display))
3160 		dbuf_state->joined_mbus = intel_de_read(display, MBUS_CTL) & MBUS_JOIN;
3161 
3162 	dbuf_state->mdclk_cdclk_ratio = intel_mdclk_cdclk_ratio(display, &display->cdclk.hw);
3163 	dbuf_state->active_pipes = 0;
3164 
3165 	for_each_intel_crtc(display->drm, crtc) {
3166 		struct intel_crtc_state *crtc_state =
3167 			to_intel_crtc_state(crtc->base.state);
3168 		enum pipe pipe = crtc->pipe;
3169 		unsigned int mbus_offset;
3170 		enum plane_id plane_id;
3171 		u8 slices;
3172 
3173 		memset(&crtc_state->wm.skl.optimal, 0,
3174 		       sizeof(crtc_state->wm.skl.optimal));
3175 		if (crtc_state->hw.active) {
3176 			skl_pipe_wm_get_hw_state(crtc, &crtc_state->wm.skl.optimal);
3177 			dbuf_state->active_pipes |= BIT(pipe);
3178 		}
3179 		crtc_state->wm.skl.raw = crtc_state->wm.skl.optimal;
3180 
3181 		memset(&dbuf_state->ddb[pipe], 0, sizeof(dbuf_state->ddb[pipe]));
3182 
3183 		for_each_plane_id_on_crtc(crtc, plane_id) {
3184 			struct skl_ddb_entry *ddb =
3185 				&crtc_state->wm.skl.plane_ddb[plane_id];
3186 			struct skl_ddb_entry *ddb_y =
3187 				&crtc_state->wm.skl.plane_ddb_y[plane_id];
3188 			u16 *min_ddb =
3189 				&crtc_state->wm.skl.plane_min_ddb[plane_id];
3190 			u16 *interim_ddb =
3191 				&crtc_state->wm.skl.plane_interim_ddb[plane_id];
3192 
3193 			if (!crtc_state->hw.active)
3194 				continue;
3195 
3196 			skl_ddb_get_hw_plane_state(i915, crtc->pipe,
3197 						   plane_id, ddb, ddb_y,
3198 						   min_ddb, interim_ddb);
3199 
3200 			skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb);
3201 			skl_ddb_entry_union(&dbuf_state->ddb[pipe], ddb_y);
3202 		}
3203 
3204 		dbuf_state->weight[pipe] = intel_crtc_ddb_weight(crtc_state);
3205 
3206 		/*
3207 		 * Used for checking overlaps, so we need absolute
3208 		 * offsets instead of MBUS relative offsets.
3209 		 */
3210 		slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes,
3211 						 dbuf_state->joined_mbus);
3212 		mbus_offset = mbus_ddb_offset(i915, slices);
3213 		crtc_state->wm.skl.ddb.start = mbus_offset + dbuf_state->ddb[pipe].start;
3214 		crtc_state->wm.skl.ddb.end = mbus_offset + dbuf_state->ddb[pipe].end;
3215 
3216 		/* The slices actually used by the planes on the pipe */
3217 		dbuf_state->slices[pipe] =
3218 			skl_ddb_dbuf_slice_mask(i915, &crtc_state->wm.skl.ddb);
3219 
3220 		drm_dbg_kms(display->drm,
3221 			    "[CRTC:%d:%s] dbuf slices 0x%x, ddb (%d - %d), active pipes 0x%x, mbus joined: %s\n",
3222 			    crtc->base.base.id, crtc->base.name,
3223 			    dbuf_state->slices[pipe], dbuf_state->ddb[pipe].start,
3224 			    dbuf_state->ddb[pipe].end, dbuf_state->active_pipes,
3225 			    str_yes_no(dbuf_state->joined_mbus));
3226 	}
3227 
3228 	dbuf_state->enabled_slices = display->dbuf.enabled_slices;
3229 }
3230 
skl_watermark_ipc_enabled(struct drm_i915_private * i915)3231 bool skl_watermark_ipc_enabled(struct drm_i915_private *i915)
3232 {
3233 	return i915->display.wm.ipc_enabled;
3234 }
3235 
skl_watermark_ipc_update(struct drm_i915_private * i915)3236 void skl_watermark_ipc_update(struct drm_i915_private *i915)
3237 {
3238 	if (!HAS_IPC(i915))
3239 		return;
3240 
3241 	intel_de_rmw(i915, DISP_ARB_CTL2, DISP_IPC_ENABLE,
3242 		     skl_watermark_ipc_enabled(i915) ? DISP_IPC_ENABLE : 0);
3243 }
3244 
skl_watermark_ipc_can_enable(struct drm_i915_private * i915)3245 static bool skl_watermark_ipc_can_enable(struct drm_i915_private *i915)
3246 {
3247 	/* Display WA #0477 WaDisableIPC: skl */
3248 	if (IS_SKYLAKE(i915))
3249 		return false;
3250 
3251 	/* Display WA #1141: SKL:all KBL:all CFL */
3252 	if (IS_KABYLAKE(i915) ||
3253 	    IS_COFFEELAKE(i915) ||
3254 	    IS_COMETLAKE(i915))
3255 		return i915->dram_info.symmetric_memory;
3256 
3257 	return true;
3258 }
3259 
skl_watermark_ipc_init(struct drm_i915_private * i915)3260 void skl_watermark_ipc_init(struct drm_i915_private *i915)
3261 {
3262 	if (!HAS_IPC(i915))
3263 		return;
3264 
3265 	i915->display.wm.ipc_enabled = skl_watermark_ipc_can_enable(i915);
3266 
3267 	skl_watermark_ipc_update(i915);
3268 }
3269 
3270 static void
adjust_wm_latency(struct drm_i915_private * i915,u16 wm[],int num_levels,int read_latency)3271 adjust_wm_latency(struct drm_i915_private *i915,
3272 		  u16 wm[], int num_levels, int read_latency)
3273 {
3274 	bool wm_lv_0_adjust_needed = i915->dram_info.wm_lv_0_adjust_needed;
3275 	int i, level;
3276 
3277 	/*
3278 	 * If a level n (n > 1) has a 0us latency, all levels m (m >= n)
3279 	 * need to be disabled. We make sure to sanitize the values out
3280 	 * of the punit to satisfy this requirement.
3281 	 */
3282 	for (level = 1; level < num_levels; level++) {
3283 		if (wm[level] == 0) {
3284 			for (i = level + 1; i < num_levels; i++)
3285 				wm[i] = 0;
3286 
3287 			num_levels = level;
3288 			break;
3289 		}
3290 	}
3291 
3292 	/*
3293 	 * WaWmMemoryReadLatency
3294 	 *
3295 	 * punit doesn't take into account the read latency so we need
3296 	 * to add proper adjustment to each valid level we retrieve
3297 	 * from the punit when level 0 response data is 0us.
3298 	 */
3299 	if (wm[0] == 0) {
3300 		for (level = 0; level < num_levels; level++)
3301 			wm[level] += read_latency;
3302 	}
3303 
3304 	/*
3305 	 * WA Level-0 adjustment for 16GB DIMMs: SKL+
3306 	 * If we could not get dimm info enable this WA to prevent from
3307 	 * any underrun. If not able to get Dimm info assume 16GB dimm
3308 	 * to avoid any underrun.
3309 	 */
3310 	if (wm_lv_0_adjust_needed)
3311 		wm[0] += 1;
3312 }
3313 
mtl_read_wm_latency(struct drm_i915_private * i915,u16 wm[])3314 static void mtl_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
3315 {
3316 	int num_levels = i915->display.wm.num_levels;
3317 	u32 val;
3318 
3319 	val = intel_de_read(i915, MTL_LATENCY_LP0_LP1);
3320 	wm[0] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
3321 	wm[1] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
3322 
3323 	val = intel_de_read(i915, MTL_LATENCY_LP2_LP3);
3324 	wm[2] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
3325 	wm[3] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
3326 
3327 	val = intel_de_read(i915, MTL_LATENCY_LP4_LP5);
3328 	wm[4] = REG_FIELD_GET(MTL_LATENCY_LEVEL_EVEN_MASK, val);
3329 	wm[5] = REG_FIELD_GET(MTL_LATENCY_LEVEL_ODD_MASK, val);
3330 
3331 	adjust_wm_latency(i915, wm, num_levels, 6);
3332 }
3333 
skl_read_wm_latency(struct drm_i915_private * i915,u16 wm[])3334 static void skl_read_wm_latency(struct drm_i915_private *i915, u16 wm[])
3335 {
3336 	int num_levels = i915->display.wm.num_levels;
3337 	int read_latency = DISPLAY_VER(i915) >= 12 ? 3 : 2;
3338 	int mult = IS_DG2(i915) ? 2 : 1;
3339 	u32 val;
3340 	int ret;
3341 
3342 	/* read the first set of memory latencies[0:3] */
3343 	val = 0; /* data0 to be programmed to 0 for first set */
3344 	ret = snb_pcode_read(&i915->uncore, GEN9_PCODE_READ_MEM_LATENCY, &val, NULL);
3345 	if (ret) {
3346 		drm_err(&i915->drm, "SKL Mailbox read error = %d\n", ret);
3347 		return;
3348 	}
3349 
3350 	wm[0] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_0_4_MASK, val) * mult;
3351 	wm[1] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_1_5_MASK, val) * mult;
3352 	wm[2] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_2_6_MASK, val) * mult;
3353 	wm[3] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_3_7_MASK, val) * mult;
3354 
3355 	/* read the second set of memory latencies[4:7] */
3356 	val = 1; /* data0 to be programmed to 1 for second set */
3357 	ret = snb_pcode_read(&i915->uncore, GEN9_PCODE_READ_MEM_LATENCY, &val, NULL);
3358 	if (ret) {
3359 		drm_err(&i915->drm, "SKL Mailbox read error = %d\n", ret);
3360 		return;
3361 	}
3362 
3363 	wm[4] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_0_4_MASK, val) * mult;
3364 	wm[5] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_1_5_MASK, val) * mult;
3365 	wm[6] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_2_6_MASK, val) * mult;
3366 	wm[7] = REG_FIELD_GET(GEN9_MEM_LATENCY_LEVEL_3_7_MASK, val) * mult;
3367 
3368 	adjust_wm_latency(i915, wm, num_levels, read_latency);
3369 }
3370 
skl_setup_wm_latency(struct drm_i915_private * i915)3371 static void skl_setup_wm_latency(struct drm_i915_private *i915)
3372 {
3373 	struct intel_display *display = &i915->display;
3374 
3375 	if (HAS_HW_SAGV_WM(display))
3376 		display->wm.num_levels = 6;
3377 	else
3378 		display->wm.num_levels = 8;
3379 
3380 	if (DISPLAY_VER(display) >= 14)
3381 		mtl_read_wm_latency(i915, display->wm.skl_latency);
3382 	else
3383 		skl_read_wm_latency(i915, display->wm.skl_latency);
3384 
3385 	intel_print_wm_latency(i915, "Gen9 Plane", display->wm.skl_latency);
3386 }
3387 
intel_dbuf_duplicate_state(struct intel_global_obj * obj)3388 static struct intel_global_state *intel_dbuf_duplicate_state(struct intel_global_obj *obj)
3389 {
3390 	struct intel_dbuf_state *dbuf_state;
3391 
3392 	dbuf_state = kmemdup(obj->state, sizeof(*dbuf_state), GFP_KERNEL);
3393 	if (!dbuf_state)
3394 		return NULL;
3395 
3396 	return &dbuf_state->base;
3397 }
3398 
intel_dbuf_destroy_state(struct intel_global_obj * obj,struct intel_global_state * state)3399 static void intel_dbuf_destroy_state(struct intel_global_obj *obj,
3400 				     struct intel_global_state *state)
3401 {
3402 	kfree(state);
3403 }
3404 
3405 static const struct intel_global_state_funcs intel_dbuf_funcs = {
3406 	.atomic_duplicate_state = intel_dbuf_duplicate_state,
3407 	.atomic_destroy_state = intel_dbuf_destroy_state,
3408 };
3409 
3410 struct intel_dbuf_state *
intel_atomic_get_dbuf_state(struct intel_atomic_state * state)3411 intel_atomic_get_dbuf_state(struct intel_atomic_state *state)
3412 {
3413 	struct drm_i915_private *i915 = to_i915(state->base.dev);
3414 	struct intel_global_state *dbuf_state;
3415 
3416 	dbuf_state = intel_atomic_get_global_obj_state(state, &i915->display.dbuf.obj);
3417 	if (IS_ERR(dbuf_state))
3418 		return ERR_CAST(dbuf_state);
3419 
3420 	return to_intel_dbuf_state(dbuf_state);
3421 }
3422 
intel_dbuf_init(struct drm_i915_private * i915)3423 int intel_dbuf_init(struct drm_i915_private *i915)
3424 {
3425 	struct intel_display *display = &i915->display;
3426 	struct intel_dbuf_state *dbuf_state;
3427 
3428 	dbuf_state = kzalloc(sizeof(*dbuf_state), GFP_KERNEL);
3429 	if (!dbuf_state)
3430 		return -ENOMEM;
3431 
3432 	intel_atomic_global_obj_init(display, &display->dbuf.obj,
3433 				     &dbuf_state->base, &intel_dbuf_funcs);
3434 
3435 	return 0;
3436 }
3437 
xelpdp_is_only_pipe_per_dbuf_bank(enum pipe pipe,u8 active_pipes)3438 static bool xelpdp_is_only_pipe_per_dbuf_bank(enum pipe pipe, u8 active_pipes)
3439 {
3440 	switch (pipe) {
3441 	case PIPE_A:
3442 	case PIPE_D:
3443 		active_pipes &= BIT(PIPE_A) | BIT(PIPE_D);
3444 		break;
3445 	case PIPE_B:
3446 	case PIPE_C:
3447 		active_pipes &= BIT(PIPE_B) | BIT(PIPE_C);
3448 		break;
3449 	default: /* to suppress compiler warning */
3450 		MISSING_CASE(pipe);
3451 		return false;
3452 	}
3453 
3454 	return is_power_of_2(active_pipes);
3455 }
3456 
pipe_mbus_dbox_ctl(const struct intel_crtc * crtc,const struct intel_dbuf_state * dbuf_state)3457 static u32 pipe_mbus_dbox_ctl(const struct intel_crtc *crtc,
3458 			      const struct intel_dbuf_state *dbuf_state)
3459 {
3460 	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
3461 	u32 val = 0;
3462 
3463 	if (DISPLAY_VER(i915) >= 14)
3464 		val |= MBUS_DBOX_I_CREDIT(2);
3465 
3466 	if (DISPLAY_VER(i915) >= 12) {
3467 		val |= MBUS_DBOX_B2B_TRANSACTIONS_MAX(16);
3468 		val |= MBUS_DBOX_B2B_TRANSACTIONS_DELAY(1);
3469 		val |= MBUS_DBOX_REGULATE_B2B_TRANSACTIONS_EN;
3470 	}
3471 
3472 	if (DISPLAY_VER(i915) >= 14)
3473 		val |= dbuf_state->joined_mbus ?
3474 			MBUS_DBOX_A_CREDIT(12) : MBUS_DBOX_A_CREDIT(8);
3475 	else if (IS_ALDERLAKE_P(i915))
3476 		/* Wa_22010947358:adl-p */
3477 		val |= dbuf_state->joined_mbus ?
3478 			MBUS_DBOX_A_CREDIT(6) : MBUS_DBOX_A_CREDIT(4);
3479 	else
3480 		val |= MBUS_DBOX_A_CREDIT(2);
3481 
3482 	if (DISPLAY_VER(i915) >= 14) {
3483 		val |= MBUS_DBOX_B_CREDIT(0xA);
3484 	} else if (IS_ALDERLAKE_P(i915)) {
3485 		val |= MBUS_DBOX_BW_CREDIT(2);
3486 		val |= MBUS_DBOX_B_CREDIT(8);
3487 	} else if (DISPLAY_VER(i915) >= 12) {
3488 		val |= MBUS_DBOX_BW_CREDIT(2);
3489 		val |= MBUS_DBOX_B_CREDIT(12);
3490 	} else {
3491 		val |= MBUS_DBOX_BW_CREDIT(1);
3492 		val |= MBUS_DBOX_B_CREDIT(8);
3493 	}
3494 
3495 	if (DISPLAY_VERx100(i915) == 1400) {
3496 		if (xelpdp_is_only_pipe_per_dbuf_bank(crtc->pipe, dbuf_state->active_pipes))
3497 			val |= MBUS_DBOX_BW_8CREDITS_MTL;
3498 		else
3499 			val |= MBUS_DBOX_BW_4CREDITS_MTL;
3500 	}
3501 
3502 	return val;
3503 }
3504 
pipe_mbus_dbox_ctl_update(struct drm_i915_private * i915,const struct intel_dbuf_state * dbuf_state)3505 static void pipe_mbus_dbox_ctl_update(struct drm_i915_private *i915,
3506 				      const struct intel_dbuf_state *dbuf_state)
3507 {
3508 	struct intel_crtc *crtc;
3509 
3510 	for_each_intel_crtc_in_pipe_mask(&i915->drm, crtc, dbuf_state->active_pipes)
3511 		intel_de_write(i915, PIPE_MBUS_DBOX_CTL(crtc->pipe),
3512 			       pipe_mbus_dbox_ctl(crtc, dbuf_state));
3513 }
3514 
intel_mbus_dbox_update(struct intel_atomic_state * state)3515 static void intel_mbus_dbox_update(struct intel_atomic_state *state)
3516 {
3517 	struct drm_i915_private *i915 = to_i915(state->base.dev);
3518 	const struct intel_dbuf_state *new_dbuf_state, *old_dbuf_state;
3519 
3520 	if (DISPLAY_VER(i915) < 11)
3521 		return;
3522 
3523 	new_dbuf_state = intel_atomic_get_new_dbuf_state(state);
3524 	old_dbuf_state = intel_atomic_get_old_dbuf_state(state);
3525 	if (!new_dbuf_state ||
3526 	    (new_dbuf_state->joined_mbus == old_dbuf_state->joined_mbus &&
3527 	     new_dbuf_state->active_pipes == old_dbuf_state->active_pipes))
3528 		return;
3529 
3530 	pipe_mbus_dbox_ctl_update(i915, new_dbuf_state);
3531 }
3532 
intel_dbuf_state_set_mdclk_cdclk_ratio(struct intel_atomic_state * state,int ratio)3533 int intel_dbuf_state_set_mdclk_cdclk_ratio(struct intel_atomic_state *state,
3534 					   int ratio)
3535 {
3536 	struct intel_dbuf_state *dbuf_state;
3537 
3538 	dbuf_state = intel_atomic_get_dbuf_state(state);
3539 	if (IS_ERR(dbuf_state))
3540 		return PTR_ERR(dbuf_state);
3541 
3542 	dbuf_state->mdclk_cdclk_ratio = ratio;
3543 
3544 	return intel_atomic_lock_global_state(&dbuf_state->base);
3545 }
3546 
intel_dbuf_mdclk_cdclk_ratio_update(struct drm_i915_private * i915,int ratio,bool joined_mbus)3547 void intel_dbuf_mdclk_cdclk_ratio_update(struct drm_i915_private *i915,
3548 					 int ratio, bool joined_mbus)
3549 {
3550 	struct intel_display *display = &i915->display;
3551 	enum dbuf_slice slice;
3552 
3553 	if (!HAS_MBUS_JOINING(display))
3554 		return;
3555 
3556 	if (DISPLAY_VER(display) >= 20)
3557 		intel_de_rmw(display, MBUS_CTL, MBUS_TRANSLATION_THROTTLE_MIN_MASK,
3558 			     MBUS_TRANSLATION_THROTTLE_MIN(ratio - 1));
3559 
3560 	if (joined_mbus)
3561 		ratio *= 2;
3562 
3563 	drm_dbg_kms(display->drm, "Updating dbuf ratio to %d (mbus joined: %s)\n",
3564 		    ratio, str_yes_no(joined_mbus));
3565 
3566 	for_each_dbuf_slice(display, slice)
3567 		intel_de_rmw(display, DBUF_CTL_S(slice),
3568 			     DBUF_MIN_TRACKER_STATE_SERVICE_MASK,
3569 			     DBUF_MIN_TRACKER_STATE_SERVICE(ratio - 1));
3570 }
3571 
intel_dbuf_mdclk_min_tracker_update(struct intel_atomic_state * state)3572 static void intel_dbuf_mdclk_min_tracker_update(struct intel_atomic_state *state)
3573 {
3574 	struct drm_i915_private *i915 = to_i915(state->base.dev);
3575 	const struct intel_dbuf_state *old_dbuf_state =
3576 		intel_atomic_get_old_dbuf_state(state);
3577 	const struct intel_dbuf_state *new_dbuf_state =
3578 		intel_atomic_get_new_dbuf_state(state);
3579 	int mdclk_cdclk_ratio;
3580 
3581 	if (intel_cdclk_is_decreasing_later(state)) {
3582 		/* cdclk/mdclk will be changed later by intel_set_cdclk_post_plane_update() */
3583 		mdclk_cdclk_ratio = old_dbuf_state->mdclk_cdclk_ratio;
3584 	} else {
3585 		/* cdclk/mdclk already changed by intel_set_cdclk_pre_plane_update() */
3586 		mdclk_cdclk_ratio = new_dbuf_state->mdclk_cdclk_ratio;
3587 	}
3588 
3589 	intel_dbuf_mdclk_cdclk_ratio_update(i915, mdclk_cdclk_ratio,
3590 					    new_dbuf_state->joined_mbus);
3591 }
3592 
intel_mbus_joined_pipe(struct intel_atomic_state * state,const struct intel_dbuf_state * dbuf_state)3593 static enum pipe intel_mbus_joined_pipe(struct intel_atomic_state *state,
3594 					const struct intel_dbuf_state *dbuf_state)
3595 {
3596 	struct intel_display *display = to_intel_display(state);
3597 	struct drm_i915_private *i915 = to_i915(state->base.dev);
3598 	enum pipe pipe = ffs(dbuf_state->active_pipes) - 1;
3599 	const struct intel_crtc_state *new_crtc_state;
3600 	struct intel_crtc *crtc;
3601 
3602 	drm_WARN_ON(&i915->drm, !dbuf_state->joined_mbus);
3603 	drm_WARN_ON(&i915->drm, !is_power_of_2(dbuf_state->active_pipes));
3604 
3605 	crtc = intel_crtc_for_pipe(display, pipe);
3606 	new_crtc_state = intel_atomic_get_new_crtc_state(state, crtc);
3607 
3608 	if (new_crtc_state && !intel_crtc_needs_modeset(new_crtc_state))
3609 		return pipe;
3610 	else
3611 		return INVALID_PIPE;
3612 }
3613 
mbus_ctl_join_update(struct drm_i915_private * i915,const struct intel_dbuf_state * dbuf_state,enum pipe pipe)3614 static void mbus_ctl_join_update(struct drm_i915_private *i915,
3615 				 const struct intel_dbuf_state *dbuf_state,
3616 				 enum pipe pipe)
3617 {
3618 	u32 mbus_ctl;
3619 
3620 	if (dbuf_state->joined_mbus)
3621 		mbus_ctl = MBUS_HASHING_MODE_1x4 | MBUS_JOIN;
3622 	else
3623 		mbus_ctl = MBUS_HASHING_MODE_2x2;
3624 
3625 	if (pipe != INVALID_PIPE)
3626 		mbus_ctl |= MBUS_JOIN_PIPE_SELECT(pipe);
3627 	else
3628 		mbus_ctl |= MBUS_JOIN_PIPE_SELECT_NONE;
3629 
3630 	intel_de_rmw(i915, MBUS_CTL,
3631 		     MBUS_HASHING_MODE_MASK | MBUS_JOIN |
3632 		     MBUS_JOIN_PIPE_SELECT_MASK, mbus_ctl);
3633 }
3634 
intel_dbuf_mbus_join_update(struct intel_atomic_state * state,enum pipe pipe)3635 static void intel_dbuf_mbus_join_update(struct intel_atomic_state *state,
3636 					enum pipe pipe)
3637 {
3638 	struct drm_i915_private *i915 = to_i915(state->base.dev);
3639 	const struct intel_dbuf_state *old_dbuf_state =
3640 		intel_atomic_get_old_dbuf_state(state);
3641 	const struct intel_dbuf_state *new_dbuf_state =
3642 		intel_atomic_get_new_dbuf_state(state);
3643 
3644 	drm_dbg_kms(&i915->drm, "Changing mbus joined: %s -> %s (pipe: %c)\n",
3645 		    str_yes_no(old_dbuf_state->joined_mbus),
3646 		    str_yes_no(new_dbuf_state->joined_mbus),
3647 		    pipe != INVALID_PIPE ? pipe_name(pipe) : '*');
3648 
3649 	mbus_ctl_join_update(i915, new_dbuf_state, pipe);
3650 }
3651 
intel_dbuf_mbus_pre_ddb_update(struct intel_atomic_state * state)3652 void intel_dbuf_mbus_pre_ddb_update(struct intel_atomic_state *state)
3653 {
3654 	const struct intel_dbuf_state *new_dbuf_state =
3655 		intel_atomic_get_new_dbuf_state(state);
3656 	const struct intel_dbuf_state *old_dbuf_state =
3657 		intel_atomic_get_old_dbuf_state(state);
3658 
3659 	if (!new_dbuf_state)
3660 		return;
3661 
3662 	if (!old_dbuf_state->joined_mbus && new_dbuf_state->joined_mbus) {
3663 		enum pipe pipe = intel_mbus_joined_pipe(state, new_dbuf_state);
3664 
3665 		WARN_ON(!new_dbuf_state->base.changed);
3666 
3667 		intel_dbuf_mbus_join_update(state, pipe);
3668 		intel_mbus_dbox_update(state);
3669 		intel_dbuf_mdclk_min_tracker_update(state);
3670 	}
3671 }
3672 
intel_dbuf_mbus_post_ddb_update(struct intel_atomic_state * state)3673 void intel_dbuf_mbus_post_ddb_update(struct intel_atomic_state *state)
3674 {
3675 	struct intel_display *display = to_intel_display(state);
3676 	const struct intel_dbuf_state *new_dbuf_state =
3677 		intel_atomic_get_new_dbuf_state(state);
3678 	const struct intel_dbuf_state *old_dbuf_state =
3679 		intel_atomic_get_old_dbuf_state(state);
3680 
3681 	if (!new_dbuf_state)
3682 		return;
3683 
3684 	if (old_dbuf_state->joined_mbus && !new_dbuf_state->joined_mbus) {
3685 		enum pipe pipe = intel_mbus_joined_pipe(state, old_dbuf_state);
3686 
3687 		WARN_ON(!new_dbuf_state->base.changed);
3688 
3689 		intel_dbuf_mdclk_min_tracker_update(state);
3690 		intel_mbus_dbox_update(state);
3691 		intel_dbuf_mbus_join_update(state, pipe);
3692 
3693 		if (pipe != INVALID_PIPE) {
3694 			struct intel_crtc *crtc = intel_crtc_for_pipe(display, pipe);
3695 
3696 			intel_crtc_wait_for_next_vblank(crtc);
3697 		}
3698 	} else if (old_dbuf_state->joined_mbus == new_dbuf_state->joined_mbus &&
3699 		   old_dbuf_state->active_pipes != new_dbuf_state->active_pipes) {
3700 		WARN_ON(!new_dbuf_state->base.changed);
3701 
3702 		intel_dbuf_mdclk_min_tracker_update(state);
3703 		intel_mbus_dbox_update(state);
3704 	}
3705 
3706 }
3707 
intel_dbuf_pre_plane_update(struct intel_atomic_state * state)3708 void intel_dbuf_pre_plane_update(struct intel_atomic_state *state)
3709 {
3710 	struct intel_display *display = to_intel_display(state);
3711 	const struct intel_dbuf_state *new_dbuf_state =
3712 		intel_atomic_get_new_dbuf_state(state);
3713 	const struct intel_dbuf_state *old_dbuf_state =
3714 		intel_atomic_get_old_dbuf_state(state);
3715 	u8 old_slices, new_slices;
3716 
3717 	if (!new_dbuf_state)
3718 		return;
3719 
3720 	old_slices = old_dbuf_state->enabled_slices;
3721 	new_slices = old_dbuf_state->enabled_slices | new_dbuf_state->enabled_slices;
3722 
3723 	if (old_slices == new_slices)
3724 		return;
3725 
3726 	WARN_ON(!new_dbuf_state->base.changed);
3727 
3728 	gen9_dbuf_slices_update(display, new_slices);
3729 }
3730 
intel_dbuf_post_plane_update(struct intel_atomic_state * state)3731 void intel_dbuf_post_plane_update(struct intel_atomic_state *state)
3732 {
3733 	struct intel_display *display = to_intel_display(state);
3734 	const struct intel_dbuf_state *new_dbuf_state =
3735 		intel_atomic_get_new_dbuf_state(state);
3736 	const struct intel_dbuf_state *old_dbuf_state =
3737 		intel_atomic_get_old_dbuf_state(state);
3738 	u8 old_slices, new_slices;
3739 
3740 	if (!new_dbuf_state)
3741 		return;
3742 
3743 	old_slices = old_dbuf_state->enabled_slices | new_dbuf_state->enabled_slices;
3744 	new_slices = new_dbuf_state->enabled_slices;
3745 
3746 	if (old_slices == new_slices)
3747 		return;
3748 
3749 	WARN_ON(!new_dbuf_state->base.changed);
3750 
3751 	gen9_dbuf_slices_update(display, new_slices);
3752 }
3753 
skl_mbus_sanitize(struct drm_i915_private * i915)3754 static void skl_mbus_sanitize(struct drm_i915_private *i915)
3755 {
3756 	struct intel_display *display = &i915->display;
3757 	struct intel_dbuf_state *dbuf_state =
3758 		to_intel_dbuf_state(display->dbuf.obj.state);
3759 
3760 	if (!HAS_MBUS_JOINING(display))
3761 		return;
3762 
3763 	if (!dbuf_state->joined_mbus ||
3764 	    adlp_check_mbus_joined(dbuf_state->active_pipes))
3765 		return;
3766 
3767 	drm_dbg_kms(display->drm, "Disabling redundant MBUS joining (active pipes 0x%x)\n",
3768 		    dbuf_state->active_pipes);
3769 
3770 	dbuf_state->joined_mbus = false;
3771 	intel_dbuf_mdclk_cdclk_ratio_update(i915,
3772 					    dbuf_state->mdclk_cdclk_ratio,
3773 					    dbuf_state->joined_mbus);
3774 	pipe_mbus_dbox_ctl_update(i915, dbuf_state);
3775 	mbus_ctl_join_update(i915, dbuf_state, INVALID_PIPE);
3776 }
3777 
skl_dbuf_is_misconfigured(struct drm_i915_private * i915)3778 static bool skl_dbuf_is_misconfigured(struct drm_i915_private *i915)
3779 {
3780 	const struct intel_dbuf_state *dbuf_state =
3781 		to_intel_dbuf_state(i915->display.dbuf.obj.state);
3782 	struct skl_ddb_entry entries[I915_MAX_PIPES] = {};
3783 	struct intel_crtc *crtc;
3784 
3785 	for_each_intel_crtc(&i915->drm, crtc) {
3786 		const struct intel_crtc_state *crtc_state =
3787 			to_intel_crtc_state(crtc->base.state);
3788 
3789 		entries[crtc->pipe] = crtc_state->wm.skl.ddb;
3790 	}
3791 
3792 	for_each_intel_crtc(&i915->drm, crtc) {
3793 		const struct intel_crtc_state *crtc_state =
3794 			to_intel_crtc_state(crtc->base.state);
3795 		u8 slices;
3796 
3797 		slices = skl_compute_dbuf_slices(crtc, dbuf_state->active_pipes,
3798 						 dbuf_state->joined_mbus);
3799 		if (dbuf_state->slices[crtc->pipe] & ~slices)
3800 			return true;
3801 
3802 		if (skl_ddb_allocation_overlaps(&crtc_state->wm.skl.ddb, entries,
3803 						I915_MAX_PIPES, crtc->pipe))
3804 			return true;
3805 	}
3806 
3807 	return false;
3808 }
3809 
skl_dbuf_sanitize(struct drm_i915_private * i915)3810 static void skl_dbuf_sanitize(struct drm_i915_private *i915)
3811 {
3812 	struct intel_crtc *crtc;
3813 
3814 	/*
3815 	 * On TGL/RKL (at least) the BIOS likes to assign the planes
3816 	 * to the wrong DBUF slices. This will cause an infinite loop
3817 	 * in skl_commit_modeset_enables() as it can't find a way to
3818 	 * transition between the old bogus DBUF layout to the new
3819 	 * proper DBUF layout without DBUF allocation overlaps between
3820 	 * the planes (which cannot be allowed or else the hardware
3821 	 * may hang). If we detect a bogus DBUF layout just turn off
3822 	 * all the planes so that skl_commit_modeset_enables() can
3823 	 * simply ignore them.
3824 	 */
3825 	if (!skl_dbuf_is_misconfigured(i915))
3826 		return;
3827 
3828 	drm_dbg_kms(&i915->drm, "BIOS has misprogrammed the DBUF, disabling all planes\n");
3829 
3830 	for_each_intel_crtc(&i915->drm, crtc) {
3831 		struct intel_plane *plane = to_intel_plane(crtc->base.primary);
3832 		const struct intel_plane_state *plane_state =
3833 			to_intel_plane_state(plane->base.state);
3834 		struct intel_crtc_state *crtc_state =
3835 			to_intel_crtc_state(crtc->base.state);
3836 
3837 		if (plane_state->uapi.visible)
3838 			intel_plane_disable_noatomic(crtc, plane);
3839 
3840 		drm_WARN_ON(&i915->drm, crtc_state->active_planes != 0);
3841 
3842 		memset(&crtc_state->wm.skl.ddb, 0, sizeof(crtc_state->wm.skl.ddb));
3843 	}
3844 }
3845 
skl_wm_sanitize(struct drm_i915_private * i915)3846 static void skl_wm_sanitize(struct drm_i915_private *i915)
3847 {
3848 	skl_mbus_sanitize(i915);
3849 	skl_dbuf_sanitize(i915);
3850 }
3851 
skl_wm_crtc_disable_noatomic(struct intel_crtc * crtc)3852 void skl_wm_crtc_disable_noatomic(struct intel_crtc *crtc)
3853 {
3854 	struct intel_display *display = to_intel_display(crtc);
3855 	struct intel_crtc_state *crtc_state =
3856 		to_intel_crtc_state(crtc->base.state);
3857 	struct intel_dbuf_state *dbuf_state =
3858 		to_intel_dbuf_state(display->dbuf.obj.state);
3859 	enum pipe pipe = crtc->pipe;
3860 
3861 	if (DISPLAY_VER(display) < 9)
3862 		return;
3863 
3864 	dbuf_state->active_pipes &= ~BIT(pipe);
3865 
3866 	dbuf_state->weight[pipe] = 0;
3867 	dbuf_state->slices[pipe] = 0;
3868 
3869 	memset(&dbuf_state->ddb[pipe], 0, sizeof(dbuf_state->ddb[pipe]));
3870 
3871 	memset(&crtc_state->wm.skl.ddb, 0, sizeof(crtc_state->wm.skl.ddb));
3872 }
3873 
skl_wm_plane_disable_noatomic(struct intel_crtc * crtc,struct intel_plane * plane)3874 void skl_wm_plane_disable_noatomic(struct intel_crtc *crtc,
3875 				   struct intel_plane *plane)
3876 {
3877 	struct intel_display *display = to_intel_display(crtc);
3878 	struct intel_crtc_state *crtc_state =
3879 		to_intel_crtc_state(crtc->base.state);
3880 
3881 	if (DISPLAY_VER(display) < 9)
3882 		return;
3883 
3884 	skl_ddb_entry_init(&crtc_state->wm.skl.plane_ddb[plane->id], 0, 0);
3885 	skl_ddb_entry_init(&crtc_state->wm.skl.plane_ddb[plane->id], 0, 0);
3886 
3887 	crtc_state->wm.skl.plane_min_ddb[plane->id] = 0;
3888 	crtc_state->wm.skl.plane_interim_ddb[plane->id] = 0;
3889 
3890 	memset(&crtc_state->wm.skl.raw.planes[plane->id], 0,
3891 	       sizeof(crtc_state->wm.skl.raw.planes[plane->id]));
3892 	memset(&crtc_state->wm.skl.optimal.planes[plane->id], 0,
3893 	       sizeof(crtc_state->wm.skl.optimal.planes[plane->id]));
3894 }
3895 
intel_wm_state_verify(struct intel_atomic_state * state,struct intel_crtc * crtc)3896 void intel_wm_state_verify(struct intel_atomic_state *state,
3897 			   struct intel_crtc *crtc)
3898 {
3899 	struct intel_display *display = to_intel_display(state);
3900 	struct drm_i915_private *i915 = to_i915(state->base.dev);
3901 	const struct intel_crtc_state *new_crtc_state =
3902 		intel_atomic_get_new_crtc_state(state, crtc);
3903 	struct skl_hw_state {
3904 		struct skl_ddb_entry ddb[I915_MAX_PLANES];
3905 		struct skl_ddb_entry ddb_y[I915_MAX_PLANES];
3906 		u16 min_ddb[I915_MAX_PLANES];
3907 		u16 interim_ddb[I915_MAX_PLANES];
3908 		struct skl_pipe_wm wm;
3909 	} *hw;
3910 	const struct skl_pipe_wm *sw_wm = &new_crtc_state->wm.skl.optimal;
3911 	struct intel_plane *plane;
3912 	u8 hw_enabled_slices;
3913 	int level;
3914 
3915 	if (DISPLAY_VER(i915) < 9 || !new_crtc_state->hw.active)
3916 		return;
3917 
3918 	hw = kzalloc(sizeof(*hw), GFP_KERNEL);
3919 	if (!hw)
3920 		return;
3921 
3922 	skl_pipe_wm_get_hw_state(crtc, &hw->wm);
3923 
3924 	skl_pipe_ddb_get_hw_state(crtc, hw->ddb, hw->ddb_y, hw->min_ddb, hw->interim_ddb);
3925 
3926 	hw_enabled_slices = intel_enabled_dbuf_slices_mask(display);
3927 
3928 	if (DISPLAY_VER(i915) >= 11 &&
3929 	    hw_enabled_slices != i915->display.dbuf.enabled_slices)
3930 		drm_err(&i915->drm,
3931 			"mismatch in DBUF Slices (expected 0x%x, got 0x%x)\n",
3932 			i915->display.dbuf.enabled_slices,
3933 			hw_enabled_slices);
3934 
3935 	for_each_intel_plane_on_crtc(&i915->drm, crtc, plane) {
3936 		const struct skl_ddb_entry *hw_ddb_entry, *sw_ddb_entry;
3937 		const struct skl_wm_level *hw_wm_level, *sw_wm_level;
3938 
3939 		/* Watermarks */
3940 		for (level = 0; level < i915->display.wm.num_levels; level++) {
3941 			hw_wm_level = &hw->wm.planes[plane->id].wm[level];
3942 			sw_wm_level = skl_plane_wm_level(sw_wm, plane->id, level);
3943 
3944 			if (skl_wm_level_equals(hw_wm_level, sw_wm_level))
3945 				continue;
3946 
3947 			drm_err(&i915->drm,
3948 				"[PLANE:%d:%s] mismatch in WM%d (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3949 				plane->base.base.id, plane->base.name, level,
3950 				sw_wm_level->enable,
3951 				sw_wm_level->blocks,
3952 				sw_wm_level->lines,
3953 				hw_wm_level->enable,
3954 				hw_wm_level->blocks,
3955 				hw_wm_level->lines);
3956 		}
3957 
3958 		hw_wm_level = &hw->wm.planes[plane->id].trans_wm;
3959 		sw_wm_level = skl_plane_trans_wm(sw_wm, plane->id);
3960 
3961 		if (!skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
3962 			drm_err(&i915->drm,
3963 				"[PLANE:%d:%s] mismatch in trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3964 				plane->base.base.id, plane->base.name,
3965 				sw_wm_level->enable,
3966 				sw_wm_level->blocks,
3967 				sw_wm_level->lines,
3968 				hw_wm_level->enable,
3969 				hw_wm_level->blocks,
3970 				hw_wm_level->lines);
3971 		}
3972 
3973 		hw_wm_level = &hw->wm.planes[plane->id].sagv.wm0;
3974 		sw_wm_level = &sw_wm->planes[plane->id].sagv.wm0;
3975 
3976 		if (HAS_HW_SAGV_WM(display) &&
3977 		    !skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
3978 			drm_err(&i915->drm,
3979 				"[PLANE:%d:%s] mismatch in SAGV WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3980 				plane->base.base.id, plane->base.name,
3981 				sw_wm_level->enable,
3982 				sw_wm_level->blocks,
3983 				sw_wm_level->lines,
3984 				hw_wm_level->enable,
3985 				hw_wm_level->blocks,
3986 				hw_wm_level->lines);
3987 		}
3988 
3989 		hw_wm_level = &hw->wm.planes[plane->id].sagv.trans_wm;
3990 		sw_wm_level = &sw_wm->planes[plane->id].sagv.trans_wm;
3991 
3992 		if (HAS_HW_SAGV_WM(display) &&
3993 		    !skl_wm_level_equals(hw_wm_level, sw_wm_level)) {
3994 			drm_err(&i915->drm,
3995 				"[PLANE:%d:%s] mismatch in SAGV trans WM (expected e=%d b=%u l=%u, got e=%d b=%u l=%u)\n",
3996 				plane->base.base.id, plane->base.name,
3997 				sw_wm_level->enable,
3998 				sw_wm_level->blocks,
3999 				sw_wm_level->lines,
4000 				hw_wm_level->enable,
4001 				hw_wm_level->blocks,
4002 				hw_wm_level->lines);
4003 		}
4004 
4005 		/* DDB */
4006 		hw_ddb_entry = &hw->ddb[PLANE_CURSOR];
4007 		sw_ddb_entry = &new_crtc_state->wm.skl.plane_ddb[PLANE_CURSOR];
4008 
4009 		if (!skl_ddb_entry_equal(hw_ddb_entry, sw_ddb_entry)) {
4010 			drm_err(&i915->drm,
4011 				"[PLANE:%d:%s] mismatch in DDB (expected (%u,%u), found (%u,%u))\n",
4012 				plane->base.base.id, plane->base.name,
4013 				sw_ddb_entry->start, sw_ddb_entry->end,
4014 				hw_ddb_entry->start, hw_ddb_entry->end);
4015 		}
4016 	}
4017 
4018 	kfree(hw);
4019 }
4020 
4021 static const struct intel_wm_funcs skl_wm_funcs = {
4022 	.compute_global_watermarks = skl_compute_wm,
4023 	.get_hw_state = skl_wm_get_hw_state,
4024 	.sanitize = skl_wm_sanitize,
4025 };
4026 
skl_wm_init(struct drm_i915_private * i915)4027 void skl_wm_init(struct drm_i915_private *i915)
4028 {
4029 	intel_sagv_init(i915);
4030 
4031 	skl_setup_wm_latency(i915);
4032 
4033 	i915->display.funcs.wm = &skl_wm_funcs;
4034 }
4035 
skl_watermark_ipc_status_show(struct seq_file * m,void * data)4036 static int skl_watermark_ipc_status_show(struct seq_file *m, void *data)
4037 {
4038 	struct drm_i915_private *i915 = m->private;
4039 
4040 	seq_printf(m, "Isochronous Priority Control: %s\n",
4041 		   str_yes_no(skl_watermark_ipc_enabled(i915)));
4042 	return 0;
4043 }
4044 
skl_watermark_ipc_status_open(struct inode * inode,struct file * file)4045 static int skl_watermark_ipc_status_open(struct inode *inode, struct file *file)
4046 {
4047 	struct drm_i915_private *i915 = inode->i_private;
4048 
4049 	return single_open(file, skl_watermark_ipc_status_show, i915);
4050 }
4051 
skl_watermark_ipc_status_write(struct file * file,const char __user * ubuf,size_t len,loff_t * offp)4052 static ssize_t skl_watermark_ipc_status_write(struct file *file,
4053 					      const char __user *ubuf,
4054 					      size_t len, loff_t *offp)
4055 {
4056 	struct seq_file *m = file->private_data;
4057 	struct drm_i915_private *i915 = m->private;
4058 	intel_wakeref_t wakeref;
4059 	bool enable;
4060 	int ret;
4061 
4062 	ret = kstrtobool_from_user(ubuf, len, &enable);
4063 	if (ret < 0)
4064 		return ret;
4065 
4066 	with_intel_runtime_pm(&i915->runtime_pm, wakeref) {
4067 		if (!skl_watermark_ipc_enabled(i915) && enable)
4068 			drm_info(&i915->drm,
4069 				 "Enabling IPC: WM will be proper only after next commit\n");
4070 		i915->display.wm.ipc_enabled = enable;
4071 		skl_watermark_ipc_update(i915);
4072 	}
4073 
4074 	return len;
4075 }
4076 
4077 static const struct file_operations skl_watermark_ipc_status_fops = {
4078 	.owner = THIS_MODULE,
4079 	.open = skl_watermark_ipc_status_open,
4080 	.read = seq_read,
4081 	.llseek = seq_lseek,
4082 	.release = single_release,
4083 	.write = skl_watermark_ipc_status_write
4084 };
4085 
intel_sagv_status_show(struct seq_file * m,void * unused)4086 static int intel_sagv_status_show(struct seq_file *m, void *unused)
4087 {
4088 	struct drm_i915_private *i915 = m->private;
4089 	static const char * const sagv_status[] = {
4090 		[I915_SAGV_UNKNOWN] = "unknown",
4091 		[I915_SAGV_DISABLED] = "disabled",
4092 		[I915_SAGV_ENABLED] = "enabled",
4093 		[I915_SAGV_NOT_CONTROLLED] = "not controlled",
4094 	};
4095 
4096 	seq_printf(m, "SAGV available: %s\n", str_yes_no(intel_has_sagv(i915)));
4097 	seq_printf(m, "SAGV modparam: %s\n",
4098 		   str_enabled_disabled(i915->display.params.enable_sagv));
4099 	seq_printf(m, "SAGV status: %s\n", sagv_status[i915->display.sagv.status]);
4100 	seq_printf(m, "SAGV block time: %d usec\n", i915->display.sagv.block_time_us);
4101 
4102 	return 0;
4103 }
4104 
4105 DEFINE_SHOW_ATTRIBUTE(intel_sagv_status);
4106 
skl_watermark_debugfs_register(struct drm_i915_private * i915)4107 void skl_watermark_debugfs_register(struct drm_i915_private *i915)
4108 {
4109 	struct intel_display *display = &i915->display;
4110 	struct drm_minor *minor = display->drm->primary;
4111 
4112 	if (HAS_IPC(display))
4113 		debugfs_create_file("i915_ipc_status", 0644, minor->debugfs_root, i915,
4114 				    &skl_watermark_ipc_status_fops);
4115 
4116 	if (HAS_SAGV(display))
4117 		debugfs_create_file("i915_sagv_status", 0444, minor->debugfs_root, i915,
4118 				    &intel_sagv_status_fops);
4119 }
4120 
skl_watermark_max_latency(struct drm_i915_private * i915,int initial_wm_level)4121 unsigned int skl_watermark_max_latency(struct drm_i915_private *i915, int initial_wm_level)
4122 {
4123 	int level;
4124 
4125 	for (level = i915->display.wm.num_levels - 1; level >= initial_wm_level; level--) {
4126 		unsigned int latency = skl_wm_latency(i915, level, NULL);
4127 
4128 		if (latency)
4129 			return latency;
4130 	}
4131 
4132 	return 0;
4133 }
4134