1 // SPDX-License-Identifier: MIT
47 	{1, 0, 800, 400, 3382, 33382, 3983, 33983},    /* DDR2-400 SC */
48 	{1, 0, 800, 667, 3354, 33354, 3807, 33807},    /* DDR2-667 SC */
49 	{1, 0, 800, 800, 3347, 33347, 3763, 33763},    /* DDR2-800 SC */
50 	{1, 1, 800, 667, 6420, 36420, 6873, 36873},    /* DDR3-667 SC */
51 	{1, 1, 800, 800, 5902, 35902, 6318, 36318},    /* DDR3-800 SC */
53 	{1, 0, 667, 400, 3400, 33400, 4021, 34021},    /* DDR2-400 SC */
54 	{1, 0, 667, 667, 3372, 33372, 3845, 33845},    /* DDR2-667 SC */
55 	{1, 0, 667, 800, 3386, 33386, 3822, 33822},    /* DDR2-800 SC */
56 	{1, 1, 667, 667, 6438, 36438, 6911, 36911},    /* DDR3-667 SC */
57 	{1, 1, 667, 800, 5941, 35941, 6377, 36377},    /* DDR3-800 SC */
59 	{1, 0, 400, 400, 3472, 33472, 4173, 34173},    /* DDR2-400 SC */
60 	{1, 0, 400, 667, 3443, 33443, 3996, 33996},    /* DDR2-667 SC */
61 	{1, 0, 400, 800, 3430, 33430, 3946, 33946},    /* DDR2-800 SC */
62 	{1, 1, 400, 667, 6509, 36509, 7062, 37062},    /* DDR3-667 SC */
63 	{1, 1, 400, 800, 5985, 35985, 6501, 36501},    /* DDR3-800 SC */
65 	{0, 0, 800, 400, 3438, 33438, 4065, 34065},    /* DDR2-400 SC */
66 	{0, 0, 800, 667, 3410, 33410, 3889, 33889},    /* DDR2-667 SC */
67 	{0, 0, 800, 800, 3403, 33403, 3845, 33845},    /* DDR2-800 SC */
68 	{0, 1, 800, 667, 6476, 36476, 6955, 36955},    /* DDR3-667 SC */
69 	{0, 1, 800, 800, 5958, 35958, 6400, 36400},    /* DDR3-800 SC */
71 	{0, 0, 667, 400, 3456, 33456, 4103, 34106},    /* DDR2-400 SC */
72 	{0, 0, 667, 667, 3428, 33428, 3927, 33927},    /* DDR2-667 SC */
73 	{0, 0, 667, 800, 3443, 33443, 3905, 33905},    /* DDR2-800 SC */
74 	{0, 1, 667, 667, 6494, 36494, 6993, 36993},    /* DDR3-667 SC */
75 	{0, 1, 667, 800, 5998, 35998, 6460, 36460},    /* DDR3-800 SC */
77 	{0, 0, 400, 400, 3528, 33528, 4255, 34255},    /* DDR2-400 SC */
78 	{0, 0, 400, 667, 3500, 33500, 4079, 34079},    /* DDR2-667 SC */
79 	{0, 0, 400, 800, 3487, 33487, 4029, 34029},    /* DDR2-800 SC */
80 	{0, 1, 400, 667, 6566, 36566, 7145, 37145},    /* DDR3-667 SC */
81 	{0, 1, 400, 800, 6042, 36042, 6584, 36584},    /* DDR3-800 SC */
92 		if (is_desktop == latency->is_desktop &&  in pnv_get_cxsr_latency()
93 		    i915->is_ddr3 == latency->is_ddr3 &&  in pnv_get_cxsr_latency()
94 		    DIV_ROUND_CLOSEST(i915->fsb_freq, 1000) == latency->fsb_freq &&  in pnv_get_cxsr_latency()
95 		    DIV_ROUND_CLOSEST(i915->mem_freq, 1000) == latency->mem_freq)  in pnv_get_cxsr_latency()
99 	drm_dbg_kms(&i915->drm,  in pnv_get_cxsr_latency()
101 		    i915->is_ddr3 ? "3" : "2", i915->fsb_freq, i915->mem_freq);  in pnv_get_cxsr_latency()
123 		drm_err(&dev_priv->drm,  in chv_set_memory_dvfs()
150 	struct intel_display *display = &dev_priv->display;  in _intel_set_memory_cxsr()
155 		was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;  in _intel_set_memory_cxsr()
156 		intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF_VLV, enable ? FW_CSPWRDWNEN : 0);  in _intel_set_memory_cxsr()
157 		intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF_VLV);  in _intel_set_memory_cxsr()
159 		was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;  in _intel_set_memory_cxsr()
160 		intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, enable ? FW_BLC_SELF_EN : 0);  in _intel_set_memory_cxsr()
161 		intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);  in _intel_set_memory_cxsr()
163 		val = intel_uncore_read(&dev_priv->uncore, DSPFW3(dev_priv));  in _intel_set_memory_cxsr()
169 		intel_uncore_write(&dev_priv->uncore, DSPFW3(dev_priv), val);  in _intel_set_memory_cxsr()
170 		intel_uncore_posting_read(&dev_priv->uncore, DSPFW3(dev_priv));  in _intel_set_memory_cxsr()
172 		was_enabled = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;  in _intel_set_memory_cxsr()
175 		intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, val);  in _intel_set_memory_cxsr()
176 		intel_uncore_posting_read(&dev_priv->uncore, FW_BLC_SELF);  in _intel_set_memory_cxsr()
183 		was_enabled = intel_uncore_read(&dev_priv->uncore, INSTPM) & INSTPM_SELF_EN;  in _intel_set_memory_cxsr()
186 		intel_uncore_write(&dev_priv->uncore, INSTPM, val);  in _intel_set_memory_cxsr()
187 		intel_uncore_posting_read(&dev_priv->uncore, INSTPM);  in _intel_set_memory_cxsr()
194 	drm_dbg_kms(&dev_priv->drm, "memory self-refresh is %s (was %s)\n",  in _intel_set_memory_cxsr()
202  * intel_set_memory_cxsr - Configure CxSR state
207  * (C-state self refresh) state. What typically happens in CxSR mode
217  * C-states will affect when/if the system actually enters/exits the
228  * Unfortunately the system will re-evaluate the CxSR conditions at
242 	mutex_lock(&dev_priv->display.wm.wm_mutex);  in intel_set_memory_cxsr()
245 		dev_priv->display.wm.vlv.cxsr = enable;  in intel_set_memory_cxsr()
247 		dev_priv->display.wm.g4x.cxsr = enable;  in intel_set_memory_cxsr()
248 	mutex_unlock(&dev_priv->display.wm.wm_mutex);  in intel_set_memory_cxsr()
255  *   - memory configuration (speed, channels)
256  *   - chipset
257  *   - current MCH state
270 	((((dsparb) >> (lo_shift)) & 0xff) | ((((dsparb2) >> (hi_shift)) & 0x1) << 8))
274 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);  in vlv_get_fifo_size()
275 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);  in vlv_get_fifo_size()
276 	struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;  in vlv_get_fifo_size()
277 	enum pipe pipe = crtc->pipe;  in vlv_get_fifo_size()
283 		dsparb = intel_uncore_read(&dev_priv->uncore,  in vlv_get_fifo_size()
285 		dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);  in vlv_get_fifo_size()
287 		sprite1_start = VLV_FIFO_START(dsparb, dsparb2, 8, 4);  in vlv_get_fifo_size()
290 		dsparb = intel_uncore_read(&dev_priv->uncore,  in vlv_get_fifo_size()
292 		dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);  in vlv_get_fifo_size()
293 		sprite0_start = VLV_FIFO_START(dsparb, dsparb2, 16, 8);  in vlv_get_fifo_size()
297 		dsparb2 = intel_uncore_read(&dev_priv->uncore, DSPARB2);  in vlv_get_fifo_size()
298 		dsparb3 = intel_uncore_read(&dev_priv->uncore, DSPARB3);  in vlv_get_fifo_size()
300 		sprite1_start = VLV_FIFO_START(dsparb3, dsparb2, 8, 20);  in vlv_get_fifo_size()
307 	fifo_state->plane[PLANE_PRIMARY] = sprite0_start;  in vlv_get_fifo_size()
308 	fifo_state->plane[PLANE_SPRITE0] = sprite1_start - sprite0_start;  in vlv_get_fifo_size()
309 	fifo_state->plane[PLANE_SPRITE1] = 511 - sprite1_start;  in vlv_get_fifo_size()
310 	fifo_state->plane[PLANE_CURSOR] = 63;  in vlv_get_fifo_size()
316 	u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB(dev_priv));  in i9xx_get_fifo_size()
321 		size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;  in i9xx_get_fifo_size()
323 	drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",  in i9xx_get_fifo_size()
332 	u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB(dev_priv));  in i830_get_fifo_size()
337 		size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;  in i830_get_fifo_size()
340 	drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",  in i830_get_fifo_size()
349 	u32 dsparb = intel_uncore_read(&dev_priv->uncore, DSPARB(dev_priv));  in i845_get_fifo_size()
355 	drm_dbg_kms(&dev_priv->drm, "FIFO size - (0x%08x) %c: %d\n",  in i845_get_fifo_size()
443  * intel_wm_method1 - Method 1 / "small buffer" watermark formula
459  * The FIFO level vs. time graph might look something like:
463  * __---__---__ (- plane active, _ blanking)
464  * -> time
469  * __----__----__ (- plane active, _ blanking)
470  * -> time
488  * intel_wm_method2 - Method 2 / "large buffer" watermark formula
506  * The FIFO level vs. time graph might look something like:
511  * __ --__--__--__--__--__--__ (- plane active, _ blanking)
512  * -> time
539  * intel_calculate_wm - calculate watermark level
547  * Calculate the watermark level (the level at which the display plane will
553  * on the pixel size.  When it reaches the watermark level, it'll start
574 	entries = DIV_ROUND_UP(entries, wm->cacheline_size) +  in intel_calculate_wm()
575 		wm->guard_size;  in intel_calculate_wm()
576 	drm_dbg_kms(&i915->drm, "FIFO entries required for mode: %d\n", entries);  in intel_calculate_wm()
578 	wm_size = fifo_size - entries;  in intel_calculate_wm()
579 	drm_dbg_kms(&i915->drm, "FIFO watermark level: %d\n", wm_size);  in intel_calculate_wm()
582 	if (wm_size > wm->max_wm)  in intel_calculate_wm()
583 		wm_size = wm->max_wm;  in intel_calculate_wm()
585 		wm_size = wm->default_wm;  in intel_calculate_wm()
590 	 * Lets go for 8 which is the burst size since certain platforms  in intel_calculate_wm()
591 	 * already use a hardcoded 8 (which is what the spec says should be  in intel_calculate_wm()
594 	if (wm_size <= 8)  in intel_calculate_wm()
595 		wm_size = 8;  in intel_calculate_wm()
618 	 * We can ditch the crtc->primary->state->fb check as soon as we can  in intel_crtc_active()
621 	 * FIXME: The intel_crtc->active here should be switched to  in intel_crtc_active()
622 	 * crtc->state->active once we have proper CRTC states wired up  in intel_crtc_active()
625 	return crtc->active && crtc->base.primary->state->fb &&  in intel_crtc_active()
626 		crtc->config->hw.adjusted_mode.crtc_clock;  in intel_crtc_active()
633 	for_each_intel_crtc(&dev_priv->drm, crtc) {  in single_enabled_crtc()
653 		drm_dbg_kms(&dev_priv->drm, "Unknown FSB/MEM, disabling CxSR\n");  in pnv_update_wm()
661 			crtc->base.primary->state->fb;  in pnv_update_wm()
662 		int pixel_rate = crtc->config->pixel_rate;  in pnv_update_wm()
663 		int cpp = fb->format->cpp[0];  in pnv_update_wm()
669 					cpp, latency->display_sr);  in pnv_update_wm()
670 		reg = intel_uncore_read(&dev_priv->uncore, DSPFW1(dev_priv));  in pnv_update_wm()
673 		intel_uncore_write(&dev_priv->uncore, DSPFW1(dev_priv), reg);  in pnv_update_wm()
674 		drm_dbg_kms(&dev_priv->drm, "DSPFW1 register is %x\n", reg);  in pnv_update_wm()
680 					4, latency->cursor_sr);  in pnv_update_wm()
681 		intel_uncore_rmw(&dev_priv->uncore, DSPFW3(dev_priv),  in pnv_update_wm()
689 					cpp, latency->display_hpll_disable);  in pnv_update_wm()
690 		intel_uncore_rmw(&dev_priv->uncore, DSPFW3(dev_priv),  in pnv_update_wm()
697 					4, latency->cursor_hpll_disable);  in pnv_update_wm()
698 		reg = intel_uncore_read(&dev_priv->uncore, DSPFW3(dev_priv));  in pnv_update_wm()
701 		intel_uncore_write(&dev_priv->uncore, DSPFW3(dev_priv), reg);  in pnv_update_wm()
702 		drm_dbg_kms(&dev_priv->drm, "DSPFW3 register is %x\n", reg);  in pnv_update_wm()
714 	if (old_plane_state->uapi.visible != new_plane_state->uapi.visible)  in i9xx_wm_need_update()
717 	if (!old_plane_state->hw.fb || !new_plane_state->hw.fb)  in i9xx_wm_need_update()
720 	if (old_plane_state->hw.fb->modifier != new_plane_state->hw.fb->modifier ||  in i9xx_wm_need_update()
721 	    old_plane_state->hw.rotation != new_plane_state->hw.rotation ||  in i9xx_wm_need_update()
722 	    drm_rect_width(&old_plane_state->uapi.src) != drm_rect_width(&new_plane_state->uapi.src) ||  in i9xx_wm_need_update()
723 	    drm_rect_height(&old_plane_state->uapi.src) != drm_rect_height(&new_plane_state->uapi.src) ||  in i9xx_wm_need_update()
724 	    drm_rect_width(&old_plane_state->uapi.dst) != drm_rect_width(&new_plane_state->uapi.dst) ||  in i9xx_wm_need_update()
725 	    drm_rect_height(&old_plane_state->uapi.dst) != drm_rect_height(&new_plane_state->uapi.dst))  in i9xx_wm_need_update()
738 	was_visible = old_plane_state->uapi.visible;  in i9xx_wm_compute()
739 	visible = new_plane_state->uapi.visible;  in i9xx_wm_compute()
749 		new_crtc_state->update_wm_pre = true;  in i9xx_wm_compute()
751 		new_crtc_state->update_wm_post = true;  in i9xx_wm_compute()
754 		new_crtc_state->update_wm_pre = true;  in i9xx_wm_compute()
755 		new_crtc_state->update_wm_post = true;  in i9xx_wm_compute()
771 		if (plane->pipe != crtc->pipe)  in i9xx_compute_watermarks()
787  *  and the size of 8 whole lines. This adjustment is always performed
792 	int tlb_miss = fifo_size * 64 - width * cpp * 8;  in g4x_tlb_miss_wa()
800 	struct intel_display *display = &dev_priv->display;  in g4x_write_wm_values()
806 	intel_uncore_write(&dev_priv->uncore, DSPFW1(dev_priv),  in g4x_write_wm_values()
807 			   FW_WM(wm->sr.plane, SR) |  in g4x_write_wm_values()
808 			   FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |  in g4x_write_wm_values()
809 			   FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |  in g4x_write_wm_values()
810 			   FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));  in g4x_write_wm_values()
811 	intel_uncore_write(&dev_priv->uncore, DSPFW2(dev_priv),  in g4x_write_wm_values()
812 			   (wm->fbc_en ? DSPFW_FBC_SR_EN : 0) |  in g4x_write_wm_values()
813 			   FW_WM(wm->sr.fbc, FBC_SR) |  in g4x_write_wm_values()
814 			   FW_WM(wm->hpll.fbc, FBC_HPLL_SR) |  in g4x_write_wm_values()
815 			   FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEB) |  in g4x_write_wm_values()
816 			   FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |  in g4x_write_wm_values()
817 			   FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));  in g4x_write_wm_values()
818 	intel_uncore_write(&dev_priv->uncore, DSPFW3(dev_priv),  in g4x_write_wm_values()
819 			   (wm->hpll_en ? DSPFW_HPLL_SR_EN : 0) |  in g4x_write_wm_values()
820 			   FW_WM(wm->sr.cursor, CURSOR_SR) |  in g4x_write_wm_values()
821 			   FW_WM(wm->hpll.cursor, HPLL_CURSOR) |  in g4x_write_wm_values()
822 			   FW_WM(wm->hpll.plane, HPLL_SR));  in g4x_write_wm_values()
824 	intel_uncore_posting_read(&dev_priv->uncore, DSPFW1(dev_priv));  in g4x_write_wm_values()
833 	struct intel_display *display = &dev_priv->display;  in vlv_write_wm_values()
839 		intel_uncore_write(&dev_priv->uncore, VLV_DDL(pipe),  in vlv_write_wm_values()
840 				   (wm->ddl[pipe].plane[PLANE_CURSOR] << DDL_CURSOR_SHIFT) |  in vlv_write_wm_values()
841 				   (wm->ddl[pipe].plane[PLANE_SPRITE1] << DDL_SPRITE_SHIFT(1)) |  in vlv_write_wm_values()
842 				   (wm->ddl[pipe].plane[PLANE_SPRITE0] << DDL_SPRITE_SHIFT(0)) |  in vlv_write_wm_values()
843 				   (wm->ddl[pipe].plane[PLANE_PRIMARY] << DDL_PLANE_SHIFT));  in vlv_write_wm_values()
851 	intel_uncore_write(&dev_priv->uncore, DSPHOWM, 0);  in vlv_write_wm_values()
852 	intel_uncore_write(&dev_priv->uncore, DSPHOWM1, 0);  in vlv_write_wm_values()
853 	intel_uncore_write(&dev_priv->uncore, DSPFW4, 0);  in vlv_write_wm_values()
854 	intel_uncore_write(&dev_priv->uncore, DSPFW5, 0);  in vlv_write_wm_values()
855 	intel_uncore_write(&dev_priv->uncore, DSPFW6, 0);  in vlv_write_wm_values()
857 	intel_uncore_write(&dev_priv->uncore, DSPFW1(dev_priv),  in vlv_write_wm_values()
858 			   FW_WM(wm->sr.plane, SR) |  in vlv_write_wm_values()
859 			   FW_WM(wm->pipe[PIPE_B].plane[PLANE_CURSOR], CURSORB) |  in vlv_write_wm_values()
860 			   FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_PRIMARY], PLANEB) |  in vlv_write_wm_values()
861 			   FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_PRIMARY], PLANEA));  in vlv_write_wm_values()
862 	intel_uncore_write(&dev_priv->uncore, DSPFW2(dev_priv),  in vlv_write_wm_values()
863 			   FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE1], SPRITEB) |  in vlv_write_wm_values()
864 			   FW_WM(wm->pipe[PIPE_A].plane[PLANE_CURSOR], CURSORA) |  in vlv_write_wm_values()
865 			   FW_WM_VLV(wm->pipe[PIPE_A].plane[PLANE_SPRITE0], SPRITEA));  in vlv_write_wm_values()
866 	intel_uncore_write(&dev_priv->uncore, DSPFW3(dev_priv),  in vlv_write_wm_values()
867 			   FW_WM(wm->sr.cursor, CURSOR_SR));  in vlv_write_wm_values()
870 		intel_uncore_write(&dev_priv->uncore, DSPFW7_CHV,  in vlv_write_wm_values()
871 				   FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |  in vlv_write_wm_values()
872 				   FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));  in vlv_write_wm_values()
873 		intel_uncore_write(&dev_priv->uncore, DSPFW8_CHV,  in vlv_write_wm_values()
874 				   FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE1], SPRITEF) |  in vlv_write_wm_values()
875 				   FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_SPRITE0], SPRITEE));  in vlv_write_wm_values()
876 		intel_uncore_write(&dev_priv->uncore, DSPFW9_CHV,  in vlv_write_wm_values()
877 				   FW_WM_VLV(wm->pipe[PIPE_C].plane[PLANE_PRIMARY], PLANEC) |  in vlv_write_wm_values()
878 				   FW_WM(wm->pipe[PIPE_C].plane[PLANE_CURSOR], CURSORC));  in vlv_write_wm_values()
879 		intel_uncore_write(&dev_priv->uncore, DSPHOWM,  in vlv_write_wm_values()
880 				   FW_WM(wm->sr.plane >> 9, SR_HI) |  in vlv_write_wm_values()
881 				   FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE1] >> 8, SPRITEF_HI) |  in vlv_write_wm_values()
882 				   FW_WM(wm->pipe[PIPE_C].plane[PLANE_SPRITE0] >> 8, SPRITEE_HI) |  in vlv_write_wm_values()
883 				   FW_WM(wm->pipe[PIPE_C].plane[PLANE_PRIMARY] >> 8, PLANEC_HI) |  in vlv_write_wm_values()
884 				   FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |  in vlv_write_wm_values()
885 				   FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |  in vlv_write_wm_values()
886 				   FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |  in vlv_write_wm_values()
887 				   FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |  in vlv_write_wm_values()
888 				   FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |  in vlv_write_wm_values()
889 				   FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));  in vlv_write_wm_values()
891 		intel_uncore_write(&dev_priv->uncore, DSPFW7,  in vlv_write_wm_values()
892 				   FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE1], SPRITED) |  in vlv_write_wm_values()
893 				   FW_WM_VLV(wm->pipe[PIPE_B].plane[PLANE_SPRITE0], SPRITEC));  in vlv_write_wm_values()
894 		intel_uncore_write(&dev_priv->uncore, DSPHOWM,  in vlv_write_wm_values()
895 				   FW_WM(wm->sr.plane >> 9, SR_HI) |  in vlv_write_wm_values()
896 				   FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE1] >> 8, SPRITED_HI) |  in vlv_write_wm_values()
897 				   FW_WM(wm->pipe[PIPE_B].plane[PLANE_SPRITE0] >> 8, SPRITEC_HI) |  in vlv_write_wm_values()
898 				   FW_WM(wm->pipe[PIPE_B].plane[PLANE_PRIMARY] >> 8, PLANEB_HI) |  in vlv_write_wm_values()
899 				   FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE1] >> 8, SPRITEB_HI) |  in vlv_write_wm_values()
900 				   FW_WM(wm->pipe[PIPE_A].plane[PLANE_SPRITE0] >> 8, SPRITEA_HI) |  in vlv_write_wm_values()
901 				   FW_WM(wm->pipe[PIPE_A].plane[PLANE_PRIMARY] >> 8, PLANEA_HI));  in vlv_write_wm_values()
904 	intel_uncore_posting_read(&dev_priv->uncore, DSPFW1(dev_priv));  in vlv_write_wm_values()
912 	dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_NORMAL] = 5;  in g4x_setup_wm_latency()
913 	dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_SR] = 12;  in g4x_setup_wm_latency()
914 	dev_priv->display.wm.pri_latency[G4X_WM_LEVEL_HPLL] = 35;  in g4x_setup_wm_latency()
916 	dev_priv->display.wm.num_levels = G4X_WM_LEVEL_HPLL + 1;  in g4x_setup_wm_latency()
919 static int g4x_plane_fifo_size(enum plane_id plane_id, int level)  in g4x_plane_fifo_size()  argument
939 		return level == G4X_WM_LEVEL_NORMAL ? 127 : 511;  in g4x_plane_fifo_size()
941 		return level == G4X_WM_LEVEL_NORMAL ? 127 : 0;  in g4x_plane_fifo_size()
948 static int g4x_fbc_fifo_size(int level)  in g4x_fbc_fifo_size()  argument
950 	switch (level) {  in g4x_fbc_fifo_size()
956 		MISSING_CASE(level);  in g4x_fbc_fifo_size()
963 			  int level)  in g4x_compute_wm()  argument
965 	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);  in g4x_compute_wm()
966 	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);  in g4x_compute_wm()
968 		&crtc_state->hw.pipe_mode;  in g4x_compute_wm()
969 	unsigned int latency = dev_priv->display.wm.pri_latency[level] * 10;  in g4x_compute_wm()
978 	cpp = plane_state->hw.fb->format->cpp[0];  in g4x_compute_wm()
987 	if (plane->id == PLANE_PRIMARY &&  in g4x_compute_wm()
988 	    level != G4X_WM_LEVEL_NORMAL)  in g4x_compute_wm()
991 	pixel_rate = crtc_state->pixel_rate;  in g4x_compute_wm()
992 	htotal = pipe_mode->crtc_htotal;  in g4x_compute_wm()
993 	width = drm_rect_width(&plane_state->uapi.src) >> 16;  in g4x_compute_wm()
995 	if (plane->id == PLANE_CURSOR) {  in g4x_compute_wm()
997 	} else if (plane->id == PLANE_PRIMARY &&  in g4x_compute_wm()
998 		   level == G4X_WM_LEVEL_NORMAL) {  in g4x_compute_wm()
1009 	wm += g4x_tlb_miss_wa(g4x_plane_fifo_size(plane->id, level),  in g4x_compute_wm()
1018 				 int level, enum plane_id plane_id, u16 value)  in g4x_raw_plane_wm_set()  argument
1020 	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);  in g4x_raw_plane_wm_set()
1023 	for (; level < dev_priv->display.wm.num_levels; level++) {  in g4x_raw_plane_wm_set()
1024 		struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];  in g4x_raw_plane_wm_set()
1026 		dirty |= raw->plane[plane_id] != value;  in g4x_raw_plane_wm_set()
1027 		raw->plane[plane_id] = value;  in g4x_raw_plane_wm_set()
1034 			       int level, u16 value)  in g4x_raw_fbc_wm_set()  argument
1036 	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);  in g4x_raw_fbc_wm_set()
1039 	/* NORMAL level doesn't have an FBC watermark */  in g4x_raw_fbc_wm_set()
1040 	level = max(level, G4X_WM_LEVEL_SR);  in g4x_raw_fbc_wm_set()
1042 	for (; level < dev_priv->display.wm.num_levels; level++) {  in g4x_raw_fbc_wm_set()
1043 		struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];  in g4x_raw_fbc_wm_set()
1045 		dirty |= raw->fbc != value;  in g4x_raw_fbc_wm_set()
1046 		raw->fbc = value;  in g4x_raw_fbc_wm_set()
1059 	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);  in g4x_raw_plane_wm_compute()
1060 	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);  in g4x_raw_plane_wm_compute()
1061 	enum plane_id plane_id = plane->id;  in g4x_raw_plane_wm_compute()
1063 	int level;  in g4x_raw_plane_wm_compute()  local
1072 	for (level = 0; level < dev_priv->display.wm.num_levels; level++) {  in g4x_raw_plane_wm_compute()
1073 		struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];  in g4x_raw_plane_wm_compute()
1076 		wm = g4x_compute_wm(crtc_state, plane_state, level);  in g4x_raw_plane_wm_compute()
1077 		max_wm = g4x_plane_fifo_size(plane_id, level);  in g4x_raw_plane_wm_compute()
1082 		dirty |= raw->plane[plane_id] != wm;  in g4x_raw_plane_wm_compute()
1083 		raw->plane[plane_id] = wm;  in g4x_raw_plane_wm_compute()
1086 		    level == G4X_WM_LEVEL_NORMAL)  in g4x_raw_plane_wm_compute()
1090 					raw->plane[plane_id]);  in g4x_raw_plane_wm_compute()
1091 		max_wm = g4x_fbc_fifo_size(level);  in g4x_raw_plane_wm_compute()
1100 		dirty |= raw->fbc != wm;  in g4x_raw_plane_wm_compute()
1101 		raw->fbc = wm;  in g4x_raw_plane_wm_compute()
1105 	dirty |= g4x_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);  in g4x_raw_plane_wm_compute()
1108 		dirty |= g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);  in g4x_raw_plane_wm_compute()
1112 		drm_dbg_kms(&dev_priv->drm,  in g4x_raw_plane_wm_compute()
1114 			    plane->base.name,  in g4x_raw_plane_wm_compute()
1115 			    crtc_state->wm.g4x.raw[G4X_WM_LEVEL_NORMAL].plane[plane_id],  in g4x_raw_plane_wm_compute()
1116 			    crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].plane[plane_id],  in g4x_raw_plane_wm_compute()
1117 			    crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].plane[plane_id]);  in g4x_raw_plane_wm_compute()
1120 			drm_dbg_kms(&dev_priv->drm,  in g4x_raw_plane_wm_compute()
1122 				    crtc_state->wm.g4x.raw[G4X_WM_LEVEL_SR].fbc,  in g4x_raw_plane_wm_compute()
1123 				    crtc_state->wm.g4x.raw[G4X_WM_LEVEL_HPLL].fbc);  in g4x_raw_plane_wm_compute()
1130 				      enum plane_id plane_id, int level)  in g4x_raw_plane_wm_is_valid()  argument
1132 	const struct g4x_pipe_wm *raw = &crtc_state->wm.g4x.raw[level];  in g4x_raw_plane_wm_is_valid()
1134 	return raw->plane[plane_id] <= g4x_plane_fifo_size(plane_id, level);  in g4x_raw_plane_wm_is_valid()
1138 				     int level)  in g4x_raw_crtc_wm_is_valid()  argument
1140 	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);  in g4x_raw_crtc_wm_is_valid()
1142 	if (level >= dev_priv->display.wm.num_levels)  in g4x_raw_crtc_wm_is_valid()
1145 	return g4x_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&  in g4x_raw_crtc_wm_is_valid()
1146 		g4x_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&  in g4x_raw_crtc_wm_is_valid()
1147 		g4x_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);  in g4x_raw_crtc_wm_is_valid()
1150 /* mark all levels starting from 'level' as invalid */
1152 			       struct g4x_wm_state *wm_state, int level)  in g4x_invalidate_wms()  argument
1154 	if (level <= G4X_WM_LEVEL_NORMAL) {  in g4x_invalidate_wms()
1158 			wm_state->wm.plane[plane_id] = USHRT_MAX;  in g4x_invalidate_wms()
1161 	if (level <= G4X_WM_LEVEL_SR) {  in g4x_invalidate_wms()
1162 		wm_state->cxsr = false;  in g4x_invalidate_wms()
1163 		wm_state->sr.cursor = USHRT_MAX;  in g4x_invalidate_wms()
1164 		wm_state->sr.plane = USHRT_MAX;  in g4x_invalidate_wms()
1165 		wm_state->sr.fbc = USHRT_MAX;  in g4x_invalidate_wms()
1168 	if (level <= G4X_WM_LEVEL_HPLL) {  in g4x_invalidate_wms()
1169 		wm_state->hpll_en = false;  in g4x_invalidate_wms()
1170 		wm_state->hpll.cursor = USHRT_MAX;  in g4x_invalidate_wms()
1171 		wm_state->hpll.plane = USHRT_MAX;  in g4x_invalidate_wms()
1172 		wm_state->hpll.fbc = USHRT_MAX;  in g4x_invalidate_wms()
1177 			       int level)  in g4x_compute_fbc_en()  argument
1179 	if (level < G4X_WM_LEVEL_SR)  in g4x_compute_fbc_en()
1182 	if (level >= G4X_WM_LEVEL_SR &&  in g4x_compute_fbc_en()
1183 	    wm_state->sr.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_SR))  in g4x_compute_fbc_en()
1186 	if (level >= G4X_WM_LEVEL_HPLL &&  in g4x_compute_fbc_en()
1187 	    wm_state->hpll.fbc > g4x_fbc_fifo_size(G4X_WM_LEVEL_HPLL))  in g4x_compute_fbc_en()
1195 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);  in _g4x_compute_pipe_wm()
1196 	struct g4x_wm_state *wm_state = &crtc_state->wm.g4x.optimal;  in _g4x_compute_pipe_wm()
1197 	u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);  in _g4x_compute_pipe_wm()
1200 	int level;  in _g4x_compute_pipe_wm()  local
1202 	level = G4X_WM_LEVEL_NORMAL;  in _g4x_compute_pipe_wm()
1203 	if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))  in _g4x_compute_pipe_wm()
1206 	raw = &crtc_state->wm.g4x.raw[level];  in _g4x_compute_pipe_wm()
1208 		wm_state->wm.plane[plane_id] = raw->plane[plane_id];  in _g4x_compute_pipe_wm()
1210 	level = G4X_WM_LEVEL_SR;  in _g4x_compute_pipe_wm()
1211 	if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))  in _g4x_compute_pipe_wm()
1214 	raw = &crtc_state->wm.g4x.raw[level];  in _g4x_compute_pipe_wm()
1215 	wm_state->sr.plane = raw->plane[PLANE_PRIMARY];  in _g4x_compute_pipe_wm()
1216 	wm_state->sr.cursor = raw->plane[PLANE_CURSOR];  in _g4x_compute_pipe_wm()
1217 	wm_state->sr.fbc = raw->fbc;  in _g4x_compute_pipe_wm()
1219 	wm_state->cxsr = active_planes == BIT(PLANE_PRIMARY);  in _g4x_compute_pipe_wm()
1221 	level = G4X_WM_LEVEL_HPLL;  in _g4x_compute_pipe_wm()
1222 	if (!g4x_raw_crtc_wm_is_valid(crtc_state, level))  in _g4x_compute_pipe_wm()
1225 	raw = &crtc_state->wm.g4x.raw[level];  in _g4x_compute_pipe_wm()
1226 	wm_state->hpll.plane = raw->plane[PLANE_PRIMARY];  in _g4x_compute_pipe_wm()
1227 	wm_state->hpll.cursor = raw->plane[PLANE_CURSOR];  in _g4x_compute_pipe_wm()
1228 	wm_state->hpll.fbc = raw->fbc;  in _g4x_compute_pipe_wm()
1230 	wm_state->hpll_en = wm_state->cxsr;  in _g4x_compute_pipe_wm()
1232 	level++;  in _g4x_compute_pipe_wm()
1235 	if (level == G4X_WM_LEVEL_NORMAL)  in _g4x_compute_pipe_wm()
1236 		return -EINVAL;  in _g4x_compute_pipe_wm()
1239 	g4x_invalidate_wms(crtc, wm_state, level);  in _g4x_compute_pipe_wm()
1245 	 * level(s) entirely. 'level-1' is the highest valid  in _g4x_compute_pipe_wm()
1246 	 * level here.  in _g4x_compute_pipe_wm()
1248 	wm_state->fbc_en = g4x_compute_fbc_en(wm_state, level - 1);  in _g4x_compute_pipe_wm()
1267 		if (new_plane_state->hw.crtc != &crtc->base &&  in g4x_compute_pipe_wm()
1268 		    old_plane_state->hw.crtc != &crtc->base)  in g4x_compute_pipe_wm()
1272 			dirty |= BIT(plane->id);  in g4x_compute_pipe_wm()
1284 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);  in g4x_compute_intermediate_wm()
1289 	struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;  in g4x_compute_intermediate_wm()
1290 	const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;  in g4x_compute_intermediate_wm()
1291 	const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;  in g4x_compute_intermediate_wm()
1294 	if (!new_crtc_state->hw.active ||  in g4x_compute_intermediate_wm()
1298 		intermediate->cxsr = false;  in g4x_compute_intermediate_wm()
1299 		intermediate->hpll_en = false;  in g4x_compute_intermediate_wm()
1303 	intermediate->cxsr = optimal->cxsr && active->cxsr &&  in g4x_compute_intermediate_wm()
1304 		!new_crtc_state->disable_cxsr;  in g4x_compute_intermediate_wm()
1305 	intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&  in g4x_compute_intermediate_wm()
1306 		!new_crtc_state->disable_cxsr;  in g4x_compute_intermediate_wm()
1307 	intermediate->fbc_en = optimal->fbc_en && active->fbc_en;  in g4x_compute_intermediate_wm()
1310 		intermediate->wm.plane[plane_id] =  in g4x_compute_intermediate_wm()
1311 			max(optimal->wm.plane[plane_id],  in g4x_compute_intermediate_wm()
1312 			    active->wm.plane[plane_id]);  in g4x_compute_intermediate_wm()
1314 		drm_WARN_ON(&dev_priv->drm, intermediate->wm.plane[plane_id] >  in g4x_compute_intermediate_wm()
1318 	intermediate->sr.plane = max(optimal->sr.plane,  in g4x_compute_intermediate_wm()
1319 				     active->sr.plane);  in g4x_compute_intermediate_wm()
1320 	intermediate->sr.cursor = max(optimal->sr.cursor,  in g4x_compute_intermediate_wm()
1321 				      active->sr.cursor);  in g4x_compute_intermediate_wm()
1322 	intermediate->sr.fbc = max(optimal->sr.fbc,  in g4x_compute_intermediate_wm()
1323 				   active->sr.fbc);  in g4x_compute_intermediate_wm()
1325 	intermediate->hpll.plane = max(optimal->hpll.plane,  in g4x_compute_intermediate_wm()
1326 				       active->hpll.plane);  in g4x_compute_intermediate_wm()
1327 	intermediate->hpll.cursor = max(optimal->hpll.cursor,  in g4x_compute_intermediate_wm()
1328 					active->hpll.cursor);  in g4x_compute_intermediate_wm()
1329 	intermediate->hpll.fbc = max(optimal->hpll.fbc,  in g4x_compute_intermediate_wm()
1330 				     active->hpll.fbc);  in g4x_compute_intermediate_wm()
1332 	drm_WARN_ON(&dev_priv->drm,  in g4x_compute_intermediate_wm()
1333 		    (intermediate->sr.plane >  in g4x_compute_intermediate_wm()
1335 		     intermediate->sr.cursor >  in g4x_compute_intermediate_wm()
1337 		    intermediate->cxsr);  in g4x_compute_intermediate_wm()
1338 	drm_WARN_ON(&dev_priv->drm,  in g4x_compute_intermediate_wm()
1339 		    (intermediate->sr.plane >  in g4x_compute_intermediate_wm()
1341 		     intermediate->sr.cursor >  in g4x_compute_intermediate_wm()
1343 		    intermediate->hpll_en);  in g4x_compute_intermediate_wm()
1345 	drm_WARN_ON(&dev_priv->drm,  in g4x_compute_intermediate_wm()
1346 		    intermediate->sr.fbc > g4x_fbc_fifo_size(1) &&  in g4x_compute_intermediate_wm()
1347 		    intermediate->fbc_en && intermediate->cxsr);  in g4x_compute_intermediate_wm()
1348 	drm_WARN_ON(&dev_priv->drm,  in g4x_compute_intermediate_wm()
1349 		    intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&  in g4x_compute_intermediate_wm()
1350 		    intermediate->fbc_en && intermediate->hpll_en);  in g4x_compute_intermediate_wm()
1355 	 * omit the post-vblank programming; only update if it's different.  in g4x_compute_intermediate_wm()
1358 		new_crtc_state->wm.need_postvbl_update = true;  in g4x_compute_intermediate_wm()
1385 	wm->cxsr = true;  in g4x_merge_wm()
1386 	wm->hpll_en = true;  in g4x_merge_wm()
1387 	wm->fbc_en = true;  in g4x_merge_wm()
1389 	for_each_intel_crtc(&dev_priv->drm, crtc) {  in g4x_merge_wm()
1390 		const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;  in g4x_merge_wm()
1392 		if (!crtc->active)  in g4x_merge_wm()
1395 		if (!wm_state->cxsr)  in g4x_merge_wm()
1396 			wm->cxsr = false;  in g4x_merge_wm()
1397 		if (!wm_state->hpll_en)  in g4x_merge_wm()
1398 			wm->hpll_en = false;  in g4x_merge_wm()
1399 		if (!wm_state->fbc_en)  in g4x_merge_wm()
1400 			wm->fbc_en = false;  in g4x_merge_wm()
1406 		wm->cxsr = false;  in g4x_merge_wm()
1407 		wm->hpll_en = false;  in g4x_merge_wm()
1408 		wm->fbc_en = false;  in g4x_merge_wm()
1411 	for_each_intel_crtc(&dev_priv->drm, crtc) {  in g4x_merge_wm()
1412 		const struct g4x_wm_state *wm_state = &crtc->wm.active.g4x;  in g4x_merge_wm()
1413 		enum pipe pipe = crtc->pipe;  in g4x_merge_wm()
1415 		wm->pipe[pipe] = wm_state->wm;  in g4x_merge_wm()
1416 		if (crtc->active && wm->cxsr)  in g4x_merge_wm()
1417 			wm->sr = wm_state->sr;  in g4x_merge_wm()
1418 		if (crtc->active && wm->hpll_en)  in g4x_merge_wm()
1419 			wm->hpll = wm_state->hpll;  in g4x_merge_wm()
1425 	struct g4x_wm_values *old_wm = &dev_priv->display.wm.g4x;  in g4x_program_watermarks()
1433 	if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))  in g4x_program_watermarks()
1438 	if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))  in g4x_program_watermarks()
1447 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);  in g4x_initial_watermarks()
1451 	mutex_lock(&dev_priv->display.wm.wm_mutex);  in g4x_initial_watermarks()
1452 	crtc->wm.active.g4x = crtc_state->wm.g4x.intermediate;  in g4x_initial_watermarks()
1454 	mutex_unlock(&dev_priv->display.wm.wm_mutex);  in g4x_initial_watermarks()
1460 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);  in g4x_optimize_watermarks()
1464 	if (!crtc_state->wm.need_postvbl_update)  in g4x_optimize_watermarks()
1467 	mutex_lock(&dev_priv->display.wm.wm_mutex);  in g4x_optimize_watermarks()
1468 	crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;  in g4x_optimize_watermarks()
1470 	mutex_unlock(&dev_priv->display.wm.wm_mutex);  in g4x_optimize_watermarks()
1492 	dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;  in vlv_setup_wm_latency()
1494 	dev_priv->display.wm.num_levels = VLV_WM_LEVEL_PM2 + 1;  in vlv_setup_wm_latency()
1497 		dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_PM5] = 12;  in vlv_setup_wm_latency()
1498 		dev_priv->display.wm.pri_latency[VLV_WM_LEVEL_DDR_DVFS] = 33;  in vlv_setup_wm_latency()
1500 		dev_priv->display.wm.num_levels = VLV_WM_LEVEL_DDR_DVFS + 1;  in vlv_setup_wm_latency()
1506 				int level)  in vlv_compute_wm_level()  argument
1508 	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);  in vlv_compute_wm_level()
1509 	struct drm_i915_private *dev_priv = to_i915(plane->base.dev);  in vlv_compute_wm_level()
1511 		&crtc_state->hw.pipe_mode;  in vlv_compute_wm_level()
1514 	if (dev_priv->display.wm.pri_latency[level] == 0)  in vlv_compute_wm_level()
1520 	cpp = plane_state->hw.fb->format->cpp[0];  in vlv_compute_wm_level()
1521 	pixel_rate = crtc_state->pixel_rate;  in vlv_compute_wm_level()
1522 	htotal = pipe_mode->crtc_htotal;  in vlv_compute_wm_level()
1523 	width = drm_rect_width(&plane_state->uapi.src) >> 16;  in vlv_compute_wm_level()
1525 	if (plane->id == PLANE_CURSOR) {  in vlv_compute_wm_level()
1535 				    dev_priv->display.wm.pri_latency[level] * 10);  in vlv_compute_wm_level()
1549 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);  in vlv_compute_fifo()
1550 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);  in vlv_compute_fifo()
1552 		&crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2];  in vlv_compute_fifo()
1553 	struct vlv_fifo_state *fifo_state = &crtc_state->wm.vlv.fifo_state;  in vlv_compute_fifo()
1554 	u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);  in vlv_compute_fifo()
1573 	total_rate = raw->plane[PLANE_PRIMARY] +  in vlv_compute_fifo()
1574 		raw->plane[PLANE_SPRITE0] +  in vlv_compute_fifo()
1575 		raw->plane[PLANE_SPRITE1] +  in vlv_compute_fifo()
1579 		return -EINVAL;  in vlv_compute_fifo()
1588 			fifo_state->plane[plane_id] = 0;  in vlv_compute_fifo()
1592 		rate = raw->plane[plane_id];  in vlv_compute_fifo()
1593 		fifo_state->plane[plane_id] = fifo_size * rate / total_rate;  in vlv_compute_fifo()
1594 		fifo_left -= fifo_state->plane[plane_id];  in vlv_compute_fifo()
1597 	fifo_state->plane[PLANE_SPRITE0] += sprite0_fifo_extra;  in vlv_compute_fifo()
1598 	fifo_left -= sprite0_fifo_extra;  in vlv_compute_fifo()
1600 	fifo_state->plane[PLANE_CURSOR] = 63;  in vlv_compute_fifo()
1615 		fifo_state->plane[plane_id] += plane_extra;  in vlv_compute_fifo()
1616 		fifo_left -= plane_extra;  in vlv_compute_fifo()
1619 	drm_WARN_ON(&dev_priv->drm, active_planes != 0 && fifo_left != 0);  in vlv_compute_fifo()
1623 		drm_WARN_ON(&dev_priv->drm, fifo_left != fifo_size);  in vlv_compute_fifo()
1624 		fifo_state->plane[PLANE_PRIMARY] = fifo_left;  in vlv_compute_fifo()
1630 /* mark all levels starting from 'level' as invalid */
1632 			       struct vlv_wm_state *wm_state, int level)  in vlv_invalidate_wms()  argument
1634 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);  in vlv_invalidate_wms()
1636 	for (; level < dev_priv->display.wm.num_levels; level++) {  in vlv_invalidate_wms()
1640 			wm_state->wm[level].plane[plane_id] = USHRT_MAX;  in vlv_invalidate_wms()
1642 		wm_state->sr[level].cursor = USHRT_MAX;  in vlv_invalidate_wms()
1643 		wm_state->sr[level].plane = USHRT_MAX;  in vlv_invalidate_wms()
1652 		return fifo_size - wm;  in vlv_invert_wm_value()
1656  * Starting from 'level' set all higher
1660 				 int level, enum plane_id plane_id, u16 value)  in vlv_raw_plane_wm_set()  argument
1662 	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);  in vlv_raw_plane_wm_set()
1665 	for (; level < dev_priv->display.wm.num_levels; level++) {  in vlv_raw_plane_wm_set()
1666 		struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];  in vlv_raw_plane_wm_set()
1668 		dirty |= raw->plane[plane_id] != value;  in vlv_raw_plane_wm_set()
1669 		raw->plane[plane_id] = value;  in vlv_raw_plane_wm_set()
1678 	struct intel_plane *plane = to_intel_plane(plane_state->uapi.plane);  in vlv_raw_plane_wm_compute()
1679 	struct drm_i915_private *dev_priv = to_i915(crtc_state->uapi.crtc->dev);  in vlv_raw_plane_wm_compute()
1680 	enum plane_id plane_id = plane->id;  in vlv_raw_plane_wm_compute()
1681 	int level;  in vlv_raw_plane_wm_compute()  local
1689 	for (level = 0; level < dev_priv->display.wm.num_levels; level++) {  in vlv_raw_plane_wm_compute()
1690 		struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];  in vlv_raw_plane_wm_compute()
1691 		int wm = vlv_compute_wm_level(crtc_state, plane_state, level);  in vlv_raw_plane_wm_compute()
1697 		dirty |= raw->plane[plane_id] != wm;  in vlv_raw_plane_wm_compute()
1698 		raw->plane[plane_id] = wm;  in vlv_raw_plane_wm_compute()
1702 	dirty |= vlv_raw_plane_wm_set(crtc_state, level, plane_id, USHRT_MAX);  in vlv_raw_plane_wm_compute()
1706 		drm_dbg_kms(&dev_priv->drm,  in vlv_raw_plane_wm_compute()
1708 			    plane->base.name,  in vlv_raw_plane_wm_compute()
1709 			    crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM2].plane[plane_id],  in vlv_raw_plane_wm_compute()
1710 			    crtc_state->wm.vlv.raw[VLV_WM_LEVEL_PM5].plane[plane_id],  in vlv_raw_plane_wm_compute()
1711 			    crtc_state->wm.vlv.raw[VLV_WM_LEVEL_DDR_DVFS].plane[plane_id]);  in vlv_raw_plane_wm_compute()
1717 				      enum plane_id plane_id, int level)  in vlv_raw_plane_wm_is_valid()  argument
1720 		&crtc_state->wm.vlv.raw[level];  in vlv_raw_plane_wm_is_valid()
1722 		&crtc_state->wm.vlv.fifo_state;  in vlv_raw_plane_wm_is_valid()
1724 	return raw->plane[plane_id] <= fifo_state->plane[plane_id];  in vlv_raw_plane_wm_is_valid()
1727 static bool vlv_raw_crtc_wm_is_valid(const struct intel_crtc_state *crtc_state, int level)  in vlv_raw_crtc_wm_is_valid()  argument
1729 	return vlv_raw_plane_wm_is_valid(crtc_state, PLANE_PRIMARY, level) &&  in vlv_raw_crtc_wm_is_valid()
1730 		vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE0, level) &&  in vlv_raw_crtc_wm_is_valid()
1731 		vlv_raw_plane_wm_is_valid(crtc_state, PLANE_SPRITE1, level) &&  in vlv_raw_crtc_wm_is_valid()
1732 		vlv_raw_plane_wm_is_valid(crtc_state, PLANE_CURSOR, level);  in vlv_raw_crtc_wm_is_valid()
1737 	struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);  in _vlv_compute_pipe_wm()
1738 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);  in _vlv_compute_pipe_wm()
1739 	struct vlv_wm_state *wm_state = &crtc_state->wm.vlv.optimal;  in _vlv_compute_pipe_wm()
1741 		&crtc_state->wm.vlv.fifo_state;  in _vlv_compute_pipe_wm()
1742 	u8 active_planes = crtc_state->active_planes & ~BIT(PLANE_CURSOR);  in _vlv_compute_pipe_wm()
1745 	int level;  in _vlv_compute_pipe_wm()  local
1748 	wm_state->num_levels = dev_priv->display.wm.num_levels;  in _vlv_compute_pipe_wm()
1754 	wm_state->cxsr = crtc->pipe != PIPE_C && num_active_planes == 1;  in _vlv_compute_pipe_wm()
1756 	for (level = 0; level < wm_state->num_levels; level++) {  in _vlv_compute_pipe_wm()
1757 		const struct g4x_pipe_wm *raw = &crtc_state->wm.vlv.raw[level];  in _vlv_compute_pipe_wm()
1758 		const int sr_fifo_size = INTEL_NUM_PIPES(dev_priv) * 512 - 1;  in _vlv_compute_pipe_wm()
1760 		if (!vlv_raw_crtc_wm_is_valid(crtc_state, level))  in _vlv_compute_pipe_wm()
1764 			wm_state->wm[level].plane[plane_id] =  in _vlv_compute_pipe_wm()
1765 				vlv_invert_wm_value(raw->plane[plane_id],  in _vlv_compute_pipe_wm()
1766 						    fifo_state->plane[plane_id]);  in _vlv_compute_pipe_wm()
1769 		wm_state->sr[level].plane =  in _vlv_compute_pipe_wm()
1770 			vlv_invert_wm_value(max3(raw->plane[PLANE_PRIMARY],  in _vlv_compute_pipe_wm()
1771 						 raw->plane[PLANE_SPRITE0],  in _vlv_compute_pipe_wm()
1772 						 raw->plane[PLANE_SPRITE1]),  in _vlv_compute_pipe_wm()
1775 		wm_state->sr[level].cursor =  in _vlv_compute_pipe_wm()
1776 			vlv_invert_wm_value(raw->plane[PLANE_CURSOR],  in _vlv_compute_pipe_wm()
1780 	if (level == 0)  in _vlv_compute_pipe_wm()
1781 		return -EINVAL;  in _vlv_compute_pipe_wm()
1784 	wm_state->num_levels = level;  in _vlv_compute_pipe_wm()
1787 	vlv_invalidate_wms(crtc, wm_state, level);  in _vlv_compute_pipe_wm()
1806 		if (new_plane_state->hw.crtc != &crtc->base &&  in vlv_compute_pipe_wm()
1807 		    old_plane_state->hw.crtc != &crtc->base)  in vlv_compute_pipe_wm()
1811 			dirty |= BIT(plane->id);  in vlv_compute_pipe_wm()
1834 			&old_crtc_state->wm.vlv.fifo_state;  in vlv_compute_pipe_wm()
1836 			&crtc_state->wm.vlv.fifo_state;  in vlv_compute_pipe_wm()
1846 			crtc_state->fifo_changed = true;  in vlv_compute_pipe_wm()
1858 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);  in vlv_atomic_update_fifo()
1859 	struct intel_uncore *uncore = &dev_priv->uncore;  in vlv_atomic_update_fifo()
1863 		&crtc_state->wm.vlv.fifo_state;  in vlv_atomic_update_fifo()
1867 	if (!crtc_state->fifo_changed)  in vlv_atomic_update_fifo()
1870 	sprite0_start = fifo_state->plane[PLANE_PRIMARY];  in vlv_atomic_update_fifo()
1871 	sprite1_start = fifo_state->plane[PLANE_SPRITE0] + sprite0_start;  in vlv_atomic_update_fifo()
1872 	fifo_size = fifo_state->plane[PLANE_SPRITE1] + sprite1_start;  in vlv_atomic_update_fifo()
1874 	drm_WARN_ON(&dev_priv->drm, fifo_state->plane[PLANE_CURSOR] != 63);  in vlv_atomic_update_fifo()
1875 	drm_WARN_ON(&dev_priv->drm, fifo_size != 511);  in vlv_atomic_update_fifo()
1888 	spin_lock(&uncore->lock);  in vlv_atomic_update_fifo()
1890 	switch (crtc->pipe) {  in vlv_atomic_update_fifo()
1902 		dsparb2 |= (VLV_FIFO(SPRITEA_HI, sprite0_start >> 8) |  in vlv_atomic_update_fifo()
1903 			   VLV_FIFO(SPRITEB_HI, sprite1_start >> 8));  in vlv_atomic_update_fifo()
1919 		dsparb2 |= (VLV_FIFO(SPRITEC_HI, sprite0_start >> 8) |  in vlv_atomic_update_fifo()
1920 			   VLV_FIFO(SPRITED_HI, sprite1_start >> 8));  in vlv_atomic_update_fifo()
1936 		dsparb2 |= (VLV_FIFO(SPRITEE_HI, sprite0_start >> 8) |  in vlv_atomic_update_fifo()
1937 			   VLV_FIFO(SPRITEF_HI, sprite1_start >> 8));  in vlv_atomic_update_fifo()
1948 	spin_unlock(&uncore->lock);  in vlv_atomic_update_fifo()
1960 	struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;  in vlv_compute_intermediate_wm()
1961 	const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;  in vlv_compute_intermediate_wm()
1962 	const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;  in vlv_compute_intermediate_wm()
1963 	int level;  in vlv_compute_intermediate_wm()  local
1965 	if (!new_crtc_state->hw.active ||  in vlv_compute_intermediate_wm()
1969 		intermediate->cxsr = false;  in vlv_compute_intermediate_wm()
1973 	intermediate->num_levels = min(optimal->num_levels, active->num_levels);  in vlv_compute_intermediate_wm()
1974 	intermediate->cxsr = optimal->cxsr && active->cxsr &&  in vlv_compute_intermediate_wm()
1975 		!new_crtc_state->disable_cxsr;  in vlv_compute_intermediate_wm()
1977 	for (level = 0; level < intermediate->num_levels; level++) {  in vlv_compute_intermediate_wm()
1981 			intermediate->wm[level].plane[plane_id] =  in vlv_compute_intermediate_wm()
1982 				min(optimal->wm[level].plane[plane_id],  in vlv_compute_intermediate_wm()
1983 				    active->wm[level].plane[plane_id]);  in vlv_compute_intermediate_wm()
1986 		intermediate->sr[level].plane = min(optimal->sr[level].plane,  in vlv_compute_intermediate_wm()
1987 						    active->sr[level].plane);  in vlv_compute_intermediate_wm()
1988 		intermediate->sr[level].cursor = min(optimal->sr[level].cursor,  in vlv_compute_intermediate_wm()
1989 						     active->sr[level].cursor);  in vlv_compute_intermediate_wm()
1992 	vlv_invalidate_wms(crtc, intermediate, level);  in vlv_compute_intermediate_wm()
1997 	 * omit the post-vblank programming; only update if it's different.  in vlv_compute_intermediate_wm()
2000 		new_crtc_state->wm.need_postvbl_update = true;  in vlv_compute_intermediate_wm()
2027 	wm->level = dev_priv->display.wm.num_levels - 1;  in vlv_merge_wm()
2028 	wm->cxsr = true;  in vlv_merge_wm()
2030 	for_each_intel_crtc(&dev_priv->drm, crtc) {  in vlv_merge_wm()
2031 		const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;  in vlv_merge_wm()
2033 		if (!crtc->active)  in vlv_merge_wm()
2036 		if (!wm_state->cxsr)  in vlv_merge_wm()
2037 			wm->cxsr = false;  in vlv_merge_wm()
2040 		wm->level = min_t(int, wm->level, wm_state->num_levels - 1);  in vlv_merge_wm()
2044 		wm->cxsr = false;  in vlv_merge_wm()
2047 		wm->level = VLV_WM_LEVEL_PM2;  in vlv_merge_wm()
2049 	for_each_intel_crtc(&dev_priv->drm, crtc) {  in vlv_merge_wm()
2050 		const struct vlv_wm_state *wm_state = &crtc->wm.active.vlv;  in vlv_merge_wm()
2051 		enum pipe pipe = crtc->pipe;  in vlv_merge_wm()
2053 		wm->pipe[pipe] = wm_state->wm[wm->level];  in vlv_merge_wm()
2054 		if (crtc->active && wm->cxsr)  in vlv_merge_wm()
2055 			wm->sr = wm_state->sr[wm->level];  in vlv_merge_wm()
2057 		wm->ddl[pipe].plane[PLANE_PRIMARY] = DDL_PRECISION_HIGH | 2;  in vlv_merge_wm()
2058 		wm->ddl[pipe].plane[PLANE_SPRITE0] = DDL_PRECISION_HIGH | 2;  in vlv_merge_wm()
2059 		wm->ddl[pipe].plane[PLANE_SPRITE1] = DDL_PRECISION_HIGH | 2;  in vlv_merge_wm()
2060 		wm->ddl[pipe].plane[PLANE_CURSOR] = DDL_PRECISION_HIGH | 2;  in vlv_merge_wm()
2066 	struct vlv_wm_values *old_wm = &dev_priv->display.wm.vlv;  in vlv_program_watermarks()
2074 	if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))  in vlv_program_watermarks()
2077 	if (is_disabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))  in vlv_program_watermarks()
2080 	if (is_disabling(old_wm->cxsr, new_wm.cxsr, true))  in vlv_program_watermarks()
2085 	if (is_enabling(old_wm->cxsr, new_wm.cxsr, true))  in vlv_program_watermarks()
2088 	if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_PM5))  in vlv_program_watermarks()
2091 	if (is_enabling(old_wm->level, new_wm.level, VLV_WM_LEVEL_DDR_DVFS))  in vlv_program_watermarks()
2100 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);  in vlv_initial_watermarks()
2104 	mutex_lock(&dev_priv->display.wm.wm_mutex);  in vlv_initial_watermarks()
2105 	crtc->wm.active.vlv = crtc_state->wm.vlv.intermediate;  in vlv_initial_watermarks()
2107 	mutex_unlock(&dev_priv->display.wm.wm_mutex);  in vlv_initial_watermarks()
2113 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);  in vlv_optimize_watermarks()
2117 	if (!crtc_state->wm.need_postvbl_update)  in vlv_optimize_watermarks()
2120 	mutex_lock(&dev_priv->display.wm.wm_mutex);  in vlv_optimize_watermarks()
2121 	crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;  in vlv_optimize_watermarks()
2123 	mutex_unlock(&dev_priv->display.wm.wm_mutex);  in vlv_optimize_watermarks()
2136 		/* self-refresh has much higher latency */  in i965_update_wm()
2139 			&crtc->config->hw.pipe_mode;  in i965_update_wm()
2141 			crtc->base.primary->state->fb;  in i965_update_wm()
2142 		int pixel_rate = crtc->config->pixel_rate;  in i965_update_wm()
2143 		int htotal = pipe_mode->crtc_htotal;  in i965_update_wm()
2144 		int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;  in i965_update_wm()
2145 		int cpp = fb->format->cpp[0];  in i965_update_wm()
2151 		srwm = I965_FIFO_SIZE - entries;  in i965_update_wm()
2155 		drm_dbg_kms(&dev_priv->drm,  in i965_update_wm()
2156 			    "self-refresh entries: %d, wm: %d\n",  in i965_update_wm()
2160 					   crtc->base.cursor->state->crtc_w, 4,  in i965_update_wm()
2166 		cursor_sr = i965_cursor_wm_info.fifo_size - entries;  in i965_update_wm()
2170 		drm_dbg_kms(&dev_priv->drm,  in i965_update_wm()
2171 			    "self-refresh watermark: display plane %d "  in i965_update_wm()
2181 	drm_dbg_kms(&dev_priv->drm,  in i965_update_wm()
2182 		    "Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",  in i965_update_wm()
2186 	intel_uncore_write(&dev_priv->uncore, DSPFW1(dev_priv),  in i965_update_wm()
2188 			   FW_WM(8, CURSORB) |  in i965_update_wm()
2189 			   FW_WM(8, PLANEB) |  in i965_update_wm()
2190 			   FW_WM(8, PLANEA));  in i965_update_wm()
2191 	intel_uncore_write(&dev_priv->uncore, DSPFW2(dev_priv),  in i965_update_wm()
2192 			   FW_WM(8, CURSORA) |  in i965_update_wm()
2193 			   FW_WM(8, PLANEC_OLD));  in i965_update_wm()
2195 	intel_uncore_write(&dev_priv->uncore, DSPFW3(dev_priv),  in i965_update_wm()
2207 	struct intel_display *display = &i915->display;  in intel_crtc_for_plane()
2210 	for_each_intel_plane(&i915->drm, plane) {  in intel_crtc_for_plane()
2211 		if (plane->id == PLANE_PRIMARY &&  in intel_crtc_for_plane()
2212 		    plane->i9xx_plane == i9xx_plane)  in intel_crtc_for_plane()
2213 			return intel_crtc_for_pipe(display, plane->pipe);  in intel_crtc_for_plane()
2243 			crtc->base.primary->state->fb;  in i9xx_update_wm()
2249 			cpp = fb->format->cpp[0];  in i9xx_update_wm()
2251 		planea_wm = intel_calculate_wm(dev_priv, crtc->config->pixel_rate,  in i9xx_update_wm()
2255 		planea_wm = fifo_size - wm_info->guard_size;  in i9xx_update_wm()
2256 		if (planea_wm > (long)wm_info->max_wm)  in i9xx_update_wm()
2257 			planea_wm = wm_info->max_wm;  in i9xx_update_wm()
2270 			crtc->base.primary->state->fb;  in i9xx_update_wm()
2276 			cpp = fb->format->cpp[0];  in i9xx_update_wm()
2278 		planeb_wm = intel_calculate_wm(dev_priv, crtc->config->pixel_rate,  in i9xx_update_wm()
2282 		planeb_wm = fifo_size - wm_info->guard_size;  in i9xx_update_wm()
2283 		if (planeb_wm > (long)wm_info->max_wm)  in i9xx_update_wm()
2284 			planeb_wm = wm_info->max_wm;  in i9xx_update_wm()
2287 	drm_dbg_kms(&dev_priv->drm,  in i9xx_update_wm()
2288 		    "FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);  in i9xx_update_wm()
2294 		obj = intel_fb_bo(crtc->base.primary->state->fb);  in i9xx_update_wm()
2296 		/* self-refresh seems busted with untiled */  in i9xx_update_wm()
2306 	/* Play safe and disable self-refresh before adjusting watermarks. */  in i9xx_update_wm()
2311 		/* self-refresh has much higher latency */  in i9xx_update_wm()
2314 			&crtc->config->hw.pipe_mode;  in i9xx_update_wm()
2316 			crtc->base.primary->state->fb;  in i9xx_update_wm()
2317 		int pixel_rate = crtc->config->pixel_rate;  in i9xx_update_wm()
2318 		int htotal = pipe_mode->crtc_htotal;  in i9xx_update_wm()
2319 		int width = drm_rect_width(&crtc->base.primary->state->src) >> 16;  in i9xx_update_wm()
2326 			cpp = fb->format->cpp[0];  in i9xx_update_wm()
2330 		entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);  in i9xx_update_wm()
2331 		drm_dbg_kms(&dev_priv->drm,  in i9xx_update_wm()
2332 			    "self-refresh entries: %d\n", entries);  in i9xx_update_wm()
2333 		srwm = wm_info->fifo_size - entries;  in i9xx_update_wm()
2338 			intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF,  in i9xx_update_wm()
2341 			intel_uncore_write(&dev_priv->uncore, FW_BLC_SELF, srwm & 0x3f);  in i9xx_update_wm()
2344 	drm_dbg_kms(&dev_priv->drm,  in i9xx_update_wm()
2345 		    "Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",  in i9xx_update_wm()
2351 	/* Set request length to 8 cachelines per fetch */  in i9xx_update_wm()
2352 	fwater_lo = fwater_lo | (1 << 24) | (1 << 8);  in i9xx_update_wm()
2353 	fwater_hi = fwater_hi | (1 << 8);  in i9xx_update_wm()
2355 	intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);  in i9xx_update_wm()
2356 	intel_uncore_write(&dev_priv->uncore, FW_BLC2, fwater_hi);  in i9xx_update_wm()
2372 	planea_wm = intel_calculate_wm(dev_priv, crtc->config->pixel_rate,  in i845_update_wm()
2376 	fwater_lo = intel_uncore_read(&dev_priv->uncore, FW_BLC) & ~0xfff;  in i845_update_wm()
2377 	fwater_lo |= (3<<8) | planea_wm;  in i845_update_wm()
2379 	drm_dbg_kms(&dev_priv->drm,  in i845_update_wm()
2380 		    "Setting FIFO watermarks - A: %d\n", planea_wm);  in i845_update_wm()
2382 	intel_uncore_write(&dev_priv->uncore, FW_BLC, fwater_lo);  in i845_update_wm()
2419 	 * extra paranoid to avoid a potential divide-by-zero if we screw up  in ilk_wm_fbc()
2454 	cpp = plane_state->hw.fb->format->cpp[0];  in ilk_compute_pri_wm()
2456 	method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);  in ilk_compute_pri_wm()
2461 	method2 = ilk_wm_method2(crtc_state->pixel_rate,  in ilk_compute_pri_wm()
2462 				 crtc_state->hw.pipe_mode.crtc_htotal,  in ilk_compute_pri_wm()
2463 				 drm_rect_width(&plane_state->uapi.src) >> 16,  in ilk_compute_pri_wm()
2486 	cpp = plane_state->hw.fb->format->cpp[0];  in ilk_compute_spr_wm()
2488 	method1 = ilk_wm_method1(crtc_state->pixel_rate, cpp, mem_value);  in ilk_compute_spr_wm()
2489 	method2 = ilk_wm_method2(crtc_state->pixel_rate,  in ilk_compute_spr_wm()
2490 				 crtc_state->hw.pipe_mode.crtc_htotal,  in ilk_compute_spr_wm()
2491 				 drm_rect_width(&plane_state->uapi.src) >> 16,  in ilk_compute_spr_wm()
2512 	cpp = plane_state->hw.fb->format->cpp[0];  in ilk_compute_cur_wm()
2514 	return ilk_wm_method2(crtc_state->pixel_rate,  in ilk_compute_cur_wm()
2515 			      crtc_state->hw.pipe_mode.crtc_htotal,  in ilk_compute_cur_wm()
2516 			      drm_rect_width(&plane_state->uapi.src) >> 16,  in ilk_compute_cur_wm()
2530 	cpp = plane_state->hw.fb->format->cpp[0];  in ilk_compute_fbc_wm()
2532 	return ilk_wm_fbc(pri_val, drm_rect_width(&plane_state->uapi.src) >> 16,  in ilk_compute_fbc_wm()
2539 	if (DISPLAY_VER(dev_priv) >= 8)  in ilk_display_fifo_size()
2549 		     int level, bool is_sprite)  in ilk_plane_wm_reg_max()  argument
2551 	if (DISPLAY_VER(dev_priv) >= 8)  in ilk_plane_wm_reg_max()
2553 		return level == 0 ? 255 : 2047;  in ilk_plane_wm_reg_max()
2556 		return level == 0 ? 127 : 1023;  in ilk_plane_wm_reg_max()
2559 		return level == 0 ? 127 : 511;  in ilk_plane_wm_reg_max()
2562 		return level == 0 ? 63 : 255;  in ilk_plane_wm_reg_max()
2566 ilk_cursor_wm_reg_max(const struct drm_i915_private *dev_priv, int level)  in ilk_cursor_wm_reg_max()  argument
2569 		return level == 0 ? 63 : 255;  in ilk_cursor_wm_reg_max()
2571 		return level == 0 ? 31 : 63;  in ilk_cursor_wm_reg_max()
2576 	if (DISPLAY_VER(dev_priv) >= 8)  in ilk_fbc_wm_reg_max()
2584 				     int level,  in ilk_plane_wm_max()  argument
2592 	if (is_sprite && !config->sprites_enabled)  in ilk_plane_wm_max()
2596 	if (level == 0 || config->num_pipes_active > 1) {  in ilk_plane_wm_max()
2608 	if (config->sprites_enabled) {  in ilk_plane_wm_max()
2609 		/* level 0 is always calculated with 1:1 split */  in ilk_plane_wm_max()
2610 		if (level > 0 && ddb_partitioning == INTEL_DDB_PART_5_6) {  in ilk_plane_wm_max()
2620 	return min(fifo_size, ilk_plane_wm_reg_max(dev_priv, level, is_sprite));  in ilk_plane_wm_max()
2625 				      int level,  in ilk_cursor_wm_max()  argument
2629 	if (level > 0 && config->num_pipes_active > 1)  in ilk_cursor_wm_max()
2633 	return ilk_cursor_wm_reg_max(dev_priv, level);  in ilk_cursor_wm_max()
2637 				    int level,  in ilk_compute_wm_maximums()  argument
2642 	max->pri = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, false);  in ilk_compute_wm_maximums()
2643 	max->spr = ilk_plane_wm_max(dev_priv, level, config, ddb_partitioning, true);  in ilk_compute_wm_maximums()
2644 	max->cur = ilk_cursor_wm_max(dev_priv, level, config);  in ilk_compute_wm_maximums()
2645 	max->fbc = ilk_fbc_wm_reg_max(dev_priv);  in ilk_compute_wm_maximums()
2649 					int level,  in ilk_compute_wm_reg_maximums()  argument
2652 	max->pri = ilk_plane_wm_reg_max(dev_priv, level, false);  in ilk_compute_wm_reg_maximums()
2653 	max->spr = ilk_plane_wm_reg_max(dev_priv, level, true);  in ilk_compute_wm_reg_maximums()
2654 	max->cur = ilk_cursor_wm_reg_max(dev_priv, level);  in ilk_compute_wm_reg_maximums()
2655 	max->fbc = ilk_fbc_wm_reg_max(dev_priv);  in ilk_compute_wm_reg_maximums()
2659 				  int level,  in ilk_validate_wm_level()  argument
2666 	if (!result->enable)  in ilk_validate_wm_level()
2669 	result->enable = result->pri_val <= max->pri &&  in ilk_validate_wm_level()
2670 			 result->spr_val <= max->spr &&  in ilk_validate_wm_level()
2671 			 result->cur_val <= max->cur;  in ilk_validate_wm_level()
2673 	ret = result->enable;  in ilk_validate_wm_level()
2676 	 * HACK until we can pre-compute everything,  in ilk_validate_wm_level()
2680 	if (level == 0 && !result->enable) {  in ilk_validate_wm_level()
2681 		if (result->pri_val > max->pri)  in ilk_validate_wm_level()
2682 			drm_dbg_kms(&i915->drm,  in ilk_validate_wm_level()
2684 				    level, result->pri_val, max->pri);  in ilk_validate_wm_level()
2685 		if (result->spr_val > max->spr)  in ilk_validate_wm_level()
2686 			drm_dbg_kms(&i915->drm,  in ilk_validate_wm_level()
2688 				    level, result->spr_val, max->spr);  in ilk_validate_wm_level()
2689 		if (result->cur_val > max->cur)  in ilk_validate_wm_level()
2690 			drm_dbg_kms(&i915->drm,  in ilk_validate_wm_level()
2692 				    level, result->cur_val, max->cur);  in ilk_validate_wm_level()
2694 		result->pri_val = min_t(u32, result->pri_val, max->pri);  in ilk_validate_wm_level()
2695 		result->spr_val = min_t(u32, result->spr_val, max->spr);  in ilk_validate_wm_level()
2696 		result->cur_val = min_t(u32, result->cur_val, max->cur);  in ilk_validate_wm_level()
2697 		result->enable = true;  in ilk_validate_wm_level()
2705 				 int level,  in ilk_compute_wm_level()  argument
2712 	u16 pri_latency = dev_priv->display.wm.pri_latency[level];  in ilk_compute_wm_level()
2713 	u16 spr_latency = dev_priv->display.wm.spr_latency[level];  in ilk_compute_wm_level()
2714 	u16 cur_latency = dev_priv->display.wm.cur_latency[level];  in ilk_compute_wm_level()
2717 	if (level > 0) {  in ilk_compute_wm_level()
2724 		result->pri_val = ilk_compute_pri_wm(crtc_state, pristate,  in ilk_compute_wm_level()
2725 						     pri_latency, level);  in ilk_compute_wm_level()
2726 		result->fbc_val = ilk_compute_fbc_wm(crtc_state, pristate, result->pri_val);  in ilk_compute_wm_level()
2730 		result->spr_val = ilk_compute_spr_wm(crtc_state, sprstate, spr_latency);  in ilk_compute_wm_level()
2733 		result->cur_val = ilk_compute_cur_wm(crtc_state, curstate, cur_latency);  in ilk_compute_wm_level()
2735 	result->enable = true;  in ilk_compute_wm_level()
2742 	i915->display.wm.num_levels = 5;  in hsw_read_wm_latency()
2744 	sskpd = intel_uncore_read64(&i915->uncore, MCH_SSKPD);  in hsw_read_wm_latency()
2759 	i915->display.wm.num_levels = 4;  in snb_read_wm_latency()
2761 	sskpd = intel_uncore_read(&i915->uncore, MCH_SSKPD);  in snb_read_wm_latency()
2773 	i915->display.wm.num_levels = 3;  in ilk_read_wm_latency()
2775 	mltr = intel_uncore_read(&i915->uncore, MLTR_ILK);  in ilk_read_wm_latency()
2802 	int level;  in ilk_increase_wm_latency()  local
2808 	for (level = 1; level < dev_priv->display.wm.num_levels; level++)  in ilk_increase_wm_latency()
2809 		wm[level] = max_t(u16, wm[level], DIV_ROUND_UP(min, 5));  in ilk_increase_wm_latency()
2822 	changed = ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.pri_latency, 12);  in snb_wm_latency_quirk()
2823 	changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.spr_latency, 12);  in snb_wm_latency_quirk()
2824 	changed |= ilk_increase_wm_latency(dev_priv, dev_priv->display.wm.cur_latency, 12);  in snb_wm_latency_quirk()
2829 	drm_dbg_kms(&dev_priv->drm,  in snb_wm_latency_quirk()
2831 	intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);  in snb_wm_latency_quirk()
2832 	intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);  in snb_wm_latency_quirk()
2833 	intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);  in snb_wm_latency_quirk()
2849 	if (dev_priv->display.wm.pri_latency[3] == 0 &&  in snb_wm_lp3_irq_quirk()
2850 	    dev_priv->display.wm.spr_latency[3] == 0 &&  in snb_wm_lp3_irq_quirk()
2851 	    dev_priv->display.wm.cur_latency[3] == 0)  in snb_wm_lp3_irq_quirk()
2854 	dev_priv->display.wm.pri_latency[3] = 0;  in snb_wm_lp3_irq_quirk()
2855 	dev_priv->display.wm.spr_latency[3] = 0;  in snb_wm_lp3_irq_quirk()
2856 	dev_priv->display.wm.cur_latency[3] = 0;  in snb_wm_lp3_irq_quirk()
2858 	drm_dbg_kms(&dev_priv->drm,  in snb_wm_lp3_irq_quirk()
2860 	intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);  in snb_wm_lp3_irq_quirk()
2861 	intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);  in snb_wm_lp3_irq_quirk()
2862 	intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);  in snb_wm_lp3_irq_quirk()
2868 		hsw_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);  in ilk_setup_wm_latency()
2870 		snb_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);  in ilk_setup_wm_latency()
2872 		ilk_read_wm_latency(dev_priv, dev_priv->display.wm.pri_latency);  in ilk_setup_wm_latency()
2874 	memcpy(dev_priv->display.wm.spr_latency, dev_priv->display.wm.pri_latency,  in ilk_setup_wm_latency()
2875 	       sizeof(dev_priv->display.wm.pri_latency));  in ilk_setup_wm_latency()
2876 	memcpy(dev_priv->display.wm.cur_latency, dev_priv->display.wm.pri_latency,  in ilk_setup_wm_latency()
2877 	       sizeof(dev_priv->display.wm.pri_latency));  in ilk_setup_wm_latency()
2879 	intel_fixup_spr_wm_latency(dev_priv, dev_priv->display.wm.spr_latency);  in ilk_setup_wm_latency()
2880 	intel_fixup_cur_wm_latency(dev_priv, dev_priv->display.wm.cur_latency);  in ilk_setup_wm_latency()
2882 	intel_print_wm_latency(dev_priv, "Primary", dev_priv->display.wm.pri_latency);  in ilk_setup_wm_latency()
2883 	intel_print_wm_latency(dev_priv, "Sprite", dev_priv->display.wm.spr_latency);  in ilk_setup_wm_latency()
2884 	intel_print_wm_latency(dev_priv, "Cursor", dev_priv->display.wm.cur_latency);  in ilk_setup_wm_latency()
2898 		.sprites_enabled = pipe_wm->sprites_enabled,  in ilk_validate_pipe_wm()
2899 		.sprites_scaled = pipe_wm->sprites_scaled,  in ilk_validate_pipe_wm()
2907 	if (!ilk_validate_wm_level(dev_priv, 0, &max, &pipe_wm->wm[0])) {  in ilk_validate_pipe_wm()
2908 		drm_dbg_kms(&dev_priv->drm, "LP0 watermark invalid\n");  in ilk_validate_pipe_wm()
2919 	struct drm_i915_private *dev_priv = to_i915(state->base.dev);  in ilk_compute_pipe_wm()
2929 	int level, usable_level;  in ilk_compute_pipe_wm()  local
2931 	pipe_wm = &crtc_state->wm.ilk.optimal;  in ilk_compute_pipe_wm()
2934 		if (plane->base.type == DRM_PLANE_TYPE_PRIMARY)  in ilk_compute_pipe_wm()
2936 		else if (plane->base.type == DRM_PLANE_TYPE_OVERLAY)  in ilk_compute_pipe_wm()
2938 		else if (plane->base.type == DRM_PLANE_TYPE_CURSOR)  in ilk_compute_pipe_wm()
2942 	pipe_wm->pipe_enabled = crtc_state->hw.active;  in ilk_compute_pipe_wm()
2943 	pipe_wm->sprites_enabled = crtc_state->active_planes & BIT(PLANE_SPRITE0);  in ilk_compute_pipe_wm()
2944 	pipe_wm->sprites_scaled = crtc_state->scaled_planes & BIT(PLANE_SPRITE0);  in ilk_compute_pipe_wm()
2946 	usable_level = dev_priv->display.wm.num_levels - 1;  in ilk_compute_pipe_wm()
2949 	if (DISPLAY_VER(dev_priv) < 7 && pipe_wm->sprites_enabled)  in ilk_compute_pipe_wm()
2953 	if (pipe_wm->sprites_scaled)  in ilk_compute_pipe_wm()
2956 	memset(&pipe_wm->wm, 0, sizeof(pipe_wm->wm));  in ilk_compute_pipe_wm()
2958 			     pristate, sprstate, curstate, &pipe_wm->wm[0]);  in ilk_compute_pipe_wm()
2961 		return -EINVAL;  in ilk_compute_pipe_wm()
2965 	for (level = 1; level <= usable_level; level++) {  in ilk_compute_pipe_wm()
2966 		struct intel_wm_level *wm = &pipe_wm->wm[level];  in ilk_compute_pipe_wm()
2968 		ilk_compute_wm_level(dev_priv, crtc, level, crtc_state,  in ilk_compute_pipe_wm()
2972 		 * Disable any watermark level that exceeds the  in ilk_compute_pipe_wm()
2976 		if (!ilk_validate_wm_level(dev_priv, level, &max, wm)) {  in ilk_compute_pipe_wm()
2993 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);  in ilk_compute_intermediate_wm()
2998 	struct intel_pipe_wm *intermediate = &new_crtc_state->wm.ilk.intermediate;  in ilk_compute_intermediate_wm()
2999 	const struct intel_pipe_wm *optimal = &new_crtc_state->wm.ilk.optimal;  in ilk_compute_intermediate_wm()
3000 	const struct intel_pipe_wm *active = &old_crtc_state->wm.ilk.optimal;  in ilk_compute_intermediate_wm()
3001 	int level;  in ilk_compute_intermediate_wm()  local
3009 	if (!new_crtc_state->hw.active ||  in ilk_compute_intermediate_wm()
3011 	    state->skip_intermediate_wm)  in ilk_compute_intermediate_wm()
3014 	intermediate->pipe_enabled |= active->pipe_enabled;  in ilk_compute_intermediate_wm()
3015 	intermediate->sprites_enabled |= active->sprites_enabled;  in ilk_compute_intermediate_wm()
3016 	intermediate->sprites_scaled |= active->sprites_scaled;  in ilk_compute_intermediate_wm()
3018 	for (level = 0; level < dev_priv->display.wm.num_levels; level++) {  in ilk_compute_intermediate_wm()
3019 		struct intel_wm_level *intermediate_wm = &intermediate->wm[level];  in ilk_compute_intermediate_wm()
3020 		const struct intel_wm_level *active_wm = &active->wm[level];  in ilk_compute_intermediate_wm()
3022 		intermediate_wm->enable &= active_wm->enable;  in ilk_compute_intermediate_wm()
3023 		intermediate_wm->pri_val = max(intermediate_wm->pri_val,  in ilk_compute_intermediate_wm()
3024 					       active_wm->pri_val);  in ilk_compute_intermediate_wm()
3025 		intermediate_wm->spr_val = max(intermediate_wm->spr_val,  in ilk_compute_intermediate_wm()
3026 					       active_wm->spr_val);  in ilk_compute_intermediate_wm()
3027 		intermediate_wm->cur_val = max(intermediate_wm->cur_val,  in ilk_compute_intermediate_wm()
3028 					       active_wm->cur_val);  in ilk_compute_intermediate_wm()
3029 		intermediate_wm->fbc_val = max(intermediate_wm->fbc_val,  in ilk_compute_intermediate_wm()
3030 					       active_wm->fbc_val);  in ilk_compute_intermediate_wm()
3040 		return -EINVAL;  in ilk_compute_intermediate_wm()
3044 	 * omit the post-vblank programming; only update if it's different.  in ilk_compute_intermediate_wm()
3047 		new_crtc_state->wm.need_postvbl_update = true;  in ilk_compute_intermediate_wm()
3069  * Merge the watermarks from all active pipes for a specific level.
3072 			       int level,  in ilk_merge_wm_level()  argument
3077 	ret_wm->enable = true;  in ilk_merge_wm_level()
3079 	for_each_intel_crtc(&dev_priv->drm, crtc) {  in ilk_merge_wm_level()
3080 		const struct intel_pipe_wm *active = &crtc->wm.active.ilk;  in ilk_merge_wm_level()
3081 		const struct intel_wm_level *wm = &active->wm[level];  in ilk_merge_wm_level()
3083 		if (!active->pipe_enabled)  in ilk_merge_wm_level()
3089 		 * time even if the level is now disabled.  in ilk_merge_wm_level()
3091 		if (!wm->enable)  in ilk_merge_wm_level()
3092 			ret_wm->enable = false;  in ilk_merge_wm_level()
3094 		ret_wm->pri_val = max(ret_wm->pri_val, wm->pri_val);  in ilk_merge_wm_level()
3095 		ret_wm->spr_val = max(ret_wm->spr_val, wm->spr_val);  in ilk_merge_wm_level()
3096 		ret_wm->cur_val = max(ret_wm->cur_val, wm->cur_val);  in ilk_merge_wm_level()
3097 		ret_wm->fbc_val = max(ret_wm->fbc_val, wm->fbc_val);  in ilk_merge_wm_level()
3109 	int level, num_levels = dev_priv->display.wm.num_levels;  in ilk_wm_merge()  local
3110 	int last_enabled_level = num_levels - 1;  in ilk_wm_merge()
3114 	    config->num_pipes_active > 1)  in ilk_wm_merge()
3118 	merged->fbc_wm_enabled = DISPLAY_VER(dev_priv) >= 6;  in ilk_wm_merge()
3120 	/* merge each WM1+ level */  in ilk_wm_merge()
3121 	for (level = 1; level < num_levels; level++) {  in ilk_wm_merge()
3122 		struct intel_wm_level *wm = &merged->wm[level];  in ilk_wm_merge()
3124 		ilk_merge_wm_level(dev_priv, level, wm);  in ilk_wm_merge()
3126 		if (level > last_enabled_level)  in ilk_wm_merge()
3127 			wm->enable = false;  in ilk_wm_merge()
3128 		else if (!ilk_validate_wm_level(dev_priv, level, max, wm))  in ilk_wm_merge()
3130 			last_enabled_level = level - 1;  in ilk_wm_merge()
3134 		 * FBC WMs instead of disabling a WM level.  in ilk_wm_merge()
3136 		if (wm->fbc_val > max->fbc) {  in ilk_wm_merge()
3137 			if (wm->enable)  in ilk_wm_merge()
3138 				merged->fbc_wm_enabled = false;  in ilk_wm_merge()
3139 			wm->fbc_val = 0;  in ilk_wm_merge()
3145 	    dev_priv->display.params.enable_fbc && !merged->fbc_wm_enabled) {  in ilk_wm_merge()
3146 		for (level = 2; level < num_levels; level++) {  in ilk_wm_merge()
3147 			struct intel_wm_level *wm = &merged->wm[level];  in ilk_wm_merge()
3149 			wm->enable = false;  in ilk_wm_merge()
3157 	return wm_lp + (wm_lp >= 2 && pipe_wm->wm[4].enable);  in ilk_wm_lp_to_level()
3162 				      int level)  in ilk_wm_lp_latency()  argument
3165 		return 2 * level;  in ilk_wm_lp_latency()
3167 		return dev_priv->display.wm.pri_latency[level];  in ilk_wm_lp_latency()
3176 	int level, wm_lp;  in ilk_compute_wm_results()  local
3178 	results->enable_fbc_wm = merged->fbc_wm_enabled;  in ilk_compute_wm_results()
3179 	results->partitioning = partitioning;  in ilk_compute_wm_results()
3185 		level = ilk_wm_lp_to_level(wm_lp, merged);  in ilk_compute_wm_results()
3187 		r = &merged->wm[level];  in ilk_compute_wm_results()
3190 		 * Maintain the watermark values even if the level is  in ilk_compute_wm_results()
3193 		results->wm_lp[wm_lp - 1] =  in ilk_compute_wm_results()
3194 			WM_LP_LATENCY(ilk_wm_lp_latency(dev_priv, level)) |  in ilk_compute_wm_results()
3195 			WM_LP_PRIMARY(r->pri_val) |  in ilk_compute_wm_results()
3196 			WM_LP_CURSOR(r->cur_val);  in ilk_compute_wm_results()
3198 		if (r->enable)  in ilk_compute_wm_results()
3199 			results->wm_lp[wm_lp - 1] |= WM_LP_ENABLE;  in ilk_compute_wm_results()
3201 		if (DISPLAY_VER(dev_priv) >= 8)  in ilk_compute_wm_results()
3202 			results->wm_lp[wm_lp - 1] |= WM_LP_FBC_BDW(r->fbc_val);  in ilk_compute_wm_results()
3204 			results->wm_lp[wm_lp - 1] |= WM_LP_FBC_ILK(r->fbc_val);  in ilk_compute_wm_results()
3206 		results->wm_lp_spr[wm_lp - 1] = WM_LP_SPRITE(r->spr_val);  in ilk_compute_wm_results()
3210 		 * level is disabled. Doing otherwise could cause underruns.  in ilk_compute_wm_results()
3212 		if (DISPLAY_VER(dev_priv) < 7 && r->spr_val) {  in ilk_compute_wm_results()
3213 			drm_WARN_ON(&dev_priv->drm, wm_lp != 1);  in ilk_compute_wm_results()
3214 			results->wm_lp_spr[wm_lp - 1] |= WM_LP_SPRITE_ENABLE;  in ilk_compute_wm_results()
3219 	for_each_intel_crtc(&dev_priv->drm, crtc) {  in ilk_compute_wm_results()
3220 		enum pipe pipe = crtc->pipe;  in ilk_compute_wm_results()
3221 		const struct intel_pipe_wm *pipe_wm = &crtc->wm.active.ilk;  in ilk_compute_wm_results()
3222 		const struct intel_wm_level *r = &pipe_wm->wm[0];  in ilk_compute_wm_results()
3224 		if (drm_WARN_ON(&dev_priv->drm, !r->enable))  in ilk_compute_wm_results()
3227 		results->wm_pipe[pipe] =  in ilk_compute_wm_results()
3228 			WM0_PIPE_PRIMARY(r->pri_val) |  in ilk_compute_wm_results()
3229 			WM0_PIPE_SPRITE(r->spr_val) |  in ilk_compute_wm_results()
3230 			WM0_PIPE_CURSOR(r->cur_val);  in ilk_compute_wm_results()
3235  * Find the result with the highest level enabled. Check for enable_fbc_wm in
3236  * case both are at the same level. Prefer r1 in case they're the same.
3243 	int level, level1 = 0, level2 = 0;  in ilk_find_best_result()  local
3245 	for (level = 1; level < dev_priv->display.wm.num_levels; level++) {  in ilk_find_best_result()
3246 		if (r1->wm[level].enable)  in ilk_find_best_result()
3247 			level1 = level;  in ilk_find_best_result()
3248 		if (r2->wm[level].enable)  in ilk_find_best_result()
3249 			level2 = level;  in ilk_find_best_result()
3253 		if (r2->fbc_wm_enabled && !r1->fbc_wm_enabled)  in ilk_find_best_result()
3280 		if (old->wm_pipe[pipe] != new->wm_pipe[pipe]) {  in ilk_compute_wm_dirty()
3287 	if (old->enable_fbc_wm != new->enable_fbc_wm) {  in ilk_compute_wm_dirty()
3293 	if (old->partitioning != new->partitioning) {  in ilk_compute_wm_dirty()
3305 		if (old->wm_lp[wm_lp - 1] != new->wm_lp[wm_lp - 1] ||  in ilk_compute_wm_dirty()
3306 		    old->wm_lp_spr[wm_lp - 1] != new->wm_lp_spr[wm_lp - 1])  in ilk_compute_wm_dirty()
3320 	struct ilk_wm_values *previous = &dev_priv->display.wm.hw;  in _ilk_disable_lp_wm()
3323 	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] & WM_LP_ENABLE) {  in _ilk_disable_lp_wm()
3324 		previous->wm_lp[2] &= ~WM_LP_ENABLE;  in _ilk_disable_lp_wm()
3325 		intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, previous->wm_lp[2]);  in _ilk_disable_lp_wm()
3328 	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] & WM_LP_ENABLE) {  in _ilk_disable_lp_wm()
3329 		previous->wm_lp[1] &= ~WM_LP_ENABLE;  in _ilk_disable_lp_wm()
3330 		intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, previous->wm_lp[1]);  in _ilk_disable_lp_wm()
3333 	if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] & WM_LP_ENABLE) {  in _ilk_disable_lp_wm()
3334 		previous->wm_lp[0] &= ~WM_LP_ENABLE;  in _ilk_disable_lp_wm()
3335 		intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, previous->wm_lp[0]);  in _ilk_disable_lp_wm()
3349  * causes WMs to be re-evaluated, expending some power.
3354 	struct ilk_wm_values *previous = &dev_priv->display.wm.hw;  in ilk_write_wm_values()
3364 		intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_A), results->wm_pipe[0]);  in ilk_write_wm_values()
3366 		intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_B), results->wm_pipe[1]);  in ilk_write_wm_values()
3368 		intel_uncore_write(&dev_priv->uncore, WM0_PIPE_ILK(PIPE_C), results->wm_pipe[2]);  in ilk_write_wm_values()
3372 			intel_uncore_rmw(&dev_priv->uncore, WM_MISC, WM_MISC_DATA_PARTITION_5_6,  in ilk_write_wm_values()
3373 					 results->partitioning == INTEL_DDB_PART_1_2 ? 0 :  in ilk_write_wm_values()
3376 			intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL2, DISP_DATA_PARTITION_5_6,  in ilk_write_wm_values()
3377 					 results->partitioning == INTEL_DDB_PART_1_2 ? 0 :  in ilk_write_wm_values()
3382 		intel_uncore_rmw(&dev_priv->uncore, DISP_ARB_CTL, DISP_FBC_WM_DIS,  in ilk_write_wm_values()
3383 				 results->enable_fbc_wm ? 0 : DISP_FBC_WM_DIS);  in ilk_write_wm_values()
3386 	    previous->wm_lp_spr[0] != results->wm_lp_spr[0])  in ilk_write_wm_values()
3387 		intel_uncore_write(&dev_priv->uncore, WM1S_LP_ILK, results->wm_lp_spr[0]);  in ilk_write_wm_values()
3390 		if (dirty & WM_DIRTY_LP(2) && previous->wm_lp_spr[1] != results->wm_lp_spr[1])  in ilk_write_wm_values()
3391 			intel_uncore_write(&dev_priv->uncore, WM2S_LP_IVB, results->wm_lp_spr[1]);  in ilk_write_wm_values()
3392 		if (dirty & WM_DIRTY_LP(3) && previous->wm_lp_spr[2] != results->wm_lp_spr[2])  in ilk_write_wm_values()
3393 			intel_uncore_write(&dev_priv->uncore, WM3S_LP_IVB, results->wm_lp_spr[2]);  in ilk_write_wm_values()
3396 	if (dirty & WM_DIRTY_LP(1) && previous->wm_lp[0] != results->wm_lp[0])  in ilk_write_wm_values()
3397 		intel_uncore_write(&dev_priv->uncore, WM1_LP_ILK, results->wm_lp[0]);  in ilk_write_wm_values()
3398 	if (dirty & WM_DIRTY_LP(2) && previous->wm_lp[1] != results->wm_lp[1])  in ilk_write_wm_values()
3399 		intel_uncore_write(&dev_priv->uncore, WM2_LP_ILK, results->wm_lp[1]);  in ilk_write_wm_values()
3400 	if (dirty & WM_DIRTY_LP(3) && previous->wm_lp[2] != results->wm_lp[2])  in ilk_write_wm_values()
3401 		intel_uncore_write(&dev_priv->uncore, WM3_LP_ILK, results->wm_lp[2]);  in ilk_write_wm_values()
3403 	dev_priv->display.wm.hw = *results;  in ilk_write_wm_values()
3417 	for_each_intel_crtc(&dev_priv->drm, crtc) {  in ilk_compute_wm_config()
3418 		const struct intel_pipe_wm *wm = &crtc->wm.active.ilk;  in ilk_compute_wm_config()
3420 		if (!wm->pipe_enabled)  in ilk_compute_wm_config()
3423 		config->sprites_enabled |= wm->sprites_enabled;  in ilk_compute_wm_config()
3424 		config->sprites_scaled |= wm->sprites_scaled;  in ilk_compute_wm_config()
3425 		config->num_pipes_active++;  in ilk_compute_wm_config()
3464 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);  in ilk_initial_watermarks()
3468 	mutex_lock(&dev_priv->display.wm.wm_mutex);  in ilk_initial_watermarks()
3469 	crtc->wm.active.ilk = crtc_state->wm.ilk.intermediate;  in ilk_initial_watermarks()
3471 	mutex_unlock(&dev_priv->display.wm.wm_mutex);  in ilk_initial_watermarks()
3477 	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);  in ilk_optimize_watermarks()
3481 	if (!crtc_state->wm.need_postvbl_update)  in ilk_optimize_watermarks()
3484 	mutex_lock(&dev_priv->display.wm.wm_mutex);  in ilk_optimize_watermarks()
3485 	crtc->wm.active.ilk = crtc_state->wm.ilk.optimal;  in ilk_optimize_watermarks()
3487 	mutex_unlock(&dev_priv->display.wm.wm_mutex);  in ilk_optimize_watermarks()
3492 	struct drm_device *dev = crtc->base.dev;  in ilk_pipe_wm_get_hw_state()
3494 	struct ilk_wm_values *hw = &dev_priv->display.wm.hw;  in ilk_pipe_wm_get_hw_state()
3495 	struct intel_crtc_state *crtc_state = to_intel_crtc_state(crtc->base.state);  in ilk_pipe_wm_get_hw_state()
3496 	struct intel_pipe_wm *active = &crtc_state->wm.ilk.optimal;  in ilk_pipe_wm_get_hw_state()
3497 	enum pipe pipe = crtc->pipe;  in ilk_pipe_wm_get_hw_state()
3499 	hw->wm_pipe[pipe] = intel_uncore_read(&dev_priv->uncore, WM0_PIPE_ILK(pipe));  in ilk_pipe_wm_get_hw_state()
3503 	active->pipe_enabled = crtc->active;  in ilk_pipe_wm_get_hw_state()
3505 	if (active->pipe_enabled) {  in ilk_pipe_wm_get_hw_state()
3506 		u32 tmp = hw->wm_pipe[pipe];  in ilk_pipe_wm_get_hw_state()
3514 		active->wm[0].enable = true;  in ilk_pipe_wm_get_hw_state()
3515 		active->wm[0].pri_val = REG_FIELD_GET(WM0_PIPE_PRIMARY_MASK, tmp);  in ilk_pipe_wm_get_hw_state()
3516 		active->wm[0].spr_val = REG_FIELD_GET(WM0_PIPE_SPRITE_MASK, tmp);  in ilk_pipe_wm_get_hw_state()
3517 		active->wm[0].cur_val = REG_FIELD_GET(WM0_PIPE_CURSOR_MASK, tmp);  in ilk_pipe_wm_get_hw_state()
3519 		int level;  in ilk_pipe_wm_get_hw_state()  local
3526 		for (level = 0; level < dev_priv->display.wm.num_levels; level++)  in ilk_pipe_wm_get_hw_state()
3527 			active->wm[level].enable = true;  in ilk_pipe_wm_get_hw_state()
3530 	crtc->wm.active.ilk = *active;  in ilk_pipe_wm_get_hw_state()
3538 	for_each_intel_crtc(state->dev, crtc) {  in ilk_sanitize_watermarks_add_affected()
3545 		if (crtc_state->hw.active) {  in ilk_sanitize_watermarks_add_affected()
3550 			crtc_state->inherited = true;  in ilk_sanitize_watermarks_add_affected()
3554 	drm_for_each_plane(plane, state->dev) {  in ilk_sanitize_watermarks_add_affected()
3586 	if (!dev_priv->display.funcs.wm->optimize_watermarks)  in ilk_wm_sanitize()
3589 	if (drm_WARN_ON(&dev_priv->drm, DISPLAY_VER(dev_priv) >= 9))  in ilk_wm_sanitize()
3592 	state = drm_atomic_state_alloc(&dev_priv->drm);  in ilk_wm_sanitize()
3593 	if (drm_WARN_ON(&dev_priv->drm, !state))  in ilk_wm_sanitize()
3600 	state->acquire_ctx = &ctx;  in ilk_wm_sanitize()
3601 	to_intel_atomic_state(state)->internal = true;  in ilk_wm_sanitize()
3610 		intel_state->skip_intermediate_wm = true;  in ilk_wm_sanitize()
3616 	ret = intel_atomic_check(&dev_priv->drm, state);  in ilk_wm_sanitize()
3622 		crtc_state->wm.need_postvbl_update = true;  in ilk_wm_sanitize()
3625 		to_intel_crtc_state(crtc->base.state)->wm = crtc_state->wm;  in ilk_wm_sanitize()
3629 	if (ret == -EDEADLK) {  in ilk_wm_sanitize()
3644 	 * BIOS-programmed watermarks untouched and hope for the best.  in ilk_wm_sanitize()
3646 	drm_WARN(&dev_priv->drm, ret,  in ilk_wm_sanitize()
3665 	tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1(dev_priv));  in g4x_read_wm_values()
3666 	wm->sr.plane = _FW_WM(tmp, SR);  in g4x_read_wm_values()
3667 	wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);  in g4x_read_wm_values()
3668 	wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEB);  in g4x_read_wm_values()
3669 	wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM(tmp, PLANEA);  in g4x_read_wm_values()
3671 	tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2(dev_priv));  in g4x_read_wm_values()
3672 	wm->fbc_en = tmp & DSPFW_FBC_SR_EN;  in g4x_read_wm_values()
3673 	wm->sr.fbc = _FW_WM(tmp, FBC_SR);  in g4x_read_wm_values()
3674 	wm->hpll.fbc = _FW_WM(tmp, FBC_HPLL_SR);  in g4x_read_wm_values()
3675 	wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEB);  in g4x_read_wm_values()
3676 	wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);  in g4x_read_wm_values()
3677 	wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM(tmp, SPRITEA);  in g4x_read_wm_values()
3679 	tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3(dev_priv));  in g4x_read_wm_values()
3680 	wm->hpll_en = tmp & DSPFW_HPLL_SR_EN;  in g4x_read_wm_values()
3681 	wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);  in g4x_read_wm_values()
3682 	wm->hpll.cursor = _FW_WM(tmp, HPLL_CURSOR);  in g4x_read_wm_values()
3683 	wm->hpll.plane = _FW_WM(tmp, HPLL_SR);  in g4x_read_wm_values()
3693 		tmp = intel_uncore_read(&dev_priv->uncore, VLV_DDL(pipe));  in vlv_read_wm_values()
3695 		wm->ddl[pipe].plane[PLANE_PRIMARY] =  in vlv_read_wm_values()
3697 		wm->ddl[pipe].plane[PLANE_CURSOR] =  in vlv_read_wm_values()
3699 		wm->ddl[pipe].plane[PLANE_SPRITE0] =  in vlv_read_wm_values()
3701 		wm->ddl[pipe].plane[PLANE_SPRITE1] =  in vlv_read_wm_values()
3705 	tmp = intel_uncore_read(&dev_priv->uncore, DSPFW1(dev_priv));  in vlv_read_wm_values()
3706 	wm->sr.plane = _FW_WM(tmp, SR);  in vlv_read_wm_values()
3707 	wm->pipe[PIPE_B].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORB);  in vlv_read_wm_values()
3708 	wm->pipe[PIPE_B].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEB);  in vlv_read_wm_values()
3709 	wm->pipe[PIPE_A].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEA);  in vlv_read_wm_values()
3711 	tmp = intel_uncore_read(&dev_priv->uncore, DSPFW2(dev_priv));  in vlv_read_wm_values()
3712 	wm->pipe[PIPE_A].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEB);  in vlv_read_wm_values()
3713 	wm->pipe[PIPE_A].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORA);  in vlv_read_wm_values()
3714 	wm->pipe[PIPE_A].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEA);  in vlv_read_wm_values()
3716 	tmp = intel_uncore_read(&dev_priv->uncore, DSPFW3(dev_priv));  in vlv_read_wm_values()
3717 	wm->sr.cursor = _FW_WM(tmp, CURSOR_SR);  in vlv_read_wm_values()
3720 		tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7_CHV);  in vlv_read_wm_values()
3721 		wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);  in vlv_read_wm_values()
3722 		wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);  in vlv_read_wm_values()
3724 		tmp = intel_uncore_read(&dev_priv->uncore, DSPFW8_CHV);  in vlv_read_wm_values()
3725 		wm->pipe[PIPE_C].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITEF);  in vlv_read_wm_values()
3726 		wm->pipe[PIPE_C].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEE);  in vlv_read_wm_values()
3728 		tmp = intel_uncore_read(&dev_priv->uncore, DSPFW9_CHV);  in vlv_read_wm_values()
3729 		wm->pipe[PIPE_C].plane[PLANE_PRIMARY] = _FW_WM_VLV(tmp, PLANEC);  in vlv_read_wm_values()
3730 		wm->pipe[PIPE_C].plane[PLANE_CURSOR] = _FW_WM(tmp, CURSORC);  in vlv_read_wm_values()
3732 		tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);  in vlv_read_wm_values()
3733 		wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;  in vlv_read_wm_values()
3734 		wm->pipe[PIPE_C].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEF_HI) << 8;  in vlv_read_wm_values()
3735 		wm->pipe[PIPE_C].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEE_HI) << 8;  in vlv_read_wm_values()
3736 		wm->pipe[PIPE_C].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEC_HI) << 8;  in vlv_read_wm_values()
3737 		wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;  in vlv_read_wm_values()
3738 		wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;  in vlv_read_wm_values()
3739 		wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;  in vlv_read_wm_values()
3740 		wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;  in vlv_read_wm_values()
3741 		wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;  in vlv_read_wm_values()
3742 		wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;  in vlv_read_wm_values()
3744 		tmp = intel_uncore_read(&dev_priv->uncore, DSPFW7);  in vlv_read_wm_values()
3745 		wm->pipe[PIPE_B].plane[PLANE_SPRITE1] = _FW_WM_VLV(tmp, SPRITED);  in vlv_read_wm_values()
3746 		wm->pipe[PIPE_B].plane[PLANE_SPRITE0] = _FW_WM_VLV(tmp, SPRITEC);  in vlv_read_wm_values()
3748 		tmp = intel_uncore_read(&dev_priv->uncore, DSPHOWM);  in vlv_read_wm_values()
3749 		wm->sr.plane |= _FW_WM(tmp, SR_HI) << 9;  in vlv_read_wm_values()
3750 		wm->pipe[PIPE_B].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITED_HI) << 8;  in vlv_read_wm_values()
3751 		wm->pipe[PIPE_B].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEC_HI) << 8;  in vlv_read_wm_values()
3752 		wm->pipe[PIPE_B].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEB_HI) << 8;  in vlv_read_wm_values()
3753 		wm->pipe[PIPE_A].plane[PLANE_SPRITE1] |= _FW_WM(tmp, SPRITEB_HI) << 8;  in vlv_read_wm_values()
3754 		wm->pipe[PIPE_A].plane[PLANE_SPRITE0] |= _FW_WM(tmp, SPRITEA_HI) << 8;  in vlv_read_wm_values()
3755 		wm->pipe[PIPE_A].plane[PLANE_PRIMARY] |= _FW_WM(tmp, PLANEA_HI) << 8;  in vlv_read_wm_values()
3764 	struct g4x_wm_values *wm = &dev_priv->display.wm.g4x;  in g4x_wm_get_hw_state()
3769 	wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF) & FW_BLC_SELF_EN;  in g4x_wm_get_hw_state()
3771 	for_each_intel_crtc(&dev_priv->drm, crtc) {  in g4x_wm_get_hw_state()
3773 			to_intel_crtc_state(crtc->base.state);  in g4x_wm_get_hw_state()
3774 		struct g4x_wm_state *active = &crtc->wm.active.g4x;  in g4x_wm_get_hw_state()
3776 		enum pipe pipe = crtc->pipe;  in g4x_wm_get_hw_state()
3778 		int level, max_level;  in g4x_wm_get_hw_state()  local
3780 		active->cxsr = wm->cxsr;  in g4x_wm_get_hw_state()
3781 		active->hpll_en = wm->hpll_en;  in g4x_wm_get_hw_state()
3782 		active->fbc_en = wm->fbc_en;  in g4x_wm_get_hw_state()
3784 		active->sr = wm->sr;  in g4x_wm_get_hw_state()
3785 		active->hpll = wm->hpll;  in g4x_wm_get_hw_state()
3788 			active->wm.plane[plane_id] =  in g4x_wm_get_hw_state()
3789 				wm->pipe[pipe].plane[plane_id];  in g4x_wm_get_hw_state()
3792 		if (wm->cxsr && wm->hpll_en)  in g4x_wm_get_hw_state()
3794 		else if (wm->cxsr)  in g4x_wm_get_hw_state()
3799 		level = G4X_WM_LEVEL_NORMAL;  in g4x_wm_get_hw_state()
3800 		raw = &crtc_state->wm.g4x.raw[level];  in g4x_wm_get_hw_state()
3802 			raw->plane[plane_id] = active->wm.plane[plane_id];  in g4x_wm_get_hw_state()
3804 		level = G4X_WM_LEVEL_SR;  in g4x_wm_get_hw_state()
3805 		if (level > max_level)  in g4x_wm_get_hw_state()
3808 		raw = &crtc_state->wm.g4x.raw[level];  in g4x_wm_get_hw_state()
3809 		raw->plane[PLANE_PRIMARY] = active->sr.plane;  in g4x_wm_get_hw_state()
3810 		raw->plane[PLANE_CURSOR] = active->sr.cursor;  in g4x_wm_get_hw_state()
3811 		raw->plane[PLANE_SPRITE0] = 0;  in g4x_wm_get_hw_state()
3812 		raw->fbc = active->sr.fbc;  in g4x_wm_get_hw_state()
3814 		level = G4X_WM_LEVEL_HPLL;  in g4x_wm_get_hw_state()
3815 		if (level > max_level)  in g4x_wm_get_hw_state()
3818 		raw = &crtc_state->wm.g4x.raw[level];  in g4x_wm_get_hw_state()
3819 		raw->plane[PLANE_PRIMARY] = active->hpll.plane;  in g4x_wm_get_hw_state()
3820 		raw->plane[PLANE_CURSOR] = active->hpll.cursor;  in g4x_wm_get_hw_state()
3821 		raw->plane[PLANE_SPRITE0] = 0;  in g4x_wm_get_hw_state()
3822 		raw->fbc = active->hpll.fbc;  in g4x_wm_get_hw_state()
3824 		level++;  in g4x_wm_get_hw_state()
3827 			g4x_raw_plane_wm_set(crtc_state, level,  in g4x_wm_get_hw_state()
3829 		g4x_raw_fbc_wm_set(crtc_state, level, USHRT_MAX);  in g4x_wm_get_hw_state()
3831 		g4x_invalidate_wms(crtc, active, level);  in g4x_wm_get_hw_state()
3833 		crtc_state->wm.g4x.optimal = *active;  in g4x_wm_get_hw_state()
3834 		crtc_state->wm.g4x.intermediate = *active;  in g4x_wm_get_hw_state()
3836 		drm_dbg_kms(&dev_priv->drm,  in g4x_wm_get_hw_state()
3839 			    wm->pipe[pipe].plane[PLANE_PRIMARY],  in g4x_wm_get_hw_state()
3840 			    wm->pipe[pipe].plane[PLANE_CURSOR],  in g4x_wm_get_hw_state()
3841 			    wm->pipe[pipe].plane[PLANE_SPRITE0]);  in g4x_wm_get_hw_state()
3844 	drm_dbg_kms(&dev_priv->drm,  in g4x_wm_get_hw_state()
3846 		    wm->sr.plane, wm->sr.cursor, wm->sr.fbc);  in g4x_wm_get_hw_state()
3847 	drm_dbg_kms(&dev_priv->drm,  in g4x_wm_get_hw_state()
3849 		    wm->hpll.plane, wm->hpll.cursor, wm->hpll.fbc);  in g4x_wm_get_hw_state()
3850 	drm_dbg_kms(&dev_priv->drm, "Initial SR=%s HPLL=%s FBC=%s\n",  in g4x_wm_get_hw_state()
3851 		    str_yes_no(wm->cxsr), str_yes_no(wm->hpll_en),  in g4x_wm_get_hw_state()
3852 		    str_yes_no(wm->fbc_en));  in g4x_wm_get_hw_state()
3857 	struct intel_display *display = &dev_priv->display;  in g4x_wm_sanitize()
3861 	mutex_lock(&dev_priv->display.wm.wm_mutex);  in g4x_wm_sanitize()
3863 	for_each_intel_plane(&dev_priv->drm, plane) {  in g4x_wm_sanitize()
3865 			intel_crtc_for_pipe(display, plane->pipe);  in g4x_wm_sanitize()
3867 			to_intel_crtc_state(crtc->base.state);  in g4x_wm_sanitize()
3869 			to_intel_plane_state(plane->base.state);  in g4x_wm_sanitize()
3870 		enum plane_id plane_id = plane->id;  in g4x_wm_sanitize()
3871 		int level;  in g4x_wm_sanitize()  local
3873 		if (plane_state->uapi.visible)  in g4x_wm_sanitize()
3876 		for (level = 0; level < dev_priv->display.wm.num_levels; level++) {  in g4x_wm_sanitize()
3878 				&crtc_state->wm.g4x.raw[level];  in g4x_wm_sanitize()
3880 			raw->plane[plane_id] = 0;  in g4x_wm_sanitize()
3883 				raw->fbc = 0;  in g4x_wm_sanitize()
3887 	for_each_intel_crtc(&dev_priv->drm, crtc) {  in g4x_wm_sanitize()
3889 			to_intel_crtc_state(crtc->base.state);  in g4x_wm_sanitize()
3893 		drm_WARN_ON(&dev_priv->drm, ret);  in g4x_wm_sanitize()
3895 		crtc_state->wm.g4x.intermediate =  in g4x_wm_sanitize()
3896 			crtc_state->wm.g4x.optimal;  in g4x_wm_sanitize()
3897 		crtc->wm.active.g4x = crtc_state->wm.g4x.optimal;  in g4x_wm_sanitize()
3902 	mutex_unlock(&dev_priv->display.wm.wm_mutex);  in g4x_wm_sanitize()
3907 	struct vlv_wm_values *wm = &dev_priv->display.wm.vlv;  in vlv_wm_get_hw_state()
3913 	wm->cxsr = intel_uncore_read(&dev_priv->uncore, FW_BLC_SELF_VLV) & FW_CSPWRDWNEN;  in vlv_wm_get_hw_state()
3914 	wm->level = VLV_WM_LEVEL_PM2;  in vlv_wm_get_hw_state()
3921 			wm->level = VLV_WM_LEVEL_PM5;  in vlv_wm_get_hw_state()
3938 			drm_dbg_kms(&dev_priv->drm,  in vlv_wm_get_hw_state()
3941 			dev_priv->display.wm.num_levels = VLV_WM_LEVEL_PM5 + 1;  in vlv_wm_get_hw_state()
3945 				wm->level = VLV_WM_LEVEL_DDR_DVFS;  in vlv_wm_get_hw_state()
3951 	for_each_intel_crtc(&dev_priv->drm, crtc) {  in vlv_wm_get_hw_state()
3953 			to_intel_crtc_state(crtc->base.state);  in vlv_wm_get_hw_state()
3954 		struct vlv_wm_state *active = &crtc->wm.active.vlv;  in vlv_wm_get_hw_state()
3956 			&crtc_state->wm.vlv.fifo_state;  in vlv_wm_get_hw_state()
3957 		enum pipe pipe = crtc->pipe;  in vlv_wm_get_hw_state()
3959 		int level;  in vlv_wm_get_hw_state()  local
3963 		active->num_levels = wm->level + 1;  in vlv_wm_get_hw_state()
3964 		active->cxsr = wm->cxsr;  in vlv_wm_get_hw_state()
3966 		for (level = 0; level < active->num_levels; level++) {  in vlv_wm_get_hw_state()
3968 				&crtc_state->wm.vlv.raw[level];  in vlv_wm_get_hw_state()
3970 			active->sr[level].plane = wm->sr.plane;  in vlv_wm_get_hw_state()
3971 			active->sr[level].cursor = wm->sr.cursor;  in vlv_wm_get_hw_state()
3974 				active->wm[level].plane[plane_id] =  in vlv_wm_get_hw_state()
3975 					wm->pipe[pipe].plane[plane_id];  in vlv_wm_get_hw_state()
3977 				raw->plane[plane_id] =  in vlv_wm_get_hw_state()
3978 					vlv_invert_wm_value(active->wm[level].plane[plane_id],  in vlv_wm_get_hw_state()
3979 							    fifo_state->plane[plane_id]);  in vlv_wm_get_hw_state()
3984 			vlv_raw_plane_wm_set(crtc_state, level,  in vlv_wm_get_hw_state()
3986 		vlv_invalidate_wms(crtc, active, level);  in vlv_wm_get_hw_state()
3988 		crtc_state->wm.vlv.optimal = *active;  in vlv_wm_get_hw_state()
3989 		crtc_state->wm.vlv.intermediate = *active;  in vlv_wm_get_hw_state()
3991 		drm_dbg_kms(&dev_priv->drm,  in vlv_wm_get_hw_state()
3994 			    wm->pipe[pipe].plane[PLANE_PRIMARY],  in vlv_wm_get_hw_state()
3995 			    wm->pipe[pipe].plane[PLANE_CURSOR],  in vlv_wm_get_hw_state()
3996 			    wm->pipe[pipe].plane[PLANE_SPRITE0],  in vlv_wm_get_hw_state()
3997 			    wm->pipe[pipe].plane[PLANE_SPRITE1]);  in vlv_wm_get_hw_state()
4000 	drm_dbg_kms(&dev_priv->drm,  in vlv_wm_get_hw_state()
4001 		    "Initial watermarks: SR plane=%d, SR cursor=%d level=%d cxsr=%d\n",  in vlv_wm_get_hw_state()
4002 		    wm->sr.plane, wm->sr.cursor, wm->level, wm->cxsr);  in vlv_wm_get_hw_state()
4007 	struct intel_display *display = &dev_priv->display;  in vlv_wm_sanitize()
4011 	mutex_lock(&dev_priv->display.wm.wm_mutex);  in vlv_wm_sanitize()
4013 	for_each_intel_plane(&dev_priv->drm, plane) {  in vlv_wm_sanitize()
4015 			intel_crtc_for_pipe(display, plane->pipe);  in vlv_wm_sanitize()
4017 			to_intel_crtc_state(crtc->base.state);  in vlv_wm_sanitize()
4019 			to_intel_plane_state(plane->base.state);  in vlv_wm_sanitize()
4020 		enum plane_id plane_id = plane->id;  in vlv_wm_sanitize()
4021 		int level;  in vlv_wm_sanitize()  local
4023 		if (plane_state->uapi.visible)  in vlv_wm_sanitize()
4026 		for (level = 0; level < dev_priv->display.wm.num_levels; level++) {  in vlv_wm_sanitize()
4028 				&crtc_state->wm.vlv.raw[level];  in vlv_wm_sanitize()
4030 			raw->plane[plane_id] = 0;  in vlv_wm_sanitize()
4034 	for_each_intel_crtc(&dev_priv->drm, crtc) {  in vlv_wm_sanitize()
4036 			to_intel_crtc_state(crtc->base.state);  in vlv_wm_sanitize()
4040 		drm_WARN_ON(&dev_priv->drm, ret);  in vlv_wm_sanitize()
4042 		crtc_state->wm.vlv.intermediate =  in vlv_wm_sanitize()
4043 			crtc_state->wm.vlv.optimal;  in vlv_wm_sanitize()
4044 		crtc->wm.active.vlv = crtc_state->wm.vlv.optimal;  in vlv_wm_sanitize()
4049 	mutex_unlock(&dev_priv->display.wm.wm_mutex);  in vlv_wm_sanitize()
4058 	intel_uncore_rmw(&dev_priv->uncore, WM3_LP_ILK, WM_LP_ENABLE, 0);  in ilk_init_lp_watermarks()
4059 	intel_uncore_rmw(&dev_priv->uncore, WM2_LP_ILK, WM_LP_ENABLE, 0);  in ilk_init_lp_watermarks()
4060 	intel_uncore_rmw(&dev_priv->uncore, WM1_LP_ILK, WM_LP_ENABLE, 0);  in ilk_init_lp_watermarks()
4070 	struct ilk_wm_values *hw = &dev_priv->display.wm.hw;  in ilk_wm_get_hw_state()
4075 	for_each_intel_crtc(&dev_priv->drm, crtc)  in ilk_wm_get_hw_state()
4078 	hw->wm_lp[0] = intel_uncore_read(&dev_priv->uncore, WM1_LP_ILK);  in ilk_wm_get_hw_state()
4079 	hw->wm_lp[1] = intel_uncore_read(&dev_priv->uncore, WM2_LP_ILK);  in ilk_wm_get_hw_state()
4080 	hw->wm_lp[2] = intel_uncore_read(&dev_priv->uncore, WM3_LP_ILK);  in ilk_wm_get_hw_state()
4082 	hw->wm_lp_spr[0] = intel_uncore_read(&dev_priv->uncore, WM1S_LP_ILK);  in ilk_wm_get_hw_state()
4084 		hw->wm_lp_spr[1] = intel_uncore_read(&dev_priv->uncore, WM2S_LP_IVB);  in ilk_wm_get_hw_state()
4085 		hw->wm_lp_spr[2] = intel_uncore_read(&dev_priv->uncore, WM3S_LP_IVB);  in ilk_wm_get_hw_state()
4089 		hw->partitioning = (intel_uncore_read(&dev_priv->uncore, WM_MISC) &  in ilk_wm_get_hw_state()
4093 		hw->partitioning = (intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL2) &  in ilk_wm_get_hw_state()
4097 	hw->enable_fbc_wm =  in ilk_wm_get_hw_state()
4098 		!(intel_uncore_read(&dev_priv->uncore, DISP_ARB_CTL) & DISP_FBC_WM_DIS);  in ilk_wm_get_hw_state()
4153 		dev_priv->display.funcs.wm = &ilk_wm_funcs;  in i9xx_wm_init()
4156 		dev_priv->display.funcs.wm = &vlv_wm_funcs;  in i9xx_wm_init()
4159 		dev_priv->display.funcs.wm = &g4x_wm_funcs;  in i9xx_wm_init()
4162 			drm_info(&dev_priv->drm, "Unknown FSB/MEM, disabling CxSR\n");  in i9xx_wm_init()
4165 			dev_priv->display.funcs.wm = &nop_funcs;  in i9xx_wm_init()
4167 			dev_priv->display.funcs.wm = &pnv_wm_funcs;  in i9xx_wm_init()
4170 		dev_priv->display.funcs.wm = &i965_wm_funcs;  in i9xx_wm_init()
4172 		dev_priv->display.funcs.wm = &i9xx_wm_funcs;  in i9xx_wm_init()
4175 			dev_priv->display.funcs.wm = &i845_wm_funcs;  in i9xx_wm_init()
4177 			dev_priv->display.funcs.wm = &i9xx_wm_funcs;  in i9xx_wm_init()
4179 		drm_err(&dev_priv->drm,  in i9xx_wm_init()
4180 			"unexpected fall-through in %s\n", __func__);  in i9xx_wm_init()
4181 		dev_priv->display.funcs.wm = &nop_funcs;  in i9xx_wm_init()