1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2019 Intel Corporation
4 */
5
6 #include <drm/drm_atomic_state_helper.h>
7
8 #include "soc/intel_dram.h"
9
10 #include "i915_drv.h"
11 #include "i915_reg.h"
12 #include "i915_utils.h"
13 #include "intel_atomic.h"
14 #include "intel_bw.h"
15 #include "intel_cdclk.h"
16 #include "intel_display_core.h"
17 #include "intel_display_regs.h"
18 #include "intel_display_types.h"
19 #include "intel_mchbar_regs.h"
20 #include "intel_pcode.h"
21 #include "intel_uncore.h"
22 #include "skl_watermark.h"
23
24 struct intel_dbuf_bw {
25 unsigned int max_bw[I915_MAX_DBUF_SLICES];
26 u8 active_planes[I915_MAX_DBUF_SLICES];
27 };
28
29 struct intel_bw_state {
30 struct intel_global_state base;
31 struct intel_dbuf_bw dbuf_bw[I915_MAX_PIPES];
32
33 /*
34 * Contains a bit mask, used to determine, whether correspondent
35 * pipe allows SAGV or not.
36 */
37 u8 pipe_sagv_reject;
38
39 /* bitmask of active pipes */
40 u8 active_pipes;
41
42 /*
43 * From MTL onwards, to lock a QGV point, punit expects the peak BW of
44 * the selected QGV point as the parameter in multiples of 100MB/s
45 */
46 u16 qgv_point_peakbw;
47
48 /*
49 * Current QGV points mask, which restricts
50 * some particular SAGV states, not to confuse
51 * with pipe_sagv_mask.
52 */
53 u16 qgv_points_mask;
54
55 unsigned int data_rate[I915_MAX_PIPES];
56 u8 num_active_planes[I915_MAX_PIPES];
57 };
58
59 /* Parameters for Qclk Geyserville (QGV) */
60 struct intel_qgv_point {
61 u16 dclk, t_rp, t_rdpre, t_rc, t_ras, t_rcd;
62 };
63
64 #define DEPROGBWPCLIMIT 60
65
66 struct intel_psf_gv_point {
67 u8 clk; /* clock in multiples of 16.6666 MHz */
68 };
69
70 struct intel_qgv_info {
71 struct intel_qgv_point points[I915_NUM_QGV_POINTS];
72 struct intel_psf_gv_point psf_points[I915_NUM_PSF_GV_POINTS];
73 u8 num_points;
74 u8 num_psf_points;
75 u8 t_bl;
76 u8 max_numchannels;
77 u8 channel_width;
78 u8 deinterleave;
79 };
80
dg1_mchbar_read_qgv_point_info(struct intel_display * display,struct intel_qgv_point * sp,int point)81 static int dg1_mchbar_read_qgv_point_info(struct intel_display *display,
82 struct intel_qgv_point *sp,
83 int point)
84 {
85 struct drm_i915_private *i915 = to_i915(display->drm);
86 u32 dclk_ratio, dclk_reference;
87 u32 val;
88
89 val = intel_uncore_read(&i915->uncore, SA_PERF_STATUS_0_0_0_MCHBAR_PC);
90 dclk_ratio = REG_FIELD_GET(DG1_QCLK_RATIO_MASK, val);
91 if (val & DG1_QCLK_REFERENCE)
92 dclk_reference = 6; /* 6 * 16.666 MHz = 100 MHz */
93 else
94 dclk_reference = 8; /* 8 * 16.666 MHz = 133 MHz */
95 sp->dclk = DIV_ROUND_UP((16667 * dclk_ratio * dclk_reference) + 500, 1000);
96
97 val = intel_uncore_read(&i915->uncore, SKL_MC_BIOS_DATA_0_0_0_MCHBAR_PCU);
98 if (val & DG1_GEAR_TYPE)
99 sp->dclk *= 2;
100
101 if (sp->dclk == 0)
102 return -EINVAL;
103
104 val = intel_uncore_read(&i915->uncore, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR);
105 sp->t_rp = REG_FIELD_GET(DG1_DRAM_T_RP_MASK, val);
106 sp->t_rdpre = REG_FIELD_GET(DG1_DRAM_T_RDPRE_MASK, val);
107
108 val = intel_uncore_read(&i915->uncore, MCHBAR_CH0_CR_TC_PRE_0_0_0_MCHBAR_HIGH);
109 sp->t_rcd = REG_FIELD_GET(DG1_DRAM_T_RCD_MASK, val);
110 sp->t_ras = REG_FIELD_GET(DG1_DRAM_T_RAS_MASK, val);
111
112 sp->t_rc = sp->t_rp + sp->t_ras;
113
114 return 0;
115 }
116
icl_pcode_read_qgv_point_info(struct intel_display * display,struct intel_qgv_point * sp,int point)117 static int icl_pcode_read_qgv_point_info(struct intel_display *display,
118 struct intel_qgv_point *sp,
119 int point)
120 {
121 u32 val = 0, val2 = 0;
122 u16 dclk;
123 int ret;
124
125 ret = intel_pcode_read(display->drm, ICL_PCODE_MEM_SUBSYSYSTEM_INFO |
126 ICL_PCODE_MEM_SS_READ_QGV_POINT_INFO(point),
127 &val, &val2);
128 if (ret)
129 return ret;
130
131 dclk = val & 0xffff;
132 sp->dclk = DIV_ROUND_UP((16667 * dclk) + (DISPLAY_VER(display) >= 12 ? 500 : 0),
133 1000);
134 sp->t_rp = (val & 0xff0000) >> 16;
135 sp->t_rcd = (val & 0xff000000) >> 24;
136
137 sp->t_rdpre = val2 & 0xff;
138 sp->t_ras = (val2 & 0xff00) >> 8;
139
140 sp->t_rc = sp->t_rp + sp->t_ras;
141
142 return 0;
143 }
144
adls_pcode_read_psf_gv_point_info(struct intel_display * display,struct intel_psf_gv_point * points)145 static int adls_pcode_read_psf_gv_point_info(struct intel_display *display,
146 struct intel_psf_gv_point *points)
147 {
148 u32 val = 0;
149 int ret;
150 int i;
151
152 ret = intel_pcode_read(display->drm, ICL_PCODE_MEM_SUBSYSYSTEM_INFO |
153 ADL_PCODE_MEM_SS_READ_PSF_GV_INFO, &val, NULL);
154 if (ret)
155 return ret;
156
157 for (i = 0; i < I915_NUM_PSF_GV_POINTS; i++) {
158 points[i].clk = val & 0xff;
159 val >>= 8;
160 }
161
162 return 0;
163 }
164
icl_qgv_points_mask(struct intel_display * display)165 static u16 icl_qgv_points_mask(struct intel_display *display)
166 {
167 unsigned int num_psf_gv_points = display->bw.max[0].num_psf_gv_points;
168 unsigned int num_qgv_points = display->bw.max[0].num_qgv_points;
169 u16 qgv_points = 0, psf_points = 0;
170
171 /*
172 * We can _not_ use the whole ADLS_QGV_PT_MASK here, as PCode rejects
173 * it with failure if we try masking any unadvertised points.
174 * So need to operate only with those returned from PCode.
175 */
176 if (num_qgv_points > 0)
177 qgv_points = GENMASK(num_qgv_points - 1, 0);
178
179 if (num_psf_gv_points > 0)
180 psf_points = GENMASK(num_psf_gv_points - 1, 0);
181
182 return ICL_PCODE_REQ_QGV_PT(qgv_points) | ADLS_PCODE_REQ_PSF_PT(psf_points);
183 }
184
is_sagv_enabled(struct intel_display * display,u16 points_mask)185 static bool is_sagv_enabled(struct intel_display *display, u16 points_mask)
186 {
187 return !is_power_of_2(~points_mask & icl_qgv_points_mask(display) &
188 ICL_PCODE_REQ_QGV_PT_MASK);
189 }
190
icl_pcode_restrict_qgv_points(struct intel_display * display,u32 points_mask)191 static int icl_pcode_restrict_qgv_points(struct intel_display *display,
192 u32 points_mask)
193 {
194 int ret;
195
196 if (DISPLAY_VER(display) >= 14)
197 return 0;
198
199 /* bspec says to keep retrying for at least 1 ms */
200 ret = intel_pcode_request(display->drm, ICL_PCODE_SAGV_DE_MEM_SS_CONFIG,
201 points_mask,
202 ICL_PCODE_REP_QGV_MASK | ADLS_PCODE_REP_PSF_MASK,
203 ICL_PCODE_REP_QGV_SAFE | ADLS_PCODE_REP_PSF_SAFE,
204 1);
205
206 if (ret < 0) {
207 drm_err(display->drm,
208 "Failed to disable qgv points (0x%x) points: 0x%x\n",
209 ret, points_mask);
210 return ret;
211 }
212
213 display->sagv.status = is_sagv_enabled(display, points_mask) ?
214 I915_SAGV_ENABLED : I915_SAGV_DISABLED;
215
216 return 0;
217 }
218
mtl_read_qgv_point_info(struct intel_display * display,struct intel_qgv_point * sp,int point)219 static int mtl_read_qgv_point_info(struct intel_display *display,
220 struct intel_qgv_point *sp, int point)
221 {
222 struct drm_i915_private *i915 = to_i915(display->drm);
223 u32 val, val2;
224 u16 dclk;
225
226 val = intel_uncore_read(&i915->uncore,
227 MTL_MEM_SS_INFO_QGV_POINT_LOW(point));
228 val2 = intel_uncore_read(&i915->uncore,
229 MTL_MEM_SS_INFO_QGV_POINT_HIGH(point));
230 dclk = REG_FIELD_GET(MTL_DCLK_MASK, val);
231 sp->dclk = DIV_ROUND_CLOSEST(16667 * dclk, 1000);
232 sp->t_rp = REG_FIELD_GET(MTL_TRP_MASK, val);
233 sp->t_rcd = REG_FIELD_GET(MTL_TRCD_MASK, val);
234
235 sp->t_rdpre = REG_FIELD_GET(MTL_TRDPRE_MASK, val2);
236 sp->t_ras = REG_FIELD_GET(MTL_TRAS_MASK, val2);
237
238 sp->t_rc = sp->t_rp + sp->t_ras;
239
240 return 0;
241 }
242
243 static int
intel_read_qgv_point_info(struct intel_display * display,struct intel_qgv_point * sp,int point)244 intel_read_qgv_point_info(struct intel_display *display,
245 struct intel_qgv_point *sp,
246 int point)
247 {
248 if (DISPLAY_VER(display) >= 14)
249 return mtl_read_qgv_point_info(display, sp, point);
250 else if (display->platform.dg1)
251 return dg1_mchbar_read_qgv_point_info(display, sp, point);
252 else
253 return icl_pcode_read_qgv_point_info(display, sp, point);
254 }
255
icl_get_qgv_points(struct intel_display * display,const struct dram_info * dram_info,struct intel_qgv_info * qi,bool is_y_tile)256 static int icl_get_qgv_points(struct intel_display *display,
257 const struct dram_info *dram_info,
258 struct intel_qgv_info *qi,
259 bool is_y_tile)
260 {
261 int i, ret;
262
263 qi->num_points = dram_info->num_qgv_points;
264 qi->num_psf_points = dram_info->num_psf_gv_points;
265
266 if (DISPLAY_VER(display) >= 14) {
267 switch (dram_info->type) {
268 case INTEL_DRAM_DDR4:
269 qi->t_bl = 4;
270 qi->max_numchannels = 2;
271 qi->channel_width = 64;
272 qi->deinterleave = 2;
273 break;
274 case INTEL_DRAM_DDR5:
275 qi->t_bl = 8;
276 qi->max_numchannels = 4;
277 qi->channel_width = 32;
278 qi->deinterleave = 2;
279 break;
280 case INTEL_DRAM_LPDDR4:
281 case INTEL_DRAM_LPDDR5:
282 qi->t_bl = 16;
283 qi->max_numchannels = 8;
284 qi->channel_width = 16;
285 qi->deinterleave = 4;
286 break;
287 case INTEL_DRAM_GDDR:
288 case INTEL_DRAM_GDDR_ECC:
289 qi->channel_width = 32;
290 break;
291 default:
292 MISSING_CASE(dram_info->type);
293 return -EINVAL;
294 }
295 } else if (DISPLAY_VER(display) >= 12) {
296 switch (dram_info->type) {
297 case INTEL_DRAM_DDR4:
298 qi->t_bl = is_y_tile ? 8 : 4;
299 qi->max_numchannels = 2;
300 qi->channel_width = 64;
301 qi->deinterleave = is_y_tile ? 1 : 2;
302 break;
303 case INTEL_DRAM_DDR5:
304 qi->t_bl = is_y_tile ? 16 : 8;
305 qi->max_numchannels = 4;
306 qi->channel_width = 32;
307 qi->deinterleave = is_y_tile ? 1 : 2;
308 break;
309 case INTEL_DRAM_LPDDR4:
310 if (display->platform.rocketlake) {
311 qi->t_bl = 8;
312 qi->max_numchannels = 4;
313 qi->channel_width = 32;
314 qi->deinterleave = 2;
315 break;
316 }
317 fallthrough;
318 case INTEL_DRAM_LPDDR5:
319 qi->t_bl = 16;
320 qi->max_numchannels = 8;
321 qi->channel_width = 16;
322 qi->deinterleave = is_y_tile ? 2 : 4;
323 break;
324 default:
325 qi->t_bl = 16;
326 qi->max_numchannels = 1;
327 break;
328 }
329 } else if (DISPLAY_VER(display) == 11) {
330 qi->t_bl = dram_info->type == INTEL_DRAM_DDR4 ? 4 : 8;
331 qi->max_numchannels = 1;
332 }
333
334 if (drm_WARN_ON(display->drm,
335 qi->num_points > ARRAY_SIZE(qi->points)))
336 qi->num_points = ARRAY_SIZE(qi->points);
337
338 for (i = 0; i < qi->num_points; i++) {
339 struct intel_qgv_point *sp = &qi->points[i];
340
341 ret = intel_read_qgv_point_info(display, sp, i);
342 if (ret) {
343 drm_dbg_kms(display->drm, "Could not read QGV %d info\n", i);
344 return ret;
345 }
346
347 drm_dbg_kms(display->drm,
348 "QGV %d: DCLK=%d tRP=%d tRDPRE=%d tRAS=%d tRCD=%d tRC=%d\n",
349 i, sp->dclk, sp->t_rp, sp->t_rdpre, sp->t_ras,
350 sp->t_rcd, sp->t_rc);
351 }
352
353 if (qi->num_psf_points > 0) {
354 ret = adls_pcode_read_psf_gv_point_info(display, qi->psf_points);
355 if (ret) {
356 drm_err(display->drm, "Failed to read PSF point data; PSF points will not be considered in bandwidth calculations.\n");
357 qi->num_psf_points = 0;
358 }
359
360 for (i = 0; i < qi->num_psf_points; i++)
361 drm_dbg_kms(display->drm,
362 "PSF GV %d: CLK=%d \n",
363 i, qi->psf_points[i].clk);
364 }
365
366 return 0;
367 }
368
adl_calc_psf_bw(int clk)369 static int adl_calc_psf_bw(int clk)
370 {
371 /*
372 * clk is multiples of 16.666MHz (100/6)
373 * According to BSpec PSF GV bandwidth is
374 * calculated as BW = 64 * clk * 16.666Mhz
375 */
376 return DIV_ROUND_CLOSEST(64 * clk * 100, 6);
377 }
378
icl_sagv_max_dclk(const struct intel_qgv_info * qi)379 static int icl_sagv_max_dclk(const struct intel_qgv_info *qi)
380 {
381 u16 dclk = 0;
382 int i;
383
384 for (i = 0; i < qi->num_points; i++)
385 dclk = max(dclk, qi->points[i].dclk);
386
387 return dclk;
388 }
389
390 struct intel_sa_info {
391 u16 displayrtids;
392 u8 deburst, deprogbwlimit, derating;
393 };
394
395 static const struct intel_sa_info icl_sa_info = {
396 .deburst = 8,
397 .deprogbwlimit = 25, /* GB/s */
398 .displayrtids = 128,
399 .derating = 10,
400 };
401
402 static const struct intel_sa_info tgl_sa_info = {
403 .deburst = 16,
404 .deprogbwlimit = 34, /* GB/s */
405 .displayrtids = 256,
406 .derating = 10,
407 };
408
409 static const struct intel_sa_info rkl_sa_info = {
410 .deburst = 8,
411 .deprogbwlimit = 20, /* GB/s */
412 .displayrtids = 128,
413 .derating = 10,
414 };
415
416 static const struct intel_sa_info adls_sa_info = {
417 .deburst = 16,
418 .deprogbwlimit = 38, /* GB/s */
419 .displayrtids = 256,
420 .derating = 10,
421 };
422
423 static const struct intel_sa_info adlp_sa_info = {
424 .deburst = 16,
425 .deprogbwlimit = 38, /* GB/s */
426 .displayrtids = 256,
427 .derating = 20,
428 };
429
430 static const struct intel_sa_info mtl_sa_info = {
431 .deburst = 32,
432 .deprogbwlimit = 38, /* GB/s */
433 .displayrtids = 256,
434 .derating = 10,
435 };
436
437 static const struct intel_sa_info xe2_hpd_sa_info = {
438 .derating = 30,
439 .deprogbwlimit = 53,
440 /* Other values not used by simplified algorithm */
441 };
442
443 static const struct intel_sa_info xe2_hpd_ecc_sa_info = {
444 .derating = 45,
445 .deprogbwlimit = 53,
446 /* Other values not used by simplified algorithm */
447 };
448
449 static const struct intel_sa_info xe3lpd_sa_info = {
450 .deburst = 32,
451 .deprogbwlimit = 65, /* GB/s */
452 .displayrtids = 256,
453 .derating = 10,
454 };
455
456 static const struct intel_sa_info xe3lpd_3002_sa_info = {
457 .deburst = 32,
458 .deprogbwlimit = 22, /* GB/s */
459 .displayrtids = 256,
460 .derating = 10,
461 };
462
icl_get_bw_info(struct intel_display * display,const struct dram_info * dram_info,const struct intel_sa_info * sa)463 static int icl_get_bw_info(struct intel_display *display,
464 const struct dram_info *dram_info,
465 const struct intel_sa_info *sa)
466 {
467 struct intel_qgv_info qi = {};
468 bool is_y_tile = true; /* assume y tile may be used */
469 int num_channels = max_t(u8, 1, dram_info->num_channels);
470 int ipqdepth, ipqdepthpch = 16;
471 int dclk_max;
472 int maxdebw;
473 int num_groups = ARRAY_SIZE(display->bw.max);
474 int i, ret;
475
476 ret = icl_get_qgv_points(display, dram_info, &qi, is_y_tile);
477 if (ret) {
478 drm_dbg_kms(display->drm,
479 "Failed to get memory subsystem information, ignoring bandwidth limits");
480 return ret;
481 }
482
483 dclk_max = icl_sagv_max_dclk(&qi);
484 maxdebw = min(sa->deprogbwlimit * 1000, dclk_max * 16 * 6 / 10);
485 ipqdepth = min(ipqdepthpch, sa->displayrtids / num_channels);
486 qi.deinterleave = DIV_ROUND_UP(num_channels, is_y_tile ? 4 : 2);
487
488 for (i = 0; i < num_groups; i++) {
489 struct intel_bw_info *bi = &display->bw.max[i];
490 int clpchgroup;
491 int j;
492
493 clpchgroup = (sa->deburst * qi.deinterleave / num_channels) << i;
494 bi->num_planes = (ipqdepth - clpchgroup) / clpchgroup + 1;
495
496 bi->num_qgv_points = qi.num_points;
497 bi->num_psf_gv_points = qi.num_psf_points;
498
499 for (j = 0; j < qi.num_points; j++) {
500 const struct intel_qgv_point *sp = &qi.points[j];
501 int ct, bw;
502
503 /*
504 * Max row cycle time
505 *
506 * FIXME what is the logic behind the
507 * assumed burst length?
508 */
509 ct = max_t(int, sp->t_rc, sp->t_rp + sp->t_rcd +
510 (clpchgroup - 1) * qi.t_bl + sp->t_rdpre);
511 bw = DIV_ROUND_UP(sp->dclk * clpchgroup * 32 * num_channels, ct);
512
513 bi->deratedbw[j] = min(maxdebw,
514 bw * (100 - sa->derating) / 100);
515
516 drm_dbg_kms(display->drm,
517 "BW%d / QGV %d: num_planes=%d deratedbw=%u\n",
518 i, j, bi->num_planes, bi->deratedbw[j]);
519 }
520 }
521 /*
522 * In case if SAGV is disabled in BIOS, we always get 1
523 * SAGV point, but we can't send PCode commands to restrict it
524 * as it will fail and pointless anyway.
525 */
526 if (qi.num_points == 1)
527 display->sagv.status = I915_SAGV_NOT_CONTROLLED;
528 else
529 display->sagv.status = I915_SAGV_ENABLED;
530
531 return 0;
532 }
533
tgl_get_bw_info(struct intel_display * display,const struct dram_info * dram_info,const struct intel_sa_info * sa)534 static int tgl_get_bw_info(struct intel_display *display,
535 const struct dram_info *dram_info,
536 const struct intel_sa_info *sa)
537 {
538 struct intel_qgv_info qi = {};
539 bool is_y_tile = true; /* assume y tile may be used */
540 int num_channels = max_t(u8, 1, dram_info->num_channels);
541 int ipqdepth, ipqdepthpch = 16;
542 int dclk_max;
543 int maxdebw, peakbw;
544 int clperchgroup;
545 int num_groups = ARRAY_SIZE(display->bw.max);
546 int i, ret;
547
548 ret = icl_get_qgv_points(display, dram_info, &qi, is_y_tile);
549 if (ret) {
550 drm_dbg_kms(display->drm,
551 "Failed to get memory subsystem information, ignoring bandwidth limits");
552 return ret;
553 }
554
555 if (DISPLAY_VER(display) < 14 &&
556 (dram_info->type == INTEL_DRAM_LPDDR4 || dram_info->type == INTEL_DRAM_LPDDR5))
557 num_channels *= 2;
558
559 qi.deinterleave = qi.deinterleave ? : DIV_ROUND_UP(num_channels, is_y_tile ? 4 : 2);
560
561 if (num_channels < qi.max_numchannels && DISPLAY_VER(display) >= 12)
562 qi.deinterleave = max(DIV_ROUND_UP(qi.deinterleave, 2), 1);
563
564 if (DISPLAY_VER(display) >= 12 && num_channels > qi.max_numchannels)
565 drm_warn(display->drm, "Number of channels exceeds max number of channels.");
566 if (qi.max_numchannels != 0)
567 num_channels = min_t(u8, num_channels, qi.max_numchannels);
568
569 dclk_max = icl_sagv_max_dclk(&qi);
570
571 peakbw = num_channels * DIV_ROUND_UP(qi.channel_width, 8) * dclk_max;
572 maxdebw = min(sa->deprogbwlimit * 1000, peakbw * DEPROGBWPCLIMIT / 100);
573
574 ipqdepth = min(ipqdepthpch, sa->displayrtids / num_channels);
575 /*
576 * clperchgroup = 4kpagespermempage * clperchperblock,
577 * clperchperblock = 8 / num_channels * interleave
578 */
579 clperchgroup = 4 * DIV_ROUND_UP(8, num_channels) * qi.deinterleave;
580
581 for (i = 0; i < num_groups; i++) {
582 struct intel_bw_info *bi = &display->bw.max[i];
583 struct intel_bw_info *bi_next;
584 int clpchgroup;
585 int j;
586
587 clpchgroup = (sa->deburst * qi.deinterleave / num_channels) << i;
588
589 if (i < num_groups - 1) {
590 bi_next = &display->bw.max[i + 1];
591
592 if (clpchgroup < clperchgroup)
593 bi_next->num_planes = (ipqdepth - clpchgroup) /
594 clpchgroup + 1;
595 else
596 bi_next->num_planes = 0;
597 }
598
599 bi->num_qgv_points = qi.num_points;
600 bi->num_psf_gv_points = qi.num_psf_points;
601
602 for (j = 0; j < qi.num_points; j++) {
603 const struct intel_qgv_point *sp = &qi.points[j];
604 int ct, bw;
605
606 /*
607 * Max row cycle time
608 *
609 * FIXME what is the logic behind the
610 * assumed burst length?
611 */
612 ct = max_t(int, sp->t_rc, sp->t_rp + sp->t_rcd +
613 (clpchgroup - 1) * qi.t_bl + sp->t_rdpre);
614 bw = DIV_ROUND_UP(sp->dclk * clpchgroup * 32 * num_channels, ct);
615
616 bi->deratedbw[j] = min(maxdebw,
617 bw * (100 - sa->derating) / 100);
618 bi->peakbw[j] = DIV_ROUND_CLOSEST(sp->dclk *
619 num_channels *
620 qi.channel_width, 8);
621
622 drm_dbg_kms(display->drm,
623 "BW%d / QGV %d: num_planes=%d deratedbw=%u peakbw: %u\n",
624 i, j, bi->num_planes, bi->deratedbw[j],
625 bi->peakbw[j]);
626 }
627
628 for (j = 0; j < qi.num_psf_points; j++) {
629 const struct intel_psf_gv_point *sp = &qi.psf_points[j];
630
631 bi->psf_bw[j] = adl_calc_psf_bw(sp->clk);
632
633 drm_dbg_kms(display->drm,
634 "BW%d / PSF GV %d: num_planes=%d bw=%u\n",
635 i, j, bi->num_planes, bi->psf_bw[j]);
636 }
637 }
638
639 /*
640 * In case if SAGV is disabled in BIOS, we always get 1
641 * SAGV point, but we can't send PCode commands to restrict it
642 * as it will fail and pointless anyway.
643 */
644 if (qi.num_points == 1)
645 display->sagv.status = I915_SAGV_NOT_CONTROLLED;
646 else
647 display->sagv.status = I915_SAGV_ENABLED;
648
649 return 0;
650 }
651
dg2_get_bw_info(struct intel_display * display)652 static void dg2_get_bw_info(struct intel_display *display)
653 {
654 unsigned int deratedbw = display->platform.dg2_g11 ? 38000 : 50000;
655 int num_groups = ARRAY_SIZE(display->bw.max);
656 int i;
657
658 /*
659 * DG2 doesn't have SAGV or QGV points, just a constant max bandwidth
660 * that doesn't depend on the number of planes enabled. So fill all the
661 * plane group with constant bw information for uniformity with other
662 * platforms. DG2-G10 platforms have a constant 50 GB/s bandwidth,
663 * whereas DG2-G11 platforms have 38 GB/s.
664 */
665 for (i = 0; i < num_groups; i++) {
666 struct intel_bw_info *bi = &display->bw.max[i];
667
668 bi->num_planes = 1;
669 /* Need only one dummy QGV point per group */
670 bi->num_qgv_points = 1;
671 bi->deratedbw[0] = deratedbw;
672 }
673
674 display->sagv.status = I915_SAGV_NOT_CONTROLLED;
675 }
676
xe2_hpd_get_bw_info(struct intel_display * display,const struct dram_info * dram_info,const struct intel_sa_info * sa)677 static int xe2_hpd_get_bw_info(struct intel_display *display,
678 const struct dram_info *dram_info,
679 const struct intel_sa_info *sa)
680 {
681 struct intel_qgv_info qi = {};
682 int num_channels = dram_info->num_channels;
683 int peakbw, maxdebw;
684 int ret, i;
685
686 ret = icl_get_qgv_points(display, dram_info, &qi, true);
687 if (ret) {
688 drm_dbg_kms(display->drm,
689 "Failed to get memory subsystem information, ignoring bandwidth limits");
690 return ret;
691 }
692
693 peakbw = num_channels * qi.channel_width / 8 * icl_sagv_max_dclk(&qi);
694 maxdebw = min(sa->deprogbwlimit * 1000, peakbw * DEPROGBWPCLIMIT / 10);
695
696 for (i = 0; i < qi.num_points; i++) {
697 const struct intel_qgv_point *point = &qi.points[i];
698 int bw = num_channels * (qi.channel_width / 8) * point->dclk;
699
700 display->bw.max[0].deratedbw[i] =
701 min(maxdebw, (100 - sa->derating) * bw / 100);
702 display->bw.max[0].peakbw[i] = bw;
703
704 drm_dbg_kms(display->drm, "QGV %d: deratedbw=%u peakbw: %u\n",
705 i, display->bw.max[0].deratedbw[i],
706 display->bw.max[0].peakbw[i]);
707 }
708
709 /* Bandwidth does not depend on # of planes; set all groups the same */
710 display->bw.max[0].num_planes = 1;
711 display->bw.max[0].num_qgv_points = qi.num_points;
712 for (i = 1; i < ARRAY_SIZE(display->bw.max); i++)
713 memcpy(&display->bw.max[i], &display->bw.max[0],
714 sizeof(display->bw.max[0]));
715
716 /*
717 * Xe2_HPD should always have exactly two QGV points representing
718 * battery and plugged-in operation.
719 */
720 drm_WARN_ON(display->drm, qi.num_points != 2);
721 display->sagv.status = I915_SAGV_ENABLED;
722
723 return 0;
724 }
725
icl_max_bw_index(struct intel_display * display,int num_planes,int qgv_point)726 static unsigned int icl_max_bw_index(struct intel_display *display,
727 int num_planes, int qgv_point)
728 {
729 int i;
730
731 /*
732 * Let's return max bw for 0 planes
733 */
734 num_planes = max(1, num_planes);
735
736 for (i = 0; i < ARRAY_SIZE(display->bw.max); i++) {
737 const struct intel_bw_info *bi =
738 &display->bw.max[i];
739
740 /*
741 * Pcode will not expose all QGV points when
742 * SAGV is forced to off/min/med/max.
743 */
744 if (qgv_point >= bi->num_qgv_points)
745 return UINT_MAX;
746
747 if (num_planes >= bi->num_planes)
748 return i;
749 }
750
751 return UINT_MAX;
752 }
753
tgl_max_bw_index(struct intel_display * display,int num_planes,int qgv_point)754 static unsigned int tgl_max_bw_index(struct intel_display *display,
755 int num_planes, int qgv_point)
756 {
757 int i;
758
759 /*
760 * Let's return max bw for 0 planes
761 */
762 num_planes = max(1, num_planes);
763
764 for (i = ARRAY_SIZE(display->bw.max) - 1; i >= 0; i--) {
765 const struct intel_bw_info *bi =
766 &display->bw.max[i];
767
768 /*
769 * Pcode will not expose all QGV points when
770 * SAGV is forced to off/min/med/max.
771 */
772 if (qgv_point >= bi->num_qgv_points)
773 return UINT_MAX;
774
775 if (num_planes <= bi->num_planes)
776 return i;
777 }
778
779 return 0;
780 }
781
adl_psf_bw(struct intel_display * display,int psf_gv_point)782 static unsigned int adl_psf_bw(struct intel_display *display,
783 int psf_gv_point)
784 {
785 const struct intel_bw_info *bi =
786 &display->bw.max[0];
787
788 return bi->psf_bw[psf_gv_point];
789 }
790
icl_qgv_bw(struct intel_display * display,int num_active_planes,int qgv_point)791 static unsigned int icl_qgv_bw(struct intel_display *display,
792 int num_active_planes, int qgv_point)
793 {
794 unsigned int idx;
795
796 if (DISPLAY_VER(display) >= 12)
797 idx = tgl_max_bw_index(display, num_active_planes, qgv_point);
798 else
799 idx = icl_max_bw_index(display, num_active_planes, qgv_point);
800
801 if (idx >= ARRAY_SIZE(display->bw.max))
802 return 0;
803
804 return display->bw.max[idx].deratedbw[qgv_point];
805 }
806
intel_bw_init_hw(struct intel_display * display)807 void intel_bw_init_hw(struct intel_display *display)
808 {
809 const struct dram_info *dram_info = intel_dram_info(display->drm);
810
811 if (!HAS_DISPLAY(display))
812 return;
813
814 if (DISPLAY_VERx100(display) >= 3002)
815 tgl_get_bw_info(display, dram_info, &xe3lpd_3002_sa_info);
816 else if (DISPLAY_VER(display) >= 30)
817 tgl_get_bw_info(display, dram_info, &xe3lpd_sa_info);
818 else if (DISPLAY_VERx100(display) >= 1401 && display->platform.dgfx &&
819 dram_info->type == INTEL_DRAM_GDDR_ECC)
820 xe2_hpd_get_bw_info(display, dram_info, &xe2_hpd_ecc_sa_info);
821 else if (DISPLAY_VERx100(display) >= 1401 && display->platform.dgfx)
822 xe2_hpd_get_bw_info(display, dram_info, &xe2_hpd_sa_info);
823 else if (DISPLAY_VER(display) >= 14)
824 tgl_get_bw_info(display, dram_info, &mtl_sa_info);
825 else if (display->platform.dg2)
826 dg2_get_bw_info(display);
827 else if (display->platform.alderlake_p)
828 tgl_get_bw_info(display, dram_info, &adlp_sa_info);
829 else if (display->platform.alderlake_s)
830 tgl_get_bw_info(display, dram_info, &adls_sa_info);
831 else if (display->platform.rocketlake)
832 tgl_get_bw_info(display, dram_info, &rkl_sa_info);
833 else if (DISPLAY_VER(display) == 12)
834 tgl_get_bw_info(display, dram_info, &tgl_sa_info);
835 else if (DISPLAY_VER(display) == 11)
836 icl_get_bw_info(display, dram_info, &icl_sa_info);
837 }
838
intel_bw_crtc_num_active_planes(const struct intel_crtc_state * crtc_state)839 static unsigned int intel_bw_crtc_num_active_planes(const struct intel_crtc_state *crtc_state)
840 {
841 /*
842 * We assume cursors are small enough
843 * to not not cause bandwidth problems.
844 */
845 return hweight8(crtc_state->active_planes & ~BIT(PLANE_CURSOR));
846 }
847
intel_bw_crtc_data_rate(const struct intel_crtc_state * crtc_state)848 static unsigned int intel_bw_crtc_data_rate(const struct intel_crtc_state *crtc_state)
849 {
850 struct intel_display *display = to_intel_display(crtc_state);
851 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
852 unsigned int data_rate = 0;
853 enum plane_id plane_id;
854
855 for_each_plane_id_on_crtc(crtc, plane_id) {
856 /*
857 * We assume cursors are small enough
858 * to not not cause bandwidth problems.
859 */
860 if (plane_id == PLANE_CURSOR)
861 continue;
862
863 data_rate += crtc_state->data_rate[plane_id];
864
865 if (DISPLAY_VER(display) < 11)
866 data_rate += crtc_state->data_rate_y[plane_id];
867 }
868
869 return data_rate;
870 }
871
872 /* "Maximum Pipe Read Bandwidth" */
intel_bw_crtc_min_cdclk(struct intel_display * display,unsigned int data_rate)873 static int intel_bw_crtc_min_cdclk(struct intel_display *display,
874 unsigned int data_rate)
875 {
876 if (DISPLAY_VER(display) < 12)
877 return 0;
878
879 return DIV_ROUND_UP_ULL(mul_u32_u32(data_rate, 10), 512);
880 }
881
intel_bw_num_active_planes(struct intel_display * display,const struct intel_bw_state * bw_state)882 static unsigned int intel_bw_num_active_planes(struct intel_display *display,
883 const struct intel_bw_state *bw_state)
884 {
885 unsigned int num_active_planes = 0;
886 enum pipe pipe;
887
888 for_each_pipe(display, pipe)
889 num_active_planes += bw_state->num_active_planes[pipe];
890
891 return num_active_planes;
892 }
893
intel_bw_data_rate(struct intel_display * display,const struct intel_bw_state * bw_state)894 static unsigned int intel_bw_data_rate(struct intel_display *display,
895 const struct intel_bw_state *bw_state)
896 {
897 struct drm_i915_private *i915 = to_i915(display->drm);
898 unsigned int data_rate = 0;
899 enum pipe pipe;
900
901 for_each_pipe(display, pipe)
902 data_rate += bw_state->data_rate[pipe];
903
904 if (DISPLAY_VER(display) >= 13 && i915_vtd_active(i915))
905 data_rate = DIV_ROUND_UP(data_rate * 105, 100);
906
907 return data_rate;
908 }
909
to_intel_bw_state(struct intel_global_state * obj_state)910 struct intel_bw_state *to_intel_bw_state(struct intel_global_state *obj_state)
911 {
912 return container_of(obj_state, struct intel_bw_state, base);
913 }
914
915 struct intel_bw_state *
intel_atomic_get_old_bw_state(struct intel_atomic_state * state)916 intel_atomic_get_old_bw_state(struct intel_atomic_state *state)
917 {
918 struct intel_display *display = to_intel_display(state);
919 struct intel_global_state *bw_state;
920
921 bw_state = intel_atomic_get_old_global_obj_state(state, &display->bw.obj);
922
923 return to_intel_bw_state(bw_state);
924 }
925
926 struct intel_bw_state *
intel_atomic_get_new_bw_state(struct intel_atomic_state * state)927 intel_atomic_get_new_bw_state(struct intel_atomic_state *state)
928 {
929 struct intel_display *display = to_intel_display(state);
930 struct intel_global_state *bw_state;
931
932 bw_state = intel_atomic_get_new_global_obj_state(state, &display->bw.obj);
933
934 return to_intel_bw_state(bw_state);
935 }
936
937 struct intel_bw_state *
intel_atomic_get_bw_state(struct intel_atomic_state * state)938 intel_atomic_get_bw_state(struct intel_atomic_state *state)
939 {
940 struct intel_display *display = to_intel_display(state);
941 struct intel_global_state *bw_state;
942
943 bw_state = intel_atomic_get_global_obj_state(state, &display->bw.obj);
944 if (IS_ERR(bw_state))
945 return ERR_CAST(bw_state);
946
947 return to_intel_bw_state(bw_state);
948 }
949
icl_max_bw_qgv_point_mask(struct intel_display * display,int num_active_planes)950 static unsigned int icl_max_bw_qgv_point_mask(struct intel_display *display,
951 int num_active_planes)
952 {
953 unsigned int num_qgv_points = display->bw.max[0].num_qgv_points;
954 unsigned int max_bw_point = 0;
955 unsigned int max_bw = 0;
956 int i;
957
958 for (i = 0; i < num_qgv_points; i++) {
959 unsigned int max_data_rate =
960 icl_qgv_bw(display, num_active_planes, i);
961
962 /*
963 * We need to know which qgv point gives us
964 * maximum bandwidth in order to disable SAGV
965 * if we find that we exceed SAGV block time
966 * with watermarks. By that moment we already
967 * have those, as it is calculated earlier in
968 * intel_atomic_check,
969 */
970 if (max_data_rate > max_bw) {
971 max_bw_point = BIT(i);
972 max_bw = max_data_rate;
973 }
974 }
975
976 return max_bw_point;
977 }
978
icl_prepare_qgv_points_mask(struct intel_display * display,unsigned int qgv_points,unsigned int psf_points)979 static u16 icl_prepare_qgv_points_mask(struct intel_display *display,
980 unsigned int qgv_points,
981 unsigned int psf_points)
982 {
983 return ~(ICL_PCODE_REQ_QGV_PT(qgv_points) |
984 ADLS_PCODE_REQ_PSF_PT(psf_points)) & icl_qgv_points_mask(display);
985 }
986
icl_max_bw_psf_gv_point_mask(struct intel_display * display)987 static unsigned int icl_max_bw_psf_gv_point_mask(struct intel_display *display)
988 {
989 unsigned int num_psf_gv_points = display->bw.max[0].num_psf_gv_points;
990 unsigned int max_bw_point_mask = 0;
991 unsigned int max_bw = 0;
992 int i;
993
994 for (i = 0; i < num_psf_gv_points; i++) {
995 unsigned int max_data_rate = adl_psf_bw(display, i);
996
997 if (max_data_rate > max_bw) {
998 max_bw_point_mask = BIT(i);
999 max_bw = max_data_rate;
1000 } else if (max_data_rate == max_bw) {
1001 max_bw_point_mask |= BIT(i);
1002 }
1003 }
1004
1005 return max_bw_point_mask;
1006 }
1007
icl_force_disable_sagv(struct intel_display * display,struct intel_bw_state * bw_state)1008 static void icl_force_disable_sagv(struct intel_display *display,
1009 struct intel_bw_state *bw_state)
1010 {
1011 unsigned int qgv_points = icl_max_bw_qgv_point_mask(display, 0);
1012 unsigned int psf_points = icl_max_bw_psf_gv_point_mask(display);
1013
1014 bw_state->qgv_points_mask = icl_prepare_qgv_points_mask(display,
1015 qgv_points,
1016 psf_points);
1017
1018 drm_dbg_kms(display->drm, "Forcing SAGV disable: mask 0x%x\n",
1019 bw_state->qgv_points_mask);
1020
1021 icl_pcode_restrict_qgv_points(display, bw_state->qgv_points_mask);
1022 }
1023
icl_sagv_pre_plane_update(struct intel_atomic_state * state)1024 void icl_sagv_pre_plane_update(struct intel_atomic_state *state)
1025 {
1026 struct intel_display *display = to_intel_display(state);
1027 const struct intel_bw_state *old_bw_state =
1028 intel_atomic_get_old_bw_state(state);
1029 const struct intel_bw_state *new_bw_state =
1030 intel_atomic_get_new_bw_state(state);
1031 u16 old_mask, new_mask;
1032
1033 if (!new_bw_state)
1034 return;
1035
1036 old_mask = old_bw_state->qgv_points_mask;
1037 new_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
1038
1039 if (old_mask == new_mask)
1040 return;
1041
1042 WARN_ON(!new_bw_state->base.changed);
1043
1044 drm_dbg_kms(display->drm, "Restricting QGV points: 0x%x -> 0x%x\n",
1045 old_mask, new_mask);
1046
1047 /*
1048 * Restrict required qgv points before updating the configuration.
1049 * According to BSpec we can't mask and unmask qgv points at the same
1050 * time. Also masking should be done before updating the configuration
1051 * and unmasking afterwards.
1052 */
1053 icl_pcode_restrict_qgv_points(display, new_mask);
1054 }
1055
icl_sagv_post_plane_update(struct intel_atomic_state * state)1056 void icl_sagv_post_plane_update(struct intel_atomic_state *state)
1057 {
1058 struct intel_display *display = to_intel_display(state);
1059 const struct intel_bw_state *old_bw_state =
1060 intel_atomic_get_old_bw_state(state);
1061 const struct intel_bw_state *new_bw_state =
1062 intel_atomic_get_new_bw_state(state);
1063 u16 old_mask, new_mask;
1064
1065 if (!new_bw_state)
1066 return;
1067
1068 old_mask = old_bw_state->qgv_points_mask | new_bw_state->qgv_points_mask;
1069 new_mask = new_bw_state->qgv_points_mask;
1070
1071 if (old_mask == new_mask)
1072 return;
1073
1074 WARN_ON(!new_bw_state->base.changed);
1075
1076 drm_dbg_kms(display->drm, "Relaxing QGV points: 0x%x -> 0x%x\n",
1077 old_mask, new_mask);
1078
1079 /*
1080 * Allow required qgv points after updating the configuration.
1081 * According to BSpec we can't mask and unmask qgv points at the same
1082 * time. Also masking should be done before updating the configuration
1083 * and unmasking afterwards.
1084 */
1085 icl_pcode_restrict_qgv_points(display, new_mask);
1086 }
1087
mtl_find_qgv_points(struct intel_display * display,unsigned int data_rate,unsigned int num_active_planes,struct intel_bw_state * new_bw_state)1088 static int mtl_find_qgv_points(struct intel_display *display,
1089 unsigned int data_rate,
1090 unsigned int num_active_planes,
1091 struct intel_bw_state *new_bw_state)
1092 {
1093 unsigned int best_rate = UINT_MAX;
1094 unsigned int num_qgv_points = display->bw.max[0].num_qgv_points;
1095 unsigned int qgv_peak_bw = 0;
1096 int i;
1097 int ret;
1098
1099 ret = intel_atomic_lock_global_state(&new_bw_state->base);
1100 if (ret)
1101 return ret;
1102
1103 /*
1104 * If SAGV cannot be enabled, disable the pcode SAGV by passing all 1's
1105 * for qgv peak bw in PM Demand request. So assign UINT_MAX if SAGV is
1106 * not enabled. PM Demand code will clamp the value for the register
1107 */
1108 if (!intel_bw_can_enable_sagv(display, new_bw_state)) {
1109 new_bw_state->qgv_point_peakbw = U16_MAX;
1110 drm_dbg_kms(display->drm, "No SAGV, use UINT_MAX as peak bw.");
1111 return 0;
1112 }
1113
1114 /*
1115 * Find the best QGV point by comparing the data_rate with max data rate
1116 * offered per plane group
1117 */
1118 for (i = 0; i < num_qgv_points; i++) {
1119 unsigned int bw_index =
1120 tgl_max_bw_index(display, num_active_planes, i);
1121 unsigned int max_data_rate;
1122
1123 if (bw_index >= ARRAY_SIZE(display->bw.max))
1124 continue;
1125
1126 max_data_rate = display->bw.max[bw_index].deratedbw[i];
1127
1128 if (max_data_rate < data_rate)
1129 continue;
1130
1131 if (max_data_rate - data_rate < best_rate) {
1132 best_rate = max_data_rate - data_rate;
1133 qgv_peak_bw = display->bw.max[bw_index].peakbw[i];
1134 }
1135
1136 drm_dbg_kms(display->drm, "QGV point %d: max bw %d required %d qgv_peak_bw: %d\n",
1137 i, max_data_rate, data_rate, qgv_peak_bw);
1138 }
1139
1140 drm_dbg_kms(display->drm, "Matching peaks QGV bw: %d for required data rate: %d\n",
1141 qgv_peak_bw, data_rate);
1142
1143 /*
1144 * The display configuration cannot be supported if no QGV point
1145 * satisfying the required data rate is found
1146 */
1147 if (qgv_peak_bw == 0) {
1148 drm_dbg_kms(display->drm, "No QGV points for bw %d for display configuration(%d active planes).\n",
1149 data_rate, num_active_planes);
1150 return -EINVAL;
1151 }
1152
1153 /* MTL PM DEMAND expects QGV BW parameter in multiples of 100 mbps */
1154 new_bw_state->qgv_point_peakbw = DIV_ROUND_CLOSEST(qgv_peak_bw, 100);
1155
1156 return 0;
1157 }
1158
icl_find_qgv_points(struct intel_display * display,unsigned int data_rate,unsigned int num_active_planes,const struct intel_bw_state * old_bw_state,struct intel_bw_state * new_bw_state)1159 static int icl_find_qgv_points(struct intel_display *display,
1160 unsigned int data_rate,
1161 unsigned int num_active_planes,
1162 const struct intel_bw_state *old_bw_state,
1163 struct intel_bw_state *new_bw_state)
1164 {
1165 unsigned int num_psf_gv_points = display->bw.max[0].num_psf_gv_points;
1166 unsigned int num_qgv_points = display->bw.max[0].num_qgv_points;
1167 u16 psf_points = 0;
1168 u16 qgv_points = 0;
1169 int i;
1170 int ret;
1171
1172 ret = intel_atomic_lock_global_state(&new_bw_state->base);
1173 if (ret)
1174 return ret;
1175
1176 for (i = 0; i < num_qgv_points; i++) {
1177 unsigned int max_data_rate = icl_qgv_bw(display,
1178 num_active_planes, i);
1179 if (max_data_rate >= data_rate)
1180 qgv_points |= BIT(i);
1181
1182 drm_dbg_kms(display->drm, "QGV point %d: max bw %d required %d\n",
1183 i, max_data_rate, data_rate);
1184 }
1185
1186 for (i = 0; i < num_psf_gv_points; i++) {
1187 unsigned int max_data_rate = adl_psf_bw(display, i);
1188
1189 if (max_data_rate >= data_rate)
1190 psf_points |= BIT(i);
1191
1192 drm_dbg_kms(display->drm, "PSF GV point %d: max bw %d"
1193 " required %d\n",
1194 i, max_data_rate, data_rate);
1195 }
1196
1197 /*
1198 * BSpec states that we always should have at least one allowed point
1199 * left, so if we couldn't - simply reject the configuration for obvious
1200 * reasons.
1201 */
1202 if (qgv_points == 0) {
1203 drm_dbg_kms(display->drm, "No QGV points provide sufficient memory"
1204 " bandwidth %d for display configuration(%d active planes).\n",
1205 data_rate, num_active_planes);
1206 return -EINVAL;
1207 }
1208
1209 if (num_psf_gv_points > 0 && psf_points == 0) {
1210 drm_dbg_kms(display->drm, "No PSF GV points provide sufficient memory"
1211 " bandwidth %d for display configuration(%d active planes).\n",
1212 data_rate, num_active_planes);
1213 return -EINVAL;
1214 }
1215
1216 /*
1217 * Leave only single point with highest bandwidth, if
1218 * we can't enable SAGV due to the increased memory latency it may
1219 * cause.
1220 */
1221 if (!intel_bw_can_enable_sagv(display, new_bw_state)) {
1222 qgv_points = icl_max_bw_qgv_point_mask(display, num_active_planes);
1223 drm_dbg_kms(display->drm, "No SAGV, using single QGV point mask 0x%x\n",
1224 qgv_points);
1225 }
1226
1227 /*
1228 * We store the ones which need to be masked as that is what PCode
1229 * actually accepts as a parameter.
1230 */
1231 new_bw_state->qgv_points_mask = icl_prepare_qgv_points_mask(display,
1232 qgv_points,
1233 psf_points);
1234 /*
1235 * If the actual mask had changed we need to make sure that
1236 * the commits are serialized(in case this is a nomodeset, nonblocking)
1237 */
1238 if (new_bw_state->qgv_points_mask != old_bw_state->qgv_points_mask) {
1239 ret = intel_atomic_serialize_global_state(&new_bw_state->base);
1240 if (ret)
1241 return ret;
1242 }
1243
1244 return 0;
1245 }
1246
intel_bw_check_qgv_points(struct intel_display * display,const struct intel_bw_state * old_bw_state,struct intel_bw_state * new_bw_state)1247 static int intel_bw_check_qgv_points(struct intel_display *display,
1248 const struct intel_bw_state *old_bw_state,
1249 struct intel_bw_state *new_bw_state)
1250 {
1251 unsigned int data_rate = intel_bw_data_rate(display, new_bw_state);
1252 unsigned int num_active_planes =
1253 intel_bw_num_active_planes(display, new_bw_state);
1254
1255 data_rate = DIV_ROUND_UP(data_rate, 1000);
1256
1257 if (DISPLAY_VER(display) >= 14)
1258 return mtl_find_qgv_points(display, data_rate, num_active_planes,
1259 new_bw_state);
1260 else
1261 return icl_find_qgv_points(display, data_rate, num_active_planes,
1262 old_bw_state, new_bw_state);
1263 }
1264
intel_dbuf_bw_changed(struct intel_display * display,const struct intel_dbuf_bw * old_dbuf_bw,const struct intel_dbuf_bw * new_dbuf_bw)1265 static bool intel_dbuf_bw_changed(struct intel_display *display,
1266 const struct intel_dbuf_bw *old_dbuf_bw,
1267 const struct intel_dbuf_bw *new_dbuf_bw)
1268 {
1269 enum dbuf_slice slice;
1270
1271 for_each_dbuf_slice(display, slice) {
1272 if (old_dbuf_bw->max_bw[slice] != new_dbuf_bw->max_bw[slice] ||
1273 old_dbuf_bw->active_planes[slice] != new_dbuf_bw->active_planes[slice])
1274 return true;
1275 }
1276
1277 return false;
1278 }
1279
intel_bw_state_changed(struct intel_display * display,const struct intel_bw_state * old_bw_state,const struct intel_bw_state * new_bw_state)1280 static bool intel_bw_state_changed(struct intel_display *display,
1281 const struct intel_bw_state *old_bw_state,
1282 const struct intel_bw_state *new_bw_state)
1283 {
1284 enum pipe pipe;
1285
1286 for_each_pipe(display, pipe) {
1287 const struct intel_dbuf_bw *old_dbuf_bw =
1288 &old_bw_state->dbuf_bw[pipe];
1289 const struct intel_dbuf_bw *new_dbuf_bw =
1290 &new_bw_state->dbuf_bw[pipe];
1291
1292 if (intel_dbuf_bw_changed(display, old_dbuf_bw, new_dbuf_bw))
1293 return true;
1294
1295 if (intel_bw_crtc_min_cdclk(display, old_bw_state->data_rate[pipe]) !=
1296 intel_bw_crtc_min_cdclk(display, new_bw_state->data_rate[pipe]))
1297 return true;
1298 }
1299
1300 return false;
1301 }
1302
skl_plane_calc_dbuf_bw(struct intel_dbuf_bw * dbuf_bw,struct intel_crtc * crtc,enum plane_id plane_id,const struct skl_ddb_entry * ddb,unsigned int data_rate)1303 static void skl_plane_calc_dbuf_bw(struct intel_dbuf_bw *dbuf_bw,
1304 struct intel_crtc *crtc,
1305 enum plane_id plane_id,
1306 const struct skl_ddb_entry *ddb,
1307 unsigned int data_rate)
1308 {
1309 struct intel_display *display = to_intel_display(crtc);
1310 unsigned int dbuf_mask = skl_ddb_dbuf_slice_mask(display, ddb);
1311 enum dbuf_slice slice;
1312
1313 /*
1314 * The arbiter can only really guarantee an
1315 * equal share of the total bw to each plane.
1316 */
1317 for_each_dbuf_slice_in_mask(display, slice, dbuf_mask) {
1318 dbuf_bw->max_bw[slice] = max(dbuf_bw->max_bw[slice], data_rate);
1319 dbuf_bw->active_planes[slice] |= BIT(plane_id);
1320 }
1321 }
1322
skl_crtc_calc_dbuf_bw(struct intel_dbuf_bw * dbuf_bw,const struct intel_crtc_state * crtc_state)1323 static void skl_crtc_calc_dbuf_bw(struct intel_dbuf_bw *dbuf_bw,
1324 const struct intel_crtc_state *crtc_state)
1325 {
1326 struct intel_display *display = to_intel_display(crtc_state);
1327 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1328 enum plane_id plane_id;
1329
1330 memset(dbuf_bw, 0, sizeof(*dbuf_bw));
1331
1332 if (!crtc_state->hw.active)
1333 return;
1334
1335 for_each_plane_id_on_crtc(crtc, plane_id) {
1336 /*
1337 * We assume cursors are small enough
1338 * to not cause bandwidth problems.
1339 */
1340 if (plane_id == PLANE_CURSOR)
1341 continue;
1342
1343 skl_plane_calc_dbuf_bw(dbuf_bw, crtc, plane_id,
1344 &crtc_state->wm.skl.plane_ddb[plane_id],
1345 crtc_state->data_rate[plane_id]);
1346
1347 if (DISPLAY_VER(display) < 11)
1348 skl_plane_calc_dbuf_bw(dbuf_bw, crtc, plane_id,
1349 &crtc_state->wm.skl.plane_ddb_y[plane_id],
1350 crtc_state->data_rate[plane_id]);
1351 }
1352 }
1353
1354 /* "Maximum Data Buffer Bandwidth" */
1355 static int
intel_bw_dbuf_min_cdclk(struct intel_display * display,const struct intel_bw_state * bw_state)1356 intel_bw_dbuf_min_cdclk(struct intel_display *display,
1357 const struct intel_bw_state *bw_state)
1358 {
1359 unsigned int total_max_bw = 0;
1360 enum dbuf_slice slice;
1361
1362 for_each_dbuf_slice(display, slice) {
1363 int num_active_planes = 0;
1364 unsigned int max_bw = 0;
1365 enum pipe pipe;
1366
1367 /*
1368 * The arbiter can only really guarantee an
1369 * equal share of the total bw to each plane.
1370 */
1371 for_each_pipe(display, pipe) {
1372 const struct intel_dbuf_bw *dbuf_bw = &bw_state->dbuf_bw[pipe];
1373
1374 max_bw = max(dbuf_bw->max_bw[slice], max_bw);
1375 num_active_planes += hweight8(dbuf_bw->active_planes[slice]);
1376 }
1377 max_bw *= num_active_planes;
1378
1379 total_max_bw = max(total_max_bw, max_bw);
1380 }
1381
1382 return DIV_ROUND_UP(total_max_bw, 64);
1383 }
1384
intel_bw_min_cdclk(struct intel_display * display,const struct intel_bw_state * bw_state)1385 int intel_bw_min_cdclk(struct intel_display *display,
1386 const struct intel_bw_state *bw_state)
1387 {
1388 enum pipe pipe;
1389 int min_cdclk;
1390
1391 min_cdclk = intel_bw_dbuf_min_cdclk(display, bw_state);
1392
1393 for_each_pipe(display, pipe)
1394 min_cdclk = max(min_cdclk,
1395 intel_bw_crtc_min_cdclk(display,
1396 bw_state->data_rate[pipe]));
1397
1398 return min_cdclk;
1399 }
1400
intel_bw_calc_min_cdclk(struct intel_atomic_state * state,bool * need_cdclk_calc)1401 int intel_bw_calc_min_cdclk(struct intel_atomic_state *state,
1402 bool *need_cdclk_calc)
1403 {
1404 struct intel_display *display = to_intel_display(state);
1405 struct intel_bw_state *new_bw_state = NULL;
1406 const struct intel_bw_state *old_bw_state = NULL;
1407 const struct intel_cdclk_state *cdclk_state;
1408 const struct intel_crtc_state *old_crtc_state;
1409 const struct intel_crtc_state *new_crtc_state;
1410 int old_min_cdclk, new_min_cdclk;
1411 struct intel_crtc *crtc;
1412 int i;
1413
1414 if (DISPLAY_VER(display) < 9)
1415 return 0;
1416
1417 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
1418 new_crtc_state, i) {
1419 struct intel_dbuf_bw old_dbuf_bw, new_dbuf_bw;
1420
1421 skl_crtc_calc_dbuf_bw(&old_dbuf_bw, old_crtc_state);
1422 skl_crtc_calc_dbuf_bw(&new_dbuf_bw, new_crtc_state);
1423
1424 if (!intel_dbuf_bw_changed(display, &old_dbuf_bw, &new_dbuf_bw))
1425 continue;
1426
1427 new_bw_state = intel_atomic_get_bw_state(state);
1428 if (IS_ERR(new_bw_state))
1429 return PTR_ERR(new_bw_state);
1430
1431 old_bw_state = intel_atomic_get_old_bw_state(state);
1432
1433 new_bw_state->dbuf_bw[crtc->pipe] = new_dbuf_bw;
1434 }
1435
1436 if (!old_bw_state)
1437 return 0;
1438
1439 if (intel_bw_state_changed(display, old_bw_state, new_bw_state)) {
1440 int ret = intel_atomic_lock_global_state(&new_bw_state->base);
1441 if (ret)
1442 return ret;
1443 }
1444
1445 old_min_cdclk = intel_bw_min_cdclk(display, old_bw_state);
1446 new_min_cdclk = intel_bw_min_cdclk(display, new_bw_state);
1447
1448 /*
1449 * No need to check against the cdclk state if
1450 * the min cdclk doesn't increase.
1451 *
1452 * Ie. we only ever increase the cdclk due to bandwidth
1453 * requirements. This can reduce back and forth
1454 * display blinking due to constant cdclk changes.
1455 */
1456 if (new_min_cdclk <= old_min_cdclk)
1457 return 0;
1458
1459 cdclk_state = intel_atomic_get_cdclk_state(state);
1460 if (IS_ERR(cdclk_state))
1461 return PTR_ERR(cdclk_state);
1462
1463 /*
1464 * No need to recalculate the cdclk state if
1465 * the min cdclk doesn't increase.
1466 *
1467 * Ie. we only ever increase the cdclk due to bandwidth
1468 * requirements. This can reduce back and forth
1469 * display blinking due to constant cdclk changes.
1470 */
1471 if (new_min_cdclk <= intel_cdclk_bw_min_cdclk(cdclk_state))
1472 return 0;
1473
1474 drm_dbg_kms(display->drm,
1475 "new bandwidth min cdclk (%d kHz) > old min cdclk (%d kHz)\n",
1476 new_min_cdclk, intel_cdclk_bw_min_cdclk(cdclk_state));
1477 *need_cdclk_calc = true;
1478
1479 return 0;
1480 }
1481
intel_bw_check_data_rate(struct intel_atomic_state * state,bool * changed)1482 static int intel_bw_check_data_rate(struct intel_atomic_state *state, bool *changed)
1483 {
1484 struct intel_display *display = to_intel_display(state);
1485 const struct intel_crtc_state *new_crtc_state, *old_crtc_state;
1486 struct intel_crtc *crtc;
1487 int i;
1488
1489 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
1490 new_crtc_state, i) {
1491 unsigned int old_data_rate =
1492 intel_bw_crtc_data_rate(old_crtc_state);
1493 unsigned int new_data_rate =
1494 intel_bw_crtc_data_rate(new_crtc_state);
1495 unsigned int old_active_planes =
1496 intel_bw_crtc_num_active_planes(old_crtc_state);
1497 unsigned int new_active_planes =
1498 intel_bw_crtc_num_active_planes(new_crtc_state);
1499 struct intel_bw_state *new_bw_state;
1500
1501 /*
1502 * Avoid locking the bw state when
1503 * nothing significant has changed.
1504 */
1505 if (old_data_rate == new_data_rate &&
1506 old_active_planes == new_active_planes)
1507 continue;
1508
1509 new_bw_state = intel_atomic_get_bw_state(state);
1510 if (IS_ERR(new_bw_state))
1511 return PTR_ERR(new_bw_state);
1512
1513 new_bw_state->data_rate[crtc->pipe] = new_data_rate;
1514 new_bw_state->num_active_planes[crtc->pipe] = new_active_planes;
1515
1516 *changed = true;
1517
1518 drm_dbg_kms(display->drm,
1519 "[CRTC:%d:%s] data rate %u num active planes %u\n",
1520 crtc->base.base.id, crtc->base.name,
1521 new_bw_state->data_rate[crtc->pipe],
1522 new_bw_state->num_active_planes[crtc->pipe]);
1523 }
1524
1525 return 0;
1526 }
1527
intel_bw_modeset_checks(struct intel_atomic_state * state)1528 static int intel_bw_modeset_checks(struct intel_atomic_state *state)
1529 {
1530 struct intel_display *display = to_intel_display(state);
1531 const struct intel_bw_state *old_bw_state;
1532 struct intel_bw_state *new_bw_state;
1533
1534 if (DISPLAY_VER(display) < 9)
1535 return 0;
1536
1537 new_bw_state = intel_atomic_get_bw_state(state);
1538 if (IS_ERR(new_bw_state))
1539 return PTR_ERR(new_bw_state);
1540
1541 old_bw_state = intel_atomic_get_old_bw_state(state);
1542
1543 new_bw_state->active_pipes =
1544 intel_calc_active_pipes(state, old_bw_state->active_pipes);
1545
1546 if (new_bw_state->active_pipes != old_bw_state->active_pipes) {
1547 int ret;
1548
1549 ret = intel_atomic_lock_global_state(&new_bw_state->base);
1550 if (ret)
1551 return ret;
1552 }
1553
1554 return 0;
1555 }
1556
intel_bw_check_sagv_mask(struct intel_atomic_state * state)1557 static int intel_bw_check_sagv_mask(struct intel_atomic_state *state)
1558 {
1559 struct intel_display *display = to_intel_display(state);
1560 const struct intel_crtc_state *old_crtc_state;
1561 const struct intel_crtc_state *new_crtc_state;
1562 const struct intel_bw_state *old_bw_state = NULL;
1563 struct intel_bw_state *new_bw_state = NULL;
1564 struct intel_crtc *crtc;
1565 int ret, i;
1566
1567 for_each_oldnew_intel_crtc_in_state(state, crtc, old_crtc_state,
1568 new_crtc_state, i) {
1569 if (intel_crtc_can_enable_sagv(old_crtc_state) ==
1570 intel_crtc_can_enable_sagv(new_crtc_state))
1571 continue;
1572
1573 new_bw_state = intel_atomic_get_bw_state(state);
1574 if (IS_ERR(new_bw_state))
1575 return PTR_ERR(new_bw_state);
1576
1577 old_bw_state = intel_atomic_get_old_bw_state(state);
1578
1579 if (intel_crtc_can_enable_sagv(new_crtc_state))
1580 new_bw_state->pipe_sagv_reject &= ~BIT(crtc->pipe);
1581 else
1582 new_bw_state->pipe_sagv_reject |= BIT(crtc->pipe);
1583 }
1584
1585 if (!new_bw_state)
1586 return 0;
1587
1588 if (intel_bw_can_enable_sagv(display, new_bw_state) !=
1589 intel_bw_can_enable_sagv(display, old_bw_state)) {
1590 ret = intel_atomic_serialize_global_state(&new_bw_state->base);
1591 if (ret)
1592 return ret;
1593 } else if (new_bw_state->pipe_sagv_reject != old_bw_state->pipe_sagv_reject) {
1594 ret = intel_atomic_lock_global_state(&new_bw_state->base);
1595 if (ret)
1596 return ret;
1597 }
1598
1599 return 0;
1600 }
1601
intel_bw_atomic_check(struct intel_atomic_state * state,bool any_ms)1602 int intel_bw_atomic_check(struct intel_atomic_state *state, bool any_ms)
1603 {
1604 struct intel_display *display = to_intel_display(state);
1605 bool changed = false;
1606 struct intel_bw_state *new_bw_state;
1607 const struct intel_bw_state *old_bw_state;
1608 int ret;
1609
1610 if (DISPLAY_VER(display) < 9)
1611 return 0;
1612
1613 if (any_ms) {
1614 ret = intel_bw_modeset_checks(state);
1615 if (ret)
1616 return ret;
1617 }
1618
1619 ret = intel_bw_check_sagv_mask(state);
1620 if (ret)
1621 return ret;
1622
1623 /* FIXME earlier gens need some checks too */
1624 if (DISPLAY_VER(display) < 11)
1625 return 0;
1626
1627 ret = intel_bw_check_data_rate(state, &changed);
1628 if (ret)
1629 return ret;
1630
1631 old_bw_state = intel_atomic_get_old_bw_state(state);
1632 new_bw_state = intel_atomic_get_new_bw_state(state);
1633
1634 if (new_bw_state &&
1635 intel_bw_can_enable_sagv(display, old_bw_state) !=
1636 intel_bw_can_enable_sagv(display, new_bw_state))
1637 changed = true;
1638
1639 /*
1640 * If none of our inputs (data rates, number of active
1641 * planes, SAGV yes/no) changed then nothing to do here.
1642 */
1643 if (!changed)
1644 return 0;
1645
1646 ret = intel_bw_check_qgv_points(display, old_bw_state, new_bw_state);
1647 if (ret)
1648 return ret;
1649
1650 return 0;
1651 }
1652
intel_bw_crtc_update(struct intel_bw_state * bw_state,const struct intel_crtc_state * crtc_state)1653 static void intel_bw_crtc_update(struct intel_bw_state *bw_state,
1654 const struct intel_crtc_state *crtc_state)
1655 {
1656 struct intel_display *display = to_intel_display(crtc_state);
1657 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
1658
1659 bw_state->data_rate[crtc->pipe] =
1660 intel_bw_crtc_data_rate(crtc_state);
1661 bw_state->num_active_planes[crtc->pipe] =
1662 intel_bw_crtc_num_active_planes(crtc_state);
1663
1664 drm_dbg_kms(display->drm, "pipe %c data rate %u num active planes %u\n",
1665 pipe_name(crtc->pipe),
1666 bw_state->data_rate[crtc->pipe],
1667 bw_state->num_active_planes[crtc->pipe]);
1668 }
1669
intel_bw_update_hw_state(struct intel_display * display)1670 void intel_bw_update_hw_state(struct intel_display *display)
1671 {
1672 struct intel_bw_state *bw_state =
1673 to_intel_bw_state(display->bw.obj.state);
1674 struct intel_crtc *crtc;
1675
1676 if (DISPLAY_VER(display) < 9)
1677 return;
1678
1679 bw_state->active_pipes = 0;
1680 bw_state->pipe_sagv_reject = 0;
1681
1682 for_each_intel_crtc(display->drm, crtc) {
1683 const struct intel_crtc_state *crtc_state =
1684 to_intel_crtc_state(crtc->base.state);
1685 enum pipe pipe = crtc->pipe;
1686
1687 if (crtc_state->hw.active)
1688 bw_state->active_pipes |= BIT(pipe);
1689
1690 if (DISPLAY_VER(display) >= 11)
1691 intel_bw_crtc_update(bw_state, crtc_state);
1692
1693 skl_crtc_calc_dbuf_bw(&bw_state->dbuf_bw[pipe], crtc_state);
1694
1695 /* initially SAGV has been forced off */
1696 bw_state->pipe_sagv_reject |= BIT(pipe);
1697 }
1698 }
1699
intel_bw_crtc_disable_noatomic(struct intel_crtc * crtc)1700 void intel_bw_crtc_disable_noatomic(struct intel_crtc *crtc)
1701 {
1702 struct intel_display *display = to_intel_display(crtc);
1703 struct intel_bw_state *bw_state =
1704 to_intel_bw_state(display->bw.obj.state);
1705 enum pipe pipe = crtc->pipe;
1706
1707 if (DISPLAY_VER(display) < 9)
1708 return;
1709
1710 bw_state->data_rate[pipe] = 0;
1711 bw_state->num_active_planes[pipe] = 0;
1712 memset(&bw_state->dbuf_bw[pipe], 0, sizeof(bw_state->dbuf_bw[pipe]));
1713 }
1714
1715 static struct intel_global_state *
intel_bw_duplicate_state(struct intel_global_obj * obj)1716 intel_bw_duplicate_state(struct intel_global_obj *obj)
1717 {
1718 struct intel_bw_state *state;
1719
1720 state = kmemdup(obj->state, sizeof(*state), GFP_KERNEL);
1721 if (!state)
1722 return NULL;
1723
1724 return &state->base;
1725 }
1726
intel_bw_destroy_state(struct intel_global_obj * obj,struct intel_global_state * state)1727 static void intel_bw_destroy_state(struct intel_global_obj *obj,
1728 struct intel_global_state *state)
1729 {
1730 kfree(state);
1731 }
1732
1733 static const struct intel_global_state_funcs intel_bw_funcs = {
1734 .atomic_duplicate_state = intel_bw_duplicate_state,
1735 .atomic_destroy_state = intel_bw_destroy_state,
1736 };
1737
intel_bw_init(struct intel_display * display)1738 int intel_bw_init(struct intel_display *display)
1739 {
1740 struct intel_bw_state *state;
1741
1742 state = kzalloc(sizeof(*state), GFP_KERNEL);
1743 if (!state)
1744 return -ENOMEM;
1745
1746 intel_atomic_global_obj_init(display, &display->bw.obj,
1747 &state->base, &intel_bw_funcs);
1748
1749 /*
1750 * Limit this only if we have SAGV. And for Display version 14 onwards
1751 * sagv is handled though pmdemand requests
1752 */
1753 if (intel_has_sagv(display) && IS_DISPLAY_VER(display, 11, 13))
1754 icl_force_disable_sagv(display, state);
1755
1756 return 0;
1757 }
1758
intel_bw_pmdemand_needs_update(struct intel_atomic_state * state)1759 bool intel_bw_pmdemand_needs_update(struct intel_atomic_state *state)
1760 {
1761 const struct intel_bw_state *new_bw_state, *old_bw_state;
1762
1763 new_bw_state = intel_atomic_get_new_bw_state(state);
1764 old_bw_state = intel_atomic_get_old_bw_state(state);
1765
1766 if (new_bw_state &&
1767 new_bw_state->qgv_point_peakbw != old_bw_state->qgv_point_peakbw)
1768 return true;
1769
1770 return false;
1771 }
1772
intel_bw_can_enable_sagv(struct intel_display * display,const struct intel_bw_state * bw_state)1773 bool intel_bw_can_enable_sagv(struct intel_display *display,
1774 const struct intel_bw_state *bw_state)
1775 {
1776 if (DISPLAY_VER(display) < 11 &&
1777 bw_state->active_pipes && !is_power_of_2(bw_state->active_pipes))
1778 return false;
1779
1780 return bw_state->pipe_sagv_reject == 0;
1781 }
1782
intel_bw_qgv_point_peakbw(const struct intel_bw_state * bw_state)1783 int intel_bw_qgv_point_peakbw(const struct intel_bw_state *bw_state)
1784 {
1785 return bw_state->qgv_point_peakbw;
1786 }
1787