1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2025 Intel Corporation
4 */
5
6 #include <linux/pci.h>
7
8 #include <drm/drm_print.h>
9
10 #include "i915_utils.h"
11 #include "intel_step.h"
12 #include "intel_crtc.h"
13 #include "intel_de.h"
14 #include "intel_display_core.h"
15 #include "intel_display_types.h"
16 #include "intel_flipq.h"
17 #include "intel_dmc.h"
18 #include "intel_dmc_regs.h"
19 #include "intel_dsb.h"
20 #include "intel_vblank.h"
21 #include "intel_vrr.h"
22
23 /**
24 * DOC: DMC Flip Queue
25 *
26 * A flip queue is a ring buffer implemented by the pipe DMC firmware.
27 * The driver inserts entries into the queues to be executed by the
28 * pipe DMC at a specified presentation timestamp (PTS).
29 *
30 * Each pipe DMC provides several queues:
31 *
32 * - 1 general queue (two DSB buffers executed per entry)
33 * - 3 plane queues (one DSB buffer executed per entry)
34 * - 1 fast queue (deprecated)
35 */
36
37 #define for_each_flipq(flipq_id) \
38 for ((flipq_id) = INTEL_FLIPQ_PLANE_1; (flipq_id) < MAX_INTEL_FLIPQ; (flipq_id)++)
39
intel_flipq_offset(enum intel_flipq_id flipq_id)40 static int intel_flipq_offset(enum intel_flipq_id flipq_id)
41 {
42 switch (flipq_id) {
43 case INTEL_FLIPQ_PLANE_1:
44 return 0x008;
45 case INTEL_FLIPQ_PLANE_2:
46 return 0x108;
47 case INTEL_FLIPQ_PLANE_3:
48 return 0x208;
49 case INTEL_FLIPQ_GENERAL:
50 return 0x308;
51 case INTEL_FLIPQ_FAST:
52 return 0x3c8;
53 default:
54 MISSING_CASE(flipq_id);
55 return 0;
56 }
57 }
58
intel_flipq_size_dw(enum intel_flipq_id flipq_id)59 static int intel_flipq_size_dw(enum intel_flipq_id flipq_id)
60 {
61 switch (flipq_id) {
62 case INTEL_FLIPQ_PLANE_1:
63 case INTEL_FLIPQ_PLANE_2:
64 case INTEL_FLIPQ_PLANE_3:
65 return 64;
66 case INTEL_FLIPQ_GENERAL:
67 case INTEL_FLIPQ_FAST:
68 return 48;
69 default:
70 MISSING_CASE(flipq_id);
71 return 1;
72 }
73 }
74
intel_flipq_elem_size_dw(enum intel_flipq_id flipq_id)75 static int intel_flipq_elem_size_dw(enum intel_flipq_id flipq_id)
76 {
77 switch (flipq_id) {
78 case INTEL_FLIPQ_PLANE_1:
79 case INTEL_FLIPQ_PLANE_2:
80 case INTEL_FLIPQ_PLANE_3:
81 return 4;
82 case INTEL_FLIPQ_GENERAL:
83 case INTEL_FLIPQ_FAST:
84 return 6;
85 default:
86 MISSING_CASE(flipq_id);
87 return 1;
88 }
89 }
90
intel_flipq_size_entries(enum intel_flipq_id flipq_id)91 static int intel_flipq_size_entries(enum intel_flipq_id flipq_id)
92 {
93 return intel_flipq_size_dw(flipq_id) / intel_flipq_elem_size_dw(flipq_id);
94 }
95
intel_flipq_crtc_init(struct intel_crtc * crtc)96 static void intel_flipq_crtc_init(struct intel_crtc *crtc)
97 {
98 struct intel_display *display = to_intel_display(crtc);
99 enum intel_flipq_id flipq_id;
100
101 for_each_flipq(flipq_id) {
102 struct intel_flipq *flipq = &crtc->flipq[flipq_id];
103
104 flipq->start_mmioaddr = intel_pipedmc_start_mmioaddr(crtc) + intel_flipq_offset(flipq_id);
105 flipq->flipq_id = flipq_id;
106
107 drm_dbg_kms(display->drm, "[CRTC:%d:%s] FQ %d: start 0x%x\n",
108 crtc->base.base.id, crtc->base.name,
109 flipq_id, flipq->start_mmioaddr);
110 }
111 }
112
intel_flipq_supported(struct intel_display * display)113 bool intel_flipq_supported(struct intel_display *display)
114 {
115 if (!display->params.enable_flipq)
116 return false;
117
118 if (!display->dmc.dmc)
119 return false;
120
121 if (DISPLAY_VER(display) == 20)
122 return true;
123
124 /* DMC firmware expects VRR timing generator to be used */
125 return DISPLAY_VER(display) >= 30 && intel_vrr_always_use_vrr_tg(display);
126 }
127
intel_flipq_init(struct intel_display * display)128 void intel_flipq_init(struct intel_display *display)
129 {
130 struct intel_crtc *crtc;
131
132 intel_dmc_wait_fw_load(display);
133
134 for_each_intel_crtc(display->drm, crtc)
135 intel_flipq_crtc_init(crtc);
136 }
137
cdclk_factor(struct intel_display * display)138 static int cdclk_factor(struct intel_display *display)
139 {
140 if (DISPLAY_VER(display) >= 30)
141 return 120;
142 else
143 return 280;
144 }
145
intel_flipq_exec_time_us(struct intel_display * display)146 int intel_flipq_exec_time_us(struct intel_display *display)
147 {
148 return intel_dsb_exec_time_us() +
149 DIV_ROUND_UP(display->cdclk.hw.cdclk * cdclk_factor(display), 540000) +
150 display->sagv.block_time_us;
151 }
152
intel_flipq_preempt_timeout_ms(struct intel_display * display)153 static int intel_flipq_preempt_timeout_ms(struct intel_display *display)
154 {
155 return DIV_ROUND_UP(intel_flipq_exec_time_us(display), 1000);
156 }
157
intel_flipq_preempt(struct intel_crtc * crtc,bool preempt)158 static void intel_flipq_preempt(struct intel_crtc *crtc, bool preempt)
159 {
160 struct intel_display *display = to_intel_display(crtc);
161
162 intel_de_rmw(display, PIPEDMC_FQ_CTRL(crtc->pipe),
163 PIPEDMC_FQ_CTRL_PREEMPT, preempt ? PIPEDMC_FQ_CTRL_PREEMPT : 0);
164
165 if (preempt &&
166 intel_de_wait_for_clear(display,
167 PIPEDMC_FQ_STATUS(crtc->pipe),
168 PIPEDMC_FQ_STATUS_BUSY,
169 intel_flipq_preempt_timeout_ms(display)))
170 drm_err(display->drm, "[CRTC:%d:%s] flip queue preempt timeout\n",
171 crtc->base.base.id, crtc->base.name);
172 }
173
intel_flipq_current_head(struct intel_crtc * crtc,enum intel_flipq_id flipq_id)174 static int intel_flipq_current_head(struct intel_crtc *crtc, enum intel_flipq_id flipq_id)
175 {
176 struct intel_display *display = to_intel_display(crtc);
177
178 return intel_de_read(display, PIPEDMC_FPQ_CHP(crtc->pipe, flipq_id));
179 }
180
intel_flipq_write_tail(struct intel_crtc * crtc)181 static void intel_flipq_write_tail(struct intel_crtc *crtc)
182 {
183 struct intel_display *display = to_intel_display(crtc);
184
185 intel_de_write(display, PIPEDMC_FPQ_ATOMIC_TP(crtc->pipe),
186 PIPEDMC_FPQ_PLANEQ_3_TP(crtc->flipq[INTEL_FLIPQ_PLANE_3].tail) |
187 PIPEDMC_FPQ_PLANEQ_2_TP(crtc->flipq[INTEL_FLIPQ_PLANE_2].tail) |
188 PIPEDMC_FPQ_PLANEQ_1_TP(crtc->flipq[INTEL_FLIPQ_PLANE_1].tail) |
189 PIPEDMC_FPQ_FASTQ_TP(crtc->flipq[INTEL_FLIPQ_FAST].tail) |
190 PIPEDMC_FPQ_GENERALQ_TP(crtc->flipq[INTEL_FLIPQ_GENERAL].tail));
191 }
192
intel_flipq_sw_dmc_wake(struct intel_crtc * crtc)193 static void intel_flipq_sw_dmc_wake(struct intel_crtc *crtc)
194 {
195 struct intel_display *display = to_intel_display(crtc);
196
197 intel_de_write(display, PIPEDMC_FPQ_CTL1(crtc->pipe), PIPEDMC_SW_DMC_WAKE);
198 }
199
intel_flipq_exec_time_lines(const struct intel_crtc_state * crtc_state)200 static int intel_flipq_exec_time_lines(const struct intel_crtc_state *crtc_state)
201 {
202 struct intel_display *display = to_intel_display(crtc_state);
203
204 return intel_usecs_to_scanlines(&crtc_state->hw.adjusted_mode,
205 intel_flipq_exec_time_us(display));
206 }
207
intel_flipq_dump(struct intel_crtc * crtc,enum intel_flipq_id flipq_id)208 void intel_flipq_dump(struct intel_crtc *crtc,
209 enum intel_flipq_id flipq_id)
210 {
211 struct intel_display *display = to_intel_display(crtc);
212 struct intel_flipq *flipq = &crtc->flipq[flipq_id];
213 u32 tmp;
214
215 drm_dbg_kms(display->drm,
216 "[CRTC:%d:%s] FQ %d @ 0x%x: ",
217 crtc->base.base.id, crtc->base.name, flipq_id,
218 flipq->start_mmioaddr);
219 for (int i = 0 ; i < intel_flipq_size_dw(flipq_id); i++) {
220 printk(KERN_CONT " 0x%08x",
221 intel_de_read(display, PIPEDMC_FQ_RAM(flipq->start_mmioaddr, i)));
222 if (i % intel_flipq_elem_size_dw(flipq_id) == intel_flipq_elem_size_dw(flipq_id) - 1)
223 printk(KERN_CONT "\n");
224 }
225
226 drm_dbg_kms(display->drm,
227 "[CRTC:%d:%s] FQ %d: chp=0x%x, hp=0x%x\n",
228 crtc->base.base.id, crtc->base.name, flipq_id,
229 intel_de_read(display, PIPEDMC_FPQ_CHP(crtc->pipe, flipq_id)),
230 intel_de_read(display, PIPEDMC_FPQ_HP(crtc->pipe, flipq_id)));
231
232 drm_dbg_kms(display->drm,
233 "[CRTC:%d:%s] FQ %d: current head %d\n",
234 crtc->base.base.id, crtc->base.name, flipq_id,
235 intel_flipq_current_head(crtc, flipq_id));
236
237 drm_dbg_kms(display->drm,
238 "[CRTC:%d:%s] flip queue timestamp: 0x%x\n",
239 crtc->base.base.id, crtc->base.name,
240 intel_de_read(display, PIPEDMC_FPQ_TS(crtc->pipe)));
241
242 tmp = intel_de_read(display, PIPEDMC_FPQ_ATOMIC_TP(crtc->pipe));
243
244 drm_dbg_kms(display->drm,
245 "[CRTC:%d:%s] flip queue atomic tails: P3 %d, P2 %d, P1 %d, G %d, F %d\n",
246 crtc->base.base.id, crtc->base.name,
247 REG_FIELD_GET(PIPEDMC_FPQ_PLANEQ_3_TP_MASK, tmp),
248 REG_FIELD_GET(PIPEDMC_FPQ_PLANEQ_2_TP_MASK, tmp),
249 REG_FIELD_GET(PIPEDMC_FPQ_PLANEQ_1_TP_MASK, tmp),
250 REG_FIELD_GET(PIPEDMC_FPQ_GENERALQ_TP_MASK, tmp),
251 REG_FIELD_GET(PIPEDMC_FPQ_FASTQ_TP_MASK, tmp));
252 }
253
intel_flipq_reset(struct intel_display * display,enum pipe pipe)254 void intel_flipq_reset(struct intel_display *display, enum pipe pipe)
255 {
256 struct intel_crtc *crtc = intel_crtc_for_pipe(display, pipe);
257 enum intel_flipq_id flipq_id;
258
259 intel_de_write(display, PIPEDMC_FQ_CTRL(pipe), 0);
260
261 intel_de_write(display, PIPEDMC_SCANLINECMPLOWER(pipe), 0);
262 intel_de_write(display, PIPEDMC_SCANLINECMPUPPER(pipe), 0);
263
264 for_each_flipq(flipq_id) {
265 struct intel_flipq *flipq = &crtc->flipq[flipq_id];
266
267 intel_de_write(display, PIPEDMC_FPQ_HP(pipe, flipq_id), 0);
268 intel_de_write(display, PIPEDMC_FPQ_CHP(pipe, flipq_id), 0);
269
270 flipq->tail = 0;
271 }
272
273 intel_de_write(display, PIPEDMC_FPQ_ATOMIC_TP(pipe), 0);
274 }
275
flipq_event_id(struct intel_display * display)276 static enum pipedmc_event_id flipq_event_id(struct intel_display *display)
277 {
278 if (DISPLAY_VER(display) >= 30)
279 return PIPEDMC_EVENT_FULL_FQ_WAKE_TRIGGER;
280 else
281 return PIPEDMC_EVENT_SCANLINE_INRANGE_FQ_TRIGGER;
282 }
283
intel_flipq_enable(const struct intel_crtc_state * crtc_state)284 void intel_flipq_enable(const struct intel_crtc_state *crtc_state)
285 {
286 struct intel_display *display = to_intel_display(crtc_state);
287 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
288 /* FIXME what to do with VRR? */
289 int scanline = intel_mode_vblank_start(&crtc_state->hw.adjusted_mode) -
290 intel_flipq_exec_time_lines(crtc_state);
291
292 if (DISPLAY_VER(display) >= 30) {
293 u32 start_mmioaddr = intel_pipedmc_start_mmioaddr(crtc);
294
295 /* undocumented magic DMC variables */
296 intel_de_write(display, PTL_PIPEDMC_EXEC_TIME_LINES(start_mmioaddr),
297 intel_flipq_exec_time_lines(crtc_state));
298 intel_de_write(display, PTL_PIPEDMC_END_OF_EXEC_GB(start_mmioaddr),
299 100);
300 }
301
302 intel_de_write(display, PIPEDMC_SCANLINECMPUPPER(crtc->pipe),
303 PIPEDMC_SCANLINE_UPPER(scanline));
304 intel_de_write(display, PIPEDMC_SCANLINECMPLOWER(crtc->pipe),
305 PIPEDMC_SCANLINEINRANGECMP_EN |
306 PIPEDMC_SCANLINE_LOWER(scanline - 2));
307
308 intel_pipedmc_enable_event(crtc, flipq_event_id(display));
309
310 intel_de_write(display, PIPEDMC_FQ_CTRL(crtc->pipe), PIPEDMC_FQ_CTRL_ENABLE);
311 }
312
intel_flipq_disable(const struct intel_crtc_state * crtc_state)313 void intel_flipq_disable(const struct intel_crtc_state *crtc_state)
314 {
315 struct intel_display *display = to_intel_display(crtc_state);
316 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
317
318 intel_flipq_preempt(crtc, true);
319
320 intel_de_write(display, PIPEDMC_FQ_CTRL(crtc->pipe), 0);
321
322 intel_pipedmc_disable_event(crtc, flipq_event_id(display));
323
324 intel_de_write(display, PIPEDMC_SCANLINECMPLOWER(crtc->pipe), 0);
325 intel_de_write(display, PIPEDMC_SCANLINECMPUPPER(crtc->pipe), 0);
326 }
327
assert_flipq_has_room(struct intel_crtc * crtc,enum intel_flipq_id flipq_id)328 static bool assert_flipq_has_room(struct intel_crtc *crtc,
329 enum intel_flipq_id flipq_id)
330 {
331 struct intel_display *display = to_intel_display(crtc);
332 struct intel_flipq *flipq = &crtc->flipq[flipq_id];
333 int head, size = intel_flipq_size_entries(flipq_id);
334
335 head = intel_flipq_current_head(crtc, flipq_id);
336
337 return !drm_WARN(display->drm,
338 (flipq->tail + size - head) % size >= size - 1,
339 "[CRTC:%d:%s] FQ %d overflow (head %d, tail %d, size %d)\n",
340 crtc->base.base.id, crtc->base.name, flipq_id,
341 head, flipq->tail, size);
342 }
343
intel_flipq_write(struct intel_display * display,struct intel_flipq * flipq,u32 data,int i)344 static void intel_flipq_write(struct intel_display *display,
345 struct intel_flipq *flipq, u32 data, int i)
346 {
347 intel_de_write(display, PIPEDMC_FQ_RAM(flipq->start_mmioaddr, flipq->tail *
348 intel_flipq_elem_size_dw(flipq->flipq_id) + i), data);
349 }
350
lnl_flipq_add(struct intel_display * display,struct intel_flipq * flipq,unsigned int pts,enum intel_dsb_id dsb_id,struct intel_dsb * dsb)351 static void lnl_flipq_add(struct intel_display *display,
352 struct intel_flipq *flipq,
353 unsigned int pts,
354 enum intel_dsb_id dsb_id,
355 struct intel_dsb *dsb)
356 {
357 int i = 0;
358
359 switch (flipq->flipq_id) {
360 case INTEL_FLIPQ_GENERAL:
361 intel_flipq_write(display, flipq, pts, i++);
362 intel_flipq_write(display, flipq, intel_dsb_head(dsb), i++);
363 intel_flipq_write(display, flipq, LNL_FQ_INTERRUPT |
364 LNL_FQ_DSB_ID(dsb_id) |
365 LNL_FQ_DSB_SIZE(intel_dsb_size(dsb) / 64), i++);
366 intel_flipq_write(display, flipq, 0, i++);
367 intel_flipq_write(display, flipq, 0, i++); /* head for second DSB */
368 intel_flipq_write(display, flipq, 0, i++); /* DSB engine + size for second DSB */
369 break;
370 case INTEL_FLIPQ_PLANE_1:
371 case INTEL_FLIPQ_PLANE_2:
372 case INTEL_FLIPQ_PLANE_3:
373 intel_flipq_write(display, flipq, pts, i++);
374 intel_flipq_write(display, flipq, intel_dsb_head(dsb), i++);
375 intel_flipq_write(display, flipq, LNL_FQ_INTERRUPT |
376 LNL_FQ_DSB_ID(dsb_id) |
377 LNL_FQ_DSB_SIZE(intel_dsb_size(dsb) / 64), i++);
378 intel_flipq_write(display, flipq, 0, i++);
379 break;
380 default:
381 MISSING_CASE(flipq->flipq_id);
382 return;
383 }
384 }
385
ptl_flipq_add(struct intel_display * display,struct intel_flipq * flipq,unsigned int pts,enum intel_dsb_id dsb_id,struct intel_dsb * dsb)386 static void ptl_flipq_add(struct intel_display *display,
387 struct intel_flipq *flipq,
388 unsigned int pts,
389 enum intel_dsb_id dsb_id,
390 struct intel_dsb *dsb)
391 {
392 int i = 0;
393
394 switch (flipq->flipq_id) {
395 case INTEL_FLIPQ_GENERAL:
396 intel_flipq_write(display, flipq, pts, i++);
397 intel_flipq_write(display, flipq, 0, i++);
398 intel_flipq_write(display, flipq, PTL_FQ_INTERRUPT |
399 PTL_FQ_DSB_ID(dsb_id) |
400 PTL_FQ_DSB_SIZE(intel_dsb_size(dsb) / 64), i++);
401 intel_flipq_write(display, flipq, intel_dsb_head(dsb), i++);
402 intel_flipq_write(display, flipq, 0, i++); /* DSB engine + size for second DSB */
403 intel_flipq_write(display, flipq, 0, i++); /* head for second DSB */
404 break;
405 case INTEL_FLIPQ_PLANE_1:
406 case INTEL_FLIPQ_PLANE_2:
407 case INTEL_FLIPQ_PLANE_3:
408 intel_flipq_write(display, flipq, pts, i++);
409 intel_flipq_write(display, flipq, 0, i++);
410 intel_flipq_write(display, flipq, PTL_FQ_INTERRUPT |
411 PTL_FQ_DSB_ID(dsb_id) |
412 PTL_FQ_DSB_SIZE(intel_dsb_size(dsb) / 64), i++);
413 intel_flipq_write(display, flipq, intel_dsb_head(dsb), i++);
414 break;
415 default:
416 MISSING_CASE(flipq->flipq_id);
417 return;
418 }
419 }
420
intel_flipq_add(struct intel_crtc * crtc,enum intel_flipq_id flipq_id,unsigned int pts,enum intel_dsb_id dsb_id,struct intel_dsb * dsb)421 void intel_flipq_add(struct intel_crtc *crtc,
422 enum intel_flipq_id flipq_id,
423 unsigned int pts,
424 enum intel_dsb_id dsb_id,
425 struct intel_dsb *dsb)
426 {
427 struct intel_display *display = to_intel_display(crtc);
428 struct intel_flipq *flipq = &crtc->flipq[flipq_id];
429
430 if (!assert_flipq_has_room(crtc, flipq_id))
431 return;
432
433 pts += intel_de_read(display, PIPEDMC_FPQ_TS(crtc->pipe));
434
435 intel_flipq_preempt(crtc, true);
436
437 if (DISPLAY_VER(display) >= 30)
438 ptl_flipq_add(display, flipq, pts, dsb_id, dsb);
439 else
440 lnl_flipq_add(display, flipq, pts, dsb_id, dsb);
441
442 flipq->tail = (flipq->tail + 1) % intel_flipq_size_entries(flipq->flipq_id);
443 intel_flipq_write_tail(crtc);
444
445 intel_flipq_preempt(crtc, false);
446
447 intel_flipq_sw_dmc_wake(crtc);
448 }
449
450 /* Wa_18034343758 */
need_dmc_halt_wa(struct intel_display * display)451 static bool need_dmc_halt_wa(struct intel_display *display)
452 {
453 return DISPLAY_VER(display) == 20 ||
454 (display->platform.pantherlake &&
455 IS_DISPLAY_STEP(display, STEP_A0, STEP_B0));
456 }
457
intel_flipq_wait_dmc_halt(struct intel_dsb * dsb,struct intel_crtc * crtc)458 void intel_flipq_wait_dmc_halt(struct intel_dsb *dsb, struct intel_crtc *crtc)
459 {
460 struct intel_display *display = to_intel_display(crtc);
461
462 if (need_dmc_halt_wa(display))
463 intel_dsb_wait_usec(dsb, 2);
464 }
465
intel_flipq_unhalt_dmc(struct intel_dsb * dsb,struct intel_crtc * crtc)466 void intel_flipq_unhalt_dmc(struct intel_dsb *dsb, struct intel_crtc *crtc)
467 {
468 struct intel_display *display = to_intel_display(crtc);
469
470 if (need_dmc_halt_wa(display))
471 intel_dsb_reg_write(dsb, PIPEDMC_CTL(crtc->pipe), 0);
472 }
473