1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2020 Intel Corporation
4 */
5 #include <linux/kernel.h>
6 #include <linux/pm_qos.h>
7 #include <linux/slab.h>
8
9 #include <drm/drm_atomic_helper.h>
10 #include <drm/drm_fourcc.h>
11 #include <drm/drm_plane.h>
12 #include <drm/drm_vblank.h>
13 #include <drm/drm_vblank_work.h>
14
15 #include "i915_drv.h"
16 #include "i915_vgpu.h"
17 #include "i9xx_plane.h"
18 #include "icl_dsi.h"
19 #include "intel_atomic.h"
20 #include "intel_atomic_plane.h"
21 #include "intel_color.h"
22 #include "intel_crtc.h"
23 #include "intel_cursor.h"
24 #include "intel_display_debugfs.h"
25 #include "intel_display_irq.h"
26 #include "intel_display_trace.h"
27 #include "intel_display_types.h"
28 #include "intel_drrs.h"
29 #include "intel_dsi.h"
30 #include "intel_fifo_underrun.h"
31 #include "intel_pipe_crc.h"
32 #include "intel_psr.h"
33 #include "intel_sprite.h"
34 #include "intel_vblank.h"
35 #include "intel_vrr.h"
36 #include "skl_universal_plane.h"
37
assert_vblank_disabled(struct drm_crtc * crtc)38 static void assert_vblank_disabled(struct drm_crtc *crtc)
39 {
40 struct intel_display *display = to_intel_display(crtc->dev);
41
42 if (INTEL_DISPLAY_STATE_WARN(display, drm_crtc_vblank_get(crtc) == 0,
43 "[CRTC:%d:%s] vblank assertion failure (expected off, current on)\n",
44 crtc->base.id, crtc->name))
45 drm_crtc_vblank_put(crtc);
46 }
47
intel_first_crtc(struct intel_display * display)48 struct intel_crtc *intel_first_crtc(struct intel_display *display)
49 {
50 return to_intel_crtc(drm_crtc_from_index(display->drm, 0));
51 }
52
intel_crtc_for_pipe(struct intel_display * display,enum pipe pipe)53 struct intel_crtc *intel_crtc_for_pipe(struct intel_display *display,
54 enum pipe pipe)
55 {
56 struct intel_crtc *crtc;
57
58 for_each_intel_crtc(display->drm, crtc) {
59 if (crtc->pipe == pipe)
60 return crtc;
61 }
62
63 return NULL;
64 }
65
intel_crtc_wait_for_next_vblank(struct intel_crtc * crtc)66 void intel_crtc_wait_for_next_vblank(struct intel_crtc *crtc)
67 {
68 drm_crtc_wait_one_vblank(&crtc->base);
69 }
70
intel_wait_for_vblank_if_active(struct intel_display * display,enum pipe pipe)71 void intel_wait_for_vblank_if_active(struct intel_display *display,
72 enum pipe pipe)
73 {
74 struct intel_crtc *crtc = intel_crtc_for_pipe(display, pipe);
75
76 if (crtc->active)
77 intel_crtc_wait_for_next_vblank(crtc);
78 }
79
intel_crtc_get_vblank_counter(struct intel_crtc * crtc)80 u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc)
81 {
82 struct drm_vblank_crtc *vblank = drm_crtc_vblank_crtc(&crtc->base);
83
84 if (!crtc->active)
85 return 0;
86
87 if (!vblank->max_vblank_count)
88 return (u32)drm_crtc_accurate_vblank_count(&crtc->base);
89
90 return crtc->base.funcs->get_vblank_counter(&crtc->base);
91 }
92
intel_crtc_max_vblank_count(const struct intel_crtc_state * crtc_state)93 u32 intel_crtc_max_vblank_count(const struct intel_crtc_state *crtc_state)
94 {
95 struct intel_display *display = to_intel_display(crtc_state);
96
97 /*
98 * From Gen 11, in case of dsi cmd mode, frame counter wouldn't
99 * have updated at the beginning of TE, if we want to use
100 * the hw counter, then we would find it updated in only
101 * the next TE, hence switching to sw counter.
102 */
103 if (crtc_state->mode_flags & (I915_MODE_FLAG_DSI_USE_TE0 |
104 I915_MODE_FLAG_DSI_USE_TE1))
105 return 0;
106
107 /*
108 * On i965gm the hardware frame counter reads
109 * zero when the TV encoder is enabled :(
110 */
111 if (display->platform.i965gm &&
112 (crtc_state->output_types & BIT(INTEL_OUTPUT_TVOUT)))
113 return 0;
114
115 if (DISPLAY_VER(display) >= 5 || display->platform.g4x)
116 return 0xffffffff; /* full 32 bit counter */
117 else if (DISPLAY_VER(display) >= 3)
118 return 0xffffff; /* only 24 bits of frame count */
119 else
120 return 0; /* Gen2 doesn't have a hardware frame counter */
121 }
122
intel_crtc_vblank_on(const struct intel_crtc_state * crtc_state)123 void intel_crtc_vblank_on(const struct intel_crtc_state *crtc_state)
124 {
125 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
126
127 crtc->block_dc_for_vblank = intel_psr_needs_block_dc_vblank(crtc_state);
128
129 assert_vblank_disabled(&crtc->base);
130 drm_crtc_set_max_vblank_count(&crtc->base,
131 intel_crtc_max_vblank_count(crtc_state));
132 drm_crtc_vblank_on(&crtc->base);
133
134 /*
135 * Should really happen exactly when we enable the pipe
136 * but we want the frame counters in the trace, and that
137 * requires vblank support on some platforms/outputs.
138 */
139 trace_intel_pipe_enable(crtc);
140 }
141
intel_crtc_vblank_off(const struct intel_crtc_state * crtc_state)142 void intel_crtc_vblank_off(const struct intel_crtc_state *crtc_state)
143 {
144 struct intel_display *display = to_intel_display(crtc_state);
145 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
146
147 /*
148 * Should really happen exactly when we disable the pipe
149 * but we want the frame counters in the trace, and that
150 * requires vblank support on some platforms/outputs.
151 */
152 trace_intel_pipe_disable(crtc);
153
154 drm_crtc_vblank_off(&crtc->base);
155 assert_vblank_disabled(&crtc->base);
156
157 crtc->block_dc_for_vblank = false;
158
159 flush_work(&display->irq.vblank_dc_work);
160 }
161
intel_crtc_state_alloc(struct intel_crtc * crtc)162 struct intel_crtc_state *intel_crtc_state_alloc(struct intel_crtc *crtc)
163 {
164 struct intel_crtc_state *crtc_state;
165
166 crtc_state = kmalloc(sizeof(*crtc_state), GFP_KERNEL);
167
168 if (crtc_state)
169 intel_crtc_state_reset(crtc_state, crtc);
170
171 return crtc_state;
172 }
173
intel_crtc_state_reset(struct intel_crtc_state * crtc_state,struct intel_crtc * crtc)174 void intel_crtc_state_reset(struct intel_crtc_state *crtc_state,
175 struct intel_crtc *crtc)
176 {
177 memset(crtc_state, 0, sizeof(*crtc_state));
178
179 __drm_atomic_helper_crtc_state_reset(&crtc_state->uapi, &crtc->base);
180
181 crtc_state->cpu_transcoder = INVALID_TRANSCODER;
182 crtc_state->master_transcoder = INVALID_TRANSCODER;
183 crtc_state->hsw_workaround_pipe = INVALID_PIPE;
184 crtc_state->scaler_state.scaler_id = -1;
185 crtc_state->mst_master_transcoder = INVALID_TRANSCODER;
186 crtc_state->max_link_bpp_x16 = INT_MAX;
187 }
188
intel_crtc_alloc(void)189 static struct intel_crtc *intel_crtc_alloc(void)
190 {
191 struct intel_crtc_state *crtc_state;
192 struct intel_crtc *crtc;
193
194 crtc = kzalloc(sizeof(*crtc), GFP_KERNEL);
195 if (!crtc)
196 return ERR_PTR(-ENOMEM);
197
198 crtc_state = intel_crtc_state_alloc(crtc);
199 if (!crtc_state) {
200 kfree(crtc);
201 return ERR_PTR(-ENOMEM);
202 }
203
204 crtc->base.state = &crtc_state->uapi;
205 crtc->config = crtc_state;
206
207 return crtc;
208 }
209
intel_crtc_free(struct intel_crtc * crtc)210 static void intel_crtc_free(struct intel_crtc *crtc)
211 {
212 intel_crtc_destroy_state(&crtc->base, crtc->base.state);
213 kfree(crtc);
214 }
215
intel_crtc_destroy(struct drm_crtc * _crtc)216 static void intel_crtc_destroy(struct drm_crtc *_crtc)
217 {
218 struct intel_crtc *crtc = to_intel_crtc(_crtc);
219
220 cpu_latency_qos_remove_request(&crtc->vblank_pm_qos);
221
222 drm_crtc_cleanup(&crtc->base);
223 kfree(crtc);
224 }
225
intel_crtc_late_register(struct drm_crtc * crtc)226 static int intel_crtc_late_register(struct drm_crtc *crtc)
227 {
228 intel_crtc_debugfs_add(to_intel_crtc(crtc));
229 return 0;
230 }
231
232 #define INTEL_CRTC_FUNCS \
233 .set_config = drm_atomic_helper_set_config, \
234 .destroy = intel_crtc_destroy, \
235 .page_flip = drm_atomic_helper_page_flip, \
236 .atomic_duplicate_state = intel_crtc_duplicate_state, \
237 .atomic_destroy_state = intel_crtc_destroy_state, \
238 .set_crc_source = intel_crtc_set_crc_source, \
239 .verify_crc_source = intel_crtc_verify_crc_source, \
240 .get_crc_sources = intel_crtc_get_crc_sources, \
241 .late_register = intel_crtc_late_register
242
243 static const struct drm_crtc_funcs bdw_crtc_funcs = {
244 INTEL_CRTC_FUNCS,
245
246 .get_vblank_counter = g4x_get_vblank_counter,
247 .enable_vblank = bdw_enable_vblank,
248 .disable_vblank = bdw_disable_vblank,
249 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
250 };
251
252 static const struct drm_crtc_funcs ilk_crtc_funcs = {
253 INTEL_CRTC_FUNCS,
254
255 .get_vblank_counter = g4x_get_vblank_counter,
256 .enable_vblank = ilk_enable_vblank,
257 .disable_vblank = ilk_disable_vblank,
258 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
259 };
260
261 static const struct drm_crtc_funcs g4x_crtc_funcs = {
262 INTEL_CRTC_FUNCS,
263
264 .get_vblank_counter = g4x_get_vblank_counter,
265 .enable_vblank = i965_enable_vblank,
266 .disable_vblank = i965_disable_vblank,
267 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
268 };
269
270 static const struct drm_crtc_funcs i965_crtc_funcs = {
271 INTEL_CRTC_FUNCS,
272
273 .get_vblank_counter = i915_get_vblank_counter,
274 .enable_vblank = i965_enable_vblank,
275 .disable_vblank = i965_disable_vblank,
276 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
277 };
278
279 static const struct drm_crtc_funcs i915gm_crtc_funcs = {
280 INTEL_CRTC_FUNCS,
281
282 .get_vblank_counter = i915_get_vblank_counter,
283 .enable_vblank = i915gm_enable_vblank,
284 .disable_vblank = i915gm_disable_vblank,
285 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
286 };
287
288 static const struct drm_crtc_funcs i915_crtc_funcs = {
289 INTEL_CRTC_FUNCS,
290
291 .get_vblank_counter = i915_get_vblank_counter,
292 .enable_vblank = i8xx_enable_vblank,
293 .disable_vblank = i8xx_disable_vblank,
294 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
295 };
296
297 static const struct drm_crtc_funcs i8xx_crtc_funcs = {
298 INTEL_CRTC_FUNCS,
299
300 /* no hw vblank counter */
301 .enable_vblank = i8xx_enable_vblank,
302 .disable_vblank = i8xx_disable_vblank,
303 .get_vblank_timestamp = intel_crtc_get_vblank_timestamp,
304 };
305
intel_crtc_init(struct intel_display * display,enum pipe pipe)306 int intel_crtc_init(struct intel_display *display, enum pipe pipe)
307 {
308 struct drm_i915_private *dev_priv = to_i915(display->drm);
309 struct intel_plane *primary, *cursor;
310 const struct drm_crtc_funcs *funcs;
311 struct intel_crtc *crtc;
312 int sprite, ret;
313
314 crtc = intel_crtc_alloc();
315 if (IS_ERR(crtc))
316 return PTR_ERR(crtc);
317
318 crtc->pipe = pipe;
319 crtc->num_scalers = DISPLAY_RUNTIME_INFO(display)->num_scalers[pipe];
320
321 if (DISPLAY_VER(display) >= 9)
322 primary = skl_universal_plane_create(display, pipe, PLANE_1);
323 else
324 primary = intel_primary_plane_create(display, pipe);
325 if (IS_ERR(primary)) {
326 ret = PTR_ERR(primary);
327 goto fail;
328 }
329 crtc->plane_ids_mask |= BIT(primary->id);
330
331 intel_init_fifo_underrun_reporting(display, crtc, false);
332
333 for_each_sprite(display, pipe, sprite) {
334 struct intel_plane *plane;
335
336 if (DISPLAY_VER(dev_priv) >= 9)
337 plane = skl_universal_plane_create(display, pipe, PLANE_2 + sprite);
338 else
339 plane = intel_sprite_plane_create(display, pipe, sprite);
340 if (IS_ERR(plane)) {
341 ret = PTR_ERR(plane);
342 goto fail;
343 }
344 crtc->plane_ids_mask |= BIT(plane->id);
345 }
346
347 cursor = intel_cursor_plane_create(display, pipe);
348 if (IS_ERR(cursor)) {
349 ret = PTR_ERR(cursor);
350 goto fail;
351 }
352 crtc->plane_ids_mask |= BIT(cursor->id);
353
354 if (HAS_GMCH(display)) {
355 if (display->platform.cherryview ||
356 display->platform.valleyview ||
357 display->platform.g4x)
358 funcs = &g4x_crtc_funcs;
359 else if (DISPLAY_VER(display) == 4)
360 funcs = &i965_crtc_funcs;
361 else if (display->platform.i945gm ||
362 display->platform.i915gm)
363 funcs = &i915gm_crtc_funcs;
364 else if (DISPLAY_VER(display) == 3)
365 funcs = &i915_crtc_funcs;
366 else
367 funcs = &i8xx_crtc_funcs;
368 } else {
369 if (DISPLAY_VER(display) >= 8)
370 funcs = &bdw_crtc_funcs;
371 else
372 funcs = &ilk_crtc_funcs;
373 }
374
375 ret = drm_crtc_init_with_planes(display->drm, &crtc->base,
376 &primary->base, &cursor->base,
377 funcs, "pipe %c", pipe_name(pipe));
378 if (ret)
379 goto fail;
380
381 if (DISPLAY_VER(display) >= 11)
382 drm_crtc_create_scaling_filter_property(&crtc->base,
383 BIT(DRM_SCALING_FILTER_DEFAULT) |
384 BIT(DRM_SCALING_FILTER_NEAREST_NEIGHBOR));
385
386 intel_color_crtc_init(crtc);
387 intel_drrs_crtc_init(crtc);
388 intel_crtc_crc_init(crtc);
389
390 cpu_latency_qos_add_request(&crtc->vblank_pm_qos, PM_QOS_DEFAULT_VALUE);
391
392 drm_WARN_ON(display->drm, drm_crtc_index(&crtc->base) != crtc->pipe);
393
394 return 0;
395
396 fail:
397 intel_crtc_free(crtc);
398
399 return ret;
400 }
401
intel_crtc_get_pipe_from_crtc_id_ioctl(struct drm_device * dev,void * data,struct drm_file * file)402 int intel_crtc_get_pipe_from_crtc_id_ioctl(struct drm_device *dev, void *data,
403 struct drm_file *file)
404 {
405 struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data;
406 struct drm_crtc *drm_crtc;
407 struct intel_crtc *crtc;
408
409 drm_crtc = drm_crtc_find(dev, file, pipe_from_crtc_id->crtc_id);
410 if (!drm_crtc)
411 return -ENOENT;
412
413 crtc = to_intel_crtc(drm_crtc);
414 pipe_from_crtc_id->pipe = crtc->pipe;
415
416 return 0;
417 }
418
intel_crtc_needs_vblank_work(const struct intel_crtc_state * crtc_state)419 static bool intel_crtc_needs_vblank_work(const struct intel_crtc_state *crtc_state)
420 {
421 return crtc_state->hw.active &&
422 !crtc_state->preload_luts &&
423 !intel_crtc_needs_modeset(crtc_state) &&
424 intel_crtc_needs_color_update(crtc_state) &&
425 !intel_color_uses_dsb(crtc_state) &&
426 !crtc_state->use_dsb;
427 }
428
intel_crtc_vblank_work(struct kthread_work * base)429 static void intel_crtc_vblank_work(struct kthread_work *base)
430 {
431 struct drm_vblank_work *work = to_drm_vblank_work(base);
432 struct intel_crtc_state *crtc_state =
433 container_of(work, typeof(*crtc_state), vblank_work);
434 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
435
436 trace_intel_crtc_vblank_work_start(crtc);
437
438 intel_color_load_luts(crtc_state);
439
440 if (crtc_state->uapi.event) {
441 spin_lock_irq(&crtc->base.dev->event_lock);
442 drm_crtc_send_vblank_event(&crtc->base, crtc_state->uapi.event);
443 spin_unlock_irq(&crtc->base.dev->event_lock);
444 crtc_state->uapi.event = NULL;
445 }
446
447 trace_intel_crtc_vblank_work_end(crtc);
448 }
449
intel_crtc_vblank_work_init(struct intel_crtc_state * crtc_state)450 static void intel_crtc_vblank_work_init(struct intel_crtc_state *crtc_state)
451 {
452 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
453
454 drm_vblank_work_init(&crtc_state->vblank_work, &crtc->base,
455 intel_crtc_vblank_work);
456 /*
457 * Interrupt latency is critical for getting the vblank
458 * work executed as early as possible during the vblank.
459 */
460 cpu_latency_qos_update_request(&crtc->vblank_pm_qos, 0);
461 }
462
intel_wait_for_vblank_workers(struct intel_atomic_state * state)463 void intel_wait_for_vblank_workers(struct intel_atomic_state *state)
464 {
465 struct intel_crtc_state *crtc_state;
466 struct intel_crtc *crtc;
467 int i;
468
469 for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
470 if (!intel_crtc_needs_vblank_work(crtc_state))
471 continue;
472
473 drm_vblank_work_flush(&crtc_state->vblank_work);
474 cpu_latency_qos_update_request(&crtc->vblank_pm_qos,
475 PM_QOS_DEFAULT_VALUE);
476 }
477 }
478
intel_usecs_to_scanlines(const struct drm_display_mode * adjusted_mode,int usecs)479 int intel_usecs_to_scanlines(const struct drm_display_mode *adjusted_mode,
480 int usecs)
481 {
482 /* paranoia */
483 if (!adjusted_mode->crtc_htotal)
484 return 1;
485
486 return DIV_ROUND_UP_ULL(mul_u32_u32(usecs, adjusted_mode->crtc_clock),
487 1000 * adjusted_mode->crtc_htotal);
488 }
489
intel_scanlines_to_usecs(const struct drm_display_mode * adjusted_mode,int scanlines)490 int intel_scanlines_to_usecs(const struct drm_display_mode *adjusted_mode,
491 int scanlines)
492 {
493 /* paranoia */
494 if (!adjusted_mode->crtc_clock)
495 return 1;
496
497 return DIV_ROUND_UP_ULL(mul_u32_u32(scanlines, adjusted_mode->crtc_htotal * 1000),
498 adjusted_mode->crtc_clock);
499 }
500
501 /**
502 * intel_pipe_update_start() - start update of a set of display registers
503 * @state: the atomic state
504 * @crtc: the crtc
505 *
506 * Mark the start of an update to pipe registers that should be updated
507 * atomically regarding vblank. If the next vblank will happens within
508 * the next 100 us, this function waits until the vblank passes.
509 *
510 * After a successful call to this function, interrupts will be disabled
511 * until a subsequent call to intel_pipe_update_end(). That is done to
512 * avoid random delays.
513 */
intel_pipe_update_start(struct intel_atomic_state * state,struct intel_crtc * crtc)514 void intel_pipe_update_start(struct intel_atomic_state *state,
515 struct intel_crtc *crtc)
516 {
517 struct intel_display *display = to_intel_display(state);
518 const struct intel_crtc_state *old_crtc_state =
519 intel_atomic_get_old_crtc_state(state, crtc);
520 struct intel_crtc_state *new_crtc_state =
521 intel_atomic_get_new_crtc_state(state, crtc);
522 struct intel_vblank_evade_ctx evade;
523 int scanline;
524
525 drm_WARN_ON(display->drm, new_crtc_state->use_dsb);
526
527 intel_psr_lock(new_crtc_state);
528
529 if (new_crtc_state->do_async_flip) {
530 intel_crtc_prepare_vblank_event(new_crtc_state,
531 &crtc->flip_done_event);
532 return;
533 }
534
535 if (intel_crtc_needs_vblank_work(new_crtc_state))
536 intel_crtc_vblank_work_init(new_crtc_state);
537
538 if (state->base.legacy_cursor_update) {
539 struct intel_plane *plane;
540 struct intel_plane_state *old_plane_state, *new_plane_state;
541 int i;
542
543 for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state,
544 new_plane_state, i) {
545 if (old_plane_state->uapi.crtc == &crtc->base)
546 intel_plane_init_cursor_vblank_work(old_plane_state,
547 new_plane_state);
548 }
549 }
550
551 intel_vblank_evade_init(old_crtc_state, new_crtc_state, &evade);
552
553 if (drm_WARN_ON(display->drm, drm_crtc_vblank_get(&crtc->base)))
554 goto irq_disable;
555
556 /*
557 * Wait for psr to idle out after enabling the VBL interrupts
558 * VBL interrupts will start the PSR exit and prevent a PSR
559 * re-entry as well.
560 */
561 intel_psr_wait_for_idle_locked(new_crtc_state);
562
563 local_irq_disable();
564
565 crtc->debug.min_vbl = evade.min;
566 crtc->debug.max_vbl = evade.max;
567 trace_intel_pipe_update_start(crtc);
568
569 scanline = intel_vblank_evade(&evade);
570
571 drm_crtc_vblank_put(&crtc->base);
572
573 crtc->debug.scanline_start = scanline;
574 crtc->debug.start_vbl_time = ktime_get();
575 crtc->debug.start_vbl_count = intel_crtc_get_vblank_counter(crtc);
576
577 trace_intel_pipe_update_vblank_evaded(crtc);
578 return;
579
580 irq_disable:
581 local_irq_disable();
582 }
583
584 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_VBLANK_EVADE)
dbg_vblank_evade(struct intel_crtc * crtc,ktime_t end)585 static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end)
586 {
587 u64 delta = ktime_to_ns(ktime_sub(end, crtc->debug.start_vbl_time));
588 unsigned int h;
589
590 h = ilog2(delta >> 9);
591 if (h >= ARRAY_SIZE(crtc->debug.vbl.times))
592 h = ARRAY_SIZE(crtc->debug.vbl.times) - 1;
593 crtc->debug.vbl.times[h]++;
594
595 crtc->debug.vbl.sum += delta;
596 if (!crtc->debug.vbl.min || delta < crtc->debug.vbl.min)
597 crtc->debug.vbl.min = delta;
598 if (delta > crtc->debug.vbl.max)
599 crtc->debug.vbl.max = delta;
600
601 if (delta > 1000 * VBLANK_EVASION_TIME_US) {
602 drm_dbg_kms(crtc->base.dev,
603 "Atomic update on pipe (%c) took %lld us, max time under evasion is %u us\n",
604 pipe_name(crtc->pipe),
605 div_u64(delta, 1000),
606 VBLANK_EVASION_TIME_US);
607 crtc->debug.vbl.over++;
608 }
609 }
610 #else
dbg_vblank_evade(struct intel_crtc * crtc,ktime_t end)611 static void dbg_vblank_evade(struct intel_crtc *crtc, ktime_t end) {}
612 #endif
613
intel_crtc_arm_vblank_event(struct intel_crtc_state * crtc_state)614 void intel_crtc_arm_vblank_event(struct intel_crtc_state *crtc_state)
615 {
616 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
617 unsigned long irqflags;
618
619 if (!crtc_state->uapi.event)
620 return;
621
622 drm_WARN_ON(crtc->base.dev, drm_crtc_vblank_get(&crtc->base) != 0);
623
624 spin_lock_irqsave(&crtc->base.dev->event_lock, irqflags);
625 drm_crtc_arm_vblank_event(&crtc->base, crtc_state->uapi.event);
626 spin_unlock_irqrestore(&crtc->base.dev->event_lock, irqflags);
627
628 crtc_state->uapi.event = NULL;
629 }
630
intel_crtc_prepare_vblank_event(struct intel_crtc_state * crtc_state,struct drm_pending_vblank_event ** event)631 void intel_crtc_prepare_vblank_event(struct intel_crtc_state *crtc_state,
632 struct drm_pending_vblank_event **event)
633 {
634 struct intel_crtc *crtc = to_intel_crtc(crtc_state->uapi.crtc);
635 unsigned long irqflags;
636
637 spin_lock_irqsave(&crtc->base.dev->event_lock, irqflags);
638 *event = crtc_state->uapi.event;
639 spin_unlock_irqrestore(&crtc->base.dev->event_lock, irqflags);
640
641 crtc_state->uapi.event = NULL;
642 }
643
644 /**
645 * intel_pipe_update_end() - end update of a set of display registers
646 * @state: the atomic state
647 * @crtc: the crtc
648 *
649 * Mark the end of an update started with intel_pipe_update_start(). This
650 * re-enables interrupts and verifies the update was actually completed
651 * before a vblank.
652 */
intel_pipe_update_end(struct intel_atomic_state * state,struct intel_crtc * crtc)653 void intel_pipe_update_end(struct intel_atomic_state *state,
654 struct intel_crtc *crtc)
655 {
656 struct intel_display *display = to_intel_display(state);
657 struct intel_crtc_state *new_crtc_state =
658 intel_atomic_get_new_crtc_state(state, crtc);
659 enum pipe pipe = crtc->pipe;
660 int scanline_end = intel_get_crtc_scanline(crtc);
661 u32 end_vbl_count = intel_crtc_get_vblank_counter(crtc);
662 ktime_t end_vbl_time = ktime_get();
663 struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
664
665 drm_WARN_ON(display->drm, new_crtc_state->use_dsb);
666
667 if (new_crtc_state->do_async_flip)
668 goto out;
669
670 trace_intel_pipe_update_end(crtc, end_vbl_count, scanline_end);
671
672 /*
673 * Incase of mipi dsi command mode, we need to set frame update
674 * request for every commit.
675 */
676 if (DISPLAY_VER(display) >= 11 &&
677 intel_crtc_has_type(new_crtc_state, INTEL_OUTPUT_DSI))
678 icl_dsi_frame_update(new_crtc_state);
679
680 /* We're still in the vblank-evade critical section, this can't race.
681 * Would be slightly nice to just grab the vblank count and arm the
682 * event outside of the critical section - the spinlock might spin for a
683 * while ... */
684 if (intel_crtc_needs_vblank_work(new_crtc_state)) {
685 drm_vblank_work_schedule(&new_crtc_state->vblank_work,
686 drm_crtc_accurate_vblank_count(&crtc->base) + 1,
687 false);
688 } else {
689 intel_crtc_arm_vblank_event(new_crtc_state);
690 }
691
692 if (state->base.legacy_cursor_update) {
693 struct intel_plane *plane;
694 struct intel_plane_state *old_plane_state;
695 int i;
696
697 for_each_old_intel_plane_in_state(state, plane, old_plane_state, i) {
698 if (old_plane_state->uapi.crtc == &crtc->base &&
699 old_plane_state->unpin_work.vblank) {
700 drm_vblank_work_schedule(&old_plane_state->unpin_work,
701 drm_crtc_accurate_vblank_count(&crtc->base) + 1,
702 false);
703
704 /* Remove plane from atomic state, cleanup/free is done from vblank worker. */
705 memset(&state->base.planes[i], 0, sizeof(state->base.planes[i]));
706 }
707 }
708 }
709
710 /*
711 * Send VRR Push to terminate Vblank. If we are already in vblank
712 * this has to be done _after_ sampling the frame counter, as
713 * otherwise the push would immediately terminate the vblank and
714 * the sampled frame counter would correspond to the next frame
715 * instead of the current frame.
716 *
717 * There is a tiny race here (iff vblank evasion failed us) where
718 * we might sample the frame counter just before vmax vblank start
719 * but the push would be sent just after it. That would cause the
720 * push to affect the next frame instead of the current frame,
721 * which would cause the next frame to terminate already at vmin
722 * vblank start instead of vmax vblank start.
723 */
724 if (!state->base.legacy_cursor_update)
725 intel_vrr_send_push(NULL, new_crtc_state);
726
727 local_irq_enable();
728
729 if (intel_vgpu_active(dev_priv))
730 goto out;
731
732 if (crtc->debug.start_vbl_count &&
733 crtc->debug.start_vbl_count != end_vbl_count) {
734 drm_err(display->drm,
735 "Atomic update failure on pipe %c (start=%u end=%u) time %lld us, min %d, max %d, scanline start %d, end %d\n",
736 pipe_name(pipe), crtc->debug.start_vbl_count,
737 end_vbl_count,
738 ktime_us_delta(end_vbl_time,
739 crtc->debug.start_vbl_time),
740 crtc->debug.min_vbl, crtc->debug.max_vbl,
741 crtc->debug.scanline_start, scanline_end);
742 }
743
744 dbg_vblank_evade(crtc, end_vbl_time);
745
746 out:
747 intel_psr_unlock(new_crtc_state);
748 }
749