1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
3 */
4
5 #define pr_fmt(fmt) "[drm:%s:%d] " fmt, __func__, __LINE__
6
7 #include <linux/debugfs.h>
8 #include <linux/errno.h>
9 #include <linux/mutex.h>
10 #include <linux/pm_opp.h>
11 #include <linux/sort.h>
12 #include <linux/clk.h>
13 #include <linux/bitmap.h>
14
15 #include "dpu_kms.h"
16 #include "dpu_trace.h"
17 #include "dpu_crtc.h"
18 #include "dpu_core_perf.h"
19
20 /**
21 * enum dpu_perf_mode - performance tuning mode
22 * @DPU_PERF_MODE_NORMAL: performance controlled by user mode client
23 * @DPU_PERF_MODE_MINIMUM: performance bounded by minimum setting
24 * @DPU_PERF_MODE_FIXED: performance bounded by fixed setting
25 * @DPU_PERF_MODE_MAX: maximum value, used for error checking
26 */
27 enum dpu_perf_mode {
28 DPU_PERF_MODE_NORMAL,
29 DPU_PERF_MODE_MINIMUM,
30 DPU_PERF_MODE_FIXED,
31 DPU_PERF_MODE_MAX
32 };
33
34 /**
35 * dpu_core_perf_adjusted_mode_clk - Adjust given mode clock rate according to
36 * the perf clock factor.
37 * @crtc_clk_rate - Unadjusted mode clock rate
38 * @perf_cfg: performance configuration
39 */
dpu_core_perf_adjusted_mode_clk(u64 mode_clk_rate,const struct dpu_perf_cfg * perf_cfg)40 u64 dpu_core_perf_adjusted_mode_clk(u64 mode_clk_rate,
41 const struct dpu_perf_cfg *perf_cfg)
42 {
43 u32 clk_factor;
44
45 clk_factor = perf_cfg->clk_inefficiency_factor;
46 if (clk_factor) {
47 mode_clk_rate *= clk_factor;
48 do_div(mode_clk_rate, 100);
49 }
50
51 return mode_clk_rate;
52 }
53
54 /**
55 * _dpu_core_perf_calc_bw() - to calculate BW per crtc
56 * @perf_cfg: performance configuration
57 * @crtc: pointer to a crtc
58 * Return: returns aggregated BW for all planes in crtc.
59 */
_dpu_core_perf_calc_bw(const struct dpu_perf_cfg * perf_cfg,struct drm_crtc * crtc)60 static u64 _dpu_core_perf_calc_bw(const struct dpu_perf_cfg *perf_cfg,
61 struct drm_crtc *crtc)
62 {
63 struct drm_plane *plane;
64 struct dpu_plane_state *pstate;
65 u64 crtc_plane_bw = 0;
66 u32 bw_factor;
67
68 drm_atomic_crtc_for_each_plane(plane, crtc) {
69 pstate = to_dpu_plane_state(plane->state);
70 if (!pstate)
71 continue;
72
73 crtc_plane_bw += pstate->plane_fetch_bw;
74 }
75
76 bw_factor = perf_cfg->bw_inefficiency_factor;
77 if (bw_factor) {
78 crtc_plane_bw *= bw_factor;
79 do_div(crtc_plane_bw, 100);
80 }
81
82 return crtc_plane_bw;
83 }
84
85 /**
86 * _dpu_core_perf_calc_clk() - to calculate clock per crtc
87 * @perf_cfg: performance configuration
88 * @crtc: pointer to a crtc
89 * @state: pointer to a crtc state
90 * Return: returns max clk for all planes in crtc.
91 */
_dpu_core_perf_calc_clk(const struct dpu_perf_cfg * perf_cfg,struct drm_crtc * crtc,struct drm_crtc_state * state)92 static u64 _dpu_core_perf_calc_clk(const struct dpu_perf_cfg *perf_cfg,
93 struct drm_crtc *crtc, struct drm_crtc_state *state)
94 {
95 struct drm_plane *plane;
96 struct dpu_plane_state *pstate;
97 struct drm_display_mode *mode;
98 u64 mode_clk;
99
100 mode = &state->adjusted_mode;
101
102 mode_clk = (u64)mode->vtotal * mode->hdisplay * drm_mode_vrefresh(mode);
103
104 drm_atomic_crtc_for_each_plane(plane, crtc) {
105 pstate = to_dpu_plane_state(plane->state);
106 if (!pstate)
107 continue;
108
109 mode_clk = max(pstate->plane_clk, mode_clk);
110 }
111
112 return dpu_core_perf_adjusted_mode_clk(mode_clk, perf_cfg);
113 }
114
_dpu_crtc_get_kms(struct drm_crtc * crtc)115 static struct dpu_kms *_dpu_crtc_get_kms(struct drm_crtc *crtc)
116 {
117 struct msm_drm_private *priv;
118 priv = crtc->dev->dev_private;
119 return to_dpu_kms(priv->kms);
120 }
121
_dpu_core_perf_calc_crtc(const struct dpu_core_perf * core_perf,struct drm_crtc * crtc,struct drm_crtc_state * state,struct dpu_core_perf_params * perf)122 static void _dpu_core_perf_calc_crtc(const struct dpu_core_perf *core_perf,
123 struct drm_crtc *crtc,
124 struct drm_crtc_state *state,
125 struct dpu_core_perf_params *perf)
126 {
127 const struct dpu_perf_cfg *perf_cfg = core_perf->perf_cfg;
128
129 if (!perf_cfg || !crtc || !state || !perf) {
130 DPU_ERROR("invalid parameters\n");
131 return;
132 }
133
134 perf->bw_ctl = _dpu_core_perf_calc_bw(perf_cfg, crtc);
135 perf->max_per_pipe_ib = perf_cfg->min_dram_ib;
136 perf->core_clk_rate = _dpu_core_perf_calc_clk(perf_cfg, crtc, state);
137 DRM_DEBUG_ATOMIC(
138 "crtc=%d clk_rate=%llu core_ib=%u core_ab=%u\n",
139 crtc->base.id, perf->core_clk_rate,
140 perf->max_per_pipe_ib,
141 (u32)DIV_ROUND_UP_ULL(perf->bw_ctl, 1000));
142 }
143
dpu_core_perf_aggregate(struct drm_device * ddev,enum dpu_crtc_client_type curr_client_type,struct dpu_core_perf_params * perf)144 static void dpu_core_perf_aggregate(struct drm_device *ddev,
145 enum dpu_crtc_client_type curr_client_type,
146 struct dpu_core_perf_params *perf)
147 {
148 struct dpu_crtc_state *dpu_cstate;
149 struct drm_crtc *tmp_crtc;
150
151 drm_for_each_crtc(tmp_crtc, ddev) {
152 if (tmp_crtc->enabled &&
153 curr_client_type == dpu_crtc_get_client_type(tmp_crtc)) {
154 dpu_cstate = to_dpu_crtc_state(tmp_crtc->state);
155
156 perf->max_per_pipe_ib = max(perf->max_per_pipe_ib,
157 dpu_cstate->new_perf.max_per_pipe_ib);
158
159 perf->bw_ctl += dpu_cstate->new_perf.bw_ctl;
160
161 DRM_DEBUG_ATOMIC("crtc=%d bw=%llu\n",
162 tmp_crtc->base.id,
163 dpu_cstate->new_perf.bw_ctl);
164 }
165 }
166 }
167
168 /**
169 * dpu_core_perf_crtc_check - validate performance of the given crtc state
170 * @crtc: Pointer to crtc
171 * @state: Pointer to new crtc state
172 * return: zero if success, or error code otherwise
173 */
dpu_core_perf_crtc_check(struct drm_crtc * crtc,struct drm_crtc_state * state)174 int dpu_core_perf_crtc_check(struct drm_crtc *crtc,
175 struct drm_crtc_state *state)
176 {
177 u32 bw, threshold;
178 struct dpu_crtc_state *dpu_cstate;
179 struct dpu_kms *kms;
180 struct dpu_core_perf_params perf = { 0 };
181
182 if (!crtc || !state) {
183 DPU_ERROR("invalid crtc\n");
184 return -EINVAL;
185 }
186
187 kms = _dpu_crtc_get_kms(crtc);
188
189 /* we only need bandwidth check on real-time clients (interfaces) */
190 if (dpu_crtc_get_client_type(crtc) == NRT_CLIENT)
191 return 0;
192
193 dpu_cstate = to_dpu_crtc_state(state);
194
195 /* obtain new values */
196 _dpu_core_perf_calc_crtc(&kms->perf, crtc, state, &dpu_cstate->new_perf);
197
198 dpu_core_perf_aggregate(crtc->dev, dpu_crtc_get_client_type(crtc), &perf);
199
200 /* convert bandwidth to kb */
201 bw = DIV_ROUND_UP_ULL(perf.bw_ctl, 1000);
202 DRM_DEBUG_ATOMIC("calculated bandwidth=%uk\n", bw);
203
204 threshold = kms->perf.perf_cfg->max_bw_high;
205
206 DRM_DEBUG_ATOMIC("final threshold bw limit = %d\n", threshold);
207
208 if (!threshold) {
209 DPU_ERROR("no bandwidth limits specified\n");
210 return -E2BIG;
211 } else if (bw > threshold) {
212 DPU_ERROR("exceeds bandwidth: %ukb > %ukb\n", bw,
213 threshold);
214 return -E2BIG;
215 }
216
217 return 0;
218 }
219
_dpu_core_perf_crtc_update_bus(struct dpu_kms * kms,struct drm_crtc * crtc)220 static int _dpu_core_perf_crtc_update_bus(struct dpu_kms *kms,
221 struct drm_crtc *crtc)
222 {
223 struct dpu_core_perf_params perf = { 0 };
224 int i, ret = 0;
225 u32 avg_bw;
226 u32 peak_bw;
227
228 if (!kms->num_paths)
229 return 0;
230
231 if (kms->perf.perf_tune.mode == DPU_PERF_MODE_MINIMUM) {
232 avg_bw = 0;
233 peak_bw = 0;
234 } else if (kms->perf.perf_tune.mode == DPU_PERF_MODE_FIXED) {
235 avg_bw = kms->perf.fix_core_ab_vote;
236 peak_bw = kms->perf.fix_core_ib_vote;
237 } else {
238 dpu_core_perf_aggregate(crtc->dev, dpu_crtc_get_client_type(crtc), &perf);
239
240 avg_bw = div_u64(perf.bw_ctl, 1000); /*Bps_to_icc*/
241 peak_bw = perf.max_per_pipe_ib;
242 }
243
244 avg_bw /= kms->num_paths;
245
246 for (i = 0; i < kms->num_paths; i++)
247 icc_set_bw(kms->path[i], avg_bw, peak_bw);
248
249 return ret;
250 }
251
252 /**
253 * dpu_core_perf_crtc_release_bw() - request zero bandwidth
254 * @crtc: pointer to a crtc
255 *
256 * Function checks a state variable for the crtc, if all pending commit
257 * requests are done, meaning no more bandwidth is needed, release
258 * bandwidth request.
259 */
dpu_core_perf_crtc_release_bw(struct drm_crtc * crtc)260 void dpu_core_perf_crtc_release_bw(struct drm_crtc *crtc)
261 {
262 struct dpu_crtc *dpu_crtc;
263 struct dpu_kms *kms;
264
265 if (!crtc) {
266 DPU_ERROR("invalid crtc\n");
267 return;
268 }
269
270 kms = _dpu_crtc_get_kms(crtc);
271 dpu_crtc = to_dpu_crtc(crtc);
272
273 if (atomic_dec_return(&kms->bandwidth_ref) > 0)
274 return;
275
276 /* Release the bandwidth */
277 if (kms->perf.enable_bw_release) {
278 trace_dpu_cmd_release_bw(crtc->base.id);
279 DRM_DEBUG_ATOMIC("Release BW crtc=%d\n", crtc->base.id);
280 dpu_crtc->cur_perf.bw_ctl = 0;
281 _dpu_core_perf_crtc_update_bus(kms, crtc);
282 }
283 }
284
_dpu_core_perf_get_core_clk_rate(struct dpu_kms * kms)285 static u64 _dpu_core_perf_get_core_clk_rate(struct dpu_kms *kms)
286 {
287 u64 clk_rate;
288 struct drm_crtc *crtc;
289 struct dpu_crtc_state *dpu_cstate;
290
291 if (kms->perf.perf_tune.mode == DPU_PERF_MODE_FIXED)
292 return kms->perf.fix_core_clk_rate;
293
294 if (kms->perf.perf_tune.mode == DPU_PERF_MODE_MINIMUM)
295 return kms->perf.max_core_clk_rate;
296
297 clk_rate = 0;
298 drm_for_each_crtc(crtc, kms->dev) {
299 if (crtc->enabled) {
300 dpu_cstate = to_dpu_crtc_state(crtc->state);
301 clk_rate = max(dpu_cstate->new_perf.core_clk_rate,
302 clk_rate);
303 }
304 }
305
306 return clk_rate;
307 }
308
309 /**
310 * dpu_core_perf_crtc_update - update performance of the given crtc
311 * @crtc: Pointer to crtc
312 * @params_changed: true if crtc parameters are modified
313 * return: zero if success, or error code otherwise
314 */
dpu_core_perf_crtc_update(struct drm_crtc * crtc,int params_changed)315 int dpu_core_perf_crtc_update(struct drm_crtc *crtc,
316 int params_changed)
317 {
318 struct dpu_core_perf_params *new, *old;
319 bool update_bus = false, update_clk = false;
320 u64 clk_rate = 0;
321 struct dpu_crtc *dpu_crtc;
322 struct dpu_crtc_state *dpu_cstate;
323 struct dpu_kms *kms;
324 int ret;
325
326 if (!crtc) {
327 DPU_ERROR("invalid crtc\n");
328 return -EINVAL;
329 }
330
331 kms = _dpu_crtc_get_kms(crtc);
332
333 dpu_crtc = to_dpu_crtc(crtc);
334 dpu_cstate = to_dpu_crtc_state(crtc->state);
335
336 DRM_DEBUG_ATOMIC("crtc:%d enabled:%d core_clk:%llu\n",
337 crtc->base.id, crtc->enabled, kms->perf.core_clk_rate);
338
339 old = &dpu_crtc->cur_perf;
340 new = &dpu_cstate->new_perf;
341
342 if (crtc->enabled) {
343 /*
344 * cases for bus bandwidth update.
345 * 1. new bandwidth vote - "ab or ib vote" is higher
346 * than current vote for update request.
347 * 2. new bandwidth vote - "ab or ib vote" is lower
348 * than current vote at end of commit or stop.
349 */
350 if ((params_changed && ((new->bw_ctl > old->bw_ctl) ||
351 (new->max_per_pipe_ib > old->max_per_pipe_ib))) ||
352 (!params_changed && ((new->bw_ctl < old->bw_ctl) ||
353 (new->max_per_pipe_ib < old->max_per_pipe_ib)))) {
354 DRM_DEBUG_ATOMIC("crtc=%d p=%d new_bw=%llu,old_bw=%llu\n",
355 crtc->base.id, params_changed,
356 new->bw_ctl, old->bw_ctl);
357 old->bw_ctl = new->bw_ctl;
358 old->max_per_pipe_ib = new->max_per_pipe_ib;
359 update_bus = true;
360 }
361
362 if ((params_changed && new->core_clk_rate > old->core_clk_rate) ||
363 (!params_changed && new->core_clk_rate < old->core_clk_rate)) {
364 old->core_clk_rate = new->core_clk_rate;
365 update_clk = true;
366 }
367 } else {
368 DRM_DEBUG_ATOMIC("crtc=%d disable\n", crtc->base.id);
369 memset(old, 0, sizeof(*old));
370 update_bus = true;
371 update_clk = true;
372 }
373
374 trace_dpu_perf_crtc_update(crtc->base.id, new->bw_ctl,
375 new->core_clk_rate, !crtc->enabled, update_bus, update_clk);
376
377 if (update_bus) {
378 ret = _dpu_core_perf_crtc_update_bus(kms, crtc);
379 if (ret) {
380 DPU_ERROR("crtc-%d: failed to update bus bw vote\n",
381 crtc->base.id);
382 return ret;
383 }
384 }
385
386 /*
387 * Update the clock after bandwidth vote to ensure
388 * bandwidth is available before clock rate is increased.
389 */
390 if (update_clk) {
391 clk_rate = _dpu_core_perf_get_core_clk_rate(kms);
392
393 DRM_DEBUG_ATOMIC("clk:%llu\n", clk_rate);
394
395 trace_dpu_core_perf_update_clk(kms->dev, !crtc->enabled, clk_rate);
396
397 clk_rate = min(clk_rate, kms->perf.max_core_clk_rate);
398 ret = dev_pm_opp_set_rate(&kms->pdev->dev, clk_rate);
399 if (ret) {
400 DPU_ERROR("failed to set core clock rate %llu\n", clk_rate);
401 return ret;
402 }
403
404 kms->perf.core_clk_rate = clk_rate;
405 DRM_DEBUG_ATOMIC("update clk rate = %lld HZ\n", clk_rate);
406 }
407 return 0;
408 }
409
410 #ifdef CONFIG_DEBUG_FS
411
_dpu_core_perf_mode_write(struct file * file,const char __user * user_buf,size_t count,loff_t * ppos)412 static ssize_t _dpu_core_perf_mode_write(struct file *file,
413 const char __user *user_buf, size_t count, loff_t *ppos)
414 {
415 struct dpu_core_perf *perf = file->private_data;
416 u32 perf_mode = 0;
417 int ret;
418
419 ret = kstrtouint_from_user(user_buf, count, 0, &perf_mode);
420 if (ret)
421 return ret;
422
423 if (perf_mode >= DPU_PERF_MODE_MAX)
424 return -EINVAL;
425
426 if (perf_mode == DPU_PERF_MODE_FIXED) {
427 DRM_INFO("fix performance mode\n");
428 } else if (perf_mode == DPU_PERF_MODE_MINIMUM) {
429 /* run the driver with max clk and BW vote */
430 DRM_INFO("minimum performance mode\n");
431 } else if (perf_mode == DPU_PERF_MODE_NORMAL) {
432 /* reset the perf tune params to 0 */
433 DRM_INFO("normal performance mode\n");
434 }
435 perf->perf_tune.mode = perf_mode;
436
437 return count;
438 }
439
_dpu_core_perf_mode_read(struct file * file,char __user * buff,size_t count,loff_t * ppos)440 static ssize_t _dpu_core_perf_mode_read(struct file *file,
441 char __user *buff, size_t count, loff_t *ppos)
442 {
443 struct dpu_core_perf *perf = file->private_data;
444 int len;
445 char buf[128];
446
447 len = scnprintf(buf, sizeof(buf),
448 "mode %d\n",
449 perf->perf_tune.mode);
450
451 return simple_read_from_buffer(buff, count, ppos, buf, len);
452 }
453
454 static const struct file_operations dpu_core_perf_mode_fops = {
455 .open = simple_open,
456 .read = _dpu_core_perf_mode_read,
457 .write = _dpu_core_perf_mode_write,
458 };
459
460 /**
461 * dpu_core_perf_debugfs_init - initialize debugfs for core performance context
462 * @dpu_kms: Pointer to the dpu_kms struct
463 * @parent: Pointer to parent debugfs
464 */
dpu_core_perf_debugfs_init(struct dpu_kms * dpu_kms,struct dentry * parent)465 int dpu_core_perf_debugfs_init(struct dpu_kms *dpu_kms, struct dentry *parent)
466 {
467 struct dpu_core_perf *perf = &dpu_kms->perf;
468 struct dentry *entry;
469
470 entry = debugfs_create_dir("core_perf", parent);
471
472 debugfs_create_u64("max_core_clk_rate", 0600, entry,
473 &perf->max_core_clk_rate);
474 debugfs_create_u64("core_clk_rate", 0600, entry,
475 &perf->core_clk_rate);
476 debugfs_create_u32("enable_bw_release", 0600, entry,
477 (u32 *)&perf->enable_bw_release);
478 debugfs_create_u32("low_core_ab", 0400, entry,
479 (u32 *)&perf->perf_cfg->max_bw_low);
480 debugfs_create_u32("max_core_ab", 0400, entry,
481 (u32 *)&perf->perf_cfg->max_bw_high);
482 debugfs_create_u32("min_core_ib", 0400, entry,
483 (u32 *)&perf->perf_cfg->min_core_ib);
484 debugfs_create_u32("min_llcc_ib", 0400, entry,
485 (u32 *)&perf->perf_cfg->min_llcc_ib);
486 debugfs_create_u32("min_dram_ib", 0400, entry,
487 (u32 *)&perf->perf_cfg->min_dram_ib);
488 debugfs_create_file("perf_mode", 0600, entry,
489 (u32 *)perf, &dpu_core_perf_mode_fops);
490 debugfs_create_u64("fix_core_clk_rate", 0600, entry,
491 &perf->fix_core_clk_rate);
492 debugfs_create_u32("fix_core_ib_vote", 0600, entry,
493 &perf->fix_core_ib_vote);
494 debugfs_create_u32("fix_core_ab_vote", 0600, entry,
495 &perf->fix_core_ab_vote);
496
497 return 0;
498 }
499 #endif
500
501 /**
502 * dpu_core_perf_init - initialize the given core performance context
503 * @perf: Pointer to core performance context
504 * @perf_cfg: Pointer to platform performance configuration
505 * @max_core_clk_rate: Maximum core clock rate
506 */
dpu_core_perf_init(struct dpu_core_perf * perf,const struct dpu_perf_cfg * perf_cfg,unsigned long max_core_clk_rate)507 int dpu_core_perf_init(struct dpu_core_perf *perf,
508 const struct dpu_perf_cfg *perf_cfg,
509 unsigned long max_core_clk_rate)
510 {
511 perf->perf_cfg = perf_cfg;
512 perf->max_core_clk_rate = max_core_clk_rate;
513
514 return 0;
515 }
516