1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
3 */
4
5 #define pr_fmt(fmt) "[drm:%s:%d] " fmt, __func__, __LINE__
6
7 #include <linux/debugfs.h>
8 #include <linux/errno.h>
9 #include <linux/mutex.h>
10 #include <linux/pm_opp.h>
11 #include <linux/sort.h>
12 #include <linux/clk.h>
13 #include <linux/bitmap.h>
14
15 #include "dpu_kms.h"
16 #include "dpu_trace.h"
17 #include "dpu_crtc.h"
18 #include "dpu_core_perf.h"
19
20 /**
21 * enum dpu_perf_mode - performance tuning mode
22 * @DPU_PERF_MODE_NORMAL: performance controlled by user mode client
23 * @DPU_PERF_MODE_MINIMUM: performance bounded by minimum setting
24 * @DPU_PERF_MODE_FIXED: performance bounded by fixed setting
25 * @DPU_PERF_MODE_MAX: maximum value, used for error checking
26 */
27 enum dpu_perf_mode {
28 DPU_PERF_MODE_NORMAL,
29 DPU_PERF_MODE_MINIMUM,
30 DPU_PERF_MODE_FIXED,
31 DPU_PERF_MODE_MAX
32 };
33
34 /**
35 * _dpu_core_perf_calc_bw() - to calculate BW per crtc
36 * @perf_cfg: performance configuration
37 * @crtc: pointer to a crtc
38 * Return: returns aggregated BW for all planes in crtc.
39 */
_dpu_core_perf_calc_bw(const struct dpu_perf_cfg * perf_cfg,struct drm_crtc * crtc)40 static u64 _dpu_core_perf_calc_bw(const struct dpu_perf_cfg *perf_cfg,
41 struct drm_crtc *crtc)
42 {
43 struct drm_plane *plane;
44 struct dpu_plane_state *pstate;
45 u64 crtc_plane_bw = 0;
46 u32 bw_factor;
47
48 drm_atomic_crtc_for_each_plane(plane, crtc) {
49 pstate = to_dpu_plane_state(plane->state);
50 if (!pstate)
51 continue;
52
53 crtc_plane_bw += pstate->plane_fetch_bw;
54 }
55
56 bw_factor = perf_cfg->bw_inefficiency_factor;
57 if (bw_factor) {
58 crtc_plane_bw *= bw_factor;
59 do_div(crtc_plane_bw, 100);
60 }
61
62 return crtc_plane_bw;
63 }
64
65 /**
66 * _dpu_core_perf_calc_clk() - to calculate clock per crtc
67 * @perf_cfg: performance configuration
68 * @crtc: pointer to a crtc
69 * @state: pointer to a crtc state
70 * Return: returns max clk for all planes in crtc.
71 */
_dpu_core_perf_calc_clk(const struct dpu_perf_cfg * perf_cfg,struct drm_crtc * crtc,struct drm_crtc_state * state)72 static u64 _dpu_core_perf_calc_clk(const struct dpu_perf_cfg *perf_cfg,
73 struct drm_crtc *crtc, struct drm_crtc_state *state)
74 {
75 struct drm_plane *plane;
76 struct dpu_plane_state *pstate;
77 struct drm_display_mode *mode;
78 u64 crtc_clk;
79 u32 clk_factor;
80
81 mode = &state->adjusted_mode;
82
83 crtc_clk = (u64)mode->vtotal * mode->hdisplay * drm_mode_vrefresh(mode);
84
85 drm_atomic_crtc_for_each_plane(plane, crtc) {
86 pstate = to_dpu_plane_state(plane->state);
87 if (!pstate)
88 continue;
89
90 crtc_clk = max(pstate->plane_clk, crtc_clk);
91 }
92
93 clk_factor = perf_cfg->clk_inefficiency_factor;
94 if (clk_factor) {
95 crtc_clk *= clk_factor;
96 do_div(crtc_clk, 100);
97 }
98
99 return crtc_clk;
100 }
101
_dpu_crtc_get_kms(struct drm_crtc * crtc)102 static struct dpu_kms *_dpu_crtc_get_kms(struct drm_crtc *crtc)
103 {
104 struct msm_drm_private *priv;
105 priv = crtc->dev->dev_private;
106 return to_dpu_kms(priv->kms);
107 }
108
_dpu_core_perf_calc_crtc(const struct dpu_core_perf * core_perf,struct drm_crtc * crtc,struct drm_crtc_state * state,struct dpu_core_perf_params * perf)109 static void _dpu_core_perf_calc_crtc(const struct dpu_core_perf *core_perf,
110 struct drm_crtc *crtc,
111 struct drm_crtc_state *state,
112 struct dpu_core_perf_params *perf)
113 {
114 const struct dpu_perf_cfg *perf_cfg = core_perf->perf_cfg;
115
116 if (!perf_cfg || !crtc || !state || !perf) {
117 DPU_ERROR("invalid parameters\n");
118 return;
119 }
120
121 perf->bw_ctl = _dpu_core_perf_calc_bw(perf_cfg, crtc);
122 perf->max_per_pipe_ib = perf_cfg->min_dram_ib;
123 perf->core_clk_rate = _dpu_core_perf_calc_clk(perf_cfg, crtc, state);
124 DRM_DEBUG_ATOMIC(
125 "crtc=%d clk_rate=%llu core_ib=%u core_ab=%u\n",
126 crtc->base.id, perf->core_clk_rate,
127 perf->max_per_pipe_ib,
128 (u32)DIV_ROUND_UP_ULL(perf->bw_ctl, 1000));
129 }
130
dpu_core_perf_aggregate(struct drm_device * ddev,enum dpu_crtc_client_type curr_client_type,struct dpu_core_perf_params * perf)131 static void dpu_core_perf_aggregate(struct drm_device *ddev,
132 enum dpu_crtc_client_type curr_client_type,
133 struct dpu_core_perf_params *perf)
134 {
135 struct dpu_crtc_state *dpu_cstate;
136 struct drm_crtc *tmp_crtc;
137
138 drm_for_each_crtc(tmp_crtc, ddev) {
139 if (tmp_crtc->enabled &&
140 curr_client_type == dpu_crtc_get_client_type(tmp_crtc)) {
141 dpu_cstate = to_dpu_crtc_state(tmp_crtc->state);
142
143 perf->max_per_pipe_ib = max(perf->max_per_pipe_ib,
144 dpu_cstate->new_perf.max_per_pipe_ib);
145
146 perf->bw_ctl += dpu_cstate->new_perf.bw_ctl;
147
148 DRM_DEBUG_ATOMIC("crtc=%d bw=%llu\n",
149 tmp_crtc->base.id,
150 dpu_cstate->new_perf.bw_ctl);
151 }
152 }
153 }
154
155 /**
156 * dpu_core_perf_crtc_check - validate performance of the given crtc state
157 * @crtc: Pointer to crtc
158 * @state: Pointer to new crtc state
159 * return: zero if success, or error code otherwise
160 */
dpu_core_perf_crtc_check(struct drm_crtc * crtc,struct drm_crtc_state * state)161 int dpu_core_perf_crtc_check(struct drm_crtc *crtc,
162 struct drm_crtc_state *state)
163 {
164 u32 bw, threshold;
165 struct dpu_crtc_state *dpu_cstate;
166 struct dpu_kms *kms;
167 struct dpu_core_perf_params perf = { 0 };
168
169 if (!crtc || !state) {
170 DPU_ERROR("invalid crtc\n");
171 return -EINVAL;
172 }
173
174 kms = _dpu_crtc_get_kms(crtc);
175
176 /* we only need bandwidth check on real-time clients (interfaces) */
177 if (dpu_crtc_get_client_type(crtc) == NRT_CLIENT)
178 return 0;
179
180 dpu_cstate = to_dpu_crtc_state(state);
181
182 /* obtain new values */
183 _dpu_core_perf_calc_crtc(&kms->perf, crtc, state, &dpu_cstate->new_perf);
184
185 dpu_core_perf_aggregate(crtc->dev, dpu_crtc_get_client_type(crtc), &perf);
186
187 /* convert bandwidth to kb */
188 bw = DIV_ROUND_UP_ULL(perf.bw_ctl, 1000);
189 DRM_DEBUG_ATOMIC("calculated bandwidth=%uk\n", bw);
190
191 threshold = kms->perf.perf_cfg->max_bw_high;
192
193 DRM_DEBUG_ATOMIC("final threshold bw limit = %d\n", threshold);
194
195 if (!threshold) {
196 DPU_ERROR("no bandwidth limits specified\n");
197 return -E2BIG;
198 } else if (bw > threshold) {
199 DPU_ERROR("exceeds bandwidth: %ukb > %ukb\n", bw,
200 threshold);
201 return -E2BIG;
202 }
203
204 return 0;
205 }
206
_dpu_core_perf_crtc_update_bus(struct dpu_kms * kms,struct drm_crtc * crtc)207 static int _dpu_core_perf_crtc_update_bus(struct dpu_kms *kms,
208 struct drm_crtc *crtc)
209 {
210 struct dpu_core_perf_params perf = { 0 };
211 int i, ret = 0;
212 u32 avg_bw;
213 u32 peak_bw;
214
215 if (!kms->num_paths)
216 return 0;
217
218 if (kms->perf.perf_tune.mode == DPU_PERF_MODE_MINIMUM) {
219 avg_bw = 0;
220 peak_bw = 0;
221 } else if (kms->perf.perf_tune.mode == DPU_PERF_MODE_FIXED) {
222 avg_bw = kms->perf.fix_core_ab_vote;
223 peak_bw = kms->perf.fix_core_ib_vote;
224 } else {
225 dpu_core_perf_aggregate(crtc->dev, dpu_crtc_get_client_type(crtc), &perf);
226
227 avg_bw = div_u64(perf.bw_ctl, 1000); /*Bps_to_icc*/
228 peak_bw = perf.max_per_pipe_ib;
229 }
230
231 avg_bw /= kms->num_paths;
232
233 for (i = 0; i < kms->num_paths; i++)
234 icc_set_bw(kms->path[i], avg_bw, peak_bw);
235
236 return ret;
237 }
238
239 /**
240 * dpu_core_perf_crtc_release_bw() - request zero bandwidth
241 * @crtc: pointer to a crtc
242 *
243 * Function checks a state variable for the crtc, if all pending commit
244 * requests are done, meaning no more bandwidth is needed, release
245 * bandwidth request.
246 */
dpu_core_perf_crtc_release_bw(struct drm_crtc * crtc)247 void dpu_core_perf_crtc_release_bw(struct drm_crtc *crtc)
248 {
249 struct dpu_crtc *dpu_crtc;
250 struct dpu_kms *kms;
251
252 if (!crtc) {
253 DPU_ERROR("invalid crtc\n");
254 return;
255 }
256
257 kms = _dpu_crtc_get_kms(crtc);
258 dpu_crtc = to_dpu_crtc(crtc);
259
260 if (atomic_dec_return(&kms->bandwidth_ref) > 0)
261 return;
262
263 /* Release the bandwidth */
264 if (kms->perf.enable_bw_release) {
265 trace_dpu_cmd_release_bw(crtc->base.id);
266 DRM_DEBUG_ATOMIC("Release BW crtc=%d\n", crtc->base.id);
267 dpu_crtc->cur_perf.bw_ctl = 0;
268 _dpu_core_perf_crtc_update_bus(kms, crtc);
269 }
270 }
271
_dpu_core_perf_get_core_clk_rate(struct dpu_kms * kms)272 static u64 _dpu_core_perf_get_core_clk_rate(struct dpu_kms *kms)
273 {
274 u64 clk_rate;
275 struct drm_crtc *crtc;
276 struct dpu_crtc_state *dpu_cstate;
277
278 if (kms->perf.perf_tune.mode == DPU_PERF_MODE_FIXED)
279 return kms->perf.fix_core_clk_rate;
280
281 if (kms->perf.perf_tune.mode == DPU_PERF_MODE_MINIMUM)
282 return kms->perf.max_core_clk_rate;
283
284 clk_rate = 0;
285 drm_for_each_crtc(crtc, kms->dev) {
286 if (crtc->enabled) {
287 dpu_cstate = to_dpu_crtc_state(crtc->state);
288 clk_rate = max(dpu_cstate->new_perf.core_clk_rate,
289 clk_rate);
290 }
291 }
292
293 return clk_rate;
294 }
295
296 /**
297 * dpu_core_perf_crtc_update - update performance of the given crtc
298 * @crtc: Pointer to crtc
299 * @params_changed: true if crtc parameters are modified
300 * return: zero if success, or error code otherwise
301 */
dpu_core_perf_crtc_update(struct drm_crtc * crtc,int params_changed)302 int dpu_core_perf_crtc_update(struct drm_crtc *crtc,
303 int params_changed)
304 {
305 struct dpu_core_perf_params *new, *old;
306 bool update_bus = false, update_clk = false;
307 u64 clk_rate = 0;
308 struct dpu_crtc *dpu_crtc;
309 struct dpu_crtc_state *dpu_cstate;
310 struct dpu_kms *kms;
311 int ret;
312
313 if (!crtc) {
314 DPU_ERROR("invalid crtc\n");
315 return -EINVAL;
316 }
317
318 kms = _dpu_crtc_get_kms(crtc);
319
320 dpu_crtc = to_dpu_crtc(crtc);
321 dpu_cstate = to_dpu_crtc_state(crtc->state);
322
323 DRM_DEBUG_ATOMIC("crtc:%d enabled:%d core_clk:%llu\n",
324 crtc->base.id, crtc->enabled, kms->perf.core_clk_rate);
325
326 old = &dpu_crtc->cur_perf;
327 new = &dpu_cstate->new_perf;
328
329 if (crtc->enabled) {
330 /*
331 * cases for bus bandwidth update.
332 * 1. new bandwidth vote - "ab or ib vote" is higher
333 * than current vote for update request.
334 * 2. new bandwidth vote - "ab or ib vote" is lower
335 * than current vote at end of commit or stop.
336 */
337 if ((params_changed && ((new->bw_ctl > old->bw_ctl) ||
338 (new->max_per_pipe_ib > old->max_per_pipe_ib))) ||
339 (!params_changed && ((new->bw_ctl < old->bw_ctl) ||
340 (new->max_per_pipe_ib < old->max_per_pipe_ib)))) {
341 DRM_DEBUG_ATOMIC("crtc=%d p=%d new_bw=%llu,old_bw=%llu\n",
342 crtc->base.id, params_changed,
343 new->bw_ctl, old->bw_ctl);
344 old->bw_ctl = new->bw_ctl;
345 old->max_per_pipe_ib = new->max_per_pipe_ib;
346 update_bus = true;
347 }
348
349 if ((params_changed && new->core_clk_rate > old->core_clk_rate) ||
350 (!params_changed && new->core_clk_rate < old->core_clk_rate)) {
351 old->core_clk_rate = new->core_clk_rate;
352 update_clk = true;
353 }
354 } else {
355 DRM_DEBUG_ATOMIC("crtc=%d disable\n", crtc->base.id);
356 memset(old, 0, sizeof(*old));
357 update_bus = true;
358 update_clk = true;
359 }
360
361 trace_dpu_perf_crtc_update(crtc->base.id, new->bw_ctl,
362 new->core_clk_rate, !crtc->enabled, update_bus, update_clk);
363
364 if (update_bus) {
365 ret = _dpu_core_perf_crtc_update_bus(kms, crtc);
366 if (ret) {
367 DPU_ERROR("crtc-%d: failed to update bus bw vote\n",
368 crtc->base.id);
369 return ret;
370 }
371 }
372
373 /*
374 * Update the clock after bandwidth vote to ensure
375 * bandwidth is available before clock rate is increased.
376 */
377 if (update_clk) {
378 clk_rate = _dpu_core_perf_get_core_clk_rate(kms);
379
380 DRM_DEBUG_ATOMIC("clk:%llu\n", clk_rate);
381
382 trace_dpu_core_perf_update_clk(kms->dev, !crtc->enabled, clk_rate);
383
384 clk_rate = min(clk_rate, kms->perf.max_core_clk_rate);
385 ret = dev_pm_opp_set_rate(&kms->pdev->dev, clk_rate);
386 if (ret) {
387 DPU_ERROR("failed to set core clock rate %llu\n", clk_rate);
388 return ret;
389 }
390
391 kms->perf.core_clk_rate = clk_rate;
392 DRM_DEBUG_ATOMIC("update clk rate = %lld HZ\n", clk_rate);
393 }
394 return 0;
395 }
396
397 #ifdef CONFIG_DEBUG_FS
398
_dpu_core_perf_mode_write(struct file * file,const char __user * user_buf,size_t count,loff_t * ppos)399 static ssize_t _dpu_core_perf_mode_write(struct file *file,
400 const char __user *user_buf, size_t count, loff_t *ppos)
401 {
402 struct dpu_core_perf *perf = file->private_data;
403 u32 perf_mode = 0;
404 int ret;
405
406 ret = kstrtouint_from_user(user_buf, count, 0, &perf_mode);
407 if (ret)
408 return ret;
409
410 if (perf_mode >= DPU_PERF_MODE_MAX)
411 return -EINVAL;
412
413 if (perf_mode == DPU_PERF_MODE_FIXED) {
414 DRM_INFO("fix performance mode\n");
415 } else if (perf_mode == DPU_PERF_MODE_MINIMUM) {
416 /* run the driver with max clk and BW vote */
417 DRM_INFO("minimum performance mode\n");
418 } else if (perf_mode == DPU_PERF_MODE_NORMAL) {
419 /* reset the perf tune params to 0 */
420 DRM_INFO("normal performance mode\n");
421 }
422 perf->perf_tune.mode = perf_mode;
423
424 return count;
425 }
426
_dpu_core_perf_mode_read(struct file * file,char __user * buff,size_t count,loff_t * ppos)427 static ssize_t _dpu_core_perf_mode_read(struct file *file,
428 char __user *buff, size_t count, loff_t *ppos)
429 {
430 struct dpu_core_perf *perf = file->private_data;
431 int len;
432 char buf[128];
433
434 len = scnprintf(buf, sizeof(buf),
435 "mode %d\n",
436 perf->perf_tune.mode);
437
438 return simple_read_from_buffer(buff, count, ppos, buf, len);
439 }
440
441 static const struct file_operations dpu_core_perf_mode_fops = {
442 .open = simple_open,
443 .read = _dpu_core_perf_mode_read,
444 .write = _dpu_core_perf_mode_write,
445 };
446
447 /**
448 * dpu_core_perf_debugfs_init - initialize debugfs for core performance context
449 * @dpu_kms: Pointer to the dpu_kms struct
450 * @parent: Pointer to parent debugfs
451 */
dpu_core_perf_debugfs_init(struct dpu_kms * dpu_kms,struct dentry * parent)452 int dpu_core_perf_debugfs_init(struct dpu_kms *dpu_kms, struct dentry *parent)
453 {
454 struct dpu_core_perf *perf = &dpu_kms->perf;
455 struct dentry *entry;
456
457 entry = debugfs_create_dir("core_perf", parent);
458
459 debugfs_create_u64("max_core_clk_rate", 0600, entry,
460 &perf->max_core_clk_rate);
461 debugfs_create_u64("core_clk_rate", 0600, entry,
462 &perf->core_clk_rate);
463 debugfs_create_u32("enable_bw_release", 0600, entry,
464 (u32 *)&perf->enable_bw_release);
465 debugfs_create_u32("low_core_ab", 0400, entry,
466 (u32 *)&perf->perf_cfg->max_bw_low);
467 debugfs_create_u32("max_core_ab", 0400, entry,
468 (u32 *)&perf->perf_cfg->max_bw_high);
469 debugfs_create_u32("min_core_ib", 0400, entry,
470 (u32 *)&perf->perf_cfg->min_core_ib);
471 debugfs_create_u32("min_llcc_ib", 0400, entry,
472 (u32 *)&perf->perf_cfg->min_llcc_ib);
473 debugfs_create_u32("min_dram_ib", 0400, entry,
474 (u32 *)&perf->perf_cfg->min_dram_ib);
475 debugfs_create_file("perf_mode", 0600, entry,
476 (u32 *)perf, &dpu_core_perf_mode_fops);
477 debugfs_create_u64("fix_core_clk_rate", 0600, entry,
478 &perf->fix_core_clk_rate);
479 debugfs_create_u32("fix_core_ib_vote", 0600, entry,
480 &perf->fix_core_ib_vote);
481 debugfs_create_u32("fix_core_ab_vote", 0600, entry,
482 &perf->fix_core_ab_vote);
483
484 return 0;
485 }
486 #endif
487
488 /**
489 * dpu_core_perf_init - initialize the given core performance context
490 * @perf: Pointer to core performance context
491 * @perf_cfg: Pointer to platform performance configuration
492 * @max_core_clk_rate: Maximum core clock rate
493 */
dpu_core_perf_init(struct dpu_core_perf * perf,const struct dpu_perf_cfg * perf_cfg,unsigned long max_core_clk_rate)494 int dpu_core_perf_init(struct dpu_core_perf *perf,
495 const struct dpu_perf_cfg *perf_cfg,
496 unsigned long max_core_clk_rate)
497 {
498 perf->perf_cfg = perf_cfg;
499 perf->max_core_clk_rate = max_core_clk_rate;
500
501 return 0;
502 }
503