1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */
3 /* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
4
5 #include <linux/clk.h>
6 #include <linux/reset.h>
7 #include <linux/platform_device.h>
8 #include <linux/pm_domain.h>
9 #include <linux/pm_runtime.h>
10 #include <linux/regulator/consumer.h>
11
12 #include "panfrost_device.h"
13 #include "panfrost_devfreq.h"
14 #include "panfrost_features.h"
15 #include "panfrost_issues.h"
16 #include "panfrost_gpu.h"
17 #include "panfrost_job.h"
18 #include "panfrost_mmu.h"
19 #include "panfrost_perfcnt.h"
20
panfrost_reset_init(struct panfrost_device * pfdev)21 static int panfrost_reset_init(struct panfrost_device *pfdev)
22 {
23 pfdev->rstc = devm_reset_control_array_get_optional_exclusive(pfdev->dev);
24 if (IS_ERR(pfdev->rstc)) {
25 dev_err(pfdev->dev, "get reset failed %ld\n", PTR_ERR(pfdev->rstc));
26 return PTR_ERR(pfdev->rstc);
27 }
28
29 return reset_control_deassert(pfdev->rstc);
30 }
31
panfrost_reset_fini(struct panfrost_device * pfdev)32 static void panfrost_reset_fini(struct panfrost_device *pfdev)
33 {
34 reset_control_assert(pfdev->rstc);
35 }
36
panfrost_clk_init(struct panfrost_device * pfdev)37 static int panfrost_clk_init(struct panfrost_device *pfdev)
38 {
39 int err;
40 unsigned long rate;
41
42 pfdev->clock = devm_clk_get(pfdev->dev, NULL);
43 if (IS_ERR(pfdev->clock)) {
44 dev_err(pfdev->dev, "get clock failed %ld\n", PTR_ERR(pfdev->clock));
45 return PTR_ERR(pfdev->clock);
46 }
47
48 rate = clk_get_rate(pfdev->clock);
49 dev_info(pfdev->dev, "clock rate = %lu\n", rate);
50
51 err = clk_prepare_enable(pfdev->clock);
52 if (err)
53 return err;
54
55 pfdev->bus_clock = devm_clk_get_optional(pfdev->dev, "bus");
56 if (IS_ERR(pfdev->bus_clock)) {
57 dev_err(pfdev->dev, "get bus_clock failed %ld\n",
58 PTR_ERR(pfdev->bus_clock));
59 err = PTR_ERR(pfdev->bus_clock);
60 goto disable_clock;
61 }
62
63 if (pfdev->bus_clock) {
64 rate = clk_get_rate(pfdev->bus_clock);
65 dev_info(pfdev->dev, "bus_clock rate = %lu\n", rate);
66
67 err = clk_prepare_enable(pfdev->bus_clock);
68 if (err)
69 goto disable_clock;
70 }
71
72 return 0;
73
74 disable_clock:
75 clk_disable_unprepare(pfdev->clock);
76
77 return err;
78 }
79
panfrost_clk_fini(struct panfrost_device * pfdev)80 static void panfrost_clk_fini(struct panfrost_device *pfdev)
81 {
82 clk_disable_unprepare(pfdev->bus_clock);
83 clk_disable_unprepare(pfdev->clock);
84 }
85
panfrost_regulator_init(struct panfrost_device * pfdev)86 static int panfrost_regulator_init(struct panfrost_device *pfdev)
87 {
88 int ret, i;
89
90 pfdev->regulators = devm_kcalloc(pfdev->dev, pfdev->comp->num_supplies,
91 sizeof(*pfdev->regulators),
92 GFP_KERNEL);
93 if (!pfdev->regulators)
94 return -ENOMEM;
95
96 for (i = 0; i < pfdev->comp->num_supplies; i++)
97 pfdev->regulators[i].supply = pfdev->comp->supply_names[i];
98
99 ret = devm_regulator_bulk_get(pfdev->dev,
100 pfdev->comp->num_supplies,
101 pfdev->regulators);
102 if (ret < 0) {
103 if (ret != -EPROBE_DEFER)
104 dev_err(pfdev->dev, "failed to get regulators: %d\n",
105 ret);
106 return ret;
107 }
108
109 ret = regulator_bulk_enable(pfdev->comp->num_supplies,
110 pfdev->regulators);
111 if (ret < 0) {
112 dev_err(pfdev->dev, "failed to enable regulators: %d\n", ret);
113 return ret;
114 }
115
116 return 0;
117 }
118
panfrost_regulator_fini(struct panfrost_device * pfdev)119 static void panfrost_regulator_fini(struct panfrost_device *pfdev)
120 {
121 if (!pfdev->regulators)
122 return;
123
124 regulator_bulk_disable(pfdev->comp->num_supplies, pfdev->regulators);
125 }
126
panfrost_pm_domain_fini(struct panfrost_device * pfdev)127 static void panfrost_pm_domain_fini(struct panfrost_device *pfdev)
128 {
129 int i;
130
131 for (i = 0; i < ARRAY_SIZE(pfdev->pm_domain_devs); i++) {
132 if (!pfdev->pm_domain_devs[i])
133 break;
134
135 if (pfdev->pm_domain_links[i])
136 device_link_del(pfdev->pm_domain_links[i]);
137
138 dev_pm_domain_detach(pfdev->pm_domain_devs[i], true);
139 }
140 }
141
panfrost_pm_domain_init(struct panfrost_device * pfdev)142 static int panfrost_pm_domain_init(struct panfrost_device *pfdev)
143 {
144 int err;
145 int i, num_domains;
146
147 num_domains = of_count_phandle_with_args(pfdev->dev->of_node,
148 "power-domains",
149 "#power-domain-cells");
150
151 /*
152 * Single domain is handled by the core, and, if only a single power
153 * the power domain is requested, the property is optional.
154 */
155 if (num_domains < 2 && pfdev->comp->num_pm_domains < 2)
156 return 0;
157
158 if (num_domains != pfdev->comp->num_pm_domains) {
159 dev_err(pfdev->dev,
160 "Incorrect number of power domains: %d provided, %d needed\n",
161 num_domains, pfdev->comp->num_pm_domains);
162 return -EINVAL;
163 }
164
165 if (WARN(num_domains > ARRAY_SIZE(pfdev->pm_domain_devs),
166 "Too many supplies in compatible structure.\n"))
167 return -EINVAL;
168
169 for (i = 0; i < num_domains; i++) {
170 pfdev->pm_domain_devs[i] =
171 dev_pm_domain_attach_by_name(pfdev->dev,
172 pfdev->comp->pm_domain_names[i]);
173 if (IS_ERR_OR_NULL(pfdev->pm_domain_devs[i])) {
174 err = PTR_ERR(pfdev->pm_domain_devs[i]) ? : -ENODATA;
175 pfdev->pm_domain_devs[i] = NULL;
176 dev_err(pfdev->dev,
177 "failed to get pm-domain %s(%d): %d\n",
178 pfdev->comp->pm_domain_names[i], i, err);
179 goto err;
180 }
181
182 pfdev->pm_domain_links[i] = device_link_add(pfdev->dev,
183 pfdev->pm_domain_devs[i], DL_FLAG_PM_RUNTIME |
184 DL_FLAG_STATELESS | DL_FLAG_RPM_ACTIVE);
185 if (!pfdev->pm_domain_links[i]) {
186 dev_err(pfdev->pm_domain_devs[i],
187 "adding device link failed!\n");
188 err = -ENODEV;
189 goto err;
190 }
191 }
192
193 return 0;
194
195 err:
196 panfrost_pm_domain_fini(pfdev);
197 return err;
198 }
199
panfrost_device_init(struct panfrost_device * pfdev)200 int panfrost_device_init(struct panfrost_device *pfdev)
201 {
202 int err;
203
204 mutex_init(&pfdev->sched_lock);
205 INIT_LIST_HEAD(&pfdev->scheduled_jobs);
206 INIT_LIST_HEAD(&pfdev->as_lru_list);
207
208 spin_lock_init(&pfdev->as_lock);
209
210 spin_lock_init(&pfdev->cycle_counter.lock);
211
212 #ifdef CONFIG_DEBUG_FS
213 mutex_init(&pfdev->debugfs.gems_lock);
214 INIT_LIST_HEAD(&pfdev->debugfs.gems_list);
215 #endif
216
217 err = panfrost_pm_domain_init(pfdev);
218 if (err)
219 return err;
220
221 err = panfrost_reset_init(pfdev);
222 if (err) {
223 dev_err(pfdev->dev, "reset init failed %d\n", err);
224 goto out_pm_domain;
225 }
226
227 err = panfrost_clk_init(pfdev);
228 if (err) {
229 dev_err(pfdev->dev, "clk init failed %d\n", err);
230 goto out_reset;
231 }
232
233 err = panfrost_devfreq_init(pfdev);
234 if (err) {
235 if (err != -EPROBE_DEFER)
236 dev_err(pfdev->dev, "devfreq init failed %d\n", err);
237 goto out_clk;
238 }
239
240 /* OPP will handle regulators */
241 if (!pfdev->pfdevfreq.opp_of_table_added) {
242 err = panfrost_regulator_init(pfdev);
243 if (err)
244 goto out_devfreq;
245 }
246
247 pfdev->iomem = devm_platform_ioremap_resource(pfdev->pdev, 0);
248 if (IS_ERR(pfdev->iomem)) {
249 err = PTR_ERR(pfdev->iomem);
250 goto out_regulator;
251 }
252
253 err = panfrost_gpu_init(pfdev);
254 if (err)
255 goto out_regulator;
256
257 err = panfrost_mmu_init(pfdev);
258 if (err)
259 goto out_gpu;
260
261 err = panfrost_job_init(pfdev);
262 if (err)
263 goto out_mmu;
264
265 err = panfrost_perfcnt_init(pfdev);
266 if (err)
267 goto out_job;
268
269 return 0;
270 out_job:
271 panfrost_job_fini(pfdev);
272 out_mmu:
273 panfrost_mmu_fini(pfdev);
274 out_gpu:
275 panfrost_gpu_fini(pfdev);
276 out_regulator:
277 panfrost_regulator_fini(pfdev);
278 out_devfreq:
279 panfrost_devfreq_fini(pfdev);
280 out_clk:
281 panfrost_clk_fini(pfdev);
282 out_reset:
283 panfrost_reset_fini(pfdev);
284 out_pm_domain:
285 panfrost_pm_domain_fini(pfdev);
286 return err;
287 }
288
panfrost_device_fini(struct panfrost_device * pfdev)289 void panfrost_device_fini(struct panfrost_device *pfdev)
290 {
291 panfrost_perfcnt_fini(pfdev);
292 panfrost_job_fini(pfdev);
293 panfrost_mmu_fini(pfdev);
294 panfrost_gpu_fini(pfdev);
295 panfrost_devfreq_fini(pfdev);
296 panfrost_regulator_fini(pfdev);
297 panfrost_clk_fini(pfdev);
298 panfrost_reset_fini(pfdev);
299 panfrost_pm_domain_fini(pfdev);
300 }
301
302 #define PANFROST_EXCEPTION(id) \
303 [DRM_PANFROST_EXCEPTION_ ## id] = { \
304 .name = #id, \
305 }
306
307 struct panfrost_exception_info {
308 const char *name;
309 };
310
311 static const struct panfrost_exception_info panfrost_exception_infos[] = {
312 PANFROST_EXCEPTION(OK),
313 PANFROST_EXCEPTION(DONE),
314 PANFROST_EXCEPTION(INTERRUPTED),
315 PANFROST_EXCEPTION(STOPPED),
316 PANFROST_EXCEPTION(TERMINATED),
317 PANFROST_EXCEPTION(KABOOM),
318 PANFROST_EXCEPTION(EUREKA),
319 PANFROST_EXCEPTION(ACTIVE),
320 PANFROST_EXCEPTION(JOB_CONFIG_FAULT),
321 PANFROST_EXCEPTION(JOB_POWER_FAULT),
322 PANFROST_EXCEPTION(JOB_READ_FAULT),
323 PANFROST_EXCEPTION(JOB_WRITE_FAULT),
324 PANFROST_EXCEPTION(JOB_AFFINITY_FAULT),
325 PANFROST_EXCEPTION(JOB_BUS_FAULT),
326 PANFROST_EXCEPTION(INSTR_INVALID_PC),
327 PANFROST_EXCEPTION(INSTR_INVALID_ENC),
328 PANFROST_EXCEPTION(INSTR_TYPE_MISMATCH),
329 PANFROST_EXCEPTION(INSTR_OPERAND_FAULT),
330 PANFROST_EXCEPTION(INSTR_TLS_FAULT),
331 PANFROST_EXCEPTION(INSTR_BARRIER_FAULT),
332 PANFROST_EXCEPTION(INSTR_ALIGN_FAULT),
333 PANFROST_EXCEPTION(DATA_INVALID_FAULT),
334 PANFROST_EXCEPTION(TILE_RANGE_FAULT),
335 PANFROST_EXCEPTION(ADDR_RANGE_FAULT),
336 PANFROST_EXCEPTION(IMPRECISE_FAULT),
337 PANFROST_EXCEPTION(OOM),
338 PANFROST_EXCEPTION(OOM_AFBC),
339 PANFROST_EXCEPTION(UNKNOWN),
340 PANFROST_EXCEPTION(DELAYED_BUS_FAULT),
341 PANFROST_EXCEPTION(GPU_SHAREABILITY_FAULT),
342 PANFROST_EXCEPTION(SYS_SHAREABILITY_FAULT),
343 PANFROST_EXCEPTION(GPU_CACHEABILITY_FAULT),
344 PANFROST_EXCEPTION(TRANSLATION_FAULT_0),
345 PANFROST_EXCEPTION(TRANSLATION_FAULT_1),
346 PANFROST_EXCEPTION(TRANSLATION_FAULT_2),
347 PANFROST_EXCEPTION(TRANSLATION_FAULT_3),
348 PANFROST_EXCEPTION(TRANSLATION_FAULT_4),
349 PANFROST_EXCEPTION(TRANSLATION_FAULT_IDENTITY),
350 PANFROST_EXCEPTION(PERM_FAULT_0),
351 PANFROST_EXCEPTION(PERM_FAULT_1),
352 PANFROST_EXCEPTION(PERM_FAULT_2),
353 PANFROST_EXCEPTION(PERM_FAULT_3),
354 PANFROST_EXCEPTION(TRANSTAB_BUS_FAULT_0),
355 PANFROST_EXCEPTION(TRANSTAB_BUS_FAULT_1),
356 PANFROST_EXCEPTION(TRANSTAB_BUS_FAULT_2),
357 PANFROST_EXCEPTION(TRANSTAB_BUS_FAULT_3),
358 PANFROST_EXCEPTION(ACCESS_FLAG_0),
359 PANFROST_EXCEPTION(ACCESS_FLAG_1),
360 PANFROST_EXCEPTION(ACCESS_FLAG_2),
361 PANFROST_EXCEPTION(ACCESS_FLAG_3),
362 PANFROST_EXCEPTION(ADDR_SIZE_FAULT_IN0),
363 PANFROST_EXCEPTION(ADDR_SIZE_FAULT_IN1),
364 PANFROST_EXCEPTION(ADDR_SIZE_FAULT_IN2),
365 PANFROST_EXCEPTION(ADDR_SIZE_FAULT_IN3),
366 PANFROST_EXCEPTION(ADDR_SIZE_FAULT_OUT0),
367 PANFROST_EXCEPTION(ADDR_SIZE_FAULT_OUT1),
368 PANFROST_EXCEPTION(ADDR_SIZE_FAULT_OUT2),
369 PANFROST_EXCEPTION(ADDR_SIZE_FAULT_OUT3),
370 PANFROST_EXCEPTION(MEM_ATTR_FAULT_0),
371 PANFROST_EXCEPTION(MEM_ATTR_FAULT_1),
372 PANFROST_EXCEPTION(MEM_ATTR_FAULT_2),
373 PANFROST_EXCEPTION(MEM_ATTR_FAULT_3),
374 PANFROST_EXCEPTION(MEM_ATTR_NONCACHE_0),
375 PANFROST_EXCEPTION(MEM_ATTR_NONCACHE_1),
376 PANFROST_EXCEPTION(MEM_ATTR_NONCACHE_2),
377 PANFROST_EXCEPTION(MEM_ATTR_NONCACHE_3),
378 };
379
panfrost_exception_name(u32 exception_code)380 const char *panfrost_exception_name(u32 exception_code)
381 {
382 if (WARN_ON(exception_code >= ARRAY_SIZE(panfrost_exception_infos) ||
383 !panfrost_exception_infos[exception_code].name))
384 return "Unknown exception type";
385
386 return panfrost_exception_infos[exception_code].name;
387 }
388
panfrost_exception_needs_reset(const struct panfrost_device * pfdev,u32 exception_code)389 bool panfrost_exception_needs_reset(const struct panfrost_device *pfdev,
390 u32 exception_code)
391 {
392 /* If an occlusion query write causes a bus fault on affected GPUs,
393 * future fragment jobs may hang. Reset to workaround.
394 */
395 if (exception_code == DRM_PANFROST_EXCEPTION_JOB_BUS_FAULT)
396 return panfrost_has_hw_issue(pfdev, HW_ISSUE_TTRX_3076);
397
398 /* No other GPUs we support need a reset */
399 return false;
400 }
401
panfrost_device_reset(struct panfrost_device * pfdev)402 void panfrost_device_reset(struct panfrost_device *pfdev)
403 {
404 panfrost_gpu_soft_reset(pfdev);
405
406 panfrost_gpu_power_on(pfdev);
407 panfrost_mmu_reset(pfdev);
408 panfrost_job_enable_interrupts(pfdev);
409 }
410
panfrost_device_runtime_resume(struct device * dev)411 static int panfrost_device_runtime_resume(struct device *dev)
412 {
413 struct panfrost_device *pfdev = dev_get_drvdata(dev);
414 int ret;
415
416 if (pfdev->comp->pm_features & BIT(GPU_PM_RT)) {
417 ret = reset_control_deassert(pfdev->rstc);
418 if (ret)
419 return ret;
420
421 ret = clk_enable(pfdev->clock);
422 if (ret)
423 goto err_clk;
424
425 if (pfdev->bus_clock) {
426 ret = clk_enable(pfdev->bus_clock);
427 if (ret)
428 goto err_bus_clk;
429 }
430 }
431
432 panfrost_device_reset(pfdev);
433 panfrost_devfreq_resume(pfdev);
434
435 return 0;
436
437 err_bus_clk:
438 if (pfdev->comp->pm_features & BIT(GPU_PM_RT))
439 clk_disable(pfdev->clock);
440 err_clk:
441 if (pfdev->comp->pm_features & BIT(GPU_PM_RT))
442 reset_control_assert(pfdev->rstc);
443 return ret;
444 }
445
panfrost_device_runtime_suspend(struct device * dev)446 static int panfrost_device_runtime_suspend(struct device *dev)
447 {
448 struct panfrost_device *pfdev = dev_get_drvdata(dev);
449
450 if (!panfrost_job_is_idle(pfdev))
451 return -EBUSY;
452
453 panfrost_devfreq_suspend(pfdev);
454 panfrost_job_suspend_irq(pfdev);
455 panfrost_mmu_suspend_irq(pfdev);
456 panfrost_gpu_suspend_irq(pfdev);
457 panfrost_gpu_power_off(pfdev);
458
459 if (pfdev->comp->pm_features & BIT(GPU_PM_RT)) {
460 if (pfdev->bus_clock)
461 clk_disable(pfdev->bus_clock);
462
463 clk_disable(pfdev->clock);
464 reset_control_assert(pfdev->rstc);
465 }
466
467 return 0;
468 }
469
panfrost_device_resume(struct device * dev)470 static int panfrost_device_resume(struct device *dev)
471 {
472 struct panfrost_device *pfdev = dev_get_drvdata(dev);
473 int ret;
474
475 if (pfdev->comp->pm_features & BIT(GPU_PM_VREG_OFF)) {
476 unsigned long freq = pfdev->pfdevfreq.fast_rate;
477 struct dev_pm_opp *opp;
478
479 opp = dev_pm_opp_find_freq_ceil(dev, &freq);
480 if (IS_ERR(opp))
481 return PTR_ERR(opp);
482 dev_pm_opp_set_opp(dev, opp);
483 dev_pm_opp_put(opp);
484 }
485
486 if (pfdev->comp->pm_features & BIT(GPU_PM_CLK_DIS)) {
487 ret = clk_enable(pfdev->clock);
488 if (ret)
489 goto err_clk;
490
491 if (pfdev->bus_clock) {
492 ret = clk_enable(pfdev->bus_clock);
493 if (ret)
494 goto err_bus_clk;
495 }
496 }
497
498 ret = pm_runtime_force_resume(dev);
499 if (ret)
500 goto err_resume;
501
502 return 0;
503
504 err_resume:
505 if (pfdev->comp->pm_features & BIT(GPU_PM_CLK_DIS) && pfdev->bus_clock)
506 clk_disable(pfdev->bus_clock);
507 err_bus_clk:
508 if (pfdev->comp->pm_features & BIT(GPU_PM_CLK_DIS))
509 clk_disable(pfdev->clock);
510 err_clk:
511 if (pfdev->comp->pm_features & BIT(GPU_PM_VREG_OFF))
512 dev_pm_opp_set_opp(dev, NULL);
513 return ret;
514 }
515
panfrost_device_suspend(struct device * dev)516 static int panfrost_device_suspend(struct device *dev)
517 {
518 struct panfrost_device *pfdev = dev_get_drvdata(dev);
519 int ret;
520
521 ret = pm_runtime_force_suspend(dev);
522 if (ret)
523 return ret;
524
525 if (pfdev->comp->pm_features & BIT(GPU_PM_CLK_DIS)) {
526 if (pfdev->bus_clock)
527 clk_disable(pfdev->bus_clock);
528
529 clk_disable(pfdev->clock);
530 }
531
532 if (pfdev->comp->pm_features & BIT(GPU_PM_VREG_OFF))
533 dev_pm_opp_set_opp(dev, NULL);
534
535 return 0;
536 }
537
538 EXPORT_GPL_DEV_PM_OPS(panfrost_pm_ops) = {
539 RUNTIME_PM_OPS(panfrost_device_runtime_suspend, panfrost_device_runtime_resume, NULL)
540 SYSTEM_SLEEP_PM_OPS(panfrost_device_suspend, panfrost_device_resume)
541 };
542