1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Driver for MediaTek MFlexGraphics Devices
4 *
5 * Copyright (C) 2025, Collabora Ltd.
6 */
7
8 #include <linux/completion.h>
9 #include <linux/clk.h>
10 #include <linux/clk-provider.h>
11 #include <linux/container_of.h>
12 #include <linux/iopoll.h>
13 #include <linux/mailbox_client.h>
14 #include <linux/module.h>
15 #include <linux/of.h>
16 #include <linux/of_address.h>
17 #include <linux/of_platform.h>
18 #include <linux/of_reserved_mem.h>
19 #include <linux/overflow.h>
20 #include <linux/platform_device.h>
21 #include <linux/pm_domain.h>
22 #include <linux/pm_opp.h>
23 #include <linux/regulator/consumer.h>
24 #include <linux/units.h>
25
26 #define GPR_LP_STATE 0x0028
27 #define EB_ON_SUSPEND 0x0
28 #define EB_ON_RESUME 0x1
29 #define GPR_IPI_MAGIC 0x34
30
31 #define RPC_PWR_CON 0x0504
32 #define PWR_ACK_M GENMASK(31, 30)
33 #define RPC_DUMMY_REG_2 0x0658
34 #define RPC_GHPM_CFG0_CON 0x0800
35 #define GHPM_ENABLE_M BIT(0)
36 #define GHPM_ON_SEQ_M BIT(2)
37 #define RPC_GHPM_RO0_CON 0x09A4
38 #define GHPM_STATE_M GENMASK(7, 0)
39 #define GHPM_PWR_STATE_M BIT(16)
40
41 #define GF_REG_MAGIC 0x0000
42 #define GF_REG_GPU_OPP_IDX 0x0004
43 #define GF_REG_STK_OPP_IDX 0x0008
44 #define GF_REG_GPU_OPP_NUM 0x000c
45 #define GF_REG_STK_OPP_NUM 0x0010
46 #define GF_REG_GPU_OPP_SNUM 0x0014
47 #define GF_REG_STK_OPP_SNUM 0x0018
48 #define GF_REG_POWER_COUNT 0x001c
49 #define GF_REG_BUCK_COUNT 0x0020
50 #define GF_REG_MTCMOS_COUNT 0x0024
51 #define GF_REG_CG_COUNT 0x0028 /* CG = Clock Gate? */
52 #define GF_REG_ACTIVE_COUNT 0x002C
53 #define GF_REG_TEMP_RAW 0x0030
54 #define GF_REG_TEMP_NORM_GPU 0x0034
55 #define GF_REG_TEMP_HIGH_GPU 0x0038
56 #define GF_REG_TEMP_NORM_STK 0x003C
57 #define GF_REG_TEMP_HIGH_STK 0x0040
58 #define GF_REG_FREQ_CUR_GPU 0x0044
59 #define GF_REG_FREQ_CUR_STK 0x0048
60 #define GF_REG_FREQ_OUT_GPU 0x004C /* Guess: actual achieved freq */
61 #define GF_REG_FREQ_OUT_STK 0x0050 /* Guess: actual achieved freq */
62 #define GF_REG_FREQ_METER_GPU 0x0054 /* Seems unused, always 0 */
63 #define GF_REG_FREQ_METER_STK 0x0058 /* Seems unused, always 0 */
64 #define GF_REG_VOLT_CUR_GPU 0x005C /* in tens of microvolts */
65 #define GF_REG_VOLT_CUR_STK 0x0060 /* in tens of microvolts */
66 #define GF_REG_VOLT_CUR_GPU_SRAM 0x0064
67 #define GF_REG_VOLT_CUR_STK_SRAM 0x0068
68 #define GF_REG_VOLT_CUR_GPU_REG 0x006C /* Seems unused, always 0 */
69 #define GF_REG_VOLT_CUR_STK_REG 0x0070 /* Seems unused, always 0 */
70 #define GF_REG_VOLT_CUR_GPU_REG_SRAM 0x0074
71 #define GF_REG_VOLT_CUR_STK_REG_SRAM 0x0078
72 #define GF_REG_PWR_CUR_GPU 0x007C /* in milliwatts */
73 #define GF_REG_PWR_CUR_STK 0x0080 /* in milliwatts */
74 #define GF_REG_PWR_MAX_GPU 0x0084 /* in milliwatts */
75 #define GF_REG_PWR_MAX_STK 0x0088 /* in milliwatts */
76 #define GF_REG_PWR_MIN_GPU 0x008C /* in milliwatts */
77 #define GF_REG_PWR_MIN_STK 0x0090 /* in milliwatts */
78 #define GF_REG_LEAKAGE_RT_GPU 0x0094 /* Unknown */
79 #define GF_REG_LEAKAGE_RT_STK 0x0098 /* Unknown */
80 #define GF_REG_LEAKAGE_RT_SRAM 0x009C /* Unknown */
81 #define GF_REG_LEAKAGE_HT_GPU 0x00A0 /* Unknown */
82 #define GF_REG_LEAKAGE_HT_STK 0x00A4 /* Unknown */
83 #define GF_REG_LEAKAGE_HT_SRAM 0x00A8 /* Unknown */
84 #define GF_REG_VOLT_DAC_LOW_GPU 0x00AC /* Seems unused, always 0 */
85 #define GF_REG_VOLT_DAC_LOW_STK 0x00B0 /* Seems unused, always 0 */
86 #define GF_REG_OPP_CUR_CEIL 0x00B4
87 #define GF_REG_OPP_CUR_FLOOR 0x00B8
88 #define GF_REG_OPP_CUR_LIMITER_CEIL 0x00BC
89 #define GF_REG_OPP_CUR_LIMITER_FLOOR 0x00C0
90 #define GF_REG_OPP_PRIORITY_CEIL 0x00C4
91 #define GF_REG_OPP_PRIORITY_FLOOR 0x00C8
92 #define GF_REG_PWR_CTL 0x00CC
93 #define GF_REG_ACTIVE_SLEEP_CTL 0x00D0
94 #define GF_REG_DVFS_STATE 0x00D4
95 #define GF_REG_SHADER_PRESENT 0x00D8
96 #define GF_REG_ASENSOR_ENABLE 0x00DC
97 #define GF_REG_AGING_LOAD 0x00E0
98 #define GF_REG_AGING_MARGIN 0x00E4
99 #define GF_REG_AVS_ENABLE 0x00E8
100 #define GF_REG_AVS_MARGIN 0x00EC
101 #define GF_REG_CHIP_TYPE 0x00F0
102 #define GF_REG_SB_VERSION 0x00F4
103 #define GF_REG_PTP_VERSION 0x00F8
104 #define GF_REG_DBG_VERSION 0x00FC
105 #define GF_REG_KDBG_VERSION 0x0100
106 #define GF_REG_GPM1_MODE 0x0104
107 #define GF_REG_GPM3_MODE 0x0108
108 #define GF_REG_DFD_MODE 0x010C
109 #define GF_REG_DUAL_BUCK 0x0110
110 #define GF_REG_SEGMENT_ID 0x0114
111 #define GF_REG_POWER_TIME_H 0x0118
112 #define GF_REG_POWER_TIME_L 0x011C
113 #define GF_REG_PWR_STATUS 0x0120
114 #define GF_REG_STRESS_TEST 0x0124
115 #define GF_REG_TEST_MODE 0x0128
116 #define GF_REG_IPS_MODE 0x012C
117 #define GF_REG_TEMP_COMP_MODE 0x0130
118 #define GF_REG_HT_TEMP_COMP_MODE 0x0134
119 #define GF_REG_PWR_TRACKER_MODE 0x0138
120 #define GF_REG_OPP_TABLE_GPU 0x0314
121 #define GF_REG_OPP_TABLE_STK 0x09A4
122 #define GF_REG_OPP_TABLE_GPU_S 0x1034
123 #define GF_REG_OPP_TABLE_STK_S 0x16c4
124 #define GF_REG_LIMIT_TABLE 0x1d54
125 #define GF_REG_GPM3_TABLE 0x223C
126
127 #define MFG_MT8196_E2_ID 0x101
128 #define GPUEB_SLEEP_MAGIC 0x55667788UL
129 #define GPUEB_MEM_MAGIC 0xBABADADAUL
130
131 #define GPUEB_TIMEOUT_US 10000UL
132 #define GPUEB_POLL_US 50
133
134 #define MAX_OPP_NUM 70
135
136 #define GPUEB_MBOX_MAX_RX_SIZE 32 /* in bytes */
137
138 /*
139 * This enum is part of the ABI of the GPUEB firmware. Don't change the
140 * numbering, as you would wreak havoc.
141 */
142 enum mtk_mfg_ipi_cmd {
143 CMD_INIT_SHARED_MEM = 0,
144 CMD_GET_FREQ_BY_IDX = 1,
145 CMD_GET_POWER_BY_IDX = 2,
146 CMD_GET_OPPIDX_BY_FREQ = 3,
147 CMD_GET_LEAKAGE_POWER = 4,
148 CMD_SET_LIMIT = 5,
149 CMD_POWER_CONTROL = 6,
150 CMD_ACTIVE_SLEEP_CONTROL = 7,
151 CMD_COMMIT = 8,
152 CMD_DUAL_COMMIT = 9,
153 CMD_PDCA_CONFIG = 10,
154 CMD_UPDATE_DEBUG_OPP_INFO = 11,
155 CMD_SWITCH_LIMIT = 12,
156 CMD_FIX_TARGET_OPPIDX = 13,
157 CMD_FIX_DUAL_TARGET_OPPIDX = 14,
158 CMD_FIX_CUSTOM_FREQ_VOLT = 15,
159 CMD_FIX_DUAL_CUSTOM_FREQ_VOLT = 16,
160 CMD_SET_MFGSYS_CONFIG = 17,
161 CMD_MSSV_COMMIT = 18,
162 CMD_NUM = 19,
163 };
164
165 /*
166 * This struct is part of the ABI of the GPUEB firmware. Changing it, or
167 * reordering fields in it, will break things, so don't do it. Thank you.
168 */
169 struct __packed mtk_mfg_ipi_msg {
170 __le32 magic;
171 __le32 cmd;
172 __le32 target;
173 /*
174 * Downstream relies on the compiler to implicitly add the following
175 * padding, as it declares the struct as non-packed.
176 */
177 __le32 reserved;
178 union {
179 s32 __bitwise oppidx;
180 s32 __bitwise return_value;
181 __le32 freq;
182 __le32 volt;
183 __le32 power;
184 __le32 power_state;
185 __le32 mode;
186 __le32 value;
187 struct {
188 __le64 base;
189 __le32 size;
190 } shared_mem;
191 struct {
192 __le32 freq;
193 __le32 volt;
194 } custom;
195 struct {
196 __le32 limiter;
197 s32 __bitwise ceiling_info;
198 s32 __bitwise floor_info;
199 } set_limit;
200 struct {
201 __le32 target;
202 __le32 val;
203 } mfg_cfg;
204 struct {
205 __le32 target;
206 __le32 val;
207 } mssv;
208 struct {
209 s32 __bitwise gpu_oppidx;
210 s32 __bitwise stack_oppidx;
211 } dual_commit;
212 struct {
213 __le32 fgpu;
214 __le32 vgpu;
215 __le32 fstack;
216 __le32 vstack;
217 } dual_custom;
218 } u;
219 };
220
221 struct __packed mtk_mfg_ipi_sleep_msg {
222 __le32 event;
223 __le32 state;
224 __le32 magic;
225 };
226
227 /**
228 * struct mtk_mfg_opp_entry - OPP table entry from firmware
229 * @freq_khz: The operating point's frequency in kilohertz
230 * @voltage_core: The operating point's core voltage in tens of microvolts
231 * @voltage_sram: The operating point's SRAM voltage in tens of microvolts
232 * @posdiv: exponent of base 2 for PLL frequency divisor used for this OPP
233 * @voltage_margin: Number of tens of microvolts the voltage can be undershot
234 * @power_mw: estimate of power usage at this operating point, in milliwatts
235 *
236 * This struct is part of the ABI with the EB firmware. Do not change it.
237 */
238 struct __packed mtk_mfg_opp_entry {
239 __le32 freq_khz;
240 __le32 voltage_core;
241 __le32 voltage_sram;
242 __le32 posdiv;
243 __le32 voltage_margin;
244 __le32 power_mw;
245 };
246
247 struct mtk_mfg_mbox {
248 struct mbox_client cl;
249 struct completion rx_done;
250 struct mtk_mfg *mfg;
251 struct mbox_chan *ch;
252 void *rx_data;
253 };
254
255 struct mtk_mfg {
256 struct generic_pm_domain pd;
257 struct platform_device *pdev;
258 struct clk *clk_eb;
259 struct clk_bulk_data *gpu_clks;
260 struct clk_hw clk_core_hw;
261 struct clk_hw clk_stack_hw;
262 struct regulator_bulk_data *gpu_regs;
263 void __iomem *rpc;
264 void __iomem *gpr;
265 void __iomem *shared_mem;
266 phys_addr_t shared_mem_phys;
267 unsigned int shared_mem_size;
268 u16 ghpm_en_reg;
269 u32 ipi_magic;
270 unsigned short num_gpu_opps;
271 unsigned short num_stack_opps;
272 struct dev_pm_opp_data *gpu_opps;
273 struct dev_pm_opp_data *stack_opps;
274 struct mtk_mfg_mbox *gf_mbox;
275 struct mtk_mfg_mbox *slp_mbox;
276 const struct mtk_mfg_variant *variant;
277 };
278
279 struct mtk_mfg_variant {
280 const char *const *clk_names;
281 unsigned int num_clks;
282 const char *const *regulator_names;
283 unsigned int num_regulators;
284 /** @turbo_below: opp indices below this value are considered turbo */
285 unsigned int turbo_below;
286 int (*init)(struct mtk_mfg *mfg);
287 };
288
mtk_mfg_from_genpd(struct generic_pm_domain * pd)289 static inline struct mtk_mfg *mtk_mfg_from_genpd(struct generic_pm_domain *pd)
290 {
291 return container_of(pd, struct mtk_mfg, pd);
292 }
293
mtk_mfg_update_reg_bits(void __iomem * addr,u32 mask,u32 val)294 static inline void mtk_mfg_update_reg_bits(void __iomem *addr, u32 mask, u32 val)
295 {
296 writel((readl(addr) & ~mask) | (val & mask), addr);
297 }
298
mtk_mfg_is_powered_on(struct mtk_mfg * mfg)299 static inline bool mtk_mfg_is_powered_on(struct mtk_mfg *mfg)
300 {
301 return (readl(mfg->rpc + RPC_PWR_CON) & PWR_ACK_M) == PWR_ACK_M;
302 }
303
mtk_mfg_recalc_rate_gpu(struct clk_hw * hw,unsigned long parent_rate)304 static unsigned long mtk_mfg_recalc_rate_gpu(struct clk_hw *hw,
305 unsigned long parent_rate)
306 {
307 struct mtk_mfg *mfg = container_of(hw, struct mtk_mfg, clk_core_hw);
308
309 return readl(mfg->shared_mem + GF_REG_FREQ_OUT_GPU) * HZ_PER_KHZ;
310 }
311
mtk_mfg_determine_rate(struct clk_hw * hw,struct clk_rate_request * req)312 static int mtk_mfg_determine_rate(struct clk_hw *hw,
313 struct clk_rate_request *req)
314 {
315 /*
316 * The determine_rate callback needs to be implemented to avoid returning
317 * the current clock frequency, rather than something even remotely
318 * close to the frequency that was asked for.
319 *
320 * Instead of writing considerable amounts of possibly slow code just to
321 * somehow figure out which of the three PLLs to round for, or even to
322 * do a search through one of two OPP tables in order to find the closest
323 * OPP of a frequency, just return the rate as-is. This avoids devfreq
324 * "rounding" a request for the lowest frequency to the possibly very
325 * high current frequency, breaking the powersave governor in the process.
326 */
327
328 return 0;
329 }
330
mtk_mfg_recalc_rate_stack(struct clk_hw * hw,unsigned long parent_rate)331 static unsigned long mtk_mfg_recalc_rate_stack(struct clk_hw *hw,
332 unsigned long parent_rate)
333 {
334 struct mtk_mfg *mfg = container_of(hw, struct mtk_mfg, clk_stack_hw);
335
336 return readl(mfg->shared_mem + GF_REG_FREQ_OUT_STK) * HZ_PER_KHZ;
337 }
338
339 static const struct clk_ops mtk_mfg_clk_gpu_ops = {
340 .recalc_rate = mtk_mfg_recalc_rate_gpu,
341 .determine_rate = mtk_mfg_determine_rate,
342 };
343
344 static const struct clk_ops mtk_mfg_clk_stack_ops = {
345 .recalc_rate = mtk_mfg_recalc_rate_stack,
346 .determine_rate = mtk_mfg_determine_rate,
347 };
348
349 static const struct clk_init_data mtk_mfg_clk_gpu_init = {
350 .name = "gpu-core",
351 .ops = &mtk_mfg_clk_gpu_ops,
352 .flags = CLK_GET_RATE_NOCACHE,
353 };
354
355 static const struct clk_init_data mtk_mfg_clk_stack_init = {
356 .name = "gpu-stack",
357 .ops = &mtk_mfg_clk_stack_ops,
358 .flags = CLK_GET_RATE_NOCACHE,
359 };
360
mtk_mfg_eb_on(struct mtk_mfg * mfg)361 static int mtk_mfg_eb_on(struct mtk_mfg *mfg)
362 {
363 struct device *dev = &mfg->pdev->dev;
364 u32 val;
365 int ret;
366
367 /*
368 * If MFG is already on from e.g. the bootloader, skip doing the
369 * power-on sequence, as it wouldn't work without powering it off first.
370 */
371 if (mtk_mfg_is_powered_on(mfg))
372 return 0;
373
374 ret = readl_poll_timeout(mfg->rpc + RPC_GHPM_RO0_CON, val,
375 !(val & (GHPM_PWR_STATE_M | GHPM_STATE_M)),
376 GPUEB_POLL_US, GPUEB_TIMEOUT_US);
377 if (ret) {
378 dev_err(dev, "timed out waiting for EB to power on\n");
379 return ret;
380 }
381
382 mtk_mfg_update_reg_bits(mfg->rpc + mfg->ghpm_en_reg, GHPM_ENABLE_M,
383 GHPM_ENABLE_M);
384
385 mtk_mfg_update_reg_bits(mfg->rpc + RPC_GHPM_CFG0_CON, GHPM_ON_SEQ_M, 0);
386 mtk_mfg_update_reg_bits(mfg->rpc + RPC_GHPM_CFG0_CON, GHPM_ON_SEQ_M,
387 GHPM_ON_SEQ_M);
388
389 mtk_mfg_update_reg_bits(mfg->rpc + mfg->ghpm_en_reg, GHPM_ENABLE_M, 0);
390
391
392 ret = readl_poll_timeout(mfg->rpc + RPC_PWR_CON, val,
393 (val & PWR_ACK_M) == PWR_ACK_M,
394 GPUEB_POLL_US, GPUEB_TIMEOUT_US);
395 if (ret) {
396 dev_err(dev, "timed out waiting for EB power ack, val = 0x%X\n",
397 val);
398 return ret;
399 }
400
401 ret = readl_poll_timeout(mfg->gpr + GPR_LP_STATE, val,
402 (val == EB_ON_RESUME),
403 GPUEB_POLL_US, GPUEB_TIMEOUT_US);
404 if (ret) {
405 dev_err(dev, "timed out waiting for EB to resume, status = 0x%X\n", val);
406 return ret;
407 }
408
409 return 0;
410 }
411
mtk_mfg_eb_off(struct mtk_mfg * mfg)412 static int mtk_mfg_eb_off(struct mtk_mfg *mfg)
413 {
414 struct device *dev = &mfg->pdev->dev;
415 struct mtk_mfg_ipi_sleep_msg msg = {
416 .event = 0,
417 .state = 0,
418 .magic = GPUEB_SLEEP_MAGIC
419 };
420 u32 val;
421 int ret;
422
423 ret = mbox_send_message(mfg->slp_mbox->ch, &msg);
424 if (ret < 0) {
425 dev_err(dev, "Cannot send sleep command: %pe\n", ERR_PTR(ret));
426 return ret;
427 }
428
429 ret = readl_poll_timeout(mfg->rpc + RPC_PWR_CON, val,
430 !(val & PWR_ACK_M), GPUEB_POLL_US,
431 GPUEB_TIMEOUT_US);
432
433 if (ret) {
434 dev_err(dev, "Timed out waiting for EB to power off, val=0x%08X\n", val);
435 return ret;
436 }
437
438 return 0;
439 }
440
441 /**
442 * mtk_mfg_send_ipi - synchronously send an IPI message on the gpufreq channel
443 * @mfg: pointer to this driver instance's private &struct mtk_mfg
444 * @msg: pointer to a message to send; will have magic filled and response assigned
445 *
446 * Send an IPI message on the gpufreq channel, and wait for a response. Once a
447 * response is received, assign a pointer to the response buffer (valid until
448 * next response is received) to @msg.
449 *
450 * Returns 0 on success, negative errno on failure.
451 */
mtk_mfg_send_ipi(struct mtk_mfg * mfg,struct mtk_mfg_ipi_msg * msg)452 static int mtk_mfg_send_ipi(struct mtk_mfg *mfg, struct mtk_mfg_ipi_msg *msg)
453 {
454 struct device *dev = &mfg->pdev->dev;
455 unsigned long wait;
456 int ret;
457
458 msg->magic = mfg->ipi_magic;
459
460 ret = mbox_send_message(mfg->gf_mbox->ch, msg);
461 if (ret < 0) {
462 dev_err(dev, "Cannot send GPUFreq IPI command: %pe\n", ERR_PTR(ret));
463 return ret;
464 }
465
466 wait = wait_for_completion_timeout(&mfg->gf_mbox->rx_done, msecs_to_jiffies(500));
467 if (!wait)
468 return -ETIMEDOUT;
469
470 msg = mfg->gf_mbox->rx_data;
471
472 if (msg->u.return_value < 0) {
473 dev_err(dev, "IPI return: %d\n", msg->u.return_value);
474 return -EPROTO;
475 }
476
477 return 0;
478 }
479
mtk_mfg_init_shared_mem(struct mtk_mfg * mfg)480 static int mtk_mfg_init_shared_mem(struct mtk_mfg *mfg)
481 {
482 struct device *dev = &mfg->pdev->dev;
483 struct mtk_mfg_ipi_msg msg = {};
484 int ret;
485
486 dev_dbg(dev, "clearing GPUEB shared memory, 0x%X bytes\n", mfg->shared_mem_size);
487 memset_io(mfg->shared_mem, 0, mfg->shared_mem_size);
488
489 msg.cmd = CMD_INIT_SHARED_MEM;
490 msg.u.shared_mem.base = mfg->shared_mem_phys;
491 msg.u.shared_mem.size = mfg->shared_mem_size;
492
493 ret = mtk_mfg_send_ipi(mfg, &msg);
494 if (ret)
495 return ret;
496
497 if (readl(mfg->shared_mem + GF_REG_MAGIC) != GPUEB_MEM_MAGIC) {
498 dev_err(dev, "EB did not initialise shared memory correctly\n");
499 return -EIO;
500 }
501
502 return 0;
503 }
504
mtk_mfg_power_control(struct mtk_mfg * mfg,bool enabled)505 static int mtk_mfg_power_control(struct mtk_mfg *mfg, bool enabled)
506 {
507 struct mtk_mfg_ipi_msg msg = {};
508
509 msg.cmd = CMD_POWER_CONTROL;
510 msg.u.power_state = enabled ? 1 : 0;
511
512 return mtk_mfg_send_ipi(mfg, &msg);
513 }
514
mtk_mfg_set_oppidx(struct mtk_mfg * mfg,unsigned int opp_idx)515 static int mtk_mfg_set_oppidx(struct mtk_mfg *mfg, unsigned int opp_idx)
516 {
517 struct mtk_mfg_ipi_msg msg = {};
518 int ret;
519
520 if (opp_idx >= mfg->num_gpu_opps)
521 return -EINVAL;
522
523 msg.cmd = CMD_FIX_DUAL_TARGET_OPPIDX;
524 msg.u.dual_commit.gpu_oppidx = opp_idx;
525 msg.u.dual_commit.stack_oppidx = opp_idx;
526
527 ret = mtk_mfg_send_ipi(mfg, &msg);
528 if (ret) {
529 dev_err(&mfg->pdev->dev, "Failed to set OPP %u: %pe\n",
530 opp_idx, ERR_PTR(ret));
531 return ret;
532 }
533
534 return 0;
535 }
536
mtk_mfg_read_opp_tables(struct mtk_mfg * mfg)537 static int mtk_mfg_read_opp_tables(struct mtk_mfg *mfg)
538 {
539 struct device *dev = &mfg->pdev->dev;
540 struct mtk_mfg_opp_entry e = {};
541 unsigned int i;
542
543 mfg->num_gpu_opps = readl(mfg->shared_mem + GF_REG_GPU_OPP_NUM);
544 mfg->num_stack_opps = readl(mfg->shared_mem + GF_REG_STK_OPP_NUM);
545
546 if (mfg->num_gpu_opps > MAX_OPP_NUM || mfg->num_gpu_opps == 0) {
547 dev_err(dev, "GPU OPP count (%u) out of range %u >= count > 0\n",
548 mfg->num_gpu_opps, MAX_OPP_NUM);
549 return -EINVAL;
550 }
551
552 if (mfg->num_stack_opps && mfg->num_stack_opps > MAX_OPP_NUM) {
553 dev_err(dev, "Stack OPP count (%u) out of range %u >= count >= 0\n",
554 mfg->num_stack_opps, MAX_OPP_NUM);
555 return -EINVAL;
556 }
557
558 mfg->gpu_opps = devm_kcalloc(dev, mfg->num_gpu_opps,
559 sizeof(struct dev_pm_opp_data), GFP_KERNEL);
560 if (!mfg->gpu_opps)
561 return -ENOMEM;
562
563 if (mfg->num_stack_opps) {
564 mfg->stack_opps = devm_kcalloc(dev, mfg->num_stack_opps,
565 sizeof(struct dev_pm_opp_data), GFP_KERNEL);
566 if (!mfg->stack_opps)
567 return -ENOMEM;
568 }
569
570 for (i = 0; i < mfg->num_gpu_opps; i++) {
571 memcpy_fromio(&e, mfg->shared_mem + GF_REG_OPP_TABLE_GPU + i * sizeof(e),
572 sizeof(e));
573 if (mem_is_zero(&e, sizeof(e))) {
574 dev_err(dev, "ran into an empty GPU OPP at index %u\n",
575 i);
576 return -EINVAL;
577 }
578 mfg->gpu_opps[i].freq = e.freq_khz * HZ_PER_KHZ;
579 mfg->gpu_opps[i].u_volt = e.voltage_core * 10;
580 mfg->gpu_opps[i].level = i;
581 if (i < mfg->variant->turbo_below)
582 mfg->gpu_opps[i].turbo = true;
583 }
584
585 for (i = 0; i < mfg->num_stack_opps; i++) {
586 memcpy_fromio(&e, mfg->shared_mem + GF_REG_OPP_TABLE_STK + i * sizeof(e),
587 sizeof(e));
588 if (mem_is_zero(&e, sizeof(e))) {
589 dev_err(dev, "ran into an empty Stack OPP at index %u\n",
590 i);
591 return -EINVAL;
592 }
593 mfg->stack_opps[i].freq = e.freq_khz * HZ_PER_KHZ;
594 mfg->stack_opps[i].u_volt = e.voltage_core * 10;
595 mfg->stack_opps[i].level = i;
596 if (i < mfg->variant->turbo_below)
597 mfg->stack_opps[i].turbo = true;
598 }
599
600 return 0;
601 }
602
603 static const char *const mtk_mfg_mt8196_clk_names[] = {
604 "core",
605 "stack0",
606 "stack1",
607 };
608
609 static const char *const mtk_mfg_mt8196_regulators[] = {
610 "core",
611 "stack",
612 "sram",
613 };
614
mtk_mfg_mt8196_init(struct mtk_mfg * mfg)615 static int mtk_mfg_mt8196_init(struct mtk_mfg *mfg)
616 {
617 void __iomem *e2_base;
618
619 e2_base = devm_platform_ioremap_resource_byname(mfg->pdev, "hw-revision");
620 if (IS_ERR(e2_base))
621 return dev_err_probe(&mfg->pdev->dev, PTR_ERR(e2_base),
622 "Couldn't get hw-revision register\n");
623
624 clk_prepare_enable(mfg->clk_eb);
625
626 if (readl(e2_base) == MFG_MT8196_E2_ID)
627 mfg->ghpm_en_reg = RPC_DUMMY_REG_2;
628 else
629 mfg->ghpm_en_reg = RPC_GHPM_CFG0_CON;
630
631 clk_disable_unprepare(mfg->clk_eb);
632
633 return 0;
634 }
635
636 static const struct mtk_mfg_variant mtk_mfg_mt8196_variant = {
637 .clk_names = mtk_mfg_mt8196_clk_names,
638 .num_clks = ARRAY_SIZE(mtk_mfg_mt8196_clk_names),
639 .regulator_names = mtk_mfg_mt8196_regulators,
640 .num_regulators = ARRAY_SIZE(mtk_mfg_mt8196_regulators),
641 .turbo_below = 7,
642 .init = mtk_mfg_mt8196_init,
643 };
644
mtk_mfg_mbox_rx_callback(struct mbox_client * cl,void * mssg)645 static void mtk_mfg_mbox_rx_callback(struct mbox_client *cl, void *mssg)
646 {
647 struct mtk_mfg_mbox *mb = container_of(cl, struct mtk_mfg_mbox, cl);
648
649 if (mb->rx_data)
650 mb->rx_data = memcpy(mb->rx_data, mssg, GPUEB_MBOX_MAX_RX_SIZE);
651 complete(&mb->rx_done);
652 }
653
mtk_mfg_attach_dev(struct generic_pm_domain * pd,struct device * dev)654 static int mtk_mfg_attach_dev(struct generic_pm_domain *pd, struct device *dev)
655 {
656 struct mtk_mfg *mfg = mtk_mfg_from_genpd(pd);
657 struct dev_pm_opp_data *so = mfg->stack_opps;
658 struct dev_pm_opp_data *go = mfg->gpu_opps;
659 struct dev_pm_opp_data *prev_o;
660 struct dev_pm_opp_data *o;
661 int i, ret;
662
663 for (i = mfg->num_gpu_opps - 1; i >= 0; i--) {
664 /*
665 * Adding the lower of the two OPPs avoids gaps of indices in
666 * situations where the GPU OPPs are duplicated a couple of
667 * times when only the Stack OPP is being lowered at that index.
668 */
669 if (i >= mfg->num_stack_opps || go[i].freq < so[i].freq)
670 o = &go[i];
671 else
672 o = &so[i];
673
674 /*
675 * Skip indices where both GPU and Stack OPPs are equal. Nominally,
676 * OPP core shouldn't care about dupes, but not doing so will cause
677 * dev_pm_opp_find_freq_ceil_indexed to -ERANGE later down the line.
678 */
679 if (prev_o && prev_o->freq == o->freq)
680 continue;
681
682 ret = dev_pm_opp_add_dynamic(dev, o);
683 if (ret) {
684 dev_err(dev, "Failed to add OPP level %u from PD %s: %pe\n",
685 o->level, pd->name, ERR_PTR(ret));
686 dev_pm_opp_remove_all_dynamic(dev);
687 return ret;
688 }
689 prev_o = o;
690 }
691
692 return 0;
693 }
694
mtk_mfg_detach_dev(struct generic_pm_domain * pd,struct device * dev)695 static void mtk_mfg_detach_dev(struct generic_pm_domain *pd, struct device *dev)
696 {
697 dev_pm_opp_remove_all_dynamic(dev);
698 }
699
mtk_mfg_set_performance(struct generic_pm_domain * pd,unsigned int state)700 static int mtk_mfg_set_performance(struct generic_pm_domain *pd,
701 unsigned int state)
702 {
703 struct mtk_mfg *mfg = mtk_mfg_from_genpd(pd);
704
705 /*
706 * pmdomain core intentionally sets a performance state before turning
707 * a domain on, and after turning it off. For the GPUEB however, it's
708 * only possible to act on performance requests when the GPUEB is
709 * powered on. To do this, return cleanly without taking action, and
710 * defer setting what pmdomain core set in mtk_mfg_power_on.
711 */
712 if (mfg->pd.status != GENPD_STATE_ON)
713 return 0;
714
715 return mtk_mfg_set_oppidx(mfg, state);
716 }
717
mtk_mfg_power_on(struct generic_pm_domain * pd)718 static int mtk_mfg_power_on(struct generic_pm_domain *pd)
719 {
720 struct mtk_mfg *mfg = mtk_mfg_from_genpd(pd);
721 int ret;
722
723 ret = regulator_bulk_enable(mfg->variant->num_regulators,
724 mfg->gpu_regs);
725 if (ret)
726 return ret;
727
728 ret = clk_prepare_enable(mfg->clk_eb);
729 if (ret)
730 goto err_disable_regulators;
731
732 ret = clk_bulk_prepare_enable(mfg->variant->num_clks, mfg->gpu_clks);
733 if (ret)
734 goto err_disable_eb_clk;
735
736 ret = mtk_mfg_eb_on(mfg);
737 if (ret)
738 goto err_disable_clks;
739
740 mfg->ipi_magic = readl(mfg->gpr + GPR_IPI_MAGIC);
741
742 ret = mtk_mfg_power_control(mfg, true);
743 if (ret)
744 goto err_eb_off;
745
746 /* Don't try to set a OPP in probe before OPPs have been read from EB */
747 if (mfg->gpu_opps) {
748 /* The aforementioned deferred setting of pmdomain's state */
749 ret = mtk_mfg_set_oppidx(mfg, pd->performance_state);
750 if (ret)
751 dev_warn(&mfg->pdev->dev, "Failed to set oppidx in %s\n", __func__);
752 }
753
754 return 0;
755
756 err_eb_off:
757 mtk_mfg_eb_off(mfg);
758 err_disable_clks:
759 clk_bulk_disable_unprepare(mfg->variant->num_clks, mfg->gpu_clks);
760 err_disable_eb_clk:
761 clk_disable_unprepare(mfg->clk_eb);
762 err_disable_regulators:
763 regulator_bulk_disable(mfg->variant->num_regulators, mfg->gpu_regs);
764
765 return ret;
766 }
767
mtk_mfg_power_off(struct generic_pm_domain * pd)768 static int mtk_mfg_power_off(struct generic_pm_domain *pd)
769 {
770 struct mtk_mfg *mfg = mtk_mfg_from_genpd(pd);
771 struct device *dev = &mfg->pdev->dev;
772 int ret;
773
774 ret = mtk_mfg_power_control(mfg, false);
775 if (ret) {
776 dev_err(dev, "power_control failed: %pe\n", ERR_PTR(ret));
777 return ret;
778 }
779
780 ret = mtk_mfg_eb_off(mfg);
781 if (ret) {
782 dev_err(dev, "eb_off failed: %pe\n", ERR_PTR(ret));
783 return ret;
784 }
785
786 clk_bulk_disable_unprepare(mfg->variant->num_clks, mfg->gpu_clks);
787 clk_disable_unprepare(mfg->clk_eb);
788 ret = regulator_bulk_disable(mfg->variant->num_regulators, mfg->gpu_regs);
789 if (ret) {
790 dev_err(dev, "Disabling regulators failed: %pe\n", ERR_PTR(ret));
791 return ret;
792 }
793
794 return 0;
795 }
796
mtk_mfg_init_mbox(struct mtk_mfg * mfg)797 static int mtk_mfg_init_mbox(struct mtk_mfg *mfg)
798 {
799 struct device *dev = &mfg->pdev->dev;
800 struct mtk_mfg_mbox *gf;
801 struct mtk_mfg_mbox *slp;
802
803 gf = devm_kzalloc(dev, sizeof(*gf), GFP_KERNEL);
804 if (!gf)
805 return -ENOMEM;
806
807 gf->rx_data = devm_kzalloc(dev, GPUEB_MBOX_MAX_RX_SIZE, GFP_KERNEL);
808 if (!gf->rx_data)
809 return -ENOMEM;
810
811 gf->mfg = mfg;
812 init_completion(&gf->rx_done);
813 gf->cl.dev = dev;
814 gf->cl.rx_callback = mtk_mfg_mbox_rx_callback;
815 gf->cl.tx_tout = GPUEB_TIMEOUT_US / USEC_PER_MSEC;
816 gf->ch = mbox_request_channel_byname(&gf->cl, "gpufreq");
817 if (IS_ERR(gf->ch))
818 return PTR_ERR(gf->ch);
819
820 mfg->gf_mbox = gf;
821
822 slp = devm_kzalloc(dev, sizeof(*slp), GFP_KERNEL);
823 if (!slp)
824 return -ENOMEM;
825
826 slp->mfg = mfg;
827 init_completion(&slp->rx_done);
828 slp->cl.dev = dev;
829 slp->cl.tx_tout = GPUEB_TIMEOUT_US / USEC_PER_MSEC;
830 slp->cl.tx_block = true;
831 slp->ch = mbox_request_channel_byname(&slp->cl, "sleep");
832 if (IS_ERR(slp->ch)) {
833 mbox_free_channel(gf->ch);
834 return PTR_ERR(slp->ch);
835 }
836
837 mfg->slp_mbox = slp;
838
839 return 0;
840 }
841
mtk_mfg_init_clk_provider(struct mtk_mfg * mfg)842 static int mtk_mfg_init_clk_provider(struct mtk_mfg *mfg)
843 {
844 struct device *dev = &mfg->pdev->dev;
845 struct clk_hw_onecell_data *clk_data;
846 int ret;
847
848 clk_data = devm_kzalloc(dev, struct_size(clk_data, hws, 2), GFP_KERNEL);
849 if (!clk_data)
850 return -ENOMEM;
851
852 clk_data->num = 2;
853
854 mfg->clk_core_hw.init = &mtk_mfg_clk_gpu_init;
855 mfg->clk_stack_hw.init = &mtk_mfg_clk_stack_init;
856
857 ret = devm_clk_hw_register(dev, &mfg->clk_core_hw);
858 if (ret)
859 return dev_err_probe(dev, ret, "Couldn't register GPU core clock\n");
860
861 ret = devm_clk_hw_register(dev, &mfg->clk_stack_hw);
862 if (ret)
863 return dev_err_probe(dev, ret, "Couldn't register GPU stack clock\n");
864
865 clk_data->hws[0] = &mfg->clk_core_hw;
866 clk_data->hws[1] = &mfg->clk_stack_hw;
867
868 ret = devm_of_clk_add_hw_provider(dev, of_clk_hw_onecell_get, clk_data);
869 if (ret)
870 return dev_err_probe(dev, ret, "Couldn't register clock provider\n");
871
872 return 0;
873 }
874
mtk_mfg_probe(struct platform_device * pdev)875 static int mtk_mfg_probe(struct platform_device *pdev)
876 {
877 struct mtk_mfg *mfg;
878 struct device *dev = &pdev->dev;
879 const struct mtk_mfg_variant *data = of_device_get_match_data(dev);
880 struct resource res;
881 int ret, i;
882
883 mfg = devm_kzalloc(dev, sizeof(*mfg), GFP_KERNEL);
884 if (!mfg)
885 return -ENOMEM;
886
887 mfg->pdev = pdev;
888 mfg->variant = data;
889
890 dev_set_drvdata(dev, mfg);
891
892 mfg->gpr = devm_platform_ioremap_resource(pdev, 0);
893 if (IS_ERR(mfg->gpr))
894 return dev_err_probe(dev, PTR_ERR(mfg->gpr),
895 "Couldn't retrieve GPR MMIO registers\n");
896
897 mfg->rpc = devm_platform_ioremap_resource(pdev, 1);
898 if (IS_ERR(mfg->rpc))
899 return dev_err_probe(dev, PTR_ERR(mfg->rpc),
900 "Couldn't retrieve RPC MMIO registers\n");
901
902 mfg->clk_eb = devm_clk_get(dev, "eb");
903 if (IS_ERR(mfg->clk_eb))
904 return dev_err_probe(dev, PTR_ERR(mfg->clk_eb),
905 "Couldn't get 'eb' clock\n");
906
907 mfg->gpu_clks = devm_kcalloc(dev, data->num_clks, sizeof(*mfg->gpu_clks),
908 GFP_KERNEL);
909 if (!mfg->gpu_clks)
910 return -ENOMEM;
911
912 for (i = 0; i < data->num_clks; i++)
913 mfg->gpu_clks[i].id = data->clk_names[i];
914
915 ret = devm_clk_bulk_get(dev, data->num_clks, mfg->gpu_clks);
916 if (ret)
917 return dev_err_probe(dev, ret, "Couldn't get GPU clocks\n");
918
919 mfg->gpu_regs = devm_kcalloc(dev, data->num_regulators,
920 sizeof(*mfg->gpu_regs), GFP_KERNEL);
921 if (!mfg->gpu_regs)
922 return -ENOMEM;
923
924 for (i = 0; i < data->num_regulators; i++)
925 mfg->gpu_regs[i].supply = data->regulator_names[i];
926
927 ret = devm_regulator_bulk_get(dev, data->num_regulators, mfg->gpu_regs);
928 if (ret)
929 return dev_err_probe(dev, ret, "Couldn't get GPU regulators\n");
930
931 ret = of_reserved_mem_region_to_resource(dev->of_node, 0, &res);
932 if (ret)
933 return dev_err_probe(dev, ret, "Couldn't get GPUEB shared memory\n");
934
935 mfg->shared_mem = devm_ioremap(dev, res.start, resource_size(&res));
936 if (!mfg->shared_mem)
937 return dev_err_probe(dev, -ENOMEM, "Can't ioremap GPUEB shared memory\n");
938 mfg->shared_mem_size = resource_size(&res);
939 mfg->shared_mem_phys = res.start;
940
941 if (data->init) {
942 ret = data->init(mfg);
943 if (ret)
944 return dev_err_probe(dev, ret, "Variant init failed\n");
945 }
946
947 mfg->pd.name = dev_name(dev);
948 mfg->pd.attach_dev = mtk_mfg_attach_dev;
949 mfg->pd.detach_dev = mtk_mfg_detach_dev;
950 mfg->pd.power_off = mtk_mfg_power_off;
951 mfg->pd.power_on = mtk_mfg_power_on;
952 mfg->pd.set_performance_state = mtk_mfg_set_performance;
953 mfg->pd.flags = GENPD_FLAG_OPP_TABLE_FW;
954
955 ret = pm_genpd_init(&mfg->pd, NULL, false);
956 if (ret)
957 return dev_err_probe(dev, ret, "Failed to initialise power domain\n");
958
959 ret = mtk_mfg_init_mbox(mfg);
960 if (ret) {
961 dev_err_probe(dev, ret, "Couldn't initialise mailbox\n");
962 goto err_remove_genpd;
963 }
964
965 ret = mtk_mfg_power_on(&mfg->pd);
966 if (ret) {
967 dev_err_probe(dev, ret, "Failed to power on MFG\n");
968 goto err_free_mbox;
969 }
970
971 ret = mtk_mfg_init_shared_mem(mfg);
972 if (ret) {
973 dev_err_probe(dev, ret, "Couldn't initialize EB shared memory\n");
974 goto err_power_off;
975 }
976
977 ret = mtk_mfg_read_opp_tables(mfg);
978 if (ret) {
979 dev_err_probe(dev, ret, "Error reading OPP tables from EB\n");
980 goto err_power_off;
981 }
982
983 ret = mtk_mfg_init_clk_provider(mfg);
984 if (ret)
985 goto err_power_off;
986
987 ret = of_genpd_add_provider_simple(dev->of_node, &mfg->pd);
988 if (ret) {
989 dev_err_probe(dev, ret, "Failed to add pmdomain provider\n");
990 goto err_power_off;
991 }
992
993 return 0;
994
995 err_power_off:
996 mtk_mfg_power_off(&mfg->pd);
997 err_free_mbox:
998 mbox_free_channel(mfg->slp_mbox->ch);
999 mfg->slp_mbox->ch = NULL;
1000 mbox_free_channel(mfg->gf_mbox->ch);
1001 mfg->gf_mbox->ch = NULL;
1002 err_remove_genpd:
1003 pm_genpd_remove(&mfg->pd);
1004
1005 return ret;
1006 }
1007
1008 static const struct of_device_id mtk_mfg_of_match[] = {
1009 { .compatible = "mediatek,mt8196-gpufreq", .data = &mtk_mfg_mt8196_variant },
1010 {}
1011 };
1012 MODULE_DEVICE_TABLE(of, mtk_mfg_of_match);
1013
mtk_mfg_remove(struct platform_device * pdev)1014 static void mtk_mfg_remove(struct platform_device *pdev)
1015 {
1016 struct mtk_mfg *mfg = dev_get_drvdata(&pdev->dev);
1017
1018 if (mtk_mfg_is_powered_on(mfg))
1019 mtk_mfg_power_off(&mfg->pd);
1020
1021 of_genpd_del_provider(pdev->dev.of_node);
1022 pm_genpd_remove(&mfg->pd);
1023
1024 mbox_free_channel(mfg->gf_mbox->ch);
1025 mfg->gf_mbox->ch = NULL;
1026
1027 mbox_free_channel(mfg->slp_mbox->ch);
1028 mfg->slp_mbox->ch = NULL;
1029 }
1030
1031 static struct platform_driver mtk_mfg_driver = {
1032 .driver = {
1033 .name = "mtk-mfg-pmdomain",
1034 .of_match_table = mtk_mfg_of_match,
1035 .suppress_bind_attrs = true,
1036 },
1037 .probe = mtk_mfg_probe,
1038 .remove = mtk_mfg_remove,
1039 };
1040 module_platform_driver(mtk_mfg_driver);
1041
1042 MODULE_AUTHOR("Nicolas Frattaroli <nicolas.frattaroli@collabora.com>");
1043 MODULE_DESCRIPTION("MediaTek MFlexGraphics Power Domain Driver");
1044 MODULE_LICENSE("GPL");
1045