1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright 2021 NXP */
3
4 #include <dt-bindings/firmware/imx/rsrc.h>
5 #include <linux/arm-smccc.h>
6 #include <linux/clk.h>
7 #include <linux/err.h>
8 #include <linux/firmware.h>
9 #include <linux/firmware/imx/sci.h>
10 #include <linux/interrupt.h>
11 #include <linux/kernel.h>
12 #include <linux/mailbox_client.h>
13 #include <linux/mfd/syscon.h>
14 #include <linux/module.h>
15 #include <linux/of.h>
16 #include <linux/of_reserved_mem.h>
17 #include <linux/platform_device.h>
18 #include <linux/pm_domain.h>
19 #include <linux/pm_runtime.h>
20 #include <linux/regmap.h>
21 #include <linux/remoteproc.h>
22 #include <linux/reset.h>
23 #include <linux/slab.h>
24
25 #include "imx_rproc.h"
26 #include "remoteproc_elf_helpers.h"
27 #include "remoteproc_internal.h"
28
29 #define DSP_RPROC_CLK_MAX 5
30
31 /*
32 * Module parameters
33 */
34 static unsigned int no_mailboxes;
35 module_param_named(no_mailboxes, no_mailboxes, int, 0644);
36 MODULE_PARM_DESC(no_mailboxes,
37 "There is no mailbox between cores, so ignore remote proc reply after start, default is 0 (off).");
38
39 #define REMOTE_IS_READY BIT(0)
40 #define REMOTE_READY_WAIT_MAX_RETRIES 500
41
42 /* att flags */
43 /* DSP own area */
44 #define ATT_OWN BIT(31)
45 /* DSP instruction area */
46 #define ATT_IRAM BIT(30)
47
48 /* Definitions for i.MX8MP */
49 /* DAP registers */
50 #define IMX8M_DAP_DEBUG 0x28800000
51 #define IMX8M_DAP_DEBUG_SIZE (64 * 1024)
52 #define IMX8M_DAP_PWRCTL (0x4000 + 0x3020)
53 #define IMX8M_PWRCTL_CORERESET BIT(16)
54
55 /* DSP audio mix registers */
56 #define IMX8M_AudioDSP_REG0 0x100
57 #define IMX8M_AudioDSP_REG1 0x104
58 #define IMX8M_AudioDSP_REG2 0x108
59 #define IMX8M_AudioDSP_REG3 0x10c
60
61 #define IMX8M_AudioDSP_REG2_RUNSTALL BIT(5)
62 #define IMX8M_AudioDSP_REG2_PWAITMODE BIT(1)
63
64 /* Definitions for i.MX8ULP */
65 #define IMX8ULP_SIM_LPAV_REG_SYSCTRL0 0x8
66 #define IMX8ULP_SYSCTRL0_DSP_DBG_RST BIT(25)
67 #define IMX8ULP_SYSCTRL0_DSP_PLAT_CLK_EN BIT(19)
68 #define IMX8ULP_SYSCTRL0_DSP_PBCLK_EN BIT(18)
69 #define IMX8ULP_SYSCTRL0_DSP_CLK_EN BIT(17)
70 #define IMX8ULP_SYSCTRL0_DSP_RST BIT(16)
71 #define IMX8ULP_SYSCTRL0_DSP_OCD_HALT BIT(14)
72 #define IMX8ULP_SYSCTRL0_DSP_STALL BIT(13)
73
74 #define IMX8ULP_SIP_HIFI_XRDC 0xc200000e
75
76 /*
77 * enum - Predefined Mailbox Messages
78 *
79 * @RP_MBOX_SUSPEND_SYSTEM: system suspend request for the remote processor
80 *
81 * @RP_MBOX_SUSPEND_ACK: successful response from remote processor for a
82 * suspend request
83 *
84 * @RP_MBOX_RESUME_SYSTEM: system resume request for the remote processor
85 *
86 * @RP_MBOX_RESUME_ACK: successful response from remote processor for a
87 * resume request
88 */
89 enum imx_dsp_rp_mbox_messages {
90 RP_MBOX_SUSPEND_SYSTEM = 0xFF11,
91 RP_MBOX_SUSPEND_ACK = 0xFF12,
92 RP_MBOX_RESUME_SYSTEM = 0xFF13,
93 RP_MBOX_RESUME_ACK = 0xFF14,
94 };
95
96 /**
97 * struct imx_dsp_rproc - DSP remote processor state
98 * @regmap: regmap handler
99 * @run_stall: reset control handle used for Run/Stall operation
100 * @rproc: rproc handler
101 * @dsp_dcfg: device configuration pointer
102 * @clks: clocks needed by this device
103 * @cl: mailbox client to request the mailbox channel
104 * @cl_rxdb: mailbox client to request the mailbox channel for doorbell
105 * @tx_ch: mailbox tx channel handle
106 * @rx_ch: mailbox rx channel handle
107 * @rxdb_ch: mailbox rx doorbell channel handle
108 * @pd_list: power domain list
109 * @ipc_handle: System Control Unit ipc handle
110 * @rproc_work: work for processing virtio interrupts
111 * @pm_comp: completion primitive to sync for suspend response
112 * @flags: control flags
113 */
114 struct imx_dsp_rproc {
115 struct regmap *regmap;
116 struct reset_control *run_stall;
117 struct rproc *rproc;
118 const struct imx_dsp_rproc_dcfg *dsp_dcfg;
119 struct clk_bulk_data clks[DSP_RPROC_CLK_MAX];
120 struct mbox_client cl;
121 struct mbox_client cl_rxdb;
122 struct mbox_chan *tx_ch;
123 struct mbox_chan *rx_ch;
124 struct mbox_chan *rxdb_ch;
125 struct dev_pm_domain_list *pd_list;
126 struct imx_sc_ipc *ipc_handle;
127 struct work_struct rproc_work;
128 struct completion pm_comp;
129 u32 flags;
130 };
131
132 /**
133 * struct imx_dsp_rproc_dcfg - DSP remote processor configuration
134 * @dcfg: imx_rproc_dcfg handler
135 * @reset: reset callback function
136 */
137 struct imx_dsp_rproc_dcfg {
138 const struct imx_rproc_dcfg *dcfg;
139 int (*reset)(struct imx_dsp_rproc *priv);
140 };
141
142 static const struct imx_rproc_att imx_dsp_rproc_att_imx8qm[] = {
143 /* dev addr , sys addr , size , flags */
144 { 0x596e8000, 0x556e8000, 0x00008000, ATT_OWN },
145 { 0x596f0000, 0x556f0000, 0x00008000, ATT_OWN },
146 { 0x596f8000, 0x556f8000, 0x00000800, ATT_OWN | ATT_IRAM},
147 { 0x55700000, 0x55700000, 0x00070000, ATT_OWN },
148 /* DDR (Data) */
149 { 0x80000000, 0x80000000, 0x60000000, 0},
150 };
151
152 static const struct imx_rproc_att imx_dsp_rproc_att_imx8qxp[] = {
153 /* dev addr , sys addr , size , flags */
154 { 0x596e8000, 0x596e8000, 0x00008000, ATT_OWN },
155 { 0x596f0000, 0x596f0000, 0x00008000, ATT_OWN },
156 { 0x596f8000, 0x596f8000, 0x00000800, ATT_OWN | ATT_IRAM},
157 { 0x59700000, 0x59700000, 0x00070000, ATT_OWN },
158 /* DDR (Data) */
159 { 0x80000000, 0x80000000, 0x60000000, 0},
160 };
161
162 static const struct imx_rproc_att imx_dsp_rproc_att_imx8mp[] = {
163 /* dev addr , sys addr , size , flags */
164 { 0x3b6e8000, 0x3b6e8000, 0x00008000, ATT_OWN },
165 { 0x3b6f0000, 0x3b6f0000, 0x00008000, ATT_OWN },
166 { 0x3b6f8000, 0x3b6f8000, 0x00000800, ATT_OWN | ATT_IRAM},
167 { 0x3b700000, 0x3b700000, 0x00040000, ATT_OWN },
168 /* DDR (Data) */
169 { 0x40000000, 0x40000000, 0x80000000, 0},
170 };
171
172 static const struct imx_rproc_att imx_dsp_rproc_att_imx8ulp[] = {
173 /* dev addr , sys addr , size , flags */
174 { 0x21170000, 0x21170000, 0x00010000, ATT_OWN | ATT_IRAM},
175 { 0x21180000, 0x21180000, 0x00010000, ATT_OWN },
176 /* DDR (Data) */
177 { 0x0c000000, 0x80000000, 0x10000000, 0},
178 { 0x30000000, 0x90000000, 0x10000000, 0},
179 };
180
181 /* Initialize the mailboxes between cores, if exists */
182 static int (*imx_dsp_rproc_mbox_init)(struct imx_dsp_rproc *priv);
183
184 /* Reset function for DSP on i.MX8MP */
imx8mp_dsp_reset(struct imx_dsp_rproc * priv)185 static int imx8mp_dsp_reset(struct imx_dsp_rproc *priv)
186 {
187 void __iomem *dap = ioremap_wc(IMX8M_DAP_DEBUG, IMX8M_DAP_DEBUG_SIZE);
188 int pwrctl;
189
190 /* Put DSP into reset and stall */
191 pwrctl = readl(dap + IMX8M_DAP_PWRCTL);
192 pwrctl |= IMX8M_PWRCTL_CORERESET;
193 writel(pwrctl, dap + IMX8M_DAP_PWRCTL);
194
195 /* Keep reset asserted for 10 cycles */
196 usleep_range(1, 2);
197
198 reset_control_assert(priv->run_stall);
199
200 /* Take the DSP out of reset and keep stalled for FW loading */
201 pwrctl = readl(dap + IMX8M_DAP_PWRCTL);
202 pwrctl &= ~IMX8M_PWRCTL_CORERESET;
203 writel(pwrctl, dap + IMX8M_DAP_PWRCTL);
204
205 iounmap(dap);
206 return 0;
207 }
208
209 /* Reset function for DSP on i.MX8ULP */
imx8ulp_dsp_reset(struct imx_dsp_rproc * priv)210 static int imx8ulp_dsp_reset(struct imx_dsp_rproc *priv)
211 {
212 struct arm_smccc_res res;
213
214 /* Put DSP into reset and stall */
215 regmap_update_bits(priv->regmap, IMX8ULP_SIM_LPAV_REG_SYSCTRL0,
216 IMX8ULP_SYSCTRL0_DSP_RST, IMX8ULP_SYSCTRL0_DSP_RST);
217 regmap_update_bits(priv->regmap, IMX8ULP_SIM_LPAV_REG_SYSCTRL0,
218 IMX8ULP_SYSCTRL0_DSP_STALL,
219 IMX8ULP_SYSCTRL0_DSP_STALL);
220
221 /* Configure resources of DSP through TFA */
222 arm_smccc_smc(IMX8ULP_SIP_HIFI_XRDC, 0, 0, 0, 0, 0, 0, 0, &res);
223
224 /* Take the DSP out of reset and keep stalled for FW loading */
225 regmap_update_bits(priv->regmap, IMX8ULP_SIM_LPAV_REG_SYSCTRL0,
226 IMX8ULP_SYSCTRL0_DSP_RST, 0);
227 regmap_update_bits(priv->regmap, IMX8ULP_SIM_LPAV_REG_SYSCTRL0,
228 IMX8ULP_SYSCTRL0_DSP_DBG_RST, 0);
229
230 return 0;
231 }
232
233 /* Specific configuration for i.MX8MP */
234 static const struct imx_rproc_dcfg dsp_rproc_cfg_imx8mp = {
235 .att = imx_dsp_rproc_att_imx8mp,
236 .att_size = ARRAY_SIZE(imx_dsp_rproc_att_imx8mp),
237 .method = IMX_RPROC_RESET_CONTROLLER,
238 };
239
240 static const struct imx_dsp_rproc_dcfg imx_dsp_rproc_cfg_imx8mp = {
241 .dcfg = &dsp_rproc_cfg_imx8mp,
242 .reset = imx8mp_dsp_reset,
243 };
244
245 /* Specific configuration for i.MX8ULP */
246 static const struct imx_rproc_dcfg dsp_rproc_cfg_imx8ulp = {
247 .src_reg = IMX8ULP_SIM_LPAV_REG_SYSCTRL0,
248 .src_mask = IMX8ULP_SYSCTRL0_DSP_STALL,
249 .src_start = 0,
250 .src_stop = IMX8ULP_SYSCTRL0_DSP_STALL,
251 .att = imx_dsp_rproc_att_imx8ulp,
252 .att_size = ARRAY_SIZE(imx_dsp_rproc_att_imx8ulp),
253 .method = IMX_RPROC_MMIO,
254 };
255
256 static const struct imx_dsp_rproc_dcfg imx_dsp_rproc_cfg_imx8ulp = {
257 .dcfg = &dsp_rproc_cfg_imx8ulp,
258 .reset = imx8ulp_dsp_reset,
259 };
260
261 /* Specific configuration for i.MX8QXP */
262 static const struct imx_rproc_dcfg dsp_rproc_cfg_imx8qxp = {
263 .att = imx_dsp_rproc_att_imx8qxp,
264 .att_size = ARRAY_SIZE(imx_dsp_rproc_att_imx8qxp),
265 .method = IMX_RPROC_SCU_API,
266 };
267
268 static const struct imx_dsp_rproc_dcfg imx_dsp_rproc_cfg_imx8qxp = {
269 .dcfg = &dsp_rproc_cfg_imx8qxp,
270 };
271
272 /* Specific configuration for i.MX8QM */
273 static const struct imx_rproc_dcfg dsp_rproc_cfg_imx8qm = {
274 .att = imx_dsp_rproc_att_imx8qm,
275 .att_size = ARRAY_SIZE(imx_dsp_rproc_att_imx8qm),
276 .method = IMX_RPROC_SCU_API,
277 };
278
279 static const struct imx_dsp_rproc_dcfg imx_dsp_rproc_cfg_imx8qm = {
280 .dcfg = &dsp_rproc_cfg_imx8qm,
281 };
282
imx_dsp_rproc_ready(struct rproc * rproc)283 static int imx_dsp_rproc_ready(struct rproc *rproc)
284 {
285 struct imx_dsp_rproc *priv = rproc->priv;
286 int i;
287
288 if (!priv->rxdb_ch)
289 return 0;
290
291 for (i = 0; i < REMOTE_READY_WAIT_MAX_RETRIES; i++) {
292 if (priv->flags & REMOTE_IS_READY)
293 return 0;
294 usleep_range(100, 200);
295 }
296
297 return -ETIMEDOUT;
298 }
299
300 /*
301 * Start function for rproc_ops
302 *
303 * There is a handshake for start procedure: when DSP starts, it
304 * will send a doorbell message to this driver, then the
305 * REMOTE_IS_READY flags is set, then driver will kick
306 * a message to DSP.
307 */
imx_dsp_rproc_start(struct rproc * rproc)308 static int imx_dsp_rproc_start(struct rproc *rproc)
309 {
310 struct imx_dsp_rproc *priv = rproc->priv;
311 const struct imx_dsp_rproc_dcfg *dsp_dcfg = priv->dsp_dcfg;
312 const struct imx_rproc_dcfg *dcfg = dsp_dcfg->dcfg;
313 struct device *dev = rproc->dev.parent;
314 int ret;
315
316 switch (dcfg->method) {
317 case IMX_RPROC_MMIO:
318 ret = regmap_update_bits(priv->regmap,
319 dcfg->src_reg,
320 dcfg->src_mask,
321 dcfg->src_start);
322 break;
323 case IMX_RPROC_SCU_API:
324 ret = imx_sc_pm_cpu_start(priv->ipc_handle,
325 IMX_SC_R_DSP,
326 true,
327 rproc->bootaddr);
328 break;
329 case IMX_RPROC_RESET_CONTROLLER:
330 ret = reset_control_deassert(priv->run_stall);
331 break;
332 default:
333 return -EOPNOTSUPP;
334 }
335
336 if (ret)
337 dev_err(dev, "Failed to enable remote core!\n");
338 else
339 ret = imx_dsp_rproc_ready(rproc);
340
341 return ret;
342 }
343
344 /*
345 * Stop function for rproc_ops
346 * It clears the REMOTE_IS_READY flags
347 */
imx_dsp_rproc_stop(struct rproc * rproc)348 static int imx_dsp_rproc_stop(struct rproc *rproc)
349 {
350 struct imx_dsp_rproc *priv = rproc->priv;
351 const struct imx_dsp_rproc_dcfg *dsp_dcfg = priv->dsp_dcfg;
352 const struct imx_rproc_dcfg *dcfg = dsp_dcfg->dcfg;
353 struct device *dev = rproc->dev.parent;
354 int ret = 0;
355
356 if (rproc->state == RPROC_CRASHED) {
357 priv->flags &= ~REMOTE_IS_READY;
358 return 0;
359 }
360
361 switch (dcfg->method) {
362 case IMX_RPROC_MMIO:
363 ret = regmap_update_bits(priv->regmap, dcfg->src_reg, dcfg->src_mask,
364 dcfg->src_stop);
365 break;
366 case IMX_RPROC_SCU_API:
367 ret = imx_sc_pm_cpu_start(priv->ipc_handle,
368 IMX_SC_R_DSP,
369 false,
370 rproc->bootaddr);
371 break;
372 case IMX_RPROC_RESET_CONTROLLER:
373 ret = reset_control_assert(priv->run_stall);
374 break;
375 default:
376 return -EOPNOTSUPP;
377 }
378
379 if (ret)
380 dev_err(dev, "Failed to stop remote core\n");
381 else
382 priv->flags &= ~REMOTE_IS_READY;
383
384 return ret;
385 }
386
387 /**
388 * imx_dsp_rproc_sys_to_da() - internal memory translation helper
389 * @priv: private data pointer
390 * @sys: system address (DDR address)
391 * @len: length of the memory buffer
392 * @da: device address to translate
393 *
394 * Convert system address (DDR address) to device address (DSP)
395 * for there may be memory remap for device.
396 */
imx_dsp_rproc_sys_to_da(struct imx_dsp_rproc * priv,u64 sys,size_t len,u64 * da)397 static int imx_dsp_rproc_sys_to_da(struct imx_dsp_rproc *priv, u64 sys,
398 size_t len, u64 *da)
399 {
400 const struct imx_dsp_rproc_dcfg *dsp_dcfg = priv->dsp_dcfg;
401 const struct imx_rproc_dcfg *dcfg = dsp_dcfg->dcfg;
402 int i;
403
404 /* Parse address translation table */
405 for (i = 0; i < dcfg->att_size; i++) {
406 const struct imx_rproc_att *att = &dcfg->att[i];
407
408 if (sys >= att->sa && sys + len <= att->sa + att->size) {
409 unsigned int offset = sys - att->sa;
410
411 *da = att->da + offset;
412 return 0;
413 }
414 }
415
416 return -ENOENT;
417 }
418
419 /* Main virtqueue message work function
420 *
421 * This function is executed upon scheduling of the i.MX DSP remoteproc
422 * driver's workqueue. The workqueue is scheduled by the mailbox rx
423 * handler.
424 *
425 * This work function processes both the Tx and Rx virtqueue indices on
426 * every invocation. The rproc_vq_interrupt function can detect if there
427 * are new unprocessed messages or not (returns IRQ_NONE vs IRQ_HANDLED),
428 * but there is no need to check for these return values. The index 0
429 * triggering will process all pending Rx buffers, and the index 1 triggering
430 * will process all newly available Tx buffers and will wakeup any potentially
431 * blocked senders.
432 *
433 * NOTE:
434 * The current logic is based on an inherent design assumption of supporting
435 * only 2 vrings, but this can be changed if needed.
436 */
imx_dsp_rproc_vq_work(struct work_struct * work)437 static void imx_dsp_rproc_vq_work(struct work_struct *work)
438 {
439 struct imx_dsp_rproc *priv = container_of(work, struct imx_dsp_rproc,
440 rproc_work);
441 struct rproc *rproc = priv->rproc;
442
443 mutex_lock(&rproc->lock);
444
445 if (rproc->state != RPROC_RUNNING)
446 goto unlock_mutex;
447
448 rproc_vq_interrupt(priv->rproc, 0);
449 rproc_vq_interrupt(priv->rproc, 1);
450
451 unlock_mutex:
452 mutex_unlock(&rproc->lock);
453 }
454
455 /**
456 * imx_dsp_rproc_rx_tx_callback() - inbound mailbox message handler
457 * @cl: mailbox client pointer used for requesting the mailbox channel
458 * @data: mailbox payload
459 *
460 * This handler is invoked by mailbox driver whenever a mailbox
461 * message is received. Usually, the SUSPEND and RESUME related messages
462 * are handled in this function, other messages are handled by remoteproc core
463 */
imx_dsp_rproc_rx_tx_callback(struct mbox_client * cl,void * data)464 static void imx_dsp_rproc_rx_tx_callback(struct mbox_client *cl, void *data)
465 {
466 struct rproc *rproc = dev_get_drvdata(cl->dev);
467 struct imx_dsp_rproc *priv = rproc->priv;
468 struct device *dev = rproc->dev.parent;
469 u32 message = (u32)(*(u32 *)data);
470
471 dev_dbg(dev, "mbox msg: 0x%x\n", message);
472
473 switch (message) {
474 case RP_MBOX_SUSPEND_ACK:
475 complete(&priv->pm_comp);
476 break;
477 case RP_MBOX_RESUME_ACK:
478 complete(&priv->pm_comp);
479 break;
480 default:
481 schedule_work(&priv->rproc_work);
482 break;
483 }
484 }
485
486 /**
487 * imx_dsp_rproc_rxdb_callback() - inbound mailbox message handler
488 * @cl: mailbox client pointer used for requesting the mailbox channel
489 * @data: mailbox payload
490 *
491 * For doorbell, there is no message specified, just set REMOTE_IS_READY
492 * flag.
493 */
imx_dsp_rproc_rxdb_callback(struct mbox_client * cl,void * data)494 static void imx_dsp_rproc_rxdb_callback(struct mbox_client *cl, void *data)
495 {
496 struct rproc *rproc = dev_get_drvdata(cl->dev);
497 struct imx_dsp_rproc *priv = rproc->priv;
498
499 /* Remote is ready after firmware is loaded and running */
500 priv->flags |= REMOTE_IS_READY;
501 }
502
503 /**
504 * imx_dsp_rproc_mbox_alloc() - request mailbox channels
505 * @priv: private data pointer
506 *
507 * Request three mailbox channels (tx, rx, rxdb).
508 */
imx_dsp_rproc_mbox_alloc(struct imx_dsp_rproc * priv)509 static int imx_dsp_rproc_mbox_alloc(struct imx_dsp_rproc *priv)
510 {
511 struct device *dev = priv->rproc->dev.parent;
512 struct mbox_client *cl;
513 int ret;
514
515 if (!of_property_present(dev->of_node, "mbox-names"))
516 return 0;
517
518 cl = &priv->cl;
519 cl->dev = dev;
520 cl->tx_block = true;
521 cl->tx_tout = 100;
522 cl->knows_txdone = false;
523 cl->rx_callback = imx_dsp_rproc_rx_tx_callback;
524
525 /* Channel for sending message */
526 priv->tx_ch = mbox_request_channel_byname(cl, "tx");
527 if (IS_ERR(priv->tx_ch)) {
528 ret = PTR_ERR(priv->tx_ch);
529 dev_dbg(cl->dev, "failed to request tx mailbox channel: %d\n",
530 ret);
531 return ret;
532 }
533
534 /* Channel for receiving message */
535 priv->rx_ch = mbox_request_channel_byname(cl, "rx");
536 if (IS_ERR(priv->rx_ch)) {
537 ret = PTR_ERR(priv->rx_ch);
538 dev_dbg(cl->dev, "failed to request rx mailbox channel: %d\n",
539 ret);
540 goto free_channel_tx;
541 }
542
543 cl = &priv->cl_rxdb;
544 cl->dev = dev;
545 cl->rx_callback = imx_dsp_rproc_rxdb_callback;
546
547 /*
548 * RX door bell is used to receive the ready signal from remote
549 * after firmware loaded.
550 */
551 priv->rxdb_ch = mbox_request_channel_byname(cl, "rxdb");
552 if (IS_ERR(priv->rxdb_ch)) {
553 ret = PTR_ERR(priv->rxdb_ch);
554 dev_dbg(cl->dev, "failed to request mbox chan rxdb, ret %d\n",
555 ret);
556 goto free_channel_rx;
557 }
558
559 return 0;
560
561 free_channel_rx:
562 mbox_free_channel(priv->rx_ch);
563 free_channel_tx:
564 mbox_free_channel(priv->tx_ch);
565 return ret;
566 }
567
568 /*
569 * imx_dsp_rproc_mbox_no_alloc()
570 *
571 * Empty function for no mailbox between cores
572 *
573 * Always return 0
574 */
imx_dsp_rproc_mbox_no_alloc(struct imx_dsp_rproc * priv)575 static int imx_dsp_rproc_mbox_no_alloc(struct imx_dsp_rproc *priv)
576 {
577 return 0;
578 }
579
imx_dsp_rproc_free_mbox(struct imx_dsp_rproc * priv)580 static void imx_dsp_rproc_free_mbox(struct imx_dsp_rproc *priv)
581 {
582 mbox_free_channel(priv->tx_ch);
583 mbox_free_channel(priv->rx_ch);
584 mbox_free_channel(priv->rxdb_ch);
585 }
586
587 /**
588 * imx_dsp_rproc_add_carveout() - request mailbox channels
589 * @priv: private data pointer
590 *
591 * This function registers specified memory entry in @rproc carveouts list
592 * The carveouts can help to mapping the memory address for DSP.
593 */
imx_dsp_rproc_add_carveout(struct imx_dsp_rproc * priv)594 static int imx_dsp_rproc_add_carveout(struct imx_dsp_rproc *priv)
595 {
596 const struct imx_dsp_rproc_dcfg *dsp_dcfg = priv->dsp_dcfg;
597 const struct imx_rproc_dcfg *dcfg = dsp_dcfg->dcfg;
598 struct rproc *rproc = priv->rproc;
599 struct device *dev = rproc->dev.parent;
600 struct device_node *np = dev->of_node;
601 struct of_phandle_iterator it;
602 struct rproc_mem_entry *mem;
603 struct reserved_mem *rmem;
604 void __iomem *cpu_addr;
605 int a;
606 u64 da;
607
608 /* Remap required addresses */
609 for (a = 0; a < dcfg->att_size; a++) {
610 const struct imx_rproc_att *att = &dcfg->att[a];
611
612 if (!(att->flags & ATT_OWN))
613 continue;
614
615 if (imx_dsp_rproc_sys_to_da(priv, att->sa, att->size, &da))
616 return -EINVAL;
617
618 cpu_addr = devm_ioremap_wc(dev, att->sa, att->size);
619 if (!cpu_addr) {
620 dev_err(dev, "failed to map memory %p\n", &att->sa);
621 return -ENOMEM;
622 }
623
624 /* Register memory region */
625 mem = rproc_mem_entry_init(dev, (void __force *)cpu_addr, (dma_addr_t)att->sa,
626 att->size, da, NULL, NULL, "dsp_mem");
627
628 if (mem)
629 rproc_coredump_add_segment(rproc, da, att->size);
630 else
631 return -ENOMEM;
632
633 rproc_add_carveout(rproc, mem);
634 }
635
636 of_phandle_iterator_init(&it, np, "memory-region", NULL, 0);
637 while (of_phandle_iterator_next(&it) == 0) {
638 /*
639 * Ignore the first memory region which will be used vdev buffer.
640 * No need to do extra handlings, rproc_add_virtio_dev will handle it.
641 */
642 if (!strcmp(it.node->name, "vdev0buffer"))
643 continue;
644
645 rmem = of_reserved_mem_lookup(it.node);
646 if (!rmem) {
647 of_node_put(it.node);
648 dev_err(dev, "unable to acquire memory-region\n");
649 return -EINVAL;
650 }
651
652 if (imx_dsp_rproc_sys_to_da(priv, rmem->base, rmem->size, &da)) {
653 of_node_put(it.node);
654 return -EINVAL;
655 }
656
657 cpu_addr = devm_ioremap_wc(dev, rmem->base, rmem->size);
658 if (!cpu_addr) {
659 of_node_put(it.node);
660 dev_err(dev, "failed to map memory %p\n", &rmem->base);
661 return -ENOMEM;
662 }
663
664 /* Register memory region */
665 mem = rproc_mem_entry_init(dev, (void __force *)cpu_addr, (dma_addr_t)rmem->base,
666 rmem->size, da, NULL, NULL, it.node->name);
667
668 if (mem) {
669 rproc_coredump_add_segment(rproc, da, rmem->size);
670 } else {
671 of_node_put(it.node);
672 return -ENOMEM;
673 }
674
675 rproc_add_carveout(rproc, mem);
676 }
677
678 return 0;
679 }
680
681 /* Prepare function for rproc_ops */
imx_dsp_rproc_prepare(struct rproc * rproc)682 static int imx_dsp_rproc_prepare(struct rproc *rproc)
683 {
684 struct imx_dsp_rproc *priv = rproc->priv;
685 struct device *dev = rproc->dev.parent;
686 struct rproc_mem_entry *carveout;
687 int ret;
688
689 ret = imx_dsp_rproc_add_carveout(priv);
690 if (ret) {
691 dev_err(dev, "failed on imx_dsp_rproc_add_carveout\n");
692 return ret;
693 }
694
695 pm_runtime_get_sync(dev);
696
697 /*
698 * Clear buffers after pm rumtime for internal ocram is not
699 * accessible if power and clock are not enabled.
700 */
701 list_for_each_entry(carveout, &rproc->carveouts, node) {
702 if (carveout->va)
703 memset(carveout->va, 0, carveout->len);
704 }
705
706 return 0;
707 }
708
709 /* Unprepare function for rproc_ops */
imx_dsp_rproc_unprepare(struct rproc * rproc)710 static int imx_dsp_rproc_unprepare(struct rproc *rproc)
711 {
712 pm_runtime_put_sync(rproc->dev.parent);
713
714 return 0;
715 }
716
717 /* Kick function for rproc_ops */
imx_dsp_rproc_kick(struct rproc * rproc,int vqid)718 static void imx_dsp_rproc_kick(struct rproc *rproc, int vqid)
719 {
720 struct imx_dsp_rproc *priv = rproc->priv;
721 struct device *dev = rproc->dev.parent;
722 int err;
723 __u32 mmsg;
724
725 if (!priv->tx_ch) {
726 dev_err(dev, "No initialized mbox tx channel\n");
727 return;
728 }
729
730 /*
731 * Send the index of the triggered virtqueue as the mu payload.
732 * Let remote processor know which virtqueue is used.
733 */
734 mmsg = vqid;
735
736 err = mbox_send_message(priv->tx_ch, (void *)&mmsg);
737 if (err < 0)
738 dev_err(dev, "%s: failed (%d, err:%d)\n", __func__, vqid, err);
739 }
740
741 /*
742 * Custom memory copy implementation for i.MX DSP Cores
743 *
744 * The IRAM is part of the HiFi DSP.
745 * According to hw specs only 32-bits writes are allowed.
746 */
imx_dsp_rproc_memcpy(void * dst,const void * src,size_t size)747 static int imx_dsp_rproc_memcpy(void *dst, const void *src, size_t size)
748 {
749 void __iomem *dest = (void __iomem *)dst;
750 const u8 *src_byte = src;
751 const u32 *source = src;
752 u32 affected_mask;
753 int i, q, r;
754 u32 tmp;
755
756 /* destination must be 32bit aligned */
757 if (!IS_ALIGNED((uintptr_t)dest, 4))
758 return -EINVAL;
759
760 q = size / 4;
761 r = size % 4;
762
763 /* copy data in units of 32 bits at a time */
764 for (i = 0; i < q; i++)
765 writel(source[i], dest + i * 4);
766
767 if (r) {
768 affected_mask = GENMASK(8 * r, 0);
769
770 /*
771 * first read the 32bit data of dest, then change affected
772 * bytes, and write back to dest.
773 * For unaffected bytes, it should not be changed
774 */
775 tmp = readl(dest + q * 4);
776 tmp &= ~affected_mask;
777
778 /* avoid reading after end of source */
779 for (i = 0; i < r; i++)
780 tmp |= (src_byte[q * 4 + i] << (8 * i));
781
782 writel(tmp, dest + q * 4);
783 }
784
785 return 0;
786 }
787
788 /*
789 * Custom memset implementation for i.MX DSP Cores
790 *
791 * The IRAM is part of the HiFi DSP.
792 * According to hw specs only 32-bits writes are allowed.
793 */
imx_dsp_rproc_memset(void * addr,u8 value,size_t size)794 static int imx_dsp_rproc_memset(void *addr, u8 value, size_t size)
795 {
796 void __iomem *tmp_dst = (void __iomem *)addr;
797 u32 tmp_val = value;
798 u32 affected_mask;
799 int q, r;
800 u32 tmp;
801
802 /* destination must be 32bit aligned */
803 if (!IS_ALIGNED((uintptr_t)addr, 4))
804 return -EINVAL;
805
806 tmp_val |= tmp_val << 8;
807 tmp_val |= tmp_val << 16;
808
809 q = size / 4;
810 r = size % 4;
811
812 while (q--)
813 writel(tmp_val, tmp_dst++);
814
815 if (r) {
816 affected_mask = GENMASK(8 * r, 0);
817
818 /*
819 * first read the 32bit data of addr, then change affected
820 * bytes, and write back to addr.
821 * For unaffected bytes, it should not be changed
822 */
823 tmp = readl(tmp_dst);
824 tmp &= ~affected_mask;
825
826 tmp |= (tmp_val & affected_mask);
827 writel(tmp, tmp_dst);
828 }
829
830 return 0;
831 }
832
833 /*
834 * imx_dsp_rproc_elf_load_segments() - load firmware segments to memory
835 * @rproc: remote processor which will be booted using these fw segments
836 * @fw: the ELF firmware image
837 *
838 * This function loads the firmware segments to memory, where the remote
839 * processor expects them.
840 *
841 * Return: 0 on success and an appropriate error code otherwise
842 */
imx_dsp_rproc_elf_load_segments(struct rproc * rproc,const struct firmware * fw)843 static int imx_dsp_rproc_elf_load_segments(struct rproc *rproc, const struct firmware *fw)
844 {
845 struct device *dev = &rproc->dev;
846 const void *ehdr, *phdr;
847 int i, ret = 0;
848 u16 phnum;
849 const u8 *elf_data = fw->data;
850 u8 class = fw_elf_get_class(fw);
851 u32 elf_phdr_get_size = elf_size_of_phdr(class);
852
853 ehdr = elf_data;
854 phnum = elf_hdr_get_e_phnum(class, ehdr);
855 phdr = elf_data + elf_hdr_get_e_phoff(class, ehdr);
856
857 /* go through the available ELF segments */
858 for (i = 0; i < phnum; i++, phdr += elf_phdr_get_size) {
859 u64 da = elf_phdr_get_p_paddr(class, phdr);
860 u64 memsz = elf_phdr_get_p_memsz(class, phdr);
861 u64 filesz = elf_phdr_get_p_filesz(class, phdr);
862 u64 offset = elf_phdr_get_p_offset(class, phdr);
863 u32 type = elf_phdr_get_p_type(class, phdr);
864 void *ptr;
865
866 if (type != PT_LOAD || !memsz)
867 continue;
868
869 dev_dbg(dev, "phdr: type %d da 0x%llx memsz 0x%llx filesz 0x%llx\n",
870 type, da, memsz, filesz);
871
872 if (filesz > memsz) {
873 dev_err(dev, "bad phdr filesz 0x%llx memsz 0x%llx\n",
874 filesz, memsz);
875 ret = -EINVAL;
876 break;
877 }
878
879 if (offset + filesz > fw->size) {
880 dev_err(dev, "truncated fw: need 0x%llx avail 0x%zx\n",
881 offset + filesz, fw->size);
882 ret = -EINVAL;
883 break;
884 }
885
886 if (!rproc_u64_fit_in_size_t(memsz)) {
887 dev_err(dev, "size (%llx) does not fit in size_t type\n",
888 memsz);
889 ret = -EOVERFLOW;
890 break;
891 }
892
893 /* grab the kernel address for this device address */
894 ptr = rproc_da_to_va(rproc, da, memsz, NULL);
895 if (!ptr) {
896 dev_err(dev, "bad phdr da 0x%llx mem 0x%llx\n", da,
897 memsz);
898 ret = -EINVAL;
899 break;
900 }
901
902 /* put the segment where the remote processor expects it */
903 if (filesz) {
904 ret = imx_dsp_rproc_memcpy(ptr, elf_data + offset, filesz);
905 if (ret) {
906 dev_err(dev, "memory copy failed for da 0x%llx memsz 0x%llx\n",
907 da, memsz);
908 break;
909 }
910 }
911
912 /* zero out remaining memory for this segment */
913 if (memsz > filesz) {
914 ret = imx_dsp_rproc_memset(ptr + filesz, 0, memsz - filesz);
915 if (ret) {
916 dev_err(dev, "memset failed for da 0x%llx memsz 0x%llx\n",
917 da, memsz);
918 break;
919 }
920 }
921 }
922
923 return ret;
924 }
925
imx_dsp_rproc_parse_fw(struct rproc * rproc,const struct firmware * fw)926 static int imx_dsp_rproc_parse_fw(struct rproc *rproc, const struct firmware *fw)
927 {
928 if (rproc_elf_load_rsc_table(rproc, fw))
929 dev_warn(&rproc->dev, "no resource table found for this firmware\n");
930
931 return 0;
932 }
933
934 static const struct rproc_ops imx_dsp_rproc_ops = {
935 .prepare = imx_dsp_rproc_prepare,
936 .unprepare = imx_dsp_rproc_unprepare,
937 .start = imx_dsp_rproc_start,
938 .stop = imx_dsp_rproc_stop,
939 .kick = imx_dsp_rproc_kick,
940 .load = imx_dsp_rproc_elf_load_segments,
941 .parse_fw = imx_dsp_rproc_parse_fw,
942 .find_loaded_rsc_table = rproc_elf_find_loaded_rsc_table,
943 .sanity_check = rproc_elf_sanity_check,
944 .get_boot_addr = rproc_elf_get_boot_addr,
945 };
946
947 /**
948 * imx_dsp_attach_pm_domains() - attach the power domains
949 * @priv: private data pointer
950 *
951 * On i.MX8QM and i.MX8QXP there is multiple power domains
952 * required, so need to link them.
953 */
imx_dsp_attach_pm_domains(struct imx_dsp_rproc * priv)954 static int imx_dsp_attach_pm_domains(struct imx_dsp_rproc *priv)
955 {
956 struct device *dev = priv->rproc->dev.parent;
957 int ret;
958
959 /* A single PM domain is already attached. */
960 if (dev->pm_domain)
961 return 0;
962
963 ret = dev_pm_domain_attach_list(dev, NULL, &priv->pd_list);
964 return ret < 0 ? ret : 0;
965 }
966
967 /**
968 * imx_dsp_rproc_detect_mode() - detect DSP control mode
969 * @priv: private data pointer
970 *
971 * Different platform has different control method for DSP, which depends
972 * on how the DSP is integrated in platform.
973 *
974 * For i.MX8QXP and i.MX8QM, DSP should be started and stopped by System
975 * Control Unit.
976 * For i.MX8MP and i.MX8ULP, DSP should be started and stopped by system
977 * integration module.
978 */
imx_dsp_rproc_detect_mode(struct imx_dsp_rproc * priv)979 static int imx_dsp_rproc_detect_mode(struct imx_dsp_rproc *priv)
980 {
981 const struct imx_dsp_rproc_dcfg *dsp_dcfg = priv->dsp_dcfg;
982 struct device *dev = priv->rproc->dev.parent;
983 struct regmap *regmap;
984 int ret = 0;
985
986 switch (dsp_dcfg->dcfg->method) {
987 case IMX_RPROC_SCU_API:
988 ret = imx_scu_get_handle(&priv->ipc_handle);
989 if (ret)
990 return ret;
991 break;
992 case IMX_RPROC_MMIO:
993 regmap = syscon_regmap_lookup_by_phandle(dev->of_node, "fsl,dsp-ctrl");
994 if (IS_ERR(regmap)) {
995 dev_err(dev, "failed to find syscon\n");
996 return PTR_ERR(regmap);
997 }
998
999 priv->regmap = regmap;
1000 break;
1001 case IMX_RPROC_RESET_CONTROLLER:
1002 priv->run_stall = devm_reset_control_get_exclusive(dev, "runstall");
1003 if (IS_ERR(priv->run_stall)) {
1004 dev_err(dev, "Failed to get DSP runstall reset control\n");
1005 return PTR_ERR(priv->run_stall);
1006 }
1007 break;
1008 default:
1009 ret = -EOPNOTSUPP;
1010 break;
1011 }
1012
1013 return ret;
1014 }
1015
1016 static const char *imx_dsp_clks_names[DSP_RPROC_CLK_MAX] = {
1017 /* DSP clocks */
1018 "core", "ocram", "debug", "ipg", "mu",
1019 };
1020
imx_dsp_rproc_clk_get(struct imx_dsp_rproc * priv)1021 static int imx_dsp_rproc_clk_get(struct imx_dsp_rproc *priv)
1022 {
1023 struct device *dev = priv->rproc->dev.parent;
1024 struct clk_bulk_data *clks = priv->clks;
1025 int i;
1026
1027 for (i = 0; i < DSP_RPROC_CLK_MAX; i++)
1028 clks[i].id = imx_dsp_clks_names[i];
1029
1030 return devm_clk_bulk_get_optional(dev, DSP_RPROC_CLK_MAX, clks);
1031 }
1032
imx_dsp_rproc_probe(struct platform_device * pdev)1033 static int imx_dsp_rproc_probe(struct platform_device *pdev)
1034 {
1035 const struct imx_dsp_rproc_dcfg *dsp_dcfg;
1036 struct device *dev = &pdev->dev;
1037 struct imx_dsp_rproc *priv;
1038 struct rproc *rproc;
1039 const char *fw_name;
1040 int ret;
1041
1042 dsp_dcfg = of_device_get_match_data(dev);
1043 if (!dsp_dcfg)
1044 return -ENODEV;
1045
1046 ret = rproc_of_parse_firmware(dev, 0, &fw_name);
1047 if (ret) {
1048 dev_err(dev, "failed to parse firmware-name property, ret = %d\n",
1049 ret);
1050 return ret;
1051 }
1052
1053 rproc = devm_rproc_alloc(dev, "imx-dsp-rproc", &imx_dsp_rproc_ops,
1054 fw_name, sizeof(*priv));
1055 if (!rproc)
1056 return -ENOMEM;
1057
1058 priv = rproc->priv;
1059 priv->rproc = rproc;
1060 priv->dsp_dcfg = dsp_dcfg;
1061
1062 if (no_mailboxes)
1063 imx_dsp_rproc_mbox_init = imx_dsp_rproc_mbox_no_alloc;
1064 else
1065 imx_dsp_rproc_mbox_init = imx_dsp_rproc_mbox_alloc;
1066
1067 dev_set_drvdata(dev, rproc);
1068
1069 INIT_WORK(&priv->rproc_work, imx_dsp_rproc_vq_work);
1070
1071 ret = imx_dsp_rproc_detect_mode(priv);
1072 if (ret) {
1073 dev_err(dev, "failed on imx_dsp_rproc_detect_mode\n");
1074 return ret;
1075 }
1076
1077 /* There are multiple power domains required by DSP on some platform */
1078 ret = imx_dsp_attach_pm_domains(priv);
1079 if (ret) {
1080 dev_err(dev, "failed on imx_dsp_attach_pm_domains\n");
1081 return ret;
1082 }
1083 /* Get clocks */
1084 ret = imx_dsp_rproc_clk_get(priv);
1085 if (ret) {
1086 dev_err(dev, "failed on imx_dsp_rproc_clk_get\n");
1087 goto err_detach_domains;
1088 }
1089
1090 init_completion(&priv->pm_comp);
1091 rproc->auto_boot = false;
1092 ret = rproc_add(rproc);
1093 if (ret) {
1094 dev_err(dev, "rproc_add failed\n");
1095 goto err_detach_domains;
1096 }
1097
1098 pm_runtime_enable(dev);
1099
1100 return 0;
1101
1102 err_detach_domains:
1103 dev_pm_domain_detach_list(priv->pd_list);
1104
1105 return ret;
1106 }
1107
imx_dsp_rproc_remove(struct platform_device * pdev)1108 static void imx_dsp_rproc_remove(struct platform_device *pdev)
1109 {
1110 struct rproc *rproc = platform_get_drvdata(pdev);
1111 struct imx_dsp_rproc *priv = rproc->priv;
1112
1113 pm_runtime_disable(&pdev->dev);
1114 rproc_del(rproc);
1115 dev_pm_domain_detach_list(priv->pd_list);
1116 }
1117
1118 /* pm runtime functions */
imx_dsp_runtime_resume(struct device * dev)1119 static int imx_dsp_runtime_resume(struct device *dev)
1120 {
1121 struct rproc *rproc = dev_get_drvdata(dev);
1122 struct imx_dsp_rproc *priv = rproc->priv;
1123 const struct imx_dsp_rproc_dcfg *dsp_dcfg = priv->dsp_dcfg;
1124 int ret;
1125
1126 /*
1127 * There is power domain attached with mailbox, if setup mailbox
1128 * in probe(), then the power of mailbox is always enabled,
1129 * the power can't be saved.
1130 * So move setup of mailbox to runtime resume.
1131 */
1132 ret = imx_dsp_rproc_mbox_init(priv);
1133 if (ret) {
1134 dev_err(dev, "failed on imx_dsp_rproc_mbox_init\n");
1135 return ret;
1136 }
1137
1138 ret = clk_bulk_prepare_enable(DSP_RPROC_CLK_MAX, priv->clks);
1139 if (ret) {
1140 dev_err(dev, "failed on clk_bulk_prepare_enable\n");
1141 return ret;
1142 }
1143
1144 /* Reset DSP if needed */
1145 if (dsp_dcfg->reset)
1146 dsp_dcfg->reset(priv);
1147
1148 return 0;
1149 }
1150
imx_dsp_runtime_suspend(struct device * dev)1151 static int imx_dsp_runtime_suspend(struct device *dev)
1152 {
1153 struct rproc *rproc = dev_get_drvdata(dev);
1154 struct imx_dsp_rproc *priv = rproc->priv;
1155
1156 clk_bulk_disable_unprepare(DSP_RPROC_CLK_MAX, priv->clks);
1157
1158 imx_dsp_rproc_free_mbox(priv);
1159
1160 return 0;
1161 }
1162
imx_dsp_load_firmware(const struct firmware * fw,void * context)1163 static void imx_dsp_load_firmware(const struct firmware *fw, void *context)
1164 {
1165 struct rproc *rproc = context;
1166 int ret;
1167
1168 /*
1169 * Same flow as start procedure.
1170 * Load the ELF segments to memory firstly.
1171 */
1172 ret = rproc_load_segments(rproc, fw);
1173 if (ret)
1174 goto out;
1175
1176 /* Start the remote processor */
1177 ret = rproc->ops->start(rproc);
1178 if (ret)
1179 goto out;
1180
1181 rproc->ops->kick(rproc, 0);
1182
1183 out:
1184 release_firmware(fw);
1185 }
1186
imx_dsp_suspend(struct device * dev)1187 static int imx_dsp_suspend(struct device *dev)
1188 {
1189 struct rproc *rproc = dev_get_drvdata(dev);
1190 struct imx_dsp_rproc *priv = rproc->priv;
1191 __u32 mmsg = RP_MBOX_SUSPEND_SYSTEM;
1192 int ret;
1193
1194 if (rproc->state != RPROC_RUNNING)
1195 goto out;
1196
1197 reinit_completion(&priv->pm_comp);
1198
1199 /* Tell DSP that suspend is happening */
1200 ret = mbox_send_message(priv->tx_ch, (void *)&mmsg);
1201 if (ret < 0) {
1202 dev_err(dev, "PM mbox_send_message failed: %d\n", ret);
1203 return ret;
1204 }
1205
1206 /*
1207 * DSP need to save the context at suspend.
1208 * Here waiting the response for DSP, then power can be disabled.
1209 */
1210 if (!wait_for_completion_timeout(&priv->pm_comp, msecs_to_jiffies(100)))
1211 return -EBUSY;
1212
1213 out:
1214 /*
1215 * The power of DSP is disabled in suspend, so force pm runtime
1216 * to be suspend, then we can reenable the power and clocks at
1217 * resume stage.
1218 */
1219 return pm_runtime_force_suspend(dev);
1220 }
1221
imx_dsp_resume(struct device * dev)1222 static int imx_dsp_resume(struct device *dev)
1223 {
1224 struct rproc *rproc = dev_get_drvdata(dev);
1225 int ret = 0;
1226
1227 ret = pm_runtime_force_resume(dev);
1228 if (ret)
1229 return ret;
1230
1231 if (rproc->state != RPROC_RUNNING)
1232 return 0;
1233
1234 /*
1235 * The power of DSP is disabled at suspend, the memory of dsp
1236 * is reset, the image segments are lost. So need to reload
1237 * firmware and restart the DSP if it is in running state.
1238 */
1239 ret = request_firmware_nowait(THIS_MODULE, FW_ACTION_UEVENT,
1240 rproc->firmware, dev, GFP_KERNEL,
1241 rproc, imx_dsp_load_firmware);
1242 if (ret < 0) {
1243 dev_err(dev, "load firmware failed: %d\n", ret);
1244 goto err;
1245 }
1246
1247 return 0;
1248
1249 err:
1250 pm_runtime_force_suspend(dev);
1251
1252 return ret;
1253 }
1254
1255 static const struct dev_pm_ops imx_dsp_rproc_pm_ops = {
1256 SYSTEM_SLEEP_PM_OPS(imx_dsp_suspend, imx_dsp_resume)
1257 RUNTIME_PM_OPS(imx_dsp_runtime_suspend, imx_dsp_runtime_resume, NULL)
1258 };
1259
1260 static const struct of_device_id imx_dsp_rproc_of_match[] = {
1261 { .compatible = "fsl,imx8qxp-hifi4", .data = &imx_dsp_rproc_cfg_imx8qxp },
1262 { .compatible = "fsl,imx8qm-hifi4", .data = &imx_dsp_rproc_cfg_imx8qm },
1263 { .compatible = "fsl,imx8mp-hifi4", .data = &imx_dsp_rproc_cfg_imx8mp },
1264 { .compatible = "fsl,imx8ulp-hifi4", .data = &imx_dsp_rproc_cfg_imx8ulp },
1265 {},
1266 };
1267 MODULE_DEVICE_TABLE(of, imx_dsp_rproc_of_match);
1268
1269 static struct platform_driver imx_dsp_rproc_driver = {
1270 .probe = imx_dsp_rproc_probe,
1271 .remove = imx_dsp_rproc_remove,
1272 .driver = {
1273 .name = "imx-dsp-rproc",
1274 .of_match_table = imx_dsp_rproc_of_match,
1275 .pm = pm_ptr(&imx_dsp_rproc_pm_ops),
1276 },
1277 };
1278 module_platform_driver(imx_dsp_rproc_driver);
1279
1280 MODULE_LICENSE("GPL v2");
1281 MODULE_DESCRIPTION("i.MX HiFi Core Remote Processor Control Driver");
1282 MODULE_AUTHOR("Shengjiu Wang <shengjiu.wang@nxp.com>");
1283