1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2012-2016, The Linux Foundation. All rights reserved.
4 * Copyright (C) 2017 Linaro Ltd.
5 */
6
7 #include <linux/delay.h>
8 #include <linux/device.h>
9 #include <linux/dma-mapping.h>
10 #include <linux/interrupt.h>
11 #include <linux/iopoll.h>
12 #include <linux/kernel.h>
13 #include <linux/slab.h>
14
15 #include "core.h"
16 #include "hfi_cmds.h"
17 #include "hfi_msgs.h"
18 #include "hfi_venus.h"
19 #include "hfi_venus_io.h"
20 #include "firmware.h"
21
22 #define HFI_MASK_QHDR_TX_TYPE 0xff000000
23 #define HFI_MASK_QHDR_RX_TYPE 0x00ff0000
24 #define HFI_MASK_QHDR_PRI_TYPE 0x0000ff00
25 #define HFI_MASK_QHDR_ID_TYPE 0x000000ff
26
27 #define HFI_HOST_TO_CTRL_CMD_Q 0
28 #define HFI_CTRL_TO_HOST_MSG_Q 1
29 #define HFI_CTRL_TO_HOST_DBG_Q 2
30 #define HFI_MASK_QHDR_STATUS 0x000000ff
31
32 #define IFACEQ_NUM 3
33 #define IFACEQ_CMD_IDX 0
34 #define IFACEQ_MSG_IDX 1
35 #define IFACEQ_DBG_IDX 2
36 #define IFACEQ_MAX_BUF_COUNT 50
37 #define IFACEQ_MAX_PARALLEL_CLNTS 16
38 #define IFACEQ_DFLT_QHDR 0x01010000
39
40 #define POLL_INTERVAL_US 50
41
42 #define IFACEQ_MAX_PKT_SIZE 1024
43 #define IFACEQ_MED_PKT_SIZE 768
44 #define IFACEQ_MIN_PKT_SIZE 8
45 #define IFACEQ_VAR_SMALL_PKT_SIZE 100
46 #define IFACEQ_VAR_LARGE_PKT_SIZE 512
47 #define IFACEQ_VAR_HUGE_PKT_SIZE (1024 * 12)
48
49 struct hfi_queue_table_header {
50 u32 version;
51 u32 size;
52 u32 qhdr0_offset;
53 u32 qhdr_size;
54 u32 num_q;
55 u32 num_active_q;
56 };
57
58 struct hfi_queue_header {
59 u32 status;
60 u32 start_addr;
61 u32 type;
62 u32 q_size;
63 u32 pkt_size;
64 u32 pkt_drop_cnt;
65 u32 rx_wm;
66 u32 tx_wm;
67 u32 rx_req;
68 u32 tx_req;
69 u32 rx_irq_status;
70 u32 tx_irq_status;
71 u32 read_idx;
72 u32 write_idx;
73 };
74
75 #define IFACEQ_TABLE_SIZE \
76 (sizeof(struct hfi_queue_table_header) + \
77 sizeof(struct hfi_queue_header) * IFACEQ_NUM)
78
79 #define IFACEQ_QUEUE_SIZE (IFACEQ_MAX_PKT_SIZE * \
80 IFACEQ_MAX_BUF_COUNT * IFACEQ_MAX_PARALLEL_CLNTS)
81
82 #define IFACEQ_GET_QHDR_START_ADDR(ptr, i) \
83 (void *)(((ptr) + sizeof(struct hfi_queue_table_header)) + \
84 ((i) * sizeof(struct hfi_queue_header)))
85
86 #define QDSS_SIZE SZ_4K
87 #define SFR_SIZE SZ_4K
88 #define QUEUE_SIZE \
89 (IFACEQ_TABLE_SIZE + (IFACEQ_QUEUE_SIZE * IFACEQ_NUM))
90
91 #define ALIGNED_QDSS_SIZE ALIGN(QDSS_SIZE, SZ_4K)
92 #define ALIGNED_SFR_SIZE ALIGN(SFR_SIZE, SZ_4K)
93 #define ALIGNED_QUEUE_SIZE ALIGN(QUEUE_SIZE, SZ_4K)
94 #define SHARED_QSIZE ALIGN(ALIGNED_SFR_SIZE + ALIGNED_QUEUE_SIZE + \
95 ALIGNED_QDSS_SIZE, SZ_1M)
96
97 struct mem_desc {
98 dma_addr_t da; /* device address */
99 void *kva; /* kernel virtual address */
100 u32 size;
101 unsigned long attrs;
102 };
103
104 struct iface_queue {
105 struct hfi_queue_header *qhdr;
106 struct mem_desc qmem;
107 };
108
109 enum venus_state {
110 VENUS_STATE_DEINIT = 1,
111 VENUS_STATE_INIT,
112 };
113
114 struct venus_hfi_device {
115 struct venus_core *core;
116 u32 irq_status;
117 u32 last_packet_type;
118 bool power_enabled;
119 bool suspended;
120 enum venus_state state;
121 /* serialize read / write to the shared memory */
122 struct mutex lock;
123 struct completion pwr_collapse_prep;
124 struct completion release_resource;
125 struct mem_desc ifaceq_table;
126 struct mem_desc sfr;
127 struct iface_queue queues[IFACEQ_NUM];
128 u8 pkt_buf[IFACEQ_VAR_HUGE_PKT_SIZE];
129 u8 dbg_buf[IFACEQ_VAR_HUGE_PKT_SIZE];
130 };
131
132 static bool venus_pkt_debug;
133 int venus_fw_debug = HFI_DEBUG_MSG_ERROR | HFI_DEBUG_MSG_FATAL;
134 static bool venus_fw_low_power_mode = true;
135 static int venus_hw_rsp_timeout = 1000;
136 static bool venus_fw_coverage;
137
venus_set_state(struct venus_hfi_device * hdev,enum venus_state state)138 static void venus_set_state(struct venus_hfi_device *hdev,
139 enum venus_state state)
140 {
141 mutex_lock(&hdev->lock);
142 hdev->state = state;
143 mutex_unlock(&hdev->lock);
144 }
145
venus_is_valid_state(struct venus_hfi_device * hdev)146 static bool venus_is_valid_state(struct venus_hfi_device *hdev)
147 {
148 return hdev->state != VENUS_STATE_DEINIT;
149 }
150
venus_dump_packet(struct venus_hfi_device * hdev,const void * packet)151 static void venus_dump_packet(struct venus_hfi_device *hdev, const void *packet)
152 {
153 size_t pkt_size = *(u32 *)packet;
154
155 if (!venus_pkt_debug)
156 return;
157
158 print_hex_dump(KERN_DEBUG, "", DUMP_PREFIX_OFFSET, 16, 1, packet,
159 pkt_size, true);
160 }
161
venus_write_queue(struct venus_hfi_device * hdev,struct iface_queue * queue,void * packet,u32 * rx_req)162 static int venus_write_queue(struct venus_hfi_device *hdev,
163 struct iface_queue *queue,
164 void *packet, u32 *rx_req)
165 {
166 struct hfi_queue_header *qhdr;
167 u32 dwords, new_wr_idx;
168 u32 empty_space, rd_idx, wr_idx, qsize;
169 u32 *wr_ptr;
170
171 if (!queue->qmem.kva)
172 return -EINVAL;
173
174 qhdr = queue->qhdr;
175 if (!qhdr)
176 return -EINVAL;
177
178 venus_dump_packet(hdev, packet);
179
180 dwords = (*(u32 *)packet) >> 2;
181 if (!dwords)
182 return -EINVAL;
183
184 rd_idx = qhdr->read_idx;
185 wr_idx = qhdr->write_idx;
186 qsize = qhdr->q_size;
187 /* ensure rd/wr indices's are read from memory */
188 rmb();
189
190 if (qsize > IFACEQ_QUEUE_SIZE / 4)
191 return -EINVAL;
192
193 if (wr_idx >= rd_idx)
194 empty_space = qsize - (wr_idx - rd_idx);
195 else
196 empty_space = rd_idx - wr_idx;
197
198 if (empty_space <= dwords) {
199 qhdr->tx_req = 1;
200 /* ensure tx_req is updated in memory */
201 wmb();
202 return -ENOSPC;
203 }
204
205 qhdr->tx_req = 0;
206 /* ensure tx_req is updated in memory */
207 wmb();
208
209 new_wr_idx = wr_idx + dwords;
210 wr_ptr = (u32 *)(queue->qmem.kva + (wr_idx << 2));
211
212 if (wr_ptr < (u32 *)queue->qmem.kva ||
213 wr_ptr > (u32 *)(queue->qmem.kva + queue->qmem.size - sizeof(*wr_ptr)))
214 return -EINVAL;
215
216 if (new_wr_idx < qsize) {
217 memcpy(wr_ptr, packet, dwords << 2);
218 } else {
219 size_t len;
220
221 new_wr_idx -= qsize;
222 len = (dwords - new_wr_idx) << 2;
223 memcpy(wr_ptr, packet, len);
224 memcpy(queue->qmem.kva, packet + len, new_wr_idx << 2);
225 }
226
227 /* make sure packet is written before updating the write index */
228 wmb();
229
230 qhdr->write_idx = new_wr_idx;
231 *rx_req = qhdr->rx_req ? 1 : 0;
232
233 /* make sure write index is updated before an interrupt is raised */
234 mb();
235
236 return 0;
237 }
238
venus_read_queue(struct venus_hfi_device * hdev,struct iface_queue * queue,void * pkt,u32 * tx_req)239 static int venus_read_queue(struct venus_hfi_device *hdev,
240 struct iface_queue *queue, void *pkt, u32 *tx_req)
241 {
242 struct hfi_pkt_hdr *pkt_hdr = NULL;
243 struct hfi_queue_header *qhdr;
244 u32 dwords, new_rd_idx;
245 u32 rd_idx, wr_idx, type, qsize;
246 u32 *rd_ptr;
247 u32 recv_request = 0;
248 int ret = 0;
249
250 if (!queue->qmem.kva)
251 return -EINVAL;
252
253 qhdr = queue->qhdr;
254 if (!qhdr)
255 return -EINVAL;
256
257 type = qhdr->type;
258 rd_idx = qhdr->read_idx;
259 wr_idx = qhdr->write_idx;
260 qsize = qhdr->q_size;
261
262 if (qsize > IFACEQ_QUEUE_SIZE / 4)
263 return -EINVAL;
264
265 /* make sure data is valid before using it */
266 rmb();
267
268 /*
269 * Do not set receive request for debug queue, if set, Venus generates
270 * interrupt for debug messages even when there is no response message
271 * available. In general debug queue will not become full as it is being
272 * emptied out for every interrupt from Venus. Venus will anyway
273 * generates interrupt if it is full.
274 */
275 if (type & HFI_CTRL_TO_HOST_MSG_Q)
276 recv_request = 1;
277
278 if (rd_idx == wr_idx) {
279 qhdr->rx_req = recv_request;
280 *tx_req = 0;
281 /* update rx_req field in memory */
282 wmb();
283 return -ENODATA;
284 }
285
286 rd_ptr = (u32 *)(queue->qmem.kva + (rd_idx << 2));
287
288 if (rd_ptr < (u32 *)queue->qmem.kva ||
289 rd_ptr > (u32 *)(queue->qmem.kva + queue->qmem.size - sizeof(*rd_ptr)))
290 return -EINVAL;
291
292 dwords = *rd_ptr >> 2;
293 if (!dwords)
294 return -EINVAL;
295
296 new_rd_idx = rd_idx + dwords;
297 if (((dwords << 2) <= IFACEQ_VAR_HUGE_PKT_SIZE) && rd_idx <= qsize) {
298 if (new_rd_idx < qsize) {
299 memcpy(pkt, rd_ptr, dwords << 2);
300 } else {
301 size_t len;
302
303 new_rd_idx -= qsize;
304 len = (dwords - new_rd_idx) << 2;
305 memcpy(pkt, rd_ptr, len);
306 memcpy(pkt + len, queue->qmem.kva, new_rd_idx << 2);
307 }
308 pkt_hdr = (struct hfi_pkt_hdr *)(pkt);
309 if ((pkt_hdr->size >> 2) != dwords)
310 return -EINVAL;
311 } else {
312 /* bad packet received, dropping */
313 new_rd_idx = qhdr->write_idx;
314 ret = -EBADMSG;
315 }
316
317 /* ensure the packet is read before updating read index */
318 rmb();
319
320 qhdr->read_idx = new_rd_idx;
321 /* ensure updating read index */
322 wmb();
323
324 rd_idx = qhdr->read_idx;
325 wr_idx = qhdr->write_idx;
326 /* ensure rd/wr indices are read from memory */
327 rmb();
328
329 if (rd_idx != wr_idx)
330 qhdr->rx_req = 0;
331 else
332 qhdr->rx_req = recv_request;
333
334 *tx_req = qhdr->tx_req ? 1 : 0;
335
336 /* ensure rx_req is stored to memory and tx_req is loaded from memory */
337 mb();
338
339 venus_dump_packet(hdev, pkt);
340
341 return ret;
342 }
343
venus_alloc(struct venus_hfi_device * hdev,struct mem_desc * desc,u32 size)344 static int venus_alloc(struct venus_hfi_device *hdev, struct mem_desc *desc,
345 u32 size)
346 {
347 struct device *dev = hdev->core->dev;
348
349 desc->attrs = DMA_ATTR_WRITE_COMBINE;
350 desc->size = ALIGN(size, SZ_4K);
351
352 desc->kva = dma_alloc_attrs(dev, desc->size, &desc->da, GFP_KERNEL,
353 desc->attrs);
354 if (!desc->kva)
355 return -ENOMEM;
356
357 return 0;
358 }
359
venus_free(struct venus_hfi_device * hdev,struct mem_desc * mem)360 static void venus_free(struct venus_hfi_device *hdev, struct mem_desc *mem)
361 {
362 struct device *dev = hdev->core->dev;
363
364 dma_free_attrs(dev, mem->size, mem->kva, mem->da, mem->attrs);
365 }
366
venus_set_registers(struct venus_hfi_device * hdev)367 static void venus_set_registers(struct venus_hfi_device *hdev)
368 {
369 const struct venus_resources *res = hdev->core->res;
370 const struct reg_val *tbl = res->reg_tbl;
371 unsigned int count = res->reg_tbl_size;
372 unsigned int i;
373
374 for (i = 0; i < count; i++)
375 writel(tbl[i].value, hdev->core->base + tbl[i].reg);
376 }
377
venus_soft_int(struct venus_hfi_device * hdev)378 static void venus_soft_int(struct venus_hfi_device *hdev)
379 {
380 void __iomem *cpu_ic_base = hdev->core->cpu_ic_base;
381 u32 clear_bit;
382
383 if (IS_V6(hdev->core))
384 clear_bit = BIT(CPU_IC_SOFTINT_H2A_SHIFT_V6);
385 else
386 clear_bit = BIT(CPU_IC_SOFTINT_H2A_SHIFT);
387
388 writel(clear_bit, cpu_ic_base + CPU_IC_SOFTINT);
389 }
390
venus_iface_cmdq_write_nolock(struct venus_hfi_device * hdev,void * pkt,bool sync)391 static int venus_iface_cmdq_write_nolock(struct venus_hfi_device *hdev,
392 void *pkt, bool sync)
393 {
394 struct device *dev = hdev->core->dev;
395 struct hfi_pkt_hdr *cmd_packet;
396 struct iface_queue *queue;
397 u32 rx_req;
398 int ret;
399
400 if (!venus_is_valid_state(hdev))
401 return -EINVAL;
402
403 cmd_packet = (struct hfi_pkt_hdr *)pkt;
404 hdev->last_packet_type = cmd_packet->pkt_type;
405
406 queue = &hdev->queues[IFACEQ_CMD_IDX];
407
408 ret = venus_write_queue(hdev, queue, pkt, &rx_req);
409 if (ret) {
410 dev_err(dev, "write to iface cmd queue failed (%d)\n", ret);
411 return ret;
412 }
413
414 if (sync) {
415 /*
416 * Inform video hardware to raise interrupt for synchronous
417 * commands
418 */
419 queue = &hdev->queues[IFACEQ_MSG_IDX];
420 queue->qhdr->rx_req = 1;
421 /* ensure rx_req is updated in memory */
422 wmb();
423 }
424
425 if (rx_req)
426 venus_soft_int(hdev);
427
428 return 0;
429 }
430
venus_iface_cmdq_write(struct venus_hfi_device * hdev,void * pkt,bool sync)431 static int venus_iface_cmdq_write(struct venus_hfi_device *hdev, void *pkt, bool sync)
432 {
433 int ret;
434
435 mutex_lock(&hdev->lock);
436 ret = venus_iface_cmdq_write_nolock(hdev, pkt, sync);
437 mutex_unlock(&hdev->lock);
438
439 return ret;
440 }
441
venus_hfi_core_set_resource(struct venus_core * core,u32 id,u32 size,u32 addr,void * cookie)442 static int venus_hfi_core_set_resource(struct venus_core *core, u32 id,
443 u32 size, u32 addr, void *cookie)
444 {
445 struct venus_hfi_device *hdev = to_hfi_priv(core);
446 struct hfi_sys_set_resource_pkt *pkt;
447 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
448 int ret;
449
450 if (id == VIDC_RESOURCE_NONE)
451 return 0;
452
453 pkt = (struct hfi_sys_set_resource_pkt *)packet;
454
455 ret = pkt_sys_set_resource(pkt, id, size, addr, cookie);
456 if (ret)
457 return ret;
458
459 ret = venus_iface_cmdq_write(hdev, pkt, false);
460 if (ret)
461 return ret;
462
463 return 0;
464 }
465
venus_boot_core(struct venus_hfi_device * hdev)466 static int venus_boot_core(struct venus_hfi_device *hdev)
467 {
468 struct device *dev = hdev->core->dev;
469 static const unsigned int max_tries = 100;
470 u32 ctrl_status = 0, mask_val = 0;
471 unsigned int count = 0;
472 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
473 void __iomem *wrapper_base = hdev->core->wrapper_base;
474 int ret = 0;
475
476 if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core)) {
477 mask_val = readl(wrapper_base + WRAPPER_INTR_MASK);
478 mask_val &= ~(WRAPPER_INTR_MASK_A2HWD_BASK_V6 |
479 WRAPPER_INTR_MASK_A2HCPU_MASK);
480 } else {
481 mask_val = WRAPPER_INTR_MASK_A2HVCODEC_MASK;
482 }
483
484 writel(mask_val, wrapper_base + WRAPPER_INTR_MASK);
485 if (IS_V1(hdev->core))
486 writel(1, cpu_cs_base + CPU_CS_SCIACMDARG3);
487
488 writel(BIT(VIDC_CTRL_INIT_CTRL_SHIFT), cpu_cs_base + VIDC_CTRL_INIT);
489 while (!ctrl_status && count < max_tries) {
490 ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
491 if ((ctrl_status & CPU_CS_SCIACMDARG0_ERROR_STATUS_MASK) == 4) {
492 dev_err(dev, "invalid setting for UC_REGION\n");
493 ret = -EINVAL;
494 break;
495 }
496
497 usleep_range(500, 1000);
498 count++;
499 }
500
501 if (count >= max_tries)
502 ret = -ETIMEDOUT;
503
504 if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core)) {
505 writel(0x1, cpu_cs_base + CPU_CS_H2XSOFTINTEN_V6);
506 writel(0x0, cpu_cs_base + CPU_CS_X2RPMH_V6);
507 }
508
509 return ret;
510 }
511
venus_hwversion(struct venus_hfi_device * hdev)512 static u32 venus_hwversion(struct venus_hfi_device *hdev)
513 {
514 struct device *dev = hdev->core->dev;
515 void __iomem *wrapper_base = hdev->core->wrapper_base;
516 u32 ver;
517 u32 major, minor, step;
518
519 ver = readl(wrapper_base + WRAPPER_HW_VERSION);
520 major = ver & WRAPPER_HW_VERSION_MAJOR_VERSION_MASK;
521 major = major >> WRAPPER_HW_VERSION_MAJOR_VERSION_SHIFT;
522 minor = ver & WRAPPER_HW_VERSION_MINOR_VERSION_MASK;
523 minor = minor >> WRAPPER_HW_VERSION_MINOR_VERSION_SHIFT;
524 step = ver & WRAPPER_HW_VERSION_STEP_VERSION_MASK;
525
526 dev_dbg(dev, VDBGL "venus hw version %x.%x.%x\n", major, minor, step);
527
528 return major;
529 }
530
venus_run(struct venus_hfi_device * hdev)531 static int venus_run(struct venus_hfi_device *hdev)
532 {
533 struct device *dev = hdev->core->dev;
534 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
535 int ret;
536
537 /*
538 * Re-program all of the registers that get reset as a result of
539 * regulator_disable() and _enable()
540 */
541 venus_set_registers(hdev);
542
543 writel(hdev->ifaceq_table.da, cpu_cs_base + UC_REGION_ADDR);
544 writel(SHARED_QSIZE, cpu_cs_base + UC_REGION_SIZE);
545 writel(hdev->ifaceq_table.da, cpu_cs_base + CPU_CS_SCIACMDARG2);
546 writel(0x01, cpu_cs_base + CPU_CS_SCIACMDARG1);
547 if (hdev->sfr.da)
548 writel(hdev->sfr.da, cpu_cs_base + SFR_ADDR);
549
550 ret = venus_boot_core(hdev);
551 if (ret) {
552 dev_err(dev, "failed to reset venus core\n");
553 return ret;
554 }
555
556 venus_hwversion(hdev);
557
558 return 0;
559 }
560
venus_halt_axi(struct venus_hfi_device * hdev)561 static int venus_halt_axi(struct venus_hfi_device *hdev)
562 {
563 void __iomem *wrapper_base = hdev->core->wrapper_base;
564 void __iomem *vbif_base = hdev->core->vbif_base;
565 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
566 void __iomem *aon_base = hdev->core->aon_base;
567 struct device *dev = hdev->core->dev;
568 u32 val;
569 u32 mask_val;
570 int ret;
571
572 if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core)) {
573 writel(0x3, cpu_cs_base + CPU_CS_X2RPMH_V6);
574
575 if (IS_IRIS2_1(hdev->core))
576 goto skip_aon_mvp_noc;
577
578 writel(0x1, aon_base + AON_WRAPPER_MVP_NOC_LPI_CONTROL);
579 ret = readl_poll_timeout(aon_base + AON_WRAPPER_MVP_NOC_LPI_STATUS,
580 val,
581 val & BIT(0),
582 POLL_INTERVAL_US,
583 VBIF_AXI_HALT_ACK_TIMEOUT_US);
584 if (ret)
585 return -ETIMEDOUT;
586
587 skip_aon_mvp_noc:
588 mask_val = (BIT(2) | BIT(1) | BIT(0));
589 writel(mask_val, wrapper_base + WRAPPER_DEBUG_BRIDGE_LPI_CONTROL_V6);
590
591 writel(0x00, wrapper_base + WRAPPER_DEBUG_BRIDGE_LPI_CONTROL_V6);
592 ret = readl_poll_timeout(wrapper_base + WRAPPER_DEBUG_BRIDGE_LPI_STATUS_V6,
593 val,
594 val == 0,
595 POLL_INTERVAL_US,
596 VBIF_AXI_HALT_ACK_TIMEOUT_US);
597
598 if (ret) {
599 dev_err(dev, "DBLP Release: lpi_status %x\n", val);
600 return -ETIMEDOUT;
601 }
602 return 0;
603 }
604
605 if (IS_V4(hdev->core)) {
606 val = readl(wrapper_base + WRAPPER_CPU_AXI_HALT);
607 val |= WRAPPER_CPU_AXI_HALT_HALT;
608 writel(val, wrapper_base + WRAPPER_CPU_AXI_HALT);
609
610 ret = readl_poll_timeout(wrapper_base + WRAPPER_CPU_AXI_HALT_STATUS,
611 val,
612 val & WRAPPER_CPU_AXI_HALT_STATUS_IDLE,
613 POLL_INTERVAL_US,
614 VBIF_AXI_HALT_ACK_TIMEOUT_US);
615 if (ret) {
616 dev_err(dev, "AXI bus port halt timeout\n");
617 return ret;
618 }
619
620 return 0;
621 }
622
623 /* Halt AXI and AXI IMEM VBIF Access */
624 val = readl(vbif_base + VBIF_AXI_HALT_CTRL0);
625 val |= VBIF_AXI_HALT_CTRL0_HALT_REQ;
626 writel(val, vbif_base + VBIF_AXI_HALT_CTRL0);
627
628 /* Request for AXI bus port halt */
629 ret = readl_poll_timeout(vbif_base + VBIF_AXI_HALT_CTRL1, val,
630 val & VBIF_AXI_HALT_CTRL1_HALT_ACK,
631 POLL_INTERVAL_US,
632 VBIF_AXI_HALT_ACK_TIMEOUT_US);
633 if (ret) {
634 dev_err(dev, "AXI bus port halt timeout\n");
635 return ret;
636 }
637
638 return 0;
639 }
640
venus_power_off(struct venus_hfi_device * hdev)641 static int venus_power_off(struct venus_hfi_device *hdev)
642 {
643 int ret;
644
645 if (!hdev->power_enabled)
646 return 0;
647
648 ret = venus_set_hw_state_suspend(hdev->core);
649 if (ret)
650 return ret;
651
652 ret = venus_halt_axi(hdev);
653 if (ret)
654 return ret;
655
656 hdev->power_enabled = false;
657
658 return 0;
659 }
660
venus_power_on(struct venus_hfi_device * hdev)661 static int venus_power_on(struct venus_hfi_device *hdev)
662 {
663 int ret;
664
665 if (hdev->power_enabled)
666 return 0;
667
668 ret = venus_set_hw_state_resume(hdev->core);
669 if (ret)
670 goto err;
671
672 ret = venus_run(hdev);
673 if (ret)
674 goto err_suspend;
675
676 hdev->power_enabled = true;
677
678 return 0;
679
680 err_suspend:
681 venus_set_hw_state_suspend(hdev->core);
682 err:
683 hdev->power_enabled = false;
684 return ret;
685 }
686
venus_iface_msgq_read_nolock(struct venus_hfi_device * hdev,void * pkt)687 static int venus_iface_msgq_read_nolock(struct venus_hfi_device *hdev,
688 void *pkt)
689 {
690 struct iface_queue *queue;
691 u32 tx_req;
692 int ret;
693
694 if (!venus_is_valid_state(hdev))
695 return -EINVAL;
696
697 queue = &hdev->queues[IFACEQ_MSG_IDX];
698
699 ret = venus_read_queue(hdev, queue, pkt, &tx_req);
700 if (ret)
701 return ret;
702
703 if (tx_req)
704 venus_soft_int(hdev);
705
706 return 0;
707 }
708
venus_iface_msgq_read(struct venus_hfi_device * hdev,void * pkt)709 static int venus_iface_msgq_read(struct venus_hfi_device *hdev, void *pkt)
710 {
711 int ret;
712
713 mutex_lock(&hdev->lock);
714 ret = venus_iface_msgq_read_nolock(hdev, pkt);
715 mutex_unlock(&hdev->lock);
716
717 return ret;
718 }
719
venus_iface_dbgq_read_nolock(struct venus_hfi_device * hdev,void * pkt)720 static int venus_iface_dbgq_read_nolock(struct venus_hfi_device *hdev,
721 void *pkt)
722 {
723 struct iface_queue *queue;
724 u32 tx_req;
725 int ret;
726
727 ret = venus_is_valid_state(hdev);
728 if (!ret)
729 return -EINVAL;
730
731 queue = &hdev->queues[IFACEQ_DBG_IDX];
732
733 ret = venus_read_queue(hdev, queue, pkt, &tx_req);
734 if (ret)
735 return ret;
736
737 if (tx_req)
738 venus_soft_int(hdev);
739
740 return 0;
741 }
742
venus_iface_dbgq_read(struct venus_hfi_device * hdev,void * pkt)743 static int venus_iface_dbgq_read(struct venus_hfi_device *hdev, void *pkt)
744 {
745 int ret;
746
747 if (!pkt)
748 return -EINVAL;
749
750 mutex_lock(&hdev->lock);
751 ret = venus_iface_dbgq_read_nolock(hdev, pkt);
752 mutex_unlock(&hdev->lock);
753
754 return ret;
755 }
756
venus_set_qhdr_defaults(struct hfi_queue_header * qhdr)757 static void venus_set_qhdr_defaults(struct hfi_queue_header *qhdr)
758 {
759 qhdr->status = 1;
760 qhdr->type = IFACEQ_DFLT_QHDR;
761 qhdr->q_size = IFACEQ_QUEUE_SIZE / 4;
762 qhdr->pkt_size = 0;
763 qhdr->rx_wm = 1;
764 qhdr->tx_wm = 1;
765 qhdr->rx_req = 1;
766 qhdr->tx_req = 0;
767 qhdr->rx_irq_status = 0;
768 qhdr->tx_irq_status = 0;
769 qhdr->read_idx = 0;
770 qhdr->write_idx = 0;
771 }
772
venus_interface_queues_release(struct venus_hfi_device * hdev)773 static void venus_interface_queues_release(struct venus_hfi_device *hdev)
774 {
775 mutex_lock(&hdev->lock);
776
777 venus_free(hdev, &hdev->ifaceq_table);
778 venus_free(hdev, &hdev->sfr);
779
780 memset(hdev->queues, 0, sizeof(hdev->queues));
781 memset(&hdev->ifaceq_table, 0, sizeof(hdev->ifaceq_table));
782 memset(&hdev->sfr, 0, sizeof(hdev->sfr));
783
784 mutex_unlock(&hdev->lock);
785 }
786
venus_interface_queues_init(struct venus_hfi_device * hdev)787 static int venus_interface_queues_init(struct venus_hfi_device *hdev)
788 {
789 struct hfi_queue_table_header *tbl_hdr;
790 struct iface_queue *queue;
791 struct hfi_sfr *sfr;
792 struct mem_desc desc = {0};
793 unsigned int offset;
794 unsigned int i;
795 int ret;
796
797 ret = venus_alloc(hdev, &desc, ALIGNED_QUEUE_SIZE);
798 if (ret)
799 return ret;
800
801 hdev->ifaceq_table = desc;
802 offset = IFACEQ_TABLE_SIZE;
803
804 for (i = 0; i < IFACEQ_NUM; i++) {
805 queue = &hdev->queues[i];
806 queue->qmem.da = desc.da + offset;
807 queue->qmem.kva = desc.kva + offset;
808 queue->qmem.size = IFACEQ_QUEUE_SIZE;
809 offset += queue->qmem.size;
810 queue->qhdr =
811 IFACEQ_GET_QHDR_START_ADDR(hdev->ifaceq_table.kva, i);
812
813 venus_set_qhdr_defaults(queue->qhdr);
814
815 queue->qhdr->start_addr = queue->qmem.da;
816
817 if (i == IFACEQ_CMD_IDX)
818 queue->qhdr->type |= HFI_HOST_TO_CTRL_CMD_Q;
819 else if (i == IFACEQ_MSG_IDX)
820 queue->qhdr->type |= HFI_CTRL_TO_HOST_MSG_Q;
821 else if (i == IFACEQ_DBG_IDX)
822 queue->qhdr->type |= HFI_CTRL_TO_HOST_DBG_Q;
823 }
824
825 tbl_hdr = hdev->ifaceq_table.kva;
826 tbl_hdr->version = 0;
827 tbl_hdr->size = IFACEQ_TABLE_SIZE;
828 tbl_hdr->qhdr0_offset = sizeof(struct hfi_queue_table_header);
829 tbl_hdr->qhdr_size = sizeof(struct hfi_queue_header);
830 tbl_hdr->num_q = IFACEQ_NUM;
831 tbl_hdr->num_active_q = IFACEQ_NUM;
832
833 /*
834 * Set receive request to zero on debug queue as there is no
835 * need of interrupt from video hardware for debug messages
836 */
837 queue = &hdev->queues[IFACEQ_DBG_IDX];
838 queue->qhdr->rx_req = 0;
839
840 ret = venus_alloc(hdev, &desc, ALIGNED_SFR_SIZE);
841 if (ret) {
842 hdev->sfr.da = 0;
843 } else {
844 hdev->sfr = desc;
845 sfr = hdev->sfr.kva;
846 sfr->buf_size = ALIGNED_SFR_SIZE;
847 }
848
849 /* ensure table and queue header structs are settled in memory */
850 wmb();
851
852 return 0;
853 }
854
venus_sys_set_debug(struct venus_hfi_device * hdev,u32 debug)855 static int venus_sys_set_debug(struct venus_hfi_device *hdev, u32 debug)
856 {
857 struct hfi_sys_set_property_pkt *pkt;
858 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
859
860 pkt = (struct hfi_sys_set_property_pkt *)packet;
861
862 pkt_sys_debug_config(pkt, HFI_DEBUG_MODE_QUEUE, debug);
863
864 return venus_iface_cmdq_write(hdev, pkt, false);
865 }
866
venus_sys_set_coverage(struct venus_hfi_device * hdev,u32 mode)867 static int venus_sys_set_coverage(struct venus_hfi_device *hdev, u32 mode)
868 {
869 struct hfi_sys_set_property_pkt *pkt;
870 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
871
872 pkt = (struct hfi_sys_set_property_pkt *)packet;
873
874 pkt_sys_coverage_config(pkt, mode);
875
876 return venus_iface_cmdq_write(hdev, pkt, false);
877 }
878
venus_sys_set_idle_message(struct venus_hfi_device * hdev,bool enable)879 static int venus_sys_set_idle_message(struct venus_hfi_device *hdev,
880 bool enable)
881 {
882 struct hfi_sys_set_property_pkt *pkt;
883 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
884
885 if (!enable)
886 return 0;
887
888 pkt = (struct hfi_sys_set_property_pkt *)packet;
889
890 pkt_sys_idle_indicator(pkt, enable);
891
892 return venus_iface_cmdq_write(hdev, pkt, false);
893 }
894
venus_sys_set_power_control(struct venus_hfi_device * hdev,bool enable)895 static int venus_sys_set_power_control(struct venus_hfi_device *hdev,
896 bool enable)
897 {
898 struct hfi_sys_set_property_pkt *pkt;
899 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
900
901 pkt = (struct hfi_sys_set_property_pkt *)packet;
902
903 pkt_sys_power_control(pkt, enable);
904
905 return venus_iface_cmdq_write(hdev, pkt, false);
906 }
907
venus_sys_set_ubwc_config(struct venus_hfi_device * hdev)908 static int venus_sys_set_ubwc_config(struct venus_hfi_device *hdev)
909 {
910 struct hfi_sys_set_property_pkt *pkt;
911 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
912 const struct venus_resources *res = hdev->core->res;
913 int ret;
914
915 pkt = (struct hfi_sys_set_property_pkt *)packet;
916
917 pkt_sys_ubwc_config(pkt, res->ubwc_conf);
918
919 ret = venus_iface_cmdq_write(hdev, pkt, false);
920 if (ret)
921 return ret;
922
923 return 0;
924 }
925
venus_get_queue_size(struct venus_hfi_device * hdev,unsigned int index)926 static int venus_get_queue_size(struct venus_hfi_device *hdev,
927 unsigned int index)
928 {
929 struct hfi_queue_header *qhdr;
930
931 if (index >= IFACEQ_NUM)
932 return -EINVAL;
933
934 qhdr = hdev->queues[index].qhdr;
935 if (!qhdr)
936 return -EINVAL;
937
938 return abs(qhdr->read_idx - qhdr->write_idx);
939 }
940
venus_sys_set_default_properties(struct venus_hfi_device * hdev)941 static int venus_sys_set_default_properties(struct venus_hfi_device *hdev)
942 {
943 struct device *dev = hdev->core->dev;
944 const struct venus_resources *res = hdev->core->res;
945 int ret;
946
947 ret = venus_sys_set_debug(hdev, venus_fw_debug);
948 if (ret)
949 dev_warn(dev, "setting fw debug msg ON failed (%d)\n", ret);
950
951 /* HFI_PROPERTY_SYS_IDLE_INDICATOR is not supported beyond 8916 (HFI V1) */
952 if (IS_V1(hdev->core)) {
953 ret = venus_sys_set_idle_message(hdev, false);
954 if (ret)
955 dev_warn(dev, "setting idle response ON failed (%d)\n", ret);
956 }
957
958 ret = venus_sys_set_power_control(hdev, venus_fw_low_power_mode);
959 if (ret)
960 dev_warn(dev, "setting hw power collapse ON failed (%d)\n",
961 ret);
962
963 /* For specific venus core, it is mandatory to set the UBWC configuration */
964 if (res->ubwc_conf) {
965 ret = venus_sys_set_ubwc_config(hdev);
966 if (ret)
967 dev_warn(dev, "setting ubwc config failed (%d)\n", ret);
968 }
969
970 return ret;
971 }
972
venus_session_cmd(struct venus_inst * inst,u32 pkt_type,bool sync)973 static int venus_session_cmd(struct venus_inst *inst, u32 pkt_type, bool sync)
974 {
975 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
976 struct hfi_session_pkt pkt;
977
978 pkt_session_cmd(&pkt, pkt_type, inst);
979
980 return venus_iface_cmdq_write(hdev, &pkt, sync);
981 }
982
venus_flush_debug_queue(struct venus_hfi_device * hdev)983 static void venus_flush_debug_queue(struct venus_hfi_device *hdev)
984 {
985 struct device *dev = hdev->core->dev;
986 void *packet = hdev->dbg_buf;
987
988 while (!venus_iface_dbgq_read(hdev, packet)) {
989 struct hfi_msg_sys_coverage_pkt *pkt = packet;
990
991 if (pkt->hdr.pkt_type != HFI_MSG_SYS_COV) {
992 struct hfi_msg_sys_debug_pkt *pkt = packet;
993
994 dev_dbg(dev, VDBGFW "%s", pkt->msg_data);
995 }
996 }
997 }
998
venus_prepare_power_collapse(struct venus_hfi_device * hdev,bool wait)999 static int venus_prepare_power_collapse(struct venus_hfi_device *hdev,
1000 bool wait)
1001 {
1002 unsigned long timeout = msecs_to_jiffies(venus_hw_rsp_timeout);
1003 struct hfi_sys_pc_prep_pkt pkt;
1004 int ret;
1005
1006 init_completion(&hdev->pwr_collapse_prep);
1007
1008 pkt_sys_pc_prep(&pkt);
1009
1010 ret = venus_iface_cmdq_write(hdev, &pkt, false);
1011 if (ret)
1012 return ret;
1013
1014 if (!wait)
1015 return 0;
1016
1017 ret = wait_for_completion_timeout(&hdev->pwr_collapse_prep, timeout);
1018 if (!ret) {
1019 venus_flush_debug_queue(hdev);
1020 return -ETIMEDOUT;
1021 }
1022
1023 return 0;
1024 }
1025
venus_are_queues_empty(struct venus_hfi_device * hdev)1026 static int venus_are_queues_empty(struct venus_hfi_device *hdev)
1027 {
1028 int ret1, ret2;
1029
1030 ret1 = venus_get_queue_size(hdev, IFACEQ_MSG_IDX);
1031 if (ret1 < 0)
1032 return ret1;
1033
1034 ret2 = venus_get_queue_size(hdev, IFACEQ_CMD_IDX);
1035 if (ret2 < 0)
1036 return ret2;
1037
1038 if (!ret1 && !ret2)
1039 return 1;
1040
1041 return 0;
1042 }
1043
venus_sfr_print(struct venus_hfi_device * hdev)1044 static void venus_sfr_print(struct venus_hfi_device *hdev)
1045 {
1046 struct device *dev = hdev->core->dev;
1047 struct hfi_sfr *sfr = hdev->sfr.kva;
1048 u32 size;
1049 void *p;
1050
1051 if (!sfr)
1052 return;
1053
1054 size = sfr->buf_size;
1055 if (!size)
1056 return;
1057
1058 if (size > ALIGNED_SFR_SIZE)
1059 size = ALIGNED_SFR_SIZE;
1060
1061 p = memchr(sfr->data, '\0', size);
1062 /*
1063 * SFR isn't guaranteed to be NULL terminated since SYS_ERROR indicates
1064 * that Venus is in the process of crashing.
1065 */
1066 if (!p)
1067 sfr->data[size - 1] = '\0';
1068
1069 dev_err_ratelimited(dev, "SFR message from FW: %s\n", sfr->data);
1070 }
1071
venus_process_msg_sys_error(struct venus_hfi_device * hdev,void * packet)1072 static void venus_process_msg_sys_error(struct venus_hfi_device *hdev,
1073 void *packet)
1074 {
1075 struct hfi_msg_event_notify_pkt *event_pkt = packet;
1076
1077 if (event_pkt->event_id != HFI_EVENT_SYS_ERROR)
1078 return;
1079
1080 venus_set_state(hdev, VENUS_STATE_DEINIT);
1081
1082 venus_sfr_print(hdev);
1083 }
1084
venus_isr_thread(struct venus_core * core)1085 static irqreturn_t venus_isr_thread(struct venus_core *core)
1086 {
1087 struct venus_hfi_device *hdev = to_hfi_priv(core);
1088 const struct venus_resources *res;
1089 void *pkt;
1090 u32 msg_ret;
1091
1092 if (!hdev)
1093 return IRQ_NONE;
1094
1095 res = hdev->core->res;
1096 pkt = hdev->pkt_buf;
1097
1098
1099 while (!venus_iface_msgq_read(hdev, pkt)) {
1100 msg_ret = hfi_process_msg_packet(core, pkt);
1101 switch (msg_ret) {
1102 case HFI_MSG_EVENT_NOTIFY:
1103 venus_process_msg_sys_error(hdev, pkt);
1104 break;
1105 case HFI_MSG_SYS_INIT:
1106 venus_hfi_core_set_resource(core, res->vmem_id,
1107 res->vmem_size,
1108 res->vmem_addr,
1109 hdev);
1110 break;
1111 case HFI_MSG_SYS_RELEASE_RESOURCE:
1112 complete(&hdev->release_resource);
1113 break;
1114 case HFI_MSG_SYS_PC_PREP:
1115 complete(&hdev->pwr_collapse_prep);
1116 break;
1117 default:
1118 break;
1119 }
1120 }
1121
1122 venus_flush_debug_queue(hdev);
1123
1124 return IRQ_HANDLED;
1125 }
1126
venus_isr(struct venus_core * core)1127 static irqreturn_t venus_isr(struct venus_core *core)
1128 {
1129 struct venus_hfi_device *hdev = to_hfi_priv(core);
1130 u32 status;
1131 void __iomem *cpu_cs_base;
1132 void __iomem *wrapper_base;
1133
1134 if (!hdev)
1135 return IRQ_NONE;
1136
1137 cpu_cs_base = hdev->core->cpu_cs_base;
1138 wrapper_base = hdev->core->wrapper_base;
1139
1140 status = readl(wrapper_base + WRAPPER_INTR_STATUS);
1141 if (IS_IRIS2(core) || IS_IRIS2_1(core)) {
1142 if (status & WRAPPER_INTR_STATUS_A2H_MASK ||
1143 status & WRAPPER_INTR_STATUS_A2HWD_MASK_V6 ||
1144 status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK)
1145 hdev->irq_status = status;
1146 } else {
1147 if (status & WRAPPER_INTR_STATUS_A2H_MASK ||
1148 status & WRAPPER_INTR_STATUS_A2HWD_MASK ||
1149 status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK)
1150 hdev->irq_status = status;
1151 }
1152 writel(1, cpu_cs_base + CPU_CS_A2HSOFTINTCLR);
1153 if (!(IS_IRIS2(core) || IS_IRIS2_1(core)))
1154 writel(status, wrapper_base + WRAPPER_INTR_CLEAR);
1155
1156 return IRQ_WAKE_THREAD;
1157 }
1158
venus_core_init(struct venus_core * core)1159 static int venus_core_init(struct venus_core *core)
1160 {
1161 struct venus_hfi_device *hdev = to_hfi_priv(core);
1162 struct device *dev = core->dev;
1163 struct hfi_sys_get_property_pkt version_pkt;
1164 struct hfi_sys_init_pkt pkt;
1165 int ret;
1166
1167 pkt_sys_init(&pkt, HFI_VIDEO_ARCH_OX);
1168
1169 venus_set_state(hdev, VENUS_STATE_INIT);
1170
1171 ret = venus_iface_cmdq_write(hdev, &pkt, false);
1172 if (ret)
1173 return ret;
1174
1175 pkt_sys_image_version(&version_pkt);
1176
1177 ret = venus_iface_cmdq_write(hdev, &version_pkt, false);
1178 if (ret)
1179 dev_warn(dev, "failed to send image version pkt to fw\n");
1180
1181 ret = venus_sys_set_default_properties(hdev);
1182 if (ret)
1183 return ret;
1184
1185 return 0;
1186 }
1187
venus_core_deinit(struct venus_core * core)1188 static int venus_core_deinit(struct venus_core *core)
1189 {
1190 struct venus_hfi_device *hdev = to_hfi_priv(core);
1191
1192 venus_set_state(hdev, VENUS_STATE_DEINIT);
1193 hdev->suspended = true;
1194 hdev->power_enabled = false;
1195
1196 return 0;
1197 }
1198
venus_core_trigger_ssr(struct venus_core * core,u32 trigger_type)1199 static int venus_core_trigger_ssr(struct venus_core *core, u32 trigger_type)
1200 {
1201 struct venus_hfi_device *hdev = to_hfi_priv(core);
1202 struct hfi_sys_test_ssr_pkt pkt;
1203 int ret;
1204
1205 ret = pkt_sys_ssr_cmd(&pkt, trigger_type);
1206 if (ret)
1207 return ret;
1208
1209 return venus_iface_cmdq_write(hdev, &pkt, false);
1210 }
1211
venus_session_init(struct venus_inst * inst,u32 session_type,u32 codec)1212 static int venus_session_init(struct venus_inst *inst, u32 session_type,
1213 u32 codec)
1214 {
1215 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1216 struct hfi_session_init_pkt pkt;
1217 int ret;
1218
1219 ret = venus_sys_set_debug(hdev, venus_fw_debug);
1220 if (ret)
1221 goto err;
1222
1223 ret = pkt_session_init(&pkt, inst, session_type, codec);
1224 if (ret)
1225 goto err;
1226
1227 ret = venus_iface_cmdq_write(hdev, &pkt, true);
1228 if (ret)
1229 goto err;
1230
1231 return 0;
1232
1233 err:
1234 venus_flush_debug_queue(hdev);
1235 return ret;
1236 }
1237
venus_session_end(struct venus_inst * inst)1238 static int venus_session_end(struct venus_inst *inst)
1239 {
1240 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1241 struct device *dev = hdev->core->dev;
1242
1243 if (venus_fw_coverage) {
1244 if (venus_sys_set_coverage(hdev, venus_fw_coverage))
1245 dev_warn(dev, "fw coverage msg ON failed\n");
1246 }
1247
1248 return venus_session_cmd(inst, HFI_CMD_SYS_SESSION_END, true);
1249 }
1250
venus_session_abort(struct venus_inst * inst)1251 static int venus_session_abort(struct venus_inst *inst)
1252 {
1253 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1254
1255 venus_flush_debug_queue(hdev);
1256
1257 return venus_session_cmd(inst, HFI_CMD_SYS_SESSION_ABORT, true);
1258 }
1259
venus_session_flush(struct venus_inst * inst,u32 flush_mode)1260 static int venus_session_flush(struct venus_inst *inst, u32 flush_mode)
1261 {
1262 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1263 struct hfi_session_flush_pkt pkt;
1264 int ret;
1265
1266 ret = pkt_session_flush(&pkt, inst, flush_mode);
1267 if (ret)
1268 return ret;
1269
1270 return venus_iface_cmdq_write(hdev, &pkt, true);
1271 }
1272
venus_session_start(struct venus_inst * inst)1273 static int venus_session_start(struct venus_inst *inst)
1274 {
1275 return venus_session_cmd(inst, HFI_CMD_SESSION_START, true);
1276 }
1277
venus_session_stop(struct venus_inst * inst)1278 static int venus_session_stop(struct venus_inst *inst)
1279 {
1280 return venus_session_cmd(inst, HFI_CMD_SESSION_STOP, true);
1281 }
1282
venus_session_continue(struct venus_inst * inst)1283 static int venus_session_continue(struct venus_inst *inst)
1284 {
1285 return venus_session_cmd(inst, HFI_CMD_SESSION_CONTINUE, false);
1286 }
1287
venus_session_etb(struct venus_inst * inst,struct hfi_frame_data * in_frame)1288 static int venus_session_etb(struct venus_inst *inst,
1289 struct hfi_frame_data *in_frame)
1290 {
1291 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1292 u32 session_type = inst->session_type;
1293 int ret;
1294
1295 if (session_type == VIDC_SESSION_TYPE_DEC) {
1296 struct hfi_session_empty_buffer_compressed_pkt pkt;
1297
1298 ret = pkt_session_etb_decoder(&pkt, inst, in_frame);
1299 if (ret)
1300 return ret;
1301
1302 ret = venus_iface_cmdq_write(hdev, &pkt, false);
1303 } else if (session_type == VIDC_SESSION_TYPE_ENC) {
1304 struct hfi_session_empty_buffer_uncompressed_plane0_pkt pkt;
1305
1306 ret = pkt_session_etb_encoder(&pkt, inst, in_frame);
1307 if (ret)
1308 return ret;
1309
1310 ret = venus_iface_cmdq_write(hdev, &pkt, false);
1311 } else {
1312 ret = -EINVAL;
1313 }
1314
1315 return ret;
1316 }
1317
venus_session_ftb(struct venus_inst * inst,struct hfi_frame_data * out_frame)1318 static int venus_session_ftb(struct venus_inst *inst,
1319 struct hfi_frame_data *out_frame)
1320 {
1321 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1322 struct hfi_session_fill_buffer_pkt pkt;
1323 int ret;
1324
1325 ret = pkt_session_ftb(&pkt, inst, out_frame);
1326 if (ret)
1327 return ret;
1328
1329 return venus_iface_cmdq_write(hdev, &pkt, false);
1330 }
1331
venus_session_set_buffers(struct venus_inst * inst,struct hfi_buffer_desc * bd)1332 static int venus_session_set_buffers(struct venus_inst *inst,
1333 struct hfi_buffer_desc *bd)
1334 {
1335 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1336 struct hfi_session_set_buffers_pkt *pkt;
1337 u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE];
1338 int ret;
1339
1340 if (bd->buffer_type == HFI_BUFFER_INPUT)
1341 return 0;
1342
1343 pkt = (struct hfi_session_set_buffers_pkt *)packet;
1344
1345 ret = pkt_session_set_buffers(pkt, inst, bd);
1346 if (ret)
1347 return ret;
1348
1349 return venus_iface_cmdq_write(hdev, pkt, false);
1350 }
1351
venus_session_unset_buffers(struct venus_inst * inst,struct hfi_buffer_desc * bd)1352 static int venus_session_unset_buffers(struct venus_inst *inst,
1353 struct hfi_buffer_desc *bd)
1354 {
1355 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1356 struct hfi_session_release_buffer_pkt *pkt;
1357 u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE];
1358 int ret;
1359
1360 if (bd->buffer_type == HFI_BUFFER_INPUT)
1361 return 0;
1362
1363 pkt = (struct hfi_session_release_buffer_pkt *)packet;
1364
1365 ret = pkt_session_unset_buffers(pkt, inst, bd);
1366 if (ret)
1367 return ret;
1368
1369 return venus_iface_cmdq_write(hdev, pkt, true);
1370 }
1371
venus_session_load_res(struct venus_inst * inst)1372 static int venus_session_load_res(struct venus_inst *inst)
1373 {
1374 return venus_session_cmd(inst, HFI_CMD_SESSION_LOAD_RESOURCES, true);
1375 }
1376
venus_session_release_res(struct venus_inst * inst)1377 static int venus_session_release_res(struct venus_inst *inst)
1378 {
1379 return venus_session_cmd(inst, HFI_CMD_SESSION_RELEASE_RESOURCES, true);
1380 }
1381
venus_session_parse_seq_hdr(struct venus_inst * inst,u32 seq_hdr,u32 seq_hdr_len)1382 static int venus_session_parse_seq_hdr(struct venus_inst *inst, u32 seq_hdr,
1383 u32 seq_hdr_len)
1384 {
1385 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1386 struct hfi_session_parse_sequence_header_pkt *pkt;
1387 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
1388 int ret;
1389
1390 pkt = (struct hfi_session_parse_sequence_header_pkt *)packet;
1391
1392 ret = pkt_session_parse_seq_header(pkt, inst, seq_hdr, seq_hdr_len);
1393 if (ret)
1394 return ret;
1395
1396 ret = venus_iface_cmdq_write(hdev, pkt, false);
1397 if (ret)
1398 return ret;
1399
1400 return 0;
1401 }
1402
venus_session_get_seq_hdr(struct venus_inst * inst,u32 seq_hdr,u32 seq_hdr_len)1403 static int venus_session_get_seq_hdr(struct venus_inst *inst, u32 seq_hdr,
1404 u32 seq_hdr_len)
1405 {
1406 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1407 struct hfi_session_get_sequence_header_pkt *pkt;
1408 u8 packet[IFACEQ_VAR_SMALL_PKT_SIZE];
1409 int ret;
1410
1411 pkt = (struct hfi_session_get_sequence_header_pkt *)packet;
1412
1413 ret = pkt_session_get_seq_hdr(pkt, inst, seq_hdr, seq_hdr_len);
1414 if (ret)
1415 return ret;
1416
1417 return venus_iface_cmdq_write(hdev, pkt, false);
1418 }
1419
venus_session_set_property(struct venus_inst * inst,u32 ptype,void * pdata)1420 static int venus_session_set_property(struct venus_inst *inst, u32 ptype,
1421 void *pdata)
1422 {
1423 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1424 struct hfi_session_set_property_pkt *pkt;
1425 u8 packet[IFACEQ_VAR_LARGE_PKT_SIZE];
1426 int ret;
1427
1428 pkt = (struct hfi_session_set_property_pkt *)packet;
1429
1430 ret = pkt_session_set_property(pkt, inst, ptype, pdata);
1431 if (ret == -ENOTSUPP)
1432 return 0;
1433 if (ret)
1434 return ret;
1435
1436 return venus_iface_cmdq_write(hdev, pkt, false);
1437 }
1438
venus_session_get_property(struct venus_inst * inst,u32 ptype)1439 static int venus_session_get_property(struct venus_inst *inst, u32 ptype)
1440 {
1441 struct venus_hfi_device *hdev = to_hfi_priv(inst->core);
1442 struct hfi_session_get_property_pkt pkt;
1443 int ret;
1444
1445 ret = pkt_session_get_property(&pkt, inst, ptype);
1446 if (ret)
1447 return ret;
1448
1449 return venus_iface_cmdq_write(hdev, &pkt, true);
1450 }
1451
venus_resume(struct venus_core * core)1452 static int venus_resume(struct venus_core *core)
1453 {
1454 struct venus_hfi_device *hdev = to_hfi_priv(core);
1455 int ret = 0;
1456
1457 mutex_lock(&hdev->lock);
1458
1459 if (!hdev->suspended)
1460 goto unlock;
1461
1462 ret = venus_power_on(hdev);
1463
1464 unlock:
1465 if (!ret)
1466 hdev->suspended = false;
1467
1468 mutex_unlock(&hdev->lock);
1469
1470 return ret;
1471 }
1472
venus_suspend_1xx(struct venus_core * core)1473 static int venus_suspend_1xx(struct venus_core *core)
1474 {
1475 struct venus_hfi_device *hdev = to_hfi_priv(core);
1476 struct device *dev = core->dev;
1477 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
1478 u32 ctrl_status;
1479 int ret;
1480
1481 if (!hdev->power_enabled || hdev->suspended)
1482 return 0;
1483
1484 mutex_lock(&hdev->lock);
1485 ret = venus_is_valid_state(hdev);
1486 mutex_unlock(&hdev->lock);
1487
1488 if (!ret) {
1489 dev_err(dev, "bad state, cannot suspend\n");
1490 return -EINVAL;
1491 }
1492
1493 ret = venus_prepare_power_collapse(hdev, true);
1494 if (ret) {
1495 dev_err(dev, "prepare for power collapse fail (%d)\n", ret);
1496 return ret;
1497 }
1498
1499 mutex_lock(&hdev->lock);
1500
1501 if (hdev->last_packet_type != HFI_CMD_SYS_PC_PREP) {
1502 mutex_unlock(&hdev->lock);
1503 return -EINVAL;
1504 }
1505
1506 ret = venus_are_queues_empty(hdev);
1507 if (ret < 0 || !ret) {
1508 mutex_unlock(&hdev->lock);
1509 return -EINVAL;
1510 }
1511
1512 ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
1513 if (!(ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)) {
1514 mutex_unlock(&hdev->lock);
1515 return -EINVAL;
1516 }
1517
1518 ret = venus_power_off(hdev);
1519 if (ret) {
1520 mutex_unlock(&hdev->lock);
1521 return ret;
1522 }
1523
1524 hdev->suspended = true;
1525
1526 mutex_unlock(&hdev->lock);
1527
1528 return 0;
1529 }
1530
venus_cpu_and_video_core_idle(struct venus_hfi_device * hdev)1531 static bool venus_cpu_and_video_core_idle(struct venus_hfi_device *hdev)
1532 {
1533 void __iomem *wrapper_base = hdev->core->wrapper_base;
1534 void __iomem *wrapper_tz_base = hdev->core->wrapper_tz_base;
1535 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
1536 u32 ctrl_status, cpu_status;
1537
1538 if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core))
1539 cpu_status = readl(wrapper_tz_base + WRAPPER_TZ_CPU_STATUS_V6);
1540 else
1541 cpu_status = readl(wrapper_base + WRAPPER_CPU_STATUS);
1542 ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
1543
1544 if (cpu_status & WRAPPER_CPU_STATUS_WFI &&
1545 ctrl_status & CPU_CS_SCIACMDARG0_INIT_IDLE_MSG_MASK)
1546 return true;
1547
1548 return false;
1549 }
1550
venus_cpu_idle_and_pc_ready(struct venus_hfi_device * hdev)1551 static bool venus_cpu_idle_and_pc_ready(struct venus_hfi_device *hdev)
1552 {
1553 void __iomem *wrapper_base = hdev->core->wrapper_base;
1554 void __iomem *wrapper_tz_base = hdev->core->wrapper_tz_base;
1555 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
1556 u32 ctrl_status, cpu_status;
1557
1558 if (IS_IRIS2(hdev->core) || IS_IRIS2_1(hdev->core))
1559 cpu_status = readl(wrapper_tz_base + WRAPPER_TZ_CPU_STATUS_V6);
1560 else
1561 cpu_status = readl(wrapper_base + WRAPPER_CPU_STATUS);
1562 ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
1563
1564 if (cpu_status & WRAPPER_CPU_STATUS_WFI &&
1565 ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)
1566 return true;
1567
1568 return false;
1569 }
1570
venus_suspend_3xx(struct venus_core * core)1571 static int venus_suspend_3xx(struct venus_core *core)
1572 {
1573 struct venus_hfi_device *hdev = to_hfi_priv(core);
1574 struct device *dev = core->dev;
1575 void __iomem *cpu_cs_base = hdev->core->cpu_cs_base;
1576 u32 ctrl_status;
1577 bool val;
1578 int ret;
1579
1580 if (!hdev->power_enabled || hdev->suspended)
1581 return 0;
1582
1583 mutex_lock(&hdev->lock);
1584 ret = venus_is_valid_state(hdev);
1585 mutex_unlock(&hdev->lock);
1586
1587 if (!ret) {
1588 dev_err(dev, "bad state, cannot suspend\n");
1589 return -EINVAL;
1590 }
1591
1592 ctrl_status = readl(cpu_cs_base + CPU_CS_SCIACMDARG0);
1593 if (ctrl_status & CPU_CS_SCIACMDARG0_PC_READY)
1594 goto power_off;
1595
1596 /*
1597 * Power collapse sequence for Venus 3xx and 4xx versions:
1598 * 1. Check for ARM9 and video core to be idle by checking WFI bit
1599 * (bit 0) in CPU status register and by checking Idle (bit 30) in
1600 * Control status register for video core.
1601 * 2. Send a command to prepare for power collapse.
1602 * 3. Check for WFI and PC_READY bits.
1603 */
1604 ret = readx_poll_timeout(venus_cpu_and_video_core_idle, hdev, val, val,
1605 1500, 100 * 1500);
1606 if (ret) {
1607 dev_err(dev, "wait for cpu and video core idle fail (%d)\n", ret);
1608 return ret;
1609 }
1610
1611 ret = venus_prepare_power_collapse(hdev, false);
1612 if (ret) {
1613 dev_err(dev, "prepare for power collapse fail (%d)\n", ret);
1614 return ret;
1615 }
1616
1617 ret = readx_poll_timeout(venus_cpu_idle_and_pc_ready, hdev, val, val,
1618 1500, 100 * 1500);
1619 if (ret)
1620 return ret;
1621
1622 power_off:
1623 mutex_lock(&hdev->lock);
1624
1625 ret = venus_power_off(hdev);
1626 if (ret) {
1627 dev_err(dev, "venus_power_off (%d)\n", ret);
1628 mutex_unlock(&hdev->lock);
1629 return ret;
1630 }
1631
1632 hdev->suspended = true;
1633
1634 mutex_unlock(&hdev->lock);
1635
1636 return 0;
1637 }
1638
venus_suspend(struct venus_core * core)1639 static int venus_suspend(struct venus_core *core)
1640 {
1641 if (IS_V3(core) || IS_V4(core) || IS_V6(core))
1642 return venus_suspend_3xx(core);
1643
1644 return venus_suspend_1xx(core);
1645 }
1646
1647 static const struct hfi_ops venus_hfi_ops = {
1648 .core_init = venus_core_init,
1649 .core_deinit = venus_core_deinit,
1650 .core_trigger_ssr = venus_core_trigger_ssr,
1651
1652 .session_init = venus_session_init,
1653 .session_end = venus_session_end,
1654 .session_abort = venus_session_abort,
1655 .session_flush = venus_session_flush,
1656 .session_start = venus_session_start,
1657 .session_stop = venus_session_stop,
1658 .session_continue = venus_session_continue,
1659 .session_etb = venus_session_etb,
1660 .session_ftb = venus_session_ftb,
1661 .session_set_buffers = venus_session_set_buffers,
1662 .session_unset_buffers = venus_session_unset_buffers,
1663 .session_load_res = venus_session_load_res,
1664 .session_release_res = venus_session_release_res,
1665 .session_parse_seq_hdr = venus_session_parse_seq_hdr,
1666 .session_get_seq_hdr = venus_session_get_seq_hdr,
1667 .session_set_property = venus_session_set_property,
1668 .session_get_property = venus_session_get_property,
1669
1670 .resume = venus_resume,
1671 .suspend = venus_suspend,
1672
1673 .isr = venus_isr,
1674 .isr_thread = venus_isr_thread,
1675 };
1676
venus_hfi_destroy(struct venus_core * core)1677 void venus_hfi_destroy(struct venus_core *core)
1678 {
1679 struct venus_hfi_device *hdev = to_hfi_priv(core);
1680
1681 core->priv = NULL;
1682 venus_interface_queues_release(hdev);
1683 mutex_destroy(&hdev->lock);
1684 kfree(hdev);
1685 disable_irq(core->irq);
1686 core->ops = NULL;
1687 }
1688
venus_hfi_create(struct venus_core * core)1689 int venus_hfi_create(struct venus_core *core)
1690 {
1691 struct venus_hfi_device *hdev;
1692 int ret;
1693
1694 hdev = kzalloc(sizeof(*hdev), GFP_KERNEL);
1695 if (!hdev)
1696 return -ENOMEM;
1697
1698 mutex_init(&hdev->lock);
1699
1700 hdev->core = core;
1701 hdev->suspended = true;
1702 core->priv = hdev;
1703 core->ops = &venus_hfi_ops;
1704
1705 ret = venus_interface_queues_init(hdev);
1706 if (ret)
1707 goto err_kfree;
1708
1709 return 0;
1710
1711 err_kfree:
1712 kfree(hdev);
1713 core->priv = NULL;
1714 core->ops = NULL;
1715 return ret;
1716 }
1717
venus_hfi_queues_reinit(struct venus_core * core)1718 void venus_hfi_queues_reinit(struct venus_core *core)
1719 {
1720 struct venus_hfi_device *hdev = to_hfi_priv(core);
1721 struct hfi_queue_table_header *tbl_hdr;
1722 struct iface_queue *queue;
1723 struct hfi_sfr *sfr;
1724 unsigned int i;
1725
1726 mutex_lock(&hdev->lock);
1727
1728 for (i = 0; i < IFACEQ_NUM; i++) {
1729 queue = &hdev->queues[i];
1730 queue->qhdr =
1731 IFACEQ_GET_QHDR_START_ADDR(hdev->ifaceq_table.kva, i);
1732
1733 venus_set_qhdr_defaults(queue->qhdr);
1734
1735 queue->qhdr->start_addr = queue->qmem.da;
1736
1737 if (i == IFACEQ_CMD_IDX)
1738 queue->qhdr->type |= HFI_HOST_TO_CTRL_CMD_Q;
1739 else if (i == IFACEQ_MSG_IDX)
1740 queue->qhdr->type |= HFI_CTRL_TO_HOST_MSG_Q;
1741 else if (i == IFACEQ_DBG_IDX)
1742 queue->qhdr->type |= HFI_CTRL_TO_HOST_DBG_Q;
1743 }
1744
1745 tbl_hdr = hdev->ifaceq_table.kva;
1746 tbl_hdr->version = 0;
1747 tbl_hdr->size = IFACEQ_TABLE_SIZE;
1748 tbl_hdr->qhdr0_offset = sizeof(struct hfi_queue_table_header);
1749 tbl_hdr->qhdr_size = sizeof(struct hfi_queue_header);
1750 tbl_hdr->num_q = IFACEQ_NUM;
1751 tbl_hdr->num_active_q = IFACEQ_NUM;
1752
1753 /*
1754 * Set receive request to zero on debug queue as there is no
1755 * need of interrupt from video hardware for debug messages
1756 */
1757 queue = &hdev->queues[IFACEQ_DBG_IDX];
1758 queue->qhdr->rx_req = 0;
1759
1760 sfr = hdev->sfr.kva;
1761 sfr->buf_size = ALIGNED_SFR_SIZE;
1762
1763 /* ensure table and queue header structs are settled in memory */
1764 wmb();
1765
1766 mutex_unlock(&hdev->lock);
1767 }
1768