1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2017-2019 The Linux Foundation. All rights reserved. */
3
4
5 #include "msm_gem.h"
6 #include "msm_mmu.h"
7 #include "msm_gpu_trace.h"
8 #include "a6xx_gpu.h"
9 #include "a6xx_gmu.xml.h"
10
11 #include <linux/bitfield.h>
12 #include <linux/devfreq.h>
13 #include <linux/pm_domain.h>
14 #include <linux/soc/qcom/llcc-qcom.h>
15
16 #define GPU_PAS_ID 13
17
_a6xx_check_idle(struct msm_gpu * gpu)18 static inline bool _a6xx_check_idle(struct msm_gpu *gpu)
19 {
20 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
21 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
22
23 /* Check that the GMU is idle */
24 if (!adreno_has_gmu_wrapper(adreno_gpu) && !a6xx_gmu_isidle(&a6xx_gpu->gmu))
25 return false;
26
27 /* Check tha the CX master is idle */
28 if (gpu_read(gpu, REG_A6XX_RBBM_STATUS) &
29 ~A6XX_RBBM_STATUS_CP_AHB_BUSY_CX_MASTER)
30 return false;
31
32 return !(gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS) &
33 A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT);
34 }
35
a6xx_idle(struct msm_gpu * gpu,struct msm_ringbuffer * ring)36 static bool a6xx_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
37 {
38 /* wait for CP to drain ringbuffer: */
39 if (!adreno_idle(gpu, ring))
40 return false;
41
42 if (spin_until(_a6xx_check_idle(gpu))) {
43 DRM_ERROR("%s: %ps: timeout waiting for GPU to idle: status %8.8X irq %8.8X rptr/wptr %d/%d\n",
44 gpu->name, __builtin_return_address(0),
45 gpu_read(gpu, REG_A6XX_RBBM_STATUS),
46 gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS),
47 gpu_read(gpu, REG_A6XX_CP_RB_RPTR),
48 gpu_read(gpu, REG_A6XX_CP_RB_WPTR));
49 return false;
50 }
51
52 return true;
53 }
54
update_shadow_rptr(struct msm_gpu * gpu,struct msm_ringbuffer * ring)55 static void update_shadow_rptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
56 {
57 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
58 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
59
60 /* Expanded APRIV doesn't need to issue the WHERE_AM_I opcode */
61 if (a6xx_gpu->has_whereami && !adreno_gpu->base.hw_apriv) {
62 OUT_PKT7(ring, CP_WHERE_AM_I, 2);
63 OUT_RING(ring, lower_32_bits(shadowptr(a6xx_gpu, ring)));
64 OUT_RING(ring, upper_32_bits(shadowptr(a6xx_gpu, ring)));
65 }
66 }
67
a6xx_flush(struct msm_gpu * gpu,struct msm_ringbuffer * ring)68 static void a6xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
69 {
70 uint32_t wptr;
71 unsigned long flags;
72
73 update_shadow_rptr(gpu, ring);
74
75 spin_lock_irqsave(&ring->preempt_lock, flags);
76
77 /* Copy the shadow to the actual register */
78 ring->cur = ring->next;
79
80 /* Make sure to wrap wptr if we need to */
81 wptr = get_wptr(ring);
82
83 spin_unlock_irqrestore(&ring->preempt_lock, flags);
84
85 /* Make sure everything is posted before making a decision */
86 mb();
87
88 gpu_write(gpu, REG_A6XX_CP_RB_WPTR, wptr);
89 }
90
get_stats_counter(struct msm_ringbuffer * ring,u32 counter,u64 iova)91 static void get_stats_counter(struct msm_ringbuffer *ring, u32 counter,
92 u64 iova)
93 {
94 OUT_PKT7(ring, CP_REG_TO_MEM, 3);
95 OUT_RING(ring, CP_REG_TO_MEM_0_REG(counter) |
96 CP_REG_TO_MEM_0_CNT(2) |
97 CP_REG_TO_MEM_0_64B);
98 OUT_RING(ring, lower_32_bits(iova));
99 OUT_RING(ring, upper_32_bits(iova));
100 }
101
a6xx_set_pagetable(struct a6xx_gpu * a6xx_gpu,struct msm_ringbuffer * ring,struct msm_file_private * ctx)102 static void a6xx_set_pagetable(struct a6xx_gpu *a6xx_gpu,
103 struct msm_ringbuffer *ring, struct msm_file_private *ctx)
104 {
105 bool sysprof = refcount_read(&a6xx_gpu->base.base.sysprof_active) > 1;
106 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
107 phys_addr_t ttbr;
108 u32 asid;
109 u64 memptr = rbmemptr(ring, ttbr0);
110
111 if (ctx->seqno == a6xx_gpu->base.base.cur_ctx_seqno)
112 return;
113
114 if (msm_iommu_pagetable_params(ctx->aspace->mmu, &ttbr, &asid))
115 return;
116
117 if (!sysprof) {
118 if (!adreno_is_a7xx(adreno_gpu)) {
119 /* Turn off protected mode to write to special registers */
120 OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1);
121 OUT_RING(ring, 0);
122 }
123
124 OUT_PKT4(ring, REG_A6XX_RBBM_PERFCTR_SRAM_INIT_CMD, 1);
125 OUT_RING(ring, 1);
126 }
127
128 /* Execute the table update */
129 OUT_PKT7(ring, CP_SMMU_TABLE_UPDATE, 4);
130 OUT_RING(ring, CP_SMMU_TABLE_UPDATE_0_TTBR0_LO(lower_32_bits(ttbr)));
131
132 OUT_RING(ring,
133 CP_SMMU_TABLE_UPDATE_1_TTBR0_HI(upper_32_bits(ttbr)) |
134 CP_SMMU_TABLE_UPDATE_1_ASID(asid));
135 OUT_RING(ring, CP_SMMU_TABLE_UPDATE_2_CONTEXTIDR(0));
136 OUT_RING(ring, CP_SMMU_TABLE_UPDATE_3_CONTEXTBANK(0));
137
138 /*
139 * Write the new TTBR0 to the memstore. This is good for debugging.
140 */
141 OUT_PKT7(ring, CP_MEM_WRITE, 4);
142 OUT_RING(ring, CP_MEM_WRITE_0_ADDR_LO(lower_32_bits(memptr)));
143 OUT_RING(ring, CP_MEM_WRITE_1_ADDR_HI(upper_32_bits(memptr)));
144 OUT_RING(ring, lower_32_bits(ttbr));
145 OUT_RING(ring, (asid << 16) | upper_32_bits(ttbr));
146
147 /*
148 * Sync both threads after switching pagetables and enable BR only
149 * to make sure BV doesn't race ahead while BR is still switching
150 * pagetables.
151 */
152 if (adreno_is_a7xx(&a6xx_gpu->base)) {
153 OUT_PKT7(ring, CP_THREAD_CONTROL, 1);
154 OUT_RING(ring, CP_THREAD_CONTROL_0_SYNC_THREADS | CP_SET_THREAD_BR);
155 }
156
157 /*
158 * And finally, trigger a uche flush to be sure there isn't anything
159 * lingering in that part of the GPU
160 */
161
162 OUT_PKT7(ring, CP_EVENT_WRITE, 1);
163 OUT_RING(ring, CACHE_INVALIDATE);
164
165 if (!sysprof) {
166 /*
167 * Wait for SRAM clear after the pgtable update, so the
168 * two can happen in parallel:
169 */
170 OUT_PKT7(ring, CP_WAIT_REG_MEM, 6);
171 OUT_RING(ring, CP_WAIT_REG_MEM_0_FUNCTION(WRITE_EQ));
172 OUT_RING(ring, CP_WAIT_REG_MEM_1_POLL_ADDR_LO(
173 REG_A6XX_RBBM_PERFCTR_SRAM_INIT_STATUS));
174 OUT_RING(ring, CP_WAIT_REG_MEM_2_POLL_ADDR_HI(0));
175 OUT_RING(ring, CP_WAIT_REG_MEM_3_REF(0x1));
176 OUT_RING(ring, CP_WAIT_REG_MEM_4_MASK(0x1));
177 OUT_RING(ring, CP_WAIT_REG_MEM_5_DELAY_LOOP_CYCLES(0));
178
179 if (!adreno_is_a7xx(adreno_gpu)) {
180 /* Re-enable protected mode: */
181 OUT_PKT7(ring, CP_SET_PROTECTED_MODE, 1);
182 OUT_RING(ring, 1);
183 }
184 }
185 }
186
a6xx_submit(struct msm_gpu * gpu,struct msm_gem_submit * submit)187 static void a6xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit)
188 {
189 unsigned int index = submit->seqno % MSM_GPU_SUBMIT_STATS_COUNT;
190 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
191 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
192 struct msm_ringbuffer *ring = submit->ring;
193 unsigned int i, ibs = 0;
194
195 a6xx_set_pagetable(a6xx_gpu, ring, submit->queue->ctx);
196
197 get_stats_counter(ring, REG_A6XX_RBBM_PERFCTR_CP(0),
198 rbmemptr_stats(ring, index, cpcycles_start));
199
200 /*
201 * For PM4 the GMU register offsets are calculated from the base of the
202 * GPU registers so we need to add 0x1a800 to the register value on A630
203 * to get the right value from PM4.
204 */
205 get_stats_counter(ring, REG_A6XX_CP_ALWAYS_ON_COUNTER,
206 rbmemptr_stats(ring, index, alwayson_start));
207
208 /* Invalidate CCU depth and color */
209 OUT_PKT7(ring, CP_EVENT_WRITE, 1);
210 OUT_RING(ring, CP_EVENT_WRITE_0_EVENT(PC_CCU_INVALIDATE_DEPTH));
211
212 OUT_PKT7(ring, CP_EVENT_WRITE, 1);
213 OUT_RING(ring, CP_EVENT_WRITE_0_EVENT(PC_CCU_INVALIDATE_COLOR));
214
215 /* Submit the commands */
216 for (i = 0; i < submit->nr_cmds; i++) {
217 switch (submit->cmd[i].type) {
218 case MSM_SUBMIT_CMD_IB_TARGET_BUF:
219 break;
220 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
221 if (gpu->cur_ctx_seqno == submit->queue->ctx->seqno)
222 break;
223 fallthrough;
224 case MSM_SUBMIT_CMD_BUF:
225 OUT_PKT7(ring, CP_INDIRECT_BUFFER_PFE, 3);
226 OUT_RING(ring, lower_32_bits(submit->cmd[i].iova));
227 OUT_RING(ring, upper_32_bits(submit->cmd[i].iova));
228 OUT_RING(ring, submit->cmd[i].size);
229 ibs++;
230 break;
231 }
232
233 /*
234 * Periodically update shadow-wptr if needed, so that we
235 * can see partial progress of submits with large # of
236 * cmds.. otherwise we could needlessly stall waiting for
237 * ringbuffer state, simply due to looking at a shadow
238 * rptr value that has not been updated
239 */
240 if ((ibs % 32) == 0)
241 update_shadow_rptr(gpu, ring);
242 }
243
244 get_stats_counter(ring, REG_A6XX_RBBM_PERFCTR_CP(0),
245 rbmemptr_stats(ring, index, cpcycles_end));
246 get_stats_counter(ring, REG_A6XX_CP_ALWAYS_ON_COUNTER,
247 rbmemptr_stats(ring, index, alwayson_end));
248
249 /* Write the fence to the scratch register */
250 OUT_PKT4(ring, REG_A6XX_CP_SCRATCH_REG(2), 1);
251 OUT_RING(ring, submit->seqno);
252
253 /*
254 * Execute a CACHE_FLUSH_TS event. This will ensure that the
255 * timestamp is written to the memory and then triggers the interrupt
256 */
257 OUT_PKT7(ring, CP_EVENT_WRITE, 4);
258 OUT_RING(ring, CP_EVENT_WRITE_0_EVENT(CACHE_FLUSH_TS) |
259 CP_EVENT_WRITE_0_IRQ);
260 OUT_RING(ring, lower_32_bits(rbmemptr(ring, fence)));
261 OUT_RING(ring, upper_32_bits(rbmemptr(ring, fence)));
262 OUT_RING(ring, submit->seqno);
263
264 trace_msm_gpu_submit_flush(submit,
265 gpu_read64(gpu, REG_A6XX_CP_ALWAYS_ON_COUNTER));
266
267 a6xx_flush(gpu, ring);
268 }
269
a7xx_submit(struct msm_gpu * gpu,struct msm_gem_submit * submit)270 static void a7xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit)
271 {
272 unsigned int index = submit->seqno % MSM_GPU_SUBMIT_STATS_COUNT;
273 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
274 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
275 struct msm_ringbuffer *ring = submit->ring;
276 unsigned int i, ibs = 0;
277
278 /*
279 * Toggle concurrent binning for pagetable switch and set the thread to
280 * BR since only it can execute the pagetable switch packets.
281 */
282 OUT_PKT7(ring, CP_THREAD_CONTROL, 1);
283 OUT_RING(ring, CP_THREAD_CONTROL_0_SYNC_THREADS | CP_SET_THREAD_BR);
284
285 a6xx_set_pagetable(a6xx_gpu, ring, submit->queue->ctx);
286
287 get_stats_counter(ring, REG_A6XX_RBBM_PERFCTR_CP(0),
288 rbmemptr_stats(ring, index, cpcycles_start));
289 get_stats_counter(ring, REG_A6XX_CP_ALWAYS_ON_COUNTER,
290 rbmemptr_stats(ring, index, alwayson_start));
291
292 OUT_PKT7(ring, CP_THREAD_CONTROL, 1);
293 OUT_RING(ring, CP_SET_THREAD_BOTH);
294
295 OUT_PKT7(ring, CP_SET_MARKER, 1);
296 OUT_RING(ring, 0x101); /* IFPC disable */
297
298 OUT_PKT7(ring, CP_SET_MARKER, 1);
299 OUT_RING(ring, 0x00d); /* IB1LIST start */
300
301 /* Submit the commands */
302 for (i = 0; i < submit->nr_cmds; i++) {
303 switch (submit->cmd[i].type) {
304 case MSM_SUBMIT_CMD_IB_TARGET_BUF:
305 break;
306 case MSM_SUBMIT_CMD_CTX_RESTORE_BUF:
307 if (gpu->cur_ctx_seqno == submit->queue->ctx->seqno)
308 break;
309 fallthrough;
310 case MSM_SUBMIT_CMD_BUF:
311 OUT_PKT7(ring, CP_INDIRECT_BUFFER_PFE, 3);
312 OUT_RING(ring, lower_32_bits(submit->cmd[i].iova));
313 OUT_RING(ring, upper_32_bits(submit->cmd[i].iova));
314 OUT_RING(ring, submit->cmd[i].size);
315 ibs++;
316 break;
317 }
318
319 /*
320 * Periodically update shadow-wptr if needed, so that we
321 * can see partial progress of submits with large # of
322 * cmds.. otherwise we could needlessly stall waiting for
323 * ringbuffer state, simply due to looking at a shadow
324 * rptr value that has not been updated
325 */
326 if ((ibs % 32) == 0)
327 update_shadow_rptr(gpu, ring);
328 }
329
330 OUT_PKT7(ring, CP_SET_MARKER, 1);
331 OUT_RING(ring, 0x00e); /* IB1LIST end */
332
333 get_stats_counter(ring, REG_A6XX_RBBM_PERFCTR_CP(0),
334 rbmemptr_stats(ring, index, cpcycles_end));
335 get_stats_counter(ring, REG_A6XX_CP_ALWAYS_ON_COUNTER,
336 rbmemptr_stats(ring, index, alwayson_end));
337
338 /* Write the fence to the scratch register */
339 OUT_PKT4(ring, REG_A6XX_CP_SCRATCH_REG(2), 1);
340 OUT_RING(ring, submit->seqno);
341
342 OUT_PKT7(ring, CP_THREAD_CONTROL, 1);
343 OUT_RING(ring, CP_SET_THREAD_BR);
344
345 OUT_PKT7(ring, CP_EVENT_WRITE, 1);
346 OUT_RING(ring, CCU_INVALIDATE_DEPTH);
347
348 OUT_PKT7(ring, CP_EVENT_WRITE, 1);
349 OUT_RING(ring, CCU_INVALIDATE_COLOR);
350
351 OUT_PKT7(ring, CP_THREAD_CONTROL, 1);
352 OUT_RING(ring, CP_SET_THREAD_BV);
353
354 /*
355 * Make sure the timestamp is committed once BV pipe is
356 * completely done with this submission.
357 */
358 OUT_PKT7(ring, CP_EVENT_WRITE, 4);
359 OUT_RING(ring, CACHE_CLEAN | BIT(27));
360 OUT_RING(ring, lower_32_bits(rbmemptr(ring, bv_fence)));
361 OUT_RING(ring, upper_32_bits(rbmemptr(ring, bv_fence)));
362 OUT_RING(ring, submit->seqno);
363
364 OUT_PKT7(ring, CP_THREAD_CONTROL, 1);
365 OUT_RING(ring, CP_SET_THREAD_BR);
366
367 /*
368 * This makes sure that BR doesn't race ahead and commit
369 * timestamp to memstore while BV is still processing
370 * this submission.
371 */
372 OUT_PKT7(ring, CP_WAIT_TIMESTAMP, 4);
373 OUT_RING(ring, 0);
374 OUT_RING(ring, lower_32_bits(rbmemptr(ring, bv_fence)));
375 OUT_RING(ring, upper_32_bits(rbmemptr(ring, bv_fence)));
376 OUT_RING(ring, submit->seqno);
377
378 /* write the ringbuffer timestamp */
379 OUT_PKT7(ring, CP_EVENT_WRITE, 4);
380 OUT_RING(ring, CACHE_CLEAN | CP_EVENT_WRITE_0_IRQ | BIT(27));
381 OUT_RING(ring, lower_32_bits(rbmemptr(ring, fence)));
382 OUT_RING(ring, upper_32_bits(rbmemptr(ring, fence)));
383 OUT_RING(ring, submit->seqno);
384
385 OUT_PKT7(ring, CP_THREAD_CONTROL, 1);
386 OUT_RING(ring, CP_SET_THREAD_BOTH);
387
388 OUT_PKT7(ring, CP_SET_MARKER, 1);
389 OUT_RING(ring, 0x100); /* IFPC enable */
390
391 trace_msm_gpu_submit_flush(submit,
392 gpu_read64(gpu, REG_A6XX_CP_ALWAYS_ON_COUNTER));
393
394 a6xx_flush(gpu, ring);
395 }
396
397 const struct adreno_reglist a612_hwcg[] = {
398 {REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x22222222},
399 {REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
400 {REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000081},
401 {REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000f3cf},
402 {REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x22222222},
403 {REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
404 {REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222},
405 {REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222},
406 {REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
407 {REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
408 {REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111},
409 {REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111},
410 {REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
411 {REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
412 {REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777},
413 {REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777},
414 {REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
415 {REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01202222},
416 {REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220},
417 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040f00},
418 {REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x05522022},
419 {REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555},
420 {REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011},
421 {REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044},
422 {REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
423 {REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
424 {REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x02222222},
425 {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002},
426 {REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222},
427 {REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
428 {REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
429 {REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
430 {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
431 {REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
432 {REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
433 {REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
434 {REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000},
435 {REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
436 {REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004},
437 {REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
438 {REG_A6XX_RBBM_ISDB_CNT, 0x00000182},
439 {REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000},
440 {REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000},
441 {REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222},
442 {REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111},
443 {REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555},
444 {},
445 };
446
447 /* For a615 family (a615, a616, a618 and a619) */
448 const struct adreno_reglist a615_hwcg[] = {
449 {REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
450 {REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
451 {REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
452 {REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
453 {REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x02222222},
454 {REG_A6XX_RBBM_CLOCK_CNTL_TP1, 0x02222222},
455 {REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
456 {REG_A6XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222},
457 {REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222},
458 {REG_A6XX_RBBM_CLOCK_CNTL3_TP1, 0x22222222},
459 {REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222},
460 {REG_A6XX_RBBM_CLOCK_CNTL4_TP1, 0x00022222},
461 {REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
462 {REG_A6XX_RBBM_CLOCK_HYST_TP1, 0x77777777},
463 {REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
464 {REG_A6XX_RBBM_CLOCK_HYST2_TP1, 0x77777777},
465 {REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777},
466 {REG_A6XX_RBBM_CLOCK_HYST3_TP1, 0x77777777},
467 {REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777},
468 {REG_A6XX_RBBM_CLOCK_HYST4_TP1, 0x00077777},
469 {REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
470 {REG_A6XX_RBBM_CLOCK_DELAY_TP1, 0x11111111},
471 {REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
472 {REG_A6XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111},
473 {REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111},
474 {REG_A6XX_RBBM_CLOCK_DELAY3_TP1, 0x11111111},
475 {REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111},
476 {REG_A6XX_RBBM_CLOCK_DELAY4_TP1, 0x00011111},
477 {REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
478 {REG_A6XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222},
479 {REG_A6XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222},
480 {REG_A6XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222},
481 {REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004},
482 {REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
483 {REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
484 {REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x00002222},
485 {REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002020},
486 {REG_A6XX_RBBM_CLOCK_CNTL_CCU1, 0x00002220},
487 {REG_A6XX_RBBM_CLOCK_CNTL_CCU2, 0x00002220},
488 {REG_A6XX_RBBM_CLOCK_CNTL_CCU3, 0x00002220},
489 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00},
490 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU1, 0x00040F00},
491 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU2, 0x00040F00},
492 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU3, 0x00040F00},
493 {REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x05022022},
494 {REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555},
495 {REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011},
496 {REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044},
497 {REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
498 {REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222},
499 {REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
500 {REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
501 {REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
502 {REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
503 {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
504 {REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
505 {REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
506 {REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
507 {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002},
508 {REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222},
509 {REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222},
510 {REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111},
511 {REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555},
512 {},
513 };
514
515 const struct adreno_reglist a630_hwcg[] = {
516 {REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x22222222},
517 {REG_A6XX_RBBM_CLOCK_CNTL_SP1, 0x22222222},
518 {REG_A6XX_RBBM_CLOCK_CNTL_SP2, 0x22222222},
519 {REG_A6XX_RBBM_CLOCK_CNTL_SP3, 0x22222222},
520 {REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02022220},
521 {REG_A6XX_RBBM_CLOCK_CNTL2_SP1, 0x02022220},
522 {REG_A6XX_RBBM_CLOCK_CNTL2_SP2, 0x02022220},
523 {REG_A6XX_RBBM_CLOCK_CNTL2_SP3, 0x02022220},
524 {REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
525 {REG_A6XX_RBBM_CLOCK_DELAY_SP1, 0x00000080},
526 {REG_A6XX_RBBM_CLOCK_DELAY_SP2, 0x00000080},
527 {REG_A6XX_RBBM_CLOCK_DELAY_SP3, 0x00000080},
528 {REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000f3cf},
529 {REG_A6XX_RBBM_CLOCK_HYST_SP1, 0x0000f3cf},
530 {REG_A6XX_RBBM_CLOCK_HYST_SP2, 0x0000f3cf},
531 {REG_A6XX_RBBM_CLOCK_HYST_SP3, 0x0000f3cf},
532 {REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x02222222},
533 {REG_A6XX_RBBM_CLOCK_CNTL_TP1, 0x02222222},
534 {REG_A6XX_RBBM_CLOCK_CNTL_TP2, 0x02222222},
535 {REG_A6XX_RBBM_CLOCK_CNTL_TP3, 0x02222222},
536 {REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
537 {REG_A6XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222},
538 {REG_A6XX_RBBM_CLOCK_CNTL2_TP2, 0x22222222},
539 {REG_A6XX_RBBM_CLOCK_CNTL2_TP3, 0x22222222},
540 {REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222},
541 {REG_A6XX_RBBM_CLOCK_CNTL3_TP1, 0x22222222},
542 {REG_A6XX_RBBM_CLOCK_CNTL3_TP2, 0x22222222},
543 {REG_A6XX_RBBM_CLOCK_CNTL3_TP3, 0x22222222},
544 {REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222},
545 {REG_A6XX_RBBM_CLOCK_CNTL4_TP1, 0x00022222},
546 {REG_A6XX_RBBM_CLOCK_CNTL4_TP2, 0x00022222},
547 {REG_A6XX_RBBM_CLOCK_CNTL4_TP3, 0x00022222},
548 {REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
549 {REG_A6XX_RBBM_CLOCK_HYST_TP1, 0x77777777},
550 {REG_A6XX_RBBM_CLOCK_HYST_TP2, 0x77777777},
551 {REG_A6XX_RBBM_CLOCK_HYST_TP3, 0x77777777},
552 {REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
553 {REG_A6XX_RBBM_CLOCK_HYST2_TP1, 0x77777777},
554 {REG_A6XX_RBBM_CLOCK_HYST2_TP2, 0x77777777},
555 {REG_A6XX_RBBM_CLOCK_HYST2_TP3, 0x77777777},
556 {REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777},
557 {REG_A6XX_RBBM_CLOCK_HYST3_TP1, 0x77777777},
558 {REG_A6XX_RBBM_CLOCK_HYST3_TP2, 0x77777777},
559 {REG_A6XX_RBBM_CLOCK_HYST3_TP3, 0x77777777},
560 {REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777},
561 {REG_A6XX_RBBM_CLOCK_HYST4_TP1, 0x00077777},
562 {REG_A6XX_RBBM_CLOCK_HYST4_TP2, 0x00077777},
563 {REG_A6XX_RBBM_CLOCK_HYST4_TP3, 0x00077777},
564 {REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
565 {REG_A6XX_RBBM_CLOCK_DELAY_TP1, 0x11111111},
566 {REG_A6XX_RBBM_CLOCK_DELAY_TP2, 0x11111111},
567 {REG_A6XX_RBBM_CLOCK_DELAY_TP3, 0x11111111},
568 {REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
569 {REG_A6XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111},
570 {REG_A6XX_RBBM_CLOCK_DELAY2_TP2, 0x11111111},
571 {REG_A6XX_RBBM_CLOCK_DELAY2_TP3, 0x11111111},
572 {REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111},
573 {REG_A6XX_RBBM_CLOCK_DELAY3_TP1, 0x11111111},
574 {REG_A6XX_RBBM_CLOCK_DELAY3_TP2, 0x11111111},
575 {REG_A6XX_RBBM_CLOCK_DELAY3_TP3, 0x11111111},
576 {REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111},
577 {REG_A6XX_RBBM_CLOCK_DELAY4_TP1, 0x00011111},
578 {REG_A6XX_RBBM_CLOCK_DELAY4_TP2, 0x00011111},
579 {REG_A6XX_RBBM_CLOCK_DELAY4_TP3, 0x00011111},
580 {REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
581 {REG_A6XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222},
582 {REG_A6XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222},
583 {REG_A6XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222},
584 {REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004},
585 {REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
586 {REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
587 {REG_A6XX_RBBM_CLOCK_CNTL_RB1, 0x22222222},
588 {REG_A6XX_RBBM_CLOCK_CNTL_RB2, 0x22222222},
589 {REG_A6XX_RBBM_CLOCK_CNTL_RB3, 0x22222222},
590 {REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x00002222},
591 {REG_A6XX_RBBM_CLOCK_CNTL2_RB1, 0x00002222},
592 {REG_A6XX_RBBM_CLOCK_CNTL2_RB2, 0x00002222},
593 {REG_A6XX_RBBM_CLOCK_CNTL2_RB3, 0x00002222},
594 {REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220},
595 {REG_A6XX_RBBM_CLOCK_CNTL_CCU1, 0x00002220},
596 {REG_A6XX_RBBM_CLOCK_CNTL_CCU2, 0x00002220},
597 {REG_A6XX_RBBM_CLOCK_CNTL_CCU3, 0x00002220},
598 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040f00},
599 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU1, 0x00040f00},
600 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU2, 0x00040f00},
601 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU3, 0x00040f00},
602 {REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x05022022},
603 {REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555},
604 {REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011},
605 {REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044},
606 {REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
607 {REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222},
608 {REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
609 {REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
610 {REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
611 {REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
612 {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
613 {REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
614 {REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
615 {REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
616 {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002},
617 {REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222},
618 {REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222},
619 {REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111},
620 {REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555},
621 {},
622 };
623
624 const struct adreno_reglist a640_hwcg[] = {
625 {REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
626 {REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
627 {REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
628 {REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
629 {REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x02222222},
630 {REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
631 {REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222},
632 {REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222},
633 {REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
634 {REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
635 {REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111},
636 {REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111},
637 {REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
638 {REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
639 {REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777},
640 {REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777},
641 {REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
642 {REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222},
643 {REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220},
644 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00},
645 {REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x05222022},
646 {REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555},
647 {REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011},
648 {REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044},
649 {REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
650 {REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
651 {REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222},
652 {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002},
653 {REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222},
654 {REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
655 {REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
656 {REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
657 {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
658 {REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
659 {REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
660 {REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
661 {REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000},
662 {REG_A6XX_RBBM_CLOCK_CNTL_TEX_FCHE, 0x00000222},
663 {REG_A6XX_RBBM_CLOCK_DELAY_TEX_FCHE, 0x00000111},
664 {REG_A6XX_RBBM_CLOCK_HYST_TEX_FCHE, 0x00000000},
665 {REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
666 {REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004},
667 {REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
668 {REG_A6XX_RBBM_ISDB_CNT, 0x00000182},
669 {REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000},
670 {REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000},
671 {REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222},
672 {REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111},
673 {REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555},
674 {},
675 };
676
677 const struct adreno_reglist a650_hwcg[] = {
678 {REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
679 {REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
680 {REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
681 {REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
682 {REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x02222222},
683 {REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
684 {REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222},
685 {REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222},
686 {REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
687 {REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
688 {REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111},
689 {REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111},
690 {REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
691 {REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
692 {REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777},
693 {REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777},
694 {REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
695 {REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222},
696 {REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220},
697 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00},
698 {REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x25222022},
699 {REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555},
700 {REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011},
701 {REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044},
702 {REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
703 {REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
704 {REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222},
705 {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002},
706 {REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222},
707 {REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
708 {REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
709 {REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
710 {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
711 {REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
712 {REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
713 {REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
714 {REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000},
715 {REG_A6XX_RBBM_CLOCK_CNTL_TEX_FCHE, 0x00000222},
716 {REG_A6XX_RBBM_CLOCK_DELAY_TEX_FCHE, 0x00000111},
717 {REG_A6XX_RBBM_CLOCK_HYST_TEX_FCHE, 0x00000777},
718 {REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
719 {REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004},
720 {REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
721 {REG_A6XX_RBBM_ISDB_CNT, 0x00000182},
722 {REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000},
723 {REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000},
724 {REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222},
725 {REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111},
726 {REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555},
727 {},
728 };
729
730 const struct adreno_reglist a660_hwcg[] = {
731 {REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
732 {REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
733 {REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
734 {REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
735 {REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x22222222},
736 {REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
737 {REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222},
738 {REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222},
739 {REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
740 {REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
741 {REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111},
742 {REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111},
743 {REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
744 {REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
745 {REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777},
746 {REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777},
747 {REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
748 {REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222},
749 {REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220},
750 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00},
751 {REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x25222022},
752 {REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555},
753 {REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011},
754 {REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044},
755 {REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
756 {REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
757 {REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222},
758 {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002},
759 {REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222},
760 {REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
761 {REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
762 {REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
763 {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
764 {REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
765 {REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
766 {REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
767 {REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000},
768 {REG_A6XX_RBBM_CLOCK_CNTL_TEX_FCHE, 0x00000222},
769 {REG_A6XX_RBBM_CLOCK_DELAY_TEX_FCHE, 0x00000111},
770 {REG_A6XX_RBBM_CLOCK_HYST_TEX_FCHE, 0x00000000},
771 {REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
772 {REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004},
773 {REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
774 {REG_A6XX_RBBM_ISDB_CNT, 0x00000182},
775 {REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000},
776 {REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000},
777 {REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222},
778 {REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111},
779 {REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555},
780 {},
781 };
782
783 const struct adreno_reglist a690_hwcg[] = {
784 {REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222},
785 {REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02222220},
786 {REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080},
787 {REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000F3CF},
788 {REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x22222222},
789 {REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222},
790 {REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222},
791 {REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222},
792 {REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111},
793 {REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111},
794 {REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111},
795 {REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111},
796 {REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777},
797 {REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777},
798 {REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777},
799 {REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777},
800 {REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222},
801 {REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222},
802 {REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220},
803 {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040F00},
804 {REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x25222022},
805 {REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555},
806 {REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011},
807 {REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044},
808 {REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222},
809 {REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222},
810 {REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222},
811 {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002},
812 {REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222},
813 {REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000},
814 {REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222},
815 {REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200},
816 {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000},
817 {REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000},
818 {REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000},
819 {REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004},
820 {REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000},
821 {REG_A6XX_RBBM_CLOCK_CNTL_TEX_FCHE, 0x00000222},
822 {REG_A6XX_RBBM_CLOCK_DELAY_TEX_FCHE, 0x00000111},
823 {REG_A6XX_RBBM_CLOCK_HYST_TEX_FCHE, 0x00000000},
824 {REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222},
825 {REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004},
826 {REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002},
827 {REG_A6XX_RBBM_CLOCK_CNTL, 0x8AA8AA82},
828 {REG_A6XX_RBBM_ISDB_CNT, 0x00000182},
829 {REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000},
830 {REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000},
831 {REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222},
832 {REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111},
833 {REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555},
834 {REG_A6XX_GPU_GMU_AO_GMU_CGC_MODE_CNTL, 0x20200},
835 {REG_A6XX_GPU_GMU_AO_GMU_CGC_DELAY_CNTL, 0x10111},
836 {REG_A6XX_GPU_GMU_AO_GMU_CGC_HYST_CNTL, 0x5555},
837 {}
838 };
839
840 const struct adreno_reglist a730_hwcg[] = {
841 { REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222 },
842 { REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02022222 },
843 { REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000f3cf },
844 { REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080 },
845 { REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x22222220 },
846 { REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222 },
847 { REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222 },
848 { REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00222222 },
849 { REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777 },
850 { REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777 },
851 { REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777 },
852 { REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777 },
853 { REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111 },
854 { REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111 },
855 { REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111 },
856 { REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111 },
857 { REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222 },
858 { REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004 },
859 { REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002 },
860 { REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222 },
861 { REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222 },
862 { REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220 },
863 { REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x44000f00 },
864 { REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x25222022 },
865 { REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00555555 },
866 { REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011 },
867 { REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00440044 },
868 { REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222 },
869 { REG_A7XX_RBBM_CLOCK_MODE2_GRAS, 0x00000222 },
870 { REG_A7XX_RBBM_CLOCK_MODE_BV_GRAS, 0x00222222 },
871 { REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x02222223 },
872 { REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222 },
873 { REG_A7XX_RBBM_CLOCK_MODE_BV_GPC, 0x00222222 },
874 { REG_A7XX_RBBM_CLOCK_MODE_BV_VFD, 0x00002222 },
875 { REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000 },
876 { REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004 },
877 { REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000 },
878 { REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000 },
879 { REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200 },
880 { REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222 },
881 { REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222 },
882 { REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000 },
883 { REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000 },
884 { REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002 },
885 { REG_A7XX_RBBM_CLOCK_MODE_BV_LRZ, 0x55555552 },
886 { REG_A7XX_RBBM_CLOCK_MODE_CP, 0x00000223 },
887 { REG_A6XX_RBBM_CLOCK_CNTL, 0x8aa8aa82 },
888 { REG_A6XX_RBBM_ISDB_CNT, 0x00000182 },
889 { REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000 },
890 { REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000 },
891 { REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222 },
892 { REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111 },
893 { REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555 },
894 {},
895 };
896
897 const struct adreno_reglist a740_hwcg[] = {
898 { REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x02222222 },
899 { REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x22022222 },
900 { REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x003cf3cf },
901 { REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080 },
902 { REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x22222220 },
903 { REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222 },
904 { REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222 },
905 { REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00222222 },
906 { REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777 },
907 { REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777 },
908 { REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777 },
909 { REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777 },
910 { REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111 },
911 { REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111 },
912 { REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111 },
913 { REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111 },
914 { REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222 },
915 { REG_A6XX_RBBM_CLOCK_CNTL2_UCHE, 0x00222222 },
916 { REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000444 },
917 { REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000222 },
918 { REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222 },
919 { REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x01002222 },
920 { REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220 },
921 { REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x44000f00 },
922 { REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x25222022 },
923 { REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00555555 },
924 { REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011 },
925 { REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00440044 },
926 { REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222 },
927 { REG_A7XX_RBBM_CLOCK_MODE2_GRAS, 0x00000222 },
928 { REG_A7XX_RBBM_CLOCK_MODE_BV_GRAS, 0x00222222 },
929 { REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x02222223 },
930 { REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00222222 },
931 { REG_A7XX_RBBM_CLOCK_MODE_BV_GPC, 0x00222222 },
932 { REG_A7XX_RBBM_CLOCK_MODE_BV_VFD, 0x00002222 },
933 { REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000 },
934 { REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004 },
935 { REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000 },
936 { REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00000000 },
937 { REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200 },
938 { REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00000000 },
939 { REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222 },
940 { REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000 },
941 { REG_A6XX_RBBM_CLOCK_HYST_HLSQ, 0x00000000 },
942 { REG_A7XX_RBBM_CLOCK_MODE_BV_LRZ, 0x55555552 },
943 { REG_A7XX_RBBM_CLOCK_HYST2_VFD, 0x00000000 },
944 { REG_A7XX_RBBM_CLOCK_MODE_CP, 0x00000222 },
945 { REG_A6XX_RBBM_CLOCK_CNTL, 0x8aa8aa82 },
946 { REG_A6XX_RBBM_ISDB_CNT, 0x00000182 },
947 { REG_A6XX_RBBM_RAC_THRESHOLD_CNT, 0x00000000 },
948 { REG_A6XX_RBBM_SP_HYST_CNT, 0x00000000 },
949 { REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222 },
950 { REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111 },
951 { REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555 },
952 {},
953 };
954
a6xx_set_hwcg(struct msm_gpu * gpu,bool state)955 static void a6xx_set_hwcg(struct msm_gpu *gpu, bool state)
956 {
957 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
958 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
959 struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
960 const struct adreno_reglist *reg;
961 unsigned int i;
962 u32 val, clock_cntl_on, cgc_mode;
963
964 if (!adreno_gpu->info->hwcg)
965 return;
966
967 if (adreno_is_a630(adreno_gpu))
968 clock_cntl_on = 0x8aa8aa02;
969 else if (adreno_is_a610(adreno_gpu))
970 clock_cntl_on = 0xaaa8aa82;
971 else
972 clock_cntl_on = 0x8aa8aa82;
973
974 if (adreno_is_a7xx(adreno_gpu)) {
975 cgc_mode = adreno_is_a740_family(adreno_gpu) ? 0x20222 : 0x20000;
976
977 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GPU_GMU_AO_GMU_CGC_MODE_CNTL,
978 state ? cgc_mode : 0);
979 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GPU_GMU_AO_GMU_CGC_DELAY_CNTL,
980 state ? 0x10111 : 0);
981 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GPU_GMU_AO_GMU_CGC_HYST_CNTL,
982 state ? 0x5555 : 0);
983 }
984
985 val = gpu_read(gpu, REG_A6XX_RBBM_CLOCK_CNTL);
986
987 /* Don't re-program the registers if they are already correct */
988 if ((!state && !val) || (state && (val == clock_cntl_on)))
989 return;
990
991 /* Disable SP clock before programming HWCG registers */
992 if (!adreno_is_a610(adreno_gpu) && !adreno_is_a7xx(adreno_gpu))
993 gmu_rmw(gmu, REG_A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL, 1, 0);
994
995 for (i = 0; (reg = &adreno_gpu->info->hwcg[i], reg->offset); i++)
996 gpu_write(gpu, reg->offset, state ? reg->value : 0);
997
998 /* Enable SP clock */
999 if (!adreno_is_a610(adreno_gpu) && !adreno_is_a7xx(adreno_gpu))
1000 gmu_rmw(gmu, REG_A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL, 0, 1);
1001
1002 gpu_write(gpu, REG_A6XX_RBBM_CLOCK_CNTL, state ? clock_cntl_on : 0);
1003 }
1004
1005 /* For a615, a616, a618, a619, a630, a640 and a680 */
1006 static const u32 a6xx_protect[] = {
1007 A6XX_PROTECT_RDONLY(0x00000, 0x04ff),
1008 A6XX_PROTECT_RDONLY(0x00501, 0x0005),
1009 A6XX_PROTECT_RDONLY(0x0050b, 0x02f4),
1010 A6XX_PROTECT_NORDWR(0x0050e, 0x0000),
1011 A6XX_PROTECT_NORDWR(0x00510, 0x0000),
1012 A6XX_PROTECT_NORDWR(0x00534, 0x0000),
1013 A6XX_PROTECT_NORDWR(0x00800, 0x0082),
1014 A6XX_PROTECT_NORDWR(0x008a0, 0x0008),
1015 A6XX_PROTECT_NORDWR(0x008ab, 0x0024),
1016 A6XX_PROTECT_RDONLY(0x008de, 0x00ae),
1017 A6XX_PROTECT_NORDWR(0x00900, 0x004d),
1018 A6XX_PROTECT_NORDWR(0x0098d, 0x0272),
1019 A6XX_PROTECT_NORDWR(0x00e00, 0x0001),
1020 A6XX_PROTECT_NORDWR(0x00e03, 0x000c),
1021 A6XX_PROTECT_NORDWR(0x03c00, 0x00c3),
1022 A6XX_PROTECT_RDONLY(0x03cc4, 0x1fff),
1023 A6XX_PROTECT_NORDWR(0x08630, 0x01cf),
1024 A6XX_PROTECT_NORDWR(0x08e00, 0x0000),
1025 A6XX_PROTECT_NORDWR(0x08e08, 0x0000),
1026 A6XX_PROTECT_NORDWR(0x08e50, 0x001f),
1027 A6XX_PROTECT_NORDWR(0x09624, 0x01db),
1028 A6XX_PROTECT_NORDWR(0x09e70, 0x0001),
1029 A6XX_PROTECT_NORDWR(0x09e78, 0x0187),
1030 A6XX_PROTECT_NORDWR(0x0a630, 0x01cf),
1031 A6XX_PROTECT_NORDWR(0x0ae02, 0x0000),
1032 A6XX_PROTECT_NORDWR(0x0ae50, 0x032f),
1033 A6XX_PROTECT_NORDWR(0x0b604, 0x0000),
1034 A6XX_PROTECT_NORDWR(0x0be02, 0x0001),
1035 A6XX_PROTECT_NORDWR(0x0be20, 0x17df),
1036 A6XX_PROTECT_NORDWR(0x0f000, 0x0bff),
1037 A6XX_PROTECT_RDONLY(0x0fc00, 0x1fff),
1038 A6XX_PROTECT_NORDWR(0x11c00, 0x0000), /* note: infinite range */
1039 };
1040
1041 /* These are for a620 and a650 */
1042 static const u32 a650_protect[] = {
1043 A6XX_PROTECT_RDONLY(0x00000, 0x04ff),
1044 A6XX_PROTECT_RDONLY(0x00501, 0x0005),
1045 A6XX_PROTECT_RDONLY(0x0050b, 0x02f4),
1046 A6XX_PROTECT_NORDWR(0x0050e, 0x0000),
1047 A6XX_PROTECT_NORDWR(0x00510, 0x0000),
1048 A6XX_PROTECT_NORDWR(0x00534, 0x0000),
1049 A6XX_PROTECT_NORDWR(0x00800, 0x0082),
1050 A6XX_PROTECT_NORDWR(0x008a0, 0x0008),
1051 A6XX_PROTECT_NORDWR(0x008ab, 0x0024),
1052 A6XX_PROTECT_RDONLY(0x008de, 0x00ae),
1053 A6XX_PROTECT_NORDWR(0x00900, 0x004d),
1054 A6XX_PROTECT_NORDWR(0x0098d, 0x0272),
1055 A6XX_PROTECT_NORDWR(0x00e00, 0x0001),
1056 A6XX_PROTECT_NORDWR(0x00e03, 0x000c),
1057 A6XX_PROTECT_NORDWR(0x03c00, 0x00c3),
1058 A6XX_PROTECT_RDONLY(0x03cc4, 0x1fff),
1059 A6XX_PROTECT_NORDWR(0x08630, 0x01cf),
1060 A6XX_PROTECT_NORDWR(0x08e00, 0x0000),
1061 A6XX_PROTECT_NORDWR(0x08e08, 0x0000),
1062 A6XX_PROTECT_NORDWR(0x08e50, 0x001f),
1063 A6XX_PROTECT_NORDWR(0x08e80, 0x027f),
1064 A6XX_PROTECT_NORDWR(0x09624, 0x01db),
1065 A6XX_PROTECT_NORDWR(0x09e60, 0x0011),
1066 A6XX_PROTECT_NORDWR(0x09e78, 0x0187),
1067 A6XX_PROTECT_NORDWR(0x0a630, 0x01cf),
1068 A6XX_PROTECT_NORDWR(0x0ae02, 0x0000),
1069 A6XX_PROTECT_NORDWR(0x0ae50, 0x032f),
1070 A6XX_PROTECT_NORDWR(0x0b604, 0x0000),
1071 A6XX_PROTECT_NORDWR(0x0b608, 0x0007),
1072 A6XX_PROTECT_NORDWR(0x0be02, 0x0001),
1073 A6XX_PROTECT_NORDWR(0x0be20, 0x17df),
1074 A6XX_PROTECT_NORDWR(0x0f000, 0x0bff),
1075 A6XX_PROTECT_RDONLY(0x0fc00, 0x1fff),
1076 A6XX_PROTECT_NORDWR(0x18400, 0x1fff),
1077 A6XX_PROTECT_NORDWR(0x1a800, 0x1fff),
1078 A6XX_PROTECT_NORDWR(0x1f400, 0x0443),
1079 A6XX_PROTECT_RDONLY(0x1f844, 0x007b),
1080 A6XX_PROTECT_NORDWR(0x1f887, 0x001b),
1081 A6XX_PROTECT_NORDWR(0x1f8c0, 0x0000), /* note: infinite range */
1082 };
1083
1084 /* These are for a635 and a660 */
1085 static const u32 a660_protect[] = {
1086 A6XX_PROTECT_RDONLY(0x00000, 0x04ff),
1087 A6XX_PROTECT_RDONLY(0x00501, 0x0005),
1088 A6XX_PROTECT_RDONLY(0x0050b, 0x02f4),
1089 A6XX_PROTECT_NORDWR(0x0050e, 0x0000),
1090 A6XX_PROTECT_NORDWR(0x00510, 0x0000),
1091 A6XX_PROTECT_NORDWR(0x00534, 0x0000),
1092 A6XX_PROTECT_NORDWR(0x00800, 0x0082),
1093 A6XX_PROTECT_NORDWR(0x008a0, 0x0008),
1094 A6XX_PROTECT_NORDWR(0x008ab, 0x0024),
1095 A6XX_PROTECT_RDONLY(0x008de, 0x00ae),
1096 A6XX_PROTECT_NORDWR(0x00900, 0x004d),
1097 A6XX_PROTECT_NORDWR(0x0098d, 0x0272),
1098 A6XX_PROTECT_NORDWR(0x00e00, 0x0001),
1099 A6XX_PROTECT_NORDWR(0x00e03, 0x000c),
1100 A6XX_PROTECT_NORDWR(0x03c00, 0x00c3),
1101 A6XX_PROTECT_RDONLY(0x03cc4, 0x1fff),
1102 A6XX_PROTECT_NORDWR(0x08630, 0x01cf),
1103 A6XX_PROTECT_NORDWR(0x08e00, 0x0000),
1104 A6XX_PROTECT_NORDWR(0x08e08, 0x0000),
1105 A6XX_PROTECT_NORDWR(0x08e50, 0x001f),
1106 A6XX_PROTECT_NORDWR(0x08e80, 0x027f),
1107 A6XX_PROTECT_NORDWR(0x09624, 0x01db),
1108 A6XX_PROTECT_NORDWR(0x09e60, 0x0011),
1109 A6XX_PROTECT_NORDWR(0x09e78, 0x0187),
1110 A6XX_PROTECT_NORDWR(0x0a630, 0x01cf),
1111 A6XX_PROTECT_NORDWR(0x0ae02, 0x0000),
1112 A6XX_PROTECT_NORDWR(0x0ae50, 0x012f),
1113 A6XX_PROTECT_NORDWR(0x0b604, 0x0000),
1114 A6XX_PROTECT_NORDWR(0x0b608, 0x0006),
1115 A6XX_PROTECT_NORDWR(0x0be02, 0x0001),
1116 A6XX_PROTECT_NORDWR(0x0be20, 0x015f),
1117 A6XX_PROTECT_NORDWR(0x0d000, 0x05ff),
1118 A6XX_PROTECT_NORDWR(0x0f000, 0x0bff),
1119 A6XX_PROTECT_RDONLY(0x0fc00, 0x1fff),
1120 A6XX_PROTECT_NORDWR(0x18400, 0x1fff),
1121 A6XX_PROTECT_NORDWR(0x1a400, 0x1fff),
1122 A6XX_PROTECT_NORDWR(0x1f400, 0x0443),
1123 A6XX_PROTECT_RDONLY(0x1f844, 0x007b),
1124 A6XX_PROTECT_NORDWR(0x1f860, 0x0000),
1125 A6XX_PROTECT_NORDWR(0x1f887, 0x001b),
1126 A6XX_PROTECT_NORDWR(0x1f8c0, 0x0000), /* note: infinite range */
1127 };
1128
1129 /* These are for a690 */
1130 static const u32 a690_protect[] = {
1131 A6XX_PROTECT_RDONLY(0x00000, 0x004ff),
1132 A6XX_PROTECT_RDONLY(0x00501, 0x00001),
1133 A6XX_PROTECT_RDONLY(0x0050b, 0x002f4),
1134 A6XX_PROTECT_NORDWR(0x0050e, 0x00000),
1135 A6XX_PROTECT_NORDWR(0x00510, 0x00000),
1136 A6XX_PROTECT_NORDWR(0x00534, 0x00000),
1137 A6XX_PROTECT_NORDWR(0x00800, 0x00082),
1138 A6XX_PROTECT_NORDWR(0x008a0, 0x00008),
1139 A6XX_PROTECT_NORDWR(0x008ab, 0x00024),
1140 A6XX_PROTECT_RDONLY(0x008de, 0x000ae),
1141 A6XX_PROTECT_NORDWR(0x00900, 0x0004d),
1142 A6XX_PROTECT_NORDWR(0x0098d, 0x00272),
1143 A6XX_PROTECT_NORDWR(0x00e00, 0x00001),
1144 A6XX_PROTECT_NORDWR(0x00e03, 0x0000c),
1145 A6XX_PROTECT_NORDWR(0x03c00, 0x000c3),
1146 A6XX_PROTECT_RDONLY(0x03cc4, 0x01fff),
1147 A6XX_PROTECT_NORDWR(0x08630, 0x001cf),
1148 A6XX_PROTECT_NORDWR(0x08e00, 0x00000),
1149 A6XX_PROTECT_NORDWR(0x08e08, 0x00007),
1150 A6XX_PROTECT_NORDWR(0x08e50, 0x0001f),
1151 A6XX_PROTECT_NORDWR(0x08e80, 0x0027f),
1152 A6XX_PROTECT_NORDWR(0x09624, 0x001db),
1153 A6XX_PROTECT_NORDWR(0x09e60, 0x00011),
1154 A6XX_PROTECT_NORDWR(0x09e78, 0x00187),
1155 A6XX_PROTECT_NORDWR(0x0a630, 0x001cf),
1156 A6XX_PROTECT_NORDWR(0x0ae02, 0x00000),
1157 A6XX_PROTECT_NORDWR(0x0ae50, 0x0012f),
1158 A6XX_PROTECT_NORDWR(0x0b604, 0x00000),
1159 A6XX_PROTECT_NORDWR(0x0b608, 0x00006),
1160 A6XX_PROTECT_NORDWR(0x0be02, 0x00001),
1161 A6XX_PROTECT_NORDWR(0x0be20, 0x0015f),
1162 A6XX_PROTECT_NORDWR(0x0d000, 0x005ff),
1163 A6XX_PROTECT_NORDWR(0x0f000, 0x00bff),
1164 A6XX_PROTECT_RDONLY(0x0fc00, 0x01fff),
1165 A6XX_PROTECT_NORDWR(0x11c00, 0x00000), /*note: infiite range */
1166 };
1167
1168 static const u32 a730_protect[] = {
1169 A6XX_PROTECT_RDONLY(0x00000, 0x04ff),
1170 A6XX_PROTECT_RDONLY(0x0050b, 0x0058),
1171 A6XX_PROTECT_NORDWR(0x0050e, 0x0000),
1172 A6XX_PROTECT_NORDWR(0x00510, 0x0000),
1173 A6XX_PROTECT_NORDWR(0x00534, 0x0000),
1174 A6XX_PROTECT_RDONLY(0x005fb, 0x009d),
1175 A6XX_PROTECT_NORDWR(0x00699, 0x01e9),
1176 A6XX_PROTECT_NORDWR(0x008a0, 0x0008),
1177 A6XX_PROTECT_NORDWR(0x008ab, 0x0024),
1178 /* 0x008d0-0x008dd are unprotected on purpose for tools like perfetto */
1179 A6XX_PROTECT_RDONLY(0x008de, 0x0154),
1180 A6XX_PROTECT_NORDWR(0x00900, 0x004d),
1181 A6XX_PROTECT_NORDWR(0x0098d, 0x00b2),
1182 A6XX_PROTECT_NORDWR(0x00a41, 0x01be),
1183 A6XX_PROTECT_NORDWR(0x00df0, 0x0001),
1184 A6XX_PROTECT_NORDWR(0x00e01, 0x0000),
1185 A6XX_PROTECT_NORDWR(0x00e07, 0x0008),
1186 A6XX_PROTECT_NORDWR(0x03c00, 0x00c3),
1187 A6XX_PROTECT_RDONLY(0x03cc4, 0x1fff),
1188 A6XX_PROTECT_NORDWR(0x08630, 0x01cf),
1189 A6XX_PROTECT_NORDWR(0x08e00, 0x0000),
1190 A6XX_PROTECT_NORDWR(0x08e08, 0x0000),
1191 A6XX_PROTECT_NORDWR(0x08e50, 0x001f),
1192 A6XX_PROTECT_NORDWR(0x08e80, 0x0280),
1193 A6XX_PROTECT_NORDWR(0x09624, 0x01db),
1194 A6XX_PROTECT_NORDWR(0x09e40, 0x0000),
1195 A6XX_PROTECT_NORDWR(0x09e64, 0x000d),
1196 A6XX_PROTECT_NORDWR(0x09e78, 0x0187),
1197 A6XX_PROTECT_NORDWR(0x0a630, 0x01cf),
1198 A6XX_PROTECT_NORDWR(0x0ae02, 0x0000),
1199 A6XX_PROTECT_NORDWR(0x0ae50, 0x000f),
1200 A6XX_PROTECT_NORDWR(0x0ae66, 0x0003),
1201 A6XX_PROTECT_NORDWR(0x0ae6f, 0x0003),
1202 A6XX_PROTECT_NORDWR(0x0b604, 0x0003),
1203 A6XX_PROTECT_NORDWR(0x0ec00, 0x0fff),
1204 A6XX_PROTECT_RDONLY(0x0fc00, 0x1fff),
1205 A6XX_PROTECT_NORDWR(0x18400, 0x0053),
1206 A6XX_PROTECT_RDONLY(0x18454, 0x0004),
1207 A6XX_PROTECT_NORDWR(0x18459, 0x1fff),
1208 A6XX_PROTECT_NORDWR(0x1a459, 0x1fff),
1209 A6XX_PROTECT_NORDWR(0x1c459, 0x1fff),
1210 A6XX_PROTECT_NORDWR(0x1f400, 0x0443),
1211 A6XX_PROTECT_RDONLY(0x1f844, 0x007b),
1212 A6XX_PROTECT_NORDWR(0x1f860, 0x0000),
1213 A6XX_PROTECT_NORDWR(0x1f878, 0x002a),
1214 /* CP_PROTECT_REG[44, 46] are left untouched! */
1215 0,
1216 0,
1217 0,
1218 A6XX_PROTECT_NORDWR(0x1f8c0, 0x00000),
1219 };
1220
a6xx_set_cp_protect(struct msm_gpu * gpu)1221 static void a6xx_set_cp_protect(struct msm_gpu *gpu)
1222 {
1223 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1224 const u32 *regs = a6xx_protect;
1225 unsigned i, count, count_max;
1226
1227 if (adreno_is_a650(adreno_gpu)) {
1228 regs = a650_protect;
1229 count = ARRAY_SIZE(a650_protect);
1230 count_max = 48;
1231 BUILD_BUG_ON(ARRAY_SIZE(a650_protect) > 48);
1232 } else if (adreno_is_a690(adreno_gpu)) {
1233 regs = a690_protect;
1234 count = ARRAY_SIZE(a690_protect);
1235 count_max = 48;
1236 BUILD_BUG_ON(ARRAY_SIZE(a690_protect) > 48);
1237 } else if (adreno_is_a660_family(adreno_gpu)) {
1238 regs = a660_protect;
1239 count = ARRAY_SIZE(a660_protect);
1240 count_max = 48;
1241 BUILD_BUG_ON(ARRAY_SIZE(a660_protect) > 48);
1242 } else if (adreno_is_a730(adreno_gpu) || adreno_is_a740(adreno_gpu)) {
1243 regs = a730_protect;
1244 count = ARRAY_SIZE(a730_protect);
1245 count_max = 48;
1246 BUILD_BUG_ON(ARRAY_SIZE(a730_protect) > 48);
1247 } else {
1248 regs = a6xx_protect;
1249 count = ARRAY_SIZE(a6xx_protect);
1250 count_max = 32;
1251 BUILD_BUG_ON(ARRAY_SIZE(a6xx_protect) > 32);
1252 }
1253
1254 /*
1255 * Enable access protection to privileged registers, fault on an access
1256 * protect violation and select the last span to protect from the start
1257 * address all the way to the end of the register address space
1258 */
1259 gpu_write(gpu, REG_A6XX_CP_PROTECT_CNTL,
1260 A6XX_CP_PROTECT_CNTL_ACCESS_PROT_EN |
1261 A6XX_CP_PROTECT_CNTL_ACCESS_FAULT_ON_VIOL_EN |
1262 A6XX_CP_PROTECT_CNTL_LAST_SPAN_INF_RANGE);
1263
1264 for (i = 0; i < count - 1; i++) {
1265 /* Intentionally skip writing to some registers */
1266 if (regs[i])
1267 gpu_write(gpu, REG_A6XX_CP_PROTECT(i), regs[i]);
1268 }
1269 /* last CP_PROTECT to have "infinite" length on the last entry */
1270 gpu_write(gpu, REG_A6XX_CP_PROTECT(count_max - 1), regs[i]);
1271 }
1272
a6xx_calc_ubwc_config(struct adreno_gpu * gpu)1273 static void a6xx_calc_ubwc_config(struct adreno_gpu *gpu)
1274 {
1275 /* Unknown, introduced with A650 family, related to UBWC mode/ver 4 */
1276 gpu->ubwc_config.rgb565_predicator = 0;
1277 /* Unknown, introduced with A650 family */
1278 gpu->ubwc_config.uavflagprd_inv = 0;
1279 /* Whether the minimum access length is 64 bits */
1280 gpu->ubwc_config.min_acc_len = 0;
1281 /* Entirely magic, per-GPU-gen value */
1282 gpu->ubwc_config.ubwc_mode = 0;
1283 /*
1284 * The Highest Bank Bit value represents the bit of the highest DDR bank.
1285 * This should ideally use DRAM type detection.
1286 */
1287 gpu->ubwc_config.highest_bank_bit = 15;
1288
1289 if (adreno_is_a610(gpu)) {
1290 gpu->ubwc_config.highest_bank_bit = 13;
1291 gpu->ubwc_config.min_acc_len = 1;
1292 gpu->ubwc_config.ubwc_mode = 1;
1293 }
1294
1295 /* a618 is using the hw default values */
1296 if (adreno_is_a618(gpu))
1297 return;
1298
1299 if (adreno_is_a619_holi(gpu))
1300 gpu->ubwc_config.highest_bank_bit = 13;
1301
1302 if (adreno_is_a640_family(gpu))
1303 gpu->ubwc_config.amsbc = 1;
1304
1305 if (adreno_is_a650(gpu) ||
1306 adreno_is_a660(gpu) ||
1307 adreno_is_a690(gpu) ||
1308 adreno_is_a730(gpu) ||
1309 adreno_is_a740_family(gpu)) {
1310 /* TODO: get ddr type from bootloader and use 2 for LPDDR4 */
1311 gpu->ubwc_config.highest_bank_bit = 16;
1312 gpu->ubwc_config.amsbc = 1;
1313 gpu->ubwc_config.rgb565_predicator = 1;
1314 gpu->ubwc_config.uavflagprd_inv = 2;
1315 }
1316
1317 if (adreno_is_7c3(gpu)) {
1318 gpu->ubwc_config.highest_bank_bit = 14;
1319 gpu->ubwc_config.amsbc = 1;
1320 gpu->ubwc_config.rgb565_predicator = 1;
1321 gpu->ubwc_config.uavflagprd_inv = 2;
1322 }
1323 }
1324
a6xx_set_ubwc_config(struct msm_gpu * gpu)1325 static void a6xx_set_ubwc_config(struct msm_gpu *gpu)
1326 {
1327 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1328 /*
1329 * We subtract 13 from the highest bank bit (13 is the minimum value
1330 * allowed by hw) and write the lowest two bits of the remaining value
1331 * as hbb_lo and the one above it as hbb_hi to the hardware.
1332 */
1333 BUG_ON(adreno_gpu->ubwc_config.highest_bank_bit < 13);
1334 u32 hbb = adreno_gpu->ubwc_config.highest_bank_bit - 13;
1335 u32 hbb_hi = hbb >> 2;
1336 u32 hbb_lo = hbb & 3;
1337
1338 gpu_write(gpu, REG_A6XX_RB_NC_MODE_CNTL,
1339 adreno_gpu->ubwc_config.rgb565_predicator << 11 |
1340 hbb_hi << 10 | adreno_gpu->ubwc_config.amsbc << 4 |
1341 adreno_gpu->ubwc_config.min_acc_len << 3 |
1342 hbb_lo << 1 | adreno_gpu->ubwc_config.ubwc_mode);
1343
1344 gpu_write(gpu, REG_A6XX_TPL1_NC_MODE_CNTL, hbb_hi << 4 |
1345 adreno_gpu->ubwc_config.min_acc_len << 3 |
1346 hbb_lo << 1 | adreno_gpu->ubwc_config.ubwc_mode);
1347
1348 gpu_write(gpu, REG_A6XX_SP_NC_MODE_CNTL, hbb_hi << 10 |
1349 adreno_gpu->ubwc_config.uavflagprd_inv << 4 |
1350 adreno_gpu->ubwc_config.min_acc_len << 3 |
1351 hbb_lo << 1 | adreno_gpu->ubwc_config.ubwc_mode);
1352
1353 if (adreno_is_a7xx(adreno_gpu))
1354 gpu_write(gpu, REG_A7XX_GRAS_NC_MODE_CNTL,
1355 FIELD_PREP(GENMASK(8, 5), hbb_lo));
1356
1357 gpu_write(gpu, REG_A6XX_UCHE_MODE_CNTL,
1358 adreno_gpu->ubwc_config.min_acc_len << 23 | hbb_lo << 21);
1359 }
1360
a6xx_cp_init(struct msm_gpu * gpu)1361 static int a6xx_cp_init(struct msm_gpu *gpu)
1362 {
1363 struct msm_ringbuffer *ring = gpu->rb[0];
1364
1365 OUT_PKT7(ring, CP_ME_INIT, 8);
1366
1367 OUT_RING(ring, 0x0000002f);
1368
1369 /* Enable multiple hardware contexts */
1370 OUT_RING(ring, 0x00000003);
1371
1372 /* Enable error detection */
1373 OUT_RING(ring, 0x20000000);
1374
1375 /* Don't enable header dump */
1376 OUT_RING(ring, 0x00000000);
1377 OUT_RING(ring, 0x00000000);
1378
1379 /* No workarounds enabled */
1380 OUT_RING(ring, 0x00000000);
1381
1382 /* Pad rest of the cmds with 0's */
1383 OUT_RING(ring, 0x00000000);
1384 OUT_RING(ring, 0x00000000);
1385
1386 a6xx_flush(gpu, ring);
1387 return a6xx_idle(gpu, ring) ? 0 : -EINVAL;
1388 }
1389
a7xx_cp_init(struct msm_gpu * gpu)1390 static int a7xx_cp_init(struct msm_gpu *gpu)
1391 {
1392 struct msm_ringbuffer *ring = gpu->rb[0];
1393 u32 mask;
1394
1395 /* Disable concurrent binning before sending CP init */
1396 OUT_PKT7(ring, CP_THREAD_CONTROL, 1);
1397 OUT_RING(ring, BIT(27));
1398
1399 OUT_PKT7(ring, CP_ME_INIT, 7);
1400
1401 /* Use multiple HW contexts */
1402 mask = BIT(0);
1403
1404 /* Enable error detection */
1405 mask |= BIT(1);
1406
1407 /* Set default reset state */
1408 mask |= BIT(3);
1409
1410 /* Disable save/restore of performance counters across preemption */
1411 mask |= BIT(6);
1412
1413 /* Enable the register init list with the spinlock */
1414 mask |= BIT(8);
1415
1416 OUT_RING(ring, mask);
1417
1418 /* Enable multiple hardware contexts */
1419 OUT_RING(ring, 0x00000003);
1420
1421 /* Enable error detection */
1422 OUT_RING(ring, 0x20000000);
1423
1424 /* Operation mode mask */
1425 OUT_RING(ring, 0x00000002);
1426
1427 /* *Don't* send a power up reg list for concurrent binning (TODO) */
1428 /* Lo address */
1429 OUT_RING(ring, 0x00000000);
1430 /* Hi address */
1431 OUT_RING(ring, 0x00000000);
1432 /* BIT(31) set => read the regs from the list */
1433 OUT_RING(ring, 0x00000000);
1434
1435 a6xx_flush(gpu, ring);
1436 return a6xx_idle(gpu, ring) ? 0 : -EINVAL;
1437 }
1438
1439 /*
1440 * Check that the microcode version is new enough to include several key
1441 * security fixes. Return true if the ucode is safe.
1442 */
a6xx_ucode_check_version(struct a6xx_gpu * a6xx_gpu,struct drm_gem_object * obj)1443 static bool a6xx_ucode_check_version(struct a6xx_gpu *a6xx_gpu,
1444 struct drm_gem_object *obj)
1445 {
1446 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
1447 struct msm_gpu *gpu = &adreno_gpu->base;
1448 const char *sqe_name = adreno_gpu->info->fw[ADRENO_FW_SQE];
1449 u32 *buf = msm_gem_get_vaddr(obj);
1450 bool ret = false;
1451
1452 if (IS_ERR(buf))
1453 return false;
1454
1455 /* A7xx is safe! */
1456 if (adreno_is_a7xx(adreno_gpu))
1457 return true;
1458
1459 /*
1460 * Targets up to a640 (a618, a630 and a640) need to check for a
1461 * microcode version that is patched to support the whereami opcode or
1462 * one that is new enough to include it by default.
1463 *
1464 * a650 tier targets don't need whereami but still need to be
1465 * equal to or newer than 0.95 for other security fixes
1466 *
1467 * a660 targets have all the critical security fixes from the start
1468 */
1469 if (!strcmp(sqe_name, "a630_sqe.fw")) {
1470 /*
1471 * If the lowest nibble is 0xa that is an indication that this
1472 * microcode has been patched. The actual version is in dword
1473 * [3] but we only care about the patchlevel which is the lowest
1474 * nibble of dword [3]
1475 *
1476 * Otherwise check that the firmware is greater than or equal
1477 * to 1.90 which was the first version that had this fix built
1478 * in
1479 */
1480 if ((((buf[0] & 0xf) == 0xa) && (buf[2] & 0xf) >= 1) ||
1481 (buf[0] & 0xfff) >= 0x190) {
1482 a6xx_gpu->has_whereami = true;
1483 ret = true;
1484 goto out;
1485 }
1486
1487 DRM_DEV_ERROR(&gpu->pdev->dev,
1488 "a630 SQE ucode is too old. Have version %x need at least %x\n",
1489 buf[0] & 0xfff, 0x190);
1490 } else if (!strcmp(sqe_name, "a650_sqe.fw")) {
1491 if ((buf[0] & 0xfff) >= 0x095) {
1492 ret = true;
1493 goto out;
1494 }
1495
1496 DRM_DEV_ERROR(&gpu->pdev->dev,
1497 "a650 SQE ucode is too old. Have version %x need at least %x\n",
1498 buf[0] & 0xfff, 0x095);
1499 } else if (!strcmp(sqe_name, "a660_sqe.fw")) {
1500 ret = true;
1501 } else {
1502 DRM_DEV_ERROR(&gpu->pdev->dev,
1503 "unknown GPU, add it to a6xx_ucode_check_version()!!\n");
1504 }
1505 out:
1506 msm_gem_put_vaddr(obj);
1507 return ret;
1508 }
1509
a6xx_ucode_load(struct msm_gpu * gpu)1510 static int a6xx_ucode_load(struct msm_gpu *gpu)
1511 {
1512 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1513 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
1514
1515 if (!a6xx_gpu->sqe_bo) {
1516 a6xx_gpu->sqe_bo = adreno_fw_create_bo(gpu,
1517 adreno_gpu->fw[ADRENO_FW_SQE], &a6xx_gpu->sqe_iova);
1518
1519 if (IS_ERR(a6xx_gpu->sqe_bo)) {
1520 int ret = PTR_ERR(a6xx_gpu->sqe_bo);
1521
1522 a6xx_gpu->sqe_bo = NULL;
1523 DRM_DEV_ERROR(&gpu->pdev->dev,
1524 "Could not allocate SQE ucode: %d\n", ret);
1525
1526 return ret;
1527 }
1528
1529 msm_gem_object_set_name(a6xx_gpu->sqe_bo, "sqefw");
1530 if (!a6xx_ucode_check_version(a6xx_gpu, a6xx_gpu->sqe_bo)) {
1531 msm_gem_unpin_iova(a6xx_gpu->sqe_bo, gpu->aspace);
1532 drm_gem_object_put(a6xx_gpu->sqe_bo);
1533
1534 a6xx_gpu->sqe_bo = NULL;
1535 return -EPERM;
1536 }
1537 }
1538
1539 /*
1540 * Expanded APRIV and targets that support WHERE_AM_I both need a
1541 * privileged buffer to store the RPTR shadow
1542 */
1543 if ((adreno_gpu->base.hw_apriv || a6xx_gpu->has_whereami) &&
1544 !a6xx_gpu->shadow_bo) {
1545 a6xx_gpu->shadow = msm_gem_kernel_new(gpu->dev,
1546 sizeof(u32) * gpu->nr_rings,
1547 MSM_BO_WC | MSM_BO_MAP_PRIV,
1548 gpu->aspace, &a6xx_gpu->shadow_bo,
1549 &a6xx_gpu->shadow_iova);
1550
1551 if (IS_ERR(a6xx_gpu->shadow))
1552 return PTR_ERR(a6xx_gpu->shadow);
1553
1554 msm_gem_object_set_name(a6xx_gpu->shadow_bo, "shadow");
1555 }
1556
1557 return 0;
1558 }
1559
a6xx_zap_shader_init(struct msm_gpu * gpu)1560 static int a6xx_zap_shader_init(struct msm_gpu *gpu)
1561 {
1562 static bool loaded;
1563 int ret;
1564
1565 if (loaded)
1566 return 0;
1567
1568 ret = adreno_zap_shader_load(gpu, GPU_PAS_ID);
1569
1570 loaded = !ret;
1571 return ret;
1572 }
1573
1574 #define A6XX_INT_MASK (A6XX_RBBM_INT_0_MASK_CP_AHB_ERROR | \
1575 A6XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNCFIFO_OVERFLOW | \
1576 A6XX_RBBM_INT_0_MASK_CP_HW_ERROR | \
1577 A6XX_RBBM_INT_0_MASK_CP_IB2 | \
1578 A6XX_RBBM_INT_0_MASK_CP_IB1 | \
1579 A6XX_RBBM_INT_0_MASK_CP_RB | \
1580 A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | \
1581 A6XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW | \
1582 A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT | \
1583 A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \
1584 A6XX_RBBM_INT_0_MASK_UCHE_TRAP_INTR)
1585
1586 #define A7XX_INT_MASK (A6XX_RBBM_INT_0_MASK_CP_AHB_ERROR | \
1587 A6XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNCFIFO_OVERFLOW | \
1588 A6XX_RBBM_INT_0_MASK_RBBM_GPC_ERROR | \
1589 A6XX_RBBM_INT_0_MASK_CP_SW | \
1590 A6XX_RBBM_INT_0_MASK_CP_HW_ERROR | \
1591 A6XX_RBBM_INT_0_MASK_PM4CPINTERRUPT | \
1592 A6XX_RBBM_INT_0_MASK_CP_RB_DONE_TS | \
1593 A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | \
1594 A6XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW | \
1595 A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT | \
1596 A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \
1597 A6XX_RBBM_INT_0_MASK_UCHE_TRAP_INTR | \
1598 A6XX_RBBM_INT_0_MASK_TSBWRITEERROR)
1599
1600 #define A7XX_APRIV_MASK (A6XX_CP_APRIV_CNTL_ICACHE | \
1601 A6XX_CP_APRIV_CNTL_RBFETCH | \
1602 A6XX_CP_APRIV_CNTL_RBPRIVLEVEL | \
1603 A6XX_CP_APRIV_CNTL_RBRPWB)
1604
1605 #define A7XX_BR_APRIVMASK (A7XX_APRIV_MASK | \
1606 A6XX_CP_APRIV_CNTL_CDREAD | \
1607 A6XX_CP_APRIV_CNTL_CDWRITE)
1608
hw_init(struct msm_gpu * gpu)1609 static int hw_init(struct msm_gpu *gpu)
1610 {
1611 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1612 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
1613 struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
1614 u64 gmem_range_min;
1615 int ret;
1616
1617 if (!adreno_has_gmu_wrapper(adreno_gpu)) {
1618 /* Make sure the GMU keeps the GPU on while we set it up */
1619 ret = a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET);
1620 if (ret)
1621 return ret;
1622 }
1623
1624 /* Clear GBIF halt in case GX domain was not collapsed */
1625 if (adreno_is_a619_holi(adreno_gpu)) {
1626 gpu_write(gpu, REG_A6XX_GBIF_HALT, 0);
1627 gpu_write(gpu, REG_A6XX_RBBM_GPR0_CNTL, 0);
1628 /* Let's make extra sure that the GPU can access the memory.. */
1629 mb();
1630 } else if (a6xx_has_gbif(adreno_gpu)) {
1631 gpu_write(gpu, REG_A6XX_GBIF_HALT, 0);
1632 gpu_write(gpu, REG_A6XX_RBBM_GBIF_HALT, 0);
1633 /* Let's make extra sure that the GPU can access the memory.. */
1634 mb();
1635 }
1636
1637 /* Some GPUs are stubborn and take their sweet time to unhalt GBIF! */
1638 if (adreno_is_a7xx(adreno_gpu) && a6xx_has_gbif(adreno_gpu))
1639 spin_until(!gpu_read(gpu, REG_A6XX_GBIF_HALT_ACK));
1640
1641 gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_CNTL, 0);
1642
1643 if (adreno_is_a619_holi(adreno_gpu))
1644 a6xx_sptprac_enable(gmu);
1645
1646 /*
1647 * Disable the trusted memory range - we don't actually supported secure
1648 * memory rendering at this point in time and we don't want to block off
1649 * part of the virtual memory space.
1650 */
1651 gpu_write64(gpu, REG_A6XX_RBBM_SECVID_TSB_TRUSTED_BASE, 0x00000000);
1652 gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_TRUSTED_SIZE, 0x00000000);
1653
1654 if (!adreno_is_a7xx(adreno_gpu)) {
1655 /* Turn on 64 bit addressing for all blocks */
1656 gpu_write(gpu, REG_A6XX_CP_ADDR_MODE_CNTL, 0x1);
1657 gpu_write(gpu, REG_A6XX_VSC_ADDR_MODE_CNTL, 0x1);
1658 gpu_write(gpu, REG_A6XX_GRAS_ADDR_MODE_CNTL, 0x1);
1659 gpu_write(gpu, REG_A6XX_RB_ADDR_MODE_CNTL, 0x1);
1660 gpu_write(gpu, REG_A6XX_PC_ADDR_MODE_CNTL, 0x1);
1661 gpu_write(gpu, REG_A6XX_HLSQ_ADDR_MODE_CNTL, 0x1);
1662 gpu_write(gpu, REG_A6XX_VFD_ADDR_MODE_CNTL, 0x1);
1663 gpu_write(gpu, REG_A6XX_VPC_ADDR_MODE_CNTL, 0x1);
1664 gpu_write(gpu, REG_A6XX_UCHE_ADDR_MODE_CNTL, 0x1);
1665 gpu_write(gpu, REG_A6XX_SP_ADDR_MODE_CNTL, 0x1);
1666 gpu_write(gpu, REG_A6XX_TPL1_ADDR_MODE_CNTL, 0x1);
1667 gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_ADDR_MODE_CNTL, 0x1);
1668 }
1669
1670 /* enable hardware clockgating */
1671 a6xx_set_hwcg(gpu, true);
1672
1673 /* VBIF/GBIF start*/
1674 if (adreno_is_a610(adreno_gpu) ||
1675 adreno_is_a640_family(adreno_gpu) ||
1676 adreno_is_a650_family(adreno_gpu) ||
1677 adreno_is_a7xx(adreno_gpu)) {
1678 gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE0, 0x00071620);
1679 gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE1, 0x00071620);
1680 gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE2, 0x00071620);
1681 gpu_write(gpu, REG_A6XX_GBIF_QSB_SIDE3, 0x00071620);
1682 gpu_write(gpu, REG_A6XX_RBBM_GBIF_CLIENT_QOS_CNTL,
1683 adreno_is_a7xx(adreno_gpu) ? 0x2120212 : 0x3);
1684 } else {
1685 gpu_write(gpu, REG_A6XX_RBBM_VBIF_CLIENT_QOS_CNTL, 0x3);
1686 }
1687
1688 if (adreno_is_a630(adreno_gpu))
1689 gpu_write(gpu, REG_A6XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000009);
1690
1691 if (adreno_is_a7xx(adreno_gpu))
1692 gpu_write(gpu, REG_A6XX_UCHE_GBIF_GX_CONFIG, 0x10240e0);
1693
1694 /* Make all blocks contribute to the GPU BUSY perf counter */
1695 gpu_write(gpu, REG_A6XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xffffffff);
1696
1697 /* Disable L2 bypass in the UCHE */
1698 if (adreno_is_a7xx(adreno_gpu)) {
1699 gpu_write64(gpu, REG_A6XX_UCHE_TRAP_BASE, 0x0001fffffffff000llu);
1700 gpu_write64(gpu, REG_A6XX_UCHE_WRITE_THRU_BASE, 0x0001fffffffff000llu);
1701 } else {
1702 gpu_write64(gpu, REG_A6XX_UCHE_WRITE_RANGE_MAX, 0x0001ffffffffffc0llu);
1703 gpu_write64(gpu, REG_A6XX_UCHE_TRAP_BASE, 0x0001fffffffff000llu);
1704 gpu_write64(gpu, REG_A6XX_UCHE_WRITE_THRU_BASE, 0x0001fffffffff000llu);
1705 }
1706
1707 if (!(adreno_is_a650_family(adreno_gpu) ||
1708 adreno_is_a730(adreno_gpu))) {
1709 gmem_range_min = adreno_is_a740_family(adreno_gpu) ? SZ_16M : SZ_1M;
1710
1711 /* Set the GMEM VA range [0x100000:0x100000 + gpu->gmem - 1] */
1712 gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MIN, gmem_range_min);
1713
1714 gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MAX,
1715 gmem_range_min + adreno_gpu->info->gmem - 1);
1716 }
1717
1718 if (adreno_is_a7xx(adreno_gpu))
1719 gpu_write(gpu, REG_A6XX_UCHE_CACHE_WAYS, BIT(23));
1720 else {
1721 gpu_write(gpu, REG_A6XX_UCHE_FILTER_CNTL, 0x804);
1722 gpu_write(gpu, REG_A6XX_UCHE_CACHE_WAYS, 0x4);
1723 }
1724
1725 if (adreno_is_a640_family(adreno_gpu) || adreno_is_a650_family(adreno_gpu)) {
1726 gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_2, 0x02000140);
1727 gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_1, 0x8040362c);
1728 } else if (adreno_is_a610(adreno_gpu)) {
1729 gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_2, 0x00800060);
1730 gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_1, 0x40201b16);
1731 } else if (!adreno_is_a7xx(adreno_gpu)) {
1732 gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_2, 0x010000c0);
1733 gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_1, 0x8040362c);
1734 }
1735
1736 if (adreno_is_a660_family(adreno_gpu))
1737 gpu_write(gpu, REG_A6XX_CP_LPAC_PROG_FIFO_SIZE, 0x00000020);
1738
1739 /* Setting the mem pool size */
1740 if (adreno_is_a610(adreno_gpu)) {
1741 gpu_write(gpu, REG_A6XX_CP_MEM_POOL_SIZE, 48);
1742 gpu_write(gpu, REG_A6XX_CP_MEM_POOL_DBG_ADDR, 47);
1743 } else if (!adreno_is_a7xx(adreno_gpu))
1744 gpu_write(gpu, REG_A6XX_CP_MEM_POOL_SIZE, 128);
1745
1746 /* Setting the primFifo thresholds default values,
1747 * and vccCacheSkipDis=1 bit (0x200) for A640 and newer
1748 */
1749 if (adreno_is_a690(adreno_gpu))
1750 gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00800200);
1751 else if (adreno_is_a650(adreno_gpu) || adreno_is_a660(adreno_gpu))
1752 gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00300200);
1753 else if (adreno_is_a640_family(adreno_gpu) || adreno_is_7c3(adreno_gpu))
1754 gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00200200);
1755 else if (adreno_is_a650(adreno_gpu) || adreno_is_a660(adreno_gpu))
1756 gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00300200);
1757 else if (adreno_is_a619(adreno_gpu))
1758 gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00018000);
1759 else if (adreno_is_a610(adreno_gpu))
1760 gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00080000);
1761 else if (!adreno_is_a7xx(adreno_gpu))
1762 gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, 0x00180000);
1763
1764 /* Set the AHB default slave response to "ERROR" */
1765 gpu_write(gpu, REG_A6XX_CP_AHB_CNTL, 0x1);
1766
1767 /* Turn on performance counters */
1768 gpu_write(gpu, REG_A6XX_RBBM_PERFCTR_CNTL, 0x1);
1769
1770 if (adreno_is_a7xx(adreno_gpu)) {
1771 /* Turn on the IFPC counter (countable 4 on XOCLK4) */
1772 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_1,
1773 FIELD_PREP(GENMASK(7, 0), 0x4));
1774 }
1775
1776 /* Select CP0 to always count cycles */
1777 gpu_write(gpu, REG_A6XX_CP_PERFCTR_CP_SEL(0), PERF_CP_ALWAYS_COUNT);
1778
1779 a6xx_set_ubwc_config(gpu);
1780
1781 /* Enable fault detection */
1782 if (adreno_is_a730(adreno_gpu) ||
1783 adreno_is_a740_family(adreno_gpu))
1784 gpu_write(gpu, REG_A6XX_RBBM_INTERFACE_HANG_INT_CNTL, (1 << 30) | 0xcfffff);
1785 else if (adreno_is_a690(adreno_gpu))
1786 gpu_write(gpu, REG_A6XX_RBBM_INTERFACE_HANG_INT_CNTL, (1 << 30) | 0x4fffff);
1787 else if (adreno_is_a619(adreno_gpu))
1788 gpu_write(gpu, REG_A6XX_RBBM_INTERFACE_HANG_INT_CNTL, (1 << 30) | 0x3fffff);
1789 else if (adreno_is_a610(adreno_gpu))
1790 gpu_write(gpu, REG_A6XX_RBBM_INTERFACE_HANG_INT_CNTL, (1 << 30) | 0x3ffff);
1791 else
1792 gpu_write(gpu, REG_A6XX_RBBM_INTERFACE_HANG_INT_CNTL, (1 << 30) | 0x1fffff);
1793
1794 gpu_write(gpu, REG_A6XX_UCHE_CLIENT_PF, BIT(7) | 0x1);
1795
1796 /* Set weights for bicubic filtering */
1797 if (adreno_is_a650_family(adreno_gpu)) {
1798 gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_0, 0);
1799 gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_1,
1800 0x3fe05ff4);
1801 gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_2,
1802 0x3fa0ebee);
1803 gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_3,
1804 0x3f5193ed);
1805 gpu_write(gpu, REG_A6XX_TPL1_BICUBIC_WEIGHTS_TABLE_4,
1806 0x3f0243f0);
1807 }
1808
1809 /* Set up the CX GMU counter 0 to count busy ticks */
1810 gmu_write(gmu, REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_MASK, 0xff000000);
1811
1812 /* Enable the power counter */
1813 gmu_rmw(gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_0, 0xff, BIT(5));
1814 gmu_write(gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE, 1);
1815
1816 /* Protect registers from the CP */
1817 a6xx_set_cp_protect(gpu);
1818
1819 if (adreno_is_a660_family(adreno_gpu)) {
1820 if (adreno_is_a690(adreno_gpu))
1821 gpu_write(gpu, REG_A6XX_CP_CHICKEN_DBG, 0x00028801);
1822 else
1823 gpu_write(gpu, REG_A6XX_CP_CHICKEN_DBG, 0x1);
1824 gpu_write(gpu, REG_A6XX_RBBM_GBIF_CLIENT_QOS_CNTL, 0x0);
1825 }
1826
1827 if (adreno_is_a690(adreno_gpu))
1828 gpu_write(gpu, REG_A6XX_UCHE_CMDQ_CONFIG, 0x90);
1829 /* Set dualQ + disable afull for A660 GPU */
1830 else if (adreno_is_a660(adreno_gpu))
1831 gpu_write(gpu, REG_A6XX_UCHE_CMDQ_CONFIG, 0x66906);
1832 else if (adreno_is_a7xx(adreno_gpu))
1833 gpu_write(gpu, REG_A6XX_UCHE_CMDQ_CONFIG,
1834 FIELD_PREP(GENMASK(19, 16), 6) |
1835 FIELD_PREP(GENMASK(15, 12), 6) |
1836 FIELD_PREP(GENMASK(11, 8), 9) |
1837 BIT(3) | BIT(2) |
1838 FIELD_PREP(GENMASK(1, 0), 2));
1839
1840 /* Enable expanded apriv for targets that support it */
1841 if (gpu->hw_apriv) {
1842 if (adreno_is_a7xx(adreno_gpu)) {
1843 gpu_write(gpu, REG_A6XX_CP_APRIV_CNTL,
1844 A7XX_BR_APRIVMASK);
1845 gpu_write(gpu, REG_A7XX_CP_BV_APRIV_CNTL,
1846 A7XX_APRIV_MASK);
1847 gpu_write(gpu, REG_A7XX_CP_LPAC_APRIV_CNTL,
1848 A7XX_APRIV_MASK);
1849 } else
1850 gpu_write(gpu, REG_A6XX_CP_APRIV_CNTL,
1851 BIT(6) | BIT(5) | BIT(3) | BIT(2) | BIT(1));
1852 }
1853
1854 /* Enable interrupts */
1855 gpu_write(gpu, REG_A6XX_RBBM_INT_0_MASK,
1856 adreno_is_a7xx(adreno_gpu) ? A7XX_INT_MASK : A6XX_INT_MASK);
1857
1858 ret = adreno_hw_init(gpu);
1859 if (ret)
1860 goto out;
1861
1862 gpu_write64(gpu, REG_A6XX_CP_SQE_INSTR_BASE, a6xx_gpu->sqe_iova);
1863
1864 /* Set the ringbuffer address */
1865 gpu_write64(gpu, REG_A6XX_CP_RB_BASE, gpu->rb[0]->iova);
1866
1867 /* Targets that support extended APRIV can use the RPTR shadow from
1868 * hardware but all the other ones need to disable the feature. Targets
1869 * that support the WHERE_AM_I opcode can use that instead
1870 */
1871 if (adreno_gpu->base.hw_apriv)
1872 gpu_write(gpu, REG_A6XX_CP_RB_CNTL, MSM_GPU_RB_CNTL_DEFAULT);
1873 else
1874 gpu_write(gpu, REG_A6XX_CP_RB_CNTL,
1875 MSM_GPU_RB_CNTL_DEFAULT | AXXX_CP_RB_CNTL_NO_UPDATE);
1876
1877 /* Configure the RPTR shadow if needed: */
1878 if (a6xx_gpu->shadow_bo) {
1879 gpu_write64(gpu, REG_A6XX_CP_RB_RPTR_ADDR,
1880 shadowptr(a6xx_gpu, gpu->rb[0]));
1881 }
1882
1883 /* ..which means "always" on A7xx, also for BV shadow */
1884 if (adreno_is_a7xx(adreno_gpu)) {
1885 gpu_write64(gpu, REG_A7XX_CP_BV_RB_RPTR_ADDR,
1886 rbmemptr(gpu->rb[0], bv_fence));
1887 }
1888
1889 /* Always come up on rb 0 */
1890 a6xx_gpu->cur_ring = gpu->rb[0];
1891
1892 gpu->cur_ctx_seqno = 0;
1893
1894 /* Enable the SQE_to start the CP engine */
1895 gpu_write(gpu, REG_A6XX_CP_SQE_CNTL, 1);
1896
1897 ret = adreno_is_a7xx(adreno_gpu) ? a7xx_cp_init(gpu) : a6xx_cp_init(gpu);
1898 if (ret)
1899 goto out;
1900
1901 /*
1902 * Try to load a zap shader into the secure world. If successful
1903 * we can use the CP to switch out of secure mode. If not then we
1904 * have no resource but to try to switch ourselves out manually. If we
1905 * guessed wrong then access to the RBBM_SECVID_TRUST_CNTL register will
1906 * be blocked and a permissions violation will soon follow.
1907 */
1908 ret = a6xx_zap_shader_init(gpu);
1909 if (!ret) {
1910 OUT_PKT7(gpu->rb[0], CP_SET_SECURE_MODE, 1);
1911 OUT_RING(gpu->rb[0], 0x00000000);
1912
1913 a6xx_flush(gpu, gpu->rb[0]);
1914 if (!a6xx_idle(gpu, gpu->rb[0]))
1915 return -EINVAL;
1916 } else if (ret == -ENODEV) {
1917 /*
1918 * This device does not use zap shader (but print a warning
1919 * just in case someone got their dt wrong.. hopefully they
1920 * have a debug UART to realize the error of their ways...
1921 * if you mess this up you are about to crash horribly)
1922 */
1923 dev_warn_once(gpu->dev->dev,
1924 "Zap shader not enabled - using SECVID_TRUST_CNTL instead\n");
1925 gpu_write(gpu, REG_A6XX_RBBM_SECVID_TRUST_CNTL, 0x0);
1926 ret = 0;
1927 } else {
1928 return ret;
1929 }
1930
1931 out:
1932 if (adreno_has_gmu_wrapper(adreno_gpu))
1933 return ret;
1934 /*
1935 * Tell the GMU that we are done touching the GPU and it can start power
1936 * management
1937 */
1938 a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET);
1939
1940 if (a6xx_gpu->gmu.legacy) {
1941 /* Take the GMU out of its special boot mode */
1942 a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_BOOT_SLUMBER);
1943 }
1944
1945 return ret;
1946 }
1947
a6xx_hw_init(struct msm_gpu * gpu)1948 static int a6xx_hw_init(struct msm_gpu *gpu)
1949 {
1950 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1951 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
1952 int ret;
1953
1954 mutex_lock(&a6xx_gpu->gmu.lock);
1955 ret = hw_init(gpu);
1956 mutex_unlock(&a6xx_gpu->gmu.lock);
1957
1958 return ret;
1959 }
1960
a6xx_dump(struct msm_gpu * gpu)1961 static void a6xx_dump(struct msm_gpu *gpu)
1962 {
1963 DRM_DEV_INFO(&gpu->pdev->dev, "status: %08x\n",
1964 gpu_read(gpu, REG_A6XX_RBBM_STATUS));
1965 adreno_dump(gpu);
1966 }
1967
a6xx_recover(struct msm_gpu * gpu)1968 static void a6xx_recover(struct msm_gpu *gpu)
1969 {
1970 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
1971 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
1972 struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
1973 int i, active_submits;
1974
1975 adreno_dump_info(gpu);
1976
1977 for (i = 0; i < 8; i++)
1978 DRM_DEV_INFO(&gpu->pdev->dev, "CP_SCRATCH_REG%d: %u\n", i,
1979 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(i)));
1980
1981 if (hang_debug)
1982 a6xx_dump(gpu);
1983
1984 /*
1985 * To handle recovery specific sequences during the rpm suspend we are
1986 * about to trigger
1987 */
1988 a6xx_gpu->hung = true;
1989
1990 /* Halt SQE first */
1991 gpu_write(gpu, REG_A6XX_CP_SQE_CNTL, 3);
1992
1993 pm_runtime_dont_use_autosuspend(&gpu->pdev->dev);
1994
1995 /* active_submit won't change until we make a submission */
1996 mutex_lock(&gpu->active_lock);
1997 active_submits = gpu->active_submits;
1998
1999 /*
2000 * Temporarily clear active_submits count to silence a WARN() in the
2001 * runtime suspend cb
2002 */
2003 gpu->active_submits = 0;
2004
2005 if (adreno_has_gmu_wrapper(adreno_gpu)) {
2006 /* Drain the outstanding traffic on memory buses */
2007 a6xx_bus_clear_pending_transactions(adreno_gpu, true);
2008
2009 /* Reset the GPU to a clean state */
2010 a6xx_gpu_sw_reset(gpu, true);
2011 a6xx_gpu_sw_reset(gpu, false);
2012 }
2013
2014 reinit_completion(&gmu->pd_gate);
2015 dev_pm_genpd_add_notifier(gmu->cxpd, &gmu->pd_nb);
2016 dev_pm_genpd_synced_poweroff(gmu->cxpd);
2017
2018 /* Drop the rpm refcount from active submits */
2019 if (active_submits)
2020 pm_runtime_put(&gpu->pdev->dev);
2021
2022 /* And the final one from recover worker */
2023 pm_runtime_put_sync(&gpu->pdev->dev);
2024
2025 if (!wait_for_completion_timeout(&gmu->pd_gate, msecs_to_jiffies(1000)))
2026 DRM_DEV_ERROR(&gpu->pdev->dev, "cx gdsc didn't collapse\n");
2027
2028 dev_pm_genpd_remove_notifier(gmu->cxpd);
2029
2030 pm_runtime_use_autosuspend(&gpu->pdev->dev);
2031
2032 if (active_submits)
2033 pm_runtime_get(&gpu->pdev->dev);
2034
2035 pm_runtime_get_sync(&gpu->pdev->dev);
2036
2037 gpu->active_submits = active_submits;
2038 mutex_unlock(&gpu->active_lock);
2039
2040 msm_gpu_hw_init(gpu);
2041 a6xx_gpu->hung = false;
2042 }
2043
a6xx_uche_fault_block(struct msm_gpu * gpu,u32 mid)2044 static const char *a6xx_uche_fault_block(struct msm_gpu *gpu, u32 mid)
2045 {
2046 static const char *uche_clients[7] = {
2047 "VFD", "SP", "VSC", "VPC", "HLSQ", "PC", "LRZ",
2048 };
2049 u32 val;
2050
2051 if (mid < 1 || mid > 3)
2052 return "UNKNOWN";
2053
2054 /*
2055 * The source of the data depends on the mid ID read from FSYNR1.
2056 * and the client ID read from the UCHE block
2057 */
2058 val = gpu_read(gpu, REG_A6XX_UCHE_CLIENT_PF);
2059
2060 /* mid = 3 is most precise and refers to only one block per client */
2061 if (mid == 3)
2062 return uche_clients[val & 7];
2063
2064 /* For mid=2 the source is TP or VFD except when the client id is 0 */
2065 if (mid == 2)
2066 return ((val & 7) == 0) ? "TP" : "TP|VFD";
2067
2068 /* For mid=1 just return "UCHE" as a catchall for everything else */
2069 return "UCHE";
2070 }
2071
a6xx_fault_block(struct msm_gpu * gpu,u32 id)2072 static const char *a6xx_fault_block(struct msm_gpu *gpu, u32 id)
2073 {
2074 if (id == 0)
2075 return "CP";
2076 else if (id == 4)
2077 return "CCU";
2078 else if (id == 6)
2079 return "CDP Prefetch";
2080
2081 return a6xx_uche_fault_block(gpu, id);
2082 }
2083
a6xx_fault_handler(void * arg,unsigned long iova,int flags,void * data)2084 static int a6xx_fault_handler(void *arg, unsigned long iova, int flags, void *data)
2085 {
2086 struct msm_gpu *gpu = arg;
2087 struct adreno_smmu_fault_info *info = data;
2088 const char *block = "unknown";
2089
2090 u32 scratch[] = {
2091 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(4)),
2092 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(5)),
2093 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(6)),
2094 gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(7)),
2095 };
2096
2097 if (info)
2098 block = a6xx_fault_block(gpu, info->fsynr1 & 0xff);
2099
2100 return adreno_fault_handler(gpu, iova, flags, info, block, scratch);
2101 }
2102
a6xx_cp_hw_err_irq(struct msm_gpu * gpu)2103 static void a6xx_cp_hw_err_irq(struct msm_gpu *gpu)
2104 {
2105 u32 status = gpu_read(gpu, REG_A6XX_CP_INTERRUPT_STATUS);
2106
2107 if (status & A6XX_CP_INT_CP_OPCODE_ERROR) {
2108 u32 val;
2109
2110 gpu_write(gpu, REG_A6XX_CP_SQE_STAT_ADDR, 1);
2111 val = gpu_read(gpu, REG_A6XX_CP_SQE_STAT_DATA);
2112 dev_err_ratelimited(&gpu->pdev->dev,
2113 "CP | opcode error | possible opcode=0x%8.8X\n",
2114 val);
2115 }
2116
2117 if (status & A6XX_CP_INT_CP_UCODE_ERROR)
2118 dev_err_ratelimited(&gpu->pdev->dev,
2119 "CP ucode error interrupt\n");
2120
2121 if (status & A6XX_CP_INT_CP_HW_FAULT_ERROR)
2122 dev_err_ratelimited(&gpu->pdev->dev, "CP | HW fault | status=0x%8.8X\n",
2123 gpu_read(gpu, REG_A6XX_CP_HW_FAULT));
2124
2125 if (status & A6XX_CP_INT_CP_REGISTER_PROTECTION_ERROR) {
2126 u32 val = gpu_read(gpu, REG_A6XX_CP_PROTECT_STATUS);
2127
2128 dev_err_ratelimited(&gpu->pdev->dev,
2129 "CP | protected mode error | %s | addr=0x%8.8X | status=0x%8.8X\n",
2130 val & (1 << 20) ? "READ" : "WRITE",
2131 (val & 0x3ffff), val);
2132 }
2133
2134 if (status & A6XX_CP_INT_CP_AHB_ERROR && !adreno_is_a7xx(to_adreno_gpu(gpu)))
2135 dev_err_ratelimited(&gpu->pdev->dev, "CP AHB error interrupt\n");
2136
2137 if (status & A6XX_CP_INT_CP_VSD_PARITY_ERROR)
2138 dev_err_ratelimited(&gpu->pdev->dev, "CP VSD decoder parity error\n");
2139
2140 if (status & A6XX_CP_INT_CP_ILLEGAL_INSTR_ERROR)
2141 dev_err_ratelimited(&gpu->pdev->dev, "CP illegal instruction error\n");
2142
2143 }
2144
a6xx_fault_detect_irq(struct msm_gpu * gpu)2145 static void a6xx_fault_detect_irq(struct msm_gpu *gpu)
2146 {
2147 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2148 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2149 struct msm_ringbuffer *ring = gpu->funcs->active_ring(gpu);
2150
2151 /*
2152 * If stalled on SMMU fault, we could trip the GPU's hang detection,
2153 * but the fault handler will trigger the devcore dump, and we want
2154 * to otherwise resume normally rather than killing the submit, so
2155 * just bail.
2156 */
2157 if (gpu_read(gpu, REG_A6XX_RBBM_STATUS3) & A6XX_RBBM_STATUS3_SMMU_STALLED_ON_FAULT)
2158 return;
2159
2160 /*
2161 * Force the GPU to stay on until after we finish
2162 * collecting information
2163 */
2164 if (!adreno_has_gmu_wrapper(adreno_gpu))
2165 gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_GMU_PWR_COL_KEEPALIVE, 1);
2166
2167 DRM_DEV_ERROR(&gpu->pdev->dev,
2168 "gpu fault ring %d fence %x status %8.8X rb %4.4x/%4.4x ib1 %16.16llX/%4.4x ib2 %16.16llX/%4.4x\n",
2169 ring ? ring->id : -1, ring ? ring->fctx->last_fence : 0,
2170 gpu_read(gpu, REG_A6XX_RBBM_STATUS),
2171 gpu_read(gpu, REG_A6XX_CP_RB_RPTR),
2172 gpu_read(gpu, REG_A6XX_CP_RB_WPTR),
2173 gpu_read64(gpu, REG_A6XX_CP_IB1_BASE),
2174 gpu_read(gpu, REG_A6XX_CP_IB1_REM_SIZE),
2175 gpu_read64(gpu, REG_A6XX_CP_IB2_BASE),
2176 gpu_read(gpu, REG_A6XX_CP_IB2_REM_SIZE));
2177
2178 /* Turn off the hangcheck timer to keep it from bothering us */
2179 del_timer(&gpu->hangcheck_timer);
2180
2181 kthread_queue_work(gpu->worker, &gpu->recover_work);
2182 }
2183
a6xx_irq(struct msm_gpu * gpu)2184 static irqreturn_t a6xx_irq(struct msm_gpu *gpu)
2185 {
2186 struct msm_drm_private *priv = gpu->dev->dev_private;
2187 u32 status = gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS);
2188
2189 gpu_write(gpu, REG_A6XX_RBBM_INT_CLEAR_CMD, status);
2190
2191 if (priv->disable_err_irq)
2192 status &= A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS;
2193
2194 if (status & A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT)
2195 a6xx_fault_detect_irq(gpu);
2196
2197 if (status & A6XX_RBBM_INT_0_MASK_CP_AHB_ERROR)
2198 dev_err_ratelimited(&gpu->pdev->dev, "CP | AHB bus error\n");
2199
2200 if (status & A6XX_RBBM_INT_0_MASK_CP_HW_ERROR)
2201 a6xx_cp_hw_err_irq(gpu);
2202
2203 if (status & A6XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNCFIFO_OVERFLOW)
2204 dev_err_ratelimited(&gpu->pdev->dev, "RBBM | ATB ASYNC overflow\n");
2205
2206 if (status & A6XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW)
2207 dev_err_ratelimited(&gpu->pdev->dev, "RBBM | ATB bus overflow\n");
2208
2209 if (status & A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS)
2210 dev_err_ratelimited(&gpu->pdev->dev, "UCHE | Out of bounds access\n");
2211
2212 if (status & A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS)
2213 msm_gpu_retire(gpu);
2214
2215 return IRQ_HANDLED;
2216 }
2217
a6xx_llc_deactivate(struct a6xx_gpu * a6xx_gpu)2218 static void a6xx_llc_deactivate(struct a6xx_gpu *a6xx_gpu)
2219 {
2220 llcc_slice_deactivate(a6xx_gpu->llc_slice);
2221 llcc_slice_deactivate(a6xx_gpu->htw_llc_slice);
2222 }
2223
a6xx_llc_activate(struct a6xx_gpu * a6xx_gpu)2224 static void a6xx_llc_activate(struct a6xx_gpu *a6xx_gpu)
2225 {
2226 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
2227 struct msm_gpu *gpu = &adreno_gpu->base;
2228 u32 cntl1_regval = 0;
2229
2230 if (IS_ERR(a6xx_gpu->llc_mmio))
2231 return;
2232
2233 if (!llcc_slice_activate(a6xx_gpu->llc_slice)) {
2234 u32 gpu_scid = llcc_get_slice_id(a6xx_gpu->llc_slice);
2235
2236 gpu_scid &= 0x1f;
2237 cntl1_regval = (gpu_scid << 0) | (gpu_scid << 5) | (gpu_scid << 10) |
2238 (gpu_scid << 15) | (gpu_scid << 20);
2239
2240 /* On A660, the SCID programming for UCHE traffic is done in
2241 * A6XX_GBIF_SCACHE_CNTL0[14:10]
2242 */
2243 if (adreno_is_a660_family(adreno_gpu))
2244 gpu_rmw(gpu, REG_A6XX_GBIF_SCACHE_CNTL0, (0x1f << 10) |
2245 (1 << 8), (gpu_scid << 10) | (1 << 8));
2246 }
2247
2248 /*
2249 * For targets with a MMU500, activate the slice but don't program the
2250 * register. The XBL will take care of that.
2251 */
2252 if (!llcc_slice_activate(a6xx_gpu->htw_llc_slice)) {
2253 if (!a6xx_gpu->have_mmu500) {
2254 u32 gpuhtw_scid = llcc_get_slice_id(a6xx_gpu->htw_llc_slice);
2255
2256 gpuhtw_scid &= 0x1f;
2257 cntl1_regval |= FIELD_PREP(GENMASK(29, 25), gpuhtw_scid);
2258 }
2259 }
2260
2261 if (!cntl1_regval)
2262 return;
2263
2264 /*
2265 * Program the slice IDs for the various GPU blocks and GPU MMU
2266 * pagetables
2267 */
2268 if (!a6xx_gpu->have_mmu500) {
2269 a6xx_llc_write(a6xx_gpu,
2270 REG_A6XX_CX_MISC_SYSTEM_CACHE_CNTL_1, cntl1_regval);
2271
2272 /*
2273 * Program cacheability overrides to not allocate cache
2274 * lines on a write miss
2275 */
2276 a6xx_llc_rmw(a6xx_gpu,
2277 REG_A6XX_CX_MISC_SYSTEM_CACHE_CNTL_0, 0xF, 0x03);
2278 return;
2279 }
2280
2281 gpu_rmw(gpu, REG_A6XX_GBIF_SCACHE_CNTL1, GENMASK(24, 0), cntl1_regval);
2282 }
2283
a7xx_llc_activate(struct a6xx_gpu * a6xx_gpu)2284 static void a7xx_llc_activate(struct a6xx_gpu *a6xx_gpu)
2285 {
2286 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
2287 struct msm_gpu *gpu = &adreno_gpu->base;
2288
2289 if (IS_ERR(a6xx_gpu->llc_mmio))
2290 return;
2291
2292 if (!llcc_slice_activate(a6xx_gpu->llc_slice)) {
2293 u32 gpu_scid = llcc_get_slice_id(a6xx_gpu->llc_slice);
2294
2295 gpu_scid &= GENMASK(4, 0);
2296
2297 gpu_write(gpu, REG_A6XX_GBIF_SCACHE_CNTL1,
2298 FIELD_PREP(GENMASK(29, 25), gpu_scid) |
2299 FIELD_PREP(GENMASK(24, 20), gpu_scid) |
2300 FIELD_PREP(GENMASK(19, 15), gpu_scid) |
2301 FIELD_PREP(GENMASK(14, 10), gpu_scid) |
2302 FIELD_PREP(GENMASK(9, 5), gpu_scid) |
2303 FIELD_PREP(GENMASK(4, 0), gpu_scid));
2304
2305 gpu_write(gpu, REG_A6XX_GBIF_SCACHE_CNTL0,
2306 FIELD_PREP(GENMASK(14, 10), gpu_scid) |
2307 BIT(8));
2308 }
2309
2310 llcc_slice_activate(a6xx_gpu->htw_llc_slice);
2311 }
2312
a6xx_llc_slices_destroy(struct a6xx_gpu * a6xx_gpu)2313 static void a6xx_llc_slices_destroy(struct a6xx_gpu *a6xx_gpu)
2314 {
2315 /* No LLCC on non-RPMh (and by extension, non-GMU) SoCs */
2316 if (adreno_has_gmu_wrapper(&a6xx_gpu->base))
2317 return;
2318
2319 llcc_slice_putd(a6xx_gpu->llc_slice);
2320 llcc_slice_putd(a6xx_gpu->htw_llc_slice);
2321 }
2322
a6xx_llc_slices_init(struct platform_device * pdev,struct a6xx_gpu * a6xx_gpu,bool is_a7xx)2323 static void a6xx_llc_slices_init(struct platform_device *pdev,
2324 struct a6xx_gpu *a6xx_gpu, bool is_a7xx)
2325 {
2326 struct device_node *phandle;
2327
2328 /* No LLCC on non-RPMh (and by extension, non-GMU) SoCs */
2329 if (adreno_has_gmu_wrapper(&a6xx_gpu->base))
2330 return;
2331
2332 /*
2333 * There is a different programming path for A6xx targets with an
2334 * mmu500 attached, so detect if that is the case
2335 */
2336 phandle = of_parse_phandle(pdev->dev.of_node, "iommus", 0);
2337 a6xx_gpu->have_mmu500 = (phandle &&
2338 of_device_is_compatible(phandle, "arm,mmu-500"));
2339 of_node_put(phandle);
2340
2341 if (is_a7xx || !a6xx_gpu->have_mmu500)
2342 a6xx_gpu->llc_mmio = msm_ioremap(pdev, "cx_mem");
2343 else
2344 a6xx_gpu->llc_mmio = NULL;
2345
2346 a6xx_gpu->llc_slice = llcc_slice_getd(LLCC_GPU);
2347 a6xx_gpu->htw_llc_slice = llcc_slice_getd(LLCC_GPUHTW);
2348
2349 if (IS_ERR_OR_NULL(a6xx_gpu->llc_slice) && IS_ERR_OR_NULL(a6xx_gpu->htw_llc_slice))
2350 a6xx_gpu->llc_mmio = ERR_PTR(-EINVAL);
2351 }
2352
2353 #define GBIF_CLIENT_HALT_MASK BIT(0)
2354 #define GBIF_ARB_HALT_MASK BIT(1)
2355 #define VBIF_XIN_HALT_CTRL0_MASK GENMASK(3, 0)
2356 #define VBIF_RESET_ACK_MASK 0xF0
2357 #define GPR0_GBIF_HALT_REQUEST 0x1E0
2358
a6xx_bus_clear_pending_transactions(struct adreno_gpu * adreno_gpu,bool gx_off)2359 void a6xx_bus_clear_pending_transactions(struct adreno_gpu *adreno_gpu, bool gx_off)
2360 {
2361 struct msm_gpu *gpu = &adreno_gpu->base;
2362
2363 if (adreno_is_a619_holi(adreno_gpu)) {
2364 gpu_write(gpu, REG_A6XX_RBBM_GPR0_CNTL, GPR0_GBIF_HALT_REQUEST);
2365 spin_until((gpu_read(gpu, REG_A6XX_RBBM_VBIF_GX_RESET_STATUS) &
2366 (VBIF_RESET_ACK_MASK)) == VBIF_RESET_ACK_MASK);
2367 } else if (!a6xx_has_gbif(adreno_gpu)) {
2368 gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, VBIF_XIN_HALT_CTRL0_MASK);
2369 spin_until((gpu_read(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL1) &
2370 (VBIF_XIN_HALT_CTRL0_MASK)) == VBIF_XIN_HALT_CTRL0_MASK);
2371 gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0);
2372
2373 return;
2374 }
2375
2376 if (gx_off) {
2377 /* Halt the gx side of GBIF */
2378 gpu_write(gpu, REG_A6XX_RBBM_GBIF_HALT, 1);
2379 spin_until(gpu_read(gpu, REG_A6XX_RBBM_GBIF_HALT_ACK) & 1);
2380 }
2381
2382 /* Halt new client requests on GBIF */
2383 gpu_write(gpu, REG_A6XX_GBIF_HALT, GBIF_CLIENT_HALT_MASK);
2384 spin_until((gpu_read(gpu, REG_A6XX_GBIF_HALT_ACK) &
2385 (GBIF_CLIENT_HALT_MASK)) == GBIF_CLIENT_HALT_MASK);
2386
2387 /* Halt all AXI requests on GBIF */
2388 gpu_write(gpu, REG_A6XX_GBIF_HALT, GBIF_ARB_HALT_MASK);
2389 spin_until((gpu_read(gpu, REG_A6XX_GBIF_HALT_ACK) &
2390 (GBIF_ARB_HALT_MASK)) == GBIF_ARB_HALT_MASK);
2391
2392 /* The GBIF halt needs to be explicitly cleared */
2393 gpu_write(gpu, REG_A6XX_GBIF_HALT, 0x0);
2394 }
2395
a6xx_gpu_sw_reset(struct msm_gpu * gpu,bool assert)2396 void a6xx_gpu_sw_reset(struct msm_gpu *gpu, bool assert)
2397 {
2398 /* 11nm chips (e.g. ones with A610) have hw issues with the reset line! */
2399 if (adreno_is_a610(to_adreno_gpu(gpu)))
2400 return;
2401
2402 gpu_write(gpu, REG_A6XX_RBBM_SW_RESET_CMD, assert);
2403 /* Perform a bogus read and add a brief delay to ensure ordering. */
2404 gpu_read(gpu, REG_A6XX_RBBM_SW_RESET_CMD);
2405 udelay(1);
2406
2407 /* The reset line needs to be asserted for at least 100 us */
2408 if (assert)
2409 udelay(100);
2410 }
2411
a6xx_gmu_pm_resume(struct msm_gpu * gpu)2412 static int a6xx_gmu_pm_resume(struct msm_gpu *gpu)
2413 {
2414 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2415 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2416 int ret;
2417
2418 gpu->needs_hw_init = true;
2419
2420 trace_msm_gpu_resume(0);
2421
2422 mutex_lock(&a6xx_gpu->gmu.lock);
2423 ret = a6xx_gmu_resume(a6xx_gpu);
2424 mutex_unlock(&a6xx_gpu->gmu.lock);
2425 if (ret)
2426 return ret;
2427
2428 msm_devfreq_resume(gpu);
2429
2430 adreno_is_a7xx(adreno_gpu) ? a7xx_llc_activate : a6xx_llc_activate(a6xx_gpu);
2431
2432 return ret;
2433 }
2434
a6xx_pm_resume(struct msm_gpu * gpu)2435 static int a6xx_pm_resume(struct msm_gpu *gpu)
2436 {
2437 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2438 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2439 struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
2440 unsigned long freq = gpu->fast_rate;
2441 struct dev_pm_opp *opp;
2442 int ret;
2443
2444 gpu->needs_hw_init = true;
2445
2446 trace_msm_gpu_resume(0);
2447
2448 mutex_lock(&a6xx_gpu->gmu.lock);
2449
2450 opp = dev_pm_opp_find_freq_ceil(&gpu->pdev->dev, &freq);
2451 if (IS_ERR(opp)) {
2452 ret = PTR_ERR(opp);
2453 goto err_set_opp;
2454 }
2455 dev_pm_opp_put(opp);
2456
2457 /* Set the core clock and bus bw, having VDD scaling in mind */
2458 dev_pm_opp_set_opp(&gpu->pdev->dev, opp);
2459
2460 pm_runtime_resume_and_get(gmu->dev);
2461 pm_runtime_resume_and_get(gmu->gxpd);
2462
2463 ret = clk_bulk_prepare_enable(gpu->nr_clocks, gpu->grp_clks);
2464 if (ret)
2465 goto err_bulk_clk;
2466
2467 if (adreno_is_a619_holi(adreno_gpu))
2468 a6xx_sptprac_enable(gmu);
2469
2470 /* If anything goes south, tear the GPU down piece by piece.. */
2471 if (ret) {
2472 err_bulk_clk:
2473 pm_runtime_put(gmu->gxpd);
2474 pm_runtime_put(gmu->dev);
2475 dev_pm_opp_set_opp(&gpu->pdev->dev, NULL);
2476 }
2477 err_set_opp:
2478 mutex_unlock(&a6xx_gpu->gmu.lock);
2479
2480 if (!ret)
2481 msm_devfreq_resume(gpu);
2482
2483 return ret;
2484 }
2485
a6xx_gmu_pm_suspend(struct msm_gpu * gpu)2486 static int a6xx_gmu_pm_suspend(struct msm_gpu *gpu)
2487 {
2488 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2489 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2490 int i, ret;
2491
2492 trace_msm_gpu_suspend(0);
2493
2494 a6xx_llc_deactivate(a6xx_gpu);
2495
2496 msm_devfreq_suspend(gpu);
2497
2498 mutex_lock(&a6xx_gpu->gmu.lock);
2499 ret = a6xx_gmu_stop(a6xx_gpu);
2500 mutex_unlock(&a6xx_gpu->gmu.lock);
2501 if (ret)
2502 return ret;
2503
2504 if (a6xx_gpu->shadow_bo)
2505 for (i = 0; i < gpu->nr_rings; i++)
2506 a6xx_gpu->shadow[i] = 0;
2507
2508 gpu->suspend_count++;
2509
2510 return 0;
2511 }
2512
a6xx_pm_suspend(struct msm_gpu * gpu)2513 static int a6xx_pm_suspend(struct msm_gpu *gpu)
2514 {
2515 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2516 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2517 struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
2518 int i;
2519
2520 trace_msm_gpu_suspend(0);
2521
2522 msm_devfreq_suspend(gpu);
2523
2524 mutex_lock(&a6xx_gpu->gmu.lock);
2525
2526 /* Drain the outstanding traffic on memory buses */
2527 a6xx_bus_clear_pending_transactions(adreno_gpu, true);
2528
2529 if (adreno_is_a619_holi(adreno_gpu))
2530 a6xx_sptprac_disable(gmu);
2531
2532 clk_bulk_disable_unprepare(gpu->nr_clocks, gpu->grp_clks);
2533
2534 pm_runtime_put_sync(gmu->gxpd);
2535 dev_pm_opp_set_opp(&gpu->pdev->dev, NULL);
2536 pm_runtime_put_sync(gmu->dev);
2537
2538 mutex_unlock(&a6xx_gpu->gmu.lock);
2539
2540 if (a6xx_gpu->shadow_bo)
2541 for (i = 0; i < gpu->nr_rings; i++)
2542 a6xx_gpu->shadow[i] = 0;
2543
2544 gpu->suspend_count++;
2545
2546 return 0;
2547 }
2548
a6xx_gmu_get_timestamp(struct msm_gpu * gpu,uint64_t * value)2549 static int a6xx_gmu_get_timestamp(struct msm_gpu *gpu, uint64_t *value)
2550 {
2551 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2552 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2553
2554 mutex_lock(&a6xx_gpu->gmu.lock);
2555
2556 /* Force the GPU power on so we can read this register */
2557 a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET);
2558
2559 *value = gpu_read64(gpu, REG_A6XX_CP_ALWAYS_ON_COUNTER);
2560
2561 a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_PERFCOUNTER_SET);
2562
2563 mutex_unlock(&a6xx_gpu->gmu.lock);
2564
2565 return 0;
2566 }
2567
a6xx_get_timestamp(struct msm_gpu * gpu,uint64_t * value)2568 static int a6xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value)
2569 {
2570 *value = gpu_read64(gpu, REG_A6XX_CP_ALWAYS_ON_COUNTER);
2571 return 0;
2572 }
2573
a6xx_active_ring(struct msm_gpu * gpu)2574 static struct msm_ringbuffer *a6xx_active_ring(struct msm_gpu *gpu)
2575 {
2576 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2577 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2578
2579 return a6xx_gpu->cur_ring;
2580 }
2581
a6xx_destroy(struct msm_gpu * gpu)2582 static void a6xx_destroy(struct msm_gpu *gpu)
2583 {
2584 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2585 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2586
2587 if (a6xx_gpu->sqe_bo) {
2588 msm_gem_unpin_iova(a6xx_gpu->sqe_bo, gpu->aspace);
2589 drm_gem_object_put(a6xx_gpu->sqe_bo);
2590 }
2591
2592 if (a6xx_gpu->shadow_bo) {
2593 msm_gem_unpin_iova(a6xx_gpu->shadow_bo, gpu->aspace);
2594 drm_gem_object_put(a6xx_gpu->shadow_bo);
2595 }
2596
2597 a6xx_llc_slices_destroy(a6xx_gpu);
2598
2599 a6xx_gmu_remove(a6xx_gpu);
2600
2601 adreno_gpu_cleanup(adreno_gpu);
2602
2603 kfree(a6xx_gpu);
2604 }
2605
a6xx_gpu_busy(struct msm_gpu * gpu,unsigned long * out_sample_rate)2606 static u64 a6xx_gpu_busy(struct msm_gpu *gpu, unsigned long *out_sample_rate)
2607 {
2608 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2609 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2610 u64 busy_cycles;
2611
2612 /* 19.2MHz */
2613 *out_sample_rate = 19200000;
2614
2615 busy_cycles = gmu_read64(&a6xx_gpu->gmu,
2616 REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_L,
2617 REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_H);
2618
2619 return busy_cycles;
2620 }
2621
a6xx_gpu_set_freq(struct msm_gpu * gpu,struct dev_pm_opp * opp,bool suspended)2622 static void a6xx_gpu_set_freq(struct msm_gpu *gpu, struct dev_pm_opp *opp,
2623 bool suspended)
2624 {
2625 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2626 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2627
2628 mutex_lock(&a6xx_gpu->gmu.lock);
2629 a6xx_gmu_set_freq(gpu, opp, suspended);
2630 mutex_unlock(&a6xx_gpu->gmu.lock);
2631 }
2632
2633 static struct msm_gem_address_space *
a6xx_create_address_space(struct msm_gpu * gpu,struct platform_device * pdev)2634 a6xx_create_address_space(struct msm_gpu *gpu, struct platform_device *pdev)
2635 {
2636 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2637 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2638 unsigned long quirks = 0;
2639
2640 /*
2641 * This allows GPU to set the bus attributes required to use system
2642 * cache on behalf of the iommu page table walker.
2643 */
2644 if (!IS_ERR_OR_NULL(a6xx_gpu->htw_llc_slice) &&
2645 !device_iommu_capable(&pdev->dev, IOMMU_CAP_CACHE_COHERENCY))
2646 quirks |= IO_PGTABLE_QUIRK_ARM_OUTER_WBWA;
2647
2648 return adreno_iommu_create_address_space(gpu, pdev, quirks);
2649 }
2650
2651 static struct msm_gem_address_space *
a6xx_create_private_address_space(struct msm_gpu * gpu)2652 a6xx_create_private_address_space(struct msm_gpu *gpu)
2653 {
2654 struct msm_mmu *mmu;
2655
2656 mmu = msm_iommu_pagetable_create(gpu->aspace->mmu);
2657
2658 if (IS_ERR(mmu))
2659 return ERR_CAST(mmu);
2660
2661 return msm_gem_address_space_create(mmu,
2662 "gpu", 0x100000000ULL,
2663 adreno_private_address_space_size(gpu));
2664 }
2665
a6xx_get_rptr(struct msm_gpu * gpu,struct msm_ringbuffer * ring)2666 static uint32_t a6xx_get_rptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
2667 {
2668 struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
2669 struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
2670
2671 if (adreno_gpu->base.hw_apriv || a6xx_gpu->has_whereami)
2672 return a6xx_gpu->shadow[ring->id];
2673
2674 return ring->memptrs->rptr = gpu_read(gpu, REG_A6XX_CP_RB_RPTR);
2675 }
2676
a6xx_progress(struct msm_gpu * gpu,struct msm_ringbuffer * ring)2677 static bool a6xx_progress(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
2678 {
2679 struct msm_cp_state cp_state = {
2680 .ib1_base = gpu_read64(gpu, REG_A6XX_CP_IB1_BASE),
2681 .ib2_base = gpu_read64(gpu, REG_A6XX_CP_IB2_BASE),
2682 .ib1_rem = gpu_read(gpu, REG_A6XX_CP_IB1_REM_SIZE),
2683 .ib2_rem = gpu_read(gpu, REG_A6XX_CP_IB2_REM_SIZE),
2684 };
2685 bool progress;
2686
2687 /*
2688 * Adjust the remaining data to account for what has already been
2689 * fetched from memory, but not yet consumed by the SQE.
2690 *
2691 * This is not *technically* correct, the amount buffered could
2692 * exceed the IB size due to hw prefetching ahead, but:
2693 *
2694 * (1) We aren't trying to find the exact position, just whether
2695 * progress has been made
2696 * (2) The CP_REG_TO_MEM at the end of a submit should be enough
2697 * to prevent prefetching into an unrelated submit. (And
2698 * either way, at some point the ROQ will be full.)
2699 */
2700 cp_state.ib1_rem += gpu_read(gpu, REG_A6XX_CP_ROQ_AVAIL_IB1) >> 16;
2701 cp_state.ib2_rem += gpu_read(gpu, REG_A6XX_CP_ROQ_AVAIL_IB2) >> 16;
2702
2703 progress = !!memcmp(&cp_state, &ring->last_cp_state, sizeof(cp_state));
2704
2705 ring->last_cp_state = cp_state;
2706
2707 return progress;
2708 }
2709
fuse_to_supp_hw(const struct adreno_info * info,u32 fuse)2710 static u32 fuse_to_supp_hw(const struct adreno_info *info, u32 fuse)
2711 {
2712 if (!info->speedbins)
2713 return UINT_MAX;
2714
2715 for (int i = 0; info->speedbins[i].fuse != SHRT_MAX; i++)
2716 if (info->speedbins[i].fuse == fuse)
2717 return BIT(info->speedbins[i].speedbin);
2718
2719 return UINT_MAX;
2720 }
2721
a6xx_set_supported_hw(struct device * dev,const struct adreno_info * info)2722 static int a6xx_set_supported_hw(struct device *dev, const struct adreno_info *info)
2723 {
2724 u32 supp_hw;
2725 u32 speedbin;
2726 int ret;
2727
2728 ret = adreno_read_speedbin(dev, &speedbin);
2729 /*
2730 * -ENOENT means that the platform doesn't support speedbin which is
2731 * fine
2732 */
2733 if (ret == -ENOENT) {
2734 return 0;
2735 } else if (ret) {
2736 dev_err_probe(dev, ret,
2737 "failed to read speed-bin. Some OPPs may not be supported by hardware\n");
2738 return ret;
2739 }
2740
2741 supp_hw = fuse_to_supp_hw(info, speedbin);
2742
2743 if (supp_hw == UINT_MAX) {
2744 DRM_DEV_ERROR(dev,
2745 "missing support for speed-bin: %u. Some OPPs may not be supported by hardware\n",
2746 speedbin);
2747 supp_hw = BIT(0); /* Default */
2748 }
2749
2750 ret = devm_pm_opp_set_supported_hw(dev, &supp_hw, 1);
2751 if (ret)
2752 return ret;
2753
2754 return 0;
2755 }
2756
2757 static const struct adreno_gpu_funcs funcs = {
2758 .base = {
2759 .get_param = adreno_get_param,
2760 .set_param = adreno_set_param,
2761 .hw_init = a6xx_hw_init,
2762 .ucode_load = a6xx_ucode_load,
2763 .pm_suspend = a6xx_gmu_pm_suspend,
2764 .pm_resume = a6xx_gmu_pm_resume,
2765 .recover = a6xx_recover,
2766 .submit = a6xx_submit,
2767 .active_ring = a6xx_active_ring,
2768 .irq = a6xx_irq,
2769 .destroy = a6xx_destroy,
2770 #if defined(CONFIG_DRM_MSM_GPU_STATE)
2771 .show = a6xx_show,
2772 #endif
2773 .gpu_busy = a6xx_gpu_busy,
2774 .gpu_get_freq = a6xx_gmu_get_freq,
2775 .gpu_set_freq = a6xx_gpu_set_freq,
2776 #if defined(CONFIG_DRM_MSM_GPU_STATE)
2777 .gpu_state_get = a6xx_gpu_state_get,
2778 .gpu_state_put = a6xx_gpu_state_put,
2779 #endif
2780 .create_address_space = a6xx_create_address_space,
2781 .create_private_address_space = a6xx_create_private_address_space,
2782 .get_rptr = a6xx_get_rptr,
2783 .progress = a6xx_progress,
2784 },
2785 .get_timestamp = a6xx_gmu_get_timestamp,
2786 };
2787
2788 static const struct adreno_gpu_funcs funcs_gmuwrapper = {
2789 .base = {
2790 .get_param = adreno_get_param,
2791 .set_param = adreno_set_param,
2792 .hw_init = a6xx_hw_init,
2793 .ucode_load = a6xx_ucode_load,
2794 .pm_suspend = a6xx_pm_suspend,
2795 .pm_resume = a6xx_pm_resume,
2796 .recover = a6xx_recover,
2797 .submit = a6xx_submit,
2798 .active_ring = a6xx_active_ring,
2799 .irq = a6xx_irq,
2800 .destroy = a6xx_destroy,
2801 #if defined(CONFIG_DRM_MSM_GPU_STATE)
2802 .show = a6xx_show,
2803 #endif
2804 .gpu_busy = a6xx_gpu_busy,
2805 #if defined(CONFIG_DRM_MSM_GPU_STATE)
2806 .gpu_state_get = a6xx_gpu_state_get,
2807 .gpu_state_put = a6xx_gpu_state_put,
2808 #endif
2809 .create_address_space = a6xx_create_address_space,
2810 .create_private_address_space = a6xx_create_private_address_space,
2811 .get_rptr = a6xx_get_rptr,
2812 .progress = a6xx_progress,
2813 },
2814 .get_timestamp = a6xx_get_timestamp,
2815 };
2816
2817 static const struct adreno_gpu_funcs funcs_a7xx = {
2818 .base = {
2819 .get_param = adreno_get_param,
2820 .set_param = adreno_set_param,
2821 .hw_init = a6xx_hw_init,
2822 .ucode_load = a6xx_ucode_load,
2823 .pm_suspend = a6xx_gmu_pm_suspend,
2824 .pm_resume = a6xx_gmu_pm_resume,
2825 .recover = a6xx_recover,
2826 .submit = a7xx_submit,
2827 .active_ring = a6xx_active_ring,
2828 .irq = a6xx_irq,
2829 .destroy = a6xx_destroy,
2830 #if defined(CONFIG_DRM_MSM_GPU_STATE)
2831 .show = a6xx_show,
2832 #endif
2833 .gpu_busy = a6xx_gpu_busy,
2834 .gpu_get_freq = a6xx_gmu_get_freq,
2835 .gpu_set_freq = a6xx_gpu_set_freq,
2836 #if defined(CONFIG_DRM_MSM_GPU_STATE)
2837 .gpu_state_get = a6xx_gpu_state_get,
2838 .gpu_state_put = a6xx_gpu_state_put,
2839 #endif
2840 .create_address_space = a6xx_create_address_space,
2841 .create_private_address_space = a6xx_create_private_address_space,
2842 .get_rptr = a6xx_get_rptr,
2843 .progress = a6xx_progress,
2844 },
2845 .get_timestamp = a6xx_gmu_get_timestamp,
2846 };
2847
a6xx_gpu_init(struct drm_device * dev)2848 struct msm_gpu *a6xx_gpu_init(struct drm_device *dev)
2849 {
2850 struct msm_drm_private *priv = dev->dev_private;
2851 struct platform_device *pdev = priv->gpu_pdev;
2852 struct adreno_platform_config *config = pdev->dev.platform_data;
2853 struct device_node *node;
2854 struct a6xx_gpu *a6xx_gpu;
2855 struct adreno_gpu *adreno_gpu;
2856 struct msm_gpu *gpu;
2857 bool is_a7xx;
2858 int ret;
2859
2860 a6xx_gpu = kzalloc(sizeof(*a6xx_gpu), GFP_KERNEL);
2861 if (!a6xx_gpu)
2862 return ERR_PTR(-ENOMEM);
2863
2864 adreno_gpu = &a6xx_gpu->base;
2865 gpu = &adreno_gpu->base;
2866
2867 mutex_init(&a6xx_gpu->gmu.lock);
2868
2869 adreno_gpu->registers = NULL;
2870
2871 /* Check if there is a GMU phandle and set it up */
2872 node = of_parse_phandle(pdev->dev.of_node, "qcom,gmu", 0);
2873 /* FIXME: How do we gracefully handle this? */
2874 BUG_ON(!node);
2875
2876 adreno_gpu->gmu_is_wrapper = of_device_is_compatible(node, "qcom,adreno-gmu-wrapper");
2877
2878 adreno_gpu->base.hw_apriv =
2879 !!(config->info->quirks & ADRENO_QUIRK_HAS_HW_APRIV);
2880
2881 /* gpu->info only gets assigned in adreno_gpu_init() */
2882 is_a7xx = config->info->family == ADRENO_7XX_GEN1 ||
2883 config->info->family == ADRENO_7XX_GEN2;
2884
2885 a6xx_llc_slices_init(pdev, a6xx_gpu, is_a7xx);
2886
2887 ret = a6xx_set_supported_hw(&pdev->dev, config->info);
2888 if (ret) {
2889 a6xx_destroy(&(a6xx_gpu->base.base));
2890 return ERR_PTR(ret);
2891 }
2892
2893 if (is_a7xx)
2894 ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs_a7xx, 1);
2895 else if (adreno_has_gmu_wrapper(adreno_gpu))
2896 ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs_gmuwrapper, 1);
2897 else
2898 ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs, 1);
2899 if (ret) {
2900 a6xx_destroy(&(a6xx_gpu->base.base));
2901 return ERR_PTR(ret);
2902 }
2903
2904 /*
2905 * For now only clamp to idle freq for devices where this is known not
2906 * to cause power supply issues:
2907 */
2908 if (adreno_is_a618(adreno_gpu) || adreno_is_7c3(adreno_gpu))
2909 priv->gpu_clamp_to_idle = true;
2910
2911 if (adreno_has_gmu_wrapper(adreno_gpu))
2912 ret = a6xx_gmu_wrapper_init(a6xx_gpu, node);
2913 else
2914 ret = a6xx_gmu_init(a6xx_gpu, node);
2915 of_node_put(node);
2916 if (ret) {
2917 a6xx_destroy(&(a6xx_gpu->base.base));
2918 return ERR_PTR(ret);
2919 }
2920
2921 if (gpu->aspace)
2922 msm_mmu_set_fault_handler(gpu->aspace->mmu, gpu,
2923 a6xx_fault_handler);
2924
2925 a6xx_calc_ubwc_config(adreno_gpu);
2926
2927 return gpu;
2928 }
2929