1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2021 Heiko Stuebner <heiko@sntech.de>
4 */
5
6 #include <linux/bug.h>
7 #include <linux/kernel.h>
8 #include <linux/memory.h>
9 #include <linux/module.h>
10 #include <linux/string.h>
11 #include <linux/uaccess.h>
12 #include <asm/alternative.h>
13 #include <asm/cacheflush.h>
14 #include <asm/cpufeature.h>
15 #include <asm/dma-noncoherent.h>
16 #include <asm/errata_list.h>
17 #include <asm/hwprobe.h>
18 #include <asm/io.h>
19 #include <asm/patch.h>
20 #include <asm/vendorid_list.h>
21
errata_probe_pbmt(unsigned int stage,unsigned long arch_id,unsigned long impid)22 static bool errata_probe_pbmt(unsigned int stage,
23 unsigned long arch_id, unsigned long impid)
24 {
25 if (!IS_ENABLED(CONFIG_ERRATA_THEAD_PBMT))
26 return false;
27
28 if (arch_id != 0 || impid != 0)
29 return false;
30
31 if (stage == RISCV_ALTERNATIVES_EARLY_BOOT ||
32 stage == RISCV_ALTERNATIVES_MODULE)
33 return true;
34
35 return false;
36 }
37
38 /*
39 * th.dcache.ipa rs1 (invalidate, physical address)
40 * | 31 - 25 | 24 - 20 | 19 - 15 | 14 - 12 | 11 - 7 | 6 - 0 |
41 * 0000001 01010 rs1 000 00000 0001011
42 * th.dcache.iva rs1 (invalidate, virtual address)
43 * 0000001 00110 rs1 000 00000 0001011
44 *
45 * th.dcache.cpa rs1 (clean, physical address)
46 * | 31 - 25 | 24 - 20 | 19 - 15 | 14 - 12 | 11 - 7 | 6 - 0 |
47 * 0000001 01001 rs1 000 00000 0001011
48 * th.dcache.cva rs1 (clean, virtual address)
49 * 0000001 00101 rs1 000 00000 0001011
50 *
51 * th.dcache.cipa rs1 (clean then invalidate, physical address)
52 * | 31 - 25 | 24 - 20 | 19 - 15 | 14 - 12 | 11 - 7 | 6 - 0 |
53 * 0000001 01011 rs1 000 00000 0001011
54 * th.dcache.civa rs1 (clean then invalidate, virtual address)
55 * 0000001 00111 rs1 000 00000 0001011
56 *
57 * th.sync.s (make sure all cache operations finished)
58 * | 31 - 25 | 24 - 20 | 19 - 15 | 14 - 12 | 11 - 7 | 6 - 0 |
59 * 0000000 11001 00000 000 00000 0001011
60 */
61 #define THEAD_INVAL_A0 ".long 0x02a5000b"
62 #define THEAD_CLEAN_A0 ".long 0x0295000b"
63 #define THEAD_FLUSH_A0 ".long 0x02b5000b"
64 #define THEAD_SYNC_S ".long 0x0190000b"
65
66 #define THEAD_CMO_OP(_op, _start, _size, _cachesize) \
67 asm volatile("mv a0, %1\n\t" \
68 "j 2f\n\t" \
69 "3:\n\t" \
70 THEAD_##_op##_A0 "\n\t" \
71 "add a0, a0, %0\n\t" \
72 "2:\n\t" \
73 "bltu a0, %2, 3b\n\t" \
74 THEAD_SYNC_S \
75 : : "r"(_cachesize), \
76 "r"((unsigned long)(_start) & ~((_cachesize) - 1UL)), \
77 "r"((unsigned long)(_start) + (_size)) \
78 : "a0")
79
thead_errata_cache_inv(phys_addr_t paddr,size_t size)80 static void thead_errata_cache_inv(phys_addr_t paddr, size_t size)
81 {
82 THEAD_CMO_OP(INVAL, paddr, size, riscv_cbom_block_size);
83 }
84
thead_errata_cache_wback(phys_addr_t paddr,size_t size)85 static void thead_errata_cache_wback(phys_addr_t paddr, size_t size)
86 {
87 THEAD_CMO_OP(CLEAN, paddr, size, riscv_cbom_block_size);
88 }
89
thead_errata_cache_wback_inv(phys_addr_t paddr,size_t size)90 static void thead_errata_cache_wback_inv(phys_addr_t paddr, size_t size)
91 {
92 THEAD_CMO_OP(FLUSH, paddr, size, riscv_cbom_block_size);
93 }
94
95 static const struct riscv_nonstd_cache_ops thead_errata_cmo_ops = {
96 .wback = &thead_errata_cache_wback,
97 .inv = &thead_errata_cache_inv,
98 .wback_inv = &thead_errata_cache_wback_inv,
99 };
100
errata_probe_cmo(unsigned int stage,unsigned long arch_id,unsigned long impid)101 static bool errata_probe_cmo(unsigned int stage,
102 unsigned long arch_id, unsigned long impid)
103 {
104 if (!IS_ENABLED(CONFIG_ERRATA_THEAD_CMO))
105 return false;
106
107 if (arch_id != 0 || impid != 0)
108 return false;
109
110 if (stage == RISCV_ALTERNATIVES_EARLY_BOOT)
111 return false;
112
113 if (stage == RISCV_ALTERNATIVES_BOOT) {
114 riscv_cbom_block_size = L1_CACHE_BYTES;
115 riscv_noncoherent_supported();
116 riscv_noncoherent_register_cache_ops(&thead_errata_cmo_ops);
117 }
118
119 return true;
120 }
121
errata_probe_pmu(unsigned int stage,unsigned long arch_id,unsigned long impid)122 static bool errata_probe_pmu(unsigned int stage,
123 unsigned long arch_id, unsigned long impid)
124 {
125 if (!IS_ENABLED(CONFIG_ERRATA_THEAD_PMU))
126 return false;
127
128 /* target-c9xx cores report arch_id and impid as 0 */
129 if (arch_id != 0 || impid != 0)
130 return false;
131
132 if (stage == RISCV_ALTERNATIVES_EARLY_BOOT)
133 return false;
134
135 return true;
136 }
137
thead_errata_probe(unsigned int stage,unsigned long archid,unsigned long impid)138 static u32 thead_errata_probe(unsigned int stage,
139 unsigned long archid, unsigned long impid)
140 {
141 u32 cpu_req_errata = 0;
142
143 if (errata_probe_pbmt(stage, archid, impid))
144 cpu_req_errata |= BIT(ERRATA_THEAD_PBMT);
145
146 errata_probe_cmo(stage, archid, impid);
147
148 if (errata_probe_pmu(stage, archid, impid))
149 cpu_req_errata |= BIT(ERRATA_THEAD_PMU);
150
151 return cpu_req_errata;
152 }
153
thead_errata_patch_func(struct alt_entry * begin,struct alt_entry * end,unsigned long archid,unsigned long impid,unsigned int stage)154 void thead_errata_patch_func(struct alt_entry *begin, struct alt_entry *end,
155 unsigned long archid, unsigned long impid,
156 unsigned int stage)
157 {
158 struct alt_entry *alt;
159 u32 cpu_req_errata = thead_errata_probe(stage, archid, impid);
160 u32 tmp;
161 void *oldptr, *altptr;
162
163 for (alt = begin; alt < end; alt++) {
164 if (alt->vendor_id != THEAD_VENDOR_ID)
165 continue;
166 if (alt->patch_id >= ERRATA_THEAD_NUMBER)
167 continue;
168
169 tmp = (1U << alt->patch_id);
170 if (cpu_req_errata & tmp) {
171 oldptr = ALT_OLD_PTR(alt);
172 altptr = ALT_ALT_PTR(alt);
173
174 /* On vm-alternatives, the mmu isn't running yet */
175 if (stage == RISCV_ALTERNATIVES_EARLY_BOOT) {
176 memcpy(oldptr, altptr, alt->alt_len);
177 } else {
178 mutex_lock(&text_mutex);
179 patch_text_nosync(oldptr, altptr, alt->alt_len);
180 mutex_unlock(&text_mutex);
181 }
182 }
183 }
184
185 if (stage == RISCV_ALTERNATIVES_EARLY_BOOT)
186 local_flush_icache_all();
187 }
188