1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Processor capabilities determination functions.
4 *
5 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
6 */
7 #include <linux/init.h>
8 #include <linux/kernel.h>
9 #include <linux/ptrace.h>
10 #include <linux/smp.h>
11 #include <linux/stddef.h>
12 #include <linux/export.h>
13 #include <linux/printk.h>
14 #include <linux/uaccess.h>
15
16 #include <asm/cpu-features.h>
17 #include <asm/elf.h>
18 #include <asm/fpu.h>
19 #include <asm/loongarch.h>
20 #include <asm/pgtable-bits.h>
21 #include <asm/setup.h>
22
23 /* Hardware capabilities */
24 unsigned int elf_hwcap __read_mostly;
25 EXPORT_SYMBOL_GPL(elf_hwcap);
26
27 /*
28 * Determine the FCSR mask for FPU hardware.
29 */
cpu_set_fpu_fcsr_mask(struct cpuinfo_loongarch * c)30 static inline void cpu_set_fpu_fcsr_mask(struct cpuinfo_loongarch *c)
31 {
32 unsigned long sr, mask, fcsr, fcsr0, fcsr1;
33
34 fcsr = c->fpu_csr0;
35 mask = FPU_CSR_ALL_X | FPU_CSR_ALL_E | FPU_CSR_ALL_S | FPU_CSR_RM;
36
37 sr = read_csr_euen();
38 enable_fpu();
39
40 fcsr0 = fcsr & mask;
41 write_fcsr(LOONGARCH_FCSR0, fcsr0);
42 fcsr0 = read_fcsr(LOONGARCH_FCSR0);
43
44 fcsr1 = fcsr | ~mask;
45 write_fcsr(LOONGARCH_FCSR0, fcsr1);
46 fcsr1 = read_fcsr(LOONGARCH_FCSR0);
47
48 write_fcsr(LOONGARCH_FCSR0, fcsr);
49
50 write_csr_euen(sr);
51
52 c->fpu_mask = ~(fcsr0 ^ fcsr1) & ~mask;
53 }
54
55 /* simd = -1/0/128/256 */
56 static unsigned int simd = -1U;
57
cpu_setup_simd(char * str)58 static int __init cpu_setup_simd(char *str)
59 {
60 get_option(&str, &simd);
61 pr_info("Set SIMD width = %u\n", simd);
62
63 return 0;
64 }
65
66 early_param("simd", cpu_setup_simd);
67
cpu_final_simd(void)68 static int __init cpu_final_simd(void)
69 {
70 struct cpuinfo_loongarch *c = &cpu_data[0];
71
72 if (simd < 128) {
73 c->options &= ~LOONGARCH_CPU_LSX;
74 elf_hwcap &= ~HWCAP_LOONGARCH_LSX;
75 }
76
77 if (simd < 256) {
78 c->options &= ~LOONGARCH_CPU_LASX;
79 elf_hwcap &= ~HWCAP_LOONGARCH_LASX;
80 }
81
82 simd = 0;
83
84 if (c->options & LOONGARCH_CPU_LSX)
85 simd = 128;
86
87 if (c->options & LOONGARCH_CPU_LASX)
88 simd = 256;
89
90 pr_info("Final SIMD width = %u\n", simd);
91
92 return 0;
93 }
94
95 arch_initcall(cpu_final_simd);
96
set_elf_platform(int cpu,const char * plat)97 static inline void set_elf_platform(int cpu, const char *plat)
98 {
99 if (cpu == 0)
100 __elf_platform = plat;
101 }
102
103 /* MAP BASE */
104 unsigned long vm_map_base;
105 EXPORT_SYMBOL(vm_map_base);
106
cpu_probe_addrbits(struct cpuinfo_loongarch * c)107 static void cpu_probe_addrbits(struct cpuinfo_loongarch *c)
108 {
109 #ifdef CONFIG_32BIT
110 c->pabits = cpu_pabits;
111 c->vabits = cpu_vabits;
112 vm_map_base = KVRANGE;
113 #else
114 c->pabits = (read_cpucfg(LOONGARCH_CPUCFG1) & CPUCFG1_PABITS) >> 4;
115 c->vabits = (read_cpucfg(LOONGARCH_CPUCFG1) & CPUCFG1_VABITS) >> 12;
116 vm_map_base = 0UL - (1UL << c->vabits);
117 #endif
118 }
119
set_isa(struct cpuinfo_loongarch * c,unsigned int isa)120 static void set_isa(struct cpuinfo_loongarch *c, unsigned int isa)
121 {
122 switch (isa) {
123 case LOONGARCH_CPU_ISA_LA64:
124 c->isa_level |= LOONGARCH_CPU_ISA_LA64;
125 fallthrough;
126 case LOONGARCH_CPU_ISA_LA32S:
127 c->isa_level |= LOONGARCH_CPU_ISA_LA32S;
128 fallthrough;
129 case LOONGARCH_CPU_ISA_LA32R:
130 c->isa_level |= LOONGARCH_CPU_ISA_LA32R;
131 break;
132 }
133 }
134
cpu_probe_common(struct cpuinfo_loongarch * c)135 static void cpu_probe_common(struct cpuinfo_loongarch *c)
136 {
137 unsigned int config;
138 unsigned long asid_mask;
139
140 c->options = LOONGARCH_CPU_CPUCFG | LOONGARCH_CPU_CSR | LOONGARCH_CPU_VINT;
141
142 elf_hwcap = HWCAP_LOONGARCH_CPUCFG;
143
144 config = read_cpucfg(LOONGARCH_CPUCFG1);
145
146 switch (config & CPUCFG1_ISA) {
147 case 0:
148 set_isa(c, LOONGARCH_CPU_ISA_LA32R);
149 break;
150 case 1:
151 set_isa(c, LOONGARCH_CPU_ISA_LA32S);
152 break;
153 case 2:
154 set_isa(c, LOONGARCH_CPU_ISA_LA64);
155 break;
156 default:
157 pr_warn("Warning: unknown ISA level\n");
158 }
159
160 if (config & CPUCFG1_PAGING)
161 c->options |= LOONGARCH_CPU_TLB;
162 if (config & CPUCFG1_IOCSR)
163 c->options |= LOONGARCH_CPU_IOCSR;
164 if (config & CPUCFG1_MSGINT)
165 c->options |= LOONGARCH_CPU_MSGINT;
166 if (config & CPUCFG1_UAL) {
167 c->options |= LOONGARCH_CPU_UAL;
168 elf_hwcap |= HWCAP_LOONGARCH_UAL;
169 }
170 if (config & CPUCFG1_CRC32) {
171 c->options |= LOONGARCH_CPU_CRC32;
172 elf_hwcap |= HWCAP_LOONGARCH_CRC32;
173 }
174
175 config = read_cpucfg(LOONGARCH_CPUCFG2);
176 if (config & CPUCFG2_LAM) {
177 c->options |= LOONGARCH_CPU_LAM;
178 elf_hwcap |= HWCAP_LOONGARCH_LAM;
179 }
180 if (config & CPUCFG2_SCQ) {
181 c->options |= LOONGARCH_CPU_SCQ;
182 elf_hwcap |= HWCAP_LOONGARCH_SCQ;
183 }
184 if (config & CPUCFG2_FP) {
185 c->options |= LOONGARCH_CPU_FPU;
186 elf_hwcap |= HWCAP_LOONGARCH_FPU;
187 }
188 #ifdef CONFIG_CPU_HAS_LSX
189 if ((config & CPUCFG2_LSX) && (simd >= 128)) {
190 c->options |= LOONGARCH_CPU_LSX;
191 elf_hwcap |= HWCAP_LOONGARCH_LSX;
192 }
193 #endif
194 #ifdef CONFIG_CPU_HAS_LASX
195 if ((config & CPUCFG2_LASX) && (simd >= 256)) {
196 c->options |= LOONGARCH_CPU_LASX;
197 elf_hwcap |= HWCAP_LOONGARCH_LASX;
198 }
199 #endif
200 if (config & CPUCFG2_COMPLEX) {
201 c->options |= LOONGARCH_CPU_COMPLEX;
202 elf_hwcap |= HWCAP_LOONGARCH_COMPLEX;
203 }
204 if (config & CPUCFG2_CRYPTO) {
205 c->options |= LOONGARCH_CPU_CRYPTO;
206 elf_hwcap |= HWCAP_LOONGARCH_CRYPTO;
207 }
208 if (config & CPUCFG2_PTW) {
209 c->options |= LOONGARCH_CPU_PTW;
210 elf_hwcap |= HWCAP_LOONGARCH_PTW;
211 }
212 if (config & CPUCFG2_LSPW) {
213 c->options |= LOONGARCH_CPU_LSPW;
214 elf_hwcap |= HWCAP_LOONGARCH_LSPW;
215 }
216 if (config & CPUCFG2_LVZP) {
217 c->options |= LOONGARCH_CPU_LVZ;
218 elf_hwcap |= HWCAP_LOONGARCH_LVZ;
219 }
220 #ifdef CONFIG_CPU_HAS_LBT
221 if (config & CPUCFG2_X86BT) {
222 c->options |= LOONGARCH_CPU_LBT_X86;
223 elf_hwcap |= HWCAP_LOONGARCH_LBT_X86;
224 }
225 if (config & CPUCFG2_ARMBT) {
226 c->options |= LOONGARCH_CPU_LBT_ARM;
227 elf_hwcap |= HWCAP_LOONGARCH_LBT_ARM;
228 }
229 if (config & CPUCFG2_MIPSBT) {
230 c->options |= LOONGARCH_CPU_LBT_MIPS;
231 elf_hwcap |= HWCAP_LOONGARCH_LBT_MIPS;
232 }
233 #endif
234
235 config = read_cpucfg(LOONGARCH_CPUCFG6);
236 if (config & CPUCFG6_PMP)
237 c->options |= LOONGARCH_CPU_PMP;
238
239 config = csr_read32(LOONGARCH_CSR_ASID);
240 config = (config & CSR_ASID_BIT) >> CSR_ASID_BIT_SHIFT;
241 asid_mask = GENMASK(config - 1, 0);
242 set_cpu_asid_mask(c, asid_mask);
243
244 config = read_csr_prcfg1();
245 c->timerbits = (config & CSR_CONF1_TMRBITS) >> CSR_CONF1_TMRBITS_SHIFT;
246 c->ksave_mask = GENMASK((config & CSR_CONF1_KSNUM) - 1, 0);
247 c->ksave_mask &= ~(EXC_KSAVE_MASK | PERCPU_KSAVE_MASK | KVM_KSAVE_MASK);
248
249 config = read_csr_prcfg3();
250 switch (config & CSR_CONF3_TLBTYPE) {
251 case 0:
252 c->tlbsizemtlb = 0;
253 c->tlbsizestlbsets = 0;
254 c->tlbsizestlbways = 0;
255 c->tlbsize = 0;
256 break;
257 case 1:
258 c->tlbsizemtlb = ((config & CSR_CONF3_MTLBSIZE) >> CSR_CONF3_MTLBSIZE_SHIFT) + 1;
259 c->tlbsizestlbsets = 0;
260 c->tlbsizestlbways = 0;
261 c->tlbsize = c->tlbsizemtlb + c->tlbsizestlbsets * c->tlbsizestlbways;
262 break;
263 case 2:
264 c->tlbsizemtlb = ((config & CSR_CONF3_MTLBSIZE) >> CSR_CONF3_MTLBSIZE_SHIFT) + 1;
265 c->tlbsizestlbsets = 1 << ((config & CSR_CONF3_STLBIDX) >> CSR_CONF3_STLBIDX_SHIFT);
266 c->tlbsizestlbways = ((config & CSR_CONF3_STLBWAYS) >> CSR_CONF3_STLBWAYS_SHIFT) + 1;
267 c->tlbsize = c->tlbsizemtlb + c->tlbsizestlbsets * c->tlbsizestlbways;
268 break;
269 default:
270 pr_warn("Warning: unknown TLB type\n");
271 }
272
273 if (get_num_brps() + get_num_wrps())
274 c->options |= LOONGARCH_CPU_WATCH;
275 }
276
277 #define MAX_NAME_LEN 32
278 #define VENDOR_OFFSET 0
279 #define CPUNAME_OFFSET 9
280
281 static char cpu_full_name[MAX_NAME_LEN] = " - ";
282
cpu_probe_loongson(struct cpuinfo_loongarch * c,unsigned int cpu)283 static inline void cpu_probe_loongson(struct cpuinfo_loongarch *c, unsigned int cpu)
284 {
285 uint32_t config;
286 uint64_t *vendor = (void *)(&cpu_full_name[VENDOR_OFFSET]);
287 uint64_t *cpuname = (void *)(&cpu_full_name[CPUNAME_OFFSET]);
288 const char *core_name = id_to_core_name(c->processor_id);
289
290 switch (BIT(fls(c->isa_level) - 1)) {
291 case LOONGARCH_CPU_ISA_LA32R:
292 case LOONGARCH_CPU_ISA_LA32S:
293 c->cputype = CPU_LOONGSON32;
294 __cpu_family[cpu] = "Loongson-32bit";
295 break;
296 case LOONGARCH_CPU_ISA_LA64:
297 c->cputype = CPU_LOONGSON64;
298 __cpu_family[cpu] = "Loongson-64bit";
299 break;
300 }
301
302 pr_info("%s Processor probed (%s Core)\n", __cpu_family[cpu], core_name);
303
304 if (!cpu_has_iocsr) {
305 __cpu_full_name[cpu] = "Unknown";
306 return;
307 }
308
309 #ifdef CONFIG_64BIT
310 *vendor = iocsr_read64(LOONGARCH_IOCSR_VENDOR);
311 *cpuname = iocsr_read64(LOONGARCH_IOCSR_CPUNAME);
312 #else
313 *vendor = iocsr_read32(LOONGARCH_IOCSR_VENDOR) |
314 (u64)iocsr_read32(LOONGARCH_IOCSR_VENDOR + 4) << 32;
315 *cpuname = iocsr_read32(LOONGARCH_IOCSR_CPUNAME) |
316 (u64)iocsr_read32(LOONGARCH_IOCSR_CPUNAME + 4) << 32;
317 #endif
318
319 if (!__cpu_full_name[cpu]) {
320 if (((char *)vendor)[0] == 0)
321 __cpu_full_name[cpu] = "Unknown";
322 else
323 __cpu_full_name[cpu] = cpu_full_name;
324 }
325
326 config = iocsr_read32(LOONGARCH_IOCSR_FEATURES);
327 if (config & IOCSRF_CSRIPI)
328 c->options |= LOONGARCH_CPU_CSRIPI;
329 if (config & IOCSRF_EXTIOI)
330 c->options |= LOONGARCH_CPU_EXTIOI;
331 if (config & IOCSRF_FREQSCALE)
332 c->options |= LOONGARCH_CPU_SCALEFREQ;
333 if (config & IOCSRF_FLATMODE)
334 c->options |= LOONGARCH_CPU_FLATMODE;
335 if (config & IOCSRF_EIODECODE)
336 c->options |= LOONGARCH_CPU_EIODECODE;
337 if (config & IOCSRF_AVEC)
338 c->options |= LOONGARCH_CPU_AVECINT;
339 if (config & IOCSRF_REDIRECT)
340 c->options |= LOONGARCH_CPU_REDIRECTINT;
341 if (config & IOCSRF_VM)
342 c->options |= LOONGARCH_CPU_HYPERVISOR;
343 }
344
345 #ifdef CONFIG_64BIT
346 /* For use by uaccess.h */
347 u64 __ua_limit;
348 EXPORT_SYMBOL(__ua_limit);
349 #endif
350
351 const char *__cpu_family[NR_CPUS];
352 const char *__cpu_full_name[NR_CPUS];
353 const char *__elf_platform;
354
cpu_report(void)355 static void cpu_report(void)
356 {
357 struct cpuinfo_loongarch *c = ¤t_cpu_data;
358
359 pr_info("CPU%d revision is: %08x (%s)\n",
360 smp_processor_id(), c->processor_id, cpu_family_string());
361 if (c->options & LOONGARCH_CPU_FPU)
362 pr_info("FPU%d revision is: %08x\n", smp_processor_id(), c->fpu_vers);
363 }
364
cpu_probe(void)365 void cpu_probe(void)
366 {
367 unsigned int cpu = smp_processor_id();
368 struct cpuinfo_loongarch *c = ¤t_cpu_data;
369
370 /*
371 * Set a default ELF platform, cpu probe may later
372 * overwrite it with a more precise value
373 */
374 set_elf_platform(cpu, "loongarch");
375
376 c->cputype = CPU_UNKNOWN;
377 c->processor_id = read_cpucfg(LOONGARCH_CPUCFG0);
378 c->fpu_vers = (read_cpucfg(LOONGARCH_CPUCFG2) & CPUCFG2_FPVERS) >> 3;
379
380 c->fpu_csr0 = FPU_CSR_RN;
381 c->fpu_mask = FPU_CSR_RSVD;
382
383 cpu_probe_common(c);
384
385 per_cpu_trap_init(cpu);
386
387 switch (c->processor_id & PRID_COMP_MASK) {
388 case PRID_COMP_LOONGSON:
389 cpu_probe_loongson(c, cpu);
390 break;
391 }
392
393 BUG_ON(!__cpu_family[cpu]);
394 BUG_ON(c->cputype == CPU_UNKNOWN);
395
396 cpu_probe_addrbits(c);
397
398 #ifdef CONFIG_64BIT
399 if (cpu == 0)
400 __ua_limit = ~((1ull << cpu_vabits) - 1);
401 #endif
402
403 cpu_report();
404 }
405