1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2020 Western Digital Corporation or its affiliates.
4 */
5 #include <linux/kernel.h>
6 #include <linux/init.h>
7 #include <linux/mm.h>
8 #include <linux/module.h>
9 #include <linux/perf_event.h>
10 #include <linux/irq.h>
11 #include <linux/stringify.h>
12
13 #include <asm/processor.h>
14 #include <asm/ptrace.h>
15 #include <asm/csr.h>
16 #include <asm/entry-common.h>
17 #include <asm/hwprobe.h>
18 #include <asm/cpufeature.h>
19 #include <asm/vector.h>
20
21 #define INSN_MATCH_LB 0x3
22 #define INSN_MASK_LB 0x707f
23 #define INSN_MATCH_LH 0x1003
24 #define INSN_MASK_LH 0x707f
25 #define INSN_MATCH_LW 0x2003
26 #define INSN_MASK_LW 0x707f
27 #define INSN_MATCH_LD 0x3003
28 #define INSN_MASK_LD 0x707f
29 #define INSN_MATCH_LBU 0x4003
30 #define INSN_MASK_LBU 0x707f
31 #define INSN_MATCH_LHU 0x5003
32 #define INSN_MASK_LHU 0x707f
33 #define INSN_MATCH_LWU 0x6003
34 #define INSN_MASK_LWU 0x707f
35 #define INSN_MATCH_SB 0x23
36 #define INSN_MASK_SB 0x707f
37 #define INSN_MATCH_SH 0x1023
38 #define INSN_MASK_SH 0x707f
39 #define INSN_MATCH_SW 0x2023
40 #define INSN_MASK_SW 0x707f
41 #define INSN_MATCH_SD 0x3023
42 #define INSN_MASK_SD 0x707f
43
44 #define INSN_MATCH_FLW 0x2007
45 #define INSN_MASK_FLW 0x707f
46 #define INSN_MATCH_FLD 0x3007
47 #define INSN_MASK_FLD 0x707f
48 #define INSN_MATCH_FLQ 0x4007
49 #define INSN_MASK_FLQ 0x707f
50 #define INSN_MATCH_FSW 0x2027
51 #define INSN_MASK_FSW 0x707f
52 #define INSN_MATCH_FSD 0x3027
53 #define INSN_MASK_FSD 0x707f
54 #define INSN_MATCH_FSQ 0x4027
55 #define INSN_MASK_FSQ 0x707f
56
57 #define INSN_MATCH_C_LD 0x6000
58 #define INSN_MASK_C_LD 0xe003
59 #define INSN_MATCH_C_SD 0xe000
60 #define INSN_MASK_C_SD 0xe003
61 #define INSN_MATCH_C_LW 0x4000
62 #define INSN_MASK_C_LW 0xe003
63 #define INSN_MATCH_C_SW 0xc000
64 #define INSN_MASK_C_SW 0xe003
65 #define INSN_MATCH_C_LDSP 0x6002
66 #define INSN_MASK_C_LDSP 0xe003
67 #define INSN_MATCH_C_SDSP 0xe002
68 #define INSN_MASK_C_SDSP 0xe003
69 #define INSN_MATCH_C_LWSP 0x4002
70 #define INSN_MASK_C_LWSP 0xe003
71 #define INSN_MATCH_C_SWSP 0xc002
72 #define INSN_MASK_C_SWSP 0xe003
73
74 #define INSN_MATCH_C_FLD 0x2000
75 #define INSN_MASK_C_FLD 0xe003
76 #define INSN_MATCH_C_FLW 0x6000
77 #define INSN_MASK_C_FLW 0xe003
78 #define INSN_MATCH_C_FSD 0xa000
79 #define INSN_MASK_C_FSD 0xe003
80 #define INSN_MATCH_C_FSW 0xe000
81 #define INSN_MASK_C_FSW 0xe003
82 #define INSN_MATCH_C_FLDSP 0x2002
83 #define INSN_MASK_C_FLDSP 0xe003
84 #define INSN_MATCH_C_FSDSP 0xa002
85 #define INSN_MASK_C_FSDSP 0xe003
86 #define INSN_MATCH_C_FLWSP 0x6002
87 #define INSN_MASK_C_FLWSP 0xe003
88 #define INSN_MATCH_C_FSWSP 0xe002
89 #define INSN_MASK_C_FSWSP 0xe003
90
91 #define INSN_MATCH_C_LHU 0x8400
92 #define INSN_MASK_C_LHU 0xfc43
93 #define INSN_MATCH_C_LH 0x8440
94 #define INSN_MASK_C_LH 0xfc43
95 #define INSN_MATCH_C_SH 0x8c00
96 #define INSN_MASK_C_SH 0xfc43
97
98 #define INSN_LEN(insn) ((((insn) & 0x3) < 0x3) ? 2 : 4)
99
100 #if defined(CONFIG_64BIT)
101 #define LOG_REGBYTES 3
102 #define XLEN 64
103 #else
104 #define LOG_REGBYTES 2
105 #define XLEN 32
106 #endif
107 #define REGBYTES (1 << LOG_REGBYTES)
108 #define XLEN_MINUS_16 ((XLEN) - 16)
109
110 #define SH_RD 7
111 #define SH_RS1 15
112 #define SH_RS2 20
113 #define SH_RS2C 2
114
115 #define RV_X(x, s, n) (((x) >> (s)) & ((1 << (n)) - 1))
116 #define RVC_LW_IMM(x) ((RV_X(x, 6, 1) << 2) | \
117 (RV_X(x, 10, 3) << 3) | \
118 (RV_X(x, 5, 1) << 6))
119 #define RVC_LD_IMM(x) ((RV_X(x, 10, 3) << 3) | \
120 (RV_X(x, 5, 2) << 6))
121 #define RVC_LWSP_IMM(x) ((RV_X(x, 4, 3) << 2) | \
122 (RV_X(x, 12, 1) << 5) | \
123 (RV_X(x, 2, 2) << 6))
124 #define RVC_LDSP_IMM(x) ((RV_X(x, 5, 2) << 3) | \
125 (RV_X(x, 12, 1) << 5) | \
126 (RV_X(x, 2, 3) << 6))
127 #define RVC_SWSP_IMM(x) ((RV_X(x, 9, 4) << 2) | \
128 (RV_X(x, 7, 2) << 6))
129 #define RVC_SDSP_IMM(x) ((RV_X(x, 10, 3) << 3) | \
130 (RV_X(x, 7, 3) << 6))
131 #define RVC_RS1S(insn) (8 + RV_X(insn, SH_RD, 3))
132 #define RVC_RS2S(insn) (8 + RV_X(insn, SH_RS2C, 3))
133 #define RVC_RS2(insn) RV_X(insn, SH_RS2C, 5)
134
135 #define SHIFT_RIGHT(x, y) \
136 ((y) < 0 ? ((x) << -(y)) : ((x) >> (y)))
137
138 #define REG_MASK \
139 ((1 << (5 + LOG_REGBYTES)) - (1 << LOG_REGBYTES))
140
141 #define REG_OFFSET(insn, pos) \
142 (SHIFT_RIGHT((insn), (pos) - LOG_REGBYTES) & REG_MASK)
143
144 #define REG_PTR(insn, pos, regs) \
145 (ulong *)((ulong)(regs) + REG_OFFSET(insn, pos))
146
147 #define GET_RS1(insn, regs) (*REG_PTR(insn, SH_RS1, regs))
148 #define GET_RS2(insn, regs) (*REG_PTR(insn, SH_RS2, regs))
149 #define GET_RS1S(insn, regs) (*REG_PTR(RVC_RS1S(insn), 0, regs))
150 #define GET_RS2S(insn, regs) (*REG_PTR(RVC_RS2S(insn), 0, regs))
151 #define GET_RS2C(insn, regs) (*REG_PTR(insn, SH_RS2C, regs))
152 #define GET_SP(regs) (*REG_PTR(2, 0, regs))
153 #define SET_RD(insn, regs, val) (*REG_PTR(insn, SH_RD, regs) = (val))
154 #define IMM_I(insn) ((s32)(insn) >> 20)
155 #define IMM_S(insn) (((s32)(insn) >> 25 << 5) | \
156 (s32)(((insn) >> 7) & 0x1f))
157 #define MASK_FUNCT3 0x7000
158
159 #define GET_PRECISION(insn) (((insn) >> 25) & 3)
160 #define GET_RM(insn) (((insn) >> 12) & 7)
161 #define PRECISION_S 0
162 #define PRECISION_D 1
163
164 #ifdef CONFIG_FPU
165
166 #define FP_GET_RD(insn) (insn >> 7 & 0x1F)
167
168 extern void put_f32_reg(unsigned long fp_reg, unsigned long value);
169
set_f32_rd(unsigned long insn,struct pt_regs * regs,unsigned long val)170 static int set_f32_rd(unsigned long insn, struct pt_regs *regs,
171 unsigned long val)
172 {
173 unsigned long fp_reg = FP_GET_RD(insn);
174
175 put_f32_reg(fp_reg, val);
176 regs->status |= SR_FS_DIRTY;
177
178 return 0;
179 }
180
181 extern void put_f64_reg(unsigned long fp_reg, unsigned long value);
182
set_f64_rd(unsigned long insn,struct pt_regs * regs,u64 val)183 static int set_f64_rd(unsigned long insn, struct pt_regs *regs, u64 val)
184 {
185 unsigned long fp_reg = FP_GET_RD(insn);
186 unsigned long value;
187
188 #if __riscv_xlen == 32
189 value = (unsigned long) &val;
190 #else
191 value = val;
192 #endif
193 put_f64_reg(fp_reg, value);
194 regs->status |= SR_FS_DIRTY;
195
196 return 0;
197 }
198
199 #if __riscv_xlen == 32
200 extern void get_f64_reg(unsigned long fp_reg, u64 *value);
201
get_f64_rs(unsigned long insn,u8 fp_reg_offset,struct pt_regs * regs)202 static u64 get_f64_rs(unsigned long insn, u8 fp_reg_offset,
203 struct pt_regs *regs)
204 {
205 unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F;
206 u64 val;
207
208 get_f64_reg(fp_reg, &val);
209 regs->status |= SR_FS_DIRTY;
210
211 return val;
212 }
213 #else
214
215 extern unsigned long get_f64_reg(unsigned long fp_reg);
216
get_f64_rs(unsigned long insn,u8 fp_reg_offset,struct pt_regs * regs)217 static unsigned long get_f64_rs(unsigned long insn, u8 fp_reg_offset,
218 struct pt_regs *regs)
219 {
220 unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F;
221 unsigned long val;
222
223 val = get_f64_reg(fp_reg);
224 regs->status |= SR_FS_DIRTY;
225
226 return val;
227 }
228
229 #endif
230
231 extern unsigned long get_f32_reg(unsigned long fp_reg);
232
get_f32_rs(unsigned long insn,u8 fp_reg_offset,struct pt_regs * regs)233 static unsigned long get_f32_rs(unsigned long insn, u8 fp_reg_offset,
234 struct pt_regs *regs)
235 {
236 unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F;
237 unsigned long val;
238
239 val = get_f32_reg(fp_reg);
240 regs->status |= SR_FS_DIRTY;
241
242 return val;
243 }
244
245 #else /* CONFIG_FPU */
set_f32_rd(unsigned long insn,struct pt_regs * regs,unsigned long val)246 static void set_f32_rd(unsigned long insn, struct pt_regs *regs,
247 unsigned long val) {}
248
set_f64_rd(unsigned long insn,struct pt_regs * regs,u64 val)249 static void set_f64_rd(unsigned long insn, struct pt_regs *regs, u64 val) {}
250
get_f64_rs(unsigned long insn,u8 fp_reg_offset,struct pt_regs * regs)251 static unsigned long get_f64_rs(unsigned long insn, u8 fp_reg_offset,
252 struct pt_regs *regs)
253 {
254 return 0;
255 }
256
get_f32_rs(unsigned long insn,u8 fp_reg_offset,struct pt_regs * regs)257 static unsigned long get_f32_rs(unsigned long insn, u8 fp_reg_offset,
258 struct pt_regs *regs)
259 {
260 return 0;
261 }
262
263 #endif
264
265 #define GET_F64_RS2(insn, regs) (get_f64_rs(insn, 20, regs))
266 #define GET_F64_RS2C(insn, regs) (get_f64_rs(insn, 2, regs))
267 #define GET_F64_RS2S(insn, regs) (get_f64_rs(RVC_RS2S(insn), 0, regs))
268
269 #define GET_F32_RS2(insn, regs) (get_f32_rs(insn, 20, regs))
270 #define GET_F32_RS2C(insn, regs) (get_f32_rs(insn, 2, regs))
271 #define GET_F32_RS2S(insn, regs) (get_f32_rs(RVC_RS2S(insn), 0, regs))
272
273 #define __read_insn(regs, insn, insn_addr, type) \
274 ({ \
275 int __ret; \
276 \
277 if (user_mode(regs)) { \
278 __ret = get_user(insn, (type __user *) insn_addr); \
279 } else { \
280 insn = *(type *)insn_addr; \
281 __ret = 0; \
282 } \
283 \
284 __ret; \
285 })
286
get_insn(struct pt_regs * regs,ulong epc,ulong * r_insn)287 static inline int get_insn(struct pt_regs *regs, ulong epc, ulong *r_insn)
288 {
289 ulong insn = 0;
290
291 if (epc & 0x2) {
292 ulong tmp = 0;
293
294 if (__read_insn(regs, insn, epc, u16))
295 return -EFAULT;
296 /* __get_user() uses regular "lw" which sign extend the loaded
297 * value make sure to clear higher order bits in case we "or" it
298 * below with the upper 16 bits half.
299 */
300 insn &= GENMASK(15, 0);
301 if ((insn & __INSN_LENGTH_MASK) != __INSN_LENGTH_32) {
302 *r_insn = insn;
303 return 0;
304 }
305 epc += sizeof(u16);
306 if (__read_insn(regs, tmp, epc, u16))
307 return -EFAULT;
308 *r_insn = (tmp << 16) | insn;
309
310 return 0;
311 } else {
312 if (__read_insn(regs, insn, epc, u32))
313 return -EFAULT;
314 if ((insn & __INSN_LENGTH_MASK) == __INSN_LENGTH_32) {
315 *r_insn = insn;
316 return 0;
317 }
318 insn &= GENMASK(15, 0);
319 *r_insn = insn;
320
321 return 0;
322 }
323 }
324
325 union reg_data {
326 u8 data_bytes[8];
327 ulong data_ulong;
328 u64 data_u64;
329 };
330
331 /* sysctl hooks */
332 int unaligned_enabled __read_mostly = 1; /* Enabled by default */
333
334 #ifdef CONFIG_RISCV_VECTOR_MISALIGNED
handle_vector_misaligned_load(struct pt_regs * regs)335 static int handle_vector_misaligned_load(struct pt_regs *regs)
336 {
337 unsigned long epc = regs->epc;
338 unsigned long insn;
339
340 if (get_insn(regs, epc, &insn))
341 return -1;
342
343 /* Only return 0 when in check_vector_unaligned_access_emulated */
344 if (*this_cpu_ptr(&vector_misaligned_access) == RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) {
345 *this_cpu_ptr(&vector_misaligned_access) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED;
346 regs->epc = epc + INSN_LEN(insn);
347 return 0;
348 }
349
350 /* If vector instruction we don't emulate it yet */
351 regs->epc = epc;
352 return -1;
353 }
354 #else
handle_vector_misaligned_load(struct pt_regs * regs)355 static int handle_vector_misaligned_load(struct pt_regs *regs)
356 {
357 return -1;
358 }
359 #endif
360
handle_scalar_misaligned_load(struct pt_regs * regs)361 static int handle_scalar_misaligned_load(struct pt_regs *regs)
362 {
363 union reg_data val;
364 unsigned long epc = regs->epc;
365 unsigned long insn;
366 unsigned long addr = regs->badaddr;
367 int fp = 0, shift = 0, len = 0;
368
369 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr);
370
371 #ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS
372 *this_cpu_ptr(&misaligned_access_speed) = RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED;
373 #endif
374
375 if (!unaligned_enabled)
376 return -1;
377
378 if (user_mode(regs) && (current->thread.align_ctl & PR_UNALIGN_SIGBUS))
379 return -1;
380
381 if (get_insn(regs, epc, &insn))
382 return -1;
383
384 regs->epc = 0;
385
386 if ((insn & INSN_MASK_LW) == INSN_MATCH_LW) {
387 len = 4;
388 shift = 8 * (sizeof(unsigned long) - len);
389 #if defined(CONFIG_64BIT)
390 } else if ((insn & INSN_MASK_LD) == INSN_MATCH_LD) {
391 len = 8;
392 shift = 8 * (sizeof(unsigned long) - len);
393 } else if ((insn & INSN_MASK_LWU) == INSN_MATCH_LWU) {
394 len = 4;
395 #endif
396 } else if ((insn & INSN_MASK_FLD) == INSN_MATCH_FLD) {
397 fp = 1;
398 len = 8;
399 } else if ((insn & INSN_MASK_FLW) == INSN_MATCH_FLW) {
400 fp = 1;
401 len = 4;
402 } else if ((insn & INSN_MASK_LH) == INSN_MATCH_LH) {
403 len = 2;
404 shift = 8 * (sizeof(unsigned long) - len);
405 } else if ((insn & INSN_MASK_LHU) == INSN_MATCH_LHU) {
406 len = 2;
407 #if defined(CONFIG_64BIT)
408 } else if ((insn & INSN_MASK_C_LD) == INSN_MATCH_C_LD) {
409 len = 8;
410 shift = 8 * (sizeof(unsigned long) - len);
411 insn = RVC_RS2S(insn) << SH_RD;
412 } else if ((insn & INSN_MASK_C_LDSP) == INSN_MATCH_C_LDSP &&
413 ((insn >> SH_RD) & 0x1f)) {
414 len = 8;
415 shift = 8 * (sizeof(unsigned long) - len);
416 #endif
417 } else if ((insn & INSN_MASK_C_LW) == INSN_MATCH_C_LW) {
418 len = 4;
419 shift = 8 * (sizeof(unsigned long) - len);
420 insn = RVC_RS2S(insn) << SH_RD;
421 } else if ((insn & INSN_MASK_C_LWSP) == INSN_MATCH_C_LWSP &&
422 ((insn >> SH_RD) & 0x1f)) {
423 len = 4;
424 shift = 8 * (sizeof(unsigned long) - len);
425 } else if ((insn & INSN_MASK_C_FLD) == INSN_MATCH_C_FLD) {
426 fp = 1;
427 len = 8;
428 insn = RVC_RS2S(insn) << SH_RD;
429 } else if ((insn & INSN_MASK_C_FLDSP) == INSN_MATCH_C_FLDSP) {
430 fp = 1;
431 len = 8;
432 #if defined(CONFIG_32BIT)
433 } else if ((insn & INSN_MASK_C_FLW) == INSN_MATCH_C_FLW) {
434 fp = 1;
435 len = 4;
436 insn = RVC_RS2S(insn) << SH_RD;
437 } else if ((insn & INSN_MASK_C_FLWSP) == INSN_MATCH_C_FLWSP) {
438 fp = 1;
439 len = 4;
440 #endif
441 } else if ((insn & INSN_MASK_C_LHU) == INSN_MATCH_C_LHU) {
442 len = 2;
443 insn = RVC_RS2S(insn) << SH_RD;
444 } else if ((insn & INSN_MASK_C_LH) == INSN_MATCH_C_LH) {
445 len = 2;
446 shift = 8 * (sizeof(ulong) - len);
447 insn = RVC_RS2S(insn) << SH_RD;
448 } else {
449 regs->epc = epc;
450 return -1;
451 }
452
453 if (!IS_ENABLED(CONFIG_FPU) && fp)
454 return -EOPNOTSUPP;
455
456 val.data_u64 = 0;
457 if (user_mode(regs)) {
458 if (copy_from_user(&val, (u8 __user *)addr, len))
459 return -1;
460 } else {
461 memcpy(&val, (u8 *)addr, len);
462 }
463
464 if (!fp)
465 SET_RD(insn, regs, val.data_ulong << shift >> shift);
466 else if (len == 8)
467 set_f64_rd(insn, regs, val.data_u64);
468 else
469 set_f32_rd(insn, regs, val.data_ulong);
470
471 regs->epc = epc + INSN_LEN(insn);
472
473 return 0;
474 }
475
handle_scalar_misaligned_store(struct pt_regs * regs)476 static int handle_scalar_misaligned_store(struct pt_regs *regs)
477 {
478 union reg_data val;
479 unsigned long epc = regs->epc;
480 unsigned long insn;
481 unsigned long addr = regs->badaddr;
482 int len = 0, fp = 0;
483
484 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr);
485
486 if (!unaligned_enabled)
487 return -1;
488
489 if (user_mode(regs) && (current->thread.align_ctl & PR_UNALIGN_SIGBUS))
490 return -1;
491
492 if (get_insn(regs, epc, &insn))
493 return -1;
494
495 regs->epc = 0;
496
497 val.data_ulong = GET_RS2(insn, regs);
498
499 if ((insn & INSN_MASK_SW) == INSN_MATCH_SW) {
500 len = 4;
501 #if defined(CONFIG_64BIT)
502 } else if ((insn & INSN_MASK_SD) == INSN_MATCH_SD) {
503 len = 8;
504 #endif
505 } else if ((insn & INSN_MASK_FSD) == INSN_MATCH_FSD) {
506 fp = 1;
507 len = 8;
508 val.data_u64 = GET_F64_RS2(insn, regs);
509 } else if ((insn & INSN_MASK_FSW) == INSN_MATCH_FSW) {
510 fp = 1;
511 len = 4;
512 val.data_ulong = GET_F32_RS2(insn, regs);
513 } else if ((insn & INSN_MASK_SH) == INSN_MATCH_SH) {
514 len = 2;
515 #if defined(CONFIG_64BIT)
516 } else if ((insn & INSN_MASK_C_SD) == INSN_MATCH_C_SD) {
517 len = 8;
518 val.data_ulong = GET_RS2S(insn, regs);
519 } else if ((insn & INSN_MASK_C_SDSP) == INSN_MATCH_C_SDSP) {
520 len = 8;
521 val.data_ulong = GET_RS2C(insn, regs);
522 #endif
523 } else if ((insn & INSN_MASK_C_SW) == INSN_MATCH_C_SW) {
524 len = 4;
525 val.data_ulong = GET_RS2S(insn, regs);
526 } else if ((insn & INSN_MASK_C_SWSP) == INSN_MATCH_C_SWSP) {
527 len = 4;
528 val.data_ulong = GET_RS2C(insn, regs);
529 } else if ((insn & INSN_MASK_C_FSD) == INSN_MATCH_C_FSD) {
530 fp = 1;
531 len = 8;
532 val.data_u64 = GET_F64_RS2S(insn, regs);
533 } else if ((insn & INSN_MASK_C_FSDSP) == INSN_MATCH_C_FSDSP) {
534 fp = 1;
535 len = 8;
536 val.data_u64 = GET_F64_RS2C(insn, regs);
537 #if !defined(CONFIG_64BIT)
538 } else if ((insn & INSN_MASK_C_FSW) == INSN_MATCH_C_FSW) {
539 fp = 1;
540 len = 4;
541 val.data_ulong = GET_F32_RS2S(insn, regs);
542 } else if ((insn & INSN_MASK_C_FSWSP) == INSN_MATCH_C_FSWSP) {
543 fp = 1;
544 len = 4;
545 val.data_ulong = GET_F32_RS2C(insn, regs);
546 #endif
547 } else if ((insn & INSN_MASK_C_SH) == INSN_MATCH_C_SH) {
548 len = 2;
549 val.data_ulong = GET_RS2S(insn, regs);
550 } else {
551 regs->epc = epc;
552 return -1;
553 }
554
555 if (!IS_ENABLED(CONFIG_FPU) && fp)
556 return -EOPNOTSUPP;
557
558 if (user_mode(regs)) {
559 if (copy_to_user((u8 __user *)addr, &val, len))
560 return -1;
561 } else {
562 memcpy((u8 *)addr, &val, len);
563 }
564
565 regs->epc = epc + INSN_LEN(insn);
566
567 return 0;
568 }
569
handle_misaligned_load(struct pt_regs * regs)570 int handle_misaligned_load(struct pt_regs *regs)
571 {
572 unsigned long epc = regs->epc;
573 unsigned long insn;
574
575 if (IS_ENABLED(CONFIG_RISCV_VECTOR_MISALIGNED)) {
576 if (get_insn(regs, epc, &insn))
577 return -1;
578
579 if (insn_is_vector(insn))
580 return handle_vector_misaligned_load(regs);
581 }
582
583 if (IS_ENABLED(CONFIG_RISCV_SCALAR_MISALIGNED))
584 return handle_scalar_misaligned_load(regs);
585
586 return -1;
587 }
588
handle_misaligned_store(struct pt_regs * regs)589 int handle_misaligned_store(struct pt_regs *regs)
590 {
591 if (IS_ENABLED(CONFIG_RISCV_SCALAR_MISALIGNED))
592 return handle_scalar_misaligned_store(regs);
593
594 return -1;
595 }
596
597 #ifdef CONFIG_RISCV_VECTOR_MISALIGNED
check_vector_unaligned_access_emulated(struct work_struct * work __always_unused)598 void check_vector_unaligned_access_emulated(struct work_struct *work __always_unused)
599 {
600 long *mas_ptr = this_cpu_ptr(&vector_misaligned_access);
601 unsigned long tmp_var;
602
603 *mas_ptr = RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN;
604
605 kernel_vector_begin();
606 /*
607 * In pre-13.0.0 versions of GCC, vector registers cannot appear in
608 * the clobber list. This inline asm clobbers v0, but since we do not
609 * currently build the kernel with V enabled, the v0 clobber arg is not
610 * needed (as the compiler will not emit vector code itself). If the kernel
611 * is changed to build with V enabled, the clobber arg will need to be
612 * added here.
613 */
614 __asm__ __volatile__ (
615 ".balign 4\n\t"
616 ".option push\n\t"
617 ".option arch, +zve32x\n\t"
618 " vsetivli zero, 1, e16, m1, ta, ma\n\t" // Vectors of 16b
619 " vle16.v v0, (%[ptr])\n\t" // Load bytes
620 ".option pop\n\t"
621 : : [ptr] "r" ((u8 *)&tmp_var + 1));
622 kernel_vector_end();
623 }
624
check_vector_unaligned_access_emulated_all_cpus(void)625 bool __init check_vector_unaligned_access_emulated_all_cpus(void)
626 {
627 int cpu;
628
629 schedule_on_each_cpu(check_vector_unaligned_access_emulated);
630
631 for_each_online_cpu(cpu)
632 if (per_cpu(vector_misaligned_access, cpu)
633 == RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN)
634 return false;
635
636 return true;
637 }
638 #else
check_vector_unaligned_access_emulated_all_cpus(void)639 bool __init check_vector_unaligned_access_emulated_all_cpus(void)
640 {
641 return false;
642 }
643 #endif
644
645 #ifdef CONFIG_RISCV_SCALAR_MISALIGNED
646
647 static bool unaligned_ctl __read_mostly;
648
check_unaligned_access_emulated(struct work_struct * work __always_unused)649 void check_unaligned_access_emulated(struct work_struct *work __always_unused)
650 {
651 int cpu = smp_processor_id();
652 long *mas_ptr = per_cpu_ptr(&misaligned_access_speed, cpu);
653 unsigned long tmp_var, tmp_val;
654
655 *mas_ptr = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN;
656
657 __asm__ __volatile__ (
658 " "REG_L" %[tmp], 1(%[ptr])\n"
659 : [tmp] "=r" (tmp_val) : [ptr] "r" (&tmp_var) : "memory");
660
661 /*
662 * If unaligned_ctl is already set, this means that we detected that all
663 * CPUS uses emulated misaligned access at boot time. If that changed
664 * when hotplugging the new cpu, this is something we don't handle.
665 */
666 if (unlikely(unaligned_ctl && (*mas_ptr != RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED))) {
667 pr_crit("CPU misaligned accesses non homogeneous (expected all emulated)\n");
668 while (true)
669 cpu_relax();
670 }
671 }
672
check_unaligned_access_emulated_all_cpus(void)673 bool __init check_unaligned_access_emulated_all_cpus(void)
674 {
675 int cpu;
676
677 /*
678 * We can only support PR_UNALIGN controls if all CPUs have misaligned
679 * accesses emulated since tasks requesting such control can run on any
680 * CPU.
681 */
682 schedule_on_each_cpu(check_unaligned_access_emulated);
683
684 for_each_online_cpu(cpu)
685 if (per_cpu(misaligned_access_speed, cpu)
686 != RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED)
687 return false;
688
689 unaligned_ctl = true;
690 return true;
691 }
692
unaligned_ctl_available(void)693 bool unaligned_ctl_available(void)
694 {
695 return unaligned_ctl;
696 }
697 #else
check_unaligned_access_emulated_all_cpus(void)698 bool __init check_unaligned_access_emulated_all_cpus(void)
699 {
700 return false;
701 }
702 #endif
703