1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (C) 2017 Arm Ltd.
3 #define pr_fmt(fmt) "sdei: " fmt
4
5 #include <linux/arm-smccc.h>
6 #include <linux/arm_sdei.h>
7 #include <linux/hardirq.h>
8 #include <linux/irqflags.h>
9 #include <linux/sched/task_stack.h>
10 #include <linux/scs.h>
11 #include <linux/uaccess.h>
12
13 #include <asm/alternative.h>
14 #include <asm/exception.h>
15 #include <asm/kprobes.h>
16 #include <asm/mmu.h>
17 #include <asm/ptrace.h>
18 #include <asm/sections.h>
19 #include <asm/stacktrace.h>
20 #include <asm/sysreg.h>
21 #include <asm/vmap_stack.h>
22
23 unsigned long sdei_exit_mode;
24
25 /*
26 * VMAP'd stacks checking for stack overflow on exception using sp as a scratch
27 * register, meaning SDEI has to switch to its own stack. We need two stacks as
28 * a critical event may interrupt a normal event that has just taken a
29 * synchronous exception, and is using sp as scratch register. For a critical
30 * event interrupting a normal event, we can't reliably tell if we were on the
31 * sdei stack.
32 * For now, we allocate stacks when the driver is probed.
33 */
34 DECLARE_PER_CPU(unsigned long *, sdei_stack_normal_ptr);
35 DECLARE_PER_CPU(unsigned long *, sdei_stack_critical_ptr);
36
37 DEFINE_PER_CPU(unsigned long *, sdei_stack_normal_ptr);
38 DEFINE_PER_CPU(unsigned long *, sdei_stack_critical_ptr);
39
40 DECLARE_PER_CPU(unsigned long *, sdei_shadow_call_stack_normal_ptr);
41 DECLARE_PER_CPU(unsigned long *, sdei_shadow_call_stack_critical_ptr);
42
43 #ifdef CONFIG_SHADOW_CALL_STACK
44 DEFINE_PER_CPU(unsigned long *, sdei_shadow_call_stack_normal_ptr);
45 DEFINE_PER_CPU(unsigned long *, sdei_shadow_call_stack_critical_ptr);
46 #endif
47
48 DEFINE_PER_CPU(struct sdei_registered_event *, sdei_active_normal_event);
49 DEFINE_PER_CPU(struct sdei_registered_event *, sdei_active_critical_event);
50
_free_sdei_stack(unsigned long * __percpu * ptr,int cpu)51 static void _free_sdei_stack(unsigned long * __percpu *ptr, int cpu)
52 {
53 unsigned long *p;
54
55 p = per_cpu(*ptr, cpu);
56 if (p) {
57 per_cpu(*ptr, cpu) = NULL;
58 vfree(p);
59 }
60 }
61
free_sdei_stacks(void)62 static void free_sdei_stacks(void)
63 {
64 int cpu;
65
66 BUILD_BUG_ON(!IS_ENABLED(CONFIG_VMAP_STACK));
67
68 for_each_possible_cpu(cpu) {
69 _free_sdei_stack(&sdei_stack_normal_ptr, cpu);
70 _free_sdei_stack(&sdei_stack_critical_ptr, cpu);
71 }
72 }
73
_init_sdei_stack(unsigned long * __percpu * ptr,int cpu)74 static int _init_sdei_stack(unsigned long * __percpu *ptr, int cpu)
75 {
76 unsigned long *p;
77
78 p = arch_alloc_vmap_stack(SDEI_STACK_SIZE, cpu_to_node(cpu));
79 if (!p)
80 return -ENOMEM;
81 per_cpu(*ptr, cpu) = p;
82
83 return 0;
84 }
85
init_sdei_stacks(void)86 static int init_sdei_stacks(void)
87 {
88 int cpu;
89 int err = 0;
90
91 BUILD_BUG_ON(!IS_ENABLED(CONFIG_VMAP_STACK));
92
93 for_each_possible_cpu(cpu) {
94 err = _init_sdei_stack(&sdei_stack_normal_ptr, cpu);
95 if (err)
96 break;
97 err = _init_sdei_stack(&sdei_stack_critical_ptr, cpu);
98 if (err)
99 break;
100 }
101
102 if (err)
103 free_sdei_stacks();
104
105 return err;
106 }
107
_free_sdei_scs(unsigned long * __percpu * ptr,int cpu)108 static void _free_sdei_scs(unsigned long * __percpu *ptr, int cpu)
109 {
110 void *s;
111
112 s = per_cpu(*ptr, cpu);
113 if (s) {
114 per_cpu(*ptr, cpu) = NULL;
115 scs_free(s);
116 }
117 }
118
free_sdei_scs(void)119 static void free_sdei_scs(void)
120 {
121 int cpu;
122
123 for_each_possible_cpu(cpu) {
124 _free_sdei_scs(&sdei_shadow_call_stack_normal_ptr, cpu);
125 _free_sdei_scs(&sdei_shadow_call_stack_critical_ptr, cpu);
126 }
127 }
128
_init_sdei_scs(unsigned long * __percpu * ptr,int cpu)129 static int _init_sdei_scs(unsigned long * __percpu *ptr, int cpu)
130 {
131 void *s;
132
133 s = scs_alloc(cpu_to_node(cpu));
134 if (!s)
135 return -ENOMEM;
136 per_cpu(*ptr, cpu) = s;
137
138 return 0;
139 }
140
init_sdei_scs(void)141 static int init_sdei_scs(void)
142 {
143 int cpu;
144 int err = 0;
145
146 if (!scs_is_enabled())
147 return 0;
148
149 for_each_possible_cpu(cpu) {
150 err = _init_sdei_scs(&sdei_shadow_call_stack_normal_ptr, cpu);
151 if (err)
152 break;
153 err = _init_sdei_scs(&sdei_shadow_call_stack_critical_ptr, cpu);
154 if (err)
155 break;
156 }
157
158 if (err)
159 free_sdei_scs();
160
161 return err;
162 }
163
sdei_arch_get_entry_point(int conduit)164 unsigned long sdei_arch_get_entry_point(int conduit)
165 {
166 /*
167 * SDEI works between adjacent exception levels. If we booted at EL1 we
168 * assume a hypervisor is marshalling events. If we booted at EL2 and
169 * dropped to EL1 because we don't support VHE, then we can't support
170 * SDEI.
171 */
172 if (is_hyp_nvhe()) {
173 pr_err("Not supported on this hardware/boot configuration\n");
174 goto out_err;
175 }
176
177 if (init_sdei_stacks())
178 goto out_err;
179
180 if (init_sdei_scs())
181 goto out_err_free_stacks;
182
183 sdei_exit_mode = (conduit == SMCCC_CONDUIT_HVC) ? SDEI_EXIT_HVC : SDEI_EXIT_SMC;
184
185 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
186 if (arm64_kernel_unmapped_at_el0()) {
187 unsigned long offset;
188
189 offset = (unsigned long)__sdei_asm_entry_trampoline -
190 (unsigned long)__entry_tramp_text_start;
191 return TRAMP_VALIAS + offset;
192 } else
193 #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
194 return (unsigned long)__sdei_asm_handler;
195
196 out_err_free_stacks:
197 free_sdei_stacks();
198 out_err:
199 return 0;
200 }
201
202 /*
203 * do_sdei_event() returns one of:
204 * SDEI_EV_HANDLED - success, return to the interrupted context.
205 * SDEI_EV_FAILED - failure, return this error code to firmare.
206 * virtual-address - success, return to this address.
207 */
do_sdei_event(struct pt_regs * regs,struct sdei_registered_event * arg)208 unsigned long __kprobes do_sdei_event(struct pt_regs *regs,
209 struct sdei_registered_event *arg)
210 {
211 u32 mode;
212 int i, err = 0;
213 int clobbered_registers = 4;
214 u64 elr = read_sysreg(elr_el1);
215 u32 kernel_mode = read_sysreg(CurrentEL) | 1; /* +SPSel */
216 unsigned long vbar = read_sysreg(vbar_el1);
217
218 if (arm64_kernel_unmapped_at_el0())
219 clobbered_registers++;
220
221 /* Retrieve the missing registers values */
222 for (i = 0; i < clobbered_registers; i++) {
223 /* from within the handler, this call always succeeds */
224 sdei_api_event_context(i, ®s->regs[i]);
225 }
226
227 err = sdei_event_handler(regs, arg);
228 if (err)
229 return SDEI_EV_FAILED;
230
231 if (elr != read_sysreg(elr_el1)) {
232 /*
233 * We took a synchronous exception from the SDEI handler.
234 * This could deadlock, and if you interrupt KVM it will
235 * hyp-panic instead.
236 */
237 pr_warn("unsafe: exception during handler\n");
238 }
239
240 mode = regs->pstate & (PSR_MODE32_BIT | PSR_MODE_MASK);
241
242 /*
243 * If we interrupted the kernel with interrupts masked, we always go
244 * back to wherever we came from.
245 */
246 if (mode == kernel_mode && !interrupts_enabled(regs))
247 return SDEI_EV_HANDLED;
248
249 /*
250 * Otherwise, we pretend this was an IRQ. This lets user space tasks
251 * receive signals before we return to them, and KVM to invoke it's
252 * world switch to do the same.
253 *
254 * See DDI0487B.a Table D1-7 'Vector offsets from vector table base
255 * address'.
256 */
257 if (mode == kernel_mode)
258 return vbar + 0x280;
259 else if (mode & PSR_MODE32_BIT)
260 return vbar + 0x680;
261
262 return vbar + 0x480;
263 }
264