1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * s390x interrupt handling
4 *
5 * Copyright (c) 2017 Red Hat Inc
6 *
7 * Authors:
8 * David Hildenbrand <david@redhat.com>
9 */
10 #include <libcflat.h>
11 #include <asm/barrier.h>
12 #include <asm/mem.h>
13 #include <asm/asm-offsets.h>
14 #include <sclp.h>
15 #include <interrupt.h>
16 #include <sie.h>
17 #include <fault.h>
18 #include <asm/page.h>
19 #include "smp.h"
20
21 /**
22 * expect_pgm_int - Expect a program interrupt on the current CPU.
23 */
expect_pgm_int(void)24 void expect_pgm_int(void)
25 {
26 THIS_CPU->pgm_int_expected = true;
27 lowcore.pgm_int_code = 0;
28 lowcore.trans_exc_id = 0;
29 mb();
30 }
31
32 /**
33 * expect_ext_int - Expect an external interrupt on the current CPU.
34 */
expect_ext_int(void)35 void expect_ext_int(void)
36 {
37 THIS_CPU->ext_int_expected = true;
38 lowcore.ext_int_code = 0;
39 mb();
40 }
41
42 /**
43 * clear_pgm_int - Clear program interrupt information
44 *
45 * Clear program interrupt information, including the expected program
46 * interrupt flag.
47 * No program interrupts are expected after calling this function.
48 *
49 * Return: the program interrupt code before clearing
50 */
clear_pgm_int(void)51 uint16_t clear_pgm_int(void)
52 {
53 uint16_t code;
54
55 mb();
56 code = lowcore.pgm_int_code;
57 lowcore.pgm_int_code = 0;
58 lowcore.trans_exc_id = 0;
59 THIS_CPU->pgm_int_expected = false;
60 return code;
61 }
62
63 /**
64 * check_pgm_int_code - Check the program interrupt code on the current CPU.
65 * @code: the expected program interrupt code on the current CPU
66 *
67 * Check and report if the program interrupt on the current CPU matches the
68 * expected one.
69 */
check_pgm_int_code(uint16_t code)70 void check_pgm_int_code(uint16_t code)
71 {
72 mb();
73 report(code == lowcore.pgm_int_code,
74 "Program interrupt: expected(%d) == received(%d)", code,
75 lowcore.pgm_int_code);
76 }
77
78 /**
79 * register_pgm_cleanup_func - Register a cleanup function for progam
80 * interrupts for the current CPU.
81 * @f: the cleanup function to be registered on the current CPU
82 *
83 * Register a cleanup function to be called at the end of the normal
84 * interrupt handling for program interrupts for this CPU.
85 *
86 * Pass NULL to unregister a previously registered cleanup function.
87 */
register_pgm_cleanup_func(void (* f)(struct stack_frame_int *))88 void register_pgm_cleanup_func(void (*f)(struct stack_frame_int *))
89 {
90 THIS_CPU->pgm_cleanup_func = f;
91 }
92
93 /**
94 * register_ext_cleanup_func - Register a cleanup function for external
95 * interrupts for the current CPU.
96 * @f: the cleanup function to be registered on the current CPU
97 *
98 * Register a cleanup function to be called at the end of the normal
99 * interrupt handling for external interrupts for this CPU.
100 *
101 * Pass NULL to unregister a previously registered cleanup function.
102 */
register_ext_cleanup_func(void (* f)(struct stack_frame_int *))103 void register_ext_cleanup_func(void (*f)(struct stack_frame_int *))
104 {
105 THIS_CPU->ext_cleanup_func = f;
106 }
107
108 /**
109 * irq_set_dat_mode - Set the DAT mode of all interrupt handlers, except for
110 * restart.
111 * @use_dat: specifies whether to use DAT or not
112 * @as: specifies the address space mode to use. Not set if use_dat is false.
113 *
114 * This will update the DAT mode and address space mode of all interrupt new
115 * PSWs.
116 *
117 * Since enabling DAT needs initialized CRs and the restart new PSW is often used
118 * to initialize CRs, the restart new PSW is never touched to avoid the chicken
119 * and egg situation.
120 */
irq_set_dat_mode(bool use_dat,enum address_space as)121 void irq_set_dat_mode(bool use_dat, enum address_space as)
122 {
123 struct psw* irq_psws[] = {
124 OPAQUE_PTR(GEN_LC_EXT_NEW_PSW),
125 OPAQUE_PTR(GEN_LC_SVC_NEW_PSW),
126 OPAQUE_PTR(GEN_LC_PGM_NEW_PSW),
127 OPAQUE_PTR(GEN_LC_MCCK_NEW_PSW),
128 OPAQUE_PTR(GEN_LC_IO_NEW_PSW),
129 };
130 struct psw *psw;
131
132 assert(as == AS_PRIM || as == AS_ACCR || as == AS_SECN || as == AS_HOME);
133
134 for (size_t i = 0; i < ARRAY_SIZE(irq_psws); i++) {
135 psw = irq_psws[i];
136 psw->dat = use_dat;
137 if (use_dat)
138 psw->as = as;
139 }
140 }
141
fixup_pgm_int(struct stack_frame_int * stack)142 static void fixup_pgm_int(struct stack_frame_int *stack)
143 {
144 /* If we have an error on SIE we directly move to sie_exit */
145 if (lowcore.pgm_old_psw.addr >= (uint64_t)&sie_entry &&
146 lowcore.pgm_old_psw.addr <= (uint64_t)&sie_exit) {
147 lowcore.pgm_old_psw.addr = (uint64_t)&sie_exit;
148 return;
149 }
150
151 switch (lowcore.pgm_int_code) {
152 case PGM_INT_CODE_PRIVILEGED_OPERATION:
153 /* Normal operation is in supervisor state, so this exception
154 * was produced intentionally and we should return to the
155 * supervisor state.
156 */
157 lowcore.pgm_old_psw.mask &= ~PSW_MASK_PSTATE;
158 break;
159 case PGM_INT_CODE_PROTECTION:
160 /* Handling for iep.c test case. */
161 if (prot_is_iep((union teid) { .val = lowcore.trans_exc_id }))
162 /*
163 * We branched to the instruction that caused
164 * the exception so we can use the return
165 * address in GR14 to jump back and continue
166 * executing test code.
167 */
168 lowcore.pgm_old_psw.addr = stack->grs0[12];
169 break;
170 case PGM_INT_CODE_SEGMENT_TRANSLATION:
171 case PGM_INT_CODE_PAGE_TRANSLATION:
172 case PGM_INT_CODE_TRACE_TABLE:
173 case PGM_INT_CODE_AFX_TRANSLATION:
174 case PGM_INT_CODE_ASX_TRANSLATION:
175 case PGM_INT_CODE_LX_TRANSLATION:
176 case PGM_INT_CODE_EX_TRANSLATION:
177 case PGM_INT_CODE_PRIMARY_AUTHORITY:
178 case PGM_INT_CODE_SECONDARY_AUTHORITY:
179 case PGM_INT_CODE_LFX_TRANSLATION:
180 case PGM_INT_CODE_LSX_TRANSLATION:
181 case PGM_INT_CODE_ALEN_TRANSLATION:
182 case PGM_INT_CODE_ALE_SEQUENCE:
183 case PGM_INT_CODE_ASTE_VALIDITY:
184 case PGM_INT_CODE_ASTE_SEQUENCE:
185 case PGM_INT_CODE_EXTENDED_AUTHORITY:
186 case PGM_INT_CODE_LSTE_SEQUENCE:
187 case PGM_INT_CODE_ASTE_INSTANCE:
188 case PGM_INT_CODE_STACK_FULL:
189 case PGM_INT_CODE_STACK_EMPTY:
190 case PGM_INT_CODE_STACK_SPECIFICATION:
191 case PGM_INT_CODE_STACK_TYPE:
192 case PGM_INT_CODE_STACK_OPERATION:
193 case PGM_INT_CODE_ASCE_TYPE:
194 case PGM_INT_CODE_REGION_FIRST_TRANS:
195 case PGM_INT_CODE_REGION_SECOND_TRANS:
196 case PGM_INT_CODE_REGION_THIRD_TRANS:
197 case PGM_INT_CODE_PER:
198 case PGM_INT_CODE_CRYPTO_OPERATION:
199 case PGM_INT_CODE_SECURE_STOR_ACCESS:
200 case PGM_INT_CODE_NON_SECURE_STOR_ACCESS:
201 case PGM_INT_CODE_SECURE_STOR_VIOLATION:
202 /* The interrupt was nullified, the old PSW points at the
203 * responsible instruction. Forward the PSW so we don't loop.
204 */
205 lowcore.pgm_old_psw.addr += lowcore.pgm_int_id;
206 }
207 /* suppressed/terminated/completed point already at the next address */
208 }
209
print_storage_exception_information(void)210 static void print_storage_exception_information(void)
211 {
212 switch (lowcore.pgm_int_code) {
213 case PGM_INT_CODE_PROTECTION:
214 case PGM_INT_CODE_PAGE_TRANSLATION:
215 case PGM_INT_CODE_SEGMENT_TRANSLATION:
216 case PGM_INT_CODE_ASCE_TYPE:
217 case PGM_INT_CODE_REGION_FIRST_TRANS:
218 case PGM_INT_CODE_REGION_SECOND_TRANS:
219 case PGM_INT_CODE_REGION_THIRD_TRANS:
220 case PGM_INT_CODE_SECURE_STOR_ACCESS:
221 case PGM_INT_CODE_NON_SECURE_STOR_ACCESS:
222 case PGM_INT_CODE_SECURE_STOR_VIOLATION:
223 print_decode_teid(lowcore.trans_exc_id);
224 break;
225 }
226 }
227
print_int_regs(struct stack_frame_int * stack,bool sie)228 static void print_int_regs(struct stack_frame_int *stack, bool sie)
229 {
230 struct kvm_s390_sie_block *sblk;
231
232 printf("\n");
233 printf("%s\n", sie ? "Guest registers:" : "Host registers:");
234 printf("GPRS:\n");
235 printf("%016lx %016lx %016lx %016lx\n",
236 stack->grs1[0], stack->grs1[1], stack->grs0[0], stack->grs0[1]);
237 printf("%016lx %016lx %016lx %016lx\n",
238 stack->grs0[2], stack->grs0[3], stack->grs0[4], stack->grs0[5]);
239 printf("%016lx %016lx %016lx %016lx\n",
240 stack->grs0[6], stack->grs0[7], stack->grs0[8], stack->grs0[9]);
241
242 if (sie) {
243 sblk = (struct kvm_s390_sie_block *)stack->grs0[12];
244 printf("%016lx %016lx %016lx %016lx\n",
245 stack->grs0[10], stack->grs0[11], sblk->gg14, sblk->gg15);
246 } else {
247 printf("%016lx %016lx %016lx %016lx\n",
248 stack->grs0[10], stack->grs0[11], stack->grs0[12], stack->grs0[13]);
249 }
250
251 printf("\n");
252 }
253
print_pgm_info(struct stack_frame_int * stack)254 static void print_pgm_info(struct stack_frame_int *stack)
255
256 {
257 bool in_sie, in_sie_gregs;
258 struct vm_save_area *vregs;
259
260 in_sie = (lowcore.pgm_old_psw.addr >= (uintptr_t)sie_entry &&
261 lowcore.pgm_old_psw.addr <= (uintptr_t)sie_exit);
262 in_sie_gregs = (lowcore.pgm_old_psw.addr >= (uintptr_t)sie_entry_gregs &&
263 lowcore.pgm_old_psw.addr < (uintptr_t)sie_exit_gregs);
264
265 printf("\n");
266 printf("Unexpected program interrupt %s: %#x on cpu %d at %#lx, ilen %d\n",
267 in_sie ? "in SIE" : "",
268 lowcore.pgm_int_code, stap(), lowcore.pgm_old_psw.addr, lowcore.pgm_int_id);
269
270 /*
271 * If we fall out of SIE before loading the host registers,
272 * then we need to do it here so we print the host registers
273 * and not the guest registers.
274 *
275 * Back tracing is actually not a problem since SIE restores gr15.
276 */
277 if (in_sie_gregs) {
278 print_int_regs(stack, true);
279 vregs = *((struct vm_save_area **)(stack->grs0[13] + __SF_SIE_SAVEAREA));
280
281 /*
282 * The grs are not linear on the interrupt stack frame.
283 * We copy 0 and 1 here and 2 - 15 with the memcopy below.
284 */
285 stack->grs1[0] = vregs->host.grs[0];
286 stack->grs1[1] = vregs->host.grs[1];
287 /* 2 - 15 */
288 memcpy(stack->grs0, &vregs->host.grs[2], sizeof(stack->grs0) - 8);
289 }
290 print_int_regs(stack, false);
291 dump_stack();
292
293 /* Dump stack doesn't end with a \n so we add it here instead */
294 printf("\n");
295 print_storage_exception_information();
296 report_summary();
297 abort();
298 }
299
handle_pgm_int(struct stack_frame_int * stack)300 void handle_pgm_int(struct stack_frame_int *stack)
301 {
302 if (THIS_CPU->in_interrupt_handler) {
303 /* Something went very wrong, stop everything now without printing anything */
304 smp_teardown();
305 disabled_wait(0xfa12edbad21);
306 }
307 if (!THIS_CPU->pgm_int_expected) {
308 /* Force sclp_busy to false, otherwise we will loop forever */
309 sclp_handle_ext();
310 print_pgm_info(stack);
311 }
312
313 THIS_CPU->pgm_int_expected = false;
314 THIS_CPU->in_interrupt_handler = true;
315
316 if (THIS_CPU->pgm_cleanup_func)
317 THIS_CPU->pgm_cleanup_func(stack);
318 else
319 fixup_pgm_int(stack);
320 THIS_CPU->in_interrupt_handler = false;
321 }
322
handle_ext_int(struct stack_frame_int * stack)323 void handle_ext_int(struct stack_frame_int *stack)
324 {
325 THIS_CPU->in_interrupt_handler = true;
326 if (!THIS_CPU->ext_int_expected && lowcore.ext_int_code != EXT_IRQ_SERVICE_SIG) {
327 report_abort("Unexpected external call interrupt (code %#x): on cpu %d at %#lx",
328 lowcore.ext_int_code, stap(), lowcore.ext_old_psw.addr);
329 return;
330 }
331
332 if (lowcore.ext_int_code == EXT_IRQ_SERVICE_SIG) {
333 stack->crs[0] &= ~(1UL << 9);
334 sclp_handle_ext();
335 } else {
336 THIS_CPU->ext_int_expected = false;
337 }
338
339 if (!(stack->crs[0] & CR0_EXTM_MASK))
340 lowcore.ext_old_psw.mask &= ~PSW_MASK_EXT;
341
342 if (THIS_CPU->ext_cleanup_func)
343 THIS_CPU->ext_cleanup_func(stack);
344 THIS_CPU->in_interrupt_handler = false;
345 }
346
handle_mcck_int(void)347 void handle_mcck_int(void)
348 {
349 report_abort("Unexpected machine check interrupt: on cpu %d at %#lx",
350 stap(), lowcore.mcck_old_psw.addr);
351 }
352
353 static void (*io_int_func)(void);
354
handle_io_int(void)355 void handle_io_int(void)
356 {
357 THIS_CPU->in_interrupt_handler = true;
358 if (io_int_func)
359 io_int_func();
360 else
361 report_abort("Unexpected io interrupt: on cpu %d at %#lx",
362 stap(), lowcore.io_old_psw.addr);
363 THIS_CPU->in_interrupt_handler = false;
364 }
365
register_io_int_func(void (* f)(void))366 int register_io_int_func(void (*f)(void))
367 {
368 if (io_int_func)
369 return -1;
370 io_int_func = f;
371 return 0;
372 }
373
unregister_io_int_func(void (* f)(void))374 int unregister_io_int_func(void (*f)(void))
375 {
376 if (io_int_func != f)
377 return -1;
378 io_int_func = NULL;
379 return 0;
380 }
381
handle_svc_int(void)382 void handle_svc_int(void)
383 {
384 uint16_t code = lowcore.svc_int_code;
385
386 switch (code) {
387 case SVC_LEAVE_PSTATE:
388 lowcore.svc_old_psw.mask &= ~PSW_MASK_PSTATE;
389 break;
390 default:
391 report_abort("Unexpected supervisor call interrupt: code %#x on cpu %d at %#lx",
392 code, stap(), lowcore.svc_old_psw.addr);
393 }
394 }
395