1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2022 - Google LLC
4  * Author: Ard Biesheuvel <ardb@google.com>
5  */
6 
7 #include <linux/errno.h>
8 #include <linux/init.h>
9 #include <linux/linkage.h>
10 #include <linux/types.h>
11 
12 #include <asm/scs.h>
13 
14 #include "pi.h"
15 
16 bool dynamic_scs_is_enabled;
17 
18 //
19 // This minimal DWARF CFI parser is partially based on the code in
20 // arch/arc/kernel/unwind.c, and on the document below:
21 // https://refspecs.linuxbase.org/LSB_4.0.0/LSB-Core-generic/LSB-Core-generic/ehframechpt.html
22 //
23 
24 #define DW_CFA_nop                          0x00
25 #define DW_CFA_set_loc                      0x01
26 #define DW_CFA_advance_loc1                 0x02
27 #define DW_CFA_advance_loc2                 0x03
28 #define DW_CFA_advance_loc4                 0x04
29 #define DW_CFA_offset_extended              0x05
30 #define DW_CFA_restore_extended             0x06
31 #define DW_CFA_undefined                    0x07
32 #define DW_CFA_same_value                   0x08
33 #define DW_CFA_register                     0x09
34 #define DW_CFA_remember_state               0x0a
35 #define DW_CFA_restore_state                0x0b
36 #define DW_CFA_def_cfa                      0x0c
37 #define DW_CFA_def_cfa_register             0x0d
38 #define DW_CFA_def_cfa_offset               0x0e
39 #define DW_CFA_def_cfa_expression           0x0f
40 #define DW_CFA_expression                   0x10
41 #define DW_CFA_offset_extended_sf           0x11
42 #define DW_CFA_def_cfa_sf                   0x12
43 #define DW_CFA_def_cfa_offset_sf            0x13
44 #define DW_CFA_val_offset                   0x14
45 #define DW_CFA_val_offset_sf                0x15
46 #define DW_CFA_val_expression               0x16
47 #define DW_CFA_lo_user                      0x1c
48 #define DW_CFA_negate_ra_state              0x2d
49 #define DW_CFA_GNU_args_size                0x2e
50 #define DW_CFA_GNU_negative_offset_extended 0x2f
51 #define DW_CFA_hi_user                      0x3f
52 
53 enum {
54 	PACIASP		= 0xd503233f,
55 	AUTIASP		= 0xd50323bf,
56 	SCS_PUSH	= 0xf800865e,
57 	SCS_POP		= 0xf85f8e5e,
58 };
59 
scs_patch_loc(u64 loc)60 static void __always_inline scs_patch_loc(u64 loc)
61 {
62 	u32 insn = le32_to_cpup((void *)loc);
63 
64 	switch (insn) {
65 	case PACIASP:
66 		*(u32 *)loc = cpu_to_le32(SCS_PUSH);
67 		break;
68 	case AUTIASP:
69 		*(u32 *)loc = cpu_to_le32(SCS_POP);
70 		break;
71 	default:
72 		/*
73 		 * While the DW_CFA_negate_ra_state directive is guaranteed to
74 		 * appear right after a PACIASP/AUTIASP instruction, it may
75 		 * also appear after a DW_CFA_restore_state directive that
76 		 * restores a state that is only partially accurate, and is
77 		 * followed by DW_CFA_negate_ra_state directive to toggle the
78 		 * PAC bit again. So we permit other instructions here, and ignore
79 		 * them.
80 		 */
81 		return;
82 	}
83 	if (IS_ENABLED(CONFIG_ARM64_WORKAROUND_CLEAN_CACHE))
84 		asm("dc civac, %0" :: "r"(loc));
85 	else
86 		asm(ALTERNATIVE("dc cvau, %0", "nop", ARM64_HAS_CACHE_IDC)
87 		    :: "r"(loc));
88 }
89 
90 /*
91  * Skip one uleb128/sleb128 encoded quantity from the opcode stream. All bytes
92  * except the last one have bit #7 set.
93  */
skip_xleb128(const u8 ** opcode,int size)94 static int __always_inline skip_xleb128(const u8 **opcode, int size)
95 {
96 	u8 c;
97 
98 	do {
99 		c = *(*opcode)++;
100 		size--;
101 	} while (c & BIT(7));
102 
103 	return size;
104 }
105 
106 struct eh_frame {
107 	/*
108 	 * The size of this frame if 0 < size < U32_MAX, 0 terminates the list.
109 	 */
110 	u32	size;
111 
112 	/*
113 	 * The first frame is a Common Information Entry (CIE) frame, followed
114 	 * by one or more Frame Description Entry (FDE) frames. In the former
115 	 * case, this field is 0, otherwise it is the negated offset relative
116 	 * to the associated CIE frame.
117 	 */
118 	u32	cie_id_or_pointer;
119 
120 	union {
121 		struct { // CIE
122 			u8	version;
123 			u8	augmentation_string[];
124 		};
125 
126 		struct { // FDE
127 			s32	initial_loc;
128 			s32	range;
129 			u8	opcodes[];
130 		};
131 	};
132 };
133 
scs_handle_fde_frame(const struct eh_frame * frame,bool fde_has_augmentation_data,int code_alignment_factor,bool dry_run)134 static int scs_handle_fde_frame(const struct eh_frame *frame,
135 				bool fde_has_augmentation_data,
136 				int code_alignment_factor,
137 				bool dry_run)
138 {
139 	int size = frame->size - offsetof(struct eh_frame, opcodes) + 4;
140 	u64 loc = (u64)offset_to_ptr(&frame->initial_loc);
141 	const u8 *opcode = frame->opcodes;
142 
143 	if (fde_has_augmentation_data) {
144 		int l;
145 
146 		// assume single byte uleb128_t
147 		if (WARN_ON(*opcode & BIT(7)))
148 			return -ENOEXEC;
149 
150 		l = *opcode++;
151 		opcode += l;
152 		size -= l + 1;
153 	}
154 
155 	/*
156 	 * Starting from 'loc', apply the CFA opcodes that advance the location
157 	 * pointer, and identify the locations of the PAC instructions.
158 	 */
159 	while (size-- > 0) {
160 		switch (*opcode++) {
161 		case DW_CFA_nop:
162 		case DW_CFA_remember_state:
163 		case DW_CFA_restore_state:
164 			break;
165 
166 		case DW_CFA_advance_loc1:
167 			loc += *opcode++ * code_alignment_factor;
168 			size--;
169 			break;
170 
171 		case DW_CFA_advance_loc2:
172 			loc += *opcode++ * code_alignment_factor;
173 			loc += (*opcode++ << 8) * code_alignment_factor;
174 			size -= 2;
175 			break;
176 
177 		case DW_CFA_def_cfa:
178 		case DW_CFA_offset_extended:
179 			size = skip_xleb128(&opcode, size);
180 			fallthrough;
181 		case DW_CFA_def_cfa_offset:
182 		case DW_CFA_def_cfa_offset_sf:
183 		case DW_CFA_def_cfa_register:
184 		case DW_CFA_same_value:
185 		case DW_CFA_restore_extended:
186 		case 0x80 ... 0xbf:
187 			size = skip_xleb128(&opcode, size);
188 			break;
189 
190 		case DW_CFA_negate_ra_state:
191 			if (!dry_run)
192 				scs_patch_loc(loc - 4);
193 			break;
194 
195 		case 0x40 ... 0x7f:
196 			// advance loc
197 			loc += (opcode[-1] & 0x3f) * code_alignment_factor;
198 			break;
199 
200 		case 0xc0 ... 0xff:
201 			break;
202 
203 		default:
204 			return -ENOEXEC;
205 		}
206 	}
207 	return 0;
208 }
209 
scs_patch(const u8 eh_frame[],int size)210 int scs_patch(const u8 eh_frame[], int size)
211 {
212 	const u8 *p = eh_frame;
213 
214 	while (size > 4) {
215 		const struct eh_frame *frame = (const void *)p;
216 		bool fde_has_augmentation_data = true;
217 		int code_alignment_factor = 1;
218 		int ret;
219 
220 		if (frame->size == 0 ||
221 		    frame->size == U32_MAX ||
222 		    frame->size > size)
223 			break;
224 
225 		if (frame->cie_id_or_pointer == 0) {
226 			const u8 *p = frame->augmentation_string;
227 
228 			/* a 'z' in the augmentation string must come first */
229 			fde_has_augmentation_data = *p == 'z';
230 
231 			/*
232 			 * The code alignment factor is a uleb128 encoded field
233 			 * but given that the only sensible values are 1 or 4,
234 			 * there is no point in decoding the whole thing.
235 			 */
236 			p += strlen(p) + 1;
237 			if (!WARN_ON(*p & BIT(7)))
238 				code_alignment_factor = *p;
239 		} else {
240 			ret = scs_handle_fde_frame(frame,
241 						   fde_has_augmentation_data,
242 						   code_alignment_factor,
243 						   true);
244 			if (ret)
245 				return ret;
246 			scs_handle_fde_frame(frame, fde_has_augmentation_data,
247 					     code_alignment_factor, false);
248 		}
249 
250 		p += sizeof(frame->size) + frame->size;
251 		size -= sizeof(frame->size) + frame->size;
252 	}
253 	return 0;
254 }
255