1 // SPDX-License-Identifier: GPL-2.0-only
2
3 #define pr_fmt(fmt) "callthunks: " fmt
4
5 #include <linux/debugfs.h>
6 #include <linux/kallsyms.h>
7 #include <linux/memory.h>
8 #include <linux/moduleloader.h>
9 #include <linux/static_call.h>
10
11 #include <asm/alternative.h>
12 #include <asm/asm-offsets.h>
13 #include <asm/cpu.h>
14 #include <asm/ftrace.h>
15 #include <asm/insn.h>
16 #include <asm/kexec.h>
17 #include <asm/nospec-branch.h>
18 #include <asm/sections.h>
19 #include <asm/switch_to.h>
20 #include <asm/sync_core.h>
21 #include <asm/text-patching.h>
22 #include <asm/xen/hypercall.h>
23
24 static int __initdata_or_module debug_callthunks;
25
26 #define MAX_PATCH_LEN (255-1)
27
28 #define prdbg(fmt, args...) \
29 do { \
30 if (debug_callthunks) \
31 printk(KERN_DEBUG pr_fmt(fmt), ##args); \
32 } while(0)
33
debug_thunks(char * str)34 static int __init debug_thunks(char *str)
35 {
36 debug_callthunks = 1;
37 return 1;
38 }
39 __setup("debug-callthunks", debug_thunks);
40
41 #ifdef CONFIG_CALL_THUNKS_DEBUG
42 DEFINE_PER_CPU(u64, __x86_call_count);
43 DEFINE_PER_CPU(u64, __x86_ret_count);
44 DEFINE_PER_CPU(u64, __x86_stuffs_count);
45 DEFINE_PER_CPU(u64, __x86_ctxsw_count);
46 EXPORT_PER_CPU_SYMBOL_GPL(__x86_ctxsw_count);
47 EXPORT_PER_CPU_SYMBOL_GPL(__x86_call_count);
48 #endif
49
50 extern s32 __call_sites[], __call_sites_end[];
51
52 struct core_text {
53 unsigned long base;
54 unsigned long end;
55 const char *name;
56 };
57
58 static bool thunks_initialized __ro_after_init;
59
60 static const struct core_text builtin_coretext = {
61 .base = (unsigned long)_text,
62 .end = (unsigned long)_etext,
63 .name = "builtin",
64 };
65
66 asm (
67 ".pushsection .rodata \n"
68 ".global skl_call_thunk_template \n"
69 "skl_call_thunk_template: \n"
70 __stringify(INCREMENT_CALL_DEPTH)" \n"
71 ".global skl_call_thunk_tail \n"
72 "skl_call_thunk_tail: \n"
73 ".popsection \n"
74 );
75
76 extern u8 skl_call_thunk_template[];
77 extern u8 skl_call_thunk_tail[];
78
79 #define SKL_TMPL_SIZE \
80 ((unsigned int)(skl_call_thunk_tail - skl_call_thunk_template))
81
82 extern void error_entry(void);
83 extern void xen_error_entry(void);
84 extern void paranoid_entry(void);
85
within_coretext(const struct core_text * ct,void * addr)86 static inline bool within_coretext(const struct core_text *ct, void *addr)
87 {
88 unsigned long p = (unsigned long)addr;
89
90 return ct->base <= p && p < ct->end;
91 }
92
within_module_coretext(void * addr)93 static inline bool within_module_coretext(void *addr)
94 {
95 bool ret = false;
96
97 #ifdef CONFIG_MODULES
98 struct module *mod;
99
100 guard(rcu)();
101 mod = __module_address((unsigned long)addr);
102 if (mod && within_module_core((unsigned long)addr, mod))
103 ret = true;
104 #endif
105 return ret;
106 }
107
is_coretext(const struct core_text * ct,void * addr)108 static bool is_coretext(const struct core_text *ct, void *addr)
109 {
110 if (ct && within_coretext(ct, addr))
111 return true;
112 if (within_coretext(&builtin_coretext, addr))
113 return true;
114 return within_module_coretext(addr);
115 }
116
skip_addr(void * dest)117 static bool skip_addr(void *dest)
118 {
119 if (dest == error_entry)
120 return true;
121 if (dest == paranoid_entry)
122 return true;
123 if (dest == xen_error_entry)
124 return true;
125 /* Does FILL_RSB... */
126 if (dest == __switch_to_asm)
127 return true;
128 /* Accounts directly */
129 if (dest == ret_from_fork)
130 return true;
131 #if defined(CONFIG_HOTPLUG_CPU) && defined(CONFIG_AMD_MEM_ENCRYPT)
132 if (dest == soft_restart_cpu)
133 return true;
134 #endif
135 #ifdef CONFIG_FUNCTION_TRACER
136 if (dest == __fentry__)
137 return true;
138 #endif
139 #ifdef CONFIG_KEXEC_CORE
140 # ifdef CONFIG_X86_64
141 if (dest >= (void *)__relocate_kernel_start &&
142 dest < (void *)__relocate_kernel_end)
143 return true;
144 # else
145 if (dest >= (void *)relocate_kernel &&
146 dest < (void*)relocate_kernel + KEXEC_CONTROL_CODE_MAX_SIZE)
147 return true;
148 # endif
149 #endif
150 return false;
151 }
152
call_get_dest(void * addr)153 static __init_or_module void *call_get_dest(void *addr)
154 {
155 struct insn insn;
156 void *dest;
157 int ret;
158
159 ret = insn_decode_kernel(&insn, addr);
160 if (ret)
161 return ERR_PTR(ret);
162
163 /* Patched out call? */
164 if (insn.opcode.bytes[0] != CALL_INSN_OPCODE)
165 return NULL;
166
167 dest = addr + insn.length + insn.immediate.value;
168 if (skip_addr(dest))
169 return NULL;
170 return dest;
171 }
172
173 static const u8 nops[] = {
174 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
175 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
176 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
177 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90, 0x90,
178 };
179
patch_dest(void * dest,bool direct)180 static void *patch_dest(void *dest, bool direct)
181 {
182 unsigned int tsize = SKL_TMPL_SIZE;
183 u8 insn_buff[MAX_PATCH_LEN];
184 u8 *pad = dest - tsize;
185
186 memcpy(insn_buff, skl_call_thunk_template, tsize);
187 text_poke_apply_relocation(insn_buff, pad, tsize, skl_call_thunk_template, tsize);
188
189 /* Already patched? */
190 if (!bcmp(pad, insn_buff, tsize))
191 return pad;
192
193 /* Ensure there are nops */
194 if (bcmp(pad, nops, tsize)) {
195 pr_warn_once("Invalid padding area for %pS\n", dest);
196 return NULL;
197 }
198
199 if (direct)
200 memcpy(pad, insn_buff, tsize);
201 else
202 text_poke_copy_locked(pad, insn_buff, tsize, true);
203 return pad;
204 }
205
patch_call(void * addr,const struct core_text * ct)206 static __init_or_module void patch_call(void *addr, const struct core_text *ct)
207 {
208 void *pad, *dest;
209 u8 bytes[8];
210
211 if (!within_coretext(ct, addr))
212 return;
213
214 dest = call_get_dest(addr);
215 if (!dest || WARN_ON_ONCE(IS_ERR(dest)))
216 return;
217
218 if (!is_coretext(ct, dest))
219 return;
220
221 pad = patch_dest(dest, within_coretext(ct, dest));
222 if (!pad)
223 return;
224
225 prdbg("Patch call at: %pS %px to %pS %px -> %px \n", addr, addr,
226 dest, dest, pad);
227 __text_gen_insn(bytes, CALL_INSN_OPCODE, addr, pad, CALL_INSN_SIZE);
228 text_poke_early(addr, bytes, CALL_INSN_SIZE);
229 }
230
231 static __init_or_module void
patch_call_sites(s32 * start,s32 * end,const struct core_text * ct)232 patch_call_sites(s32 *start, s32 *end, const struct core_text *ct)
233 {
234 s32 *s;
235
236 for (s = start; s < end; s++)
237 patch_call((void *)s + *s, ct);
238 }
239
240 static __init_or_module void
callthunks_setup(struct callthunk_sites * cs,const struct core_text * ct)241 callthunks_setup(struct callthunk_sites *cs, const struct core_text *ct)
242 {
243 prdbg("Patching call sites %s\n", ct->name);
244 patch_call_sites(cs->call_start, cs->call_end, ct);
245 prdbg("Patching call sites done%s\n", ct->name);
246 }
247
callthunks_patch_builtin_calls(void)248 void __init callthunks_patch_builtin_calls(void)
249 {
250 struct callthunk_sites cs = {
251 .call_start = __call_sites,
252 .call_end = __call_sites_end,
253 };
254
255 if (!cpu_feature_enabled(X86_FEATURE_CALL_DEPTH))
256 return;
257
258 pr_info("Setting up call depth tracking\n");
259 mutex_lock(&text_mutex);
260 callthunks_setup(&cs, &builtin_coretext);
261 thunks_initialized = true;
262 mutex_unlock(&text_mutex);
263 }
264
callthunks_translate_call_dest(void * dest)265 void *callthunks_translate_call_dest(void *dest)
266 {
267 void *target;
268
269 lockdep_assert_held(&text_mutex);
270
271 if (!thunks_initialized || skip_addr(dest))
272 return dest;
273
274 if (!is_coretext(NULL, dest))
275 return dest;
276
277 target = patch_dest(dest, false);
278 return target ? : dest;
279 }
280
281 #ifdef CONFIG_BPF_JIT
is_callthunk(void * addr)282 static bool is_callthunk(void *addr)
283 {
284 unsigned int tmpl_size = SKL_TMPL_SIZE;
285 u8 insn_buff[MAX_PATCH_LEN];
286 unsigned long dest;
287 u8 *pad;
288
289 dest = roundup((unsigned long)addr, CONFIG_FUNCTION_ALIGNMENT);
290 if (!thunks_initialized || skip_addr((void *)dest))
291 return false;
292
293 pad = (void *)(dest - tmpl_size);
294
295 memcpy(insn_buff, skl_call_thunk_template, tmpl_size);
296 text_poke_apply_relocation(insn_buff, pad, tmpl_size, skl_call_thunk_template, tmpl_size);
297
298 return !bcmp(pad, insn_buff, tmpl_size);
299 }
300
x86_call_depth_emit_accounting(u8 ** pprog,void * func,void * ip)301 int x86_call_depth_emit_accounting(u8 **pprog, void *func, void *ip)
302 {
303 unsigned int tmpl_size = SKL_TMPL_SIZE;
304 u8 insn_buff[MAX_PATCH_LEN];
305
306 if (!thunks_initialized)
307 return 0;
308
309 /* Is function call target a thunk? */
310 if (func && is_callthunk(func))
311 return 0;
312
313 memcpy(insn_buff, skl_call_thunk_template, tmpl_size);
314 text_poke_apply_relocation(insn_buff, ip, tmpl_size, skl_call_thunk_template, tmpl_size);
315
316 memcpy(*pprog, insn_buff, tmpl_size);
317 *pprog += tmpl_size;
318 return tmpl_size;
319 }
320 #endif
321
322 #ifdef CONFIG_MODULES
callthunks_patch_module_calls(struct callthunk_sites * cs,struct module * mod)323 void noinline callthunks_patch_module_calls(struct callthunk_sites *cs,
324 struct module *mod)
325 {
326 struct core_text ct = {
327 .base = (unsigned long)mod->mem[MOD_TEXT].base,
328 .end = (unsigned long)mod->mem[MOD_TEXT].base + mod->mem[MOD_TEXT].size,
329 .name = mod->name,
330 };
331
332 if (!thunks_initialized)
333 return;
334
335 mutex_lock(&text_mutex);
336 callthunks_setup(cs, &ct);
337 mutex_unlock(&text_mutex);
338 }
339 #endif /* CONFIG_MODULES */
340
341 #if defined(CONFIG_CALL_THUNKS_DEBUG) && defined(CONFIG_DEBUG_FS)
callthunks_debug_show(struct seq_file * m,void * p)342 static int callthunks_debug_show(struct seq_file *m, void *p)
343 {
344 unsigned long cpu = (unsigned long)m->private;
345
346 seq_printf(m, "C: %16llu R: %16llu S: %16llu X: %16llu\n,",
347 per_cpu(__x86_call_count, cpu),
348 per_cpu(__x86_ret_count, cpu),
349 per_cpu(__x86_stuffs_count, cpu),
350 per_cpu(__x86_ctxsw_count, cpu));
351 return 0;
352 }
353
callthunks_debug_open(struct inode * inode,struct file * file)354 static int callthunks_debug_open(struct inode *inode, struct file *file)
355 {
356 return single_open(file, callthunks_debug_show, inode->i_private);
357 }
358
359 static const struct file_operations dfs_ops = {
360 .open = callthunks_debug_open,
361 .read = seq_read,
362 .llseek = seq_lseek,
363 .release = single_release,
364 };
365
callthunks_debugfs_init(void)366 static int __init callthunks_debugfs_init(void)
367 {
368 struct dentry *dir;
369 unsigned long cpu;
370
371 dir = debugfs_create_dir("callthunks", NULL);
372 for_each_possible_cpu(cpu) {
373 void *arg = (void *)cpu;
374 char name [10];
375
376 sprintf(name, "cpu%lu", cpu);
377 debugfs_create_file(name, 0644, dir, arg, &dfs_ops);
378 }
379 return 0;
380 }
381 __initcall(callthunks_debugfs_init);
382 #endif
383