1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * This file contains core hardware tag-based KASAN code.
4 *
5 * Copyright (c) 2020 Google, Inc.
6 * Author: Andrey Konovalov <andreyknvl@google.com>
7 */
8
9 #define pr_fmt(fmt) "kasan: " fmt
10
11 #include <linux/init.h>
12 #include <linux/kasan.h>
13 #include <linux/kernel.h>
14 #include <linux/memory.h>
15 #include <linux/mm.h>
16 #include <linux/static_key.h>
17 #include <linux/string.h>
18 #include <linux/types.h>
19
20 #include "kasan.h"
21
22 enum kasan_arg {
23 KASAN_ARG_DEFAULT,
24 KASAN_ARG_OFF,
25 KASAN_ARG_ON,
26 };
27
28 enum kasan_arg_mode {
29 KASAN_ARG_MODE_DEFAULT,
30 KASAN_ARG_MODE_SYNC,
31 KASAN_ARG_MODE_ASYNC,
32 KASAN_ARG_MODE_ASYMM,
33 };
34
35 enum kasan_arg_vmalloc {
36 KASAN_ARG_VMALLOC_DEFAULT,
37 KASAN_ARG_VMALLOC_OFF,
38 KASAN_ARG_VMALLOC_ON,
39 };
40
41 static enum kasan_arg kasan_arg __ro_after_init;
42 static enum kasan_arg_mode kasan_arg_mode __ro_after_init;
43 static enum kasan_arg_vmalloc kasan_arg_vmalloc __initdata;
44
45 /*
46 * Whether KASAN is enabled at all.
47 * The value remains false until KASAN is initialized by kasan_init_hw_tags().
48 */
49 DEFINE_STATIC_KEY_FALSE(kasan_flag_enabled);
50 EXPORT_SYMBOL(kasan_flag_enabled);
51
52 /*
53 * Whether the selected mode is synchronous, asynchronous, or asymmetric.
54 * Defaults to KASAN_MODE_SYNC.
55 */
56 enum kasan_mode kasan_mode __ro_after_init;
57 EXPORT_SYMBOL_GPL(kasan_mode);
58
59 /* Whether to enable vmalloc tagging. */
60 #ifdef CONFIG_KASAN_VMALLOC
61 DEFINE_STATIC_KEY_TRUE(kasan_flag_vmalloc);
62 #else
63 DEFINE_STATIC_KEY_FALSE(kasan_flag_vmalloc);
64 #endif
65 EXPORT_SYMBOL_GPL(kasan_flag_vmalloc);
66
67 #define PAGE_ALLOC_SAMPLE_DEFAULT 1
68 #define PAGE_ALLOC_SAMPLE_ORDER_DEFAULT 3
69
70 /*
71 * Sampling interval of page_alloc allocation (un)poisoning.
72 * Defaults to no sampling.
73 */
74 unsigned long kasan_page_alloc_sample = PAGE_ALLOC_SAMPLE_DEFAULT;
75
76 /*
77 * Minimum order of page_alloc allocations to be affected by sampling.
78 * The default value is chosen to match both
79 * PAGE_ALLOC_COSTLY_ORDER and SKB_FRAG_PAGE_ORDER.
80 */
81 unsigned int kasan_page_alloc_sample_order = PAGE_ALLOC_SAMPLE_ORDER_DEFAULT;
82
83 DEFINE_PER_CPU(long, kasan_page_alloc_skip);
84
85 /* kasan=off/on */
early_kasan_flag(char * arg)86 static int __init early_kasan_flag(char *arg)
87 {
88 if (!arg)
89 return -EINVAL;
90
91 if (!strcmp(arg, "off"))
92 kasan_arg = KASAN_ARG_OFF;
93 else if (!strcmp(arg, "on"))
94 kasan_arg = KASAN_ARG_ON;
95 else
96 return -EINVAL;
97
98 return 0;
99 }
100 early_param("kasan", early_kasan_flag);
101
102 /* kasan.mode=sync/async/asymm */
early_kasan_mode(char * arg)103 static int __init early_kasan_mode(char *arg)
104 {
105 if (!arg)
106 return -EINVAL;
107
108 if (!strcmp(arg, "sync"))
109 kasan_arg_mode = KASAN_ARG_MODE_SYNC;
110 else if (!strcmp(arg, "async"))
111 kasan_arg_mode = KASAN_ARG_MODE_ASYNC;
112 else if (!strcmp(arg, "asymm"))
113 kasan_arg_mode = KASAN_ARG_MODE_ASYMM;
114 else
115 return -EINVAL;
116
117 return 0;
118 }
119 early_param("kasan.mode", early_kasan_mode);
120
121 /* kasan.vmalloc=off/on */
early_kasan_flag_vmalloc(char * arg)122 static int __init early_kasan_flag_vmalloc(char *arg)
123 {
124 if (!arg)
125 return -EINVAL;
126
127 if (!IS_ENABLED(CONFIG_KASAN_VMALLOC))
128 return 0;
129
130 if (!strcmp(arg, "off"))
131 kasan_arg_vmalloc = KASAN_ARG_VMALLOC_OFF;
132 else if (!strcmp(arg, "on"))
133 kasan_arg_vmalloc = KASAN_ARG_VMALLOC_ON;
134 else
135 return -EINVAL;
136
137 return 0;
138 }
139 early_param("kasan.vmalloc", early_kasan_flag_vmalloc);
140
kasan_mode_info(void)141 static inline const char *kasan_mode_info(void)
142 {
143 if (kasan_mode == KASAN_MODE_ASYNC)
144 return "async";
145 else if (kasan_mode == KASAN_MODE_ASYMM)
146 return "asymm";
147 else
148 return "sync";
149 }
150
151 /* kasan.page_alloc.sample=<sampling interval> */
early_kasan_flag_page_alloc_sample(char * arg)152 static int __init early_kasan_flag_page_alloc_sample(char *arg)
153 {
154 int rv;
155
156 if (!arg)
157 return -EINVAL;
158
159 rv = kstrtoul(arg, 0, &kasan_page_alloc_sample);
160 if (rv)
161 return rv;
162
163 if (!kasan_page_alloc_sample || kasan_page_alloc_sample > LONG_MAX) {
164 kasan_page_alloc_sample = PAGE_ALLOC_SAMPLE_DEFAULT;
165 return -EINVAL;
166 }
167
168 return 0;
169 }
170 early_param("kasan.page_alloc.sample", early_kasan_flag_page_alloc_sample);
171
172 /* kasan.page_alloc.sample.order=<minimum page order> */
early_kasan_flag_page_alloc_sample_order(char * arg)173 static int __init early_kasan_flag_page_alloc_sample_order(char *arg)
174 {
175 int rv;
176
177 if (!arg)
178 return -EINVAL;
179
180 rv = kstrtouint(arg, 0, &kasan_page_alloc_sample_order);
181 if (rv)
182 return rv;
183
184 if (kasan_page_alloc_sample_order > INT_MAX) {
185 kasan_page_alloc_sample_order = PAGE_ALLOC_SAMPLE_ORDER_DEFAULT;
186 return -EINVAL;
187 }
188
189 return 0;
190 }
191 early_param("kasan.page_alloc.sample.order", early_kasan_flag_page_alloc_sample_order);
192
193 /*
194 * kasan_init_hw_tags_cpu() is called for each CPU.
195 * Not marked as __init as a CPU can be hot-plugged after boot.
196 */
kasan_init_hw_tags_cpu(void)197 void kasan_init_hw_tags_cpu(void)
198 {
199 /*
200 * There's no need to check that the hardware is MTE-capable here,
201 * as this function is only called for MTE-capable hardware.
202 */
203
204 /*
205 * If KASAN is disabled via command line, don't initialize it.
206 * When this function is called, kasan_flag_enabled is not yet
207 * set by kasan_init_hw_tags(). Thus, check kasan_arg instead.
208 */
209 if (kasan_arg == KASAN_ARG_OFF)
210 return;
211
212 /*
213 * Enable async or asymm modes only when explicitly requested
214 * through the command line.
215 */
216 kasan_enable_hw_tags();
217 }
218
219 /* kasan_init_hw_tags() is called once on boot CPU. */
kasan_init_hw_tags(void)220 void __init kasan_init_hw_tags(void)
221 {
222 /* If hardware doesn't support MTE, don't initialize KASAN. */
223 if (!system_supports_mte())
224 return;
225
226 /* If KASAN is disabled via command line, don't initialize it. */
227 if (kasan_arg == KASAN_ARG_OFF)
228 return;
229
230 switch (kasan_arg_mode) {
231 case KASAN_ARG_MODE_DEFAULT:
232 /* Default is specified by kasan_mode definition. */
233 break;
234 case KASAN_ARG_MODE_SYNC:
235 kasan_mode = KASAN_MODE_SYNC;
236 break;
237 case KASAN_ARG_MODE_ASYNC:
238 kasan_mode = KASAN_MODE_ASYNC;
239 break;
240 case KASAN_ARG_MODE_ASYMM:
241 kasan_mode = KASAN_MODE_ASYMM;
242 break;
243 }
244
245 switch (kasan_arg_vmalloc) {
246 case KASAN_ARG_VMALLOC_DEFAULT:
247 /* Default is specified by kasan_flag_vmalloc definition. */
248 break;
249 case KASAN_ARG_VMALLOC_OFF:
250 static_branch_disable(&kasan_flag_vmalloc);
251 break;
252 case KASAN_ARG_VMALLOC_ON:
253 static_branch_enable(&kasan_flag_vmalloc);
254 break;
255 }
256
257 kasan_init_tags();
258
259 /* KASAN is now initialized, enable it. */
260 static_branch_enable(&kasan_flag_enabled);
261
262 pr_info("KernelAddressSanitizer initialized (hw-tags, mode=%s, vmalloc=%s, stacktrace=%s)\n",
263 kasan_mode_info(),
264 kasan_vmalloc_enabled() ? "on" : "off",
265 kasan_stack_collection_enabled() ? "on" : "off");
266 }
267
268 #ifdef CONFIG_KASAN_VMALLOC
269
unpoison_vmalloc_pages(const void * addr,u8 tag)270 static void unpoison_vmalloc_pages(const void *addr, u8 tag)
271 {
272 struct vm_struct *area;
273 int i;
274
275 /*
276 * As hardware tag-based KASAN only tags VM_ALLOC vmalloc allocations
277 * (see the comment in __kasan_unpoison_vmalloc), all of the pages
278 * should belong to a single area.
279 */
280 area = find_vm_area((void *)addr);
281 if (WARN_ON(!area))
282 return;
283
284 for (i = 0; i < area->nr_pages; i++) {
285 struct page *page = area->pages[i];
286
287 page_kasan_tag_set(page, tag);
288 }
289 }
290
init_vmalloc_pages(const void * start,unsigned long size)291 static void init_vmalloc_pages(const void *start, unsigned long size)
292 {
293 const void *addr;
294
295 for (addr = start; addr < start + size; addr += PAGE_SIZE) {
296 struct page *page = vmalloc_to_page(addr);
297
298 clear_highpage_kasan_tagged(page);
299 }
300 }
301
__kasan_unpoison_vmalloc(const void * start,unsigned long size,kasan_vmalloc_flags_t flags)302 void *__kasan_unpoison_vmalloc(const void *start, unsigned long size,
303 kasan_vmalloc_flags_t flags)
304 {
305 u8 tag;
306 unsigned long redzone_start, redzone_size;
307
308 if (!kasan_vmalloc_enabled()) {
309 if (flags & KASAN_VMALLOC_INIT)
310 init_vmalloc_pages(start, size);
311 return (void *)start;
312 }
313
314 /*
315 * Don't tag non-VM_ALLOC mappings, as:
316 *
317 * 1. Unlike the software KASAN modes, hardware tag-based KASAN only
318 * supports tagging physical memory. Therefore, it can only tag a
319 * single mapping of normal physical pages.
320 * 2. Hardware tag-based KASAN can only tag memory mapped with special
321 * mapping protection bits, see arch_vmap_pgprot_tagged().
322 * As non-VM_ALLOC mappings can be mapped outside of vmalloc code,
323 * providing these bits would require tracking all non-VM_ALLOC
324 * mappers.
325 *
326 * Thus, for VM_ALLOC mappings, hardware tag-based KASAN only tags
327 * the first virtual mapping, which is created by vmalloc().
328 * Tagging the page_alloc memory backing that vmalloc() allocation is
329 * skipped, see ___GFP_SKIP_KASAN.
330 *
331 * For non-VM_ALLOC allocations, page_alloc memory is tagged as usual.
332 */
333 if (!(flags & KASAN_VMALLOC_VM_ALLOC)) {
334 WARN_ON(flags & KASAN_VMALLOC_INIT);
335 return (void *)start;
336 }
337
338 /*
339 * Don't tag executable memory.
340 * The kernel doesn't tolerate having the PC register tagged.
341 */
342 if (!(flags & KASAN_VMALLOC_PROT_NORMAL)) {
343 WARN_ON(flags & KASAN_VMALLOC_INIT);
344 return (void *)start;
345 }
346
347 tag = kasan_random_tag();
348 start = set_tag(start, tag);
349
350 /* Unpoison and initialize memory up to size. */
351 kasan_unpoison(start, size, flags & KASAN_VMALLOC_INIT);
352
353 /*
354 * Explicitly poison and initialize the in-page vmalloc() redzone.
355 * Unlike software KASAN modes, hardware tag-based KASAN doesn't
356 * unpoison memory when populating shadow for vmalloc() space.
357 */
358 redzone_start = round_up((unsigned long)start + size,
359 KASAN_GRANULE_SIZE);
360 redzone_size = round_up(redzone_start, PAGE_SIZE) - redzone_start;
361 kasan_poison((void *)redzone_start, redzone_size, KASAN_TAG_INVALID,
362 flags & KASAN_VMALLOC_INIT);
363
364 /*
365 * Set per-page tag flags to allow accessing physical memory for the
366 * vmalloc() mapping through page_address(vmalloc_to_page()).
367 */
368 unpoison_vmalloc_pages(start, tag);
369
370 return (void *)start;
371 }
372
__kasan_poison_vmalloc(const void * start,unsigned long size)373 void __kasan_poison_vmalloc(const void *start, unsigned long size)
374 {
375 /*
376 * No tagging here.
377 * The physical pages backing the vmalloc() allocation are poisoned
378 * through the usual page_alloc paths.
379 */
380 }
381
382 #endif
383
kasan_enable_hw_tags(void)384 void kasan_enable_hw_tags(void)
385 {
386 if (kasan_arg_mode == KASAN_ARG_MODE_ASYNC)
387 hw_enable_tag_checks_async();
388 else if (kasan_arg_mode == KASAN_ARG_MODE_ASYMM)
389 hw_enable_tag_checks_asymm();
390 else
391 hw_enable_tag_checks_sync();
392 }
393
394 #if IS_ENABLED(CONFIG_KASAN_KUNIT_TEST)
395
396 EXPORT_SYMBOL_GPL(kasan_enable_hw_tags);
397
kasan_force_async_fault(void)398 void kasan_force_async_fault(void)
399 {
400 hw_force_async_tag_fault();
401 }
402 EXPORT_SYMBOL_GPL(kasan_force_async_fault);
403
404 #endif
405