1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * This file contains core hardware tag-based KASAN code.
4  *
5  * Copyright (c) 2020 Google, Inc.
6  * Author: Andrey Konovalov <andreyknvl@google.com>
7  */
8 
9 #define pr_fmt(fmt) "kasan: " fmt
10 
11 #include <kunit/visibility.h>
12 #include <linux/init.h>
13 #include <linux/kasan.h>
14 #include <linux/kernel.h>
15 #include <linux/memory.h>
16 #include <linux/mm.h>
17 #include <linux/static_key.h>
18 #include <linux/string.h>
19 #include <linux/string_choices.h>
20 #include <linux/types.h>
21 #include <linux/vmalloc.h>
22 
23 #include "kasan.h"
24 
25 enum kasan_arg {
26 	KASAN_ARG_DEFAULT,
27 	KASAN_ARG_OFF,
28 	KASAN_ARG_ON,
29 };
30 
31 enum kasan_arg_mode {
32 	KASAN_ARG_MODE_DEFAULT,
33 	KASAN_ARG_MODE_SYNC,
34 	KASAN_ARG_MODE_ASYNC,
35 	KASAN_ARG_MODE_ASYMM,
36 };
37 
38 enum kasan_arg_vmalloc {
39 	KASAN_ARG_VMALLOC_DEFAULT,
40 	KASAN_ARG_VMALLOC_OFF,
41 	KASAN_ARG_VMALLOC_ON,
42 };
43 
44 static enum kasan_arg kasan_arg __ro_after_init;
45 static enum kasan_arg_mode kasan_arg_mode __ro_after_init;
46 static enum kasan_arg_vmalloc kasan_arg_vmalloc __initdata;
47 
48 /*
49  * Whether KASAN is enabled at all.
50  * The value remains false until KASAN is initialized by kasan_init_hw_tags().
51  */
52 DEFINE_STATIC_KEY_FALSE(kasan_flag_enabled);
53 EXPORT_SYMBOL(kasan_flag_enabled);
54 
55 /*
56  * Whether the selected mode is synchronous, asynchronous, or asymmetric.
57  * Defaults to KASAN_MODE_SYNC.
58  */
59 enum kasan_mode kasan_mode __ro_after_init;
60 EXPORT_SYMBOL_GPL(kasan_mode);
61 
62 /* Whether to enable vmalloc tagging. */
63 #ifdef CONFIG_KASAN_VMALLOC
64 DEFINE_STATIC_KEY_TRUE(kasan_flag_vmalloc);
65 #else
66 DEFINE_STATIC_KEY_FALSE(kasan_flag_vmalloc);
67 #endif
68 EXPORT_SYMBOL_GPL(kasan_flag_vmalloc);
69 
70 #define PAGE_ALLOC_SAMPLE_DEFAULT	1
71 #define PAGE_ALLOC_SAMPLE_ORDER_DEFAULT	3
72 
73 /*
74  * Sampling interval of page_alloc allocation (un)poisoning.
75  * Defaults to no sampling.
76  */
77 unsigned long kasan_page_alloc_sample = PAGE_ALLOC_SAMPLE_DEFAULT;
78 
79 /*
80  * Minimum order of page_alloc allocations to be affected by sampling.
81  * The default value is chosen to match both
82  * PAGE_ALLOC_COSTLY_ORDER and SKB_FRAG_PAGE_ORDER.
83  */
84 unsigned int kasan_page_alloc_sample_order = PAGE_ALLOC_SAMPLE_ORDER_DEFAULT;
85 
86 DEFINE_PER_CPU(long, kasan_page_alloc_skip);
87 
88 /* kasan=off/on */
89 static int __init early_kasan_flag(char *arg)
90 {
91 	if (!arg)
92 		return -EINVAL;
93 
94 	if (!strcmp(arg, "off"))
95 		kasan_arg = KASAN_ARG_OFF;
96 	else if (!strcmp(arg, "on"))
97 		kasan_arg = KASAN_ARG_ON;
98 	else
99 		return -EINVAL;
100 
101 	return 0;
102 }
103 early_param("kasan", early_kasan_flag);
104 
105 /* kasan.mode=sync/async/asymm */
106 static int __init early_kasan_mode(char *arg)
107 {
108 	if (!arg)
109 		return -EINVAL;
110 
111 	if (!strcmp(arg, "sync"))
112 		kasan_arg_mode = KASAN_ARG_MODE_SYNC;
113 	else if (!strcmp(arg, "async"))
114 		kasan_arg_mode = KASAN_ARG_MODE_ASYNC;
115 	else if (!strcmp(arg, "asymm"))
116 		kasan_arg_mode = KASAN_ARG_MODE_ASYMM;
117 	else
118 		return -EINVAL;
119 
120 	return 0;
121 }
122 early_param("kasan.mode", early_kasan_mode);
123 
124 /* kasan.vmalloc=off/on */
125 static int __init early_kasan_flag_vmalloc(char *arg)
126 {
127 	if (!arg)
128 		return -EINVAL;
129 
130 	if (!IS_ENABLED(CONFIG_KASAN_VMALLOC))
131 		return 0;
132 
133 	if (!strcmp(arg, "off"))
134 		kasan_arg_vmalloc = KASAN_ARG_VMALLOC_OFF;
135 	else if (!strcmp(arg, "on"))
136 		kasan_arg_vmalloc = KASAN_ARG_VMALLOC_ON;
137 	else
138 		return -EINVAL;
139 
140 	return 0;
141 }
142 early_param("kasan.vmalloc", early_kasan_flag_vmalloc);
143 
144 static inline const char *kasan_mode_info(void)
145 {
146 	if (kasan_mode == KASAN_MODE_ASYNC)
147 		return "async";
148 	else if (kasan_mode == KASAN_MODE_ASYMM)
149 		return "asymm";
150 	else
151 		return "sync";
152 }
153 
154 /* kasan.page_alloc.sample=<sampling interval> */
155 static int __init early_kasan_flag_page_alloc_sample(char *arg)
156 {
157 	int rv;
158 
159 	if (!arg)
160 		return -EINVAL;
161 
162 	rv = kstrtoul(arg, 0, &kasan_page_alloc_sample);
163 	if (rv)
164 		return rv;
165 
166 	if (!kasan_page_alloc_sample || kasan_page_alloc_sample > LONG_MAX) {
167 		kasan_page_alloc_sample = PAGE_ALLOC_SAMPLE_DEFAULT;
168 		return -EINVAL;
169 	}
170 
171 	return 0;
172 }
173 early_param("kasan.page_alloc.sample", early_kasan_flag_page_alloc_sample);
174 
175 /* kasan.page_alloc.sample.order=<minimum page order> */
176 static int __init early_kasan_flag_page_alloc_sample_order(char *arg)
177 {
178 	int rv;
179 
180 	if (!arg)
181 		return -EINVAL;
182 
183 	rv = kstrtouint(arg, 0, &kasan_page_alloc_sample_order);
184 	if (rv)
185 		return rv;
186 
187 	if (kasan_page_alloc_sample_order > INT_MAX) {
188 		kasan_page_alloc_sample_order = PAGE_ALLOC_SAMPLE_ORDER_DEFAULT;
189 		return -EINVAL;
190 	}
191 
192 	return 0;
193 }
194 early_param("kasan.page_alloc.sample.order", early_kasan_flag_page_alloc_sample_order);
195 
196 /*
197  * kasan_init_hw_tags_cpu() is called for each CPU.
198  * Not marked as __init as a CPU can be hot-plugged after boot.
199  */
200 void kasan_init_hw_tags_cpu(void)
201 {
202 	/*
203 	 * There's no need to check that the hardware is MTE-capable here,
204 	 * as this function is only called for MTE-capable hardware.
205 	 */
206 
207 	/*
208 	 * If KASAN is disabled via command line, don't initialize it.
209 	 * When this function is called, kasan_flag_enabled is not yet
210 	 * set by kasan_init_hw_tags(). Thus, check kasan_arg instead.
211 	 */
212 	if (kasan_arg == KASAN_ARG_OFF)
213 		return;
214 
215 	/*
216 	 * Enable async or asymm modes only when explicitly requested
217 	 * through the command line.
218 	 */
219 	kasan_enable_hw_tags();
220 }
221 
222 /* kasan_init_hw_tags() is called once on boot CPU. */
223 void __init kasan_init_hw_tags(void)
224 {
225 	/* If hardware doesn't support MTE, don't initialize KASAN. */
226 	if (!system_supports_mte())
227 		return;
228 
229 	/* If KASAN is disabled via command line, don't initialize it. */
230 	if (kasan_arg == KASAN_ARG_OFF)
231 		return;
232 
233 	switch (kasan_arg_mode) {
234 	case KASAN_ARG_MODE_DEFAULT:
235 		/* Default is specified by kasan_mode definition. */
236 		break;
237 	case KASAN_ARG_MODE_SYNC:
238 		kasan_mode = KASAN_MODE_SYNC;
239 		break;
240 	case KASAN_ARG_MODE_ASYNC:
241 		kasan_mode = KASAN_MODE_ASYNC;
242 		break;
243 	case KASAN_ARG_MODE_ASYMM:
244 		kasan_mode = KASAN_MODE_ASYMM;
245 		break;
246 	}
247 
248 	switch (kasan_arg_vmalloc) {
249 	case KASAN_ARG_VMALLOC_DEFAULT:
250 		/* Default is specified by kasan_flag_vmalloc definition. */
251 		break;
252 	case KASAN_ARG_VMALLOC_OFF:
253 		static_branch_disable(&kasan_flag_vmalloc);
254 		break;
255 	case KASAN_ARG_VMALLOC_ON:
256 		static_branch_enable(&kasan_flag_vmalloc);
257 		break;
258 	}
259 
260 	kasan_init_tags();
261 
262 	/* KASAN is now initialized, enable it. */
263 	static_branch_enable(&kasan_flag_enabled);
264 
265 	pr_info("KernelAddressSanitizer initialized (hw-tags, mode=%s, vmalloc=%s, stacktrace=%s)\n",
266 		kasan_mode_info(),
267 		str_on_off(kasan_vmalloc_enabled()),
268 		str_on_off(kasan_stack_collection_enabled()));
269 }
270 
271 #ifdef CONFIG_KASAN_VMALLOC
272 
273 static void unpoison_vmalloc_pages(const void *addr, u8 tag)
274 {
275 	struct vm_struct *area;
276 	int i;
277 
278 	/*
279 	 * As hardware tag-based KASAN only tags VM_ALLOC vmalloc allocations
280 	 * (see the comment in __kasan_unpoison_vmalloc), all of the pages
281 	 * should belong to a single area.
282 	 */
283 	area = find_vm_area((void *)addr);
284 	if (WARN_ON(!area))
285 		return;
286 
287 	for (i = 0; i < area->nr_pages; i++) {
288 		struct page *page = area->pages[i];
289 
290 		page_kasan_tag_set(page, tag);
291 	}
292 }
293 
294 static void init_vmalloc_pages(const void *start, unsigned long size)
295 {
296 	const void *addr;
297 
298 	for (addr = start; addr < start + size; addr += PAGE_SIZE) {
299 		struct page *page = vmalloc_to_page(addr);
300 
301 		clear_highpage_kasan_tagged(page);
302 	}
303 }
304 
305 void *__kasan_unpoison_vmalloc(const void *start, unsigned long size,
306 				kasan_vmalloc_flags_t flags)
307 {
308 	u8 tag;
309 	unsigned long redzone_start, redzone_size;
310 
311 	if (!kasan_vmalloc_enabled()) {
312 		if (flags & KASAN_VMALLOC_INIT)
313 			init_vmalloc_pages(start, size);
314 		return (void *)start;
315 	}
316 
317 	/*
318 	 * Don't tag non-VM_ALLOC mappings, as:
319 	 *
320 	 * 1. Unlike the software KASAN modes, hardware tag-based KASAN only
321 	 *    supports tagging physical memory. Therefore, it can only tag a
322 	 *    single mapping of normal physical pages.
323 	 * 2. Hardware tag-based KASAN can only tag memory mapped with special
324 	 *    mapping protection bits, see arch_vmap_pgprot_tagged().
325 	 *    As non-VM_ALLOC mappings can be mapped outside of vmalloc code,
326 	 *    providing these bits would require tracking all non-VM_ALLOC
327 	 *    mappers.
328 	 *
329 	 * Thus, for VM_ALLOC mappings, hardware tag-based KASAN only tags
330 	 * the first virtual mapping, which is created by vmalloc().
331 	 * Tagging the page_alloc memory backing that vmalloc() allocation is
332 	 * skipped, see ___GFP_SKIP_KASAN.
333 	 *
334 	 * For non-VM_ALLOC allocations, page_alloc memory is tagged as usual.
335 	 */
336 	if (!(flags & KASAN_VMALLOC_VM_ALLOC)) {
337 		WARN_ON(flags & KASAN_VMALLOC_INIT);
338 		return (void *)start;
339 	}
340 
341 	/*
342 	 * Don't tag executable memory.
343 	 * The kernel doesn't tolerate having the PC register tagged.
344 	 */
345 	if (!(flags & KASAN_VMALLOC_PROT_NORMAL)) {
346 		WARN_ON(flags & KASAN_VMALLOC_INIT);
347 		return (void *)start;
348 	}
349 
350 	tag = kasan_random_tag();
351 	start = set_tag(start, tag);
352 
353 	/* Unpoison and initialize memory up to size. */
354 	kasan_unpoison(start, size, flags & KASAN_VMALLOC_INIT);
355 
356 	/*
357 	 * Explicitly poison and initialize the in-page vmalloc() redzone.
358 	 * Unlike software KASAN modes, hardware tag-based KASAN doesn't
359 	 * unpoison memory when populating shadow for vmalloc() space.
360 	 */
361 	redzone_start = round_up((unsigned long)start + size,
362 				 KASAN_GRANULE_SIZE);
363 	redzone_size = round_up(redzone_start, PAGE_SIZE) - redzone_start;
364 	kasan_poison((void *)redzone_start, redzone_size, KASAN_TAG_INVALID,
365 		     flags & KASAN_VMALLOC_INIT);
366 
367 	/*
368 	 * Set per-page tag flags to allow accessing physical memory for the
369 	 * vmalloc() mapping through page_address(vmalloc_to_page()).
370 	 */
371 	unpoison_vmalloc_pages(start, tag);
372 
373 	return (void *)start;
374 }
375 
376 void __kasan_poison_vmalloc(const void *start, unsigned long size)
377 {
378 	/*
379 	 * No tagging here.
380 	 * The physical pages backing the vmalloc() allocation are poisoned
381 	 * through the usual page_alloc paths.
382 	 */
383 }
384 
385 #endif
386 
387 void kasan_enable_hw_tags(void)
388 {
389 	if (kasan_arg_mode == KASAN_ARG_MODE_ASYNC)
390 		hw_enable_tag_checks_async();
391 	else if (kasan_arg_mode == KASAN_ARG_MODE_ASYMM)
392 		hw_enable_tag_checks_asymm();
393 	else
394 		hw_enable_tag_checks_sync();
395 }
396 
397 #if IS_ENABLED(CONFIG_KASAN_KUNIT_TEST)
398 
399 EXPORT_SYMBOL_IF_KUNIT(kasan_enable_hw_tags);
400 
401 VISIBLE_IF_KUNIT void kasan_force_async_fault(void)
402 {
403 	hw_force_async_tag_fault();
404 }
405 EXPORT_SYMBOL_IF_KUNIT(kasan_force_async_fault);
406 
407 #endif
408