1 /* $NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $ */
2
3 /*
4 * Copyright (c) 2019-2020 Maxime Villard, m00nbsd.net
5 * All rights reserved.
6 * Copyright (c) 2021 The FreeBSD Foundation
7 *
8 * Portions of this software were developed by Mark Johnston under sponsorship
9 * from the FreeBSD Foundation.
10 *
11 * This code is part of the KMSAN subsystem of the NetBSD kernel.
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 * 1. Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
23 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
24 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
25 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 */
34
35 #define SAN_RUNTIME
36
37 #include <sys/cdefs.h>
38 #if 0
39 __KERNEL_RCSID(0, "$NetBSD: subr_msan.c,v 1.14 2020/09/09 16:29:59 maxv Exp $");
40 #endif
41
42 #include <sys/param.h>
43 #include <sys/systm.h>
44 #include <sys/bio.h>
45 #include <sys/buf.h>
46 #include <sys/conf.h>
47 #include <sys/kdb.h>
48 #include <sys/kernel.h>
49 #include <sys/linker.h>
50 #include <sys/malloc.h>
51 #include <sys/mbuf.h>
52 #include <sys/memdesc.h>
53 #include <sys/msan.h>
54 #include <sys/proc.h>
55 #include <sys/stack.h>
56 #include <sys/stdarg.h>
57 #include <sys/sysctl.h>
58 #include <sys/uio.h>
59
60 #include <vm/vm.h>
61 #include <vm/pmap.h>
62
63 #include <machine/msan.h>
64
65 void kmsan_init_arg(size_t);
66 void kmsan_init_ret(size_t);
67
68 /* -------------------------------------------------------------------------- */
69
70 /*
71 * Part of the compiler ABI.
72 */
73
74 typedef struct {
75 uint8_t *shad;
76 msan_orig_t *orig;
77 } msan_meta_t;
78
79 #define MSAN_PARAM_SIZE 800
80 #define MSAN_RETVAL_SIZE 800
81 typedef struct {
82 uint8_t param_shadow[MSAN_PARAM_SIZE];
83 uint8_t retval_shadow[MSAN_RETVAL_SIZE];
84 uint8_t va_arg_shadow[MSAN_PARAM_SIZE];
85 uint8_t va_arg_origin[MSAN_PARAM_SIZE];
86 uint64_t va_arg_overflow_size;
87 msan_orig_t param_origin[MSAN_PARAM_SIZE / sizeof(msan_orig_t)];
88 msan_orig_t retval_origin;
89 } msan_tls_t;
90
91 /* -------------------------------------------------------------------------- */
92
93 #define MSAN_NCONTEXT 4
94 #define MSAN_ORIG_MASK (~0x3)
95
96 typedef struct kmsan_td {
97 size_t ctx;
98 msan_tls_t tls[MSAN_NCONTEXT];
99 } msan_td_t;
100
101 static msan_tls_t dummy_tls;
102
103 /*
104 * Use separate dummy regions for loads and stores: stores may mark the region
105 * as uninitialized, and that can trigger false positives.
106 */
107 static uint8_t msan_dummy_shad[PAGE_SIZE] __aligned(PAGE_SIZE);
108 static uint8_t msan_dummy_write_shad[PAGE_SIZE] __aligned(PAGE_SIZE);
109 static uint8_t msan_dummy_orig[PAGE_SIZE] __aligned(PAGE_SIZE);
110 static msan_td_t msan_thread0;
111 static bool kmsan_reporting = false;
112
113 /*
114 * Avoid clobbering any thread-local state before we panic.
115 */
116 #define kmsan_panic(f, ...) do { \
117 kmsan_disabled = true; \
118 panic(f, __VA_ARGS__); \
119 } while (0)
120
121 #define REPORT(f, ...) do { \
122 if (panic_on_violation) { \
123 kmsan_panic(f, __VA_ARGS__); \
124 } else { \
125 struct stack st; \
126 \
127 stack_save(&st); \
128 printf(f "\n", __VA_ARGS__); \
129 stack_print_ddb(&st); \
130 } \
131 } while (0)
132
133 FEATURE(kmsan, "Kernel memory sanitizer");
134
135 static SYSCTL_NODE(_debug, OID_AUTO, kmsan, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
136 "KMSAN options");
137
138 static bool panic_on_violation = 1;
139 SYSCTL_BOOL(_debug_kmsan, OID_AUTO, panic_on_violation, CTLFLAG_RWTUN,
140 &panic_on_violation, 0,
141 "Panic if an invalid access is detected");
142
143 static bool kmsan_disabled __read_mostly = true;
144 #define kmsan_enabled (!kmsan_disabled)
145 SYSCTL_BOOL(_debug_kmsan, OID_AUTO, disabled, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
146 &kmsan_disabled, 0, "KMSAN is disabled");
147
148 static MALLOC_DEFINE(M_KMSAN, "kmsan", "Kernel memory sanitizer");
149
150 /* -------------------------------------------------------------------------- */
151
152 static inline const char *
kmsan_orig_name(int type)153 kmsan_orig_name(int type)
154 {
155 switch (type) {
156 case KMSAN_TYPE_STACK:
157 return ("stack");
158 case KMSAN_TYPE_KMEM:
159 return ("kmem");
160 case KMSAN_TYPE_MALLOC:
161 return ("malloc");
162 case KMSAN_TYPE_UMA:
163 return ("UMA");
164 default:
165 return ("unknown");
166 }
167 }
168
169 static void
kmsan_report_hook(const void * addr,msan_orig_t * orig,size_t size,size_t off,const char * hook)170 kmsan_report_hook(const void *addr, msan_orig_t *orig, size_t size, size_t off,
171 const char *hook)
172 {
173 const char *typename;
174 char *var, *fn;
175 uintptr_t ptr;
176 long foff;
177 char buf[128];
178 int type;
179
180 if (__predict_false(KERNEL_PANICKED() || kdb_active || kmsan_reporting))
181 return;
182 if (__predict_false(curthread != NULL &&
183 (curthread->td_pflags2 & TDP2_SAN_QUIET) != 0))
184 return;
185
186 kmsan_reporting = true;
187 __compiler_membar();
188
189 if (*orig == 0) {
190 REPORT("MSan: Uninitialized memory in %s, offset %zu",
191 hook, off);
192 goto out;
193 }
194
195 kmsan_md_orig_decode(*orig, &type, &ptr);
196 typename = kmsan_orig_name(type);
197
198 if (linker_ddb_search_symbol_name((caddr_t)ptr, buf,
199 sizeof(buf), &foff) == 0) {
200 REPORT("MSan: Uninitialized %s memory in %s, "
201 "offset %zu/%zu, addr %p, from %s+%#lx",
202 typename, hook, off, size, addr, buf, foff);
203 } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) {
204 /*
205 * The format of the string is: "----var@function". Parse it to
206 * display a nice warning.
207 */
208 var = (char *)ptr + 4;
209 strlcpy(buf, var, sizeof(buf));
210 var = buf;
211 fn = strchr(buf, '@');
212 *fn++ = '\0';
213 REPORT("MSan: Uninitialized %s memory in %s, offset %zu, "
214 "variable '%s' from %s", typename, hook, off, var, fn);
215 } else {
216 REPORT("MSan: Uninitialized %s memory in %s, "
217 "offset %zu/%zu, addr %p, PC %p",
218 typename, hook, off, size, addr, (void *)ptr);
219 }
220
221 out:
222 __compiler_membar();
223 kmsan_reporting = false;
224 }
225
226 static void
kmsan_report_inline(msan_orig_t orig,unsigned long pc)227 kmsan_report_inline(msan_orig_t orig, unsigned long pc)
228 {
229 const char *typename;
230 char *var, *fn;
231 uintptr_t ptr;
232 char buf[128];
233 long foff;
234 int type;
235
236 if (__predict_false(KERNEL_PANICKED() || kdb_active || kmsan_reporting))
237 return;
238 if (__predict_false(curthread != NULL &&
239 (curthread->td_pflags2 & TDP2_SAN_QUIET) != 0))
240 return;
241
242 kmsan_reporting = true;
243 __compiler_membar();
244
245 if (orig == 0) {
246 REPORT("MSan: uninitialized variable in %p", (void *)pc);
247 goto out;
248 }
249
250 kmsan_md_orig_decode(orig, &type, &ptr);
251 typename = kmsan_orig_name(type);
252
253 if (linker_ddb_search_symbol_name((caddr_t)ptr, buf,
254 sizeof(buf), &foff) == 0) {
255 REPORT("MSan: Uninitialized %s memory from %s+%#lx",
256 typename, buf, foff);
257 } else if (__builtin_memcmp((void *)ptr, "----", 4) == 0) {
258 /*
259 * The format of the string is: "----var@function". Parse it to
260 * display a nice warning.
261 */
262 var = (char *)ptr + 4;
263 strlcpy(buf, var, sizeof(buf));
264 var = buf;
265 fn = strchr(buf, '@');
266 *fn++ = '\0';
267 REPORT("MSan: Uninitialized variable '%s' from %s", var, fn);
268 } else {
269 REPORT("MSan: Uninitialized %s memory, origin %x",
270 typename, orig);
271 }
272
273 out:
274 __compiler_membar();
275 kmsan_reporting = false;
276 }
277
278 /* -------------------------------------------------------------------------- */
279
280 static inline msan_meta_t
kmsan_meta_get(const void * addr,size_t size,const bool write)281 kmsan_meta_get(const void *addr, size_t size, const bool write)
282 {
283 msan_meta_t ret;
284
285 if (__predict_false(!kmsan_enabled)) {
286 ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad;
287 ret.orig = (msan_orig_t *)msan_dummy_orig;
288 } else if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr))) {
289 ret.shad = write ? msan_dummy_write_shad : msan_dummy_shad;
290 ret.orig = (msan_orig_t *)msan_dummy_orig;
291 } else {
292 ret.shad = (void *)kmsan_md_addr_to_shad((vm_offset_t)addr);
293 ret.orig =
294 (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr);
295 ret.orig = (msan_orig_t *)((uintptr_t)ret.orig &
296 MSAN_ORIG_MASK);
297 }
298
299 return (ret);
300 }
301
302 static inline void
kmsan_origin_fill(const void * addr,msan_orig_t o,size_t size)303 kmsan_origin_fill(const void *addr, msan_orig_t o, size_t size)
304 {
305 msan_orig_t *orig;
306 size_t i;
307
308 if (__predict_false(!kmsan_enabled))
309 return;
310 if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr)))
311 return;
312
313 orig = (msan_orig_t *)kmsan_md_addr_to_orig((vm_offset_t)addr);
314 size += ((uintptr_t)orig & (sizeof(*orig) - 1));
315 orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK);
316
317 for (i = 0; i < size; i += 4) {
318 orig[i / 4] = o;
319 }
320 }
321
322 static inline void
kmsan_shadow_fill(uintptr_t addr,uint8_t c,size_t size)323 kmsan_shadow_fill(uintptr_t addr, uint8_t c, size_t size)
324 {
325 uint8_t *shad;
326
327 if (__predict_false(!kmsan_enabled))
328 return;
329 if (__predict_false(kmsan_md_unsupported(addr)))
330 return;
331
332 shad = (uint8_t *)kmsan_md_addr_to_shad(addr);
333 __builtin_memset(shad, c, size);
334 }
335
336 static inline void
kmsan_meta_copy(void * dst,const void * src,size_t size)337 kmsan_meta_copy(void *dst, const void *src, size_t size)
338 {
339 uint8_t *orig_src, *orig_dst;
340 uint8_t *shad_src, *shad_dst;
341 msan_orig_t *_src, *_dst;
342 size_t i;
343
344 if (__predict_false(!kmsan_enabled))
345 return;
346 if (__predict_false(kmsan_md_unsupported((vm_offset_t)dst)))
347 return;
348 if (__predict_false(kmsan_md_unsupported((vm_offset_t)src))) {
349 kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, size);
350 return;
351 }
352
353 shad_src = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)src);
354 shad_dst = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)dst);
355 __builtin_memmove(shad_dst, shad_src, size);
356
357 orig_src = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)src);
358 orig_dst = (uint8_t *)kmsan_md_addr_to_orig((vm_offset_t)dst);
359 for (i = 0; i < size; i++) {
360 _src = (msan_orig_t *)((uintptr_t)orig_src & MSAN_ORIG_MASK);
361 _dst = (msan_orig_t *)((uintptr_t)orig_dst & MSAN_ORIG_MASK);
362 *_dst = *_src;
363 orig_src++;
364 orig_dst++;
365 }
366 }
367
368 static inline void
kmsan_shadow_check(uintptr_t addr,size_t size,const char * hook)369 kmsan_shadow_check(uintptr_t addr, size_t size, const char *hook)
370 {
371 msan_orig_t *orig;
372 uint8_t *shad;
373 size_t i;
374
375 if (__predict_false(!kmsan_enabled))
376 return;
377 if (__predict_false(kmsan_md_unsupported(addr)))
378 return;
379
380 shad = (uint8_t *)kmsan_md_addr_to_shad(addr);
381 for (i = 0; i < size; i++) {
382 if (__predict_true(shad[i] == 0))
383 continue;
384 orig = (msan_orig_t *)kmsan_md_addr_to_orig(addr + i);
385 orig = (msan_orig_t *)((uintptr_t)orig & MSAN_ORIG_MASK);
386 kmsan_report_hook((const char *)addr + i, orig, size, i, hook);
387 break;
388 }
389 }
390
391 void
kmsan_init_arg(size_t n)392 kmsan_init_arg(size_t n)
393 {
394 msan_td_t *mtd;
395 uint8_t *arg;
396
397 if (__predict_false(!kmsan_enabled))
398 return;
399 if (__predict_false(curthread == NULL))
400 return;
401 mtd = curthread->td_kmsan;
402 arg = mtd->tls[mtd->ctx].param_shadow;
403 __builtin_memset(arg, 0, n);
404 }
405
406 void
kmsan_init_ret(size_t n)407 kmsan_init_ret(size_t n)
408 {
409 msan_td_t *mtd;
410 uint8_t *arg;
411
412 if (__predict_false(!kmsan_enabled))
413 return;
414 if (__predict_false(curthread == NULL))
415 return;
416 mtd = curthread->td_kmsan;
417 arg = mtd->tls[mtd->ctx].retval_shadow;
418 __builtin_memset(arg, 0, n);
419 }
420
421 static void
kmsan_check_arg(size_t size,const char * hook)422 kmsan_check_arg(size_t size, const char *hook)
423 {
424 msan_orig_t *orig;
425 msan_td_t *mtd;
426 uint8_t *arg;
427 size_t ctx, i;
428
429 if (__predict_false(!kmsan_enabled))
430 return;
431 if (__predict_false(curthread == NULL))
432 return;
433 mtd = curthread->td_kmsan;
434 ctx = mtd->ctx;
435 arg = mtd->tls[ctx].param_shadow;
436
437 for (i = 0; i < size; i++) {
438 if (__predict_true(arg[i] == 0))
439 continue;
440 orig = &mtd->tls[ctx].param_origin[i / sizeof(msan_orig_t)];
441 kmsan_report_hook((const char *)arg + i, orig, size, i, hook);
442 break;
443 }
444 }
445
446 void
kmsan_thread_alloc(struct thread * td)447 kmsan_thread_alloc(struct thread *td)
448 {
449 msan_td_t *mtd;
450
451 if (!kmsan_enabled)
452 return;
453
454 mtd = td->td_kmsan;
455 if (mtd == NULL) {
456 /* We might be recycling a thread. */
457 kmsan_init_arg(sizeof(size_t) + sizeof(struct malloc_type *) +
458 sizeof(int));
459 mtd = malloc(sizeof(*mtd), M_KMSAN, M_WAITOK);
460 }
461 __builtin_memset(mtd, 0, sizeof(*mtd));
462 mtd->ctx = 0;
463
464 if (td->td_kstack != 0)
465 kmsan_mark((void *)td->td_kstack, ptoa(td->td_kstack_pages),
466 KMSAN_STATE_UNINIT);
467
468 td->td_kmsan = mtd;
469 }
470
471 void
kmsan_thread_free(struct thread * td)472 kmsan_thread_free(struct thread *td)
473 {
474 msan_td_t *mtd;
475
476 if (!kmsan_enabled)
477 return;
478 if (__predict_false(td == curthread))
479 kmsan_panic("%s: freeing KMSAN TLS for curthread", __func__);
480
481 mtd = td->td_kmsan;
482 kmsan_init_arg(sizeof(void *) + sizeof(struct malloc_type *));
483 free(mtd, M_KMSAN);
484 td->td_kmsan = NULL;
485 }
486
487 void kmsan_intr_enter(void);
488 void kmsan_intr_leave(void);
489
490 void
kmsan_intr_enter(void)491 kmsan_intr_enter(void)
492 {
493 msan_td_t *mtd;
494
495 if (__predict_false(!kmsan_enabled))
496 return;
497
498 mtd = curthread->td_kmsan;
499 mtd->ctx++;
500 if (__predict_false(mtd->ctx >= MSAN_NCONTEXT))
501 kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx);
502 }
503
504 void
kmsan_intr_leave(void)505 kmsan_intr_leave(void)
506 {
507 msan_td_t *mtd;
508
509 if (__predict_false(!kmsan_enabled))
510 return;
511
512 mtd = curthread->td_kmsan;
513 if (__predict_false(mtd->ctx == 0))
514 kmsan_panic("%s: mtd->ctx = %zu", __func__, mtd->ctx);
515 mtd->ctx--;
516 }
517
518 /* -------------------------------------------------------------------------- */
519
520 void
kmsan_shadow_map(vm_offset_t addr,size_t size)521 kmsan_shadow_map(vm_offset_t addr, size_t size)
522 {
523 size_t npages, i;
524 vm_offset_t va;
525
526 MPASS(addr % PAGE_SIZE == 0);
527 MPASS(size % PAGE_SIZE == 0);
528
529 if (!kmsan_enabled)
530 return;
531
532 npages = atop(size);
533
534 va = kmsan_md_addr_to_shad(addr);
535 for (i = 0; i < npages; i++) {
536 pmap_san_enter(va + ptoa(i));
537 }
538
539 va = kmsan_md_addr_to_orig(addr);
540 for (i = 0; i < npages; i++) {
541 pmap_san_enter(va + ptoa(i));
542 }
543 }
544
545 void
kmsan_orig(const void * addr,size_t size,int type,uintptr_t pc)546 kmsan_orig(const void *addr, size_t size, int type, uintptr_t pc)
547 {
548 msan_orig_t orig;
549
550 orig = kmsan_md_orig_encode(type, pc);
551 kmsan_origin_fill(addr, orig, size);
552 }
553
554 void
kmsan_mark(const void * addr,size_t size,uint8_t c)555 kmsan_mark(const void *addr, size_t size, uint8_t c)
556 {
557 kmsan_shadow_fill((uintptr_t)addr, c, size);
558 }
559
560 void
kmsan_mark_bio(const struct bio * bp,uint8_t c)561 kmsan_mark_bio(const struct bio *bp, uint8_t c)
562 {
563 kmsan_mark(bp->bio_data, bp->bio_length, c);
564 }
565
566 void
kmsan_mark_mbuf(const struct mbuf * m,uint8_t c)567 kmsan_mark_mbuf(const struct mbuf *m, uint8_t c)
568 {
569 do {
570 if ((m->m_flags & M_EXTPG) == 0)
571 kmsan_mark(m->m_data, m->m_len, c);
572 m = m->m_next;
573 } while (m != NULL);
574 }
575
576 void
kmsan_check(const void * p,size_t sz,const char * descr)577 kmsan_check(const void *p, size_t sz, const char *descr)
578 {
579 kmsan_shadow_check((uintptr_t)p, sz, descr);
580 }
581
582 void
kmsan_check_bio(const struct bio * bp,const char * descr)583 kmsan_check_bio(const struct bio *bp, const char *descr)
584 {
585 kmsan_shadow_check((uintptr_t)bp->bio_data, bp->bio_length, descr);
586 }
587
588 void
kmsan_check_mbuf(const struct mbuf * m,const char * descr)589 kmsan_check_mbuf(const struct mbuf *m, const char *descr)
590 {
591 do {
592 kmsan_shadow_check((uintptr_t)mtod(m, void *), m->m_len, descr);
593 } while ((m = m->m_next) != NULL);
594 }
595
596 void
kmsan_check_uio(const struct uio * uio,const char * descr)597 kmsan_check_uio(const struct uio *uio, const char *descr)
598 {
599 for (int i = 0; i < uio->uio_iovcnt; i++) {
600 kmsan_check(uio->uio_iov[i].iov_base, uio->uio_iov[i].iov_len,
601 descr);
602 }
603 }
604
605 void
kmsan_init(void)606 kmsan_init(void)
607 {
608 int disabled;
609
610 disabled = 0;
611 TUNABLE_INT_FETCH("debug.kmsan.disabled", &disabled);
612 if (disabled)
613 return;
614
615 /* Initialize the TLS for curthread. */
616 msan_thread0.ctx = 0;
617 thread0.td_kmsan = &msan_thread0;
618
619 /* Now officially enabled. */
620 kmsan_disabled = false;
621 }
622
623 /* -------------------------------------------------------------------------- */
624
625 msan_meta_t __msan_metadata_ptr_for_load_n(void *, size_t);
626 msan_meta_t __msan_metadata_ptr_for_store_n(void *, size_t);
627
628 msan_meta_t
__msan_metadata_ptr_for_load_n(void * addr,size_t size)629 __msan_metadata_ptr_for_load_n(void *addr, size_t size)
630 {
631 return (kmsan_meta_get(addr, size, false));
632 }
633
634 msan_meta_t
__msan_metadata_ptr_for_store_n(void * addr,size_t size)635 __msan_metadata_ptr_for_store_n(void *addr, size_t size)
636 {
637 return (kmsan_meta_get(addr, size, true));
638 }
639
640 #define MSAN_META_FUNC(size) \
641 msan_meta_t __msan_metadata_ptr_for_load_##size(void *); \
642 msan_meta_t __msan_metadata_ptr_for_load_##size(void *addr) \
643 { \
644 return (kmsan_meta_get(addr, size, false)); \
645 } \
646 msan_meta_t __msan_metadata_ptr_for_store_##size(void *); \
647 msan_meta_t __msan_metadata_ptr_for_store_##size(void *addr) \
648 { \
649 return (kmsan_meta_get(addr, size, true)); \
650 }
651
652 MSAN_META_FUNC(1)
653 MSAN_META_FUNC(2)
654 MSAN_META_FUNC(4)
655 MSAN_META_FUNC(8)
656
657 void __msan_instrument_asm_store(const void *, size_t);
658 msan_orig_t __msan_chain_origin(msan_orig_t);
659 void __msan_poison(const void *, size_t);
660 void __msan_unpoison(const void *, size_t);
661 void __msan_poison_alloca(const void *, uint64_t, const char *);
662 void __msan_unpoison_alloca(const void *, uint64_t);
663 void __msan_warning(msan_orig_t);
664 msan_tls_t *__msan_get_context_state(void);
665 intptr_t __msan_test_shadow(const void *, size_t);
666
667 void
__msan_instrument_asm_store(const void * addr,size_t size)668 __msan_instrument_asm_store(const void *addr, size_t size)
669 {
670 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size);
671 }
672
673 msan_orig_t
__msan_chain_origin(msan_orig_t origin)674 __msan_chain_origin(msan_orig_t origin)
675 {
676 return (origin);
677 }
678
679 void
__msan_poison(const void * addr,size_t size)680 __msan_poison(const void *addr, size_t size)
681 {
682 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size);
683 }
684
685 void
__msan_unpoison(const void * addr,size_t size)686 __msan_unpoison(const void *addr, size_t size)
687 {
688 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size);
689 }
690
691 void
__msan_poison_alloca(const void * addr,uint64_t size,const char * descr)692 __msan_poison_alloca(const void *addr, uint64_t size, const char *descr)
693 {
694 msan_orig_t orig;
695
696 orig = kmsan_md_orig_encode(KMSAN_TYPE_STACK, (uintptr_t)descr);
697 kmsan_origin_fill(addr, orig, size);
698 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_UNINIT, size);
699 }
700
701 void
__msan_unpoison_alloca(const void * addr,uint64_t size)702 __msan_unpoison_alloca(const void *addr, uint64_t size)
703 {
704 kmsan_shadow_fill((uintptr_t)addr, KMSAN_STATE_INITED, size);
705 }
706
707 void
__msan_warning(msan_orig_t origin)708 __msan_warning(msan_orig_t origin)
709 {
710 if (__predict_false(!kmsan_enabled))
711 return;
712 kmsan_report_inline(origin, KMSAN_RET_ADDR);
713 }
714
715 msan_tls_t *
__msan_get_context_state(void)716 __msan_get_context_state(void)
717 {
718 msan_td_t *mtd;
719
720 /*
721 * When APs are started, they execute some C code before curthread is
722 * set. We have to handle that here.
723 */
724 if (__predict_false(!kmsan_enabled || curthread == NULL))
725 return (&dummy_tls);
726 mtd = curthread->td_kmsan;
727 return (&mtd->tls[mtd->ctx]);
728 }
729
730 intptr_t
__msan_test_shadow(const void * addr,size_t len)731 __msan_test_shadow(const void *addr, size_t len)
732 {
733 uint8_t *shad;
734
735 if (__predict_false(!kmsan_enabled))
736 return (-1);
737 if (__predict_false(kmsan_md_unsupported((vm_offset_t)addr)))
738 return (-1);
739
740 shad = (uint8_t *)kmsan_md_addr_to_shad((vm_offset_t)addr);
741 for (size_t i = 0; i < len; i++)
742 if (shad[i] != 0)
743 return (i);
744 return (-1);
745 }
746
747 /* -------------------------------------------------------------------------- */
748
749 /*
750 * Function hooks. Mostly ASM functions which need KMSAN wrappers to handle
751 * initialized areas properly.
752 */
753
754 void *
kmsan_memcpy(void * dst,const void * src,size_t len)755 kmsan_memcpy(void *dst, const void *src, size_t len)
756 {
757 /* No kmsan_check_arg, because inlined. */
758 kmsan_init_ret(sizeof(void *));
759 if (__predict_true(len != 0)) {
760 kmsan_meta_copy(dst, src, len);
761 }
762 return (__builtin_memcpy(dst, src, len));
763 }
764
765 int
kmsan_memcmp(const void * b1,const void * b2,size_t len)766 kmsan_memcmp(const void *b1, const void *b2, size_t len)
767 {
768 const uint8_t *_b1 = b1, *_b2 = b2;
769 size_t i;
770
771 kmsan_check_arg(sizeof(b1) + sizeof(b2) + sizeof(len),
772 "memcmp():args");
773 kmsan_init_ret(sizeof(int));
774
775 for (i = 0; i < len; i++) {
776 if (*_b1 != *_b2) {
777 kmsan_shadow_check((uintptr_t)b1, i + 1,
778 "memcmp():arg1");
779 kmsan_shadow_check((uintptr_t)b2, i + 1,
780 "memcmp():arg2");
781 return (*_b1 - *_b2);
782 }
783 _b1++, _b2++;
784 }
785
786 return (0);
787 }
788
789 void *
kmsan_memset(void * dst,int c,size_t len)790 kmsan_memset(void *dst, int c, size_t len)
791 {
792 /* No kmsan_check_arg, because inlined. */
793 kmsan_shadow_fill((uintptr_t)dst, KMSAN_STATE_INITED, len);
794 kmsan_init_ret(sizeof(void *));
795 return (__builtin_memset(dst, c, len));
796 }
797
798 void *
kmsan_memmove(void * dst,const void * src,size_t len)799 kmsan_memmove(void *dst, const void *src, size_t len)
800 {
801 /* No kmsan_check_arg, because inlined. */
802 if (__predict_true(len != 0)) {
803 kmsan_meta_copy(dst, src, len);
804 }
805 kmsan_init_ret(sizeof(void *));
806 return (__builtin_memmove(dst, src, len));
807 }
808
809 __strong_reference(kmsan_memcpy, __msan_memcpy);
810 __strong_reference(kmsan_memset, __msan_memset);
811 __strong_reference(kmsan_memmove, __msan_memmove);
812
813 char *
kmsan_strcpy(char * dst,const char * src)814 kmsan_strcpy(char *dst, const char *src)
815 {
816 const char *_src = src;
817 char *_dst = dst;
818 size_t len = 0;
819
820 kmsan_check_arg(sizeof(dst) + sizeof(src), "strcpy():args");
821
822 while (1) {
823 len++;
824 *dst = *src;
825 if (*src == '\0')
826 break;
827 src++, dst++;
828 }
829
830 kmsan_shadow_check((uintptr_t)_src, len, "strcpy():arg2");
831 kmsan_shadow_fill((uintptr_t)_dst, KMSAN_STATE_INITED, len);
832 kmsan_init_ret(sizeof(char *));
833 return (_dst);
834 }
835
836 int
kmsan_strcmp(const char * s1,const char * s2)837 kmsan_strcmp(const char *s1, const char *s2)
838 {
839 const char *_s1 = s1, *_s2 = s2;
840 size_t len = 0;
841
842 kmsan_check_arg(sizeof(s1) + sizeof(s2), "strcmp():args");
843 kmsan_init_ret(sizeof(int));
844
845 while (1) {
846 len++;
847 if (*s1 != *s2)
848 break;
849 if (*s1 == '\0') {
850 kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1");
851 kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2");
852 return (0);
853 }
854 s1++, s2++;
855 }
856
857 kmsan_shadow_check((uintptr_t)_s1, len, "strcmp():arg1");
858 kmsan_shadow_check((uintptr_t)_s2, len, "strcmp():arg2");
859
860 return (*(const unsigned char *)s1 - *(const unsigned char *)s2);
861 }
862
863 size_t
kmsan_strlen(const char * str)864 kmsan_strlen(const char *str)
865 {
866 const char *s;
867
868 kmsan_check_arg(sizeof(str), "strlen():args");
869
870 s = str;
871 while (1) {
872 if (*s == '\0')
873 break;
874 s++;
875 }
876
877 kmsan_shadow_check((uintptr_t)str, (size_t)(s - str) + 1, "strlen():arg1");
878 kmsan_init_ret(sizeof(size_t));
879 return (s - str);
880 }
881
882 int kmsan_copyin(const void *, void *, size_t);
883 int kmsan_copyout(const void *, void *, size_t);
884 int kmsan_copyinstr(const void *, void *, size_t, size_t *);
885
886 int
kmsan_copyin(const void * uaddr,void * kaddr,size_t len)887 kmsan_copyin(const void *uaddr, void *kaddr, size_t len)
888 {
889 int ret;
890
891 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) + sizeof(len),
892 "copyin():args");
893 ret = copyin(uaddr, kaddr, len);
894 if (ret == 0)
895 kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, len);
896 kmsan_init_ret(sizeof(int));
897 return (ret);
898 }
899
900 int
kmsan_copyout(const void * kaddr,void * uaddr,size_t len)901 kmsan_copyout(const void *kaddr, void *uaddr, size_t len)
902 {
903 kmsan_check_arg(sizeof(kaddr) + sizeof(uaddr) + sizeof(len),
904 "copyout():args");
905 kmsan_shadow_check((uintptr_t)kaddr, len, "copyout():arg1");
906 kmsan_init_ret(sizeof(int));
907 return (copyout(kaddr, uaddr, len));
908 }
909
910 int
kmsan_copyinstr(const void * uaddr,void * kaddr,size_t len,size_t * done)911 kmsan_copyinstr(const void *uaddr, void *kaddr, size_t len, size_t *done)
912 {
913 size_t _done;
914 int ret;
915
916 kmsan_check_arg(sizeof(uaddr) + sizeof(kaddr) +
917 sizeof(len) + sizeof(done), "copyinstr():args");
918 ret = copyinstr(uaddr, kaddr, len, &_done);
919 if (ret == 0)
920 kmsan_shadow_fill((uintptr_t)kaddr, KMSAN_STATE_INITED, _done);
921 if (done != NULL) {
922 *done = _done;
923 kmsan_shadow_fill((uintptr_t)done, KMSAN_STATE_INITED, sizeof(size_t));
924 }
925 kmsan_init_ret(sizeof(int));
926 return (ret);
927 }
928
929 /* -------------------------------------------------------------------------- */
930
931 int
kmsan_fubyte(volatile const void * base)932 kmsan_fubyte(volatile const void *base)
933 {
934 int ret;
935
936 kmsan_check_arg(sizeof(base), "fubyte(): args");
937 ret = fubyte(base);
938 kmsan_init_ret(sizeof(int));
939 return (ret);
940 }
941
942 int
kmsan_fuword16(volatile const void * base)943 kmsan_fuword16(volatile const void *base)
944 {
945 int ret;
946
947 kmsan_check_arg(sizeof(base), "fuword16(): args");
948 ret = fuword16(base);
949 kmsan_init_ret(sizeof(int));
950 return (ret);
951 }
952
953 int
kmsan_fueword(volatile const void * base,long * val)954 kmsan_fueword(volatile const void *base, long *val)
955 {
956 int ret;
957
958 kmsan_check_arg(sizeof(base) + sizeof(val), "fueword(): args");
959 ret = fueword(base, val);
960 if (ret == 0)
961 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED,
962 sizeof(*val));
963 kmsan_init_ret(sizeof(int));
964 return (ret);
965 }
966
967 int
kmsan_fueword32(volatile const void * base,int32_t * val)968 kmsan_fueword32(volatile const void *base, int32_t *val)
969 {
970 int ret;
971
972 kmsan_check_arg(sizeof(base) + sizeof(val), "fueword32(): args");
973 ret = fueword32(base, val);
974 if (ret == 0)
975 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED,
976 sizeof(*val));
977 kmsan_init_ret(sizeof(int));
978 return (ret);
979 }
980
981 int
kmsan_fueword64(volatile const void * base,int64_t * val)982 kmsan_fueword64(volatile const void *base, int64_t *val)
983 {
984 int ret;
985
986 kmsan_check_arg(sizeof(base) + sizeof(val), "fueword64(): args");
987 ret = fueword64(base, val);
988 if (ret == 0)
989 kmsan_shadow_fill((uintptr_t)val, KMSAN_STATE_INITED,
990 sizeof(*val));
991 kmsan_init_ret(sizeof(int));
992 return (ret);
993 }
994
995 int
kmsan_subyte(volatile void * base,int byte)996 kmsan_subyte(volatile void *base, int byte)
997 {
998 int ret;
999
1000 kmsan_check_arg(sizeof(base) + sizeof(byte), "subyte():args");
1001 ret = subyte(base, byte);
1002 kmsan_init_ret(sizeof(int));
1003 return (ret);
1004 }
1005
1006 int
kmsan_suword(volatile void * base,long word)1007 kmsan_suword(volatile void *base, long word)
1008 {
1009 int ret;
1010
1011 kmsan_check_arg(sizeof(base) + sizeof(word), "suword():args");
1012 ret = suword(base, word);
1013 kmsan_init_ret(sizeof(int));
1014 return (ret);
1015 }
1016
1017 int
kmsan_suword16(volatile void * base,int word)1018 kmsan_suword16(volatile void *base, int word)
1019 {
1020 int ret;
1021
1022 kmsan_check_arg(sizeof(base) + sizeof(word), "suword16():args");
1023 ret = suword16(base, word);
1024 kmsan_init_ret(sizeof(int));
1025 return (ret);
1026 }
1027
1028 int
kmsan_suword32(volatile void * base,int32_t word)1029 kmsan_suword32(volatile void *base, int32_t word)
1030 {
1031 int ret;
1032
1033 kmsan_check_arg(sizeof(base) + sizeof(word), "suword32():args");
1034 ret = suword32(base, word);
1035 kmsan_init_ret(sizeof(int));
1036 return (ret);
1037 }
1038
1039 int
kmsan_suword64(volatile void * base,int64_t word)1040 kmsan_suword64(volatile void *base, int64_t word)
1041 {
1042 int ret;
1043
1044 kmsan_check_arg(sizeof(base) + sizeof(word), "suword64():args");
1045 ret = suword64(base, word);
1046 kmsan_init_ret(sizeof(int));
1047 return (ret);
1048 }
1049
1050 int
kmsan_casueword32(volatile uint32_t * base,uint32_t oldval,uint32_t * oldvalp,uint32_t newval)1051 kmsan_casueword32(volatile uint32_t *base, uint32_t oldval, uint32_t *oldvalp,
1052 uint32_t newval)
1053 {
1054 int ret;
1055
1056 kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) +
1057 sizeof(newval), "casueword32(): args");
1058 ret = casueword32(base, oldval, oldvalp, newval);
1059 kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED,
1060 sizeof(*oldvalp));
1061 kmsan_init_ret(sizeof(int));
1062 return (ret);
1063 }
1064
1065 int
kmsan_casueword(volatile u_long * base,u_long oldval,u_long * oldvalp,u_long newval)1066 kmsan_casueword(volatile u_long *base, u_long oldval, u_long *oldvalp,
1067 u_long newval)
1068 {
1069 int ret;
1070
1071 kmsan_check_arg(sizeof(base) + sizeof(oldval) + sizeof(oldvalp) +
1072 sizeof(newval), "casueword32(): args");
1073 ret = casueword(base, oldval, oldvalp, newval);
1074 kmsan_shadow_fill((uintptr_t)oldvalp, KMSAN_STATE_INITED,
1075 sizeof(*oldvalp));
1076 kmsan_init_ret(sizeof(int));
1077 return (ret);
1078 }
1079
1080 /* -------------------------------------------------------------------------- */
1081
1082 #include <machine/atomic.h>
1083 #include <sys/atomic_san.h>
1084
1085 #define _MSAN_ATOMIC_FUNC_ADD(name, type) \
1086 void kmsan_atomic_add_##name(volatile type *ptr, type val) \
1087 { \
1088 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1089 "atomic_add_" #name "():args"); \
1090 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1091 "atomic_add_" #name "():ptr"); \
1092 atomic_add_##name(ptr, val); \
1093 }
1094
1095 #define MSAN_ATOMIC_FUNC_ADD(name, type) \
1096 _MSAN_ATOMIC_FUNC_ADD(name, type) \
1097 _MSAN_ATOMIC_FUNC_ADD(acq_##name, type) \
1098 _MSAN_ATOMIC_FUNC_ADD(rel_##name, type)
1099
1100 #define _MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \
1101 void kmsan_atomic_subtract_##name(volatile type *ptr, type val) \
1102 { \
1103 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1104 "atomic_subtract_" #name "():args"); \
1105 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1106 "atomic_subtract_" #name "():ptr"); \
1107 atomic_subtract_##name(ptr, val); \
1108 }
1109
1110 #define MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \
1111 _MSAN_ATOMIC_FUNC_SUBTRACT(name, type) \
1112 _MSAN_ATOMIC_FUNC_SUBTRACT(acq_##name, type) \
1113 _MSAN_ATOMIC_FUNC_SUBTRACT(rel_##name, type)
1114
1115 #define _MSAN_ATOMIC_FUNC_SET(name, type) \
1116 void kmsan_atomic_set_##name(volatile type *ptr, type val) \
1117 { \
1118 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1119 "atomic_set_" #name "():args"); \
1120 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1121 "atomic_set_" #name "():ptr"); \
1122 atomic_set_##name(ptr, val); \
1123 }
1124
1125 #define MSAN_ATOMIC_FUNC_SET(name, type) \
1126 _MSAN_ATOMIC_FUNC_SET(name, type) \
1127 _MSAN_ATOMIC_FUNC_SET(acq_##name, type) \
1128 _MSAN_ATOMIC_FUNC_SET(rel_##name, type)
1129
1130 #define _MSAN_ATOMIC_FUNC_CLEAR(name, type) \
1131 void kmsan_atomic_clear_##name(volatile type *ptr, type val) \
1132 { \
1133 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1134 "atomic_clear_" #name "():args"); \
1135 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1136 "atomic_clear_" #name "():ptr"); \
1137 atomic_clear_##name(ptr, val); \
1138 }
1139
1140 #define MSAN_ATOMIC_FUNC_CLEAR(name, type) \
1141 _MSAN_ATOMIC_FUNC_CLEAR(name, type) \
1142 _MSAN_ATOMIC_FUNC_CLEAR(acq_##name, type) \
1143 _MSAN_ATOMIC_FUNC_CLEAR(rel_##name, type)
1144
1145 #define MSAN_ATOMIC_FUNC_FETCHADD(name, type) \
1146 type kmsan_atomic_fetchadd_##name(volatile type *ptr, type val) \
1147 { \
1148 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1149 "atomic_fetchadd_" #name "():args"); \
1150 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1151 "atomic_fetchadd_" #name "():ptr"); \
1152 kmsan_init_ret(sizeof(type)); \
1153 return (atomic_fetchadd_##name(ptr, val)); \
1154 }
1155
1156 #define MSAN_ATOMIC_FUNC_READANDCLEAR(name, type) \
1157 type kmsan_atomic_readandclear_##name(volatile type *ptr) \
1158 { \
1159 kmsan_check_arg(sizeof(ptr), \
1160 "atomic_readandclear_" #name "():args"); \
1161 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1162 "atomic_readandclear_" #name "():ptr"); \
1163 kmsan_init_ret(sizeof(type)); \
1164 return (atomic_readandclear_##name(ptr)); \
1165 }
1166
1167 #define MSAN_ATOMIC_FUNC_TESTANDCLEAR(name, type) \
1168 int kmsan_atomic_testandclear_##name(volatile type *ptr, u_int v) \
1169 { \
1170 kmsan_check_arg(sizeof(ptr) + sizeof(v), \
1171 "atomic_testandclear_" #name "():args"); \
1172 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1173 "atomic_testandclear_" #name "():ptr"); \
1174 kmsan_init_ret(sizeof(int)); \
1175 return (atomic_testandclear_##name(ptr, v)); \
1176 }
1177
1178 #define MSAN_ATOMIC_FUNC_TESTANDSET(name, type) \
1179 int kmsan_atomic_testandset_##name(volatile type *ptr, u_int v) \
1180 { \
1181 kmsan_check_arg(sizeof(ptr) + sizeof(v), \
1182 "atomic_testandset_" #name "():args"); \
1183 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1184 "atomic_testandset_" #name "():ptr"); \
1185 kmsan_init_ret(sizeof(int)); \
1186 return (atomic_testandset_##name(ptr, v)); \
1187 }
1188
1189 #define MSAN_ATOMIC_FUNC_SWAP(name, type) \
1190 type kmsan_atomic_swap_##name(volatile type *ptr, type val) \
1191 { \
1192 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1193 "atomic_swap_" #name "():args"); \
1194 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1195 "atomic_swap_" #name "():ptr"); \
1196 kmsan_init_ret(sizeof(type)); \
1197 return (atomic_swap_##name(ptr, val)); \
1198 }
1199
1200 #define _MSAN_ATOMIC_FUNC_CMPSET(name, type) \
1201 int kmsan_atomic_cmpset_##name(volatile type *ptr, type oval, \
1202 type nval) \
1203 { \
1204 kmsan_check_arg(sizeof(ptr) + sizeof(oval) + \
1205 sizeof(nval), "atomic_cmpset_" #name "():args"); \
1206 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1207 "atomic_cmpset_" #name "():ptr"); \
1208 kmsan_init_ret(sizeof(int)); \
1209 return (atomic_cmpset_##name(ptr, oval, nval)); \
1210 }
1211
1212 #define MSAN_ATOMIC_FUNC_CMPSET(name, type) \
1213 _MSAN_ATOMIC_FUNC_CMPSET(name, type) \
1214 _MSAN_ATOMIC_FUNC_CMPSET(acq_##name, type) \
1215 _MSAN_ATOMIC_FUNC_CMPSET(rel_##name, type)
1216
1217 #define _MSAN_ATOMIC_FUNC_FCMPSET(name, type) \
1218 int kmsan_atomic_fcmpset_##name(volatile type *ptr, type *oval, \
1219 type nval) \
1220 { \
1221 kmsan_check_arg(sizeof(ptr) + sizeof(oval) + \
1222 sizeof(nval), "atomic_fcmpset_" #name "():args"); \
1223 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1224 "atomic_fcmpset_" #name "():ptr"); \
1225 kmsan_init_ret(sizeof(int)); \
1226 return (atomic_fcmpset_##name(ptr, oval, nval)); \
1227 }
1228
1229 #define MSAN_ATOMIC_FUNC_FCMPSET(name, type) \
1230 _MSAN_ATOMIC_FUNC_FCMPSET(name, type) \
1231 _MSAN_ATOMIC_FUNC_FCMPSET(acq_##name, type) \
1232 _MSAN_ATOMIC_FUNC_FCMPSET(rel_##name, type)
1233
1234 #define MSAN_ATOMIC_FUNC_THREAD_FENCE(name) \
1235 void kmsan_atomic_thread_fence_##name(void) \
1236 { \
1237 atomic_thread_fence_##name(); \
1238 }
1239
1240 #define _MSAN_ATOMIC_FUNC_LOAD(name, type) \
1241 type kmsan_atomic_load_##name(const volatile type *ptr) \
1242 { \
1243 kmsan_check_arg(sizeof(ptr), \
1244 "atomic_load_" #name "():args"); \
1245 kmsan_shadow_check((uintptr_t)ptr, sizeof(type), \
1246 "atomic_load_" #name "():ptr"); \
1247 kmsan_init_ret(sizeof(type)); \
1248 return (atomic_load_##name(ptr)); \
1249 }
1250
1251 #define MSAN_ATOMIC_FUNC_LOAD(name, type) \
1252 _MSAN_ATOMIC_FUNC_LOAD(name, type) \
1253 _MSAN_ATOMIC_FUNC_LOAD(acq_##name, type)
1254
1255 #define _MSAN_ATOMIC_FUNC_STORE(name, type) \
1256 void kmsan_atomic_store_##name(volatile type *ptr, type val) \
1257 { \
1258 kmsan_check_arg(sizeof(ptr) + sizeof(val), \
1259 "atomic_store_" #name "():args"); \
1260 kmsan_shadow_fill((uintptr_t)ptr, KMSAN_STATE_INITED, \
1261 sizeof(type)); \
1262 atomic_store_##name(ptr, val); \
1263 }
1264
1265 #define MSAN_ATOMIC_FUNC_STORE(name, type) \
1266 _MSAN_ATOMIC_FUNC_STORE(name, type) \
1267 _MSAN_ATOMIC_FUNC_STORE(rel_##name, type)
1268
1269 MSAN_ATOMIC_FUNC_ADD(8, uint8_t);
1270 MSAN_ATOMIC_FUNC_ADD(16, uint16_t);
1271 MSAN_ATOMIC_FUNC_ADD(32, uint32_t);
1272 MSAN_ATOMIC_FUNC_ADD(64, uint64_t);
1273 MSAN_ATOMIC_FUNC_ADD(int, u_int);
1274 MSAN_ATOMIC_FUNC_ADD(long, u_long);
1275 MSAN_ATOMIC_FUNC_ADD(ptr, uintptr_t);
1276
1277 MSAN_ATOMIC_FUNC_SUBTRACT(8, uint8_t);
1278 MSAN_ATOMIC_FUNC_SUBTRACT(16, uint16_t);
1279 MSAN_ATOMIC_FUNC_SUBTRACT(32, uint32_t);
1280 MSAN_ATOMIC_FUNC_SUBTRACT(64, uint64_t);
1281 MSAN_ATOMIC_FUNC_SUBTRACT(int, u_int);
1282 MSAN_ATOMIC_FUNC_SUBTRACT(long, u_long);
1283 MSAN_ATOMIC_FUNC_SUBTRACT(ptr, uintptr_t);
1284
1285 MSAN_ATOMIC_FUNC_SET(8, uint8_t);
1286 MSAN_ATOMIC_FUNC_SET(16, uint16_t);
1287 MSAN_ATOMIC_FUNC_SET(32, uint32_t);
1288 MSAN_ATOMIC_FUNC_SET(64, uint64_t);
1289 MSAN_ATOMIC_FUNC_SET(int, u_int);
1290 MSAN_ATOMIC_FUNC_SET(long, u_long);
1291 MSAN_ATOMIC_FUNC_SET(ptr, uintptr_t);
1292
1293 MSAN_ATOMIC_FUNC_CLEAR(8, uint8_t);
1294 MSAN_ATOMIC_FUNC_CLEAR(16, uint16_t);
1295 MSAN_ATOMIC_FUNC_CLEAR(32, uint32_t);
1296 MSAN_ATOMIC_FUNC_CLEAR(64, uint64_t);
1297 MSAN_ATOMIC_FUNC_CLEAR(int, u_int);
1298 MSAN_ATOMIC_FUNC_CLEAR(long, u_long);
1299 MSAN_ATOMIC_FUNC_CLEAR(ptr, uintptr_t);
1300
1301 MSAN_ATOMIC_FUNC_FETCHADD(32, uint32_t);
1302 MSAN_ATOMIC_FUNC_FETCHADD(64, uint64_t);
1303 MSAN_ATOMIC_FUNC_FETCHADD(int, u_int);
1304 MSAN_ATOMIC_FUNC_FETCHADD(long, u_long);
1305
1306 MSAN_ATOMIC_FUNC_READANDCLEAR(32, uint32_t);
1307 MSAN_ATOMIC_FUNC_READANDCLEAR(64, uint64_t);
1308 MSAN_ATOMIC_FUNC_READANDCLEAR(int, u_int);
1309 MSAN_ATOMIC_FUNC_READANDCLEAR(long, u_long);
1310 MSAN_ATOMIC_FUNC_READANDCLEAR(ptr, uintptr_t);
1311
1312 MSAN_ATOMIC_FUNC_TESTANDCLEAR(32, uint32_t);
1313 MSAN_ATOMIC_FUNC_TESTANDCLEAR(64, uint64_t);
1314 MSAN_ATOMIC_FUNC_TESTANDCLEAR(int, u_int);
1315 MSAN_ATOMIC_FUNC_TESTANDCLEAR(long, u_long);
1316 MSAN_ATOMIC_FUNC_TESTANDCLEAR(ptr, uintptr_t);
1317
1318 MSAN_ATOMIC_FUNC_TESTANDSET(32, uint32_t);
1319 MSAN_ATOMIC_FUNC_TESTANDSET(64, uint64_t);
1320 MSAN_ATOMIC_FUNC_TESTANDSET(int, u_int);
1321 MSAN_ATOMIC_FUNC_TESTANDSET(long, u_long);
1322 MSAN_ATOMIC_FUNC_TESTANDSET(acq_long, u_long);
1323 MSAN_ATOMIC_FUNC_TESTANDSET(ptr, uintptr_t);
1324
1325 MSAN_ATOMIC_FUNC_SWAP(32, uint32_t);
1326 MSAN_ATOMIC_FUNC_SWAP(64, uint64_t);
1327 MSAN_ATOMIC_FUNC_SWAP(int, u_int);
1328 MSAN_ATOMIC_FUNC_SWAP(long, u_long);
1329 MSAN_ATOMIC_FUNC_SWAP(ptr, uintptr_t);
1330
1331 MSAN_ATOMIC_FUNC_CMPSET(8, uint8_t);
1332 MSAN_ATOMIC_FUNC_CMPSET(16, uint16_t);
1333 MSAN_ATOMIC_FUNC_CMPSET(32, uint32_t);
1334 MSAN_ATOMIC_FUNC_CMPSET(64, uint64_t);
1335 MSAN_ATOMIC_FUNC_CMPSET(int, u_int);
1336 MSAN_ATOMIC_FUNC_CMPSET(long, u_long);
1337 MSAN_ATOMIC_FUNC_CMPSET(ptr, uintptr_t);
1338
1339 MSAN_ATOMIC_FUNC_FCMPSET(8, uint8_t);
1340 MSAN_ATOMIC_FUNC_FCMPSET(16, uint16_t);
1341 MSAN_ATOMIC_FUNC_FCMPSET(32, uint32_t);
1342 MSAN_ATOMIC_FUNC_FCMPSET(64, uint64_t);
1343 MSAN_ATOMIC_FUNC_FCMPSET(int, u_int);
1344 MSAN_ATOMIC_FUNC_FCMPSET(long, u_long);
1345 MSAN_ATOMIC_FUNC_FCMPSET(ptr, uintptr_t);
1346
1347 _MSAN_ATOMIC_FUNC_LOAD(bool, bool);
1348 MSAN_ATOMIC_FUNC_LOAD(8, uint8_t);
1349 MSAN_ATOMIC_FUNC_LOAD(16, uint16_t);
1350 MSAN_ATOMIC_FUNC_LOAD(32, uint32_t);
1351 MSAN_ATOMIC_FUNC_LOAD(64, uint64_t);
1352 MSAN_ATOMIC_FUNC_LOAD(char, u_char);
1353 MSAN_ATOMIC_FUNC_LOAD(short, u_short);
1354 MSAN_ATOMIC_FUNC_LOAD(int, u_int);
1355 MSAN_ATOMIC_FUNC_LOAD(long, u_long);
1356 MSAN_ATOMIC_FUNC_LOAD(ptr, uintptr_t);
1357
1358 _MSAN_ATOMIC_FUNC_STORE(bool, bool);
1359 MSAN_ATOMIC_FUNC_STORE(8, uint8_t);
1360 MSAN_ATOMIC_FUNC_STORE(16, uint16_t);
1361 MSAN_ATOMIC_FUNC_STORE(32, uint32_t);
1362 MSAN_ATOMIC_FUNC_STORE(64, uint64_t);
1363 MSAN_ATOMIC_FUNC_STORE(char, u_char);
1364 MSAN_ATOMIC_FUNC_STORE(short, u_short);
1365 MSAN_ATOMIC_FUNC_STORE(int, u_int);
1366 MSAN_ATOMIC_FUNC_STORE(long, u_long);
1367 MSAN_ATOMIC_FUNC_STORE(ptr, uintptr_t);
1368
1369 MSAN_ATOMIC_FUNC_THREAD_FENCE(acq);
1370 MSAN_ATOMIC_FUNC_THREAD_FENCE(rel);
1371 MSAN_ATOMIC_FUNC_THREAD_FENCE(acq_rel);
1372 MSAN_ATOMIC_FUNC_THREAD_FENCE(seq_cst);
1373
1374 void
kmsan_atomic_interrupt_fence(void)1375 kmsan_atomic_interrupt_fence(void)
1376 {
1377 atomic_interrupt_fence();
1378 }
1379
1380 /* -------------------------------------------------------------------------- */
1381
1382 #include <sys/bus.h>
1383 #include <machine/bus.h>
1384 #include <sys/bus_san.h>
1385
1386 int
kmsan_bus_space_map(bus_space_tag_t tag,bus_addr_t hnd,bus_size_t size,int flags,bus_space_handle_t * handlep)1387 kmsan_bus_space_map(bus_space_tag_t tag, bus_addr_t hnd, bus_size_t size,
1388 int flags, bus_space_handle_t *handlep)
1389 {
1390 return (bus_space_map(tag, hnd, size, flags, handlep));
1391 }
1392
1393 void
kmsan_bus_space_unmap(bus_space_tag_t tag,bus_space_handle_t hnd,bus_size_t size)1394 kmsan_bus_space_unmap(bus_space_tag_t tag, bus_space_handle_t hnd,
1395 bus_size_t size)
1396 {
1397 bus_space_unmap(tag, hnd, size);
1398 }
1399
1400 int
kmsan_bus_space_subregion(bus_space_tag_t tag,bus_space_handle_t hnd,bus_size_t offset,bus_size_t size,bus_space_handle_t * handlep)1401 kmsan_bus_space_subregion(bus_space_tag_t tag, bus_space_handle_t hnd,
1402 bus_size_t offset, bus_size_t size, bus_space_handle_t *handlep)
1403 {
1404 return (bus_space_subregion(tag, hnd, offset, size, handlep));
1405 }
1406
1407 void
kmsan_bus_space_free(bus_space_tag_t tag,bus_space_handle_t hnd,bus_size_t size)1408 kmsan_bus_space_free(bus_space_tag_t tag, bus_space_handle_t hnd,
1409 bus_size_t size)
1410 {
1411 bus_space_free(tag, hnd, size);
1412 }
1413
1414 void
kmsan_bus_space_barrier(bus_space_tag_t tag,bus_space_handle_t hnd,bus_size_t offset,bus_size_t size,int flags)1415 kmsan_bus_space_barrier(bus_space_tag_t tag, bus_space_handle_t hnd,
1416 bus_size_t offset, bus_size_t size, int flags)
1417 {
1418 bus_space_barrier(tag, hnd, offset, size, flags);
1419 }
1420
1421 #if defined(__amd64__)
1422 #define BUS_SPACE_IO(tag) ((tag) == X86_BUS_SPACE_IO)
1423 #else
1424 #define BUS_SPACE_IO(tag) (false)
1425 #endif
1426
1427 #define MSAN_BUS_READ_FUNC(func, width, type) \
1428 type kmsan_bus_space_read##func##_##width(bus_space_tag_t tag, \
1429 bus_space_handle_t hnd, bus_size_t offset) \
1430 { \
1431 type ret; \
1432 if (!BUS_SPACE_IO(tag)) \
1433 kmsan_shadow_fill((uintptr_t)(hnd + offset), \
1434 KMSAN_STATE_INITED, (width)); \
1435 ret = bus_space_read##func##_##width(tag, hnd, offset); \
1436 kmsan_init_ret(sizeof(type)); \
1437 return (ret); \
1438 } \
1439
1440 #define MSAN_BUS_READ_PTR_FUNC(func, width, type) \
1441 void kmsan_bus_space_read_##func##_##width(bus_space_tag_t tag, \
1442 bus_space_handle_t hnd, bus_size_t size, type *buf, \
1443 bus_size_t count) \
1444 { \
1445 kmsan_shadow_fill((uintptr_t)buf, KMSAN_STATE_INITED, \
1446 (width) * count); \
1447 bus_space_read_##func##_##width(tag, hnd, size, buf, \
1448 count); \
1449 }
1450
1451 MSAN_BUS_READ_FUNC(, 1, uint8_t)
1452 MSAN_BUS_READ_FUNC(_stream, 1, uint8_t)
1453 MSAN_BUS_READ_PTR_FUNC(multi, 1, uint8_t)
1454 MSAN_BUS_READ_PTR_FUNC(multi_stream, 1, uint8_t)
1455 MSAN_BUS_READ_PTR_FUNC(region, 1, uint8_t)
1456 MSAN_BUS_READ_PTR_FUNC(region_stream, 1, uint8_t)
1457
1458 MSAN_BUS_READ_FUNC(, 2, uint16_t)
1459 MSAN_BUS_READ_FUNC(_stream, 2, uint16_t)
1460 MSAN_BUS_READ_PTR_FUNC(multi, 2, uint16_t)
1461 MSAN_BUS_READ_PTR_FUNC(multi_stream, 2, uint16_t)
1462 MSAN_BUS_READ_PTR_FUNC(region, 2, uint16_t)
1463 MSAN_BUS_READ_PTR_FUNC(region_stream, 2, uint16_t)
1464
1465 MSAN_BUS_READ_FUNC(, 4, uint32_t)
1466 MSAN_BUS_READ_FUNC(_stream, 4, uint32_t)
1467 MSAN_BUS_READ_PTR_FUNC(multi, 4, uint32_t)
1468 MSAN_BUS_READ_PTR_FUNC(multi_stream, 4, uint32_t)
1469 MSAN_BUS_READ_PTR_FUNC(region, 4, uint32_t)
1470 MSAN_BUS_READ_PTR_FUNC(region_stream, 4, uint32_t)
1471
1472 MSAN_BUS_READ_FUNC(, 8, uint64_t)
1473 #ifndef __amd64__
1474 MSAN_BUS_READ_FUNC(_stream, 8, uint64_t)
1475 MSAN_BUS_READ_PTR_FUNC(multi, 8, uint64_t)
1476 MSAN_BUS_READ_PTR_FUNC(multi_stream, 8, uint64_t)
1477 MSAN_BUS_READ_PTR_FUNC(region, 8, uint64_t)
1478 MSAN_BUS_READ_PTR_FUNC(region_stream, 8, uint64_t)
1479 #endif
1480
1481 #define MSAN_BUS_WRITE_FUNC(func, width, type) \
1482 void kmsan_bus_space_write##func##_##width(bus_space_tag_t tag, \
1483 bus_space_handle_t hnd, bus_size_t offset, type value) \
1484 { \
1485 bus_space_write##func##_##width(tag, hnd, offset, value);\
1486 } \
1487
1488 #define MSAN_BUS_WRITE_PTR_FUNC(func, width, type) \
1489 void kmsan_bus_space_write_##func##_##width(bus_space_tag_t tag,\
1490 bus_space_handle_t hnd, bus_size_t size, const type *buf, \
1491 bus_size_t count) \
1492 { \
1493 kmsan_shadow_check((uintptr_t)buf, sizeof(type) * count,\
1494 "bus_space_write()"); \
1495 bus_space_write_##func##_##width(tag, hnd, size, buf, \
1496 count); \
1497 }
1498
1499 MSAN_BUS_WRITE_FUNC(, 1, uint8_t)
1500 MSAN_BUS_WRITE_FUNC(_stream, 1, uint8_t)
1501 MSAN_BUS_WRITE_PTR_FUNC(multi, 1, uint8_t)
1502 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 1, uint8_t)
1503 MSAN_BUS_WRITE_PTR_FUNC(region, 1, uint8_t)
1504 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 1, uint8_t)
1505
1506 MSAN_BUS_WRITE_FUNC(, 2, uint16_t)
1507 MSAN_BUS_WRITE_FUNC(_stream, 2, uint16_t)
1508 MSAN_BUS_WRITE_PTR_FUNC(multi, 2, uint16_t)
1509 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 2, uint16_t)
1510 MSAN_BUS_WRITE_PTR_FUNC(region, 2, uint16_t)
1511 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 2, uint16_t)
1512
1513 MSAN_BUS_WRITE_FUNC(, 4, uint32_t)
1514 MSAN_BUS_WRITE_FUNC(_stream, 4, uint32_t)
1515 MSAN_BUS_WRITE_PTR_FUNC(multi, 4, uint32_t)
1516 MSAN_BUS_WRITE_PTR_FUNC(multi_stream, 4, uint32_t)
1517 MSAN_BUS_WRITE_PTR_FUNC(region, 4, uint32_t)
1518 MSAN_BUS_WRITE_PTR_FUNC(region_stream, 4, uint32_t)
1519
1520 MSAN_BUS_WRITE_FUNC(, 8, uint64_t)
1521
1522 #define MSAN_BUS_SET_FUNC(func, width, type) \
1523 void kmsan_bus_space_set_##func##_##width(bus_space_tag_t tag, \
1524 bus_space_handle_t hnd, bus_size_t offset, type value, \
1525 bus_size_t count) \
1526 { \
1527 bus_space_set_##func##_##width(tag, hnd, offset, value, \
1528 count); \
1529 }
1530
1531 MSAN_BUS_SET_FUNC(multi, 1, uint8_t)
1532 MSAN_BUS_SET_FUNC(region, 1, uint8_t)
1533 MSAN_BUS_SET_FUNC(multi_stream, 1, uint8_t)
1534 MSAN_BUS_SET_FUNC(region_stream, 1, uint8_t)
1535
1536 MSAN_BUS_SET_FUNC(multi, 2, uint16_t)
1537 MSAN_BUS_SET_FUNC(region, 2, uint16_t)
1538 MSAN_BUS_SET_FUNC(multi_stream, 2, uint16_t)
1539 MSAN_BUS_SET_FUNC(region_stream, 2, uint16_t)
1540
1541 MSAN_BUS_SET_FUNC(multi, 4, uint32_t)
1542 MSAN_BUS_SET_FUNC(region, 4, uint32_t)
1543 MSAN_BUS_SET_FUNC(multi_stream, 4, uint32_t)
1544 MSAN_BUS_SET_FUNC(region_stream, 4, uint32_t)
1545
1546 #define MSAN_BUS_PEEK_FUNC(width, type) \
1547 int kmsan_bus_space_peek_##width(bus_space_tag_t tag, \
1548 bus_space_handle_t hnd, bus_size_t offset, type *value) \
1549 { \
1550 return (bus_space_peek_##width(tag, hnd, offset, value)); \
1551 }
1552
1553 MSAN_BUS_PEEK_FUNC(1, uint8_t)
1554 MSAN_BUS_PEEK_FUNC(2, uint16_t)
1555 MSAN_BUS_PEEK_FUNC(4, uint32_t)
1556
1557 #define MSAN_BUS_POKE_FUNC(width, type) \
1558 int kmsan_bus_space_poke_##width(bus_space_tag_t tag, \
1559 bus_space_handle_t hnd, bus_size_t offset, type value) \
1560 { \
1561 return (bus_space_poke_##width(tag, hnd, offset, value)); \
1562 }
1563
1564 MSAN_BUS_POKE_FUNC(1, uint8_t)
1565 MSAN_BUS_POKE_FUNC(2, uint16_t)
1566 MSAN_BUS_POKE_FUNC(4, uint32_t)
1567
1568 /* -------------------------------------------------------------------------- */
1569
1570 void
kmsan_bus_dmamap_sync(struct memdesc * desc,bus_dmasync_op_t op)1571 kmsan_bus_dmamap_sync(struct memdesc *desc, bus_dmasync_op_t op)
1572 {
1573 /*
1574 * Some drivers, e.g., nvme, use the same code path for loading device
1575 * read and write requests, and will thus specify both flags. In this
1576 * case we should not do any checking since it will generally lead to
1577 * false positives.
1578 */
1579 if ((op & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) ==
1580 BUS_DMASYNC_PREWRITE) {
1581 switch (desc->md_type) {
1582 case MEMDESC_VADDR:
1583 kmsan_check(desc->u.md_vaddr, desc->md_len,
1584 "dmasync");
1585 break;
1586 case MEMDESC_MBUF:
1587 kmsan_check_mbuf(desc->u.md_mbuf, "dmasync");
1588 break;
1589 case 0:
1590 break;
1591 default:
1592 kmsan_panic("%s: unhandled memdesc type %d", __func__,
1593 desc->md_type);
1594 }
1595 }
1596 if ((op & BUS_DMASYNC_POSTREAD) != 0) {
1597 switch (desc->md_type) {
1598 case MEMDESC_VADDR:
1599 kmsan_mark(desc->u.md_vaddr, desc->md_len,
1600 KMSAN_STATE_INITED);
1601 break;
1602 case MEMDESC_MBUF:
1603 kmsan_mark_mbuf(desc->u.md_mbuf, KMSAN_STATE_INITED);
1604 break;
1605 case 0:
1606 break;
1607 default:
1608 kmsan_panic("%s: unhandled memdesc type %d", __func__,
1609 desc->md_type);
1610 }
1611 }
1612 }
1613