1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * This file contains common tag-based KASAN code. 4 * 5 * Copyright (c) 2018 Google, Inc. 6 * Copyright (c) 2020 Google, Inc. 7 */ 8 9 #include <linux/atomic.h> 10 #include <linux/init.h> 11 #include <linux/kasan.h> 12 #include <linux/kernel.h> 13 #include <linux/memblock.h> 14 #include <linux/memory.h> 15 #include <linux/mm.h> 16 #include <linux/sched/clock.h> 17 #include <linux/stackdepot.h> 18 #include <linux/static_key.h> 19 #include <linux/string.h> 20 #include <linux/types.h> 21 22 #include "kasan.h" 23 #include "../slab.h" 24 25 #define KASAN_STACK_RING_SIZE_DEFAULT (32 << 10) 26 27 enum kasan_arg_stacktrace { 28 KASAN_ARG_STACKTRACE_DEFAULT, 29 KASAN_ARG_STACKTRACE_OFF, 30 KASAN_ARG_STACKTRACE_ON, 31 }; 32 33 static enum kasan_arg_stacktrace kasan_arg_stacktrace __initdata; 34 35 /* Whether to collect alloc/free stack traces. */ 36 DEFINE_STATIC_KEY_TRUE(kasan_flag_stacktrace); 37 38 /* Non-zero, as initial pointer values are 0. */ 39 #define STACK_RING_BUSY_PTR ((void *)1) 40 41 struct kasan_stack_ring stack_ring = { 42 .lock = __RW_LOCK_UNLOCKED(stack_ring.lock) 43 }; 44 45 /* kasan.stacktrace=off/on */ 46 static int __init early_kasan_flag_stacktrace(char *arg) 47 { 48 if (!arg) 49 return -EINVAL; 50 51 if (!strcmp(arg, "off")) 52 kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_OFF; 53 else if (!strcmp(arg, "on")) 54 kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_ON; 55 else 56 return -EINVAL; 57 58 return 0; 59 } 60 early_param("kasan.stacktrace", early_kasan_flag_stacktrace); 61 62 /* kasan.stack_ring_size=<number of entries> */ 63 static int __init early_kasan_flag_stack_ring_size(char *arg) 64 { 65 if (!arg) 66 return -EINVAL; 67 68 return kstrtoul(arg, 0, &stack_ring.size); 69 } 70 early_param("kasan.stack_ring_size", early_kasan_flag_stack_ring_size); 71 72 void __init kasan_init_tags(void) 73 { 74 switch (kasan_arg_stacktrace) { 75 case KASAN_ARG_STACKTRACE_DEFAULT: 76 /* Default is specified by kasan_flag_stacktrace definition. */ 77 break; 78 case KASAN_ARG_STACKTRACE_OFF: 79 static_branch_disable(&kasan_flag_stacktrace); 80 break; 81 case KASAN_ARG_STACKTRACE_ON: 82 static_branch_enable(&kasan_flag_stacktrace); 83 break; 84 } 85 86 if (kasan_stack_collection_enabled()) { 87 if (!stack_ring.size) 88 stack_ring.size = KASAN_STACK_RING_SIZE_DEFAULT; 89 stack_ring.entries = memblock_alloc( 90 sizeof(stack_ring.entries[0]) * stack_ring.size, 91 SMP_CACHE_BYTES); 92 if (WARN_ON(!stack_ring.entries)) 93 static_branch_disable(&kasan_flag_stacktrace); 94 } 95 } 96 97 static void save_stack_info(struct kmem_cache *cache, void *object, 98 gfp_t gfp_flags, bool is_free) 99 { 100 unsigned long flags; 101 depot_stack_handle_t stack, old_stack; 102 u64 pos; 103 struct kasan_stack_ring_entry *entry; 104 void *old_ptr; 105 106 stack = kasan_save_stack(gfp_flags, 107 STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_GET); 108 109 /* 110 * Prevent save_stack_info() from modifying stack ring 111 * when kasan_complete_mode_report_info() is walking it. 112 */ 113 read_lock_irqsave(&stack_ring.lock, flags); 114 115 next: 116 pos = atomic64_fetch_add(1, &stack_ring.pos); 117 entry = &stack_ring.entries[pos % stack_ring.size]; 118 119 /* Detect stack ring entry slots that are being written to. */ 120 old_ptr = READ_ONCE(entry->ptr); 121 if (old_ptr == STACK_RING_BUSY_PTR) 122 goto next; /* Busy slot. */ 123 if (!try_cmpxchg(&entry->ptr, &old_ptr, STACK_RING_BUSY_PTR)) 124 goto next; /* Busy slot. */ 125 126 old_stack = entry->track.stack; 127 128 entry->size = cache->object_size; 129 kasan_set_track(&entry->track, stack); 130 entry->is_free = is_free; 131 132 entry->ptr = object; 133 134 read_unlock_irqrestore(&stack_ring.lock, flags); 135 136 if (old_stack) 137 stack_depot_put(old_stack); 138 } 139 140 void kasan_save_alloc_info(struct kmem_cache *cache, void *object, gfp_t flags) 141 { 142 save_stack_info(cache, object, flags, false); 143 } 144 145 void kasan_save_free_info(struct kmem_cache *cache, void *object) 146 { 147 save_stack_info(cache, object, 0, true); 148 } 149