1 /* SPDX-License-Identifier: GPL-2.0 */
2
3 /*
4 * This header provides generic wrappers for memory access instrumentation that
5 * the compiler cannot emit for: KASAN, KCSAN, KMSAN.
6 */
7 #ifndef _LINUX_INSTRUMENTED_H
8 #define _LINUX_INSTRUMENTED_H
9
10 #include <linux/bug.h>
11 #include <linux/compiler.h>
12 #include <linux/kasan-checks.h>
13 #include <linux/kcsan-checks.h>
14 #include <linux/kmsan-checks.h>
15 #include <linux/types.h>
16
17 /**
18 * instrument_read - instrument regular read access
19 * @v: address of access
20 * @size: size of access
21 *
22 * Instrument a regular read access. The instrumentation should be inserted
23 * before the actual read happens.
24 */
instrument_read(const volatile void * v,size_t size)25 static __always_inline void instrument_read(const volatile void *v, size_t size)
26 {
27 kasan_check_read(v, size);
28 kcsan_check_read(v, size);
29 }
30
31 /**
32 * instrument_write - instrument regular write access
33 * @v: address of access
34 * @size: size of access
35 *
36 * Instrument a regular write access. The instrumentation should be inserted
37 * before the actual write happens.
38 */
instrument_write(const volatile void * v,size_t size)39 static __always_inline void instrument_write(const volatile void *v, size_t size)
40 {
41 kasan_check_write(v, size);
42 kcsan_check_write(v, size);
43 }
44
45 /**
46 * instrument_read_write - instrument regular read-write access
47 * @v: address of access
48 * @size: size of access
49 *
50 * Instrument a regular write access. The instrumentation should be inserted
51 * before the actual write happens.
52 */
instrument_read_write(const volatile void * v,size_t size)53 static __always_inline void instrument_read_write(const volatile void *v, size_t size)
54 {
55 kasan_check_write(v, size);
56 kcsan_check_read_write(v, size);
57 }
58
instrument_atomic_check_alignment(const volatile void * v,size_t size)59 static __always_inline void instrument_atomic_check_alignment(const volatile void *v, size_t size)
60 {
61 #ifndef __DISABLE_EXPORTS
62 if (IS_ENABLED(CONFIG_DEBUG_ATOMIC)) {
63 unsigned int mask = size - 1;
64
65 if (IS_ENABLED(CONFIG_DEBUG_ATOMIC_LARGEST_ALIGN))
66 mask &= sizeof(struct { long x; } __aligned_largest) - 1;
67 WARN_ON_ONCE((unsigned long)v & mask);
68 }
69 #endif
70 }
71
72 /**
73 * instrument_atomic_read - instrument atomic read access
74 * @v: address of access
75 * @size: size of access
76 *
77 * Instrument an atomic read access. The instrumentation should be inserted
78 * before the actual read happens.
79 */
instrument_atomic_read(const volatile void * v,size_t size)80 static __always_inline void instrument_atomic_read(const volatile void *v, size_t size)
81 {
82 kasan_check_read(v, size);
83 kcsan_check_atomic_read(v, size);
84 instrument_atomic_check_alignment(v, size);
85 }
86
87 /**
88 * instrument_atomic_write - instrument atomic write access
89 * @v: address of access
90 * @size: size of access
91 *
92 * Instrument an atomic write access. The instrumentation should be inserted
93 * before the actual write happens.
94 */
instrument_atomic_write(const volatile void * v,size_t size)95 static __always_inline void instrument_atomic_write(const volatile void *v, size_t size)
96 {
97 kasan_check_write(v, size);
98 kcsan_check_atomic_write(v, size);
99 instrument_atomic_check_alignment(v, size);
100 }
101
102 /**
103 * instrument_atomic_read_write - instrument atomic read-write access
104 * @v: address of access
105 * @size: size of access
106 *
107 * Instrument an atomic read-write access. The instrumentation should be
108 * inserted before the actual write happens.
109 */
instrument_atomic_read_write(const volatile void * v,size_t size)110 static __always_inline void instrument_atomic_read_write(const volatile void *v, size_t size)
111 {
112 kasan_check_write(v, size);
113 kcsan_check_atomic_read_write(v, size);
114 instrument_atomic_check_alignment(v, size);
115 }
116
117 /**
118 * instrument_copy_to_user - instrument reads of copy_to_user
119 * @to: destination address
120 * @from: source address
121 * @n: number of bytes to copy
122 *
123 * Instrument reads from kernel memory, that are due to copy_to_user (and
124 * variants). The instrumentation must be inserted before the accesses.
125 */
126 static __always_inline void
instrument_copy_to_user(void __user * to,const void * from,unsigned long n)127 instrument_copy_to_user(void __user *to, const void *from, unsigned long n)
128 {
129 kasan_check_read(from, n);
130 kcsan_check_read(from, n);
131 kmsan_copy_to_user(to, from, n, 0);
132 }
133
134 /**
135 * instrument_copy_from_user_before - add instrumentation before copy_from_user
136 * @to: destination address
137 * @from: source address
138 * @n: number of bytes to copy
139 *
140 * Instrument writes to kernel memory, that are due to copy_from_user (and
141 * variants). The instrumentation should be inserted before the accesses.
142 */
143 static __always_inline void
instrument_copy_from_user_before(const void * to,const void __user * from,unsigned long n)144 instrument_copy_from_user_before(const void *to, const void __user *from, unsigned long n)
145 {
146 kasan_check_write(to, n);
147 kcsan_check_write(to, n);
148 }
149
150 /**
151 * instrument_copy_from_user_after - add instrumentation after copy_from_user
152 * @to: destination address
153 * @from: source address
154 * @n: number of bytes to copy
155 * @left: number of bytes not copied (as returned by copy_from_user)
156 *
157 * Instrument writes to kernel memory, that are due to copy_from_user (and
158 * variants). The instrumentation should be inserted after the accesses.
159 */
160 static __always_inline void
instrument_copy_from_user_after(const void * to,const void __user * from,unsigned long n,unsigned long left)161 instrument_copy_from_user_after(const void *to, const void __user *from,
162 unsigned long n, unsigned long left)
163 {
164 kmsan_unpoison_memory(to, n - left);
165 }
166
167 /**
168 * instrument_memcpy_before - add instrumentation before non-instrumented memcpy
169 * @to: destination address
170 * @from: source address
171 * @n: number of bytes to copy
172 *
173 * Instrument memory accesses that happen in custom memcpy implementations. The
174 * instrumentation should be inserted before the memcpy call.
175 */
instrument_memcpy_before(void * to,const void * from,unsigned long n)176 static __always_inline void instrument_memcpy_before(void *to, const void *from,
177 unsigned long n)
178 {
179 kasan_check_write(to, n);
180 kasan_check_read(from, n);
181 kcsan_check_write(to, n);
182 kcsan_check_read(from, n);
183 }
184
185 /**
186 * instrument_memcpy_after - add instrumentation after non-instrumented memcpy
187 * @to: destination address
188 * @from: source address
189 * @n: number of bytes to copy
190 * @left: number of bytes not copied (if known)
191 *
192 * Instrument memory accesses that happen in custom memcpy implementations. The
193 * instrumentation should be inserted after the memcpy call.
194 */
instrument_memcpy_after(void * to,const void * from,unsigned long n,unsigned long left)195 static __always_inline void instrument_memcpy_after(void *to, const void *from,
196 unsigned long n,
197 unsigned long left)
198 {
199 kmsan_memmove(to, from, n - left);
200 }
201
202 /**
203 * instrument_get_user() - add instrumentation to get_user()-like macros
204 * @to: destination variable, may not be address-taken
205 *
206 * get_user() and friends are fragile, so it may depend on the implementation
207 * whether the instrumentation happens before or after the data is copied from
208 * the userspace.
209 */
210 #define instrument_get_user(to) \
211 ({ \
212 u64 __tmp = (u64)(to); \
213 kmsan_unpoison_memory(&__tmp, sizeof(__tmp)); \
214 to = __tmp; \
215 })
216
217
218 /**
219 * instrument_put_user() - add instrumentation to put_user()-like macros
220 * @from: source address
221 * @ptr: userspace pointer to copy to
222 * @size: number of bytes to copy
223 *
224 * put_user() and friends are fragile, so it may depend on the implementation
225 * whether the instrumentation happens before or after the data is copied from
226 * the userspace.
227 */
228 #define instrument_put_user(from, ptr, size) \
229 ({ \
230 kmsan_copy_to_user(ptr, &from, sizeof(from), 0); \
231 })
232
233 #endif /* _LINUX_INSTRUMENTED_H */
234