1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * CDDL HEADER START
4 *
5 * The contents of this file are subject to the terms of the
6 * Common Development and Distribution License (the "License").
7 * You may not use this file except in compliance with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or https://opensource.org/licenses/CDDL-1.0.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22
23 /*
24 * Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
25 * Copyright (c) 2026, TrueNAS.
26 */
27
28 #include <sys/simd.h>
29 #include <sys/zfs_context.h>
30 #include <sys/zfs_impl.h>
31 #include <sys/sha2.h>
32
33 #include <sha2/sha2_impl.h>
34 #include <sys/asm_linkage.h>
35
36 #define TF(E, N) \
37 extern void ASMABI E(uint64_t s[8], const void *, size_t); \
38 static inline void N(uint64_t s[8], const void *d, size_t b) { \
39 kfpu_begin(); E(s, d, b); kfpu_end(); \
40 }
41
42 #if defined(__x86_64) || defined(__aarch64__) || defined(__arm__) || \
43 defined(__aarch64__) || defined(__arm__) || defined(__PPC64__)
44 /* some implementation is always okay */
sha2_is_supported(void)45 static inline boolean_t sha2_is_supported(void)
46 {
47 return (B_TRUE);
48 }
49 #endif
50
51 #if defined(__x86_64)
52
53 /* Users of ASMABI requires all calls to be from wrappers */
54 extern void ASMABI
55 zfs_sha512_transform_x64(uint64_t s[8], const void *, size_t);
56
57 static inline void
tf_sha512_transform_x64(uint64_t s[8],const void * d,size_t b)58 tf_sha512_transform_x64(uint64_t s[8], const void *d, size_t b)
59 {
60 zfs_sha512_transform_x64(s, d, b);
61 }
62 const sha512_ops_t sha512_x64_impl = {
63 .is_supported = sha2_is_supported,
64 .transform = tf_sha512_transform_x64,
65 .name = "x64"
66 };
67
68 #if defined(HAVE_AVX)
sha2_have_avx(void)69 static boolean_t sha2_have_avx(void)
70 {
71 return (kfpu_allowed() && zfs_avx_available());
72 }
73
74 TF(zfs_sha512_transform_avx, tf_sha512_avx);
75 const sha512_ops_t sha512_avx_impl = {
76 .is_supported = sha2_have_avx,
77 .transform = tf_sha512_avx,
78 .name = "avx"
79 };
80 #endif
81
82 #if defined(HAVE_AVX2)
sha2_have_avx2(void)83 static boolean_t sha2_have_avx2(void)
84 {
85 return (kfpu_allowed() && zfs_avx2_available());
86 }
87
88 TF(zfs_sha512_transform_avx2, tf_sha512_avx2);
89 const sha512_ops_t sha512_avx2_impl = {
90 .is_supported = sha2_have_avx2,
91 .transform = tf_sha512_avx2,
92 .name = "avx2"
93 };
94 #endif
95
96 #if defined(HAVE_SHA512EXT)
sha2_have_sha512ext(void)97 static boolean_t sha2_have_sha512ext(void)
98 {
99 return (kfpu_allowed() && zfs_sha512ext_available());
100 }
101
102 TF(zfs_sha512_transform_sha512ext, tf_sha512_sha512ext);
103 const sha512_ops_t sha512_sha512ext_impl = {
104 .is_supported = sha2_have_sha512ext,
105 .transform = tf_sha512_sha512ext,
106 .name = "sha512ext"
107 };
108 #endif
109
110 #elif defined(__aarch64__) || defined(__arm__)
111 extern void zfs_sha512_block_armv7(uint64_t s[8], const void *, size_t);
112 const sha512_ops_t sha512_armv7_impl = {
113 .is_supported = sha2_is_supported,
114 .transform = zfs_sha512_block_armv7,
115 .name = "armv7"
116 };
117
118 #if defined(__aarch64__)
sha512_have_armv8ce(void)119 static boolean_t sha512_have_armv8ce(void)
120 {
121 return (kfpu_allowed() && zfs_sha512_available());
122 }
123
124 TF(zfs_sha512_block_armv8, tf_sha512_armv8ce);
125 const sha512_ops_t sha512_armv8_impl = {
126 .is_supported = sha512_have_armv8ce,
127 .transform = tf_sha512_armv8ce,
128 .name = "armv8-ce"
129 };
130 #endif
131
132 #if defined(__arm__) && __ARM_ARCH > 6
sha512_have_neon(void)133 static boolean_t sha512_have_neon(void)
134 {
135 return (kfpu_allowed() && zfs_neon_available());
136 }
137
138 TF(zfs_sha512_block_neon, tf_sha512_neon);
139 const sha512_ops_t sha512_neon_impl = {
140 .is_supported = sha512_have_neon,
141 .transform = tf_sha512_neon,
142 .name = "neon"
143 };
144 #endif
145
146 #elif defined(__PPC64__)
147 TF(zfs_sha512_ppc, tf_sha512_ppc);
148 const sha512_ops_t sha512_ppc_impl = {
149 .is_supported = sha2_is_supported,
150 .transform = tf_sha512_ppc,
151 .name = "ppc"
152 };
153
sha512_have_isa207(void)154 static boolean_t sha512_have_isa207(void)
155 {
156 return (kfpu_allowed() && zfs_isa207_available());
157 }
158
159 TF(zfs_sha512_power8, tf_sha512_power8);
160 const sha512_ops_t sha512_power8_impl = {
161 .is_supported = sha512_have_isa207,
162 .transform = tf_sha512_power8,
163 .name = "power8"
164 };
165 #endif /* __PPC64__ */
166
167 /* the two generic ones */
168 extern const sha512_ops_t sha512_generic_impl;
169
170 /* array with all sha512 implementations */
171 static const sha512_ops_t *const sha512_impls[] = {
172 &sha512_generic_impl,
173 #if defined(__x86_64)
174 &sha512_x64_impl,
175 #endif
176 #if defined(__x86_64) && defined(HAVE_AVX)
177 &sha512_avx_impl,
178 #endif
179 #if defined(__x86_64) && defined(HAVE_AVX2)
180 &sha512_avx2_impl,
181 #endif
182 #if defined(__x86_64) && defined(HAVE_SHA512EXT)
183 &sha512_sha512ext_impl,
184 #endif
185 #if defined(__aarch64__) || defined(__arm__)
186 &sha512_armv7_impl,
187 #if defined(__aarch64__)
188 &sha512_armv8_impl,
189 #endif
190 #if defined(__arm__) && __ARM_ARCH > 6
191 &sha512_neon_impl,
192 #endif
193 #endif
194 #if defined(__PPC64__)
195 &sha512_ppc_impl,
196 &sha512_power8_impl,
197 #endif /* __PPC64__ */
198 };
199
200 /* use the generic implementation functions */
201 #define IMPL_NAME "sha512"
202 #define IMPL_OPS_T sha512_ops_t
203 #define IMPL_ARRAY sha512_impls
204 #define IMPL_GET_OPS sha512_get_ops
205 #define ZFS_IMPL_OPS zfs_sha512_ops
206 #include <generic_impl.c>
207
208 #ifdef _KERNEL
209
210 #define IMPL_FMT(impl, i) (((impl) == (i)) ? "[%s] " : "%s ")
211
212 #if defined(__linux__)
213
214 static int
sha512_param_get(char * buffer,zfs_kernel_param_t * unused)215 sha512_param_get(char *buffer, zfs_kernel_param_t *unused)
216 {
217 const uint32_t impl = IMPL_READ(generic_impl_chosen);
218 char *fmt;
219 int cnt = 0;
220
221 /* cycling */
222 fmt = IMPL_FMT(impl, IMPL_CYCLE);
223 cnt += sprintf(buffer + cnt, fmt, "cycle");
224
225 /* list fastest */
226 fmt = IMPL_FMT(impl, IMPL_FASTEST);
227 cnt += sprintf(buffer + cnt, fmt, "fastest");
228
229 /* list all supported implementations */
230 generic_impl_init();
231 for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) {
232 fmt = IMPL_FMT(impl, i);
233 cnt += sprintf(buffer + cnt, fmt,
234 generic_supp_impls[i]->name);
235 }
236
237 return (cnt);
238 }
239
240 static int
sha512_param_set(const char * val,zfs_kernel_param_t * unused)241 sha512_param_set(const char *val, zfs_kernel_param_t *unused)
242 {
243 (void) unused;
244 return (generic_impl_setname(val));
245 }
246
247 #elif defined(__FreeBSD__)
248
249 #include <sys/sbuf.h>
250
251 static int
sha512_param(ZFS_MODULE_PARAM_ARGS)252 sha512_param(ZFS_MODULE_PARAM_ARGS)
253 {
254 int err;
255
256 generic_impl_init();
257 if (req->newptr == NULL) {
258 const uint32_t impl = IMPL_READ(generic_impl_chosen);
259 const int init_buflen = 64;
260 const char *fmt;
261 struct sbuf *s;
262
263 s = sbuf_new_for_sysctl(NULL, NULL, init_buflen, req);
264
265 /* cycling */
266 fmt = IMPL_FMT(impl, IMPL_CYCLE);
267 (void) sbuf_printf(s, fmt, "cycle");
268
269 /* list fastest */
270 fmt = IMPL_FMT(impl, IMPL_FASTEST);
271 (void) sbuf_printf(s, fmt, "fastest");
272
273 /* list all supported implementations */
274 for (uint32_t i = 0; i < generic_supp_impls_cnt; ++i) {
275 fmt = IMPL_FMT(impl, i);
276 (void) sbuf_printf(s, fmt, generic_supp_impls[i]->name);
277 }
278
279 err = sbuf_finish(s);
280 sbuf_delete(s);
281
282 return (err);
283 }
284
285 /* we got module parameter */
286 char buf[16];
287
288 err = sysctl_handle_string(oidp, buf, sizeof (buf), req);
289 if (err) {
290 return (err);
291 }
292
293 return (-generic_impl_setname(buf));
294 }
295 #endif
296
297 #undef IMPL_FMT
298
299 ZFS_MODULE_VIRTUAL_PARAM_CALL(zfs, zfs_, sha512_impl,
300 sha512_param_set, sha512_param_get, ZMOD_RW, \
301 "Select SHA512 implementation.");
302 #endif
303
304 #undef TF
305