1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * CDDL HEADER START
4 *
5 * The contents of this file are subject to the terms of the
6 * Common Development and Distribution License (the "License").
7 * You may not use this file except in compliance with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or https://opensource.org/licenses/CDDL-1.0.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22 /*
23 * Copyright 2024 Google, Inc. All rights reserved.
24 */
25 #include <sys/zfs_context.h>
26 #include <sys/kstat.h>
27 #include <sys/simd.h>
28
29
30 #ifdef _KERNEL
31 #ifdef __linux__
32 #include <linux/simd.h>
33 #endif /* __linux__ */
34 kstat_t *simd_stat_kstat;
35 #endif /* _KERNEL */
36
37 #ifdef _KERNEL
38 /* Sometimes, we don't define these at all. */
39 #ifndef HAVE_KERNEL_FPU
40 #define HAVE_KERNEL_FPU (0)
41 #endif
42 #ifndef HAVE_KERNEL_NEON
43 #define HAVE_KERNEL_NEON (0)
44 #endif
45 #ifndef HAVE_KERNEL_FPU_INTERNAL
46 #define HAVE_KERNEL_FPU_INTERNAL (0)
47 #endif
48 #ifndef HAVE_UNDERSCORE_KERNEL_FPU
49 #define HAVE_UNDERSCORE_KERNEL_FPU (0)
50 #endif
51
52 #define SIMD_STAT_PRINT(s, feat, val) \
53 kmem_scnprintf(s + off, MAX(4095-off, 0), "%-16s\t%1d\n", feat, (val))
54
55 static int
simd_stat_kstat_data(char * buf,size_t size,void * data)56 simd_stat_kstat_data(char *buf, size_t size, void *data)
57 {
58 (void) data;
59
60 static char simd_stat_kstat_payload[4096] = {0};
61 static int off = 0;
62 #ifdef __linux__
63 if (off == 0) {
64 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
65 "kfpu_allowed", kfpu_allowed());
66 #if defined(__x86_64__) || defined(__i386__)
67 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
68 "kfpu", HAVE_KERNEL_FPU);
69 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
70 "kfpu_internal", HAVE_KERNEL_FPU_INTERNAL);
71 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
72 "__kernel_fpu", HAVE_UNDERSCORE_KERNEL_FPU);
73 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
74 "sse", zfs_sse_available());
75 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
76 "sse2", zfs_sse2_available());
77 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
78 "sse3", zfs_sse3_available());
79 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
80 "ssse3", zfs_ssse3_available());
81 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
82 "sse41", zfs_sse4_1_available());
83 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
84 "sse42", zfs_sse4_2_available());
85 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
86 "avx", zfs_avx_available());
87 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
88 "avx2", zfs_avx2_available());
89 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
90 "avx512f", zfs_avx512f_available());
91 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
92 "avx512cd", zfs_avx512cd_available());
93 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
94 "avx512er", zfs_avx512er_available());
95 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
96 "avx512pf", zfs_avx512pf_available());
97 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
98 "avx512bw", zfs_avx512bw_available());
99 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
100 "avx512dq", zfs_avx512dq_available());
101 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
102 "avx512vl", zfs_avx512vl_available());
103 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
104 "avx512ifma", zfs_avx512ifma_available());
105 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
106 "avx512vbmi", zfs_avx512vbmi_available());
107 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
108 "ymm", __ymm_enabled());
109 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
110 "zmm", __zmm_enabled());
111 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
112 "bmi1", zfs_bmi1_available());
113 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
114 "bmi2", zfs_bmi2_available());
115 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
116 "aes", zfs_aes_available());
117 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
118 "pclmulqdq", zfs_pclmulqdq_available());
119 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
120 "movbe", zfs_movbe_available());
121 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
122 "vaes", zfs_vaes_available());
123 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
124 "vpclmulqdq", zfs_vpclmulqdq_available());
125 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
126 "sha512ext", zfs_sha512ext_available());
127
128 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
129 "osxsave", boot_cpu_has(X86_FEATURE_OSXSAVE));
130 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
131 "xsaves", static_cpu_has(X86_FEATURE_XSAVES));
132 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
133 "xsaveopt", static_cpu_has(X86_FEATURE_XSAVEOPT));
134 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
135 "xsave", static_cpu_has(X86_FEATURE_XSAVE));
136 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
137 "fxsr", static_cpu_has(X86_FEATURE_FXSR));
138 #endif /* __x86__ */
139 #if defined(__arm__) || defined(__aarch64__)
140 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
141 "kernel_neon", HAVE_KERNEL_NEON);
142 #if defined(CONFIG_KERNEL_MODE_NEON)
143 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
144 "kernel_mode_neon", CONFIG_KERNEL_MODE_NEON);
145 #endif /* CONFIG_KERNEL_MODE_NEON */
146 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
147 "neon", zfs_neon_available());
148 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
149 "sha256", zfs_sha256_available());
150 #if defined(__aarch64__)
151 /*
152 * This technically can exist on 32b ARM but we don't
153 * define hooks to check for it and I didn't want to
154 * learn enough ARM ASM to add one.
155 */
156 off += SIMD_STAT_PRINT(simd_stat_kstat_payload,
157 "sha512", zfs_sha512_available());
158 #endif /* __aarch64__ */
159 #endif /* __arm__ */
160 /* We want to short-circuit this on unsupported platforms. */
161 off += 1;
162 }
163
164 kmem_scnprintf(buf, MIN(off, size), "%s", simd_stat_kstat_payload);
165 #endif /* __linux__ */
166 return (0);
167 }
168 #endif /* _KERNEL */
169
170 void
simd_stat_init(void)171 simd_stat_init(void)
172 {
173 static boolean_t simd_stat_initialized = B_FALSE;
174
175 if (!simd_stat_initialized) {
176 #if defined(_KERNEL)
177 /* Install kstats for all implementations */
178 simd_stat_kstat = kstat_create("zfs", 0, "simd", "misc",
179 KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VIRTUAL);
180
181
182 if (simd_stat_kstat != NULL) {
183 simd_stat_kstat->ks_data = (void*)(uintptr_t)1;
184 simd_stat_kstat->ks_ndata = 1;
185 simd_stat_kstat->ks_flags |= KSTAT_FLAG_NO_HEADERS;
186 kstat_set_raw_ops(simd_stat_kstat,
187 NULL,
188 simd_stat_kstat_data,
189 NULL);
190 kstat_install(simd_stat_kstat);
191 }
192 #endif /* _KERNEL */
193 }
194 /* Finish initialization */
195 simd_stat_initialized = B_TRUE;
196 }
197
198 void
simd_stat_fini(void)199 simd_stat_fini(void)
200 {
201 #if defined(_KERNEL)
202 if (simd_stat_kstat != NULL) {
203 kstat_delete(simd_stat_kstat);
204 simd_stat_kstat = NULL;
205 }
206 #endif
207 }
208
209 #ifdef _KERNEL
210 EXPORT_SYMBOL(simd_stat_init);
211 EXPORT_SYMBOL(simd_stat_fini);
212 #endif
213