1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Copyright (C) 2023 Arm Limited.
4 * All rights reserved.
5 */
6
7 #include <libcflat.h>
8 #include <asm/smp.h>
9 #include <stdlib.h>
10
11 #define CPU0_ID 0
12 #define CPU1_ID (CPU0_ID + 1)
13 #define CPUS_MAX (CPU1_ID + 1)
14 #define FPU_QREG_MAX 32
15 #define FPU_RESULT_PASS (-1U)
16
17 /*
18 * Write 8 bytes of random data in random. Returns true on success, false on
19 * failure.
20 */
arch_collect_entropy(uint64_t * random)21 static inline bool arch_collect_entropy(uint64_t *random)
22 {
23 unsigned long ret;
24
25 asm volatile(
26 " mrs %[ptr], " xstr(RNDR) "\n"
27 " cset %[ret], ne\n" /* RNDR sets NZCV to 0b0100 on failure */
28 :
29 [ret] "=r" (ret),
30 [ptr] "=r" (*random)
31 :
32 : "cc"
33 );
34
35 return ret == 1;
36 }
37
38 #define fpu_reg_read(val) \
39 ({ \
40 uint64_t *__val = (val); \
41 asm volatile(".arch_extension fp\n" \
42 "stp q0, q1, [%0], #32\n\t" \
43 "stp q2, q3, [%0], #32\n\t" \
44 "stp q4, q5, [%0], #32\n\t" \
45 "stp q6, q7, [%0], #32\n\t" \
46 "stp q8, q9, [%0], #32\n\t" \
47 "stp q10, q11, [%0], #32\n\t" \
48 "stp q12, q13, [%0], #32\n\t" \
49 "stp q14, q15, [%0], #32\n\t" \
50 "stp q16, q17, [%0], #32\n\t" \
51 "stp q18, q19, [%0], #32\n\t" \
52 "stp q20, q21, [%0], #32\n\t" \
53 "stp q22, q23, [%0], #32\n\t" \
54 "stp q24, q25, [%0], #32\n\t" \
55 "stp q26, q27, [%0], #32\n\t" \
56 "stp q28, q29, [%0], #32\n\t" \
57 "stp q30, q31, [%0], #32\n\t" \
58 : "+r" (__val) \
59 : \
60 : "v0", "v1", "v2", "v3", \
61 "v4", "v5", "v6", "v7", \
62 "v8", "v9", "v10", "v11", \
63 "v12", "v13", "v14", \
64 "v15", "v16", "v17", \
65 "v18", "v19", "v20", \
66 "v21", "v22", "v23", \
67 "v24", "v25", "v26", \
68 "v27", "v28", "v29", \
69 "v30", "v31", "memory"); \
70 })
71
72 #define fpu_reg_write(val) \
73 do { \
74 uint64_t *__val = (val); \
75 asm volatile(".arch_extension fp\n" \
76 "ldp q0, q1, [%0], #32\n\t" \
77 "ldp q2, q3, [%0], #32\n\t" \
78 "ldp q4, q5, [%0], #32\n\t" \
79 "ldp q6, q7, [%0], #32\n\t" \
80 "ldp q8, q9, [%0], #32\n\t" \
81 "ldp q10, q11, [%0], #32\n\t" \
82 "ldp q12, q13, [%0], #32\n\t" \
83 "ldp q14, q15, [%0], #32\n\t" \
84 "ldp q16, q17, [%0], #32\n\t" \
85 "ldp q18, q19, [%0], #32\n\t" \
86 "ldp q20, q21, [%0], #32\n\t" \
87 "ldp q22, q23, [%0], #32\n\t" \
88 "ldp q24, q25, [%0], #32\n\t" \
89 "ldp q26, q27, [%0], #32\n\t" \
90 "ldp q28, q29, [%0], #32\n\t" \
91 "ldp q30, q31, [%0], #32\n\t" \
92 : "+r" (__val) \
93 : \
94 : "v0", "v1", "v2", "v3", \
95 "v4", "v5", "v6", "v7", \
96 "v8", "v9", "v10", "v11", \
97 "v12", "v13", "v14", \
98 "v15", "v16", "v17", \
99 "v18", "v19", "v20", \
100 "v21", "v22", "v23", \
101 "v24", "v25", "v26", \
102 "v27", "v28", "v29", \
103 "v30", "v31", "memory"); \
104 } while (0)
105
106 #ifdef CC_HAS_SVE
107 #define sve_reg_read(val) \
108 ({ \
109 uint64_t *__val = (val); \
110 asm volatile(".arch_extension sve\n" \
111 "str z0, [%0, #0, MUL VL]\n" \
112 "str z1, [%0, #1, MUL VL]\n" \
113 "str z2, [%0, #2, MUL VL]\n" \
114 "str z3, [%0, #3, MUL VL]\n" \
115 "str z4, [%0, #4, MUL VL]\n" \
116 "str z5, [%0, #5, MUL VL]\n" \
117 "str z6, [%0, #6, MUL VL]\n" \
118 "str z7, [%0, #7, MUL VL]\n" \
119 "str z8, [%0, #8, MUL VL]\n" \
120 "str z9, [%0, #9, MUL VL]\n" \
121 "str z10, [%0, #10, MUL VL]\n" \
122 "str z11, [%0, #11, MUL VL]\n" \
123 "str z12, [%0, #12, MUL VL]\n" \
124 "str z13, [%0, #13, MUL VL]\n" \
125 "str z14, [%0, #14, MUL VL]\n" \
126 "str z15, [%0, #15, MUL VL]\n" \
127 "str z16, [%0, #16, MUL VL]\n" \
128 "str z17, [%0, #17, MUL VL]\n" \
129 "str z18, [%0, #18, MUL VL]\n" \
130 "str z19, [%0, #19, MUL VL]\n" \
131 "str z20, [%0, #20, MUL VL]\n" \
132 "str z21, [%0, #21, MUL VL]\n" \
133 "str z22, [%0, #22, MUL VL]\n" \
134 "str z23, [%0, #23, MUL VL]\n" \
135 "str z24, [%0, #24, MUL VL]\n" \
136 "str z25, [%0, #25, MUL VL]\n" \
137 "str z26, [%0, #26, MUL VL]\n" \
138 "str z27, [%0, #27, MUL VL]\n" \
139 "str z28, [%0, #28, MUL VL]\n" \
140 "str z29, [%0, #29, MUL VL]\n" \
141 "str z30, [%0, #30, MUL VL]\n" \
142 "str z31, [%0, #31, MUL VL]\n" \
143 : \
144 : "r" (__val) \
145 : "z0", "z1", "z2", "z3", \
146 "z4", "z5", "z6", "z7", \
147 "z8", "z9", "z10", "z11", \
148 "z12", "z13", "z14", \
149 "z15", "z16", "z17", \
150 "z18", "z19", "z20", \
151 "z21", "z22", "z23", \
152 "z24", "z25", "z26", \
153 "z27", "z28", "z29", \
154 "z30", "z31", "memory"); \
155 })
156
157 #define sve_reg_write(val) \
158 ({ \
159 uint64_t *__val = (val); \
160 asm volatile(".arch_extension sve\n" \
161 "ldr z0, [%0, #0, MUL VL]\n" \
162 "ldr z1, [%0, #1, MUL VL]\n" \
163 "ldr z2, [%0, #2, MUL VL]\n" \
164 "ldr z3, [%0, #3, MUL VL]\n" \
165 "ldr z4, [%0, #4, MUL VL]\n" \
166 "ldr z5, [%0, #5, MUL VL]\n" \
167 "ldr z6, [%0, #6, MUL VL]\n" \
168 "ldr z7, [%0, #7, MUL VL]\n" \
169 "ldr z8, [%0, #8, MUL VL]\n" \
170 "ldr z9, [%0, #9, MUL VL]\n" \
171 "ldr z10, [%0, #10, MUL VL]\n" \
172 "ldr z11, [%0, #11, MUL VL]\n" \
173 "ldr z12, [%0, #12, MUL VL]\n" \
174 "ldr z13, [%0, #13, MUL VL]\n" \
175 "ldr z14, [%0, #14, MUL VL]\n" \
176 "ldr z15, [%0, #15, MUL VL]\n" \
177 "ldr z16, [%0, #16, MUL VL]\n" \
178 "ldr z17, [%0, #17, MUL VL]\n" \
179 "ldr z18, [%0, #18, MUL VL]\n" \
180 "ldr z19, [%0, #19, MUL VL]\n" \
181 "ldr z20, [%0, #20, MUL VL]\n" \
182 "ldr z21, [%0, #21, MUL VL]\n" \
183 "ldr z22, [%0, #22, MUL VL]\n" \
184 "ldr z23, [%0, #23, MUL VL]\n" \
185 "ldr z24, [%0, #24, MUL VL]\n" \
186 "ldr z25, [%0, #25, MUL VL]\n" \
187 "ldr z26, [%0, #26, MUL VL]\n" \
188 "ldr z27, [%0, #27, MUL VL]\n" \
189 "ldr z28, [%0, #28, MUL VL]\n" \
190 "ldr z29, [%0, #29, MUL VL]\n" \
191 "ldr z30, [%0, #30, MUL VL]\n" \
192 "ldr z31, [%0, #31, MUL VL]\n" \
193 : \
194 : "r" (__val) \
195 : "z0", "z1", "z2", "z3", \
196 "z4", "z5", "z6", "z7", \
197 "z8", "z9", "z10", "z11", \
198 "z12", "z13", "z14", \
199 "z15", "z16", "z17", \
200 "z18", "z19", "z20", \
201 "z21", "z22", "z23", \
202 "z24", "z25", "z26", \
203 "z27", "z28", "z29", \
204 "z30", "z31", "memory"); \
205 })
206 #else
207 #define sve_reg_read(val) report_abort("SVE: not supported")
208 #define sve_reg_write(val) report_abort("SVE: not supported")
209 #endif
210
nr_cpu_check(int nr)211 static void nr_cpu_check(int nr)
212 {
213 if (nr_cpus < nr)
214 report_abort("At least %d cpus required", nr);
215 }
216
217 /*
218 * check if the FPU/SIMD/SVE register contents are the same as
219 * the input data provided.
220 */
__fpuregs_testall(uint64_t * indata,int sve)221 static uint32_t __fpuregs_testall(uint64_t *indata, int sve)
222 {
223 /* 128b aligned array to read data into */
224 uint64_t outdata[FPU_QREG_MAX * 2]
225 __attribute__((aligned(sizeof(__uint128_t)))) = {
226 [0 ... ((FPU_QREG_MAX * 2) - 1)] = 0 };
227 uint8_t regcnt = 0;
228 uint32_t result = 0;
229
230 if (indata == NULL)
231 report_abort("invalid data pointer received");
232
233 /* Read data from FPU/SVE registers */
234 if (sve)
235 sve_reg_read(outdata);
236 else
237 fpu_reg_read(outdata);
238
239 /* Check is the data is the same */
240 for (regcnt = 0; regcnt < (FPU_QREG_MAX * 2); regcnt += 2) {
241 if ((outdata[regcnt] != indata[regcnt]) ||
242 (outdata[regcnt + 1] != indata[regcnt + 1])) {
243 report_info(
244 "%s save/restore failed for reg: %c%u expected: %lx_%lx received: %lx_%lx\n",
245 sve ? "SVE" : "FPU/SIMD",
246 sve ? 'z' : 'q',
247 regcnt / 2,
248 indata[regcnt + 1], indata[regcnt],
249 outdata[regcnt + 1], outdata[regcnt]);
250 } else {
251 /* populate a bitmask indicating which
252 * registers passed/failed
253 */
254 result |= (1 << (regcnt / 2));
255 }
256 }
257
258 return result;
259 }
260
261 /*
262 * Write randomly sampled data into the FPU/SIMD registers.
263 */
__fpuregs_writeall_random(uint64_t ** indata,int sve)264 static void __fpuregs_writeall_random(uint64_t **indata, int sve)
265 {
266 /* allocate 128b aligned memory */
267 *indata = memalign(sizeof(__uint128_t), sizeof(uint64_t) * FPU_QREG_MAX);
268
269 if (system_supports_rndr()) {
270 /* Populate memory with random data */
271 for (unsigned int i = 0; i < (FPU_QREG_MAX * 2); i++)
272 while (!arch_collect_entropy(&(*indata)[i])) {}
273 } else {
274 /* Populate memory with data from the counter register */
275 for (unsigned int i = 0; i < (FPU_QREG_MAX * 2); i++)
276 (*indata)[i] = get_cntvct();
277 }
278
279 /* Write data into FPU registers */
280 if (sve)
281 sve_reg_write(*indata);
282 else
283 fpu_reg_write(*indata);
284 }
285
fpuregs_writeall_run(void * data)286 static void fpuregs_writeall_run(void *data)
287 {
288 uint64_t **indata = (uint64_t **)data;
289
290 __fpuregs_writeall_random(indata, 0);
291 }
292
sveregs_writeall_run(void * data)293 static void sveregs_writeall_run(void *data)
294 {
295 uint64_t **indata = (uint64_t **)data;
296
297 __fpuregs_writeall_random(indata, 1);
298 }
299
fpuregs_testall_run(void * data)300 static void fpuregs_testall_run(void *data)
301 {
302 uint64_t *indata = (uint64_t *)data;
303 uint32_t result = 0;
304
305 result = __fpuregs_testall(indata, 0);
306 report((result == FPU_RESULT_PASS),
307 "FPU/SIMD register save/restore mask: 0x%x", result);
308 }
309
sveregs_testall_run(void * data)310 static void sveregs_testall_run(void *data)
311 {
312 uint64_t *indata = (uint64_t *)data;
313 uint32_t result = 0;
314
315 result = __fpuregs_testall(indata, 1);
316 report((result == FPU_RESULT_PASS),
317 "SVE register save/restore mask: 0x%x", result);
318 }
319
320 /*
321 * This test uses two CPUs to test FPU/SIMD save/restore
322 * CPU1 writes random data into FPU/SIMD registers,
323 * CPU0 corrupts/overwrites the data and finally CPU1 checks
324 * if the data remains unchanged in its context.
325 */
fpuregs_context_switch_cpu1(int sve)326 static void fpuregs_context_switch_cpu1(int sve)
327 {
328 int target = CPU1_ID;
329 uint64_t *indata_remote = NULL;
330 uint64_t *indata_local = NULL;
331
332 /* write data from CPU1 */
333 on_cpu(target, sve ? sveregs_writeall_run
334 : fpuregs_writeall_run,
335 &indata_remote);
336
337 /* Overwrite from CPU0 */
338 __fpuregs_writeall_random(&indata_local, sve);
339
340 /* Check data consistency */
341 on_cpu(target, sve ? sveregs_testall_run
342 : fpuregs_testall_run,
343 indata_remote);
344
345 free(indata_remote);
346 free(indata_local);
347 }
348
349 /*
350 * This test uses two CPUs to test FPU/SIMD save/restore
351 * CPU0 writes random data into FPU/SIMD registers,
352 * CPU1 corrupts/overwrites the data and finally CPU0 checks if
353 * the data remains unchanged in its context.
354 */
fpuregs_context_switch_cpu0(int sve)355 static void fpuregs_context_switch_cpu0(int sve)
356 {
357 int target = CPU1_ID;
358 uint64_t *indata_local = NULL;
359 uint64_t *indata_remote = NULL;
360 uint32_t result = 0;
361
362 /* write data from CPU0 */
363 __fpuregs_writeall_random(&indata_local, sve);
364
365 /* Overwrite from CPU1 */
366 on_cpu(target, sve ? sveregs_writeall_run
367 : fpuregs_writeall_run,
368 &indata_remote);
369
370 /* Check data consistency */
371 result = __fpuregs_testall(indata_local, sve);
372 report((result == FPU_RESULT_PASS),
373 "%s register save/restore mask: 0x%x", sve ? "SVE" : "FPU/SIMD", result);
374
375 free(indata_remote);
376 free(indata_local);
377 }
378
379 /*
380 * Checks if during context switch, FPU/SIMD registers
381 * are saved/restored.
382 */
fpuregs_context_switch(void)383 static void fpuregs_context_switch(void)
384 {
385 fpuregs_context_switch_cpu0(0);
386 fpuregs_context_switch_cpu1(0);
387 }
388
389 /*
390 * Checks if during context switch, SVE registers
391 * are saved/restored.
392 */
sveregs_context_switch(void)393 static void sveregs_context_switch(void)
394 {
395 unsigned long zcr = read_sysreg(ZCR_EL1);
396
397 // Set the SVE vector length to 128-bits
398 write_sysreg(zcr & ~ZCR_EL1_LEN, ZCR_EL1);
399
400 fpuregs_context_switch_cpu0(1);
401 fpuregs_context_switch_cpu1(1);
402 }
403
should_run_sve_tests(void)404 static bool should_run_sve_tests(void)
405 {
406 #ifdef CC_HAS_SVE
407 if (system_supports_sve())
408 return true;
409 #endif
410 return false;
411 }
412
main(int argc,char ** argv)413 int main(int argc, char **argv)
414 {
415 report_prefix_pushf("fpu");
416
417 nr_cpu_check(CPUS_MAX);
418 fpuregs_context_switch();
419
420 if (should_run_sve_tests())
421 sveregs_context_switch();
422
423 return report_summary();
424 }
425