1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Copyright (C) 2023 Arm Limited. 4 * All rights reserved. 5 */ 6 7 #include <libcflat.h> 8 #include <asm/smp.h> 9 #include <stdlib.h> 10 11 #define CPU0_ID 0 12 #define CPU1_ID (CPU0_ID + 1) 13 #define CPUS_MAX (CPU1_ID + 1) 14 #define FPU_QREG_MAX 32 15 #define FPU_RESULT_PASS (-1U) 16 17 /* 18 * Write 8 bytes of random data in random. Returns true on success, false on 19 * failure. 20 */ 21 static inline bool arch_collect_entropy(uint64_t *random) 22 { 23 unsigned long ret; 24 25 asm volatile( 26 " mrs %[ptr], " xstr(RNDR) "\n" 27 " cset %[ret], ne\n" /* RNDR sets NZCV to 0b0100 on failure */ 28 : 29 [ret] "=r" (ret), 30 [ptr] "=r" (*random) 31 : 32 : "cc" 33 ); 34 35 return ret == 1; 36 } 37 38 #define fpu_reg_read(val) \ 39 ({ \ 40 uint64_t *__val = (val); \ 41 asm volatile(".arch_extension fp\n" \ 42 "stp q0, q1, [%0], #32\n\t" \ 43 "stp q2, q3, [%0], #32\n\t" \ 44 "stp q4, q5, [%0], #32\n\t" \ 45 "stp q6, q7, [%0], #32\n\t" \ 46 "stp q8, q9, [%0], #32\n\t" \ 47 "stp q10, q11, [%0], #32\n\t" \ 48 "stp q12, q13, [%0], #32\n\t" \ 49 "stp q14, q15, [%0], #32\n\t" \ 50 "stp q16, q17, [%0], #32\n\t" \ 51 "stp q18, q19, [%0], #32\n\t" \ 52 "stp q20, q21, [%0], #32\n\t" \ 53 "stp q22, q23, [%0], #32\n\t" \ 54 "stp q24, q25, [%0], #32\n\t" \ 55 "stp q26, q27, [%0], #32\n\t" \ 56 "stp q28, q29, [%0], #32\n\t" \ 57 "stp q30, q31, [%0], #32\n\t" \ 58 : "+r" (__val) \ 59 : \ 60 : "v0", "v1", "v2", "v3", \ 61 "v4", "v5", "v6", "v7", \ 62 "v8", "v9", "v10", "v11", \ 63 "v12", "v13", "v14", \ 64 "v15", "v16", "v17", \ 65 "v18", "v19", "v20", \ 66 "v21", "v22", "v23", \ 67 "v24", "v25", "v26", \ 68 "v27", "v28", "v29", \ 69 "v30", "v31", "memory"); \ 70 }) 71 72 #define fpu_reg_write(val) \ 73 do { \ 74 uint64_t *__val = (val); \ 75 asm volatile(".arch_extension fp\n" \ 76 "ldp q0, q1, [%0], #32\n\t" \ 77 "ldp q2, q3, [%0], #32\n\t" \ 78 "ldp q4, q5, [%0], #32\n\t" \ 79 "ldp q6, q7, [%0], #32\n\t" \ 80 "ldp q8, q9, [%0], #32\n\t" \ 81 "ldp q10, q11, [%0], #32\n\t" \ 82 "ldp q12, q13, [%0], #32\n\t" \ 83 "ldp q14, q15, [%0], #32\n\t" \ 84 "ldp q16, q17, [%0], #32\n\t" \ 85 "ldp q18, q19, [%0], #32\n\t" \ 86 "ldp q20, q21, [%0], #32\n\t" \ 87 "ldp q22, q23, [%0], #32\n\t" \ 88 "ldp q24, q25, [%0], #32\n\t" \ 89 "ldp q26, q27, [%0], #32\n\t" \ 90 "ldp q28, q29, [%0], #32\n\t" \ 91 "ldp q30, q31, [%0], #32\n\t" \ 92 : "+r" (__val) \ 93 : \ 94 : "v0", "v1", "v2", "v3", \ 95 "v4", "v5", "v6", "v7", \ 96 "v8", "v9", "v10", "v11", \ 97 "v12", "v13", "v14", \ 98 "v15", "v16", "v17", \ 99 "v18", "v19", "v20", \ 100 "v21", "v22", "v23", \ 101 "v24", "v25", "v26", \ 102 "v27", "v28", "v29", \ 103 "v30", "v31", "memory"); \ 104 } while (0) 105 106 #ifdef CC_HAS_SVE 107 #define sve_reg_read(val) \ 108 ({ \ 109 uint64_t *__val = (val); \ 110 asm volatile(".arch_extension sve\n" \ 111 "str z0, [%0, #0, MUL VL]\n" \ 112 "str z1, [%0, #1, MUL VL]\n" \ 113 "str z2, [%0, #2, MUL VL]\n" \ 114 "str z3, [%0, #3, MUL VL]\n" \ 115 "str z4, [%0, #4, MUL VL]\n" \ 116 "str z5, [%0, #5, MUL VL]\n" \ 117 "str z6, [%0, #6, MUL VL]\n" \ 118 "str z7, [%0, #7, MUL VL]\n" \ 119 "str z8, [%0, #8, MUL VL]\n" \ 120 "str z9, [%0, #9, MUL VL]\n" \ 121 "str z10, [%0, #10, MUL VL]\n" \ 122 "str z11, [%0, #11, MUL VL]\n" \ 123 "str z12, [%0, #12, MUL VL]\n" \ 124 "str z13, [%0, #13, MUL VL]\n" \ 125 "str z14, [%0, #14, MUL VL]\n" \ 126 "str z15, [%0, #15, MUL VL]\n" \ 127 "str z16, [%0, #16, MUL VL]\n" \ 128 "str z17, [%0, #17, MUL VL]\n" \ 129 "str z18, [%0, #18, MUL VL]\n" \ 130 "str z19, [%0, #19, MUL VL]\n" \ 131 "str z20, [%0, #20, MUL VL]\n" \ 132 "str z21, [%0, #21, MUL VL]\n" \ 133 "str z22, [%0, #22, MUL VL]\n" \ 134 "str z23, [%0, #23, MUL VL]\n" \ 135 "str z24, [%0, #24, MUL VL]\n" \ 136 "str z25, [%0, #25, MUL VL]\n" \ 137 "str z26, [%0, #26, MUL VL]\n" \ 138 "str z27, [%0, #27, MUL VL]\n" \ 139 "str z28, [%0, #28, MUL VL]\n" \ 140 "str z29, [%0, #29, MUL VL]\n" \ 141 "str z30, [%0, #30, MUL VL]\n" \ 142 "str z31, [%0, #31, MUL VL]\n" \ 143 : \ 144 : "r" (__val) \ 145 : "z0", "z1", "z2", "z3", \ 146 "z4", "z5", "z6", "z7", \ 147 "z8", "z9", "z10", "z11", \ 148 "z12", "z13", "z14", \ 149 "z15", "z16", "z17", \ 150 "z18", "z19", "z20", \ 151 "z21", "z22", "z23", \ 152 "z24", "z25", "z26", \ 153 "z27", "z28", "z29", \ 154 "z30", "z31", "memory"); \ 155 }) 156 157 #define sve_reg_write(val) \ 158 ({ \ 159 uint64_t *__val = (val); \ 160 asm volatile(".arch_extension sve\n" \ 161 "ldr z0, [%0, #0, MUL VL]\n" \ 162 "ldr z1, [%0, #1, MUL VL]\n" \ 163 "ldr z2, [%0, #2, MUL VL]\n" \ 164 "ldr z3, [%0, #3, MUL VL]\n" \ 165 "ldr z4, [%0, #4, MUL VL]\n" \ 166 "ldr z5, [%0, #5, MUL VL]\n" \ 167 "ldr z6, [%0, #6, MUL VL]\n" \ 168 "ldr z7, [%0, #7, MUL VL]\n" \ 169 "ldr z8, [%0, #8, MUL VL]\n" \ 170 "ldr z9, [%0, #9, MUL VL]\n" \ 171 "ldr z10, [%0, #10, MUL VL]\n" \ 172 "ldr z11, [%0, #11, MUL VL]\n" \ 173 "ldr z12, [%0, #12, MUL VL]\n" \ 174 "ldr z13, [%0, #13, MUL VL]\n" \ 175 "ldr z14, [%0, #14, MUL VL]\n" \ 176 "ldr z15, [%0, #15, MUL VL]\n" \ 177 "ldr z16, [%0, #16, MUL VL]\n" \ 178 "ldr z17, [%0, #17, MUL VL]\n" \ 179 "ldr z18, [%0, #18, MUL VL]\n" \ 180 "ldr z19, [%0, #19, MUL VL]\n" \ 181 "ldr z20, [%0, #20, MUL VL]\n" \ 182 "ldr z21, [%0, #21, MUL VL]\n" \ 183 "ldr z22, [%0, #22, MUL VL]\n" \ 184 "ldr z23, [%0, #23, MUL VL]\n" \ 185 "ldr z24, [%0, #24, MUL VL]\n" \ 186 "ldr z25, [%0, #25, MUL VL]\n" \ 187 "ldr z26, [%0, #26, MUL VL]\n" \ 188 "ldr z27, [%0, #27, MUL VL]\n" \ 189 "ldr z28, [%0, #28, MUL VL]\n" \ 190 "ldr z29, [%0, #29, MUL VL]\n" \ 191 "ldr z30, [%0, #30, MUL VL]\n" \ 192 "ldr z31, [%0, #31, MUL VL]\n" \ 193 : \ 194 : "r" (__val) \ 195 : "z0", "z1", "z2", "z3", \ 196 "z4", "z5", "z6", "z7", \ 197 "z8", "z9", "z10", "z11", \ 198 "z12", "z13", "z14", \ 199 "z15", "z16", "z17", \ 200 "z18", "z19", "z20", \ 201 "z21", "z22", "z23", \ 202 "z24", "z25", "z26", \ 203 "z27", "z28", "z29", \ 204 "z30", "z31", "memory"); \ 205 }) 206 #else 207 #define sve_reg_read(val) report_abort("SVE: not supported") 208 #define sve_reg_write(val) report_abort("SVE: not supported") 209 #endif 210 211 static void nr_cpu_check(int nr) 212 { 213 if (nr_cpus < nr) 214 report_abort("At least %d cpus required", nr); 215 } 216 217 /* 218 * check if the FPU/SIMD/SVE register contents are the same as 219 * the input data provided. 220 */ 221 static uint32_t __fpuregs_testall(uint64_t *indata, int sve) 222 { 223 /* 128b aligned array to read data into */ 224 uint64_t outdata[FPU_QREG_MAX * 2] 225 __attribute__((aligned(sizeof(__uint128_t)))) = { 226 [0 ... ((FPU_QREG_MAX * 2) - 1)] = 0 }; 227 uint8_t regcnt = 0; 228 uint32_t result = 0; 229 230 if (indata == NULL) 231 report_abort("invalid data pointer received"); 232 233 /* Read data from FPU/SVE registers */ 234 if (sve) 235 sve_reg_read(outdata); 236 else 237 fpu_reg_read(outdata); 238 239 /* Check is the data is the same */ 240 for (regcnt = 0; regcnt < (FPU_QREG_MAX * 2); regcnt += 2) { 241 if ((outdata[regcnt] != indata[regcnt]) || 242 (outdata[regcnt + 1] != indata[regcnt + 1])) { 243 report_info( 244 "%s save/restore failed for reg: %c%u expected: %lx_%lx received: %lx_%lx\n", 245 sve ? "SVE" : "FPU/SIMD", 246 sve ? 'z' : 'q', 247 regcnt / 2, 248 indata[regcnt + 1], indata[regcnt], 249 outdata[regcnt + 1], outdata[regcnt]); 250 } else { 251 /* populate a bitmask indicating which 252 * registers passed/failed 253 */ 254 result |= (1 << (regcnt / 2)); 255 } 256 } 257 258 return result; 259 } 260 261 /* 262 * Write randomly sampled data into the FPU/SIMD registers. 263 */ 264 static void __fpuregs_writeall_random(uint64_t **indata, int sve) 265 { 266 /* allocate 128b aligned memory */ 267 *indata = memalign(sizeof(__uint128_t), sizeof(uint64_t) * FPU_QREG_MAX); 268 269 if (system_supports_rndr()) { 270 /* Populate memory with random data */ 271 for (unsigned int i = 0; i < (FPU_QREG_MAX * 2); i++) 272 while (!arch_collect_entropy(&(*indata)[i])) {} 273 } else { 274 /* Populate memory with data from the counter register */ 275 for (unsigned int i = 0; i < (FPU_QREG_MAX * 2); i++) 276 (*indata)[i] = get_cntvct(); 277 } 278 279 /* Write data into FPU registers */ 280 if (sve) 281 sve_reg_write(*indata); 282 else 283 fpu_reg_write(*indata); 284 } 285 286 static void fpuregs_writeall_run(void *data) 287 { 288 uint64_t **indata = (uint64_t **)data; 289 290 __fpuregs_writeall_random(indata, 0); 291 } 292 293 static void sveregs_writeall_run(void *data) 294 { 295 uint64_t **indata = (uint64_t **)data; 296 297 __fpuregs_writeall_random(indata, 1); 298 } 299 300 static void fpuregs_testall_run(void *data) 301 { 302 uint64_t *indata = (uint64_t *)data; 303 uint32_t result = 0; 304 305 result = __fpuregs_testall(indata, 0); 306 report((result == FPU_RESULT_PASS), 307 "FPU/SIMD register save/restore mask: 0x%x", result); 308 } 309 310 static void sveregs_testall_run(void *data) 311 { 312 uint64_t *indata = (uint64_t *)data; 313 uint32_t result = 0; 314 315 result = __fpuregs_testall(indata, 1); 316 report((result == FPU_RESULT_PASS), 317 "SVE register save/restore mask: 0x%x", result); 318 } 319 320 /* 321 * This test uses two CPUs to test FPU/SIMD save/restore 322 * CPU1 writes random data into FPU/SIMD registers, 323 * CPU0 corrupts/overwrites the data and finally CPU1 checks 324 * if the data remains unchanged in its context. 325 */ 326 static void fpuregs_context_switch_cpu1(int sve) 327 { 328 int target = CPU1_ID; 329 uint64_t *indata_remote = NULL; 330 uint64_t *indata_local = NULL; 331 332 /* write data from CPU1 */ 333 on_cpu(target, sve ? sveregs_writeall_run 334 : fpuregs_writeall_run, 335 &indata_remote); 336 337 /* Overwrite from CPU0 */ 338 __fpuregs_writeall_random(&indata_local, sve); 339 340 /* Check data consistency */ 341 on_cpu(target, sve ? sveregs_testall_run 342 : fpuregs_testall_run, 343 indata_remote); 344 345 free(indata_remote); 346 free(indata_local); 347 } 348 349 /* 350 * This test uses two CPUs to test FPU/SIMD save/restore 351 * CPU0 writes random data into FPU/SIMD registers, 352 * CPU1 corrupts/overwrites the data and finally CPU0 checks if 353 * the data remains unchanged in its context. 354 */ 355 static void fpuregs_context_switch_cpu0(int sve) 356 { 357 int target = CPU1_ID; 358 uint64_t *indata_local = NULL; 359 uint64_t *indata_remote = NULL; 360 uint32_t result = 0; 361 362 /* write data from CPU0 */ 363 __fpuregs_writeall_random(&indata_local, sve); 364 365 /* Overwrite from CPU1 */ 366 on_cpu(target, sve ? sveregs_writeall_run 367 : fpuregs_writeall_run, 368 &indata_remote); 369 370 /* Check data consistency */ 371 result = __fpuregs_testall(indata_local, sve); 372 report((result == FPU_RESULT_PASS), 373 "%s register save/restore mask: 0x%x", sve ? "SVE" : "FPU/SIMD", result); 374 375 free(indata_remote); 376 free(indata_local); 377 } 378 379 /* 380 * Checks if during context switch, FPU/SIMD registers 381 * are saved/restored. 382 */ 383 static void fpuregs_context_switch(void) 384 { 385 fpuregs_context_switch_cpu0(0); 386 fpuregs_context_switch_cpu1(0); 387 } 388 389 /* 390 * Checks if during context switch, SVE registers 391 * are saved/restored. 392 */ 393 static void sveregs_context_switch(void) 394 { 395 unsigned long zcr = read_sysreg(ZCR_EL1); 396 397 // Set the SVE vector length to 128-bits 398 write_sysreg(zcr & ~ZCR_EL1_LEN, ZCR_EL1); 399 400 fpuregs_context_switch_cpu0(1); 401 fpuregs_context_switch_cpu1(1); 402 } 403 404 static bool should_run_sve_tests(void) 405 { 406 #ifdef CC_HAS_SVE 407 if (system_supports_sve()) 408 return true; 409 #endif 410 return false; 411 } 412 413 int main(int argc, char **argv) 414 { 415 report_prefix_pushf("fpu"); 416 417 nr_cpu_check(CPUS_MAX); 418 fpuregs_context_switch(); 419 420 if (should_run_sve_tests()) 421 sveregs_context_switch(); 422 423 return report_summary(); 424 } 425