1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Copyright (C) 2023 Arm Limited. 4 * All rights reserved. 5 */ 6 7 #include <libcflat.h> 8 #include <asm/smp.h> 9 #include <stdlib.h> 10 11 #define CPU0_ID 0 12 #define CPU1_ID (CPU0_ID + 1) 13 #define CPUS_MAX (CPU1_ID + 1) 14 #define FPU_QREG_MAX 32 15 #define FPU_RESULT_PASS (-1U) 16 17 /* 18 * Write 8 bytes of random data in random. Returns true on success, false on 19 * failure. 20 */ 21 static inline bool arch_collect_entropy(uint64_t *random) 22 { 23 unsigned long ret; 24 25 asm volatile( 26 " mrs %[ptr], " xstr(RNDR) "\n" 27 " cset %[ret], ne\n" /* RNDR sets NZCV to 0b0100 on failure */ 28 : 29 [ret] "=r" (ret), 30 [ptr] "=r" (*random) 31 : 32 : "cc" 33 ); 34 35 return ret == 1; 36 } 37 38 #define fpu_reg_read(val) \ 39 ({ \ 40 uint64_t *__val = (val); \ 41 asm volatile("stp q0, q1, [%0], #32\n\t" \ 42 "stp q2, q3, [%0], #32\n\t" \ 43 "stp q4, q5, [%0], #32\n\t" \ 44 "stp q6, q7, [%0], #32\n\t" \ 45 "stp q8, q9, [%0], #32\n\t" \ 46 "stp q10, q11, [%0], #32\n\t" \ 47 "stp q12, q13, [%0], #32\n\t" \ 48 "stp q14, q15, [%0], #32\n\t" \ 49 "stp q16, q17, [%0], #32\n\t" \ 50 "stp q18, q19, [%0], #32\n\t" \ 51 "stp q20, q21, [%0], #32\n\t" \ 52 "stp q22, q23, [%0], #32\n\t" \ 53 "stp q24, q25, [%0], #32\n\t" \ 54 "stp q26, q27, [%0], #32\n\t" \ 55 "stp q28, q29, [%0], #32\n\t" \ 56 "stp q30, q31, [%0], #32\n\t" \ 57 : "=r" (__val) \ 58 : \ 59 : "q0", "q1", "q2", "q3", \ 60 "q4", "q5", "q6", "q7", \ 61 "q8", "q9", "q10", "q11", \ 62 "q12", "q13", "q14", \ 63 "q15", "q16", "q17", \ 64 "q18", "q19", "q20", \ 65 "q21", "q22", "q23", \ 66 "q24", "q25", "q26", \ 67 "q27", "q28", "q29", \ 68 "q30", "q31", "memory"); \ 69 }) 70 71 #define fpu_reg_write(val) \ 72 do { \ 73 uint64_t *__val = (val); \ 74 asm volatile("ldp q0, q1, [%0], #32\n\t" \ 75 "ldp q2, q3, [%0], #32\n\t" \ 76 "ldp q4, q5, [%0], #32\n\t" \ 77 "ldp q6, q7, [%0], #32\n\t" \ 78 "ldp q8, q9, [%0], #32\n\t" \ 79 "ldp q10, q11, [%0], #32\n\t" \ 80 "ldp q12, q13, [%0], #32\n\t" \ 81 "ldp q14, q15, [%0], #32\n\t" \ 82 "ldp q16, q17, [%0], #32\n\t" \ 83 "ldp q18, q19, [%0], #32\n\t" \ 84 "ldp q20, q21, [%0], #32\n\t" \ 85 "ldp q22, q23, [%0], #32\n\t" \ 86 "ldp q24, q25, [%0], #32\n\t" \ 87 "ldp q26, q27, [%0], #32\n\t" \ 88 "ldp q28, q29, [%0], #32\n\t" \ 89 "ldp q30, q31, [%0], #32\n\t" \ 90 : \ 91 : "r" (__val) \ 92 : "q0", "q1", "q2", "q3", \ 93 "q4", "q5", "q6", "q7", \ 94 "q8", "q9", "q10", "q11", \ 95 "q12", "q13", "q14", \ 96 "q15", "q16", "q17", \ 97 "q18", "q19", "q20", \ 98 "q21", "q22", "q23", \ 99 "q24", "q25", "q26", \ 100 "q27", "q28", "q29", \ 101 "q30", "q31", "memory"); \ 102 } while (0) 103 104 #ifdef CC_HAS_SVE 105 #define sve_reg_read(val) \ 106 ({ \ 107 uint64_t *__val = (val); \ 108 asm volatile(".arch_extension sve\n" \ 109 "str z0, [%0, #0, MUL VL]\n" \ 110 "str z1, [%0, #1, MUL VL]\n" \ 111 "str z2, [%0, #2, MUL VL]\n" \ 112 "str z3, [%0, #3, MUL VL]\n" \ 113 "str z4, [%0, #4, MUL VL]\n" \ 114 "str z5, [%0, #5, MUL VL]\n" \ 115 "str z6, [%0, #6, MUL VL]\n" \ 116 "str z7, [%0, #7, MUL VL]\n" \ 117 "str z8, [%0, #8, MUL VL]\n" \ 118 "str z9, [%0, #9, MUL VL]\n" \ 119 "str z10, [%0, #10, MUL VL]\n" \ 120 "str z11, [%0, #11, MUL VL]\n" \ 121 "str z12, [%0, #12, MUL VL]\n" \ 122 "str z13, [%0, #13, MUL VL]\n" \ 123 "str z14, [%0, #14, MUL VL]\n" \ 124 "str z15, [%0, #15, MUL VL]\n" \ 125 "str z16, [%0, #16, MUL VL]\n" \ 126 "str z17, [%0, #17, MUL VL]\n" \ 127 "str z18, [%0, #18, MUL VL]\n" \ 128 "str z19, [%0, #19, MUL VL]\n" \ 129 "str z20, [%0, #20, MUL VL]\n" \ 130 "str z21, [%0, #21, MUL VL]\n" \ 131 "str z22, [%0, #22, MUL VL]\n" \ 132 "str z23, [%0, #23, MUL VL]\n" \ 133 "str z24, [%0, #24, MUL VL]\n" \ 134 "str z25, [%0, #25, MUL VL]\n" \ 135 "str z26, [%0, #26, MUL VL]\n" \ 136 "str z27, [%0, #27, MUL VL]\n" \ 137 "str z28, [%0, #28, MUL VL]\n" \ 138 "str z29, [%0, #29, MUL VL]\n" \ 139 "str z30, [%0, #30, MUL VL]\n" \ 140 "str z31, [%0, #31, MUL VL]\n" \ 141 : "=r" (__val) \ 142 : \ 143 : "z0", "z1", "z2", "z3", \ 144 "z4", "z5", "z6", "z7", \ 145 "z8", "z9", "z10", "z11", \ 146 "z12", "z13", "z14", \ 147 "z15", "z16", "z17", \ 148 "z18", "z19", "z20", \ 149 "z21", "z22", "z23", \ 150 "z24", "z25", "z26", \ 151 "z27", "z28", "z29", \ 152 "z30", "z31", "memory"); \ 153 }) 154 155 #define sve_reg_write(val) \ 156 ({ \ 157 uint64_t *__val = (val); \ 158 asm volatile(".arch_extension sve\n" \ 159 "ldr z0, [%0, #0, MUL VL]\n" \ 160 "ldr z1, [%0, #1, MUL VL]\n" \ 161 "ldr z2, [%0, #2, MUL VL]\n" \ 162 "ldr z3, [%0, #3, MUL VL]\n" \ 163 "ldr z4, [%0, #4, MUL VL]\n" \ 164 "ldr z5, [%0, #5, MUL VL]\n" \ 165 "ldr z6, [%0, #6, MUL VL]\n" \ 166 "ldr z7, [%0, #7, MUL VL]\n" \ 167 "ldr z8, [%0, #8, MUL VL]\n" \ 168 "ldr z9, [%0, #9, MUL VL]\n" \ 169 "ldr z10, [%0, #10, MUL VL]\n" \ 170 "ldr z11, [%0, #11, MUL VL]\n" \ 171 "ldr z12, [%0, #12, MUL VL]\n" \ 172 "ldr z13, [%0, #13, MUL VL]\n" \ 173 "ldr z14, [%0, #14, MUL VL]\n" \ 174 "ldr z15, [%0, #15, MUL VL]\n" \ 175 "ldr z16, [%0, #16, MUL VL]\n" \ 176 "ldr z17, [%0, #17, MUL VL]\n" \ 177 "ldr z18, [%0, #18, MUL VL]\n" \ 178 "ldr z19, [%0, #19, MUL VL]\n" \ 179 "ldr z20, [%0, #20, MUL VL]\n" \ 180 "ldr z21, [%0, #21, MUL VL]\n" \ 181 "ldr z22, [%0, #22, MUL VL]\n" \ 182 "ldr z23, [%0, #23, MUL VL]\n" \ 183 "ldr z24, [%0, #24, MUL VL]\n" \ 184 "ldr z25, [%0, #25, MUL VL]\n" \ 185 "ldr z26, [%0, #26, MUL VL]\n" \ 186 "ldr z27, [%0, #27, MUL VL]\n" \ 187 "ldr z28, [%0, #28, MUL VL]\n" \ 188 "ldr z29, [%0, #29, MUL VL]\n" \ 189 "ldr z30, [%0, #30, MUL VL]\n" \ 190 "ldr z31, [%0, #31, MUL VL]\n" \ 191 : \ 192 : "r" (__val) \ 193 : "z0", "z1", "z2", "z3", \ 194 "z4", "z5", "z6", "z7", \ 195 "z8", "z9", "z10", "z11", \ 196 "z12", "z13", "z14", \ 197 "z15", "z16", "z17", \ 198 "z18", "z19", "z20", \ 199 "z21", "z22", "z23", \ 200 "z24", "z25", "z26", \ 201 "z27", "z28", "z29", \ 202 "z30", "z31", "memory"); \ 203 }) 204 #else 205 #define sve_reg_read(val) report_abort("SVE: not supported") 206 #define sve_reg_write(val) report_abort("SVE: not supported") 207 #endif 208 209 static void nr_cpu_check(int nr) 210 { 211 if (nr_cpus < nr) 212 report_abort("At least %d cpus required", nr); 213 } 214 215 /* 216 * check if the FPU/SIMD/SVE register contents are the same as 217 * the input data provided. 218 */ 219 static uint32_t __fpuregs_testall(uint64_t *indata, int sve) 220 { 221 /* 128b aligned array to read data into */ 222 uint64_t outdata[FPU_QREG_MAX * 2] 223 __attribute__((aligned(sizeof(__uint128_t)))) = { 224 [0 ... ((FPU_QREG_MAX * 2) - 1)] = 0 }; 225 uint8_t regcnt = 0; 226 uint32_t result = 0; 227 228 if (indata == NULL) 229 report_abort("invalid data pointer received"); 230 231 /* Read data from FPU/SVE registers */ 232 if (sve) 233 sve_reg_read(outdata); 234 else 235 fpu_reg_read(outdata); 236 237 /* Check is the data is the same */ 238 for (regcnt = 0; regcnt < (FPU_QREG_MAX * 2); regcnt += 2) { 239 if ((outdata[regcnt] != indata[regcnt]) || 240 (outdata[regcnt + 1] != indata[regcnt + 1])) { 241 report_info( 242 "%s save/restore failed for reg: %c%u expected: %lx_%lx received: %lx_%lx\n", 243 sve ? "SVE" : "FPU/SIMD", 244 sve ? 'z' : 'q', 245 regcnt / 2, 246 indata[regcnt + 1], indata[regcnt], 247 outdata[regcnt + 1], outdata[regcnt]); 248 } else { 249 /* populate a bitmask indicating which 250 * registers passed/failed 251 */ 252 result |= (1 << (regcnt / 2)); 253 } 254 } 255 256 return result; 257 } 258 259 /* 260 * Write randomly sampled data into the FPU/SIMD registers. 261 */ 262 static void __fpuregs_writeall_random(uint64_t **indata, int sve) 263 { 264 /* allocate 128b aligned memory */ 265 *indata = memalign(sizeof(__uint128_t), sizeof(uint64_t) * FPU_QREG_MAX); 266 267 if (system_supports_rndr()) { 268 /* Populate memory with random data */ 269 for (unsigned int i = 0; i < (FPU_QREG_MAX * 2); i++) 270 while (!arch_collect_entropy(&(*indata)[i])) {} 271 } else { 272 /* Populate memory with data from the counter register */ 273 for (unsigned int i = 0; i < (FPU_QREG_MAX * 2); i++) 274 (*indata)[i] = get_cntvct(); 275 } 276 277 /* Write data into FPU registers */ 278 if (sve) 279 sve_reg_write(*indata); 280 else 281 fpu_reg_write(*indata); 282 } 283 284 static void fpuregs_writeall_run(void *data) 285 { 286 uint64_t **indata = (uint64_t **)data; 287 288 __fpuregs_writeall_random(indata, 0); 289 } 290 291 static void sveregs_writeall_run(void *data) 292 { 293 uint64_t **indata = (uint64_t **)data; 294 295 __fpuregs_writeall_random(indata, 1); 296 } 297 298 static void fpuregs_testall_run(void *data) 299 { 300 uint64_t *indata = (uint64_t *)data; 301 uint32_t result = 0; 302 303 result = __fpuregs_testall(indata, 0); 304 report((result == FPU_RESULT_PASS), 305 "FPU/SIMD register save/restore mask: 0x%x", result); 306 } 307 308 static void sveregs_testall_run(void *data) 309 { 310 uint64_t *indata = (uint64_t *)data; 311 uint32_t result = 0; 312 313 result = __fpuregs_testall(indata, 1); 314 report((result == FPU_RESULT_PASS), 315 "SVE register save/restore mask: 0x%x", result); 316 } 317 318 /* 319 * This test uses two CPUs to test FPU/SIMD save/restore 320 * CPU1 writes random data into FPU/SIMD registers, 321 * CPU0 corrupts/overwrites the data and finally CPU1 checks 322 * if the data remains unchanged in its context. 323 */ 324 static void fpuregs_context_switch_cpu1(int sve) 325 { 326 int target = CPU1_ID; 327 uint64_t *indata_remote = NULL; 328 uint64_t *indata_local = NULL; 329 330 /* write data from CPU1 */ 331 on_cpu(target, sve ? sveregs_writeall_run 332 : fpuregs_writeall_run, 333 &indata_remote); 334 335 /* Overwrite from CPU0 */ 336 __fpuregs_writeall_random(&indata_local, sve); 337 338 /* Check data consistency */ 339 on_cpu(target, sve ? sveregs_testall_run 340 : fpuregs_testall_run, 341 indata_remote); 342 343 free(indata_remote); 344 free(indata_local); 345 } 346 347 /* 348 * This test uses two CPUs to test FPU/SIMD save/restore 349 * CPU0 writes random data into FPU/SIMD registers, 350 * CPU1 corrupts/overwrites the data and finally CPU0 checks if 351 * the data remains unchanged in its context. 352 */ 353 static void fpuregs_context_switch_cpu0(int sve) 354 { 355 int target = CPU1_ID; 356 uint64_t *indata_local = NULL; 357 uint64_t *indata_remote = NULL; 358 uint32_t result = 0; 359 360 /* write data from CPU0 */ 361 __fpuregs_writeall_random(&indata_local, sve); 362 363 /* Overwrite from CPU1 */ 364 on_cpu(target, sve ? sveregs_writeall_run 365 : fpuregs_writeall_run, 366 &indata_remote); 367 368 /* Check data consistency */ 369 result = __fpuregs_testall(indata_local, sve); 370 report((result == FPU_RESULT_PASS), 371 "%s register save/restore mask: 0x%x", sve ? "SVE" : "FPU/SIMD", result); 372 373 free(indata_remote); 374 free(indata_local); 375 } 376 377 /* 378 * Checks if during context switch, FPU/SIMD registers 379 * are saved/restored. 380 */ 381 static void fpuregs_context_switch(void) 382 { 383 fpuregs_context_switch_cpu0(0); 384 fpuregs_context_switch_cpu1(0); 385 } 386 387 /* 388 * Checks if during context switch, SVE registers 389 * are saved/restored. 390 */ 391 static void sveregs_context_switch(void) 392 { 393 unsigned long zcr = read_sysreg(ZCR_EL1); 394 395 // Set the SVE vector length to 128-bits 396 write_sysreg(zcr & ~ZCR_EL1_LEN, ZCR_EL1); 397 398 fpuregs_context_switch_cpu0(1); 399 fpuregs_context_switch_cpu1(1); 400 } 401 402 static bool should_run_sve_tests(void) 403 { 404 #ifdef CC_HAS_SVE 405 if (system_supports_sve()) 406 return true; 407 #endif 408 return false; 409 } 410 411 int main(int argc, char **argv) 412 { 413 report_prefix_pushf("fpu"); 414 415 nr_cpu_check(CPUS_MAX); 416 fpuregs_context_switch(); 417 418 if (should_run_sve_tests()) 419 sveregs_context_switch(); 420 421 return report_summary(); 422 } 423