1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * FP/SIMD context switching and fault handling 4 * 5 * Copyright (C) 2012 ARM Ltd. 6 * Author: Catalin Marinas <catalin.marinas@arm.com> 7 */ 8 9 #include <linux/bitmap.h> 10 #include <linux/bitops.h> 11 #include <linux/bottom_half.h> 12 #include <linux/bug.h> 13 #include <linux/cache.h> 14 #include <linux/compat.h> 15 #include <linux/compiler.h> 16 #include <linux/cpu.h> 17 #include <linux/cpu_pm.h> 18 #include <linux/ctype.h> 19 #include <linux/kernel.h> 20 #include <linux/linkage.h> 21 #include <linux/irqflags.h> 22 #include <linux/init.h> 23 #include <linux/percpu.h> 24 #include <linux/prctl.h> 25 #include <linux/preempt.h> 26 #include <linux/ptrace.h> 27 #include <linux/sched/signal.h> 28 #include <linux/sched/task_stack.h> 29 #include <linux/signal.h> 30 #include <linux/slab.h> 31 #include <linux/stddef.h> 32 #include <linux/sysctl.h> 33 #include <linux/swab.h> 34 35 #include <asm/esr.h> 36 #include <asm/exception.h> 37 #include <asm/fpsimd.h> 38 #include <asm/cpufeature.h> 39 #include <asm/cputype.h> 40 #include <asm/neon.h> 41 #include <asm/processor.h> 42 #include <asm/simd.h> 43 #include <asm/sigcontext.h> 44 #include <asm/sysreg.h> 45 #include <asm/traps.h> 46 #include <asm/virt.h> 47 48 #define FPEXC_IOF (1 << 0) 49 #define FPEXC_DZF (1 << 1) 50 #define FPEXC_OFF (1 << 2) 51 #define FPEXC_UFF (1 << 3) 52 #define FPEXC_IXF (1 << 4) 53 #define FPEXC_IDF (1 << 7) 54 55 /* 56 * (Note: in this discussion, statements about FPSIMD apply equally to SVE.) 57 * 58 * In order to reduce the number of times the FPSIMD state is needlessly saved 59 * and restored, we need to keep track of two things: 60 * (a) for each task, we need to remember which CPU was the last one to have 61 * the task's FPSIMD state loaded into its FPSIMD registers; 62 * (b) for each CPU, we need to remember which task's userland FPSIMD state has 63 * been loaded into its FPSIMD registers most recently, or whether it has 64 * been used to perform kernel mode NEON in the meantime. 65 * 66 * For (a), we add a fpsimd_cpu field to thread_struct, which gets updated to 67 * the id of the current CPU every time the state is loaded onto a CPU. For (b), 68 * we add the per-cpu variable 'fpsimd_last_state' (below), which contains the 69 * address of the userland FPSIMD state of the task that was loaded onto the CPU 70 * the most recently, or NULL if kernel mode NEON has been performed after that. 71 * 72 * With this in place, we no longer have to restore the next FPSIMD state right 73 * when switching between tasks. Instead, we can defer this check to userland 74 * resume, at which time we verify whether the CPU's fpsimd_last_state and the 75 * task's fpsimd_cpu are still mutually in sync. If this is the case, we 76 * can omit the FPSIMD restore. 77 * 78 * As an optimization, we use the thread_info flag TIF_FOREIGN_FPSTATE to 79 * indicate whether or not the userland FPSIMD state of the current task is 80 * present in the registers. The flag is set unless the FPSIMD registers of this 81 * CPU currently contain the most recent userland FPSIMD state of the current 82 * task. If the task is behaving as a VMM, then this is will be managed by 83 * KVM which will clear it to indicate that the vcpu FPSIMD state is currently 84 * loaded on the CPU, allowing the state to be saved if a FPSIMD-aware 85 * softirq kicks in. Upon vcpu_put(), KVM will save the vcpu FP state and 86 * flag the register state as invalid. 87 * 88 * In order to allow softirq handlers to use FPSIMD, kernel_neon_begin() may be 89 * called from softirq context, which will save the task's FPSIMD context back 90 * to task_struct. To prevent this from racing with the manipulation of the 91 * task's FPSIMD state from task context and thereby corrupting the state, it 92 * is necessary to protect any manipulation of a task's fpsimd_state or 93 * TIF_FOREIGN_FPSTATE flag with get_cpu_fpsimd_context(), which will suspend 94 * softirq servicing entirely until put_cpu_fpsimd_context() is called. 95 * 96 * For a certain task, the sequence may look something like this: 97 * - the task gets scheduled in; if both the task's fpsimd_cpu field 98 * contains the id of the current CPU, and the CPU's fpsimd_last_state per-cpu 99 * variable points to the task's fpsimd_state, the TIF_FOREIGN_FPSTATE flag is 100 * cleared, otherwise it is set; 101 * 102 * - the task returns to userland; if TIF_FOREIGN_FPSTATE is set, the task's 103 * userland FPSIMD state is copied from memory to the registers, the task's 104 * fpsimd_cpu field is set to the id of the current CPU, the current 105 * CPU's fpsimd_last_state pointer is set to this task's fpsimd_state and the 106 * TIF_FOREIGN_FPSTATE flag is cleared; 107 * 108 * - the task executes an ordinary syscall; upon return to userland, the 109 * TIF_FOREIGN_FPSTATE flag will still be cleared, so no FPSIMD state is 110 * restored; 111 * 112 * - the task executes a syscall which executes some NEON instructions; this is 113 * preceded by a call to kernel_neon_begin(), which copies the task's FPSIMD 114 * register contents to memory, clears the fpsimd_last_state per-cpu variable 115 * and sets the TIF_FOREIGN_FPSTATE flag; 116 * 117 * - the task gets preempted after kernel_neon_end() is called; as we have not 118 * returned from the 2nd syscall yet, TIF_FOREIGN_FPSTATE is still set so 119 * whatever is in the FPSIMD registers is not saved to memory, but discarded. 120 */ 121 122 DEFINE_PER_CPU(struct cpu_fp_state, fpsimd_last_state); 123 124 __ro_after_init struct vl_info vl_info[ARM64_VEC_MAX] = { 125 #ifdef CONFIG_ARM64_SVE 126 [ARM64_VEC_SVE] = { 127 .type = ARM64_VEC_SVE, 128 .name = "SVE", 129 .min_vl = SVE_VL_MIN, 130 .max_vl = SVE_VL_MIN, 131 .max_virtualisable_vl = SVE_VL_MIN, 132 }, 133 #endif 134 #ifdef CONFIG_ARM64_SME 135 [ARM64_VEC_SME] = { 136 .type = ARM64_VEC_SME, 137 .name = "SME", 138 }, 139 #endif 140 }; 141 142 static unsigned int vec_vl_inherit_flag(enum vec_type type) 143 { 144 switch (type) { 145 case ARM64_VEC_SVE: 146 return TIF_SVE_VL_INHERIT; 147 case ARM64_VEC_SME: 148 return TIF_SME_VL_INHERIT; 149 default: 150 WARN_ON_ONCE(1); 151 return 0; 152 } 153 } 154 155 struct vl_config { 156 int __default_vl; /* Default VL for tasks */ 157 }; 158 159 static struct vl_config vl_config[ARM64_VEC_MAX]; 160 161 static inline int get_default_vl(enum vec_type type) 162 { 163 return READ_ONCE(vl_config[type].__default_vl); 164 } 165 166 #ifdef CONFIG_ARM64_SVE 167 168 static inline int get_sve_default_vl(void) 169 { 170 return get_default_vl(ARM64_VEC_SVE); 171 } 172 173 static inline void set_default_vl(enum vec_type type, int val) 174 { 175 WRITE_ONCE(vl_config[type].__default_vl, val); 176 } 177 178 static inline void set_sve_default_vl(int val) 179 { 180 set_default_vl(ARM64_VEC_SVE, val); 181 } 182 183 static u8 *efi_sve_state; 184 185 #else /* ! CONFIG_ARM64_SVE */ 186 187 /* Dummy declaration for code that will be optimised out: */ 188 extern u8 *efi_sve_state; 189 190 #endif /* ! CONFIG_ARM64_SVE */ 191 192 #ifdef CONFIG_ARM64_SME 193 194 static int get_sme_default_vl(void) 195 { 196 return get_default_vl(ARM64_VEC_SME); 197 } 198 199 static void set_sme_default_vl(int val) 200 { 201 set_default_vl(ARM64_VEC_SME, val); 202 } 203 204 static void sme_free(struct task_struct *); 205 206 #else 207 208 static inline void sme_free(struct task_struct *t) { } 209 210 #endif 211 212 static void fpsimd_bind_task_to_cpu(void); 213 214 /* 215 * Claim ownership of the CPU FPSIMD context for use by the calling context. 216 * 217 * The caller may freely manipulate the FPSIMD context metadata until 218 * put_cpu_fpsimd_context() is called. 219 * 220 * On RT kernels local_bh_disable() is not sufficient because it only 221 * serializes soft interrupt related sections via a local lock, but stays 222 * preemptible. Disabling preemption is the right choice here as bottom 223 * half processing is always in thread context on RT kernels so it 224 * implicitly prevents bottom half processing as well. 225 */ 226 static void get_cpu_fpsimd_context(void) 227 { 228 if (!IS_ENABLED(CONFIG_PREEMPT_RT)) 229 local_bh_disable(); 230 else 231 preempt_disable(); 232 } 233 234 /* 235 * Release the CPU FPSIMD context. 236 * 237 * Must be called from a context in which get_cpu_fpsimd_context() was 238 * previously called, with no call to put_cpu_fpsimd_context() in the 239 * meantime. 240 */ 241 static void put_cpu_fpsimd_context(void) 242 { 243 if (!IS_ENABLED(CONFIG_PREEMPT_RT)) 244 local_bh_enable(); 245 else 246 preempt_enable(); 247 } 248 249 unsigned int task_get_vl(const struct task_struct *task, enum vec_type type) 250 { 251 return task->thread.vl[type]; 252 } 253 254 void task_set_vl(struct task_struct *task, enum vec_type type, 255 unsigned long vl) 256 { 257 task->thread.vl[type] = vl; 258 } 259 260 unsigned int task_get_vl_onexec(const struct task_struct *task, 261 enum vec_type type) 262 { 263 return task->thread.vl_onexec[type]; 264 } 265 266 void task_set_vl_onexec(struct task_struct *task, enum vec_type type, 267 unsigned long vl) 268 { 269 task->thread.vl_onexec[type] = vl; 270 } 271 272 /* 273 * TIF_SME controls whether a task can use SME without trapping while 274 * in userspace, when TIF_SME is set then we must have storage 275 * allocated in sve_state and sme_state to store the contents of both ZA 276 * and the SVE registers for both streaming and non-streaming modes. 277 * 278 * If both SVCR.ZA and SVCR.SM are disabled then at any point we 279 * may disable TIF_SME and reenable traps. 280 */ 281 282 283 /* 284 * TIF_SVE controls whether a task can use SVE without trapping while 285 * in userspace, and also (together with TIF_SME) the way a task's 286 * FPSIMD/SVE state is stored in thread_struct. 287 * 288 * The kernel uses this flag to track whether a user task is actively 289 * using SVE, and therefore whether full SVE register state needs to 290 * be tracked. If not, the cheaper FPSIMD context handling code can 291 * be used instead of the more costly SVE equivalents. 292 * 293 * * TIF_SVE or SVCR.SM set: 294 * 295 * The task can execute SVE instructions while in userspace without 296 * trapping to the kernel. 297 * 298 * During any syscall, the kernel may optionally clear TIF_SVE and 299 * discard the vector state except for the FPSIMD subset. 300 * 301 * * TIF_SVE clear: 302 * 303 * An attempt by the user task to execute an SVE instruction causes 304 * do_sve_acc() to be called, which does some preparation and then 305 * sets TIF_SVE. 306 * 307 * During any syscall, the kernel may optionally clear TIF_SVE and 308 * discard the vector state except for the FPSIMD subset. 309 * 310 * The data will be stored in one of two formats: 311 * 312 * * FPSIMD only - FP_STATE_FPSIMD: 313 * 314 * When the FPSIMD only state stored task->thread.fp_type is set to 315 * FP_STATE_FPSIMD, the FPSIMD registers V0-V31 are encoded in 316 * task->thread.uw.fpsimd_state; bits [max : 128] for each of Z0-Z31 are 317 * logically zero but not stored anywhere; P0-P15 and FFR are not 318 * stored and have unspecified values from userspace's point of 319 * view. For hygiene purposes, the kernel zeroes them on next use, 320 * but userspace is discouraged from relying on this. 321 * 322 * task->thread.sve_state does not need to be non-NULL, valid or any 323 * particular size: it must not be dereferenced and any data stored 324 * there should be considered stale and not referenced. 325 * 326 * * SVE state - FP_STATE_SVE: 327 * 328 * When the full SVE state is stored task->thread.fp_type is set to 329 * FP_STATE_SVE and Z0-Z31 (incorporating Vn in bits[127:0] or the 330 * corresponding Zn), P0-P15 and FFR are encoded in in 331 * task->thread.sve_state, formatted appropriately for vector 332 * length task->thread.sve_vl or, if SVCR.SM is set, 333 * task->thread.sme_vl. The storage for the vector registers in 334 * task->thread.uw.fpsimd_state should be ignored. 335 * 336 * task->thread.sve_state must point to a valid buffer at least 337 * sve_state_size(task) bytes in size. The data stored in 338 * task->thread.uw.fpsimd_state.vregs should be considered stale 339 * and not referenced. 340 * 341 * * FPSR and FPCR are always stored in task->thread.uw.fpsimd_state 342 * irrespective of whether TIF_SVE is clear or set, since these are 343 * not vector length dependent. 344 */ 345 346 /* 347 * Update current's FPSIMD/SVE registers from thread_struct. 348 * 349 * This function should be called only when the FPSIMD/SVE state in 350 * thread_struct is known to be up to date, when preparing to enter 351 * userspace. 352 */ 353 static void task_fpsimd_load(void) 354 { 355 bool restore_sve_regs = false; 356 bool restore_ffr; 357 358 WARN_ON(!system_supports_fpsimd()); 359 WARN_ON(preemptible()); 360 WARN_ON(test_thread_flag(TIF_KERNEL_FPSTATE)); 361 362 if (system_supports_sve() || system_supports_sme()) { 363 switch (current->thread.fp_type) { 364 case FP_STATE_FPSIMD: 365 /* Stop tracking SVE for this task until next use. */ 366 clear_thread_flag(TIF_SVE); 367 break; 368 case FP_STATE_SVE: 369 if (!thread_sm_enabled(¤t->thread)) 370 WARN_ON_ONCE(!test_and_set_thread_flag(TIF_SVE)); 371 372 if (test_thread_flag(TIF_SVE)) 373 sve_set_vq(sve_vq_from_vl(task_get_sve_vl(current)) - 1); 374 375 restore_sve_regs = true; 376 restore_ffr = true; 377 break; 378 default: 379 /* 380 * This indicates either a bug in 381 * fpsimd_save_user_state() or memory corruption, we 382 * should always record an explicit format 383 * when we save. We always at least have the 384 * memory allocated for FPSIMD registers so 385 * try that and hope for the best. 386 */ 387 WARN_ON_ONCE(1); 388 clear_thread_flag(TIF_SVE); 389 break; 390 } 391 } 392 393 /* Restore SME, override SVE register configuration if needed */ 394 if (system_supports_sme()) { 395 unsigned long sme_vl = task_get_sme_vl(current); 396 397 /* Ensure VL is set up for restoring data */ 398 if (test_thread_flag(TIF_SME)) 399 sme_set_vq(sve_vq_from_vl(sme_vl) - 1); 400 401 write_sysreg_s(current->thread.svcr, SYS_SVCR); 402 403 if (thread_za_enabled(¤t->thread)) 404 sme_load_state(current->thread.sme_state, 405 system_supports_sme2()); 406 407 if (thread_sm_enabled(¤t->thread)) 408 restore_ffr = system_supports_fa64(); 409 } 410 411 if (system_supports_fpmr()) 412 write_sysreg_s(current->thread.uw.fpmr, SYS_FPMR); 413 414 if (restore_sve_regs) { 415 WARN_ON_ONCE(current->thread.fp_type != FP_STATE_SVE); 416 sve_load_state(sve_pffr(¤t->thread), 417 ¤t->thread.uw.fpsimd_state.fpsr, 418 restore_ffr); 419 } else { 420 WARN_ON_ONCE(current->thread.fp_type != FP_STATE_FPSIMD); 421 fpsimd_load_state(¤t->thread.uw.fpsimd_state); 422 } 423 } 424 425 /* 426 * Ensure FPSIMD/SVE storage in memory for the loaded context is up to 427 * date with respect to the CPU registers. Note carefully that the 428 * current context is the context last bound to the CPU stored in 429 * last, if KVM is involved this may be the guest VM context rather 430 * than the host thread for the VM pointed to by current. This means 431 * that we must always reference the state storage via last rather 432 * than via current, if we are saving KVM state then it will have 433 * ensured that the type of registers to save is set in last->to_save. 434 */ 435 static void fpsimd_save_user_state(void) 436 { 437 struct cpu_fp_state const *last = 438 this_cpu_ptr(&fpsimd_last_state); 439 /* set by fpsimd_bind_task_to_cpu() or fpsimd_bind_state_to_cpu() */ 440 bool save_sve_regs = false; 441 bool save_ffr; 442 unsigned int vl; 443 444 WARN_ON(!system_supports_fpsimd()); 445 WARN_ON(preemptible()); 446 447 if (test_thread_flag(TIF_FOREIGN_FPSTATE)) 448 return; 449 450 if (system_supports_fpmr()) 451 *(last->fpmr) = read_sysreg_s(SYS_FPMR); 452 453 /* 454 * Save SVE state if it is live. 455 * 456 * The syscall ABI discards live SVE state at syscall entry. When 457 * entering a syscall, fpsimd_syscall_enter() sets to_save to 458 * FP_STATE_FPSIMD to allow the SVE state to be lazily discarded until 459 * either new SVE state is loaded+bound or fpsimd_syscall_exit() is 460 * called prior to a return to userspace. 461 */ 462 if ((last->to_save == FP_STATE_CURRENT && test_thread_flag(TIF_SVE)) || 463 last->to_save == FP_STATE_SVE) { 464 save_sve_regs = true; 465 save_ffr = true; 466 vl = last->sve_vl; 467 } 468 469 if (system_supports_sme()) { 470 u64 *svcr = last->svcr; 471 472 *svcr = read_sysreg_s(SYS_SVCR); 473 474 if (*svcr & SVCR_ZA_MASK) 475 sme_save_state(last->sme_state, 476 system_supports_sme2()); 477 478 /* If we are in streaming mode override regular SVE. */ 479 if (*svcr & SVCR_SM_MASK) { 480 save_sve_regs = true; 481 save_ffr = system_supports_fa64(); 482 vl = last->sme_vl; 483 } 484 } 485 486 if (IS_ENABLED(CONFIG_ARM64_SVE) && save_sve_regs) { 487 /* Get the configured VL from RDVL, will account for SM */ 488 if (WARN_ON(sve_get_vl() != vl)) { 489 /* 490 * Can't save the user regs, so current would 491 * re-enter user with corrupt state. 492 * There's no way to recover, so kill it: 493 */ 494 force_signal_inject(SIGKILL, SI_KERNEL, 0, 0); 495 return; 496 } 497 498 sve_save_state((char *)last->sve_state + 499 sve_ffr_offset(vl), 500 &last->st->fpsr, save_ffr); 501 *last->fp_type = FP_STATE_SVE; 502 } else { 503 fpsimd_save_state(last->st); 504 *last->fp_type = FP_STATE_FPSIMD; 505 } 506 } 507 508 /* 509 * All vector length selection from userspace comes through here. 510 * We're on a slow path, so some sanity-checks are included. 511 * If things go wrong there's a bug somewhere, but try to fall back to a 512 * safe choice. 513 */ 514 static unsigned int find_supported_vector_length(enum vec_type type, 515 unsigned int vl) 516 { 517 struct vl_info *info = &vl_info[type]; 518 int bit; 519 int max_vl = info->max_vl; 520 521 if (WARN_ON(!sve_vl_valid(vl))) 522 vl = info->min_vl; 523 524 if (WARN_ON(!sve_vl_valid(max_vl))) 525 max_vl = info->min_vl; 526 527 if (vl > max_vl) 528 vl = max_vl; 529 if (vl < info->min_vl) 530 vl = info->min_vl; 531 532 bit = find_next_bit(info->vq_map, SVE_VQ_MAX, 533 __vq_to_bit(sve_vq_from_vl(vl))); 534 return sve_vl_from_vq(__bit_to_vq(bit)); 535 } 536 537 #if defined(CONFIG_ARM64_SVE) && defined(CONFIG_SYSCTL) 538 539 static int vec_proc_do_default_vl(const struct ctl_table *table, int write, 540 void *buffer, size_t *lenp, loff_t *ppos) 541 { 542 struct vl_info *info = table->extra1; 543 enum vec_type type = info->type; 544 int ret; 545 int vl = get_default_vl(type); 546 struct ctl_table tmp_table = { 547 .data = &vl, 548 .maxlen = sizeof(vl), 549 }; 550 551 ret = proc_dointvec(&tmp_table, write, buffer, lenp, ppos); 552 if (ret || !write) 553 return ret; 554 555 /* Writing -1 has the special meaning "set to max": */ 556 if (vl == -1) 557 vl = info->max_vl; 558 559 if (!sve_vl_valid(vl)) 560 return -EINVAL; 561 562 set_default_vl(type, find_supported_vector_length(type, vl)); 563 return 0; 564 } 565 566 static const struct ctl_table sve_default_vl_table[] = { 567 { 568 .procname = "sve_default_vector_length", 569 .mode = 0644, 570 .proc_handler = vec_proc_do_default_vl, 571 .extra1 = &vl_info[ARM64_VEC_SVE], 572 }, 573 }; 574 575 static int __init sve_sysctl_init(void) 576 { 577 if (system_supports_sve()) 578 if (!register_sysctl("abi", sve_default_vl_table)) 579 return -EINVAL; 580 581 return 0; 582 } 583 584 #else /* ! (CONFIG_ARM64_SVE && CONFIG_SYSCTL) */ 585 static int __init sve_sysctl_init(void) { return 0; } 586 #endif /* ! (CONFIG_ARM64_SVE && CONFIG_SYSCTL) */ 587 588 #if defined(CONFIG_ARM64_SME) && defined(CONFIG_SYSCTL) 589 static const struct ctl_table sme_default_vl_table[] = { 590 { 591 .procname = "sme_default_vector_length", 592 .mode = 0644, 593 .proc_handler = vec_proc_do_default_vl, 594 .extra1 = &vl_info[ARM64_VEC_SME], 595 }, 596 }; 597 598 static int __init sme_sysctl_init(void) 599 { 600 if (system_supports_sme()) 601 if (!register_sysctl("abi", sme_default_vl_table)) 602 return -EINVAL; 603 604 return 0; 605 } 606 607 #else /* ! (CONFIG_ARM64_SME && CONFIG_SYSCTL) */ 608 static int __init sme_sysctl_init(void) { return 0; } 609 #endif /* ! (CONFIG_ARM64_SME && CONFIG_SYSCTL) */ 610 611 #define ZREG(sve_state, vq, n) ((char *)(sve_state) + \ 612 (SVE_SIG_ZREG_OFFSET(vq, n) - SVE_SIG_REGS_OFFSET)) 613 614 #ifdef CONFIG_CPU_BIG_ENDIAN 615 static __uint128_t arm64_cpu_to_le128(__uint128_t x) 616 { 617 u64 a = swab64(x); 618 u64 b = swab64(x >> 64); 619 620 return ((__uint128_t)a << 64) | b; 621 } 622 #else 623 static __uint128_t arm64_cpu_to_le128(__uint128_t x) 624 { 625 return x; 626 } 627 #endif 628 629 #define arm64_le128_to_cpu(x) arm64_cpu_to_le128(x) 630 631 static void __fpsimd_to_sve(void *sst, struct user_fpsimd_state const *fst, 632 unsigned int vq) 633 { 634 unsigned int i; 635 __uint128_t *p; 636 637 for (i = 0; i < SVE_NUM_ZREGS; ++i) { 638 p = (__uint128_t *)ZREG(sst, vq, i); 639 *p = arm64_cpu_to_le128(fst->vregs[i]); 640 } 641 } 642 643 /* 644 * Transfer the FPSIMD state in task->thread.uw.fpsimd_state to 645 * task->thread.sve_state. 646 * 647 * Task can be a non-runnable task, or current. In the latter case, 648 * the caller must have ownership of the cpu FPSIMD context before calling 649 * this function. 650 * task->thread.sve_state must point to at least sve_state_size(task) 651 * bytes of allocated kernel memory. 652 * task->thread.uw.fpsimd_state must be up to date before calling this 653 * function. 654 */ 655 static inline void fpsimd_to_sve(struct task_struct *task) 656 { 657 unsigned int vq; 658 void *sst = task->thread.sve_state; 659 struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state; 660 661 if (!system_supports_sve() && !system_supports_sme()) 662 return; 663 664 vq = sve_vq_from_vl(thread_get_cur_vl(&task->thread)); 665 __fpsimd_to_sve(sst, fst, vq); 666 } 667 668 /* 669 * Transfer the SVE state in task->thread.sve_state to 670 * task->thread.uw.fpsimd_state. 671 * 672 * Task can be a non-runnable task, or current. In the latter case, 673 * the caller must have ownership of the cpu FPSIMD context before calling 674 * this function. 675 * task->thread.sve_state must point to at least sve_state_size(task) 676 * bytes of allocated kernel memory. 677 * task->thread.sve_state must be up to date before calling this function. 678 */ 679 static inline void sve_to_fpsimd(struct task_struct *task) 680 { 681 unsigned int vq, vl; 682 void const *sst = task->thread.sve_state; 683 struct user_fpsimd_state *fst = &task->thread.uw.fpsimd_state; 684 unsigned int i; 685 __uint128_t const *p; 686 687 if (!system_supports_sve() && !system_supports_sme()) 688 return; 689 690 vl = thread_get_cur_vl(&task->thread); 691 vq = sve_vq_from_vl(vl); 692 for (i = 0; i < SVE_NUM_ZREGS; ++i) { 693 p = (__uint128_t const *)ZREG(sst, vq, i); 694 fst->vregs[i] = arm64_le128_to_cpu(*p); 695 } 696 } 697 698 static inline void __fpsimd_zero_vregs(struct user_fpsimd_state *fpsimd) 699 { 700 memset(&fpsimd->vregs, 0, sizeof(fpsimd->vregs)); 701 } 702 703 /* 704 * Simulate the effects of an SMSTOP SM instruction. 705 */ 706 void task_smstop_sm(struct task_struct *task) 707 { 708 if (!thread_sm_enabled(&task->thread)) 709 return; 710 711 __fpsimd_zero_vregs(&task->thread.uw.fpsimd_state); 712 task->thread.uw.fpsimd_state.fpsr = 0x0800009f; 713 if (system_supports_fpmr()) 714 task->thread.uw.fpmr = 0; 715 716 task->thread.svcr &= ~SVCR_SM_MASK; 717 task->thread.fp_type = FP_STATE_FPSIMD; 718 } 719 720 void cpu_enable_fpmr(const struct arm64_cpu_capabilities *__always_unused p) 721 { 722 write_sysreg_s(read_sysreg_s(SYS_SCTLR_EL1) | SCTLR_EL1_EnFPM_MASK, 723 SYS_SCTLR_EL1); 724 } 725 726 #ifdef CONFIG_ARM64_SVE 727 static void sve_free(struct task_struct *task) 728 { 729 kfree(task->thread.sve_state); 730 task->thread.sve_state = NULL; 731 } 732 733 /* 734 * Ensure that task->thread.sve_state is allocated and sufficiently large. 735 * 736 * This function should be used only in preparation for replacing 737 * task->thread.sve_state with new data. The memory is always zeroed 738 * here to prevent stale data from showing through: this is done in 739 * the interest of testability and predictability: except in the 740 * do_sve_acc() case, there is no ABI requirement to hide stale data 741 * written previously be task. 742 */ 743 void sve_alloc(struct task_struct *task, bool flush) 744 { 745 if (task->thread.sve_state) { 746 if (flush) 747 memset(task->thread.sve_state, 0, 748 sve_state_size(task)); 749 return; 750 } 751 752 /* This is a small allocation (maximum ~8KB) and Should Not Fail. */ 753 task->thread.sve_state = 754 kzalloc(sve_state_size(task), GFP_KERNEL); 755 } 756 757 /* 758 * Ensure that task->thread.uw.fpsimd_state is up to date with respect to the 759 * task's currently effective FPSIMD/SVE state. 760 * 761 * The task's FPSIMD/SVE/SME state must not be subject to concurrent 762 * manipulation. 763 */ 764 void fpsimd_sync_from_effective_state(struct task_struct *task) 765 { 766 if (task->thread.fp_type == FP_STATE_SVE) 767 sve_to_fpsimd(task); 768 } 769 770 /* 771 * Ensure that the task's currently effective FPSIMD/SVE state is up to date 772 * with respect to task->thread.uw.fpsimd_state, zeroing any effective 773 * non-FPSIMD (S)SVE state. 774 * 775 * The task's FPSIMD/SVE/SME state must not be subject to concurrent 776 * manipulation. 777 */ 778 void fpsimd_sync_to_effective_state_zeropad(struct task_struct *task) 779 { 780 unsigned int vq; 781 void *sst = task->thread.sve_state; 782 struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state; 783 784 if (task->thread.fp_type != FP_STATE_SVE) 785 return; 786 787 vq = sve_vq_from_vl(thread_get_cur_vl(&task->thread)); 788 789 memset(sst, 0, SVE_SIG_REGS_SIZE(vq)); 790 __fpsimd_to_sve(sst, fst, vq); 791 } 792 793 static int change_live_vector_length(struct task_struct *task, 794 enum vec_type type, 795 unsigned long vl) 796 { 797 unsigned int sve_vl = task_get_sve_vl(task); 798 unsigned int sme_vl = task_get_sme_vl(task); 799 void *sve_state = NULL, *sme_state = NULL; 800 801 if (type == ARM64_VEC_SME) 802 sme_vl = vl; 803 else 804 sve_vl = vl; 805 806 /* 807 * Allocate the new sve_state and sme_state before freeing the old 808 * copies so that allocation failure can be handled without needing to 809 * mutate the task's state in any way. 810 * 811 * Changes to the SVE vector length must not discard live ZA state or 812 * clear PSTATE.ZA, as userspace code which is unaware of the AAPCS64 813 * ZA lazy saving scheme may attempt to change the SVE vector length 814 * while unsaved/dormant ZA state exists. 815 */ 816 sve_state = kzalloc(__sve_state_size(sve_vl, sme_vl), GFP_KERNEL); 817 if (!sve_state) 818 goto out_mem; 819 820 if (type == ARM64_VEC_SME) { 821 sme_state = kzalloc(__sme_state_size(sme_vl), GFP_KERNEL); 822 if (!sme_state) 823 goto out_mem; 824 } 825 826 if (task == current) 827 fpsimd_save_and_flush_current_state(); 828 else 829 fpsimd_flush_task_state(task); 830 831 /* 832 * Always preserve PSTATE.SM and the effective FPSIMD state, zeroing 833 * other SVE state. 834 */ 835 fpsimd_sync_from_effective_state(task); 836 task_set_vl(task, type, vl); 837 kfree(task->thread.sve_state); 838 task->thread.sve_state = sve_state; 839 fpsimd_sync_to_effective_state_zeropad(task); 840 841 if (type == ARM64_VEC_SME) { 842 task->thread.svcr &= ~SVCR_ZA_MASK; 843 kfree(task->thread.sme_state); 844 task->thread.sme_state = sme_state; 845 } 846 847 return 0; 848 849 out_mem: 850 kfree(sve_state); 851 kfree(sme_state); 852 return -ENOMEM; 853 } 854 855 int vec_set_vector_length(struct task_struct *task, enum vec_type type, 856 unsigned long vl, unsigned long flags) 857 { 858 bool onexec = flags & PR_SVE_SET_VL_ONEXEC; 859 bool inherit = flags & PR_SVE_VL_INHERIT; 860 861 if (flags & ~(unsigned long)(PR_SVE_VL_INHERIT | 862 PR_SVE_SET_VL_ONEXEC)) 863 return -EINVAL; 864 865 if (!sve_vl_valid(vl)) 866 return -EINVAL; 867 868 /* 869 * Clamp to the maximum vector length that VL-agnostic code 870 * can work with. A flag may be assigned in the future to 871 * allow setting of larger vector lengths without confusing 872 * older software. 873 */ 874 if (vl > VL_ARCH_MAX) 875 vl = VL_ARCH_MAX; 876 877 vl = find_supported_vector_length(type, vl); 878 879 if (!onexec && vl != task_get_vl(task, type)) { 880 if (change_live_vector_length(task, type, vl)) 881 return -ENOMEM; 882 } 883 884 if (onexec || inherit) 885 task_set_vl_onexec(task, type, vl); 886 else 887 /* Reset VL to system default on next exec: */ 888 task_set_vl_onexec(task, type, 0); 889 890 update_tsk_thread_flag(task, vec_vl_inherit_flag(type), 891 flags & PR_SVE_VL_INHERIT); 892 893 return 0; 894 } 895 896 /* 897 * Encode the current vector length and flags for return. 898 * This is only required for prctl(): ptrace has separate fields. 899 * SVE and SME use the same bits for _ONEXEC and _INHERIT. 900 * 901 * flags are as for vec_set_vector_length(). 902 */ 903 static int vec_prctl_status(enum vec_type type, unsigned long flags) 904 { 905 int ret; 906 907 if (flags & PR_SVE_SET_VL_ONEXEC) 908 ret = task_get_vl_onexec(current, type); 909 else 910 ret = task_get_vl(current, type); 911 912 if (test_thread_flag(vec_vl_inherit_flag(type))) 913 ret |= PR_SVE_VL_INHERIT; 914 915 return ret; 916 } 917 918 /* PR_SVE_SET_VL */ 919 int sve_set_current_vl(unsigned long arg) 920 { 921 unsigned long vl, flags; 922 int ret; 923 924 vl = arg & PR_SVE_VL_LEN_MASK; 925 flags = arg & ~vl; 926 927 if (!system_supports_sve() || is_compat_task()) 928 return -EINVAL; 929 930 ret = vec_set_vector_length(current, ARM64_VEC_SVE, vl, flags); 931 if (ret) 932 return ret; 933 934 return vec_prctl_status(ARM64_VEC_SVE, flags); 935 } 936 937 /* PR_SVE_GET_VL */ 938 int sve_get_current_vl(void) 939 { 940 if (!system_supports_sve() || is_compat_task()) 941 return -EINVAL; 942 943 return vec_prctl_status(ARM64_VEC_SVE, 0); 944 } 945 946 #ifdef CONFIG_ARM64_SME 947 /* PR_SME_SET_VL */ 948 int sme_set_current_vl(unsigned long arg) 949 { 950 unsigned long vl, flags; 951 int ret; 952 953 vl = arg & PR_SME_VL_LEN_MASK; 954 flags = arg & ~vl; 955 956 if (!system_supports_sme() || is_compat_task()) 957 return -EINVAL; 958 959 ret = vec_set_vector_length(current, ARM64_VEC_SME, vl, flags); 960 if (ret) 961 return ret; 962 963 return vec_prctl_status(ARM64_VEC_SME, flags); 964 } 965 966 /* PR_SME_GET_VL */ 967 int sme_get_current_vl(void) 968 { 969 if (!system_supports_sme() || is_compat_task()) 970 return -EINVAL; 971 972 return vec_prctl_status(ARM64_VEC_SME, 0); 973 } 974 #endif /* CONFIG_ARM64_SME */ 975 976 static void vec_probe_vqs(struct vl_info *info, 977 DECLARE_BITMAP(map, SVE_VQ_MAX)) 978 { 979 unsigned int vq, vl; 980 981 bitmap_zero(map, SVE_VQ_MAX); 982 983 for (vq = SVE_VQ_MAX; vq >= SVE_VQ_MIN; --vq) { 984 write_vl(info->type, vq - 1); /* self-syncing */ 985 986 switch (info->type) { 987 case ARM64_VEC_SVE: 988 vl = sve_get_vl(); 989 break; 990 case ARM64_VEC_SME: 991 vl = sme_get_vl(); 992 break; 993 default: 994 vl = 0; 995 break; 996 } 997 998 /* Minimum VL identified? */ 999 if (sve_vq_from_vl(vl) > vq) 1000 break; 1001 1002 vq = sve_vq_from_vl(vl); /* skip intervening lengths */ 1003 set_bit(__vq_to_bit(vq), map); 1004 } 1005 } 1006 1007 /* 1008 * Initialise the set of known supported VQs for the boot CPU. 1009 * This is called during kernel boot, before secondary CPUs are brought up. 1010 */ 1011 void __init vec_init_vq_map(enum vec_type type) 1012 { 1013 struct vl_info *info = &vl_info[type]; 1014 vec_probe_vqs(info, info->vq_map); 1015 bitmap_copy(info->vq_partial_map, info->vq_map, SVE_VQ_MAX); 1016 } 1017 1018 /* 1019 * If we haven't committed to the set of supported VQs yet, filter out 1020 * those not supported by the current CPU. 1021 * This function is called during the bring-up of early secondary CPUs only. 1022 */ 1023 void vec_update_vq_map(enum vec_type type) 1024 { 1025 struct vl_info *info = &vl_info[type]; 1026 DECLARE_BITMAP(tmp_map, SVE_VQ_MAX); 1027 1028 vec_probe_vqs(info, tmp_map); 1029 bitmap_and(info->vq_map, info->vq_map, tmp_map, SVE_VQ_MAX); 1030 bitmap_or(info->vq_partial_map, info->vq_partial_map, tmp_map, 1031 SVE_VQ_MAX); 1032 } 1033 1034 /* 1035 * Check whether the current CPU supports all VQs in the committed set. 1036 * This function is called during the bring-up of late secondary CPUs only. 1037 */ 1038 int vec_verify_vq_map(enum vec_type type) 1039 { 1040 struct vl_info *info = &vl_info[type]; 1041 DECLARE_BITMAP(tmp_map, SVE_VQ_MAX); 1042 unsigned long b; 1043 1044 vec_probe_vqs(info, tmp_map); 1045 1046 bitmap_complement(tmp_map, tmp_map, SVE_VQ_MAX); 1047 if (bitmap_intersects(tmp_map, info->vq_map, SVE_VQ_MAX)) { 1048 pr_warn("%s: cpu%d: Required vector length(s) missing\n", 1049 info->name, smp_processor_id()); 1050 return -EINVAL; 1051 } 1052 1053 if (!IS_ENABLED(CONFIG_KVM) || !is_hyp_mode_available()) 1054 return 0; 1055 1056 /* 1057 * For KVM, it is necessary to ensure that this CPU doesn't 1058 * support any vector length that guests may have probed as 1059 * unsupported. 1060 */ 1061 1062 /* Recover the set of supported VQs: */ 1063 bitmap_complement(tmp_map, tmp_map, SVE_VQ_MAX); 1064 /* Find VQs supported that are not globally supported: */ 1065 bitmap_andnot(tmp_map, tmp_map, info->vq_map, SVE_VQ_MAX); 1066 1067 /* Find the lowest such VQ, if any: */ 1068 b = find_last_bit(tmp_map, SVE_VQ_MAX); 1069 if (b >= SVE_VQ_MAX) 1070 return 0; /* no mismatches */ 1071 1072 /* 1073 * Mismatches above sve_max_virtualisable_vl are fine, since 1074 * no guest is allowed to configure ZCR_EL2.LEN to exceed this: 1075 */ 1076 if (sve_vl_from_vq(__bit_to_vq(b)) <= info->max_virtualisable_vl) { 1077 pr_warn("%s: cpu%d: Unsupported vector length(s) present\n", 1078 info->name, smp_processor_id()); 1079 return -EINVAL; 1080 } 1081 1082 return 0; 1083 } 1084 1085 static void __init sve_efi_setup(void) 1086 { 1087 int max_vl = 0; 1088 int i; 1089 1090 if (!IS_ENABLED(CONFIG_EFI)) 1091 return; 1092 1093 for (i = 0; i < ARRAY_SIZE(vl_info); i++) 1094 max_vl = max(vl_info[i].max_vl, max_vl); 1095 1096 /* 1097 * alloc_percpu() warns and prints a backtrace if this goes wrong. 1098 * This is evidence of a crippled system and we are returning void, 1099 * so no attempt is made to handle this situation here. 1100 */ 1101 if (!sve_vl_valid(max_vl)) 1102 goto fail; 1103 1104 efi_sve_state = kmalloc(SVE_SIG_REGS_SIZE(sve_vq_from_vl(max_vl)), 1105 GFP_KERNEL); 1106 if (!efi_sve_state) 1107 goto fail; 1108 1109 return; 1110 1111 fail: 1112 panic("Cannot allocate memory for EFI SVE save/restore"); 1113 } 1114 1115 void cpu_enable_sve(const struct arm64_cpu_capabilities *__always_unused p) 1116 { 1117 write_sysreg(read_sysreg(CPACR_EL1) | CPACR_EL1_ZEN_EL1EN, CPACR_EL1); 1118 isb(); 1119 1120 write_sysreg_s(0, SYS_ZCR_EL1); 1121 } 1122 1123 void __init sve_setup(void) 1124 { 1125 struct vl_info *info = &vl_info[ARM64_VEC_SVE]; 1126 DECLARE_BITMAP(tmp_map, SVE_VQ_MAX); 1127 unsigned long b; 1128 int max_bit; 1129 1130 if (!system_supports_sve()) 1131 return; 1132 1133 /* 1134 * The SVE architecture mandates support for 128-bit vectors, 1135 * so sve_vq_map must have at least SVE_VQ_MIN set. 1136 * If something went wrong, at least try to patch it up: 1137 */ 1138 if (WARN_ON(!test_bit(__vq_to_bit(SVE_VQ_MIN), info->vq_map))) 1139 set_bit(__vq_to_bit(SVE_VQ_MIN), info->vq_map); 1140 1141 max_bit = find_first_bit(info->vq_map, SVE_VQ_MAX); 1142 info->max_vl = sve_vl_from_vq(__bit_to_vq(max_bit)); 1143 1144 /* 1145 * For the default VL, pick the maximum supported value <= 64. 1146 * VL == 64 is guaranteed not to grow the signal frame. 1147 */ 1148 set_sve_default_vl(find_supported_vector_length(ARM64_VEC_SVE, 64)); 1149 1150 bitmap_andnot(tmp_map, info->vq_partial_map, info->vq_map, 1151 SVE_VQ_MAX); 1152 1153 b = find_last_bit(tmp_map, SVE_VQ_MAX); 1154 if (b >= SVE_VQ_MAX) 1155 /* No non-virtualisable VLs found */ 1156 info->max_virtualisable_vl = SVE_VQ_MAX; 1157 else if (WARN_ON(b == SVE_VQ_MAX - 1)) 1158 /* No virtualisable VLs? This is architecturally forbidden. */ 1159 info->max_virtualisable_vl = SVE_VQ_MIN; 1160 else /* b + 1 < SVE_VQ_MAX */ 1161 info->max_virtualisable_vl = sve_vl_from_vq(__bit_to_vq(b + 1)); 1162 1163 if (info->max_virtualisable_vl > info->max_vl) 1164 info->max_virtualisable_vl = info->max_vl; 1165 1166 pr_info("%s: maximum available vector length %u bytes per vector\n", 1167 info->name, info->max_vl); 1168 pr_info("%s: default vector length %u bytes per vector\n", 1169 info->name, get_sve_default_vl()); 1170 1171 /* KVM decides whether to support mismatched systems. Just warn here: */ 1172 if (sve_max_virtualisable_vl() < sve_max_vl()) 1173 pr_warn("%s: unvirtualisable vector lengths present\n", 1174 info->name); 1175 1176 sve_efi_setup(); 1177 } 1178 1179 /* 1180 * Called from the put_task_struct() path, which cannot get here 1181 * unless dead_task is really dead and not schedulable. 1182 */ 1183 void fpsimd_release_task(struct task_struct *dead_task) 1184 { 1185 sve_free(dead_task); 1186 sme_free(dead_task); 1187 } 1188 1189 #endif /* CONFIG_ARM64_SVE */ 1190 1191 #ifdef CONFIG_ARM64_SME 1192 1193 /* 1194 * Ensure that task->thread.sme_state is allocated and sufficiently large. 1195 * 1196 * This function should be used only in preparation for replacing 1197 * task->thread.sme_state with new data. The memory is always zeroed 1198 * here to prevent stale data from showing through: this is done in 1199 * the interest of testability and predictability, the architecture 1200 * guarantees that when ZA is enabled it will be zeroed. 1201 */ 1202 void sme_alloc(struct task_struct *task, bool flush) 1203 { 1204 if (task->thread.sme_state) { 1205 if (flush) 1206 memset(task->thread.sme_state, 0, 1207 sme_state_size(task)); 1208 return; 1209 } 1210 1211 /* This could potentially be up to 64K. */ 1212 task->thread.sme_state = 1213 kzalloc(sme_state_size(task), GFP_KERNEL); 1214 } 1215 1216 static void sme_free(struct task_struct *task) 1217 { 1218 kfree(task->thread.sme_state); 1219 task->thread.sme_state = NULL; 1220 } 1221 1222 void cpu_enable_sme(const struct arm64_cpu_capabilities *__always_unused p) 1223 { 1224 /* Set priority for all PEs to architecturally defined minimum */ 1225 write_sysreg_s(read_sysreg_s(SYS_SMPRI_EL1) & ~SMPRI_EL1_PRIORITY_MASK, 1226 SYS_SMPRI_EL1); 1227 1228 /* Allow SME in kernel */ 1229 write_sysreg(read_sysreg(CPACR_EL1) | CPACR_EL1_SMEN_EL1EN, CPACR_EL1); 1230 isb(); 1231 1232 /* Ensure all bits in SMCR are set to known values */ 1233 write_sysreg_s(0, SYS_SMCR_EL1); 1234 1235 /* Allow EL0 to access TPIDR2 */ 1236 write_sysreg(read_sysreg(SCTLR_EL1) | SCTLR_ELx_ENTP2, SCTLR_EL1); 1237 isb(); 1238 } 1239 1240 void cpu_enable_sme2(const struct arm64_cpu_capabilities *__always_unused p) 1241 { 1242 /* This must be enabled after SME */ 1243 BUILD_BUG_ON(ARM64_SME2 <= ARM64_SME); 1244 1245 /* Allow use of ZT0 */ 1246 write_sysreg_s(read_sysreg_s(SYS_SMCR_EL1) | SMCR_ELx_EZT0_MASK, 1247 SYS_SMCR_EL1); 1248 } 1249 1250 void cpu_enable_fa64(const struct arm64_cpu_capabilities *__always_unused p) 1251 { 1252 /* This must be enabled after SME */ 1253 BUILD_BUG_ON(ARM64_SME_FA64 <= ARM64_SME); 1254 1255 /* Allow use of FA64 */ 1256 write_sysreg_s(read_sysreg_s(SYS_SMCR_EL1) | SMCR_ELx_FA64_MASK, 1257 SYS_SMCR_EL1); 1258 } 1259 1260 void __init sme_setup(void) 1261 { 1262 struct vl_info *info = &vl_info[ARM64_VEC_SME]; 1263 int min_bit, max_bit; 1264 1265 if (!system_supports_sme()) 1266 return; 1267 1268 /* 1269 * SME doesn't require any particular vector length be 1270 * supported but it does require at least one. We should have 1271 * disabled the feature entirely while bringing up CPUs but 1272 * let's double check here. The bitmap is SVE_VQ_MAP sized for 1273 * sharing with SVE. 1274 */ 1275 WARN_ON(bitmap_empty(info->vq_map, SVE_VQ_MAX)); 1276 1277 min_bit = find_last_bit(info->vq_map, SVE_VQ_MAX); 1278 info->min_vl = sve_vl_from_vq(__bit_to_vq(min_bit)); 1279 1280 max_bit = find_first_bit(info->vq_map, SVE_VQ_MAX); 1281 info->max_vl = sve_vl_from_vq(__bit_to_vq(max_bit)); 1282 1283 WARN_ON(info->min_vl > info->max_vl); 1284 1285 /* 1286 * For the default VL, pick the maximum supported value <= 32 1287 * (256 bits) if there is one since this is guaranteed not to 1288 * grow the signal frame when in streaming mode, otherwise the 1289 * minimum available VL will be used. 1290 */ 1291 set_sme_default_vl(find_supported_vector_length(ARM64_VEC_SME, 32)); 1292 1293 pr_info("SME: minimum available vector length %u bytes per vector\n", 1294 info->min_vl); 1295 pr_info("SME: maximum available vector length %u bytes per vector\n", 1296 info->max_vl); 1297 pr_info("SME: default vector length %u bytes per vector\n", 1298 get_sme_default_vl()); 1299 } 1300 1301 void sme_suspend_exit(void) 1302 { 1303 u64 smcr = 0; 1304 1305 if (!system_supports_sme()) 1306 return; 1307 1308 if (system_supports_fa64()) 1309 smcr |= SMCR_ELx_FA64; 1310 if (system_supports_sme2()) 1311 smcr |= SMCR_ELx_EZT0; 1312 1313 write_sysreg_s(smcr, SYS_SMCR_EL1); 1314 write_sysreg_s(0, SYS_SMPRI_EL1); 1315 } 1316 1317 #endif /* CONFIG_ARM64_SME */ 1318 1319 static void sve_init_regs(void) 1320 { 1321 /* 1322 * Convert the FPSIMD state to SVE, zeroing all the state that 1323 * is not shared with FPSIMD. If (as is likely) the current 1324 * state is live in the registers then do this there and 1325 * update our metadata for the current task including 1326 * disabling the trap, otherwise update our in-memory copy. 1327 * We are guaranteed to not be in streaming mode, we can only 1328 * take a SVE trap when not in streaming mode and we can't be 1329 * in streaming mode when taking a SME trap. 1330 */ 1331 if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) { 1332 unsigned long vq_minus_one = 1333 sve_vq_from_vl(task_get_sve_vl(current)) - 1; 1334 sve_set_vq(vq_minus_one); 1335 sve_flush_live(true, vq_minus_one); 1336 fpsimd_bind_task_to_cpu(); 1337 } else { 1338 fpsimd_to_sve(current); 1339 current->thread.fp_type = FP_STATE_SVE; 1340 fpsimd_flush_task_state(current); 1341 } 1342 } 1343 1344 /* 1345 * Trapped SVE access 1346 * 1347 * Storage is allocated for the full SVE state, the current FPSIMD 1348 * register contents are migrated across, and the access trap is 1349 * disabled. 1350 * 1351 * TIF_SVE should be clear on entry: otherwise, fpsimd_restore_current_state() 1352 * would have disabled the SVE access trap for userspace during 1353 * ret_to_user, making an SVE access trap impossible in that case. 1354 */ 1355 void do_sve_acc(unsigned long esr, struct pt_regs *regs) 1356 { 1357 /* Even if we chose not to use SVE, the hardware could still trap: */ 1358 if (unlikely(!system_supports_sve()) || WARN_ON(is_compat_task())) { 1359 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0); 1360 return; 1361 } 1362 1363 sve_alloc(current, true); 1364 if (!current->thread.sve_state) { 1365 force_sig(SIGKILL); 1366 return; 1367 } 1368 1369 get_cpu_fpsimd_context(); 1370 1371 if (test_and_set_thread_flag(TIF_SVE)) 1372 WARN_ON(1); /* SVE access shouldn't have trapped */ 1373 1374 /* 1375 * Even if the task can have used streaming mode we can only 1376 * generate SVE access traps in normal SVE mode and 1377 * transitioning out of streaming mode may discard any 1378 * streaming mode state. Always clear the high bits to avoid 1379 * any potential errors tracking what is properly initialised. 1380 */ 1381 sve_init_regs(); 1382 1383 put_cpu_fpsimd_context(); 1384 } 1385 1386 /* 1387 * Trapped SME access 1388 * 1389 * Storage is allocated for the full SVE and SME state, the current 1390 * FPSIMD register contents are migrated to SVE if SVE is not already 1391 * active, and the access trap is disabled. 1392 * 1393 * TIF_SME should be clear on entry: otherwise, fpsimd_restore_current_state() 1394 * would have disabled the SME access trap for userspace during 1395 * ret_to_user, making an SME access trap impossible in that case. 1396 */ 1397 void do_sme_acc(unsigned long esr, struct pt_regs *regs) 1398 { 1399 /* Even if we chose not to use SME, the hardware could still trap: */ 1400 if (unlikely(!system_supports_sme()) || WARN_ON(is_compat_task())) { 1401 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0); 1402 return; 1403 } 1404 1405 /* 1406 * If this not a trap due to SME being disabled then something 1407 * is being used in the wrong mode, report as SIGILL. 1408 */ 1409 if (ESR_ELx_SME_ISS_SMTC(esr) != ESR_ELx_SME_ISS_SMTC_SME_DISABLED) { 1410 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0); 1411 return; 1412 } 1413 1414 sve_alloc(current, false); 1415 sme_alloc(current, true); 1416 if (!current->thread.sve_state || !current->thread.sme_state) { 1417 force_sig(SIGKILL); 1418 return; 1419 } 1420 1421 get_cpu_fpsimd_context(); 1422 1423 /* With TIF_SME userspace shouldn't generate any traps */ 1424 if (test_and_set_thread_flag(TIF_SME)) 1425 WARN_ON(1); 1426 1427 if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) { 1428 unsigned long vq_minus_one = 1429 sve_vq_from_vl(task_get_sme_vl(current)) - 1; 1430 sme_set_vq(vq_minus_one); 1431 1432 fpsimd_bind_task_to_cpu(); 1433 } else { 1434 fpsimd_flush_task_state(current); 1435 } 1436 1437 put_cpu_fpsimd_context(); 1438 } 1439 1440 /* 1441 * Trapped FP/ASIMD access. 1442 */ 1443 void do_fpsimd_acc(unsigned long esr, struct pt_regs *regs) 1444 { 1445 /* Even if we chose not to use FPSIMD, the hardware could still trap: */ 1446 if (!system_supports_fpsimd()) { 1447 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0); 1448 return; 1449 } 1450 1451 /* 1452 * When FPSIMD is enabled, we should never take a trap unless something 1453 * has gone very wrong. 1454 */ 1455 BUG(); 1456 } 1457 1458 /* 1459 * Raise a SIGFPE for the current process. 1460 */ 1461 void do_fpsimd_exc(unsigned long esr, struct pt_regs *regs) 1462 { 1463 unsigned int si_code = FPE_FLTUNK; 1464 1465 if (esr & ESR_ELx_FP_EXC_TFV) { 1466 if (esr & FPEXC_IOF) 1467 si_code = FPE_FLTINV; 1468 else if (esr & FPEXC_DZF) 1469 si_code = FPE_FLTDIV; 1470 else if (esr & FPEXC_OFF) 1471 si_code = FPE_FLTOVF; 1472 else if (esr & FPEXC_UFF) 1473 si_code = FPE_FLTUND; 1474 else if (esr & FPEXC_IXF) 1475 si_code = FPE_FLTRES; 1476 } 1477 1478 send_sig_fault(SIGFPE, si_code, 1479 (void __user *)instruction_pointer(regs), 1480 current); 1481 } 1482 1483 static void fpsimd_load_kernel_state(struct task_struct *task) 1484 { 1485 struct cpu_fp_state *last = this_cpu_ptr(&fpsimd_last_state); 1486 1487 /* 1488 * Elide the load if this CPU holds the most recent kernel mode 1489 * FPSIMD context of the current task. 1490 */ 1491 if (last->st == &task->thread.kernel_fpsimd_state && 1492 task->thread.kernel_fpsimd_cpu == smp_processor_id()) 1493 return; 1494 1495 fpsimd_load_state(&task->thread.kernel_fpsimd_state); 1496 } 1497 1498 static void fpsimd_save_kernel_state(struct task_struct *task) 1499 { 1500 struct cpu_fp_state cpu_fp_state = { 1501 .st = &task->thread.kernel_fpsimd_state, 1502 .to_save = FP_STATE_FPSIMD, 1503 }; 1504 1505 fpsimd_save_state(&task->thread.kernel_fpsimd_state); 1506 fpsimd_bind_state_to_cpu(&cpu_fp_state); 1507 1508 task->thread.kernel_fpsimd_cpu = smp_processor_id(); 1509 } 1510 1511 /* 1512 * Invalidate any task's FPSIMD state that is present on this cpu. 1513 * The FPSIMD context should be acquired with get_cpu_fpsimd_context() 1514 * before calling this function. 1515 */ 1516 static void fpsimd_flush_cpu_state(void) 1517 { 1518 WARN_ON(!system_supports_fpsimd()); 1519 __this_cpu_write(fpsimd_last_state.st, NULL); 1520 1521 /* 1522 * Leaving streaming mode enabled will cause issues for any kernel 1523 * NEON and leaving streaming mode or ZA enabled may increase power 1524 * consumption. 1525 */ 1526 if (system_supports_sme()) 1527 sme_smstop(); 1528 1529 set_thread_flag(TIF_FOREIGN_FPSTATE); 1530 } 1531 1532 void fpsimd_thread_switch(struct task_struct *next) 1533 { 1534 bool wrong_task, wrong_cpu; 1535 1536 if (!system_supports_fpsimd()) 1537 return; 1538 1539 WARN_ON_ONCE(!irqs_disabled()); 1540 1541 /* Save unsaved fpsimd state, if any: */ 1542 if (test_thread_flag(TIF_KERNEL_FPSTATE)) 1543 fpsimd_save_kernel_state(current); 1544 else 1545 fpsimd_save_user_state(); 1546 1547 if (test_tsk_thread_flag(next, TIF_KERNEL_FPSTATE)) { 1548 fpsimd_flush_cpu_state(); 1549 fpsimd_load_kernel_state(next); 1550 } else { 1551 /* 1552 * Fix up TIF_FOREIGN_FPSTATE to correctly describe next's 1553 * state. For kernel threads, FPSIMD registers are never 1554 * loaded with user mode FPSIMD state and so wrong_task and 1555 * wrong_cpu will always be true. 1556 */ 1557 wrong_task = __this_cpu_read(fpsimd_last_state.st) != 1558 &next->thread.uw.fpsimd_state; 1559 wrong_cpu = next->thread.fpsimd_cpu != smp_processor_id(); 1560 1561 update_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE, 1562 wrong_task || wrong_cpu); 1563 } 1564 } 1565 1566 static void fpsimd_flush_thread_vl(enum vec_type type) 1567 { 1568 int vl, supported_vl; 1569 1570 /* 1571 * Reset the task vector length as required. This is where we 1572 * ensure that all user tasks have a valid vector length 1573 * configured: no kernel task can become a user task without 1574 * an exec and hence a call to this function. By the time the 1575 * first call to this function is made, all early hardware 1576 * probing is complete, so __sve_default_vl should be valid. 1577 * If a bug causes this to go wrong, we make some noise and 1578 * try to fudge thread.sve_vl to a safe value here. 1579 */ 1580 vl = task_get_vl_onexec(current, type); 1581 if (!vl) 1582 vl = get_default_vl(type); 1583 1584 if (WARN_ON(!sve_vl_valid(vl))) 1585 vl = vl_info[type].min_vl; 1586 1587 supported_vl = find_supported_vector_length(type, vl); 1588 if (WARN_ON(supported_vl != vl)) 1589 vl = supported_vl; 1590 1591 task_set_vl(current, type, vl); 1592 1593 /* 1594 * If the task is not set to inherit, ensure that the vector 1595 * length will be reset by a subsequent exec: 1596 */ 1597 if (!test_thread_flag(vec_vl_inherit_flag(type))) 1598 task_set_vl_onexec(current, type, 0); 1599 } 1600 1601 void fpsimd_flush_thread(void) 1602 { 1603 void *sve_state = NULL; 1604 void *sme_state = NULL; 1605 1606 if (!system_supports_fpsimd()) 1607 return; 1608 1609 get_cpu_fpsimd_context(); 1610 1611 fpsimd_flush_task_state(current); 1612 memset(¤t->thread.uw.fpsimd_state, 0, 1613 sizeof(current->thread.uw.fpsimd_state)); 1614 1615 if (system_supports_sve()) { 1616 clear_thread_flag(TIF_SVE); 1617 1618 /* Defer kfree() while in atomic context */ 1619 sve_state = current->thread.sve_state; 1620 current->thread.sve_state = NULL; 1621 1622 fpsimd_flush_thread_vl(ARM64_VEC_SVE); 1623 } 1624 1625 if (system_supports_sme()) { 1626 clear_thread_flag(TIF_SME); 1627 1628 /* Defer kfree() while in atomic context */ 1629 sme_state = current->thread.sme_state; 1630 current->thread.sme_state = NULL; 1631 1632 fpsimd_flush_thread_vl(ARM64_VEC_SME); 1633 current->thread.svcr = 0; 1634 } 1635 1636 if (system_supports_fpmr()) 1637 current->thread.uw.fpmr = 0; 1638 1639 current->thread.fp_type = FP_STATE_FPSIMD; 1640 1641 put_cpu_fpsimd_context(); 1642 kfree(sve_state); 1643 kfree(sme_state); 1644 } 1645 1646 /* 1647 * Save the userland FPSIMD state of 'current' to memory, but only if the state 1648 * currently held in the registers does in fact belong to 'current' 1649 */ 1650 void fpsimd_preserve_current_state(void) 1651 { 1652 if (!system_supports_fpsimd()) 1653 return; 1654 1655 get_cpu_fpsimd_context(); 1656 fpsimd_save_user_state(); 1657 put_cpu_fpsimd_context(); 1658 } 1659 1660 /* 1661 * Associate current's FPSIMD context with this cpu 1662 * The caller must have ownership of the cpu FPSIMD context before calling 1663 * this function. 1664 */ 1665 static void fpsimd_bind_task_to_cpu(void) 1666 { 1667 struct cpu_fp_state *last = this_cpu_ptr(&fpsimd_last_state); 1668 1669 WARN_ON(!system_supports_fpsimd()); 1670 last->st = ¤t->thread.uw.fpsimd_state; 1671 last->sve_state = current->thread.sve_state; 1672 last->sme_state = current->thread.sme_state; 1673 last->sve_vl = task_get_sve_vl(current); 1674 last->sme_vl = task_get_sme_vl(current); 1675 last->svcr = ¤t->thread.svcr; 1676 last->fpmr = ¤t->thread.uw.fpmr; 1677 last->fp_type = ¤t->thread.fp_type; 1678 last->to_save = FP_STATE_CURRENT; 1679 current->thread.fpsimd_cpu = smp_processor_id(); 1680 1681 /* 1682 * Toggle SVE and SME trapping for userspace if needed, these 1683 * are serialsied by ret_to_user(). 1684 */ 1685 if (system_supports_sme()) { 1686 if (test_thread_flag(TIF_SME)) 1687 sme_user_enable(); 1688 else 1689 sme_user_disable(); 1690 } 1691 1692 if (system_supports_sve()) { 1693 if (test_thread_flag(TIF_SVE)) 1694 sve_user_enable(); 1695 else 1696 sve_user_disable(); 1697 } 1698 } 1699 1700 void fpsimd_bind_state_to_cpu(struct cpu_fp_state *state) 1701 { 1702 struct cpu_fp_state *last = this_cpu_ptr(&fpsimd_last_state); 1703 1704 WARN_ON(!system_supports_fpsimd()); 1705 WARN_ON(!in_softirq() && !irqs_disabled()); 1706 1707 *last = *state; 1708 } 1709 1710 /* 1711 * Load the userland FPSIMD state of 'current' from memory, but only if the 1712 * FPSIMD state already held in the registers is /not/ the most recent FPSIMD 1713 * state of 'current'. This is called when we are preparing to return to 1714 * userspace to ensure that userspace sees a good register state. 1715 */ 1716 void fpsimd_restore_current_state(void) 1717 { 1718 /* 1719 * TIF_FOREIGN_FPSTATE is set on the init task and copied by 1720 * arch_dup_task_struct() regardless of whether FP/SIMD is detected. 1721 * Thus user threads can have this set even when FP/SIMD hasn't been 1722 * detected. 1723 * 1724 * When FP/SIMD is detected, begin_new_exec() will set 1725 * TIF_FOREIGN_FPSTATE via flush_thread() -> fpsimd_flush_thread(), 1726 * and fpsimd_thread_switch() will set TIF_FOREIGN_FPSTATE when 1727 * switching tasks. We detect FP/SIMD before we exec the first user 1728 * process, ensuring this has TIF_FOREIGN_FPSTATE set and 1729 * do_notify_resume() will call fpsimd_restore_current_state() to 1730 * install the user FP/SIMD context. 1731 * 1732 * When FP/SIMD is not detected, nothing else will clear or set 1733 * TIF_FOREIGN_FPSTATE prior to the first return to userspace, and 1734 * we must clear TIF_FOREIGN_FPSTATE to avoid do_notify_resume() 1735 * looping forever calling fpsimd_restore_current_state(). 1736 */ 1737 if (!system_supports_fpsimd()) { 1738 clear_thread_flag(TIF_FOREIGN_FPSTATE); 1739 return; 1740 } 1741 1742 get_cpu_fpsimd_context(); 1743 1744 if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) { 1745 task_fpsimd_load(); 1746 fpsimd_bind_task_to_cpu(); 1747 } 1748 1749 put_cpu_fpsimd_context(); 1750 } 1751 1752 void fpsimd_update_current_state(struct user_fpsimd_state const *state) 1753 { 1754 if (WARN_ON(!system_supports_fpsimd())) 1755 return; 1756 1757 current->thread.uw.fpsimd_state = *state; 1758 if (current->thread.fp_type == FP_STATE_SVE) 1759 fpsimd_to_sve(current); 1760 } 1761 1762 /* 1763 * Invalidate live CPU copies of task t's FPSIMD state 1764 * 1765 * This function may be called with preemption enabled. The barrier() 1766 * ensures that the assignment to fpsimd_cpu is visible to any 1767 * preemption/softirq that could race with set_tsk_thread_flag(), so 1768 * that TIF_FOREIGN_FPSTATE cannot be spuriously re-cleared. 1769 * 1770 * The final barrier ensures that TIF_FOREIGN_FPSTATE is seen set by any 1771 * subsequent code. 1772 */ 1773 void fpsimd_flush_task_state(struct task_struct *t) 1774 { 1775 t->thread.fpsimd_cpu = NR_CPUS; 1776 /* 1777 * If we don't support fpsimd, bail out after we have 1778 * reset the fpsimd_cpu for this task and clear the 1779 * FPSTATE. 1780 */ 1781 if (!system_supports_fpsimd()) 1782 return; 1783 barrier(); 1784 set_tsk_thread_flag(t, TIF_FOREIGN_FPSTATE); 1785 1786 barrier(); 1787 } 1788 1789 void fpsimd_save_and_flush_current_state(void) 1790 { 1791 if (!system_supports_fpsimd()) 1792 return; 1793 1794 get_cpu_fpsimd_context(); 1795 fpsimd_save_user_state(); 1796 fpsimd_flush_task_state(current); 1797 put_cpu_fpsimd_context(); 1798 } 1799 1800 /* 1801 * Save the FPSIMD state to memory and invalidate cpu view. 1802 * This function must be called with preemption disabled. 1803 */ 1804 void fpsimd_save_and_flush_cpu_state(void) 1805 { 1806 unsigned long flags; 1807 1808 if (!system_supports_fpsimd()) 1809 return; 1810 WARN_ON(preemptible()); 1811 local_irq_save(flags); 1812 fpsimd_save_user_state(); 1813 fpsimd_flush_cpu_state(); 1814 local_irq_restore(flags); 1815 } 1816 1817 #ifdef CONFIG_KERNEL_MODE_NEON 1818 1819 /* 1820 * Kernel-side NEON support functions 1821 */ 1822 1823 /* 1824 * kernel_neon_begin(): obtain the CPU FPSIMD registers for use by the calling 1825 * context 1826 * 1827 * Must not be called unless may_use_simd() returns true. 1828 * Task context in the FPSIMD registers is saved back to memory as necessary. 1829 * 1830 * A matching call to kernel_neon_end() must be made before returning from the 1831 * calling context. 1832 * 1833 * The caller may freely use the FPSIMD registers until kernel_neon_end() is 1834 * called. 1835 */ 1836 void kernel_neon_begin(void) 1837 { 1838 if (WARN_ON(!system_supports_fpsimd())) 1839 return; 1840 1841 BUG_ON(!may_use_simd()); 1842 1843 get_cpu_fpsimd_context(); 1844 1845 /* Save unsaved fpsimd state, if any: */ 1846 if (test_thread_flag(TIF_KERNEL_FPSTATE)) { 1847 BUG_ON(IS_ENABLED(CONFIG_PREEMPT_RT) || !in_serving_softirq()); 1848 fpsimd_save_kernel_state(current); 1849 } else { 1850 fpsimd_save_user_state(); 1851 1852 /* 1853 * Set the thread flag so that the kernel mode FPSIMD state 1854 * will be context switched along with the rest of the task 1855 * state. 1856 * 1857 * On non-PREEMPT_RT, softirqs may interrupt task level kernel 1858 * mode FPSIMD, but the task will not be preemptible so setting 1859 * TIF_KERNEL_FPSTATE for those would be both wrong (as it 1860 * would mark the task context FPSIMD state as requiring a 1861 * context switch) and unnecessary. 1862 * 1863 * On PREEMPT_RT, softirqs are serviced from a separate thread, 1864 * which is scheduled as usual, and this guarantees that these 1865 * softirqs are not interrupting use of the FPSIMD in kernel 1866 * mode in task context. So in this case, setting the flag here 1867 * is always appropriate. 1868 */ 1869 if (IS_ENABLED(CONFIG_PREEMPT_RT) || !in_serving_softirq()) 1870 set_thread_flag(TIF_KERNEL_FPSTATE); 1871 } 1872 1873 /* Invalidate any task state remaining in the fpsimd regs: */ 1874 fpsimd_flush_cpu_state(); 1875 1876 put_cpu_fpsimd_context(); 1877 } 1878 EXPORT_SYMBOL_GPL(kernel_neon_begin); 1879 1880 /* 1881 * kernel_neon_end(): give the CPU FPSIMD registers back to the current task 1882 * 1883 * Must be called from a context in which kernel_neon_begin() was previously 1884 * called, with no call to kernel_neon_end() in the meantime. 1885 * 1886 * The caller must not use the FPSIMD registers after this function is called, 1887 * unless kernel_neon_begin() is called again in the meantime. 1888 */ 1889 void kernel_neon_end(void) 1890 { 1891 if (!system_supports_fpsimd()) 1892 return; 1893 1894 /* 1895 * If we are returning from a nested use of kernel mode FPSIMD, restore 1896 * the task context kernel mode FPSIMD state. This can only happen when 1897 * running in softirq context on non-PREEMPT_RT. 1898 */ 1899 if (!IS_ENABLED(CONFIG_PREEMPT_RT) && in_serving_softirq() && 1900 test_thread_flag(TIF_KERNEL_FPSTATE)) 1901 fpsimd_load_kernel_state(current); 1902 else 1903 clear_thread_flag(TIF_KERNEL_FPSTATE); 1904 } 1905 EXPORT_SYMBOL_GPL(kernel_neon_end); 1906 1907 #ifdef CONFIG_EFI 1908 1909 static struct user_fpsimd_state efi_fpsimd_state; 1910 static bool efi_fpsimd_state_used; 1911 static bool efi_sve_state_used; 1912 static bool efi_sm_state; 1913 1914 /* 1915 * EFI runtime services support functions 1916 * 1917 * The ABI for EFI runtime services allows EFI to use FPSIMD during the call. 1918 * This means that for EFI (and only for EFI), we have to assume that FPSIMD 1919 * is always used rather than being an optional accelerator. 1920 * 1921 * These functions provide the necessary support for ensuring FPSIMD 1922 * save/restore in the contexts from which EFI is used. 1923 * 1924 * Do not use them for any other purpose -- if tempted to do so, you are 1925 * either doing something wrong or you need to propose some refactoring. 1926 */ 1927 1928 /* 1929 * __efi_fpsimd_begin(): prepare FPSIMD for making an EFI runtime services call 1930 */ 1931 void __efi_fpsimd_begin(void) 1932 { 1933 if (!system_supports_fpsimd()) 1934 return; 1935 1936 WARN_ON(preemptible()); 1937 1938 if (may_use_simd()) { 1939 kernel_neon_begin(); 1940 } else { 1941 /* 1942 * If !efi_sve_state, SVE can't be in use yet and doesn't need 1943 * preserving: 1944 */ 1945 if (system_supports_sve() && efi_sve_state != NULL) { 1946 bool ffr = true; 1947 u64 svcr; 1948 1949 efi_sve_state_used = true; 1950 1951 if (system_supports_sme()) { 1952 svcr = read_sysreg_s(SYS_SVCR); 1953 1954 efi_sm_state = svcr & SVCR_SM_MASK; 1955 1956 /* 1957 * Unless we have FA64 FFR does not 1958 * exist in streaming mode. 1959 */ 1960 if (!system_supports_fa64()) 1961 ffr = !(svcr & SVCR_SM_MASK); 1962 } 1963 1964 sve_save_state(efi_sve_state + sve_ffr_offset(sve_max_vl()), 1965 &efi_fpsimd_state.fpsr, ffr); 1966 1967 if (system_supports_sme()) 1968 sysreg_clear_set_s(SYS_SVCR, 1969 SVCR_SM_MASK, 0); 1970 1971 } else { 1972 fpsimd_save_state(&efi_fpsimd_state); 1973 } 1974 1975 efi_fpsimd_state_used = true; 1976 } 1977 } 1978 1979 /* 1980 * __efi_fpsimd_end(): clean up FPSIMD after an EFI runtime services call 1981 */ 1982 void __efi_fpsimd_end(void) 1983 { 1984 if (!system_supports_fpsimd()) 1985 return; 1986 1987 if (!efi_fpsimd_state_used) { 1988 kernel_neon_end(); 1989 } else { 1990 if (system_supports_sve() && efi_sve_state_used) { 1991 bool ffr = true; 1992 1993 /* 1994 * Restore streaming mode; EFI calls are 1995 * normal function calls so should not return in 1996 * streaming mode. 1997 */ 1998 if (system_supports_sme()) { 1999 if (efi_sm_state) { 2000 sysreg_clear_set_s(SYS_SVCR, 2001 0, 2002 SVCR_SM_MASK); 2003 2004 /* 2005 * Unless we have FA64 FFR does not 2006 * exist in streaming mode. 2007 */ 2008 if (!system_supports_fa64()) 2009 ffr = false; 2010 } 2011 } 2012 2013 sve_load_state(efi_sve_state + sve_ffr_offset(sve_max_vl()), 2014 &efi_fpsimd_state.fpsr, ffr); 2015 2016 efi_sve_state_used = false; 2017 } else { 2018 fpsimd_load_state(&efi_fpsimd_state); 2019 } 2020 2021 efi_fpsimd_state_used = false; 2022 } 2023 } 2024 2025 #endif /* CONFIG_EFI */ 2026 2027 #endif /* CONFIG_KERNEL_MODE_NEON */ 2028 2029 #ifdef CONFIG_CPU_PM 2030 static int fpsimd_cpu_pm_notifier(struct notifier_block *self, 2031 unsigned long cmd, void *v) 2032 { 2033 switch (cmd) { 2034 case CPU_PM_ENTER: 2035 fpsimd_save_and_flush_cpu_state(); 2036 break; 2037 case CPU_PM_EXIT: 2038 break; 2039 case CPU_PM_ENTER_FAILED: 2040 default: 2041 return NOTIFY_DONE; 2042 } 2043 return NOTIFY_OK; 2044 } 2045 2046 static struct notifier_block fpsimd_cpu_pm_notifier_block = { 2047 .notifier_call = fpsimd_cpu_pm_notifier, 2048 }; 2049 2050 static void __init fpsimd_pm_init(void) 2051 { 2052 cpu_pm_register_notifier(&fpsimd_cpu_pm_notifier_block); 2053 } 2054 2055 #else 2056 static inline void fpsimd_pm_init(void) { } 2057 #endif /* CONFIG_CPU_PM */ 2058 2059 #ifdef CONFIG_HOTPLUG_CPU 2060 static int fpsimd_cpu_dead(unsigned int cpu) 2061 { 2062 per_cpu(fpsimd_last_state.st, cpu) = NULL; 2063 return 0; 2064 } 2065 2066 static inline void fpsimd_hotplug_init(void) 2067 { 2068 cpuhp_setup_state_nocalls(CPUHP_ARM64_FPSIMD_DEAD, "arm64/fpsimd:dead", 2069 NULL, fpsimd_cpu_dead); 2070 } 2071 2072 #else 2073 static inline void fpsimd_hotplug_init(void) { } 2074 #endif 2075 2076 void cpu_enable_fpsimd(const struct arm64_cpu_capabilities *__always_unused p) 2077 { 2078 unsigned long enable = CPACR_EL1_FPEN_EL1EN | CPACR_EL1_FPEN_EL0EN; 2079 write_sysreg(read_sysreg(CPACR_EL1) | enable, CPACR_EL1); 2080 isb(); 2081 } 2082 2083 /* 2084 * FP/SIMD support code initialisation. 2085 */ 2086 static int __init fpsimd_init(void) 2087 { 2088 if (cpu_have_named_feature(FP)) { 2089 fpsimd_pm_init(); 2090 fpsimd_hotplug_init(); 2091 } else { 2092 pr_notice("Floating-point is not implemented\n"); 2093 } 2094 2095 if (!cpu_have_named_feature(ASIMD)) 2096 pr_notice("Advanced SIMD is not implemented\n"); 2097 2098 2099 sve_sysctl_init(); 2100 sme_sysctl_init(); 2101 2102 return 0; 2103 } 2104 core_initcall(fpsimd_init); 2105