1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * FPU data structures: 4 */ 5 #ifndef _ASM_X86_FPU_H 6 #define _ASM_X86_FPU_H 7 8 #include <asm/page_types.h> 9 10 /* 11 * The legacy x87 FPU state format, as saved by FSAVE and 12 * restored by the FRSTOR instructions: 13 */ 14 struct fregs_state { 15 u32 cwd; /* FPU Control Word */ 16 u32 swd; /* FPU Status Word */ 17 u32 twd; /* FPU Tag Word */ 18 u32 fip; /* FPU IP Offset */ 19 u32 fcs; /* FPU IP Selector */ 20 u32 foo; /* FPU Operand Pointer Offset */ 21 u32 fos; /* FPU Operand Pointer Selector */ 22 23 /* 8*10 bytes for each FP-reg = 80 bytes: */ 24 u32 st_space[20]; 25 26 /* Software status information [not touched by FSAVE]: */ 27 u32 status; 28 }; 29 30 /* 31 * The legacy fx SSE/MMX FPU state format, as saved by FXSAVE and 32 * restored by the FXRSTOR instructions. It's similar to the FSAVE 33 * format, but differs in some areas, plus has extensions at 34 * the end for the XMM registers. 35 */ 36 struct fxregs_state { 37 u16 cwd; /* Control Word */ 38 u16 swd; /* Status Word */ 39 u16 twd; /* Tag Word */ 40 u16 fop; /* Last Instruction Opcode */ 41 union { 42 struct { 43 u64 rip; /* Instruction Pointer */ 44 u64 rdp; /* Data Pointer */ 45 }; 46 struct { 47 u32 fip; /* FPU IP Offset */ 48 u32 fcs; /* FPU IP Selector */ 49 u32 foo; /* FPU Operand Offset */ 50 u32 fos; /* FPU Operand Selector */ 51 }; 52 }; 53 u32 mxcsr; /* MXCSR Register State */ 54 u32 mxcsr_mask; /* MXCSR Mask */ 55 56 /* 8*16 bytes for each FP-reg = 128 bytes: */ 57 u32 st_space[32]; 58 59 /* 16*16 bytes for each XMM-reg = 256 bytes: */ 60 u32 xmm_space[64]; 61 62 u32 padding[12]; 63 64 union { 65 u32 padding1[12]; 66 u32 sw_reserved[12]; 67 }; 68 69 } __attribute__((aligned(16))); 70 71 /* Default value for fxregs_state.mxcsr: */ 72 #define MXCSR_DEFAULT 0x1f80 73 74 /* Copy both mxcsr & mxcsr_flags with a single u64 memcpy: */ 75 #define MXCSR_AND_FLAGS_SIZE sizeof(u64) 76 77 /* 78 * Software based FPU emulation state. This is arbitrary really, 79 * it matches the x87 format to make it easier to understand: 80 */ 81 struct swregs_state { 82 u32 cwd; 83 u32 swd; 84 u32 twd; 85 u32 fip; 86 u32 fcs; 87 u32 foo; 88 u32 fos; 89 /* 8*10 bytes for each FP-reg = 80 bytes: */ 90 u32 st_space[20]; 91 u8 ftop; 92 u8 changed; 93 u8 lookahead; 94 u8 no_update; 95 u8 rm; 96 u8 alimit; 97 struct math_emu_info *info; 98 u32 entry_eip; 99 }; 100 101 /* 102 * List of XSAVE features Linux knows about: 103 */ 104 enum xfeature { 105 XFEATURE_FP, 106 XFEATURE_SSE, 107 /* 108 * Values above here are "legacy states". 109 * Those below are "extended states". 110 */ 111 XFEATURE_YMM, 112 XFEATURE_BNDREGS, 113 XFEATURE_BNDCSR, 114 XFEATURE_OPMASK, 115 XFEATURE_ZMM_Hi256, 116 XFEATURE_Hi16_ZMM, 117 XFEATURE_PT_UNIMPLEMENTED_SO_FAR, 118 XFEATURE_PKRU, 119 XFEATURE_PASID, 120 XFEATURE_CET_USER, 121 XFEATURE_CET_KERNEL_UNUSED, 122 XFEATURE_RSRVD_COMP_13, 123 XFEATURE_RSRVD_COMP_14, 124 XFEATURE_LBR, 125 XFEATURE_RSRVD_COMP_16, 126 XFEATURE_XTILE_CFG, 127 XFEATURE_XTILE_DATA, 128 129 XFEATURE_MAX, 130 }; 131 132 #define XFEATURE_MASK_FP (1 << XFEATURE_FP) 133 #define XFEATURE_MASK_SSE (1 << XFEATURE_SSE) 134 #define XFEATURE_MASK_YMM (1 << XFEATURE_YMM) 135 #define XFEATURE_MASK_BNDREGS (1 << XFEATURE_BNDREGS) 136 #define XFEATURE_MASK_BNDCSR (1 << XFEATURE_BNDCSR) 137 #define XFEATURE_MASK_OPMASK (1 << XFEATURE_OPMASK) 138 #define XFEATURE_MASK_ZMM_Hi256 (1 << XFEATURE_ZMM_Hi256) 139 #define XFEATURE_MASK_Hi16_ZMM (1 << XFEATURE_Hi16_ZMM) 140 #define XFEATURE_MASK_PT (1 << XFEATURE_PT_UNIMPLEMENTED_SO_FAR) 141 #define XFEATURE_MASK_PKRU (1 << XFEATURE_PKRU) 142 #define XFEATURE_MASK_PASID (1 << XFEATURE_PASID) 143 #define XFEATURE_MASK_CET_USER (1 << XFEATURE_CET_USER) 144 #define XFEATURE_MASK_CET_KERNEL (1 << XFEATURE_CET_KERNEL_UNUSED) 145 #define XFEATURE_MASK_LBR (1 << XFEATURE_LBR) 146 #define XFEATURE_MASK_XTILE_CFG (1 << XFEATURE_XTILE_CFG) 147 #define XFEATURE_MASK_XTILE_DATA (1 << XFEATURE_XTILE_DATA) 148 149 #define XFEATURE_MASK_FPSSE (XFEATURE_MASK_FP | XFEATURE_MASK_SSE) 150 #define XFEATURE_MASK_AVX512 (XFEATURE_MASK_OPMASK \ 151 | XFEATURE_MASK_ZMM_Hi256 \ 152 | XFEATURE_MASK_Hi16_ZMM) 153 154 #ifdef CONFIG_X86_64 155 # define XFEATURE_MASK_XTILE (XFEATURE_MASK_XTILE_DATA \ 156 | XFEATURE_MASK_XTILE_CFG) 157 #else 158 # define XFEATURE_MASK_XTILE (0) 159 #endif 160 161 #define FIRST_EXTENDED_XFEATURE XFEATURE_YMM 162 163 struct reg_128_bit { 164 u8 regbytes[128/8]; 165 }; 166 struct reg_256_bit { 167 u8 regbytes[256/8]; 168 }; 169 struct reg_512_bit { 170 u8 regbytes[512/8]; 171 }; 172 struct reg_1024_byte { 173 u8 regbytes[1024]; 174 }; 175 176 /* 177 * State component 2: 178 * 179 * There are 16x 256-bit AVX registers named YMM0-YMM15. 180 * The low 128 bits are aliased to the 16 SSE registers (XMM0-XMM15) 181 * and are stored in 'struct fxregs_state::xmm_space[]' in the 182 * "legacy" area. 183 * 184 * The high 128 bits are stored here. 185 */ 186 struct ymmh_struct { 187 struct reg_128_bit hi_ymm[16]; 188 } __packed; 189 190 /* Intel MPX support: */ 191 192 struct mpx_bndreg { 193 u64 lower_bound; 194 u64 upper_bound; 195 } __packed; 196 /* 197 * State component 3 is used for the 4 128-bit bounds registers 198 */ 199 struct mpx_bndreg_state { 200 struct mpx_bndreg bndreg[4]; 201 } __packed; 202 203 /* 204 * State component 4 is used for the 64-bit user-mode MPX 205 * configuration register BNDCFGU and the 64-bit MPX status 206 * register BNDSTATUS. We call the pair "BNDCSR". 207 */ 208 struct mpx_bndcsr { 209 u64 bndcfgu; 210 u64 bndstatus; 211 } __packed; 212 213 /* 214 * The BNDCSR state is padded out to be 64-bytes in size. 215 */ 216 struct mpx_bndcsr_state { 217 union { 218 struct mpx_bndcsr bndcsr; 219 u8 pad_to_64_bytes[64]; 220 }; 221 } __packed; 222 223 /* AVX-512 Components: */ 224 225 /* 226 * State component 5 is used for the 8 64-bit opmask registers 227 * k0-k7 (opmask state). 228 */ 229 struct avx_512_opmask_state { 230 u64 opmask_reg[8]; 231 } __packed; 232 233 /* 234 * State component 6 is used for the upper 256 bits of the 235 * registers ZMM0-ZMM15. These 16 256-bit values are denoted 236 * ZMM0_H-ZMM15_H (ZMM_Hi256 state). 237 */ 238 struct avx_512_zmm_uppers_state { 239 struct reg_256_bit zmm_upper[16]; 240 } __packed; 241 242 /* 243 * State component 7 is used for the 16 512-bit registers 244 * ZMM16-ZMM31 (Hi16_ZMM state). 245 */ 246 struct avx_512_hi16_state { 247 struct reg_512_bit hi16_zmm[16]; 248 } __packed; 249 250 /* 251 * State component 9: 32-bit PKRU register. The state is 252 * 8 bytes long but only 4 bytes is used currently. 253 */ 254 struct pkru_state { 255 u32 pkru; 256 u32 pad; 257 } __packed; 258 259 /* 260 * State component 11 is Control-flow Enforcement user states 261 */ 262 struct cet_user_state { 263 /* user control-flow settings */ 264 u64 user_cet; 265 /* user shadow stack pointer */ 266 u64 user_ssp; 267 }; 268 269 /* 270 * State component 15: Architectural LBR configuration state. 271 * The size of Arch LBR state depends on the number of LBRs (lbr_depth). 272 */ 273 274 struct lbr_entry { 275 u64 from; 276 u64 to; 277 u64 info; 278 }; 279 280 struct arch_lbr_state { 281 u64 lbr_ctl; 282 u64 lbr_depth; 283 u64 ler_from; 284 u64 ler_to; 285 u64 ler_info; 286 struct lbr_entry entries[]; 287 }; 288 289 /* 290 * State component 17: 64-byte tile configuration register. 291 */ 292 struct xtile_cfg { 293 u64 tcfg[8]; 294 } __packed; 295 296 /* 297 * State component 18: 1KB tile data register. 298 * Each register represents 16 64-byte rows of the matrix 299 * data. But the number of registers depends on the actual 300 * implementation. 301 */ 302 struct xtile_data { 303 struct reg_1024_byte tmm; 304 } __packed; 305 306 /* 307 * State component 10 is supervisor state used for context-switching the 308 * PASID state. 309 */ 310 struct ia32_pasid_state { 311 u64 pasid; 312 } __packed; 313 314 struct xstate_header { 315 u64 xfeatures; 316 u64 xcomp_bv; 317 u64 reserved[6]; 318 } __attribute__((packed)); 319 320 /* 321 * xstate_header.xcomp_bv[63] indicates that the extended_state_area 322 * is in compacted format. 323 */ 324 #define XCOMP_BV_COMPACTED_FORMAT ((u64)1 << 63) 325 326 /* 327 * This is our most modern FPU state format, as saved by the XSAVE 328 * and restored by the XRSTOR instructions. 329 * 330 * It consists of a legacy fxregs portion, an xstate header and 331 * subsequent areas as defined by the xstate header. Not all CPUs 332 * support all the extensions, so the size of the extended area 333 * can vary quite a bit between CPUs. 334 */ 335 struct xregs_state { 336 struct fxregs_state i387; 337 struct xstate_header header; 338 u8 extended_state_area[]; 339 } __attribute__ ((packed, aligned (64))); 340 341 /* 342 * This is a union of all the possible FPU state formats 343 * put together, so that we can pick the right one runtime. 344 * 345 * The size of the structure is determined by the largest 346 * member - which is the xsave area. The padding is there 347 * to ensure that statically-allocated task_structs (just 348 * the init_task today) have enough space. 349 */ 350 union fpregs_state { 351 struct fregs_state fsave; 352 struct fxregs_state fxsave; 353 struct swregs_state soft; 354 struct xregs_state xsave; 355 u8 __padding[PAGE_SIZE]; 356 }; 357 358 struct fpstate { 359 /* @kernel_size: The size of the kernel register image */ 360 unsigned int size; 361 362 /* @user_size: The size in non-compacted UABI format */ 363 unsigned int user_size; 364 365 /* @xfeatures: xfeatures for which the storage is sized */ 366 u64 xfeatures; 367 368 /* @user_xfeatures: xfeatures valid in UABI buffers */ 369 u64 user_xfeatures; 370 371 /* @xfd: xfeatures disabled to trap userspace use. */ 372 u64 xfd; 373 374 /* @is_valloc: Indicator for dynamically allocated state */ 375 unsigned int is_valloc : 1; 376 377 /* @is_guest: Indicator for guest state (KVM) */ 378 unsigned int is_guest : 1; 379 380 /* 381 * @is_confidential: Indicator for KVM confidential mode. 382 * The FPU registers are restored by the 383 * vmentry firmware from encrypted guest 384 * memory. On vmexit the FPU registers are 385 * saved by firmware to encrypted guest memory 386 * and the registers are scrubbed before 387 * returning to the host. So there is no 388 * content which is worth saving and restoring. 389 * The fpstate has to be there so that 390 * preemption and softirq FPU usage works 391 * without special casing. 392 */ 393 unsigned int is_confidential : 1; 394 395 /* @in_use: State is in use */ 396 unsigned int in_use : 1; 397 398 /* @regs: The register state union for all supported formats */ 399 union fpregs_state regs; 400 401 /* @regs is dynamically sized! Don't add anything after @regs! */ 402 } __aligned(64); 403 404 #define FPU_GUEST_PERM_LOCKED BIT_ULL(63) 405 406 struct fpu_state_perm { 407 /* 408 * @__state_perm: 409 * 410 * This bitmap indicates the permission for state components, which 411 * are available to a thread group. The permission prctl() sets the 412 * enabled state bits in thread_group_leader()->thread.fpu. 413 * 414 * All run time operations use the per thread information in the 415 * currently active fpu.fpstate which contains the xfeature masks 416 * and sizes for kernel and user space. 417 * 418 * This master permission field is only to be used when 419 * task.fpu.fpstate based checks fail to validate whether the task 420 * is allowed to expand its xfeatures set which requires to 421 * allocate a larger sized fpstate buffer. 422 * 423 * Do not access this field directly. Use the provided helper 424 * function. Unlocked access is possible for quick checks. 425 */ 426 u64 __state_perm; 427 428 /* 429 * @__state_size: 430 * 431 * The size required for @__state_perm. Only valid to access 432 * with sighand locked. 433 */ 434 unsigned int __state_size; 435 436 /* 437 * @__user_state_size: 438 * 439 * The size required for @__state_perm user part. Only valid to 440 * access with sighand locked. 441 */ 442 unsigned int __user_state_size; 443 }; 444 445 /* 446 * Highest level per task FPU state data structure that 447 * contains the FPU register state plus various FPU 448 * state fields: 449 */ 450 struct fpu { 451 /* 452 * @last_cpu: 453 * 454 * Records the last CPU on which this context was loaded into 455 * FPU registers. (In the lazy-restore case we might be 456 * able to reuse FPU registers across multiple context switches 457 * this way, if no intermediate task used the FPU.) 458 * 459 * A value of -1 is used to indicate that the FPU state in context 460 * memory is newer than the FPU state in registers, and that the 461 * FPU state should be reloaded next time the task is run. 462 */ 463 unsigned int last_cpu; 464 465 /* 466 * @avx512_timestamp: 467 * 468 * Records the timestamp of AVX512 use during last context switch. 469 */ 470 unsigned long avx512_timestamp; 471 472 /* 473 * @fpstate: 474 * 475 * Pointer to the active struct fpstate. Initialized to 476 * point at @__fpstate below. 477 */ 478 struct fpstate *fpstate; 479 480 /* 481 * @__task_fpstate: 482 * 483 * Pointer to an inactive struct fpstate. Initialized to NULL. Is 484 * used only for KVM support to swap out the regular task fpstate. 485 */ 486 struct fpstate *__task_fpstate; 487 488 /* 489 * @perm: 490 * 491 * Permission related information 492 */ 493 struct fpu_state_perm perm; 494 495 /* 496 * @guest_perm: 497 * 498 * Permission related information for guest pseudo FPUs 499 */ 500 struct fpu_state_perm guest_perm; 501 502 /* 503 * @__fpstate: 504 * 505 * Initial in-memory storage for FPU registers which are saved in 506 * context switch and when the kernel uses the FPU. The registers 507 * are restored from this storage on return to user space if they 508 * are not longer containing the tasks FPU register state. 509 */ 510 struct fpstate __fpstate; 511 /* 512 * WARNING: '__fpstate' is dynamically-sized. Do not put 513 * anything after it here. 514 */ 515 }; 516 517 /* 518 * Guest pseudo FPU container 519 */ 520 struct fpu_guest { 521 /* 522 * @xfeatures: xfeature bitmap of features which are 523 * currently enabled for the guest vCPU. 524 */ 525 u64 xfeatures; 526 527 /* 528 * @perm: xfeature bitmap of features which are 529 * permitted to be enabled for the guest 530 * vCPU. 531 */ 532 u64 perm; 533 534 /* 535 * @xfd_err: Save the guest value. 536 */ 537 u64 xfd_err; 538 539 /* 540 * @uabi_size: Size required for save/restore 541 */ 542 unsigned int uabi_size; 543 544 /* 545 * @fpstate: Pointer to the allocated guest fpstate 546 */ 547 struct fpstate *fpstate; 548 }; 549 550 /* 551 * FPU state configuration data. Initialized at boot time. Read only after init. 552 */ 553 struct fpu_state_config { 554 /* 555 * @max_size: 556 * 557 * The maximum size of the register state buffer. Includes all 558 * supported features except independent managed features. 559 */ 560 unsigned int max_size; 561 562 /* 563 * @default_size: 564 * 565 * The default size of the register state buffer. Includes all 566 * supported features except independent managed features and 567 * features which have to be requested by user space before usage. 568 */ 569 unsigned int default_size; 570 571 /* 572 * @max_features: 573 * 574 * The maximum supported features bitmap. Does not include 575 * independent managed features. 576 */ 577 u64 max_features; 578 579 /* 580 * @default_features: 581 * 582 * The default supported features bitmap. Does not include 583 * independent managed features and features which have to 584 * be requested by user space before usage. 585 */ 586 u64 default_features; 587 /* 588 * @legacy_features: 589 * 590 * Features which can be reported back to user space 591 * even without XSAVE support, i.e. legacy features FP + SSE 592 */ 593 u64 legacy_features; 594 }; 595 596 /* FPU state configuration information */ 597 extern struct fpu_state_config fpu_kernel_cfg, fpu_user_cfg; 598 599 #endif /* _ASM_X86_FPU_H */ 600