1 /* 2 * AArch64 specific helpers 3 * 4 * Copyright (c) 2013 Alexander Graf <agraf@suse.de> 5 * 6 * This library is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU Lesser General Public 8 * License as published by the Free Software Foundation; either 9 * version 2.1 of the License, or (at your option) any later version. 10 * 11 * This library is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * Lesser General Public License for more details. 15 * 16 * You should have received a copy of the GNU Lesser General Public 17 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 18 */ 19 20 #include "qemu/osdep.h" 21 #include "qemu/units.h" 22 #include "cpu.h" 23 #include "gdbstub/helpers.h" 24 #include "exec/helper-proto.h" 25 #include "qemu/host-utils.h" 26 #include "qemu/log.h" 27 #include "qemu/main-loop.h" 28 #include "qemu/bitops.h" 29 #include "internals.h" 30 #include "qemu/crc32c.h" 31 #include "exec/cpu-common.h" 32 #include "exec/exec-all.h" 33 #include "exec/cpu_ldst.h" 34 #include "qemu/int128.h" 35 #include "qemu/atomic128.h" 36 #include "fpu/softfloat.h" 37 #include <zlib.h> /* for crc32 */ 38 #ifdef CONFIG_USER_ONLY 39 #include "user/page-protection.h" 40 #endif 41 #include "vec_internal.h" 42 43 /* C2.4.7 Multiply and divide */ 44 /* special cases for 0 and LLONG_MIN are mandated by the standard */ 45 uint64_t HELPER(udiv64)(uint64_t num, uint64_t den) 46 { 47 if (den == 0) { 48 return 0; 49 } 50 return num / den; 51 } 52 53 int64_t HELPER(sdiv64)(int64_t num, int64_t den) 54 { 55 if (den == 0) { 56 return 0; 57 } 58 if (num == LLONG_MIN && den == -1) { 59 return LLONG_MIN; 60 } 61 return num / den; 62 } 63 64 uint64_t HELPER(rbit64)(uint64_t x) 65 { 66 return revbit64(x); 67 } 68 69 void HELPER(msr_i_spsel)(CPUARMState *env, uint32_t imm) 70 { 71 update_spsel(env, imm); 72 } 73 74 void HELPER(msr_set_allint_el1)(CPUARMState *env) 75 { 76 /* ALLINT update to PSTATE. */ 77 if (arm_hcrx_el2_eff(env) & HCRX_TALLINT) { 78 raise_exception_ra(env, EXCP_UDEF, 79 syn_aa64_sysregtrap(0, 1, 0, 4, 1, 0x1f, 0), 2, 80 GETPC()); 81 } 82 83 env->pstate |= PSTATE_ALLINT; 84 } 85 86 static void daif_check(CPUARMState *env, uint32_t op, 87 uint32_t imm, uintptr_t ra) 88 { 89 /* DAIF update to PSTATE. This is OK from EL0 only if UMA is set. */ 90 if (arm_current_el(env) == 0 && !(arm_sctlr(env, 0) & SCTLR_UMA)) { 91 raise_exception_ra(env, EXCP_UDEF, 92 syn_aa64_sysregtrap(0, extract32(op, 0, 3), 93 extract32(op, 3, 3), 4, 94 imm, 0x1f, 0), 95 exception_target_el(env), ra); 96 } 97 } 98 99 void HELPER(msr_i_daifset)(CPUARMState *env, uint32_t imm) 100 { 101 daif_check(env, 0x1e, imm, GETPC()); 102 env->daif |= (imm << 6) & PSTATE_DAIF; 103 arm_rebuild_hflags(env); 104 } 105 106 void HELPER(msr_i_daifclear)(CPUARMState *env, uint32_t imm) 107 { 108 daif_check(env, 0x1f, imm, GETPC()); 109 env->daif &= ~((imm << 6) & PSTATE_DAIF); 110 arm_rebuild_hflags(env); 111 } 112 113 /* Convert a softfloat float_relation_ (as returned by 114 * the float*_compare functions) to the correct ARM 115 * NZCV flag state. 116 */ 117 static inline uint32_t float_rel_to_flags(int res) 118 { 119 uint64_t flags; 120 switch (res) { 121 case float_relation_equal: 122 flags = PSTATE_Z | PSTATE_C; 123 break; 124 case float_relation_less: 125 flags = PSTATE_N; 126 break; 127 case float_relation_greater: 128 flags = PSTATE_C; 129 break; 130 case float_relation_unordered: 131 default: 132 flags = PSTATE_C | PSTATE_V; 133 break; 134 } 135 return flags; 136 } 137 138 uint64_t HELPER(vfp_cmph_a64)(uint32_t x, uint32_t y, float_status *fp_status) 139 { 140 return float_rel_to_flags(float16_compare_quiet(x, y, fp_status)); 141 } 142 143 uint64_t HELPER(vfp_cmpeh_a64)(uint32_t x, uint32_t y, float_status *fp_status) 144 { 145 return float_rel_to_flags(float16_compare(x, y, fp_status)); 146 } 147 148 uint64_t HELPER(vfp_cmps_a64)(float32 x, float32 y, float_status *fp_status) 149 { 150 return float_rel_to_flags(float32_compare_quiet(x, y, fp_status)); 151 } 152 153 uint64_t HELPER(vfp_cmpes_a64)(float32 x, float32 y, float_status *fp_status) 154 { 155 return float_rel_to_flags(float32_compare(x, y, fp_status)); 156 } 157 158 uint64_t HELPER(vfp_cmpd_a64)(float64 x, float64 y, float_status *fp_status) 159 { 160 return float_rel_to_flags(float64_compare_quiet(x, y, fp_status)); 161 } 162 163 uint64_t HELPER(vfp_cmped_a64)(float64 x, float64 y, float_status *fp_status) 164 { 165 return float_rel_to_flags(float64_compare(x, y, fp_status)); 166 } 167 168 float32 HELPER(vfp_mulxs)(float32 a, float32 b, float_status *fpst) 169 { 170 a = float32_squash_input_denormal(a, fpst); 171 b = float32_squash_input_denormal(b, fpst); 172 173 if ((float32_is_zero(a) && float32_is_infinity(b)) || 174 (float32_is_infinity(a) && float32_is_zero(b))) { 175 /* 2.0 with the sign bit set to sign(A) XOR sign(B) */ 176 return make_float32((1U << 30) | 177 ((float32_val(a) ^ float32_val(b)) & (1U << 31))); 178 } 179 return float32_mul(a, b, fpst); 180 } 181 182 float64 HELPER(vfp_mulxd)(float64 a, float64 b, float_status *fpst) 183 { 184 a = float64_squash_input_denormal(a, fpst); 185 b = float64_squash_input_denormal(b, fpst); 186 187 if ((float64_is_zero(a) && float64_is_infinity(b)) || 188 (float64_is_infinity(a) && float64_is_zero(b))) { 189 /* 2.0 with the sign bit set to sign(A) XOR sign(B) */ 190 return make_float64((1ULL << 62) | 191 ((float64_val(a) ^ float64_val(b)) & (1ULL << 63))); 192 } 193 return float64_mul(a, b, fpst); 194 } 195 196 /* 64bit/double versions of the neon float compare functions */ 197 uint64_t HELPER(neon_ceq_f64)(float64 a, float64 b, float_status *fpst) 198 { 199 return -float64_eq_quiet(a, b, fpst); 200 } 201 202 uint64_t HELPER(neon_cge_f64)(float64 a, float64 b, float_status *fpst) 203 { 204 return -float64_le(b, a, fpst); 205 } 206 207 uint64_t HELPER(neon_cgt_f64)(float64 a, float64 b, float_status *fpst) 208 { 209 return -float64_lt(b, a, fpst); 210 } 211 212 /* 213 * Reciprocal step and sqrt step. Note that unlike the A32/T32 214 * versions, these do a fully fused multiply-add or 215 * multiply-add-and-halve. 216 * The FPCR.AH == 1 versions need to avoid flipping the sign of NaN. 217 */ 218 #define DO_RECPS(NAME, CTYPE, FLOATTYPE, CHSFN) \ 219 CTYPE HELPER(NAME)(CTYPE a, CTYPE b, float_status *fpst) \ 220 { \ 221 a = FLOATTYPE ## _squash_input_denormal(a, fpst); \ 222 b = FLOATTYPE ## _squash_input_denormal(b, fpst); \ 223 a = FLOATTYPE ## _ ## CHSFN(a); \ 224 if ((FLOATTYPE ## _is_infinity(a) && FLOATTYPE ## _is_zero(b)) || \ 225 (FLOATTYPE ## _is_infinity(b) && FLOATTYPE ## _is_zero(a))) { \ 226 return FLOATTYPE ## _two; \ 227 } \ 228 return FLOATTYPE ## _muladd(a, b, FLOATTYPE ## _two, 0, fpst); \ 229 } 230 231 DO_RECPS(recpsf_f16, uint32_t, float16, chs) 232 DO_RECPS(recpsf_f32, float32, float32, chs) 233 DO_RECPS(recpsf_f64, float64, float64, chs) 234 DO_RECPS(recpsf_ah_f16, uint32_t, float16, ah_chs) 235 DO_RECPS(recpsf_ah_f32, float32, float32, ah_chs) 236 DO_RECPS(recpsf_ah_f64, float64, float64, ah_chs) 237 238 #define DO_RSQRTSF(NAME, CTYPE, FLOATTYPE, CHSFN) \ 239 CTYPE HELPER(NAME)(CTYPE a, CTYPE b, float_status *fpst) \ 240 { \ 241 a = FLOATTYPE ## _squash_input_denormal(a, fpst); \ 242 b = FLOATTYPE ## _squash_input_denormal(b, fpst); \ 243 a = FLOATTYPE ## _ ## CHSFN(a); \ 244 if ((FLOATTYPE ## _is_infinity(a) && FLOATTYPE ## _is_zero(b)) || \ 245 (FLOATTYPE ## _is_infinity(b) && FLOATTYPE ## _is_zero(a))) { \ 246 return FLOATTYPE ## _one_point_five; \ 247 } \ 248 return FLOATTYPE ## _muladd_scalbn(a, b, FLOATTYPE ## _three, \ 249 -1, 0, fpst); \ 250 } \ 251 252 DO_RSQRTSF(rsqrtsf_f16, uint32_t, float16, chs) 253 DO_RSQRTSF(rsqrtsf_f32, float32, float32, chs) 254 DO_RSQRTSF(rsqrtsf_f64, float64, float64, chs) 255 DO_RSQRTSF(rsqrtsf_ah_f16, uint32_t, float16, ah_chs) 256 DO_RSQRTSF(rsqrtsf_ah_f32, float32, float32, ah_chs) 257 DO_RSQRTSF(rsqrtsf_ah_f64, float64, float64, ah_chs) 258 259 /* Floating-point reciprocal exponent - see FPRecpX in ARM ARM */ 260 uint32_t HELPER(frecpx_f16)(uint32_t a, float_status *fpst) 261 { 262 uint16_t val16, sbit; 263 int16_t exp; 264 265 if (float16_is_any_nan(a)) { 266 float16 nan = a; 267 if (float16_is_signaling_nan(a, fpst)) { 268 float_raise(float_flag_invalid, fpst); 269 if (!fpst->default_nan_mode) { 270 nan = float16_silence_nan(a, fpst); 271 } 272 } 273 if (fpst->default_nan_mode) { 274 nan = float16_default_nan(fpst); 275 } 276 return nan; 277 } 278 279 a = float16_squash_input_denormal(a, fpst); 280 281 val16 = float16_val(a); 282 sbit = 0x8000 & val16; 283 exp = extract32(val16, 10, 5); 284 285 if (exp == 0) { 286 return make_float16(deposit32(sbit, 10, 5, 0x1e)); 287 } else { 288 return make_float16(deposit32(sbit, 10, 5, ~exp)); 289 } 290 } 291 292 float32 HELPER(frecpx_f32)(float32 a, float_status *fpst) 293 { 294 uint32_t val32, sbit; 295 int32_t exp; 296 297 if (float32_is_any_nan(a)) { 298 float32 nan = a; 299 if (float32_is_signaling_nan(a, fpst)) { 300 float_raise(float_flag_invalid, fpst); 301 if (!fpst->default_nan_mode) { 302 nan = float32_silence_nan(a, fpst); 303 } 304 } 305 if (fpst->default_nan_mode) { 306 nan = float32_default_nan(fpst); 307 } 308 return nan; 309 } 310 311 a = float32_squash_input_denormal(a, fpst); 312 313 val32 = float32_val(a); 314 sbit = 0x80000000ULL & val32; 315 exp = extract32(val32, 23, 8); 316 317 if (exp == 0) { 318 return make_float32(sbit | (0xfe << 23)); 319 } else { 320 return make_float32(sbit | (~exp & 0xff) << 23); 321 } 322 } 323 324 float64 HELPER(frecpx_f64)(float64 a, float_status *fpst) 325 { 326 uint64_t val64, sbit; 327 int64_t exp; 328 329 if (float64_is_any_nan(a)) { 330 float64 nan = a; 331 if (float64_is_signaling_nan(a, fpst)) { 332 float_raise(float_flag_invalid, fpst); 333 if (!fpst->default_nan_mode) { 334 nan = float64_silence_nan(a, fpst); 335 } 336 } 337 if (fpst->default_nan_mode) { 338 nan = float64_default_nan(fpst); 339 } 340 return nan; 341 } 342 343 a = float64_squash_input_denormal(a, fpst); 344 345 val64 = float64_val(a); 346 sbit = 0x8000000000000000ULL & val64; 347 exp = extract64(float64_val(a), 52, 11); 348 349 if (exp == 0) { 350 return make_float64(sbit | (0x7feULL << 52)); 351 } else { 352 return make_float64(sbit | (~exp & 0x7ffULL) << 52); 353 } 354 } 355 356 float32 HELPER(fcvtx_f64_to_f32)(float64 a, float_status *fpst) 357 { 358 float32 r; 359 int old = get_float_rounding_mode(fpst); 360 361 set_float_rounding_mode(float_round_to_odd, fpst); 362 r = float64_to_float32(a, fpst); 363 set_float_rounding_mode(old, fpst); 364 return r; 365 } 366 367 /* 368 * AH=1 min/max have some odd special cases: 369 * comparing two zeroes (regardless of sign), (NaN, anything), 370 * or (anything, NaN) should return the second argument (possibly 371 * squashed to zero). 372 * Also, denormal outputs are not squashed to zero regardless of FZ or FZ16. 373 */ 374 #define AH_MINMAX_HELPER(NAME, CTYPE, FLOATTYPE, MINMAX) \ 375 CTYPE HELPER(NAME)(CTYPE a, CTYPE b, float_status *fpst) \ 376 { \ 377 bool save; \ 378 CTYPE r; \ 379 a = FLOATTYPE ## _squash_input_denormal(a, fpst); \ 380 b = FLOATTYPE ## _squash_input_denormal(b, fpst); \ 381 if (FLOATTYPE ## _is_zero(a) && FLOATTYPE ## _is_zero(b)) { \ 382 return b; \ 383 } \ 384 if (FLOATTYPE ## _is_any_nan(a) || \ 385 FLOATTYPE ## _is_any_nan(b)) { \ 386 float_raise(float_flag_invalid, fpst); \ 387 return b; \ 388 } \ 389 save = get_flush_to_zero(fpst); \ 390 set_flush_to_zero(false, fpst); \ 391 r = FLOATTYPE ## _ ## MINMAX(a, b, fpst); \ 392 set_flush_to_zero(save, fpst); \ 393 return r; \ 394 } 395 396 AH_MINMAX_HELPER(vfp_ah_minh, dh_ctype_f16, float16, min) 397 AH_MINMAX_HELPER(vfp_ah_mins, float32, float32, min) 398 AH_MINMAX_HELPER(vfp_ah_mind, float64, float64, min) 399 AH_MINMAX_HELPER(vfp_ah_maxh, dh_ctype_f16, float16, max) 400 AH_MINMAX_HELPER(vfp_ah_maxs, float32, float32, max) 401 AH_MINMAX_HELPER(vfp_ah_maxd, float64, float64, max) 402 403 /* 64-bit versions of the CRC helpers. Note that although the operation 404 * (and the prototypes of crc32c() and crc32() mean that only the bottom 405 * 32 bits of the accumulator and result are used, we pass and return 406 * uint64_t for convenience of the generated code. Unlike the 32-bit 407 * instruction set versions, val may genuinely have 64 bits of data in it. 408 * The upper bytes of val (above the number specified by 'bytes') must have 409 * been zeroed out by the caller. 410 */ 411 uint64_t HELPER(crc32_64)(uint64_t acc, uint64_t val, uint32_t bytes) 412 { 413 uint8_t buf[8]; 414 415 stq_le_p(buf, val); 416 417 /* zlib crc32 converts the accumulator and output to one's complement. */ 418 return crc32(acc ^ 0xffffffff, buf, bytes) ^ 0xffffffff; 419 } 420 421 uint64_t HELPER(crc32c_64)(uint64_t acc, uint64_t val, uint32_t bytes) 422 { 423 uint8_t buf[8]; 424 425 stq_le_p(buf, val); 426 427 /* Linux crc32c converts the output to one's complement. */ 428 return crc32c(acc, buf, bytes) ^ 0xffffffff; 429 } 430 431 /* 432 * AdvSIMD half-precision 433 */ 434 435 #define ADVSIMD_HELPER(name, suffix) HELPER(glue(glue(advsimd_, name), suffix)) 436 437 #define ADVSIMD_HALFOP(name) \ 438 uint32_t ADVSIMD_HELPER(name, h)(uint32_t a, uint32_t b, float_status *fpst) \ 439 { \ 440 return float16_ ## name(a, b, fpst); \ 441 } 442 443 #define ADVSIMD_TWOHALFOP(name) \ 444 uint32_t ADVSIMD_HELPER(name, 2h)(uint32_t two_a, uint32_t two_b, \ 445 float_status *fpst) \ 446 { \ 447 float16 a1, a2, b1, b2; \ 448 uint32_t r1, r2; \ 449 a1 = extract32(two_a, 0, 16); \ 450 a2 = extract32(two_a, 16, 16); \ 451 b1 = extract32(two_b, 0, 16); \ 452 b2 = extract32(two_b, 16, 16); \ 453 r1 = float16_ ## name(a1, b1, fpst); \ 454 r2 = float16_ ## name(a2, b2, fpst); \ 455 return deposit32(r1, 16, 16, r2); \ 456 } 457 458 ADVSIMD_TWOHALFOP(add) 459 ADVSIMD_TWOHALFOP(sub) 460 ADVSIMD_TWOHALFOP(mul) 461 ADVSIMD_TWOHALFOP(div) 462 ADVSIMD_TWOHALFOP(min) 463 ADVSIMD_TWOHALFOP(max) 464 ADVSIMD_TWOHALFOP(minnum) 465 ADVSIMD_TWOHALFOP(maxnum) 466 467 /* Data processing - scalar floating-point and advanced SIMD */ 468 static float16 float16_mulx(float16 a, float16 b, float_status *fpst) 469 { 470 a = float16_squash_input_denormal(a, fpst); 471 b = float16_squash_input_denormal(b, fpst); 472 473 if ((float16_is_zero(a) && float16_is_infinity(b)) || 474 (float16_is_infinity(a) && float16_is_zero(b))) { 475 /* 2.0 with the sign bit set to sign(A) XOR sign(B) */ 476 return make_float16((1U << 14) | 477 ((float16_val(a) ^ float16_val(b)) & (1U << 15))); 478 } 479 return float16_mul(a, b, fpst); 480 } 481 482 ADVSIMD_HALFOP(mulx) 483 ADVSIMD_TWOHALFOP(mulx) 484 485 /* fused multiply-accumulate */ 486 uint32_t HELPER(advsimd_muladdh)(uint32_t a, uint32_t b, uint32_t c, 487 float_status *fpst) 488 { 489 return float16_muladd(a, b, c, 0, fpst); 490 } 491 492 uint32_t HELPER(advsimd_muladd2h)(uint32_t two_a, uint32_t two_b, 493 uint32_t two_c, float_status *fpst) 494 { 495 float16 a1, a2, b1, b2, c1, c2; 496 uint32_t r1, r2; 497 a1 = extract32(two_a, 0, 16); 498 a2 = extract32(two_a, 16, 16); 499 b1 = extract32(two_b, 0, 16); 500 b2 = extract32(two_b, 16, 16); 501 c1 = extract32(two_c, 0, 16); 502 c2 = extract32(two_c, 16, 16); 503 r1 = float16_muladd(a1, b1, c1, 0, fpst); 504 r2 = float16_muladd(a2, b2, c2, 0, fpst); 505 return deposit32(r1, 16, 16, r2); 506 } 507 508 /* 509 * Floating point comparisons produce an integer result. Softfloat 510 * routines return float_relation types which we convert to the 0/-1 511 * Neon requires. 512 */ 513 514 #define ADVSIMD_CMPRES(test) (test) ? 0xffff : 0 515 516 uint32_t HELPER(advsimd_ceq_f16)(uint32_t a, uint32_t b, float_status *fpst) 517 { 518 int compare = float16_compare_quiet(a, b, fpst); 519 return ADVSIMD_CMPRES(compare == float_relation_equal); 520 } 521 522 uint32_t HELPER(advsimd_cge_f16)(uint32_t a, uint32_t b, float_status *fpst) 523 { 524 int compare = float16_compare(a, b, fpst); 525 return ADVSIMD_CMPRES(compare == float_relation_greater || 526 compare == float_relation_equal); 527 } 528 529 uint32_t HELPER(advsimd_cgt_f16)(uint32_t a, uint32_t b, float_status *fpst) 530 { 531 int compare = float16_compare(a, b, fpst); 532 return ADVSIMD_CMPRES(compare == float_relation_greater); 533 } 534 535 uint32_t HELPER(advsimd_acge_f16)(uint32_t a, uint32_t b, float_status *fpst) 536 { 537 float16 f0 = float16_abs(a); 538 float16 f1 = float16_abs(b); 539 int compare = float16_compare(f0, f1, fpst); 540 return ADVSIMD_CMPRES(compare == float_relation_greater || 541 compare == float_relation_equal); 542 } 543 544 uint32_t HELPER(advsimd_acgt_f16)(uint32_t a, uint32_t b, float_status *fpst) 545 { 546 float16 f0 = float16_abs(a); 547 float16 f1 = float16_abs(b); 548 int compare = float16_compare(f0, f1, fpst); 549 return ADVSIMD_CMPRES(compare == float_relation_greater); 550 } 551 552 /* round to integral */ 553 uint32_t HELPER(advsimd_rinth_exact)(uint32_t x, float_status *fp_status) 554 { 555 return float16_round_to_int(x, fp_status); 556 } 557 558 uint32_t HELPER(advsimd_rinth)(uint32_t x, float_status *fp_status) 559 { 560 int old_flags = get_float_exception_flags(fp_status), new_flags; 561 float16 ret; 562 563 ret = float16_round_to_int(x, fp_status); 564 565 /* Suppress any inexact exceptions the conversion produced */ 566 if (!(old_flags & float_flag_inexact)) { 567 new_flags = get_float_exception_flags(fp_status); 568 set_float_exception_flags(new_flags & ~float_flag_inexact, fp_status); 569 } 570 571 return ret; 572 } 573 574 static int el_from_spsr(uint32_t spsr) 575 { 576 /* Return the exception level that this SPSR is requesting a return to, 577 * or -1 if it is invalid (an illegal return) 578 */ 579 if (spsr & PSTATE_nRW) { 580 switch (spsr & CPSR_M) { 581 case ARM_CPU_MODE_USR: 582 return 0; 583 case ARM_CPU_MODE_HYP: 584 return 2; 585 case ARM_CPU_MODE_FIQ: 586 case ARM_CPU_MODE_IRQ: 587 case ARM_CPU_MODE_SVC: 588 case ARM_CPU_MODE_ABT: 589 case ARM_CPU_MODE_UND: 590 case ARM_CPU_MODE_SYS: 591 return 1; 592 case ARM_CPU_MODE_MON: 593 /* Returning to Mon from AArch64 is never possible, 594 * so this is an illegal return. 595 */ 596 default: 597 return -1; 598 } 599 } else { 600 if (extract32(spsr, 1, 1)) { 601 /* Return with reserved M[1] bit set */ 602 return -1; 603 } 604 if (extract32(spsr, 0, 4) == 1) { 605 /* return to EL0 with M[0] bit set */ 606 return -1; 607 } 608 return extract32(spsr, 2, 2); 609 } 610 } 611 612 static void cpsr_write_from_spsr_elx(CPUARMState *env, 613 uint32_t val) 614 { 615 uint32_t mask; 616 617 /* Save SPSR_ELx.SS into PSTATE. */ 618 env->pstate = (env->pstate & ~PSTATE_SS) | (val & PSTATE_SS); 619 val &= ~PSTATE_SS; 620 621 /* Move DIT to the correct location for CPSR */ 622 if (val & PSTATE_DIT) { 623 val &= ~PSTATE_DIT; 624 val |= CPSR_DIT; 625 } 626 627 mask = aarch32_cpsr_valid_mask(env->features, \ 628 &env_archcpu(env)->isar); 629 cpsr_write(env, val, mask, CPSRWriteRaw); 630 } 631 632 void HELPER(exception_return)(CPUARMState *env, uint64_t new_pc) 633 { 634 int cur_el = arm_current_el(env); 635 unsigned int spsr_idx = aarch64_banked_spsr_index(cur_el); 636 uint32_t spsr = env->banked_spsr[spsr_idx]; 637 int new_el; 638 bool return_to_aa64 = (spsr & PSTATE_nRW) == 0; 639 640 aarch64_save_sp(env, cur_el); 641 642 arm_clear_exclusive(env); 643 644 /* We must squash the PSTATE.SS bit to zero unless both of the 645 * following hold: 646 * 1. debug exceptions are currently disabled 647 * 2. singlestep will be active in the EL we return to 648 * We check 1 here and 2 after we've done the pstate/cpsr write() to 649 * transition to the EL we're going to. 650 */ 651 if (arm_generate_debug_exceptions(env)) { 652 spsr &= ~PSTATE_SS; 653 } 654 655 /* 656 * FEAT_RME forbids return from EL3 with an invalid security state. 657 * We don't need an explicit check for FEAT_RME here because we enforce 658 * in scr_write() that you can't set the NSE bit without it. 659 */ 660 if (cur_el == 3 && (env->cp15.scr_el3 & (SCR_NS | SCR_NSE)) == SCR_NSE) { 661 goto illegal_return; 662 } 663 664 new_el = el_from_spsr(spsr); 665 if (new_el == -1) { 666 goto illegal_return; 667 } 668 if (new_el > cur_el || (new_el == 2 && !arm_is_el2_enabled(env))) { 669 /* Disallow return to an EL which is unimplemented or higher 670 * than the current one. 671 */ 672 goto illegal_return; 673 } 674 675 if (new_el != 0 && arm_el_is_aa64(env, new_el) != return_to_aa64) { 676 /* Return to an EL which is configured for a different register width */ 677 goto illegal_return; 678 } 679 680 if (new_el == 1 && (arm_hcr_el2_eff(env) & HCR_TGE)) { 681 goto illegal_return; 682 } 683 684 bql_lock(); 685 arm_call_pre_el_change_hook(env_archcpu(env)); 686 bql_unlock(); 687 688 if (!return_to_aa64) { 689 env->aarch64 = false; 690 /* We do a raw CPSR write because aarch64_sync_64_to_32() 691 * will sort the register banks out for us, and we've already 692 * caught all the bad-mode cases in el_from_spsr(). 693 */ 694 cpsr_write_from_spsr_elx(env, spsr); 695 if (!arm_singlestep_active(env)) { 696 env->pstate &= ~PSTATE_SS; 697 } 698 aarch64_sync_64_to_32(env); 699 700 if (spsr & CPSR_T) { 701 env->regs[15] = new_pc & ~0x1; 702 } else { 703 env->regs[15] = new_pc & ~0x3; 704 } 705 helper_rebuild_hflags_a32(env, new_el); 706 qemu_log_mask(CPU_LOG_INT, "Exception return from AArch64 EL%d to " 707 "AArch32 EL%d PC 0x%" PRIx32 "\n", 708 cur_el, new_el, env->regs[15]); 709 } else { 710 int tbii; 711 712 env->aarch64 = true; 713 spsr &= aarch64_pstate_valid_mask(&env_archcpu(env)->isar); 714 pstate_write(env, spsr); 715 if (!arm_singlestep_active(env)) { 716 env->pstate &= ~PSTATE_SS; 717 } 718 aarch64_restore_sp(env, new_el); 719 helper_rebuild_hflags_a64(env, new_el); 720 721 /* 722 * Apply TBI to the exception return address. We had to delay this 723 * until after we selected the new EL, so that we could select the 724 * correct TBI+TBID bits. This is made easier by waiting until after 725 * the hflags rebuild, since we can pull the composite TBII field 726 * from there. 727 */ 728 tbii = EX_TBFLAG_A64(env->hflags, TBII); 729 if ((tbii >> extract64(new_pc, 55, 1)) & 1) { 730 /* TBI is enabled. */ 731 int core_mmu_idx = arm_env_mmu_index(env); 732 if (regime_has_2_ranges(core_to_aa64_mmu_idx(core_mmu_idx))) { 733 new_pc = sextract64(new_pc, 0, 56); 734 } else { 735 new_pc = extract64(new_pc, 0, 56); 736 } 737 } 738 env->pc = new_pc; 739 740 qemu_log_mask(CPU_LOG_INT, "Exception return from AArch64 EL%d to " 741 "AArch64 EL%d PC 0x%" PRIx64 "\n", 742 cur_el, new_el, env->pc); 743 } 744 745 /* 746 * Note that cur_el can never be 0. If new_el is 0, then 747 * el0_a64 is return_to_aa64, else el0_a64 is ignored. 748 */ 749 aarch64_sve_change_el(env, cur_el, new_el, return_to_aa64); 750 751 bql_lock(); 752 arm_call_el_change_hook(env_archcpu(env)); 753 bql_unlock(); 754 755 return; 756 757 illegal_return: 758 /* Illegal return events of various kinds have architecturally 759 * mandated behaviour: 760 * restore NZCV and DAIF from SPSR_ELx 761 * set PSTATE.IL 762 * restore PC from ELR_ELx 763 * no change to exception level, execution state or stack pointer 764 */ 765 env->pstate |= PSTATE_IL; 766 env->pc = new_pc; 767 spsr &= PSTATE_NZCV | PSTATE_DAIF | PSTATE_ALLINT; 768 spsr |= pstate_read(env) & ~(PSTATE_NZCV | PSTATE_DAIF | PSTATE_ALLINT); 769 pstate_write(env, spsr); 770 if (!arm_singlestep_active(env)) { 771 env->pstate &= ~PSTATE_SS; 772 } 773 helper_rebuild_hflags_a64(env, cur_el); 774 qemu_log_mask(LOG_GUEST_ERROR, "Illegal exception return at EL%d: " 775 "resuming execution at 0x%" PRIx64 "\n", cur_el, env->pc); 776 } 777 778 void HELPER(dc_zva)(CPUARMState *env, uint64_t vaddr_in) 779 { 780 uintptr_t ra = GETPC(); 781 782 /* 783 * Implement DC ZVA, which zeroes a fixed-length block of memory. 784 * Note that we do not implement the (architecturally mandated) 785 * alignment fault for attempts to use this on Device memory 786 * (which matches the usual QEMU behaviour of not implementing either 787 * alignment faults or any memory attribute handling). 788 */ 789 int blocklen = 4 << env_archcpu(env)->dcz_blocksize; 790 uint64_t vaddr = vaddr_in & ~(blocklen - 1); 791 int mmu_idx = arm_env_mmu_index(env); 792 void *mem; 793 794 /* 795 * Trapless lookup. In addition to actual invalid page, may 796 * return NULL for I/O, watchpoints, clean pages, etc. 797 */ 798 mem = tlb_vaddr_to_host(env, vaddr, MMU_DATA_STORE, mmu_idx); 799 800 #ifndef CONFIG_USER_ONLY 801 if (unlikely(!mem)) { 802 /* 803 * Trap if accessing an invalid page. DC_ZVA requires that we supply 804 * the original pointer for an invalid page. But watchpoints require 805 * that we probe the actual space. So do both. 806 */ 807 (void) probe_write(env, vaddr_in, 1, mmu_idx, ra); 808 mem = probe_write(env, vaddr, blocklen, mmu_idx, ra); 809 810 if (unlikely(!mem)) { 811 /* 812 * The only remaining reason for mem == NULL is I/O. 813 * Just do a series of byte writes as the architecture demands. 814 */ 815 for (int i = 0; i < blocklen; i++) { 816 cpu_stb_mmuidx_ra(env, vaddr + i, 0, mmu_idx, ra); 817 } 818 return; 819 } 820 } 821 #endif 822 823 set_helper_retaddr(ra); 824 memset(mem, 0, blocklen); 825 clear_helper_retaddr(); 826 } 827 828 void HELPER(unaligned_access)(CPUARMState *env, uint64_t addr, 829 uint32_t access_type, uint32_t mmu_idx) 830 { 831 arm_cpu_do_unaligned_access(env_cpu(env), addr, access_type, 832 mmu_idx, GETPC()); 833 } 834 835 /* Memory operations (memset, memmove, memcpy) */ 836 837 /* 838 * Return true if the CPY* and SET* insns can execute; compare 839 * pseudocode CheckMOPSEnabled(), though we refactor it a little. 840 */ 841 static bool mops_enabled(CPUARMState *env) 842 { 843 int el = arm_current_el(env); 844 845 if (el < 2 && 846 (arm_hcr_el2_eff(env) & (HCR_E2H | HCR_TGE)) != (HCR_E2H | HCR_TGE) && 847 !(arm_hcrx_el2_eff(env) & HCRX_MSCEN)) { 848 return false; 849 } 850 851 if (el == 0) { 852 if (!el_is_in_host(env, 0)) { 853 return env->cp15.sctlr_el[1] & SCTLR_MSCEN; 854 } else { 855 return env->cp15.sctlr_el[2] & SCTLR_MSCEN; 856 } 857 } 858 return true; 859 } 860 861 static void check_mops_enabled(CPUARMState *env, uintptr_t ra) 862 { 863 if (!mops_enabled(env)) { 864 raise_exception_ra(env, EXCP_UDEF, syn_uncategorized(), 865 exception_target_el(env), ra); 866 } 867 } 868 869 /* 870 * Return the target exception level for an exception due 871 * to mismatched arguments in a FEAT_MOPS copy or set. 872 * Compare pseudocode MismatchedCpySetTargetEL() 873 */ 874 static int mops_mismatch_exception_target_el(CPUARMState *env) 875 { 876 int el = arm_current_el(env); 877 878 if (el > 1) { 879 return el; 880 } 881 if (el == 0 && (arm_hcr_el2_eff(env) & HCR_TGE)) { 882 return 2; 883 } 884 if (el == 1 && (arm_hcrx_el2_eff(env) & HCRX_MCE2)) { 885 return 2; 886 } 887 return 1; 888 } 889 890 /* 891 * Check whether an M or E instruction was executed with a CF value 892 * indicating the wrong option for this implementation. 893 * Assumes we are always Option A. 894 */ 895 static void check_mops_wrong_option(CPUARMState *env, uint32_t syndrome, 896 uintptr_t ra) 897 { 898 if (env->CF != 0) { 899 syndrome |= 1 << 17; /* Set the wrong-option bit */ 900 raise_exception_ra(env, EXCP_UDEF, syndrome, 901 mops_mismatch_exception_target_el(env), ra); 902 } 903 } 904 905 /* 906 * Return the maximum number of bytes we can transfer starting at addr 907 * without crossing a page boundary. 908 */ 909 static uint64_t page_limit(uint64_t addr) 910 { 911 return TARGET_PAGE_ALIGN(addr + 1) - addr; 912 } 913 914 /* 915 * Return the number of bytes we can copy starting from addr and working 916 * backwards without crossing a page boundary. 917 */ 918 static uint64_t page_limit_rev(uint64_t addr) 919 { 920 return (addr & ~TARGET_PAGE_MASK) + 1; 921 } 922 923 /* 924 * Perform part of a memory set on an area of guest memory starting at 925 * toaddr (a dirty address) and extending for setsize bytes. 926 * 927 * Returns the number of bytes actually set, which might be less than 928 * setsize; the caller should loop until the whole set has been done. 929 * The caller should ensure that the guest registers are correct 930 * for the possibility that the first byte of the set encounters 931 * an exception or watchpoint. We guarantee not to take any faults 932 * for bytes other than the first. 933 */ 934 static uint64_t set_step(CPUARMState *env, uint64_t toaddr, 935 uint64_t setsize, uint32_t data, int memidx, 936 uint32_t *mtedesc, uintptr_t ra) 937 { 938 void *mem; 939 940 setsize = MIN(setsize, page_limit(toaddr)); 941 if (*mtedesc) { 942 uint64_t mtesize = mte_mops_probe(env, toaddr, setsize, *mtedesc); 943 if (mtesize == 0) { 944 /* Trap, or not. All CPU state is up to date */ 945 mte_check_fail(env, *mtedesc, toaddr, ra); 946 /* Continue, with no further MTE checks required */ 947 *mtedesc = 0; 948 } else { 949 /* Advance to the end, or to the tag mismatch */ 950 setsize = MIN(setsize, mtesize); 951 } 952 } 953 954 toaddr = useronly_clean_ptr(toaddr); 955 /* 956 * Trapless lookup: returns NULL for invalid page, I/O, 957 * watchpoints, clean pages, etc. 958 */ 959 mem = tlb_vaddr_to_host(env, toaddr, MMU_DATA_STORE, memidx); 960 961 #ifndef CONFIG_USER_ONLY 962 if (unlikely(!mem)) { 963 /* 964 * Slow-path: just do one byte write. This will handle the 965 * watchpoint, invalid page, etc handling correctly. 966 * For clean code pages, the next iteration will see 967 * the page dirty and will use the fast path. 968 */ 969 cpu_stb_mmuidx_ra(env, toaddr, data, memidx, ra); 970 return 1; 971 } 972 #endif 973 /* Easy case: just memset the host memory */ 974 set_helper_retaddr(ra); 975 memset(mem, data, setsize); 976 clear_helper_retaddr(); 977 return setsize; 978 } 979 980 /* 981 * Similar, but setting tags. The architecture requires us to do this 982 * in 16-byte chunks. SETP accesses are not tag checked; they set 983 * the tags. 984 */ 985 static uint64_t set_step_tags(CPUARMState *env, uint64_t toaddr, 986 uint64_t setsize, uint32_t data, int memidx, 987 uint32_t *mtedesc, uintptr_t ra) 988 { 989 void *mem; 990 uint64_t cleanaddr; 991 992 setsize = MIN(setsize, page_limit(toaddr)); 993 994 cleanaddr = useronly_clean_ptr(toaddr); 995 /* 996 * Trapless lookup: returns NULL for invalid page, I/O, 997 * watchpoints, clean pages, etc. 998 */ 999 mem = tlb_vaddr_to_host(env, cleanaddr, MMU_DATA_STORE, memidx); 1000 1001 #ifndef CONFIG_USER_ONLY 1002 if (unlikely(!mem)) { 1003 /* 1004 * Slow-path: just do one write. This will handle the 1005 * watchpoint, invalid page, etc handling correctly. 1006 * The architecture requires that we do 16 bytes at a time, 1007 * and we know both ptr and size are 16 byte aligned. 1008 * For clean code pages, the next iteration will see 1009 * the page dirty and will use the fast path. 1010 */ 1011 uint64_t repldata = data * 0x0101010101010101ULL; 1012 MemOpIdx oi16 = make_memop_idx(MO_TE | MO_128, memidx); 1013 cpu_st16_mmu(env, toaddr, int128_make128(repldata, repldata), oi16, ra); 1014 mte_mops_set_tags(env, toaddr, 16, *mtedesc); 1015 return 16; 1016 } 1017 #endif 1018 /* Easy case: just memset the host memory */ 1019 set_helper_retaddr(ra); 1020 memset(mem, data, setsize); 1021 clear_helper_retaddr(); 1022 mte_mops_set_tags(env, toaddr, setsize, *mtedesc); 1023 return setsize; 1024 } 1025 1026 typedef uint64_t StepFn(CPUARMState *env, uint64_t toaddr, 1027 uint64_t setsize, uint32_t data, 1028 int memidx, uint32_t *mtedesc, uintptr_t ra); 1029 1030 /* Extract register numbers from a MOPS exception syndrome value */ 1031 static int mops_destreg(uint32_t syndrome) 1032 { 1033 return extract32(syndrome, 10, 5); 1034 } 1035 1036 static int mops_srcreg(uint32_t syndrome) 1037 { 1038 return extract32(syndrome, 5, 5); 1039 } 1040 1041 static int mops_sizereg(uint32_t syndrome) 1042 { 1043 return extract32(syndrome, 0, 5); 1044 } 1045 1046 /* 1047 * Return true if TCMA and TBI bits mean we need to do MTE checks. 1048 * We only need to do this once per MOPS insn, not for every page. 1049 */ 1050 static bool mte_checks_needed(uint64_t ptr, uint32_t desc) 1051 { 1052 int bit55 = extract64(ptr, 55, 1); 1053 1054 /* 1055 * Note that tbi_check() returns true for "access checked" but 1056 * tcma_check() returns true for "access unchecked". 1057 */ 1058 if (!tbi_check(desc, bit55)) { 1059 return false; 1060 } 1061 return !tcma_check(desc, bit55, allocation_tag_from_addr(ptr)); 1062 } 1063 1064 /* Take an exception if the SETG addr/size are not granule aligned */ 1065 static void check_setg_alignment(CPUARMState *env, uint64_t ptr, uint64_t size, 1066 uint32_t memidx, uintptr_t ra) 1067 { 1068 if ((size != 0 && !QEMU_IS_ALIGNED(ptr, TAG_GRANULE)) || 1069 !QEMU_IS_ALIGNED(size, TAG_GRANULE)) { 1070 arm_cpu_do_unaligned_access(env_cpu(env), ptr, MMU_DATA_STORE, 1071 memidx, ra); 1072 1073 } 1074 } 1075 1076 static uint64_t arm_reg_or_xzr(CPUARMState *env, int reg) 1077 { 1078 /* 1079 * Runtime equivalent of cpu_reg() -- return the CPU register value, 1080 * for contexts when index 31 means XZR (not SP). 1081 */ 1082 return reg == 31 ? 0 : env->xregs[reg]; 1083 } 1084 1085 /* 1086 * For the Memory Set operation, our implementation chooses 1087 * always to use "option A", where we update Xd to the final 1088 * address in the SETP insn, and set Xn to be -(bytes remaining). 1089 * On SETM and SETE insns we only need update Xn. 1090 * 1091 * @env: CPU 1092 * @syndrome: syndrome value for mismatch exceptions 1093 * (also contains the register numbers we need to use) 1094 * @mtedesc: MTE descriptor word 1095 * @stepfn: function which does a single part of the set operation 1096 * @is_setg: true if this is the tag-setting SETG variant 1097 */ 1098 static void do_setp(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc, 1099 StepFn *stepfn, bool is_setg, uintptr_t ra) 1100 { 1101 /* Prologue: we choose to do up to the next page boundary */ 1102 int rd = mops_destreg(syndrome); 1103 int rs = mops_srcreg(syndrome); 1104 int rn = mops_sizereg(syndrome); 1105 uint8_t data = arm_reg_or_xzr(env, rs); 1106 uint32_t memidx = FIELD_EX32(mtedesc, MTEDESC, MIDX); 1107 uint64_t toaddr = env->xregs[rd]; 1108 uint64_t setsize = env->xregs[rn]; 1109 uint64_t stagesetsize, step; 1110 1111 check_mops_enabled(env, ra); 1112 1113 if (setsize > INT64_MAX) { 1114 setsize = INT64_MAX; 1115 if (is_setg) { 1116 setsize &= ~0xf; 1117 } 1118 } 1119 1120 if (unlikely(is_setg)) { 1121 check_setg_alignment(env, toaddr, setsize, memidx, ra); 1122 } else if (!mte_checks_needed(toaddr, mtedesc)) { 1123 mtedesc = 0; 1124 } 1125 1126 stagesetsize = MIN(setsize, page_limit(toaddr)); 1127 while (stagesetsize) { 1128 env->xregs[rd] = toaddr; 1129 env->xregs[rn] = setsize; 1130 step = stepfn(env, toaddr, stagesetsize, data, memidx, &mtedesc, ra); 1131 toaddr += step; 1132 setsize -= step; 1133 stagesetsize -= step; 1134 } 1135 /* Insn completed, so update registers to the Option A format */ 1136 env->xregs[rd] = toaddr + setsize; 1137 env->xregs[rn] = -setsize; 1138 1139 /* Set NZCV = 0000 to indicate we are an Option A implementation */ 1140 env->NF = 0; 1141 env->ZF = 1; /* our env->ZF encoding is inverted */ 1142 env->CF = 0; 1143 env->VF = 0; 1144 return; 1145 } 1146 1147 void HELPER(setp)(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc) 1148 { 1149 do_setp(env, syndrome, mtedesc, set_step, false, GETPC()); 1150 } 1151 1152 void HELPER(setgp)(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc) 1153 { 1154 do_setp(env, syndrome, mtedesc, set_step_tags, true, GETPC()); 1155 } 1156 1157 static void do_setm(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc, 1158 StepFn *stepfn, bool is_setg, uintptr_t ra) 1159 { 1160 /* Main: we choose to do all the full-page chunks */ 1161 CPUState *cs = env_cpu(env); 1162 int rd = mops_destreg(syndrome); 1163 int rs = mops_srcreg(syndrome); 1164 int rn = mops_sizereg(syndrome); 1165 uint8_t data = arm_reg_or_xzr(env, rs); 1166 uint64_t toaddr = env->xregs[rd] + env->xregs[rn]; 1167 uint64_t setsize = -env->xregs[rn]; 1168 uint32_t memidx = FIELD_EX32(mtedesc, MTEDESC, MIDX); 1169 uint64_t step, stagesetsize; 1170 1171 check_mops_enabled(env, ra); 1172 1173 /* 1174 * We're allowed to NOP out "no data to copy" before the consistency 1175 * checks; we choose to do so. 1176 */ 1177 if (env->xregs[rn] == 0) { 1178 return; 1179 } 1180 1181 check_mops_wrong_option(env, syndrome, ra); 1182 1183 /* 1184 * Our implementation will work fine even if we have an unaligned 1185 * destination address, and because we update Xn every time around 1186 * the loop below and the return value from stepfn() may be less 1187 * than requested, we might find toaddr is unaligned. So we don't 1188 * have an IMPDEF check for alignment here. 1189 */ 1190 1191 if (unlikely(is_setg)) { 1192 check_setg_alignment(env, toaddr, setsize, memidx, ra); 1193 } else if (!mte_checks_needed(toaddr, mtedesc)) { 1194 mtedesc = 0; 1195 } 1196 1197 /* Do the actual memset: we leave the last partial page to SETE */ 1198 stagesetsize = setsize & TARGET_PAGE_MASK; 1199 while (stagesetsize > 0) { 1200 step = stepfn(env, toaddr, stagesetsize, data, memidx, &mtedesc, ra); 1201 toaddr += step; 1202 setsize -= step; 1203 stagesetsize -= step; 1204 env->xregs[rn] = -setsize; 1205 if (stagesetsize > 0 && unlikely(cpu_loop_exit_requested(cs))) { 1206 cpu_loop_exit_restore(cs, ra); 1207 } 1208 } 1209 } 1210 1211 void HELPER(setm)(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc) 1212 { 1213 do_setm(env, syndrome, mtedesc, set_step, false, GETPC()); 1214 } 1215 1216 void HELPER(setgm)(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc) 1217 { 1218 do_setm(env, syndrome, mtedesc, set_step_tags, true, GETPC()); 1219 } 1220 1221 static void do_sete(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc, 1222 StepFn *stepfn, bool is_setg, uintptr_t ra) 1223 { 1224 /* Epilogue: do the last partial page */ 1225 int rd = mops_destreg(syndrome); 1226 int rs = mops_srcreg(syndrome); 1227 int rn = mops_sizereg(syndrome); 1228 uint8_t data = arm_reg_or_xzr(env, rs); 1229 uint64_t toaddr = env->xregs[rd] + env->xregs[rn]; 1230 uint64_t setsize = -env->xregs[rn]; 1231 uint32_t memidx = FIELD_EX32(mtedesc, MTEDESC, MIDX); 1232 uint64_t step; 1233 1234 check_mops_enabled(env, ra); 1235 1236 /* 1237 * We're allowed to NOP out "no data to copy" before the consistency 1238 * checks; we choose to do so. 1239 */ 1240 if (setsize == 0) { 1241 return; 1242 } 1243 1244 check_mops_wrong_option(env, syndrome, ra); 1245 1246 /* 1247 * Our implementation has no address alignment requirements, but 1248 * we do want to enforce the "less than a page" size requirement, 1249 * so we don't need to have the "check for interrupts" here. 1250 */ 1251 if (setsize >= TARGET_PAGE_SIZE) { 1252 raise_exception_ra(env, EXCP_UDEF, syndrome, 1253 mops_mismatch_exception_target_el(env), ra); 1254 } 1255 1256 if (unlikely(is_setg)) { 1257 check_setg_alignment(env, toaddr, setsize, memidx, ra); 1258 } else if (!mte_checks_needed(toaddr, mtedesc)) { 1259 mtedesc = 0; 1260 } 1261 1262 /* Do the actual memset */ 1263 while (setsize > 0) { 1264 step = stepfn(env, toaddr, setsize, data, memidx, &mtedesc, ra); 1265 toaddr += step; 1266 setsize -= step; 1267 env->xregs[rn] = -setsize; 1268 } 1269 } 1270 1271 void HELPER(sete)(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc) 1272 { 1273 do_sete(env, syndrome, mtedesc, set_step, false, GETPC()); 1274 } 1275 1276 void HELPER(setge)(CPUARMState *env, uint32_t syndrome, uint32_t mtedesc) 1277 { 1278 do_sete(env, syndrome, mtedesc, set_step_tags, true, GETPC()); 1279 } 1280 1281 /* 1282 * Perform part of a memory copy from the guest memory at fromaddr 1283 * and extending for copysize bytes, to the guest memory at 1284 * toaddr. Both addresses are dirty. 1285 * 1286 * Returns the number of bytes actually set, which might be less than 1287 * copysize; the caller should loop until the whole copy has been done. 1288 * The caller should ensure that the guest registers are correct 1289 * for the possibility that the first byte of the copy encounters 1290 * an exception or watchpoint. We guarantee not to take any faults 1291 * for bytes other than the first. 1292 */ 1293 static uint64_t copy_step(CPUARMState *env, uint64_t toaddr, uint64_t fromaddr, 1294 uint64_t copysize, int wmemidx, int rmemidx, 1295 uint32_t *wdesc, uint32_t *rdesc, uintptr_t ra) 1296 { 1297 void *rmem; 1298 void *wmem; 1299 1300 /* Don't cross a page boundary on either source or destination */ 1301 copysize = MIN(copysize, page_limit(toaddr)); 1302 copysize = MIN(copysize, page_limit(fromaddr)); 1303 /* 1304 * Handle MTE tag checks: either handle the tag mismatch for byte 0, 1305 * or else copy up to but not including the byte with the mismatch. 1306 */ 1307 if (*rdesc) { 1308 uint64_t mtesize = mte_mops_probe(env, fromaddr, copysize, *rdesc); 1309 if (mtesize == 0) { 1310 mte_check_fail(env, *rdesc, fromaddr, ra); 1311 *rdesc = 0; 1312 } else { 1313 copysize = MIN(copysize, mtesize); 1314 } 1315 } 1316 if (*wdesc) { 1317 uint64_t mtesize = mte_mops_probe(env, toaddr, copysize, *wdesc); 1318 if (mtesize == 0) { 1319 mte_check_fail(env, *wdesc, toaddr, ra); 1320 *wdesc = 0; 1321 } else { 1322 copysize = MIN(copysize, mtesize); 1323 } 1324 } 1325 1326 toaddr = useronly_clean_ptr(toaddr); 1327 fromaddr = useronly_clean_ptr(fromaddr); 1328 /* Trapless lookup of whether we can get a host memory pointer */ 1329 wmem = tlb_vaddr_to_host(env, toaddr, MMU_DATA_STORE, wmemidx); 1330 rmem = tlb_vaddr_to_host(env, fromaddr, MMU_DATA_LOAD, rmemidx); 1331 1332 #ifndef CONFIG_USER_ONLY 1333 /* 1334 * If we don't have host memory for both source and dest then just 1335 * do a single byte copy. This will handle watchpoints, invalid pages, 1336 * etc correctly. For clean code pages, the next iteration will see 1337 * the page dirty and will use the fast path. 1338 */ 1339 if (unlikely(!rmem || !wmem)) { 1340 uint8_t byte; 1341 if (rmem) { 1342 byte = *(uint8_t *)rmem; 1343 } else { 1344 byte = cpu_ldub_mmuidx_ra(env, fromaddr, rmemidx, ra); 1345 } 1346 if (wmem) { 1347 *(uint8_t *)wmem = byte; 1348 } else { 1349 cpu_stb_mmuidx_ra(env, toaddr, byte, wmemidx, ra); 1350 } 1351 return 1; 1352 } 1353 #endif 1354 /* Easy case: just memmove the host memory */ 1355 set_helper_retaddr(ra); 1356 memmove(wmem, rmem, copysize); 1357 clear_helper_retaddr(); 1358 return copysize; 1359 } 1360 1361 /* 1362 * Do part of a backwards memory copy. Here toaddr and fromaddr point 1363 * to the *last* byte to be copied. 1364 */ 1365 static uint64_t copy_step_rev(CPUARMState *env, uint64_t toaddr, 1366 uint64_t fromaddr, 1367 uint64_t copysize, int wmemidx, int rmemidx, 1368 uint32_t *wdesc, uint32_t *rdesc, uintptr_t ra) 1369 { 1370 void *rmem; 1371 void *wmem; 1372 1373 /* Don't cross a page boundary on either source or destination */ 1374 copysize = MIN(copysize, page_limit_rev(toaddr)); 1375 copysize = MIN(copysize, page_limit_rev(fromaddr)); 1376 1377 /* 1378 * Handle MTE tag checks: either handle the tag mismatch for byte 0, 1379 * or else copy up to but not including the byte with the mismatch. 1380 */ 1381 if (*rdesc) { 1382 uint64_t mtesize = mte_mops_probe_rev(env, fromaddr, copysize, *rdesc); 1383 if (mtesize == 0) { 1384 mte_check_fail(env, *rdesc, fromaddr, ra); 1385 *rdesc = 0; 1386 } else { 1387 copysize = MIN(copysize, mtesize); 1388 } 1389 } 1390 if (*wdesc) { 1391 uint64_t mtesize = mte_mops_probe_rev(env, toaddr, copysize, *wdesc); 1392 if (mtesize == 0) { 1393 mte_check_fail(env, *wdesc, toaddr, ra); 1394 *wdesc = 0; 1395 } else { 1396 copysize = MIN(copysize, mtesize); 1397 } 1398 } 1399 1400 toaddr = useronly_clean_ptr(toaddr); 1401 fromaddr = useronly_clean_ptr(fromaddr); 1402 /* Trapless lookup of whether we can get a host memory pointer */ 1403 wmem = tlb_vaddr_to_host(env, toaddr, MMU_DATA_STORE, wmemidx); 1404 rmem = tlb_vaddr_to_host(env, fromaddr, MMU_DATA_LOAD, rmemidx); 1405 1406 #ifndef CONFIG_USER_ONLY 1407 /* 1408 * If we don't have host memory for both source and dest then just 1409 * do a single byte copy. This will handle watchpoints, invalid pages, 1410 * etc correctly. For clean code pages, the next iteration will see 1411 * the page dirty and will use the fast path. 1412 */ 1413 if (unlikely(!rmem || !wmem)) { 1414 uint8_t byte; 1415 if (rmem) { 1416 byte = *(uint8_t *)rmem; 1417 } else { 1418 byte = cpu_ldub_mmuidx_ra(env, fromaddr, rmemidx, ra); 1419 } 1420 if (wmem) { 1421 *(uint8_t *)wmem = byte; 1422 } else { 1423 cpu_stb_mmuidx_ra(env, toaddr, byte, wmemidx, ra); 1424 } 1425 return 1; 1426 } 1427 #endif 1428 /* 1429 * Easy case: just memmove the host memory. Note that wmem and 1430 * rmem here point to the *last* byte to copy. 1431 */ 1432 set_helper_retaddr(ra); 1433 memmove(wmem - (copysize - 1), rmem - (copysize - 1), copysize); 1434 clear_helper_retaddr(); 1435 return copysize; 1436 } 1437 1438 /* 1439 * for the Memory Copy operation, our implementation chooses always 1440 * to use "option A", where we update Xd and Xs to the final addresses 1441 * in the CPYP insn, and then in CPYM and CPYE only need to update Xn. 1442 * 1443 * @env: CPU 1444 * @syndrome: syndrome value for mismatch exceptions 1445 * (also contains the register numbers we need to use) 1446 * @wdesc: MTE descriptor for the writes (destination) 1447 * @rdesc: MTE descriptor for the reads (source) 1448 * @move: true if this is CPY (memmove), false for CPYF (memcpy forwards) 1449 */ 1450 static void do_cpyp(CPUARMState *env, uint32_t syndrome, uint32_t wdesc, 1451 uint32_t rdesc, uint32_t move, uintptr_t ra) 1452 { 1453 int rd = mops_destreg(syndrome); 1454 int rs = mops_srcreg(syndrome); 1455 int rn = mops_sizereg(syndrome); 1456 uint32_t rmemidx = FIELD_EX32(rdesc, MTEDESC, MIDX); 1457 uint32_t wmemidx = FIELD_EX32(wdesc, MTEDESC, MIDX); 1458 bool forwards = true; 1459 uint64_t toaddr = env->xregs[rd]; 1460 uint64_t fromaddr = env->xregs[rs]; 1461 uint64_t copysize = env->xregs[rn]; 1462 uint64_t stagecopysize, step; 1463 1464 check_mops_enabled(env, ra); 1465 1466 1467 if (move) { 1468 /* 1469 * Copy backwards if necessary. The direction for a non-overlapping 1470 * copy is IMPDEF; we choose forwards. 1471 */ 1472 if (copysize > 0x007FFFFFFFFFFFFFULL) { 1473 copysize = 0x007FFFFFFFFFFFFFULL; 1474 } 1475 uint64_t fs = extract64(fromaddr, 0, 56); 1476 uint64_t ts = extract64(toaddr, 0, 56); 1477 uint64_t fe = extract64(fromaddr + copysize, 0, 56); 1478 1479 if (fs < ts && fe > ts) { 1480 forwards = false; 1481 } 1482 } else { 1483 if (copysize > INT64_MAX) { 1484 copysize = INT64_MAX; 1485 } 1486 } 1487 1488 if (!mte_checks_needed(fromaddr, rdesc)) { 1489 rdesc = 0; 1490 } 1491 if (!mte_checks_needed(toaddr, wdesc)) { 1492 wdesc = 0; 1493 } 1494 1495 if (forwards) { 1496 stagecopysize = MIN(copysize, page_limit(toaddr)); 1497 stagecopysize = MIN(stagecopysize, page_limit(fromaddr)); 1498 while (stagecopysize) { 1499 env->xregs[rd] = toaddr; 1500 env->xregs[rs] = fromaddr; 1501 env->xregs[rn] = copysize; 1502 step = copy_step(env, toaddr, fromaddr, stagecopysize, 1503 wmemidx, rmemidx, &wdesc, &rdesc, ra); 1504 toaddr += step; 1505 fromaddr += step; 1506 copysize -= step; 1507 stagecopysize -= step; 1508 } 1509 /* Insn completed, so update registers to the Option A format */ 1510 env->xregs[rd] = toaddr + copysize; 1511 env->xregs[rs] = fromaddr + copysize; 1512 env->xregs[rn] = -copysize; 1513 } else { 1514 /* 1515 * In a reverse copy the to and from addrs in Xs and Xd are the start 1516 * of the range, but it's more convenient for us to work with pointers 1517 * to the last byte being copied. 1518 */ 1519 toaddr += copysize - 1; 1520 fromaddr += copysize - 1; 1521 stagecopysize = MIN(copysize, page_limit_rev(toaddr)); 1522 stagecopysize = MIN(stagecopysize, page_limit_rev(fromaddr)); 1523 while (stagecopysize) { 1524 env->xregs[rn] = copysize; 1525 step = copy_step_rev(env, toaddr, fromaddr, stagecopysize, 1526 wmemidx, rmemidx, &wdesc, &rdesc, ra); 1527 copysize -= step; 1528 stagecopysize -= step; 1529 toaddr -= step; 1530 fromaddr -= step; 1531 } 1532 /* 1533 * Insn completed, so update registers to the Option A format. 1534 * For a reverse copy this is no different to the CPYP input format. 1535 */ 1536 env->xregs[rn] = copysize; 1537 } 1538 1539 /* Set NZCV = 0000 to indicate we are an Option A implementation */ 1540 env->NF = 0; 1541 env->ZF = 1; /* our env->ZF encoding is inverted */ 1542 env->CF = 0; 1543 env->VF = 0; 1544 return; 1545 } 1546 1547 void HELPER(cpyp)(CPUARMState *env, uint32_t syndrome, uint32_t wdesc, 1548 uint32_t rdesc) 1549 { 1550 do_cpyp(env, syndrome, wdesc, rdesc, true, GETPC()); 1551 } 1552 1553 void HELPER(cpyfp)(CPUARMState *env, uint32_t syndrome, uint32_t wdesc, 1554 uint32_t rdesc) 1555 { 1556 do_cpyp(env, syndrome, wdesc, rdesc, false, GETPC()); 1557 } 1558 1559 static void do_cpym(CPUARMState *env, uint32_t syndrome, uint32_t wdesc, 1560 uint32_t rdesc, uint32_t move, uintptr_t ra) 1561 { 1562 /* Main: we choose to copy until less than a page remaining */ 1563 CPUState *cs = env_cpu(env); 1564 int rd = mops_destreg(syndrome); 1565 int rs = mops_srcreg(syndrome); 1566 int rn = mops_sizereg(syndrome); 1567 uint32_t rmemidx = FIELD_EX32(rdesc, MTEDESC, MIDX); 1568 uint32_t wmemidx = FIELD_EX32(wdesc, MTEDESC, MIDX); 1569 bool forwards = true; 1570 uint64_t toaddr, fromaddr, copysize, step; 1571 1572 check_mops_enabled(env, ra); 1573 1574 /* We choose to NOP out "no data to copy" before consistency checks */ 1575 if (env->xregs[rn] == 0) { 1576 return; 1577 } 1578 1579 check_mops_wrong_option(env, syndrome, ra); 1580 1581 if (move) { 1582 forwards = (int64_t)env->xregs[rn] < 0; 1583 } 1584 1585 if (forwards) { 1586 toaddr = env->xregs[rd] + env->xregs[rn]; 1587 fromaddr = env->xregs[rs] + env->xregs[rn]; 1588 copysize = -env->xregs[rn]; 1589 } else { 1590 copysize = env->xregs[rn]; 1591 /* This toaddr and fromaddr point to the *last* byte to copy */ 1592 toaddr = env->xregs[rd] + copysize - 1; 1593 fromaddr = env->xregs[rs] + copysize - 1; 1594 } 1595 1596 if (!mte_checks_needed(fromaddr, rdesc)) { 1597 rdesc = 0; 1598 } 1599 if (!mte_checks_needed(toaddr, wdesc)) { 1600 wdesc = 0; 1601 } 1602 1603 /* Our implementation has no particular parameter requirements for CPYM */ 1604 1605 /* Do the actual memmove */ 1606 if (forwards) { 1607 while (copysize >= TARGET_PAGE_SIZE) { 1608 step = copy_step(env, toaddr, fromaddr, copysize, 1609 wmemidx, rmemidx, &wdesc, &rdesc, ra); 1610 toaddr += step; 1611 fromaddr += step; 1612 copysize -= step; 1613 env->xregs[rn] = -copysize; 1614 if (copysize >= TARGET_PAGE_SIZE && 1615 unlikely(cpu_loop_exit_requested(cs))) { 1616 cpu_loop_exit_restore(cs, ra); 1617 } 1618 } 1619 } else { 1620 while (copysize >= TARGET_PAGE_SIZE) { 1621 step = copy_step_rev(env, toaddr, fromaddr, copysize, 1622 wmemidx, rmemidx, &wdesc, &rdesc, ra); 1623 toaddr -= step; 1624 fromaddr -= step; 1625 copysize -= step; 1626 env->xregs[rn] = copysize; 1627 if (copysize >= TARGET_PAGE_SIZE && 1628 unlikely(cpu_loop_exit_requested(cs))) { 1629 cpu_loop_exit_restore(cs, ra); 1630 } 1631 } 1632 } 1633 } 1634 1635 void HELPER(cpym)(CPUARMState *env, uint32_t syndrome, uint32_t wdesc, 1636 uint32_t rdesc) 1637 { 1638 do_cpym(env, syndrome, wdesc, rdesc, true, GETPC()); 1639 } 1640 1641 void HELPER(cpyfm)(CPUARMState *env, uint32_t syndrome, uint32_t wdesc, 1642 uint32_t rdesc) 1643 { 1644 do_cpym(env, syndrome, wdesc, rdesc, false, GETPC()); 1645 } 1646 1647 static void do_cpye(CPUARMState *env, uint32_t syndrome, uint32_t wdesc, 1648 uint32_t rdesc, uint32_t move, uintptr_t ra) 1649 { 1650 /* Epilogue: do the last partial page */ 1651 int rd = mops_destreg(syndrome); 1652 int rs = mops_srcreg(syndrome); 1653 int rn = mops_sizereg(syndrome); 1654 uint32_t rmemidx = FIELD_EX32(rdesc, MTEDESC, MIDX); 1655 uint32_t wmemidx = FIELD_EX32(wdesc, MTEDESC, MIDX); 1656 bool forwards = true; 1657 uint64_t toaddr, fromaddr, copysize, step; 1658 1659 check_mops_enabled(env, ra); 1660 1661 /* We choose to NOP out "no data to copy" before consistency checks */ 1662 if (env->xregs[rn] == 0) { 1663 return; 1664 } 1665 1666 check_mops_wrong_option(env, syndrome, ra); 1667 1668 if (move) { 1669 forwards = (int64_t)env->xregs[rn] < 0; 1670 } 1671 1672 if (forwards) { 1673 toaddr = env->xregs[rd] + env->xregs[rn]; 1674 fromaddr = env->xregs[rs] + env->xregs[rn]; 1675 copysize = -env->xregs[rn]; 1676 } else { 1677 copysize = env->xregs[rn]; 1678 /* This toaddr and fromaddr point to the *last* byte to copy */ 1679 toaddr = env->xregs[rd] + copysize - 1; 1680 fromaddr = env->xregs[rs] + copysize - 1; 1681 } 1682 1683 if (!mte_checks_needed(fromaddr, rdesc)) { 1684 rdesc = 0; 1685 } 1686 if (!mte_checks_needed(toaddr, wdesc)) { 1687 wdesc = 0; 1688 } 1689 1690 /* Check the size; we don't want to have do a check-for-interrupts */ 1691 if (copysize >= TARGET_PAGE_SIZE) { 1692 raise_exception_ra(env, EXCP_UDEF, syndrome, 1693 mops_mismatch_exception_target_el(env), ra); 1694 } 1695 1696 /* Do the actual memmove */ 1697 if (forwards) { 1698 while (copysize > 0) { 1699 step = copy_step(env, toaddr, fromaddr, copysize, 1700 wmemidx, rmemidx, &wdesc, &rdesc, ra); 1701 toaddr += step; 1702 fromaddr += step; 1703 copysize -= step; 1704 env->xregs[rn] = -copysize; 1705 } 1706 } else { 1707 while (copysize > 0) { 1708 step = copy_step_rev(env, toaddr, fromaddr, copysize, 1709 wmemidx, rmemidx, &wdesc, &rdesc, ra); 1710 toaddr -= step; 1711 fromaddr -= step; 1712 copysize -= step; 1713 env->xregs[rn] = copysize; 1714 } 1715 } 1716 } 1717 1718 void HELPER(cpye)(CPUARMState *env, uint32_t syndrome, uint32_t wdesc, 1719 uint32_t rdesc) 1720 { 1721 do_cpye(env, syndrome, wdesc, rdesc, true, GETPC()); 1722 } 1723 1724 void HELPER(cpyfe)(CPUARMState *env, uint32_t syndrome, uint32_t wdesc, 1725 uint32_t rdesc) 1726 { 1727 do_cpye(env, syndrome, wdesc, rdesc, false, GETPC()); 1728 } 1729 1730 static bool is_guarded_page(CPUARMState *env, target_ulong addr, uintptr_t ra) 1731 { 1732 #ifdef CONFIG_USER_ONLY 1733 return page_get_flags(addr) & PAGE_BTI; 1734 #else 1735 CPUTLBEntryFull *full; 1736 void *host; 1737 int mmu_idx = cpu_mmu_index(env_cpu(env), true); 1738 int flags = probe_access_full(env, addr, 0, MMU_INST_FETCH, mmu_idx, 1739 false, &host, &full, ra); 1740 1741 assert(!(flags & TLB_INVALID_MASK)); 1742 return full->extra.arm.guarded; 1743 #endif 1744 } 1745 1746 void HELPER(guarded_page_check)(CPUARMState *env) 1747 { 1748 /* 1749 * We have already verified that bti is enabled, and that the 1750 * instruction at PC is not ok for BTYPE. This is always at 1751 * the beginning of a block, so PC is always up-to-date and 1752 * no unwind is required. 1753 */ 1754 if (is_guarded_page(env, env->pc, 0)) { 1755 raise_exception(env, EXCP_UDEF, syn_btitrap(env->btype), 1756 exception_target_el(env)); 1757 } 1758 } 1759 1760 void HELPER(guarded_page_br)(CPUARMState *env, target_ulong pc) 1761 { 1762 /* 1763 * We have already checked for branch via x16 and x17. 1764 * What remains for choosing BTYPE is checking for a guarded page. 1765 */ 1766 env->btype = is_guarded_page(env, pc, GETPC()) ? 3 : 1; 1767 } 1768