1 /* 2 * AArch64 SVE translation 3 * 4 * Copyright (c) 2018 Linaro, Ltd 5 * 6 * This library is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU Lesser General Public 8 * License as published by the Free Software Foundation; either 9 * version 2.1 of the License, or (at your option) any later version. 10 * 11 * This library is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * Lesser General Public License for more details. 15 * 16 * You should have received a copy of the GNU Lesser General Public 17 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 18 */ 19 20 #include "qemu/osdep.h" 21 #include "translate.h" 22 #include "translate-a64.h" 23 #include "fpu/softfloat.h" 24 25 26 typedef void GVecGen2sFn(unsigned, uint32_t, uint32_t, 27 TCGv_i64, uint32_t, uint32_t); 28 29 typedef void gen_helper_gvec_flags_3(TCGv_i32, TCGv_ptr, TCGv_ptr, 30 TCGv_ptr, TCGv_i32); 31 typedef void gen_helper_gvec_flags_4(TCGv_i32, TCGv_ptr, TCGv_ptr, 32 TCGv_ptr, TCGv_ptr, TCGv_i32); 33 34 typedef void gen_helper_gvec_mem(TCGv_env, TCGv_ptr, TCGv_i64, TCGv_i32); 35 typedef void gen_helper_gvec_mem_scatter(TCGv_env, TCGv_ptr, TCGv_ptr, 36 TCGv_ptr, TCGv_i64, TCGv_i32); 37 38 /* 39 * Helpers for extracting complex instruction fields. 40 */ 41 42 /* See e.g. ASR (immediate, predicated). 43 * Returns -1 for unallocated encoding; diagnose later. 44 */ 45 static int tszimm_esz(DisasContext *s, int x) 46 { 47 x >>= 3; /* discard imm3 */ 48 return 31 - clz32(x); 49 } 50 51 static int tszimm_shr(DisasContext *s, int x) 52 { 53 /* 54 * We won't use the tszimm_shr() value if tszimm_esz() returns -1 (the 55 * trans function will check for esz < 0), so we can return any 56 * value we like from here in that case as long as we avoid UB. 57 */ 58 int esz = tszimm_esz(s, x); 59 if (esz < 0) { 60 return esz; 61 } 62 return (16 << esz) - x; 63 } 64 65 /* See e.g. LSL (immediate, predicated). */ 66 static int tszimm_shl(DisasContext *s, int x) 67 { 68 /* As with tszimm_shr(), value will be unused if esz < 0 */ 69 int esz = tszimm_esz(s, x); 70 if (esz < 0) { 71 return esz; 72 } 73 return x - (8 << esz); 74 } 75 76 /* The SH bit is in bit 8. Extract the low 8 and shift. */ 77 static inline int expand_imm_sh8s(DisasContext *s, int x) 78 { 79 return (int8_t)x << (x & 0x100 ? 8 : 0); 80 } 81 82 static inline int expand_imm_sh8u(DisasContext *s, int x) 83 { 84 return (uint8_t)x << (x & 0x100 ? 8 : 0); 85 } 86 87 /* Convert a 2-bit memory size (msz) to a 4-bit data type (dtype) 88 * with unsigned data. C.f. SVE Memory Contiguous Load Group. 89 */ 90 static inline int msz_dtype(DisasContext *s, int msz) 91 { 92 static const uint8_t dtype[4] = { 0, 5, 10, 15 }; 93 return dtype[msz]; 94 } 95 96 /* 97 * Include the generated decoder. 98 */ 99 100 #include "decode-sve.c.inc" 101 102 /* 103 * Implement all of the translator functions referenced by the decoder. 104 */ 105 106 /* Invoke an out-of-line helper on 2 Zregs. */ 107 static bool gen_gvec_ool_zz(DisasContext *s, gen_helper_gvec_2 *fn, 108 int rd, int rn, int data) 109 { 110 if (fn == NULL) { 111 return false; 112 } 113 if (sve_access_check(s)) { 114 unsigned vsz = vec_full_reg_size(s); 115 tcg_gen_gvec_2_ool(vec_full_reg_offset(s, rd), 116 vec_full_reg_offset(s, rn), 117 vsz, vsz, data, fn); 118 } 119 return true; 120 } 121 122 static bool gen_gvec_fpst_zz(DisasContext *s, gen_helper_gvec_2_ptr *fn, 123 int rd, int rn, int data, 124 ARMFPStatusFlavour flavour) 125 { 126 if (fn == NULL) { 127 return false; 128 } 129 if (sve_access_check(s)) { 130 unsigned vsz = vec_full_reg_size(s); 131 TCGv_ptr status = fpstatus_ptr(flavour); 132 133 tcg_gen_gvec_2_ptr(vec_full_reg_offset(s, rd), 134 vec_full_reg_offset(s, rn), 135 status, vsz, vsz, data, fn); 136 } 137 return true; 138 } 139 140 static bool gen_gvec_fpst_ah_arg_zz(DisasContext *s, gen_helper_gvec_2_ptr *fn, 141 arg_rr_esz *a, int data) 142 { 143 return gen_gvec_fpst_zz(s, fn, a->rd, a->rn, data, 144 select_ah_fpst(s, a->esz)); 145 } 146 147 /* Invoke an out-of-line helper on 3 Zregs. */ 148 static bool gen_gvec_ool_zzz(DisasContext *s, gen_helper_gvec_3 *fn, 149 int rd, int rn, int rm, int data) 150 { 151 if (fn == NULL) { 152 return false; 153 } 154 if (sve_access_check(s)) { 155 unsigned vsz = vec_full_reg_size(s); 156 tcg_gen_gvec_3_ool(vec_full_reg_offset(s, rd), 157 vec_full_reg_offset(s, rn), 158 vec_full_reg_offset(s, rm), 159 vsz, vsz, data, fn); 160 } 161 return true; 162 } 163 164 static bool gen_gvec_ool_arg_zzz(DisasContext *s, gen_helper_gvec_3 *fn, 165 arg_rrr_esz *a, int data) 166 { 167 return gen_gvec_ool_zzz(s, fn, a->rd, a->rn, a->rm, data); 168 } 169 170 /* Invoke an out-of-line helper on 3 Zregs, plus float_status. */ 171 static bool gen_gvec_fpst_zzz(DisasContext *s, gen_helper_gvec_3_ptr *fn, 172 int rd, int rn, int rm, 173 int data, ARMFPStatusFlavour flavour) 174 { 175 if (fn == NULL) { 176 return false; 177 } 178 if (sve_access_check(s)) { 179 unsigned vsz = vec_full_reg_size(s); 180 TCGv_ptr status = fpstatus_ptr(flavour); 181 182 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, rd), 183 vec_full_reg_offset(s, rn), 184 vec_full_reg_offset(s, rm), 185 status, vsz, vsz, data, fn); 186 } 187 return true; 188 } 189 190 static bool gen_gvec_fpst_arg_zzz(DisasContext *s, gen_helper_gvec_3_ptr *fn, 191 arg_rrr_esz *a, int data) 192 { 193 return gen_gvec_fpst_zzz(s, fn, a->rd, a->rn, a->rm, data, 194 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64); 195 } 196 197 static bool gen_gvec_fpst_ah_arg_zzz(DisasContext *s, gen_helper_gvec_3_ptr *fn, 198 arg_rrr_esz *a, int data) 199 { 200 return gen_gvec_fpst_zzz(s, fn, a->rd, a->rn, a->rm, data, 201 select_ah_fpst(s, a->esz)); 202 } 203 204 /* Invoke an out-of-line helper on 4 Zregs. */ 205 static bool gen_gvec_ool_zzzz(DisasContext *s, gen_helper_gvec_4 *fn, 206 int rd, int rn, int rm, int ra, int data) 207 { 208 if (fn == NULL) { 209 return false; 210 } 211 if (sve_access_check(s)) { 212 unsigned vsz = vec_full_reg_size(s); 213 tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd), 214 vec_full_reg_offset(s, rn), 215 vec_full_reg_offset(s, rm), 216 vec_full_reg_offset(s, ra), 217 vsz, vsz, data, fn); 218 } 219 return true; 220 } 221 222 static bool gen_gvec_ool_arg_zzzz(DisasContext *s, gen_helper_gvec_4 *fn, 223 arg_rrrr_esz *a, int data) 224 { 225 return gen_gvec_ool_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, data); 226 } 227 228 static bool gen_gvec_ool_arg_zzxz(DisasContext *s, gen_helper_gvec_4 *fn, 229 arg_rrxr_esz *a) 230 { 231 return gen_gvec_ool_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, a->index); 232 } 233 234 /* Invoke an out-of-line helper on 4 Zregs, plus a pointer. */ 235 static bool gen_gvec_ptr_zzzz(DisasContext *s, gen_helper_gvec_4_ptr *fn, 236 int rd, int rn, int rm, int ra, 237 int data, TCGv_ptr ptr) 238 { 239 if (fn == NULL) { 240 return false; 241 } 242 if (sve_access_check(s)) { 243 unsigned vsz = vec_full_reg_size(s); 244 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, rd), 245 vec_full_reg_offset(s, rn), 246 vec_full_reg_offset(s, rm), 247 vec_full_reg_offset(s, ra), 248 ptr, vsz, vsz, data, fn); 249 } 250 return true; 251 } 252 253 static bool gen_gvec_fpst_zzzz(DisasContext *s, gen_helper_gvec_4_ptr *fn, 254 int rd, int rn, int rm, int ra, 255 int data, ARMFPStatusFlavour flavour) 256 { 257 TCGv_ptr status = fpstatus_ptr(flavour); 258 bool ret = gen_gvec_ptr_zzzz(s, fn, rd, rn, rm, ra, data, status); 259 return ret; 260 } 261 262 static bool gen_gvec_env_zzzz(DisasContext *s, gen_helper_gvec_4_ptr *fn, 263 int rd, int rn, int rm, int ra, 264 int data) 265 { 266 return gen_gvec_ptr_zzzz(s, fn, rd, rn, rm, ra, data, tcg_env); 267 } 268 269 static bool gen_gvec_env_arg_zzzz(DisasContext *s, gen_helper_gvec_4_ptr *fn, 270 arg_rrrr_esz *a, int data) 271 { 272 return gen_gvec_env_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, data); 273 } 274 275 static bool gen_gvec_env_arg_zzxz(DisasContext *s, gen_helper_gvec_4_ptr *fn, 276 arg_rrxr_esz *a) 277 { 278 return gen_gvec_env_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, a->index); 279 } 280 281 /* Invoke an out-of-line helper on 4 Zregs, 1 Preg, plus fpst. */ 282 static bool gen_gvec_fpst_zzzzp(DisasContext *s, gen_helper_gvec_5_ptr *fn, 283 int rd, int rn, int rm, int ra, int pg, 284 int data, ARMFPStatusFlavour flavour) 285 { 286 if (fn == NULL) { 287 return false; 288 } 289 if (sve_access_check(s)) { 290 unsigned vsz = vec_full_reg_size(s); 291 TCGv_ptr status = fpstatus_ptr(flavour); 292 293 tcg_gen_gvec_5_ptr(vec_full_reg_offset(s, rd), 294 vec_full_reg_offset(s, rn), 295 vec_full_reg_offset(s, rm), 296 vec_full_reg_offset(s, ra), 297 pred_full_reg_offset(s, pg), 298 status, vsz, vsz, data, fn); 299 } 300 return true; 301 } 302 303 /* Invoke an out-of-line helper on 2 Zregs and a predicate. */ 304 static bool gen_gvec_ool_zzp(DisasContext *s, gen_helper_gvec_3 *fn, 305 int rd, int rn, int pg, int data) 306 { 307 if (fn == NULL) { 308 return false; 309 } 310 if (sve_access_check(s)) { 311 unsigned vsz = vec_full_reg_size(s); 312 tcg_gen_gvec_3_ool(vec_full_reg_offset(s, rd), 313 vec_full_reg_offset(s, rn), 314 pred_full_reg_offset(s, pg), 315 vsz, vsz, data, fn); 316 } 317 return true; 318 } 319 320 static bool gen_gvec_ool_arg_zpz(DisasContext *s, gen_helper_gvec_3 *fn, 321 arg_rpr_esz *a, int data) 322 { 323 return gen_gvec_ool_zzp(s, fn, a->rd, a->rn, a->pg, data); 324 } 325 326 static bool gen_gvec_ool_arg_zpzi(DisasContext *s, gen_helper_gvec_3 *fn, 327 arg_rpri_esz *a) 328 { 329 return gen_gvec_ool_zzp(s, fn, a->rd, a->rn, a->pg, a->imm); 330 } 331 332 static bool gen_gvec_fpst_zzp(DisasContext *s, gen_helper_gvec_3_ptr *fn, 333 int rd, int rn, int pg, int data, 334 ARMFPStatusFlavour flavour) 335 { 336 if (fn == NULL) { 337 return false; 338 } 339 if (sve_access_check(s)) { 340 unsigned vsz = vec_full_reg_size(s); 341 TCGv_ptr status = fpstatus_ptr(flavour); 342 343 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, rd), 344 vec_full_reg_offset(s, rn), 345 pred_full_reg_offset(s, pg), 346 status, vsz, vsz, data, fn); 347 } 348 return true; 349 } 350 351 static bool gen_gvec_fpst_arg_zpz(DisasContext *s, gen_helper_gvec_3_ptr *fn, 352 arg_rpr_esz *a, int data, 353 ARMFPStatusFlavour flavour) 354 { 355 return gen_gvec_fpst_zzp(s, fn, a->rd, a->rn, a->pg, data, flavour); 356 } 357 358 /* Invoke an out-of-line helper on 3 Zregs and a predicate. */ 359 static bool gen_gvec_ool_zzzp(DisasContext *s, gen_helper_gvec_4 *fn, 360 int rd, int rn, int rm, int pg, int data) 361 { 362 if (fn == NULL) { 363 return false; 364 } 365 if (sve_access_check(s)) { 366 unsigned vsz = vec_full_reg_size(s); 367 tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd), 368 vec_full_reg_offset(s, rn), 369 vec_full_reg_offset(s, rm), 370 pred_full_reg_offset(s, pg), 371 vsz, vsz, data, fn); 372 } 373 return true; 374 } 375 376 static bool gen_gvec_ool_arg_zpzz(DisasContext *s, gen_helper_gvec_4 *fn, 377 arg_rprr_esz *a, int data) 378 { 379 return gen_gvec_ool_zzzp(s, fn, a->rd, a->rn, a->rm, a->pg, data); 380 } 381 382 /* Invoke an out-of-line helper on 3 Zregs and a predicate. */ 383 static bool gen_gvec_fpst_zzzp(DisasContext *s, gen_helper_gvec_4_ptr *fn, 384 int rd, int rn, int rm, int pg, int data, 385 ARMFPStatusFlavour flavour) 386 { 387 if (fn == NULL) { 388 return false; 389 } 390 if (sve_access_check(s)) { 391 unsigned vsz = vec_full_reg_size(s); 392 TCGv_ptr status = fpstatus_ptr(flavour); 393 394 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, rd), 395 vec_full_reg_offset(s, rn), 396 vec_full_reg_offset(s, rm), 397 pred_full_reg_offset(s, pg), 398 status, vsz, vsz, data, fn); 399 } 400 return true; 401 } 402 403 static bool gen_gvec_fpst_arg_zpzz(DisasContext *s, gen_helper_gvec_4_ptr *fn, 404 arg_rprr_esz *a) 405 { 406 return gen_gvec_fpst_zzzp(s, fn, a->rd, a->rn, a->rm, a->pg, 0, 407 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64); 408 } 409 410 /* Invoke a vector expander on two Zregs and an immediate. */ 411 static bool gen_gvec_fn_zzi(DisasContext *s, GVecGen2iFn *gvec_fn, 412 int esz, int rd, int rn, uint64_t imm) 413 { 414 if (gvec_fn == NULL) { 415 return false; 416 } 417 if (sve_access_check(s)) { 418 unsigned vsz = vec_full_reg_size(s); 419 gvec_fn(esz, vec_full_reg_offset(s, rd), 420 vec_full_reg_offset(s, rn), imm, vsz, vsz); 421 } 422 return true; 423 } 424 425 static bool gen_gvec_fn_arg_zzi(DisasContext *s, GVecGen2iFn *gvec_fn, 426 arg_rri_esz *a) 427 { 428 if (a->esz < 0) { 429 /* Invalid tsz encoding -- see tszimm_esz. */ 430 return false; 431 } 432 return gen_gvec_fn_zzi(s, gvec_fn, a->esz, a->rd, a->rn, a->imm); 433 } 434 435 /* Invoke a vector expander on three Zregs. */ 436 static bool gen_gvec_fn_zzz(DisasContext *s, GVecGen3Fn *gvec_fn, 437 int esz, int rd, int rn, int rm) 438 { 439 if (gvec_fn == NULL) { 440 return false; 441 } 442 if (sve_access_check(s)) { 443 unsigned vsz = vec_full_reg_size(s); 444 gvec_fn(esz, vec_full_reg_offset(s, rd), 445 vec_full_reg_offset(s, rn), 446 vec_full_reg_offset(s, rm), vsz, vsz); 447 } 448 return true; 449 } 450 451 static bool gen_gvec_fn_arg_zzz(DisasContext *s, GVecGen3Fn *fn, 452 arg_rrr_esz *a) 453 { 454 return gen_gvec_fn_zzz(s, fn, a->esz, a->rd, a->rn, a->rm); 455 } 456 457 /* Invoke a vector expander on four Zregs. */ 458 static bool gen_gvec_fn_arg_zzzz(DisasContext *s, GVecGen4Fn *gvec_fn, 459 arg_rrrr_esz *a) 460 { 461 if (gvec_fn == NULL) { 462 return false; 463 } 464 if (sve_access_check(s)) { 465 unsigned vsz = vec_full_reg_size(s); 466 gvec_fn(a->esz, vec_full_reg_offset(s, a->rd), 467 vec_full_reg_offset(s, a->rn), 468 vec_full_reg_offset(s, a->rm), 469 vec_full_reg_offset(s, a->ra), vsz, vsz); 470 } 471 return true; 472 } 473 474 /* Invoke a vector move on two Zregs. */ 475 static bool do_mov_z(DisasContext *s, int rd, int rn) 476 { 477 if (sve_access_check(s)) { 478 unsigned vsz = vec_full_reg_size(s); 479 tcg_gen_gvec_mov(MO_8, vec_full_reg_offset(s, rd), 480 vec_full_reg_offset(s, rn), vsz, vsz); 481 } 482 return true; 483 } 484 485 /* Initialize a Zreg with replications of a 64-bit immediate. */ 486 static void do_dupi_z(DisasContext *s, int rd, uint64_t word) 487 { 488 unsigned vsz = vec_full_reg_size(s); 489 tcg_gen_gvec_dup_imm(MO_64, vec_full_reg_offset(s, rd), vsz, vsz, word); 490 } 491 492 /* Invoke a vector expander on three Pregs. */ 493 static bool gen_gvec_fn_ppp(DisasContext *s, GVecGen3Fn *gvec_fn, 494 int rd, int rn, int rm) 495 { 496 if (sve_access_check(s)) { 497 unsigned psz = pred_gvec_reg_size(s); 498 gvec_fn(MO_64, pred_full_reg_offset(s, rd), 499 pred_full_reg_offset(s, rn), 500 pred_full_reg_offset(s, rm), psz, psz); 501 } 502 return true; 503 } 504 505 /* Invoke a vector move on two Pregs. */ 506 static bool do_mov_p(DisasContext *s, int rd, int rn) 507 { 508 if (sve_access_check(s)) { 509 unsigned psz = pred_gvec_reg_size(s); 510 tcg_gen_gvec_mov(MO_8, pred_full_reg_offset(s, rd), 511 pred_full_reg_offset(s, rn), psz, psz); 512 } 513 return true; 514 } 515 516 /* Set the cpu flags as per a return from an SVE helper. */ 517 static void do_pred_flags(TCGv_i32 t) 518 { 519 tcg_gen_mov_i32(cpu_NF, t); 520 tcg_gen_andi_i32(cpu_ZF, t, 2); 521 tcg_gen_andi_i32(cpu_CF, t, 1); 522 tcg_gen_movi_i32(cpu_VF, 0); 523 } 524 525 /* Subroutines computing the ARM PredTest psuedofunction. */ 526 static void do_predtest1(TCGv_i64 d, TCGv_i64 g) 527 { 528 TCGv_i32 t = tcg_temp_new_i32(); 529 530 gen_helper_sve_predtest1(t, d, g); 531 do_pred_flags(t); 532 } 533 534 static void do_predtest(DisasContext *s, int dofs, int gofs, int words) 535 { 536 TCGv_ptr dptr = tcg_temp_new_ptr(); 537 TCGv_ptr gptr = tcg_temp_new_ptr(); 538 TCGv_i32 t = tcg_temp_new_i32(); 539 540 tcg_gen_addi_ptr(dptr, tcg_env, dofs); 541 tcg_gen_addi_ptr(gptr, tcg_env, gofs); 542 543 gen_helper_sve_predtest(t, dptr, gptr, tcg_constant_i32(words)); 544 545 do_pred_flags(t); 546 } 547 548 /* For each element size, the bits within a predicate word that are active. */ 549 const uint64_t pred_esz_masks[5] = { 550 0xffffffffffffffffull, 0x5555555555555555ull, 551 0x1111111111111111ull, 0x0101010101010101ull, 552 0x0001000100010001ull, 553 }; 554 555 static bool trans_INVALID(DisasContext *s, arg_INVALID *a) 556 { 557 unallocated_encoding(s); 558 return true; 559 } 560 561 /* 562 *** SVE Logical - Unpredicated Group 563 */ 564 565 TRANS_FEAT(AND_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_and, a) 566 TRANS_FEAT(ORR_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_or, a) 567 TRANS_FEAT(EOR_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_xor, a) 568 TRANS_FEAT(BIC_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_andc, a) 569 570 static bool trans_XAR(DisasContext *s, arg_rrri_esz *a) 571 { 572 if (a->esz < 0 || !dc_isar_feature(aa64_sve2, s)) { 573 return false; 574 } 575 if (sve_access_check(s)) { 576 unsigned vsz = vec_full_reg_size(s); 577 gen_gvec_xar(a->esz, vec_full_reg_offset(s, a->rd), 578 vec_full_reg_offset(s, a->rn), 579 vec_full_reg_offset(s, a->rm), a->imm, vsz, vsz); 580 } 581 return true; 582 } 583 584 TRANS_FEAT(EOR3, aa64_sve2, gen_gvec_fn_arg_zzzz, gen_gvec_eor3, a) 585 TRANS_FEAT(BCAX, aa64_sve2, gen_gvec_fn_arg_zzzz, gen_gvec_bcax, a) 586 587 static void gen_bsl(unsigned vece, uint32_t d, uint32_t n, uint32_t m, 588 uint32_t a, uint32_t oprsz, uint32_t maxsz) 589 { 590 /* BSL differs from the generic bitsel in argument ordering. */ 591 tcg_gen_gvec_bitsel(vece, d, a, n, m, oprsz, maxsz); 592 } 593 594 TRANS_FEAT(BSL, aa64_sve2, gen_gvec_fn_arg_zzzz, gen_bsl, a) 595 596 static void gen_bsl1n_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 k) 597 { 598 tcg_gen_andc_i64(n, k, n); 599 tcg_gen_andc_i64(m, m, k); 600 tcg_gen_or_i64(d, n, m); 601 } 602 603 static void gen_bsl1n_vec(unsigned vece, TCGv_vec d, TCGv_vec n, 604 TCGv_vec m, TCGv_vec k) 605 { 606 tcg_gen_not_vec(vece, n, n); 607 tcg_gen_bitsel_vec(vece, d, k, n, m); 608 } 609 610 static void gen_bsl1n(unsigned vece, uint32_t d, uint32_t n, uint32_t m, 611 uint32_t a, uint32_t oprsz, uint32_t maxsz) 612 { 613 static const GVecGen4 op = { 614 .fni8 = gen_bsl1n_i64, 615 .fniv = gen_bsl1n_vec, 616 .fno = gen_helper_sve2_bsl1n, 617 .vece = MO_64, 618 .prefer_i64 = TCG_TARGET_REG_BITS == 64, 619 }; 620 tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op); 621 } 622 623 TRANS_FEAT(BSL1N, aa64_sve2, gen_gvec_fn_arg_zzzz, gen_bsl1n, a) 624 625 static void gen_bsl2n_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 k) 626 { 627 /* 628 * Z[dn] = (n & k) | (~m & ~k) 629 * = | ~(m | k) 630 */ 631 tcg_gen_and_i64(n, n, k); 632 if (tcg_op_supported(INDEX_op_orc_i64, TCG_TYPE_I64, 0)) { 633 tcg_gen_or_i64(m, m, k); 634 tcg_gen_orc_i64(d, n, m); 635 } else { 636 tcg_gen_nor_i64(m, m, k); 637 tcg_gen_or_i64(d, n, m); 638 } 639 } 640 641 static void gen_bsl2n_vec(unsigned vece, TCGv_vec d, TCGv_vec n, 642 TCGv_vec m, TCGv_vec k) 643 { 644 tcg_gen_not_vec(vece, m, m); 645 tcg_gen_bitsel_vec(vece, d, k, n, m); 646 } 647 648 static void gen_bsl2n(unsigned vece, uint32_t d, uint32_t n, uint32_t m, 649 uint32_t a, uint32_t oprsz, uint32_t maxsz) 650 { 651 static const GVecGen4 op = { 652 .fni8 = gen_bsl2n_i64, 653 .fniv = gen_bsl2n_vec, 654 .fno = gen_helper_sve2_bsl2n, 655 .vece = MO_64, 656 .prefer_i64 = TCG_TARGET_REG_BITS == 64, 657 }; 658 tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op); 659 } 660 661 TRANS_FEAT(BSL2N, aa64_sve2, gen_gvec_fn_arg_zzzz, gen_bsl2n, a) 662 663 static void gen_nbsl_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 k) 664 { 665 tcg_gen_and_i64(n, n, k); 666 tcg_gen_andc_i64(m, m, k); 667 tcg_gen_nor_i64(d, n, m); 668 } 669 670 static void gen_nbsl_vec(unsigned vece, TCGv_vec d, TCGv_vec n, 671 TCGv_vec m, TCGv_vec k) 672 { 673 tcg_gen_bitsel_vec(vece, d, k, n, m); 674 tcg_gen_not_vec(vece, d, d); 675 } 676 677 static void gen_nbsl(unsigned vece, uint32_t d, uint32_t n, uint32_t m, 678 uint32_t a, uint32_t oprsz, uint32_t maxsz) 679 { 680 static const GVecGen4 op = { 681 .fni8 = gen_nbsl_i64, 682 .fniv = gen_nbsl_vec, 683 .fno = gen_helper_sve2_nbsl, 684 .vece = MO_64, 685 .prefer_i64 = TCG_TARGET_REG_BITS == 64, 686 }; 687 tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op); 688 } 689 690 TRANS_FEAT(NBSL, aa64_sve2, gen_gvec_fn_arg_zzzz, gen_nbsl, a) 691 692 /* 693 *** SVE Integer Arithmetic - Unpredicated Group 694 */ 695 696 TRANS_FEAT(ADD_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_add, a) 697 TRANS_FEAT(SUB_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_sub, a) 698 TRANS_FEAT(SQADD_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_ssadd, a) 699 TRANS_FEAT(SQSUB_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_sssub, a) 700 TRANS_FEAT(UQADD_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_usadd, a) 701 TRANS_FEAT(UQSUB_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_ussub, a) 702 703 /* 704 *** SVE Integer Arithmetic - Binary Predicated Group 705 */ 706 707 /* Select active elememnts from Zn and inactive elements from Zm, 708 * storing the result in Zd. 709 */ 710 static bool do_sel_z(DisasContext *s, int rd, int rn, int rm, int pg, int esz) 711 { 712 static gen_helper_gvec_4 * const fns[4] = { 713 gen_helper_sve_sel_zpzz_b, gen_helper_sve_sel_zpzz_h, 714 gen_helper_sve_sel_zpzz_s, gen_helper_sve_sel_zpzz_d 715 }; 716 return gen_gvec_ool_zzzp(s, fns[esz], rd, rn, rm, pg, 0); 717 } 718 719 #define DO_ZPZZ(NAME, FEAT, name) \ 720 static gen_helper_gvec_4 * const name##_zpzz_fns[4] = { \ 721 gen_helper_##name##_zpzz_b, gen_helper_##name##_zpzz_h, \ 722 gen_helper_##name##_zpzz_s, gen_helper_##name##_zpzz_d, \ 723 }; \ 724 TRANS_FEAT(NAME, FEAT, gen_gvec_ool_arg_zpzz, \ 725 name##_zpzz_fns[a->esz], a, 0) 726 727 DO_ZPZZ(AND_zpzz, aa64_sve, sve_and) 728 DO_ZPZZ(EOR_zpzz, aa64_sve, sve_eor) 729 DO_ZPZZ(ORR_zpzz, aa64_sve, sve_orr) 730 DO_ZPZZ(BIC_zpzz, aa64_sve, sve_bic) 731 732 DO_ZPZZ(ADD_zpzz, aa64_sve, sve_add) 733 DO_ZPZZ(SUB_zpzz, aa64_sve, sve_sub) 734 735 DO_ZPZZ(SMAX_zpzz, aa64_sve, sve_smax) 736 DO_ZPZZ(UMAX_zpzz, aa64_sve, sve_umax) 737 DO_ZPZZ(SMIN_zpzz, aa64_sve, sve_smin) 738 DO_ZPZZ(UMIN_zpzz, aa64_sve, sve_umin) 739 DO_ZPZZ(SABD_zpzz, aa64_sve, sve_sabd) 740 DO_ZPZZ(UABD_zpzz, aa64_sve, sve_uabd) 741 742 DO_ZPZZ(MUL_zpzz, aa64_sve, sve_mul) 743 DO_ZPZZ(SMULH_zpzz, aa64_sve, sve_smulh) 744 DO_ZPZZ(UMULH_zpzz, aa64_sve, sve_umulh) 745 746 DO_ZPZZ(ASR_zpzz, aa64_sve, sve_asr) 747 DO_ZPZZ(LSR_zpzz, aa64_sve, sve_lsr) 748 DO_ZPZZ(LSL_zpzz, aa64_sve, sve_lsl) 749 750 static gen_helper_gvec_4 * const sdiv_fns[4] = { 751 NULL, NULL, gen_helper_sve_sdiv_zpzz_s, gen_helper_sve_sdiv_zpzz_d 752 }; 753 TRANS_FEAT(SDIV_zpzz, aa64_sve, gen_gvec_ool_arg_zpzz, sdiv_fns[a->esz], a, 0) 754 755 static gen_helper_gvec_4 * const udiv_fns[4] = { 756 NULL, NULL, gen_helper_sve_udiv_zpzz_s, gen_helper_sve_udiv_zpzz_d 757 }; 758 TRANS_FEAT(UDIV_zpzz, aa64_sve, gen_gvec_ool_arg_zpzz, udiv_fns[a->esz], a, 0) 759 760 TRANS_FEAT(SEL_zpzz, aa64_sve, do_sel_z, a->rd, a->rn, a->rm, a->pg, a->esz) 761 762 /* 763 *** SVE Integer Arithmetic - Unary Predicated Group 764 */ 765 766 #define DO_ZPZ(NAME, FEAT, name) \ 767 static gen_helper_gvec_3 * const name##_fns[4] = { \ 768 gen_helper_##name##_b, gen_helper_##name##_h, \ 769 gen_helper_##name##_s, gen_helper_##name##_d, \ 770 }; \ 771 TRANS_FEAT(NAME, FEAT, gen_gvec_ool_arg_zpz, name##_fns[a->esz], a, 0) 772 773 DO_ZPZ(CLS, aa64_sve, sve_cls) 774 DO_ZPZ(CLZ, aa64_sve, sve_clz) 775 DO_ZPZ(CNT_zpz, aa64_sve, sve_cnt_zpz) 776 DO_ZPZ(CNOT, aa64_sve, sve_cnot) 777 DO_ZPZ(NOT_zpz, aa64_sve, sve_not_zpz) 778 DO_ZPZ(ABS, aa64_sve, sve_abs) 779 DO_ZPZ(NEG, aa64_sve, sve_neg) 780 DO_ZPZ(RBIT, aa64_sve, sve_rbit) 781 782 static gen_helper_gvec_3 * const fabs_fns[4] = { 783 NULL, gen_helper_sve_fabs_h, 784 gen_helper_sve_fabs_s, gen_helper_sve_fabs_d, 785 }; 786 static gen_helper_gvec_3 * const fabs_ah_fns[4] = { 787 NULL, gen_helper_sve_ah_fabs_h, 788 gen_helper_sve_ah_fabs_s, gen_helper_sve_ah_fabs_d, 789 }; 790 TRANS_FEAT(FABS, aa64_sve, gen_gvec_ool_arg_zpz, 791 s->fpcr_ah ? fabs_ah_fns[a->esz] : fabs_fns[a->esz], a, 0) 792 793 static gen_helper_gvec_3 * const fneg_fns[4] = { 794 NULL, gen_helper_sve_fneg_h, 795 gen_helper_sve_fneg_s, gen_helper_sve_fneg_d, 796 }; 797 static gen_helper_gvec_3 * const fneg_ah_fns[4] = { 798 NULL, gen_helper_sve_ah_fneg_h, 799 gen_helper_sve_ah_fneg_s, gen_helper_sve_ah_fneg_d, 800 }; 801 TRANS_FEAT(FNEG, aa64_sve, gen_gvec_ool_arg_zpz, 802 s->fpcr_ah ? fneg_ah_fns[a->esz] : fneg_fns[a->esz], a, 0) 803 804 static gen_helper_gvec_3 * const sxtb_fns[4] = { 805 NULL, gen_helper_sve_sxtb_h, 806 gen_helper_sve_sxtb_s, gen_helper_sve_sxtb_d, 807 }; 808 TRANS_FEAT(SXTB, aa64_sve, gen_gvec_ool_arg_zpz, sxtb_fns[a->esz], a, 0) 809 810 static gen_helper_gvec_3 * const uxtb_fns[4] = { 811 NULL, gen_helper_sve_uxtb_h, 812 gen_helper_sve_uxtb_s, gen_helper_sve_uxtb_d, 813 }; 814 TRANS_FEAT(UXTB, aa64_sve, gen_gvec_ool_arg_zpz, uxtb_fns[a->esz], a, 0) 815 816 static gen_helper_gvec_3 * const sxth_fns[4] = { 817 NULL, NULL, gen_helper_sve_sxth_s, gen_helper_sve_sxth_d 818 }; 819 TRANS_FEAT(SXTH, aa64_sve, gen_gvec_ool_arg_zpz, sxth_fns[a->esz], a, 0) 820 821 static gen_helper_gvec_3 * const uxth_fns[4] = { 822 NULL, NULL, gen_helper_sve_uxth_s, gen_helper_sve_uxth_d 823 }; 824 TRANS_FEAT(UXTH, aa64_sve, gen_gvec_ool_arg_zpz, uxth_fns[a->esz], a, 0) 825 826 TRANS_FEAT(SXTW, aa64_sve, gen_gvec_ool_arg_zpz, 827 a->esz == 3 ? gen_helper_sve_sxtw_d : NULL, a, 0) 828 TRANS_FEAT(UXTW, aa64_sve, gen_gvec_ool_arg_zpz, 829 a->esz == 3 ? gen_helper_sve_uxtw_d : NULL, a, 0) 830 831 /* 832 *** SVE Integer Reduction Group 833 */ 834 835 typedef void gen_helper_gvec_reduc(TCGv_i64, TCGv_ptr, TCGv_ptr, TCGv_i32); 836 static bool do_vpz_ool(DisasContext *s, arg_rpr_esz *a, 837 gen_helper_gvec_reduc *fn) 838 { 839 unsigned vsz = vec_full_reg_size(s); 840 TCGv_ptr t_zn, t_pg; 841 TCGv_i32 desc; 842 TCGv_i64 temp; 843 844 if (fn == NULL) { 845 return false; 846 } 847 if (!sve_access_check(s)) { 848 return true; 849 } 850 851 desc = tcg_constant_i32(simd_desc(vsz, vsz, 0)); 852 temp = tcg_temp_new_i64(); 853 t_zn = tcg_temp_new_ptr(); 854 t_pg = tcg_temp_new_ptr(); 855 856 tcg_gen_addi_ptr(t_zn, tcg_env, vec_full_reg_offset(s, a->rn)); 857 tcg_gen_addi_ptr(t_pg, tcg_env, pred_full_reg_offset(s, a->pg)); 858 fn(temp, t_zn, t_pg, desc); 859 860 write_fp_dreg(s, a->rd, temp); 861 return true; 862 } 863 864 #define DO_VPZ(NAME, name) \ 865 static gen_helper_gvec_reduc * const name##_fns[4] = { \ 866 gen_helper_sve_##name##_b, gen_helper_sve_##name##_h, \ 867 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \ 868 }; \ 869 TRANS_FEAT(NAME, aa64_sve, do_vpz_ool, a, name##_fns[a->esz]) 870 871 DO_VPZ(ORV, orv) 872 DO_VPZ(ANDV, andv) 873 DO_VPZ(EORV, eorv) 874 875 DO_VPZ(UADDV, uaddv) 876 DO_VPZ(SMAXV, smaxv) 877 DO_VPZ(UMAXV, umaxv) 878 DO_VPZ(SMINV, sminv) 879 DO_VPZ(UMINV, uminv) 880 881 static gen_helper_gvec_reduc * const saddv_fns[4] = { 882 gen_helper_sve_saddv_b, gen_helper_sve_saddv_h, 883 gen_helper_sve_saddv_s, NULL 884 }; 885 TRANS_FEAT(SADDV, aa64_sve, do_vpz_ool, a, saddv_fns[a->esz]) 886 887 #undef DO_VPZ 888 889 /* 890 *** SVE Shift by Immediate - Predicated Group 891 */ 892 893 /* 894 * Copy Zn into Zd, storing zeros into inactive elements. 895 * If invert, store zeros into the active elements. 896 */ 897 static bool do_movz_zpz(DisasContext *s, int rd, int rn, int pg, 898 int esz, bool invert) 899 { 900 static gen_helper_gvec_3 * const fns[4] = { 901 gen_helper_sve_movz_b, gen_helper_sve_movz_h, 902 gen_helper_sve_movz_s, gen_helper_sve_movz_d, 903 }; 904 return gen_gvec_ool_zzp(s, fns[esz], rd, rn, pg, invert); 905 } 906 907 static bool do_shift_zpzi(DisasContext *s, arg_rpri_esz *a, bool asr, 908 gen_helper_gvec_3 * const fns[4]) 909 { 910 int max; 911 912 if (a->esz < 0) { 913 /* Invalid tsz encoding -- see tszimm_esz. */ 914 return false; 915 } 916 917 /* 918 * Shift by element size is architecturally valid. 919 * For arithmetic right-shift, it's the same as by one less. 920 * For logical shifts and ASRD, it is a zeroing operation. 921 */ 922 max = 8 << a->esz; 923 if (a->imm >= max) { 924 if (asr) { 925 a->imm = max - 1; 926 } else { 927 return do_movz_zpz(s, a->rd, a->rd, a->pg, a->esz, true); 928 } 929 } 930 return gen_gvec_ool_arg_zpzi(s, fns[a->esz], a); 931 } 932 933 static gen_helper_gvec_3 * const asr_zpzi_fns[4] = { 934 gen_helper_sve_asr_zpzi_b, gen_helper_sve_asr_zpzi_h, 935 gen_helper_sve_asr_zpzi_s, gen_helper_sve_asr_zpzi_d, 936 }; 937 TRANS_FEAT(ASR_zpzi, aa64_sve, do_shift_zpzi, a, true, asr_zpzi_fns) 938 939 static gen_helper_gvec_3 * const lsr_zpzi_fns[4] = { 940 gen_helper_sve_lsr_zpzi_b, gen_helper_sve_lsr_zpzi_h, 941 gen_helper_sve_lsr_zpzi_s, gen_helper_sve_lsr_zpzi_d, 942 }; 943 TRANS_FEAT(LSR_zpzi, aa64_sve, do_shift_zpzi, a, false, lsr_zpzi_fns) 944 945 static gen_helper_gvec_3 * const lsl_zpzi_fns[4] = { 946 gen_helper_sve_lsl_zpzi_b, gen_helper_sve_lsl_zpzi_h, 947 gen_helper_sve_lsl_zpzi_s, gen_helper_sve_lsl_zpzi_d, 948 }; 949 TRANS_FEAT(LSL_zpzi, aa64_sve, do_shift_zpzi, a, false, lsl_zpzi_fns) 950 951 static gen_helper_gvec_3 * const asrd_fns[4] = { 952 gen_helper_sve_asrd_b, gen_helper_sve_asrd_h, 953 gen_helper_sve_asrd_s, gen_helper_sve_asrd_d, 954 }; 955 TRANS_FEAT(ASRD, aa64_sve, do_shift_zpzi, a, false, asrd_fns) 956 957 static gen_helper_gvec_3 * const sqshl_zpzi_fns[4] = { 958 gen_helper_sve2_sqshl_zpzi_b, gen_helper_sve2_sqshl_zpzi_h, 959 gen_helper_sve2_sqshl_zpzi_s, gen_helper_sve2_sqshl_zpzi_d, 960 }; 961 TRANS_FEAT(SQSHL_zpzi, aa64_sve2, gen_gvec_ool_arg_zpzi, 962 a->esz < 0 ? NULL : sqshl_zpzi_fns[a->esz], a) 963 964 static gen_helper_gvec_3 * const uqshl_zpzi_fns[4] = { 965 gen_helper_sve2_uqshl_zpzi_b, gen_helper_sve2_uqshl_zpzi_h, 966 gen_helper_sve2_uqshl_zpzi_s, gen_helper_sve2_uqshl_zpzi_d, 967 }; 968 TRANS_FEAT(UQSHL_zpzi, aa64_sve2, gen_gvec_ool_arg_zpzi, 969 a->esz < 0 ? NULL : uqshl_zpzi_fns[a->esz], a) 970 971 static gen_helper_gvec_3 * const srshr_fns[4] = { 972 gen_helper_sve2_srshr_b, gen_helper_sve2_srshr_h, 973 gen_helper_sve2_srshr_s, gen_helper_sve2_srshr_d, 974 }; 975 TRANS_FEAT(SRSHR, aa64_sve2, gen_gvec_ool_arg_zpzi, 976 a->esz < 0 ? NULL : srshr_fns[a->esz], a) 977 978 static gen_helper_gvec_3 * const urshr_fns[4] = { 979 gen_helper_sve2_urshr_b, gen_helper_sve2_urshr_h, 980 gen_helper_sve2_urshr_s, gen_helper_sve2_urshr_d, 981 }; 982 TRANS_FEAT(URSHR, aa64_sve2, gen_gvec_ool_arg_zpzi, 983 a->esz < 0 ? NULL : urshr_fns[a->esz], a) 984 985 static gen_helper_gvec_3 * const sqshlu_fns[4] = { 986 gen_helper_sve2_sqshlu_b, gen_helper_sve2_sqshlu_h, 987 gen_helper_sve2_sqshlu_s, gen_helper_sve2_sqshlu_d, 988 }; 989 TRANS_FEAT(SQSHLU, aa64_sve2, gen_gvec_ool_arg_zpzi, 990 a->esz < 0 ? NULL : sqshlu_fns[a->esz], a) 991 992 /* 993 *** SVE Bitwise Shift - Predicated Group 994 */ 995 996 #define DO_ZPZW(NAME, name) \ 997 static gen_helper_gvec_4 * const name##_zpzw_fns[4] = { \ 998 gen_helper_sve_##name##_zpzw_b, gen_helper_sve_##name##_zpzw_h, \ 999 gen_helper_sve_##name##_zpzw_s, NULL \ 1000 }; \ 1001 TRANS_FEAT(NAME##_zpzw, aa64_sve, gen_gvec_ool_arg_zpzz, \ 1002 a->esz < 0 ? NULL : name##_zpzw_fns[a->esz], a, 0) 1003 1004 DO_ZPZW(ASR, asr) 1005 DO_ZPZW(LSR, lsr) 1006 DO_ZPZW(LSL, lsl) 1007 1008 #undef DO_ZPZW 1009 1010 /* 1011 *** SVE Bitwise Shift - Unpredicated Group 1012 */ 1013 1014 static bool do_shift_imm(DisasContext *s, arg_rri_esz *a, bool asr, 1015 void (*gvec_fn)(unsigned, uint32_t, uint32_t, 1016 int64_t, uint32_t, uint32_t)) 1017 { 1018 if (a->esz < 0) { 1019 /* Invalid tsz encoding -- see tszimm_esz. */ 1020 return false; 1021 } 1022 if (sve_access_check(s)) { 1023 unsigned vsz = vec_full_reg_size(s); 1024 /* Shift by element size is architecturally valid. For 1025 arithmetic right-shift, it's the same as by one less. 1026 Otherwise it is a zeroing operation. */ 1027 if (a->imm >= 8 << a->esz) { 1028 if (asr) { 1029 a->imm = (8 << a->esz) - 1; 1030 } else { 1031 do_dupi_z(s, a->rd, 0); 1032 return true; 1033 } 1034 } 1035 gvec_fn(a->esz, vec_full_reg_offset(s, a->rd), 1036 vec_full_reg_offset(s, a->rn), a->imm, vsz, vsz); 1037 } 1038 return true; 1039 } 1040 1041 TRANS_FEAT(ASR_zzi, aa64_sve, do_shift_imm, a, true, tcg_gen_gvec_sari) 1042 TRANS_FEAT(LSR_zzi, aa64_sve, do_shift_imm, a, false, tcg_gen_gvec_shri) 1043 TRANS_FEAT(LSL_zzi, aa64_sve, do_shift_imm, a, false, tcg_gen_gvec_shli) 1044 1045 #define DO_ZZW(NAME, name) \ 1046 static gen_helper_gvec_3 * const name##_zzw_fns[4] = { \ 1047 gen_helper_sve_##name##_zzw_b, gen_helper_sve_##name##_zzw_h, \ 1048 gen_helper_sve_##name##_zzw_s, NULL \ 1049 }; \ 1050 TRANS_FEAT(NAME, aa64_sve, gen_gvec_ool_arg_zzz, \ 1051 name##_zzw_fns[a->esz], a, 0) 1052 1053 DO_ZZW(ASR_zzw, asr) 1054 DO_ZZW(LSR_zzw, lsr) 1055 DO_ZZW(LSL_zzw, lsl) 1056 1057 #undef DO_ZZW 1058 1059 /* 1060 *** SVE Integer Multiply-Add Group 1061 */ 1062 1063 static bool do_zpzzz_ool(DisasContext *s, arg_rprrr_esz *a, 1064 gen_helper_gvec_5 *fn) 1065 { 1066 if (sve_access_check(s)) { 1067 unsigned vsz = vec_full_reg_size(s); 1068 tcg_gen_gvec_5_ool(vec_full_reg_offset(s, a->rd), 1069 vec_full_reg_offset(s, a->ra), 1070 vec_full_reg_offset(s, a->rn), 1071 vec_full_reg_offset(s, a->rm), 1072 pred_full_reg_offset(s, a->pg), 1073 vsz, vsz, 0, fn); 1074 } 1075 return true; 1076 } 1077 1078 static gen_helper_gvec_5 * const mla_fns[4] = { 1079 gen_helper_sve_mla_b, gen_helper_sve_mla_h, 1080 gen_helper_sve_mla_s, gen_helper_sve_mla_d, 1081 }; 1082 TRANS_FEAT(MLA, aa64_sve, do_zpzzz_ool, a, mla_fns[a->esz]) 1083 1084 static gen_helper_gvec_5 * const mls_fns[4] = { 1085 gen_helper_sve_mls_b, gen_helper_sve_mls_h, 1086 gen_helper_sve_mls_s, gen_helper_sve_mls_d, 1087 }; 1088 TRANS_FEAT(MLS, aa64_sve, do_zpzzz_ool, a, mls_fns[a->esz]) 1089 1090 /* 1091 *** SVE Index Generation Group 1092 */ 1093 1094 static bool do_index(DisasContext *s, int esz, int rd, 1095 TCGv_i64 start, TCGv_i64 incr) 1096 { 1097 unsigned vsz; 1098 TCGv_i32 desc; 1099 TCGv_ptr t_zd; 1100 1101 if (!sve_access_check(s)) { 1102 return true; 1103 } 1104 1105 vsz = vec_full_reg_size(s); 1106 desc = tcg_constant_i32(simd_desc(vsz, vsz, 0)); 1107 t_zd = tcg_temp_new_ptr(); 1108 1109 tcg_gen_addi_ptr(t_zd, tcg_env, vec_full_reg_offset(s, rd)); 1110 if (esz == 3) { 1111 gen_helper_sve_index_d(t_zd, start, incr, desc); 1112 } else { 1113 typedef void index_fn(TCGv_ptr, TCGv_i32, TCGv_i32, TCGv_i32); 1114 static index_fn * const fns[3] = { 1115 gen_helper_sve_index_b, 1116 gen_helper_sve_index_h, 1117 gen_helper_sve_index_s, 1118 }; 1119 TCGv_i32 s32 = tcg_temp_new_i32(); 1120 TCGv_i32 i32 = tcg_temp_new_i32(); 1121 1122 tcg_gen_extrl_i64_i32(s32, start); 1123 tcg_gen_extrl_i64_i32(i32, incr); 1124 fns[esz](t_zd, s32, i32, desc); 1125 } 1126 return true; 1127 } 1128 1129 TRANS_FEAT(INDEX_ii, aa64_sve, do_index, a->esz, a->rd, 1130 tcg_constant_i64(a->imm1), tcg_constant_i64(a->imm2)) 1131 TRANS_FEAT(INDEX_ir, aa64_sve, do_index, a->esz, a->rd, 1132 tcg_constant_i64(a->imm), cpu_reg(s, a->rm)) 1133 TRANS_FEAT(INDEX_ri, aa64_sve, do_index, a->esz, a->rd, 1134 cpu_reg(s, a->rn), tcg_constant_i64(a->imm)) 1135 TRANS_FEAT(INDEX_rr, aa64_sve, do_index, a->esz, a->rd, 1136 cpu_reg(s, a->rn), cpu_reg(s, a->rm)) 1137 1138 /* 1139 *** SVE Stack Allocation Group 1140 */ 1141 1142 static bool trans_ADDVL(DisasContext *s, arg_ADDVL *a) 1143 { 1144 if (!dc_isar_feature(aa64_sve, s)) { 1145 return false; 1146 } 1147 if (sve_access_check(s)) { 1148 TCGv_i64 rd = cpu_reg_sp(s, a->rd); 1149 TCGv_i64 rn = cpu_reg_sp(s, a->rn); 1150 tcg_gen_addi_i64(rd, rn, a->imm * vec_full_reg_size(s)); 1151 } 1152 return true; 1153 } 1154 1155 static bool trans_ADDSVL(DisasContext *s, arg_ADDSVL *a) 1156 { 1157 if (!dc_isar_feature(aa64_sme, s)) { 1158 return false; 1159 } 1160 if (sme_enabled_check(s)) { 1161 TCGv_i64 rd = cpu_reg_sp(s, a->rd); 1162 TCGv_i64 rn = cpu_reg_sp(s, a->rn); 1163 tcg_gen_addi_i64(rd, rn, a->imm * streaming_vec_reg_size(s)); 1164 } 1165 return true; 1166 } 1167 1168 static bool trans_ADDPL(DisasContext *s, arg_ADDPL *a) 1169 { 1170 if (!dc_isar_feature(aa64_sve, s)) { 1171 return false; 1172 } 1173 if (sve_access_check(s)) { 1174 TCGv_i64 rd = cpu_reg_sp(s, a->rd); 1175 TCGv_i64 rn = cpu_reg_sp(s, a->rn); 1176 tcg_gen_addi_i64(rd, rn, a->imm * pred_full_reg_size(s)); 1177 } 1178 return true; 1179 } 1180 1181 static bool trans_ADDSPL(DisasContext *s, arg_ADDSPL *a) 1182 { 1183 if (!dc_isar_feature(aa64_sme, s)) { 1184 return false; 1185 } 1186 if (sme_enabled_check(s)) { 1187 TCGv_i64 rd = cpu_reg_sp(s, a->rd); 1188 TCGv_i64 rn = cpu_reg_sp(s, a->rn); 1189 tcg_gen_addi_i64(rd, rn, a->imm * streaming_pred_reg_size(s)); 1190 } 1191 return true; 1192 } 1193 1194 static bool trans_RDVL(DisasContext *s, arg_RDVL *a) 1195 { 1196 if (!dc_isar_feature(aa64_sve, s)) { 1197 return false; 1198 } 1199 if (sve_access_check(s)) { 1200 TCGv_i64 reg = cpu_reg(s, a->rd); 1201 tcg_gen_movi_i64(reg, a->imm * vec_full_reg_size(s)); 1202 } 1203 return true; 1204 } 1205 1206 static bool trans_RDSVL(DisasContext *s, arg_RDSVL *a) 1207 { 1208 if (!dc_isar_feature(aa64_sme, s)) { 1209 return false; 1210 } 1211 if (sme_enabled_check(s)) { 1212 TCGv_i64 reg = cpu_reg(s, a->rd); 1213 tcg_gen_movi_i64(reg, a->imm * streaming_vec_reg_size(s)); 1214 } 1215 return true; 1216 } 1217 1218 /* 1219 *** SVE Compute Vector Address Group 1220 */ 1221 1222 static bool do_adr(DisasContext *s, arg_rrri *a, gen_helper_gvec_3 *fn) 1223 { 1224 return gen_gvec_ool_zzz(s, fn, a->rd, a->rn, a->rm, a->imm); 1225 } 1226 1227 TRANS_FEAT_NONSTREAMING(ADR_p32, aa64_sve, do_adr, a, gen_helper_sve_adr_p32) 1228 TRANS_FEAT_NONSTREAMING(ADR_p64, aa64_sve, do_adr, a, gen_helper_sve_adr_p64) 1229 TRANS_FEAT_NONSTREAMING(ADR_s32, aa64_sve, do_adr, a, gen_helper_sve_adr_s32) 1230 TRANS_FEAT_NONSTREAMING(ADR_u32, aa64_sve, do_adr, a, gen_helper_sve_adr_u32) 1231 1232 /* 1233 *** SVE Integer Misc - Unpredicated Group 1234 */ 1235 1236 static gen_helper_gvec_2 * const fexpa_fns[4] = { 1237 NULL, gen_helper_sve_fexpa_h, 1238 gen_helper_sve_fexpa_s, gen_helper_sve_fexpa_d, 1239 }; 1240 TRANS_FEAT_NONSTREAMING(FEXPA, aa64_sve, gen_gvec_ool_zz, 1241 fexpa_fns[a->esz], a->rd, a->rn, s->fpcr_ah) 1242 1243 static gen_helper_gvec_3 * const ftssel_fns[4] = { 1244 NULL, gen_helper_sve_ftssel_h, 1245 gen_helper_sve_ftssel_s, gen_helper_sve_ftssel_d, 1246 }; 1247 TRANS_FEAT_NONSTREAMING(FTSSEL, aa64_sve, gen_gvec_ool_arg_zzz, 1248 ftssel_fns[a->esz], a, s->fpcr_ah) 1249 1250 /* 1251 *** SVE Predicate Logical Operations Group 1252 */ 1253 1254 static bool do_pppp_flags(DisasContext *s, arg_rprr_s *a, 1255 const GVecGen4 *gvec_op) 1256 { 1257 if (!sve_access_check(s)) { 1258 return true; 1259 } 1260 1261 unsigned psz = pred_gvec_reg_size(s); 1262 int dofs = pred_full_reg_offset(s, a->rd); 1263 int nofs = pred_full_reg_offset(s, a->rn); 1264 int mofs = pred_full_reg_offset(s, a->rm); 1265 int gofs = pred_full_reg_offset(s, a->pg); 1266 1267 if (!a->s) { 1268 tcg_gen_gvec_4(dofs, nofs, mofs, gofs, psz, psz, gvec_op); 1269 return true; 1270 } 1271 1272 if (psz == 8) { 1273 /* Do the operation and the flags generation in temps. */ 1274 TCGv_i64 pd = tcg_temp_new_i64(); 1275 TCGv_i64 pn = tcg_temp_new_i64(); 1276 TCGv_i64 pm = tcg_temp_new_i64(); 1277 TCGv_i64 pg = tcg_temp_new_i64(); 1278 1279 tcg_gen_ld_i64(pn, tcg_env, nofs); 1280 tcg_gen_ld_i64(pm, tcg_env, mofs); 1281 tcg_gen_ld_i64(pg, tcg_env, gofs); 1282 1283 gvec_op->fni8(pd, pn, pm, pg); 1284 tcg_gen_st_i64(pd, tcg_env, dofs); 1285 1286 do_predtest1(pd, pg); 1287 } else { 1288 /* The operation and flags generation is large. The computation 1289 * of the flags depends on the original contents of the guarding 1290 * predicate. If the destination overwrites the guarding predicate, 1291 * then the easiest way to get this right is to save a copy. 1292 */ 1293 int tofs = gofs; 1294 if (a->rd == a->pg) { 1295 tofs = offsetof(CPUARMState, vfp.preg_tmp); 1296 tcg_gen_gvec_mov(0, tofs, gofs, psz, psz); 1297 } 1298 1299 tcg_gen_gvec_4(dofs, nofs, mofs, gofs, psz, psz, gvec_op); 1300 do_predtest(s, dofs, tofs, psz / 8); 1301 } 1302 return true; 1303 } 1304 1305 static void gen_and_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg) 1306 { 1307 tcg_gen_and_i64(pd, pn, pm); 1308 tcg_gen_and_i64(pd, pd, pg); 1309 } 1310 1311 static void gen_and_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn, 1312 TCGv_vec pm, TCGv_vec pg) 1313 { 1314 tcg_gen_and_vec(vece, pd, pn, pm); 1315 tcg_gen_and_vec(vece, pd, pd, pg); 1316 } 1317 1318 static bool trans_AND_pppp(DisasContext *s, arg_rprr_s *a) 1319 { 1320 static const GVecGen4 op = { 1321 .fni8 = gen_and_pg_i64, 1322 .fniv = gen_and_pg_vec, 1323 .fno = gen_helper_sve_and_pppp, 1324 .prefer_i64 = TCG_TARGET_REG_BITS == 64, 1325 }; 1326 1327 if (!dc_isar_feature(aa64_sve, s)) { 1328 return false; 1329 } 1330 if (!a->s) { 1331 if (a->rn == a->rm) { 1332 if (a->pg == a->rn) { 1333 return do_mov_p(s, a->rd, a->rn); 1334 } 1335 return gen_gvec_fn_ppp(s, tcg_gen_gvec_and, a->rd, a->rn, a->pg); 1336 } else if (a->pg == a->rn || a->pg == a->rm) { 1337 return gen_gvec_fn_ppp(s, tcg_gen_gvec_and, a->rd, a->rn, a->rm); 1338 } 1339 } 1340 return do_pppp_flags(s, a, &op); 1341 } 1342 1343 static void gen_bic_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg) 1344 { 1345 tcg_gen_andc_i64(pd, pn, pm); 1346 tcg_gen_and_i64(pd, pd, pg); 1347 } 1348 1349 static void gen_bic_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn, 1350 TCGv_vec pm, TCGv_vec pg) 1351 { 1352 tcg_gen_andc_vec(vece, pd, pn, pm); 1353 tcg_gen_and_vec(vece, pd, pd, pg); 1354 } 1355 1356 static bool trans_BIC_pppp(DisasContext *s, arg_rprr_s *a) 1357 { 1358 static const GVecGen4 op = { 1359 .fni8 = gen_bic_pg_i64, 1360 .fniv = gen_bic_pg_vec, 1361 .fno = gen_helper_sve_bic_pppp, 1362 .prefer_i64 = TCG_TARGET_REG_BITS == 64, 1363 }; 1364 1365 if (!dc_isar_feature(aa64_sve, s)) { 1366 return false; 1367 } 1368 if (!a->s && a->pg == a->rn) { 1369 return gen_gvec_fn_ppp(s, tcg_gen_gvec_andc, a->rd, a->rn, a->rm); 1370 } 1371 return do_pppp_flags(s, a, &op); 1372 } 1373 1374 static void gen_eor_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg) 1375 { 1376 tcg_gen_xor_i64(pd, pn, pm); 1377 tcg_gen_and_i64(pd, pd, pg); 1378 } 1379 1380 static void gen_eor_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn, 1381 TCGv_vec pm, TCGv_vec pg) 1382 { 1383 tcg_gen_xor_vec(vece, pd, pn, pm); 1384 tcg_gen_and_vec(vece, pd, pd, pg); 1385 } 1386 1387 static bool trans_EOR_pppp(DisasContext *s, arg_rprr_s *a) 1388 { 1389 static const GVecGen4 op = { 1390 .fni8 = gen_eor_pg_i64, 1391 .fniv = gen_eor_pg_vec, 1392 .fno = gen_helper_sve_eor_pppp, 1393 .prefer_i64 = TCG_TARGET_REG_BITS == 64, 1394 }; 1395 1396 if (!dc_isar_feature(aa64_sve, s)) { 1397 return false; 1398 } 1399 /* Alias NOT (predicate) is EOR Pd.B, Pg/Z, Pn.B, Pg.B */ 1400 if (!a->s && a->pg == a->rm) { 1401 return gen_gvec_fn_ppp(s, tcg_gen_gvec_andc, a->rd, a->pg, a->rn); 1402 } 1403 return do_pppp_flags(s, a, &op); 1404 } 1405 1406 static bool trans_SEL_pppp(DisasContext *s, arg_rprr_s *a) 1407 { 1408 if (a->s || !dc_isar_feature(aa64_sve, s)) { 1409 return false; 1410 } 1411 if (sve_access_check(s)) { 1412 unsigned psz = pred_gvec_reg_size(s); 1413 tcg_gen_gvec_bitsel(MO_8, pred_full_reg_offset(s, a->rd), 1414 pred_full_reg_offset(s, a->pg), 1415 pred_full_reg_offset(s, a->rn), 1416 pred_full_reg_offset(s, a->rm), psz, psz); 1417 } 1418 return true; 1419 } 1420 1421 static void gen_orr_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg) 1422 { 1423 tcg_gen_or_i64(pd, pn, pm); 1424 tcg_gen_and_i64(pd, pd, pg); 1425 } 1426 1427 static void gen_orr_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn, 1428 TCGv_vec pm, TCGv_vec pg) 1429 { 1430 tcg_gen_or_vec(vece, pd, pn, pm); 1431 tcg_gen_and_vec(vece, pd, pd, pg); 1432 } 1433 1434 static bool trans_ORR_pppp(DisasContext *s, arg_rprr_s *a) 1435 { 1436 static const GVecGen4 op = { 1437 .fni8 = gen_orr_pg_i64, 1438 .fniv = gen_orr_pg_vec, 1439 .fno = gen_helper_sve_orr_pppp, 1440 .prefer_i64 = TCG_TARGET_REG_BITS == 64, 1441 }; 1442 1443 if (!dc_isar_feature(aa64_sve, s)) { 1444 return false; 1445 } 1446 if (!a->s && a->pg == a->rn && a->rn == a->rm) { 1447 return do_mov_p(s, a->rd, a->rn); 1448 } 1449 return do_pppp_flags(s, a, &op); 1450 } 1451 1452 static void gen_orn_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg) 1453 { 1454 tcg_gen_orc_i64(pd, pn, pm); 1455 tcg_gen_and_i64(pd, pd, pg); 1456 } 1457 1458 static void gen_orn_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn, 1459 TCGv_vec pm, TCGv_vec pg) 1460 { 1461 tcg_gen_orc_vec(vece, pd, pn, pm); 1462 tcg_gen_and_vec(vece, pd, pd, pg); 1463 } 1464 1465 static bool trans_ORN_pppp(DisasContext *s, arg_rprr_s *a) 1466 { 1467 static const GVecGen4 op = { 1468 .fni8 = gen_orn_pg_i64, 1469 .fniv = gen_orn_pg_vec, 1470 .fno = gen_helper_sve_orn_pppp, 1471 .prefer_i64 = TCG_TARGET_REG_BITS == 64, 1472 }; 1473 1474 if (!dc_isar_feature(aa64_sve, s)) { 1475 return false; 1476 } 1477 return do_pppp_flags(s, a, &op); 1478 } 1479 1480 static void gen_nor_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg) 1481 { 1482 tcg_gen_or_i64(pd, pn, pm); 1483 tcg_gen_andc_i64(pd, pg, pd); 1484 } 1485 1486 static void gen_nor_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn, 1487 TCGv_vec pm, TCGv_vec pg) 1488 { 1489 tcg_gen_or_vec(vece, pd, pn, pm); 1490 tcg_gen_andc_vec(vece, pd, pg, pd); 1491 } 1492 1493 static bool trans_NOR_pppp(DisasContext *s, arg_rprr_s *a) 1494 { 1495 static const GVecGen4 op = { 1496 .fni8 = gen_nor_pg_i64, 1497 .fniv = gen_nor_pg_vec, 1498 .fno = gen_helper_sve_nor_pppp, 1499 .prefer_i64 = TCG_TARGET_REG_BITS == 64, 1500 }; 1501 1502 if (!dc_isar_feature(aa64_sve, s)) { 1503 return false; 1504 } 1505 return do_pppp_flags(s, a, &op); 1506 } 1507 1508 static void gen_nand_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg) 1509 { 1510 tcg_gen_and_i64(pd, pn, pm); 1511 tcg_gen_andc_i64(pd, pg, pd); 1512 } 1513 1514 static void gen_nand_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn, 1515 TCGv_vec pm, TCGv_vec pg) 1516 { 1517 tcg_gen_and_vec(vece, pd, pn, pm); 1518 tcg_gen_andc_vec(vece, pd, pg, pd); 1519 } 1520 1521 static bool trans_NAND_pppp(DisasContext *s, arg_rprr_s *a) 1522 { 1523 static const GVecGen4 op = { 1524 .fni8 = gen_nand_pg_i64, 1525 .fniv = gen_nand_pg_vec, 1526 .fno = gen_helper_sve_nand_pppp, 1527 .prefer_i64 = TCG_TARGET_REG_BITS == 64, 1528 }; 1529 1530 if (!dc_isar_feature(aa64_sve, s)) { 1531 return false; 1532 } 1533 return do_pppp_flags(s, a, &op); 1534 } 1535 1536 /* 1537 *** SVE Predicate Misc Group 1538 */ 1539 1540 static bool trans_PTEST(DisasContext *s, arg_PTEST *a) 1541 { 1542 if (!dc_isar_feature(aa64_sve, s)) { 1543 return false; 1544 } 1545 if (sve_access_check(s)) { 1546 int nofs = pred_full_reg_offset(s, a->rn); 1547 int gofs = pred_full_reg_offset(s, a->pg); 1548 int words = DIV_ROUND_UP(pred_full_reg_size(s), 8); 1549 1550 if (words == 1) { 1551 TCGv_i64 pn = tcg_temp_new_i64(); 1552 TCGv_i64 pg = tcg_temp_new_i64(); 1553 1554 tcg_gen_ld_i64(pn, tcg_env, nofs); 1555 tcg_gen_ld_i64(pg, tcg_env, gofs); 1556 do_predtest1(pn, pg); 1557 } else { 1558 do_predtest(s, nofs, gofs, words); 1559 } 1560 } 1561 return true; 1562 } 1563 1564 /* See the ARM pseudocode DecodePredCount. */ 1565 static unsigned decode_pred_count(unsigned fullsz, int pattern, int esz) 1566 { 1567 unsigned elements = fullsz >> esz; 1568 unsigned bound; 1569 1570 switch (pattern) { 1571 case 0x0: /* POW2 */ 1572 return pow2floor(elements); 1573 case 0x1: /* VL1 */ 1574 case 0x2: /* VL2 */ 1575 case 0x3: /* VL3 */ 1576 case 0x4: /* VL4 */ 1577 case 0x5: /* VL5 */ 1578 case 0x6: /* VL6 */ 1579 case 0x7: /* VL7 */ 1580 case 0x8: /* VL8 */ 1581 bound = pattern; 1582 break; 1583 case 0x9: /* VL16 */ 1584 case 0xa: /* VL32 */ 1585 case 0xb: /* VL64 */ 1586 case 0xc: /* VL128 */ 1587 case 0xd: /* VL256 */ 1588 bound = 16 << (pattern - 9); 1589 break; 1590 case 0x1d: /* MUL4 */ 1591 return elements - elements % 4; 1592 case 0x1e: /* MUL3 */ 1593 return elements - elements % 3; 1594 case 0x1f: /* ALL */ 1595 return elements; 1596 default: /* #uimm5 */ 1597 return 0; 1598 } 1599 return elements >= bound ? bound : 0; 1600 } 1601 1602 /* This handles all of the predicate initialization instructions, 1603 * PTRUE, PFALSE, SETFFR. For PFALSE, we will have set PAT == 32 1604 * so that decode_pred_count returns 0. For SETFFR, we will have 1605 * set RD == 16 == FFR. 1606 */ 1607 static bool do_predset(DisasContext *s, int esz, int rd, int pat, bool setflag) 1608 { 1609 if (!sve_access_check(s)) { 1610 return true; 1611 } 1612 1613 unsigned fullsz = vec_full_reg_size(s); 1614 unsigned ofs = pred_full_reg_offset(s, rd); 1615 unsigned numelem, setsz, i; 1616 uint64_t word, lastword; 1617 TCGv_i64 t; 1618 1619 numelem = decode_pred_count(fullsz, pat, esz); 1620 1621 /* Determine what we must store into each bit, and how many. */ 1622 if (numelem == 0) { 1623 lastword = word = 0; 1624 setsz = fullsz; 1625 } else { 1626 setsz = numelem << esz; 1627 lastword = word = pred_esz_masks[esz]; 1628 if (setsz % 64) { 1629 lastword &= MAKE_64BIT_MASK(0, setsz % 64); 1630 } 1631 } 1632 1633 t = tcg_temp_new_i64(); 1634 if (fullsz <= 64) { 1635 tcg_gen_movi_i64(t, lastword); 1636 tcg_gen_st_i64(t, tcg_env, ofs); 1637 goto done; 1638 } 1639 1640 if (word == lastword) { 1641 unsigned maxsz = size_for_gvec(fullsz / 8); 1642 unsigned oprsz = size_for_gvec(setsz / 8); 1643 1644 if (oprsz * 8 == setsz) { 1645 tcg_gen_gvec_dup_imm(MO_64, ofs, oprsz, maxsz, word); 1646 goto done; 1647 } 1648 } 1649 1650 setsz /= 8; 1651 fullsz /= 8; 1652 1653 tcg_gen_movi_i64(t, word); 1654 for (i = 0; i < QEMU_ALIGN_DOWN(setsz, 8); i += 8) { 1655 tcg_gen_st_i64(t, tcg_env, ofs + i); 1656 } 1657 if (lastword != word) { 1658 tcg_gen_movi_i64(t, lastword); 1659 tcg_gen_st_i64(t, tcg_env, ofs + i); 1660 i += 8; 1661 } 1662 if (i < fullsz) { 1663 tcg_gen_movi_i64(t, 0); 1664 for (; i < fullsz; i += 8) { 1665 tcg_gen_st_i64(t, tcg_env, ofs + i); 1666 } 1667 } 1668 1669 done: 1670 /* PTRUES */ 1671 if (setflag) { 1672 tcg_gen_movi_i32(cpu_NF, -(word != 0)); 1673 tcg_gen_movi_i32(cpu_CF, word == 0); 1674 tcg_gen_movi_i32(cpu_VF, 0); 1675 tcg_gen_mov_i32(cpu_ZF, cpu_NF); 1676 } 1677 return true; 1678 } 1679 1680 TRANS_FEAT(PTRUE, aa64_sve, do_predset, a->esz, a->rd, a->pat, a->s) 1681 1682 /* Note pat == 31 is #all, to set all elements. */ 1683 TRANS_FEAT_NONSTREAMING(SETFFR, aa64_sve, 1684 do_predset, 0, FFR_PRED_NUM, 31, false) 1685 1686 /* Note pat == 32 is #unimp, to set no elements. */ 1687 TRANS_FEAT(PFALSE, aa64_sve, do_predset, 0, a->rd, 32, false) 1688 1689 static bool trans_RDFFR_p(DisasContext *s, arg_RDFFR_p *a) 1690 { 1691 /* The path through do_pppp_flags is complicated enough to want to avoid 1692 * duplication. Frob the arguments into the form of a predicated AND. 1693 */ 1694 arg_rprr_s alt_a = { 1695 .rd = a->rd, .pg = a->pg, .s = a->s, 1696 .rn = FFR_PRED_NUM, .rm = FFR_PRED_NUM, 1697 }; 1698 1699 s->is_nonstreaming = true; 1700 return trans_AND_pppp(s, &alt_a); 1701 } 1702 1703 TRANS_FEAT_NONSTREAMING(RDFFR, aa64_sve, do_mov_p, a->rd, FFR_PRED_NUM) 1704 TRANS_FEAT_NONSTREAMING(WRFFR, aa64_sve, do_mov_p, FFR_PRED_NUM, a->rn) 1705 1706 static bool do_pfirst_pnext(DisasContext *s, arg_rr_esz *a, 1707 void (*gen_fn)(TCGv_i32, TCGv_ptr, 1708 TCGv_ptr, TCGv_i32)) 1709 { 1710 if (!sve_access_check(s)) { 1711 return true; 1712 } 1713 1714 TCGv_ptr t_pd = tcg_temp_new_ptr(); 1715 TCGv_ptr t_pg = tcg_temp_new_ptr(); 1716 TCGv_i32 t; 1717 unsigned desc = 0; 1718 1719 desc = FIELD_DP32(desc, PREDDESC, OPRSZ, pred_full_reg_size(s)); 1720 desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz); 1721 1722 tcg_gen_addi_ptr(t_pd, tcg_env, pred_full_reg_offset(s, a->rd)); 1723 tcg_gen_addi_ptr(t_pg, tcg_env, pred_full_reg_offset(s, a->rn)); 1724 t = tcg_temp_new_i32(); 1725 1726 gen_fn(t, t_pd, t_pg, tcg_constant_i32(desc)); 1727 1728 do_pred_flags(t); 1729 return true; 1730 } 1731 1732 TRANS_FEAT(PFIRST, aa64_sve, do_pfirst_pnext, a, gen_helper_sve_pfirst) 1733 TRANS_FEAT(PNEXT, aa64_sve, do_pfirst_pnext, a, gen_helper_sve_pnext) 1734 1735 /* 1736 *** SVE Element Count Group 1737 */ 1738 1739 /* Perform an inline saturating addition of a 32-bit value within 1740 * a 64-bit register. The second operand is known to be positive, 1741 * which halves the comparisons we must perform to bound the result. 1742 */ 1743 static void do_sat_addsub_32(TCGv_i64 reg, TCGv_i64 val, bool u, bool d) 1744 { 1745 int64_t ibound; 1746 1747 /* Use normal 64-bit arithmetic to detect 32-bit overflow. */ 1748 if (u) { 1749 tcg_gen_ext32u_i64(reg, reg); 1750 } else { 1751 tcg_gen_ext32s_i64(reg, reg); 1752 } 1753 if (d) { 1754 tcg_gen_sub_i64(reg, reg, val); 1755 ibound = (u ? 0 : INT32_MIN); 1756 tcg_gen_smax_i64(reg, reg, tcg_constant_i64(ibound)); 1757 } else { 1758 tcg_gen_add_i64(reg, reg, val); 1759 ibound = (u ? UINT32_MAX : INT32_MAX); 1760 tcg_gen_smin_i64(reg, reg, tcg_constant_i64(ibound)); 1761 } 1762 } 1763 1764 /* Similarly with 64-bit values. */ 1765 static void do_sat_addsub_64(TCGv_i64 reg, TCGv_i64 val, bool u, bool d) 1766 { 1767 TCGv_i64 t0 = tcg_temp_new_i64(); 1768 TCGv_i64 t2; 1769 1770 if (u) { 1771 if (d) { 1772 tcg_gen_sub_i64(t0, reg, val); 1773 t2 = tcg_constant_i64(0); 1774 tcg_gen_movcond_i64(TCG_COND_LTU, reg, reg, val, t2, t0); 1775 } else { 1776 tcg_gen_add_i64(t0, reg, val); 1777 t2 = tcg_constant_i64(-1); 1778 tcg_gen_movcond_i64(TCG_COND_LTU, reg, t0, reg, t2, t0); 1779 } 1780 } else { 1781 TCGv_i64 t1 = tcg_temp_new_i64(); 1782 if (d) { 1783 /* Detect signed overflow for subtraction. */ 1784 tcg_gen_xor_i64(t0, reg, val); 1785 tcg_gen_sub_i64(t1, reg, val); 1786 tcg_gen_xor_i64(reg, reg, t1); 1787 tcg_gen_and_i64(t0, t0, reg); 1788 1789 /* Bound the result. */ 1790 tcg_gen_movi_i64(reg, INT64_MIN); 1791 t2 = tcg_constant_i64(0); 1792 tcg_gen_movcond_i64(TCG_COND_LT, reg, t0, t2, reg, t1); 1793 } else { 1794 /* Detect signed overflow for addition. */ 1795 tcg_gen_xor_i64(t0, reg, val); 1796 tcg_gen_add_i64(reg, reg, val); 1797 tcg_gen_xor_i64(t1, reg, val); 1798 tcg_gen_andc_i64(t0, t1, t0); 1799 1800 /* Bound the result. */ 1801 tcg_gen_movi_i64(t1, INT64_MAX); 1802 t2 = tcg_constant_i64(0); 1803 tcg_gen_movcond_i64(TCG_COND_LT, reg, t0, t2, t1, reg); 1804 } 1805 } 1806 } 1807 1808 /* Similarly with a vector and a scalar operand. */ 1809 static void do_sat_addsub_vec(DisasContext *s, int esz, int rd, int rn, 1810 TCGv_i64 val, bool u, bool d) 1811 { 1812 unsigned vsz = vec_full_reg_size(s); 1813 TCGv_ptr dptr, nptr; 1814 TCGv_i32 t32, desc; 1815 TCGv_i64 t64; 1816 1817 dptr = tcg_temp_new_ptr(); 1818 nptr = tcg_temp_new_ptr(); 1819 tcg_gen_addi_ptr(dptr, tcg_env, vec_full_reg_offset(s, rd)); 1820 tcg_gen_addi_ptr(nptr, tcg_env, vec_full_reg_offset(s, rn)); 1821 desc = tcg_constant_i32(simd_desc(vsz, vsz, 0)); 1822 1823 switch (esz) { 1824 case MO_8: 1825 t32 = tcg_temp_new_i32(); 1826 tcg_gen_extrl_i64_i32(t32, val); 1827 if (d) { 1828 tcg_gen_neg_i32(t32, t32); 1829 } 1830 if (u) { 1831 gen_helper_sve_uqaddi_b(dptr, nptr, t32, desc); 1832 } else { 1833 gen_helper_sve_sqaddi_b(dptr, nptr, t32, desc); 1834 } 1835 break; 1836 1837 case MO_16: 1838 t32 = tcg_temp_new_i32(); 1839 tcg_gen_extrl_i64_i32(t32, val); 1840 if (d) { 1841 tcg_gen_neg_i32(t32, t32); 1842 } 1843 if (u) { 1844 gen_helper_sve_uqaddi_h(dptr, nptr, t32, desc); 1845 } else { 1846 gen_helper_sve_sqaddi_h(dptr, nptr, t32, desc); 1847 } 1848 break; 1849 1850 case MO_32: 1851 t64 = tcg_temp_new_i64(); 1852 if (d) { 1853 tcg_gen_neg_i64(t64, val); 1854 } else { 1855 tcg_gen_mov_i64(t64, val); 1856 } 1857 if (u) { 1858 gen_helper_sve_uqaddi_s(dptr, nptr, t64, desc); 1859 } else { 1860 gen_helper_sve_sqaddi_s(dptr, nptr, t64, desc); 1861 } 1862 break; 1863 1864 case MO_64: 1865 if (u) { 1866 if (d) { 1867 gen_helper_sve_uqsubi_d(dptr, nptr, val, desc); 1868 } else { 1869 gen_helper_sve_uqaddi_d(dptr, nptr, val, desc); 1870 } 1871 } else if (d) { 1872 t64 = tcg_temp_new_i64(); 1873 tcg_gen_neg_i64(t64, val); 1874 gen_helper_sve_sqaddi_d(dptr, nptr, t64, desc); 1875 } else { 1876 gen_helper_sve_sqaddi_d(dptr, nptr, val, desc); 1877 } 1878 break; 1879 1880 default: 1881 g_assert_not_reached(); 1882 } 1883 } 1884 1885 static bool trans_CNT_r(DisasContext *s, arg_CNT_r *a) 1886 { 1887 if (!dc_isar_feature(aa64_sve, s)) { 1888 return false; 1889 } 1890 if (sve_access_check(s)) { 1891 unsigned fullsz = vec_full_reg_size(s); 1892 unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz); 1893 tcg_gen_movi_i64(cpu_reg(s, a->rd), numelem * a->imm); 1894 } 1895 return true; 1896 } 1897 1898 static bool trans_INCDEC_r(DisasContext *s, arg_incdec_cnt *a) 1899 { 1900 if (!dc_isar_feature(aa64_sve, s)) { 1901 return false; 1902 } 1903 if (sve_access_check(s)) { 1904 unsigned fullsz = vec_full_reg_size(s); 1905 unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz); 1906 int inc = numelem * a->imm * (a->d ? -1 : 1); 1907 TCGv_i64 reg = cpu_reg(s, a->rd); 1908 1909 tcg_gen_addi_i64(reg, reg, inc); 1910 } 1911 return true; 1912 } 1913 1914 static bool trans_SINCDEC_r_32(DisasContext *s, arg_incdec_cnt *a) 1915 { 1916 if (!dc_isar_feature(aa64_sve, s)) { 1917 return false; 1918 } 1919 if (!sve_access_check(s)) { 1920 return true; 1921 } 1922 1923 unsigned fullsz = vec_full_reg_size(s); 1924 unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz); 1925 int inc = numelem * a->imm; 1926 TCGv_i64 reg = cpu_reg(s, a->rd); 1927 1928 /* Use normal 64-bit arithmetic to detect 32-bit overflow. */ 1929 if (inc == 0) { 1930 if (a->u) { 1931 tcg_gen_ext32u_i64(reg, reg); 1932 } else { 1933 tcg_gen_ext32s_i64(reg, reg); 1934 } 1935 } else { 1936 do_sat_addsub_32(reg, tcg_constant_i64(inc), a->u, a->d); 1937 } 1938 return true; 1939 } 1940 1941 static bool trans_SINCDEC_r_64(DisasContext *s, arg_incdec_cnt *a) 1942 { 1943 if (!dc_isar_feature(aa64_sve, s)) { 1944 return false; 1945 } 1946 if (!sve_access_check(s)) { 1947 return true; 1948 } 1949 1950 unsigned fullsz = vec_full_reg_size(s); 1951 unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz); 1952 int inc = numelem * a->imm; 1953 TCGv_i64 reg = cpu_reg(s, a->rd); 1954 1955 if (inc != 0) { 1956 do_sat_addsub_64(reg, tcg_constant_i64(inc), a->u, a->d); 1957 } 1958 return true; 1959 } 1960 1961 static bool trans_INCDEC_v(DisasContext *s, arg_incdec2_cnt *a) 1962 { 1963 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) { 1964 return false; 1965 } 1966 1967 unsigned fullsz = vec_full_reg_size(s); 1968 unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz); 1969 int inc = numelem * a->imm; 1970 1971 if (inc != 0) { 1972 if (sve_access_check(s)) { 1973 tcg_gen_gvec_adds(a->esz, vec_full_reg_offset(s, a->rd), 1974 vec_full_reg_offset(s, a->rn), 1975 tcg_constant_i64(a->d ? -inc : inc), 1976 fullsz, fullsz); 1977 } 1978 } else { 1979 do_mov_z(s, a->rd, a->rn); 1980 } 1981 return true; 1982 } 1983 1984 static bool trans_SINCDEC_v(DisasContext *s, arg_incdec2_cnt *a) 1985 { 1986 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) { 1987 return false; 1988 } 1989 1990 unsigned fullsz = vec_full_reg_size(s); 1991 unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz); 1992 int inc = numelem * a->imm; 1993 1994 if (inc != 0) { 1995 if (sve_access_check(s)) { 1996 do_sat_addsub_vec(s, a->esz, a->rd, a->rn, 1997 tcg_constant_i64(inc), a->u, a->d); 1998 } 1999 } else { 2000 do_mov_z(s, a->rd, a->rn); 2001 } 2002 return true; 2003 } 2004 2005 /* 2006 *** SVE Bitwise Immediate Group 2007 */ 2008 2009 static bool do_zz_dbm(DisasContext *s, arg_rr_dbm *a, GVecGen2iFn *gvec_fn) 2010 { 2011 uint64_t imm; 2012 if (!logic_imm_decode_wmask(&imm, extract32(a->dbm, 12, 1), 2013 extract32(a->dbm, 0, 6), 2014 extract32(a->dbm, 6, 6))) { 2015 return false; 2016 } 2017 return gen_gvec_fn_zzi(s, gvec_fn, MO_64, a->rd, a->rn, imm); 2018 } 2019 2020 TRANS_FEAT(AND_zzi, aa64_sve, do_zz_dbm, a, tcg_gen_gvec_andi) 2021 TRANS_FEAT(ORR_zzi, aa64_sve, do_zz_dbm, a, tcg_gen_gvec_ori) 2022 TRANS_FEAT(EOR_zzi, aa64_sve, do_zz_dbm, a, tcg_gen_gvec_xori) 2023 2024 static bool trans_DUPM(DisasContext *s, arg_DUPM *a) 2025 { 2026 uint64_t imm; 2027 2028 if (!dc_isar_feature(aa64_sve, s)) { 2029 return false; 2030 } 2031 if (!logic_imm_decode_wmask(&imm, extract32(a->dbm, 12, 1), 2032 extract32(a->dbm, 0, 6), 2033 extract32(a->dbm, 6, 6))) { 2034 return false; 2035 } 2036 if (sve_access_check(s)) { 2037 do_dupi_z(s, a->rd, imm); 2038 } 2039 return true; 2040 } 2041 2042 /* 2043 *** SVE Integer Wide Immediate - Predicated Group 2044 */ 2045 2046 /* Implement all merging copies. This is used for CPY (immediate), 2047 * FCPY, CPY (scalar), CPY (SIMD&FP scalar). 2048 */ 2049 static void do_cpy_m(DisasContext *s, int esz, int rd, int rn, int pg, 2050 TCGv_i64 val) 2051 { 2052 typedef void gen_cpy(TCGv_ptr, TCGv_ptr, TCGv_ptr, TCGv_i64, TCGv_i32); 2053 static gen_cpy * const fns[4] = { 2054 gen_helper_sve_cpy_m_b, gen_helper_sve_cpy_m_h, 2055 gen_helper_sve_cpy_m_s, gen_helper_sve_cpy_m_d, 2056 }; 2057 unsigned vsz = vec_full_reg_size(s); 2058 TCGv_i32 desc = tcg_constant_i32(simd_desc(vsz, vsz, 0)); 2059 TCGv_ptr t_zd = tcg_temp_new_ptr(); 2060 TCGv_ptr t_zn = tcg_temp_new_ptr(); 2061 TCGv_ptr t_pg = tcg_temp_new_ptr(); 2062 2063 tcg_gen_addi_ptr(t_zd, tcg_env, vec_full_reg_offset(s, rd)); 2064 tcg_gen_addi_ptr(t_zn, tcg_env, vec_full_reg_offset(s, rn)); 2065 tcg_gen_addi_ptr(t_pg, tcg_env, pred_full_reg_offset(s, pg)); 2066 2067 fns[esz](t_zd, t_zn, t_pg, val, desc); 2068 } 2069 2070 static bool trans_FCPY(DisasContext *s, arg_FCPY *a) 2071 { 2072 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) { 2073 return false; 2074 } 2075 if (sve_access_check(s)) { 2076 /* Decode the VFP immediate. */ 2077 uint64_t imm = vfp_expand_imm(a->esz, a->imm); 2078 do_cpy_m(s, a->esz, a->rd, a->rn, a->pg, tcg_constant_i64(imm)); 2079 } 2080 return true; 2081 } 2082 2083 static bool trans_CPY_m_i(DisasContext *s, arg_rpri_esz *a) 2084 { 2085 if (!dc_isar_feature(aa64_sve, s)) { 2086 return false; 2087 } 2088 if (sve_access_check(s)) { 2089 do_cpy_m(s, a->esz, a->rd, a->rn, a->pg, tcg_constant_i64(a->imm)); 2090 } 2091 return true; 2092 } 2093 2094 static bool trans_CPY_z_i(DisasContext *s, arg_CPY_z_i *a) 2095 { 2096 static gen_helper_gvec_2i * const fns[4] = { 2097 gen_helper_sve_cpy_z_b, gen_helper_sve_cpy_z_h, 2098 gen_helper_sve_cpy_z_s, gen_helper_sve_cpy_z_d, 2099 }; 2100 2101 if (!dc_isar_feature(aa64_sve, s)) { 2102 return false; 2103 } 2104 if (sve_access_check(s)) { 2105 unsigned vsz = vec_full_reg_size(s); 2106 tcg_gen_gvec_2i_ool(vec_full_reg_offset(s, a->rd), 2107 pred_full_reg_offset(s, a->pg), 2108 tcg_constant_i64(a->imm), 2109 vsz, vsz, 0, fns[a->esz]); 2110 } 2111 return true; 2112 } 2113 2114 /* 2115 *** SVE Permute Extract Group 2116 */ 2117 2118 static bool do_EXT(DisasContext *s, int rd, int rn, int rm, int imm) 2119 { 2120 if (!sve_access_check(s)) { 2121 return true; 2122 } 2123 2124 unsigned vsz = vec_full_reg_size(s); 2125 unsigned n_ofs = imm >= vsz ? 0 : imm; 2126 unsigned n_siz = vsz - n_ofs; 2127 unsigned d = vec_full_reg_offset(s, rd); 2128 unsigned n = vec_full_reg_offset(s, rn); 2129 unsigned m = vec_full_reg_offset(s, rm); 2130 2131 /* Use host vector move insns if we have appropriate sizes 2132 * and no unfortunate overlap. 2133 */ 2134 if (m != d 2135 && n_ofs == size_for_gvec(n_ofs) 2136 && n_siz == size_for_gvec(n_siz) 2137 && (d != n || n_siz <= n_ofs)) { 2138 tcg_gen_gvec_mov(0, d, n + n_ofs, n_siz, n_siz); 2139 if (n_ofs != 0) { 2140 tcg_gen_gvec_mov(0, d + n_siz, m, n_ofs, n_ofs); 2141 } 2142 } else { 2143 tcg_gen_gvec_3_ool(d, n, m, vsz, vsz, n_ofs, gen_helper_sve_ext); 2144 } 2145 return true; 2146 } 2147 2148 TRANS_FEAT(EXT, aa64_sve, do_EXT, a->rd, a->rn, a->rm, a->imm) 2149 TRANS_FEAT(EXT_sve2, aa64_sve2, do_EXT, a->rd, a->rn, (a->rn + 1) % 32, a->imm) 2150 2151 /* 2152 *** SVE Permute - Unpredicated Group 2153 */ 2154 2155 static bool trans_DUP_s(DisasContext *s, arg_DUP_s *a) 2156 { 2157 if (!dc_isar_feature(aa64_sve, s)) { 2158 return false; 2159 } 2160 if (sve_access_check(s)) { 2161 unsigned vsz = vec_full_reg_size(s); 2162 tcg_gen_gvec_dup_i64(a->esz, vec_full_reg_offset(s, a->rd), 2163 vsz, vsz, cpu_reg_sp(s, a->rn)); 2164 } 2165 return true; 2166 } 2167 2168 static bool trans_DUP_x(DisasContext *s, arg_DUP_x *a) 2169 { 2170 if (!dc_isar_feature(aa64_sve, s)) { 2171 return false; 2172 } 2173 if ((a->imm & 0x1f) == 0) { 2174 return false; 2175 } 2176 if (sve_access_check(s)) { 2177 unsigned vsz = vec_full_reg_size(s); 2178 unsigned dofs = vec_full_reg_offset(s, a->rd); 2179 unsigned esz, index; 2180 2181 esz = ctz32(a->imm); 2182 index = a->imm >> (esz + 1); 2183 2184 if ((index << esz) < vsz) { 2185 unsigned nofs = vec_reg_offset(s, a->rn, index, esz); 2186 tcg_gen_gvec_dup_mem(esz, dofs, nofs, vsz, vsz); 2187 } else { 2188 /* 2189 * While dup_mem handles 128-bit elements, dup_imm does not. 2190 * Thankfully element size doesn't matter for splatting zero. 2191 */ 2192 tcg_gen_gvec_dup_imm(MO_64, dofs, vsz, vsz, 0); 2193 } 2194 } 2195 return true; 2196 } 2197 2198 static void do_insr_i64(DisasContext *s, arg_rrr_esz *a, TCGv_i64 val) 2199 { 2200 typedef void gen_insr(TCGv_ptr, TCGv_ptr, TCGv_i64, TCGv_i32); 2201 static gen_insr * const fns[4] = { 2202 gen_helper_sve_insr_b, gen_helper_sve_insr_h, 2203 gen_helper_sve_insr_s, gen_helper_sve_insr_d, 2204 }; 2205 unsigned vsz = vec_full_reg_size(s); 2206 TCGv_i32 desc = tcg_constant_i32(simd_desc(vsz, vsz, 0)); 2207 TCGv_ptr t_zd = tcg_temp_new_ptr(); 2208 TCGv_ptr t_zn = tcg_temp_new_ptr(); 2209 2210 tcg_gen_addi_ptr(t_zd, tcg_env, vec_full_reg_offset(s, a->rd)); 2211 tcg_gen_addi_ptr(t_zn, tcg_env, vec_full_reg_offset(s, a->rn)); 2212 2213 fns[a->esz](t_zd, t_zn, val, desc); 2214 } 2215 2216 static bool trans_INSR_f(DisasContext *s, arg_rrr_esz *a) 2217 { 2218 if (!dc_isar_feature(aa64_sve, s)) { 2219 return false; 2220 } 2221 if (sve_access_check(s)) { 2222 TCGv_i64 t = tcg_temp_new_i64(); 2223 tcg_gen_ld_i64(t, tcg_env, vec_reg_offset(s, a->rm, 0, MO_64)); 2224 do_insr_i64(s, a, t); 2225 } 2226 return true; 2227 } 2228 2229 static bool trans_INSR_r(DisasContext *s, arg_rrr_esz *a) 2230 { 2231 if (!dc_isar_feature(aa64_sve, s)) { 2232 return false; 2233 } 2234 if (sve_access_check(s)) { 2235 do_insr_i64(s, a, cpu_reg(s, a->rm)); 2236 } 2237 return true; 2238 } 2239 2240 static gen_helper_gvec_2 * const rev_fns[4] = { 2241 gen_helper_sve_rev_b, gen_helper_sve_rev_h, 2242 gen_helper_sve_rev_s, gen_helper_sve_rev_d 2243 }; 2244 TRANS_FEAT(REV_v, aa64_sve, gen_gvec_ool_zz, rev_fns[a->esz], a->rd, a->rn, 0) 2245 2246 static gen_helper_gvec_3 * const sve_tbl_fns[4] = { 2247 gen_helper_sve_tbl_b, gen_helper_sve_tbl_h, 2248 gen_helper_sve_tbl_s, gen_helper_sve_tbl_d 2249 }; 2250 TRANS_FEAT(TBL, aa64_sve, gen_gvec_ool_arg_zzz, sve_tbl_fns[a->esz], a, 0) 2251 2252 static gen_helper_gvec_4 * const sve2_tbl_fns[4] = { 2253 gen_helper_sve2_tbl_b, gen_helper_sve2_tbl_h, 2254 gen_helper_sve2_tbl_s, gen_helper_sve2_tbl_d 2255 }; 2256 TRANS_FEAT(TBL_sve2, aa64_sve2, gen_gvec_ool_zzzz, sve2_tbl_fns[a->esz], 2257 a->rd, a->rn, (a->rn + 1) % 32, a->rm, 0) 2258 2259 static gen_helper_gvec_3 * const tbx_fns[4] = { 2260 gen_helper_sve2_tbx_b, gen_helper_sve2_tbx_h, 2261 gen_helper_sve2_tbx_s, gen_helper_sve2_tbx_d 2262 }; 2263 TRANS_FEAT(TBX, aa64_sve2, gen_gvec_ool_arg_zzz, tbx_fns[a->esz], a, 0) 2264 2265 static bool trans_UNPK(DisasContext *s, arg_UNPK *a) 2266 { 2267 static gen_helper_gvec_2 * const fns[4][2] = { 2268 { NULL, NULL }, 2269 { gen_helper_sve_sunpk_h, gen_helper_sve_uunpk_h }, 2270 { gen_helper_sve_sunpk_s, gen_helper_sve_uunpk_s }, 2271 { gen_helper_sve_sunpk_d, gen_helper_sve_uunpk_d }, 2272 }; 2273 2274 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) { 2275 return false; 2276 } 2277 if (sve_access_check(s)) { 2278 unsigned vsz = vec_full_reg_size(s); 2279 tcg_gen_gvec_2_ool(vec_full_reg_offset(s, a->rd), 2280 vec_full_reg_offset(s, a->rn) 2281 + (a->h ? vsz / 2 : 0), 2282 vsz, vsz, 0, fns[a->esz][a->u]); 2283 } 2284 return true; 2285 } 2286 2287 /* 2288 *** SVE Permute - Predicates Group 2289 */ 2290 2291 static bool do_perm_pred3(DisasContext *s, arg_rrr_esz *a, bool high_odd, 2292 gen_helper_gvec_3 *fn) 2293 { 2294 if (!sve_access_check(s)) { 2295 return true; 2296 } 2297 2298 unsigned vsz = pred_full_reg_size(s); 2299 2300 TCGv_ptr t_d = tcg_temp_new_ptr(); 2301 TCGv_ptr t_n = tcg_temp_new_ptr(); 2302 TCGv_ptr t_m = tcg_temp_new_ptr(); 2303 uint32_t desc = 0; 2304 2305 desc = FIELD_DP32(desc, PREDDESC, OPRSZ, vsz); 2306 desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz); 2307 desc = FIELD_DP32(desc, PREDDESC, DATA, high_odd); 2308 2309 tcg_gen_addi_ptr(t_d, tcg_env, pred_full_reg_offset(s, a->rd)); 2310 tcg_gen_addi_ptr(t_n, tcg_env, pred_full_reg_offset(s, a->rn)); 2311 tcg_gen_addi_ptr(t_m, tcg_env, pred_full_reg_offset(s, a->rm)); 2312 2313 fn(t_d, t_n, t_m, tcg_constant_i32(desc)); 2314 return true; 2315 } 2316 2317 static bool do_perm_pred2(DisasContext *s, arg_rr_esz *a, bool high_odd, 2318 gen_helper_gvec_2 *fn) 2319 { 2320 if (!sve_access_check(s)) { 2321 return true; 2322 } 2323 2324 unsigned vsz = pred_full_reg_size(s); 2325 TCGv_ptr t_d = tcg_temp_new_ptr(); 2326 TCGv_ptr t_n = tcg_temp_new_ptr(); 2327 uint32_t desc = 0; 2328 2329 tcg_gen_addi_ptr(t_d, tcg_env, pred_full_reg_offset(s, a->rd)); 2330 tcg_gen_addi_ptr(t_n, tcg_env, pred_full_reg_offset(s, a->rn)); 2331 2332 desc = FIELD_DP32(desc, PREDDESC, OPRSZ, vsz); 2333 desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz); 2334 desc = FIELD_DP32(desc, PREDDESC, DATA, high_odd); 2335 2336 fn(t_d, t_n, tcg_constant_i32(desc)); 2337 return true; 2338 } 2339 2340 TRANS_FEAT(ZIP1_p, aa64_sve, do_perm_pred3, a, 0, gen_helper_sve_zip_p) 2341 TRANS_FEAT(ZIP2_p, aa64_sve, do_perm_pred3, a, 1, gen_helper_sve_zip_p) 2342 TRANS_FEAT(UZP1_p, aa64_sve, do_perm_pred3, a, 0, gen_helper_sve_uzp_p) 2343 TRANS_FEAT(UZP2_p, aa64_sve, do_perm_pred3, a, 1, gen_helper_sve_uzp_p) 2344 TRANS_FEAT(TRN1_p, aa64_sve, do_perm_pred3, a, 0, gen_helper_sve_trn_p) 2345 TRANS_FEAT(TRN2_p, aa64_sve, do_perm_pred3, a, 1, gen_helper_sve_trn_p) 2346 2347 TRANS_FEAT(REV_p, aa64_sve, do_perm_pred2, a, 0, gen_helper_sve_rev_p) 2348 TRANS_FEAT(PUNPKLO, aa64_sve, do_perm_pred2, a, 0, gen_helper_sve_punpk_p) 2349 TRANS_FEAT(PUNPKHI, aa64_sve, do_perm_pred2, a, 1, gen_helper_sve_punpk_p) 2350 2351 /* 2352 *** SVE Permute - Interleaving Group 2353 */ 2354 2355 static gen_helper_gvec_3 * const zip_fns[4] = { 2356 gen_helper_sve_zip_b, gen_helper_sve_zip_h, 2357 gen_helper_sve_zip_s, gen_helper_sve_zip_d, 2358 }; 2359 TRANS_FEAT(ZIP1_z, aa64_sve, gen_gvec_ool_arg_zzz, 2360 zip_fns[a->esz], a, 0) 2361 TRANS_FEAT(ZIP2_z, aa64_sve, gen_gvec_ool_arg_zzz, 2362 zip_fns[a->esz], a, vec_full_reg_size(s) / 2) 2363 2364 TRANS_FEAT(ZIP1_q, aa64_sve_f64mm, gen_gvec_ool_arg_zzz, 2365 gen_helper_sve2_zip_q, a, 0) 2366 TRANS_FEAT(ZIP2_q, aa64_sve_f64mm, gen_gvec_ool_arg_zzz, 2367 gen_helper_sve2_zip_q, a, 2368 QEMU_ALIGN_DOWN(vec_full_reg_size(s), 32) / 2) 2369 2370 static gen_helper_gvec_3 * const uzp_fns[4] = { 2371 gen_helper_sve_uzp_b, gen_helper_sve_uzp_h, 2372 gen_helper_sve_uzp_s, gen_helper_sve_uzp_d, 2373 }; 2374 2375 TRANS_FEAT(UZP1_z, aa64_sve, gen_gvec_ool_arg_zzz, 2376 uzp_fns[a->esz], a, 0) 2377 TRANS_FEAT(UZP2_z, aa64_sve, gen_gvec_ool_arg_zzz, 2378 uzp_fns[a->esz], a, 1 << a->esz) 2379 2380 TRANS_FEAT(UZP1_q, aa64_sve_f64mm, gen_gvec_ool_arg_zzz, 2381 gen_helper_sve2_uzp_q, a, 0) 2382 TRANS_FEAT(UZP2_q, aa64_sve_f64mm, gen_gvec_ool_arg_zzz, 2383 gen_helper_sve2_uzp_q, a, 16) 2384 2385 static gen_helper_gvec_3 * const trn_fns[4] = { 2386 gen_helper_sve_trn_b, gen_helper_sve_trn_h, 2387 gen_helper_sve_trn_s, gen_helper_sve_trn_d, 2388 }; 2389 2390 TRANS_FEAT(TRN1_z, aa64_sve, gen_gvec_ool_arg_zzz, 2391 trn_fns[a->esz], a, 0) 2392 TRANS_FEAT(TRN2_z, aa64_sve, gen_gvec_ool_arg_zzz, 2393 trn_fns[a->esz], a, 1 << a->esz) 2394 2395 TRANS_FEAT(TRN1_q, aa64_sve_f64mm, gen_gvec_ool_arg_zzz, 2396 gen_helper_sve2_trn_q, a, 0) 2397 TRANS_FEAT(TRN2_q, aa64_sve_f64mm, gen_gvec_ool_arg_zzz, 2398 gen_helper_sve2_trn_q, a, 16) 2399 2400 /* 2401 *** SVE Permute Vector - Predicated Group 2402 */ 2403 2404 static gen_helper_gvec_3 * const compact_fns[4] = { 2405 NULL, NULL, gen_helper_sve_compact_s, gen_helper_sve_compact_d 2406 }; 2407 TRANS_FEAT_NONSTREAMING(COMPACT, aa64_sve, gen_gvec_ool_arg_zpz, 2408 compact_fns[a->esz], a, 0) 2409 2410 /* Call the helper that computes the ARM LastActiveElement pseudocode 2411 * function, scaled by the element size. This includes the not found 2412 * indication; e.g. not found for esz=3 is -8. 2413 */ 2414 static void find_last_active(DisasContext *s, TCGv_i32 ret, int esz, int pg) 2415 { 2416 /* Predicate sizes may be smaller and cannot use simd_desc. We cannot 2417 * round up, as we do elsewhere, because we need the exact size. 2418 */ 2419 TCGv_ptr t_p = tcg_temp_new_ptr(); 2420 unsigned desc = 0; 2421 2422 desc = FIELD_DP32(desc, PREDDESC, OPRSZ, pred_full_reg_size(s)); 2423 desc = FIELD_DP32(desc, PREDDESC, ESZ, esz); 2424 2425 tcg_gen_addi_ptr(t_p, tcg_env, pred_full_reg_offset(s, pg)); 2426 2427 gen_helper_sve_last_active_element(ret, t_p, tcg_constant_i32(desc)); 2428 } 2429 2430 /* Increment LAST to the offset of the next element in the vector, 2431 * wrapping around to 0. 2432 */ 2433 static void incr_last_active(DisasContext *s, TCGv_i32 last, int esz) 2434 { 2435 unsigned vsz = vec_full_reg_size(s); 2436 2437 tcg_gen_addi_i32(last, last, 1 << esz); 2438 if (is_power_of_2(vsz)) { 2439 tcg_gen_andi_i32(last, last, vsz - 1); 2440 } else { 2441 TCGv_i32 max = tcg_constant_i32(vsz); 2442 TCGv_i32 zero = tcg_constant_i32(0); 2443 tcg_gen_movcond_i32(TCG_COND_GEU, last, last, max, zero, last); 2444 } 2445 } 2446 2447 /* If LAST < 0, set LAST to the offset of the last element in the vector. */ 2448 static void wrap_last_active(DisasContext *s, TCGv_i32 last, int esz) 2449 { 2450 unsigned vsz = vec_full_reg_size(s); 2451 2452 if (is_power_of_2(vsz)) { 2453 tcg_gen_andi_i32(last, last, vsz - 1); 2454 } else { 2455 TCGv_i32 max = tcg_constant_i32(vsz - (1 << esz)); 2456 TCGv_i32 zero = tcg_constant_i32(0); 2457 tcg_gen_movcond_i32(TCG_COND_LT, last, last, zero, max, last); 2458 } 2459 } 2460 2461 /* Load an unsigned element of ESZ from BASE+OFS. */ 2462 static TCGv_i64 load_esz(TCGv_ptr base, int ofs, int esz) 2463 { 2464 TCGv_i64 r = tcg_temp_new_i64(); 2465 2466 switch (esz) { 2467 case 0: 2468 tcg_gen_ld8u_i64(r, base, ofs); 2469 break; 2470 case 1: 2471 tcg_gen_ld16u_i64(r, base, ofs); 2472 break; 2473 case 2: 2474 tcg_gen_ld32u_i64(r, base, ofs); 2475 break; 2476 case 3: 2477 tcg_gen_ld_i64(r, base, ofs); 2478 break; 2479 default: 2480 g_assert_not_reached(); 2481 } 2482 return r; 2483 } 2484 2485 /* Load an unsigned element of ESZ from RM[LAST]. */ 2486 static TCGv_i64 load_last_active(DisasContext *s, TCGv_i32 last, 2487 int rm, int esz) 2488 { 2489 TCGv_ptr p = tcg_temp_new_ptr(); 2490 2491 /* Convert offset into vector into offset into ENV. 2492 * The final adjustment for the vector register base 2493 * is added via constant offset to the load. 2494 */ 2495 #if HOST_BIG_ENDIAN 2496 /* Adjust for element ordering. See vec_reg_offset. */ 2497 if (esz < 3) { 2498 tcg_gen_xori_i32(last, last, 8 - (1 << esz)); 2499 } 2500 #endif 2501 tcg_gen_ext_i32_ptr(p, last); 2502 tcg_gen_add_ptr(p, p, tcg_env); 2503 2504 return load_esz(p, vec_full_reg_offset(s, rm), esz); 2505 } 2506 2507 /* Compute CLAST for a Zreg. */ 2508 static bool do_clast_vector(DisasContext *s, arg_rprr_esz *a, bool before) 2509 { 2510 TCGv_i32 last; 2511 TCGLabel *over; 2512 TCGv_i64 ele; 2513 unsigned vsz, esz = a->esz; 2514 2515 if (!sve_access_check(s)) { 2516 return true; 2517 } 2518 2519 last = tcg_temp_new_i32(); 2520 over = gen_new_label(); 2521 2522 find_last_active(s, last, esz, a->pg); 2523 2524 /* There is of course no movcond for a 2048-bit vector, 2525 * so we must branch over the actual store. 2526 */ 2527 tcg_gen_brcondi_i32(TCG_COND_LT, last, 0, over); 2528 2529 if (!before) { 2530 incr_last_active(s, last, esz); 2531 } 2532 2533 ele = load_last_active(s, last, a->rm, esz); 2534 2535 vsz = vec_full_reg_size(s); 2536 tcg_gen_gvec_dup_i64(esz, vec_full_reg_offset(s, a->rd), vsz, vsz, ele); 2537 2538 /* If this insn used MOVPRFX, we may need a second move. */ 2539 if (a->rd != a->rn) { 2540 TCGLabel *done = gen_new_label(); 2541 tcg_gen_br(done); 2542 2543 gen_set_label(over); 2544 do_mov_z(s, a->rd, a->rn); 2545 2546 gen_set_label(done); 2547 } else { 2548 gen_set_label(over); 2549 } 2550 return true; 2551 } 2552 2553 TRANS_FEAT(CLASTA_z, aa64_sve, do_clast_vector, a, false) 2554 TRANS_FEAT(CLASTB_z, aa64_sve, do_clast_vector, a, true) 2555 2556 /* Compute CLAST for a scalar. */ 2557 static void do_clast_scalar(DisasContext *s, int esz, int pg, int rm, 2558 bool before, TCGv_i64 reg_val) 2559 { 2560 TCGv_i32 last = tcg_temp_new_i32(); 2561 TCGv_i64 ele, cmp; 2562 2563 find_last_active(s, last, esz, pg); 2564 2565 /* Extend the original value of last prior to incrementing. */ 2566 cmp = tcg_temp_new_i64(); 2567 tcg_gen_ext_i32_i64(cmp, last); 2568 2569 if (!before) { 2570 incr_last_active(s, last, esz); 2571 } 2572 2573 /* The conceit here is that while last < 0 indicates not found, after 2574 * adjusting for tcg_env->vfp.zregs[rm], it is still a valid address 2575 * from which we can load garbage. We then discard the garbage with 2576 * a conditional move. 2577 */ 2578 ele = load_last_active(s, last, rm, esz); 2579 2580 tcg_gen_movcond_i64(TCG_COND_GE, reg_val, cmp, tcg_constant_i64(0), 2581 ele, reg_val); 2582 } 2583 2584 /* Compute CLAST for a Vreg. */ 2585 static bool do_clast_fp(DisasContext *s, arg_rpr_esz *a, bool before) 2586 { 2587 if (sve_access_check(s)) { 2588 int esz = a->esz; 2589 int ofs = vec_reg_offset(s, a->rd, 0, esz); 2590 TCGv_i64 reg = load_esz(tcg_env, ofs, esz); 2591 2592 do_clast_scalar(s, esz, a->pg, a->rn, before, reg); 2593 write_fp_dreg(s, a->rd, reg); 2594 } 2595 return true; 2596 } 2597 2598 TRANS_FEAT(CLASTA_v, aa64_sve, do_clast_fp, a, false) 2599 TRANS_FEAT(CLASTB_v, aa64_sve, do_clast_fp, a, true) 2600 2601 /* Compute CLAST for a Xreg. */ 2602 static bool do_clast_general(DisasContext *s, arg_rpr_esz *a, bool before) 2603 { 2604 TCGv_i64 reg; 2605 2606 if (!sve_access_check(s)) { 2607 return true; 2608 } 2609 2610 reg = cpu_reg(s, a->rd); 2611 switch (a->esz) { 2612 case 0: 2613 tcg_gen_ext8u_i64(reg, reg); 2614 break; 2615 case 1: 2616 tcg_gen_ext16u_i64(reg, reg); 2617 break; 2618 case 2: 2619 tcg_gen_ext32u_i64(reg, reg); 2620 break; 2621 case 3: 2622 break; 2623 default: 2624 g_assert_not_reached(); 2625 } 2626 2627 do_clast_scalar(s, a->esz, a->pg, a->rn, before, reg); 2628 return true; 2629 } 2630 2631 TRANS_FEAT(CLASTA_r, aa64_sve, do_clast_general, a, false) 2632 TRANS_FEAT(CLASTB_r, aa64_sve, do_clast_general, a, true) 2633 2634 /* Compute LAST for a scalar. */ 2635 static TCGv_i64 do_last_scalar(DisasContext *s, int esz, 2636 int pg, int rm, bool before) 2637 { 2638 TCGv_i32 last = tcg_temp_new_i32(); 2639 2640 find_last_active(s, last, esz, pg); 2641 if (before) { 2642 wrap_last_active(s, last, esz); 2643 } else { 2644 incr_last_active(s, last, esz); 2645 } 2646 2647 return load_last_active(s, last, rm, esz); 2648 } 2649 2650 /* Compute LAST for a Vreg. */ 2651 static bool do_last_fp(DisasContext *s, arg_rpr_esz *a, bool before) 2652 { 2653 if (sve_access_check(s)) { 2654 TCGv_i64 val = do_last_scalar(s, a->esz, a->pg, a->rn, before); 2655 write_fp_dreg(s, a->rd, val); 2656 } 2657 return true; 2658 } 2659 2660 TRANS_FEAT(LASTA_v, aa64_sve, do_last_fp, a, false) 2661 TRANS_FEAT(LASTB_v, aa64_sve, do_last_fp, a, true) 2662 2663 /* Compute LAST for a Xreg. */ 2664 static bool do_last_general(DisasContext *s, arg_rpr_esz *a, bool before) 2665 { 2666 if (sve_access_check(s)) { 2667 TCGv_i64 val = do_last_scalar(s, a->esz, a->pg, a->rn, before); 2668 tcg_gen_mov_i64(cpu_reg(s, a->rd), val); 2669 } 2670 return true; 2671 } 2672 2673 TRANS_FEAT(LASTA_r, aa64_sve, do_last_general, a, false) 2674 TRANS_FEAT(LASTB_r, aa64_sve, do_last_general, a, true) 2675 2676 static bool trans_CPY_m_r(DisasContext *s, arg_rpr_esz *a) 2677 { 2678 if (!dc_isar_feature(aa64_sve, s)) { 2679 return false; 2680 } 2681 if (sve_access_check(s)) { 2682 do_cpy_m(s, a->esz, a->rd, a->rd, a->pg, cpu_reg_sp(s, a->rn)); 2683 } 2684 return true; 2685 } 2686 2687 static bool trans_CPY_m_v(DisasContext *s, arg_rpr_esz *a) 2688 { 2689 if (!dc_isar_feature(aa64_sve, s)) { 2690 return false; 2691 } 2692 if (sve_access_check(s)) { 2693 int ofs = vec_reg_offset(s, a->rn, 0, a->esz); 2694 TCGv_i64 t = load_esz(tcg_env, ofs, a->esz); 2695 do_cpy_m(s, a->esz, a->rd, a->rd, a->pg, t); 2696 } 2697 return true; 2698 } 2699 2700 static gen_helper_gvec_3 * const revb_fns[4] = { 2701 NULL, gen_helper_sve_revb_h, 2702 gen_helper_sve_revb_s, gen_helper_sve_revb_d, 2703 }; 2704 TRANS_FEAT(REVB, aa64_sve, gen_gvec_ool_arg_zpz, revb_fns[a->esz], a, 0) 2705 2706 static gen_helper_gvec_3 * const revh_fns[4] = { 2707 NULL, NULL, gen_helper_sve_revh_s, gen_helper_sve_revh_d, 2708 }; 2709 TRANS_FEAT(REVH, aa64_sve, gen_gvec_ool_arg_zpz, revh_fns[a->esz], a, 0) 2710 2711 TRANS_FEAT(REVW, aa64_sve, gen_gvec_ool_arg_zpz, 2712 a->esz == 3 ? gen_helper_sve_revw_d : NULL, a, 0) 2713 2714 TRANS_FEAT(REVD, aa64_sme, gen_gvec_ool_arg_zpz, gen_helper_sme_revd_q, a, 0) 2715 2716 TRANS_FEAT(SPLICE, aa64_sve, gen_gvec_ool_arg_zpzz, 2717 gen_helper_sve_splice, a, a->esz) 2718 2719 TRANS_FEAT(SPLICE_sve2, aa64_sve2, gen_gvec_ool_zzzp, gen_helper_sve_splice, 2720 a->rd, a->rn, (a->rn + 1) % 32, a->pg, a->esz) 2721 2722 /* 2723 *** SVE Integer Compare - Vectors Group 2724 */ 2725 2726 static bool do_ppzz_flags(DisasContext *s, arg_rprr_esz *a, 2727 gen_helper_gvec_flags_4 *gen_fn) 2728 { 2729 TCGv_ptr pd, zn, zm, pg; 2730 unsigned vsz; 2731 TCGv_i32 t; 2732 2733 if (gen_fn == NULL) { 2734 return false; 2735 } 2736 if (!sve_access_check(s)) { 2737 return true; 2738 } 2739 2740 vsz = vec_full_reg_size(s); 2741 t = tcg_temp_new_i32(); 2742 pd = tcg_temp_new_ptr(); 2743 zn = tcg_temp_new_ptr(); 2744 zm = tcg_temp_new_ptr(); 2745 pg = tcg_temp_new_ptr(); 2746 2747 tcg_gen_addi_ptr(pd, tcg_env, pred_full_reg_offset(s, a->rd)); 2748 tcg_gen_addi_ptr(zn, tcg_env, vec_full_reg_offset(s, a->rn)); 2749 tcg_gen_addi_ptr(zm, tcg_env, vec_full_reg_offset(s, a->rm)); 2750 tcg_gen_addi_ptr(pg, tcg_env, pred_full_reg_offset(s, a->pg)); 2751 2752 gen_fn(t, pd, zn, zm, pg, tcg_constant_i32(simd_desc(vsz, vsz, 0))); 2753 2754 do_pred_flags(t); 2755 return true; 2756 } 2757 2758 #define DO_PPZZ(NAME, name) \ 2759 static gen_helper_gvec_flags_4 * const name##_ppzz_fns[4] = { \ 2760 gen_helper_sve_##name##_ppzz_b, gen_helper_sve_##name##_ppzz_h, \ 2761 gen_helper_sve_##name##_ppzz_s, gen_helper_sve_##name##_ppzz_d, \ 2762 }; \ 2763 TRANS_FEAT(NAME##_ppzz, aa64_sve, do_ppzz_flags, \ 2764 a, name##_ppzz_fns[a->esz]) 2765 2766 DO_PPZZ(CMPEQ, cmpeq) 2767 DO_PPZZ(CMPNE, cmpne) 2768 DO_PPZZ(CMPGT, cmpgt) 2769 DO_PPZZ(CMPGE, cmpge) 2770 DO_PPZZ(CMPHI, cmphi) 2771 DO_PPZZ(CMPHS, cmphs) 2772 2773 #undef DO_PPZZ 2774 2775 #define DO_PPZW(NAME, name) \ 2776 static gen_helper_gvec_flags_4 * const name##_ppzw_fns[4] = { \ 2777 gen_helper_sve_##name##_ppzw_b, gen_helper_sve_##name##_ppzw_h, \ 2778 gen_helper_sve_##name##_ppzw_s, NULL \ 2779 }; \ 2780 TRANS_FEAT(NAME##_ppzw, aa64_sve, do_ppzz_flags, \ 2781 a, name##_ppzw_fns[a->esz]) 2782 2783 DO_PPZW(CMPEQ, cmpeq) 2784 DO_PPZW(CMPNE, cmpne) 2785 DO_PPZW(CMPGT, cmpgt) 2786 DO_PPZW(CMPGE, cmpge) 2787 DO_PPZW(CMPHI, cmphi) 2788 DO_PPZW(CMPHS, cmphs) 2789 DO_PPZW(CMPLT, cmplt) 2790 DO_PPZW(CMPLE, cmple) 2791 DO_PPZW(CMPLO, cmplo) 2792 DO_PPZW(CMPLS, cmpls) 2793 2794 #undef DO_PPZW 2795 2796 /* 2797 *** SVE Integer Compare - Immediate Groups 2798 */ 2799 2800 static bool do_ppzi_flags(DisasContext *s, arg_rpri_esz *a, 2801 gen_helper_gvec_flags_3 *gen_fn) 2802 { 2803 TCGv_ptr pd, zn, pg; 2804 unsigned vsz; 2805 TCGv_i32 t; 2806 2807 if (gen_fn == NULL) { 2808 return false; 2809 } 2810 if (!sve_access_check(s)) { 2811 return true; 2812 } 2813 2814 vsz = vec_full_reg_size(s); 2815 t = tcg_temp_new_i32(); 2816 pd = tcg_temp_new_ptr(); 2817 zn = tcg_temp_new_ptr(); 2818 pg = tcg_temp_new_ptr(); 2819 2820 tcg_gen_addi_ptr(pd, tcg_env, pred_full_reg_offset(s, a->rd)); 2821 tcg_gen_addi_ptr(zn, tcg_env, vec_full_reg_offset(s, a->rn)); 2822 tcg_gen_addi_ptr(pg, tcg_env, pred_full_reg_offset(s, a->pg)); 2823 2824 gen_fn(t, pd, zn, pg, tcg_constant_i32(simd_desc(vsz, vsz, a->imm))); 2825 2826 do_pred_flags(t); 2827 return true; 2828 } 2829 2830 #define DO_PPZI(NAME, name) \ 2831 static gen_helper_gvec_flags_3 * const name##_ppzi_fns[4] = { \ 2832 gen_helper_sve_##name##_ppzi_b, gen_helper_sve_##name##_ppzi_h, \ 2833 gen_helper_sve_##name##_ppzi_s, gen_helper_sve_##name##_ppzi_d, \ 2834 }; \ 2835 TRANS_FEAT(NAME##_ppzi, aa64_sve, do_ppzi_flags, a, \ 2836 name##_ppzi_fns[a->esz]) 2837 2838 DO_PPZI(CMPEQ, cmpeq) 2839 DO_PPZI(CMPNE, cmpne) 2840 DO_PPZI(CMPGT, cmpgt) 2841 DO_PPZI(CMPGE, cmpge) 2842 DO_PPZI(CMPHI, cmphi) 2843 DO_PPZI(CMPHS, cmphs) 2844 DO_PPZI(CMPLT, cmplt) 2845 DO_PPZI(CMPLE, cmple) 2846 DO_PPZI(CMPLO, cmplo) 2847 DO_PPZI(CMPLS, cmpls) 2848 2849 #undef DO_PPZI 2850 2851 /* 2852 *** SVE Partition Break Group 2853 */ 2854 2855 static bool do_brk3(DisasContext *s, arg_rprr_s *a, 2856 gen_helper_gvec_4 *fn, gen_helper_gvec_flags_4 *fn_s) 2857 { 2858 if (!sve_access_check(s)) { 2859 return true; 2860 } 2861 2862 unsigned vsz = pred_full_reg_size(s); 2863 2864 /* Predicate sizes may be smaller and cannot use simd_desc. */ 2865 TCGv_ptr d = tcg_temp_new_ptr(); 2866 TCGv_ptr n = tcg_temp_new_ptr(); 2867 TCGv_ptr m = tcg_temp_new_ptr(); 2868 TCGv_ptr g = tcg_temp_new_ptr(); 2869 TCGv_i32 desc = tcg_constant_i32(FIELD_DP32(0, PREDDESC, OPRSZ, vsz)); 2870 2871 tcg_gen_addi_ptr(d, tcg_env, pred_full_reg_offset(s, a->rd)); 2872 tcg_gen_addi_ptr(n, tcg_env, pred_full_reg_offset(s, a->rn)); 2873 tcg_gen_addi_ptr(m, tcg_env, pred_full_reg_offset(s, a->rm)); 2874 tcg_gen_addi_ptr(g, tcg_env, pred_full_reg_offset(s, a->pg)); 2875 2876 if (a->s) { 2877 TCGv_i32 t = tcg_temp_new_i32(); 2878 fn_s(t, d, n, m, g, desc); 2879 do_pred_flags(t); 2880 } else { 2881 fn(d, n, m, g, desc); 2882 } 2883 return true; 2884 } 2885 2886 static bool do_brk2(DisasContext *s, arg_rpr_s *a, 2887 gen_helper_gvec_3 *fn, gen_helper_gvec_flags_3 *fn_s) 2888 { 2889 if (!sve_access_check(s)) { 2890 return true; 2891 } 2892 2893 unsigned vsz = pred_full_reg_size(s); 2894 2895 /* Predicate sizes may be smaller and cannot use simd_desc. */ 2896 TCGv_ptr d = tcg_temp_new_ptr(); 2897 TCGv_ptr n = tcg_temp_new_ptr(); 2898 TCGv_ptr g = tcg_temp_new_ptr(); 2899 TCGv_i32 desc = tcg_constant_i32(FIELD_DP32(0, PREDDESC, OPRSZ, vsz)); 2900 2901 tcg_gen_addi_ptr(d, tcg_env, pred_full_reg_offset(s, a->rd)); 2902 tcg_gen_addi_ptr(n, tcg_env, pred_full_reg_offset(s, a->rn)); 2903 tcg_gen_addi_ptr(g, tcg_env, pred_full_reg_offset(s, a->pg)); 2904 2905 if (a->s) { 2906 TCGv_i32 t = tcg_temp_new_i32(); 2907 fn_s(t, d, n, g, desc); 2908 do_pred_flags(t); 2909 } else { 2910 fn(d, n, g, desc); 2911 } 2912 return true; 2913 } 2914 2915 TRANS_FEAT(BRKPA, aa64_sve, do_brk3, a, 2916 gen_helper_sve_brkpa, gen_helper_sve_brkpas) 2917 TRANS_FEAT(BRKPB, aa64_sve, do_brk3, a, 2918 gen_helper_sve_brkpb, gen_helper_sve_brkpbs) 2919 2920 TRANS_FEAT(BRKA_m, aa64_sve, do_brk2, a, 2921 gen_helper_sve_brka_m, gen_helper_sve_brkas_m) 2922 TRANS_FEAT(BRKB_m, aa64_sve, do_brk2, a, 2923 gen_helper_sve_brkb_m, gen_helper_sve_brkbs_m) 2924 2925 TRANS_FEAT(BRKA_z, aa64_sve, do_brk2, a, 2926 gen_helper_sve_brka_z, gen_helper_sve_brkas_z) 2927 TRANS_FEAT(BRKB_z, aa64_sve, do_brk2, a, 2928 gen_helper_sve_brkb_z, gen_helper_sve_brkbs_z) 2929 2930 TRANS_FEAT(BRKN, aa64_sve, do_brk2, a, 2931 gen_helper_sve_brkn, gen_helper_sve_brkns) 2932 2933 /* 2934 *** SVE Predicate Count Group 2935 */ 2936 2937 static void do_cntp(DisasContext *s, TCGv_i64 val, int esz, int pn, int pg) 2938 { 2939 unsigned psz = pred_full_reg_size(s); 2940 2941 if (psz <= 8) { 2942 uint64_t psz_mask; 2943 2944 tcg_gen_ld_i64(val, tcg_env, pred_full_reg_offset(s, pn)); 2945 if (pn != pg) { 2946 TCGv_i64 g = tcg_temp_new_i64(); 2947 tcg_gen_ld_i64(g, tcg_env, pred_full_reg_offset(s, pg)); 2948 tcg_gen_and_i64(val, val, g); 2949 } 2950 2951 /* Reduce the pred_esz_masks value simply to reduce the 2952 * size of the code generated here. 2953 */ 2954 psz_mask = MAKE_64BIT_MASK(0, psz * 8); 2955 tcg_gen_andi_i64(val, val, pred_esz_masks[esz] & psz_mask); 2956 2957 tcg_gen_ctpop_i64(val, val); 2958 } else { 2959 TCGv_ptr t_pn = tcg_temp_new_ptr(); 2960 TCGv_ptr t_pg = tcg_temp_new_ptr(); 2961 unsigned desc = 0; 2962 2963 desc = FIELD_DP32(desc, PREDDESC, OPRSZ, psz); 2964 desc = FIELD_DP32(desc, PREDDESC, ESZ, esz); 2965 2966 tcg_gen_addi_ptr(t_pn, tcg_env, pred_full_reg_offset(s, pn)); 2967 tcg_gen_addi_ptr(t_pg, tcg_env, pred_full_reg_offset(s, pg)); 2968 2969 gen_helper_sve_cntp(val, t_pn, t_pg, tcg_constant_i32(desc)); 2970 } 2971 } 2972 2973 static bool trans_CNTP(DisasContext *s, arg_CNTP *a) 2974 { 2975 if (!dc_isar_feature(aa64_sve, s)) { 2976 return false; 2977 } 2978 if (sve_access_check(s)) { 2979 do_cntp(s, cpu_reg(s, a->rd), a->esz, a->rn, a->pg); 2980 } 2981 return true; 2982 } 2983 2984 static bool trans_INCDECP_r(DisasContext *s, arg_incdec_pred *a) 2985 { 2986 if (!dc_isar_feature(aa64_sve, s)) { 2987 return false; 2988 } 2989 if (sve_access_check(s)) { 2990 TCGv_i64 reg = cpu_reg(s, a->rd); 2991 TCGv_i64 val = tcg_temp_new_i64(); 2992 2993 do_cntp(s, val, a->esz, a->pg, a->pg); 2994 if (a->d) { 2995 tcg_gen_sub_i64(reg, reg, val); 2996 } else { 2997 tcg_gen_add_i64(reg, reg, val); 2998 } 2999 } 3000 return true; 3001 } 3002 3003 static bool trans_INCDECP_z(DisasContext *s, arg_incdec2_pred *a) 3004 { 3005 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) { 3006 return false; 3007 } 3008 if (sve_access_check(s)) { 3009 unsigned vsz = vec_full_reg_size(s); 3010 TCGv_i64 val = tcg_temp_new_i64(); 3011 GVecGen2sFn *gvec_fn = a->d ? tcg_gen_gvec_subs : tcg_gen_gvec_adds; 3012 3013 do_cntp(s, val, a->esz, a->pg, a->pg); 3014 gvec_fn(a->esz, vec_full_reg_offset(s, a->rd), 3015 vec_full_reg_offset(s, a->rn), val, vsz, vsz); 3016 } 3017 return true; 3018 } 3019 3020 static bool trans_SINCDECP_r_32(DisasContext *s, arg_incdec_pred *a) 3021 { 3022 if (!dc_isar_feature(aa64_sve, s)) { 3023 return false; 3024 } 3025 if (sve_access_check(s)) { 3026 TCGv_i64 reg = cpu_reg(s, a->rd); 3027 TCGv_i64 val = tcg_temp_new_i64(); 3028 3029 do_cntp(s, val, a->esz, a->pg, a->pg); 3030 do_sat_addsub_32(reg, val, a->u, a->d); 3031 } 3032 return true; 3033 } 3034 3035 static bool trans_SINCDECP_r_64(DisasContext *s, arg_incdec_pred *a) 3036 { 3037 if (!dc_isar_feature(aa64_sve, s)) { 3038 return false; 3039 } 3040 if (sve_access_check(s)) { 3041 TCGv_i64 reg = cpu_reg(s, a->rd); 3042 TCGv_i64 val = tcg_temp_new_i64(); 3043 3044 do_cntp(s, val, a->esz, a->pg, a->pg); 3045 do_sat_addsub_64(reg, val, a->u, a->d); 3046 } 3047 return true; 3048 } 3049 3050 static bool trans_SINCDECP_z(DisasContext *s, arg_incdec2_pred *a) 3051 { 3052 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) { 3053 return false; 3054 } 3055 if (sve_access_check(s)) { 3056 TCGv_i64 val = tcg_temp_new_i64(); 3057 do_cntp(s, val, a->esz, a->pg, a->pg); 3058 do_sat_addsub_vec(s, a->esz, a->rd, a->rn, val, a->u, a->d); 3059 } 3060 return true; 3061 } 3062 3063 /* 3064 *** SVE Integer Compare Scalars Group 3065 */ 3066 3067 static bool trans_CTERM(DisasContext *s, arg_CTERM *a) 3068 { 3069 if (!dc_isar_feature(aa64_sve, s)) { 3070 return false; 3071 } 3072 if (!sve_access_check(s)) { 3073 return true; 3074 } 3075 3076 TCGCond cond = (a->ne ? TCG_COND_NE : TCG_COND_EQ); 3077 TCGv_i64 rn = read_cpu_reg(s, a->rn, a->sf); 3078 TCGv_i64 rm = read_cpu_reg(s, a->rm, a->sf); 3079 TCGv_i64 cmp = tcg_temp_new_i64(); 3080 3081 tcg_gen_setcond_i64(cond, cmp, rn, rm); 3082 tcg_gen_extrl_i64_i32(cpu_NF, cmp); 3083 3084 /* VF = !NF & !CF. */ 3085 tcg_gen_xori_i32(cpu_VF, cpu_NF, 1); 3086 tcg_gen_andc_i32(cpu_VF, cpu_VF, cpu_CF); 3087 3088 /* Both NF and VF actually look at bit 31. */ 3089 tcg_gen_neg_i32(cpu_NF, cpu_NF); 3090 tcg_gen_neg_i32(cpu_VF, cpu_VF); 3091 return true; 3092 } 3093 3094 static bool trans_WHILE(DisasContext *s, arg_WHILE *a) 3095 { 3096 TCGv_i64 op0, op1, t0, t1, tmax; 3097 TCGv_i32 t2; 3098 TCGv_ptr ptr; 3099 unsigned vsz = vec_full_reg_size(s); 3100 unsigned desc = 0; 3101 TCGCond cond; 3102 uint64_t maxval; 3103 /* Note that GE/HS has a->eq == 0 and GT/HI has a->eq == 1. */ 3104 bool eq = a->eq == a->lt; 3105 3106 /* The greater-than conditions are all SVE2. */ 3107 if (a->lt 3108 ? !dc_isar_feature(aa64_sve, s) 3109 : !dc_isar_feature(aa64_sve2, s)) { 3110 return false; 3111 } 3112 if (!sve_access_check(s)) { 3113 return true; 3114 } 3115 3116 op0 = read_cpu_reg(s, a->rn, 1); 3117 op1 = read_cpu_reg(s, a->rm, 1); 3118 3119 if (!a->sf) { 3120 if (a->u) { 3121 tcg_gen_ext32u_i64(op0, op0); 3122 tcg_gen_ext32u_i64(op1, op1); 3123 } else { 3124 tcg_gen_ext32s_i64(op0, op0); 3125 tcg_gen_ext32s_i64(op1, op1); 3126 } 3127 } 3128 3129 /* For the helper, compress the different conditions into a computation 3130 * of how many iterations for which the condition is true. 3131 */ 3132 t0 = tcg_temp_new_i64(); 3133 t1 = tcg_temp_new_i64(); 3134 3135 if (a->lt) { 3136 tcg_gen_sub_i64(t0, op1, op0); 3137 if (a->u) { 3138 maxval = a->sf ? UINT64_MAX : UINT32_MAX; 3139 cond = eq ? TCG_COND_LEU : TCG_COND_LTU; 3140 } else { 3141 maxval = a->sf ? INT64_MAX : INT32_MAX; 3142 cond = eq ? TCG_COND_LE : TCG_COND_LT; 3143 } 3144 } else { 3145 tcg_gen_sub_i64(t0, op0, op1); 3146 if (a->u) { 3147 maxval = 0; 3148 cond = eq ? TCG_COND_GEU : TCG_COND_GTU; 3149 } else { 3150 maxval = a->sf ? INT64_MIN : INT32_MIN; 3151 cond = eq ? TCG_COND_GE : TCG_COND_GT; 3152 } 3153 } 3154 3155 tmax = tcg_constant_i64(vsz >> a->esz); 3156 if (eq) { 3157 /* Equality means one more iteration. */ 3158 tcg_gen_addi_i64(t0, t0, 1); 3159 3160 /* 3161 * For the less-than while, if op1 is maxval (and the only time 3162 * the addition above could overflow), then we produce an all-true 3163 * predicate by setting the count to the vector length. This is 3164 * because the pseudocode is described as an increment + compare 3165 * loop, and the maximum integer would always compare true. 3166 * Similarly, the greater-than while has the same issue with the 3167 * minimum integer due to the decrement + compare loop. 3168 */ 3169 tcg_gen_movi_i64(t1, maxval); 3170 tcg_gen_movcond_i64(TCG_COND_EQ, t0, op1, t1, tmax, t0); 3171 } 3172 3173 /* Bound to the maximum. */ 3174 tcg_gen_umin_i64(t0, t0, tmax); 3175 3176 /* Set the count to zero if the condition is false. */ 3177 tcg_gen_movi_i64(t1, 0); 3178 tcg_gen_movcond_i64(cond, t0, op0, op1, t0, t1); 3179 3180 /* Since we're bounded, pass as a 32-bit type. */ 3181 t2 = tcg_temp_new_i32(); 3182 tcg_gen_extrl_i64_i32(t2, t0); 3183 3184 /* Scale elements to bits. */ 3185 tcg_gen_shli_i32(t2, t2, a->esz); 3186 3187 desc = FIELD_DP32(desc, PREDDESC, OPRSZ, vsz / 8); 3188 desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz); 3189 3190 ptr = tcg_temp_new_ptr(); 3191 tcg_gen_addi_ptr(ptr, tcg_env, pred_full_reg_offset(s, a->rd)); 3192 3193 if (a->lt) { 3194 gen_helper_sve_whilel(t2, ptr, t2, tcg_constant_i32(desc)); 3195 } else { 3196 gen_helper_sve_whileg(t2, ptr, t2, tcg_constant_i32(desc)); 3197 } 3198 do_pred_flags(t2); 3199 return true; 3200 } 3201 3202 static bool trans_WHILE_ptr(DisasContext *s, arg_WHILE_ptr *a) 3203 { 3204 TCGv_i64 op0, op1, diff, t1, tmax; 3205 TCGv_i32 t2; 3206 TCGv_ptr ptr; 3207 unsigned vsz = vec_full_reg_size(s); 3208 unsigned desc = 0; 3209 3210 if (!dc_isar_feature(aa64_sve2, s)) { 3211 return false; 3212 } 3213 if (!sve_access_check(s)) { 3214 return true; 3215 } 3216 3217 op0 = read_cpu_reg(s, a->rn, 1); 3218 op1 = read_cpu_reg(s, a->rm, 1); 3219 3220 tmax = tcg_constant_i64(vsz); 3221 diff = tcg_temp_new_i64(); 3222 3223 if (a->rw) { 3224 /* WHILERW */ 3225 /* diff = abs(op1 - op0), noting that op0/1 are unsigned. */ 3226 t1 = tcg_temp_new_i64(); 3227 tcg_gen_sub_i64(diff, op0, op1); 3228 tcg_gen_sub_i64(t1, op1, op0); 3229 tcg_gen_movcond_i64(TCG_COND_GEU, diff, op0, op1, diff, t1); 3230 /* Round down to a multiple of ESIZE. */ 3231 tcg_gen_andi_i64(diff, diff, -1 << a->esz); 3232 /* If op1 == op0, diff == 0, and the condition is always true. */ 3233 tcg_gen_movcond_i64(TCG_COND_EQ, diff, op0, op1, tmax, diff); 3234 } else { 3235 /* WHILEWR */ 3236 tcg_gen_sub_i64(diff, op1, op0); 3237 /* Round down to a multiple of ESIZE. */ 3238 tcg_gen_andi_i64(diff, diff, -1 << a->esz); 3239 /* If op0 >= op1, diff <= 0, the condition is always true. */ 3240 tcg_gen_movcond_i64(TCG_COND_GEU, diff, op0, op1, tmax, diff); 3241 } 3242 3243 /* Bound to the maximum. */ 3244 tcg_gen_umin_i64(diff, diff, tmax); 3245 3246 /* Since we're bounded, pass as a 32-bit type. */ 3247 t2 = tcg_temp_new_i32(); 3248 tcg_gen_extrl_i64_i32(t2, diff); 3249 3250 desc = FIELD_DP32(desc, PREDDESC, OPRSZ, vsz / 8); 3251 desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz); 3252 3253 ptr = tcg_temp_new_ptr(); 3254 tcg_gen_addi_ptr(ptr, tcg_env, pred_full_reg_offset(s, a->rd)); 3255 3256 gen_helper_sve_whilel(t2, ptr, t2, tcg_constant_i32(desc)); 3257 do_pred_flags(t2); 3258 return true; 3259 } 3260 3261 /* 3262 *** SVE Integer Wide Immediate - Unpredicated Group 3263 */ 3264 3265 static bool trans_FDUP(DisasContext *s, arg_FDUP *a) 3266 { 3267 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) { 3268 return false; 3269 } 3270 if (sve_access_check(s)) { 3271 unsigned vsz = vec_full_reg_size(s); 3272 int dofs = vec_full_reg_offset(s, a->rd); 3273 uint64_t imm; 3274 3275 /* Decode the VFP immediate. */ 3276 imm = vfp_expand_imm(a->esz, a->imm); 3277 tcg_gen_gvec_dup_imm(a->esz, dofs, vsz, vsz, imm); 3278 } 3279 return true; 3280 } 3281 3282 static bool trans_DUP_i(DisasContext *s, arg_DUP_i *a) 3283 { 3284 if (!dc_isar_feature(aa64_sve, s)) { 3285 return false; 3286 } 3287 if (sve_access_check(s)) { 3288 unsigned vsz = vec_full_reg_size(s); 3289 int dofs = vec_full_reg_offset(s, a->rd); 3290 tcg_gen_gvec_dup_imm(a->esz, dofs, vsz, vsz, a->imm); 3291 } 3292 return true; 3293 } 3294 3295 TRANS_FEAT(ADD_zzi, aa64_sve, gen_gvec_fn_arg_zzi, tcg_gen_gvec_addi, a) 3296 3297 static bool trans_SUB_zzi(DisasContext *s, arg_rri_esz *a) 3298 { 3299 a->imm = -a->imm; 3300 return trans_ADD_zzi(s, a); 3301 } 3302 3303 static bool trans_SUBR_zzi(DisasContext *s, arg_rri_esz *a) 3304 { 3305 static const TCGOpcode vecop_list[] = { INDEX_op_sub_vec, 0 }; 3306 static const GVecGen2s op[4] = { 3307 { .fni8 = tcg_gen_vec_sub8_i64, 3308 .fniv = tcg_gen_sub_vec, 3309 .fno = gen_helper_sve_subri_b, 3310 .opt_opc = vecop_list, 3311 .vece = MO_8, 3312 .scalar_first = true }, 3313 { .fni8 = tcg_gen_vec_sub16_i64, 3314 .fniv = tcg_gen_sub_vec, 3315 .fno = gen_helper_sve_subri_h, 3316 .opt_opc = vecop_list, 3317 .vece = MO_16, 3318 .scalar_first = true }, 3319 { .fni4 = tcg_gen_sub_i32, 3320 .fniv = tcg_gen_sub_vec, 3321 .fno = gen_helper_sve_subri_s, 3322 .opt_opc = vecop_list, 3323 .vece = MO_32, 3324 .scalar_first = true }, 3325 { .fni8 = tcg_gen_sub_i64, 3326 .fniv = tcg_gen_sub_vec, 3327 .fno = gen_helper_sve_subri_d, 3328 .opt_opc = vecop_list, 3329 .prefer_i64 = TCG_TARGET_REG_BITS == 64, 3330 .vece = MO_64, 3331 .scalar_first = true } 3332 }; 3333 3334 if (!dc_isar_feature(aa64_sve, s)) { 3335 return false; 3336 } 3337 if (sve_access_check(s)) { 3338 unsigned vsz = vec_full_reg_size(s); 3339 tcg_gen_gvec_2s(vec_full_reg_offset(s, a->rd), 3340 vec_full_reg_offset(s, a->rn), 3341 vsz, vsz, tcg_constant_i64(a->imm), &op[a->esz]); 3342 } 3343 return true; 3344 } 3345 3346 TRANS_FEAT(MUL_zzi, aa64_sve, gen_gvec_fn_arg_zzi, tcg_gen_gvec_muli, a) 3347 3348 static bool do_zzi_sat(DisasContext *s, arg_rri_esz *a, bool u, bool d) 3349 { 3350 if (sve_access_check(s)) { 3351 do_sat_addsub_vec(s, a->esz, a->rd, a->rn, 3352 tcg_constant_i64(a->imm), u, d); 3353 } 3354 return true; 3355 } 3356 3357 TRANS_FEAT(SQADD_zzi, aa64_sve, do_zzi_sat, a, false, false) 3358 TRANS_FEAT(UQADD_zzi, aa64_sve, do_zzi_sat, a, true, false) 3359 TRANS_FEAT(SQSUB_zzi, aa64_sve, do_zzi_sat, a, false, true) 3360 TRANS_FEAT(UQSUB_zzi, aa64_sve, do_zzi_sat, a, true, true) 3361 3362 static bool do_zzi_ool(DisasContext *s, arg_rri_esz *a, gen_helper_gvec_2i *fn) 3363 { 3364 if (sve_access_check(s)) { 3365 unsigned vsz = vec_full_reg_size(s); 3366 tcg_gen_gvec_2i_ool(vec_full_reg_offset(s, a->rd), 3367 vec_full_reg_offset(s, a->rn), 3368 tcg_constant_i64(a->imm), vsz, vsz, 0, fn); 3369 } 3370 return true; 3371 } 3372 3373 #define DO_ZZI(NAME, name) \ 3374 static gen_helper_gvec_2i * const name##i_fns[4] = { \ 3375 gen_helper_sve_##name##i_b, gen_helper_sve_##name##i_h, \ 3376 gen_helper_sve_##name##i_s, gen_helper_sve_##name##i_d, \ 3377 }; \ 3378 TRANS_FEAT(NAME##_zzi, aa64_sve, do_zzi_ool, a, name##i_fns[a->esz]) 3379 3380 DO_ZZI(SMAX, smax) 3381 DO_ZZI(UMAX, umax) 3382 DO_ZZI(SMIN, smin) 3383 DO_ZZI(UMIN, umin) 3384 3385 #undef DO_ZZI 3386 3387 static gen_helper_gvec_4 * const dot_fns[2][2] = { 3388 { gen_helper_gvec_sdot_b, gen_helper_gvec_sdot_h }, 3389 { gen_helper_gvec_udot_b, gen_helper_gvec_udot_h } 3390 }; 3391 TRANS_FEAT(DOT_zzzz, aa64_sve, gen_gvec_ool_zzzz, 3392 dot_fns[a->u][a->sz], a->rd, a->rn, a->rm, a->ra, 0) 3393 3394 /* 3395 * SVE Multiply - Indexed 3396 */ 3397 3398 TRANS_FEAT(SDOT_zzxw_s, aa64_sve, gen_gvec_ool_arg_zzxz, 3399 gen_helper_gvec_sdot_idx_b, a) 3400 TRANS_FEAT(SDOT_zzxw_d, aa64_sve, gen_gvec_ool_arg_zzxz, 3401 gen_helper_gvec_sdot_idx_h, a) 3402 TRANS_FEAT(UDOT_zzxw_s, aa64_sve, gen_gvec_ool_arg_zzxz, 3403 gen_helper_gvec_udot_idx_b, a) 3404 TRANS_FEAT(UDOT_zzxw_d, aa64_sve, gen_gvec_ool_arg_zzxz, 3405 gen_helper_gvec_udot_idx_h, a) 3406 3407 TRANS_FEAT(SUDOT_zzxw_s, aa64_sve_i8mm, gen_gvec_ool_arg_zzxz, 3408 gen_helper_gvec_sudot_idx_b, a) 3409 TRANS_FEAT(USDOT_zzxw_s, aa64_sve_i8mm, gen_gvec_ool_arg_zzxz, 3410 gen_helper_gvec_usdot_idx_b, a) 3411 3412 #define DO_SVE2_RRX(NAME, FUNC) \ 3413 TRANS_FEAT(NAME, aa64_sve, gen_gvec_ool_zzz, FUNC, \ 3414 a->rd, a->rn, a->rm, a->index) 3415 3416 DO_SVE2_RRX(MUL_zzx_h, gen_helper_gvec_mul_idx_h) 3417 DO_SVE2_RRX(MUL_zzx_s, gen_helper_gvec_mul_idx_s) 3418 DO_SVE2_RRX(MUL_zzx_d, gen_helper_gvec_mul_idx_d) 3419 3420 DO_SVE2_RRX(SQDMULH_zzx_h, gen_helper_sve2_sqdmulh_idx_h) 3421 DO_SVE2_RRX(SQDMULH_zzx_s, gen_helper_sve2_sqdmulh_idx_s) 3422 DO_SVE2_RRX(SQDMULH_zzx_d, gen_helper_sve2_sqdmulh_idx_d) 3423 3424 DO_SVE2_RRX(SQRDMULH_zzx_h, gen_helper_sve2_sqrdmulh_idx_h) 3425 DO_SVE2_RRX(SQRDMULH_zzx_s, gen_helper_sve2_sqrdmulh_idx_s) 3426 DO_SVE2_RRX(SQRDMULH_zzx_d, gen_helper_sve2_sqrdmulh_idx_d) 3427 3428 #undef DO_SVE2_RRX 3429 3430 #define DO_SVE2_RRX_TB(NAME, FUNC, TOP) \ 3431 TRANS_FEAT(NAME, aa64_sve, gen_gvec_ool_zzz, FUNC, \ 3432 a->rd, a->rn, a->rm, (a->index << 1) | TOP) 3433 3434 DO_SVE2_RRX_TB(SQDMULLB_zzx_s, gen_helper_sve2_sqdmull_idx_s, false) 3435 DO_SVE2_RRX_TB(SQDMULLB_zzx_d, gen_helper_sve2_sqdmull_idx_d, false) 3436 DO_SVE2_RRX_TB(SQDMULLT_zzx_s, gen_helper_sve2_sqdmull_idx_s, true) 3437 DO_SVE2_RRX_TB(SQDMULLT_zzx_d, gen_helper_sve2_sqdmull_idx_d, true) 3438 3439 DO_SVE2_RRX_TB(SMULLB_zzx_s, gen_helper_sve2_smull_idx_s, false) 3440 DO_SVE2_RRX_TB(SMULLB_zzx_d, gen_helper_sve2_smull_idx_d, false) 3441 DO_SVE2_RRX_TB(SMULLT_zzx_s, gen_helper_sve2_smull_idx_s, true) 3442 DO_SVE2_RRX_TB(SMULLT_zzx_d, gen_helper_sve2_smull_idx_d, true) 3443 3444 DO_SVE2_RRX_TB(UMULLB_zzx_s, gen_helper_sve2_umull_idx_s, false) 3445 DO_SVE2_RRX_TB(UMULLB_zzx_d, gen_helper_sve2_umull_idx_d, false) 3446 DO_SVE2_RRX_TB(UMULLT_zzx_s, gen_helper_sve2_umull_idx_s, true) 3447 DO_SVE2_RRX_TB(UMULLT_zzx_d, gen_helper_sve2_umull_idx_d, true) 3448 3449 #undef DO_SVE2_RRX_TB 3450 3451 #define DO_SVE2_RRXR(NAME, FUNC) \ 3452 TRANS_FEAT(NAME, aa64_sve2, gen_gvec_ool_arg_zzxz, FUNC, a) 3453 3454 DO_SVE2_RRXR(MLA_zzxz_h, gen_helper_gvec_mla_idx_h) 3455 DO_SVE2_RRXR(MLA_zzxz_s, gen_helper_gvec_mla_idx_s) 3456 DO_SVE2_RRXR(MLA_zzxz_d, gen_helper_gvec_mla_idx_d) 3457 3458 DO_SVE2_RRXR(MLS_zzxz_h, gen_helper_gvec_mls_idx_h) 3459 DO_SVE2_RRXR(MLS_zzxz_s, gen_helper_gvec_mls_idx_s) 3460 DO_SVE2_RRXR(MLS_zzxz_d, gen_helper_gvec_mls_idx_d) 3461 3462 DO_SVE2_RRXR(SQRDMLAH_zzxz_h, gen_helper_sve2_sqrdmlah_idx_h) 3463 DO_SVE2_RRXR(SQRDMLAH_zzxz_s, gen_helper_sve2_sqrdmlah_idx_s) 3464 DO_SVE2_RRXR(SQRDMLAH_zzxz_d, gen_helper_sve2_sqrdmlah_idx_d) 3465 3466 DO_SVE2_RRXR(SQRDMLSH_zzxz_h, gen_helper_sve2_sqrdmlsh_idx_h) 3467 DO_SVE2_RRXR(SQRDMLSH_zzxz_s, gen_helper_sve2_sqrdmlsh_idx_s) 3468 DO_SVE2_RRXR(SQRDMLSH_zzxz_d, gen_helper_sve2_sqrdmlsh_idx_d) 3469 3470 #undef DO_SVE2_RRXR 3471 3472 #define DO_SVE2_RRXR_TB(NAME, FUNC, TOP) \ 3473 TRANS_FEAT(NAME, aa64_sve2, gen_gvec_ool_zzzz, FUNC, \ 3474 a->rd, a->rn, a->rm, a->ra, (a->index << 1) | TOP) 3475 3476 DO_SVE2_RRXR_TB(SQDMLALB_zzxw_s, gen_helper_sve2_sqdmlal_idx_s, false) 3477 DO_SVE2_RRXR_TB(SQDMLALB_zzxw_d, gen_helper_sve2_sqdmlal_idx_d, false) 3478 DO_SVE2_RRXR_TB(SQDMLALT_zzxw_s, gen_helper_sve2_sqdmlal_idx_s, true) 3479 DO_SVE2_RRXR_TB(SQDMLALT_zzxw_d, gen_helper_sve2_sqdmlal_idx_d, true) 3480 3481 DO_SVE2_RRXR_TB(SQDMLSLB_zzxw_s, gen_helper_sve2_sqdmlsl_idx_s, false) 3482 DO_SVE2_RRXR_TB(SQDMLSLB_zzxw_d, gen_helper_sve2_sqdmlsl_idx_d, false) 3483 DO_SVE2_RRXR_TB(SQDMLSLT_zzxw_s, gen_helper_sve2_sqdmlsl_idx_s, true) 3484 DO_SVE2_RRXR_TB(SQDMLSLT_zzxw_d, gen_helper_sve2_sqdmlsl_idx_d, true) 3485 3486 DO_SVE2_RRXR_TB(SMLALB_zzxw_s, gen_helper_sve2_smlal_idx_s, false) 3487 DO_SVE2_RRXR_TB(SMLALB_zzxw_d, gen_helper_sve2_smlal_idx_d, false) 3488 DO_SVE2_RRXR_TB(SMLALT_zzxw_s, gen_helper_sve2_smlal_idx_s, true) 3489 DO_SVE2_RRXR_TB(SMLALT_zzxw_d, gen_helper_sve2_smlal_idx_d, true) 3490 3491 DO_SVE2_RRXR_TB(UMLALB_zzxw_s, gen_helper_sve2_umlal_idx_s, false) 3492 DO_SVE2_RRXR_TB(UMLALB_zzxw_d, gen_helper_sve2_umlal_idx_d, false) 3493 DO_SVE2_RRXR_TB(UMLALT_zzxw_s, gen_helper_sve2_umlal_idx_s, true) 3494 DO_SVE2_RRXR_TB(UMLALT_zzxw_d, gen_helper_sve2_umlal_idx_d, true) 3495 3496 DO_SVE2_RRXR_TB(SMLSLB_zzxw_s, gen_helper_sve2_smlsl_idx_s, false) 3497 DO_SVE2_RRXR_TB(SMLSLB_zzxw_d, gen_helper_sve2_smlsl_idx_d, false) 3498 DO_SVE2_RRXR_TB(SMLSLT_zzxw_s, gen_helper_sve2_smlsl_idx_s, true) 3499 DO_SVE2_RRXR_TB(SMLSLT_zzxw_d, gen_helper_sve2_smlsl_idx_d, true) 3500 3501 DO_SVE2_RRXR_TB(UMLSLB_zzxw_s, gen_helper_sve2_umlsl_idx_s, false) 3502 DO_SVE2_RRXR_TB(UMLSLB_zzxw_d, gen_helper_sve2_umlsl_idx_d, false) 3503 DO_SVE2_RRXR_TB(UMLSLT_zzxw_s, gen_helper_sve2_umlsl_idx_s, true) 3504 DO_SVE2_RRXR_TB(UMLSLT_zzxw_d, gen_helper_sve2_umlsl_idx_d, true) 3505 3506 #undef DO_SVE2_RRXR_TB 3507 3508 #define DO_SVE2_RRXR_ROT(NAME, FUNC) \ 3509 TRANS_FEAT(NAME, aa64_sve2, gen_gvec_ool_zzzz, FUNC, \ 3510 a->rd, a->rn, a->rm, a->ra, (a->index << 2) | a->rot) 3511 3512 DO_SVE2_RRXR_ROT(CMLA_zzxz_h, gen_helper_sve2_cmla_idx_h) 3513 DO_SVE2_RRXR_ROT(CMLA_zzxz_s, gen_helper_sve2_cmla_idx_s) 3514 3515 DO_SVE2_RRXR_ROT(SQRDCMLAH_zzxz_h, gen_helper_sve2_sqrdcmlah_idx_h) 3516 DO_SVE2_RRXR_ROT(SQRDCMLAH_zzxz_s, gen_helper_sve2_sqrdcmlah_idx_s) 3517 3518 DO_SVE2_RRXR_ROT(CDOT_zzxw_s, gen_helper_sve2_cdot_idx_s) 3519 DO_SVE2_RRXR_ROT(CDOT_zzxw_d, gen_helper_sve2_cdot_idx_d) 3520 3521 #undef DO_SVE2_RRXR_ROT 3522 3523 /* 3524 *** SVE Floating Point Multiply-Add Indexed Group 3525 */ 3526 3527 static gen_helper_gvec_4_ptr * const fmla_idx_fns[4] = { 3528 NULL, gen_helper_gvec_fmla_idx_h, 3529 gen_helper_gvec_fmla_idx_s, gen_helper_gvec_fmla_idx_d 3530 }; 3531 TRANS_FEAT(FMLA_zzxz, aa64_sve, gen_gvec_fpst_zzzz, 3532 fmla_idx_fns[a->esz], a->rd, a->rn, a->rm, a->ra, a->index, 3533 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64) 3534 3535 static gen_helper_gvec_4_ptr * const fmls_idx_fns[4][2] = { 3536 { NULL, NULL }, 3537 { gen_helper_gvec_fmls_idx_h, gen_helper_gvec_ah_fmls_idx_h }, 3538 { gen_helper_gvec_fmls_idx_s, gen_helper_gvec_ah_fmls_idx_s }, 3539 { gen_helper_gvec_fmls_idx_d, gen_helper_gvec_ah_fmls_idx_d }, 3540 }; 3541 TRANS_FEAT(FMLS_zzxz, aa64_sve, gen_gvec_fpst_zzzz, 3542 fmls_idx_fns[a->esz][s->fpcr_ah], 3543 a->rd, a->rn, a->rm, a->ra, a->index, 3544 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64) 3545 3546 /* 3547 *** SVE Floating Point Multiply Indexed Group 3548 */ 3549 3550 static gen_helper_gvec_3_ptr * const fmul_idx_fns[4] = { 3551 NULL, gen_helper_gvec_fmul_idx_h, 3552 gen_helper_gvec_fmul_idx_s, gen_helper_gvec_fmul_idx_d, 3553 }; 3554 TRANS_FEAT(FMUL_zzx, aa64_sve, gen_gvec_fpst_zzz, 3555 fmul_idx_fns[a->esz], a->rd, a->rn, a->rm, a->index, 3556 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64) 3557 3558 /* 3559 *** SVE Floating Point Fast Reduction Group 3560 */ 3561 3562 typedef void gen_helper_fp_reduce(TCGv_i64, TCGv_ptr, TCGv_ptr, 3563 TCGv_ptr, TCGv_i32); 3564 3565 static bool do_reduce(DisasContext *s, arg_rpr_esz *a, 3566 gen_helper_fp_reduce *fn) 3567 { 3568 unsigned vsz, p2vsz; 3569 TCGv_i32 t_desc; 3570 TCGv_ptr t_zn, t_pg, status; 3571 TCGv_i64 temp; 3572 3573 if (fn == NULL) { 3574 return false; 3575 } 3576 if (!sve_access_check(s)) { 3577 return true; 3578 } 3579 3580 vsz = vec_full_reg_size(s); 3581 p2vsz = pow2ceil(vsz); 3582 t_desc = tcg_constant_i32(simd_desc(vsz, vsz, p2vsz)); 3583 temp = tcg_temp_new_i64(); 3584 t_zn = tcg_temp_new_ptr(); 3585 t_pg = tcg_temp_new_ptr(); 3586 3587 tcg_gen_addi_ptr(t_zn, tcg_env, vec_full_reg_offset(s, a->rn)); 3588 tcg_gen_addi_ptr(t_pg, tcg_env, pred_full_reg_offset(s, a->pg)); 3589 status = fpstatus_ptr(a->esz == MO_16 ? FPST_A64_F16 : FPST_A64); 3590 3591 fn(temp, t_zn, t_pg, status, t_desc); 3592 3593 write_fp_dreg(s, a->rd, temp); 3594 return true; 3595 } 3596 3597 #define DO_VPZ(NAME, name) \ 3598 static gen_helper_fp_reduce * const name##_fns[4] = { \ 3599 NULL, gen_helper_sve_##name##_h, \ 3600 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \ 3601 }; \ 3602 TRANS_FEAT(NAME, aa64_sve, do_reduce, a, name##_fns[a->esz]) 3603 3604 #define DO_VPZ_AH(NAME, name) \ 3605 static gen_helper_fp_reduce * const name##_fns[4] = { \ 3606 NULL, gen_helper_sve_##name##_h, \ 3607 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \ 3608 }; \ 3609 static gen_helper_fp_reduce * const name##_ah_fns[4] = { \ 3610 NULL, gen_helper_sve_ah_##name##_h, \ 3611 gen_helper_sve_ah_##name##_s, gen_helper_sve_ah_##name##_d, \ 3612 }; \ 3613 TRANS_FEAT(NAME, aa64_sve, do_reduce, a, \ 3614 s->fpcr_ah ? name##_ah_fns[a->esz] : name##_fns[a->esz]) 3615 3616 DO_VPZ(FADDV, faddv) 3617 DO_VPZ(FMINNMV, fminnmv) 3618 DO_VPZ(FMAXNMV, fmaxnmv) 3619 DO_VPZ_AH(FMINV, fminv) 3620 DO_VPZ_AH(FMAXV, fmaxv) 3621 3622 #undef DO_VPZ 3623 3624 /* 3625 *** SVE Floating Point Unary Operations - Unpredicated Group 3626 */ 3627 3628 static gen_helper_gvec_2_ptr * const frecpe_fns[] = { 3629 NULL, gen_helper_gvec_frecpe_h, 3630 gen_helper_gvec_frecpe_s, gen_helper_gvec_frecpe_d, 3631 }; 3632 static gen_helper_gvec_2_ptr * const frecpe_rpres_fns[] = { 3633 NULL, gen_helper_gvec_frecpe_h, 3634 gen_helper_gvec_frecpe_rpres_s, gen_helper_gvec_frecpe_d, 3635 }; 3636 TRANS_FEAT(FRECPE, aa64_sve, gen_gvec_fpst_ah_arg_zz, 3637 s->fpcr_ah && dc_isar_feature(aa64_rpres, s) ? 3638 frecpe_rpres_fns[a->esz] : frecpe_fns[a->esz], a, 0) 3639 3640 static gen_helper_gvec_2_ptr * const frsqrte_fns[] = { 3641 NULL, gen_helper_gvec_frsqrte_h, 3642 gen_helper_gvec_frsqrte_s, gen_helper_gvec_frsqrte_d, 3643 }; 3644 static gen_helper_gvec_2_ptr * const frsqrte_rpres_fns[] = { 3645 NULL, gen_helper_gvec_frsqrte_h, 3646 gen_helper_gvec_frsqrte_rpres_s, gen_helper_gvec_frsqrte_d, 3647 }; 3648 TRANS_FEAT(FRSQRTE, aa64_sve, gen_gvec_fpst_ah_arg_zz, 3649 s->fpcr_ah && dc_isar_feature(aa64_rpres, s) ? 3650 frsqrte_rpres_fns[a->esz] : frsqrte_fns[a->esz], a, 0) 3651 3652 /* 3653 *** SVE Floating Point Compare with Zero Group 3654 */ 3655 3656 static bool do_ppz_fp(DisasContext *s, arg_rpr_esz *a, 3657 gen_helper_gvec_3_ptr *fn) 3658 { 3659 if (fn == NULL) { 3660 return false; 3661 } 3662 if (sve_access_check(s)) { 3663 unsigned vsz = vec_full_reg_size(s); 3664 TCGv_ptr status = 3665 fpstatus_ptr(a->esz == MO_16 ? FPST_A64_F16 : FPST_A64); 3666 3667 tcg_gen_gvec_3_ptr(pred_full_reg_offset(s, a->rd), 3668 vec_full_reg_offset(s, a->rn), 3669 pred_full_reg_offset(s, a->pg), 3670 status, vsz, vsz, 0, fn); 3671 } 3672 return true; 3673 } 3674 3675 #define DO_PPZ(NAME, name) \ 3676 static gen_helper_gvec_3_ptr * const name##_fns[] = { \ 3677 NULL, gen_helper_sve_##name##_h, \ 3678 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \ 3679 }; \ 3680 TRANS_FEAT(NAME, aa64_sve, do_ppz_fp, a, name##_fns[a->esz]) 3681 3682 DO_PPZ(FCMGE_ppz0, fcmge0) 3683 DO_PPZ(FCMGT_ppz0, fcmgt0) 3684 DO_PPZ(FCMLE_ppz0, fcmle0) 3685 DO_PPZ(FCMLT_ppz0, fcmlt0) 3686 DO_PPZ(FCMEQ_ppz0, fcmeq0) 3687 DO_PPZ(FCMNE_ppz0, fcmne0) 3688 3689 #undef DO_PPZ 3690 3691 /* 3692 *** SVE floating-point trig multiply-add coefficient 3693 */ 3694 3695 static gen_helper_gvec_3_ptr * const ftmad_fns[4] = { 3696 NULL, gen_helper_sve_ftmad_h, 3697 gen_helper_sve_ftmad_s, gen_helper_sve_ftmad_d, 3698 }; 3699 TRANS_FEAT_NONSTREAMING(FTMAD, aa64_sve, gen_gvec_fpst_zzz, 3700 ftmad_fns[a->esz], a->rd, a->rn, a->rm, 3701 a->imm | (s->fpcr_ah << 3), 3702 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64) 3703 3704 /* 3705 *** SVE Floating Point Accumulating Reduction Group 3706 */ 3707 3708 static bool trans_FADDA(DisasContext *s, arg_rprr_esz *a) 3709 { 3710 typedef void fadda_fn(TCGv_i64, TCGv_i64, TCGv_ptr, 3711 TCGv_ptr, TCGv_ptr, TCGv_i32); 3712 static fadda_fn * const fns[3] = { 3713 gen_helper_sve_fadda_h, 3714 gen_helper_sve_fadda_s, 3715 gen_helper_sve_fadda_d, 3716 }; 3717 unsigned vsz = vec_full_reg_size(s); 3718 TCGv_ptr t_rm, t_pg, t_fpst; 3719 TCGv_i64 t_val; 3720 TCGv_i32 t_desc; 3721 3722 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) { 3723 return false; 3724 } 3725 s->is_nonstreaming = true; 3726 if (!sve_access_check(s)) { 3727 return true; 3728 } 3729 3730 t_val = load_esz(tcg_env, vec_reg_offset(s, a->rn, 0, a->esz), a->esz); 3731 t_rm = tcg_temp_new_ptr(); 3732 t_pg = tcg_temp_new_ptr(); 3733 tcg_gen_addi_ptr(t_rm, tcg_env, vec_full_reg_offset(s, a->rm)); 3734 tcg_gen_addi_ptr(t_pg, tcg_env, pred_full_reg_offset(s, a->pg)); 3735 t_fpst = fpstatus_ptr(a->esz == MO_16 ? FPST_A64_F16 : FPST_A64); 3736 t_desc = tcg_constant_i32(simd_desc(vsz, vsz, 0)); 3737 3738 fns[a->esz - 1](t_val, t_val, t_rm, t_pg, t_fpst, t_desc); 3739 3740 write_fp_dreg(s, a->rd, t_val); 3741 return true; 3742 } 3743 3744 /* 3745 *** SVE Floating Point Arithmetic - Unpredicated Group 3746 */ 3747 3748 #define DO_FP3(NAME, name) \ 3749 static gen_helper_gvec_3_ptr * const name##_fns[4] = { \ 3750 NULL, gen_helper_gvec_##name##_h, \ 3751 gen_helper_gvec_##name##_s, gen_helper_gvec_##name##_d \ 3752 }; \ 3753 TRANS_FEAT(NAME, aa64_sve, gen_gvec_fpst_arg_zzz, name##_fns[a->esz], a, 0) 3754 3755 #define DO_FP3_AH(NAME, name) \ 3756 static gen_helper_gvec_3_ptr * const name##_fns[4] = { \ 3757 NULL, gen_helper_gvec_##name##_h, \ 3758 gen_helper_gvec_##name##_s, gen_helper_gvec_##name##_d \ 3759 }; \ 3760 static gen_helper_gvec_3_ptr * const name##_ah_fns[4] = { \ 3761 NULL, gen_helper_gvec_ah_##name##_h, \ 3762 gen_helper_gvec_ah_##name##_s, gen_helper_gvec_ah_##name##_d \ 3763 }; \ 3764 TRANS_FEAT(NAME, aa64_sve, gen_gvec_fpst_ah_arg_zzz, \ 3765 s->fpcr_ah ? name##_ah_fns[a->esz] : name##_fns[a->esz], a, 0) 3766 3767 DO_FP3(FADD_zzz, fadd) 3768 DO_FP3(FSUB_zzz, fsub) 3769 DO_FP3(FMUL_zzz, fmul) 3770 DO_FP3_AH(FRECPS, recps) 3771 DO_FP3_AH(FRSQRTS, rsqrts) 3772 3773 #undef DO_FP3 3774 3775 static gen_helper_gvec_3_ptr * const ftsmul_fns[4] = { 3776 NULL, gen_helper_gvec_ftsmul_h, 3777 gen_helper_gvec_ftsmul_s, gen_helper_gvec_ftsmul_d 3778 }; 3779 TRANS_FEAT_NONSTREAMING(FTSMUL, aa64_sve, gen_gvec_fpst_arg_zzz, 3780 ftsmul_fns[a->esz], a, 0) 3781 3782 /* 3783 *** SVE Floating Point Arithmetic - Predicated Group 3784 */ 3785 3786 #define DO_ZPZZ_FP(NAME, FEAT, name) \ 3787 static gen_helper_gvec_4_ptr * const name##_zpzz_fns[4] = { \ 3788 NULL, gen_helper_##name##_h, \ 3789 gen_helper_##name##_s, gen_helper_##name##_d \ 3790 }; \ 3791 TRANS_FEAT(NAME, FEAT, gen_gvec_fpst_arg_zpzz, name##_zpzz_fns[a->esz], a) 3792 3793 #define DO_ZPZZ_AH_FP(NAME, FEAT, name, ah_name) \ 3794 static gen_helper_gvec_4_ptr * const name##_zpzz_fns[4] = { \ 3795 NULL, gen_helper_##name##_h, \ 3796 gen_helper_##name##_s, gen_helper_##name##_d \ 3797 }; \ 3798 static gen_helper_gvec_4_ptr * const name##_ah_zpzz_fns[4] = { \ 3799 NULL, gen_helper_##ah_name##_h, \ 3800 gen_helper_##ah_name##_s, gen_helper_##ah_name##_d \ 3801 }; \ 3802 TRANS_FEAT(NAME, FEAT, gen_gvec_fpst_arg_zpzz, \ 3803 s->fpcr_ah ? name##_ah_zpzz_fns[a->esz] : \ 3804 name##_zpzz_fns[a->esz], a) 3805 3806 DO_ZPZZ_FP(FADD_zpzz, aa64_sve, sve_fadd) 3807 DO_ZPZZ_FP(FSUB_zpzz, aa64_sve, sve_fsub) 3808 DO_ZPZZ_FP(FMUL_zpzz, aa64_sve, sve_fmul) 3809 DO_ZPZZ_AH_FP(FMIN_zpzz, aa64_sve, sve_fmin, sve_ah_fmin) 3810 DO_ZPZZ_AH_FP(FMAX_zpzz, aa64_sve, sve_fmax, sve_ah_fmax) 3811 DO_ZPZZ_FP(FMINNM_zpzz, aa64_sve, sve_fminnum) 3812 DO_ZPZZ_FP(FMAXNM_zpzz, aa64_sve, sve_fmaxnum) 3813 DO_ZPZZ_AH_FP(FABD, aa64_sve, sve_fabd, sve_ah_fabd) 3814 DO_ZPZZ_FP(FSCALE, aa64_sve, sve_fscalbn) 3815 DO_ZPZZ_FP(FDIV, aa64_sve, sve_fdiv) 3816 DO_ZPZZ_FP(FMULX, aa64_sve, sve_fmulx) 3817 3818 typedef void gen_helper_sve_fp2scalar(TCGv_ptr, TCGv_ptr, TCGv_ptr, 3819 TCGv_i64, TCGv_ptr, TCGv_i32); 3820 3821 static void do_fp_scalar(DisasContext *s, int zd, int zn, int pg, bool is_fp16, 3822 TCGv_i64 scalar, gen_helper_sve_fp2scalar *fn) 3823 { 3824 unsigned vsz = vec_full_reg_size(s); 3825 TCGv_ptr t_zd, t_zn, t_pg, status; 3826 TCGv_i32 desc; 3827 3828 t_zd = tcg_temp_new_ptr(); 3829 t_zn = tcg_temp_new_ptr(); 3830 t_pg = tcg_temp_new_ptr(); 3831 tcg_gen_addi_ptr(t_zd, tcg_env, vec_full_reg_offset(s, zd)); 3832 tcg_gen_addi_ptr(t_zn, tcg_env, vec_full_reg_offset(s, zn)); 3833 tcg_gen_addi_ptr(t_pg, tcg_env, pred_full_reg_offset(s, pg)); 3834 3835 status = fpstatus_ptr(is_fp16 ? FPST_A64_F16 : FPST_A64); 3836 desc = tcg_constant_i32(simd_desc(vsz, vsz, 0)); 3837 fn(t_zd, t_zn, t_pg, scalar, status, desc); 3838 } 3839 3840 static bool do_fp_imm(DisasContext *s, arg_rpri_esz *a, uint64_t imm, 3841 gen_helper_sve_fp2scalar *fn) 3842 { 3843 if (fn == NULL) { 3844 return false; 3845 } 3846 if (sve_access_check(s)) { 3847 do_fp_scalar(s, a->rd, a->rn, a->pg, a->esz == MO_16, 3848 tcg_constant_i64(imm), fn); 3849 } 3850 return true; 3851 } 3852 3853 #define DO_FP_IMM(NAME, name, const0, const1) \ 3854 static gen_helper_sve_fp2scalar * const name##_fns[4] = { \ 3855 NULL, gen_helper_sve_##name##_h, \ 3856 gen_helper_sve_##name##_s, \ 3857 gen_helper_sve_##name##_d \ 3858 }; \ 3859 static uint64_t const name##_const[4][2] = { \ 3860 { -1, -1 }, \ 3861 { float16_##const0, float16_##const1 }, \ 3862 { float32_##const0, float32_##const1 }, \ 3863 { float64_##const0, float64_##const1 }, \ 3864 }; \ 3865 TRANS_FEAT(NAME##_zpzi, aa64_sve, do_fp_imm, a, \ 3866 name##_const[a->esz][a->imm], name##_fns[a->esz]) 3867 3868 #define DO_FP_AH_IMM(NAME, name, const0, const1) \ 3869 static gen_helper_sve_fp2scalar * const name##_fns[4] = { \ 3870 NULL, gen_helper_sve_##name##_h, \ 3871 gen_helper_sve_##name##_s, \ 3872 gen_helper_sve_##name##_d \ 3873 }; \ 3874 static gen_helper_sve_fp2scalar * const name##_ah_fns[4] = { \ 3875 NULL, gen_helper_sve_ah_##name##_h, \ 3876 gen_helper_sve_ah_##name##_s, \ 3877 gen_helper_sve_ah_##name##_d \ 3878 }; \ 3879 static uint64_t const name##_const[4][2] = { \ 3880 { -1, -1 }, \ 3881 { float16_##const0, float16_##const1 }, \ 3882 { float32_##const0, float32_##const1 }, \ 3883 { float64_##const0, float64_##const1 }, \ 3884 }; \ 3885 TRANS_FEAT(NAME##_zpzi, aa64_sve, do_fp_imm, a, \ 3886 name##_const[a->esz][a->imm], \ 3887 s->fpcr_ah ? name##_ah_fns[a->esz] : name##_fns[a->esz]) 3888 3889 DO_FP_IMM(FADD, fadds, half, one) 3890 DO_FP_IMM(FSUB, fsubs, half, one) 3891 DO_FP_IMM(FMUL, fmuls, half, two) 3892 DO_FP_IMM(FSUBR, fsubrs, half, one) 3893 DO_FP_IMM(FMAXNM, fmaxnms, zero, one) 3894 DO_FP_IMM(FMINNM, fminnms, zero, one) 3895 DO_FP_AH_IMM(FMAX, fmaxs, zero, one) 3896 DO_FP_AH_IMM(FMIN, fmins, zero, one) 3897 3898 #undef DO_FP_IMM 3899 3900 static bool do_fp_cmp(DisasContext *s, arg_rprr_esz *a, 3901 gen_helper_gvec_4_ptr *fn) 3902 { 3903 if (fn == NULL) { 3904 return false; 3905 } 3906 if (sve_access_check(s)) { 3907 unsigned vsz = vec_full_reg_size(s); 3908 TCGv_ptr status = fpstatus_ptr(a->esz == MO_16 ? FPST_A64_F16 : FPST_A64); 3909 tcg_gen_gvec_4_ptr(pred_full_reg_offset(s, a->rd), 3910 vec_full_reg_offset(s, a->rn), 3911 vec_full_reg_offset(s, a->rm), 3912 pred_full_reg_offset(s, a->pg), 3913 status, vsz, vsz, 0, fn); 3914 } 3915 return true; 3916 } 3917 3918 #define DO_FPCMP(NAME, name) \ 3919 static gen_helper_gvec_4_ptr * const name##_fns[4] = { \ 3920 NULL, gen_helper_sve_##name##_h, \ 3921 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \ 3922 }; \ 3923 TRANS_FEAT(NAME##_ppzz, aa64_sve, do_fp_cmp, a, name##_fns[a->esz]) 3924 3925 DO_FPCMP(FCMGE, fcmge) 3926 DO_FPCMP(FCMGT, fcmgt) 3927 DO_FPCMP(FCMEQ, fcmeq) 3928 DO_FPCMP(FCMNE, fcmne) 3929 DO_FPCMP(FCMUO, fcmuo) 3930 DO_FPCMP(FACGE, facge) 3931 DO_FPCMP(FACGT, facgt) 3932 3933 #undef DO_FPCMP 3934 3935 static gen_helper_gvec_4_ptr * const fcadd_fns[] = { 3936 NULL, gen_helper_sve_fcadd_h, 3937 gen_helper_sve_fcadd_s, gen_helper_sve_fcadd_d, 3938 }; 3939 TRANS_FEAT(FCADD, aa64_sve, gen_gvec_fpst_zzzp, fcadd_fns[a->esz], 3940 a->rd, a->rn, a->rm, a->pg, a->rot | (s->fpcr_ah << 1), 3941 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64) 3942 3943 #define DO_FMLA(NAME, name, ah_name) \ 3944 static gen_helper_gvec_5_ptr * const name##_fns[4] = { \ 3945 NULL, gen_helper_sve_##name##_h, \ 3946 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \ 3947 }; \ 3948 static gen_helper_gvec_5_ptr * const name##_ah_fns[4] = { \ 3949 NULL, gen_helper_sve_##ah_name##_h, \ 3950 gen_helper_sve_##ah_name##_s, gen_helper_sve_##ah_name##_d \ 3951 }; \ 3952 TRANS_FEAT(NAME, aa64_sve, gen_gvec_fpst_zzzzp, \ 3953 s->fpcr_ah ? name##_ah_fns[a->esz] : name##_fns[a->esz], \ 3954 a->rd, a->rn, a->rm, a->ra, a->pg, 0, \ 3955 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64) 3956 3957 /* We don't need an ah_fmla_zpzzz because fmla doesn't negate anything */ 3958 DO_FMLA(FMLA_zpzzz, fmla_zpzzz, fmla_zpzzz) 3959 DO_FMLA(FMLS_zpzzz, fmls_zpzzz, ah_fmls_zpzzz) 3960 DO_FMLA(FNMLA_zpzzz, fnmla_zpzzz, ah_fnmla_zpzzz) 3961 DO_FMLA(FNMLS_zpzzz, fnmls_zpzzz, ah_fnmls_zpzzz) 3962 3963 #undef DO_FMLA 3964 3965 static gen_helper_gvec_5_ptr * const fcmla_fns[4] = { 3966 NULL, gen_helper_sve_fcmla_zpzzz_h, 3967 gen_helper_sve_fcmla_zpzzz_s, gen_helper_sve_fcmla_zpzzz_d, 3968 }; 3969 TRANS_FEAT(FCMLA_zpzzz, aa64_sve, gen_gvec_fpst_zzzzp, fcmla_fns[a->esz], 3970 a->rd, a->rn, a->rm, a->ra, a->pg, a->rot | (s->fpcr_ah << 2), 3971 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64) 3972 3973 static gen_helper_gvec_4_ptr * const fcmla_idx_fns[4] = { 3974 NULL, gen_helper_gvec_fcmlah_idx, gen_helper_gvec_fcmlas_idx, NULL 3975 }; 3976 TRANS_FEAT(FCMLA_zzxz, aa64_sve, gen_gvec_fpst_zzzz, fcmla_idx_fns[a->esz], 3977 a->rd, a->rn, a->rm, a->ra, a->index * 4 + a->rot, 3978 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64) 3979 3980 /* 3981 *** SVE Floating Point Unary Operations Predicated Group 3982 */ 3983 3984 TRANS_FEAT(FCVT_sh, aa64_sve, gen_gvec_fpst_arg_zpz, 3985 gen_helper_sve_fcvt_sh, a, 0, FPST_A64) 3986 TRANS_FEAT(FCVT_hs, aa64_sve, gen_gvec_fpst_arg_zpz, 3987 gen_helper_sve_fcvt_hs, a, 0, FPST_A64_F16) 3988 3989 TRANS_FEAT(BFCVT, aa64_sve_bf16, gen_gvec_fpst_arg_zpz, 3990 gen_helper_sve_bfcvt, a, 0, 3991 s->fpcr_ah ? FPST_AH : FPST_A64) 3992 3993 TRANS_FEAT(FCVT_dh, aa64_sve, gen_gvec_fpst_arg_zpz, 3994 gen_helper_sve_fcvt_dh, a, 0, FPST_A64) 3995 TRANS_FEAT(FCVT_hd, aa64_sve, gen_gvec_fpst_arg_zpz, 3996 gen_helper_sve_fcvt_hd, a, 0, FPST_A64_F16) 3997 TRANS_FEAT(FCVT_ds, aa64_sve, gen_gvec_fpst_arg_zpz, 3998 gen_helper_sve_fcvt_ds, a, 0, FPST_A64) 3999 TRANS_FEAT(FCVT_sd, aa64_sve, gen_gvec_fpst_arg_zpz, 4000 gen_helper_sve_fcvt_sd, a, 0, FPST_A64) 4001 4002 TRANS_FEAT(FCVTZS_hh, aa64_sve, gen_gvec_fpst_arg_zpz, 4003 gen_helper_sve_fcvtzs_hh, a, 0, FPST_A64_F16) 4004 TRANS_FEAT(FCVTZU_hh, aa64_sve, gen_gvec_fpst_arg_zpz, 4005 gen_helper_sve_fcvtzu_hh, a, 0, FPST_A64_F16) 4006 TRANS_FEAT(FCVTZS_hs, aa64_sve, gen_gvec_fpst_arg_zpz, 4007 gen_helper_sve_fcvtzs_hs, a, 0, FPST_A64_F16) 4008 TRANS_FEAT(FCVTZU_hs, aa64_sve, gen_gvec_fpst_arg_zpz, 4009 gen_helper_sve_fcvtzu_hs, a, 0, FPST_A64_F16) 4010 TRANS_FEAT(FCVTZS_hd, aa64_sve, gen_gvec_fpst_arg_zpz, 4011 gen_helper_sve_fcvtzs_hd, a, 0, FPST_A64_F16) 4012 TRANS_FEAT(FCVTZU_hd, aa64_sve, gen_gvec_fpst_arg_zpz, 4013 gen_helper_sve_fcvtzu_hd, a, 0, FPST_A64_F16) 4014 4015 TRANS_FEAT(FCVTZS_ss, aa64_sve, gen_gvec_fpst_arg_zpz, 4016 gen_helper_sve_fcvtzs_ss, a, 0, FPST_A64) 4017 TRANS_FEAT(FCVTZU_ss, aa64_sve, gen_gvec_fpst_arg_zpz, 4018 gen_helper_sve_fcvtzu_ss, a, 0, FPST_A64) 4019 TRANS_FEAT(FCVTZS_sd, aa64_sve, gen_gvec_fpst_arg_zpz, 4020 gen_helper_sve_fcvtzs_sd, a, 0, FPST_A64) 4021 TRANS_FEAT(FCVTZU_sd, aa64_sve, gen_gvec_fpst_arg_zpz, 4022 gen_helper_sve_fcvtzu_sd, a, 0, FPST_A64) 4023 TRANS_FEAT(FCVTZS_ds, aa64_sve, gen_gvec_fpst_arg_zpz, 4024 gen_helper_sve_fcvtzs_ds, a, 0, FPST_A64) 4025 TRANS_FEAT(FCVTZU_ds, aa64_sve, gen_gvec_fpst_arg_zpz, 4026 gen_helper_sve_fcvtzu_ds, a, 0, FPST_A64) 4027 4028 TRANS_FEAT(FCVTZS_dd, aa64_sve, gen_gvec_fpst_arg_zpz, 4029 gen_helper_sve_fcvtzs_dd, a, 0, FPST_A64) 4030 TRANS_FEAT(FCVTZU_dd, aa64_sve, gen_gvec_fpst_arg_zpz, 4031 gen_helper_sve_fcvtzu_dd, a, 0, FPST_A64) 4032 4033 static gen_helper_gvec_3_ptr * const frint_fns[] = { 4034 NULL, 4035 gen_helper_sve_frint_h, 4036 gen_helper_sve_frint_s, 4037 gen_helper_sve_frint_d 4038 }; 4039 TRANS_FEAT(FRINTI, aa64_sve, gen_gvec_fpst_arg_zpz, frint_fns[a->esz], 4040 a, 0, a->esz == MO_16 ? FPST_A64_F16 : FPST_A64) 4041 4042 static gen_helper_gvec_3_ptr * const frintx_fns[] = { 4043 NULL, 4044 gen_helper_sve_frintx_h, 4045 gen_helper_sve_frintx_s, 4046 gen_helper_sve_frintx_d 4047 }; 4048 TRANS_FEAT(FRINTX, aa64_sve, gen_gvec_fpst_arg_zpz, frintx_fns[a->esz], 4049 a, 0, a->esz == MO_16 ? FPST_A64_F16 : FPST_A64); 4050 4051 static bool do_frint_mode(DisasContext *s, arg_rpr_esz *a, 4052 ARMFPRounding mode, gen_helper_gvec_3_ptr *fn) 4053 { 4054 unsigned vsz; 4055 TCGv_i32 tmode; 4056 TCGv_ptr status; 4057 4058 if (fn == NULL) { 4059 return false; 4060 } 4061 if (!sve_access_check(s)) { 4062 return true; 4063 } 4064 4065 vsz = vec_full_reg_size(s); 4066 status = fpstatus_ptr(a->esz == MO_16 ? FPST_A64_F16 : FPST_A64); 4067 tmode = gen_set_rmode(mode, status); 4068 4069 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, a->rd), 4070 vec_full_reg_offset(s, a->rn), 4071 pred_full_reg_offset(s, a->pg), 4072 status, vsz, vsz, 0, fn); 4073 4074 gen_restore_rmode(tmode, status); 4075 return true; 4076 } 4077 4078 TRANS_FEAT(FRINTN, aa64_sve, do_frint_mode, a, 4079 FPROUNDING_TIEEVEN, frint_fns[a->esz]) 4080 TRANS_FEAT(FRINTP, aa64_sve, do_frint_mode, a, 4081 FPROUNDING_POSINF, frint_fns[a->esz]) 4082 TRANS_FEAT(FRINTM, aa64_sve, do_frint_mode, a, 4083 FPROUNDING_NEGINF, frint_fns[a->esz]) 4084 TRANS_FEAT(FRINTZ, aa64_sve, do_frint_mode, a, 4085 FPROUNDING_ZERO, frint_fns[a->esz]) 4086 TRANS_FEAT(FRINTA, aa64_sve, do_frint_mode, a, 4087 FPROUNDING_TIEAWAY, frint_fns[a->esz]) 4088 4089 static gen_helper_gvec_3_ptr * const frecpx_fns[] = { 4090 NULL, gen_helper_sve_frecpx_h, 4091 gen_helper_sve_frecpx_s, gen_helper_sve_frecpx_d, 4092 }; 4093 TRANS_FEAT(FRECPX, aa64_sve, gen_gvec_fpst_arg_zpz, frecpx_fns[a->esz], 4094 a, 0, select_ah_fpst(s, a->esz)) 4095 4096 static gen_helper_gvec_3_ptr * const fsqrt_fns[] = { 4097 NULL, gen_helper_sve_fsqrt_h, 4098 gen_helper_sve_fsqrt_s, gen_helper_sve_fsqrt_d, 4099 }; 4100 TRANS_FEAT(FSQRT, aa64_sve, gen_gvec_fpst_arg_zpz, fsqrt_fns[a->esz], 4101 a, 0, a->esz == MO_16 ? FPST_A64_F16 : FPST_A64) 4102 4103 TRANS_FEAT(SCVTF_hh, aa64_sve, gen_gvec_fpst_arg_zpz, 4104 gen_helper_sve_scvt_hh, a, 0, FPST_A64_F16) 4105 TRANS_FEAT(SCVTF_sh, aa64_sve, gen_gvec_fpst_arg_zpz, 4106 gen_helper_sve_scvt_sh, a, 0, FPST_A64_F16) 4107 TRANS_FEAT(SCVTF_dh, aa64_sve, gen_gvec_fpst_arg_zpz, 4108 gen_helper_sve_scvt_dh, a, 0, FPST_A64_F16) 4109 4110 TRANS_FEAT(SCVTF_ss, aa64_sve, gen_gvec_fpst_arg_zpz, 4111 gen_helper_sve_scvt_ss, a, 0, FPST_A64) 4112 TRANS_FEAT(SCVTF_ds, aa64_sve, gen_gvec_fpst_arg_zpz, 4113 gen_helper_sve_scvt_ds, a, 0, FPST_A64) 4114 4115 TRANS_FEAT(SCVTF_sd, aa64_sve, gen_gvec_fpst_arg_zpz, 4116 gen_helper_sve_scvt_sd, a, 0, FPST_A64) 4117 TRANS_FEAT(SCVTF_dd, aa64_sve, gen_gvec_fpst_arg_zpz, 4118 gen_helper_sve_scvt_dd, a, 0, FPST_A64) 4119 4120 TRANS_FEAT(UCVTF_hh, aa64_sve, gen_gvec_fpst_arg_zpz, 4121 gen_helper_sve_ucvt_hh, a, 0, FPST_A64_F16) 4122 TRANS_FEAT(UCVTF_sh, aa64_sve, gen_gvec_fpst_arg_zpz, 4123 gen_helper_sve_ucvt_sh, a, 0, FPST_A64_F16) 4124 TRANS_FEAT(UCVTF_dh, aa64_sve, gen_gvec_fpst_arg_zpz, 4125 gen_helper_sve_ucvt_dh, a, 0, FPST_A64_F16) 4126 4127 TRANS_FEAT(UCVTF_ss, aa64_sve, gen_gvec_fpst_arg_zpz, 4128 gen_helper_sve_ucvt_ss, a, 0, FPST_A64) 4129 TRANS_FEAT(UCVTF_ds, aa64_sve, gen_gvec_fpst_arg_zpz, 4130 gen_helper_sve_ucvt_ds, a, 0, FPST_A64) 4131 TRANS_FEAT(UCVTF_sd, aa64_sve, gen_gvec_fpst_arg_zpz, 4132 gen_helper_sve_ucvt_sd, a, 0, FPST_A64) 4133 4134 TRANS_FEAT(UCVTF_dd, aa64_sve, gen_gvec_fpst_arg_zpz, 4135 gen_helper_sve_ucvt_dd, a, 0, FPST_A64) 4136 4137 /* 4138 *** SVE Memory - 32-bit Gather and Unsized Contiguous Group 4139 */ 4140 4141 /* Subroutine loading a vector register at VOFS of LEN bytes. 4142 * The load should begin at the address Rn + IMM. 4143 */ 4144 4145 void gen_sve_ldr(DisasContext *s, TCGv_ptr base, int vofs, 4146 int len, int rn, int imm) 4147 { 4148 int len_align = QEMU_ALIGN_DOWN(len, 16); 4149 int len_remain = len % 16; 4150 int nparts = len / 16 + ctpop8(len_remain); 4151 int midx = get_mem_index(s); 4152 TCGv_i64 dirty_addr, clean_addr, t0, t1; 4153 TCGv_i128 t16; 4154 4155 dirty_addr = tcg_temp_new_i64(); 4156 tcg_gen_addi_i64(dirty_addr, cpu_reg_sp(s, rn), imm); 4157 clean_addr = gen_mte_checkN(s, dirty_addr, false, rn != 31, len, MO_8); 4158 4159 /* 4160 * Note that unpredicated load/store of vector/predicate registers 4161 * are defined as a stream of bytes, which equates to little-endian 4162 * operations on larger quantities. 4163 * Attempt to keep code expansion to a minimum by limiting the 4164 * amount of unrolling done. 4165 */ 4166 if (nparts <= 4) { 4167 int i; 4168 4169 t0 = tcg_temp_new_i64(); 4170 t1 = tcg_temp_new_i64(); 4171 t16 = tcg_temp_new_i128(); 4172 4173 for (i = 0; i < len_align; i += 16) { 4174 tcg_gen_qemu_ld_i128(t16, clean_addr, midx, 4175 MO_LE | MO_128 | MO_ATOM_NONE); 4176 tcg_gen_extr_i128_i64(t0, t1, t16); 4177 tcg_gen_st_i64(t0, base, vofs + i); 4178 tcg_gen_st_i64(t1, base, vofs + i + 8); 4179 tcg_gen_addi_i64(clean_addr, clean_addr, 16); 4180 } 4181 } else { 4182 TCGLabel *loop = gen_new_label(); 4183 TCGv_ptr tp, i = tcg_temp_new_ptr(); 4184 4185 tcg_gen_movi_ptr(i, 0); 4186 gen_set_label(loop); 4187 4188 t16 = tcg_temp_new_i128(); 4189 tcg_gen_qemu_ld_i128(t16, clean_addr, midx, 4190 MO_LE | MO_128 | MO_ATOM_NONE); 4191 tcg_gen_addi_i64(clean_addr, clean_addr, 16); 4192 4193 tp = tcg_temp_new_ptr(); 4194 tcg_gen_add_ptr(tp, base, i); 4195 tcg_gen_addi_ptr(i, i, 16); 4196 4197 t0 = tcg_temp_new_i64(); 4198 t1 = tcg_temp_new_i64(); 4199 tcg_gen_extr_i128_i64(t0, t1, t16); 4200 4201 tcg_gen_st_i64(t0, tp, vofs); 4202 tcg_gen_st_i64(t1, tp, vofs + 8); 4203 4204 tcg_gen_brcondi_ptr(TCG_COND_LTU, i, len_align, loop); 4205 } 4206 4207 /* 4208 * Predicate register loads can be any multiple of 2. 4209 * Note that we still store the entire 64-bit unit into tcg_env. 4210 */ 4211 if (len_remain >= 8) { 4212 t0 = tcg_temp_new_i64(); 4213 tcg_gen_qemu_ld_i64(t0, clean_addr, midx, MO_LEUQ | MO_ATOM_NONE); 4214 tcg_gen_st_i64(t0, base, vofs + len_align); 4215 len_remain -= 8; 4216 len_align += 8; 4217 if (len_remain) { 4218 tcg_gen_addi_i64(clean_addr, clean_addr, 8); 4219 } 4220 } 4221 if (len_remain) { 4222 t0 = tcg_temp_new_i64(); 4223 switch (len_remain) { 4224 case 2: 4225 case 4: 4226 case 8: 4227 tcg_gen_qemu_ld_i64(t0, clean_addr, midx, 4228 MO_LE | ctz32(len_remain) | MO_ATOM_NONE); 4229 break; 4230 4231 case 6: 4232 t1 = tcg_temp_new_i64(); 4233 tcg_gen_qemu_ld_i64(t0, clean_addr, midx, MO_LEUL | MO_ATOM_NONE); 4234 tcg_gen_addi_i64(clean_addr, clean_addr, 4); 4235 tcg_gen_qemu_ld_i64(t1, clean_addr, midx, MO_LEUW | MO_ATOM_NONE); 4236 tcg_gen_deposit_i64(t0, t0, t1, 32, 32); 4237 break; 4238 4239 default: 4240 g_assert_not_reached(); 4241 } 4242 tcg_gen_st_i64(t0, base, vofs + len_align); 4243 } 4244 } 4245 4246 /* Similarly for stores. */ 4247 void gen_sve_str(DisasContext *s, TCGv_ptr base, int vofs, 4248 int len, int rn, int imm) 4249 { 4250 int len_align = QEMU_ALIGN_DOWN(len, 16); 4251 int len_remain = len % 16; 4252 int nparts = len / 16 + ctpop8(len_remain); 4253 int midx = get_mem_index(s); 4254 TCGv_i64 dirty_addr, clean_addr, t0, t1; 4255 TCGv_i128 t16; 4256 4257 dirty_addr = tcg_temp_new_i64(); 4258 tcg_gen_addi_i64(dirty_addr, cpu_reg_sp(s, rn), imm); 4259 clean_addr = gen_mte_checkN(s, dirty_addr, false, rn != 31, len, MO_8); 4260 4261 /* Note that unpredicated load/store of vector/predicate registers 4262 * are defined as a stream of bytes, which equates to little-endian 4263 * operations on larger quantities. There is no nice way to force 4264 * a little-endian store for aarch64_be-linux-user out of line. 4265 * 4266 * Attempt to keep code expansion to a minimum by limiting the 4267 * amount of unrolling done. 4268 */ 4269 if (nparts <= 4) { 4270 int i; 4271 4272 t0 = tcg_temp_new_i64(); 4273 t1 = tcg_temp_new_i64(); 4274 t16 = tcg_temp_new_i128(); 4275 for (i = 0; i < len_align; i += 16) { 4276 tcg_gen_ld_i64(t0, base, vofs + i); 4277 tcg_gen_ld_i64(t1, base, vofs + i + 8); 4278 tcg_gen_concat_i64_i128(t16, t0, t1); 4279 tcg_gen_qemu_st_i128(t16, clean_addr, midx, 4280 MO_LE | MO_128 | MO_ATOM_NONE); 4281 tcg_gen_addi_i64(clean_addr, clean_addr, 16); 4282 } 4283 } else { 4284 TCGLabel *loop = gen_new_label(); 4285 TCGv_ptr tp, i = tcg_temp_new_ptr(); 4286 4287 tcg_gen_movi_ptr(i, 0); 4288 gen_set_label(loop); 4289 4290 t0 = tcg_temp_new_i64(); 4291 t1 = tcg_temp_new_i64(); 4292 tp = tcg_temp_new_ptr(); 4293 tcg_gen_add_ptr(tp, base, i); 4294 tcg_gen_ld_i64(t0, tp, vofs); 4295 tcg_gen_ld_i64(t1, tp, vofs + 8); 4296 tcg_gen_addi_ptr(i, i, 16); 4297 4298 t16 = tcg_temp_new_i128(); 4299 tcg_gen_concat_i64_i128(t16, t0, t1); 4300 4301 tcg_gen_qemu_st_i128(t16, clean_addr, midx, 4302 MO_LE | MO_128 | MO_ATOM_NONE); 4303 tcg_gen_addi_i64(clean_addr, clean_addr, 16); 4304 4305 tcg_gen_brcondi_ptr(TCG_COND_LTU, i, len_align, loop); 4306 } 4307 4308 /* Predicate register stores can be any multiple of 2. */ 4309 if (len_remain >= 8) { 4310 t0 = tcg_temp_new_i64(); 4311 tcg_gen_ld_i64(t0, base, vofs + len_align); 4312 tcg_gen_qemu_st_i64(t0, clean_addr, midx, MO_LEUQ | MO_ATOM_NONE); 4313 len_remain -= 8; 4314 len_align += 8; 4315 if (len_remain) { 4316 tcg_gen_addi_i64(clean_addr, clean_addr, 8); 4317 } 4318 } 4319 if (len_remain) { 4320 t0 = tcg_temp_new_i64(); 4321 tcg_gen_ld_i64(t0, base, vofs + len_align); 4322 4323 switch (len_remain) { 4324 case 2: 4325 case 4: 4326 case 8: 4327 tcg_gen_qemu_st_i64(t0, clean_addr, midx, 4328 MO_LE | ctz32(len_remain) | MO_ATOM_NONE); 4329 break; 4330 4331 case 6: 4332 tcg_gen_qemu_st_i64(t0, clean_addr, midx, MO_LEUL | MO_ATOM_NONE); 4333 tcg_gen_addi_i64(clean_addr, clean_addr, 4); 4334 tcg_gen_shri_i64(t0, t0, 32); 4335 tcg_gen_qemu_st_i64(t0, clean_addr, midx, MO_LEUW | MO_ATOM_NONE); 4336 break; 4337 4338 default: 4339 g_assert_not_reached(); 4340 } 4341 } 4342 } 4343 4344 static bool trans_LDR_zri(DisasContext *s, arg_rri *a) 4345 { 4346 if (!dc_isar_feature(aa64_sve, s)) { 4347 return false; 4348 } 4349 if (sve_access_check(s)) { 4350 int size = vec_full_reg_size(s); 4351 int off = vec_full_reg_offset(s, a->rd); 4352 gen_sve_ldr(s, tcg_env, off, size, a->rn, a->imm * size); 4353 } 4354 return true; 4355 } 4356 4357 static bool trans_LDR_pri(DisasContext *s, arg_rri *a) 4358 { 4359 if (!dc_isar_feature(aa64_sve, s)) { 4360 return false; 4361 } 4362 if (sve_access_check(s)) { 4363 int size = pred_full_reg_size(s); 4364 int off = pred_full_reg_offset(s, a->rd); 4365 gen_sve_ldr(s, tcg_env, off, size, a->rn, a->imm * size); 4366 } 4367 return true; 4368 } 4369 4370 static bool trans_STR_zri(DisasContext *s, arg_rri *a) 4371 { 4372 if (!dc_isar_feature(aa64_sve, s)) { 4373 return false; 4374 } 4375 if (sve_access_check(s)) { 4376 int size = vec_full_reg_size(s); 4377 int off = vec_full_reg_offset(s, a->rd); 4378 gen_sve_str(s, tcg_env, off, size, a->rn, a->imm * size); 4379 } 4380 return true; 4381 } 4382 4383 static bool trans_STR_pri(DisasContext *s, arg_rri *a) 4384 { 4385 if (!dc_isar_feature(aa64_sve, s)) { 4386 return false; 4387 } 4388 if (sve_access_check(s)) { 4389 int size = pred_full_reg_size(s); 4390 int off = pred_full_reg_offset(s, a->rd); 4391 gen_sve_str(s, tcg_env, off, size, a->rn, a->imm * size); 4392 } 4393 return true; 4394 } 4395 4396 /* 4397 *** SVE Memory - Contiguous Load Group 4398 */ 4399 4400 /* The memory mode of the dtype. */ 4401 static const MemOp dtype_mop[16] = { 4402 MO_UB, MO_UB, MO_UB, MO_UB, 4403 MO_SL, MO_UW, MO_UW, MO_UW, 4404 MO_SW, MO_SW, MO_UL, MO_UL, 4405 MO_SB, MO_SB, MO_SB, MO_UQ 4406 }; 4407 4408 #define dtype_msz(x) (dtype_mop[x] & MO_SIZE) 4409 4410 /* The vector element size of dtype. */ 4411 static const uint8_t dtype_esz[16] = { 4412 0, 1, 2, 3, 4413 3, 1, 2, 3, 4414 3, 2, 2, 3, 4415 3, 2, 1, 3 4416 }; 4417 4418 uint32_t make_svemte_desc(DisasContext *s, unsigned vsz, uint32_t nregs, 4419 uint32_t msz, bool is_write, uint32_t data) 4420 { 4421 uint32_t sizem1; 4422 uint32_t desc = 0; 4423 4424 /* Assert all of the data fits, with or without MTE enabled. */ 4425 assert(nregs >= 1 && nregs <= 4); 4426 sizem1 = (nregs << msz) - 1; 4427 assert(sizem1 <= R_MTEDESC_SIZEM1_MASK >> R_MTEDESC_SIZEM1_SHIFT); 4428 assert(data < 1u << SVE_MTEDESC_SHIFT); 4429 4430 if (s->mte_active[0]) { 4431 desc = FIELD_DP32(desc, MTEDESC, MIDX, get_mem_index(s)); 4432 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid); 4433 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma); 4434 desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write); 4435 desc = FIELD_DP32(desc, MTEDESC, SIZEM1, sizem1); 4436 desc <<= SVE_MTEDESC_SHIFT; 4437 } 4438 return simd_desc(vsz, vsz, desc | data); 4439 } 4440 4441 static void do_mem_zpa(DisasContext *s, int zt, int pg, TCGv_i64 addr, 4442 int dtype, uint32_t nregs, bool is_write, 4443 gen_helper_gvec_mem *fn) 4444 { 4445 TCGv_ptr t_pg; 4446 uint32_t desc; 4447 4448 if (!s->mte_active[0]) { 4449 addr = clean_data_tbi(s, addr); 4450 } 4451 4452 /* 4453 * For e.g. LD4, there are not enough arguments to pass all 4 4454 * registers as pointers, so encode the regno into the data field. 4455 * For consistency, do this even for LD1. 4456 */ 4457 desc = make_svemte_desc(s, vec_full_reg_size(s), nregs, 4458 dtype_msz(dtype), is_write, zt); 4459 t_pg = tcg_temp_new_ptr(); 4460 4461 tcg_gen_addi_ptr(t_pg, tcg_env, pred_full_reg_offset(s, pg)); 4462 fn(tcg_env, t_pg, addr, tcg_constant_i32(desc)); 4463 } 4464 4465 /* Indexed by [mte][be][dtype][nreg] */ 4466 static gen_helper_gvec_mem * const ldr_fns[2][2][16][4] = { 4467 { /* mte inactive, little-endian */ 4468 { { gen_helper_sve_ld1bb_r, gen_helper_sve_ld2bb_r, 4469 gen_helper_sve_ld3bb_r, gen_helper_sve_ld4bb_r }, 4470 { gen_helper_sve_ld1bhu_r, NULL, NULL, NULL }, 4471 { gen_helper_sve_ld1bsu_r, NULL, NULL, NULL }, 4472 { gen_helper_sve_ld1bdu_r, NULL, NULL, NULL }, 4473 4474 { gen_helper_sve_ld1sds_le_r, NULL, NULL, NULL }, 4475 { gen_helper_sve_ld1hh_le_r, gen_helper_sve_ld2hh_le_r, 4476 gen_helper_sve_ld3hh_le_r, gen_helper_sve_ld4hh_le_r }, 4477 { gen_helper_sve_ld1hsu_le_r, NULL, NULL, NULL }, 4478 { gen_helper_sve_ld1hdu_le_r, NULL, NULL, NULL }, 4479 4480 { gen_helper_sve_ld1hds_le_r, NULL, NULL, NULL }, 4481 { gen_helper_sve_ld1hss_le_r, NULL, NULL, NULL }, 4482 { gen_helper_sve_ld1ss_le_r, gen_helper_sve_ld2ss_le_r, 4483 gen_helper_sve_ld3ss_le_r, gen_helper_sve_ld4ss_le_r }, 4484 { gen_helper_sve_ld1sdu_le_r, NULL, NULL, NULL }, 4485 4486 { gen_helper_sve_ld1bds_r, NULL, NULL, NULL }, 4487 { gen_helper_sve_ld1bss_r, NULL, NULL, NULL }, 4488 { gen_helper_sve_ld1bhs_r, NULL, NULL, NULL }, 4489 { gen_helper_sve_ld1dd_le_r, gen_helper_sve_ld2dd_le_r, 4490 gen_helper_sve_ld3dd_le_r, gen_helper_sve_ld4dd_le_r } }, 4491 4492 /* mte inactive, big-endian */ 4493 { { gen_helper_sve_ld1bb_r, gen_helper_sve_ld2bb_r, 4494 gen_helper_sve_ld3bb_r, gen_helper_sve_ld4bb_r }, 4495 { gen_helper_sve_ld1bhu_r, NULL, NULL, NULL }, 4496 { gen_helper_sve_ld1bsu_r, NULL, NULL, NULL }, 4497 { gen_helper_sve_ld1bdu_r, NULL, NULL, NULL }, 4498 4499 { gen_helper_sve_ld1sds_be_r, NULL, NULL, NULL }, 4500 { gen_helper_sve_ld1hh_be_r, gen_helper_sve_ld2hh_be_r, 4501 gen_helper_sve_ld3hh_be_r, gen_helper_sve_ld4hh_be_r }, 4502 { gen_helper_sve_ld1hsu_be_r, NULL, NULL, NULL }, 4503 { gen_helper_sve_ld1hdu_be_r, NULL, NULL, NULL }, 4504 4505 { gen_helper_sve_ld1hds_be_r, NULL, NULL, NULL }, 4506 { gen_helper_sve_ld1hss_be_r, NULL, NULL, NULL }, 4507 { gen_helper_sve_ld1ss_be_r, gen_helper_sve_ld2ss_be_r, 4508 gen_helper_sve_ld3ss_be_r, gen_helper_sve_ld4ss_be_r }, 4509 { gen_helper_sve_ld1sdu_be_r, NULL, NULL, NULL }, 4510 4511 { gen_helper_sve_ld1bds_r, NULL, NULL, NULL }, 4512 { gen_helper_sve_ld1bss_r, NULL, NULL, NULL }, 4513 { gen_helper_sve_ld1bhs_r, NULL, NULL, NULL }, 4514 { gen_helper_sve_ld1dd_be_r, gen_helper_sve_ld2dd_be_r, 4515 gen_helper_sve_ld3dd_be_r, gen_helper_sve_ld4dd_be_r } } }, 4516 4517 { /* mte active, little-endian */ 4518 { { gen_helper_sve_ld1bb_r_mte, 4519 gen_helper_sve_ld2bb_r_mte, 4520 gen_helper_sve_ld3bb_r_mte, 4521 gen_helper_sve_ld4bb_r_mte }, 4522 { gen_helper_sve_ld1bhu_r_mte, NULL, NULL, NULL }, 4523 { gen_helper_sve_ld1bsu_r_mte, NULL, NULL, NULL }, 4524 { gen_helper_sve_ld1bdu_r_mte, NULL, NULL, NULL }, 4525 4526 { gen_helper_sve_ld1sds_le_r_mte, NULL, NULL, NULL }, 4527 { gen_helper_sve_ld1hh_le_r_mte, 4528 gen_helper_sve_ld2hh_le_r_mte, 4529 gen_helper_sve_ld3hh_le_r_mte, 4530 gen_helper_sve_ld4hh_le_r_mte }, 4531 { gen_helper_sve_ld1hsu_le_r_mte, NULL, NULL, NULL }, 4532 { gen_helper_sve_ld1hdu_le_r_mte, NULL, NULL, NULL }, 4533 4534 { gen_helper_sve_ld1hds_le_r_mte, NULL, NULL, NULL }, 4535 { gen_helper_sve_ld1hss_le_r_mte, NULL, NULL, NULL }, 4536 { gen_helper_sve_ld1ss_le_r_mte, 4537 gen_helper_sve_ld2ss_le_r_mte, 4538 gen_helper_sve_ld3ss_le_r_mte, 4539 gen_helper_sve_ld4ss_le_r_mte }, 4540 { gen_helper_sve_ld1sdu_le_r_mte, NULL, NULL, NULL }, 4541 4542 { gen_helper_sve_ld1bds_r_mte, NULL, NULL, NULL }, 4543 { gen_helper_sve_ld1bss_r_mte, NULL, NULL, NULL }, 4544 { gen_helper_sve_ld1bhs_r_mte, NULL, NULL, NULL }, 4545 { gen_helper_sve_ld1dd_le_r_mte, 4546 gen_helper_sve_ld2dd_le_r_mte, 4547 gen_helper_sve_ld3dd_le_r_mte, 4548 gen_helper_sve_ld4dd_le_r_mte } }, 4549 4550 /* mte active, big-endian */ 4551 { { gen_helper_sve_ld1bb_r_mte, 4552 gen_helper_sve_ld2bb_r_mte, 4553 gen_helper_sve_ld3bb_r_mte, 4554 gen_helper_sve_ld4bb_r_mte }, 4555 { gen_helper_sve_ld1bhu_r_mte, NULL, NULL, NULL }, 4556 { gen_helper_sve_ld1bsu_r_mte, NULL, NULL, NULL }, 4557 { gen_helper_sve_ld1bdu_r_mte, NULL, NULL, NULL }, 4558 4559 { gen_helper_sve_ld1sds_be_r_mte, NULL, NULL, NULL }, 4560 { gen_helper_sve_ld1hh_be_r_mte, 4561 gen_helper_sve_ld2hh_be_r_mte, 4562 gen_helper_sve_ld3hh_be_r_mte, 4563 gen_helper_sve_ld4hh_be_r_mte }, 4564 { gen_helper_sve_ld1hsu_be_r_mte, NULL, NULL, NULL }, 4565 { gen_helper_sve_ld1hdu_be_r_mte, NULL, NULL, NULL }, 4566 4567 { gen_helper_sve_ld1hds_be_r_mte, NULL, NULL, NULL }, 4568 { gen_helper_sve_ld1hss_be_r_mte, NULL, NULL, NULL }, 4569 { gen_helper_sve_ld1ss_be_r_mte, 4570 gen_helper_sve_ld2ss_be_r_mte, 4571 gen_helper_sve_ld3ss_be_r_mte, 4572 gen_helper_sve_ld4ss_be_r_mte }, 4573 { gen_helper_sve_ld1sdu_be_r_mte, NULL, NULL, NULL }, 4574 4575 { gen_helper_sve_ld1bds_r_mte, NULL, NULL, NULL }, 4576 { gen_helper_sve_ld1bss_r_mte, NULL, NULL, NULL }, 4577 { gen_helper_sve_ld1bhs_r_mte, NULL, NULL, NULL }, 4578 { gen_helper_sve_ld1dd_be_r_mte, 4579 gen_helper_sve_ld2dd_be_r_mte, 4580 gen_helper_sve_ld3dd_be_r_mte, 4581 gen_helper_sve_ld4dd_be_r_mte } } }, 4582 }; 4583 4584 static void do_ld_zpa(DisasContext *s, int zt, int pg, 4585 TCGv_i64 addr, int dtype, int nreg) 4586 { 4587 gen_helper_gvec_mem *fn 4588 = ldr_fns[s->mte_active[0]][s->be_data == MO_BE][dtype][nreg]; 4589 4590 /* 4591 * While there are holes in the table, they are not 4592 * accessible via the instruction encoding. 4593 */ 4594 assert(fn != NULL); 4595 do_mem_zpa(s, zt, pg, addr, dtype, nreg + 1, false, fn); 4596 } 4597 4598 static bool trans_LD_zprr(DisasContext *s, arg_rprr_load *a) 4599 { 4600 if (a->rm == 31 || !dc_isar_feature(aa64_sve, s)) { 4601 return false; 4602 } 4603 if (sve_access_check(s)) { 4604 TCGv_i64 addr = tcg_temp_new_i64(); 4605 tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), dtype_msz(a->dtype)); 4606 tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn)); 4607 do_ld_zpa(s, a->rd, a->pg, addr, a->dtype, a->nreg); 4608 } 4609 return true; 4610 } 4611 4612 static bool trans_LD_zpri(DisasContext *s, arg_rpri_load *a) 4613 { 4614 if (!dc_isar_feature(aa64_sve, s)) { 4615 return false; 4616 } 4617 if (sve_access_check(s)) { 4618 int vsz = vec_full_reg_size(s); 4619 int elements = vsz >> dtype_esz[a->dtype]; 4620 TCGv_i64 addr = tcg_temp_new_i64(); 4621 4622 tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn), 4623 (a->imm * elements * (a->nreg + 1)) 4624 << dtype_msz(a->dtype)); 4625 do_ld_zpa(s, a->rd, a->pg, addr, a->dtype, a->nreg); 4626 } 4627 return true; 4628 } 4629 4630 static bool trans_LDFF1_zprr(DisasContext *s, arg_rprr_load *a) 4631 { 4632 static gen_helper_gvec_mem * const fns[2][2][16] = { 4633 { /* mte inactive, little-endian */ 4634 { gen_helper_sve_ldff1bb_r, 4635 gen_helper_sve_ldff1bhu_r, 4636 gen_helper_sve_ldff1bsu_r, 4637 gen_helper_sve_ldff1bdu_r, 4638 4639 gen_helper_sve_ldff1sds_le_r, 4640 gen_helper_sve_ldff1hh_le_r, 4641 gen_helper_sve_ldff1hsu_le_r, 4642 gen_helper_sve_ldff1hdu_le_r, 4643 4644 gen_helper_sve_ldff1hds_le_r, 4645 gen_helper_sve_ldff1hss_le_r, 4646 gen_helper_sve_ldff1ss_le_r, 4647 gen_helper_sve_ldff1sdu_le_r, 4648 4649 gen_helper_sve_ldff1bds_r, 4650 gen_helper_sve_ldff1bss_r, 4651 gen_helper_sve_ldff1bhs_r, 4652 gen_helper_sve_ldff1dd_le_r }, 4653 4654 /* mte inactive, big-endian */ 4655 { gen_helper_sve_ldff1bb_r, 4656 gen_helper_sve_ldff1bhu_r, 4657 gen_helper_sve_ldff1bsu_r, 4658 gen_helper_sve_ldff1bdu_r, 4659 4660 gen_helper_sve_ldff1sds_be_r, 4661 gen_helper_sve_ldff1hh_be_r, 4662 gen_helper_sve_ldff1hsu_be_r, 4663 gen_helper_sve_ldff1hdu_be_r, 4664 4665 gen_helper_sve_ldff1hds_be_r, 4666 gen_helper_sve_ldff1hss_be_r, 4667 gen_helper_sve_ldff1ss_be_r, 4668 gen_helper_sve_ldff1sdu_be_r, 4669 4670 gen_helper_sve_ldff1bds_r, 4671 gen_helper_sve_ldff1bss_r, 4672 gen_helper_sve_ldff1bhs_r, 4673 gen_helper_sve_ldff1dd_be_r } }, 4674 4675 { /* mte active, little-endian */ 4676 { gen_helper_sve_ldff1bb_r_mte, 4677 gen_helper_sve_ldff1bhu_r_mte, 4678 gen_helper_sve_ldff1bsu_r_mte, 4679 gen_helper_sve_ldff1bdu_r_mte, 4680 4681 gen_helper_sve_ldff1sds_le_r_mte, 4682 gen_helper_sve_ldff1hh_le_r_mte, 4683 gen_helper_sve_ldff1hsu_le_r_mte, 4684 gen_helper_sve_ldff1hdu_le_r_mte, 4685 4686 gen_helper_sve_ldff1hds_le_r_mte, 4687 gen_helper_sve_ldff1hss_le_r_mte, 4688 gen_helper_sve_ldff1ss_le_r_mte, 4689 gen_helper_sve_ldff1sdu_le_r_mte, 4690 4691 gen_helper_sve_ldff1bds_r_mte, 4692 gen_helper_sve_ldff1bss_r_mte, 4693 gen_helper_sve_ldff1bhs_r_mte, 4694 gen_helper_sve_ldff1dd_le_r_mte }, 4695 4696 /* mte active, big-endian */ 4697 { gen_helper_sve_ldff1bb_r_mte, 4698 gen_helper_sve_ldff1bhu_r_mte, 4699 gen_helper_sve_ldff1bsu_r_mte, 4700 gen_helper_sve_ldff1bdu_r_mte, 4701 4702 gen_helper_sve_ldff1sds_be_r_mte, 4703 gen_helper_sve_ldff1hh_be_r_mte, 4704 gen_helper_sve_ldff1hsu_be_r_mte, 4705 gen_helper_sve_ldff1hdu_be_r_mte, 4706 4707 gen_helper_sve_ldff1hds_be_r_mte, 4708 gen_helper_sve_ldff1hss_be_r_mte, 4709 gen_helper_sve_ldff1ss_be_r_mte, 4710 gen_helper_sve_ldff1sdu_be_r_mte, 4711 4712 gen_helper_sve_ldff1bds_r_mte, 4713 gen_helper_sve_ldff1bss_r_mte, 4714 gen_helper_sve_ldff1bhs_r_mte, 4715 gen_helper_sve_ldff1dd_be_r_mte } }, 4716 }; 4717 4718 if (!dc_isar_feature(aa64_sve, s)) { 4719 return false; 4720 } 4721 s->is_nonstreaming = true; 4722 if (sve_access_check(s)) { 4723 TCGv_i64 addr = tcg_temp_new_i64(); 4724 tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), dtype_msz(a->dtype)); 4725 tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn)); 4726 do_mem_zpa(s, a->rd, a->pg, addr, a->dtype, 1, false, 4727 fns[s->mte_active[0]][s->be_data == MO_BE][a->dtype]); 4728 } 4729 return true; 4730 } 4731 4732 static bool trans_LDNF1_zpri(DisasContext *s, arg_rpri_load *a) 4733 { 4734 static gen_helper_gvec_mem * const fns[2][2][16] = { 4735 { /* mte inactive, little-endian */ 4736 { gen_helper_sve_ldnf1bb_r, 4737 gen_helper_sve_ldnf1bhu_r, 4738 gen_helper_sve_ldnf1bsu_r, 4739 gen_helper_sve_ldnf1bdu_r, 4740 4741 gen_helper_sve_ldnf1sds_le_r, 4742 gen_helper_sve_ldnf1hh_le_r, 4743 gen_helper_sve_ldnf1hsu_le_r, 4744 gen_helper_sve_ldnf1hdu_le_r, 4745 4746 gen_helper_sve_ldnf1hds_le_r, 4747 gen_helper_sve_ldnf1hss_le_r, 4748 gen_helper_sve_ldnf1ss_le_r, 4749 gen_helper_sve_ldnf1sdu_le_r, 4750 4751 gen_helper_sve_ldnf1bds_r, 4752 gen_helper_sve_ldnf1bss_r, 4753 gen_helper_sve_ldnf1bhs_r, 4754 gen_helper_sve_ldnf1dd_le_r }, 4755 4756 /* mte inactive, big-endian */ 4757 { gen_helper_sve_ldnf1bb_r, 4758 gen_helper_sve_ldnf1bhu_r, 4759 gen_helper_sve_ldnf1bsu_r, 4760 gen_helper_sve_ldnf1bdu_r, 4761 4762 gen_helper_sve_ldnf1sds_be_r, 4763 gen_helper_sve_ldnf1hh_be_r, 4764 gen_helper_sve_ldnf1hsu_be_r, 4765 gen_helper_sve_ldnf1hdu_be_r, 4766 4767 gen_helper_sve_ldnf1hds_be_r, 4768 gen_helper_sve_ldnf1hss_be_r, 4769 gen_helper_sve_ldnf1ss_be_r, 4770 gen_helper_sve_ldnf1sdu_be_r, 4771 4772 gen_helper_sve_ldnf1bds_r, 4773 gen_helper_sve_ldnf1bss_r, 4774 gen_helper_sve_ldnf1bhs_r, 4775 gen_helper_sve_ldnf1dd_be_r } }, 4776 4777 { /* mte inactive, little-endian */ 4778 { gen_helper_sve_ldnf1bb_r_mte, 4779 gen_helper_sve_ldnf1bhu_r_mte, 4780 gen_helper_sve_ldnf1bsu_r_mte, 4781 gen_helper_sve_ldnf1bdu_r_mte, 4782 4783 gen_helper_sve_ldnf1sds_le_r_mte, 4784 gen_helper_sve_ldnf1hh_le_r_mte, 4785 gen_helper_sve_ldnf1hsu_le_r_mte, 4786 gen_helper_sve_ldnf1hdu_le_r_mte, 4787 4788 gen_helper_sve_ldnf1hds_le_r_mte, 4789 gen_helper_sve_ldnf1hss_le_r_mte, 4790 gen_helper_sve_ldnf1ss_le_r_mte, 4791 gen_helper_sve_ldnf1sdu_le_r_mte, 4792 4793 gen_helper_sve_ldnf1bds_r_mte, 4794 gen_helper_sve_ldnf1bss_r_mte, 4795 gen_helper_sve_ldnf1bhs_r_mte, 4796 gen_helper_sve_ldnf1dd_le_r_mte }, 4797 4798 /* mte inactive, big-endian */ 4799 { gen_helper_sve_ldnf1bb_r_mte, 4800 gen_helper_sve_ldnf1bhu_r_mte, 4801 gen_helper_sve_ldnf1bsu_r_mte, 4802 gen_helper_sve_ldnf1bdu_r_mte, 4803 4804 gen_helper_sve_ldnf1sds_be_r_mte, 4805 gen_helper_sve_ldnf1hh_be_r_mte, 4806 gen_helper_sve_ldnf1hsu_be_r_mte, 4807 gen_helper_sve_ldnf1hdu_be_r_mte, 4808 4809 gen_helper_sve_ldnf1hds_be_r_mte, 4810 gen_helper_sve_ldnf1hss_be_r_mte, 4811 gen_helper_sve_ldnf1ss_be_r_mte, 4812 gen_helper_sve_ldnf1sdu_be_r_mte, 4813 4814 gen_helper_sve_ldnf1bds_r_mte, 4815 gen_helper_sve_ldnf1bss_r_mte, 4816 gen_helper_sve_ldnf1bhs_r_mte, 4817 gen_helper_sve_ldnf1dd_be_r_mte } }, 4818 }; 4819 4820 if (!dc_isar_feature(aa64_sve, s)) { 4821 return false; 4822 } 4823 s->is_nonstreaming = true; 4824 if (sve_access_check(s)) { 4825 int vsz = vec_full_reg_size(s); 4826 int elements = vsz >> dtype_esz[a->dtype]; 4827 int off = (a->imm * elements) << dtype_msz(a->dtype); 4828 TCGv_i64 addr = tcg_temp_new_i64(); 4829 4830 tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn), off); 4831 do_mem_zpa(s, a->rd, a->pg, addr, a->dtype, 1, false, 4832 fns[s->mte_active[0]][s->be_data == MO_BE][a->dtype]); 4833 } 4834 return true; 4835 } 4836 4837 static void do_ldrq(DisasContext *s, int zt, int pg, TCGv_i64 addr, int dtype) 4838 { 4839 unsigned vsz = vec_full_reg_size(s); 4840 TCGv_ptr t_pg; 4841 int poff; 4842 uint32_t desc; 4843 4844 /* Load the first quadword using the normal predicated load helpers. */ 4845 if (!s->mte_active[0]) { 4846 addr = clean_data_tbi(s, addr); 4847 } 4848 4849 poff = pred_full_reg_offset(s, pg); 4850 if (vsz > 16) { 4851 /* 4852 * Zero-extend the first 16 bits of the predicate into a temporary. 4853 * This avoids triggering an assert making sure we don't have bits 4854 * set within a predicate beyond VQ, but we have lowered VQ to 1 4855 * for this load operation. 4856 */ 4857 TCGv_i64 tmp = tcg_temp_new_i64(); 4858 #if HOST_BIG_ENDIAN 4859 poff += 6; 4860 #endif 4861 tcg_gen_ld16u_i64(tmp, tcg_env, poff); 4862 4863 poff = offsetof(CPUARMState, vfp.preg_tmp); 4864 tcg_gen_st_i64(tmp, tcg_env, poff); 4865 } 4866 4867 t_pg = tcg_temp_new_ptr(); 4868 tcg_gen_addi_ptr(t_pg, tcg_env, poff); 4869 4870 gen_helper_gvec_mem *fn 4871 = ldr_fns[s->mte_active[0]][s->be_data == MO_BE][dtype][0]; 4872 desc = make_svemte_desc(s, 16, 1, dtype_msz(dtype), false, zt); 4873 fn(tcg_env, t_pg, addr, tcg_constant_i32(desc)); 4874 4875 /* Replicate that first quadword. */ 4876 if (vsz > 16) { 4877 int doff = vec_full_reg_offset(s, zt); 4878 tcg_gen_gvec_dup_mem(4, doff + 16, doff, vsz - 16, vsz - 16); 4879 } 4880 } 4881 4882 static bool trans_LD1RQ_zprr(DisasContext *s, arg_rprr_load *a) 4883 { 4884 if (a->rm == 31 || !dc_isar_feature(aa64_sve, s)) { 4885 return false; 4886 } 4887 if (sve_access_check(s)) { 4888 int msz = dtype_msz(a->dtype); 4889 TCGv_i64 addr = tcg_temp_new_i64(); 4890 tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), msz); 4891 tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn)); 4892 do_ldrq(s, a->rd, a->pg, addr, a->dtype); 4893 } 4894 return true; 4895 } 4896 4897 static bool trans_LD1RQ_zpri(DisasContext *s, arg_rpri_load *a) 4898 { 4899 if (!dc_isar_feature(aa64_sve, s)) { 4900 return false; 4901 } 4902 if (sve_access_check(s)) { 4903 TCGv_i64 addr = tcg_temp_new_i64(); 4904 tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn), a->imm * 16); 4905 do_ldrq(s, a->rd, a->pg, addr, a->dtype); 4906 } 4907 return true; 4908 } 4909 4910 static void do_ldro(DisasContext *s, int zt, int pg, TCGv_i64 addr, int dtype) 4911 { 4912 unsigned vsz = vec_full_reg_size(s); 4913 unsigned vsz_r32; 4914 TCGv_ptr t_pg; 4915 int poff, doff; 4916 uint32_t desc; 4917 4918 if (vsz < 32) { 4919 /* 4920 * Note that this UNDEFINED check comes after CheckSVEEnabled() 4921 * in the ARM pseudocode, which is the sve_access_check() done 4922 * in our caller. We should not now return false from the caller. 4923 */ 4924 unallocated_encoding(s); 4925 return; 4926 } 4927 4928 /* Load the first octaword using the normal predicated load helpers. */ 4929 if (!s->mte_active[0]) { 4930 addr = clean_data_tbi(s, addr); 4931 } 4932 4933 poff = pred_full_reg_offset(s, pg); 4934 if (vsz > 32) { 4935 /* 4936 * Zero-extend the first 32 bits of the predicate into a temporary. 4937 * This avoids triggering an assert making sure we don't have bits 4938 * set within a predicate beyond VQ, but we have lowered VQ to 2 4939 * for this load operation. 4940 */ 4941 TCGv_i64 tmp = tcg_temp_new_i64(); 4942 #if HOST_BIG_ENDIAN 4943 poff += 4; 4944 #endif 4945 tcg_gen_ld32u_i64(tmp, tcg_env, poff); 4946 4947 poff = offsetof(CPUARMState, vfp.preg_tmp); 4948 tcg_gen_st_i64(tmp, tcg_env, poff); 4949 } 4950 4951 t_pg = tcg_temp_new_ptr(); 4952 tcg_gen_addi_ptr(t_pg, tcg_env, poff); 4953 4954 gen_helper_gvec_mem *fn 4955 = ldr_fns[s->mte_active[0]][s->be_data == MO_BE][dtype][0]; 4956 desc = make_svemte_desc(s, 32, 1, dtype_msz(dtype), false, zt); 4957 fn(tcg_env, t_pg, addr, tcg_constant_i32(desc)); 4958 4959 /* 4960 * Replicate that first octaword. 4961 * The replication happens in units of 32; if the full vector size 4962 * is not a multiple of 32, the final bits are zeroed. 4963 */ 4964 doff = vec_full_reg_offset(s, zt); 4965 vsz_r32 = QEMU_ALIGN_DOWN(vsz, 32); 4966 if (vsz >= 64) { 4967 tcg_gen_gvec_dup_mem(5, doff + 32, doff, vsz_r32 - 32, vsz_r32 - 32); 4968 } 4969 vsz -= vsz_r32; 4970 if (vsz) { 4971 tcg_gen_gvec_dup_imm(MO_64, doff + vsz_r32, vsz, vsz, 0); 4972 } 4973 } 4974 4975 static bool trans_LD1RO_zprr(DisasContext *s, arg_rprr_load *a) 4976 { 4977 if (!dc_isar_feature(aa64_sve_f64mm, s)) { 4978 return false; 4979 } 4980 if (a->rm == 31) { 4981 return false; 4982 } 4983 s->is_nonstreaming = true; 4984 if (sve_access_check(s)) { 4985 TCGv_i64 addr = tcg_temp_new_i64(); 4986 tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), dtype_msz(a->dtype)); 4987 tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn)); 4988 do_ldro(s, a->rd, a->pg, addr, a->dtype); 4989 } 4990 return true; 4991 } 4992 4993 static bool trans_LD1RO_zpri(DisasContext *s, arg_rpri_load *a) 4994 { 4995 if (!dc_isar_feature(aa64_sve_f64mm, s)) { 4996 return false; 4997 } 4998 s->is_nonstreaming = true; 4999 if (sve_access_check(s)) { 5000 TCGv_i64 addr = tcg_temp_new_i64(); 5001 tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn), a->imm * 32); 5002 do_ldro(s, a->rd, a->pg, addr, a->dtype); 5003 } 5004 return true; 5005 } 5006 5007 /* Load and broadcast element. */ 5008 static bool trans_LD1R_zpri(DisasContext *s, arg_rpri_load *a) 5009 { 5010 unsigned vsz = vec_full_reg_size(s); 5011 unsigned psz = pred_full_reg_size(s); 5012 unsigned esz = dtype_esz[a->dtype]; 5013 unsigned msz = dtype_msz(a->dtype); 5014 TCGLabel *over; 5015 TCGv_i64 temp, clean_addr; 5016 MemOp memop; 5017 5018 if (!dc_isar_feature(aa64_sve, s)) { 5019 return false; 5020 } 5021 if (!sve_access_check(s)) { 5022 return true; 5023 } 5024 5025 over = gen_new_label(); 5026 5027 /* If the guarding predicate has no bits set, no load occurs. */ 5028 if (psz <= 8) { 5029 /* Reduce the pred_esz_masks value simply to reduce the 5030 * size of the code generated here. 5031 */ 5032 uint64_t psz_mask = MAKE_64BIT_MASK(0, psz * 8); 5033 temp = tcg_temp_new_i64(); 5034 tcg_gen_ld_i64(temp, tcg_env, pred_full_reg_offset(s, a->pg)); 5035 tcg_gen_andi_i64(temp, temp, pred_esz_masks[esz] & psz_mask); 5036 tcg_gen_brcondi_i64(TCG_COND_EQ, temp, 0, over); 5037 } else { 5038 TCGv_i32 t32 = tcg_temp_new_i32(); 5039 find_last_active(s, t32, esz, a->pg); 5040 tcg_gen_brcondi_i32(TCG_COND_LT, t32, 0, over); 5041 } 5042 5043 /* Load the data. */ 5044 temp = tcg_temp_new_i64(); 5045 tcg_gen_addi_i64(temp, cpu_reg_sp(s, a->rn), a->imm << msz); 5046 5047 memop = finalize_memop(s, dtype_mop[a->dtype]); 5048 clean_addr = gen_mte_check1(s, temp, false, true, memop); 5049 tcg_gen_qemu_ld_i64(temp, clean_addr, get_mem_index(s), memop); 5050 5051 /* Broadcast to *all* elements. */ 5052 tcg_gen_gvec_dup_i64(esz, vec_full_reg_offset(s, a->rd), 5053 vsz, vsz, temp); 5054 5055 /* Zero the inactive elements. */ 5056 gen_set_label(over); 5057 return do_movz_zpz(s, a->rd, a->rd, a->pg, esz, false); 5058 } 5059 5060 static void do_st_zpa(DisasContext *s, int zt, int pg, TCGv_i64 addr, 5061 int msz, int esz, int nreg) 5062 { 5063 static gen_helper_gvec_mem * const fn_single[2][2][4][4] = { 5064 { { { gen_helper_sve_st1bb_r, 5065 gen_helper_sve_st1bh_r, 5066 gen_helper_sve_st1bs_r, 5067 gen_helper_sve_st1bd_r }, 5068 { NULL, 5069 gen_helper_sve_st1hh_le_r, 5070 gen_helper_sve_st1hs_le_r, 5071 gen_helper_sve_st1hd_le_r }, 5072 { NULL, NULL, 5073 gen_helper_sve_st1ss_le_r, 5074 gen_helper_sve_st1sd_le_r }, 5075 { NULL, NULL, NULL, 5076 gen_helper_sve_st1dd_le_r } }, 5077 { { gen_helper_sve_st1bb_r, 5078 gen_helper_sve_st1bh_r, 5079 gen_helper_sve_st1bs_r, 5080 gen_helper_sve_st1bd_r }, 5081 { NULL, 5082 gen_helper_sve_st1hh_be_r, 5083 gen_helper_sve_st1hs_be_r, 5084 gen_helper_sve_st1hd_be_r }, 5085 { NULL, NULL, 5086 gen_helper_sve_st1ss_be_r, 5087 gen_helper_sve_st1sd_be_r }, 5088 { NULL, NULL, NULL, 5089 gen_helper_sve_st1dd_be_r } } }, 5090 5091 { { { gen_helper_sve_st1bb_r_mte, 5092 gen_helper_sve_st1bh_r_mte, 5093 gen_helper_sve_st1bs_r_mte, 5094 gen_helper_sve_st1bd_r_mte }, 5095 { NULL, 5096 gen_helper_sve_st1hh_le_r_mte, 5097 gen_helper_sve_st1hs_le_r_mte, 5098 gen_helper_sve_st1hd_le_r_mte }, 5099 { NULL, NULL, 5100 gen_helper_sve_st1ss_le_r_mte, 5101 gen_helper_sve_st1sd_le_r_mte }, 5102 { NULL, NULL, NULL, 5103 gen_helper_sve_st1dd_le_r_mte } }, 5104 { { gen_helper_sve_st1bb_r_mte, 5105 gen_helper_sve_st1bh_r_mte, 5106 gen_helper_sve_st1bs_r_mte, 5107 gen_helper_sve_st1bd_r_mte }, 5108 { NULL, 5109 gen_helper_sve_st1hh_be_r_mte, 5110 gen_helper_sve_st1hs_be_r_mte, 5111 gen_helper_sve_st1hd_be_r_mte }, 5112 { NULL, NULL, 5113 gen_helper_sve_st1ss_be_r_mte, 5114 gen_helper_sve_st1sd_be_r_mte }, 5115 { NULL, NULL, NULL, 5116 gen_helper_sve_st1dd_be_r_mte } } }, 5117 }; 5118 static gen_helper_gvec_mem * const fn_multiple[2][2][3][4] = { 5119 { { { gen_helper_sve_st2bb_r, 5120 gen_helper_sve_st2hh_le_r, 5121 gen_helper_sve_st2ss_le_r, 5122 gen_helper_sve_st2dd_le_r }, 5123 { gen_helper_sve_st3bb_r, 5124 gen_helper_sve_st3hh_le_r, 5125 gen_helper_sve_st3ss_le_r, 5126 gen_helper_sve_st3dd_le_r }, 5127 { gen_helper_sve_st4bb_r, 5128 gen_helper_sve_st4hh_le_r, 5129 gen_helper_sve_st4ss_le_r, 5130 gen_helper_sve_st4dd_le_r } }, 5131 { { gen_helper_sve_st2bb_r, 5132 gen_helper_sve_st2hh_be_r, 5133 gen_helper_sve_st2ss_be_r, 5134 gen_helper_sve_st2dd_be_r }, 5135 { gen_helper_sve_st3bb_r, 5136 gen_helper_sve_st3hh_be_r, 5137 gen_helper_sve_st3ss_be_r, 5138 gen_helper_sve_st3dd_be_r }, 5139 { gen_helper_sve_st4bb_r, 5140 gen_helper_sve_st4hh_be_r, 5141 gen_helper_sve_st4ss_be_r, 5142 gen_helper_sve_st4dd_be_r } } }, 5143 { { { gen_helper_sve_st2bb_r_mte, 5144 gen_helper_sve_st2hh_le_r_mte, 5145 gen_helper_sve_st2ss_le_r_mte, 5146 gen_helper_sve_st2dd_le_r_mte }, 5147 { gen_helper_sve_st3bb_r_mte, 5148 gen_helper_sve_st3hh_le_r_mte, 5149 gen_helper_sve_st3ss_le_r_mte, 5150 gen_helper_sve_st3dd_le_r_mte }, 5151 { gen_helper_sve_st4bb_r_mte, 5152 gen_helper_sve_st4hh_le_r_mte, 5153 gen_helper_sve_st4ss_le_r_mte, 5154 gen_helper_sve_st4dd_le_r_mte } }, 5155 { { gen_helper_sve_st2bb_r_mte, 5156 gen_helper_sve_st2hh_be_r_mte, 5157 gen_helper_sve_st2ss_be_r_mte, 5158 gen_helper_sve_st2dd_be_r_mte }, 5159 { gen_helper_sve_st3bb_r_mte, 5160 gen_helper_sve_st3hh_be_r_mte, 5161 gen_helper_sve_st3ss_be_r_mte, 5162 gen_helper_sve_st3dd_be_r_mte }, 5163 { gen_helper_sve_st4bb_r_mte, 5164 gen_helper_sve_st4hh_be_r_mte, 5165 gen_helper_sve_st4ss_be_r_mte, 5166 gen_helper_sve_st4dd_be_r_mte } } }, 5167 }; 5168 gen_helper_gvec_mem *fn; 5169 int be = s->be_data == MO_BE; 5170 5171 if (nreg == 0) { 5172 /* ST1 */ 5173 fn = fn_single[s->mte_active[0]][be][msz][esz]; 5174 } else { 5175 /* ST2, ST3, ST4 -- msz == esz, enforced by encoding */ 5176 assert(msz == esz); 5177 fn = fn_multiple[s->mte_active[0]][be][nreg - 1][msz]; 5178 } 5179 assert(fn != NULL); 5180 do_mem_zpa(s, zt, pg, addr, msz_dtype(s, msz), nreg + 1, true, fn); 5181 } 5182 5183 static bool trans_ST_zprr(DisasContext *s, arg_rprr_store *a) 5184 { 5185 if (!dc_isar_feature(aa64_sve, s)) { 5186 return false; 5187 } 5188 if (a->rm == 31 || a->msz > a->esz) { 5189 return false; 5190 } 5191 if (sve_access_check(s)) { 5192 TCGv_i64 addr = tcg_temp_new_i64(); 5193 tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), a->msz); 5194 tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn)); 5195 do_st_zpa(s, a->rd, a->pg, addr, a->msz, a->esz, a->nreg); 5196 } 5197 return true; 5198 } 5199 5200 static bool trans_ST_zpri(DisasContext *s, arg_rpri_store *a) 5201 { 5202 if (!dc_isar_feature(aa64_sve, s)) { 5203 return false; 5204 } 5205 if (a->msz > a->esz) { 5206 return false; 5207 } 5208 if (sve_access_check(s)) { 5209 int vsz = vec_full_reg_size(s); 5210 int elements = vsz >> a->esz; 5211 TCGv_i64 addr = tcg_temp_new_i64(); 5212 5213 tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn), 5214 (a->imm * elements * (a->nreg + 1)) << a->msz); 5215 do_st_zpa(s, a->rd, a->pg, addr, a->msz, a->esz, a->nreg); 5216 } 5217 return true; 5218 } 5219 5220 /* 5221 *** SVE gather loads / scatter stores 5222 */ 5223 5224 static void do_mem_zpz(DisasContext *s, int zt, int pg, int zm, 5225 int scale, TCGv_i64 scalar, int msz, bool is_write, 5226 gen_helper_gvec_mem_scatter *fn) 5227 { 5228 TCGv_ptr t_zm = tcg_temp_new_ptr(); 5229 TCGv_ptr t_pg = tcg_temp_new_ptr(); 5230 TCGv_ptr t_zt = tcg_temp_new_ptr(); 5231 uint32_t desc; 5232 5233 tcg_gen_addi_ptr(t_pg, tcg_env, pred_full_reg_offset(s, pg)); 5234 tcg_gen_addi_ptr(t_zm, tcg_env, vec_full_reg_offset(s, zm)); 5235 tcg_gen_addi_ptr(t_zt, tcg_env, vec_full_reg_offset(s, zt)); 5236 5237 desc = make_svemte_desc(s, vec_full_reg_size(s), 1, msz, is_write, scale); 5238 fn(tcg_env, t_zt, t_pg, t_zm, scalar, tcg_constant_i32(desc)); 5239 } 5240 5241 /* Indexed by [mte][be][ff][xs][u][msz]. */ 5242 static gen_helper_gvec_mem_scatter * const 5243 gather_load_fn32[2][2][2][2][2][3] = { 5244 { /* MTE Inactive */ 5245 { /* Little-endian */ 5246 { { { gen_helper_sve_ldbss_zsu, 5247 gen_helper_sve_ldhss_le_zsu, 5248 NULL, }, 5249 { gen_helper_sve_ldbsu_zsu, 5250 gen_helper_sve_ldhsu_le_zsu, 5251 gen_helper_sve_ldss_le_zsu, } }, 5252 { { gen_helper_sve_ldbss_zss, 5253 gen_helper_sve_ldhss_le_zss, 5254 NULL, }, 5255 { gen_helper_sve_ldbsu_zss, 5256 gen_helper_sve_ldhsu_le_zss, 5257 gen_helper_sve_ldss_le_zss, } } }, 5258 5259 /* First-fault */ 5260 { { { gen_helper_sve_ldffbss_zsu, 5261 gen_helper_sve_ldffhss_le_zsu, 5262 NULL, }, 5263 { gen_helper_sve_ldffbsu_zsu, 5264 gen_helper_sve_ldffhsu_le_zsu, 5265 gen_helper_sve_ldffss_le_zsu, } }, 5266 { { gen_helper_sve_ldffbss_zss, 5267 gen_helper_sve_ldffhss_le_zss, 5268 NULL, }, 5269 { gen_helper_sve_ldffbsu_zss, 5270 gen_helper_sve_ldffhsu_le_zss, 5271 gen_helper_sve_ldffss_le_zss, } } } }, 5272 5273 { /* Big-endian */ 5274 { { { gen_helper_sve_ldbss_zsu, 5275 gen_helper_sve_ldhss_be_zsu, 5276 NULL, }, 5277 { gen_helper_sve_ldbsu_zsu, 5278 gen_helper_sve_ldhsu_be_zsu, 5279 gen_helper_sve_ldss_be_zsu, } }, 5280 { { gen_helper_sve_ldbss_zss, 5281 gen_helper_sve_ldhss_be_zss, 5282 NULL, }, 5283 { gen_helper_sve_ldbsu_zss, 5284 gen_helper_sve_ldhsu_be_zss, 5285 gen_helper_sve_ldss_be_zss, } } }, 5286 5287 /* First-fault */ 5288 { { { gen_helper_sve_ldffbss_zsu, 5289 gen_helper_sve_ldffhss_be_zsu, 5290 NULL, }, 5291 { gen_helper_sve_ldffbsu_zsu, 5292 gen_helper_sve_ldffhsu_be_zsu, 5293 gen_helper_sve_ldffss_be_zsu, } }, 5294 { { gen_helper_sve_ldffbss_zss, 5295 gen_helper_sve_ldffhss_be_zss, 5296 NULL, }, 5297 { gen_helper_sve_ldffbsu_zss, 5298 gen_helper_sve_ldffhsu_be_zss, 5299 gen_helper_sve_ldffss_be_zss, } } } } }, 5300 { /* MTE Active */ 5301 { /* Little-endian */ 5302 { { { gen_helper_sve_ldbss_zsu_mte, 5303 gen_helper_sve_ldhss_le_zsu_mte, 5304 NULL, }, 5305 { gen_helper_sve_ldbsu_zsu_mte, 5306 gen_helper_sve_ldhsu_le_zsu_mte, 5307 gen_helper_sve_ldss_le_zsu_mte, } }, 5308 { { gen_helper_sve_ldbss_zss_mte, 5309 gen_helper_sve_ldhss_le_zss_mte, 5310 NULL, }, 5311 { gen_helper_sve_ldbsu_zss_mte, 5312 gen_helper_sve_ldhsu_le_zss_mte, 5313 gen_helper_sve_ldss_le_zss_mte, } } }, 5314 5315 /* First-fault */ 5316 { { { gen_helper_sve_ldffbss_zsu_mte, 5317 gen_helper_sve_ldffhss_le_zsu_mte, 5318 NULL, }, 5319 { gen_helper_sve_ldffbsu_zsu_mte, 5320 gen_helper_sve_ldffhsu_le_zsu_mte, 5321 gen_helper_sve_ldffss_le_zsu_mte, } }, 5322 { { gen_helper_sve_ldffbss_zss_mte, 5323 gen_helper_sve_ldffhss_le_zss_mte, 5324 NULL, }, 5325 { gen_helper_sve_ldffbsu_zss_mte, 5326 gen_helper_sve_ldffhsu_le_zss_mte, 5327 gen_helper_sve_ldffss_le_zss_mte, } } } }, 5328 5329 { /* Big-endian */ 5330 { { { gen_helper_sve_ldbss_zsu_mte, 5331 gen_helper_sve_ldhss_be_zsu_mte, 5332 NULL, }, 5333 { gen_helper_sve_ldbsu_zsu_mte, 5334 gen_helper_sve_ldhsu_be_zsu_mte, 5335 gen_helper_sve_ldss_be_zsu_mte, } }, 5336 { { gen_helper_sve_ldbss_zss_mte, 5337 gen_helper_sve_ldhss_be_zss_mte, 5338 NULL, }, 5339 { gen_helper_sve_ldbsu_zss_mte, 5340 gen_helper_sve_ldhsu_be_zss_mte, 5341 gen_helper_sve_ldss_be_zss_mte, } } }, 5342 5343 /* First-fault */ 5344 { { { gen_helper_sve_ldffbss_zsu_mte, 5345 gen_helper_sve_ldffhss_be_zsu_mte, 5346 NULL, }, 5347 { gen_helper_sve_ldffbsu_zsu_mte, 5348 gen_helper_sve_ldffhsu_be_zsu_mte, 5349 gen_helper_sve_ldffss_be_zsu_mte, } }, 5350 { { gen_helper_sve_ldffbss_zss_mte, 5351 gen_helper_sve_ldffhss_be_zss_mte, 5352 NULL, }, 5353 { gen_helper_sve_ldffbsu_zss_mte, 5354 gen_helper_sve_ldffhsu_be_zss_mte, 5355 gen_helper_sve_ldffss_be_zss_mte, } } } } }, 5356 }; 5357 5358 /* Note that we overload xs=2 to indicate 64-bit offset. */ 5359 static gen_helper_gvec_mem_scatter * const 5360 gather_load_fn64[2][2][2][3][2][4] = { 5361 { /* MTE Inactive */ 5362 { /* Little-endian */ 5363 { { { gen_helper_sve_ldbds_zsu, 5364 gen_helper_sve_ldhds_le_zsu, 5365 gen_helper_sve_ldsds_le_zsu, 5366 NULL, }, 5367 { gen_helper_sve_ldbdu_zsu, 5368 gen_helper_sve_ldhdu_le_zsu, 5369 gen_helper_sve_ldsdu_le_zsu, 5370 gen_helper_sve_lddd_le_zsu, } }, 5371 { { gen_helper_sve_ldbds_zss, 5372 gen_helper_sve_ldhds_le_zss, 5373 gen_helper_sve_ldsds_le_zss, 5374 NULL, }, 5375 { gen_helper_sve_ldbdu_zss, 5376 gen_helper_sve_ldhdu_le_zss, 5377 gen_helper_sve_ldsdu_le_zss, 5378 gen_helper_sve_lddd_le_zss, } }, 5379 { { gen_helper_sve_ldbds_zd, 5380 gen_helper_sve_ldhds_le_zd, 5381 gen_helper_sve_ldsds_le_zd, 5382 NULL, }, 5383 { gen_helper_sve_ldbdu_zd, 5384 gen_helper_sve_ldhdu_le_zd, 5385 gen_helper_sve_ldsdu_le_zd, 5386 gen_helper_sve_lddd_le_zd, } } }, 5387 5388 /* First-fault */ 5389 { { { gen_helper_sve_ldffbds_zsu, 5390 gen_helper_sve_ldffhds_le_zsu, 5391 gen_helper_sve_ldffsds_le_zsu, 5392 NULL, }, 5393 { gen_helper_sve_ldffbdu_zsu, 5394 gen_helper_sve_ldffhdu_le_zsu, 5395 gen_helper_sve_ldffsdu_le_zsu, 5396 gen_helper_sve_ldffdd_le_zsu, } }, 5397 { { gen_helper_sve_ldffbds_zss, 5398 gen_helper_sve_ldffhds_le_zss, 5399 gen_helper_sve_ldffsds_le_zss, 5400 NULL, }, 5401 { gen_helper_sve_ldffbdu_zss, 5402 gen_helper_sve_ldffhdu_le_zss, 5403 gen_helper_sve_ldffsdu_le_zss, 5404 gen_helper_sve_ldffdd_le_zss, } }, 5405 { { gen_helper_sve_ldffbds_zd, 5406 gen_helper_sve_ldffhds_le_zd, 5407 gen_helper_sve_ldffsds_le_zd, 5408 NULL, }, 5409 { gen_helper_sve_ldffbdu_zd, 5410 gen_helper_sve_ldffhdu_le_zd, 5411 gen_helper_sve_ldffsdu_le_zd, 5412 gen_helper_sve_ldffdd_le_zd, } } } }, 5413 { /* Big-endian */ 5414 { { { gen_helper_sve_ldbds_zsu, 5415 gen_helper_sve_ldhds_be_zsu, 5416 gen_helper_sve_ldsds_be_zsu, 5417 NULL, }, 5418 { gen_helper_sve_ldbdu_zsu, 5419 gen_helper_sve_ldhdu_be_zsu, 5420 gen_helper_sve_ldsdu_be_zsu, 5421 gen_helper_sve_lddd_be_zsu, } }, 5422 { { gen_helper_sve_ldbds_zss, 5423 gen_helper_sve_ldhds_be_zss, 5424 gen_helper_sve_ldsds_be_zss, 5425 NULL, }, 5426 { gen_helper_sve_ldbdu_zss, 5427 gen_helper_sve_ldhdu_be_zss, 5428 gen_helper_sve_ldsdu_be_zss, 5429 gen_helper_sve_lddd_be_zss, } }, 5430 { { gen_helper_sve_ldbds_zd, 5431 gen_helper_sve_ldhds_be_zd, 5432 gen_helper_sve_ldsds_be_zd, 5433 NULL, }, 5434 { gen_helper_sve_ldbdu_zd, 5435 gen_helper_sve_ldhdu_be_zd, 5436 gen_helper_sve_ldsdu_be_zd, 5437 gen_helper_sve_lddd_be_zd, } } }, 5438 5439 /* First-fault */ 5440 { { { gen_helper_sve_ldffbds_zsu, 5441 gen_helper_sve_ldffhds_be_zsu, 5442 gen_helper_sve_ldffsds_be_zsu, 5443 NULL, }, 5444 { gen_helper_sve_ldffbdu_zsu, 5445 gen_helper_sve_ldffhdu_be_zsu, 5446 gen_helper_sve_ldffsdu_be_zsu, 5447 gen_helper_sve_ldffdd_be_zsu, } }, 5448 { { gen_helper_sve_ldffbds_zss, 5449 gen_helper_sve_ldffhds_be_zss, 5450 gen_helper_sve_ldffsds_be_zss, 5451 NULL, }, 5452 { gen_helper_sve_ldffbdu_zss, 5453 gen_helper_sve_ldffhdu_be_zss, 5454 gen_helper_sve_ldffsdu_be_zss, 5455 gen_helper_sve_ldffdd_be_zss, } }, 5456 { { gen_helper_sve_ldffbds_zd, 5457 gen_helper_sve_ldffhds_be_zd, 5458 gen_helper_sve_ldffsds_be_zd, 5459 NULL, }, 5460 { gen_helper_sve_ldffbdu_zd, 5461 gen_helper_sve_ldffhdu_be_zd, 5462 gen_helper_sve_ldffsdu_be_zd, 5463 gen_helper_sve_ldffdd_be_zd, } } } } }, 5464 { /* MTE Active */ 5465 { /* Little-endian */ 5466 { { { gen_helper_sve_ldbds_zsu_mte, 5467 gen_helper_sve_ldhds_le_zsu_mte, 5468 gen_helper_sve_ldsds_le_zsu_mte, 5469 NULL, }, 5470 { gen_helper_sve_ldbdu_zsu_mte, 5471 gen_helper_sve_ldhdu_le_zsu_mte, 5472 gen_helper_sve_ldsdu_le_zsu_mte, 5473 gen_helper_sve_lddd_le_zsu_mte, } }, 5474 { { gen_helper_sve_ldbds_zss_mte, 5475 gen_helper_sve_ldhds_le_zss_mte, 5476 gen_helper_sve_ldsds_le_zss_mte, 5477 NULL, }, 5478 { gen_helper_sve_ldbdu_zss_mte, 5479 gen_helper_sve_ldhdu_le_zss_mte, 5480 gen_helper_sve_ldsdu_le_zss_mte, 5481 gen_helper_sve_lddd_le_zss_mte, } }, 5482 { { gen_helper_sve_ldbds_zd_mte, 5483 gen_helper_sve_ldhds_le_zd_mte, 5484 gen_helper_sve_ldsds_le_zd_mte, 5485 NULL, }, 5486 { gen_helper_sve_ldbdu_zd_mte, 5487 gen_helper_sve_ldhdu_le_zd_mte, 5488 gen_helper_sve_ldsdu_le_zd_mte, 5489 gen_helper_sve_lddd_le_zd_mte, } } }, 5490 5491 /* First-fault */ 5492 { { { gen_helper_sve_ldffbds_zsu_mte, 5493 gen_helper_sve_ldffhds_le_zsu_mte, 5494 gen_helper_sve_ldffsds_le_zsu_mte, 5495 NULL, }, 5496 { gen_helper_sve_ldffbdu_zsu_mte, 5497 gen_helper_sve_ldffhdu_le_zsu_mte, 5498 gen_helper_sve_ldffsdu_le_zsu_mte, 5499 gen_helper_sve_ldffdd_le_zsu_mte, } }, 5500 { { gen_helper_sve_ldffbds_zss_mte, 5501 gen_helper_sve_ldffhds_le_zss_mte, 5502 gen_helper_sve_ldffsds_le_zss_mte, 5503 NULL, }, 5504 { gen_helper_sve_ldffbdu_zss_mte, 5505 gen_helper_sve_ldffhdu_le_zss_mte, 5506 gen_helper_sve_ldffsdu_le_zss_mte, 5507 gen_helper_sve_ldffdd_le_zss_mte, } }, 5508 { { gen_helper_sve_ldffbds_zd_mte, 5509 gen_helper_sve_ldffhds_le_zd_mte, 5510 gen_helper_sve_ldffsds_le_zd_mte, 5511 NULL, }, 5512 { gen_helper_sve_ldffbdu_zd_mte, 5513 gen_helper_sve_ldffhdu_le_zd_mte, 5514 gen_helper_sve_ldffsdu_le_zd_mte, 5515 gen_helper_sve_ldffdd_le_zd_mte, } } } }, 5516 { /* Big-endian */ 5517 { { { gen_helper_sve_ldbds_zsu_mte, 5518 gen_helper_sve_ldhds_be_zsu_mte, 5519 gen_helper_sve_ldsds_be_zsu_mte, 5520 NULL, }, 5521 { gen_helper_sve_ldbdu_zsu_mte, 5522 gen_helper_sve_ldhdu_be_zsu_mte, 5523 gen_helper_sve_ldsdu_be_zsu_mte, 5524 gen_helper_sve_lddd_be_zsu_mte, } }, 5525 { { gen_helper_sve_ldbds_zss_mte, 5526 gen_helper_sve_ldhds_be_zss_mte, 5527 gen_helper_sve_ldsds_be_zss_mte, 5528 NULL, }, 5529 { gen_helper_sve_ldbdu_zss_mte, 5530 gen_helper_sve_ldhdu_be_zss_mte, 5531 gen_helper_sve_ldsdu_be_zss_mte, 5532 gen_helper_sve_lddd_be_zss_mte, } }, 5533 { { gen_helper_sve_ldbds_zd_mte, 5534 gen_helper_sve_ldhds_be_zd_mte, 5535 gen_helper_sve_ldsds_be_zd_mte, 5536 NULL, }, 5537 { gen_helper_sve_ldbdu_zd_mte, 5538 gen_helper_sve_ldhdu_be_zd_mte, 5539 gen_helper_sve_ldsdu_be_zd_mte, 5540 gen_helper_sve_lddd_be_zd_mte, } } }, 5541 5542 /* First-fault */ 5543 { { { gen_helper_sve_ldffbds_zsu_mte, 5544 gen_helper_sve_ldffhds_be_zsu_mte, 5545 gen_helper_sve_ldffsds_be_zsu_mte, 5546 NULL, }, 5547 { gen_helper_sve_ldffbdu_zsu_mte, 5548 gen_helper_sve_ldffhdu_be_zsu_mte, 5549 gen_helper_sve_ldffsdu_be_zsu_mte, 5550 gen_helper_sve_ldffdd_be_zsu_mte, } }, 5551 { { gen_helper_sve_ldffbds_zss_mte, 5552 gen_helper_sve_ldffhds_be_zss_mte, 5553 gen_helper_sve_ldffsds_be_zss_mte, 5554 NULL, }, 5555 { gen_helper_sve_ldffbdu_zss_mte, 5556 gen_helper_sve_ldffhdu_be_zss_mte, 5557 gen_helper_sve_ldffsdu_be_zss_mte, 5558 gen_helper_sve_ldffdd_be_zss_mte, } }, 5559 { { gen_helper_sve_ldffbds_zd_mte, 5560 gen_helper_sve_ldffhds_be_zd_mte, 5561 gen_helper_sve_ldffsds_be_zd_mte, 5562 NULL, }, 5563 { gen_helper_sve_ldffbdu_zd_mte, 5564 gen_helper_sve_ldffhdu_be_zd_mte, 5565 gen_helper_sve_ldffsdu_be_zd_mte, 5566 gen_helper_sve_ldffdd_be_zd_mte, } } } } }, 5567 }; 5568 5569 static bool trans_LD1_zprz(DisasContext *s, arg_LD1_zprz *a) 5570 { 5571 gen_helper_gvec_mem_scatter *fn = NULL; 5572 bool be = s->be_data == MO_BE; 5573 bool mte = s->mte_active[0]; 5574 5575 if (!dc_isar_feature(aa64_sve, s)) { 5576 return false; 5577 } 5578 s->is_nonstreaming = true; 5579 if (!sve_access_check(s)) { 5580 return true; 5581 } 5582 5583 switch (a->esz) { 5584 case MO_32: 5585 fn = gather_load_fn32[mte][be][a->ff][a->xs][a->u][a->msz]; 5586 break; 5587 case MO_64: 5588 fn = gather_load_fn64[mte][be][a->ff][a->xs][a->u][a->msz]; 5589 break; 5590 } 5591 assert(fn != NULL); 5592 5593 do_mem_zpz(s, a->rd, a->pg, a->rm, a->scale * a->msz, 5594 cpu_reg_sp(s, a->rn), a->msz, false, fn); 5595 return true; 5596 } 5597 5598 static bool trans_LD1_zpiz(DisasContext *s, arg_LD1_zpiz *a) 5599 { 5600 gen_helper_gvec_mem_scatter *fn = NULL; 5601 bool be = s->be_data == MO_BE; 5602 bool mte = s->mte_active[0]; 5603 5604 if (a->esz < a->msz || (a->esz == a->msz && !a->u)) { 5605 return false; 5606 } 5607 if (!dc_isar_feature(aa64_sve, s)) { 5608 return false; 5609 } 5610 s->is_nonstreaming = true; 5611 if (!sve_access_check(s)) { 5612 return true; 5613 } 5614 5615 switch (a->esz) { 5616 case MO_32: 5617 fn = gather_load_fn32[mte][be][a->ff][0][a->u][a->msz]; 5618 break; 5619 case MO_64: 5620 fn = gather_load_fn64[mte][be][a->ff][2][a->u][a->msz]; 5621 break; 5622 } 5623 assert(fn != NULL); 5624 5625 /* Treat LD1_zpiz (zn[x] + imm) the same way as LD1_zprz (rn + zm[x]) 5626 * by loading the immediate into the scalar parameter. 5627 */ 5628 do_mem_zpz(s, a->rd, a->pg, a->rn, 0, 5629 tcg_constant_i64(a->imm << a->msz), a->msz, false, fn); 5630 return true; 5631 } 5632 5633 static bool trans_LDNT1_zprz(DisasContext *s, arg_LD1_zprz *a) 5634 { 5635 gen_helper_gvec_mem_scatter *fn = NULL; 5636 bool be = s->be_data == MO_BE; 5637 bool mte = s->mte_active[0]; 5638 5639 if (a->esz < a->msz + !a->u) { 5640 return false; 5641 } 5642 if (!dc_isar_feature(aa64_sve2, s)) { 5643 return false; 5644 } 5645 s->is_nonstreaming = true; 5646 if (!sve_access_check(s)) { 5647 return true; 5648 } 5649 5650 switch (a->esz) { 5651 case MO_32: 5652 fn = gather_load_fn32[mte][be][0][0][a->u][a->msz]; 5653 break; 5654 case MO_64: 5655 fn = gather_load_fn64[mte][be][0][2][a->u][a->msz]; 5656 break; 5657 } 5658 assert(fn != NULL); 5659 5660 do_mem_zpz(s, a->rd, a->pg, a->rn, 0, 5661 cpu_reg(s, a->rm), a->msz, false, fn); 5662 return true; 5663 } 5664 5665 /* Indexed by [mte][be][xs][msz]. */ 5666 static gen_helper_gvec_mem_scatter * const scatter_store_fn32[2][2][2][3] = { 5667 { /* MTE Inactive */ 5668 { /* Little-endian */ 5669 { gen_helper_sve_stbs_zsu, 5670 gen_helper_sve_sths_le_zsu, 5671 gen_helper_sve_stss_le_zsu, }, 5672 { gen_helper_sve_stbs_zss, 5673 gen_helper_sve_sths_le_zss, 5674 gen_helper_sve_stss_le_zss, } }, 5675 { /* Big-endian */ 5676 { gen_helper_sve_stbs_zsu, 5677 gen_helper_sve_sths_be_zsu, 5678 gen_helper_sve_stss_be_zsu, }, 5679 { gen_helper_sve_stbs_zss, 5680 gen_helper_sve_sths_be_zss, 5681 gen_helper_sve_stss_be_zss, } } }, 5682 { /* MTE Active */ 5683 { /* Little-endian */ 5684 { gen_helper_sve_stbs_zsu_mte, 5685 gen_helper_sve_sths_le_zsu_mte, 5686 gen_helper_sve_stss_le_zsu_mte, }, 5687 { gen_helper_sve_stbs_zss_mte, 5688 gen_helper_sve_sths_le_zss_mte, 5689 gen_helper_sve_stss_le_zss_mte, } }, 5690 { /* Big-endian */ 5691 { gen_helper_sve_stbs_zsu_mte, 5692 gen_helper_sve_sths_be_zsu_mte, 5693 gen_helper_sve_stss_be_zsu_mte, }, 5694 { gen_helper_sve_stbs_zss_mte, 5695 gen_helper_sve_sths_be_zss_mte, 5696 gen_helper_sve_stss_be_zss_mte, } } }, 5697 }; 5698 5699 /* Note that we overload xs=2 to indicate 64-bit offset. */ 5700 static gen_helper_gvec_mem_scatter * const scatter_store_fn64[2][2][3][4] = { 5701 { /* MTE Inactive */ 5702 { /* Little-endian */ 5703 { gen_helper_sve_stbd_zsu, 5704 gen_helper_sve_sthd_le_zsu, 5705 gen_helper_sve_stsd_le_zsu, 5706 gen_helper_sve_stdd_le_zsu, }, 5707 { gen_helper_sve_stbd_zss, 5708 gen_helper_sve_sthd_le_zss, 5709 gen_helper_sve_stsd_le_zss, 5710 gen_helper_sve_stdd_le_zss, }, 5711 { gen_helper_sve_stbd_zd, 5712 gen_helper_sve_sthd_le_zd, 5713 gen_helper_sve_stsd_le_zd, 5714 gen_helper_sve_stdd_le_zd, } }, 5715 { /* Big-endian */ 5716 { gen_helper_sve_stbd_zsu, 5717 gen_helper_sve_sthd_be_zsu, 5718 gen_helper_sve_stsd_be_zsu, 5719 gen_helper_sve_stdd_be_zsu, }, 5720 { gen_helper_sve_stbd_zss, 5721 gen_helper_sve_sthd_be_zss, 5722 gen_helper_sve_stsd_be_zss, 5723 gen_helper_sve_stdd_be_zss, }, 5724 { gen_helper_sve_stbd_zd, 5725 gen_helper_sve_sthd_be_zd, 5726 gen_helper_sve_stsd_be_zd, 5727 gen_helper_sve_stdd_be_zd, } } }, 5728 { /* MTE Inactive */ 5729 { /* Little-endian */ 5730 { gen_helper_sve_stbd_zsu_mte, 5731 gen_helper_sve_sthd_le_zsu_mte, 5732 gen_helper_sve_stsd_le_zsu_mte, 5733 gen_helper_sve_stdd_le_zsu_mte, }, 5734 { gen_helper_sve_stbd_zss_mte, 5735 gen_helper_sve_sthd_le_zss_mte, 5736 gen_helper_sve_stsd_le_zss_mte, 5737 gen_helper_sve_stdd_le_zss_mte, }, 5738 { gen_helper_sve_stbd_zd_mte, 5739 gen_helper_sve_sthd_le_zd_mte, 5740 gen_helper_sve_stsd_le_zd_mte, 5741 gen_helper_sve_stdd_le_zd_mte, } }, 5742 { /* Big-endian */ 5743 { gen_helper_sve_stbd_zsu_mte, 5744 gen_helper_sve_sthd_be_zsu_mte, 5745 gen_helper_sve_stsd_be_zsu_mte, 5746 gen_helper_sve_stdd_be_zsu_mte, }, 5747 { gen_helper_sve_stbd_zss_mte, 5748 gen_helper_sve_sthd_be_zss_mte, 5749 gen_helper_sve_stsd_be_zss_mte, 5750 gen_helper_sve_stdd_be_zss_mte, }, 5751 { gen_helper_sve_stbd_zd_mte, 5752 gen_helper_sve_sthd_be_zd_mte, 5753 gen_helper_sve_stsd_be_zd_mte, 5754 gen_helper_sve_stdd_be_zd_mte, } } }, 5755 }; 5756 5757 static bool trans_ST1_zprz(DisasContext *s, arg_ST1_zprz *a) 5758 { 5759 gen_helper_gvec_mem_scatter *fn; 5760 bool be = s->be_data == MO_BE; 5761 bool mte = s->mte_active[0]; 5762 5763 if (a->esz < a->msz || (a->msz == 0 && a->scale)) { 5764 return false; 5765 } 5766 if (!dc_isar_feature(aa64_sve, s)) { 5767 return false; 5768 } 5769 s->is_nonstreaming = true; 5770 if (!sve_access_check(s)) { 5771 return true; 5772 } 5773 switch (a->esz) { 5774 case MO_32: 5775 fn = scatter_store_fn32[mte][be][a->xs][a->msz]; 5776 break; 5777 case MO_64: 5778 fn = scatter_store_fn64[mte][be][a->xs][a->msz]; 5779 break; 5780 default: 5781 g_assert_not_reached(); 5782 } 5783 do_mem_zpz(s, a->rd, a->pg, a->rm, a->scale * a->msz, 5784 cpu_reg_sp(s, a->rn), a->msz, true, fn); 5785 return true; 5786 } 5787 5788 static bool trans_ST1_zpiz(DisasContext *s, arg_ST1_zpiz *a) 5789 { 5790 gen_helper_gvec_mem_scatter *fn = NULL; 5791 bool be = s->be_data == MO_BE; 5792 bool mte = s->mte_active[0]; 5793 5794 if (a->esz < a->msz) { 5795 return false; 5796 } 5797 if (!dc_isar_feature(aa64_sve, s)) { 5798 return false; 5799 } 5800 s->is_nonstreaming = true; 5801 if (!sve_access_check(s)) { 5802 return true; 5803 } 5804 5805 switch (a->esz) { 5806 case MO_32: 5807 fn = scatter_store_fn32[mte][be][0][a->msz]; 5808 break; 5809 case MO_64: 5810 fn = scatter_store_fn64[mte][be][2][a->msz]; 5811 break; 5812 } 5813 assert(fn != NULL); 5814 5815 /* Treat ST1_zpiz (zn[x] + imm) the same way as ST1_zprz (rn + zm[x]) 5816 * by loading the immediate into the scalar parameter. 5817 */ 5818 do_mem_zpz(s, a->rd, a->pg, a->rn, 0, 5819 tcg_constant_i64(a->imm << a->msz), a->msz, true, fn); 5820 return true; 5821 } 5822 5823 static bool trans_STNT1_zprz(DisasContext *s, arg_ST1_zprz *a) 5824 { 5825 gen_helper_gvec_mem_scatter *fn; 5826 bool be = s->be_data == MO_BE; 5827 bool mte = s->mte_active[0]; 5828 5829 if (a->esz < a->msz) { 5830 return false; 5831 } 5832 if (!dc_isar_feature(aa64_sve2, s)) { 5833 return false; 5834 } 5835 s->is_nonstreaming = true; 5836 if (!sve_access_check(s)) { 5837 return true; 5838 } 5839 5840 switch (a->esz) { 5841 case MO_32: 5842 fn = scatter_store_fn32[mte][be][0][a->msz]; 5843 break; 5844 case MO_64: 5845 fn = scatter_store_fn64[mte][be][2][a->msz]; 5846 break; 5847 default: 5848 g_assert_not_reached(); 5849 } 5850 5851 do_mem_zpz(s, a->rd, a->pg, a->rn, 0, 5852 cpu_reg(s, a->rm), a->msz, true, fn); 5853 return true; 5854 } 5855 5856 /* 5857 * Prefetches 5858 */ 5859 5860 static bool trans_PRF(DisasContext *s, arg_PRF *a) 5861 { 5862 if (!dc_isar_feature(aa64_sve, s)) { 5863 return false; 5864 } 5865 /* Prefetch is a nop within QEMU. */ 5866 (void)sve_access_check(s); 5867 return true; 5868 } 5869 5870 static bool trans_PRF_rr(DisasContext *s, arg_PRF_rr *a) 5871 { 5872 if (a->rm == 31 || !dc_isar_feature(aa64_sve, s)) { 5873 return false; 5874 } 5875 /* Prefetch is a nop within QEMU. */ 5876 (void)sve_access_check(s); 5877 return true; 5878 } 5879 5880 static bool trans_PRF_ns(DisasContext *s, arg_PRF_ns *a) 5881 { 5882 if (!dc_isar_feature(aa64_sve, s)) { 5883 return false; 5884 } 5885 /* Prefetch is a nop within QEMU. */ 5886 s->is_nonstreaming = true; 5887 (void)sve_access_check(s); 5888 return true; 5889 } 5890 5891 /* 5892 * Move Prefix 5893 * 5894 * TODO: The implementation so far could handle predicated merging movprfx. 5895 * The helper functions as written take an extra source register to 5896 * use in the operation, but the result is only written when predication 5897 * succeeds. For unpredicated movprfx, we need to rearrange the helpers 5898 * to allow the final write back to the destination to be unconditional. 5899 * For predicated zeroing movprfx, we need to rearrange the helpers to 5900 * allow the final write back to zero inactives. 5901 * 5902 * In the meantime, just emit the moves. 5903 */ 5904 5905 TRANS_FEAT(MOVPRFX, aa64_sve, do_mov_z, a->rd, a->rn) 5906 TRANS_FEAT(MOVPRFX_m, aa64_sve, do_sel_z, a->rd, a->rn, a->rd, a->pg, a->esz) 5907 TRANS_FEAT(MOVPRFX_z, aa64_sve, do_movz_zpz, a->rd, a->rn, a->pg, a->esz, false) 5908 5909 /* 5910 * SVE2 Integer Multiply - Unpredicated 5911 */ 5912 5913 TRANS_FEAT(MUL_zzz, aa64_sve2, gen_gvec_fn_arg_zzz, tcg_gen_gvec_mul, a) 5914 5915 static gen_helper_gvec_3 * const smulh_zzz_fns[4] = { 5916 gen_helper_gvec_smulh_b, gen_helper_gvec_smulh_h, 5917 gen_helper_gvec_smulh_s, gen_helper_gvec_smulh_d, 5918 }; 5919 TRANS_FEAT(SMULH_zzz, aa64_sve2, gen_gvec_ool_arg_zzz, 5920 smulh_zzz_fns[a->esz], a, 0) 5921 5922 static gen_helper_gvec_3 * const umulh_zzz_fns[4] = { 5923 gen_helper_gvec_umulh_b, gen_helper_gvec_umulh_h, 5924 gen_helper_gvec_umulh_s, gen_helper_gvec_umulh_d, 5925 }; 5926 TRANS_FEAT(UMULH_zzz, aa64_sve2, gen_gvec_ool_arg_zzz, 5927 umulh_zzz_fns[a->esz], a, 0) 5928 5929 TRANS_FEAT(PMUL_zzz, aa64_sve2, gen_gvec_ool_arg_zzz, 5930 gen_helper_gvec_pmul_b, a, 0) 5931 5932 static gen_helper_gvec_3 * const sqdmulh_zzz_fns[4] = { 5933 gen_helper_sve2_sqdmulh_b, gen_helper_sve2_sqdmulh_h, 5934 gen_helper_sve2_sqdmulh_s, gen_helper_sve2_sqdmulh_d, 5935 }; 5936 TRANS_FEAT(SQDMULH_zzz, aa64_sve2, gen_gvec_ool_arg_zzz, 5937 sqdmulh_zzz_fns[a->esz], a, 0) 5938 5939 static gen_helper_gvec_3 * const sqrdmulh_zzz_fns[4] = { 5940 gen_helper_sve2_sqrdmulh_b, gen_helper_sve2_sqrdmulh_h, 5941 gen_helper_sve2_sqrdmulh_s, gen_helper_sve2_sqrdmulh_d, 5942 }; 5943 TRANS_FEAT(SQRDMULH_zzz, aa64_sve2, gen_gvec_ool_arg_zzz, 5944 sqrdmulh_zzz_fns[a->esz], a, 0) 5945 5946 /* 5947 * SVE2 Integer - Predicated 5948 */ 5949 5950 static gen_helper_gvec_4 * const sadlp_fns[4] = { 5951 NULL, gen_helper_sve2_sadalp_zpzz_h, 5952 gen_helper_sve2_sadalp_zpzz_s, gen_helper_sve2_sadalp_zpzz_d, 5953 }; 5954 TRANS_FEAT(SADALP_zpzz, aa64_sve2, gen_gvec_ool_arg_zpzz, 5955 sadlp_fns[a->esz], a, 0) 5956 5957 static gen_helper_gvec_4 * const uadlp_fns[4] = { 5958 NULL, gen_helper_sve2_uadalp_zpzz_h, 5959 gen_helper_sve2_uadalp_zpzz_s, gen_helper_sve2_uadalp_zpzz_d, 5960 }; 5961 TRANS_FEAT(UADALP_zpzz, aa64_sve2, gen_gvec_ool_arg_zpzz, 5962 uadlp_fns[a->esz], a, 0) 5963 5964 /* 5965 * SVE2 integer unary operations (predicated) 5966 */ 5967 5968 TRANS_FEAT(URECPE, aa64_sve2, gen_gvec_ool_arg_zpz, 5969 a->esz == 2 ? gen_helper_sve2_urecpe_s : NULL, a, 0) 5970 5971 TRANS_FEAT(URSQRTE, aa64_sve2, gen_gvec_ool_arg_zpz, 5972 a->esz == 2 ? gen_helper_sve2_ursqrte_s : NULL, a, 0) 5973 5974 static gen_helper_gvec_3 * const sqabs_fns[4] = { 5975 gen_helper_sve2_sqabs_b, gen_helper_sve2_sqabs_h, 5976 gen_helper_sve2_sqabs_s, gen_helper_sve2_sqabs_d, 5977 }; 5978 TRANS_FEAT(SQABS, aa64_sve2, gen_gvec_ool_arg_zpz, sqabs_fns[a->esz], a, 0) 5979 5980 static gen_helper_gvec_3 * const sqneg_fns[4] = { 5981 gen_helper_sve2_sqneg_b, gen_helper_sve2_sqneg_h, 5982 gen_helper_sve2_sqneg_s, gen_helper_sve2_sqneg_d, 5983 }; 5984 TRANS_FEAT(SQNEG, aa64_sve2, gen_gvec_ool_arg_zpz, sqneg_fns[a->esz], a, 0) 5985 5986 DO_ZPZZ(SQSHL, aa64_sve2, sve2_sqshl) 5987 DO_ZPZZ(SQRSHL, aa64_sve2, sve2_sqrshl) 5988 DO_ZPZZ(SRSHL, aa64_sve2, sve2_srshl) 5989 5990 DO_ZPZZ(UQSHL, aa64_sve2, sve2_uqshl) 5991 DO_ZPZZ(UQRSHL, aa64_sve2, sve2_uqrshl) 5992 DO_ZPZZ(URSHL, aa64_sve2, sve2_urshl) 5993 5994 DO_ZPZZ(SHADD, aa64_sve2, sve2_shadd) 5995 DO_ZPZZ(SRHADD, aa64_sve2, sve2_srhadd) 5996 DO_ZPZZ(SHSUB, aa64_sve2, sve2_shsub) 5997 5998 DO_ZPZZ(UHADD, aa64_sve2, sve2_uhadd) 5999 DO_ZPZZ(URHADD, aa64_sve2, sve2_urhadd) 6000 DO_ZPZZ(UHSUB, aa64_sve2, sve2_uhsub) 6001 6002 DO_ZPZZ(ADDP, aa64_sve2, sve2_addp) 6003 DO_ZPZZ(SMAXP, aa64_sve2, sve2_smaxp) 6004 DO_ZPZZ(UMAXP, aa64_sve2, sve2_umaxp) 6005 DO_ZPZZ(SMINP, aa64_sve2, sve2_sminp) 6006 DO_ZPZZ(UMINP, aa64_sve2, sve2_uminp) 6007 6008 DO_ZPZZ(SQADD_zpzz, aa64_sve2, sve2_sqadd) 6009 DO_ZPZZ(UQADD_zpzz, aa64_sve2, sve2_uqadd) 6010 DO_ZPZZ(SQSUB_zpzz, aa64_sve2, sve2_sqsub) 6011 DO_ZPZZ(UQSUB_zpzz, aa64_sve2, sve2_uqsub) 6012 DO_ZPZZ(SUQADD, aa64_sve2, sve2_suqadd) 6013 DO_ZPZZ(USQADD, aa64_sve2, sve2_usqadd) 6014 6015 /* 6016 * SVE2 Widening Integer Arithmetic 6017 */ 6018 6019 static gen_helper_gvec_3 * const saddl_fns[4] = { 6020 NULL, gen_helper_sve2_saddl_h, 6021 gen_helper_sve2_saddl_s, gen_helper_sve2_saddl_d, 6022 }; 6023 TRANS_FEAT(SADDLB, aa64_sve2, gen_gvec_ool_arg_zzz, 6024 saddl_fns[a->esz], a, 0) 6025 TRANS_FEAT(SADDLT, aa64_sve2, gen_gvec_ool_arg_zzz, 6026 saddl_fns[a->esz], a, 3) 6027 TRANS_FEAT(SADDLBT, aa64_sve2, gen_gvec_ool_arg_zzz, 6028 saddl_fns[a->esz], a, 2) 6029 6030 static gen_helper_gvec_3 * const ssubl_fns[4] = { 6031 NULL, gen_helper_sve2_ssubl_h, 6032 gen_helper_sve2_ssubl_s, gen_helper_sve2_ssubl_d, 6033 }; 6034 TRANS_FEAT(SSUBLB, aa64_sve2, gen_gvec_ool_arg_zzz, 6035 ssubl_fns[a->esz], a, 0) 6036 TRANS_FEAT(SSUBLT, aa64_sve2, gen_gvec_ool_arg_zzz, 6037 ssubl_fns[a->esz], a, 3) 6038 TRANS_FEAT(SSUBLBT, aa64_sve2, gen_gvec_ool_arg_zzz, 6039 ssubl_fns[a->esz], a, 2) 6040 TRANS_FEAT(SSUBLTB, aa64_sve2, gen_gvec_ool_arg_zzz, 6041 ssubl_fns[a->esz], a, 1) 6042 6043 static gen_helper_gvec_3 * const sabdl_fns[4] = { 6044 NULL, gen_helper_sve2_sabdl_h, 6045 gen_helper_sve2_sabdl_s, gen_helper_sve2_sabdl_d, 6046 }; 6047 TRANS_FEAT(SABDLB, aa64_sve2, gen_gvec_ool_arg_zzz, 6048 sabdl_fns[a->esz], a, 0) 6049 TRANS_FEAT(SABDLT, aa64_sve2, gen_gvec_ool_arg_zzz, 6050 sabdl_fns[a->esz], a, 3) 6051 6052 static gen_helper_gvec_3 * const uaddl_fns[4] = { 6053 NULL, gen_helper_sve2_uaddl_h, 6054 gen_helper_sve2_uaddl_s, gen_helper_sve2_uaddl_d, 6055 }; 6056 TRANS_FEAT(UADDLB, aa64_sve2, gen_gvec_ool_arg_zzz, 6057 uaddl_fns[a->esz], a, 0) 6058 TRANS_FEAT(UADDLT, aa64_sve2, gen_gvec_ool_arg_zzz, 6059 uaddl_fns[a->esz], a, 3) 6060 6061 static gen_helper_gvec_3 * const usubl_fns[4] = { 6062 NULL, gen_helper_sve2_usubl_h, 6063 gen_helper_sve2_usubl_s, gen_helper_sve2_usubl_d, 6064 }; 6065 TRANS_FEAT(USUBLB, aa64_sve2, gen_gvec_ool_arg_zzz, 6066 usubl_fns[a->esz], a, 0) 6067 TRANS_FEAT(USUBLT, aa64_sve2, gen_gvec_ool_arg_zzz, 6068 usubl_fns[a->esz], a, 3) 6069 6070 static gen_helper_gvec_3 * const uabdl_fns[4] = { 6071 NULL, gen_helper_sve2_uabdl_h, 6072 gen_helper_sve2_uabdl_s, gen_helper_sve2_uabdl_d, 6073 }; 6074 TRANS_FEAT(UABDLB, aa64_sve2, gen_gvec_ool_arg_zzz, 6075 uabdl_fns[a->esz], a, 0) 6076 TRANS_FEAT(UABDLT, aa64_sve2, gen_gvec_ool_arg_zzz, 6077 uabdl_fns[a->esz], a, 3) 6078 6079 static gen_helper_gvec_3 * const sqdmull_fns[4] = { 6080 NULL, gen_helper_sve2_sqdmull_zzz_h, 6081 gen_helper_sve2_sqdmull_zzz_s, gen_helper_sve2_sqdmull_zzz_d, 6082 }; 6083 TRANS_FEAT(SQDMULLB_zzz, aa64_sve2, gen_gvec_ool_arg_zzz, 6084 sqdmull_fns[a->esz], a, 0) 6085 TRANS_FEAT(SQDMULLT_zzz, aa64_sve2, gen_gvec_ool_arg_zzz, 6086 sqdmull_fns[a->esz], a, 3) 6087 6088 static gen_helper_gvec_3 * const smull_fns[4] = { 6089 NULL, gen_helper_sve2_smull_zzz_h, 6090 gen_helper_sve2_smull_zzz_s, gen_helper_sve2_smull_zzz_d, 6091 }; 6092 TRANS_FEAT(SMULLB_zzz, aa64_sve2, gen_gvec_ool_arg_zzz, 6093 smull_fns[a->esz], a, 0) 6094 TRANS_FEAT(SMULLT_zzz, aa64_sve2, gen_gvec_ool_arg_zzz, 6095 smull_fns[a->esz], a, 3) 6096 6097 static gen_helper_gvec_3 * const umull_fns[4] = { 6098 NULL, gen_helper_sve2_umull_zzz_h, 6099 gen_helper_sve2_umull_zzz_s, gen_helper_sve2_umull_zzz_d, 6100 }; 6101 TRANS_FEAT(UMULLB_zzz, aa64_sve2, gen_gvec_ool_arg_zzz, 6102 umull_fns[a->esz], a, 0) 6103 TRANS_FEAT(UMULLT_zzz, aa64_sve2, gen_gvec_ool_arg_zzz, 6104 umull_fns[a->esz], a, 3) 6105 6106 static gen_helper_gvec_3 * const eoril_fns[4] = { 6107 gen_helper_sve2_eoril_b, gen_helper_sve2_eoril_h, 6108 gen_helper_sve2_eoril_s, gen_helper_sve2_eoril_d, 6109 }; 6110 TRANS_FEAT(EORBT, aa64_sve2, gen_gvec_ool_arg_zzz, eoril_fns[a->esz], a, 2) 6111 TRANS_FEAT(EORTB, aa64_sve2, gen_gvec_ool_arg_zzz, eoril_fns[a->esz], a, 1) 6112 6113 static bool do_trans_pmull(DisasContext *s, arg_rrr_esz *a, bool sel) 6114 { 6115 static gen_helper_gvec_3 * const fns[4] = { 6116 gen_helper_gvec_pmull_q, gen_helper_sve2_pmull_h, 6117 NULL, gen_helper_sve2_pmull_d, 6118 }; 6119 6120 if (a->esz == 0) { 6121 if (!dc_isar_feature(aa64_sve2_pmull128, s)) { 6122 return false; 6123 } 6124 s->is_nonstreaming = true; 6125 } else if (!dc_isar_feature(aa64_sve, s)) { 6126 return false; 6127 } 6128 return gen_gvec_ool_arg_zzz(s, fns[a->esz], a, sel); 6129 } 6130 6131 TRANS_FEAT(PMULLB, aa64_sve2, do_trans_pmull, a, false) 6132 TRANS_FEAT(PMULLT, aa64_sve2, do_trans_pmull, a, true) 6133 6134 static gen_helper_gvec_3 * const saddw_fns[4] = { 6135 NULL, gen_helper_sve2_saddw_h, 6136 gen_helper_sve2_saddw_s, gen_helper_sve2_saddw_d, 6137 }; 6138 TRANS_FEAT(SADDWB, aa64_sve2, gen_gvec_ool_arg_zzz, saddw_fns[a->esz], a, 0) 6139 TRANS_FEAT(SADDWT, aa64_sve2, gen_gvec_ool_arg_zzz, saddw_fns[a->esz], a, 1) 6140 6141 static gen_helper_gvec_3 * const ssubw_fns[4] = { 6142 NULL, gen_helper_sve2_ssubw_h, 6143 gen_helper_sve2_ssubw_s, gen_helper_sve2_ssubw_d, 6144 }; 6145 TRANS_FEAT(SSUBWB, aa64_sve2, gen_gvec_ool_arg_zzz, ssubw_fns[a->esz], a, 0) 6146 TRANS_FEAT(SSUBWT, aa64_sve2, gen_gvec_ool_arg_zzz, ssubw_fns[a->esz], a, 1) 6147 6148 static gen_helper_gvec_3 * const uaddw_fns[4] = { 6149 NULL, gen_helper_sve2_uaddw_h, 6150 gen_helper_sve2_uaddw_s, gen_helper_sve2_uaddw_d, 6151 }; 6152 TRANS_FEAT(UADDWB, aa64_sve2, gen_gvec_ool_arg_zzz, uaddw_fns[a->esz], a, 0) 6153 TRANS_FEAT(UADDWT, aa64_sve2, gen_gvec_ool_arg_zzz, uaddw_fns[a->esz], a, 1) 6154 6155 static gen_helper_gvec_3 * const usubw_fns[4] = { 6156 NULL, gen_helper_sve2_usubw_h, 6157 gen_helper_sve2_usubw_s, gen_helper_sve2_usubw_d, 6158 }; 6159 TRANS_FEAT(USUBWB, aa64_sve2, gen_gvec_ool_arg_zzz, usubw_fns[a->esz], a, 0) 6160 TRANS_FEAT(USUBWT, aa64_sve2, gen_gvec_ool_arg_zzz, usubw_fns[a->esz], a, 1) 6161 6162 static void gen_sshll_vec(unsigned vece, TCGv_vec d, TCGv_vec n, int64_t imm) 6163 { 6164 int top = imm & 1; 6165 int shl = imm >> 1; 6166 int halfbits = 4 << vece; 6167 6168 if (top) { 6169 if (shl == halfbits) { 6170 tcg_gen_and_vec(vece, d, n, 6171 tcg_constant_vec_matching(d, vece, 6172 MAKE_64BIT_MASK(halfbits, halfbits))); 6173 } else { 6174 tcg_gen_sari_vec(vece, d, n, halfbits); 6175 tcg_gen_shli_vec(vece, d, d, shl); 6176 } 6177 } else { 6178 tcg_gen_shli_vec(vece, d, n, halfbits); 6179 tcg_gen_sari_vec(vece, d, d, halfbits - shl); 6180 } 6181 } 6182 6183 static void gen_ushll_i64(unsigned vece, TCGv_i64 d, TCGv_i64 n, int imm) 6184 { 6185 int halfbits = 4 << vece; 6186 int top = imm & 1; 6187 int shl = (imm >> 1); 6188 int shift; 6189 uint64_t mask; 6190 6191 mask = MAKE_64BIT_MASK(0, halfbits); 6192 mask <<= shl; 6193 mask = dup_const(vece, mask); 6194 6195 shift = shl - top * halfbits; 6196 if (shift < 0) { 6197 tcg_gen_shri_i64(d, n, -shift); 6198 } else { 6199 tcg_gen_shli_i64(d, n, shift); 6200 } 6201 tcg_gen_andi_i64(d, d, mask); 6202 } 6203 6204 static void gen_ushll16_i64(TCGv_i64 d, TCGv_i64 n, int64_t imm) 6205 { 6206 gen_ushll_i64(MO_16, d, n, imm); 6207 } 6208 6209 static void gen_ushll32_i64(TCGv_i64 d, TCGv_i64 n, int64_t imm) 6210 { 6211 gen_ushll_i64(MO_32, d, n, imm); 6212 } 6213 6214 static void gen_ushll64_i64(TCGv_i64 d, TCGv_i64 n, int64_t imm) 6215 { 6216 gen_ushll_i64(MO_64, d, n, imm); 6217 } 6218 6219 static void gen_ushll_vec(unsigned vece, TCGv_vec d, TCGv_vec n, int64_t imm) 6220 { 6221 int halfbits = 4 << vece; 6222 int top = imm & 1; 6223 int shl = imm >> 1; 6224 6225 if (top) { 6226 if (shl == halfbits) { 6227 tcg_gen_and_vec(vece, d, n, 6228 tcg_constant_vec_matching(d, vece, 6229 MAKE_64BIT_MASK(halfbits, halfbits))); 6230 } else { 6231 tcg_gen_shri_vec(vece, d, n, halfbits); 6232 tcg_gen_shli_vec(vece, d, d, shl); 6233 } 6234 } else { 6235 if (shl == 0) { 6236 tcg_gen_and_vec(vece, d, n, 6237 tcg_constant_vec_matching(d, vece, 6238 MAKE_64BIT_MASK(0, halfbits))); 6239 } else { 6240 tcg_gen_shli_vec(vece, d, n, halfbits); 6241 tcg_gen_shri_vec(vece, d, d, halfbits - shl); 6242 } 6243 } 6244 } 6245 6246 static bool do_shll_tb(DisasContext *s, arg_rri_esz *a, 6247 const GVecGen2i ops[3], bool sel) 6248 { 6249 6250 if (a->esz < 0 || a->esz > 2) { 6251 return false; 6252 } 6253 if (sve_access_check(s)) { 6254 unsigned vsz = vec_full_reg_size(s); 6255 tcg_gen_gvec_2i(vec_full_reg_offset(s, a->rd), 6256 vec_full_reg_offset(s, a->rn), 6257 vsz, vsz, (a->imm << 1) | sel, 6258 &ops[a->esz]); 6259 } 6260 return true; 6261 } 6262 6263 static const TCGOpcode sshll_list[] = { 6264 INDEX_op_shli_vec, INDEX_op_sari_vec, 0 6265 }; 6266 static const GVecGen2i sshll_ops[3] = { 6267 { .fniv = gen_sshll_vec, 6268 .opt_opc = sshll_list, 6269 .fno = gen_helper_sve2_sshll_h, 6270 .vece = MO_16 }, 6271 { .fniv = gen_sshll_vec, 6272 .opt_opc = sshll_list, 6273 .fno = gen_helper_sve2_sshll_s, 6274 .vece = MO_32 }, 6275 { .fniv = gen_sshll_vec, 6276 .opt_opc = sshll_list, 6277 .fno = gen_helper_sve2_sshll_d, 6278 .vece = MO_64 } 6279 }; 6280 TRANS_FEAT(SSHLLB, aa64_sve2, do_shll_tb, a, sshll_ops, false) 6281 TRANS_FEAT(SSHLLT, aa64_sve2, do_shll_tb, a, sshll_ops, true) 6282 6283 static const TCGOpcode ushll_list[] = { 6284 INDEX_op_shli_vec, INDEX_op_shri_vec, 0 6285 }; 6286 static const GVecGen2i ushll_ops[3] = { 6287 { .fni8 = gen_ushll16_i64, 6288 .fniv = gen_ushll_vec, 6289 .opt_opc = ushll_list, 6290 .fno = gen_helper_sve2_ushll_h, 6291 .vece = MO_16 }, 6292 { .fni8 = gen_ushll32_i64, 6293 .fniv = gen_ushll_vec, 6294 .opt_opc = ushll_list, 6295 .fno = gen_helper_sve2_ushll_s, 6296 .vece = MO_32 }, 6297 { .fni8 = gen_ushll64_i64, 6298 .fniv = gen_ushll_vec, 6299 .opt_opc = ushll_list, 6300 .fno = gen_helper_sve2_ushll_d, 6301 .vece = MO_64 }, 6302 }; 6303 TRANS_FEAT(USHLLB, aa64_sve2, do_shll_tb, a, ushll_ops, false) 6304 TRANS_FEAT(USHLLT, aa64_sve2, do_shll_tb, a, ushll_ops, true) 6305 6306 static gen_helper_gvec_3 * const bext_fns[4] = { 6307 gen_helper_sve2_bext_b, gen_helper_sve2_bext_h, 6308 gen_helper_sve2_bext_s, gen_helper_sve2_bext_d, 6309 }; 6310 TRANS_FEAT_NONSTREAMING(BEXT, aa64_sve2_bitperm, gen_gvec_ool_arg_zzz, 6311 bext_fns[a->esz], a, 0) 6312 6313 static gen_helper_gvec_3 * const bdep_fns[4] = { 6314 gen_helper_sve2_bdep_b, gen_helper_sve2_bdep_h, 6315 gen_helper_sve2_bdep_s, gen_helper_sve2_bdep_d, 6316 }; 6317 TRANS_FEAT_NONSTREAMING(BDEP, aa64_sve2_bitperm, gen_gvec_ool_arg_zzz, 6318 bdep_fns[a->esz], a, 0) 6319 6320 static gen_helper_gvec_3 * const bgrp_fns[4] = { 6321 gen_helper_sve2_bgrp_b, gen_helper_sve2_bgrp_h, 6322 gen_helper_sve2_bgrp_s, gen_helper_sve2_bgrp_d, 6323 }; 6324 TRANS_FEAT_NONSTREAMING(BGRP, aa64_sve2_bitperm, gen_gvec_ool_arg_zzz, 6325 bgrp_fns[a->esz], a, 0) 6326 6327 static gen_helper_gvec_3 * const cadd_fns[4] = { 6328 gen_helper_sve2_cadd_b, gen_helper_sve2_cadd_h, 6329 gen_helper_sve2_cadd_s, gen_helper_sve2_cadd_d, 6330 }; 6331 TRANS_FEAT(CADD_rot90, aa64_sve2, gen_gvec_ool_arg_zzz, 6332 cadd_fns[a->esz], a, 0) 6333 TRANS_FEAT(CADD_rot270, aa64_sve2, gen_gvec_ool_arg_zzz, 6334 cadd_fns[a->esz], a, 1) 6335 6336 static gen_helper_gvec_3 * const sqcadd_fns[4] = { 6337 gen_helper_sve2_sqcadd_b, gen_helper_sve2_sqcadd_h, 6338 gen_helper_sve2_sqcadd_s, gen_helper_sve2_sqcadd_d, 6339 }; 6340 TRANS_FEAT(SQCADD_rot90, aa64_sve2, gen_gvec_ool_arg_zzz, 6341 sqcadd_fns[a->esz], a, 0) 6342 TRANS_FEAT(SQCADD_rot270, aa64_sve2, gen_gvec_ool_arg_zzz, 6343 sqcadd_fns[a->esz], a, 1) 6344 6345 static gen_helper_gvec_4 * const sabal_fns[4] = { 6346 NULL, gen_helper_sve2_sabal_h, 6347 gen_helper_sve2_sabal_s, gen_helper_sve2_sabal_d, 6348 }; 6349 TRANS_FEAT(SABALB, aa64_sve2, gen_gvec_ool_arg_zzzz, sabal_fns[a->esz], a, 0) 6350 TRANS_FEAT(SABALT, aa64_sve2, gen_gvec_ool_arg_zzzz, sabal_fns[a->esz], a, 1) 6351 6352 static gen_helper_gvec_4 * const uabal_fns[4] = { 6353 NULL, gen_helper_sve2_uabal_h, 6354 gen_helper_sve2_uabal_s, gen_helper_sve2_uabal_d, 6355 }; 6356 TRANS_FEAT(UABALB, aa64_sve2, gen_gvec_ool_arg_zzzz, uabal_fns[a->esz], a, 0) 6357 TRANS_FEAT(UABALT, aa64_sve2, gen_gvec_ool_arg_zzzz, uabal_fns[a->esz], a, 1) 6358 6359 static bool do_adcl(DisasContext *s, arg_rrrr_esz *a, bool sel) 6360 { 6361 static gen_helper_gvec_4 * const fns[2] = { 6362 gen_helper_sve2_adcl_s, 6363 gen_helper_sve2_adcl_d, 6364 }; 6365 /* 6366 * Note that in this case the ESZ field encodes both size and sign. 6367 * Split out 'subtract' into bit 1 of the data field for the helper. 6368 */ 6369 return gen_gvec_ool_arg_zzzz(s, fns[a->esz & 1], a, (a->esz & 2) | sel); 6370 } 6371 6372 TRANS_FEAT(ADCLB, aa64_sve2, do_adcl, a, false) 6373 TRANS_FEAT(ADCLT, aa64_sve2, do_adcl, a, true) 6374 6375 TRANS_FEAT(SSRA, aa64_sve2, gen_gvec_fn_arg_zzi, gen_gvec_ssra, a) 6376 TRANS_FEAT(USRA, aa64_sve2, gen_gvec_fn_arg_zzi, gen_gvec_usra, a) 6377 TRANS_FEAT(SRSRA, aa64_sve2, gen_gvec_fn_arg_zzi, gen_gvec_srsra, a) 6378 TRANS_FEAT(URSRA, aa64_sve2, gen_gvec_fn_arg_zzi, gen_gvec_ursra, a) 6379 TRANS_FEAT(SRI, aa64_sve2, gen_gvec_fn_arg_zzi, gen_gvec_sri, a) 6380 TRANS_FEAT(SLI, aa64_sve2, gen_gvec_fn_arg_zzi, gen_gvec_sli, a) 6381 6382 TRANS_FEAT(SABA, aa64_sve2, gen_gvec_fn_arg_zzz, gen_gvec_saba, a) 6383 TRANS_FEAT(UABA, aa64_sve2, gen_gvec_fn_arg_zzz, gen_gvec_uaba, a) 6384 6385 static bool do_narrow_extract(DisasContext *s, arg_rri_esz *a, 6386 const GVecGen2 ops[3]) 6387 { 6388 if (a->esz < 0 || a->esz > MO_32 || a->imm != 0) { 6389 return false; 6390 } 6391 if (sve_access_check(s)) { 6392 unsigned vsz = vec_full_reg_size(s); 6393 tcg_gen_gvec_2(vec_full_reg_offset(s, a->rd), 6394 vec_full_reg_offset(s, a->rn), 6395 vsz, vsz, &ops[a->esz]); 6396 } 6397 return true; 6398 } 6399 6400 static const TCGOpcode sqxtn_list[] = { 6401 INDEX_op_shli_vec, INDEX_op_smin_vec, INDEX_op_smax_vec, 0 6402 }; 6403 6404 static void gen_sqxtnb_vec(unsigned vece, TCGv_vec d, TCGv_vec n) 6405 { 6406 int halfbits = 4 << vece; 6407 int64_t mask = (1ull << halfbits) - 1; 6408 int64_t min = -1ull << (halfbits - 1); 6409 int64_t max = -min - 1; 6410 6411 tcg_gen_smax_vec(vece, d, n, tcg_constant_vec_matching(d, vece, min)); 6412 tcg_gen_smin_vec(vece, d, d, tcg_constant_vec_matching(d, vece, max)); 6413 tcg_gen_and_vec(vece, d, d, tcg_constant_vec_matching(d, vece, mask)); 6414 } 6415 6416 static const GVecGen2 sqxtnb_ops[3] = { 6417 { .fniv = gen_sqxtnb_vec, 6418 .opt_opc = sqxtn_list, 6419 .fno = gen_helper_sve2_sqxtnb_h, 6420 .vece = MO_16 }, 6421 { .fniv = gen_sqxtnb_vec, 6422 .opt_opc = sqxtn_list, 6423 .fno = gen_helper_sve2_sqxtnb_s, 6424 .vece = MO_32 }, 6425 { .fniv = gen_sqxtnb_vec, 6426 .opt_opc = sqxtn_list, 6427 .fno = gen_helper_sve2_sqxtnb_d, 6428 .vece = MO_64 }, 6429 }; 6430 TRANS_FEAT(SQXTNB, aa64_sve2, do_narrow_extract, a, sqxtnb_ops) 6431 6432 static void gen_sqxtnt_vec(unsigned vece, TCGv_vec d, TCGv_vec n) 6433 { 6434 int halfbits = 4 << vece; 6435 int64_t mask = (1ull << halfbits) - 1; 6436 int64_t min = -1ull << (halfbits - 1); 6437 int64_t max = -min - 1; 6438 6439 tcg_gen_smax_vec(vece, n, n, tcg_constant_vec_matching(d, vece, min)); 6440 tcg_gen_smin_vec(vece, n, n, tcg_constant_vec_matching(d, vece, max)); 6441 tcg_gen_shli_vec(vece, n, n, halfbits); 6442 tcg_gen_bitsel_vec(vece, d, tcg_constant_vec_matching(d, vece, mask), d, n); 6443 } 6444 6445 static const GVecGen2 sqxtnt_ops[3] = { 6446 { .fniv = gen_sqxtnt_vec, 6447 .opt_opc = sqxtn_list, 6448 .load_dest = true, 6449 .fno = gen_helper_sve2_sqxtnt_h, 6450 .vece = MO_16 }, 6451 { .fniv = gen_sqxtnt_vec, 6452 .opt_opc = sqxtn_list, 6453 .load_dest = true, 6454 .fno = gen_helper_sve2_sqxtnt_s, 6455 .vece = MO_32 }, 6456 { .fniv = gen_sqxtnt_vec, 6457 .opt_opc = sqxtn_list, 6458 .load_dest = true, 6459 .fno = gen_helper_sve2_sqxtnt_d, 6460 .vece = MO_64 }, 6461 }; 6462 TRANS_FEAT(SQXTNT, aa64_sve2, do_narrow_extract, a, sqxtnt_ops) 6463 6464 static const TCGOpcode uqxtn_list[] = { 6465 INDEX_op_shli_vec, INDEX_op_umin_vec, 0 6466 }; 6467 6468 static void gen_uqxtnb_vec(unsigned vece, TCGv_vec d, TCGv_vec n) 6469 { 6470 int halfbits = 4 << vece; 6471 int64_t max = (1ull << halfbits) - 1; 6472 6473 tcg_gen_umin_vec(vece, d, n, tcg_constant_vec_matching(d, vece, max)); 6474 } 6475 6476 static const GVecGen2 uqxtnb_ops[3] = { 6477 { .fniv = gen_uqxtnb_vec, 6478 .opt_opc = uqxtn_list, 6479 .fno = gen_helper_sve2_uqxtnb_h, 6480 .vece = MO_16 }, 6481 { .fniv = gen_uqxtnb_vec, 6482 .opt_opc = uqxtn_list, 6483 .fno = gen_helper_sve2_uqxtnb_s, 6484 .vece = MO_32 }, 6485 { .fniv = gen_uqxtnb_vec, 6486 .opt_opc = uqxtn_list, 6487 .fno = gen_helper_sve2_uqxtnb_d, 6488 .vece = MO_64 }, 6489 }; 6490 TRANS_FEAT(UQXTNB, aa64_sve2, do_narrow_extract, a, uqxtnb_ops) 6491 6492 static void gen_uqxtnt_vec(unsigned vece, TCGv_vec d, TCGv_vec n) 6493 { 6494 int halfbits = 4 << vece; 6495 int64_t max = (1ull << halfbits) - 1; 6496 TCGv_vec maxv = tcg_constant_vec_matching(d, vece, max); 6497 6498 tcg_gen_umin_vec(vece, n, n, maxv); 6499 tcg_gen_shli_vec(vece, n, n, halfbits); 6500 tcg_gen_bitsel_vec(vece, d, maxv, d, n); 6501 } 6502 6503 static const GVecGen2 uqxtnt_ops[3] = { 6504 { .fniv = gen_uqxtnt_vec, 6505 .opt_opc = uqxtn_list, 6506 .load_dest = true, 6507 .fno = gen_helper_sve2_uqxtnt_h, 6508 .vece = MO_16 }, 6509 { .fniv = gen_uqxtnt_vec, 6510 .opt_opc = uqxtn_list, 6511 .load_dest = true, 6512 .fno = gen_helper_sve2_uqxtnt_s, 6513 .vece = MO_32 }, 6514 { .fniv = gen_uqxtnt_vec, 6515 .opt_opc = uqxtn_list, 6516 .load_dest = true, 6517 .fno = gen_helper_sve2_uqxtnt_d, 6518 .vece = MO_64 }, 6519 }; 6520 TRANS_FEAT(UQXTNT, aa64_sve2, do_narrow_extract, a, uqxtnt_ops) 6521 6522 static const TCGOpcode sqxtun_list[] = { 6523 INDEX_op_shli_vec, INDEX_op_umin_vec, INDEX_op_smax_vec, 0 6524 }; 6525 6526 static void gen_sqxtunb_vec(unsigned vece, TCGv_vec d, TCGv_vec n) 6527 { 6528 int halfbits = 4 << vece; 6529 int64_t max = (1ull << halfbits) - 1; 6530 6531 tcg_gen_smax_vec(vece, d, n, tcg_constant_vec_matching(d, vece, 0)); 6532 tcg_gen_umin_vec(vece, d, d, tcg_constant_vec_matching(d, vece, max)); 6533 } 6534 6535 static const GVecGen2 sqxtunb_ops[3] = { 6536 { .fniv = gen_sqxtunb_vec, 6537 .opt_opc = sqxtun_list, 6538 .fno = gen_helper_sve2_sqxtunb_h, 6539 .vece = MO_16 }, 6540 { .fniv = gen_sqxtunb_vec, 6541 .opt_opc = sqxtun_list, 6542 .fno = gen_helper_sve2_sqxtunb_s, 6543 .vece = MO_32 }, 6544 { .fniv = gen_sqxtunb_vec, 6545 .opt_opc = sqxtun_list, 6546 .fno = gen_helper_sve2_sqxtunb_d, 6547 .vece = MO_64 }, 6548 }; 6549 TRANS_FEAT(SQXTUNB, aa64_sve2, do_narrow_extract, a, sqxtunb_ops) 6550 6551 static void gen_sqxtunt_vec(unsigned vece, TCGv_vec d, TCGv_vec n) 6552 { 6553 int halfbits = 4 << vece; 6554 int64_t max = (1ull << halfbits) - 1; 6555 TCGv_vec maxv = tcg_constant_vec_matching(d, vece, max); 6556 6557 tcg_gen_smax_vec(vece, n, n, tcg_constant_vec_matching(d, vece, 0)); 6558 tcg_gen_umin_vec(vece, n, n, maxv); 6559 tcg_gen_shli_vec(vece, n, n, halfbits); 6560 tcg_gen_bitsel_vec(vece, d, maxv, d, n); 6561 } 6562 6563 static const GVecGen2 sqxtunt_ops[3] = { 6564 { .fniv = gen_sqxtunt_vec, 6565 .opt_opc = sqxtun_list, 6566 .load_dest = true, 6567 .fno = gen_helper_sve2_sqxtunt_h, 6568 .vece = MO_16 }, 6569 { .fniv = gen_sqxtunt_vec, 6570 .opt_opc = sqxtun_list, 6571 .load_dest = true, 6572 .fno = gen_helper_sve2_sqxtunt_s, 6573 .vece = MO_32 }, 6574 { .fniv = gen_sqxtunt_vec, 6575 .opt_opc = sqxtun_list, 6576 .load_dest = true, 6577 .fno = gen_helper_sve2_sqxtunt_d, 6578 .vece = MO_64 }, 6579 }; 6580 TRANS_FEAT(SQXTUNT, aa64_sve2, do_narrow_extract, a, sqxtunt_ops) 6581 6582 static bool do_shr_narrow(DisasContext *s, arg_rri_esz *a, 6583 const GVecGen2i ops[3]) 6584 { 6585 if (a->esz < 0 || a->esz > MO_32) { 6586 return false; 6587 } 6588 assert(a->imm > 0 && a->imm <= (8 << a->esz)); 6589 if (sve_access_check(s)) { 6590 unsigned vsz = vec_full_reg_size(s); 6591 tcg_gen_gvec_2i(vec_full_reg_offset(s, a->rd), 6592 vec_full_reg_offset(s, a->rn), 6593 vsz, vsz, a->imm, &ops[a->esz]); 6594 } 6595 return true; 6596 } 6597 6598 static void gen_shrnb_i64(unsigned vece, TCGv_i64 d, TCGv_i64 n, int shr) 6599 { 6600 int halfbits = 4 << vece; 6601 uint64_t mask = dup_const(vece, MAKE_64BIT_MASK(0, halfbits)); 6602 6603 tcg_gen_shri_i64(d, n, shr); 6604 tcg_gen_andi_i64(d, d, mask); 6605 } 6606 6607 static void gen_shrnb16_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr) 6608 { 6609 gen_shrnb_i64(MO_16, d, n, shr); 6610 } 6611 6612 static void gen_shrnb32_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr) 6613 { 6614 gen_shrnb_i64(MO_32, d, n, shr); 6615 } 6616 6617 static void gen_shrnb64_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr) 6618 { 6619 gen_shrnb_i64(MO_64, d, n, shr); 6620 } 6621 6622 static void gen_shrnb_vec(unsigned vece, TCGv_vec d, TCGv_vec n, int64_t shr) 6623 { 6624 int halfbits = 4 << vece; 6625 uint64_t mask = MAKE_64BIT_MASK(0, halfbits); 6626 6627 tcg_gen_shri_vec(vece, n, n, shr); 6628 tcg_gen_and_vec(vece, d, n, tcg_constant_vec_matching(d, vece, mask)); 6629 } 6630 6631 static const TCGOpcode shrnb_vec_list[] = { INDEX_op_shri_vec, 0 }; 6632 static const GVecGen2i shrnb_ops[3] = { 6633 { .fni8 = gen_shrnb16_i64, 6634 .fniv = gen_shrnb_vec, 6635 .opt_opc = shrnb_vec_list, 6636 .fno = gen_helper_sve2_shrnb_h, 6637 .vece = MO_16 }, 6638 { .fni8 = gen_shrnb32_i64, 6639 .fniv = gen_shrnb_vec, 6640 .opt_opc = shrnb_vec_list, 6641 .fno = gen_helper_sve2_shrnb_s, 6642 .vece = MO_32 }, 6643 { .fni8 = gen_shrnb64_i64, 6644 .fniv = gen_shrnb_vec, 6645 .opt_opc = shrnb_vec_list, 6646 .fno = gen_helper_sve2_shrnb_d, 6647 .vece = MO_64 }, 6648 }; 6649 TRANS_FEAT(SHRNB, aa64_sve2, do_shr_narrow, a, shrnb_ops) 6650 6651 static void gen_shrnt_i64(unsigned vece, TCGv_i64 d, TCGv_i64 n, int shr) 6652 { 6653 int halfbits = 4 << vece; 6654 uint64_t mask = dup_const(vece, MAKE_64BIT_MASK(0, halfbits)); 6655 6656 tcg_gen_shli_i64(n, n, halfbits - shr); 6657 tcg_gen_andi_i64(n, n, ~mask); 6658 tcg_gen_andi_i64(d, d, mask); 6659 tcg_gen_or_i64(d, d, n); 6660 } 6661 6662 static void gen_shrnt16_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr) 6663 { 6664 gen_shrnt_i64(MO_16, d, n, shr); 6665 } 6666 6667 static void gen_shrnt32_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr) 6668 { 6669 gen_shrnt_i64(MO_32, d, n, shr); 6670 } 6671 6672 static void gen_shrnt64_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr) 6673 { 6674 tcg_gen_shri_i64(n, n, shr); 6675 tcg_gen_deposit_i64(d, d, n, 32, 32); 6676 } 6677 6678 static void gen_shrnt_vec(unsigned vece, TCGv_vec d, TCGv_vec n, int64_t shr) 6679 { 6680 int halfbits = 4 << vece; 6681 uint64_t mask = MAKE_64BIT_MASK(0, halfbits); 6682 6683 tcg_gen_shli_vec(vece, n, n, halfbits - shr); 6684 tcg_gen_bitsel_vec(vece, d, tcg_constant_vec_matching(d, vece, mask), d, n); 6685 } 6686 6687 static const TCGOpcode shrnt_vec_list[] = { INDEX_op_shli_vec, 0 }; 6688 static const GVecGen2i shrnt_ops[3] = { 6689 { .fni8 = gen_shrnt16_i64, 6690 .fniv = gen_shrnt_vec, 6691 .opt_opc = shrnt_vec_list, 6692 .load_dest = true, 6693 .fno = gen_helper_sve2_shrnt_h, 6694 .vece = MO_16 }, 6695 { .fni8 = gen_shrnt32_i64, 6696 .fniv = gen_shrnt_vec, 6697 .opt_opc = shrnt_vec_list, 6698 .load_dest = true, 6699 .fno = gen_helper_sve2_shrnt_s, 6700 .vece = MO_32 }, 6701 { .fni8 = gen_shrnt64_i64, 6702 .fniv = gen_shrnt_vec, 6703 .opt_opc = shrnt_vec_list, 6704 .load_dest = true, 6705 .fno = gen_helper_sve2_shrnt_d, 6706 .vece = MO_64 }, 6707 }; 6708 TRANS_FEAT(SHRNT, aa64_sve2, do_shr_narrow, a, shrnt_ops) 6709 6710 static const GVecGen2i rshrnb_ops[3] = { 6711 { .fno = gen_helper_sve2_rshrnb_h }, 6712 { .fno = gen_helper_sve2_rshrnb_s }, 6713 { .fno = gen_helper_sve2_rshrnb_d }, 6714 }; 6715 TRANS_FEAT(RSHRNB, aa64_sve2, do_shr_narrow, a, rshrnb_ops) 6716 6717 static const GVecGen2i rshrnt_ops[3] = { 6718 { .fno = gen_helper_sve2_rshrnt_h }, 6719 { .fno = gen_helper_sve2_rshrnt_s }, 6720 { .fno = gen_helper_sve2_rshrnt_d }, 6721 }; 6722 TRANS_FEAT(RSHRNT, aa64_sve2, do_shr_narrow, a, rshrnt_ops) 6723 6724 static void gen_sqshrunb_vec(unsigned vece, TCGv_vec d, 6725 TCGv_vec n, int64_t shr) 6726 { 6727 int halfbits = 4 << vece; 6728 uint64_t max = MAKE_64BIT_MASK(0, halfbits); 6729 6730 tcg_gen_sari_vec(vece, n, n, shr); 6731 tcg_gen_smax_vec(vece, n, n, tcg_constant_vec_matching(d, vece, 0)); 6732 tcg_gen_umin_vec(vece, d, n, tcg_constant_vec_matching(d, vece, max)); 6733 } 6734 6735 static const TCGOpcode sqshrunb_vec_list[] = { 6736 INDEX_op_sari_vec, INDEX_op_smax_vec, INDEX_op_umin_vec, 0 6737 }; 6738 static const GVecGen2i sqshrunb_ops[3] = { 6739 { .fniv = gen_sqshrunb_vec, 6740 .opt_opc = sqshrunb_vec_list, 6741 .fno = gen_helper_sve2_sqshrunb_h, 6742 .vece = MO_16 }, 6743 { .fniv = gen_sqshrunb_vec, 6744 .opt_opc = sqshrunb_vec_list, 6745 .fno = gen_helper_sve2_sqshrunb_s, 6746 .vece = MO_32 }, 6747 { .fniv = gen_sqshrunb_vec, 6748 .opt_opc = sqshrunb_vec_list, 6749 .fno = gen_helper_sve2_sqshrunb_d, 6750 .vece = MO_64 }, 6751 }; 6752 TRANS_FEAT(SQSHRUNB, aa64_sve2, do_shr_narrow, a, sqshrunb_ops) 6753 6754 static void gen_sqshrunt_vec(unsigned vece, TCGv_vec d, 6755 TCGv_vec n, int64_t shr) 6756 { 6757 int halfbits = 4 << vece; 6758 uint64_t max = MAKE_64BIT_MASK(0, halfbits); 6759 TCGv_vec maxv = tcg_constant_vec_matching(d, vece, max); 6760 6761 tcg_gen_sari_vec(vece, n, n, shr); 6762 tcg_gen_smax_vec(vece, n, n, tcg_constant_vec_matching(d, vece, 0)); 6763 tcg_gen_umin_vec(vece, n, n, maxv); 6764 tcg_gen_shli_vec(vece, n, n, halfbits); 6765 tcg_gen_bitsel_vec(vece, d, maxv, d, n); 6766 } 6767 6768 static const TCGOpcode sqshrunt_vec_list[] = { 6769 INDEX_op_shli_vec, INDEX_op_sari_vec, 6770 INDEX_op_smax_vec, INDEX_op_umin_vec, 0 6771 }; 6772 static const GVecGen2i sqshrunt_ops[3] = { 6773 { .fniv = gen_sqshrunt_vec, 6774 .opt_opc = sqshrunt_vec_list, 6775 .load_dest = true, 6776 .fno = gen_helper_sve2_sqshrunt_h, 6777 .vece = MO_16 }, 6778 { .fniv = gen_sqshrunt_vec, 6779 .opt_opc = sqshrunt_vec_list, 6780 .load_dest = true, 6781 .fno = gen_helper_sve2_sqshrunt_s, 6782 .vece = MO_32 }, 6783 { .fniv = gen_sqshrunt_vec, 6784 .opt_opc = sqshrunt_vec_list, 6785 .load_dest = true, 6786 .fno = gen_helper_sve2_sqshrunt_d, 6787 .vece = MO_64 }, 6788 }; 6789 TRANS_FEAT(SQSHRUNT, aa64_sve2, do_shr_narrow, a, sqshrunt_ops) 6790 6791 static const GVecGen2i sqrshrunb_ops[3] = { 6792 { .fno = gen_helper_sve2_sqrshrunb_h }, 6793 { .fno = gen_helper_sve2_sqrshrunb_s }, 6794 { .fno = gen_helper_sve2_sqrshrunb_d }, 6795 }; 6796 TRANS_FEAT(SQRSHRUNB, aa64_sve2, do_shr_narrow, a, sqrshrunb_ops) 6797 6798 static const GVecGen2i sqrshrunt_ops[3] = { 6799 { .fno = gen_helper_sve2_sqrshrunt_h }, 6800 { .fno = gen_helper_sve2_sqrshrunt_s }, 6801 { .fno = gen_helper_sve2_sqrshrunt_d }, 6802 }; 6803 TRANS_FEAT(SQRSHRUNT, aa64_sve2, do_shr_narrow, a, sqrshrunt_ops) 6804 6805 static void gen_sqshrnb_vec(unsigned vece, TCGv_vec d, 6806 TCGv_vec n, int64_t shr) 6807 { 6808 int halfbits = 4 << vece; 6809 int64_t max = MAKE_64BIT_MASK(0, halfbits - 1); 6810 int64_t min = -max - 1; 6811 int64_t mask = MAKE_64BIT_MASK(0, halfbits); 6812 6813 tcg_gen_sari_vec(vece, n, n, shr); 6814 tcg_gen_smax_vec(vece, n, n, tcg_constant_vec_matching(d, vece, min)); 6815 tcg_gen_smin_vec(vece, n, n, tcg_constant_vec_matching(d, vece, max)); 6816 tcg_gen_and_vec(vece, d, n, tcg_constant_vec_matching(d, vece, mask)); 6817 } 6818 6819 static const TCGOpcode sqshrnb_vec_list[] = { 6820 INDEX_op_sari_vec, INDEX_op_smax_vec, INDEX_op_smin_vec, 0 6821 }; 6822 static const GVecGen2i sqshrnb_ops[3] = { 6823 { .fniv = gen_sqshrnb_vec, 6824 .opt_opc = sqshrnb_vec_list, 6825 .fno = gen_helper_sve2_sqshrnb_h, 6826 .vece = MO_16 }, 6827 { .fniv = gen_sqshrnb_vec, 6828 .opt_opc = sqshrnb_vec_list, 6829 .fno = gen_helper_sve2_sqshrnb_s, 6830 .vece = MO_32 }, 6831 { .fniv = gen_sqshrnb_vec, 6832 .opt_opc = sqshrnb_vec_list, 6833 .fno = gen_helper_sve2_sqshrnb_d, 6834 .vece = MO_64 }, 6835 }; 6836 TRANS_FEAT(SQSHRNB, aa64_sve2, do_shr_narrow, a, sqshrnb_ops) 6837 6838 static void gen_sqshrnt_vec(unsigned vece, TCGv_vec d, 6839 TCGv_vec n, int64_t shr) 6840 { 6841 int halfbits = 4 << vece; 6842 int64_t max = MAKE_64BIT_MASK(0, halfbits - 1); 6843 int64_t min = -max - 1; 6844 int64_t mask = MAKE_64BIT_MASK(0, halfbits); 6845 6846 tcg_gen_sari_vec(vece, n, n, shr); 6847 tcg_gen_smax_vec(vece, n, n, tcg_constant_vec_matching(d, vece, min)); 6848 tcg_gen_smin_vec(vece, n, n, tcg_constant_vec_matching(d, vece, max)); 6849 tcg_gen_shli_vec(vece, n, n, halfbits); 6850 tcg_gen_bitsel_vec(vece, d, tcg_constant_vec_matching(d, vece, mask), d, n); 6851 } 6852 6853 static const TCGOpcode sqshrnt_vec_list[] = { 6854 INDEX_op_shli_vec, INDEX_op_sari_vec, 6855 INDEX_op_smax_vec, INDEX_op_smin_vec, 0 6856 }; 6857 static const GVecGen2i sqshrnt_ops[3] = { 6858 { .fniv = gen_sqshrnt_vec, 6859 .opt_opc = sqshrnt_vec_list, 6860 .load_dest = true, 6861 .fno = gen_helper_sve2_sqshrnt_h, 6862 .vece = MO_16 }, 6863 { .fniv = gen_sqshrnt_vec, 6864 .opt_opc = sqshrnt_vec_list, 6865 .load_dest = true, 6866 .fno = gen_helper_sve2_sqshrnt_s, 6867 .vece = MO_32 }, 6868 { .fniv = gen_sqshrnt_vec, 6869 .opt_opc = sqshrnt_vec_list, 6870 .load_dest = true, 6871 .fno = gen_helper_sve2_sqshrnt_d, 6872 .vece = MO_64 }, 6873 }; 6874 TRANS_FEAT(SQSHRNT, aa64_sve2, do_shr_narrow, a, sqshrnt_ops) 6875 6876 static const GVecGen2i sqrshrnb_ops[3] = { 6877 { .fno = gen_helper_sve2_sqrshrnb_h }, 6878 { .fno = gen_helper_sve2_sqrshrnb_s }, 6879 { .fno = gen_helper_sve2_sqrshrnb_d }, 6880 }; 6881 TRANS_FEAT(SQRSHRNB, aa64_sve2, do_shr_narrow, a, sqrshrnb_ops) 6882 6883 static const GVecGen2i sqrshrnt_ops[3] = { 6884 { .fno = gen_helper_sve2_sqrshrnt_h }, 6885 { .fno = gen_helper_sve2_sqrshrnt_s }, 6886 { .fno = gen_helper_sve2_sqrshrnt_d }, 6887 }; 6888 TRANS_FEAT(SQRSHRNT, aa64_sve2, do_shr_narrow, a, sqrshrnt_ops) 6889 6890 static void gen_uqshrnb_vec(unsigned vece, TCGv_vec d, 6891 TCGv_vec n, int64_t shr) 6892 { 6893 int halfbits = 4 << vece; 6894 int64_t max = MAKE_64BIT_MASK(0, halfbits); 6895 6896 tcg_gen_shri_vec(vece, n, n, shr); 6897 tcg_gen_umin_vec(vece, d, n, tcg_constant_vec_matching(d, vece, max)); 6898 } 6899 6900 static const TCGOpcode uqshrnb_vec_list[] = { 6901 INDEX_op_shri_vec, INDEX_op_umin_vec, 0 6902 }; 6903 static const GVecGen2i uqshrnb_ops[3] = { 6904 { .fniv = gen_uqshrnb_vec, 6905 .opt_opc = uqshrnb_vec_list, 6906 .fno = gen_helper_sve2_uqshrnb_h, 6907 .vece = MO_16 }, 6908 { .fniv = gen_uqshrnb_vec, 6909 .opt_opc = uqshrnb_vec_list, 6910 .fno = gen_helper_sve2_uqshrnb_s, 6911 .vece = MO_32 }, 6912 { .fniv = gen_uqshrnb_vec, 6913 .opt_opc = uqshrnb_vec_list, 6914 .fno = gen_helper_sve2_uqshrnb_d, 6915 .vece = MO_64 }, 6916 }; 6917 TRANS_FEAT(UQSHRNB, aa64_sve2, do_shr_narrow, a, uqshrnb_ops) 6918 6919 static void gen_uqshrnt_vec(unsigned vece, TCGv_vec d, 6920 TCGv_vec n, int64_t shr) 6921 { 6922 int halfbits = 4 << vece; 6923 int64_t max = MAKE_64BIT_MASK(0, halfbits); 6924 TCGv_vec maxv = tcg_constant_vec_matching(d, vece, max); 6925 6926 tcg_gen_shri_vec(vece, n, n, shr); 6927 tcg_gen_umin_vec(vece, n, n, maxv); 6928 tcg_gen_shli_vec(vece, n, n, halfbits); 6929 tcg_gen_bitsel_vec(vece, d, maxv, d, n); 6930 } 6931 6932 static const TCGOpcode uqshrnt_vec_list[] = { 6933 INDEX_op_shli_vec, INDEX_op_shri_vec, INDEX_op_umin_vec, 0 6934 }; 6935 static const GVecGen2i uqshrnt_ops[3] = { 6936 { .fniv = gen_uqshrnt_vec, 6937 .opt_opc = uqshrnt_vec_list, 6938 .load_dest = true, 6939 .fno = gen_helper_sve2_uqshrnt_h, 6940 .vece = MO_16 }, 6941 { .fniv = gen_uqshrnt_vec, 6942 .opt_opc = uqshrnt_vec_list, 6943 .load_dest = true, 6944 .fno = gen_helper_sve2_uqshrnt_s, 6945 .vece = MO_32 }, 6946 { .fniv = gen_uqshrnt_vec, 6947 .opt_opc = uqshrnt_vec_list, 6948 .load_dest = true, 6949 .fno = gen_helper_sve2_uqshrnt_d, 6950 .vece = MO_64 }, 6951 }; 6952 TRANS_FEAT(UQSHRNT, aa64_sve2, do_shr_narrow, a, uqshrnt_ops) 6953 6954 static const GVecGen2i uqrshrnb_ops[3] = { 6955 { .fno = gen_helper_sve2_uqrshrnb_h }, 6956 { .fno = gen_helper_sve2_uqrshrnb_s }, 6957 { .fno = gen_helper_sve2_uqrshrnb_d }, 6958 }; 6959 TRANS_FEAT(UQRSHRNB, aa64_sve2, do_shr_narrow, a, uqrshrnb_ops) 6960 6961 static const GVecGen2i uqrshrnt_ops[3] = { 6962 { .fno = gen_helper_sve2_uqrshrnt_h }, 6963 { .fno = gen_helper_sve2_uqrshrnt_s }, 6964 { .fno = gen_helper_sve2_uqrshrnt_d }, 6965 }; 6966 TRANS_FEAT(UQRSHRNT, aa64_sve2, do_shr_narrow, a, uqrshrnt_ops) 6967 6968 #define DO_SVE2_ZZZ_NARROW(NAME, name) \ 6969 static gen_helper_gvec_3 * const name##_fns[4] = { \ 6970 NULL, gen_helper_sve2_##name##_h, \ 6971 gen_helper_sve2_##name##_s, gen_helper_sve2_##name##_d, \ 6972 }; \ 6973 TRANS_FEAT(NAME, aa64_sve2, gen_gvec_ool_arg_zzz, \ 6974 name##_fns[a->esz], a, 0) 6975 6976 DO_SVE2_ZZZ_NARROW(ADDHNB, addhnb) 6977 DO_SVE2_ZZZ_NARROW(ADDHNT, addhnt) 6978 DO_SVE2_ZZZ_NARROW(RADDHNB, raddhnb) 6979 DO_SVE2_ZZZ_NARROW(RADDHNT, raddhnt) 6980 6981 DO_SVE2_ZZZ_NARROW(SUBHNB, subhnb) 6982 DO_SVE2_ZZZ_NARROW(SUBHNT, subhnt) 6983 DO_SVE2_ZZZ_NARROW(RSUBHNB, rsubhnb) 6984 DO_SVE2_ZZZ_NARROW(RSUBHNT, rsubhnt) 6985 6986 static gen_helper_gvec_flags_4 * const match_fns[4] = { 6987 gen_helper_sve2_match_ppzz_b, gen_helper_sve2_match_ppzz_h, NULL, NULL 6988 }; 6989 TRANS_FEAT_NONSTREAMING(MATCH, aa64_sve2, do_ppzz_flags, a, match_fns[a->esz]) 6990 6991 static gen_helper_gvec_flags_4 * const nmatch_fns[4] = { 6992 gen_helper_sve2_nmatch_ppzz_b, gen_helper_sve2_nmatch_ppzz_h, NULL, NULL 6993 }; 6994 TRANS_FEAT_NONSTREAMING(NMATCH, aa64_sve2, do_ppzz_flags, a, nmatch_fns[a->esz]) 6995 6996 static gen_helper_gvec_4 * const histcnt_fns[4] = { 6997 NULL, NULL, gen_helper_sve2_histcnt_s, gen_helper_sve2_histcnt_d 6998 }; 6999 TRANS_FEAT_NONSTREAMING(HISTCNT, aa64_sve2, gen_gvec_ool_arg_zpzz, 7000 histcnt_fns[a->esz], a, 0) 7001 7002 TRANS_FEAT_NONSTREAMING(HISTSEG, aa64_sve2, gen_gvec_ool_arg_zzz, 7003 a->esz == 0 ? gen_helper_sve2_histseg : NULL, a, 0) 7004 7005 DO_ZPZZ_FP(FADDP, aa64_sve2, sve2_faddp_zpzz) 7006 DO_ZPZZ_FP(FMAXNMP, aa64_sve2, sve2_fmaxnmp_zpzz) 7007 DO_ZPZZ_FP(FMINNMP, aa64_sve2, sve2_fminnmp_zpzz) 7008 DO_ZPZZ_FP(FMAXP, aa64_sve2, sve2_fmaxp_zpzz) 7009 DO_ZPZZ_FP(FMINP, aa64_sve2, sve2_fminp_zpzz) 7010 7011 /* 7012 * SVE Integer Multiply-Add (unpredicated) 7013 */ 7014 7015 TRANS_FEAT_NONSTREAMING(FMMLA_s, aa64_sve_f32mm, gen_gvec_fpst_zzzz, 7016 gen_helper_fmmla_s, a->rd, a->rn, a->rm, a->ra, 7017 0, FPST_A64) 7018 TRANS_FEAT_NONSTREAMING(FMMLA_d, aa64_sve_f64mm, gen_gvec_fpst_zzzz, 7019 gen_helper_fmmla_d, a->rd, a->rn, a->rm, a->ra, 7020 0, FPST_A64) 7021 7022 static gen_helper_gvec_4 * const sqdmlal_zzzw_fns[] = { 7023 NULL, gen_helper_sve2_sqdmlal_zzzw_h, 7024 gen_helper_sve2_sqdmlal_zzzw_s, gen_helper_sve2_sqdmlal_zzzw_d, 7025 }; 7026 TRANS_FEAT(SQDMLALB_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz, 7027 sqdmlal_zzzw_fns[a->esz], a, 0) 7028 TRANS_FEAT(SQDMLALT_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz, 7029 sqdmlal_zzzw_fns[a->esz], a, 3) 7030 TRANS_FEAT(SQDMLALBT, aa64_sve2, gen_gvec_ool_arg_zzzz, 7031 sqdmlal_zzzw_fns[a->esz], a, 2) 7032 7033 static gen_helper_gvec_4 * const sqdmlsl_zzzw_fns[] = { 7034 NULL, gen_helper_sve2_sqdmlsl_zzzw_h, 7035 gen_helper_sve2_sqdmlsl_zzzw_s, gen_helper_sve2_sqdmlsl_zzzw_d, 7036 }; 7037 TRANS_FEAT(SQDMLSLB_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz, 7038 sqdmlsl_zzzw_fns[a->esz], a, 0) 7039 TRANS_FEAT(SQDMLSLT_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz, 7040 sqdmlsl_zzzw_fns[a->esz], a, 3) 7041 TRANS_FEAT(SQDMLSLBT, aa64_sve2, gen_gvec_ool_arg_zzzz, 7042 sqdmlsl_zzzw_fns[a->esz], a, 2) 7043 7044 static gen_helper_gvec_4 * const sqrdmlah_fns[] = { 7045 gen_helper_sve2_sqrdmlah_b, gen_helper_sve2_sqrdmlah_h, 7046 gen_helper_sve2_sqrdmlah_s, gen_helper_sve2_sqrdmlah_d, 7047 }; 7048 TRANS_FEAT(SQRDMLAH_zzzz, aa64_sve2, gen_gvec_ool_arg_zzzz, 7049 sqrdmlah_fns[a->esz], a, 0) 7050 7051 static gen_helper_gvec_4 * const sqrdmlsh_fns[] = { 7052 gen_helper_sve2_sqrdmlsh_b, gen_helper_sve2_sqrdmlsh_h, 7053 gen_helper_sve2_sqrdmlsh_s, gen_helper_sve2_sqrdmlsh_d, 7054 }; 7055 TRANS_FEAT(SQRDMLSH_zzzz, aa64_sve2, gen_gvec_ool_arg_zzzz, 7056 sqrdmlsh_fns[a->esz], a, 0) 7057 7058 static gen_helper_gvec_4 * const smlal_zzzw_fns[] = { 7059 NULL, gen_helper_sve2_smlal_zzzw_h, 7060 gen_helper_sve2_smlal_zzzw_s, gen_helper_sve2_smlal_zzzw_d, 7061 }; 7062 TRANS_FEAT(SMLALB_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz, 7063 smlal_zzzw_fns[a->esz], a, 0) 7064 TRANS_FEAT(SMLALT_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz, 7065 smlal_zzzw_fns[a->esz], a, 1) 7066 7067 static gen_helper_gvec_4 * const umlal_zzzw_fns[] = { 7068 NULL, gen_helper_sve2_umlal_zzzw_h, 7069 gen_helper_sve2_umlal_zzzw_s, gen_helper_sve2_umlal_zzzw_d, 7070 }; 7071 TRANS_FEAT(UMLALB_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz, 7072 umlal_zzzw_fns[a->esz], a, 0) 7073 TRANS_FEAT(UMLALT_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz, 7074 umlal_zzzw_fns[a->esz], a, 1) 7075 7076 static gen_helper_gvec_4 * const smlsl_zzzw_fns[] = { 7077 NULL, gen_helper_sve2_smlsl_zzzw_h, 7078 gen_helper_sve2_smlsl_zzzw_s, gen_helper_sve2_smlsl_zzzw_d, 7079 }; 7080 TRANS_FEAT(SMLSLB_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz, 7081 smlsl_zzzw_fns[a->esz], a, 0) 7082 TRANS_FEAT(SMLSLT_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz, 7083 smlsl_zzzw_fns[a->esz], a, 1) 7084 7085 static gen_helper_gvec_4 * const umlsl_zzzw_fns[] = { 7086 NULL, gen_helper_sve2_umlsl_zzzw_h, 7087 gen_helper_sve2_umlsl_zzzw_s, gen_helper_sve2_umlsl_zzzw_d, 7088 }; 7089 TRANS_FEAT(UMLSLB_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz, 7090 umlsl_zzzw_fns[a->esz], a, 0) 7091 TRANS_FEAT(UMLSLT_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz, 7092 umlsl_zzzw_fns[a->esz], a, 1) 7093 7094 static gen_helper_gvec_4 * const cmla_fns[] = { 7095 gen_helper_sve2_cmla_zzzz_b, gen_helper_sve2_cmla_zzzz_h, 7096 gen_helper_sve2_cmla_zzzz_s, gen_helper_sve2_cmla_zzzz_d, 7097 }; 7098 TRANS_FEAT(CMLA_zzzz, aa64_sve2, gen_gvec_ool_zzzz, 7099 cmla_fns[a->esz], a->rd, a->rn, a->rm, a->ra, a->rot) 7100 7101 static gen_helper_gvec_4 * const cdot_fns[] = { 7102 NULL, NULL, gen_helper_sve2_cdot_zzzz_s, gen_helper_sve2_cdot_zzzz_d 7103 }; 7104 TRANS_FEAT(CDOT_zzzz, aa64_sve2, gen_gvec_ool_zzzz, 7105 cdot_fns[a->esz], a->rd, a->rn, a->rm, a->ra, a->rot) 7106 7107 static gen_helper_gvec_4 * const sqrdcmlah_fns[] = { 7108 gen_helper_sve2_sqrdcmlah_zzzz_b, gen_helper_sve2_sqrdcmlah_zzzz_h, 7109 gen_helper_sve2_sqrdcmlah_zzzz_s, gen_helper_sve2_sqrdcmlah_zzzz_d, 7110 }; 7111 TRANS_FEAT(SQRDCMLAH_zzzz, aa64_sve2, gen_gvec_ool_zzzz, 7112 sqrdcmlah_fns[a->esz], a->rd, a->rn, a->rm, a->ra, a->rot) 7113 7114 TRANS_FEAT(USDOT_zzzz, aa64_sve_i8mm, gen_gvec_ool_arg_zzzz, 7115 a->esz == 2 ? gen_helper_gvec_usdot_b : NULL, a, 0) 7116 7117 TRANS_FEAT_NONSTREAMING(AESMC, aa64_sve2_aes, gen_gvec_ool_zz, 7118 gen_helper_crypto_aesmc, a->rd, a->rd, 0) 7119 TRANS_FEAT_NONSTREAMING(AESIMC, aa64_sve2_aes, gen_gvec_ool_zz, 7120 gen_helper_crypto_aesimc, a->rd, a->rd, 0) 7121 7122 TRANS_FEAT_NONSTREAMING(AESE, aa64_sve2_aes, gen_gvec_ool_arg_zzz, 7123 gen_helper_crypto_aese, a, 0) 7124 TRANS_FEAT_NONSTREAMING(AESD, aa64_sve2_aes, gen_gvec_ool_arg_zzz, 7125 gen_helper_crypto_aesd, a, 0) 7126 7127 TRANS_FEAT_NONSTREAMING(SM4E, aa64_sve2_sm4, gen_gvec_ool_arg_zzz, 7128 gen_helper_crypto_sm4e, a, 0) 7129 TRANS_FEAT_NONSTREAMING(SM4EKEY, aa64_sve2_sm4, gen_gvec_ool_arg_zzz, 7130 gen_helper_crypto_sm4ekey, a, 0) 7131 7132 TRANS_FEAT_NONSTREAMING(RAX1, aa64_sve2_sha3, gen_gvec_fn_arg_zzz, 7133 gen_gvec_rax1, a) 7134 7135 TRANS_FEAT(FCVTNT_sh, aa64_sve2, gen_gvec_fpst_arg_zpz, 7136 gen_helper_sve2_fcvtnt_sh, a, 0, FPST_A64) 7137 TRANS_FEAT(FCVTNT_ds, aa64_sve2, gen_gvec_fpst_arg_zpz, 7138 gen_helper_sve2_fcvtnt_ds, a, 0, FPST_A64) 7139 7140 TRANS_FEAT(BFCVTNT, aa64_sve_bf16, gen_gvec_fpst_arg_zpz, 7141 gen_helper_sve_bfcvtnt, a, 0, 7142 s->fpcr_ah ? FPST_AH : FPST_A64) 7143 7144 TRANS_FEAT(FCVTLT_hs, aa64_sve2, gen_gvec_fpst_arg_zpz, 7145 gen_helper_sve2_fcvtlt_hs, a, 0, FPST_A64) 7146 TRANS_FEAT(FCVTLT_sd, aa64_sve2, gen_gvec_fpst_arg_zpz, 7147 gen_helper_sve2_fcvtlt_sd, a, 0, FPST_A64) 7148 7149 TRANS_FEAT(FCVTX_ds, aa64_sve2, do_frint_mode, a, 7150 FPROUNDING_ODD, gen_helper_sve_fcvt_ds) 7151 TRANS_FEAT(FCVTXNT_ds, aa64_sve2, do_frint_mode, a, 7152 FPROUNDING_ODD, gen_helper_sve2_fcvtnt_ds) 7153 7154 static gen_helper_gvec_3_ptr * const flogb_fns[] = { 7155 NULL, gen_helper_flogb_h, 7156 gen_helper_flogb_s, gen_helper_flogb_d 7157 }; 7158 TRANS_FEAT(FLOGB, aa64_sve2, gen_gvec_fpst_arg_zpz, flogb_fns[a->esz], 7159 a, 0, a->esz == MO_16 ? FPST_A64_F16 : FPST_A64) 7160 7161 static bool do_FMLAL_zzzw(DisasContext *s, arg_rrrr_esz *a, bool sub, bool sel) 7162 { 7163 return gen_gvec_ptr_zzzz(s, gen_helper_sve2_fmlal_zzzw_s, 7164 a->rd, a->rn, a->rm, a->ra, 7165 (sel << 1) | sub, tcg_env); 7166 } 7167 7168 TRANS_FEAT(FMLALB_zzzw, aa64_sve2, do_FMLAL_zzzw, a, false, false) 7169 TRANS_FEAT(FMLALT_zzzw, aa64_sve2, do_FMLAL_zzzw, a, false, true) 7170 TRANS_FEAT(FMLSLB_zzzw, aa64_sve2, do_FMLAL_zzzw, a, true, false) 7171 TRANS_FEAT(FMLSLT_zzzw, aa64_sve2, do_FMLAL_zzzw, a, true, true) 7172 7173 static bool do_FMLAL_zzxw(DisasContext *s, arg_rrxr_esz *a, bool sub, bool sel) 7174 { 7175 return gen_gvec_ptr_zzzz(s, gen_helper_sve2_fmlal_zzxw_s, 7176 a->rd, a->rn, a->rm, a->ra, 7177 (a->index << 2) | (sel << 1) | sub, tcg_env); 7178 } 7179 7180 TRANS_FEAT(FMLALB_zzxw, aa64_sve2, do_FMLAL_zzxw, a, false, false) 7181 TRANS_FEAT(FMLALT_zzxw, aa64_sve2, do_FMLAL_zzxw, a, false, true) 7182 TRANS_FEAT(FMLSLB_zzxw, aa64_sve2, do_FMLAL_zzxw, a, true, false) 7183 TRANS_FEAT(FMLSLT_zzxw, aa64_sve2, do_FMLAL_zzxw, a, true, true) 7184 7185 TRANS_FEAT_NONSTREAMING(SMMLA, aa64_sve_i8mm, gen_gvec_ool_arg_zzzz, 7186 gen_helper_gvec_smmla_b, a, 0) 7187 TRANS_FEAT_NONSTREAMING(USMMLA, aa64_sve_i8mm, gen_gvec_ool_arg_zzzz, 7188 gen_helper_gvec_usmmla_b, a, 0) 7189 TRANS_FEAT_NONSTREAMING(UMMLA, aa64_sve_i8mm, gen_gvec_ool_arg_zzzz, 7190 gen_helper_gvec_ummla_b, a, 0) 7191 7192 TRANS_FEAT(BFDOT_zzzz, aa64_sve_bf16, gen_gvec_env_arg_zzzz, 7193 gen_helper_gvec_bfdot, a, 0) 7194 TRANS_FEAT(BFDOT_zzxz, aa64_sve_bf16, gen_gvec_env_arg_zzxz, 7195 gen_helper_gvec_bfdot_idx, a) 7196 7197 TRANS_FEAT_NONSTREAMING(BFMMLA, aa64_sve_bf16, gen_gvec_env_arg_zzzz, 7198 gen_helper_gvec_bfmmla, a, 0) 7199 7200 static bool do_BFMLAL_zzzw(DisasContext *s, arg_rrrr_esz *a, bool sel) 7201 { 7202 return gen_gvec_fpst_zzzz(s, gen_helper_gvec_bfmlal, 7203 a->rd, a->rn, a->rm, a->ra, sel, 7204 s->fpcr_ah ? FPST_AH : FPST_A64); 7205 } 7206 7207 TRANS_FEAT(BFMLALB_zzzw, aa64_sve_bf16, do_BFMLAL_zzzw, a, false) 7208 TRANS_FEAT(BFMLALT_zzzw, aa64_sve_bf16, do_BFMLAL_zzzw, a, true) 7209 7210 static bool do_BFMLAL_zzxw(DisasContext *s, arg_rrxr_esz *a, bool sel) 7211 { 7212 return gen_gvec_fpst_zzzz(s, gen_helper_gvec_bfmlal_idx, 7213 a->rd, a->rn, a->rm, a->ra, 7214 (a->index << 1) | sel, 7215 s->fpcr_ah ? FPST_AH : FPST_A64); 7216 } 7217 7218 TRANS_FEAT(BFMLALB_zzxw, aa64_sve_bf16, do_BFMLAL_zzxw, a, false) 7219 TRANS_FEAT(BFMLALT_zzxw, aa64_sve_bf16, do_BFMLAL_zzxw, a, true) 7220 7221 static bool trans_PSEL(DisasContext *s, arg_psel *a) 7222 { 7223 int vl = vec_full_reg_size(s); 7224 int pl = pred_gvec_reg_size(s); 7225 int elements = vl >> a->esz; 7226 TCGv_i64 tmp, didx, dbit; 7227 TCGv_ptr ptr; 7228 7229 if (!dc_isar_feature(aa64_sme, s)) { 7230 return false; 7231 } 7232 if (!sve_access_check(s)) { 7233 return true; 7234 } 7235 7236 tmp = tcg_temp_new_i64(); 7237 dbit = tcg_temp_new_i64(); 7238 didx = tcg_temp_new_i64(); 7239 ptr = tcg_temp_new_ptr(); 7240 7241 /* Compute the predicate element. */ 7242 tcg_gen_addi_i64(tmp, cpu_reg(s, a->rv), a->imm); 7243 if (is_power_of_2(elements)) { 7244 tcg_gen_andi_i64(tmp, tmp, elements - 1); 7245 } else { 7246 tcg_gen_remu_i64(tmp, tmp, tcg_constant_i64(elements)); 7247 } 7248 7249 /* Extract the predicate byte and bit indices. */ 7250 tcg_gen_shli_i64(tmp, tmp, a->esz); 7251 tcg_gen_andi_i64(dbit, tmp, 7); 7252 tcg_gen_shri_i64(didx, tmp, 3); 7253 if (HOST_BIG_ENDIAN) { 7254 tcg_gen_xori_i64(didx, didx, 7); 7255 } 7256 7257 /* Load the predicate word. */ 7258 tcg_gen_trunc_i64_ptr(ptr, didx); 7259 tcg_gen_add_ptr(ptr, ptr, tcg_env); 7260 tcg_gen_ld8u_i64(tmp, ptr, pred_full_reg_offset(s, a->pm)); 7261 7262 /* Extract the predicate bit and replicate to MO_64. */ 7263 tcg_gen_shr_i64(tmp, tmp, dbit); 7264 tcg_gen_andi_i64(tmp, tmp, 1); 7265 tcg_gen_neg_i64(tmp, tmp); 7266 7267 /* Apply to either copy the source, or write zeros. */ 7268 tcg_gen_gvec_ands(MO_64, pred_full_reg_offset(s, a->pd), 7269 pred_full_reg_offset(s, a->pn), tmp, pl, pl); 7270 return true; 7271 } 7272 7273 static void gen_sclamp_i32(TCGv_i32 d, TCGv_i32 n, TCGv_i32 m, TCGv_i32 a) 7274 { 7275 tcg_gen_smax_i32(d, a, n); 7276 tcg_gen_smin_i32(d, d, m); 7277 } 7278 7279 static void gen_sclamp_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 a) 7280 { 7281 tcg_gen_smax_i64(d, a, n); 7282 tcg_gen_smin_i64(d, d, m); 7283 } 7284 7285 static void gen_sclamp_vec(unsigned vece, TCGv_vec d, TCGv_vec n, 7286 TCGv_vec m, TCGv_vec a) 7287 { 7288 tcg_gen_smax_vec(vece, d, a, n); 7289 tcg_gen_smin_vec(vece, d, d, m); 7290 } 7291 7292 static void gen_sclamp(unsigned vece, uint32_t d, uint32_t n, uint32_t m, 7293 uint32_t a, uint32_t oprsz, uint32_t maxsz) 7294 { 7295 static const TCGOpcode vecop[] = { 7296 INDEX_op_smin_vec, INDEX_op_smax_vec, 0 7297 }; 7298 static const GVecGen4 ops[4] = { 7299 { .fniv = gen_sclamp_vec, 7300 .fno = gen_helper_gvec_sclamp_b, 7301 .opt_opc = vecop, 7302 .vece = MO_8 }, 7303 { .fniv = gen_sclamp_vec, 7304 .fno = gen_helper_gvec_sclamp_h, 7305 .opt_opc = vecop, 7306 .vece = MO_16 }, 7307 { .fni4 = gen_sclamp_i32, 7308 .fniv = gen_sclamp_vec, 7309 .fno = gen_helper_gvec_sclamp_s, 7310 .opt_opc = vecop, 7311 .vece = MO_32 }, 7312 { .fni8 = gen_sclamp_i64, 7313 .fniv = gen_sclamp_vec, 7314 .fno = gen_helper_gvec_sclamp_d, 7315 .opt_opc = vecop, 7316 .vece = MO_64, 7317 .prefer_i64 = TCG_TARGET_REG_BITS == 64 } 7318 }; 7319 tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &ops[vece]); 7320 } 7321 7322 TRANS_FEAT(SCLAMP, aa64_sme, gen_gvec_fn_arg_zzzz, gen_sclamp, a) 7323 7324 static void gen_uclamp_i32(TCGv_i32 d, TCGv_i32 n, TCGv_i32 m, TCGv_i32 a) 7325 { 7326 tcg_gen_umax_i32(d, a, n); 7327 tcg_gen_umin_i32(d, d, m); 7328 } 7329 7330 static void gen_uclamp_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 a) 7331 { 7332 tcg_gen_umax_i64(d, a, n); 7333 tcg_gen_umin_i64(d, d, m); 7334 } 7335 7336 static void gen_uclamp_vec(unsigned vece, TCGv_vec d, TCGv_vec n, 7337 TCGv_vec m, TCGv_vec a) 7338 { 7339 tcg_gen_umax_vec(vece, d, a, n); 7340 tcg_gen_umin_vec(vece, d, d, m); 7341 } 7342 7343 static void gen_uclamp(unsigned vece, uint32_t d, uint32_t n, uint32_t m, 7344 uint32_t a, uint32_t oprsz, uint32_t maxsz) 7345 { 7346 static const TCGOpcode vecop[] = { 7347 INDEX_op_umin_vec, INDEX_op_umax_vec, 0 7348 }; 7349 static const GVecGen4 ops[4] = { 7350 { .fniv = gen_uclamp_vec, 7351 .fno = gen_helper_gvec_uclamp_b, 7352 .opt_opc = vecop, 7353 .vece = MO_8 }, 7354 { .fniv = gen_uclamp_vec, 7355 .fno = gen_helper_gvec_uclamp_h, 7356 .opt_opc = vecop, 7357 .vece = MO_16 }, 7358 { .fni4 = gen_uclamp_i32, 7359 .fniv = gen_uclamp_vec, 7360 .fno = gen_helper_gvec_uclamp_s, 7361 .opt_opc = vecop, 7362 .vece = MO_32 }, 7363 { .fni8 = gen_uclamp_i64, 7364 .fniv = gen_uclamp_vec, 7365 .fno = gen_helper_gvec_uclamp_d, 7366 .opt_opc = vecop, 7367 .vece = MO_64, 7368 .prefer_i64 = TCG_TARGET_REG_BITS == 64 } 7369 }; 7370 tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &ops[vece]); 7371 } 7372 7373 TRANS_FEAT(UCLAMP, aa64_sme, gen_gvec_fn_arg_zzzz, gen_uclamp, a) 7374