Lines Matching +full:rx +full:- +full:eq
22 #include "translate-a64.h"
74 #include "decode-sme-fa64.c.inc"
75 #include "decode-a64.c.inc"
112 ARMMMUIdx useridx = s->mmu_idx; in get_a64_user_mem_index()
114 if (unpriv && s->unpriv) { in get_a64_user_mem_index()
116 * We have pre-computed the condition for AccType_UNPRIV. in get_a64_user_mem_index()
144 /* BTYPE is a 2-bit field, and 0 should be done with reset_btype. */ in set_btype()
147 s->btype = -1; in set_btype()
152 if (s->btype != 0) { in reset_btype()
154 s->btype = 0; in reset_btype()
160 assert(s->pc_save != -1); in gen_pc_plus_diff()
161 if (tb_cflags(s->base.tb) & CF_PCREL) { in gen_pc_plus_diff()
162 tcg_gen_addi_i64(dest, cpu_pc, (s->pc_curr - s->pc_save) + diff); in gen_pc_plus_diff()
164 tcg_gen_movi_i64(dest, s->pc_curr + diff); in gen_pc_plus_diff()
171 s->pc_save = s->pc_curr + diff; in gen_a64_update_pc()
179 * then the address is zero-extended, clearing bits [63:56]
183 * the address is sign-extended from bit 55 into bits [63:56]
193 } else if (!regime_has_2_ranges(s->mmu_idx)) { in gen_top_byte_ignore()
197 /* Sign-extend from bit 55. */ in gen_top_byte_ignore()
224 gen_top_byte_ignore(s, cpu_pc, src, s->tbii); in gen_a64_set_pc()
225 s->pc_save = -1; in gen_a64_set_pc()
232 * for the tag to be present in the FAR_ELx register. But for user-only
237 * of the write-back address.
244 gen_top_byte_ignore(s, clean, addr, s->tbid); in clean_data_tbi()
277 if (tag_checked && s->mte_active[is_unpriv]) { in gen_mte_check1_mmuidx()
282 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid); in gen_mte_check1_mmuidx()
283 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma); in gen_mte_check1_mmuidx()
286 desc = FIELD_DP32(desc, MTEDESC, SIZEM1, memop_size(memop) - 1); in gen_mte_check1_mmuidx()
309 if (tag_checked && s->mte_active[0]) { in gen_mte_checkN()
314 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid); in gen_mte_checkN()
315 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma); in gen_mte_checkN()
318 desc = FIELD_DP32(desc, MTEDESC, SIZEM1, total_size - 1); in gen_mte_checkN()
331 * naturally aligned, but it must not cross a 16-byte boundary.
374 * If size == MO_128, this is a LDXP, and the operation is single-copy in check_atomic_align()
405 } else if (!s->naa) { in check_ordered_align()
423 * Sign-extend the 32-bit value so that the GE/LT comparisons work in a64_test_cc()
424 * properly. The NE/EQ comparisons are also fine with this choice. in a64_test_cc()
426 c64->cond = c32.cond; in a64_test_cc()
427 c64->value = tcg_temp_new_i64(); in a64_test_cc()
428 tcg_gen_ext_i32_i64(c64->value, c32.value); in a64_test_cc()
433 gen_helper_rebuild_hflags_a64(tcg_env, tcg_constant_i32(s->current_el)); in gen_rebuild_hflags()
446 s->base.is_jmp = DISAS_NORETURN; in gen_exception_internal_insn()
453 s->base.is_jmp = DISAS_NORETURN; in gen_exception_bkpt_insn()
458 /* We just completed step of an insn. Move from Active-not-pending in gen_step_complete_exception()
459 * to Active-pending, and then also take the swstep exception. in gen_step_complete_exception()
468 gen_swstep_exception(s, 1, s->is_ldex); in gen_step_complete_exception()
469 s->base.is_jmp = DISAS_NORETURN; in gen_step_complete_exception()
474 if (s->ss_active) { in use_goto_tb()
477 return translator_use_goto_tb(&s->base, dest); in use_goto_tb()
482 if (use_goto_tb(s, s->pc_curr + diff)) { in gen_goto_tb()
484 * For pcrel, the pc must always be up-to-date on entry to in gen_goto_tb()
491 if (tb_cflags(s->base.tb) & CF_PCREL) { in gen_goto_tb()
498 tcg_gen_exit_tb(s->base.tb, n); in gen_goto_tb()
499 s->base.is_jmp = DISAS_NORETURN; in gen_goto_tb()
502 if (s->ss_active) { in gen_goto_tb()
506 s->base.is_jmp = DISAS_NORETURN; in gen_goto_tb()
544 * representing the register contents. This TCGv is an auto-freed
628 * Clear the bits above an N-bit vector, for N = (is_q ? 128 : 64).
658 * - if FPCR.NEP == 0, clear the high elements of reg
659 * - if FPCR.NEP == 1, set the high elements of reg from mergereg
666 if (!s->fpcr_nep) { in write_fp_dreg_merging()
682 * Write a single-prec result, but only clear the higher elements
688 if (!s->fpcr_nep) { in write_fp_sreg_merging()
700 * Write a half-prec result, but only clear the higher elements
707 if (!s->fpcr_nep) { in write_fp_hreg_merging()
718 /* Expand a 2-operand AdvSIMD vector operation using an expander function. */
726 /* Expand a 2-operand + immediate AdvSIMD vector operation using
736 /* Expand a 3-operand AdvSIMD vector operation using an expander function. */
744 /* Expand a 4-operand AdvSIMD vector operation using an expander function. */
746 int rx, GVecGen4Fn *gvec_fn, int vece) in gen_gvec_fn4() argument
749 vec_full_reg_offset(s, rm), vec_full_reg_offset(s, rx), in gen_gvec_fn4()
753 /* Expand a 2-operand operation using an out-of-line helper. */
762 /* Expand a 3-operand operation using an out-of-line helper. */
772 /* Expand a 3-operand + fpstatus pointer + simd data value operation using
773 * an out-of-line helper.
786 /* Expand a 4-operand operation using an out-of-line helper. */
798 * Expand a 4-operand operation using an out-of-line helper that takes
814 * Expand a 4-operand + fpstatus pointer + simd data value operation using
815 * an out-of-line helper.
910 if (dc->fpcr_ah) { in gen_vfp_maybe_ah_negh()
919 if (dc->fpcr_ah) { in gen_vfp_maybe_ah_negs()
928 if (dc->fpcr_ah) { in gen_vfp_maybe_ah_negd()
1006 /* dest = T0 - T1; compute C, N, V and Z flags */
1212 /* This always zero-extends and writes to a full 128 bit wide vector */ in do_fp_ld()
1372 * unallocated-encoding checks (otherwise the syndrome information
1377 if (s->fp_excp_el) { in fp_access_check_only()
1378 assert(!s->fp_access_checked); in fp_access_check_only()
1379 s->fp_access_checked = -1; in fp_access_check_only()
1383 s->fp_excp_el); in fp_access_check_only()
1386 s->fp_access_checked = 1; in fp_access_check_only()
1395 if (s->sme_trap_nonstreaming && s->is_nonstreaming) { in fp_access_check()
1404 * Return <0 for non-supported element sizes, with MO_16 controlled by
1415 return -1; in fp_access_check_scalar_hsd()
1419 return -1; in fp_access_check_scalar_hsd()
1430 return -1; in fp_access_check_vector_hsd()
1437 return -1; in fp_access_check_vector_hsd()
1441 return -1; in fp_access_check_vector_hsd()
1453 if (s->pstate_sm || !dc_isar_feature(aa64_sve, s)) { in sve_access_check()
1458 s->sve_access_checked = (ret ? 1 : -1); in sve_access_check()
1461 if (s->sve_excp_el) { in sve_access_check()
1463 assert(!s->sve_access_checked); in sve_access_check()
1465 syn_sve_access_trap(), s->sve_excp_el); in sve_access_check()
1466 s->sve_access_checked = -1; in sve_access_check()
1469 s->sve_access_checked = 1; in sve_access_check()
1480 if (s->sme_excp_el) { in sme_access_check()
1483 s->sme_excp_el); in sme_access_check()
1497 if (!s->fp_excp_el || s->sme_excp_el < s->fp_excp_el) { in sme_enabled_check()
1499 s->fp_access_checked = (ret ? 1 : -1); in sme_enabled_check()
1511 if (FIELD_EX64(req, SVCR, SM) && !s->pstate_sm) { in sme_enabled_check_with_svcr()
1516 if (FIELD_EX64(req, SVCR, ZA) && !s->pstate_za) { in sme_enabled_check_with_svcr()
1531 if (!a->q && a->esz == MO_64) { in do_gvec_op2_ool()
1535 gen_gvec_op2_ool(s, a->q, a->rd, a->rn, data, fn); in do_gvec_op2_ool()
1543 if (!a->q && a->esz == MO_64) { in do_gvec_op3_ool()
1547 gen_gvec_op3_ool(s, a->q, a->rd, a->rn, a->rm, data, fn); in do_gvec_op3_ool()
1554 if (!a->q && a->esz == MO_64) { in do_gvec_fn3()
1558 gen_gvec_fn3(s, a->q, a->rd, a->rn, a->rm, fn, a->esz); in do_gvec_fn3()
1565 if (a->esz == MO_64) { in do_gvec_fn3_no64()
1569 gen_gvec_fn3(s, a->q, a->rd, a->rn, a->rm, fn, a->esz); in do_gvec_fn3_no64()
1576 if (a->esz == MO_8) { in do_gvec_fn3_no8_no64()
1584 if (!a->q && a->esz == MO_64) { in do_gvec_fn4()
1588 gen_gvec_fn4(s, a->q, a->rd, a->rn, a->rm, a->ra, fn, a->esz); in do_gvec_fn4()
1611 * or SCTLR bits) there is a check that SP is 16-aligned on every in gen_check_sp_alignment()
1612 * SP-relative load or store (with an exception generated if it is not). in gen_check_sp_alignment()
1632 gen_goto_tb(s, 0, a->imm); in trans_B()
1640 gen_goto_tb(s, 0, a->imm); in trans_BL()
1650 tcg_cmp = read_cpu_reg(s, a->rt, a->sf); in trans_CBZ()
1654 tcg_gen_brcondi_i64(a->nz ? TCG_COND_NE : TCG_COND_EQ, in trans_CBZ()
1658 gen_goto_tb(s, 1, a->imm); in trans_CBZ()
1668 tcg_gen_andi_i64(tcg_cmp, cpu_reg(s, a->rt), 1ULL << a->bitpos); in trans_TBZ()
1673 tcg_gen_brcondi_i64(a->nz ? TCG_COND_NE : TCG_COND_EQ, in trans_TBZ()
1677 gen_goto_tb(s, 1, a->imm); in trans_TBZ()
1684 if (a->c && !dc_isar_feature(aa64_hbc, s)) { in trans_B_cond()
1688 if (a->cond < 0x0e) { in trans_B_cond()
1691 arm_gen_test_cc(a->cond, match.label); in trans_B_cond()
1694 gen_goto_tb(s, 1, a->imm); in trans_B_cond()
1697 gen_goto_tb(s, 0, a->imm); in trans_B_cond()
1705 /* BR to {x16,x17} or !guard -> 1, else 3. */ in set_btype_for_br()
1712 s->btype = -1; in set_btype_for_br()
1727 set_btype_for_br(s, a->rn); in trans_BR()
1728 gen_a64_set_pc(s, cpu_reg(s, a->rn)); in trans_BR()
1729 s->base.is_jmp = DISAS_JUMP; in trans_BR()
1735 TCGv_i64 dst = cpu_reg(s, a->rn); in trans_BLR()
1745 s->base.is_jmp = DISAS_JUMP; in trans_BLR()
1751 gen_a64_set_pc(s, cpu_reg(s, a->rn)); in trans_RET()
1752 s->base.is_jmp = DISAS_JUMP; in trans_RET()
1765 if (!s->pauth_active) { in auth_branch_target()
1786 dst = auth_branch_target(s, cpu_reg(s, a->rn), tcg_constant_i64(0), !a->m); in trans_BRAZ()
1787 set_btype_for_br(s, a->rn); in trans_BRAZ()
1789 s->base.is_jmp = DISAS_JUMP; in trans_BRAZ()
1801 dst = auth_branch_target(s, cpu_reg(s, a->rn), tcg_constant_i64(0), !a->m); in trans_BLRAZ()
1811 s->base.is_jmp = DISAS_JUMP; in trans_BLRAZ()
1819 dst = auth_branch_target(s, cpu_reg(s, 30), cpu_X[31], !a->m); in trans_RETA()
1821 s->base.is_jmp = DISAS_JUMP; in trans_RETA()
1832 dst = auth_branch_target(s, cpu_reg(s,a->rn), cpu_reg_sp(s, a->rm), !a->m); in trans_BRA()
1834 set_btype_for_br(s, a->rn); in trans_BRA()
1835 s->base.is_jmp = DISAS_JUMP; in trans_BRA()
1846 dst = auth_branch_target(s, cpu_reg(s, a->rn), cpu_reg_sp(s, a->rm), !a->m); in trans_BLRA()
1856 s->base.is_jmp = DISAS_JUMP; in trans_BLRA()
1864 if (s->current_el == 0) { in trans_ERET()
1867 if (s->trap_eret) { in trans_ERET()
1873 offsetof(CPUARMState, elr_el[s->current_el])); in trans_ERET()
1875 translator_io_start(&s->base); in trans_ERET()
1878 /* Must exit loop to check un-masked IRQs */ in trans_ERET()
1879 s->base.is_jmp = DISAS_EXIT; in trans_ERET()
1890 if (s->current_el == 0) { in trans_ERETA()
1894 if (s->trap_eret) { in trans_ERETA()
1895 gen_exception_insn_el(s, 0, EXCP_UDEF, syn_erettrap(a->m ? 3 : 2), 2); in trans_ERETA()
1900 offsetof(CPUARMState, elr_el[s->current_el])); in trans_ERETA()
1902 dst = auth_branch_target(s, dst, cpu_X[31], !a->m); in trans_ERETA()
1904 translator_io_start(&s->base); in trans_ERETA()
1907 /* Must exit loop to check un-masked IRQs */ in trans_ERETA()
1908 s->base.is_jmp = DISAS_EXIT; in trans_ERETA()
1925 if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) { in trans_YIELD()
1926 s->base.is_jmp = DISAS_YIELD; in trans_YIELD()
1933 s->base.is_jmp = DISAS_WFI; in trans_WFI()
1945 if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) { in trans_WFE()
1946 s->base.is_jmp = DISAS_WFE; in trans_WFE()
1961 * check ss_active so that single-stepping a WFIT doesn't halt. in trans_WFIT()
1963 if (s->ss_active) { in trans_WFIT()
1969 gen_helper_wfit(tcg_env, cpu_reg(s, a->rd)); in trans_WFIT()
1971 s->base.is_jmp = DISAS_EXIT; in trans_WFIT()
1986 if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) { in trans_WFET()
1987 s->base.is_jmp = DISAS_WFE; in trans_WFET()
1994 if (s->pauth_active) { in trans_XPACLRI()
2002 if (s->pauth_active) { in trans_PACIA1716()
2010 if (s->pauth_active) { in trans_PACIB1716()
2018 if (s->pauth_active) { in trans_AUTIA1716()
2026 if (s->pauth_active) { in trans_AUTIB1716()
2045 if (s->current_el <= 1 && arm_dc_feature(s, ARM_FEATURE_EL2)) { in trans_ESB()
2054 if (s->pauth_active) { in trans_PACIAZ()
2062 if (s->pauth_active) { in trans_PACIASP()
2070 if (s->pauth_active) { in trans_PACIBZ()
2078 if (s->pauth_active) { in trans_PACIBSP()
2086 if (s->pauth_active) { in trans_AUTIAZ()
2094 if (s->pauth_active) { in trans_AUTIASP()
2102 if (s->pauth_active) { in trans_AUTIBZ()
2110 if (s->pauth_active) { in trans_AUTIBSP()
2118 tcg_gen_movi_i64(cpu_exclusive_addr, -1); in trans_CLREX()
2127 switch (a->types) { in trans_DSB_DMB()
2155 * self-modifying code correctly and also to take in trans_ISB()
2202 * ~-(C | Z) in trans_XAFLAG()
2203 * (C | Z) - 1 in trans_XAFLAG()
2212 /* (!C & Z) << 31 -> -(Z & ~C) */ in trans_XAFLAG()
2228 tcg_gen_sari_i32(cpu_VF, cpu_VF, 31); /* V ? -1 : 0 */ in trans_AXFLAG()
2231 /* !(Z | V) -> !(!ZF | V) -> ZF & !V -> ZF & ~VF */ in trans_AXFLAG()
2242 if (!dc_isar_feature(aa64_uao, s) || s->current_el == 0) { in trans_MSR_i_UAO()
2245 if (a->imm & 1) { in trans_MSR_i_UAO()
2251 s->base.is_jmp = DISAS_TOO_MANY; in trans_MSR_i_UAO()
2257 if (!dc_isar_feature(aa64_pan, s) || s->current_el == 0) { in trans_MSR_i_PAN()
2260 if (a->imm & 1) { in trans_MSR_i_PAN()
2266 s->base.is_jmp = DISAS_TOO_MANY; in trans_MSR_i_PAN()
2272 if (s->current_el == 0) { in trans_MSR_i_SPSEL()
2275 gen_helper_msr_i_spsel(tcg_env, tcg_constant_i32(a->imm & PSTATE_SP)); in trans_MSR_i_SPSEL()
2276 s->base.is_jmp = DISAS_TOO_MANY; in trans_MSR_i_SPSEL()
2285 if (a->imm & 1) { in trans_MSR_i_SBSS()
2291 s->base.is_jmp = DISAS_TOO_MANY; in trans_MSR_i_SBSS()
2300 if (a->imm & 1) { in trans_MSR_i_DIT()
2306 s->base.is_jmp = DISAS_TOO_MANY; in trans_MSR_i_DIT()
2313 /* Full MTE is enabled -- set the TCO bit as directed. */ in trans_MSR_i_TCO()
2314 if (a->imm & 1) { in trans_MSR_i_TCO()
2321 s->base.is_jmp = DISAS_UPDATE_NOCHAIN; in trans_MSR_i_TCO()
2324 /* Only "instructions accessible at EL0" -- PSTATE.TCO is WI. */ in trans_MSR_i_TCO()
2334 gen_helper_msr_i_daifset(tcg_env, tcg_constant_i32(a->imm)); in trans_MSR_i_DAIFSET()
2335 s->base.is_jmp = DISAS_TOO_MANY; in trans_MSR_i_DAIFSET()
2341 gen_helper_msr_i_daifclear(tcg_env, tcg_constant_i32(a->imm)); in trans_MSR_i_DAIFCLEAR()
2342 /* Exit the cpu loop to re-evaluate pending IRQs. */ in trans_MSR_i_DAIFCLEAR()
2343 s->base.is_jmp = DISAS_UPDATE_EXIT; in trans_MSR_i_DAIFCLEAR()
2349 if (!dc_isar_feature(aa64_nmi, s) || s->current_el == 0) { in trans_MSR_i_ALLINT()
2353 if (a->imm == 0) { in trans_MSR_i_ALLINT()
2355 } else if (s->current_el > 1) { in trans_MSR_i_ALLINT()
2361 /* Exit the cpu loop to re-evaluate pending IRQs. */ in trans_MSR_i_ALLINT()
2362 s->base.is_jmp = DISAS_UPDATE_EXIT; in trans_MSR_i_ALLINT()
2368 if (!dc_isar_feature(aa64_sme, s) || a->mask == 0) { in trans_MSR_i_SVCR()
2372 int old = s->pstate_sm | (s->pstate_za << 1); in trans_MSR_i_SVCR()
2373 int new = a->imm * 3; in trans_MSR_i_SVCR()
2375 if ((old ^ new) & a->mask) { in trans_MSR_i_SVCR()
2378 tcg_constant_i32(a->mask)); in trans_MSR_i_SVCR()
2379 s->base.is_jmp = DISAS_TOO_MANY; in trans_MSR_i_SVCR()
2447 /* MRS - move from system register
2448 * MSR (register) - move to system register
2460 const ARMCPRegInfo *ri = get_arm_cp_reginfo(s->cp_regs, key); in handle_sys()
2475 switch (s->current_el) { in handle_sys()
2498 if (s->nv2 && ri->nv2_redirect_offset) { in handle_sys()
2504 if (ri->nv2_redirect_offset & NV2_REDIR_NV1) { in handle_sys()
2505 nv2_mem_redirect = s->nv1; in handle_sys()
2506 } else if (ri->nv2_redirect_offset & NV2_REDIR_NO_NV1) { in handle_sys()
2507 nv2_mem_redirect = !s->nv1; in handle_sys()
2514 if (!cp_access_ok(s->current_el, ri, isread)) { in handle_sys()
2521 if (s->nv2 && (ri->type & ARM_CP_NV2_REDIRECT)) { in handle_sys()
2531 * NV2 redirect-to-memory takes precedence over trap to EL2 or in handle_sys()
2534 } else if (s->nv && arm_cpreg_traps_in_nv(ri)) { in handle_sys()
2541 * over the trap-to-EL2. Most trapped-by-FEAT_NV registers have in handle_sys()
2543 * the trap-to-EL3 controls which would apply to that register in handle_sys()
2544 * at EL2 don't take priority over the FEAT_NV trap-to-EL2.) in handle_sys()
2553 if (ri->accessfn || (ri->fgt && s->fgt_active)) { in handle_sys()
2563 } else if (ri->type & ARM_CP_RAISES_EXC) { in handle_sys()
2572 if ((ri->type & ARM_CP_FPU) && !fp_access_check_only(s)) { in handle_sys()
2574 } else if ((ri->type & ARM_CP_SVE) && !sve_access_check(s)) { in handle_sys()
2576 } else if ((ri->type & ARM_CP_SME) && !sme_access_check(s)) { in handle_sys()
2593 * fine-grained-traps on EL1 also do not apply here. in handle_sys()
2597 ri = get_arm_cp_reginfo(s->cp_regs, key); in handle_sys()
2599 assert(cp_access_ok(s->current_el, ri, isread)); in handle_sys()
2604 assert(!(ri->type & ARM_CP_RAISES_EXC)); in handle_sys()
2613 * The access is 64-bit single copy atomic, guaranteed aligned because in handle_sys()
2622 ARMMMUIdx armmemidx = s->nv2_mem_e20 ? ARMMMUIdx_E20_2 : ARMMMUIdx_E2; in handle_sys()
2626 mop |= (s->nv2_mem_be ? MO_BE : MO_LE); in handle_sys()
2630 (ri->nv2_redirect_offset & ~NV2_REDIR_FLAG_MASK)); in handle_sys()
2644 switch (ri->type & ARM_CP_SPECIAL_MASK) { in handle_sys()
2664 int el = s->nv ? 2 : s->current_el; in handle_sys()
2671 if (s->mte_active[0]) { in handle_sys()
2675 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid); in handle_sys()
2676 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma); in handle_sys()
2698 if (s->ata[0]) { in handle_sys()
2715 if (s->ata[0]) { in handle_sys()
2727 if (ri->type & ARM_CP_IO) { in handle_sys()
2729 need_exit_tb = translator_io_start(&s->base); in handle_sys()
2735 if (ri->type & ARM_CP_CONST) { in handle_sys()
2736 tcg_gen_movi_i64(tcg_rt, ri->resetvalue); in handle_sys()
2737 } else if (ri->readfn) { in handle_sys()
2743 tcg_gen_ld_i64(tcg_rt, tcg_env, ri->fieldoffset); in handle_sys()
2746 if (ri->type & ARM_CP_CONST) { in handle_sys()
2749 } else if (ri->writefn) { in handle_sys()
2755 tcg_gen_st_i64(tcg_rt, tcg_env, ri->fieldoffset); in handle_sys()
2759 if (!isread && !(ri->type & ARM_CP_SUPPRESS_TB_END)) { in handle_sys()
2773 s->base.is_jmp = DISAS_UPDATE_EXIT; in handle_sys()
2779 handle_sys(s, a->l, a->op0, a->op1, a->op2, a->crn, a->crm, a->rt); in trans_SYS()
2786 * For SVC, HVC and SMC we advance the single-step state in trans_SVC()
2788 * mandated, to ensure that single-stepping a system call in trans_SVC()
2791 uint32_t syndrome = syn_aa64_svc(a->imm); in trans_SVC()
2792 if (s->fgt_svc) { in trans_SVC()
2803 int target_el = s->current_el == 3 ? 3 : 2; in trans_HVC()
2805 if (s->current_el == 0) { in trans_HVC()
2817 gen_exception_insn_el(s, 4, EXCP_HVC, syn_aa64_hvc(a->imm), target_el); in trans_HVC()
2823 if (s->current_el == 0) { in trans_SMC()
2828 gen_helper_pre_smc(tcg_env, tcg_constant_i32(syn_aa64_smc(a->imm))); in trans_SMC()
2831 gen_exception_insn_el(s, 4, EXCP_SMC, syn_aa64_smc(a->imm), 3); in trans_SMC()
2837 gen_exception_bkpt_insn(s, syn_aa64_bkpt(a->imm)); in trans_BRK()
2850 if (semihosting_enabled(s->current_el == 0) && a->imm == 0xf000) { in trans_HLT()
2866 * races in multi-threaded linux-user and when MTTCG softmmu is
2876 s->is_ldex = true; in gen_load_exclusive()
2885 if (s->be_data == MO_LE) { in gen_load_exclusive()
2897 if (s->be_data == MO_LE) { in gen_load_exclusive()
2917 /* if (env->exclusive_addr == addr && env->exclusive_val == [addr] in gen_store_exclusive()
2918 * && (!is_pair || env->exclusive_high == [addr + datasize])) { in gen_store_exclusive()
2927 * env->exclusive_addr = -1; in gen_store_exclusive()
2953 * It is possible to trigger an alignment fault pre-LSE2, e.g. with an in gen_store_exclusive()
2954 * unaligned 4-byte write within the range of an aligned 8-byte load. in gen_store_exclusive()
2955 * With LSE2, the store would need to cross a 16-byte boundary when the in gen_store_exclusive()
2975 if (s->be_data == MO_LE) { in gen_store_exclusive()
2989 if (s->be_data == MO_LE) { in gen_store_exclusive()
3004 if (s->be_data == MO_LE) { in gen_store_exclusive()
3027 tcg_gen_movi_i64(cpu_exclusive_addr, -1); in gen_store_exclusive()
3071 if (s->be_data == MO_LE) { in gen_compare_and_swap_pair()
3081 if (s->be_data == MO_LE) { in gen_compare_and_swap_pair()
3090 if (s->be_data == MO_LE) { in gen_compare_and_swap_pair()
3100 if (s->be_data == MO_LE) { in gen_compare_and_swap_pair()
3111 * is accessing a 32-bit or 64-bit register. This logic is derived
3120 * do a zero-extend from 32 to 64 after the load. in ldst_iss_sf()
3132 if (a->rn == 31) { in trans_STXR()
3135 if (a->lasr) { in trans_STXR()
3138 gen_store_exclusive(s, a->rs, a->rt, a->rt2, a->rn, a->sz, false); in trans_STXR()
3144 if (a->rn == 31) { in trans_LDXR()
3147 gen_load_exclusive(s, a->rt, a->rt2, a->rn, a->sz, false); in trans_LDXR()
3148 if (a->lasr) { in trans_LDXR()
3158 bool iss_sf = ldst_iss_sf(a->sz, false, false); in trans_STLR()
3161 * StoreLORelease is the same as Store-Release for QEMU, but in trans_STLR()
3162 * needs the feature-test. in trans_STLR()
3164 if (!a->lasr && !dc_isar_feature(aa64_lor, s)) { in trans_STLR()
3167 /* Generate ISS for non-exclusive accesses including LASR. */ in trans_STLR()
3168 if (a->rn == 31) { in trans_STLR()
3172 memop = check_ordered_align(s, a->rn, 0, true, a->sz); in trans_STLR()
3173 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn), in trans_STLR()
3174 true, a->rn != 31, memop); in trans_STLR()
3175 do_gpr_st(s, cpu_reg(s, a->rt), clean_addr, memop, true, a->rt, in trans_STLR()
3176 iss_sf, a->lasr); in trans_STLR()
3184 bool iss_sf = ldst_iss_sf(a->sz, false, false); in trans_LDAR()
3186 /* LoadLOAcquire is the same as Load-Acquire for QEMU. */ in trans_LDAR()
3187 if (!a->lasr && !dc_isar_feature(aa64_lor, s)) { in trans_LDAR()
3190 /* Generate ISS for non-exclusive accesses including LASR. */ in trans_LDAR()
3191 if (a->rn == 31) { in trans_LDAR()
3194 memop = check_ordered_align(s, a->rn, 0, false, a->sz); in trans_LDAR()
3195 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn), in trans_LDAR()
3196 false, a->rn != 31, memop); in trans_LDAR()
3197 do_gpr_ld(s, cpu_reg(s, a->rt), clean_addr, memop, false, true, in trans_LDAR()
3198 a->rt, iss_sf, a->lasr); in trans_LDAR()
3205 if (a->rn == 31) { in trans_STXP()
3208 if (a->lasr) { in trans_STXP()
3211 gen_store_exclusive(s, a->rs, a->rt, a->rt2, a->rn, a->sz, true); in trans_STXP()
3217 if (a->rn == 31) { in trans_LDXP()
3220 gen_load_exclusive(s, a->rt, a->rt2, a->rn, a->sz, true); in trans_LDXP()
3221 if (a->lasr) { in trans_LDXP()
3232 if (((a->rt | a->rs) & 1) != 0) { in trans_CASP()
3236 gen_compare_and_swap_pair(s, a->rs, a->rt, a->rn, a->sz); in trans_CASP()
3245 gen_compare_and_swap(s, a->rs, a->rt, a->rn, a->sz); in trans_CAS()
3251 bool iss_sf = ldst_iss_sf(a->sz, a->sign, false); in trans_LD_lit()
3252 TCGv_i64 tcg_rt = cpu_reg(s, a->rt); in trans_LD_lit()
3254 MemOp memop = finalize_memop(s, a->sz + a->sign * MO_SIGN); in trans_LD_lit()
3256 gen_pc_plus_diff(s, clean_addr, a->imm); in trans_LD_lit()
3258 false, true, a->rt, iss_sf, false); in trans_LD_lit()
3271 memop = finalize_memop_asimd(s, a->sz); in trans_LD_lit_v()
3273 gen_pc_plus_diff(s, clean_addr, a->imm); in trans_LD_lit_v()
3274 do_fp_ld(s, a->rt, clean_addr, memop); in trans_LD_lit_v()
3282 if (a->rn == 31) { in op_addr_ldstpair_pre()
3286 *dirty_addr = read_cpu_reg_sp(s, a->rn, 1); in op_addr_ldstpair_pre()
3287 if (!a->p) { in op_addr_ldstpair_pre()
3292 (a->w || a->rn != 31), 2 << a->sz, mop); in op_addr_ldstpair_pre()
3298 if (a->w) { in op_addr_ldstpair_post()
3299 if (a->p) { in op_addr_ldstpair_post()
3302 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), dirty_addr); in op_addr_ldstpair_post()
3308 uint64_t offset = a->imm << a->sz; in trans_STP()
3310 MemOp mop = finalize_memop(s, a->sz); in trans_STP()
3313 tcg_rt = cpu_reg(s, a->rt); in trans_STP()
3314 tcg_rt2 = cpu_reg(s, a->rt2); in trans_STP()
3316 * We built mop above for the single logical access -- rebuild it in trans_STP()
3319 * With LSE2, non-sign-extending pairs are treated atomically if in trans_STP()
3321 * within a 16-byte block and that element will be atomic. in trans_STP()
3325 mop = a->sz + 1; in trans_STP()
3326 if (s->align_mem) { in trans_STP()
3327 mop |= (a->sz == 2 ? MO_ALIGN_4 : MO_ALIGN_8); in trans_STP()
3330 if (a->sz == 2) { in trans_STP()
3333 if (s->be_data == MO_LE) { in trans_STP()
3342 if (s->be_data == MO_LE) { in trans_STP()
3355 uint64_t offset = a->imm << a->sz; in trans_LDP()
3357 MemOp mop = finalize_memop(s, a->sz); in trans_LDP()
3360 tcg_rt = cpu_reg(s, a->rt); in trans_LDP()
3361 tcg_rt2 = cpu_reg(s, a->rt2); in trans_LDP()
3364 * We built mop above for the single logical access -- rebuild it in trans_LDP()
3367 * With LSE2, non-sign-extending pairs are treated atomically if in trans_LDP()
3369 * within a 16-byte block and that element will be atomic. in trans_LDP()
3373 * This treats sign-extending loads like zero-extending loads, in trans_LDP()
3376 mop = a->sz + 1; in trans_LDP()
3377 if (s->align_mem) { in trans_LDP()
3378 mop |= (a->sz == 2 ? MO_ALIGN_4 : MO_ALIGN_8); in trans_LDP()
3381 if (a->sz == 2) { in trans_LDP()
3382 int o2 = s->be_data == MO_LE ? 32 : 0; in trans_LDP()
3386 if (a->sign) { in trans_LDP()
3397 if (s->be_data == MO_LE) { in trans_LDP()
3409 uint64_t offset = a->imm << a->sz; in trans_STP_v()
3418 mop = finalize_memop_asimd(s, a->sz); in trans_STP_v()
3420 do_fp_st(s, a->rt, clean_addr, mop); in trans_STP_v()
3421 tcg_gen_addi_i64(clean_addr, clean_addr, 1 << a->sz); in trans_STP_v()
3422 do_fp_st(s, a->rt2, clean_addr, mop); in trans_STP_v()
3429 uint64_t offset = a->imm << a->sz; in trans_LDP_v()
3438 mop = finalize_memop_asimd(s, a->sz); in trans_LDP_v()
3440 do_fp_ld(s, a->rt, clean_addr, mop); in trans_LDP_v()
3441 tcg_gen_addi_i64(clean_addr, clean_addr, 1 << a->sz); in trans_LDP_v()
3442 do_fp_ld(s, a->rt2, clean_addr, mop); in trans_LDP_v()
3450 uint64_t offset = a->imm << LOG2_TAG_GRANULE; in trans_STGP()
3455 tcg_debug_assert(a->sz == MO_64); in trans_STGP()
3461 if (a->rn == 31) { in trans_STGP()
3465 dirty_addr = read_cpu_reg_sp(s, a->rn, 1); in trans_STGP()
3466 if (!a->p) { in trans_STGP()
3471 tcg_rt = cpu_reg(s, a->rt); in trans_STGP()
3472 tcg_rt2 = cpu_reg(s, a->rt2); in trans_STGP()
3475 * STGP is defined as two 8-byte memory operations, aligned to TAG_GRANULE, in trans_STGP()
3476 * and one tag operation. We implement it as one single aligned 16-byte in trans_STGP()
3478 * MO_ATOM_IFALIGN_PAIR produces 8-byte atomicity for the memory store. in trans_STGP()
3483 if (s->be_data == MO_LE) { in trans_STGP()
3491 if (s->ata[0]) { in trans_STGP()
3492 if (tb_cflags(s->base.tb) & CF_PARALLEL) { in trans_STGP()
3509 if (a->rn == 31) { in op_addr_ldst_imm_pre()
3513 *dirty_addr = read_cpu_reg_sp(s, a->rn, 1); in op_addr_ldst_imm_pre()
3514 if (!a->p) { in op_addr_ldst_imm_pre()
3517 memidx = get_a64_user_mem_index(s, a->unpriv); in op_addr_ldst_imm_pre()
3519 a->w || a->rn != 31, in op_addr_ldst_imm_pre()
3520 mop, a->unpriv, memidx); in op_addr_ldst_imm_pre()
3526 if (a->w) { in op_addr_ldst_imm_post()
3527 if (a->p) { in op_addr_ldst_imm_post()
3530 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), dirty_addr); in op_addr_ldst_imm_post()
3536 bool iss_sf, iss_valid = !a->w; in trans_STR_i()
3538 int memidx = get_a64_user_mem_index(s, a->unpriv); in trans_STR_i()
3539 MemOp mop = finalize_memop(s, a->sz + a->sign * MO_SIGN); in trans_STR_i()
3541 op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, true, mop); in trans_STR_i()
3543 tcg_rt = cpu_reg(s, a->rt); in trans_STR_i()
3544 iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext); in trans_STR_i()
3547 iss_valid, a->rt, iss_sf, false); in trans_STR_i()
3548 op_addr_ldst_imm_post(s, a, dirty_addr, a->imm); in trans_STR_i()
3554 bool iss_sf, iss_valid = !a->w; in trans_LDR_i()
3556 int memidx = get_a64_user_mem_index(s, a->unpriv); in trans_LDR_i()
3557 MemOp mop = finalize_memop(s, a->sz + a->sign * MO_SIGN); in trans_LDR_i()
3559 op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, false, mop); in trans_LDR_i()
3561 tcg_rt = cpu_reg(s, a->rt); in trans_LDR_i()
3562 iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext); in trans_LDR_i()
3565 a->ext, memidx, iss_valid, a->rt, iss_sf, false); in trans_LDR_i()
3566 op_addr_ldst_imm_post(s, a, dirty_addr, a->imm); in trans_LDR_i()
3578 mop = finalize_memop_asimd(s, a->sz); in trans_STR_v_i()
3579 op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, true, mop); in trans_STR_v_i()
3580 do_fp_st(s, a->rt, clean_addr, mop); in trans_STR_v_i()
3581 op_addr_ldst_imm_post(s, a, dirty_addr, a->imm); in trans_STR_v_i()
3593 mop = finalize_memop_asimd(s, a->sz); in trans_LDR_v_i()
3594 op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, false, mop); in trans_LDR_v_i()
3595 do_fp_ld(s, a->rt, clean_addr, mop); in trans_LDR_v_i()
3596 op_addr_ldst_imm_post(s, a, dirty_addr, a->imm); in trans_LDR_v_i()
3606 if (a->rn == 31) { in op_addr_ldst_pre()
3609 *dirty_addr = read_cpu_reg_sp(s, a->rn, 1); in op_addr_ldst_pre()
3611 tcg_rm = read_cpu_reg(s, a->rm, 1); in op_addr_ldst_pre()
3612 ext_and_shift_reg(tcg_rm, tcg_rm, a->opt, a->s ? a->sz : 0); in op_addr_ldst_pre()
3621 bool iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext); in trans_LDR()
3624 if (extract32(a->opt, 1, 1) == 0) { in trans_LDR()
3628 memop = finalize_memop(s, a->sz + a->sign * MO_SIGN); in trans_LDR()
3630 tcg_rt = cpu_reg(s, a->rt); in trans_LDR()
3632 a->ext, true, a->rt, iss_sf, false); in trans_LDR()
3639 bool iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext); in trans_STR()
3642 if (extract32(a->opt, 1, 1) == 0) { in trans_STR()
3646 memop = finalize_memop(s, a->sz); in trans_STR()
3648 tcg_rt = cpu_reg(s, a->rt); in trans_STR()
3649 do_gpr_st(s, tcg_rt, clean_addr, memop, true, a->rt, iss_sf, false); in trans_STR()
3658 if (extract32(a->opt, 1, 1) == 0) { in trans_LDR_v()
3666 memop = finalize_memop_asimd(s, a->sz); in trans_LDR_v()
3668 do_fp_ld(s, a->rt, clean_addr, memop); in trans_LDR_v()
3677 if (extract32(a->opt, 1, 1) == 0) { in trans_STR_v()
3685 memop = finalize_memop_asimd(s, a->sz); in trans_STR_v()
3687 do_fp_st(s, a->rt, clean_addr, memop); in trans_STR_v()
3695 MemOp mop = a->sz | sign; in do_atomic_ld()
3698 if (a->rn == 31) { in do_atomic_ld()
3701 mop = check_atomic_align(s, a->rn, mop); in do_atomic_ld()
3702 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn), false, in do_atomic_ld()
3703 a->rn != 31, mop); in do_atomic_ld()
3704 tcg_rs = read_cpu_reg(s, a->rs, true); in do_atomic_ld()
3705 tcg_rt = cpu_reg(s, a->rt); in do_atomic_ld()
3716 switch (a->sz) { in do_atomic_ld()
3747 bool iss_sf = ldst_iss_sf(a->sz, false, false); in TRANS_FEAT()
3755 if (a->rn == 31) { in TRANS_FEAT()
3758 mop = check_ordered_align(s, a->rn, 0, false, a->sz); in TRANS_FEAT()
3759 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn), false, in TRANS_FEAT()
3760 a->rn != 31, mop); in TRANS_FEAT()
3763 * fetch-and-do-something op. in TRANS_FEAT()
3765 * full load-acquire (we only need "load-acquire processor consistent"), in TRANS_FEAT()
3768 do_gpr_ld(s, cpu_reg(s, a->rt), clean_addr, mop, false, in TRANS_FEAT()
3769 true, a->rt, iss_sf, true); in TRANS_FEAT()
3784 if (a->rn == 31) { in trans_LDRA()
3787 dirty_addr = read_cpu_reg_sp(s, a->rn, 1); in trans_LDRA()
3789 if (s->pauth_active) { in trans_LDRA()
3790 if (!a->m) { in trans_LDRA()
3799 tcg_gen_addi_i64(dirty_addr, dirty_addr, a->imm); in trans_LDRA()
3805 a->w || a->rn != 31, memop); in trans_LDRA()
3807 tcg_rt = cpu_reg(s, a->rt); in trans_LDRA()
3809 /* extend */ false, /* iss_valid */ !a->w, in trans_LDRA()
3810 /* iss_srt */ a->rt, /* iss_sf */ true, /* iss_ar */ false); in trans_LDRA()
3812 if (a->w) { in trans_LDRA()
3813 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), dirty_addr); in trans_LDRA()
3821 MemOp mop = a->sz | (a->sign ? MO_SIGN : 0); in trans_LDAPR_i()
3822 bool iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext); in trans_LDAPR_i()
3828 if (a->rn == 31) { in trans_LDAPR_i()
3832 mop = check_ordered_align(s, a->rn, a->imm, false, mop); in trans_LDAPR_i()
3833 dirty_addr = read_cpu_reg_sp(s, a->rn, 1); in trans_LDAPR_i()
3834 tcg_gen_addi_i64(dirty_addr, dirty_addr, a->imm); in trans_LDAPR_i()
3838 * Load-AcquirePC semantics; we implement as the slightly more in trans_LDAPR_i()
3839 * restrictive Load-Acquire. in trans_LDAPR_i()
3841 do_gpr_ld(s, cpu_reg(s, a->rt), clean_addr, mop, a->ext, true, in trans_LDAPR_i()
3842 a->rt, iss_sf, true); in trans_LDAPR_i()
3850 MemOp mop = a->sz; in trans_STLR_i()
3851 bool iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext); in trans_STLR_i()
3857 /* TODO: ARMv8.4-LSE SCTLR.nAA */ in trans_STLR_i()
3859 if (a->rn == 31) { in trans_STLR_i()
3863 mop = check_ordered_align(s, a->rn, a->imm, true, mop); in trans_STLR_i()
3864 dirty_addr = read_cpu_reg_sp(s, a->rn, 1); in trans_STLR_i()
3865 tcg_gen_addi_i64(dirty_addr, dirty_addr, a->imm); in trans_STLR_i()
3868 /* Store-Release semantics */ in trans_STLR_i()
3870 do_gpr_st(s, cpu_reg(s, a->rt), clean_addr, mop, true, a->rt, iss_sf, true); in trans_STLR_i()
3882 int size = a->sz; in trans_LD_mult()
3884 if (!a->p && a->rm != 0) { in trans_LD_mult()
3885 /* For non-postindexed accesses the Rm field must be 0 */ in trans_LD_mult()
3888 if (size == 3 && !a->q && a->selem != 1) { in trans_LD_mult()
3895 if (a->rn == 31) { in trans_LD_mult()
3899 /* For our purposes, bytes are always little-endian. */ in trans_LD_mult()
3900 endian = s->be_data; in trans_LD_mult()
3905 total = a->rpt * a->selem * (a->q ? 16 : 8); in trans_LD_mult()
3906 tcg_rn = cpu_reg_sp(s, a->rn); in trans_LD_mult()
3910 * promote consecutive little-endian elements below. in trans_LD_mult()
3912 clean_addr = gen_mte_checkN(s, tcg_rn, false, a->p || a->rn != 31, total, in trans_LD_mult()
3916 * Consecutive little-endian elements from a single register in trans_LD_mult()
3917 * can be promoted to a larger little-endian operation. in trans_LD_mult()
3920 if (a->selem == 1 && endian == MO_LE) { in trans_LD_mult()
3924 if (!s->align_mem) { in trans_LD_mult()
3929 elements = (a->q ? 16 : 8) >> size; in trans_LD_mult()
3931 for (r = 0; r < a->rpt; r++) { in trans_LD_mult()
3935 for (xs = 0; xs < a->selem; xs++) { in trans_LD_mult()
3936 int tt = (a->rt + r + xs) % 32; in trans_LD_mult()
3944 * For non-quad operations, setting a slice of the low 64 bits of in trans_LD_mult()
3950 for (r = 0; r < a->rpt * a->selem; r++) { in trans_LD_mult()
3951 int tt = (a->rt + r) % 32; in trans_LD_mult()
3952 clear_vec_high(s, a->q, tt); in trans_LD_mult()
3955 if (a->p) { in trans_LD_mult()
3956 if (a->rm == 31) { in trans_LD_mult()
3959 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm)); in trans_LD_mult()
3973 int size = a->sz; in trans_ST_mult()
3975 if (!a->p && a->rm != 0) { in trans_ST_mult()
3976 /* For non-postindexed accesses the Rm field must be 0 */ in trans_ST_mult()
3979 if (size == 3 && !a->q && a->selem != 1) { in trans_ST_mult()
3986 if (a->rn == 31) { in trans_ST_mult()
3990 /* For our purposes, bytes are always little-endian. */ in trans_ST_mult()
3991 endian = s->be_data; in trans_ST_mult()
3996 total = a->rpt * a->selem * (a->q ? 16 : 8); in trans_ST_mult()
3997 tcg_rn = cpu_reg_sp(s, a->rn); in trans_ST_mult()
4001 * promote consecutive little-endian elements below. in trans_ST_mult()
4003 clean_addr = gen_mte_checkN(s, tcg_rn, true, a->p || a->rn != 31, total, in trans_ST_mult()
4007 * Consecutive little-endian elements from a single register in trans_ST_mult()
4008 * can be promoted to a larger little-endian operation. in trans_ST_mult()
4011 if (a->selem == 1 && endian == MO_LE) { in trans_ST_mult()
4015 if (!s->align_mem) { in trans_ST_mult()
4020 elements = (a->q ? 16 : 8) >> size; in trans_ST_mult()
4022 for (r = 0; r < a->rpt; r++) { in trans_ST_mult()
4026 for (xs = 0; xs < a->selem; xs++) { in trans_ST_mult()
4027 int tt = (a->rt + r + xs) % 32; in trans_ST_mult()
4034 if (a->p) { in trans_ST_mult()
4035 if (a->rm == 31) { in trans_ST_mult()
4038 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm)); in trans_ST_mult()
4050 if (!a->p && a->rm != 0) { in trans_ST_single()
4057 if (a->rn == 31) { in trans_ST_single()
4061 total = a->selem << a->scale; in trans_ST_single()
4062 tcg_rn = cpu_reg_sp(s, a->rn); in trans_ST_single()
4064 mop = finalize_memop_asimd(s, a->scale); in trans_ST_single()
4065 clean_addr = gen_mte_checkN(s, tcg_rn, true, a->p || a->rn != 31, in trans_ST_single()
4068 tcg_ebytes = tcg_constant_i64(1 << a->scale); in trans_ST_single()
4069 for (xs = 0, rt = a->rt; xs < a->selem; xs++, rt = (rt + 1) % 32) { in trans_ST_single()
4070 do_vec_st(s, rt, a->index, clean_addr, mop); in trans_ST_single()
4074 if (a->p) { in trans_ST_single()
4075 if (a->rm == 31) { in trans_ST_single()
4078 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm)); in trans_ST_single()
4090 if (!a->p && a->rm != 0) { in trans_LD_single()
4097 if (a->rn == 31) { in trans_LD_single()
4101 total = a->selem << a->scale; in trans_LD_single()
4102 tcg_rn = cpu_reg_sp(s, a->rn); in trans_LD_single()
4104 mop = finalize_memop_asimd(s, a->scale); in trans_LD_single()
4105 clean_addr = gen_mte_checkN(s, tcg_rn, false, a->p || a->rn != 31, in trans_LD_single()
4108 tcg_ebytes = tcg_constant_i64(1 << a->scale); in trans_LD_single()
4109 for (xs = 0, rt = a->rt; xs < a->selem; xs++, rt = (rt + 1) % 32) { in trans_LD_single()
4110 do_vec_ld(s, rt, a->index, clean_addr, mop); in trans_LD_single()
4114 if (a->p) { in trans_LD_single()
4115 if (a->rm == 31) { in trans_LD_single()
4118 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm)); in trans_LD_single()
4130 if (!a->p && a->rm != 0) { in trans_LD_single_repl()
4137 if (a->rn == 31) { in trans_LD_single_repl()
4141 total = a->selem << a->scale; in trans_LD_single_repl()
4142 tcg_rn = cpu_reg_sp(s, a->rn); in trans_LD_single_repl()
4144 mop = finalize_memop_asimd(s, a->scale); in trans_LD_single_repl()
4145 clean_addr = gen_mte_checkN(s, tcg_rn, false, a->p || a->rn != 31, in trans_LD_single_repl()
4148 tcg_ebytes = tcg_constant_i64(1 << a->scale); in trans_LD_single_repl()
4149 for (xs = 0, rt = a->rt; xs < a->selem; xs++, rt = (rt + 1) % 32) { in trans_LD_single_repl()
4154 tcg_gen_gvec_dup_i64(a->scale, vec_full_reg_offset(s, rt), in trans_LD_single_repl()
4155 (a->q + 1) * 8, vec_full_reg_size(s), tcg_tmp); in trans_LD_single_repl()
4159 if (a->p) { in trans_LD_single_repl()
4160 if (a->rm == 31) { in trans_LD_single_repl()
4163 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm)); in trans_LD_single_repl()
4172 int size = 4 << s->dcz_blocksize; in trans_STZGM()
4177 if (s->current_el == 0) { in trans_STZGM()
4181 if (a->rn == 31) { in trans_STZGM()
4185 addr = read_cpu_reg_sp(s, a->rn, true); in trans_STZGM()
4186 tcg_gen_addi_i64(addr, addr, a->imm); in trans_STZGM()
4187 tcg_rt = cpu_reg(s, a->rt); in trans_STZGM()
4189 if (s->ata[0]) { in trans_STZGM()
4193 * The non-tags portion of STZGM is mostly like DC_ZVA, in trans_STZGM()
4197 tcg_gen_andi_i64(clean_addr, clean_addr, -size); in trans_STZGM()
4209 if (s->current_el == 0) { in trans_STGM()
4213 if (a->rn == 31) { in trans_STGM()
4217 addr = read_cpu_reg_sp(s, a->rn, true); in trans_STGM()
4218 tcg_gen_addi_i64(addr, addr, a->imm); in trans_STGM()
4219 tcg_rt = cpu_reg(s, a->rt); in trans_STGM()
4221 if (s->ata[0]) { in trans_STGM()
4225 int size = 4 << s->gm_blocksize; in trans_STGM()
4228 tcg_gen_andi_i64(clean_addr, clean_addr, -size); in trans_STGM()
4241 if (s->current_el == 0) { in trans_LDGM()
4245 if (a->rn == 31) { in trans_LDGM()
4249 addr = read_cpu_reg_sp(s, a->rn, true); in trans_LDGM()
4250 tcg_gen_addi_i64(addr, addr, a->imm); in trans_LDGM()
4251 tcg_rt = cpu_reg(s, a->rt); in trans_LDGM()
4253 if (s->ata[0]) { in trans_LDGM()
4257 int size = 4 << s->gm_blocksize; in trans_LDGM()
4260 tcg_gen_andi_i64(clean_addr, clean_addr, -size); in trans_LDGM()
4276 if (a->rn == 31) { in trans_LDG()
4280 addr = read_cpu_reg_sp(s, a->rn, true); in trans_LDG()
4281 if (!a->p) { in trans_LDG()
4282 /* pre-index or signed offset */ in trans_LDG()
4283 tcg_gen_addi_i64(addr, addr, a->imm); in trans_LDG()
4286 tcg_gen_andi_i64(addr, addr, -TAG_GRANULE); in trans_LDG()
4287 tcg_rt = cpu_reg(s, a->rt); in trans_LDG()
4288 if (s->ata[0]) { in trans_LDG()
4300 if (a->w) { in trans_LDG()
4301 /* pre-index or post-index */ in trans_LDG()
4302 if (a->p) { in trans_LDG()
4303 /* post-index */ in trans_LDG()
4304 tcg_gen_addi_i64(addr, addr, a->imm); in trans_LDG()
4306 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), addr); in trans_LDG()
4315 if (a->rn == 31) { in do_STG()
4319 addr = read_cpu_reg_sp(s, a->rn, true); in do_STG()
4320 if (!a->p) { in do_STG()
4321 /* pre-index or signed offset */ in do_STG()
4322 tcg_gen_addi_i64(addr, addr, a->imm); in do_STG()
4324 tcg_rt = cpu_reg_sp(s, a->rt); in do_STG()
4325 if (!s->ata[0]) { in do_STG()
4329 * at least for system mode; user-only won't enforce alignment. in do_STG()
4336 } else if (tb_cflags(s->base.tb) & CF_PARALLEL) { in do_STG()
4359 /* This is 1 or 2 atomic 16-byte operations. */ in do_STG()
4367 if (a->w) { in do_STG()
4368 /* pre-index or post-index */ in do_STG()
4369 if (a->p) { in do_STG()
4370 /* post-index */ in do_STG()
4371 tcg_gen_addi_i64(addr, addr, a->imm); in do_STG()
4373 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), addr); in do_STG()
4400 if (a->rs == a->rn || a->rs == a->rd || a->rn == a->rd || in do_SET()
4401 a->rd == 31 || a->rn == 31) { in do_SET()
4405 memidx = get_a64_user_mem_index(s, a->unpriv); in do_SET()
4411 syndrome = syn_mop(true, is_setg, (a->nontemp << 1) | a->unpriv, in do_SET()
4412 is_epilogue, false, true, a->rd, a->rs, a->rn); in do_SET()
4414 if (is_setg ? s->ata[a->unpriv] : s->mte_active[a->unpriv]) { in do_SET()
4416 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid); in do_SET()
4417 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma); in do_SET()
4446 bool wunpriv = extract32(a->options, 0, 1); in do_CPY()
4447 bool runpriv = extract32(a->options, 1, 1); in do_CPY()
4454 if (a->rs == a->rn || a->rs == a->rd || a->rn == a->rd || in do_CPY()
4455 a->rd == 31 || a->rs == 31 || a->rn == 31) { in do_CPY()
4466 syndrome = syn_mop(false, false, a->options, is_epilogue, in do_CPY()
4467 false, true, a->rd, a->rs, a->rn); in do_CPY()
4470 if (s->mte_active[runpriv]) { in do_CPY()
4471 rdesc = FIELD_DP32(rdesc, MTEDESC, TBI, s->tbid); in do_CPY()
4472 rdesc = FIELD_DP32(rdesc, MTEDESC, TCMA, s->tcma); in do_CPY()
4474 if (s->mte_active[wunpriv]) { in do_CPY()
4475 wdesc = FIELD_DP32(wdesc, MTEDESC, TBI, s->tbid); in do_CPY()
4476 wdesc = FIELD_DP32(wdesc, MTEDESC, TCMA, s->tcma); in do_CPY()
4505 TCGv_i64 tcg_rn = rn_sp ? cpu_reg_sp(s, a->rn) : cpu_reg(s, a->rn); in gen_rri()
4506 TCGv_i64 tcg_rd = rd_sp ? cpu_reg_sp(s, a->rd) : cpu_reg(s, a->rd); in gen_rri()
4507 TCGv_i64 tcg_imm = tcg_constant_i64(a->imm); in gen_rri()
4510 if (!a->sf) { in gen_rri()
4517 * PC-rel. addressing
4522 gen_pc_plus_diff(s, cpu_reg(s, a->rd), a->imm); in trans_ADR()
4528 int64_t offset = (int64_t)a->imm << 12; in trans_ADRP()
4531 offset -= s->pc_curr & 0xfff; in trans_ADRP()
4532 gen_pc_plus_diff(s, cpu_reg(s, a->rd), offset); in trans_ADRP()
4541 TRANS(ADDS_i, gen_rri, a, 0, 1, a->sf ? gen_add64_CC : gen_add32_CC)
4542 TRANS(SUBS_i, gen_rri, a, 0, 1, a->sf ? gen_sub64_CC : gen_sub32_CC)
4554 imm = a->uimm6 << LOG2_TAG_GRANULE; in gen_add_sub_imm_with_tags()
4556 imm = -imm; in gen_add_sub_imm_with_tags()
4559 tcg_rn = cpu_reg_sp(s, a->rn); in gen_add_sub_imm_with_tags()
4560 tcg_rd = cpu_reg_sp(s, a->rd); in gen_add_sub_imm_with_tags()
4562 if (s->ata[0]) { in gen_add_sub_imm_with_tags()
4565 tcg_constant_i32(a->uimm4)); in gen_add_sub_imm_with_tags()
4612 * of between 1 and e-1 non-zero bits, rotated within the in logic_imm_decode_wmask()
4613 * element by between 0 and e-1 bits. in logic_imm_decode_wmask()
4617 * 64 bit elements: immn = 1, imms = <length of run - 1> in logic_imm_decode_wmask()
4618 * 32 bit elements: immn = 0, imms = 0 : <length of run - 1> in logic_imm_decode_wmask()
4619 * 16 bit elements: immn = 0, imms = 10 : <length of run - 1> in logic_imm_decode_wmask()
4620 * 8 bit elements: immn = 0, imms = 110 : <length of run - 1> in logic_imm_decode_wmask()
4621 * 4 bit elements: immn = 0, imms = 1110 : <length of run - 1> in logic_imm_decode_wmask()
4622 * 2 bit elements: immn = 0, imms = 11110 : <length of run - 1> in logic_imm_decode_wmask()
4625 * Further, <length of run - 1> all-ones is a reserved pattern. in logic_imm_decode_wmask()
4631 len = 31 - clz32((immn << 6) | (~imms & 0x3f)); in logic_imm_decode_wmask()
4638 levels = e - 1; in logic_imm_decode_wmask()
4643 /* <length of run - 1> mustn't be all-ones. */ in logic_imm_decode_wmask()
4652 mask = (mask >> r) | (mask << (e - r)); in logic_imm_decode_wmask()
4668 if (!logic_imm_decode_wmask(&imm, extract32(a->dbm, 12, 1), in gen_rri_log()
4669 extract32(a->dbm, 0, 6), in gen_rri_log()
4670 extract32(a->dbm, 6, 6))) { in gen_rri_log()
4673 if (!a->sf) { in gen_rri_log()
4677 tcg_rd = set_cc ? cpu_reg(s, a->rd) : cpu_reg_sp(s, a->rd); in gen_rri_log()
4678 tcg_rn = cpu_reg(s, a->rn); in gen_rri_log()
4682 gen_logic_CC(a->sf, tcg_rd); in gen_rri_log()
4684 if (!a->sf) { in gen_rri_log()
4701 int pos = a->hw << 4; in TRANS()
4702 tcg_gen_movi_i64(cpu_reg(s, a->rd), (uint64_t)a->imm << pos); in TRANS()
4708 int pos = a->hw << 4; in trans_MOVN()
4709 uint64_t imm = a->imm; in trans_MOVN()
4712 if (!a->sf) { in trans_MOVN()
4715 tcg_gen_movi_i64(cpu_reg(s, a->rd), imm); in trans_MOVN()
4721 int pos = a->hw << 4; in trans_MOVK()
4724 tcg_rd = cpu_reg(s, a->rd); in trans_MOVK()
4725 tcg_im = tcg_constant_i64(a->imm); in trans_MOVK()
4727 if (!a->sf) { in trans_MOVK()
4739 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); in trans_SBFM()
4740 TCGv_i64 tcg_tmp = read_cpu_reg(s, a->rn, 1); in trans_SBFM()
4741 unsigned int bitsize = a->sf ? 64 : 32; in trans_SBFM()
4742 unsigned int ri = a->immr; in trans_SBFM()
4743 unsigned int si = a->imms; in trans_SBFM()
4747 /* Wd<s-r:0> = Wn<s:r> */ in trans_SBFM()
4748 len = (si - ri) + 1; in trans_SBFM()
4750 if (!a->sf) { in trans_SBFM()
4754 /* Wd<32+s-r,32-r> = Wn<s:0> */ in trans_SBFM()
4756 pos = (bitsize - ri) & (bitsize - 1); in trans_SBFM()
4770 * bitsize, therefore no final zero-extension is unneeded for !sf. in trans_SBFM()
4779 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); in trans_UBFM()
4780 TCGv_i64 tcg_tmp = read_cpu_reg(s, a->rn, 1); in trans_UBFM()
4781 unsigned int bitsize = a->sf ? 64 : 32; in trans_UBFM()
4782 unsigned int ri = a->immr; in trans_UBFM()
4783 unsigned int si = a->imms; in trans_UBFM()
4786 tcg_rd = cpu_reg(s, a->rd); in trans_UBFM()
4787 tcg_tmp = read_cpu_reg(s, a->rn, 1); in trans_UBFM()
4790 /* Wd<s-r:0> = Wn<s:r> */ in trans_UBFM()
4791 len = (si - ri) + 1; in trans_UBFM()
4794 /* Wd<32+s-r,32-r> = Wn<s:0> */ in trans_UBFM()
4796 pos = (bitsize - ri) & (bitsize - 1); in trans_UBFM()
4804 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); in trans_BFM()
4805 TCGv_i64 tcg_tmp = read_cpu_reg(s, a->rn, 1); in trans_BFM()
4806 unsigned int bitsize = a->sf ? 64 : 32; in trans_BFM()
4807 unsigned int ri = a->immr; in trans_BFM()
4808 unsigned int si = a->imms; in trans_BFM()
4811 tcg_rd = cpu_reg(s, a->rd); in trans_BFM()
4812 tcg_tmp = read_cpu_reg(s, a->rn, 1); in trans_BFM()
4815 /* Wd<s-r:0> = Wn<s:r> */ in trans_BFM()
4817 len = (si - ri) + 1; in trans_BFM()
4820 /* Wd<32+s-r,32-r> = Wn<s:0> */ in trans_BFM()
4822 pos = (bitsize - ri) & (bitsize - 1); in trans_BFM()
4826 if (!a->sf) { in trans_BFM()
4836 tcg_rd = cpu_reg(s, a->rd); in trans_EXTR()
4838 if (unlikely(a->imm == 0)) { in trans_EXTR()
4843 if (a->sf) { in trans_EXTR()
4844 tcg_gen_mov_i64(tcg_rd, cpu_reg(s, a->rm)); in trans_EXTR()
4846 tcg_gen_ext32u_i64(tcg_rd, cpu_reg(s, a->rm)); in trans_EXTR()
4849 tcg_rm = cpu_reg(s, a->rm); in trans_EXTR()
4850 tcg_rn = cpu_reg(s, a->rn); in trans_EXTR()
4852 if (a->sf) { in trans_EXTR()
4854 tcg_gen_extract2_i64(tcg_rd, tcg_rm, tcg_rn, a->imm); in trans_EXTR()
4859 if (a->rm == a->rn) { in trans_EXTR()
4860 tcg_gen_rotri_i32(t0, t0, a->imm); in trans_EXTR()
4864 tcg_gen_extract2_i32(t0, t0, t1, a->imm); in trans_EXTR()
4875 int len = (a->len + 1) * 16; in trans_TBL_TBX()
4877 tcg_gen_gvec_2_ptr(vec_full_reg_offset(s, a->rd), in trans_TBL_TBX()
4878 vec_full_reg_offset(s, a->rm), tcg_env, in trans_TBL_TBX()
4879 a->q ? 16 : 8, vec_full_reg_size(s), in trans_TBL_TBX()
4880 (len << 6) | (a->tbx << 5) | a->rn, in trans_TBL_TBX()
4891 MemOp esz = a->esz; in do_simd_permute()
4892 int datasize = a->q ? 16 : 8; in do_simd_permute()
4896 if (esz == MO_64 && !a->q) { in do_simd_permute()
4904 tcg_res[1] = a->q ? tcg_temp_new_i64() : NULL; in do_simd_permute()
4911 read_vec_element(s, tcg_ele, (idx & elements ? a->rm : a->rn), in do_simd_permute()
4912 idx & (elements - 1), esz); in do_simd_permute()
4923 for (int i = a->q; i >= 0; --i) { in do_simd_permute()
4924 write_vec_element(s, tcg_res[i], a->rd, i, MO_64); in do_simd_permute()
4926 clear_vec_high(s, a->q, a->rd); in do_simd_permute()
5001 read_vec_element_i32(s, tcg_op1, a->rn, 3, MO_32); in TRANS_FEAT()
5002 read_vec_element_i32(s, tcg_op2, a->rm, 3, MO_32); in TRANS_FEAT()
5003 read_vec_element_i32(s, tcg_op3, a->ra, 3, MO_32); in TRANS_FEAT()
5011 clear_vec(s, a->rd); in TRANS_FEAT()
5012 write_vec_element_i32(s, tcg_res, a->rd, 3, MO_32); in TRANS_FEAT()
5020 gen_gvec_op3_ool(s, true, a->rd, a->rn, a->rm, a->imm, fn); in do_crypto3i()
5035 gen_gvec_xar(MO_64, vec_full_reg_offset(s, a->rd), in TRANS_FEAT()
5036 vec_full_reg_offset(s, a->rn), in TRANS_FEAT()
5037 vec_full_reg_offset(s, a->rm), a->imm, 16, in TRANS_FEAT()
5063 if (!decode_esz_idx(a->imm, &esz, &idx)) { in trans_DUP_element_s()
5069 * zero-extends it into the bottom of the destination register. in trans_DUP_element_s()
5072 read_vec_element(s, tmp, a->rn, idx, esz); in trans_DUP_element_s()
5073 write_fp_dreg(s, a->rd, tmp); in trans_DUP_element_s()
5083 if (!decode_esz_idx(a->imm, &esz, &idx)) { in trans_DUP_element_v()
5086 if (esz == MO_64 && !a->q) { in trans_DUP_element_v()
5090 tcg_gen_gvec_dup_mem(esz, vec_full_reg_offset(s, a->rd), in trans_DUP_element_v()
5091 vec_reg_offset(s, a->rn, idx, esz), in trans_DUP_element_v()
5092 a->q ? 16 : 8, vec_full_reg_size(s)); in trans_DUP_element_v()
5102 if (!decode_esz_idx(a->imm, &esz, &idx)) { in trans_DUP_general()
5105 if (esz == MO_64 && !a->q) { in trans_DUP_general()
5109 tcg_gen_gvec_dup_i64(esz, vec_full_reg_offset(s, a->rd), in trans_DUP_general()
5110 a->q ? 16 : 8, vec_full_reg_size(s), in trans_DUP_general()
5111 cpu_reg(s, a->rn)); in trans_DUP_general()
5121 if (!decode_esz_idx(a->imm, &esz, &idx)) { in do_smov_umov()
5125 if (esz == MO_64 || (esz == MO_32 && !a->q)) { in do_smov_umov()
5129 if (esz == MO_64 ? !a->q : a->q) { in do_smov_umov()
5134 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); in do_smov_umov()
5135 read_vec_element(s, tcg_rd, a->rn, idx, esz | is_signed); in do_smov_umov()
5136 if (is_signed && !a->q) { in do_smov_umov()
5151 if (!decode_esz_idx(a->imm, &esz, &idx)) { in TRANS()
5155 write_vec_element(s, cpu_reg(s, a->rn), a->rd, idx, esz); in TRANS()
5156 clear_vec_high(s, true, a->rd); in TRANS()
5166 if (!decode_esz_idx(a->di, &esz, &didx)) { in trans_INS_element()
5169 sidx = a->si >> esz; in trans_INS_element()
5173 read_vec_element(s, tmp, a->rn, sidx, esz); in trans_INS_element()
5174 write_vec_element(s, tmp, a->rd, didx, esz); in trans_INS_element()
5176 /* INS is considered a 128-bit write for SVE. */ in trans_INS_element()
5177 clear_vec_high(s, true, a->rd); in trans_INS_element()
5196 switch (a->esz) { in do_fp3_scalar_with_fpsttype()
5199 TCGv_i64 t0 = read_fp_dreg(s, a->rn); in do_fp3_scalar_with_fpsttype()
5200 TCGv_i64 t1 = read_fp_dreg(s, a->rm); in do_fp3_scalar_with_fpsttype()
5201 f->gen_d(t0, t0, t1, fpstatus_ptr(fpsttype)); in do_fp3_scalar_with_fpsttype()
5202 write_fp_dreg_merging(s, a->rd, mergereg, t0); in do_fp3_scalar_with_fpsttype()
5207 TCGv_i32 t0 = read_fp_sreg(s, a->rn); in do_fp3_scalar_with_fpsttype()
5208 TCGv_i32 t1 = read_fp_sreg(s, a->rm); in do_fp3_scalar_with_fpsttype()
5209 f->gen_s(t0, t0, t1, fpstatus_ptr(fpsttype)); in do_fp3_scalar_with_fpsttype()
5210 write_fp_sreg_merging(s, a->rd, mergereg, t0); in do_fp3_scalar_with_fpsttype()
5218 TCGv_i32 t0 = read_fp_hreg(s, a->rn); in do_fp3_scalar_with_fpsttype()
5219 TCGv_i32 t1 = read_fp_hreg(s, a->rm); in do_fp3_scalar_with_fpsttype()
5220 f->gen_h(t0, t0, t1, fpstatus_ptr(fpsttype)); in do_fp3_scalar_with_fpsttype()
5221 write_fp_hreg_merging(s, a->rd, mergereg, t0); in do_fp3_scalar_with_fpsttype()
5234 a->esz == MO_16 ? in do_fp3_scalar()
5242 return do_fp3_scalar_with_fpsttype(s, a, s->fpcr_ah ? fah : fnormal, in do_fp3_scalar_ah_2fn()
5243 mergereg, select_ah_fpst(s, a->esz)); in do_fp3_scalar_ah_2fn()
5252 return do_fp3_scalar(s, a, s->fpcr_ah ? fah : fnormal, mergereg); in do_fp3_scalar_2fn()
5260 TRANS(FADD_s, do_fp3_scalar, a, &f_scalar_fadd, a->rn)
5267 TRANS(FSUB_s, do_fp3_scalar, a, &f_scalar_fsub, a->rn)
5274 TRANS(FDIV_s, do_fp3_scalar, a, &f_scalar_fdiv, a->rn)
5281 TRANS(FMUL_s, do_fp3_scalar, a, &f_scalar_fmul, a->rn)
5293 TRANS(FMAX_s, do_fp3_scalar_2fn, a, &f_scalar_fmax, &f_scalar_fmax_ah, a->rn)
5305 TRANS(FMIN_s, do_fp3_scalar_2fn, a, &f_scalar_fmin, &f_scalar_fmin_ah, a->rn)
5312 TRANS(FMAXNM_s, do_fp3_scalar, a, &f_scalar_fmaxnm, a->rn)
5319 TRANS(FMINNM_s, do_fp3_scalar, a, &f_scalar_fminnm, a->rn)
5326 TRANS(FMULX_s, do_fp3_scalar, a, &f_scalar_fmulx, a->rn)
5374 TRANS(FNMUL_s, do_fp3_scalar_2fn, a, &f_scalar_fnmul, &f_scalar_ah_fnmul, a->rn)
5381 TRANS(FCMEQ_s, do_fp3_scalar, a, &f_scalar_fcmeq, a->rm)
5388 TRANS(FCMGE_s, do_fp3_scalar, a, &f_scalar_fcmge, a->rm)
5395 TRANS(FCMGT_s, do_fp3_scalar, a, &f_scalar_fcmgt, a->rm)
5402 TRANS(FACGE_s, do_fp3_scalar, a, &f_scalar_facge, a->rm)
5409 TRANS(FACGT_s, do_fp3_scalar, a, &f_scalar_facgt, a->rm)
5457 TRANS(FABD_s, do_fp3_scalar_2fn, a, &f_scalar_fabd, &f_scalar_ah_fabd, a->rn)
5470 &f_scalar_frecps, &f_scalar_ah_frecps, a->rn)
5483 &f_scalar_frsqrts, &f_scalar_ah_frsqrts, a->rn)
5488 switch (a->esz) { in do_fcmp0_s()
5491 TCGv_i64 t0 = read_fp_dreg(s, a->rn); in do_fcmp0_s()
5494 f->gen_d(t0, t1, t0, fpstatus_ptr(FPST_A64)); in do_fcmp0_s()
5496 f->gen_d(t0, t0, t1, fpstatus_ptr(FPST_A64)); in do_fcmp0_s()
5498 write_fp_dreg(s, a->rd, t0); in do_fcmp0_s()
5503 TCGv_i32 t0 = read_fp_sreg(s, a->rn); in do_fcmp0_s()
5506 f->gen_s(t0, t1, t0, fpstatus_ptr(FPST_A64)); in do_fcmp0_s()
5508 f->gen_s(t0, t0, t1, fpstatus_ptr(FPST_A64)); in do_fcmp0_s()
5510 write_fp_sreg(s, a->rd, t0); in do_fcmp0_s()
5518 TCGv_i32 t0 = read_fp_hreg(s, a->rn); in do_fcmp0_s()
5521 f->gen_h(t0, t1, t0, fpstatus_ptr(FPST_A64_F16)); in do_fcmp0_s()
5523 f->gen_h(t0, t0, t1, fpstatus_ptr(FPST_A64_F16)); in do_fcmp0_s()
5525 write_fp_sreg(s, a->rd, t0); in do_fcmp0_s()
5546 MemOp esz = a->esz; in do_satacc_s()
5556 read_vec_element(s, t1, a->rn, 0, esz | sgn_n); in do_satacc_s()
5557 read_vec_element(s, t2, a->rm, 0, esz | sgn_m); in do_satacc_s()
5567 write_fp_dreg(s, a->rd, t0); in do_satacc_s()
5586 read_vec_element(s, t0, a->rn, 0, MO_64); in TRANS()
5587 read_vec_element(s, t1, a->rm, 0, MO_64); in TRANS()
5589 write_fp_dreg(s, a->rd, t0); in TRANS()
5611 if (a->esz == MO_64) { in do_env_scalar2()
5612 TCGv_i64 t0 = read_fp_dreg(s, a->rn); in do_env_scalar2()
5613 TCGv_i64 t1 = read_fp_dreg(s, a->rm); in do_env_scalar2()
5614 f->gen_d(t0, tcg_env, t0, t1); in do_env_scalar2()
5615 write_fp_dreg(s, a->rd, t0); in do_env_scalar2()
5620 read_vec_element_i32(s, t0, a->rn, 0, a->esz); in do_env_scalar2()
5621 read_vec_element_i32(s, t1, a->rm, 0, a->esz); in do_env_scalar2()
5622 f->gen_bhs[a->esz](t0, tcg_env, t0, t1); in do_env_scalar2()
5623 write_fp_sreg(s, a->rd, t0); in do_env_scalar2()
5663 if (a->esz == MO_16 || a->esz == MO_32) { in do_env_scalar2_hs()
5688 if (a->esz != MO_16 && a->esz != MO_32) { in do_env_scalar3_hs()
5698 read_vec_element_i32(s, t0, a->rn, 0, a->esz); in do_env_scalar3_hs()
5699 read_vec_element_i32(s, t1, a->rm, 0, a->esz); in do_env_scalar3_hs()
5700 read_vec_element_i32(s, t2, a->rd, 0, a->esz); in do_env_scalar3_hs()
5701 f->gen_hs[a->esz - 1](t0, tcg_env, t0, t1, t2); in do_env_scalar3_hs()
5702 write_fp_sreg(s, a->rd, t0); in do_env_scalar3_hs()
5719 TCGv_i64 t0 = read_fp_dreg(s, a->rn); in do_cmop_d()
5720 TCGv_i64 t1 = read_fp_dreg(s, a->rm); in do_cmop_d()
5722 write_fp_dreg(s, a->rd, t0); in do_cmop_d()
5739 MemOp esz = a->esz; in TRANS()
5740 int check = fp_access_check_vector_hsd(s, a->q, esz); in TRANS()
5746 gen_gvec_op3_fpst(s, a->q, a->rd, a->rn, a->rm, fpsttype, in TRANS()
5747 data, fns[esz - 1]); in TRANS()
5755 a->esz == MO_16 ? in do_fp3_vector()
5763 return do_fp3_vector(s, a, data, s->fpcr_ah ? fah : fnormal); in do_fp3_vector_2fn()
5770 return do_fp3_vector_with_fpsttype(s, a, data, s->fpcr_ah ? fah : fnormal, in do_fp3_vector_ah_2fn()
5771 select_ah_fpst(s, a->esz)); in do_fp3_vector_ah_2fn()
5986 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, a->rd), in do_fmlal()
5987 vec_full_reg_offset(s, a->rn), in do_fmlal()
5988 vec_full_reg_offset(s, a->rm), tcg_env, in do_fmlal()
5989 a->q ? 16 : 8, vec_full_reg_size(s), in do_fmlal()
6020 TRANS(BSL_v, do_bitsel, a->q, a->rd, a->rd, a->rn, a->rm)
6021 TRANS(BIT_v, do_bitsel, a->q, a->rd, a->rm, a->rn, a->rd)
6022 TRANS(BIF_v, do_bitsel, a->q, a->rd, a->rm, a->rd, a->rn)
6063 if (a->esz == MO_64 && !a->q) { in TRANS()
6067 tcg_gen_gvec_cmp(cond, a->esz, in TRANS()
6068 vec_full_reg_offset(s, a->rd), in TRANS()
6069 vec_full_reg_offset(s, a->rn), in TRANS()
6070 vec_full_reg_offset(s, a->rm), in TRANS()
6071 a->q ? 16 : 8, vec_full_reg_size(s)); in TRANS()
6092 gen_gvec_op4_ool(s, a->q, a->rd, a->rn, a->rm, a->rd, 0, fn); in TRANS()
6101 gen_gvec_op4_env(s, a->q, a->rd, a->rn, a->rm, a->rd, 0, fn); in do_dot_vector_env()
6122 gen_gvec_op4_fpst(s, true, a->rd, a->rn, a->rm, a->rd, in TRANS_FEAT()
6123 s->fpcr_ah ? FPST_AH : FPST_A64, a->q, in TRANS_FEAT()
6138 TRANS_FEAT(FCADD_90, aa64_fcma, do_fp3_vector, a, 0 | (s->fpcr_ah << 1),
6140 TRANS_FEAT(FCADD_270, aa64_fcma, do_fp3_vector, a, 1 | (s->fpcr_ah << 1),
6156 check = fp_access_check_vector_hsd(s, a->q, a->esz); in trans_FCMLA_v()
6161 gen_gvec_op4_fpst(s, a->q, a->rd, a->rn, a->rm, a->rd, in trans_FCMLA_v()
6162 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64, in trans_FCMLA_v()
6163 a->rot | (s->fpcr_ah << 2), fn[a->esz]); in trans_FCMLA_v()
6170 * These read from the top or bottom half of a 128-bit vector.
6171 * After widening, optionally accumulate with a 128-bit vector.
6192 /* There are no 64x64->128 bit operations. */ in do_3op_widening()
6210 top_swap = top ? 0 : half - 1; in do_3op_widening()
6245 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6248 a->esz, a->q, a->rd, a->rn, a->rm, -1,
6251 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6254 a->esz, a->q, a->rd, a->rn, a->rm, -1,
6257 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6260 a->esz, a->q, a->rd, a->rn, a->rm, -1,
6264 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, a->idx,
6267 a->esz, a->q, a->rd, a->rn, a->rm, a->idx,
6270 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, a->idx,
6273 a->esz, a->q, a->rd, a->rn, a->rm, a->idx,
6276 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, a->idx,
6279 a->esz, a->q, a->rd, a->rn, a->rm, a->idx,
6317 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6320 a->esz, a->q, a->rd, a->rn, a->rm, -1,
6323 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6326 a->esz, a->q, a->rd, a->rn, a->rm, -1,
6329 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6332 a->esz, a->q, a->rd, a->rn, a->rm, -1,
6335 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6338 a->esz, a->q, a->rd, a->rn, a->rm, -1,
6392 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6393 a->esz == MO_16 ? gen_sqdmull_h : gen_sqdmull_s, false)
6395 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6396 a->esz == MO_16 ? gen_sqdmlal_h : gen_sqdmlal_s, true)
6398 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6399 a->esz == MO_16 ? gen_sqdmlsl_h : gen_sqdmlsl_s, true)
6402 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, a->idx,
6403 a->esz == MO_16 ? gen_sqdmull_h : gen_sqdmull_s, false)
6405 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, a->idx,
6406 a->esz == MO_16 ? gen_sqdmlal_h : gen_sqdmlal_s, true)
6408 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, a->idx,
6409 a->esz == MO_16 ? gen_sqdmlsl_h : gen_sqdmlsl_s, true)
6415 MemOp esz = a->esz; in do_addsub_wide()
6417 bool top = a->q; in do_addsub_wide()
6418 int top_swap = top ? 0 : half - 1; in do_addsub_wide()
6421 /* There are no 64x64->128 bit operations. */ in do_addsub_wide()
6434 read_vec_element(s, tcg_op1, a->rm, elt + top_half, esz | sign); in do_addsub_wide()
6435 read_vec_element(s, tcg_op0, a->rn, elt, esz + 1); in do_addsub_wide()
6441 write_vec_element(s, tcg_op0, a->rd, elt, esz + 1); in do_addsub_wide()
6443 clear_vec_high(s, 1, a->rd); in do_addsub_wide()
6456 MemOp esz = a->esz; in TRANS()
6458 bool top = a->q; in TRANS()
6460 uint64_t rbit = 1ull << (ebits - 1); in TRANS()
6463 /* There are no 128x128->64 bit operations. */ in TRANS()
6478 top_swap = top ? half - 1 : 0; in TRANS()
6484 read_vec_element(s, tcg_op1, a->rm, elt, esz + 1); in TRANS()
6485 read_vec_element(s, tcg_op0, a->rn, elt, esz + 1); in TRANS()
6495 write_vec_element(s, tcg_op0, a->rd, elt + top_half, esz); in TRANS()
6497 clear_vec_high(s, top, a->rd); in TRANS()
6510 gen_gvec_op3_ool(s, true, a->rd, a->rn, a->rm, a->q, fn); in TRANS()
6524 switch (a->esz) { in TRANS()
6527 TCGv_i64 t0 = read_fp_dreg(s, a->rn); in TRANS()
6530 read_vec_element(s, t1, a->rm, a->idx, MO_64); in TRANS()
6531 f->gen_d(t0, t0, t1, fpstatus_ptr(FPST_A64)); in TRANS()
6532 write_fp_dreg_merging(s, a->rd, a->rn, t0); in TRANS()
6537 TCGv_i32 t0 = read_fp_sreg(s, a->rn); in TRANS()
6540 read_vec_element_i32(s, t1, a->rm, a->idx, MO_32); in TRANS()
6541 f->gen_s(t0, t0, t1, fpstatus_ptr(FPST_A64)); in TRANS()
6542 write_fp_sreg_merging(s, a->rd, a->rn, t0); in TRANS()
6550 TCGv_i32 t0 = read_fp_hreg(s, a->rn); in TRANS()
6553 read_vec_element_i32(s, t1, a->rm, a->idx, MO_16); in TRANS()
6554 f->gen_h(t0, t0, t1, fpstatus_ptr(FPST_A64_F16)); in TRANS()
6555 write_fp_hreg_merging(s, a->rd, a->rn, t0); in TRANS()
6569 switch (a->esz) { in do_fmla_scalar_idx()
6572 TCGv_i64 t0 = read_fp_dreg(s, a->rd); in do_fmla_scalar_idx()
6573 TCGv_i64 t1 = read_fp_dreg(s, a->rn); in do_fmla_scalar_idx()
6576 read_vec_element(s, t2, a->rm, a->idx, MO_64); in do_fmla_scalar_idx()
6581 write_fp_dreg_merging(s, a->rd, a->rd, t0); in do_fmla_scalar_idx()
6586 TCGv_i32 t0 = read_fp_sreg(s, a->rd); in do_fmla_scalar_idx()
6587 TCGv_i32 t1 = read_fp_sreg(s, a->rn); in do_fmla_scalar_idx()
6590 read_vec_element_i32(s, t2, a->rm, a->idx, MO_32); in do_fmla_scalar_idx()
6595 write_fp_sreg_merging(s, a->rd, a->rd, t0); in do_fmla_scalar_idx()
6603 TCGv_i32 t0 = read_fp_hreg(s, a->rd); in do_fmla_scalar_idx()
6604 TCGv_i32 t1 = read_fp_hreg(s, a->rn); in do_fmla_scalar_idx()
6607 read_vec_element_i32(s, t2, a->rm, a->idx, MO_16); in do_fmla_scalar_idx()
6613 write_fp_hreg_merging(s, a->rd, a->rd, t0); in do_fmla_scalar_idx()
6628 if (a->esz < MO_16 || a->esz > MO_32) { in TRANS()
6635 read_vec_element_i32(s, t0, a->rn, 0, a->esz); in TRANS()
6636 read_vec_element_i32(s, t1, a->rm, a->idx, a->esz); in TRANS()
6637 f->gen_bhs[a->esz](t0, tcg_env, t0, t1); in TRANS()
6638 write_fp_sreg(s, a->rd, t0); in TRANS()
6649 if (a->esz < MO_16 || a->esz > MO_32) { in do_env_scalar3_idx_hs()
6657 read_vec_element_i32(s, t0, a->rn, 0, a->esz); in do_env_scalar3_idx_hs()
6658 read_vec_element_i32(s, t1, a->rm, a->idx, a->esz); in do_env_scalar3_idx_hs()
6659 read_vec_element_i32(s, t2, a->rd, 0, a->esz); in do_env_scalar3_idx_hs()
6660 f->gen_hs[a->esz - 1](t0, tcg_env, t0, t1, t2); in do_env_scalar3_idx_hs()
6661 write_fp_sreg(s, a->rd, t0); in do_env_scalar3_idx_hs()
6678 read_vec_element(s, t0, a->rd, 0, a->esz + 1); in do_scalar_muladd_widening_idx()
6680 read_vec_element(s, t1, a->rn, 0, a->esz | MO_SIGN); in do_scalar_muladd_widening_idx()
6681 read_vec_element(s, t2, a->rm, a->idx, a->esz | MO_SIGN); in do_scalar_muladd_widening_idx()
6685 clear_vec(s, a->rd); in do_scalar_muladd_widening_idx()
6686 write_vec_element(s, t0, a->rd, 0, a->esz + 1); in do_scalar_muladd_widening_idx()
6692 a->esz == MO_16 ? gen_sqdmull_h : gen_sqdmull_s, false)
6694 a->esz == MO_16 ? gen_sqdmlal_h : gen_sqdmlal_s, true)
6696 a->esz == MO_16 ? gen_sqdmlsl_h : gen_sqdmlsl_s, true)
6701 MemOp esz = a->esz; in do_fp3_vector_idx()
6702 int check = fp_access_check_vector_hsd(s, a->q, esz); in do_fp3_vector_idx()
6708 gen_gvec_op3_fpst(s, a->q, a->rd, a->rn, a->rm, in do_fp3_vector_idx()
6710 a->idx, fns[esz - 1]); in do_fp3_vector_idx()
6741 MemOp esz = a->esz; in TRANS()
6742 int check = fp_access_check_vector_hsd(s, a->q, esz); in TRANS()
6748 gen_gvec_op4_fpst(s, a->q, a->rd, a->rn, a->rm, a->rd, in TRANS()
6750 a->idx, fns[neg ? 1 + s->fpcr_ah : 0][esz - 1]); in TRANS()
6760 int data = (a->idx << 2) | (is_2 << 1) | is_s; in TRANS()
6761 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, a->rd), in TRANS()
6762 vec_full_reg_offset(s, a->rn), in TRANS()
6763 vec_full_reg_offset(s, a->rm), tcg_env, in TRANS()
6764 a->q ? 16 : 8, vec_full_reg_size(s), in TRANS()
6778 assert(a->esz == MO_16 || a->esz == MO_32); in TRANS_FEAT()
6780 gen_gvec_op3_ool(s, a->q, a->rd, a->rn, a->rm, a->idx, fns[a->esz - 1]); in TRANS_FEAT()
6798 assert(a->esz == MO_16 || a->esz == MO_32); in TRANS()
6800 gen_gvec_op4_ool(s, a->q, a->rd, a->rn, a->rm, a->rd, in TRANS()
6801 a->idx, fns[a->esz - 1][sub]); in TRANS()
6812 assert(a->esz == MO_16 || a->esz == MO_32); in TRANS()
6814 tcg_gen_gvec_4_ool(vec_full_reg_offset(s, a->rd), in TRANS()
6815 vec_full_reg_offset(s, a->rn), in TRANS()
6816 vec_full_reg_offset(s, a->rm), in TRANS()
6818 a->q ? 16 : 8, vec_full_reg_size(s), in TRANS()
6819 a->idx, fns[a->esz - 1]); in TRANS()
6854 gen_gvec_op4_ool(s, a->q, a->rd, a->rn, a->rm, a->rd, a->idx, fn); in TRANS_FEAT()
6863 gen_gvec_op4_env(s, a->q, a->rd, a->rn, a->rm, a->rd, a->idx, fn); in do_dot_vector_idx_env()
6884 gen_gvec_op4_fpst(s, true, a->rd, a->rn, a->rm, a->rd, in TRANS_FEAT()
6885 s->fpcr_ah ? FPST_AH : FPST_A64, in TRANS_FEAT()
6886 (a->idx << 1) | a->q, in TRANS_FEAT()
6899 switch (a->esz) { in trans_FCMLA_vi()
6913 gen_gvec_op4_fpst(s, a->q, a->rd, a->rn, a->rm, a->rd, in trans_FCMLA_vi()
6914 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64, in trans_FCMLA_vi()
6915 (s->fpcr_ah << 4) | (a->idx << 2) | a->rot, fn); in trans_FCMLA_vi()
6926 switch (a->esz) { in do_fp3_scalar_pair()
6932 read_vec_element(s, t0, a->rn, 0, MO_64); in do_fp3_scalar_pair()
6933 read_vec_element(s, t1, a->rn, 1, MO_64); in do_fp3_scalar_pair()
6934 f->gen_d(t0, t0, t1, fpstatus_ptr(FPST_A64)); in do_fp3_scalar_pair()
6935 write_fp_dreg(s, a->rd, t0); in do_fp3_scalar_pair()
6943 read_vec_element_i32(s, t0, a->rn, 0, MO_32); in do_fp3_scalar_pair()
6944 read_vec_element_i32(s, t1, a->rn, 1, MO_32); in do_fp3_scalar_pair()
6945 f->gen_s(t0, t0, t1, fpstatus_ptr(FPST_A64)); in do_fp3_scalar_pair()
6946 write_fp_sreg(s, a->rd, t0); in do_fp3_scalar_pair()
6957 read_vec_element_i32(s, t0, a->rn, 0, MO_16); in do_fp3_scalar_pair()
6958 read_vec_element_i32(s, t1, a->rn, 1, MO_16); in do_fp3_scalar_pair()
6959 f->gen_h(t0, t0, t1, fpstatus_ptr(FPST_A64_F16)); in do_fp3_scalar_pair()
6960 write_fp_sreg(s, a->rd, t0); in do_fp3_scalar_pair()
6973 return do_fp3_scalar_pair(s, a, s->fpcr_ah ? fah : fnormal); in do_fp3_scalar_pair_2fn()
6988 read_vec_element(s, t0, a->rn, 0, MO_64); in trans_ADDP_s()
6989 read_vec_element(s, t1, a->rn, 1, MO_64); in trans_ADDP_s()
6991 write_fp_dreg(s, a->rd, t0); in trans_ADDP_s()
6997 * Floating-point conditional select
7004 int check = fp_access_check_scalar_hsd(s, a->esz); in trans_FCSEL()
7013 read_vec_element(s, t_true, a->rn, 0, a->esz); in trans_FCSEL()
7014 read_vec_element(s, t_false, a->rm, 0, a->esz); in trans_FCSEL()
7016 a64_test_cc(&c, a->cond); in trans_FCSEL()
7022 * and we've already done the zero-extension. in trans_FCSEL()
7024 write_fp_dreg(s, a->rd, t_true); in trans_FCSEL()
7035 TCGv_i64 lo = read_fp_dreg(s, a->rn); in trans_EXT_d()
7036 if (a->imm != 0) { in trans_EXT_d()
7037 TCGv_i64 hi = read_fp_dreg(s, a->rm); in trans_EXT_d()
7038 tcg_gen_extract2_i64(lo, lo, hi, a->imm * 8); in trans_EXT_d()
7040 write_fp_dreg(s, a->rd, lo); in trans_EXT_d()
7048 int pos = (a->imm & 7) * 8; in trans_EXT_q()
7049 int elt = a->imm >> 3; in trans_EXT_q()
7058 read_vec_element(s, lo, a->rn, elt, MO_64); in trans_EXT_q()
7060 read_vec_element(s, hi, elt & 2 ? a->rm : a->rn, elt & 1, MO_64); in trans_EXT_q()
7066 read_vec_element(s, hh, a->rm, elt & 1, MO_64); in trans_EXT_q()
7070 write_vec_element(s, lo, a->rd, 0, MO_64); in trans_EXT_q()
7071 write_vec_element(s, hi, a->rd, 1, MO_64); in trans_EXT_q()
7072 clear_vec_high(s, true, a->rd); in trans_EXT_q()
7077 * Floating-point data-processing (3 source)
7085 * These are fused multiply-add. Note that doing the negations here in do_fmadd()
7087 * its sign bit flipped if it is a negated-input. in do_fmadd()
7089 switch (a->esz) { in do_fmadd()
7092 TCGv_i64 tn = read_fp_dreg(s, a->rn); in do_fmadd()
7093 TCGv_i64 tm = read_fp_dreg(s, a->rm); in do_fmadd()
7094 TCGv_i64 ta = read_fp_dreg(s, a->ra); in do_fmadd()
7104 write_fp_dreg_merging(s, a->rd, a->ra, ta); in do_fmadd()
7110 TCGv_i32 tn = read_fp_sreg(s, a->rn); in do_fmadd()
7111 TCGv_i32 tm = read_fp_sreg(s, a->rm); in do_fmadd()
7112 TCGv_i32 ta = read_fp_sreg(s, a->ra); in do_fmadd()
7122 write_fp_sreg_merging(s, a->rd, a->ra, ta); in do_fmadd()
7131 TCGv_i32 tn = read_fp_hreg(s, a->rn); in do_fmadd()
7132 TCGv_i32 tm = read_fp_hreg(s, a->rm); in do_fmadd()
7133 TCGv_i32 ta = read_fp_hreg(s, a->ra); in do_fmadd()
7143 write_fp_hreg_merging(s, a->rd, a->ra, ta); in do_fmadd()
7166 MemOp src_mop = a->esz | src_sign; in TRANS()
7167 int elements = (a->q ? 16 : 8) >> a->esz; in TRANS()
7180 read_vec_element(s, tcg_res, a->rn, 0, src_mop); in TRANS()
7182 read_vec_element(s, tcg_elt, a->rn, i, src_mop); in TRANS()
7186 tcg_gen_ext_i64(tcg_res, tcg_res, a->esz + widen); in TRANS()
7187 write_fp_dreg(s, a->rd, tcg_res); in TRANS()
7234 MemOp esz = a->esz; in do_fp_reduction()
7235 int elts = (a->q ? 16 : 8) >> esz; in do_fp_reduction()
7237 TCGv_i32 res = do_reduction_op(s, a->rn, esz, 0, elts, fpst, in do_fp_reduction()
7238 s->fpcr_ah ? fah : fnormal); in do_fp_reduction()
7239 write_fp_sreg(s, a->rd, res); in do_fp_reduction()
7261 * Floating-point Immediate in TRANS_FEAT()
7266 int check = fp_access_check_scalar_hsd(s, a->esz); in TRANS_FEAT()
7273 imm = vfp_expand_imm(a->esz, a->imm); in TRANS_FEAT()
7274 write_fp_dreg(s, a->rd, tcg_constant_i64(imm)); in TRANS_FEAT()
7340 int check = fp_access_check_scalar_hsd(s, a->esz); in trans_FCMP()
7346 handle_fp_compare(s, a->esz, a->rn, a->rm, a->z, a->e); in trans_FCMP()
7354 int check = fp_access_check_scalar_hsd(s, a->esz); in trans_FCCMP()
7360 if (a->cond < 0x0e) { /* not always */ in trans_FCCMP()
7363 arm_gen_test_cc(a->cond, label_match); in trans_FCCMP()
7365 gen_set_nzcv(tcg_constant_i64(a->nzcv << 28)); in trans_FCCMP()
7370 handle_fp_compare(s, a->esz, a->rn, a->rm, false, a->e); in trans_FCCMP()
7388 tcg_gen_gvec_dup_imm(MO_16, vec_full_reg_offset(s, a->rd), in trans_FMOVI_v_h()
7389 a->q ? 16 : 8, vec_full_reg_size(s), in trans_FMOVI_v_h()
7390 vfp_expand_imm(MO_16, a->abcdefgh)); in trans_FMOVI_v_h()
7406 if ((a->cmode & 1) && a->cmode < 12) { in trans_Vimm()
7408 fn = a->op ? tcg_gen_gvec_andi : tcg_gen_gvec_ori; in trans_Vimm()
7411 if (a->cmode == 15 && a->op == 1 && a->q == 0) { in trans_Vimm()
7418 uint64_t imm = asimd_imm_const(a->abcdefgh, a->cmode, a->op); in trans_Vimm()
7419 gen_gvec_fn2i(s, a->q, a->rd, a->rd, imm, fn, MO_64); in trans_Vimm()
7431 gen_gvec_fn2i(s, a->q, a->rd, a->rn, a->imm, fn, a->esz); in do_vec_shift_imm()
7454 int esz = a->esz; in TRANS()
7468 read_vec_element(s, tcg_rn, a->rn, a->q, MO_64); in TRANS()
7477 tcg_gen_shli_i64(tcg_rd, tcg_rd, a->imm); in TRANS()
7478 write_vec_element(s, tcg_rd, a->rd, i, esz + 1); in TRANS()
7480 clear_vec_high(s, true, a->rd); in TRANS()
7519 TCGv_i64 rnd = tcg_constant_i64(1ull << (shift - 1)); in gen_srshr_bhs()
7531 TCGv_i64 rnd = tcg_constant_i64(1ull << (shift - 1)); in gen_urshr_bhs()
7545 /* Extension of sign bit (0,-1) plus sign bit (0,1) is zero. */ in gen_srshr_d()
7549 tcg_gen_extract_i64(rnd, src, shift - 1, 1); in gen_srshr_d()
7565 tcg_gen_extract_i64(rnd, src, shift - 1, 1); in gen_urshr_d()
7588 tcg_gen_deposit_i64(dst, dst, src, 0, 64 - shift); in gen_sri_d()
7594 tcg_gen_deposit_i64(dst, dst, src, shift, 64 - shift); in gen_sli_d()
7601 int esz = a->esz; in do_vec_shift_imm_narrow()
7618 read_vec_element(s, tcg_rn, a->rn, i, (esz + 1) | sign); in do_vec_shift_imm_narrow()
7619 fn(tcg_rn, tcg_rn, a->imm); in do_vec_shift_imm_narrow()
7623 write_vec_element(s, tcg_rd, a->rd, a->q, MO_64); in do_vec_shift_imm_narrow()
7624 clear_vec_high(s, a->q, a->rd); in do_vec_shift_imm_narrow()
7812 read_vec_element(s, rn, a->rn, 0, a->esz | sign); in TRANS()
7814 read_vec_element(s, rd, a->rd, 0, a->esz | sign); in TRANS()
7816 fn(rd, rn, a->imm); in TRANS()
7817 write_fp_dreg(s, a->rd, rd); in TRANS()
7916 /* Note that the helpers sign-extend their inputs, so don't do it here. */
7917 TRANS(SQSHL_si, do_scalar_shift_imm, a, f_scalar_sqshli[a->esz], false, 0)
7918 TRANS(UQSHL_si, do_scalar_shift_imm, a, f_scalar_uqshli[a->esz], false, 0)
7919 TRANS(SQSHLU_si, do_scalar_shift_imm, a, f_scalar_sqshlui[a->esz], false, 0)
7925 MemOp esz = a->esz; in do_scalar_shift_imm_narrow()
7933 read_vec_element(s, rn, a->rn, 0, (esz + 1) | sign); in do_scalar_shift_imm_narrow()
7934 fns[esz](rd, rn, a->imm); in do_scalar_shift_imm_narrow()
7938 write_fp_dreg(s, a->rd, rd); in do_scalar_shift_imm_narrow()
7953 tcg_rd = cpu_reg(s, a->rd); in TRANS()
7955 if (!a->sf && is_signed) { in TRANS()
7958 tcg_gen_ext32s_i64(tcg_n, cpu_reg(s, a->rn)); in TRANS()
7959 tcg_gen_ext32s_i64(tcg_m, cpu_reg(s, a->rm)); in TRANS()
7961 tcg_n = read_cpu_reg(s, a->rn, a->sf); in TRANS()
7962 tcg_m = read_cpu_reg(s, a->rm, a->sf); in TRANS()
7971 if (!a->sf) { /* zero extend final result */ in TRANS()
8044 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); in do_shift_reg()
8045 TCGv_i64 tcg_rn = read_cpu_reg(s, a->rn, a->sf); in do_shift_reg()
8047 tcg_gen_andi_i64(tcg_shift, cpu_reg(s, a->rm), a->sf ? 63 : 31); in do_shift_reg()
8048 shift_reg(tcg_rd, tcg_rn, a->sf, shift_type, tcg_shift); in do_shift_reg()
8062 switch (a->esz) { in TRANS()
8067 tcg_gen_extract_i64(tcg_val, cpu_reg(s, a->rm), 0, 8 << a->esz); in TRANS()
8070 tcg_val = cpu_reg(s, a->rm); in TRANS()
8075 tcg_acc = cpu_reg(s, a->rn); in TRANS()
8076 tcg_bytes = tcg_constant_i32(1 << a->esz); in TRANS()
8077 tcg_rd = cpu_reg(s, a->rd); in TRANS()
8092 TCGv_i64 tcg_n = read_cpu_reg_sp(s, a->rn, true); in TRANS_FEAT()
8093 TCGv_i64 tcg_m = read_cpu_reg_sp(s, a->rm, true); in TRANS_FEAT()
8094 TCGv_i64 tcg_d = cpu_reg(s, a->rd); in TRANS_FEAT()
8113 TCGv_i64 tcg_rd = cpu_reg_sp(s, a->rd); in TRANS_FEAT()
8114 TCGv_i64 tcg_rn = cpu_reg_sp(s, a->rn); in TRANS_FEAT()
8116 if (s->ata[0]) { in TRANS_FEAT()
8117 gen_helper_irg(tcg_rd, tcg_env, tcg_rn, cpu_reg(s, a->rm)); in TRANS_FEAT()
8131 tcg_gen_extract_i64(t, cpu_reg_sp(s, a->rn), 56, 4); in trans_GMI()
8133 tcg_gen_or_i64(cpu_reg(s, a->rd), cpu_reg(s, a->rm), t); in trans_GMI()
8142 gen_helper_pacga(cpu_reg(s, a->rd), tcg_env, in trans_PACGA()
8143 cpu_reg(s, a->rn), cpu_reg_sp(s, a->rm)); in trans_PACGA()
8198 TRANS(RBIT, gen_rr, a->rd, a->rn, a->sf ? gen_helper_rbit64 : gen_rbit32)
8199 TRANS(REV16, gen_rr, a->rd, a->rn, a->sf ? gen_rev16_64 : gen_rev16_32)
8200 TRANS(REV32, gen_rr, a->rd, a->rn, a->sf ? gen_rev32 : gen_rev_32)
8201 TRANS(REV64, gen_rr, a->rd, a->rn, tcg_gen_bswap64_i64)
8226 TRANS(CLZ, gen_rr, a->rd, a->rn, a->sf ? gen_clz64 : gen_clz32)
8227 TRANS(CLS, gen_rr, a->rd, a->rn, a->sf ? tcg_gen_clrsb_i64 : gen_cls32)
8233 if (a->z) { in gen_pacaut()
8234 if (a->rn != 31) { in gen_pacaut()
8239 tcg_rn = cpu_reg_sp(s, a->rn); in gen_pacaut()
8241 if (s->pauth_active) { in gen_pacaut()
8242 tcg_rd = cpu_reg(s, a->rd); in gen_pacaut()
8260 if (s->pauth_active) { in TRANS_FEAT()
8267 TRANS_FEAT(XPACI, aa64_pauth, do_xpac, a->rd, gen_helper_xpaci)
8268 TRANS_FEAT(XPACD, aa64_pauth, do_xpac, a->rd, gen_helper_xpacd)
8275 if (!a->sf && (a->sa & (1 << 5))) { in do_logic_reg()
8279 tcg_rd = cpu_reg(s, a->rd); in do_logic_reg()
8280 tcg_rn = cpu_reg(s, a->rn); in do_logic_reg()
8282 tcg_rm = read_cpu_reg(s, a->rm, a->sf); in do_logic_reg()
8283 if (a->sa) { in do_logic_reg()
8284 shift_reg_imm(tcg_rm, tcg_rm, a->sf, a->st, a->sa); in do_logic_reg()
8287 (a->n ? inv_fn : fn)(tcg_rd, tcg_rn, tcg_rm); in do_logic_reg()
8288 if (!a->sf) { in do_logic_reg()
8292 gen_logic_CC(a->sf, tcg_rd); in do_logic_reg()
8301 * register-register MOV and MVN, so it is worth special casing. in trans_ORR_r()
8303 if (a->sa == 0 && a->st == 0 && a->rn == 31) { in trans_ORR_r()
8304 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); in trans_ORR_r()
8305 TCGv_i64 tcg_rm = cpu_reg(s, a->rm); in trans_ORR_r()
8307 if (a->n) { in trans_ORR_r()
8309 if (!a->sf) { in trans_ORR_r()
8313 if (a->sf) { in trans_ORR_r()
8334 if (a->sa > 4) { in TRANS()
8338 /* non-flag setting ops may use SP */ in TRANS()
8340 tcg_rd = cpu_reg_sp(s, a->rd); in TRANS()
8342 tcg_rd = cpu_reg(s, a->rd); in TRANS()
8344 tcg_rn = read_cpu_reg_sp(s, a->rn, a->sf); in TRANS()
8346 tcg_rm = read_cpu_reg(s, a->rm, a->sf); in TRANS()
8347 ext_and_shift_reg(tcg_rm, tcg_rm, a->st, a->sa); in TRANS()
8358 gen_sub_CC(a->sf, tcg_result, tcg_rn, tcg_rm); in TRANS()
8360 gen_add_CC(a->sf, tcg_result, tcg_rn, tcg_rm); in TRANS()
8364 if (a->sf) { in TRANS()
8382 if (a->st == 3 || (!a->sf && (a->sa & 32))) { in TRANS()
8386 tcg_rd = cpu_reg(s, a->rd); in TRANS()
8387 tcg_rn = read_cpu_reg(s, a->rn, a->sf); in TRANS()
8388 tcg_rm = read_cpu_reg(s, a->rm, a->sf); in TRANS()
8390 shift_reg_imm(tcg_rm, tcg_rm, a->sf, a->st, a->sa); in TRANS()
8401 gen_sub_CC(a->sf, tcg_result, tcg_rn, tcg_rm); in TRANS()
8403 gen_add_CC(a->sf, tcg_result, tcg_rn, tcg_rm); in TRANS()
8407 if (a->sf) { in TRANS()
8424 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); in TRANS()
8425 TCGv_i64 tcg_rn = cpu_reg(s, a->rn); in TRANS()
8426 TCGv_i64 tcg_rm = cpu_reg(s, a->rm); in TRANS()
8438 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); in TRANS()
8442 tcg_op1 = cpu_reg(s, a->rn); in TRANS()
8443 tcg_op2 = cpu_reg(s, a->rm); in TRANS()
8447 tcg_gen_ext_i64(tcg_op1, cpu_reg(s, a->rn), mop); in TRANS()
8448 tcg_gen_ext_i64(tcg_op2, cpu_reg(s, a->rm), mop); in TRANS()
8451 if (a->ra == 31 && !is_sub) { in TRANS()
8452 /* Special-case MADD with rA == XZR; it is the standard MUL alias */ in TRANS()
8456 TCGv_i64 tcg_ra = cpu_reg(s, a->ra); in TRANS()
8487 tcg_rd = cpu_reg(s, a->rd); in TRANS()
8488 tcg_rn = cpu_reg(s, a->rn); in TRANS()
8492 tcg_gen_not_i64(tcg_y, cpu_reg(s, a->rm)); in TRANS()
8494 tcg_y = cpu_reg(s, a->rm); in TRANS()
8498 gen_adc_CC(a->sf, tcg_rd, tcg_rn, tcg_y); in TRANS()
8500 gen_adc(a->sf, tcg_rd, tcg_rn, tcg_y); in TRANS()
8512 int mask = a->mask; in TRANS()
8520 tcg_rn = read_cpu_reg(s, a->rn, 1); in TRANS()
8521 tcg_gen_rotri_i64(tcg_rn, tcg_rn, a->imm); in TRANS()
8527 tcg_gen_shli_i32(cpu_NF, nzcv, 31 - 3); in TRANS()
8537 tcg_gen_shli_i32(cpu_VF, nzcv, 31 - 0); in TRANS()
8548 tcg_gen_shli_i32(cpu_VF, tmp, shift - 1); in do_setf()
8554 TRANS_FEAT(SETF8, aa64_condm_4, do_setf, a->rn, 24)
8555 TRANS_FEAT(SETF16, aa64_condm_4, do_setf, a->rn, 16)
8570 arm_test_cc(&c, a->cond); in trans_CCMP()
8574 if (a->imm) { in trans_CCMP()
8575 tcg_y = tcg_constant_i64(a->y); in trans_CCMP()
8577 tcg_y = cpu_reg(s, a->y); in trans_CCMP()
8579 tcg_rn = cpu_reg(s, a->rn); in trans_CCMP()
8582 if (a->op) { in trans_CCMP()
8583 gen_sub_CC(a->sf, tcg_tmp, tcg_rn, tcg_y); in trans_CCMP()
8585 gen_add_CC(a->sf, tcg_tmp, tcg_rn, tcg_y); in trans_CCMP()
8590 * to help with this: T1 = (COND ? 0 : -1), T2 = (COND ? -1 : 0). in trans_CCMP()
8597 nzcv = a->nzcv; in trans_CCMP()
8640 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); in trans_CSEL()
8644 a64_test_cc(&c, a->cond); in trans_CSEL()
8646 if (a->rn == 31 && a->rm == 31 && (a->else_inc ^ a->else_inv)) { in trans_CSEL()
8648 if (a->else_inv) { in trans_CSEL()
8656 TCGv_i64 t_true = cpu_reg(s, a->rn); in trans_CSEL()
8657 TCGv_i64 t_false = read_cpu_reg(s, a->rm, 1); in trans_CSEL()
8659 if (a->else_inv && a->else_inc) { in trans_CSEL()
8661 } else if (a->else_inv) { in trans_CSEL()
8663 } else if (a->else_inc) { in trans_CSEL()
8669 if (!a->sf) { in trans_CSEL()
8685 switch (a->esz) { in do_fp1_scalar_int()
8688 TCGv_i64 t = read_fp_dreg(s, a->rn); in do_fp1_scalar_int()
8689 f->gen_d(t, t); in do_fp1_scalar_int()
8691 write_fp_dreg_merging(s, a->rd, a->rd, t); in do_fp1_scalar_int()
8693 write_fp_dreg(s, a->rd, t); in do_fp1_scalar_int()
8699 TCGv_i32 t = read_fp_sreg(s, a->rn); in do_fp1_scalar_int()
8700 f->gen_s(t, t); in do_fp1_scalar_int()
8702 write_fp_sreg_merging(s, a->rd, a->rd, t); in do_fp1_scalar_int()
8704 write_fp_sreg(s, a->rd, t); in do_fp1_scalar_int()
8713 TCGv_i32 t = read_fp_hreg(s, a->rn); in do_fp1_scalar_int()
8714 f->gen_h(t, t); in do_fp1_scalar_int()
8716 write_fp_hreg_merging(s, a->rd, a->rd, t); in do_fp1_scalar_int()
8718 write_fp_sreg(s, a->rd, t); in do_fp1_scalar_int()
8732 return do_fp1_scalar_int(s, a, s->fpcr_ah ? fah : fnormal, true); in do_fp1_scalar_int_2fn()
8780 int check = fp_access_check_scalar_hsd(s, a->esz); in do_fp1_scalar_with_fpsttype()
8791 switch (a->esz) { in do_fp1_scalar_with_fpsttype()
8793 t64 = read_fp_dreg(s, a->rn); in do_fp1_scalar_with_fpsttype()
8794 f->gen_d(t64, t64, fpst); in do_fp1_scalar_with_fpsttype()
8795 write_fp_dreg_merging(s, a->rd, a->rd, t64); in do_fp1_scalar_with_fpsttype()
8798 t32 = read_fp_sreg(s, a->rn); in do_fp1_scalar_with_fpsttype()
8799 f->gen_s(t32, t32, fpst); in do_fp1_scalar_with_fpsttype()
8800 write_fp_sreg_merging(s, a->rd, a->rd, t32); in do_fp1_scalar_with_fpsttype()
8803 t32 = read_fp_hreg(s, a->rn); in do_fp1_scalar_with_fpsttype()
8804 f->gen_h(t32, t32, fpst); in do_fp1_scalar_with_fpsttype()
8805 write_fp_hreg_merging(s, a->rd, a->rd, t32); in do_fp1_scalar_with_fpsttype()
8821 a->esz == MO_16 ? in do_fp1_scalar()
8828 return do_fp1_scalar_with_fpsttype(s, a, f, rmode, select_ah_fpst(s, a->esz)); in do_fp1_scalar_ah()
8836 TRANS(FSQRT_s, do_fp1_scalar, a, &f_scalar_fsqrt, -1)
8848 TRANS(FRINTI_s, do_fp1_scalar, a, &f_scalar_frint, -1)
8855 TRANS(FRINTX_s, do_fp1_scalar, a, &f_scalar_frintx, -1)
8859 ARMFPStatusFlavour fpsttype = s->fpcr_ah ? FPST_AH : FPST_A64; in trans_BFCVT_s()
8867 check = fp_access_check_scalar_hsd(s, a->esz); in trans_BFCVT_s()
8873 t32 = read_fp_sreg(s, a->rn); in trans_BFCVT_s()
8875 write_fp_hreg_merging(s, a->rd, a->rd, t32); in trans_BFCVT_s()
8886 TRANS_FEAT(FRINT32X_s, aa64_frint, do_fp1_scalar, a, &f_scalar_frint32, -1)
8895 TRANS_FEAT(FRINT64X_s, aa64_frint, do_fp1_scalar, a, &f_scalar_frint64, -1)
8908 s->fpcr_ah && dc_isar_feature(aa64_rpres, s) ?
8909 &f_scalar_frecpe_rpres : &f_scalar_frecpe, -1)
8916 TRANS(FRECPX_s, do_fp1_scalar_ah, a, &f_scalar_frecpx, -1)
8929 s->fpcr_ah && dc_isar_feature(aa64_rpres, s) ?
8930 &f_scalar_frsqrte_rpres : &f_scalar_frsqrte, -1)
8935 TCGv_i32 tcg_rn = read_fp_sreg(s, a->rn); in trans_FCVT_s_ds()
8940 write_fp_dreg_merging(s, a->rd, a->rd, tcg_rd); in trans_FCVT_s_ds()
8948 TCGv_i32 tmp = read_fp_sreg(s, a->rn); in trans_FCVT_s_hs()
8954 write_fp_hreg_merging(s, a->rd, a->rd, tmp); in trans_FCVT_s_hs()
8962 TCGv_i64 tcg_rn = read_fp_dreg(s, a->rn); in trans_FCVT_s_sd()
8967 write_fp_sreg_merging(s, a->rd, a->rd, tcg_rd); in trans_FCVT_s_sd()
8975 TCGv_i64 tcg_rn = read_fp_dreg(s, a->rn); in trans_FCVT_s_hd()
8982 write_fp_hreg_merging(s, a->rd, a->rd, tcg_rd); in trans_FCVT_s_hd()
8990 TCGv_i32 tcg_rn = read_fp_hreg(s, a->rn); in trans_FCVT_s_sh()
8996 write_fp_sreg_merging(s, a->rd, a->rd, tcg_rd); in trans_FCVT_s_sh()
9004 TCGv_i32 tcg_rn = read_fp_hreg(s, a->rn); in trans_FCVT_s_dh()
9010 write_fp_dreg_merging(s, a->rd, a->rd, tcg_rd); in trans_FCVT_s_dh()
9065 int check = fp_access_check_scalar_hsd(s, a->esz); in do_cvtf_g()
9071 if (a->sf) { in do_cvtf_g()
9072 tcg_int = cpu_reg(s, a->rn); in do_cvtf_g()
9074 tcg_int = read_cpu_reg(s, a->rn, true); in do_cvtf_g()
9081 return do_cvtf_scalar(s, a->esz, a->rd, a->shift, tcg_int, is_signed); in do_cvtf_g()
9096 int check = fp_access_check_scalar_hsd(s, a->esz); in TRANS()
9103 read_vec_element(s, tcg_int, a->rn, 0, a->esz | (is_signed ? MO_SIGN : 0)); in TRANS()
9104 return do_cvtf_scalar(s, a->esz, a->rd, a->shift, tcg_int, is_signed); in TRANS()
9211 int check = fp_access_check_scalar_hsd(s, a->esz); in do_fcvt_g()
9217 tcg_int = cpu_reg(s, a->rd); in do_fcvt_g()
9218 do_fcvt_scalar(s, (a->sf ? MO_64 : MO_32) | (is_signed ? MO_SIGN : 0), in do_fcvt_g()
9219 a->esz, tcg_int, a->shift, a->rn, rmode); in do_fcvt_g()
9221 if (!a->sf) { in do_fcvt_g()
9248 int check = fp_access_check_scalar_hsd(s, a->esz); in TRANS()
9255 do_fcvt_scalar(s, a->esz | (is_signed ? MO_SIGN : 0), in TRANS()
9256 a->esz, tcg_int, a->shift, a->rn, rmode); in TRANS()
9258 if (!s->fpcr_nep) { in TRANS()
9259 clear_vec(s, a->rd); in TRANS()
9261 write_vec_element(s, tcg_int, a->rd, 0, a->esz); in TRANS()
9282 TCGv_i64 t = read_fp_dreg(s, a->rn); in TRANS()
9287 tcg_gen_ext32u_i64(cpu_reg(s, a->rd), t); in TRANS()
9302 TCGv_i64 tcg_rn = cpu_reg(s, a->rn); in trans_FMOV_hx()
9305 write_fp_dreg(s, a->rd, tmp); in trans_FMOV_hx()
9313 TCGv_i64 tcg_rn = cpu_reg(s, a->rn); in trans_FMOV_sw()
9316 write_fp_dreg(s, a->rd, tmp); in trans_FMOV_sw()
9324 TCGv_i64 tcg_rn = cpu_reg(s, a->rn); in trans_FMOV_dx()
9325 write_fp_dreg(s, a->rd, tcg_rn); in trans_FMOV_dx()
9333 TCGv_i64 tcg_rn = cpu_reg(s, a->rn); in trans_FMOV_ux()
9334 tcg_gen_st_i64(tcg_rn, tcg_env, fp_reg_hi_offset(s, a->rd)); in trans_FMOV_ux()
9335 clear_vec_high(s, true, a->rd); in trans_FMOV_ux()
9346 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); in trans_FMOV_xh()
9347 tcg_gen_ld16u_i64(tcg_rd, tcg_env, fp_reg_offset(s, a->rn, MO_16)); in trans_FMOV_xh()
9355 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); in trans_FMOV_ws()
9356 tcg_gen_ld32u_i64(tcg_rd, tcg_env, fp_reg_offset(s, a->rn, MO_32)); in trans_FMOV_ws()
9364 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); in trans_FMOV_xd()
9365 tcg_gen_ld_i64(tcg_rd, tcg_env, fp_reg_offset(s, a->rn, MO_64)); in trans_FMOV_xd()
9373 TCGv_i64 tcg_rd = cpu_reg(s, a->rd); in trans_FMOV_xu()
9374 tcg_gen_ld_i64(tcg_rd, tcg_env, fp_reg_hi_offset(s, a->rn)); in trans_FMOV_xu()
9389 if (a->esz == MO_64) { in do_env_scalar1()
9390 TCGv_i64 t = read_fp_dreg(s, a->rn); in do_env_scalar1()
9391 f->gen_d(t, tcg_env, t); in do_env_scalar1()
9392 write_fp_dreg(s, a->rd, t); in do_env_scalar1()
9396 read_vec_element_i32(s, t, a->rn, 0, a->esz); in do_env_scalar1()
9397 f->gen_bhs[a->esz](t, tcg_env, t); in do_env_scalar1()
9398 write_fp_sreg(s, a->rd, t); in do_env_scalar1()
9405 if (a->esz == MO_64 && !a->q) { in do_env_vector1()
9411 if (a->esz == MO_64) { in do_env_vector1()
9415 read_vec_element(s, t, a->rn, i, MO_64); in do_env_vector1()
9416 f->gen_d(t, tcg_env, t); in do_env_vector1()
9417 write_vec_element(s, t, a->rd, i, MO_64); in do_env_vector1()
9421 int n = (a->q ? 16 : 8) >> a->esz; in do_env_vector1()
9424 read_vec_element_i32(s, t, a->rn, i, a->esz); in do_env_vector1()
9425 f->gen_bhs[a->esz](t, tcg_env, t); in do_env_vector1()
9426 write_vec_element_i32(s, t, a->rd, i, a->esz); in do_env_vector1()
9429 clear_vec_high(s, a->q, a->rd); in do_env_vector1()
9454 TCGv_i64 t = read_fp_dreg(s, a->rn); in do_scalar1_d()
9456 write_fp_dreg(s, a->rd, t); in do_scalar1_d()
9467 TCGv_i64 t = read_fp_dreg(s, a->rn); in TRANS()
9469 write_fp_dreg(s, a->rd, t); in TRANS()
9483 if (a->esz == MO_64) { in TRANS()
9489 read_vec_element(s, t, a->rn, 0, a->esz + 1); in TRANS()
9490 fn[a->esz](t, t); in TRANS()
9491 clear_vec(s, a->rd); in TRANS()
9492 write_vec_element(s, t, a->rd, 0, a->esz); in TRANS()
9541 TCGv_i64 src = read_fp_dreg(s, a->rn); in TRANS()
9544 write_fp_sreg_merging(s, a->rd, a->rd, dst); in TRANS()
9553 if (!a->q && a->esz == MO_64) { in do_gvec_fn2()
9557 gen_gvec_fn2(s, a->q, a->rd, a->rn, fn, a->esz); in do_gvec_fn2()
9579 if (a->esz == MO_64) { in TRANS()
9583 gen_gvec_fn2(s, a->q, a->rd, a->rn, fn, a->esz); in TRANS()
9599 if (a->esz == MO_64) { in TRANS()
9606 read_vec_element(s, t0, a->rn, 0, MO_64); in TRANS()
9607 read_vec_element(s, t1, a->rn, 1, MO_64); in TRANS()
9608 fn[a->esz](t0, t0); in TRANS()
9609 fn[a->esz](t1, t1); in TRANS()
9610 write_vec_element(s, t0, a->rd, a->q ? 2 : 0, MO_32); in TRANS()
9611 write_vec_element(s, t1, a->rd, a->q ? 3 : 1, MO_32); in TRANS()
9612 clear_vec_high(s, a->q, a->rd); in TRANS()
9702 f_vector_bfcvtn[s->fpcr_ah]) in TRANS_FEAT()
9716 if (a->esz == MO_64) { in TRANS_FEAT()
9724 widenfn = widenfns[a->esz]; in TRANS_FEAT()
9725 part = a->q ? 2 : 0; in TRANS_FEAT()
9728 read_vec_element_i32(s, tcg_op, a->rn, part + pass, MO_32); in TRANS_FEAT()
9731 tcg_gen_shli_i64(tcg_res[pass], tcg_res[pass], 8 << a->esz); in TRANS_FEAT()
9735 write_vec_element(s, tcg_res[pass], a->rd, pass, MO_64); in TRANS_FEAT()
9742 int check = fp_access_check_vector_hsd(s, a->q, a->esz); in do_fabs_fneg_v()
9748 gen_gvec_fn2(s, a->q, a->rd, a->rn, fn, a->esz); in do_fabs_fneg_v()
9760 int check = fp_access_check_vector_hsd(s, a->q, a->esz); in TRANS()
9766 fpst = fpstatus_ptr(a->esz == MO_16 ? FPST_A64_F16 : FPST_A64); in TRANS()
9771 if (a->esz == MO_64) { in TRANS()
9775 read_vec_element(s, t64, a->rn, pass, MO_64); in TRANS()
9776 f->gen_d(t64, t64, fpst); in TRANS()
9777 write_vec_element(s, t64, a->rd, pass, MO_64); in TRANS()
9782 = (a->esz == MO_16 ? f->gen_h : f->gen_s); in TRANS()
9784 for (int pass = 0, n = (a->q ? 16 : 8) >> a->esz; pass < n; ++pass) { in TRANS()
9785 read_vec_element_i32(s, t32, a->rn, pass, a->esz); in TRANS()
9787 write_vec_element_i32(s, t32, a->rd, pass, a->esz); in TRANS()
9790 clear_vec_high(s, a->q, a->rd); in TRANS()
9798 TRANS(FSQRT_v, do_fp1_vector, a, &f_scalar_fsqrt, -1)
9805 TRANS(FRINTI_v, do_fp1_vector, a, &f_scalar_frint, -1)
9806 TRANS(FRINTX_v, do_fp1_vector, a, &f_scalar_frintx, -1)
9810 TRANS_FEAT(FRINT32X_v, aa64_frint, do_fp1_vector, a, &f_scalar_frint32, -1)
9813 TRANS_FEAT(FRINT64X_v, aa64_frint, do_fp1_vector, a, &f_scalar_frint64, -1)
9831 data, fns[esz - 1]); in do_gvec_op2_fpst_with_fpsttype()
9858 a->esz, a->q, a->rd, a->rn, 0, f_scvtf_v)
9860 a->esz, a->q, a->rd, a->rn, a->shift, f_scvtf_v)
9868 a->esz, a->q, a->rd, a->rn, 0, f_ucvtf_v)
9870 a->esz, a->q, a->rd, a->rn, a->shift, f_ucvtf_v)
9878 a->esz, a->q, a->rd, a->rn, a->shift, f_fcvtzs_vf)
9886 a->esz, a->q, a->rd, a->rn, a->shift, f_fcvtzu_vf)
9901 a->esz, a->q, a->rd, a->rn, float_round_nearest_even, f_fcvt_s_vi)
9903 a->esz, a->q, a->rd, a->rn, float_round_nearest_even, f_fcvt_u_vi)
9905 a->esz, a->q, a->rd, a->rn, float_round_up, f_fcvt_s_vi)
9907 a->esz, a->q, a->rd, a->rn, float_round_up, f_fcvt_u_vi)
9909 a->esz, a->q, a->rd, a->rn, float_round_down, f_fcvt_s_vi)
9911 a->esz, a->q, a->rd, a->rn, float_round_down, f_fcvt_u_vi)
9913 a->esz, a->q, a->rd, a->rn, float_round_to_zero, f_fcvt_s_vi)
9915 a->esz, a->q, a->rd, a->rn, float_round_to_zero, f_fcvt_u_vi)
9917 a->esz, a->q, a->rd, a->rn, float_round_ties_away, f_fcvt_s_vi)
9919 a->esz, a->q, a->rd, a->rn, float_round_ties_away, f_fcvt_u_vi)
9926 TRANS(FCMEQ0_v, do_gvec_op2_fpst, a->esz, a->q, a->rd, a->rn, 0, f_fceq0)
9933 TRANS(FCMGT0_v, do_gvec_op2_fpst, a->esz, a->q, a->rd, a->rn, 0, f_fcgt0)
9940 TRANS(FCMGE0_v, do_gvec_op2_fpst, a->esz, a->q, a->rd, a->rn, 0, f_fcge0)
9947 TRANS(FCMLT0_v, do_gvec_op2_fpst, a->esz, a->q, a->rd, a->rn, 0, f_fclt0)
9954 TRANS(FCMLE0_v, do_gvec_op2_fpst, a->esz, a->q, a->rd, a->rn, 0, f_fcle0)
9966 TRANS(FRECPE_v, do_gvec_op2_ah_fpst, a->esz, a->q, a->rd, a->rn, 0,
9967 s->fpcr_ah && dc_isar_feature(aa64_rpres, s) ? f_frecpe_rpres : f_frecpe)
9979 TRANS(FRSQRTE_v, do_gvec_op2_ah_fpst, a->esz, a->q, a->rd, a->rn, 0,
9980 s->fpcr_ah && dc_isar_feature(aa64_rpres, s) ? f_frsqrte_rpres : f_frsqrte)
9984 /* Handle 2-reg-misc ops which are widening (so each size element in trans_FCVTL_v()
9995 if (a->esz == MO_64) { in trans_FCVTL_v()
9996 /* 32 -> 64 bit fp conversion */ in trans_FCVTL_v()
9999 int srcelt = a->q ? 2 : 0; in trans_FCVTL_v()
10005 read_vec_element_i32(s, tcg_op, a->rn, srcelt + pass, MO_32); in trans_FCVTL_v()
10009 write_vec_element(s, tcg_res[pass], a->rd, pass, MO_64); in trans_FCVTL_v()
10012 /* 16 -> 32 bit fp conversion */ in trans_FCVTL_v()
10013 int srcelt = a->q ? 4 : 0; in trans_FCVTL_v()
10021 read_vec_element_i32(s, tcg_res[pass], a->rn, srcelt + pass, MO_16); in trans_FCVTL_v()
10026 write_vec_element_i32(s, tcg_res[pass], a->rd, pass, MO_32); in trans_FCVTL_v()
10029 clear_vec_high(s, true, a->rd); in trans_FCVTL_v()
10040 s->is_nonstreaming = true; in trans_FAIL()
10048 * @btype: PSTATE.BTYPE, and is non-zero
10052 * - branch target identifiers,
10053 * - paciasp, pacibsp,
10054 * - BRK insn
10055 * - HLT insn
10102 CPUARMTBFlags tb_flags = arm_tbflags_from_tb(dc->base.tb); in aarch64_tr_init_disas_context()
10105 dc->isar = &arm_cpu->isar; in aarch64_tr_init_disas_context()
10106 dc->condjmp = 0; in aarch64_tr_init_disas_context()
10107 dc->pc_save = dc->base.pc_first; in aarch64_tr_init_disas_context()
10108 dc->aarch64 = true; in aarch64_tr_init_disas_context()
10109 dc->thumb = false; in aarch64_tr_init_disas_context()
10110 dc->sctlr_b = 0; in aarch64_tr_init_disas_context()
10111 dc->be_data = EX_TBFLAG_ANY(tb_flags, BE_DATA) ? MO_BE : MO_LE; in aarch64_tr_init_disas_context()
10112 dc->condexec_mask = 0; in aarch64_tr_init_disas_context()
10113 dc->condexec_cond = 0; in aarch64_tr_init_disas_context()
10115 dc->mmu_idx = core_to_aa64_mmu_idx(core_mmu_idx); in aarch64_tr_init_disas_context()
10116 dc->tbii = EX_TBFLAG_A64(tb_flags, TBII); in aarch64_tr_init_disas_context()
10117 dc->tbid = EX_TBFLAG_A64(tb_flags, TBID); in aarch64_tr_init_disas_context()
10118 dc->tcma = EX_TBFLAG_A64(tb_flags, TCMA); in aarch64_tr_init_disas_context()
10119 dc->current_el = arm_mmu_idx_to_el(dc->mmu_idx); in aarch64_tr_init_disas_context()
10121 dc->user = (dc->current_el == 0); in aarch64_tr_init_disas_context()
10123 dc->fp_excp_el = EX_TBFLAG_ANY(tb_flags, FPEXC_EL); in aarch64_tr_init_disas_context()
10124 dc->align_mem = EX_TBFLAG_ANY(tb_flags, ALIGN_MEM); in aarch64_tr_init_disas_context()
10125 dc->pstate_il = EX_TBFLAG_ANY(tb_flags, PSTATE__IL); in aarch64_tr_init_disas_context()
10126 dc->fgt_active = EX_TBFLAG_ANY(tb_flags, FGT_ACTIVE); in aarch64_tr_init_disas_context()
10127 dc->fgt_svc = EX_TBFLAG_ANY(tb_flags, FGT_SVC); in aarch64_tr_init_disas_context()
10128 dc->trap_eret = EX_TBFLAG_A64(tb_flags, TRAP_ERET); in aarch64_tr_init_disas_context()
10129 dc->sve_excp_el = EX_TBFLAG_A64(tb_flags, SVEEXC_EL); in aarch64_tr_init_disas_context()
10130 dc->sme_excp_el = EX_TBFLAG_A64(tb_flags, SMEEXC_EL); in aarch64_tr_init_disas_context()
10131 dc->vl = (EX_TBFLAG_A64(tb_flags, VL) + 1) * 16; in aarch64_tr_init_disas_context()
10132 dc->svl = (EX_TBFLAG_A64(tb_flags, SVL) + 1) * 16; in aarch64_tr_init_disas_context()
10133 dc->pauth_active = EX_TBFLAG_A64(tb_flags, PAUTH_ACTIVE); in aarch64_tr_init_disas_context()
10134 dc->bt = EX_TBFLAG_A64(tb_flags, BT); in aarch64_tr_init_disas_context()
10135 dc->btype = EX_TBFLAG_A64(tb_flags, BTYPE); in aarch64_tr_init_disas_context()
10136 dc->unpriv = EX_TBFLAG_A64(tb_flags, UNPRIV); in aarch64_tr_init_disas_context()
10137 dc->ata[0] = EX_TBFLAG_A64(tb_flags, ATA); in aarch64_tr_init_disas_context()
10138 dc->ata[1] = EX_TBFLAG_A64(tb_flags, ATA0); in aarch64_tr_init_disas_context()
10139 dc->mte_active[0] = EX_TBFLAG_A64(tb_flags, MTE_ACTIVE); in aarch64_tr_init_disas_context()
10140 dc->mte_active[1] = EX_TBFLAG_A64(tb_flags, MTE0_ACTIVE); in aarch64_tr_init_disas_context()
10141 dc->pstate_sm = EX_TBFLAG_A64(tb_flags, PSTATE_SM); in aarch64_tr_init_disas_context()
10142 dc->pstate_za = EX_TBFLAG_A64(tb_flags, PSTATE_ZA); in aarch64_tr_init_disas_context()
10143 dc->sme_trap_nonstreaming = EX_TBFLAG_A64(tb_flags, SME_TRAP_NONSTREAMING); in aarch64_tr_init_disas_context()
10144 dc->naa = EX_TBFLAG_A64(tb_flags, NAA); in aarch64_tr_init_disas_context()
10145 dc->nv = EX_TBFLAG_A64(tb_flags, NV); in aarch64_tr_init_disas_context()
10146 dc->nv1 = EX_TBFLAG_A64(tb_flags, NV1); in aarch64_tr_init_disas_context()
10147 dc->nv2 = EX_TBFLAG_A64(tb_flags, NV2); in aarch64_tr_init_disas_context()
10148 dc->nv2_mem_e20 = EX_TBFLAG_A64(tb_flags, NV2_MEM_E20); in aarch64_tr_init_disas_context()
10149 dc->nv2_mem_be = EX_TBFLAG_A64(tb_flags, NV2_MEM_BE); in aarch64_tr_init_disas_context()
10150 dc->fpcr_ah = EX_TBFLAG_A64(tb_flags, AH); in aarch64_tr_init_disas_context()
10151 dc->fpcr_nep = EX_TBFLAG_A64(tb_flags, NEP); in aarch64_tr_init_disas_context()
10152 dc->vec_len = 0; in aarch64_tr_init_disas_context()
10153 dc->vec_stride = 0; in aarch64_tr_init_disas_context()
10154 dc->cp_regs = arm_cpu->cp_regs; in aarch64_tr_init_disas_context()
10155 dc->features = env->features; in aarch64_tr_init_disas_context()
10156 dc->dcz_blocksize = arm_cpu->dcz_blocksize; in aarch64_tr_init_disas_context()
10157 dc->gm_blocksize = arm_cpu->gm_blocksize; in aarch64_tr_init_disas_context()
10161 tcg_debug_assert(dc->tbid & 1); in aarch64_tr_init_disas_context()
10164 dc->lse2 = dc_isar_feature(aa64_lse2, dc); in aarch64_tr_init_disas_context()
10166 /* Single step state. The code-generation logic here is: in aarch64_tr_init_disas_context()
10168 * generate code with no special handling for single-stepping (except in aarch64_tr_init_disas_context()
10172 * SS_ACTIVE == 1, PSTATE.SS == 1: (active-not-pending) in aarch64_tr_init_disas_context()
10177 * SS_ACTIVE == 1, PSTATE.SS == 0: (active-pending) in aarch64_tr_init_disas_context()
10181 dc->ss_active = EX_TBFLAG_ANY(tb_flags, SS_ACTIVE); in aarch64_tr_init_disas_context()
10182 dc->pstate_ss = EX_TBFLAG_ANY(tb_flags, PSTATE__SS); in aarch64_tr_init_disas_context()
10183 dc->is_ldex = false; in aarch64_tr_init_disas_context()
10186 bound = -(dc->base.pc_first | TARGET_PAGE_MASK) / 4; in aarch64_tr_init_disas_context()
10189 if (dc->ss_active) { in aarch64_tr_init_disas_context()
10192 dc->base.max_insns = MIN(dc->base.max_insns, bound); in aarch64_tr_init_disas_context()
10202 target_ulong pc_arg = dc->base.pc_next; in aarch64_tr_insn_start()
10204 if (tb_cflags(dcbase->tb) & CF_PCREL) { in aarch64_tr_insn_start()
10208 dc->insn_start_updated = false; in aarch64_tr_insn_start()
10215 uint64_t pc = s->base.pc_next; in aarch64_tr_translate_insn()
10219 if (s->ss_active && !s->pstate_ss) { in aarch64_tr_translate_insn()
10220 /* Singlestep state is Active-pending. in aarch64_tr_translate_insn()
10230 assert(s->base.num_insns == 1); in aarch64_tr_translate_insn()
10232 s->base.is_jmp = DISAS_NORETURN; in aarch64_tr_translate_insn()
10233 s->base.pc_next = pc + 4; in aarch64_tr_translate_insn()
10244 assert(s->base.num_insns == 1); in aarch64_tr_translate_insn()
10246 s->base.is_jmp = DISAS_NORETURN; in aarch64_tr_translate_insn()
10247 s->base.pc_next = QEMU_ALIGN_UP(pc, 4); in aarch64_tr_translate_insn()
10251 s->pc_curr = pc; in aarch64_tr_translate_insn()
10252 insn = arm_ldl_code(env, &s->base, pc, s->sctlr_b); in aarch64_tr_translate_insn()
10253 s->insn = insn; in aarch64_tr_translate_insn()
10254 s->base.pc_next = pc + 4; in aarch64_tr_translate_insn()
10256 s->fp_access_checked = 0; in aarch64_tr_translate_insn()
10257 s->sve_access_checked = 0; in aarch64_tr_translate_insn()
10259 if (s->pstate_il) { in aarch64_tr_translate_insn()
10269 if (s->base.num_insns == 1) { in aarch64_tr_translate_insn()
10270 /* First insn can have btype set to non-zero. */ in aarch64_tr_translate_insn()
10271 tcg_debug_assert(s->btype >= 0); in aarch64_tr_translate_insn()
10275 * priority -- below debugging exceptions but above most in aarch64_tr_translate_insn()
10282 if (s->btype != 0 in aarch64_tr_translate_insn()
10283 && !btype_destination_ok(insn, s->bt, s->btype)) { in aarch64_tr_translate_insn()
10288 tcg_debug_assert(s->btype == 0); in aarch64_tr_translate_insn()
10292 s->is_nonstreaming = false; in aarch64_tr_translate_insn()
10293 if (s->sme_trap_nonstreaming) { in aarch64_tr_translate_insn()
10305 * Note that we set btype == -1 when the insn sets btype. in aarch64_tr_translate_insn()
10307 if (s->btype > 0 && s->base.is_jmp != DISAS_NORETURN) { in aarch64_tr_translate_insn()
10316 if (unlikely(dc->ss_active)) { in aarch64_tr_tb_stop()
10320 * (and thus a tb-jump is not possible when singlestepping). in aarch64_tr_tb_stop()
10322 switch (dc->base.is_jmp) { in aarch64_tr_tb_stop()
10334 switch (dc->base.is_jmp) { in aarch64_tr_tb_stop()