Lines Matching refs:vg
102 uint32_t HELPER(sve_predtest)(void *vd, void *vg, uint32_t words) in HELPER()
105 uint64_t *d = vd, *g = vg; in HELPER()
127 void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \
130 uint64_t *d = vd, *n = vn, *m = vm, *g = vg; \
174 void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \ in LOGICAL_PPPP()
178 uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \ in LOGICAL_PPPP()
192 void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \
196 uint8_t *pg = vg; \
669 void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \ in DO_ZPZZ()
673 uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \ in DO_ZPZZ()
693 void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \
697 uint8_t *pg = vg; \
739 void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, \
744 uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \
789 void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \
793 uint8_t pg = *(uint8_t *)(vg + H1(i >> 3)); \
822 void HELPER(NAME)(void *vd, void *vn, void *vg, uint32_t desc) \
826 uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \
839 void HELPER(NAME)(void *vd, void *vn, void *vg, uint32_t desc) \
843 uint8_t *pg = vg; \
959 void HELPER(sme_revd_q)(void *vd, void *vn, void *vg, uint32_t desc)
963 uint8_t *pg = vg;
1756 uint64_t HELPER(NAME)(void *vn, void *vg, uint32_t desc) \
1761 uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \
1774 uint64_t HELPER(NAME)(void *vn, void *vg, uint32_t desc) \
1778 uint8_t *pg = vg; \
1909 uint32_t HELPER(sve_pfirst)(void *vd, void *vg, uint32_t pred_desc) in HELPER()
1913 uint64_t *d = vd, *g = vg; in HELPER()
1933 uint32_t HELPER(sve_pnext)(void *vd, void *vg, uint32_t pred_desc) in HELPER()
1938 uint64_t *d = vd, *g = vg, esz_mask; in HELPER()
1981 void HELPER(sve_movz_b)(void *vd, void *vn, void *vg, uint32_t desc) in HELPER()
1986 uint8_t *pg = vg; in HELPER()
1993 void HELPER(sve_movz_h)(void *vd, void *vn, void *vg, uint32_t desc) in HELPER()
1998 uint8_t *pg = vg; in HELPER()
2005 void HELPER(sve_movz_s)(void *vd, void *vn, void *vg, uint32_t desc) in HELPER()
2010 uint8_t *pg = vg; in HELPER()
2017 void HELPER(sve_movz_d)(void *vd, void *vn, void *vg, uint32_t desc) in HELPER()
2021 uint8_t *pg = vg; in HELPER()
2032 void HELPER(NAME)(void *vd, void *vn, void *vg, uint32_t desc) \
2037 uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \
2050 void HELPER(NAME)(void *vd, void *vn, void *vg, uint32_t desc) \
2055 uint8_t *pg = vg; \
2336 void *vg, uint32_t desc) \
2340 uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \
2356 void *vg, uint32_t desc) \
2360 uint8_t *pg = vg; \
2705 void HELPER(sve_cpy_m_b)(void *vd, void *vn, void *vg, in HELPER()
2710 uint8_t *pg = vg; in HELPER()
2720 void HELPER(sve_cpy_m_h)(void *vd, void *vn, void *vg, in HELPER()
2725 uint8_t *pg = vg; in HELPER()
2735 void HELPER(sve_cpy_m_s)(void *vd, void *vn, void *vg, in HELPER()
2740 uint8_t *pg = vg; in HELPER()
2750 void HELPER(sve_cpy_m_d)(void *vd, void *vn, void *vg, in HELPER()
2755 uint8_t *pg = vg; in HELPER()
2763 void HELPER(sve_cpy_z_b)(void *vd, void *vg, uint64_t val, uint32_t desc) in HELPER()
2767 uint8_t *pg = vg; in HELPER()
2775 void HELPER(sve_cpy_z_h)(void *vd, void *vg, uint64_t val, uint32_t desc) in HELPER()
2779 uint8_t *pg = vg; in HELPER()
2787 void HELPER(sve_cpy_z_s)(void *vd, void *vg, uint64_t val, uint32_t desc) in HELPER()
2791 uint8_t *pg = vg; in HELPER()
2799 void HELPER(sve_cpy_z_d)(void *vd, void *vg, uint64_t val, uint32_t desc) in HELPER()
2803 uint8_t *pg = vg; in HELPER()
3483 void HELPER(sve_compact_s)(void *vd, void *vn, void *vg, uint32_t desc)
3487 uint8_t *pg = vg;
3500 void HELPER(sve_compact_d)(void *vd, void *vn, void *vg, uint32_t desc) in HELPER()
3504 uint8_t *pg = vg; in HELPER()
3521 int32_t HELPER(sve_last_active_element)(void *vg, uint32_t pred_desc) in HELPER()
3526 return last_active_element(vg, words, esz); in HELPER()
3529 void HELPER(sve_splice)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) in HELPER()
3542 pg = *(uint64_t *)(vg + i) & mask; in HELPER()
3567 void *vg, uint32_t desc) in HELPER()
3571 uint8_t *pg = vg; in HELPER()
3581 void *vg, uint32_t desc) in HELPER()
3585 uint8_t *pg = vg; in HELPER()
3595 void *vg, uint32_t desc) in HELPER()
3599 uint8_t *pg = vg; in HELPER()
3609 void *vg, uint32_t desc) in HELPER()
3613 uint8_t *pg = vg; in HELPER()
3622 void *vg, uint32_t desc) in HELPER()
3626 uint16_t *pg = vg; in HELPER()
3655 uint32_t HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \
3668 pg = *(uint64_t *)(vg + (i >> 3)) & MASK; \
3723 uint32_t HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \
3738 pg = *(uint64_t *)(vg + (i >> 3)) & MASK; \
3800 uint32_t HELPER(NAME)(void *vd, void *vn, void *vg, uint32_t desc) \
3813 pg = *(uint64_t *)(vg + (i >> 3)) & MASK; \
3887 static bool last_active_pred(void *vd, void *vg, intptr_t oprsz) in last_active_pred() argument
3892 uint64_t pg = *(uint64_t *)(vg + i); in last_active_pred()
4005 void HELPER(sve_brkpa)(void *vd, void *vn, void *vm, void *vg, in HELPER()
4009 if (last_active_pred(vn, vg, oprsz)) { in HELPER()
4010 compute_brk_z(vd, vm, vg, oprsz, true); in HELPER()
4016 uint32_t HELPER(sve_brkpas)(void *vd, void *vn, void *vm, void *vg, in HELPER()
4020 if (last_active_pred(vn, vg, oprsz)) { in HELPER()
4021 return compute_brks_z(vd, vm, vg, oprsz, true); in HELPER()
4027 void HELPER(sve_brkpb)(void *vd, void *vn, void *vm, void *vg, in HELPER()
4031 if (last_active_pred(vn, vg, oprsz)) { in HELPER()
4032 compute_brk_z(vd, vm, vg, oprsz, false); in HELPER()
4038 uint32_t HELPER(sve_brkpbs)(void *vd, void *vn, void *vm, void *vg, in HELPER()
4042 if (last_active_pred(vn, vg, oprsz)) { in HELPER()
4043 return compute_brks_z(vd, vm, vg, oprsz, false); in HELPER()
4049 void HELPER(sve_brka_z)(void *vd, void *vn, void *vg, uint32_t pred_desc) in HELPER()
4052 compute_brk_z(vd, vn, vg, oprsz, true); in HELPER()
4055 uint32_t HELPER(sve_brkas_z)(void *vd, void *vn, void *vg, uint32_t pred_desc) in HELPER()
4058 return compute_brks_z(vd, vn, vg, oprsz, true); in HELPER()
4061 void HELPER(sve_brkb_z)(void *vd, void *vn, void *vg, uint32_t pred_desc) in HELPER()
4064 compute_brk_z(vd, vn, vg, oprsz, false); in HELPER()
4067 uint32_t HELPER(sve_brkbs_z)(void *vd, void *vn, void *vg, uint32_t pred_desc) in HELPER()
4070 return compute_brks_z(vd, vn, vg, oprsz, false); in HELPER()
4073 void HELPER(sve_brka_m)(void *vd, void *vn, void *vg, uint32_t pred_desc) in HELPER()
4076 compute_brk_m(vd, vn, vg, oprsz, true); in HELPER()
4079 uint32_t HELPER(sve_brkas_m)(void *vd, void *vn, void *vg, uint32_t pred_desc) in HELPER()
4082 return compute_brks_m(vd, vn, vg, oprsz, true); in HELPER()
4085 void HELPER(sve_brkb_m)(void *vd, void *vn, void *vg, uint32_t pred_desc) in HELPER()
4088 compute_brk_m(vd, vn, vg, oprsz, false); in HELPER()
4091 uint32_t HELPER(sve_brkbs_m)(void *vd, void *vn, void *vg, uint32_t pred_desc) in HELPER()
4094 return compute_brks_m(vd, vn, vg, oprsz, false); in HELPER()
4097 void HELPER(sve_brkn)(void *vd, void *vn, void *vg, uint32_t pred_desc) in HELPER()
4100 if (!last_active_pred(vn, vg, oprsz)) { in HELPER()
4122 uint32_t HELPER(sve_brkns)(void *vd, void *vn, void *vg, uint32_t pred_desc) in HELPER()
4125 if (last_active_pred(vn, vg, oprsz)) { in HELPER()
4132 uint64_t HELPER(sve_cntp)(void *vn, void *vg, uint32_t pred_desc) in HELPER()
4136 uint64_t *n = vn, *g = vg, sum = 0, mask = pred_esz_masks[esz]; in HELPER()
4228 uint64_t HELPER(NAME)(void *vn, void *vg, float_status *s, uint32_t desc) \
4233 uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); \
4280 uint64_t HELPER(sve_fadda_h)(uint64_t nn, void *vm, void *vg, in DO_REDUCE()
4287 uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); in DO_REDUCE()
4300 uint64_t HELPER(sve_fadda_s)(uint64_t nn, void *vm, void *vg, in HELPER()
4307 uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)); in HELPER()
4320 uint64_t HELPER(sve_fadda_d)(uint64_t nn, void *vm, void *vg, in HELPER()
4325 uint8_t *pg = vg; in HELPER()
4340 void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, \
4344 uint64_t *g = vg; \
4459 void HELPER(NAME)(void *vd, void *vn, void *vg, uint64_t scalar, \ in DO_ZPZZ_FP()
4463 uint64_t *g = vg; \ in DO_ZPZZ_FP()
4536 void HELPER(NAME)(void *vd, void *vn, void *vg, \ in DO_ZPZS_FP()
4540 uint64_t *g = vg; \ in DO_ZPZS_FP()
4819 static void do_fmla_zpzzz_h(void *vd, void *vn, void *vm, void *va, void *vg, in DO_ZPZ_FP()
4824 uint64_t *g = vg; in DO_ZPZ_FP()
4844 void *vg, float_status *status, uint32_t desc) in HELPER()
4846 do_fmla_zpzzz_h(vd, vn, vm, va, vg, status, desc, 0, 0, 0); in HELPER()
4850 void *vg, float_status *status, uint32_t desc) in HELPER()
4852 do_fmla_zpzzz_h(vd, vn, vm, va, vg, status, desc, 0x8000, 0, 0); in HELPER()
4856 void *vg, float_status *status, uint32_t desc) in HELPER()
4858 do_fmla_zpzzz_h(vd, vn, vm, va, vg, status, desc, 0x8000, 0x8000, 0); in HELPER()
4862 void *vg, float_status *status, uint32_t desc) in HELPER()
4864 do_fmla_zpzzz_h(vd, vn, vm, va, vg, status, desc, 0, 0x8000, 0); in HELPER()
4868 void *vg, float_status *status, uint32_t desc) in HELPER()
4870 do_fmla_zpzzz_h(vd, vn, vm, va, vg, status, desc, 0, 0, in HELPER()
4875 void *vg, float_status *status, uint32_t desc) in HELPER()
4877 do_fmla_zpzzz_h(vd, vn, vm, va, vg, status, desc, 0, 0, in HELPER()
4882 void *vg, float_status *status, uint32_t desc) in HELPER()
4884 do_fmla_zpzzz_h(vd, vn, vm, va, vg, status, desc, 0, 0, in HELPER()
4888 static void do_fmla_zpzzz_s(void *vd, void *vn, void *vm, void *va, void *vg, in do_fmla_zpzzz_s() argument
4893 uint64_t *g = vg; in do_fmla_zpzzz_s()
4913 void *vg, float_status *status, uint32_t desc) in HELPER()
4915 do_fmla_zpzzz_s(vd, vn, vm, va, vg, status, desc, 0, 0, 0); in HELPER()
4919 void *vg, float_status *status, uint32_t desc) in HELPER()
4921 do_fmla_zpzzz_s(vd, vn, vm, va, vg, status, desc, 0x80000000, 0, 0); in HELPER()
4925 void *vg, float_status *status, uint32_t desc) in HELPER()
4927 do_fmla_zpzzz_s(vd, vn, vm, va, vg, status, desc, 0x80000000, 0x80000000, 0); in HELPER()
4931 void *vg, float_status *status, uint32_t desc) in HELPER()
4933 do_fmla_zpzzz_s(vd, vn, vm, va, vg, status, desc, 0, 0x80000000, 0); in HELPER()
4937 void *vg, float_status *status, uint32_t desc) in HELPER()
4939 do_fmla_zpzzz_s(vd, vn, vm, va, vg, status, desc, 0, 0, in HELPER()
4944 void *vg, float_status *status, uint32_t desc) in HELPER()
4946 do_fmla_zpzzz_s(vd, vn, vm, va, vg, status, desc, 0, 0, in HELPER()
4951 void *vg, float_status *status, uint32_t desc) in HELPER()
4953 do_fmla_zpzzz_s(vd, vn, vm, va, vg, status, desc, 0, 0, in HELPER()
4957 static void do_fmla_zpzzz_d(void *vd, void *vn, void *vm, void *va, void *vg, in do_fmla_zpzzz_d() argument
4962 uint64_t *g = vg; in do_fmla_zpzzz_d()
4982 void *vg, float_status *status, uint32_t desc) in HELPER()
4984 do_fmla_zpzzz_d(vd, vn, vm, va, vg, status, desc, 0, 0, 0); in HELPER()
4988 void *vg, float_status *status, uint32_t desc) in HELPER()
4990 do_fmla_zpzzz_d(vd, vn, vm, va, vg, status, desc, INT64_MIN, 0, 0); in HELPER()
4994 void *vg, float_status *status, uint32_t desc) in HELPER()
4996 do_fmla_zpzzz_d(vd, vn, vm, va, vg, status, desc, INT64_MIN, INT64_MIN, 0); in HELPER()
5000 void *vg, float_status *status, uint32_t desc) in HELPER()
5002 do_fmla_zpzzz_d(vd, vn, vm, va, vg, status, desc, 0, INT64_MIN, 0); in HELPER()
5006 void *vg, float_status *status, uint32_t desc) in HELPER()
5008 do_fmla_zpzzz_d(vd, vn, vm, va, vg, status, desc, 0, 0, in HELPER()
5013 void *vg, float_status *status, uint32_t desc) in HELPER()
5015 do_fmla_zpzzz_d(vd, vn, vm, va, vg, status, desc, 0, 0, in HELPER()
5020 void *vg, float_status *status, uint32_t desc) in HELPER()
5022 do_fmla_zpzzz_d(vd, vn, vm, va, vg, status, desc, 0, 0, in HELPER()
5032 void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, \
5036 uint64_t *d = vd, *g = vg; \
5094 void HELPER(NAME)(void *vd, void *vn, void *vg, \ in DO_FPCMP_PPZZ_ALL()
5098 uint64_t *d = vd, *g = vg; \ in DO_FPCMP_PPZZ_ALL()
5232 void HELPER(sve_fcadd_h)(void *vd, void *vn, void *vm, void *vg, in HELPER()
5236 uint64_t *g = vg; in HELPER()
5270 void HELPER(sve_fcadd_s)(void *vd, void *vn, void *vm, void *vg, in HELPER()
5274 uint64_t *g = vg; in HELPER()
5308 void HELPER(sve_fcadd_d)(void *vd, void *vn, void *vm, void *vg, in HELPER()
5312 uint64_t *g = vg; in HELPER()
5351 void *vg, float_status *status, uint32_t desc) in HELPER()
5359 uint64_t *g = vg; in HELPER()
5401 void *vg, float_status *status, uint32_t desc) in HELPER()
5409 uint64_t *g = vg; in HELPER()
5451 void *vg, float_status *status, uint32_t desc) in HELPER()
5459 uint64_t *g = vg; in HELPER()
5509 static intptr_t find_next_active(uint64_t *vg, intptr_t reg_off, in find_next_active() argument
5513 uint64_t pg = (vg[reg_off >> 6] & pg_mask) >> (reg_off & 63); in find_next_active()
5528 pg = vg[reg_off >> 6] & pg_mask; in find_next_active()
5598 bool sve_cont_ldst_elements(SVEContLdSt *info, target_ulong addr, uint64_t *vg, in sve_cont_ldst_elements() argument
5615 uint64_t pg = vg[i] & pg_mask; in sve_cont_ldst_elements()
5659 if ((vg[reg_off_split >> 6] >> (reg_off_split & 63)) & 1) { in sve_cont_ldst_elements()
5676 reg_off_split = find_next_active(vg, reg_off_split, reg_max, esz); in sve_cont_ldst_elements()
5749 uint64_t *vg, target_ulong addr, in sve_cont_ldst_watchpoints() argument
5771 uint64_t pg = vg[reg_off >> 6]; in sve_cont_ldst_watchpoints()
5796 uint64_t pg = vg[reg_off >> 6]; in sve_cont_ldst_watchpoints()
5812 uint64_t *vg, target_ulong addr, int esize, in sve_cont_ldst_mte_check() argument
5827 uint64_t pg = vg[reg_off >> 6]; in sve_cont_ldst_mte_check()
5844 uint64_t pg = vg[reg_off >> 6]; in sve_cont_ldst_mte_check()
5860 void sve_ldN_r(CPUARMState *env, uint64_t *vg, const target_ulong addr, in sve_ldN_r() argument
5874 if (!sve_cont_ldst_elements(&info, addr, vg, reg_max, esz, N << msz)) { in sve_ldN_r()
5886 sve_cont_ldst_watchpoints(&info, env, vg, addr, 1 << esz, N << msz, in sve_ldN_r()
5894 sve_cont_ldst_mte_check(&info, env, vg, addr, 1 << esz, N << msz, in sve_ldN_r()
5919 uint64_t pg = vg[reg_off >> 6]; in sve_ldN_r()
5952 uint64_t pg = vg[reg_off >> 6]; in sve_ldN_r()
5989 uint64_t pg = vg[reg_off >> 6]; in sve_ldN_r()
6007 void sve_ldN_r_mte(CPUARMState *env, uint64_t *vg, target_ulong addr, in sve_ldN_r_mte() argument
6025 sve_ldN_r(env, vg, addr, desc, ra, esz, msz, N, mtedesc, host_fn, tlb_fn); in sve_ldN_r_mte()
6029 void HELPER(sve_##NAME##_r)(CPUARMState *env, void *vg, \
6032 sve_ldN_r(env, vg, addr, desc, GETPC(), ESZ, MO_8, 1, 0, \
6035 void HELPER(sve_##NAME##_r_mte)(CPUARMState *env, void *vg, \
6038 sve_ldN_r_mte(env, vg, addr, desc, GETPC(), ESZ, MO_8, 1, \
6043 void HELPER(sve_##NAME##_le_r)(CPUARMState *env, void *vg, \
6046 sve_ldN_r(env, vg, addr, desc, GETPC(), ESZ, MSZ, 1, 0, \
6049 void HELPER(sve_##NAME##_be_r)(CPUARMState *env, void *vg, \
6052 sve_ldN_r(env, vg, addr, desc, GETPC(), ESZ, MSZ, 1, 0, \
6055 void HELPER(sve_##NAME##_le_r_mte)(CPUARMState *env, void *vg, \
6058 sve_ldN_r_mte(env, vg, addr, desc, GETPC(), ESZ, MSZ, 1, \
6061 void HELPER(sve_##NAME##_be_r_mte)(CPUARMState *env, void *vg, \
6064 sve_ldN_r_mte(env, vg, addr, desc, GETPC(), ESZ, MSZ, 1, \
6092 void HELPER(sve_ld##N##bb_r)(CPUARMState *env, void *vg, \ in DO_LD1_1()
6095 sve_ldN_r(env, vg, addr, desc, GETPC(), MO_8, MO_8, N, 0, \ in DO_LD1_1()
6098 void HELPER(sve_ld##N##bb_r_mte)(CPUARMState *env, void *vg, \
6101 sve_ldN_r_mte(env, vg, addr, desc, GETPC(), MO_8, MO_8, N, \
6106 void HELPER(sve_ld##N##SUFF##_le_r)(CPUARMState *env, void *vg, \
6109 sve_ldN_r(env, vg, addr, desc, GETPC(), ESZ, ESZ, N, 0, \
6112 void HELPER(sve_ld##N##SUFF##_be_r)(CPUARMState *env, void *vg, \
6115 sve_ldN_r(env, vg, addr, desc, GETPC(), ESZ, ESZ, N, 0, \
6118 void HELPER(sve_ld##N##SUFF##_le_r_mte)(CPUARMState *env, void *vg, \
6121 sve_ldN_r_mte(env, vg, addr, desc, GETPC(), ESZ, ESZ, N, \
6124 void HELPER(sve_ld##N##SUFF##_be_r_mte)(CPUARMState *env, void *vg, \
6127 sve_ldN_r_mte(env, vg, addr, desc, GETPC(), ESZ, ESZ, N, \
6181 void sve_ldnfff1_r(CPUARMState *env, void *vg, const target_ulong addr, in sve_ldnfff1_r() argument
6196 if (!sve_cont_ldst_elements(&info, addr, vg, reg_max, esz, 1 << msz)) { in sve_ldnfff1_r()
6311 uint64_t pg = *(uint64_t *)(vg + (reg_off >> 3)); in sve_ldnfff1_r()
6365 void sve_ldnfff1_r_mte(CPUARMState *env, void *vg, target_ulong addr, in sve_ldnfff1_r_mte() argument
6383 sve_ldnfff1_r(env, vg, addr, desc, retaddr, mtedesc, in sve_ldnfff1_r_mte()
6388 void HELPER(sve_ldff1##PART##_r)(CPUARMState *env, void *vg, \
6391 sve_ldnfff1_r(env, vg, addr, desc, GETPC(), 0, ESZ, MO_8, FAULT_FIRST, \
6394 void HELPER(sve_ldnf1##PART##_r)(CPUARMState *env, void *vg, \
6397 sve_ldnfff1_r(env, vg, addr, desc, GETPC(), 0, ESZ, MO_8, FAULT_NO, \
6400 void HELPER(sve_ldff1##PART##_r_mte)(CPUARMState *env, void *vg, \
6403 sve_ldnfff1_r_mte(env, vg, addr, desc, GETPC(), ESZ, MO_8, FAULT_FIRST, \
6406 void HELPER(sve_ldnf1##PART##_r_mte)(CPUARMState *env, void *vg, \
6409 sve_ldnfff1_r_mte(env, vg, addr, desc, GETPC(), ESZ, MO_8, FAULT_NO, \
6414 void HELPER(sve_ldff1##PART##_le_r)(CPUARMState *env, void *vg, \
6417 sve_ldnfff1_r(env, vg, addr, desc, GETPC(), 0, ESZ, MSZ, FAULT_FIRST, \
6420 void HELPER(sve_ldnf1##PART##_le_r)(CPUARMState *env, void *vg, \
6423 sve_ldnfff1_r(env, vg, addr, desc, GETPC(), 0, ESZ, MSZ, FAULT_NO, \
6426 void HELPER(sve_ldff1##PART##_be_r)(CPUARMState *env, void *vg, \
6429 sve_ldnfff1_r(env, vg, addr, desc, GETPC(), 0, ESZ, MSZ, FAULT_FIRST, \
6432 void HELPER(sve_ldnf1##PART##_be_r)(CPUARMState *env, void *vg, \
6435 sve_ldnfff1_r(env, vg, addr, desc, GETPC(), 0, ESZ, MSZ, FAULT_NO, \
6438 void HELPER(sve_ldff1##PART##_le_r_mte)(CPUARMState *env, void *vg, \
6441 sve_ldnfff1_r_mte(env, vg, addr, desc, GETPC(), ESZ, MSZ, FAULT_FIRST, \
6444 void HELPER(sve_ldnf1##PART##_le_r_mte)(CPUARMState *env, void *vg, \
6447 sve_ldnfff1_r_mte(env, vg, addr, desc, GETPC(), ESZ, MSZ, FAULT_NO, \
6450 void HELPER(sve_ldff1##PART##_be_r_mte)(CPUARMState *env, void *vg, \
6453 sve_ldnfff1_r_mte(env, vg, addr, desc, GETPC(), ESZ, MSZ, FAULT_FIRST, \
6456 void HELPER(sve_ldnf1##PART##_be_r_mte)(CPUARMState *env, void *vg, \
6459 sve_ldnfff1_r_mte(env, vg, addr, desc, GETPC(), ESZ, MSZ, FAULT_NO, \
6491 void sve_stN_r(CPUARMState *env, uint64_t *vg, target_ulong addr, in DO_LDFF1_LDNF1_1()
6505 if (!sve_cont_ldst_elements(&info, addr, vg, reg_max, esz, N << msz)) { in DO_LDFF1_LDNF1_1()
6514 sve_cont_ldst_watchpoints(&info, env, vg, addr, 1 << esz, N << msz, in DO_LDFF1_LDNF1_1()
6522 sve_cont_ldst_mte_check(&info, env, vg, addr, 1 << esz, N << msz, in DO_LDFF1_LDNF1_1()
6545 uint64_t pg = vg[reg_off >> 6]; in DO_LDFF1_LDNF1_1()
6568 uint64_t pg = vg[reg_off >> 6]; in DO_LDFF1_LDNF1_1()
6605 uint64_t pg = vg[reg_off >> 6]; in DO_LDFF1_LDNF1_1()
6623 void sve_stN_r_mte(CPUARMState *env, uint64_t *vg, target_ulong addr, in sve_stN_r_mte() argument
6641 sve_stN_r(env, vg, addr, desc, ra, esz, msz, N, mtedesc, host_fn, tlb_fn); in sve_stN_r_mte()
6645 void HELPER(sve_st##N##NAME##_r)(CPUARMState *env, void *vg, \
6648 sve_stN_r(env, vg, addr, desc, GETPC(), ESZ, MO_8, N, 0, \
6651 void HELPER(sve_st##N##NAME##_r_mte)(CPUARMState *env, void *vg, \
6654 sve_stN_r_mte(env, vg, addr, desc, GETPC(), ESZ, MO_8, N, \
6659 void HELPER(sve_st##N##NAME##_le_r)(CPUARMState *env, void *vg, \
6662 sve_stN_r(env, vg, addr, desc, GETPC(), ESZ, MSZ, N, 0, \
6665 void HELPER(sve_st##N##NAME##_be_r)(CPUARMState *env, void *vg, \
6668 sve_stN_r(env, vg, addr, desc, GETPC(), ESZ, MSZ, N, 0, \
6671 void HELPER(sve_st##N##NAME##_le_r_mte)(CPUARMState *env, void *vg, \
6674 sve_stN_r_mte(env, vg, addr, desc, GETPC(), ESZ, MSZ, N, \
6677 void HELPER(sve_st##N##NAME##_be_r_mte)(CPUARMState *env, void *vg, \
6680 sve_stN_r_mte(env, vg, addr, desc, GETPC(), ESZ, MSZ, N, \
6748 void sve_ld1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm, in sve_ld1_z() argument
6765 uint64_t pg = vg[reg_off >> 6]; in sve_ld1_z()
6814 void sve_ld1_z_mte(CPUARMState *env, void *vd, uint64_t *vg, void *vm, in sve_ld1_z_mte() argument
6830 sve_ld1_z(env, vd, vg, vm, base, desc, retaddr, mtedesc, in sve_ld1_z_mte()
6835 void HELPER(sve_ld##MEM##_##OFS)(CPUARMState *env, void *vd, void *vg, \
6838 sve_ld1_z(env, vd, vg, vm, base, desc, GETPC(), 0, 4, 1 << MSZ, \
6841 void HELPER(sve_ld##MEM##_##OFS##_mte)(CPUARMState *env, void *vd, void *vg, \
6844 sve_ld1_z_mte(env, vd, vg, vm, base, desc, GETPC(), 4, 1 << MSZ, \
6849 void HELPER(sve_ld##MEM##_##OFS)(CPUARMState *env, void *vd, void *vg, \
6852 sve_ld1_z(env, vd, vg, vm, base, desc, GETPC(), 0, 8, 1 << MSZ, \
6855 void HELPER(sve_ld##MEM##_##OFS##_mte)(CPUARMState *env, void *vd, void *vg, \
6858 sve_ld1_z_mte(env, vd, vg, vm, base, desc, GETPC(), 8, 1 << MSZ, \
6936 void sve_ldff1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm, in DO_LD1_ZPZ_S()
6954 reg_off = find_next_active(vg, 0, reg_max, esz); in DO_LD1_ZPZ_S()
6984 uint64_t pg = vg[reg_off >> 6]; in DO_LD1_ZPZ_S()
7023 void sve_ldff1_z_mte(CPUARMState *env, void *vd, uint64_t *vg, void *vm, in sve_ldff1_z_mte() argument
7040 sve_ldff1_z(env, vd, vg, vm, base, desc, retaddr, mtedesc, in sve_ldff1_z_mte()
7046 (CPUARMState *env, void *vd, void *vg, \
7049 sve_ldff1_z(env, vd, vg, vm, base, desc, GETPC(), 0, MO_32, MSZ, \
7053 (CPUARMState *env, void *vd, void *vg, \
7056 sve_ldff1_z_mte(env, vd, vg, vm, base, desc, GETPC(), MO_32, MSZ, \
7062 (CPUARMState *env, void *vd, void *vg, \
7065 sve_ldff1_z(env, vd, vg, vm, base, desc, GETPC(), 0, MO_64, MSZ, \
7069 (CPUARMState *env, void *vd, void *vg, \
7072 sve_ldff1_z_mte(env, vd, vg, vm, base, desc, GETPC(), MO_64, MSZ, \
7143 void sve_st1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm, in DO_LDFF1_ZPZ_S()
7162 uint64_t pg = vg[reg_off >> 6]; in DO_LDFF1_ZPZ_S()
7218 } else if ((vg[reg_off >> 6] >> (reg_off & 63)) & 1) { in DO_LDFF1_ZPZ_S()
7228 void sve_st1_z_mte(CPUARMState *env, void *vd, uint64_t *vg, void *vm, in sve_st1_z_mte() argument
7244 sve_st1_z(env, vd, vg, vm, base, desc, retaddr, mtedesc, in sve_st1_z_mte()
7249 void HELPER(sve_st##MEM##_##OFS)(CPUARMState *env, void *vd, void *vg, \
7252 sve_st1_z(env, vd, vg, vm, base, desc, GETPC(), 0, 4, 1 << MSZ, \
7255 void HELPER(sve_st##MEM##_##OFS##_mte)(CPUARMState *env, void *vd, void *vg, \
7258 sve_st1_z_mte(env, vd, vg, vm, base, desc, GETPC(), 4, 1 << MSZ, \
7263 void HELPER(sve_st##MEM##_##OFS)(CPUARMState *env, void *vd, void *vg, \
7266 sve_st1_z(env, vd, vg, vm, base, desc, GETPC(), 0, 8, 1 << MSZ, \
7269 void HELPER(sve_st##MEM##_##OFS##_mte)(CPUARMState *env, void *vd, void *vg, \
7272 sve_st1_z_mte(env, vd, vg, vm, base, desc, GETPC(), 8, 1 << MSZ, \
7385 static inline uint32_t do_match(void *vd, void *vn, void *vm, void *vg, in do_match() argument
7396 uint16_t pg = *(uint16_t *)(vg + H1_2(i >> 3)) & esz_mask; in do_match()
7416 uint32_t HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \
7418 return do_match(vd, vn, vm, vg, desc, ESZ, INV); \
7429 void HELPER(sve2_histcnt_s)(void *vd, void *vn, void *vm, void *vg, in DO_PPZZ_MATCH()
7436 uint8_t *pg = vg; in DO_PPZZ_MATCH()
7466 void HELPER(sve2_histcnt_d)(void *vd, void *vn, void *vm, void *vg, in HELPER()
7473 uint8_t *pg = vg; in HELPER()
7674 void HELPER(NAME)(void *vd, void *vn, void *vg, \
7678 uint64_t *g = vg; \
7696 void HELPER(NAME)(void *vd, void *vn, void *vg, \
7700 uint64_t *g = vg; \