Lines Matching refs:vd
102 uint32_t HELPER(sve_predtest)(void *vd, void *vg, uint32_t words) in HELPER()
105 uint64_t *d = vd, *g = vg; in HELPER()
127 void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \
130 uint64_t *d = vd, *n = vn, *m = vm, *g = vg; \
174 void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \ in LOGICAL_PPPP()
183 *(TYPE *)(vd + H(i)) = OP(nn, mm); \ in LOGICAL_PPPP()
192 void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \
195 TYPE *d = vd, *n = vn, *m = vm; \
669 void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \ in DO_ZPZZ()
680 *(TYPE *)(vd + H(i)) = OP(n0, n1); \ in DO_ZPZZ()
684 *(TYPE *)(vd + H(i)) = OP(m0, m1); \ in DO_ZPZZ()
693 void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \
696 TYPE *d = vd, *n = vn, *m = vm; \
739 void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, \
751 *(TYPE *)(vd + H(i)) = OP(n0, n1, status); \
755 *(TYPE *)(vd + H(i)) = OP(m0, m1, status); \
789 void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \
798 *(TYPE *)(vd + H(i)) = OP(nn, mm); \
822 void HELPER(NAME)(void *vd, void *vn, void *vg, uint32_t desc) \
830 *(TYPE *)(vd + H(i)) = OP(nn); \
839 void HELPER(NAME)(void *vd, void *vn, void *vg, uint32_t desc) \
842 TYPE *d = vd, *n = vn; \
959 void HELPER(sme_revd_q)(void *vd, void *vn, void *vg, uint32_t desc)
962 uint64_t *d = vd, *n = vn;
1004 void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
1011 *(TYPE *)(vd + H(i)) = OP(nn, mm); \
1048 void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
1056 *(TYPEW *)(vd + HW(i)) = OP(nn, mm); \
1118 void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \ in DO_ZZZ_TB()
1125 *(TYPEW *)(vd + HW(i)) = OP(nn, mm); \ in DO_ZZZ_TB()
1148 void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
1156 *(TYPE *)(vd + H(i + sel1)) = OP(nn, mm); \
1168 void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \
1176 *(TYPEW *)(vd + HW(i)) = OP(nn, mm) + aa; \
1209 void HELPER(NAME)(void *vd, void *vn, uint32_t desc) \
1215 *(TYPE *)(vd + i) = nn; \
1220 void HELPER(NAME)(void *vd, void *vn, uint32_t desc) \
1225 *(TYPEN *)(vd + i + odd) = OP(nn); \
1264 void HELPER(sve2_adcl_s)(void *vd, void *vn, void *vm, void *va, uint32_t desc)
1270 uint64_t *d = vd, *m = vm;
1281 void HELPER(sve2_adcl_d)(void *vd, void *vn, void *vm, void *va, uint32_t desc) in HELPER()
1286 uint64_t *d = vd, *a = va, *n = vn, *m = vm; in HELPER()
1299 void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \
1308 *(TYPEW *)(vd + HW(i)) = SUM_OP(aa, DMUL_OP(nn, mm)); \
1329 void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \ in DO_SQDMLAL()
1336 TYPE *d = vd, *n = vn, *m = vm, *a = va; \ in DO_SQDMLAL()
1368 void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \
1376 TYPE *d = vd, *n = vn, *m = vm, *a = va; \
1431 void HELPER(sve2_cdot_zzzz_s)(void *vd, void *vn, void *vm, in HELPER()
1439 uint32_t *d = vd, *n = vn, *m = vm, *a = va; in HELPER()
1446 void HELPER(sve2_cdot_zzzz_d)(void *vd, void *vn, void *vm, in HELPER()
1454 uint64_t *d = vd, *n = vn, *m = vm, *a = va; in HELPER()
1461 void HELPER(sve2_cdot_idx_s)(void *vd, void *vn, void *vm, in HELPER()
1470 uint32_t *d = vd, *n = vn, *m = vm, *a = va; in HELPER()
1481 void HELPER(sve2_cdot_idx_d)(void *vd, void *vn, void *vm, in HELPER()
1490 uint64_t *d = vd, *n = vn, *m = vm, *a = va; in HELPER()
1502 void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \
1506 TYPE *d = vd, *a = va, *n = vn, *m = (TYPE *)vm + H(idx); \
1538 void HELPER(NAME)(void *vd, void *vn, void *vm, void *va, uint32_t desc) \
1548 *(TYPEW *)(vd + HW(i + j)) = OP(nn, mm, aa); \
1584 void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
1593 *(TYPEW *)(vd + HW(i + j)) = OP(nn, mm); \
1610 void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
1616 *(TYPE *)(vd + i) = OP(nn, mm, sizeof(TYPE) * 8); \
1683 void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \ in DO_BITPERM()
1695 *(TYPE *)(vd + H(i)) = acc_r; \ in DO_BITPERM()
1696 *(TYPE *)(vd + H(i + sizeof(TYPE))) = acc_i; \ in DO_BITPERM()
1706 *(TYPE *)(vd + H(i)) = acc_r; \ in DO_BITPERM()
1707 *(TYPE *)(vd + H(i + sizeof(TYPE))) = acc_i; \ in DO_BITPERM()
1725 void HELPER(NAME)(void *vd, void *vn, uint32_t desc) \
1732 *(TYPEW *)(vd + HW(i)) = nn << shift; \
1838 void HELPER(NAME)(void *vd, void *vn, uint64_t s64, uint32_t desc) \
1841 TYPE s = s64, *d = vd, *n = vn; \
1909 uint32_t HELPER(sve_pfirst)(void *vd, void *vg, uint32_t pred_desc) in HELPER()
1913 uint64_t *d = vd, *g = vg; in HELPER()
1933 uint32_t HELPER(sve_pnext)(void *vd, void *vg, uint32_t pred_desc) in HELPER()
1938 uint64_t *d = vd, *g = vg, esz_mask; in HELPER()
1941 next = last_active_element(vd, words, esz) + (1 << esz); in HELPER()
1981 void HELPER(sve_movz_b)(void *vd, void *vn, void *vg, uint32_t desc) in HELPER()
1985 uint64_t *d = vd, *n = vn; in HELPER()
1993 void HELPER(sve_movz_h)(void *vd, void *vn, void *vg, uint32_t desc) in HELPER()
1997 uint64_t *d = vd, *n = vn; in HELPER()
2005 void HELPER(sve_movz_s)(void *vd, void *vn, void *vg, uint32_t desc) in HELPER()
2009 uint64_t *d = vd, *n = vn; in HELPER()
2017 void HELPER(sve_movz_d)(void *vd, void *vn, void *vg, uint32_t desc) in HELPER()
2020 uint64_t *d = vd, *n = vn; in HELPER()
2032 void HELPER(NAME)(void *vd, void *vn, void *vg, uint32_t desc) \
2041 *(TYPE *)(vd + H(i)) = OP(nn, imm); \
2050 void HELPER(NAME)(void *vd, void *vn, void *vg, uint32_t desc) \
2053 TYPE *d = vd, *n = vn; \
2153 void HELPER(NAME)(void *vd, void *vn, uint32_t desc) \
2159 *(TYPEW *)(vd + i) = (TYPEN)OP(nn, shift); \
2164 void HELPER(NAME)(void *vd, void *vn, uint32_t desc) \
2170 *(TYPEN *)(vd + HN(i + sizeof(TYPEN))) = OP(nn, shift); \
2267 void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
2273 *(TYPEW *)(vd + i) = (TYPEN)OP(nn, mm, SHIFT); \
2278 void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
2284 *(TYPEN *)(vd + HN(i + sizeof(TYPEN))) = OP(nn, mm, SHIFT); \
2335 void HELPER(NAME)(void *vd, void *va, void *vn, void *vm, \
2346 *(TYPE *)(vd + H(i)) = OP(aa, nn, mm); \
2355 void HELPER(NAME)(void *vd, void *va, void *vn, void *vm, \
2359 TYPE *d = vd, *a = va, *n = vn, *m = vm; \
2389 void HELPER(sve_index_b)(void *vd, uint32_t start,
2393 uint8_t *d = vd;
2399 void HELPER(sve_index_h)(void *vd, uint32_t start, in HELPER()
2403 uint16_t *d = vd; in HELPER()
2409 void HELPER(sve_index_s)(void *vd, uint32_t start, in HELPER()
2413 uint32_t *d = vd; in HELPER()
2419 void HELPER(sve_index_d)(void *vd, uint64_t start, in HELPER()
2423 uint64_t *d = vd; in HELPER()
2429 void HELPER(sve_adr_p32)(void *vd, void *vn, void *vm, uint32_t desc) in HELPER()
2433 uint32_t *d = vd, *n = vn, *m = vm; in HELPER()
2439 void HELPER(sve_adr_p64)(void *vd, void *vn, void *vm, uint32_t desc) in HELPER()
2443 uint64_t *d = vd, *n = vn, *m = vm; in HELPER()
2449 void HELPER(sve_adr_s32)(void *vd, void *vn, void *vm, uint32_t desc) in HELPER()
2453 uint64_t *d = vd, *n = vn, *m = vm; in HELPER()
2459 void HELPER(sve_adr_u32)(void *vd, void *vn, void *vm, uint32_t desc) in HELPER()
2463 uint64_t *d = vd, *n = vn, *m = vm; in HELPER()
2469 void HELPER(sve_fexpa_h)(void *vd, void *vn, uint32_t desc) in HELPER()
2479 uint16_t *d = vd, *n = vn; in HELPER()
2489 void HELPER(sve_fexpa_s)(void *vd, void *vn, uint32_t desc) in HELPER()
2511 uint32_t *d = vd, *n = vn; in HELPER()
2521 void HELPER(sve_fexpa_d)(void *vd, void *vn, uint32_t desc) in HELPER()
2549 uint64_t *d = vd, *n = vn; in HELPER()
2559 void HELPER(sve_ftssel_h)(void *vd, void *vn, void *vm, uint32_t desc) in HELPER()
2563 uint16_t *d = vd, *n = vn, *m = vm; in HELPER()
2577 void HELPER(sve_ftssel_s)(void *vd, void *vn, void *vm, uint32_t desc) in HELPER()
2581 uint32_t *d = vd, *n = vn, *m = vm; in HELPER()
2595 void HELPER(sve_ftssel_d)(void *vd, void *vn, void *vm, uint32_t desc) in HELPER()
2599 uint64_t *d = vd, *n = vn, *m = vm; in HELPER()
2705 void HELPER(sve_cpy_m_b)(void *vd, void *vn, void *vg, in HELPER()
2709 uint64_t *d = vd, *n = vn; in HELPER()
2720 void HELPER(sve_cpy_m_h)(void *vd, void *vn, void *vg, in HELPER()
2724 uint64_t *d = vd, *n = vn; in HELPER()
2735 void HELPER(sve_cpy_m_s)(void *vd, void *vn, void *vg, in HELPER()
2739 uint64_t *d = vd, *n = vn; in HELPER()
2750 void HELPER(sve_cpy_m_d)(void *vd, void *vn, void *vg, in HELPER()
2754 uint64_t *d = vd, *n = vn; in HELPER()
2763 void HELPER(sve_cpy_z_b)(void *vd, void *vg, uint64_t val, uint32_t desc) in HELPER()
2766 uint64_t *d = vd; in HELPER()
2775 void HELPER(sve_cpy_z_h)(void *vd, void *vg, uint64_t val, uint32_t desc) in HELPER()
2778 uint64_t *d = vd; in HELPER()
2787 void HELPER(sve_cpy_z_s)(void *vd, void *vg, uint64_t val, uint32_t desc) in HELPER()
2790 uint64_t *d = vd; in HELPER()
2799 void HELPER(sve_cpy_z_d)(void *vd, void *vg, uint64_t val, uint32_t desc) in HELPER()
2802 uint64_t *d = vd; in HELPER()
2813 static void swap_memmove(void *vd, void *vs, size_t n) in swap_memmove() argument
2815 uintptr_t d = (uintptr_t)vd; in swap_memmove()
2825 memmove(vd, vs, n); in swap_memmove()
2871 static void swap_memzero(void *vd, size_t n) in swap_memzero() argument
2873 uintptr_t d = (uintptr_t)vd; in swap_memzero()
2887 memset(vd, 0, n); in swap_memzero()
2911 void HELPER(sve_ext)(void *vd, void *vn, void *vm, uint32_t desc) in HELPER()
2917 if (vd != vm) { in HELPER()
2918 swap_memmove(vd, vn + n_ofs, n_siz); in HELPER()
2919 swap_memmove(vd + n_siz, vm, n_ofs); in HELPER()
2920 } else if (vd != vn) { in HELPER()
2921 swap_memmove(vd + n_siz, vd, n_ofs); in HELPER()
2922 swap_memmove(vd, vn + n_ofs, n_siz); in HELPER()
2927 swap_memmove(vd, vd + n_ofs, n_siz); in HELPER()
2928 memcpy(vd + n_siz, &tmp, n_ofs); in HELPER()
2933 void HELPER(NAME)(void *vd, void *vn, uint64_t val, uint32_t desc) \
2936 swap_memmove(vd + sizeof(TYPE), vn, opr_sz - sizeof(TYPE)); \
2937 *(TYPE *)(vd + H(0)) = val; \
2947 void HELPER(sve_rev_b)(void *vd, void *vn, uint32_t desc) in DO_INSR()
2953 *(uint64_t *)(vd + i) = bswap64(b); in DO_INSR()
2954 *(uint64_t *)(vd + j) = bswap64(f); in DO_INSR()
2958 void HELPER(sve_rev_h)(void *vd, void *vn, uint32_t desc) in HELPER()
2964 *(uint64_t *)(vd + i) = hswap64(b); in HELPER()
2965 *(uint64_t *)(vd + j) = hswap64(f); in HELPER()
2969 void HELPER(sve_rev_s)(void *vd, void *vn, uint32_t desc) in HELPER()
2975 *(uint64_t *)(vd + i) = rol64(b, 32); in HELPER()
2976 *(uint64_t *)(vd + j) = rol64(f, 32); in HELPER()
2980 void HELPER(sve_rev_d)(void *vd, void *vn, uint32_t desc) in HELPER()
2986 *(uint64_t *)(vd + i) = b; in HELPER()
2987 *(uint64_t *)(vd + j) = f; in HELPER()
2993 static inline void do_tbl1(void *vd, void *vn, void *vm, uint32_t desc, in do_tbl1() argument
2999 if (unlikely(vd == vn)) { in do_tbl1()
3003 fn(vd, vn, NULL, vm, oprsz, is_tbx); in do_tbl1()
3006 static inline void do_tbl2(void *vd, void *vn0, void *vn1, void *vm, in do_tbl2() argument
3012 if (unlikely(vd == vn0)) { in do_tbl2()
3014 if (vd == vn1) { in do_tbl2()
3017 } else if (unlikely(vd == vn1)) { in do_tbl2()
3021 fn(vd, vn0, vn1, vm, oprsz, is_tbx); in do_tbl2()
3025 static inline void do_tb_##SUFF(void *vd, void *vt0, void *vt1, \
3028 TYPE *d = vd, *tbl0 = vt0, *tbl1 = vt1, *indexes = vm; \
3045 void HELPER(sve_tbl_##SUFF)(void *vd, void *vn, void *vm, uint32_t desc) \
3047 do_tbl1(vd, vn, vm, desc, false, do_tb_##SUFF); \
3049 void HELPER(sve2_tbl_##SUFF)(void *vd, void *vn0, void *vn1, \
3052 do_tbl2(vd, vn0, vn1, vm, desc, false, do_tb_##SUFF); \
3054 void HELPER(sve2_tbx_##SUFF)(void *vd, void *vn, void *vm, uint32_t desc) \
3056 do_tbl1(vd, vn, vm, desc, true, do_tb_##SUFF); \
3067 void HELPER(NAME)(void *vd, void *vn, uint32_t desc) \
3070 TYPED *d = vd; \
3073 if (unlikely(vn - vd < opr_sz)) { \
3137 void HELPER(sve_zip_p)(void *vd, void *vn, void *vm, uint32_t pred_desc) in HELPER()
3143 uint64_t *d = vd; in HELPER()
3161 if (vd == vn) { in HELPER()
3163 if (vd == vm) { in HELPER()
3166 } else if (vd == vm) { in HELPER()
3187 uint16_t *d16 = vd; in HELPER()
3201 void HELPER(sve_uzp_p)(void *vd, void *vn, void *vm, uint32_t pred_desc) in HELPER()
3206 uint64_t *d = vd, *n = vn, *m = vm; in HELPER()
3218 if ((vm - vd) < (uintptr_t)oprsz) { in HELPER()
3257 swap_memmove(vd + oprsz / 2, &tmp_m, oprsz / 2); in HELPER()
3270 void HELPER(sve_trn_p)(void *vd, void *vn, void *vm, uint32_t pred_desc) in HELPER()
3275 uint64_t *d = vd, *n = vn, *m = vm; in HELPER()
3320 void HELPER(sve_rev_p)(void *vd, void *vn, uint32_t pred_desc) in HELPER()
3329 *(uint64_t *)vd = l; in HELPER()
3335 *(uint64_t *)(vd + i) = h; in HELPER()
3336 *(uint64_t *)(vd + ih) = l; in HELPER()
3344 *(uint8_t *)(vd + il) = h; in HELPER()
3345 *(uint8_t *)(vd + ih) = l; in HELPER()
3350 void HELPER(sve_punpk_p)(void *vd, void *vn, uint32_t pred_desc) in HELPER()
3354 uint64_t *d = vd; in HELPER()
3369 if ((vn - vd) < (uintptr_t)oprsz) { in HELPER()
3385 uint16_t *d16 = vd; in HELPER()
3397 void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
3405 if (unlikely((vn - vd) < (uintptr_t)oprsz)) { \
3408 if (unlikely((vm - vd) < (uintptr_t)oprsz)) { \
3412 *(TYPE *)(vd + H(2 * i + 0)) = *(TYPE *)(vn + odd_ofs + H(i)); \
3413 *(TYPE *)(vd + H(2 * i + sizeof(TYPE))) = \
3417 memset(vd + oprsz - 16, 0, 16); \
3428 void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \ in DO_ZIP()
3434 if (unlikely((vm - vd) < (uintptr_t)oprsz)) { \ in DO_ZIP()
3439 *(TYPE *)(vd + H(i)) = *(TYPE *)(vn + H(p)); \ in DO_ZIP()
3444 *(TYPE *)(vd + H(i)) = *(TYPE *)(vm + H(p)); \ in DO_ZIP()
3457 void HELPER(NAME)(void *vd, void *vn, void *vm, uint32_t desc) \
3465 *(TYPE *)(vd + H(i + 0)) = ae; \
3466 *(TYPE *)(vd + H(i + sizeof(TYPE))) = be; \
3469 memset(vd + oprsz - 16, 0, 16); \
3483 void HELPER(sve_compact_s)(void *vd, void *vn, void *vg, uint32_t desc)
3486 uint32_t *d = vd, *n = vn;
3500 void HELPER(sve_compact_d)(void *vd, void *vn, void *vg, uint32_t desc) in HELPER()
3503 uint64_t *d = vd, *n = vn; in HELPER()
3529 void HELPER(sve_splice)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) in HELPER()
3558 if (vd == vm) { in HELPER()
3561 swap_memmove(vd, vn + first_i, len); in HELPER()
3563 swap_memmove(vd + len, vm, opr_sz * 8 - len); in HELPER()
3566 void HELPER(sve_sel_zpzz_b)(void *vd, void *vn, void *vm, in HELPER()
3570 uint64_t *d = vd, *n = vn, *m = vm; in HELPER()
3580 void HELPER(sve_sel_zpzz_h)(void *vd, void *vn, void *vm, in HELPER()
3584 uint64_t *d = vd, *n = vn, *m = vm; in HELPER()
3594 void HELPER(sve_sel_zpzz_s)(void *vd, void *vn, void *vm, in HELPER()
3598 uint64_t *d = vd, *n = vn, *m = vm; in HELPER()
3608 void HELPER(sve_sel_zpzz_d)(void *vd, void *vn, void *vm, in HELPER()
3612 uint64_t *d = vd, *n = vn, *m = vm; in HELPER()
3621 void HELPER(sve_sel_zpzz_q)(void *vd, void *vn, void *vm, in HELPER()
3625 Int128 *d = vd, *n = vn, *m = vm; in HELPER()
3655 uint32_t HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \
3670 *(uint64_t *)(vd + (i >> 3)) = out; \
3723 uint32_t HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \
3740 *(uint64_t *)(vd + (i >> 3)) = out; \
3800 uint32_t HELPER(NAME)(void *vd, void *vn, void *vg, uint32_t desc) \
3815 *(uint64_t *)(vd + (i >> 3)) = out; \
3887 static bool last_active_pred(void *vd, void *vg, intptr_t oprsz) in last_active_pred() argument
3894 return (pow2floor(pg) & *(uint64_t *)(vd + i)) != 0; in last_active_pred()
4005 void HELPER(sve_brkpa)(void *vd, void *vn, void *vm, void *vg, in HELPER()
4010 compute_brk_z(vd, vm, vg, oprsz, true); in HELPER()
4012 do_zero(vd, oprsz); in HELPER()
4016 uint32_t HELPER(sve_brkpas)(void *vd, void *vn, void *vm, void *vg, in HELPER()
4021 return compute_brks_z(vd, vm, vg, oprsz, true); in HELPER()
4023 return do_zero(vd, oprsz); in HELPER()
4027 void HELPER(sve_brkpb)(void *vd, void *vn, void *vm, void *vg, in HELPER()
4032 compute_brk_z(vd, vm, vg, oprsz, false); in HELPER()
4034 do_zero(vd, oprsz); in HELPER()
4038 uint32_t HELPER(sve_brkpbs)(void *vd, void *vn, void *vm, void *vg, in HELPER()
4043 return compute_brks_z(vd, vm, vg, oprsz, false); in HELPER()
4045 return do_zero(vd, oprsz); in HELPER()
4049 void HELPER(sve_brka_z)(void *vd, void *vn, void *vg, uint32_t pred_desc) in HELPER()
4052 compute_brk_z(vd, vn, vg, oprsz, true); in HELPER()
4055 uint32_t HELPER(sve_brkas_z)(void *vd, void *vn, void *vg, uint32_t pred_desc) in HELPER()
4058 return compute_brks_z(vd, vn, vg, oprsz, true); in HELPER()
4061 void HELPER(sve_brkb_z)(void *vd, void *vn, void *vg, uint32_t pred_desc) in HELPER()
4064 compute_brk_z(vd, vn, vg, oprsz, false); in HELPER()
4067 uint32_t HELPER(sve_brkbs_z)(void *vd, void *vn, void *vg, uint32_t pred_desc) in HELPER()
4070 return compute_brks_z(vd, vn, vg, oprsz, false); in HELPER()
4073 void HELPER(sve_brka_m)(void *vd, void *vn, void *vg, uint32_t pred_desc) in HELPER()
4076 compute_brk_m(vd, vn, vg, oprsz, true); in HELPER()
4079 uint32_t HELPER(sve_brkas_m)(void *vd, void *vn, void *vg, uint32_t pred_desc) in HELPER()
4082 return compute_brks_m(vd, vn, vg, oprsz, true); in HELPER()
4085 void HELPER(sve_brkb_m)(void *vd, void *vn, void *vg, uint32_t pred_desc) in HELPER()
4088 compute_brk_m(vd, vn, vg, oprsz, false); in HELPER()
4091 uint32_t HELPER(sve_brkbs_m)(void *vd, void *vn, void *vg, uint32_t pred_desc) in HELPER()
4094 return compute_brks_m(vd, vn, vg, oprsz, false); in HELPER()
4097 void HELPER(sve_brkn)(void *vd, void *vn, void *vg, uint32_t pred_desc) in HELPER()
4101 do_zero(vd, oprsz); in HELPER()
4122 uint32_t HELPER(sve_brkns)(void *vd, void *vn, void *vg, uint32_t pred_desc) in HELPER()
4126 return predtest_ones(vd, oprsz, -1); in HELPER()
4128 return do_zero(vd, oprsz); in HELPER()
4146 uint32_t HELPER(sve_whilel)(void *vd, uint32_t count, uint32_t pred_desc) in HELPER()
4151 ARMPredicateReg *d = vd; in HELPER()
4172 uint32_t HELPER(sve_whileg)(void *vd, uint32_t count, uint32_t pred_desc) in HELPER()
4177 ARMPredicateReg *d = vd; in HELPER()
4340 void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, \
4352 *(TYPE *)(vd + H(i)) = OP(nn, mm, status); \
4459 void HELPER(NAME)(void *vd, void *vn, void *vg, uint64_t scalar, \ in DO_ZPZZ_FP()
4471 *(TYPE *)(vd + H(i)) = OP(nn, mm, status); \ in DO_ZPZZ_FP()
4536 void HELPER(NAME)(void *vd, void *vn, void *vg, \ in DO_ZPZS_FP()
4547 *(TYPE *)(vd + H(i)) = OP(nn, status); \ in DO_ZPZS_FP()
4819 static void do_fmla_zpzzz_h(void *vd, void *vn, void *vm, void *va, void *vg, in DO_ZPZ_FP()
4837 *(uint16_t *)(vd + H1_2(i)) = r; in DO_ZPZ_FP()
4843 void HELPER(sve_fmla_zpzzz_h)(void *vd, void *vn, void *vm, void *va, in HELPER()
4846 do_fmla_zpzzz_h(vd, vn, vm, va, vg, status, desc, 0, 0, 0); in HELPER()
4849 void HELPER(sve_fmls_zpzzz_h)(void *vd, void *vn, void *vm, void *va, in HELPER()
4852 do_fmla_zpzzz_h(vd, vn, vm, va, vg, status, desc, 0x8000, 0, 0); in HELPER()
4855 void HELPER(sve_fnmla_zpzzz_h)(void *vd, void *vn, void *vm, void *va, in HELPER()
4858 do_fmla_zpzzz_h(vd, vn, vm, va, vg, status, desc, 0x8000, 0x8000, 0); in HELPER()
4861 void HELPER(sve_fnmls_zpzzz_h)(void *vd, void *vn, void *vm, void *va, in HELPER()
4864 do_fmla_zpzzz_h(vd, vn, vm, va, vg, status, desc, 0, 0x8000, 0); in HELPER()
4867 void HELPER(sve_ah_fmls_zpzzz_h)(void *vd, void *vn, void *vm, void *va, in HELPER()
4870 do_fmla_zpzzz_h(vd, vn, vm, va, vg, status, desc, 0, 0, in HELPER()
4874 void HELPER(sve_ah_fnmla_zpzzz_h)(void *vd, void *vn, void *vm, void *va, in HELPER()
4877 do_fmla_zpzzz_h(vd, vn, vm, va, vg, status, desc, 0, 0, in HELPER()
4881 void HELPER(sve_ah_fnmls_zpzzz_h)(void *vd, void *vn, void *vm, void *va, in HELPER()
4884 do_fmla_zpzzz_h(vd, vn, vm, va, vg, status, desc, 0, 0, in HELPER()
4888 static void do_fmla_zpzzz_s(void *vd, void *vn, void *vm, void *va, void *vg, in do_fmla_zpzzz_s() argument
4906 *(uint32_t *)(vd + H1_4(i)) = r; in do_fmla_zpzzz_s()
4912 void HELPER(sve_fmla_zpzzz_s)(void *vd, void *vn, void *vm, void *va, in HELPER()
4915 do_fmla_zpzzz_s(vd, vn, vm, va, vg, status, desc, 0, 0, 0); in HELPER()
4918 void HELPER(sve_fmls_zpzzz_s)(void *vd, void *vn, void *vm, void *va, in HELPER()
4921 do_fmla_zpzzz_s(vd, vn, vm, va, vg, status, desc, 0x80000000, 0, 0); in HELPER()
4924 void HELPER(sve_fnmla_zpzzz_s)(void *vd, void *vn, void *vm, void *va, in HELPER()
4927 do_fmla_zpzzz_s(vd, vn, vm, va, vg, status, desc, 0x80000000, 0x80000000, 0); in HELPER()
4930 void HELPER(sve_fnmls_zpzzz_s)(void *vd, void *vn, void *vm, void *va, in HELPER()
4933 do_fmla_zpzzz_s(vd, vn, vm, va, vg, status, desc, 0, 0x80000000, 0); in HELPER()
4936 void HELPER(sve_ah_fmls_zpzzz_s)(void *vd, void *vn, void *vm, void *va, in HELPER()
4939 do_fmla_zpzzz_s(vd, vn, vm, va, vg, status, desc, 0, 0, in HELPER()
4943 void HELPER(sve_ah_fnmla_zpzzz_s)(void *vd, void *vn, void *vm, void *va, in HELPER()
4946 do_fmla_zpzzz_s(vd, vn, vm, va, vg, status, desc, 0, 0, in HELPER()
4950 void HELPER(sve_ah_fnmls_zpzzz_s)(void *vd, void *vn, void *vm, void *va, in HELPER()
4953 do_fmla_zpzzz_s(vd, vn, vm, va, vg, status, desc, 0, 0, in HELPER()
4957 static void do_fmla_zpzzz_d(void *vd, void *vn, void *vm, void *va, void *vg, in do_fmla_zpzzz_d() argument
4975 *(uint64_t *)(vd + i) = r; in do_fmla_zpzzz_d()
4981 void HELPER(sve_fmla_zpzzz_d)(void *vd, void *vn, void *vm, void *va, in HELPER()
4984 do_fmla_zpzzz_d(vd, vn, vm, va, vg, status, desc, 0, 0, 0); in HELPER()
4987 void HELPER(sve_fmls_zpzzz_d)(void *vd, void *vn, void *vm, void *va, in HELPER()
4990 do_fmla_zpzzz_d(vd, vn, vm, va, vg, status, desc, INT64_MIN, 0, 0); in HELPER()
4993 void HELPER(sve_fnmla_zpzzz_d)(void *vd, void *vn, void *vm, void *va, in HELPER()
4996 do_fmla_zpzzz_d(vd, vn, vm, va, vg, status, desc, INT64_MIN, INT64_MIN, 0); in HELPER()
4999 void HELPER(sve_fnmls_zpzzz_d)(void *vd, void *vn, void *vm, void *va, in HELPER()
5002 do_fmla_zpzzz_d(vd, vn, vm, va, vg, status, desc, 0, INT64_MIN, 0); in HELPER()
5005 void HELPER(sve_ah_fmls_zpzzz_d)(void *vd, void *vn, void *vm, void *va, in HELPER()
5008 do_fmla_zpzzz_d(vd, vn, vm, va, vg, status, desc, 0, 0, in HELPER()
5012 void HELPER(sve_ah_fnmla_zpzzz_d)(void *vd, void *vn, void *vm, void *va, in HELPER()
5015 do_fmla_zpzzz_d(vd, vn, vm, va, vg, status, desc, 0, 0, in HELPER()
5019 void HELPER(sve_ah_fnmls_zpzzz_d)(void *vd, void *vn, void *vm, void *va, in HELPER()
5022 do_fmla_zpzzz_d(vd, vn, vm, va, vg, status, desc, 0, 0, in HELPER()
5032 void HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, \
5036 uint64_t *d = vd, *g = vg; \
5094 void HELPER(NAME)(void *vd, void *vn, void *vg, \ in DO_FPCMP_PPZZ_ALL()
5098 uint64_t *d = vd, *g = vg; \ in DO_FPCMP_PPZZ_ALL()
5133 void HELPER(sve_ftmad_h)(void *vd, void *vn, void *vm,
5143 float16 *d = vd, *n = vn, *m = vm;
5162 void HELPER(sve_ftmad_s)(void *vd, void *vn, void *vm, in HELPER()
5174 float32 *d = vd, *n = vn, *m = vm; in HELPER()
5193 void HELPER(sve_ftmad_d)(void *vd, void *vn, void *vm, in HELPER()
5209 float64 *d = vd, *n = vn, *m = vm; in HELPER()
5232 void HELPER(sve_fcadd_h)(void *vd, void *vn, void *vm, void *vg, in HELPER()
5261 *(float16 *)(vd + H1_2(i)) = float16_add(e0, e1, s); in HELPER()
5264 *(float16 *)(vd + H1_2(j)) = float16_add(e2, e3, s); in HELPER()
5270 void HELPER(sve_fcadd_s)(void *vd, void *vn, void *vm, void *vg, in HELPER()
5299 *(float32 *)(vd + H1_2(i)) = float32_add(e0, e1, s); in HELPER()
5302 *(float32 *)(vd + H1_2(j)) = float32_add(e2, e3, s); in HELPER()
5308 void HELPER(sve_fcadd_d)(void *vd, void *vn, void *vm, void *vg, in HELPER()
5337 *(float64 *)(vd + H1_2(i)) = float64_add(e0, e1, s); in HELPER()
5340 *(float64 *)(vd + H1_2(j)) = float64_add(e2, e3, s); in HELPER()
5350 void HELPER(sve_fcmla_zpzzz_h)(void *vd, void *vn, void *vm, void *va, in HELPER()
5389 *(float16 *)(vd + H1_2(i)) = d; in HELPER()
5394 *(float16 *)(vd + H1_2(j)) = d; in HELPER()
5400 void HELPER(sve_fcmla_zpzzz_s)(void *vd, void *vn, void *vm, void *va, in HELPER()
5439 *(float32 *)(vd + H1_2(i)) = d; in HELPER()
5444 *(float32 *)(vd + H1_2(j)) = d; in HELPER()
5450 void HELPER(sve_fcmla_zpzzz_d)(void *vd, void *vn, void *vm, void *va, in HELPER()
5489 *(float64 *)(vd + H1_2(i)) = d; in HELPER()
5494 *(float64 *)(vd + H1_2(j)) = d; in HELPER()
6188 void *vd = &env->vfp.zregs[rd]; in sve_ldnfff1_r() local
6198 memset(vd, 0, reg_max); in sve_ldnfff1_r()
6207 memset(vd, 0, reg_max); in sve_ldnfff1_r()
6238 tlb_fn(env, vd, reg_off, addr + mem_off, retaddr); in sve_ldnfff1_r()
6241 swap_memzero(vd, reg_off); in sve_ldnfff1_r()
6244 swap_memzero(vd + reg_off, reg_max - reg_off); in sve_ldnfff1_r()
6250 memset(vd, 0, reg_max); in sve_ldnfff1_r()
6253 memset(vd, 0, reg_max); in sve_ldnfff1_r()
6275 tlb_fn(env, vd, reg_off, addr + mem_off, retaddr); in sve_ldnfff1_r()
6325 host_fn(vd, reg_off, host + mem_off); in sve_ldnfff1_r()
6748 void sve_ld1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm, in sve_ld1_z() argument
6810 memcpy(vd, &scratch, reg_max); in sve_ld1_z()
6814 void sve_ld1_z_mte(CPUARMState *env, void *vd, uint64_t *vg, void *vm, in sve_ld1_z_mte() argument
6830 sve_ld1_z(env, vd, vg, vm, base, desc, retaddr, mtedesc, in sve_ld1_z_mte()
6835 void HELPER(sve_ld##MEM##_##OFS)(CPUARMState *env, void *vd, void *vg, \
6838 sve_ld1_z(env, vd, vg, vm, base, desc, GETPC(), 0, 4, 1 << MSZ, \
6841 void HELPER(sve_ld##MEM##_##OFS##_mte)(CPUARMState *env, void *vd, void *vg, \
6844 sve_ld1_z_mte(env, vd, vg, vm, base, desc, GETPC(), 4, 1 << MSZ, \
6849 void HELPER(sve_ld##MEM##_##OFS)(CPUARMState *env, void *vd, void *vg, \
6852 sve_ld1_z(env, vd, vg, vm, base, desc, GETPC(), 0, 8, 1 << MSZ, \
6855 void HELPER(sve_ld##MEM##_##OFS##_mte)(CPUARMState *env, void *vd, void *vg, \
6858 sve_ld1_z_mte(env, vd, vg, vm, base, desc, GETPC(), 8, 1 << MSZ, \
6936 void sve_ldff1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm, in DO_LD1_ZPZ_S()
6957 memset(vd, 0, reg_max); in DO_LD1_ZPZ_S()
6962 if (unlikely(vd == vm)) { in DO_LD1_ZPZ_S()
6973 tlb_fn(env, vd, reg_off, addr, retaddr); in DO_LD1_ZPZ_S()
6976 swap_memzero(vd, reg_off); in DO_LD1_ZPZ_S()
6978 swap_memzero(vd + reg_off, reg_max - reg_off); in DO_LD1_ZPZ_S()
7010 host_fn(vd, reg_off, info.host); in DO_LD1_ZPZ_S()
7023 void sve_ldff1_z_mte(CPUARMState *env, void *vd, uint64_t *vg, void *vm, in sve_ldff1_z_mte() argument
7040 sve_ldff1_z(env, vd, vg, vm, base, desc, retaddr, mtedesc, in sve_ldff1_z_mte()
7046 (CPUARMState *env, void *vd, void *vg, \
7049 sve_ldff1_z(env, vd, vg, vm, base, desc, GETPC(), 0, MO_32, MSZ, \
7053 (CPUARMState *env, void *vd, void *vg, \
7056 sve_ldff1_z_mte(env, vd, vg, vm, base, desc, GETPC(), MO_32, MSZ, \
7062 (CPUARMState *env, void *vd, void *vg, \
7065 sve_ldff1_z(env, vd, vg, vm, base, desc, GETPC(), 0, MO_64, MSZ, \
7069 (CPUARMState *env, void *vd, void *vg, \
7072 sve_ldff1_z_mte(env, vd, vg, vm, base, desc, GETPC(), MO_64, MSZ, \
7143 void sve_st1_z(CPUARMState *env, void *vd, uint64_t *vg, void *vm, in DO_LDFF1_ZPZ_S()
7216 host_fn(vd, reg_off, h); in DO_LDFF1_ZPZ_S()
7220 tlb_fn(env, vd, reg_off, addr, retaddr); in DO_LDFF1_ZPZ_S()
7228 void sve_st1_z_mte(CPUARMState *env, void *vd, uint64_t *vg, void *vm, in sve_st1_z_mte() argument
7244 sve_st1_z(env, vd, vg, vm, base, desc, retaddr, mtedesc, in sve_st1_z_mte()
7249 void HELPER(sve_st##MEM##_##OFS)(CPUARMState *env, void *vd, void *vg, \
7252 sve_st1_z(env, vd, vg, vm, base, desc, GETPC(), 0, 4, 1 << MSZ, \
7255 void HELPER(sve_st##MEM##_##OFS##_mte)(CPUARMState *env, void *vd, void *vg, \
7258 sve_st1_z_mte(env, vd, vg, vm, base, desc, GETPC(), 4, 1 << MSZ, \
7263 void HELPER(sve_st##MEM##_##OFS)(CPUARMState *env, void *vd, void *vg, \
7266 sve_st1_z(env, vd, vg, vm, base, desc, GETPC(), 0, 8, 1 << MSZ, \
7269 void HELPER(sve_st##MEM##_##OFS##_mte)(CPUARMState *env, void *vd, void *vg, \
7272 sve_st1_z_mte(env, vd, vg, vm, base, desc, GETPC(), 8, 1 << MSZ, \
7315 void HELPER(sve2_eor3)(void *vd, void *vn, void *vm, void *vk, uint32_t desc) in DO_ST1_ZPZ_S()
7318 uint64_t *d = vd, *n = vn, *m = vm, *k = vk; in DO_ST1_ZPZ_S()
7325 void HELPER(sve2_bcax)(void *vd, void *vn, void *vm, void *vk, uint32_t desc) in HELPER()
7328 uint64_t *d = vd, *n = vn, *m = vm, *k = vk; in HELPER()
7335 void HELPER(sve2_bsl1n)(void *vd, void *vn, void *vm, void *vk, uint32_t desc) in HELPER()
7338 uint64_t *d = vd, *n = vn, *m = vm, *k = vk; in HELPER()
7345 void HELPER(sve2_bsl2n)(void *vd, void *vn, void *vm, void *vk, uint32_t desc) in HELPER()
7348 uint64_t *d = vd, *n = vn, *m = vm, *k = vk; in HELPER()
7355 void HELPER(sve2_nbsl)(void *vd, void *vn, void *vm, void *vk, uint32_t desc) in HELPER()
7358 uint64_t *d = vd, *n = vn, *m = vm, *k = vk; in HELPER()
7385 static inline uint32_t do_match(void *vd, void *vn, void *vm, void *vg, in do_match() argument
7409 *(uint16_t *)(vd + H1_2(i >> 3)) = out; in do_match()
7416 uint32_t HELPER(NAME)(void *vd, void *vn, void *vm, void *vg, uint32_t desc) \
7418 return do_match(vd, vn, vm, vg, desc, ESZ, INV); \
7429 void HELPER(sve2_histcnt_s)(void *vd, void *vn, void *vm, void *vg, in DO_PPZZ_MATCH()
7435 uint32_t *d = vd, *n = vn, *m = vm; in DO_PPZZ_MATCH()
7466 void HELPER(sve2_histcnt_d)(void *vd, void *vn, void *vm, void *vg, in HELPER()
7472 uint64_t *d = vd, *n = vn, *m = vm; in HELPER()
7535 void HELPER(sve2_histseg)(void *vd, void *vn, void *vm, uint32_t desc) in HELPER()
7555 *(uint64_t *)(vd + i) = out0; in HELPER()
7556 *(uint64_t *)(vd + i + 8) = out1; in HELPER()
7560 void HELPER(sve2_xar_b)(void *vd, void *vn, void *vm, uint32_t desc) in HELPER()
7566 uint64_t *d = vd, *n = vn, *m = vm; in HELPER()
7574 void HELPER(sve2_xar_h)(void *vd, void *vn, void *vm, uint32_t desc) in HELPER()
7580 uint64_t *d = vd, *n = vn, *m = vm; in HELPER()
7588 void HELPER(sve2_xar_s)(void *vd, void *vn, void *vm, uint32_t desc) in HELPER()
7592 uint32_t *d = vd, *n = vn, *m = vm; in HELPER()
7599 void HELPER(fmmla_s)(void *vd, void *vn, void *vm, void *va, in HELPER()
7608 float32 *d = vd + s * sizeof(float32) * 4; in HELPER()
7637 void HELPER(fmmla_d)(void *vd, void *vn, void *vm, void *va, in HELPER()
7646 float64 *d = vd + s * sizeof(float64) * 4; in HELPER()
7674 void HELPER(NAME)(void *vd, void *vn, void *vg, \
7685 *(TYPEN *)(vd + HN(i + sizeof(TYPEN))) = OP(nn, status); \
7696 void HELPER(NAME)(void *vd, void *vn, void *vg, \
7707 *(TYPEW *)(vd + HW(i)) = OP(nn, status); \