xref: /qemu/target/arm/tcg/translate-mve.c (revision c2d8f6bb28ffc5c9e4e465604ccdf7a08b7e3568)
1 /*
2  *  ARM translation: M-profile MVE instructions
3  *
4  *  Copyright (c) 2021 Linaro, Ltd.
5  *
6  * This library is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * This library is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18  */
19 
20 #include "qemu/osdep.h"
21 #include "tcg/tcg-op.h"
22 #include "tcg/tcg-op-gvec.h"
23 #include "exec/exec-all.h"
24 #include "exec/gen-icount.h"
25 #include "translate.h"
26 #include "translate-a32.h"
27 
28 static inline int vidup_imm(DisasContext *s, int x)
29 {
30     return 1 << x;
31 }
32 
33 /* Include the generated decoder */
34 #include "decode-mve.c.inc"
35 
36 typedef void MVEGenLdStFn(TCGv_ptr, TCGv_ptr, TCGv_i32);
37 typedef void MVEGenLdStSGFn(TCGv_ptr, TCGv_ptr, TCGv_ptr, TCGv_i32);
38 typedef void MVEGenLdStIlFn(TCGv_ptr, TCGv_i32, TCGv_i32);
39 typedef void MVEGenOneOpFn(TCGv_ptr, TCGv_ptr, TCGv_ptr);
40 typedef void MVEGenTwoOpFn(TCGv_ptr, TCGv_ptr, TCGv_ptr, TCGv_ptr);
41 typedef void MVEGenTwoOpScalarFn(TCGv_ptr, TCGv_ptr, TCGv_ptr, TCGv_i32);
42 typedef void MVEGenTwoOpShiftFn(TCGv_ptr, TCGv_ptr, TCGv_ptr, TCGv_i32);
43 typedef void MVEGenLongDualAccOpFn(TCGv_i64, TCGv_ptr, TCGv_ptr, TCGv_ptr, TCGv_i64);
44 typedef void MVEGenVADDVFn(TCGv_i32, TCGv_ptr, TCGv_ptr, TCGv_i32);
45 typedef void MVEGenOneOpImmFn(TCGv_ptr, TCGv_ptr, TCGv_i64);
46 typedef void MVEGenVIDUPFn(TCGv_i32, TCGv_ptr, TCGv_ptr, TCGv_i32, TCGv_i32);
47 typedef void MVEGenVIWDUPFn(TCGv_i32, TCGv_ptr, TCGv_ptr, TCGv_i32, TCGv_i32, TCGv_i32);
48 typedef void MVEGenCmpFn(TCGv_ptr, TCGv_ptr, TCGv_ptr);
49 typedef void MVEGenScalarCmpFn(TCGv_ptr, TCGv_ptr, TCGv_i32);
50 typedef void MVEGenVABAVFn(TCGv_i32, TCGv_ptr, TCGv_ptr, TCGv_ptr, TCGv_i32);
51 typedef void MVEGenDualAccOpFn(TCGv_i32, TCGv_ptr, TCGv_ptr, TCGv_ptr, TCGv_i32);
52 
53 /* Return the offset of a Qn register (same semantics as aa32_vfp_qreg()) */
54 static inline long mve_qreg_offset(unsigned reg)
55 {
56     return offsetof(CPUARMState, vfp.zregs[reg].d[0]);
57 }
58 
59 static TCGv_ptr mve_qreg_ptr(unsigned reg)
60 {
61     TCGv_ptr ret = tcg_temp_new_ptr();
62     tcg_gen_addi_ptr(ret, cpu_env, mve_qreg_offset(reg));
63     return ret;
64 }
65 
66 static bool mve_check_qreg_bank(DisasContext *s, int qmask)
67 {
68     /*
69      * Check whether Qregs are in range. For v8.1M only Q0..Q7
70      * are supported, see VFPSmallRegisterBank().
71      */
72     return qmask < 8;
73 }
74 
75 bool mve_eci_check(DisasContext *s)
76 {
77     /*
78      * This is a beatwise insn: check that ECI is valid (not a
79      * reserved value) and note that we are handling it.
80      * Return true if OK, false if we generated an exception.
81      */
82     s->eci_handled = true;
83     switch (s->eci) {
84     case ECI_NONE:
85     case ECI_A0:
86     case ECI_A0A1:
87     case ECI_A0A1A2:
88     case ECI_A0A1A2B0:
89         return true;
90     default:
91         /* Reserved value: INVSTATE UsageFault */
92         gen_exception_insn(s, s->pc_curr, EXCP_INVSTATE, syn_uncategorized(),
93                            default_exception_el(s));
94         return false;
95     }
96 }
97 
98 void mve_update_eci(DisasContext *s)
99 {
100     /*
101      * The helper function will always update the CPUState field,
102      * so we only need to update the DisasContext field.
103      */
104     if (s->eci) {
105         s->eci = (s->eci == ECI_A0A1A2B0) ? ECI_A0 : ECI_NONE;
106     }
107 }
108 
109 void mve_update_and_store_eci(DisasContext *s)
110 {
111     /*
112      * For insns which don't call a helper function that will call
113      * mve_advance_vpt(), this version updates s->eci and also stores
114      * it out to the CPUState field.
115      */
116     if (s->eci) {
117         mve_update_eci(s);
118         store_cpu_field(tcg_constant_i32(s->eci << 4), condexec_bits);
119     }
120 }
121 
122 static bool mve_skip_first_beat(DisasContext *s)
123 {
124     /* Return true if PSR.ECI says we must skip the first beat of this insn */
125     switch (s->eci) {
126     case ECI_NONE:
127         return false;
128     case ECI_A0:
129     case ECI_A0A1:
130     case ECI_A0A1A2:
131     case ECI_A0A1A2B0:
132         return true;
133     default:
134         g_assert_not_reached();
135     }
136 }
137 
138 static bool do_ldst(DisasContext *s, arg_VLDR_VSTR *a, MVEGenLdStFn *fn,
139                     unsigned msize)
140 {
141     TCGv_i32 addr;
142     uint32_t offset;
143     TCGv_ptr qreg;
144 
145     if (!dc_isar_feature(aa32_mve, s) ||
146         !mve_check_qreg_bank(s, a->qd) ||
147         !fn) {
148         return false;
149     }
150 
151     /* CONSTRAINED UNPREDICTABLE: we choose to UNDEF */
152     if (a->rn == 15 || (a->rn == 13 && a->w)) {
153         return false;
154     }
155 
156     if (!mve_eci_check(s) || !vfp_access_check(s)) {
157         return true;
158     }
159 
160     offset = a->imm << msize;
161     if (!a->a) {
162         offset = -offset;
163     }
164     addr = load_reg(s, a->rn);
165     if (a->p) {
166         tcg_gen_addi_i32(addr, addr, offset);
167     }
168 
169     qreg = mve_qreg_ptr(a->qd);
170     fn(cpu_env, qreg, addr);
171     tcg_temp_free_ptr(qreg);
172 
173     /*
174      * Writeback always happens after the last beat of the insn,
175      * regardless of predication
176      */
177     if (a->w) {
178         if (!a->p) {
179             tcg_gen_addi_i32(addr, addr, offset);
180         }
181         store_reg(s, a->rn, addr);
182     } else {
183         tcg_temp_free_i32(addr);
184     }
185     mve_update_eci(s);
186     return true;
187 }
188 
189 static bool trans_VLDR_VSTR(DisasContext *s, arg_VLDR_VSTR *a)
190 {
191     static MVEGenLdStFn * const ldstfns[4][2] = {
192         { gen_helper_mve_vstrb, gen_helper_mve_vldrb },
193         { gen_helper_mve_vstrh, gen_helper_mve_vldrh },
194         { gen_helper_mve_vstrw, gen_helper_mve_vldrw },
195         { NULL, NULL }
196     };
197     return do_ldst(s, a, ldstfns[a->size][a->l], a->size);
198 }
199 
200 #define DO_VLDST_WIDE_NARROW(OP, SLD, ULD, ST, MSIZE)           \
201     static bool trans_##OP(DisasContext *s, arg_VLDR_VSTR *a)   \
202     {                                                           \
203         static MVEGenLdStFn * const ldstfns[2][2] = {           \
204             { gen_helper_mve_##ST, gen_helper_mve_##SLD },      \
205             { NULL, gen_helper_mve_##ULD },                     \
206         };                                                      \
207         return do_ldst(s, a, ldstfns[a->u][a->l], MSIZE);       \
208     }
209 
210 DO_VLDST_WIDE_NARROW(VLDSTB_H, vldrb_sh, vldrb_uh, vstrb_h, MO_8)
211 DO_VLDST_WIDE_NARROW(VLDSTB_W, vldrb_sw, vldrb_uw, vstrb_w, MO_8)
212 DO_VLDST_WIDE_NARROW(VLDSTH_W, vldrh_sw, vldrh_uw, vstrh_w, MO_16)
213 
214 static bool do_ldst_sg(DisasContext *s, arg_vldst_sg *a, MVEGenLdStSGFn fn)
215 {
216     TCGv_i32 addr;
217     TCGv_ptr qd, qm;
218 
219     if (!dc_isar_feature(aa32_mve, s) ||
220         !mve_check_qreg_bank(s, a->qd | a->qm) ||
221         !fn || a->rn == 15) {
222         /* Rn case is UNPREDICTABLE */
223         return false;
224     }
225 
226     if (!mve_eci_check(s) || !vfp_access_check(s)) {
227         return true;
228     }
229 
230     addr = load_reg(s, a->rn);
231 
232     qd = mve_qreg_ptr(a->qd);
233     qm = mve_qreg_ptr(a->qm);
234     fn(cpu_env, qd, qm, addr);
235     tcg_temp_free_ptr(qd);
236     tcg_temp_free_ptr(qm);
237     tcg_temp_free_i32(addr);
238     mve_update_eci(s);
239     return true;
240 }
241 
242 /*
243  * The naming scheme here is "vldrb_sg_sh == in-memory byte loads
244  * signextended to halfword elements in register". _os_ indicates that
245  * the offsets in Qm should be scaled by the element size.
246  */
247 /* This macro is just to make the arrays more compact in these functions */
248 #define F(N) gen_helper_mve_##N
249 
250 /* VLDRB/VSTRB (ie msize 1) with OS=1 is UNPREDICTABLE; we UNDEF */
251 static bool trans_VLDR_S_sg(DisasContext *s, arg_vldst_sg *a)
252 {
253     static MVEGenLdStSGFn * const fns[2][4][4] = { {
254             { NULL, F(vldrb_sg_sh), F(vldrb_sg_sw), NULL },
255             { NULL, NULL,           F(vldrh_sg_sw), NULL },
256             { NULL, NULL,           NULL,           NULL },
257             { NULL, NULL,           NULL,           NULL }
258         }, {
259             { NULL, NULL,              NULL,              NULL },
260             { NULL, NULL,              F(vldrh_sg_os_sw), NULL },
261             { NULL, NULL,              NULL,              NULL },
262             { NULL, NULL,              NULL,              NULL }
263         }
264     };
265     if (a->qd == a->qm) {
266         return false; /* UNPREDICTABLE */
267     }
268     return do_ldst_sg(s, a, fns[a->os][a->msize][a->size]);
269 }
270 
271 static bool trans_VLDR_U_sg(DisasContext *s, arg_vldst_sg *a)
272 {
273     static MVEGenLdStSGFn * const fns[2][4][4] = { {
274             { F(vldrb_sg_ub), F(vldrb_sg_uh), F(vldrb_sg_uw), NULL },
275             { NULL,           F(vldrh_sg_uh), F(vldrh_sg_uw), NULL },
276             { NULL,           NULL,           F(vldrw_sg_uw), NULL },
277             { NULL,           NULL,           NULL,           F(vldrd_sg_ud) }
278         }, {
279             { NULL, NULL,              NULL,              NULL },
280             { NULL, F(vldrh_sg_os_uh), F(vldrh_sg_os_uw), NULL },
281             { NULL, NULL,              F(vldrw_sg_os_uw), NULL },
282             { NULL, NULL,              NULL,              F(vldrd_sg_os_ud) }
283         }
284     };
285     if (a->qd == a->qm) {
286         return false; /* UNPREDICTABLE */
287     }
288     return do_ldst_sg(s, a, fns[a->os][a->msize][a->size]);
289 }
290 
291 static bool trans_VSTR_sg(DisasContext *s, arg_vldst_sg *a)
292 {
293     static MVEGenLdStSGFn * const fns[2][4][4] = { {
294             { F(vstrb_sg_ub), F(vstrb_sg_uh), F(vstrb_sg_uw), NULL },
295             { NULL,           F(vstrh_sg_uh), F(vstrh_sg_uw), NULL },
296             { NULL,           NULL,           F(vstrw_sg_uw), NULL },
297             { NULL,           NULL,           NULL,           F(vstrd_sg_ud) }
298         }, {
299             { NULL, NULL,              NULL,              NULL },
300             { NULL, F(vstrh_sg_os_uh), F(vstrh_sg_os_uw), NULL },
301             { NULL, NULL,              F(vstrw_sg_os_uw), NULL },
302             { NULL, NULL,              NULL,              F(vstrd_sg_os_ud) }
303         }
304     };
305     return do_ldst_sg(s, a, fns[a->os][a->msize][a->size]);
306 }
307 
308 #undef F
309 
310 static bool do_ldst_sg_imm(DisasContext *s, arg_vldst_sg_imm *a,
311                            MVEGenLdStSGFn *fn, unsigned msize)
312 {
313     uint32_t offset;
314     TCGv_ptr qd, qm;
315 
316     if (!dc_isar_feature(aa32_mve, s) ||
317         !mve_check_qreg_bank(s, a->qd | a->qm) ||
318         !fn) {
319         return false;
320     }
321 
322     if (!mve_eci_check(s) || !vfp_access_check(s)) {
323         return true;
324     }
325 
326     offset = a->imm << msize;
327     if (!a->a) {
328         offset = -offset;
329     }
330 
331     qd = mve_qreg_ptr(a->qd);
332     qm = mve_qreg_ptr(a->qm);
333     fn(cpu_env, qd, qm, tcg_constant_i32(offset));
334     tcg_temp_free_ptr(qd);
335     tcg_temp_free_ptr(qm);
336     mve_update_eci(s);
337     return true;
338 }
339 
340 static bool trans_VLDRW_sg_imm(DisasContext *s, arg_vldst_sg_imm *a)
341 {
342     static MVEGenLdStSGFn * const fns[] = {
343         gen_helper_mve_vldrw_sg_uw,
344         gen_helper_mve_vldrw_sg_wb_uw,
345     };
346     if (a->qd == a->qm) {
347         return false; /* UNPREDICTABLE */
348     }
349     return do_ldst_sg_imm(s, a, fns[a->w], MO_32);
350 }
351 
352 static bool trans_VLDRD_sg_imm(DisasContext *s, arg_vldst_sg_imm *a)
353 {
354     static MVEGenLdStSGFn * const fns[] = {
355         gen_helper_mve_vldrd_sg_ud,
356         gen_helper_mve_vldrd_sg_wb_ud,
357     };
358     if (a->qd == a->qm) {
359         return false; /* UNPREDICTABLE */
360     }
361     return do_ldst_sg_imm(s, a, fns[a->w], MO_64);
362 }
363 
364 static bool trans_VSTRW_sg_imm(DisasContext *s, arg_vldst_sg_imm *a)
365 {
366     static MVEGenLdStSGFn * const fns[] = {
367         gen_helper_mve_vstrw_sg_uw,
368         gen_helper_mve_vstrw_sg_wb_uw,
369     };
370     return do_ldst_sg_imm(s, a, fns[a->w], MO_32);
371 }
372 
373 static bool trans_VSTRD_sg_imm(DisasContext *s, arg_vldst_sg_imm *a)
374 {
375     static MVEGenLdStSGFn * const fns[] = {
376         gen_helper_mve_vstrd_sg_ud,
377         gen_helper_mve_vstrd_sg_wb_ud,
378     };
379     return do_ldst_sg_imm(s, a, fns[a->w], MO_64);
380 }
381 
382 static bool do_vldst_il(DisasContext *s, arg_vldst_il *a, MVEGenLdStIlFn *fn,
383                         int addrinc)
384 {
385     TCGv_i32 rn;
386 
387     if (!dc_isar_feature(aa32_mve, s) ||
388         !mve_check_qreg_bank(s, a->qd) ||
389         !fn || (a->rn == 13 && a->w) || a->rn == 15) {
390         /* Variously UNPREDICTABLE or UNDEF or related-encoding */
391         return false;
392     }
393     if (!mve_eci_check(s) || !vfp_access_check(s)) {
394         return true;
395     }
396 
397     rn = load_reg(s, a->rn);
398     /*
399      * We pass the index of Qd, not a pointer, because the helper must
400      * access multiple Q registers starting at Qd and working up.
401      */
402     fn(cpu_env, tcg_constant_i32(a->qd), rn);
403 
404     if (a->w) {
405         tcg_gen_addi_i32(rn, rn, addrinc);
406         store_reg(s, a->rn, rn);
407     } else {
408         tcg_temp_free_i32(rn);
409     }
410     mve_update_and_store_eci(s);
411     return true;
412 }
413 
414 /* This macro is just to make the arrays more compact in these functions */
415 #define F(N) gen_helper_mve_##N
416 
417 static bool trans_VLD2(DisasContext *s, arg_vldst_il *a)
418 {
419     static MVEGenLdStIlFn * const fns[4][4] = {
420         { F(vld20b), F(vld20h), F(vld20w), NULL, },
421         { F(vld21b), F(vld21h), F(vld21w), NULL, },
422         { NULL, NULL, NULL, NULL },
423         { NULL, NULL, NULL, NULL },
424     };
425     if (a->qd > 6) {
426         return false;
427     }
428     return do_vldst_il(s, a, fns[a->pat][a->size], 32);
429 }
430 
431 static bool trans_VLD4(DisasContext *s, arg_vldst_il *a)
432 {
433     static MVEGenLdStIlFn * const fns[4][4] = {
434         { F(vld40b), F(vld40h), F(vld40w), NULL, },
435         { F(vld41b), F(vld41h), F(vld41w), NULL, },
436         { F(vld42b), F(vld42h), F(vld42w), NULL, },
437         { F(vld43b), F(vld43h), F(vld43w), NULL, },
438     };
439     if (a->qd > 4) {
440         return false;
441     }
442     return do_vldst_il(s, a, fns[a->pat][a->size], 64);
443 }
444 
445 static bool trans_VST2(DisasContext *s, arg_vldst_il *a)
446 {
447     static MVEGenLdStIlFn * const fns[4][4] = {
448         { F(vst20b), F(vst20h), F(vst20w), NULL, },
449         { F(vst21b), F(vst21h), F(vst21w), NULL, },
450         { NULL, NULL, NULL, NULL },
451         { NULL, NULL, NULL, NULL },
452     };
453     if (a->qd > 6) {
454         return false;
455     }
456     return do_vldst_il(s, a, fns[a->pat][a->size], 32);
457 }
458 
459 static bool trans_VST4(DisasContext *s, arg_vldst_il *a)
460 {
461     static MVEGenLdStIlFn * const fns[4][4] = {
462         { F(vst40b), F(vst40h), F(vst40w), NULL, },
463         { F(vst41b), F(vst41h), F(vst41w), NULL, },
464         { F(vst42b), F(vst42h), F(vst42w), NULL, },
465         { F(vst43b), F(vst43h), F(vst43w), NULL, },
466     };
467     if (a->qd > 4) {
468         return false;
469     }
470     return do_vldst_il(s, a, fns[a->pat][a->size], 64);
471 }
472 
473 #undef F
474 
475 static bool trans_VDUP(DisasContext *s, arg_VDUP *a)
476 {
477     TCGv_ptr qd;
478     TCGv_i32 rt;
479 
480     if (!dc_isar_feature(aa32_mve, s) ||
481         !mve_check_qreg_bank(s, a->qd)) {
482         return false;
483     }
484     if (a->rt == 13 || a->rt == 15) {
485         /* UNPREDICTABLE; we choose to UNDEF */
486         return false;
487     }
488     if (!mve_eci_check(s) || !vfp_access_check(s)) {
489         return true;
490     }
491 
492     qd = mve_qreg_ptr(a->qd);
493     rt = load_reg(s, a->rt);
494     tcg_gen_dup_i32(a->size, rt, rt);
495     gen_helper_mve_vdup(cpu_env, qd, rt);
496     tcg_temp_free_ptr(qd);
497     tcg_temp_free_i32(rt);
498     mve_update_eci(s);
499     return true;
500 }
501 
502 static bool do_1op(DisasContext *s, arg_1op *a, MVEGenOneOpFn fn)
503 {
504     TCGv_ptr qd, qm;
505 
506     if (!dc_isar_feature(aa32_mve, s) ||
507         !mve_check_qreg_bank(s, a->qd | a->qm) ||
508         !fn) {
509         return false;
510     }
511 
512     if (!mve_eci_check(s) || !vfp_access_check(s)) {
513         return true;
514     }
515 
516     qd = mve_qreg_ptr(a->qd);
517     qm = mve_qreg_ptr(a->qm);
518     fn(cpu_env, qd, qm);
519     tcg_temp_free_ptr(qd);
520     tcg_temp_free_ptr(qm);
521     mve_update_eci(s);
522     return true;
523 }
524 
525 #define DO_1OP(INSN, FN)                                        \
526     static bool trans_##INSN(DisasContext *s, arg_1op *a)       \
527     {                                                           \
528         static MVEGenOneOpFn * const fns[] = {                  \
529             gen_helper_mve_##FN##b,                             \
530             gen_helper_mve_##FN##h,                             \
531             gen_helper_mve_##FN##w,                             \
532             NULL,                                               \
533         };                                                      \
534         return do_1op(s, a, fns[a->size]);                      \
535     }
536 
537 DO_1OP(VCLZ, vclz)
538 DO_1OP(VCLS, vcls)
539 DO_1OP(VABS, vabs)
540 DO_1OP(VNEG, vneg)
541 DO_1OP(VQABS, vqabs)
542 DO_1OP(VQNEG, vqneg)
543 DO_1OP(VMAXA, vmaxa)
544 DO_1OP(VMINA, vmina)
545 
546 /* Narrowing moves: only size 0 and 1 are valid */
547 #define DO_VMOVN(INSN, FN) \
548     static bool trans_##INSN(DisasContext *s, arg_1op *a)       \
549     {                                                           \
550         static MVEGenOneOpFn * const fns[] = {                  \
551             gen_helper_mve_##FN##b,                             \
552             gen_helper_mve_##FN##h,                             \
553             NULL,                                               \
554             NULL,                                               \
555         };                                                      \
556         return do_1op(s, a, fns[a->size]);                      \
557     }
558 
559 DO_VMOVN(VMOVNB, vmovnb)
560 DO_VMOVN(VMOVNT, vmovnt)
561 DO_VMOVN(VQMOVUNB, vqmovunb)
562 DO_VMOVN(VQMOVUNT, vqmovunt)
563 DO_VMOVN(VQMOVN_BS, vqmovnbs)
564 DO_VMOVN(VQMOVN_TS, vqmovnts)
565 DO_VMOVN(VQMOVN_BU, vqmovnbu)
566 DO_VMOVN(VQMOVN_TU, vqmovntu)
567 
568 static bool trans_VREV16(DisasContext *s, arg_1op *a)
569 {
570     static MVEGenOneOpFn * const fns[] = {
571         gen_helper_mve_vrev16b,
572         NULL,
573         NULL,
574         NULL,
575     };
576     return do_1op(s, a, fns[a->size]);
577 }
578 
579 static bool trans_VREV32(DisasContext *s, arg_1op *a)
580 {
581     static MVEGenOneOpFn * const fns[] = {
582         gen_helper_mve_vrev32b,
583         gen_helper_mve_vrev32h,
584         NULL,
585         NULL,
586     };
587     return do_1op(s, a, fns[a->size]);
588 }
589 
590 static bool trans_VREV64(DisasContext *s, arg_1op *a)
591 {
592     static MVEGenOneOpFn * const fns[] = {
593         gen_helper_mve_vrev64b,
594         gen_helper_mve_vrev64h,
595         gen_helper_mve_vrev64w,
596         NULL,
597     };
598     return do_1op(s, a, fns[a->size]);
599 }
600 
601 static bool trans_VMVN(DisasContext *s, arg_1op *a)
602 {
603     return do_1op(s, a, gen_helper_mve_vmvn);
604 }
605 
606 static bool trans_VABS_fp(DisasContext *s, arg_1op *a)
607 {
608     static MVEGenOneOpFn * const fns[] = {
609         NULL,
610         gen_helper_mve_vfabsh,
611         gen_helper_mve_vfabss,
612         NULL,
613     };
614     if (!dc_isar_feature(aa32_mve_fp, s)) {
615         return false;
616     }
617     return do_1op(s, a, fns[a->size]);
618 }
619 
620 static bool trans_VNEG_fp(DisasContext *s, arg_1op *a)
621 {
622     static MVEGenOneOpFn * const fns[] = {
623         NULL,
624         gen_helper_mve_vfnegh,
625         gen_helper_mve_vfnegs,
626         NULL,
627     };
628     if (!dc_isar_feature(aa32_mve_fp, s)) {
629         return false;
630     }
631     return do_1op(s, a, fns[a->size]);
632 }
633 
634 static bool do_2op(DisasContext *s, arg_2op *a, MVEGenTwoOpFn fn)
635 {
636     TCGv_ptr qd, qn, qm;
637 
638     if (!dc_isar_feature(aa32_mve, s) ||
639         !mve_check_qreg_bank(s, a->qd | a->qn | a->qm) ||
640         !fn) {
641         return false;
642     }
643     if (!mve_eci_check(s) || !vfp_access_check(s)) {
644         return true;
645     }
646 
647     qd = mve_qreg_ptr(a->qd);
648     qn = mve_qreg_ptr(a->qn);
649     qm = mve_qreg_ptr(a->qm);
650     fn(cpu_env, qd, qn, qm);
651     tcg_temp_free_ptr(qd);
652     tcg_temp_free_ptr(qn);
653     tcg_temp_free_ptr(qm);
654     mve_update_eci(s);
655     return true;
656 }
657 
658 #define DO_LOGIC(INSN, HELPER)                                  \
659     static bool trans_##INSN(DisasContext *s, arg_2op *a)       \
660     {                                                           \
661         return do_2op(s, a, HELPER);                            \
662     }
663 
664 DO_LOGIC(VAND, gen_helper_mve_vand)
665 DO_LOGIC(VBIC, gen_helper_mve_vbic)
666 DO_LOGIC(VORR, gen_helper_mve_vorr)
667 DO_LOGIC(VORN, gen_helper_mve_vorn)
668 DO_LOGIC(VEOR, gen_helper_mve_veor)
669 
670 DO_LOGIC(VPSEL, gen_helper_mve_vpsel)
671 
672 #define DO_2OP(INSN, FN) \
673     static bool trans_##INSN(DisasContext *s, arg_2op *a)       \
674     {                                                           \
675         static MVEGenTwoOpFn * const fns[] = {                  \
676             gen_helper_mve_##FN##b,                             \
677             gen_helper_mve_##FN##h,                             \
678             gen_helper_mve_##FN##w,                             \
679             NULL,                                               \
680         };                                                      \
681         return do_2op(s, a, fns[a->size]);                      \
682     }
683 
684 DO_2OP(VADD, vadd)
685 DO_2OP(VSUB, vsub)
686 DO_2OP(VMUL, vmul)
687 DO_2OP(VMULH_S, vmulhs)
688 DO_2OP(VMULH_U, vmulhu)
689 DO_2OP(VRMULH_S, vrmulhs)
690 DO_2OP(VRMULH_U, vrmulhu)
691 DO_2OP(VMAX_S, vmaxs)
692 DO_2OP(VMAX_U, vmaxu)
693 DO_2OP(VMIN_S, vmins)
694 DO_2OP(VMIN_U, vminu)
695 DO_2OP(VABD_S, vabds)
696 DO_2OP(VABD_U, vabdu)
697 DO_2OP(VHADD_S, vhadds)
698 DO_2OP(VHADD_U, vhaddu)
699 DO_2OP(VHSUB_S, vhsubs)
700 DO_2OP(VHSUB_U, vhsubu)
701 DO_2OP(VMULL_BS, vmullbs)
702 DO_2OP(VMULL_BU, vmullbu)
703 DO_2OP(VMULL_TS, vmullts)
704 DO_2OP(VMULL_TU, vmulltu)
705 DO_2OP(VQDMULH, vqdmulh)
706 DO_2OP(VQRDMULH, vqrdmulh)
707 DO_2OP(VQADD_S, vqadds)
708 DO_2OP(VQADD_U, vqaddu)
709 DO_2OP(VQSUB_S, vqsubs)
710 DO_2OP(VQSUB_U, vqsubu)
711 DO_2OP(VSHL_S, vshls)
712 DO_2OP(VSHL_U, vshlu)
713 DO_2OP(VRSHL_S, vrshls)
714 DO_2OP(VRSHL_U, vrshlu)
715 DO_2OP(VQSHL_S, vqshls)
716 DO_2OP(VQSHL_U, vqshlu)
717 DO_2OP(VQRSHL_S, vqrshls)
718 DO_2OP(VQRSHL_U, vqrshlu)
719 DO_2OP(VQDMLADH, vqdmladh)
720 DO_2OP(VQDMLADHX, vqdmladhx)
721 DO_2OP(VQRDMLADH, vqrdmladh)
722 DO_2OP(VQRDMLADHX, vqrdmladhx)
723 DO_2OP(VQDMLSDH, vqdmlsdh)
724 DO_2OP(VQDMLSDHX, vqdmlsdhx)
725 DO_2OP(VQRDMLSDH, vqrdmlsdh)
726 DO_2OP(VQRDMLSDHX, vqrdmlsdhx)
727 DO_2OP(VRHADD_S, vrhadds)
728 DO_2OP(VRHADD_U, vrhaddu)
729 /*
730  * VCADD Qd == Qm at size MO_32 is UNPREDICTABLE; we choose not to diagnose
731  * so we can reuse the DO_2OP macro. (Our implementation calculates the
732  * "expected" results in this case.) Similarly for VHCADD.
733  */
734 DO_2OP(VCADD90, vcadd90)
735 DO_2OP(VCADD270, vcadd270)
736 DO_2OP(VHCADD90, vhcadd90)
737 DO_2OP(VHCADD270, vhcadd270)
738 
739 static bool trans_VQDMULLB(DisasContext *s, arg_2op *a)
740 {
741     static MVEGenTwoOpFn * const fns[] = {
742         NULL,
743         gen_helper_mve_vqdmullbh,
744         gen_helper_mve_vqdmullbw,
745         NULL,
746     };
747     if (a->size == MO_32 && (a->qd == a->qm || a->qd == a->qn)) {
748         /* UNPREDICTABLE; we choose to undef */
749         return false;
750     }
751     return do_2op(s, a, fns[a->size]);
752 }
753 
754 static bool trans_VQDMULLT(DisasContext *s, arg_2op *a)
755 {
756     static MVEGenTwoOpFn * const fns[] = {
757         NULL,
758         gen_helper_mve_vqdmullth,
759         gen_helper_mve_vqdmulltw,
760         NULL,
761     };
762     if (a->size == MO_32 && (a->qd == a->qm || a->qd == a->qn)) {
763         /* UNPREDICTABLE; we choose to undef */
764         return false;
765     }
766     return do_2op(s, a, fns[a->size]);
767 }
768 
769 static bool trans_VMULLP_B(DisasContext *s, arg_2op *a)
770 {
771     /*
772      * Note that a->size indicates the output size, ie VMULL.P8
773      * is the 8x8->16 operation and a->size is MO_16; VMULL.P16
774      * is the 16x16->32 operation and a->size is MO_32.
775      */
776     static MVEGenTwoOpFn * const fns[] = {
777         NULL,
778         gen_helper_mve_vmullpbh,
779         gen_helper_mve_vmullpbw,
780         NULL,
781     };
782     return do_2op(s, a, fns[a->size]);
783 }
784 
785 static bool trans_VMULLP_T(DisasContext *s, arg_2op *a)
786 {
787     /* a->size is as for trans_VMULLP_B */
788     static MVEGenTwoOpFn * const fns[] = {
789         NULL,
790         gen_helper_mve_vmullpth,
791         gen_helper_mve_vmullptw,
792         NULL,
793     };
794     return do_2op(s, a, fns[a->size]);
795 }
796 
797 /*
798  * VADC and VSBC: these perform an add-with-carry or subtract-with-carry
799  * of the 32-bit elements in each lane of the input vectors, where the
800  * carry-out of each add is the carry-in of the next.  The initial carry
801  * input is either fixed (0 for VADCI, 1 for VSBCI) or is from FPSCR.C
802  * (for VADC and VSBC); the carry out at the end is written back to FPSCR.C.
803  * These insns are subject to beat-wise execution.  Partial execution
804  * of an I=1 (initial carry input fixed) insn which does not
805  * execute the first beat must start with the current FPSCR.NZCV
806  * value, not the fixed constant input.
807  */
808 static bool trans_VADC(DisasContext *s, arg_2op *a)
809 {
810     return do_2op(s, a, gen_helper_mve_vadc);
811 }
812 
813 static bool trans_VADCI(DisasContext *s, arg_2op *a)
814 {
815     if (mve_skip_first_beat(s)) {
816         return trans_VADC(s, a);
817     }
818     return do_2op(s, a, gen_helper_mve_vadci);
819 }
820 
821 static bool trans_VSBC(DisasContext *s, arg_2op *a)
822 {
823     return do_2op(s, a, gen_helper_mve_vsbc);
824 }
825 
826 static bool trans_VSBCI(DisasContext *s, arg_2op *a)
827 {
828     if (mve_skip_first_beat(s)) {
829         return trans_VSBC(s, a);
830     }
831     return do_2op(s, a, gen_helper_mve_vsbci);
832 }
833 
834 #define DO_2OP_FP(INSN, FN)                                     \
835     static bool trans_##INSN(DisasContext *s, arg_2op *a)       \
836     {                                                           \
837         static MVEGenTwoOpFn * const fns[] = {                  \
838             NULL,                                               \
839             gen_helper_mve_##FN##h,                             \
840             gen_helper_mve_##FN##s,                             \
841             NULL,                                               \
842         };                                                      \
843         if (!dc_isar_feature(aa32_mve_fp, s)) {                 \
844             return false;                                       \
845         }                                                       \
846         return do_2op(s, a, fns[a->size]);                      \
847     }
848 
849 DO_2OP_FP(VADD_fp, vfadd)
850 DO_2OP_FP(VSUB_fp, vfsub)
851 DO_2OP_FP(VMUL_fp, vfmul)
852 DO_2OP_FP(VABD_fp, vfabd)
853 DO_2OP_FP(VMAXNM, vmaxnm)
854 DO_2OP_FP(VMINNM, vminnm)
855 DO_2OP_FP(VCADD90_fp, vfcadd90)
856 DO_2OP_FP(VCADD270_fp, vfcadd270)
857 DO_2OP_FP(VFMA, vfma)
858 DO_2OP_FP(VFMS, vfms)
859 DO_2OP_FP(VCMUL0, vcmul0)
860 DO_2OP_FP(VCMUL90, vcmul90)
861 DO_2OP_FP(VCMUL180, vcmul180)
862 DO_2OP_FP(VCMUL270, vcmul270)
863 DO_2OP_FP(VCMLA0, vcmla0)
864 DO_2OP_FP(VCMLA90, vcmla90)
865 DO_2OP_FP(VCMLA180, vcmla180)
866 DO_2OP_FP(VCMLA270, vcmla270)
867 DO_2OP_FP(VMAXNMA, vmaxnma)
868 DO_2OP_FP(VMINNMA, vminnma)
869 
870 static bool do_2op_scalar(DisasContext *s, arg_2scalar *a,
871                           MVEGenTwoOpScalarFn fn)
872 {
873     TCGv_ptr qd, qn;
874     TCGv_i32 rm;
875 
876     if (!dc_isar_feature(aa32_mve, s) ||
877         !mve_check_qreg_bank(s, a->qd | a->qn) ||
878         !fn) {
879         return false;
880     }
881     if (a->rm == 13 || a->rm == 15) {
882         /* UNPREDICTABLE */
883         return false;
884     }
885     if (!mve_eci_check(s) || !vfp_access_check(s)) {
886         return true;
887     }
888 
889     qd = mve_qreg_ptr(a->qd);
890     qn = mve_qreg_ptr(a->qn);
891     rm = load_reg(s, a->rm);
892     fn(cpu_env, qd, qn, rm);
893     tcg_temp_free_i32(rm);
894     tcg_temp_free_ptr(qd);
895     tcg_temp_free_ptr(qn);
896     mve_update_eci(s);
897     return true;
898 }
899 
900 #define DO_2OP_SCALAR(INSN, FN)                                 \
901     static bool trans_##INSN(DisasContext *s, arg_2scalar *a)   \
902     {                                                           \
903         static MVEGenTwoOpScalarFn * const fns[] = {            \
904             gen_helper_mve_##FN##b,                             \
905             gen_helper_mve_##FN##h,                             \
906             gen_helper_mve_##FN##w,                             \
907             NULL,                                               \
908         };                                                      \
909         return do_2op_scalar(s, a, fns[a->size]);               \
910     }
911 
912 DO_2OP_SCALAR(VADD_scalar, vadd_scalar)
913 DO_2OP_SCALAR(VSUB_scalar, vsub_scalar)
914 DO_2OP_SCALAR(VMUL_scalar, vmul_scalar)
915 DO_2OP_SCALAR(VHADD_S_scalar, vhadds_scalar)
916 DO_2OP_SCALAR(VHADD_U_scalar, vhaddu_scalar)
917 DO_2OP_SCALAR(VHSUB_S_scalar, vhsubs_scalar)
918 DO_2OP_SCALAR(VHSUB_U_scalar, vhsubu_scalar)
919 DO_2OP_SCALAR(VQADD_S_scalar, vqadds_scalar)
920 DO_2OP_SCALAR(VQADD_U_scalar, vqaddu_scalar)
921 DO_2OP_SCALAR(VQSUB_S_scalar, vqsubs_scalar)
922 DO_2OP_SCALAR(VQSUB_U_scalar, vqsubu_scalar)
923 DO_2OP_SCALAR(VQDMULH_scalar, vqdmulh_scalar)
924 DO_2OP_SCALAR(VQRDMULH_scalar, vqrdmulh_scalar)
925 DO_2OP_SCALAR(VBRSR, vbrsr)
926 DO_2OP_SCALAR(VMLA, vmla)
927 DO_2OP_SCALAR(VMLAS, vmlas)
928 DO_2OP_SCALAR(VQDMLAH, vqdmlah)
929 DO_2OP_SCALAR(VQRDMLAH, vqrdmlah)
930 DO_2OP_SCALAR(VQDMLASH, vqdmlash)
931 DO_2OP_SCALAR(VQRDMLASH, vqrdmlash)
932 
933 static bool trans_VQDMULLB_scalar(DisasContext *s, arg_2scalar *a)
934 {
935     static MVEGenTwoOpScalarFn * const fns[] = {
936         NULL,
937         gen_helper_mve_vqdmullb_scalarh,
938         gen_helper_mve_vqdmullb_scalarw,
939         NULL,
940     };
941     if (a->qd == a->qn && a->size == MO_32) {
942         /* UNPREDICTABLE; we choose to undef */
943         return false;
944     }
945     return do_2op_scalar(s, a, fns[a->size]);
946 }
947 
948 static bool trans_VQDMULLT_scalar(DisasContext *s, arg_2scalar *a)
949 {
950     static MVEGenTwoOpScalarFn * const fns[] = {
951         NULL,
952         gen_helper_mve_vqdmullt_scalarh,
953         gen_helper_mve_vqdmullt_scalarw,
954         NULL,
955     };
956     if (a->qd == a->qn && a->size == MO_32) {
957         /* UNPREDICTABLE; we choose to undef */
958         return false;
959     }
960     return do_2op_scalar(s, a, fns[a->size]);
961 }
962 
963 
964 #define DO_2OP_FP_SCALAR(INSN, FN)                              \
965     static bool trans_##INSN(DisasContext *s, arg_2scalar *a)   \
966     {                                                           \
967         static MVEGenTwoOpScalarFn * const fns[] = {            \
968             NULL,                                               \
969             gen_helper_mve_##FN##h,                             \
970             gen_helper_mve_##FN##s,                             \
971             NULL,                                               \
972         };                                                      \
973         if (!dc_isar_feature(aa32_mve_fp, s)) {                 \
974             return false;                                       \
975         }                                                       \
976         return do_2op_scalar(s, a, fns[a->size]);               \
977     }
978 
979 DO_2OP_FP_SCALAR(VADD_fp_scalar, vfadd_scalar)
980 DO_2OP_FP_SCALAR(VSUB_fp_scalar, vfsub_scalar)
981 DO_2OP_FP_SCALAR(VMUL_fp_scalar, vfmul_scalar)
982 DO_2OP_FP_SCALAR(VFMA_scalar, vfma_scalar)
983 DO_2OP_FP_SCALAR(VFMAS_scalar, vfmas_scalar)
984 
985 static bool do_long_dual_acc(DisasContext *s, arg_vmlaldav *a,
986                              MVEGenLongDualAccOpFn *fn)
987 {
988     TCGv_ptr qn, qm;
989     TCGv_i64 rda;
990     TCGv_i32 rdalo, rdahi;
991 
992     if (!dc_isar_feature(aa32_mve, s) ||
993         !mve_check_qreg_bank(s, a->qn | a->qm) ||
994         !fn) {
995         return false;
996     }
997     /*
998      * rdahi == 13 is UNPREDICTABLE; rdahi == 15 is a related
999      * encoding; rdalo always has bit 0 clear so cannot be 13 or 15.
1000      */
1001     if (a->rdahi == 13 || a->rdahi == 15) {
1002         return false;
1003     }
1004     if (!mve_eci_check(s) || !vfp_access_check(s)) {
1005         return true;
1006     }
1007 
1008     qn = mve_qreg_ptr(a->qn);
1009     qm = mve_qreg_ptr(a->qm);
1010 
1011     /*
1012      * This insn is subject to beat-wise execution. Partial execution
1013      * of an A=0 (no-accumulate) insn which does not execute the first
1014      * beat must start with the current rda value, not 0.
1015      */
1016     if (a->a || mve_skip_first_beat(s)) {
1017         rda = tcg_temp_new_i64();
1018         rdalo = load_reg(s, a->rdalo);
1019         rdahi = load_reg(s, a->rdahi);
1020         tcg_gen_concat_i32_i64(rda, rdalo, rdahi);
1021         tcg_temp_free_i32(rdalo);
1022         tcg_temp_free_i32(rdahi);
1023     } else {
1024         rda = tcg_const_i64(0);
1025     }
1026 
1027     fn(rda, cpu_env, qn, qm, rda);
1028     tcg_temp_free_ptr(qn);
1029     tcg_temp_free_ptr(qm);
1030 
1031     rdalo = tcg_temp_new_i32();
1032     rdahi = tcg_temp_new_i32();
1033     tcg_gen_extrl_i64_i32(rdalo, rda);
1034     tcg_gen_extrh_i64_i32(rdahi, rda);
1035     store_reg(s, a->rdalo, rdalo);
1036     store_reg(s, a->rdahi, rdahi);
1037     tcg_temp_free_i64(rda);
1038     mve_update_eci(s);
1039     return true;
1040 }
1041 
1042 static bool trans_VMLALDAV_S(DisasContext *s, arg_vmlaldav *a)
1043 {
1044     static MVEGenLongDualAccOpFn * const fns[4][2] = {
1045         { NULL, NULL },
1046         { gen_helper_mve_vmlaldavsh, gen_helper_mve_vmlaldavxsh },
1047         { gen_helper_mve_vmlaldavsw, gen_helper_mve_vmlaldavxsw },
1048         { NULL, NULL },
1049     };
1050     return do_long_dual_acc(s, a, fns[a->size][a->x]);
1051 }
1052 
1053 static bool trans_VMLALDAV_U(DisasContext *s, arg_vmlaldav *a)
1054 {
1055     static MVEGenLongDualAccOpFn * const fns[4][2] = {
1056         { NULL, NULL },
1057         { gen_helper_mve_vmlaldavuh, NULL },
1058         { gen_helper_mve_vmlaldavuw, NULL },
1059         { NULL, NULL },
1060     };
1061     return do_long_dual_acc(s, a, fns[a->size][a->x]);
1062 }
1063 
1064 static bool trans_VMLSLDAV(DisasContext *s, arg_vmlaldav *a)
1065 {
1066     static MVEGenLongDualAccOpFn * const fns[4][2] = {
1067         { NULL, NULL },
1068         { gen_helper_mve_vmlsldavsh, gen_helper_mve_vmlsldavxsh },
1069         { gen_helper_mve_vmlsldavsw, gen_helper_mve_vmlsldavxsw },
1070         { NULL, NULL },
1071     };
1072     return do_long_dual_acc(s, a, fns[a->size][a->x]);
1073 }
1074 
1075 static bool trans_VRMLALDAVH_S(DisasContext *s, arg_vmlaldav *a)
1076 {
1077     static MVEGenLongDualAccOpFn * const fns[] = {
1078         gen_helper_mve_vrmlaldavhsw, gen_helper_mve_vrmlaldavhxsw,
1079     };
1080     return do_long_dual_acc(s, a, fns[a->x]);
1081 }
1082 
1083 static bool trans_VRMLALDAVH_U(DisasContext *s, arg_vmlaldav *a)
1084 {
1085     static MVEGenLongDualAccOpFn * const fns[] = {
1086         gen_helper_mve_vrmlaldavhuw, NULL,
1087     };
1088     return do_long_dual_acc(s, a, fns[a->x]);
1089 }
1090 
1091 static bool trans_VRMLSLDAVH(DisasContext *s, arg_vmlaldav *a)
1092 {
1093     static MVEGenLongDualAccOpFn * const fns[] = {
1094         gen_helper_mve_vrmlsldavhsw, gen_helper_mve_vrmlsldavhxsw,
1095     };
1096     return do_long_dual_acc(s, a, fns[a->x]);
1097 }
1098 
1099 static bool do_dual_acc(DisasContext *s, arg_vmladav *a, MVEGenDualAccOpFn *fn)
1100 {
1101     TCGv_ptr qn, qm;
1102     TCGv_i32 rda;
1103 
1104     if (!dc_isar_feature(aa32_mve, s) ||
1105         !mve_check_qreg_bank(s, a->qn) ||
1106         !fn) {
1107         return false;
1108     }
1109     if (!mve_eci_check(s) || !vfp_access_check(s)) {
1110         return true;
1111     }
1112 
1113     qn = mve_qreg_ptr(a->qn);
1114     qm = mve_qreg_ptr(a->qm);
1115 
1116     /*
1117      * This insn is subject to beat-wise execution. Partial execution
1118      * of an A=0 (no-accumulate) insn which does not execute the first
1119      * beat must start with the current rda value, not 0.
1120      */
1121     if (a->a || mve_skip_first_beat(s)) {
1122         rda = load_reg(s, a->rda);
1123     } else {
1124         rda = tcg_const_i32(0);
1125     }
1126 
1127     fn(rda, cpu_env, qn, qm, rda);
1128     store_reg(s, a->rda, rda);
1129     tcg_temp_free_ptr(qn);
1130     tcg_temp_free_ptr(qm);
1131 
1132     mve_update_eci(s);
1133     return true;
1134 }
1135 
1136 #define DO_DUAL_ACC(INSN, FN)                                           \
1137     static bool trans_##INSN(DisasContext *s, arg_vmladav *a)           \
1138     {                                                                   \
1139         static MVEGenDualAccOpFn * const fns[4][2] = {                  \
1140             { gen_helper_mve_##FN##b, gen_helper_mve_##FN##xb },        \
1141             { gen_helper_mve_##FN##h, gen_helper_mve_##FN##xh },        \
1142             { gen_helper_mve_##FN##w, gen_helper_mve_##FN##xw },        \
1143             { NULL, NULL },                                             \
1144         };                                                              \
1145         return do_dual_acc(s, a, fns[a->size][a->x]);                   \
1146     }
1147 
1148 DO_DUAL_ACC(VMLADAV_S, vmladavs)
1149 DO_DUAL_ACC(VMLSDAV, vmlsdav)
1150 
1151 static bool trans_VMLADAV_U(DisasContext *s, arg_vmladav *a)
1152 {
1153     static MVEGenDualAccOpFn * const fns[4][2] = {
1154         { gen_helper_mve_vmladavub, NULL },
1155         { gen_helper_mve_vmladavuh, NULL },
1156         { gen_helper_mve_vmladavuw, NULL },
1157         { NULL, NULL },
1158     };
1159     return do_dual_acc(s, a, fns[a->size][a->x]);
1160 }
1161 
1162 static void gen_vpst(DisasContext *s, uint32_t mask)
1163 {
1164     /*
1165      * Set the VPR mask fields. We take advantage of MASK01 and MASK23
1166      * being adjacent fields in the register.
1167      *
1168      * Updating the masks is not predicated, but it is subject to beat-wise
1169      * execution, and the mask is updated on the odd-numbered beats.
1170      * So if PSR.ECI says we should skip beat 1, we mustn't update the
1171      * 01 mask field.
1172      */
1173     TCGv_i32 vpr = load_cpu_field(v7m.vpr);
1174     switch (s->eci) {
1175     case ECI_NONE:
1176     case ECI_A0:
1177         /* Update both 01 and 23 fields */
1178         tcg_gen_deposit_i32(vpr, vpr,
1179                             tcg_constant_i32(mask | (mask << 4)),
1180                             R_V7M_VPR_MASK01_SHIFT,
1181                             R_V7M_VPR_MASK01_LENGTH + R_V7M_VPR_MASK23_LENGTH);
1182         break;
1183     case ECI_A0A1:
1184     case ECI_A0A1A2:
1185     case ECI_A0A1A2B0:
1186         /* Update only the 23 mask field */
1187         tcg_gen_deposit_i32(vpr, vpr,
1188                             tcg_constant_i32(mask),
1189                             R_V7M_VPR_MASK23_SHIFT, R_V7M_VPR_MASK23_LENGTH);
1190         break;
1191     default:
1192         g_assert_not_reached();
1193     }
1194     store_cpu_field(vpr, v7m.vpr);
1195 }
1196 
1197 static bool trans_VPST(DisasContext *s, arg_VPST *a)
1198 {
1199     /* mask == 0 is a "related encoding" */
1200     if (!dc_isar_feature(aa32_mve, s) || !a->mask) {
1201         return false;
1202     }
1203     if (!mve_eci_check(s) || !vfp_access_check(s)) {
1204         return true;
1205     }
1206     gen_vpst(s, a->mask);
1207     mve_update_and_store_eci(s);
1208     return true;
1209 }
1210 
1211 static bool trans_VPNOT(DisasContext *s, arg_VPNOT *a)
1212 {
1213     /*
1214      * Invert the predicate in VPR.P0. We have call out to
1215      * a helper because this insn itself is beatwise and can
1216      * be predicated.
1217      */
1218     if (!dc_isar_feature(aa32_mve, s)) {
1219         return false;
1220     }
1221     if (!mve_eci_check(s) || !vfp_access_check(s)) {
1222         return true;
1223     }
1224 
1225     gen_helper_mve_vpnot(cpu_env);
1226     mve_update_eci(s);
1227     return true;
1228 }
1229 
1230 static bool trans_VADDV(DisasContext *s, arg_VADDV *a)
1231 {
1232     /* VADDV: vector add across vector */
1233     static MVEGenVADDVFn * const fns[4][2] = {
1234         { gen_helper_mve_vaddvsb, gen_helper_mve_vaddvub },
1235         { gen_helper_mve_vaddvsh, gen_helper_mve_vaddvuh },
1236         { gen_helper_mve_vaddvsw, gen_helper_mve_vaddvuw },
1237         { NULL, NULL }
1238     };
1239     TCGv_ptr qm;
1240     TCGv_i32 rda;
1241 
1242     if (!dc_isar_feature(aa32_mve, s) ||
1243         a->size == 3) {
1244         return false;
1245     }
1246     if (!mve_eci_check(s) || !vfp_access_check(s)) {
1247         return true;
1248     }
1249 
1250     /*
1251      * This insn is subject to beat-wise execution. Partial execution
1252      * of an A=0 (no-accumulate) insn which does not execute the first
1253      * beat must start with the current value of Rda, not zero.
1254      */
1255     if (a->a || mve_skip_first_beat(s)) {
1256         /* Accumulate input from Rda */
1257         rda = load_reg(s, a->rda);
1258     } else {
1259         /* Accumulate starting at zero */
1260         rda = tcg_const_i32(0);
1261     }
1262 
1263     qm = mve_qreg_ptr(a->qm);
1264     fns[a->size][a->u](rda, cpu_env, qm, rda);
1265     store_reg(s, a->rda, rda);
1266     tcg_temp_free_ptr(qm);
1267 
1268     mve_update_eci(s);
1269     return true;
1270 }
1271 
1272 static bool trans_VADDLV(DisasContext *s, arg_VADDLV *a)
1273 {
1274     /*
1275      * Vector Add Long Across Vector: accumulate the 32-bit
1276      * elements of the vector into a 64-bit result stored in
1277      * a pair of general-purpose registers.
1278      * No need to check Qm's bank: it is only 3 bits in decode.
1279      */
1280     TCGv_ptr qm;
1281     TCGv_i64 rda;
1282     TCGv_i32 rdalo, rdahi;
1283 
1284     if (!dc_isar_feature(aa32_mve, s)) {
1285         return false;
1286     }
1287     /*
1288      * rdahi == 13 is UNPREDICTABLE; rdahi == 15 is a related
1289      * encoding; rdalo always has bit 0 clear so cannot be 13 or 15.
1290      */
1291     if (a->rdahi == 13 || a->rdahi == 15) {
1292         return false;
1293     }
1294     if (!mve_eci_check(s) || !vfp_access_check(s)) {
1295         return true;
1296     }
1297 
1298     /*
1299      * This insn is subject to beat-wise execution. Partial execution
1300      * of an A=0 (no-accumulate) insn which does not execute the first
1301      * beat must start with the current value of RdaHi:RdaLo, not zero.
1302      */
1303     if (a->a || mve_skip_first_beat(s)) {
1304         /* Accumulate input from RdaHi:RdaLo */
1305         rda = tcg_temp_new_i64();
1306         rdalo = load_reg(s, a->rdalo);
1307         rdahi = load_reg(s, a->rdahi);
1308         tcg_gen_concat_i32_i64(rda, rdalo, rdahi);
1309         tcg_temp_free_i32(rdalo);
1310         tcg_temp_free_i32(rdahi);
1311     } else {
1312         /* Accumulate starting at zero */
1313         rda = tcg_const_i64(0);
1314     }
1315 
1316     qm = mve_qreg_ptr(a->qm);
1317     if (a->u) {
1318         gen_helper_mve_vaddlv_u(rda, cpu_env, qm, rda);
1319     } else {
1320         gen_helper_mve_vaddlv_s(rda, cpu_env, qm, rda);
1321     }
1322     tcg_temp_free_ptr(qm);
1323 
1324     rdalo = tcg_temp_new_i32();
1325     rdahi = tcg_temp_new_i32();
1326     tcg_gen_extrl_i64_i32(rdalo, rda);
1327     tcg_gen_extrh_i64_i32(rdahi, rda);
1328     store_reg(s, a->rdalo, rdalo);
1329     store_reg(s, a->rdahi, rdahi);
1330     tcg_temp_free_i64(rda);
1331     mve_update_eci(s);
1332     return true;
1333 }
1334 
1335 static bool do_1imm(DisasContext *s, arg_1imm *a, MVEGenOneOpImmFn *fn)
1336 {
1337     TCGv_ptr qd;
1338     uint64_t imm;
1339 
1340     if (!dc_isar_feature(aa32_mve, s) ||
1341         !mve_check_qreg_bank(s, a->qd) ||
1342         !fn) {
1343         return false;
1344     }
1345     if (!mve_eci_check(s) || !vfp_access_check(s)) {
1346         return true;
1347     }
1348 
1349     imm = asimd_imm_const(a->imm, a->cmode, a->op);
1350 
1351     qd = mve_qreg_ptr(a->qd);
1352     fn(cpu_env, qd, tcg_constant_i64(imm));
1353     tcg_temp_free_ptr(qd);
1354     mve_update_eci(s);
1355     return true;
1356 }
1357 
1358 static bool trans_Vimm_1r(DisasContext *s, arg_1imm *a)
1359 {
1360     /* Handle decode of cmode/op here between VORR/VBIC/VMOV */
1361     MVEGenOneOpImmFn *fn;
1362 
1363     if ((a->cmode & 1) && a->cmode < 12) {
1364         if (a->op) {
1365             /*
1366              * For op=1, the immediate will be inverted by asimd_imm_const(),
1367              * so the VBIC becomes a logical AND operation.
1368              */
1369             fn = gen_helper_mve_vandi;
1370         } else {
1371             fn = gen_helper_mve_vorri;
1372         }
1373     } else {
1374         /* There is one unallocated cmode/op combination in this space */
1375         if (a->cmode == 15 && a->op == 1) {
1376             return false;
1377         }
1378         /* asimd_imm_const() sorts out VMVNI vs VMOVI for us */
1379         fn = gen_helper_mve_vmovi;
1380     }
1381     return do_1imm(s, a, fn);
1382 }
1383 
1384 static bool do_2shift(DisasContext *s, arg_2shift *a, MVEGenTwoOpShiftFn fn,
1385                       bool negateshift)
1386 {
1387     TCGv_ptr qd, qm;
1388     int shift = a->shift;
1389 
1390     if (!dc_isar_feature(aa32_mve, s) ||
1391         !mve_check_qreg_bank(s, a->qd | a->qm) ||
1392         !fn) {
1393         return false;
1394     }
1395     if (!mve_eci_check(s) || !vfp_access_check(s)) {
1396         return true;
1397     }
1398 
1399     /*
1400      * When we handle a right shift insn using a left-shift helper
1401      * which permits a negative shift count to indicate a right-shift,
1402      * we must negate the shift count.
1403      */
1404     if (negateshift) {
1405         shift = -shift;
1406     }
1407 
1408     qd = mve_qreg_ptr(a->qd);
1409     qm = mve_qreg_ptr(a->qm);
1410     fn(cpu_env, qd, qm, tcg_constant_i32(shift));
1411     tcg_temp_free_ptr(qd);
1412     tcg_temp_free_ptr(qm);
1413     mve_update_eci(s);
1414     return true;
1415 }
1416 
1417 #define DO_2SHIFT(INSN, FN, NEGATESHIFT)                         \
1418     static bool trans_##INSN(DisasContext *s, arg_2shift *a)    \
1419     {                                                           \
1420         static MVEGenTwoOpShiftFn * const fns[] = {             \
1421             gen_helper_mve_##FN##b,                             \
1422             gen_helper_mve_##FN##h,                             \
1423             gen_helper_mve_##FN##w,                             \
1424             NULL,                                               \
1425         };                                                      \
1426         return do_2shift(s, a, fns[a->size], NEGATESHIFT);      \
1427     }
1428 
1429 DO_2SHIFT(VSHLI, vshli_u, false)
1430 DO_2SHIFT(VQSHLI_S, vqshli_s, false)
1431 DO_2SHIFT(VQSHLI_U, vqshli_u, false)
1432 DO_2SHIFT(VQSHLUI, vqshlui_s, false)
1433 /* These right shifts use a left-shift helper with negated shift count */
1434 DO_2SHIFT(VSHRI_S, vshli_s, true)
1435 DO_2SHIFT(VSHRI_U, vshli_u, true)
1436 DO_2SHIFT(VRSHRI_S, vrshli_s, true)
1437 DO_2SHIFT(VRSHRI_U, vrshli_u, true)
1438 
1439 DO_2SHIFT(VSRI, vsri, false)
1440 DO_2SHIFT(VSLI, vsli, false)
1441 
1442 static bool do_2shift_scalar(DisasContext *s, arg_shl_scalar *a,
1443                              MVEGenTwoOpShiftFn *fn)
1444 {
1445     TCGv_ptr qda;
1446     TCGv_i32 rm;
1447 
1448     if (!dc_isar_feature(aa32_mve, s) ||
1449         !mve_check_qreg_bank(s, a->qda) ||
1450         a->rm == 13 || a->rm == 15 || !fn) {
1451         /* Rm cases are UNPREDICTABLE */
1452         return false;
1453     }
1454     if (!mve_eci_check(s) || !vfp_access_check(s)) {
1455         return true;
1456     }
1457 
1458     qda = mve_qreg_ptr(a->qda);
1459     rm = load_reg(s, a->rm);
1460     fn(cpu_env, qda, qda, rm);
1461     tcg_temp_free_ptr(qda);
1462     tcg_temp_free_i32(rm);
1463     mve_update_eci(s);
1464     return true;
1465 }
1466 
1467 #define DO_2SHIFT_SCALAR(INSN, FN)                                      \
1468     static bool trans_##INSN(DisasContext *s, arg_shl_scalar *a)        \
1469     {                                                                   \
1470         static MVEGenTwoOpShiftFn * const fns[] = {                     \
1471             gen_helper_mve_##FN##b,                                     \
1472             gen_helper_mve_##FN##h,                                     \
1473             gen_helper_mve_##FN##w,                                     \
1474             NULL,                                                       \
1475         };                                                              \
1476         return do_2shift_scalar(s, a, fns[a->size]);                    \
1477     }
1478 
1479 DO_2SHIFT_SCALAR(VSHL_S_scalar, vshli_s)
1480 DO_2SHIFT_SCALAR(VSHL_U_scalar, vshli_u)
1481 DO_2SHIFT_SCALAR(VRSHL_S_scalar, vrshli_s)
1482 DO_2SHIFT_SCALAR(VRSHL_U_scalar, vrshli_u)
1483 DO_2SHIFT_SCALAR(VQSHL_S_scalar, vqshli_s)
1484 DO_2SHIFT_SCALAR(VQSHL_U_scalar, vqshli_u)
1485 DO_2SHIFT_SCALAR(VQRSHL_S_scalar, vqrshli_s)
1486 DO_2SHIFT_SCALAR(VQRSHL_U_scalar, vqrshli_u)
1487 
1488 #define DO_VSHLL(INSN, FN)                                      \
1489     static bool trans_##INSN(DisasContext *s, arg_2shift *a)    \
1490     {                                                           \
1491         static MVEGenTwoOpShiftFn * const fns[] = {             \
1492             gen_helper_mve_##FN##b,                             \
1493             gen_helper_mve_##FN##h,                             \
1494         };                                                      \
1495         return do_2shift(s, a, fns[a->size], false);            \
1496     }
1497 
1498 DO_VSHLL(VSHLL_BS, vshllbs)
1499 DO_VSHLL(VSHLL_BU, vshllbu)
1500 DO_VSHLL(VSHLL_TS, vshllts)
1501 DO_VSHLL(VSHLL_TU, vshlltu)
1502 
1503 #define DO_2SHIFT_N(INSN, FN)                                   \
1504     static bool trans_##INSN(DisasContext *s, arg_2shift *a)    \
1505     {                                                           \
1506         static MVEGenTwoOpShiftFn * const fns[] = {             \
1507             gen_helper_mve_##FN##b,                             \
1508             gen_helper_mve_##FN##h,                             \
1509         };                                                      \
1510         return do_2shift(s, a, fns[a->size], false);            \
1511     }
1512 
1513 DO_2SHIFT_N(VSHRNB, vshrnb)
1514 DO_2SHIFT_N(VSHRNT, vshrnt)
1515 DO_2SHIFT_N(VRSHRNB, vrshrnb)
1516 DO_2SHIFT_N(VRSHRNT, vrshrnt)
1517 DO_2SHIFT_N(VQSHRNB_S, vqshrnb_s)
1518 DO_2SHIFT_N(VQSHRNT_S, vqshrnt_s)
1519 DO_2SHIFT_N(VQSHRNB_U, vqshrnb_u)
1520 DO_2SHIFT_N(VQSHRNT_U, vqshrnt_u)
1521 DO_2SHIFT_N(VQSHRUNB, vqshrunb)
1522 DO_2SHIFT_N(VQSHRUNT, vqshrunt)
1523 DO_2SHIFT_N(VQRSHRNB_S, vqrshrnb_s)
1524 DO_2SHIFT_N(VQRSHRNT_S, vqrshrnt_s)
1525 DO_2SHIFT_N(VQRSHRNB_U, vqrshrnb_u)
1526 DO_2SHIFT_N(VQRSHRNT_U, vqrshrnt_u)
1527 DO_2SHIFT_N(VQRSHRUNB, vqrshrunb)
1528 DO_2SHIFT_N(VQRSHRUNT, vqrshrunt)
1529 
1530 static bool trans_VSHLC(DisasContext *s, arg_VSHLC *a)
1531 {
1532     /*
1533      * Whole Vector Left Shift with Carry. The carry is taken
1534      * from a general purpose register and written back there.
1535      * An imm of 0 means "shift by 32".
1536      */
1537     TCGv_ptr qd;
1538     TCGv_i32 rdm;
1539 
1540     if (!dc_isar_feature(aa32_mve, s) || !mve_check_qreg_bank(s, a->qd)) {
1541         return false;
1542     }
1543     if (a->rdm == 13 || a->rdm == 15) {
1544         /* CONSTRAINED UNPREDICTABLE: we UNDEF */
1545         return false;
1546     }
1547     if (!mve_eci_check(s) || !vfp_access_check(s)) {
1548         return true;
1549     }
1550 
1551     qd = mve_qreg_ptr(a->qd);
1552     rdm = load_reg(s, a->rdm);
1553     gen_helper_mve_vshlc(rdm, cpu_env, qd, rdm, tcg_constant_i32(a->imm));
1554     store_reg(s, a->rdm, rdm);
1555     tcg_temp_free_ptr(qd);
1556     mve_update_eci(s);
1557     return true;
1558 }
1559 
1560 static bool do_vidup(DisasContext *s, arg_vidup *a, MVEGenVIDUPFn *fn)
1561 {
1562     TCGv_ptr qd;
1563     TCGv_i32 rn;
1564 
1565     /*
1566      * Vector increment/decrement with wrap and duplicate (VIDUP, VDDUP).
1567      * This fills the vector with elements of successively increasing
1568      * or decreasing values, starting from Rn.
1569      */
1570     if (!dc_isar_feature(aa32_mve, s) || !mve_check_qreg_bank(s, a->qd)) {
1571         return false;
1572     }
1573     if (a->size == MO_64) {
1574         /* size 0b11 is another encoding */
1575         return false;
1576     }
1577     if (!mve_eci_check(s) || !vfp_access_check(s)) {
1578         return true;
1579     }
1580 
1581     qd = mve_qreg_ptr(a->qd);
1582     rn = load_reg(s, a->rn);
1583     fn(rn, cpu_env, qd, rn, tcg_constant_i32(a->imm));
1584     store_reg(s, a->rn, rn);
1585     tcg_temp_free_ptr(qd);
1586     mve_update_eci(s);
1587     return true;
1588 }
1589 
1590 static bool do_viwdup(DisasContext *s, arg_viwdup *a, MVEGenVIWDUPFn *fn)
1591 {
1592     TCGv_ptr qd;
1593     TCGv_i32 rn, rm;
1594 
1595     /*
1596      * Vector increment/decrement with wrap and duplicate (VIWDUp, VDWDUP)
1597      * This fills the vector with elements of successively increasing
1598      * or decreasing values, starting from Rn. Rm specifies a point where
1599      * the count wraps back around to 0. The updated offset is written back
1600      * to Rn.
1601      */
1602     if (!dc_isar_feature(aa32_mve, s) || !mve_check_qreg_bank(s, a->qd)) {
1603         return false;
1604     }
1605     if (!fn || a->rm == 13 || a->rm == 15) {
1606         /*
1607          * size 0b11 is another encoding; Rm == 13 is UNPREDICTABLE;
1608          * Rm == 13 is VIWDUP, VDWDUP.
1609          */
1610         return false;
1611     }
1612     if (!mve_eci_check(s) || !vfp_access_check(s)) {
1613         return true;
1614     }
1615 
1616     qd = mve_qreg_ptr(a->qd);
1617     rn = load_reg(s, a->rn);
1618     rm = load_reg(s, a->rm);
1619     fn(rn, cpu_env, qd, rn, rm, tcg_constant_i32(a->imm));
1620     store_reg(s, a->rn, rn);
1621     tcg_temp_free_ptr(qd);
1622     tcg_temp_free_i32(rm);
1623     mve_update_eci(s);
1624     return true;
1625 }
1626 
1627 static bool trans_VIDUP(DisasContext *s, arg_vidup *a)
1628 {
1629     static MVEGenVIDUPFn * const fns[] = {
1630         gen_helper_mve_vidupb,
1631         gen_helper_mve_viduph,
1632         gen_helper_mve_vidupw,
1633         NULL,
1634     };
1635     return do_vidup(s, a, fns[a->size]);
1636 }
1637 
1638 static bool trans_VDDUP(DisasContext *s, arg_vidup *a)
1639 {
1640     static MVEGenVIDUPFn * const fns[] = {
1641         gen_helper_mve_vidupb,
1642         gen_helper_mve_viduph,
1643         gen_helper_mve_vidupw,
1644         NULL,
1645     };
1646     /* VDDUP is just like VIDUP but with a negative immediate */
1647     a->imm = -a->imm;
1648     return do_vidup(s, a, fns[a->size]);
1649 }
1650 
1651 static bool trans_VIWDUP(DisasContext *s, arg_viwdup *a)
1652 {
1653     static MVEGenVIWDUPFn * const fns[] = {
1654         gen_helper_mve_viwdupb,
1655         gen_helper_mve_viwduph,
1656         gen_helper_mve_viwdupw,
1657         NULL,
1658     };
1659     return do_viwdup(s, a, fns[a->size]);
1660 }
1661 
1662 static bool trans_VDWDUP(DisasContext *s, arg_viwdup *a)
1663 {
1664     static MVEGenVIWDUPFn * const fns[] = {
1665         gen_helper_mve_vdwdupb,
1666         gen_helper_mve_vdwduph,
1667         gen_helper_mve_vdwdupw,
1668         NULL,
1669     };
1670     return do_viwdup(s, a, fns[a->size]);
1671 }
1672 
1673 static bool do_vcmp(DisasContext *s, arg_vcmp *a, MVEGenCmpFn *fn)
1674 {
1675     TCGv_ptr qn, qm;
1676 
1677     if (!dc_isar_feature(aa32_mve, s) || !mve_check_qreg_bank(s, a->qm) ||
1678         !fn) {
1679         return false;
1680     }
1681     if (!mve_eci_check(s) || !vfp_access_check(s)) {
1682         return true;
1683     }
1684 
1685     qn = mve_qreg_ptr(a->qn);
1686     qm = mve_qreg_ptr(a->qm);
1687     fn(cpu_env, qn, qm);
1688     tcg_temp_free_ptr(qn);
1689     tcg_temp_free_ptr(qm);
1690     if (a->mask) {
1691         /* VPT */
1692         gen_vpst(s, a->mask);
1693     }
1694     mve_update_eci(s);
1695     return true;
1696 }
1697 
1698 static bool do_vcmp_scalar(DisasContext *s, arg_vcmp_scalar *a,
1699                            MVEGenScalarCmpFn *fn)
1700 {
1701     TCGv_ptr qn;
1702     TCGv_i32 rm;
1703 
1704     if (!dc_isar_feature(aa32_mve, s) || !fn || a->rm == 13) {
1705         return false;
1706     }
1707     if (!mve_eci_check(s) || !vfp_access_check(s)) {
1708         return true;
1709     }
1710 
1711     qn = mve_qreg_ptr(a->qn);
1712     if (a->rm == 15) {
1713         /* Encoding Rm=0b1111 means "constant zero" */
1714         rm = tcg_constant_i32(0);
1715     } else {
1716         rm = load_reg(s, a->rm);
1717     }
1718     fn(cpu_env, qn, rm);
1719     tcg_temp_free_ptr(qn);
1720     tcg_temp_free_i32(rm);
1721     if (a->mask) {
1722         /* VPT */
1723         gen_vpst(s, a->mask);
1724     }
1725     mve_update_eci(s);
1726     return true;
1727 }
1728 
1729 #define DO_VCMP(INSN, FN)                                       \
1730     static bool trans_##INSN(DisasContext *s, arg_vcmp *a)      \
1731     {                                                           \
1732         static MVEGenCmpFn * const fns[] = {                    \
1733             gen_helper_mve_##FN##b,                             \
1734             gen_helper_mve_##FN##h,                             \
1735             gen_helper_mve_##FN##w,                             \
1736             NULL,                                               \
1737         };                                                      \
1738         return do_vcmp(s, a, fns[a->size]);                     \
1739     }                                                           \
1740     static bool trans_##INSN##_scalar(DisasContext *s,          \
1741                                       arg_vcmp_scalar *a)       \
1742     {                                                           \
1743         static MVEGenScalarCmpFn * const fns[] = {              \
1744             gen_helper_mve_##FN##_scalarb,                      \
1745             gen_helper_mve_##FN##_scalarh,                      \
1746             gen_helper_mve_##FN##_scalarw,                      \
1747             NULL,                                               \
1748         };                                                      \
1749         return do_vcmp_scalar(s, a, fns[a->size]);              \
1750     }
1751 
1752 DO_VCMP(VCMPEQ, vcmpeq)
1753 DO_VCMP(VCMPNE, vcmpne)
1754 DO_VCMP(VCMPCS, vcmpcs)
1755 DO_VCMP(VCMPHI, vcmphi)
1756 DO_VCMP(VCMPGE, vcmpge)
1757 DO_VCMP(VCMPLT, vcmplt)
1758 DO_VCMP(VCMPGT, vcmpgt)
1759 DO_VCMP(VCMPLE, vcmple)
1760 
1761 #define DO_VCMP_FP(INSN, FN)                                    \
1762     static bool trans_##INSN(DisasContext *s, arg_vcmp *a)      \
1763     {                                                           \
1764         static MVEGenCmpFn * const fns[] = {                    \
1765             NULL,                                               \
1766             gen_helper_mve_##FN##h,                             \
1767             gen_helper_mve_##FN##s,                             \
1768             NULL,                                               \
1769         };                                                      \
1770         if (!dc_isar_feature(aa32_mve_fp, s)) {                 \
1771             return false;                                       \
1772         }                                                       \
1773         return do_vcmp(s, a, fns[a->size]);                     \
1774     }                                                           \
1775     static bool trans_##INSN##_scalar(DisasContext *s,          \
1776                                       arg_vcmp_scalar *a)       \
1777     {                                                           \
1778         static MVEGenScalarCmpFn * const fns[] = {              \
1779             NULL,                                               \
1780             gen_helper_mve_##FN##_scalarh,                      \
1781             gen_helper_mve_##FN##_scalars,                      \
1782             NULL,                                               \
1783         };                                                      \
1784         if (!dc_isar_feature(aa32_mve_fp, s)) {                 \
1785             return false;                                       \
1786         }                                                       \
1787         return do_vcmp_scalar(s, a, fns[a->size]);              \
1788     }
1789 
1790 DO_VCMP_FP(VCMPEQ_fp, vfcmpeq)
1791 DO_VCMP_FP(VCMPNE_fp, vfcmpne)
1792 DO_VCMP_FP(VCMPGE_fp, vfcmpge)
1793 DO_VCMP_FP(VCMPLT_fp, vfcmplt)
1794 DO_VCMP_FP(VCMPGT_fp, vfcmpgt)
1795 DO_VCMP_FP(VCMPLE_fp, vfcmple)
1796 
1797 static bool do_vmaxv(DisasContext *s, arg_vmaxv *a, MVEGenVADDVFn fn)
1798 {
1799     /*
1800      * MIN/MAX operations across a vector: compute the min or
1801      * max of the initial value in a general purpose register
1802      * and all the elements in the vector, and store it back
1803      * into the general purpose register.
1804      */
1805     TCGv_ptr qm;
1806     TCGv_i32 rda;
1807 
1808     if (!dc_isar_feature(aa32_mve, s) || !mve_check_qreg_bank(s, a->qm) ||
1809         !fn || a->rda == 13 || a->rda == 15) {
1810         /* Rda cases are UNPREDICTABLE */
1811         return false;
1812     }
1813     if (!mve_eci_check(s) || !vfp_access_check(s)) {
1814         return true;
1815     }
1816 
1817     qm = mve_qreg_ptr(a->qm);
1818     rda = load_reg(s, a->rda);
1819     fn(rda, cpu_env, qm, rda);
1820     store_reg(s, a->rda, rda);
1821     tcg_temp_free_ptr(qm);
1822     mve_update_eci(s);
1823     return true;
1824 }
1825 
1826 #define DO_VMAXV(INSN, FN)                                      \
1827     static bool trans_##INSN(DisasContext *s, arg_vmaxv *a)     \
1828     {                                                           \
1829         static MVEGenVADDVFn * const fns[] = {                  \
1830             gen_helper_mve_##FN##b,                             \
1831             gen_helper_mve_##FN##h,                             \
1832             gen_helper_mve_##FN##w,                             \
1833             NULL,                                               \
1834         };                                                      \
1835         return do_vmaxv(s, a, fns[a->size]);                    \
1836     }
1837 
1838 DO_VMAXV(VMAXV_S, vmaxvs)
1839 DO_VMAXV(VMAXV_U, vmaxvu)
1840 DO_VMAXV(VMAXAV, vmaxav)
1841 DO_VMAXV(VMINV_S, vminvs)
1842 DO_VMAXV(VMINV_U, vminvu)
1843 DO_VMAXV(VMINAV, vminav)
1844 
1845 #define DO_VMAXV_FP(INSN, FN)                                   \
1846     static bool trans_##INSN(DisasContext *s, arg_vmaxv *a)     \
1847     {                                                           \
1848         static MVEGenVADDVFn * const fns[] = {                  \
1849             NULL,                                               \
1850             gen_helper_mve_##FN##h,                             \
1851             gen_helper_mve_##FN##s,                             \
1852             NULL,                                               \
1853         };                                                      \
1854         if (!dc_isar_feature(aa32_mve_fp, s)) {                 \
1855             return false;                                       \
1856         }                                                       \
1857         return do_vmaxv(s, a, fns[a->size]);                    \
1858     }
1859 
1860 DO_VMAXV_FP(VMAXNMV, vmaxnmv)
1861 DO_VMAXV_FP(VMINNMV, vminnmv)
1862 DO_VMAXV_FP(VMAXNMAV, vmaxnmav)
1863 DO_VMAXV_FP(VMINNMAV, vminnmav)
1864 
1865 static bool do_vabav(DisasContext *s, arg_vabav *a, MVEGenVABAVFn *fn)
1866 {
1867     /* Absolute difference accumulated across vector */
1868     TCGv_ptr qn, qm;
1869     TCGv_i32 rda;
1870 
1871     if (!dc_isar_feature(aa32_mve, s) ||
1872         !mve_check_qreg_bank(s, a->qm | a->qn) ||
1873         !fn || a->rda == 13 || a->rda == 15) {
1874         /* Rda cases are UNPREDICTABLE */
1875         return false;
1876     }
1877     if (!mve_eci_check(s) || !vfp_access_check(s)) {
1878         return true;
1879     }
1880 
1881     qm = mve_qreg_ptr(a->qm);
1882     qn = mve_qreg_ptr(a->qn);
1883     rda = load_reg(s, a->rda);
1884     fn(rda, cpu_env, qn, qm, rda);
1885     store_reg(s, a->rda, rda);
1886     tcg_temp_free_ptr(qm);
1887     tcg_temp_free_ptr(qn);
1888     mve_update_eci(s);
1889     return true;
1890 }
1891 
1892 #define DO_VABAV(INSN, FN)                                      \
1893     static bool trans_##INSN(DisasContext *s, arg_vabav *a)     \
1894     {                                                           \
1895         static MVEGenVABAVFn * const fns[] = {                  \
1896             gen_helper_mve_##FN##b,                             \
1897             gen_helper_mve_##FN##h,                             \
1898             gen_helper_mve_##FN##w,                             \
1899             NULL,                                               \
1900         };                                                      \
1901         return do_vabav(s, a, fns[a->size]);                    \
1902     }
1903 
1904 DO_VABAV(VABAV_S, vabavs)
1905 DO_VABAV(VABAV_U, vabavu)
1906 
1907 static bool trans_VMOV_to_2gp(DisasContext *s, arg_VMOV_to_2gp *a)
1908 {
1909     /*
1910      * VMOV two 32-bit vector lanes to two general-purpose registers.
1911      * This insn is not predicated but it is subject to beat-wise
1912      * execution if it is not in an IT block. For us this means
1913      * only that if PSR.ECI says we should not be executing the beat
1914      * corresponding to the lane of the vector register being accessed
1915      * then we should skip perfoming the move, and that we need to do
1916      * the usual check for bad ECI state and advance of ECI state.
1917      * (If PSR.ECI is non-zero then we cannot be in an IT block.)
1918      */
1919     TCGv_i32 tmp;
1920     int vd;
1921 
1922     if (!dc_isar_feature(aa32_mve, s) || !mve_check_qreg_bank(s, a->qd) ||
1923         a->rt == 13 || a->rt == 15 || a->rt2 == 13 || a->rt2 == 15 ||
1924         a->rt == a->rt2) {
1925         /* Rt/Rt2 cases are UNPREDICTABLE */
1926         return false;
1927     }
1928     if (!mve_eci_check(s) || !vfp_access_check(s)) {
1929         return true;
1930     }
1931 
1932     /* Convert Qreg index to Dreg for read_neon_element32() etc */
1933     vd = a->qd * 2;
1934 
1935     if (!mve_skip_vmov(s, vd, a->idx, MO_32)) {
1936         tmp = tcg_temp_new_i32();
1937         read_neon_element32(tmp, vd, a->idx, MO_32);
1938         store_reg(s, a->rt, tmp);
1939     }
1940     if (!mve_skip_vmov(s, vd + 1, a->idx, MO_32)) {
1941         tmp = tcg_temp_new_i32();
1942         read_neon_element32(tmp, vd + 1, a->idx, MO_32);
1943         store_reg(s, a->rt2, tmp);
1944     }
1945 
1946     mve_update_and_store_eci(s);
1947     return true;
1948 }
1949 
1950 static bool trans_VMOV_from_2gp(DisasContext *s, arg_VMOV_to_2gp *a)
1951 {
1952     /*
1953      * VMOV two general-purpose registers to two 32-bit vector lanes.
1954      * This insn is not predicated but it is subject to beat-wise
1955      * execution if it is not in an IT block. For us this means
1956      * only that if PSR.ECI says we should not be executing the beat
1957      * corresponding to the lane of the vector register being accessed
1958      * then we should skip perfoming the move, and that we need to do
1959      * the usual check for bad ECI state and advance of ECI state.
1960      * (If PSR.ECI is non-zero then we cannot be in an IT block.)
1961      */
1962     TCGv_i32 tmp;
1963     int vd;
1964 
1965     if (!dc_isar_feature(aa32_mve, s) || !mve_check_qreg_bank(s, a->qd) ||
1966         a->rt == 13 || a->rt == 15 || a->rt2 == 13 || a->rt2 == 15) {
1967         /* Rt/Rt2 cases are UNPREDICTABLE */
1968         return false;
1969     }
1970     if (!mve_eci_check(s) || !vfp_access_check(s)) {
1971         return true;
1972     }
1973 
1974     /* Convert Qreg idx to Dreg for read_neon_element32() etc */
1975     vd = a->qd * 2;
1976 
1977     if (!mve_skip_vmov(s, vd, a->idx, MO_32)) {
1978         tmp = load_reg(s, a->rt);
1979         write_neon_element32(tmp, vd, a->idx, MO_32);
1980         tcg_temp_free_i32(tmp);
1981     }
1982     if (!mve_skip_vmov(s, vd + 1, a->idx, MO_32)) {
1983         tmp = load_reg(s, a->rt2);
1984         write_neon_element32(tmp, vd + 1, a->idx, MO_32);
1985         tcg_temp_free_i32(tmp);
1986     }
1987 
1988     mve_update_and_store_eci(s);
1989     return true;
1990 }
1991