1 /*
2 * AArch64 SVE translation
3 *
4 * Copyright (c) 2018 Linaro, Ltd
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19
20 #include "qemu/osdep.h"
21 #include "translate.h"
22 #include "translate-a64.h"
23 #include "fpu/softfloat.h"
24
25
26 typedef void GVecGen2sFn(unsigned, uint32_t, uint32_t,
27 TCGv_i64, uint32_t, uint32_t);
28
29 typedef void gen_helper_gvec_flags_3(TCGv_i32, TCGv_ptr, TCGv_ptr,
30 TCGv_ptr, TCGv_i32);
31 typedef void gen_helper_gvec_flags_4(TCGv_i32, TCGv_ptr, TCGv_ptr,
32 TCGv_ptr, TCGv_ptr, TCGv_i32);
33
34 typedef void gen_helper_gvec_mem(TCGv_env, TCGv_ptr, TCGv_i64, TCGv_i32);
35 typedef void gen_helper_gvec_mem_scatter(TCGv_env, TCGv_ptr, TCGv_ptr,
36 TCGv_ptr, TCGv_i64, TCGv_i32);
37
38 /*
39 * Helpers for extracting complex instruction fields.
40 */
41
42 /* See e.g. ASR (immediate, predicated).
43 * Returns -1 for unallocated encoding; diagnose later.
44 */
tszimm_esz(DisasContext * s,int x)45 static int tszimm_esz(DisasContext *s, int x)
46 {
47 x >>= 3; /* discard imm3 */
48 return 31 - clz32(x);
49 }
50
tszimm_shr(DisasContext * s,int x)51 static int tszimm_shr(DisasContext *s, int x)
52 {
53 /*
54 * We won't use the tszimm_shr() value if tszimm_esz() returns -1 (the
55 * trans function will check for esz < 0), so we can return any
56 * value we like from here in that case as long as we avoid UB.
57 */
58 int esz = tszimm_esz(s, x);
59 if (esz < 0) {
60 return esz;
61 }
62 return (16 << esz) - x;
63 }
64
65 /* See e.g. LSL (immediate, predicated). */
tszimm_shl(DisasContext * s,int x)66 static int tszimm_shl(DisasContext *s, int x)
67 {
68 /* As with tszimm_shr(), value will be unused if esz < 0 */
69 int esz = tszimm_esz(s, x);
70 if (esz < 0) {
71 return esz;
72 }
73 return x - (8 << esz);
74 }
75
76 /* The SH bit is in bit 8. Extract the low 8 and shift. */
expand_imm_sh8s(DisasContext * s,int x)77 static inline int expand_imm_sh8s(DisasContext *s, int x)
78 {
79 return (int8_t)x << (x & 0x100 ? 8 : 0);
80 }
81
expand_imm_sh8u(DisasContext * s,int x)82 static inline int expand_imm_sh8u(DisasContext *s, int x)
83 {
84 return (uint8_t)x << (x & 0x100 ? 8 : 0);
85 }
86
87 /* Convert a 2-bit memory size (msz) to a 4-bit data type (dtype)
88 * with unsigned data. C.f. SVE Memory Contiguous Load Group.
89 */
msz_dtype(DisasContext * s,int msz)90 static inline int msz_dtype(DisasContext *s, int msz)
91 {
92 static const uint8_t dtype[4] = { 0, 5, 10, 15 };
93 return dtype[msz];
94 }
95
96 /*
97 * Include the generated decoder.
98 */
99
100 #include "decode-sve.c.inc"
101
102 /*
103 * Implement all of the translator functions referenced by the decoder.
104 */
105
106 /* Invoke an out-of-line helper on 2 Zregs. */
gen_gvec_ool_zz(DisasContext * s,gen_helper_gvec_2 * fn,int rd,int rn,int data)107 static bool gen_gvec_ool_zz(DisasContext *s, gen_helper_gvec_2 *fn,
108 int rd, int rn, int data)
109 {
110 if (fn == NULL) {
111 return false;
112 }
113 if (sve_access_check(s)) {
114 unsigned vsz = vec_full_reg_size(s);
115 tcg_gen_gvec_2_ool(vec_full_reg_offset(s, rd),
116 vec_full_reg_offset(s, rn),
117 vsz, vsz, data, fn);
118 }
119 return true;
120 }
121
gen_gvec_fpst_zz(DisasContext * s,gen_helper_gvec_2_ptr * fn,int rd,int rn,int data,ARMFPStatusFlavour flavour)122 static bool gen_gvec_fpst_zz(DisasContext *s, gen_helper_gvec_2_ptr *fn,
123 int rd, int rn, int data,
124 ARMFPStatusFlavour flavour)
125 {
126 if (fn == NULL) {
127 return false;
128 }
129 if (sve_access_check(s)) {
130 unsigned vsz = vec_full_reg_size(s);
131 TCGv_ptr status = fpstatus_ptr(flavour);
132
133 tcg_gen_gvec_2_ptr(vec_full_reg_offset(s, rd),
134 vec_full_reg_offset(s, rn),
135 status, vsz, vsz, data, fn);
136 }
137 return true;
138 }
139
gen_gvec_fpst_ah_arg_zz(DisasContext * s,gen_helper_gvec_2_ptr * fn,arg_rr_esz * a,int data)140 static bool gen_gvec_fpst_ah_arg_zz(DisasContext *s, gen_helper_gvec_2_ptr *fn,
141 arg_rr_esz *a, int data)
142 {
143 return gen_gvec_fpst_zz(s, fn, a->rd, a->rn, data,
144 select_ah_fpst(s, a->esz));
145 }
146
147 /* Invoke an out-of-line helper on 3 Zregs. */
gen_gvec_ool_zzz(DisasContext * s,gen_helper_gvec_3 * fn,int rd,int rn,int rm,int data)148 static bool gen_gvec_ool_zzz(DisasContext *s, gen_helper_gvec_3 *fn,
149 int rd, int rn, int rm, int data)
150 {
151 if (fn == NULL) {
152 return false;
153 }
154 if (sve_access_check(s)) {
155 unsigned vsz = vec_full_reg_size(s);
156 tcg_gen_gvec_3_ool(vec_full_reg_offset(s, rd),
157 vec_full_reg_offset(s, rn),
158 vec_full_reg_offset(s, rm),
159 vsz, vsz, data, fn);
160 }
161 return true;
162 }
163
gen_gvec_ool_arg_zzz(DisasContext * s,gen_helper_gvec_3 * fn,arg_rrr_esz * a,int data)164 static bool gen_gvec_ool_arg_zzz(DisasContext *s, gen_helper_gvec_3 *fn,
165 arg_rrr_esz *a, int data)
166 {
167 return gen_gvec_ool_zzz(s, fn, a->rd, a->rn, a->rm, data);
168 }
169
170 /* Invoke an out-of-line helper on 3 Zregs, plus float_status. */
gen_gvec_fpst_zzz(DisasContext * s,gen_helper_gvec_3_ptr * fn,int rd,int rn,int rm,int data,ARMFPStatusFlavour flavour)171 static bool gen_gvec_fpst_zzz(DisasContext *s, gen_helper_gvec_3_ptr *fn,
172 int rd, int rn, int rm,
173 int data, ARMFPStatusFlavour flavour)
174 {
175 if (fn == NULL) {
176 return false;
177 }
178 if (sve_access_check(s)) {
179 unsigned vsz = vec_full_reg_size(s);
180 TCGv_ptr status = fpstatus_ptr(flavour);
181
182 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, rd),
183 vec_full_reg_offset(s, rn),
184 vec_full_reg_offset(s, rm),
185 status, vsz, vsz, data, fn);
186 }
187 return true;
188 }
189
gen_gvec_fpst_arg_zzz(DisasContext * s,gen_helper_gvec_3_ptr * fn,arg_rrr_esz * a,int data)190 static bool gen_gvec_fpst_arg_zzz(DisasContext *s, gen_helper_gvec_3_ptr *fn,
191 arg_rrr_esz *a, int data)
192 {
193 return gen_gvec_fpst_zzz(s, fn, a->rd, a->rn, a->rm, data,
194 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64);
195 }
196
gen_gvec_fpst_ah_arg_zzz(DisasContext * s,gen_helper_gvec_3_ptr * fn,arg_rrr_esz * a,int data)197 static bool gen_gvec_fpst_ah_arg_zzz(DisasContext *s, gen_helper_gvec_3_ptr *fn,
198 arg_rrr_esz *a, int data)
199 {
200 return gen_gvec_fpst_zzz(s, fn, a->rd, a->rn, a->rm, data,
201 select_ah_fpst(s, a->esz));
202 }
203
204 /* Invoke an out-of-line helper on 4 Zregs. */
gen_gvec_ool_zzzz(DisasContext * s,gen_helper_gvec_4 * fn,int rd,int rn,int rm,int ra,int data)205 static bool gen_gvec_ool_zzzz(DisasContext *s, gen_helper_gvec_4 *fn,
206 int rd, int rn, int rm, int ra, int data)
207 {
208 if (fn == NULL) {
209 return false;
210 }
211 if (sve_access_check(s)) {
212 unsigned vsz = vec_full_reg_size(s);
213 tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd),
214 vec_full_reg_offset(s, rn),
215 vec_full_reg_offset(s, rm),
216 vec_full_reg_offset(s, ra),
217 vsz, vsz, data, fn);
218 }
219 return true;
220 }
221
gen_gvec_ool_arg_zzzz(DisasContext * s,gen_helper_gvec_4 * fn,arg_rrrr_esz * a,int data)222 static bool gen_gvec_ool_arg_zzzz(DisasContext *s, gen_helper_gvec_4 *fn,
223 arg_rrrr_esz *a, int data)
224 {
225 return gen_gvec_ool_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, data);
226 }
227
gen_gvec_ool_arg_zzxz(DisasContext * s,gen_helper_gvec_4 * fn,arg_rrxr_esz * a)228 static bool gen_gvec_ool_arg_zzxz(DisasContext *s, gen_helper_gvec_4 *fn,
229 arg_rrxr_esz *a)
230 {
231 return gen_gvec_ool_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, a->index);
232 }
233
234 /* Invoke an out-of-line helper on 4 Zregs, plus a pointer. */
gen_gvec_ptr_zzzz(DisasContext * s,gen_helper_gvec_4_ptr * fn,int rd,int rn,int rm,int ra,int data,TCGv_ptr ptr)235 static bool gen_gvec_ptr_zzzz(DisasContext *s, gen_helper_gvec_4_ptr *fn,
236 int rd, int rn, int rm, int ra,
237 int data, TCGv_ptr ptr)
238 {
239 if (fn == NULL) {
240 return false;
241 }
242 if (sve_access_check(s)) {
243 unsigned vsz = vec_full_reg_size(s);
244 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, rd),
245 vec_full_reg_offset(s, rn),
246 vec_full_reg_offset(s, rm),
247 vec_full_reg_offset(s, ra),
248 ptr, vsz, vsz, data, fn);
249 }
250 return true;
251 }
252
gen_gvec_fpst_zzzz(DisasContext * s,gen_helper_gvec_4_ptr * fn,int rd,int rn,int rm,int ra,int data,ARMFPStatusFlavour flavour)253 static bool gen_gvec_fpst_zzzz(DisasContext *s, gen_helper_gvec_4_ptr *fn,
254 int rd, int rn, int rm, int ra,
255 int data, ARMFPStatusFlavour flavour)
256 {
257 TCGv_ptr status = fpstatus_ptr(flavour);
258 bool ret = gen_gvec_ptr_zzzz(s, fn, rd, rn, rm, ra, data, status);
259 return ret;
260 }
261
gen_gvec_env_zzzz(DisasContext * s,gen_helper_gvec_4_ptr * fn,int rd,int rn,int rm,int ra,int data)262 static bool gen_gvec_env_zzzz(DisasContext *s, gen_helper_gvec_4_ptr *fn,
263 int rd, int rn, int rm, int ra,
264 int data)
265 {
266 return gen_gvec_ptr_zzzz(s, fn, rd, rn, rm, ra, data, tcg_env);
267 }
268
gen_gvec_env_arg_zzzz(DisasContext * s,gen_helper_gvec_4_ptr * fn,arg_rrrr_esz * a,int data)269 static bool gen_gvec_env_arg_zzzz(DisasContext *s, gen_helper_gvec_4_ptr *fn,
270 arg_rrrr_esz *a, int data)
271 {
272 return gen_gvec_env_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, data);
273 }
274
gen_gvec_env_arg_zzxz(DisasContext * s,gen_helper_gvec_4_ptr * fn,arg_rrxr_esz * a)275 static bool gen_gvec_env_arg_zzxz(DisasContext *s, gen_helper_gvec_4_ptr *fn,
276 arg_rrxr_esz *a)
277 {
278 return gen_gvec_env_zzzz(s, fn, a->rd, a->rn, a->rm, a->ra, a->index);
279 }
280
281 /* Invoke an out-of-line helper on 4 Zregs, 1 Preg, plus fpst. */
gen_gvec_fpst_zzzzp(DisasContext * s,gen_helper_gvec_5_ptr * fn,int rd,int rn,int rm,int ra,int pg,int data,ARMFPStatusFlavour flavour)282 static bool gen_gvec_fpst_zzzzp(DisasContext *s, gen_helper_gvec_5_ptr *fn,
283 int rd, int rn, int rm, int ra, int pg,
284 int data, ARMFPStatusFlavour flavour)
285 {
286 if (fn == NULL) {
287 return false;
288 }
289 if (sve_access_check(s)) {
290 unsigned vsz = vec_full_reg_size(s);
291 TCGv_ptr status = fpstatus_ptr(flavour);
292
293 tcg_gen_gvec_5_ptr(vec_full_reg_offset(s, rd),
294 vec_full_reg_offset(s, rn),
295 vec_full_reg_offset(s, rm),
296 vec_full_reg_offset(s, ra),
297 pred_full_reg_offset(s, pg),
298 status, vsz, vsz, data, fn);
299 }
300 return true;
301 }
302
303 /* Invoke an out-of-line helper on 2 Zregs and a predicate. */
gen_gvec_ool_zzp(DisasContext * s,gen_helper_gvec_3 * fn,int rd,int rn,int pg,int data)304 static bool gen_gvec_ool_zzp(DisasContext *s, gen_helper_gvec_3 *fn,
305 int rd, int rn, int pg, int data)
306 {
307 if (fn == NULL) {
308 return false;
309 }
310 if (sve_access_check(s)) {
311 unsigned vsz = vec_full_reg_size(s);
312 tcg_gen_gvec_3_ool(vec_full_reg_offset(s, rd),
313 vec_full_reg_offset(s, rn),
314 pred_full_reg_offset(s, pg),
315 vsz, vsz, data, fn);
316 }
317 return true;
318 }
319
gen_gvec_ool_arg_zpz(DisasContext * s,gen_helper_gvec_3 * fn,arg_rpr_esz * a,int data)320 static bool gen_gvec_ool_arg_zpz(DisasContext *s, gen_helper_gvec_3 *fn,
321 arg_rpr_esz *a, int data)
322 {
323 return gen_gvec_ool_zzp(s, fn, a->rd, a->rn, a->pg, data);
324 }
325
gen_gvec_ool_arg_zpzi(DisasContext * s,gen_helper_gvec_3 * fn,arg_rpri_esz * a)326 static bool gen_gvec_ool_arg_zpzi(DisasContext *s, gen_helper_gvec_3 *fn,
327 arg_rpri_esz *a)
328 {
329 return gen_gvec_ool_zzp(s, fn, a->rd, a->rn, a->pg, a->imm);
330 }
331
gen_gvec_fpst_zzp(DisasContext * s,gen_helper_gvec_3_ptr * fn,int rd,int rn,int pg,int data,ARMFPStatusFlavour flavour)332 static bool gen_gvec_fpst_zzp(DisasContext *s, gen_helper_gvec_3_ptr *fn,
333 int rd, int rn, int pg, int data,
334 ARMFPStatusFlavour flavour)
335 {
336 if (fn == NULL) {
337 return false;
338 }
339 if (sve_access_check(s)) {
340 unsigned vsz = vec_full_reg_size(s);
341 TCGv_ptr status = fpstatus_ptr(flavour);
342
343 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, rd),
344 vec_full_reg_offset(s, rn),
345 pred_full_reg_offset(s, pg),
346 status, vsz, vsz, data, fn);
347 }
348 return true;
349 }
350
gen_gvec_fpst_arg_zpz(DisasContext * s,gen_helper_gvec_3_ptr * fn,arg_rpr_esz * a,int data,ARMFPStatusFlavour flavour)351 static bool gen_gvec_fpst_arg_zpz(DisasContext *s, gen_helper_gvec_3_ptr *fn,
352 arg_rpr_esz *a, int data,
353 ARMFPStatusFlavour flavour)
354 {
355 return gen_gvec_fpst_zzp(s, fn, a->rd, a->rn, a->pg, data, flavour);
356 }
357
358 /* Invoke an out-of-line helper on 3 Zregs and a predicate. */
gen_gvec_ool_zzzp(DisasContext * s,gen_helper_gvec_4 * fn,int rd,int rn,int rm,int pg,int data)359 static bool gen_gvec_ool_zzzp(DisasContext *s, gen_helper_gvec_4 *fn,
360 int rd, int rn, int rm, int pg, int data)
361 {
362 if (fn == NULL) {
363 return false;
364 }
365 if (sve_access_check(s)) {
366 unsigned vsz = vec_full_reg_size(s);
367 tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd),
368 vec_full_reg_offset(s, rn),
369 vec_full_reg_offset(s, rm),
370 pred_full_reg_offset(s, pg),
371 vsz, vsz, data, fn);
372 }
373 return true;
374 }
375
gen_gvec_ool_arg_zpzz(DisasContext * s,gen_helper_gvec_4 * fn,arg_rprr_esz * a,int data)376 static bool gen_gvec_ool_arg_zpzz(DisasContext *s, gen_helper_gvec_4 *fn,
377 arg_rprr_esz *a, int data)
378 {
379 return gen_gvec_ool_zzzp(s, fn, a->rd, a->rn, a->rm, a->pg, data);
380 }
381
382 /* Invoke an out-of-line helper on 3 Zregs and a predicate. */
gen_gvec_fpst_zzzp(DisasContext * s,gen_helper_gvec_4_ptr * fn,int rd,int rn,int rm,int pg,int data,ARMFPStatusFlavour flavour)383 static bool gen_gvec_fpst_zzzp(DisasContext *s, gen_helper_gvec_4_ptr *fn,
384 int rd, int rn, int rm, int pg, int data,
385 ARMFPStatusFlavour flavour)
386 {
387 if (fn == NULL) {
388 return false;
389 }
390 if (sve_access_check(s)) {
391 unsigned vsz = vec_full_reg_size(s);
392 TCGv_ptr status = fpstatus_ptr(flavour);
393
394 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, rd),
395 vec_full_reg_offset(s, rn),
396 vec_full_reg_offset(s, rm),
397 pred_full_reg_offset(s, pg),
398 status, vsz, vsz, data, fn);
399 }
400 return true;
401 }
402
gen_gvec_fpst_arg_zpzz(DisasContext * s,gen_helper_gvec_4_ptr * fn,arg_rprr_esz * a)403 static bool gen_gvec_fpst_arg_zpzz(DisasContext *s, gen_helper_gvec_4_ptr *fn,
404 arg_rprr_esz *a)
405 {
406 return gen_gvec_fpst_zzzp(s, fn, a->rd, a->rn, a->rm, a->pg, 0,
407 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64);
408 }
409
410 /* Invoke a vector expander on two Zregs and an immediate. */
gen_gvec_fn_zzi(DisasContext * s,GVecGen2iFn * gvec_fn,int esz,int rd,int rn,uint64_t imm)411 static bool gen_gvec_fn_zzi(DisasContext *s, GVecGen2iFn *gvec_fn,
412 int esz, int rd, int rn, uint64_t imm)
413 {
414 if (gvec_fn == NULL) {
415 return false;
416 }
417 if (sve_access_check(s)) {
418 unsigned vsz = vec_full_reg_size(s);
419 gvec_fn(esz, vec_full_reg_offset(s, rd),
420 vec_full_reg_offset(s, rn), imm, vsz, vsz);
421 }
422 return true;
423 }
424
gen_gvec_fn_arg_zzi(DisasContext * s,GVecGen2iFn * gvec_fn,arg_rri_esz * a)425 static bool gen_gvec_fn_arg_zzi(DisasContext *s, GVecGen2iFn *gvec_fn,
426 arg_rri_esz *a)
427 {
428 if (a->esz < 0) {
429 /* Invalid tsz encoding -- see tszimm_esz. */
430 return false;
431 }
432 return gen_gvec_fn_zzi(s, gvec_fn, a->esz, a->rd, a->rn, a->imm);
433 }
434
435 /* Invoke a vector expander on three Zregs. */
gen_gvec_fn_zzz(DisasContext * s,GVecGen3Fn * gvec_fn,int esz,int rd,int rn,int rm)436 static bool gen_gvec_fn_zzz(DisasContext *s, GVecGen3Fn *gvec_fn,
437 int esz, int rd, int rn, int rm)
438 {
439 if (gvec_fn == NULL) {
440 return false;
441 }
442 if (sve_access_check(s)) {
443 unsigned vsz = vec_full_reg_size(s);
444 gvec_fn(esz, vec_full_reg_offset(s, rd),
445 vec_full_reg_offset(s, rn),
446 vec_full_reg_offset(s, rm), vsz, vsz);
447 }
448 return true;
449 }
450
gen_gvec_fn_arg_zzz(DisasContext * s,GVecGen3Fn * fn,arg_rrr_esz * a)451 static bool gen_gvec_fn_arg_zzz(DisasContext *s, GVecGen3Fn *fn,
452 arg_rrr_esz *a)
453 {
454 return gen_gvec_fn_zzz(s, fn, a->esz, a->rd, a->rn, a->rm);
455 }
456
457 /* Invoke a vector expander on four Zregs. */
gen_gvec_fn_arg_zzzz(DisasContext * s,GVecGen4Fn * gvec_fn,arg_rrrr_esz * a)458 static bool gen_gvec_fn_arg_zzzz(DisasContext *s, GVecGen4Fn *gvec_fn,
459 arg_rrrr_esz *a)
460 {
461 if (gvec_fn == NULL) {
462 return false;
463 }
464 if (sve_access_check(s)) {
465 unsigned vsz = vec_full_reg_size(s);
466 gvec_fn(a->esz, vec_full_reg_offset(s, a->rd),
467 vec_full_reg_offset(s, a->rn),
468 vec_full_reg_offset(s, a->rm),
469 vec_full_reg_offset(s, a->ra), vsz, vsz);
470 }
471 return true;
472 }
473
474 /* Invoke a vector move on two Zregs. */
do_mov_z(DisasContext * s,int rd,int rn)475 static bool do_mov_z(DisasContext *s, int rd, int rn)
476 {
477 if (sve_access_check(s)) {
478 unsigned vsz = vec_full_reg_size(s);
479 tcg_gen_gvec_mov(MO_8, vec_full_reg_offset(s, rd),
480 vec_full_reg_offset(s, rn), vsz, vsz);
481 }
482 return true;
483 }
484
485 /* Initialize a Zreg with replications of a 64-bit immediate. */
do_dupi_z(DisasContext * s,int rd,uint64_t word)486 static void do_dupi_z(DisasContext *s, int rd, uint64_t word)
487 {
488 unsigned vsz = vec_full_reg_size(s);
489 tcg_gen_gvec_dup_imm(MO_64, vec_full_reg_offset(s, rd), vsz, vsz, word);
490 }
491
492 /* Invoke a vector expander on three Pregs. */
gen_gvec_fn_ppp(DisasContext * s,GVecGen3Fn * gvec_fn,int rd,int rn,int rm)493 static bool gen_gvec_fn_ppp(DisasContext *s, GVecGen3Fn *gvec_fn,
494 int rd, int rn, int rm)
495 {
496 if (sve_access_check(s)) {
497 unsigned psz = pred_gvec_reg_size(s);
498 gvec_fn(MO_64, pred_full_reg_offset(s, rd),
499 pred_full_reg_offset(s, rn),
500 pred_full_reg_offset(s, rm), psz, psz);
501 }
502 return true;
503 }
504
505 /* Invoke a vector move on two Pregs. */
do_mov_p(DisasContext * s,int rd,int rn)506 static bool do_mov_p(DisasContext *s, int rd, int rn)
507 {
508 if (sve_access_check(s)) {
509 unsigned psz = pred_gvec_reg_size(s);
510 tcg_gen_gvec_mov(MO_8, pred_full_reg_offset(s, rd),
511 pred_full_reg_offset(s, rn), psz, psz);
512 }
513 return true;
514 }
515
516 /* Set the cpu flags as per a return from an SVE helper. */
do_pred_flags(TCGv_i32 t)517 static void do_pred_flags(TCGv_i32 t)
518 {
519 tcg_gen_mov_i32(cpu_NF, t);
520 tcg_gen_andi_i32(cpu_ZF, t, 2);
521 tcg_gen_andi_i32(cpu_CF, t, 1);
522 tcg_gen_movi_i32(cpu_VF, 0);
523 }
524
525 /* Subroutines computing the ARM PredTest psuedofunction. */
do_predtest1(TCGv_i64 d,TCGv_i64 g)526 static void do_predtest1(TCGv_i64 d, TCGv_i64 g)
527 {
528 TCGv_i32 t = tcg_temp_new_i32();
529
530 gen_helper_sve_predtest1(t, d, g);
531 do_pred_flags(t);
532 }
533
do_predtest(DisasContext * s,int dofs,int gofs,int words)534 static void do_predtest(DisasContext *s, int dofs, int gofs, int words)
535 {
536 TCGv_ptr dptr = tcg_temp_new_ptr();
537 TCGv_ptr gptr = tcg_temp_new_ptr();
538 TCGv_i32 t = tcg_temp_new_i32();
539
540 tcg_gen_addi_ptr(dptr, tcg_env, dofs);
541 tcg_gen_addi_ptr(gptr, tcg_env, gofs);
542
543 gen_helper_sve_predtest(t, dptr, gptr, tcg_constant_i32(words));
544
545 do_pred_flags(t);
546 }
547
548 /* For each element size, the bits within a predicate word that are active. */
549 const uint64_t pred_esz_masks[5] = {
550 0xffffffffffffffffull, 0x5555555555555555ull,
551 0x1111111111111111ull, 0x0101010101010101ull,
552 0x0001000100010001ull,
553 };
554
trans_INVALID(DisasContext * s,arg_INVALID * a)555 static bool trans_INVALID(DisasContext *s, arg_INVALID *a)
556 {
557 unallocated_encoding(s);
558 return true;
559 }
560
561 /*
562 *** SVE Logical - Unpredicated Group
563 */
564
TRANS_FEAT(AND_zzz,aa64_sve,gen_gvec_fn_arg_zzz,tcg_gen_gvec_and,a)565 TRANS_FEAT(AND_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_and, a)
566 TRANS_FEAT(ORR_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_or, a)
567 TRANS_FEAT(EOR_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_xor, a)
568 TRANS_FEAT(BIC_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_andc, a)
569
570 static bool trans_XAR(DisasContext *s, arg_rrri_esz *a)
571 {
572 if (a->esz < 0 || !dc_isar_feature(aa64_sve2, s)) {
573 return false;
574 }
575 if (sve_access_check(s)) {
576 unsigned vsz = vec_full_reg_size(s);
577 gen_gvec_xar(a->esz, vec_full_reg_offset(s, a->rd),
578 vec_full_reg_offset(s, a->rn),
579 vec_full_reg_offset(s, a->rm), a->imm, vsz, vsz);
580 }
581 return true;
582 }
583
TRANS_FEAT(EOR3,aa64_sve2,gen_gvec_fn_arg_zzzz,gen_gvec_eor3,a)584 TRANS_FEAT(EOR3, aa64_sve2, gen_gvec_fn_arg_zzzz, gen_gvec_eor3, a)
585 TRANS_FEAT(BCAX, aa64_sve2, gen_gvec_fn_arg_zzzz, gen_gvec_bcax, a)
586
587 static void gen_bsl(unsigned vece, uint32_t d, uint32_t n, uint32_t m,
588 uint32_t a, uint32_t oprsz, uint32_t maxsz)
589 {
590 /* BSL differs from the generic bitsel in argument ordering. */
591 tcg_gen_gvec_bitsel(vece, d, a, n, m, oprsz, maxsz);
592 }
593
TRANS_FEAT(BSL,aa64_sve2,gen_gvec_fn_arg_zzzz,gen_bsl,a)594 TRANS_FEAT(BSL, aa64_sve2, gen_gvec_fn_arg_zzzz, gen_bsl, a)
595
596 static void gen_bsl1n_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 k)
597 {
598 tcg_gen_andc_i64(n, k, n);
599 tcg_gen_andc_i64(m, m, k);
600 tcg_gen_or_i64(d, n, m);
601 }
602
gen_bsl1n_vec(unsigned vece,TCGv_vec d,TCGv_vec n,TCGv_vec m,TCGv_vec k)603 static void gen_bsl1n_vec(unsigned vece, TCGv_vec d, TCGv_vec n,
604 TCGv_vec m, TCGv_vec k)
605 {
606 tcg_gen_not_vec(vece, n, n);
607 tcg_gen_bitsel_vec(vece, d, k, n, m);
608 }
609
gen_bsl1n(unsigned vece,uint32_t d,uint32_t n,uint32_t m,uint32_t a,uint32_t oprsz,uint32_t maxsz)610 static void gen_bsl1n(unsigned vece, uint32_t d, uint32_t n, uint32_t m,
611 uint32_t a, uint32_t oprsz, uint32_t maxsz)
612 {
613 static const GVecGen4 op = {
614 .fni8 = gen_bsl1n_i64,
615 .fniv = gen_bsl1n_vec,
616 .fno = gen_helper_sve2_bsl1n,
617 .vece = MO_64,
618 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
619 };
620 tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op);
621 }
622
TRANS_FEAT(BSL1N,aa64_sve2,gen_gvec_fn_arg_zzzz,gen_bsl1n,a)623 TRANS_FEAT(BSL1N, aa64_sve2, gen_gvec_fn_arg_zzzz, gen_bsl1n, a)
624
625 static void gen_bsl2n_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 k)
626 {
627 /*
628 * Z[dn] = (n & k) | (~m & ~k)
629 * = | ~(m | k)
630 */
631 tcg_gen_and_i64(n, n, k);
632 if (tcg_op_supported(INDEX_op_orc, TCG_TYPE_I64, 0)) {
633 tcg_gen_or_i64(m, m, k);
634 tcg_gen_orc_i64(d, n, m);
635 } else {
636 tcg_gen_nor_i64(m, m, k);
637 tcg_gen_or_i64(d, n, m);
638 }
639 }
640
gen_bsl2n_vec(unsigned vece,TCGv_vec d,TCGv_vec n,TCGv_vec m,TCGv_vec k)641 static void gen_bsl2n_vec(unsigned vece, TCGv_vec d, TCGv_vec n,
642 TCGv_vec m, TCGv_vec k)
643 {
644 tcg_gen_not_vec(vece, m, m);
645 tcg_gen_bitsel_vec(vece, d, k, n, m);
646 }
647
gen_bsl2n(unsigned vece,uint32_t d,uint32_t n,uint32_t m,uint32_t a,uint32_t oprsz,uint32_t maxsz)648 static void gen_bsl2n(unsigned vece, uint32_t d, uint32_t n, uint32_t m,
649 uint32_t a, uint32_t oprsz, uint32_t maxsz)
650 {
651 static const GVecGen4 op = {
652 .fni8 = gen_bsl2n_i64,
653 .fniv = gen_bsl2n_vec,
654 .fno = gen_helper_sve2_bsl2n,
655 .vece = MO_64,
656 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
657 };
658 tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op);
659 }
660
TRANS_FEAT(BSL2N,aa64_sve2,gen_gvec_fn_arg_zzzz,gen_bsl2n,a)661 TRANS_FEAT(BSL2N, aa64_sve2, gen_gvec_fn_arg_zzzz, gen_bsl2n, a)
662
663 static void gen_nbsl_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 k)
664 {
665 tcg_gen_and_i64(n, n, k);
666 tcg_gen_andc_i64(m, m, k);
667 tcg_gen_nor_i64(d, n, m);
668 }
669
gen_nbsl_vec(unsigned vece,TCGv_vec d,TCGv_vec n,TCGv_vec m,TCGv_vec k)670 static void gen_nbsl_vec(unsigned vece, TCGv_vec d, TCGv_vec n,
671 TCGv_vec m, TCGv_vec k)
672 {
673 tcg_gen_bitsel_vec(vece, d, k, n, m);
674 tcg_gen_not_vec(vece, d, d);
675 }
676
gen_nbsl(unsigned vece,uint32_t d,uint32_t n,uint32_t m,uint32_t a,uint32_t oprsz,uint32_t maxsz)677 static void gen_nbsl(unsigned vece, uint32_t d, uint32_t n, uint32_t m,
678 uint32_t a, uint32_t oprsz, uint32_t maxsz)
679 {
680 static const GVecGen4 op = {
681 .fni8 = gen_nbsl_i64,
682 .fniv = gen_nbsl_vec,
683 .fno = gen_helper_sve2_nbsl,
684 .vece = MO_64,
685 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
686 };
687 tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &op);
688 }
689
TRANS_FEAT(NBSL,aa64_sve2,gen_gvec_fn_arg_zzzz,gen_nbsl,a)690 TRANS_FEAT(NBSL, aa64_sve2, gen_gvec_fn_arg_zzzz, gen_nbsl, a)
691
692 /*
693 *** SVE Integer Arithmetic - Unpredicated Group
694 */
695
696 TRANS_FEAT(ADD_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_add, a)
697 TRANS_FEAT(SUB_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_sub, a)
698 TRANS_FEAT(SQADD_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_ssadd, a)
699 TRANS_FEAT(SQSUB_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_sssub, a)
700 TRANS_FEAT(UQADD_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_usadd, a)
701 TRANS_FEAT(UQSUB_zzz, aa64_sve, gen_gvec_fn_arg_zzz, tcg_gen_gvec_ussub, a)
702
703 /*
704 *** SVE Integer Arithmetic - Binary Predicated Group
705 */
706
707 /* Select active elememnts from Zn and inactive elements from Zm,
708 * storing the result in Zd.
709 */
710 static bool do_sel_z(DisasContext *s, int rd, int rn, int rm, int pg, int esz)
711 {
712 static gen_helper_gvec_4 * const fns[4] = {
713 gen_helper_sve_sel_zpzz_b, gen_helper_sve_sel_zpzz_h,
714 gen_helper_sve_sel_zpzz_s, gen_helper_sve_sel_zpzz_d
715 };
716 return gen_gvec_ool_zzzp(s, fns[esz], rd, rn, rm, pg, 0);
717 }
718
719 #define DO_ZPZZ(NAME, FEAT, name) \
720 static gen_helper_gvec_4 * const name##_zpzz_fns[4] = { \
721 gen_helper_##name##_zpzz_b, gen_helper_##name##_zpzz_h, \
722 gen_helper_##name##_zpzz_s, gen_helper_##name##_zpzz_d, \
723 }; \
724 TRANS_FEAT(NAME, FEAT, gen_gvec_ool_arg_zpzz, \
725 name##_zpzz_fns[a->esz], a, 0)
726
727 DO_ZPZZ(AND_zpzz, aa64_sve, sve_and)
728 DO_ZPZZ(EOR_zpzz, aa64_sve, sve_eor)
729 DO_ZPZZ(ORR_zpzz, aa64_sve, sve_orr)
730 DO_ZPZZ(BIC_zpzz, aa64_sve, sve_bic)
731
732 DO_ZPZZ(ADD_zpzz, aa64_sve, sve_add)
733 DO_ZPZZ(SUB_zpzz, aa64_sve, sve_sub)
734
735 DO_ZPZZ(SMAX_zpzz, aa64_sve, sve_smax)
736 DO_ZPZZ(UMAX_zpzz, aa64_sve, sve_umax)
737 DO_ZPZZ(SMIN_zpzz, aa64_sve, sve_smin)
738 DO_ZPZZ(UMIN_zpzz, aa64_sve, sve_umin)
739 DO_ZPZZ(SABD_zpzz, aa64_sve, sve_sabd)
740 DO_ZPZZ(UABD_zpzz, aa64_sve, sve_uabd)
741
742 DO_ZPZZ(MUL_zpzz, aa64_sve, sve_mul)
743 DO_ZPZZ(SMULH_zpzz, aa64_sve, sve_smulh)
744 DO_ZPZZ(UMULH_zpzz, aa64_sve, sve_umulh)
745
746 DO_ZPZZ(ASR_zpzz, aa64_sve, sve_asr)
747 DO_ZPZZ(LSR_zpzz, aa64_sve, sve_lsr)
748 DO_ZPZZ(LSL_zpzz, aa64_sve, sve_lsl)
749
750 static gen_helper_gvec_4 * const sdiv_fns[4] = {
751 NULL, NULL, gen_helper_sve_sdiv_zpzz_s, gen_helper_sve_sdiv_zpzz_d
752 };
753 TRANS_FEAT(SDIV_zpzz, aa64_sve, gen_gvec_ool_arg_zpzz, sdiv_fns[a->esz], a, 0)
754
755 static gen_helper_gvec_4 * const udiv_fns[4] = {
756 NULL, NULL, gen_helper_sve_udiv_zpzz_s, gen_helper_sve_udiv_zpzz_d
757 };
758 TRANS_FEAT(UDIV_zpzz, aa64_sve, gen_gvec_ool_arg_zpzz, udiv_fns[a->esz], a, 0)
759
760 TRANS_FEAT(SEL_zpzz, aa64_sve, do_sel_z, a->rd, a->rn, a->rm, a->pg, a->esz)
761
762 /*
763 *** SVE Integer Arithmetic - Unary Predicated Group
764 */
765
766 #define DO_ZPZ(NAME, FEAT, name) \
767 static gen_helper_gvec_3 * const name##_fns[4] = { \
768 gen_helper_##name##_b, gen_helper_##name##_h, \
769 gen_helper_##name##_s, gen_helper_##name##_d, \
770 }; \
771 TRANS_FEAT(NAME, FEAT, gen_gvec_ool_arg_zpz, name##_fns[a->esz], a, 0)
772
773 DO_ZPZ(CLS, aa64_sve, sve_cls)
774 DO_ZPZ(CLZ, aa64_sve, sve_clz)
775 DO_ZPZ(CNT_zpz, aa64_sve, sve_cnt_zpz)
776 DO_ZPZ(CNOT, aa64_sve, sve_cnot)
777 DO_ZPZ(NOT_zpz, aa64_sve, sve_not_zpz)
778 DO_ZPZ(ABS, aa64_sve, sve_abs)
779 DO_ZPZ(NEG, aa64_sve, sve_neg)
780 DO_ZPZ(RBIT, aa64_sve, sve_rbit)
781
782 static gen_helper_gvec_3 * const fabs_fns[4] = {
783 NULL, gen_helper_sve_fabs_h,
784 gen_helper_sve_fabs_s, gen_helper_sve_fabs_d,
785 };
786 static gen_helper_gvec_3 * const fabs_ah_fns[4] = {
787 NULL, gen_helper_sve_ah_fabs_h,
788 gen_helper_sve_ah_fabs_s, gen_helper_sve_ah_fabs_d,
789 };
790 TRANS_FEAT(FABS, aa64_sve, gen_gvec_ool_arg_zpz,
791 s->fpcr_ah ? fabs_ah_fns[a->esz] : fabs_fns[a->esz], a, 0)
792
793 static gen_helper_gvec_3 * const fneg_fns[4] = {
794 NULL, gen_helper_sve_fneg_h,
795 gen_helper_sve_fneg_s, gen_helper_sve_fneg_d,
796 };
797 static gen_helper_gvec_3 * const fneg_ah_fns[4] = {
798 NULL, gen_helper_sve_ah_fneg_h,
799 gen_helper_sve_ah_fneg_s, gen_helper_sve_ah_fneg_d,
800 };
801 TRANS_FEAT(FNEG, aa64_sve, gen_gvec_ool_arg_zpz,
802 s->fpcr_ah ? fneg_ah_fns[a->esz] : fneg_fns[a->esz], a, 0)
803
804 static gen_helper_gvec_3 * const sxtb_fns[4] = {
805 NULL, gen_helper_sve_sxtb_h,
806 gen_helper_sve_sxtb_s, gen_helper_sve_sxtb_d,
807 };
808 TRANS_FEAT(SXTB, aa64_sve, gen_gvec_ool_arg_zpz, sxtb_fns[a->esz], a, 0)
809
810 static gen_helper_gvec_3 * const uxtb_fns[4] = {
811 NULL, gen_helper_sve_uxtb_h,
812 gen_helper_sve_uxtb_s, gen_helper_sve_uxtb_d,
813 };
814 TRANS_FEAT(UXTB, aa64_sve, gen_gvec_ool_arg_zpz, uxtb_fns[a->esz], a, 0)
815
816 static gen_helper_gvec_3 * const sxth_fns[4] = {
817 NULL, NULL, gen_helper_sve_sxth_s, gen_helper_sve_sxth_d
818 };
819 TRANS_FEAT(SXTH, aa64_sve, gen_gvec_ool_arg_zpz, sxth_fns[a->esz], a, 0)
820
821 static gen_helper_gvec_3 * const uxth_fns[4] = {
822 NULL, NULL, gen_helper_sve_uxth_s, gen_helper_sve_uxth_d
823 };
824 TRANS_FEAT(UXTH, aa64_sve, gen_gvec_ool_arg_zpz, uxth_fns[a->esz], a, 0)
825
826 TRANS_FEAT(SXTW, aa64_sve, gen_gvec_ool_arg_zpz,
827 a->esz == 3 ? gen_helper_sve_sxtw_d : NULL, a, 0)
828 TRANS_FEAT(UXTW, aa64_sve, gen_gvec_ool_arg_zpz,
829 a->esz == 3 ? gen_helper_sve_uxtw_d : NULL, a, 0)
830
831 /*
832 *** SVE Integer Reduction Group
833 */
834
835 typedef void gen_helper_gvec_reduc(TCGv_i64, TCGv_ptr, TCGv_ptr, TCGv_i32);
do_vpz_ool(DisasContext * s,arg_rpr_esz * a,gen_helper_gvec_reduc * fn)836 static bool do_vpz_ool(DisasContext *s, arg_rpr_esz *a,
837 gen_helper_gvec_reduc *fn)
838 {
839 unsigned vsz = vec_full_reg_size(s);
840 TCGv_ptr t_zn, t_pg;
841 TCGv_i32 desc;
842 TCGv_i64 temp;
843
844 if (fn == NULL) {
845 return false;
846 }
847 if (!sve_access_check(s)) {
848 return true;
849 }
850
851 desc = tcg_constant_i32(simd_desc(vsz, vsz, 0));
852 temp = tcg_temp_new_i64();
853 t_zn = tcg_temp_new_ptr();
854 t_pg = tcg_temp_new_ptr();
855
856 tcg_gen_addi_ptr(t_zn, tcg_env, vec_full_reg_offset(s, a->rn));
857 tcg_gen_addi_ptr(t_pg, tcg_env, pred_full_reg_offset(s, a->pg));
858 fn(temp, t_zn, t_pg, desc);
859
860 write_fp_dreg(s, a->rd, temp);
861 return true;
862 }
863
864 #define DO_VPZ(NAME, name) \
865 static gen_helper_gvec_reduc * const name##_fns[4] = { \
866 gen_helper_sve_##name##_b, gen_helper_sve_##name##_h, \
867 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \
868 }; \
869 TRANS_FEAT(NAME, aa64_sve, do_vpz_ool, a, name##_fns[a->esz])
870
871 DO_VPZ(ORV, orv)
872 DO_VPZ(ANDV, andv)
873 DO_VPZ(EORV, eorv)
874
875 DO_VPZ(UADDV, uaddv)
876 DO_VPZ(SMAXV, smaxv)
877 DO_VPZ(UMAXV, umaxv)
878 DO_VPZ(SMINV, sminv)
879 DO_VPZ(UMINV, uminv)
880
881 static gen_helper_gvec_reduc * const saddv_fns[4] = {
882 gen_helper_sve_saddv_b, gen_helper_sve_saddv_h,
883 gen_helper_sve_saddv_s, NULL
884 };
TRANS_FEAT(SADDV,aa64_sve,do_vpz_ool,a,saddv_fns[a->esz])885 TRANS_FEAT(SADDV, aa64_sve, do_vpz_ool, a, saddv_fns[a->esz])
886
887 #undef DO_VPZ
888
889 /*
890 *** SVE Shift by Immediate - Predicated Group
891 */
892
893 /*
894 * Copy Zn into Zd, storing zeros into inactive elements.
895 * If invert, store zeros into the active elements.
896 */
897 static bool do_movz_zpz(DisasContext *s, int rd, int rn, int pg,
898 int esz, bool invert)
899 {
900 static gen_helper_gvec_3 * const fns[4] = {
901 gen_helper_sve_movz_b, gen_helper_sve_movz_h,
902 gen_helper_sve_movz_s, gen_helper_sve_movz_d,
903 };
904 return gen_gvec_ool_zzp(s, fns[esz], rd, rn, pg, invert);
905 }
906
do_shift_zpzi(DisasContext * s,arg_rpri_esz * a,bool asr,gen_helper_gvec_3 * const fns[4])907 static bool do_shift_zpzi(DisasContext *s, arg_rpri_esz *a, bool asr,
908 gen_helper_gvec_3 * const fns[4])
909 {
910 int max;
911
912 if (a->esz < 0) {
913 /* Invalid tsz encoding -- see tszimm_esz. */
914 return false;
915 }
916
917 /*
918 * Shift by element size is architecturally valid.
919 * For arithmetic right-shift, it's the same as by one less.
920 * For logical shifts and ASRD, it is a zeroing operation.
921 */
922 max = 8 << a->esz;
923 if (a->imm >= max) {
924 if (asr) {
925 a->imm = max - 1;
926 } else {
927 return do_movz_zpz(s, a->rd, a->rd, a->pg, a->esz, true);
928 }
929 }
930 return gen_gvec_ool_arg_zpzi(s, fns[a->esz], a);
931 }
932
933 static gen_helper_gvec_3 * const asr_zpzi_fns[4] = {
934 gen_helper_sve_asr_zpzi_b, gen_helper_sve_asr_zpzi_h,
935 gen_helper_sve_asr_zpzi_s, gen_helper_sve_asr_zpzi_d,
936 };
937 TRANS_FEAT(ASR_zpzi, aa64_sve, do_shift_zpzi, a, true, asr_zpzi_fns)
938
939 static gen_helper_gvec_3 * const lsr_zpzi_fns[4] = {
940 gen_helper_sve_lsr_zpzi_b, gen_helper_sve_lsr_zpzi_h,
941 gen_helper_sve_lsr_zpzi_s, gen_helper_sve_lsr_zpzi_d,
942 };
943 TRANS_FEAT(LSR_zpzi, aa64_sve, do_shift_zpzi, a, false, lsr_zpzi_fns)
944
945 static gen_helper_gvec_3 * const lsl_zpzi_fns[4] = {
946 gen_helper_sve_lsl_zpzi_b, gen_helper_sve_lsl_zpzi_h,
947 gen_helper_sve_lsl_zpzi_s, gen_helper_sve_lsl_zpzi_d,
948 };
949 TRANS_FEAT(LSL_zpzi, aa64_sve, do_shift_zpzi, a, false, lsl_zpzi_fns)
950
951 static gen_helper_gvec_3 * const asrd_fns[4] = {
952 gen_helper_sve_asrd_b, gen_helper_sve_asrd_h,
953 gen_helper_sve_asrd_s, gen_helper_sve_asrd_d,
954 };
955 TRANS_FEAT(ASRD, aa64_sve, do_shift_zpzi, a, false, asrd_fns)
956
957 static gen_helper_gvec_3 * const sqshl_zpzi_fns[4] = {
958 gen_helper_sve2_sqshl_zpzi_b, gen_helper_sve2_sqshl_zpzi_h,
959 gen_helper_sve2_sqshl_zpzi_s, gen_helper_sve2_sqshl_zpzi_d,
960 };
961 TRANS_FEAT(SQSHL_zpzi, aa64_sve2, gen_gvec_ool_arg_zpzi,
962 a->esz < 0 ? NULL : sqshl_zpzi_fns[a->esz], a)
963
964 static gen_helper_gvec_3 * const uqshl_zpzi_fns[4] = {
965 gen_helper_sve2_uqshl_zpzi_b, gen_helper_sve2_uqshl_zpzi_h,
966 gen_helper_sve2_uqshl_zpzi_s, gen_helper_sve2_uqshl_zpzi_d,
967 };
968 TRANS_FEAT(UQSHL_zpzi, aa64_sve2, gen_gvec_ool_arg_zpzi,
969 a->esz < 0 ? NULL : uqshl_zpzi_fns[a->esz], a)
970
971 static gen_helper_gvec_3 * const srshr_fns[4] = {
972 gen_helper_sve2_srshr_b, gen_helper_sve2_srshr_h,
973 gen_helper_sve2_srshr_s, gen_helper_sve2_srshr_d,
974 };
975 TRANS_FEAT(SRSHR, aa64_sve2, gen_gvec_ool_arg_zpzi,
976 a->esz < 0 ? NULL : srshr_fns[a->esz], a)
977
978 static gen_helper_gvec_3 * const urshr_fns[4] = {
979 gen_helper_sve2_urshr_b, gen_helper_sve2_urshr_h,
980 gen_helper_sve2_urshr_s, gen_helper_sve2_urshr_d,
981 };
982 TRANS_FEAT(URSHR, aa64_sve2, gen_gvec_ool_arg_zpzi,
983 a->esz < 0 ? NULL : urshr_fns[a->esz], a)
984
985 static gen_helper_gvec_3 * const sqshlu_fns[4] = {
986 gen_helper_sve2_sqshlu_b, gen_helper_sve2_sqshlu_h,
987 gen_helper_sve2_sqshlu_s, gen_helper_sve2_sqshlu_d,
988 };
989 TRANS_FEAT(SQSHLU, aa64_sve2, gen_gvec_ool_arg_zpzi,
990 a->esz < 0 ? NULL : sqshlu_fns[a->esz], a)
991
992 /*
993 *** SVE Bitwise Shift - Predicated Group
994 */
995
996 #define DO_ZPZW(NAME, name) \
997 static gen_helper_gvec_4 * const name##_zpzw_fns[4] = { \
998 gen_helper_sve_##name##_zpzw_b, gen_helper_sve_##name##_zpzw_h, \
999 gen_helper_sve_##name##_zpzw_s, NULL \
1000 }; \
1001 TRANS_FEAT(NAME##_zpzw, aa64_sve, gen_gvec_ool_arg_zpzz, \
1002 a->esz < 0 ? NULL : name##_zpzw_fns[a->esz], a, 0)
1003
DO_ZPZW(ASR,asr)1004 DO_ZPZW(ASR, asr)
1005 DO_ZPZW(LSR, lsr)
1006 DO_ZPZW(LSL, lsl)
1007
1008 #undef DO_ZPZW
1009
1010 /*
1011 *** SVE Bitwise Shift - Unpredicated Group
1012 */
1013
1014 static bool do_shift_imm(DisasContext *s, arg_rri_esz *a, bool asr,
1015 void (*gvec_fn)(unsigned, uint32_t, uint32_t,
1016 int64_t, uint32_t, uint32_t))
1017 {
1018 if (a->esz < 0) {
1019 /* Invalid tsz encoding -- see tszimm_esz. */
1020 return false;
1021 }
1022 if (sve_access_check(s)) {
1023 unsigned vsz = vec_full_reg_size(s);
1024 /* Shift by element size is architecturally valid. For
1025 arithmetic right-shift, it's the same as by one less.
1026 Otherwise it is a zeroing operation. */
1027 if (a->imm >= 8 << a->esz) {
1028 if (asr) {
1029 a->imm = (8 << a->esz) - 1;
1030 } else {
1031 do_dupi_z(s, a->rd, 0);
1032 return true;
1033 }
1034 }
1035 gvec_fn(a->esz, vec_full_reg_offset(s, a->rd),
1036 vec_full_reg_offset(s, a->rn), a->imm, vsz, vsz);
1037 }
1038 return true;
1039 }
1040
TRANS_FEAT(ASR_zzi,aa64_sve,do_shift_imm,a,true,tcg_gen_gvec_sari)1041 TRANS_FEAT(ASR_zzi, aa64_sve, do_shift_imm, a, true, tcg_gen_gvec_sari)
1042 TRANS_FEAT(LSR_zzi, aa64_sve, do_shift_imm, a, false, tcg_gen_gvec_shri)
1043 TRANS_FEAT(LSL_zzi, aa64_sve, do_shift_imm, a, false, tcg_gen_gvec_shli)
1044
1045 #define DO_ZZW(NAME, name) \
1046 static gen_helper_gvec_3 * const name##_zzw_fns[4] = { \
1047 gen_helper_sve_##name##_zzw_b, gen_helper_sve_##name##_zzw_h, \
1048 gen_helper_sve_##name##_zzw_s, NULL \
1049 }; \
1050 TRANS_FEAT(NAME, aa64_sve, gen_gvec_ool_arg_zzz, \
1051 name##_zzw_fns[a->esz], a, 0)
1052
1053 DO_ZZW(ASR_zzw, asr)
1054 DO_ZZW(LSR_zzw, lsr)
1055 DO_ZZW(LSL_zzw, lsl)
1056
1057 #undef DO_ZZW
1058
1059 /*
1060 *** SVE Integer Multiply-Add Group
1061 */
1062
1063 static bool do_zpzzz_ool(DisasContext *s, arg_rprrr_esz *a,
1064 gen_helper_gvec_5 *fn)
1065 {
1066 if (sve_access_check(s)) {
1067 unsigned vsz = vec_full_reg_size(s);
1068 tcg_gen_gvec_5_ool(vec_full_reg_offset(s, a->rd),
1069 vec_full_reg_offset(s, a->ra),
1070 vec_full_reg_offset(s, a->rn),
1071 vec_full_reg_offset(s, a->rm),
1072 pred_full_reg_offset(s, a->pg),
1073 vsz, vsz, 0, fn);
1074 }
1075 return true;
1076 }
1077
1078 static gen_helper_gvec_5 * const mla_fns[4] = {
1079 gen_helper_sve_mla_b, gen_helper_sve_mla_h,
1080 gen_helper_sve_mla_s, gen_helper_sve_mla_d,
1081 };
1082 TRANS_FEAT(MLA, aa64_sve, do_zpzzz_ool, a, mla_fns[a->esz])
1083
1084 static gen_helper_gvec_5 * const mls_fns[4] = {
1085 gen_helper_sve_mls_b, gen_helper_sve_mls_h,
1086 gen_helper_sve_mls_s, gen_helper_sve_mls_d,
1087 };
TRANS_FEAT(MLS,aa64_sve,do_zpzzz_ool,a,mls_fns[a->esz])1088 TRANS_FEAT(MLS, aa64_sve, do_zpzzz_ool, a, mls_fns[a->esz])
1089
1090 /*
1091 *** SVE Index Generation Group
1092 */
1093
1094 static bool do_index(DisasContext *s, int esz, int rd,
1095 TCGv_i64 start, TCGv_i64 incr)
1096 {
1097 unsigned vsz;
1098 TCGv_i32 desc;
1099 TCGv_ptr t_zd;
1100
1101 if (!sve_access_check(s)) {
1102 return true;
1103 }
1104
1105 vsz = vec_full_reg_size(s);
1106 desc = tcg_constant_i32(simd_desc(vsz, vsz, 0));
1107 t_zd = tcg_temp_new_ptr();
1108
1109 tcg_gen_addi_ptr(t_zd, tcg_env, vec_full_reg_offset(s, rd));
1110 if (esz == 3) {
1111 gen_helper_sve_index_d(t_zd, start, incr, desc);
1112 } else {
1113 typedef void index_fn(TCGv_ptr, TCGv_i32, TCGv_i32, TCGv_i32);
1114 static index_fn * const fns[3] = {
1115 gen_helper_sve_index_b,
1116 gen_helper_sve_index_h,
1117 gen_helper_sve_index_s,
1118 };
1119 TCGv_i32 s32 = tcg_temp_new_i32();
1120 TCGv_i32 i32 = tcg_temp_new_i32();
1121
1122 tcg_gen_extrl_i64_i32(s32, start);
1123 tcg_gen_extrl_i64_i32(i32, incr);
1124 fns[esz](t_zd, s32, i32, desc);
1125 }
1126 return true;
1127 }
1128
1129 TRANS_FEAT(INDEX_ii, aa64_sve, do_index, a->esz, a->rd,
1130 tcg_constant_i64(a->imm1), tcg_constant_i64(a->imm2))
1131 TRANS_FEAT(INDEX_ir, aa64_sve, do_index, a->esz, a->rd,
1132 tcg_constant_i64(a->imm), cpu_reg(s, a->rm))
1133 TRANS_FEAT(INDEX_ri, aa64_sve, do_index, a->esz, a->rd,
1134 cpu_reg(s, a->rn), tcg_constant_i64(a->imm))
1135 TRANS_FEAT(INDEX_rr, aa64_sve, do_index, a->esz, a->rd,
1136 cpu_reg(s, a->rn), cpu_reg(s, a->rm))
1137
1138 /*
1139 *** SVE Stack Allocation Group
1140 */
1141
trans_ADDVL(DisasContext * s,arg_ADDVL * a)1142 static bool trans_ADDVL(DisasContext *s, arg_ADDVL *a)
1143 {
1144 if (!dc_isar_feature(aa64_sve, s)) {
1145 return false;
1146 }
1147 if (sve_access_check(s)) {
1148 TCGv_i64 rd = cpu_reg_sp(s, a->rd);
1149 TCGv_i64 rn = cpu_reg_sp(s, a->rn);
1150 tcg_gen_addi_i64(rd, rn, a->imm * vec_full_reg_size(s));
1151 }
1152 return true;
1153 }
1154
trans_ADDSVL(DisasContext * s,arg_ADDSVL * a)1155 static bool trans_ADDSVL(DisasContext *s, arg_ADDSVL *a)
1156 {
1157 if (!dc_isar_feature(aa64_sme, s)) {
1158 return false;
1159 }
1160 if (sme_enabled_check(s)) {
1161 TCGv_i64 rd = cpu_reg_sp(s, a->rd);
1162 TCGv_i64 rn = cpu_reg_sp(s, a->rn);
1163 tcg_gen_addi_i64(rd, rn, a->imm * streaming_vec_reg_size(s));
1164 }
1165 return true;
1166 }
1167
trans_ADDPL(DisasContext * s,arg_ADDPL * a)1168 static bool trans_ADDPL(DisasContext *s, arg_ADDPL *a)
1169 {
1170 if (!dc_isar_feature(aa64_sve, s)) {
1171 return false;
1172 }
1173 if (sve_access_check(s)) {
1174 TCGv_i64 rd = cpu_reg_sp(s, a->rd);
1175 TCGv_i64 rn = cpu_reg_sp(s, a->rn);
1176 tcg_gen_addi_i64(rd, rn, a->imm * pred_full_reg_size(s));
1177 }
1178 return true;
1179 }
1180
trans_ADDSPL(DisasContext * s,arg_ADDSPL * a)1181 static bool trans_ADDSPL(DisasContext *s, arg_ADDSPL *a)
1182 {
1183 if (!dc_isar_feature(aa64_sme, s)) {
1184 return false;
1185 }
1186 if (sme_enabled_check(s)) {
1187 TCGv_i64 rd = cpu_reg_sp(s, a->rd);
1188 TCGv_i64 rn = cpu_reg_sp(s, a->rn);
1189 tcg_gen_addi_i64(rd, rn, a->imm * streaming_pred_reg_size(s));
1190 }
1191 return true;
1192 }
1193
trans_RDVL(DisasContext * s,arg_RDVL * a)1194 static bool trans_RDVL(DisasContext *s, arg_RDVL *a)
1195 {
1196 if (!dc_isar_feature(aa64_sve, s)) {
1197 return false;
1198 }
1199 if (sve_access_check(s)) {
1200 TCGv_i64 reg = cpu_reg(s, a->rd);
1201 tcg_gen_movi_i64(reg, a->imm * vec_full_reg_size(s));
1202 }
1203 return true;
1204 }
1205
trans_RDSVL(DisasContext * s,arg_RDSVL * a)1206 static bool trans_RDSVL(DisasContext *s, arg_RDSVL *a)
1207 {
1208 if (!dc_isar_feature(aa64_sme, s)) {
1209 return false;
1210 }
1211 if (sme_enabled_check(s)) {
1212 TCGv_i64 reg = cpu_reg(s, a->rd);
1213 tcg_gen_movi_i64(reg, a->imm * streaming_vec_reg_size(s));
1214 }
1215 return true;
1216 }
1217
1218 /*
1219 *** SVE Compute Vector Address Group
1220 */
1221
do_adr(DisasContext * s,arg_rrri * a,gen_helper_gvec_3 * fn)1222 static bool do_adr(DisasContext *s, arg_rrri *a, gen_helper_gvec_3 *fn)
1223 {
1224 return gen_gvec_ool_zzz(s, fn, a->rd, a->rn, a->rm, a->imm);
1225 }
1226
1227 TRANS_FEAT_NONSTREAMING(ADR_p32, aa64_sve, do_adr, a, gen_helper_sve_adr_p32)
1228 TRANS_FEAT_NONSTREAMING(ADR_p64, aa64_sve, do_adr, a, gen_helper_sve_adr_p64)
1229 TRANS_FEAT_NONSTREAMING(ADR_s32, aa64_sve, do_adr, a, gen_helper_sve_adr_s32)
1230 TRANS_FEAT_NONSTREAMING(ADR_u32, aa64_sve, do_adr, a, gen_helper_sve_adr_u32)
1231
1232 /*
1233 *** SVE Integer Misc - Unpredicated Group
1234 */
1235
1236 static gen_helper_gvec_2 * const fexpa_fns[4] = {
1237 NULL, gen_helper_sve_fexpa_h,
1238 gen_helper_sve_fexpa_s, gen_helper_sve_fexpa_d,
1239 };
1240 TRANS_FEAT_NONSTREAMING(FEXPA, aa64_sve, gen_gvec_ool_zz,
1241 fexpa_fns[a->esz], a->rd, a->rn, s->fpcr_ah)
1242
1243 static gen_helper_gvec_3 * const ftssel_fns[4] = {
1244 NULL, gen_helper_sve_ftssel_h,
1245 gen_helper_sve_ftssel_s, gen_helper_sve_ftssel_d,
1246 };
1247 TRANS_FEAT_NONSTREAMING(FTSSEL, aa64_sve, gen_gvec_ool_arg_zzz,
1248 ftssel_fns[a->esz], a, s->fpcr_ah)
1249
1250 /*
1251 *** SVE Predicate Logical Operations Group
1252 */
1253
do_pppp_flags(DisasContext * s,arg_rprr_s * a,const GVecGen4 * gvec_op)1254 static bool do_pppp_flags(DisasContext *s, arg_rprr_s *a,
1255 const GVecGen4 *gvec_op)
1256 {
1257 if (!sve_access_check(s)) {
1258 return true;
1259 }
1260
1261 unsigned psz = pred_gvec_reg_size(s);
1262 int dofs = pred_full_reg_offset(s, a->rd);
1263 int nofs = pred_full_reg_offset(s, a->rn);
1264 int mofs = pred_full_reg_offset(s, a->rm);
1265 int gofs = pred_full_reg_offset(s, a->pg);
1266
1267 if (!a->s) {
1268 tcg_gen_gvec_4(dofs, nofs, mofs, gofs, psz, psz, gvec_op);
1269 return true;
1270 }
1271
1272 if (psz == 8) {
1273 /* Do the operation and the flags generation in temps. */
1274 TCGv_i64 pd = tcg_temp_new_i64();
1275 TCGv_i64 pn = tcg_temp_new_i64();
1276 TCGv_i64 pm = tcg_temp_new_i64();
1277 TCGv_i64 pg = tcg_temp_new_i64();
1278
1279 tcg_gen_ld_i64(pn, tcg_env, nofs);
1280 tcg_gen_ld_i64(pm, tcg_env, mofs);
1281 tcg_gen_ld_i64(pg, tcg_env, gofs);
1282
1283 gvec_op->fni8(pd, pn, pm, pg);
1284 tcg_gen_st_i64(pd, tcg_env, dofs);
1285
1286 do_predtest1(pd, pg);
1287 } else {
1288 /* The operation and flags generation is large. The computation
1289 * of the flags depends on the original contents of the guarding
1290 * predicate. If the destination overwrites the guarding predicate,
1291 * then the easiest way to get this right is to save a copy.
1292 */
1293 int tofs = gofs;
1294 if (a->rd == a->pg) {
1295 tofs = offsetof(CPUARMState, vfp.preg_tmp);
1296 tcg_gen_gvec_mov(0, tofs, gofs, psz, psz);
1297 }
1298
1299 tcg_gen_gvec_4(dofs, nofs, mofs, gofs, psz, psz, gvec_op);
1300 do_predtest(s, dofs, tofs, psz / 8);
1301 }
1302 return true;
1303 }
1304
gen_and_pg_i64(TCGv_i64 pd,TCGv_i64 pn,TCGv_i64 pm,TCGv_i64 pg)1305 static void gen_and_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg)
1306 {
1307 tcg_gen_and_i64(pd, pn, pm);
1308 tcg_gen_and_i64(pd, pd, pg);
1309 }
1310
gen_and_pg_vec(unsigned vece,TCGv_vec pd,TCGv_vec pn,TCGv_vec pm,TCGv_vec pg)1311 static void gen_and_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn,
1312 TCGv_vec pm, TCGv_vec pg)
1313 {
1314 tcg_gen_and_vec(vece, pd, pn, pm);
1315 tcg_gen_and_vec(vece, pd, pd, pg);
1316 }
1317
trans_AND_pppp(DisasContext * s,arg_rprr_s * a)1318 static bool trans_AND_pppp(DisasContext *s, arg_rprr_s *a)
1319 {
1320 static const GVecGen4 op = {
1321 .fni8 = gen_and_pg_i64,
1322 .fniv = gen_and_pg_vec,
1323 .fno = gen_helper_sve_and_pppp,
1324 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1325 };
1326
1327 if (!dc_isar_feature(aa64_sve, s)) {
1328 return false;
1329 }
1330 if (!a->s) {
1331 if (a->rn == a->rm) {
1332 if (a->pg == a->rn) {
1333 return do_mov_p(s, a->rd, a->rn);
1334 }
1335 return gen_gvec_fn_ppp(s, tcg_gen_gvec_and, a->rd, a->rn, a->pg);
1336 } else if (a->pg == a->rn || a->pg == a->rm) {
1337 return gen_gvec_fn_ppp(s, tcg_gen_gvec_and, a->rd, a->rn, a->rm);
1338 }
1339 }
1340 return do_pppp_flags(s, a, &op);
1341 }
1342
gen_bic_pg_i64(TCGv_i64 pd,TCGv_i64 pn,TCGv_i64 pm,TCGv_i64 pg)1343 static void gen_bic_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg)
1344 {
1345 tcg_gen_andc_i64(pd, pn, pm);
1346 tcg_gen_and_i64(pd, pd, pg);
1347 }
1348
gen_bic_pg_vec(unsigned vece,TCGv_vec pd,TCGv_vec pn,TCGv_vec pm,TCGv_vec pg)1349 static void gen_bic_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn,
1350 TCGv_vec pm, TCGv_vec pg)
1351 {
1352 tcg_gen_andc_vec(vece, pd, pn, pm);
1353 tcg_gen_and_vec(vece, pd, pd, pg);
1354 }
1355
trans_BIC_pppp(DisasContext * s,arg_rprr_s * a)1356 static bool trans_BIC_pppp(DisasContext *s, arg_rprr_s *a)
1357 {
1358 static const GVecGen4 op = {
1359 .fni8 = gen_bic_pg_i64,
1360 .fniv = gen_bic_pg_vec,
1361 .fno = gen_helper_sve_bic_pppp,
1362 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1363 };
1364
1365 if (!dc_isar_feature(aa64_sve, s)) {
1366 return false;
1367 }
1368 if (!a->s && a->pg == a->rn) {
1369 return gen_gvec_fn_ppp(s, tcg_gen_gvec_andc, a->rd, a->rn, a->rm);
1370 }
1371 return do_pppp_flags(s, a, &op);
1372 }
1373
gen_eor_pg_i64(TCGv_i64 pd,TCGv_i64 pn,TCGv_i64 pm,TCGv_i64 pg)1374 static void gen_eor_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg)
1375 {
1376 tcg_gen_xor_i64(pd, pn, pm);
1377 tcg_gen_and_i64(pd, pd, pg);
1378 }
1379
gen_eor_pg_vec(unsigned vece,TCGv_vec pd,TCGv_vec pn,TCGv_vec pm,TCGv_vec pg)1380 static void gen_eor_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn,
1381 TCGv_vec pm, TCGv_vec pg)
1382 {
1383 tcg_gen_xor_vec(vece, pd, pn, pm);
1384 tcg_gen_and_vec(vece, pd, pd, pg);
1385 }
1386
trans_EOR_pppp(DisasContext * s,arg_rprr_s * a)1387 static bool trans_EOR_pppp(DisasContext *s, arg_rprr_s *a)
1388 {
1389 static const GVecGen4 op = {
1390 .fni8 = gen_eor_pg_i64,
1391 .fniv = gen_eor_pg_vec,
1392 .fno = gen_helper_sve_eor_pppp,
1393 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1394 };
1395
1396 if (!dc_isar_feature(aa64_sve, s)) {
1397 return false;
1398 }
1399 /* Alias NOT (predicate) is EOR Pd.B, Pg/Z, Pn.B, Pg.B */
1400 if (!a->s && a->pg == a->rm) {
1401 return gen_gvec_fn_ppp(s, tcg_gen_gvec_andc, a->rd, a->pg, a->rn);
1402 }
1403 return do_pppp_flags(s, a, &op);
1404 }
1405
trans_SEL_pppp(DisasContext * s,arg_rprr_s * a)1406 static bool trans_SEL_pppp(DisasContext *s, arg_rprr_s *a)
1407 {
1408 if (a->s || !dc_isar_feature(aa64_sve, s)) {
1409 return false;
1410 }
1411 if (sve_access_check(s)) {
1412 unsigned psz = pred_gvec_reg_size(s);
1413 tcg_gen_gvec_bitsel(MO_8, pred_full_reg_offset(s, a->rd),
1414 pred_full_reg_offset(s, a->pg),
1415 pred_full_reg_offset(s, a->rn),
1416 pred_full_reg_offset(s, a->rm), psz, psz);
1417 }
1418 return true;
1419 }
1420
gen_orr_pg_i64(TCGv_i64 pd,TCGv_i64 pn,TCGv_i64 pm,TCGv_i64 pg)1421 static void gen_orr_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg)
1422 {
1423 tcg_gen_or_i64(pd, pn, pm);
1424 tcg_gen_and_i64(pd, pd, pg);
1425 }
1426
gen_orr_pg_vec(unsigned vece,TCGv_vec pd,TCGv_vec pn,TCGv_vec pm,TCGv_vec pg)1427 static void gen_orr_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn,
1428 TCGv_vec pm, TCGv_vec pg)
1429 {
1430 tcg_gen_or_vec(vece, pd, pn, pm);
1431 tcg_gen_and_vec(vece, pd, pd, pg);
1432 }
1433
trans_ORR_pppp(DisasContext * s,arg_rprr_s * a)1434 static bool trans_ORR_pppp(DisasContext *s, arg_rprr_s *a)
1435 {
1436 static const GVecGen4 op = {
1437 .fni8 = gen_orr_pg_i64,
1438 .fniv = gen_orr_pg_vec,
1439 .fno = gen_helper_sve_orr_pppp,
1440 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1441 };
1442
1443 if (!dc_isar_feature(aa64_sve, s)) {
1444 return false;
1445 }
1446 if (!a->s && a->pg == a->rn && a->rn == a->rm) {
1447 return do_mov_p(s, a->rd, a->rn);
1448 }
1449 return do_pppp_flags(s, a, &op);
1450 }
1451
gen_orn_pg_i64(TCGv_i64 pd,TCGv_i64 pn,TCGv_i64 pm,TCGv_i64 pg)1452 static void gen_orn_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg)
1453 {
1454 tcg_gen_orc_i64(pd, pn, pm);
1455 tcg_gen_and_i64(pd, pd, pg);
1456 }
1457
gen_orn_pg_vec(unsigned vece,TCGv_vec pd,TCGv_vec pn,TCGv_vec pm,TCGv_vec pg)1458 static void gen_orn_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn,
1459 TCGv_vec pm, TCGv_vec pg)
1460 {
1461 tcg_gen_orc_vec(vece, pd, pn, pm);
1462 tcg_gen_and_vec(vece, pd, pd, pg);
1463 }
1464
trans_ORN_pppp(DisasContext * s,arg_rprr_s * a)1465 static bool trans_ORN_pppp(DisasContext *s, arg_rprr_s *a)
1466 {
1467 static const GVecGen4 op = {
1468 .fni8 = gen_orn_pg_i64,
1469 .fniv = gen_orn_pg_vec,
1470 .fno = gen_helper_sve_orn_pppp,
1471 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1472 };
1473
1474 if (!dc_isar_feature(aa64_sve, s)) {
1475 return false;
1476 }
1477 return do_pppp_flags(s, a, &op);
1478 }
1479
gen_nor_pg_i64(TCGv_i64 pd,TCGv_i64 pn,TCGv_i64 pm,TCGv_i64 pg)1480 static void gen_nor_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg)
1481 {
1482 tcg_gen_or_i64(pd, pn, pm);
1483 tcg_gen_andc_i64(pd, pg, pd);
1484 }
1485
gen_nor_pg_vec(unsigned vece,TCGv_vec pd,TCGv_vec pn,TCGv_vec pm,TCGv_vec pg)1486 static void gen_nor_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn,
1487 TCGv_vec pm, TCGv_vec pg)
1488 {
1489 tcg_gen_or_vec(vece, pd, pn, pm);
1490 tcg_gen_andc_vec(vece, pd, pg, pd);
1491 }
1492
trans_NOR_pppp(DisasContext * s,arg_rprr_s * a)1493 static bool trans_NOR_pppp(DisasContext *s, arg_rprr_s *a)
1494 {
1495 static const GVecGen4 op = {
1496 .fni8 = gen_nor_pg_i64,
1497 .fniv = gen_nor_pg_vec,
1498 .fno = gen_helper_sve_nor_pppp,
1499 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1500 };
1501
1502 if (!dc_isar_feature(aa64_sve, s)) {
1503 return false;
1504 }
1505 return do_pppp_flags(s, a, &op);
1506 }
1507
gen_nand_pg_i64(TCGv_i64 pd,TCGv_i64 pn,TCGv_i64 pm,TCGv_i64 pg)1508 static void gen_nand_pg_i64(TCGv_i64 pd, TCGv_i64 pn, TCGv_i64 pm, TCGv_i64 pg)
1509 {
1510 tcg_gen_and_i64(pd, pn, pm);
1511 tcg_gen_andc_i64(pd, pg, pd);
1512 }
1513
gen_nand_pg_vec(unsigned vece,TCGv_vec pd,TCGv_vec pn,TCGv_vec pm,TCGv_vec pg)1514 static void gen_nand_pg_vec(unsigned vece, TCGv_vec pd, TCGv_vec pn,
1515 TCGv_vec pm, TCGv_vec pg)
1516 {
1517 tcg_gen_and_vec(vece, pd, pn, pm);
1518 tcg_gen_andc_vec(vece, pd, pg, pd);
1519 }
1520
trans_NAND_pppp(DisasContext * s,arg_rprr_s * a)1521 static bool trans_NAND_pppp(DisasContext *s, arg_rprr_s *a)
1522 {
1523 static const GVecGen4 op = {
1524 .fni8 = gen_nand_pg_i64,
1525 .fniv = gen_nand_pg_vec,
1526 .fno = gen_helper_sve_nand_pppp,
1527 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1528 };
1529
1530 if (!dc_isar_feature(aa64_sve, s)) {
1531 return false;
1532 }
1533 return do_pppp_flags(s, a, &op);
1534 }
1535
1536 /*
1537 *** SVE Predicate Misc Group
1538 */
1539
trans_PTEST(DisasContext * s,arg_PTEST * a)1540 static bool trans_PTEST(DisasContext *s, arg_PTEST *a)
1541 {
1542 if (!dc_isar_feature(aa64_sve, s)) {
1543 return false;
1544 }
1545 if (sve_access_check(s)) {
1546 int nofs = pred_full_reg_offset(s, a->rn);
1547 int gofs = pred_full_reg_offset(s, a->pg);
1548 int words = DIV_ROUND_UP(pred_full_reg_size(s), 8);
1549
1550 if (words == 1) {
1551 TCGv_i64 pn = tcg_temp_new_i64();
1552 TCGv_i64 pg = tcg_temp_new_i64();
1553
1554 tcg_gen_ld_i64(pn, tcg_env, nofs);
1555 tcg_gen_ld_i64(pg, tcg_env, gofs);
1556 do_predtest1(pn, pg);
1557 } else {
1558 do_predtest(s, nofs, gofs, words);
1559 }
1560 }
1561 return true;
1562 }
1563
1564 /* See the ARM pseudocode DecodePredCount. */
decode_pred_count(unsigned fullsz,int pattern,int esz)1565 static unsigned decode_pred_count(unsigned fullsz, int pattern, int esz)
1566 {
1567 unsigned elements = fullsz >> esz;
1568 unsigned bound;
1569
1570 switch (pattern) {
1571 case 0x0: /* POW2 */
1572 return pow2floor(elements);
1573 case 0x1: /* VL1 */
1574 case 0x2: /* VL2 */
1575 case 0x3: /* VL3 */
1576 case 0x4: /* VL4 */
1577 case 0x5: /* VL5 */
1578 case 0x6: /* VL6 */
1579 case 0x7: /* VL7 */
1580 case 0x8: /* VL8 */
1581 bound = pattern;
1582 break;
1583 case 0x9: /* VL16 */
1584 case 0xa: /* VL32 */
1585 case 0xb: /* VL64 */
1586 case 0xc: /* VL128 */
1587 case 0xd: /* VL256 */
1588 bound = 16 << (pattern - 9);
1589 break;
1590 case 0x1d: /* MUL4 */
1591 return elements - elements % 4;
1592 case 0x1e: /* MUL3 */
1593 return elements - elements % 3;
1594 case 0x1f: /* ALL */
1595 return elements;
1596 default: /* #uimm5 */
1597 return 0;
1598 }
1599 return elements >= bound ? bound : 0;
1600 }
1601
1602 /* This handles all of the predicate initialization instructions,
1603 * PTRUE, PFALSE, SETFFR. For PFALSE, we will have set PAT == 32
1604 * so that decode_pred_count returns 0. For SETFFR, we will have
1605 * set RD == 16 == FFR.
1606 */
do_predset(DisasContext * s,int esz,int rd,int pat,bool setflag)1607 static bool do_predset(DisasContext *s, int esz, int rd, int pat, bool setflag)
1608 {
1609 if (!sve_access_check(s)) {
1610 return true;
1611 }
1612
1613 unsigned fullsz = vec_full_reg_size(s);
1614 unsigned ofs = pred_full_reg_offset(s, rd);
1615 unsigned numelem, setsz, i;
1616 uint64_t word, lastword;
1617 TCGv_i64 t;
1618
1619 numelem = decode_pred_count(fullsz, pat, esz);
1620
1621 /* Determine what we must store into each bit, and how many. */
1622 if (numelem == 0) {
1623 lastword = word = 0;
1624 setsz = fullsz;
1625 } else {
1626 setsz = numelem << esz;
1627 lastword = word = pred_esz_masks[esz];
1628 if (setsz % 64) {
1629 lastword &= MAKE_64BIT_MASK(0, setsz % 64);
1630 }
1631 }
1632
1633 t = tcg_temp_new_i64();
1634 if (fullsz <= 64) {
1635 tcg_gen_movi_i64(t, lastword);
1636 tcg_gen_st_i64(t, tcg_env, ofs);
1637 goto done;
1638 }
1639
1640 if (word == lastword) {
1641 unsigned maxsz = size_for_gvec(fullsz / 8);
1642 unsigned oprsz = size_for_gvec(setsz / 8);
1643
1644 if (oprsz * 8 == setsz) {
1645 tcg_gen_gvec_dup_imm(MO_64, ofs, oprsz, maxsz, word);
1646 goto done;
1647 }
1648 }
1649
1650 setsz /= 8;
1651 fullsz /= 8;
1652
1653 tcg_gen_movi_i64(t, word);
1654 for (i = 0; i < QEMU_ALIGN_DOWN(setsz, 8); i += 8) {
1655 tcg_gen_st_i64(t, tcg_env, ofs + i);
1656 }
1657 if (lastword != word) {
1658 tcg_gen_movi_i64(t, lastword);
1659 tcg_gen_st_i64(t, tcg_env, ofs + i);
1660 i += 8;
1661 }
1662 if (i < fullsz) {
1663 tcg_gen_movi_i64(t, 0);
1664 for (; i < fullsz; i += 8) {
1665 tcg_gen_st_i64(t, tcg_env, ofs + i);
1666 }
1667 }
1668
1669 done:
1670 /* PTRUES */
1671 if (setflag) {
1672 tcg_gen_movi_i32(cpu_NF, -(word != 0));
1673 tcg_gen_movi_i32(cpu_CF, word == 0);
1674 tcg_gen_movi_i32(cpu_VF, 0);
1675 tcg_gen_mov_i32(cpu_ZF, cpu_NF);
1676 }
1677 return true;
1678 }
1679
1680 TRANS_FEAT(PTRUE, aa64_sve, do_predset, a->esz, a->rd, a->pat, a->s)
1681
1682 /* Note pat == 31 is #all, to set all elements. */
1683 TRANS_FEAT_NONSTREAMING(SETFFR, aa64_sve,
1684 do_predset, 0, FFR_PRED_NUM, 31, false)
1685
1686 /* Note pat == 32 is #unimp, to set no elements. */
1687 TRANS_FEAT(PFALSE, aa64_sve, do_predset, 0, a->rd, 32, false)
1688
trans_RDFFR_p(DisasContext * s,arg_RDFFR_p * a)1689 static bool trans_RDFFR_p(DisasContext *s, arg_RDFFR_p *a)
1690 {
1691 /* The path through do_pppp_flags is complicated enough to want to avoid
1692 * duplication. Frob the arguments into the form of a predicated AND.
1693 */
1694 arg_rprr_s alt_a = {
1695 .rd = a->rd, .pg = a->pg, .s = a->s,
1696 .rn = FFR_PRED_NUM, .rm = FFR_PRED_NUM,
1697 };
1698
1699 s->is_nonstreaming = true;
1700 return trans_AND_pppp(s, &alt_a);
1701 }
1702
1703 TRANS_FEAT_NONSTREAMING(RDFFR, aa64_sve, do_mov_p, a->rd, FFR_PRED_NUM)
1704 TRANS_FEAT_NONSTREAMING(WRFFR, aa64_sve, do_mov_p, FFR_PRED_NUM, a->rn)
1705
do_pfirst_pnext(DisasContext * s,arg_rr_esz * a,void (* gen_fn)(TCGv_i32,TCGv_ptr,TCGv_ptr,TCGv_i32))1706 static bool do_pfirst_pnext(DisasContext *s, arg_rr_esz *a,
1707 void (*gen_fn)(TCGv_i32, TCGv_ptr,
1708 TCGv_ptr, TCGv_i32))
1709 {
1710 if (!sve_access_check(s)) {
1711 return true;
1712 }
1713
1714 TCGv_ptr t_pd = tcg_temp_new_ptr();
1715 TCGv_ptr t_pg = tcg_temp_new_ptr();
1716 TCGv_i32 t;
1717 unsigned desc = 0;
1718
1719 desc = FIELD_DP32(desc, PREDDESC, OPRSZ, pred_full_reg_size(s));
1720 desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz);
1721
1722 tcg_gen_addi_ptr(t_pd, tcg_env, pred_full_reg_offset(s, a->rd));
1723 tcg_gen_addi_ptr(t_pg, tcg_env, pred_full_reg_offset(s, a->rn));
1724 t = tcg_temp_new_i32();
1725
1726 gen_fn(t, t_pd, t_pg, tcg_constant_i32(desc));
1727
1728 do_pred_flags(t);
1729 return true;
1730 }
1731
TRANS_FEAT(PFIRST,aa64_sve,do_pfirst_pnext,a,gen_helper_sve_pfirst)1732 TRANS_FEAT(PFIRST, aa64_sve, do_pfirst_pnext, a, gen_helper_sve_pfirst)
1733 TRANS_FEAT(PNEXT, aa64_sve, do_pfirst_pnext, a, gen_helper_sve_pnext)
1734
1735 /*
1736 *** SVE Element Count Group
1737 */
1738
1739 /* Perform an inline saturating addition of a 32-bit value within
1740 * a 64-bit register. The second operand is known to be positive,
1741 * which halves the comparisons we must perform to bound the result.
1742 */
1743 static void do_sat_addsub_32(TCGv_i64 reg, TCGv_i64 val, bool u, bool d)
1744 {
1745 int64_t ibound;
1746
1747 /* Use normal 64-bit arithmetic to detect 32-bit overflow. */
1748 if (u) {
1749 tcg_gen_ext32u_i64(reg, reg);
1750 } else {
1751 tcg_gen_ext32s_i64(reg, reg);
1752 }
1753 if (d) {
1754 tcg_gen_sub_i64(reg, reg, val);
1755 ibound = (u ? 0 : INT32_MIN);
1756 tcg_gen_smax_i64(reg, reg, tcg_constant_i64(ibound));
1757 } else {
1758 tcg_gen_add_i64(reg, reg, val);
1759 ibound = (u ? UINT32_MAX : INT32_MAX);
1760 tcg_gen_smin_i64(reg, reg, tcg_constant_i64(ibound));
1761 }
1762 }
1763
1764 /* Similarly with 64-bit values. */
do_sat_addsub_64(TCGv_i64 reg,TCGv_i64 val,bool u,bool d)1765 static void do_sat_addsub_64(TCGv_i64 reg, TCGv_i64 val, bool u, bool d)
1766 {
1767 TCGv_i64 t0 = tcg_temp_new_i64();
1768 TCGv_i64 t2;
1769
1770 if (u) {
1771 if (d) {
1772 tcg_gen_sub_i64(t0, reg, val);
1773 t2 = tcg_constant_i64(0);
1774 tcg_gen_movcond_i64(TCG_COND_LTU, reg, reg, val, t2, t0);
1775 } else {
1776 tcg_gen_add_i64(t0, reg, val);
1777 t2 = tcg_constant_i64(-1);
1778 tcg_gen_movcond_i64(TCG_COND_LTU, reg, t0, reg, t2, t0);
1779 }
1780 } else {
1781 TCGv_i64 t1 = tcg_temp_new_i64();
1782 if (d) {
1783 /* Detect signed overflow for subtraction. */
1784 tcg_gen_xor_i64(t0, reg, val);
1785 tcg_gen_sub_i64(t1, reg, val);
1786 tcg_gen_xor_i64(reg, reg, t1);
1787 tcg_gen_and_i64(t0, t0, reg);
1788
1789 /* Bound the result. */
1790 tcg_gen_movi_i64(reg, INT64_MIN);
1791 t2 = tcg_constant_i64(0);
1792 tcg_gen_movcond_i64(TCG_COND_LT, reg, t0, t2, reg, t1);
1793 } else {
1794 /* Detect signed overflow for addition. */
1795 tcg_gen_xor_i64(t0, reg, val);
1796 tcg_gen_add_i64(reg, reg, val);
1797 tcg_gen_xor_i64(t1, reg, val);
1798 tcg_gen_andc_i64(t0, t1, t0);
1799
1800 /* Bound the result. */
1801 tcg_gen_movi_i64(t1, INT64_MAX);
1802 t2 = tcg_constant_i64(0);
1803 tcg_gen_movcond_i64(TCG_COND_LT, reg, t0, t2, t1, reg);
1804 }
1805 }
1806 }
1807
1808 /* Similarly with a vector and a scalar operand. */
do_sat_addsub_vec(DisasContext * s,int esz,int rd,int rn,TCGv_i64 val,bool u,bool d)1809 static void do_sat_addsub_vec(DisasContext *s, int esz, int rd, int rn,
1810 TCGv_i64 val, bool u, bool d)
1811 {
1812 unsigned vsz = vec_full_reg_size(s);
1813 TCGv_ptr dptr, nptr;
1814 TCGv_i32 t32, desc;
1815 TCGv_i64 t64;
1816
1817 dptr = tcg_temp_new_ptr();
1818 nptr = tcg_temp_new_ptr();
1819 tcg_gen_addi_ptr(dptr, tcg_env, vec_full_reg_offset(s, rd));
1820 tcg_gen_addi_ptr(nptr, tcg_env, vec_full_reg_offset(s, rn));
1821 desc = tcg_constant_i32(simd_desc(vsz, vsz, 0));
1822
1823 switch (esz) {
1824 case MO_8:
1825 t32 = tcg_temp_new_i32();
1826 tcg_gen_extrl_i64_i32(t32, val);
1827 if (d) {
1828 tcg_gen_neg_i32(t32, t32);
1829 }
1830 if (u) {
1831 gen_helper_sve_uqaddi_b(dptr, nptr, t32, desc);
1832 } else {
1833 gen_helper_sve_sqaddi_b(dptr, nptr, t32, desc);
1834 }
1835 break;
1836
1837 case MO_16:
1838 t32 = tcg_temp_new_i32();
1839 tcg_gen_extrl_i64_i32(t32, val);
1840 if (d) {
1841 tcg_gen_neg_i32(t32, t32);
1842 }
1843 if (u) {
1844 gen_helper_sve_uqaddi_h(dptr, nptr, t32, desc);
1845 } else {
1846 gen_helper_sve_sqaddi_h(dptr, nptr, t32, desc);
1847 }
1848 break;
1849
1850 case MO_32:
1851 t64 = tcg_temp_new_i64();
1852 if (d) {
1853 tcg_gen_neg_i64(t64, val);
1854 } else {
1855 tcg_gen_mov_i64(t64, val);
1856 }
1857 if (u) {
1858 gen_helper_sve_uqaddi_s(dptr, nptr, t64, desc);
1859 } else {
1860 gen_helper_sve_sqaddi_s(dptr, nptr, t64, desc);
1861 }
1862 break;
1863
1864 case MO_64:
1865 if (u) {
1866 if (d) {
1867 gen_helper_sve_uqsubi_d(dptr, nptr, val, desc);
1868 } else {
1869 gen_helper_sve_uqaddi_d(dptr, nptr, val, desc);
1870 }
1871 } else if (d) {
1872 t64 = tcg_temp_new_i64();
1873 tcg_gen_neg_i64(t64, val);
1874 gen_helper_sve_sqaddi_d(dptr, nptr, t64, desc);
1875 } else {
1876 gen_helper_sve_sqaddi_d(dptr, nptr, val, desc);
1877 }
1878 break;
1879
1880 default:
1881 g_assert_not_reached();
1882 }
1883 }
1884
trans_CNT_r(DisasContext * s,arg_CNT_r * a)1885 static bool trans_CNT_r(DisasContext *s, arg_CNT_r *a)
1886 {
1887 if (!dc_isar_feature(aa64_sve, s)) {
1888 return false;
1889 }
1890 if (sve_access_check(s)) {
1891 unsigned fullsz = vec_full_reg_size(s);
1892 unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz);
1893 tcg_gen_movi_i64(cpu_reg(s, a->rd), numelem * a->imm);
1894 }
1895 return true;
1896 }
1897
trans_INCDEC_r(DisasContext * s,arg_incdec_cnt * a)1898 static bool trans_INCDEC_r(DisasContext *s, arg_incdec_cnt *a)
1899 {
1900 if (!dc_isar_feature(aa64_sve, s)) {
1901 return false;
1902 }
1903 if (sve_access_check(s)) {
1904 unsigned fullsz = vec_full_reg_size(s);
1905 unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz);
1906 int inc = numelem * a->imm * (a->d ? -1 : 1);
1907 TCGv_i64 reg = cpu_reg(s, a->rd);
1908
1909 tcg_gen_addi_i64(reg, reg, inc);
1910 }
1911 return true;
1912 }
1913
trans_SINCDEC_r_32(DisasContext * s,arg_incdec_cnt * a)1914 static bool trans_SINCDEC_r_32(DisasContext *s, arg_incdec_cnt *a)
1915 {
1916 if (!dc_isar_feature(aa64_sve, s)) {
1917 return false;
1918 }
1919 if (!sve_access_check(s)) {
1920 return true;
1921 }
1922
1923 unsigned fullsz = vec_full_reg_size(s);
1924 unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz);
1925 int inc = numelem * a->imm;
1926 TCGv_i64 reg = cpu_reg(s, a->rd);
1927
1928 /* Use normal 64-bit arithmetic to detect 32-bit overflow. */
1929 if (inc == 0) {
1930 if (a->u) {
1931 tcg_gen_ext32u_i64(reg, reg);
1932 } else {
1933 tcg_gen_ext32s_i64(reg, reg);
1934 }
1935 } else {
1936 do_sat_addsub_32(reg, tcg_constant_i64(inc), a->u, a->d);
1937 }
1938 return true;
1939 }
1940
trans_SINCDEC_r_64(DisasContext * s,arg_incdec_cnt * a)1941 static bool trans_SINCDEC_r_64(DisasContext *s, arg_incdec_cnt *a)
1942 {
1943 if (!dc_isar_feature(aa64_sve, s)) {
1944 return false;
1945 }
1946 if (!sve_access_check(s)) {
1947 return true;
1948 }
1949
1950 unsigned fullsz = vec_full_reg_size(s);
1951 unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz);
1952 int inc = numelem * a->imm;
1953 TCGv_i64 reg = cpu_reg(s, a->rd);
1954
1955 if (inc != 0) {
1956 do_sat_addsub_64(reg, tcg_constant_i64(inc), a->u, a->d);
1957 }
1958 return true;
1959 }
1960
trans_INCDEC_v(DisasContext * s,arg_incdec2_cnt * a)1961 static bool trans_INCDEC_v(DisasContext *s, arg_incdec2_cnt *a)
1962 {
1963 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) {
1964 return false;
1965 }
1966
1967 unsigned fullsz = vec_full_reg_size(s);
1968 unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz);
1969 int inc = numelem * a->imm;
1970
1971 if (inc != 0) {
1972 if (sve_access_check(s)) {
1973 tcg_gen_gvec_adds(a->esz, vec_full_reg_offset(s, a->rd),
1974 vec_full_reg_offset(s, a->rn),
1975 tcg_constant_i64(a->d ? -inc : inc),
1976 fullsz, fullsz);
1977 }
1978 } else {
1979 do_mov_z(s, a->rd, a->rn);
1980 }
1981 return true;
1982 }
1983
trans_SINCDEC_v(DisasContext * s,arg_incdec2_cnt * a)1984 static bool trans_SINCDEC_v(DisasContext *s, arg_incdec2_cnt *a)
1985 {
1986 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) {
1987 return false;
1988 }
1989
1990 unsigned fullsz = vec_full_reg_size(s);
1991 unsigned numelem = decode_pred_count(fullsz, a->pat, a->esz);
1992 int inc = numelem * a->imm;
1993
1994 if (inc != 0) {
1995 if (sve_access_check(s)) {
1996 do_sat_addsub_vec(s, a->esz, a->rd, a->rn,
1997 tcg_constant_i64(inc), a->u, a->d);
1998 }
1999 } else {
2000 do_mov_z(s, a->rd, a->rn);
2001 }
2002 return true;
2003 }
2004
2005 /*
2006 *** SVE Bitwise Immediate Group
2007 */
2008
do_zz_dbm(DisasContext * s,arg_rr_dbm * a,GVecGen2iFn * gvec_fn)2009 static bool do_zz_dbm(DisasContext *s, arg_rr_dbm *a, GVecGen2iFn *gvec_fn)
2010 {
2011 uint64_t imm;
2012 if (!logic_imm_decode_wmask(&imm, extract32(a->dbm, 12, 1),
2013 extract32(a->dbm, 0, 6),
2014 extract32(a->dbm, 6, 6))) {
2015 return false;
2016 }
2017 return gen_gvec_fn_zzi(s, gvec_fn, MO_64, a->rd, a->rn, imm);
2018 }
2019
TRANS_FEAT(AND_zzi,aa64_sve,do_zz_dbm,a,tcg_gen_gvec_andi)2020 TRANS_FEAT(AND_zzi, aa64_sve, do_zz_dbm, a, tcg_gen_gvec_andi)
2021 TRANS_FEAT(ORR_zzi, aa64_sve, do_zz_dbm, a, tcg_gen_gvec_ori)
2022 TRANS_FEAT(EOR_zzi, aa64_sve, do_zz_dbm, a, tcg_gen_gvec_xori)
2023
2024 static bool trans_DUPM(DisasContext *s, arg_DUPM *a)
2025 {
2026 uint64_t imm;
2027
2028 if (!dc_isar_feature(aa64_sve, s)) {
2029 return false;
2030 }
2031 if (!logic_imm_decode_wmask(&imm, extract32(a->dbm, 12, 1),
2032 extract32(a->dbm, 0, 6),
2033 extract32(a->dbm, 6, 6))) {
2034 return false;
2035 }
2036 if (sve_access_check(s)) {
2037 do_dupi_z(s, a->rd, imm);
2038 }
2039 return true;
2040 }
2041
2042 /*
2043 *** SVE Integer Wide Immediate - Predicated Group
2044 */
2045
2046 /* Implement all merging copies. This is used for CPY (immediate),
2047 * FCPY, CPY (scalar), CPY (SIMD&FP scalar).
2048 */
do_cpy_m(DisasContext * s,int esz,int rd,int rn,int pg,TCGv_i64 val)2049 static void do_cpy_m(DisasContext *s, int esz, int rd, int rn, int pg,
2050 TCGv_i64 val)
2051 {
2052 typedef void gen_cpy(TCGv_ptr, TCGv_ptr, TCGv_ptr, TCGv_i64, TCGv_i32);
2053 static gen_cpy * const fns[4] = {
2054 gen_helper_sve_cpy_m_b, gen_helper_sve_cpy_m_h,
2055 gen_helper_sve_cpy_m_s, gen_helper_sve_cpy_m_d,
2056 };
2057 unsigned vsz = vec_full_reg_size(s);
2058 TCGv_i32 desc = tcg_constant_i32(simd_desc(vsz, vsz, 0));
2059 TCGv_ptr t_zd = tcg_temp_new_ptr();
2060 TCGv_ptr t_zn = tcg_temp_new_ptr();
2061 TCGv_ptr t_pg = tcg_temp_new_ptr();
2062
2063 tcg_gen_addi_ptr(t_zd, tcg_env, vec_full_reg_offset(s, rd));
2064 tcg_gen_addi_ptr(t_zn, tcg_env, vec_full_reg_offset(s, rn));
2065 tcg_gen_addi_ptr(t_pg, tcg_env, pred_full_reg_offset(s, pg));
2066
2067 fns[esz](t_zd, t_zn, t_pg, val, desc);
2068 }
2069
trans_FCPY(DisasContext * s,arg_FCPY * a)2070 static bool trans_FCPY(DisasContext *s, arg_FCPY *a)
2071 {
2072 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) {
2073 return false;
2074 }
2075 if (sve_access_check(s)) {
2076 /* Decode the VFP immediate. */
2077 uint64_t imm = vfp_expand_imm(a->esz, a->imm);
2078 do_cpy_m(s, a->esz, a->rd, a->rn, a->pg, tcg_constant_i64(imm));
2079 }
2080 return true;
2081 }
2082
trans_CPY_m_i(DisasContext * s,arg_rpri_esz * a)2083 static bool trans_CPY_m_i(DisasContext *s, arg_rpri_esz *a)
2084 {
2085 if (!dc_isar_feature(aa64_sve, s)) {
2086 return false;
2087 }
2088 if (sve_access_check(s)) {
2089 do_cpy_m(s, a->esz, a->rd, a->rn, a->pg, tcg_constant_i64(a->imm));
2090 }
2091 return true;
2092 }
2093
trans_CPY_z_i(DisasContext * s,arg_CPY_z_i * a)2094 static bool trans_CPY_z_i(DisasContext *s, arg_CPY_z_i *a)
2095 {
2096 static gen_helper_gvec_2i * const fns[4] = {
2097 gen_helper_sve_cpy_z_b, gen_helper_sve_cpy_z_h,
2098 gen_helper_sve_cpy_z_s, gen_helper_sve_cpy_z_d,
2099 };
2100
2101 if (!dc_isar_feature(aa64_sve, s)) {
2102 return false;
2103 }
2104 if (sve_access_check(s)) {
2105 unsigned vsz = vec_full_reg_size(s);
2106 tcg_gen_gvec_2i_ool(vec_full_reg_offset(s, a->rd),
2107 pred_full_reg_offset(s, a->pg),
2108 tcg_constant_i64(a->imm),
2109 vsz, vsz, 0, fns[a->esz]);
2110 }
2111 return true;
2112 }
2113
2114 /*
2115 *** SVE Permute Extract Group
2116 */
2117
do_EXT(DisasContext * s,int rd,int rn,int rm,int imm)2118 static bool do_EXT(DisasContext *s, int rd, int rn, int rm, int imm)
2119 {
2120 if (!sve_access_check(s)) {
2121 return true;
2122 }
2123
2124 unsigned vsz = vec_full_reg_size(s);
2125 unsigned n_ofs = imm >= vsz ? 0 : imm;
2126 unsigned n_siz = vsz - n_ofs;
2127 unsigned d = vec_full_reg_offset(s, rd);
2128 unsigned n = vec_full_reg_offset(s, rn);
2129 unsigned m = vec_full_reg_offset(s, rm);
2130
2131 /* Use host vector move insns if we have appropriate sizes
2132 * and no unfortunate overlap.
2133 */
2134 if (m != d
2135 && n_ofs == size_for_gvec(n_ofs)
2136 && n_siz == size_for_gvec(n_siz)
2137 && (d != n || n_siz <= n_ofs)) {
2138 tcg_gen_gvec_mov(0, d, n + n_ofs, n_siz, n_siz);
2139 if (n_ofs != 0) {
2140 tcg_gen_gvec_mov(0, d + n_siz, m, n_ofs, n_ofs);
2141 }
2142 } else {
2143 tcg_gen_gvec_3_ool(d, n, m, vsz, vsz, n_ofs, gen_helper_sve_ext);
2144 }
2145 return true;
2146 }
2147
2148 TRANS_FEAT(EXT, aa64_sve, do_EXT, a->rd, a->rn, a->rm, a->imm)
2149 TRANS_FEAT(EXT_sve2, aa64_sve2, do_EXT, a->rd, a->rn, (a->rn + 1) % 32, a->imm)
2150
2151 /*
2152 *** SVE Permute - Unpredicated Group
2153 */
2154
trans_DUP_s(DisasContext * s,arg_DUP_s * a)2155 static bool trans_DUP_s(DisasContext *s, arg_DUP_s *a)
2156 {
2157 if (!dc_isar_feature(aa64_sve, s)) {
2158 return false;
2159 }
2160 if (sve_access_check(s)) {
2161 unsigned vsz = vec_full_reg_size(s);
2162 tcg_gen_gvec_dup_i64(a->esz, vec_full_reg_offset(s, a->rd),
2163 vsz, vsz, cpu_reg_sp(s, a->rn));
2164 }
2165 return true;
2166 }
2167
trans_DUP_x(DisasContext * s,arg_DUP_x * a)2168 static bool trans_DUP_x(DisasContext *s, arg_DUP_x *a)
2169 {
2170 if (!dc_isar_feature(aa64_sve, s)) {
2171 return false;
2172 }
2173 if ((a->imm & 0x1f) == 0) {
2174 return false;
2175 }
2176 if (sve_access_check(s)) {
2177 unsigned vsz = vec_full_reg_size(s);
2178 unsigned dofs = vec_full_reg_offset(s, a->rd);
2179 unsigned esz, index;
2180
2181 esz = ctz32(a->imm);
2182 index = a->imm >> (esz + 1);
2183
2184 if ((index << esz) < vsz) {
2185 unsigned nofs = vec_reg_offset(s, a->rn, index, esz);
2186 tcg_gen_gvec_dup_mem(esz, dofs, nofs, vsz, vsz);
2187 } else {
2188 /*
2189 * While dup_mem handles 128-bit elements, dup_imm does not.
2190 * Thankfully element size doesn't matter for splatting zero.
2191 */
2192 tcg_gen_gvec_dup_imm(MO_64, dofs, vsz, vsz, 0);
2193 }
2194 }
2195 return true;
2196 }
2197
do_insr_i64(DisasContext * s,arg_rrr_esz * a,TCGv_i64 val)2198 static void do_insr_i64(DisasContext *s, arg_rrr_esz *a, TCGv_i64 val)
2199 {
2200 typedef void gen_insr(TCGv_ptr, TCGv_ptr, TCGv_i64, TCGv_i32);
2201 static gen_insr * const fns[4] = {
2202 gen_helper_sve_insr_b, gen_helper_sve_insr_h,
2203 gen_helper_sve_insr_s, gen_helper_sve_insr_d,
2204 };
2205 unsigned vsz = vec_full_reg_size(s);
2206 TCGv_i32 desc = tcg_constant_i32(simd_desc(vsz, vsz, 0));
2207 TCGv_ptr t_zd = tcg_temp_new_ptr();
2208 TCGv_ptr t_zn = tcg_temp_new_ptr();
2209
2210 tcg_gen_addi_ptr(t_zd, tcg_env, vec_full_reg_offset(s, a->rd));
2211 tcg_gen_addi_ptr(t_zn, tcg_env, vec_full_reg_offset(s, a->rn));
2212
2213 fns[a->esz](t_zd, t_zn, val, desc);
2214 }
2215
trans_INSR_f(DisasContext * s,arg_rrr_esz * a)2216 static bool trans_INSR_f(DisasContext *s, arg_rrr_esz *a)
2217 {
2218 if (!dc_isar_feature(aa64_sve, s)) {
2219 return false;
2220 }
2221 if (sve_access_check(s)) {
2222 TCGv_i64 t = tcg_temp_new_i64();
2223 tcg_gen_ld_i64(t, tcg_env, vec_reg_offset(s, a->rm, 0, MO_64));
2224 do_insr_i64(s, a, t);
2225 }
2226 return true;
2227 }
2228
trans_INSR_r(DisasContext * s,arg_rrr_esz * a)2229 static bool trans_INSR_r(DisasContext *s, arg_rrr_esz *a)
2230 {
2231 if (!dc_isar_feature(aa64_sve, s)) {
2232 return false;
2233 }
2234 if (sve_access_check(s)) {
2235 do_insr_i64(s, a, cpu_reg(s, a->rm));
2236 }
2237 return true;
2238 }
2239
2240 static gen_helper_gvec_2 * const rev_fns[4] = {
2241 gen_helper_sve_rev_b, gen_helper_sve_rev_h,
2242 gen_helper_sve_rev_s, gen_helper_sve_rev_d
2243 };
2244 TRANS_FEAT(REV_v, aa64_sve, gen_gvec_ool_zz, rev_fns[a->esz], a->rd, a->rn, 0)
2245
2246 static gen_helper_gvec_3 * const sve_tbl_fns[4] = {
2247 gen_helper_sve_tbl_b, gen_helper_sve_tbl_h,
2248 gen_helper_sve_tbl_s, gen_helper_sve_tbl_d
2249 };
2250 TRANS_FEAT(TBL, aa64_sve, gen_gvec_ool_arg_zzz, sve_tbl_fns[a->esz], a, 0)
2251
2252 static gen_helper_gvec_4 * const sve2_tbl_fns[4] = {
2253 gen_helper_sve2_tbl_b, gen_helper_sve2_tbl_h,
2254 gen_helper_sve2_tbl_s, gen_helper_sve2_tbl_d
2255 };
2256 TRANS_FEAT(TBL_sve2, aa64_sve2, gen_gvec_ool_zzzz, sve2_tbl_fns[a->esz],
2257 a->rd, a->rn, (a->rn + 1) % 32, a->rm, 0)
2258
2259 static gen_helper_gvec_3 * const tbx_fns[4] = {
2260 gen_helper_sve2_tbx_b, gen_helper_sve2_tbx_h,
2261 gen_helper_sve2_tbx_s, gen_helper_sve2_tbx_d
2262 };
2263 TRANS_FEAT(TBX, aa64_sve2, gen_gvec_ool_arg_zzz, tbx_fns[a->esz], a, 0)
2264
trans_UNPK(DisasContext * s,arg_UNPK * a)2265 static bool trans_UNPK(DisasContext *s, arg_UNPK *a)
2266 {
2267 static gen_helper_gvec_2 * const fns[4][2] = {
2268 { NULL, NULL },
2269 { gen_helper_sve_sunpk_h, gen_helper_sve_uunpk_h },
2270 { gen_helper_sve_sunpk_s, gen_helper_sve_uunpk_s },
2271 { gen_helper_sve_sunpk_d, gen_helper_sve_uunpk_d },
2272 };
2273
2274 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) {
2275 return false;
2276 }
2277 if (sve_access_check(s)) {
2278 unsigned vsz = vec_full_reg_size(s);
2279 tcg_gen_gvec_2_ool(vec_full_reg_offset(s, a->rd),
2280 vec_full_reg_offset(s, a->rn)
2281 + (a->h ? vsz / 2 : 0),
2282 vsz, vsz, 0, fns[a->esz][a->u]);
2283 }
2284 return true;
2285 }
2286
2287 /*
2288 *** SVE Permute - Predicates Group
2289 */
2290
do_perm_pred3(DisasContext * s,arg_rrr_esz * a,bool high_odd,gen_helper_gvec_3 * fn)2291 static bool do_perm_pred3(DisasContext *s, arg_rrr_esz *a, bool high_odd,
2292 gen_helper_gvec_3 *fn)
2293 {
2294 if (!sve_access_check(s)) {
2295 return true;
2296 }
2297
2298 unsigned vsz = pred_full_reg_size(s);
2299
2300 TCGv_ptr t_d = tcg_temp_new_ptr();
2301 TCGv_ptr t_n = tcg_temp_new_ptr();
2302 TCGv_ptr t_m = tcg_temp_new_ptr();
2303 uint32_t desc = 0;
2304
2305 desc = FIELD_DP32(desc, PREDDESC, OPRSZ, vsz);
2306 desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz);
2307 desc = FIELD_DP32(desc, PREDDESC, DATA, high_odd);
2308
2309 tcg_gen_addi_ptr(t_d, tcg_env, pred_full_reg_offset(s, a->rd));
2310 tcg_gen_addi_ptr(t_n, tcg_env, pred_full_reg_offset(s, a->rn));
2311 tcg_gen_addi_ptr(t_m, tcg_env, pred_full_reg_offset(s, a->rm));
2312
2313 fn(t_d, t_n, t_m, tcg_constant_i32(desc));
2314 return true;
2315 }
2316
do_perm_pred2(DisasContext * s,arg_rr_esz * a,bool high_odd,gen_helper_gvec_2 * fn)2317 static bool do_perm_pred2(DisasContext *s, arg_rr_esz *a, bool high_odd,
2318 gen_helper_gvec_2 *fn)
2319 {
2320 if (!sve_access_check(s)) {
2321 return true;
2322 }
2323
2324 unsigned vsz = pred_full_reg_size(s);
2325 TCGv_ptr t_d = tcg_temp_new_ptr();
2326 TCGv_ptr t_n = tcg_temp_new_ptr();
2327 uint32_t desc = 0;
2328
2329 tcg_gen_addi_ptr(t_d, tcg_env, pred_full_reg_offset(s, a->rd));
2330 tcg_gen_addi_ptr(t_n, tcg_env, pred_full_reg_offset(s, a->rn));
2331
2332 desc = FIELD_DP32(desc, PREDDESC, OPRSZ, vsz);
2333 desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz);
2334 desc = FIELD_DP32(desc, PREDDESC, DATA, high_odd);
2335
2336 fn(t_d, t_n, tcg_constant_i32(desc));
2337 return true;
2338 }
2339
2340 TRANS_FEAT(ZIP1_p, aa64_sve, do_perm_pred3, a, 0, gen_helper_sve_zip_p)
2341 TRANS_FEAT(ZIP2_p, aa64_sve, do_perm_pred3, a, 1, gen_helper_sve_zip_p)
2342 TRANS_FEAT(UZP1_p, aa64_sve, do_perm_pred3, a, 0, gen_helper_sve_uzp_p)
2343 TRANS_FEAT(UZP2_p, aa64_sve, do_perm_pred3, a, 1, gen_helper_sve_uzp_p)
2344 TRANS_FEAT(TRN1_p, aa64_sve, do_perm_pred3, a, 0, gen_helper_sve_trn_p)
2345 TRANS_FEAT(TRN2_p, aa64_sve, do_perm_pred3, a, 1, gen_helper_sve_trn_p)
2346
2347 TRANS_FEAT(REV_p, aa64_sve, do_perm_pred2, a, 0, gen_helper_sve_rev_p)
2348 TRANS_FEAT(PUNPKLO, aa64_sve, do_perm_pred2, a, 0, gen_helper_sve_punpk_p)
2349 TRANS_FEAT(PUNPKHI, aa64_sve, do_perm_pred2, a, 1, gen_helper_sve_punpk_p)
2350
2351 /*
2352 *** SVE Permute - Interleaving Group
2353 */
2354
2355 static gen_helper_gvec_3 * const zip_fns[4] = {
2356 gen_helper_sve_zip_b, gen_helper_sve_zip_h,
2357 gen_helper_sve_zip_s, gen_helper_sve_zip_d,
2358 };
2359 TRANS_FEAT(ZIP1_z, aa64_sve, gen_gvec_ool_arg_zzz,
2360 zip_fns[a->esz], a, 0)
2361 TRANS_FEAT(ZIP2_z, aa64_sve, gen_gvec_ool_arg_zzz,
2362 zip_fns[a->esz], a, vec_full_reg_size(s) / 2)
2363
2364 TRANS_FEAT(ZIP1_q, aa64_sve_f64mm, gen_gvec_ool_arg_zzz,
2365 gen_helper_sve2_zip_q, a, 0)
2366 TRANS_FEAT(ZIP2_q, aa64_sve_f64mm, gen_gvec_ool_arg_zzz,
2367 gen_helper_sve2_zip_q, a,
2368 QEMU_ALIGN_DOWN(vec_full_reg_size(s), 32) / 2)
2369
2370 static gen_helper_gvec_3 * const uzp_fns[4] = {
2371 gen_helper_sve_uzp_b, gen_helper_sve_uzp_h,
2372 gen_helper_sve_uzp_s, gen_helper_sve_uzp_d,
2373 };
2374
2375 TRANS_FEAT(UZP1_z, aa64_sve, gen_gvec_ool_arg_zzz,
2376 uzp_fns[a->esz], a, 0)
2377 TRANS_FEAT(UZP2_z, aa64_sve, gen_gvec_ool_arg_zzz,
2378 uzp_fns[a->esz], a, 1 << a->esz)
2379
2380 TRANS_FEAT(UZP1_q, aa64_sve_f64mm, gen_gvec_ool_arg_zzz,
2381 gen_helper_sve2_uzp_q, a, 0)
2382 TRANS_FEAT(UZP2_q, aa64_sve_f64mm, gen_gvec_ool_arg_zzz,
2383 gen_helper_sve2_uzp_q, a, 16)
2384
2385 static gen_helper_gvec_3 * const trn_fns[4] = {
2386 gen_helper_sve_trn_b, gen_helper_sve_trn_h,
2387 gen_helper_sve_trn_s, gen_helper_sve_trn_d,
2388 };
2389
2390 TRANS_FEAT(TRN1_z, aa64_sve, gen_gvec_ool_arg_zzz,
2391 trn_fns[a->esz], a, 0)
2392 TRANS_FEAT(TRN2_z, aa64_sve, gen_gvec_ool_arg_zzz,
2393 trn_fns[a->esz], a, 1 << a->esz)
2394
2395 TRANS_FEAT(TRN1_q, aa64_sve_f64mm, gen_gvec_ool_arg_zzz,
2396 gen_helper_sve2_trn_q, a, 0)
2397 TRANS_FEAT(TRN2_q, aa64_sve_f64mm, gen_gvec_ool_arg_zzz,
2398 gen_helper_sve2_trn_q, a, 16)
2399
2400 /*
2401 *** SVE Permute Vector - Predicated Group
2402 */
2403
2404 static gen_helper_gvec_3 * const compact_fns[4] = {
2405 NULL, NULL, gen_helper_sve_compact_s, gen_helper_sve_compact_d
2406 };
2407 TRANS_FEAT_NONSTREAMING(COMPACT, aa64_sve, gen_gvec_ool_arg_zpz,
2408 compact_fns[a->esz], a, 0)
2409
2410 /* Call the helper that computes the ARM LastActiveElement pseudocode
2411 * function, scaled by the element size. This includes the not found
2412 * indication; e.g. not found for esz=3 is -8.
2413 */
find_last_active(DisasContext * s,TCGv_i32 ret,int esz,int pg)2414 static void find_last_active(DisasContext *s, TCGv_i32 ret, int esz, int pg)
2415 {
2416 /* Predicate sizes may be smaller and cannot use simd_desc. We cannot
2417 * round up, as we do elsewhere, because we need the exact size.
2418 */
2419 TCGv_ptr t_p = tcg_temp_new_ptr();
2420 unsigned desc = 0;
2421
2422 desc = FIELD_DP32(desc, PREDDESC, OPRSZ, pred_full_reg_size(s));
2423 desc = FIELD_DP32(desc, PREDDESC, ESZ, esz);
2424
2425 tcg_gen_addi_ptr(t_p, tcg_env, pred_full_reg_offset(s, pg));
2426
2427 gen_helper_sve_last_active_element(ret, t_p, tcg_constant_i32(desc));
2428 }
2429
2430 /* Increment LAST to the offset of the next element in the vector,
2431 * wrapping around to 0.
2432 */
incr_last_active(DisasContext * s,TCGv_i32 last,int esz)2433 static void incr_last_active(DisasContext *s, TCGv_i32 last, int esz)
2434 {
2435 unsigned vsz = vec_full_reg_size(s);
2436
2437 tcg_gen_addi_i32(last, last, 1 << esz);
2438 if (is_power_of_2(vsz)) {
2439 tcg_gen_andi_i32(last, last, vsz - 1);
2440 } else {
2441 TCGv_i32 max = tcg_constant_i32(vsz);
2442 TCGv_i32 zero = tcg_constant_i32(0);
2443 tcg_gen_movcond_i32(TCG_COND_GEU, last, last, max, zero, last);
2444 }
2445 }
2446
2447 /* If LAST < 0, set LAST to the offset of the last element in the vector. */
wrap_last_active(DisasContext * s,TCGv_i32 last,int esz)2448 static void wrap_last_active(DisasContext *s, TCGv_i32 last, int esz)
2449 {
2450 unsigned vsz = vec_full_reg_size(s);
2451
2452 if (is_power_of_2(vsz)) {
2453 tcg_gen_andi_i32(last, last, vsz - 1);
2454 } else {
2455 TCGv_i32 max = tcg_constant_i32(vsz - (1 << esz));
2456 TCGv_i32 zero = tcg_constant_i32(0);
2457 tcg_gen_movcond_i32(TCG_COND_LT, last, last, zero, max, last);
2458 }
2459 }
2460
2461 /* Load an unsigned element of ESZ from BASE+OFS. */
load_esz(TCGv_ptr base,int ofs,int esz)2462 static TCGv_i64 load_esz(TCGv_ptr base, int ofs, int esz)
2463 {
2464 TCGv_i64 r = tcg_temp_new_i64();
2465
2466 switch (esz) {
2467 case 0:
2468 tcg_gen_ld8u_i64(r, base, ofs);
2469 break;
2470 case 1:
2471 tcg_gen_ld16u_i64(r, base, ofs);
2472 break;
2473 case 2:
2474 tcg_gen_ld32u_i64(r, base, ofs);
2475 break;
2476 case 3:
2477 tcg_gen_ld_i64(r, base, ofs);
2478 break;
2479 default:
2480 g_assert_not_reached();
2481 }
2482 return r;
2483 }
2484
2485 /* Load an unsigned element of ESZ from RM[LAST]. */
load_last_active(DisasContext * s,TCGv_i32 last,int rm,int esz)2486 static TCGv_i64 load_last_active(DisasContext *s, TCGv_i32 last,
2487 int rm, int esz)
2488 {
2489 TCGv_ptr p = tcg_temp_new_ptr();
2490
2491 /* Convert offset into vector into offset into ENV.
2492 * The final adjustment for the vector register base
2493 * is added via constant offset to the load.
2494 */
2495 #if HOST_BIG_ENDIAN
2496 /* Adjust for element ordering. See vec_reg_offset. */
2497 if (esz < 3) {
2498 tcg_gen_xori_i32(last, last, 8 - (1 << esz));
2499 }
2500 #endif
2501 tcg_gen_ext_i32_ptr(p, last);
2502 tcg_gen_add_ptr(p, p, tcg_env);
2503
2504 return load_esz(p, vec_full_reg_offset(s, rm), esz);
2505 }
2506
2507 /* Compute CLAST for a Zreg. */
do_clast_vector(DisasContext * s,arg_rprr_esz * a,bool before)2508 static bool do_clast_vector(DisasContext *s, arg_rprr_esz *a, bool before)
2509 {
2510 TCGv_i32 last;
2511 TCGLabel *over;
2512 TCGv_i64 ele;
2513 unsigned vsz, esz = a->esz;
2514
2515 if (!sve_access_check(s)) {
2516 return true;
2517 }
2518
2519 last = tcg_temp_new_i32();
2520 over = gen_new_label();
2521
2522 find_last_active(s, last, esz, a->pg);
2523
2524 /* There is of course no movcond for a 2048-bit vector,
2525 * so we must branch over the actual store.
2526 */
2527 tcg_gen_brcondi_i32(TCG_COND_LT, last, 0, over);
2528
2529 if (!before) {
2530 incr_last_active(s, last, esz);
2531 }
2532
2533 ele = load_last_active(s, last, a->rm, esz);
2534
2535 vsz = vec_full_reg_size(s);
2536 tcg_gen_gvec_dup_i64(esz, vec_full_reg_offset(s, a->rd), vsz, vsz, ele);
2537
2538 /* If this insn used MOVPRFX, we may need a second move. */
2539 if (a->rd != a->rn) {
2540 TCGLabel *done = gen_new_label();
2541 tcg_gen_br(done);
2542
2543 gen_set_label(over);
2544 do_mov_z(s, a->rd, a->rn);
2545
2546 gen_set_label(done);
2547 } else {
2548 gen_set_label(over);
2549 }
2550 return true;
2551 }
2552
TRANS_FEAT(CLASTA_z,aa64_sve,do_clast_vector,a,false)2553 TRANS_FEAT(CLASTA_z, aa64_sve, do_clast_vector, a, false)
2554 TRANS_FEAT(CLASTB_z, aa64_sve, do_clast_vector, a, true)
2555
2556 /* Compute CLAST for a scalar. */
2557 static void do_clast_scalar(DisasContext *s, int esz, int pg, int rm,
2558 bool before, TCGv_i64 reg_val)
2559 {
2560 TCGv_i32 last = tcg_temp_new_i32();
2561 TCGv_i64 ele, cmp;
2562
2563 find_last_active(s, last, esz, pg);
2564
2565 /* Extend the original value of last prior to incrementing. */
2566 cmp = tcg_temp_new_i64();
2567 tcg_gen_ext_i32_i64(cmp, last);
2568
2569 if (!before) {
2570 incr_last_active(s, last, esz);
2571 }
2572
2573 /* The conceit here is that while last < 0 indicates not found, after
2574 * adjusting for tcg_env->vfp.zregs[rm], it is still a valid address
2575 * from which we can load garbage. We then discard the garbage with
2576 * a conditional move.
2577 */
2578 ele = load_last_active(s, last, rm, esz);
2579
2580 tcg_gen_movcond_i64(TCG_COND_GE, reg_val, cmp, tcg_constant_i64(0),
2581 ele, reg_val);
2582 }
2583
2584 /* Compute CLAST for a Vreg. */
do_clast_fp(DisasContext * s,arg_rpr_esz * a,bool before)2585 static bool do_clast_fp(DisasContext *s, arg_rpr_esz *a, bool before)
2586 {
2587 if (sve_access_check(s)) {
2588 int esz = a->esz;
2589 int ofs = vec_reg_offset(s, a->rd, 0, esz);
2590 TCGv_i64 reg = load_esz(tcg_env, ofs, esz);
2591
2592 do_clast_scalar(s, esz, a->pg, a->rn, before, reg);
2593 write_fp_dreg(s, a->rd, reg);
2594 }
2595 return true;
2596 }
2597
TRANS_FEAT(CLASTA_v,aa64_sve,do_clast_fp,a,false)2598 TRANS_FEAT(CLASTA_v, aa64_sve, do_clast_fp, a, false)
2599 TRANS_FEAT(CLASTB_v, aa64_sve, do_clast_fp, a, true)
2600
2601 /* Compute CLAST for a Xreg. */
2602 static bool do_clast_general(DisasContext *s, arg_rpr_esz *a, bool before)
2603 {
2604 TCGv_i64 reg;
2605
2606 if (!sve_access_check(s)) {
2607 return true;
2608 }
2609
2610 reg = cpu_reg(s, a->rd);
2611 switch (a->esz) {
2612 case 0:
2613 tcg_gen_ext8u_i64(reg, reg);
2614 break;
2615 case 1:
2616 tcg_gen_ext16u_i64(reg, reg);
2617 break;
2618 case 2:
2619 tcg_gen_ext32u_i64(reg, reg);
2620 break;
2621 case 3:
2622 break;
2623 default:
2624 g_assert_not_reached();
2625 }
2626
2627 do_clast_scalar(s, a->esz, a->pg, a->rn, before, reg);
2628 return true;
2629 }
2630
TRANS_FEAT(CLASTA_r,aa64_sve,do_clast_general,a,false)2631 TRANS_FEAT(CLASTA_r, aa64_sve, do_clast_general, a, false)
2632 TRANS_FEAT(CLASTB_r, aa64_sve, do_clast_general, a, true)
2633
2634 /* Compute LAST for a scalar. */
2635 static TCGv_i64 do_last_scalar(DisasContext *s, int esz,
2636 int pg, int rm, bool before)
2637 {
2638 TCGv_i32 last = tcg_temp_new_i32();
2639
2640 find_last_active(s, last, esz, pg);
2641 if (before) {
2642 wrap_last_active(s, last, esz);
2643 } else {
2644 incr_last_active(s, last, esz);
2645 }
2646
2647 return load_last_active(s, last, rm, esz);
2648 }
2649
2650 /* Compute LAST for a Vreg. */
do_last_fp(DisasContext * s,arg_rpr_esz * a,bool before)2651 static bool do_last_fp(DisasContext *s, arg_rpr_esz *a, bool before)
2652 {
2653 if (sve_access_check(s)) {
2654 TCGv_i64 val = do_last_scalar(s, a->esz, a->pg, a->rn, before);
2655 write_fp_dreg(s, a->rd, val);
2656 }
2657 return true;
2658 }
2659
TRANS_FEAT(LASTA_v,aa64_sve,do_last_fp,a,false)2660 TRANS_FEAT(LASTA_v, aa64_sve, do_last_fp, a, false)
2661 TRANS_FEAT(LASTB_v, aa64_sve, do_last_fp, a, true)
2662
2663 /* Compute LAST for a Xreg. */
2664 static bool do_last_general(DisasContext *s, arg_rpr_esz *a, bool before)
2665 {
2666 if (sve_access_check(s)) {
2667 TCGv_i64 val = do_last_scalar(s, a->esz, a->pg, a->rn, before);
2668 tcg_gen_mov_i64(cpu_reg(s, a->rd), val);
2669 }
2670 return true;
2671 }
2672
TRANS_FEAT(LASTA_r,aa64_sve,do_last_general,a,false)2673 TRANS_FEAT(LASTA_r, aa64_sve, do_last_general, a, false)
2674 TRANS_FEAT(LASTB_r, aa64_sve, do_last_general, a, true)
2675
2676 static bool trans_CPY_m_r(DisasContext *s, arg_rpr_esz *a)
2677 {
2678 if (!dc_isar_feature(aa64_sve, s)) {
2679 return false;
2680 }
2681 if (sve_access_check(s)) {
2682 do_cpy_m(s, a->esz, a->rd, a->rd, a->pg, cpu_reg_sp(s, a->rn));
2683 }
2684 return true;
2685 }
2686
trans_CPY_m_v(DisasContext * s,arg_rpr_esz * a)2687 static bool trans_CPY_m_v(DisasContext *s, arg_rpr_esz *a)
2688 {
2689 if (!dc_isar_feature(aa64_sve, s)) {
2690 return false;
2691 }
2692 if (sve_access_check(s)) {
2693 int ofs = vec_reg_offset(s, a->rn, 0, a->esz);
2694 TCGv_i64 t = load_esz(tcg_env, ofs, a->esz);
2695 do_cpy_m(s, a->esz, a->rd, a->rd, a->pg, t);
2696 }
2697 return true;
2698 }
2699
2700 static gen_helper_gvec_3 * const revb_fns[4] = {
2701 NULL, gen_helper_sve_revb_h,
2702 gen_helper_sve_revb_s, gen_helper_sve_revb_d,
2703 };
2704 TRANS_FEAT(REVB, aa64_sve, gen_gvec_ool_arg_zpz, revb_fns[a->esz], a, 0)
2705
2706 static gen_helper_gvec_3 * const revh_fns[4] = {
2707 NULL, NULL, gen_helper_sve_revh_s, gen_helper_sve_revh_d,
2708 };
2709 TRANS_FEAT(REVH, aa64_sve, gen_gvec_ool_arg_zpz, revh_fns[a->esz], a, 0)
2710
2711 TRANS_FEAT(REVW, aa64_sve, gen_gvec_ool_arg_zpz,
2712 a->esz == 3 ? gen_helper_sve_revw_d : NULL, a, 0)
2713
2714 TRANS_FEAT(REVD, aa64_sme, gen_gvec_ool_arg_zpz, gen_helper_sme_revd_q, a, 0)
2715
2716 TRANS_FEAT(SPLICE, aa64_sve, gen_gvec_ool_arg_zpzz,
2717 gen_helper_sve_splice, a, a->esz)
2718
2719 TRANS_FEAT(SPLICE_sve2, aa64_sve2, gen_gvec_ool_zzzp, gen_helper_sve_splice,
2720 a->rd, a->rn, (a->rn + 1) % 32, a->pg, a->esz)
2721
2722 /*
2723 *** SVE Integer Compare - Vectors Group
2724 */
2725
do_ppzz_flags(DisasContext * s,arg_rprr_esz * a,gen_helper_gvec_flags_4 * gen_fn)2726 static bool do_ppzz_flags(DisasContext *s, arg_rprr_esz *a,
2727 gen_helper_gvec_flags_4 *gen_fn)
2728 {
2729 TCGv_ptr pd, zn, zm, pg;
2730 unsigned vsz;
2731 TCGv_i32 t;
2732
2733 if (gen_fn == NULL) {
2734 return false;
2735 }
2736 if (!sve_access_check(s)) {
2737 return true;
2738 }
2739
2740 vsz = vec_full_reg_size(s);
2741 t = tcg_temp_new_i32();
2742 pd = tcg_temp_new_ptr();
2743 zn = tcg_temp_new_ptr();
2744 zm = tcg_temp_new_ptr();
2745 pg = tcg_temp_new_ptr();
2746
2747 tcg_gen_addi_ptr(pd, tcg_env, pred_full_reg_offset(s, a->rd));
2748 tcg_gen_addi_ptr(zn, tcg_env, vec_full_reg_offset(s, a->rn));
2749 tcg_gen_addi_ptr(zm, tcg_env, vec_full_reg_offset(s, a->rm));
2750 tcg_gen_addi_ptr(pg, tcg_env, pred_full_reg_offset(s, a->pg));
2751
2752 gen_fn(t, pd, zn, zm, pg, tcg_constant_i32(simd_desc(vsz, vsz, 0)));
2753
2754 do_pred_flags(t);
2755 return true;
2756 }
2757
2758 #define DO_PPZZ(NAME, name) \
2759 static gen_helper_gvec_flags_4 * const name##_ppzz_fns[4] = { \
2760 gen_helper_sve_##name##_ppzz_b, gen_helper_sve_##name##_ppzz_h, \
2761 gen_helper_sve_##name##_ppzz_s, gen_helper_sve_##name##_ppzz_d, \
2762 }; \
2763 TRANS_FEAT(NAME##_ppzz, aa64_sve, do_ppzz_flags, \
2764 a, name##_ppzz_fns[a->esz])
2765
DO_PPZZ(CMPEQ,cmpeq)2766 DO_PPZZ(CMPEQ, cmpeq)
2767 DO_PPZZ(CMPNE, cmpne)
2768 DO_PPZZ(CMPGT, cmpgt)
2769 DO_PPZZ(CMPGE, cmpge)
2770 DO_PPZZ(CMPHI, cmphi)
2771 DO_PPZZ(CMPHS, cmphs)
2772
2773 #undef DO_PPZZ
2774
2775 #define DO_PPZW(NAME, name) \
2776 static gen_helper_gvec_flags_4 * const name##_ppzw_fns[4] = { \
2777 gen_helper_sve_##name##_ppzw_b, gen_helper_sve_##name##_ppzw_h, \
2778 gen_helper_sve_##name##_ppzw_s, NULL \
2779 }; \
2780 TRANS_FEAT(NAME##_ppzw, aa64_sve, do_ppzz_flags, \
2781 a, name##_ppzw_fns[a->esz])
2782
2783 DO_PPZW(CMPEQ, cmpeq)
2784 DO_PPZW(CMPNE, cmpne)
2785 DO_PPZW(CMPGT, cmpgt)
2786 DO_PPZW(CMPGE, cmpge)
2787 DO_PPZW(CMPHI, cmphi)
2788 DO_PPZW(CMPHS, cmphs)
2789 DO_PPZW(CMPLT, cmplt)
2790 DO_PPZW(CMPLE, cmple)
2791 DO_PPZW(CMPLO, cmplo)
2792 DO_PPZW(CMPLS, cmpls)
2793
2794 #undef DO_PPZW
2795
2796 /*
2797 *** SVE Integer Compare - Immediate Groups
2798 */
2799
2800 static bool do_ppzi_flags(DisasContext *s, arg_rpri_esz *a,
2801 gen_helper_gvec_flags_3 *gen_fn)
2802 {
2803 TCGv_ptr pd, zn, pg;
2804 unsigned vsz;
2805 TCGv_i32 t;
2806
2807 if (gen_fn == NULL) {
2808 return false;
2809 }
2810 if (!sve_access_check(s)) {
2811 return true;
2812 }
2813
2814 vsz = vec_full_reg_size(s);
2815 t = tcg_temp_new_i32();
2816 pd = tcg_temp_new_ptr();
2817 zn = tcg_temp_new_ptr();
2818 pg = tcg_temp_new_ptr();
2819
2820 tcg_gen_addi_ptr(pd, tcg_env, pred_full_reg_offset(s, a->rd));
2821 tcg_gen_addi_ptr(zn, tcg_env, vec_full_reg_offset(s, a->rn));
2822 tcg_gen_addi_ptr(pg, tcg_env, pred_full_reg_offset(s, a->pg));
2823
2824 gen_fn(t, pd, zn, pg, tcg_constant_i32(simd_desc(vsz, vsz, a->imm)));
2825
2826 do_pred_flags(t);
2827 return true;
2828 }
2829
2830 #define DO_PPZI(NAME, name) \
2831 static gen_helper_gvec_flags_3 * const name##_ppzi_fns[4] = { \
2832 gen_helper_sve_##name##_ppzi_b, gen_helper_sve_##name##_ppzi_h, \
2833 gen_helper_sve_##name##_ppzi_s, gen_helper_sve_##name##_ppzi_d, \
2834 }; \
2835 TRANS_FEAT(NAME##_ppzi, aa64_sve, do_ppzi_flags, a, \
2836 name##_ppzi_fns[a->esz])
2837
DO_PPZI(CMPEQ,cmpeq)2838 DO_PPZI(CMPEQ, cmpeq)
2839 DO_PPZI(CMPNE, cmpne)
2840 DO_PPZI(CMPGT, cmpgt)
2841 DO_PPZI(CMPGE, cmpge)
2842 DO_PPZI(CMPHI, cmphi)
2843 DO_PPZI(CMPHS, cmphs)
2844 DO_PPZI(CMPLT, cmplt)
2845 DO_PPZI(CMPLE, cmple)
2846 DO_PPZI(CMPLO, cmplo)
2847 DO_PPZI(CMPLS, cmpls)
2848
2849 #undef DO_PPZI
2850
2851 /*
2852 *** SVE Partition Break Group
2853 */
2854
2855 static bool do_brk3(DisasContext *s, arg_rprr_s *a,
2856 gen_helper_gvec_4 *fn, gen_helper_gvec_flags_4 *fn_s)
2857 {
2858 if (!sve_access_check(s)) {
2859 return true;
2860 }
2861
2862 unsigned vsz = pred_full_reg_size(s);
2863
2864 /* Predicate sizes may be smaller and cannot use simd_desc. */
2865 TCGv_ptr d = tcg_temp_new_ptr();
2866 TCGv_ptr n = tcg_temp_new_ptr();
2867 TCGv_ptr m = tcg_temp_new_ptr();
2868 TCGv_ptr g = tcg_temp_new_ptr();
2869 TCGv_i32 desc = tcg_constant_i32(FIELD_DP32(0, PREDDESC, OPRSZ, vsz));
2870
2871 tcg_gen_addi_ptr(d, tcg_env, pred_full_reg_offset(s, a->rd));
2872 tcg_gen_addi_ptr(n, tcg_env, pred_full_reg_offset(s, a->rn));
2873 tcg_gen_addi_ptr(m, tcg_env, pred_full_reg_offset(s, a->rm));
2874 tcg_gen_addi_ptr(g, tcg_env, pred_full_reg_offset(s, a->pg));
2875
2876 if (a->s) {
2877 TCGv_i32 t = tcg_temp_new_i32();
2878 fn_s(t, d, n, m, g, desc);
2879 do_pred_flags(t);
2880 } else {
2881 fn(d, n, m, g, desc);
2882 }
2883 return true;
2884 }
2885
do_brk2(DisasContext * s,arg_rpr_s * a,gen_helper_gvec_3 * fn,gen_helper_gvec_flags_3 * fn_s)2886 static bool do_brk2(DisasContext *s, arg_rpr_s *a,
2887 gen_helper_gvec_3 *fn, gen_helper_gvec_flags_3 *fn_s)
2888 {
2889 if (!sve_access_check(s)) {
2890 return true;
2891 }
2892
2893 unsigned vsz = pred_full_reg_size(s);
2894
2895 /* Predicate sizes may be smaller and cannot use simd_desc. */
2896 TCGv_ptr d = tcg_temp_new_ptr();
2897 TCGv_ptr n = tcg_temp_new_ptr();
2898 TCGv_ptr g = tcg_temp_new_ptr();
2899 TCGv_i32 desc = tcg_constant_i32(FIELD_DP32(0, PREDDESC, OPRSZ, vsz));
2900
2901 tcg_gen_addi_ptr(d, tcg_env, pred_full_reg_offset(s, a->rd));
2902 tcg_gen_addi_ptr(n, tcg_env, pred_full_reg_offset(s, a->rn));
2903 tcg_gen_addi_ptr(g, tcg_env, pred_full_reg_offset(s, a->pg));
2904
2905 if (a->s) {
2906 TCGv_i32 t = tcg_temp_new_i32();
2907 fn_s(t, d, n, g, desc);
2908 do_pred_flags(t);
2909 } else {
2910 fn(d, n, g, desc);
2911 }
2912 return true;
2913 }
2914
TRANS_FEAT(BRKPA,aa64_sve,do_brk3,a,gen_helper_sve_brkpa,gen_helper_sve_brkpas)2915 TRANS_FEAT(BRKPA, aa64_sve, do_brk3, a,
2916 gen_helper_sve_brkpa, gen_helper_sve_brkpas)
2917 TRANS_FEAT(BRKPB, aa64_sve, do_brk3, a,
2918 gen_helper_sve_brkpb, gen_helper_sve_brkpbs)
2919
2920 TRANS_FEAT(BRKA_m, aa64_sve, do_brk2, a,
2921 gen_helper_sve_brka_m, gen_helper_sve_brkas_m)
2922 TRANS_FEAT(BRKB_m, aa64_sve, do_brk2, a,
2923 gen_helper_sve_brkb_m, gen_helper_sve_brkbs_m)
2924
2925 TRANS_FEAT(BRKA_z, aa64_sve, do_brk2, a,
2926 gen_helper_sve_brka_z, gen_helper_sve_brkas_z)
2927 TRANS_FEAT(BRKB_z, aa64_sve, do_brk2, a,
2928 gen_helper_sve_brkb_z, gen_helper_sve_brkbs_z)
2929
2930 TRANS_FEAT(BRKN, aa64_sve, do_brk2, a,
2931 gen_helper_sve_brkn, gen_helper_sve_brkns)
2932
2933 /*
2934 *** SVE Predicate Count Group
2935 */
2936
2937 static void do_cntp(DisasContext *s, TCGv_i64 val, int esz, int pn, int pg)
2938 {
2939 unsigned psz = pred_full_reg_size(s);
2940
2941 if (psz <= 8) {
2942 uint64_t psz_mask;
2943
2944 tcg_gen_ld_i64(val, tcg_env, pred_full_reg_offset(s, pn));
2945 if (pn != pg) {
2946 TCGv_i64 g = tcg_temp_new_i64();
2947 tcg_gen_ld_i64(g, tcg_env, pred_full_reg_offset(s, pg));
2948 tcg_gen_and_i64(val, val, g);
2949 }
2950
2951 /* Reduce the pred_esz_masks value simply to reduce the
2952 * size of the code generated here.
2953 */
2954 psz_mask = MAKE_64BIT_MASK(0, psz * 8);
2955 tcg_gen_andi_i64(val, val, pred_esz_masks[esz] & psz_mask);
2956
2957 tcg_gen_ctpop_i64(val, val);
2958 } else {
2959 TCGv_ptr t_pn = tcg_temp_new_ptr();
2960 TCGv_ptr t_pg = tcg_temp_new_ptr();
2961 unsigned desc = 0;
2962
2963 desc = FIELD_DP32(desc, PREDDESC, OPRSZ, psz);
2964 desc = FIELD_DP32(desc, PREDDESC, ESZ, esz);
2965
2966 tcg_gen_addi_ptr(t_pn, tcg_env, pred_full_reg_offset(s, pn));
2967 tcg_gen_addi_ptr(t_pg, tcg_env, pred_full_reg_offset(s, pg));
2968
2969 gen_helper_sve_cntp(val, t_pn, t_pg, tcg_constant_i32(desc));
2970 }
2971 }
2972
trans_CNTP(DisasContext * s,arg_CNTP * a)2973 static bool trans_CNTP(DisasContext *s, arg_CNTP *a)
2974 {
2975 if (!dc_isar_feature(aa64_sve, s)) {
2976 return false;
2977 }
2978 if (sve_access_check(s)) {
2979 do_cntp(s, cpu_reg(s, a->rd), a->esz, a->rn, a->pg);
2980 }
2981 return true;
2982 }
2983
trans_INCDECP_r(DisasContext * s,arg_incdec_pred * a)2984 static bool trans_INCDECP_r(DisasContext *s, arg_incdec_pred *a)
2985 {
2986 if (!dc_isar_feature(aa64_sve, s)) {
2987 return false;
2988 }
2989 if (sve_access_check(s)) {
2990 TCGv_i64 reg = cpu_reg(s, a->rd);
2991 TCGv_i64 val = tcg_temp_new_i64();
2992
2993 do_cntp(s, val, a->esz, a->pg, a->pg);
2994 if (a->d) {
2995 tcg_gen_sub_i64(reg, reg, val);
2996 } else {
2997 tcg_gen_add_i64(reg, reg, val);
2998 }
2999 }
3000 return true;
3001 }
3002
trans_INCDECP_z(DisasContext * s,arg_incdec2_pred * a)3003 static bool trans_INCDECP_z(DisasContext *s, arg_incdec2_pred *a)
3004 {
3005 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) {
3006 return false;
3007 }
3008 if (sve_access_check(s)) {
3009 unsigned vsz = vec_full_reg_size(s);
3010 TCGv_i64 val = tcg_temp_new_i64();
3011 GVecGen2sFn *gvec_fn = a->d ? tcg_gen_gvec_subs : tcg_gen_gvec_adds;
3012
3013 do_cntp(s, val, a->esz, a->pg, a->pg);
3014 gvec_fn(a->esz, vec_full_reg_offset(s, a->rd),
3015 vec_full_reg_offset(s, a->rn), val, vsz, vsz);
3016 }
3017 return true;
3018 }
3019
trans_SINCDECP_r_32(DisasContext * s,arg_incdec_pred * a)3020 static bool trans_SINCDECP_r_32(DisasContext *s, arg_incdec_pred *a)
3021 {
3022 if (!dc_isar_feature(aa64_sve, s)) {
3023 return false;
3024 }
3025 if (sve_access_check(s)) {
3026 TCGv_i64 reg = cpu_reg(s, a->rd);
3027 TCGv_i64 val = tcg_temp_new_i64();
3028
3029 do_cntp(s, val, a->esz, a->pg, a->pg);
3030 do_sat_addsub_32(reg, val, a->u, a->d);
3031 }
3032 return true;
3033 }
3034
trans_SINCDECP_r_64(DisasContext * s,arg_incdec_pred * a)3035 static bool trans_SINCDECP_r_64(DisasContext *s, arg_incdec_pred *a)
3036 {
3037 if (!dc_isar_feature(aa64_sve, s)) {
3038 return false;
3039 }
3040 if (sve_access_check(s)) {
3041 TCGv_i64 reg = cpu_reg(s, a->rd);
3042 TCGv_i64 val = tcg_temp_new_i64();
3043
3044 do_cntp(s, val, a->esz, a->pg, a->pg);
3045 do_sat_addsub_64(reg, val, a->u, a->d);
3046 }
3047 return true;
3048 }
3049
trans_SINCDECP_z(DisasContext * s,arg_incdec2_pred * a)3050 static bool trans_SINCDECP_z(DisasContext *s, arg_incdec2_pred *a)
3051 {
3052 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) {
3053 return false;
3054 }
3055 if (sve_access_check(s)) {
3056 TCGv_i64 val = tcg_temp_new_i64();
3057 do_cntp(s, val, a->esz, a->pg, a->pg);
3058 do_sat_addsub_vec(s, a->esz, a->rd, a->rn, val, a->u, a->d);
3059 }
3060 return true;
3061 }
3062
3063 /*
3064 *** SVE Integer Compare Scalars Group
3065 */
3066
trans_CTERM(DisasContext * s,arg_CTERM * a)3067 static bool trans_CTERM(DisasContext *s, arg_CTERM *a)
3068 {
3069 if (!dc_isar_feature(aa64_sve, s)) {
3070 return false;
3071 }
3072 if (!sve_access_check(s)) {
3073 return true;
3074 }
3075
3076 TCGCond cond = (a->ne ? TCG_COND_NE : TCG_COND_EQ);
3077 TCGv_i64 rn = read_cpu_reg(s, a->rn, a->sf);
3078 TCGv_i64 rm = read_cpu_reg(s, a->rm, a->sf);
3079 TCGv_i64 cmp = tcg_temp_new_i64();
3080
3081 tcg_gen_setcond_i64(cond, cmp, rn, rm);
3082 tcg_gen_extrl_i64_i32(cpu_NF, cmp);
3083
3084 /* VF = !NF & !CF. */
3085 tcg_gen_xori_i32(cpu_VF, cpu_NF, 1);
3086 tcg_gen_andc_i32(cpu_VF, cpu_VF, cpu_CF);
3087
3088 /* Both NF and VF actually look at bit 31. */
3089 tcg_gen_neg_i32(cpu_NF, cpu_NF);
3090 tcg_gen_neg_i32(cpu_VF, cpu_VF);
3091 return true;
3092 }
3093
trans_WHILE(DisasContext * s,arg_WHILE * a)3094 static bool trans_WHILE(DisasContext *s, arg_WHILE *a)
3095 {
3096 TCGv_i64 op0, op1, t0, t1, tmax;
3097 TCGv_i32 t2;
3098 TCGv_ptr ptr;
3099 unsigned vsz = vec_full_reg_size(s);
3100 unsigned desc = 0;
3101 TCGCond cond;
3102 uint64_t maxval;
3103 /* Note that GE/HS has a->eq == 0 and GT/HI has a->eq == 1. */
3104 bool eq = a->eq == a->lt;
3105
3106 /* The greater-than conditions are all SVE2. */
3107 if (a->lt
3108 ? !dc_isar_feature(aa64_sve, s)
3109 : !dc_isar_feature(aa64_sve2, s)) {
3110 return false;
3111 }
3112 if (!sve_access_check(s)) {
3113 return true;
3114 }
3115
3116 op0 = read_cpu_reg(s, a->rn, 1);
3117 op1 = read_cpu_reg(s, a->rm, 1);
3118
3119 if (!a->sf) {
3120 if (a->u) {
3121 tcg_gen_ext32u_i64(op0, op0);
3122 tcg_gen_ext32u_i64(op1, op1);
3123 } else {
3124 tcg_gen_ext32s_i64(op0, op0);
3125 tcg_gen_ext32s_i64(op1, op1);
3126 }
3127 }
3128
3129 /* For the helper, compress the different conditions into a computation
3130 * of how many iterations for which the condition is true.
3131 */
3132 t0 = tcg_temp_new_i64();
3133 t1 = tcg_temp_new_i64();
3134
3135 if (a->lt) {
3136 tcg_gen_sub_i64(t0, op1, op0);
3137 if (a->u) {
3138 maxval = a->sf ? UINT64_MAX : UINT32_MAX;
3139 cond = eq ? TCG_COND_LEU : TCG_COND_LTU;
3140 } else {
3141 maxval = a->sf ? INT64_MAX : INT32_MAX;
3142 cond = eq ? TCG_COND_LE : TCG_COND_LT;
3143 }
3144 } else {
3145 tcg_gen_sub_i64(t0, op0, op1);
3146 if (a->u) {
3147 maxval = 0;
3148 cond = eq ? TCG_COND_GEU : TCG_COND_GTU;
3149 } else {
3150 maxval = a->sf ? INT64_MIN : INT32_MIN;
3151 cond = eq ? TCG_COND_GE : TCG_COND_GT;
3152 }
3153 }
3154
3155 tmax = tcg_constant_i64(vsz >> a->esz);
3156 if (eq) {
3157 /* Equality means one more iteration. */
3158 tcg_gen_addi_i64(t0, t0, 1);
3159
3160 /*
3161 * For the less-than while, if op1 is maxval (and the only time
3162 * the addition above could overflow), then we produce an all-true
3163 * predicate by setting the count to the vector length. This is
3164 * because the pseudocode is described as an increment + compare
3165 * loop, and the maximum integer would always compare true.
3166 * Similarly, the greater-than while has the same issue with the
3167 * minimum integer due to the decrement + compare loop.
3168 */
3169 tcg_gen_movi_i64(t1, maxval);
3170 tcg_gen_movcond_i64(TCG_COND_EQ, t0, op1, t1, tmax, t0);
3171 }
3172
3173 /* Bound to the maximum. */
3174 tcg_gen_umin_i64(t0, t0, tmax);
3175
3176 /* Set the count to zero if the condition is false. */
3177 tcg_gen_movi_i64(t1, 0);
3178 tcg_gen_movcond_i64(cond, t0, op0, op1, t0, t1);
3179
3180 /* Since we're bounded, pass as a 32-bit type. */
3181 t2 = tcg_temp_new_i32();
3182 tcg_gen_extrl_i64_i32(t2, t0);
3183
3184 /* Scale elements to bits. */
3185 tcg_gen_shli_i32(t2, t2, a->esz);
3186
3187 desc = FIELD_DP32(desc, PREDDESC, OPRSZ, vsz / 8);
3188 desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz);
3189
3190 ptr = tcg_temp_new_ptr();
3191 tcg_gen_addi_ptr(ptr, tcg_env, pred_full_reg_offset(s, a->rd));
3192
3193 if (a->lt) {
3194 gen_helper_sve_whilel(t2, ptr, t2, tcg_constant_i32(desc));
3195 } else {
3196 gen_helper_sve_whileg(t2, ptr, t2, tcg_constant_i32(desc));
3197 }
3198 do_pred_flags(t2);
3199 return true;
3200 }
3201
trans_WHILE_ptr(DisasContext * s,arg_WHILE_ptr * a)3202 static bool trans_WHILE_ptr(DisasContext *s, arg_WHILE_ptr *a)
3203 {
3204 TCGv_i64 op0, op1, diff, t1, tmax;
3205 TCGv_i32 t2;
3206 TCGv_ptr ptr;
3207 unsigned vsz = vec_full_reg_size(s);
3208 unsigned desc = 0;
3209
3210 if (!dc_isar_feature(aa64_sve2, s)) {
3211 return false;
3212 }
3213 if (!sve_access_check(s)) {
3214 return true;
3215 }
3216
3217 op0 = read_cpu_reg(s, a->rn, 1);
3218 op1 = read_cpu_reg(s, a->rm, 1);
3219
3220 tmax = tcg_constant_i64(vsz);
3221 diff = tcg_temp_new_i64();
3222
3223 if (a->rw) {
3224 /* WHILERW */
3225 /* diff = abs(op1 - op0), noting that op0/1 are unsigned. */
3226 t1 = tcg_temp_new_i64();
3227 tcg_gen_sub_i64(diff, op0, op1);
3228 tcg_gen_sub_i64(t1, op1, op0);
3229 tcg_gen_movcond_i64(TCG_COND_GEU, diff, op0, op1, diff, t1);
3230 /* Round down to a multiple of ESIZE. */
3231 tcg_gen_andi_i64(diff, diff, -1 << a->esz);
3232 /* If op1 == op0, diff == 0, and the condition is always true. */
3233 tcg_gen_movcond_i64(TCG_COND_EQ, diff, op0, op1, tmax, diff);
3234 } else {
3235 /* WHILEWR */
3236 tcg_gen_sub_i64(diff, op1, op0);
3237 /* Round down to a multiple of ESIZE. */
3238 tcg_gen_andi_i64(diff, diff, -1 << a->esz);
3239 /* If op0 >= op1, diff <= 0, the condition is always true. */
3240 tcg_gen_movcond_i64(TCG_COND_GEU, diff, op0, op1, tmax, diff);
3241 }
3242
3243 /* Bound to the maximum. */
3244 tcg_gen_umin_i64(diff, diff, tmax);
3245
3246 /* Since we're bounded, pass as a 32-bit type. */
3247 t2 = tcg_temp_new_i32();
3248 tcg_gen_extrl_i64_i32(t2, diff);
3249
3250 desc = FIELD_DP32(desc, PREDDESC, OPRSZ, vsz / 8);
3251 desc = FIELD_DP32(desc, PREDDESC, ESZ, a->esz);
3252
3253 ptr = tcg_temp_new_ptr();
3254 tcg_gen_addi_ptr(ptr, tcg_env, pred_full_reg_offset(s, a->rd));
3255
3256 gen_helper_sve_whilel(t2, ptr, t2, tcg_constant_i32(desc));
3257 do_pred_flags(t2);
3258 return true;
3259 }
3260
3261 /*
3262 *** SVE Integer Wide Immediate - Unpredicated Group
3263 */
3264
trans_FDUP(DisasContext * s,arg_FDUP * a)3265 static bool trans_FDUP(DisasContext *s, arg_FDUP *a)
3266 {
3267 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) {
3268 return false;
3269 }
3270 if (sve_access_check(s)) {
3271 unsigned vsz = vec_full_reg_size(s);
3272 int dofs = vec_full_reg_offset(s, a->rd);
3273 uint64_t imm;
3274
3275 /* Decode the VFP immediate. */
3276 imm = vfp_expand_imm(a->esz, a->imm);
3277 tcg_gen_gvec_dup_imm(a->esz, dofs, vsz, vsz, imm);
3278 }
3279 return true;
3280 }
3281
trans_DUP_i(DisasContext * s,arg_DUP_i * a)3282 static bool trans_DUP_i(DisasContext *s, arg_DUP_i *a)
3283 {
3284 if (!dc_isar_feature(aa64_sve, s)) {
3285 return false;
3286 }
3287 if (sve_access_check(s)) {
3288 unsigned vsz = vec_full_reg_size(s);
3289 int dofs = vec_full_reg_offset(s, a->rd);
3290 tcg_gen_gvec_dup_imm(a->esz, dofs, vsz, vsz, a->imm);
3291 }
3292 return true;
3293 }
3294
TRANS_FEAT(ADD_zzi,aa64_sve,gen_gvec_fn_arg_zzi,tcg_gen_gvec_addi,a)3295 TRANS_FEAT(ADD_zzi, aa64_sve, gen_gvec_fn_arg_zzi, tcg_gen_gvec_addi, a)
3296
3297 static bool trans_SUB_zzi(DisasContext *s, arg_rri_esz *a)
3298 {
3299 a->imm = -a->imm;
3300 return trans_ADD_zzi(s, a);
3301 }
3302
trans_SUBR_zzi(DisasContext * s,arg_rri_esz * a)3303 static bool trans_SUBR_zzi(DisasContext *s, arg_rri_esz *a)
3304 {
3305 static const TCGOpcode vecop_list[] = { INDEX_op_sub_vec, 0 };
3306 static const GVecGen2s op[4] = {
3307 { .fni8 = tcg_gen_vec_sub8_i64,
3308 .fniv = tcg_gen_sub_vec,
3309 .fno = gen_helper_sve_subri_b,
3310 .opt_opc = vecop_list,
3311 .vece = MO_8,
3312 .scalar_first = true },
3313 { .fni8 = tcg_gen_vec_sub16_i64,
3314 .fniv = tcg_gen_sub_vec,
3315 .fno = gen_helper_sve_subri_h,
3316 .opt_opc = vecop_list,
3317 .vece = MO_16,
3318 .scalar_first = true },
3319 { .fni4 = tcg_gen_sub_i32,
3320 .fniv = tcg_gen_sub_vec,
3321 .fno = gen_helper_sve_subri_s,
3322 .opt_opc = vecop_list,
3323 .vece = MO_32,
3324 .scalar_first = true },
3325 { .fni8 = tcg_gen_sub_i64,
3326 .fniv = tcg_gen_sub_vec,
3327 .fno = gen_helper_sve_subri_d,
3328 .opt_opc = vecop_list,
3329 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
3330 .vece = MO_64,
3331 .scalar_first = true }
3332 };
3333
3334 if (!dc_isar_feature(aa64_sve, s)) {
3335 return false;
3336 }
3337 if (sve_access_check(s)) {
3338 unsigned vsz = vec_full_reg_size(s);
3339 tcg_gen_gvec_2s(vec_full_reg_offset(s, a->rd),
3340 vec_full_reg_offset(s, a->rn),
3341 vsz, vsz, tcg_constant_i64(a->imm), &op[a->esz]);
3342 }
3343 return true;
3344 }
3345
TRANS_FEAT(MUL_zzi,aa64_sve,gen_gvec_fn_arg_zzi,tcg_gen_gvec_muli,a)3346 TRANS_FEAT(MUL_zzi, aa64_sve, gen_gvec_fn_arg_zzi, tcg_gen_gvec_muli, a)
3347
3348 static bool do_zzi_sat(DisasContext *s, arg_rri_esz *a, bool u, bool d)
3349 {
3350 if (sve_access_check(s)) {
3351 do_sat_addsub_vec(s, a->esz, a->rd, a->rn,
3352 tcg_constant_i64(a->imm), u, d);
3353 }
3354 return true;
3355 }
3356
TRANS_FEAT(SQADD_zzi,aa64_sve,do_zzi_sat,a,false,false)3357 TRANS_FEAT(SQADD_zzi, aa64_sve, do_zzi_sat, a, false, false)
3358 TRANS_FEAT(UQADD_zzi, aa64_sve, do_zzi_sat, a, true, false)
3359 TRANS_FEAT(SQSUB_zzi, aa64_sve, do_zzi_sat, a, false, true)
3360 TRANS_FEAT(UQSUB_zzi, aa64_sve, do_zzi_sat, a, true, true)
3361
3362 static bool do_zzi_ool(DisasContext *s, arg_rri_esz *a, gen_helper_gvec_2i *fn)
3363 {
3364 if (sve_access_check(s)) {
3365 unsigned vsz = vec_full_reg_size(s);
3366 tcg_gen_gvec_2i_ool(vec_full_reg_offset(s, a->rd),
3367 vec_full_reg_offset(s, a->rn),
3368 tcg_constant_i64(a->imm), vsz, vsz, 0, fn);
3369 }
3370 return true;
3371 }
3372
3373 #define DO_ZZI(NAME, name) \
3374 static gen_helper_gvec_2i * const name##i_fns[4] = { \
3375 gen_helper_sve_##name##i_b, gen_helper_sve_##name##i_h, \
3376 gen_helper_sve_##name##i_s, gen_helper_sve_##name##i_d, \
3377 }; \
3378 TRANS_FEAT(NAME##_zzi, aa64_sve, do_zzi_ool, a, name##i_fns[a->esz])
3379
3380 DO_ZZI(SMAX, smax)
3381 DO_ZZI(UMAX, umax)
3382 DO_ZZI(SMIN, smin)
3383 DO_ZZI(UMIN, umin)
3384
3385 #undef DO_ZZI
3386
3387 static gen_helper_gvec_4 * const dot_fns[2][2] = {
3388 { gen_helper_gvec_sdot_b, gen_helper_gvec_sdot_h },
3389 { gen_helper_gvec_udot_b, gen_helper_gvec_udot_h }
3390 };
3391 TRANS_FEAT(DOT_zzzz, aa64_sve, gen_gvec_ool_zzzz,
3392 dot_fns[a->u][a->sz], a->rd, a->rn, a->rm, a->ra, 0)
3393
3394 /*
3395 * SVE Multiply - Indexed
3396 */
3397
3398 TRANS_FEAT(SDOT_zzxw_s, aa64_sve, gen_gvec_ool_arg_zzxz,
3399 gen_helper_gvec_sdot_idx_b, a)
3400 TRANS_FEAT(SDOT_zzxw_d, aa64_sve, gen_gvec_ool_arg_zzxz,
3401 gen_helper_gvec_sdot_idx_h, a)
3402 TRANS_FEAT(UDOT_zzxw_s, aa64_sve, gen_gvec_ool_arg_zzxz,
3403 gen_helper_gvec_udot_idx_b, a)
3404 TRANS_FEAT(UDOT_zzxw_d, aa64_sve, gen_gvec_ool_arg_zzxz,
3405 gen_helper_gvec_udot_idx_h, a)
3406
3407 TRANS_FEAT(SUDOT_zzxw_s, aa64_sve_i8mm, gen_gvec_ool_arg_zzxz,
3408 gen_helper_gvec_sudot_idx_b, a)
3409 TRANS_FEAT(USDOT_zzxw_s, aa64_sve_i8mm, gen_gvec_ool_arg_zzxz,
3410 gen_helper_gvec_usdot_idx_b, a)
3411
3412 #define DO_SVE2_RRX(NAME, FUNC) \
3413 TRANS_FEAT(NAME, aa64_sve, gen_gvec_ool_zzz, FUNC, \
3414 a->rd, a->rn, a->rm, a->index)
3415
3416 DO_SVE2_RRX(MUL_zzx_h, gen_helper_gvec_mul_idx_h)
3417 DO_SVE2_RRX(MUL_zzx_s, gen_helper_gvec_mul_idx_s)
3418 DO_SVE2_RRX(MUL_zzx_d, gen_helper_gvec_mul_idx_d)
3419
3420 DO_SVE2_RRX(SQDMULH_zzx_h, gen_helper_sve2_sqdmulh_idx_h)
3421 DO_SVE2_RRX(SQDMULH_zzx_s, gen_helper_sve2_sqdmulh_idx_s)
3422 DO_SVE2_RRX(SQDMULH_zzx_d, gen_helper_sve2_sqdmulh_idx_d)
3423
3424 DO_SVE2_RRX(SQRDMULH_zzx_h, gen_helper_sve2_sqrdmulh_idx_h)
3425 DO_SVE2_RRX(SQRDMULH_zzx_s, gen_helper_sve2_sqrdmulh_idx_s)
3426 DO_SVE2_RRX(SQRDMULH_zzx_d, gen_helper_sve2_sqrdmulh_idx_d)
3427
3428 #undef DO_SVE2_RRX
3429
3430 #define DO_SVE2_RRX_TB(NAME, FUNC, TOP) \
3431 TRANS_FEAT(NAME, aa64_sve, gen_gvec_ool_zzz, FUNC, \
3432 a->rd, a->rn, a->rm, (a->index << 1) | TOP)
3433
3434 DO_SVE2_RRX_TB(SQDMULLB_zzx_s, gen_helper_sve2_sqdmull_idx_s, false)
3435 DO_SVE2_RRX_TB(SQDMULLB_zzx_d, gen_helper_sve2_sqdmull_idx_d, false)
3436 DO_SVE2_RRX_TB(SQDMULLT_zzx_s, gen_helper_sve2_sqdmull_idx_s, true)
3437 DO_SVE2_RRX_TB(SQDMULLT_zzx_d, gen_helper_sve2_sqdmull_idx_d, true)
3438
3439 DO_SVE2_RRX_TB(SMULLB_zzx_s, gen_helper_sve2_smull_idx_s, false)
3440 DO_SVE2_RRX_TB(SMULLB_zzx_d, gen_helper_sve2_smull_idx_d, false)
3441 DO_SVE2_RRX_TB(SMULLT_zzx_s, gen_helper_sve2_smull_idx_s, true)
3442 DO_SVE2_RRX_TB(SMULLT_zzx_d, gen_helper_sve2_smull_idx_d, true)
3443
3444 DO_SVE2_RRX_TB(UMULLB_zzx_s, gen_helper_sve2_umull_idx_s, false)
3445 DO_SVE2_RRX_TB(UMULLB_zzx_d, gen_helper_sve2_umull_idx_d, false)
3446 DO_SVE2_RRX_TB(UMULLT_zzx_s, gen_helper_sve2_umull_idx_s, true)
3447 DO_SVE2_RRX_TB(UMULLT_zzx_d, gen_helper_sve2_umull_idx_d, true)
3448
3449 #undef DO_SVE2_RRX_TB
3450
3451 #define DO_SVE2_RRXR(NAME, FUNC) \
3452 TRANS_FEAT(NAME, aa64_sve2, gen_gvec_ool_arg_zzxz, FUNC, a)
3453
3454 DO_SVE2_RRXR(MLA_zzxz_h, gen_helper_gvec_mla_idx_h)
3455 DO_SVE2_RRXR(MLA_zzxz_s, gen_helper_gvec_mla_idx_s)
3456 DO_SVE2_RRXR(MLA_zzxz_d, gen_helper_gvec_mla_idx_d)
3457
3458 DO_SVE2_RRXR(MLS_zzxz_h, gen_helper_gvec_mls_idx_h)
3459 DO_SVE2_RRXR(MLS_zzxz_s, gen_helper_gvec_mls_idx_s)
3460 DO_SVE2_RRXR(MLS_zzxz_d, gen_helper_gvec_mls_idx_d)
3461
3462 DO_SVE2_RRXR(SQRDMLAH_zzxz_h, gen_helper_sve2_sqrdmlah_idx_h)
3463 DO_SVE2_RRXR(SQRDMLAH_zzxz_s, gen_helper_sve2_sqrdmlah_idx_s)
3464 DO_SVE2_RRXR(SQRDMLAH_zzxz_d, gen_helper_sve2_sqrdmlah_idx_d)
3465
3466 DO_SVE2_RRXR(SQRDMLSH_zzxz_h, gen_helper_sve2_sqrdmlsh_idx_h)
3467 DO_SVE2_RRXR(SQRDMLSH_zzxz_s, gen_helper_sve2_sqrdmlsh_idx_s)
3468 DO_SVE2_RRXR(SQRDMLSH_zzxz_d, gen_helper_sve2_sqrdmlsh_idx_d)
3469
3470 #undef DO_SVE2_RRXR
3471
3472 #define DO_SVE2_RRXR_TB(NAME, FUNC, TOP) \
3473 TRANS_FEAT(NAME, aa64_sve2, gen_gvec_ool_zzzz, FUNC, \
3474 a->rd, a->rn, a->rm, a->ra, (a->index << 1) | TOP)
3475
3476 DO_SVE2_RRXR_TB(SQDMLALB_zzxw_s, gen_helper_sve2_sqdmlal_idx_s, false)
3477 DO_SVE2_RRXR_TB(SQDMLALB_zzxw_d, gen_helper_sve2_sqdmlal_idx_d, false)
3478 DO_SVE2_RRXR_TB(SQDMLALT_zzxw_s, gen_helper_sve2_sqdmlal_idx_s, true)
3479 DO_SVE2_RRXR_TB(SQDMLALT_zzxw_d, gen_helper_sve2_sqdmlal_idx_d, true)
3480
3481 DO_SVE2_RRXR_TB(SQDMLSLB_zzxw_s, gen_helper_sve2_sqdmlsl_idx_s, false)
3482 DO_SVE2_RRXR_TB(SQDMLSLB_zzxw_d, gen_helper_sve2_sqdmlsl_idx_d, false)
3483 DO_SVE2_RRXR_TB(SQDMLSLT_zzxw_s, gen_helper_sve2_sqdmlsl_idx_s, true)
3484 DO_SVE2_RRXR_TB(SQDMLSLT_zzxw_d, gen_helper_sve2_sqdmlsl_idx_d, true)
3485
3486 DO_SVE2_RRXR_TB(SMLALB_zzxw_s, gen_helper_sve2_smlal_idx_s, false)
3487 DO_SVE2_RRXR_TB(SMLALB_zzxw_d, gen_helper_sve2_smlal_idx_d, false)
3488 DO_SVE2_RRXR_TB(SMLALT_zzxw_s, gen_helper_sve2_smlal_idx_s, true)
3489 DO_SVE2_RRXR_TB(SMLALT_zzxw_d, gen_helper_sve2_smlal_idx_d, true)
3490
3491 DO_SVE2_RRXR_TB(UMLALB_zzxw_s, gen_helper_sve2_umlal_idx_s, false)
3492 DO_SVE2_RRXR_TB(UMLALB_zzxw_d, gen_helper_sve2_umlal_idx_d, false)
3493 DO_SVE2_RRXR_TB(UMLALT_zzxw_s, gen_helper_sve2_umlal_idx_s, true)
3494 DO_SVE2_RRXR_TB(UMLALT_zzxw_d, gen_helper_sve2_umlal_idx_d, true)
3495
3496 DO_SVE2_RRXR_TB(SMLSLB_zzxw_s, gen_helper_sve2_smlsl_idx_s, false)
3497 DO_SVE2_RRXR_TB(SMLSLB_zzxw_d, gen_helper_sve2_smlsl_idx_d, false)
3498 DO_SVE2_RRXR_TB(SMLSLT_zzxw_s, gen_helper_sve2_smlsl_idx_s, true)
3499 DO_SVE2_RRXR_TB(SMLSLT_zzxw_d, gen_helper_sve2_smlsl_idx_d, true)
3500
3501 DO_SVE2_RRXR_TB(UMLSLB_zzxw_s, gen_helper_sve2_umlsl_idx_s, false)
3502 DO_SVE2_RRXR_TB(UMLSLB_zzxw_d, gen_helper_sve2_umlsl_idx_d, false)
3503 DO_SVE2_RRXR_TB(UMLSLT_zzxw_s, gen_helper_sve2_umlsl_idx_s, true)
3504 DO_SVE2_RRXR_TB(UMLSLT_zzxw_d, gen_helper_sve2_umlsl_idx_d, true)
3505
3506 #undef DO_SVE2_RRXR_TB
3507
3508 #define DO_SVE2_RRXR_ROT(NAME, FUNC) \
3509 TRANS_FEAT(NAME, aa64_sve2, gen_gvec_ool_zzzz, FUNC, \
3510 a->rd, a->rn, a->rm, a->ra, (a->index << 2) | a->rot)
3511
3512 DO_SVE2_RRXR_ROT(CMLA_zzxz_h, gen_helper_sve2_cmla_idx_h)
3513 DO_SVE2_RRXR_ROT(CMLA_zzxz_s, gen_helper_sve2_cmla_idx_s)
3514
3515 DO_SVE2_RRXR_ROT(SQRDCMLAH_zzxz_h, gen_helper_sve2_sqrdcmlah_idx_h)
3516 DO_SVE2_RRXR_ROT(SQRDCMLAH_zzxz_s, gen_helper_sve2_sqrdcmlah_idx_s)
3517
3518 DO_SVE2_RRXR_ROT(CDOT_zzxw_s, gen_helper_sve2_cdot_idx_s)
3519 DO_SVE2_RRXR_ROT(CDOT_zzxw_d, gen_helper_sve2_cdot_idx_d)
3520
3521 #undef DO_SVE2_RRXR_ROT
3522
3523 /*
3524 *** SVE Floating Point Multiply-Add Indexed Group
3525 */
3526
3527 static gen_helper_gvec_4_ptr * const fmla_idx_fns[4] = {
3528 NULL, gen_helper_gvec_fmla_idx_h,
3529 gen_helper_gvec_fmla_idx_s, gen_helper_gvec_fmla_idx_d
3530 };
3531 TRANS_FEAT(FMLA_zzxz, aa64_sve, gen_gvec_fpst_zzzz,
3532 fmla_idx_fns[a->esz], a->rd, a->rn, a->rm, a->ra, a->index,
3533 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
3534
3535 static gen_helper_gvec_4_ptr * const fmls_idx_fns[4][2] = {
3536 { NULL, NULL },
3537 { gen_helper_gvec_fmls_idx_h, gen_helper_gvec_ah_fmls_idx_h },
3538 { gen_helper_gvec_fmls_idx_s, gen_helper_gvec_ah_fmls_idx_s },
3539 { gen_helper_gvec_fmls_idx_d, gen_helper_gvec_ah_fmls_idx_d },
3540 };
3541 TRANS_FEAT(FMLS_zzxz, aa64_sve, gen_gvec_fpst_zzzz,
3542 fmls_idx_fns[a->esz][s->fpcr_ah],
3543 a->rd, a->rn, a->rm, a->ra, a->index,
3544 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
3545
3546 /*
3547 *** SVE Floating Point Multiply Indexed Group
3548 */
3549
3550 static gen_helper_gvec_3_ptr * const fmul_idx_fns[4] = {
3551 NULL, gen_helper_gvec_fmul_idx_h,
3552 gen_helper_gvec_fmul_idx_s, gen_helper_gvec_fmul_idx_d,
3553 };
3554 TRANS_FEAT(FMUL_zzx, aa64_sve, gen_gvec_fpst_zzz,
3555 fmul_idx_fns[a->esz], a->rd, a->rn, a->rm, a->index,
3556 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
3557
3558 /*
3559 *** SVE Floating Point Fast Reduction Group
3560 */
3561
3562 typedef void gen_helper_fp_reduce(TCGv_i64, TCGv_ptr, TCGv_ptr,
3563 TCGv_ptr, TCGv_i32);
3564
do_reduce(DisasContext * s,arg_rpr_esz * a,gen_helper_fp_reduce * fn)3565 static bool do_reduce(DisasContext *s, arg_rpr_esz *a,
3566 gen_helper_fp_reduce *fn)
3567 {
3568 unsigned vsz, p2vsz;
3569 TCGv_i32 t_desc;
3570 TCGv_ptr t_zn, t_pg, status;
3571 TCGv_i64 temp;
3572
3573 if (fn == NULL) {
3574 return false;
3575 }
3576 if (!sve_access_check(s)) {
3577 return true;
3578 }
3579
3580 vsz = vec_full_reg_size(s);
3581 p2vsz = pow2ceil(vsz);
3582 t_desc = tcg_constant_i32(simd_desc(vsz, vsz, p2vsz));
3583 temp = tcg_temp_new_i64();
3584 t_zn = tcg_temp_new_ptr();
3585 t_pg = tcg_temp_new_ptr();
3586
3587 tcg_gen_addi_ptr(t_zn, tcg_env, vec_full_reg_offset(s, a->rn));
3588 tcg_gen_addi_ptr(t_pg, tcg_env, pred_full_reg_offset(s, a->pg));
3589 status = fpstatus_ptr(a->esz == MO_16 ? FPST_A64_F16 : FPST_A64);
3590
3591 fn(temp, t_zn, t_pg, status, t_desc);
3592
3593 write_fp_dreg(s, a->rd, temp);
3594 return true;
3595 }
3596
3597 #define DO_VPZ(NAME, name) \
3598 static gen_helper_fp_reduce * const name##_fns[4] = { \
3599 NULL, gen_helper_sve_##name##_h, \
3600 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \
3601 }; \
3602 TRANS_FEAT(NAME, aa64_sve, do_reduce, a, name##_fns[a->esz])
3603
3604 #define DO_VPZ_AH(NAME, name) \
3605 static gen_helper_fp_reduce * const name##_fns[4] = { \
3606 NULL, gen_helper_sve_##name##_h, \
3607 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \
3608 }; \
3609 static gen_helper_fp_reduce * const name##_ah_fns[4] = { \
3610 NULL, gen_helper_sve_ah_##name##_h, \
3611 gen_helper_sve_ah_##name##_s, gen_helper_sve_ah_##name##_d, \
3612 }; \
3613 TRANS_FEAT(NAME, aa64_sve, do_reduce, a, \
3614 s->fpcr_ah ? name##_ah_fns[a->esz] : name##_fns[a->esz])
3615
3616 DO_VPZ(FADDV, faddv)
3617 DO_VPZ(FMINNMV, fminnmv)
3618 DO_VPZ(FMAXNMV, fmaxnmv)
3619 DO_VPZ_AH(FMINV, fminv)
3620 DO_VPZ_AH(FMAXV, fmaxv)
3621
3622 #undef DO_VPZ
3623
3624 /*
3625 *** SVE Floating Point Unary Operations - Unpredicated Group
3626 */
3627
3628 static gen_helper_gvec_2_ptr * const frecpe_fns[] = {
3629 NULL, gen_helper_gvec_frecpe_h,
3630 gen_helper_gvec_frecpe_s, gen_helper_gvec_frecpe_d,
3631 };
3632 static gen_helper_gvec_2_ptr * const frecpe_rpres_fns[] = {
3633 NULL, gen_helper_gvec_frecpe_h,
3634 gen_helper_gvec_frecpe_rpres_s, gen_helper_gvec_frecpe_d,
3635 };
3636 TRANS_FEAT(FRECPE, aa64_sve, gen_gvec_fpst_ah_arg_zz,
3637 s->fpcr_ah && dc_isar_feature(aa64_rpres, s) ?
3638 frecpe_rpres_fns[a->esz] : frecpe_fns[a->esz], a, 0)
3639
3640 static gen_helper_gvec_2_ptr * const frsqrte_fns[] = {
3641 NULL, gen_helper_gvec_frsqrte_h,
3642 gen_helper_gvec_frsqrte_s, gen_helper_gvec_frsqrte_d,
3643 };
3644 static gen_helper_gvec_2_ptr * const frsqrte_rpres_fns[] = {
3645 NULL, gen_helper_gvec_frsqrte_h,
3646 gen_helper_gvec_frsqrte_rpres_s, gen_helper_gvec_frsqrte_d,
3647 };
3648 TRANS_FEAT(FRSQRTE, aa64_sve, gen_gvec_fpst_ah_arg_zz,
3649 s->fpcr_ah && dc_isar_feature(aa64_rpres, s) ?
3650 frsqrte_rpres_fns[a->esz] : frsqrte_fns[a->esz], a, 0)
3651
3652 /*
3653 *** SVE Floating Point Compare with Zero Group
3654 */
3655
do_ppz_fp(DisasContext * s,arg_rpr_esz * a,gen_helper_gvec_3_ptr * fn)3656 static bool do_ppz_fp(DisasContext *s, arg_rpr_esz *a,
3657 gen_helper_gvec_3_ptr *fn)
3658 {
3659 if (fn == NULL) {
3660 return false;
3661 }
3662 if (sve_access_check(s)) {
3663 unsigned vsz = vec_full_reg_size(s);
3664 TCGv_ptr status =
3665 fpstatus_ptr(a->esz == MO_16 ? FPST_A64_F16 : FPST_A64);
3666
3667 tcg_gen_gvec_3_ptr(pred_full_reg_offset(s, a->rd),
3668 vec_full_reg_offset(s, a->rn),
3669 pred_full_reg_offset(s, a->pg),
3670 status, vsz, vsz, 0, fn);
3671 }
3672 return true;
3673 }
3674
3675 #define DO_PPZ(NAME, name) \
3676 static gen_helper_gvec_3_ptr * const name##_fns[] = { \
3677 NULL, gen_helper_sve_##name##_h, \
3678 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d, \
3679 }; \
3680 TRANS_FEAT(NAME, aa64_sve, do_ppz_fp, a, name##_fns[a->esz])
3681
3682 DO_PPZ(FCMGE_ppz0, fcmge0)
3683 DO_PPZ(FCMGT_ppz0, fcmgt0)
3684 DO_PPZ(FCMLE_ppz0, fcmle0)
3685 DO_PPZ(FCMLT_ppz0, fcmlt0)
3686 DO_PPZ(FCMEQ_ppz0, fcmeq0)
3687 DO_PPZ(FCMNE_ppz0, fcmne0)
3688
3689 #undef DO_PPZ
3690
3691 /*
3692 *** SVE floating-point trig multiply-add coefficient
3693 */
3694
3695 static gen_helper_gvec_3_ptr * const ftmad_fns[4] = {
3696 NULL, gen_helper_sve_ftmad_h,
3697 gen_helper_sve_ftmad_s, gen_helper_sve_ftmad_d,
3698 };
3699 TRANS_FEAT_NONSTREAMING(FTMAD, aa64_sve, gen_gvec_fpst_zzz,
3700 ftmad_fns[a->esz], a->rd, a->rn, a->rm,
3701 a->imm | (s->fpcr_ah << 3),
3702 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
3703
3704 /*
3705 *** SVE Floating Point Accumulating Reduction Group
3706 */
3707
trans_FADDA(DisasContext * s,arg_rprr_esz * a)3708 static bool trans_FADDA(DisasContext *s, arg_rprr_esz *a)
3709 {
3710 typedef void fadda_fn(TCGv_i64, TCGv_i64, TCGv_ptr,
3711 TCGv_ptr, TCGv_ptr, TCGv_i32);
3712 static fadda_fn * const fns[3] = {
3713 gen_helper_sve_fadda_h,
3714 gen_helper_sve_fadda_s,
3715 gen_helper_sve_fadda_d,
3716 };
3717 unsigned vsz = vec_full_reg_size(s);
3718 TCGv_ptr t_rm, t_pg, t_fpst;
3719 TCGv_i64 t_val;
3720 TCGv_i32 t_desc;
3721
3722 if (a->esz == 0 || !dc_isar_feature(aa64_sve, s)) {
3723 return false;
3724 }
3725 s->is_nonstreaming = true;
3726 if (!sve_access_check(s)) {
3727 return true;
3728 }
3729
3730 t_val = load_esz(tcg_env, vec_reg_offset(s, a->rn, 0, a->esz), a->esz);
3731 t_rm = tcg_temp_new_ptr();
3732 t_pg = tcg_temp_new_ptr();
3733 tcg_gen_addi_ptr(t_rm, tcg_env, vec_full_reg_offset(s, a->rm));
3734 tcg_gen_addi_ptr(t_pg, tcg_env, pred_full_reg_offset(s, a->pg));
3735 t_fpst = fpstatus_ptr(a->esz == MO_16 ? FPST_A64_F16 : FPST_A64);
3736 t_desc = tcg_constant_i32(simd_desc(vsz, vsz, 0));
3737
3738 fns[a->esz - 1](t_val, t_val, t_rm, t_pg, t_fpst, t_desc);
3739
3740 write_fp_dreg(s, a->rd, t_val);
3741 return true;
3742 }
3743
3744 /*
3745 *** SVE Floating Point Arithmetic - Unpredicated Group
3746 */
3747
3748 #define DO_FP3(NAME, name) \
3749 static gen_helper_gvec_3_ptr * const name##_fns[4] = { \
3750 NULL, gen_helper_gvec_##name##_h, \
3751 gen_helper_gvec_##name##_s, gen_helper_gvec_##name##_d \
3752 }; \
3753 TRANS_FEAT(NAME, aa64_sve, gen_gvec_fpst_arg_zzz, name##_fns[a->esz], a, 0)
3754
3755 #define DO_FP3_AH(NAME, name) \
3756 static gen_helper_gvec_3_ptr * const name##_fns[4] = { \
3757 NULL, gen_helper_gvec_##name##_h, \
3758 gen_helper_gvec_##name##_s, gen_helper_gvec_##name##_d \
3759 }; \
3760 static gen_helper_gvec_3_ptr * const name##_ah_fns[4] = { \
3761 NULL, gen_helper_gvec_ah_##name##_h, \
3762 gen_helper_gvec_ah_##name##_s, gen_helper_gvec_ah_##name##_d \
3763 }; \
3764 TRANS_FEAT(NAME, aa64_sve, gen_gvec_fpst_ah_arg_zzz, \
3765 s->fpcr_ah ? name##_ah_fns[a->esz] : name##_fns[a->esz], a, 0)
3766
3767 DO_FP3(FADD_zzz, fadd)
3768 DO_FP3(FSUB_zzz, fsub)
3769 DO_FP3(FMUL_zzz, fmul)
3770 DO_FP3_AH(FRECPS, recps)
3771 DO_FP3_AH(FRSQRTS, rsqrts)
3772
3773 #undef DO_FP3
3774
3775 static gen_helper_gvec_3_ptr * const ftsmul_fns[4] = {
3776 NULL, gen_helper_gvec_ftsmul_h,
3777 gen_helper_gvec_ftsmul_s, gen_helper_gvec_ftsmul_d
3778 };
3779 TRANS_FEAT_NONSTREAMING(FTSMUL, aa64_sve, gen_gvec_fpst_arg_zzz,
3780 ftsmul_fns[a->esz], a, 0)
3781
3782 /*
3783 *** SVE Floating Point Arithmetic - Predicated Group
3784 */
3785
3786 #define DO_ZPZZ_FP(NAME, FEAT, name) \
3787 static gen_helper_gvec_4_ptr * const name##_zpzz_fns[4] = { \
3788 NULL, gen_helper_##name##_h, \
3789 gen_helper_##name##_s, gen_helper_##name##_d \
3790 }; \
3791 TRANS_FEAT(NAME, FEAT, gen_gvec_fpst_arg_zpzz, name##_zpzz_fns[a->esz], a)
3792
3793 #define DO_ZPZZ_AH_FP(NAME, FEAT, name, ah_name) \
3794 static gen_helper_gvec_4_ptr * const name##_zpzz_fns[4] = { \
3795 NULL, gen_helper_##name##_h, \
3796 gen_helper_##name##_s, gen_helper_##name##_d \
3797 }; \
3798 static gen_helper_gvec_4_ptr * const name##_ah_zpzz_fns[4] = { \
3799 NULL, gen_helper_##ah_name##_h, \
3800 gen_helper_##ah_name##_s, gen_helper_##ah_name##_d \
3801 }; \
3802 TRANS_FEAT(NAME, FEAT, gen_gvec_fpst_arg_zpzz, \
3803 s->fpcr_ah ? name##_ah_zpzz_fns[a->esz] : \
3804 name##_zpzz_fns[a->esz], a)
3805
3806 DO_ZPZZ_FP(FADD_zpzz, aa64_sve, sve_fadd)
3807 DO_ZPZZ_FP(FSUB_zpzz, aa64_sve, sve_fsub)
3808 DO_ZPZZ_FP(FMUL_zpzz, aa64_sve, sve_fmul)
3809 DO_ZPZZ_AH_FP(FMIN_zpzz, aa64_sve, sve_fmin, sve_ah_fmin)
3810 DO_ZPZZ_AH_FP(FMAX_zpzz, aa64_sve, sve_fmax, sve_ah_fmax)
3811 DO_ZPZZ_FP(FMINNM_zpzz, aa64_sve, sve_fminnum)
3812 DO_ZPZZ_FP(FMAXNM_zpzz, aa64_sve, sve_fmaxnum)
3813 DO_ZPZZ_AH_FP(FABD, aa64_sve, sve_fabd, sve_ah_fabd)
3814 DO_ZPZZ_FP(FSCALE, aa64_sve, sve_fscalbn)
3815 DO_ZPZZ_FP(FDIV, aa64_sve, sve_fdiv)
3816 DO_ZPZZ_FP(FMULX, aa64_sve, sve_fmulx)
3817
3818 typedef void gen_helper_sve_fp2scalar(TCGv_ptr, TCGv_ptr, TCGv_ptr,
3819 TCGv_i64, TCGv_ptr, TCGv_i32);
3820
do_fp_scalar(DisasContext * s,int zd,int zn,int pg,bool is_fp16,TCGv_i64 scalar,gen_helper_sve_fp2scalar * fn)3821 static void do_fp_scalar(DisasContext *s, int zd, int zn, int pg, bool is_fp16,
3822 TCGv_i64 scalar, gen_helper_sve_fp2scalar *fn)
3823 {
3824 unsigned vsz = vec_full_reg_size(s);
3825 TCGv_ptr t_zd, t_zn, t_pg, status;
3826 TCGv_i32 desc;
3827
3828 t_zd = tcg_temp_new_ptr();
3829 t_zn = tcg_temp_new_ptr();
3830 t_pg = tcg_temp_new_ptr();
3831 tcg_gen_addi_ptr(t_zd, tcg_env, vec_full_reg_offset(s, zd));
3832 tcg_gen_addi_ptr(t_zn, tcg_env, vec_full_reg_offset(s, zn));
3833 tcg_gen_addi_ptr(t_pg, tcg_env, pred_full_reg_offset(s, pg));
3834
3835 status = fpstatus_ptr(is_fp16 ? FPST_A64_F16 : FPST_A64);
3836 desc = tcg_constant_i32(simd_desc(vsz, vsz, 0));
3837 fn(t_zd, t_zn, t_pg, scalar, status, desc);
3838 }
3839
do_fp_imm(DisasContext * s,arg_rpri_esz * a,uint64_t imm,gen_helper_sve_fp2scalar * fn)3840 static bool do_fp_imm(DisasContext *s, arg_rpri_esz *a, uint64_t imm,
3841 gen_helper_sve_fp2scalar *fn)
3842 {
3843 if (fn == NULL) {
3844 return false;
3845 }
3846 if (sve_access_check(s)) {
3847 do_fp_scalar(s, a->rd, a->rn, a->pg, a->esz == MO_16,
3848 tcg_constant_i64(imm), fn);
3849 }
3850 return true;
3851 }
3852
3853 #define DO_FP_IMM(NAME, name, const0, const1) \
3854 static gen_helper_sve_fp2scalar * const name##_fns[4] = { \
3855 NULL, gen_helper_sve_##name##_h, \
3856 gen_helper_sve_##name##_s, \
3857 gen_helper_sve_##name##_d \
3858 }; \
3859 static uint64_t const name##_const[4][2] = { \
3860 { -1, -1 }, \
3861 { float16_##const0, float16_##const1 }, \
3862 { float32_##const0, float32_##const1 }, \
3863 { float64_##const0, float64_##const1 }, \
3864 }; \
3865 TRANS_FEAT(NAME##_zpzi, aa64_sve, do_fp_imm, a, \
3866 name##_const[a->esz][a->imm], name##_fns[a->esz])
3867
3868 #define DO_FP_AH_IMM(NAME, name, const0, const1) \
3869 static gen_helper_sve_fp2scalar * const name##_fns[4] = { \
3870 NULL, gen_helper_sve_##name##_h, \
3871 gen_helper_sve_##name##_s, \
3872 gen_helper_sve_##name##_d \
3873 }; \
3874 static gen_helper_sve_fp2scalar * const name##_ah_fns[4] = { \
3875 NULL, gen_helper_sve_ah_##name##_h, \
3876 gen_helper_sve_ah_##name##_s, \
3877 gen_helper_sve_ah_##name##_d \
3878 }; \
3879 static uint64_t const name##_const[4][2] = { \
3880 { -1, -1 }, \
3881 { float16_##const0, float16_##const1 }, \
3882 { float32_##const0, float32_##const1 }, \
3883 { float64_##const0, float64_##const1 }, \
3884 }; \
3885 TRANS_FEAT(NAME##_zpzi, aa64_sve, do_fp_imm, a, \
3886 name##_const[a->esz][a->imm], \
3887 s->fpcr_ah ? name##_ah_fns[a->esz] : name##_fns[a->esz])
3888
DO_FP_IMM(FADD,fadds,half,one)3889 DO_FP_IMM(FADD, fadds, half, one)
3890 DO_FP_IMM(FSUB, fsubs, half, one)
3891 DO_FP_IMM(FMUL, fmuls, half, two)
3892 DO_FP_IMM(FSUBR, fsubrs, half, one)
3893 DO_FP_IMM(FMAXNM, fmaxnms, zero, one)
3894 DO_FP_IMM(FMINNM, fminnms, zero, one)
3895 DO_FP_AH_IMM(FMAX, fmaxs, zero, one)
3896 DO_FP_AH_IMM(FMIN, fmins, zero, one)
3897
3898 #undef DO_FP_IMM
3899
3900 static bool do_fp_cmp(DisasContext *s, arg_rprr_esz *a,
3901 gen_helper_gvec_4_ptr *fn)
3902 {
3903 if (fn == NULL) {
3904 return false;
3905 }
3906 if (sve_access_check(s)) {
3907 unsigned vsz = vec_full_reg_size(s);
3908 TCGv_ptr status = fpstatus_ptr(a->esz == MO_16 ? FPST_A64_F16 : FPST_A64);
3909 tcg_gen_gvec_4_ptr(pred_full_reg_offset(s, a->rd),
3910 vec_full_reg_offset(s, a->rn),
3911 vec_full_reg_offset(s, a->rm),
3912 pred_full_reg_offset(s, a->pg),
3913 status, vsz, vsz, 0, fn);
3914 }
3915 return true;
3916 }
3917
3918 #define DO_FPCMP(NAME, name) \
3919 static gen_helper_gvec_4_ptr * const name##_fns[4] = { \
3920 NULL, gen_helper_sve_##name##_h, \
3921 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
3922 }; \
3923 TRANS_FEAT(NAME##_ppzz, aa64_sve, do_fp_cmp, a, name##_fns[a->esz])
3924
3925 DO_FPCMP(FCMGE, fcmge)
3926 DO_FPCMP(FCMGT, fcmgt)
3927 DO_FPCMP(FCMEQ, fcmeq)
3928 DO_FPCMP(FCMNE, fcmne)
3929 DO_FPCMP(FCMUO, fcmuo)
3930 DO_FPCMP(FACGE, facge)
3931 DO_FPCMP(FACGT, facgt)
3932
3933 #undef DO_FPCMP
3934
3935 static gen_helper_gvec_4_ptr * const fcadd_fns[] = {
3936 NULL, gen_helper_sve_fcadd_h,
3937 gen_helper_sve_fcadd_s, gen_helper_sve_fcadd_d,
3938 };
3939 TRANS_FEAT(FCADD, aa64_sve, gen_gvec_fpst_zzzp, fcadd_fns[a->esz],
3940 a->rd, a->rn, a->rm, a->pg, a->rot | (s->fpcr_ah << 1),
3941 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
3942
3943 #define DO_FMLA(NAME, name, ah_name) \
3944 static gen_helper_gvec_5_ptr * const name##_fns[4] = { \
3945 NULL, gen_helper_sve_##name##_h, \
3946 gen_helper_sve_##name##_s, gen_helper_sve_##name##_d \
3947 }; \
3948 static gen_helper_gvec_5_ptr * const name##_ah_fns[4] = { \
3949 NULL, gen_helper_sve_##ah_name##_h, \
3950 gen_helper_sve_##ah_name##_s, gen_helper_sve_##ah_name##_d \
3951 }; \
3952 TRANS_FEAT(NAME, aa64_sve, gen_gvec_fpst_zzzzp, \
3953 s->fpcr_ah ? name##_ah_fns[a->esz] : name##_fns[a->esz], \
3954 a->rd, a->rn, a->rm, a->ra, a->pg, 0, \
3955 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
3956
3957 /* We don't need an ah_fmla_zpzzz because fmla doesn't negate anything */
3958 DO_FMLA(FMLA_zpzzz, fmla_zpzzz, fmla_zpzzz)
3959 DO_FMLA(FMLS_zpzzz, fmls_zpzzz, ah_fmls_zpzzz)
3960 DO_FMLA(FNMLA_zpzzz, fnmla_zpzzz, ah_fnmla_zpzzz)
3961 DO_FMLA(FNMLS_zpzzz, fnmls_zpzzz, ah_fnmls_zpzzz)
3962
3963 #undef DO_FMLA
3964
3965 static gen_helper_gvec_5_ptr * const fcmla_fns[4] = {
3966 NULL, gen_helper_sve_fcmla_zpzzz_h,
3967 gen_helper_sve_fcmla_zpzzz_s, gen_helper_sve_fcmla_zpzzz_d,
3968 };
3969 TRANS_FEAT(FCMLA_zpzzz, aa64_sve, gen_gvec_fpst_zzzzp, fcmla_fns[a->esz],
3970 a->rd, a->rn, a->rm, a->ra, a->pg, a->rot | (s->fpcr_ah << 2),
3971 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
3972
3973 static gen_helper_gvec_4_ptr * const fcmla_idx_fns[4] = {
3974 NULL, gen_helper_gvec_fcmlah_idx, gen_helper_gvec_fcmlas_idx, NULL
3975 };
3976 TRANS_FEAT(FCMLA_zzxz, aa64_sve, gen_gvec_fpst_zzzz, fcmla_idx_fns[a->esz],
3977 a->rd, a->rn, a->rm, a->ra, a->index * 4 + a->rot,
3978 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
3979
3980 /*
3981 *** SVE Floating Point Unary Operations Predicated Group
3982 */
3983
3984 TRANS_FEAT(FCVT_sh, aa64_sve, gen_gvec_fpst_arg_zpz,
3985 gen_helper_sve_fcvt_sh, a, 0, FPST_A64)
3986 TRANS_FEAT(FCVT_hs, aa64_sve, gen_gvec_fpst_arg_zpz,
3987 gen_helper_sve_fcvt_hs, a, 0, FPST_A64_F16)
3988
3989 TRANS_FEAT(BFCVT, aa64_sve_bf16, gen_gvec_fpst_arg_zpz,
3990 gen_helper_sve_bfcvt, a, 0,
3991 s->fpcr_ah ? FPST_AH : FPST_A64)
3992
3993 TRANS_FEAT(FCVT_dh, aa64_sve, gen_gvec_fpst_arg_zpz,
3994 gen_helper_sve_fcvt_dh, a, 0, FPST_A64)
3995 TRANS_FEAT(FCVT_hd, aa64_sve, gen_gvec_fpst_arg_zpz,
3996 gen_helper_sve_fcvt_hd, a, 0, FPST_A64_F16)
3997 TRANS_FEAT(FCVT_ds, aa64_sve, gen_gvec_fpst_arg_zpz,
3998 gen_helper_sve_fcvt_ds, a, 0, FPST_A64)
3999 TRANS_FEAT(FCVT_sd, aa64_sve, gen_gvec_fpst_arg_zpz,
4000 gen_helper_sve_fcvt_sd, a, 0, FPST_A64)
4001
4002 TRANS_FEAT(FCVTZS_hh, aa64_sve, gen_gvec_fpst_arg_zpz,
4003 gen_helper_sve_fcvtzs_hh, a, 0, FPST_A64_F16)
4004 TRANS_FEAT(FCVTZU_hh, aa64_sve, gen_gvec_fpst_arg_zpz,
4005 gen_helper_sve_fcvtzu_hh, a, 0, FPST_A64_F16)
4006 TRANS_FEAT(FCVTZS_hs, aa64_sve, gen_gvec_fpst_arg_zpz,
4007 gen_helper_sve_fcvtzs_hs, a, 0, FPST_A64_F16)
4008 TRANS_FEAT(FCVTZU_hs, aa64_sve, gen_gvec_fpst_arg_zpz,
4009 gen_helper_sve_fcvtzu_hs, a, 0, FPST_A64_F16)
4010 TRANS_FEAT(FCVTZS_hd, aa64_sve, gen_gvec_fpst_arg_zpz,
4011 gen_helper_sve_fcvtzs_hd, a, 0, FPST_A64_F16)
4012 TRANS_FEAT(FCVTZU_hd, aa64_sve, gen_gvec_fpst_arg_zpz,
4013 gen_helper_sve_fcvtzu_hd, a, 0, FPST_A64_F16)
4014
4015 TRANS_FEAT(FCVTZS_ss, aa64_sve, gen_gvec_fpst_arg_zpz,
4016 gen_helper_sve_fcvtzs_ss, a, 0, FPST_A64)
4017 TRANS_FEAT(FCVTZU_ss, aa64_sve, gen_gvec_fpst_arg_zpz,
4018 gen_helper_sve_fcvtzu_ss, a, 0, FPST_A64)
4019 TRANS_FEAT(FCVTZS_sd, aa64_sve, gen_gvec_fpst_arg_zpz,
4020 gen_helper_sve_fcvtzs_sd, a, 0, FPST_A64)
4021 TRANS_FEAT(FCVTZU_sd, aa64_sve, gen_gvec_fpst_arg_zpz,
4022 gen_helper_sve_fcvtzu_sd, a, 0, FPST_A64)
4023 TRANS_FEAT(FCVTZS_ds, aa64_sve, gen_gvec_fpst_arg_zpz,
4024 gen_helper_sve_fcvtzs_ds, a, 0, FPST_A64)
4025 TRANS_FEAT(FCVTZU_ds, aa64_sve, gen_gvec_fpst_arg_zpz,
4026 gen_helper_sve_fcvtzu_ds, a, 0, FPST_A64)
4027
4028 TRANS_FEAT(FCVTZS_dd, aa64_sve, gen_gvec_fpst_arg_zpz,
4029 gen_helper_sve_fcvtzs_dd, a, 0, FPST_A64)
4030 TRANS_FEAT(FCVTZU_dd, aa64_sve, gen_gvec_fpst_arg_zpz,
4031 gen_helper_sve_fcvtzu_dd, a, 0, FPST_A64)
4032
4033 static gen_helper_gvec_3_ptr * const frint_fns[] = {
4034 NULL,
4035 gen_helper_sve_frint_h,
4036 gen_helper_sve_frint_s,
4037 gen_helper_sve_frint_d
4038 };
4039 TRANS_FEAT(FRINTI, aa64_sve, gen_gvec_fpst_arg_zpz, frint_fns[a->esz],
4040 a, 0, a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
4041
4042 static gen_helper_gvec_3_ptr * const frintx_fns[] = {
4043 NULL,
4044 gen_helper_sve_frintx_h,
4045 gen_helper_sve_frintx_s,
4046 gen_helper_sve_frintx_d
4047 };
4048 TRANS_FEAT(FRINTX, aa64_sve, gen_gvec_fpst_arg_zpz, frintx_fns[a->esz],
4049 a, 0, a->esz == MO_16 ? FPST_A64_F16 : FPST_A64);
4050
do_frint_mode(DisasContext * s,arg_rpr_esz * a,ARMFPRounding mode,gen_helper_gvec_3_ptr * fn)4051 static bool do_frint_mode(DisasContext *s, arg_rpr_esz *a,
4052 ARMFPRounding mode, gen_helper_gvec_3_ptr *fn)
4053 {
4054 unsigned vsz;
4055 TCGv_i32 tmode;
4056 TCGv_ptr status;
4057
4058 if (fn == NULL) {
4059 return false;
4060 }
4061 if (!sve_access_check(s)) {
4062 return true;
4063 }
4064
4065 vsz = vec_full_reg_size(s);
4066 status = fpstatus_ptr(a->esz == MO_16 ? FPST_A64_F16 : FPST_A64);
4067 tmode = gen_set_rmode(mode, status);
4068
4069 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, a->rd),
4070 vec_full_reg_offset(s, a->rn),
4071 pred_full_reg_offset(s, a->pg),
4072 status, vsz, vsz, 0, fn);
4073
4074 gen_restore_rmode(tmode, status);
4075 return true;
4076 }
4077
4078 TRANS_FEAT(FRINTN, aa64_sve, do_frint_mode, a,
4079 FPROUNDING_TIEEVEN, frint_fns[a->esz])
4080 TRANS_FEAT(FRINTP, aa64_sve, do_frint_mode, a,
4081 FPROUNDING_POSINF, frint_fns[a->esz])
4082 TRANS_FEAT(FRINTM, aa64_sve, do_frint_mode, a,
4083 FPROUNDING_NEGINF, frint_fns[a->esz])
4084 TRANS_FEAT(FRINTZ, aa64_sve, do_frint_mode, a,
4085 FPROUNDING_ZERO, frint_fns[a->esz])
4086 TRANS_FEAT(FRINTA, aa64_sve, do_frint_mode, a,
4087 FPROUNDING_TIEAWAY, frint_fns[a->esz])
4088
4089 static gen_helper_gvec_3_ptr * const frecpx_fns[] = {
4090 NULL, gen_helper_sve_frecpx_h,
4091 gen_helper_sve_frecpx_s, gen_helper_sve_frecpx_d,
4092 };
4093 TRANS_FEAT(FRECPX, aa64_sve, gen_gvec_fpst_arg_zpz, frecpx_fns[a->esz],
4094 a, 0, select_ah_fpst(s, a->esz))
4095
4096 static gen_helper_gvec_3_ptr * const fsqrt_fns[] = {
4097 NULL, gen_helper_sve_fsqrt_h,
4098 gen_helper_sve_fsqrt_s, gen_helper_sve_fsqrt_d,
4099 };
4100 TRANS_FEAT(FSQRT, aa64_sve, gen_gvec_fpst_arg_zpz, fsqrt_fns[a->esz],
4101 a, 0, a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
4102
4103 TRANS_FEAT(SCVTF_hh, aa64_sve, gen_gvec_fpst_arg_zpz,
4104 gen_helper_sve_scvt_hh, a, 0, FPST_A64_F16)
4105 TRANS_FEAT(SCVTF_sh, aa64_sve, gen_gvec_fpst_arg_zpz,
4106 gen_helper_sve_scvt_sh, a, 0, FPST_A64_F16)
4107 TRANS_FEAT(SCVTF_dh, aa64_sve, gen_gvec_fpst_arg_zpz,
4108 gen_helper_sve_scvt_dh, a, 0, FPST_A64_F16)
4109
4110 TRANS_FEAT(SCVTF_ss, aa64_sve, gen_gvec_fpst_arg_zpz,
4111 gen_helper_sve_scvt_ss, a, 0, FPST_A64)
4112 TRANS_FEAT(SCVTF_ds, aa64_sve, gen_gvec_fpst_arg_zpz,
4113 gen_helper_sve_scvt_ds, a, 0, FPST_A64)
4114
4115 TRANS_FEAT(SCVTF_sd, aa64_sve, gen_gvec_fpst_arg_zpz,
4116 gen_helper_sve_scvt_sd, a, 0, FPST_A64)
4117 TRANS_FEAT(SCVTF_dd, aa64_sve, gen_gvec_fpst_arg_zpz,
4118 gen_helper_sve_scvt_dd, a, 0, FPST_A64)
4119
4120 TRANS_FEAT(UCVTF_hh, aa64_sve, gen_gvec_fpst_arg_zpz,
4121 gen_helper_sve_ucvt_hh, a, 0, FPST_A64_F16)
4122 TRANS_FEAT(UCVTF_sh, aa64_sve, gen_gvec_fpst_arg_zpz,
4123 gen_helper_sve_ucvt_sh, a, 0, FPST_A64_F16)
4124 TRANS_FEAT(UCVTF_dh, aa64_sve, gen_gvec_fpst_arg_zpz,
4125 gen_helper_sve_ucvt_dh, a, 0, FPST_A64_F16)
4126
4127 TRANS_FEAT(UCVTF_ss, aa64_sve, gen_gvec_fpst_arg_zpz,
4128 gen_helper_sve_ucvt_ss, a, 0, FPST_A64)
4129 TRANS_FEAT(UCVTF_ds, aa64_sve, gen_gvec_fpst_arg_zpz,
4130 gen_helper_sve_ucvt_ds, a, 0, FPST_A64)
4131 TRANS_FEAT(UCVTF_sd, aa64_sve, gen_gvec_fpst_arg_zpz,
4132 gen_helper_sve_ucvt_sd, a, 0, FPST_A64)
4133
4134 TRANS_FEAT(UCVTF_dd, aa64_sve, gen_gvec_fpst_arg_zpz,
4135 gen_helper_sve_ucvt_dd, a, 0, FPST_A64)
4136
4137 /*
4138 *** SVE Memory - 32-bit Gather and Unsized Contiguous Group
4139 */
4140
4141 /* Subroutine loading a vector register at VOFS of LEN bytes.
4142 * The load should begin at the address Rn + IMM.
4143 */
4144
gen_sve_ldr(DisasContext * s,TCGv_ptr base,int vofs,int len,int rn,int imm)4145 void gen_sve_ldr(DisasContext *s, TCGv_ptr base, int vofs,
4146 int len, int rn, int imm)
4147 {
4148 int len_align = QEMU_ALIGN_DOWN(len, 16);
4149 int len_remain = len % 16;
4150 int nparts = len / 16 + ctpop8(len_remain);
4151 int midx = get_mem_index(s);
4152 TCGv_i64 dirty_addr, clean_addr, t0, t1;
4153 TCGv_i128 t16;
4154
4155 dirty_addr = tcg_temp_new_i64();
4156 tcg_gen_addi_i64(dirty_addr, cpu_reg_sp(s, rn), imm);
4157 clean_addr = gen_mte_checkN(s, dirty_addr, false, rn != 31, len, MO_8);
4158
4159 /*
4160 * Note that unpredicated load/store of vector/predicate registers
4161 * are defined as a stream of bytes, which equates to little-endian
4162 * operations on larger quantities.
4163 * Attempt to keep code expansion to a minimum by limiting the
4164 * amount of unrolling done.
4165 */
4166 if (nparts <= 4) {
4167 int i;
4168
4169 t0 = tcg_temp_new_i64();
4170 t1 = tcg_temp_new_i64();
4171 t16 = tcg_temp_new_i128();
4172
4173 for (i = 0; i < len_align; i += 16) {
4174 tcg_gen_qemu_ld_i128(t16, clean_addr, midx,
4175 MO_LE | MO_128 | MO_ATOM_NONE);
4176 tcg_gen_extr_i128_i64(t0, t1, t16);
4177 tcg_gen_st_i64(t0, base, vofs + i);
4178 tcg_gen_st_i64(t1, base, vofs + i + 8);
4179 tcg_gen_addi_i64(clean_addr, clean_addr, 16);
4180 }
4181 } else {
4182 TCGLabel *loop = gen_new_label();
4183 TCGv_ptr tp, i = tcg_temp_new_ptr();
4184
4185 tcg_gen_movi_ptr(i, 0);
4186 gen_set_label(loop);
4187
4188 t16 = tcg_temp_new_i128();
4189 tcg_gen_qemu_ld_i128(t16, clean_addr, midx,
4190 MO_LE | MO_128 | MO_ATOM_NONE);
4191 tcg_gen_addi_i64(clean_addr, clean_addr, 16);
4192
4193 tp = tcg_temp_new_ptr();
4194 tcg_gen_add_ptr(tp, base, i);
4195 tcg_gen_addi_ptr(i, i, 16);
4196
4197 t0 = tcg_temp_new_i64();
4198 t1 = tcg_temp_new_i64();
4199 tcg_gen_extr_i128_i64(t0, t1, t16);
4200
4201 tcg_gen_st_i64(t0, tp, vofs);
4202 tcg_gen_st_i64(t1, tp, vofs + 8);
4203
4204 tcg_gen_brcondi_ptr(TCG_COND_LTU, i, len_align, loop);
4205 }
4206
4207 /*
4208 * Predicate register loads can be any multiple of 2.
4209 * Note that we still store the entire 64-bit unit into tcg_env.
4210 */
4211 if (len_remain >= 8) {
4212 t0 = tcg_temp_new_i64();
4213 tcg_gen_qemu_ld_i64(t0, clean_addr, midx, MO_LEUQ | MO_ATOM_NONE);
4214 tcg_gen_st_i64(t0, base, vofs + len_align);
4215 len_remain -= 8;
4216 len_align += 8;
4217 if (len_remain) {
4218 tcg_gen_addi_i64(clean_addr, clean_addr, 8);
4219 }
4220 }
4221 if (len_remain) {
4222 t0 = tcg_temp_new_i64();
4223 switch (len_remain) {
4224 case 2:
4225 case 4:
4226 case 8:
4227 tcg_gen_qemu_ld_i64(t0, clean_addr, midx,
4228 MO_LE | ctz32(len_remain) | MO_ATOM_NONE);
4229 break;
4230
4231 case 6:
4232 t1 = tcg_temp_new_i64();
4233 tcg_gen_qemu_ld_i64(t0, clean_addr, midx, MO_LEUL | MO_ATOM_NONE);
4234 tcg_gen_addi_i64(clean_addr, clean_addr, 4);
4235 tcg_gen_qemu_ld_i64(t1, clean_addr, midx, MO_LEUW | MO_ATOM_NONE);
4236 tcg_gen_deposit_i64(t0, t0, t1, 32, 32);
4237 break;
4238
4239 default:
4240 g_assert_not_reached();
4241 }
4242 tcg_gen_st_i64(t0, base, vofs + len_align);
4243 }
4244 }
4245
4246 /* Similarly for stores. */
gen_sve_str(DisasContext * s,TCGv_ptr base,int vofs,int len,int rn,int imm)4247 void gen_sve_str(DisasContext *s, TCGv_ptr base, int vofs,
4248 int len, int rn, int imm)
4249 {
4250 int len_align = QEMU_ALIGN_DOWN(len, 16);
4251 int len_remain = len % 16;
4252 int nparts = len / 16 + ctpop8(len_remain);
4253 int midx = get_mem_index(s);
4254 TCGv_i64 dirty_addr, clean_addr, t0, t1;
4255 TCGv_i128 t16;
4256
4257 dirty_addr = tcg_temp_new_i64();
4258 tcg_gen_addi_i64(dirty_addr, cpu_reg_sp(s, rn), imm);
4259 clean_addr = gen_mte_checkN(s, dirty_addr, false, rn != 31, len, MO_8);
4260
4261 /* Note that unpredicated load/store of vector/predicate registers
4262 * are defined as a stream of bytes, which equates to little-endian
4263 * operations on larger quantities. There is no nice way to force
4264 * a little-endian store for aarch64_be-linux-user out of line.
4265 *
4266 * Attempt to keep code expansion to a minimum by limiting the
4267 * amount of unrolling done.
4268 */
4269 if (nparts <= 4) {
4270 int i;
4271
4272 t0 = tcg_temp_new_i64();
4273 t1 = tcg_temp_new_i64();
4274 t16 = tcg_temp_new_i128();
4275 for (i = 0; i < len_align; i += 16) {
4276 tcg_gen_ld_i64(t0, base, vofs + i);
4277 tcg_gen_ld_i64(t1, base, vofs + i + 8);
4278 tcg_gen_concat_i64_i128(t16, t0, t1);
4279 tcg_gen_qemu_st_i128(t16, clean_addr, midx,
4280 MO_LE | MO_128 | MO_ATOM_NONE);
4281 tcg_gen_addi_i64(clean_addr, clean_addr, 16);
4282 }
4283 } else {
4284 TCGLabel *loop = gen_new_label();
4285 TCGv_ptr tp, i = tcg_temp_new_ptr();
4286
4287 tcg_gen_movi_ptr(i, 0);
4288 gen_set_label(loop);
4289
4290 t0 = tcg_temp_new_i64();
4291 t1 = tcg_temp_new_i64();
4292 tp = tcg_temp_new_ptr();
4293 tcg_gen_add_ptr(tp, base, i);
4294 tcg_gen_ld_i64(t0, tp, vofs);
4295 tcg_gen_ld_i64(t1, tp, vofs + 8);
4296 tcg_gen_addi_ptr(i, i, 16);
4297
4298 t16 = tcg_temp_new_i128();
4299 tcg_gen_concat_i64_i128(t16, t0, t1);
4300
4301 tcg_gen_qemu_st_i128(t16, clean_addr, midx,
4302 MO_LE | MO_128 | MO_ATOM_NONE);
4303 tcg_gen_addi_i64(clean_addr, clean_addr, 16);
4304
4305 tcg_gen_brcondi_ptr(TCG_COND_LTU, i, len_align, loop);
4306 }
4307
4308 /* Predicate register stores can be any multiple of 2. */
4309 if (len_remain >= 8) {
4310 t0 = tcg_temp_new_i64();
4311 tcg_gen_ld_i64(t0, base, vofs + len_align);
4312 tcg_gen_qemu_st_i64(t0, clean_addr, midx, MO_LEUQ | MO_ATOM_NONE);
4313 len_remain -= 8;
4314 len_align += 8;
4315 if (len_remain) {
4316 tcg_gen_addi_i64(clean_addr, clean_addr, 8);
4317 }
4318 }
4319 if (len_remain) {
4320 t0 = tcg_temp_new_i64();
4321 tcg_gen_ld_i64(t0, base, vofs + len_align);
4322
4323 switch (len_remain) {
4324 case 2:
4325 case 4:
4326 case 8:
4327 tcg_gen_qemu_st_i64(t0, clean_addr, midx,
4328 MO_LE | ctz32(len_remain) | MO_ATOM_NONE);
4329 break;
4330
4331 case 6:
4332 tcg_gen_qemu_st_i64(t0, clean_addr, midx, MO_LEUL | MO_ATOM_NONE);
4333 tcg_gen_addi_i64(clean_addr, clean_addr, 4);
4334 tcg_gen_shri_i64(t0, t0, 32);
4335 tcg_gen_qemu_st_i64(t0, clean_addr, midx, MO_LEUW | MO_ATOM_NONE);
4336 break;
4337
4338 default:
4339 g_assert_not_reached();
4340 }
4341 }
4342 }
4343
trans_LDR_zri(DisasContext * s,arg_rri * a)4344 static bool trans_LDR_zri(DisasContext *s, arg_rri *a)
4345 {
4346 if (!dc_isar_feature(aa64_sve, s)) {
4347 return false;
4348 }
4349 if (sve_access_check(s)) {
4350 int size = vec_full_reg_size(s);
4351 int off = vec_full_reg_offset(s, a->rd);
4352 gen_sve_ldr(s, tcg_env, off, size, a->rn, a->imm * size);
4353 }
4354 return true;
4355 }
4356
trans_LDR_pri(DisasContext * s,arg_rri * a)4357 static bool trans_LDR_pri(DisasContext *s, arg_rri *a)
4358 {
4359 if (!dc_isar_feature(aa64_sve, s)) {
4360 return false;
4361 }
4362 if (sve_access_check(s)) {
4363 int size = pred_full_reg_size(s);
4364 int off = pred_full_reg_offset(s, a->rd);
4365 gen_sve_ldr(s, tcg_env, off, size, a->rn, a->imm * size);
4366 }
4367 return true;
4368 }
4369
trans_STR_zri(DisasContext * s,arg_rri * a)4370 static bool trans_STR_zri(DisasContext *s, arg_rri *a)
4371 {
4372 if (!dc_isar_feature(aa64_sve, s)) {
4373 return false;
4374 }
4375 if (sve_access_check(s)) {
4376 int size = vec_full_reg_size(s);
4377 int off = vec_full_reg_offset(s, a->rd);
4378 gen_sve_str(s, tcg_env, off, size, a->rn, a->imm * size);
4379 }
4380 return true;
4381 }
4382
trans_STR_pri(DisasContext * s,arg_rri * a)4383 static bool trans_STR_pri(DisasContext *s, arg_rri *a)
4384 {
4385 if (!dc_isar_feature(aa64_sve, s)) {
4386 return false;
4387 }
4388 if (sve_access_check(s)) {
4389 int size = pred_full_reg_size(s);
4390 int off = pred_full_reg_offset(s, a->rd);
4391 gen_sve_str(s, tcg_env, off, size, a->rn, a->imm * size);
4392 }
4393 return true;
4394 }
4395
4396 /*
4397 *** SVE Memory - Contiguous Load Group
4398 */
4399
4400 /* The memory mode of the dtype. */
4401 static const MemOp dtype_mop[16] = {
4402 MO_UB, MO_UB, MO_UB, MO_UB,
4403 MO_SL, MO_UW, MO_UW, MO_UW,
4404 MO_SW, MO_SW, MO_UL, MO_UL,
4405 MO_SB, MO_SB, MO_SB, MO_UQ
4406 };
4407
4408 #define dtype_msz(x) (dtype_mop[x] & MO_SIZE)
4409
4410 /* The vector element size of dtype. */
4411 static const uint8_t dtype_esz[16] = {
4412 0, 1, 2, 3,
4413 3, 1, 2, 3,
4414 3, 2, 2, 3,
4415 3, 2, 1, 3
4416 };
4417
make_svemte_desc(DisasContext * s,unsigned vsz,uint32_t nregs,uint32_t msz,bool is_write,uint32_t data)4418 uint32_t make_svemte_desc(DisasContext *s, unsigned vsz, uint32_t nregs,
4419 uint32_t msz, bool is_write, uint32_t data)
4420 {
4421 uint32_t sizem1;
4422 uint32_t desc = 0;
4423
4424 /* Assert all of the data fits, with or without MTE enabled. */
4425 assert(nregs >= 1 && nregs <= 4);
4426 sizem1 = (nregs << msz) - 1;
4427 assert(sizem1 <= R_MTEDESC_SIZEM1_MASK >> R_MTEDESC_SIZEM1_SHIFT);
4428 assert(data < 1u << SVE_MTEDESC_SHIFT);
4429
4430 if (s->mte_active[0]) {
4431 desc = FIELD_DP32(desc, MTEDESC, MIDX, get_mem_index(s));
4432 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid);
4433 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma);
4434 desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write);
4435 desc = FIELD_DP32(desc, MTEDESC, SIZEM1, sizem1);
4436 desc <<= SVE_MTEDESC_SHIFT;
4437 }
4438 return simd_desc(vsz, vsz, desc | data);
4439 }
4440
do_mem_zpa(DisasContext * s,int zt,int pg,TCGv_i64 addr,int dtype,uint32_t nregs,bool is_write,gen_helper_gvec_mem * fn)4441 static void do_mem_zpa(DisasContext *s, int zt, int pg, TCGv_i64 addr,
4442 int dtype, uint32_t nregs, bool is_write,
4443 gen_helper_gvec_mem *fn)
4444 {
4445 TCGv_ptr t_pg;
4446 uint32_t desc;
4447
4448 if (!s->mte_active[0]) {
4449 addr = clean_data_tbi(s, addr);
4450 }
4451
4452 /*
4453 * For e.g. LD4, there are not enough arguments to pass all 4
4454 * registers as pointers, so encode the regno into the data field.
4455 * For consistency, do this even for LD1.
4456 */
4457 desc = make_svemte_desc(s, vec_full_reg_size(s), nregs,
4458 dtype_msz(dtype), is_write, zt);
4459 t_pg = tcg_temp_new_ptr();
4460
4461 tcg_gen_addi_ptr(t_pg, tcg_env, pred_full_reg_offset(s, pg));
4462 fn(tcg_env, t_pg, addr, tcg_constant_i32(desc));
4463 }
4464
4465 /* Indexed by [mte][be][dtype][nreg] */
4466 static gen_helper_gvec_mem * const ldr_fns[2][2][16][4] = {
4467 { /* mte inactive, little-endian */
4468 { { gen_helper_sve_ld1bb_r, gen_helper_sve_ld2bb_r,
4469 gen_helper_sve_ld3bb_r, gen_helper_sve_ld4bb_r },
4470 { gen_helper_sve_ld1bhu_r, NULL, NULL, NULL },
4471 { gen_helper_sve_ld1bsu_r, NULL, NULL, NULL },
4472 { gen_helper_sve_ld1bdu_r, NULL, NULL, NULL },
4473
4474 { gen_helper_sve_ld1sds_le_r, NULL, NULL, NULL },
4475 { gen_helper_sve_ld1hh_le_r, gen_helper_sve_ld2hh_le_r,
4476 gen_helper_sve_ld3hh_le_r, gen_helper_sve_ld4hh_le_r },
4477 { gen_helper_sve_ld1hsu_le_r, NULL, NULL, NULL },
4478 { gen_helper_sve_ld1hdu_le_r, NULL, NULL, NULL },
4479
4480 { gen_helper_sve_ld1hds_le_r, NULL, NULL, NULL },
4481 { gen_helper_sve_ld1hss_le_r, NULL, NULL, NULL },
4482 { gen_helper_sve_ld1ss_le_r, gen_helper_sve_ld2ss_le_r,
4483 gen_helper_sve_ld3ss_le_r, gen_helper_sve_ld4ss_le_r },
4484 { gen_helper_sve_ld1sdu_le_r, NULL, NULL, NULL },
4485
4486 { gen_helper_sve_ld1bds_r, NULL, NULL, NULL },
4487 { gen_helper_sve_ld1bss_r, NULL, NULL, NULL },
4488 { gen_helper_sve_ld1bhs_r, NULL, NULL, NULL },
4489 { gen_helper_sve_ld1dd_le_r, gen_helper_sve_ld2dd_le_r,
4490 gen_helper_sve_ld3dd_le_r, gen_helper_sve_ld4dd_le_r } },
4491
4492 /* mte inactive, big-endian */
4493 { { gen_helper_sve_ld1bb_r, gen_helper_sve_ld2bb_r,
4494 gen_helper_sve_ld3bb_r, gen_helper_sve_ld4bb_r },
4495 { gen_helper_sve_ld1bhu_r, NULL, NULL, NULL },
4496 { gen_helper_sve_ld1bsu_r, NULL, NULL, NULL },
4497 { gen_helper_sve_ld1bdu_r, NULL, NULL, NULL },
4498
4499 { gen_helper_sve_ld1sds_be_r, NULL, NULL, NULL },
4500 { gen_helper_sve_ld1hh_be_r, gen_helper_sve_ld2hh_be_r,
4501 gen_helper_sve_ld3hh_be_r, gen_helper_sve_ld4hh_be_r },
4502 { gen_helper_sve_ld1hsu_be_r, NULL, NULL, NULL },
4503 { gen_helper_sve_ld1hdu_be_r, NULL, NULL, NULL },
4504
4505 { gen_helper_sve_ld1hds_be_r, NULL, NULL, NULL },
4506 { gen_helper_sve_ld1hss_be_r, NULL, NULL, NULL },
4507 { gen_helper_sve_ld1ss_be_r, gen_helper_sve_ld2ss_be_r,
4508 gen_helper_sve_ld3ss_be_r, gen_helper_sve_ld4ss_be_r },
4509 { gen_helper_sve_ld1sdu_be_r, NULL, NULL, NULL },
4510
4511 { gen_helper_sve_ld1bds_r, NULL, NULL, NULL },
4512 { gen_helper_sve_ld1bss_r, NULL, NULL, NULL },
4513 { gen_helper_sve_ld1bhs_r, NULL, NULL, NULL },
4514 { gen_helper_sve_ld1dd_be_r, gen_helper_sve_ld2dd_be_r,
4515 gen_helper_sve_ld3dd_be_r, gen_helper_sve_ld4dd_be_r } } },
4516
4517 { /* mte active, little-endian */
4518 { { gen_helper_sve_ld1bb_r_mte,
4519 gen_helper_sve_ld2bb_r_mte,
4520 gen_helper_sve_ld3bb_r_mte,
4521 gen_helper_sve_ld4bb_r_mte },
4522 { gen_helper_sve_ld1bhu_r_mte, NULL, NULL, NULL },
4523 { gen_helper_sve_ld1bsu_r_mte, NULL, NULL, NULL },
4524 { gen_helper_sve_ld1bdu_r_mte, NULL, NULL, NULL },
4525
4526 { gen_helper_sve_ld1sds_le_r_mte, NULL, NULL, NULL },
4527 { gen_helper_sve_ld1hh_le_r_mte,
4528 gen_helper_sve_ld2hh_le_r_mte,
4529 gen_helper_sve_ld3hh_le_r_mte,
4530 gen_helper_sve_ld4hh_le_r_mte },
4531 { gen_helper_sve_ld1hsu_le_r_mte, NULL, NULL, NULL },
4532 { gen_helper_sve_ld1hdu_le_r_mte, NULL, NULL, NULL },
4533
4534 { gen_helper_sve_ld1hds_le_r_mte, NULL, NULL, NULL },
4535 { gen_helper_sve_ld1hss_le_r_mte, NULL, NULL, NULL },
4536 { gen_helper_sve_ld1ss_le_r_mte,
4537 gen_helper_sve_ld2ss_le_r_mte,
4538 gen_helper_sve_ld3ss_le_r_mte,
4539 gen_helper_sve_ld4ss_le_r_mte },
4540 { gen_helper_sve_ld1sdu_le_r_mte, NULL, NULL, NULL },
4541
4542 { gen_helper_sve_ld1bds_r_mte, NULL, NULL, NULL },
4543 { gen_helper_sve_ld1bss_r_mte, NULL, NULL, NULL },
4544 { gen_helper_sve_ld1bhs_r_mte, NULL, NULL, NULL },
4545 { gen_helper_sve_ld1dd_le_r_mte,
4546 gen_helper_sve_ld2dd_le_r_mte,
4547 gen_helper_sve_ld3dd_le_r_mte,
4548 gen_helper_sve_ld4dd_le_r_mte } },
4549
4550 /* mte active, big-endian */
4551 { { gen_helper_sve_ld1bb_r_mte,
4552 gen_helper_sve_ld2bb_r_mte,
4553 gen_helper_sve_ld3bb_r_mte,
4554 gen_helper_sve_ld4bb_r_mte },
4555 { gen_helper_sve_ld1bhu_r_mte, NULL, NULL, NULL },
4556 { gen_helper_sve_ld1bsu_r_mte, NULL, NULL, NULL },
4557 { gen_helper_sve_ld1bdu_r_mte, NULL, NULL, NULL },
4558
4559 { gen_helper_sve_ld1sds_be_r_mte, NULL, NULL, NULL },
4560 { gen_helper_sve_ld1hh_be_r_mte,
4561 gen_helper_sve_ld2hh_be_r_mte,
4562 gen_helper_sve_ld3hh_be_r_mte,
4563 gen_helper_sve_ld4hh_be_r_mte },
4564 { gen_helper_sve_ld1hsu_be_r_mte, NULL, NULL, NULL },
4565 { gen_helper_sve_ld1hdu_be_r_mte, NULL, NULL, NULL },
4566
4567 { gen_helper_sve_ld1hds_be_r_mte, NULL, NULL, NULL },
4568 { gen_helper_sve_ld1hss_be_r_mte, NULL, NULL, NULL },
4569 { gen_helper_sve_ld1ss_be_r_mte,
4570 gen_helper_sve_ld2ss_be_r_mte,
4571 gen_helper_sve_ld3ss_be_r_mte,
4572 gen_helper_sve_ld4ss_be_r_mte },
4573 { gen_helper_sve_ld1sdu_be_r_mte, NULL, NULL, NULL },
4574
4575 { gen_helper_sve_ld1bds_r_mte, NULL, NULL, NULL },
4576 { gen_helper_sve_ld1bss_r_mte, NULL, NULL, NULL },
4577 { gen_helper_sve_ld1bhs_r_mte, NULL, NULL, NULL },
4578 { gen_helper_sve_ld1dd_be_r_mte,
4579 gen_helper_sve_ld2dd_be_r_mte,
4580 gen_helper_sve_ld3dd_be_r_mte,
4581 gen_helper_sve_ld4dd_be_r_mte } } },
4582 };
4583
do_ld_zpa(DisasContext * s,int zt,int pg,TCGv_i64 addr,int dtype,int nreg)4584 static void do_ld_zpa(DisasContext *s, int zt, int pg,
4585 TCGv_i64 addr, int dtype, int nreg)
4586 {
4587 gen_helper_gvec_mem *fn
4588 = ldr_fns[s->mte_active[0]][s->be_data == MO_BE][dtype][nreg];
4589
4590 /*
4591 * While there are holes in the table, they are not
4592 * accessible via the instruction encoding.
4593 */
4594 assert(fn != NULL);
4595 do_mem_zpa(s, zt, pg, addr, dtype, nreg + 1, false, fn);
4596 }
4597
trans_LD_zprr(DisasContext * s,arg_rprr_load * a)4598 static bool trans_LD_zprr(DisasContext *s, arg_rprr_load *a)
4599 {
4600 if (a->rm == 31 || !dc_isar_feature(aa64_sve, s)) {
4601 return false;
4602 }
4603 if (sve_access_check(s)) {
4604 TCGv_i64 addr = tcg_temp_new_i64();
4605 tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), dtype_msz(a->dtype));
4606 tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn));
4607 do_ld_zpa(s, a->rd, a->pg, addr, a->dtype, a->nreg);
4608 }
4609 return true;
4610 }
4611
trans_LD_zpri(DisasContext * s,arg_rpri_load * a)4612 static bool trans_LD_zpri(DisasContext *s, arg_rpri_load *a)
4613 {
4614 if (!dc_isar_feature(aa64_sve, s)) {
4615 return false;
4616 }
4617 if (sve_access_check(s)) {
4618 int vsz = vec_full_reg_size(s);
4619 int elements = vsz >> dtype_esz[a->dtype];
4620 TCGv_i64 addr = tcg_temp_new_i64();
4621
4622 tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn),
4623 (a->imm * elements * (a->nreg + 1))
4624 << dtype_msz(a->dtype));
4625 do_ld_zpa(s, a->rd, a->pg, addr, a->dtype, a->nreg);
4626 }
4627 return true;
4628 }
4629
trans_LDFF1_zprr(DisasContext * s,arg_rprr_load * a)4630 static bool trans_LDFF1_zprr(DisasContext *s, arg_rprr_load *a)
4631 {
4632 static gen_helper_gvec_mem * const fns[2][2][16] = {
4633 { /* mte inactive, little-endian */
4634 { gen_helper_sve_ldff1bb_r,
4635 gen_helper_sve_ldff1bhu_r,
4636 gen_helper_sve_ldff1bsu_r,
4637 gen_helper_sve_ldff1bdu_r,
4638
4639 gen_helper_sve_ldff1sds_le_r,
4640 gen_helper_sve_ldff1hh_le_r,
4641 gen_helper_sve_ldff1hsu_le_r,
4642 gen_helper_sve_ldff1hdu_le_r,
4643
4644 gen_helper_sve_ldff1hds_le_r,
4645 gen_helper_sve_ldff1hss_le_r,
4646 gen_helper_sve_ldff1ss_le_r,
4647 gen_helper_sve_ldff1sdu_le_r,
4648
4649 gen_helper_sve_ldff1bds_r,
4650 gen_helper_sve_ldff1bss_r,
4651 gen_helper_sve_ldff1bhs_r,
4652 gen_helper_sve_ldff1dd_le_r },
4653
4654 /* mte inactive, big-endian */
4655 { gen_helper_sve_ldff1bb_r,
4656 gen_helper_sve_ldff1bhu_r,
4657 gen_helper_sve_ldff1bsu_r,
4658 gen_helper_sve_ldff1bdu_r,
4659
4660 gen_helper_sve_ldff1sds_be_r,
4661 gen_helper_sve_ldff1hh_be_r,
4662 gen_helper_sve_ldff1hsu_be_r,
4663 gen_helper_sve_ldff1hdu_be_r,
4664
4665 gen_helper_sve_ldff1hds_be_r,
4666 gen_helper_sve_ldff1hss_be_r,
4667 gen_helper_sve_ldff1ss_be_r,
4668 gen_helper_sve_ldff1sdu_be_r,
4669
4670 gen_helper_sve_ldff1bds_r,
4671 gen_helper_sve_ldff1bss_r,
4672 gen_helper_sve_ldff1bhs_r,
4673 gen_helper_sve_ldff1dd_be_r } },
4674
4675 { /* mte active, little-endian */
4676 { gen_helper_sve_ldff1bb_r_mte,
4677 gen_helper_sve_ldff1bhu_r_mte,
4678 gen_helper_sve_ldff1bsu_r_mte,
4679 gen_helper_sve_ldff1bdu_r_mte,
4680
4681 gen_helper_sve_ldff1sds_le_r_mte,
4682 gen_helper_sve_ldff1hh_le_r_mte,
4683 gen_helper_sve_ldff1hsu_le_r_mte,
4684 gen_helper_sve_ldff1hdu_le_r_mte,
4685
4686 gen_helper_sve_ldff1hds_le_r_mte,
4687 gen_helper_sve_ldff1hss_le_r_mte,
4688 gen_helper_sve_ldff1ss_le_r_mte,
4689 gen_helper_sve_ldff1sdu_le_r_mte,
4690
4691 gen_helper_sve_ldff1bds_r_mte,
4692 gen_helper_sve_ldff1bss_r_mte,
4693 gen_helper_sve_ldff1bhs_r_mte,
4694 gen_helper_sve_ldff1dd_le_r_mte },
4695
4696 /* mte active, big-endian */
4697 { gen_helper_sve_ldff1bb_r_mte,
4698 gen_helper_sve_ldff1bhu_r_mte,
4699 gen_helper_sve_ldff1bsu_r_mte,
4700 gen_helper_sve_ldff1bdu_r_mte,
4701
4702 gen_helper_sve_ldff1sds_be_r_mte,
4703 gen_helper_sve_ldff1hh_be_r_mte,
4704 gen_helper_sve_ldff1hsu_be_r_mte,
4705 gen_helper_sve_ldff1hdu_be_r_mte,
4706
4707 gen_helper_sve_ldff1hds_be_r_mte,
4708 gen_helper_sve_ldff1hss_be_r_mte,
4709 gen_helper_sve_ldff1ss_be_r_mte,
4710 gen_helper_sve_ldff1sdu_be_r_mte,
4711
4712 gen_helper_sve_ldff1bds_r_mte,
4713 gen_helper_sve_ldff1bss_r_mte,
4714 gen_helper_sve_ldff1bhs_r_mte,
4715 gen_helper_sve_ldff1dd_be_r_mte } },
4716 };
4717
4718 if (!dc_isar_feature(aa64_sve, s)) {
4719 return false;
4720 }
4721 s->is_nonstreaming = true;
4722 if (sve_access_check(s)) {
4723 TCGv_i64 addr = tcg_temp_new_i64();
4724 tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), dtype_msz(a->dtype));
4725 tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn));
4726 do_mem_zpa(s, a->rd, a->pg, addr, a->dtype, 1, false,
4727 fns[s->mte_active[0]][s->be_data == MO_BE][a->dtype]);
4728 }
4729 return true;
4730 }
4731
trans_LDNF1_zpri(DisasContext * s,arg_rpri_load * a)4732 static bool trans_LDNF1_zpri(DisasContext *s, arg_rpri_load *a)
4733 {
4734 static gen_helper_gvec_mem * const fns[2][2][16] = {
4735 { /* mte inactive, little-endian */
4736 { gen_helper_sve_ldnf1bb_r,
4737 gen_helper_sve_ldnf1bhu_r,
4738 gen_helper_sve_ldnf1bsu_r,
4739 gen_helper_sve_ldnf1bdu_r,
4740
4741 gen_helper_sve_ldnf1sds_le_r,
4742 gen_helper_sve_ldnf1hh_le_r,
4743 gen_helper_sve_ldnf1hsu_le_r,
4744 gen_helper_sve_ldnf1hdu_le_r,
4745
4746 gen_helper_sve_ldnf1hds_le_r,
4747 gen_helper_sve_ldnf1hss_le_r,
4748 gen_helper_sve_ldnf1ss_le_r,
4749 gen_helper_sve_ldnf1sdu_le_r,
4750
4751 gen_helper_sve_ldnf1bds_r,
4752 gen_helper_sve_ldnf1bss_r,
4753 gen_helper_sve_ldnf1bhs_r,
4754 gen_helper_sve_ldnf1dd_le_r },
4755
4756 /* mte inactive, big-endian */
4757 { gen_helper_sve_ldnf1bb_r,
4758 gen_helper_sve_ldnf1bhu_r,
4759 gen_helper_sve_ldnf1bsu_r,
4760 gen_helper_sve_ldnf1bdu_r,
4761
4762 gen_helper_sve_ldnf1sds_be_r,
4763 gen_helper_sve_ldnf1hh_be_r,
4764 gen_helper_sve_ldnf1hsu_be_r,
4765 gen_helper_sve_ldnf1hdu_be_r,
4766
4767 gen_helper_sve_ldnf1hds_be_r,
4768 gen_helper_sve_ldnf1hss_be_r,
4769 gen_helper_sve_ldnf1ss_be_r,
4770 gen_helper_sve_ldnf1sdu_be_r,
4771
4772 gen_helper_sve_ldnf1bds_r,
4773 gen_helper_sve_ldnf1bss_r,
4774 gen_helper_sve_ldnf1bhs_r,
4775 gen_helper_sve_ldnf1dd_be_r } },
4776
4777 { /* mte inactive, little-endian */
4778 { gen_helper_sve_ldnf1bb_r_mte,
4779 gen_helper_sve_ldnf1bhu_r_mte,
4780 gen_helper_sve_ldnf1bsu_r_mte,
4781 gen_helper_sve_ldnf1bdu_r_mte,
4782
4783 gen_helper_sve_ldnf1sds_le_r_mte,
4784 gen_helper_sve_ldnf1hh_le_r_mte,
4785 gen_helper_sve_ldnf1hsu_le_r_mte,
4786 gen_helper_sve_ldnf1hdu_le_r_mte,
4787
4788 gen_helper_sve_ldnf1hds_le_r_mte,
4789 gen_helper_sve_ldnf1hss_le_r_mte,
4790 gen_helper_sve_ldnf1ss_le_r_mte,
4791 gen_helper_sve_ldnf1sdu_le_r_mte,
4792
4793 gen_helper_sve_ldnf1bds_r_mte,
4794 gen_helper_sve_ldnf1bss_r_mte,
4795 gen_helper_sve_ldnf1bhs_r_mte,
4796 gen_helper_sve_ldnf1dd_le_r_mte },
4797
4798 /* mte inactive, big-endian */
4799 { gen_helper_sve_ldnf1bb_r_mte,
4800 gen_helper_sve_ldnf1bhu_r_mte,
4801 gen_helper_sve_ldnf1bsu_r_mte,
4802 gen_helper_sve_ldnf1bdu_r_mte,
4803
4804 gen_helper_sve_ldnf1sds_be_r_mte,
4805 gen_helper_sve_ldnf1hh_be_r_mte,
4806 gen_helper_sve_ldnf1hsu_be_r_mte,
4807 gen_helper_sve_ldnf1hdu_be_r_mte,
4808
4809 gen_helper_sve_ldnf1hds_be_r_mte,
4810 gen_helper_sve_ldnf1hss_be_r_mte,
4811 gen_helper_sve_ldnf1ss_be_r_mte,
4812 gen_helper_sve_ldnf1sdu_be_r_mte,
4813
4814 gen_helper_sve_ldnf1bds_r_mte,
4815 gen_helper_sve_ldnf1bss_r_mte,
4816 gen_helper_sve_ldnf1bhs_r_mte,
4817 gen_helper_sve_ldnf1dd_be_r_mte } },
4818 };
4819
4820 if (!dc_isar_feature(aa64_sve, s)) {
4821 return false;
4822 }
4823 s->is_nonstreaming = true;
4824 if (sve_access_check(s)) {
4825 int vsz = vec_full_reg_size(s);
4826 int elements = vsz >> dtype_esz[a->dtype];
4827 int off = (a->imm * elements) << dtype_msz(a->dtype);
4828 TCGv_i64 addr = tcg_temp_new_i64();
4829
4830 tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn), off);
4831 do_mem_zpa(s, a->rd, a->pg, addr, a->dtype, 1, false,
4832 fns[s->mte_active[0]][s->be_data == MO_BE][a->dtype]);
4833 }
4834 return true;
4835 }
4836
do_ldrq(DisasContext * s,int zt,int pg,TCGv_i64 addr,int dtype)4837 static void do_ldrq(DisasContext *s, int zt, int pg, TCGv_i64 addr, int dtype)
4838 {
4839 unsigned vsz = vec_full_reg_size(s);
4840 TCGv_ptr t_pg;
4841 int poff;
4842 uint32_t desc;
4843
4844 /* Load the first quadword using the normal predicated load helpers. */
4845 if (!s->mte_active[0]) {
4846 addr = clean_data_tbi(s, addr);
4847 }
4848
4849 poff = pred_full_reg_offset(s, pg);
4850 if (vsz > 16) {
4851 /*
4852 * Zero-extend the first 16 bits of the predicate into a temporary.
4853 * This avoids triggering an assert making sure we don't have bits
4854 * set within a predicate beyond VQ, but we have lowered VQ to 1
4855 * for this load operation.
4856 */
4857 TCGv_i64 tmp = tcg_temp_new_i64();
4858 #if HOST_BIG_ENDIAN
4859 poff += 6;
4860 #endif
4861 tcg_gen_ld16u_i64(tmp, tcg_env, poff);
4862
4863 poff = offsetof(CPUARMState, vfp.preg_tmp);
4864 tcg_gen_st_i64(tmp, tcg_env, poff);
4865 }
4866
4867 t_pg = tcg_temp_new_ptr();
4868 tcg_gen_addi_ptr(t_pg, tcg_env, poff);
4869
4870 gen_helper_gvec_mem *fn
4871 = ldr_fns[s->mte_active[0]][s->be_data == MO_BE][dtype][0];
4872 desc = make_svemte_desc(s, 16, 1, dtype_msz(dtype), false, zt);
4873 fn(tcg_env, t_pg, addr, tcg_constant_i32(desc));
4874
4875 /* Replicate that first quadword. */
4876 if (vsz > 16) {
4877 int doff = vec_full_reg_offset(s, zt);
4878 tcg_gen_gvec_dup_mem(4, doff + 16, doff, vsz - 16, vsz - 16);
4879 }
4880 }
4881
trans_LD1RQ_zprr(DisasContext * s,arg_rprr_load * a)4882 static bool trans_LD1RQ_zprr(DisasContext *s, arg_rprr_load *a)
4883 {
4884 if (a->rm == 31 || !dc_isar_feature(aa64_sve, s)) {
4885 return false;
4886 }
4887 if (sve_access_check(s)) {
4888 int msz = dtype_msz(a->dtype);
4889 TCGv_i64 addr = tcg_temp_new_i64();
4890 tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), msz);
4891 tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn));
4892 do_ldrq(s, a->rd, a->pg, addr, a->dtype);
4893 }
4894 return true;
4895 }
4896
trans_LD1RQ_zpri(DisasContext * s,arg_rpri_load * a)4897 static bool trans_LD1RQ_zpri(DisasContext *s, arg_rpri_load *a)
4898 {
4899 if (!dc_isar_feature(aa64_sve, s)) {
4900 return false;
4901 }
4902 if (sve_access_check(s)) {
4903 TCGv_i64 addr = tcg_temp_new_i64();
4904 tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn), a->imm * 16);
4905 do_ldrq(s, a->rd, a->pg, addr, a->dtype);
4906 }
4907 return true;
4908 }
4909
do_ldro(DisasContext * s,int zt,int pg,TCGv_i64 addr,int dtype)4910 static void do_ldro(DisasContext *s, int zt, int pg, TCGv_i64 addr, int dtype)
4911 {
4912 unsigned vsz = vec_full_reg_size(s);
4913 unsigned vsz_r32;
4914 TCGv_ptr t_pg;
4915 int poff, doff;
4916 uint32_t desc;
4917
4918 if (vsz < 32) {
4919 /*
4920 * Note that this UNDEFINED check comes after CheckSVEEnabled()
4921 * in the ARM pseudocode, which is the sve_access_check() done
4922 * in our caller. We should not now return false from the caller.
4923 */
4924 unallocated_encoding(s);
4925 return;
4926 }
4927
4928 /* Load the first octaword using the normal predicated load helpers. */
4929 if (!s->mte_active[0]) {
4930 addr = clean_data_tbi(s, addr);
4931 }
4932
4933 poff = pred_full_reg_offset(s, pg);
4934 if (vsz > 32) {
4935 /*
4936 * Zero-extend the first 32 bits of the predicate into a temporary.
4937 * This avoids triggering an assert making sure we don't have bits
4938 * set within a predicate beyond VQ, but we have lowered VQ to 2
4939 * for this load operation.
4940 */
4941 TCGv_i64 tmp = tcg_temp_new_i64();
4942 #if HOST_BIG_ENDIAN
4943 poff += 4;
4944 #endif
4945 tcg_gen_ld32u_i64(tmp, tcg_env, poff);
4946
4947 poff = offsetof(CPUARMState, vfp.preg_tmp);
4948 tcg_gen_st_i64(tmp, tcg_env, poff);
4949 }
4950
4951 t_pg = tcg_temp_new_ptr();
4952 tcg_gen_addi_ptr(t_pg, tcg_env, poff);
4953
4954 gen_helper_gvec_mem *fn
4955 = ldr_fns[s->mte_active[0]][s->be_data == MO_BE][dtype][0];
4956 desc = make_svemte_desc(s, 32, 1, dtype_msz(dtype), false, zt);
4957 fn(tcg_env, t_pg, addr, tcg_constant_i32(desc));
4958
4959 /*
4960 * Replicate that first octaword.
4961 * The replication happens in units of 32; if the full vector size
4962 * is not a multiple of 32, the final bits are zeroed.
4963 */
4964 doff = vec_full_reg_offset(s, zt);
4965 vsz_r32 = QEMU_ALIGN_DOWN(vsz, 32);
4966 if (vsz >= 64) {
4967 tcg_gen_gvec_dup_mem(5, doff + 32, doff, vsz_r32 - 32, vsz_r32 - 32);
4968 }
4969 vsz -= vsz_r32;
4970 if (vsz) {
4971 tcg_gen_gvec_dup_imm(MO_64, doff + vsz_r32, vsz, vsz, 0);
4972 }
4973 }
4974
trans_LD1RO_zprr(DisasContext * s,arg_rprr_load * a)4975 static bool trans_LD1RO_zprr(DisasContext *s, arg_rprr_load *a)
4976 {
4977 if (!dc_isar_feature(aa64_sve_f64mm, s)) {
4978 return false;
4979 }
4980 if (a->rm == 31) {
4981 return false;
4982 }
4983 s->is_nonstreaming = true;
4984 if (sve_access_check(s)) {
4985 TCGv_i64 addr = tcg_temp_new_i64();
4986 tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), dtype_msz(a->dtype));
4987 tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn));
4988 do_ldro(s, a->rd, a->pg, addr, a->dtype);
4989 }
4990 return true;
4991 }
4992
trans_LD1RO_zpri(DisasContext * s,arg_rpri_load * a)4993 static bool trans_LD1RO_zpri(DisasContext *s, arg_rpri_load *a)
4994 {
4995 if (!dc_isar_feature(aa64_sve_f64mm, s)) {
4996 return false;
4997 }
4998 s->is_nonstreaming = true;
4999 if (sve_access_check(s)) {
5000 TCGv_i64 addr = tcg_temp_new_i64();
5001 tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn), a->imm * 32);
5002 do_ldro(s, a->rd, a->pg, addr, a->dtype);
5003 }
5004 return true;
5005 }
5006
5007 /* Load and broadcast element. */
trans_LD1R_zpri(DisasContext * s,arg_rpri_load * a)5008 static bool trans_LD1R_zpri(DisasContext *s, arg_rpri_load *a)
5009 {
5010 unsigned vsz = vec_full_reg_size(s);
5011 unsigned psz = pred_full_reg_size(s);
5012 unsigned esz = dtype_esz[a->dtype];
5013 unsigned msz = dtype_msz(a->dtype);
5014 TCGLabel *over;
5015 TCGv_i64 temp, clean_addr;
5016 MemOp memop;
5017
5018 if (!dc_isar_feature(aa64_sve, s)) {
5019 return false;
5020 }
5021 if (!sve_access_check(s)) {
5022 return true;
5023 }
5024
5025 over = gen_new_label();
5026
5027 /* If the guarding predicate has no bits set, no load occurs. */
5028 if (psz <= 8) {
5029 /* Reduce the pred_esz_masks value simply to reduce the
5030 * size of the code generated here.
5031 */
5032 uint64_t psz_mask = MAKE_64BIT_MASK(0, psz * 8);
5033 temp = tcg_temp_new_i64();
5034 tcg_gen_ld_i64(temp, tcg_env, pred_full_reg_offset(s, a->pg));
5035 tcg_gen_andi_i64(temp, temp, pred_esz_masks[esz] & psz_mask);
5036 tcg_gen_brcondi_i64(TCG_COND_EQ, temp, 0, over);
5037 } else {
5038 TCGv_i32 t32 = tcg_temp_new_i32();
5039 find_last_active(s, t32, esz, a->pg);
5040 tcg_gen_brcondi_i32(TCG_COND_LT, t32, 0, over);
5041 }
5042
5043 /* Load the data. */
5044 temp = tcg_temp_new_i64();
5045 tcg_gen_addi_i64(temp, cpu_reg_sp(s, a->rn), a->imm << msz);
5046
5047 memop = finalize_memop(s, dtype_mop[a->dtype]);
5048 clean_addr = gen_mte_check1(s, temp, false, true, memop);
5049 tcg_gen_qemu_ld_i64(temp, clean_addr, get_mem_index(s), memop);
5050
5051 /* Broadcast to *all* elements. */
5052 tcg_gen_gvec_dup_i64(esz, vec_full_reg_offset(s, a->rd),
5053 vsz, vsz, temp);
5054
5055 /* Zero the inactive elements. */
5056 gen_set_label(over);
5057 return do_movz_zpz(s, a->rd, a->rd, a->pg, esz, false);
5058 }
5059
do_st_zpa(DisasContext * s,int zt,int pg,TCGv_i64 addr,int msz,int esz,int nreg)5060 static void do_st_zpa(DisasContext *s, int zt, int pg, TCGv_i64 addr,
5061 int msz, int esz, int nreg)
5062 {
5063 static gen_helper_gvec_mem * const fn_single[2][2][4][4] = {
5064 { { { gen_helper_sve_st1bb_r,
5065 gen_helper_sve_st1bh_r,
5066 gen_helper_sve_st1bs_r,
5067 gen_helper_sve_st1bd_r },
5068 { NULL,
5069 gen_helper_sve_st1hh_le_r,
5070 gen_helper_sve_st1hs_le_r,
5071 gen_helper_sve_st1hd_le_r },
5072 { NULL, NULL,
5073 gen_helper_sve_st1ss_le_r,
5074 gen_helper_sve_st1sd_le_r },
5075 { NULL, NULL, NULL,
5076 gen_helper_sve_st1dd_le_r } },
5077 { { gen_helper_sve_st1bb_r,
5078 gen_helper_sve_st1bh_r,
5079 gen_helper_sve_st1bs_r,
5080 gen_helper_sve_st1bd_r },
5081 { NULL,
5082 gen_helper_sve_st1hh_be_r,
5083 gen_helper_sve_st1hs_be_r,
5084 gen_helper_sve_st1hd_be_r },
5085 { NULL, NULL,
5086 gen_helper_sve_st1ss_be_r,
5087 gen_helper_sve_st1sd_be_r },
5088 { NULL, NULL, NULL,
5089 gen_helper_sve_st1dd_be_r } } },
5090
5091 { { { gen_helper_sve_st1bb_r_mte,
5092 gen_helper_sve_st1bh_r_mte,
5093 gen_helper_sve_st1bs_r_mte,
5094 gen_helper_sve_st1bd_r_mte },
5095 { NULL,
5096 gen_helper_sve_st1hh_le_r_mte,
5097 gen_helper_sve_st1hs_le_r_mte,
5098 gen_helper_sve_st1hd_le_r_mte },
5099 { NULL, NULL,
5100 gen_helper_sve_st1ss_le_r_mte,
5101 gen_helper_sve_st1sd_le_r_mte },
5102 { NULL, NULL, NULL,
5103 gen_helper_sve_st1dd_le_r_mte } },
5104 { { gen_helper_sve_st1bb_r_mte,
5105 gen_helper_sve_st1bh_r_mte,
5106 gen_helper_sve_st1bs_r_mte,
5107 gen_helper_sve_st1bd_r_mte },
5108 { NULL,
5109 gen_helper_sve_st1hh_be_r_mte,
5110 gen_helper_sve_st1hs_be_r_mte,
5111 gen_helper_sve_st1hd_be_r_mte },
5112 { NULL, NULL,
5113 gen_helper_sve_st1ss_be_r_mte,
5114 gen_helper_sve_st1sd_be_r_mte },
5115 { NULL, NULL, NULL,
5116 gen_helper_sve_st1dd_be_r_mte } } },
5117 };
5118 static gen_helper_gvec_mem * const fn_multiple[2][2][3][4] = {
5119 { { { gen_helper_sve_st2bb_r,
5120 gen_helper_sve_st2hh_le_r,
5121 gen_helper_sve_st2ss_le_r,
5122 gen_helper_sve_st2dd_le_r },
5123 { gen_helper_sve_st3bb_r,
5124 gen_helper_sve_st3hh_le_r,
5125 gen_helper_sve_st3ss_le_r,
5126 gen_helper_sve_st3dd_le_r },
5127 { gen_helper_sve_st4bb_r,
5128 gen_helper_sve_st4hh_le_r,
5129 gen_helper_sve_st4ss_le_r,
5130 gen_helper_sve_st4dd_le_r } },
5131 { { gen_helper_sve_st2bb_r,
5132 gen_helper_sve_st2hh_be_r,
5133 gen_helper_sve_st2ss_be_r,
5134 gen_helper_sve_st2dd_be_r },
5135 { gen_helper_sve_st3bb_r,
5136 gen_helper_sve_st3hh_be_r,
5137 gen_helper_sve_st3ss_be_r,
5138 gen_helper_sve_st3dd_be_r },
5139 { gen_helper_sve_st4bb_r,
5140 gen_helper_sve_st4hh_be_r,
5141 gen_helper_sve_st4ss_be_r,
5142 gen_helper_sve_st4dd_be_r } } },
5143 { { { gen_helper_sve_st2bb_r_mte,
5144 gen_helper_sve_st2hh_le_r_mte,
5145 gen_helper_sve_st2ss_le_r_mte,
5146 gen_helper_sve_st2dd_le_r_mte },
5147 { gen_helper_sve_st3bb_r_mte,
5148 gen_helper_sve_st3hh_le_r_mte,
5149 gen_helper_sve_st3ss_le_r_mte,
5150 gen_helper_sve_st3dd_le_r_mte },
5151 { gen_helper_sve_st4bb_r_mte,
5152 gen_helper_sve_st4hh_le_r_mte,
5153 gen_helper_sve_st4ss_le_r_mte,
5154 gen_helper_sve_st4dd_le_r_mte } },
5155 { { gen_helper_sve_st2bb_r_mte,
5156 gen_helper_sve_st2hh_be_r_mte,
5157 gen_helper_sve_st2ss_be_r_mte,
5158 gen_helper_sve_st2dd_be_r_mte },
5159 { gen_helper_sve_st3bb_r_mte,
5160 gen_helper_sve_st3hh_be_r_mte,
5161 gen_helper_sve_st3ss_be_r_mte,
5162 gen_helper_sve_st3dd_be_r_mte },
5163 { gen_helper_sve_st4bb_r_mte,
5164 gen_helper_sve_st4hh_be_r_mte,
5165 gen_helper_sve_st4ss_be_r_mte,
5166 gen_helper_sve_st4dd_be_r_mte } } },
5167 };
5168 gen_helper_gvec_mem *fn;
5169 int be = s->be_data == MO_BE;
5170
5171 if (nreg == 0) {
5172 /* ST1 */
5173 fn = fn_single[s->mte_active[0]][be][msz][esz];
5174 } else {
5175 /* ST2, ST3, ST4 -- msz == esz, enforced by encoding */
5176 assert(msz == esz);
5177 fn = fn_multiple[s->mte_active[0]][be][nreg - 1][msz];
5178 }
5179 assert(fn != NULL);
5180 do_mem_zpa(s, zt, pg, addr, msz_dtype(s, msz), nreg + 1, true, fn);
5181 }
5182
trans_ST_zprr(DisasContext * s,arg_rprr_store * a)5183 static bool trans_ST_zprr(DisasContext *s, arg_rprr_store *a)
5184 {
5185 if (!dc_isar_feature(aa64_sve, s)) {
5186 return false;
5187 }
5188 if (a->rm == 31 || a->msz > a->esz) {
5189 return false;
5190 }
5191 if (sve_access_check(s)) {
5192 TCGv_i64 addr = tcg_temp_new_i64();
5193 tcg_gen_shli_i64(addr, cpu_reg(s, a->rm), a->msz);
5194 tcg_gen_add_i64(addr, addr, cpu_reg_sp(s, a->rn));
5195 do_st_zpa(s, a->rd, a->pg, addr, a->msz, a->esz, a->nreg);
5196 }
5197 return true;
5198 }
5199
trans_ST_zpri(DisasContext * s,arg_rpri_store * a)5200 static bool trans_ST_zpri(DisasContext *s, arg_rpri_store *a)
5201 {
5202 if (!dc_isar_feature(aa64_sve, s)) {
5203 return false;
5204 }
5205 if (a->msz > a->esz) {
5206 return false;
5207 }
5208 if (sve_access_check(s)) {
5209 int vsz = vec_full_reg_size(s);
5210 int elements = vsz >> a->esz;
5211 TCGv_i64 addr = tcg_temp_new_i64();
5212
5213 tcg_gen_addi_i64(addr, cpu_reg_sp(s, a->rn),
5214 (a->imm * elements * (a->nreg + 1)) << a->msz);
5215 do_st_zpa(s, a->rd, a->pg, addr, a->msz, a->esz, a->nreg);
5216 }
5217 return true;
5218 }
5219
5220 /*
5221 *** SVE gather loads / scatter stores
5222 */
5223
do_mem_zpz(DisasContext * s,int zt,int pg,int zm,int scale,TCGv_i64 scalar,int msz,bool is_write,gen_helper_gvec_mem_scatter * fn)5224 static void do_mem_zpz(DisasContext *s, int zt, int pg, int zm,
5225 int scale, TCGv_i64 scalar, int msz, bool is_write,
5226 gen_helper_gvec_mem_scatter *fn)
5227 {
5228 TCGv_ptr t_zm = tcg_temp_new_ptr();
5229 TCGv_ptr t_pg = tcg_temp_new_ptr();
5230 TCGv_ptr t_zt = tcg_temp_new_ptr();
5231 uint32_t desc;
5232
5233 tcg_gen_addi_ptr(t_pg, tcg_env, pred_full_reg_offset(s, pg));
5234 tcg_gen_addi_ptr(t_zm, tcg_env, vec_full_reg_offset(s, zm));
5235 tcg_gen_addi_ptr(t_zt, tcg_env, vec_full_reg_offset(s, zt));
5236
5237 desc = make_svemte_desc(s, vec_full_reg_size(s), 1, msz, is_write, scale);
5238 fn(tcg_env, t_zt, t_pg, t_zm, scalar, tcg_constant_i32(desc));
5239 }
5240
5241 /* Indexed by [mte][be][ff][xs][u][msz]. */
5242 static gen_helper_gvec_mem_scatter * const
5243 gather_load_fn32[2][2][2][2][2][3] = {
5244 { /* MTE Inactive */
5245 { /* Little-endian */
5246 { { { gen_helper_sve_ldbss_zsu,
5247 gen_helper_sve_ldhss_le_zsu,
5248 NULL, },
5249 { gen_helper_sve_ldbsu_zsu,
5250 gen_helper_sve_ldhsu_le_zsu,
5251 gen_helper_sve_ldss_le_zsu, } },
5252 { { gen_helper_sve_ldbss_zss,
5253 gen_helper_sve_ldhss_le_zss,
5254 NULL, },
5255 { gen_helper_sve_ldbsu_zss,
5256 gen_helper_sve_ldhsu_le_zss,
5257 gen_helper_sve_ldss_le_zss, } } },
5258
5259 /* First-fault */
5260 { { { gen_helper_sve_ldffbss_zsu,
5261 gen_helper_sve_ldffhss_le_zsu,
5262 NULL, },
5263 { gen_helper_sve_ldffbsu_zsu,
5264 gen_helper_sve_ldffhsu_le_zsu,
5265 gen_helper_sve_ldffss_le_zsu, } },
5266 { { gen_helper_sve_ldffbss_zss,
5267 gen_helper_sve_ldffhss_le_zss,
5268 NULL, },
5269 { gen_helper_sve_ldffbsu_zss,
5270 gen_helper_sve_ldffhsu_le_zss,
5271 gen_helper_sve_ldffss_le_zss, } } } },
5272
5273 { /* Big-endian */
5274 { { { gen_helper_sve_ldbss_zsu,
5275 gen_helper_sve_ldhss_be_zsu,
5276 NULL, },
5277 { gen_helper_sve_ldbsu_zsu,
5278 gen_helper_sve_ldhsu_be_zsu,
5279 gen_helper_sve_ldss_be_zsu, } },
5280 { { gen_helper_sve_ldbss_zss,
5281 gen_helper_sve_ldhss_be_zss,
5282 NULL, },
5283 { gen_helper_sve_ldbsu_zss,
5284 gen_helper_sve_ldhsu_be_zss,
5285 gen_helper_sve_ldss_be_zss, } } },
5286
5287 /* First-fault */
5288 { { { gen_helper_sve_ldffbss_zsu,
5289 gen_helper_sve_ldffhss_be_zsu,
5290 NULL, },
5291 { gen_helper_sve_ldffbsu_zsu,
5292 gen_helper_sve_ldffhsu_be_zsu,
5293 gen_helper_sve_ldffss_be_zsu, } },
5294 { { gen_helper_sve_ldffbss_zss,
5295 gen_helper_sve_ldffhss_be_zss,
5296 NULL, },
5297 { gen_helper_sve_ldffbsu_zss,
5298 gen_helper_sve_ldffhsu_be_zss,
5299 gen_helper_sve_ldffss_be_zss, } } } } },
5300 { /* MTE Active */
5301 { /* Little-endian */
5302 { { { gen_helper_sve_ldbss_zsu_mte,
5303 gen_helper_sve_ldhss_le_zsu_mte,
5304 NULL, },
5305 { gen_helper_sve_ldbsu_zsu_mte,
5306 gen_helper_sve_ldhsu_le_zsu_mte,
5307 gen_helper_sve_ldss_le_zsu_mte, } },
5308 { { gen_helper_sve_ldbss_zss_mte,
5309 gen_helper_sve_ldhss_le_zss_mte,
5310 NULL, },
5311 { gen_helper_sve_ldbsu_zss_mte,
5312 gen_helper_sve_ldhsu_le_zss_mte,
5313 gen_helper_sve_ldss_le_zss_mte, } } },
5314
5315 /* First-fault */
5316 { { { gen_helper_sve_ldffbss_zsu_mte,
5317 gen_helper_sve_ldffhss_le_zsu_mte,
5318 NULL, },
5319 { gen_helper_sve_ldffbsu_zsu_mte,
5320 gen_helper_sve_ldffhsu_le_zsu_mte,
5321 gen_helper_sve_ldffss_le_zsu_mte, } },
5322 { { gen_helper_sve_ldffbss_zss_mte,
5323 gen_helper_sve_ldffhss_le_zss_mte,
5324 NULL, },
5325 { gen_helper_sve_ldffbsu_zss_mte,
5326 gen_helper_sve_ldffhsu_le_zss_mte,
5327 gen_helper_sve_ldffss_le_zss_mte, } } } },
5328
5329 { /* Big-endian */
5330 { { { gen_helper_sve_ldbss_zsu_mte,
5331 gen_helper_sve_ldhss_be_zsu_mte,
5332 NULL, },
5333 { gen_helper_sve_ldbsu_zsu_mte,
5334 gen_helper_sve_ldhsu_be_zsu_mte,
5335 gen_helper_sve_ldss_be_zsu_mte, } },
5336 { { gen_helper_sve_ldbss_zss_mte,
5337 gen_helper_sve_ldhss_be_zss_mte,
5338 NULL, },
5339 { gen_helper_sve_ldbsu_zss_mte,
5340 gen_helper_sve_ldhsu_be_zss_mte,
5341 gen_helper_sve_ldss_be_zss_mte, } } },
5342
5343 /* First-fault */
5344 { { { gen_helper_sve_ldffbss_zsu_mte,
5345 gen_helper_sve_ldffhss_be_zsu_mte,
5346 NULL, },
5347 { gen_helper_sve_ldffbsu_zsu_mte,
5348 gen_helper_sve_ldffhsu_be_zsu_mte,
5349 gen_helper_sve_ldffss_be_zsu_mte, } },
5350 { { gen_helper_sve_ldffbss_zss_mte,
5351 gen_helper_sve_ldffhss_be_zss_mte,
5352 NULL, },
5353 { gen_helper_sve_ldffbsu_zss_mte,
5354 gen_helper_sve_ldffhsu_be_zss_mte,
5355 gen_helper_sve_ldffss_be_zss_mte, } } } } },
5356 };
5357
5358 /* Note that we overload xs=2 to indicate 64-bit offset. */
5359 static gen_helper_gvec_mem_scatter * const
5360 gather_load_fn64[2][2][2][3][2][4] = {
5361 { /* MTE Inactive */
5362 { /* Little-endian */
5363 { { { gen_helper_sve_ldbds_zsu,
5364 gen_helper_sve_ldhds_le_zsu,
5365 gen_helper_sve_ldsds_le_zsu,
5366 NULL, },
5367 { gen_helper_sve_ldbdu_zsu,
5368 gen_helper_sve_ldhdu_le_zsu,
5369 gen_helper_sve_ldsdu_le_zsu,
5370 gen_helper_sve_lddd_le_zsu, } },
5371 { { gen_helper_sve_ldbds_zss,
5372 gen_helper_sve_ldhds_le_zss,
5373 gen_helper_sve_ldsds_le_zss,
5374 NULL, },
5375 { gen_helper_sve_ldbdu_zss,
5376 gen_helper_sve_ldhdu_le_zss,
5377 gen_helper_sve_ldsdu_le_zss,
5378 gen_helper_sve_lddd_le_zss, } },
5379 { { gen_helper_sve_ldbds_zd,
5380 gen_helper_sve_ldhds_le_zd,
5381 gen_helper_sve_ldsds_le_zd,
5382 NULL, },
5383 { gen_helper_sve_ldbdu_zd,
5384 gen_helper_sve_ldhdu_le_zd,
5385 gen_helper_sve_ldsdu_le_zd,
5386 gen_helper_sve_lddd_le_zd, } } },
5387
5388 /* First-fault */
5389 { { { gen_helper_sve_ldffbds_zsu,
5390 gen_helper_sve_ldffhds_le_zsu,
5391 gen_helper_sve_ldffsds_le_zsu,
5392 NULL, },
5393 { gen_helper_sve_ldffbdu_zsu,
5394 gen_helper_sve_ldffhdu_le_zsu,
5395 gen_helper_sve_ldffsdu_le_zsu,
5396 gen_helper_sve_ldffdd_le_zsu, } },
5397 { { gen_helper_sve_ldffbds_zss,
5398 gen_helper_sve_ldffhds_le_zss,
5399 gen_helper_sve_ldffsds_le_zss,
5400 NULL, },
5401 { gen_helper_sve_ldffbdu_zss,
5402 gen_helper_sve_ldffhdu_le_zss,
5403 gen_helper_sve_ldffsdu_le_zss,
5404 gen_helper_sve_ldffdd_le_zss, } },
5405 { { gen_helper_sve_ldffbds_zd,
5406 gen_helper_sve_ldffhds_le_zd,
5407 gen_helper_sve_ldffsds_le_zd,
5408 NULL, },
5409 { gen_helper_sve_ldffbdu_zd,
5410 gen_helper_sve_ldffhdu_le_zd,
5411 gen_helper_sve_ldffsdu_le_zd,
5412 gen_helper_sve_ldffdd_le_zd, } } } },
5413 { /* Big-endian */
5414 { { { gen_helper_sve_ldbds_zsu,
5415 gen_helper_sve_ldhds_be_zsu,
5416 gen_helper_sve_ldsds_be_zsu,
5417 NULL, },
5418 { gen_helper_sve_ldbdu_zsu,
5419 gen_helper_sve_ldhdu_be_zsu,
5420 gen_helper_sve_ldsdu_be_zsu,
5421 gen_helper_sve_lddd_be_zsu, } },
5422 { { gen_helper_sve_ldbds_zss,
5423 gen_helper_sve_ldhds_be_zss,
5424 gen_helper_sve_ldsds_be_zss,
5425 NULL, },
5426 { gen_helper_sve_ldbdu_zss,
5427 gen_helper_sve_ldhdu_be_zss,
5428 gen_helper_sve_ldsdu_be_zss,
5429 gen_helper_sve_lddd_be_zss, } },
5430 { { gen_helper_sve_ldbds_zd,
5431 gen_helper_sve_ldhds_be_zd,
5432 gen_helper_sve_ldsds_be_zd,
5433 NULL, },
5434 { gen_helper_sve_ldbdu_zd,
5435 gen_helper_sve_ldhdu_be_zd,
5436 gen_helper_sve_ldsdu_be_zd,
5437 gen_helper_sve_lddd_be_zd, } } },
5438
5439 /* First-fault */
5440 { { { gen_helper_sve_ldffbds_zsu,
5441 gen_helper_sve_ldffhds_be_zsu,
5442 gen_helper_sve_ldffsds_be_zsu,
5443 NULL, },
5444 { gen_helper_sve_ldffbdu_zsu,
5445 gen_helper_sve_ldffhdu_be_zsu,
5446 gen_helper_sve_ldffsdu_be_zsu,
5447 gen_helper_sve_ldffdd_be_zsu, } },
5448 { { gen_helper_sve_ldffbds_zss,
5449 gen_helper_sve_ldffhds_be_zss,
5450 gen_helper_sve_ldffsds_be_zss,
5451 NULL, },
5452 { gen_helper_sve_ldffbdu_zss,
5453 gen_helper_sve_ldffhdu_be_zss,
5454 gen_helper_sve_ldffsdu_be_zss,
5455 gen_helper_sve_ldffdd_be_zss, } },
5456 { { gen_helper_sve_ldffbds_zd,
5457 gen_helper_sve_ldffhds_be_zd,
5458 gen_helper_sve_ldffsds_be_zd,
5459 NULL, },
5460 { gen_helper_sve_ldffbdu_zd,
5461 gen_helper_sve_ldffhdu_be_zd,
5462 gen_helper_sve_ldffsdu_be_zd,
5463 gen_helper_sve_ldffdd_be_zd, } } } } },
5464 { /* MTE Active */
5465 { /* Little-endian */
5466 { { { gen_helper_sve_ldbds_zsu_mte,
5467 gen_helper_sve_ldhds_le_zsu_mte,
5468 gen_helper_sve_ldsds_le_zsu_mte,
5469 NULL, },
5470 { gen_helper_sve_ldbdu_zsu_mte,
5471 gen_helper_sve_ldhdu_le_zsu_mte,
5472 gen_helper_sve_ldsdu_le_zsu_mte,
5473 gen_helper_sve_lddd_le_zsu_mte, } },
5474 { { gen_helper_sve_ldbds_zss_mte,
5475 gen_helper_sve_ldhds_le_zss_mte,
5476 gen_helper_sve_ldsds_le_zss_mte,
5477 NULL, },
5478 { gen_helper_sve_ldbdu_zss_mte,
5479 gen_helper_sve_ldhdu_le_zss_mte,
5480 gen_helper_sve_ldsdu_le_zss_mte,
5481 gen_helper_sve_lddd_le_zss_mte, } },
5482 { { gen_helper_sve_ldbds_zd_mte,
5483 gen_helper_sve_ldhds_le_zd_mte,
5484 gen_helper_sve_ldsds_le_zd_mte,
5485 NULL, },
5486 { gen_helper_sve_ldbdu_zd_mte,
5487 gen_helper_sve_ldhdu_le_zd_mte,
5488 gen_helper_sve_ldsdu_le_zd_mte,
5489 gen_helper_sve_lddd_le_zd_mte, } } },
5490
5491 /* First-fault */
5492 { { { gen_helper_sve_ldffbds_zsu_mte,
5493 gen_helper_sve_ldffhds_le_zsu_mte,
5494 gen_helper_sve_ldffsds_le_zsu_mte,
5495 NULL, },
5496 { gen_helper_sve_ldffbdu_zsu_mte,
5497 gen_helper_sve_ldffhdu_le_zsu_mte,
5498 gen_helper_sve_ldffsdu_le_zsu_mte,
5499 gen_helper_sve_ldffdd_le_zsu_mte, } },
5500 { { gen_helper_sve_ldffbds_zss_mte,
5501 gen_helper_sve_ldffhds_le_zss_mte,
5502 gen_helper_sve_ldffsds_le_zss_mte,
5503 NULL, },
5504 { gen_helper_sve_ldffbdu_zss_mte,
5505 gen_helper_sve_ldffhdu_le_zss_mte,
5506 gen_helper_sve_ldffsdu_le_zss_mte,
5507 gen_helper_sve_ldffdd_le_zss_mte, } },
5508 { { gen_helper_sve_ldffbds_zd_mte,
5509 gen_helper_sve_ldffhds_le_zd_mte,
5510 gen_helper_sve_ldffsds_le_zd_mte,
5511 NULL, },
5512 { gen_helper_sve_ldffbdu_zd_mte,
5513 gen_helper_sve_ldffhdu_le_zd_mte,
5514 gen_helper_sve_ldffsdu_le_zd_mte,
5515 gen_helper_sve_ldffdd_le_zd_mte, } } } },
5516 { /* Big-endian */
5517 { { { gen_helper_sve_ldbds_zsu_mte,
5518 gen_helper_sve_ldhds_be_zsu_mte,
5519 gen_helper_sve_ldsds_be_zsu_mte,
5520 NULL, },
5521 { gen_helper_sve_ldbdu_zsu_mte,
5522 gen_helper_sve_ldhdu_be_zsu_mte,
5523 gen_helper_sve_ldsdu_be_zsu_mte,
5524 gen_helper_sve_lddd_be_zsu_mte, } },
5525 { { gen_helper_sve_ldbds_zss_mte,
5526 gen_helper_sve_ldhds_be_zss_mte,
5527 gen_helper_sve_ldsds_be_zss_mte,
5528 NULL, },
5529 { gen_helper_sve_ldbdu_zss_mte,
5530 gen_helper_sve_ldhdu_be_zss_mte,
5531 gen_helper_sve_ldsdu_be_zss_mte,
5532 gen_helper_sve_lddd_be_zss_mte, } },
5533 { { gen_helper_sve_ldbds_zd_mte,
5534 gen_helper_sve_ldhds_be_zd_mte,
5535 gen_helper_sve_ldsds_be_zd_mte,
5536 NULL, },
5537 { gen_helper_sve_ldbdu_zd_mte,
5538 gen_helper_sve_ldhdu_be_zd_mte,
5539 gen_helper_sve_ldsdu_be_zd_mte,
5540 gen_helper_sve_lddd_be_zd_mte, } } },
5541
5542 /* First-fault */
5543 { { { gen_helper_sve_ldffbds_zsu_mte,
5544 gen_helper_sve_ldffhds_be_zsu_mte,
5545 gen_helper_sve_ldffsds_be_zsu_mte,
5546 NULL, },
5547 { gen_helper_sve_ldffbdu_zsu_mte,
5548 gen_helper_sve_ldffhdu_be_zsu_mte,
5549 gen_helper_sve_ldffsdu_be_zsu_mte,
5550 gen_helper_sve_ldffdd_be_zsu_mte, } },
5551 { { gen_helper_sve_ldffbds_zss_mte,
5552 gen_helper_sve_ldffhds_be_zss_mte,
5553 gen_helper_sve_ldffsds_be_zss_mte,
5554 NULL, },
5555 { gen_helper_sve_ldffbdu_zss_mte,
5556 gen_helper_sve_ldffhdu_be_zss_mte,
5557 gen_helper_sve_ldffsdu_be_zss_mte,
5558 gen_helper_sve_ldffdd_be_zss_mte, } },
5559 { { gen_helper_sve_ldffbds_zd_mte,
5560 gen_helper_sve_ldffhds_be_zd_mte,
5561 gen_helper_sve_ldffsds_be_zd_mte,
5562 NULL, },
5563 { gen_helper_sve_ldffbdu_zd_mte,
5564 gen_helper_sve_ldffhdu_be_zd_mte,
5565 gen_helper_sve_ldffsdu_be_zd_mte,
5566 gen_helper_sve_ldffdd_be_zd_mte, } } } } },
5567 };
5568
trans_LD1_zprz(DisasContext * s,arg_LD1_zprz * a)5569 static bool trans_LD1_zprz(DisasContext *s, arg_LD1_zprz *a)
5570 {
5571 gen_helper_gvec_mem_scatter *fn = NULL;
5572 bool be = s->be_data == MO_BE;
5573 bool mte = s->mte_active[0];
5574
5575 if (!dc_isar_feature(aa64_sve, s)) {
5576 return false;
5577 }
5578 s->is_nonstreaming = true;
5579 if (!sve_access_check(s)) {
5580 return true;
5581 }
5582
5583 switch (a->esz) {
5584 case MO_32:
5585 fn = gather_load_fn32[mte][be][a->ff][a->xs][a->u][a->msz];
5586 break;
5587 case MO_64:
5588 fn = gather_load_fn64[mte][be][a->ff][a->xs][a->u][a->msz];
5589 break;
5590 }
5591 assert(fn != NULL);
5592
5593 do_mem_zpz(s, a->rd, a->pg, a->rm, a->scale * a->msz,
5594 cpu_reg_sp(s, a->rn), a->msz, false, fn);
5595 return true;
5596 }
5597
trans_LD1_zpiz(DisasContext * s,arg_LD1_zpiz * a)5598 static bool trans_LD1_zpiz(DisasContext *s, arg_LD1_zpiz *a)
5599 {
5600 gen_helper_gvec_mem_scatter *fn = NULL;
5601 bool be = s->be_data == MO_BE;
5602 bool mte = s->mte_active[0];
5603
5604 if (a->esz < a->msz || (a->esz == a->msz && !a->u)) {
5605 return false;
5606 }
5607 if (!dc_isar_feature(aa64_sve, s)) {
5608 return false;
5609 }
5610 s->is_nonstreaming = true;
5611 if (!sve_access_check(s)) {
5612 return true;
5613 }
5614
5615 switch (a->esz) {
5616 case MO_32:
5617 fn = gather_load_fn32[mte][be][a->ff][0][a->u][a->msz];
5618 break;
5619 case MO_64:
5620 fn = gather_load_fn64[mte][be][a->ff][2][a->u][a->msz];
5621 break;
5622 }
5623 assert(fn != NULL);
5624
5625 /* Treat LD1_zpiz (zn[x] + imm) the same way as LD1_zprz (rn + zm[x])
5626 * by loading the immediate into the scalar parameter.
5627 */
5628 do_mem_zpz(s, a->rd, a->pg, a->rn, 0,
5629 tcg_constant_i64(a->imm << a->msz), a->msz, false, fn);
5630 return true;
5631 }
5632
trans_LDNT1_zprz(DisasContext * s,arg_LD1_zprz * a)5633 static bool trans_LDNT1_zprz(DisasContext *s, arg_LD1_zprz *a)
5634 {
5635 gen_helper_gvec_mem_scatter *fn = NULL;
5636 bool be = s->be_data == MO_BE;
5637 bool mte = s->mte_active[0];
5638
5639 if (a->esz < a->msz + !a->u) {
5640 return false;
5641 }
5642 if (!dc_isar_feature(aa64_sve2, s)) {
5643 return false;
5644 }
5645 s->is_nonstreaming = true;
5646 if (!sve_access_check(s)) {
5647 return true;
5648 }
5649
5650 switch (a->esz) {
5651 case MO_32:
5652 fn = gather_load_fn32[mte][be][0][0][a->u][a->msz];
5653 break;
5654 case MO_64:
5655 fn = gather_load_fn64[mte][be][0][2][a->u][a->msz];
5656 break;
5657 }
5658 assert(fn != NULL);
5659
5660 do_mem_zpz(s, a->rd, a->pg, a->rn, 0,
5661 cpu_reg(s, a->rm), a->msz, false, fn);
5662 return true;
5663 }
5664
5665 /* Indexed by [mte][be][xs][msz]. */
5666 static gen_helper_gvec_mem_scatter * const scatter_store_fn32[2][2][2][3] = {
5667 { /* MTE Inactive */
5668 { /* Little-endian */
5669 { gen_helper_sve_stbs_zsu,
5670 gen_helper_sve_sths_le_zsu,
5671 gen_helper_sve_stss_le_zsu, },
5672 { gen_helper_sve_stbs_zss,
5673 gen_helper_sve_sths_le_zss,
5674 gen_helper_sve_stss_le_zss, } },
5675 { /* Big-endian */
5676 { gen_helper_sve_stbs_zsu,
5677 gen_helper_sve_sths_be_zsu,
5678 gen_helper_sve_stss_be_zsu, },
5679 { gen_helper_sve_stbs_zss,
5680 gen_helper_sve_sths_be_zss,
5681 gen_helper_sve_stss_be_zss, } } },
5682 { /* MTE Active */
5683 { /* Little-endian */
5684 { gen_helper_sve_stbs_zsu_mte,
5685 gen_helper_sve_sths_le_zsu_mte,
5686 gen_helper_sve_stss_le_zsu_mte, },
5687 { gen_helper_sve_stbs_zss_mte,
5688 gen_helper_sve_sths_le_zss_mte,
5689 gen_helper_sve_stss_le_zss_mte, } },
5690 { /* Big-endian */
5691 { gen_helper_sve_stbs_zsu_mte,
5692 gen_helper_sve_sths_be_zsu_mte,
5693 gen_helper_sve_stss_be_zsu_mte, },
5694 { gen_helper_sve_stbs_zss_mte,
5695 gen_helper_sve_sths_be_zss_mte,
5696 gen_helper_sve_stss_be_zss_mte, } } },
5697 };
5698
5699 /* Note that we overload xs=2 to indicate 64-bit offset. */
5700 static gen_helper_gvec_mem_scatter * const scatter_store_fn64[2][2][3][4] = {
5701 { /* MTE Inactive */
5702 { /* Little-endian */
5703 { gen_helper_sve_stbd_zsu,
5704 gen_helper_sve_sthd_le_zsu,
5705 gen_helper_sve_stsd_le_zsu,
5706 gen_helper_sve_stdd_le_zsu, },
5707 { gen_helper_sve_stbd_zss,
5708 gen_helper_sve_sthd_le_zss,
5709 gen_helper_sve_stsd_le_zss,
5710 gen_helper_sve_stdd_le_zss, },
5711 { gen_helper_sve_stbd_zd,
5712 gen_helper_sve_sthd_le_zd,
5713 gen_helper_sve_stsd_le_zd,
5714 gen_helper_sve_stdd_le_zd, } },
5715 { /* Big-endian */
5716 { gen_helper_sve_stbd_zsu,
5717 gen_helper_sve_sthd_be_zsu,
5718 gen_helper_sve_stsd_be_zsu,
5719 gen_helper_sve_stdd_be_zsu, },
5720 { gen_helper_sve_stbd_zss,
5721 gen_helper_sve_sthd_be_zss,
5722 gen_helper_sve_stsd_be_zss,
5723 gen_helper_sve_stdd_be_zss, },
5724 { gen_helper_sve_stbd_zd,
5725 gen_helper_sve_sthd_be_zd,
5726 gen_helper_sve_stsd_be_zd,
5727 gen_helper_sve_stdd_be_zd, } } },
5728 { /* MTE Inactive */
5729 { /* Little-endian */
5730 { gen_helper_sve_stbd_zsu_mte,
5731 gen_helper_sve_sthd_le_zsu_mte,
5732 gen_helper_sve_stsd_le_zsu_mte,
5733 gen_helper_sve_stdd_le_zsu_mte, },
5734 { gen_helper_sve_stbd_zss_mte,
5735 gen_helper_sve_sthd_le_zss_mte,
5736 gen_helper_sve_stsd_le_zss_mte,
5737 gen_helper_sve_stdd_le_zss_mte, },
5738 { gen_helper_sve_stbd_zd_mte,
5739 gen_helper_sve_sthd_le_zd_mte,
5740 gen_helper_sve_stsd_le_zd_mte,
5741 gen_helper_sve_stdd_le_zd_mte, } },
5742 { /* Big-endian */
5743 { gen_helper_sve_stbd_zsu_mte,
5744 gen_helper_sve_sthd_be_zsu_mte,
5745 gen_helper_sve_stsd_be_zsu_mte,
5746 gen_helper_sve_stdd_be_zsu_mte, },
5747 { gen_helper_sve_stbd_zss_mte,
5748 gen_helper_sve_sthd_be_zss_mte,
5749 gen_helper_sve_stsd_be_zss_mte,
5750 gen_helper_sve_stdd_be_zss_mte, },
5751 { gen_helper_sve_stbd_zd_mte,
5752 gen_helper_sve_sthd_be_zd_mte,
5753 gen_helper_sve_stsd_be_zd_mte,
5754 gen_helper_sve_stdd_be_zd_mte, } } },
5755 };
5756
trans_ST1_zprz(DisasContext * s,arg_ST1_zprz * a)5757 static bool trans_ST1_zprz(DisasContext *s, arg_ST1_zprz *a)
5758 {
5759 gen_helper_gvec_mem_scatter *fn;
5760 bool be = s->be_data == MO_BE;
5761 bool mte = s->mte_active[0];
5762
5763 if (a->esz < a->msz || (a->msz == 0 && a->scale)) {
5764 return false;
5765 }
5766 if (!dc_isar_feature(aa64_sve, s)) {
5767 return false;
5768 }
5769 s->is_nonstreaming = true;
5770 if (!sve_access_check(s)) {
5771 return true;
5772 }
5773 switch (a->esz) {
5774 case MO_32:
5775 fn = scatter_store_fn32[mte][be][a->xs][a->msz];
5776 break;
5777 case MO_64:
5778 fn = scatter_store_fn64[mte][be][a->xs][a->msz];
5779 break;
5780 default:
5781 g_assert_not_reached();
5782 }
5783 do_mem_zpz(s, a->rd, a->pg, a->rm, a->scale * a->msz,
5784 cpu_reg_sp(s, a->rn), a->msz, true, fn);
5785 return true;
5786 }
5787
trans_ST1_zpiz(DisasContext * s,arg_ST1_zpiz * a)5788 static bool trans_ST1_zpiz(DisasContext *s, arg_ST1_zpiz *a)
5789 {
5790 gen_helper_gvec_mem_scatter *fn = NULL;
5791 bool be = s->be_data == MO_BE;
5792 bool mte = s->mte_active[0];
5793
5794 if (a->esz < a->msz) {
5795 return false;
5796 }
5797 if (!dc_isar_feature(aa64_sve, s)) {
5798 return false;
5799 }
5800 s->is_nonstreaming = true;
5801 if (!sve_access_check(s)) {
5802 return true;
5803 }
5804
5805 switch (a->esz) {
5806 case MO_32:
5807 fn = scatter_store_fn32[mte][be][0][a->msz];
5808 break;
5809 case MO_64:
5810 fn = scatter_store_fn64[mte][be][2][a->msz];
5811 break;
5812 }
5813 assert(fn != NULL);
5814
5815 /* Treat ST1_zpiz (zn[x] + imm) the same way as ST1_zprz (rn + zm[x])
5816 * by loading the immediate into the scalar parameter.
5817 */
5818 do_mem_zpz(s, a->rd, a->pg, a->rn, 0,
5819 tcg_constant_i64(a->imm << a->msz), a->msz, true, fn);
5820 return true;
5821 }
5822
trans_STNT1_zprz(DisasContext * s,arg_ST1_zprz * a)5823 static bool trans_STNT1_zprz(DisasContext *s, arg_ST1_zprz *a)
5824 {
5825 gen_helper_gvec_mem_scatter *fn;
5826 bool be = s->be_data == MO_BE;
5827 bool mte = s->mte_active[0];
5828
5829 if (a->esz < a->msz) {
5830 return false;
5831 }
5832 if (!dc_isar_feature(aa64_sve2, s)) {
5833 return false;
5834 }
5835 s->is_nonstreaming = true;
5836 if (!sve_access_check(s)) {
5837 return true;
5838 }
5839
5840 switch (a->esz) {
5841 case MO_32:
5842 fn = scatter_store_fn32[mte][be][0][a->msz];
5843 break;
5844 case MO_64:
5845 fn = scatter_store_fn64[mte][be][2][a->msz];
5846 break;
5847 default:
5848 g_assert_not_reached();
5849 }
5850
5851 do_mem_zpz(s, a->rd, a->pg, a->rn, 0,
5852 cpu_reg(s, a->rm), a->msz, true, fn);
5853 return true;
5854 }
5855
5856 /*
5857 * Prefetches
5858 */
5859
trans_PRF(DisasContext * s,arg_PRF * a)5860 static bool trans_PRF(DisasContext *s, arg_PRF *a)
5861 {
5862 if (!dc_isar_feature(aa64_sve, s)) {
5863 return false;
5864 }
5865 /* Prefetch is a nop within QEMU. */
5866 (void)sve_access_check(s);
5867 return true;
5868 }
5869
trans_PRF_rr(DisasContext * s,arg_PRF_rr * a)5870 static bool trans_PRF_rr(DisasContext *s, arg_PRF_rr *a)
5871 {
5872 if (a->rm == 31 || !dc_isar_feature(aa64_sve, s)) {
5873 return false;
5874 }
5875 /* Prefetch is a nop within QEMU. */
5876 (void)sve_access_check(s);
5877 return true;
5878 }
5879
trans_PRF_ns(DisasContext * s,arg_PRF_ns * a)5880 static bool trans_PRF_ns(DisasContext *s, arg_PRF_ns *a)
5881 {
5882 if (!dc_isar_feature(aa64_sve, s)) {
5883 return false;
5884 }
5885 /* Prefetch is a nop within QEMU. */
5886 s->is_nonstreaming = true;
5887 (void)sve_access_check(s);
5888 return true;
5889 }
5890
5891 /*
5892 * Move Prefix
5893 *
5894 * TODO: The implementation so far could handle predicated merging movprfx.
5895 * The helper functions as written take an extra source register to
5896 * use in the operation, but the result is only written when predication
5897 * succeeds. For unpredicated movprfx, we need to rearrange the helpers
5898 * to allow the final write back to the destination to be unconditional.
5899 * For predicated zeroing movprfx, we need to rearrange the helpers to
5900 * allow the final write back to zero inactives.
5901 *
5902 * In the meantime, just emit the moves.
5903 */
5904
5905 TRANS_FEAT(MOVPRFX, aa64_sve, do_mov_z, a->rd, a->rn)
5906 TRANS_FEAT(MOVPRFX_m, aa64_sve, do_sel_z, a->rd, a->rn, a->rd, a->pg, a->esz)
5907 TRANS_FEAT(MOVPRFX_z, aa64_sve, do_movz_zpz, a->rd, a->rn, a->pg, a->esz, false)
5908
5909 /*
5910 * SVE2 Integer Multiply - Unpredicated
5911 */
5912
5913 TRANS_FEAT(MUL_zzz, aa64_sve2, gen_gvec_fn_arg_zzz, tcg_gen_gvec_mul, a)
5914
5915 static gen_helper_gvec_3 * const smulh_zzz_fns[4] = {
5916 gen_helper_gvec_smulh_b, gen_helper_gvec_smulh_h,
5917 gen_helper_gvec_smulh_s, gen_helper_gvec_smulh_d,
5918 };
5919 TRANS_FEAT(SMULH_zzz, aa64_sve2, gen_gvec_ool_arg_zzz,
5920 smulh_zzz_fns[a->esz], a, 0)
5921
5922 static gen_helper_gvec_3 * const umulh_zzz_fns[4] = {
5923 gen_helper_gvec_umulh_b, gen_helper_gvec_umulh_h,
5924 gen_helper_gvec_umulh_s, gen_helper_gvec_umulh_d,
5925 };
5926 TRANS_FEAT(UMULH_zzz, aa64_sve2, gen_gvec_ool_arg_zzz,
5927 umulh_zzz_fns[a->esz], a, 0)
5928
5929 TRANS_FEAT(PMUL_zzz, aa64_sve2, gen_gvec_ool_arg_zzz,
5930 gen_helper_gvec_pmul_b, a, 0)
5931
5932 static gen_helper_gvec_3 * const sqdmulh_zzz_fns[4] = {
5933 gen_helper_sve2_sqdmulh_b, gen_helper_sve2_sqdmulh_h,
5934 gen_helper_sve2_sqdmulh_s, gen_helper_sve2_sqdmulh_d,
5935 };
5936 TRANS_FEAT(SQDMULH_zzz, aa64_sve2, gen_gvec_ool_arg_zzz,
5937 sqdmulh_zzz_fns[a->esz], a, 0)
5938
5939 static gen_helper_gvec_3 * const sqrdmulh_zzz_fns[4] = {
5940 gen_helper_sve2_sqrdmulh_b, gen_helper_sve2_sqrdmulh_h,
5941 gen_helper_sve2_sqrdmulh_s, gen_helper_sve2_sqrdmulh_d,
5942 };
5943 TRANS_FEAT(SQRDMULH_zzz, aa64_sve2, gen_gvec_ool_arg_zzz,
5944 sqrdmulh_zzz_fns[a->esz], a, 0)
5945
5946 /*
5947 * SVE2 Integer - Predicated
5948 */
5949
5950 static gen_helper_gvec_4 * const sadlp_fns[4] = {
5951 NULL, gen_helper_sve2_sadalp_zpzz_h,
5952 gen_helper_sve2_sadalp_zpzz_s, gen_helper_sve2_sadalp_zpzz_d,
5953 };
5954 TRANS_FEAT(SADALP_zpzz, aa64_sve2, gen_gvec_ool_arg_zpzz,
5955 sadlp_fns[a->esz], a, 0)
5956
5957 static gen_helper_gvec_4 * const uadlp_fns[4] = {
5958 NULL, gen_helper_sve2_uadalp_zpzz_h,
5959 gen_helper_sve2_uadalp_zpzz_s, gen_helper_sve2_uadalp_zpzz_d,
5960 };
5961 TRANS_FEAT(UADALP_zpzz, aa64_sve2, gen_gvec_ool_arg_zpzz,
5962 uadlp_fns[a->esz], a, 0)
5963
5964 /*
5965 * SVE2 integer unary operations (predicated)
5966 */
5967
5968 TRANS_FEAT(URECPE, aa64_sve2, gen_gvec_ool_arg_zpz,
5969 a->esz == 2 ? gen_helper_sve2_urecpe_s : NULL, a, 0)
5970
5971 TRANS_FEAT(URSQRTE, aa64_sve2, gen_gvec_ool_arg_zpz,
5972 a->esz == 2 ? gen_helper_sve2_ursqrte_s : NULL, a, 0)
5973
5974 static gen_helper_gvec_3 * const sqabs_fns[4] = {
5975 gen_helper_sve2_sqabs_b, gen_helper_sve2_sqabs_h,
5976 gen_helper_sve2_sqabs_s, gen_helper_sve2_sqabs_d,
5977 };
5978 TRANS_FEAT(SQABS, aa64_sve2, gen_gvec_ool_arg_zpz, sqabs_fns[a->esz], a, 0)
5979
5980 static gen_helper_gvec_3 * const sqneg_fns[4] = {
5981 gen_helper_sve2_sqneg_b, gen_helper_sve2_sqneg_h,
5982 gen_helper_sve2_sqneg_s, gen_helper_sve2_sqneg_d,
5983 };
5984 TRANS_FEAT(SQNEG, aa64_sve2, gen_gvec_ool_arg_zpz, sqneg_fns[a->esz], a, 0)
5985
5986 DO_ZPZZ(SQSHL, aa64_sve2, sve2_sqshl)
5987 DO_ZPZZ(SQRSHL, aa64_sve2, sve2_sqrshl)
5988 DO_ZPZZ(SRSHL, aa64_sve2, sve2_srshl)
5989
5990 DO_ZPZZ(UQSHL, aa64_sve2, sve2_uqshl)
5991 DO_ZPZZ(UQRSHL, aa64_sve2, sve2_uqrshl)
5992 DO_ZPZZ(URSHL, aa64_sve2, sve2_urshl)
5993
5994 DO_ZPZZ(SHADD, aa64_sve2, sve2_shadd)
5995 DO_ZPZZ(SRHADD, aa64_sve2, sve2_srhadd)
5996 DO_ZPZZ(SHSUB, aa64_sve2, sve2_shsub)
5997
5998 DO_ZPZZ(UHADD, aa64_sve2, sve2_uhadd)
5999 DO_ZPZZ(URHADD, aa64_sve2, sve2_urhadd)
6000 DO_ZPZZ(UHSUB, aa64_sve2, sve2_uhsub)
6001
6002 DO_ZPZZ(ADDP, aa64_sve2, sve2_addp)
6003 DO_ZPZZ(SMAXP, aa64_sve2, sve2_smaxp)
6004 DO_ZPZZ(UMAXP, aa64_sve2, sve2_umaxp)
6005 DO_ZPZZ(SMINP, aa64_sve2, sve2_sminp)
6006 DO_ZPZZ(UMINP, aa64_sve2, sve2_uminp)
6007
6008 DO_ZPZZ(SQADD_zpzz, aa64_sve2, sve2_sqadd)
6009 DO_ZPZZ(UQADD_zpzz, aa64_sve2, sve2_uqadd)
6010 DO_ZPZZ(SQSUB_zpzz, aa64_sve2, sve2_sqsub)
6011 DO_ZPZZ(UQSUB_zpzz, aa64_sve2, sve2_uqsub)
6012 DO_ZPZZ(SUQADD, aa64_sve2, sve2_suqadd)
6013 DO_ZPZZ(USQADD, aa64_sve2, sve2_usqadd)
6014
6015 /*
6016 * SVE2 Widening Integer Arithmetic
6017 */
6018
6019 static gen_helper_gvec_3 * const saddl_fns[4] = {
6020 NULL, gen_helper_sve2_saddl_h,
6021 gen_helper_sve2_saddl_s, gen_helper_sve2_saddl_d,
6022 };
6023 TRANS_FEAT(SADDLB, aa64_sve2, gen_gvec_ool_arg_zzz,
6024 saddl_fns[a->esz], a, 0)
6025 TRANS_FEAT(SADDLT, aa64_sve2, gen_gvec_ool_arg_zzz,
6026 saddl_fns[a->esz], a, 3)
6027 TRANS_FEAT(SADDLBT, aa64_sve2, gen_gvec_ool_arg_zzz,
6028 saddl_fns[a->esz], a, 2)
6029
6030 static gen_helper_gvec_3 * const ssubl_fns[4] = {
6031 NULL, gen_helper_sve2_ssubl_h,
6032 gen_helper_sve2_ssubl_s, gen_helper_sve2_ssubl_d,
6033 };
6034 TRANS_FEAT(SSUBLB, aa64_sve2, gen_gvec_ool_arg_zzz,
6035 ssubl_fns[a->esz], a, 0)
6036 TRANS_FEAT(SSUBLT, aa64_sve2, gen_gvec_ool_arg_zzz,
6037 ssubl_fns[a->esz], a, 3)
6038 TRANS_FEAT(SSUBLBT, aa64_sve2, gen_gvec_ool_arg_zzz,
6039 ssubl_fns[a->esz], a, 2)
6040 TRANS_FEAT(SSUBLTB, aa64_sve2, gen_gvec_ool_arg_zzz,
6041 ssubl_fns[a->esz], a, 1)
6042
6043 static gen_helper_gvec_3 * const sabdl_fns[4] = {
6044 NULL, gen_helper_sve2_sabdl_h,
6045 gen_helper_sve2_sabdl_s, gen_helper_sve2_sabdl_d,
6046 };
6047 TRANS_FEAT(SABDLB, aa64_sve2, gen_gvec_ool_arg_zzz,
6048 sabdl_fns[a->esz], a, 0)
6049 TRANS_FEAT(SABDLT, aa64_sve2, gen_gvec_ool_arg_zzz,
6050 sabdl_fns[a->esz], a, 3)
6051
6052 static gen_helper_gvec_3 * const uaddl_fns[4] = {
6053 NULL, gen_helper_sve2_uaddl_h,
6054 gen_helper_sve2_uaddl_s, gen_helper_sve2_uaddl_d,
6055 };
6056 TRANS_FEAT(UADDLB, aa64_sve2, gen_gvec_ool_arg_zzz,
6057 uaddl_fns[a->esz], a, 0)
6058 TRANS_FEAT(UADDLT, aa64_sve2, gen_gvec_ool_arg_zzz,
6059 uaddl_fns[a->esz], a, 3)
6060
6061 static gen_helper_gvec_3 * const usubl_fns[4] = {
6062 NULL, gen_helper_sve2_usubl_h,
6063 gen_helper_sve2_usubl_s, gen_helper_sve2_usubl_d,
6064 };
6065 TRANS_FEAT(USUBLB, aa64_sve2, gen_gvec_ool_arg_zzz,
6066 usubl_fns[a->esz], a, 0)
6067 TRANS_FEAT(USUBLT, aa64_sve2, gen_gvec_ool_arg_zzz,
6068 usubl_fns[a->esz], a, 3)
6069
6070 static gen_helper_gvec_3 * const uabdl_fns[4] = {
6071 NULL, gen_helper_sve2_uabdl_h,
6072 gen_helper_sve2_uabdl_s, gen_helper_sve2_uabdl_d,
6073 };
6074 TRANS_FEAT(UABDLB, aa64_sve2, gen_gvec_ool_arg_zzz,
6075 uabdl_fns[a->esz], a, 0)
6076 TRANS_FEAT(UABDLT, aa64_sve2, gen_gvec_ool_arg_zzz,
6077 uabdl_fns[a->esz], a, 3)
6078
6079 static gen_helper_gvec_3 * const sqdmull_fns[4] = {
6080 NULL, gen_helper_sve2_sqdmull_zzz_h,
6081 gen_helper_sve2_sqdmull_zzz_s, gen_helper_sve2_sqdmull_zzz_d,
6082 };
6083 TRANS_FEAT(SQDMULLB_zzz, aa64_sve2, gen_gvec_ool_arg_zzz,
6084 sqdmull_fns[a->esz], a, 0)
6085 TRANS_FEAT(SQDMULLT_zzz, aa64_sve2, gen_gvec_ool_arg_zzz,
6086 sqdmull_fns[a->esz], a, 3)
6087
6088 static gen_helper_gvec_3 * const smull_fns[4] = {
6089 NULL, gen_helper_sve2_smull_zzz_h,
6090 gen_helper_sve2_smull_zzz_s, gen_helper_sve2_smull_zzz_d,
6091 };
6092 TRANS_FEAT(SMULLB_zzz, aa64_sve2, gen_gvec_ool_arg_zzz,
6093 smull_fns[a->esz], a, 0)
6094 TRANS_FEAT(SMULLT_zzz, aa64_sve2, gen_gvec_ool_arg_zzz,
6095 smull_fns[a->esz], a, 3)
6096
6097 static gen_helper_gvec_3 * const umull_fns[4] = {
6098 NULL, gen_helper_sve2_umull_zzz_h,
6099 gen_helper_sve2_umull_zzz_s, gen_helper_sve2_umull_zzz_d,
6100 };
6101 TRANS_FEAT(UMULLB_zzz, aa64_sve2, gen_gvec_ool_arg_zzz,
6102 umull_fns[a->esz], a, 0)
6103 TRANS_FEAT(UMULLT_zzz, aa64_sve2, gen_gvec_ool_arg_zzz,
6104 umull_fns[a->esz], a, 3)
6105
6106 static gen_helper_gvec_3 * const eoril_fns[4] = {
6107 gen_helper_sve2_eoril_b, gen_helper_sve2_eoril_h,
6108 gen_helper_sve2_eoril_s, gen_helper_sve2_eoril_d,
6109 };
6110 TRANS_FEAT(EORBT, aa64_sve2, gen_gvec_ool_arg_zzz, eoril_fns[a->esz], a, 2)
6111 TRANS_FEAT(EORTB, aa64_sve2, gen_gvec_ool_arg_zzz, eoril_fns[a->esz], a, 1)
6112
do_trans_pmull(DisasContext * s,arg_rrr_esz * a,bool sel)6113 static bool do_trans_pmull(DisasContext *s, arg_rrr_esz *a, bool sel)
6114 {
6115 static gen_helper_gvec_3 * const fns[4] = {
6116 gen_helper_gvec_pmull_q, gen_helper_sve2_pmull_h,
6117 NULL, gen_helper_sve2_pmull_d,
6118 };
6119
6120 if (a->esz == 0) {
6121 if (!dc_isar_feature(aa64_sve2_pmull128, s)) {
6122 return false;
6123 }
6124 s->is_nonstreaming = true;
6125 } else if (!dc_isar_feature(aa64_sve, s)) {
6126 return false;
6127 }
6128 return gen_gvec_ool_arg_zzz(s, fns[a->esz], a, sel);
6129 }
6130
6131 TRANS_FEAT(PMULLB, aa64_sve2, do_trans_pmull, a, false)
6132 TRANS_FEAT(PMULLT, aa64_sve2, do_trans_pmull, a, true)
6133
6134 static gen_helper_gvec_3 * const saddw_fns[4] = {
6135 NULL, gen_helper_sve2_saddw_h,
6136 gen_helper_sve2_saddw_s, gen_helper_sve2_saddw_d,
6137 };
6138 TRANS_FEAT(SADDWB, aa64_sve2, gen_gvec_ool_arg_zzz, saddw_fns[a->esz], a, 0)
6139 TRANS_FEAT(SADDWT, aa64_sve2, gen_gvec_ool_arg_zzz, saddw_fns[a->esz], a, 1)
6140
6141 static gen_helper_gvec_3 * const ssubw_fns[4] = {
6142 NULL, gen_helper_sve2_ssubw_h,
6143 gen_helper_sve2_ssubw_s, gen_helper_sve2_ssubw_d,
6144 };
6145 TRANS_FEAT(SSUBWB, aa64_sve2, gen_gvec_ool_arg_zzz, ssubw_fns[a->esz], a, 0)
6146 TRANS_FEAT(SSUBWT, aa64_sve2, gen_gvec_ool_arg_zzz, ssubw_fns[a->esz], a, 1)
6147
6148 static gen_helper_gvec_3 * const uaddw_fns[4] = {
6149 NULL, gen_helper_sve2_uaddw_h,
6150 gen_helper_sve2_uaddw_s, gen_helper_sve2_uaddw_d,
6151 };
6152 TRANS_FEAT(UADDWB, aa64_sve2, gen_gvec_ool_arg_zzz, uaddw_fns[a->esz], a, 0)
6153 TRANS_FEAT(UADDWT, aa64_sve2, gen_gvec_ool_arg_zzz, uaddw_fns[a->esz], a, 1)
6154
6155 static gen_helper_gvec_3 * const usubw_fns[4] = {
6156 NULL, gen_helper_sve2_usubw_h,
6157 gen_helper_sve2_usubw_s, gen_helper_sve2_usubw_d,
6158 };
6159 TRANS_FEAT(USUBWB, aa64_sve2, gen_gvec_ool_arg_zzz, usubw_fns[a->esz], a, 0)
6160 TRANS_FEAT(USUBWT, aa64_sve2, gen_gvec_ool_arg_zzz, usubw_fns[a->esz], a, 1)
6161
gen_sshll_vec(unsigned vece,TCGv_vec d,TCGv_vec n,int64_t imm)6162 static void gen_sshll_vec(unsigned vece, TCGv_vec d, TCGv_vec n, int64_t imm)
6163 {
6164 int top = imm & 1;
6165 int shl = imm >> 1;
6166 int halfbits = 4 << vece;
6167
6168 if (top) {
6169 if (shl == halfbits) {
6170 tcg_gen_and_vec(vece, d, n,
6171 tcg_constant_vec_matching(d, vece,
6172 MAKE_64BIT_MASK(halfbits, halfbits)));
6173 } else {
6174 tcg_gen_sari_vec(vece, d, n, halfbits);
6175 tcg_gen_shli_vec(vece, d, d, shl);
6176 }
6177 } else {
6178 tcg_gen_shli_vec(vece, d, n, halfbits);
6179 tcg_gen_sari_vec(vece, d, d, halfbits - shl);
6180 }
6181 }
6182
gen_ushll_i64(unsigned vece,TCGv_i64 d,TCGv_i64 n,int imm)6183 static void gen_ushll_i64(unsigned vece, TCGv_i64 d, TCGv_i64 n, int imm)
6184 {
6185 int halfbits = 4 << vece;
6186 int top = imm & 1;
6187 int shl = (imm >> 1);
6188 int shift;
6189 uint64_t mask;
6190
6191 mask = MAKE_64BIT_MASK(0, halfbits);
6192 mask <<= shl;
6193 mask = dup_const(vece, mask);
6194
6195 shift = shl - top * halfbits;
6196 if (shift < 0) {
6197 tcg_gen_shri_i64(d, n, -shift);
6198 } else {
6199 tcg_gen_shli_i64(d, n, shift);
6200 }
6201 tcg_gen_andi_i64(d, d, mask);
6202 }
6203
gen_ushll16_i64(TCGv_i64 d,TCGv_i64 n,int64_t imm)6204 static void gen_ushll16_i64(TCGv_i64 d, TCGv_i64 n, int64_t imm)
6205 {
6206 gen_ushll_i64(MO_16, d, n, imm);
6207 }
6208
gen_ushll32_i64(TCGv_i64 d,TCGv_i64 n,int64_t imm)6209 static void gen_ushll32_i64(TCGv_i64 d, TCGv_i64 n, int64_t imm)
6210 {
6211 gen_ushll_i64(MO_32, d, n, imm);
6212 }
6213
gen_ushll64_i64(TCGv_i64 d,TCGv_i64 n,int64_t imm)6214 static void gen_ushll64_i64(TCGv_i64 d, TCGv_i64 n, int64_t imm)
6215 {
6216 gen_ushll_i64(MO_64, d, n, imm);
6217 }
6218
gen_ushll_vec(unsigned vece,TCGv_vec d,TCGv_vec n,int64_t imm)6219 static void gen_ushll_vec(unsigned vece, TCGv_vec d, TCGv_vec n, int64_t imm)
6220 {
6221 int halfbits = 4 << vece;
6222 int top = imm & 1;
6223 int shl = imm >> 1;
6224
6225 if (top) {
6226 if (shl == halfbits) {
6227 tcg_gen_and_vec(vece, d, n,
6228 tcg_constant_vec_matching(d, vece,
6229 MAKE_64BIT_MASK(halfbits, halfbits)));
6230 } else {
6231 tcg_gen_shri_vec(vece, d, n, halfbits);
6232 tcg_gen_shli_vec(vece, d, d, shl);
6233 }
6234 } else {
6235 if (shl == 0) {
6236 tcg_gen_and_vec(vece, d, n,
6237 tcg_constant_vec_matching(d, vece,
6238 MAKE_64BIT_MASK(0, halfbits)));
6239 } else {
6240 tcg_gen_shli_vec(vece, d, n, halfbits);
6241 tcg_gen_shri_vec(vece, d, d, halfbits - shl);
6242 }
6243 }
6244 }
6245
do_shll_tb(DisasContext * s,arg_rri_esz * a,const GVecGen2i ops[3],bool sel)6246 static bool do_shll_tb(DisasContext *s, arg_rri_esz *a,
6247 const GVecGen2i ops[3], bool sel)
6248 {
6249
6250 if (a->esz < 0 || a->esz > 2) {
6251 return false;
6252 }
6253 if (sve_access_check(s)) {
6254 unsigned vsz = vec_full_reg_size(s);
6255 tcg_gen_gvec_2i(vec_full_reg_offset(s, a->rd),
6256 vec_full_reg_offset(s, a->rn),
6257 vsz, vsz, (a->imm << 1) | sel,
6258 &ops[a->esz]);
6259 }
6260 return true;
6261 }
6262
6263 static const TCGOpcode sshll_list[] = {
6264 INDEX_op_shli_vec, INDEX_op_sari_vec, 0
6265 };
6266 static const GVecGen2i sshll_ops[3] = {
6267 { .fniv = gen_sshll_vec,
6268 .opt_opc = sshll_list,
6269 .fno = gen_helper_sve2_sshll_h,
6270 .vece = MO_16 },
6271 { .fniv = gen_sshll_vec,
6272 .opt_opc = sshll_list,
6273 .fno = gen_helper_sve2_sshll_s,
6274 .vece = MO_32 },
6275 { .fniv = gen_sshll_vec,
6276 .opt_opc = sshll_list,
6277 .fno = gen_helper_sve2_sshll_d,
6278 .vece = MO_64 }
6279 };
6280 TRANS_FEAT(SSHLLB, aa64_sve2, do_shll_tb, a, sshll_ops, false)
6281 TRANS_FEAT(SSHLLT, aa64_sve2, do_shll_tb, a, sshll_ops, true)
6282
6283 static const TCGOpcode ushll_list[] = {
6284 INDEX_op_shli_vec, INDEX_op_shri_vec, 0
6285 };
6286 static const GVecGen2i ushll_ops[3] = {
6287 { .fni8 = gen_ushll16_i64,
6288 .fniv = gen_ushll_vec,
6289 .opt_opc = ushll_list,
6290 .fno = gen_helper_sve2_ushll_h,
6291 .vece = MO_16 },
6292 { .fni8 = gen_ushll32_i64,
6293 .fniv = gen_ushll_vec,
6294 .opt_opc = ushll_list,
6295 .fno = gen_helper_sve2_ushll_s,
6296 .vece = MO_32 },
6297 { .fni8 = gen_ushll64_i64,
6298 .fniv = gen_ushll_vec,
6299 .opt_opc = ushll_list,
6300 .fno = gen_helper_sve2_ushll_d,
6301 .vece = MO_64 },
6302 };
6303 TRANS_FEAT(USHLLB, aa64_sve2, do_shll_tb, a, ushll_ops, false)
6304 TRANS_FEAT(USHLLT, aa64_sve2, do_shll_tb, a, ushll_ops, true)
6305
6306 static gen_helper_gvec_3 * const bext_fns[4] = {
6307 gen_helper_sve2_bext_b, gen_helper_sve2_bext_h,
6308 gen_helper_sve2_bext_s, gen_helper_sve2_bext_d,
6309 };
6310 TRANS_FEAT_NONSTREAMING(BEXT, aa64_sve2_bitperm, gen_gvec_ool_arg_zzz,
6311 bext_fns[a->esz], a, 0)
6312
6313 static gen_helper_gvec_3 * const bdep_fns[4] = {
6314 gen_helper_sve2_bdep_b, gen_helper_sve2_bdep_h,
6315 gen_helper_sve2_bdep_s, gen_helper_sve2_bdep_d,
6316 };
6317 TRANS_FEAT_NONSTREAMING(BDEP, aa64_sve2_bitperm, gen_gvec_ool_arg_zzz,
6318 bdep_fns[a->esz], a, 0)
6319
6320 static gen_helper_gvec_3 * const bgrp_fns[4] = {
6321 gen_helper_sve2_bgrp_b, gen_helper_sve2_bgrp_h,
6322 gen_helper_sve2_bgrp_s, gen_helper_sve2_bgrp_d,
6323 };
6324 TRANS_FEAT_NONSTREAMING(BGRP, aa64_sve2_bitperm, gen_gvec_ool_arg_zzz,
6325 bgrp_fns[a->esz], a, 0)
6326
6327 static gen_helper_gvec_3 * const cadd_fns[4] = {
6328 gen_helper_sve2_cadd_b, gen_helper_sve2_cadd_h,
6329 gen_helper_sve2_cadd_s, gen_helper_sve2_cadd_d,
6330 };
6331 TRANS_FEAT(CADD_rot90, aa64_sve2, gen_gvec_ool_arg_zzz,
6332 cadd_fns[a->esz], a, 0)
6333 TRANS_FEAT(CADD_rot270, aa64_sve2, gen_gvec_ool_arg_zzz,
6334 cadd_fns[a->esz], a, 1)
6335
6336 static gen_helper_gvec_3 * const sqcadd_fns[4] = {
6337 gen_helper_sve2_sqcadd_b, gen_helper_sve2_sqcadd_h,
6338 gen_helper_sve2_sqcadd_s, gen_helper_sve2_sqcadd_d,
6339 };
6340 TRANS_FEAT(SQCADD_rot90, aa64_sve2, gen_gvec_ool_arg_zzz,
6341 sqcadd_fns[a->esz], a, 0)
6342 TRANS_FEAT(SQCADD_rot270, aa64_sve2, gen_gvec_ool_arg_zzz,
6343 sqcadd_fns[a->esz], a, 1)
6344
6345 static gen_helper_gvec_4 * const sabal_fns[4] = {
6346 NULL, gen_helper_sve2_sabal_h,
6347 gen_helper_sve2_sabal_s, gen_helper_sve2_sabal_d,
6348 };
6349 TRANS_FEAT(SABALB, aa64_sve2, gen_gvec_ool_arg_zzzz, sabal_fns[a->esz], a, 0)
6350 TRANS_FEAT(SABALT, aa64_sve2, gen_gvec_ool_arg_zzzz, sabal_fns[a->esz], a, 1)
6351
6352 static gen_helper_gvec_4 * const uabal_fns[4] = {
6353 NULL, gen_helper_sve2_uabal_h,
6354 gen_helper_sve2_uabal_s, gen_helper_sve2_uabal_d,
6355 };
6356 TRANS_FEAT(UABALB, aa64_sve2, gen_gvec_ool_arg_zzzz, uabal_fns[a->esz], a, 0)
6357 TRANS_FEAT(UABALT, aa64_sve2, gen_gvec_ool_arg_zzzz, uabal_fns[a->esz], a, 1)
6358
do_adcl(DisasContext * s,arg_rrrr_esz * a,bool sel)6359 static bool do_adcl(DisasContext *s, arg_rrrr_esz *a, bool sel)
6360 {
6361 static gen_helper_gvec_4 * const fns[2] = {
6362 gen_helper_sve2_adcl_s,
6363 gen_helper_sve2_adcl_d,
6364 };
6365 /*
6366 * Note that in this case the ESZ field encodes both size and sign.
6367 * Split out 'subtract' into bit 1 of the data field for the helper.
6368 */
6369 return gen_gvec_ool_arg_zzzz(s, fns[a->esz & 1], a, (a->esz & 2) | sel);
6370 }
6371
TRANS_FEAT(ADCLB,aa64_sve2,do_adcl,a,false)6372 TRANS_FEAT(ADCLB, aa64_sve2, do_adcl, a, false)
6373 TRANS_FEAT(ADCLT, aa64_sve2, do_adcl, a, true)
6374
6375 TRANS_FEAT(SSRA, aa64_sve2, gen_gvec_fn_arg_zzi, gen_gvec_ssra, a)
6376 TRANS_FEAT(USRA, aa64_sve2, gen_gvec_fn_arg_zzi, gen_gvec_usra, a)
6377 TRANS_FEAT(SRSRA, aa64_sve2, gen_gvec_fn_arg_zzi, gen_gvec_srsra, a)
6378 TRANS_FEAT(URSRA, aa64_sve2, gen_gvec_fn_arg_zzi, gen_gvec_ursra, a)
6379 TRANS_FEAT(SRI, aa64_sve2, gen_gvec_fn_arg_zzi, gen_gvec_sri, a)
6380 TRANS_FEAT(SLI, aa64_sve2, gen_gvec_fn_arg_zzi, gen_gvec_sli, a)
6381
6382 TRANS_FEAT(SABA, aa64_sve2, gen_gvec_fn_arg_zzz, gen_gvec_saba, a)
6383 TRANS_FEAT(UABA, aa64_sve2, gen_gvec_fn_arg_zzz, gen_gvec_uaba, a)
6384
6385 static bool do_narrow_extract(DisasContext *s, arg_rri_esz *a,
6386 const GVecGen2 ops[3])
6387 {
6388 if (a->esz < 0 || a->esz > MO_32 || a->imm != 0) {
6389 return false;
6390 }
6391 if (sve_access_check(s)) {
6392 unsigned vsz = vec_full_reg_size(s);
6393 tcg_gen_gvec_2(vec_full_reg_offset(s, a->rd),
6394 vec_full_reg_offset(s, a->rn),
6395 vsz, vsz, &ops[a->esz]);
6396 }
6397 return true;
6398 }
6399
6400 static const TCGOpcode sqxtn_list[] = {
6401 INDEX_op_shli_vec, INDEX_op_smin_vec, INDEX_op_smax_vec, 0
6402 };
6403
gen_sqxtnb_vec(unsigned vece,TCGv_vec d,TCGv_vec n)6404 static void gen_sqxtnb_vec(unsigned vece, TCGv_vec d, TCGv_vec n)
6405 {
6406 int halfbits = 4 << vece;
6407 int64_t mask = (1ull << halfbits) - 1;
6408 int64_t min = -1ull << (halfbits - 1);
6409 int64_t max = -min - 1;
6410
6411 tcg_gen_smax_vec(vece, d, n, tcg_constant_vec_matching(d, vece, min));
6412 tcg_gen_smin_vec(vece, d, d, tcg_constant_vec_matching(d, vece, max));
6413 tcg_gen_and_vec(vece, d, d, tcg_constant_vec_matching(d, vece, mask));
6414 }
6415
6416 static const GVecGen2 sqxtnb_ops[3] = {
6417 { .fniv = gen_sqxtnb_vec,
6418 .opt_opc = sqxtn_list,
6419 .fno = gen_helper_sve2_sqxtnb_h,
6420 .vece = MO_16 },
6421 { .fniv = gen_sqxtnb_vec,
6422 .opt_opc = sqxtn_list,
6423 .fno = gen_helper_sve2_sqxtnb_s,
6424 .vece = MO_32 },
6425 { .fniv = gen_sqxtnb_vec,
6426 .opt_opc = sqxtn_list,
6427 .fno = gen_helper_sve2_sqxtnb_d,
6428 .vece = MO_64 },
6429 };
TRANS_FEAT(SQXTNB,aa64_sve2,do_narrow_extract,a,sqxtnb_ops)6430 TRANS_FEAT(SQXTNB, aa64_sve2, do_narrow_extract, a, sqxtnb_ops)
6431
6432 static void gen_sqxtnt_vec(unsigned vece, TCGv_vec d, TCGv_vec n)
6433 {
6434 int halfbits = 4 << vece;
6435 int64_t mask = (1ull << halfbits) - 1;
6436 int64_t min = -1ull << (halfbits - 1);
6437 int64_t max = -min - 1;
6438
6439 tcg_gen_smax_vec(vece, n, n, tcg_constant_vec_matching(d, vece, min));
6440 tcg_gen_smin_vec(vece, n, n, tcg_constant_vec_matching(d, vece, max));
6441 tcg_gen_shli_vec(vece, n, n, halfbits);
6442 tcg_gen_bitsel_vec(vece, d, tcg_constant_vec_matching(d, vece, mask), d, n);
6443 }
6444
6445 static const GVecGen2 sqxtnt_ops[3] = {
6446 { .fniv = gen_sqxtnt_vec,
6447 .opt_opc = sqxtn_list,
6448 .load_dest = true,
6449 .fno = gen_helper_sve2_sqxtnt_h,
6450 .vece = MO_16 },
6451 { .fniv = gen_sqxtnt_vec,
6452 .opt_opc = sqxtn_list,
6453 .load_dest = true,
6454 .fno = gen_helper_sve2_sqxtnt_s,
6455 .vece = MO_32 },
6456 { .fniv = gen_sqxtnt_vec,
6457 .opt_opc = sqxtn_list,
6458 .load_dest = true,
6459 .fno = gen_helper_sve2_sqxtnt_d,
6460 .vece = MO_64 },
6461 };
6462 TRANS_FEAT(SQXTNT, aa64_sve2, do_narrow_extract, a, sqxtnt_ops)
6463
6464 static const TCGOpcode uqxtn_list[] = {
6465 INDEX_op_shli_vec, INDEX_op_umin_vec, 0
6466 };
6467
gen_uqxtnb_vec(unsigned vece,TCGv_vec d,TCGv_vec n)6468 static void gen_uqxtnb_vec(unsigned vece, TCGv_vec d, TCGv_vec n)
6469 {
6470 int halfbits = 4 << vece;
6471 int64_t max = (1ull << halfbits) - 1;
6472
6473 tcg_gen_umin_vec(vece, d, n, tcg_constant_vec_matching(d, vece, max));
6474 }
6475
6476 static const GVecGen2 uqxtnb_ops[3] = {
6477 { .fniv = gen_uqxtnb_vec,
6478 .opt_opc = uqxtn_list,
6479 .fno = gen_helper_sve2_uqxtnb_h,
6480 .vece = MO_16 },
6481 { .fniv = gen_uqxtnb_vec,
6482 .opt_opc = uqxtn_list,
6483 .fno = gen_helper_sve2_uqxtnb_s,
6484 .vece = MO_32 },
6485 { .fniv = gen_uqxtnb_vec,
6486 .opt_opc = uqxtn_list,
6487 .fno = gen_helper_sve2_uqxtnb_d,
6488 .vece = MO_64 },
6489 };
TRANS_FEAT(UQXTNB,aa64_sve2,do_narrow_extract,a,uqxtnb_ops)6490 TRANS_FEAT(UQXTNB, aa64_sve2, do_narrow_extract, a, uqxtnb_ops)
6491
6492 static void gen_uqxtnt_vec(unsigned vece, TCGv_vec d, TCGv_vec n)
6493 {
6494 int halfbits = 4 << vece;
6495 int64_t max = (1ull << halfbits) - 1;
6496 TCGv_vec maxv = tcg_constant_vec_matching(d, vece, max);
6497
6498 tcg_gen_umin_vec(vece, n, n, maxv);
6499 tcg_gen_shli_vec(vece, n, n, halfbits);
6500 tcg_gen_bitsel_vec(vece, d, maxv, d, n);
6501 }
6502
6503 static const GVecGen2 uqxtnt_ops[3] = {
6504 { .fniv = gen_uqxtnt_vec,
6505 .opt_opc = uqxtn_list,
6506 .load_dest = true,
6507 .fno = gen_helper_sve2_uqxtnt_h,
6508 .vece = MO_16 },
6509 { .fniv = gen_uqxtnt_vec,
6510 .opt_opc = uqxtn_list,
6511 .load_dest = true,
6512 .fno = gen_helper_sve2_uqxtnt_s,
6513 .vece = MO_32 },
6514 { .fniv = gen_uqxtnt_vec,
6515 .opt_opc = uqxtn_list,
6516 .load_dest = true,
6517 .fno = gen_helper_sve2_uqxtnt_d,
6518 .vece = MO_64 },
6519 };
6520 TRANS_FEAT(UQXTNT, aa64_sve2, do_narrow_extract, a, uqxtnt_ops)
6521
6522 static const TCGOpcode sqxtun_list[] = {
6523 INDEX_op_shli_vec, INDEX_op_umin_vec, INDEX_op_smax_vec, 0
6524 };
6525
gen_sqxtunb_vec(unsigned vece,TCGv_vec d,TCGv_vec n)6526 static void gen_sqxtunb_vec(unsigned vece, TCGv_vec d, TCGv_vec n)
6527 {
6528 int halfbits = 4 << vece;
6529 int64_t max = (1ull << halfbits) - 1;
6530
6531 tcg_gen_smax_vec(vece, d, n, tcg_constant_vec_matching(d, vece, 0));
6532 tcg_gen_umin_vec(vece, d, d, tcg_constant_vec_matching(d, vece, max));
6533 }
6534
6535 static const GVecGen2 sqxtunb_ops[3] = {
6536 { .fniv = gen_sqxtunb_vec,
6537 .opt_opc = sqxtun_list,
6538 .fno = gen_helper_sve2_sqxtunb_h,
6539 .vece = MO_16 },
6540 { .fniv = gen_sqxtunb_vec,
6541 .opt_opc = sqxtun_list,
6542 .fno = gen_helper_sve2_sqxtunb_s,
6543 .vece = MO_32 },
6544 { .fniv = gen_sqxtunb_vec,
6545 .opt_opc = sqxtun_list,
6546 .fno = gen_helper_sve2_sqxtunb_d,
6547 .vece = MO_64 },
6548 };
TRANS_FEAT(SQXTUNB,aa64_sve2,do_narrow_extract,a,sqxtunb_ops)6549 TRANS_FEAT(SQXTUNB, aa64_sve2, do_narrow_extract, a, sqxtunb_ops)
6550
6551 static void gen_sqxtunt_vec(unsigned vece, TCGv_vec d, TCGv_vec n)
6552 {
6553 int halfbits = 4 << vece;
6554 int64_t max = (1ull << halfbits) - 1;
6555 TCGv_vec maxv = tcg_constant_vec_matching(d, vece, max);
6556
6557 tcg_gen_smax_vec(vece, n, n, tcg_constant_vec_matching(d, vece, 0));
6558 tcg_gen_umin_vec(vece, n, n, maxv);
6559 tcg_gen_shli_vec(vece, n, n, halfbits);
6560 tcg_gen_bitsel_vec(vece, d, maxv, d, n);
6561 }
6562
6563 static const GVecGen2 sqxtunt_ops[3] = {
6564 { .fniv = gen_sqxtunt_vec,
6565 .opt_opc = sqxtun_list,
6566 .load_dest = true,
6567 .fno = gen_helper_sve2_sqxtunt_h,
6568 .vece = MO_16 },
6569 { .fniv = gen_sqxtunt_vec,
6570 .opt_opc = sqxtun_list,
6571 .load_dest = true,
6572 .fno = gen_helper_sve2_sqxtunt_s,
6573 .vece = MO_32 },
6574 { .fniv = gen_sqxtunt_vec,
6575 .opt_opc = sqxtun_list,
6576 .load_dest = true,
6577 .fno = gen_helper_sve2_sqxtunt_d,
6578 .vece = MO_64 },
6579 };
TRANS_FEAT(SQXTUNT,aa64_sve2,do_narrow_extract,a,sqxtunt_ops)6580 TRANS_FEAT(SQXTUNT, aa64_sve2, do_narrow_extract, a, sqxtunt_ops)
6581
6582 static bool do_shr_narrow(DisasContext *s, arg_rri_esz *a,
6583 const GVecGen2i ops[3])
6584 {
6585 if (a->esz < 0 || a->esz > MO_32) {
6586 return false;
6587 }
6588 assert(a->imm > 0 && a->imm <= (8 << a->esz));
6589 if (sve_access_check(s)) {
6590 unsigned vsz = vec_full_reg_size(s);
6591 tcg_gen_gvec_2i(vec_full_reg_offset(s, a->rd),
6592 vec_full_reg_offset(s, a->rn),
6593 vsz, vsz, a->imm, &ops[a->esz]);
6594 }
6595 return true;
6596 }
6597
gen_shrnb_i64(unsigned vece,TCGv_i64 d,TCGv_i64 n,int shr)6598 static void gen_shrnb_i64(unsigned vece, TCGv_i64 d, TCGv_i64 n, int shr)
6599 {
6600 int halfbits = 4 << vece;
6601 uint64_t mask = dup_const(vece, MAKE_64BIT_MASK(0, halfbits));
6602
6603 tcg_gen_shri_i64(d, n, shr);
6604 tcg_gen_andi_i64(d, d, mask);
6605 }
6606
gen_shrnb16_i64(TCGv_i64 d,TCGv_i64 n,int64_t shr)6607 static void gen_shrnb16_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr)
6608 {
6609 gen_shrnb_i64(MO_16, d, n, shr);
6610 }
6611
gen_shrnb32_i64(TCGv_i64 d,TCGv_i64 n,int64_t shr)6612 static void gen_shrnb32_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr)
6613 {
6614 gen_shrnb_i64(MO_32, d, n, shr);
6615 }
6616
gen_shrnb64_i64(TCGv_i64 d,TCGv_i64 n,int64_t shr)6617 static void gen_shrnb64_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr)
6618 {
6619 gen_shrnb_i64(MO_64, d, n, shr);
6620 }
6621
gen_shrnb_vec(unsigned vece,TCGv_vec d,TCGv_vec n,int64_t shr)6622 static void gen_shrnb_vec(unsigned vece, TCGv_vec d, TCGv_vec n, int64_t shr)
6623 {
6624 int halfbits = 4 << vece;
6625 uint64_t mask = MAKE_64BIT_MASK(0, halfbits);
6626
6627 tcg_gen_shri_vec(vece, n, n, shr);
6628 tcg_gen_and_vec(vece, d, n, tcg_constant_vec_matching(d, vece, mask));
6629 }
6630
6631 static const TCGOpcode shrnb_vec_list[] = { INDEX_op_shri_vec, 0 };
6632 static const GVecGen2i shrnb_ops[3] = {
6633 { .fni8 = gen_shrnb16_i64,
6634 .fniv = gen_shrnb_vec,
6635 .opt_opc = shrnb_vec_list,
6636 .fno = gen_helper_sve2_shrnb_h,
6637 .vece = MO_16 },
6638 { .fni8 = gen_shrnb32_i64,
6639 .fniv = gen_shrnb_vec,
6640 .opt_opc = shrnb_vec_list,
6641 .fno = gen_helper_sve2_shrnb_s,
6642 .vece = MO_32 },
6643 { .fni8 = gen_shrnb64_i64,
6644 .fniv = gen_shrnb_vec,
6645 .opt_opc = shrnb_vec_list,
6646 .fno = gen_helper_sve2_shrnb_d,
6647 .vece = MO_64 },
6648 };
TRANS_FEAT(SHRNB,aa64_sve2,do_shr_narrow,a,shrnb_ops)6649 TRANS_FEAT(SHRNB, aa64_sve2, do_shr_narrow, a, shrnb_ops)
6650
6651 static void gen_shrnt_i64(unsigned vece, TCGv_i64 d, TCGv_i64 n, int shr)
6652 {
6653 int halfbits = 4 << vece;
6654 uint64_t mask = dup_const(vece, MAKE_64BIT_MASK(0, halfbits));
6655
6656 tcg_gen_shli_i64(n, n, halfbits - shr);
6657 tcg_gen_andi_i64(n, n, ~mask);
6658 tcg_gen_andi_i64(d, d, mask);
6659 tcg_gen_or_i64(d, d, n);
6660 }
6661
gen_shrnt16_i64(TCGv_i64 d,TCGv_i64 n,int64_t shr)6662 static void gen_shrnt16_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr)
6663 {
6664 gen_shrnt_i64(MO_16, d, n, shr);
6665 }
6666
gen_shrnt32_i64(TCGv_i64 d,TCGv_i64 n,int64_t shr)6667 static void gen_shrnt32_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr)
6668 {
6669 gen_shrnt_i64(MO_32, d, n, shr);
6670 }
6671
gen_shrnt64_i64(TCGv_i64 d,TCGv_i64 n,int64_t shr)6672 static void gen_shrnt64_i64(TCGv_i64 d, TCGv_i64 n, int64_t shr)
6673 {
6674 tcg_gen_shri_i64(n, n, shr);
6675 tcg_gen_deposit_i64(d, d, n, 32, 32);
6676 }
6677
gen_shrnt_vec(unsigned vece,TCGv_vec d,TCGv_vec n,int64_t shr)6678 static void gen_shrnt_vec(unsigned vece, TCGv_vec d, TCGv_vec n, int64_t shr)
6679 {
6680 int halfbits = 4 << vece;
6681 uint64_t mask = MAKE_64BIT_MASK(0, halfbits);
6682
6683 tcg_gen_shli_vec(vece, n, n, halfbits - shr);
6684 tcg_gen_bitsel_vec(vece, d, tcg_constant_vec_matching(d, vece, mask), d, n);
6685 }
6686
6687 static const TCGOpcode shrnt_vec_list[] = { INDEX_op_shli_vec, 0 };
6688 static const GVecGen2i shrnt_ops[3] = {
6689 { .fni8 = gen_shrnt16_i64,
6690 .fniv = gen_shrnt_vec,
6691 .opt_opc = shrnt_vec_list,
6692 .load_dest = true,
6693 .fno = gen_helper_sve2_shrnt_h,
6694 .vece = MO_16 },
6695 { .fni8 = gen_shrnt32_i64,
6696 .fniv = gen_shrnt_vec,
6697 .opt_opc = shrnt_vec_list,
6698 .load_dest = true,
6699 .fno = gen_helper_sve2_shrnt_s,
6700 .vece = MO_32 },
6701 { .fni8 = gen_shrnt64_i64,
6702 .fniv = gen_shrnt_vec,
6703 .opt_opc = shrnt_vec_list,
6704 .load_dest = true,
6705 .fno = gen_helper_sve2_shrnt_d,
6706 .vece = MO_64 },
6707 };
6708 TRANS_FEAT(SHRNT, aa64_sve2, do_shr_narrow, a, shrnt_ops)
6709
6710 static const GVecGen2i rshrnb_ops[3] = {
6711 { .fno = gen_helper_sve2_rshrnb_h },
6712 { .fno = gen_helper_sve2_rshrnb_s },
6713 { .fno = gen_helper_sve2_rshrnb_d },
6714 };
6715 TRANS_FEAT(RSHRNB, aa64_sve2, do_shr_narrow, a, rshrnb_ops)
6716
6717 static const GVecGen2i rshrnt_ops[3] = {
6718 { .fno = gen_helper_sve2_rshrnt_h },
6719 { .fno = gen_helper_sve2_rshrnt_s },
6720 { .fno = gen_helper_sve2_rshrnt_d },
6721 };
TRANS_FEAT(RSHRNT,aa64_sve2,do_shr_narrow,a,rshrnt_ops)6722 TRANS_FEAT(RSHRNT, aa64_sve2, do_shr_narrow, a, rshrnt_ops)
6723
6724 static void gen_sqshrunb_vec(unsigned vece, TCGv_vec d,
6725 TCGv_vec n, int64_t shr)
6726 {
6727 int halfbits = 4 << vece;
6728 uint64_t max = MAKE_64BIT_MASK(0, halfbits);
6729
6730 tcg_gen_sari_vec(vece, n, n, shr);
6731 tcg_gen_smax_vec(vece, n, n, tcg_constant_vec_matching(d, vece, 0));
6732 tcg_gen_umin_vec(vece, d, n, tcg_constant_vec_matching(d, vece, max));
6733 }
6734
6735 static const TCGOpcode sqshrunb_vec_list[] = {
6736 INDEX_op_sari_vec, INDEX_op_smax_vec, INDEX_op_umin_vec, 0
6737 };
6738 static const GVecGen2i sqshrunb_ops[3] = {
6739 { .fniv = gen_sqshrunb_vec,
6740 .opt_opc = sqshrunb_vec_list,
6741 .fno = gen_helper_sve2_sqshrunb_h,
6742 .vece = MO_16 },
6743 { .fniv = gen_sqshrunb_vec,
6744 .opt_opc = sqshrunb_vec_list,
6745 .fno = gen_helper_sve2_sqshrunb_s,
6746 .vece = MO_32 },
6747 { .fniv = gen_sqshrunb_vec,
6748 .opt_opc = sqshrunb_vec_list,
6749 .fno = gen_helper_sve2_sqshrunb_d,
6750 .vece = MO_64 },
6751 };
TRANS_FEAT(SQSHRUNB,aa64_sve2,do_shr_narrow,a,sqshrunb_ops)6752 TRANS_FEAT(SQSHRUNB, aa64_sve2, do_shr_narrow, a, sqshrunb_ops)
6753
6754 static void gen_sqshrunt_vec(unsigned vece, TCGv_vec d,
6755 TCGv_vec n, int64_t shr)
6756 {
6757 int halfbits = 4 << vece;
6758 uint64_t max = MAKE_64BIT_MASK(0, halfbits);
6759 TCGv_vec maxv = tcg_constant_vec_matching(d, vece, max);
6760
6761 tcg_gen_sari_vec(vece, n, n, shr);
6762 tcg_gen_smax_vec(vece, n, n, tcg_constant_vec_matching(d, vece, 0));
6763 tcg_gen_umin_vec(vece, n, n, maxv);
6764 tcg_gen_shli_vec(vece, n, n, halfbits);
6765 tcg_gen_bitsel_vec(vece, d, maxv, d, n);
6766 }
6767
6768 static const TCGOpcode sqshrunt_vec_list[] = {
6769 INDEX_op_shli_vec, INDEX_op_sari_vec,
6770 INDEX_op_smax_vec, INDEX_op_umin_vec, 0
6771 };
6772 static const GVecGen2i sqshrunt_ops[3] = {
6773 { .fniv = gen_sqshrunt_vec,
6774 .opt_opc = sqshrunt_vec_list,
6775 .load_dest = true,
6776 .fno = gen_helper_sve2_sqshrunt_h,
6777 .vece = MO_16 },
6778 { .fniv = gen_sqshrunt_vec,
6779 .opt_opc = sqshrunt_vec_list,
6780 .load_dest = true,
6781 .fno = gen_helper_sve2_sqshrunt_s,
6782 .vece = MO_32 },
6783 { .fniv = gen_sqshrunt_vec,
6784 .opt_opc = sqshrunt_vec_list,
6785 .load_dest = true,
6786 .fno = gen_helper_sve2_sqshrunt_d,
6787 .vece = MO_64 },
6788 };
6789 TRANS_FEAT(SQSHRUNT, aa64_sve2, do_shr_narrow, a, sqshrunt_ops)
6790
6791 static const GVecGen2i sqrshrunb_ops[3] = {
6792 { .fno = gen_helper_sve2_sqrshrunb_h },
6793 { .fno = gen_helper_sve2_sqrshrunb_s },
6794 { .fno = gen_helper_sve2_sqrshrunb_d },
6795 };
6796 TRANS_FEAT(SQRSHRUNB, aa64_sve2, do_shr_narrow, a, sqrshrunb_ops)
6797
6798 static const GVecGen2i sqrshrunt_ops[3] = {
6799 { .fno = gen_helper_sve2_sqrshrunt_h },
6800 { .fno = gen_helper_sve2_sqrshrunt_s },
6801 { .fno = gen_helper_sve2_sqrshrunt_d },
6802 };
TRANS_FEAT(SQRSHRUNT,aa64_sve2,do_shr_narrow,a,sqrshrunt_ops)6803 TRANS_FEAT(SQRSHRUNT, aa64_sve2, do_shr_narrow, a, sqrshrunt_ops)
6804
6805 static void gen_sqshrnb_vec(unsigned vece, TCGv_vec d,
6806 TCGv_vec n, int64_t shr)
6807 {
6808 int halfbits = 4 << vece;
6809 int64_t max = MAKE_64BIT_MASK(0, halfbits - 1);
6810 int64_t min = -max - 1;
6811 int64_t mask = MAKE_64BIT_MASK(0, halfbits);
6812
6813 tcg_gen_sari_vec(vece, n, n, shr);
6814 tcg_gen_smax_vec(vece, n, n, tcg_constant_vec_matching(d, vece, min));
6815 tcg_gen_smin_vec(vece, n, n, tcg_constant_vec_matching(d, vece, max));
6816 tcg_gen_and_vec(vece, d, n, tcg_constant_vec_matching(d, vece, mask));
6817 }
6818
6819 static const TCGOpcode sqshrnb_vec_list[] = {
6820 INDEX_op_sari_vec, INDEX_op_smax_vec, INDEX_op_smin_vec, 0
6821 };
6822 static const GVecGen2i sqshrnb_ops[3] = {
6823 { .fniv = gen_sqshrnb_vec,
6824 .opt_opc = sqshrnb_vec_list,
6825 .fno = gen_helper_sve2_sqshrnb_h,
6826 .vece = MO_16 },
6827 { .fniv = gen_sqshrnb_vec,
6828 .opt_opc = sqshrnb_vec_list,
6829 .fno = gen_helper_sve2_sqshrnb_s,
6830 .vece = MO_32 },
6831 { .fniv = gen_sqshrnb_vec,
6832 .opt_opc = sqshrnb_vec_list,
6833 .fno = gen_helper_sve2_sqshrnb_d,
6834 .vece = MO_64 },
6835 };
TRANS_FEAT(SQSHRNB,aa64_sve2,do_shr_narrow,a,sqshrnb_ops)6836 TRANS_FEAT(SQSHRNB, aa64_sve2, do_shr_narrow, a, sqshrnb_ops)
6837
6838 static void gen_sqshrnt_vec(unsigned vece, TCGv_vec d,
6839 TCGv_vec n, int64_t shr)
6840 {
6841 int halfbits = 4 << vece;
6842 int64_t max = MAKE_64BIT_MASK(0, halfbits - 1);
6843 int64_t min = -max - 1;
6844 int64_t mask = MAKE_64BIT_MASK(0, halfbits);
6845
6846 tcg_gen_sari_vec(vece, n, n, shr);
6847 tcg_gen_smax_vec(vece, n, n, tcg_constant_vec_matching(d, vece, min));
6848 tcg_gen_smin_vec(vece, n, n, tcg_constant_vec_matching(d, vece, max));
6849 tcg_gen_shli_vec(vece, n, n, halfbits);
6850 tcg_gen_bitsel_vec(vece, d, tcg_constant_vec_matching(d, vece, mask), d, n);
6851 }
6852
6853 static const TCGOpcode sqshrnt_vec_list[] = {
6854 INDEX_op_shli_vec, INDEX_op_sari_vec,
6855 INDEX_op_smax_vec, INDEX_op_smin_vec, 0
6856 };
6857 static const GVecGen2i sqshrnt_ops[3] = {
6858 { .fniv = gen_sqshrnt_vec,
6859 .opt_opc = sqshrnt_vec_list,
6860 .load_dest = true,
6861 .fno = gen_helper_sve2_sqshrnt_h,
6862 .vece = MO_16 },
6863 { .fniv = gen_sqshrnt_vec,
6864 .opt_opc = sqshrnt_vec_list,
6865 .load_dest = true,
6866 .fno = gen_helper_sve2_sqshrnt_s,
6867 .vece = MO_32 },
6868 { .fniv = gen_sqshrnt_vec,
6869 .opt_opc = sqshrnt_vec_list,
6870 .load_dest = true,
6871 .fno = gen_helper_sve2_sqshrnt_d,
6872 .vece = MO_64 },
6873 };
6874 TRANS_FEAT(SQSHRNT, aa64_sve2, do_shr_narrow, a, sqshrnt_ops)
6875
6876 static const GVecGen2i sqrshrnb_ops[3] = {
6877 { .fno = gen_helper_sve2_sqrshrnb_h },
6878 { .fno = gen_helper_sve2_sqrshrnb_s },
6879 { .fno = gen_helper_sve2_sqrshrnb_d },
6880 };
6881 TRANS_FEAT(SQRSHRNB, aa64_sve2, do_shr_narrow, a, sqrshrnb_ops)
6882
6883 static const GVecGen2i sqrshrnt_ops[3] = {
6884 { .fno = gen_helper_sve2_sqrshrnt_h },
6885 { .fno = gen_helper_sve2_sqrshrnt_s },
6886 { .fno = gen_helper_sve2_sqrshrnt_d },
6887 };
TRANS_FEAT(SQRSHRNT,aa64_sve2,do_shr_narrow,a,sqrshrnt_ops)6888 TRANS_FEAT(SQRSHRNT, aa64_sve2, do_shr_narrow, a, sqrshrnt_ops)
6889
6890 static void gen_uqshrnb_vec(unsigned vece, TCGv_vec d,
6891 TCGv_vec n, int64_t shr)
6892 {
6893 int halfbits = 4 << vece;
6894 int64_t max = MAKE_64BIT_MASK(0, halfbits);
6895
6896 tcg_gen_shri_vec(vece, n, n, shr);
6897 tcg_gen_umin_vec(vece, d, n, tcg_constant_vec_matching(d, vece, max));
6898 }
6899
6900 static const TCGOpcode uqshrnb_vec_list[] = {
6901 INDEX_op_shri_vec, INDEX_op_umin_vec, 0
6902 };
6903 static const GVecGen2i uqshrnb_ops[3] = {
6904 { .fniv = gen_uqshrnb_vec,
6905 .opt_opc = uqshrnb_vec_list,
6906 .fno = gen_helper_sve2_uqshrnb_h,
6907 .vece = MO_16 },
6908 { .fniv = gen_uqshrnb_vec,
6909 .opt_opc = uqshrnb_vec_list,
6910 .fno = gen_helper_sve2_uqshrnb_s,
6911 .vece = MO_32 },
6912 { .fniv = gen_uqshrnb_vec,
6913 .opt_opc = uqshrnb_vec_list,
6914 .fno = gen_helper_sve2_uqshrnb_d,
6915 .vece = MO_64 },
6916 };
TRANS_FEAT(UQSHRNB,aa64_sve2,do_shr_narrow,a,uqshrnb_ops)6917 TRANS_FEAT(UQSHRNB, aa64_sve2, do_shr_narrow, a, uqshrnb_ops)
6918
6919 static void gen_uqshrnt_vec(unsigned vece, TCGv_vec d,
6920 TCGv_vec n, int64_t shr)
6921 {
6922 int halfbits = 4 << vece;
6923 int64_t max = MAKE_64BIT_MASK(0, halfbits);
6924 TCGv_vec maxv = tcg_constant_vec_matching(d, vece, max);
6925
6926 tcg_gen_shri_vec(vece, n, n, shr);
6927 tcg_gen_umin_vec(vece, n, n, maxv);
6928 tcg_gen_shli_vec(vece, n, n, halfbits);
6929 tcg_gen_bitsel_vec(vece, d, maxv, d, n);
6930 }
6931
6932 static const TCGOpcode uqshrnt_vec_list[] = {
6933 INDEX_op_shli_vec, INDEX_op_shri_vec, INDEX_op_umin_vec, 0
6934 };
6935 static const GVecGen2i uqshrnt_ops[3] = {
6936 { .fniv = gen_uqshrnt_vec,
6937 .opt_opc = uqshrnt_vec_list,
6938 .load_dest = true,
6939 .fno = gen_helper_sve2_uqshrnt_h,
6940 .vece = MO_16 },
6941 { .fniv = gen_uqshrnt_vec,
6942 .opt_opc = uqshrnt_vec_list,
6943 .load_dest = true,
6944 .fno = gen_helper_sve2_uqshrnt_s,
6945 .vece = MO_32 },
6946 { .fniv = gen_uqshrnt_vec,
6947 .opt_opc = uqshrnt_vec_list,
6948 .load_dest = true,
6949 .fno = gen_helper_sve2_uqshrnt_d,
6950 .vece = MO_64 },
6951 };
6952 TRANS_FEAT(UQSHRNT, aa64_sve2, do_shr_narrow, a, uqshrnt_ops)
6953
6954 static const GVecGen2i uqrshrnb_ops[3] = {
6955 { .fno = gen_helper_sve2_uqrshrnb_h },
6956 { .fno = gen_helper_sve2_uqrshrnb_s },
6957 { .fno = gen_helper_sve2_uqrshrnb_d },
6958 };
6959 TRANS_FEAT(UQRSHRNB, aa64_sve2, do_shr_narrow, a, uqrshrnb_ops)
6960
6961 static const GVecGen2i uqrshrnt_ops[3] = {
6962 { .fno = gen_helper_sve2_uqrshrnt_h },
6963 { .fno = gen_helper_sve2_uqrshrnt_s },
6964 { .fno = gen_helper_sve2_uqrshrnt_d },
6965 };
6966 TRANS_FEAT(UQRSHRNT, aa64_sve2, do_shr_narrow, a, uqrshrnt_ops)
6967
6968 #define DO_SVE2_ZZZ_NARROW(NAME, name) \
6969 static gen_helper_gvec_3 * const name##_fns[4] = { \
6970 NULL, gen_helper_sve2_##name##_h, \
6971 gen_helper_sve2_##name##_s, gen_helper_sve2_##name##_d, \
6972 }; \
6973 TRANS_FEAT(NAME, aa64_sve2, gen_gvec_ool_arg_zzz, \
6974 name##_fns[a->esz], a, 0)
6975
6976 DO_SVE2_ZZZ_NARROW(ADDHNB, addhnb)
6977 DO_SVE2_ZZZ_NARROW(ADDHNT, addhnt)
6978 DO_SVE2_ZZZ_NARROW(RADDHNB, raddhnb)
6979 DO_SVE2_ZZZ_NARROW(RADDHNT, raddhnt)
6980
6981 DO_SVE2_ZZZ_NARROW(SUBHNB, subhnb)
6982 DO_SVE2_ZZZ_NARROW(SUBHNT, subhnt)
6983 DO_SVE2_ZZZ_NARROW(RSUBHNB, rsubhnb)
6984 DO_SVE2_ZZZ_NARROW(RSUBHNT, rsubhnt)
6985
6986 static gen_helper_gvec_flags_4 * const match_fns[4] = {
6987 gen_helper_sve2_match_ppzz_b, gen_helper_sve2_match_ppzz_h, NULL, NULL
6988 };
6989 TRANS_FEAT_NONSTREAMING(MATCH, aa64_sve2, do_ppzz_flags, a, match_fns[a->esz])
6990
6991 static gen_helper_gvec_flags_4 * const nmatch_fns[4] = {
6992 gen_helper_sve2_nmatch_ppzz_b, gen_helper_sve2_nmatch_ppzz_h, NULL, NULL
6993 };
6994 TRANS_FEAT_NONSTREAMING(NMATCH, aa64_sve2, do_ppzz_flags, a, nmatch_fns[a->esz])
6995
6996 static gen_helper_gvec_4 * const histcnt_fns[4] = {
6997 NULL, NULL, gen_helper_sve2_histcnt_s, gen_helper_sve2_histcnt_d
6998 };
6999 TRANS_FEAT_NONSTREAMING(HISTCNT, aa64_sve2, gen_gvec_ool_arg_zpzz,
7000 histcnt_fns[a->esz], a, 0)
7001
7002 TRANS_FEAT_NONSTREAMING(HISTSEG, aa64_sve2, gen_gvec_ool_arg_zzz,
7003 a->esz == 0 ? gen_helper_sve2_histseg : NULL, a, 0)
7004
7005 DO_ZPZZ_FP(FADDP, aa64_sve2, sve2_faddp_zpzz)
7006 DO_ZPZZ_FP(FMAXNMP, aa64_sve2, sve2_fmaxnmp_zpzz)
7007 DO_ZPZZ_FP(FMINNMP, aa64_sve2, sve2_fminnmp_zpzz)
7008 DO_ZPZZ_FP(FMAXP, aa64_sve2, sve2_fmaxp_zpzz)
7009 DO_ZPZZ_FP(FMINP, aa64_sve2, sve2_fminp_zpzz)
7010
7011 /*
7012 * SVE Integer Multiply-Add (unpredicated)
7013 */
7014
7015 TRANS_FEAT_NONSTREAMING(FMMLA_s, aa64_sve_f32mm, gen_gvec_fpst_zzzz,
7016 gen_helper_fmmla_s, a->rd, a->rn, a->rm, a->ra,
7017 0, FPST_A64)
7018 TRANS_FEAT_NONSTREAMING(FMMLA_d, aa64_sve_f64mm, gen_gvec_fpst_zzzz,
7019 gen_helper_fmmla_d, a->rd, a->rn, a->rm, a->ra,
7020 0, FPST_A64)
7021
7022 static gen_helper_gvec_4 * const sqdmlal_zzzw_fns[] = {
7023 NULL, gen_helper_sve2_sqdmlal_zzzw_h,
7024 gen_helper_sve2_sqdmlal_zzzw_s, gen_helper_sve2_sqdmlal_zzzw_d,
7025 };
7026 TRANS_FEAT(SQDMLALB_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz,
7027 sqdmlal_zzzw_fns[a->esz], a, 0)
7028 TRANS_FEAT(SQDMLALT_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz,
7029 sqdmlal_zzzw_fns[a->esz], a, 3)
7030 TRANS_FEAT(SQDMLALBT, aa64_sve2, gen_gvec_ool_arg_zzzz,
7031 sqdmlal_zzzw_fns[a->esz], a, 2)
7032
7033 static gen_helper_gvec_4 * const sqdmlsl_zzzw_fns[] = {
7034 NULL, gen_helper_sve2_sqdmlsl_zzzw_h,
7035 gen_helper_sve2_sqdmlsl_zzzw_s, gen_helper_sve2_sqdmlsl_zzzw_d,
7036 };
7037 TRANS_FEAT(SQDMLSLB_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz,
7038 sqdmlsl_zzzw_fns[a->esz], a, 0)
7039 TRANS_FEAT(SQDMLSLT_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz,
7040 sqdmlsl_zzzw_fns[a->esz], a, 3)
7041 TRANS_FEAT(SQDMLSLBT, aa64_sve2, gen_gvec_ool_arg_zzzz,
7042 sqdmlsl_zzzw_fns[a->esz], a, 2)
7043
7044 static gen_helper_gvec_4 * const sqrdmlah_fns[] = {
7045 gen_helper_sve2_sqrdmlah_b, gen_helper_sve2_sqrdmlah_h,
7046 gen_helper_sve2_sqrdmlah_s, gen_helper_sve2_sqrdmlah_d,
7047 };
7048 TRANS_FEAT(SQRDMLAH_zzzz, aa64_sve2, gen_gvec_ool_arg_zzzz,
7049 sqrdmlah_fns[a->esz], a, 0)
7050
7051 static gen_helper_gvec_4 * const sqrdmlsh_fns[] = {
7052 gen_helper_sve2_sqrdmlsh_b, gen_helper_sve2_sqrdmlsh_h,
7053 gen_helper_sve2_sqrdmlsh_s, gen_helper_sve2_sqrdmlsh_d,
7054 };
7055 TRANS_FEAT(SQRDMLSH_zzzz, aa64_sve2, gen_gvec_ool_arg_zzzz,
7056 sqrdmlsh_fns[a->esz], a, 0)
7057
7058 static gen_helper_gvec_4 * const smlal_zzzw_fns[] = {
7059 NULL, gen_helper_sve2_smlal_zzzw_h,
7060 gen_helper_sve2_smlal_zzzw_s, gen_helper_sve2_smlal_zzzw_d,
7061 };
7062 TRANS_FEAT(SMLALB_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz,
7063 smlal_zzzw_fns[a->esz], a, 0)
7064 TRANS_FEAT(SMLALT_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz,
7065 smlal_zzzw_fns[a->esz], a, 1)
7066
7067 static gen_helper_gvec_4 * const umlal_zzzw_fns[] = {
7068 NULL, gen_helper_sve2_umlal_zzzw_h,
7069 gen_helper_sve2_umlal_zzzw_s, gen_helper_sve2_umlal_zzzw_d,
7070 };
7071 TRANS_FEAT(UMLALB_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz,
7072 umlal_zzzw_fns[a->esz], a, 0)
7073 TRANS_FEAT(UMLALT_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz,
7074 umlal_zzzw_fns[a->esz], a, 1)
7075
7076 static gen_helper_gvec_4 * const smlsl_zzzw_fns[] = {
7077 NULL, gen_helper_sve2_smlsl_zzzw_h,
7078 gen_helper_sve2_smlsl_zzzw_s, gen_helper_sve2_smlsl_zzzw_d,
7079 };
7080 TRANS_FEAT(SMLSLB_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz,
7081 smlsl_zzzw_fns[a->esz], a, 0)
7082 TRANS_FEAT(SMLSLT_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz,
7083 smlsl_zzzw_fns[a->esz], a, 1)
7084
7085 static gen_helper_gvec_4 * const umlsl_zzzw_fns[] = {
7086 NULL, gen_helper_sve2_umlsl_zzzw_h,
7087 gen_helper_sve2_umlsl_zzzw_s, gen_helper_sve2_umlsl_zzzw_d,
7088 };
7089 TRANS_FEAT(UMLSLB_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz,
7090 umlsl_zzzw_fns[a->esz], a, 0)
7091 TRANS_FEAT(UMLSLT_zzzw, aa64_sve2, gen_gvec_ool_arg_zzzz,
7092 umlsl_zzzw_fns[a->esz], a, 1)
7093
7094 static gen_helper_gvec_4 * const cmla_fns[] = {
7095 gen_helper_sve2_cmla_zzzz_b, gen_helper_sve2_cmla_zzzz_h,
7096 gen_helper_sve2_cmla_zzzz_s, gen_helper_sve2_cmla_zzzz_d,
7097 };
7098 TRANS_FEAT(CMLA_zzzz, aa64_sve2, gen_gvec_ool_zzzz,
7099 cmla_fns[a->esz], a->rd, a->rn, a->rm, a->ra, a->rot)
7100
7101 static gen_helper_gvec_4 * const cdot_fns[] = {
7102 NULL, NULL, gen_helper_sve2_cdot_zzzz_s, gen_helper_sve2_cdot_zzzz_d
7103 };
7104 TRANS_FEAT(CDOT_zzzz, aa64_sve2, gen_gvec_ool_zzzz,
7105 cdot_fns[a->esz], a->rd, a->rn, a->rm, a->ra, a->rot)
7106
7107 static gen_helper_gvec_4 * const sqrdcmlah_fns[] = {
7108 gen_helper_sve2_sqrdcmlah_zzzz_b, gen_helper_sve2_sqrdcmlah_zzzz_h,
7109 gen_helper_sve2_sqrdcmlah_zzzz_s, gen_helper_sve2_sqrdcmlah_zzzz_d,
7110 };
7111 TRANS_FEAT(SQRDCMLAH_zzzz, aa64_sve2, gen_gvec_ool_zzzz,
7112 sqrdcmlah_fns[a->esz], a->rd, a->rn, a->rm, a->ra, a->rot)
7113
7114 TRANS_FEAT(USDOT_zzzz, aa64_sve_i8mm, gen_gvec_ool_arg_zzzz,
7115 a->esz == 2 ? gen_helper_gvec_usdot_b : NULL, a, 0)
7116
7117 TRANS_FEAT_NONSTREAMING(AESMC, aa64_sve2_aes, gen_gvec_ool_zz,
7118 gen_helper_crypto_aesmc, a->rd, a->rd, 0)
7119 TRANS_FEAT_NONSTREAMING(AESIMC, aa64_sve2_aes, gen_gvec_ool_zz,
7120 gen_helper_crypto_aesimc, a->rd, a->rd, 0)
7121
7122 TRANS_FEAT_NONSTREAMING(AESE, aa64_sve2_aes, gen_gvec_ool_arg_zzz,
7123 gen_helper_crypto_aese, a, 0)
7124 TRANS_FEAT_NONSTREAMING(AESD, aa64_sve2_aes, gen_gvec_ool_arg_zzz,
7125 gen_helper_crypto_aesd, a, 0)
7126
7127 TRANS_FEAT_NONSTREAMING(SM4E, aa64_sve2_sm4, gen_gvec_ool_arg_zzz,
7128 gen_helper_crypto_sm4e, a, 0)
7129 TRANS_FEAT_NONSTREAMING(SM4EKEY, aa64_sve2_sm4, gen_gvec_ool_arg_zzz,
7130 gen_helper_crypto_sm4ekey, a, 0)
7131
7132 TRANS_FEAT_NONSTREAMING(RAX1, aa64_sve2_sha3, gen_gvec_fn_arg_zzz,
7133 gen_gvec_rax1, a)
7134
7135 TRANS_FEAT(FCVTNT_sh, aa64_sve2, gen_gvec_fpst_arg_zpz,
7136 gen_helper_sve2_fcvtnt_sh, a, 0, FPST_A64)
7137 TRANS_FEAT(FCVTNT_ds, aa64_sve2, gen_gvec_fpst_arg_zpz,
7138 gen_helper_sve2_fcvtnt_ds, a, 0, FPST_A64)
7139
7140 TRANS_FEAT(BFCVTNT, aa64_sve_bf16, gen_gvec_fpst_arg_zpz,
7141 gen_helper_sve_bfcvtnt, a, 0,
7142 s->fpcr_ah ? FPST_AH : FPST_A64)
7143
7144 TRANS_FEAT(FCVTLT_hs, aa64_sve2, gen_gvec_fpst_arg_zpz,
7145 gen_helper_sve2_fcvtlt_hs, a, 0, FPST_A64)
7146 TRANS_FEAT(FCVTLT_sd, aa64_sve2, gen_gvec_fpst_arg_zpz,
7147 gen_helper_sve2_fcvtlt_sd, a, 0, FPST_A64)
7148
7149 TRANS_FEAT(FCVTX_ds, aa64_sve2, do_frint_mode, a,
7150 FPROUNDING_ODD, gen_helper_sve_fcvt_ds)
7151 TRANS_FEAT(FCVTXNT_ds, aa64_sve2, do_frint_mode, a,
7152 FPROUNDING_ODD, gen_helper_sve2_fcvtnt_ds)
7153
7154 static gen_helper_gvec_3_ptr * const flogb_fns[] = {
7155 NULL, gen_helper_flogb_h,
7156 gen_helper_flogb_s, gen_helper_flogb_d
7157 };
7158 TRANS_FEAT(FLOGB, aa64_sve2, gen_gvec_fpst_arg_zpz, flogb_fns[a->esz],
7159 a, 0, a->esz == MO_16 ? FPST_A64_F16 : FPST_A64)
7160
do_FMLAL_zzzw(DisasContext * s,arg_rrrr_esz * a,bool sub,bool sel)7161 static bool do_FMLAL_zzzw(DisasContext *s, arg_rrrr_esz *a, bool sub, bool sel)
7162 {
7163 return gen_gvec_ptr_zzzz(s, gen_helper_sve2_fmlal_zzzw_s,
7164 a->rd, a->rn, a->rm, a->ra,
7165 (sel << 1) | sub, tcg_env);
7166 }
7167
TRANS_FEAT(FMLALB_zzzw,aa64_sve2,do_FMLAL_zzzw,a,false,false)7168 TRANS_FEAT(FMLALB_zzzw, aa64_sve2, do_FMLAL_zzzw, a, false, false)
7169 TRANS_FEAT(FMLALT_zzzw, aa64_sve2, do_FMLAL_zzzw, a, false, true)
7170 TRANS_FEAT(FMLSLB_zzzw, aa64_sve2, do_FMLAL_zzzw, a, true, false)
7171 TRANS_FEAT(FMLSLT_zzzw, aa64_sve2, do_FMLAL_zzzw, a, true, true)
7172
7173 static bool do_FMLAL_zzxw(DisasContext *s, arg_rrxr_esz *a, bool sub, bool sel)
7174 {
7175 return gen_gvec_ptr_zzzz(s, gen_helper_sve2_fmlal_zzxw_s,
7176 a->rd, a->rn, a->rm, a->ra,
7177 (a->index << 2) | (sel << 1) | sub, tcg_env);
7178 }
7179
TRANS_FEAT(FMLALB_zzxw,aa64_sve2,do_FMLAL_zzxw,a,false,false)7180 TRANS_FEAT(FMLALB_zzxw, aa64_sve2, do_FMLAL_zzxw, a, false, false)
7181 TRANS_FEAT(FMLALT_zzxw, aa64_sve2, do_FMLAL_zzxw, a, false, true)
7182 TRANS_FEAT(FMLSLB_zzxw, aa64_sve2, do_FMLAL_zzxw, a, true, false)
7183 TRANS_FEAT(FMLSLT_zzxw, aa64_sve2, do_FMLAL_zzxw, a, true, true)
7184
7185 TRANS_FEAT_NONSTREAMING(SMMLA, aa64_sve_i8mm, gen_gvec_ool_arg_zzzz,
7186 gen_helper_gvec_smmla_b, a, 0)
7187 TRANS_FEAT_NONSTREAMING(USMMLA, aa64_sve_i8mm, gen_gvec_ool_arg_zzzz,
7188 gen_helper_gvec_usmmla_b, a, 0)
7189 TRANS_FEAT_NONSTREAMING(UMMLA, aa64_sve_i8mm, gen_gvec_ool_arg_zzzz,
7190 gen_helper_gvec_ummla_b, a, 0)
7191
7192 TRANS_FEAT(BFDOT_zzzz, aa64_sve_bf16, gen_gvec_env_arg_zzzz,
7193 gen_helper_gvec_bfdot, a, 0)
7194 TRANS_FEAT(BFDOT_zzxz, aa64_sve_bf16, gen_gvec_env_arg_zzxz,
7195 gen_helper_gvec_bfdot_idx, a)
7196
7197 TRANS_FEAT_NONSTREAMING(BFMMLA, aa64_sve_bf16, gen_gvec_env_arg_zzzz,
7198 gen_helper_gvec_bfmmla, a, 0)
7199
7200 static bool do_BFMLAL_zzzw(DisasContext *s, arg_rrrr_esz *a, bool sel)
7201 {
7202 return gen_gvec_fpst_zzzz(s, gen_helper_gvec_bfmlal,
7203 a->rd, a->rn, a->rm, a->ra, sel,
7204 s->fpcr_ah ? FPST_AH : FPST_A64);
7205 }
7206
TRANS_FEAT(BFMLALB_zzzw,aa64_sve_bf16,do_BFMLAL_zzzw,a,false)7207 TRANS_FEAT(BFMLALB_zzzw, aa64_sve_bf16, do_BFMLAL_zzzw, a, false)
7208 TRANS_FEAT(BFMLALT_zzzw, aa64_sve_bf16, do_BFMLAL_zzzw, a, true)
7209
7210 static bool do_BFMLAL_zzxw(DisasContext *s, arg_rrxr_esz *a, bool sel)
7211 {
7212 return gen_gvec_fpst_zzzz(s, gen_helper_gvec_bfmlal_idx,
7213 a->rd, a->rn, a->rm, a->ra,
7214 (a->index << 1) | sel,
7215 s->fpcr_ah ? FPST_AH : FPST_A64);
7216 }
7217
TRANS_FEAT(BFMLALB_zzxw,aa64_sve_bf16,do_BFMLAL_zzxw,a,false)7218 TRANS_FEAT(BFMLALB_zzxw, aa64_sve_bf16, do_BFMLAL_zzxw, a, false)
7219 TRANS_FEAT(BFMLALT_zzxw, aa64_sve_bf16, do_BFMLAL_zzxw, a, true)
7220
7221 static bool trans_PSEL(DisasContext *s, arg_psel *a)
7222 {
7223 int vl = vec_full_reg_size(s);
7224 int pl = pred_gvec_reg_size(s);
7225 int elements = vl >> a->esz;
7226 TCGv_i64 tmp, didx, dbit;
7227 TCGv_ptr ptr;
7228
7229 if (!dc_isar_feature(aa64_sme, s)) {
7230 return false;
7231 }
7232 if (!sve_access_check(s)) {
7233 return true;
7234 }
7235
7236 tmp = tcg_temp_new_i64();
7237 dbit = tcg_temp_new_i64();
7238 didx = tcg_temp_new_i64();
7239 ptr = tcg_temp_new_ptr();
7240
7241 /* Compute the predicate element. */
7242 tcg_gen_addi_i64(tmp, cpu_reg(s, a->rv), a->imm);
7243 if (is_power_of_2(elements)) {
7244 tcg_gen_andi_i64(tmp, tmp, elements - 1);
7245 } else {
7246 tcg_gen_remu_i64(tmp, tmp, tcg_constant_i64(elements));
7247 }
7248
7249 /* Extract the predicate byte and bit indices. */
7250 tcg_gen_shli_i64(tmp, tmp, a->esz);
7251 tcg_gen_andi_i64(dbit, tmp, 7);
7252 tcg_gen_shri_i64(didx, tmp, 3);
7253 if (HOST_BIG_ENDIAN) {
7254 tcg_gen_xori_i64(didx, didx, 7);
7255 }
7256
7257 /* Load the predicate word. */
7258 tcg_gen_trunc_i64_ptr(ptr, didx);
7259 tcg_gen_add_ptr(ptr, ptr, tcg_env);
7260 tcg_gen_ld8u_i64(tmp, ptr, pred_full_reg_offset(s, a->pm));
7261
7262 /* Extract the predicate bit and replicate to MO_64. */
7263 tcg_gen_shr_i64(tmp, tmp, dbit);
7264 tcg_gen_andi_i64(tmp, tmp, 1);
7265 tcg_gen_neg_i64(tmp, tmp);
7266
7267 /* Apply to either copy the source, or write zeros. */
7268 tcg_gen_gvec_ands(MO_64, pred_full_reg_offset(s, a->pd),
7269 pred_full_reg_offset(s, a->pn), tmp, pl, pl);
7270 return true;
7271 }
7272
gen_sclamp_i32(TCGv_i32 d,TCGv_i32 n,TCGv_i32 m,TCGv_i32 a)7273 static void gen_sclamp_i32(TCGv_i32 d, TCGv_i32 n, TCGv_i32 m, TCGv_i32 a)
7274 {
7275 tcg_gen_smax_i32(d, a, n);
7276 tcg_gen_smin_i32(d, d, m);
7277 }
7278
gen_sclamp_i64(TCGv_i64 d,TCGv_i64 n,TCGv_i64 m,TCGv_i64 a)7279 static void gen_sclamp_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 a)
7280 {
7281 tcg_gen_smax_i64(d, a, n);
7282 tcg_gen_smin_i64(d, d, m);
7283 }
7284
gen_sclamp_vec(unsigned vece,TCGv_vec d,TCGv_vec n,TCGv_vec m,TCGv_vec a)7285 static void gen_sclamp_vec(unsigned vece, TCGv_vec d, TCGv_vec n,
7286 TCGv_vec m, TCGv_vec a)
7287 {
7288 tcg_gen_smax_vec(vece, d, a, n);
7289 tcg_gen_smin_vec(vece, d, d, m);
7290 }
7291
gen_sclamp(unsigned vece,uint32_t d,uint32_t n,uint32_t m,uint32_t a,uint32_t oprsz,uint32_t maxsz)7292 static void gen_sclamp(unsigned vece, uint32_t d, uint32_t n, uint32_t m,
7293 uint32_t a, uint32_t oprsz, uint32_t maxsz)
7294 {
7295 static const TCGOpcode vecop[] = {
7296 INDEX_op_smin_vec, INDEX_op_smax_vec, 0
7297 };
7298 static const GVecGen4 ops[4] = {
7299 { .fniv = gen_sclamp_vec,
7300 .fno = gen_helper_gvec_sclamp_b,
7301 .opt_opc = vecop,
7302 .vece = MO_8 },
7303 { .fniv = gen_sclamp_vec,
7304 .fno = gen_helper_gvec_sclamp_h,
7305 .opt_opc = vecop,
7306 .vece = MO_16 },
7307 { .fni4 = gen_sclamp_i32,
7308 .fniv = gen_sclamp_vec,
7309 .fno = gen_helper_gvec_sclamp_s,
7310 .opt_opc = vecop,
7311 .vece = MO_32 },
7312 { .fni8 = gen_sclamp_i64,
7313 .fniv = gen_sclamp_vec,
7314 .fno = gen_helper_gvec_sclamp_d,
7315 .opt_opc = vecop,
7316 .vece = MO_64,
7317 .prefer_i64 = TCG_TARGET_REG_BITS == 64 }
7318 };
7319 tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &ops[vece]);
7320 }
7321
TRANS_FEAT(SCLAMP,aa64_sme,gen_gvec_fn_arg_zzzz,gen_sclamp,a)7322 TRANS_FEAT(SCLAMP, aa64_sme, gen_gvec_fn_arg_zzzz, gen_sclamp, a)
7323
7324 static void gen_uclamp_i32(TCGv_i32 d, TCGv_i32 n, TCGv_i32 m, TCGv_i32 a)
7325 {
7326 tcg_gen_umax_i32(d, a, n);
7327 tcg_gen_umin_i32(d, d, m);
7328 }
7329
gen_uclamp_i64(TCGv_i64 d,TCGv_i64 n,TCGv_i64 m,TCGv_i64 a)7330 static void gen_uclamp_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_i64 a)
7331 {
7332 tcg_gen_umax_i64(d, a, n);
7333 tcg_gen_umin_i64(d, d, m);
7334 }
7335
gen_uclamp_vec(unsigned vece,TCGv_vec d,TCGv_vec n,TCGv_vec m,TCGv_vec a)7336 static void gen_uclamp_vec(unsigned vece, TCGv_vec d, TCGv_vec n,
7337 TCGv_vec m, TCGv_vec a)
7338 {
7339 tcg_gen_umax_vec(vece, d, a, n);
7340 tcg_gen_umin_vec(vece, d, d, m);
7341 }
7342
gen_uclamp(unsigned vece,uint32_t d,uint32_t n,uint32_t m,uint32_t a,uint32_t oprsz,uint32_t maxsz)7343 static void gen_uclamp(unsigned vece, uint32_t d, uint32_t n, uint32_t m,
7344 uint32_t a, uint32_t oprsz, uint32_t maxsz)
7345 {
7346 static const TCGOpcode vecop[] = {
7347 INDEX_op_umin_vec, INDEX_op_umax_vec, 0
7348 };
7349 static const GVecGen4 ops[4] = {
7350 { .fniv = gen_uclamp_vec,
7351 .fno = gen_helper_gvec_uclamp_b,
7352 .opt_opc = vecop,
7353 .vece = MO_8 },
7354 { .fniv = gen_uclamp_vec,
7355 .fno = gen_helper_gvec_uclamp_h,
7356 .opt_opc = vecop,
7357 .vece = MO_16 },
7358 { .fni4 = gen_uclamp_i32,
7359 .fniv = gen_uclamp_vec,
7360 .fno = gen_helper_gvec_uclamp_s,
7361 .opt_opc = vecop,
7362 .vece = MO_32 },
7363 { .fni8 = gen_uclamp_i64,
7364 .fniv = gen_uclamp_vec,
7365 .fno = gen_helper_gvec_uclamp_d,
7366 .opt_opc = vecop,
7367 .vece = MO_64,
7368 .prefer_i64 = TCG_TARGET_REG_BITS == 64 }
7369 };
7370 tcg_gen_gvec_4(d, n, m, a, oprsz, maxsz, &ops[vece]);
7371 }
7372
7373 TRANS_FEAT(UCLAMP, aa64_sme, gen_gvec_fn_arg_zzzz, gen_uclamp, a)
7374