1 /*
2 * AArch64 translation
3 *
4 * Copyright (c) 2013 Alexander Graf <agraf@suse.de>
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19 #include "qemu/osdep.h"
20 #include "exec/target_page.h"
21 #include "translate.h"
22 #include "translate-a64.h"
23 #include "qemu/log.h"
24 #include "arm_ldst.h"
25 #include "semihosting/semihost.h"
26 #include "cpregs.h"
27
28 static TCGv_i64 cpu_X[32];
29 static TCGv_i64 cpu_pc;
30
31 /* Load/store exclusive handling */
32 static TCGv_i64 cpu_exclusive_high;
33
34 static const char *regnames[] = {
35 "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7",
36 "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15",
37 "x16", "x17", "x18", "x19", "x20", "x21", "x22", "x23",
38 "x24", "x25", "x26", "x27", "x28", "x29", "lr", "sp"
39 };
40
41 enum a64_shift_type {
42 A64_SHIFT_TYPE_LSL = 0,
43 A64_SHIFT_TYPE_LSR = 1,
44 A64_SHIFT_TYPE_ASR = 2,
45 A64_SHIFT_TYPE_ROR = 3
46 };
47
48 /*
49 * Helpers for extracting complex instruction fields
50 */
51
52 /*
53 * For load/store with an unsigned 12 bit immediate scaled by the element
54 * size. The input has the immediate field in bits [14:3] and the element
55 * size in [2:0].
56 */
uimm_scaled(DisasContext * s,int x)57 static int uimm_scaled(DisasContext *s, int x)
58 {
59 unsigned imm = x >> 3;
60 unsigned scale = extract32(x, 0, 3);
61 return imm << scale;
62 }
63
64 /* For load/store memory tags: scale offset by LOG2_TAG_GRANULE */
scale_by_log2_tag_granule(DisasContext * s,int x)65 static int scale_by_log2_tag_granule(DisasContext *s, int x)
66 {
67 return x << LOG2_TAG_GRANULE;
68 }
69
70 /*
71 * Include the generated decoders.
72 */
73
74 #include "decode-sme-fa64.c.inc"
75 #include "decode-a64.c.inc"
76
77 /* initialize TCG globals. */
a64_translate_init(void)78 void a64_translate_init(void)
79 {
80 int i;
81
82 cpu_pc = tcg_global_mem_new_i64(tcg_env,
83 offsetof(CPUARMState, pc),
84 "pc");
85 for (i = 0; i < 32; i++) {
86 cpu_X[i] = tcg_global_mem_new_i64(tcg_env,
87 offsetof(CPUARMState, xregs[i]),
88 regnames[i]);
89 }
90
91 cpu_exclusive_high = tcg_global_mem_new_i64(tcg_env,
92 offsetof(CPUARMState, exclusive_high), "exclusive_high");
93 }
94
95 /*
96 * Return the core mmu_idx to use for A64 load/store insns which
97 * have a "unprivileged load/store" variant. Those insns access
98 * EL0 if executed from an EL which has control over EL0 (usually
99 * EL1) but behave like normal loads and stores if executed from
100 * elsewhere (eg EL3).
101 *
102 * @unpriv : true for the unprivileged encoding; false for the
103 * normal encoding (in which case we will return the same
104 * thing as get_mem_index().
105 */
get_a64_user_mem_index(DisasContext * s,bool unpriv)106 static int get_a64_user_mem_index(DisasContext *s, bool unpriv)
107 {
108 /*
109 * If AccType_UNPRIV is not used, the insn uses AccType_NORMAL,
110 * which is the usual mmu_idx for this cpu state.
111 */
112 ARMMMUIdx useridx = s->mmu_idx;
113
114 if (unpriv && s->unpriv) {
115 /*
116 * We have pre-computed the condition for AccType_UNPRIV.
117 * Therefore we should never get here with a mmu_idx for
118 * which we do not know the corresponding user mmu_idx.
119 */
120 switch (useridx) {
121 case ARMMMUIdx_E10_1:
122 case ARMMMUIdx_E10_1_PAN:
123 useridx = ARMMMUIdx_E10_0;
124 break;
125 case ARMMMUIdx_E20_2:
126 case ARMMMUIdx_E20_2_PAN:
127 useridx = ARMMMUIdx_E20_0;
128 break;
129 default:
130 g_assert_not_reached();
131 }
132 }
133 return arm_to_core_mmu_idx(useridx);
134 }
135
set_btype_raw(int val)136 static void set_btype_raw(int val)
137 {
138 tcg_gen_st_i32(tcg_constant_i32(val), tcg_env,
139 offsetof(CPUARMState, btype));
140 }
141
set_btype(DisasContext * s,int val)142 static void set_btype(DisasContext *s, int val)
143 {
144 /* BTYPE is a 2-bit field, and 0 should be done with reset_btype. */
145 tcg_debug_assert(val >= 1 && val <= 3);
146 set_btype_raw(val);
147 s->btype = -1;
148 }
149
reset_btype(DisasContext * s)150 static void reset_btype(DisasContext *s)
151 {
152 if (s->btype != 0) {
153 set_btype_raw(0);
154 s->btype = 0;
155 }
156 }
157
gen_pc_plus_diff(DisasContext * s,TCGv_i64 dest,target_long diff)158 static void gen_pc_plus_diff(DisasContext *s, TCGv_i64 dest, target_long diff)
159 {
160 assert(s->pc_save != -1);
161 if (tb_cflags(s->base.tb) & CF_PCREL) {
162 tcg_gen_addi_i64(dest, cpu_pc, (s->pc_curr - s->pc_save) + diff);
163 } else {
164 tcg_gen_movi_i64(dest, s->pc_curr + diff);
165 }
166 }
167
gen_a64_update_pc(DisasContext * s,target_long diff)168 void gen_a64_update_pc(DisasContext *s, target_long diff)
169 {
170 gen_pc_plus_diff(s, cpu_pc, diff);
171 s->pc_save = s->pc_curr + diff;
172 }
173
174 /*
175 * Handle Top Byte Ignore (TBI) bits.
176 *
177 * If address tagging is enabled via the TCR TBI bits:
178 * + for EL2 and EL3 there is only one TBI bit, and if it is set
179 * then the address is zero-extended, clearing bits [63:56]
180 * + for EL0 and EL1, TBI0 controls addresses with bit 55 == 0
181 * and TBI1 controls addresses with bit 55 == 1.
182 * If the appropriate TBI bit is set for the address then
183 * the address is sign-extended from bit 55 into bits [63:56]
184 *
185 * Here We have concatenated TBI{1,0} into tbi.
186 */
gen_top_byte_ignore(DisasContext * s,TCGv_i64 dst,TCGv_i64 src,int tbi)187 static void gen_top_byte_ignore(DisasContext *s, TCGv_i64 dst,
188 TCGv_i64 src, int tbi)
189 {
190 if (tbi == 0) {
191 /* Load unmodified address */
192 tcg_gen_mov_i64(dst, src);
193 } else if (!regime_has_2_ranges(s->mmu_idx)) {
194 /* Force tag byte to all zero */
195 tcg_gen_extract_i64(dst, src, 0, 56);
196 } else {
197 /* Sign-extend from bit 55. */
198 tcg_gen_sextract_i64(dst, src, 0, 56);
199
200 switch (tbi) {
201 case 1:
202 /* tbi0 but !tbi1: only use the extension if positive */
203 tcg_gen_and_i64(dst, dst, src);
204 break;
205 case 2:
206 /* !tbi0 but tbi1: only use the extension if negative */
207 tcg_gen_or_i64(dst, dst, src);
208 break;
209 case 3:
210 /* tbi0 and tbi1: always use the extension */
211 break;
212 default:
213 g_assert_not_reached();
214 }
215 }
216 }
217
gen_a64_set_pc(DisasContext * s,TCGv_i64 src)218 static void gen_a64_set_pc(DisasContext *s, TCGv_i64 src)
219 {
220 /*
221 * If address tagging is enabled for instructions via the TCR TBI bits,
222 * then loading an address into the PC will clear out any tag.
223 */
224 gen_top_byte_ignore(s, cpu_pc, src, s->tbii);
225 s->pc_save = -1;
226 }
227
228 /*
229 * Handle MTE and/or TBI.
230 *
231 * For TBI, ideally, we would do nothing. Proper behaviour on fault is
232 * for the tag to be present in the FAR_ELx register. But for user-only
233 * mode we do not have a TLB with which to implement this, so we must
234 * remove the top byte now.
235 *
236 * Always return a fresh temporary that we can increment independently
237 * of the write-back address.
238 */
239
clean_data_tbi(DisasContext * s,TCGv_i64 addr)240 TCGv_i64 clean_data_tbi(DisasContext *s, TCGv_i64 addr)
241 {
242 TCGv_i64 clean = tcg_temp_new_i64();
243 #ifdef CONFIG_USER_ONLY
244 gen_top_byte_ignore(s, clean, addr, s->tbid);
245 #else
246 tcg_gen_mov_i64(clean, addr);
247 #endif
248 return clean;
249 }
250
251 /* Insert a zero tag into src, with the result at dst. */
gen_address_with_allocation_tag0(TCGv_i64 dst,TCGv_i64 src)252 static void gen_address_with_allocation_tag0(TCGv_i64 dst, TCGv_i64 src)
253 {
254 tcg_gen_andi_i64(dst, src, ~MAKE_64BIT_MASK(56, 4));
255 }
256
gen_probe_access(DisasContext * s,TCGv_i64 ptr,MMUAccessType acc,int log2_size)257 static void gen_probe_access(DisasContext *s, TCGv_i64 ptr,
258 MMUAccessType acc, int log2_size)
259 {
260 gen_helper_probe_access(tcg_env, ptr,
261 tcg_constant_i32(acc),
262 tcg_constant_i32(get_mem_index(s)),
263 tcg_constant_i32(1 << log2_size));
264 }
265
266 /*
267 * For MTE, check a single logical or atomic access. This probes a single
268 * address, the exact one specified. The size and alignment of the access
269 * is not relevant to MTE, per se, but watchpoints do require the size,
270 * and we want to recognize those before making any other changes to state.
271 */
gen_mte_check1_mmuidx(DisasContext * s,TCGv_i64 addr,bool is_write,bool tag_checked,MemOp memop,bool is_unpriv,int core_idx)272 static TCGv_i64 gen_mte_check1_mmuidx(DisasContext *s, TCGv_i64 addr,
273 bool is_write, bool tag_checked,
274 MemOp memop, bool is_unpriv,
275 int core_idx)
276 {
277 if (tag_checked && s->mte_active[is_unpriv]) {
278 TCGv_i64 ret;
279 int desc = 0;
280
281 desc = FIELD_DP32(desc, MTEDESC, MIDX, core_idx);
282 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid);
283 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma);
284 desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write);
285 desc = FIELD_DP32(desc, MTEDESC, ALIGN, memop_alignment_bits(memop));
286 desc = FIELD_DP32(desc, MTEDESC, SIZEM1, memop_size(memop) - 1);
287
288 ret = tcg_temp_new_i64();
289 gen_helper_mte_check(ret, tcg_env, tcg_constant_i32(desc), addr);
290
291 return ret;
292 }
293 return clean_data_tbi(s, addr);
294 }
295
gen_mte_check1(DisasContext * s,TCGv_i64 addr,bool is_write,bool tag_checked,MemOp memop)296 TCGv_i64 gen_mte_check1(DisasContext *s, TCGv_i64 addr, bool is_write,
297 bool tag_checked, MemOp memop)
298 {
299 return gen_mte_check1_mmuidx(s, addr, is_write, tag_checked, memop,
300 false, get_mem_index(s));
301 }
302
303 /*
304 * For MTE, check multiple logical sequential accesses.
305 */
gen_mte_checkN(DisasContext * s,TCGv_i64 addr,bool is_write,bool tag_checked,int total_size,MemOp single_mop)306 TCGv_i64 gen_mte_checkN(DisasContext *s, TCGv_i64 addr, bool is_write,
307 bool tag_checked, int total_size, MemOp single_mop)
308 {
309 if (tag_checked && s->mte_active[0]) {
310 TCGv_i64 ret;
311 int desc = 0;
312
313 desc = FIELD_DP32(desc, MTEDESC, MIDX, get_mem_index(s));
314 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid);
315 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma);
316 desc = FIELD_DP32(desc, MTEDESC, WRITE, is_write);
317 desc = FIELD_DP32(desc, MTEDESC, ALIGN, memop_alignment_bits(single_mop));
318 desc = FIELD_DP32(desc, MTEDESC, SIZEM1, total_size - 1);
319
320 ret = tcg_temp_new_i64();
321 gen_helper_mte_check(ret, tcg_env, tcg_constant_i32(desc), addr);
322
323 return ret;
324 }
325 return clean_data_tbi(s, addr);
326 }
327
328 /*
329 * Generate the special alignment check that applies to AccType_ATOMIC
330 * and AccType_ORDERED insns under FEAT_LSE2: the access need not be
331 * naturally aligned, but it must not cross a 16-byte boundary.
332 * See AArch64.CheckAlignment().
333 */
check_lse2_align(DisasContext * s,int rn,int imm,bool is_write,MemOp mop)334 static void check_lse2_align(DisasContext *s, int rn, int imm,
335 bool is_write, MemOp mop)
336 {
337 TCGv_i32 tmp;
338 TCGv_i64 addr;
339 TCGLabel *over_label;
340 MMUAccessType type;
341 int mmu_idx;
342
343 tmp = tcg_temp_new_i32();
344 tcg_gen_extrl_i64_i32(tmp, cpu_reg_sp(s, rn));
345 tcg_gen_addi_i32(tmp, tmp, imm & 15);
346 tcg_gen_andi_i32(tmp, tmp, 15);
347 tcg_gen_addi_i32(tmp, tmp, memop_size(mop));
348
349 over_label = gen_new_label();
350 tcg_gen_brcondi_i32(TCG_COND_LEU, tmp, 16, over_label);
351
352 addr = tcg_temp_new_i64();
353 tcg_gen_addi_i64(addr, cpu_reg_sp(s, rn), imm);
354
355 type = is_write ? MMU_DATA_STORE : MMU_DATA_LOAD,
356 mmu_idx = get_mem_index(s);
357 gen_helper_unaligned_access(tcg_env, addr, tcg_constant_i32(type),
358 tcg_constant_i32(mmu_idx));
359
360 gen_set_label(over_label);
361
362 }
363
364 /* Handle the alignment check for AccType_ATOMIC instructions. */
check_atomic_align(DisasContext * s,int rn,MemOp mop)365 static MemOp check_atomic_align(DisasContext *s, int rn, MemOp mop)
366 {
367 MemOp size = mop & MO_SIZE;
368
369 if (size == MO_8) {
370 return mop;
371 }
372
373 /*
374 * If size == MO_128, this is a LDXP, and the operation is single-copy
375 * atomic for each doubleword, not the entire quadword; it still must
376 * be quadword aligned.
377 */
378 if (size == MO_128) {
379 return finalize_memop_atom(s, MO_128 | MO_ALIGN,
380 MO_ATOM_IFALIGN_PAIR);
381 }
382 if (dc_isar_feature(aa64_lse2, s)) {
383 check_lse2_align(s, rn, 0, true, mop);
384 } else {
385 mop |= MO_ALIGN;
386 }
387 return finalize_memop(s, mop);
388 }
389
390 /* Handle the alignment check for AccType_ORDERED instructions. */
check_ordered_align(DisasContext * s,int rn,int imm,bool is_write,MemOp mop)391 static MemOp check_ordered_align(DisasContext *s, int rn, int imm,
392 bool is_write, MemOp mop)
393 {
394 MemOp size = mop & MO_SIZE;
395
396 if (size == MO_8) {
397 return mop;
398 }
399 if (size == MO_128) {
400 return finalize_memop_atom(s, MO_128 | MO_ALIGN,
401 MO_ATOM_IFALIGN_PAIR);
402 }
403 if (!dc_isar_feature(aa64_lse2, s)) {
404 mop |= MO_ALIGN;
405 } else if (!s->naa) {
406 check_lse2_align(s, rn, imm, is_write, mop);
407 }
408 return finalize_memop(s, mop);
409 }
410
411 typedef struct DisasCompare64 {
412 TCGCond cond;
413 TCGv_i64 value;
414 } DisasCompare64;
415
a64_test_cc(DisasCompare64 * c64,int cc)416 static void a64_test_cc(DisasCompare64 *c64, int cc)
417 {
418 DisasCompare c32;
419
420 arm_test_cc(&c32, cc);
421
422 /*
423 * Sign-extend the 32-bit value so that the GE/LT comparisons work
424 * properly. The NE/EQ comparisons are also fine with this choice.
425 */
426 c64->cond = c32.cond;
427 c64->value = tcg_temp_new_i64();
428 tcg_gen_ext_i32_i64(c64->value, c32.value);
429 }
430
gen_rebuild_hflags(DisasContext * s)431 static void gen_rebuild_hflags(DisasContext *s)
432 {
433 gen_helper_rebuild_hflags_a64(tcg_env, tcg_constant_i32(s->current_el));
434 }
435
gen_exception_internal_insn(DisasContext * s,int excp)436 static void gen_exception_internal_insn(DisasContext *s, int excp)
437 {
438 gen_a64_update_pc(s, 0);
439 gen_exception_internal(excp);
440 s->base.is_jmp = DISAS_NORETURN;
441 }
442
gen_exception_bkpt_insn(DisasContext * s,uint32_t syndrome)443 static void gen_exception_bkpt_insn(DisasContext *s, uint32_t syndrome)
444 {
445 gen_a64_update_pc(s, 0);
446 gen_helper_exception_bkpt_insn(tcg_env, tcg_constant_i32(syndrome));
447 s->base.is_jmp = DISAS_NORETURN;
448 }
449
gen_step_complete_exception(DisasContext * s)450 static void gen_step_complete_exception(DisasContext *s)
451 {
452 /* We just completed step of an insn. Move from Active-not-pending
453 * to Active-pending, and then also take the swstep exception.
454 * This corresponds to making the (IMPDEF) choice to prioritize
455 * swstep exceptions over asynchronous exceptions taken to an exception
456 * level where debug is disabled. This choice has the advantage that
457 * we do not need to maintain internal state corresponding to the
458 * ISV/EX syndrome bits between completion of the step and generation
459 * of the exception, and our syndrome information is always correct.
460 */
461 gen_ss_advance(s);
462 gen_swstep_exception(s, 1, s->is_ldex);
463 s->base.is_jmp = DISAS_NORETURN;
464 }
465
use_goto_tb(DisasContext * s,uint64_t dest)466 static inline bool use_goto_tb(DisasContext *s, uint64_t dest)
467 {
468 if (s->ss_active) {
469 return false;
470 }
471 return translator_use_goto_tb(&s->base, dest);
472 }
473
gen_goto_tb(DisasContext * s,int n,int64_t diff)474 static void gen_goto_tb(DisasContext *s, int n, int64_t diff)
475 {
476 if (use_goto_tb(s, s->pc_curr + diff)) {
477 /*
478 * For pcrel, the pc must always be up-to-date on entry to
479 * the linked TB, so that it can use simple additions for all
480 * further adjustments. For !pcrel, the linked TB is compiled
481 * to know its full virtual address, so we can delay the
482 * update to pc to the unlinked path. A long chain of links
483 * can thus avoid many updates to the PC.
484 */
485 if (tb_cflags(s->base.tb) & CF_PCREL) {
486 gen_a64_update_pc(s, diff);
487 tcg_gen_goto_tb(n);
488 } else {
489 tcg_gen_goto_tb(n);
490 gen_a64_update_pc(s, diff);
491 }
492 tcg_gen_exit_tb(s->base.tb, n);
493 s->base.is_jmp = DISAS_NORETURN;
494 } else {
495 gen_a64_update_pc(s, diff);
496 if (s->ss_active) {
497 gen_step_complete_exception(s);
498 } else {
499 tcg_gen_lookup_and_goto_ptr();
500 s->base.is_jmp = DISAS_NORETURN;
501 }
502 }
503 }
504
505 /*
506 * Register access functions
507 *
508 * These functions are used for directly accessing a register in where
509 * changes to the final register value are likely to be made. If you
510 * need to use a register for temporary calculation (e.g. index type
511 * operations) use the read_* form.
512 *
513 * B1.2.1 Register mappings
514 *
515 * In instruction register encoding 31 can refer to ZR (zero register) or
516 * the SP (stack pointer) depending on context. In QEMU's case we map SP
517 * to cpu_X[31] and ZR accesses to a temporary which can be discarded.
518 * This is the point of the _sp forms.
519 */
cpu_reg(DisasContext * s,int reg)520 TCGv_i64 cpu_reg(DisasContext *s, int reg)
521 {
522 if (reg == 31) {
523 TCGv_i64 t = tcg_temp_new_i64();
524 tcg_gen_movi_i64(t, 0);
525 return t;
526 } else {
527 return cpu_X[reg];
528 }
529 }
530
531 /* register access for when 31 == SP */
cpu_reg_sp(DisasContext * s,int reg)532 TCGv_i64 cpu_reg_sp(DisasContext *s, int reg)
533 {
534 return cpu_X[reg];
535 }
536
537 /* read a cpu register in 32bit/64bit mode. Returns a TCGv_i64
538 * representing the register contents. This TCGv is an auto-freed
539 * temporary so it need not be explicitly freed, and may be modified.
540 */
read_cpu_reg(DisasContext * s,int reg,int sf)541 TCGv_i64 read_cpu_reg(DisasContext *s, int reg, int sf)
542 {
543 TCGv_i64 v = tcg_temp_new_i64();
544 if (reg != 31) {
545 if (sf) {
546 tcg_gen_mov_i64(v, cpu_X[reg]);
547 } else {
548 tcg_gen_ext32u_i64(v, cpu_X[reg]);
549 }
550 } else {
551 tcg_gen_movi_i64(v, 0);
552 }
553 return v;
554 }
555
read_cpu_reg_sp(DisasContext * s,int reg,int sf)556 TCGv_i64 read_cpu_reg_sp(DisasContext *s, int reg, int sf)
557 {
558 TCGv_i64 v = tcg_temp_new_i64();
559 if (sf) {
560 tcg_gen_mov_i64(v, cpu_X[reg]);
561 } else {
562 tcg_gen_ext32u_i64(v, cpu_X[reg]);
563 }
564 return v;
565 }
566
567 /* Return the offset into CPUARMState of a slice (from
568 * the least significant end) of FP register Qn (ie
569 * Dn, Sn, Hn or Bn).
570 * (Note that this is not the same mapping as for A32; see cpu.h)
571 */
fp_reg_offset(DisasContext * s,int regno,MemOp size)572 static inline int fp_reg_offset(DisasContext *s, int regno, MemOp size)
573 {
574 return vec_reg_offset(s, regno, 0, size);
575 }
576
577 /* Offset of the high half of the 128 bit vector Qn */
fp_reg_hi_offset(DisasContext * s,int regno)578 static inline int fp_reg_hi_offset(DisasContext *s, int regno)
579 {
580 return vec_reg_offset(s, regno, 1, MO_64);
581 }
582
583 /* Convenience accessors for reading and writing single and double
584 * FP registers. Writing clears the upper parts of the associated
585 * 128 bit vector register, as required by the architecture.
586 * Note that unlike the GP register accessors, the values returned
587 * by the read functions must be manually freed.
588 */
read_fp_dreg(DisasContext * s,int reg)589 static TCGv_i64 read_fp_dreg(DisasContext *s, int reg)
590 {
591 TCGv_i64 v = tcg_temp_new_i64();
592
593 tcg_gen_ld_i64(v, tcg_env, fp_reg_offset(s, reg, MO_64));
594 return v;
595 }
596
read_fp_sreg(DisasContext * s,int reg)597 static TCGv_i32 read_fp_sreg(DisasContext *s, int reg)
598 {
599 TCGv_i32 v = tcg_temp_new_i32();
600
601 tcg_gen_ld_i32(v, tcg_env, fp_reg_offset(s, reg, MO_32));
602 return v;
603 }
604
read_fp_hreg(DisasContext * s,int reg)605 static TCGv_i32 read_fp_hreg(DisasContext *s, int reg)
606 {
607 TCGv_i32 v = tcg_temp_new_i32();
608
609 tcg_gen_ld16u_i32(v, tcg_env, fp_reg_offset(s, reg, MO_16));
610 return v;
611 }
612
clear_vec(DisasContext * s,int rd)613 static void clear_vec(DisasContext *s, int rd)
614 {
615 unsigned ofs = fp_reg_offset(s, rd, MO_64);
616 unsigned vsz = vec_full_reg_size(s);
617
618 tcg_gen_gvec_dup_imm(MO_64, ofs, vsz, vsz, 0);
619 }
620
621 /*
622 * Clear the bits above an N-bit vector, for N = (is_q ? 128 : 64).
623 * If SVE is not enabled, then there are only 128 bits in the vector.
624 */
clear_vec_high(DisasContext * s,bool is_q,int rd)625 static void clear_vec_high(DisasContext *s, bool is_q, int rd)
626 {
627 unsigned ofs = fp_reg_offset(s, rd, MO_64);
628 unsigned vsz = vec_full_reg_size(s);
629
630 /* Nop move, with side effect of clearing the tail. */
631 tcg_gen_gvec_mov(MO_64, ofs, ofs, is_q ? 16 : 8, vsz);
632 }
633
write_fp_dreg(DisasContext * s,int reg,TCGv_i64 v)634 void write_fp_dreg(DisasContext *s, int reg, TCGv_i64 v)
635 {
636 unsigned ofs = fp_reg_offset(s, reg, MO_64);
637
638 tcg_gen_st_i64(v, tcg_env, ofs);
639 clear_vec_high(s, false, reg);
640 }
641
write_fp_sreg(DisasContext * s,int reg,TCGv_i32 v)642 static void write_fp_sreg(DisasContext *s, int reg, TCGv_i32 v)
643 {
644 TCGv_i64 tmp = tcg_temp_new_i64();
645
646 tcg_gen_extu_i32_i64(tmp, v);
647 write_fp_dreg(s, reg, tmp);
648 }
649
650 /*
651 * Write a double result to 128 bit vector register reg, honouring FPCR.NEP:
652 * - if FPCR.NEP == 0, clear the high elements of reg
653 * - if FPCR.NEP == 1, set the high elements of reg from mergereg
654 * (i.e. merge the result with those high elements)
655 * In either case, SVE register bits above 128 are zeroed (per R_WKYLB).
656 */
write_fp_dreg_merging(DisasContext * s,int reg,int mergereg,TCGv_i64 v)657 static void write_fp_dreg_merging(DisasContext *s, int reg, int mergereg,
658 TCGv_i64 v)
659 {
660 if (!s->fpcr_nep) {
661 write_fp_dreg(s, reg, v);
662 return;
663 }
664
665 /*
666 * Move from mergereg to reg; this sets the high elements and
667 * clears the bits above 128 as a side effect.
668 */
669 tcg_gen_gvec_mov(MO_64, vec_full_reg_offset(s, reg),
670 vec_full_reg_offset(s, mergereg),
671 16, vec_full_reg_size(s));
672 tcg_gen_st_i64(v, tcg_env, vec_full_reg_offset(s, reg));
673 }
674
675 /*
676 * Write a single-prec result, but only clear the higher elements
677 * of the destination register if FPCR.NEP is 0; otherwise preserve them.
678 */
write_fp_sreg_merging(DisasContext * s,int reg,int mergereg,TCGv_i32 v)679 static void write_fp_sreg_merging(DisasContext *s, int reg, int mergereg,
680 TCGv_i32 v)
681 {
682 if (!s->fpcr_nep) {
683 write_fp_sreg(s, reg, v);
684 return;
685 }
686
687 tcg_gen_gvec_mov(MO_64, vec_full_reg_offset(s, reg),
688 vec_full_reg_offset(s, mergereg),
689 16, vec_full_reg_size(s));
690 tcg_gen_st_i32(v, tcg_env, fp_reg_offset(s, reg, MO_32));
691 }
692
693 /*
694 * Write a half-prec result, but only clear the higher elements
695 * of the destination register if FPCR.NEP is 0; otherwise preserve them.
696 * The caller must ensure that the top 16 bits of v are zero.
697 */
write_fp_hreg_merging(DisasContext * s,int reg,int mergereg,TCGv_i32 v)698 static void write_fp_hreg_merging(DisasContext *s, int reg, int mergereg,
699 TCGv_i32 v)
700 {
701 if (!s->fpcr_nep) {
702 write_fp_sreg(s, reg, v);
703 return;
704 }
705
706 tcg_gen_gvec_mov(MO_64, vec_full_reg_offset(s, reg),
707 vec_full_reg_offset(s, mergereg),
708 16, vec_full_reg_size(s));
709 tcg_gen_st16_i32(v, tcg_env, fp_reg_offset(s, reg, MO_16));
710 }
711
712 /* Expand a 2-operand AdvSIMD vector operation using an expander function. */
gen_gvec_fn2(DisasContext * s,bool is_q,int rd,int rn,GVecGen2Fn * gvec_fn,int vece)713 static void gen_gvec_fn2(DisasContext *s, bool is_q, int rd, int rn,
714 GVecGen2Fn *gvec_fn, int vece)
715 {
716 gvec_fn(vece, vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn),
717 is_q ? 16 : 8, vec_full_reg_size(s));
718 }
719
720 /* Expand a 2-operand + immediate AdvSIMD vector operation using
721 * an expander function.
722 */
gen_gvec_fn2i(DisasContext * s,bool is_q,int rd,int rn,int64_t imm,GVecGen2iFn * gvec_fn,int vece)723 static void gen_gvec_fn2i(DisasContext *s, bool is_q, int rd, int rn,
724 int64_t imm, GVecGen2iFn *gvec_fn, int vece)
725 {
726 gvec_fn(vece, vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn),
727 imm, is_q ? 16 : 8, vec_full_reg_size(s));
728 }
729
730 /* Expand a 3-operand AdvSIMD vector operation using an expander function. */
gen_gvec_fn3(DisasContext * s,bool is_q,int rd,int rn,int rm,GVecGen3Fn * gvec_fn,int vece)731 static void gen_gvec_fn3(DisasContext *s, bool is_q, int rd, int rn, int rm,
732 GVecGen3Fn *gvec_fn, int vece)
733 {
734 gvec_fn(vece, vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn),
735 vec_full_reg_offset(s, rm), is_q ? 16 : 8, vec_full_reg_size(s));
736 }
737
738 /* Expand a 4-operand AdvSIMD vector operation using an expander function. */
gen_gvec_fn4(DisasContext * s,bool is_q,int rd,int rn,int rm,int rx,GVecGen4Fn * gvec_fn,int vece)739 static void gen_gvec_fn4(DisasContext *s, bool is_q, int rd, int rn, int rm,
740 int rx, GVecGen4Fn *gvec_fn, int vece)
741 {
742 gvec_fn(vece, vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn),
743 vec_full_reg_offset(s, rm), vec_full_reg_offset(s, rx),
744 is_q ? 16 : 8, vec_full_reg_size(s));
745 }
746
747 /* Expand a 2-operand operation using an out-of-line helper. */
gen_gvec_op2_ool(DisasContext * s,bool is_q,int rd,int rn,int data,gen_helper_gvec_2 * fn)748 static void gen_gvec_op2_ool(DisasContext *s, bool is_q, int rd,
749 int rn, int data, gen_helper_gvec_2 *fn)
750 {
751 tcg_gen_gvec_2_ool(vec_full_reg_offset(s, rd),
752 vec_full_reg_offset(s, rn),
753 is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
754 }
755
756 /* Expand a 3-operand operation using an out-of-line helper. */
gen_gvec_op3_ool(DisasContext * s,bool is_q,int rd,int rn,int rm,int data,gen_helper_gvec_3 * fn)757 static void gen_gvec_op3_ool(DisasContext *s, bool is_q, int rd,
758 int rn, int rm, int data, gen_helper_gvec_3 *fn)
759 {
760 tcg_gen_gvec_3_ool(vec_full_reg_offset(s, rd),
761 vec_full_reg_offset(s, rn),
762 vec_full_reg_offset(s, rm),
763 is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
764 }
765
766 /* Expand a 3-operand + fpstatus pointer + simd data value operation using
767 * an out-of-line helper.
768 */
gen_gvec_op3_fpst(DisasContext * s,bool is_q,int rd,int rn,int rm,ARMFPStatusFlavour fpsttype,int data,gen_helper_gvec_3_ptr * fn)769 static void gen_gvec_op3_fpst(DisasContext *s, bool is_q, int rd, int rn,
770 int rm, ARMFPStatusFlavour fpsttype, int data,
771 gen_helper_gvec_3_ptr *fn)
772 {
773 TCGv_ptr fpst = fpstatus_ptr(fpsttype);
774 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, rd),
775 vec_full_reg_offset(s, rn),
776 vec_full_reg_offset(s, rm), fpst,
777 is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
778 }
779
780 /* Expand a 4-operand operation using an out-of-line helper. */
gen_gvec_op4_ool(DisasContext * s,bool is_q,int rd,int rn,int rm,int ra,int data,gen_helper_gvec_4 * fn)781 static void gen_gvec_op4_ool(DisasContext *s, bool is_q, int rd, int rn,
782 int rm, int ra, int data, gen_helper_gvec_4 *fn)
783 {
784 tcg_gen_gvec_4_ool(vec_full_reg_offset(s, rd),
785 vec_full_reg_offset(s, rn),
786 vec_full_reg_offset(s, rm),
787 vec_full_reg_offset(s, ra),
788 is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
789 }
790
791 /*
792 * Expand a 4-operand operation using an out-of-line helper that takes
793 * a pointer to the CPU env.
794 */
gen_gvec_op4_env(DisasContext * s,bool is_q,int rd,int rn,int rm,int ra,int data,gen_helper_gvec_4_ptr * fn)795 static void gen_gvec_op4_env(DisasContext *s, bool is_q, int rd, int rn,
796 int rm, int ra, int data,
797 gen_helper_gvec_4_ptr *fn)
798 {
799 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, rd),
800 vec_full_reg_offset(s, rn),
801 vec_full_reg_offset(s, rm),
802 vec_full_reg_offset(s, ra),
803 tcg_env,
804 is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
805 }
806
807 /*
808 * Expand a 4-operand + fpstatus pointer + simd data value operation using
809 * an out-of-line helper.
810 */
gen_gvec_op4_fpst(DisasContext * s,bool is_q,int rd,int rn,int rm,int ra,ARMFPStatusFlavour fpsttype,int data,gen_helper_gvec_4_ptr * fn)811 static void gen_gvec_op4_fpst(DisasContext *s, bool is_q, int rd, int rn,
812 int rm, int ra, ARMFPStatusFlavour fpsttype,
813 int data,
814 gen_helper_gvec_4_ptr *fn)
815 {
816 TCGv_ptr fpst = fpstatus_ptr(fpsttype);
817 tcg_gen_gvec_4_ptr(vec_full_reg_offset(s, rd),
818 vec_full_reg_offset(s, rn),
819 vec_full_reg_offset(s, rm),
820 vec_full_reg_offset(s, ra), fpst,
821 is_q ? 16 : 8, vec_full_reg_size(s), data, fn);
822 }
823
824 /*
825 * When FPCR.AH == 1, NEG and ABS do not flip the sign bit of a NaN.
826 * These functions implement
827 * d = floatN_is_any_nan(s) ? s : floatN_chs(s)
828 * which for float32 is
829 * d = (s & ~(1 << 31)) > 0x7f800000UL) ? s : (s ^ (1 << 31))
830 * and similarly for the other float sizes.
831 */
gen_vfp_ah_negh(TCGv_i32 d,TCGv_i32 s)832 static void gen_vfp_ah_negh(TCGv_i32 d, TCGv_i32 s)
833 {
834 TCGv_i32 abs_s = tcg_temp_new_i32(), chs_s = tcg_temp_new_i32();
835
836 gen_vfp_negh(chs_s, s);
837 gen_vfp_absh(abs_s, s);
838 tcg_gen_movcond_i32(TCG_COND_GTU, d,
839 abs_s, tcg_constant_i32(0x7c00),
840 s, chs_s);
841 }
842
gen_vfp_ah_negs(TCGv_i32 d,TCGv_i32 s)843 static void gen_vfp_ah_negs(TCGv_i32 d, TCGv_i32 s)
844 {
845 TCGv_i32 abs_s = tcg_temp_new_i32(), chs_s = tcg_temp_new_i32();
846
847 gen_vfp_negs(chs_s, s);
848 gen_vfp_abss(abs_s, s);
849 tcg_gen_movcond_i32(TCG_COND_GTU, d,
850 abs_s, tcg_constant_i32(0x7f800000UL),
851 s, chs_s);
852 }
853
gen_vfp_ah_negd(TCGv_i64 d,TCGv_i64 s)854 static void gen_vfp_ah_negd(TCGv_i64 d, TCGv_i64 s)
855 {
856 TCGv_i64 abs_s = tcg_temp_new_i64(), chs_s = tcg_temp_new_i64();
857
858 gen_vfp_negd(chs_s, s);
859 gen_vfp_absd(abs_s, s);
860 tcg_gen_movcond_i64(TCG_COND_GTU, d,
861 abs_s, tcg_constant_i64(0x7ff0000000000000ULL),
862 s, chs_s);
863 }
864
865 /*
866 * These functions implement
867 * d = floatN_is_any_nan(s) ? s : floatN_abs(s)
868 * which for float32 is
869 * d = (s & ~(1 << 31)) > 0x7f800000UL) ? s : (s & ~(1 << 31))
870 * and similarly for the other float sizes.
871 */
gen_vfp_ah_absh(TCGv_i32 d,TCGv_i32 s)872 static void gen_vfp_ah_absh(TCGv_i32 d, TCGv_i32 s)
873 {
874 TCGv_i32 abs_s = tcg_temp_new_i32();
875
876 gen_vfp_absh(abs_s, s);
877 tcg_gen_movcond_i32(TCG_COND_GTU, d,
878 abs_s, tcg_constant_i32(0x7c00),
879 s, abs_s);
880 }
881
gen_vfp_ah_abss(TCGv_i32 d,TCGv_i32 s)882 static void gen_vfp_ah_abss(TCGv_i32 d, TCGv_i32 s)
883 {
884 TCGv_i32 abs_s = tcg_temp_new_i32();
885
886 gen_vfp_abss(abs_s, s);
887 tcg_gen_movcond_i32(TCG_COND_GTU, d,
888 abs_s, tcg_constant_i32(0x7f800000UL),
889 s, abs_s);
890 }
891
gen_vfp_ah_absd(TCGv_i64 d,TCGv_i64 s)892 static void gen_vfp_ah_absd(TCGv_i64 d, TCGv_i64 s)
893 {
894 TCGv_i64 abs_s = tcg_temp_new_i64();
895
896 gen_vfp_absd(abs_s, s);
897 tcg_gen_movcond_i64(TCG_COND_GTU, d,
898 abs_s, tcg_constant_i64(0x7ff0000000000000ULL),
899 s, abs_s);
900 }
901
gen_vfp_maybe_ah_negh(DisasContext * dc,TCGv_i32 d,TCGv_i32 s)902 static void gen_vfp_maybe_ah_negh(DisasContext *dc, TCGv_i32 d, TCGv_i32 s)
903 {
904 if (dc->fpcr_ah) {
905 gen_vfp_ah_negh(d, s);
906 } else {
907 gen_vfp_negh(d, s);
908 }
909 }
910
gen_vfp_maybe_ah_negs(DisasContext * dc,TCGv_i32 d,TCGv_i32 s)911 static void gen_vfp_maybe_ah_negs(DisasContext *dc, TCGv_i32 d, TCGv_i32 s)
912 {
913 if (dc->fpcr_ah) {
914 gen_vfp_ah_negs(d, s);
915 } else {
916 gen_vfp_negs(d, s);
917 }
918 }
919
gen_vfp_maybe_ah_negd(DisasContext * dc,TCGv_i64 d,TCGv_i64 s)920 static void gen_vfp_maybe_ah_negd(DisasContext *dc, TCGv_i64 d, TCGv_i64 s)
921 {
922 if (dc->fpcr_ah) {
923 gen_vfp_ah_negd(d, s);
924 } else {
925 gen_vfp_negd(d, s);
926 }
927 }
928
929 /* Set ZF and NF based on a 64 bit result. This is alas fiddlier
930 * than the 32 bit equivalent.
931 */
gen_set_NZ64(TCGv_i64 result)932 static inline void gen_set_NZ64(TCGv_i64 result)
933 {
934 tcg_gen_extr_i64_i32(cpu_ZF, cpu_NF, result);
935 tcg_gen_or_i32(cpu_ZF, cpu_ZF, cpu_NF);
936 }
937
938 /* Set NZCV as for a logical operation: NZ as per result, CV cleared. */
gen_logic_CC(int sf,TCGv_i64 result)939 static inline void gen_logic_CC(int sf, TCGv_i64 result)
940 {
941 if (sf) {
942 gen_set_NZ64(result);
943 } else {
944 tcg_gen_extrl_i64_i32(cpu_ZF, result);
945 tcg_gen_mov_i32(cpu_NF, cpu_ZF);
946 }
947 tcg_gen_movi_i32(cpu_CF, 0);
948 tcg_gen_movi_i32(cpu_VF, 0);
949 }
950
951 /* dest = T0 + T1; compute C, N, V and Z flags */
gen_add64_CC(TCGv_i64 dest,TCGv_i64 t0,TCGv_i64 t1)952 static void gen_add64_CC(TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
953 {
954 TCGv_i64 result, flag, tmp;
955 result = tcg_temp_new_i64();
956 flag = tcg_temp_new_i64();
957 tmp = tcg_temp_new_i64();
958
959 tcg_gen_movi_i64(tmp, 0);
960 tcg_gen_add2_i64(result, flag, t0, tmp, t1, tmp);
961
962 tcg_gen_extrl_i64_i32(cpu_CF, flag);
963
964 gen_set_NZ64(result);
965
966 tcg_gen_xor_i64(flag, result, t0);
967 tcg_gen_xor_i64(tmp, t0, t1);
968 tcg_gen_andc_i64(flag, flag, tmp);
969 tcg_gen_extrh_i64_i32(cpu_VF, flag);
970
971 tcg_gen_mov_i64(dest, result);
972 }
973
gen_add32_CC(TCGv_i64 dest,TCGv_i64 t0,TCGv_i64 t1)974 static void gen_add32_CC(TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
975 {
976 TCGv_i32 t0_32 = tcg_temp_new_i32();
977 TCGv_i32 t1_32 = tcg_temp_new_i32();
978 TCGv_i32 tmp = tcg_temp_new_i32();
979
980 tcg_gen_movi_i32(tmp, 0);
981 tcg_gen_extrl_i64_i32(t0_32, t0);
982 tcg_gen_extrl_i64_i32(t1_32, t1);
983 tcg_gen_add2_i32(cpu_NF, cpu_CF, t0_32, tmp, t1_32, tmp);
984 tcg_gen_mov_i32(cpu_ZF, cpu_NF);
985 tcg_gen_xor_i32(cpu_VF, cpu_NF, t0_32);
986 tcg_gen_xor_i32(tmp, t0_32, t1_32);
987 tcg_gen_andc_i32(cpu_VF, cpu_VF, tmp);
988 tcg_gen_extu_i32_i64(dest, cpu_NF);
989 }
990
gen_add_CC(int sf,TCGv_i64 dest,TCGv_i64 t0,TCGv_i64 t1)991 static void gen_add_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
992 {
993 if (sf) {
994 gen_add64_CC(dest, t0, t1);
995 } else {
996 gen_add32_CC(dest, t0, t1);
997 }
998 }
999
1000 /* dest = T0 - T1; compute C, N, V and Z flags */
gen_sub64_CC(TCGv_i64 dest,TCGv_i64 t0,TCGv_i64 t1)1001 static void gen_sub64_CC(TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
1002 {
1003 /* 64 bit arithmetic */
1004 TCGv_i64 result, flag, tmp;
1005
1006 result = tcg_temp_new_i64();
1007 flag = tcg_temp_new_i64();
1008 tcg_gen_sub_i64(result, t0, t1);
1009
1010 gen_set_NZ64(result);
1011
1012 tcg_gen_setcond_i64(TCG_COND_GEU, flag, t0, t1);
1013 tcg_gen_extrl_i64_i32(cpu_CF, flag);
1014
1015 tcg_gen_xor_i64(flag, result, t0);
1016 tmp = tcg_temp_new_i64();
1017 tcg_gen_xor_i64(tmp, t0, t1);
1018 tcg_gen_and_i64(flag, flag, tmp);
1019 tcg_gen_extrh_i64_i32(cpu_VF, flag);
1020 tcg_gen_mov_i64(dest, result);
1021 }
1022
gen_sub32_CC(TCGv_i64 dest,TCGv_i64 t0,TCGv_i64 t1)1023 static void gen_sub32_CC(TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
1024 {
1025 /* 32 bit arithmetic */
1026 TCGv_i32 t0_32 = tcg_temp_new_i32();
1027 TCGv_i32 t1_32 = tcg_temp_new_i32();
1028 TCGv_i32 tmp;
1029
1030 tcg_gen_extrl_i64_i32(t0_32, t0);
1031 tcg_gen_extrl_i64_i32(t1_32, t1);
1032 tcg_gen_sub_i32(cpu_NF, t0_32, t1_32);
1033 tcg_gen_mov_i32(cpu_ZF, cpu_NF);
1034 tcg_gen_setcond_i32(TCG_COND_GEU, cpu_CF, t0_32, t1_32);
1035 tcg_gen_xor_i32(cpu_VF, cpu_NF, t0_32);
1036 tmp = tcg_temp_new_i32();
1037 tcg_gen_xor_i32(tmp, t0_32, t1_32);
1038 tcg_gen_and_i32(cpu_VF, cpu_VF, tmp);
1039 tcg_gen_extu_i32_i64(dest, cpu_NF);
1040 }
1041
gen_sub_CC(int sf,TCGv_i64 dest,TCGv_i64 t0,TCGv_i64 t1)1042 static void gen_sub_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
1043 {
1044 if (sf) {
1045 gen_sub64_CC(dest, t0, t1);
1046 } else {
1047 gen_sub32_CC(dest, t0, t1);
1048 }
1049 }
1050
1051 /* dest = T0 + T1 + CF; do not compute flags. */
gen_adc(int sf,TCGv_i64 dest,TCGv_i64 t0,TCGv_i64 t1)1052 static void gen_adc(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
1053 {
1054 TCGv_i64 flag = tcg_temp_new_i64();
1055 tcg_gen_extu_i32_i64(flag, cpu_CF);
1056 tcg_gen_add_i64(dest, t0, t1);
1057 tcg_gen_add_i64(dest, dest, flag);
1058
1059 if (!sf) {
1060 tcg_gen_ext32u_i64(dest, dest);
1061 }
1062 }
1063
1064 /* dest = T0 + T1 + CF; compute C, N, V and Z flags. */
gen_adc_CC(int sf,TCGv_i64 dest,TCGv_i64 t0,TCGv_i64 t1)1065 static void gen_adc_CC(int sf, TCGv_i64 dest, TCGv_i64 t0, TCGv_i64 t1)
1066 {
1067 if (sf) {
1068 TCGv_i64 result = tcg_temp_new_i64();
1069 TCGv_i64 cf_64 = tcg_temp_new_i64();
1070 TCGv_i64 vf_64 = tcg_temp_new_i64();
1071 TCGv_i64 tmp = tcg_temp_new_i64();
1072
1073 tcg_gen_extu_i32_i64(cf_64, cpu_CF);
1074 tcg_gen_addcio_i64(result, cf_64, t0, t1, cf_64);
1075 tcg_gen_extrl_i64_i32(cpu_CF, cf_64);
1076 gen_set_NZ64(result);
1077
1078 tcg_gen_xor_i64(vf_64, result, t0);
1079 tcg_gen_xor_i64(tmp, t0, t1);
1080 tcg_gen_andc_i64(vf_64, vf_64, tmp);
1081 tcg_gen_extrh_i64_i32(cpu_VF, vf_64);
1082
1083 tcg_gen_mov_i64(dest, result);
1084 } else {
1085 TCGv_i32 t0_32 = tcg_temp_new_i32();
1086 TCGv_i32 t1_32 = tcg_temp_new_i32();
1087 TCGv_i32 tmp = tcg_temp_new_i32();
1088
1089 tcg_gen_extrl_i64_i32(t0_32, t0);
1090 tcg_gen_extrl_i64_i32(t1_32, t1);
1091 tcg_gen_addcio_i32(cpu_NF, cpu_CF, t0_32, t1_32, cpu_CF);
1092
1093 tcg_gen_mov_i32(cpu_ZF, cpu_NF);
1094 tcg_gen_xor_i32(cpu_VF, cpu_NF, t0_32);
1095 tcg_gen_xor_i32(tmp, t0_32, t1_32);
1096 tcg_gen_andc_i32(cpu_VF, cpu_VF, tmp);
1097 tcg_gen_extu_i32_i64(dest, cpu_NF);
1098 }
1099 }
1100
1101 /*
1102 * Load/Store generators
1103 */
1104
1105 /*
1106 * Store from GPR register to memory.
1107 */
do_gpr_st_memidx(DisasContext * s,TCGv_i64 source,TCGv_i64 tcg_addr,MemOp memop,int memidx,bool iss_valid,unsigned int iss_srt,bool iss_sf,bool iss_ar)1108 static void do_gpr_st_memidx(DisasContext *s, TCGv_i64 source,
1109 TCGv_i64 tcg_addr, MemOp memop, int memidx,
1110 bool iss_valid,
1111 unsigned int iss_srt,
1112 bool iss_sf, bool iss_ar)
1113 {
1114 tcg_gen_qemu_st_i64(source, tcg_addr, memidx, memop);
1115
1116 if (iss_valid) {
1117 uint32_t syn;
1118
1119 syn = syn_data_abort_with_iss(0,
1120 (memop & MO_SIZE),
1121 false,
1122 iss_srt,
1123 iss_sf,
1124 iss_ar,
1125 0, 0, 0, 0, 0, false);
1126 disas_set_insn_syndrome(s, syn);
1127 }
1128 }
1129
do_gpr_st(DisasContext * s,TCGv_i64 source,TCGv_i64 tcg_addr,MemOp memop,bool iss_valid,unsigned int iss_srt,bool iss_sf,bool iss_ar)1130 static void do_gpr_st(DisasContext *s, TCGv_i64 source,
1131 TCGv_i64 tcg_addr, MemOp memop,
1132 bool iss_valid,
1133 unsigned int iss_srt,
1134 bool iss_sf, bool iss_ar)
1135 {
1136 do_gpr_st_memidx(s, source, tcg_addr, memop, get_mem_index(s),
1137 iss_valid, iss_srt, iss_sf, iss_ar);
1138 }
1139
1140 /*
1141 * Load from memory to GPR register
1142 */
do_gpr_ld_memidx(DisasContext * s,TCGv_i64 dest,TCGv_i64 tcg_addr,MemOp memop,bool extend,int memidx,bool iss_valid,unsigned int iss_srt,bool iss_sf,bool iss_ar)1143 static void do_gpr_ld_memidx(DisasContext *s, TCGv_i64 dest, TCGv_i64 tcg_addr,
1144 MemOp memop, bool extend, int memidx,
1145 bool iss_valid, unsigned int iss_srt,
1146 bool iss_sf, bool iss_ar)
1147 {
1148 tcg_gen_qemu_ld_i64(dest, tcg_addr, memidx, memop);
1149
1150 if (extend && (memop & MO_SIGN)) {
1151 g_assert((memop & MO_SIZE) <= MO_32);
1152 tcg_gen_ext32u_i64(dest, dest);
1153 }
1154
1155 if (iss_valid) {
1156 uint32_t syn;
1157
1158 syn = syn_data_abort_with_iss(0,
1159 (memop & MO_SIZE),
1160 (memop & MO_SIGN) != 0,
1161 iss_srt,
1162 iss_sf,
1163 iss_ar,
1164 0, 0, 0, 0, 0, false);
1165 disas_set_insn_syndrome(s, syn);
1166 }
1167 }
1168
do_gpr_ld(DisasContext * s,TCGv_i64 dest,TCGv_i64 tcg_addr,MemOp memop,bool extend,bool iss_valid,unsigned int iss_srt,bool iss_sf,bool iss_ar)1169 static void do_gpr_ld(DisasContext *s, TCGv_i64 dest, TCGv_i64 tcg_addr,
1170 MemOp memop, bool extend,
1171 bool iss_valid, unsigned int iss_srt,
1172 bool iss_sf, bool iss_ar)
1173 {
1174 do_gpr_ld_memidx(s, dest, tcg_addr, memop, extend, get_mem_index(s),
1175 iss_valid, iss_srt, iss_sf, iss_ar);
1176 }
1177
1178 /*
1179 * Store from FP register to memory
1180 */
do_fp_st(DisasContext * s,int srcidx,TCGv_i64 tcg_addr,MemOp mop)1181 static void do_fp_st(DisasContext *s, int srcidx, TCGv_i64 tcg_addr, MemOp mop)
1182 {
1183 /* This writes the bottom N bits of a 128 bit wide vector to memory */
1184 TCGv_i64 tmplo = tcg_temp_new_i64();
1185
1186 tcg_gen_ld_i64(tmplo, tcg_env, fp_reg_offset(s, srcidx, MO_64));
1187
1188 if ((mop & MO_SIZE) < MO_128) {
1189 tcg_gen_qemu_st_i64(tmplo, tcg_addr, get_mem_index(s), mop);
1190 } else {
1191 TCGv_i64 tmphi = tcg_temp_new_i64();
1192 TCGv_i128 t16 = tcg_temp_new_i128();
1193
1194 tcg_gen_ld_i64(tmphi, tcg_env, fp_reg_hi_offset(s, srcidx));
1195 tcg_gen_concat_i64_i128(t16, tmplo, tmphi);
1196
1197 tcg_gen_qemu_st_i128(t16, tcg_addr, get_mem_index(s), mop);
1198 }
1199 }
1200
1201 /*
1202 * Load from memory to FP register
1203 */
do_fp_ld(DisasContext * s,int destidx,TCGv_i64 tcg_addr,MemOp mop)1204 static void do_fp_ld(DisasContext *s, int destidx, TCGv_i64 tcg_addr, MemOp mop)
1205 {
1206 /* This always zero-extends and writes to a full 128 bit wide vector */
1207 TCGv_i64 tmplo = tcg_temp_new_i64();
1208 TCGv_i64 tmphi = NULL;
1209
1210 if ((mop & MO_SIZE) < MO_128) {
1211 tcg_gen_qemu_ld_i64(tmplo, tcg_addr, get_mem_index(s), mop);
1212 } else {
1213 TCGv_i128 t16 = tcg_temp_new_i128();
1214
1215 tcg_gen_qemu_ld_i128(t16, tcg_addr, get_mem_index(s), mop);
1216
1217 tmphi = tcg_temp_new_i64();
1218 tcg_gen_extr_i128_i64(tmplo, tmphi, t16);
1219 }
1220
1221 tcg_gen_st_i64(tmplo, tcg_env, fp_reg_offset(s, destidx, MO_64));
1222
1223 if (tmphi) {
1224 tcg_gen_st_i64(tmphi, tcg_env, fp_reg_hi_offset(s, destidx));
1225 }
1226 clear_vec_high(s, tmphi != NULL, destidx);
1227 }
1228
1229 /*
1230 * Vector load/store helpers.
1231 *
1232 * The principal difference between this and a FP load is that we don't
1233 * zero extend as we are filling a partial chunk of the vector register.
1234 * These functions don't support 128 bit loads/stores, which would be
1235 * normal load/store operations.
1236 *
1237 * The _i32 versions are useful when operating on 32 bit quantities
1238 * (eg for floating point single or using Neon helper functions).
1239 */
1240
1241 /* Get value of an element within a vector register */
read_vec_element(DisasContext * s,TCGv_i64 tcg_dest,int srcidx,int element,MemOp memop)1242 static void read_vec_element(DisasContext *s, TCGv_i64 tcg_dest, int srcidx,
1243 int element, MemOp memop)
1244 {
1245 int vect_off = vec_reg_offset(s, srcidx, element, memop & MO_SIZE);
1246 switch ((unsigned)memop) {
1247 case MO_8:
1248 tcg_gen_ld8u_i64(tcg_dest, tcg_env, vect_off);
1249 break;
1250 case MO_16:
1251 tcg_gen_ld16u_i64(tcg_dest, tcg_env, vect_off);
1252 break;
1253 case MO_32:
1254 tcg_gen_ld32u_i64(tcg_dest, tcg_env, vect_off);
1255 break;
1256 case MO_8|MO_SIGN:
1257 tcg_gen_ld8s_i64(tcg_dest, tcg_env, vect_off);
1258 break;
1259 case MO_16|MO_SIGN:
1260 tcg_gen_ld16s_i64(tcg_dest, tcg_env, vect_off);
1261 break;
1262 case MO_32|MO_SIGN:
1263 tcg_gen_ld32s_i64(tcg_dest, tcg_env, vect_off);
1264 break;
1265 case MO_64:
1266 case MO_64|MO_SIGN:
1267 tcg_gen_ld_i64(tcg_dest, tcg_env, vect_off);
1268 break;
1269 default:
1270 g_assert_not_reached();
1271 }
1272 }
1273
read_vec_element_i32(DisasContext * s,TCGv_i32 tcg_dest,int srcidx,int element,MemOp memop)1274 static void read_vec_element_i32(DisasContext *s, TCGv_i32 tcg_dest, int srcidx,
1275 int element, MemOp memop)
1276 {
1277 int vect_off = vec_reg_offset(s, srcidx, element, memop & MO_SIZE);
1278 switch (memop) {
1279 case MO_8:
1280 tcg_gen_ld8u_i32(tcg_dest, tcg_env, vect_off);
1281 break;
1282 case MO_16:
1283 tcg_gen_ld16u_i32(tcg_dest, tcg_env, vect_off);
1284 break;
1285 case MO_8|MO_SIGN:
1286 tcg_gen_ld8s_i32(tcg_dest, tcg_env, vect_off);
1287 break;
1288 case MO_16|MO_SIGN:
1289 tcg_gen_ld16s_i32(tcg_dest, tcg_env, vect_off);
1290 break;
1291 case MO_32:
1292 case MO_32|MO_SIGN:
1293 tcg_gen_ld_i32(tcg_dest, tcg_env, vect_off);
1294 break;
1295 default:
1296 g_assert_not_reached();
1297 }
1298 }
1299
1300 /* Set value of an element within a vector register */
write_vec_element(DisasContext * s,TCGv_i64 tcg_src,int destidx,int element,MemOp memop)1301 static void write_vec_element(DisasContext *s, TCGv_i64 tcg_src, int destidx,
1302 int element, MemOp memop)
1303 {
1304 int vect_off = vec_reg_offset(s, destidx, element, memop & MO_SIZE);
1305 switch (memop) {
1306 case MO_8:
1307 tcg_gen_st8_i64(tcg_src, tcg_env, vect_off);
1308 break;
1309 case MO_16:
1310 tcg_gen_st16_i64(tcg_src, tcg_env, vect_off);
1311 break;
1312 case MO_32:
1313 tcg_gen_st32_i64(tcg_src, tcg_env, vect_off);
1314 break;
1315 case MO_64:
1316 tcg_gen_st_i64(tcg_src, tcg_env, vect_off);
1317 break;
1318 default:
1319 g_assert_not_reached();
1320 }
1321 }
1322
write_vec_element_i32(DisasContext * s,TCGv_i32 tcg_src,int destidx,int element,MemOp memop)1323 static void write_vec_element_i32(DisasContext *s, TCGv_i32 tcg_src,
1324 int destidx, int element, MemOp memop)
1325 {
1326 int vect_off = vec_reg_offset(s, destidx, element, memop & MO_SIZE);
1327 switch (memop) {
1328 case MO_8:
1329 tcg_gen_st8_i32(tcg_src, tcg_env, vect_off);
1330 break;
1331 case MO_16:
1332 tcg_gen_st16_i32(tcg_src, tcg_env, vect_off);
1333 break;
1334 case MO_32:
1335 tcg_gen_st_i32(tcg_src, tcg_env, vect_off);
1336 break;
1337 default:
1338 g_assert_not_reached();
1339 }
1340 }
1341
1342 /* Store from vector register to memory */
do_vec_st(DisasContext * s,int srcidx,int element,TCGv_i64 tcg_addr,MemOp mop)1343 static void do_vec_st(DisasContext *s, int srcidx, int element,
1344 TCGv_i64 tcg_addr, MemOp mop)
1345 {
1346 TCGv_i64 tcg_tmp = tcg_temp_new_i64();
1347
1348 read_vec_element(s, tcg_tmp, srcidx, element, mop & MO_SIZE);
1349 tcg_gen_qemu_st_i64(tcg_tmp, tcg_addr, get_mem_index(s), mop);
1350 }
1351
1352 /* Load from memory to vector register */
do_vec_ld(DisasContext * s,int destidx,int element,TCGv_i64 tcg_addr,MemOp mop)1353 static void do_vec_ld(DisasContext *s, int destidx, int element,
1354 TCGv_i64 tcg_addr, MemOp mop)
1355 {
1356 TCGv_i64 tcg_tmp = tcg_temp_new_i64();
1357
1358 tcg_gen_qemu_ld_i64(tcg_tmp, tcg_addr, get_mem_index(s), mop);
1359 write_vec_element(s, tcg_tmp, destidx, element, mop & MO_SIZE);
1360 }
1361
1362 /* Check that FP/Neon access is enabled. If it is, return
1363 * true. If not, emit code to generate an appropriate exception,
1364 * and return false; the caller should not emit any code for
1365 * the instruction. Note that this check must happen after all
1366 * unallocated-encoding checks (otherwise the syndrome information
1367 * for the resulting exception will be incorrect).
1368 */
fp_access_check_only(DisasContext * s)1369 static bool fp_access_check_only(DisasContext *s)
1370 {
1371 if (s->fp_excp_el) {
1372 assert(!s->fp_access_checked);
1373 s->fp_access_checked = -1;
1374
1375 gen_exception_insn_el(s, 0, EXCP_UDEF,
1376 syn_fp_access_trap(1, 0xe, false, 0),
1377 s->fp_excp_el);
1378 return false;
1379 }
1380 s->fp_access_checked = 1;
1381 return true;
1382 }
1383
fp_access_check(DisasContext * s)1384 static bool fp_access_check(DisasContext *s)
1385 {
1386 if (!fp_access_check_only(s)) {
1387 return false;
1388 }
1389 if (s->sme_trap_nonstreaming && s->is_nonstreaming) {
1390 gen_exception_insn(s, 0, EXCP_UDEF,
1391 syn_smetrap(SME_ET_Streaming, false));
1392 return false;
1393 }
1394 return true;
1395 }
1396
1397 /*
1398 * Return <0 for non-supported element sizes, with MO_16 controlled by
1399 * FEAT_FP16; return 0 for fp disabled; otherwise return >0 for success.
1400 */
fp_access_check_scalar_hsd(DisasContext * s,MemOp esz)1401 static int fp_access_check_scalar_hsd(DisasContext *s, MemOp esz)
1402 {
1403 switch (esz) {
1404 case MO_64:
1405 case MO_32:
1406 break;
1407 case MO_16:
1408 if (!dc_isar_feature(aa64_fp16, s)) {
1409 return -1;
1410 }
1411 break;
1412 default:
1413 return -1;
1414 }
1415 return fp_access_check(s);
1416 }
1417
1418 /* Likewise, but vector MO_64 must have two elements. */
fp_access_check_vector_hsd(DisasContext * s,bool is_q,MemOp esz)1419 static int fp_access_check_vector_hsd(DisasContext *s, bool is_q, MemOp esz)
1420 {
1421 switch (esz) {
1422 case MO_64:
1423 if (!is_q) {
1424 return -1;
1425 }
1426 break;
1427 case MO_32:
1428 break;
1429 case MO_16:
1430 if (!dc_isar_feature(aa64_fp16, s)) {
1431 return -1;
1432 }
1433 break;
1434 default:
1435 return -1;
1436 }
1437 return fp_access_check(s);
1438 }
1439
1440 /*
1441 * Check that SVE access is enabled. If it is, return true.
1442 * If not, emit code to generate an appropriate exception and return false.
1443 * This function corresponds to CheckSVEEnabled().
1444 */
sve_access_check(DisasContext * s)1445 bool sve_access_check(DisasContext *s)
1446 {
1447 if (s->pstate_sm || !dc_isar_feature(aa64_sve, s)) {
1448 bool ret;
1449
1450 assert(dc_isar_feature(aa64_sme, s));
1451 ret = sme_sm_enabled_check(s);
1452 s->sve_access_checked = (ret ? 1 : -1);
1453 return ret;
1454 }
1455 if (s->sve_excp_el) {
1456 /* Assert that we only raise one exception per instruction. */
1457 assert(!s->sve_access_checked);
1458 gen_exception_insn_el(s, 0, EXCP_UDEF,
1459 syn_sve_access_trap(), s->sve_excp_el);
1460 s->sve_access_checked = -1;
1461 return false;
1462 }
1463 s->sve_access_checked = 1;
1464 return fp_access_check(s);
1465 }
1466
1467 /*
1468 * Check that SME access is enabled, raise an exception if not.
1469 * Note that this function corresponds to CheckSMEAccess and is
1470 * only used directly for cpregs.
1471 */
sme_access_check(DisasContext * s)1472 static bool sme_access_check(DisasContext *s)
1473 {
1474 if (s->sme_excp_el) {
1475 gen_exception_insn_el(s, 0, EXCP_UDEF,
1476 syn_smetrap(SME_ET_AccessTrap, false),
1477 s->sme_excp_el);
1478 return false;
1479 }
1480 return true;
1481 }
1482
1483 /* This function corresponds to CheckSMEEnabled. */
sme_enabled_check(DisasContext * s)1484 bool sme_enabled_check(DisasContext *s)
1485 {
1486 /*
1487 * Note that unlike sve_excp_el, we have not constrained sme_excp_el
1488 * to be zero when fp_excp_el has priority. This is because we need
1489 * sme_excp_el by itself for cpregs access checks.
1490 */
1491 if (!s->fp_excp_el || s->sme_excp_el < s->fp_excp_el) {
1492 bool ret = sme_access_check(s);
1493 s->fp_access_checked = (ret ? 1 : -1);
1494 return ret;
1495 }
1496 return fp_access_check_only(s);
1497 }
1498
1499 /* Common subroutine for CheckSMEAnd*Enabled. */
sme_enabled_check_with_svcr(DisasContext * s,unsigned req)1500 bool sme_enabled_check_with_svcr(DisasContext *s, unsigned req)
1501 {
1502 if (!sme_enabled_check(s)) {
1503 return false;
1504 }
1505 if (FIELD_EX64(req, SVCR, SM) && !s->pstate_sm) {
1506 gen_exception_insn(s, 0, EXCP_UDEF,
1507 syn_smetrap(SME_ET_NotStreaming, false));
1508 return false;
1509 }
1510 if (FIELD_EX64(req, SVCR, ZA) && !s->pstate_za) {
1511 gen_exception_insn(s, 0, EXCP_UDEF,
1512 syn_smetrap(SME_ET_InactiveZA, false));
1513 return false;
1514 }
1515 return true;
1516 }
1517
1518 /*
1519 * Expanders for AdvSIMD translation functions.
1520 */
1521
do_gvec_op2_ool(DisasContext * s,arg_qrr_e * a,int data,gen_helper_gvec_2 * fn)1522 static bool do_gvec_op2_ool(DisasContext *s, arg_qrr_e *a, int data,
1523 gen_helper_gvec_2 *fn)
1524 {
1525 if (!a->q && a->esz == MO_64) {
1526 return false;
1527 }
1528 if (fp_access_check(s)) {
1529 gen_gvec_op2_ool(s, a->q, a->rd, a->rn, data, fn);
1530 }
1531 return true;
1532 }
1533
do_gvec_op3_ool(DisasContext * s,arg_qrrr_e * a,int data,gen_helper_gvec_3 * fn)1534 static bool do_gvec_op3_ool(DisasContext *s, arg_qrrr_e *a, int data,
1535 gen_helper_gvec_3 *fn)
1536 {
1537 if (!a->q && a->esz == MO_64) {
1538 return false;
1539 }
1540 if (fp_access_check(s)) {
1541 gen_gvec_op3_ool(s, a->q, a->rd, a->rn, a->rm, data, fn);
1542 }
1543 return true;
1544 }
1545
do_gvec_fn3(DisasContext * s,arg_qrrr_e * a,GVecGen3Fn * fn)1546 static bool do_gvec_fn3(DisasContext *s, arg_qrrr_e *a, GVecGen3Fn *fn)
1547 {
1548 if (!a->q && a->esz == MO_64) {
1549 return false;
1550 }
1551 if (fp_access_check(s)) {
1552 gen_gvec_fn3(s, a->q, a->rd, a->rn, a->rm, fn, a->esz);
1553 }
1554 return true;
1555 }
1556
do_gvec_fn3_no64(DisasContext * s,arg_qrrr_e * a,GVecGen3Fn * fn)1557 static bool do_gvec_fn3_no64(DisasContext *s, arg_qrrr_e *a, GVecGen3Fn *fn)
1558 {
1559 if (a->esz == MO_64) {
1560 return false;
1561 }
1562 if (fp_access_check(s)) {
1563 gen_gvec_fn3(s, a->q, a->rd, a->rn, a->rm, fn, a->esz);
1564 }
1565 return true;
1566 }
1567
do_gvec_fn3_no8_no64(DisasContext * s,arg_qrrr_e * a,GVecGen3Fn * fn)1568 static bool do_gvec_fn3_no8_no64(DisasContext *s, arg_qrrr_e *a, GVecGen3Fn *fn)
1569 {
1570 if (a->esz == MO_8) {
1571 return false;
1572 }
1573 return do_gvec_fn3_no64(s, a, fn);
1574 }
1575
do_gvec_fn4(DisasContext * s,arg_qrrrr_e * a,GVecGen4Fn * fn)1576 static bool do_gvec_fn4(DisasContext *s, arg_qrrrr_e *a, GVecGen4Fn *fn)
1577 {
1578 if (!a->q && a->esz == MO_64) {
1579 return false;
1580 }
1581 if (fp_access_check(s)) {
1582 gen_gvec_fn4(s, a->q, a->rd, a->rn, a->rm, a->ra, fn, a->esz);
1583 }
1584 return true;
1585 }
1586
1587 /*
1588 * This utility function is for doing register extension with an
1589 * optional shift. You will likely want to pass a temporary for the
1590 * destination register. See DecodeRegExtend() in the ARM ARM.
1591 */
ext_and_shift_reg(TCGv_i64 tcg_out,TCGv_i64 tcg_in,int option,unsigned int shift)1592 static void ext_and_shift_reg(TCGv_i64 tcg_out, TCGv_i64 tcg_in,
1593 int option, unsigned int shift)
1594 {
1595 int extsize = extract32(option, 0, 2);
1596 bool is_signed = extract32(option, 2, 1);
1597
1598 tcg_gen_ext_i64(tcg_out, tcg_in, extsize | (is_signed ? MO_SIGN : 0));
1599 tcg_gen_shli_i64(tcg_out, tcg_out, shift);
1600 }
1601
gen_check_sp_alignment(DisasContext * s)1602 static inline void gen_check_sp_alignment(DisasContext *s)
1603 {
1604 /* The AArch64 architecture mandates that (if enabled via PSTATE
1605 * or SCTLR bits) there is a check that SP is 16-aligned on every
1606 * SP-relative load or store (with an exception generated if it is not).
1607 * In line with general QEMU practice regarding misaligned accesses,
1608 * we omit these checks for the sake of guest program performance.
1609 * This function is provided as a hook so we can more easily add these
1610 * checks in future (possibly as a "favour catching guest program bugs
1611 * over speed" user selectable option).
1612 */
1613 }
1614
1615 /*
1616 * The instruction disassembly implemented here matches
1617 * the instruction encoding classifications in chapter C4
1618 * of the ARM Architecture Reference Manual (DDI0487B_a);
1619 * classification names and decode diagrams here should generally
1620 * match up with those in the manual.
1621 */
1622
trans_B(DisasContext * s,arg_i * a)1623 static bool trans_B(DisasContext *s, arg_i *a)
1624 {
1625 reset_btype(s);
1626 gen_goto_tb(s, 0, a->imm);
1627 return true;
1628 }
1629
trans_BL(DisasContext * s,arg_i * a)1630 static bool trans_BL(DisasContext *s, arg_i *a)
1631 {
1632 gen_pc_plus_diff(s, cpu_reg(s, 30), curr_insn_len(s));
1633 reset_btype(s);
1634 gen_goto_tb(s, 0, a->imm);
1635 return true;
1636 }
1637
1638
trans_CBZ(DisasContext * s,arg_cbz * a)1639 static bool trans_CBZ(DisasContext *s, arg_cbz *a)
1640 {
1641 DisasLabel match;
1642 TCGv_i64 tcg_cmp;
1643
1644 tcg_cmp = read_cpu_reg(s, a->rt, a->sf);
1645 reset_btype(s);
1646
1647 match = gen_disas_label(s);
1648 tcg_gen_brcondi_i64(a->nz ? TCG_COND_NE : TCG_COND_EQ,
1649 tcg_cmp, 0, match.label);
1650 gen_goto_tb(s, 0, 4);
1651 set_disas_label(s, match);
1652 gen_goto_tb(s, 1, a->imm);
1653 return true;
1654 }
1655
trans_TBZ(DisasContext * s,arg_tbz * a)1656 static bool trans_TBZ(DisasContext *s, arg_tbz *a)
1657 {
1658 DisasLabel match;
1659 TCGv_i64 tcg_cmp;
1660
1661 tcg_cmp = tcg_temp_new_i64();
1662 tcg_gen_andi_i64(tcg_cmp, cpu_reg(s, a->rt), 1ULL << a->bitpos);
1663
1664 reset_btype(s);
1665
1666 match = gen_disas_label(s);
1667 tcg_gen_brcondi_i64(a->nz ? TCG_COND_NE : TCG_COND_EQ,
1668 tcg_cmp, 0, match.label);
1669 gen_goto_tb(s, 0, 4);
1670 set_disas_label(s, match);
1671 gen_goto_tb(s, 1, a->imm);
1672 return true;
1673 }
1674
trans_B_cond(DisasContext * s,arg_B_cond * a)1675 static bool trans_B_cond(DisasContext *s, arg_B_cond *a)
1676 {
1677 /* BC.cond is only present with FEAT_HBC */
1678 if (a->c && !dc_isar_feature(aa64_hbc, s)) {
1679 return false;
1680 }
1681 reset_btype(s);
1682 if (a->cond < 0x0e) {
1683 /* genuinely conditional branches */
1684 DisasLabel match = gen_disas_label(s);
1685 arm_gen_test_cc(a->cond, match.label);
1686 gen_goto_tb(s, 0, 4);
1687 set_disas_label(s, match);
1688 gen_goto_tb(s, 1, a->imm);
1689 } else {
1690 /* 0xe and 0xf are both "always" conditions */
1691 gen_goto_tb(s, 0, a->imm);
1692 }
1693 return true;
1694 }
1695
set_btype_for_br(DisasContext * s,int rn)1696 static void set_btype_for_br(DisasContext *s, int rn)
1697 {
1698 if (dc_isar_feature(aa64_bti, s)) {
1699 /* BR to {x16,x17} or !guard -> 1, else 3. */
1700 if (rn == 16 || rn == 17) {
1701 set_btype(s, 1);
1702 } else {
1703 TCGv_i64 pc = tcg_temp_new_i64();
1704 gen_pc_plus_diff(s, pc, 0);
1705 gen_helper_guarded_page_br(tcg_env, pc);
1706 s->btype = -1;
1707 }
1708 }
1709 }
1710
set_btype_for_blr(DisasContext * s)1711 static void set_btype_for_blr(DisasContext *s)
1712 {
1713 if (dc_isar_feature(aa64_bti, s)) {
1714 /* BLR sets BTYPE to 2, regardless of source guarded page. */
1715 set_btype(s, 2);
1716 }
1717 }
1718
trans_BR(DisasContext * s,arg_r * a)1719 static bool trans_BR(DisasContext *s, arg_r *a)
1720 {
1721 set_btype_for_br(s, a->rn);
1722 gen_a64_set_pc(s, cpu_reg(s, a->rn));
1723 s->base.is_jmp = DISAS_JUMP;
1724 return true;
1725 }
1726
trans_BLR(DisasContext * s,arg_r * a)1727 static bool trans_BLR(DisasContext *s, arg_r *a)
1728 {
1729 TCGv_i64 dst = cpu_reg(s, a->rn);
1730 TCGv_i64 lr = cpu_reg(s, 30);
1731 if (dst == lr) {
1732 TCGv_i64 tmp = tcg_temp_new_i64();
1733 tcg_gen_mov_i64(tmp, dst);
1734 dst = tmp;
1735 }
1736 gen_pc_plus_diff(s, lr, curr_insn_len(s));
1737 gen_a64_set_pc(s, dst);
1738 set_btype_for_blr(s);
1739 s->base.is_jmp = DISAS_JUMP;
1740 return true;
1741 }
1742
trans_RET(DisasContext * s,arg_r * a)1743 static bool trans_RET(DisasContext *s, arg_r *a)
1744 {
1745 gen_a64_set_pc(s, cpu_reg(s, a->rn));
1746 s->base.is_jmp = DISAS_JUMP;
1747 return true;
1748 }
1749
auth_branch_target(DisasContext * s,TCGv_i64 dst,TCGv_i64 modifier,bool use_key_a)1750 static TCGv_i64 auth_branch_target(DisasContext *s, TCGv_i64 dst,
1751 TCGv_i64 modifier, bool use_key_a)
1752 {
1753 TCGv_i64 truedst;
1754 /*
1755 * Return the branch target for a BRAA/RETA/etc, which is either
1756 * just the destination dst, or that value with the pauth check
1757 * done and the code removed from the high bits.
1758 */
1759 if (!s->pauth_active) {
1760 return dst;
1761 }
1762
1763 truedst = tcg_temp_new_i64();
1764 if (use_key_a) {
1765 gen_helper_autia_combined(truedst, tcg_env, dst, modifier);
1766 } else {
1767 gen_helper_autib_combined(truedst, tcg_env, dst, modifier);
1768 }
1769 return truedst;
1770 }
1771
trans_BRAZ(DisasContext * s,arg_braz * a)1772 static bool trans_BRAZ(DisasContext *s, arg_braz *a)
1773 {
1774 TCGv_i64 dst;
1775
1776 if (!dc_isar_feature(aa64_pauth, s)) {
1777 return false;
1778 }
1779
1780 dst = auth_branch_target(s, cpu_reg(s, a->rn), tcg_constant_i64(0), !a->m);
1781 set_btype_for_br(s, a->rn);
1782 gen_a64_set_pc(s, dst);
1783 s->base.is_jmp = DISAS_JUMP;
1784 return true;
1785 }
1786
trans_BLRAZ(DisasContext * s,arg_braz * a)1787 static bool trans_BLRAZ(DisasContext *s, arg_braz *a)
1788 {
1789 TCGv_i64 dst, lr;
1790
1791 if (!dc_isar_feature(aa64_pauth, s)) {
1792 return false;
1793 }
1794
1795 dst = auth_branch_target(s, cpu_reg(s, a->rn), tcg_constant_i64(0), !a->m);
1796 lr = cpu_reg(s, 30);
1797 if (dst == lr) {
1798 TCGv_i64 tmp = tcg_temp_new_i64();
1799 tcg_gen_mov_i64(tmp, dst);
1800 dst = tmp;
1801 }
1802 gen_pc_plus_diff(s, lr, curr_insn_len(s));
1803 gen_a64_set_pc(s, dst);
1804 set_btype_for_blr(s);
1805 s->base.is_jmp = DISAS_JUMP;
1806 return true;
1807 }
1808
trans_RETA(DisasContext * s,arg_reta * a)1809 static bool trans_RETA(DisasContext *s, arg_reta *a)
1810 {
1811 TCGv_i64 dst;
1812
1813 if (!dc_isar_feature(aa64_pauth, s)) {
1814 return false;
1815 }
1816
1817 dst = auth_branch_target(s, cpu_reg(s, 30), cpu_X[31], !a->m);
1818 gen_a64_set_pc(s, dst);
1819 s->base.is_jmp = DISAS_JUMP;
1820 return true;
1821 }
1822
trans_BRA(DisasContext * s,arg_bra * a)1823 static bool trans_BRA(DisasContext *s, arg_bra *a)
1824 {
1825 TCGv_i64 dst;
1826
1827 if (!dc_isar_feature(aa64_pauth, s)) {
1828 return false;
1829 }
1830 dst = auth_branch_target(s, cpu_reg(s,a->rn), cpu_reg_sp(s, a->rm), !a->m);
1831 gen_a64_set_pc(s, dst);
1832 set_btype_for_br(s, a->rn);
1833 s->base.is_jmp = DISAS_JUMP;
1834 return true;
1835 }
1836
trans_BLRA(DisasContext * s,arg_bra * a)1837 static bool trans_BLRA(DisasContext *s, arg_bra *a)
1838 {
1839 TCGv_i64 dst, lr;
1840
1841 if (!dc_isar_feature(aa64_pauth, s)) {
1842 return false;
1843 }
1844 dst = auth_branch_target(s, cpu_reg(s, a->rn), cpu_reg_sp(s, a->rm), !a->m);
1845 lr = cpu_reg(s, 30);
1846 if (dst == lr) {
1847 TCGv_i64 tmp = tcg_temp_new_i64();
1848 tcg_gen_mov_i64(tmp, dst);
1849 dst = tmp;
1850 }
1851 gen_pc_plus_diff(s, lr, curr_insn_len(s));
1852 gen_a64_set_pc(s, dst);
1853 set_btype_for_blr(s);
1854 s->base.is_jmp = DISAS_JUMP;
1855 return true;
1856 }
1857
trans_ERET(DisasContext * s,arg_ERET * a)1858 static bool trans_ERET(DisasContext *s, arg_ERET *a)
1859 {
1860 TCGv_i64 dst;
1861
1862 if (s->current_el == 0) {
1863 return false;
1864 }
1865 if (s->trap_eret) {
1866 gen_exception_insn_el(s, 0, EXCP_UDEF, syn_erettrap(0), 2);
1867 return true;
1868 }
1869 dst = tcg_temp_new_i64();
1870 tcg_gen_ld_i64(dst, tcg_env,
1871 offsetof(CPUARMState, elr_el[s->current_el]));
1872
1873 translator_io_start(&s->base);
1874
1875 gen_helper_exception_return(tcg_env, dst);
1876 /* Must exit loop to check un-masked IRQs */
1877 s->base.is_jmp = DISAS_EXIT;
1878 return true;
1879 }
1880
trans_ERETA(DisasContext * s,arg_reta * a)1881 static bool trans_ERETA(DisasContext *s, arg_reta *a)
1882 {
1883 TCGv_i64 dst;
1884
1885 if (!dc_isar_feature(aa64_pauth, s)) {
1886 return false;
1887 }
1888 if (s->current_el == 0) {
1889 return false;
1890 }
1891 /* The FGT trap takes precedence over an auth trap. */
1892 if (s->trap_eret) {
1893 gen_exception_insn_el(s, 0, EXCP_UDEF, syn_erettrap(a->m ? 3 : 2), 2);
1894 return true;
1895 }
1896 dst = tcg_temp_new_i64();
1897 tcg_gen_ld_i64(dst, tcg_env,
1898 offsetof(CPUARMState, elr_el[s->current_el]));
1899
1900 dst = auth_branch_target(s, dst, cpu_X[31], !a->m);
1901
1902 translator_io_start(&s->base);
1903
1904 gen_helper_exception_return(tcg_env, dst);
1905 /* Must exit loop to check un-masked IRQs */
1906 s->base.is_jmp = DISAS_EXIT;
1907 return true;
1908 }
1909
trans_NOP(DisasContext * s,arg_NOP * a)1910 static bool trans_NOP(DisasContext *s, arg_NOP *a)
1911 {
1912 return true;
1913 }
1914
trans_YIELD(DisasContext * s,arg_YIELD * a)1915 static bool trans_YIELD(DisasContext *s, arg_YIELD *a)
1916 {
1917 /*
1918 * When running in MTTCG we don't generate jumps to the yield and
1919 * WFE helpers as it won't affect the scheduling of other vCPUs.
1920 * If we wanted to more completely model WFE/SEV so we don't busy
1921 * spin unnecessarily we would need to do something more involved.
1922 */
1923 if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) {
1924 s->base.is_jmp = DISAS_YIELD;
1925 }
1926 return true;
1927 }
1928
trans_WFI(DisasContext * s,arg_WFI * a)1929 static bool trans_WFI(DisasContext *s, arg_WFI *a)
1930 {
1931 s->base.is_jmp = DISAS_WFI;
1932 return true;
1933 }
1934
trans_WFE(DisasContext * s,arg_WFI * a)1935 static bool trans_WFE(DisasContext *s, arg_WFI *a)
1936 {
1937 /*
1938 * When running in MTTCG we don't generate jumps to the yield and
1939 * WFE helpers as it won't affect the scheduling of other vCPUs.
1940 * If we wanted to more completely model WFE/SEV so we don't busy
1941 * spin unnecessarily we would need to do something more involved.
1942 */
1943 if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) {
1944 s->base.is_jmp = DISAS_WFE;
1945 }
1946 return true;
1947 }
1948
trans_WFIT(DisasContext * s,arg_WFIT * a)1949 static bool trans_WFIT(DisasContext *s, arg_WFIT *a)
1950 {
1951 if (!dc_isar_feature(aa64_wfxt, s)) {
1952 return false;
1953 }
1954
1955 /*
1956 * Because we need to pass the register value to the helper,
1957 * it's easier to emit the code now, unlike trans_WFI which
1958 * defers it to aarch64_tr_tb_stop(). That means we need to
1959 * check ss_active so that single-stepping a WFIT doesn't halt.
1960 */
1961 if (s->ss_active) {
1962 /* Act like a NOP under architectural singlestep */
1963 return true;
1964 }
1965
1966 gen_a64_update_pc(s, 4);
1967 gen_helper_wfit(tcg_env, cpu_reg(s, a->rd));
1968 /* Go back to the main loop to check for interrupts */
1969 s->base.is_jmp = DISAS_EXIT;
1970 return true;
1971 }
1972
trans_WFET(DisasContext * s,arg_WFET * a)1973 static bool trans_WFET(DisasContext *s, arg_WFET *a)
1974 {
1975 if (!dc_isar_feature(aa64_wfxt, s)) {
1976 return false;
1977 }
1978
1979 /*
1980 * We rely here on our WFE implementation being a NOP, so we
1981 * don't need to do anything different to handle the WFET timeout
1982 * from what trans_WFE does.
1983 */
1984 if (!(tb_cflags(s->base.tb) & CF_PARALLEL)) {
1985 s->base.is_jmp = DISAS_WFE;
1986 }
1987 return true;
1988 }
1989
trans_XPACLRI(DisasContext * s,arg_XPACLRI * a)1990 static bool trans_XPACLRI(DisasContext *s, arg_XPACLRI *a)
1991 {
1992 if (s->pauth_active) {
1993 gen_helper_xpaci(cpu_X[30], tcg_env, cpu_X[30]);
1994 }
1995 return true;
1996 }
1997
trans_PACIA1716(DisasContext * s,arg_PACIA1716 * a)1998 static bool trans_PACIA1716(DisasContext *s, arg_PACIA1716 *a)
1999 {
2000 if (s->pauth_active) {
2001 gen_helper_pacia(cpu_X[17], tcg_env, cpu_X[17], cpu_X[16]);
2002 }
2003 return true;
2004 }
2005
trans_PACIB1716(DisasContext * s,arg_PACIB1716 * a)2006 static bool trans_PACIB1716(DisasContext *s, arg_PACIB1716 *a)
2007 {
2008 if (s->pauth_active) {
2009 gen_helper_pacib(cpu_X[17], tcg_env, cpu_X[17], cpu_X[16]);
2010 }
2011 return true;
2012 }
2013
trans_AUTIA1716(DisasContext * s,arg_AUTIA1716 * a)2014 static bool trans_AUTIA1716(DisasContext *s, arg_AUTIA1716 *a)
2015 {
2016 if (s->pauth_active) {
2017 gen_helper_autia(cpu_X[17], tcg_env, cpu_X[17], cpu_X[16]);
2018 }
2019 return true;
2020 }
2021
trans_AUTIB1716(DisasContext * s,arg_AUTIB1716 * a)2022 static bool trans_AUTIB1716(DisasContext *s, arg_AUTIB1716 *a)
2023 {
2024 if (s->pauth_active) {
2025 gen_helper_autib(cpu_X[17], tcg_env, cpu_X[17], cpu_X[16]);
2026 }
2027 return true;
2028 }
2029
trans_ESB(DisasContext * s,arg_ESB * a)2030 static bool trans_ESB(DisasContext *s, arg_ESB *a)
2031 {
2032 /* Without RAS, we must implement this as NOP. */
2033 if (dc_isar_feature(aa64_ras, s)) {
2034 /*
2035 * QEMU does not have a source of physical SErrors,
2036 * so we are only concerned with virtual SErrors.
2037 * The pseudocode in the ARM for this case is
2038 * if PSTATE.EL IN {EL0, EL1} && EL2Enabled() then
2039 * AArch64.vESBOperation();
2040 * Most of the condition can be evaluated at translation time.
2041 * Test for EL2 present, and defer test for SEL2 to runtime.
2042 */
2043 if (s->current_el <= 1 && arm_dc_feature(s, ARM_FEATURE_EL2)) {
2044 gen_helper_vesb(tcg_env);
2045 }
2046 }
2047 return true;
2048 }
2049
trans_PACIAZ(DisasContext * s,arg_PACIAZ * a)2050 static bool trans_PACIAZ(DisasContext *s, arg_PACIAZ *a)
2051 {
2052 if (s->pauth_active) {
2053 gen_helper_pacia(cpu_X[30], tcg_env, cpu_X[30], tcg_constant_i64(0));
2054 }
2055 return true;
2056 }
2057
trans_PACIASP(DisasContext * s,arg_PACIASP * a)2058 static bool trans_PACIASP(DisasContext *s, arg_PACIASP *a)
2059 {
2060 if (s->pauth_active) {
2061 gen_helper_pacia(cpu_X[30], tcg_env, cpu_X[30], cpu_X[31]);
2062 }
2063 return true;
2064 }
2065
trans_PACIBZ(DisasContext * s,arg_PACIBZ * a)2066 static bool trans_PACIBZ(DisasContext *s, arg_PACIBZ *a)
2067 {
2068 if (s->pauth_active) {
2069 gen_helper_pacib(cpu_X[30], tcg_env, cpu_X[30], tcg_constant_i64(0));
2070 }
2071 return true;
2072 }
2073
trans_PACIBSP(DisasContext * s,arg_PACIBSP * a)2074 static bool trans_PACIBSP(DisasContext *s, arg_PACIBSP *a)
2075 {
2076 if (s->pauth_active) {
2077 gen_helper_pacib(cpu_X[30], tcg_env, cpu_X[30], cpu_X[31]);
2078 }
2079 return true;
2080 }
2081
trans_AUTIAZ(DisasContext * s,arg_AUTIAZ * a)2082 static bool trans_AUTIAZ(DisasContext *s, arg_AUTIAZ *a)
2083 {
2084 if (s->pauth_active) {
2085 gen_helper_autia(cpu_X[30], tcg_env, cpu_X[30], tcg_constant_i64(0));
2086 }
2087 return true;
2088 }
2089
trans_AUTIASP(DisasContext * s,arg_AUTIASP * a)2090 static bool trans_AUTIASP(DisasContext *s, arg_AUTIASP *a)
2091 {
2092 if (s->pauth_active) {
2093 gen_helper_autia(cpu_X[30], tcg_env, cpu_X[30], cpu_X[31]);
2094 }
2095 return true;
2096 }
2097
trans_AUTIBZ(DisasContext * s,arg_AUTIBZ * a)2098 static bool trans_AUTIBZ(DisasContext *s, arg_AUTIBZ *a)
2099 {
2100 if (s->pauth_active) {
2101 gen_helper_autib(cpu_X[30], tcg_env, cpu_X[30], tcg_constant_i64(0));
2102 }
2103 return true;
2104 }
2105
trans_AUTIBSP(DisasContext * s,arg_AUTIBSP * a)2106 static bool trans_AUTIBSP(DisasContext *s, arg_AUTIBSP *a)
2107 {
2108 if (s->pauth_active) {
2109 gen_helper_autib(cpu_X[30], tcg_env, cpu_X[30], cpu_X[31]);
2110 }
2111 return true;
2112 }
2113
trans_CLREX(DisasContext * s,arg_CLREX * a)2114 static bool trans_CLREX(DisasContext *s, arg_CLREX *a)
2115 {
2116 tcg_gen_movi_i64(cpu_exclusive_addr, -1);
2117 return true;
2118 }
2119
trans_DSB_DMB(DisasContext * s,arg_DSB_DMB * a)2120 static bool trans_DSB_DMB(DisasContext *s, arg_DSB_DMB *a)
2121 {
2122 /* We handle DSB and DMB the same way */
2123 TCGBar bar;
2124
2125 switch (a->types) {
2126 case 1: /* MBReqTypes_Reads */
2127 bar = TCG_BAR_SC | TCG_MO_LD_LD | TCG_MO_LD_ST;
2128 break;
2129 case 2: /* MBReqTypes_Writes */
2130 bar = TCG_BAR_SC | TCG_MO_ST_ST;
2131 break;
2132 default: /* MBReqTypes_All */
2133 bar = TCG_BAR_SC | TCG_MO_ALL;
2134 break;
2135 }
2136 tcg_gen_mb(bar);
2137 return true;
2138 }
2139
trans_DSB_nXS(DisasContext * s,arg_DSB_nXS * a)2140 static bool trans_DSB_nXS(DisasContext *s, arg_DSB_nXS *a)
2141 {
2142 if (!dc_isar_feature(aa64_xs, s)) {
2143 return false;
2144 }
2145 tcg_gen_mb(TCG_BAR_SC | TCG_MO_ALL);
2146 return true;
2147 }
2148
trans_ISB(DisasContext * s,arg_ISB * a)2149 static bool trans_ISB(DisasContext *s, arg_ISB *a)
2150 {
2151 /*
2152 * We need to break the TB after this insn to execute
2153 * self-modifying code correctly and also to take
2154 * any pending interrupts immediately.
2155 */
2156 reset_btype(s);
2157 gen_goto_tb(s, 0, 4);
2158 return true;
2159 }
2160
trans_SB(DisasContext * s,arg_SB * a)2161 static bool trans_SB(DisasContext *s, arg_SB *a)
2162 {
2163 if (!dc_isar_feature(aa64_sb, s)) {
2164 return false;
2165 }
2166 /*
2167 * TODO: There is no speculation barrier opcode for TCG;
2168 * MB and end the TB instead.
2169 */
2170 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_SC);
2171 gen_goto_tb(s, 0, 4);
2172 return true;
2173 }
2174
trans_CFINV(DisasContext * s,arg_CFINV * a)2175 static bool trans_CFINV(DisasContext *s, arg_CFINV *a)
2176 {
2177 if (!dc_isar_feature(aa64_condm_4, s)) {
2178 return false;
2179 }
2180 tcg_gen_xori_i32(cpu_CF, cpu_CF, 1);
2181 return true;
2182 }
2183
trans_XAFLAG(DisasContext * s,arg_XAFLAG * a)2184 static bool trans_XAFLAG(DisasContext *s, arg_XAFLAG *a)
2185 {
2186 TCGv_i32 z;
2187
2188 if (!dc_isar_feature(aa64_condm_5, s)) {
2189 return false;
2190 }
2191
2192 z = tcg_temp_new_i32();
2193
2194 tcg_gen_setcondi_i32(TCG_COND_EQ, z, cpu_ZF, 0);
2195
2196 /*
2197 * (!C & !Z) << 31
2198 * (!(C | Z)) << 31
2199 * ~((C | Z) << 31)
2200 * ~-(C | Z)
2201 * (C | Z) - 1
2202 */
2203 tcg_gen_or_i32(cpu_NF, cpu_CF, z);
2204 tcg_gen_subi_i32(cpu_NF, cpu_NF, 1);
2205
2206 /* !(Z & C) */
2207 tcg_gen_and_i32(cpu_ZF, z, cpu_CF);
2208 tcg_gen_xori_i32(cpu_ZF, cpu_ZF, 1);
2209
2210 /* (!C & Z) << 31 -> -(Z & ~C) */
2211 tcg_gen_andc_i32(cpu_VF, z, cpu_CF);
2212 tcg_gen_neg_i32(cpu_VF, cpu_VF);
2213
2214 /* C | Z */
2215 tcg_gen_or_i32(cpu_CF, cpu_CF, z);
2216
2217 return true;
2218 }
2219
trans_AXFLAG(DisasContext * s,arg_AXFLAG * a)2220 static bool trans_AXFLAG(DisasContext *s, arg_AXFLAG *a)
2221 {
2222 if (!dc_isar_feature(aa64_condm_5, s)) {
2223 return false;
2224 }
2225
2226 tcg_gen_sari_i32(cpu_VF, cpu_VF, 31); /* V ? -1 : 0 */
2227 tcg_gen_andc_i32(cpu_CF, cpu_CF, cpu_VF); /* C & !V */
2228
2229 /* !(Z | V) -> !(!ZF | V) -> ZF & !V -> ZF & ~VF */
2230 tcg_gen_andc_i32(cpu_ZF, cpu_ZF, cpu_VF);
2231
2232 tcg_gen_movi_i32(cpu_NF, 0);
2233 tcg_gen_movi_i32(cpu_VF, 0);
2234
2235 return true;
2236 }
2237
trans_MSR_i_UAO(DisasContext * s,arg_i * a)2238 static bool trans_MSR_i_UAO(DisasContext *s, arg_i *a)
2239 {
2240 if (!dc_isar_feature(aa64_uao, s) || s->current_el == 0) {
2241 return false;
2242 }
2243 if (a->imm & 1) {
2244 set_pstate_bits(PSTATE_UAO);
2245 } else {
2246 clear_pstate_bits(PSTATE_UAO);
2247 }
2248 gen_rebuild_hflags(s);
2249 s->base.is_jmp = DISAS_TOO_MANY;
2250 return true;
2251 }
2252
trans_MSR_i_PAN(DisasContext * s,arg_i * a)2253 static bool trans_MSR_i_PAN(DisasContext *s, arg_i *a)
2254 {
2255 if (!dc_isar_feature(aa64_pan, s) || s->current_el == 0) {
2256 return false;
2257 }
2258 if (a->imm & 1) {
2259 set_pstate_bits(PSTATE_PAN);
2260 } else {
2261 clear_pstate_bits(PSTATE_PAN);
2262 }
2263 gen_rebuild_hflags(s);
2264 s->base.is_jmp = DISAS_TOO_MANY;
2265 return true;
2266 }
2267
trans_MSR_i_SPSEL(DisasContext * s,arg_i * a)2268 static bool trans_MSR_i_SPSEL(DisasContext *s, arg_i *a)
2269 {
2270 if (s->current_el == 0) {
2271 return false;
2272 }
2273 gen_helper_msr_i_spsel(tcg_env, tcg_constant_i32(a->imm & PSTATE_SP));
2274 s->base.is_jmp = DISAS_TOO_MANY;
2275 return true;
2276 }
2277
trans_MSR_i_SBSS(DisasContext * s,arg_i * a)2278 static bool trans_MSR_i_SBSS(DisasContext *s, arg_i *a)
2279 {
2280 if (!dc_isar_feature(aa64_ssbs, s)) {
2281 return false;
2282 }
2283 if (a->imm & 1) {
2284 set_pstate_bits(PSTATE_SSBS);
2285 } else {
2286 clear_pstate_bits(PSTATE_SSBS);
2287 }
2288 /* Don't need to rebuild hflags since SSBS is a nop */
2289 s->base.is_jmp = DISAS_TOO_MANY;
2290 return true;
2291 }
2292
trans_MSR_i_DIT(DisasContext * s,arg_i * a)2293 static bool trans_MSR_i_DIT(DisasContext *s, arg_i *a)
2294 {
2295 if (!dc_isar_feature(aa64_dit, s)) {
2296 return false;
2297 }
2298 if (a->imm & 1) {
2299 set_pstate_bits(PSTATE_DIT);
2300 } else {
2301 clear_pstate_bits(PSTATE_DIT);
2302 }
2303 /* There's no need to rebuild hflags because DIT is a nop */
2304 s->base.is_jmp = DISAS_TOO_MANY;
2305 return true;
2306 }
2307
trans_MSR_i_TCO(DisasContext * s,arg_i * a)2308 static bool trans_MSR_i_TCO(DisasContext *s, arg_i *a)
2309 {
2310 if (dc_isar_feature(aa64_mte, s)) {
2311 /* Full MTE is enabled -- set the TCO bit as directed. */
2312 if (a->imm & 1) {
2313 set_pstate_bits(PSTATE_TCO);
2314 } else {
2315 clear_pstate_bits(PSTATE_TCO);
2316 }
2317 gen_rebuild_hflags(s);
2318 /* Many factors, including TCO, go into MTE_ACTIVE. */
2319 s->base.is_jmp = DISAS_UPDATE_NOCHAIN;
2320 return true;
2321 } else if (dc_isar_feature(aa64_mte_insn_reg, s)) {
2322 /* Only "instructions accessible at EL0" -- PSTATE.TCO is WI. */
2323 return true;
2324 } else {
2325 /* Insn not present */
2326 return false;
2327 }
2328 }
2329
trans_MSR_i_DAIFSET(DisasContext * s,arg_i * a)2330 static bool trans_MSR_i_DAIFSET(DisasContext *s, arg_i *a)
2331 {
2332 gen_helper_msr_i_daifset(tcg_env, tcg_constant_i32(a->imm));
2333 s->base.is_jmp = DISAS_TOO_MANY;
2334 return true;
2335 }
2336
trans_MSR_i_DAIFCLEAR(DisasContext * s,arg_i * a)2337 static bool trans_MSR_i_DAIFCLEAR(DisasContext *s, arg_i *a)
2338 {
2339 gen_helper_msr_i_daifclear(tcg_env, tcg_constant_i32(a->imm));
2340 /* Exit the cpu loop to re-evaluate pending IRQs. */
2341 s->base.is_jmp = DISAS_UPDATE_EXIT;
2342 return true;
2343 }
2344
trans_MSR_i_ALLINT(DisasContext * s,arg_i * a)2345 static bool trans_MSR_i_ALLINT(DisasContext *s, arg_i *a)
2346 {
2347 if (!dc_isar_feature(aa64_nmi, s) || s->current_el == 0) {
2348 return false;
2349 }
2350
2351 if (a->imm == 0) {
2352 clear_pstate_bits(PSTATE_ALLINT);
2353 } else if (s->current_el > 1) {
2354 set_pstate_bits(PSTATE_ALLINT);
2355 } else {
2356 gen_helper_msr_set_allint_el1(tcg_env);
2357 }
2358
2359 /* Exit the cpu loop to re-evaluate pending IRQs. */
2360 s->base.is_jmp = DISAS_UPDATE_EXIT;
2361 return true;
2362 }
2363
trans_MSR_i_SVCR(DisasContext * s,arg_MSR_i_SVCR * a)2364 static bool trans_MSR_i_SVCR(DisasContext *s, arg_MSR_i_SVCR *a)
2365 {
2366 if (!dc_isar_feature(aa64_sme, s) || a->mask == 0) {
2367 return false;
2368 }
2369 if (sme_access_check(s)) {
2370 int old = s->pstate_sm | (s->pstate_za << 1);
2371 int new = a->imm * 3;
2372
2373 if ((old ^ new) & a->mask) {
2374 /* At least one bit changes. */
2375 gen_helper_set_svcr(tcg_env, tcg_constant_i32(new),
2376 tcg_constant_i32(a->mask));
2377 s->base.is_jmp = DISAS_TOO_MANY;
2378 }
2379 }
2380 return true;
2381 }
2382
gen_get_nzcv(TCGv_i64 tcg_rt)2383 static void gen_get_nzcv(TCGv_i64 tcg_rt)
2384 {
2385 TCGv_i32 tmp = tcg_temp_new_i32();
2386 TCGv_i32 nzcv = tcg_temp_new_i32();
2387
2388 /* build bit 31, N */
2389 tcg_gen_andi_i32(nzcv, cpu_NF, (1U << 31));
2390 /* build bit 30, Z */
2391 tcg_gen_setcondi_i32(TCG_COND_EQ, tmp, cpu_ZF, 0);
2392 tcg_gen_deposit_i32(nzcv, nzcv, tmp, 30, 1);
2393 /* build bit 29, C */
2394 tcg_gen_deposit_i32(nzcv, nzcv, cpu_CF, 29, 1);
2395 /* build bit 28, V */
2396 tcg_gen_shri_i32(tmp, cpu_VF, 31);
2397 tcg_gen_deposit_i32(nzcv, nzcv, tmp, 28, 1);
2398 /* generate result */
2399 tcg_gen_extu_i32_i64(tcg_rt, nzcv);
2400 }
2401
gen_set_nzcv(TCGv_i64 tcg_rt)2402 static void gen_set_nzcv(TCGv_i64 tcg_rt)
2403 {
2404 TCGv_i32 nzcv = tcg_temp_new_i32();
2405
2406 /* take NZCV from R[t] */
2407 tcg_gen_extrl_i64_i32(nzcv, tcg_rt);
2408
2409 /* bit 31, N */
2410 tcg_gen_andi_i32(cpu_NF, nzcv, (1U << 31));
2411 /* bit 30, Z */
2412 tcg_gen_andi_i32(cpu_ZF, nzcv, (1 << 30));
2413 tcg_gen_setcondi_i32(TCG_COND_EQ, cpu_ZF, cpu_ZF, 0);
2414 /* bit 29, C */
2415 tcg_gen_andi_i32(cpu_CF, nzcv, (1 << 29));
2416 tcg_gen_shri_i32(cpu_CF, cpu_CF, 29);
2417 /* bit 28, V */
2418 tcg_gen_andi_i32(cpu_VF, nzcv, (1 << 28));
2419 tcg_gen_shli_i32(cpu_VF, cpu_VF, 3);
2420 }
2421
gen_sysreg_undef(DisasContext * s,bool isread,uint8_t op0,uint8_t op1,uint8_t op2,uint8_t crn,uint8_t crm,uint8_t rt)2422 static void gen_sysreg_undef(DisasContext *s, bool isread,
2423 uint8_t op0, uint8_t op1, uint8_t op2,
2424 uint8_t crn, uint8_t crm, uint8_t rt)
2425 {
2426 /*
2427 * Generate code to emit an UNDEF with correct syndrome
2428 * information for a failed system register access.
2429 * This is EC_UNCATEGORIZED (ie a standard UNDEF) in most cases,
2430 * but if FEAT_IDST is implemented then read accesses to registers
2431 * in the feature ID space are reported with the EC_SYSTEMREGISTERTRAP
2432 * syndrome.
2433 */
2434 uint32_t syndrome;
2435
2436 if (isread && dc_isar_feature(aa64_ids, s) &&
2437 arm_cpreg_encoding_in_idspace(op0, op1, op2, crn, crm)) {
2438 syndrome = syn_aa64_sysregtrap(op0, op1, op2, crn, crm, rt, isread);
2439 } else {
2440 syndrome = syn_uncategorized();
2441 }
2442 gen_exception_insn(s, 0, EXCP_UDEF, syndrome);
2443 }
2444
2445 /* MRS - move from system register
2446 * MSR (register) - move to system register
2447 * SYS
2448 * SYSL
2449 * These are all essentially the same insn in 'read' and 'write'
2450 * versions, with varying op0 fields.
2451 */
handle_sys(DisasContext * s,bool isread,unsigned int op0,unsigned int op1,unsigned int op2,unsigned int crn,unsigned int crm,unsigned int rt)2452 static void handle_sys(DisasContext *s, bool isread,
2453 unsigned int op0, unsigned int op1, unsigned int op2,
2454 unsigned int crn, unsigned int crm, unsigned int rt)
2455 {
2456 uint32_t key = ENCODE_AA64_CP_REG(CP_REG_ARM64_SYSREG_CP,
2457 crn, crm, op0, op1, op2);
2458 const ARMCPRegInfo *ri = get_arm_cp_reginfo(s->cp_regs, key);
2459 bool need_exit_tb = false;
2460 bool nv_trap_to_el2 = false;
2461 bool nv_redirect_reg = false;
2462 bool skip_fp_access_checks = false;
2463 bool nv2_mem_redirect = false;
2464 TCGv_ptr tcg_ri = NULL;
2465 TCGv_i64 tcg_rt;
2466 uint32_t syndrome = syn_aa64_sysregtrap(op0, op1, op2, crn, crm, rt, isread);
2467
2468 if (crn == 11 || crn == 15) {
2469 /*
2470 * Check for TIDCP trap, which must take precedence over
2471 * the UNDEF for "no such register" etc.
2472 */
2473 switch (s->current_el) {
2474 case 0:
2475 if (dc_isar_feature(aa64_tidcp1, s)) {
2476 gen_helper_tidcp_el0(tcg_env, tcg_constant_i32(syndrome));
2477 }
2478 break;
2479 case 1:
2480 gen_helper_tidcp_el1(tcg_env, tcg_constant_i32(syndrome));
2481 break;
2482 }
2483 }
2484
2485 if (!ri) {
2486 /* Unknown register; this might be a guest error or a QEMU
2487 * unimplemented feature.
2488 */
2489 qemu_log_mask(LOG_UNIMP, "%s access to unsupported AArch64 "
2490 "system register op0:%d op1:%d crn:%d crm:%d op2:%d\n",
2491 isread ? "read" : "write", op0, op1, crn, crm, op2);
2492 gen_sysreg_undef(s, isread, op0, op1, op2, crn, crm, rt);
2493 return;
2494 }
2495
2496 if (s->nv2 && ri->nv2_redirect_offset) {
2497 /*
2498 * Some registers always redirect to memory; some only do so if
2499 * HCR_EL2.NV1 is 0, and some only if NV1 is 1 (these come in
2500 * pairs which share an offset; see the table in R_CSRPQ).
2501 */
2502 if (ri->nv2_redirect_offset & NV2_REDIR_NV1) {
2503 nv2_mem_redirect = s->nv1;
2504 } else if (ri->nv2_redirect_offset & NV2_REDIR_NO_NV1) {
2505 nv2_mem_redirect = !s->nv1;
2506 } else {
2507 nv2_mem_redirect = true;
2508 }
2509 }
2510
2511 /* Check access permissions */
2512 if (!cp_access_ok(s->current_el, ri, isread)) {
2513 /*
2514 * FEAT_NV/NV2 handling does not do the usual FP access checks
2515 * for registers only accessible at EL2 (though it *does* do them
2516 * for registers accessible at EL1).
2517 */
2518 skip_fp_access_checks = true;
2519 if (s->nv2 && (ri->type & ARM_CP_NV2_REDIRECT)) {
2520 /*
2521 * This is one of the few EL2 registers which should redirect
2522 * to the equivalent EL1 register. We do that after running
2523 * the EL2 register's accessfn.
2524 */
2525 nv_redirect_reg = true;
2526 assert(!nv2_mem_redirect);
2527 } else if (nv2_mem_redirect) {
2528 /*
2529 * NV2 redirect-to-memory takes precedence over trap to EL2 or
2530 * UNDEF to EL1.
2531 */
2532 } else if (s->nv && arm_cpreg_traps_in_nv(ri)) {
2533 /*
2534 * This register / instruction exists and is an EL2 register, so
2535 * we must trap to EL2 if accessed in nested virtualization EL1
2536 * instead of UNDEFing. We'll do that after the usual access checks.
2537 * (This makes a difference only for a couple of registers like
2538 * VSTTBR_EL2 where the "UNDEF if NonSecure" should take priority
2539 * over the trap-to-EL2. Most trapped-by-FEAT_NV registers have
2540 * an accessfn which does nothing when called from EL1, because
2541 * the trap-to-EL3 controls which would apply to that register
2542 * at EL2 don't take priority over the FEAT_NV trap-to-EL2.)
2543 */
2544 nv_trap_to_el2 = true;
2545 } else {
2546 gen_sysreg_undef(s, isread, op0, op1, op2, crn, crm, rt);
2547 return;
2548 }
2549 }
2550
2551 if (ri->accessfn || (ri->fgt && s->fgt_active)) {
2552 /* Emit code to perform further access permissions checks at
2553 * runtime; this may result in an exception.
2554 */
2555 gen_a64_update_pc(s, 0);
2556 tcg_ri = tcg_temp_new_ptr();
2557 gen_helper_access_check_cp_reg(tcg_ri, tcg_env,
2558 tcg_constant_i32(key),
2559 tcg_constant_i32(syndrome),
2560 tcg_constant_i32(isread));
2561 } else if (ri->type & ARM_CP_RAISES_EXC) {
2562 /*
2563 * The readfn or writefn might raise an exception;
2564 * synchronize the CPU state in case it does.
2565 */
2566 gen_a64_update_pc(s, 0);
2567 }
2568
2569 if (!skip_fp_access_checks) {
2570 if ((ri->type & ARM_CP_FPU) && !fp_access_check_only(s)) {
2571 return;
2572 } else if ((ri->type & ARM_CP_SVE) && !sve_access_check(s)) {
2573 return;
2574 } else if ((ri->type & ARM_CP_SME) && !sme_access_check(s)) {
2575 return;
2576 }
2577 }
2578
2579 if (nv_trap_to_el2) {
2580 gen_exception_insn_el(s, 0, EXCP_UDEF, syndrome, 2);
2581 return;
2582 }
2583
2584 if (nv_redirect_reg) {
2585 /*
2586 * FEAT_NV2 redirection of an EL2 register to an EL1 register.
2587 * Conveniently in all cases the encoding of the EL1 register is
2588 * identical to the EL2 register except that opc1 is 0.
2589 * Get the reginfo for the EL1 register to use for the actual access.
2590 * We don't use the EL1 register's access function, and
2591 * fine-grained-traps on EL1 also do not apply here.
2592 */
2593 key = ENCODE_AA64_CP_REG(CP_REG_ARM64_SYSREG_CP,
2594 crn, crm, op0, 0, op2);
2595 ri = get_arm_cp_reginfo(s->cp_regs, key);
2596 assert(ri);
2597 assert(cp_access_ok(s->current_el, ri, isread));
2598 /*
2599 * We might not have done an update_pc earlier, so check we don't
2600 * need it. We could support this in future if necessary.
2601 */
2602 assert(!(ri->type & ARM_CP_RAISES_EXC));
2603 }
2604
2605 if (nv2_mem_redirect) {
2606 /*
2607 * This system register is being redirected into an EL2 memory access.
2608 * This means it is not an IO operation, doesn't change hflags,
2609 * and need not end the TB, because it has no side effects.
2610 *
2611 * The access is 64-bit single copy atomic, guaranteed aligned because
2612 * of the definition of VCNR_EL2. Its endianness depends on
2613 * SCTLR_EL2.EE, not on the data endianness of EL1.
2614 * It is done under either the EL2 translation regime or the EL2&0
2615 * translation regime, depending on HCR_EL2.E2H. It behaves as if
2616 * PSTATE.PAN is 0.
2617 */
2618 TCGv_i64 ptr = tcg_temp_new_i64();
2619 MemOp mop = MO_64 | MO_ALIGN | MO_ATOM_IFALIGN;
2620 ARMMMUIdx armmemidx = s->nv2_mem_e20 ? ARMMMUIdx_E20_2 : ARMMMUIdx_E2;
2621 int memidx = arm_to_core_mmu_idx(armmemidx);
2622 uint32_t syn;
2623
2624 mop |= (s->nv2_mem_be ? MO_BE : MO_LE);
2625
2626 tcg_gen_ld_i64(ptr, tcg_env, offsetof(CPUARMState, cp15.vncr_el2));
2627 tcg_gen_addi_i64(ptr, ptr,
2628 (ri->nv2_redirect_offset & ~NV2_REDIR_FLAG_MASK));
2629 tcg_rt = cpu_reg(s, rt);
2630
2631 syn = syn_data_abort_vncr(0, !isread, 0);
2632 disas_set_insn_syndrome(s, syn);
2633 if (isread) {
2634 tcg_gen_qemu_ld_i64(tcg_rt, ptr, memidx, mop);
2635 } else {
2636 tcg_gen_qemu_st_i64(tcg_rt, ptr, memidx, mop);
2637 }
2638 return;
2639 }
2640
2641 /* Handle special cases first */
2642 switch (ri->type & ARM_CP_SPECIAL_MASK) {
2643 case 0:
2644 break;
2645 case ARM_CP_NOP:
2646 return;
2647 case ARM_CP_NZCV:
2648 tcg_rt = cpu_reg(s, rt);
2649 if (isread) {
2650 gen_get_nzcv(tcg_rt);
2651 } else {
2652 gen_set_nzcv(tcg_rt);
2653 }
2654 return;
2655 case ARM_CP_CURRENTEL:
2656 {
2657 /*
2658 * Reads as current EL value from pstate, which is
2659 * guaranteed to be constant by the tb flags.
2660 * For nested virt we should report EL2.
2661 */
2662 int el = s->nv ? 2 : s->current_el;
2663 tcg_rt = cpu_reg(s, rt);
2664 tcg_gen_movi_i64(tcg_rt, el << 2);
2665 return;
2666 }
2667 case ARM_CP_DC_ZVA:
2668 /* Writes clear the aligned block of memory which rt points into. */
2669 if (s->mte_active[0]) {
2670 int desc = 0;
2671
2672 desc = FIELD_DP32(desc, MTEDESC, MIDX, get_mem_index(s));
2673 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid);
2674 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma);
2675
2676 tcg_rt = tcg_temp_new_i64();
2677 gen_helper_mte_check_zva(tcg_rt, tcg_env,
2678 tcg_constant_i32(desc), cpu_reg(s, rt));
2679 } else {
2680 tcg_rt = clean_data_tbi(s, cpu_reg(s, rt));
2681 }
2682 gen_helper_dc_zva(tcg_env, tcg_rt);
2683 return;
2684 case ARM_CP_DC_GVA:
2685 {
2686 TCGv_i64 clean_addr, tag;
2687
2688 /*
2689 * DC_GVA, like DC_ZVA, requires that we supply the original
2690 * pointer for an invalid page. Probe that address first.
2691 */
2692 tcg_rt = cpu_reg(s, rt);
2693 clean_addr = clean_data_tbi(s, tcg_rt);
2694 gen_probe_access(s, clean_addr, MMU_DATA_STORE, MO_8);
2695
2696 if (s->ata[0]) {
2697 /* Extract the tag from the register to match STZGM. */
2698 tag = tcg_temp_new_i64();
2699 tcg_gen_shri_i64(tag, tcg_rt, 56);
2700 gen_helper_stzgm_tags(tcg_env, clean_addr, tag);
2701 }
2702 }
2703 return;
2704 case ARM_CP_DC_GZVA:
2705 {
2706 TCGv_i64 clean_addr, tag;
2707
2708 /* For DC_GZVA, we can rely on DC_ZVA for the proper fault. */
2709 tcg_rt = cpu_reg(s, rt);
2710 clean_addr = clean_data_tbi(s, tcg_rt);
2711 gen_helper_dc_zva(tcg_env, clean_addr);
2712
2713 if (s->ata[0]) {
2714 /* Extract the tag from the register to match STZGM. */
2715 tag = tcg_temp_new_i64();
2716 tcg_gen_shri_i64(tag, tcg_rt, 56);
2717 gen_helper_stzgm_tags(tcg_env, clean_addr, tag);
2718 }
2719 }
2720 return;
2721 default:
2722 g_assert_not_reached();
2723 }
2724
2725 if (ri->type & ARM_CP_IO) {
2726 /* I/O operations must end the TB here (whether read or write) */
2727 need_exit_tb = translator_io_start(&s->base);
2728 }
2729
2730 tcg_rt = cpu_reg(s, rt);
2731
2732 if (isread) {
2733 if (ri->type & ARM_CP_CONST) {
2734 tcg_gen_movi_i64(tcg_rt, ri->resetvalue);
2735 } else if (ri->readfn) {
2736 if (!tcg_ri) {
2737 tcg_ri = gen_lookup_cp_reg(key);
2738 }
2739 gen_helper_get_cp_reg64(tcg_rt, tcg_env, tcg_ri);
2740 } else {
2741 tcg_gen_ld_i64(tcg_rt, tcg_env, ri->fieldoffset);
2742 }
2743 } else {
2744 if (ri->type & ARM_CP_CONST) {
2745 /* If not forbidden by access permissions, treat as WI */
2746 return;
2747 } else if (ri->writefn) {
2748 if (!tcg_ri) {
2749 tcg_ri = gen_lookup_cp_reg(key);
2750 }
2751 gen_helper_set_cp_reg64(tcg_env, tcg_ri, tcg_rt);
2752 } else {
2753 tcg_gen_st_i64(tcg_rt, tcg_env, ri->fieldoffset);
2754 }
2755 }
2756
2757 if (!isread && !(ri->type & ARM_CP_SUPPRESS_TB_END)) {
2758 /*
2759 * A write to any coprocessor register that ends a TB
2760 * must rebuild the hflags for the next TB.
2761 */
2762 gen_rebuild_hflags(s);
2763 /*
2764 * We default to ending the TB on a coprocessor register write,
2765 * but allow this to be suppressed by the register definition
2766 * (usually only necessary to work around guest bugs).
2767 */
2768 need_exit_tb = true;
2769 }
2770 if (need_exit_tb) {
2771 s->base.is_jmp = DISAS_UPDATE_EXIT;
2772 }
2773 }
2774
trans_SYS(DisasContext * s,arg_SYS * a)2775 static bool trans_SYS(DisasContext *s, arg_SYS *a)
2776 {
2777 handle_sys(s, a->l, a->op0, a->op1, a->op2, a->crn, a->crm, a->rt);
2778 return true;
2779 }
2780
trans_SVC(DisasContext * s,arg_i * a)2781 static bool trans_SVC(DisasContext *s, arg_i *a)
2782 {
2783 /*
2784 * For SVC, HVC and SMC we advance the single-step state
2785 * machine before taking the exception. This is architecturally
2786 * mandated, to ensure that single-stepping a system call
2787 * instruction works properly.
2788 */
2789 uint32_t syndrome = syn_aa64_svc(a->imm);
2790 if (s->fgt_svc) {
2791 gen_exception_insn_el(s, 0, EXCP_UDEF, syndrome, 2);
2792 return true;
2793 }
2794 gen_ss_advance(s);
2795 gen_exception_insn(s, 4, EXCP_SWI, syndrome);
2796 return true;
2797 }
2798
trans_HVC(DisasContext * s,arg_i * a)2799 static bool trans_HVC(DisasContext *s, arg_i *a)
2800 {
2801 int target_el = s->current_el == 3 ? 3 : 2;
2802
2803 if (s->current_el == 0) {
2804 unallocated_encoding(s);
2805 return true;
2806 }
2807 /*
2808 * The pre HVC helper handles cases when HVC gets trapped
2809 * as an undefined insn by runtime configuration.
2810 */
2811 gen_a64_update_pc(s, 0);
2812 gen_helper_pre_hvc(tcg_env);
2813 /* Architecture requires ss advance before we do the actual work */
2814 gen_ss_advance(s);
2815 gen_exception_insn_el(s, 4, EXCP_HVC, syn_aa64_hvc(a->imm), target_el);
2816 return true;
2817 }
2818
trans_SMC(DisasContext * s,arg_i * a)2819 static bool trans_SMC(DisasContext *s, arg_i *a)
2820 {
2821 if (s->current_el == 0) {
2822 unallocated_encoding(s);
2823 return true;
2824 }
2825 gen_a64_update_pc(s, 0);
2826 gen_helper_pre_smc(tcg_env, tcg_constant_i32(syn_aa64_smc(a->imm)));
2827 /* Architecture requires ss advance before we do the actual work */
2828 gen_ss_advance(s);
2829 gen_exception_insn_el(s, 4, EXCP_SMC, syn_aa64_smc(a->imm), 3);
2830 return true;
2831 }
2832
trans_BRK(DisasContext * s,arg_i * a)2833 static bool trans_BRK(DisasContext *s, arg_i *a)
2834 {
2835 gen_exception_bkpt_insn(s, syn_aa64_bkpt(a->imm));
2836 return true;
2837 }
2838
trans_HLT(DisasContext * s,arg_i * a)2839 static bool trans_HLT(DisasContext *s, arg_i *a)
2840 {
2841 /*
2842 * HLT. This has two purposes.
2843 * Architecturally, it is an external halting debug instruction.
2844 * Since QEMU doesn't implement external debug, we treat this as
2845 * it is required for halting debug disabled: it will UNDEF.
2846 * Secondly, "HLT 0xf000" is the A64 semihosting syscall instruction.
2847 */
2848 if (semihosting_enabled(s->current_el == 0) && a->imm == 0xf000) {
2849 gen_exception_internal_insn(s, EXCP_SEMIHOST);
2850 } else {
2851 unallocated_encoding(s);
2852 }
2853 return true;
2854 }
2855
2856 /*
2857 * Load/Store exclusive instructions are implemented by remembering
2858 * the value/address loaded, and seeing if these are the same
2859 * when the store is performed. This is not actually the architecturally
2860 * mandated semantics, but it works for typical guest code sequences
2861 * and avoids having to monitor regular stores.
2862 *
2863 * The store exclusive uses the atomic cmpxchg primitives to avoid
2864 * races in multi-threaded linux-user and when MTTCG softmmu is
2865 * enabled.
2866 */
gen_load_exclusive(DisasContext * s,int rt,int rt2,int rn,int size,bool is_pair)2867 static void gen_load_exclusive(DisasContext *s, int rt, int rt2, int rn,
2868 int size, bool is_pair)
2869 {
2870 int idx = get_mem_index(s);
2871 TCGv_i64 dirty_addr, clean_addr;
2872 MemOp memop = check_atomic_align(s, rn, size + is_pair);
2873
2874 s->is_ldex = true;
2875 dirty_addr = cpu_reg_sp(s, rn);
2876 clean_addr = gen_mte_check1(s, dirty_addr, false, rn != 31, memop);
2877
2878 g_assert(size <= 3);
2879 if (is_pair) {
2880 g_assert(size >= 2);
2881 if (size == 2) {
2882 tcg_gen_qemu_ld_i64(cpu_exclusive_val, clean_addr, idx, memop);
2883 if (s->be_data == MO_LE) {
2884 tcg_gen_extract_i64(cpu_reg(s, rt), cpu_exclusive_val, 0, 32);
2885 tcg_gen_extract_i64(cpu_reg(s, rt2), cpu_exclusive_val, 32, 32);
2886 } else {
2887 tcg_gen_extract_i64(cpu_reg(s, rt), cpu_exclusive_val, 32, 32);
2888 tcg_gen_extract_i64(cpu_reg(s, rt2), cpu_exclusive_val, 0, 32);
2889 }
2890 } else {
2891 TCGv_i128 t16 = tcg_temp_new_i128();
2892
2893 tcg_gen_qemu_ld_i128(t16, clean_addr, idx, memop);
2894
2895 if (s->be_data == MO_LE) {
2896 tcg_gen_extr_i128_i64(cpu_exclusive_val,
2897 cpu_exclusive_high, t16);
2898 } else {
2899 tcg_gen_extr_i128_i64(cpu_exclusive_high,
2900 cpu_exclusive_val, t16);
2901 }
2902 tcg_gen_mov_i64(cpu_reg(s, rt), cpu_exclusive_val);
2903 tcg_gen_mov_i64(cpu_reg(s, rt2), cpu_exclusive_high);
2904 }
2905 } else {
2906 tcg_gen_qemu_ld_i64(cpu_exclusive_val, clean_addr, idx, memop);
2907 tcg_gen_mov_i64(cpu_reg(s, rt), cpu_exclusive_val);
2908 }
2909 tcg_gen_mov_i64(cpu_exclusive_addr, clean_addr);
2910 }
2911
gen_store_exclusive(DisasContext * s,int rd,int rt,int rt2,int rn,int size,int is_pair)2912 static void gen_store_exclusive(DisasContext *s, int rd, int rt, int rt2,
2913 int rn, int size, int is_pair)
2914 {
2915 /* if (env->exclusive_addr == addr && env->exclusive_val == [addr]
2916 * && (!is_pair || env->exclusive_high == [addr + datasize])) {
2917 * [addr] = {Rt};
2918 * if (is_pair) {
2919 * [addr + datasize] = {Rt2};
2920 * }
2921 * {Rd} = 0;
2922 * } else {
2923 * {Rd} = 1;
2924 * }
2925 * env->exclusive_addr = -1;
2926 */
2927 TCGLabel *fail_label = gen_new_label();
2928 TCGLabel *done_label = gen_new_label();
2929 TCGv_i64 tmp, clean_addr;
2930 MemOp memop;
2931
2932 /*
2933 * FIXME: We are out of spec here. We have recorded only the address
2934 * from load_exclusive, not the entire range, and we assume that the
2935 * size of the access on both sides match. The architecture allows the
2936 * store to be smaller than the load, so long as the stored bytes are
2937 * within the range recorded by the load.
2938 */
2939
2940 /* See AArch64.ExclusiveMonitorsPass() and AArch64.IsExclusiveVA(). */
2941 clean_addr = clean_data_tbi(s, cpu_reg_sp(s, rn));
2942 tcg_gen_brcond_i64(TCG_COND_NE, clean_addr, cpu_exclusive_addr, fail_label);
2943
2944 /*
2945 * The write, and any associated faults, only happen if the virtual
2946 * and physical addresses pass the exclusive monitor check. These
2947 * faults are exceedingly unlikely, because normally the guest uses
2948 * the exact same address register for the load_exclusive, and we
2949 * would have recognized these faults there.
2950 *
2951 * It is possible to trigger an alignment fault pre-LSE2, e.g. with an
2952 * unaligned 4-byte write within the range of an aligned 8-byte load.
2953 * With LSE2, the store would need to cross a 16-byte boundary when the
2954 * load did not, which would mean the store is outside the range
2955 * recorded for the monitor, which would have failed a corrected monitor
2956 * check above. For now, we assume no size change and retain the
2957 * MO_ALIGN to let tcg know what we checked in the load_exclusive.
2958 *
2959 * It is possible to trigger an MTE fault, by performing the load with
2960 * a virtual address with a valid tag and performing the store with the
2961 * same virtual address and a different invalid tag.
2962 */
2963 memop = size + is_pair;
2964 if (memop == MO_128 || !dc_isar_feature(aa64_lse2, s)) {
2965 memop |= MO_ALIGN;
2966 }
2967 memop = finalize_memop(s, memop);
2968 gen_mte_check1(s, cpu_reg_sp(s, rn), true, rn != 31, memop);
2969
2970 tmp = tcg_temp_new_i64();
2971 if (is_pair) {
2972 if (size == 2) {
2973 if (s->be_data == MO_LE) {
2974 tcg_gen_concat32_i64(tmp, cpu_reg(s, rt), cpu_reg(s, rt2));
2975 } else {
2976 tcg_gen_concat32_i64(tmp, cpu_reg(s, rt2), cpu_reg(s, rt));
2977 }
2978 tcg_gen_atomic_cmpxchg_i64(tmp, cpu_exclusive_addr,
2979 cpu_exclusive_val, tmp,
2980 get_mem_index(s), memop);
2981 tcg_gen_setcond_i64(TCG_COND_NE, tmp, tmp, cpu_exclusive_val);
2982 } else {
2983 TCGv_i128 t16 = tcg_temp_new_i128();
2984 TCGv_i128 c16 = tcg_temp_new_i128();
2985 TCGv_i64 a, b;
2986
2987 if (s->be_data == MO_LE) {
2988 tcg_gen_concat_i64_i128(t16, cpu_reg(s, rt), cpu_reg(s, rt2));
2989 tcg_gen_concat_i64_i128(c16, cpu_exclusive_val,
2990 cpu_exclusive_high);
2991 } else {
2992 tcg_gen_concat_i64_i128(t16, cpu_reg(s, rt2), cpu_reg(s, rt));
2993 tcg_gen_concat_i64_i128(c16, cpu_exclusive_high,
2994 cpu_exclusive_val);
2995 }
2996
2997 tcg_gen_atomic_cmpxchg_i128(t16, cpu_exclusive_addr, c16, t16,
2998 get_mem_index(s), memop);
2999
3000 a = tcg_temp_new_i64();
3001 b = tcg_temp_new_i64();
3002 if (s->be_data == MO_LE) {
3003 tcg_gen_extr_i128_i64(a, b, t16);
3004 } else {
3005 tcg_gen_extr_i128_i64(b, a, t16);
3006 }
3007
3008 tcg_gen_xor_i64(a, a, cpu_exclusive_val);
3009 tcg_gen_xor_i64(b, b, cpu_exclusive_high);
3010 tcg_gen_or_i64(tmp, a, b);
3011
3012 tcg_gen_setcondi_i64(TCG_COND_NE, tmp, tmp, 0);
3013 }
3014 } else {
3015 tcg_gen_atomic_cmpxchg_i64(tmp, cpu_exclusive_addr, cpu_exclusive_val,
3016 cpu_reg(s, rt), get_mem_index(s), memop);
3017 tcg_gen_setcond_i64(TCG_COND_NE, tmp, tmp, cpu_exclusive_val);
3018 }
3019 tcg_gen_mov_i64(cpu_reg(s, rd), tmp);
3020 tcg_gen_br(done_label);
3021
3022 gen_set_label(fail_label);
3023 tcg_gen_movi_i64(cpu_reg(s, rd), 1);
3024 gen_set_label(done_label);
3025 tcg_gen_movi_i64(cpu_exclusive_addr, -1);
3026 }
3027
gen_compare_and_swap(DisasContext * s,int rs,int rt,int rn,int size)3028 static void gen_compare_and_swap(DisasContext *s, int rs, int rt,
3029 int rn, int size)
3030 {
3031 TCGv_i64 tcg_rs = cpu_reg(s, rs);
3032 TCGv_i64 tcg_rt = cpu_reg(s, rt);
3033 int memidx = get_mem_index(s);
3034 TCGv_i64 clean_addr;
3035 MemOp memop;
3036
3037 if (rn == 31) {
3038 gen_check_sp_alignment(s);
3039 }
3040 memop = check_atomic_align(s, rn, size);
3041 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn), true, rn != 31, memop);
3042 tcg_gen_atomic_cmpxchg_i64(tcg_rs, clean_addr, tcg_rs, tcg_rt,
3043 memidx, memop);
3044 }
3045
gen_compare_and_swap_pair(DisasContext * s,int rs,int rt,int rn,int size)3046 static void gen_compare_and_swap_pair(DisasContext *s, int rs, int rt,
3047 int rn, int size)
3048 {
3049 TCGv_i64 s1 = cpu_reg(s, rs);
3050 TCGv_i64 s2 = cpu_reg(s, rs + 1);
3051 TCGv_i64 t1 = cpu_reg(s, rt);
3052 TCGv_i64 t2 = cpu_reg(s, rt + 1);
3053 TCGv_i64 clean_addr;
3054 int memidx = get_mem_index(s);
3055 MemOp memop;
3056
3057 if (rn == 31) {
3058 gen_check_sp_alignment(s);
3059 }
3060
3061 /* This is a single atomic access, despite the "pair". */
3062 memop = check_atomic_align(s, rn, size + 1);
3063 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, rn), true, rn != 31, memop);
3064
3065 if (size == 2) {
3066 TCGv_i64 cmp = tcg_temp_new_i64();
3067 TCGv_i64 val = tcg_temp_new_i64();
3068
3069 if (s->be_data == MO_LE) {
3070 tcg_gen_concat32_i64(val, t1, t2);
3071 tcg_gen_concat32_i64(cmp, s1, s2);
3072 } else {
3073 tcg_gen_concat32_i64(val, t2, t1);
3074 tcg_gen_concat32_i64(cmp, s2, s1);
3075 }
3076
3077 tcg_gen_atomic_cmpxchg_i64(cmp, clean_addr, cmp, val, memidx, memop);
3078
3079 if (s->be_data == MO_LE) {
3080 tcg_gen_extr32_i64(s1, s2, cmp);
3081 } else {
3082 tcg_gen_extr32_i64(s2, s1, cmp);
3083 }
3084 } else {
3085 TCGv_i128 cmp = tcg_temp_new_i128();
3086 TCGv_i128 val = tcg_temp_new_i128();
3087
3088 if (s->be_data == MO_LE) {
3089 tcg_gen_concat_i64_i128(val, t1, t2);
3090 tcg_gen_concat_i64_i128(cmp, s1, s2);
3091 } else {
3092 tcg_gen_concat_i64_i128(val, t2, t1);
3093 tcg_gen_concat_i64_i128(cmp, s2, s1);
3094 }
3095
3096 tcg_gen_atomic_cmpxchg_i128(cmp, clean_addr, cmp, val, memidx, memop);
3097
3098 if (s->be_data == MO_LE) {
3099 tcg_gen_extr_i128_i64(s1, s2, cmp);
3100 } else {
3101 tcg_gen_extr_i128_i64(s2, s1, cmp);
3102 }
3103 }
3104 }
3105
3106 /*
3107 * Compute the ISS.SF bit for syndrome information if an exception
3108 * is taken on a load or store. This indicates whether the instruction
3109 * is accessing a 32-bit or 64-bit register. This logic is derived
3110 * from the ARMv8 specs for LDR (Shared decode for all encodings).
3111 */
ldst_iss_sf(int size,bool sign,bool ext)3112 static bool ldst_iss_sf(int size, bool sign, bool ext)
3113 {
3114
3115 if (sign) {
3116 /*
3117 * Signed loads are 64 bit results if we are not going to
3118 * do a zero-extend from 32 to 64 after the load.
3119 * (For a store, sign and ext are always false.)
3120 */
3121 return !ext;
3122 } else {
3123 /* Unsigned loads/stores work at the specified size */
3124 return size == MO_64;
3125 }
3126 }
3127
trans_STXR(DisasContext * s,arg_stxr * a)3128 static bool trans_STXR(DisasContext *s, arg_stxr *a)
3129 {
3130 if (a->rn == 31) {
3131 gen_check_sp_alignment(s);
3132 }
3133 if (a->lasr) {
3134 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_STRL);
3135 }
3136 gen_store_exclusive(s, a->rs, a->rt, a->rt2, a->rn, a->sz, false);
3137 return true;
3138 }
3139
trans_LDXR(DisasContext * s,arg_stxr * a)3140 static bool trans_LDXR(DisasContext *s, arg_stxr *a)
3141 {
3142 if (a->rn == 31) {
3143 gen_check_sp_alignment(s);
3144 }
3145 gen_load_exclusive(s, a->rt, a->rt2, a->rn, a->sz, false);
3146 if (a->lasr) {
3147 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ);
3148 }
3149 return true;
3150 }
3151
trans_STLR(DisasContext * s,arg_stlr * a)3152 static bool trans_STLR(DisasContext *s, arg_stlr *a)
3153 {
3154 TCGv_i64 clean_addr;
3155 MemOp memop;
3156 bool iss_sf = ldst_iss_sf(a->sz, false, false);
3157
3158 /*
3159 * StoreLORelease is the same as Store-Release for QEMU, but
3160 * needs the feature-test.
3161 */
3162 if (!a->lasr && !dc_isar_feature(aa64_lor, s)) {
3163 return false;
3164 }
3165 /* Generate ISS for non-exclusive accesses including LASR. */
3166 if (a->rn == 31) {
3167 gen_check_sp_alignment(s);
3168 }
3169 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_STRL);
3170 memop = check_ordered_align(s, a->rn, 0, true, a->sz);
3171 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn),
3172 true, a->rn != 31, memop);
3173 do_gpr_st(s, cpu_reg(s, a->rt), clean_addr, memop, true, a->rt,
3174 iss_sf, a->lasr);
3175 return true;
3176 }
3177
trans_LDAR(DisasContext * s,arg_stlr * a)3178 static bool trans_LDAR(DisasContext *s, arg_stlr *a)
3179 {
3180 TCGv_i64 clean_addr;
3181 MemOp memop;
3182 bool iss_sf = ldst_iss_sf(a->sz, false, false);
3183
3184 /* LoadLOAcquire is the same as Load-Acquire for QEMU. */
3185 if (!a->lasr && !dc_isar_feature(aa64_lor, s)) {
3186 return false;
3187 }
3188 /* Generate ISS for non-exclusive accesses including LASR. */
3189 if (a->rn == 31) {
3190 gen_check_sp_alignment(s);
3191 }
3192 memop = check_ordered_align(s, a->rn, 0, false, a->sz);
3193 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn),
3194 false, a->rn != 31, memop);
3195 do_gpr_ld(s, cpu_reg(s, a->rt), clean_addr, memop, false, true,
3196 a->rt, iss_sf, a->lasr);
3197 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ);
3198 return true;
3199 }
3200
trans_STXP(DisasContext * s,arg_stxr * a)3201 static bool trans_STXP(DisasContext *s, arg_stxr *a)
3202 {
3203 if (a->rn == 31) {
3204 gen_check_sp_alignment(s);
3205 }
3206 if (a->lasr) {
3207 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_STRL);
3208 }
3209 gen_store_exclusive(s, a->rs, a->rt, a->rt2, a->rn, a->sz, true);
3210 return true;
3211 }
3212
trans_LDXP(DisasContext * s,arg_stxr * a)3213 static bool trans_LDXP(DisasContext *s, arg_stxr *a)
3214 {
3215 if (a->rn == 31) {
3216 gen_check_sp_alignment(s);
3217 }
3218 gen_load_exclusive(s, a->rt, a->rt2, a->rn, a->sz, true);
3219 if (a->lasr) {
3220 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ);
3221 }
3222 return true;
3223 }
3224
trans_CASP(DisasContext * s,arg_CASP * a)3225 static bool trans_CASP(DisasContext *s, arg_CASP *a)
3226 {
3227 if (!dc_isar_feature(aa64_atomics, s)) {
3228 return false;
3229 }
3230 if (((a->rt | a->rs) & 1) != 0) {
3231 return false;
3232 }
3233
3234 gen_compare_and_swap_pair(s, a->rs, a->rt, a->rn, a->sz);
3235 return true;
3236 }
3237
trans_CAS(DisasContext * s,arg_CAS * a)3238 static bool trans_CAS(DisasContext *s, arg_CAS *a)
3239 {
3240 if (!dc_isar_feature(aa64_atomics, s)) {
3241 return false;
3242 }
3243 gen_compare_and_swap(s, a->rs, a->rt, a->rn, a->sz);
3244 return true;
3245 }
3246
trans_LD_lit(DisasContext * s,arg_ldlit * a)3247 static bool trans_LD_lit(DisasContext *s, arg_ldlit *a)
3248 {
3249 bool iss_sf = ldst_iss_sf(a->sz, a->sign, false);
3250 TCGv_i64 tcg_rt = cpu_reg(s, a->rt);
3251 TCGv_i64 clean_addr = tcg_temp_new_i64();
3252 MemOp memop = finalize_memop(s, a->sz + a->sign * MO_SIGN);
3253
3254 gen_pc_plus_diff(s, clean_addr, a->imm);
3255 do_gpr_ld(s, tcg_rt, clean_addr, memop,
3256 false, true, a->rt, iss_sf, false);
3257 return true;
3258 }
3259
trans_LD_lit_v(DisasContext * s,arg_ldlit * a)3260 static bool trans_LD_lit_v(DisasContext *s, arg_ldlit *a)
3261 {
3262 /* Load register (literal), vector version */
3263 TCGv_i64 clean_addr;
3264 MemOp memop;
3265
3266 if (!fp_access_check(s)) {
3267 return true;
3268 }
3269 memop = finalize_memop_asimd(s, a->sz);
3270 clean_addr = tcg_temp_new_i64();
3271 gen_pc_plus_diff(s, clean_addr, a->imm);
3272 do_fp_ld(s, a->rt, clean_addr, memop);
3273 return true;
3274 }
3275
op_addr_ldstpair_pre(DisasContext * s,arg_ldstpair * a,TCGv_i64 * clean_addr,TCGv_i64 * dirty_addr,uint64_t offset,bool is_store,MemOp mop)3276 static void op_addr_ldstpair_pre(DisasContext *s, arg_ldstpair *a,
3277 TCGv_i64 *clean_addr, TCGv_i64 *dirty_addr,
3278 uint64_t offset, bool is_store, MemOp mop)
3279 {
3280 if (a->rn == 31) {
3281 gen_check_sp_alignment(s);
3282 }
3283
3284 *dirty_addr = read_cpu_reg_sp(s, a->rn, 1);
3285 if (!a->p) {
3286 tcg_gen_addi_i64(*dirty_addr, *dirty_addr, offset);
3287 }
3288
3289 *clean_addr = gen_mte_checkN(s, *dirty_addr, is_store,
3290 (a->w || a->rn != 31), 2 << a->sz, mop);
3291 }
3292
op_addr_ldstpair_post(DisasContext * s,arg_ldstpair * a,TCGv_i64 dirty_addr,uint64_t offset)3293 static void op_addr_ldstpair_post(DisasContext *s, arg_ldstpair *a,
3294 TCGv_i64 dirty_addr, uint64_t offset)
3295 {
3296 if (a->w) {
3297 if (a->p) {
3298 tcg_gen_addi_i64(dirty_addr, dirty_addr, offset);
3299 }
3300 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), dirty_addr);
3301 }
3302 }
3303
trans_STP(DisasContext * s,arg_ldstpair * a)3304 static bool trans_STP(DisasContext *s, arg_ldstpair *a)
3305 {
3306 uint64_t offset = a->imm << a->sz;
3307 TCGv_i64 clean_addr, dirty_addr, tcg_rt, tcg_rt2;
3308 MemOp mop = finalize_memop(s, a->sz);
3309
3310 op_addr_ldstpair_pre(s, a, &clean_addr, &dirty_addr, offset, true, mop);
3311 tcg_rt = cpu_reg(s, a->rt);
3312 tcg_rt2 = cpu_reg(s, a->rt2);
3313 /*
3314 * We built mop above for the single logical access -- rebuild it
3315 * now for the paired operation.
3316 *
3317 * With LSE2, non-sign-extending pairs are treated atomically if
3318 * aligned, and if unaligned one of the pair will be completely
3319 * within a 16-byte block and that element will be atomic.
3320 * Otherwise each element is separately atomic.
3321 * In all cases, issue one operation with the correct atomicity.
3322 */
3323 mop = a->sz + 1;
3324 if (s->align_mem) {
3325 mop |= (a->sz == 2 ? MO_ALIGN_4 : MO_ALIGN_8);
3326 }
3327 mop = finalize_memop_pair(s, mop);
3328 if (a->sz == 2) {
3329 TCGv_i64 tmp = tcg_temp_new_i64();
3330
3331 if (s->be_data == MO_LE) {
3332 tcg_gen_concat32_i64(tmp, tcg_rt, tcg_rt2);
3333 } else {
3334 tcg_gen_concat32_i64(tmp, tcg_rt2, tcg_rt);
3335 }
3336 tcg_gen_qemu_st_i64(tmp, clean_addr, get_mem_index(s), mop);
3337 } else {
3338 TCGv_i128 tmp = tcg_temp_new_i128();
3339
3340 if (s->be_data == MO_LE) {
3341 tcg_gen_concat_i64_i128(tmp, tcg_rt, tcg_rt2);
3342 } else {
3343 tcg_gen_concat_i64_i128(tmp, tcg_rt2, tcg_rt);
3344 }
3345 tcg_gen_qemu_st_i128(tmp, clean_addr, get_mem_index(s), mop);
3346 }
3347 op_addr_ldstpair_post(s, a, dirty_addr, offset);
3348 return true;
3349 }
3350
trans_LDP(DisasContext * s,arg_ldstpair * a)3351 static bool trans_LDP(DisasContext *s, arg_ldstpair *a)
3352 {
3353 uint64_t offset = a->imm << a->sz;
3354 TCGv_i64 clean_addr, dirty_addr, tcg_rt, tcg_rt2;
3355 MemOp mop = finalize_memop(s, a->sz);
3356
3357 op_addr_ldstpair_pre(s, a, &clean_addr, &dirty_addr, offset, false, mop);
3358 tcg_rt = cpu_reg(s, a->rt);
3359 tcg_rt2 = cpu_reg(s, a->rt2);
3360
3361 /*
3362 * We built mop above for the single logical access -- rebuild it
3363 * now for the paired operation.
3364 *
3365 * With LSE2, non-sign-extending pairs are treated atomically if
3366 * aligned, and if unaligned one of the pair will be completely
3367 * within a 16-byte block and that element will be atomic.
3368 * Otherwise each element is separately atomic.
3369 * In all cases, issue one operation with the correct atomicity.
3370 *
3371 * This treats sign-extending loads like zero-extending loads,
3372 * since that reuses the most code below.
3373 */
3374 mop = a->sz + 1;
3375 if (s->align_mem) {
3376 mop |= (a->sz == 2 ? MO_ALIGN_4 : MO_ALIGN_8);
3377 }
3378 mop = finalize_memop_pair(s, mop);
3379 if (a->sz == 2) {
3380 int o2 = s->be_data == MO_LE ? 32 : 0;
3381 int o1 = o2 ^ 32;
3382
3383 tcg_gen_qemu_ld_i64(tcg_rt, clean_addr, get_mem_index(s), mop);
3384 if (a->sign) {
3385 tcg_gen_sextract_i64(tcg_rt2, tcg_rt, o2, 32);
3386 tcg_gen_sextract_i64(tcg_rt, tcg_rt, o1, 32);
3387 } else {
3388 tcg_gen_extract_i64(tcg_rt2, tcg_rt, o2, 32);
3389 tcg_gen_extract_i64(tcg_rt, tcg_rt, o1, 32);
3390 }
3391 } else {
3392 TCGv_i128 tmp = tcg_temp_new_i128();
3393
3394 tcg_gen_qemu_ld_i128(tmp, clean_addr, get_mem_index(s), mop);
3395 if (s->be_data == MO_LE) {
3396 tcg_gen_extr_i128_i64(tcg_rt, tcg_rt2, tmp);
3397 } else {
3398 tcg_gen_extr_i128_i64(tcg_rt2, tcg_rt, tmp);
3399 }
3400 }
3401 op_addr_ldstpair_post(s, a, dirty_addr, offset);
3402 return true;
3403 }
3404
trans_STP_v(DisasContext * s,arg_ldstpair * a)3405 static bool trans_STP_v(DisasContext *s, arg_ldstpair *a)
3406 {
3407 uint64_t offset = a->imm << a->sz;
3408 TCGv_i64 clean_addr, dirty_addr;
3409 MemOp mop;
3410
3411 if (!fp_access_check(s)) {
3412 return true;
3413 }
3414
3415 /* LSE2 does not merge FP pairs; leave these as separate operations. */
3416 mop = finalize_memop_asimd(s, a->sz);
3417 op_addr_ldstpair_pre(s, a, &clean_addr, &dirty_addr, offset, true, mop);
3418 do_fp_st(s, a->rt, clean_addr, mop);
3419 tcg_gen_addi_i64(clean_addr, clean_addr, 1 << a->sz);
3420 do_fp_st(s, a->rt2, clean_addr, mop);
3421 op_addr_ldstpair_post(s, a, dirty_addr, offset);
3422 return true;
3423 }
3424
trans_LDP_v(DisasContext * s,arg_ldstpair * a)3425 static bool trans_LDP_v(DisasContext *s, arg_ldstpair *a)
3426 {
3427 uint64_t offset = a->imm << a->sz;
3428 TCGv_i64 clean_addr, dirty_addr;
3429 MemOp mop;
3430
3431 if (!fp_access_check(s)) {
3432 return true;
3433 }
3434
3435 /* LSE2 does not merge FP pairs; leave these as separate operations. */
3436 mop = finalize_memop_asimd(s, a->sz);
3437 op_addr_ldstpair_pre(s, a, &clean_addr, &dirty_addr, offset, false, mop);
3438 do_fp_ld(s, a->rt, clean_addr, mop);
3439 tcg_gen_addi_i64(clean_addr, clean_addr, 1 << a->sz);
3440 do_fp_ld(s, a->rt2, clean_addr, mop);
3441 op_addr_ldstpair_post(s, a, dirty_addr, offset);
3442 return true;
3443 }
3444
trans_STGP(DisasContext * s,arg_ldstpair * a)3445 static bool trans_STGP(DisasContext *s, arg_ldstpair *a)
3446 {
3447 TCGv_i64 clean_addr, dirty_addr, tcg_rt, tcg_rt2;
3448 uint64_t offset = a->imm << LOG2_TAG_GRANULE;
3449 MemOp mop;
3450 TCGv_i128 tmp;
3451
3452 /* STGP only comes in one size. */
3453 tcg_debug_assert(a->sz == MO_64);
3454
3455 if (!dc_isar_feature(aa64_mte_insn_reg, s)) {
3456 return false;
3457 }
3458
3459 if (a->rn == 31) {
3460 gen_check_sp_alignment(s);
3461 }
3462
3463 dirty_addr = read_cpu_reg_sp(s, a->rn, 1);
3464 if (!a->p) {
3465 tcg_gen_addi_i64(dirty_addr, dirty_addr, offset);
3466 }
3467
3468 clean_addr = clean_data_tbi(s, dirty_addr);
3469 tcg_rt = cpu_reg(s, a->rt);
3470 tcg_rt2 = cpu_reg(s, a->rt2);
3471
3472 /*
3473 * STGP is defined as two 8-byte memory operations, aligned to TAG_GRANULE,
3474 * and one tag operation. We implement it as one single aligned 16-byte
3475 * memory operation for convenience. Note that the alignment ensures
3476 * MO_ATOM_IFALIGN_PAIR produces 8-byte atomicity for the memory store.
3477 */
3478 mop = finalize_memop_atom(s, MO_128 | MO_ALIGN, MO_ATOM_IFALIGN_PAIR);
3479
3480 tmp = tcg_temp_new_i128();
3481 if (s->be_data == MO_LE) {
3482 tcg_gen_concat_i64_i128(tmp, tcg_rt, tcg_rt2);
3483 } else {
3484 tcg_gen_concat_i64_i128(tmp, tcg_rt2, tcg_rt);
3485 }
3486 tcg_gen_qemu_st_i128(tmp, clean_addr, get_mem_index(s), mop);
3487
3488 /* Perform the tag store, if tag access enabled. */
3489 if (s->ata[0]) {
3490 if (tb_cflags(s->base.tb) & CF_PARALLEL) {
3491 gen_helper_stg_parallel(tcg_env, dirty_addr, dirty_addr);
3492 } else {
3493 gen_helper_stg(tcg_env, dirty_addr, dirty_addr);
3494 }
3495 }
3496
3497 op_addr_ldstpair_post(s, a, dirty_addr, offset);
3498 return true;
3499 }
3500
op_addr_ldst_imm_pre(DisasContext * s,arg_ldst_imm * a,TCGv_i64 * clean_addr,TCGv_i64 * dirty_addr,uint64_t offset,bool is_store,MemOp mop)3501 static void op_addr_ldst_imm_pre(DisasContext *s, arg_ldst_imm *a,
3502 TCGv_i64 *clean_addr, TCGv_i64 *dirty_addr,
3503 uint64_t offset, bool is_store, MemOp mop)
3504 {
3505 int memidx;
3506
3507 if (a->rn == 31) {
3508 gen_check_sp_alignment(s);
3509 }
3510
3511 *dirty_addr = read_cpu_reg_sp(s, a->rn, 1);
3512 if (!a->p) {
3513 tcg_gen_addi_i64(*dirty_addr, *dirty_addr, offset);
3514 }
3515 memidx = get_a64_user_mem_index(s, a->unpriv);
3516 *clean_addr = gen_mte_check1_mmuidx(s, *dirty_addr, is_store,
3517 a->w || a->rn != 31,
3518 mop, a->unpriv, memidx);
3519 }
3520
op_addr_ldst_imm_post(DisasContext * s,arg_ldst_imm * a,TCGv_i64 dirty_addr,uint64_t offset)3521 static void op_addr_ldst_imm_post(DisasContext *s, arg_ldst_imm *a,
3522 TCGv_i64 dirty_addr, uint64_t offset)
3523 {
3524 if (a->w) {
3525 if (a->p) {
3526 tcg_gen_addi_i64(dirty_addr, dirty_addr, offset);
3527 }
3528 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), dirty_addr);
3529 }
3530 }
3531
trans_STR_i(DisasContext * s,arg_ldst_imm * a)3532 static bool trans_STR_i(DisasContext *s, arg_ldst_imm *a)
3533 {
3534 bool iss_sf, iss_valid = !a->w;
3535 TCGv_i64 clean_addr, dirty_addr, tcg_rt;
3536 int memidx = get_a64_user_mem_index(s, a->unpriv);
3537 MemOp mop = finalize_memop(s, a->sz + a->sign * MO_SIGN);
3538
3539 op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, true, mop);
3540
3541 tcg_rt = cpu_reg(s, a->rt);
3542 iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext);
3543
3544 do_gpr_st_memidx(s, tcg_rt, clean_addr, mop, memidx,
3545 iss_valid, a->rt, iss_sf, false);
3546 op_addr_ldst_imm_post(s, a, dirty_addr, a->imm);
3547 return true;
3548 }
3549
trans_LDR_i(DisasContext * s,arg_ldst_imm * a)3550 static bool trans_LDR_i(DisasContext *s, arg_ldst_imm *a)
3551 {
3552 bool iss_sf, iss_valid = !a->w;
3553 TCGv_i64 clean_addr, dirty_addr, tcg_rt;
3554 int memidx = get_a64_user_mem_index(s, a->unpriv);
3555 MemOp mop = finalize_memop(s, a->sz + a->sign * MO_SIGN);
3556
3557 op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, false, mop);
3558
3559 tcg_rt = cpu_reg(s, a->rt);
3560 iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext);
3561
3562 do_gpr_ld_memidx(s, tcg_rt, clean_addr, mop,
3563 a->ext, memidx, iss_valid, a->rt, iss_sf, false);
3564 op_addr_ldst_imm_post(s, a, dirty_addr, a->imm);
3565 return true;
3566 }
3567
trans_STR_v_i(DisasContext * s,arg_ldst_imm * a)3568 static bool trans_STR_v_i(DisasContext *s, arg_ldst_imm *a)
3569 {
3570 TCGv_i64 clean_addr, dirty_addr;
3571 MemOp mop;
3572
3573 if (!fp_access_check(s)) {
3574 return true;
3575 }
3576 mop = finalize_memop_asimd(s, a->sz);
3577 op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, true, mop);
3578 do_fp_st(s, a->rt, clean_addr, mop);
3579 op_addr_ldst_imm_post(s, a, dirty_addr, a->imm);
3580 return true;
3581 }
3582
trans_LDR_v_i(DisasContext * s,arg_ldst_imm * a)3583 static bool trans_LDR_v_i(DisasContext *s, arg_ldst_imm *a)
3584 {
3585 TCGv_i64 clean_addr, dirty_addr;
3586 MemOp mop;
3587
3588 if (!fp_access_check(s)) {
3589 return true;
3590 }
3591 mop = finalize_memop_asimd(s, a->sz);
3592 op_addr_ldst_imm_pre(s, a, &clean_addr, &dirty_addr, a->imm, false, mop);
3593 do_fp_ld(s, a->rt, clean_addr, mop);
3594 op_addr_ldst_imm_post(s, a, dirty_addr, a->imm);
3595 return true;
3596 }
3597
op_addr_ldst_pre(DisasContext * s,arg_ldst * a,TCGv_i64 * clean_addr,TCGv_i64 * dirty_addr,bool is_store,MemOp memop)3598 static void op_addr_ldst_pre(DisasContext *s, arg_ldst *a,
3599 TCGv_i64 *clean_addr, TCGv_i64 *dirty_addr,
3600 bool is_store, MemOp memop)
3601 {
3602 TCGv_i64 tcg_rm;
3603
3604 if (a->rn == 31) {
3605 gen_check_sp_alignment(s);
3606 }
3607 *dirty_addr = read_cpu_reg_sp(s, a->rn, 1);
3608
3609 tcg_rm = read_cpu_reg(s, a->rm, 1);
3610 ext_and_shift_reg(tcg_rm, tcg_rm, a->opt, a->s ? a->sz : 0);
3611
3612 tcg_gen_add_i64(*dirty_addr, *dirty_addr, tcg_rm);
3613 *clean_addr = gen_mte_check1(s, *dirty_addr, is_store, true, memop);
3614 }
3615
trans_LDR(DisasContext * s,arg_ldst * a)3616 static bool trans_LDR(DisasContext *s, arg_ldst *a)
3617 {
3618 TCGv_i64 clean_addr, dirty_addr, tcg_rt;
3619 bool iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext);
3620 MemOp memop;
3621
3622 if (extract32(a->opt, 1, 1) == 0) {
3623 return false;
3624 }
3625
3626 memop = finalize_memop(s, a->sz + a->sign * MO_SIGN);
3627 op_addr_ldst_pre(s, a, &clean_addr, &dirty_addr, false, memop);
3628 tcg_rt = cpu_reg(s, a->rt);
3629 do_gpr_ld(s, tcg_rt, clean_addr, memop,
3630 a->ext, true, a->rt, iss_sf, false);
3631 return true;
3632 }
3633
trans_STR(DisasContext * s,arg_ldst * a)3634 static bool trans_STR(DisasContext *s, arg_ldst *a)
3635 {
3636 TCGv_i64 clean_addr, dirty_addr, tcg_rt;
3637 bool iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext);
3638 MemOp memop;
3639
3640 if (extract32(a->opt, 1, 1) == 0) {
3641 return false;
3642 }
3643
3644 memop = finalize_memop(s, a->sz);
3645 op_addr_ldst_pre(s, a, &clean_addr, &dirty_addr, true, memop);
3646 tcg_rt = cpu_reg(s, a->rt);
3647 do_gpr_st(s, tcg_rt, clean_addr, memop, true, a->rt, iss_sf, false);
3648 return true;
3649 }
3650
trans_LDR_v(DisasContext * s,arg_ldst * a)3651 static bool trans_LDR_v(DisasContext *s, arg_ldst *a)
3652 {
3653 TCGv_i64 clean_addr, dirty_addr;
3654 MemOp memop;
3655
3656 if (extract32(a->opt, 1, 1) == 0) {
3657 return false;
3658 }
3659
3660 if (!fp_access_check(s)) {
3661 return true;
3662 }
3663
3664 memop = finalize_memop_asimd(s, a->sz);
3665 op_addr_ldst_pre(s, a, &clean_addr, &dirty_addr, false, memop);
3666 do_fp_ld(s, a->rt, clean_addr, memop);
3667 return true;
3668 }
3669
trans_STR_v(DisasContext * s,arg_ldst * a)3670 static bool trans_STR_v(DisasContext *s, arg_ldst *a)
3671 {
3672 TCGv_i64 clean_addr, dirty_addr;
3673 MemOp memop;
3674
3675 if (extract32(a->opt, 1, 1) == 0) {
3676 return false;
3677 }
3678
3679 if (!fp_access_check(s)) {
3680 return true;
3681 }
3682
3683 memop = finalize_memop_asimd(s, a->sz);
3684 op_addr_ldst_pre(s, a, &clean_addr, &dirty_addr, true, memop);
3685 do_fp_st(s, a->rt, clean_addr, memop);
3686 return true;
3687 }
3688
3689
do_atomic_ld(DisasContext * s,arg_atomic * a,AtomicThreeOpFn * fn,int sign,bool invert)3690 static bool do_atomic_ld(DisasContext *s, arg_atomic *a, AtomicThreeOpFn *fn,
3691 int sign, bool invert)
3692 {
3693 MemOp mop = a->sz | sign;
3694 TCGv_i64 clean_addr, tcg_rs, tcg_rt;
3695
3696 if (a->rn == 31) {
3697 gen_check_sp_alignment(s);
3698 }
3699 mop = check_atomic_align(s, a->rn, mop);
3700 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn), false,
3701 a->rn != 31, mop);
3702 tcg_rs = read_cpu_reg(s, a->rs, true);
3703 tcg_rt = cpu_reg(s, a->rt);
3704 if (invert) {
3705 tcg_gen_not_i64(tcg_rs, tcg_rs);
3706 }
3707 /*
3708 * The tcg atomic primitives are all full barriers. Therefore we
3709 * can ignore the Acquire and Release bits of this instruction.
3710 */
3711 fn(tcg_rt, clean_addr, tcg_rs, get_mem_index(s), mop);
3712
3713 if (mop & MO_SIGN) {
3714 switch (a->sz) {
3715 case MO_8:
3716 tcg_gen_ext8u_i64(tcg_rt, tcg_rt);
3717 break;
3718 case MO_16:
3719 tcg_gen_ext16u_i64(tcg_rt, tcg_rt);
3720 break;
3721 case MO_32:
3722 tcg_gen_ext32u_i64(tcg_rt, tcg_rt);
3723 break;
3724 case MO_64:
3725 break;
3726 default:
3727 g_assert_not_reached();
3728 }
3729 }
3730 return true;
3731 }
3732
3733 TRANS_FEAT(LDADD, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_add_i64, 0, false)
3734 TRANS_FEAT(LDCLR, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_and_i64, 0, true)
3735 TRANS_FEAT(LDEOR, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_xor_i64, 0, false)
3736 TRANS_FEAT(LDSET, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_or_i64, 0, false)
TRANS_FEAT(LDSMAX,aa64_atomics,do_atomic_ld,a,tcg_gen_atomic_fetch_smax_i64,MO_SIGN,false)3737 TRANS_FEAT(LDSMAX, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_smax_i64, MO_SIGN, false)
3738 TRANS_FEAT(LDSMIN, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_smin_i64, MO_SIGN, false)
3739 TRANS_FEAT(LDUMAX, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_umax_i64, 0, false)
3740 TRANS_FEAT(LDUMIN, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_fetch_umin_i64, 0, false)
3741 TRANS_FEAT(SWP, aa64_atomics, do_atomic_ld, a, tcg_gen_atomic_xchg_i64, 0, false)
3742
3743 static bool trans_LDAPR(DisasContext *s, arg_LDAPR *a)
3744 {
3745 bool iss_sf = ldst_iss_sf(a->sz, false, false);
3746 TCGv_i64 clean_addr;
3747 MemOp mop;
3748
3749 if (!dc_isar_feature(aa64_atomics, s) ||
3750 !dc_isar_feature(aa64_rcpc_8_3, s)) {
3751 return false;
3752 }
3753 if (a->rn == 31) {
3754 gen_check_sp_alignment(s);
3755 }
3756 mop = check_ordered_align(s, a->rn, 0, false, a->sz);
3757 clean_addr = gen_mte_check1(s, cpu_reg_sp(s, a->rn), false,
3758 a->rn != 31, mop);
3759 /*
3760 * LDAPR* are a special case because they are a simple load, not a
3761 * fetch-and-do-something op.
3762 * The architectural consistency requirements here are weaker than
3763 * full load-acquire (we only need "load-acquire processor consistent"),
3764 * but we choose to implement them as full LDAQ.
3765 */
3766 do_gpr_ld(s, cpu_reg(s, a->rt), clean_addr, mop, false,
3767 true, a->rt, iss_sf, true);
3768 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ);
3769 return true;
3770 }
3771
trans_LDRA(DisasContext * s,arg_LDRA * a)3772 static bool trans_LDRA(DisasContext *s, arg_LDRA *a)
3773 {
3774 TCGv_i64 clean_addr, dirty_addr, tcg_rt;
3775 MemOp memop;
3776
3777 /* Load with pointer authentication */
3778 if (!dc_isar_feature(aa64_pauth, s)) {
3779 return false;
3780 }
3781
3782 if (a->rn == 31) {
3783 gen_check_sp_alignment(s);
3784 }
3785 dirty_addr = read_cpu_reg_sp(s, a->rn, 1);
3786
3787 if (s->pauth_active) {
3788 if (!a->m) {
3789 gen_helper_autda_combined(dirty_addr, tcg_env, dirty_addr,
3790 tcg_constant_i64(0));
3791 } else {
3792 gen_helper_autdb_combined(dirty_addr, tcg_env, dirty_addr,
3793 tcg_constant_i64(0));
3794 }
3795 }
3796
3797 tcg_gen_addi_i64(dirty_addr, dirty_addr, a->imm);
3798
3799 memop = finalize_memop(s, MO_64);
3800
3801 /* Note that "clean" and "dirty" here refer to TBI not PAC. */
3802 clean_addr = gen_mte_check1(s, dirty_addr, false,
3803 a->w || a->rn != 31, memop);
3804
3805 tcg_rt = cpu_reg(s, a->rt);
3806 do_gpr_ld(s, tcg_rt, clean_addr, memop,
3807 /* extend */ false, /* iss_valid */ !a->w,
3808 /* iss_srt */ a->rt, /* iss_sf */ true, /* iss_ar */ false);
3809
3810 if (a->w) {
3811 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), dirty_addr);
3812 }
3813 return true;
3814 }
3815
trans_LDAPR_i(DisasContext * s,arg_ldapr_stlr_i * a)3816 static bool trans_LDAPR_i(DisasContext *s, arg_ldapr_stlr_i *a)
3817 {
3818 TCGv_i64 clean_addr, dirty_addr;
3819 MemOp mop = a->sz | (a->sign ? MO_SIGN : 0);
3820 bool iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext);
3821
3822 if (!dc_isar_feature(aa64_rcpc_8_4, s)) {
3823 return false;
3824 }
3825
3826 if (a->rn == 31) {
3827 gen_check_sp_alignment(s);
3828 }
3829
3830 mop = check_ordered_align(s, a->rn, a->imm, false, mop);
3831 dirty_addr = read_cpu_reg_sp(s, a->rn, 1);
3832 tcg_gen_addi_i64(dirty_addr, dirty_addr, a->imm);
3833 clean_addr = clean_data_tbi(s, dirty_addr);
3834
3835 /*
3836 * Load-AcquirePC semantics; we implement as the slightly more
3837 * restrictive Load-Acquire.
3838 */
3839 do_gpr_ld(s, cpu_reg(s, a->rt), clean_addr, mop, a->ext, true,
3840 a->rt, iss_sf, true);
3841 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_LDAQ);
3842 return true;
3843 }
3844
trans_STLR_i(DisasContext * s,arg_ldapr_stlr_i * a)3845 static bool trans_STLR_i(DisasContext *s, arg_ldapr_stlr_i *a)
3846 {
3847 TCGv_i64 clean_addr, dirty_addr;
3848 MemOp mop = a->sz;
3849 bool iss_sf = ldst_iss_sf(a->sz, a->sign, a->ext);
3850
3851 if (!dc_isar_feature(aa64_rcpc_8_4, s)) {
3852 return false;
3853 }
3854
3855 /* TODO: ARMv8.4-LSE SCTLR.nAA */
3856
3857 if (a->rn == 31) {
3858 gen_check_sp_alignment(s);
3859 }
3860
3861 mop = check_ordered_align(s, a->rn, a->imm, true, mop);
3862 dirty_addr = read_cpu_reg_sp(s, a->rn, 1);
3863 tcg_gen_addi_i64(dirty_addr, dirty_addr, a->imm);
3864 clean_addr = clean_data_tbi(s, dirty_addr);
3865
3866 /* Store-Release semantics */
3867 tcg_gen_mb(TCG_MO_ALL | TCG_BAR_STRL);
3868 do_gpr_st(s, cpu_reg(s, a->rt), clean_addr, mop, true, a->rt, iss_sf, true);
3869 return true;
3870 }
3871
trans_LD_mult(DisasContext * s,arg_ldst_mult * a)3872 static bool trans_LD_mult(DisasContext *s, arg_ldst_mult *a)
3873 {
3874 TCGv_i64 clean_addr, tcg_rn, tcg_ebytes;
3875 MemOp endian, align, mop;
3876
3877 int total; /* total bytes */
3878 int elements; /* elements per vector */
3879 int r;
3880 int size = a->sz;
3881
3882 if (!a->p && a->rm != 0) {
3883 /* For non-postindexed accesses the Rm field must be 0 */
3884 return false;
3885 }
3886 if (size == 3 && !a->q && a->selem != 1) {
3887 return false;
3888 }
3889 if (!fp_access_check(s)) {
3890 return true;
3891 }
3892
3893 if (a->rn == 31) {
3894 gen_check_sp_alignment(s);
3895 }
3896
3897 /* For our purposes, bytes are always little-endian. */
3898 endian = s->be_data;
3899 if (size == 0) {
3900 endian = MO_LE;
3901 }
3902
3903 total = a->rpt * a->selem * (a->q ? 16 : 8);
3904 tcg_rn = cpu_reg_sp(s, a->rn);
3905
3906 /*
3907 * Issue the MTE check vs the logical repeat count, before we
3908 * promote consecutive little-endian elements below.
3909 */
3910 clean_addr = gen_mte_checkN(s, tcg_rn, false, a->p || a->rn != 31, total,
3911 finalize_memop_asimd(s, size));
3912
3913 /*
3914 * Consecutive little-endian elements from a single register
3915 * can be promoted to a larger little-endian operation.
3916 */
3917 align = MO_ALIGN;
3918 if (a->selem == 1 && endian == MO_LE) {
3919 align = pow2_align(size);
3920 size = 3;
3921 }
3922 if (!s->align_mem) {
3923 align = 0;
3924 }
3925 mop = endian | size | align;
3926
3927 elements = (a->q ? 16 : 8) >> size;
3928 tcg_ebytes = tcg_constant_i64(1 << size);
3929 for (r = 0; r < a->rpt; r++) {
3930 int e;
3931 for (e = 0; e < elements; e++) {
3932 int xs;
3933 for (xs = 0; xs < a->selem; xs++) {
3934 int tt = (a->rt + r + xs) % 32;
3935 do_vec_ld(s, tt, e, clean_addr, mop);
3936 tcg_gen_add_i64(clean_addr, clean_addr, tcg_ebytes);
3937 }
3938 }
3939 }
3940
3941 /*
3942 * For non-quad operations, setting a slice of the low 64 bits of
3943 * the register clears the high 64 bits (in the ARM ARM pseudocode
3944 * this is implicit in the fact that 'rval' is a 64 bit wide
3945 * variable). For quad operations, we might still need to zero
3946 * the high bits of SVE.
3947 */
3948 for (r = 0; r < a->rpt * a->selem; r++) {
3949 int tt = (a->rt + r) % 32;
3950 clear_vec_high(s, a->q, tt);
3951 }
3952
3953 if (a->p) {
3954 if (a->rm == 31) {
3955 tcg_gen_addi_i64(tcg_rn, tcg_rn, total);
3956 } else {
3957 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm));
3958 }
3959 }
3960 return true;
3961 }
3962
trans_ST_mult(DisasContext * s,arg_ldst_mult * a)3963 static bool trans_ST_mult(DisasContext *s, arg_ldst_mult *a)
3964 {
3965 TCGv_i64 clean_addr, tcg_rn, tcg_ebytes;
3966 MemOp endian, align, mop;
3967
3968 int total; /* total bytes */
3969 int elements; /* elements per vector */
3970 int r;
3971 int size = a->sz;
3972
3973 if (!a->p && a->rm != 0) {
3974 /* For non-postindexed accesses the Rm field must be 0 */
3975 return false;
3976 }
3977 if (size == 3 && !a->q && a->selem != 1) {
3978 return false;
3979 }
3980 if (!fp_access_check(s)) {
3981 return true;
3982 }
3983
3984 if (a->rn == 31) {
3985 gen_check_sp_alignment(s);
3986 }
3987
3988 /* For our purposes, bytes are always little-endian. */
3989 endian = s->be_data;
3990 if (size == 0) {
3991 endian = MO_LE;
3992 }
3993
3994 total = a->rpt * a->selem * (a->q ? 16 : 8);
3995 tcg_rn = cpu_reg_sp(s, a->rn);
3996
3997 /*
3998 * Issue the MTE check vs the logical repeat count, before we
3999 * promote consecutive little-endian elements below.
4000 */
4001 clean_addr = gen_mte_checkN(s, tcg_rn, true, a->p || a->rn != 31, total,
4002 finalize_memop_asimd(s, size));
4003
4004 /*
4005 * Consecutive little-endian elements from a single register
4006 * can be promoted to a larger little-endian operation.
4007 */
4008 align = MO_ALIGN;
4009 if (a->selem == 1 && endian == MO_LE) {
4010 align = pow2_align(size);
4011 size = 3;
4012 }
4013 if (!s->align_mem) {
4014 align = 0;
4015 }
4016 mop = endian | size | align;
4017
4018 elements = (a->q ? 16 : 8) >> size;
4019 tcg_ebytes = tcg_constant_i64(1 << size);
4020 for (r = 0; r < a->rpt; r++) {
4021 int e;
4022 for (e = 0; e < elements; e++) {
4023 int xs;
4024 for (xs = 0; xs < a->selem; xs++) {
4025 int tt = (a->rt + r + xs) % 32;
4026 do_vec_st(s, tt, e, clean_addr, mop);
4027 tcg_gen_add_i64(clean_addr, clean_addr, tcg_ebytes);
4028 }
4029 }
4030 }
4031
4032 if (a->p) {
4033 if (a->rm == 31) {
4034 tcg_gen_addi_i64(tcg_rn, tcg_rn, total);
4035 } else {
4036 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm));
4037 }
4038 }
4039 return true;
4040 }
4041
trans_ST_single(DisasContext * s,arg_ldst_single * a)4042 static bool trans_ST_single(DisasContext *s, arg_ldst_single *a)
4043 {
4044 int xs, total, rt;
4045 TCGv_i64 clean_addr, tcg_rn, tcg_ebytes;
4046 MemOp mop;
4047
4048 if (!a->p && a->rm != 0) {
4049 return false;
4050 }
4051 if (!fp_access_check(s)) {
4052 return true;
4053 }
4054
4055 if (a->rn == 31) {
4056 gen_check_sp_alignment(s);
4057 }
4058
4059 total = a->selem << a->scale;
4060 tcg_rn = cpu_reg_sp(s, a->rn);
4061
4062 mop = finalize_memop_asimd(s, a->scale);
4063 clean_addr = gen_mte_checkN(s, tcg_rn, true, a->p || a->rn != 31,
4064 total, mop);
4065
4066 tcg_ebytes = tcg_constant_i64(1 << a->scale);
4067 for (xs = 0, rt = a->rt; xs < a->selem; xs++, rt = (rt + 1) % 32) {
4068 do_vec_st(s, rt, a->index, clean_addr, mop);
4069 tcg_gen_add_i64(clean_addr, clean_addr, tcg_ebytes);
4070 }
4071
4072 if (a->p) {
4073 if (a->rm == 31) {
4074 tcg_gen_addi_i64(tcg_rn, tcg_rn, total);
4075 } else {
4076 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm));
4077 }
4078 }
4079 return true;
4080 }
4081
trans_LD_single(DisasContext * s,arg_ldst_single * a)4082 static bool trans_LD_single(DisasContext *s, arg_ldst_single *a)
4083 {
4084 int xs, total, rt;
4085 TCGv_i64 clean_addr, tcg_rn, tcg_ebytes;
4086 MemOp mop;
4087
4088 if (!a->p && a->rm != 0) {
4089 return false;
4090 }
4091 if (!fp_access_check(s)) {
4092 return true;
4093 }
4094
4095 if (a->rn == 31) {
4096 gen_check_sp_alignment(s);
4097 }
4098
4099 total = a->selem << a->scale;
4100 tcg_rn = cpu_reg_sp(s, a->rn);
4101
4102 mop = finalize_memop_asimd(s, a->scale);
4103 clean_addr = gen_mte_checkN(s, tcg_rn, false, a->p || a->rn != 31,
4104 total, mop);
4105
4106 tcg_ebytes = tcg_constant_i64(1 << a->scale);
4107 for (xs = 0, rt = a->rt; xs < a->selem; xs++, rt = (rt + 1) % 32) {
4108 do_vec_ld(s, rt, a->index, clean_addr, mop);
4109 tcg_gen_add_i64(clean_addr, clean_addr, tcg_ebytes);
4110 }
4111
4112 if (a->p) {
4113 if (a->rm == 31) {
4114 tcg_gen_addi_i64(tcg_rn, tcg_rn, total);
4115 } else {
4116 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm));
4117 }
4118 }
4119 return true;
4120 }
4121
trans_LD_single_repl(DisasContext * s,arg_LD_single_repl * a)4122 static bool trans_LD_single_repl(DisasContext *s, arg_LD_single_repl *a)
4123 {
4124 int xs, total, rt;
4125 TCGv_i64 clean_addr, tcg_rn, tcg_ebytes;
4126 MemOp mop;
4127
4128 if (!a->p && a->rm != 0) {
4129 return false;
4130 }
4131 if (!fp_access_check(s)) {
4132 return true;
4133 }
4134
4135 if (a->rn == 31) {
4136 gen_check_sp_alignment(s);
4137 }
4138
4139 total = a->selem << a->scale;
4140 tcg_rn = cpu_reg_sp(s, a->rn);
4141
4142 mop = finalize_memop_asimd(s, a->scale);
4143 clean_addr = gen_mte_checkN(s, tcg_rn, false, a->p || a->rn != 31,
4144 total, mop);
4145
4146 tcg_ebytes = tcg_constant_i64(1 << a->scale);
4147 for (xs = 0, rt = a->rt; xs < a->selem; xs++, rt = (rt + 1) % 32) {
4148 /* Load and replicate to all elements */
4149 TCGv_i64 tcg_tmp = tcg_temp_new_i64();
4150
4151 tcg_gen_qemu_ld_i64(tcg_tmp, clean_addr, get_mem_index(s), mop);
4152 tcg_gen_gvec_dup_i64(a->scale, vec_full_reg_offset(s, rt),
4153 (a->q + 1) * 8, vec_full_reg_size(s), tcg_tmp);
4154 tcg_gen_add_i64(clean_addr, clean_addr, tcg_ebytes);
4155 }
4156
4157 if (a->p) {
4158 if (a->rm == 31) {
4159 tcg_gen_addi_i64(tcg_rn, tcg_rn, total);
4160 } else {
4161 tcg_gen_add_i64(tcg_rn, tcg_rn, cpu_reg(s, a->rm));
4162 }
4163 }
4164 return true;
4165 }
4166
trans_STZGM(DisasContext * s,arg_ldst_tag * a)4167 static bool trans_STZGM(DisasContext *s, arg_ldst_tag *a)
4168 {
4169 TCGv_i64 addr, clean_addr, tcg_rt;
4170 int size = 4 << s->dcz_blocksize;
4171
4172 if (!dc_isar_feature(aa64_mte, s)) {
4173 return false;
4174 }
4175 if (s->current_el == 0) {
4176 return false;
4177 }
4178
4179 if (a->rn == 31) {
4180 gen_check_sp_alignment(s);
4181 }
4182
4183 addr = read_cpu_reg_sp(s, a->rn, true);
4184 tcg_gen_addi_i64(addr, addr, a->imm);
4185 tcg_rt = cpu_reg(s, a->rt);
4186
4187 if (s->ata[0]) {
4188 gen_helper_stzgm_tags(tcg_env, addr, tcg_rt);
4189 }
4190 /*
4191 * The non-tags portion of STZGM is mostly like DC_ZVA,
4192 * except the alignment happens before the access.
4193 */
4194 clean_addr = clean_data_tbi(s, addr);
4195 tcg_gen_andi_i64(clean_addr, clean_addr, -size);
4196 gen_helper_dc_zva(tcg_env, clean_addr);
4197 return true;
4198 }
4199
trans_STGM(DisasContext * s,arg_ldst_tag * a)4200 static bool trans_STGM(DisasContext *s, arg_ldst_tag *a)
4201 {
4202 TCGv_i64 addr, clean_addr, tcg_rt;
4203
4204 if (!dc_isar_feature(aa64_mte, s)) {
4205 return false;
4206 }
4207 if (s->current_el == 0) {
4208 return false;
4209 }
4210
4211 if (a->rn == 31) {
4212 gen_check_sp_alignment(s);
4213 }
4214
4215 addr = read_cpu_reg_sp(s, a->rn, true);
4216 tcg_gen_addi_i64(addr, addr, a->imm);
4217 tcg_rt = cpu_reg(s, a->rt);
4218
4219 if (s->ata[0]) {
4220 gen_helper_stgm(tcg_env, addr, tcg_rt);
4221 } else {
4222 MMUAccessType acc = MMU_DATA_STORE;
4223 int size = 4 << s->gm_blocksize;
4224
4225 clean_addr = clean_data_tbi(s, addr);
4226 tcg_gen_andi_i64(clean_addr, clean_addr, -size);
4227 gen_probe_access(s, clean_addr, acc, size);
4228 }
4229 return true;
4230 }
4231
trans_LDGM(DisasContext * s,arg_ldst_tag * a)4232 static bool trans_LDGM(DisasContext *s, arg_ldst_tag *a)
4233 {
4234 TCGv_i64 addr, clean_addr, tcg_rt;
4235
4236 if (!dc_isar_feature(aa64_mte, s)) {
4237 return false;
4238 }
4239 if (s->current_el == 0) {
4240 return false;
4241 }
4242
4243 if (a->rn == 31) {
4244 gen_check_sp_alignment(s);
4245 }
4246
4247 addr = read_cpu_reg_sp(s, a->rn, true);
4248 tcg_gen_addi_i64(addr, addr, a->imm);
4249 tcg_rt = cpu_reg(s, a->rt);
4250
4251 if (s->ata[0]) {
4252 gen_helper_ldgm(tcg_rt, tcg_env, addr);
4253 } else {
4254 MMUAccessType acc = MMU_DATA_LOAD;
4255 int size = 4 << s->gm_blocksize;
4256
4257 clean_addr = clean_data_tbi(s, addr);
4258 tcg_gen_andi_i64(clean_addr, clean_addr, -size);
4259 gen_probe_access(s, clean_addr, acc, size);
4260 /* The result tags are zeros. */
4261 tcg_gen_movi_i64(tcg_rt, 0);
4262 }
4263 return true;
4264 }
4265
trans_LDG(DisasContext * s,arg_ldst_tag * a)4266 static bool trans_LDG(DisasContext *s, arg_ldst_tag *a)
4267 {
4268 TCGv_i64 addr, clean_addr, tcg_rt;
4269
4270 if (!dc_isar_feature(aa64_mte_insn_reg, s)) {
4271 return false;
4272 }
4273
4274 if (a->rn == 31) {
4275 gen_check_sp_alignment(s);
4276 }
4277
4278 addr = read_cpu_reg_sp(s, a->rn, true);
4279 if (!a->p) {
4280 /* pre-index or signed offset */
4281 tcg_gen_addi_i64(addr, addr, a->imm);
4282 }
4283
4284 tcg_gen_andi_i64(addr, addr, -TAG_GRANULE);
4285 tcg_rt = cpu_reg(s, a->rt);
4286 if (s->ata[0]) {
4287 gen_helper_ldg(tcg_rt, tcg_env, addr, tcg_rt);
4288 } else {
4289 /*
4290 * Tag access disabled: we must check for aborts on the load
4291 * load from [rn+offset], and then insert a 0 tag into rt.
4292 */
4293 clean_addr = clean_data_tbi(s, addr);
4294 gen_probe_access(s, clean_addr, MMU_DATA_LOAD, MO_8);
4295 gen_address_with_allocation_tag0(tcg_rt, tcg_rt);
4296 }
4297
4298 if (a->w) {
4299 /* pre-index or post-index */
4300 if (a->p) {
4301 /* post-index */
4302 tcg_gen_addi_i64(addr, addr, a->imm);
4303 }
4304 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), addr);
4305 }
4306 return true;
4307 }
4308
do_STG(DisasContext * s,arg_ldst_tag * a,bool is_zero,bool is_pair)4309 static bool do_STG(DisasContext *s, arg_ldst_tag *a, bool is_zero, bool is_pair)
4310 {
4311 TCGv_i64 addr, tcg_rt;
4312
4313 if (a->rn == 31) {
4314 gen_check_sp_alignment(s);
4315 }
4316
4317 addr = read_cpu_reg_sp(s, a->rn, true);
4318 if (!a->p) {
4319 /* pre-index or signed offset */
4320 tcg_gen_addi_i64(addr, addr, a->imm);
4321 }
4322 tcg_rt = cpu_reg_sp(s, a->rt);
4323 if (!s->ata[0]) {
4324 /*
4325 * For STG and ST2G, we need to check alignment and probe memory.
4326 * TODO: For STZG and STZ2G, we could rely on the stores below,
4327 * at least for system mode; user-only won't enforce alignment.
4328 */
4329 if (is_pair) {
4330 gen_helper_st2g_stub(tcg_env, addr);
4331 } else {
4332 gen_helper_stg_stub(tcg_env, addr);
4333 }
4334 } else if (tb_cflags(s->base.tb) & CF_PARALLEL) {
4335 if (is_pair) {
4336 gen_helper_st2g_parallel(tcg_env, addr, tcg_rt);
4337 } else {
4338 gen_helper_stg_parallel(tcg_env, addr, tcg_rt);
4339 }
4340 } else {
4341 if (is_pair) {
4342 gen_helper_st2g(tcg_env, addr, tcg_rt);
4343 } else {
4344 gen_helper_stg(tcg_env, addr, tcg_rt);
4345 }
4346 }
4347
4348 if (is_zero) {
4349 TCGv_i64 clean_addr = clean_data_tbi(s, addr);
4350 TCGv_i64 zero64 = tcg_constant_i64(0);
4351 TCGv_i128 zero128 = tcg_temp_new_i128();
4352 int mem_index = get_mem_index(s);
4353 MemOp mop = finalize_memop(s, MO_128 | MO_ALIGN);
4354
4355 tcg_gen_concat_i64_i128(zero128, zero64, zero64);
4356
4357 /* This is 1 or 2 atomic 16-byte operations. */
4358 tcg_gen_qemu_st_i128(zero128, clean_addr, mem_index, mop);
4359 if (is_pair) {
4360 tcg_gen_addi_i64(clean_addr, clean_addr, 16);
4361 tcg_gen_qemu_st_i128(zero128, clean_addr, mem_index, mop);
4362 }
4363 }
4364
4365 if (a->w) {
4366 /* pre-index or post-index */
4367 if (a->p) {
4368 /* post-index */
4369 tcg_gen_addi_i64(addr, addr, a->imm);
4370 }
4371 tcg_gen_mov_i64(cpu_reg_sp(s, a->rn), addr);
4372 }
4373 return true;
4374 }
4375
4376 TRANS_FEAT(STG, aa64_mte_insn_reg, do_STG, a, false, false)
4377 TRANS_FEAT(STZG, aa64_mte_insn_reg, do_STG, a, true, false)
4378 TRANS_FEAT(ST2G, aa64_mte_insn_reg, do_STG, a, false, true)
4379 TRANS_FEAT(STZ2G, aa64_mte_insn_reg, do_STG, a, true, true)
4380
4381 typedef void SetFn(TCGv_env, TCGv_i32, TCGv_i32);
4382
do_SET(DisasContext * s,arg_set * a,bool is_epilogue,bool is_setg,SetFn fn)4383 static bool do_SET(DisasContext *s, arg_set *a, bool is_epilogue,
4384 bool is_setg, SetFn fn)
4385 {
4386 int memidx;
4387 uint32_t syndrome, desc = 0;
4388
4389 if (is_setg && !dc_isar_feature(aa64_mte, s)) {
4390 return false;
4391 }
4392
4393 /*
4394 * UNPREDICTABLE cases: we choose to UNDEF, which allows
4395 * us to pull this check before the CheckMOPSEnabled() test
4396 * (which we do in the helper function)
4397 */
4398 if (a->rs == a->rn || a->rs == a->rd || a->rn == a->rd ||
4399 a->rd == 31 || a->rn == 31) {
4400 return false;
4401 }
4402
4403 memidx = get_a64_user_mem_index(s, a->unpriv);
4404
4405 /*
4406 * We pass option_a == true, matching our implementation;
4407 * we pass wrong_option == false: helper function may set that bit.
4408 */
4409 syndrome = syn_mop(true, is_setg, (a->nontemp << 1) | a->unpriv,
4410 is_epilogue, false, true, a->rd, a->rs, a->rn);
4411
4412 if (is_setg ? s->ata[a->unpriv] : s->mte_active[a->unpriv]) {
4413 /* We may need to do MTE tag checking, so assemble the descriptor */
4414 desc = FIELD_DP32(desc, MTEDESC, TBI, s->tbid);
4415 desc = FIELD_DP32(desc, MTEDESC, TCMA, s->tcma);
4416 desc = FIELD_DP32(desc, MTEDESC, WRITE, true);
4417 /* SIZEM1 and ALIGN we leave 0 (byte write) */
4418 }
4419 /* The helper function always needs the memidx even with MTE disabled */
4420 desc = FIELD_DP32(desc, MTEDESC, MIDX, memidx);
4421
4422 /*
4423 * The helper needs the register numbers, but since they're in
4424 * the syndrome anyway, we let it extract them from there rather
4425 * than passing in an extra three integer arguments.
4426 */
4427 fn(tcg_env, tcg_constant_i32(syndrome), tcg_constant_i32(desc));
4428 return true;
4429 }
4430
4431 TRANS_FEAT(SETP, aa64_mops, do_SET, a, false, false, gen_helper_setp)
4432 TRANS_FEAT(SETM, aa64_mops, do_SET, a, false, false, gen_helper_setm)
4433 TRANS_FEAT(SETE, aa64_mops, do_SET, a, true, false, gen_helper_sete)
4434 TRANS_FEAT(SETGP, aa64_mops, do_SET, a, false, true, gen_helper_setgp)
4435 TRANS_FEAT(SETGM, aa64_mops, do_SET, a, false, true, gen_helper_setgm)
4436 TRANS_FEAT(SETGE, aa64_mops, do_SET, a, true, true, gen_helper_setge)
4437
4438 typedef void CpyFn(TCGv_env, TCGv_i32, TCGv_i32, TCGv_i32);
4439
do_CPY(DisasContext * s,arg_cpy * a,bool is_epilogue,CpyFn fn)4440 static bool do_CPY(DisasContext *s, arg_cpy *a, bool is_epilogue, CpyFn fn)
4441 {
4442 int rmemidx, wmemidx;
4443 uint32_t syndrome, rdesc = 0, wdesc = 0;
4444 bool wunpriv = extract32(a->options, 0, 1);
4445 bool runpriv = extract32(a->options, 1, 1);
4446
4447 /*
4448 * UNPREDICTABLE cases: we choose to UNDEF, which allows
4449 * us to pull this check before the CheckMOPSEnabled() test
4450 * (which we do in the helper function)
4451 */
4452 if (a->rs == a->rn || a->rs == a->rd || a->rn == a->rd ||
4453 a->rd == 31 || a->rs == 31 || a->rn == 31) {
4454 return false;
4455 }
4456
4457 rmemidx = get_a64_user_mem_index(s, runpriv);
4458 wmemidx = get_a64_user_mem_index(s, wunpriv);
4459
4460 /*
4461 * We pass option_a == true, matching our implementation;
4462 * we pass wrong_option == false: helper function may set that bit.
4463 */
4464 syndrome = syn_mop(false, false, a->options, is_epilogue,
4465 false, true, a->rd, a->rs, a->rn);
4466
4467 /* If we need to do MTE tag checking, assemble the descriptors */
4468 if (s->mte_active[runpriv]) {
4469 rdesc = FIELD_DP32(rdesc, MTEDESC, TBI, s->tbid);
4470 rdesc = FIELD_DP32(rdesc, MTEDESC, TCMA, s->tcma);
4471 }
4472 if (s->mte_active[wunpriv]) {
4473 wdesc = FIELD_DP32(wdesc, MTEDESC, TBI, s->tbid);
4474 wdesc = FIELD_DP32(wdesc, MTEDESC, TCMA, s->tcma);
4475 wdesc = FIELD_DP32(wdesc, MTEDESC, WRITE, true);
4476 }
4477 /* The helper function needs these parts of the descriptor regardless */
4478 rdesc = FIELD_DP32(rdesc, MTEDESC, MIDX, rmemidx);
4479 wdesc = FIELD_DP32(wdesc, MTEDESC, MIDX, wmemidx);
4480
4481 /*
4482 * The helper needs the register numbers, but since they're in
4483 * the syndrome anyway, we let it extract them from there rather
4484 * than passing in an extra three integer arguments.
4485 */
4486 fn(tcg_env, tcg_constant_i32(syndrome), tcg_constant_i32(wdesc),
4487 tcg_constant_i32(rdesc));
4488 return true;
4489 }
4490
4491 TRANS_FEAT(CPYP, aa64_mops, do_CPY, a, false, gen_helper_cpyp)
4492 TRANS_FEAT(CPYM, aa64_mops, do_CPY, a, false, gen_helper_cpym)
4493 TRANS_FEAT(CPYE, aa64_mops, do_CPY, a, true, gen_helper_cpye)
4494 TRANS_FEAT(CPYFP, aa64_mops, do_CPY, a, false, gen_helper_cpyfp)
4495 TRANS_FEAT(CPYFM, aa64_mops, do_CPY, a, false, gen_helper_cpyfm)
4496 TRANS_FEAT(CPYFE, aa64_mops, do_CPY, a, true, gen_helper_cpyfe)
4497
4498 typedef void ArithTwoOp(TCGv_i64, TCGv_i64, TCGv_i64);
4499
gen_rri(DisasContext * s,arg_rri_sf * a,bool rd_sp,bool rn_sp,ArithTwoOp * fn)4500 static bool gen_rri(DisasContext *s, arg_rri_sf *a,
4501 bool rd_sp, bool rn_sp, ArithTwoOp *fn)
4502 {
4503 TCGv_i64 tcg_rn = rn_sp ? cpu_reg_sp(s, a->rn) : cpu_reg(s, a->rn);
4504 TCGv_i64 tcg_rd = rd_sp ? cpu_reg_sp(s, a->rd) : cpu_reg(s, a->rd);
4505 TCGv_i64 tcg_imm = tcg_constant_i64(a->imm);
4506
4507 fn(tcg_rd, tcg_rn, tcg_imm);
4508 if (!a->sf) {
4509 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
4510 }
4511 return true;
4512 }
4513
4514 /*
4515 * PC-rel. addressing
4516 */
4517
trans_ADR(DisasContext * s,arg_ri * a)4518 static bool trans_ADR(DisasContext *s, arg_ri *a)
4519 {
4520 gen_pc_plus_diff(s, cpu_reg(s, a->rd), a->imm);
4521 return true;
4522 }
4523
trans_ADRP(DisasContext * s,arg_ri * a)4524 static bool trans_ADRP(DisasContext *s, arg_ri *a)
4525 {
4526 int64_t offset = (int64_t)a->imm << 12;
4527
4528 /* The page offset is ok for CF_PCREL. */
4529 offset -= s->pc_curr & 0xfff;
4530 gen_pc_plus_diff(s, cpu_reg(s, a->rd), offset);
4531 return true;
4532 }
4533
4534 /*
4535 * Add/subtract (immediate)
4536 */
4537 TRANS(ADD_i, gen_rri, a, 1, 1, tcg_gen_add_i64)
4538 TRANS(SUB_i, gen_rri, a, 1, 1, tcg_gen_sub_i64)
4539 TRANS(ADDS_i, gen_rri, a, 0, 1, a->sf ? gen_add64_CC : gen_add32_CC)
4540 TRANS(SUBS_i, gen_rri, a, 0, 1, a->sf ? gen_sub64_CC : gen_sub32_CC)
4541
4542 /*
4543 * Add/subtract (immediate, with tags)
4544 */
4545
gen_add_sub_imm_with_tags(DisasContext * s,arg_rri_tag * a,bool sub_op)4546 static bool gen_add_sub_imm_with_tags(DisasContext *s, arg_rri_tag *a,
4547 bool sub_op)
4548 {
4549 TCGv_i64 tcg_rn, tcg_rd;
4550 int imm;
4551
4552 imm = a->uimm6 << LOG2_TAG_GRANULE;
4553 if (sub_op) {
4554 imm = -imm;
4555 }
4556
4557 tcg_rn = cpu_reg_sp(s, a->rn);
4558 tcg_rd = cpu_reg_sp(s, a->rd);
4559
4560 if (s->ata[0]) {
4561 gen_helper_addsubg(tcg_rd, tcg_env, tcg_rn,
4562 tcg_constant_i32(imm),
4563 tcg_constant_i32(a->uimm4));
4564 } else {
4565 tcg_gen_addi_i64(tcg_rd, tcg_rn, imm);
4566 gen_address_with_allocation_tag0(tcg_rd, tcg_rd);
4567 }
4568 return true;
4569 }
4570
TRANS_FEAT(ADDG_i,aa64_mte_insn_reg,gen_add_sub_imm_with_tags,a,false)4571 TRANS_FEAT(ADDG_i, aa64_mte_insn_reg, gen_add_sub_imm_with_tags, a, false)
4572 TRANS_FEAT(SUBG_i, aa64_mte_insn_reg, gen_add_sub_imm_with_tags, a, true)
4573
4574 /* The input should be a value in the bottom e bits (with higher
4575 * bits zero); returns that value replicated into every element
4576 * of size e in a 64 bit integer.
4577 */
4578 static uint64_t bitfield_replicate(uint64_t mask, unsigned int e)
4579 {
4580 assert(e != 0);
4581 while (e < 64) {
4582 mask |= mask << e;
4583 e *= 2;
4584 }
4585 return mask;
4586 }
4587
4588 /*
4589 * Logical (immediate)
4590 */
4591
4592 /*
4593 * Simplified variant of pseudocode DecodeBitMasks() for the case where we
4594 * only require the wmask. Returns false if the imms/immr/immn are a reserved
4595 * value (ie should cause a guest UNDEF exception), and true if they are
4596 * valid, in which case the decoded bit pattern is written to result.
4597 */
logic_imm_decode_wmask(uint64_t * result,unsigned int immn,unsigned int imms,unsigned int immr)4598 bool logic_imm_decode_wmask(uint64_t *result, unsigned int immn,
4599 unsigned int imms, unsigned int immr)
4600 {
4601 uint64_t mask;
4602 unsigned e, levels, s, r;
4603 int len;
4604
4605 assert(immn < 2 && imms < 64 && immr < 64);
4606
4607 /* The bit patterns we create here are 64 bit patterns which
4608 * are vectors of identical elements of size e = 2, 4, 8, 16, 32 or
4609 * 64 bits each. Each element contains the same value: a run
4610 * of between 1 and e-1 non-zero bits, rotated within the
4611 * element by between 0 and e-1 bits.
4612 *
4613 * The element size and run length are encoded into immn (1 bit)
4614 * and imms (6 bits) as follows:
4615 * 64 bit elements: immn = 1, imms = <length of run - 1>
4616 * 32 bit elements: immn = 0, imms = 0 : <length of run - 1>
4617 * 16 bit elements: immn = 0, imms = 10 : <length of run - 1>
4618 * 8 bit elements: immn = 0, imms = 110 : <length of run - 1>
4619 * 4 bit elements: immn = 0, imms = 1110 : <length of run - 1>
4620 * 2 bit elements: immn = 0, imms = 11110 : <length of run - 1>
4621 * Notice that immn = 0, imms = 11111x is the only combination
4622 * not covered by one of the above options; this is reserved.
4623 * Further, <length of run - 1> all-ones is a reserved pattern.
4624 *
4625 * In all cases the rotation is by immr % e (and immr is 6 bits).
4626 */
4627
4628 /* First determine the element size */
4629 len = 31 - clz32((immn << 6) | (~imms & 0x3f));
4630 if (len < 1) {
4631 /* This is the immn == 0, imms == 0x11111x case */
4632 return false;
4633 }
4634 e = 1 << len;
4635
4636 levels = e - 1;
4637 s = imms & levels;
4638 r = immr & levels;
4639
4640 if (s == levels) {
4641 /* <length of run - 1> mustn't be all-ones. */
4642 return false;
4643 }
4644
4645 /* Create the value of one element: s+1 set bits rotated
4646 * by r within the element (which is e bits wide)...
4647 */
4648 mask = MAKE_64BIT_MASK(0, s + 1);
4649 if (r) {
4650 mask = (mask >> r) | (mask << (e - r));
4651 mask &= MAKE_64BIT_MASK(0, e);
4652 }
4653 /* ...then replicate the element over the whole 64 bit value */
4654 mask = bitfield_replicate(mask, e);
4655 *result = mask;
4656 return true;
4657 }
4658
gen_rri_log(DisasContext * s,arg_rri_log * a,bool set_cc,void (* fn)(TCGv_i64,TCGv_i64,int64_t))4659 static bool gen_rri_log(DisasContext *s, arg_rri_log *a, bool set_cc,
4660 void (*fn)(TCGv_i64, TCGv_i64, int64_t))
4661 {
4662 TCGv_i64 tcg_rd, tcg_rn;
4663 uint64_t imm;
4664
4665 /* Some immediate field values are reserved. */
4666 if (!logic_imm_decode_wmask(&imm, extract32(a->dbm, 12, 1),
4667 extract32(a->dbm, 0, 6),
4668 extract32(a->dbm, 6, 6))) {
4669 return false;
4670 }
4671 if (!a->sf) {
4672 imm &= 0xffffffffull;
4673 }
4674
4675 tcg_rd = set_cc ? cpu_reg(s, a->rd) : cpu_reg_sp(s, a->rd);
4676 tcg_rn = cpu_reg(s, a->rn);
4677
4678 fn(tcg_rd, tcg_rn, imm);
4679 if (set_cc) {
4680 gen_logic_CC(a->sf, tcg_rd);
4681 }
4682 if (!a->sf) {
4683 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
4684 }
4685 return true;
4686 }
4687
TRANS(AND_i,gen_rri_log,a,false,tcg_gen_andi_i64)4688 TRANS(AND_i, gen_rri_log, a, false, tcg_gen_andi_i64)
4689 TRANS(ORR_i, gen_rri_log, a, false, tcg_gen_ori_i64)
4690 TRANS(EOR_i, gen_rri_log, a, false, tcg_gen_xori_i64)
4691 TRANS(ANDS_i, gen_rri_log, a, true, tcg_gen_andi_i64)
4692
4693 /*
4694 * Move wide (immediate)
4695 */
4696
4697 static bool trans_MOVZ(DisasContext *s, arg_movw *a)
4698 {
4699 int pos = a->hw << 4;
4700 tcg_gen_movi_i64(cpu_reg(s, a->rd), (uint64_t)a->imm << pos);
4701 return true;
4702 }
4703
trans_MOVN(DisasContext * s,arg_movw * a)4704 static bool trans_MOVN(DisasContext *s, arg_movw *a)
4705 {
4706 int pos = a->hw << 4;
4707 uint64_t imm = a->imm;
4708
4709 imm = ~(imm << pos);
4710 if (!a->sf) {
4711 imm = (uint32_t)imm;
4712 }
4713 tcg_gen_movi_i64(cpu_reg(s, a->rd), imm);
4714 return true;
4715 }
4716
trans_MOVK(DisasContext * s,arg_movw * a)4717 static bool trans_MOVK(DisasContext *s, arg_movw *a)
4718 {
4719 int pos = a->hw << 4;
4720 TCGv_i64 tcg_rd, tcg_im;
4721
4722 tcg_rd = cpu_reg(s, a->rd);
4723 tcg_im = tcg_constant_i64(a->imm);
4724 tcg_gen_deposit_i64(tcg_rd, tcg_rd, tcg_im, pos, 16);
4725 if (!a->sf) {
4726 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
4727 }
4728 return true;
4729 }
4730
4731 /*
4732 * Bitfield
4733 */
4734
trans_SBFM(DisasContext * s,arg_SBFM * a)4735 static bool trans_SBFM(DisasContext *s, arg_SBFM *a)
4736 {
4737 TCGv_i64 tcg_rd = cpu_reg(s, a->rd);
4738 TCGv_i64 tcg_tmp = read_cpu_reg(s, a->rn, 1);
4739 unsigned int bitsize = a->sf ? 64 : 32;
4740 unsigned int ri = a->immr;
4741 unsigned int si = a->imms;
4742 unsigned int pos, len;
4743
4744 if (si >= ri) {
4745 /* Wd<s-r:0> = Wn<s:r> */
4746 len = (si - ri) + 1;
4747 tcg_gen_sextract_i64(tcg_rd, tcg_tmp, ri, len);
4748 if (!a->sf) {
4749 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
4750 }
4751 } else {
4752 /* Wd<32+s-r,32-r> = Wn<s:0> */
4753 len = si + 1;
4754 pos = (bitsize - ri) & (bitsize - 1);
4755
4756 if (len < ri) {
4757 /*
4758 * Sign extend the destination field from len to fill the
4759 * balance of the word. Let the deposit below insert all
4760 * of those sign bits.
4761 */
4762 tcg_gen_sextract_i64(tcg_tmp, tcg_tmp, 0, len);
4763 len = ri;
4764 }
4765
4766 /*
4767 * We start with zero, and we haven't modified any bits outside
4768 * bitsize, therefore no final zero-extension is unneeded for !sf.
4769 */
4770 tcg_gen_deposit_z_i64(tcg_rd, tcg_tmp, pos, len);
4771 }
4772 return true;
4773 }
4774
trans_UBFM(DisasContext * s,arg_UBFM * a)4775 static bool trans_UBFM(DisasContext *s, arg_UBFM *a)
4776 {
4777 TCGv_i64 tcg_rd = cpu_reg(s, a->rd);
4778 TCGv_i64 tcg_tmp = read_cpu_reg(s, a->rn, 1);
4779 unsigned int bitsize = a->sf ? 64 : 32;
4780 unsigned int ri = a->immr;
4781 unsigned int si = a->imms;
4782 unsigned int pos, len;
4783
4784 tcg_rd = cpu_reg(s, a->rd);
4785 tcg_tmp = read_cpu_reg(s, a->rn, 1);
4786
4787 if (si >= ri) {
4788 /* Wd<s-r:0> = Wn<s:r> */
4789 len = (si - ri) + 1;
4790 tcg_gen_extract_i64(tcg_rd, tcg_tmp, ri, len);
4791 } else {
4792 /* Wd<32+s-r,32-r> = Wn<s:0> */
4793 len = si + 1;
4794 pos = (bitsize - ri) & (bitsize - 1);
4795 tcg_gen_deposit_z_i64(tcg_rd, tcg_tmp, pos, len);
4796 }
4797 return true;
4798 }
4799
trans_BFM(DisasContext * s,arg_BFM * a)4800 static bool trans_BFM(DisasContext *s, arg_BFM *a)
4801 {
4802 TCGv_i64 tcg_rd = cpu_reg(s, a->rd);
4803 TCGv_i64 tcg_tmp = read_cpu_reg(s, a->rn, 1);
4804 unsigned int bitsize = a->sf ? 64 : 32;
4805 unsigned int ri = a->immr;
4806 unsigned int si = a->imms;
4807 unsigned int pos, len;
4808
4809 tcg_rd = cpu_reg(s, a->rd);
4810 tcg_tmp = read_cpu_reg(s, a->rn, 1);
4811
4812 if (si >= ri) {
4813 /* Wd<s-r:0> = Wn<s:r> */
4814 tcg_gen_shri_i64(tcg_tmp, tcg_tmp, ri);
4815 len = (si - ri) + 1;
4816 pos = 0;
4817 } else {
4818 /* Wd<32+s-r,32-r> = Wn<s:0> */
4819 len = si + 1;
4820 pos = (bitsize - ri) & (bitsize - 1);
4821 }
4822
4823 tcg_gen_deposit_i64(tcg_rd, tcg_rd, tcg_tmp, pos, len);
4824 if (!a->sf) {
4825 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
4826 }
4827 return true;
4828 }
4829
trans_EXTR(DisasContext * s,arg_extract * a)4830 static bool trans_EXTR(DisasContext *s, arg_extract *a)
4831 {
4832 TCGv_i64 tcg_rd, tcg_rm, tcg_rn;
4833
4834 tcg_rd = cpu_reg(s, a->rd);
4835
4836 if (unlikely(a->imm == 0)) {
4837 /*
4838 * tcg shl_i32/shl_i64 is undefined for 32/64 bit shifts,
4839 * so an extract from bit 0 is a special case.
4840 */
4841 if (a->sf) {
4842 tcg_gen_mov_i64(tcg_rd, cpu_reg(s, a->rm));
4843 } else {
4844 tcg_gen_ext32u_i64(tcg_rd, cpu_reg(s, a->rm));
4845 }
4846 } else {
4847 tcg_rm = cpu_reg(s, a->rm);
4848 tcg_rn = cpu_reg(s, a->rn);
4849
4850 if (a->sf) {
4851 /* Specialization to ROR happens in EXTRACT2. */
4852 tcg_gen_extract2_i64(tcg_rd, tcg_rm, tcg_rn, a->imm);
4853 } else {
4854 TCGv_i32 t0 = tcg_temp_new_i32();
4855
4856 tcg_gen_extrl_i64_i32(t0, tcg_rm);
4857 if (a->rm == a->rn) {
4858 tcg_gen_rotri_i32(t0, t0, a->imm);
4859 } else {
4860 TCGv_i32 t1 = tcg_temp_new_i32();
4861 tcg_gen_extrl_i64_i32(t1, tcg_rn);
4862 tcg_gen_extract2_i32(t0, t0, t1, a->imm);
4863 }
4864 tcg_gen_extu_i32_i64(tcg_rd, t0);
4865 }
4866 }
4867 return true;
4868 }
4869
trans_TBL_TBX(DisasContext * s,arg_TBL_TBX * a)4870 static bool trans_TBL_TBX(DisasContext *s, arg_TBL_TBX *a)
4871 {
4872 if (fp_access_check(s)) {
4873 int len = (a->len + 1) * 16;
4874
4875 tcg_gen_gvec_2_ptr(vec_full_reg_offset(s, a->rd),
4876 vec_full_reg_offset(s, a->rm), tcg_env,
4877 a->q ? 16 : 8, vec_full_reg_size(s),
4878 (len << 6) | (a->tbx << 5) | a->rn,
4879 gen_helper_simd_tblx);
4880 }
4881 return true;
4882 }
4883
4884 typedef int simd_permute_idx_fn(int i, int part, int elements);
4885
do_simd_permute(DisasContext * s,arg_qrrr_e * a,simd_permute_idx_fn * fn,int part)4886 static bool do_simd_permute(DisasContext *s, arg_qrrr_e *a,
4887 simd_permute_idx_fn *fn, int part)
4888 {
4889 MemOp esz = a->esz;
4890 int datasize = a->q ? 16 : 8;
4891 int elements = datasize >> esz;
4892 TCGv_i64 tcg_res[2], tcg_ele;
4893
4894 if (esz == MO_64 && !a->q) {
4895 return false;
4896 }
4897 if (!fp_access_check(s)) {
4898 return true;
4899 }
4900
4901 tcg_res[0] = tcg_temp_new_i64();
4902 tcg_res[1] = a->q ? tcg_temp_new_i64() : NULL;
4903 tcg_ele = tcg_temp_new_i64();
4904
4905 for (int i = 0; i < elements; i++) {
4906 int o, w, idx;
4907
4908 idx = fn(i, part, elements);
4909 read_vec_element(s, tcg_ele, (idx & elements ? a->rm : a->rn),
4910 idx & (elements - 1), esz);
4911
4912 w = (i << (esz + 3)) / 64;
4913 o = (i << (esz + 3)) % 64;
4914 if (o == 0) {
4915 tcg_gen_mov_i64(tcg_res[w], tcg_ele);
4916 } else {
4917 tcg_gen_deposit_i64(tcg_res[w], tcg_res[w], tcg_ele, o, 8 << esz);
4918 }
4919 }
4920
4921 for (int i = a->q; i >= 0; --i) {
4922 write_vec_element(s, tcg_res[i], a->rd, i, MO_64);
4923 }
4924 clear_vec_high(s, a->q, a->rd);
4925 return true;
4926 }
4927
permute_load_uzp(int i,int part,int elements)4928 static int permute_load_uzp(int i, int part, int elements)
4929 {
4930 return 2 * i + part;
4931 }
4932
4933 TRANS(UZP1, do_simd_permute, a, permute_load_uzp, 0)
4934 TRANS(UZP2, do_simd_permute, a, permute_load_uzp, 1)
4935
permute_load_trn(int i,int part,int elements)4936 static int permute_load_trn(int i, int part, int elements)
4937 {
4938 return (i & 1) * elements + (i & ~1) + part;
4939 }
4940
4941 TRANS(TRN1, do_simd_permute, a, permute_load_trn, 0)
4942 TRANS(TRN2, do_simd_permute, a, permute_load_trn, 1)
4943
permute_load_zip(int i,int part,int elements)4944 static int permute_load_zip(int i, int part, int elements)
4945 {
4946 return (i & 1) * elements + ((part * elements + i) >> 1);
4947 }
4948
4949 TRANS(ZIP1, do_simd_permute, a, permute_load_zip, 0)
4950 TRANS(ZIP2, do_simd_permute, a, permute_load_zip, 1)
4951
4952 /*
4953 * Cryptographic AES, SHA, SHA512
4954 */
4955
4956 TRANS_FEAT(AESE, aa64_aes, do_gvec_op3_ool, a, 0, gen_helper_crypto_aese)
4957 TRANS_FEAT(AESD, aa64_aes, do_gvec_op3_ool, a, 0, gen_helper_crypto_aesd)
4958 TRANS_FEAT(AESMC, aa64_aes, do_gvec_op2_ool, a, 0, gen_helper_crypto_aesmc)
4959 TRANS_FEAT(AESIMC, aa64_aes, do_gvec_op2_ool, a, 0, gen_helper_crypto_aesimc)
4960
4961 TRANS_FEAT(SHA1C, aa64_sha1, do_gvec_op3_ool, a, 0, gen_helper_crypto_sha1c)
4962 TRANS_FEAT(SHA1P, aa64_sha1, do_gvec_op3_ool, a, 0, gen_helper_crypto_sha1p)
4963 TRANS_FEAT(SHA1M, aa64_sha1, do_gvec_op3_ool, a, 0, gen_helper_crypto_sha1m)
4964 TRANS_FEAT(SHA1SU0, aa64_sha1, do_gvec_op3_ool, a, 0, gen_helper_crypto_sha1su0)
4965
4966 TRANS_FEAT(SHA256H, aa64_sha256, do_gvec_op3_ool, a, 0, gen_helper_crypto_sha256h)
4967 TRANS_FEAT(SHA256H2, aa64_sha256, do_gvec_op3_ool, a, 0, gen_helper_crypto_sha256h2)
4968 TRANS_FEAT(SHA256SU1, aa64_sha256, do_gvec_op3_ool, a, 0, gen_helper_crypto_sha256su1)
4969
4970 TRANS_FEAT(SHA1H, aa64_sha1, do_gvec_op2_ool, a, 0, gen_helper_crypto_sha1h)
4971 TRANS_FEAT(SHA1SU1, aa64_sha1, do_gvec_op2_ool, a, 0, gen_helper_crypto_sha1su1)
4972 TRANS_FEAT(SHA256SU0, aa64_sha256, do_gvec_op2_ool, a, 0, gen_helper_crypto_sha256su0)
4973
4974 TRANS_FEAT(SHA512H, aa64_sha512, do_gvec_op3_ool, a, 0, gen_helper_crypto_sha512h)
4975 TRANS_FEAT(SHA512H2, aa64_sha512, do_gvec_op3_ool, a, 0, gen_helper_crypto_sha512h2)
4976 TRANS_FEAT(SHA512SU1, aa64_sha512, do_gvec_op3_ool, a, 0, gen_helper_crypto_sha512su1)
TRANS_FEAT(RAX1,aa64_sha3,do_gvec_fn3,a,gen_gvec_rax1)4977 TRANS_FEAT(RAX1, aa64_sha3, do_gvec_fn3, a, gen_gvec_rax1)
4978 TRANS_FEAT(SM3PARTW1, aa64_sm3, do_gvec_op3_ool, a, 0, gen_helper_crypto_sm3partw1)
4979 TRANS_FEAT(SM3PARTW2, aa64_sm3, do_gvec_op3_ool, a, 0, gen_helper_crypto_sm3partw2)
4980 TRANS_FEAT(SM4EKEY, aa64_sm4, do_gvec_op3_ool, a, 0, gen_helper_crypto_sm4ekey)
4981
4982 TRANS_FEAT(SHA512SU0, aa64_sha512, do_gvec_op2_ool, a, 0, gen_helper_crypto_sha512su0)
4983 TRANS_FEAT(SM4E, aa64_sm4, do_gvec_op3_ool, a, 0, gen_helper_crypto_sm4e)
4984
4985 TRANS_FEAT(EOR3, aa64_sha3, do_gvec_fn4, a, gen_gvec_eor3)
4986 TRANS_FEAT(BCAX, aa64_sha3, do_gvec_fn4, a, gen_gvec_bcax)
4987
4988 static bool trans_SM3SS1(DisasContext *s, arg_SM3SS1 *a)
4989 {
4990 if (!dc_isar_feature(aa64_sm3, s)) {
4991 return false;
4992 }
4993 if (fp_access_check(s)) {
4994 TCGv_i32 tcg_op1 = tcg_temp_new_i32();
4995 TCGv_i32 tcg_op2 = tcg_temp_new_i32();
4996 TCGv_i32 tcg_op3 = tcg_temp_new_i32();
4997 TCGv_i32 tcg_res = tcg_temp_new_i32();
4998
4999 read_vec_element_i32(s, tcg_op1, a->rn, 3, MO_32);
5000 read_vec_element_i32(s, tcg_op2, a->rm, 3, MO_32);
5001 read_vec_element_i32(s, tcg_op3, a->ra, 3, MO_32);
5002
5003 tcg_gen_rotri_i32(tcg_res, tcg_op1, 20);
5004 tcg_gen_add_i32(tcg_res, tcg_res, tcg_op2);
5005 tcg_gen_add_i32(tcg_res, tcg_res, tcg_op3);
5006 tcg_gen_rotri_i32(tcg_res, tcg_res, 25);
5007
5008 /* Clear the whole register first, then store bits [127:96]. */
5009 clear_vec(s, a->rd);
5010 write_vec_element_i32(s, tcg_res, a->rd, 3, MO_32);
5011 }
5012 return true;
5013 }
5014
do_crypto3i(DisasContext * s,arg_crypto3i * a,gen_helper_gvec_3 * fn)5015 static bool do_crypto3i(DisasContext *s, arg_crypto3i *a, gen_helper_gvec_3 *fn)
5016 {
5017 if (fp_access_check(s)) {
5018 gen_gvec_op3_ool(s, true, a->rd, a->rn, a->rm, a->imm, fn);
5019 }
5020 return true;
5021 }
TRANS_FEAT(SM3TT1A,aa64_sm3,do_crypto3i,a,gen_helper_crypto_sm3tt1a)5022 TRANS_FEAT(SM3TT1A, aa64_sm3, do_crypto3i, a, gen_helper_crypto_sm3tt1a)
5023 TRANS_FEAT(SM3TT1B, aa64_sm3, do_crypto3i, a, gen_helper_crypto_sm3tt1b)
5024 TRANS_FEAT(SM3TT2A, aa64_sm3, do_crypto3i, a, gen_helper_crypto_sm3tt2a)
5025 TRANS_FEAT(SM3TT2B, aa64_sm3, do_crypto3i, a, gen_helper_crypto_sm3tt2b)
5026
5027 static bool trans_XAR(DisasContext *s, arg_XAR *a)
5028 {
5029 if (!dc_isar_feature(aa64_sha3, s)) {
5030 return false;
5031 }
5032 if (fp_access_check(s)) {
5033 gen_gvec_xar(MO_64, vec_full_reg_offset(s, a->rd),
5034 vec_full_reg_offset(s, a->rn),
5035 vec_full_reg_offset(s, a->rm), a->imm, 16,
5036 vec_full_reg_size(s));
5037 }
5038 return true;
5039 }
5040
5041 /*
5042 * Advanced SIMD copy
5043 */
5044
decode_esz_idx(int imm,MemOp * pesz,unsigned * pidx)5045 static bool decode_esz_idx(int imm, MemOp *pesz, unsigned *pidx)
5046 {
5047 unsigned esz = ctz32(imm);
5048 if (esz <= MO_64) {
5049 *pesz = esz;
5050 *pidx = imm >> (esz + 1);
5051 return true;
5052 }
5053 return false;
5054 }
5055
trans_DUP_element_s(DisasContext * s,arg_DUP_element_s * a)5056 static bool trans_DUP_element_s(DisasContext *s, arg_DUP_element_s *a)
5057 {
5058 MemOp esz;
5059 unsigned idx;
5060
5061 if (!decode_esz_idx(a->imm, &esz, &idx)) {
5062 return false;
5063 }
5064 if (fp_access_check(s)) {
5065 /*
5066 * This instruction just extracts the specified element and
5067 * zero-extends it into the bottom of the destination register.
5068 */
5069 TCGv_i64 tmp = tcg_temp_new_i64();
5070 read_vec_element(s, tmp, a->rn, idx, esz);
5071 write_fp_dreg(s, a->rd, tmp);
5072 }
5073 return true;
5074 }
5075
trans_DUP_element_v(DisasContext * s,arg_DUP_element_v * a)5076 static bool trans_DUP_element_v(DisasContext *s, arg_DUP_element_v *a)
5077 {
5078 MemOp esz;
5079 unsigned idx;
5080
5081 if (!decode_esz_idx(a->imm, &esz, &idx)) {
5082 return false;
5083 }
5084 if (esz == MO_64 && !a->q) {
5085 return false;
5086 }
5087 if (fp_access_check(s)) {
5088 tcg_gen_gvec_dup_mem(esz, vec_full_reg_offset(s, a->rd),
5089 vec_reg_offset(s, a->rn, idx, esz),
5090 a->q ? 16 : 8, vec_full_reg_size(s));
5091 }
5092 return true;
5093 }
5094
trans_DUP_general(DisasContext * s,arg_DUP_general * a)5095 static bool trans_DUP_general(DisasContext *s, arg_DUP_general *a)
5096 {
5097 MemOp esz;
5098 unsigned idx;
5099
5100 if (!decode_esz_idx(a->imm, &esz, &idx)) {
5101 return false;
5102 }
5103 if (esz == MO_64 && !a->q) {
5104 return false;
5105 }
5106 if (fp_access_check(s)) {
5107 tcg_gen_gvec_dup_i64(esz, vec_full_reg_offset(s, a->rd),
5108 a->q ? 16 : 8, vec_full_reg_size(s),
5109 cpu_reg(s, a->rn));
5110 }
5111 return true;
5112 }
5113
do_smov_umov(DisasContext * s,arg_SMOV * a,MemOp is_signed)5114 static bool do_smov_umov(DisasContext *s, arg_SMOV *a, MemOp is_signed)
5115 {
5116 MemOp esz;
5117 unsigned idx;
5118
5119 if (!decode_esz_idx(a->imm, &esz, &idx)) {
5120 return false;
5121 }
5122 if (is_signed) {
5123 if (esz == MO_64 || (esz == MO_32 && !a->q)) {
5124 return false;
5125 }
5126 } else {
5127 if (esz == MO_64 ? !a->q : a->q) {
5128 return false;
5129 }
5130 }
5131 if (fp_access_check(s)) {
5132 TCGv_i64 tcg_rd = cpu_reg(s, a->rd);
5133 read_vec_element(s, tcg_rd, a->rn, idx, esz | is_signed);
5134 if (is_signed && !a->q) {
5135 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
5136 }
5137 }
5138 return true;
5139 }
5140
TRANS(SMOV,do_smov_umov,a,MO_SIGN)5141 TRANS(SMOV, do_smov_umov, a, MO_SIGN)
5142 TRANS(UMOV, do_smov_umov, a, 0)
5143
5144 static bool trans_INS_general(DisasContext *s, arg_INS_general *a)
5145 {
5146 MemOp esz;
5147 unsigned idx;
5148
5149 if (!decode_esz_idx(a->imm, &esz, &idx)) {
5150 return false;
5151 }
5152 if (fp_access_check(s)) {
5153 write_vec_element(s, cpu_reg(s, a->rn), a->rd, idx, esz);
5154 clear_vec_high(s, true, a->rd);
5155 }
5156 return true;
5157 }
5158
trans_INS_element(DisasContext * s,arg_INS_element * a)5159 static bool trans_INS_element(DisasContext *s, arg_INS_element *a)
5160 {
5161 MemOp esz;
5162 unsigned didx, sidx;
5163
5164 if (!decode_esz_idx(a->di, &esz, &didx)) {
5165 return false;
5166 }
5167 sidx = a->si >> esz;
5168 if (fp_access_check(s)) {
5169 TCGv_i64 tmp = tcg_temp_new_i64();
5170
5171 read_vec_element(s, tmp, a->rn, sidx, esz);
5172 write_vec_element(s, tmp, a->rd, didx, esz);
5173
5174 /* INS is considered a 128-bit write for SVE. */
5175 clear_vec_high(s, true, a->rd);
5176 }
5177 return true;
5178 }
5179
5180 /*
5181 * Advanced SIMD three same
5182 */
5183
5184 typedef struct FPScalar {
5185 void (*gen_h)(TCGv_i32, TCGv_i32, TCGv_i32, TCGv_ptr);
5186 void (*gen_s)(TCGv_i32, TCGv_i32, TCGv_i32, TCGv_ptr);
5187 void (*gen_d)(TCGv_i64, TCGv_i64, TCGv_i64, TCGv_ptr);
5188 } FPScalar;
5189
do_fp3_scalar_with_fpsttype(DisasContext * s,arg_rrr_e * a,const FPScalar * f,int mergereg,ARMFPStatusFlavour fpsttype)5190 static bool do_fp3_scalar_with_fpsttype(DisasContext *s, arg_rrr_e *a,
5191 const FPScalar *f, int mergereg,
5192 ARMFPStatusFlavour fpsttype)
5193 {
5194 switch (a->esz) {
5195 case MO_64:
5196 if (fp_access_check(s)) {
5197 TCGv_i64 t0 = read_fp_dreg(s, a->rn);
5198 TCGv_i64 t1 = read_fp_dreg(s, a->rm);
5199 f->gen_d(t0, t0, t1, fpstatus_ptr(fpsttype));
5200 write_fp_dreg_merging(s, a->rd, mergereg, t0);
5201 }
5202 break;
5203 case MO_32:
5204 if (fp_access_check(s)) {
5205 TCGv_i32 t0 = read_fp_sreg(s, a->rn);
5206 TCGv_i32 t1 = read_fp_sreg(s, a->rm);
5207 f->gen_s(t0, t0, t1, fpstatus_ptr(fpsttype));
5208 write_fp_sreg_merging(s, a->rd, mergereg, t0);
5209 }
5210 break;
5211 case MO_16:
5212 if (!dc_isar_feature(aa64_fp16, s)) {
5213 return false;
5214 }
5215 if (fp_access_check(s)) {
5216 TCGv_i32 t0 = read_fp_hreg(s, a->rn);
5217 TCGv_i32 t1 = read_fp_hreg(s, a->rm);
5218 f->gen_h(t0, t0, t1, fpstatus_ptr(fpsttype));
5219 write_fp_hreg_merging(s, a->rd, mergereg, t0);
5220 }
5221 break;
5222 default:
5223 return false;
5224 }
5225 return true;
5226 }
5227
do_fp3_scalar(DisasContext * s,arg_rrr_e * a,const FPScalar * f,int mergereg)5228 static bool do_fp3_scalar(DisasContext *s, arg_rrr_e *a, const FPScalar *f,
5229 int mergereg)
5230 {
5231 return do_fp3_scalar_with_fpsttype(s, a, f, mergereg,
5232 a->esz == MO_16 ?
5233 FPST_A64_F16 : FPST_A64);
5234 }
5235
do_fp3_scalar_ah_2fn(DisasContext * s,arg_rrr_e * a,const FPScalar * fnormal,const FPScalar * fah,int mergereg)5236 static bool do_fp3_scalar_ah_2fn(DisasContext *s, arg_rrr_e *a,
5237 const FPScalar *fnormal, const FPScalar *fah,
5238 int mergereg)
5239 {
5240 return do_fp3_scalar_with_fpsttype(s, a, s->fpcr_ah ? fah : fnormal,
5241 mergereg, select_ah_fpst(s, a->esz));
5242 }
5243
5244 /* Some insns need to call different helpers when FPCR.AH == 1 */
do_fp3_scalar_2fn(DisasContext * s,arg_rrr_e * a,const FPScalar * fnormal,const FPScalar * fah,int mergereg)5245 static bool do_fp3_scalar_2fn(DisasContext *s, arg_rrr_e *a,
5246 const FPScalar *fnormal,
5247 const FPScalar *fah,
5248 int mergereg)
5249 {
5250 return do_fp3_scalar(s, a, s->fpcr_ah ? fah : fnormal, mergereg);
5251 }
5252
5253 static const FPScalar f_scalar_fadd = {
5254 gen_helper_vfp_addh,
5255 gen_helper_vfp_adds,
5256 gen_helper_vfp_addd,
5257 };
5258 TRANS(FADD_s, do_fp3_scalar, a, &f_scalar_fadd, a->rn)
5259
5260 static const FPScalar f_scalar_fsub = {
5261 gen_helper_vfp_subh,
5262 gen_helper_vfp_subs,
5263 gen_helper_vfp_subd,
5264 };
5265 TRANS(FSUB_s, do_fp3_scalar, a, &f_scalar_fsub, a->rn)
5266
5267 static const FPScalar f_scalar_fdiv = {
5268 gen_helper_vfp_divh,
5269 gen_helper_vfp_divs,
5270 gen_helper_vfp_divd,
5271 };
5272 TRANS(FDIV_s, do_fp3_scalar, a, &f_scalar_fdiv, a->rn)
5273
5274 static const FPScalar f_scalar_fmul = {
5275 gen_helper_vfp_mulh,
5276 gen_helper_vfp_muls,
5277 gen_helper_vfp_muld,
5278 };
5279 TRANS(FMUL_s, do_fp3_scalar, a, &f_scalar_fmul, a->rn)
5280
5281 static const FPScalar f_scalar_fmax = {
5282 gen_helper_vfp_maxh,
5283 gen_helper_vfp_maxs,
5284 gen_helper_vfp_maxd,
5285 };
5286 static const FPScalar f_scalar_fmax_ah = {
5287 gen_helper_vfp_ah_maxh,
5288 gen_helper_vfp_ah_maxs,
5289 gen_helper_vfp_ah_maxd,
5290 };
5291 TRANS(FMAX_s, do_fp3_scalar_2fn, a, &f_scalar_fmax, &f_scalar_fmax_ah, a->rn)
5292
5293 static const FPScalar f_scalar_fmin = {
5294 gen_helper_vfp_minh,
5295 gen_helper_vfp_mins,
5296 gen_helper_vfp_mind,
5297 };
5298 static const FPScalar f_scalar_fmin_ah = {
5299 gen_helper_vfp_ah_minh,
5300 gen_helper_vfp_ah_mins,
5301 gen_helper_vfp_ah_mind,
5302 };
5303 TRANS(FMIN_s, do_fp3_scalar_2fn, a, &f_scalar_fmin, &f_scalar_fmin_ah, a->rn)
5304
5305 static const FPScalar f_scalar_fmaxnm = {
5306 gen_helper_vfp_maxnumh,
5307 gen_helper_vfp_maxnums,
5308 gen_helper_vfp_maxnumd,
5309 };
5310 TRANS(FMAXNM_s, do_fp3_scalar, a, &f_scalar_fmaxnm, a->rn)
5311
5312 static const FPScalar f_scalar_fminnm = {
5313 gen_helper_vfp_minnumh,
5314 gen_helper_vfp_minnums,
5315 gen_helper_vfp_minnumd,
5316 };
5317 TRANS(FMINNM_s, do_fp3_scalar, a, &f_scalar_fminnm, a->rn)
5318
5319 static const FPScalar f_scalar_fmulx = {
5320 gen_helper_advsimd_mulxh,
5321 gen_helper_vfp_mulxs,
5322 gen_helper_vfp_mulxd,
5323 };
5324 TRANS(FMULX_s, do_fp3_scalar, a, &f_scalar_fmulx, a->rn)
5325
gen_fnmul_h(TCGv_i32 d,TCGv_i32 n,TCGv_i32 m,TCGv_ptr s)5326 static void gen_fnmul_h(TCGv_i32 d, TCGv_i32 n, TCGv_i32 m, TCGv_ptr s)
5327 {
5328 gen_helper_vfp_mulh(d, n, m, s);
5329 gen_vfp_negh(d, d);
5330 }
5331
gen_fnmul_s(TCGv_i32 d,TCGv_i32 n,TCGv_i32 m,TCGv_ptr s)5332 static void gen_fnmul_s(TCGv_i32 d, TCGv_i32 n, TCGv_i32 m, TCGv_ptr s)
5333 {
5334 gen_helper_vfp_muls(d, n, m, s);
5335 gen_vfp_negs(d, d);
5336 }
5337
gen_fnmul_d(TCGv_i64 d,TCGv_i64 n,TCGv_i64 m,TCGv_ptr s)5338 static void gen_fnmul_d(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_ptr s)
5339 {
5340 gen_helper_vfp_muld(d, n, m, s);
5341 gen_vfp_negd(d, d);
5342 }
5343
gen_fnmul_ah_h(TCGv_i32 d,TCGv_i32 n,TCGv_i32 m,TCGv_ptr s)5344 static void gen_fnmul_ah_h(TCGv_i32 d, TCGv_i32 n, TCGv_i32 m, TCGv_ptr s)
5345 {
5346 gen_helper_vfp_mulh(d, n, m, s);
5347 gen_vfp_ah_negh(d, d);
5348 }
5349
gen_fnmul_ah_s(TCGv_i32 d,TCGv_i32 n,TCGv_i32 m,TCGv_ptr s)5350 static void gen_fnmul_ah_s(TCGv_i32 d, TCGv_i32 n, TCGv_i32 m, TCGv_ptr s)
5351 {
5352 gen_helper_vfp_muls(d, n, m, s);
5353 gen_vfp_ah_negs(d, d);
5354 }
5355
gen_fnmul_ah_d(TCGv_i64 d,TCGv_i64 n,TCGv_i64 m,TCGv_ptr s)5356 static void gen_fnmul_ah_d(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_ptr s)
5357 {
5358 gen_helper_vfp_muld(d, n, m, s);
5359 gen_vfp_ah_negd(d, d);
5360 }
5361
5362 static const FPScalar f_scalar_fnmul = {
5363 gen_fnmul_h,
5364 gen_fnmul_s,
5365 gen_fnmul_d,
5366 };
5367 static const FPScalar f_scalar_ah_fnmul = {
5368 gen_fnmul_ah_h,
5369 gen_fnmul_ah_s,
5370 gen_fnmul_ah_d,
5371 };
5372 TRANS(FNMUL_s, do_fp3_scalar_2fn, a, &f_scalar_fnmul, &f_scalar_ah_fnmul, a->rn)
5373
5374 static const FPScalar f_scalar_fcmeq = {
5375 gen_helper_advsimd_ceq_f16,
5376 gen_helper_neon_ceq_f32,
5377 gen_helper_neon_ceq_f64,
5378 };
5379 TRANS(FCMEQ_s, do_fp3_scalar, a, &f_scalar_fcmeq, a->rm)
5380
5381 static const FPScalar f_scalar_fcmge = {
5382 gen_helper_advsimd_cge_f16,
5383 gen_helper_neon_cge_f32,
5384 gen_helper_neon_cge_f64,
5385 };
5386 TRANS(FCMGE_s, do_fp3_scalar, a, &f_scalar_fcmge, a->rm)
5387
5388 static const FPScalar f_scalar_fcmgt = {
5389 gen_helper_advsimd_cgt_f16,
5390 gen_helper_neon_cgt_f32,
5391 gen_helper_neon_cgt_f64,
5392 };
5393 TRANS(FCMGT_s, do_fp3_scalar, a, &f_scalar_fcmgt, a->rm)
5394
5395 static const FPScalar f_scalar_facge = {
5396 gen_helper_advsimd_acge_f16,
5397 gen_helper_neon_acge_f32,
5398 gen_helper_neon_acge_f64,
5399 };
5400 TRANS(FACGE_s, do_fp3_scalar, a, &f_scalar_facge, a->rm)
5401
5402 static const FPScalar f_scalar_facgt = {
5403 gen_helper_advsimd_acgt_f16,
5404 gen_helper_neon_acgt_f32,
5405 gen_helper_neon_acgt_f64,
5406 };
5407 TRANS(FACGT_s, do_fp3_scalar, a, &f_scalar_facgt, a->rm)
5408
gen_fabd_h(TCGv_i32 d,TCGv_i32 n,TCGv_i32 m,TCGv_ptr s)5409 static void gen_fabd_h(TCGv_i32 d, TCGv_i32 n, TCGv_i32 m, TCGv_ptr s)
5410 {
5411 gen_helper_vfp_subh(d, n, m, s);
5412 gen_vfp_absh(d, d);
5413 }
5414
gen_fabd_s(TCGv_i32 d,TCGv_i32 n,TCGv_i32 m,TCGv_ptr s)5415 static void gen_fabd_s(TCGv_i32 d, TCGv_i32 n, TCGv_i32 m, TCGv_ptr s)
5416 {
5417 gen_helper_vfp_subs(d, n, m, s);
5418 gen_vfp_abss(d, d);
5419 }
5420
gen_fabd_d(TCGv_i64 d,TCGv_i64 n,TCGv_i64 m,TCGv_ptr s)5421 static void gen_fabd_d(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_ptr s)
5422 {
5423 gen_helper_vfp_subd(d, n, m, s);
5424 gen_vfp_absd(d, d);
5425 }
5426
gen_fabd_ah_h(TCGv_i32 d,TCGv_i32 n,TCGv_i32 m,TCGv_ptr s)5427 static void gen_fabd_ah_h(TCGv_i32 d, TCGv_i32 n, TCGv_i32 m, TCGv_ptr s)
5428 {
5429 gen_helper_vfp_subh(d, n, m, s);
5430 gen_vfp_ah_absh(d, d);
5431 }
5432
gen_fabd_ah_s(TCGv_i32 d,TCGv_i32 n,TCGv_i32 m,TCGv_ptr s)5433 static void gen_fabd_ah_s(TCGv_i32 d, TCGv_i32 n, TCGv_i32 m, TCGv_ptr s)
5434 {
5435 gen_helper_vfp_subs(d, n, m, s);
5436 gen_vfp_ah_abss(d, d);
5437 }
5438
gen_fabd_ah_d(TCGv_i64 d,TCGv_i64 n,TCGv_i64 m,TCGv_ptr s)5439 static void gen_fabd_ah_d(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m, TCGv_ptr s)
5440 {
5441 gen_helper_vfp_subd(d, n, m, s);
5442 gen_vfp_ah_absd(d, d);
5443 }
5444
5445 static const FPScalar f_scalar_fabd = {
5446 gen_fabd_h,
5447 gen_fabd_s,
5448 gen_fabd_d,
5449 };
5450 static const FPScalar f_scalar_ah_fabd = {
5451 gen_fabd_ah_h,
5452 gen_fabd_ah_s,
5453 gen_fabd_ah_d,
5454 };
5455 TRANS(FABD_s, do_fp3_scalar_2fn, a, &f_scalar_fabd, &f_scalar_ah_fabd, a->rn)
5456
5457 static const FPScalar f_scalar_frecps = {
5458 gen_helper_recpsf_f16,
5459 gen_helper_recpsf_f32,
5460 gen_helper_recpsf_f64,
5461 };
5462 static const FPScalar f_scalar_ah_frecps = {
5463 gen_helper_recpsf_ah_f16,
5464 gen_helper_recpsf_ah_f32,
5465 gen_helper_recpsf_ah_f64,
5466 };
5467 TRANS(FRECPS_s, do_fp3_scalar_ah_2fn, a,
5468 &f_scalar_frecps, &f_scalar_ah_frecps, a->rn)
5469
5470 static const FPScalar f_scalar_frsqrts = {
5471 gen_helper_rsqrtsf_f16,
5472 gen_helper_rsqrtsf_f32,
5473 gen_helper_rsqrtsf_f64,
5474 };
5475 static const FPScalar f_scalar_ah_frsqrts = {
5476 gen_helper_rsqrtsf_ah_f16,
5477 gen_helper_rsqrtsf_ah_f32,
5478 gen_helper_rsqrtsf_ah_f64,
5479 };
5480 TRANS(FRSQRTS_s, do_fp3_scalar_ah_2fn, a,
5481 &f_scalar_frsqrts, &f_scalar_ah_frsqrts, a->rn)
5482
do_fcmp0_s(DisasContext * s,arg_rr_e * a,const FPScalar * f,bool swap)5483 static bool do_fcmp0_s(DisasContext *s, arg_rr_e *a,
5484 const FPScalar *f, bool swap)
5485 {
5486 switch (a->esz) {
5487 case MO_64:
5488 if (fp_access_check(s)) {
5489 TCGv_i64 t0 = read_fp_dreg(s, a->rn);
5490 TCGv_i64 t1 = tcg_constant_i64(0);
5491 if (swap) {
5492 f->gen_d(t0, t1, t0, fpstatus_ptr(FPST_A64));
5493 } else {
5494 f->gen_d(t0, t0, t1, fpstatus_ptr(FPST_A64));
5495 }
5496 write_fp_dreg(s, a->rd, t0);
5497 }
5498 break;
5499 case MO_32:
5500 if (fp_access_check(s)) {
5501 TCGv_i32 t0 = read_fp_sreg(s, a->rn);
5502 TCGv_i32 t1 = tcg_constant_i32(0);
5503 if (swap) {
5504 f->gen_s(t0, t1, t0, fpstatus_ptr(FPST_A64));
5505 } else {
5506 f->gen_s(t0, t0, t1, fpstatus_ptr(FPST_A64));
5507 }
5508 write_fp_sreg(s, a->rd, t0);
5509 }
5510 break;
5511 case MO_16:
5512 if (!dc_isar_feature(aa64_fp16, s)) {
5513 return false;
5514 }
5515 if (fp_access_check(s)) {
5516 TCGv_i32 t0 = read_fp_hreg(s, a->rn);
5517 TCGv_i32 t1 = tcg_constant_i32(0);
5518 if (swap) {
5519 f->gen_h(t0, t1, t0, fpstatus_ptr(FPST_A64_F16));
5520 } else {
5521 f->gen_h(t0, t0, t1, fpstatus_ptr(FPST_A64_F16));
5522 }
5523 write_fp_sreg(s, a->rd, t0);
5524 }
5525 break;
5526 default:
5527 return false;
5528 }
5529 return true;
5530 }
5531
5532 TRANS(FCMEQ0_s, do_fcmp0_s, a, &f_scalar_fcmeq, false)
5533 TRANS(FCMGT0_s, do_fcmp0_s, a, &f_scalar_fcmgt, false)
5534 TRANS(FCMGE0_s, do_fcmp0_s, a, &f_scalar_fcmge, false)
5535 TRANS(FCMLT0_s, do_fcmp0_s, a, &f_scalar_fcmgt, true)
5536 TRANS(FCMLE0_s, do_fcmp0_s, a, &f_scalar_fcmge, true)
5537
do_satacc_s(DisasContext * s,arg_rrr_e * a,MemOp sgn_n,MemOp sgn_m,void (* gen_bhs)(TCGv_i64,TCGv_i64,TCGv_i64,TCGv_i64,MemOp),void (* gen_d)(TCGv_i64,TCGv_i64,TCGv_i64,TCGv_i64))5538 static bool do_satacc_s(DisasContext *s, arg_rrr_e *a,
5539 MemOp sgn_n, MemOp sgn_m,
5540 void (*gen_bhs)(TCGv_i64, TCGv_i64, TCGv_i64, TCGv_i64, MemOp),
5541 void (*gen_d)(TCGv_i64, TCGv_i64, TCGv_i64, TCGv_i64))
5542 {
5543 TCGv_i64 t0, t1, t2, qc;
5544 MemOp esz = a->esz;
5545
5546 if (!fp_access_check(s)) {
5547 return true;
5548 }
5549
5550 t0 = tcg_temp_new_i64();
5551 t1 = tcg_temp_new_i64();
5552 t2 = tcg_temp_new_i64();
5553 qc = tcg_temp_new_i64();
5554 read_vec_element(s, t1, a->rn, 0, esz | sgn_n);
5555 read_vec_element(s, t2, a->rm, 0, esz | sgn_m);
5556 tcg_gen_ld_i64(qc, tcg_env, offsetof(CPUARMState, vfp.qc));
5557
5558 if (esz == MO_64) {
5559 gen_d(t0, qc, t1, t2);
5560 } else {
5561 gen_bhs(t0, qc, t1, t2, esz);
5562 tcg_gen_ext_i64(t0, t0, esz);
5563 }
5564
5565 write_fp_dreg(s, a->rd, t0);
5566 tcg_gen_st_i64(qc, tcg_env, offsetof(CPUARMState, vfp.qc));
5567 return true;
5568 }
5569
TRANS(SQADD_s,do_satacc_s,a,MO_SIGN,MO_SIGN,gen_sqadd_bhs,gen_sqadd_d)5570 TRANS(SQADD_s, do_satacc_s, a, MO_SIGN, MO_SIGN, gen_sqadd_bhs, gen_sqadd_d)
5571 TRANS(SQSUB_s, do_satacc_s, a, MO_SIGN, MO_SIGN, gen_sqsub_bhs, gen_sqsub_d)
5572 TRANS(UQADD_s, do_satacc_s, a, 0, 0, gen_uqadd_bhs, gen_uqadd_d)
5573 TRANS(UQSUB_s, do_satacc_s, a, 0, 0, gen_uqsub_bhs, gen_uqsub_d)
5574 TRANS(SUQADD_s, do_satacc_s, a, MO_SIGN, 0, gen_suqadd_bhs, gen_suqadd_d)
5575 TRANS(USQADD_s, do_satacc_s, a, 0, MO_SIGN, gen_usqadd_bhs, gen_usqadd_d)
5576
5577 static bool do_int3_scalar_d(DisasContext *s, arg_rrr_e *a,
5578 void (*fn)(TCGv_i64, TCGv_i64, TCGv_i64))
5579 {
5580 if (fp_access_check(s)) {
5581 TCGv_i64 t0 = tcg_temp_new_i64();
5582 TCGv_i64 t1 = tcg_temp_new_i64();
5583
5584 read_vec_element(s, t0, a->rn, 0, MO_64);
5585 read_vec_element(s, t1, a->rm, 0, MO_64);
5586 fn(t0, t0, t1);
5587 write_fp_dreg(s, a->rd, t0);
5588 }
5589 return true;
5590 }
5591
5592 TRANS(SSHL_s, do_int3_scalar_d, a, gen_sshl_i64)
5593 TRANS(USHL_s, do_int3_scalar_d, a, gen_ushl_i64)
5594 TRANS(SRSHL_s, do_int3_scalar_d, a, gen_helper_neon_rshl_s64)
5595 TRANS(URSHL_s, do_int3_scalar_d, a, gen_helper_neon_rshl_u64)
5596 TRANS(ADD_s, do_int3_scalar_d, a, tcg_gen_add_i64)
5597 TRANS(SUB_s, do_int3_scalar_d, a, tcg_gen_sub_i64)
5598
5599 typedef struct ENVScalar2 {
5600 NeonGenTwoOpEnvFn *gen_bhs[3];
5601 NeonGenTwo64OpEnvFn *gen_d;
5602 } ENVScalar2;
5603
do_env_scalar2(DisasContext * s,arg_rrr_e * a,const ENVScalar2 * f)5604 static bool do_env_scalar2(DisasContext *s, arg_rrr_e *a, const ENVScalar2 *f)
5605 {
5606 if (!fp_access_check(s)) {
5607 return true;
5608 }
5609 if (a->esz == MO_64) {
5610 TCGv_i64 t0 = read_fp_dreg(s, a->rn);
5611 TCGv_i64 t1 = read_fp_dreg(s, a->rm);
5612 f->gen_d(t0, tcg_env, t0, t1);
5613 write_fp_dreg(s, a->rd, t0);
5614 } else {
5615 TCGv_i32 t0 = tcg_temp_new_i32();
5616 TCGv_i32 t1 = tcg_temp_new_i32();
5617
5618 read_vec_element_i32(s, t0, a->rn, 0, a->esz);
5619 read_vec_element_i32(s, t1, a->rm, 0, a->esz);
5620 f->gen_bhs[a->esz](t0, tcg_env, t0, t1);
5621 write_fp_sreg(s, a->rd, t0);
5622 }
5623 return true;
5624 }
5625
5626 static const ENVScalar2 f_scalar_sqshl = {
5627 { gen_helper_neon_qshl_s8,
5628 gen_helper_neon_qshl_s16,
5629 gen_helper_neon_qshl_s32 },
5630 gen_helper_neon_qshl_s64,
5631 };
5632 TRANS(SQSHL_s, do_env_scalar2, a, &f_scalar_sqshl)
5633
5634 static const ENVScalar2 f_scalar_uqshl = {
5635 { gen_helper_neon_qshl_u8,
5636 gen_helper_neon_qshl_u16,
5637 gen_helper_neon_qshl_u32 },
5638 gen_helper_neon_qshl_u64,
5639 };
5640 TRANS(UQSHL_s, do_env_scalar2, a, &f_scalar_uqshl)
5641
5642 static const ENVScalar2 f_scalar_sqrshl = {
5643 { gen_helper_neon_qrshl_s8,
5644 gen_helper_neon_qrshl_s16,
5645 gen_helper_neon_qrshl_s32 },
5646 gen_helper_neon_qrshl_s64,
5647 };
5648 TRANS(SQRSHL_s, do_env_scalar2, a, &f_scalar_sqrshl)
5649
5650 static const ENVScalar2 f_scalar_uqrshl = {
5651 { gen_helper_neon_qrshl_u8,
5652 gen_helper_neon_qrshl_u16,
5653 gen_helper_neon_qrshl_u32 },
5654 gen_helper_neon_qrshl_u64,
5655 };
5656 TRANS(UQRSHL_s, do_env_scalar2, a, &f_scalar_uqrshl)
5657
do_env_scalar2_hs(DisasContext * s,arg_rrr_e * a,const ENVScalar2 * f)5658 static bool do_env_scalar2_hs(DisasContext *s, arg_rrr_e *a,
5659 const ENVScalar2 *f)
5660 {
5661 if (a->esz == MO_16 || a->esz == MO_32) {
5662 return do_env_scalar2(s, a, f);
5663 }
5664 return false;
5665 }
5666
5667 static const ENVScalar2 f_scalar_sqdmulh = {
5668 { NULL, gen_helper_neon_qdmulh_s16, gen_helper_neon_qdmulh_s32 }
5669 };
5670 TRANS(SQDMULH_s, do_env_scalar2_hs, a, &f_scalar_sqdmulh)
5671
5672 static const ENVScalar2 f_scalar_sqrdmulh = {
5673 { NULL, gen_helper_neon_qrdmulh_s16, gen_helper_neon_qrdmulh_s32 }
5674 };
5675 TRANS(SQRDMULH_s, do_env_scalar2_hs, a, &f_scalar_sqrdmulh)
5676
5677 typedef struct ENVScalar3 {
5678 NeonGenThreeOpEnvFn *gen_hs[2];
5679 } ENVScalar3;
5680
do_env_scalar3_hs(DisasContext * s,arg_rrr_e * a,const ENVScalar3 * f)5681 static bool do_env_scalar3_hs(DisasContext *s, arg_rrr_e *a,
5682 const ENVScalar3 *f)
5683 {
5684 TCGv_i32 t0, t1, t2;
5685
5686 if (a->esz != MO_16 && a->esz != MO_32) {
5687 return false;
5688 }
5689 if (!fp_access_check(s)) {
5690 return true;
5691 }
5692
5693 t0 = tcg_temp_new_i32();
5694 t1 = tcg_temp_new_i32();
5695 t2 = tcg_temp_new_i32();
5696 read_vec_element_i32(s, t0, a->rn, 0, a->esz);
5697 read_vec_element_i32(s, t1, a->rm, 0, a->esz);
5698 read_vec_element_i32(s, t2, a->rd, 0, a->esz);
5699 f->gen_hs[a->esz - 1](t0, tcg_env, t0, t1, t2);
5700 write_fp_sreg(s, a->rd, t0);
5701 return true;
5702 }
5703
5704 static const ENVScalar3 f_scalar_sqrdmlah = {
5705 { gen_helper_neon_qrdmlah_s16, gen_helper_neon_qrdmlah_s32 }
5706 };
5707 TRANS_FEAT(SQRDMLAH_s, aa64_rdm, do_env_scalar3_hs, a, &f_scalar_sqrdmlah)
5708
5709 static const ENVScalar3 f_scalar_sqrdmlsh = {
5710 { gen_helper_neon_qrdmlsh_s16, gen_helper_neon_qrdmlsh_s32 }
5711 };
5712 TRANS_FEAT(SQRDMLSH_s, aa64_rdm, do_env_scalar3_hs, a, &f_scalar_sqrdmlsh)
5713
do_cmop_d(DisasContext * s,arg_rrr_e * a,TCGCond cond)5714 static bool do_cmop_d(DisasContext *s, arg_rrr_e *a, TCGCond cond)
5715 {
5716 if (fp_access_check(s)) {
5717 TCGv_i64 t0 = read_fp_dreg(s, a->rn);
5718 TCGv_i64 t1 = read_fp_dreg(s, a->rm);
5719 tcg_gen_negsetcond_i64(cond, t0, t0, t1);
5720 write_fp_dreg(s, a->rd, t0);
5721 }
5722 return true;
5723 }
5724
TRANS(CMGT_s,do_cmop_d,a,TCG_COND_GT)5725 TRANS(CMGT_s, do_cmop_d, a, TCG_COND_GT)
5726 TRANS(CMHI_s, do_cmop_d, a, TCG_COND_GTU)
5727 TRANS(CMGE_s, do_cmop_d, a, TCG_COND_GE)
5728 TRANS(CMHS_s, do_cmop_d, a, TCG_COND_GEU)
5729 TRANS(CMEQ_s, do_cmop_d, a, TCG_COND_EQ)
5730 TRANS(CMTST_s, do_cmop_d, a, TCG_COND_TSTNE)
5731
5732 static bool do_fp3_vector_with_fpsttype(DisasContext *s, arg_qrrr_e *a,
5733 int data,
5734 gen_helper_gvec_3_ptr * const fns[3],
5735 ARMFPStatusFlavour fpsttype)
5736 {
5737 MemOp esz = a->esz;
5738 int check = fp_access_check_vector_hsd(s, a->q, esz);
5739
5740 if (check <= 0) {
5741 return check == 0;
5742 }
5743
5744 gen_gvec_op3_fpst(s, a->q, a->rd, a->rn, a->rm, fpsttype,
5745 data, fns[esz - 1]);
5746 return true;
5747 }
5748
do_fp3_vector(DisasContext * s,arg_qrrr_e * a,int data,gen_helper_gvec_3_ptr * const fns[3])5749 static bool do_fp3_vector(DisasContext *s, arg_qrrr_e *a, int data,
5750 gen_helper_gvec_3_ptr * const fns[3])
5751 {
5752 return do_fp3_vector_with_fpsttype(s, a, data, fns,
5753 a->esz == MO_16 ?
5754 FPST_A64_F16 : FPST_A64);
5755 }
5756
do_fp3_vector_2fn(DisasContext * s,arg_qrrr_e * a,int data,gen_helper_gvec_3_ptr * const fnormal[3],gen_helper_gvec_3_ptr * const fah[3])5757 static bool do_fp3_vector_2fn(DisasContext *s, arg_qrrr_e *a, int data,
5758 gen_helper_gvec_3_ptr * const fnormal[3],
5759 gen_helper_gvec_3_ptr * const fah[3])
5760 {
5761 return do_fp3_vector(s, a, data, s->fpcr_ah ? fah : fnormal);
5762 }
5763
do_fp3_vector_ah_2fn(DisasContext * s,arg_qrrr_e * a,int data,gen_helper_gvec_3_ptr * const fnormal[3],gen_helper_gvec_3_ptr * const fah[3])5764 static bool do_fp3_vector_ah_2fn(DisasContext *s, arg_qrrr_e *a, int data,
5765 gen_helper_gvec_3_ptr * const fnormal[3],
5766 gen_helper_gvec_3_ptr * const fah[3])
5767 {
5768 return do_fp3_vector_with_fpsttype(s, a, data, s->fpcr_ah ? fah : fnormal,
5769 select_ah_fpst(s, a->esz));
5770 }
5771
5772 static gen_helper_gvec_3_ptr * const f_vector_fadd[3] = {
5773 gen_helper_gvec_fadd_h,
5774 gen_helper_gvec_fadd_s,
5775 gen_helper_gvec_fadd_d,
5776 };
5777 TRANS(FADD_v, do_fp3_vector, a, 0, f_vector_fadd)
5778
5779 static gen_helper_gvec_3_ptr * const f_vector_fsub[3] = {
5780 gen_helper_gvec_fsub_h,
5781 gen_helper_gvec_fsub_s,
5782 gen_helper_gvec_fsub_d,
5783 };
5784 TRANS(FSUB_v, do_fp3_vector, a, 0, f_vector_fsub)
5785
5786 static gen_helper_gvec_3_ptr * const f_vector_fdiv[3] = {
5787 gen_helper_gvec_fdiv_h,
5788 gen_helper_gvec_fdiv_s,
5789 gen_helper_gvec_fdiv_d,
5790 };
5791 TRANS(FDIV_v, do_fp3_vector, a, 0, f_vector_fdiv)
5792
5793 static gen_helper_gvec_3_ptr * const f_vector_fmul[3] = {
5794 gen_helper_gvec_fmul_h,
5795 gen_helper_gvec_fmul_s,
5796 gen_helper_gvec_fmul_d,
5797 };
5798 TRANS(FMUL_v, do_fp3_vector, a, 0, f_vector_fmul)
5799
5800 static gen_helper_gvec_3_ptr * const f_vector_fmax[3] = {
5801 gen_helper_gvec_fmax_h,
5802 gen_helper_gvec_fmax_s,
5803 gen_helper_gvec_fmax_d,
5804 };
5805 static gen_helper_gvec_3_ptr * const f_vector_fmax_ah[3] = {
5806 gen_helper_gvec_ah_fmax_h,
5807 gen_helper_gvec_ah_fmax_s,
5808 gen_helper_gvec_ah_fmax_d,
5809 };
5810 TRANS(FMAX_v, do_fp3_vector_2fn, a, 0, f_vector_fmax, f_vector_fmax_ah)
5811
5812 static gen_helper_gvec_3_ptr * const f_vector_fmin[3] = {
5813 gen_helper_gvec_fmin_h,
5814 gen_helper_gvec_fmin_s,
5815 gen_helper_gvec_fmin_d,
5816 };
5817 static gen_helper_gvec_3_ptr * const f_vector_fmin_ah[3] = {
5818 gen_helper_gvec_ah_fmin_h,
5819 gen_helper_gvec_ah_fmin_s,
5820 gen_helper_gvec_ah_fmin_d,
5821 };
5822 TRANS(FMIN_v, do_fp3_vector_2fn, a, 0, f_vector_fmin, f_vector_fmin_ah)
5823
5824 static gen_helper_gvec_3_ptr * const f_vector_fmaxnm[3] = {
5825 gen_helper_gvec_fmaxnum_h,
5826 gen_helper_gvec_fmaxnum_s,
5827 gen_helper_gvec_fmaxnum_d,
5828 };
5829 TRANS(FMAXNM_v, do_fp3_vector, a, 0, f_vector_fmaxnm)
5830
5831 static gen_helper_gvec_3_ptr * const f_vector_fminnm[3] = {
5832 gen_helper_gvec_fminnum_h,
5833 gen_helper_gvec_fminnum_s,
5834 gen_helper_gvec_fminnum_d,
5835 };
5836 TRANS(FMINNM_v, do_fp3_vector, a, 0, f_vector_fminnm)
5837
5838 static gen_helper_gvec_3_ptr * const f_vector_fmulx[3] = {
5839 gen_helper_gvec_fmulx_h,
5840 gen_helper_gvec_fmulx_s,
5841 gen_helper_gvec_fmulx_d,
5842 };
5843 TRANS(FMULX_v, do_fp3_vector, a, 0, f_vector_fmulx)
5844
5845 static gen_helper_gvec_3_ptr * const f_vector_fmla[3] = {
5846 gen_helper_gvec_vfma_h,
5847 gen_helper_gvec_vfma_s,
5848 gen_helper_gvec_vfma_d,
5849 };
5850 TRANS(FMLA_v, do_fp3_vector, a, 0, f_vector_fmla)
5851
5852 static gen_helper_gvec_3_ptr * const f_vector_fmls[3] = {
5853 gen_helper_gvec_vfms_h,
5854 gen_helper_gvec_vfms_s,
5855 gen_helper_gvec_vfms_d,
5856 };
5857 static gen_helper_gvec_3_ptr * const f_vector_fmls_ah[3] = {
5858 gen_helper_gvec_ah_vfms_h,
5859 gen_helper_gvec_ah_vfms_s,
5860 gen_helper_gvec_ah_vfms_d,
5861 };
5862 TRANS(FMLS_v, do_fp3_vector_2fn, a, 0, f_vector_fmls, f_vector_fmls_ah)
5863
5864 static gen_helper_gvec_3_ptr * const f_vector_fcmeq[3] = {
5865 gen_helper_gvec_fceq_h,
5866 gen_helper_gvec_fceq_s,
5867 gen_helper_gvec_fceq_d,
5868 };
5869 TRANS(FCMEQ_v, do_fp3_vector, a, 0, f_vector_fcmeq)
5870
5871 static gen_helper_gvec_3_ptr * const f_vector_fcmge[3] = {
5872 gen_helper_gvec_fcge_h,
5873 gen_helper_gvec_fcge_s,
5874 gen_helper_gvec_fcge_d,
5875 };
5876 TRANS(FCMGE_v, do_fp3_vector, a, 0, f_vector_fcmge)
5877
5878 static gen_helper_gvec_3_ptr * const f_vector_fcmgt[3] = {
5879 gen_helper_gvec_fcgt_h,
5880 gen_helper_gvec_fcgt_s,
5881 gen_helper_gvec_fcgt_d,
5882 };
5883 TRANS(FCMGT_v, do_fp3_vector, a, 0, f_vector_fcmgt)
5884
5885 static gen_helper_gvec_3_ptr * const f_vector_facge[3] = {
5886 gen_helper_gvec_facge_h,
5887 gen_helper_gvec_facge_s,
5888 gen_helper_gvec_facge_d,
5889 };
5890 TRANS(FACGE_v, do_fp3_vector, a, 0, f_vector_facge)
5891
5892 static gen_helper_gvec_3_ptr * const f_vector_facgt[3] = {
5893 gen_helper_gvec_facgt_h,
5894 gen_helper_gvec_facgt_s,
5895 gen_helper_gvec_facgt_d,
5896 };
5897 TRANS(FACGT_v, do_fp3_vector, a, 0, f_vector_facgt)
5898
5899 static gen_helper_gvec_3_ptr * const f_vector_fabd[3] = {
5900 gen_helper_gvec_fabd_h,
5901 gen_helper_gvec_fabd_s,
5902 gen_helper_gvec_fabd_d,
5903 };
5904 static gen_helper_gvec_3_ptr * const f_vector_ah_fabd[3] = {
5905 gen_helper_gvec_ah_fabd_h,
5906 gen_helper_gvec_ah_fabd_s,
5907 gen_helper_gvec_ah_fabd_d,
5908 };
5909 TRANS(FABD_v, do_fp3_vector_2fn, a, 0, f_vector_fabd, f_vector_ah_fabd)
5910
5911 static gen_helper_gvec_3_ptr * const f_vector_frecps[3] = {
5912 gen_helper_gvec_recps_h,
5913 gen_helper_gvec_recps_s,
5914 gen_helper_gvec_recps_d,
5915 };
5916 static gen_helper_gvec_3_ptr * const f_vector_ah_frecps[3] = {
5917 gen_helper_gvec_ah_recps_h,
5918 gen_helper_gvec_ah_recps_s,
5919 gen_helper_gvec_ah_recps_d,
5920 };
5921 TRANS(FRECPS_v, do_fp3_vector_ah_2fn, a, 0, f_vector_frecps, f_vector_ah_frecps)
5922
5923 static gen_helper_gvec_3_ptr * const f_vector_frsqrts[3] = {
5924 gen_helper_gvec_rsqrts_h,
5925 gen_helper_gvec_rsqrts_s,
5926 gen_helper_gvec_rsqrts_d,
5927 };
5928 static gen_helper_gvec_3_ptr * const f_vector_ah_frsqrts[3] = {
5929 gen_helper_gvec_ah_rsqrts_h,
5930 gen_helper_gvec_ah_rsqrts_s,
5931 gen_helper_gvec_ah_rsqrts_d,
5932 };
5933 TRANS(FRSQRTS_v, do_fp3_vector_ah_2fn, a, 0, f_vector_frsqrts, f_vector_ah_frsqrts)
5934
5935 static gen_helper_gvec_3_ptr * const f_vector_faddp[3] = {
5936 gen_helper_gvec_faddp_h,
5937 gen_helper_gvec_faddp_s,
5938 gen_helper_gvec_faddp_d,
5939 };
5940 TRANS(FADDP_v, do_fp3_vector, a, 0, f_vector_faddp)
5941
5942 static gen_helper_gvec_3_ptr * const f_vector_fmaxp[3] = {
5943 gen_helper_gvec_fmaxp_h,
5944 gen_helper_gvec_fmaxp_s,
5945 gen_helper_gvec_fmaxp_d,
5946 };
5947 static gen_helper_gvec_3_ptr * const f_vector_ah_fmaxp[3] = {
5948 gen_helper_gvec_ah_fmaxp_h,
5949 gen_helper_gvec_ah_fmaxp_s,
5950 gen_helper_gvec_ah_fmaxp_d,
5951 };
5952 TRANS(FMAXP_v, do_fp3_vector_2fn, a, 0, f_vector_fmaxp, f_vector_ah_fmaxp)
5953
5954 static gen_helper_gvec_3_ptr * const f_vector_fminp[3] = {
5955 gen_helper_gvec_fminp_h,
5956 gen_helper_gvec_fminp_s,
5957 gen_helper_gvec_fminp_d,
5958 };
5959 static gen_helper_gvec_3_ptr * const f_vector_ah_fminp[3] = {
5960 gen_helper_gvec_ah_fminp_h,
5961 gen_helper_gvec_ah_fminp_s,
5962 gen_helper_gvec_ah_fminp_d,
5963 };
5964 TRANS(FMINP_v, do_fp3_vector_2fn, a, 0, f_vector_fminp, f_vector_ah_fminp)
5965
5966 static gen_helper_gvec_3_ptr * const f_vector_fmaxnmp[3] = {
5967 gen_helper_gvec_fmaxnump_h,
5968 gen_helper_gvec_fmaxnump_s,
5969 gen_helper_gvec_fmaxnump_d,
5970 };
5971 TRANS(FMAXNMP_v, do_fp3_vector, a, 0, f_vector_fmaxnmp)
5972
5973 static gen_helper_gvec_3_ptr * const f_vector_fminnmp[3] = {
5974 gen_helper_gvec_fminnump_h,
5975 gen_helper_gvec_fminnump_s,
5976 gen_helper_gvec_fminnump_d,
5977 };
5978 TRANS(FMINNMP_v, do_fp3_vector, a, 0, f_vector_fminnmp)
5979
do_fmlal(DisasContext * s,arg_qrrr_e * a,bool is_s,bool is_2)5980 static bool do_fmlal(DisasContext *s, arg_qrrr_e *a, bool is_s, bool is_2)
5981 {
5982 if (fp_access_check(s)) {
5983 int data = (is_2 << 1) | is_s;
5984 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, a->rd),
5985 vec_full_reg_offset(s, a->rn),
5986 vec_full_reg_offset(s, a->rm), tcg_env,
5987 a->q ? 16 : 8, vec_full_reg_size(s),
5988 data, gen_helper_gvec_fmlal_a64);
5989 }
5990 return true;
5991 }
5992
TRANS_FEAT(FMLAL_v,aa64_fhm,do_fmlal,a,false,false)5993 TRANS_FEAT(FMLAL_v, aa64_fhm, do_fmlal, a, false, false)
5994 TRANS_FEAT(FMLSL_v, aa64_fhm, do_fmlal, a, true, false)
5995 TRANS_FEAT(FMLAL2_v, aa64_fhm, do_fmlal, a, false, true)
5996 TRANS_FEAT(FMLSL2_v, aa64_fhm, do_fmlal, a, true, true)
5997
5998 TRANS(ADDP_v, do_gvec_fn3, a, gen_gvec_addp)
5999 TRANS(SMAXP_v, do_gvec_fn3_no64, a, gen_gvec_smaxp)
6000 TRANS(SMINP_v, do_gvec_fn3_no64, a, gen_gvec_sminp)
6001 TRANS(UMAXP_v, do_gvec_fn3_no64, a, gen_gvec_umaxp)
6002 TRANS(UMINP_v, do_gvec_fn3_no64, a, gen_gvec_uminp)
6003
6004 TRANS(AND_v, do_gvec_fn3, a, tcg_gen_gvec_and)
6005 TRANS(BIC_v, do_gvec_fn3, a, tcg_gen_gvec_andc)
6006 TRANS(ORR_v, do_gvec_fn3, a, tcg_gen_gvec_or)
6007 TRANS(ORN_v, do_gvec_fn3, a, tcg_gen_gvec_orc)
6008 TRANS(EOR_v, do_gvec_fn3, a, tcg_gen_gvec_xor)
6009
6010 static bool do_bitsel(DisasContext *s, bool is_q, int d, int a, int b, int c)
6011 {
6012 if (fp_access_check(s)) {
6013 gen_gvec_fn4(s, is_q, d, a, b, c, tcg_gen_gvec_bitsel, 0);
6014 }
6015 return true;
6016 }
6017
6018 TRANS(BSL_v, do_bitsel, a->q, a->rd, a->rd, a->rn, a->rm)
6019 TRANS(BIT_v, do_bitsel, a->q, a->rd, a->rm, a->rn, a->rd)
6020 TRANS(BIF_v, do_bitsel, a->q, a->rd, a->rm, a->rd, a->rn)
6021
TRANS(SQADD_v,do_gvec_fn3,a,gen_gvec_sqadd_qc)6022 TRANS(SQADD_v, do_gvec_fn3, a, gen_gvec_sqadd_qc)
6023 TRANS(UQADD_v, do_gvec_fn3, a, gen_gvec_uqadd_qc)
6024 TRANS(SQSUB_v, do_gvec_fn3, a, gen_gvec_sqsub_qc)
6025 TRANS(UQSUB_v, do_gvec_fn3, a, gen_gvec_uqsub_qc)
6026 TRANS(SUQADD_v, do_gvec_fn3, a, gen_gvec_suqadd_qc)
6027 TRANS(USQADD_v, do_gvec_fn3, a, gen_gvec_usqadd_qc)
6028
6029 TRANS(SSHL_v, do_gvec_fn3, a, gen_gvec_sshl)
6030 TRANS(USHL_v, do_gvec_fn3, a, gen_gvec_ushl)
6031 TRANS(SRSHL_v, do_gvec_fn3, a, gen_gvec_srshl)
6032 TRANS(URSHL_v, do_gvec_fn3, a, gen_gvec_urshl)
6033 TRANS(SQSHL_v, do_gvec_fn3, a, gen_neon_sqshl)
6034 TRANS(UQSHL_v, do_gvec_fn3, a, gen_neon_uqshl)
6035 TRANS(SQRSHL_v, do_gvec_fn3, a, gen_neon_sqrshl)
6036 TRANS(UQRSHL_v, do_gvec_fn3, a, gen_neon_uqrshl)
6037
6038 TRANS(ADD_v, do_gvec_fn3, a, tcg_gen_gvec_add)
6039 TRANS(SUB_v, do_gvec_fn3, a, tcg_gen_gvec_sub)
6040 TRANS(SHADD_v, do_gvec_fn3_no64, a, gen_gvec_shadd)
6041 TRANS(UHADD_v, do_gvec_fn3_no64, a, gen_gvec_uhadd)
6042 TRANS(SHSUB_v, do_gvec_fn3_no64, a, gen_gvec_shsub)
6043 TRANS(UHSUB_v, do_gvec_fn3_no64, a, gen_gvec_uhsub)
6044 TRANS(SRHADD_v, do_gvec_fn3_no64, a, gen_gvec_srhadd)
6045 TRANS(URHADD_v, do_gvec_fn3_no64, a, gen_gvec_urhadd)
6046 TRANS(SMAX_v, do_gvec_fn3_no64, a, tcg_gen_gvec_smax)
6047 TRANS(UMAX_v, do_gvec_fn3_no64, a, tcg_gen_gvec_umax)
6048 TRANS(SMIN_v, do_gvec_fn3_no64, a, tcg_gen_gvec_smin)
6049 TRANS(UMIN_v, do_gvec_fn3_no64, a, tcg_gen_gvec_umin)
6050 TRANS(SABA_v, do_gvec_fn3_no64, a, gen_gvec_saba)
6051 TRANS(UABA_v, do_gvec_fn3_no64, a, gen_gvec_uaba)
6052 TRANS(SABD_v, do_gvec_fn3_no64, a, gen_gvec_sabd)
6053 TRANS(UABD_v, do_gvec_fn3_no64, a, gen_gvec_uabd)
6054 TRANS(MUL_v, do_gvec_fn3_no64, a, tcg_gen_gvec_mul)
6055 TRANS(PMUL_v, do_gvec_op3_ool, a, 0, gen_helper_gvec_pmul_b)
6056 TRANS(MLA_v, do_gvec_fn3_no64, a, gen_gvec_mla)
6057 TRANS(MLS_v, do_gvec_fn3_no64, a, gen_gvec_mls)
6058
6059 static bool do_cmop_v(DisasContext *s, arg_qrrr_e *a, TCGCond cond)
6060 {
6061 if (a->esz == MO_64 && !a->q) {
6062 return false;
6063 }
6064 if (fp_access_check(s)) {
6065 tcg_gen_gvec_cmp(cond, a->esz,
6066 vec_full_reg_offset(s, a->rd),
6067 vec_full_reg_offset(s, a->rn),
6068 vec_full_reg_offset(s, a->rm),
6069 a->q ? 16 : 8, vec_full_reg_size(s));
6070 }
6071 return true;
6072 }
6073
TRANS(CMGT_v,do_cmop_v,a,TCG_COND_GT)6074 TRANS(CMGT_v, do_cmop_v, a, TCG_COND_GT)
6075 TRANS(CMHI_v, do_cmop_v, a, TCG_COND_GTU)
6076 TRANS(CMGE_v, do_cmop_v, a, TCG_COND_GE)
6077 TRANS(CMHS_v, do_cmop_v, a, TCG_COND_GEU)
6078 TRANS(CMEQ_v, do_cmop_v, a, TCG_COND_EQ)
6079 TRANS(CMTST_v, do_gvec_fn3, a, gen_gvec_cmtst)
6080
6081 TRANS(SQDMULH_v, do_gvec_fn3_no8_no64, a, gen_gvec_sqdmulh_qc)
6082 TRANS(SQRDMULH_v, do_gvec_fn3_no8_no64, a, gen_gvec_sqrdmulh_qc)
6083 TRANS_FEAT(SQRDMLAH_v, aa64_rdm, do_gvec_fn3_no8_no64, a, gen_gvec_sqrdmlah_qc)
6084 TRANS_FEAT(SQRDMLSH_v, aa64_rdm, do_gvec_fn3_no8_no64, a, gen_gvec_sqrdmlsh_qc)
6085
6086 static bool do_dot_vector(DisasContext *s, arg_qrrr_e *a,
6087 gen_helper_gvec_4 *fn)
6088 {
6089 if (fp_access_check(s)) {
6090 gen_gvec_op4_ool(s, a->q, a->rd, a->rn, a->rm, a->rd, 0, fn);
6091 }
6092 return true;
6093 }
6094
do_dot_vector_env(DisasContext * s,arg_qrrr_e * a,gen_helper_gvec_4_ptr * fn)6095 static bool do_dot_vector_env(DisasContext *s, arg_qrrr_e *a,
6096 gen_helper_gvec_4_ptr *fn)
6097 {
6098 if (fp_access_check(s)) {
6099 gen_gvec_op4_env(s, a->q, a->rd, a->rn, a->rm, a->rd, 0, fn);
6100 }
6101 return true;
6102 }
6103
TRANS_FEAT(SDOT_v,aa64_dp,do_dot_vector,a,gen_helper_gvec_sdot_b)6104 TRANS_FEAT(SDOT_v, aa64_dp, do_dot_vector, a, gen_helper_gvec_sdot_b)
6105 TRANS_FEAT(UDOT_v, aa64_dp, do_dot_vector, a, gen_helper_gvec_udot_b)
6106 TRANS_FEAT(USDOT_v, aa64_i8mm, do_dot_vector, a, gen_helper_gvec_usdot_b)
6107 TRANS_FEAT(BFDOT_v, aa64_bf16, do_dot_vector_env, a, gen_helper_gvec_bfdot)
6108 TRANS_FEAT(BFMMLA, aa64_bf16, do_dot_vector_env, a, gen_helper_gvec_bfmmla)
6109 TRANS_FEAT(SMMLA, aa64_i8mm, do_dot_vector, a, gen_helper_gvec_smmla_b)
6110 TRANS_FEAT(UMMLA, aa64_i8mm, do_dot_vector, a, gen_helper_gvec_ummla_b)
6111 TRANS_FEAT(USMMLA, aa64_i8mm, do_dot_vector, a, gen_helper_gvec_usmmla_b)
6112
6113 static bool trans_BFMLAL_v(DisasContext *s, arg_qrrr_e *a)
6114 {
6115 if (!dc_isar_feature(aa64_bf16, s)) {
6116 return false;
6117 }
6118 if (fp_access_check(s)) {
6119 /* Q bit selects BFMLALB vs BFMLALT. */
6120 gen_gvec_op4_fpst(s, true, a->rd, a->rn, a->rm, a->rd,
6121 s->fpcr_ah ? FPST_AH : FPST_A64, a->q,
6122 gen_helper_gvec_bfmlal);
6123 }
6124 return true;
6125 }
6126
6127 static gen_helper_gvec_3_ptr * const f_vector_fcadd[3] = {
6128 gen_helper_gvec_fcaddh,
6129 gen_helper_gvec_fcadds,
6130 gen_helper_gvec_fcaddd,
6131 };
6132 /*
6133 * Encode FPCR.AH into the data so the helper knows whether the
6134 * negations it does should avoid flipping the sign bit on a NaN
6135 */
6136 TRANS_FEAT(FCADD_90, aa64_fcma, do_fp3_vector, a, 0 | (s->fpcr_ah << 1),
6137 f_vector_fcadd)
6138 TRANS_FEAT(FCADD_270, aa64_fcma, do_fp3_vector, a, 1 | (s->fpcr_ah << 1),
6139 f_vector_fcadd)
6140
trans_FCMLA_v(DisasContext * s,arg_FCMLA_v * a)6141 static bool trans_FCMLA_v(DisasContext *s, arg_FCMLA_v *a)
6142 {
6143 static gen_helper_gvec_4_ptr * const fn[] = {
6144 [MO_16] = gen_helper_gvec_fcmlah,
6145 [MO_32] = gen_helper_gvec_fcmlas,
6146 [MO_64] = gen_helper_gvec_fcmlad,
6147 };
6148 int check;
6149
6150 if (!dc_isar_feature(aa64_fcma, s)) {
6151 return false;
6152 }
6153
6154 check = fp_access_check_vector_hsd(s, a->q, a->esz);
6155 if (check <= 0) {
6156 return check == 0;
6157 }
6158
6159 gen_gvec_op4_fpst(s, a->q, a->rd, a->rn, a->rm, a->rd,
6160 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64,
6161 a->rot | (s->fpcr_ah << 2), fn[a->esz]);
6162 return true;
6163 }
6164
6165 /*
6166 * Widening vector x vector/indexed.
6167 *
6168 * These read from the top or bottom half of a 128-bit vector.
6169 * After widening, optionally accumulate with a 128-bit vector.
6170 * Implement these inline, as the number of elements are limited
6171 * and the related SVE and SME operations on larger vectors use
6172 * even/odd elements instead of top/bottom half.
6173 *
6174 * If idx >= 0, operand 2 is indexed, otherwise vector.
6175 * If acc, operand 0 is loaded with rd.
6176 */
6177
6178 /* For low half, iterating up. */
do_3op_widening(DisasContext * s,MemOp memop,int top,int rd,int rn,int rm,int idx,NeonGenTwo64OpFn * fn,bool acc)6179 static bool do_3op_widening(DisasContext *s, MemOp memop, int top,
6180 int rd, int rn, int rm, int idx,
6181 NeonGenTwo64OpFn *fn, bool acc)
6182 {
6183 TCGv_i64 tcg_op0 = tcg_temp_new_i64();
6184 TCGv_i64 tcg_op1 = tcg_temp_new_i64();
6185 TCGv_i64 tcg_op2 = tcg_temp_new_i64();
6186 MemOp esz = memop & MO_SIZE;
6187 int half = 8 >> esz;
6188 int top_swap, top_half;
6189
6190 /* There are no 64x64->128 bit operations. */
6191 if (esz >= MO_64) {
6192 return false;
6193 }
6194 if (!fp_access_check(s)) {
6195 return true;
6196 }
6197
6198 if (idx >= 0) {
6199 read_vec_element(s, tcg_op2, rm, idx, memop);
6200 }
6201
6202 /*
6203 * For top half inputs, iterate forward; backward for bottom half.
6204 * This means the store to the destination will not occur until
6205 * overlapping input inputs are consumed.
6206 * Use top_swap to conditionally invert the forward iteration index.
6207 */
6208 top_swap = top ? 0 : half - 1;
6209 top_half = top ? half : 0;
6210
6211 for (int elt_fwd = 0; elt_fwd < half; ++elt_fwd) {
6212 int elt = elt_fwd ^ top_swap;
6213
6214 read_vec_element(s, tcg_op1, rn, elt + top_half, memop);
6215 if (idx < 0) {
6216 read_vec_element(s, tcg_op2, rm, elt + top_half, memop);
6217 }
6218 if (acc) {
6219 read_vec_element(s, tcg_op0, rd, elt, memop + 1);
6220 }
6221 fn(tcg_op0, tcg_op1, tcg_op2);
6222 write_vec_element(s, tcg_op0, rd, elt, esz + 1);
6223 }
6224 clear_vec_high(s, 1, rd);
6225 return true;
6226 }
6227
gen_muladd_i64(TCGv_i64 d,TCGv_i64 n,TCGv_i64 m)6228 static void gen_muladd_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m)
6229 {
6230 TCGv_i64 t = tcg_temp_new_i64();
6231 tcg_gen_mul_i64(t, n, m);
6232 tcg_gen_add_i64(d, d, t);
6233 }
6234
gen_mulsub_i64(TCGv_i64 d,TCGv_i64 n,TCGv_i64 m)6235 static void gen_mulsub_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m)
6236 {
6237 TCGv_i64 t = tcg_temp_new_i64();
6238 tcg_gen_mul_i64(t, n, m);
6239 tcg_gen_sub_i64(d, d, t);
6240 }
6241
6242 TRANS(SMULL_v, do_3op_widening,
6243 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6244 tcg_gen_mul_i64, false)
6245 TRANS(UMULL_v, do_3op_widening,
6246 a->esz, a->q, a->rd, a->rn, a->rm, -1,
6247 tcg_gen_mul_i64, false)
6248 TRANS(SMLAL_v, do_3op_widening,
6249 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6250 gen_muladd_i64, true)
6251 TRANS(UMLAL_v, do_3op_widening,
6252 a->esz, a->q, a->rd, a->rn, a->rm, -1,
6253 gen_muladd_i64, true)
6254 TRANS(SMLSL_v, do_3op_widening,
6255 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6256 gen_mulsub_i64, true)
6257 TRANS(UMLSL_v, do_3op_widening,
6258 a->esz, a->q, a->rd, a->rn, a->rm, -1,
6259 gen_mulsub_i64, true)
6260
6261 TRANS(SMULL_vi, do_3op_widening,
6262 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, a->idx,
6263 tcg_gen_mul_i64, false)
6264 TRANS(UMULL_vi, do_3op_widening,
6265 a->esz, a->q, a->rd, a->rn, a->rm, a->idx,
6266 tcg_gen_mul_i64, false)
6267 TRANS(SMLAL_vi, do_3op_widening,
6268 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, a->idx,
6269 gen_muladd_i64, true)
6270 TRANS(UMLAL_vi, do_3op_widening,
6271 a->esz, a->q, a->rd, a->rn, a->rm, a->idx,
6272 gen_muladd_i64, true)
6273 TRANS(SMLSL_vi, do_3op_widening,
6274 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, a->idx,
6275 gen_mulsub_i64, true)
6276 TRANS(UMLSL_vi, do_3op_widening,
6277 a->esz, a->q, a->rd, a->rn, a->rm, a->idx,
6278 gen_mulsub_i64, true)
6279
gen_sabd_i64(TCGv_i64 d,TCGv_i64 n,TCGv_i64 m)6280 static void gen_sabd_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m)
6281 {
6282 TCGv_i64 t1 = tcg_temp_new_i64();
6283 TCGv_i64 t2 = tcg_temp_new_i64();
6284
6285 tcg_gen_sub_i64(t1, n, m);
6286 tcg_gen_sub_i64(t2, m, n);
6287 tcg_gen_movcond_i64(TCG_COND_GE, d, n, m, t1, t2);
6288 }
6289
gen_uabd_i64(TCGv_i64 d,TCGv_i64 n,TCGv_i64 m)6290 static void gen_uabd_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m)
6291 {
6292 TCGv_i64 t1 = tcg_temp_new_i64();
6293 TCGv_i64 t2 = tcg_temp_new_i64();
6294
6295 tcg_gen_sub_i64(t1, n, m);
6296 tcg_gen_sub_i64(t2, m, n);
6297 tcg_gen_movcond_i64(TCG_COND_GEU, d, n, m, t1, t2);
6298 }
6299
gen_saba_i64(TCGv_i64 d,TCGv_i64 n,TCGv_i64 m)6300 static void gen_saba_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m)
6301 {
6302 TCGv_i64 t = tcg_temp_new_i64();
6303 gen_sabd_i64(t, n, m);
6304 tcg_gen_add_i64(d, d, t);
6305 }
6306
gen_uaba_i64(TCGv_i64 d,TCGv_i64 n,TCGv_i64 m)6307 static void gen_uaba_i64(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m)
6308 {
6309 TCGv_i64 t = tcg_temp_new_i64();
6310 gen_uabd_i64(t, n, m);
6311 tcg_gen_add_i64(d, d, t);
6312 }
6313
6314 TRANS(SADDL_v, do_3op_widening,
6315 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6316 tcg_gen_add_i64, false)
6317 TRANS(UADDL_v, do_3op_widening,
6318 a->esz, a->q, a->rd, a->rn, a->rm, -1,
6319 tcg_gen_add_i64, false)
6320 TRANS(SSUBL_v, do_3op_widening,
6321 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6322 tcg_gen_sub_i64, false)
6323 TRANS(USUBL_v, do_3op_widening,
6324 a->esz, a->q, a->rd, a->rn, a->rm, -1,
6325 tcg_gen_sub_i64, false)
6326 TRANS(SABDL_v, do_3op_widening,
6327 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6328 gen_sabd_i64, false)
6329 TRANS(UABDL_v, do_3op_widening,
6330 a->esz, a->q, a->rd, a->rn, a->rm, -1,
6331 gen_uabd_i64, false)
6332 TRANS(SABAL_v, do_3op_widening,
6333 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6334 gen_saba_i64, true)
6335 TRANS(UABAL_v, do_3op_widening,
6336 a->esz, a->q, a->rd, a->rn, a->rm, -1,
6337 gen_uaba_i64, true)
6338
gen_sqdmull_h(TCGv_i64 d,TCGv_i64 n,TCGv_i64 m)6339 static void gen_sqdmull_h(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m)
6340 {
6341 tcg_gen_mul_i64(d, n, m);
6342 gen_helper_neon_addl_saturate_s32(d, tcg_env, d, d);
6343 }
6344
gen_sqdmull_s(TCGv_i64 d,TCGv_i64 n,TCGv_i64 m)6345 static void gen_sqdmull_s(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m)
6346 {
6347 tcg_gen_mul_i64(d, n, m);
6348 gen_helper_neon_addl_saturate_s64(d, tcg_env, d, d);
6349 }
6350
gen_sqdmlal_h(TCGv_i64 d,TCGv_i64 n,TCGv_i64 m)6351 static void gen_sqdmlal_h(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m)
6352 {
6353 TCGv_i64 t = tcg_temp_new_i64();
6354
6355 tcg_gen_mul_i64(t, n, m);
6356 gen_helper_neon_addl_saturate_s32(t, tcg_env, t, t);
6357 gen_helper_neon_addl_saturate_s32(d, tcg_env, d, t);
6358 }
6359
gen_sqdmlal_s(TCGv_i64 d,TCGv_i64 n,TCGv_i64 m)6360 static void gen_sqdmlal_s(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m)
6361 {
6362 TCGv_i64 t = tcg_temp_new_i64();
6363
6364 tcg_gen_mul_i64(t, n, m);
6365 gen_helper_neon_addl_saturate_s64(t, tcg_env, t, t);
6366 gen_helper_neon_addl_saturate_s64(d, tcg_env, d, t);
6367 }
6368
gen_sqdmlsl_h(TCGv_i64 d,TCGv_i64 n,TCGv_i64 m)6369 static void gen_sqdmlsl_h(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m)
6370 {
6371 TCGv_i64 t = tcg_temp_new_i64();
6372
6373 tcg_gen_mul_i64(t, n, m);
6374 gen_helper_neon_addl_saturate_s32(t, tcg_env, t, t);
6375 tcg_gen_neg_i64(t, t);
6376 gen_helper_neon_addl_saturate_s32(d, tcg_env, d, t);
6377 }
6378
gen_sqdmlsl_s(TCGv_i64 d,TCGv_i64 n,TCGv_i64 m)6379 static void gen_sqdmlsl_s(TCGv_i64 d, TCGv_i64 n, TCGv_i64 m)
6380 {
6381 TCGv_i64 t = tcg_temp_new_i64();
6382
6383 tcg_gen_mul_i64(t, n, m);
6384 gen_helper_neon_addl_saturate_s64(t, tcg_env, t, t);
6385 tcg_gen_neg_i64(t, t);
6386 gen_helper_neon_addl_saturate_s64(d, tcg_env, d, t);
6387 }
6388
6389 TRANS(SQDMULL_v, do_3op_widening,
6390 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6391 a->esz == MO_16 ? gen_sqdmull_h : gen_sqdmull_s, false)
6392 TRANS(SQDMLAL_v, do_3op_widening,
6393 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6394 a->esz == MO_16 ? gen_sqdmlal_h : gen_sqdmlal_s, true)
6395 TRANS(SQDMLSL_v, do_3op_widening,
6396 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, -1,
6397 a->esz == MO_16 ? gen_sqdmlsl_h : gen_sqdmlsl_s, true)
6398
6399 TRANS(SQDMULL_vi, do_3op_widening,
6400 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, a->idx,
6401 a->esz == MO_16 ? gen_sqdmull_h : gen_sqdmull_s, false)
6402 TRANS(SQDMLAL_vi, do_3op_widening,
6403 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, a->idx,
6404 a->esz == MO_16 ? gen_sqdmlal_h : gen_sqdmlal_s, true)
6405 TRANS(SQDMLSL_vi, do_3op_widening,
6406 a->esz | MO_SIGN, a->q, a->rd, a->rn, a->rm, a->idx,
6407 a->esz == MO_16 ? gen_sqdmlsl_h : gen_sqdmlsl_s, true)
6408
do_addsub_wide(DisasContext * s,arg_qrrr_e * a,MemOp sign,bool sub)6409 static bool do_addsub_wide(DisasContext *s, arg_qrrr_e *a,
6410 MemOp sign, bool sub)
6411 {
6412 TCGv_i64 tcg_op0, tcg_op1;
6413 MemOp esz = a->esz;
6414 int half = 8 >> esz;
6415 bool top = a->q;
6416 int top_swap = top ? 0 : half - 1;
6417 int top_half = top ? half : 0;
6418
6419 /* There are no 64x64->128 bit operations. */
6420 if (esz >= MO_64) {
6421 return false;
6422 }
6423 if (!fp_access_check(s)) {
6424 return true;
6425 }
6426 tcg_op0 = tcg_temp_new_i64();
6427 tcg_op1 = tcg_temp_new_i64();
6428
6429 for (int elt_fwd = 0; elt_fwd < half; ++elt_fwd) {
6430 int elt = elt_fwd ^ top_swap;
6431
6432 read_vec_element(s, tcg_op1, a->rm, elt + top_half, esz | sign);
6433 read_vec_element(s, tcg_op0, a->rn, elt, esz + 1);
6434 if (sub) {
6435 tcg_gen_sub_i64(tcg_op0, tcg_op0, tcg_op1);
6436 } else {
6437 tcg_gen_add_i64(tcg_op0, tcg_op0, tcg_op1);
6438 }
6439 write_vec_element(s, tcg_op0, a->rd, elt, esz + 1);
6440 }
6441 clear_vec_high(s, 1, a->rd);
6442 return true;
6443 }
6444
TRANS(SADDW,do_addsub_wide,a,MO_SIGN,false)6445 TRANS(SADDW, do_addsub_wide, a, MO_SIGN, false)
6446 TRANS(UADDW, do_addsub_wide, a, 0, false)
6447 TRANS(SSUBW, do_addsub_wide, a, MO_SIGN, true)
6448 TRANS(USUBW, do_addsub_wide, a, 0, true)
6449
6450 static bool do_addsub_highnarrow(DisasContext *s, arg_qrrr_e *a,
6451 bool sub, bool round)
6452 {
6453 TCGv_i64 tcg_op0, tcg_op1;
6454 MemOp esz = a->esz;
6455 int half = 8 >> esz;
6456 bool top = a->q;
6457 int ebits = 8 << esz;
6458 uint64_t rbit = 1ull << (ebits - 1);
6459 int top_swap, top_half;
6460
6461 /* There are no 128x128->64 bit operations. */
6462 if (esz >= MO_64) {
6463 return false;
6464 }
6465 if (!fp_access_check(s)) {
6466 return true;
6467 }
6468 tcg_op0 = tcg_temp_new_i64();
6469 tcg_op1 = tcg_temp_new_i64();
6470
6471 /*
6472 * For top half inputs, iterate backward; forward for bottom half.
6473 * This means the store to the destination will not occur until
6474 * overlapping input inputs are consumed.
6475 */
6476 top_swap = top ? half - 1 : 0;
6477 top_half = top ? half : 0;
6478
6479 for (int elt_fwd = 0; elt_fwd < half; ++elt_fwd) {
6480 int elt = elt_fwd ^ top_swap;
6481
6482 read_vec_element(s, tcg_op1, a->rm, elt, esz + 1);
6483 read_vec_element(s, tcg_op0, a->rn, elt, esz + 1);
6484 if (sub) {
6485 tcg_gen_sub_i64(tcg_op0, tcg_op0, tcg_op1);
6486 } else {
6487 tcg_gen_add_i64(tcg_op0, tcg_op0, tcg_op1);
6488 }
6489 if (round) {
6490 tcg_gen_addi_i64(tcg_op0, tcg_op0, rbit);
6491 }
6492 tcg_gen_shri_i64(tcg_op0, tcg_op0, ebits);
6493 write_vec_element(s, tcg_op0, a->rd, elt + top_half, esz);
6494 }
6495 clear_vec_high(s, top, a->rd);
6496 return true;
6497 }
6498
TRANS(ADDHN,do_addsub_highnarrow,a,false,false)6499 TRANS(ADDHN, do_addsub_highnarrow, a, false, false)
6500 TRANS(SUBHN, do_addsub_highnarrow, a, true, false)
6501 TRANS(RADDHN, do_addsub_highnarrow, a, false, true)
6502 TRANS(RSUBHN, do_addsub_highnarrow, a, true, true)
6503
6504 static bool do_pmull(DisasContext *s, arg_qrrr_e *a, gen_helper_gvec_3 *fn)
6505 {
6506 if (fp_access_check(s)) {
6507 /* The Q field specifies lo/hi half input for these insns. */
6508 gen_gvec_op3_ool(s, true, a->rd, a->rn, a->rm, a->q, fn);
6509 }
6510 return true;
6511 }
6512
TRANS(PMULL_p8,do_pmull,a,gen_helper_neon_pmull_h)6513 TRANS(PMULL_p8, do_pmull, a, gen_helper_neon_pmull_h)
6514 TRANS_FEAT(PMULL_p64, aa64_pmull, do_pmull, a, gen_helper_gvec_pmull_q)
6515
6516 /*
6517 * Advanced SIMD scalar/vector x indexed element
6518 */
6519
6520 static bool do_fp3_scalar_idx(DisasContext *s, arg_rrx_e *a, const FPScalar *f)
6521 {
6522 switch (a->esz) {
6523 case MO_64:
6524 if (fp_access_check(s)) {
6525 TCGv_i64 t0 = read_fp_dreg(s, a->rn);
6526 TCGv_i64 t1 = tcg_temp_new_i64();
6527
6528 read_vec_element(s, t1, a->rm, a->idx, MO_64);
6529 f->gen_d(t0, t0, t1, fpstatus_ptr(FPST_A64));
6530 write_fp_dreg_merging(s, a->rd, a->rn, t0);
6531 }
6532 break;
6533 case MO_32:
6534 if (fp_access_check(s)) {
6535 TCGv_i32 t0 = read_fp_sreg(s, a->rn);
6536 TCGv_i32 t1 = tcg_temp_new_i32();
6537
6538 read_vec_element_i32(s, t1, a->rm, a->idx, MO_32);
6539 f->gen_s(t0, t0, t1, fpstatus_ptr(FPST_A64));
6540 write_fp_sreg_merging(s, a->rd, a->rn, t0);
6541 }
6542 break;
6543 case MO_16:
6544 if (!dc_isar_feature(aa64_fp16, s)) {
6545 return false;
6546 }
6547 if (fp_access_check(s)) {
6548 TCGv_i32 t0 = read_fp_hreg(s, a->rn);
6549 TCGv_i32 t1 = tcg_temp_new_i32();
6550
6551 read_vec_element_i32(s, t1, a->rm, a->idx, MO_16);
6552 f->gen_h(t0, t0, t1, fpstatus_ptr(FPST_A64_F16));
6553 write_fp_hreg_merging(s, a->rd, a->rn, t0);
6554 }
6555 break;
6556 default:
6557 g_assert_not_reached();
6558 }
6559 return true;
6560 }
6561
6562 TRANS(FMUL_si, do_fp3_scalar_idx, a, &f_scalar_fmul)
6563 TRANS(FMULX_si, do_fp3_scalar_idx, a, &f_scalar_fmulx)
6564
do_fmla_scalar_idx(DisasContext * s,arg_rrx_e * a,bool neg)6565 static bool do_fmla_scalar_idx(DisasContext *s, arg_rrx_e *a, bool neg)
6566 {
6567 switch (a->esz) {
6568 case MO_64:
6569 if (fp_access_check(s)) {
6570 TCGv_i64 t0 = read_fp_dreg(s, a->rd);
6571 TCGv_i64 t1 = read_fp_dreg(s, a->rn);
6572 TCGv_i64 t2 = tcg_temp_new_i64();
6573
6574 read_vec_element(s, t2, a->rm, a->idx, MO_64);
6575 if (neg) {
6576 gen_vfp_maybe_ah_negd(s, t1, t1);
6577 }
6578 gen_helper_vfp_muladdd(t0, t1, t2, t0, fpstatus_ptr(FPST_A64));
6579 write_fp_dreg_merging(s, a->rd, a->rd, t0);
6580 }
6581 break;
6582 case MO_32:
6583 if (fp_access_check(s)) {
6584 TCGv_i32 t0 = read_fp_sreg(s, a->rd);
6585 TCGv_i32 t1 = read_fp_sreg(s, a->rn);
6586 TCGv_i32 t2 = tcg_temp_new_i32();
6587
6588 read_vec_element_i32(s, t2, a->rm, a->idx, MO_32);
6589 if (neg) {
6590 gen_vfp_maybe_ah_negs(s, t1, t1);
6591 }
6592 gen_helper_vfp_muladds(t0, t1, t2, t0, fpstatus_ptr(FPST_A64));
6593 write_fp_sreg_merging(s, a->rd, a->rd, t0);
6594 }
6595 break;
6596 case MO_16:
6597 if (!dc_isar_feature(aa64_fp16, s)) {
6598 return false;
6599 }
6600 if (fp_access_check(s)) {
6601 TCGv_i32 t0 = read_fp_hreg(s, a->rd);
6602 TCGv_i32 t1 = read_fp_hreg(s, a->rn);
6603 TCGv_i32 t2 = tcg_temp_new_i32();
6604
6605 read_vec_element_i32(s, t2, a->rm, a->idx, MO_16);
6606 if (neg) {
6607 gen_vfp_maybe_ah_negh(s, t1, t1);
6608 }
6609 gen_helper_advsimd_muladdh(t0, t1, t2, t0,
6610 fpstatus_ptr(FPST_A64_F16));
6611 write_fp_hreg_merging(s, a->rd, a->rd, t0);
6612 }
6613 break;
6614 default:
6615 g_assert_not_reached();
6616 }
6617 return true;
6618 }
6619
TRANS(FMLA_si,do_fmla_scalar_idx,a,false)6620 TRANS(FMLA_si, do_fmla_scalar_idx, a, false)
6621 TRANS(FMLS_si, do_fmla_scalar_idx, a, true)
6622
6623 static bool do_env_scalar2_idx_hs(DisasContext *s, arg_rrx_e *a,
6624 const ENVScalar2 *f)
6625 {
6626 if (a->esz < MO_16 || a->esz > MO_32) {
6627 return false;
6628 }
6629 if (fp_access_check(s)) {
6630 TCGv_i32 t0 = tcg_temp_new_i32();
6631 TCGv_i32 t1 = tcg_temp_new_i32();
6632
6633 read_vec_element_i32(s, t0, a->rn, 0, a->esz);
6634 read_vec_element_i32(s, t1, a->rm, a->idx, a->esz);
6635 f->gen_bhs[a->esz](t0, tcg_env, t0, t1);
6636 write_fp_sreg(s, a->rd, t0);
6637 }
6638 return true;
6639 }
6640
6641 TRANS(SQDMULH_si, do_env_scalar2_idx_hs, a, &f_scalar_sqdmulh)
6642 TRANS(SQRDMULH_si, do_env_scalar2_idx_hs, a, &f_scalar_sqrdmulh)
6643
do_env_scalar3_idx_hs(DisasContext * s,arg_rrx_e * a,const ENVScalar3 * f)6644 static bool do_env_scalar3_idx_hs(DisasContext *s, arg_rrx_e *a,
6645 const ENVScalar3 *f)
6646 {
6647 if (a->esz < MO_16 || a->esz > MO_32) {
6648 return false;
6649 }
6650 if (fp_access_check(s)) {
6651 TCGv_i32 t0 = tcg_temp_new_i32();
6652 TCGv_i32 t1 = tcg_temp_new_i32();
6653 TCGv_i32 t2 = tcg_temp_new_i32();
6654
6655 read_vec_element_i32(s, t0, a->rn, 0, a->esz);
6656 read_vec_element_i32(s, t1, a->rm, a->idx, a->esz);
6657 read_vec_element_i32(s, t2, a->rd, 0, a->esz);
6658 f->gen_hs[a->esz - 1](t0, tcg_env, t0, t1, t2);
6659 write_fp_sreg(s, a->rd, t0);
6660 }
6661 return true;
6662 }
6663
6664 TRANS_FEAT(SQRDMLAH_si, aa64_rdm, do_env_scalar3_idx_hs, a, &f_scalar_sqrdmlah)
6665 TRANS_FEAT(SQRDMLSH_si, aa64_rdm, do_env_scalar3_idx_hs, a, &f_scalar_sqrdmlsh)
6666
do_scalar_muladd_widening_idx(DisasContext * s,arg_rrx_e * a,NeonGenTwo64OpFn * fn,bool acc)6667 static bool do_scalar_muladd_widening_idx(DisasContext *s, arg_rrx_e *a,
6668 NeonGenTwo64OpFn *fn, bool acc)
6669 {
6670 if (fp_access_check(s)) {
6671 TCGv_i64 t0 = tcg_temp_new_i64();
6672 TCGv_i64 t1 = tcg_temp_new_i64();
6673 TCGv_i64 t2 = tcg_temp_new_i64();
6674
6675 if (acc) {
6676 read_vec_element(s, t0, a->rd, 0, a->esz + 1);
6677 }
6678 read_vec_element(s, t1, a->rn, 0, a->esz | MO_SIGN);
6679 read_vec_element(s, t2, a->rm, a->idx, a->esz | MO_SIGN);
6680 fn(t0, t1, t2);
6681
6682 /* Clear the whole register first, then store scalar. */
6683 clear_vec(s, a->rd);
6684 write_vec_element(s, t0, a->rd, 0, a->esz + 1);
6685 }
6686 return true;
6687 }
6688
6689 TRANS(SQDMULL_si, do_scalar_muladd_widening_idx, a,
6690 a->esz == MO_16 ? gen_sqdmull_h : gen_sqdmull_s, false)
6691 TRANS(SQDMLAL_si, do_scalar_muladd_widening_idx, a,
6692 a->esz == MO_16 ? gen_sqdmlal_h : gen_sqdmlal_s, true)
6693 TRANS(SQDMLSL_si, do_scalar_muladd_widening_idx, a,
6694 a->esz == MO_16 ? gen_sqdmlsl_h : gen_sqdmlsl_s, true)
6695
do_fp3_vector_idx(DisasContext * s,arg_qrrx_e * a,gen_helper_gvec_3_ptr * const fns[3])6696 static bool do_fp3_vector_idx(DisasContext *s, arg_qrrx_e *a,
6697 gen_helper_gvec_3_ptr * const fns[3])
6698 {
6699 MemOp esz = a->esz;
6700 int check = fp_access_check_vector_hsd(s, a->q, esz);
6701
6702 if (check <= 0) {
6703 return check == 0;
6704 }
6705
6706 gen_gvec_op3_fpst(s, a->q, a->rd, a->rn, a->rm,
6707 esz == MO_16 ? FPST_A64_F16 : FPST_A64,
6708 a->idx, fns[esz - 1]);
6709 return true;
6710 }
6711
6712 static gen_helper_gvec_3_ptr * const f_vector_idx_fmul[3] = {
6713 gen_helper_gvec_fmul_idx_h,
6714 gen_helper_gvec_fmul_idx_s,
6715 gen_helper_gvec_fmul_idx_d,
6716 };
6717 TRANS(FMUL_vi, do_fp3_vector_idx, a, f_vector_idx_fmul)
6718
6719 static gen_helper_gvec_3_ptr * const f_vector_idx_fmulx[3] = {
6720 gen_helper_gvec_fmulx_idx_h,
6721 gen_helper_gvec_fmulx_idx_s,
6722 gen_helper_gvec_fmulx_idx_d,
6723 };
TRANS(FMULX_vi,do_fp3_vector_idx,a,f_vector_idx_fmulx)6724 TRANS(FMULX_vi, do_fp3_vector_idx, a, f_vector_idx_fmulx)
6725
6726 static bool do_fmla_vector_idx(DisasContext *s, arg_qrrx_e *a, bool neg)
6727 {
6728 static gen_helper_gvec_4_ptr * const fns[3][3] = {
6729 { gen_helper_gvec_fmla_idx_h,
6730 gen_helper_gvec_fmla_idx_s,
6731 gen_helper_gvec_fmla_idx_d },
6732 { gen_helper_gvec_fmls_idx_h,
6733 gen_helper_gvec_fmls_idx_s,
6734 gen_helper_gvec_fmls_idx_d },
6735 { gen_helper_gvec_ah_fmls_idx_h,
6736 gen_helper_gvec_ah_fmls_idx_s,
6737 gen_helper_gvec_ah_fmls_idx_d },
6738 };
6739 MemOp esz = a->esz;
6740 int check = fp_access_check_vector_hsd(s, a->q, esz);
6741
6742 if (check <= 0) {
6743 return check == 0;
6744 }
6745
6746 gen_gvec_op4_fpst(s, a->q, a->rd, a->rn, a->rm, a->rd,
6747 esz == MO_16 ? FPST_A64_F16 : FPST_A64,
6748 a->idx, fns[neg ? 1 + s->fpcr_ah : 0][esz - 1]);
6749 return true;
6750 }
6751
TRANS(FMLA_vi,do_fmla_vector_idx,a,false)6752 TRANS(FMLA_vi, do_fmla_vector_idx, a, false)
6753 TRANS(FMLS_vi, do_fmla_vector_idx, a, true)
6754
6755 static bool do_fmlal_idx(DisasContext *s, arg_qrrx_e *a, bool is_s, bool is_2)
6756 {
6757 if (fp_access_check(s)) {
6758 int data = (a->idx << 2) | (is_2 << 1) | is_s;
6759 tcg_gen_gvec_3_ptr(vec_full_reg_offset(s, a->rd),
6760 vec_full_reg_offset(s, a->rn),
6761 vec_full_reg_offset(s, a->rm), tcg_env,
6762 a->q ? 16 : 8, vec_full_reg_size(s),
6763 data, gen_helper_gvec_fmlal_idx_a64);
6764 }
6765 return true;
6766 }
6767
TRANS_FEAT(FMLAL_vi,aa64_fhm,do_fmlal_idx,a,false,false)6768 TRANS_FEAT(FMLAL_vi, aa64_fhm, do_fmlal_idx, a, false, false)
6769 TRANS_FEAT(FMLSL_vi, aa64_fhm, do_fmlal_idx, a, true, false)
6770 TRANS_FEAT(FMLAL2_vi, aa64_fhm, do_fmlal_idx, a, false, true)
6771 TRANS_FEAT(FMLSL2_vi, aa64_fhm, do_fmlal_idx, a, true, true)
6772
6773 static bool do_int3_vector_idx(DisasContext *s, arg_qrrx_e *a,
6774 gen_helper_gvec_3 * const fns[2])
6775 {
6776 assert(a->esz == MO_16 || a->esz == MO_32);
6777 if (fp_access_check(s)) {
6778 gen_gvec_op3_ool(s, a->q, a->rd, a->rn, a->rm, a->idx, fns[a->esz - 1]);
6779 }
6780 return true;
6781 }
6782
6783 static gen_helper_gvec_3 * const f_vector_idx_mul[2] = {
6784 gen_helper_gvec_mul_idx_h,
6785 gen_helper_gvec_mul_idx_s,
6786 };
TRANS(MUL_vi,do_int3_vector_idx,a,f_vector_idx_mul)6787 TRANS(MUL_vi, do_int3_vector_idx, a, f_vector_idx_mul)
6788
6789 static bool do_mla_vector_idx(DisasContext *s, arg_qrrx_e *a, bool sub)
6790 {
6791 static gen_helper_gvec_4 * const fns[2][2] = {
6792 { gen_helper_gvec_mla_idx_h, gen_helper_gvec_mls_idx_h },
6793 { gen_helper_gvec_mla_idx_s, gen_helper_gvec_mls_idx_s },
6794 };
6795
6796 assert(a->esz == MO_16 || a->esz == MO_32);
6797 if (fp_access_check(s)) {
6798 gen_gvec_op4_ool(s, a->q, a->rd, a->rn, a->rm, a->rd,
6799 a->idx, fns[a->esz - 1][sub]);
6800 }
6801 return true;
6802 }
6803
TRANS(MLA_vi,do_mla_vector_idx,a,false)6804 TRANS(MLA_vi, do_mla_vector_idx, a, false)
6805 TRANS(MLS_vi, do_mla_vector_idx, a, true)
6806
6807 static bool do_int3_qc_vector_idx(DisasContext *s, arg_qrrx_e *a,
6808 gen_helper_gvec_4 * const fns[2])
6809 {
6810 assert(a->esz == MO_16 || a->esz == MO_32);
6811 if (fp_access_check(s)) {
6812 tcg_gen_gvec_4_ool(vec_full_reg_offset(s, a->rd),
6813 vec_full_reg_offset(s, a->rn),
6814 vec_full_reg_offset(s, a->rm),
6815 offsetof(CPUARMState, vfp.qc),
6816 a->q ? 16 : 8, vec_full_reg_size(s),
6817 a->idx, fns[a->esz - 1]);
6818 }
6819 return true;
6820 }
6821
6822 static gen_helper_gvec_4 * const f_vector_idx_sqdmulh[2] = {
6823 gen_helper_neon_sqdmulh_idx_h,
6824 gen_helper_neon_sqdmulh_idx_s,
6825 };
6826 TRANS(SQDMULH_vi, do_int3_qc_vector_idx, a, f_vector_idx_sqdmulh)
6827
6828 static gen_helper_gvec_4 * const f_vector_idx_sqrdmulh[2] = {
6829 gen_helper_neon_sqrdmulh_idx_h,
6830 gen_helper_neon_sqrdmulh_idx_s,
6831 };
6832 TRANS(SQRDMULH_vi, do_int3_qc_vector_idx, a, f_vector_idx_sqrdmulh)
6833
6834 static gen_helper_gvec_4 * const f_vector_idx_sqrdmlah[2] = {
6835 gen_helper_neon_sqrdmlah_idx_h,
6836 gen_helper_neon_sqrdmlah_idx_s,
6837 };
6838 TRANS_FEAT(SQRDMLAH_vi, aa64_rdm, do_int3_qc_vector_idx, a,
6839 f_vector_idx_sqrdmlah)
6840
6841 static gen_helper_gvec_4 * const f_vector_idx_sqrdmlsh[2] = {
6842 gen_helper_neon_sqrdmlsh_idx_h,
6843 gen_helper_neon_sqrdmlsh_idx_s,
6844 };
TRANS_FEAT(SQRDMLSH_vi,aa64_rdm,do_int3_qc_vector_idx,a,f_vector_idx_sqrdmlsh)6845 TRANS_FEAT(SQRDMLSH_vi, aa64_rdm, do_int3_qc_vector_idx, a,
6846 f_vector_idx_sqrdmlsh)
6847
6848 static bool do_dot_vector_idx(DisasContext *s, arg_qrrx_e *a,
6849 gen_helper_gvec_4 *fn)
6850 {
6851 if (fp_access_check(s)) {
6852 gen_gvec_op4_ool(s, a->q, a->rd, a->rn, a->rm, a->rd, a->idx, fn);
6853 }
6854 return true;
6855 }
6856
do_dot_vector_idx_env(DisasContext * s,arg_qrrx_e * a,gen_helper_gvec_4_ptr * fn)6857 static bool do_dot_vector_idx_env(DisasContext *s, arg_qrrx_e *a,
6858 gen_helper_gvec_4_ptr *fn)
6859 {
6860 if (fp_access_check(s)) {
6861 gen_gvec_op4_env(s, a->q, a->rd, a->rn, a->rm, a->rd, a->idx, fn);
6862 }
6863 return true;
6864 }
6865
TRANS_FEAT(SDOT_vi,aa64_dp,do_dot_vector_idx,a,gen_helper_gvec_sdot_idx_b)6866 TRANS_FEAT(SDOT_vi, aa64_dp, do_dot_vector_idx, a, gen_helper_gvec_sdot_idx_b)
6867 TRANS_FEAT(UDOT_vi, aa64_dp, do_dot_vector_idx, a, gen_helper_gvec_udot_idx_b)
6868 TRANS_FEAT(SUDOT_vi, aa64_i8mm, do_dot_vector_idx, a,
6869 gen_helper_gvec_sudot_idx_b)
6870 TRANS_FEAT(USDOT_vi, aa64_i8mm, do_dot_vector_idx, a,
6871 gen_helper_gvec_usdot_idx_b)
6872 TRANS_FEAT(BFDOT_vi, aa64_bf16, do_dot_vector_idx_env, a,
6873 gen_helper_gvec_bfdot_idx)
6874
6875 static bool trans_BFMLAL_vi(DisasContext *s, arg_qrrx_e *a)
6876 {
6877 if (!dc_isar_feature(aa64_bf16, s)) {
6878 return false;
6879 }
6880 if (fp_access_check(s)) {
6881 /* Q bit selects BFMLALB vs BFMLALT. */
6882 gen_gvec_op4_fpst(s, true, a->rd, a->rn, a->rm, a->rd,
6883 s->fpcr_ah ? FPST_AH : FPST_A64,
6884 (a->idx << 1) | a->q,
6885 gen_helper_gvec_bfmlal_idx);
6886 }
6887 return true;
6888 }
6889
trans_FCMLA_vi(DisasContext * s,arg_FCMLA_vi * a)6890 static bool trans_FCMLA_vi(DisasContext *s, arg_FCMLA_vi *a)
6891 {
6892 gen_helper_gvec_4_ptr *fn;
6893
6894 if (!dc_isar_feature(aa64_fcma, s)) {
6895 return false;
6896 }
6897 switch (a->esz) {
6898 case MO_16:
6899 if (!dc_isar_feature(aa64_fp16, s)) {
6900 return false;
6901 }
6902 fn = gen_helper_gvec_fcmlah_idx;
6903 break;
6904 case MO_32:
6905 fn = gen_helper_gvec_fcmlas_idx;
6906 break;
6907 default:
6908 g_assert_not_reached();
6909 }
6910 if (fp_access_check(s)) {
6911 gen_gvec_op4_fpst(s, a->q, a->rd, a->rn, a->rm, a->rd,
6912 a->esz == MO_16 ? FPST_A64_F16 : FPST_A64,
6913 (s->fpcr_ah << 4) | (a->idx << 2) | a->rot, fn);
6914 }
6915 return true;
6916 }
6917
6918 /*
6919 * Advanced SIMD scalar pairwise
6920 */
6921
do_fp3_scalar_pair(DisasContext * s,arg_rr_e * a,const FPScalar * f)6922 static bool do_fp3_scalar_pair(DisasContext *s, arg_rr_e *a, const FPScalar *f)
6923 {
6924 switch (a->esz) {
6925 case MO_64:
6926 if (fp_access_check(s)) {
6927 TCGv_i64 t0 = tcg_temp_new_i64();
6928 TCGv_i64 t1 = tcg_temp_new_i64();
6929
6930 read_vec_element(s, t0, a->rn, 0, MO_64);
6931 read_vec_element(s, t1, a->rn, 1, MO_64);
6932 f->gen_d(t0, t0, t1, fpstatus_ptr(FPST_A64));
6933 write_fp_dreg(s, a->rd, t0);
6934 }
6935 break;
6936 case MO_32:
6937 if (fp_access_check(s)) {
6938 TCGv_i32 t0 = tcg_temp_new_i32();
6939 TCGv_i32 t1 = tcg_temp_new_i32();
6940
6941 read_vec_element_i32(s, t0, a->rn, 0, MO_32);
6942 read_vec_element_i32(s, t1, a->rn, 1, MO_32);
6943 f->gen_s(t0, t0, t1, fpstatus_ptr(FPST_A64));
6944 write_fp_sreg(s, a->rd, t0);
6945 }
6946 break;
6947 case MO_16:
6948 if (!dc_isar_feature(aa64_fp16, s)) {
6949 return false;
6950 }
6951 if (fp_access_check(s)) {
6952 TCGv_i32 t0 = tcg_temp_new_i32();
6953 TCGv_i32 t1 = tcg_temp_new_i32();
6954
6955 read_vec_element_i32(s, t0, a->rn, 0, MO_16);
6956 read_vec_element_i32(s, t1, a->rn, 1, MO_16);
6957 f->gen_h(t0, t0, t1, fpstatus_ptr(FPST_A64_F16));
6958 write_fp_sreg(s, a->rd, t0);
6959 }
6960 break;
6961 default:
6962 g_assert_not_reached();
6963 }
6964 return true;
6965 }
6966
do_fp3_scalar_pair_2fn(DisasContext * s,arg_rr_e * a,const FPScalar * fnormal,const FPScalar * fah)6967 static bool do_fp3_scalar_pair_2fn(DisasContext *s, arg_rr_e *a,
6968 const FPScalar *fnormal,
6969 const FPScalar *fah)
6970 {
6971 return do_fp3_scalar_pair(s, a, s->fpcr_ah ? fah : fnormal);
6972 }
6973
6974 TRANS(FADDP_s, do_fp3_scalar_pair, a, &f_scalar_fadd)
6975 TRANS(FMAXP_s, do_fp3_scalar_pair_2fn, a, &f_scalar_fmax, &f_scalar_fmax_ah)
6976 TRANS(FMINP_s, do_fp3_scalar_pair_2fn, a, &f_scalar_fmin, &f_scalar_fmin_ah)
6977 TRANS(FMAXNMP_s, do_fp3_scalar_pair, a, &f_scalar_fmaxnm)
6978 TRANS(FMINNMP_s, do_fp3_scalar_pair, a, &f_scalar_fminnm)
6979
trans_ADDP_s(DisasContext * s,arg_rr_e * a)6980 static bool trans_ADDP_s(DisasContext *s, arg_rr_e *a)
6981 {
6982 if (fp_access_check(s)) {
6983 TCGv_i64 t0 = tcg_temp_new_i64();
6984 TCGv_i64 t1 = tcg_temp_new_i64();
6985
6986 read_vec_element(s, t0, a->rn, 0, MO_64);
6987 read_vec_element(s, t1, a->rn, 1, MO_64);
6988 tcg_gen_add_i64(t0, t0, t1);
6989 write_fp_dreg(s, a->rd, t0);
6990 }
6991 return true;
6992 }
6993
6994 /*
6995 * Floating-point conditional select
6996 */
6997
trans_FCSEL(DisasContext * s,arg_FCSEL * a)6998 static bool trans_FCSEL(DisasContext *s, arg_FCSEL *a)
6999 {
7000 TCGv_i64 t_true, t_false;
7001 DisasCompare64 c;
7002 int check = fp_access_check_scalar_hsd(s, a->esz);
7003
7004 if (check <= 0) {
7005 return check == 0;
7006 }
7007
7008 /* Zero extend sreg & hreg inputs to 64 bits now. */
7009 t_true = tcg_temp_new_i64();
7010 t_false = tcg_temp_new_i64();
7011 read_vec_element(s, t_true, a->rn, 0, a->esz);
7012 read_vec_element(s, t_false, a->rm, 0, a->esz);
7013
7014 a64_test_cc(&c, a->cond);
7015 tcg_gen_movcond_i64(c.cond, t_true, c.value, tcg_constant_i64(0),
7016 t_true, t_false);
7017
7018 /*
7019 * Note that sregs & hregs write back zeros to the high bits,
7020 * and we've already done the zero-extension.
7021 */
7022 write_fp_dreg(s, a->rd, t_true);
7023 return true;
7024 }
7025
7026 /*
7027 * Advanced SIMD Extract
7028 */
7029
trans_EXT_d(DisasContext * s,arg_EXT_d * a)7030 static bool trans_EXT_d(DisasContext *s, arg_EXT_d *a)
7031 {
7032 if (fp_access_check(s)) {
7033 TCGv_i64 lo = read_fp_dreg(s, a->rn);
7034 if (a->imm != 0) {
7035 TCGv_i64 hi = read_fp_dreg(s, a->rm);
7036 tcg_gen_extract2_i64(lo, lo, hi, a->imm * 8);
7037 }
7038 write_fp_dreg(s, a->rd, lo);
7039 }
7040 return true;
7041 }
7042
trans_EXT_q(DisasContext * s,arg_EXT_q * a)7043 static bool trans_EXT_q(DisasContext *s, arg_EXT_q *a)
7044 {
7045 TCGv_i64 lo, hi;
7046 int pos = (a->imm & 7) * 8;
7047 int elt = a->imm >> 3;
7048
7049 if (!fp_access_check(s)) {
7050 return true;
7051 }
7052
7053 lo = tcg_temp_new_i64();
7054 hi = tcg_temp_new_i64();
7055
7056 read_vec_element(s, lo, a->rn, elt, MO_64);
7057 elt++;
7058 read_vec_element(s, hi, elt & 2 ? a->rm : a->rn, elt & 1, MO_64);
7059 elt++;
7060
7061 if (pos != 0) {
7062 TCGv_i64 hh = tcg_temp_new_i64();
7063 tcg_gen_extract2_i64(lo, lo, hi, pos);
7064 read_vec_element(s, hh, a->rm, elt & 1, MO_64);
7065 tcg_gen_extract2_i64(hi, hi, hh, pos);
7066 }
7067
7068 write_vec_element(s, lo, a->rd, 0, MO_64);
7069 write_vec_element(s, hi, a->rd, 1, MO_64);
7070 clear_vec_high(s, true, a->rd);
7071 return true;
7072 }
7073
7074 /*
7075 * Floating-point data-processing (3 source)
7076 */
7077
do_fmadd(DisasContext * s,arg_rrrr_e * a,bool neg_a,bool neg_n)7078 static bool do_fmadd(DisasContext *s, arg_rrrr_e *a, bool neg_a, bool neg_n)
7079 {
7080 TCGv_ptr fpst;
7081
7082 /*
7083 * These are fused multiply-add. Note that doing the negations here
7084 * as separate steps is correct: an input NaN should come out with
7085 * its sign bit flipped if it is a negated-input.
7086 */
7087 switch (a->esz) {
7088 case MO_64:
7089 if (fp_access_check(s)) {
7090 TCGv_i64 tn = read_fp_dreg(s, a->rn);
7091 TCGv_i64 tm = read_fp_dreg(s, a->rm);
7092 TCGv_i64 ta = read_fp_dreg(s, a->ra);
7093
7094 if (neg_a) {
7095 gen_vfp_maybe_ah_negd(s, ta, ta);
7096 }
7097 if (neg_n) {
7098 gen_vfp_maybe_ah_negd(s, tn, tn);
7099 }
7100 fpst = fpstatus_ptr(FPST_A64);
7101 gen_helper_vfp_muladdd(ta, tn, tm, ta, fpst);
7102 write_fp_dreg_merging(s, a->rd, a->ra, ta);
7103 }
7104 break;
7105
7106 case MO_32:
7107 if (fp_access_check(s)) {
7108 TCGv_i32 tn = read_fp_sreg(s, a->rn);
7109 TCGv_i32 tm = read_fp_sreg(s, a->rm);
7110 TCGv_i32 ta = read_fp_sreg(s, a->ra);
7111
7112 if (neg_a) {
7113 gen_vfp_maybe_ah_negs(s, ta, ta);
7114 }
7115 if (neg_n) {
7116 gen_vfp_maybe_ah_negs(s, tn, tn);
7117 }
7118 fpst = fpstatus_ptr(FPST_A64);
7119 gen_helper_vfp_muladds(ta, tn, tm, ta, fpst);
7120 write_fp_sreg_merging(s, a->rd, a->ra, ta);
7121 }
7122 break;
7123
7124 case MO_16:
7125 if (!dc_isar_feature(aa64_fp16, s)) {
7126 return false;
7127 }
7128 if (fp_access_check(s)) {
7129 TCGv_i32 tn = read_fp_hreg(s, a->rn);
7130 TCGv_i32 tm = read_fp_hreg(s, a->rm);
7131 TCGv_i32 ta = read_fp_hreg(s, a->ra);
7132
7133 if (neg_a) {
7134 gen_vfp_maybe_ah_negh(s, ta, ta);
7135 }
7136 if (neg_n) {
7137 gen_vfp_maybe_ah_negh(s, tn, tn);
7138 }
7139 fpst = fpstatus_ptr(FPST_A64_F16);
7140 gen_helper_advsimd_muladdh(ta, tn, tm, ta, fpst);
7141 write_fp_hreg_merging(s, a->rd, a->ra, ta);
7142 }
7143 break;
7144
7145 default:
7146 return false;
7147 }
7148 return true;
7149 }
7150
TRANS(FMADD,do_fmadd,a,false,false)7151 TRANS(FMADD, do_fmadd, a, false, false)
7152 TRANS(FNMADD, do_fmadd, a, true, true)
7153 TRANS(FMSUB, do_fmadd, a, false, true)
7154 TRANS(FNMSUB, do_fmadd, a, true, false)
7155
7156 /*
7157 * Advanced SIMD Across Lanes
7158 */
7159
7160 static bool do_int_reduction(DisasContext *s, arg_qrr_e *a, bool widen,
7161 MemOp src_sign, NeonGenTwo64OpFn *fn)
7162 {
7163 TCGv_i64 tcg_res, tcg_elt;
7164 MemOp src_mop = a->esz | src_sign;
7165 int elements = (a->q ? 16 : 8) >> a->esz;
7166
7167 /* Reject MO_64, and MO_32 without Q: a minimum of 4 elements. */
7168 if (elements < 4) {
7169 return false;
7170 }
7171 if (!fp_access_check(s)) {
7172 return true;
7173 }
7174
7175 tcg_res = tcg_temp_new_i64();
7176 tcg_elt = tcg_temp_new_i64();
7177
7178 read_vec_element(s, tcg_res, a->rn, 0, src_mop);
7179 for (int i = 1; i < elements; i++) {
7180 read_vec_element(s, tcg_elt, a->rn, i, src_mop);
7181 fn(tcg_res, tcg_res, tcg_elt);
7182 }
7183
7184 tcg_gen_ext_i64(tcg_res, tcg_res, a->esz + widen);
7185 write_fp_dreg(s, a->rd, tcg_res);
7186 return true;
7187 }
7188
7189 TRANS(ADDV, do_int_reduction, a, false, 0, tcg_gen_add_i64)
TRANS(SADDLV,do_int_reduction,a,true,MO_SIGN,tcg_gen_add_i64)7190 TRANS(SADDLV, do_int_reduction, a, true, MO_SIGN, tcg_gen_add_i64)
7191 TRANS(UADDLV, do_int_reduction, a, true, 0, tcg_gen_add_i64)
7192 TRANS(SMAXV, do_int_reduction, a, false, MO_SIGN, tcg_gen_smax_i64)
7193 TRANS(UMAXV, do_int_reduction, a, false, 0, tcg_gen_umax_i64)
7194 TRANS(SMINV, do_int_reduction, a, false, MO_SIGN, tcg_gen_smin_i64)
7195 TRANS(UMINV, do_int_reduction, a, false, 0, tcg_gen_umin_i64)
7196
7197 /*
7198 * do_fp_reduction helper
7199 *
7200 * This mirrors the Reduce() pseudocode in the ARM ARM. It is
7201 * important for correct NaN propagation that we do these
7202 * operations in exactly the order specified by the pseudocode.
7203 *
7204 * This is a recursive function.
7205 */
7206 static TCGv_i32 do_reduction_op(DisasContext *s, int rn, MemOp esz,
7207 int ebase, int ecount, TCGv_ptr fpst,
7208 NeonGenTwoSingleOpFn *fn)
7209 {
7210 if (ecount == 1) {
7211 TCGv_i32 tcg_elem = tcg_temp_new_i32();
7212 read_vec_element_i32(s, tcg_elem, rn, ebase, esz);
7213 return tcg_elem;
7214 } else {
7215 int half = ecount >> 1;
7216 TCGv_i32 tcg_hi, tcg_lo, tcg_res;
7217
7218 tcg_hi = do_reduction_op(s, rn, esz, ebase + half, half, fpst, fn);
7219 tcg_lo = do_reduction_op(s, rn, esz, ebase, half, fpst, fn);
7220 tcg_res = tcg_temp_new_i32();
7221
7222 fn(tcg_res, tcg_lo, tcg_hi, fpst);
7223 return tcg_res;
7224 }
7225 }
7226
do_fp_reduction(DisasContext * s,arg_qrr_e * a,NeonGenTwoSingleOpFn * fnormal,NeonGenTwoSingleOpFn * fah)7227 static bool do_fp_reduction(DisasContext *s, arg_qrr_e *a,
7228 NeonGenTwoSingleOpFn *fnormal,
7229 NeonGenTwoSingleOpFn *fah)
7230 {
7231 if (fp_access_check(s)) {
7232 MemOp esz = a->esz;
7233 int elts = (a->q ? 16 : 8) >> esz;
7234 TCGv_ptr fpst = fpstatus_ptr(esz == MO_16 ? FPST_A64_F16 : FPST_A64);
7235 TCGv_i32 res = do_reduction_op(s, a->rn, esz, 0, elts, fpst,
7236 s->fpcr_ah ? fah : fnormal);
7237 write_fp_sreg(s, a->rd, res);
7238 }
7239 return true;
7240 }
7241
TRANS_FEAT(FMAXNMV_h,aa64_fp16,do_fp_reduction,a,gen_helper_vfp_maxnumh,gen_helper_vfp_maxnumh)7242 TRANS_FEAT(FMAXNMV_h, aa64_fp16, do_fp_reduction, a,
7243 gen_helper_vfp_maxnumh, gen_helper_vfp_maxnumh)
7244 TRANS_FEAT(FMINNMV_h, aa64_fp16, do_fp_reduction, a,
7245 gen_helper_vfp_minnumh, gen_helper_vfp_minnumh)
7246 TRANS_FEAT(FMAXV_h, aa64_fp16, do_fp_reduction, a,
7247 gen_helper_vfp_maxh, gen_helper_vfp_ah_maxh)
7248 TRANS_FEAT(FMINV_h, aa64_fp16, do_fp_reduction, a,
7249 gen_helper_vfp_minh, gen_helper_vfp_ah_minh)
7250
7251 TRANS(FMAXNMV_s, do_fp_reduction, a,
7252 gen_helper_vfp_maxnums, gen_helper_vfp_maxnums)
7253 TRANS(FMINNMV_s, do_fp_reduction, a,
7254 gen_helper_vfp_minnums, gen_helper_vfp_minnums)
7255 TRANS(FMAXV_s, do_fp_reduction, a, gen_helper_vfp_maxs, gen_helper_vfp_ah_maxs)
7256 TRANS(FMINV_s, do_fp_reduction, a, gen_helper_vfp_mins, gen_helper_vfp_ah_mins)
7257
7258 /*
7259 * Floating-point Immediate
7260 */
7261
7262 static bool trans_FMOVI_s(DisasContext *s, arg_FMOVI_s *a)
7263 {
7264 int check = fp_access_check_scalar_hsd(s, a->esz);
7265 uint64_t imm;
7266
7267 if (check <= 0) {
7268 return check == 0;
7269 }
7270
7271 imm = vfp_expand_imm(a->esz, a->imm);
7272 write_fp_dreg(s, a->rd, tcg_constant_i64(imm));
7273 return true;
7274 }
7275
7276 /*
7277 * Floating point compare, conditional compare
7278 */
7279
handle_fp_compare(DisasContext * s,int size,unsigned int rn,unsigned int rm,bool cmp_with_zero,bool signal_all_nans)7280 static void handle_fp_compare(DisasContext *s, int size,
7281 unsigned int rn, unsigned int rm,
7282 bool cmp_with_zero, bool signal_all_nans)
7283 {
7284 TCGv_i64 tcg_flags = tcg_temp_new_i64();
7285 TCGv_ptr fpst = fpstatus_ptr(size == MO_16 ? FPST_A64_F16 : FPST_A64);
7286
7287 if (size == MO_64) {
7288 TCGv_i64 tcg_vn, tcg_vm;
7289
7290 tcg_vn = read_fp_dreg(s, rn);
7291 if (cmp_with_zero) {
7292 tcg_vm = tcg_constant_i64(0);
7293 } else {
7294 tcg_vm = read_fp_dreg(s, rm);
7295 }
7296 if (signal_all_nans) {
7297 gen_helper_vfp_cmped_a64(tcg_flags, tcg_vn, tcg_vm, fpst);
7298 } else {
7299 gen_helper_vfp_cmpd_a64(tcg_flags, tcg_vn, tcg_vm, fpst);
7300 }
7301 } else {
7302 TCGv_i32 tcg_vn = tcg_temp_new_i32();
7303 TCGv_i32 tcg_vm = tcg_temp_new_i32();
7304
7305 read_vec_element_i32(s, tcg_vn, rn, 0, size);
7306 if (cmp_with_zero) {
7307 tcg_gen_movi_i32(tcg_vm, 0);
7308 } else {
7309 read_vec_element_i32(s, tcg_vm, rm, 0, size);
7310 }
7311
7312 switch (size) {
7313 case MO_32:
7314 if (signal_all_nans) {
7315 gen_helper_vfp_cmpes_a64(tcg_flags, tcg_vn, tcg_vm, fpst);
7316 } else {
7317 gen_helper_vfp_cmps_a64(tcg_flags, tcg_vn, tcg_vm, fpst);
7318 }
7319 break;
7320 case MO_16:
7321 if (signal_all_nans) {
7322 gen_helper_vfp_cmpeh_a64(tcg_flags, tcg_vn, tcg_vm, fpst);
7323 } else {
7324 gen_helper_vfp_cmph_a64(tcg_flags, tcg_vn, tcg_vm, fpst);
7325 }
7326 break;
7327 default:
7328 g_assert_not_reached();
7329 }
7330 }
7331
7332 gen_set_nzcv(tcg_flags);
7333 }
7334
7335 /* FCMP, FCMPE */
trans_FCMP(DisasContext * s,arg_FCMP * a)7336 static bool trans_FCMP(DisasContext *s, arg_FCMP *a)
7337 {
7338 int check = fp_access_check_scalar_hsd(s, a->esz);
7339
7340 if (check <= 0) {
7341 return check == 0;
7342 }
7343
7344 handle_fp_compare(s, a->esz, a->rn, a->rm, a->z, a->e);
7345 return true;
7346 }
7347
7348 /* FCCMP, FCCMPE */
trans_FCCMP(DisasContext * s,arg_FCCMP * a)7349 static bool trans_FCCMP(DisasContext *s, arg_FCCMP *a)
7350 {
7351 TCGLabel *label_continue = NULL;
7352 int check = fp_access_check_scalar_hsd(s, a->esz);
7353
7354 if (check <= 0) {
7355 return check == 0;
7356 }
7357
7358 if (a->cond < 0x0e) { /* not always */
7359 TCGLabel *label_match = gen_new_label();
7360 label_continue = gen_new_label();
7361 arm_gen_test_cc(a->cond, label_match);
7362 /* nomatch: */
7363 gen_set_nzcv(tcg_constant_i64(a->nzcv << 28));
7364 tcg_gen_br(label_continue);
7365 gen_set_label(label_match);
7366 }
7367
7368 handle_fp_compare(s, a->esz, a->rn, a->rm, false, a->e);
7369
7370 if (label_continue) {
7371 gen_set_label(label_continue);
7372 }
7373 return true;
7374 }
7375
7376 /*
7377 * Advanced SIMD Modified Immediate
7378 */
7379
trans_FMOVI_v_h(DisasContext * s,arg_FMOVI_v_h * a)7380 static bool trans_FMOVI_v_h(DisasContext *s, arg_FMOVI_v_h *a)
7381 {
7382 if (!dc_isar_feature(aa64_fp16, s)) {
7383 return false;
7384 }
7385 if (fp_access_check(s)) {
7386 tcg_gen_gvec_dup_imm(MO_16, vec_full_reg_offset(s, a->rd),
7387 a->q ? 16 : 8, vec_full_reg_size(s),
7388 vfp_expand_imm(MO_16, a->abcdefgh));
7389 }
7390 return true;
7391 }
7392
gen_movi(unsigned vece,uint32_t dofs,uint32_t aofs,int64_t c,uint32_t oprsz,uint32_t maxsz)7393 static void gen_movi(unsigned vece, uint32_t dofs, uint32_t aofs,
7394 int64_t c, uint32_t oprsz, uint32_t maxsz)
7395 {
7396 tcg_gen_gvec_dup_imm(MO_64, dofs, oprsz, maxsz, c);
7397 }
7398
trans_Vimm(DisasContext * s,arg_Vimm * a)7399 static bool trans_Vimm(DisasContext *s, arg_Vimm *a)
7400 {
7401 GVecGen2iFn *fn;
7402
7403 /* Handle decode of cmode/op here between ORR/BIC/MOVI */
7404 if ((a->cmode & 1) && a->cmode < 12) {
7405 /* For op=1, the imm will be inverted, so BIC becomes AND. */
7406 fn = a->op ? tcg_gen_gvec_andi : tcg_gen_gvec_ori;
7407 } else {
7408 /* There is one unallocated cmode/op combination in this space */
7409 if (a->cmode == 15 && a->op == 1 && a->q == 0) {
7410 return false;
7411 }
7412 fn = gen_movi;
7413 }
7414
7415 if (fp_access_check(s)) {
7416 uint64_t imm = asimd_imm_const(a->abcdefgh, a->cmode, a->op);
7417 gen_gvec_fn2i(s, a->q, a->rd, a->rd, imm, fn, MO_64);
7418 }
7419 return true;
7420 }
7421
7422 /*
7423 * Advanced SIMD Shift by Immediate
7424 */
7425
do_vec_shift_imm(DisasContext * s,arg_qrri_e * a,GVecGen2iFn * fn)7426 static bool do_vec_shift_imm(DisasContext *s, arg_qrri_e *a, GVecGen2iFn *fn)
7427 {
7428 if (fp_access_check(s)) {
7429 gen_gvec_fn2i(s, a->q, a->rd, a->rn, a->imm, fn, a->esz);
7430 }
7431 return true;
7432 }
7433
7434 TRANS(SSHR_v, do_vec_shift_imm, a, gen_gvec_sshr)
7435 TRANS(USHR_v, do_vec_shift_imm, a, gen_gvec_ushr)
7436 TRANS(SSRA_v, do_vec_shift_imm, a, gen_gvec_ssra)
7437 TRANS(USRA_v, do_vec_shift_imm, a, gen_gvec_usra)
7438 TRANS(SRSHR_v, do_vec_shift_imm, a, gen_gvec_srshr)
7439 TRANS(URSHR_v, do_vec_shift_imm, a, gen_gvec_urshr)
7440 TRANS(SRSRA_v, do_vec_shift_imm, a, gen_gvec_srsra)
7441 TRANS(URSRA_v, do_vec_shift_imm, a, gen_gvec_ursra)
7442 TRANS(SRI_v, do_vec_shift_imm, a, gen_gvec_sri)
7443 TRANS(SHL_v, do_vec_shift_imm, a, tcg_gen_gvec_shli)
7444 TRANS(SLI_v, do_vec_shift_imm, a, gen_gvec_sli);
TRANS(SQSHL_vi,do_vec_shift_imm,a,gen_neon_sqshli)7445 TRANS(SQSHL_vi, do_vec_shift_imm, a, gen_neon_sqshli)
7446 TRANS(UQSHL_vi, do_vec_shift_imm, a, gen_neon_uqshli)
7447 TRANS(SQSHLU_vi, do_vec_shift_imm, a, gen_neon_sqshlui)
7448
7449 static bool do_vec_shift_imm_wide(DisasContext *s, arg_qrri_e *a, bool is_u)
7450 {
7451 TCGv_i64 tcg_rn, tcg_rd;
7452 int esz = a->esz;
7453 int esize;
7454
7455 if (!fp_access_check(s)) {
7456 return true;
7457 }
7458
7459 /*
7460 * For the LL variants the store is larger than the load,
7461 * so if rd == rn we would overwrite parts of our input.
7462 * So load everything right now and use shifts in the main loop.
7463 */
7464 tcg_rd = tcg_temp_new_i64();
7465 tcg_rn = tcg_temp_new_i64();
7466 read_vec_element(s, tcg_rn, a->rn, a->q, MO_64);
7467
7468 esize = 8 << esz;
7469 for (int i = 0, elements = 8 >> esz; i < elements; i++) {
7470 if (is_u) {
7471 tcg_gen_extract_i64(tcg_rd, tcg_rn, i * esize, esize);
7472 } else {
7473 tcg_gen_sextract_i64(tcg_rd, tcg_rn, i * esize, esize);
7474 }
7475 tcg_gen_shli_i64(tcg_rd, tcg_rd, a->imm);
7476 write_vec_element(s, tcg_rd, a->rd, i, esz + 1);
7477 }
7478 clear_vec_high(s, true, a->rd);
7479 return true;
7480 }
7481
TRANS(SSHLL_v,do_vec_shift_imm_wide,a,false)7482 TRANS(SSHLL_v, do_vec_shift_imm_wide, a, false)
7483 TRANS(USHLL_v, do_vec_shift_imm_wide, a, true)
7484
7485 static void gen_sshr_d(TCGv_i64 dst, TCGv_i64 src, int64_t shift)
7486 {
7487 assert(shift >= 0 && shift <= 64);
7488 tcg_gen_sari_i64(dst, src, MIN(shift, 63));
7489 }
7490
gen_ushr_d(TCGv_i64 dst,TCGv_i64 src,int64_t shift)7491 static void gen_ushr_d(TCGv_i64 dst, TCGv_i64 src, int64_t shift)
7492 {
7493 assert(shift >= 0 && shift <= 64);
7494 if (shift == 64) {
7495 tcg_gen_movi_i64(dst, 0);
7496 } else {
7497 tcg_gen_shri_i64(dst, src, shift);
7498 }
7499 }
7500
gen_ssra_d(TCGv_i64 dst,TCGv_i64 src,int64_t shift)7501 static void gen_ssra_d(TCGv_i64 dst, TCGv_i64 src, int64_t shift)
7502 {
7503 gen_sshr_d(src, src, shift);
7504 tcg_gen_add_i64(dst, dst, src);
7505 }
7506
gen_usra_d(TCGv_i64 dst,TCGv_i64 src,int64_t shift)7507 static void gen_usra_d(TCGv_i64 dst, TCGv_i64 src, int64_t shift)
7508 {
7509 gen_ushr_d(src, src, shift);
7510 tcg_gen_add_i64(dst, dst, src);
7511 }
7512
gen_srshr_bhs(TCGv_i64 dst,TCGv_i64 src,int64_t shift)7513 static void gen_srshr_bhs(TCGv_i64 dst, TCGv_i64 src, int64_t shift)
7514 {
7515 assert(shift >= 0 && shift <= 32);
7516 if (shift) {
7517 TCGv_i64 rnd = tcg_constant_i64(1ull << (shift - 1));
7518 tcg_gen_add_i64(dst, src, rnd);
7519 tcg_gen_sari_i64(dst, dst, shift);
7520 } else {
7521 tcg_gen_mov_i64(dst, src);
7522 }
7523 }
7524
gen_urshr_bhs(TCGv_i64 dst,TCGv_i64 src,int64_t shift)7525 static void gen_urshr_bhs(TCGv_i64 dst, TCGv_i64 src, int64_t shift)
7526 {
7527 assert(shift >= 0 && shift <= 32);
7528 if (shift) {
7529 TCGv_i64 rnd = tcg_constant_i64(1ull << (shift - 1));
7530 tcg_gen_add_i64(dst, src, rnd);
7531 tcg_gen_shri_i64(dst, dst, shift);
7532 } else {
7533 tcg_gen_mov_i64(dst, src);
7534 }
7535 }
7536
gen_srshr_d(TCGv_i64 dst,TCGv_i64 src,int64_t shift)7537 static void gen_srshr_d(TCGv_i64 dst, TCGv_i64 src, int64_t shift)
7538 {
7539 assert(shift >= 0 && shift <= 64);
7540 if (shift == 0) {
7541 tcg_gen_mov_i64(dst, src);
7542 } else if (shift == 64) {
7543 /* Extension of sign bit (0,-1) plus sign bit (0,1) is zero. */
7544 tcg_gen_movi_i64(dst, 0);
7545 } else {
7546 TCGv_i64 rnd = tcg_temp_new_i64();
7547 tcg_gen_extract_i64(rnd, src, shift - 1, 1);
7548 tcg_gen_sari_i64(dst, src, shift);
7549 tcg_gen_add_i64(dst, dst, rnd);
7550 }
7551 }
7552
gen_urshr_d(TCGv_i64 dst,TCGv_i64 src,int64_t shift)7553 static void gen_urshr_d(TCGv_i64 dst, TCGv_i64 src, int64_t shift)
7554 {
7555 assert(shift >= 0 && shift <= 64);
7556 if (shift == 0) {
7557 tcg_gen_mov_i64(dst, src);
7558 } else if (shift == 64) {
7559 /* Rounding will propagate bit 63 into bit 64. */
7560 tcg_gen_shri_i64(dst, src, 63);
7561 } else {
7562 TCGv_i64 rnd = tcg_temp_new_i64();
7563 tcg_gen_extract_i64(rnd, src, shift - 1, 1);
7564 tcg_gen_shri_i64(dst, src, shift);
7565 tcg_gen_add_i64(dst, dst, rnd);
7566 }
7567 }
7568
gen_srsra_d(TCGv_i64 dst,TCGv_i64 src,int64_t shift)7569 static void gen_srsra_d(TCGv_i64 dst, TCGv_i64 src, int64_t shift)
7570 {
7571 gen_srshr_d(src, src, shift);
7572 tcg_gen_add_i64(dst, dst, src);
7573 }
7574
gen_ursra_d(TCGv_i64 dst,TCGv_i64 src,int64_t shift)7575 static void gen_ursra_d(TCGv_i64 dst, TCGv_i64 src, int64_t shift)
7576 {
7577 gen_urshr_d(src, src, shift);
7578 tcg_gen_add_i64(dst, dst, src);
7579 }
7580
gen_sri_d(TCGv_i64 dst,TCGv_i64 src,int64_t shift)7581 static void gen_sri_d(TCGv_i64 dst, TCGv_i64 src, int64_t shift)
7582 {
7583 /* If shift is 64, dst is unchanged. */
7584 if (shift != 64) {
7585 tcg_gen_shri_i64(src, src, shift);
7586 tcg_gen_deposit_i64(dst, dst, src, 0, 64 - shift);
7587 }
7588 }
7589
gen_sli_d(TCGv_i64 dst,TCGv_i64 src,int64_t shift)7590 static void gen_sli_d(TCGv_i64 dst, TCGv_i64 src, int64_t shift)
7591 {
7592 tcg_gen_deposit_i64(dst, dst, src, shift, 64 - shift);
7593 }
7594
do_vec_shift_imm_narrow(DisasContext * s,arg_qrri_e * a,WideShiftImmFn * const fns[3],MemOp sign)7595 static bool do_vec_shift_imm_narrow(DisasContext *s, arg_qrri_e *a,
7596 WideShiftImmFn * const fns[3], MemOp sign)
7597 {
7598 TCGv_i64 tcg_rn, tcg_rd;
7599 int esz = a->esz;
7600 int esize;
7601 WideShiftImmFn *fn;
7602
7603 tcg_debug_assert(esz >= MO_8 && esz <= MO_32);
7604
7605 if (!fp_access_check(s)) {
7606 return true;
7607 }
7608
7609 tcg_rn = tcg_temp_new_i64();
7610 tcg_rd = tcg_temp_new_i64();
7611 tcg_gen_movi_i64(tcg_rd, 0);
7612
7613 fn = fns[esz];
7614 esize = 8 << esz;
7615 for (int i = 0, elements = 8 >> esz; i < elements; i++) {
7616 read_vec_element(s, tcg_rn, a->rn, i, (esz + 1) | sign);
7617 fn(tcg_rn, tcg_rn, a->imm);
7618 tcg_gen_deposit_i64(tcg_rd, tcg_rd, tcg_rn, esize * i, esize);
7619 }
7620
7621 write_vec_element(s, tcg_rd, a->rd, a->q, MO_64);
7622 clear_vec_high(s, a->q, a->rd);
7623 return true;
7624 }
7625
gen_sqshrn_b(TCGv_i64 d,TCGv_i64 s,int64_t i)7626 static void gen_sqshrn_b(TCGv_i64 d, TCGv_i64 s, int64_t i)
7627 {
7628 tcg_gen_sari_i64(d, s, i);
7629 tcg_gen_ext16u_i64(d, d);
7630 gen_helper_neon_narrow_sat_s8(d, tcg_env, d);
7631 }
7632
gen_sqshrn_h(TCGv_i64 d,TCGv_i64 s,int64_t i)7633 static void gen_sqshrn_h(TCGv_i64 d, TCGv_i64 s, int64_t i)
7634 {
7635 tcg_gen_sari_i64(d, s, i);
7636 tcg_gen_ext32u_i64(d, d);
7637 gen_helper_neon_narrow_sat_s16(d, tcg_env, d);
7638 }
7639
gen_sqshrn_s(TCGv_i64 d,TCGv_i64 s,int64_t i)7640 static void gen_sqshrn_s(TCGv_i64 d, TCGv_i64 s, int64_t i)
7641 {
7642 gen_sshr_d(d, s, i);
7643 gen_helper_neon_narrow_sat_s32(d, tcg_env, d);
7644 }
7645
gen_uqshrn_b(TCGv_i64 d,TCGv_i64 s,int64_t i)7646 static void gen_uqshrn_b(TCGv_i64 d, TCGv_i64 s, int64_t i)
7647 {
7648 tcg_gen_shri_i64(d, s, i);
7649 gen_helper_neon_narrow_sat_u8(d, tcg_env, d);
7650 }
7651
gen_uqshrn_h(TCGv_i64 d,TCGv_i64 s,int64_t i)7652 static void gen_uqshrn_h(TCGv_i64 d, TCGv_i64 s, int64_t i)
7653 {
7654 tcg_gen_shri_i64(d, s, i);
7655 gen_helper_neon_narrow_sat_u16(d, tcg_env, d);
7656 }
7657
gen_uqshrn_s(TCGv_i64 d,TCGv_i64 s,int64_t i)7658 static void gen_uqshrn_s(TCGv_i64 d, TCGv_i64 s, int64_t i)
7659 {
7660 gen_ushr_d(d, s, i);
7661 gen_helper_neon_narrow_sat_u32(d, tcg_env, d);
7662 }
7663
gen_sqshrun_b(TCGv_i64 d,TCGv_i64 s,int64_t i)7664 static void gen_sqshrun_b(TCGv_i64 d, TCGv_i64 s, int64_t i)
7665 {
7666 tcg_gen_sari_i64(d, s, i);
7667 tcg_gen_ext16u_i64(d, d);
7668 gen_helper_neon_unarrow_sat8(d, tcg_env, d);
7669 }
7670
gen_sqshrun_h(TCGv_i64 d,TCGv_i64 s,int64_t i)7671 static void gen_sqshrun_h(TCGv_i64 d, TCGv_i64 s, int64_t i)
7672 {
7673 tcg_gen_sari_i64(d, s, i);
7674 tcg_gen_ext32u_i64(d, d);
7675 gen_helper_neon_unarrow_sat16(d, tcg_env, d);
7676 }
7677
gen_sqshrun_s(TCGv_i64 d,TCGv_i64 s,int64_t i)7678 static void gen_sqshrun_s(TCGv_i64 d, TCGv_i64 s, int64_t i)
7679 {
7680 gen_sshr_d(d, s, i);
7681 gen_helper_neon_unarrow_sat32(d, tcg_env, d);
7682 }
7683
gen_sqrshrn_b(TCGv_i64 d,TCGv_i64 s,int64_t i)7684 static void gen_sqrshrn_b(TCGv_i64 d, TCGv_i64 s, int64_t i)
7685 {
7686 gen_srshr_bhs(d, s, i);
7687 tcg_gen_ext16u_i64(d, d);
7688 gen_helper_neon_narrow_sat_s8(d, tcg_env, d);
7689 }
7690
gen_sqrshrn_h(TCGv_i64 d,TCGv_i64 s,int64_t i)7691 static void gen_sqrshrn_h(TCGv_i64 d, TCGv_i64 s, int64_t i)
7692 {
7693 gen_srshr_bhs(d, s, i);
7694 tcg_gen_ext32u_i64(d, d);
7695 gen_helper_neon_narrow_sat_s16(d, tcg_env, d);
7696 }
7697
gen_sqrshrn_s(TCGv_i64 d,TCGv_i64 s,int64_t i)7698 static void gen_sqrshrn_s(TCGv_i64 d, TCGv_i64 s, int64_t i)
7699 {
7700 gen_srshr_d(d, s, i);
7701 gen_helper_neon_narrow_sat_s32(d, tcg_env, d);
7702 }
7703
gen_uqrshrn_b(TCGv_i64 d,TCGv_i64 s,int64_t i)7704 static void gen_uqrshrn_b(TCGv_i64 d, TCGv_i64 s, int64_t i)
7705 {
7706 gen_urshr_bhs(d, s, i);
7707 gen_helper_neon_narrow_sat_u8(d, tcg_env, d);
7708 }
7709
gen_uqrshrn_h(TCGv_i64 d,TCGv_i64 s,int64_t i)7710 static void gen_uqrshrn_h(TCGv_i64 d, TCGv_i64 s, int64_t i)
7711 {
7712 gen_urshr_bhs(d, s, i);
7713 gen_helper_neon_narrow_sat_u16(d, tcg_env, d);
7714 }
7715
gen_uqrshrn_s(TCGv_i64 d,TCGv_i64 s,int64_t i)7716 static void gen_uqrshrn_s(TCGv_i64 d, TCGv_i64 s, int64_t i)
7717 {
7718 gen_urshr_d(d, s, i);
7719 gen_helper_neon_narrow_sat_u32(d, tcg_env, d);
7720 }
7721
gen_sqrshrun_b(TCGv_i64 d,TCGv_i64 s,int64_t i)7722 static void gen_sqrshrun_b(TCGv_i64 d, TCGv_i64 s, int64_t i)
7723 {
7724 gen_srshr_bhs(d, s, i);
7725 tcg_gen_ext16u_i64(d, d);
7726 gen_helper_neon_unarrow_sat8(d, tcg_env, d);
7727 }
7728
gen_sqrshrun_h(TCGv_i64 d,TCGv_i64 s,int64_t i)7729 static void gen_sqrshrun_h(TCGv_i64 d, TCGv_i64 s, int64_t i)
7730 {
7731 gen_srshr_bhs(d, s, i);
7732 tcg_gen_ext32u_i64(d, d);
7733 gen_helper_neon_unarrow_sat16(d, tcg_env, d);
7734 }
7735
gen_sqrshrun_s(TCGv_i64 d,TCGv_i64 s,int64_t i)7736 static void gen_sqrshrun_s(TCGv_i64 d, TCGv_i64 s, int64_t i)
7737 {
7738 gen_srshr_d(d, s, i);
7739 gen_helper_neon_unarrow_sat32(d, tcg_env, d);
7740 }
7741
7742 static WideShiftImmFn * const shrn_fns[] = {
7743 tcg_gen_shri_i64,
7744 tcg_gen_shri_i64,
7745 gen_ushr_d,
7746 };
7747 TRANS(SHRN_v, do_vec_shift_imm_narrow, a, shrn_fns, 0)
7748
7749 static WideShiftImmFn * const rshrn_fns[] = {
7750 gen_urshr_bhs,
7751 gen_urshr_bhs,
7752 gen_urshr_d,
7753 };
7754 TRANS(RSHRN_v, do_vec_shift_imm_narrow, a, rshrn_fns, 0)
7755
7756 static WideShiftImmFn * const sqshrn_fns[] = {
7757 gen_sqshrn_b,
7758 gen_sqshrn_h,
7759 gen_sqshrn_s,
7760 };
7761 TRANS(SQSHRN_v, do_vec_shift_imm_narrow, a, sqshrn_fns, MO_SIGN)
7762
7763 static WideShiftImmFn * const uqshrn_fns[] = {
7764 gen_uqshrn_b,
7765 gen_uqshrn_h,
7766 gen_uqshrn_s,
7767 };
7768 TRANS(UQSHRN_v, do_vec_shift_imm_narrow, a, uqshrn_fns, 0)
7769
7770 static WideShiftImmFn * const sqshrun_fns[] = {
7771 gen_sqshrun_b,
7772 gen_sqshrun_h,
7773 gen_sqshrun_s,
7774 };
7775 TRANS(SQSHRUN_v, do_vec_shift_imm_narrow, a, sqshrun_fns, MO_SIGN)
7776
7777 static WideShiftImmFn * const sqrshrn_fns[] = {
7778 gen_sqrshrn_b,
7779 gen_sqrshrn_h,
7780 gen_sqrshrn_s,
7781 };
7782 TRANS(SQRSHRN_v, do_vec_shift_imm_narrow, a, sqrshrn_fns, MO_SIGN)
7783
7784 static WideShiftImmFn * const uqrshrn_fns[] = {
7785 gen_uqrshrn_b,
7786 gen_uqrshrn_h,
7787 gen_uqrshrn_s,
7788 };
7789 TRANS(UQRSHRN_v, do_vec_shift_imm_narrow, a, uqrshrn_fns, 0)
7790
7791 static WideShiftImmFn * const sqrshrun_fns[] = {
7792 gen_sqrshrun_b,
7793 gen_sqrshrun_h,
7794 gen_sqrshrun_s,
7795 };
TRANS(SQRSHRUN_v,do_vec_shift_imm_narrow,a,sqrshrun_fns,MO_SIGN)7796 TRANS(SQRSHRUN_v, do_vec_shift_imm_narrow, a, sqrshrun_fns, MO_SIGN)
7797
7798 /*
7799 * Advanced SIMD Scalar Shift by Immediate
7800 */
7801
7802 static bool do_scalar_shift_imm(DisasContext *s, arg_rri_e *a,
7803 WideShiftImmFn *fn, bool accumulate,
7804 MemOp sign)
7805 {
7806 if (fp_access_check(s)) {
7807 TCGv_i64 rd = tcg_temp_new_i64();
7808 TCGv_i64 rn = tcg_temp_new_i64();
7809
7810 read_vec_element(s, rn, a->rn, 0, a->esz | sign);
7811 if (accumulate) {
7812 read_vec_element(s, rd, a->rd, 0, a->esz | sign);
7813 }
7814 fn(rd, rn, a->imm);
7815 write_fp_dreg(s, a->rd, rd);
7816 }
7817 return true;
7818 }
7819
7820 TRANS(SSHR_s, do_scalar_shift_imm, a, gen_sshr_d, false, 0)
7821 TRANS(USHR_s, do_scalar_shift_imm, a, gen_ushr_d, false, 0)
7822 TRANS(SSRA_s, do_scalar_shift_imm, a, gen_ssra_d, true, 0)
7823 TRANS(USRA_s, do_scalar_shift_imm, a, gen_usra_d, true, 0)
7824 TRANS(SRSHR_s, do_scalar_shift_imm, a, gen_srshr_d, false, 0)
7825 TRANS(URSHR_s, do_scalar_shift_imm, a, gen_urshr_d, false, 0)
7826 TRANS(SRSRA_s, do_scalar_shift_imm, a, gen_srsra_d, true, 0)
7827 TRANS(URSRA_s, do_scalar_shift_imm, a, gen_ursra_d, true, 0)
7828 TRANS(SRI_s, do_scalar_shift_imm, a, gen_sri_d, true, 0)
7829
7830 TRANS(SHL_s, do_scalar_shift_imm, a, tcg_gen_shli_i64, false, 0)
7831 TRANS(SLI_s, do_scalar_shift_imm, a, gen_sli_d, true, 0)
7832
trunc_i64_env_imm(TCGv_i64 d,TCGv_i64 s,int64_t i,NeonGenTwoOpEnvFn * fn)7833 static void trunc_i64_env_imm(TCGv_i64 d, TCGv_i64 s, int64_t i,
7834 NeonGenTwoOpEnvFn *fn)
7835 {
7836 TCGv_i32 t = tcg_temp_new_i32();
7837 tcg_gen_extrl_i64_i32(t, s);
7838 fn(t, tcg_env, t, tcg_constant_i32(i));
7839 tcg_gen_extu_i32_i64(d, t);
7840 }
7841
gen_sqshli_b(TCGv_i64 d,TCGv_i64 s,int64_t i)7842 static void gen_sqshli_b(TCGv_i64 d, TCGv_i64 s, int64_t i)
7843 {
7844 trunc_i64_env_imm(d, s, i, gen_helper_neon_qshl_s8);
7845 }
7846
gen_sqshli_h(TCGv_i64 d,TCGv_i64 s,int64_t i)7847 static void gen_sqshli_h(TCGv_i64 d, TCGv_i64 s, int64_t i)
7848 {
7849 trunc_i64_env_imm(d, s, i, gen_helper_neon_qshl_s16);
7850 }
7851
gen_sqshli_s(TCGv_i64 d,TCGv_i64 s,int64_t i)7852 static void gen_sqshli_s(TCGv_i64 d, TCGv_i64 s, int64_t i)
7853 {
7854 trunc_i64_env_imm(d, s, i, gen_helper_neon_qshl_s32);
7855 }
7856
gen_sqshli_d(TCGv_i64 d,TCGv_i64 s,int64_t i)7857 static void gen_sqshli_d(TCGv_i64 d, TCGv_i64 s, int64_t i)
7858 {
7859 gen_helper_neon_qshl_s64(d, tcg_env, s, tcg_constant_i64(i));
7860 }
7861
gen_uqshli_b(TCGv_i64 d,TCGv_i64 s,int64_t i)7862 static void gen_uqshli_b(TCGv_i64 d, TCGv_i64 s, int64_t i)
7863 {
7864 trunc_i64_env_imm(d, s, i, gen_helper_neon_qshl_u8);
7865 }
7866
gen_uqshli_h(TCGv_i64 d,TCGv_i64 s,int64_t i)7867 static void gen_uqshli_h(TCGv_i64 d, TCGv_i64 s, int64_t i)
7868 {
7869 trunc_i64_env_imm(d, s, i, gen_helper_neon_qshl_u16);
7870 }
7871
gen_uqshli_s(TCGv_i64 d,TCGv_i64 s,int64_t i)7872 static void gen_uqshli_s(TCGv_i64 d, TCGv_i64 s, int64_t i)
7873 {
7874 trunc_i64_env_imm(d, s, i, gen_helper_neon_qshl_u32);
7875 }
7876
gen_uqshli_d(TCGv_i64 d,TCGv_i64 s,int64_t i)7877 static void gen_uqshli_d(TCGv_i64 d, TCGv_i64 s, int64_t i)
7878 {
7879 gen_helper_neon_qshl_u64(d, tcg_env, s, tcg_constant_i64(i));
7880 }
7881
gen_sqshlui_b(TCGv_i64 d,TCGv_i64 s,int64_t i)7882 static void gen_sqshlui_b(TCGv_i64 d, TCGv_i64 s, int64_t i)
7883 {
7884 trunc_i64_env_imm(d, s, i, gen_helper_neon_qshlu_s8);
7885 }
7886
gen_sqshlui_h(TCGv_i64 d,TCGv_i64 s,int64_t i)7887 static void gen_sqshlui_h(TCGv_i64 d, TCGv_i64 s, int64_t i)
7888 {
7889 trunc_i64_env_imm(d, s, i, gen_helper_neon_qshlu_s16);
7890 }
7891
gen_sqshlui_s(TCGv_i64 d,TCGv_i64 s,int64_t i)7892 static void gen_sqshlui_s(TCGv_i64 d, TCGv_i64 s, int64_t i)
7893 {
7894 trunc_i64_env_imm(d, s, i, gen_helper_neon_qshlu_s32);
7895 }
7896
gen_sqshlui_d(TCGv_i64 d,TCGv_i64 s,int64_t i)7897 static void gen_sqshlui_d(TCGv_i64 d, TCGv_i64 s, int64_t i)
7898 {
7899 gen_helper_neon_qshlu_s64(d, tcg_env, s, tcg_constant_i64(i));
7900 }
7901
7902 static WideShiftImmFn * const f_scalar_sqshli[] = {
7903 gen_sqshli_b, gen_sqshli_h, gen_sqshli_s, gen_sqshli_d
7904 };
7905
7906 static WideShiftImmFn * const f_scalar_uqshli[] = {
7907 gen_uqshli_b, gen_uqshli_h, gen_uqshli_s, gen_uqshli_d
7908 };
7909
7910 static WideShiftImmFn * const f_scalar_sqshlui[] = {
7911 gen_sqshlui_b, gen_sqshlui_h, gen_sqshlui_s, gen_sqshlui_d
7912 };
7913
7914 /* Note that the helpers sign-extend their inputs, so don't do it here. */
7915 TRANS(SQSHL_si, do_scalar_shift_imm, a, f_scalar_sqshli[a->esz], false, 0)
7916 TRANS(UQSHL_si, do_scalar_shift_imm, a, f_scalar_uqshli[a->esz], false, 0)
7917 TRANS(SQSHLU_si, do_scalar_shift_imm, a, f_scalar_sqshlui[a->esz], false, 0)
7918
do_scalar_shift_imm_narrow(DisasContext * s,arg_rri_e * a,WideShiftImmFn * const fns[3],MemOp sign,bool zext)7919 static bool do_scalar_shift_imm_narrow(DisasContext *s, arg_rri_e *a,
7920 WideShiftImmFn * const fns[3],
7921 MemOp sign, bool zext)
7922 {
7923 MemOp esz = a->esz;
7924
7925 tcg_debug_assert(esz >= MO_8 && esz <= MO_32);
7926
7927 if (fp_access_check(s)) {
7928 TCGv_i64 rd = tcg_temp_new_i64();
7929 TCGv_i64 rn = tcg_temp_new_i64();
7930
7931 read_vec_element(s, rn, a->rn, 0, (esz + 1) | sign);
7932 fns[esz](rd, rn, a->imm);
7933 if (zext) {
7934 tcg_gen_ext_i64(rd, rd, esz);
7935 }
7936 write_fp_dreg(s, a->rd, rd);
7937 }
7938 return true;
7939 }
7940
TRANS(SQSHRN_si,do_scalar_shift_imm_narrow,a,sqshrn_fns,MO_SIGN,true)7941 TRANS(SQSHRN_si, do_scalar_shift_imm_narrow, a, sqshrn_fns, MO_SIGN, true)
7942 TRANS(SQRSHRN_si, do_scalar_shift_imm_narrow, a, sqrshrn_fns, MO_SIGN, true)
7943 TRANS(UQSHRN_si, do_scalar_shift_imm_narrow, a, uqshrn_fns, 0, false)
7944 TRANS(UQRSHRN_si, do_scalar_shift_imm_narrow, a, uqrshrn_fns, 0, false)
7945 TRANS(SQSHRUN_si, do_scalar_shift_imm_narrow, a, sqshrun_fns, MO_SIGN, false)
7946 TRANS(SQRSHRUN_si, do_scalar_shift_imm_narrow, a, sqrshrun_fns, MO_SIGN, false)
7947
7948 static bool do_div(DisasContext *s, arg_rrr_sf *a, bool is_signed)
7949 {
7950 TCGv_i64 tcg_n, tcg_m, tcg_rd;
7951 tcg_rd = cpu_reg(s, a->rd);
7952
7953 if (!a->sf && is_signed) {
7954 tcg_n = tcg_temp_new_i64();
7955 tcg_m = tcg_temp_new_i64();
7956 tcg_gen_ext32s_i64(tcg_n, cpu_reg(s, a->rn));
7957 tcg_gen_ext32s_i64(tcg_m, cpu_reg(s, a->rm));
7958 } else {
7959 tcg_n = read_cpu_reg(s, a->rn, a->sf);
7960 tcg_m = read_cpu_reg(s, a->rm, a->sf);
7961 }
7962
7963 if (is_signed) {
7964 gen_helper_sdiv64(tcg_rd, tcg_n, tcg_m);
7965 } else {
7966 gen_helper_udiv64(tcg_rd, tcg_n, tcg_m);
7967 }
7968
7969 if (!a->sf) { /* zero extend final result */
7970 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
7971 }
7972 return true;
7973 }
7974
TRANS(SDIV,do_div,a,true)7975 TRANS(SDIV, do_div, a, true)
7976 TRANS(UDIV, do_div, a, false)
7977
7978 /* Shift a TCGv src by TCGv shift_amount, put result in dst.
7979 * Note that it is the caller's responsibility to ensure that the
7980 * shift amount is in range (ie 0..31 or 0..63) and provide the ARM
7981 * mandated semantics for out of range shifts.
7982 */
7983 static void shift_reg(TCGv_i64 dst, TCGv_i64 src, int sf,
7984 enum a64_shift_type shift_type, TCGv_i64 shift_amount)
7985 {
7986 switch (shift_type) {
7987 case A64_SHIFT_TYPE_LSL:
7988 tcg_gen_shl_i64(dst, src, shift_amount);
7989 break;
7990 case A64_SHIFT_TYPE_LSR:
7991 tcg_gen_shr_i64(dst, src, shift_amount);
7992 break;
7993 case A64_SHIFT_TYPE_ASR:
7994 if (!sf) {
7995 tcg_gen_ext32s_i64(dst, src);
7996 }
7997 tcg_gen_sar_i64(dst, sf ? src : dst, shift_amount);
7998 break;
7999 case A64_SHIFT_TYPE_ROR:
8000 if (sf) {
8001 tcg_gen_rotr_i64(dst, src, shift_amount);
8002 } else {
8003 TCGv_i32 t0, t1;
8004 t0 = tcg_temp_new_i32();
8005 t1 = tcg_temp_new_i32();
8006 tcg_gen_extrl_i64_i32(t0, src);
8007 tcg_gen_extrl_i64_i32(t1, shift_amount);
8008 tcg_gen_rotr_i32(t0, t0, t1);
8009 tcg_gen_extu_i32_i64(dst, t0);
8010 }
8011 break;
8012 default:
8013 assert(FALSE); /* all shift types should be handled */
8014 break;
8015 }
8016
8017 if (!sf) { /* zero extend final result */
8018 tcg_gen_ext32u_i64(dst, dst);
8019 }
8020 }
8021
8022 /* Shift a TCGv src by immediate, put result in dst.
8023 * The shift amount must be in range (this should always be true as the
8024 * relevant instructions will UNDEF on bad shift immediates).
8025 */
shift_reg_imm(TCGv_i64 dst,TCGv_i64 src,int sf,enum a64_shift_type shift_type,unsigned int shift_i)8026 static void shift_reg_imm(TCGv_i64 dst, TCGv_i64 src, int sf,
8027 enum a64_shift_type shift_type, unsigned int shift_i)
8028 {
8029 assert(shift_i < (sf ? 64 : 32));
8030
8031 if (shift_i == 0) {
8032 tcg_gen_mov_i64(dst, src);
8033 } else {
8034 shift_reg(dst, src, sf, shift_type, tcg_constant_i64(shift_i));
8035 }
8036 }
8037
do_shift_reg(DisasContext * s,arg_rrr_sf * a,enum a64_shift_type shift_type)8038 static bool do_shift_reg(DisasContext *s, arg_rrr_sf *a,
8039 enum a64_shift_type shift_type)
8040 {
8041 TCGv_i64 tcg_shift = tcg_temp_new_i64();
8042 TCGv_i64 tcg_rd = cpu_reg(s, a->rd);
8043 TCGv_i64 tcg_rn = read_cpu_reg(s, a->rn, a->sf);
8044
8045 tcg_gen_andi_i64(tcg_shift, cpu_reg(s, a->rm), a->sf ? 63 : 31);
8046 shift_reg(tcg_rd, tcg_rn, a->sf, shift_type, tcg_shift);
8047 return true;
8048 }
8049
TRANS(LSLV,do_shift_reg,a,A64_SHIFT_TYPE_LSL)8050 TRANS(LSLV, do_shift_reg, a, A64_SHIFT_TYPE_LSL)
8051 TRANS(LSRV, do_shift_reg, a, A64_SHIFT_TYPE_LSR)
8052 TRANS(ASRV, do_shift_reg, a, A64_SHIFT_TYPE_ASR)
8053 TRANS(RORV, do_shift_reg, a, A64_SHIFT_TYPE_ROR)
8054
8055 static bool do_crc32(DisasContext *s, arg_rrr_e *a, bool crc32c)
8056 {
8057 TCGv_i64 tcg_acc, tcg_val, tcg_rd;
8058 TCGv_i32 tcg_bytes;
8059
8060 switch (a->esz) {
8061 case MO_8:
8062 case MO_16:
8063 case MO_32:
8064 tcg_val = tcg_temp_new_i64();
8065 tcg_gen_extract_i64(tcg_val, cpu_reg(s, a->rm), 0, 8 << a->esz);
8066 break;
8067 case MO_64:
8068 tcg_val = cpu_reg(s, a->rm);
8069 break;
8070 default:
8071 g_assert_not_reached();
8072 }
8073 tcg_acc = cpu_reg(s, a->rn);
8074 tcg_bytes = tcg_constant_i32(1 << a->esz);
8075 tcg_rd = cpu_reg(s, a->rd);
8076
8077 if (crc32c) {
8078 gen_helper_crc32c_64(tcg_rd, tcg_acc, tcg_val, tcg_bytes);
8079 } else {
8080 gen_helper_crc32_64(tcg_rd, tcg_acc, tcg_val, tcg_bytes);
8081 }
8082 return true;
8083 }
8084
TRANS_FEAT(CRC32,aa64_crc32,do_crc32,a,false)8085 TRANS_FEAT(CRC32, aa64_crc32, do_crc32, a, false)
8086 TRANS_FEAT(CRC32C, aa64_crc32, do_crc32, a, true)
8087
8088 static bool do_subp(DisasContext *s, arg_rrr *a, bool setflag)
8089 {
8090 TCGv_i64 tcg_n = read_cpu_reg_sp(s, a->rn, true);
8091 TCGv_i64 tcg_m = read_cpu_reg_sp(s, a->rm, true);
8092 TCGv_i64 tcg_d = cpu_reg(s, a->rd);
8093
8094 tcg_gen_sextract_i64(tcg_n, tcg_n, 0, 56);
8095 tcg_gen_sextract_i64(tcg_m, tcg_m, 0, 56);
8096
8097 if (setflag) {
8098 gen_sub_CC(true, tcg_d, tcg_n, tcg_m);
8099 } else {
8100 tcg_gen_sub_i64(tcg_d, tcg_n, tcg_m);
8101 }
8102 return true;
8103 }
8104
TRANS_FEAT(SUBP,aa64_mte_insn_reg,do_subp,a,false)8105 TRANS_FEAT(SUBP, aa64_mte_insn_reg, do_subp, a, false)
8106 TRANS_FEAT(SUBPS, aa64_mte_insn_reg, do_subp, a, true)
8107
8108 static bool trans_IRG(DisasContext *s, arg_rrr *a)
8109 {
8110 if (dc_isar_feature(aa64_mte_insn_reg, s)) {
8111 TCGv_i64 tcg_rd = cpu_reg_sp(s, a->rd);
8112 TCGv_i64 tcg_rn = cpu_reg_sp(s, a->rn);
8113
8114 if (s->ata[0]) {
8115 gen_helper_irg(tcg_rd, tcg_env, tcg_rn, cpu_reg(s, a->rm));
8116 } else {
8117 gen_address_with_allocation_tag0(tcg_rd, tcg_rn);
8118 }
8119 return true;
8120 }
8121 return false;
8122 }
8123
trans_GMI(DisasContext * s,arg_rrr * a)8124 static bool trans_GMI(DisasContext *s, arg_rrr *a)
8125 {
8126 if (dc_isar_feature(aa64_mte_insn_reg, s)) {
8127 TCGv_i64 t = tcg_temp_new_i64();
8128
8129 tcg_gen_extract_i64(t, cpu_reg_sp(s, a->rn), 56, 4);
8130 tcg_gen_shl_i64(t, tcg_constant_i64(1), t);
8131 tcg_gen_or_i64(cpu_reg(s, a->rd), cpu_reg(s, a->rm), t);
8132 return true;
8133 }
8134 return false;
8135 }
8136
trans_PACGA(DisasContext * s,arg_rrr * a)8137 static bool trans_PACGA(DisasContext *s, arg_rrr *a)
8138 {
8139 if (dc_isar_feature(aa64_pauth, s)) {
8140 gen_helper_pacga(cpu_reg(s, a->rd), tcg_env,
8141 cpu_reg(s, a->rn), cpu_reg_sp(s, a->rm));
8142 return true;
8143 }
8144 return false;
8145 }
8146
8147 typedef void ArithOneOp(TCGv_i64, TCGv_i64);
8148
gen_rr(DisasContext * s,int rd,int rn,ArithOneOp fn)8149 static bool gen_rr(DisasContext *s, int rd, int rn, ArithOneOp fn)
8150 {
8151 fn(cpu_reg(s, rd), cpu_reg(s, rn));
8152 return true;
8153 }
8154
gen_rbit32(TCGv_i64 tcg_rd,TCGv_i64 tcg_rn)8155 static void gen_rbit32(TCGv_i64 tcg_rd, TCGv_i64 tcg_rn)
8156 {
8157 TCGv_i32 t32 = tcg_temp_new_i32();
8158
8159 tcg_gen_extrl_i64_i32(t32, tcg_rn);
8160 gen_helper_rbit(t32, t32);
8161 tcg_gen_extu_i32_i64(tcg_rd, t32);
8162 }
8163
gen_rev16_xx(TCGv_i64 tcg_rd,TCGv_i64 tcg_rn,TCGv_i64 mask)8164 static void gen_rev16_xx(TCGv_i64 tcg_rd, TCGv_i64 tcg_rn, TCGv_i64 mask)
8165 {
8166 TCGv_i64 tcg_tmp = tcg_temp_new_i64();
8167
8168 tcg_gen_shri_i64(tcg_tmp, tcg_rn, 8);
8169 tcg_gen_and_i64(tcg_rd, tcg_rn, mask);
8170 tcg_gen_and_i64(tcg_tmp, tcg_tmp, mask);
8171 tcg_gen_shli_i64(tcg_rd, tcg_rd, 8);
8172 tcg_gen_or_i64(tcg_rd, tcg_rd, tcg_tmp);
8173 }
8174
gen_rev16_32(TCGv_i64 tcg_rd,TCGv_i64 tcg_rn)8175 static void gen_rev16_32(TCGv_i64 tcg_rd, TCGv_i64 tcg_rn)
8176 {
8177 gen_rev16_xx(tcg_rd, tcg_rn, tcg_constant_i64(0x00ff00ff));
8178 }
8179
gen_rev16_64(TCGv_i64 tcg_rd,TCGv_i64 tcg_rn)8180 static void gen_rev16_64(TCGv_i64 tcg_rd, TCGv_i64 tcg_rn)
8181 {
8182 gen_rev16_xx(tcg_rd, tcg_rn, tcg_constant_i64(0x00ff00ff00ff00ffull));
8183 }
8184
gen_rev_32(TCGv_i64 tcg_rd,TCGv_i64 tcg_rn)8185 static void gen_rev_32(TCGv_i64 tcg_rd, TCGv_i64 tcg_rn)
8186 {
8187 tcg_gen_bswap32_i64(tcg_rd, tcg_rn, TCG_BSWAP_OZ);
8188 }
8189
gen_rev32(TCGv_i64 tcg_rd,TCGv_i64 tcg_rn)8190 static void gen_rev32(TCGv_i64 tcg_rd, TCGv_i64 tcg_rn)
8191 {
8192 tcg_gen_bswap64_i64(tcg_rd, tcg_rn);
8193 tcg_gen_rotri_i64(tcg_rd, tcg_rd, 32);
8194 }
8195
8196 TRANS(RBIT, gen_rr, a->rd, a->rn, a->sf ? gen_helper_rbit64 : gen_rbit32)
8197 TRANS(REV16, gen_rr, a->rd, a->rn, a->sf ? gen_rev16_64 : gen_rev16_32)
8198 TRANS(REV32, gen_rr, a->rd, a->rn, a->sf ? gen_rev32 : gen_rev_32)
8199 TRANS(REV64, gen_rr, a->rd, a->rn, tcg_gen_bswap64_i64)
8200
gen_clz32(TCGv_i64 tcg_rd,TCGv_i64 tcg_rn)8201 static void gen_clz32(TCGv_i64 tcg_rd, TCGv_i64 tcg_rn)
8202 {
8203 TCGv_i32 t32 = tcg_temp_new_i32();
8204
8205 tcg_gen_extrl_i64_i32(t32, tcg_rn);
8206 tcg_gen_clzi_i32(t32, t32, 32);
8207 tcg_gen_extu_i32_i64(tcg_rd, t32);
8208 }
8209
gen_clz64(TCGv_i64 tcg_rd,TCGv_i64 tcg_rn)8210 static void gen_clz64(TCGv_i64 tcg_rd, TCGv_i64 tcg_rn)
8211 {
8212 tcg_gen_clzi_i64(tcg_rd, tcg_rn, 64);
8213 }
8214
gen_cls32(TCGv_i64 tcg_rd,TCGv_i64 tcg_rn)8215 static void gen_cls32(TCGv_i64 tcg_rd, TCGv_i64 tcg_rn)
8216 {
8217 TCGv_i32 t32 = tcg_temp_new_i32();
8218
8219 tcg_gen_extrl_i64_i32(t32, tcg_rn);
8220 tcg_gen_clrsb_i32(t32, t32);
8221 tcg_gen_extu_i32_i64(tcg_rd, t32);
8222 }
8223
8224 TRANS(CLZ, gen_rr, a->rd, a->rn, a->sf ? gen_clz64 : gen_clz32)
8225 TRANS(CLS, gen_rr, a->rd, a->rn, a->sf ? tcg_gen_clrsb_i64 : gen_cls32)
8226
gen_pacaut(DisasContext * s,arg_pacaut * a,NeonGenTwo64OpEnvFn fn)8227 static bool gen_pacaut(DisasContext *s, arg_pacaut *a, NeonGenTwo64OpEnvFn fn)
8228 {
8229 TCGv_i64 tcg_rd, tcg_rn;
8230
8231 if (a->z) {
8232 if (a->rn != 31) {
8233 return false;
8234 }
8235 tcg_rn = tcg_constant_i64(0);
8236 } else {
8237 tcg_rn = cpu_reg_sp(s, a->rn);
8238 }
8239 if (s->pauth_active) {
8240 tcg_rd = cpu_reg(s, a->rd);
8241 fn(tcg_rd, tcg_env, tcg_rd, tcg_rn);
8242 }
8243 return true;
8244 }
8245
TRANS_FEAT(PACIA,aa64_pauth,gen_pacaut,a,gen_helper_pacia)8246 TRANS_FEAT(PACIA, aa64_pauth, gen_pacaut, a, gen_helper_pacia)
8247 TRANS_FEAT(PACIB, aa64_pauth, gen_pacaut, a, gen_helper_pacib)
8248 TRANS_FEAT(PACDA, aa64_pauth, gen_pacaut, a, gen_helper_pacda)
8249 TRANS_FEAT(PACDB, aa64_pauth, gen_pacaut, a, gen_helper_pacdb)
8250
8251 TRANS_FEAT(AUTIA, aa64_pauth, gen_pacaut, a, gen_helper_autia)
8252 TRANS_FEAT(AUTIB, aa64_pauth, gen_pacaut, a, gen_helper_autib)
8253 TRANS_FEAT(AUTDA, aa64_pauth, gen_pacaut, a, gen_helper_autda)
8254 TRANS_FEAT(AUTDB, aa64_pauth, gen_pacaut, a, gen_helper_autdb)
8255
8256 static bool do_xpac(DisasContext *s, int rd, NeonGenOne64OpEnvFn *fn)
8257 {
8258 if (s->pauth_active) {
8259 TCGv_i64 tcg_rd = cpu_reg(s, rd);
8260 fn(tcg_rd, tcg_env, tcg_rd);
8261 }
8262 return true;
8263 }
8264
8265 TRANS_FEAT(XPACI, aa64_pauth, do_xpac, a->rd, gen_helper_xpaci)
8266 TRANS_FEAT(XPACD, aa64_pauth, do_xpac, a->rd, gen_helper_xpacd)
8267
do_logic_reg(DisasContext * s,arg_logic_shift * a,ArithTwoOp * fn,ArithTwoOp * inv_fn,bool setflags)8268 static bool do_logic_reg(DisasContext *s, arg_logic_shift *a,
8269 ArithTwoOp *fn, ArithTwoOp *inv_fn, bool setflags)
8270 {
8271 TCGv_i64 tcg_rd, tcg_rn, tcg_rm;
8272
8273 if (!a->sf && (a->sa & (1 << 5))) {
8274 return false;
8275 }
8276
8277 tcg_rd = cpu_reg(s, a->rd);
8278 tcg_rn = cpu_reg(s, a->rn);
8279
8280 tcg_rm = read_cpu_reg(s, a->rm, a->sf);
8281 if (a->sa) {
8282 shift_reg_imm(tcg_rm, tcg_rm, a->sf, a->st, a->sa);
8283 }
8284
8285 (a->n ? inv_fn : fn)(tcg_rd, tcg_rn, tcg_rm);
8286 if (!a->sf) {
8287 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
8288 }
8289 if (setflags) {
8290 gen_logic_CC(a->sf, tcg_rd);
8291 }
8292 return true;
8293 }
8294
trans_ORR_r(DisasContext * s,arg_logic_shift * a)8295 static bool trans_ORR_r(DisasContext *s, arg_logic_shift *a)
8296 {
8297 /*
8298 * Unshifted ORR and ORN with WZR/XZR is the standard encoding for
8299 * register-register MOV and MVN, so it is worth special casing.
8300 */
8301 if (a->sa == 0 && a->st == 0 && a->rn == 31) {
8302 TCGv_i64 tcg_rd = cpu_reg(s, a->rd);
8303 TCGv_i64 tcg_rm = cpu_reg(s, a->rm);
8304
8305 if (a->n) {
8306 tcg_gen_not_i64(tcg_rd, tcg_rm);
8307 if (!a->sf) {
8308 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
8309 }
8310 } else {
8311 if (a->sf) {
8312 tcg_gen_mov_i64(tcg_rd, tcg_rm);
8313 } else {
8314 tcg_gen_ext32u_i64(tcg_rd, tcg_rm);
8315 }
8316 }
8317 return true;
8318 }
8319
8320 return do_logic_reg(s, a, tcg_gen_or_i64, tcg_gen_orc_i64, false);
8321 }
8322
TRANS(AND_r,do_logic_reg,a,tcg_gen_and_i64,tcg_gen_andc_i64,false)8323 TRANS(AND_r, do_logic_reg, a, tcg_gen_and_i64, tcg_gen_andc_i64, false)
8324 TRANS(ANDS_r, do_logic_reg, a, tcg_gen_and_i64, tcg_gen_andc_i64, true)
8325 TRANS(EOR_r, do_logic_reg, a, tcg_gen_xor_i64, tcg_gen_eqv_i64, false)
8326
8327 static bool do_addsub_ext(DisasContext *s, arg_addsub_ext *a,
8328 bool sub_op, bool setflags)
8329 {
8330 TCGv_i64 tcg_rm, tcg_rn, tcg_rd, tcg_result;
8331
8332 if (a->sa > 4) {
8333 return false;
8334 }
8335
8336 /* non-flag setting ops may use SP */
8337 if (!setflags) {
8338 tcg_rd = cpu_reg_sp(s, a->rd);
8339 } else {
8340 tcg_rd = cpu_reg(s, a->rd);
8341 }
8342 tcg_rn = read_cpu_reg_sp(s, a->rn, a->sf);
8343
8344 tcg_rm = read_cpu_reg(s, a->rm, a->sf);
8345 ext_and_shift_reg(tcg_rm, tcg_rm, a->st, a->sa);
8346
8347 tcg_result = tcg_temp_new_i64();
8348 if (!setflags) {
8349 if (sub_op) {
8350 tcg_gen_sub_i64(tcg_result, tcg_rn, tcg_rm);
8351 } else {
8352 tcg_gen_add_i64(tcg_result, tcg_rn, tcg_rm);
8353 }
8354 } else {
8355 if (sub_op) {
8356 gen_sub_CC(a->sf, tcg_result, tcg_rn, tcg_rm);
8357 } else {
8358 gen_add_CC(a->sf, tcg_result, tcg_rn, tcg_rm);
8359 }
8360 }
8361
8362 if (a->sf) {
8363 tcg_gen_mov_i64(tcg_rd, tcg_result);
8364 } else {
8365 tcg_gen_ext32u_i64(tcg_rd, tcg_result);
8366 }
8367 return true;
8368 }
8369
TRANS(ADD_ext,do_addsub_ext,a,false,false)8370 TRANS(ADD_ext, do_addsub_ext, a, false, false)
8371 TRANS(SUB_ext, do_addsub_ext, a, true, false)
8372 TRANS(ADDS_ext, do_addsub_ext, a, false, true)
8373 TRANS(SUBS_ext, do_addsub_ext, a, true, true)
8374
8375 static bool do_addsub_reg(DisasContext *s, arg_addsub_shift *a,
8376 bool sub_op, bool setflags)
8377 {
8378 TCGv_i64 tcg_rd, tcg_rn, tcg_rm, tcg_result;
8379
8380 if (a->st == 3 || (!a->sf && (a->sa & 32))) {
8381 return false;
8382 }
8383
8384 tcg_rd = cpu_reg(s, a->rd);
8385 tcg_rn = read_cpu_reg(s, a->rn, a->sf);
8386 tcg_rm = read_cpu_reg(s, a->rm, a->sf);
8387
8388 shift_reg_imm(tcg_rm, tcg_rm, a->sf, a->st, a->sa);
8389
8390 tcg_result = tcg_temp_new_i64();
8391 if (!setflags) {
8392 if (sub_op) {
8393 tcg_gen_sub_i64(tcg_result, tcg_rn, tcg_rm);
8394 } else {
8395 tcg_gen_add_i64(tcg_result, tcg_rn, tcg_rm);
8396 }
8397 } else {
8398 if (sub_op) {
8399 gen_sub_CC(a->sf, tcg_result, tcg_rn, tcg_rm);
8400 } else {
8401 gen_add_CC(a->sf, tcg_result, tcg_rn, tcg_rm);
8402 }
8403 }
8404
8405 if (a->sf) {
8406 tcg_gen_mov_i64(tcg_rd, tcg_result);
8407 } else {
8408 tcg_gen_ext32u_i64(tcg_rd, tcg_result);
8409 }
8410 return true;
8411 }
8412
TRANS(ADD_r,do_addsub_reg,a,false,false)8413 TRANS(ADD_r, do_addsub_reg, a, false, false)
8414 TRANS(SUB_r, do_addsub_reg, a, true, false)
8415 TRANS(ADDS_r, do_addsub_reg, a, false, true)
8416 TRANS(SUBS_r, do_addsub_reg, a, true, true)
8417
8418 static bool do_mulh(DisasContext *s, arg_rrr *a,
8419 void (*fn)(TCGv_i64, TCGv_i64, TCGv_i64, TCGv_i64))
8420 {
8421 TCGv_i64 discard = tcg_temp_new_i64();
8422 TCGv_i64 tcg_rd = cpu_reg(s, a->rd);
8423 TCGv_i64 tcg_rn = cpu_reg(s, a->rn);
8424 TCGv_i64 tcg_rm = cpu_reg(s, a->rm);
8425
8426 fn(discard, tcg_rd, tcg_rn, tcg_rm);
8427 return true;
8428 }
8429
TRANS(SMULH,do_mulh,a,tcg_gen_muls2_i64)8430 TRANS(SMULH, do_mulh, a, tcg_gen_muls2_i64)
8431 TRANS(UMULH, do_mulh, a, tcg_gen_mulu2_i64)
8432
8433 static bool do_muladd(DisasContext *s, arg_rrrr *a,
8434 bool sf, bool is_sub, MemOp mop)
8435 {
8436 TCGv_i64 tcg_rd = cpu_reg(s, a->rd);
8437 TCGv_i64 tcg_op1, tcg_op2;
8438
8439 if (mop == MO_64) {
8440 tcg_op1 = cpu_reg(s, a->rn);
8441 tcg_op2 = cpu_reg(s, a->rm);
8442 } else {
8443 tcg_op1 = tcg_temp_new_i64();
8444 tcg_op2 = tcg_temp_new_i64();
8445 tcg_gen_ext_i64(tcg_op1, cpu_reg(s, a->rn), mop);
8446 tcg_gen_ext_i64(tcg_op2, cpu_reg(s, a->rm), mop);
8447 }
8448
8449 if (a->ra == 31 && !is_sub) {
8450 /* Special-case MADD with rA == XZR; it is the standard MUL alias */
8451 tcg_gen_mul_i64(tcg_rd, tcg_op1, tcg_op2);
8452 } else {
8453 TCGv_i64 tcg_tmp = tcg_temp_new_i64();
8454 TCGv_i64 tcg_ra = cpu_reg(s, a->ra);
8455
8456 tcg_gen_mul_i64(tcg_tmp, tcg_op1, tcg_op2);
8457 if (is_sub) {
8458 tcg_gen_sub_i64(tcg_rd, tcg_ra, tcg_tmp);
8459 } else {
8460 tcg_gen_add_i64(tcg_rd, tcg_ra, tcg_tmp);
8461 }
8462 }
8463
8464 if (!sf) {
8465 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
8466 }
8467 return true;
8468 }
8469
TRANS(MADD_w,do_muladd,a,false,false,MO_64)8470 TRANS(MADD_w, do_muladd, a, false, false, MO_64)
8471 TRANS(MSUB_w, do_muladd, a, false, true, MO_64)
8472 TRANS(MADD_x, do_muladd, a, true, false, MO_64)
8473 TRANS(MSUB_x, do_muladd, a, true, true, MO_64)
8474
8475 TRANS(SMADDL, do_muladd, a, true, false, MO_SL)
8476 TRANS(SMSUBL, do_muladd, a, true, true, MO_SL)
8477 TRANS(UMADDL, do_muladd, a, true, false, MO_UL)
8478 TRANS(UMSUBL, do_muladd, a, true, true, MO_UL)
8479
8480 static bool do_adc_sbc(DisasContext *s, arg_rrr_sf *a,
8481 bool is_sub, bool setflags)
8482 {
8483 TCGv_i64 tcg_y, tcg_rn, tcg_rd;
8484
8485 tcg_rd = cpu_reg(s, a->rd);
8486 tcg_rn = cpu_reg(s, a->rn);
8487
8488 if (is_sub) {
8489 tcg_y = tcg_temp_new_i64();
8490 tcg_gen_not_i64(tcg_y, cpu_reg(s, a->rm));
8491 } else {
8492 tcg_y = cpu_reg(s, a->rm);
8493 }
8494
8495 if (setflags) {
8496 gen_adc_CC(a->sf, tcg_rd, tcg_rn, tcg_y);
8497 } else {
8498 gen_adc(a->sf, tcg_rd, tcg_rn, tcg_y);
8499 }
8500 return true;
8501 }
8502
TRANS(ADC,do_adc_sbc,a,false,false)8503 TRANS(ADC, do_adc_sbc, a, false, false)
8504 TRANS(SBC, do_adc_sbc, a, true, false)
8505 TRANS(ADCS, do_adc_sbc, a, false, true)
8506 TRANS(SBCS, do_adc_sbc, a, true, true)
8507
8508 static bool trans_RMIF(DisasContext *s, arg_RMIF *a)
8509 {
8510 int mask = a->mask;
8511 TCGv_i64 tcg_rn;
8512 TCGv_i32 nzcv;
8513
8514 if (!dc_isar_feature(aa64_condm_4, s)) {
8515 return false;
8516 }
8517
8518 tcg_rn = read_cpu_reg(s, a->rn, 1);
8519 tcg_gen_rotri_i64(tcg_rn, tcg_rn, a->imm);
8520
8521 nzcv = tcg_temp_new_i32();
8522 tcg_gen_extrl_i64_i32(nzcv, tcg_rn);
8523
8524 if (mask & 8) { /* N */
8525 tcg_gen_shli_i32(cpu_NF, nzcv, 31 - 3);
8526 }
8527 if (mask & 4) { /* Z */
8528 tcg_gen_not_i32(cpu_ZF, nzcv);
8529 tcg_gen_andi_i32(cpu_ZF, cpu_ZF, 4);
8530 }
8531 if (mask & 2) { /* C */
8532 tcg_gen_extract_i32(cpu_CF, nzcv, 1, 1);
8533 }
8534 if (mask & 1) { /* V */
8535 tcg_gen_shli_i32(cpu_VF, nzcv, 31 - 0);
8536 }
8537 return true;
8538 }
8539
do_setf(DisasContext * s,int rn,int shift)8540 static bool do_setf(DisasContext *s, int rn, int shift)
8541 {
8542 TCGv_i32 tmp = tcg_temp_new_i32();
8543
8544 tcg_gen_extrl_i64_i32(tmp, cpu_reg(s, rn));
8545 tcg_gen_shli_i32(cpu_NF, tmp, shift);
8546 tcg_gen_shli_i32(cpu_VF, tmp, shift - 1);
8547 tcg_gen_mov_i32(cpu_ZF, cpu_NF);
8548 tcg_gen_xor_i32(cpu_VF, cpu_VF, cpu_NF);
8549 return true;
8550 }
8551
8552 TRANS_FEAT(SETF8, aa64_condm_4, do_setf, a->rn, 24)
8553 TRANS_FEAT(SETF16, aa64_condm_4, do_setf, a->rn, 16)
8554
8555 /* CCMP, CCMN */
trans_CCMP(DisasContext * s,arg_CCMP * a)8556 static bool trans_CCMP(DisasContext *s, arg_CCMP *a)
8557 {
8558 TCGv_i32 tcg_t0 = tcg_temp_new_i32();
8559 TCGv_i32 tcg_t1 = tcg_temp_new_i32();
8560 TCGv_i32 tcg_t2 = tcg_temp_new_i32();
8561 TCGv_i64 tcg_tmp = tcg_temp_new_i64();
8562 TCGv_i64 tcg_rn, tcg_y;
8563 DisasCompare c;
8564 unsigned nzcv;
8565 bool has_andc;
8566
8567 /* Set T0 = !COND. */
8568 arm_test_cc(&c, a->cond);
8569 tcg_gen_setcondi_i32(tcg_invert_cond(c.cond), tcg_t0, c.value, 0);
8570
8571 /* Load the arguments for the new comparison. */
8572 if (a->imm) {
8573 tcg_y = tcg_constant_i64(a->y);
8574 } else {
8575 tcg_y = cpu_reg(s, a->y);
8576 }
8577 tcg_rn = cpu_reg(s, a->rn);
8578
8579 /* Set the flags for the new comparison. */
8580 if (a->op) {
8581 gen_sub_CC(a->sf, tcg_tmp, tcg_rn, tcg_y);
8582 } else {
8583 gen_add_CC(a->sf, tcg_tmp, tcg_rn, tcg_y);
8584 }
8585
8586 /*
8587 * If COND was false, force the flags to #nzcv. Compute two masks
8588 * to help with this: T1 = (COND ? 0 : -1), T2 = (COND ? -1 : 0).
8589 * For tcg hosts that support ANDC, we can make do with just T1.
8590 * In either case, allow the tcg optimizer to delete any unused mask.
8591 */
8592 tcg_gen_neg_i32(tcg_t1, tcg_t0);
8593 tcg_gen_subi_i32(tcg_t2, tcg_t0, 1);
8594
8595 nzcv = a->nzcv;
8596 has_andc = tcg_op_supported(INDEX_op_andc, TCG_TYPE_I32, 0);
8597 if (nzcv & 8) { /* N */
8598 tcg_gen_or_i32(cpu_NF, cpu_NF, tcg_t1);
8599 } else {
8600 if (has_andc) {
8601 tcg_gen_andc_i32(cpu_NF, cpu_NF, tcg_t1);
8602 } else {
8603 tcg_gen_and_i32(cpu_NF, cpu_NF, tcg_t2);
8604 }
8605 }
8606 if (nzcv & 4) { /* Z */
8607 if (has_andc) {
8608 tcg_gen_andc_i32(cpu_ZF, cpu_ZF, tcg_t1);
8609 } else {
8610 tcg_gen_and_i32(cpu_ZF, cpu_ZF, tcg_t2);
8611 }
8612 } else {
8613 tcg_gen_or_i32(cpu_ZF, cpu_ZF, tcg_t0);
8614 }
8615 if (nzcv & 2) { /* C */
8616 tcg_gen_or_i32(cpu_CF, cpu_CF, tcg_t0);
8617 } else {
8618 if (has_andc) {
8619 tcg_gen_andc_i32(cpu_CF, cpu_CF, tcg_t1);
8620 } else {
8621 tcg_gen_and_i32(cpu_CF, cpu_CF, tcg_t2);
8622 }
8623 }
8624 if (nzcv & 1) { /* V */
8625 tcg_gen_or_i32(cpu_VF, cpu_VF, tcg_t1);
8626 } else {
8627 if (has_andc) {
8628 tcg_gen_andc_i32(cpu_VF, cpu_VF, tcg_t1);
8629 } else {
8630 tcg_gen_and_i32(cpu_VF, cpu_VF, tcg_t2);
8631 }
8632 }
8633 return true;
8634 }
8635
trans_CSEL(DisasContext * s,arg_CSEL * a)8636 static bool trans_CSEL(DisasContext *s, arg_CSEL *a)
8637 {
8638 TCGv_i64 tcg_rd = cpu_reg(s, a->rd);
8639 TCGv_i64 zero = tcg_constant_i64(0);
8640 DisasCompare64 c;
8641
8642 a64_test_cc(&c, a->cond);
8643
8644 if (a->rn == 31 && a->rm == 31 && (a->else_inc ^ a->else_inv)) {
8645 /* CSET & CSETM. */
8646 if (a->else_inv) {
8647 tcg_gen_negsetcond_i64(tcg_invert_cond(c.cond),
8648 tcg_rd, c.value, zero);
8649 } else {
8650 tcg_gen_setcond_i64(tcg_invert_cond(c.cond),
8651 tcg_rd, c.value, zero);
8652 }
8653 } else {
8654 TCGv_i64 t_true = cpu_reg(s, a->rn);
8655 TCGv_i64 t_false = read_cpu_reg(s, a->rm, 1);
8656
8657 if (a->else_inv && a->else_inc) {
8658 tcg_gen_neg_i64(t_false, t_false);
8659 } else if (a->else_inv) {
8660 tcg_gen_not_i64(t_false, t_false);
8661 } else if (a->else_inc) {
8662 tcg_gen_addi_i64(t_false, t_false, 1);
8663 }
8664 tcg_gen_movcond_i64(c.cond, tcg_rd, c.value, zero, t_true, t_false);
8665 }
8666
8667 if (!a->sf) {
8668 tcg_gen_ext32u_i64(tcg_rd, tcg_rd);
8669 }
8670 return true;
8671 }
8672
8673 typedef struct FPScalar1Int {
8674 void (*gen_h)(TCGv_i32, TCGv_i32);
8675 void (*gen_s)(TCGv_i32, TCGv_i32);
8676 void (*gen_d)(TCGv_i64, TCGv_i64);
8677 } FPScalar1Int;
8678
do_fp1_scalar_int(DisasContext * s,arg_rr_e * a,const FPScalar1Int * f,bool merging)8679 static bool do_fp1_scalar_int(DisasContext *s, arg_rr_e *a,
8680 const FPScalar1Int *f,
8681 bool merging)
8682 {
8683 switch (a->esz) {
8684 case MO_64:
8685 if (fp_access_check(s)) {
8686 TCGv_i64 t = read_fp_dreg(s, a->rn);
8687 f->gen_d(t, t);
8688 if (merging) {
8689 write_fp_dreg_merging(s, a->rd, a->rd, t);
8690 } else {
8691 write_fp_dreg(s, a->rd, t);
8692 }
8693 }
8694 break;
8695 case MO_32:
8696 if (fp_access_check(s)) {
8697 TCGv_i32 t = read_fp_sreg(s, a->rn);
8698 f->gen_s(t, t);
8699 if (merging) {
8700 write_fp_sreg_merging(s, a->rd, a->rd, t);
8701 } else {
8702 write_fp_sreg(s, a->rd, t);
8703 }
8704 }
8705 break;
8706 case MO_16:
8707 if (!dc_isar_feature(aa64_fp16, s)) {
8708 return false;
8709 }
8710 if (fp_access_check(s)) {
8711 TCGv_i32 t = read_fp_hreg(s, a->rn);
8712 f->gen_h(t, t);
8713 if (merging) {
8714 write_fp_hreg_merging(s, a->rd, a->rd, t);
8715 } else {
8716 write_fp_sreg(s, a->rd, t);
8717 }
8718 }
8719 break;
8720 default:
8721 return false;
8722 }
8723 return true;
8724 }
8725
do_fp1_scalar_int_2fn(DisasContext * s,arg_rr_e * a,const FPScalar1Int * fnormal,const FPScalar1Int * fah)8726 static bool do_fp1_scalar_int_2fn(DisasContext *s, arg_rr_e *a,
8727 const FPScalar1Int *fnormal,
8728 const FPScalar1Int *fah)
8729 {
8730 return do_fp1_scalar_int(s, a, s->fpcr_ah ? fah : fnormal, true);
8731 }
8732
8733 static const FPScalar1Int f_scalar_fmov = {
8734 tcg_gen_mov_i32,
8735 tcg_gen_mov_i32,
8736 tcg_gen_mov_i64,
8737 };
8738 TRANS(FMOV_s, do_fp1_scalar_int, a, &f_scalar_fmov, false)
8739
8740 static const FPScalar1Int f_scalar_fabs = {
8741 gen_vfp_absh,
8742 gen_vfp_abss,
8743 gen_vfp_absd,
8744 };
8745 static const FPScalar1Int f_scalar_ah_fabs = {
8746 gen_vfp_ah_absh,
8747 gen_vfp_ah_abss,
8748 gen_vfp_ah_absd,
8749 };
8750 TRANS(FABS_s, do_fp1_scalar_int_2fn, a, &f_scalar_fabs, &f_scalar_ah_fabs)
8751
8752 static const FPScalar1Int f_scalar_fneg = {
8753 gen_vfp_negh,
8754 gen_vfp_negs,
8755 gen_vfp_negd,
8756 };
8757 static const FPScalar1Int f_scalar_ah_fneg = {
8758 gen_vfp_ah_negh,
8759 gen_vfp_ah_negs,
8760 gen_vfp_ah_negd,
8761 };
8762 TRANS(FNEG_s, do_fp1_scalar_int_2fn, a, &f_scalar_fneg, &f_scalar_ah_fneg)
8763
8764 typedef struct FPScalar1 {
8765 void (*gen_h)(TCGv_i32, TCGv_i32, TCGv_ptr);
8766 void (*gen_s)(TCGv_i32, TCGv_i32, TCGv_ptr);
8767 void (*gen_d)(TCGv_i64, TCGv_i64, TCGv_ptr);
8768 } FPScalar1;
8769
do_fp1_scalar_with_fpsttype(DisasContext * s,arg_rr_e * a,const FPScalar1 * f,int rmode,ARMFPStatusFlavour fpsttype)8770 static bool do_fp1_scalar_with_fpsttype(DisasContext *s, arg_rr_e *a,
8771 const FPScalar1 *f, int rmode,
8772 ARMFPStatusFlavour fpsttype)
8773 {
8774 TCGv_i32 tcg_rmode = NULL;
8775 TCGv_ptr fpst;
8776 TCGv_i64 t64;
8777 TCGv_i32 t32;
8778 int check = fp_access_check_scalar_hsd(s, a->esz);
8779
8780 if (check <= 0) {
8781 return check == 0;
8782 }
8783
8784 fpst = fpstatus_ptr(fpsttype);
8785 if (rmode >= 0) {
8786 tcg_rmode = gen_set_rmode(rmode, fpst);
8787 }
8788
8789 switch (a->esz) {
8790 case MO_64:
8791 t64 = read_fp_dreg(s, a->rn);
8792 f->gen_d(t64, t64, fpst);
8793 write_fp_dreg_merging(s, a->rd, a->rd, t64);
8794 break;
8795 case MO_32:
8796 t32 = read_fp_sreg(s, a->rn);
8797 f->gen_s(t32, t32, fpst);
8798 write_fp_sreg_merging(s, a->rd, a->rd, t32);
8799 break;
8800 case MO_16:
8801 t32 = read_fp_hreg(s, a->rn);
8802 f->gen_h(t32, t32, fpst);
8803 write_fp_hreg_merging(s, a->rd, a->rd, t32);
8804 break;
8805 default:
8806 g_assert_not_reached();
8807 }
8808
8809 if (rmode >= 0) {
8810 gen_restore_rmode(tcg_rmode, fpst);
8811 }
8812 return true;
8813 }
8814
do_fp1_scalar(DisasContext * s,arg_rr_e * a,const FPScalar1 * f,int rmode)8815 static bool do_fp1_scalar(DisasContext *s, arg_rr_e *a,
8816 const FPScalar1 *f, int rmode)
8817 {
8818 return do_fp1_scalar_with_fpsttype(s, a, f, rmode,
8819 a->esz == MO_16 ?
8820 FPST_A64_F16 : FPST_A64);
8821 }
8822
do_fp1_scalar_ah(DisasContext * s,arg_rr_e * a,const FPScalar1 * f,int rmode)8823 static bool do_fp1_scalar_ah(DisasContext *s, arg_rr_e *a,
8824 const FPScalar1 *f, int rmode)
8825 {
8826 return do_fp1_scalar_with_fpsttype(s, a, f, rmode, select_ah_fpst(s, a->esz));
8827 }
8828
8829 static const FPScalar1 f_scalar_fsqrt = {
8830 gen_helper_vfp_sqrth,
8831 gen_helper_vfp_sqrts,
8832 gen_helper_vfp_sqrtd,
8833 };
8834 TRANS(FSQRT_s, do_fp1_scalar, a, &f_scalar_fsqrt, -1)
8835
8836 static const FPScalar1 f_scalar_frint = {
8837 gen_helper_advsimd_rinth,
8838 gen_helper_rints,
8839 gen_helper_rintd,
8840 };
8841 TRANS(FRINTN_s, do_fp1_scalar, a, &f_scalar_frint, FPROUNDING_TIEEVEN)
8842 TRANS(FRINTP_s, do_fp1_scalar, a, &f_scalar_frint, FPROUNDING_POSINF)
8843 TRANS(FRINTM_s, do_fp1_scalar, a, &f_scalar_frint, FPROUNDING_NEGINF)
8844 TRANS(FRINTZ_s, do_fp1_scalar, a, &f_scalar_frint, FPROUNDING_ZERO)
8845 TRANS(FRINTA_s, do_fp1_scalar, a, &f_scalar_frint, FPROUNDING_TIEAWAY)
8846 TRANS(FRINTI_s, do_fp1_scalar, a, &f_scalar_frint, -1)
8847
8848 static const FPScalar1 f_scalar_frintx = {
8849 gen_helper_advsimd_rinth_exact,
8850 gen_helper_rints_exact,
8851 gen_helper_rintd_exact,
8852 };
8853 TRANS(FRINTX_s, do_fp1_scalar, a, &f_scalar_frintx, -1)
8854
trans_BFCVT_s(DisasContext * s,arg_rr_e * a)8855 static bool trans_BFCVT_s(DisasContext *s, arg_rr_e *a)
8856 {
8857 ARMFPStatusFlavour fpsttype = s->fpcr_ah ? FPST_AH : FPST_A64;
8858 TCGv_i32 t32;
8859 int check;
8860
8861 if (!dc_isar_feature(aa64_bf16, s)) {
8862 return false;
8863 }
8864
8865 check = fp_access_check_scalar_hsd(s, a->esz);
8866
8867 if (check <= 0) {
8868 return check == 0;
8869 }
8870
8871 t32 = read_fp_sreg(s, a->rn);
8872 gen_helper_bfcvt(t32, t32, fpstatus_ptr(fpsttype));
8873 write_fp_hreg_merging(s, a->rd, a->rd, t32);
8874 return true;
8875 }
8876
8877 static const FPScalar1 f_scalar_frint32 = {
8878 NULL,
8879 gen_helper_frint32_s,
8880 gen_helper_frint32_d,
8881 };
8882 TRANS_FEAT(FRINT32Z_s, aa64_frint, do_fp1_scalar, a,
8883 &f_scalar_frint32, FPROUNDING_ZERO)
8884 TRANS_FEAT(FRINT32X_s, aa64_frint, do_fp1_scalar, a, &f_scalar_frint32, -1)
8885
8886 static const FPScalar1 f_scalar_frint64 = {
8887 NULL,
8888 gen_helper_frint64_s,
8889 gen_helper_frint64_d,
8890 };
8891 TRANS_FEAT(FRINT64Z_s, aa64_frint, do_fp1_scalar, a,
8892 &f_scalar_frint64, FPROUNDING_ZERO)
8893 TRANS_FEAT(FRINT64X_s, aa64_frint, do_fp1_scalar, a, &f_scalar_frint64, -1)
8894
8895 static const FPScalar1 f_scalar_frecpe = {
8896 gen_helper_recpe_f16,
8897 gen_helper_recpe_f32,
8898 gen_helper_recpe_f64,
8899 };
8900 static const FPScalar1 f_scalar_frecpe_rpres = {
8901 gen_helper_recpe_f16,
8902 gen_helper_recpe_rpres_f32,
8903 gen_helper_recpe_f64,
8904 };
8905 TRANS(FRECPE_s, do_fp1_scalar_ah, a,
8906 s->fpcr_ah && dc_isar_feature(aa64_rpres, s) ?
8907 &f_scalar_frecpe_rpres : &f_scalar_frecpe, -1)
8908
8909 static const FPScalar1 f_scalar_frecpx = {
8910 gen_helper_frecpx_f16,
8911 gen_helper_frecpx_f32,
8912 gen_helper_frecpx_f64,
8913 };
8914 TRANS(FRECPX_s, do_fp1_scalar_ah, a, &f_scalar_frecpx, -1)
8915
8916 static const FPScalar1 f_scalar_frsqrte = {
8917 gen_helper_rsqrte_f16,
8918 gen_helper_rsqrte_f32,
8919 gen_helper_rsqrte_f64,
8920 };
8921 static const FPScalar1 f_scalar_frsqrte_rpres = {
8922 gen_helper_rsqrte_f16,
8923 gen_helper_rsqrte_rpres_f32,
8924 gen_helper_rsqrte_f64,
8925 };
8926 TRANS(FRSQRTE_s, do_fp1_scalar_ah, a,
8927 s->fpcr_ah && dc_isar_feature(aa64_rpres, s) ?
8928 &f_scalar_frsqrte_rpres : &f_scalar_frsqrte, -1)
8929
trans_FCVT_s_ds(DisasContext * s,arg_rr * a)8930 static bool trans_FCVT_s_ds(DisasContext *s, arg_rr *a)
8931 {
8932 if (fp_access_check(s)) {
8933 TCGv_i32 tcg_rn = read_fp_sreg(s, a->rn);
8934 TCGv_i64 tcg_rd = tcg_temp_new_i64();
8935 TCGv_ptr fpst = fpstatus_ptr(FPST_A64);
8936
8937 gen_helper_vfp_fcvtds(tcg_rd, tcg_rn, fpst);
8938 write_fp_dreg_merging(s, a->rd, a->rd, tcg_rd);
8939 }
8940 return true;
8941 }
8942
trans_FCVT_s_hs(DisasContext * s,arg_rr * a)8943 static bool trans_FCVT_s_hs(DisasContext *s, arg_rr *a)
8944 {
8945 if (fp_access_check(s)) {
8946 TCGv_i32 tmp = read_fp_sreg(s, a->rn);
8947 TCGv_i32 ahp = get_ahp_flag();
8948 TCGv_ptr fpst = fpstatus_ptr(FPST_A64);
8949
8950 gen_helper_vfp_fcvt_f32_to_f16(tmp, tmp, fpst, ahp);
8951 /* write_fp_hreg_merging is OK here because top half of result is zero */
8952 write_fp_hreg_merging(s, a->rd, a->rd, tmp);
8953 }
8954 return true;
8955 }
8956
trans_FCVT_s_sd(DisasContext * s,arg_rr * a)8957 static bool trans_FCVT_s_sd(DisasContext *s, arg_rr *a)
8958 {
8959 if (fp_access_check(s)) {
8960 TCGv_i64 tcg_rn = read_fp_dreg(s, a->rn);
8961 TCGv_i32 tcg_rd = tcg_temp_new_i32();
8962 TCGv_ptr fpst = fpstatus_ptr(FPST_A64);
8963
8964 gen_helper_vfp_fcvtsd(tcg_rd, tcg_rn, fpst);
8965 write_fp_sreg_merging(s, a->rd, a->rd, tcg_rd);
8966 }
8967 return true;
8968 }
8969
trans_FCVT_s_hd(DisasContext * s,arg_rr * a)8970 static bool trans_FCVT_s_hd(DisasContext *s, arg_rr *a)
8971 {
8972 if (fp_access_check(s)) {
8973 TCGv_i64 tcg_rn = read_fp_dreg(s, a->rn);
8974 TCGv_i32 tcg_rd = tcg_temp_new_i32();
8975 TCGv_i32 ahp = get_ahp_flag();
8976 TCGv_ptr fpst = fpstatus_ptr(FPST_A64);
8977
8978 gen_helper_vfp_fcvt_f64_to_f16(tcg_rd, tcg_rn, fpst, ahp);
8979 /* write_fp_hreg_merging is OK here because top half of tcg_rd is zero */
8980 write_fp_hreg_merging(s, a->rd, a->rd, tcg_rd);
8981 }
8982 return true;
8983 }
8984
trans_FCVT_s_sh(DisasContext * s,arg_rr * a)8985 static bool trans_FCVT_s_sh(DisasContext *s, arg_rr *a)
8986 {
8987 if (fp_access_check(s)) {
8988 TCGv_i32 tcg_rn = read_fp_hreg(s, a->rn);
8989 TCGv_i32 tcg_rd = tcg_temp_new_i32();
8990 TCGv_ptr tcg_fpst = fpstatus_ptr(FPST_A64_F16);
8991 TCGv_i32 tcg_ahp = get_ahp_flag();
8992
8993 gen_helper_vfp_fcvt_f16_to_f32(tcg_rd, tcg_rn, tcg_fpst, tcg_ahp);
8994 write_fp_sreg_merging(s, a->rd, a->rd, tcg_rd);
8995 }
8996 return true;
8997 }
8998
trans_FCVT_s_dh(DisasContext * s,arg_rr * a)8999 static bool trans_FCVT_s_dh(DisasContext *s, arg_rr *a)
9000 {
9001 if (fp_access_check(s)) {
9002 TCGv_i32 tcg_rn = read_fp_hreg(s, a->rn);
9003 TCGv_i64 tcg_rd = tcg_temp_new_i64();
9004 TCGv_ptr tcg_fpst = fpstatus_ptr(FPST_A64_F16);
9005 TCGv_i32 tcg_ahp = get_ahp_flag();
9006
9007 gen_helper_vfp_fcvt_f16_to_f64(tcg_rd, tcg_rn, tcg_fpst, tcg_ahp);
9008 write_fp_dreg_merging(s, a->rd, a->rd, tcg_rd);
9009 }
9010 return true;
9011 }
9012
do_cvtf_scalar(DisasContext * s,MemOp esz,int rd,int shift,TCGv_i64 tcg_int,bool is_signed)9013 static bool do_cvtf_scalar(DisasContext *s, MemOp esz, int rd, int shift,
9014 TCGv_i64 tcg_int, bool is_signed)
9015 {
9016 TCGv_ptr tcg_fpstatus;
9017 TCGv_i32 tcg_shift, tcg_single;
9018 TCGv_i64 tcg_double;
9019
9020 tcg_fpstatus = fpstatus_ptr(esz == MO_16 ? FPST_A64_F16 : FPST_A64);
9021 tcg_shift = tcg_constant_i32(shift);
9022
9023 switch (esz) {
9024 case MO_64:
9025 tcg_double = tcg_temp_new_i64();
9026 if (is_signed) {
9027 gen_helper_vfp_sqtod(tcg_double, tcg_int, tcg_shift, tcg_fpstatus);
9028 } else {
9029 gen_helper_vfp_uqtod(tcg_double, tcg_int, tcg_shift, tcg_fpstatus);
9030 }
9031 write_fp_dreg_merging(s, rd, rd, tcg_double);
9032 break;
9033
9034 case MO_32:
9035 tcg_single = tcg_temp_new_i32();
9036 if (is_signed) {
9037 gen_helper_vfp_sqtos(tcg_single, tcg_int, tcg_shift, tcg_fpstatus);
9038 } else {
9039 gen_helper_vfp_uqtos(tcg_single, tcg_int, tcg_shift, tcg_fpstatus);
9040 }
9041 write_fp_sreg_merging(s, rd, rd, tcg_single);
9042 break;
9043
9044 case MO_16:
9045 tcg_single = tcg_temp_new_i32();
9046 if (is_signed) {
9047 gen_helper_vfp_sqtoh(tcg_single, tcg_int, tcg_shift, tcg_fpstatus);
9048 } else {
9049 gen_helper_vfp_uqtoh(tcg_single, tcg_int, tcg_shift, tcg_fpstatus);
9050 }
9051 write_fp_hreg_merging(s, rd, rd, tcg_single);
9052 break;
9053
9054 default:
9055 g_assert_not_reached();
9056 }
9057 return true;
9058 }
9059
do_cvtf_g(DisasContext * s,arg_fcvt * a,bool is_signed)9060 static bool do_cvtf_g(DisasContext *s, arg_fcvt *a, bool is_signed)
9061 {
9062 TCGv_i64 tcg_int;
9063 int check = fp_access_check_scalar_hsd(s, a->esz);
9064
9065 if (check <= 0) {
9066 return check == 0;
9067 }
9068
9069 if (a->sf) {
9070 tcg_int = cpu_reg(s, a->rn);
9071 } else {
9072 tcg_int = read_cpu_reg(s, a->rn, true);
9073 if (is_signed) {
9074 tcg_gen_ext32s_i64(tcg_int, tcg_int);
9075 } else {
9076 tcg_gen_ext32u_i64(tcg_int, tcg_int);
9077 }
9078 }
9079 return do_cvtf_scalar(s, a->esz, a->rd, a->shift, tcg_int, is_signed);
9080 }
9081
TRANS(SCVTF_g,do_cvtf_g,a,true)9082 TRANS(SCVTF_g, do_cvtf_g, a, true)
9083 TRANS(UCVTF_g, do_cvtf_g, a, false)
9084
9085 /*
9086 * [US]CVTF (vector), scalar version.
9087 * Which sounds weird, but really just means input from fp register
9088 * instead of input from general register. Input and output element
9089 * size are always equal.
9090 */
9091 static bool do_cvtf_f(DisasContext *s, arg_fcvt *a, bool is_signed)
9092 {
9093 TCGv_i64 tcg_int;
9094 int check = fp_access_check_scalar_hsd(s, a->esz);
9095
9096 if (check <= 0) {
9097 return check == 0;
9098 }
9099
9100 tcg_int = tcg_temp_new_i64();
9101 read_vec_element(s, tcg_int, a->rn, 0, a->esz | (is_signed ? MO_SIGN : 0));
9102 return do_cvtf_scalar(s, a->esz, a->rd, a->shift, tcg_int, is_signed);
9103 }
9104
TRANS(SCVTF_f,do_cvtf_f,a,true)9105 TRANS(SCVTF_f, do_cvtf_f, a, true)
9106 TRANS(UCVTF_f, do_cvtf_f, a, false)
9107
9108 static void do_fcvt_scalar(DisasContext *s, MemOp out, MemOp esz,
9109 TCGv_i64 tcg_out, int shift, int rn,
9110 ARMFPRounding rmode)
9111 {
9112 TCGv_ptr tcg_fpstatus;
9113 TCGv_i32 tcg_shift, tcg_rmode, tcg_single;
9114
9115 tcg_fpstatus = fpstatus_ptr(esz == MO_16 ? FPST_A64_F16 : FPST_A64);
9116 tcg_shift = tcg_constant_i32(shift);
9117 tcg_rmode = gen_set_rmode(rmode, tcg_fpstatus);
9118
9119 switch (esz) {
9120 case MO_64:
9121 read_vec_element(s, tcg_out, rn, 0, MO_64);
9122 switch (out) {
9123 case MO_64 | MO_SIGN:
9124 gen_helper_vfp_tosqd(tcg_out, tcg_out, tcg_shift, tcg_fpstatus);
9125 break;
9126 case MO_64:
9127 gen_helper_vfp_touqd(tcg_out, tcg_out, tcg_shift, tcg_fpstatus);
9128 break;
9129 case MO_32 | MO_SIGN:
9130 gen_helper_vfp_tosld(tcg_out, tcg_out, tcg_shift, tcg_fpstatus);
9131 break;
9132 case MO_32:
9133 gen_helper_vfp_tould(tcg_out, tcg_out, tcg_shift, tcg_fpstatus);
9134 break;
9135 default:
9136 g_assert_not_reached();
9137 }
9138 break;
9139
9140 case MO_32:
9141 tcg_single = read_fp_sreg(s, rn);
9142 switch (out) {
9143 case MO_64 | MO_SIGN:
9144 gen_helper_vfp_tosqs(tcg_out, tcg_single, tcg_shift, tcg_fpstatus);
9145 break;
9146 case MO_64:
9147 gen_helper_vfp_touqs(tcg_out, tcg_single, tcg_shift, tcg_fpstatus);
9148 break;
9149 case MO_32 | MO_SIGN:
9150 gen_helper_vfp_tosls(tcg_single, tcg_single,
9151 tcg_shift, tcg_fpstatus);
9152 tcg_gen_extu_i32_i64(tcg_out, tcg_single);
9153 break;
9154 case MO_32:
9155 gen_helper_vfp_touls(tcg_single, tcg_single,
9156 tcg_shift, tcg_fpstatus);
9157 tcg_gen_extu_i32_i64(tcg_out, tcg_single);
9158 break;
9159 default:
9160 g_assert_not_reached();
9161 }
9162 break;
9163
9164 case MO_16:
9165 tcg_single = read_fp_hreg(s, rn);
9166 switch (out) {
9167 case MO_64 | MO_SIGN:
9168 gen_helper_vfp_tosqh(tcg_out, tcg_single, tcg_shift, tcg_fpstatus);
9169 break;
9170 case MO_64:
9171 gen_helper_vfp_touqh(tcg_out, tcg_single, tcg_shift, tcg_fpstatus);
9172 break;
9173 case MO_32 | MO_SIGN:
9174 gen_helper_vfp_toslh(tcg_single, tcg_single,
9175 tcg_shift, tcg_fpstatus);
9176 tcg_gen_extu_i32_i64(tcg_out, tcg_single);
9177 break;
9178 case MO_32:
9179 gen_helper_vfp_toulh(tcg_single, tcg_single,
9180 tcg_shift, tcg_fpstatus);
9181 tcg_gen_extu_i32_i64(tcg_out, tcg_single);
9182 break;
9183 case MO_16 | MO_SIGN:
9184 gen_helper_vfp_toshh(tcg_single, tcg_single,
9185 tcg_shift, tcg_fpstatus);
9186 tcg_gen_extu_i32_i64(tcg_out, tcg_single);
9187 break;
9188 case MO_16:
9189 gen_helper_vfp_touhh(tcg_single, tcg_single,
9190 tcg_shift, tcg_fpstatus);
9191 tcg_gen_extu_i32_i64(tcg_out, tcg_single);
9192 break;
9193 default:
9194 g_assert_not_reached();
9195 }
9196 break;
9197
9198 default:
9199 g_assert_not_reached();
9200 }
9201
9202 gen_restore_rmode(tcg_rmode, tcg_fpstatus);
9203 }
9204
do_fcvt_g(DisasContext * s,arg_fcvt * a,ARMFPRounding rmode,bool is_signed)9205 static bool do_fcvt_g(DisasContext *s, arg_fcvt *a,
9206 ARMFPRounding rmode, bool is_signed)
9207 {
9208 TCGv_i64 tcg_int;
9209 int check = fp_access_check_scalar_hsd(s, a->esz);
9210
9211 if (check <= 0) {
9212 return check == 0;
9213 }
9214
9215 tcg_int = cpu_reg(s, a->rd);
9216 do_fcvt_scalar(s, (a->sf ? MO_64 : MO_32) | (is_signed ? MO_SIGN : 0),
9217 a->esz, tcg_int, a->shift, a->rn, rmode);
9218
9219 if (!a->sf) {
9220 tcg_gen_ext32u_i64(tcg_int, tcg_int);
9221 }
9222 return true;
9223 }
9224
TRANS(FCVTNS_g,do_fcvt_g,a,FPROUNDING_TIEEVEN,true)9225 TRANS(FCVTNS_g, do_fcvt_g, a, FPROUNDING_TIEEVEN, true)
9226 TRANS(FCVTNU_g, do_fcvt_g, a, FPROUNDING_TIEEVEN, false)
9227 TRANS(FCVTPS_g, do_fcvt_g, a, FPROUNDING_POSINF, true)
9228 TRANS(FCVTPU_g, do_fcvt_g, a, FPROUNDING_POSINF, false)
9229 TRANS(FCVTMS_g, do_fcvt_g, a, FPROUNDING_NEGINF, true)
9230 TRANS(FCVTMU_g, do_fcvt_g, a, FPROUNDING_NEGINF, false)
9231 TRANS(FCVTZS_g, do_fcvt_g, a, FPROUNDING_ZERO, true)
9232 TRANS(FCVTZU_g, do_fcvt_g, a, FPROUNDING_ZERO, false)
9233 TRANS(FCVTAS_g, do_fcvt_g, a, FPROUNDING_TIEAWAY, true)
9234 TRANS(FCVTAU_g, do_fcvt_g, a, FPROUNDING_TIEAWAY, false)
9235
9236 /*
9237 * FCVT* (vector), scalar version.
9238 * Which sounds weird, but really just means output to fp register
9239 * instead of output to general register. Input and output element
9240 * size are always equal.
9241 */
9242 static bool do_fcvt_f(DisasContext *s, arg_fcvt *a,
9243 ARMFPRounding rmode, bool is_signed)
9244 {
9245 TCGv_i64 tcg_int;
9246 int check = fp_access_check_scalar_hsd(s, a->esz);
9247
9248 if (check <= 0) {
9249 return check == 0;
9250 }
9251
9252 tcg_int = tcg_temp_new_i64();
9253 do_fcvt_scalar(s, a->esz | (is_signed ? MO_SIGN : 0),
9254 a->esz, tcg_int, a->shift, a->rn, rmode);
9255
9256 if (!s->fpcr_nep) {
9257 clear_vec(s, a->rd);
9258 }
9259 write_vec_element(s, tcg_int, a->rd, 0, a->esz);
9260 return true;
9261 }
9262
TRANS(FCVTNS_f,do_fcvt_f,a,FPROUNDING_TIEEVEN,true)9263 TRANS(FCVTNS_f, do_fcvt_f, a, FPROUNDING_TIEEVEN, true)
9264 TRANS(FCVTNU_f, do_fcvt_f, a, FPROUNDING_TIEEVEN, false)
9265 TRANS(FCVTPS_f, do_fcvt_f, a, FPROUNDING_POSINF, true)
9266 TRANS(FCVTPU_f, do_fcvt_f, a, FPROUNDING_POSINF, false)
9267 TRANS(FCVTMS_f, do_fcvt_f, a, FPROUNDING_NEGINF, true)
9268 TRANS(FCVTMU_f, do_fcvt_f, a, FPROUNDING_NEGINF, false)
9269 TRANS(FCVTZS_f, do_fcvt_f, a, FPROUNDING_ZERO, true)
9270 TRANS(FCVTZU_f, do_fcvt_f, a, FPROUNDING_ZERO, false)
9271 TRANS(FCVTAS_f, do_fcvt_f, a, FPROUNDING_TIEAWAY, true)
9272 TRANS(FCVTAU_f, do_fcvt_f, a, FPROUNDING_TIEAWAY, false)
9273
9274 static bool trans_FJCVTZS(DisasContext *s, arg_FJCVTZS *a)
9275 {
9276 if (!dc_isar_feature(aa64_jscvt, s)) {
9277 return false;
9278 }
9279 if (fp_access_check(s)) {
9280 TCGv_i64 t = read_fp_dreg(s, a->rn);
9281 TCGv_ptr fpstatus = fpstatus_ptr(FPST_A64);
9282
9283 gen_helper_fjcvtzs(t, t, fpstatus);
9284
9285 tcg_gen_ext32u_i64(cpu_reg(s, a->rd), t);
9286 tcg_gen_extrh_i64_i32(cpu_ZF, t);
9287 tcg_gen_movi_i32(cpu_CF, 0);
9288 tcg_gen_movi_i32(cpu_NF, 0);
9289 tcg_gen_movi_i32(cpu_VF, 0);
9290 }
9291 return true;
9292 }
9293
trans_FMOV_hx(DisasContext * s,arg_rr * a)9294 static bool trans_FMOV_hx(DisasContext *s, arg_rr *a)
9295 {
9296 if (!dc_isar_feature(aa64_fp16, s)) {
9297 return false;
9298 }
9299 if (fp_access_check(s)) {
9300 TCGv_i64 tcg_rn = cpu_reg(s, a->rn);
9301 TCGv_i64 tmp = tcg_temp_new_i64();
9302 tcg_gen_ext16u_i64(tmp, tcg_rn);
9303 write_fp_dreg(s, a->rd, tmp);
9304 }
9305 return true;
9306 }
9307
trans_FMOV_sw(DisasContext * s,arg_rr * a)9308 static bool trans_FMOV_sw(DisasContext *s, arg_rr *a)
9309 {
9310 if (fp_access_check(s)) {
9311 TCGv_i64 tcg_rn = cpu_reg(s, a->rn);
9312 TCGv_i64 tmp = tcg_temp_new_i64();
9313 tcg_gen_ext32u_i64(tmp, tcg_rn);
9314 write_fp_dreg(s, a->rd, tmp);
9315 }
9316 return true;
9317 }
9318
trans_FMOV_dx(DisasContext * s,arg_rr * a)9319 static bool trans_FMOV_dx(DisasContext *s, arg_rr *a)
9320 {
9321 if (fp_access_check(s)) {
9322 TCGv_i64 tcg_rn = cpu_reg(s, a->rn);
9323 write_fp_dreg(s, a->rd, tcg_rn);
9324 }
9325 return true;
9326 }
9327
trans_FMOV_ux(DisasContext * s,arg_rr * a)9328 static bool trans_FMOV_ux(DisasContext *s, arg_rr *a)
9329 {
9330 if (fp_access_check(s)) {
9331 TCGv_i64 tcg_rn = cpu_reg(s, a->rn);
9332 tcg_gen_st_i64(tcg_rn, tcg_env, fp_reg_hi_offset(s, a->rd));
9333 clear_vec_high(s, true, a->rd);
9334 }
9335 return true;
9336 }
9337
trans_FMOV_xh(DisasContext * s,arg_rr * a)9338 static bool trans_FMOV_xh(DisasContext *s, arg_rr *a)
9339 {
9340 if (!dc_isar_feature(aa64_fp16, s)) {
9341 return false;
9342 }
9343 if (fp_access_check(s)) {
9344 TCGv_i64 tcg_rd = cpu_reg(s, a->rd);
9345 tcg_gen_ld16u_i64(tcg_rd, tcg_env, fp_reg_offset(s, a->rn, MO_16));
9346 }
9347 return true;
9348 }
9349
trans_FMOV_ws(DisasContext * s,arg_rr * a)9350 static bool trans_FMOV_ws(DisasContext *s, arg_rr *a)
9351 {
9352 if (fp_access_check(s)) {
9353 TCGv_i64 tcg_rd = cpu_reg(s, a->rd);
9354 tcg_gen_ld32u_i64(tcg_rd, tcg_env, fp_reg_offset(s, a->rn, MO_32));
9355 }
9356 return true;
9357 }
9358
trans_FMOV_xd(DisasContext * s,arg_rr * a)9359 static bool trans_FMOV_xd(DisasContext *s, arg_rr *a)
9360 {
9361 if (fp_access_check(s)) {
9362 TCGv_i64 tcg_rd = cpu_reg(s, a->rd);
9363 tcg_gen_ld_i64(tcg_rd, tcg_env, fp_reg_offset(s, a->rn, MO_64));
9364 }
9365 return true;
9366 }
9367
trans_FMOV_xu(DisasContext * s,arg_rr * a)9368 static bool trans_FMOV_xu(DisasContext *s, arg_rr *a)
9369 {
9370 if (fp_access_check(s)) {
9371 TCGv_i64 tcg_rd = cpu_reg(s, a->rd);
9372 tcg_gen_ld_i64(tcg_rd, tcg_env, fp_reg_hi_offset(s, a->rn));
9373 }
9374 return true;
9375 }
9376
9377 typedef struct ENVScalar1 {
9378 NeonGenOneOpEnvFn *gen_bhs[3];
9379 NeonGenOne64OpEnvFn *gen_d;
9380 } ENVScalar1;
9381
do_env_scalar1(DisasContext * s,arg_rr_e * a,const ENVScalar1 * f)9382 static bool do_env_scalar1(DisasContext *s, arg_rr_e *a, const ENVScalar1 *f)
9383 {
9384 if (!fp_access_check(s)) {
9385 return true;
9386 }
9387 if (a->esz == MO_64) {
9388 TCGv_i64 t = read_fp_dreg(s, a->rn);
9389 f->gen_d(t, tcg_env, t);
9390 write_fp_dreg(s, a->rd, t);
9391 } else {
9392 TCGv_i32 t = tcg_temp_new_i32();
9393
9394 read_vec_element_i32(s, t, a->rn, 0, a->esz);
9395 f->gen_bhs[a->esz](t, tcg_env, t);
9396 write_fp_sreg(s, a->rd, t);
9397 }
9398 return true;
9399 }
9400
do_env_vector1(DisasContext * s,arg_qrr_e * a,const ENVScalar1 * f)9401 static bool do_env_vector1(DisasContext *s, arg_qrr_e *a, const ENVScalar1 *f)
9402 {
9403 if (a->esz == MO_64 && !a->q) {
9404 return false;
9405 }
9406 if (!fp_access_check(s)) {
9407 return true;
9408 }
9409 if (a->esz == MO_64) {
9410 TCGv_i64 t = tcg_temp_new_i64();
9411
9412 for (int i = 0; i < 2; ++i) {
9413 read_vec_element(s, t, a->rn, i, MO_64);
9414 f->gen_d(t, tcg_env, t);
9415 write_vec_element(s, t, a->rd, i, MO_64);
9416 }
9417 } else {
9418 TCGv_i32 t = tcg_temp_new_i32();
9419 int n = (a->q ? 16 : 8) >> a->esz;
9420
9421 for (int i = 0; i < n; ++i) {
9422 read_vec_element_i32(s, t, a->rn, i, a->esz);
9423 f->gen_bhs[a->esz](t, tcg_env, t);
9424 write_vec_element_i32(s, t, a->rd, i, a->esz);
9425 }
9426 }
9427 clear_vec_high(s, a->q, a->rd);
9428 return true;
9429 }
9430
9431 static const ENVScalar1 f_scalar_sqabs = {
9432 { gen_helper_neon_qabs_s8,
9433 gen_helper_neon_qabs_s16,
9434 gen_helper_neon_qabs_s32 },
9435 gen_helper_neon_qabs_s64,
9436 };
9437 TRANS(SQABS_s, do_env_scalar1, a, &f_scalar_sqabs)
9438 TRANS(SQABS_v, do_env_vector1, a, &f_scalar_sqabs)
9439
9440 static const ENVScalar1 f_scalar_sqneg = {
9441 { gen_helper_neon_qneg_s8,
9442 gen_helper_neon_qneg_s16,
9443 gen_helper_neon_qneg_s32 },
9444 gen_helper_neon_qneg_s64,
9445 };
9446 TRANS(SQNEG_s, do_env_scalar1, a, &f_scalar_sqneg)
9447 TRANS(SQNEG_v, do_env_vector1, a, &f_scalar_sqneg)
9448
do_scalar1_d(DisasContext * s,arg_rr * a,ArithOneOp * f)9449 static bool do_scalar1_d(DisasContext *s, arg_rr *a, ArithOneOp *f)
9450 {
9451 if (fp_access_check(s)) {
9452 TCGv_i64 t = read_fp_dreg(s, a->rn);
9453 f(t, t);
9454 write_fp_dreg(s, a->rd, t);
9455 }
9456 return true;
9457 }
9458
TRANS(ABS_s,do_scalar1_d,a,tcg_gen_abs_i64)9459 TRANS(ABS_s, do_scalar1_d, a, tcg_gen_abs_i64)
9460 TRANS(NEG_s, do_scalar1_d, a, tcg_gen_neg_i64)
9461
9462 static bool do_cmop0_d(DisasContext *s, arg_rr *a, TCGCond cond)
9463 {
9464 if (fp_access_check(s)) {
9465 TCGv_i64 t = read_fp_dreg(s, a->rn);
9466 tcg_gen_negsetcond_i64(cond, t, t, tcg_constant_i64(0));
9467 write_fp_dreg(s, a->rd, t);
9468 }
9469 return true;
9470 }
9471
TRANS(CMGT0_s,do_cmop0_d,a,TCG_COND_GT)9472 TRANS(CMGT0_s, do_cmop0_d, a, TCG_COND_GT)
9473 TRANS(CMGE0_s, do_cmop0_d, a, TCG_COND_GE)
9474 TRANS(CMLE0_s, do_cmop0_d, a, TCG_COND_LE)
9475 TRANS(CMLT0_s, do_cmop0_d, a, TCG_COND_LT)
9476 TRANS(CMEQ0_s, do_cmop0_d, a, TCG_COND_EQ)
9477
9478 static bool do_2misc_narrow_scalar(DisasContext *s, arg_rr_e *a,
9479 ArithOneOp * const fn[3])
9480 {
9481 if (a->esz == MO_64) {
9482 return false;
9483 }
9484 if (fp_access_check(s)) {
9485 TCGv_i64 t = tcg_temp_new_i64();
9486
9487 read_vec_element(s, t, a->rn, 0, a->esz + 1);
9488 fn[a->esz](t, t);
9489 clear_vec(s, a->rd);
9490 write_vec_element(s, t, a->rd, 0, a->esz);
9491 }
9492 return true;
9493 }
9494
9495 #define WRAP_ENV(NAME) \
9496 static void gen_##NAME(TCGv_i64 d, TCGv_i64 n) \
9497 { gen_helper_##NAME(d, tcg_env, n); }
9498
9499 WRAP_ENV(neon_unarrow_sat8)
9500 WRAP_ENV(neon_unarrow_sat16)
9501 WRAP_ENV(neon_unarrow_sat32)
9502
9503 static ArithOneOp * const f_scalar_sqxtun[] = {
9504 gen_neon_unarrow_sat8,
9505 gen_neon_unarrow_sat16,
9506 gen_neon_unarrow_sat32,
9507 };
9508 TRANS(SQXTUN_s, do_2misc_narrow_scalar, a, f_scalar_sqxtun)
9509
9510 WRAP_ENV(neon_narrow_sat_s8)
9511 WRAP_ENV(neon_narrow_sat_s16)
9512 WRAP_ENV(neon_narrow_sat_s32)
9513
9514 static ArithOneOp * const f_scalar_sqxtn[] = {
9515 gen_neon_narrow_sat_s8,
9516 gen_neon_narrow_sat_s16,
9517 gen_neon_narrow_sat_s32,
9518 };
9519 TRANS(SQXTN_s, do_2misc_narrow_scalar, a, f_scalar_sqxtn)
9520
9521 WRAP_ENV(neon_narrow_sat_u8)
9522 WRAP_ENV(neon_narrow_sat_u16)
9523 WRAP_ENV(neon_narrow_sat_u32)
9524
9525 static ArithOneOp * const f_scalar_uqxtn[] = {
9526 gen_neon_narrow_sat_u8,
9527 gen_neon_narrow_sat_u16,
9528 gen_neon_narrow_sat_u32,
9529 };
TRANS(UQXTN_s,do_2misc_narrow_scalar,a,f_scalar_uqxtn)9530 TRANS(UQXTN_s, do_2misc_narrow_scalar, a, f_scalar_uqxtn)
9531
9532 static bool trans_FCVTXN_s(DisasContext *s, arg_rr_e *a)
9533 {
9534 if (fp_access_check(s)) {
9535 /*
9536 * 64 bit to 32 bit float conversion
9537 * with von Neumann rounding (round to odd)
9538 */
9539 TCGv_i64 src = read_fp_dreg(s, a->rn);
9540 TCGv_i32 dst = tcg_temp_new_i32();
9541 gen_helper_fcvtx_f64_to_f32(dst, src, fpstatus_ptr(FPST_A64));
9542 write_fp_sreg_merging(s, a->rd, a->rd, dst);
9543 }
9544 return true;
9545 }
9546
9547 #undef WRAP_ENV
9548
do_gvec_fn2(DisasContext * s,arg_qrr_e * a,GVecGen2Fn * fn)9549 static bool do_gvec_fn2(DisasContext *s, arg_qrr_e *a, GVecGen2Fn *fn)
9550 {
9551 if (!a->q && a->esz == MO_64) {
9552 return false;
9553 }
9554 if (fp_access_check(s)) {
9555 gen_gvec_fn2(s, a->q, a->rd, a->rn, fn, a->esz);
9556 }
9557 return true;
9558 }
9559
TRANS(ABS_v,do_gvec_fn2,a,tcg_gen_gvec_abs)9560 TRANS(ABS_v, do_gvec_fn2, a, tcg_gen_gvec_abs)
9561 TRANS(NEG_v, do_gvec_fn2, a, tcg_gen_gvec_neg)
9562 TRANS(NOT_v, do_gvec_fn2, a, tcg_gen_gvec_not)
9563 TRANS(CNT_v, do_gvec_fn2, a, gen_gvec_cnt)
9564 TRANS(RBIT_v, do_gvec_fn2, a, gen_gvec_rbit)
9565 TRANS(CMGT0_v, do_gvec_fn2, a, gen_gvec_cgt0)
9566 TRANS(CMGE0_v, do_gvec_fn2, a, gen_gvec_cge0)
9567 TRANS(CMLT0_v, do_gvec_fn2, a, gen_gvec_clt0)
9568 TRANS(CMLE0_v, do_gvec_fn2, a, gen_gvec_cle0)
9569 TRANS(CMEQ0_v, do_gvec_fn2, a, gen_gvec_ceq0)
9570 TRANS(REV16_v, do_gvec_fn2, a, gen_gvec_rev16)
9571 TRANS(REV32_v, do_gvec_fn2, a, gen_gvec_rev32)
9572 TRANS(URECPE_v, do_gvec_fn2, a, gen_gvec_urecpe)
9573 TRANS(URSQRTE_v, do_gvec_fn2, a, gen_gvec_ursqrte)
9574
9575 static bool do_gvec_fn2_bhs(DisasContext *s, arg_qrr_e *a, GVecGen2Fn *fn)
9576 {
9577 if (a->esz == MO_64) {
9578 return false;
9579 }
9580 if (fp_access_check(s)) {
9581 gen_gvec_fn2(s, a->q, a->rd, a->rn, fn, a->esz);
9582 }
9583 return true;
9584 }
9585
TRANS(CLS_v,do_gvec_fn2_bhs,a,gen_gvec_cls)9586 TRANS(CLS_v, do_gvec_fn2_bhs, a, gen_gvec_cls)
9587 TRANS(CLZ_v, do_gvec_fn2_bhs, a, gen_gvec_clz)
9588 TRANS(REV64_v, do_gvec_fn2_bhs, a, gen_gvec_rev64)
9589 TRANS(SADDLP_v, do_gvec_fn2_bhs, a, gen_gvec_saddlp)
9590 TRANS(UADDLP_v, do_gvec_fn2_bhs, a, gen_gvec_uaddlp)
9591 TRANS(SADALP_v, do_gvec_fn2_bhs, a, gen_gvec_sadalp)
9592 TRANS(UADALP_v, do_gvec_fn2_bhs, a, gen_gvec_uadalp)
9593
9594 static bool do_2misc_narrow_vector(DisasContext *s, arg_qrr_e *a,
9595 ArithOneOp * const fn[3])
9596 {
9597 if (a->esz == MO_64) {
9598 return false;
9599 }
9600 if (fp_access_check(s)) {
9601 TCGv_i64 t0 = tcg_temp_new_i64();
9602 TCGv_i64 t1 = tcg_temp_new_i64();
9603
9604 read_vec_element(s, t0, a->rn, 0, MO_64);
9605 read_vec_element(s, t1, a->rn, 1, MO_64);
9606 fn[a->esz](t0, t0);
9607 fn[a->esz](t1, t1);
9608 write_vec_element(s, t0, a->rd, a->q ? 2 : 0, MO_32);
9609 write_vec_element(s, t1, a->rd, a->q ? 3 : 1, MO_32);
9610 clear_vec_high(s, a->q, a->rd);
9611 }
9612 return true;
9613 }
9614
9615 static ArithOneOp * const f_scalar_xtn[] = {
9616 gen_helper_neon_narrow_u8,
9617 gen_helper_neon_narrow_u16,
9618 tcg_gen_ext32u_i64,
9619 };
TRANS(XTN,do_2misc_narrow_vector,a,f_scalar_xtn)9620 TRANS(XTN, do_2misc_narrow_vector, a, f_scalar_xtn)
9621 TRANS(SQXTUN_v, do_2misc_narrow_vector, a, f_scalar_sqxtun)
9622 TRANS(SQXTN_v, do_2misc_narrow_vector, a, f_scalar_sqxtn)
9623 TRANS(UQXTN_v, do_2misc_narrow_vector, a, f_scalar_uqxtn)
9624
9625 static void gen_fcvtn_hs(TCGv_i64 d, TCGv_i64 n)
9626 {
9627 TCGv_i32 tcg_lo = tcg_temp_new_i32();
9628 TCGv_i32 tcg_hi = tcg_temp_new_i32();
9629 TCGv_ptr fpst = fpstatus_ptr(FPST_A64);
9630 TCGv_i32 ahp = get_ahp_flag();
9631
9632 tcg_gen_extr_i64_i32(tcg_lo, tcg_hi, n);
9633 gen_helper_vfp_fcvt_f32_to_f16(tcg_lo, tcg_lo, fpst, ahp);
9634 gen_helper_vfp_fcvt_f32_to_f16(tcg_hi, tcg_hi, fpst, ahp);
9635 tcg_gen_deposit_i32(tcg_lo, tcg_lo, tcg_hi, 16, 16);
9636 tcg_gen_extu_i32_i64(d, tcg_lo);
9637 }
9638
gen_fcvtn_sd(TCGv_i64 d,TCGv_i64 n)9639 static void gen_fcvtn_sd(TCGv_i64 d, TCGv_i64 n)
9640 {
9641 TCGv_i32 tmp = tcg_temp_new_i32();
9642 TCGv_ptr fpst = fpstatus_ptr(FPST_A64);
9643
9644 gen_helper_vfp_fcvtsd(tmp, n, fpst);
9645 tcg_gen_extu_i32_i64(d, tmp);
9646 }
9647
gen_fcvtxn_sd(TCGv_i64 d,TCGv_i64 n)9648 static void gen_fcvtxn_sd(TCGv_i64 d, TCGv_i64 n)
9649 {
9650 /*
9651 * 64 bit to 32 bit float conversion
9652 * with von Neumann rounding (round to odd)
9653 */
9654 TCGv_i32 tmp = tcg_temp_new_i32();
9655 gen_helper_fcvtx_f64_to_f32(tmp, n, fpstatus_ptr(FPST_A64));
9656 tcg_gen_extu_i32_i64(d, tmp);
9657 }
9658
9659 static ArithOneOp * const f_vector_fcvtn[] = {
9660 NULL,
9661 gen_fcvtn_hs,
9662 gen_fcvtn_sd,
9663 };
9664 static ArithOneOp * const f_scalar_fcvtxn[] = {
9665 NULL,
9666 NULL,
9667 gen_fcvtxn_sd,
9668 };
TRANS(FCVTN_v,do_2misc_narrow_vector,a,f_vector_fcvtn)9669 TRANS(FCVTN_v, do_2misc_narrow_vector, a, f_vector_fcvtn)
9670 TRANS(FCVTXN_v, do_2misc_narrow_vector, a, f_scalar_fcvtxn)
9671
9672 static void gen_bfcvtn_hs(TCGv_i64 d, TCGv_i64 n)
9673 {
9674 TCGv_ptr fpst = fpstatus_ptr(FPST_A64);
9675 TCGv_i32 tmp = tcg_temp_new_i32();
9676 gen_helper_bfcvt_pair(tmp, n, fpst);
9677 tcg_gen_extu_i32_i64(d, tmp);
9678 }
9679
gen_bfcvtn_ah_hs(TCGv_i64 d,TCGv_i64 n)9680 static void gen_bfcvtn_ah_hs(TCGv_i64 d, TCGv_i64 n)
9681 {
9682 TCGv_ptr fpst = fpstatus_ptr(FPST_AH);
9683 TCGv_i32 tmp = tcg_temp_new_i32();
9684 gen_helper_bfcvt_pair(tmp, n, fpst);
9685 tcg_gen_extu_i32_i64(d, tmp);
9686 }
9687
9688 static ArithOneOp * const f_vector_bfcvtn[2][3] = {
9689 {
9690 NULL,
9691 gen_bfcvtn_hs,
9692 NULL,
9693 }, {
9694 NULL,
9695 gen_bfcvtn_ah_hs,
9696 NULL,
9697 }
9698 };
TRANS_FEAT(BFCVTN_v,aa64_bf16,do_2misc_narrow_vector,a,f_vector_bfcvtn[s->fpcr_ah])9699 TRANS_FEAT(BFCVTN_v, aa64_bf16, do_2misc_narrow_vector, a,
9700 f_vector_bfcvtn[s->fpcr_ah])
9701
9702 static bool trans_SHLL_v(DisasContext *s, arg_qrr_e *a)
9703 {
9704 static NeonGenWidenFn * const widenfns[3] = {
9705 gen_helper_neon_widen_u8,
9706 gen_helper_neon_widen_u16,
9707 tcg_gen_extu_i32_i64,
9708 };
9709 NeonGenWidenFn *widenfn;
9710 TCGv_i64 tcg_res[2];
9711 TCGv_i32 tcg_op;
9712 int part, pass;
9713
9714 if (a->esz == MO_64) {
9715 return false;
9716 }
9717 if (!fp_access_check(s)) {
9718 return true;
9719 }
9720
9721 tcg_op = tcg_temp_new_i32();
9722 widenfn = widenfns[a->esz];
9723 part = a->q ? 2 : 0;
9724
9725 for (pass = 0; pass < 2; pass++) {
9726 read_vec_element_i32(s, tcg_op, a->rn, part + pass, MO_32);
9727 tcg_res[pass] = tcg_temp_new_i64();
9728 widenfn(tcg_res[pass], tcg_op);
9729 tcg_gen_shli_i64(tcg_res[pass], tcg_res[pass], 8 << a->esz);
9730 }
9731
9732 for (pass = 0; pass < 2; pass++) {
9733 write_vec_element(s, tcg_res[pass], a->rd, pass, MO_64);
9734 }
9735 return true;
9736 }
9737
do_fabs_fneg_v(DisasContext * s,arg_qrr_e * a,GVecGen2Fn * fn)9738 static bool do_fabs_fneg_v(DisasContext *s, arg_qrr_e *a, GVecGen2Fn *fn)
9739 {
9740 int check = fp_access_check_vector_hsd(s, a->q, a->esz);
9741
9742 if (check <= 0) {
9743 return check == 0;
9744 }
9745
9746 gen_gvec_fn2(s, a->q, a->rd, a->rn, fn, a->esz);
9747 return true;
9748 }
9749
TRANS(FABS_v,do_fabs_fneg_v,a,gen_gvec_fabs)9750 TRANS(FABS_v, do_fabs_fneg_v, a, gen_gvec_fabs)
9751 TRANS(FNEG_v, do_fabs_fneg_v, a, gen_gvec_fneg)
9752
9753 static bool do_fp1_vector(DisasContext *s, arg_qrr_e *a,
9754 const FPScalar1 *f, int rmode)
9755 {
9756 TCGv_i32 tcg_rmode = NULL;
9757 TCGv_ptr fpst;
9758 int check = fp_access_check_vector_hsd(s, a->q, a->esz);
9759
9760 if (check <= 0) {
9761 return check == 0;
9762 }
9763
9764 fpst = fpstatus_ptr(a->esz == MO_16 ? FPST_A64_F16 : FPST_A64);
9765 if (rmode >= 0) {
9766 tcg_rmode = gen_set_rmode(rmode, fpst);
9767 }
9768
9769 if (a->esz == MO_64) {
9770 TCGv_i64 t64 = tcg_temp_new_i64();
9771
9772 for (int pass = 0; pass < 2; ++pass) {
9773 read_vec_element(s, t64, a->rn, pass, MO_64);
9774 f->gen_d(t64, t64, fpst);
9775 write_vec_element(s, t64, a->rd, pass, MO_64);
9776 }
9777 } else {
9778 TCGv_i32 t32 = tcg_temp_new_i32();
9779 void (*gen)(TCGv_i32, TCGv_i32, TCGv_ptr)
9780 = (a->esz == MO_16 ? f->gen_h : f->gen_s);
9781
9782 for (int pass = 0, n = (a->q ? 16 : 8) >> a->esz; pass < n; ++pass) {
9783 read_vec_element_i32(s, t32, a->rn, pass, a->esz);
9784 gen(t32, t32, fpst);
9785 write_vec_element_i32(s, t32, a->rd, pass, a->esz);
9786 }
9787 }
9788 clear_vec_high(s, a->q, a->rd);
9789
9790 if (rmode >= 0) {
9791 gen_restore_rmode(tcg_rmode, fpst);
9792 }
9793 return true;
9794 }
9795
9796 TRANS(FSQRT_v, do_fp1_vector, a, &f_scalar_fsqrt, -1)
9797
9798 TRANS(FRINTN_v, do_fp1_vector, a, &f_scalar_frint, FPROUNDING_TIEEVEN)
9799 TRANS(FRINTP_v, do_fp1_vector, a, &f_scalar_frint, FPROUNDING_POSINF)
9800 TRANS(FRINTM_v, do_fp1_vector, a, &f_scalar_frint, FPROUNDING_NEGINF)
9801 TRANS(FRINTZ_v, do_fp1_vector, a, &f_scalar_frint, FPROUNDING_ZERO)
9802 TRANS(FRINTA_v, do_fp1_vector, a, &f_scalar_frint, FPROUNDING_TIEAWAY)
9803 TRANS(FRINTI_v, do_fp1_vector, a, &f_scalar_frint, -1)
9804 TRANS(FRINTX_v, do_fp1_vector, a, &f_scalar_frintx, -1)
9805
9806 TRANS_FEAT(FRINT32Z_v, aa64_frint, do_fp1_vector, a,
9807 &f_scalar_frint32, FPROUNDING_ZERO)
9808 TRANS_FEAT(FRINT32X_v, aa64_frint, do_fp1_vector, a, &f_scalar_frint32, -1)
9809 TRANS_FEAT(FRINT64Z_v, aa64_frint, do_fp1_vector, a,
9810 &f_scalar_frint64, FPROUNDING_ZERO)
9811 TRANS_FEAT(FRINT64X_v, aa64_frint, do_fp1_vector, a, &f_scalar_frint64, -1)
9812
do_gvec_op2_fpst_with_fpsttype(DisasContext * s,MemOp esz,bool is_q,int rd,int rn,int data,gen_helper_gvec_2_ptr * const fns[3],ARMFPStatusFlavour fpsttype)9813 static bool do_gvec_op2_fpst_with_fpsttype(DisasContext *s, MemOp esz,
9814 bool is_q, int rd, int rn, int data,
9815 gen_helper_gvec_2_ptr * const fns[3],
9816 ARMFPStatusFlavour fpsttype)
9817 {
9818 int check = fp_access_check_vector_hsd(s, is_q, esz);
9819 TCGv_ptr fpst;
9820
9821 if (check <= 0) {
9822 return check == 0;
9823 }
9824
9825 fpst = fpstatus_ptr(fpsttype);
9826 tcg_gen_gvec_2_ptr(vec_full_reg_offset(s, rd),
9827 vec_full_reg_offset(s, rn), fpst,
9828 is_q ? 16 : 8, vec_full_reg_size(s),
9829 data, fns[esz - 1]);
9830 return true;
9831 }
9832
do_gvec_op2_fpst(DisasContext * s,MemOp esz,bool is_q,int rd,int rn,int data,gen_helper_gvec_2_ptr * const fns[3])9833 static bool do_gvec_op2_fpst(DisasContext *s, MemOp esz, bool is_q,
9834 int rd, int rn, int data,
9835 gen_helper_gvec_2_ptr * const fns[3])
9836 {
9837 return do_gvec_op2_fpst_with_fpsttype(s, esz, is_q, rd, rn, data, fns,
9838 esz == MO_16 ? FPST_A64_F16 :
9839 FPST_A64);
9840 }
9841
do_gvec_op2_ah_fpst(DisasContext * s,MemOp esz,bool is_q,int rd,int rn,int data,gen_helper_gvec_2_ptr * const fns[3])9842 static bool do_gvec_op2_ah_fpst(DisasContext *s, MemOp esz, bool is_q,
9843 int rd, int rn, int data,
9844 gen_helper_gvec_2_ptr * const fns[3])
9845 {
9846 return do_gvec_op2_fpst_with_fpsttype(s, esz, is_q, rd, rn, data,
9847 fns, select_ah_fpst(s, esz));
9848 }
9849
9850 static gen_helper_gvec_2_ptr * const f_scvtf_v[] = {
9851 gen_helper_gvec_vcvt_sh,
9852 gen_helper_gvec_vcvt_sf,
9853 gen_helper_gvec_vcvt_sd,
9854 };
9855 TRANS(SCVTF_vi, do_gvec_op2_fpst,
9856 a->esz, a->q, a->rd, a->rn, 0, f_scvtf_v)
9857 TRANS(SCVTF_vf, do_gvec_op2_fpst,
9858 a->esz, a->q, a->rd, a->rn, a->shift, f_scvtf_v)
9859
9860 static gen_helper_gvec_2_ptr * const f_ucvtf_v[] = {
9861 gen_helper_gvec_vcvt_uh,
9862 gen_helper_gvec_vcvt_uf,
9863 gen_helper_gvec_vcvt_ud,
9864 };
9865 TRANS(UCVTF_vi, do_gvec_op2_fpst,
9866 a->esz, a->q, a->rd, a->rn, 0, f_ucvtf_v)
9867 TRANS(UCVTF_vf, do_gvec_op2_fpst,
9868 a->esz, a->q, a->rd, a->rn, a->shift, f_ucvtf_v)
9869
9870 static gen_helper_gvec_2_ptr * const f_fcvtzs_vf[] = {
9871 gen_helper_gvec_vcvt_rz_hs,
9872 gen_helper_gvec_vcvt_rz_fs,
9873 gen_helper_gvec_vcvt_rz_ds,
9874 };
9875 TRANS(FCVTZS_vf, do_gvec_op2_fpst,
9876 a->esz, a->q, a->rd, a->rn, a->shift, f_fcvtzs_vf)
9877
9878 static gen_helper_gvec_2_ptr * const f_fcvtzu_vf[] = {
9879 gen_helper_gvec_vcvt_rz_hu,
9880 gen_helper_gvec_vcvt_rz_fu,
9881 gen_helper_gvec_vcvt_rz_du,
9882 };
9883 TRANS(FCVTZU_vf, do_gvec_op2_fpst,
9884 a->esz, a->q, a->rd, a->rn, a->shift, f_fcvtzu_vf)
9885
9886 static gen_helper_gvec_2_ptr * const f_fcvt_s_vi[] = {
9887 gen_helper_gvec_vcvt_rm_sh,
9888 gen_helper_gvec_vcvt_rm_ss,
9889 gen_helper_gvec_vcvt_rm_sd,
9890 };
9891
9892 static gen_helper_gvec_2_ptr * const f_fcvt_u_vi[] = {
9893 gen_helper_gvec_vcvt_rm_uh,
9894 gen_helper_gvec_vcvt_rm_us,
9895 gen_helper_gvec_vcvt_rm_ud,
9896 };
9897
9898 TRANS(FCVTNS_vi, do_gvec_op2_fpst,
9899 a->esz, a->q, a->rd, a->rn, float_round_nearest_even, f_fcvt_s_vi)
9900 TRANS(FCVTNU_vi, do_gvec_op2_fpst,
9901 a->esz, a->q, a->rd, a->rn, float_round_nearest_even, f_fcvt_u_vi)
9902 TRANS(FCVTPS_vi, do_gvec_op2_fpst,
9903 a->esz, a->q, a->rd, a->rn, float_round_up, f_fcvt_s_vi)
9904 TRANS(FCVTPU_vi, do_gvec_op2_fpst,
9905 a->esz, a->q, a->rd, a->rn, float_round_up, f_fcvt_u_vi)
9906 TRANS(FCVTMS_vi, do_gvec_op2_fpst,
9907 a->esz, a->q, a->rd, a->rn, float_round_down, f_fcvt_s_vi)
9908 TRANS(FCVTMU_vi, do_gvec_op2_fpst,
9909 a->esz, a->q, a->rd, a->rn, float_round_down, f_fcvt_u_vi)
9910 TRANS(FCVTZS_vi, do_gvec_op2_fpst,
9911 a->esz, a->q, a->rd, a->rn, float_round_to_zero, f_fcvt_s_vi)
9912 TRANS(FCVTZU_vi, do_gvec_op2_fpst,
9913 a->esz, a->q, a->rd, a->rn, float_round_to_zero, f_fcvt_u_vi)
9914 TRANS(FCVTAS_vi, do_gvec_op2_fpst,
9915 a->esz, a->q, a->rd, a->rn, float_round_ties_away, f_fcvt_s_vi)
9916 TRANS(FCVTAU_vi, do_gvec_op2_fpst,
9917 a->esz, a->q, a->rd, a->rn, float_round_ties_away, f_fcvt_u_vi)
9918
9919 static gen_helper_gvec_2_ptr * const f_fceq0[] = {
9920 gen_helper_gvec_fceq0_h,
9921 gen_helper_gvec_fceq0_s,
9922 gen_helper_gvec_fceq0_d,
9923 };
9924 TRANS(FCMEQ0_v, do_gvec_op2_fpst, a->esz, a->q, a->rd, a->rn, 0, f_fceq0)
9925
9926 static gen_helper_gvec_2_ptr * const f_fcgt0[] = {
9927 gen_helper_gvec_fcgt0_h,
9928 gen_helper_gvec_fcgt0_s,
9929 gen_helper_gvec_fcgt0_d,
9930 };
9931 TRANS(FCMGT0_v, do_gvec_op2_fpst, a->esz, a->q, a->rd, a->rn, 0, f_fcgt0)
9932
9933 static gen_helper_gvec_2_ptr * const f_fcge0[] = {
9934 gen_helper_gvec_fcge0_h,
9935 gen_helper_gvec_fcge0_s,
9936 gen_helper_gvec_fcge0_d,
9937 };
9938 TRANS(FCMGE0_v, do_gvec_op2_fpst, a->esz, a->q, a->rd, a->rn, 0, f_fcge0)
9939
9940 static gen_helper_gvec_2_ptr * const f_fclt0[] = {
9941 gen_helper_gvec_fclt0_h,
9942 gen_helper_gvec_fclt0_s,
9943 gen_helper_gvec_fclt0_d,
9944 };
9945 TRANS(FCMLT0_v, do_gvec_op2_fpst, a->esz, a->q, a->rd, a->rn, 0, f_fclt0)
9946
9947 static gen_helper_gvec_2_ptr * const f_fcle0[] = {
9948 gen_helper_gvec_fcle0_h,
9949 gen_helper_gvec_fcle0_s,
9950 gen_helper_gvec_fcle0_d,
9951 };
9952 TRANS(FCMLE0_v, do_gvec_op2_fpst, a->esz, a->q, a->rd, a->rn, 0, f_fcle0)
9953
9954 static gen_helper_gvec_2_ptr * const f_frecpe[] = {
9955 gen_helper_gvec_frecpe_h,
9956 gen_helper_gvec_frecpe_s,
9957 gen_helper_gvec_frecpe_d,
9958 };
9959 static gen_helper_gvec_2_ptr * const f_frecpe_rpres[] = {
9960 gen_helper_gvec_frecpe_h,
9961 gen_helper_gvec_frecpe_rpres_s,
9962 gen_helper_gvec_frecpe_d,
9963 };
9964 TRANS(FRECPE_v, do_gvec_op2_ah_fpst, a->esz, a->q, a->rd, a->rn, 0,
9965 s->fpcr_ah && dc_isar_feature(aa64_rpres, s) ? f_frecpe_rpres : f_frecpe)
9966
9967 static gen_helper_gvec_2_ptr * const f_frsqrte[] = {
9968 gen_helper_gvec_frsqrte_h,
9969 gen_helper_gvec_frsqrte_s,
9970 gen_helper_gvec_frsqrte_d,
9971 };
9972 static gen_helper_gvec_2_ptr * const f_frsqrte_rpres[] = {
9973 gen_helper_gvec_frsqrte_h,
9974 gen_helper_gvec_frsqrte_rpres_s,
9975 gen_helper_gvec_frsqrte_d,
9976 };
9977 TRANS(FRSQRTE_v, do_gvec_op2_ah_fpst, a->esz, a->q, a->rd, a->rn, 0,
9978 s->fpcr_ah && dc_isar_feature(aa64_rpres, s) ? f_frsqrte_rpres : f_frsqrte)
9979
trans_FCVTL_v(DisasContext * s,arg_qrr_e * a)9980 static bool trans_FCVTL_v(DisasContext *s, arg_qrr_e *a)
9981 {
9982 /* Handle 2-reg-misc ops which are widening (so each size element
9983 * in the source becomes a 2*size element in the destination.
9984 * The only instruction like this is FCVTL.
9985 */
9986 int pass;
9987 TCGv_ptr fpst;
9988
9989 if (!fp_access_check(s)) {
9990 return true;
9991 }
9992
9993 if (a->esz == MO_64) {
9994 /* 32 -> 64 bit fp conversion */
9995 TCGv_i64 tcg_res[2];
9996 TCGv_i32 tcg_op = tcg_temp_new_i32();
9997 int srcelt = a->q ? 2 : 0;
9998
9999 fpst = fpstatus_ptr(FPST_A64);
10000
10001 for (pass = 0; pass < 2; pass++) {
10002 tcg_res[pass] = tcg_temp_new_i64();
10003 read_vec_element_i32(s, tcg_op, a->rn, srcelt + pass, MO_32);
10004 gen_helper_vfp_fcvtds(tcg_res[pass], tcg_op, fpst);
10005 }
10006 for (pass = 0; pass < 2; pass++) {
10007 write_vec_element(s, tcg_res[pass], a->rd, pass, MO_64);
10008 }
10009 } else {
10010 /* 16 -> 32 bit fp conversion */
10011 int srcelt = a->q ? 4 : 0;
10012 TCGv_i32 tcg_res[4];
10013 TCGv_i32 ahp = get_ahp_flag();
10014
10015 fpst = fpstatus_ptr(FPST_A64_F16);
10016
10017 for (pass = 0; pass < 4; pass++) {
10018 tcg_res[pass] = tcg_temp_new_i32();
10019 read_vec_element_i32(s, tcg_res[pass], a->rn, srcelt + pass, MO_16);
10020 gen_helper_vfp_fcvt_f16_to_f32(tcg_res[pass], tcg_res[pass],
10021 fpst, ahp);
10022 }
10023 for (pass = 0; pass < 4; pass++) {
10024 write_vec_element_i32(s, tcg_res[pass], a->rd, pass, MO_32);
10025 }
10026 }
10027 clear_vec_high(s, true, a->rd);
10028 return true;
10029 }
10030
trans_OK(DisasContext * s,arg_OK * a)10031 static bool trans_OK(DisasContext *s, arg_OK *a)
10032 {
10033 return true;
10034 }
10035
trans_FAIL(DisasContext * s,arg_OK * a)10036 static bool trans_FAIL(DisasContext *s, arg_OK *a)
10037 {
10038 s->is_nonstreaming = true;
10039 return true;
10040 }
10041
10042 /**
10043 * btype_destination_ok:
10044 * @insn: The instruction at the branch destination
10045 * @bt: SCTLR_ELx.BT
10046 * @btype: PSTATE.BTYPE, and is non-zero
10047 *
10048 * On a guarded page, there are a limited number of insns
10049 * that may be present at the branch target:
10050 * - branch target identifiers,
10051 * - paciasp, pacibsp,
10052 * - BRK insn
10053 * - HLT insn
10054 * Anything else causes a Branch Target Exception.
10055 *
10056 * Return true if the branch is compatible, false to raise BTITRAP.
10057 */
btype_destination_ok(uint32_t insn,bool bt,int btype)10058 static bool btype_destination_ok(uint32_t insn, bool bt, int btype)
10059 {
10060 if ((insn & 0xfffff01fu) == 0xd503201fu) {
10061 /* HINT space */
10062 switch (extract32(insn, 5, 7)) {
10063 case 0b011001: /* PACIASP */
10064 case 0b011011: /* PACIBSP */
10065 /*
10066 * If SCTLR_ELx.BT, then PACI*SP are not compatible
10067 * with btype == 3. Otherwise all btype are ok.
10068 */
10069 return !bt || btype != 3;
10070 case 0b100000: /* BTI */
10071 /* Not compatible with any btype. */
10072 return false;
10073 case 0b100010: /* BTI c */
10074 /* Not compatible with btype == 3 */
10075 return btype != 3;
10076 case 0b100100: /* BTI j */
10077 /* Not compatible with btype == 2 */
10078 return btype != 2;
10079 case 0b100110: /* BTI jc */
10080 /* Compatible with any btype. */
10081 return true;
10082 }
10083 } else {
10084 switch (insn & 0xffe0001fu) {
10085 case 0xd4200000u: /* BRK */
10086 case 0xd4400000u: /* HLT */
10087 /* Give priority to the breakpoint exception. */
10088 return true;
10089 }
10090 }
10091 return false;
10092 }
10093
aarch64_tr_init_disas_context(DisasContextBase * dcbase,CPUState * cpu)10094 static void aarch64_tr_init_disas_context(DisasContextBase *dcbase,
10095 CPUState *cpu)
10096 {
10097 DisasContext *dc = container_of(dcbase, DisasContext, base);
10098 CPUARMState *env = cpu_env(cpu);
10099 ARMCPU *arm_cpu = env_archcpu(env);
10100 CPUARMTBFlags tb_flags = arm_tbflags_from_tb(dc->base.tb);
10101 int bound, core_mmu_idx;
10102
10103 dc->isar = &arm_cpu->isar;
10104 dc->condjmp = 0;
10105 dc->pc_save = dc->base.pc_first;
10106 dc->aarch64 = true;
10107 dc->thumb = false;
10108 dc->sctlr_b = 0;
10109 dc->be_data = EX_TBFLAG_ANY(tb_flags, BE_DATA) ? MO_BE : MO_LE;
10110 dc->condexec_mask = 0;
10111 dc->condexec_cond = 0;
10112 core_mmu_idx = EX_TBFLAG_ANY(tb_flags, MMUIDX);
10113 dc->mmu_idx = core_to_aa64_mmu_idx(core_mmu_idx);
10114 dc->tbii = EX_TBFLAG_A64(tb_flags, TBII);
10115 dc->tbid = EX_TBFLAG_A64(tb_flags, TBID);
10116 dc->tcma = EX_TBFLAG_A64(tb_flags, TCMA);
10117 dc->current_el = arm_mmu_idx_to_el(dc->mmu_idx);
10118 #if !defined(CONFIG_USER_ONLY)
10119 dc->user = (dc->current_el == 0);
10120 #endif
10121 dc->fp_excp_el = EX_TBFLAG_ANY(tb_flags, FPEXC_EL);
10122 dc->align_mem = EX_TBFLAG_ANY(tb_flags, ALIGN_MEM);
10123 dc->pstate_il = EX_TBFLAG_ANY(tb_flags, PSTATE__IL);
10124 dc->fgt_active = EX_TBFLAG_ANY(tb_flags, FGT_ACTIVE);
10125 dc->fgt_svc = EX_TBFLAG_ANY(tb_flags, FGT_SVC);
10126 dc->trap_eret = EX_TBFLAG_A64(tb_flags, TRAP_ERET);
10127 dc->sve_excp_el = EX_TBFLAG_A64(tb_flags, SVEEXC_EL);
10128 dc->sme_excp_el = EX_TBFLAG_A64(tb_flags, SMEEXC_EL);
10129 dc->vl = (EX_TBFLAG_A64(tb_flags, VL) + 1) * 16;
10130 dc->svl = (EX_TBFLAG_A64(tb_flags, SVL) + 1) * 16;
10131 dc->pauth_active = EX_TBFLAG_A64(tb_flags, PAUTH_ACTIVE);
10132 dc->bt = EX_TBFLAG_A64(tb_flags, BT);
10133 dc->btype = EX_TBFLAG_A64(tb_flags, BTYPE);
10134 dc->unpriv = EX_TBFLAG_A64(tb_flags, UNPRIV);
10135 dc->ata[0] = EX_TBFLAG_A64(tb_flags, ATA);
10136 dc->ata[1] = EX_TBFLAG_A64(tb_flags, ATA0);
10137 dc->mte_active[0] = EX_TBFLAG_A64(tb_flags, MTE_ACTIVE);
10138 dc->mte_active[1] = EX_TBFLAG_A64(tb_flags, MTE0_ACTIVE);
10139 dc->pstate_sm = EX_TBFLAG_A64(tb_flags, PSTATE_SM);
10140 dc->pstate_za = EX_TBFLAG_A64(tb_flags, PSTATE_ZA);
10141 dc->sme_trap_nonstreaming = EX_TBFLAG_A64(tb_flags, SME_TRAP_NONSTREAMING);
10142 dc->naa = EX_TBFLAG_A64(tb_flags, NAA);
10143 dc->nv = EX_TBFLAG_A64(tb_flags, NV);
10144 dc->nv1 = EX_TBFLAG_A64(tb_flags, NV1);
10145 dc->nv2 = EX_TBFLAG_A64(tb_flags, NV2);
10146 dc->nv2_mem_e20 = EX_TBFLAG_A64(tb_flags, NV2_MEM_E20);
10147 dc->nv2_mem_be = EX_TBFLAG_A64(tb_flags, NV2_MEM_BE);
10148 dc->fpcr_ah = EX_TBFLAG_A64(tb_flags, AH);
10149 dc->fpcr_nep = EX_TBFLAG_A64(tb_flags, NEP);
10150 dc->vec_len = 0;
10151 dc->vec_stride = 0;
10152 dc->cp_regs = arm_cpu->cp_regs;
10153 dc->features = env->features;
10154 dc->dcz_blocksize = arm_cpu->dcz_blocksize;
10155 dc->gm_blocksize = arm_cpu->gm_blocksize;
10156
10157 #ifdef CONFIG_USER_ONLY
10158 /* In sve_probe_page, we assume TBI is enabled. */
10159 tcg_debug_assert(dc->tbid & 1);
10160 #endif
10161
10162 dc->lse2 = dc_isar_feature(aa64_lse2, dc);
10163
10164 /* Single step state. The code-generation logic here is:
10165 * SS_ACTIVE == 0:
10166 * generate code with no special handling for single-stepping (except
10167 * that anything that can make us go to SS_ACTIVE == 1 must end the TB;
10168 * this happens anyway because those changes are all system register or
10169 * PSTATE writes).
10170 * SS_ACTIVE == 1, PSTATE.SS == 1: (active-not-pending)
10171 * emit code for one insn
10172 * emit code to clear PSTATE.SS
10173 * emit code to generate software step exception for completed step
10174 * end TB (as usual for having generated an exception)
10175 * SS_ACTIVE == 1, PSTATE.SS == 0: (active-pending)
10176 * emit code to generate a software step exception
10177 * end the TB
10178 */
10179 dc->ss_active = EX_TBFLAG_ANY(tb_flags, SS_ACTIVE);
10180 dc->pstate_ss = EX_TBFLAG_ANY(tb_flags, PSTATE__SS);
10181 dc->is_ldex = false;
10182
10183 /* Bound the number of insns to execute to those left on the page. */
10184 bound = -(dc->base.pc_first | TARGET_PAGE_MASK) / 4;
10185
10186 /* If architectural single step active, limit to 1. */
10187 if (dc->ss_active) {
10188 bound = 1;
10189 }
10190 dc->base.max_insns = MIN(dc->base.max_insns, bound);
10191 }
10192
aarch64_tr_tb_start(DisasContextBase * db,CPUState * cpu)10193 static void aarch64_tr_tb_start(DisasContextBase *db, CPUState *cpu)
10194 {
10195 }
10196
aarch64_tr_insn_start(DisasContextBase * dcbase,CPUState * cpu)10197 static void aarch64_tr_insn_start(DisasContextBase *dcbase, CPUState *cpu)
10198 {
10199 DisasContext *dc = container_of(dcbase, DisasContext, base);
10200 target_ulong pc_arg = dc->base.pc_next;
10201
10202 if (tb_cflags(dcbase->tb) & CF_PCREL) {
10203 pc_arg &= ~TARGET_PAGE_MASK;
10204 }
10205 tcg_gen_insn_start(pc_arg, 0, 0);
10206 dc->insn_start_updated = false;
10207 }
10208
aarch64_tr_translate_insn(DisasContextBase * dcbase,CPUState * cpu)10209 static void aarch64_tr_translate_insn(DisasContextBase *dcbase, CPUState *cpu)
10210 {
10211 DisasContext *s = container_of(dcbase, DisasContext, base);
10212 CPUARMState *env = cpu_env(cpu);
10213 uint64_t pc = s->base.pc_next;
10214 uint32_t insn;
10215
10216 /* Singlestep exceptions have the highest priority. */
10217 if (s->ss_active && !s->pstate_ss) {
10218 /* Singlestep state is Active-pending.
10219 * If we're in this state at the start of a TB then either
10220 * a) we just took an exception to an EL which is being debugged
10221 * and this is the first insn in the exception handler
10222 * b) debug exceptions were masked and we just unmasked them
10223 * without changing EL (eg by clearing PSTATE.D)
10224 * In either case we're going to take a swstep exception in the
10225 * "did not step an insn" case, and so the syndrome ISV and EX
10226 * bits should be zero.
10227 */
10228 assert(s->base.num_insns == 1);
10229 gen_swstep_exception(s, 0, 0);
10230 s->base.is_jmp = DISAS_NORETURN;
10231 s->base.pc_next = pc + 4;
10232 return;
10233 }
10234
10235 if (pc & 3) {
10236 /*
10237 * PC alignment fault. This has priority over the instruction abort
10238 * that we would receive from a translation fault via arm_ldl_code.
10239 * This should only be possible after an indirect branch, at the
10240 * start of the TB.
10241 */
10242 assert(s->base.num_insns == 1);
10243 gen_helper_exception_pc_alignment(tcg_env, tcg_constant_vaddr(pc));
10244 s->base.is_jmp = DISAS_NORETURN;
10245 s->base.pc_next = QEMU_ALIGN_UP(pc, 4);
10246 return;
10247 }
10248
10249 s->pc_curr = pc;
10250 insn = arm_ldl_code(env, &s->base, pc, s->sctlr_b);
10251 s->insn = insn;
10252 s->base.pc_next = pc + 4;
10253
10254 s->fp_access_checked = 0;
10255 s->sve_access_checked = 0;
10256
10257 if (s->pstate_il) {
10258 /*
10259 * Illegal execution state. This has priority over BTI
10260 * exceptions, but comes after instruction abort exceptions.
10261 */
10262 gen_exception_insn(s, 0, EXCP_UDEF, syn_illegalstate());
10263 return;
10264 }
10265
10266 if (dc_isar_feature(aa64_bti, s)) {
10267 if (s->base.num_insns == 1) {
10268 /* First insn can have btype set to non-zero. */
10269 tcg_debug_assert(s->btype >= 0);
10270
10271 /*
10272 * Note that the Branch Target Exception has fairly high
10273 * priority -- below debugging exceptions but above most
10274 * everything else. This allows us to handle this now
10275 * instead of waiting until the insn is otherwise decoded.
10276 *
10277 * We can check all but the guarded page check here;
10278 * defer the latter to a helper.
10279 */
10280 if (s->btype != 0
10281 && !btype_destination_ok(insn, s->bt, s->btype)) {
10282 gen_helper_guarded_page_check(tcg_env);
10283 }
10284 } else {
10285 /* Not the first insn: btype must be 0. */
10286 tcg_debug_assert(s->btype == 0);
10287 }
10288 }
10289
10290 s->is_nonstreaming = false;
10291 if (s->sme_trap_nonstreaming) {
10292 disas_sme_fa64(s, insn);
10293 }
10294
10295 if (!disas_a64(s, insn) &&
10296 !disas_sme(s, insn) &&
10297 !disas_sve(s, insn)) {
10298 unallocated_encoding(s);
10299 }
10300
10301 /*
10302 * After execution of most insns, btype is reset to 0.
10303 * Note that we set btype == -1 when the insn sets btype.
10304 */
10305 if (s->btype > 0 && s->base.is_jmp != DISAS_NORETURN) {
10306 reset_btype(s);
10307 }
10308 }
10309
aarch64_tr_tb_stop(DisasContextBase * dcbase,CPUState * cpu)10310 static void aarch64_tr_tb_stop(DisasContextBase *dcbase, CPUState *cpu)
10311 {
10312 DisasContext *dc = container_of(dcbase, DisasContext, base);
10313
10314 if (unlikely(dc->ss_active)) {
10315 /* Note that this means single stepping WFI doesn't halt the CPU.
10316 * For conditional branch insns this is harmless unreachable code as
10317 * gen_goto_tb() has already handled emitting the debug exception
10318 * (and thus a tb-jump is not possible when singlestepping).
10319 */
10320 switch (dc->base.is_jmp) {
10321 default:
10322 gen_a64_update_pc(dc, 4);
10323 /* fall through */
10324 case DISAS_EXIT:
10325 case DISAS_JUMP:
10326 gen_step_complete_exception(dc);
10327 break;
10328 case DISAS_NORETURN:
10329 break;
10330 }
10331 } else {
10332 switch (dc->base.is_jmp) {
10333 case DISAS_NEXT:
10334 case DISAS_TOO_MANY:
10335 gen_goto_tb(dc, 1, 4);
10336 break;
10337 default:
10338 case DISAS_UPDATE_EXIT:
10339 gen_a64_update_pc(dc, 4);
10340 /* fall through */
10341 case DISAS_EXIT:
10342 tcg_gen_exit_tb(NULL, 0);
10343 break;
10344 case DISAS_UPDATE_NOCHAIN:
10345 gen_a64_update_pc(dc, 4);
10346 /* fall through */
10347 case DISAS_JUMP:
10348 tcg_gen_lookup_and_goto_ptr();
10349 break;
10350 case DISAS_NORETURN:
10351 case DISAS_SWI:
10352 break;
10353 case DISAS_WFE:
10354 gen_a64_update_pc(dc, 4);
10355 gen_helper_wfe(tcg_env);
10356 break;
10357 case DISAS_YIELD:
10358 gen_a64_update_pc(dc, 4);
10359 gen_helper_yield(tcg_env);
10360 break;
10361 case DISAS_WFI:
10362 /*
10363 * This is a special case because we don't want to just halt
10364 * the CPU if trying to debug across a WFI.
10365 */
10366 gen_a64_update_pc(dc, 4);
10367 gen_helper_wfi(tcg_env, tcg_constant_i32(4));
10368 /*
10369 * The helper doesn't necessarily throw an exception, but we
10370 * must go back to the main loop to check for interrupts anyway.
10371 */
10372 tcg_gen_exit_tb(NULL, 0);
10373 break;
10374 }
10375 }
10376 }
10377
10378 const TranslatorOps aarch64_translator_ops = {
10379 .init_disas_context = aarch64_tr_init_disas_context,
10380 .tb_start = aarch64_tr_tb_start,
10381 .insn_start = aarch64_tr_insn_start,
10382 .translate_insn = aarch64_tr_translate_insn,
10383 .tb_stop = aarch64_tr_tb_stop,
10384 };
10385