1 /*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
25 #include "qemu/osdep.h"
26
27 /* Define to jump the ELF file used to communicate with GDB. */
28 #undef DEBUG_JIT
29
30 #include "qemu/error-report.h"
31 #include "qemu/cutils.h"
32 #include "qemu/host-utils.h"
33 #include "qemu/qemu-print.h"
34 #include "qemu/cacheflush.h"
35 #include "qemu/cacheinfo.h"
36 #include "qemu/timer.h"
37 #include "exec/target_page.h"
38 #include "exec/translation-block.h"
39 #include "exec/tlb-common.h"
40 #include "tcg/startup.h"
41 #include "tcg/tcg-op-common.h"
42
43 #if UINTPTR_MAX == UINT32_MAX
44 # define ELF_CLASS ELFCLASS32
45 #else
46 # define ELF_CLASS ELFCLASS64
47 #endif
48 #if HOST_BIG_ENDIAN
49 # define ELF_DATA ELFDATA2MSB
50 #else
51 # define ELF_DATA ELFDATA2LSB
52 #endif
53
54 #include "elf.h"
55 #include "exec/log.h"
56 #include "tcg/tcg-ldst.h"
57 #include "tcg/tcg-temp-internal.h"
58 #include "tcg-internal.h"
59 #include "tcg/perf.h"
60 #include "tcg-has.h"
61 #ifdef CONFIG_USER_ONLY
62 #include "user/guest-base.h"
63 #endif
64
65 /* Forward declarations for functions declared in tcg-target.c.inc and
66 used here. */
67 static void tcg_target_init(TCGContext *s);
68 static void tcg_target_qemu_prologue(TCGContext *s);
69 static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
70 intptr_t value, intptr_t addend);
71 static void tcg_out_nop_fill(tcg_insn_unit *p, int count);
72
73 typedef struct TCGLabelQemuLdst TCGLabelQemuLdst;
74 static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l);
75 static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l);
76
77 /* The CIE and FDE header definitions will be common to all hosts. */
78 typedef struct {
79 uint32_t len __attribute__((aligned((sizeof(void *)))));
80 uint32_t id;
81 uint8_t version;
82 char augmentation[1];
83 uint8_t code_align;
84 uint8_t data_align;
85 uint8_t return_column;
86 } DebugFrameCIE;
87
88 typedef struct QEMU_PACKED {
89 uint32_t len __attribute__((aligned((sizeof(void *)))));
90 uint32_t cie_offset;
91 uintptr_t func_start;
92 uintptr_t func_len;
93 } DebugFrameFDEHeader;
94
95 typedef struct QEMU_PACKED {
96 DebugFrameCIE cie;
97 DebugFrameFDEHeader fde;
98 } DebugFrameHeader;
99
100 struct TCGLabelQemuLdst {
101 bool is_ld; /* qemu_ld: true, qemu_st: false */
102 MemOpIdx oi;
103 TCGType type; /* result type of a load */
104 TCGReg addr_reg; /* reg index for guest virtual addr */
105 TCGReg datalo_reg; /* reg index for low word to be loaded or stored */
106 TCGReg datahi_reg; /* reg index for high word to be loaded or stored */
107 const tcg_insn_unit *raddr; /* addr of the next IR of qemu_ld/st IR */
108 tcg_insn_unit *label_ptr[2]; /* label pointers to be updated */
109 QSIMPLEQ_ENTRY(TCGLabelQemuLdst) next;
110 };
111
112 static void tcg_register_jit_int(const void *buf, size_t size,
113 const void *debug_frame,
114 size_t debug_frame_size)
115 __attribute__((unused));
116
117 /* Forward declarations for functions declared and used in tcg-target.c.inc. */
118 static void tcg_out_tb_start(TCGContext *s);
119 static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
120 intptr_t arg2);
121 static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
122 static void tcg_out_movi(TCGContext *s, TCGType type,
123 TCGReg ret, tcg_target_long arg);
124 static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
125 static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
126 static void tcg_out_ext8u(TCGContext *s, TCGReg ret, TCGReg arg);
127 static void tcg_out_ext16u(TCGContext *s, TCGReg ret, TCGReg arg);
128 static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg);
129 static void tcg_out_ext32u(TCGContext *s, TCGReg ret, TCGReg arg);
130 static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg);
131 static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg);
132 static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg ret, TCGReg arg);
133 static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long);
134 static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2);
135 static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg);
136 static void tcg_out_goto_tb(TCGContext *s, int which);
137 static void tcg_out_goto_ptr(TCGContext *s, TCGReg dest);
138 static void tcg_out_mb(TCGContext *s, unsigned bar);
139 static void tcg_out_br(TCGContext *s, TCGLabel *l);
140 static void tcg_out_set_carry(TCGContext *s);
141 static void tcg_out_set_borrow(TCGContext *s);
142 #if TCG_TARGET_MAYBE_vec
143 static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
144 TCGReg dst, TCGReg src);
145 static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
146 TCGReg dst, TCGReg base, intptr_t offset);
147 static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
148 TCGReg dst, int64_t arg);
149 static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
150 unsigned vecl, unsigned vece,
151 const TCGArg args[TCG_MAX_OP_ARGS],
152 const int const_args[TCG_MAX_OP_ARGS]);
153 #else
tcg_out_dup_vec(TCGContext * s,TCGType type,unsigned vece,TCGReg dst,TCGReg src)154 static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
155 TCGReg dst, TCGReg src)
156 {
157 g_assert_not_reached();
158 }
tcg_out_dupm_vec(TCGContext * s,TCGType type,unsigned vece,TCGReg dst,TCGReg base,intptr_t offset)159 static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
160 TCGReg dst, TCGReg base, intptr_t offset)
161 {
162 g_assert_not_reached();
163 }
tcg_out_dupi_vec(TCGContext * s,TCGType type,unsigned vece,TCGReg dst,int64_t arg)164 static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
165 TCGReg dst, int64_t arg)
166 {
167 g_assert_not_reached();
168 }
tcg_out_vec_op(TCGContext * s,TCGOpcode opc,unsigned vecl,unsigned vece,const TCGArg args[TCG_MAX_OP_ARGS],const int const_args[TCG_MAX_OP_ARGS])169 static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
170 unsigned vecl, unsigned vece,
171 const TCGArg args[TCG_MAX_OP_ARGS],
172 const int const_args[TCG_MAX_OP_ARGS])
173 {
174 g_assert_not_reached();
175 }
tcg_can_emit_vec_op(TCGOpcode o,TCGType t,unsigned ve)176 int tcg_can_emit_vec_op(TCGOpcode o, TCGType t, unsigned ve)
177 {
178 return 0;
179 }
180 #endif
181 static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
182 intptr_t arg2);
183 static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
184 TCGReg base, intptr_t ofs);
185 static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target,
186 const TCGHelperInfo *info);
187 static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot);
188 static bool tcg_target_const_match(int64_t val, int ct,
189 TCGType type, TCGCond cond, int vece);
190
191 #ifndef CONFIG_USER_ONLY
192 #define guest_base ({ qemu_build_not_reached(); (uintptr_t)0; })
193 #endif
194
195 typedef struct TCGLdstHelperParam {
196 TCGReg (*ra_gen)(TCGContext *s, const TCGLabelQemuLdst *l, int arg_reg);
197 unsigned ntmp;
198 int tmp[3];
199 } TCGLdstHelperParam;
200
201 static void tcg_out_ld_helper_args(TCGContext *s, const TCGLabelQemuLdst *l,
202 const TCGLdstHelperParam *p)
203 __attribute__((unused));
204 static void tcg_out_ld_helper_ret(TCGContext *s, const TCGLabelQemuLdst *l,
205 bool load_sign, const TCGLdstHelperParam *p)
206 __attribute__((unused));
207 static void tcg_out_st_helper_args(TCGContext *s, const TCGLabelQemuLdst *l,
208 const TCGLdstHelperParam *p)
209 __attribute__((unused));
210
211 static void * const qemu_ld_helpers[MO_SSIZE + 1] __attribute__((unused)) = {
212 [MO_UB] = helper_ldub_mmu,
213 [MO_SB] = helper_ldsb_mmu,
214 [MO_UW] = helper_lduw_mmu,
215 [MO_SW] = helper_ldsw_mmu,
216 [MO_UL] = helper_ldul_mmu,
217 [MO_UQ] = helper_ldq_mmu,
218 #if TCG_TARGET_REG_BITS == 64
219 [MO_SL] = helper_ldsl_mmu,
220 [MO_128] = helper_ld16_mmu,
221 #endif
222 };
223
224 static void * const qemu_st_helpers[MO_SIZE + 1] __attribute__((unused)) = {
225 [MO_8] = helper_stb_mmu,
226 [MO_16] = helper_stw_mmu,
227 [MO_32] = helper_stl_mmu,
228 [MO_64] = helper_stq_mmu,
229 #if TCG_TARGET_REG_BITS == 64
230 [MO_128] = helper_st16_mmu,
231 #endif
232 };
233
234 typedef struct {
235 MemOp atom; /* lg2 bits of atomicity required */
236 MemOp align; /* lg2 bits of alignment to use */
237 } TCGAtomAlign;
238
239 static TCGAtomAlign atom_and_align_for_opc(TCGContext *s, MemOp opc,
240 MemOp host_atom, bool allow_two_ops)
241 __attribute__((unused));
242
243 #ifdef CONFIG_USER_ONLY
244 bool tcg_use_softmmu;
245 #endif
246
247 TCGContext tcg_init_ctx;
248 __thread TCGContext *tcg_ctx;
249
250 TCGContext **tcg_ctxs;
251 unsigned int tcg_cur_ctxs;
252 unsigned int tcg_max_ctxs;
253 TCGv_env tcg_env;
254 const void *tcg_code_gen_epilogue;
255 uintptr_t tcg_splitwx_diff;
256
257 #ifndef CONFIG_TCG_INTERPRETER
258 tcg_prologue_fn *tcg_qemu_tb_exec;
259 #endif
260
261 static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT];
262 static TCGRegSet tcg_target_call_clobber_regs;
263
264 #if TCG_TARGET_INSN_UNIT_SIZE == 1
tcg_out8(TCGContext * s,uint8_t v)265 static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v)
266 {
267 *s->code_ptr++ = v;
268 }
269
tcg_patch8(tcg_insn_unit * p,uint8_t v)270 static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p,
271 uint8_t v)
272 {
273 *p = v;
274 }
275 #endif
276
277 #if TCG_TARGET_INSN_UNIT_SIZE <= 2
tcg_out16(TCGContext * s,uint16_t v)278 static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v)
279 {
280 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
281 *s->code_ptr++ = v;
282 } else {
283 tcg_insn_unit *p = s->code_ptr;
284 memcpy(p, &v, sizeof(v));
285 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE);
286 }
287 }
288
tcg_patch16(tcg_insn_unit * p,uint16_t v)289 static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p,
290 uint16_t v)
291 {
292 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
293 *p = v;
294 } else {
295 memcpy(p, &v, sizeof(v));
296 }
297 }
298 #endif
299
300 #if TCG_TARGET_INSN_UNIT_SIZE <= 4
tcg_out32(TCGContext * s,uint32_t v)301 static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v)
302 {
303 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
304 *s->code_ptr++ = v;
305 } else {
306 tcg_insn_unit *p = s->code_ptr;
307 memcpy(p, &v, sizeof(v));
308 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE);
309 }
310 }
311
tcg_patch32(tcg_insn_unit * p,uint32_t v)312 static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p,
313 uint32_t v)
314 {
315 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
316 *p = v;
317 } else {
318 memcpy(p, &v, sizeof(v));
319 }
320 }
321 #endif
322
323 #if TCG_TARGET_INSN_UNIT_SIZE <= 8
tcg_out64(TCGContext * s,uint64_t v)324 static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v)
325 {
326 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
327 *s->code_ptr++ = v;
328 } else {
329 tcg_insn_unit *p = s->code_ptr;
330 memcpy(p, &v, sizeof(v));
331 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE);
332 }
333 }
334
tcg_patch64(tcg_insn_unit * p,uint64_t v)335 static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p,
336 uint64_t v)
337 {
338 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
339 *p = v;
340 } else {
341 memcpy(p, &v, sizeof(v));
342 }
343 }
344 #endif
345
346 /* label relocation processing */
347
tcg_out_reloc(TCGContext * s,tcg_insn_unit * code_ptr,int type,TCGLabel * l,intptr_t addend)348 static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
349 TCGLabel *l, intptr_t addend)
350 {
351 TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation));
352
353 r->type = type;
354 r->ptr = code_ptr;
355 r->addend = addend;
356 QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next);
357 }
358
tcg_out_label(TCGContext * s,TCGLabel * l)359 static void tcg_out_label(TCGContext *s, TCGLabel *l)
360 {
361 tcg_debug_assert(!l->has_value);
362 l->has_value = 1;
363 l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr);
364 }
365
gen_new_label(void)366 TCGLabel *gen_new_label(void)
367 {
368 TCGContext *s = tcg_ctx;
369 TCGLabel *l = tcg_malloc(sizeof(TCGLabel));
370
371 memset(l, 0, sizeof(TCGLabel));
372 l->id = s->nb_labels++;
373 QSIMPLEQ_INIT(&l->branches);
374 QSIMPLEQ_INIT(&l->relocs);
375
376 QSIMPLEQ_INSERT_TAIL(&s->labels, l, next);
377
378 return l;
379 }
380
tcg_resolve_relocs(TCGContext * s)381 static bool tcg_resolve_relocs(TCGContext *s)
382 {
383 TCGLabel *l;
384
385 QSIMPLEQ_FOREACH(l, &s->labels, next) {
386 TCGRelocation *r;
387 uintptr_t value = l->u.value;
388
389 QSIMPLEQ_FOREACH(r, &l->relocs, next) {
390 if (!patch_reloc(r->ptr, r->type, value, r->addend)) {
391 return false;
392 }
393 }
394 }
395 return true;
396 }
397
set_jmp_reset_offset(TCGContext * s,int which)398 static void set_jmp_reset_offset(TCGContext *s, int which)
399 {
400 /*
401 * We will check for overflow at the end of the opcode loop in
402 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
403 */
404 s->gen_tb->jmp_reset_offset[which] = tcg_current_code_size(s);
405 }
406
set_jmp_insn_offset(TCGContext * s,int which)407 static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which)
408 {
409 /*
410 * We will check for overflow at the end of the opcode loop in
411 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
412 */
413 s->gen_tb->jmp_insn_offset[which] = tcg_current_code_size(s);
414 }
415
get_jmp_target_addr(TCGContext * s,int which)416 static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which)
417 {
418 /*
419 * Return the read-execute version of the pointer, for the benefit
420 * of any pc-relative addressing mode.
421 */
422 return (uintptr_t)tcg_splitwx_to_rx(&s->gen_tb->jmp_target_addr[which]);
423 }
424
425 static int __attribute__((unused))
tlb_mask_table_ofs(TCGContext * s,int which)426 tlb_mask_table_ofs(TCGContext *s, int which)
427 {
428 return (offsetof(CPUNegativeOffsetState, tlb.f[which]) -
429 sizeof(CPUNegativeOffsetState));
430 }
431
432 /* Signal overflow, starting over with fewer guest insns. */
433 static G_NORETURN
tcg_raise_tb_overflow(TCGContext * s)434 void tcg_raise_tb_overflow(TCGContext *s)
435 {
436 siglongjmp(s->jmp_trans, -2);
437 }
438
439 /*
440 * Used by tcg_out_movext{1,2} to hold the arguments for tcg_out_movext.
441 * By the time we arrive at tcg_out_movext1, @dst is always a TCGReg.
442 *
443 * However, tcg_out_helper_load_slots reuses this field to hold an
444 * argument slot number (which may designate a argument register or an
445 * argument stack slot), converting to TCGReg once all arguments that
446 * are destined for the stack are processed.
447 */
448 typedef struct TCGMovExtend {
449 unsigned dst;
450 TCGReg src;
451 TCGType dst_type;
452 TCGType src_type;
453 MemOp src_ext;
454 } TCGMovExtend;
455
456 /**
457 * tcg_out_movext -- move and extend
458 * @s: tcg context
459 * @dst_type: integral type for destination
460 * @dst: destination register
461 * @src_type: integral type for source
462 * @src_ext: extension to apply to source
463 * @src: source register
464 *
465 * Move or extend @src into @dst, depending on @src_ext and the types.
466 */
tcg_out_movext(TCGContext * s,TCGType dst_type,TCGReg dst,TCGType src_type,MemOp src_ext,TCGReg src)467 static void tcg_out_movext(TCGContext *s, TCGType dst_type, TCGReg dst,
468 TCGType src_type, MemOp src_ext, TCGReg src)
469 {
470 switch (src_ext) {
471 case MO_UB:
472 tcg_out_ext8u(s, dst, src);
473 break;
474 case MO_SB:
475 tcg_out_ext8s(s, dst_type, dst, src);
476 break;
477 case MO_UW:
478 tcg_out_ext16u(s, dst, src);
479 break;
480 case MO_SW:
481 tcg_out_ext16s(s, dst_type, dst, src);
482 break;
483 case MO_UL:
484 case MO_SL:
485 if (dst_type == TCG_TYPE_I32) {
486 if (src_type == TCG_TYPE_I32) {
487 tcg_out_mov(s, TCG_TYPE_I32, dst, src);
488 } else {
489 tcg_out_extrl_i64_i32(s, dst, src);
490 }
491 } else if (src_type == TCG_TYPE_I32) {
492 if (src_ext & MO_SIGN) {
493 tcg_out_exts_i32_i64(s, dst, src);
494 } else {
495 tcg_out_extu_i32_i64(s, dst, src);
496 }
497 } else {
498 if (src_ext & MO_SIGN) {
499 tcg_out_ext32s(s, dst, src);
500 } else {
501 tcg_out_ext32u(s, dst, src);
502 }
503 }
504 break;
505 case MO_UQ:
506 tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
507 if (dst_type == TCG_TYPE_I32) {
508 tcg_out_extrl_i64_i32(s, dst, src);
509 } else {
510 tcg_out_mov(s, TCG_TYPE_I64, dst, src);
511 }
512 break;
513 default:
514 g_assert_not_reached();
515 }
516 }
517
518 /* Minor variations on a theme, using a structure. */
tcg_out_movext1_new_src(TCGContext * s,const TCGMovExtend * i,TCGReg src)519 static void tcg_out_movext1_new_src(TCGContext *s, const TCGMovExtend *i,
520 TCGReg src)
521 {
522 tcg_out_movext(s, i->dst_type, i->dst, i->src_type, i->src_ext, src);
523 }
524
tcg_out_movext1(TCGContext * s,const TCGMovExtend * i)525 static void tcg_out_movext1(TCGContext *s, const TCGMovExtend *i)
526 {
527 tcg_out_movext1_new_src(s, i, i->src);
528 }
529
530 /**
531 * tcg_out_movext2 -- move and extend two pair
532 * @s: tcg context
533 * @i1: first move description
534 * @i2: second move description
535 * @scratch: temporary register, or -1 for none
536 *
537 * As tcg_out_movext, for both @i1 and @i2, caring for overlap
538 * between the sources and destinations.
539 */
540
tcg_out_movext2(TCGContext * s,const TCGMovExtend * i1,const TCGMovExtend * i2,int scratch)541 static void tcg_out_movext2(TCGContext *s, const TCGMovExtend *i1,
542 const TCGMovExtend *i2, int scratch)
543 {
544 TCGReg src1 = i1->src;
545 TCGReg src2 = i2->src;
546
547 if (i1->dst != src2) {
548 tcg_out_movext1(s, i1);
549 tcg_out_movext1(s, i2);
550 return;
551 }
552 if (i2->dst == src1) {
553 TCGType src1_type = i1->src_type;
554 TCGType src2_type = i2->src_type;
555
556 if (tcg_out_xchg(s, MAX(src1_type, src2_type), src1, src2)) {
557 /* The data is now in the correct registers, now extend. */
558 src1 = i2->src;
559 src2 = i1->src;
560 } else {
561 tcg_debug_assert(scratch >= 0);
562 tcg_out_mov(s, src1_type, scratch, src1);
563 src1 = scratch;
564 }
565 }
566 tcg_out_movext1_new_src(s, i2, src2);
567 tcg_out_movext1_new_src(s, i1, src1);
568 }
569
570 /**
571 * tcg_out_movext3 -- move and extend three pair
572 * @s: tcg context
573 * @i1: first move description
574 * @i2: second move description
575 * @i3: third move description
576 * @scratch: temporary register, or -1 for none
577 *
578 * As tcg_out_movext, for all of @i1, @i2 and @i3, caring for overlap
579 * between the sources and destinations.
580 */
581
tcg_out_movext3(TCGContext * s,const TCGMovExtend * i1,const TCGMovExtend * i2,const TCGMovExtend * i3,int scratch)582 static void tcg_out_movext3(TCGContext *s, const TCGMovExtend *i1,
583 const TCGMovExtend *i2, const TCGMovExtend *i3,
584 int scratch)
585 {
586 TCGReg src1 = i1->src;
587 TCGReg src2 = i2->src;
588 TCGReg src3 = i3->src;
589
590 if (i1->dst != src2 && i1->dst != src3) {
591 tcg_out_movext1(s, i1);
592 tcg_out_movext2(s, i2, i3, scratch);
593 return;
594 }
595 if (i2->dst != src1 && i2->dst != src3) {
596 tcg_out_movext1(s, i2);
597 tcg_out_movext2(s, i1, i3, scratch);
598 return;
599 }
600 if (i3->dst != src1 && i3->dst != src2) {
601 tcg_out_movext1(s, i3);
602 tcg_out_movext2(s, i1, i2, scratch);
603 return;
604 }
605
606 /*
607 * There is a cycle. Since there are only 3 nodes, the cycle is
608 * either "clockwise" or "anti-clockwise", and can be solved with
609 * a single scratch or two xchg.
610 */
611 if (i1->dst == src2 && i2->dst == src3 && i3->dst == src1) {
612 /* "Clockwise" */
613 if (tcg_out_xchg(s, MAX(i1->src_type, i2->src_type), src1, src2)) {
614 tcg_out_xchg(s, MAX(i2->src_type, i3->src_type), src2, src3);
615 /* The data is now in the correct registers, now extend. */
616 tcg_out_movext1_new_src(s, i1, i1->dst);
617 tcg_out_movext1_new_src(s, i2, i2->dst);
618 tcg_out_movext1_new_src(s, i3, i3->dst);
619 } else {
620 tcg_debug_assert(scratch >= 0);
621 tcg_out_mov(s, i1->src_type, scratch, src1);
622 tcg_out_movext1(s, i3);
623 tcg_out_movext1(s, i2);
624 tcg_out_movext1_new_src(s, i1, scratch);
625 }
626 } else if (i1->dst == src3 && i2->dst == src1 && i3->dst == src2) {
627 /* "Anti-clockwise" */
628 if (tcg_out_xchg(s, MAX(i2->src_type, i3->src_type), src2, src3)) {
629 tcg_out_xchg(s, MAX(i1->src_type, i2->src_type), src1, src2);
630 /* The data is now in the correct registers, now extend. */
631 tcg_out_movext1_new_src(s, i1, i1->dst);
632 tcg_out_movext1_new_src(s, i2, i2->dst);
633 tcg_out_movext1_new_src(s, i3, i3->dst);
634 } else {
635 tcg_debug_assert(scratch >= 0);
636 tcg_out_mov(s, i1->src_type, scratch, src1);
637 tcg_out_movext1(s, i2);
638 tcg_out_movext1(s, i3);
639 tcg_out_movext1_new_src(s, i1, scratch);
640 }
641 } else {
642 g_assert_not_reached();
643 }
644 }
645
646 /*
647 * Allocate a new TCGLabelQemuLdst entry.
648 */
649
650 __attribute__((unused))
new_ldst_label(TCGContext * s)651 static TCGLabelQemuLdst *new_ldst_label(TCGContext *s)
652 {
653 TCGLabelQemuLdst *l = tcg_malloc(sizeof(*l));
654
655 memset(l, 0, sizeof(*l));
656 QSIMPLEQ_INSERT_TAIL(&s->ldst_labels, l, next);
657
658 return l;
659 }
660
661 /*
662 * Allocate new constant pool entries.
663 */
664
665 typedef struct TCGLabelPoolData {
666 struct TCGLabelPoolData *next;
667 tcg_insn_unit *label;
668 intptr_t addend;
669 int rtype;
670 unsigned nlong;
671 tcg_target_ulong data[];
672 } TCGLabelPoolData;
673
new_pool_alloc(TCGContext * s,int nlong,int rtype,tcg_insn_unit * label,intptr_t addend)674 static TCGLabelPoolData *new_pool_alloc(TCGContext *s, int nlong, int rtype,
675 tcg_insn_unit *label, intptr_t addend)
676 {
677 TCGLabelPoolData *n = tcg_malloc(sizeof(TCGLabelPoolData)
678 + sizeof(tcg_target_ulong) * nlong);
679
680 n->label = label;
681 n->addend = addend;
682 n->rtype = rtype;
683 n->nlong = nlong;
684 return n;
685 }
686
new_pool_insert(TCGContext * s,TCGLabelPoolData * n)687 static void new_pool_insert(TCGContext *s, TCGLabelPoolData *n)
688 {
689 TCGLabelPoolData *i, **pp;
690 int nlong = n->nlong;
691
692 /* Insertion sort on the pool. */
693 for (pp = &s->pool_labels; (i = *pp) != NULL; pp = &i->next) {
694 if (nlong > i->nlong) {
695 break;
696 }
697 if (nlong < i->nlong) {
698 continue;
699 }
700 if (memcmp(n->data, i->data, sizeof(tcg_target_ulong) * nlong) >= 0) {
701 break;
702 }
703 }
704 n->next = *pp;
705 *pp = n;
706 }
707
708 /* The "usual" for generic integer code. */
709 __attribute__((unused))
new_pool_label(TCGContext * s,tcg_target_ulong d,int rtype,tcg_insn_unit * label,intptr_t addend)710 static void new_pool_label(TCGContext *s, tcg_target_ulong d, int rtype,
711 tcg_insn_unit *label, intptr_t addend)
712 {
713 TCGLabelPoolData *n = new_pool_alloc(s, 1, rtype, label, addend);
714 n->data[0] = d;
715 new_pool_insert(s, n);
716 }
717
718 /* For v64 or v128, depending on the host. */
719 __attribute__((unused))
new_pool_l2(TCGContext * s,int rtype,tcg_insn_unit * label,intptr_t addend,tcg_target_ulong d0,tcg_target_ulong d1)720 static void new_pool_l2(TCGContext *s, int rtype, tcg_insn_unit *label,
721 intptr_t addend, tcg_target_ulong d0,
722 tcg_target_ulong d1)
723 {
724 TCGLabelPoolData *n = new_pool_alloc(s, 2, rtype, label, addend);
725 n->data[0] = d0;
726 n->data[1] = d1;
727 new_pool_insert(s, n);
728 }
729
730 /* For v128 or v256, depending on the host. */
731 __attribute__((unused))
new_pool_l4(TCGContext * s,int rtype,tcg_insn_unit * label,intptr_t addend,tcg_target_ulong d0,tcg_target_ulong d1,tcg_target_ulong d2,tcg_target_ulong d3)732 static void new_pool_l4(TCGContext *s, int rtype, tcg_insn_unit *label,
733 intptr_t addend, tcg_target_ulong d0,
734 tcg_target_ulong d1, tcg_target_ulong d2,
735 tcg_target_ulong d3)
736 {
737 TCGLabelPoolData *n = new_pool_alloc(s, 4, rtype, label, addend);
738 n->data[0] = d0;
739 n->data[1] = d1;
740 n->data[2] = d2;
741 n->data[3] = d3;
742 new_pool_insert(s, n);
743 }
744
745 /* For v256, for 32-bit host. */
746 __attribute__((unused))
new_pool_l8(TCGContext * s,int rtype,tcg_insn_unit * label,intptr_t addend,tcg_target_ulong d0,tcg_target_ulong d1,tcg_target_ulong d2,tcg_target_ulong d3,tcg_target_ulong d4,tcg_target_ulong d5,tcg_target_ulong d6,tcg_target_ulong d7)747 static void new_pool_l8(TCGContext *s, int rtype, tcg_insn_unit *label,
748 intptr_t addend, tcg_target_ulong d0,
749 tcg_target_ulong d1, tcg_target_ulong d2,
750 tcg_target_ulong d3, tcg_target_ulong d4,
751 tcg_target_ulong d5, tcg_target_ulong d6,
752 tcg_target_ulong d7)
753 {
754 TCGLabelPoolData *n = new_pool_alloc(s, 8, rtype, label, addend);
755 n->data[0] = d0;
756 n->data[1] = d1;
757 n->data[2] = d2;
758 n->data[3] = d3;
759 n->data[4] = d4;
760 n->data[5] = d5;
761 n->data[6] = d6;
762 n->data[7] = d7;
763 new_pool_insert(s, n);
764 }
765
766 /*
767 * Generate TB finalization at the end of block
768 */
769
tcg_out_ldst_finalize(TCGContext * s)770 static int tcg_out_ldst_finalize(TCGContext *s)
771 {
772 TCGLabelQemuLdst *lb;
773
774 /* qemu_ld/st slow paths */
775 QSIMPLEQ_FOREACH(lb, &s->ldst_labels, next) {
776 if (lb->is_ld
777 ? !tcg_out_qemu_ld_slow_path(s, lb)
778 : !tcg_out_qemu_st_slow_path(s, lb)) {
779 return -2;
780 }
781
782 /*
783 * Test for (pending) buffer overflow. The assumption is that any
784 * one operation beginning below the high water mark cannot overrun
785 * the buffer completely. Thus we can test for overflow after
786 * generating code without having to check during generation.
787 */
788 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
789 return -1;
790 }
791 }
792 return 0;
793 }
794
tcg_out_pool_finalize(TCGContext * s)795 static int tcg_out_pool_finalize(TCGContext *s)
796 {
797 TCGLabelPoolData *p = s->pool_labels;
798 TCGLabelPoolData *l = NULL;
799 void *a;
800
801 if (p == NULL) {
802 return 0;
803 }
804
805 /*
806 * ??? Round up to qemu_icache_linesize, but then do not round
807 * again when allocating the next TranslationBlock structure.
808 */
809 a = (void *)ROUND_UP((uintptr_t)s->code_ptr,
810 sizeof(tcg_target_ulong) * p->nlong);
811 tcg_out_nop_fill(s->code_ptr, (tcg_insn_unit *)a - s->code_ptr);
812 s->data_gen_ptr = a;
813
814 for (; p != NULL; p = p->next) {
815 size_t size = sizeof(tcg_target_ulong) * p->nlong;
816 uintptr_t value;
817
818 if (!l || l->nlong != p->nlong || memcmp(l->data, p->data, size)) {
819 if (unlikely(a > s->code_gen_highwater)) {
820 return -1;
821 }
822 memcpy(a, p->data, size);
823 a += size;
824 l = p;
825 }
826
827 value = (uintptr_t)tcg_splitwx_to_rx(a) - size;
828 if (!patch_reloc(p->label, p->rtype, value, p->addend)) {
829 return -2;
830 }
831 }
832
833 s->code_ptr = a;
834 return 0;
835 }
836
837 #define C_PFX1(P, A) P##A
838 #define C_PFX2(P, A, B) P##A##_##B
839 #define C_PFX3(P, A, B, C) P##A##_##B##_##C
840 #define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D
841 #define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E
842 #define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F
843
844 /* Define an enumeration for the various combinations. */
845
846 #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1),
847 #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2),
848 #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3),
849 #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4),
850
851 #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1),
852 #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2),
853 #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3),
854 #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4),
855
856 #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2),
857 #define C_N1O1_I1(O1, O2, I1) C_PFX3(c_n1o1_i1_, O1, O2, I1),
858 #define C_N2_I1(O1, O2, I1) C_PFX3(c_n2_i1_, O1, O2, I1),
859
860 #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1),
861 #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2),
862 #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3),
863 #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4),
864 #define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_n1_o1_i4_, O1, O2, I1, I2, I3, I4),
865
866 typedef enum {
867 C_Dynamic = -2,
868 C_NotImplemented = -1,
869 #include "tcg-target-con-set.h"
870 } TCGConstraintSetIndex;
871
872 static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode, TCGType, unsigned);
873
874 #undef C_O0_I1
875 #undef C_O0_I2
876 #undef C_O0_I3
877 #undef C_O0_I4
878 #undef C_O1_I1
879 #undef C_O1_I2
880 #undef C_O1_I3
881 #undef C_O1_I4
882 #undef C_N1_I2
883 #undef C_N1O1_I1
884 #undef C_N2_I1
885 #undef C_O2_I1
886 #undef C_O2_I2
887 #undef C_O2_I3
888 #undef C_O2_I4
889 #undef C_N1_O1_I4
890
891 /* Put all of the constraint sets into an array, indexed by the enum. */
892
893 typedef struct TCGConstraintSet {
894 uint8_t nb_oargs, nb_iargs;
895 const char *args_ct_str[TCG_MAX_OP_ARGS];
896 } TCGConstraintSet;
897
898 #define C_O0_I1(I1) { 0, 1, { #I1 } },
899 #define C_O0_I2(I1, I2) { 0, 2, { #I1, #I2 } },
900 #define C_O0_I3(I1, I2, I3) { 0, 3, { #I1, #I2, #I3 } },
901 #define C_O0_I4(I1, I2, I3, I4) { 0, 4, { #I1, #I2, #I3, #I4 } },
902
903 #define C_O1_I1(O1, I1) { 1, 1, { #O1, #I1 } },
904 #define C_O1_I2(O1, I1, I2) { 1, 2, { #O1, #I1, #I2 } },
905 #define C_O1_I3(O1, I1, I2, I3) { 1, 3, { #O1, #I1, #I2, #I3 } },
906 #define C_O1_I4(O1, I1, I2, I3, I4) { 1, 4, { #O1, #I1, #I2, #I3, #I4 } },
907
908 #define C_N1_I2(O1, I1, I2) { 1, 2, { "&" #O1, #I1, #I2 } },
909 #define C_N1O1_I1(O1, O2, I1) { 2, 1, { "&" #O1, #O2, #I1 } },
910 #define C_N2_I1(O1, O2, I1) { 2, 1, { "&" #O1, "&" #O2, #I1 } },
911
912 #define C_O2_I1(O1, O2, I1) { 2, 1, { #O1, #O2, #I1 } },
913 #define C_O2_I2(O1, O2, I1, I2) { 2, 2, { #O1, #O2, #I1, #I2 } },
914 #define C_O2_I3(O1, O2, I1, I2, I3) { 2, 3, { #O1, #O2, #I1, #I2, #I3 } },
915 #define C_O2_I4(O1, O2, I1, I2, I3, I4) { 2, 4, { #O1, #O2, #I1, #I2, #I3, #I4 } },
916 #define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) { 2, 4, { "&" #O1, #O2, #I1, #I2, #I3, #I4 } },
917
918 static const TCGConstraintSet constraint_sets[] = {
919 #include "tcg-target-con-set.h"
920 };
921
922 #undef C_O0_I1
923 #undef C_O0_I2
924 #undef C_O0_I3
925 #undef C_O0_I4
926 #undef C_O1_I1
927 #undef C_O1_I2
928 #undef C_O1_I3
929 #undef C_O1_I4
930 #undef C_N1_I2
931 #undef C_N1O1_I1
932 #undef C_N2_I1
933 #undef C_O2_I1
934 #undef C_O2_I2
935 #undef C_O2_I3
936 #undef C_O2_I4
937 #undef C_N1_O1_I4
938
939 /* Expand the enumerator to be returned from tcg_target_op_def(). */
940
941 #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1)
942 #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2)
943 #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3)
944 #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4)
945
946 #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1)
947 #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2)
948 #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3)
949 #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4)
950
951 #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2)
952 #define C_N1O1_I1(O1, O2, I1) C_PFX3(c_n1o1_i1_, O1, O2, I1)
953 #define C_N2_I1(O1, O2, I1) C_PFX3(c_n2_i1_, O1, O2, I1)
954
955 #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1)
956 #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2)
957 #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3)
958 #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4)
959 #define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_n1_o1_i4_, O1, O2, I1, I2, I3, I4)
960
961 /*
962 * TCGOutOp is the base class for a set of structures that describe how
963 * to generate code for a given TCGOpcode.
964 *
965 * @static_constraint:
966 * C_NotImplemented: The TCGOpcode is not supported by the backend.
967 * C_Dynamic: Use @dynamic_constraint to select a constraint set
968 * based on any of @type, @flags, or host isa.
969 * Otherwise: The register allocation constrains for the TCGOpcode.
970 *
971 * Subclasses of TCGOutOp will define a set of output routines that may
972 * be used. Such routines will often be selected by the set of registers
973 * and constants that come out of register allocation. The set of
974 * routines that are provided will guide the set of constraints that are
975 * legal. In particular, assume that tcg_optimize() has done its job in
976 * swapping commutative operands and folding operations for which all
977 * operands are constant.
978 */
979 typedef struct TCGOutOp {
980 TCGConstraintSetIndex static_constraint;
981 TCGConstraintSetIndex (*dynamic_constraint)(TCGType type, unsigned flags);
982 } TCGOutOp;
983
984 typedef struct TCGOutOpAddSubCarry {
985 TCGOutOp base;
986 void (*out_rrr)(TCGContext *s, TCGType type,
987 TCGReg a0, TCGReg a1, TCGReg a2);
988 void (*out_rri)(TCGContext *s, TCGType type,
989 TCGReg a0, TCGReg a1, tcg_target_long a2);
990 void (*out_rir)(TCGContext *s, TCGType type,
991 TCGReg a0, tcg_target_long a1, TCGReg a2);
992 void (*out_rii)(TCGContext *s, TCGType type,
993 TCGReg a0, tcg_target_long a1, tcg_target_long a2);
994 } TCGOutOpAddSubCarry;
995
996 typedef struct TCGOutOpBinary {
997 TCGOutOp base;
998 void (*out_rrr)(TCGContext *s, TCGType type,
999 TCGReg a0, TCGReg a1, TCGReg a2);
1000 void (*out_rri)(TCGContext *s, TCGType type,
1001 TCGReg a0, TCGReg a1, tcg_target_long a2);
1002 } TCGOutOpBinary;
1003
1004 typedef struct TCGOutOpBrcond {
1005 TCGOutOp base;
1006 void (*out_rr)(TCGContext *s, TCGType type, TCGCond cond,
1007 TCGReg a1, TCGReg a2, TCGLabel *label);
1008 void (*out_ri)(TCGContext *s, TCGType type, TCGCond cond,
1009 TCGReg a1, tcg_target_long a2, TCGLabel *label);
1010 } TCGOutOpBrcond;
1011
1012 typedef struct TCGOutOpBrcond2 {
1013 TCGOutOp base;
1014 void (*out)(TCGContext *s, TCGCond cond, TCGReg al, TCGReg ah,
1015 TCGArg bl, bool const_bl,
1016 TCGArg bh, bool const_bh, TCGLabel *l);
1017 } TCGOutOpBrcond2;
1018
1019 typedef struct TCGOutOpBswap {
1020 TCGOutOp base;
1021 void (*out_rr)(TCGContext *s, TCGType type,
1022 TCGReg a0, TCGReg a1, unsigned flags);
1023 } TCGOutOpBswap;
1024
1025 typedef struct TCGOutOpDeposit {
1026 TCGOutOp base;
1027 void (*out_rrr)(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1,
1028 TCGReg a2, unsigned ofs, unsigned len);
1029 void (*out_rri)(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1,
1030 tcg_target_long a2, unsigned ofs, unsigned len);
1031 void (*out_rzr)(TCGContext *s, TCGType type, TCGReg a0,
1032 TCGReg a2, unsigned ofs, unsigned len);
1033 } TCGOutOpDeposit;
1034
1035 typedef struct TCGOutOpDivRem {
1036 TCGOutOp base;
1037 void (*out_rr01r)(TCGContext *s, TCGType type,
1038 TCGReg a0, TCGReg a1, TCGReg a4);
1039 } TCGOutOpDivRem;
1040
1041 typedef struct TCGOutOpExtract {
1042 TCGOutOp base;
1043 void (*out_rr)(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1,
1044 unsigned ofs, unsigned len);
1045 } TCGOutOpExtract;
1046
1047 typedef struct TCGOutOpExtract2 {
1048 TCGOutOp base;
1049 void (*out_rrr)(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1,
1050 TCGReg a2, unsigned shr);
1051 } TCGOutOpExtract2;
1052
1053 typedef struct TCGOutOpLoad {
1054 TCGOutOp base;
1055 void (*out)(TCGContext *s, TCGType type, TCGReg dest,
1056 TCGReg base, intptr_t offset);
1057 } TCGOutOpLoad;
1058
1059 typedef struct TCGOutOpMovcond {
1060 TCGOutOp base;
1061 void (*out)(TCGContext *s, TCGType type, TCGCond cond,
1062 TCGReg ret, TCGReg c1, TCGArg c2, bool const_c2,
1063 TCGArg vt, bool const_vt, TCGArg vf, bool consf_vf);
1064 } TCGOutOpMovcond;
1065
1066 typedef struct TCGOutOpMul2 {
1067 TCGOutOp base;
1068 void (*out_rrrr)(TCGContext *s, TCGType type,
1069 TCGReg a0, TCGReg a1, TCGReg a2, TCGReg a3);
1070 } TCGOutOpMul2;
1071
1072 typedef struct TCGOutOpQemuLdSt {
1073 TCGOutOp base;
1074 void (*out)(TCGContext *s, TCGType type, TCGReg dest,
1075 TCGReg addr, MemOpIdx oi);
1076 } TCGOutOpQemuLdSt;
1077
1078 typedef struct TCGOutOpQemuLdSt2 {
1079 TCGOutOp base;
1080 void (*out)(TCGContext *s, TCGType type, TCGReg dlo, TCGReg dhi,
1081 TCGReg addr, MemOpIdx oi);
1082 } TCGOutOpQemuLdSt2;
1083
1084 typedef struct TCGOutOpUnary {
1085 TCGOutOp base;
1086 void (*out_rr)(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1);
1087 } TCGOutOpUnary;
1088
1089 typedef struct TCGOutOpSetcond {
1090 TCGOutOp base;
1091 void (*out_rrr)(TCGContext *s, TCGType type, TCGCond cond,
1092 TCGReg ret, TCGReg a1, TCGReg a2);
1093 void (*out_rri)(TCGContext *s, TCGType type, TCGCond cond,
1094 TCGReg ret, TCGReg a1, tcg_target_long a2);
1095 } TCGOutOpSetcond;
1096
1097 typedef struct TCGOutOpSetcond2 {
1098 TCGOutOp base;
1099 void (*out)(TCGContext *s, TCGCond cond, TCGReg ret, TCGReg al, TCGReg ah,
1100 TCGArg bl, bool const_bl, TCGArg bh, bool const_bh);
1101 } TCGOutOpSetcond2;
1102
1103 typedef struct TCGOutOpStore {
1104 TCGOutOp base;
1105 void (*out_r)(TCGContext *s, TCGType type, TCGReg data,
1106 TCGReg base, intptr_t offset);
1107 void (*out_i)(TCGContext *s, TCGType type, tcg_target_long data,
1108 TCGReg base, intptr_t offset);
1109 } TCGOutOpStore;
1110
1111 typedef struct TCGOutOpSubtract {
1112 TCGOutOp base;
1113 void (*out_rrr)(TCGContext *s, TCGType type,
1114 TCGReg a0, TCGReg a1, TCGReg a2);
1115 void (*out_rir)(TCGContext *s, TCGType type,
1116 TCGReg a0, tcg_target_long a1, TCGReg a2);
1117 } TCGOutOpSubtract;
1118
1119 #include "tcg-target.c.inc"
1120
1121 #ifndef CONFIG_TCG_INTERPRETER
1122 /* Validate CPUTLBDescFast placement. */
1123 QEMU_BUILD_BUG_ON((int)(offsetof(CPUNegativeOffsetState, tlb.f[0]) -
1124 sizeof(CPUNegativeOffsetState))
1125 < MIN_TLB_MASK_TABLE_OFS);
1126 #endif
1127
1128 #if TCG_TARGET_REG_BITS == 64
1129 /*
1130 * We require these functions for slow-path function calls.
1131 * Adapt them generically for opcode output.
1132 */
1133
tgen_exts_i32_i64(TCGContext * s,TCGType t,TCGReg a0,TCGReg a1)1134 static void tgen_exts_i32_i64(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1)
1135 {
1136 tcg_out_exts_i32_i64(s, a0, a1);
1137 }
1138
1139 static const TCGOutOpUnary outop_exts_i32_i64 = {
1140 .base.static_constraint = C_O1_I1(r, r),
1141 .out_rr = tgen_exts_i32_i64,
1142 };
1143
tgen_extu_i32_i64(TCGContext * s,TCGType t,TCGReg a0,TCGReg a1)1144 static void tgen_extu_i32_i64(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1)
1145 {
1146 tcg_out_extu_i32_i64(s, a0, a1);
1147 }
1148
1149 static const TCGOutOpUnary outop_extu_i32_i64 = {
1150 .base.static_constraint = C_O1_I1(r, r),
1151 .out_rr = tgen_extu_i32_i64,
1152 };
1153
tgen_extrl_i64_i32(TCGContext * s,TCGType t,TCGReg a0,TCGReg a1)1154 static void tgen_extrl_i64_i32(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1)
1155 {
1156 tcg_out_extrl_i64_i32(s, a0, a1);
1157 }
1158
1159 static const TCGOutOpUnary outop_extrl_i64_i32 = {
1160 .base.static_constraint = C_O1_I1(r, r),
1161 .out_rr = TCG_TARGET_HAS_extr_i64_i32 ? tgen_extrl_i64_i32 : NULL,
1162 };
1163 #endif
1164
1165 static const TCGOutOp outop_goto_ptr = {
1166 .static_constraint = C_O0_I1(r),
1167 };
1168
1169 static const TCGOutOpLoad outop_ld = {
1170 .base.static_constraint = C_O1_I1(r, r),
1171 .out = tcg_out_ld,
1172 };
1173
1174 /*
1175 * Register V as the TCGOutOp for O.
1176 * This verifies that V is of type T, otherwise give a nice compiler error.
1177 * This prevents trivial mistakes within each arch/tcg-target.c.inc.
1178 */
1179 #define OUTOP(O, T, V) [O] = _Generic(V, T: &V.base)
1180
1181 /* Register allocation descriptions for every TCGOpcode. */
1182 static const TCGOutOp * const all_outop[NB_OPS] = {
1183 OUTOP(INDEX_op_add, TCGOutOpBinary, outop_add),
1184 OUTOP(INDEX_op_addci, TCGOutOpAddSubCarry, outop_addci),
1185 OUTOP(INDEX_op_addcio, TCGOutOpBinary, outop_addcio),
1186 OUTOP(INDEX_op_addco, TCGOutOpBinary, outop_addco),
1187 /* addc1o is implemented with set_carry + addcio */
1188 OUTOP(INDEX_op_addc1o, TCGOutOpBinary, outop_addcio),
1189 OUTOP(INDEX_op_and, TCGOutOpBinary, outop_and),
1190 OUTOP(INDEX_op_andc, TCGOutOpBinary, outop_andc),
1191 OUTOP(INDEX_op_brcond, TCGOutOpBrcond, outop_brcond),
1192 OUTOP(INDEX_op_bswap16, TCGOutOpBswap, outop_bswap16),
1193 OUTOP(INDEX_op_bswap32, TCGOutOpBswap, outop_bswap32),
1194 OUTOP(INDEX_op_clz, TCGOutOpBinary, outop_clz),
1195 OUTOP(INDEX_op_ctpop, TCGOutOpUnary, outop_ctpop),
1196 OUTOP(INDEX_op_ctz, TCGOutOpBinary, outop_ctz),
1197 OUTOP(INDEX_op_deposit, TCGOutOpDeposit, outop_deposit),
1198 OUTOP(INDEX_op_divs, TCGOutOpBinary, outop_divs),
1199 OUTOP(INDEX_op_divu, TCGOutOpBinary, outop_divu),
1200 OUTOP(INDEX_op_divs2, TCGOutOpDivRem, outop_divs2),
1201 OUTOP(INDEX_op_divu2, TCGOutOpDivRem, outop_divu2),
1202 OUTOP(INDEX_op_eqv, TCGOutOpBinary, outop_eqv),
1203 OUTOP(INDEX_op_extract, TCGOutOpExtract, outop_extract),
1204 OUTOP(INDEX_op_extract2, TCGOutOpExtract2, outop_extract2),
1205 OUTOP(INDEX_op_ld8u, TCGOutOpLoad, outop_ld8u),
1206 OUTOP(INDEX_op_ld8s, TCGOutOpLoad, outop_ld8s),
1207 OUTOP(INDEX_op_ld16u, TCGOutOpLoad, outop_ld16u),
1208 OUTOP(INDEX_op_ld16s, TCGOutOpLoad, outop_ld16s),
1209 OUTOP(INDEX_op_ld, TCGOutOpLoad, outop_ld),
1210 OUTOP(INDEX_op_movcond, TCGOutOpMovcond, outop_movcond),
1211 OUTOP(INDEX_op_mul, TCGOutOpBinary, outop_mul),
1212 OUTOP(INDEX_op_muls2, TCGOutOpMul2, outop_muls2),
1213 OUTOP(INDEX_op_mulsh, TCGOutOpBinary, outop_mulsh),
1214 OUTOP(INDEX_op_mulu2, TCGOutOpMul2, outop_mulu2),
1215 OUTOP(INDEX_op_muluh, TCGOutOpBinary, outop_muluh),
1216 OUTOP(INDEX_op_nand, TCGOutOpBinary, outop_nand),
1217 OUTOP(INDEX_op_neg, TCGOutOpUnary, outop_neg),
1218 OUTOP(INDEX_op_negsetcond, TCGOutOpSetcond, outop_negsetcond),
1219 OUTOP(INDEX_op_nor, TCGOutOpBinary, outop_nor),
1220 OUTOP(INDEX_op_not, TCGOutOpUnary, outop_not),
1221 OUTOP(INDEX_op_or, TCGOutOpBinary, outop_or),
1222 OUTOP(INDEX_op_orc, TCGOutOpBinary, outop_orc),
1223 OUTOP(INDEX_op_qemu_ld, TCGOutOpQemuLdSt, outop_qemu_ld),
1224 OUTOP(INDEX_op_qemu_ld2, TCGOutOpQemuLdSt2, outop_qemu_ld2),
1225 OUTOP(INDEX_op_qemu_st, TCGOutOpQemuLdSt, outop_qemu_st),
1226 OUTOP(INDEX_op_qemu_st2, TCGOutOpQemuLdSt2, outop_qemu_st2),
1227 OUTOP(INDEX_op_rems, TCGOutOpBinary, outop_rems),
1228 OUTOP(INDEX_op_remu, TCGOutOpBinary, outop_remu),
1229 OUTOP(INDEX_op_rotl, TCGOutOpBinary, outop_rotl),
1230 OUTOP(INDEX_op_rotr, TCGOutOpBinary, outop_rotr),
1231 OUTOP(INDEX_op_sar, TCGOutOpBinary, outop_sar),
1232 OUTOP(INDEX_op_setcond, TCGOutOpSetcond, outop_setcond),
1233 OUTOP(INDEX_op_sextract, TCGOutOpExtract, outop_sextract),
1234 OUTOP(INDEX_op_shl, TCGOutOpBinary, outop_shl),
1235 OUTOP(INDEX_op_shr, TCGOutOpBinary, outop_shr),
1236 OUTOP(INDEX_op_st, TCGOutOpStore, outop_st),
1237 OUTOP(INDEX_op_st8, TCGOutOpStore, outop_st8),
1238 OUTOP(INDEX_op_st16, TCGOutOpStore, outop_st16),
1239 OUTOP(INDEX_op_sub, TCGOutOpSubtract, outop_sub),
1240 OUTOP(INDEX_op_subbi, TCGOutOpAddSubCarry, outop_subbi),
1241 OUTOP(INDEX_op_subbio, TCGOutOpAddSubCarry, outop_subbio),
1242 OUTOP(INDEX_op_subbo, TCGOutOpAddSubCarry, outop_subbo),
1243 /* subb1o is implemented with set_borrow + subbio */
1244 OUTOP(INDEX_op_subb1o, TCGOutOpAddSubCarry, outop_subbio),
1245 OUTOP(INDEX_op_xor, TCGOutOpBinary, outop_xor),
1246
1247 [INDEX_op_goto_ptr] = &outop_goto_ptr,
1248
1249 #if TCG_TARGET_REG_BITS == 32
1250 OUTOP(INDEX_op_brcond2_i32, TCGOutOpBrcond2, outop_brcond2),
1251 OUTOP(INDEX_op_setcond2_i32, TCGOutOpSetcond2, outop_setcond2),
1252 #else
1253 OUTOP(INDEX_op_bswap64, TCGOutOpUnary, outop_bswap64),
1254 OUTOP(INDEX_op_ext_i32_i64, TCGOutOpUnary, outop_exts_i32_i64),
1255 OUTOP(INDEX_op_extu_i32_i64, TCGOutOpUnary, outop_extu_i32_i64),
1256 OUTOP(INDEX_op_extrl_i64_i32, TCGOutOpUnary, outop_extrl_i64_i32),
1257 OUTOP(INDEX_op_extrh_i64_i32, TCGOutOpUnary, outop_extrh_i64_i32),
1258 OUTOP(INDEX_op_ld32u, TCGOutOpLoad, outop_ld32u),
1259 OUTOP(INDEX_op_ld32s, TCGOutOpLoad, outop_ld32s),
1260 OUTOP(INDEX_op_st32, TCGOutOpStore, outop_st),
1261 #endif
1262 };
1263
1264 #undef OUTOP
1265
1266 /*
1267 * All TCG threads except the parent (i.e. the one that called tcg_context_init
1268 * and registered the target's TCG globals) must register with this function
1269 * before initiating translation.
1270 *
1271 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation
1272 * of tcg_region_init() for the reasoning behind this.
1273 *
1274 * In system-mode each caller registers its context in tcg_ctxs[]. Note that in
1275 * system-mode tcg_ctxs[] does not track tcg_ctx_init, since the initial context
1276 * is not used anymore for translation once this function is called.
1277 *
1278 * Not tracking tcg_init_ctx in tcg_ctxs[] in system-mode keeps code that
1279 * iterates over the array (e.g. tcg_code_size() the same for both system/user
1280 * modes.
1281 */
1282 #ifdef CONFIG_USER_ONLY
tcg_register_thread(void)1283 void tcg_register_thread(void)
1284 {
1285 tcg_ctx = &tcg_init_ctx;
1286 }
1287 #else
tcg_register_thread(void)1288 void tcg_register_thread(void)
1289 {
1290 TCGContext *s = g_malloc(sizeof(*s));
1291 unsigned int i, n;
1292
1293 *s = tcg_init_ctx;
1294
1295 /* Relink mem_base. */
1296 for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) {
1297 if (tcg_init_ctx.temps[i].mem_base) {
1298 ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps;
1299 tcg_debug_assert(b >= 0 && b < n);
1300 s->temps[i].mem_base = &s->temps[b];
1301 }
1302 }
1303
1304 /* Claim an entry in tcg_ctxs */
1305 n = qatomic_fetch_inc(&tcg_cur_ctxs);
1306 g_assert(n < tcg_max_ctxs);
1307 qatomic_set(&tcg_ctxs[n], s);
1308
1309 if (n > 0) {
1310 tcg_region_initial_alloc(s);
1311 }
1312
1313 tcg_ctx = s;
1314 }
1315 #endif /* !CONFIG_USER_ONLY */
1316
1317 /* pool based memory allocation */
tcg_malloc_internal(TCGContext * s,int size)1318 void *tcg_malloc_internal(TCGContext *s, int size)
1319 {
1320 TCGPool *p;
1321 int pool_size;
1322
1323 if (size > TCG_POOL_CHUNK_SIZE) {
1324 /* big malloc: insert a new pool (XXX: could optimize) */
1325 p = g_malloc(sizeof(TCGPool) + size);
1326 p->size = size;
1327 p->next = s->pool_first_large;
1328 s->pool_first_large = p;
1329 return p->data;
1330 } else {
1331 p = s->pool_current;
1332 if (!p) {
1333 p = s->pool_first;
1334 if (!p)
1335 goto new_pool;
1336 } else {
1337 if (!p->next) {
1338 new_pool:
1339 pool_size = TCG_POOL_CHUNK_SIZE;
1340 p = g_malloc(sizeof(TCGPool) + pool_size);
1341 p->size = pool_size;
1342 p->next = NULL;
1343 if (s->pool_current) {
1344 s->pool_current->next = p;
1345 } else {
1346 s->pool_first = p;
1347 }
1348 } else {
1349 p = p->next;
1350 }
1351 }
1352 }
1353 s->pool_current = p;
1354 s->pool_cur = p->data + size;
1355 s->pool_end = p->data + p->size;
1356 return p->data;
1357 }
1358
tcg_pool_reset(TCGContext * s)1359 void tcg_pool_reset(TCGContext *s)
1360 {
1361 TCGPool *p, *t;
1362 for (p = s->pool_first_large; p; p = t) {
1363 t = p->next;
1364 g_free(p);
1365 }
1366 s->pool_first_large = NULL;
1367 s->pool_cur = s->pool_end = NULL;
1368 s->pool_current = NULL;
1369 }
1370
1371 /*
1372 * Create TCGHelperInfo structures for "tcg/tcg-ldst.h" functions,
1373 * akin to what "exec/helper-tcg.h" does with DEF_HELPER_FLAGS_N.
1374 * We only use these for layout in tcg_out_ld_helper_ret and
1375 * tcg_out_st_helper_args, and share them between several of
1376 * the helpers, with the end result that it's easier to build manually.
1377 */
1378
1379 #if TCG_TARGET_REG_BITS == 32
1380 # define dh_typecode_ttl dh_typecode_i32
1381 #else
1382 # define dh_typecode_ttl dh_typecode_i64
1383 #endif
1384
1385 static TCGHelperInfo info_helper_ld32_mmu = {
1386 .flags = TCG_CALL_NO_WG,
1387 .typemask = dh_typemask(ttl, 0) /* return tcg_target_ulong */
1388 | dh_typemask(env, 1)
1389 | dh_typemask(i64, 2) /* uint64_t addr */
1390 | dh_typemask(i32, 3) /* unsigned oi */
1391 | dh_typemask(ptr, 4) /* uintptr_t ra */
1392 };
1393
1394 static TCGHelperInfo info_helper_ld64_mmu = {
1395 .flags = TCG_CALL_NO_WG,
1396 .typemask = dh_typemask(i64, 0) /* return uint64_t */
1397 | dh_typemask(env, 1)
1398 | dh_typemask(i64, 2) /* uint64_t addr */
1399 | dh_typemask(i32, 3) /* unsigned oi */
1400 | dh_typemask(ptr, 4) /* uintptr_t ra */
1401 };
1402
1403 static TCGHelperInfo info_helper_ld128_mmu = {
1404 .flags = TCG_CALL_NO_WG,
1405 .typemask = dh_typemask(i128, 0) /* return Int128 */
1406 | dh_typemask(env, 1)
1407 | dh_typemask(i64, 2) /* uint64_t addr */
1408 | dh_typemask(i32, 3) /* unsigned oi */
1409 | dh_typemask(ptr, 4) /* uintptr_t ra */
1410 };
1411
1412 static TCGHelperInfo info_helper_st32_mmu = {
1413 .flags = TCG_CALL_NO_WG,
1414 .typemask = dh_typemask(void, 0)
1415 | dh_typemask(env, 1)
1416 | dh_typemask(i64, 2) /* uint64_t addr */
1417 | dh_typemask(i32, 3) /* uint32_t data */
1418 | dh_typemask(i32, 4) /* unsigned oi */
1419 | dh_typemask(ptr, 5) /* uintptr_t ra */
1420 };
1421
1422 static TCGHelperInfo info_helper_st64_mmu = {
1423 .flags = TCG_CALL_NO_WG,
1424 .typemask = dh_typemask(void, 0)
1425 | dh_typemask(env, 1)
1426 | dh_typemask(i64, 2) /* uint64_t addr */
1427 | dh_typemask(i64, 3) /* uint64_t data */
1428 | dh_typemask(i32, 4) /* unsigned oi */
1429 | dh_typemask(ptr, 5) /* uintptr_t ra */
1430 };
1431
1432 static TCGHelperInfo info_helper_st128_mmu = {
1433 .flags = TCG_CALL_NO_WG,
1434 .typemask = dh_typemask(void, 0)
1435 | dh_typemask(env, 1)
1436 | dh_typemask(i64, 2) /* uint64_t addr */
1437 | dh_typemask(i128, 3) /* Int128 data */
1438 | dh_typemask(i32, 4) /* unsigned oi */
1439 | dh_typemask(ptr, 5) /* uintptr_t ra */
1440 };
1441
1442 #ifdef CONFIG_TCG_INTERPRETER
typecode_to_ffi(int argmask)1443 static ffi_type *typecode_to_ffi(int argmask)
1444 {
1445 /*
1446 * libffi does not support __int128_t, so we have forced Int128
1447 * to use the structure definition instead of the builtin type.
1448 */
1449 static ffi_type *ffi_type_i128_elements[3] = {
1450 &ffi_type_uint64,
1451 &ffi_type_uint64,
1452 NULL
1453 };
1454 static ffi_type ffi_type_i128 = {
1455 .size = 16,
1456 .alignment = __alignof__(Int128),
1457 .type = FFI_TYPE_STRUCT,
1458 .elements = ffi_type_i128_elements,
1459 };
1460
1461 switch (argmask) {
1462 case dh_typecode_void:
1463 return &ffi_type_void;
1464 case dh_typecode_i32:
1465 return &ffi_type_uint32;
1466 case dh_typecode_s32:
1467 return &ffi_type_sint32;
1468 case dh_typecode_i64:
1469 return &ffi_type_uint64;
1470 case dh_typecode_s64:
1471 return &ffi_type_sint64;
1472 case dh_typecode_ptr:
1473 return &ffi_type_pointer;
1474 case dh_typecode_i128:
1475 return &ffi_type_i128;
1476 }
1477 g_assert_not_reached();
1478 }
1479
init_ffi_layout(TCGHelperInfo * info)1480 static ffi_cif *init_ffi_layout(TCGHelperInfo *info)
1481 {
1482 unsigned typemask = info->typemask;
1483 struct {
1484 ffi_cif cif;
1485 ffi_type *args[];
1486 } *ca;
1487 ffi_status status;
1488 int nargs;
1489
1490 /* Ignoring the return type, find the last non-zero field. */
1491 nargs = 32 - clz32(typemask >> 3);
1492 nargs = DIV_ROUND_UP(nargs, 3);
1493 assert(nargs <= MAX_CALL_IARGS);
1494
1495 ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *));
1496 ca->cif.rtype = typecode_to_ffi(typemask & 7);
1497 ca->cif.nargs = nargs;
1498
1499 if (nargs != 0) {
1500 ca->cif.arg_types = ca->args;
1501 for (int j = 0; j < nargs; ++j) {
1502 int typecode = extract32(typemask, (j + 1) * 3, 3);
1503 ca->args[j] = typecode_to_ffi(typecode);
1504 }
1505 }
1506
1507 status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs,
1508 ca->cif.rtype, ca->cif.arg_types);
1509 assert(status == FFI_OK);
1510
1511 return &ca->cif;
1512 }
1513
1514 #define HELPER_INFO_INIT(I) (&(I)->cif)
1515 #define HELPER_INFO_INIT_VAL(I) init_ffi_layout(I)
1516 #else
1517 #define HELPER_INFO_INIT(I) (&(I)->init)
1518 #define HELPER_INFO_INIT_VAL(I) 1
1519 #endif /* CONFIG_TCG_INTERPRETER */
1520
arg_slot_reg_p(unsigned arg_slot)1521 static inline bool arg_slot_reg_p(unsigned arg_slot)
1522 {
1523 /*
1524 * Split the sizeof away from the comparison to avoid Werror from
1525 * "unsigned < 0 is always false", when iarg_regs is empty.
1526 */
1527 unsigned nreg = ARRAY_SIZE(tcg_target_call_iarg_regs);
1528 return arg_slot < nreg;
1529 }
1530
arg_slot_stk_ofs(unsigned arg_slot)1531 static inline int arg_slot_stk_ofs(unsigned arg_slot)
1532 {
1533 unsigned max = TCG_STATIC_CALL_ARGS_SIZE / sizeof(tcg_target_long);
1534 unsigned stk_slot = arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs);
1535
1536 tcg_debug_assert(stk_slot < max);
1537 return TCG_TARGET_CALL_STACK_OFFSET + stk_slot * sizeof(tcg_target_long);
1538 }
1539
1540 typedef struct TCGCumulativeArgs {
1541 int arg_idx; /* tcg_gen_callN args[] */
1542 int info_in_idx; /* TCGHelperInfo in[] */
1543 int arg_slot; /* regs+stack slot */
1544 int ref_slot; /* stack slots for references */
1545 } TCGCumulativeArgs;
1546
layout_arg_even(TCGCumulativeArgs * cum)1547 static void layout_arg_even(TCGCumulativeArgs *cum)
1548 {
1549 cum->arg_slot += cum->arg_slot & 1;
1550 }
1551
layout_arg_1(TCGCumulativeArgs * cum,TCGHelperInfo * info,TCGCallArgumentKind kind)1552 static void layout_arg_1(TCGCumulativeArgs *cum, TCGHelperInfo *info,
1553 TCGCallArgumentKind kind)
1554 {
1555 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
1556
1557 *loc = (TCGCallArgumentLoc){
1558 .kind = kind,
1559 .arg_idx = cum->arg_idx,
1560 .arg_slot = cum->arg_slot,
1561 };
1562 cum->info_in_idx++;
1563 cum->arg_slot++;
1564 }
1565
layout_arg_normal_n(TCGCumulativeArgs * cum,TCGHelperInfo * info,int n)1566 static void layout_arg_normal_n(TCGCumulativeArgs *cum,
1567 TCGHelperInfo *info, int n)
1568 {
1569 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
1570
1571 for (int i = 0; i < n; ++i) {
1572 /* Layout all using the same arg_idx, adjusting the subindex. */
1573 loc[i] = (TCGCallArgumentLoc){
1574 .kind = TCG_CALL_ARG_NORMAL,
1575 .arg_idx = cum->arg_idx,
1576 .tmp_subindex = i,
1577 .arg_slot = cum->arg_slot + i,
1578 };
1579 }
1580 cum->info_in_idx += n;
1581 cum->arg_slot += n;
1582 }
1583
layout_arg_by_ref(TCGCumulativeArgs * cum,TCGHelperInfo * info)1584 static void layout_arg_by_ref(TCGCumulativeArgs *cum, TCGHelperInfo *info)
1585 {
1586 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
1587 int n = 128 / TCG_TARGET_REG_BITS;
1588
1589 /* The first subindex carries the pointer. */
1590 layout_arg_1(cum, info, TCG_CALL_ARG_BY_REF);
1591
1592 /*
1593 * The callee is allowed to clobber memory associated with
1594 * structure pass by-reference. Therefore we must make copies.
1595 * Allocate space from "ref_slot", which will be adjusted to
1596 * follow the parameters on the stack.
1597 */
1598 loc[0].ref_slot = cum->ref_slot;
1599
1600 /*
1601 * Subsequent words also go into the reference slot, but
1602 * do not accumulate into the regular arguments.
1603 */
1604 for (int i = 1; i < n; ++i) {
1605 loc[i] = (TCGCallArgumentLoc){
1606 .kind = TCG_CALL_ARG_BY_REF_N,
1607 .arg_idx = cum->arg_idx,
1608 .tmp_subindex = i,
1609 .ref_slot = cum->ref_slot + i,
1610 };
1611 }
1612 cum->info_in_idx += n - 1; /* i=0 accounted for in layout_arg_1 */
1613 cum->ref_slot += n;
1614 }
1615
init_call_layout(TCGHelperInfo * info)1616 static void init_call_layout(TCGHelperInfo *info)
1617 {
1618 int max_reg_slots = ARRAY_SIZE(tcg_target_call_iarg_regs);
1619 int max_stk_slots = TCG_STATIC_CALL_ARGS_SIZE / sizeof(tcg_target_long);
1620 unsigned typemask = info->typemask;
1621 unsigned typecode;
1622 TCGCumulativeArgs cum = { };
1623
1624 /*
1625 * Parse and place any function return value.
1626 */
1627 typecode = typemask & 7;
1628 switch (typecode) {
1629 case dh_typecode_void:
1630 info->nr_out = 0;
1631 break;
1632 case dh_typecode_i32:
1633 case dh_typecode_s32:
1634 case dh_typecode_ptr:
1635 info->nr_out = 1;
1636 info->out_kind = TCG_CALL_RET_NORMAL;
1637 break;
1638 case dh_typecode_i64:
1639 case dh_typecode_s64:
1640 info->nr_out = 64 / TCG_TARGET_REG_BITS;
1641 info->out_kind = TCG_CALL_RET_NORMAL;
1642 /* Query the last register now to trigger any assert early. */
1643 tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1);
1644 break;
1645 case dh_typecode_i128:
1646 info->nr_out = 128 / TCG_TARGET_REG_BITS;
1647 info->out_kind = TCG_TARGET_CALL_RET_I128;
1648 switch (TCG_TARGET_CALL_RET_I128) {
1649 case TCG_CALL_RET_NORMAL:
1650 /* Query the last register now to trigger any assert early. */
1651 tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1);
1652 break;
1653 case TCG_CALL_RET_BY_VEC:
1654 /* Query the single register now to trigger any assert early. */
1655 tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0);
1656 break;
1657 case TCG_CALL_RET_BY_REF:
1658 /*
1659 * Allocate the first argument to the output.
1660 * We don't need to store this anywhere, just make it
1661 * unavailable for use in the input loop below.
1662 */
1663 cum.arg_slot = 1;
1664 break;
1665 default:
1666 qemu_build_not_reached();
1667 }
1668 break;
1669 default:
1670 g_assert_not_reached();
1671 }
1672
1673 /*
1674 * Parse and place function arguments.
1675 */
1676 for (typemask >>= 3; typemask; typemask >>= 3, cum.arg_idx++) {
1677 TCGCallArgumentKind kind;
1678 TCGType type;
1679
1680 typecode = typemask & 7;
1681 switch (typecode) {
1682 case dh_typecode_i32:
1683 case dh_typecode_s32:
1684 type = TCG_TYPE_I32;
1685 break;
1686 case dh_typecode_i64:
1687 case dh_typecode_s64:
1688 type = TCG_TYPE_I64;
1689 break;
1690 case dh_typecode_ptr:
1691 type = TCG_TYPE_PTR;
1692 break;
1693 case dh_typecode_i128:
1694 type = TCG_TYPE_I128;
1695 break;
1696 default:
1697 g_assert_not_reached();
1698 }
1699
1700 switch (type) {
1701 case TCG_TYPE_I32:
1702 switch (TCG_TARGET_CALL_ARG_I32) {
1703 case TCG_CALL_ARG_EVEN:
1704 layout_arg_even(&cum);
1705 /* fall through */
1706 case TCG_CALL_ARG_NORMAL:
1707 layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL);
1708 break;
1709 case TCG_CALL_ARG_EXTEND:
1710 kind = TCG_CALL_ARG_EXTEND_U + (typecode & 1);
1711 layout_arg_1(&cum, info, kind);
1712 break;
1713 default:
1714 qemu_build_not_reached();
1715 }
1716 break;
1717
1718 case TCG_TYPE_I64:
1719 switch (TCG_TARGET_CALL_ARG_I64) {
1720 case TCG_CALL_ARG_EVEN:
1721 layout_arg_even(&cum);
1722 /* fall through */
1723 case TCG_CALL_ARG_NORMAL:
1724 if (TCG_TARGET_REG_BITS == 32) {
1725 layout_arg_normal_n(&cum, info, 2);
1726 } else {
1727 layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL);
1728 }
1729 break;
1730 default:
1731 qemu_build_not_reached();
1732 }
1733 break;
1734
1735 case TCG_TYPE_I128:
1736 switch (TCG_TARGET_CALL_ARG_I128) {
1737 case TCG_CALL_ARG_EVEN:
1738 layout_arg_even(&cum);
1739 /* fall through */
1740 case TCG_CALL_ARG_NORMAL:
1741 layout_arg_normal_n(&cum, info, 128 / TCG_TARGET_REG_BITS);
1742 break;
1743 case TCG_CALL_ARG_BY_REF:
1744 layout_arg_by_ref(&cum, info);
1745 break;
1746 default:
1747 qemu_build_not_reached();
1748 }
1749 break;
1750
1751 default:
1752 g_assert_not_reached();
1753 }
1754 }
1755 info->nr_in = cum.info_in_idx;
1756
1757 /* Validate that we didn't overrun the input array. */
1758 assert(cum.info_in_idx <= ARRAY_SIZE(info->in));
1759 /* Validate the backend has enough argument space. */
1760 assert(cum.arg_slot <= max_reg_slots + max_stk_slots);
1761
1762 /*
1763 * Relocate the "ref_slot" area to the end of the parameters.
1764 * Minimizing this stack offset helps code size for x86,
1765 * which has a signed 8-bit offset encoding.
1766 */
1767 if (cum.ref_slot != 0) {
1768 int ref_base = 0;
1769
1770 if (cum.arg_slot > max_reg_slots) {
1771 int align = __alignof(Int128) / sizeof(tcg_target_long);
1772
1773 ref_base = cum.arg_slot - max_reg_slots;
1774 if (align > 1) {
1775 ref_base = ROUND_UP(ref_base, align);
1776 }
1777 }
1778 assert(ref_base + cum.ref_slot <= max_stk_slots);
1779 ref_base += max_reg_slots;
1780
1781 if (ref_base != 0) {
1782 for (int i = cum.info_in_idx - 1; i >= 0; --i) {
1783 TCGCallArgumentLoc *loc = &info->in[i];
1784 switch (loc->kind) {
1785 case TCG_CALL_ARG_BY_REF:
1786 case TCG_CALL_ARG_BY_REF_N:
1787 loc->ref_slot += ref_base;
1788 break;
1789 default:
1790 break;
1791 }
1792 }
1793 }
1794 }
1795 }
1796
1797 static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)];
1798 static void process_constraint_sets(void);
1799 static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
1800 TCGReg reg, const char *name);
1801
tcg_context_init(unsigned max_threads)1802 static void tcg_context_init(unsigned max_threads)
1803 {
1804 TCGContext *s = &tcg_init_ctx;
1805 int n, i;
1806 TCGTemp *ts;
1807
1808 memset(s, 0, sizeof(*s));
1809 s->nb_globals = 0;
1810
1811 init_call_layout(&info_helper_ld32_mmu);
1812 init_call_layout(&info_helper_ld64_mmu);
1813 init_call_layout(&info_helper_ld128_mmu);
1814 init_call_layout(&info_helper_st32_mmu);
1815 init_call_layout(&info_helper_st64_mmu);
1816 init_call_layout(&info_helper_st128_mmu);
1817
1818 tcg_target_init(s);
1819 process_constraint_sets();
1820
1821 /* Reverse the order of the saved registers, assuming they're all at
1822 the start of tcg_target_reg_alloc_order. */
1823 for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) {
1824 int r = tcg_target_reg_alloc_order[n];
1825 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) {
1826 break;
1827 }
1828 }
1829 for (i = 0; i < n; ++i) {
1830 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i];
1831 }
1832 for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) {
1833 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i];
1834 }
1835
1836 tcg_ctx = s;
1837 /*
1838 * In user-mode we simply share the init context among threads, since we
1839 * use a single region. See the documentation tcg_region_init() for the
1840 * reasoning behind this.
1841 * In system-mode we will have at most max_threads TCG threads.
1842 */
1843 #ifdef CONFIG_USER_ONLY
1844 tcg_ctxs = &tcg_ctx;
1845 tcg_cur_ctxs = 1;
1846 tcg_max_ctxs = 1;
1847 #else
1848 tcg_max_ctxs = max_threads;
1849 tcg_ctxs = g_new0(TCGContext *, max_threads);
1850 #endif
1851
1852 tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0));
1853 ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env");
1854 tcg_env = temp_tcgv_ptr(ts);
1855 }
1856
tcg_init(size_t tb_size,int splitwx,unsigned max_threads)1857 void tcg_init(size_t tb_size, int splitwx, unsigned max_threads)
1858 {
1859 tcg_context_init(max_threads);
1860 tcg_region_init(tb_size, splitwx, max_threads);
1861 }
1862
1863 /*
1864 * Allocate TBs right before their corresponding translated code, making
1865 * sure that TBs and code are on different cache lines.
1866 */
tcg_tb_alloc(TCGContext * s)1867 TranslationBlock *tcg_tb_alloc(TCGContext *s)
1868 {
1869 uintptr_t align = qemu_icache_linesize;
1870 TranslationBlock *tb;
1871 void *next;
1872
1873 retry:
1874 tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align);
1875 next = (void *)ROUND_UP((uintptr_t)(tb + 1), align);
1876
1877 if (unlikely(next > s->code_gen_highwater)) {
1878 if (tcg_region_alloc(s)) {
1879 return NULL;
1880 }
1881 goto retry;
1882 }
1883 qatomic_set(&s->code_gen_ptr, next);
1884 return tb;
1885 }
1886
tcg_prologue_init(void)1887 void tcg_prologue_init(void)
1888 {
1889 TCGContext *s = tcg_ctx;
1890 size_t prologue_size;
1891
1892 s->code_ptr = s->code_gen_ptr;
1893 s->code_buf = s->code_gen_ptr;
1894 s->data_gen_ptr = NULL;
1895
1896 #ifndef CONFIG_TCG_INTERPRETER
1897 tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr);
1898 #endif
1899
1900 s->pool_labels = NULL;
1901
1902 qemu_thread_jit_write();
1903 /* Generate the prologue. */
1904 tcg_target_qemu_prologue(s);
1905
1906 /* Allow the prologue to put e.g. guest_base into a pool entry. */
1907 {
1908 int result = tcg_out_pool_finalize(s);
1909 tcg_debug_assert(result == 0);
1910 }
1911
1912 prologue_size = tcg_current_code_size(s);
1913 perf_report_prologue(s->code_gen_ptr, prologue_size);
1914
1915 #ifndef CONFIG_TCG_INTERPRETER
1916 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
1917 (uintptr_t)s->code_buf, prologue_size);
1918 #endif
1919
1920 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
1921 FILE *logfile = qemu_log_trylock();
1922 if (logfile) {
1923 fprintf(logfile, "PROLOGUE: [size=%zu]\n", prologue_size);
1924 if (s->data_gen_ptr) {
1925 size_t code_size = s->data_gen_ptr - s->code_gen_ptr;
1926 size_t data_size = prologue_size - code_size;
1927 size_t i;
1928
1929 disas(logfile, s->code_gen_ptr, code_size);
1930
1931 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) {
1932 if (sizeof(tcg_target_ulong) == 8) {
1933 fprintf(logfile,
1934 "0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n",
1935 (uintptr_t)s->data_gen_ptr + i,
1936 *(uint64_t *)(s->data_gen_ptr + i));
1937 } else {
1938 fprintf(logfile,
1939 "0x%08" PRIxPTR ": .long 0x%08x\n",
1940 (uintptr_t)s->data_gen_ptr + i,
1941 *(uint32_t *)(s->data_gen_ptr + i));
1942 }
1943 }
1944 } else {
1945 disas(logfile, s->code_gen_ptr, prologue_size);
1946 }
1947 fprintf(logfile, "\n");
1948 qemu_log_unlock(logfile);
1949 }
1950 }
1951
1952 #ifndef CONFIG_TCG_INTERPRETER
1953 /*
1954 * Assert that goto_ptr is implemented completely, setting an epilogue.
1955 * For tci, we use NULL as the signal to return from the interpreter,
1956 * so skip this check.
1957 */
1958 tcg_debug_assert(tcg_code_gen_epilogue != NULL);
1959 #endif
1960
1961 tcg_region_prologue_set(s);
1962 }
1963
tcg_func_start(TCGContext * s)1964 void tcg_func_start(TCGContext *s)
1965 {
1966 tcg_pool_reset(s);
1967 s->nb_temps = s->nb_globals;
1968
1969 /* No temps have been previously allocated for size or locality. */
1970 tcg_temp_ebb_reset_freed(s);
1971
1972 /* No constant temps have been previously allocated. */
1973 for (int i = 0; i < TCG_TYPE_COUNT; ++i) {
1974 if (s->const_table[i]) {
1975 g_hash_table_remove_all(s->const_table[i]);
1976 }
1977 }
1978
1979 s->nb_ops = 0;
1980 s->nb_labels = 0;
1981 s->current_frame_offset = s->frame_start;
1982
1983 #ifdef CONFIG_DEBUG_TCG
1984 s->goto_tb_issue_mask = 0;
1985 #endif
1986
1987 QTAILQ_INIT(&s->ops);
1988 QTAILQ_INIT(&s->free_ops);
1989 s->emit_before_op = NULL;
1990 QSIMPLEQ_INIT(&s->labels);
1991
1992 tcg_debug_assert(s->addr_type <= TCG_TYPE_REG);
1993 }
1994
tcg_temp_alloc(TCGContext * s)1995 static TCGTemp *tcg_temp_alloc(TCGContext *s)
1996 {
1997 int n = s->nb_temps++;
1998
1999 if (n >= TCG_MAX_TEMPS) {
2000 tcg_raise_tb_overflow(s);
2001 }
2002 return memset(&s->temps[n], 0, sizeof(TCGTemp));
2003 }
2004
tcg_global_alloc(TCGContext * s)2005 static TCGTemp *tcg_global_alloc(TCGContext *s)
2006 {
2007 TCGTemp *ts;
2008
2009 tcg_debug_assert(s->nb_globals == s->nb_temps);
2010 tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS);
2011 s->nb_globals++;
2012 ts = tcg_temp_alloc(s);
2013 ts->kind = TEMP_GLOBAL;
2014
2015 return ts;
2016 }
2017
tcg_global_reg_new_internal(TCGContext * s,TCGType type,TCGReg reg,const char * name)2018 static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
2019 TCGReg reg, const char *name)
2020 {
2021 TCGTemp *ts;
2022
2023 tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32);
2024
2025 ts = tcg_global_alloc(s);
2026 ts->base_type = type;
2027 ts->type = type;
2028 ts->kind = TEMP_FIXED;
2029 ts->reg = reg;
2030 ts->name = name;
2031 tcg_regset_set_reg(s->reserved_regs, reg);
2032
2033 return ts;
2034 }
2035
tcg_set_frame(TCGContext * s,TCGReg reg,intptr_t start,intptr_t size)2036 void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size)
2037 {
2038 s->frame_start = start;
2039 s->frame_end = start + size;
2040 s->frame_temp
2041 = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame");
2042 }
2043
tcg_global_mem_new_internal(TCGv_ptr base,intptr_t offset,const char * name,TCGType type)2044 static TCGTemp *tcg_global_mem_new_internal(TCGv_ptr base, intptr_t offset,
2045 const char *name, TCGType type)
2046 {
2047 TCGContext *s = tcg_ctx;
2048 TCGTemp *base_ts = tcgv_ptr_temp(base);
2049 TCGTemp *ts = tcg_global_alloc(s);
2050 int indirect_reg = 0;
2051
2052 switch (base_ts->kind) {
2053 case TEMP_FIXED:
2054 break;
2055 case TEMP_GLOBAL:
2056 /* We do not support double-indirect registers. */
2057 tcg_debug_assert(!base_ts->indirect_reg);
2058 base_ts->indirect_base = 1;
2059 s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64
2060 ? 2 : 1);
2061 indirect_reg = 1;
2062 break;
2063 default:
2064 g_assert_not_reached();
2065 }
2066
2067 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
2068 TCGTemp *ts2 = tcg_global_alloc(s);
2069 char buf[64];
2070
2071 ts->base_type = TCG_TYPE_I64;
2072 ts->type = TCG_TYPE_I32;
2073 ts->indirect_reg = indirect_reg;
2074 ts->mem_allocated = 1;
2075 ts->mem_base = base_ts;
2076 ts->mem_offset = offset;
2077 pstrcpy(buf, sizeof(buf), name);
2078 pstrcat(buf, sizeof(buf), "_0");
2079 ts->name = strdup(buf);
2080
2081 tcg_debug_assert(ts2 == ts + 1);
2082 ts2->base_type = TCG_TYPE_I64;
2083 ts2->type = TCG_TYPE_I32;
2084 ts2->indirect_reg = indirect_reg;
2085 ts2->mem_allocated = 1;
2086 ts2->mem_base = base_ts;
2087 ts2->mem_offset = offset + 4;
2088 ts2->temp_subindex = 1;
2089 pstrcpy(buf, sizeof(buf), name);
2090 pstrcat(buf, sizeof(buf), "_1");
2091 ts2->name = strdup(buf);
2092 } else {
2093 ts->base_type = type;
2094 ts->type = type;
2095 ts->indirect_reg = indirect_reg;
2096 ts->mem_allocated = 1;
2097 ts->mem_base = base_ts;
2098 ts->mem_offset = offset;
2099 ts->name = name;
2100 }
2101 return ts;
2102 }
2103
tcg_global_mem_new_i32(TCGv_ptr reg,intptr_t off,const char * name)2104 TCGv_i32 tcg_global_mem_new_i32(TCGv_ptr reg, intptr_t off, const char *name)
2105 {
2106 TCGTemp *ts = tcg_global_mem_new_internal(reg, off, name, TCG_TYPE_I32);
2107 return temp_tcgv_i32(ts);
2108 }
2109
tcg_global_mem_new_i64(TCGv_ptr reg,intptr_t off,const char * name)2110 TCGv_i64 tcg_global_mem_new_i64(TCGv_ptr reg, intptr_t off, const char *name)
2111 {
2112 TCGTemp *ts = tcg_global_mem_new_internal(reg, off, name, TCG_TYPE_I64);
2113 return temp_tcgv_i64(ts);
2114 }
2115
tcg_global_mem_new_ptr(TCGv_ptr reg,intptr_t off,const char * name)2116 TCGv_ptr tcg_global_mem_new_ptr(TCGv_ptr reg, intptr_t off, const char *name)
2117 {
2118 TCGTemp *ts = tcg_global_mem_new_internal(reg, off, name, TCG_TYPE_PTR);
2119 return temp_tcgv_ptr(ts);
2120 }
2121
tcg_temp_new_internal(TCGType type,TCGTempKind kind)2122 TCGTemp *tcg_temp_new_internal(TCGType type, TCGTempKind kind)
2123 {
2124 TCGContext *s = tcg_ctx;
2125 TCGTemp *ts;
2126 int n;
2127
2128 if (kind == TEMP_EBB) {
2129 int idx = find_first_bit(s->free_temps[type].l, TCG_MAX_TEMPS);
2130
2131 if (idx < TCG_MAX_TEMPS) {
2132 /* There is already an available temp with the right type. */
2133 clear_bit(idx, s->free_temps[type].l);
2134
2135 ts = &s->temps[idx];
2136 ts->temp_allocated = 1;
2137 tcg_debug_assert(ts->base_type == type);
2138 tcg_debug_assert(ts->kind == kind);
2139 return ts;
2140 }
2141 } else {
2142 tcg_debug_assert(kind == TEMP_TB);
2143 }
2144
2145 switch (type) {
2146 case TCG_TYPE_I32:
2147 case TCG_TYPE_V64:
2148 case TCG_TYPE_V128:
2149 case TCG_TYPE_V256:
2150 n = 1;
2151 break;
2152 case TCG_TYPE_I64:
2153 n = 64 / TCG_TARGET_REG_BITS;
2154 break;
2155 case TCG_TYPE_I128:
2156 n = 128 / TCG_TARGET_REG_BITS;
2157 break;
2158 default:
2159 g_assert_not_reached();
2160 }
2161
2162 ts = tcg_temp_alloc(s);
2163 ts->base_type = type;
2164 ts->temp_allocated = 1;
2165 ts->kind = kind;
2166
2167 if (n == 1) {
2168 ts->type = type;
2169 } else {
2170 ts->type = TCG_TYPE_REG;
2171
2172 for (int i = 1; i < n; ++i) {
2173 TCGTemp *ts2 = tcg_temp_alloc(s);
2174
2175 tcg_debug_assert(ts2 == ts + i);
2176 ts2->base_type = type;
2177 ts2->type = TCG_TYPE_REG;
2178 ts2->temp_allocated = 1;
2179 ts2->temp_subindex = i;
2180 ts2->kind = kind;
2181 }
2182 }
2183 return ts;
2184 }
2185
tcg_temp_new_i32(void)2186 TCGv_i32 tcg_temp_new_i32(void)
2187 {
2188 return temp_tcgv_i32(tcg_temp_new_internal(TCG_TYPE_I32, TEMP_TB));
2189 }
2190
tcg_temp_ebb_new_i32(void)2191 TCGv_i32 tcg_temp_ebb_new_i32(void)
2192 {
2193 return temp_tcgv_i32(tcg_temp_new_internal(TCG_TYPE_I32, TEMP_EBB));
2194 }
2195
tcg_temp_new_i64(void)2196 TCGv_i64 tcg_temp_new_i64(void)
2197 {
2198 return temp_tcgv_i64(tcg_temp_new_internal(TCG_TYPE_I64, TEMP_TB));
2199 }
2200
tcg_temp_ebb_new_i64(void)2201 TCGv_i64 tcg_temp_ebb_new_i64(void)
2202 {
2203 return temp_tcgv_i64(tcg_temp_new_internal(TCG_TYPE_I64, TEMP_EBB));
2204 }
2205
tcg_temp_new_ptr(void)2206 TCGv_ptr tcg_temp_new_ptr(void)
2207 {
2208 return temp_tcgv_ptr(tcg_temp_new_internal(TCG_TYPE_PTR, TEMP_TB));
2209 }
2210
tcg_temp_ebb_new_ptr(void)2211 TCGv_ptr tcg_temp_ebb_new_ptr(void)
2212 {
2213 return temp_tcgv_ptr(tcg_temp_new_internal(TCG_TYPE_PTR, TEMP_EBB));
2214 }
2215
tcg_temp_new_i128(void)2216 TCGv_i128 tcg_temp_new_i128(void)
2217 {
2218 return temp_tcgv_i128(tcg_temp_new_internal(TCG_TYPE_I128, TEMP_TB));
2219 }
2220
tcg_temp_ebb_new_i128(void)2221 TCGv_i128 tcg_temp_ebb_new_i128(void)
2222 {
2223 return temp_tcgv_i128(tcg_temp_new_internal(TCG_TYPE_I128, TEMP_EBB));
2224 }
2225
tcg_temp_new_vec(TCGType type)2226 TCGv_vec tcg_temp_new_vec(TCGType type)
2227 {
2228 TCGTemp *t;
2229
2230 #ifdef CONFIG_DEBUG_TCG
2231 switch (type) {
2232 case TCG_TYPE_V64:
2233 assert(TCG_TARGET_HAS_v64);
2234 break;
2235 case TCG_TYPE_V128:
2236 assert(TCG_TARGET_HAS_v128);
2237 break;
2238 case TCG_TYPE_V256:
2239 assert(TCG_TARGET_HAS_v256);
2240 break;
2241 default:
2242 g_assert_not_reached();
2243 }
2244 #endif
2245
2246 t = tcg_temp_new_internal(type, TEMP_EBB);
2247 return temp_tcgv_vec(t);
2248 }
2249
2250 /* Create a new temp of the same type as an existing temp. */
tcg_temp_new_vec_matching(TCGv_vec match)2251 TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match)
2252 {
2253 TCGTemp *t = tcgv_vec_temp(match);
2254
2255 tcg_debug_assert(t->temp_allocated != 0);
2256
2257 t = tcg_temp_new_internal(t->base_type, TEMP_EBB);
2258 return temp_tcgv_vec(t);
2259 }
2260
tcg_temp_free_internal(TCGTemp * ts)2261 void tcg_temp_free_internal(TCGTemp *ts)
2262 {
2263 TCGContext *s = tcg_ctx;
2264
2265 switch (ts->kind) {
2266 case TEMP_CONST:
2267 case TEMP_TB:
2268 /* Silently ignore free. */
2269 break;
2270 case TEMP_EBB:
2271 tcg_debug_assert(ts->temp_allocated != 0);
2272 ts->temp_allocated = 0;
2273 set_bit(temp_idx(ts), s->free_temps[ts->base_type].l);
2274 break;
2275 default:
2276 /* It never made sense to free TEMP_FIXED or TEMP_GLOBAL. */
2277 g_assert_not_reached();
2278 }
2279 }
2280
tcg_temp_free_i32(TCGv_i32 arg)2281 void tcg_temp_free_i32(TCGv_i32 arg)
2282 {
2283 tcg_temp_free_internal(tcgv_i32_temp(arg));
2284 }
2285
tcg_temp_free_i64(TCGv_i64 arg)2286 void tcg_temp_free_i64(TCGv_i64 arg)
2287 {
2288 tcg_temp_free_internal(tcgv_i64_temp(arg));
2289 }
2290
tcg_temp_free_i128(TCGv_i128 arg)2291 void tcg_temp_free_i128(TCGv_i128 arg)
2292 {
2293 tcg_temp_free_internal(tcgv_i128_temp(arg));
2294 }
2295
tcg_temp_free_ptr(TCGv_ptr arg)2296 void tcg_temp_free_ptr(TCGv_ptr arg)
2297 {
2298 tcg_temp_free_internal(tcgv_ptr_temp(arg));
2299 }
2300
tcg_temp_free_vec(TCGv_vec arg)2301 void tcg_temp_free_vec(TCGv_vec arg)
2302 {
2303 tcg_temp_free_internal(tcgv_vec_temp(arg));
2304 }
2305
tcg_constant_internal(TCGType type,int64_t val)2306 TCGTemp *tcg_constant_internal(TCGType type, int64_t val)
2307 {
2308 TCGContext *s = tcg_ctx;
2309 GHashTable *h = s->const_table[type];
2310 TCGTemp *ts;
2311
2312 if (h == NULL) {
2313 h = g_hash_table_new(g_int64_hash, g_int64_equal);
2314 s->const_table[type] = h;
2315 }
2316
2317 ts = g_hash_table_lookup(h, &val);
2318 if (ts == NULL) {
2319 int64_t *val_ptr;
2320
2321 ts = tcg_temp_alloc(s);
2322
2323 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
2324 TCGTemp *ts2 = tcg_temp_alloc(s);
2325
2326 tcg_debug_assert(ts2 == ts + 1);
2327
2328 ts->base_type = TCG_TYPE_I64;
2329 ts->type = TCG_TYPE_I32;
2330 ts->kind = TEMP_CONST;
2331 ts->temp_allocated = 1;
2332
2333 ts2->base_type = TCG_TYPE_I64;
2334 ts2->type = TCG_TYPE_I32;
2335 ts2->kind = TEMP_CONST;
2336 ts2->temp_allocated = 1;
2337 ts2->temp_subindex = 1;
2338
2339 /*
2340 * Retain the full value of the 64-bit constant in the low
2341 * part, so that the hash table works. Actual uses will
2342 * truncate the value to the low part.
2343 */
2344 ts[HOST_BIG_ENDIAN].val = val;
2345 ts[!HOST_BIG_ENDIAN].val = val >> 32;
2346 val_ptr = &ts[HOST_BIG_ENDIAN].val;
2347 } else {
2348 ts->base_type = type;
2349 ts->type = type;
2350 ts->kind = TEMP_CONST;
2351 ts->temp_allocated = 1;
2352 ts->val = val;
2353 val_ptr = &ts->val;
2354 }
2355 g_hash_table_insert(h, val_ptr, ts);
2356 }
2357
2358 return ts;
2359 }
2360
tcg_constant_i32(int32_t val)2361 TCGv_i32 tcg_constant_i32(int32_t val)
2362 {
2363 return temp_tcgv_i32(tcg_constant_internal(TCG_TYPE_I32, val));
2364 }
2365
tcg_constant_i64(int64_t val)2366 TCGv_i64 tcg_constant_i64(int64_t val)
2367 {
2368 return temp_tcgv_i64(tcg_constant_internal(TCG_TYPE_I64, val));
2369 }
2370
tcg_constant_vaddr(uintptr_t val)2371 TCGv_vaddr tcg_constant_vaddr(uintptr_t val)
2372 {
2373 return temp_tcgv_vaddr(tcg_constant_internal(TCG_TYPE_PTR, val));
2374 }
2375
tcg_constant_ptr_int(intptr_t val)2376 TCGv_ptr tcg_constant_ptr_int(intptr_t val)
2377 {
2378 return temp_tcgv_ptr(tcg_constant_internal(TCG_TYPE_PTR, val));
2379 }
2380
tcg_constant_vec(TCGType type,unsigned vece,int64_t val)2381 TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val)
2382 {
2383 val = dup_const(vece, val);
2384 return temp_tcgv_vec(tcg_constant_internal(type, val));
2385 }
2386
tcg_constant_vec_matching(TCGv_vec match,unsigned vece,int64_t val)2387 TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val)
2388 {
2389 TCGTemp *t = tcgv_vec_temp(match);
2390
2391 tcg_debug_assert(t->temp_allocated != 0);
2392 return tcg_constant_vec(t->base_type, vece, val);
2393 }
2394
2395 #ifdef CONFIG_DEBUG_TCG
temp_idx(TCGTemp * ts)2396 size_t temp_idx(TCGTemp *ts)
2397 {
2398 ptrdiff_t n = ts - tcg_ctx->temps;
2399 assert(n >= 0 && n < tcg_ctx->nb_temps);
2400 return n;
2401 }
2402
tcgv_i32_temp(TCGv_i32 v)2403 TCGTemp *tcgv_i32_temp(TCGv_i32 v)
2404 {
2405 uintptr_t o = (uintptr_t)v - offsetof(TCGContext, temps);
2406
2407 assert(o < sizeof(TCGTemp) * tcg_ctx->nb_temps);
2408 assert(o % sizeof(TCGTemp) == 0);
2409
2410 return (void *)tcg_ctx + (uintptr_t)v;
2411 }
2412 #endif /* CONFIG_DEBUG_TCG */
2413
2414 /*
2415 * Return true if OP may appear in the opcode stream with TYPE.
2416 * Test the runtime variable that controls each opcode.
2417 */
tcg_op_supported(TCGOpcode op,TCGType type,unsigned flags)2418 bool tcg_op_supported(TCGOpcode op, TCGType type, unsigned flags)
2419 {
2420 bool has_type;
2421
2422 switch (type) {
2423 case TCG_TYPE_I32:
2424 has_type = true;
2425 break;
2426 case TCG_TYPE_I64:
2427 has_type = TCG_TARGET_REG_BITS == 64;
2428 break;
2429 case TCG_TYPE_V64:
2430 has_type = TCG_TARGET_HAS_v64;
2431 break;
2432 case TCG_TYPE_V128:
2433 has_type = TCG_TARGET_HAS_v128;
2434 break;
2435 case TCG_TYPE_V256:
2436 has_type = TCG_TARGET_HAS_v256;
2437 break;
2438 default:
2439 has_type = false;
2440 break;
2441 }
2442
2443 switch (op) {
2444 case INDEX_op_discard:
2445 case INDEX_op_set_label:
2446 case INDEX_op_call:
2447 case INDEX_op_br:
2448 case INDEX_op_mb:
2449 case INDEX_op_insn_start:
2450 case INDEX_op_exit_tb:
2451 case INDEX_op_goto_tb:
2452 case INDEX_op_goto_ptr:
2453 return true;
2454
2455 case INDEX_op_qemu_ld:
2456 case INDEX_op_qemu_st:
2457 tcg_debug_assert(type <= TCG_TYPE_REG);
2458 return true;
2459
2460 case INDEX_op_qemu_ld2:
2461 case INDEX_op_qemu_st2:
2462 if (TCG_TARGET_REG_BITS == 32) {
2463 tcg_debug_assert(type == TCG_TYPE_I64);
2464 return true;
2465 }
2466 tcg_debug_assert(type == TCG_TYPE_I128);
2467 goto do_lookup;
2468
2469 case INDEX_op_add:
2470 case INDEX_op_and:
2471 case INDEX_op_brcond:
2472 case INDEX_op_deposit:
2473 case INDEX_op_extract:
2474 case INDEX_op_ld8u:
2475 case INDEX_op_ld8s:
2476 case INDEX_op_ld16u:
2477 case INDEX_op_ld16s:
2478 case INDEX_op_ld:
2479 case INDEX_op_mov:
2480 case INDEX_op_movcond:
2481 case INDEX_op_negsetcond:
2482 case INDEX_op_or:
2483 case INDEX_op_setcond:
2484 case INDEX_op_sextract:
2485 case INDEX_op_st8:
2486 case INDEX_op_st16:
2487 case INDEX_op_st:
2488 case INDEX_op_xor:
2489 return has_type;
2490
2491 case INDEX_op_brcond2_i32:
2492 case INDEX_op_setcond2_i32:
2493 return TCG_TARGET_REG_BITS == 32;
2494
2495 case INDEX_op_ld32u:
2496 case INDEX_op_ld32s:
2497 case INDEX_op_st32:
2498 case INDEX_op_ext_i32_i64:
2499 case INDEX_op_extu_i32_i64:
2500 case INDEX_op_extrl_i64_i32:
2501 case INDEX_op_extrh_i64_i32:
2502 return TCG_TARGET_REG_BITS == 64;
2503
2504 case INDEX_op_mov_vec:
2505 case INDEX_op_dup_vec:
2506 case INDEX_op_dupm_vec:
2507 case INDEX_op_ld_vec:
2508 case INDEX_op_st_vec:
2509 case INDEX_op_add_vec:
2510 case INDEX_op_sub_vec:
2511 case INDEX_op_and_vec:
2512 case INDEX_op_or_vec:
2513 case INDEX_op_xor_vec:
2514 case INDEX_op_cmp_vec:
2515 return has_type;
2516 case INDEX_op_dup2_vec:
2517 return has_type && TCG_TARGET_REG_BITS == 32;
2518 case INDEX_op_not_vec:
2519 return has_type && TCG_TARGET_HAS_not_vec;
2520 case INDEX_op_neg_vec:
2521 return has_type && TCG_TARGET_HAS_neg_vec;
2522 case INDEX_op_abs_vec:
2523 return has_type && TCG_TARGET_HAS_abs_vec;
2524 case INDEX_op_andc_vec:
2525 return has_type && TCG_TARGET_HAS_andc_vec;
2526 case INDEX_op_orc_vec:
2527 return has_type && TCG_TARGET_HAS_orc_vec;
2528 case INDEX_op_nand_vec:
2529 return has_type && TCG_TARGET_HAS_nand_vec;
2530 case INDEX_op_nor_vec:
2531 return has_type && TCG_TARGET_HAS_nor_vec;
2532 case INDEX_op_eqv_vec:
2533 return has_type && TCG_TARGET_HAS_eqv_vec;
2534 case INDEX_op_mul_vec:
2535 return has_type && TCG_TARGET_HAS_mul_vec;
2536 case INDEX_op_shli_vec:
2537 case INDEX_op_shri_vec:
2538 case INDEX_op_sari_vec:
2539 return has_type && TCG_TARGET_HAS_shi_vec;
2540 case INDEX_op_shls_vec:
2541 case INDEX_op_shrs_vec:
2542 case INDEX_op_sars_vec:
2543 return has_type && TCG_TARGET_HAS_shs_vec;
2544 case INDEX_op_shlv_vec:
2545 case INDEX_op_shrv_vec:
2546 case INDEX_op_sarv_vec:
2547 return has_type && TCG_TARGET_HAS_shv_vec;
2548 case INDEX_op_rotli_vec:
2549 return has_type && TCG_TARGET_HAS_roti_vec;
2550 case INDEX_op_rotls_vec:
2551 return has_type && TCG_TARGET_HAS_rots_vec;
2552 case INDEX_op_rotlv_vec:
2553 case INDEX_op_rotrv_vec:
2554 return has_type && TCG_TARGET_HAS_rotv_vec;
2555 case INDEX_op_ssadd_vec:
2556 case INDEX_op_usadd_vec:
2557 case INDEX_op_sssub_vec:
2558 case INDEX_op_ussub_vec:
2559 return has_type && TCG_TARGET_HAS_sat_vec;
2560 case INDEX_op_smin_vec:
2561 case INDEX_op_umin_vec:
2562 case INDEX_op_smax_vec:
2563 case INDEX_op_umax_vec:
2564 return has_type && TCG_TARGET_HAS_minmax_vec;
2565 case INDEX_op_bitsel_vec:
2566 return has_type && TCG_TARGET_HAS_bitsel_vec;
2567 case INDEX_op_cmpsel_vec:
2568 return has_type && TCG_TARGET_HAS_cmpsel_vec;
2569
2570 default:
2571 if (op < INDEX_op_last_generic) {
2572 const TCGOutOp *outop;
2573 TCGConstraintSetIndex con_set;
2574
2575 if (!has_type) {
2576 return false;
2577 }
2578
2579 do_lookup:
2580 outop = all_outop[op];
2581 tcg_debug_assert(outop != NULL);
2582
2583 con_set = outop->static_constraint;
2584 if (con_set == C_Dynamic) {
2585 con_set = outop->dynamic_constraint(type, flags);
2586 }
2587 if (con_set >= 0) {
2588 return true;
2589 }
2590 tcg_debug_assert(con_set == C_NotImplemented);
2591 return false;
2592 }
2593 tcg_debug_assert(op < NB_OPS);
2594 return true;
2595
2596 case INDEX_op_last_generic:
2597 g_assert_not_reached();
2598 }
2599 }
2600
tcg_op_deposit_valid(TCGType type,unsigned ofs,unsigned len)2601 bool tcg_op_deposit_valid(TCGType type, unsigned ofs, unsigned len)
2602 {
2603 unsigned width;
2604
2605 tcg_debug_assert(type == TCG_TYPE_I32 || type == TCG_TYPE_I64);
2606 width = (type == TCG_TYPE_I32 ? 32 : 64);
2607
2608 tcg_debug_assert(ofs < width);
2609 tcg_debug_assert(len > 0);
2610 tcg_debug_assert(len <= width - ofs);
2611
2612 return TCG_TARGET_deposit_valid(type, ofs, len);
2613 }
2614
2615 static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs);
2616
tcg_gen_callN(void * func,TCGHelperInfo * info,TCGTemp * ret,TCGTemp ** args)2617 static void tcg_gen_callN(void *func, TCGHelperInfo *info,
2618 TCGTemp *ret, TCGTemp **args)
2619 {
2620 TCGv_i64 extend_free[MAX_CALL_IARGS];
2621 int n_extend = 0;
2622 TCGOp *op;
2623 int i, n, pi = 0, total_args;
2624
2625 if (unlikely(g_once_init_enter(HELPER_INFO_INIT(info)))) {
2626 init_call_layout(info);
2627 g_once_init_leave(HELPER_INFO_INIT(info), HELPER_INFO_INIT_VAL(info));
2628 }
2629
2630 total_args = info->nr_out + info->nr_in + 2;
2631 op = tcg_op_alloc(INDEX_op_call, total_args);
2632
2633 #ifdef CONFIG_PLUGIN
2634 /* Flag helpers that may affect guest state */
2635 if (tcg_ctx->plugin_insn && !(info->flags & TCG_CALL_NO_SIDE_EFFECTS)) {
2636 tcg_ctx->plugin_insn->calls_helpers = true;
2637 }
2638 #endif
2639
2640 TCGOP_CALLO(op) = n = info->nr_out;
2641 switch (n) {
2642 case 0:
2643 tcg_debug_assert(ret == NULL);
2644 break;
2645 case 1:
2646 tcg_debug_assert(ret != NULL);
2647 op->args[pi++] = temp_arg(ret);
2648 break;
2649 case 2:
2650 case 4:
2651 tcg_debug_assert(ret != NULL);
2652 tcg_debug_assert(ret->base_type == ret->type + ctz32(n));
2653 tcg_debug_assert(ret->temp_subindex == 0);
2654 for (i = 0; i < n; ++i) {
2655 op->args[pi++] = temp_arg(ret + i);
2656 }
2657 break;
2658 default:
2659 g_assert_not_reached();
2660 }
2661
2662 TCGOP_CALLI(op) = n = info->nr_in;
2663 for (i = 0; i < n; i++) {
2664 const TCGCallArgumentLoc *loc = &info->in[i];
2665 TCGTemp *ts = args[loc->arg_idx] + loc->tmp_subindex;
2666
2667 switch (loc->kind) {
2668 case TCG_CALL_ARG_NORMAL:
2669 case TCG_CALL_ARG_BY_REF:
2670 case TCG_CALL_ARG_BY_REF_N:
2671 op->args[pi++] = temp_arg(ts);
2672 break;
2673
2674 case TCG_CALL_ARG_EXTEND_U:
2675 case TCG_CALL_ARG_EXTEND_S:
2676 {
2677 TCGv_i64 temp = tcg_temp_ebb_new_i64();
2678 TCGv_i32 orig = temp_tcgv_i32(ts);
2679
2680 if (loc->kind == TCG_CALL_ARG_EXTEND_S) {
2681 tcg_gen_ext_i32_i64(temp, orig);
2682 } else {
2683 tcg_gen_extu_i32_i64(temp, orig);
2684 }
2685 op->args[pi++] = tcgv_i64_arg(temp);
2686 extend_free[n_extend++] = temp;
2687 }
2688 break;
2689
2690 default:
2691 g_assert_not_reached();
2692 }
2693 }
2694 op->args[pi++] = (uintptr_t)func;
2695 op->args[pi++] = (uintptr_t)info;
2696 tcg_debug_assert(pi == total_args);
2697
2698 if (tcg_ctx->emit_before_op) {
2699 QTAILQ_INSERT_BEFORE(tcg_ctx->emit_before_op, op, link);
2700 } else {
2701 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
2702 }
2703
2704 tcg_debug_assert(n_extend < ARRAY_SIZE(extend_free));
2705 for (i = 0; i < n_extend; ++i) {
2706 tcg_temp_free_i64(extend_free[i]);
2707 }
2708 }
2709
tcg_gen_call0(void * func,TCGHelperInfo * info,TCGTemp * ret)2710 void tcg_gen_call0(void *func, TCGHelperInfo *info, TCGTemp *ret)
2711 {
2712 tcg_gen_callN(func, info, ret, NULL);
2713 }
2714
tcg_gen_call1(void * func,TCGHelperInfo * info,TCGTemp * ret,TCGTemp * t1)2715 void tcg_gen_call1(void *func, TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1)
2716 {
2717 tcg_gen_callN(func, info, ret, &t1);
2718 }
2719
tcg_gen_call2(void * func,TCGHelperInfo * info,TCGTemp * ret,TCGTemp * t1,TCGTemp * t2)2720 void tcg_gen_call2(void *func, TCGHelperInfo *info, TCGTemp *ret,
2721 TCGTemp *t1, TCGTemp *t2)
2722 {
2723 TCGTemp *args[2] = { t1, t2 };
2724 tcg_gen_callN(func, info, ret, args);
2725 }
2726
tcg_gen_call3(void * func,TCGHelperInfo * info,TCGTemp * ret,TCGTemp * t1,TCGTemp * t2,TCGTemp * t3)2727 void tcg_gen_call3(void *func, TCGHelperInfo *info, TCGTemp *ret,
2728 TCGTemp *t1, TCGTemp *t2, TCGTemp *t3)
2729 {
2730 TCGTemp *args[3] = { t1, t2, t3 };
2731 tcg_gen_callN(func, info, ret, args);
2732 }
2733
tcg_gen_call4(void * func,TCGHelperInfo * info,TCGTemp * ret,TCGTemp * t1,TCGTemp * t2,TCGTemp * t3,TCGTemp * t4)2734 void tcg_gen_call4(void *func, TCGHelperInfo *info, TCGTemp *ret,
2735 TCGTemp *t1, TCGTemp *t2, TCGTemp *t3, TCGTemp *t4)
2736 {
2737 TCGTemp *args[4] = { t1, t2, t3, t4 };
2738 tcg_gen_callN(func, info, ret, args);
2739 }
2740
tcg_gen_call5(void * func,TCGHelperInfo * info,TCGTemp * ret,TCGTemp * t1,TCGTemp * t2,TCGTemp * t3,TCGTemp * t4,TCGTemp * t5)2741 void tcg_gen_call5(void *func, TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1,
2742 TCGTemp *t2, TCGTemp *t3, TCGTemp *t4, TCGTemp *t5)
2743 {
2744 TCGTemp *args[5] = { t1, t2, t3, t4, t5 };
2745 tcg_gen_callN(func, info, ret, args);
2746 }
2747
tcg_gen_call6(void * func,TCGHelperInfo * info,TCGTemp * ret,TCGTemp * t1,TCGTemp * t2,TCGTemp * t3,TCGTemp * t4,TCGTemp * t5,TCGTemp * t6)2748 void tcg_gen_call6(void *func, TCGHelperInfo *info, TCGTemp *ret,
2749 TCGTemp *t1, TCGTemp *t2, TCGTemp *t3,
2750 TCGTemp *t4, TCGTemp *t5, TCGTemp *t6)
2751 {
2752 TCGTemp *args[6] = { t1, t2, t3, t4, t5, t6 };
2753 tcg_gen_callN(func, info, ret, args);
2754 }
2755
tcg_gen_call7(void * func,TCGHelperInfo * info,TCGTemp * ret,TCGTemp * t1,TCGTemp * t2,TCGTemp * t3,TCGTemp * t4,TCGTemp * t5,TCGTemp * t6,TCGTemp * t7)2756 void tcg_gen_call7(void *func, TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1,
2757 TCGTemp *t2, TCGTemp *t3, TCGTemp *t4,
2758 TCGTemp *t5, TCGTemp *t6, TCGTemp *t7)
2759 {
2760 TCGTemp *args[7] = { t1, t2, t3, t4, t5, t6, t7 };
2761 tcg_gen_callN(func, info, ret, args);
2762 }
2763
tcg_reg_alloc_start(TCGContext * s)2764 static void tcg_reg_alloc_start(TCGContext *s)
2765 {
2766 int i, n;
2767
2768 for (i = 0, n = s->nb_temps; i < n; i++) {
2769 TCGTemp *ts = &s->temps[i];
2770 TCGTempVal val = TEMP_VAL_MEM;
2771
2772 switch (ts->kind) {
2773 case TEMP_CONST:
2774 val = TEMP_VAL_CONST;
2775 break;
2776 case TEMP_FIXED:
2777 val = TEMP_VAL_REG;
2778 break;
2779 case TEMP_GLOBAL:
2780 break;
2781 case TEMP_EBB:
2782 val = TEMP_VAL_DEAD;
2783 /* fall through */
2784 case TEMP_TB:
2785 ts->mem_allocated = 0;
2786 break;
2787 default:
2788 g_assert_not_reached();
2789 }
2790 ts->val_type = val;
2791 }
2792
2793 memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp));
2794 }
2795
tcg_get_arg_str_ptr(TCGContext * s,char * buf,int buf_size,TCGTemp * ts)2796 static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size,
2797 TCGTemp *ts)
2798 {
2799 int idx = temp_idx(ts);
2800
2801 switch (ts->kind) {
2802 case TEMP_FIXED:
2803 case TEMP_GLOBAL:
2804 pstrcpy(buf, buf_size, ts->name);
2805 break;
2806 case TEMP_TB:
2807 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals);
2808 break;
2809 case TEMP_EBB:
2810 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals);
2811 break;
2812 case TEMP_CONST:
2813 switch (ts->type) {
2814 case TCG_TYPE_I32:
2815 snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val);
2816 break;
2817 #if TCG_TARGET_REG_BITS > 32
2818 case TCG_TYPE_I64:
2819 snprintf(buf, buf_size, "$0x%" PRIx64, ts->val);
2820 break;
2821 #endif
2822 case TCG_TYPE_V64:
2823 case TCG_TYPE_V128:
2824 case TCG_TYPE_V256:
2825 snprintf(buf, buf_size, "v%d$0x%" PRIx64,
2826 64 << (ts->type - TCG_TYPE_V64), ts->val);
2827 break;
2828 default:
2829 g_assert_not_reached();
2830 }
2831 break;
2832 }
2833 return buf;
2834 }
2835
tcg_get_arg_str(TCGContext * s,char * buf,int buf_size,TCGArg arg)2836 static char *tcg_get_arg_str(TCGContext *s, char *buf,
2837 int buf_size, TCGArg arg)
2838 {
2839 return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg));
2840 }
2841
2842 static const char * const cond_name[] =
2843 {
2844 [TCG_COND_NEVER] = "never",
2845 [TCG_COND_ALWAYS] = "always",
2846 [TCG_COND_EQ] = "eq",
2847 [TCG_COND_NE] = "ne",
2848 [TCG_COND_LT] = "lt",
2849 [TCG_COND_GE] = "ge",
2850 [TCG_COND_LE] = "le",
2851 [TCG_COND_GT] = "gt",
2852 [TCG_COND_LTU] = "ltu",
2853 [TCG_COND_GEU] = "geu",
2854 [TCG_COND_LEU] = "leu",
2855 [TCG_COND_GTU] = "gtu",
2856 [TCG_COND_TSTEQ] = "tsteq",
2857 [TCG_COND_TSTNE] = "tstne",
2858 };
2859
2860 static const char * const ldst_name[(MO_BSWAP | MO_SSIZE) + 1] =
2861 {
2862 [MO_UB] = "ub",
2863 [MO_SB] = "sb",
2864 [MO_LEUW] = "leuw",
2865 [MO_LESW] = "lesw",
2866 [MO_LEUL] = "leul",
2867 [MO_LESL] = "lesl",
2868 [MO_LEUQ] = "leq",
2869 [MO_BEUW] = "beuw",
2870 [MO_BESW] = "besw",
2871 [MO_BEUL] = "beul",
2872 [MO_BESL] = "besl",
2873 [MO_BEUQ] = "beq",
2874 [MO_128 + MO_BE] = "beo",
2875 [MO_128 + MO_LE] = "leo",
2876 };
2877
2878 static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = {
2879 [MO_UNALN >> MO_ASHIFT] = "un+",
2880 [MO_ALIGN >> MO_ASHIFT] = "al+",
2881 [MO_ALIGN_2 >> MO_ASHIFT] = "al2+",
2882 [MO_ALIGN_4 >> MO_ASHIFT] = "al4+",
2883 [MO_ALIGN_8 >> MO_ASHIFT] = "al8+",
2884 [MO_ALIGN_16 >> MO_ASHIFT] = "al16+",
2885 [MO_ALIGN_32 >> MO_ASHIFT] = "al32+",
2886 [MO_ALIGN_64 >> MO_ASHIFT] = "al64+",
2887 };
2888
2889 static const char * const atom_name[(MO_ATOM_MASK >> MO_ATOM_SHIFT) + 1] = {
2890 [MO_ATOM_IFALIGN >> MO_ATOM_SHIFT] = "",
2891 [MO_ATOM_IFALIGN_PAIR >> MO_ATOM_SHIFT] = "pair+",
2892 [MO_ATOM_WITHIN16 >> MO_ATOM_SHIFT] = "w16+",
2893 [MO_ATOM_WITHIN16_PAIR >> MO_ATOM_SHIFT] = "w16p+",
2894 [MO_ATOM_SUBALIGN >> MO_ATOM_SHIFT] = "sub+",
2895 [MO_ATOM_NONE >> MO_ATOM_SHIFT] = "noat+",
2896 };
2897
2898 static const char bswap_flag_name[][6] = {
2899 [TCG_BSWAP_IZ] = "iz",
2900 [TCG_BSWAP_OZ] = "oz",
2901 [TCG_BSWAP_OS] = "os",
2902 [TCG_BSWAP_IZ | TCG_BSWAP_OZ] = "iz,oz",
2903 [TCG_BSWAP_IZ | TCG_BSWAP_OS] = "iz,os",
2904 };
2905
2906 #ifdef CONFIG_PLUGIN
2907 static const char * const plugin_from_name[] = {
2908 "from-tb",
2909 "from-insn",
2910 "after-insn",
2911 "after-tb",
2912 };
2913 #endif
2914
tcg_regset_single(TCGRegSet d)2915 static inline bool tcg_regset_single(TCGRegSet d)
2916 {
2917 return (d & (d - 1)) == 0;
2918 }
2919
tcg_regset_first(TCGRegSet d)2920 static inline TCGReg tcg_regset_first(TCGRegSet d)
2921 {
2922 if (TCG_TARGET_NB_REGS <= 32) {
2923 return ctz32(d);
2924 } else {
2925 return ctz64(d);
2926 }
2927 }
2928
2929 /* Return only the number of characters output -- no error return. */
2930 #define ne_fprintf(...) \
2931 ({ int ret_ = fprintf(__VA_ARGS__); ret_ >= 0 ? ret_ : 0; })
2932
tcg_dump_ops(TCGContext * s,FILE * f,bool have_prefs)2933 void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs)
2934 {
2935 char buf[128];
2936 TCGOp *op;
2937
2938 QTAILQ_FOREACH(op, &s->ops, link) {
2939 int i, k, nb_oargs, nb_iargs, nb_cargs;
2940 const TCGOpDef *def;
2941 TCGOpcode c;
2942 int col = 0;
2943
2944 c = op->opc;
2945 def = &tcg_op_defs[c];
2946
2947 if (c == INDEX_op_insn_start) {
2948 nb_oargs = 0;
2949 col += ne_fprintf(f, "\n ----");
2950
2951 for (i = 0, k = INSN_START_WORDS; i < k; ++i) {
2952 col += ne_fprintf(f, " %016" PRIx64,
2953 tcg_get_insn_start_param(op, i));
2954 }
2955 } else if (c == INDEX_op_call) {
2956 const TCGHelperInfo *info = tcg_call_info(op);
2957 void *func = tcg_call_func(op);
2958
2959 /* variable number of arguments */
2960 nb_oargs = TCGOP_CALLO(op);
2961 nb_iargs = TCGOP_CALLI(op);
2962 nb_cargs = def->nb_cargs;
2963
2964 col += ne_fprintf(f, " %s ", def->name);
2965
2966 /*
2967 * Print the function name from TCGHelperInfo, if available.
2968 * Note that plugins have a template function for the info,
2969 * but the actual function pointer comes from the plugin.
2970 */
2971 if (func == info->func) {
2972 col += ne_fprintf(f, "%s", info->name);
2973 } else {
2974 col += ne_fprintf(f, "plugin(%p)", func);
2975 }
2976
2977 col += ne_fprintf(f, ",$0x%x,$%d", info->flags, nb_oargs);
2978 for (i = 0; i < nb_oargs; i++) {
2979 col += ne_fprintf(f, ",%s", tcg_get_arg_str(s, buf, sizeof(buf),
2980 op->args[i]));
2981 }
2982 for (i = 0; i < nb_iargs; i++) {
2983 TCGArg arg = op->args[nb_oargs + i];
2984 const char *t = tcg_get_arg_str(s, buf, sizeof(buf), arg);
2985 col += ne_fprintf(f, ",%s", t);
2986 }
2987 } else {
2988 if (def->flags & TCG_OPF_INT) {
2989 col += ne_fprintf(f, " %s_i%d ",
2990 def->name,
2991 8 * tcg_type_size(TCGOP_TYPE(op)));
2992 } else if (def->flags & TCG_OPF_VECTOR) {
2993 col += ne_fprintf(f, "%s v%d,e%d,",
2994 def->name,
2995 8 * tcg_type_size(TCGOP_TYPE(op)),
2996 8 << TCGOP_VECE(op));
2997 } else {
2998 col += ne_fprintf(f, " %s ", def->name);
2999 }
3000
3001 nb_oargs = def->nb_oargs;
3002 nb_iargs = def->nb_iargs;
3003 nb_cargs = def->nb_cargs;
3004
3005 k = 0;
3006 for (i = 0; i < nb_oargs; i++) {
3007 const char *sep = k ? "," : "";
3008 col += ne_fprintf(f, "%s%s", sep,
3009 tcg_get_arg_str(s, buf, sizeof(buf),
3010 op->args[k++]));
3011 }
3012 for (i = 0; i < nb_iargs; i++) {
3013 const char *sep = k ? "," : "";
3014 col += ne_fprintf(f, "%s%s", sep,
3015 tcg_get_arg_str(s, buf, sizeof(buf),
3016 op->args[k++]));
3017 }
3018 switch (c) {
3019 case INDEX_op_brcond:
3020 case INDEX_op_setcond:
3021 case INDEX_op_negsetcond:
3022 case INDEX_op_movcond:
3023 case INDEX_op_brcond2_i32:
3024 case INDEX_op_setcond2_i32:
3025 case INDEX_op_cmp_vec:
3026 case INDEX_op_cmpsel_vec:
3027 if (op->args[k] < ARRAY_SIZE(cond_name)
3028 && cond_name[op->args[k]]) {
3029 col += ne_fprintf(f, ",%s", cond_name[op->args[k++]]);
3030 } else {
3031 col += ne_fprintf(f, ",$0x%" TCG_PRIlx, op->args[k++]);
3032 }
3033 i = 1;
3034 break;
3035 case INDEX_op_qemu_ld:
3036 case INDEX_op_qemu_st:
3037 case INDEX_op_qemu_ld2:
3038 case INDEX_op_qemu_st2:
3039 {
3040 const char *s_al, *s_op, *s_at;
3041 MemOpIdx oi = op->args[k++];
3042 MemOp mop = get_memop(oi);
3043 unsigned ix = get_mmuidx(oi);
3044
3045 s_al = alignment_name[(mop & MO_AMASK) >> MO_ASHIFT];
3046 s_op = ldst_name[mop & (MO_BSWAP | MO_SSIZE)];
3047 s_at = atom_name[(mop & MO_ATOM_MASK) >> MO_ATOM_SHIFT];
3048 mop &= ~(MO_AMASK | MO_BSWAP | MO_SSIZE | MO_ATOM_MASK);
3049
3050 /* If all fields are accounted for, print symbolically. */
3051 if (!mop && s_al && s_op && s_at) {
3052 col += ne_fprintf(f, ",%s%s%s,%u",
3053 s_at, s_al, s_op, ix);
3054 } else {
3055 mop = get_memop(oi);
3056 col += ne_fprintf(f, ",$0x%x,%u", mop, ix);
3057 }
3058 i = 1;
3059 }
3060 break;
3061 case INDEX_op_bswap16:
3062 case INDEX_op_bswap32:
3063 case INDEX_op_bswap64:
3064 {
3065 TCGArg flags = op->args[k];
3066 const char *name = NULL;
3067
3068 if (flags < ARRAY_SIZE(bswap_flag_name)) {
3069 name = bswap_flag_name[flags];
3070 }
3071 if (name) {
3072 col += ne_fprintf(f, ",%s", name);
3073 } else {
3074 col += ne_fprintf(f, ",$0x%" TCG_PRIlx, flags);
3075 }
3076 i = k = 1;
3077 }
3078 break;
3079 #ifdef CONFIG_PLUGIN
3080 case INDEX_op_plugin_cb:
3081 {
3082 TCGArg from = op->args[k++];
3083 const char *name = NULL;
3084
3085 if (from < ARRAY_SIZE(plugin_from_name)) {
3086 name = plugin_from_name[from];
3087 }
3088 if (name) {
3089 col += ne_fprintf(f, "%s", name);
3090 } else {
3091 col += ne_fprintf(f, "$0x%" TCG_PRIlx, from);
3092 }
3093 i = 1;
3094 }
3095 break;
3096 #endif
3097 default:
3098 i = 0;
3099 break;
3100 }
3101 switch (c) {
3102 case INDEX_op_set_label:
3103 case INDEX_op_br:
3104 case INDEX_op_brcond:
3105 case INDEX_op_brcond2_i32:
3106 col += ne_fprintf(f, "%s$L%d", k ? "," : "",
3107 arg_label(op->args[k])->id);
3108 i++, k++;
3109 break;
3110 case INDEX_op_mb:
3111 {
3112 TCGBar membar = op->args[k];
3113 const char *b_op, *m_op;
3114
3115 switch (membar & TCG_BAR_SC) {
3116 case 0:
3117 b_op = "none";
3118 break;
3119 case TCG_BAR_LDAQ:
3120 b_op = "acq";
3121 break;
3122 case TCG_BAR_STRL:
3123 b_op = "rel";
3124 break;
3125 case TCG_BAR_SC:
3126 b_op = "seq";
3127 break;
3128 default:
3129 g_assert_not_reached();
3130 }
3131
3132 switch (membar & TCG_MO_ALL) {
3133 case 0:
3134 m_op = "none";
3135 break;
3136 case TCG_MO_LD_LD:
3137 m_op = "rr";
3138 break;
3139 case TCG_MO_LD_ST:
3140 m_op = "rw";
3141 break;
3142 case TCG_MO_ST_LD:
3143 m_op = "wr";
3144 break;
3145 case TCG_MO_ST_ST:
3146 m_op = "ww";
3147 break;
3148 case TCG_MO_LD_LD | TCG_MO_LD_ST:
3149 m_op = "rr+rw";
3150 break;
3151 case TCG_MO_LD_LD | TCG_MO_ST_LD:
3152 m_op = "rr+wr";
3153 break;
3154 case TCG_MO_LD_LD | TCG_MO_ST_ST:
3155 m_op = "rr+ww";
3156 break;
3157 case TCG_MO_LD_ST | TCG_MO_ST_LD:
3158 m_op = "rw+wr";
3159 break;
3160 case TCG_MO_LD_ST | TCG_MO_ST_ST:
3161 m_op = "rw+ww";
3162 break;
3163 case TCG_MO_ST_LD | TCG_MO_ST_ST:
3164 m_op = "wr+ww";
3165 break;
3166 case TCG_MO_LD_LD | TCG_MO_LD_ST | TCG_MO_ST_LD:
3167 m_op = "rr+rw+wr";
3168 break;
3169 case TCG_MO_LD_LD | TCG_MO_LD_ST | TCG_MO_ST_ST:
3170 m_op = "rr+rw+ww";
3171 break;
3172 case TCG_MO_LD_LD | TCG_MO_ST_LD | TCG_MO_ST_ST:
3173 m_op = "rr+wr+ww";
3174 break;
3175 case TCG_MO_LD_ST | TCG_MO_ST_LD | TCG_MO_ST_ST:
3176 m_op = "rw+wr+ww";
3177 break;
3178 case TCG_MO_ALL:
3179 m_op = "all";
3180 break;
3181 default:
3182 g_assert_not_reached();
3183 }
3184
3185 col += ne_fprintf(f, "%s%s:%s", (k ? "," : ""), b_op, m_op);
3186 i++, k++;
3187 }
3188 break;
3189 default:
3190 break;
3191 }
3192 for (; i < nb_cargs; i++, k++) {
3193 col += ne_fprintf(f, "%s$0x%" TCG_PRIlx, k ? "," : "",
3194 op->args[k]);
3195 }
3196 }
3197
3198 if (have_prefs || op->life) {
3199 for (; col < 40; ++col) {
3200 putc(' ', f);
3201 }
3202 }
3203
3204 if (op->life) {
3205 unsigned life = op->life;
3206
3207 if (life & (SYNC_ARG * 3)) {
3208 ne_fprintf(f, " sync:");
3209 for (i = 0; i < 2; ++i) {
3210 if (life & (SYNC_ARG << i)) {
3211 ne_fprintf(f, " %d", i);
3212 }
3213 }
3214 }
3215 life /= DEAD_ARG;
3216 if (life) {
3217 ne_fprintf(f, " dead:");
3218 for (i = 0; life; ++i, life >>= 1) {
3219 if (life & 1) {
3220 ne_fprintf(f, " %d", i);
3221 }
3222 }
3223 }
3224 }
3225
3226 if (have_prefs) {
3227 for (i = 0; i < nb_oargs; ++i) {
3228 TCGRegSet set = output_pref(op, i);
3229
3230 if (i == 0) {
3231 ne_fprintf(f, " pref=");
3232 } else {
3233 ne_fprintf(f, ",");
3234 }
3235 if (set == 0) {
3236 ne_fprintf(f, "none");
3237 } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) {
3238 ne_fprintf(f, "all");
3239 #ifdef CONFIG_DEBUG_TCG
3240 } else if (tcg_regset_single(set)) {
3241 TCGReg reg = tcg_regset_first(set);
3242 ne_fprintf(f, "%s", tcg_target_reg_names[reg]);
3243 #endif
3244 } else if (TCG_TARGET_NB_REGS <= 32) {
3245 ne_fprintf(f, "0x%x", (uint32_t)set);
3246 } else {
3247 ne_fprintf(f, "0x%" PRIx64, (uint64_t)set);
3248 }
3249 }
3250 }
3251
3252 putc('\n', f);
3253 }
3254 }
3255
3256 /* we give more priority to constraints with less registers */
get_constraint_priority(const TCGArgConstraint * arg_ct,int k)3257 static int get_constraint_priority(const TCGArgConstraint *arg_ct, int k)
3258 {
3259 int n;
3260
3261 arg_ct += k;
3262 n = ctpop64(arg_ct->regs);
3263
3264 /*
3265 * Sort constraints of a single register first, which includes output
3266 * aliases (which must exactly match the input already allocated).
3267 */
3268 if (n == 1 || arg_ct->oalias) {
3269 return INT_MAX;
3270 }
3271
3272 /*
3273 * Sort register pairs next, first then second immediately after.
3274 * Arbitrarily sort multiple pairs by the index of the first reg;
3275 * there shouldn't be many pairs.
3276 */
3277 switch (arg_ct->pair) {
3278 case 1:
3279 case 3:
3280 return (k + 1) * 2;
3281 case 2:
3282 return (arg_ct->pair_index + 1) * 2 - 1;
3283 }
3284
3285 /* Finally, sort by decreasing register count. */
3286 assert(n > 1);
3287 return -n;
3288 }
3289
3290 /* sort from highest priority to lowest */
sort_constraints(TCGArgConstraint * a,int start,int n)3291 static void sort_constraints(TCGArgConstraint *a, int start, int n)
3292 {
3293 int i, j;
3294
3295 for (i = 0; i < n; i++) {
3296 a[start + i].sort_index = start + i;
3297 }
3298 if (n <= 1) {
3299 return;
3300 }
3301 for (i = 0; i < n - 1; i++) {
3302 for (j = i + 1; j < n; j++) {
3303 int p1 = get_constraint_priority(a, a[start + i].sort_index);
3304 int p2 = get_constraint_priority(a, a[start + j].sort_index);
3305 if (p1 < p2) {
3306 int tmp = a[start + i].sort_index;
3307 a[start + i].sort_index = a[start + j].sort_index;
3308 a[start + j].sort_index = tmp;
3309 }
3310 }
3311 }
3312 }
3313
3314 static const TCGArgConstraint empty_cts[TCG_MAX_OP_ARGS];
3315 static TCGArgConstraint all_cts[ARRAY_SIZE(constraint_sets)][TCG_MAX_OP_ARGS];
3316
process_constraint_sets(void)3317 static void process_constraint_sets(void)
3318 {
3319 for (size_t c = 0; c < ARRAY_SIZE(constraint_sets); ++c) {
3320 const TCGConstraintSet *tdefs = &constraint_sets[c];
3321 TCGArgConstraint *args_ct = all_cts[c];
3322 int nb_oargs = tdefs->nb_oargs;
3323 int nb_iargs = tdefs->nb_iargs;
3324 int nb_args = nb_oargs + nb_iargs;
3325 bool saw_alias_pair = false;
3326
3327 for (int i = 0; i < nb_args; i++) {
3328 const char *ct_str = tdefs->args_ct_str[i];
3329 bool input_p = i >= nb_oargs;
3330 int o;
3331
3332 switch (*ct_str) {
3333 case '0' ... '9':
3334 o = *ct_str - '0';
3335 tcg_debug_assert(input_p);
3336 tcg_debug_assert(o < nb_oargs);
3337 tcg_debug_assert(args_ct[o].regs != 0);
3338 tcg_debug_assert(!args_ct[o].oalias);
3339 args_ct[i] = args_ct[o];
3340 /* The output sets oalias. */
3341 args_ct[o].oalias = 1;
3342 args_ct[o].alias_index = i;
3343 /* The input sets ialias. */
3344 args_ct[i].ialias = 1;
3345 args_ct[i].alias_index = o;
3346 if (args_ct[i].pair) {
3347 saw_alias_pair = true;
3348 }
3349 tcg_debug_assert(ct_str[1] == '\0');
3350 continue;
3351
3352 case '&':
3353 tcg_debug_assert(!input_p);
3354 args_ct[i].newreg = true;
3355 ct_str++;
3356 break;
3357
3358 case 'p': /* plus */
3359 /* Allocate to the register after the previous. */
3360 tcg_debug_assert(i > (input_p ? nb_oargs : 0));
3361 o = i - 1;
3362 tcg_debug_assert(!args_ct[o].pair);
3363 tcg_debug_assert(!args_ct[o].ct);
3364 args_ct[i] = (TCGArgConstraint){
3365 .pair = 2,
3366 .pair_index = o,
3367 .regs = args_ct[o].regs << 1,
3368 .newreg = args_ct[o].newreg,
3369 };
3370 args_ct[o].pair = 1;
3371 args_ct[o].pair_index = i;
3372 tcg_debug_assert(ct_str[1] == '\0');
3373 continue;
3374
3375 case 'm': /* minus */
3376 /* Allocate to the register before the previous. */
3377 tcg_debug_assert(i > (input_p ? nb_oargs : 0));
3378 o = i - 1;
3379 tcg_debug_assert(!args_ct[o].pair);
3380 tcg_debug_assert(!args_ct[o].ct);
3381 args_ct[i] = (TCGArgConstraint){
3382 .pair = 1,
3383 .pair_index = o,
3384 .regs = args_ct[o].regs >> 1,
3385 .newreg = args_ct[o].newreg,
3386 };
3387 args_ct[o].pair = 2;
3388 args_ct[o].pair_index = i;
3389 tcg_debug_assert(ct_str[1] == '\0');
3390 continue;
3391 }
3392
3393 do {
3394 switch (*ct_str) {
3395 case 'i':
3396 args_ct[i].ct |= TCG_CT_CONST;
3397 break;
3398 #ifdef TCG_REG_ZERO
3399 case 'z':
3400 args_ct[i].ct |= TCG_CT_REG_ZERO;
3401 break;
3402 #endif
3403
3404 /* Include all of the target-specific constraints. */
3405
3406 #undef CONST
3407 #define CONST(CASE, MASK) \
3408 case CASE: args_ct[i].ct |= MASK; break;
3409 #define REGS(CASE, MASK) \
3410 case CASE: args_ct[i].regs |= MASK; break;
3411
3412 #include "tcg-target-con-str.h"
3413
3414 #undef REGS
3415 #undef CONST
3416 default:
3417 case '0' ... '9':
3418 case '&':
3419 case 'p':
3420 case 'm':
3421 /* Typo in TCGConstraintSet constraint. */
3422 g_assert_not_reached();
3423 }
3424 } while (*++ct_str != '\0');
3425 }
3426
3427 /*
3428 * Fix up output pairs that are aliased with inputs.
3429 * When we created the alias, we copied pair from the output.
3430 * There are three cases:
3431 * (1a) Pairs of inputs alias pairs of outputs.
3432 * (1b) One input aliases the first of a pair of outputs.
3433 * (2) One input aliases the second of a pair of outputs.
3434 *
3435 * Case 1a is handled by making sure that the pair_index'es are
3436 * properly updated so that they appear the same as a pair of inputs.
3437 *
3438 * Case 1b is handled by setting the pair_index of the input to
3439 * itself, simply so it doesn't point to an unrelated argument.
3440 * Since we don't encounter the "second" during the input allocation
3441 * phase, nothing happens with the second half of the input pair.
3442 *
3443 * Case 2 is handled by setting the second input to pair=3, the
3444 * first output to pair=3, and the pair_index'es to match.
3445 */
3446 if (saw_alias_pair) {
3447 for (int i = nb_oargs; i < nb_args; i++) {
3448 int o, o2, i2;
3449
3450 /*
3451 * Since [0-9pm] must be alone in the constraint string,
3452 * the only way they can both be set is if the pair comes
3453 * from the output alias.
3454 */
3455 if (!args_ct[i].ialias) {
3456 continue;
3457 }
3458 switch (args_ct[i].pair) {
3459 case 0:
3460 break;
3461 case 1:
3462 o = args_ct[i].alias_index;
3463 o2 = args_ct[o].pair_index;
3464 tcg_debug_assert(args_ct[o].pair == 1);
3465 tcg_debug_assert(args_ct[o2].pair == 2);
3466 if (args_ct[o2].oalias) {
3467 /* Case 1a */
3468 i2 = args_ct[o2].alias_index;
3469 tcg_debug_assert(args_ct[i2].pair == 2);
3470 args_ct[i2].pair_index = i;
3471 args_ct[i].pair_index = i2;
3472 } else {
3473 /* Case 1b */
3474 args_ct[i].pair_index = i;
3475 }
3476 break;
3477 case 2:
3478 o = args_ct[i].alias_index;
3479 o2 = args_ct[o].pair_index;
3480 tcg_debug_assert(args_ct[o].pair == 2);
3481 tcg_debug_assert(args_ct[o2].pair == 1);
3482 if (args_ct[o2].oalias) {
3483 /* Case 1a */
3484 i2 = args_ct[o2].alias_index;
3485 tcg_debug_assert(args_ct[i2].pair == 1);
3486 args_ct[i2].pair_index = i;
3487 args_ct[i].pair_index = i2;
3488 } else {
3489 /* Case 2 */
3490 args_ct[i].pair = 3;
3491 args_ct[o2].pair = 3;
3492 args_ct[i].pair_index = o2;
3493 args_ct[o2].pair_index = i;
3494 }
3495 break;
3496 default:
3497 g_assert_not_reached();
3498 }
3499 }
3500 }
3501
3502 /* sort the constraints (XXX: this is just an heuristic) */
3503 sort_constraints(args_ct, 0, nb_oargs);
3504 sort_constraints(args_ct, nb_oargs, nb_iargs);
3505 }
3506 }
3507
opcode_args_ct(const TCGOp * op)3508 static const TCGArgConstraint *opcode_args_ct(const TCGOp *op)
3509 {
3510 TCGOpcode opc = op->opc;
3511 TCGType type = TCGOP_TYPE(op);
3512 unsigned flags = TCGOP_FLAGS(op);
3513 const TCGOpDef *def = &tcg_op_defs[opc];
3514 const TCGOutOp *outop = all_outop[opc];
3515 TCGConstraintSetIndex con_set;
3516
3517 if (def->flags & TCG_OPF_NOT_PRESENT) {
3518 return empty_cts;
3519 }
3520
3521 if (outop) {
3522 con_set = outop->static_constraint;
3523 if (con_set == C_Dynamic) {
3524 con_set = outop->dynamic_constraint(type, flags);
3525 }
3526 } else {
3527 con_set = tcg_target_op_def(opc, type, flags);
3528 }
3529 tcg_debug_assert(con_set >= 0);
3530 tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets));
3531
3532 /* The constraint arguments must match TCGOpcode arguments. */
3533 tcg_debug_assert(constraint_sets[con_set].nb_oargs == def->nb_oargs);
3534 tcg_debug_assert(constraint_sets[con_set].nb_iargs == def->nb_iargs);
3535
3536 return all_cts[con_set];
3537 }
3538
remove_label_use(TCGOp * op,int idx)3539 static void remove_label_use(TCGOp *op, int idx)
3540 {
3541 TCGLabel *label = arg_label(op->args[idx]);
3542 TCGLabelUse *use;
3543
3544 QSIMPLEQ_FOREACH(use, &label->branches, next) {
3545 if (use->op == op) {
3546 QSIMPLEQ_REMOVE(&label->branches, use, TCGLabelUse, next);
3547 return;
3548 }
3549 }
3550 g_assert_not_reached();
3551 }
3552
tcg_op_remove(TCGContext * s,TCGOp * op)3553 void tcg_op_remove(TCGContext *s, TCGOp *op)
3554 {
3555 switch (op->opc) {
3556 case INDEX_op_br:
3557 remove_label_use(op, 0);
3558 break;
3559 case INDEX_op_brcond:
3560 remove_label_use(op, 3);
3561 break;
3562 case INDEX_op_brcond2_i32:
3563 remove_label_use(op, 5);
3564 break;
3565 default:
3566 break;
3567 }
3568
3569 QTAILQ_REMOVE(&s->ops, op, link);
3570 QTAILQ_INSERT_TAIL(&s->free_ops, op, link);
3571 s->nb_ops--;
3572 }
3573
tcg_remove_ops_after(TCGOp * op)3574 void tcg_remove_ops_after(TCGOp *op)
3575 {
3576 TCGContext *s = tcg_ctx;
3577
3578 while (true) {
3579 TCGOp *last = tcg_last_op();
3580 if (last == op) {
3581 return;
3582 }
3583 tcg_op_remove(s, last);
3584 }
3585 }
3586
tcg_op_alloc(TCGOpcode opc,unsigned nargs)3587 static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs)
3588 {
3589 TCGContext *s = tcg_ctx;
3590 TCGOp *op = NULL;
3591
3592 if (unlikely(!QTAILQ_EMPTY(&s->free_ops))) {
3593 QTAILQ_FOREACH(op, &s->free_ops, link) {
3594 if (nargs <= op->nargs) {
3595 QTAILQ_REMOVE(&s->free_ops, op, link);
3596 nargs = op->nargs;
3597 goto found;
3598 }
3599 }
3600 }
3601
3602 /* Most opcodes have 3 or 4 operands: reduce fragmentation. */
3603 nargs = MAX(4, nargs);
3604 op = tcg_malloc(sizeof(TCGOp) + sizeof(TCGArg) * nargs);
3605
3606 found:
3607 memset(op, 0, offsetof(TCGOp, link));
3608 op->opc = opc;
3609 op->nargs = nargs;
3610
3611 /* Check for bitfield overflow. */
3612 tcg_debug_assert(op->nargs == nargs);
3613
3614 s->nb_ops++;
3615 return op;
3616 }
3617
tcg_emit_op(TCGOpcode opc,unsigned nargs)3618 TCGOp *tcg_emit_op(TCGOpcode opc, unsigned nargs)
3619 {
3620 TCGOp *op = tcg_op_alloc(opc, nargs);
3621
3622 if (tcg_ctx->emit_before_op) {
3623 QTAILQ_INSERT_BEFORE(tcg_ctx->emit_before_op, op, link);
3624 } else {
3625 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
3626 }
3627 return op;
3628 }
3629
tcg_op_insert_before(TCGContext * s,TCGOp * old_op,TCGOpcode opc,TCGType type,unsigned nargs)3630 TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op,
3631 TCGOpcode opc, TCGType type, unsigned nargs)
3632 {
3633 TCGOp *new_op = tcg_op_alloc(opc, nargs);
3634
3635 TCGOP_TYPE(new_op) = type;
3636 QTAILQ_INSERT_BEFORE(old_op, new_op, link);
3637 return new_op;
3638 }
3639
tcg_op_insert_after(TCGContext * s,TCGOp * old_op,TCGOpcode opc,TCGType type,unsigned nargs)3640 TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op,
3641 TCGOpcode opc, TCGType type, unsigned nargs)
3642 {
3643 TCGOp *new_op = tcg_op_alloc(opc, nargs);
3644
3645 TCGOP_TYPE(new_op) = type;
3646 QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link);
3647 return new_op;
3648 }
3649
move_label_uses(TCGLabel * to,TCGLabel * from)3650 static void move_label_uses(TCGLabel *to, TCGLabel *from)
3651 {
3652 TCGLabelUse *u;
3653
3654 QSIMPLEQ_FOREACH(u, &from->branches, next) {
3655 TCGOp *op = u->op;
3656 switch (op->opc) {
3657 case INDEX_op_br:
3658 op->args[0] = label_arg(to);
3659 break;
3660 case INDEX_op_brcond:
3661 op->args[3] = label_arg(to);
3662 break;
3663 case INDEX_op_brcond2_i32:
3664 op->args[5] = label_arg(to);
3665 break;
3666 default:
3667 g_assert_not_reached();
3668 }
3669 }
3670
3671 QSIMPLEQ_CONCAT(&to->branches, &from->branches);
3672 }
3673
3674 /* Reachable analysis : remove unreachable code. */
3675 static void __attribute__((noinline))
reachable_code_pass(TCGContext * s)3676 reachable_code_pass(TCGContext *s)
3677 {
3678 TCGOp *op, *op_next, *op_prev;
3679 bool dead = false;
3680
3681 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
3682 bool remove = dead;
3683 TCGLabel *label;
3684
3685 switch (op->opc) {
3686 case INDEX_op_set_label:
3687 label = arg_label(op->args[0]);
3688
3689 /*
3690 * Note that the first op in the TB is always a load,
3691 * so there is always something before a label.
3692 */
3693 op_prev = QTAILQ_PREV(op, link);
3694
3695 /*
3696 * If we find two sequential labels, move all branches to
3697 * reference the second label and remove the first label.
3698 * Do this before branch to next optimization, so that the
3699 * middle label is out of the way.
3700 */
3701 if (op_prev->opc == INDEX_op_set_label) {
3702 move_label_uses(label, arg_label(op_prev->args[0]));
3703 tcg_op_remove(s, op_prev);
3704 op_prev = QTAILQ_PREV(op, link);
3705 }
3706
3707 /*
3708 * Optimization can fold conditional branches to unconditional.
3709 * If we find a label which is preceded by an unconditional
3710 * branch to next, remove the branch. We couldn't do this when
3711 * processing the branch because any dead code between the branch
3712 * and label had not yet been removed.
3713 */
3714 if (op_prev->opc == INDEX_op_br &&
3715 label == arg_label(op_prev->args[0])) {
3716 tcg_op_remove(s, op_prev);
3717 /* Fall through means insns become live again. */
3718 dead = false;
3719 }
3720
3721 if (QSIMPLEQ_EMPTY(&label->branches)) {
3722 /*
3723 * While there is an occasional backward branch, virtually
3724 * all branches generated by the translators are forward.
3725 * Which means that generally we will have already removed
3726 * all references to the label that will be, and there is
3727 * little to be gained by iterating.
3728 */
3729 remove = true;
3730 } else {
3731 /* Once we see a label, insns become live again. */
3732 dead = false;
3733 remove = false;
3734 }
3735 break;
3736
3737 case INDEX_op_br:
3738 case INDEX_op_exit_tb:
3739 case INDEX_op_goto_ptr:
3740 /* Unconditional branches; everything following is dead. */
3741 dead = true;
3742 break;
3743
3744 case INDEX_op_call:
3745 /* Notice noreturn helper calls, raising exceptions. */
3746 if (tcg_call_flags(op) & TCG_CALL_NO_RETURN) {
3747 dead = true;
3748 }
3749 break;
3750
3751 case INDEX_op_insn_start:
3752 /* Never remove -- we need to keep these for unwind. */
3753 remove = false;
3754 break;
3755
3756 default:
3757 break;
3758 }
3759
3760 if (remove) {
3761 tcg_op_remove(s, op);
3762 }
3763 }
3764 }
3765
3766 #define TS_DEAD 1
3767 #define TS_MEM 2
3768
3769 #define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n)))
3770 #define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n)))
3771
3772 /* For liveness_pass_1, the register preferences for a given temp. */
la_temp_pref(TCGTemp * ts)3773 static inline TCGRegSet *la_temp_pref(TCGTemp *ts)
3774 {
3775 return ts->state_ptr;
3776 }
3777
3778 /* For liveness_pass_1, reset the preferences for a given temp to the
3779 * maximal regset for its type.
3780 */
la_reset_pref(TCGTemp * ts)3781 static inline void la_reset_pref(TCGTemp *ts)
3782 {
3783 *la_temp_pref(ts)
3784 = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]);
3785 }
3786
3787 /* liveness analysis: end of function: all temps are dead, and globals
3788 should be in memory. */
la_func_end(TCGContext * s,int ng,int nt)3789 static void la_func_end(TCGContext *s, int ng, int nt)
3790 {
3791 int i;
3792
3793 for (i = 0; i < ng; ++i) {
3794 s->temps[i].state = TS_DEAD | TS_MEM;
3795 la_reset_pref(&s->temps[i]);
3796 }
3797 for (i = ng; i < nt; ++i) {
3798 s->temps[i].state = TS_DEAD;
3799 la_reset_pref(&s->temps[i]);
3800 }
3801 }
3802
3803 /* liveness analysis: end of basic block: all temps are dead, globals
3804 and local temps should be in memory. */
la_bb_end(TCGContext * s,int ng,int nt)3805 static void la_bb_end(TCGContext *s, int ng, int nt)
3806 {
3807 int i;
3808
3809 for (i = 0; i < nt; ++i) {
3810 TCGTemp *ts = &s->temps[i];
3811 int state;
3812
3813 switch (ts->kind) {
3814 case TEMP_FIXED:
3815 case TEMP_GLOBAL:
3816 case TEMP_TB:
3817 state = TS_DEAD | TS_MEM;
3818 break;
3819 case TEMP_EBB:
3820 case TEMP_CONST:
3821 state = TS_DEAD;
3822 break;
3823 default:
3824 g_assert_not_reached();
3825 }
3826 ts->state = state;
3827 la_reset_pref(ts);
3828 }
3829 }
3830
3831 /* liveness analysis: sync globals back to memory. */
la_global_sync(TCGContext * s,int ng)3832 static void la_global_sync(TCGContext *s, int ng)
3833 {
3834 int i;
3835
3836 for (i = 0; i < ng; ++i) {
3837 int state = s->temps[i].state;
3838 s->temps[i].state = state | TS_MEM;
3839 if (state == TS_DEAD) {
3840 /* If the global was previously dead, reset prefs. */
3841 la_reset_pref(&s->temps[i]);
3842 }
3843 }
3844 }
3845
3846 /*
3847 * liveness analysis: conditional branch: all temps are dead unless
3848 * explicitly live-across-conditional-branch, globals and local temps
3849 * should be synced.
3850 */
la_bb_sync(TCGContext * s,int ng,int nt)3851 static void la_bb_sync(TCGContext *s, int ng, int nt)
3852 {
3853 la_global_sync(s, ng);
3854
3855 for (int i = ng; i < nt; ++i) {
3856 TCGTemp *ts = &s->temps[i];
3857 int state;
3858
3859 switch (ts->kind) {
3860 case TEMP_TB:
3861 state = ts->state;
3862 ts->state = state | TS_MEM;
3863 if (state != TS_DEAD) {
3864 continue;
3865 }
3866 break;
3867 case TEMP_EBB:
3868 case TEMP_CONST:
3869 continue;
3870 default:
3871 g_assert_not_reached();
3872 }
3873 la_reset_pref(&s->temps[i]);
3874 }
3875 }
3876
3877 /* liveness analysis: sync globals back to memory and kill. */
la_global_kill(TCGContext * s,int ng)3878 static void la_global_kill(TCGContext *s, int ng)
3879 {
3880 int i;
3881
3882 for (i = 0; i < ng; i++) {
3883 s->temps[i].state = TS_DEAD | TS_MEM;
3884 la_reset_pref(&s->temps[i]);
3885 }
3886 }
3887
3888 /* liveness analysis: note live globals crossing calls. */
la_cross_call(TCGContext * s,int nt)3889 static void la_cross_call(TCGContext *s, int nt)
3890 {
3891 TCGRegSet mask = ~tcg_target_call_clobber_regs;
3892 int i;
3893
3894 for (i = 0; i < nt; i++) {
3895 TCGTemp *ts = &s->temps[i];
3896 if (!(ts->state & TS_DEAD)) {
3897 TCGRegSet *pset = la_temp_pref(ts);
3898 TCGRegSet set = *pset;
3899
3900 set &= mask;
3901 /* If the combination is not possible, restart. */
3902 if (set == 0) {
3903 set = tcg_target_available_regs[ts->type] & mask;
3904 }
3905 *pset = set;
3906 }
3907 }
3908 }
3909
3910 /*
3911 * Liveness analysis: Verify the lifetime of TEMP_TB, and reduce
3912 * to TEMP_EBB, if possible.
3913 */
3914 static void __attribute__((noinline))
liveness_pass_0(TCGContext * s)3915 liveness_pass_0(TCGContext *s)
3916 {
3917 void * const multiple_ebb = (void *)(uintptr_t)-1;
3918 int nb_temps = s->nb_temps;
3919 TCGOp *op, *ebb;
3920
3921 for (int i = s->nb_globals; i < nb_temps; ++i) {
3922 s->temps[i].state_ptr = NULL;
3923 }
3924
3925 /*
3926 * Represent each EBB by the op at which it begins. In the case of
3927 * the first EBB, this is the first op, otherwise it is a label.
3928 * Collect the uses of each TEMP_TB: NULL for unused, EBB for use
3929 * within a single EBB, else MULTIPLE_EBB.
3930 */
3931 ebb = QTAILQ_FIRST(&s->ops);
3932 QTAILQ_FOREACH(op, &s->ops, link) {
3933 const TCGOpDef *def;
3934 int nb_oargs, nb_iargs;
3935
3936 switch (op->opc) {
3937 case INDEX_op_set_label:
3938 ebb = op;
3939 continue;
3940 case INDEX_op_discard:
3941 continue;
3942 case INDEX_op_call:
3943 nb_oargs = TCGOP_CALLO(op);
3944 nb_iargs = TCGOP_CALLI(op);
3945 break;
3946 default:
3947 def = &tcg_op_defs[op->opc];
3948 nb_oargs = def->nb_oargs;
3949 nb_iargs = def->nb_iargs;
3950 break;
3951 }
3952
3953 for (int i = 0; i < nb_oargs + nb_iargs; ++i) {
3954 TCGTemp *ts = arg_temp(op->args[i]);
3955
3956 if (ts->kind != TEMP_TB) {
3957 continue;
3958 }
3959 if (ts->state_ptr == NULL) {
3960 ts->state_ptr = ebb;
3961 } else if (ts->state_ptr != ebb) {
3962 ts->state_ptr = multiple_ebb;
3963 }
3964 }
3965 }
3966
3967 /*
3968 * For TEMP_TB that turned out not to be used beyond one EBB,
3969 * reduce the liveness to TEMP_EBB.
3970 */
3971 for (int i = s->nb_globals; i < nb_temps; ++i) {
3972 TCGTemp *ts = &s->temps[i];
3973 if (ts->kind == TEMP_TB && ts->state_ptr != multiple_ebb) {
3974 ts->kind = TEMP_EBB;
3975 }
3976 }
3977 }
3978
assert_carry_dead(TCGContext * s)3979 static void assert_carry_dead(TCGContext *s)
3980 {
3981 /*
3982 * Carry operations can be separated by a few insns like mov,
3983 * load or store, but they should always be "close", and
3984 * carry-out operations should always be paired with carry-in.
3985 * At various boundaries, carry must have been consumed.
3986 */
3987 tcg_debug_assert(!s->carry_live);
3988 }
3989
3990 /* Liveness analysis : update the opc_arg_life array to tell if a
3991 given input arguments is dead. Instructions updating dead
3992 temporaries are removed. */
3993 static void __attribute__((noinline))
liveness_pass_1(TCGContext * s)3994 liveness_pass_1(TCGContext *s)
3995 {
3996 int nb_globals = s->nb_globals;
3997 int nb_temps = s->nb_temps;
3998 TCGOp *op, *op_prev;
3999 TCGRegSet *prefs;
4000
4001 prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps);
4002 for (int i = 0; i < nb_temps; ++i) {
4003 s->temps[i].state_ptr = prefs + i;
4004 }
4005
4006 /* ??? Should be redundant with the exit_tb that ends the TB. */
4007 la_func_end(s, nb_globals, nb_temps);
4008
4009 s->carry_live = false;
4010 QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) {
4011 int nb_iargs, nb_oargs;
4012 TCGOpcode opc_new, opc_new2;
4013 TCGLifeData arg_life = 0;
4014 TCGTemp *ts;
4015 TCGOpcode opc = op->opc;
4016 const TCGOpDef *def;
4017 const TCGArgConstraint *args_ct;
4018
4019 switch (opc) {
4020 case INDEX_op_call:
4021 assert_carry_dead(s);
4022 {
4023 const TCGHelperInfo *info = tcg_call_info(op);
4024 int call_flags = tcg_call_flags(op);
4025
4026 nb_oargs = TCGOP_CALLO(op);
4027 nb_iargs = TCGOP_CALLI(op);
4028
4029 /* pure functions can be removed if their result is unused */
4030 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) {
4031 for (int i = 0; i < nb_oargs; i++) {
4032 ts = arg_temp(op->args[i]);
4033 if (ts->state != TS_DEAD) {
4034 goto do_not_remove_call;
4035 }
4036 }
4037 goto do_remove;
4038 }
4039 do_not_remove_call:
4040
4041 /* Output args are dead. */
4042 for (int i = 0; i < nb_oargs; i++) {
4043 ts = arg_temp(op->args[i]);
4044 if (ts->state & TS_DEAD) {
4045 arg_life |= DEAD_ARG << i;
4046 }
4047 if (ts->state & TS_MEM) {
4048 arg_life |= SYNC_ARG << i;
4049 }
4050 ts->state = TS_DEAD;
4051 la_reset_pref(ts);
4052 }
4053
4054 /* Not used -- it will be tcg_target_call_oarg_reg(). */
4055 memset(op->output_pref, 0, sizeof(op->output_pref));
4056
4057 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS |
4058 TCG_CALL_NO_READ_GLOBALS))) {
4059 la_global_kill(s, nb_globals);
4060 } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) {
4061 la_global_sync(s, nb_globals);
4062 }
4063
4064 /* Record arguments that die in this helper. */
4065 for (int i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
4066 ts = arg_temp(op->args[i]);
4067 if (ts->state & TS_DEAD) {
4068 arg_life |= DEAD_ARG << i;
4069 }
4070 }
4071
4072 /* For all live registers, remove call-clobbered prefs. */
4073 la_cross_call(s, nb_temps);
4074
4075 /*
4076 * Input arguments are live for preceding opcodes.
4077 *
4078 * For those arguments that die, and will be allocated in
4079 * registers, clear the register set for that arg, to be
4080 * filled in below. For args that will be on the stack,
4081 * reset to any available reg. Process arguments in reverse
4082 * order so that if a temp is used more than once, the stack
4083 * reset to max happens before the register reset to 0.
4084 */
4085 for (int i = nb_iargs - 1; i >= 0; i--) {
4086 const TCGCallArgumentLoc *loc = &info->in[i];
4087 ts = arg_temp(op->args[nb_oargs + i]);
4088
4089 if (ts->state & TS_DEAD) {
4090 switch (loc->kind) {
4091 case TCG_CALL_ARG_NORMAL:
4092 case TCG_CALL_ARG_EXTEND_U:
4093 case TCG_CALL_ARG_EXTEND_S:
4094 if (arg_slot_reg_p(loc->arg_slot)) {
4095 *la_temp_pref(ts) = 0;
4096 break;
4097 }
4098 /* fall through */
4099 default:
4100 *la_temp_pref(ts) =
4101 tcg_target_available_regs[ts->type];
4102 break;
4103 }
4104 ts->state &= ~TS_DEAD;
4105 }
4106 }
4107
4108 /*
4109 * For each input argument, add its input register to prefs.
4110 * If a temp is used once, this produces a single set bit;
4111 * if a temp is used multiple times, this produces a set.
4112 */
4113 for (int i = 0; i < nb_iargs; i++) {
4114 const TCGCallArgumentLoc *loc = &info->in[i];
4115 ts = arg_temp(op->args[nb_oargs + i]);
4116
4117 switch (loc->kind) {
4118 case TCG_CALL_ARG_NORMAL:
4119 case TCG_CALL_ARG_EXTEND_U:
4120 case TCG_CALL_ARG_EXTEND_S:
4121 if (arg_slot_reg_p(loc->arg_slot)) {
4122 tcg_regset_set_reg(*la_temp_pref(ts),
4123 tcg_target_call_iarg_regs[loc->arg_slot]);
4124 }
4125 break;
4126 default:
4127 break;
4128 }
4129 }
4130 }
4131 break;
4132 case INDEX_op_insn_start:
4133 assert_carry_dead(s);
4134 break;
4135 case INDEX_op_discard:
4136 /* mark the temporary as dead */
4137 ts = arg_temp(op->args[0]);
4138 ts->state = TS_DEAD;
4139 la_reset_pref(ts);
4140 break;
4141
4142 case INDEX_op_muls2:
4143 opc_new = INDEX_op_mul;
4144 opc_new2 = INDEX_op_mulsh;
4145 goto do_mul2;
4146 case INDEX_op_mulu2:
4147 opc_new = INDEX_op_mul;
4148 opc_new2 = INDEX_op_muluh;
4149 do_mul2:
4150 assert_carry_dead(s);
4151 if (arg_temp(op->args[1])->state == TS_DEAD) {
4152 if (arg_temp(op->args[0])->state == TS_DEAD) {
4153 /* Both parts of the operation are dead. */
4154 goto do_remove;
4155 }
4156 /* The high part of the operation is dead; generate the low. */
4157 op->opc = opc = opc_new;
4158 op->args[1] = op->args[2];
4159 op->args[2] = op->args[3];
4160 } else if (arg_temp(op->args[0])->state == TS_DEAD &&
4161 tcg_op_supported(opc_new2, TCGOP_TYPE(op), 0)) {
4162 /* The low part of the operation is dead; generate the high. */
4163 op->opc = opc = opc_new2;
4164 op->args[0] = op->args[1];
4165 op->args[1] = op->args[2];
4166 op->args[2] = op->args[3];
4167 } else {
4168 goto do_not_remove;
4169 }
4170 /* Mark the single-word operation live. */
4171 goto do_not_remove;
4172
4173 case INDEX_op_addco:
4174 if (s->carry_live) {
4175 goto do_not_remove;
4176 }
4177 op->opc = opc = INDEX_op_add;
4178 goto do_default;
4179
4180 case INDEX_op_addcio:
4181 if (s->carry_live) {
4182 goto do_not_remove;
4183 }
4184 op->opc = opc = INDEX_op_addci;
4185 goto do_default;
4186
4187 case INDEX_op_subbo:
4188 if (s->carry_live) {
4189 goto do_not_remove;
4190 }
4191 /* Lower to sub, but this may also require canonicalization. */
4192 op->opc = opc = INDEX_op_sub;
4193 ts = arg_temp(op->args[2]);
4194 if (ts->kind == TEMP_CONST) {
4195 ts = tcg_constant_internal(ts->type, -ts->val);
4196 if (ts->state_ptr == NULL) {
4197 tcg_debug_assert(temp_idx(ts) == nb_temps);
4198 nb_temps++;
4199 ts->state_ptr = tcg_malloc(sizeof(TCGRegSet));
4200 ts->state = TS_DEAD;
4201 la_reset_pref(ts);
4202 }
4203 op->args[2] = temp_arg(ts);
4204 op->opc = opc = INDEX_op_add;
4205 }
4206 goto do_default;
4207
4208 case INDEX_op_subbio:
4209 if (s->carry_live) {
4210 goto do_not_remove;
4211 }
4212 op->opc = opc = INDEX_op_subbi;
4213 goto do_default;
4214
4215 case INDEX_op_addc1o:
4216 if (s->carry_live) {
4217 goto do_not_remove;
4218 }
4219 /* Lower to add, add +1. */
4220 op_prev = tcg_op_insert_before(s, op, INDEX_op_add,
4221 TCGOP_TYPE(op), 3);
4222 op_prev->args[0] = op->args[0];
4223 op_prev->args[1] = op->args[1];
4224 op_prev->args[2] = op->args[2];
4225 op->opc = opc = INDEX_op_add;
4226 op->args[1] = op->args[0];
4227 ts = arg_temp(op->args[0]);
4228 ts = tcg_constant_internal(ts->type, 1);
4229 op->args[2] = temp_arg(ts);
4230 goto do_default;
4231
4232 case INDEX_op_subb1o:
4233 if (s->carry_live) {
4234 goto do_not_remove;
4235 }
4236 /* Lower to sub, add -1. */
4237 op_prev = tcg_op_insert_before(s, op, INDEX_op_sub,
4238 TCGOP_TYPE(op), 3);
4239 op_prev->args[0] = op->args[0];
4240 op_prev->args[1] = op->args[1];
4241 op_prev->args[2] = op->args[2];
4242 op->opc = opc = INDEX_op_add;
4243 op->args[1] = op->args[0];
4244 ts = arg_temp(op->args[0]);
4245 ts = tcg_constant_internal(ts->type, -1);
4246 op->args[2] = temp_arg(ts);
4247 goto do_default;
4248
4249 default:
4250 do_default:
4251 /*
4252 * Test if the operation can be removed because all
4253 * its outputs are dead. We assume that nb_oargs == 0
4254 * implies side effects.
4255 */
4256 def = &tcg_op_defs[opc];
4257 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && def->nb_oargs != 0) {
4258 for (int i = def->nb_oargs - 1; i >= 0; i--) {
4259 if (arg_temp(op->args[i])->state != TS_DEAD) {
4260 goto do_not_remove;
4261 }
4262 }
4263 goto do_remove;
4264 }
4265 goto do_not_remove;
4266
4267 do_remove:
4268 tcg_op_remove(s, op);
4269 break;
4270
4271 do_not_remove:
4272 def = &tcg_op_defs[opc];
4273 nb_iargs = def->nb_iargs;
4274 nb_oargs = def->nb_oargs;
4275
4276 for (int i = 0; i < nb_oargs; i++) {
4277 ts = arg_temp(op->args[i]);
4278
4279 /* Remember the preference of the uses that followed. */
4280 if (i < ARRAY_SIZE(op->output_pref)) {
4281 op->output_pref[i] = *la_temp_pref(ts);
4282 }
4283
4284 /* Output args are dead. */
4285 if (ts->state & TS_DEAD) {
4286 arg_life |= DEAD_ARG << i;
4287 }
4288 if (ts->state & TS_MEM) {
4289 arg_life |= SYNC_ARG << i;
4290 }
4291 ts->state = TS_DEAD;
4292 la_reset_pref(ts);
4293 }
4294
4295 /* If end of basic block, update. */
4296 if (def->flags & TCG_OPF_BB_EXIT) {
4297 assert_carry_dead(s);
4298 la_func_end(s, nb_globals, nb_temps);
4299 } else if (def->flags & TCG_OPF_COND_BRANCH) {
4300 assert_carry_dead(s);
4301 la_bb_sync(s, nb_globals, nb_temps);
4302 } else if (def->flags & TCG_OPF_BB_END) {
4303 assert_carry_dead(s);
4304 la_bb_end(s, nb_globals, nb_temps);
4305 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
4306 assert_carry_dead(s);
4307 la_global_sync(s, nb_globals);
4308 if (def->flags & TCG_OPF_CALL_CLOBBER) {
4309 la_cross_call(s, nb_temps);
4310 }
4311 }
4312
4313 /* Record arguments that die in this opcode. */
4314 for (int i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
4315 ts = arg_temp(op->args[i]);
4316 if (ts->state & TS_DEAD) {
4317 arg_life |= DEAD_ARG << i;
4318 }
4319 }
4320 if (def->flags & TCG_OPF_CARRY_OUT) {
4321 s->carry_live = false;
4322 }
4323
4324 /* Input arguments are live for preceding opcodes. */
4325 for (int i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
4326 ts = arg_temp(op->args[i]);
4327 if (ts->state & TS_DEAD) {
4328 /* For operands that were dead, initially allow
4329 all regs for the type. */
4330 *la_temp_pref(ts) = tcg_target_available_regs[ts->type];
4331 ts->state &= ~TS_DEAD;
4332 }
4333 }
4334 if (def->flags & TCG_OPF_CARRY_IN) {
4335 s->carry_live = true;
4336 }
4337
4338 /* Incorporate constraints for this operand. */
4339 switch (opc) {
4340 case INDEX_op_mov:
4341 /* Note that these are TCG_OPF_NOT_PRESENT and do not
4342 have proper constraints. That said, special case
4343 moves to propagate preferences backward. */
4344 if (IS_DEAD_ARG(1)) {
4345 *la_temp_pref(arg_temp(op->args[0]))
4346 = *la_temp_pref(arg_temp(op->args[1]));
4347 }
4348 break;
4349
4350 default:
4351 args_ct = opcode_args_ct(op);
4352 for (int i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
4353 const TCGArgConstraint *ct = &args_ct[i];
4354 TCGRegSet set, *pset;
4355
4356 ts = arg_temp(op->args[i]);
4357 pset = la_temp_pref(ts);
4358 set = *pset;
4359
4360 set &= ct->regs;
4361 if (ct->ialias) {
4362 set &= output_pref(op, ct->alias_index);
4363 }
4364 /* If the combination is not possible, restart. */
4365 if (set == 0) {
4366 set = ct->regs;
4367 }
4368 *pset = set;
4369 }
4370 break;
4371 }
4372 break;
4373 }
4374 op->life = arg_life;
4375 }
4376 assert_carry_dead(s);
4377 }
4378
4379 /* Liveness analysis: Convert indirect regs to direct temporaries. */
4380 static bool __attribute__((noinline))
liveness_pass_2(TCGContext * s)4381 liveness_pass_2(TCGContext *s)
4382 {
4383 int nb_globals = s->nb_globals;
4384 int nb_temps, i;
4385 bool changes = false;
4386 TCGOp *op, *op_next;
4387
4388 /* Create a temporary for each indirect global. */
4389 for (i = 0; i < nb_globals; ++i) {
4390 TCGTemp *its = &s->temps[i];
4391 if (its->indirect_reg) {
4392 TCGTemp *dts = tcg_temp_alloc(s);
4393 dts->type = its->type;
4394 dts->base_type = its->base_type;
4395 dts->temp_subindex = its->temp_subindex;
4396 dts->kind = TEMP_EBB;
4397 its->state_ptr = dts;
4398 } else {
4399 its->state_ptr = NULL;
4400 }
4401 /* All globals begin dead. */
4402 its->state = TS_DEAD;
4403 }
4404 for (nb_temps = s->nb_temps; i < nb_temps; ++i) {
4405 TCGTemp *its = &s->temps[i];
4406 its->state_ptr = NULL;
4407 its->state = TS_DEAD;
4408 }
4409
4410 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
4411 TCGOpcode opc = op->opc;
4412 const TCGOpDef *def = &tcg_op_defs[opc];
4413 TCGLifeData arg_life = op->life;
4414 int nb_iargs, nb_oargs, call_flags;
4415 TCGTemp *arg_ts, *dir_ts;
4416
4417 if (opc == INDEX_op_call) {
4418 nb_oargs = TCGOP_CALLO(op);
4419 nb_iargs = TCGOP_CALLI(op);
4420 call_flags = tcg_call_flags(op);
4421 } else {
4422 nb_iargs = def->nb_iargs;
4423 nb_oargs = def->nb_oargs;
4424
4425 /* Set flags similar to how calls require. */
4426 if (def->flags & TCG_OPF_COND_BRANCH) {
4427 /* Like reading globals: sync_globals */
4428 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
4429 } else if (def->flags & TCG_OPF_BB_END) {
4430 /* Like writing globals: save_globals */
4431 call_flags = 0;
4432 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
4433 /* Like reading globals: sync_globals */
4434 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
4435 } else {
4436 /* No effect on globals. */
4437 call_flags = (TCG_CALL_NO_READ_GLOBALS |
4438 TCG_CALL_NO_WRITE_GLOBALS);
4439 }
4440 }
4441
4442 /* Make sure that input arguments are available. */
4443 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
4444 arg_ts = arg_temp(op->args[i]);
4445 dir_ts = arg_ts->state_ptr;
4446 if (dir_ts && arg_ts->state == TS_DEAD) {
4447 TCGOp *lop = tcg_op_insert_before(s, op, INDEX_op_ld,
4448 arg_ts->type, 3);
4449
4450 lop->args[0] = temp_arg(dir_ts);
4451 lop->args[1] = temp_arg(arg_ts->mem_base);
4452 lop->args[2] = arg_ts->mem_offset;
4453
4454 /* Loaded, but synced with memory. */
4455 arg_ts->state = TS_MEM;
4456 }
4457 }
4458
4459 /* Perform input replacement, and mark inputs that became dead.
4460 No action is required except keeping temp_state up to date
4461 so that we reload when needed. */
4462 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
4463 arg_ts = arg_temp(op->args[i]);
4464 dir_ts = arg_ts->state_ptr;
4465 if (dir_ts) {
4466 op->args[i] = temp_arg(dir_ts);
4467 changes = true;
4468 if (IS_DEAD_ARG(i)) {
4469 arg_ts->state = TS_DEAD;
4470 }
4471 }
4472 }
4473
4474 /* Liveness analysis should ensure that the following are
4475 all correct, for call sites and basic block end points. */
4476 if (call_flags & TCG_CALL_NO_READ_GLOBALS) {
4477 /* Nothing to do */
4478 } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) {
4479 for (i = 0; i < nb_globals; ++i) {
4480 /* Liveness should see that globals are synced back,
4481 that is, either TS_DEAD or TS_MEM. */
4482 arg_ts = &s->temps[i];
4483 tcg_debug_assert(arg_ts->state_ptr == 0
4484 || arg_ts->state != 0);
4485 }
4486 } else {
4487 for (i = 0; i < nb_globals; ++i) {
4488 /* Liveness should see that globals are saved back,
4489 that is, TS_DEAD, waiting to be reloaded. */
4490 arg_ts = &s->temps[i];
4491 tcg_debug_assert(arg_ts->state_ptr == 0
4492 || arg_ts->state == TS_DEAD);
4493 }
4494 }
4495
4496 /* Outputs become available. */
4497 if (opc == INDEX_op_mov) {
4498 arg_ts = arg_temp(op->args[0]);
4499 dir_ts = arg_ts->state_ptr;
4500 if (dir_ts) {
4501 op->args[0] = temp_arg(dir_ts);
4502 changes = true;
4503
4504 /* The output is now live and modified. */
4505 arg_ts->state = 0;
4506
4507 if (NEED_SYNC_ARG(0)) {
4508 TCGOp *sop = tcg_op_insert_after(s, op, INDEX_op_st,
4509 arg_ts->type, 3);
4510 TCGTemp *out_ts = dir_ts;
4511
4512 if (IS_DEAD_ARG(0)) {
4513 out_ts = arg_temp(op->args[1]);
4514 arg_ts->state = TS_DEAD;
4515 tcg_op_remove(s, op);
4516 } else {
4517 arg_ts->state = TS_MEM;
4518 }
4519
4520 sop->args[0] = temp_arg(out_ts);
4521 sop->args[1] = temp_arg(arg_ts->mem_base);
4522 sop->args[2] = arg_ts->mem_offset;
4523 } else {
4524 tcg_debug_assert(!IS_DEAD_ARG(0));
4525 }
4526 }
4527 } else {
4528 for (i = 0; i < nb_oargs; i++) {
4529 arg_ts = arg_temp(op->args[i]);
4530 dir_ts = arg_ts->state_ptr;
4531 if (!dir_ts) {
4532 continue;
4533 }
4534 op->args[i] = temp_arg(dir_ts);
4535 changes = true;
4536
4537 /* The output is now live and modified. */
4538 arg_ts->state = 0;
4539
4540 /* Sync outputs upon their last write. */
4541 if (NEED_SYNC_ARG(i)) {
4542 TCGOp *sop = tcg_op_insert_after(s, op, INDEX_op_st,
4543 arg_ts->type, 3);
4544
4545 sop->args[0] = temp_arg(dir_ts);
4546 sop->args[1] = temp_arg(arg_ts->mem_base);
4547 sop->args[2] = arg_ts->mem_offset;
4548
4549 arg_ts->state = TS_MEM;
4550 }
4551 /* Drop outputs that are dead. */
4552 if (IS_DEAD_ARG(i)) {
4553 arg_ts->state = TS_DEAD;
4554 }
4555 }
4556 }
4557 }
4558
4559 return changes;
4560 }
4561
temp_allocate_frame(TCGContext * s,TCGTemp * ts)4562 static void temp_allocate_frame(TCGContext *s, TCGTemp *ts)
4563 {
4564 intptr_t off;
4565 int size, align;
4566
4567 /* When allocating an object, look at the full type. */
4568 size = tcg_type_size(ts->base_type);
4569 switch (ts->base_type) {
4570 case TCG_TYPE_I32:
4571 align = 4;
4572 break;
4573 case TCG_TYPE_I64:
4574 case TCG_TYPE_V64:
4575 align = 8;
4576 break;
4577 case TCG_TYPE_I128:
4578 case TCG_TYPE_V128:
4579 case TCG_TYPE_V256:
4580 /*
4581 * Note that we do not require aligned storage for V256,
4582 * and that we provide alignment for I128 to match V128,
4583 * even if that's above what the host ABI requires.
4584 */
4585 align = 16;
4586 break;
4587 default:
4588 g_assert_not_reached();
4589 }
4590
4591 /*
4592 * Assume the stack is sufficiently aligned.
4593 * This affects e.g. ARM NEON, where we have 8 byte stack alignment
4594 * and do not require 16 byte vector alignment. This seems slightly
4595 * easier than fully parameterizing the above switch statement.
4596 */
4597 align = MIN(TCG_TARGET_STACK_ALIGN, align);
4598 off = ROUND_UP(s->current_frame_offset, align);
4599
4600 /* If we've exhausted the stack frame, restart with a smaller TB. */
4601 if (off + size > s->frame_end) {
4602 tcg_raise_tb_overflow(s);
4603 }
4604 s->current_frame_offset = off + size;
4605 #if defined(__sparc__)
4606 off += TCG_TARGET_STACK_BIAS;
4607 #endif
4608
4609 /* If the object was subdivided, assign memory to all the parts. */
4610 if (ts->base_type != ts->type) {
4611 int part_size = tcg_type_size(ts->type);
4612 int part_count = size / part_size;
4613
4614 /*
4615 * Each part is allocated sequentially in tcg_temp_new_internal.
4616 * Jump back to the first part by subtracting the current index.
4617 */
4618 ts -= ts->temp_subindex;
4619 for (int i = 0; i < part_count; ++i) {
4620 ts[i].mem_offset = off + i * part_size;
4621 ts[i].mem_base = s->frame_temp;
4622 ts[i].mem_allocated = 1;
4623 }
4624 } else {
4625 ts->mem_offset = off;
4626 ts->mem_base = s->frame_temp;
4627 ts->mem_allocated = 1;
4628 }
4629 }
4630
4631 /* Assign @reg to @ts, and update reg_to_temp[]. */
set_temp_val_reg(TCGContext * s,TCGTemp * ts,TCGReg reg)4632 static void set_temp_val_reg(TCGContext *s, TCGTemp *ts, TCGReg reg)
4633 {
4634 if (ts->val_type == TEMP_VAL_REG) {
4635 TCGReg old = ts->reg;
4636 tcg_debug_assert(s->reg_to_temp[old] == ts);
4637 if (old == reg) {
4638 return;
4639 }
4640 s->reg_to_temp[old] = NULL;
4641 }
4642 tcg_debug_assert(s->reg_to_temp[reg] == NULL);
4643 s->reg_to_temp[reg] = ts;
4644 ts->val_type = TEMP_VAL_REG;
4645 ts->reg = reg;
4646 }
4647
4648 /* Assign a non-register value type to @ts, and update reg_to_temp[]. */
set_temp_val_nonreg(TCGContext * s,TCGTemp * ts,TCGTempVal type)4649 static void set_temp_val_nonreg(TCGContext *s, TCGTemp *ts, TCGTempVal type)
4650 {
4651 tcg_debug_assert(type != TEMP_VAL_REG);
4652 if (ts->val_type == TEMP_VAL_REG) {
4653 TCGReg reg = ts->reg;
4654 tcg_debug_assert(s->reg_to_temp[reg] == ts);
4655 s->reg_to_temp[reg] = NULL;
4656 }
4657 ts->val_type = type;
4658 }
4659
4660 static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet);
4661
4662 /* Mark a temporary as free or dead. If 'free_or_dead' is negative,
4663 mark it free; otherwise mark it dead. */
temp_free_or_dead(TCGContext * s,TCGTemp * ts,int free_or_dead)4664 static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead)
4665 {
4666 TCGTempVal new_type;
4667
4668 switch (ts->kind) {
4669 case TEMP_FIXED:
4670 return;
4671 case TEMP_GLOBAL:
4672 case TEMP_TB:
4673 new_type = TEMP_VAL_MEM;
4674 break;
4675 case TEMP_EBB:
4676 new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD;
4677 break;
4678 case TEMP_CONST:
4679 new_type = TEMP_VAL_CONST;
4680 break;
4681 default:
4682 g_assert_not_reached();
4683 }
4684 set_temp_val_nonreg(s, ts, new_type);
4685 }
4686
4687 /* Mark a temporary as dead. */
temp_dead(TCGContext * s,TCGTemp * ts)4688 static inline void temp_dead(TCGContext *s, TCGTemp *ts)
4689 {
4690 temp_free_or_dead(s, ts, 1);
4691 }
4692
4693 /* Sync a temporary to memory. 'allocated_regs' is used in case a temporary
4694 registers needs to be allocated to store a constant. If 'free_or_dead'
4695 is non-zero, subsequently release the temporary; if it is positive, the
4696 temp is dead; if it is negative, the temp is free. */
temp_sync(TCGContext * s,TCGTemp * ts,TCGRegSet allocated_regs,TCGRegSet preferred_regs,int free_or_dead)4697 static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs,
4698 TCGRegSet preferred_regs, int free_or_dead)
4699 {
4700 if (!temp_readonly(ts) && !ts->mem_coherent) {
4701 if (!ts->mem_allocated) {
4702 temp_allocate_frame(s, ts);
4703 }
4704 switch (ts->val_type) {
4705 case TEMP_VAL_CONST:
4706 /* If we're going to free the temp immediately, then we won't
4707 require it later in a register, so attempt to store the
4708 constant to memory directly. */
4709 if (free_or_dead
4710 && tcg_out_sti(s, ts->type, ts->val,
4711 ts->mem_base->reg, ts->mem_offset)) {
4712 break;
4713 }
4714 temp_load(s, ts, tcg_target_available_regs[ts->type],
4715 allocated_regs, preferred_regs);
4716 /* fallthrough */
4717
4718 case TEMP_VAL_REG:
4719 tcg_out_st(s, ts->type, ts->reg,
4720 ts->mem_base->reg, ts->mem_offset);
4721 break;
4722
4723 case TEMP_VAL_MEM:
4724 break;
4725
4726 case TEMP_VAL_DEAD:
4727 default:
4728 g_assert_not_reached();
4729 }
4730 ts->mem_coherent = 1;
4731 }
4732 if (free_or_dead) {
4733 temp_free_or_dead(s, ts, free_or_dead);
4734 }
4735 }
4736
4737 /* free register 'reg' by spilling the corresponding temporary if necessary */
tcg_reg_free(TCGContext * s,TCGReg reg,TCGRegSet allocated_regs)4738 static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs)
4739 {
4740 TCGTemp *ts = s->reg_to_temp[reg];
4741 if (ts != NULL) {
4742 temp_sync(s, ts, allocated_regs, 0, -1);
4743 }
4744 }
4745
4746 /**
4747 * tcg_reg_alloc:
4748 * @required_regs: Set of registers in which we must allocate.
4749 * @allocated_regs: Set of registers which must be avoided.
4750 * @preferred_regs: Set of registers we should prefer.
4751 * @rev: True if we search the registers in "indirect" order.
4752 *
4753 * The allocated register must be in @required_regs & ~@allocated_regs,
4754 * but if we can put it in @preferred_regs we may save a move later.
4755 */
tcg_reg_alloc(TCGContext * s,TCGRegSet required_regs,TCGRegSet allocated_regs,TCGRegSet preferred_regs,bool rev)4756 static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs,
4757 TCGRegSet allocated_regs,
4758 TCGRegSet preferred_regs, bool rev)
4759 {
4760 int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
4761 TCGRegSet reg_ct[2];
4762 const int *order;
4763
4764 reg_ct[1] = required_regs & ~allocated_regs;
4765 tcg_debug_assert(reg_ct[1] != 0);
4766 reg_ct[0] = reg_ct[1] & preferred_regs;
4767
4768 /* Skip the preferred_regs option if it cannot be satisfied,
4769 or if the preference made no difference. */
4770 f = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1];
4771
4772 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
4773
4774 /* Try free registers, preferences first. */
4775 for (j = f; j < 2; j++) {
4776 TCGRegSet set = reg_ct[j];
4777
4778 if (tcg_regset_single(set)) {
4779 /* One register in the set. */
4780 TCGReg reg = tcg_regset_first(set);
4781 if (s->reg_to_temp[reg] == NULL) {
4782 return reg;
4783 }
4784 } else {
4785 for (i = 0; i < n; i++) {
4786 TCGReg reg = order[i];
4787 if (s->reg_to_temp[reg] == NULL &&
4788 tcg_regset_test_reg(set, reg)) {
4789 return reg;
4790 }
4791 }
4792 }
4793 }
4794
4795 /* We must spill something. */
4796 for (j = f; j < 2; j++) {
4797 TCGRegSet set = reg_ct[j];
4798
4799 if (tcg_regset_single(set)) {
4800 /* One register in the set. */
4801 TCGReg reg = tcg_regset_first(set);
4802 tcg_reg_free(s, reg, allocated_regs);
4803 return reg;
4804 } else {
4805 for (i = 0; i < n; i++) {
4806 TCGReg reg = order[i];
4807 if (tcg_regset_test_reg(set, reg)) {
4808 tcg_reg_free(s, reg, allocated_regs);
4809 return reg;
4810 }
4811 }
4812 }
4813 }
4814
4815 g_assert_not_reached();
4816 }
4817
tcg_reg_alloc_pair(TCGContext * s,TCGRegSet required_regs,TCGRegSet allocated_regs,TCGRegSet preferred_regs,bool rev)4818 static TCGReg tcg_reg_alloc_pair(TCGContext *s, TCGRegSet required_regs,
4819 TCGRegSet allocated_regs,
4820 TCGRegSet preferred_regs, bool rev)
4821 {
4822 int i, j, k, fmin, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
4823 TCGRegSet reg_ct[2];
4824 const int *order;
4825
4826 /* Ensure that if I is not in allocated_regs, I+1 is not either. */
4827 reg_ct[1] = required_regs & ~(allocated_regs | (allocated_regs >> 1));
4828 tcg_debug_assert(reg_ct[1] != 0);
4829 reg_ct[0] = reg_ct[1] & preferred_regs;
4830
4831 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
4832
4833 /*
4834 * Skip the preferred_regs option if it cannot be satisfied,
4835 * or if the preference made no difference.
4836 */
4837 k = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1];
4838
4839 /*
4840 * Minimize the number of flushes by looking for 2 free registers first,
4841 * then a single flush, then two flushes.
4842 */
4843 for (fmin = 2; fmin >= 0; fmin--) {
4844 for (j = k; j < 2; j++) {
4845 TCGRegSet set = reg_ct[j];
4846
4847 for (i = 0; i < n; i++) {
4848 TCGReg reg = order[i];
4849
4850 if (tcg_regset_test_reg(set, reg)) {
4851 int f = !s->reg_to_temp[reg] + !s->reg_to_temp[reg + 1];
4852 if (f >= fmin) {
4853 tcg_reg_free(s, reg, allocated_regs);
4854 tcg_reg_free(s, reg + 1, allocated_regs);
4855 return reg;
4856 }
4857 }
4858 }
4859 }
4860 }
4861 g_assert_not_reached();
4862 }
4863
4864 /* Make sure the temporary is in a register. If needed, allocate the register
4865 from DESIRED while avoiding ALLOCATED. */
temp_load(TCGContext * s,TCGTemp * ts,TCGRegSet desired_regs,TCGRegSet allocated_regs,TCGRegSet preferred_regs)4866 static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs,
4867 TCGRegSet allocated_regs, TCGRegSet preferred_regs)
4868 {
4869 TCGReg reg;
4870
4871 switch (ts->val_type) {
4872 case TEMP_VAL_REG:
4873 return;
4874 case TEMP_VAL_CONST:
4875 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
4876 preferred_regs, ts->indirect_base);
4877 if (ts->type <= TCG_TYPE_I64) {
4878 tcg_out_movi(s, ts->type, reg, ts->val);
4879 } else {
4880 uint64_t val = ts->val;
4881 MemOp vece = MO_64;
4882
4883 /*
4884 * Find the minimal vector element that matches the constant.
4885 * The targets will, in general, have to do this search anyway,
4886 * do this generically.
4887 */
4888 if (val == dup_const(MO_8, val)) {
4889 vece = MO_8;
4890 } else if (val == dup_const(MO_16, val)) {
4891 vece = MO_16;
4892 } else if (val == dup_const(MO_32, val)) {
4893 vece = MO_32;
4894 }
4895
4896 tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val);
4897 }
4898 ts->mem_coherent = 0;
4899 break;
4900 case TEMP_VAL_MEM:
4901 if (!ts->mem_allocated) {
4902 temp_allocate_frame(s, ts);
4903 }
4904 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
4905 preferred_regs, ts->indirect_base);
4906 tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset);
4907 ts->mem_coherent = 1;
4908 break;
4909 case TEMP_VAL_DEAD:
4910 default:
4911 g_assert_not_reached();
4912 }
4913 set_temp_val_reg(s, ts, reg);
4914 }
4915
4916 /* Save a temporary to memory. 'allocated_regs' is used in case a
4917 temporary registers needs to be allocated to store a constant. */
temp_save(TCGContext * s,TCGTemp * ts,TCGRegSet allocated_regs)4918 static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs)
4919 {
4920 /* The liveness analysis already ensures that globals are back
4921 in memory. Keep an tcg_debug_assert for safety. */
4922 tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts));
4923 }
4924
4925 /* save globals to their canonical location and assume they can be
4926 modified be the following code. 'allocated_regs' is used in case a
4927 temporary registers needs to be allocated to store a constant. */
save_globals(TCGContext * s,TCGRegSet allocated_regs)4928 static void save_globals(TCGContext *s, TCGRegSet allocated_regs)
4929 {
4930 int i, n;
4931
4932 for (i = 0, n = s->nb_globals; i < n; i++) {
4933 temp_save(s, &s->temps[i], allocated_regs);
4934 }
4935 }
4936
4937 /* sync globals to their canonical location and assume they can be
4938 read by the following code. 'allocated_regs' is used in case a
4939 temporary registers needs to be allocated to store a constant. */
sync_globals(TCGContext * s,TCGRegSet allocated_regs)4940 static void sync_globals(TCGContext *s, TCGRegSet allocated_regs)
4941 {
4942 int i, n;
4943
4944 for (i = 0, n = s->nb_globals; i < n; i++) {
4945 TCGTemp *ts = &s->temps[i];
4946 tcg_debug_assert(ts->val_type != TEMP_VAL_REG
4947 || ts->kind == TEMP_FIXED
4948 || ts->mem_coherent);
4949 }
4950 }
4951
4952 /* at the end of a basic block, we assume all temporaries are dead and
4953 all globals are stored at their canonical location. */
tcg_reg_alloc_bb_end(TCGContext * s,TCGRegSet allocated_regs)4954 static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs)
4955 {
4956 assert_carry_dead(s);
4957 for (int i = s->nb_globals; i < s->nb_temps; i++) {
4958 TCGTemp *ts = &s->temps[i];
4959
4960 switch (ts->kind) {
4961 case TEMP_TB:
4962 temp_save(s, ts, allocated_regs);
4963 break;
4964 case TEMP_EBB:
4965 /* The liveness analysis already ensures that temps are dead.
4966 Keep an tcg_debug_assert for safety. */
4967 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
4968 break;
4969 case TEMP_CONST:
4970 /* Similarly, we should have freed any allocated register. */
4971 tcg_debug_assert(ts->val_type == TEMP_VAL_CONST);
4972 break;
4973 default:
4974 g_assert_not_reached();
4975 }
4976 }
4977
4978 save_globals(s, allocated_regs);
4979 }
4980
4981 /*
4982 * At a conditional branch, we assume all temporaries are dead unless
4983 * explicitly live-across-conditional-branch; all globals and local
4984 * temps are synced to their location.
4985 */
tcg_reg_alloc_cbranch(TCGContext * s,TCGRegSet allocated_regs)4986 static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs)
4987 {
4988 assert_carry_dead(s);
4989 sync_globals(s, allocated_regs);
4990
4991 for (int i = s->nb_globals; i < s->nb_temps; i++) {
4992 TCGTemp *ts = &s->temps[i];
4993 /*
4994 * The liveness analysis already ensures that temps are dead.
4995 * Keep tcg_debug_asserts for safety.
4996 */
4997 switch (ts->kind) {
4998 case TEMP_TB:
4999 tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent);
5000 break;
5001 case TEMP_EBB:
5002 case TEMP_CONST:
5003 break;
5004 default:
5005 g_assert_not_reached();
5006 }
5007 }
5008 }
5009
5010 /*
5011 * Specialized code generation for INDEX_op_mov_* with a constant.
5012 */
tcg_reg_alloc_do_movi(TCGContext * s,TCGTemp * ots,tcg_target_ulong val,TCGLifeData arg_life,TCGRegSet preferred_regs)5013 static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots,
5014 tcg_target_ulong val, TCGLifeData arg_life,
5015 TCGRegSet preferred_regs)
5016 {
5017 /* ENV should not be modified. */
5018 tcg_debug_assert(!temp_readonly(ots));
5019
5020 /* The movi is not explicitly generated here. */
5021 set_temp_val_nonreg(s, ots, TEMP_VAL_CONST);
5022 ots->val = val;
5023 ots->mem_coherent = 0;
5024 if (NEED_SYNC_ARG(0)) {
5025 temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0));
5026 } else if (IS_DEAD_ARG(0)) {
5027 temp_dead(s, ots);
5028 }
5029 }
5030
5031 /*
5032 * Specialized code generation for INDEX_op_mov_*.
5033 */
tcg_reg_alloc_mov(TCGContext * s,const TCGOp * op)5034 static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op)
5035 {
5036 const TCGLifeData arg_life = op->life;
5037 TCGRegSet allocated_regs, preferred_regs;
5038 TCGTemp *ts, *ots;
5039 TCGType otype, itype;
5040 TCGReg oreg, ireg;
5041
5042 allocated_regs = s->reserved_regs;
5043 preferred_regs = output_pref(op, 0);
5044 ots = arg_temp(op->args[0]);
5045 ts = arg_temp(op->args[1]);
5046
5047 /* ENV should not be modified. */
5048 tcg_debug_assert(!temp_readonly(ots));
5049
5050 /* Note that otype != itype for no-op truncation. */
5051 otype = ots->type;
5052 itype = ts->type;
5053
5054 if (ts->val_type == TEMP_VAL_CONST) {
5055 /* propagate constant or generate sti */
5056 tcg_target_ulong val = ts->val;
5057 if (IS_DEAD_ARG(1)) {
5058 temp_dead(s, ts);
5059 }
5060 tcg_reg_alloc_do_movi(s, ots, val, arg_life, preferred_regs);
5061 return;
5062 }
5063
5064 /* If the source value is in memory we're going to be forced
5065 to have it in a register in order to perform the copy. Copy
5066 the SOURCE value into its own register first, that way we
5067 don't have to reload SOURCE the next time it is used. */
5068 if (ts->val_type == TEMP_VAL_MEM) {
5069 temp_load(s, ts, tcg_target_available_regs[itype],
5070 allocated_regs, preferred_regs);
5071 }
5072 tcg_debug_assert(ts->val_type == TEMP_VAL_REG);
5073 ireg = ts->reg;
5074
5075 if (IS_DEAD_ARG(0)) {
5076 /* mov to a non-saved dead register makes no sense (even with
5077 liveness analysis disabled). */
5078 tcg_debug_assert(NEED_SYNC_ARG(0));
5079 if (!ots->mem_allocated) {
5080 temp_allocate_frame(s, ots);
5081 }
5082 tcg_out_st(s, otype, ireg, ots->mem_base->reg, ots->mem_offset);
5083 if (IS_DEAD_ARG(1)) {
5084 temp_dead(s, ts);
5085 }
5086 temp_dead(s, ots);
5087 return;
5088 }
5089
5090 if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) {
5091 /*
5092 * The mov can be suppressed. Kill input first, so that it
5093 * is unlinked from reg_to_temp, then set the output to the
5094 * reg that we saved from the input.
5095 */
5096 temp_dead(s, ts);
5097 oreg = ireg;
5098 } else {
5099 if (ots->val_type == TEMP_VAL_REG) {
5100 oreg = ots->reg;
5101 } else {
5102 /* Make sure to not spill the input register during allocation. */
5103 oreg = tcg_reg_alloc(s, tcg_target_available_regs[otype],
5104 allocated_regs | ((TCGRegSet)1 << ireg),
5105 preferred_regs, ots->indirect_base);
5106 }
5107 if (!tcg_out_mov(s, otype, oreg, ireg)) {
5108 /*
5109 * Cross register class move not supported.
5110 * Store the source register into the destination slot
5111 * and leave the destination temp as TEMP_VAL_MEM.
5112 */
5113 assert(!temp_readonly(ots));
5114 if (!ts->mem_allocated) {
5115 temp_allocate_frame(s, ots);
5116 }
5117 tcg_out_st(s, ts->type, ireg, ots->mem_base->reg, ots->mem_offset);
5118 set_temp_val_nonreg(s, ts, TEMP_VAL_MEM);
5119 ots->mem_coherent = 1;
5120 return;
5121 }
5122 }
5123 set_temp_val_reg(s, ots, oreg);
5124 ots->mem_coherent = 0;
5125
5126 if (NEED_SYNC_ARG(0)) {
5127 temp_sync(s, ots, allocated_regs, 0, 0);
5128 }
5129 }
5130
5131 /*
5132 * Specialized code generation for INDEX_op_dup_vec.
5133 */
tcg_reg_alloc_dup(TCGContext * s,const TCGOp * op)5134 static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op)
5135 {
5136 const TCGLifeData arg_life = op->life;
5137 TCGRegSet dup_out_regs, dup_in_regs;
5138 const TCGArgConstraint *dup_args_ct;
5139 TCGTemp *its, *ots;
5140 TCGType itype, vtype;
5141 unsigned vece;
5142 int lowpart_ofs;
5143 bool ok;
5144
5145 ots = arg_temp(op->args[0]);
5146 its = arg_temp(op->args[1]);
5147
5148 /* ENV should not be modified. */
5149 tcg_debug_assert(!temp_readonly(ots));
5150
5151 itype = its->type;
5152 vece = TCGOP_VECE(op);
5153 vtype = TCGOP_TYPE(op);
5154
5155 if (its->val_type == TEMP_VAL_CONST) {
5156 /* Propagate constant via movi -> dupi. */
5157 tcg_target_ulong val = its->val;
5158 if (IS_DEAD_ARG(1)) {
5159 temp_dead(s, its);
5160 }
5161 tcg_reg_alloc_do_movi(s, ots, val, arg_life, output_pref(op, 0));
5162 return;
5163 }
5164
5165 dup_args_ct = opcode_args_ct(op);
5166 dup_out_regs = dup_args_ct[0].regs;
5167 dup_in_regs = dup_args_ct[1].regs;
5168
5169 /* Allocate the output register now. */
5170 if (ots->val_type != TEMP_VAL_REG) {
5171 TCGRegSet allocated_regs = s->reserved_regs;
5172 TCGReg oreg;
5173
5174 if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) {
5175 /* Make sure to not spill the input register. */
5176 tcg_regset_set_reg(allocated_regs, its->reg);
5177 }
5178 oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
5179 output_pref(op, 0), ots->indirect_base);
5180 set_temp_val_reg(s, ots, oreg);
5181 }
5182
5183 switch (its->val_type) {
5184 case TEMP_VAL_REG:
5185 /*
5186 * The dup constriaints must be broad, covering all possible VECE.
5187 * However, tcg_op_dup_vec() gets to see the VECE and we allow it
5188 * to fail, indicating that extra moves are required for that case.
5189 */
5190 if (tcg_regset_test_reg(dup_in_regs, its->reg)) {
5191 if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) {
5192 goto done;
5193 }
5194 /* Try again from memory or a vector input register. */
5195 }
5196 if (!its->mem_coherent) {
5197 /*
5198 * The input register is not synced, and so an extra store
5199 * would be required to use memory. Attempt an integer-vector
5200 * register move first. We do not have a TCGRegSet for this.
5201 */
5202 if (tcg_out_mov(s, itype, ots->reg, its->reg)) {
5203 break;
5204 }
5205 /* Sync the temp back to its slot and load from there. */
5206 temp_sync(s, its, s->reserved_regs, 0, 0);
5207 }
5208 /* fall through */
5209
5210 case TEMP_VAL_MEM:
5211 lowpart_ofs = 0;
5212 if (HOST_BIG_ENDIAN) {
5213 lowpart_ofs = tcg_type_size(itype) - (1 << vece);
5214 }
5215 if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg,
5216 its->mem_offset + lowpart_ofs)) {
5217 goto done;
5218 }
5219 /* Load the input into the destination vector register. */
5220 tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset);
5221 break;
5222
5223 default:
5224 g_assert_not_reached();
5225 }
5226
5227 /* We now have a vector input register, so dup must succeed. */
5228 ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg);
5229 tcg_debug_assert(ok);
5230
5231 done:
5232 ots->mem_coherent = 0;
5233 if (IS_DEAD_ARG(1)) {
5234 temp_dead(s, its);
5235 }
5236 if (NEED_SYNC_ARG(0)) {
5237 temp_sync(s, ots, s->reserved_regs, 0, 0);
5238 }
5239 if (IS_DEAD_ARG(0)) {
5240 temp_dead(s, ots);
5241 }
5242 }
5243
tcg_reg_alloc_op(TCGContext * s,const TCGOp * op)5244 static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
5245 {
5246 const TCGLifeData arg_life = op->life;
5247 const TCGOpDef * const def = &tcg_op_defs[op->opc];
5248 TCGRegSet i_allocated_regs;
5249 TCGRegSet o_allocated_regs;
5250 int i, k, nb_iargs, nb_oargs;
5251 TCGReg reg;
5252 TCGArg arg;
5253 const TCGArgConstraint *args_ct;
5254 const TCGArgConstraint *arg_ct;
5255 TCGTemp *ts;
5256 TCGArg new_args[TCG_MAX_OP_ARGS];
5257 int const_args[TCG_MAX_OP_ARGS];
5258 TCGCond op_cond;
5259
5260 if (def->flags & TCG_OPF_CARRY_IN) {
5261 tcg_debug_assert(s->carry_live);
5262 }
5263
5264 nb_oargs = def->nb_oargs;
5265 nb_iargs = def->nb_iargs;
5266
5267 /* copy constants */
5268 memcpy(new_args + nb_oargs + nb_iargs,
5269 op->args + nb_oargs + nb_iargs,
5270 sizeof(TCGArg) * def->nb_cargs);
5271
5272 i_allocated_regs = s->reserved_regs;
5273 o_allocated_regs = s->reserved_regs;
5274
5275 switch (op->opc) {
5276 case INDEX_op_brcond:
5277 op_cond = op->args[2];
5278 break;
5279 case INDEX_op_setcond:
5280 case INDEX_op_negsetcond:
5281 case INDEX_op_cmp_vec:
5282 op_cond = op->args[3];
5283 break;
5284 case INDEX_op_brcond2_i32:
5285 op_cond = op->args[4];
5286 break;
5287 case INDEX_op_movcond:
5288 case INDEX_op_setcond2_i32:
5289 case INDEX_op_cmpsel_vec:
5290 op_cond = op->args[5];
5291 break;
5292 default:
5293 /* No condition within opcode. */
5294 op_cond = TCG_COND_ALWAYS;
5295 break;
5296 }
5297
5298 args_ct = opcode_args_ct(op);
5299
5300 /* satisfy input constraints */
5301 for (k = 0; k < nb_iargs; k++) {
5302 TCGRegSet i_preferred_regs, i_required_regs;
5303 bool allocate_new_reg, copyto_new_reg;
5304 TCGTemp *ts2;
5305 int i1, i2;
5306
5307 i = args_ct[nb_oargs + k].sort_index;
5308 arg = op->args[i];
5309 arg_ct = &args_ct[i];
5310 ts = arg_temp(arg);
5311
5312 if (ts->val_type == TEMP_VAL_CONST) {
5313 #ifdef TCG_REG_ZERO
5314 if (ts->val == 0 && (arg_ct->ct & TCG_CT_REG_ZERO)) {
5315 /* Hardware zero register: indicate register via non-const. */
5316 const_args[i] = 0;
5317 new_args[i] = TCG_REG_ZERO;
5318 continue;
5319 }
5320 #endif
5321
5322 if (tcg_target_const_match(ts->val, arg_ct->ct, ts->type,
5323 op_cond, TCGOP_VECE(op))) {
5324 /* constant is OK for instruction */
5325 const_args[i] = 1;
5326 new_args[i] = ts->val;
5327 continue;
5328 }
5329 }
5330
5331 reg = ts->reg;
5332 i_preferred_regs = 0;
5333 i_required_regs = arg_ct->regs;
5334 allocate_new_reg = false;
5335 copyto_new_reg = false;
5336
5337 switch (arg_ct->pair) {
5338 case 0: /* not paired */
5339 if (arg_ct->ialias) {
5340 i_preferred_regs = output_pref(op, arg_ct->alias_index);
5341
5342 /*
5343 * If the input is readonly, then it cannot also be an
5344 * output and aliased to itself. If the input is not
5345 * dead after the instruction, we must allocate a new
5346 * register and move it.
5347 */
5348 if (temp_readonly(ts) || !IS_DEAD_ARG(i)
5349 || args_ct[arg_ct->alias_index].newreg) {
5350 allocate_new_reg = true;
5351 } else if (ts->val_type == TEMP_VAL_REG) {
5352 /*
5353 * Check if the current register has already been
5354 * allocated for another input.
5355 */
5356 allocate_new_reg =
5357 tcg_regset_test_reg(i_allocated_regs, reg);
5358 }
5359 }
5360 if (!allocate_new_reg) {
5361 temp_load(s, ts, i_required_regs, i_allocated_regs,
5362 i_preferred_regs);
5363 reg = ts->reg;
5364 allocate_new_reg = !tcg_regset_test_reg(i_required_regs, reg);
5365 }
5366 if (allocate_new_reg) {
5367 /*
5368 * Allocate a new register matching the constraint
5369 * and move the temporary register into it.
5370 */
5371 temp_load(s, ts, tcg_target_available_regs[ts->type],
5372 i_allocated_regs, 0);
5373 reg = tcg_reg_alloc(s, i_required_regs, i_allocated_regs,
5374 i_preferred_regs, ts->indirect_base);
5375 copyto_new_reg = true;
5376 }
5377 break;
5378
5379 case 1:
5380 /* First of an input pair; if i1 == i2, the second is an output. */
5381 i1 = i;
5382 i2 = arg_ct->pair_index;
5383 ts2 = i1 != i2 ? arg_temp(op->args[i2]) : NULL;
5384
5385 /*
5386 * It is easier to default to allocating a new pair
5387 * and to identify a few cases where it's not required.
5388 */
5389 if (arg_ct->ialias) {
5390 i_preferred_regs = output_pref(op, arg_ct->alias_index);
5391 if (IS_DEAD_ARG(i1) &&
5392 IS_DEAD_ARG(i2) &&
5393 !temp_readonly(ts) &&
5394 ts->val_type == TEMP_VAL_REG &&
5395 ts->reg < TCG_TARGET_NB_REGS - 1 &&
5396 tcg_regset_test_reg(i_required_regs, reg) &&
5397 !tcg_regset_test_reg(i_allocated_regs, reg) &&
5398 !tcg_regset_test_reg(i_allocated_regs, reg + 1) &&
5399 (ts2
5400 ? ts2->val_type == TEMP_VAL_REG &&
5401 ts2->reg == reg + 1 &&
5402 !temp_readonly(ts2)
5403 : s->reg_to_temp[reg + 1] == NULL)) {
5404 break;
5405 }
5406 } else {
5407 /* Without aliasing, the pair must also be an input. */
5408 tcg_debug_assert(ts2);
5409 if (ts->val_type == TEMP_VAL_REG &&
5410 ts2->val_type == TEMP_VAL_REG &&
5411 ts2->reg == reg + 1 &&
5412 tcg_regset_test_reg(i_required_regs, reg)) {
5413 break;
5414 }
5415 }
5416 reg = tcg_reg_alloc_pair(s, i_required_regs, i_allocated_regs,
5417 0, ts->indirect_base);
5418 goto do_pair;
5419
5420 case 2: /* pair second */
5421 reg = new_args[arg_ct->pair_index] + 1;
5422 goto do_pair;
5423
5424 case 3: /* ialias with second output, no first input */
5425 tcg_debug_assert(arg_ct->ialias);
5426 i_preferred_regs = output_pref(op, arg_ct->alias_index);
5427
5428 if (IS_DEAD_ARG(i) &&
5429 !temp_readonly(ts) &&
5430 ts->val_type == TEMP_VAL_REG &&
5431 reg > 0 &&
5432 s->reg_to_temp[reg - 1] == NULL &&
5433 tcg_regset_test_reg(i_required_regs, reg) &&
5434 !tcg_regset_test_reg(i_allocated_regs, reg) &&
5435 !tcg_regset_test_reg(i_allocated_regs, reg - 1)) {
5436 tcg_regset_set_reg(i_allocated_regs, reg - 1);
5437 break;
5438 }
5439 reg = tcg_reg_alloc_pair(s, i_required_regs >> 1,
5440 i_allocated_regs, 0,
5441 ts->indirect_base);
5442 tcg_regset_set_reg(i_allocated_regs, reg);
5443 reg += 1;
5444 goto do_pair;
5445
5446 do_pair:
5447 /*
5448 * If an aliased input is not dead after the instruction,
5449 * we must allocate a new register and move it.
5450 */
5451 if (arg_ct->ialias && (!IS_DEAD_ARG(i) || temp_readonly(ts))) {
5452 TCGRegSet t_allocated_regs = i_allocated_regs;
5453
5454 /*
5455 * Because of the alias, and the continued life, make sure
5456 * that the temp is somewhere *other* than the reg pair,
5457 * and we get a copy in reg.
5458 */
5459 tcg_regset_set_reg(t_allocated_regs, reg);
5460 tcg_regset_set_reg(t_allocated_regs, reg + 1);
5461 if (ts->val_type == TEMP_VAL_REG && ts->reg == reg) {
5462 /* If ts was already in reg, copy it somewhere else. */
5463 TCGReg nr;
5464 bool ok;
5465
5466 tcg_debug_assert(ts->kind != TEMP_FIXED);
5467 nr = tcg_reg_alloc(s, tcg_target_available_regs[ts->type],
5468 t_allocated_regs, 0, ts->indirect_base);
5469 ok = tcg_out_mov(s, ts->type, nr, reg);
5470 tcg_debug_assert(ok);
5471
5472 set_temp_val_reg(s, ts, nr);
5473 } else {
5474 temp_load(s, ts, tcg_target_available_regs[ts->type],
5475 t_allocated_regs, 0);
5476 copyto_new_reg = true;
5477 }
5478 } else {
5479 /* Preferably allocate to reg, otherwise copy. */
5480 i_required_regs = (TCGRegSet)1 << reg;
5481 temp_load(s, ts, i_required_regs, i_allocated_regs,
5482 i_preferred_regs);
5483 copyto_new_reg = ts->reg != reg;
5484 }
5485 break;
5486
5487 default:
5488 g_assert_not_reached();
5489 }
5490
5491 if (copyto_new_reg) {
5492 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
5493 /*
5494 * Cross register class move not supported. Sync the
5495 * temp back to its slot and load from there.
5496 */
5497 temp_sync(s, ts, i_allocated_regs, 0, 0);
5498 tcg_out_ld(s, ts->type, reg,
5499 ts->mem_base->reg, ts->mem_offset);
5500 }
5501 }
5502 new_args[i] = reg;
5503 const_args[i] = 0;
5504 tcg_regset_set_reg(i_allocated_regs, reg);
5505 }
5506
5507 /* mark dead temporaries and free the associated registers */
5508 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
5509 if (IS_DEAD_ARG(i)) {
5510 temp_dead(s, arg_temp(op->args[i]));
5511 }
5512 }
5513
5514 if (def->flags & TCG_OPF_COND_BRANCH) {
5515 tcg_reg_alloc_cbranch(s, i_allocated_regs);
5516 } else if (def->flags & TCG_OPF_BB_END) {
5517 tcg_reg_alloc_bb_end(s, i_allocated_regs);
5518 } else {
5519 if (def->flags & TCG_OPF_CALL_CLOBBER) {
5520 assert_carry_dead(s);
5521 /* XXX: permit generic clobber register list ? */
5522 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
5523 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
5524 tcg_reg_free(s, i, i_allocated_regs);
5525 }
5526 }
5527 }
5528 if (def->flags & TCG_OPF_SIDE_EFFECTS) {
5529 /* sync globals if the op has side effects and might trigger
5530 an exception. */
5531 sync_globals(s, i_allocated_regs);
5532 }
5533
5534 /* satisfy the output constraints */
5535 for (k = 0; k < nb_oargs; k++) {
5536 i = args_ct[k].sort_index;
5537 arg = op->args[i];
5538 arg_ct = &args_ct[i];
5539 ts = arg_temp(arg);
5540
5541 /* ENV should not be modified. */
5542 tcg_debug_assert(!temp_readonly(ts));
5543
5544 switch (arg_ct->pair) {
5545 case 0: /* not paired */
5546 if (arg_ct->oalias && !const_args[arg_ct->alias_index]) {
5547 reg = new_args[arg_ct->alias_index];
5548 } else if (arg_ct->newreg) {
5549 reg = tcg_reg_alloc(s, arg_ct->regs,
5550 i_allocated_regs | o_allocated_regs,
5551 output_pref(op, k), ts->indirect_base);
5552 } else {
5553 reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs,
5554 output_pref(op, k), ts->indirect_base);
5555 }
5556 break;
5557
5558 case 1: /* first of pair */
5559 if (arg_ct->oalias) {
5560 reg = new_args[arg_ct->alias_index];
5561 } else if (arg_ct->newreg) {
5562 reg = tcg_reg_alloc_pair(s, arg_ct->regs,
5563 i_allocated_regs | o_allocated_regs,
5564 output_pref(op, k),
5565 ts->indirect_base);
5566 } else {
5567 reg = tcg_reg_alloc_pair(s, arg_ct->regs, o_allocated_regs,
5568 output_pref(op, k),
5569 ts->indirect_base);
5570 }
5571 break;
5572
5573 case 2: /* second of pair */
5574 if (arg_ct->oalias) {
5575 reg = new_args[arg_ct->alias_index];
5576 } else {
5577 reg = new_args[arg_ct->pair_index] + 1;
5578 }
5579 break;
5580
5581 case 3: /* first of pair, aliasing with a second input */
5582 tcg_debug_assert(!arg_ct->newreg);
5583 reg = new_args[arg_ct->pair_index] - 1;
5584 break;
5585
5586 default:
5587 g_assert_not_reached();
5588 }
5589 tcg_regset_set_reg(o_allocated_regs, reg);
5590 set_temp_val_reg(s, ts, reg);
5591 ts->mem_coherent = 0;
5592 new_args[i] = reg;
5593 }
5594 }
5595
5596 /* emit instruction */
5597 TCGType type = TCGOP_TYPE(op);
5598 switch (op->opc) {
5599 case INDEX_op_addc1o:
5600 tcg_out_set_carry(s);
5601 /* fall through */
5602 case INDEX_op_add:
5603 case INDEX_op_addcio:
5604 case INDEX_op_addco:
5605 case INDEX_op_and:
5606 case INDEX_op_andc:
5607 case INDEX_op_clz:
5608 case INDEX_op_ctz:
5609 case INDEX_op_divs:
5610 case INDEX_op_divu:
5611 case INDEX_op_eqv:
5612 case INDEX_op_mul:
5613 case INDEX_op_mulsh:
5614 case INDEX_op_muluh:
5615 case INDEX_op_nand:
5616 case INDEX_op_nor:
5617 case INDEX_op_or:
5618 case INDEX_op_orc:
5619 case INDEX_op_rems:
5620 case INDEX_op_remu:
5621 case INDEX_op_rotl:
5622 case INDEX_op_rotr:
5623 case INDEX_op_sar:
5624 case INDEX_op_shl:
5625 case INDEX_op_shr:
5626 case INDEX_op_xor:
5627 {
5628 const TCGOutOpBinary *out =
5629 container_of(all_outop[op->opc], TCGOutOpBinary, base);
5630
5631 /* Constants should never appear in the first source operand. */
5632 tcg_debug_assert(!const_args[1]);
5633 if (const_args[2]) {
5634 out->out_rri(s, type, new_args[0], new_args[1], new_args[2]);
5635 } else {
5636 out->out_rrr(s, type, new_args[0], new_args[1], new_args[2]);
5637 }
5638 }
5639 break;
5640
5641 case INDEX_op_sub:
5642 {
5643 const TCGOutOpSubtract *out = &outop_sub;
5644
5645 /*
5646 * Constants should never appear in the second source operand.
5647 * These are folded to add with negative constant.
5648 */
5649 tcg_debug_assert(!const_args[2]);
5650 if (const_args[1]) {
5651 out->out_rir(s, type, new_args[0], new_args[1], new_args[2]);
5652 } else {
5653 out->out_rrr(s, type, new_args[0], new_args[1], new_args[2]);
5654 }
5655 }
5656 break;
5657
5658 case INDEX_op_subb1o:
5659 tcg_out_set_borrow(s);
5660 /* fall through */
5661 case INDEX_op_addci:
5662 case INDEX_op_subbi:
5663 case INDEX_op_subbio:
5664 case INDEX_op_subbo:
5665 {
5666 const TCGOutOpAddSubCarry *out =
5667 container_of(all_outop[op->opc], TCGOutOpAddSubCarry, base);
5668
5669 if (const_args[2]) {
5670 if (const_args[1]) {
5671 out->out_rii(s, type, new_args[0],
5672 new_args[1], new_args[2]);
5673 } else {
5674 out->out_rri(s, type, new_args[0],
5675 new_args[1], new_args[2]);
5676 }
5677 } else if (const_args[1]) {
5678 out->out_rir(s, type, new_args[0], new_args[1], new_args[2]);
5679 } else {
5680 out->out_rrr(s, type, new_args[0], new_args[1], new_args[2]);
5681 }
5682 }
5683 break;
5684
5685 case INDEX_op_bswap64:
5686 case INDEX_op_ext_i32_i64:
5687 case INDEX_op_extu_i32_i64:
5688 case INDEX_op_extrl_i64_i32:
5689 case INDEX_op_extrh_i64_i32:
5690 assert(TCG_TARGET_REG_BITS == 64);
5691 /* fall through */
5692 case INDEX_op_ctpop:
5693 case INDEX_op_neg:
5694 case INDEX_op_not:
5695 {
5696 const TCGOutOpUnary *out =
5697 container_of(all_outop[op->opc], TCGOutOpUnary, base);
5698
5699 /* Constants should have been folded. */
5700 tcg_debug_assert(!const_args[1]);
5701 out->out_rr(s, type, new_args[0], new_args[1]);
5702 }
5703 break;
5704
5705 case INDEX_op_bswap16:
5706 case INDEX_op_bswap32:
5707 {
5708 const TCGOutOpBswap *out =
5709 container_of(all_outop[op->opc], TCGOutOpBswap, base);
5710
5711 tcg_debug_assert(!const_args[1]);
5712 out->out_rr(s, type, new_args[0], new_args[1], new_args[2]);
5713 }
5714 break;
5715
5716 case INDEX_op_deposit:
5717 {
5718 const TCGOutOpDeposit *out = &outop_deposit;
5719
5720 if (const_args[2]) {
5721 tcg_debug_assert(!const_args[1]);
5722 out->out_rri(s, type, new_args[0], new_args[1],
5723 new_args[2], new_args[3], new_args[4]);
5724 } else if (const_args[1]) {
5725 tcg_debug_assert(new_args[1] == 0);
5726 tcg_debug_assert(!const_args[2]);
5727 out->out_rzr(s, type, new_args[0], new_args[2],
5728 new_args[3], new_args[4]);
5729 } else {
5730 out->out_rrr(s, type, new_args[0], new_args[1],
5731 new_args[2], new_args[3], new_args[4]);
5732 }
5733 }
5734 break;
5735
5736 case INDEX_op_divs2:
5737 case INDEX_op_divu2:
5738 {
5739 const TCGOutOpDivRem *out =
5740 container_of(all_outop[op->opc], TCGOutOpDivRem, base);
5741
5742 /* Only used by x86 and s390x, which use matching constraints. */
5743 tcg_debug_assert(new_args[0] == new_args[2]);
5744 tcg_debug_assert(new_args[1] == new_args[3]);
5745 tcg_debug_assert(!const_args[4]);
5746 out->out_rr01r(s, type, new_args[0], new_args[1], new_args[4]);
5747 }
5748 break;
5749
5750 case INDEX_op_extract:
5751 case INDEX_op_sextract:
5752 {
5753 const TCGOutOpExtract *out =
5754 container_of(all_outop[op->opc], TCGOutOpExtract, base);
5755
5756 tcg_debug_assert(!const_args[1]);
5757 out->out_rr(s, type, new_args[0], new_args[1],
5758 new_args[2], new_args[3]);
5759 }
5760 break;
5761
5762 case INDEX_op_extract2:
5763 {
5764 const TCGOutOpExtract2 *out = &outop_extract2;
5765
5766 tcg_debug_assert(!const_args[1]);
5767 tcg_debug_assert(!const_args[2]);
5768 out->out_rrr(s, type, new_args[0], new_args[1],
5769 new_args[2], new_args[3]);
5770 }
5771 break;
5772
5773 case INDEX_op_ld8u:
5774 case INDEX_op_ld8s:
5775 case INDEX_op_ld16u:
5776 case INDEX_op_ld16s:
5777 case INDEX_op_ld32u:
5778 case INDEX_op_ld32s:
5779 case INDEX_op_ld:
5780 {
5781 const TCGOutOpLoad *out =
5782 container_of(all_outop[op->opc], TCGOutOpLoad, base);
5783
5784 tcg_debug_assert(!const_args[1]);
5785 out->out(s, type, new_args[0], new_args[1], new_args[2]);
5786 }
5787 break;
5788
5789 case INDEX_op_muls2:
5790 case INDEX_op_mulu2:
5791 {
5792 const TCGOutOpMul2 *out =
5793 container_of(all_outop[op->opc], TCGOutOpMul2, base);
5794
5795 tcg_debug_assert(!const_args[2]);
5796 tcg_debug_assert(!const_args[3]);
5797 out->out_rrrr(s, type, new_args[0], new_args[1],
5798 new_args[2], new_args[3]);
5799 }
5800 break;
5801
5802 case INDEX_op_st32:
5803 /* Use tcg_op_st w/ I32. */
5804 type = TCG_TYPE_I32;
5805 /* fall through */
5806 case INDEX_op_st:
5807 case INDEX_op_st8:
5808 case INDEX_op_st16:
5809 {
5810 const TCGOutOpStore *out =
5811 container_of(all_outop[op->opc], TCGOutOpStore, base);
5812
5813 if (const_args[0]) {
5814 out->out_i(s, type, new_args[0], new_args[1], new_args[2]);
5815 } else {
5816 out->out_r(s, type, new_args[0], new_args[1], new_args[2]);
5817 }
5818 }
5819 break;
5820
5821 case INDEX_op_qemu_ld:
5822 case INDEX_op_qemu_st:
5823 {
5824 const TCGOutOpQemuLdSt *out =
5825 container_of(all_outop[op->opc], TCGOutOpQemuLdSt, base);
5826
5827 out->out(s, type, new_args[0], new_args[1], new_args[2]);
5828 }
5829 break;
5830
5831 case INDEX_op_qemu_ld2:
5832 case INDEX_op_qemu_st2:
5833 {
5834 const TCGOutOpQemuLdSt2 *out =
5835 container_of(all_outop[op->opc], TCGOutOpQemuLdSt2, base);
5836
5837 out->out(s, type, new_args[0], new_args[1],
5838 new_args[2], new_args[3]);
5839 }
5840 break;
5841
5842 case INDEX_op_brcond:
5843 {
5844 const TCGOutOpBrcond *out = &outop_brcond;
5845 TCGCond cond = new_args[2];
5846 TCGLabel *label = arg_label(new_args[3]);
5847
5848 tcg_debug_assert(!const_args[0]);
5849 if (const_args[1]) {
5850 out->out_ri(s, type, cond, new_args[0], new_args[1], label);
5851 } else {
5852 out->out_rr(s, type, cond, new_args[0], new_args[1], label);
5853 }
5854 }
5855 break;
5856
5857 case INDEX_op_movcond:
5858 {
5859 const TCGOutOpMovcond *out = &outop_movcond;
5860 TCGCond cond = new_args[5];
5861
5862 tcg_debug_assert(!const_args[1]);
5863 out->out(s, type, cond, new_args[0],
5864 new_args[1], new_args[2], const_args[2],
5865 new_args[3], const_args[3],
5866 new_args[4], const_args[4]);
5867 }
5868 break;
5869
5870 case INDEX_op_setcond:
5871 case INDEX_op_negsetcond:
5872 {
5873 const TCGOutOpSetcond *out =
5874 container_of(all_outop[op->opc], TCGOutOpSetcond, base);
5875 TCGCond cond = new_args[3];
5876
5877 tcg_debug_assert(!const_args[1]);
5878 if (const_args[2]) {
5879 out->out_rri(s, type, cond,
5880 new_args[0], new_args[1], new_args[2]);
5881 } else {
5882 out->out_rrr(s, type, cond,
5883 new_args[0], new_args[1], new_args[2]);
5884 }
5885 }
5886 break;
5887
5888 #if TCG_TARGET_REG_BITS == 32
5889 case INDEX_op_brcond2_i32:
5890 {
5891 const TCGOutOpBrcond2 *out = &outop_brcond2;
5892 TCGCond cond = new_args[4];
5893 TCGLabel *label = arg_label(new_args[5]);
5894
5895 tcg_debug_assert(!const_args[0]);
5896 tcg_debug_assert(!const_args[1]);
5897 out->out(s, cond, new_args[0], new_args[1],
5898 new_args[2], const_args[2],
5899 new_args[3], const_args[3], label);
5900 }
5901 break;
5902 case INDEX_op_setcond2_i32:
5903 {
5904 const TCGOutOpSetcond2 *out = &outop_setcond2;
5905 TCGCond cond = new_args[5];
5906
5907 tcg_debug_assert(!const_args[1]);
5908 tcg_debug_assert(!const_args[2]);
5909 out->out(s, cond, new_args[0], new_args[1], new_args[2],
5910 new_args[3], const_args[3], new_args[4], const_args[4]);
5911 }
5912 break;
5913 #else
5914 case INDEX_op_brcond2_i32:
5915 case INDEX_op_setcond2_i32:
5916 g_assert_not_reached();
5917 #endif
5918
5919 case INDEX_op_goto_ptr:
5920 tcg_debug_assert(!const_args[0]);
5921 tcg_out_goto_ptr(s, new_args[0]);
5922 break;
5923
5924 default:
5925 tcg_debug_assert(def->flags & TCG_OPF_VECTOR);
5926 tcg_out_vec_op(s, op->opc, type - TCG_TYPE_V64,
5927 TCGOP_VECE(op), new_args, const_args);
5928 break;
5929 }
5930
5931 if (def->flags & TCG_OPF_CARRY_IN) {
5932 s->carry_live = false;
5933 }
5934 if (def->flags & TCG_OPF_CARRY_OUT) {
5935 s->carry_live = true;
5936 }
5937
5938 /* move the outputs in the correct register if needed */
5939 for(i = 0; i < nb_oargs; i++) {
5940 ts = arg_temp(op->args[i]);
5941
5942 /* ENV should not be modified. */
5943 tcg_debug_assert(!temp_readonly(ts));
5944
5945 if (NEED_SYNC_ARG(i)) {
5946 temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i));
5947 } else if (IS_DEAD_ARG(i)) {
5948 temp_dead(s, ts);
5949 }
5950 }
5951 }
5952
tcg_reg_alloc_dup2(TCGContext * s,const TCGOp * op)5953 static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op)
5954 {
5955 const TCGLifeData arg_life = op->life;
5956 TCGTemp *ots, *itsl, *itsh;
5957 TCGType vtype = TCGOP_TYPE(op);
5958
5959 /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */
5960 tcg_debug_assert(TCG_TARGET_REG_BITS == 32);
5961 tcg_debug_assert(TCGOP_VECE(op) == MO_64);
5962
5963 ots = arg_temp(op->args[0]);
5964 itsl = arg_temp(op->args[1]);
5965 itsh = arg_temp(op->args[2]);
5966
5967 /* ENV should not be modified. */
5968 tcg_debug_assert(!temp_readonly(ots));
5969
5970 /* Allocate the output register now. */
5971 if (ots->val_type != TEMP_VAL_REG) {
5972 TCGRegSet allocated_regs = s->reserved_regs;
5973 TCGRegSet dup_out_regs = opcode_args_ct(op)[0].regs;
5974 TCGReg oreg;
5975
5976 /* Make sure to not spill the input registers. */
5977 if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) {
5978 tcg_regset_set_reg(allocated_regs, itsl->reg);
5979 }
5980 if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) {
5981 tcg_regset_set_reg(allocated_regs, itsh->reg);
5982 }
5983
5984 oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
5985 output_pref(op, 0), ots->indirect_base);
5986 set_temp_val_reg(s, ots, oreg);
5987 }
5988
5989 /* Promote dup2 of immediates to dupi_vec. */
5990 if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) {
5991 uint64_t val = deposit64(itsl->val, 32, 32, itsh->val);
5992 MemOp vece = MO_64;
5993
5994 if (val == dup_const(MO_8, val)) {
5995 vece = MO_8;
5996 } else if (val == dup_const(MO_16, val)) {
5997 vece = MO_16;
5998 } else if (val == dup_const(MO_32, val)) {
5999 vece = MO_32;
6000 }
6001
6002 tcg_out_dupi_vec(s, vtype, vece, ots->reg, val);
6003 goto done;
6004 }
6005
6006 /* If the two inputs form one 64-bit value, try dupm_vec. */
6007 if (itsl->temp_subindex == HOST_BIG_ENDIAN &&
6008 itsh->temp_subindex == !HOST_BIG_ENDIAN &&
6009 itsl == itsh + (HOST_BIG_ENDIAN ? 1 : -1)) {
6010 TCGTemp *its = itsl - HOST_BIG_ENDIAN;
6011
6012 temp_sync(s, its + 0, s->reserved_regs, 0, 0);
6013 temp_sync(s, its + 1, s->reserved_regs, 0, 0);
6014
6015 if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg,
6016 its->mem_base->reg, its->mem_offset)) {
6017 goto done;
6018 }
6019 }
6020
6021 /* Fall back to generic expansion. */
6022 return false;
6023
6024 done:
6025 ots->mem_coherent = 0;
6026 if (IS_DEAD_ARG(1)) {
6027 temp_dead(s, itsl);
6028 }
6029 if (IS_DEAD_ARG(2)) {
6030 temp_dead(s, itsh);
6031 }
6032 if (NEED_SYNC_ARG(0)) {
6033 temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0));
6034 } else if (IS_DEAD_ARG(0)) {
6035 temp_dead(s, ots);
6036 }
6037 return true;
6038 }
6039
load_arg_reg(TCGContext * s,TCGReg reg,TCGTemp * ts,TCGRegSet allocated_regs)6040 static void load_arg_reg(TCGContext *s, TCGReg reg, TCGTemp *ts,
6041 TCGRegSet allocated_regs)
6042 {
6043 if (ts->val_type == TEMP_VAL_REG) {
6044 if (ts->reg != reg) {
6045 tcg_reg_free(s, reg, allocated_regs);
6046 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
6047 /*
6048 * Cross register class move not supported. Sync the
6049 * temp back to its slot and load from there.
6050 */
6051 temp_sync(s, ts, allocated_regs, 0, 0);
6052 tcg_out_ld(s, ts->type, reg,
6053 ts->mem_base->reg, ts->mem_offset);
6054 }
6055 }
6056 } else {
6057 TCGRegSet arg_set = 0;
6058
6059 tcg_reg_free(s, reg, allocated_regs);
6060 tcg_regset_set_reg(arg_set, reg);
6061 temp_load(s, ts, arg_set, allocated_regs, 0);
6062 }
6063 }
6064
load_arg_stk(TCGContext * s,unsigned arg_slot,TCGTemp * ts,TCGRegSet allocated_regs)6065 static void load_arg_stk(TCGContext *s, unsigned arg_slot, TCGTemp *ts,
6066 TCGRegSet allocated_regs)
6067 {
6068 /*
6069 * When the destination is on the stack, load up the temp and store.
6070 * If there are many call-saved registers, the temp might live to
6071 * see another use; otherwise it'll be discarded.
6072 */
6073 temp_load(s, ts, tcg_target_available_regs[ts->type], allocated_regs, 0);
6074 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK,
6075 arg_slot_stk_ofs(arg_slot));
6076 }
6077
load_arg_normal(TCGContext * s,const TCGCallArgumentLoc * l,TCGTemp * ts,TCGRegSet * allocated_regs)6078 static void load_arg_normal(TCGContext *s, const TCGCallArgumentLoc *l,
6079 TCGTemp *ts, TCGRegSet *allocated_regs)
6080 {
6081 if (arg_slot_reg_p(l->arg_slot)) {
6082 TCGReg reg = tcg_target_call_iarg_regs[l->arg_slot];
6083 load_arg_reg(s, reg, ts, *allocated_regs);
6084 tcg_regset_set_reg(*allocated_regs, reg);
6085 } else {
6086 load_arg_stk(s, l->arg_slot, ts, *allocated_regs);
6087 }
6088 }
6089
load_arg_ref(TCGContext * s,unsigned arg_slot,TCGReg ref_base,intptr_t ref_off,TCGRegSet * allocated_regs)6090 static void load_arg_ref(TCGContext *s, unsigned arg_slot, TCGReg ref_base,
6091 intptr_t ref_off, TCGRegSet *allocated_regs)
6092 {
6093 TCGReg reg;
6094
6095 if (arg_slot_reg_p(arg_slot)) {
6096 reg = tcg_target_call_iarg_regs[arg_slot];
6097 tcg_reg_free(s, reg, *allocated_regs);
6098 tcg_out_addi_ptr(s, reg, ref_base, ref_off);
6099 tcg_regset_set_reg(*allocated_regs, reg);
6100 } else {
6101 reg = tcg_reg_alloc(s, tcg_target_available_regs[TCG_TYPE_PTR],
6102 *allocated_regs, 0, false);
6103 tcg_out_addi_ptr(s, reg, ref_base, ref_off);
6104 tcg_out_st(s, TCG_TYPE_PTR, reg, TCG_REG_CALL_STACK,
6105 arg_slot_stk_ofs(arg_slot));
6106 }
6107 }
6108
tcg_reg_alloc_call(TCGContext * s,TCGOp * op)6109 static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op)
6110 {
6111 const int nb_oargs = TCGOP_CALLO(op);
6112 const int nb_iargs = TCGOP_CALLI(op);
6113 const TCGLifeData arg_life = op->life;
6114 const TCGHelperInfo *info = tcg_call_info(op);
6115 TCGRegSet allocated_regs = s->reserved_regs;
6116 int i;
6117
6118 /*
6119 * Move inputs into place in reverse order,
6120 * so that we place stacked arguments first.
6121 */
6122 for (i = nb_iargs - 1; i >= 0; --i) {
6123 const TCGCallArgumentLoc *loc = &info->in[i];
6124 TCGTemp *ts = arg_temp(op->args[nb_oargs + i]);
6125
6126 switch (loc->kind) {
6127 case TCG_CALL_ARG_NORMAL:
6128 case TCG_CALL_ARG_EXTEND_U:
6129 case TCG_CALL_ARG_EXTEND_S:
6130 load_arg_normal(s, loc, ts, &allocated_regs);
6131 break;
6132 case TCG_CALL_ARG_BY_REF:
6133 load_arg_stk(s, loc->ref_slot, ts, allocated_regs);
6134 load_arg_ref(s, loc->arg_slot, TCG_REG_CALL_STACK,
6135 arg_slot_stk_ofs(loc->ref_slot),
6136 &allocated_regs);
6137 break;
6138 case TCG_CALL_ARG_BY_REF_N:
6139 load_arg_stk(s, loc->ref_slot, ts, allocated_regs);
6140 break;
6141 default:
6142 g_assert_not_reached();
6143 }
6144 }
6145
6146 /* Mark dead temporaries and free the associated registers. */
6147 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
6148 if (IS_DEAD_ARG(i)) {
6149 temp_dead(s, arg_temp(op->args[i]));
6150 }
6151 }
6152
6153 /* Clobber call registers. */
6154 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
6155 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
6156 tcg_reg_free(s, i, allocated_regs);
6157 }
6158 }
6159
6160 /*
6161 * Save globals if they might be written by the helper,
6162 * sync them if they might be read.
6163 */
6164 if (info->flags & TCG_CALL_NO_READ_GLOBALS) {
6165 /* Nothing to do */
6166 } else if (info->flags & TCG_CALL_NO_WRITE_GLOBALS) {
6167 sync_globals(s, allocated_regs);
6168 } else {
6169 save_globals(s, allocated_regs);
6170 }
6171
6172 /*
6173 * If the ABI passes a pointer to the returned struct as the first
6174 * argument, load that now. Pass a pointer to the output home slot.
6175 */
6176 if (info->out_kind == TCG_CALL_RET_BY_REF) {
6177 TCGTemp *ts = arg_temp(op->args[0]);
6178
6179 if (!ts->mem_allocated) {
6180 temp_allocate_frame(s, ts);
6181 }
6182 load_arg_ref(s, 0, ts->mem_base->reg, ts->mem_offset, &allocated_regs);
6183 }
6184
6185 tcg_out_call(s, tcg_call_func(op), info);
6186
6187 /* Assign output registers and emit moves if needed. */
6188 switch (info->out_kind) {
6189 case TCG_CALL_RET_NORMAL:
6190 for (i = 0; i < nb_oargs; i++) {
6191 TCGTemp *ts = arg_temp(op->args[i]);
6192 TCGReg reg = tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, i);
6193
6194 /* ENV should not be modified. */
6195 tcg_debug_assert(!temp_readonly(ts));
6196
6197 set_temp_val_reg(s, ts, reg);
6198 ts->mem_coherent = 0;
6199 }
6200 break;
6201
6202 case TCG_CALL_RET_BY_VEC:
6203 {
6204 TCGTemp *ts = arg_temp(op->args[0]);
6205
6206 tcg_debug_assert(ts->base_type == TCG_TYPE_I128);
6207 tcg_debug_assert(ts->temp_subindex == 0);
6208 if (!ts->mem_allocated) {
6209 temp_allocate_frame(s, ts);
6210 }
6211 tcg_out_st(s, TCG_TYPE_V128,
6212 tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0),
6213 ts->mem_base->reg, ts->mem_offset);
6214 }
6215 /* fall through to mark all parts in memory */
6216
6217 case TCG_CALL_RET_BY_REF:
6218 /* The callee has performed a write through the reference. */
6219 for (i = 0; i < nb_oargs; i++) {
6220 TCGTemp *ts = arg_temp(op->args[i]);
6221 ts->val_type = TEMP_VAL_MEM;
6222 }
6223 break;
6224
6225 default:
6226 g_assert_not_reached();
6227 }
6228
6229 /* Flush or discard output registers as needed. */
6230 for (i = 0; i < nb_oargs; i++) {
6231 TCGTemp *ts = arg_temp(op->args[i]);
6232 if (NEED_SYNC_ARG(i)) {
6233 temp_sync(s, ts, s->reserved_regs, 0, IS_DEAD_ARG(i));
6234 } else if (IS_DEAD_ARG(i)) {
6235 temp_dead(s, ts);
6236 }
6237 }
6238 }
6239
6240 /**
6241 * atom_and_align_for_opc:
6242 * @s: tcg context
6243 * @opc: memory operation code
6244 * @host_atom: MO_ATOM_{IFALIGN,WITHIN16,SUBALIGN} for host operations
6245 * @allow_two_ops: true if we are prepared to issue two operations
6246 *
6247 * Return the alignment and atomicity to use for the inline fast path
6248 * for the given memory operation. The alignment may be larger than
6249 * that specified in @opc, and the correct alignment will be diagnosed
6250 * by the slow path helper.
6251 *
6252 * If @allow_two_ops, the host is prepared to test for 2x alignment,
6253 * and issue two loads or stores for subalignment.
6254 */
atom_and_align_for_opc(TCGContext * s,MemOp opc,MemOp host_atom,bool allow_two_ops)6255 static TCGAtomAlign atom_and_align_for_opc(TCGContext *s, MemOp opc,
6256 MemOp host_atom, bool allow_two_ops)
6257 {
6258 MemOp align = memop_alignment_bits(opc);
6259 MemOp size = opc & MO_SIZE;
6260 MemOp half = size ? size - 1 : 0;
6261 MemOp atom = opc & MO_ATOM_MASK;
6262 MemOp atmax;
6263
6264 switch (atom) {
6265 case MO_ATOM_NONE:
6266 /* The operation requires no specific atomicity. */
6267 atmax = MO_8;
6268 break;
6269
6270 case MO_ATOM_IFALIGN:
6271 atmax = size;
6272 break;
6273
6274 case MO_ATOM_IFALIGN_PAIR:
6275 atmax = half;
6276 break;
6277
6278 case MO_ATOM_WITHIN16:
6279 atmax = size;
6280 if (size == MO_128) {
6281 /* Misalignment implies !within16, and therefore no atomicity. */
6282 } else if (host_atom != MO_ATOM_WITHIN16) {
6283 /* The host does not implement within16, so require alignment. */
6284 align = MAX(align, size);
6285 }
6286 break;
6287
6288 case MO_ATOM_WITHIN16_PAIR:
6289 atmax = size;
6290 /*
6291 * Misalignment implies !within16, and therefore half atomicity.
6292 * Any host prepared for two operations can implement this with
6293 * half alignment.
6294 */
6295 if (host_atom != MO_ATOM_WITHIN16 && allow_two_ops) {
6296 align = MAX(align, half);
6297 }
6298 break;
6299
6300 case MO_ATOM_SUBALIGN:
6301 atmax = size;
6302 if (host_atom != MO_ATOM_SUBALIGN) {
6303 /* If unaligned but not odd, there are subobjects up to half. */
6304 if (allow_two_ops) {
6305 align = MAX(align, half);
6306 } else {
6307 align = MAX(align, size);
6308 }
6309 }
6310 break;
6311
6312 default:
6313 g_assert_not_reached();
6314 }
6315
6316 return (TCGAtomAlign){ .atom = atmax, .align = align };
6317 }
6318
6319 /*
6320 * Similarly for qemu_ld/st slow path helpers.
6321 * We must re-implement tcg_gen_callN and tcg_reg_alloc_call simultaneously,
6322 * using only the provided backend tcg_out_* functions.
6323 */
6324
tcg_out_helper_stk_ofs(TCGType type,unsigned slot)6325 static int tcg_out_helper_stk_ofs(TCGType type, unsigned slot)
6326 {
6327 int ofs = arg_slot_stk_ofs(slot);
6328
6329 /*
6330 * Each stack slot is TCG_TARGET_LONG_BITS. If the host does not
6331 * require extension to uint64_t, adjust the address for uint32_t.
6332 */
6333 if (HOST_BIG_ENDIAN &&
6334 TCG_TARGET_REG_BITS == 64 &&
6335 type == TCG_TYPE_I32) {
6336 ofs += 4;
6337 }
6338 return ofs;
6339 }
6340
tcg_out_helper_load_slots(TCGContext * s,unsigned nmov,TCGMovExtend * mov,const TCGLdstHelperParam * parm)6341 static void tcg_out_helper_load_slots(TCGContext *s,
6342 unsigned nmov, TCGMovExtend *mov,
6343 const TCGLdstHelperParam *parm)
6344 {
6345 unsigned i;
6346 TCGReg dst3;
6347
6348 /*
6349 * Start from the end, storing to the stack first.
6350 * This frees those registers, so we need not consider overlap.
6351 */
6352 for (i = nmov; i-- > 0; ) {
6353 unsigned slot = mov[i].dst;
6354
6355 if (arg_slot_reg_p(slot)) {
6356 goto found_reg;
6357 }
6358
6359 TCGReg src = mov[i].src;
6360 TCGType dst_type = mov[i].dst_type;
6361 MemOp dst_mo = dst_type == TCG_TYPE_I32 ? MO_32 : MO_64;
6362
6363 /* The argument is going onto the stack; extend into scratch. */
6364 if ((mov[i].src_ext & MO_SIZE) != dst_mo) {
6365 tcg_debug_assert(parm->ntmp != 0);
6366 mov[i].dst = src = parm->tmp[0];
6367 tcg_out_movext1(s, &mov[i]);
6368 }
6369
6370 tcg_out_st(s, dst_type, src, TCG_REG_CALL_STACK,
6371 tcg_out_helper_stk_ofs(dst_type, slot));
6372 }
6373 return;
6374
6375 found_reg:
6376 /*
6377 * The remaining arguments are in registers.
6378 * Convert slot numbers to argument registers.
6379 */
6380 nmov = i + 1;
6381 for (i = 0; i < nmov; ++i) {
6382 mov[i].dst = tcg_target_call_iarg_regs[mov[i].dst];
6383 }
6384
6385 switch (nmov) {
6386 case 4:
6387 /* The backend must have provided enough temps for the worst case. */
6388 tcg_debug_assert(parm->ntmp >= 2);
6389
6390 dst3 = mov[3].dst;
6391 for (unsigned j = 0; j < 3; ++j) {
6392 if (dst3 == mov[j].src) {
6393 /*
6394 * Conflict. Copy the source to a temporary, perform the
6395 * remaining moves, then the extension from our scratch
6396 * on the way out.
6397 */
6398 TCGReg scratch = parm->tmp[1];
6399
6400 tcg_out_mov(s, mov[3].src_type, scratch, mov[3].src);
6401 tcg_out_movext3(s, mov, mov + 1, mov + 2, parm->tmp[0]);
6402 tcg_out_movext1_new_src(s, &mov[3], scratch);
6403 break;
6404 }
6405 }
6406
6407 /* No conflicts: perform this move and continue. */
6408 tcg_out_movext1(s, &mov[3]);
6409 /* fall through */
6410
6411 case 3:
6412 tcg_out_movext3(s, mov, mov + 1, mov + 2,
6413 parm->ntmp ? parm->tmp[0] : -1);
6414 break;
6415 case 2:
6416 tcg_out_movext2(s, mov, mov + 1,
6417 parm->ntmp ? parm->tmp[0] : -1);
6418 break;
6419 case 1:
6420 tcg_out_movext1(s, mov);
6421 break;
6422 default:
6423 g_assert_not_reached();
6424 }
6425 }
6426
tcg_out_helper_load_imm(TCGContext * s,unsigned slot,TCGType type,tcg_target_long imm,const TCGLdstHelperParam * parm)6427 static void tcg_out_helper_load_imm(TCGContext *s, unsigned slot,
6428 TCGType type, tcg_target_long imm,
6429 const TCGLdstHelperParam *parm)
6430 {
6431 if (arg_slot_reg_p(slot)) {
6432 tcg_out_movi(s, type, tcg_target_call_iarg_regs[slot], imm);
6433 } else {
6434 int ofs = tcg_out_helper_stk_ofs(type, slot);
6435 if (!tcg_out_sti(s, type, imm, TCG_REG_CALL_STACK, ofs)) {
6436 tcg_debug_assert(parm->ntmp != 0);
6437 tcg_out_movi(s, type, parm->tmp[0], imm);
6438 tcg_out_st(s, type, parm->tmp[0], TCG_REG_CALL_STACK, ofs);
6439 }
6440 }
6441 }
6442
tcg_out_helper_load_common_args(TCGContext * s,const TCGLabelQemuLdst * ldst,const TCGLdstHelperParam * parm,const TCGHelperInfo * info,unsigned next_arg)6443 static void tcg_out_helper_load_common_args(TCGContext *s,
6444 const TCGLabelQemuLdst *ldst,
6445 const TCGLdstHelperParam *parm,
6446 const TCGHelperInfo *info,
6447 unsigned next_arg)
6448 {
6449 TCGMovExtend ptr_mov = {
6450 .dst_type = TCG_TYPE_PTR,
6451 .src_type = TCG_TYPE_PTR,
6452 .src_ext = sizeof(void *) == 4 ? MO_32 : MO_64
6453 };
6454 const TCGCallArgumentLoc *loc = &info->in[0];
6455 TCGType type;
6456 unsigned slot;
6457 tcg_target_ulong imm;
6458
6459 /*
6460 * Handle env, which is always first.
6461 */
6462 ptr_mov.dst = loc->arg_slot;
6463 ptr_mov.src = TCG_AREG0;
6464 tcg_out_helper_load_slots(s, 1, &ptr_mov, parm);
6465
6466 /*
6467 * Handle oi.
6468 */
6469 imm = ldst->oi;
6470 loc = &info->in[next_arg];
6471 type = TCG_TYPE_I32;
6472 switch (loc->kind) {
6473 case TCG_CALL_ARG_NORMAL:
6474 break;
6475 case TCG_CALL_ARG_EXTEND_U:
6476 case TCG_CALL_ARG_EXTEND_S:
6477 /* No extension required for MemOpIdx. */
6478 tcg_debug_assert(imm <= INT32_MAX);
6479 type = TCG_TYPE_REG;
6480 break;
6481 default:
6482 g_assert_not_reached();
6483 }
6484 tcg_out_helper_load_imm(s, loc->arg_slot, type, imm, parm);
6485 next_arg++;
6486
6487 /*
6488 * Handle ra.
6489 */
6490 loc = &info->in[next_arg];
6491 slot = loc->arg_slot;
6492 if (parm->ra_gen) {
6493 int arg_reg = -1;
6494 TCGReg ra_reg;
6495
6496 if (arg_slot_reg_p(slot)) {
6497 arg_reg = tcg_target_call_iarg_regs[slot];
6498 }
6499 ra_reg = parm->ra_gen(s, ldst, arg_reg);
6500
6501 ptr_mov.dst = slot;
6502 ptr_mov.src = ra_reg;
6503 tcg_out_helper_load_slots(s, 1, &ptr_mov, parm);
6504 } else {
6505 imm = (uintptr_t)ldst->raddr;
6506 tcg_out_helper_load_imm(s, slot, TCG_TYPE_PTR, imm, parm);
6507 }
6508 }
6509
tcg_out_helper_add_mov(TCGMovExtend * mov,const TCGCallArgumentLoc * loc,TCGType dst_type,TCGType src_type,TCGReg lo,TCGReg hi)6510 static unsigned tcg_out_helper_add_mov(TCGMovExtend *mov,
6511 const TCGCallArgumentLoc *loc,
6512 TCGType dst_type, TCGType src_type,
6513 TCGReg lo, TCGReg hi)
6514 {
6515 MemOp reg_mo;
6516
6517 if (dst_type <= TCG_TYPE_REG) {
6518 MemOp src_ext;
6519
6520 switch (loc->kind) {
6521 case TCG_CALL_ARG_NORMAL:
6522 src_ext = src_type == TCG_TYPE_I32 ? MO_32 : MO_64;
6523 break;
6524 case TCG_CALL_ARG_EXTEND_U:
6525 dst_type = TCG_TYPE_REG;
6526 src_ext = MO_UL;
6527 break;
6528 case TCG_CALL_ARG_EXTEND_S:
6529 dst_type = TCG_TYPE_REG;
6530 src_ext = MO_SL;
6531 break;
6532 default:
6533 g_assert_not_reached();
6534 }
6535
6536 mov[0].dst = loc->arg_slot;
6537 mov[0].dst_type = dst_type;
6538 mov[0].src = lo;
6539 mov[0].src_type = src_type;
6540 mov[0].src_ext = src_ext;
6541 return 1;
6542 }
6543
6544 if (TCG_TARGET_REG_BITS == 32) {
6545 assert(dst_type == TCG_TYPE_I64);
6546 reg_mo = MO_32;
6547 } else {
6548 assert(dst_type == TCG_TYPE_I128);
6549 reg_mo = MO_64;
6550 }
6551
6552 mov[0].dst = loc[HOST_BIG_ENDIAN].arg_slot;
6553 mov[0].src = lo;
6554 mov[0].dst_type = TCG_TYPE_REG;
6555 mov[0].src_type = TCG_TYPE_REG;
6556 mov[0].src_ext = reg_mo;
6557
6558 mov[1].dst = loc[!HOST_BIG_ENDIAN].arg_slot;
6559 mov[1].src = hi;
6560 mov[1].dst_type = TCG_TYPE_REG;
6561 mov[1].src_type = TCG_TYPE_REG;
6562 mov[1].src_ext = reg_mo;
6563
6564 return 2;
6565 }
6566
tcg_out_ld_helper_args(TCGContext * s,const TCGLabelQemuLdst * ldst,const TCGLdstHelperParam * parm)6567 static void tcg_out_ld_helper_args(TCGContext *s, const TCGLabelQemuLdst *ldst,
6568 const TCGLdstHelperParam *parm)
6569 {
6570 const TCGHelperInfo *info;
6571 const TCGCallArgumentLoc *loc;
6572 TCGMovExtend mov[2];
6573 unsigned next_arg, nmov;
6574 MemOp mop = get_memop(ldst->oi);
6575
6576 switch (mop & MO_SIZE) {
6577 case MO_8:
6578 case MO_16:
6579 case MO_32:
6580 info = &info_helper_ld32_mmu;
6581 break;
6582 case MO_64:
6583 info = &info_helper_ld64_mmu;
6584 break;
6585 case MO_128:
6586 info = &info_helper_ld128_mmu;
6587 break;
6588 default:
6589 g_assert_not_reached();
6590 }
6591
6592 /* Defer env argument. */
6593 next_arg = 1;
6594
6595 loc = &info->in[next_arg];
6596 if (TCG_TARGET_REG_BITS == 32 && s->addr_type == TCG_TYPE_I32) {
6597 /*
6598 * 32-bit host with 32-bit guest: zero-extend the guest address
6599 * to 64-bits for the helper by storing the low part, then
6600 * load a zero for the high part.
6601 */
6602 tcg_out_helper_add_mov(mov, loc + HOST_BIG_ENDIAN,
6603 TCG_TYPE_I32, TCG_TYPE_I32,
6604 ldst->addr_reg, -1);
6605 tcg_out_helper_load_slots(s, 1, mov, parm);
6606
6607 tcg_out_helper_load_imm(s, loc[!HOST_BIG_ENDIAN].arg_slot,
6608 TCG_TYPE_I32, 0, parm);
6609 next_arg += 2;
6610 } else {
6611 nmov = tcg_out_helper_add_mov(mov, loc, TCG_TYPE_I64, s->addr_type,
6612 ldst->addr_reg, -1);
6613 tcg_out_helper_load_slots(s, nmov, mov, parm);
6614 next_arg += nmov;
6615 }
6616
6617 switch (info->out_kind) {
6618 case TCG_CALL_RET_NORMAL:
6619 case TCG_CALL_RET_BY_VEC:
6620 break;
6621 case TCG_CALL_RET_BY_REF:
6622 /*
6623 * The return reference is in the first argument slot.
6624 * We need memory in which to return: re-use the top of stack.
6625 */
6626 {
6627 int ofs_slot0 = TCG_TARGET_CALL_STACK_OFFSET;
6628
6629 if (arg_slot_reg_p(0)) {
6630 tcg_out_addi_ptr(s, tcg_target_call_iarg_regs[0],
6631 TCG_REG_CALL_STACK, ofs_slot0);
6632 } else {
6633 tcg_debug_assert(parm->ntmp != 0);
6634 tcg_out_addi_ptr(s, parm->tmp[0],
6635 TCG_REG_CALL_STACK, ofs_slot0);
6636 tcg_out_st(s, TCG_TYPE_PTR, parm->tmp[0],
6637 TCG_REG_CALL_STACK, ofs_slot0);
6638 }
6639 }
6640 break;
6641 default:
6642 g_assert_not_reached();
6643 }
6644
6645 tcg_out_helper_load_common_args(s, ldst, parm, info, next_arg);
6646 }
6647
tcg_out_ld_helper_ret(TCGContext * s,const TCGLabelQemuLdst * ldst,bool load_sign,const TCGLdstHelperParam * parm)6648 static void tcg_out_ld_helper_ret(TCGContext *s, const TCGLabelQemuLdst *ldst,
6649 bool load_sign,
6650 const TCGLdstHelperParam *parm)
6651 {
6652 MemOp mop = get_memop(ldst->oi);
6653 TCGMovExtend mov[2];
6654 int ofs_slot0;
6655
6656 switch (ldst->type) {
6657 case TCG_TYPE_I64:
6658 if (TCG_TARGET_REG_BITS == 32) {
6659 break;
6660 }
6661 /* fall through */
6662
6663 case TCG_TYPE_I32:
6664 mov[0].dst = ldst->datalo_reg;
6665 mov[0].src = tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, 0);
6666 mov[0].dst_type = ldst->type;
6667 mov[0].src_type = TCG_TYPE_REG;
6668
6669 /*
6670 * If load_sign, then we allowed the helper to perform the
6671 * appropriate sign extension to tcg_target_ulong, and all
6672 * we need now is a plain move.
6673 *
6674 * If they do not, then we expect the relevant extension
6675 * instruction to be no more expensive than a move, and
6676 * we thus save the icache etc by only using one of two
6677 * helper functions.
6678 */
6679 if (load_sign || !(mop & MO_SIGN)) {
6680 if (TCG_TARGET_REG_BITS == 32 || ldst->type == TCG_TYPE_I32) {
6681 mov[0].src_ext = MO_32;
6682 } else {
6683 mov[0].src_ext = MO_64;
6684 }
6685 } else {
6686 mov[0].src_ext = mop & MO_SSIZE;
6687 }
6688 tcg_out_movext1(s, mov);
6689 return;
6690
6691 case TCG_TYPE_I128:
6692 tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
6693 ofs_slot0 = TCG_TARGET_CALL_STACK_OFFSET;
6694 switch (TCG_TARGET_CALL_RET_I128) {
6695 case TCG_CALL_RET_NORMAL:
6696 break;
6697 case TCG_CALL_RET_BY_VEC:
6698 tcg_out_st(s, TCG_TYPE_V128,
6699 tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0),
6700 TCG_REG_CALL_STACK, ofs_slot0);
6701 /* fall through */
6702 case TCG_CALL_RET_BY_REF:
6703 tcg_out_ld(s, TCG_TYPE_I64, ldst->datalo_reg,
6704 TCG_REG_CALL_STACK, ofs_slot0 + 8 * HOST_BIG_ENDIAN);
6705 tcg_out_ld(s, TCG_TYPE_I64, ldst->datahi_reg,
6706 TCG_REG_CALL_STACK, ofs_slot0 + 8 * !HOST_BIG_ENDIAN);
6707 return;
6708 default:
6709 g_assert_not_reached();
6710 }
6711 break;
6712
6713 default:
6714 g_assert_not_reached();
6715 }
6716
6717 mov[0].dst = ldst->datalo_reg;
6718 mov[0].src =
6719 tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, HOST_BIG_ENDIAN);
6720 mov[0].dst_type = TCG_TYPE_REG;
6721 mov[0].src_type = TCG_TYPE_REG;
6722 mov[0].src_ext = TCG_TARGET_REG_BITS == 32 ? MO_32 : MO_64;
6723
6724 mov[1].dst = ldst->datahi_reg;
6725 mov[1].src =
6726 tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, !HOST_BIG_ENDIAN);
6727 mov[1].dst_type = TCG_TYPE_REG;
6728 mov[1].src_type = TCG_TYPE_REG;
6729 mov[1].src_ext = TCG_TARGET_REG_BITS == 32 ? MO_32 : MO_64;
6730
6731 tcg_out_movext2(s, mov, mov + 1, parm->ntmp ? parm->tmp[0] : -1);
6732 }
6733
tcg_out_st_helper_args(TCGContext * s,const TCGLabelQemuLdst * ldst,const TCGLdstHelperParam * parm)6734 static void tcg_out_st_helper_args(TCGContext *s, const TCGLabelQemuLdst *ldst,
6735 const TCGLdstHelperParam *parm)
6736 {
6737 const TCGHelperInfo *info;
6738 const TCGCallArgumentLoc *loc;
6739 TCGMovExtend mov[4];
6740 TCGType data_type;
6741 unsigned next_arg, nmov, n;
6742 MemOp mop = get_memop(ldst->oi);
6743
6744 switch (mop & MO_SIZE) {
6745 case MO_8:
6746 case MO_16:
6747 case MO_32:
6748 info = &info_helper_st32_mmu;
6749 data_type = TCG_TYPE_I32;
6750 break;
6751 case MO_64:
6752 info = &info_helper_st64_mmu;
6753 data_type = TCG_TYPE_I64;
6754 break;
6755 case MO_128:
6756 info = &info_helper_st128_mmu;
6757 data_type = TCG_TYPE_I128;
6758 break;
6759 default:
6760 g_assert_not_reached();
6761 }
6762
6763 /* Defer env argument. */
6764 next_arg = 1;
6765 nmov = 0;
6766
6767 /* Handle addr argument. */
6768 loc = &info->in[next_arg];
6769 tcg_debug_assert(s->addr_type <= TCG_TYPE_REG);
6770 if (TCG_TARGET_REG_BITS == 32) {
6771 /*
6772 * 32-bit host (and thus 32-bit guest): zero-extend the guest address
6773 * to 64-bits for the helper by storing the low part. Later,
6774 * after we have processed the register inputs, we will load a
6775 * zero for the high part.
6776 */
6777 tcg_out_helper_add_mov(mov, loc + HOST_BIG_ENDIAN,
6778 TCG_TYPE_I32, TCG_TYPE_I32,
6779 ldst->addr_reg, -1);
6780 next_arg += 2;
6781 nmov += 1;
6782 } else {
6783 n = tcg_out_helper_add_mov(mov, loc, TCG_TYPE_I64, s->addr_type,
6784 ldst->addr_reg, -1);
6785 next_arg += n;
6786 nmov += n;
6787 }
6788
6789 /* Handle data argument. */
6790 loc = &info->in[next_arg];
6791 switch (loc->kind) {
6792 case TCG_CALL_ARG_NORMAL:
6793 case TCG_CALL_ARG_EXTEND_U:
6794 case TCG_CALL_ARG_EXTEND_S:
6795 n = tcg_out_helper_add_mov(mov + nmov, loc, data_type, ldst->type,
6796 ldst->datalo_reg, ldst->datahi_reg);
6797 next_arg += n;
6798 nmov += n;
6799 tcg_out_helper_load_slots(s, nmov, mov, parm);
6800 break;
6801
6802 case TCG_CALL_ARG_BY_REF:
6803 tcg_debug_assert(TCG_TARGET_REG_BITS == 64);
6804 tcg_debug_assert(data_type == TCG_TYPE_I128);
6805 tcg_out_st(s, TCG_TYPE_I64,
6806 HOST_BIG_ENDIAN ? ldst->datahi_reg : ldst->datalo_reg,
6807 TCG_REG_CALL_STACK, arg_slot_stk_ofs(loc[0].ref_slot));
6808 tcg_out_st(s, TCG_TYPE_I64,
6809 HOST_BIG_ENDIAN ? ldst->datalo_reg : ldst->datahi_reg,
6810 TCG_REG_CALL_STACK, arg_slot_stk_ofs(loc[1].ref_slot));
6811
6812 tcg_out_helper_load_slots(s, nmov, mov, parm);
6813
6814 if (arg_slot_reg_p(loc->arg_slot)) {
6815 tcg_out_addi_ptr(s, tcg_target_call_iarg_regs[loc->arg_slot],
6816 TCG_REG_CALL_STACK,
6817 arg_slot_stk_ofs(loc->ref_slot));
6818 } else {
6819 tcg_debug_assert(parm->ntmp != 0);
6820 tcg_out_addi_ptr(s, parm->tmp[0], TCG_REG_CALL_STACK,
6821 arg_slot_stk_ofs(loc->ref_slot));
6822 tcg_out_st(s, TCG_TYPE_PTR, parm->tmp[0],
6823 TCG_REG_CALL_STACK, arg_slot_stk_ofs(loc->arg_slot));
6824 }
6825 next_arg += 2;
6826 break;
6827
6828 default:
6829 g_assert_not_reached();
6830 }
6831
6832 if (TCG_TARGET_REG_BITS == 32) {
6833 /* Zero extend the address by loading a zero for the high part. */
6834 loc = &info->in[1 + !HOST_BIG_ENDIAN];
6835 tcg_out_helper_load_imm(s, loc->arg_slot, TCG_TYPE_I32, 0, parm);
6836 }
6837
6838 tcg_out_helper_load_common_args(s, ldst, parm, info, next_arg);
6839 }
6840
tcg_gen_code(TCGContext * s,TranslationBlock * tb,uint64_t pc_start)6841 int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start)
6842 {
6843 int i, num_insns;
6844 TCGOp *op;
6845
6846 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)
6847 && qemu_log_in_addr_range(pc_start))) {
6848 FILE *logfile = qemu_log_trylock();
6849 if (logfile) {
6850 fprintf(logfile, "OP:\n");
6851 tcg_dump_ops(s, logfile, false);
6852 fprintf(logfile, "\n");
6853 qemu_log_unlock(logfile);
6854 }
6855 }
6856
6857 #ifdef CONFIG_DEBUG_TCG
6858 /* Ensure all labels referenced have been emitted. */
6859 {
6860 TCGLabel *l;
6861 bool error = false;
6862
6863 QSIMPLEQ_FOREACH(l, &s->labels, next) {
6864 if (unlikely(!l->present) && !QSIMPLEQ_EMPTY(&l->branches)) {
6865 qemu_log_mask(CPU_LOG_TB_OP,
6866 "$L%d referenced but not present.\n", l->id);
6867 error = true;
6868 }
6869 }
6870 assert(!error);
6871 }
6872 #endif
6873
6874 /* Do not reuse any EBB that may be allocated within the TB. */
6875 tcg_temp_ebb_reset_freed(s);
6876
6877 tcg_optimize(s);
6878
6879 reachable_code_pass(s);
6880 liveness_pass_0(s);
6881 liveness_pass_1(s);
6882
6883 if (s->nb_indirects > 0) {
6884 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND)
6885 && qemu_log_in_addr_range(pc_start))) {
6886 FILE *logfile = qemu_log_trylock();
6887 if (logfile) {
6888 fprintf(logfile, "OP before indirect lowering:\n");
6889 tcg_dump_ops(s, logfile, false);
6890 fprintf(logfile, "\n");
6891 qemu_log_unlock(logfile);
6892 }
6893 }
6894
6895 /* Replace indirect temps with direct temps. */
6896 if (liveness_pass_2(s)) {
6897 /* If changes were made, re-run liveness. */
6898 liveness_pass_1(s);
6899 }
6900 }
6901
6902 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT)
6903 && qemu_log_in_addr_range(pc_start))) {
6904 FILE *logfile = qemu_log_trylock();
6905 if (logfile) {
6906 fprintf(logfile, "OP after optimization and liveness analysis:\n");
6907 tcg_dump_ops(s, logfile, true);
6908 fprintf(logfile, "\n");
6909 qemu_log_unlock(logfile);
6910 }
6911 }
6912
6913 /* Initialize goto_tb jump offsets. */
6914 tb->jmp_reset_offset[0] = TB_JMP_OFFSET_INVALID;
6915 tb->jmp_reset_offset[1] = TB_JMP_OFFSET_INVALID;
6916 tb->jmp_insn_offset[0] = TB_JMP_OFFSET_INVALID;
6917 tb->jmp_insn_offset[1] = TB_JMP_OFFSET_INVALID;
6918
6919 tcg_reg_alloc_start(s);
6920
6921 /*
6922 * Reset the buffer pointers when restarting after overflow.
6923 * TODO: Move this into translate-all.c with the rest of the
6924 * buffer management. Having only this done here is confusing.
6925 */
6926 s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr);
6927 s->code_ptr = s->code_buf;
6928 s->data_gen_ptr = NULL;
6929
6930 QSIMPLEQ_INIT(&s->ldst_labels);
6931 s->pool_labels = NULL;
6932
6933 s->gen_insn_data =
6934 tcg_malloc(sizeof(uint64_t) * s->gen_tb->icount * INSN_START_WORDS);
6935
6936 tcg_out_tb_start(s);
6937
6938 num_insns = -1;
6939 s->carry_live = false;
6940 QTAILQ_FOREACH(op, &s->ops, link) {
6941 TCGOpcode opc = op->opc;
6942
6943 switch (opc) {
6944 case INDEX_op_extrl_i64_i32:
6945 assert(TCG_TARGET_REG_BITS == 64);
6946 /*
6947 * If TCG_TYPE_I32 is represented in some canonical form,
6948 * e.g. zero or sign-extended, then emit as a unary op.
6949 * Otherwise we can treat this as a plain move.
6950 * If the output dies, treat this as a plain move, because
6951 * this will be implemented with a store.
6952 */
6953 if (TCG_TARGET_HAS_extr_i64_i32) {
6954 TCGLifeData arg_life = op->life;
6955 if (!IS_DEAD_ARG(0)) {
6956 goto do_default;
6957 }
6958 }
6959 /* fall through */
6960 case INDEX_op_mov:
6961 case INDEX_op_mov_vec:
6962 tcg_reg_alloc_mov(s, op);
6963 break;
6964 case INDEX_op_dup_vec:
6965 tcg_reg_alloc_dup(s, op);
6966 break;
6967 case INDEX_op_insn_start:
6968 assert_carry_dead(s);
6969 if (num_insns >= 0) {
6970 size_t off = tcg_current_code_size(s);
6971 s->gen_insn_end_off[num_insns] = off;
6972 /* Assert that we do not overflow our stored offset. */
6973 assert(s->gen_insn_end_off[num_insns] == off);
6974 }
6975 num_insns++;
6976 for (i = 0; i < INSN_START_WORDS; ++i) {
6977 s->gen_insn_data[num_insns * INSN_START_WORDS + i] =
6978 tcg_get_insn_start_param(op, i);
6979 }
6980 break;
6981 case INDEX_op_discard:
6982 temp_dead(s, arg_temp(op->args[0]));
6983 break;
6984 case INDEX_op_set_label:
6985 tcg_reg_alloc_bb_end(s, s->reserved_regs);
6986 tcg_out_label(s, arg_label(op->args[0]));
6987 break;
6988 case INDEX_op_call:
6989 assert_carry_dead(s);
6990 tcg_reg_alloc_call(s, op);
6991 break;
6992 case INDEX_op_exit_tb:
6993 tcg_out_exit_tb(s, op->args[0]);
6994 break;
6995 case INDEX_op_goto_tb:
6996 tcg_out_goto_tb(s, op->args[0]);
6997 break;
6998 case INDEX_op_br:
6999 tcg_out_br(s, arg_label(op->args[0]));
7000 break;
7001 case INDEX_op_mb:
7002 tcg_out_mb(s, op->args[0]);
7003 break;
7004 case INDEX_op_dup2_vec:
7005 if (tcg_reg_alloc_dup2(s, op)) {
7006 break;
7007 }
7008 /* fall through */
7009 default:
7010 do_default:
7011 /* Sanity check that we've not introduced any unhandled opcodes. */
7012 tcg_debug_assert(tcg_op_supported(opc, TCGOP_TYPE(op),
7013 TCGOP_FLAGS(op)));
7014 /* Note: in order to speed up the code, it would be much
7015 faster to have specialized register allocator functions for
7016 some common argument patterns */
7017 tcg_reg_alloc_op(s, op);
7018 break;
7019 }
7020 /* Test for (pending) buffer overflow. The assumption is that any
7021 one operation beginning below the high water mark cannot overrun
7022 the buffer completely. Thus we can test for overflow after
7023 generating code without having to check during generation. */
7024 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
7025 return -1;
7026 }
7027 /* Test for TB overflow, as seen by gen_insn_end_off. */
7028 if (unlikely(tcg_current_code_size(s) > UINT16_MAX)) {
7029 return -2;
7030 }
7031 }
7032 assert_carry_dead(s);
7033
7034 tcg_debug_assert(num_insns + 1 == s->gen_tb->icount);
7035 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s);
7036
7037 /* Generate TB finalization at the end of block */
7038 i = tcg_out_ldst_finalize(s);
7039 if (i < 0) {
7040 return i;
7041 }
7042 i = tcg_out_pool_finalize(s);
7043 if (i < 0) {
7044 return i;
7045 }
7046 if (!tcg_resolve_relocs(s)) {
7047 return -2;
7048 }
7049
7050 #ifndef CONFIG_TCG_INTERPRETER
7051 /* flush instruction cache */
7052 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
7053 (uintptr_t)s->code_buf,
7054 tcg_ptr_byte_diff(s->code_ptr, s->code_buf));
7055 #endif
7056
7057 return tcg_current_code_size(s);
7058 }
7059
7060 #ifdef ELF_HOST_MACHINE
7061 /* In order to use this feature, the backend needs to do three things:
7062
7063 (1) Define ELF_HOST_MACHINE to indicate both what value to
7064 put into the ELF image and to indicate support for the feature.
7065
7066 (2) Define tcg_register_jit. This should create a buffer containing
7067 the contents of a .debug_frame section that describes the post-
7068 prologue unwind info for the tcg machine.
7069
7070 (3) Call tcg_register_jit_int, with the constructed .debug_frame.
7071 */
7072
7073 /* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */
7074 typedef enum {
7075 JIT_NOACTION = 0,
7076 JIT_REGISTER_FN,
7077 JIT_UNREGISTER_FN
7078 } jit_actions_t;
7079
7080 struct jit_code_entry {
7081 struct jit_code_entry *next_entry;
7082 struct jit_code_entry *prev_entry;
7083 const void *symfile_addr;
7084 uint64_t symfile_size;
7085 };
7086
7087 struct jit_descriptor {
7088 uint32_t version;
7089 uint32_t action_flag;
7090 struct jit_code_entry *relevant_entry;
7091 struct jit_code_entry *first_entry;
7092 };
7093
7094 void __jit_debug_register_code(void) __attribute__((noinline));
__jit_debug_register_code(void)7095 void __jit_debug_register_code(void)
7096 {
7097 asm("");
7098 }
7099
7100 /* Must statically initialize the version, because GDB may check
7101 the version before we can set it. */
7102 struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 };
7103
7104 /* End GDB interface. */
7105
find_string(const char * strtab,const char * str)7106 static int find_string(const char *strtab, const char *str)
7107 {
7108 const char *p = strtab + 1;
7109
7110 while (1) {
7111 if (strcmp(p, str) == 0) {
7112 return p - strtab;
7113 }
7114 p += strlen(p) + 1;
7115 }
7116 }
7117
tcg_register_jit_int(const void * buf_ptr,size_t buf_size,const void * debug_frame,size_t debug_frame_size)7118 static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size,
7119 const void *debug_frame,
7120 size_t debug_frame_size)
7121 {
7122 struct __attribute__((packed)) DebugInfo {
7123 uint32_t len;
7124 uint16_t version;
7125 uint32_t abbrev;
7126 uint8_t ptr_size;
7127 uint8_t cu_die;
7128 uint16_t cu_lang;
7129 uintptr_t cu_low_pc;
7130 uintptr_t cu_high_pc;
7131 uint8_t fn_die;
7132 char fn_name[16];
7133 uintptr_t fn_low_pc;
7134 uintptr_t fn_high_pc;
7135 uint8_t cu_eoc;
7136 };
7137
7138 struct ElfImage {
7139 ElfW(Ehdr) ehdr;
7140 ElfW(Phdr) phdr;
7141 ElfW(Shdr) shdr[7];
7142 ElfW(Sym) sym[2];
7143 struct DebugInfo di;
7144 uint8_t da[24];
7145 char str[80];
7146 };
7147
7148 struct ElfImage *img;
7149
7150 static const struct ElfImage img_template = {
7151 .ehdr = {
7152 .e_ident[EI_MAG0] = ELFMAG0,
7153 .e_ident[EI_MAG1] = ELFMAG1,
7154 .e_ident[EI_MAG2] = ELFMAG2,
7155 .e_ident[EI_MAG3] = ELFMAG3,
7156 .e_ident[EI_CLASS] = ELF_CLASS,
7157 .e_ident[EI_DATA] = ELF_DATA,
7158 .e_ident[EI_VERSION] = EV_CURRENT,
7159 .e_type = ET_EXEC,
7160 .e_machine = ELF_HOST_MACHINE,
7161 .e_version = EV_CURRENT,
7162 .e_phoff = offsetof(struct ElfImage, phdr),
7163 .e_shoff = offsetof(struct ElfImage, shdr),
7164 .e_ehsize = sizeof(ElfW(Shdr)),
7165 .e_phentsize = sizeof(ElfW(Phdr)),
7166 .e_phnum = 1,
7167 .e_shentsize = sizeof(ElfW(Shdr)),
7168 .e_shnum = ARRAY_SIZE(img->shdr),
7169 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1,
7170 #ifdef ELF_HOST_FLAGS
7171 .e_flags = ELF_HOST_FLAGS,
7172 #endif
7173 #ifdef ELF_OSABI
7174 .e_ident[EI_OSABI] = ELF_OSABI,
7175 #endif
7176 },
7177 .phdr = {
7178 .p_type = PT_LOAD,
7179 .p_flags = PF_X,
7180 },
7181 .shdr = {
7182 [0] = { .sh_type = SHT_NULL },
7183 /* Trick: The contents of code_gen_buffer are not present in
7184 this fake ELF file; that got allocated elsewhere. Therefore
7185 we mark .text as SHT_NOBITS (similar to .bss) so that readers
7186 will not look for contents. We can record any address. */
7187 [1] = { /* .text */
7188 .sh_type = SHT_NOBITS,
7189 .sh_flags = SHF_EXECINSTR | SHF_ALLOC,
7190 },
7191 [2] = { /* .debug_info */
7192 .sh_type = SHT_PROGBITS,
7193 .sh_offset = offsetof(struct ElfImage, di),
7194 .sh_size = sizeof(struct DebugInfo),
7195 },
7196 [3] = { /* .debug_abbrev */
7197 .sh_type = SHT_PROGBITS,
7198 .sh_offset = offsetof(struct ElfImage, da),
7199 .sh_size = sizeof(img->da),
7200 },
7201 [4] = { /* .debug_frame */
7202 .sh_type = SHT_PROGBITS,
7203 .sh_offset = sizeof(struct ElfImage),
7204 },
7205 [5] = { /* .symtab */
7206 .sh_type = SHT_SYMTAB,
7207 .sh_offset = offsetof(struct ElfImage, sym),
7208 .sh_size = sizeof(img->sym),
7209 .sh_info = 1,
7210 .sh_link = ARRAY_SIZE(img->shdr) - 1,
7211 .sh_entsize = sizeof(ElfW(Sym)),
7212 },
7213 [6] = { /* .strtab */
7214 .sh_type = SHT_STRTAB,
7215 .sh_offset = offsetof(struct ElfImage, str),
7216 .sh_size = sizeof(img->str),
7217 }
7218 },
7219 .sym = {
7220 [1] = { /* code_gen_buffer */
7221 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC),
7222 .st_shndx = 1,
7223 }
7224 },
7225 .di = {
7226 .len = sizeof(struct DebugInfo) - 4,
7227 .version = 2,
7228 .ptr_size = sizeof(void *),
7229 .cu_die = 1,
7230 .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */
7231 .fn_die = 2,
7232 .fn_name = "code_gen_buffer"
7233 },
7234 .da = {
7235 1, /* abbrev number (the cu) */
7236 0x11, 1, /* DW_TAG_compile_unit, has children */
7237 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */
7238 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
7239 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
7240 0, 0, /* end of abbrev */
7241 2, /* abbrev number (the fn) */
7242 0x2e, 0, /* DW_TAG_subprogram, no children */
7243 0x3, 0x8, /* DW_AT_name, DW_FORM_string */
7244 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
7245 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
7246 0, 0, /* end of abbrev */
7247 0 /* no more abbrev */
7248 },
7249 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0"
7250 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer",
7251 };
7252
7253 /* We only need a single jit entry; statically allocate it. */
7254 static struct jit_code_entry one_entry;
7255
7256 uintptr_t buf = (uintptr_t)buf_ptr;
7257 size_t img_size = sizeof(struct ElfImage) + debug_frame_size;
7258 DebugFrameHeader *dfh;
7259
7260 img = g_malloc(img_size);
7261 *img = img_template;
7262
7263 img->phdr.p_vaddr = buf;
7264 img->phdr.p_paddr = buf;
7265 img->phdr.p_memsz = buf_size;
7266
7267 img->shdr[1].sh_name = find_string(img->str, ".text");
7268 img->shdr[1].sh_addr = buf;
7269 img->shdr[1].sh_size = buf_size;
7270
7271 img->shdr[2].sh_name = find_string(img->str, ".debug_info");
7272 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev");
7273
7274 img->shdr[4].sh_name = find_string(img->str, ".debug_frame");
7275 img->shdr[4].sh_size = debug_frame_size;
7276
7277 img->shdr[5].sh_name = find_string(img->str, ".symtab");
7278 img->shdr[6].sh_name = find_string(img->str, ".strtab");
7279
7280 img->sym[1].st_name = find_string(img->str, "code_gen_buffer");
7281 img->sym[1].st_value = buf;
7282 img->sym[1].st_size = buf_size;
7283
7284 img->di.cu_low_pc = buf;
7285 img->di.cu_high_pc = buf + buf_size;
7286 img->di.fn_low_pc = buf;
7287 img->di.fn_high_pc = buf + buf_size;
7288
7289 dfh = (DebugFrameHeader *)(img + 1);
7290 memcpy(dfh, debug_frame, debug_frame_size);
7291 dfh->fde.func_start = buf;
7292 dfh->fde.func_len = buf_size;
7293
7294 #ifdef DEBUG_JIT
7295 /* Enable this block to be able to debug the ELF image file creation.
7296 One can use readelf, objdump, or other inspection utilities. */
7297 {
7298 g_autofree char *jit = g_strdup_printf("%s/qemu.jit", g_get_tmp_dir());
7299 FILE *f = fopen(jit, "w+b");
7300 if (f) {
7301 if (fwrite(img, img_size, 1, f) != img_size) {
7302 /* Avoid stupid unused return value warning for fwrite. */
7303 }
7304 fclose(f);
7305 }
7306 }
7307 #endif
7308
7309 one_entry.symfile_addr = img;
7310 one_entry.symfile_size = img_size;
7311
7312 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN;
7313 __jit_debug_descriptor.relevant_entry = &one_entry;
7314 __jit_debug_descriptor.first_entry = &one_entry;
7315 __jit_debug_register_code();
7316 }
7317 #else
7318 /* No support for the feature. Provide the entry point expected by exec.c,
7319 and implement the internal function we declared earlier. */
7320
tcg_register_jit_int(const void * buf,size_t size,const void * debug_frame,size_t debug_frame_size)7321 static void tcg_register_jit_int(const void *buf, size_t size,
7322 const void *debug_frame,
7323 size_t debug_frame_size)
7324 {
7325 }
7326
tcg_register_jit(const void * buf,size_t buf_size)7327 void tcg_register_jit(const void *buf, size_t buf_size)
7328 {
7329 }
7330 #endif /* ELF_HOST_MACHINE */
7331
7332 #if !TCG_TARGET_MAYBE_vec
tcg_expand_vec_op(TCGOpcode o,TCGType t,unsigned e,TCGArg a0,...)7333 void tcg_expand_vec_op(TCGOpcode o, TCGType t, unsigned e, TCGArg a0, ...)
7334 {
7335 g_assert_not_reached();
7336 }
7337 #endif
7338