Lines Matching +full:cross +full:- +full:s390x +full:- +full:user
30 #include "qemu/error-report.h"
32 #include "qemu/host-utils.h"
33 #include "qemu/qemu-print.h"
38 #include "exec/translation-block.h"
39 #include "exec/tlb-common.h"
41 #include "tcg/tcg-op-common.h"
56 #include "tcg/tcg-ldst.h"
57 #include "tcg/tcg-temp-internal.h"
58 #include "tcg-internal.h"
60 #include "tcg-has.h"
62 #include "user/guest-base.h"
65 /* Forward declarations for functions declared in tcg-target.c.inc and
117 /* Forward declarations for functions declared and used in tcg-target.c.inc. */
267 *s->code_ptr++ = v; in tcg_out8()
281 *s->code_ptr++ = v; in tcg_out16()
283 tcg_insn_unit *p = s->code_ptr; in tcg_out16()
285 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE); in tcg_out16()
304 *s->code_ptr++ = v; in tcg_out32()
306 tcg_insn_unit *p = s->code_ptr; in tcg_out32()
308 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE); in tcg_out32()
327 *s->code_ptr++ = v; in tcg_out64()
329 tcg_insn_unit *p = s->code_ptr; in tcg_out64()
331 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE); in tcg_out64()
353 r->type = type; in tcg_out_reloc()
354 r->ptr = code_ptr; in tcg_out_reloc()
355 r->addend = addend; in tcg_out_reloc()
356 QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next); in tcg_out_reloc()
361 tcg_debug_assert(!l->has_value); in tcg_out_label()
362 l->has_value = 1; in tcg_out_label()
363 l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr); in tcg_out_label()
372 l->id = s->nb_labels++; in gen_new_label()
373 QSIMPLEQ_INIT(&l->branches); in gen_new_label()
374 QSIMPLEQ_INIT(&l->relocs); in gen_new_label()
376 QSIMPLEQ_INSERT_TAIL(&s->labels, l, next); in gen_new_label()
385 QSIMPLEQ_FOREACH(l, &s->labels, next) { in tcg_resolve_relocs()
387 uintptr_t value = l->u.value; in tcg_resolve_relocs()
389 QSIMPLEQ_FOREACH(r, &l->relocs, next) { in tcg_resolve_relocs()
390 if (!patch_reloc(r->ptr, r->type, value, r->addend)) { in tcg_resolve_relocs()
404 s->gen_tb->jmp_reset_offset[which] = tcg_current_code_size(s); in set_jmp_reset_offset()
413 s->gen_tb->jmp_insn_offset[which] = tcg_current_code_size(s); in set_jmp_insn_offset()
419 * Return the read-execute version of the pointer, for the benefit in get_jmp_target_addr()
420 * of any pc-relative addressing mode. in get_jmp_target_addr()
422 return (uintptr_t)tcg_splitwx_to_rx(&s->gen_tb->jmp_target_addr[which]); in get_jmp_target_addr()
428 return (offsetof(CPUNegativeOffsetState, tlb.f[which]) - in tlb_mask_table_ofs()
436 siglongjmp(s->jmp_trans, -2); in tcg_raise_tb_overflow()
457 * tcg_out_movext -- move and extend
522 tcg_out_movext(s, i->dst_type, i->dst, i->src_type, i->src_ext, src); in tcg_out_movext1_new_src()
527 tcg_out_movext1_new_src(s, i, i->src); in tcg_out_movext1()
531 * tcg_out_movext2 -- move and extend two pair
535 * @scratch: temporary register, or -1 for none
544 TCGReg src1 = i1->src; in tcg_out_movext2()
545 TCGReg src2 = i2->src; in tcg_out_movext2()
547 if (i1->dst != src2) { in tcg_out_movext2()
552 if (i2->dst == src1) { in tcg_out_movext2()
553 TCGType src1_type = i1->src_type; in tcg_out_movext2()
554 TCGType src2_type = i2->src_type; in tcg_out_movext2()
558 src1 = i2->src; in tcg_out_movext2()
559 src2 = i1->src; in tcg_out_movext2()
571 * tcg_out_movext3 -- move and extend three pair
576 * @scratch: temporary register, or -1 for none
586 TCGReg src1 = i1->src; in tcg_out_movext3()
587 TCGReg src2 = i2->src; in tcg_out_movext3()
588 TCGReg src3 = i3->src; in tcg_out_movext3()
590 if (i1->dst != src2 && i1->dst != src3) { in tcg_out_movext3()
595 if (i2->dst != src1 && i2->dst != src3) { in tcg_out_movext3()
600 if (i3->dst != src1 && i3->dst != src2) { in tcg_out_movext3()
608 * either "clockwise" or "anti-clockwise", and can be solved with in tcg_out_movext3()
611 if (i1->dst == src2 && i2->dst == src3 && i3->dst == src1) { in tcg_out_movext3()
613 if (tcg_out_xchg(s, MAX(i1->src_type, i2->src_type), src1, src2)) { in tcg_out_movext3()
614 tcg_out_xchg(s, MAX(i2->src_type, i3->src_type), src2, src3); in tcg_out_movext3()
616 tcg_out_movext1_new_src(s, i1, i1->dst); in tcg_out_movext3()
617 tcg_out_movext1_new_src(s, i2, i2->dst); in tcg_out_movext3()
618 tcg_out_movext1_new_src(s, i3, i3->dst); in tcg_out_movext3()
621 tcg_out_mov(s, i1->src_type, scratch, src1); in tcg_out_movext3()
626 } else if (i1->dst == src3 && i2->dst == src1 && i3->dst == src2) { in tcg_out_movext3()
627 /* "Anti-clockwise" */ in tcg_out_movext3()
628 if (tcg_out_xchg(s, MAX(i2->src_type, i3->src_type), src2, src3)) { in tcg_out_movext3()
629 tcg_out_xchg(s, MAX(i1->src_type, i2->src_type), src1, src2); in tcg_out_movext3()
631 tcg_out_movext1_new_src(s, i1, i1->dst); in tcg_out_movext3()
632 tcg_out_movext1_new_src(s, i2, i2->dst); in tcg_out_movext3()
633 tcg_out_movext1_new_src(s, i3, i3->dst); in tcg_out_movext3()
636 tcg_out_mov(s, i1->src_type, scratch, src1); in tcg_out_movext3()
656 QSIMPLEQ_INSERT_TAIL(&s->ldst_labels, l, next); in new_ldst_label()
680 n->label = label; in new_pool_alloc()
681 n->addend = addend; in new_pool_alloc()
682 n->rtype = rtype; in new_pool_alloc()
683 n->nlong = nlong; in new_pool_alloc()
690 int nlong = n->nlong; in new_pool_insert()
693 for (pp = &s->pool_labels; (i = *pp) != NULL; pp = &i->next) { in new_pool_insert()
694 if (nlong > i->nlong) { in new_pool_insert()
697 if (nlong < i->nlong) { in new_pool_insert()
700 if (memcmp(n->data, i->data, sizeof(tcg_target_ulong) * nlong) >= 0) { in new_pool_insert()
704 n->next = *pp; in new_pool_insert()
714 n->data[0] = d; in new_pool_label()
725 n->data[0] = d0; in new_pool_l2()
726 n->data[1] = d1; in new_pool_l2()
738 n->data[0] = d0; in new_pool_l4()
739 n->data[1] = d1; in new_pool_l4()
740 n->data[2] = d2; in new_pool_l4()
741 n->data[3] = d3; in new_pool_l4()
745 /* For v256, for 32-bit host. */
755 n->data[0] = d0; in new_pool_l8()
756 n->data[1] = d1; in new_pool_l8()
757 n->data[2] = d2; in new_pool_l8()
758 n->data[3] = d3; in new_pool_l8()
759 n->data[4] = d4; in new_pool_l8()
760 n->data[5] = d5; in new_pool_l8()
761 n->data[6] = d6; in new_pool_l8()
762 n->data[7] = d7; in new_pool_l8()
775 QSIMPLEQ_FOREACH(lb, &s->ldst_labels, next) { in tcg_out_ldst_finalize()
776 if (lb->is_ld in tcg_out_ldst_finalize()
779 return -2; in tcg_out_ldst_finalize()
788 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) { in tcg_out_ldst_finalize()
789 return -1; in tcg_out_ldst_finalize()
797 TCGLabelPoolData *p = s->pool_labels; in tcg_out_pool_finalize()
809 a = (void *)ROUND_UP((uintptr_t)s->code_ptr, in tcg_out_pool_finalize()
810 sizeof(tcg_target_ulong) * p->nlong); in tcg_out_pool_finalize()
811 tcg_out_nop_fill(s->code_ptr, (tcg_insn_unit *)a - s->code_ptr); in tcg_out_pool_finalize()
812 s->data_gen_ptr = a; in tcg_out_pool_finalize()
814 for (; p != NULL; p = p->next) { in tcg_out_pool_finalize()
815 size_t size = sizeof(tcg_target_ulong) * p->nlong; in tcg_out_pool_finalize()
818 if (!l || l->nlong != p->nlong || memcmp(l->data, p->data, size)) { in tcg_out_pool_finalize()
819 if (unlikely(a > s->code_gen_highwater)) { in tcg_out_pool_finalize()
820 return -1; in tcg_out_pool_finalize()
822 memcpy(a, p->data, size); in tcg_out_pool_finalize()
827 value = (uintptr_t)tcg_splitwx_to_rx(a) - size; in tcg_out_pool_finalize()
828 if (!patch_reloc(p->label, p->rtype, value, p->addend)) { in tcg_out_pool_finalize()
829 return -2; in tcg_out_pool_finalize()
833 s->code_ptr = a; in tcg_out_pool_finalize()
867 C_Dynamic = -2,
868 C_NotImplemented = -1,
869 #include "tcg-target-con-set.h"
919 #include "tcg-target-con-set.h"
1119 #include "tcg-target.c.inc"
1123 QEMU_BUILD_BUG_ON((int)(offsetof(CPUNegativeOffsetState, tlb.f[0]) -
1130 * We require these functions for slow-path function calls.
1177 * This prevents trivial mistakes within each arch/tcg-target.c.inc.
1271 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation
1274 * In system-mode each caller registers its context in tcg_ctxs[]. Note that in
1275 * system-mode tcg_ctxs[] does not track tcg_ctx_init, since the initial context
1278 * Not tracking tcg_init_ctx in tcg_ctxs[] in system-mode keeps code that
1279 * iterates over the array (e.g. tcg_code_size() the same for both system/user
1298 ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps; in tcg_register_thread()
1300 s->temps[i].mem_base = &s->temps[b]; in tcg_register_thread()
1326 p->size = size; in tcg_malloc_internal()
1327 p->next = s->pool_first_large; in tcg_malloc_internal()
1328 s->pool_first_large = p; in tcg_malloc_internal()
1329 return p->data; in tcg_malloc_internal()
1331 p = s->pool_current; in tcg_malloc_internal()
1333 p = s->pool_first; in tcg_malloc_internal()
1337 if (!p->next) { in tcg_malloc_internal()
1341 p->size = pool_size; in tcg_malloc_internal()
1342 p->next = NULL; in tcg_malloc_internal()
1343 if (s->pool_current) { in tcg_malloc_internal()
1344 s->pool_current->next = p; in tcg_malloc_internal()
1346 s->pool_first = p; in tcg_malloc_internal()
1349 p = p->next; in tcg_malloc_internal()
1353 s->pool_current = p; in tcg_malloc_internal()
1354 s->pool_cur = p->data + size; in tcg_malloc_internal()
1355 s->pool_end = p->data + p->size; in tcg_malloc_internal()
1356 return p->data; in tcg_malloc_internal()
1362 for (p = s->pool_first_large; p; p = t) { in tcg_pool_reset()
1363 t = p->next; in tcg_pool_reset()
1366 s->pool_first_large = NULL; in tcg_pool_reset()
1367 s->pool_cur = s->pool_end = NULL; in tcg_pool_reset()
1368 s->pool_current = NULL; in tcg_pool_reset()
1372 * Create TCGHelperInfo structures for "tcg/tcg-ldst.h" functions,
1373 * akin to what "exec/helper-tcg.h" does with DEF_HELPER_FLAGS_N.
1482 unsigned typemask = info->typemask; in init_ffi_layout()
1490 /* Ignoring the return type, find the last non-zero field. */ in init_ffi_layout()
1491 nargs = 32 - clz32(typemask >> 3); in init_ffi_layout()
1496 ca->cif.rtype = typecode_to_ffi(typemask & 7); in init_ffi_layout()
1497 ca->cif.nargs = nargs; in init_ffi_layout()
1500 ca->cif.arg_types = ca->args; in init_ffi_layout()
1503 ca->args[j] = typecode_to_ffi(typecode); in init_ffi_layout()
1507 status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs, in init_ffi_layout()
1508 ca->cif.rtype, ca->cif.arg_types); in init_ffi_layout()
1511 return &ca->cif; in init_ffi_layout()
1514 #define HELPER_INFO_INIT(I) (&(I)->cif)
1517 #define HELPER_INFO_INIT(I) (&(I)->init)
1534 unsigned stk_slot = arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs); in arg_slot_stk_ofs()
1549 cum->arg_slot += cum->arg_slot & 1; in layout_arg_even()
1555 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx]; in layout_arg_1()
1559 .arg_idx = cum->arg_idx, in layout_arg_1()
1560 .arg_slot = cum->arg_slot, in layout_arg_1()
1562 cum->info_in_idx++; in layout_arg_1()
1563 cum->arg_slot++; in layout_arg_1()
1569 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx]; in layout_arg_normal_n()
1575 .arg_idx = cum->arg_idx, in layout_arg_normal_n()
1577 .arg_slot = cum->arg_slot + i, in layout_arg_normal_n()
1580 cum->info_in_idx += n; in layout_arg_normal_n()
1581 cum->arg_slot += n; in layout_arg_normal_n()
1586 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx]; in layout_arg_by_ref()
1594 * structure pass by-reference. Therefore we must make copies. in layout_arg_by_ref()
1598 loc[0].ref_slot = cum->ref_slot; in layout_arg_by_ref()
1607 .arg_idx = cum->arg_idx, in layout_arg_by_ref()
1609 .ref_slot = cum->ref_slot + i, in layout_arg_by_ref()
1612 cum->info_in_idx += n - 1; /* i=0 accounted for in layout_arg_1 */ in layout_arg_by_ref()
1613 cum->ref_slot += n; in layout_arg_by_ref()
1620 unsigned typemask = info->typemask; in init_call_layout()
1630 info->nr_out = 0; in init_call_layout()
1635 info->nr_out = 1; in init_call_layout()
1636 info->out_kind = TCG_CALL_RET_NORMAL; in init_call_layout()
1640 info->nr_out = 64 / TCG_TARGET_REG_BITS; in init_call_layout()
1641 info->out_kind = TCG_CALL_RET_NORMAL; in init_call_layout()
1643 tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1); in init_call_layout()
1646 info->nr_out = 128 / TCG_TARGET_REG_BITS; in init_call_layout()
1647 info->out_kind = TCG_TARGET_CALL_RET_I128; in init_call_layout()
1651 tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1); in init_call_layout()
1755 info->nr_in = cum.info_in_idx; in init_call_layout()
1758 assert(cum.info_in_idx <= ARRAY_SIZE(info->in)); in init_call_layout()
1765 * which has a signed 8-bit offset encoding. in init_call_layout()
1773 ref_base = cum.arg_slot - max_reg_slots; in init_call_layout()
1782 for (int i = cum.info_in_idx - 1; i >= 0; --i) { in init_call_layout()
1783 TCGCallArgumentLoc *loc = &info->in[i]; in init_call_layout()
1784 switch (loc->kind) { in init_call_layout()
1787 loc->ref_slot += ref_base; in init_call_layout()
1809 s->nb_globals = 0; in tcg_context_init()
1830 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i]; in tcg_context_init()
1838 * In user-mode we simply share the init context among threads, since we in tcg_context_init()
1841 * In system-mode we will have at most max_threads TCG threads. in tcg_context_init()
1852 tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0)); in tcg_context_init()
1874 tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align); in tcg_tb_alloc()
1877 if (unlikely(next > s->code_gen_highwater)) { in tcg_tb_alloc()
1883 qatomic_set(&s->code_gen_ptr, next); in tcg_tb_alloc()
1892 s->code_ptr = s->code_gen_ptr; in tcg_prologue_init()
1893 s->code_buf = s->code_gen_ptr; in tcg_prologue_init()
1894 s->data_gen_ptr = NULL; in tcg_prologue_init()
1897 tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr); in tcg_prologue_init()
1900 s->pool_labels = NULL; in tcg_prologue_init()
1913 perf_report_prologue(s->code_gen_ptr, prologue_size); in tcg_prologue_init()
1916 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf), in tcg_prologue_init()
1917 (uintptr_t)s->code_buf, prologue_size); in tcg_prologue_init()
1924 if (s->data_gen_ptr) { in tcg_prologue_init()
1925 size_t code_size = s->data_gen_ptr - s->code_gen_ptr; in tcg_prologue_init()
1926 size_t data_size = prologue_size - code_size; in tcg_prologue_init()
1929 disas(logfile, s->code_gen_ptr, code_size); in tcg_prologue_init()
1935 (uintptr_t)s->data_gen_ptr + i, in tcg_prologue_init()
1936 *(uint64_t *)(s->data_gen_ptr + i)); in tcg_prologue_init()
1940 (uintptr_t)s->data_gen_ptr + i, in tcg_prologue_init()
1941 *(uint32_t *)(s->data_gen_ptr + i)); in tcg_prologue_init()
1945 disas(logfile, s->code_gen_ptr, prologue_size); in tcg_prologue_init()
1967 s->nb_temps = s->nb_globals; in tcg_func_start()
1974 if (s->const_table[i]) { in tcg_func_start()
1975 g_hash_table_remove_all(s->const_table[i]); in tcg_func_start()
1979 s->nb_ops = 0; in tcg_func_start()
1980 s->nb_labels = 0; in tcg_func_start()
1981 s->current_frame_offset = s->frame_start; in tcg_func_start()
1984 s->goto_tb_issue_mask = 0; in tcg_func_start()
1987 QTAILQ_INIT(&s->ops); in tcg_func_start()
1988 QTAILQ_INIT(&s->free_ops); in tcg_func_start()
1989 s->emit_before_op = NULL; in tcg_func_start()
1990 QSIMPLEQ_INIT(&s->labels); in tcg_func_start()
1992 tcg_debug_assert(s->addr_type <= TCG_TYPE_REG); in tcg_func_start()
1997 int n = s->nb_temps++; in tcg_temp_alloc()
2002 return memset(&s->temps[n], 0, sizeof(TCGTemp)); in tcg_temp_alloc()
2009 tcg_debug_assert(s->nb_globals == s->nb_temps); in tcg_global_alloc()
2010 tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS); in tcg_global_alloc()
2011 s->nb_globals++; in tcg_global_alloc()
2013 ts->kind = TEMP_GLOBAL; in tcg_global_alloc()
2026 ts->base_type = type; in tcg_global_reg_new_internal()
2027 ts->type = type; in tcg_global_reg_new_internal()
2028 ts->kind = TEMP_FIXED; in tcg_global_reg_new_internal()
2029 ts->reg = reg; in tcg_global_reg_new_internal()
2030 ts->name = name; in tcg_global_reg_new_internal()
2031 tcg_regset_set_reg(s->reserved_regs, reg); in tcg_global_reg_new_internal()
2038 s->frame_start = start; in tcg_set_frame()
2039 s->frame_end = start + size; in tcg_set_frame()
2040 s->frame_temp in tcg_set_frame()
2052 switch (base_ts->kind) { in tcg_global_mem_new_internal()
2056 /* We do not support double-indirect registers. */ in tcg_global_mem_new_internal()
2057 tcg_debug_assert(!base_ts->indirect_reg); in tcg_global_mem_new_internal()
2058 base_ts->indirect_base = 1; in tcg_global_mem_new_internal()
2059 s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64 in tcg_global_mem_new_internal()
2071 ts->base_type = TCG_TYPE_I64; in tcg_global_mem_new_internal()
2072 ts->type = TCG_TYPE_I32; in tcg_global_mem_new_internal()
2073 ts->indirect_reg = indirect_reg; in tcg_global_mem_new_internal()
2074 ts->mem_allocated = 1; in tcg_global_mem_new_internal()
2075 ts->mem_base = base_ts; in tcg_global_mem_new_internal()
2076 ts->mem_offset = offset; in tcg_global_mem_new_internal()
2079 ts->name = strdup(buf); in tcg_global_mem_new_internal()
2082 ts2->base_type = TCG_TYPE_I64; in tcg_global_mem_new_internal()
2083 ts2->type = TCG_TYPE_I32; in tcg_global_mem_new_internal()
2084 ts2->indirect_reg = indirect_reg; in tcg_global_mem_new_internal()
2085 ts2->mem_allocated = 1; in tcg_global_mem_new_internal()
2086 ts2->mem_base = base_ts; in tcg_global_mem_new_internal()
2087 ts2->mem_offset = offset + 4; in tcg_global_mem_new_internal()
2088 ts2->temp_subindex = 1; in tcg_global_mem_new_internal()
2091 ts2->name = strdup(buf); in tcg_global_mem_new_internal()
2093 ts->base_type = type; in tcg_global_mem_new_internal()
2094 ts->type = type; in tcg_global_mem_new_internal()
2095 ts->indirect_reg = indirect_reg; in tcg_global_mem_new_internal()
2096 ts->mem_allocated = 1; in tcg_global_mem_new_internal()
2097 ts->mem_base = base_ts; in tcg_global_mem_new_internal()
2098 ts->mem_offset = offset; in tcg_global_mem_new_internal()
2099 ts->name = name; in tcg_global_mem_new_internal()
2129 int idx = find_first_bit(s->free_temps[type].l, TCG_MAX_TEMPS); in tcg_temp_new_internal()
2133 clear_bit(idx, s->free_temps[type].l); in tcg_temp_new_internal()
2135 ts = &s->temps[idx]; in tcg_temp_new_internal()
2136 ts->temp_allocated = 1; in tcg_temp_new_internal()
2137 tcg_debug_assert(ts->base_type == type); in tcg_temp_new_internal()
2138 tcg_debug_assert(ts->kind == kind); in tcg_temp_new_internal()
2163 ts->base_type = type; in tcg_temp_new_internal()
2164 ts->temp_allocated = 1; in tcg_temp_new_internal()
2165 ts->kind = kind; in tcg_temp_new_internal()
2168 ts->type = type; in tcg_temp_new_internal()
2170 ts->type = TCG_TYPE_REG; in tcg_temp_new_internal()
2176 ts2->base_type = type; in tcg_temp_new_internal()
2177 ts2->type = TCG_TYPE_REG; in tcg_temp_new_internal()
2178 ts2->temp_allocated = 1; in tcg_temp_new_internal()
2179 ts2->temp_subindex = i; in tcg_temp_new_internal()
2180 ts2->kind = kind; in tcg_temp_new_internal()
2255 tcg_debug_assert(t->temp_allocated != 0); in tcg_temp_new_vec_matching()
2257 t = tcg_temp_new_internal(t->base_type, TEMP_EBB); in tcg_temp_new_vec_matching()
2265 switch (ts->kind) { in tcg_temp_free_internal()
2271 tcg_debug_assert(ts->temp_allocated != 0); in tcg_temp_free_internal()
2272 ts->temp_allocated = 0; in tcg_temp_free_internal()
2273 set_bit(temp_idx(ts), s->free_temps[ts->base_type].l); in tcg_temp_free_internal()
2309 GHashTable *h = s->const_table[type]; in tcg_constant_internal()
2314 s->const_table[type] = h; in tcg_constant_internal()
2328 ts->base_type = TCG_TYPE_I64; in tcg_constant_internal()
2329 ts->type = TCG_TYPE_I32; in tcg_constant_internal()
2330 ts->kind = TEMP_CONST; in tcg_constant_internal()
2331 ts->temp_allocated = 1; in tcg_constant_internal()
2333 ts2->base_type = TCG_TYPE_I64; in tcg_constant_internal()
2334 ts2->type = TCG_TYPE_I32; in tcg_constant_internal()
2335 ts2->kind = TEMP_CONST; in tcg_constant_internal()
2336 ts2->temp_allocated = 1; in tcg_constant_internal()
2337 ts2->temp_subindex = 1; in tcg_constant_internal()
2340 * Retain the full value of the 64-bit constant in the low in tcg_constant_internal()
2348 ts->base_type = type; in tcg_constant_internal()
2349 ts->type = type; in tcg_constant_internal()
2350 ts->kind = TEMP_CONST; in tcg_constant_internal()
2351 ts->temp_allocated = 1; in tcg_constant_internal()
2352 ts->val = val; in tcg_constant_internal()
2353 val_ptr = &ts->val; in tcg_constant_internal()
2391 tcg_debug_assert(t->temp_allocated != 0); in tcg_constant_vec_matching()
2392 return tcg_constant_vec(t->base_type, vece, val); in tcg_constant_vec_matching()
2398 ptrdiff_t n = ts - tcg_ctx->temps; in temp_idx()
2399 assert(n >= 0 && n < tcg_ctx->nb_temps); in temp_idx()
2405 uintptr_t o = (uintptr_t)v - offsetof(TCGContext, temps); in tcgv_i32_temp()
2407 assert(o < sizeof(TCGTemp) * tcg_ctx->nb_temps); in tcgv_i32_temp()
2583 con_set = outop->static_constraint; in tcg_op_supported()
2585 con_set = outop->dynamic_constraint(type, flags); in tcg_op_supported()
2610 tcg_debug_assert(len <= width - ofs); in tcg_op_deposit_valid()
2630 total_args = info->nr_out + info->nr_in + 2; in tcg_gen_callN()
2635 if (tcg_ctx->plugin_insn && !(info->flags & TCG_CALL_NO_SIDE_EFFECTS)) { in tcg_gen_callN()
2636 tcg_ctx->plugin_insn->calls_helpers = true; in tcg_gen_callN()
2640 TCGOP_CALLO(op) = n = info->nr_out; in tcg_gen_callN()
2647 op->args[pi++] = temp_arg(ret); in tcg_gen_callN()
2652 tcg_debug_assert(ret->base_type == ret->type + ctz32(n)); in tcg_gen_callN()
2653 tcg_debug_assert(ret->temp_subindex == 0); in tcg_gen_callN()
2655 op->args[pi++] = temp_arg(ret + i); in tcg_gen_callN()
2662 TCGOP_CALLI(op) = n = info->nr_in; in tcg_gen_callN()
2664 const TCGCallArgumentLoc *loc = &info->in[i]; in tcg_gen_callN()
2665 TCGTemp *ts = args[loc->arg_idx] + loc->tmp_subindex; in tcg_gen_callN()
2667 switch (loc->kind) { in tcg_gen_callN()
2671 op->args[pi++] = temp_arg(ts); in tcg_gen_callN()
2680 if (loc->kind == TCG_CALL_ARG_EXTEND_S) { in tcg_gen_callN()
2685 op->args[pi++] = tcgv_i64_arg(temp); in tcg_gen_callN()
2694 op->args[pi++] = (uintptr_t)func; in tcg_gen_callN()
2695 op->args[pi++] = (uintptr_t)info; in tcg_gen_callN()
2698 if (tcg_ctx->emit_before_op) { in tcg_gen_callN()
2699 QTAILQ_INSERT_BEFORE(tcg_ctx->emit_before_op, op, link); in tcg_gen_callN()
2701 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link); in tcg_gen_callN()
2768 for (i = 0, n = s->nb_temps; i < n; i++) { in tcg_reg_alloc_start()
2769 TCGTemp *ts = &s->temps[i]; in tcg_reg_alloc_start()
2772 switch (ts->kind) { in tcg_reg_alloc_start()
2785 ts->mem_allocated = 0; in tcg_reg_alloc_start()
2790 ts->val_type = val; in tcg_reg_alloc_start()
2793 memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp)); in tcg_reg_alloc_start()
2801 switch (ts->kind) { in tcg_get_arg_str_ptr()
2804 pstrcpy(buf, buf_size, ts->name); in tcg_get_arg_str_ptr()
2807 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals); in tcg_get_arg_str_ptr()
2810 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals); in tcg_get_arg_str_ptr()
2813 switch (ts->type) { in tcg_get_arg_str_ptr()
2815 snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val); in tcg_get_arg_str_ptr()
2819 snprintf(buf, buf_size, "$0x%" PRIx64, ts->val); in tcg_get_arg_str_ptr()
2826 64 << (ts->type - TCG_TYPE_V64), ts->val); in tcg_get_arg_str_ptr()
2908 "from-tb",
2909 "from-insn",
2910 "after-insn",
2911 "after-tb",
2917 return (d & (d - 1)) == 0; in tcg_regset_single()
2929 /* Return only the number of characters output -- no error return. */
2938 QTAILQ_FOREACH(op, &s->ops, link) { in tcg_dump_ops()
2944 c = op->opc; in tcg_dump_ops()
2949 col += ne_fprintf(f, "\n ----"); in tcg_dump_ops()
2962 nb_cargs = def->nb_cargs; in tcg_dump_ops()
2964 col += ne_fprintf(f, " %s ", def->name); in tcg_dump_ops()
2971 if (func == info->func) { in tcg_dump_ops()
2972 col += ne_fprintf(f, "%s", info->name); in tcg_dump_ops()
2977 col += ne_fprintf(f, ",$0x%x,$%d", info->flags, nb_oargs); in tcg_dump_ops()
2980 op->args[i])); in tcg_dump_ops()
2983 TCGArg arg = op->args[nb_oargs + i]; in tcg_dump_ops()
2988 if (def->flags & TCG_OPF_INT) { in tcg_dump_ops()
2990 def->name, in tcg_dump_ops()
2992 } else if (def->flags & TCG_OPF_VECTOR) { in tcg_dump_ops()
2994 def->name, in tcg_dump_ops()
2998 col += ne_fprintf(f, " %s ", def->name); in tcg_dump_ops()
3001 nb_oargs = def->nb_oargs; in tcg_dump_ops()
3002 nb_iargs = def->nb_iargs; in tcg_dump_ops()
3003 nb_cargs = def->nb_cargs; in tcg_dump_ops()
3010 op->args[k++])); in tcg_dump_ops()
3016 op->args[k++])); in tcg_dump_ops()
3027 if (op->args[k] < ARRAY_SIZE(cond_name) in tcg_dump_ops()
3028 && cond_name[op->args[k]]) { in tcg_dump_ops()
3029 col += ne_fprintf(f, ",%s", cond_name[op->args[k++]]); in tcg_dump_ops()
3031 col += ne_fprintf(f, ",$0x%" TCG_PRIlx, op->args[k++]); in tcg_dump_ops()
3041 MemOpIdx oi = op->args[k++]; in tcg_dump_ops()
3065 TCGArg flags = op->args[k]; in tcg_dump_ops()
3082 TCGArg from = op->args[k++]; in tcg_dump_ops()
3107 arg_label(op->args[k])->id); in tcg_dump_ops()
3112 TCGBar membar = op->args[k]; in tcg_dump_ops()
3194 op->args[k]); in tcg_dump_ops()
3198 if (have_prefs || op->life) { in tcg_dump_ops()
3204 if (op->life) { in tcg_dump_ops()
3205 unsigned life = op->life; in tcg_dump_ops()
3262 n = ctpop64(arg_ct->regs); in get_constraint_priority()
3268 if (n == 1 || arg_ct->oalias) { in get_constraint_priority()
3277 switch (arg_ct->pair) { in get_constraint_priority()
3282 return (arg_ct->pair_index + 1) * 2 - 1; in get_constraint_priority()
3287 return -n; in get_constraint_priority()
3301 for (i = 0; i < n - 1; i++) { in sort_constraints()
3322 int nb_oargs = tdefs->nb_oargs; in process_constraint_sets()
3323 int nb_iargs = tdefs->nb_iargs; in process_constraint_sets()
3328 const char *ct_str = tdefs->args_ct_str[i]; in process_constraint_sets()
3334 o = *ct_str - '0'; in process_constraint_sets()
3361 o = i - 1; in process_constraint_sets()
3378 o = i - 1; in process_constraint_sets()
3404 /* Include all of the target-specific constraints. */ in process_constraint_sets()
3412 #include "tcg-target-con-str.h" in process_constraint_sets()
3451 * Since [0-9pm] must be alone in the constraint string, in process_constraint_sets()
3510 TCGOpcode opc = op->opc; in opcode_args_ct()
3517 if (def->flags & TCG_OPF_NOT_PRESENT) { in opcode_args_ct()
3522 con_set = outop->static_constraint; in opcode_args_ct()
3524 con_set = outop->dynamic_constraint(type, flags); in opcode_args_ct()
3533 tcg_debug_assert(constraint_sets[con_set].nb_oargs == def->nb_oargs); in opcode_args_ct()
3534 tcg_debug_assert(constraint_sets[con_set].nb_iargs == def->nb_iargs); in opcode_args_ct()
3541 TCGLabel *label = arg_label(op->args[idx]); in remove_label_use()
3544 QSIMPLEQ_FOREACH(use, &label->branches, next) { in remove_label_use()
3545 if (use->op == op) { in remove_label_use()
3546 QSIMPLEQ_REMOVE(&label->branches, use, TCGLabelUse, next); in remove_label_use()
3555 switch (op->opc) { in tcg_op_remove()
3569 QTAILQ_REMOVE(&s->ops, op, link); in tcg_op_remove()
3570 QTAILQ_INSERT_TAIL(&s->free_ops, op, link); in tcg_op_remove()
3571 s->nb_ops--; in tcg_op_remove()
3592 if (unlikely(!QTAILQ_EMPTY(&s->free_ops))) { in tcg_op_alloc()
3593 QTAILQ_FOREACH(op, &s->free_ops, link) { in tcg_op_alloc()
3594 if (nargs <= op->nargs) { in tcg_op_alloc()
3595 QTAILQ_REMOVE(&s->free_ops, op, link); in tcg_op_alloc()
3596 nargs = op->nargs; in tcg_op_alloc()
3608 op->opc = opc; in tcg_op_alloc()
3609 op->nargs = nargs; in tcg_op_alloc()
3612 tcg_debug_assert(op->nargs == nargs); in tcg_op_alloc()
3614 s->nb_ops++; in tcg_op_alloc()
3622 if (tcg_ctx->emit_before_op) { in tcg_emit_op()
3623 QTAILQ_INSERT_BEFORE(tcg_ctx->emit_before_op, op, link); in tcg_emit_op()
3625 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link); in tcg_emit_op()
3646 QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link); in tcg_op_insert_after()
3654 QSIMPLEQ_FOREACH(u, &from->branches, next) { in move_label_uses()
3655 TCGOp *op = u->op; in move_label_uses()
3656 switch (op->opc) { in move_label_uses()
3658 op->args[0] = label_arg(to); in move_label_uses()
3661 op->args[3] = label_arg(to); in move_label_uses()
3664 op->args[5] = label_arg(to); in move_label_uses()
3671 QSIMPLEQ_CONCAT(&to->branches, &from->branches); in move_label_uses()
3681 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { in reachable_code_pass()
3685 switch (op->opc) { in reachable_code_pass()
3687 label = arg_label(op->args[0]); in reachable_code_pass()
3701 if (op_prev->opc == INDEX_op_set_label) { in reachable_code_pass()
3702 move_label_uses(label, arg_label(op_prev->args[0])); in reachable_code_pass()
3714 if (op_prev->opc == INDEX_op_br && in reachable_code_pass()
3715 label == arg_label(op_prev->args[0])) { in reachable_code_pass()
3721 if (QSIMPLEQ_EMPTY(&label->branches)) { in reachable_code_pass()
3752 /* Never remove -- we need to keep these for unwind. */ in reachable_code_pass()
3775 return ts->state_ptr; in la_temp_pref()
3784 = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]); in la_reset_pref()
3794 s->temps[i].state = TS_DEAD | TS_MEM; in la_func_end()
3795 la_reset_pref(&s->temps[i]); in la_func_end()
3798 s->temps[i].state = TS_DEAD; in la_func_end()
3799 la_reset_pref(&s->temps[i]); in la_func_end()
3810 TCGTemp *ts = &s->temps[i]; in la_bb_end()
3813 switch (ts->kind) { in la_bb_end()
3826 ts->state = state; in la_bb_end()
3837 int state = s->temps[i].state; in la_global_sync()
3838 s->temps[i].state = state | TS_MEM; in la_global_sync()
3841 la_reset_pref(&s->temps[i]); in la_global_sync()
3848 * explicitly live-across-conditional-branch, globals and local temps
3856 TCGTemp *ts = &s->temps[i]; in la_bb_sync()
3859 switch (ts->kind) { in la_bb_sync()
3861 state = ts->state; in la_bb_sync()
3862 ts->state = state | TS_MEM; in la_bb_sync()
3873 la_reset_pref(&s->temps[i]); in la_bb_sync()
3883 s->temps[i].state = TS_DEAD | TS_MEM; in la_global_kill()
3884 la_reset_pref(&s->temps[i]); in la_global_kill()
3895 TCGTemp *ts = &s->temps[i]; in la_cross_call()
3896 if (!(ts->state & TS_DEAD)) { in la_cross_call()
3903 set = tcg_target_available_regs[ts->type] & mask; in la_cross_call()
3917 void * const multiple_ebb = (void *)(uintptr_t)-1; in liveness_pass_0()
3918 int nb_temps = s->nb_temps; in liveness_pass_0()
3921 for (int i = s->nb_globals; i < nb_temps; ++i) { in liveness_pass_0()
3922 s->temps[i].state_ptr = NULL; in liveness_pass_0()
3931 ebb = QTAILQ_FIRST(&s->ops); in liveness_pass_0()
3932 QTAILQ_FOREACH(op, &s->ops, link) { in liveness_pass_0()
3936 switch (op->opc) { in liveness_pass_0()
3947 def = &tcg_op_defs[op->opc]; in liveness_pass_0()
3948 nb_oargs = def->nb_oargs; in liveness_pass_0()
3949 nb_iargs = def->nb_iargs; in liveness_pass_0()
3954 TCGTemp *ts = arg_temp(op->args[i]); in liveness_pass_0()
3956 if (ts->kind != TEMP_TB) { in liveness_pass_0()
3959 if (ts->state_ptr == NULL) { in liveness_pass_0()
3960 ts->state_ptr = ebb; in liveness_pass_0()
3961 } else if (ts->state_ptr != ebb) { in liveness_pass_0()
3962 ts->state_ptr = multiple_ebb; in liveness_pass_0()
3971 for (int i = s->nb_globals; i < nb_temps; ++i) { in liveness_pass_0()
3972 TCGTemp *ts = &s->temps[i]; in liveness_pass_0()
3973 if (ts->kind == TEMP_TB && ts->state_ptr != multiple_ebb) { in liveness_pass_0()
3974 ts->kind = TEMP_EBB; in liveness_pass_0()
3984 * carry-out operations should always be paired with carry-in. in assert_carry_dead()
3987 tcg_debug_assert(!s->carry_live); in assert_carry_dead()
3996 int nb_globals = s->nb_globals; in liveness_pass_1()
3997 int nb_temps = s->nb_temps; in liveness_pass_1()
4003 s->temps[i].state_ptr = prefs + i; in liveness_pass_1()
4009 s->carry_live = false; in liveness_pass_1()
4010 QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) { in liveness_pass_1()
4015 TCGOpcode opc = op->opc; in liveness_pass_1()
4032 ts = arg_temp(op->args[i]); in liveness_pass_1()
4033 if (ts->state != TS_DEAD) { in liveness_pass_1()
4043 ts = arg_temp(op->args[i]); in liveness_pass_1()
4044 if (ts->state & TS_DEAD) { in liveness_pass_1()
4047 if (ts->state & TS_MEM) { in liveness_pass_1()
4050 ts->state = TS_DEAD; in liveness_pass_1()
4054 /* Not used -- it will be tcg_target_call_oarg_reg(). */ in liveness_pass_1()
4055 memset(op->output_pref, 0, sizeof(op->output_pref)); in liveness_pass_1()
4066 ts = arg_temp(op->args[i]); in liveness_pass_1()
4067 if (ts->state & TS_DEAD) { in liveness_pass_1()
4072 /* For all live registers, remove call-clobbered prefs. */ in liveness_pass_1()
4085 for (int i = nb_iargs - 1; i >= 0; i--) { in liveness_pass_1()
4086 const TCGCallArgumentLoc *loc = &info->in[i]; in liveness_pass_1()
4087 ts = arg_temp(op->args[nb_oargs + i]); in liveness_pass_1()
4089 if (ts->state & TS_DEAD) { in liveness_pass_1()
4090 switch (loc->kind) { in liveness_pass_1()
4094 if (arg_slot_reg_p(loc->arg_slot)) { in liveness_pass_1()
4101 tcg_target_available_regs[ts->type]; in liveness_pass_1()
4104 ts->state &= ~TS_DEAD; in liveness_pass_1()
4114 const TCGCallArgumentLoc *loc = &info->in[i]; in liveness_pass_1()
4115 ts = arg_temp(op->args[nb_oargs + i]); in liveness_pass_1()
4117 switch (loc->kind) { in liveness_pass_1()
4121 if (arg_slot_reg_p(loc->arg_slot)) { in liveness_pass_1()
4123 tcg_target_call_iarg_regs[loc->arg_slot]); in liveness_pass_1()
4137 ts = arg_temp(op->args[0]); in liveness_pass_1()
4138 ts->state = TS_DEAD; in liveness_pass_1()
4151 if (arg_temp(op->args[1])->state == TS_DEAD) { in liveness_pass_1()
4152 if (arg_temp(op->args[0])->state == TS_DEAD) { in liveness_pass_1()
4157 op->opc = opc = opc_new; in liveness_pass_1()
4158 op->args[1] = op->args[2]; in liveness_pass_1()
4159 op->args[2] = op->args[3]; in liveness_pass_1()
4160 } else if (arg_temp(op->args[0])->state == TS_DEAD && in liveness_pass_1()
4163 op->opc = opc = opc_new2; in liveness_pass_1()
4164 op->args[0] = op->args[1]; in liveness_pass_1()
4165 op->args[1] = op->args[2]; in liveness_pass_1()
4166 op->args[2] = op->args[3]; in liveness_pass_1()
4170 /* Mark the single-word operation live. */ in liveness_pass_1()
4174 if (s->carry_live) { in liveness_pass_1()
4177 op->opc = opc = INDEX_op_add; in liveness_pass_1()
4181 if (s->carry_live) { in liveness_pass_1()
4184 op->opc = opc = INDEX_op_addci; in liveness_pass_1()
4188 if (s->carry_live) { in liveness_pass_1()
4192 op->opc = opc = INDEX_op_sub; in liveness_pass_1()
4193 ts = arg_temp(op->args[2]); in liveness_pass_1()
4194 if (ts->kind == TEMP_CONST) { in liveness_pass_1()
4195 ts = tcg_constant_internal(ts->type, -ts->val); in liveness_pass_1()
4196 if (ts->state_ptr == NULL) { in liveness_pass_1()
4199 ts->state_ptr = tcg_malloc(sizeof(TCGRegSet)); in liveness_pass_1()
4200 ts->state = TS_DEAD; in liveness_pass_1()
4203 op->args[2] = temp_arg(ts); in liveness_pass_1()
4204 op->opc = opc = INDEX_op_add; in liveness_pass_1()
4209 if (s->carry_live) { in liveness_pass_1()
4212 op->opc = opc = INDEX_op_subbi; in liveness_pass_1()
4216 if (s->carry_live) { in liveness_pass_1()
4222 op_prev->args[0] = op->args[0]; in liveness_pass_1()
4223 op_prev->args[1] = op->args[1]; in liveness_pass_1()
4224 op_prev->args[2] = op->args[2]; in liveness_pass_1()
4225 op->opc = opc = INDEX_op_add; in liveness_pass_1()
4226 op->args[1] = op->args[0]; in liveness_pass_1()
4227 ts = arg_temp(op->args[0]); in liveness_pass_1()
4228 ts = tcg_constant_internal(ts->type, 1); in liveness_pass_1()
4229 op->args[2] = temp_arg(ts); in liveness_pass_1()
4233 if (s->carry_live) { in liveness_pass_1()
4236 /* Lower to sub, add -1. */ in liveness_pass_1()
4239 op_prev->args[0] = op->args[0]; in liveness_pass_1()
4240 op_prev->args[1] = op->args[1]; in liveness_pass_1()
4241 op_prev->args[2] = op->args[2]; in liveness_pass_1()
4242 op->opc = opc = INDEX_op_add; in liveness_pass_1()
4243 op->args[1] = op->args[0]; in liveness_pass_1()
4244 ts = arg_temp(op->args[0]); in liveness_pass_1()
4245 ts = tcg_constant_internal(ts->type, -1); in liveness_pass_1()
4246 op->args[2] = temp_arg(ts); in liveness_pass_1()
4257 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && def->nb_oargs != 0) { in liveness_pass_1()
4258 for (int i = def->nb_oargs - 1; i >= 0; i--) { in liveness_pass_1()
4259 if (arg_temp(op->args[i])->state != TS_DEAD) { in liveness_pass_1()
4273 nb_iargs = def->nb_iargs; in liveness_pass_1()
4274 nb_oargs = def->nb_oargs; in liveness_pass_1()
4277 ts = arg_temp(op->args[i]); in liveness_pass_1()
4280 if (i < ARRAY_SIZE(op->output_pref)) { in liveness_pass_1()
4281 op->output_pref[i] = *la_temp_pref(ts); in liveness_pass_1()
4285 if (ts->state & TS_DEAD) { in liveness_pass_1()
4288 if (ts->state & TS_MEM) { in liveness_pass_1()
4291 ts->state = TS_DEAD; in liveness_pass_1()
4296 if (def->flags & TCG_OPF_BB_EXIT) { in liveness_pass_1()
4299 } else if (def->flags & TCG_OPF_COND_BRANCH) { in liveness_pass_1()
4302 } else if (def->flags & TCG_OPF_BB_END) { in liveness_pass_1()
4305 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { in liveness_pass_1()
4308 if (def->flags & TCG_OPF_CALL_CLOBBER) { in liveness_pass_1()
4315 ts = arg_temp(op->args[i]); in liveness_pass_1()
4316 if (ts->state & TS_DEAD) { in liveness_pass_1()
4320 if (def->flags & TCG_OPF_CARRY_OUT) { in liveness_pass_1()
4321 s->carry_live = false; in liveness_pass_1()
4326 ts = arg_temp(op->args[i]); in liveness_pass_1()
4327 if (ts->state & TS_DEAD) { in liveness_pass_1()
4330 *la_temp_pref(ts) = tcg_target_available_regs[ts->type]; in liveness_pass_1()
4331 ts->state &= ~TS_DEAD; in liveness_pass_1()
4334 if (def->flags & TCG_OPF_CARRY_IN) { in liveness_pass_1()
4335 s->carry_live = true; in liveness_pass_1()
4345 *la_temp_pref(arg_temp(op->args[0])) in liveness_pass_1()
4346 = *la_temp_pref(arg_temp(op->args[1])); in liveness_pass_1()
4356 ts = arg_temp(op->args[i]); in liveness_pass_1()
4360 set &= ct->regs; in liveness_pass_1()
4361 if (ct->ialias) { in liveness_pass_1()
4362 set &= output_pref(op, ct->alias_index); in liveness_pass_1()
4366 set = ct->regs; in liveness_pass_1()
4374 op->life = arg_life; in liveness_pass_1()
4383 int nb_globals = s->nb_globals; in liveness_pass_2()
4390 TCGTemp *its = &s->temps[i]; in liveness_pass_2()
4391 if (its->indirect_reg) { in liveness_pass_2()
4393 dts->type = its->type; in liveness_pass_2()
4394 dts->base_type = its->base_type; in liveness_pass_2()
4395 dts->temp_subindex = its->temp_subindex; in liveness_pass_2()
4396 dts->kind = TEMP_EBB; in liveness_pass_2()
4397 its->state_ptr = dts; in liveness_pass_2()
4399 its->state_ptr = NULL; in liveness_pass_2()
4402 its->state = TS_DEAD; in liveness_pass_2()
4404 for (nb_temps = s->nb_temps; i < nb_temps; ++i) { in liveness_pass_2()
4405 TCGTemp *its = &s->temps[i]; in liveness_pass_2()
4406 its->state_ptr = NULL; in liveness_pass_2()
4407 its->state = TS_DEAD; in liveness_pass_2()
4410 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { in liveness_pass_2()
4411 TCGOpcode opc = op->opc; in liveness_pass_2()
4413 TCGLifeData arg_life = op->life; in liveness_pass_2()
4422 nb_iargs = def->nb_iargs; in liveness_pass_2()
4423 nb_oargs = def->nb_oargs; in liveness_pass_2()
4426 if (def->flags & TCG_OPF_COND_BRANCH) { in liveness_pass_2()
4429 } else if (def->flags & TCG_OPF_BB_END) { in liveness_pass_2()
4432 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { in liveness_pass_2()
4444 arg_ts = arg_temp(op->args[i]); in liveness_pass_2()
4445 dir_ts = arg_ts->state_ptr; in liveness_pass_2()
4446 if (dir_ts && arg_ts->state == TS_DEAD) { in liveness_pass_2()
4448 arg_ts->type, 3); in liveness_pass_2()
4450 lop->args[0] = temp_arg(dir_ts); in liveness_pass_2()
4451 lop->args[1] = temp_arg(arg_ts->mem_base); in liveness_pass_2()
4452 lop->args[2] = arg_ts->mem_offset; in liveness_pass_2()
4455 arg_ts->state = TS_MEM; in liveness_pass_2()
4463 arg_ts = arg_temp(op->args[i]); in liveness_pass_2()
4464 dir_ts = arg_ts->state_ptr; in liveness_pass_2()
4466 op->args[i] = temp_arg(dir_ts); in liveness_pass_2()
4469 arg_ts->state = TS_DEAD; in liveness_pass_2()
4482 arg_ts = &s->temps[i]; in liveness_pass_2()
4483 tcg_debug_assert(arg_ts->state_ptr == 0 in liveness_pass_2()
4484 || arg_ts->state != 0); in liveness_pass_2()
4490 arg_ts = &s->temps[i]; in liveness_pass_2()
4491 tcg_debug_assert(arg_ts->state_ptr == 0 in liveness_pass_2()
4492 || arg_ts->state == TS_DEAD); in liveness_pass_2()
4498 arg_ts = arg_temp(op->args[0]); in liveness_pass_2()
4499 dir_ts = arg_ts->state_ptr; in liveness_pass_2()
4501 op->args[0] = temp_arg(dir_ts); in liveness_pass_2()
4505 arg_ts->state = 0; in liveness_pass_2()
4509 arg_ts->type, 3); in liveness_pass_2()
4513 out_ts = arg_temp(op->args[1]); in liveness_pass_2()
4514 arg_ts->state = TS_DEAD; in liveness_pass_2()
4517 arg_ts->state = TS_MEM; in liveness_pass_2()
4520 sop->args[0] = temp_arg(out_ts); in liveness_pass_2()
4521 sop->args[1] = temp_arg(arg_ts->mem_base); in liveness_pass_2()
4522 sop->args[2] = arg_ts->mem_offset; in liveness_pass_2()
4529 arg_ts = arg_temp(op->args[i]); in liveness_pass_2()
4530 dir_ts = arg_ts->state_ptr; in liveness_pass_2()
4534 op->args[i] = temp_arg(dir_ts); in liveness_pass_2()
4538 arg_ts->state = 0; in liveness_pass_2()
4543 arg_ts->type, 3); in liveness_pass_2()
4545 sop->args[0] = temp_arg(dir_ts); in liveness_pass_2()
4546 sop->args[1] = temp_arg(arg_ts->mem_base); in liveness_pass_2()
4547 sop->args[2] = arg_ts->mem_offset; in liveness_pass_2()
4549 arg_ts->state = TS_MEM; in liveness_pass_2()
4553 arg_ts->state = TS_DEAD; in liveness_pass_2()
4568 size = tcg_type_size(ts->base_type); in temp_allocate_frame()
4569 switch (ts->base_type) { in temp_allocate_frame()
4598 off = ROUND_UP(s->current_frame_offset, align); in temp_allocate_frame()
4601 if (off + size > s->frame_end) { in temp_allocate_frame()
4604 s->current_frame_offset = off + size; in temp_allocate_frame()
4610 if (ts->base_type != ts->type) { in temp_allocate_frame()
4611 int part_size = tcg_type_size(ts->type); in temp_allocate_frame()
4618 ts -= ts->temp_subindex; in temp_allocate_frame()
4621 ts[i].mem_base = s->frame_temp; in temp_allocate_frame()
4625 ts->mem_offset = off; in temp_allocate_frame()
4626 ts->mem_base = s->frame_temp; in temp_allocate_frame()
4627 ts->mem_allocated = 1; in temp_allocate_frame()
4634 if (ts->val_type == TEMP_VAL_REG) { in set_temp_val_reg()
4635 TCGReg old = ts->reg; in set_temp_val_reg()
4636 tcg_debug_assert(s->reg_to_temp[old] == ts); in set_temp_val_reg()
4640 s->reg_to_temp[old] = NULL; in set_temp_val_reg()
4642 tcg_debug_assert(s->reg_to_temp[reg] == NULL); in set_temp_val_reg()
4643 s->reg_to_temp[reg] = ts; in set_temp_val_reg()
4644 ts->val_type = TEMP_VAL_REG; in set_temp_val_reg()
4645 ts->reg = reg; in set_temp_val_reg()
4648 /* Assign a non-register value type to @ts, and update reg_to_temp[]. */
4652 if (ts->val_type == TEMP_VAL_REG) { in set_temp_val_nonreg()
4653 TCGReg reg = ts->reg; in set_temp_val_nonreg()
4654 tcg_debug_assert(s->reg_to_temp[reg] == ts); in set_temp_val_nonreg()
4655 s->reg_to_temp[reg] = NULL; in set_temp_val_nonreg()
4657 ts->val_type = type; in set_temp_val_nonreg()
4668 switch (ts->kind) { in temp_free_or_dead()
4695 is non-zero, subsequently release the temporary; if it is positive, the
4700 if (!temp_readonly(ts) && !ts->mem_coherent) { in temp_sync()
4701 if (!ts->mem_allocated) { in temp_sync()
4704 switch (ts->val_type) { in temp_sync()
4710 && tcg_out_sti(s, ts->type, ts->val, in temp_sync()
4711 ts->mem_base->reg, ts->mem_offset)) { in temp_sync()
4714 temp_load(s, ts, tcg_target_available_regs[ts->type], in temp_sync()
4719 tcg_out_st(s, ts->type, ts->reg, in temp_sync()
4720 ts->mem_base->reg, ts->mem_offset); in temp_sync()
4730 ts->mem_coherent = 1; in temp_sync()
4740 TCGTemp *ts = s->reg_to_temp[reg]; in tcg_reg_free()
4742 temp_sync(s, ts, allocated_regs, 0, -1); in tcg_reg_free()
4781 if (s->reg_to_temp[reg] == NULL) { in tcg_reg_alloc()
4787 if (s->reg_to_temp[reg] == NULL && in tcg_reg_alloc()
4843 for (fmin = 2; fmin >= 0; fmin--) { in tcg_reg_alloc_pair()
4851 int f = !s->reg_to_temp[reg] + !s->reg_to_temp[reg + 1]; in tcg_reg_alloc_pair()
4871 switch (ts->val_type) { in temp_load()
4876 preferred_regs, ts->indirect_base); in temp_load()
4877 if (ts->type <= TCG_TYPE_I64) { in temp_load()
4878 tcg_out_movi(s, ts->type, reg, ts->val); in temp_load()
4880 uint64_t val = ts->val; in temp_load()
4896 tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val); in temp_load()
4898 ts->mem_coherent = 0; in temp_load()
4901 if (!ts->mem_allocated) { in temp_load()
4905 preferred_regs, ts->indirect_base); in temp_load()
4906 tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset); in temp_load()
4907 ts->mem_coherent = 1; in temp_load()
4922 tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts)); in temp_save()
4932 for (i = 0, n = s->nb_globals; i < n; i++) { in save_globals()
4933 temp_save(s, &s->temps[i], allocated_regs); in save_globals()
4944 for (i = 0, n = s->nb_globals; i < n; i++) { in sync_globals()
4945 TCGTemp *ts = &s->temps[i]; in sync_globals()
4946 tcg_debug_assert(ts->val_type != TEMP_VAL_REG in sync_globals()
4947 || ts->kind == TEMP_FIXED in sync_globals()
4948 || ts->mem_coherent); in sync_globals()
4957 for (int i = s->nb_globals; i < s->nb_temps; i++) { in tcg_reg_alloc_bb_end()
4958 TCGTemp *ts = &s->temps[i]; in tcg_reg_alloc_bb_end()
4960 switch (ts->kind) { in tcg_reg_alloc_bb_end()
4967 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD); in tcg_reg_alloc_bb_end()
4971 tcg_debug_assert(ts->val_type == TEMP_VAL_CONST); in tcg_reg_alloc_bb_end()
4983 * explicitly live-across-conditional-branch; all globals and local
4991 for (int i = s->nb_globals; i < s->nb_temps; i++) { in tcg_reg_alloc_cbranch()
4992 TCGTemp *ts = &s->temps[i]; in tcg_reg_alloc_cbranch()
4997 switch (ts->kind) { in tcg_reg_alloc_cbranch()
4999 tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent); in tcg_reg_alloc_cbranch()
5022 ots->val = val; in tcg_reg_alloc_do_movi()
5023 ots->mem_coherent = 0; in tcg_reg_alloc_do_movi()
5025 temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0)); in tcg_reg_alloc_do_movi()
5036 const TCGLifeData arg_life = op->life; in tcg_reg_alloc_mov()
5042 allocated_regs = s->reserved_regs; in tcg_reg_alloc_mov()
5044 ots = arg_temp(op->args[0]); in tcg_reg_alloc_mov()
5045 ts = arg_temp(op->args[1]); in tcg_reg_alloc_mov()
5050 /* Note that otype != itype for no-op truncation. */ in tcg_reg_alloc_mov()
5051 otype = ots->type; in tcg_reg_alloc_mov()
5052 itype = ts->type; in tcg_reg_alloc_mov()
5054 if (ts->val_type == TEMP_VAL_CONST) { in tcg_reg_alloc_mov()
5056 tcg_target_ulong val = ts->val; in tcg_reg_alloc_mov()
5068 if (ts->val_type == TEMP_VAL_MEM) { in tcg_reg_alloc_mov()
5072 tcg_debug_assert(ts->val_type == TEMP_VAL_REG); in tcg_reg_alloc_mov()
5073 ireg = ts->reg; in tcg_reg_alloc_mov()
5076 /* mov to a non-saved dead register makes no sense (even with in tcg_reg_alloc_mov()
5079 if (!ots->mem_allocated) { in tcg_reg_alloc_mov()
5082 tcg_out_st(s, otype, ireg, ots->mem_base->reg, ots->mem_offset); in tcg_reg_alloc_mov()
5090 if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) { in tcg_reg_alloc_mov()
5099 if (ots->val_type == TEMP_VAL_REG) { in tcg_reg_alloc_mov()
5100 oreg = ots->reg; in tcg_reg_alloc_mov()
5105 preferred_regs, ots->indirect_base); in tcg_reg_alloc_mov()
5109 * Cross register class move not supported. in tcg_reg_alloc_mov()
5114 if (!ts->mem_allocated) { in tcg_reg_alloc_mov()
5117 tcg_out_st(s, ts->type, ireg, ots->mem_base->reg, ots->mem_offset); in tcg_reg_alloc_mov()
5119 ots->mem_coherent = 1; in tcg_reg_alloc_mov()
5124 ots->mem_coherent = 0; in tcg_reg_alloc_mov()
5136 const TCGLifeData arg_life = op->life; in tcg_reg_alloc_dup()
5145 ots = arg_temp(op->args[0]); in tcg_reg_alloc_dup()
5146 its = arg_temp(op->args[1]); in tcg_reg_alloc_dup()
5151 itype = its->type; in tcg_reg_alloc_dup()
5155 if (its->val_type == TEMP_VAL_CONST) { in tcg_reg_alloc_dup()
5156 /* Propagate constant via movi -> dupi. */ in tcg_reg_alloc_dup()
5157 tcg_target_ulong val = its->val; in tcg_reg_alloc_dup()
5170 if (ots->val_type != TEMP_VAL_REG) { in tcg_reg_alloc_dup()
5171 TCGRegSet allocated_regs = s->reserved_regs; in tcg_reg_alloc_dup()
5174 if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) { in tcg_reg_alloc_dup()
5176 tcg_regset_set_reg(allocated_regs, its->reg); in tcg_reg_alloc_dup()
5179 output_pref(op, 0), ots->indirect_base); in tcg_reg_alloc_dup()
5183 switch (its->val_type) { in tcg_reg_alloc_dup()
5190 if (tcg_regset_test_reg(dup_in_regs, its->reg)) { in tcg_reg_alloc_dup()
5191 if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) { in tcg_reg_alloc_dup()
5196 if (!its->mem_coherent) { in tcg_reg_alloc_dup()
5199 * would be required to use memory. Attempt an integer-vector in tcg_reg_alloc_dup()
5202 if (tcg_out_mov(s, itype, ots->reg, its->reg)) { in tcg_reg_alloc_dup()
5206 temp_sync(s, its, s->reserved_regs, 0, 0); in tcg_reg_alloc_dup()
5213 lowpart_ofs = tcg_type_size(itype) - (1 << vece); in tcg_reg_alloc_dup()
5215 if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg, in tcg_reg_alloc_dup()
5216 its->mem_offset + lowpart_ofs)) { in tcg_reg_alloc_dup()
5220 tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset); in tcg_reg_alloc_dup()
5228 ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg); in tcg_reg_alloc_dup()
5232 ots->mem_coherent = 0; in tcg_reg_alloc_dup()
5237 temp_sync(s, ots, s->reserved_regs, 0, 0); in tcg_reg_alloc_dup()
5246 const TCGLifeData arg_life = op->life; in tcg_reg_alloc_op()
5247 const TCGOpDef * const def = &tcg_op_defs[op->opc]; in tcg_reg_alloc_op()
5260 if (def->flags & TCG_OPF_CARRY_IN) { in tcg_reg_alloc_op()
5261 tcg_debug_assert(s->carry_live); in tcg_reg_alloc_op()
5264 nb_oargs = def->nb_oargs; in tcg_reg_alloc_op()
5265 nb_iargs = def->nb_iargs; in tcg_reg_alloc_op()
5269 op->args + nb_oargs + nb_iargs, in tcg_reg_alloc_op()
5270 sizeof(TCGArg) * def->nb_cargs); in tcg_reg_alloc_op()
5272 i_allocated_regs = s->reserved_regs; in tcg_reg_alloc_op()
5273 o_allocated_regs = s->reserved_regs; in tcg_reg_alloc_op()
5275 switch (op->opc) { in tcg_reg_alloc_op()
5277 op_cond = op->args[2]; in tcg_reg_alloc_op()
5282 op_cond = op->args[3]; in tcg_reg_alloc_op()
5285 op_cond = op->args[4]; in tcg_reg_alloc_op()
5290 op_cond = op->args[5]; in tcg_reg_alloc_op()
5308 arg = op->args[i]; in tcg_reg_alloc_op()
5312 if (ts->val_type == TEMP_VAL_CONST) { in tcg_reg_alloc_op()
5314 if (ts->val == 0 && (arg_ct->ct & TCG_CT_REG_ZERO)) { in tcg_reg_alloc_op()
5315 /* Hardware zero register: indicate register via non-const. */ in tcg_reg_alloc_op()
5322 if (tcg_target_const_match(ts->val, arg_ct->ct, ts->type, in tcg_reg_alloc_op()
5326 new_args[i] = ts->val; in tcg_reg_alloc_op()
5331 reg = ts->reg; in tcg_reg_alloc_op()
5333 i_required_regs = arg_ct->regs; in tcg_reg_alloc_op()
5337 switch (arg_ct->pair) { in tcg_reg_alloc_op()
5339 if (arg_ct->ialias) { in tcg_reg_alloc_op()
5340 i_preferred_regs = output_pref(op, arg_ct->alias_index); in tcg_reg_alloc_op()
5349 || args_ct[arg_ct->alias_index].newreg) { in tcg_reg_alloc_op()
5351 } else if (ts->val_type == TEMP_VAL_REG) { in tcg_reg_alloc_op()
5363 reg = ts->reg; in tcg_reg_alloc_op()
5371 temp_load(s, ts, tcg_target_available_regs[ts->type], in tcg_reg_alloc_op()
5374 i_preferred_regs, ts->indirect_base); in tcg_reg_alloc_op()
5382 i2 = arg_ct->pair_index; in tcg_reg_alloc_op()
5383 ts2 = i1 != i2 ? arg_temp(op->args[i2]) : NULL; in tcg_reg_alloc_op()
5389 if (arg_ct->ialias) { in tcg_reg_alloc_op()
5390 i_preferred_regs = output_pref(op, arg_ct->alias_index); in tcg_reg_alloc_op()
5394 ts->val_type == TEMP_VAL_REG && in tcg_reg_alloc_op()
5395 ts->reg < TCG_TARGET_NB_REGS - 1 && in tcg_reg_alloc_op()
5400 ? ts2->val_type == TEMP_VAL_REG && in tcg_reg_alloc_op()
5401 ts2->reg == reg + 1 && in tcg_reg_alloc_op()
5403 : s->reg_to_temp[reg + 1] == NULL)) { in tcg_reg_alloc_op()
5409 if (ts->val_type == TEMP_VAL_REG && in tcg_reg_alloc_op()
5410 ts2->val_type == TEMP_VAL_REG && in tcg_reg_alloc_op()
5411 ts2->reg == reg + 1 && in tcg_reg_alloc_op()
5417 0, ts->indirect_base); in tcg_reg_alloc_op()
5421 reg = new_args[arg_ct->pair_index] + 1; in tcg_reg_alloc_op()
5425 tcg_debug_assert(arg_ct->ialias); in tcg_reg_alloc_op()
5426 i_preferred_regs = output_pref(op, arg_ct->alias_index); in tcg_reg_alloc_op()
5430 ts->val_type == TEMP_VAL_REG && in tcg_reg_alloc_op()
5432 s->reg_to_temp[reg - 1] == NULL && in tcg_reg_alloc_op()
5435 !tcg_regset_test_reg(i_allocated_regs, reg - 1)) { in tcg_reg_alloc_op()
5436 tcg_regset_set_reg(i_allocated_regs, reg - 1); in tcg_reg_alloc_op()
5441 ts->indirect_base); in tcg_reg_alloc_op()
5451 if (arg_ct->ialias && (!IS_DEAD_ARG(i) || temp_readonly(ts))) { in tcg_reg_alloc_op()
5461 if (ts->val_type == TEMP_VAL_REG && ts->reg == reg) { in tcg_reg_alloc_op()
5466 tcg_debug_assert(ts->kind != TEMP_FIXED); in tcg_reg_alloc_op()
5467 nr = tcg_reg_alloc(s, tcg_target_available_regs[ts->type], in tcg_reg_alloc_op()
5468 t_allocated_regs, 0, ts->indirect_base); in tcg_reg_alloc_op()
5469 ok = tcg_out_mov(s, ts->type, nr, reg); in tcg_reg_alloc_op()
5474 temp_load(s, ts, tcg_target_available_regs[ts->type], in tcg_reg_alloc_op()
5483 copyto_new_reg = ts->reg != reg; in tcg_reg_alloc_op()
5492 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) { in tcg_reg_alloc_op()
5494 * Cross register class move not supported. Sync the in tcg_reg_alloc_op()
5498 tcg_out_ld(s, ts->type, reg, in tcg_reg_alloc_op()
5499 ts->mem_base->reg, ts->mem_offset); in tcg_reg_alloc_op()
5510 temp_dead(s, arg_temp(op->args[i])); in tcg_reg_alloc_op()
5514 if (def->flags & TCG_OPF_COND_BRANCH) { in tcg_reg_alloc_op()
5516 } else if (def->flags & TCG_OPF_BB_END) { in tcg_reg_alloc_op()
5519 if (def->flags & TCG_OPF_CALL_CLOBBER) { in tcg_reg_alloc_op()
5528 if (def->flags & TCG_OPF_SIDE_EFFECTS) { in tcg_reg_alloc_op()
5537 arg = op->args[i]; in tcg_reg_alloc_op()
5544 switch (arg_ct->pair) { in tcg_reg_alloc_op()
5546 if (arg_ct->oalias && !const_args[arg_ct->alias_index]) { in tcg_reg_alloc_op()
5547 reg = new_args[arg_ct->alias_index]; in tcg_reg_alloc_op()
5548 } else if (arg_ct->newreg) { in tcg_reg_alloc_op()
5549 reg = tcg_reg_alloc(s, arg_ct->regs, in tcg_reg_alloc_op()
5551 output_pref(op, k), ts->indirect_base); in tcg_reg_alloc_op()
5553 reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs, in tcg_reg_alloc_op()
5554 output_pref(op, k), ts->indirect_base); in tcg_reg_alloc_op()
5559 if (arg_ct->oalias) { in tcg_reg_alloc_op()
5560 reg = new_args[arg_ct->alias_index]; in tcg_reg_alloc_op()
5561 } else if (arg_ct->newreg) { in tcg_reg_alloc_op()
5562 reg = tcg_reg_alloc_pair(s, arg_ct->regs, in tcg_reg_alloc_op()
5565 ts->indirect_base); in tcg_reg_alloc_op()
5567 reg = tcg_reg_alloc_pair(s, arg_ct->regs, o_allocated_regs, in tcg_reg_alloc_op()
5569 ts->indirect_base); in tcg_reg_alloc_op()
5574 if (arg_ct->oalias) { in tcg_reg_alloc_op()
5575 reg = new_args[arg_ct->alias_index]; in tcg_reg_alloc_op()
5577 reg = new_args[arg_ct->pair_index] + 1; in tcg_reg_alloc_op()
5582 tcg_debug_assert(!arg_ct->newreg); in tcg_reg_alloc_op()
5583 reg = new_args[arg_ct->pair_index] - 1; in tcg_reg_alloc_op()
5591 ts->mem_coherent = 0; in tcg_reg_alloc_op()
5598 switch (op->opc) { in tcg_reg_alloc_op()
5629 container_of(all_outop[op->opc], TCGOutOpBinary, base); in tcg_reg_alloc_op()
5634 out->out_rri(s, type, new_args[0], new_args[1], new_args[2]); in tcg_reg_alloc_op()
5636 out->out_rrr(s, type, new_args[0], new_args[1], new_args[2]); in tcg_reg_alloc_op()
5651 out->out_rir(s, type, new_args[0], new_args[1], new_args[2]); in tcg_reg_alloc_op()
5653 out->out_rrr(s, type, new_args[0], new_args[1], new_args[2]); in tcg_reg_alloc_op()
5667 container_of(all_outop[op->opc], TCGOutOpAddSubCarry, base); in tcg_reg_alloc_op()
5671 out->out_rii(s, type, new_args[0], in tcg_reg_alloc_op()
5674 out->out_rri(s, type, new_args[0], in tcg_reg_alloc_op()
5678 out->out_rir(s, type, new_args[0], new_args[1], new_args[2]); in tcg_reg_alloc_op()
5680 out->out_rrr(s, type, new_args[0], new_args[1], new_args[2]); in tcg_reg_alloc_op()
5697 container_of(all_outop[op->opc], TCGOutOpUnary, base); in tcg_reg_alloc_op()
5701 out->out_rr(s, type, new_args[0], new_args[1]); in tcg_reg_alloc_op()
5709 container_of(all_outop[op->opc], TCGOutOpBswap, base); in tcg_reg_alloc_op()
5712 out->out_rr(s, type, new_args[0], new_args[1], new_args[2]); in tcg_reg_alloc_op()
5722 out->out_rri(s, type, new_args[0], new_args[1], in tcg_reg_alloc_op()
5727 out->out_rzr(s, type, new_args[0], new_args[2], in tcg_reg_alloc_op()
5730 out->out_rrr(s, type, new_args[0], new_args[1], in tcg_reg_alloc_op()
5740 container_of(all_outop[op->opc], TCGOutOpDivRem, base); in tcg_reg_alloc_op()
5742 /* Only used by x86 and s390x, which use matching constraints. */ in tcg_reg_alloc_op()
5746 out->out_rr01r(s, type, new_args[0], new_args[1], new_args[4]); in tcg_reg_alloc_op()
5754 container_of(all_outop[op->opc], TCGOutOpExtract, base); in tcg_reg_alloc_op()
5757 out->out_rr(s, type, new_args[0], new_args[1], in tcg_reg_alloc_op()
5768 out->out_rrr(s, type, new_args[0], new_args[1], in tcg_reg_alloc_op()
5782 container_of(all_outop[op->opc], TCGOutOpLoad, base); in tcg_reg_alloc_op()
5785 out->out(s, type, new_args[0], new_args[1], new_args[2]); in tcg_reg_alloc_op()
5793 container_of(all_outop[op->opc], TCGOutOpMul2, base); in tcg_reg_alloc_op()
5797 out->out_rrrr(s, type, new_args[0], new_args[1], in tcg_reg_alloc_op()
5811 container_of(all_outop[op->opc], TCGOutOpStore, base); in tcg_reg_alloc_op()
5814 out->out_i(s, type, new_args[0], new_args[1], new_args[2]); in tcg_reg_alloc_op()
5816 out->out_r(s, type, new_args[0], new_args[1], new_args[2]); in tcg_reg_alloc_op()
5825 container_of(all_outop[op->opc], TCGOutOpQemuLdSt, base); in tcg_reg_alloc_op()
5827 out->out(s, type, new_args[0], new_args[1], new_args[2]); in tcg_reg_alloc_op()
5835 container_of(all_outop[op->opc], TCGOutOpQemuLdSt2, base); in tcg_reg_alloc_op()
5837 out->out(s, type, new_args[0], new_args[1], in tcg_reg_alloc_op()
5850 out->out_ri(s, type, cond, new_args[0], new_args[1], label); in tcg_reg_alloc_op()
5852 out->out_rr(s, type, cond, new_args[0], new_args[1], label); in tcg_reg_alloc_op()
5863 out->out(s, type, cond, new_args[0], in tcg_reg_alloc_op()
5874 container_of(all_outop[op->opc], TCGOutOpSetcond, base); in tcg_reg_alloc_op()
5879 out->out_rri(s, type, cond, in tcg_reg_alloc_op()
5882 out->out_rrr(s, type, cond, in tcg_reg_alloc_op()
5897 out->out(s, cond, new_args[0], new_args[1], in tcg_reg_alloc_op()
5909 out->out(s, cond, new_args[0], new_args[1], new_args[2], in tcg_reg_alloc_op()
5925 tcg_debug_assert(def->flags & TCG_OPF_VECTOR); in tcg_reg_alloc_op()
5926 tcg_out_vec_op(s, op->opc, type - TCG_TYPE_V64, in tcg_reg_alloc_op()
5931 if (def->flags & TCG_OPF_CARRY_IN) { in tcg_reg_alloc_op()
5932 s->carry_live = false; in tcg_reg_alloc_op()
5934 if (def->flags & TCG_OPF_CARRY_OUT) { in tcg_reg_alloc_op()
5935 s->carry_live = true; in tcg_reg_alloc_op()
5940 ts = arg_temp(op->args[i]); in tcg_reg_alloc_op()
5955 const TCGLifeData arg_life = op->life; in tcg_reg_alloc_dup2()
5959 /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */ in tcg_reg_alloc_dup2()
5963 ots = arg_temp(op->args[0]); in tcg_reg_alloc_dup2()
5964 itsl = arg_temp(op->args[1]); in tcg_reg_alloc_dup2()
5965 itsh = arg_temp(op->args[2]); in tcg_reg_alloc_dup2()
5971 if (ots->val_type != TEMP_VAL_REG) { in tcg_reg_alloc_dup2()
5972 TCGRegSet allocated_regs = s->reserved_regs; in tcg_reg_alloc_dup2()
5977 if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) { in tcg_reg_alloc_dup2()
5978 tcg_regset_set_reg(allocated_regs, itsl->reg); in tcg_reg_alloc_dup2()
5980 if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) { in tcg_reg_alloc_dup2()
5981 tcg_regset_set_reg(allocated_regs, itsh->reg); in tcg_reg_alloc_dup2()
5985 output_pref(op, 0), ots->indirect_base); in tcg_reg_alloc_dup2()
5990 if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) { in tcg_reg_alloc_dup2()
5991 uint64_t val = deposit64(itsl->val, 32, 32, itsh->val); in tcg_reg_alloc_dup2()
6002 tcg_out_dupi_vec(s, vtype, vece, ots->reg, val); in tcg_reg_alloc_dup2()
6006 /* If the two inputs form one 64-bit value, try dupm_vec. */ in tcg_reg_alloc_dup2()
6007 if (itsl->temp_subindex == HOST_BIG_ENDIAN && in tcg_reg_alloc_dup2()
6008 itsh->temp_subindex == !HOST_BIG_ENDIAN && in tcg_reg_alloc_dup2()
6009 itsl == itsh + (HOST_BIG_ENDIAN ? 1 : -1)) { in tcg_reg_alloc_dup2()
6010 TCGTemp *its = itsl - HOST_BIG_ENDIAN; in tcg_reg_alloc_dup2()
6012 temp_sync(s, its + 0, s->reserved_regs, 0, 0); in tcg_reg_alloc_dup2()
6013 temp_sync(s, its + 1, s->reserved_regs, 0, 0); in tcg_reg_alloc_dup2()
6015 if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg, in tcg_reg_alloc_dup2()
6016 its->mem_base->reg, its->mem_offset)) { in tcg_reg_alloc_dup2()
6025 ots->mem_coherent = 0; in tcg_reg_alloc_dup2()
6033 temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0)); in tcg_reg_alloc_dup2()
6043 if (ts->val_type == TEMP_VAL_REG) { in load_arg_reg()
6044 if (ts->reg != reg) { in load_arg_reg()
6046 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) { in load_arg_reg()
6048 * Cross register class move not supported. Sync the in load_arg_reg()
6052 tcg_out_ld(s, ts->type, reg, in load_arg_reg()
6053 ts->mem_base->reg, ts->mem_offset); in load_arg_reg()
6070 * If there are many call-saved registers, the temp might live to in load_arg_stk()
6073 temp_load(s, ts, tcg_target_available_regs[ts->type], allocated_regs, 0); in load_arg_stk()
6074 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, in load_arg_stk()
6081 if (arg_slot_reg_p(l->arg_slot)) { in load_arg_normal()
6082 TCGReg reg = tcg_target_call_iarg_regs[l->arg_slot]; in load_arg_normal()
6086 load_arg_stk(s, l->arg_slot, ts, *allocated_regs); in load_arg_normal()
6113 const TCGLifeData arg_life = op->life; in tcg_reg_alloc_call()
6115 TCGRegSet allocated_regs = s->reserved_regs; in tcg_reg_alloc_call()
6122 for (i = nb_iargs - 1; i >= 0; --i) { in tcg_reg_alloc_call()
6123 const TCGCallArgumentLoc *loc = &info->in[i]; in tcg_reg_alloc_call()
6124 TCGTemp *ts = arg_temp(op->args[nb_oargs + i]); in tcg_reg_alloc_call()
6126 switch (loc->kind) { in tcg_reg_alloc_call()
6133 load_arg_stk(s, loc->ref_slot, ts, allocated_regs); in tcg_reg_alloc_call()
6134 load_arg_ref(s, loc->arg_slot, TCG_REG_CALL_STACK, in tcg_reg_alloc_call()
6135 arg_slot_stk_ofs(loc->ref_slot), in tcg_reg_alloc_call()
6139 load_arg_stk(s, loc->ref_slot, ts, allocated_regs); in tcg_reg_alloc_call()
6149 temp_dead(s, arg_temp(op->args[i])); in tcg_reg_alloc_call()
6164 if (info->flags & TCG_CALL_NO_READ_GLOBALS) { in tcg_reg_alloc_call()
6166 } else if (info->flags & TCG_CALL_NO_WRITE_GLOBALS) { in tcg_reg_alloc_call()
6176 if (info->out_kind == TCG_CALL_RET_BY_REF) { in tcg_reg_alloc_call()
6177 TCGTemp *ts = arg_temp(op->args[0]); in tcg_reg_alloc_call()
6179 if (!ts->mem_allocated) { in tcg_reg_alloc_call()
6182 load_arg_ref(s, 0, ts->mem_base->reg, ts->mem_offset, &allocated_regs); in tcg_reg_alloc_call()
6188 switch (info->out_kind) { in tcg_reg_alloc_call()
6191 TCGTemp *ts = arg_temp(op->args[i]); in tcg_reg_alloc_call()
6198 ts->mem_coherent = 0; in tcg_reg_alloc_call()
6204 TCGTemp *ts = arg_temp(op->args[0]); in tcg_reg_alloc_call()
6206 tcg_debug_assert(ts->base_type == TCG_TYPE_I128); in tcg_reg_alloc_call()
6207 tcg_debug_assert(ts->temp_subindex == 0); in tcg_reg_alloc_call()
6208 if (!ts->mem_allocated) { in tcg_reg_alloc_call()
6213 ts->mem_base->reg, ts->mem_offset); in tcg_reg_alloc_call()
6220 TCGTemp *ts = arg_temp(op->args[i]); in tcg_reg_alloc_call()
6221 ts->val_type = TEMP_VAL_MEM; in tcg_reg_alloc_call()
6231 TCGTemp *ts = arg_temp(op->args[i]); in tcg_reg_alloc_call()
6233 temp_sync(s, ts, s->reserved_regs, 0, IS_DEAD_ARG(i)); in tcg_reg_alloc_call()
6260 MemOp half = size ? size - 1 : 0; in atom_and_align_for_opc()
6321 * We must re-implement tcg_gen_callN and tcg_reg_alloc_call simultaneously,
6352 for (i = nmov; i-- > 0; ) { in tcg_out_helper_load_slots()
6365 tcg_debug_assert(parm->ntmp != 0); in tcg_out_helper_load_slots()
6366 mov[i].dst = src = parm->tmp[0]; in tcg_out_helper_load_slots()
6388 tcg_debug_assert(parm->ntmp >= 2); in tcg_out_helper_load_slots()
6398 TCGReg scratch = parm->tmp[1]; in tcg_out_helper_load_slots()
6401 tcg_out_movext3(s, mov, mov + 1, mov + 2, parm->tmp[0]); in tcg_out_helper_load_slots()
6413 parm->ntmp ? parm->tmp[0] : -1); in tcg_out_helper_load_slots()
6417 parm->ntmp ? parm->tmp[0] : -1); in tcg_out_helper_load_slots()
6436 tcg_debug_assert(parm->ntmp != 0); in tcg_out_helper_load_imm()
6437 tcg_out_movi(s, type, parm->tmp[0], imm); in tcg_out_helper_load_imm()
6438 tcg_out_st(s, type, parm->tmp[0], TCG_REG_CALL_STACK, ofs); in tcg_out_helper_load_imm()
6454 const TCGCallArgumentLoc *loc = &info->in[0]; in tcg_out_helper_load_common_args()
6462 ptr_mov.dst = loc->arg_slot; in tcg_out_helper_load_common_args()
6469 imm = ldst->oi; in tcg_out_helper_load_common_args()
6470 loc = &info->in[next_arg]; in tcg_out_helper_load_common_args()
6472 switch (loc->kind) { in tcg_out_helper_load_common_args()
6484 tcg_out_helper_load_imm(s, loc->arg_slot, type, imm, parm); in tcg_out_helper_load_common_args()
6490 loc = &info->in[next_arg]; in tcg_out_helper_load_common_args()
6491 slot = loc->arg_slot; in tcg_out_helper_load_common_args()
6492 if (parm->ra_gen) { in tcg_out_helper_load_common_args()
6493 int arg_reg = -1; in tcg_out_helper_load_common_args()
6499 ra_reg = parm->ra_gen(s, ldst, arg_reg); in tcg_out_helper_load_common_args()
6505 imm = (uintptr_t)ldst->raddr; in tcg_out_helper_load_common_args()
6520 switch (loc->kind) { in tcg_out_helper_add_mov()
6536 mov[0].dst = loc->arg_slot; in tcg_out_helper_add_mov()
6574 MemOp mop = get_memop(ldst->oi); in tcg_out_ld_helper_args()
6595 loc = &info->in[next_arg]; in tcg_out_ld_helper_args()
6596 if (TCG_TARGET_REG_BITS == 32 && s->addr_type == TCG_TYPE_I32) { in tcg_out_ld_helper_args()
6598 * 32-bit host with 32-bit guest: zero-extend the guest address in tcg_out_ld_helper_args()
6599 * to 64-bits for the helper by storing the low part, then in tcg_out_ld_helper_args()
6604 ldst->addr_reg, -1); in tcg_out_ld_helper_args()
6611 nmov = tcg_out_helper_add_mov(mov, loc, TCG_TYPE_I64, s->addr_type, in tcg_out_ld_helper_args()
6612 ldst->addr_reg, -1); in tcg_out_ld_helper_args()
6617 switch (info->out_kind) { in tcg_out_ld_helper_args()
6624 * We need memory in which to return: re-use the top of stack. in tcg_out_ld_helper_args()
6633 tcg_debug_assert(parm->ntmp != 0); in tcg_out_ld_helper_args()
6634 tcg_out_addi_ptr(s, parm->tmp[0], in tcg_out_ld_helper_args()
6636 tcg_out_st(s, TCG_TYPE_PTR, parm->tmp[0], in tcg_out_ld_helper_args()
6652 MemOp mop = get_memop(ldst->oi); in tcg_out_ld_helper_ret()
6656 switch (ldst->type) { in tcg_out_ld_helper_ret()
6664 mov[0].dst = ldst->datalo_reg; in tcg_out_ld_helper_ret()
6666 mov[0].dst_type = ldst->type; in tcg_out_ld_helper_ret()
6680 if (TCG_TARGET_REG_BITS == 32 || ldst->type == TCG_TYPE_I32) { in tcg_out_ld_helper_ret()
6703 tcg_out_ld(s, TCG_TYPE_I64, ldst->datalo_reg, in tcg_out_ld_helper_ret()
6705 tcg_out_ld(s, TCG_TYPE_I64, ldst->datahi_reg, in tcg_out_ld_helper_ret()
6717 mov[0].dst = ldst->datalo_reg; in tcg_out_ld_helper_ret()
6724 mov[1].dst = ldst->datahi_reg; in tcg_out_ld_helper_ret()
6731 tcg_out_movext2(s, mov, mov + 1, parm->ntmp ? parm->tmp[0] : -1); in tcg_out_ld_helper_ret()
6742 MemOp mop = get_memop(ldst->oi); in tcg_out_st_helper_args()
6768 loc = &info->in[next_arg]; in tcg_out_st_helper_args()
6769 tcg_debug_assert(s->addr_type <= TCG_TYPE_REG); in tcg_out_st_helper_args()
6772 * 32-bit host (and thus 32-bit guest): zero-extend the guest address in tcg_out_st_helper_args()
6773 * to 64-bits for the helper by storing the low part. Later, in tcg_out_st_helper_args()
6779 ldst->addr_reg, -1); in tcg_out_st_helper_args()
6783 n = tcg_out_helper_add_mov(mov, loc, TCG_TYPE_I64, s->addr_type, in tcg_out_st_helper_args()
6784 ldst->addr_reg, -1); in tcg_out_st_helper_args()
6790 loc = &info->in[next_arg]; in tcg_out_st_helper_args()
6791 switch (loc->kind) { in tcg_out_st_helper_args()
6795 n = tcg_out_helper_add_mov(mov + nmov, loc, data_type, ldst->type, in tcg_out_st_helper_args()
6796 ldst->datalo_reg, ldst->datahi_reg); in tcg_out_st_helper_args()
6806 HOST_BIG_ENDIAN ? ldst->datahi_reg : ldst->datalo_reg, in tcg_out_st_helper_args()
6809 HOST_BIG_ENDIAN ? ldst->datalo_reg : ldst->datahi_reg, in tcg_out_st_helper_args()
6814 if (arg_slot_reg_p(loc->arg_slot)) { in tcg_out_st_helper_args()
6815 tcg_out_addi_ptr(s, tcg_target_call_iarg_regs[loc->arg_slot], in tcg_out_st_helper_args()
6817 arg_slot_stk_ofs(loc->ref_slot)); in tcg_out_st_helper_args()
6819 tcg_debug_assert(parm->ntmp != 0); in tcg_out_st_helper_args()
6820 tcg_out_addi_ptr(s, parm->tmp[0], TCG_REG_CALL_STACK, in tcg_out_st_helper_args()
6821 arg_slot_stk_ofs(loc->ref_slot)); in tcg_out_st_helper_args()
6822 tcg_out_st(s, TCG_TYPE_PTR, parm->tmp[0], in tcg_out_st_helper_args()
6823 TCG_REG_CALL_STACK, arg_slot_stk_ofs(loc->arg_slot)); in tcg_out_st_helper_args()
6834 loc = &info->in[1 + !HOST_BIG_ENDIAN]; in tcg_out_st_helper_args()
6835 tcg_out_helper_load_imm(s, loc->arg_slot, TCG_TYPE_I32, 0, parm); in tcg_out_st_helper_args()
6863 QSIMPLEQ_FOREACH(l, &s->labels, next) { in tcg_gen_code()
6864 if (unlikely(!l->present) && !QSIMPLEQ_EMPTY(&l->branches)) { in tcg_gen_code()
6866 "$L%d referenced but not present.\n", l->id); in tcg_gen_code()
6883 if (s->nb_indirects > 0) { in tcg_gen_code()
6897 /* If changes were made, re-run liveness. */ in tcg_gen_code()
6914 tb->jmp_reset_offset[0] = TB_JMP_OFFSET_INVALID; in tcg_gen_code()
6915 tb->jmp_reset_offset[1] = TB_JMP_OFFSET_INVALID; in tcg_gen_code()
6916 tb->jmp_insn_offset[0] = TB_JMP_OFFSET_INVALID; in tcg_gen_code()
6917 tb->jmp_insn_offset[1] = TB_JMP_OFFSET_INVALID; in tcg_gen_code()
6923 * TODO: Move this into translate-all.c with the rest of the in tcg_gen_code()
6926 s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr); in tcg_gen_code()
6927 s->code_ptr = s->code_buf; in tcg_gen_code()
6928 s->data_gen_ptr = NULL; in tcg_gen_code()
6930 QSIMPLEQ_INIT(&s->ldst_labels); in tcg_gen_code()
6931 s->pool_labels = NULL; in tcg_gen_code()
6933 s->gen_insn_data = in tcg_gen_code()
6934 tcg_malloc(sizeof(uint64_t) * s->gen_tb->icount * INSN_START_WORDS); in tcg_gen_code()
6938 num_insns = -1; in tcg_gen_code()
6939 s->carry_live = false; in tcg_gen_code()
6940 QTAILQ_FOREACH(op, &s->ops, link) { in tcg_gen_code()
6941 TCGOpcode opc = op->opc; in tcg_gen_code()
6948 * e.g. zero or sign-extended, then emit as a unary op. in tcg_gen_code()
6954 TCGLifeData arg_life = op->life; in tcg_gen_code()
6971 s->gen_insn_end_off[num_insns] = off; in tcg_gen_code()
6973 assert(s->gen_insn_end_off[num_insns] == off); in tcg_gen_code()
6977 s->gen_insn_data[num_insns * INSN_START_WORDS + i] = in tcg_gen_code()
6982 temp_dead(s, arg_temp(op->args[0])); in tcg_gen_code()
6985 tcg_reg_alloc_bb_end(s, s->reserved_regs); in tcg_gen_code()
6986 tcg_out_label(s, arg_label(op->args[0])); in tcg_gen_code()
6993 tcg_out_exit_tb(s, op->args[0]); in tcg_gen_code()
6996 tcg_out_goto_tb(s, op->args[0]); in tcg_gen_code()
6999 tcg_out_br(s, arg_label(op->args[0])); in tcg_gen_code()
7002 tcg_out_mb(s, op->args[0]); in tcg_gen_code()
7024 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) { in tcg_gen_code()
7025 return -1; in tcg_gen_code()
7029 return -2; in tcg_gen_code()
7034 tcg_debug_assert(num_insns + 1 == s->gen_tb->icount); in tcg_gen_code()
7035 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s); in tcg_gen_code()
7047 return -2; in tcg_gen_code()
7052 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf), in tcg_gen_code()
7053 (uintptr_t)s->code_buf, in tcg_gen_code()
7054 tcg_ptr_byte_diff(s->code_ptr, s->code_buf)); in tcg_gen_code()
7067 the contents of a .debug_frame section that describes the post-
7112 return p - strtab; in find_string()
7168 .e_shnum = ARRAY_SIZE(img->shdr), in tcg_register_jit_int()
7169 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1, in tcg_register_jit_int()
7199 .sh_size = sizeof(img->da), in tcg_register_jit_int()
7208 .sh_size = sizeof(img->sym), in tcg_register_jit_int()
7210 .sh_link = ARRAY_SIZE(img->shdr) - 1, in tcg_register_jit_int()
7216 .sh_size = sizeof(img->str), in tcg_register_jit_int()
7226 .len = sizeof(struct DebugInfo) - 4, in tcg_register_jit_int()
7263 img->phdr.p_vaddr = buf; in tcg_register_jit_int()
7264 img->phdr.p_paddr = buf; in tcg_register_jit_int()
7265 img->phdr.p_memsz = buf_size; in tcg_register_jit_int()
7267 img->shdr[1].sh_name = find_string(img->str, ".text"); in tcg_register_jit_int()
7268 img->shdr[1].sh_addr = buf; in tcg_register_jit_int()
7269 img->shdr[1].sh_size = buf_size; in tcg_register_jit_int()
7271 img->shdr[2].sh_name = find_string(img->str, ".debug_info"); in tcg_register_jit_int()
7272 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev"); in tcg_register_jit_int()
7274 img->shdr[4].sh_name = find_string(img->str, ".debug_frame"); in tcg_register_jit_int()
7275 img->shdr[4].sh_size = debug_frame_size; in tcg_register_jit_int()
7277 img->shdr[5].sh_name = find_string(img->str, ".symtab"); in tcg_register_jit_int()
7278 img->shdr[6].sh_name = find_string(img->str, ".strtab"); in tcg_register_jit_int()
7280 img->sym[1].st_name = find_string(img->str, "code_gen_buffer"); in tcg_register_jit_int()
7281 img->sym[1].st_value = buf; in tcg_register_jit_int()
7282 img->sym[1].st_size = buf_size; in tcg_register_jit_int()
7284 img->di.cu_low_pc = buf; in tcg_register_jit_int()
7285 img->di.cu_high_pc = buf + buf_size; in tcg_register_jit_int()
7286 img->di.fn_low_pc = buf; in tcg_register_jit_int()
7287 img->di.fn_high_pc = buf + buf_size; in tcg_register_jit_int()
7291 dfh->fde.func_start = buf; in tcg_register_jit_int()
7292 dfh->fde.func_len = buf_size; in tcg_register_jit_int()