Lines Matching +full:post +full:- +full:processing
30 #include "qemu/error-report.h"
32 #include "qemu/host-utils.h"
33 #include "qemu/qemu-print.h"
38 #include "exec/translation-block.h"
39 #include "exec/tlb-common.h"
41 #include "tcg/tcg-op-common.h"
56 #include "tcg/tcg-ldst.h"
57 #include "tcg/tcg-temp-internal.h"
58 #include "tcg-internal.h"
60 #include "tcg-has.h"
62 #include "user/guest-base.h"
65 /* Forward declarations for functions declared in tcg-target.c.inc and
117 /* Forward declarations for functions declared and used in tcg-target.c.inc. */
267 *s->code_ptr++ = v;
281 *s->code_ptr++ = v;
283 tcg_insn_unit *p = s->code_ptr;
285 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE);
304 *s->code_ptr++ = v;
306 tcg_insn_unit *p = s->code_ptr;
308 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE);
327 *s->code_ptr++ = v;
329 tcg_insn_unit *p = s->code_ptr;
331 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE);
346 /* label relocation processing */
353 r->type = type;
354 r->ptr = code_ptr;
355 r->addend = addend;
356 QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next);
361 tcg_debug_assert(!l->has_value);
362 l->has_value = 1;
363 l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr);
372 l->id = s->nb_labels++;
373 QSIMPLEQ_INIT(&l->branches);
374 QSIMPLEQ_INIT(&l->relocs);
376 QSIMPLEQ_INSERT_TAIL(&s->labels, l, next);
385 QSIMPLEQ_FOREACH(l, &s->labels, next) {
387 uintptr_t value = l->u.value;
389 QSIMPLEQ_FOREACH(r, &l->relocs, next) {
390 if (!patch_reloc(r->ptr, r->type, value, r->addend)) {
404 s->gen_tb->jmp_reset_offset[which] = tcg_current_code_size(s);
413 s->gen_tb->jmp_insn_offset[which] = tcg_current_code_size(s);
419 * Return the read-execute version of the pointer, for the benefit
420 * of any pc-relative addressing mode.
422 return (uintptr_t)tcg_splitwx_to_rx(&s->gen_tb->jmp_target_addr[which]);
428 return (offsetof(CPUNegativeOffsetState, tlb.f[which]) -
436 siglongjmp(s->jmp_trans, -2);
457 * tcg_out_movext -- move and extend
522 tcg_out_movext(s, i->dst_type, i->dst, i->src_type, i->src_ext, src);
527 tcg_out_movext1_new_src(s, i, i->src);
531 * tcg_out_movext2 -- move and extend two pair
535 * @scratch: temporary register, or -1 for none
544 TCGReg src1 = i1->src;
545 TCGReg src2 = i2->src;
547 if (i1->dst != src2) {
552 if (i2->dst == src1) {
553 TCGType src1_type = i1->src_type;
554 TCGType src2_type = i2->src_type;
558 src1 = i2->src;
559 src2 = i1->src;
571 * tcg_out_movext3 -- move and extend three pair
576 * @scratch: temporary register, or -1 for none
586 TCGReg src1 = i1->src;
587 TCGReg src2 = i2->src;
588 TCGReg src3 = i3->src;
590 if (i1->dst != src2 && i1->dst != src3) {
595 if (i2->dst != src1 && i2->dst != src3) {
600 if (i3->dst != src1 && i3->dst != src2) {
608 * either "clockwise" or "anti-clockwise", and can be solved with
611 if (i1->dst == src2 && i2->dst == src3 && i3->dst == src1) {
613 if (tcg_out_xchg(s, MAX(i1->src_type, i2->src_type), src1, src2)) {
614 tcg_out_xchg(s, MAX(i2->src_type, i3->src_type), src2, src3);
616 tcg_out_movext1_new_src(s, i1, i1->dst);
617 tcg_out_movext1_new_src(s, i2, i2->dst);
618 tcg_out_movext1_new_src(s, i3, i3->dst);
621 tcg_out_mov(s, i1->src_type, scratch, src1);
626 } else if (i1->dst == src3 && i2->dst == src1 && i3->dst == src2) {
627 /* "Anti-clockwise" */
628 if (tcg_out_xchg(s, MAX(i2->src_type, i3->src_type), src2, src3)) {
629 tcg_out_xchg(s, MAX(i1->src_type, i2->src_type), src1, src2);
631 tcg_out_movext1_new_src(s, i1, i1->dst);
632 tcg_out_movext1_new_src(s, i2, i2->dst);
633 tcg_out_movext1_new_src(s, i3, i3->dst);
636 tcg_out_mov(s, i1->src_type, scratch, src1);
656 QSIMPLEQ_INSERT_TAIL(&s->ldst_labels, l, next);
680 n->label = label;
681 n->addend = addend;
682 n->rtype = rtype;
683 n->nlong = nlong;
690 int nlong = n->nlong;
693 for (pp = &s->pool_labels; (i = *pp) != NULL; pp = &i->next) {
694 if (nlong > i->nlong) {
697 if (nlong < i->nlong) {
700 if (memcmp(n->data, i->data, sizeof(tcg_target_ulong) * nlong) >= 0) {
704 n->next = *pp;
714 n->data[0] = d;
725 n->data[0] = d0;
726 n->data[1] = d1;
738 n->data[0] = d0;
739 n->data[1] = d1;
740 n->data[2] = d2;
741 n->data[3] = d3;
745 /* For v256, for 32-bit host. */
755 n->data[0] = d0;
756 n->data[1] = d1;
757 n->data[2] = d2;
758 n->data[3] = d3;
759 n->data[4] = d4;
760 n->data[5] = d5;
761 n->data[6] = d6;
762 n->data[7] = d7;
775 QSIMPLEQ_FOREACH(lb, &s->ldst_labels, next) {
776 if (lb->is_ld
779 return -2;
788 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
789 return -1;
797 TCGLabelPoolData *p = s->pool_labels;
809 a = (void *)ROUND_UP((uintptr_t)s->code_ptr,
810 sizeof(tcg_target_ulong) * p->nlong);
811 tcg_out_nop_fill(s->code_ptr, (tcg_insn_unit *)a - s->code_ptr);
812 s->data_gen_ptr = a;
814 for (; p != NULL; p = p->next) {
815 size_t size = sizeof(tcg_target_ulong) * p->nlong;
818 if (!l || l->nlong != p->nlong || memcmp(l->data, p->data, size)) {
819 if (unlikely(a > s->code_gen_highwater)) {
820 return -1;
822 memcpy(a, p->data, size);
827 value = (uintptr_t)tcg_splitwx_to_rx(a) - size;
828 if (!patch_reloc(p->label, p->rtype, value, p->addend)) {
829 return -2;
833 s->code_ptr = a;
867 C_Dynamic = -2,
868 C_NotImplemented = -1,
869 #include "tcg-target-con-set.h"
919 #include "tcg-target-con-set.h"
1119 #include "tcg-target.c.inc"
1123 QEMU_BUILD_BUG_ON((int)(offsetof(CPUNegativeOffsetState, tlb.f[0]) -
1130 * We require these functions for slow-path function calls.
1177 * This prevents trivial mistakes within each arch/tcg-target.c.inc.
1271 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation
1274 * In system-mode each caller registers its context in tcg_ctxs[]. Note that in
1275 * system-mode tcg_ctxs[] does not track tcg_ctx_init, since the initial context
1278 * Not tracking tcg_init_ctx in tcg_ctxs[] in system-mode keeps code that
1298 ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps;
1300 s->temps[i].mem_base = &s->temps[b];
1326 p->size = size;
1327 p->next = s->pool_first_large;
1328 s->pool_first_large = p;
1329 return p->data;
1331 p = s->pool_current;
1333 p = s->pool_first;
1337 if (!p->next) {
1341 p->size = pool_size;
1342 p->next = NULL;
1343 if (s->pool_current) {
1344 s->pool_current->next = p;
1346 s->pool_first = p;
1349 p = p->next;
1353 s->pool_current = p;
1354 s->pool_cur = p->data + size;
1355 s->pool_end = p->data + p->size;
1356 return p->data;
1362 for (p = s->pool_first_large; p; p = t) {
1363 t = p->next;
1366 s->pool_first_large = NULL;
1367 s->pool_cur = s->pool_end = NULL;
1368 s->pool_current = NULL;
1372 * Create TCGHelperInfo structures for "tcg/tcg-ldst.h" functions,
1373 * akin to what "exec/helper-tcg.h" does with DEF_HELPER_FLAGS_N.
1482 unsigned typemask = info->typemask;
1490 /* Ignoring the return type, find the last non-zero field. */
1491 nargs = 32 - clz32(typemask >> 3);
1496 ca->cif.rtype = typecode_to_ffi(typemask & 7);
1497 ca->cif.nargs = nargs;
1500 ca->cif.arg_types = ca->args;
1503 ca->args[j] = typecode_to_ffi(typecode);
1507 status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs,
1508 ca->cif.rtype, ca->cif.arg_types);
1511 return &ca->cif;
1514 #define HELPER_INFO_INIT(I) (&(I)->cif)
1517 #define HELPER_INFO_INIT(I) (&(I)->init)
1534 unsigned stk_slot = arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs);
1549 cum->arg_slot += cum->arg_slot & 1;
1555 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
1559 .arg_idx = cum->arg_idx,
1560 .arg_slot = cum->arg_slot,
1562 cum->info_in_idx++;
1563 cum->arg_slot++;
1569 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
1575 .arg_idx = cum->arg_idx,
1577 .arg_slot = cum->arg_slot + i,
1580 cum->info_in_idx += n;
1581 cum->arg_slot += n;
1586 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx];
1594 * structure pass by-reference. Therefore we must make copies.
1598 loc[0].ref_slot = cum->ref_slot;
1607 .arg_idx = cum->arg_idx,
1609 .ref_slot = cum->ref_slot + i,
1612 cum->info_in_idx += n - 1; /* i=0 accounted for in layout_arg_1 */
1613 cum->ref_slot += n;
1620 unsigned typemask = info->typemask;
1630 info->nr_out = 0;
1635 info->nr_out = 1;
1636 info->out_kind = TCG_CALL_RET_NORMAL;
1640 info->nr_out = 64 / TCG_TARGET_REG_BITS;
1641 info->out_kind = TCG_CALL_RET_NORMAL;
1643 tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1);
1646 info->nr_out = 128 / TCG_TARGET_REG_BITS;
1647 info->out_kind = TCG_TARGET_CALL_RET_I128;
1651 tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1);
1755 info->nr_in = cum.info_in_idx;
1758 assert(cum.info_in_idx <= ARRAY_SIZE(info->in));
1765 * which has a signed 8-bit offset encoding.
1773 ref_base = cum.arg_slot - max_reg_slots;
1782 for (int i = cum.info_in_idx - 1; i >= 0; --i) {
1783 TCGCallArgumentLoc *loc = &info->in[i];
1784 switch (loc->kind) {
1787 loc->ref_slot += ref_base;
1809 s->nb_globals = 0;
1830 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i];
1838 * In user-mode we simply share the init context among threads, since we
1841 * In system-mode we will have at most max_threads TCG threads.
1852 tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0));
1874 tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align);
1877 if (unlikely(next > s->code_gen_highwater)) {
1883 qatomic_set(&s->code_gen_ptr, next);
1892 s->code_ptr = s->code_gen_ptr;
1893 s->code_buf = s->code_gen_ptr;
1894 s->data_gen_ptr = NULL;
1897 tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr);
1900 s->pool_labels = NULL;
1913 perf_report_prologue(s->code_gen_ptr, prologue_size);
1916 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
1917 (uintptr_t)s->code_buf, prologue_size);
1924 if (s->data_gen_ptr) {
1925 size_t code_size = s->data_gen_ptr - s->code_gen_ptr;
1926 size_t data_size = prologue_size - code_size;
1929 disas(logfile, s->code_gen_ptr, code_size);
1935 (uintptr_t)s->data_gen_ptr + i,
1936 *(uint64_t *)(s->data_gen_ptr + i));
1940 (uintptr_t)s->data_gen_ptr + i,
1941 *(uint32_t *)(s->data_gen_ptr + i));
1945 disas(logfile, s->code_gen_ptr, prologue_size);
1967 s->nb_temps = s->nb_globals;
1974 if (s->const_table[i]) {
1975 g_hash_table_remove_all(s->const_table[i]);
1979 s->nb_ops = 0;
1980 s->nb_labels = 0;
1981 s->current_frame_offset = s->frame_start;
1984 s->goto_tb_issue_mask = 0;
1987 QTAILQ_INIT(&s->ops);
1988 QTAILQ_INIT(&s->free_ops);
1989 s->emit_before_op = NULL;
1990 QSIMPLEQ_INIT(&s->labels);
1992 tcg_debug_assert(s->addr_type <= TCG_TYPE_REG);
1997 int n = s->nb_temps++;
2002 return memset(&s->temps[n], 0, sizeof(TCGTemp));
2009 tcg_debug_assert(s->nb_globals == s->nb_temps);
2010 tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS);
2011 s->nb_globals++;
2013 ts->kind = TEMP_GLOBAL;
2026 ts->base_type = type;
2027 ts->type = type;
2028 ts->kind = TEMP_FIXED;
2029 ts->reg = reg;
2030 ts->name = name;
2031 tcg_regset_set_reg(s->reserved_regs, reg);
2038 s->frame_start = start;
2039 s->frame_end = start + size;
2040 s->frame_temp
2052 switch (base_ts->kind) {
2056 /* We do not support double-indirect registers. */
2057 tcg_debug_assert(!base_ts->indirect_reg);
2058 base_ts->indirect_base = 1;
2059 s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64
2071 ts->base_type = TCG_TYPE_I64;
2072 ts->type = TCG_TYPE_I32;
2073 ts->indirect_reg = indirect_reg;
2074 ts->mem_allocated = 1;
2075 ts->mem_base = base_ts;
2076 ts->mem_offset = offset;
2079 ts->name = strdup(buf);
2082 ts2->base_type = TCG_TYPE_I64;
2083 ts2->type = TCG_TYPE_I32;
2084 ts2->indirect_reg = indirect_reg;
2085 ts2->mem_allocated = 1;
2086 ts2->mem_base = base_ts;
2087 ts2->mem_offset = offset + 4;
2088 ts2->temp_subindex = 1;
2091 ts2->name = strdup(buf);
2093 ts->base_type = type;
2094 ts->type = type;
2095 ts->indirect_reg = indirect_reg;
2096 ts->mem_allocated = 1;
2097 ts->mem_base = base_ts;
2098 ts->mem_offset = offset;
2099 ts->name = name;
2129 int idx = find_first_bit(s->free_temps[type].l, TCG_MAX_TEMPS);
2133 clear_bit(idx, s->free_temps[type].l);
2135 ts = &s->temps[idx];
2136 ts->temp_allocated = 1;
2137 tcg_debug_assert(ts->base_type == type);
2138 tcg_debug_assert(ts->kind == kind);
2163 ts->base_type = type;
2164 ts->temp_allocated = 1;
2165 ts->kind = kind;
2168 ts->type = type;
2170 ts->type = TCG_TYPE_REG;
2176 ts2->base_type = type;
2177 ts2->type = TCG_TYPE_REG;
2178 ts2->temp_allocated = 1;
2179 ts2->temp_subindex = i;
2180 ts2->kind = kind;
2255 tcg_debug_assert(t->temp_allocated != 0);
2257 t = tcg_temp_new_internal(t->base_type, TEMP_EBB);
2265 switch (ts->kind) {
2271 tcg_debug_assert(ts->temp_allocated != 0);
2272 ts->temp_allocated = 0;
2273 set_bit(temp_idx(ts), s->free_temps[ts->base_type].l);
2309 GHashTable *h = s->const_table[type];
2314 s->const_table[type] = h;
2328 ts->base_type = TCG_TYPE_I64;
2329 ts->type = TCG_TYPE_I32;
2330 ts->kind = TEMP_CONST;
2331 ts->temp_allocated = 1;
2333 ts2->base_type = TCG_TYPE_I64;
2334 ts2->type = TCG_TYPE_I32;
2335 ts2->kind = TEMP_CONST;
2336 ts2->temp_allocated = 1;
2337 ts2->temp_subindex = 1;
2340 * Retain the full value of the 64-bit constant in the low
2348 ts->base_type = type;
2349 ts->type = type;
2350 ts->kind = TEMP_CONST;
2351 ts->temp_allocated = 1;
2352 ts->val = val;
2353 val_ptr = &ts->val;
2391 tcg_debug_assert(t->temp_allocated != 0);
2392 return tcg_constant_vec(t->base_type, vece, val);
2398 ptrdiff_t n = ts - tcg_ctx->temps;
2399 assert(n >= 0 && n < tcg_ctx->nb_temps);
2405 uintptr_t o = (uintptr_t)v - offsetof(TCGContext, temps);
2407 assert(o < sizeof(TCGTemp) * tcg_ctx->nb_temps);
2583 con_set = outop->static_constraint;
2585 con_set = outop->dynamic_constraint(type, flags);
2610 tcg_debug_assert(len <= width - ofs);
2630 total_args = info->nr_out + info->nr_in + 2;
2635 if (tcg_ctx->plugin_insn && !(info->flags & TCG_CALL_NO_SIDE_EFFECTS)) {
2636 tcg_ctx->plugin_insn->calls_helpers = true;
2640 TCGOP_CALLO(op) = n = info->nr_out;
2647 op->args[pi++] = temp_arg(ret);
2652 tcg_debug_assert(ret->base_type == ret->type + ctz32(n));
2653 tcg_debug_assert(ret->temp_subindex == 0);
2655 op->args[pi++] = temp_arg(ret + i);
2662 TCGOP_CALLI(op) = n = info->nr_in;
2664 const TCGCallArgumentLoc *loc = &info->in[i];
2665 TCGTemp *ts = args[loc->arg_idx] + loc->tmp_subindex;
2667 switch (loc->kind) {
2671 op->args[pi++] = temp_arg(ts);
2680 if (loc->kind == TCG_CALL_ARG_EXTEND_S) {
2685 op->args[pi++] = tcgv_i64_arg(temp);
2694 op->args[pi++] = (uintptr_t)func;
2695 op->args[pi++] = (uintptr_t)info;
2698 if (tcg_ctx->emit_before_op) {
2699 QTAILQ_INSERT_BEFORE(tcg_ctx->emit_before_op, op, link);
2701 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
2768 for (i = 0, n = s->nb_temps; i < n; i++) {
2769 TCGTemp *ts = &s->temps[i];
2772 switch (ts->kind) {
2785 ts->mem_allocated = 0;
2790 ts->val_type = val;
2793 memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp));
2801 switch (ts->kind) {
2804 pstrcpy(buf, buf_size, ts->name);
2807 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals);
2810 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals);
2813 switch (ts->type) {
2815 snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val);
2819 snprintf(buf, buf_size, "$0x%" PRIx64, ts->val);
2826 64 << (ts->type - TCG_TYPE_V64), ts->val);
2908 "from-tb",
2909 "from-insn",
2910 "after-insn",
2911 "after-tb",
2917 return (d & (d - 1)) == 0;
2929 /* Return only the number of characters output -- no error return. */
2938 QTAILQ_FOREACH(op, &s->ops, link) {
2944 c = op->opc;
2949 col += ne_fprintf(f, "\n ----");
2962 nb_cargs = def->nb_cargs;
2964 col += ne_fprintf(f, " %s ", def->name);
2971 if (func == info->func) {
2972 col += ne_fprintf(f, "%s", info->name);
2977 col += ne_fprintf(f, ",$0x%x,$%d", info->flags, nb_oargs);
2980 op->args[i]));
2983 TCGArg arg = op->args[nb_oargs + i];
2988 if (def->flags & TCG_OPF_INT) {
2990 def->name,
2992 } else if (def->flags & TCG_OPF_VECTOR) {
2994 def->name,
2998 col += ne_fprintf(f, " %s ", def->name);
3001 nb_oargs = def->nb_oargs;
3002 nb_iargs = def->nb_iargs;
3003 nb_cargs = def->nb_cargs;
3010 op->args[k++]));
3016 op->args[k++]));
3027 if (op->args[k] < ARRAY_SIZE(cond_name)
3028 && cond_name[op->args[k]]) {
3029 col += ne_fprintf(f, ",%s", cond_name[op->args[k++]]);
3031 col += ne_fprintf(f, ",$0x%" TCG_PRIlx, op->args[k++]);
3041 MemOpIdx oi = op->args[k++];
3065 TCGArg flags = op->args[k];
3082 TCGArg from = op->args[k++];
3107 arg_label(op->args[k])->id);
3112 TCGBar membar = op->args[k];
3194 op->args[k]);
3198 if (have_prefs || op->life) {
3204 if (op->life) {
3205 unsigned life = op->life;
3262 n = ctpop64(arg_ct->regs);
3268 if (n == 1 || arg_ct->oalias) {
3277 switch (arg_ct->pair) {
3282 return (arg_ct->pair_index + 1) * 2 - 1;
3287 return -n;
3301 for (i = 0; i < n - 1; i++) {
3322 int nb_oargs = tdefs->nb_oargs;
3323 int nb_iargs = tdefs->nb_iargs;
3328 const char *ct_str = tdefs->args_ct_str[i];
3334 o = *ct_str - '0';
3361 o = i - 1;
3378 o = i - 1;
3404 /* Include all of the target-specific constraints. */
3412 #include "tcg-target-con-str.h"
3451 * Since [0-9pm] must be alone in the constraint string,
3510 TCGOpcode opc = op->opc;
3517 if (def->flags & TCG_OPF_NOT_PRESENT) {
3522 con_set = outop->static_constraint;
3524 con_set = outop->dynamic_constraint(type, flags);
3533 tcg_debug_assert(constraint_sets[con_set].nb_oargs == def->nb_oargs);
3534 tcg_debug_assert(constraint_sets[con_set].nb_iargs == def->nb_iargs);
3541 TCGLabel *label = arg_label(op->args[idx]);
3544 QSIMPLEQ_FOREACH(use, &label->branches, next) {
3545 if (use->op == op) {
3546 QSIMPLEQ_REMOVE(&label->branches, use, TCGLabelUse, next);
3555 switch (op->opc) {
3569 QTAILQ_REMOVE(&s->ops, op, link);
3570 QTAILQ_INSERT_TAIL(&s->free_ops, op, link);
3571 s->nb_ops--;
3592 if (unlikely(!QTAILQ_EMPTY(&s->free_ops))) {
3593 QTAILQ_FOREACH(op, &s->free_ops, link) {
3594 if (nargs <= op->nargs) {
3595 QTAILQ_REMOVE(&s->free_ops, op, link);
3596 nargs = op->nargs;
3608 op->opc = opc;
3609 op->nargs = nargs;
3612 tcg_debug_assert(op->nargs == nargs);
3614 s->nb_ops++;
3622 if (tcg_ctx->emit_before_op) {
3623 QTAILQ_INSERT_BEFORE(tcg_ctx->emit_before_op, op, link);
3625 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
3646 QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link);
3654 QSIMPLEQ_FOREACH(u, &from->branches, next) {
3655 TCGOp *op = u->op;
3656 switch (op->opc) {
3658 op->args[0] = label_arg(to);
3661 op->args[3] = label_arg(to);
3664 op->args[5] = label_arg(to);
3671 QSIMPLEQ_CONCAT(&to->branches, &from->branches);
3681 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
3685 switch (op->opc) {
3687 label = arg_label(op->args[0]);
3701 if (op_prev->opc == INDEX_op_set_label) {
3702 move_label_uses(label, arg_label(op_prev->args[0]));
3711 * processing the branch because any dead code between the branch
3714 if (op_prev->opc == INDEX_op_br &&
3715 label == arg_label(op_prev->args[0])) {
3721 if (QSIMPLEQ_EMPTY(&label->branches)) {
3752 /* Never remove -- we need to keep these for unwind. */
3775 return ts->state_ptr;
3784 = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]);
3794 s->temps[i].state = TS_DEAD | TS_MEM;
3795 la_reset_pref(&s->temps[i]);
3798 s->temps[i].state = TS_DEAD;
3799 la_reset_pref(&s->temps[i]);
3810 TCGTemp *ts = &s->temps[i];
3813 switch (ts->kind) {
3826 ts->state = state;
3837 int state = s->temps[i].state;
3838 s->temps[i].state = state | TS_MEM;
3841 la_reset_pref(&s->temps[i]);
3848 * explicitly live-across-conditional-branch, globals and local temps
3856 TCGTemp *ts = &s->temps[i];
3859 switch (ts->kind) {
3861 state = ts->state;
3862 ts->state = state | TS_MEM;
3873 la_reset_pref(&s->temps[i]);
3883 s->temps[i].state = TS_DEAD | TS_MEM;
3884 la_reset_pref(&s->temps[i]);
3895 TCGTemp *ts = &s->temps[i];
3896 if (!(ts->state & TS_DEAD)) {
3903 set = tcg_target_available_regs[ts->type] & mask;
3917 void * const multiple_ebb = (void *)(uintptr_t)-1;
3918 int nb_temps = s->nb_temps;
3921 for (int i = s->nb_globals; i < nb_temps; ++i) {
3922 s->temps[i].state_ptr = NULL;
3931 ebb = QTAILQ_FIRST(&s->ops);
3932 QTAILQ_FOREACH(op, &s->ops, link) {
3936 switch (op->opc) {
3947 def = &tcg_op_defs[op->opc];
3948 nb_oargs = def->nb_oargs;
3949 nb_iargs = def->nb_iargs;
3954 TCGTemp *ts = arg_temp(op->args[i]);
3956 if (ts->kind != TEMP_TB) {
3959 if (ts->state_ptr == NULL) {
3960 ts->state_ptr = ebb;
3961 } else if (ts->state_ptr != ebb) {
3962 ts->state_ptr = multiple_ebb;
3971 for (int i = s->nb_globals; i < nb_temps; ++i) {
3972 TCGTemp *ts = &s->temps[i];
3973 if (ts->kind == TEMP_TB && ts->state_ptr != multiple_ebb) {
3974 ts->kind = TEMP_EBB;
3984 * carry-out operations should always be paired with carry-in.
3987 tcg_debug_assert(!s->carry_live);
3996 int nb_globals = s->nb_globals;
3997 int nb_temps = s->nb_temps;
4003 s->temps[i].state_ptr = prefs + i;
4009 s->carry_live = false;
4010 QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) {
4015 TCGOpcode opc = op->opc;
4032 ts = arg_temp(op->args[i]);
4033 if (ts->state != TS_DEAD) {
4043 ts = arg_temp(op->args[i]);
4044 if (ts->state & TS_DEAD) {
4047 if (ts->state & TS_MEM) {
4050 ts->state = TS_DEAD;
4054 /* Not used -- it will be tcg_target_call_oarg_reg(). */
4055 memset(op->output_pref, 0, sizeof(op->output_pref));
4066 ts = arg_temp(op->args[i]);
4067 if (ts->state & TS_DEAD) {
4072 /* For all live registers, remove call-clobbered prefs. */
4085 for (int i = nb_iargs - 1; i >= 0; i--) {
4086 const TCGCallArgumentLoc *loc = &info->in[i];
4087 ts = arg_temp(op->args[nb_oargs + i]);
4089 if (ts->state & TS_DEAD) {
4090 switch (loc->kind) {
4094 if (arg_slot_reg_p(loc->arg_slot)) {
4101 tcg_target_available_regs[ts->type];
4104 ts->state &= ~TS_DEAD;
4114 const TCGCallArgumentLoc *loc = &info->in[i];
4115 ts = arg_temp(op->args[nb_oargs + i]);
4117 switch (loc->kind) {
4121 if (arg_slot_reg_p(loc->arg_slot)) {
4123 tcg_target_call_iarg_regs[loc->arg_slot]);
4137 ts = arg_temp(op->args[0]);
4138 ts->state = TS_DEAD;
4151 if (arg_temp(op->args[1])->state == TS_DEAD) {
4152 if (arg_temp(op->args[0])->state == TS_DEAD) {
4157 op->opc = opc = opc_new;
4158 op->args[1] = op->args[2];
4159 op->args[2] = op->args[3];
4160 } else if (arg_temp(op->args[0])->state == TS_DEAD &&
4163 op->opc = opc = opc_new2;
4164 op->args[0] = op->args[1];
4165 op->args[1] = op->args[2];
4166 op->args[2] = op->args[3];
4170 /* Mark the single-word operation live. */
4174 if (s->carry_live) {
4177 op->opc = opc = INDEX_op_add;
4181 if (s->carry_live) {
4184 op->opc = opc = INDEX_op_addci;
4188 if (s->carry_live) {
4192 op->opc = opc = INDEX_op_sub;
4193 ts = arg_temp(op->args[2]);
4194 if (ts->kind == TEMP_CONST) {
4195 ts = tcg_constant_internal(ts->type, -ts->val);
4196 if (ts->state_ptr == NULL) {
4199 ts->state_ptr = tcg_malloc(sizeof(TCGRegSet));
4200 ts->state = TS_DEAD;
4203 op->args[2] = temp_arg(ts);
4204 op->opc = opc = INDEX_op_add;
4209 if (s->carry_live) {
4212 op->opc = opc = INDEX_op_subbi;
4216 if (s->carry_live) {
4222 op_prev->args[0] = op->args[0];
4223 op_prev->args[1] = op->args[1];
4224 op_prev->args[2] = op->args[2];
4225 op->opc = opc = INDEX_op_add;
4226 op->args[1] = op->args[0];
4227 ts = arg_temp(op->args[0]);
4228 ts = tcg_constant_internal(ts->type, 1);
4229 op->args[2] = temp_arg(ts);
4233 if (s->carry_live) {
4236 /* Lower to sub, add -1. */
4239 op_prev->args[0] = op->args[0];
4240 op_prev->args[1] = op->args[1];
4241 op_prev->args[2] = op->args[2];
4242 op->opc = opc = INDEX_op_add;
4243 op->args[1] = op->args[0];
4244 ts = arg_temp(op->args[0]);
4245 ts = tcg_constant_internal(ts->type, -1);
4246 op->args[2] = temp_arg(ts);
4257 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && def->nb_oargs != 0) {
4258 for (int i = def->nb_oargs - 1; i >= 0; i--) {
4259 if (arg_temp(op->args[i])->state != TS_DEAD) {
4273 nb_iargs = def->nb_iargs;
4274 nb_oargs = def->nb_oargs;
4277 ts = arg_temp(op->args[i]);
4280 if (i < ARRAY_SIZE(op->output_pref)) {
4281 op->output_pref[i] = *la_temp_pref(ts);
4285 if (ts->state & TS_DEAD) {
4288 if (ts->state & TS_MEM) {
4291 ts->state = TS_DEAD;
4296 if (def->flags & TCG_OPF_BB_EXIT) {
4299 } else if (def->flags & TCG_OPF_COND_BRANCH) {
4302 } else if (def->flags & TCG_OPF_BB_END) {
4305 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
4308 if (def->flags & TCG_OPF_CALL_CLOBBER) {
4315 ts = arg_temp(op->args[i]);
4316 if (ts->state & TS_DEAD) {
4320 if (def->flags & TCG_OPF_CARRY_OUT) {
4321 s->carry_live = false;
4326 ts = arg_temp(op->args[i]);
4327 if (ts->state & TS_DEAD) {
4330 *la_temp_pref(ts) = tcg_target_available_regs[ts->type];
4331 ts->state &= ~TS_DEAD;
4334 if (def->flags & TCG_OPF_CARRY_IN) {
4335 s->carry_live = true;
4345 *la_temp_pref(arg_temp(op->args[0]))
4346 = *la_temp_pref(arg_temp(op->args[1]));
4356 ts = arg_temp(op->args[i]);
4360 set &= ct->regs;
4361 if (ct->ialias) {
4362 set &= output_pref(op, ct->alias_index);
4366 set = ct->regs;
4374 op->life = arg_life;
4383 int nb_globals = s->nb_globals;
4390 TCGTemp *its = &s->temps[i];
4391 if (its->indirect_reg) {
4393 dts->type = its->type;
4394 dts->base_type = its->base_type;
4395 dts->temp_subindex = its->temp_subindex;
4396 dts->kind = TEMP_EBB;
4397 its->state_ptr = dts;
4399 its->state_ptr = NULL;
4402 its->state = TS_DEAD;
4404 for (nb_temps = s->nb_temps; i < nb_temps; ++i) {
4405 TCGTemp *its = &s->temps[i];
4406 its->state_ptr = NULL;
4407 its->state = TS_DEAD;
4410 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
4411 TCGOpcode opc = op->opc;
4413 TCGLifeData arg_life = op->life;
4422 nb_iargs = def->nb_iargs;
4423 nb_oargs = def->nb_oargs;
4426 if (def->flags & TCG_OPF_COND_BRANCH) {
4429 } else if (def->flags & TCG_OPF_BB_END) {
4432 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
4444 arg_ts = arg_temp(op->args[i]);
4445 dir_ts = arg_ts->state_ptr;
4446 if (dir_ts && arg_ts->state == TS_DEAD) {
4448 arg_ts->type, 3);
4450 lop->args[0] = temp_arg(dir_ts);
4451 lop->args[1] = temp_arg(arg_ts->mem_base);
4452 lop->args[2] = arg_ts->mem_offset;
4455 arg_ts->state = TS_MEM;
4463 arg_ts = arg_temp(op->args[i]);
4464 dir_ts = arg_ts->state_ptr;
4466 op->args[i] = temp_arg(dir_ts);
4469 arg_ts->state = TS_DEAD;
4482 arg_ts = &s->temps[i];
4483 tcg_debug_assert(arg_ts->state_ptr == 0
4484 || arg_ts->state != 0);
4490 arg_ts = &s->temps[i];
4491 tcg_debug_assert(arg_ts->state_ptr == 0
4492 || arg_ts->state == TS_DEAD);
4498 arg_ts = arg_temp(op->args[0]);
4499 dir_ts = arg_ts->state_ptr;
4501 op->args[0] = temp_arg(dir_ts);
4505 arg_ts->state = 0;
4509 arg_ts->type, 3);
4513 out_ts = arg_temp(op->args[1]);
4514 arg_ts->state = TS_DEAD;
4517 arg_ts->state = TS_MEM;
4520 sop->args[0] = temp_arg(out_ts);
4521 sop->args[1] = temp_arg(arg_ts->mem_base);
4522 sop->args[2] = arg_ts->mem_offset;
4529 arg_ts = arg_temp(op->args[i]);
4530 dir_ts = arg_ts->state_ptr;
4534 op->args[i] = temp_arg(dir_ts);
4538 arg_ts->state = 0;
4543 arg_ts->type, 3);
4545 sop->args[0] = temp_arg(dir_ts);
4546 sop->args[1] = temp_arg(arg_ts->mem_base);
4547 sop->args[2] = arg_ts->mem_offset;
4549 arg_ts->state = TS_MEM;
4553 arg_ts->state = TS_DEAD;
4568 size = tcg_type_size(ts->base_type);
4569 switch (ts->base_type) {
4598 off = ROUND_UP(s->current_frame_offset, align);
4601 if (off + size > s->frame_end) {
4604 s->current_frame_offset = off + size;
4610 if (ts->base_type != ts->type) {
4611 int part_size = tcg_type_size(ts->type);
4618 ts -= ts->temp_subindex;
4621 ts[i].mem_base = s->frame_temp;
4625 ts->mem_offset = off;
4626 ts->mem_base = s->frame_temp;
4627 ts->mem_allocated = 1;
4634 if (ts->val_type == TEMP_VAL_REG) {
4635 TCGReg old = ts->reg;
4636 tcg_debug_assert(s->reg_to_temp[old] == ts);
4640 s->reg_to_temp[old] = NULL;
4642 tcg_debug_assert(s->reg_to_temp[reg] == NULL);
4643 s->reg_to_temp[reg] = ts;
4644 ts->val_type = TEMP_VAL_REG;
4645 ts->reg = reg;
4648 /* Assign a non-register value type to @ts, and update reg_to_temp[]. */
4652 if (ts->val_type == TEMP_VAL_REG) {
4653 TCGReg reg = ts->reg;
4654 tcg_debug_assert(s->reg_to_temp[reg] == ts);
4655 s->reg_to_temp[reg] = NULL;
4657 ts->val_type = type;
4668 switch (ts->kind) {
4695 is non-zero, subsequently release the temporary; if it is positive, the
4700 if (!temp_readonly(ts) && !ts->mem_coherent) {
4701 if (!ts->mem_allocated) {
4704 switch (ts->val_type) {
4710 && tcg_out_sti(s, ts->type, ts->val,
4711 ts->mem_base->reg, ts->mem_offset)) {
4714 temp_load(s, ts, tcg_target_available_regs[ts->type],
4719 tcg_out_st(s, ts->type, ts->reg,
4720 ts->mem_base->reg, ts->mem_offset);
4730 ts->mem_coherent = 1;
4740 TCGTemp *ts = s->reg_to_temp[reg];
4742 temp_sync(s, ts, allocated_regs, 0, -1);
4781 if (s->reg_to_temp[reg] == NULL) {
4787 if (s->reg_to_temp[reg] == NULL &&
4843 for (fmin = 2; fmin >= 0; fmin--) {
4851 int f = !s->reg_to_temp[reg] + !s->reg_to_temp[reg + 1];
4871 switch (ts->val_type) {
4876 preferred_regs, ts->indirect_base);
4877 if (ts->type <= TCG_TYPE_I64) {
4878 tcg_out_movi(s, ts->type, reg, ts->val);
4880 uint64_t val = ts->val;
4896 tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val);
4898 ts->mem_coherent = 0;
4901 if (!ts->mem_allocated) {
4905 preferred_regs, ts->indirect_base);
4906 tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset);
4907 ts->mem_coherent = 1;
4922 tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts));
4932 for (i = 0, n = s->nb_globals; i < n; i++) {
4933 temp_save(s, &s->temps[i], allocated_regs);
4944 for (i = 0, n = s->nb_globals; i < n; i++) {
4945 TCGTemp *ts = &s->temps[i];
4946 tcg_debug_assert(ts->val_type != TEMP_VAL_REG
4947 || ts->kind == TEMP_FIXED
4948 || ts->mem_coherent);
4957 for (int i = s->nb_globals; i < s->nb_temps; i++) {
4958 TCGTemp *ts = &s->temps[i];
4960 switch (ts->kind) {
4967 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
4971 tcg_debug_assert(ts->val_type == TEMP_VAL_CONST);
4983 * explicitly live-across-conditional-branch; all globals and local
4991 for (int i = s->nb_globals; i < s->nb_temps; i++) {
4992 TCGTemp *ts = &s->temps[i];
4997 switch (ts->kind) {
4999 tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent);
5022 ots->val = val;
5023 ots->mem_coherent = 0;
5025 temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0));
5036 const TCGLifeData arg_life = op->life;
5042 allocated_regs = s->reserved_regs;
5044 ots = arg_temp(op->args[0]);
5045 ts = arg_temp(op->args[1]);
5050 /* Note that otype != itype for no-op truncation. */
5051 otype = ots->type;
5052 itype = ts->type;
5054 if (ts->val_type == TEMP_VAL_CONST) {
5056 tcg_target_ulong val = ts->val;
5068 if (ts->val_type == TEMP_VAL_MEM) {
5072 tcg_debug_assert(ts->val_type == TEMP_VAL_REG);
5073 ireg = ts->reg;
5076 /* mov to a non-saved dead register makes no sense (even with
5079 if (!ots->mem_allocated) {
5082 tcg_out_st(s, otype, ireg, ots->mem_base->reg, ots->mem_offset);
5090 if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) {
5099 if (ots->val_type == TEMP_VAL_REG) {
5100 oreg = ots->reg;
5105 preferred_regs, ots->indirect_base);
5114 if (!ts->mem_allocated) {
5117 tcg_out_st(s, ts->type, ireg, ots->mem_base->reg, ots->mem_offset);
5119 ots->mem_coherent = 1;
5124 ots->mem_coherent = 0;
5136 const TCGLifeData arg_life = op->life;
5145 ots = arg_temp(op->args[0]);
5146 its = arg_temp(op->args[1]);
5151 itype = its->type;
5155 if (its->val_type == TEMP_VAL_CONST) {
5156 /* Propagate constant via movi -> dupi. */
5157 tcg_target_ulong val = its->val;
5170 if (ots->val_type != TEMP_VAL_REG) {
5171 TCGRegSet allocated_regs = s->reserved_regs;
5174 if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) {
5176 tcg_regset_set_reg(allocated_regs, its->reg);
5179 output_pref(op, 0), ots->indirect_base);
5183 switch (its->val_type) {
5190 if (tcg_regset_test_reg(dup_in_regs, its->reg)) {
5191 if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) {
5196 if (!its->mem_coherent) {
5199 * would be required to use memory. Attempt an integer-vector
5202 if (tcg_out_mov(s, itype, ots->reg, its->reg)) {
5206 temp_sync(s, its, s->reserved_regs, 0, 0);
5213 lowpart_ofs = tcg_type_size(itype) - (1 << vece);
5215 if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg,
5216 its->mem_offset + lowpart_ofs)) {
5220 tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset);
5228 ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg);
5232 ots->mem_coherent = 0;
5237 temp_sync(s, ots, s->reserved_regs, 0, 0);
5246 const TCGLifeData arg_life = op->life;
5247 const TCGOpDef * const def = &tcg_op_defs[op->opc];
5260 if (def->flags & TCG_OPF_CARRY_IN) {
5261 tcg_debug_assert(s->carry_live);
5264 nb_oargs = def->nb_oargs;
5265 nb_iargs = def->nb_iargs;
5269 op->args + nb_oargs + nb_iargs,
5270 sizeof(TCGArg) * def->nb_cargs);
5272 i_allocated_regs = s->reserved_regs;
5273 o_allocated_regs = s->reserved_regs;
5275 switch (op->opc) {
5277 op_cond = op->args[2];
5282 op_cond = op->args[3];
5285 op_cond = op->args[4];
5290 op_cond = op->args[5];
5308 arg = op->args[i];
5312 if (ts->val_type == TEMP_VAL_CONST) {
5314 if (ts->val == 0 && (arg_ct->ct & TCG_CT_REG_ZERO)) {
5315 /* Hardware zero register: indicate register via non-const. */
5322 if (tcg_target_const_match(ts->val, arg_ct->ct, ts->type,
5326 new_args[i] = ts->val;
5331 reg = ts->reg;
5333 i_required_regs = arg_ct->regs;
5337 switch (arg_ct->pair) {
5339 if (arg_ct->ialias) {
5340 i_preferred_regs = output_pref(op, arg_ct->alias_index);
5349 || args_ct[arg_ct->alias_index].newreg) {
5351 } else if (ts->val_type == TEMP_VAL_REG) {
5363 reg = ts->reg;
5371 temp_load(s, ts, tcg_target_available_regs[ts->type],
5374 i_preferred_regs, ts->indirect_base);
5382 i2 = arg_ct->pair_index;
5383 ts2 = i1 != i2 ? arg_temp(op->args[i2]) : NULL;
5389 if (arg_ct->ialias) {
5390 i_preferred_regs = output_pref(op, arg_ct->alias_index);
5394 ts->val_type == TEMP_VAL_REG &&
5395 ts->reg < TCG_TARGET_NB_REGS - 1 &&
5400 ? ts2->val_type == TEMP_VAL_REG &&
5401 ts2->reg == reg + 1 &&
5403 : s->reg_to_temp[reg + 1] == NULL)) {
5409 if (ts->val_type == TEMP_VAL_REG &&
5410 ts2->val_type == TEMP_VAL_REG &&
5411 ts2->reg == reg + 1 &&
5417 0, ts->indirect_base);
5421 reg = new_args[arg_ct->pair_index] + 1;
5425 tcg_debug_assert(arg_ct->ialias);
5426 i_preferred_regs = output_pref(op, arg_ct->alias_index);
5430 ts->val_type == TEMP_VAL_REG &&
5432 s->reg_to_temp[reg - 1] == NULL &&
5435 !tcg_regset_test_reg(i_allocated_regs, reg - 1)) {
5436 tcg_regset_set_reg(i_allocated_regs, reg - 1);
5441 ts->indirect_base);
5451 if (arg_ct->ialias && (!IS_DEAD_ARG(i) || temp_readonly(ts))) {
5461 if (ts->val_type == TEMP_VAL_REG && ts->reg == reg) {
5466 tcg_debug_assert(ts->kind != TEMP_FIXED);
5467 nr = tcg_reg_alloc(s, tcg_target_available_regs[ts->type],
5468 t_allocated_regs, 0, ts->indirect_base);
5469 ok = tcg_out_mov(s, ts->type, nr, reg);
5474 temp_load(s, ts, tcg_target_available_regs[ts->type],
5483 copyto_new_reg = ts->reg != reg;
5492 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
5498 tcg_out_ld(s, ts->type, reg,
5499 ts->mem_base->reg, ts->mem_offset);
5510 temp_dead(s, arg_temp(op->args[i]));
5514 if (def->flags & TCG_OPF_COND_BRANCH) {
5516 } else if (def->flags & TCG_OPF_BB_END) {
5519 if (def->flags & TCG_OPF_CALL_CLOBBER) {
5528 if (def->flags & TCG_OPF_SIDE_EFFECTS) {
5537 arg = op->args[i];
5544 switch (arg_ct->pair) {
5546 if (arg_ct->oalias && !const_args[arg_ct->alias_index]) {
5547 reg = new_args[arg_ct->alias_index];
5548 } else if (arg_ct->newreg) {
5549 reg = tcg_reg_alloc(s, arg_ct->regs,
5551 output_pref(op, k), ts->indirect_base);
5553 reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs,
5554 output_pref(op, k), ts->indirect_base);
5559 if (arg_ct->oalias) {
5560 reg = new_args[arg_ct->alias_index];
5561 } else if (arg_ct->newreg) {
5562 reg = tcg_reg_alloc_pair(s, arg_ct->regs,
5565 ts->indirect_base);
5567 reg = tcg_reg_alloc_pair(s, arg_ct->regs, o_allocated_regs,
5569 ts->indirect_base);
5574 if (arg_ct->oalias) {
5575 reg = new_args[arg_ct->alias_index];
5577 reg = new_args[arg_ct->pair_index] + 1;
5582 tcg_debug_assert(!arg_ct->newreg);
5583 reg = new_args[arg_ct->pair_index] - 1;
5591 ts->mem_coherent = 0;
5598 switch (op->opc) {
5629 container_of(all_outop[op->opc], TCGOutOpBinary, base);
5634 out->out_rri(s, type, new_args[0], new_args[1], new_args[2]);
5636 out->out_rrr(s, type, new_args[0], new_args[1], new_args[2]);
5651 out->out_rir(s, type, new_args[0], new_args[1], new_args[2]);
5653 out->out_rrr(s, type, new_args[0], new_args[1], new_args[2]);
5667 container_of(all_outop[op->opc], TCGOutOpAddSubCarry, base);
5671 out->out_rii(s, type, new_args[0],
5674 out->out_rri(s, type, new_args[0],
5678 out->out_rir(s, type, new_args[0], new_args[1], new_args[2]);
5680 out->out_rrr(s, type, new_args[0], new_args[1], new_args[2]);
5697 container_of(all_outop[op->opc], TCGOutOpUnary, base);
5701 out->out_rr(s, type, new_args[0], new_args[1]);
5709 container_of(all_outop[op->opc], TCGOutOpBswap, base);
5712 out->out_rr(s, type, new_args[0], new_args[1], new_args[2]);
5722 out->out_rri(s, type, new_args[0], new_args[1],
5727 out->out_rzr(s, type, new_args[0], new_args[2],
5730 out->out_rrr(s, type, new_args[0], new_args[1],
5740 container_of(all_outop[op->opc], TCGOutOpDivRem, base);
5746 out->out_rr01r(s, type, new_args[0], new_args[1], new_args[4]);
5754 container_of(all_outop[op->opc], TCGOutOpExtract, base);
5757 out->out_rr(s, type, new_args[0], new_args[1],
5768 out->out_rrr(s, type, new_args[0], new_args[1],
5782 container_of(all_outop[op->opc], TCGOutOpLoad, base);
5785 out->out(s, type, new_args[0], new_args[1], new_args[2]);
5793 container_of(all_outop[op->opc], TCGOutOpMul2, base);
5797 out->out_rrrr(s, type, new_args[0], new_args[1],
5811 container_of(all_outop[op->opc], TCGOutOpStore, base);
5814 out->out_i(s, type, new_args[0], new_args[1], new_args[2]);
5816 out->out_r(s, type, new_args[0], new_args[1], new_args[2]);
5825 container_of(all_outop[op->opc], TCGOutOpQemuLdSt, base);
5827 out->out(s, type, new_args[0], new_args[1], new_args[2]);
5835 container_of(all_outop[op->opc], TCGOutOpQemuLdSt2, base);
5837 out->out(s, type, new_args[0], new_args[1],
5850 out->out_ri(s, type, cond, new_args[0], new_args[1], label);
5852 out->out_rr(s, type, cond, new_args[0], new_args[1], label);
5863 out->out(s, type, cond, new_args[0],
5874 container_of(all_outop[op->opc], TCGOutOpSetcond, base);
5879 out->out_rri(s, type, cond,
5882 out->out_rrr(s, type, cond,
5897 out->out(s, cond, new_args[0], new_args[1],
5909 out->out(s, cond, new_args[0], new_args[1], new_args[2],
5925 tcg_debug_assert(def->flags & TCG_OPF_VECTOR);
5926 tcg_out_vec_op(s, op->opc, type - TCG_TYPE_V64,
5931 if (def->flags & TCG_OPF_CARRY_IN) {
5932 s->carry_live = false;
5934 if (def->flags & TCG_OPF_CARRY_OUT) {
5935 s->carry_live = true;
5940 ts = arg_temp(op->args[i]);
5955 const TCGLifeData arg_life = op->life;
5959 /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */
5963 ots = arg_temp(op->args[0]);
5964 itsl = arg_temp(op->args[1]);
5965 itsh = arg_temp(op->args[2]);
5971 if (ots->val_type != TEMP_VAL_REG) {
5972 TCGRegSet allocated_regs = s->reserved_regs;
5977 if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) {
5978 tcg_regset_set_reg(allocated_regs, itsl->reg);
5980 if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) {
5981 tcg_regset_set_reg(allocated_regs, itsh->reg);
5985 output_pref(op, 0), ots->indirect_base);
5990 if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) {
5991 uint64_t val = deposit64(itsl->val, 32, 32, itsh->val);
6002 tcg_out_dupi_vec(s, vtype, vece, ots->reg, val);
6006 /* If the two inputs form one 64-bit value, try dupm_vec. */
6007 if (itsl->temp_subindex == HOST_BIG_ENDIAN &&
6008 itsh->temp_subindex == !HOST_BIG_ENDIAN &&
6009 itsl == itsh + (HOST_BIG_ENDIAN ? 1 : -1)) {
6010 TCGTemp *its = itsl - HOST_BIG_ENDIAN;
6012 temp_sync(s, its + 0, s->reserved_regs, 0, 0);
6013 temp_sync(s, its + 1, s->reserved_regs, 0, 0);
6015 if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg,
6016 its->mem_base->reg, its->mem_offset)) {
6025 ots->mem_coherent = 0;
6033 temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0));
6043 if (ts->val_type == TEMP_VAL_REG) {
6044 if (ts->reg != reg) {
6046 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
6052 tcg_out_ld(s, ts->type, reg,
6053 ts->mem_base->reg, ts->mem_offset);
6070 * If there are many call-saved registers, the temp might live to
6073 temp_load(s, ts, tcg_target_available_regs[ts->type], allocated_regs, 0);
6074 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK,
6081 if (arg_slot_reg_p(l->arg_slot)) {
6082 TCGReg reg = tcg_target_call_iarg_regs[l->arg_slot];
6086 load_arg_stk(s, l->arg_slot, ts, *allocated_regs);
6113 const TCGLifeData arg_life = op->life;
6115 TCGRegSet allocated_regs = s->reserved_regs;
6122 for (i = nb_iargs - 1; i >= 0; --i) {
6123 const TCGCallArgumentLoc *loc = &info->in[i];
6124 TCGTemp *ts = arg_temp(op->args[nb_oargs + i]);
6126 switch (loc->kind) {
6133 load_arg_stk(s, loc->ref_slot, ts, allocated_regs);
6134 load_arg_ref(s, loc->arg_slot, TCG_REG_CALL_STACK,
6135 arg_slot_stk_ofs(loc->ref_slot),
6139 load_arg_stk(s, loc->ref_slot, ts, allocated_regs);
6149 temp_dead(s, arg_temp(op->args[i]));
6164 if (info->flags & TCG_CALL_NO_READ_GLOBALS) {
6166 } else if (info->flags & TCG_CALL_NO_WRITE_GLOBALS) {
6176 if (info->out_kind == TCG_CALL_RET_BY_REF) {
6177 TCGTemp *ts = arg_temp(op->args[0]);
6179 if (!ts->mem_allocated) {
6182 load_arg_ref(s, 0, ts->mem_base->reg, ts->mem_offset, &allocated_regs);
6188 switch (info->out_kind) {
6191 TCGTemp *ts = arg_temp(op->args[i]);
6198 ts->mem_coherent = 0;
6204 TCGTemp *ts = arg_temp(op->args[0]);
6206 tcg_debug_assert(ts->base_type == TCG_TYPE_I128);
6207 tcg_debug_assert(ts->temp_subindex == 0);
6208 if (!ts->mem_allocated) {
6213 ts->mem_base->reg, ts->mem_offset);
6220 TCGTemp *ts = arg_temp(op->args[i]);
6221 ts->val_type = TEMP_VAL_MEM;
6231 TCGTemp *ts = arg_temp(op->args[i]);
6233 temp_sync(s, ts, s->reserved_regs, 0, IS_DEAD_ARG(i));
6260 MemOp half = size ? size - 1 : 0;
6321 * We must re-implement tcg_gen_callN and tcg_reg_alloc_call simultaneously,
6352 for (i = nmov; i-- > 0; ) {
6365 tcg_debug_assert(parm->ntmp != 0);
6366 mov[i].dst = src = parm->tmp[0];
6388 tcg_debug_assert(parm->ntmp >= 2);
6398 TCGReg scratch = parm->tmp[1];
6401 tcg_out_movext3(s, mov, mov + 1, mov + 2, parm->tmp[0]);
6413 parm->ntmp ? parm->tmp[0] : -1);
6417 parm->ntmp ? parm->tmp[0] : -1);
6436 tcg_debug_assert(parm->ntmp != 0);
6437 tcg_out_movi(s, type, parm->tmp[0], imm);
6438 tcg_out_st(s, type, parm->tmp[0], TCG_REG_CALL_STACK, ofs);
6454 const TCGCallArgumentLoc *loc = &info->in[0];
6462 ptr_mov.dst = loc->arg_slot;
6469 imm = ldst->oi;
6470 loc = &info->in[next_arg];
6472 switch (loc->kind) {
6484 tcg_out_helper_load_imm(s, loc->arg_slot, type, imm, parm);
6490 loc = &info->in[next_arg];
6491 slot = loc->arg_slot;
6492 if (parm->ra_gen) {
6493 int arg_reg = -1;
6499 ra_reg = parm->ra_gen(s, ldst, arg_reg);
6505 imm = (uintptr_t)ldst->raddr;
6520 switch (loc->kind) {
6536 mov[0].dst = loc->arg_slot;
6574 MemOp mop = get_memop(ldst->oi);
6595 loc = &info->in[next_arg];
6596 if (TCG_TARGET_REG_BITS == 32 && s->addr_type == TCG_TYPE_I32) {
6598 * 32-bit host with 32-bit guest: zero-extend the guest address
6599 * to 64-bits for the helper by storing the low part, then
6604 ldst->addr_reg, -1);
6611 nmov = tcg_out_helper_add_mov(mov, loc, TCG_TYPE_I64, s->addr_type,
6612 ldst->addr_reg, -1);
6617 switch (info->out_kind) {
6624 * We need memory in which to return: re-use the top of stack.
6633 tcg_debug_assert(parm->ntmp != 0);
6634 tcg_out_addi_ptr(s, parm->tmp[0],
6636 tcg_out_st(s, TCG_TYPE_PTR, parm->tmp[0],
6652 MemOp mop = get_memop(ldst->oi);
6656 switch (ldst->type) {
6664 mov[0].dst = ldst->datalo_reg;
6666 mov[0].dst_type = ldst->type;
6680 if (TCG_TARGET_REG_BITS == 32 || ldst->type == TCG_TYPE_I32) {
6703 tcg_out_ld(s, TCG_TYPE_I64, ldst->datalo_reg,
6705 tcg_out_ld(s, TCG_TYPE_I64, ldst->datahi_reg,
6717 mov[0].dst = ldst->datalo_reg;
6724 mov[1].dst = ldst->datahi_reg;
6731 tcg_out_movext2(s, mov, mov + 1, parm->ntmp ? parm->tmp[0] : -1);
6742 MemOp mop = get_memop(ldst->oi);
6768 loc = &info->in[next_arg];
6769 tcg_debug_assert(s->addr_type <= TCG_TYPE_REG);
6772 * 32-bit host (and thus 32-bit guest): zero-extend the guest address
6773 * to 64-bits for the helper by storing the low part. Later,
6779 ldst->addr_reg, -1);
6783 n = tcg_out_helper_add_mov(mov, loc, TCG_TYPE_I64, s->addr_type,
6784 ldst->addr_reg, -1);
6790 loc = &info->in[next_arg];
6791 switch (loc->kind) {
6795 n = tcg_out_helper_add_mov(mov + nmov, loc, data_type, ldst->type,
6796 ldst->datalo_reg, ldst->datahi_reg);
6806 HOST_BIG_ENDIAN ? ldst->datahi_reg : ldst->datalo_reg,
6809 HOST_BIG_ENDIAN ? ldst->datalo_reg : ldst->datahi_reg,
6814 if (arg_slot_reg_p(loc->arg_slot)) {
6815 tcg_out_addi_ptr(s, tcg_target_call_iarg_regs[loc->arg_slot],
6817 arg_slot_stk_ofs(loc->ref_slot));
6819 tcg_debug_assert(parm->ntmp != 0);
6820 tcg_out_addi_ptr(s, parm->tmp[0], TCG_REG_CALL_STACK,
6821 arg_slot_stk_ofs(loc->ref_slot));
6822 tcg_out_st(s, TCG_TYPE_PTR, parm->tmp[0],
6823 TCG_REG_CALL_STACK, arg_slot_stk_ofs(loc->arg_slot));
6834 loc = &info->in[1 + !HOST_BIG_ENDIAN];
6835 tcg_out_helper_load_imm(s, loc->arg_slot, TCG_TYPE_I32, 0, parm);
6863 QSIMPLEQ_FOREACH(l, &s->labels, next) {
6864 if (unlikely(!l->present) && !QSIMPLEQ_EMPTY(&l->branches)) {
6866 "$L%d referenced but not present.\n", l->id);
6883 if (s->nb_indirects > 0) {
6897 /* If changes were made, re-run liveness. */
6914 tb->jmp_reset_offset[0] = TB_JMP_OFFSET_INVALID;
6915 tb->jmp_reset_offset[1] = TB_JMP_OFFSET_INVALID;
6916 tb->jmp_insn_offset[0] = TB_JMP_OFFSET_INVALID;
6917 tb->jmp_insn_offset[1] = TB_JMP_OFFSET_INVALID;
6923 * TODO: Move this into translate-all.c with the rest of the
6926 s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr);
6927 s->code_ptr = s->code_buf;
6928 s->data_gen_ptr = NULL;
6930 QSIMPLEQ_INIT(&s->ldst_labels);
6931 s->pool_labels = NULL;
6933 s->gen_insn_data =
6934 tcg_malloc(sizeof(uint64_t) * s->gen_tb->icount * INSN_START_WORDS);
6938 num_insns = -1;
6939 s->carry_live = false;
6940 QTAILQ_FOREACH(op, &s->ops, link) {
6941 TCGOpcode opc = op->opc;
6948 * e.g. zero or sign-extended, then emit as a unary op.
6954 TCGLifeData arg_life = op->life;
6971 s->gen_insn_end_off[num_insns] = off;
6973 assert(s->gen_insn_end_off[num_insns] == off);
6977 s->gen_insn_data[num_insns * INSN_START_WORDS + i] =
6982 temp_dead(s, arg_temp(op->args[0]));
6985 tcg_reg_alloc_bb_end(s, s->reserved_regs);
6986 tcg_out_label(s, arg_label(op->args[0]));
6993 tcg_out_exit_tb(s, op->args[0]);
6996 tcg_out_goto_tb(s, op->args[0]);
6999 tcg_out_br(s, arg_label(op->args[0]));
7002 tcg_out_mb(s, op->args[0]);
7024 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
7025 return -1;
7029 return -2;
7034 tcg_debug_assert(num_insns + 1 == s->gen_tb->icount);
7035 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s);
7047 return -2;
7052 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
7053 (uintptr_t)s->code_buf,
7054 tcg_ptr_byte_diff(s->code_ptr, s->code_buf));
7067 the contents of a .debug_frame section that describes the post-
7112 return p - strtab;
7168 .e_shnum = ARRAY_SIZE(img->shdr),
7169 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1,
7199 .sh_size = sizeof(img->da),
7208 .sh_size = sizeof(img->sym),
7210 .sh_link = ARRAY_SIZE(img->shdr) - 1,
7216 .sh_size = sizeof(img->str),
7226 .len = sizeof(struct DebugInfo) - 4,
7263 img->phdr.p_vaddr = buf;
7264 img->phdr.p_paddr = buf;
7265 img->phdr.p_memsz = buf_size;
7267 img->shdr[1].sh_name = find_string(img->str, ".text");
7268 img->shdr[1].sh_addr = buf;
7269 img->shdr[1].sh_size = buf_size;
7271 img->shdr[2].sh_name = find_string(img->str, ".debug_info");
7272 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev");
7274 img->shdr[4].sh_name = find_string(img->str, ".debug_frame");
7275 img->shdr[4].sh_size = debug_frame_size;
7277 img->shdr[5].sh_name = find_string(img->str, ".symtab");
7278 img->shdr[6].sh_name = find_string(img->str, ".strtab");
7280 img->sym[1].st_name = find_string(img->str, "code_gen_buffer");
7281 img->sym[1].st_value = buf;
7282 img->sym[1].st_size = buf_size;
7284 img->di.cu_low_pc = buf;
7285 img->di.cu_high_pc = buf + buf_size;
7286 img->di.fn_low_pc = buf;
7287 img->di.fn_high_pc = buf + buf_size;
7291 dfh->fde.func_start = buf;
7292 dfh->fde.func_len = buf_size;