1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 #include "qemu/osdep.h" 26 27 /* Define to jump the ELF file used to communicate with GDB. */ 28 #undef DEBUG_JIT 29 30 #include "qemu/error-report.h" 31 #include "qemu/cutils.h" 32 #include "qemu/host-utils.h" 33 #include "qemu/qemu-print.h" 34 #include "qemu/cacheflush.h" 35 #include "qemu/cacheinfo.h" 36 #include "qemu/timer.h" 37 #include "exec/translation-block.h" 38 #include "exec/tlb-common.h" 39 #include "tcg/startup.h" 40 #include "tcg/tcg-op-common.h" 41 42 #if UINTPTR_MAX == UINT32_MAX 43 # define ELF_CLASS ELFCLASS32 44 #else 45 # define ELF_CLASS ELFCLASS64 46 #endif 47 #if HOST_BIG_ENDIAN 48 # define ELF_DATA ELFDATA2MSB 49 #else 50 # define ELF_DATA ELFDATA2LSB 51 #endif 52 53 #include "elf.h" 54 #include "exec/log.h" 55 #include "tcg/tcg-ldst.h" 56 #include "tcg/tcg-temp-internal.h" 57 #include "tcg-internal.h" 58 #include "tcg/perf.h" 59 #include "tcg-has.h" 60 #ifdef CONFIG_USER_ONLY 61 #include "user/guest-base.h" 62 #endif 63 64 /* Forward declarations for functions declared in tcg-target.c.inc and 65 used here. */ 66 static void tcg_target_init(TCGContext *s); 67 static void tcg_target_qemu_prologue(TCGContext *s); 68 static bool patch_reloc(tcg_insn_unit *code_ptr, int type, 69 intptr_t value, intptr_t addend); 70 static void tcg_out_nop_fill(tcg_insn_unit *p, int count); 71 72 typedef struct TCGLabelQemuLdst TCGLabelQemuLdst; 73 static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l); 74 static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l); 75 76 /* The CIE and FDE header definitions will be common to all hosts. */ 77 typedef struct { 78 uint32_t len __attribute__((aligned((sizeof(void *))))); 79 uint32_t id; 80 uint8_t version; 81 char augmentation[1]; 82 uint8_t code_align; 83 uint8_t data_align; 84 uint8_t return_column; 85 } DebugFrameCIE; 86 87 typedef struct QEMU_PACKED { 88 uint32_t len __attribute__((aligned((sizeof(void *))))); 89 uint32_t cie_offset; 90 uintptr_t func_start; 91 uintptr_t func_len; 92 } DebugFrameFDEHeader; 93 94 typedef struct QEMU_PACKED { 95 DebugFrameCIE cie; 96 DebugFrameFDEHeader fde; 97 } DebugFrameHeader; 98 99 struct TCGLabelQemuLdst { 100 bool is_ld; /* qemu_ld: true, qemu_st: false */ 101 MemOpIdx oi; 102 TCGType type; /* result type of a load */ 103 TCGReg addr_reg; /* reg index for guest virtual addr */ 104 TCGReg datalo_reg; /* reg index for low word to be loaded or stored */ 105 TCGReg datahi_reg; /* reg index for high word to be loaded or stored */ 106 const tcg_insn_unit *raddr; /* addr of the next IR of qemu_ld/st IR */ 107 tcg_insn_unit *label_ptr[2]; /* label pointers to be updated */ 108 QSIMPLEQ_ENTRY(TCGLabelQemuLdst) next; 109 }; 110 111 static void tcg_register_jit_int(const void *buf, size_t size, 112 const void *debug_frame, 113 size_t debug_frame_size) 114 __attribute__((unused)); 115 116 /* Forward declarations for functions declared and used in tcg-target.c.inc. */ 117 static void tcg_out_tb_start(TCGContext *s); 118 static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1, 119 intptr_t arg2); 120 static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); 121 static void tcg_out_movi(TCGContext *s, TCGType type, 122 TCGReg ret, tcg_target_long arg); 123 static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); 124 static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); 125 static void tcg_out_ext8u(TCGContext *s, TCGReg ret, TCGReg arg); 126 static void tcg_out_ext16u(TCGContext *s, TCGReg ret, TCGReg arg); 127 static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg); 128 static void tcg_out_ext32u(TCGContext *s, TCGReg ret, TCGReg arg); 129 static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg); 130 static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg); 131 static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg ret, TCGReg arg); 132 static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long); 133 static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2); 134 static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg); 135 static void tcg_out_goto_tb(TCGContext *s, int which); 136 static void tcg_out_goto_ptr(TCGContext *s, TCGReg dest); 137 static void tcg_out_mb(TCGContext *s, unsigned bar); 138 static void tcg_out_br(TCGContext *s, TCGLabel *l); 139 static void tcg_out_set_carry(TCGContext *s); 140 static void tcg_out_set_borrow(TCGContext *s); 141 #if TCG_TARGET_MAYBE_vec 142 static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece, 143 TCGReg dst, TCGReg src); 144 static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece, 145 TCGReg dst, TCGReg base, intptr_t offset); 146 static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, 147 TCGReg dst, int64_t arg); 148 static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, 149 unsigned vecl, unsigned vece, 150 const TCGArg args[TCG_MAX_OP_ARGS], 151 const int const_args[TCG_MAX_OP_ARGS]); 152 #else 153 static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece, 154 TCGReg dst, TCGReg src) 155 { 156 g_assert_not_reached(); 157 } 158 static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece, 159 TCGReg dst, TCGReg base, intptr_t offset) 160 { 161 g_assert_not_reached(); 162 } 163 static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, 164 TCGReg dst, int64_t arg) 165 { 166 g_assert_not_reached(); 167 } 168 static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, 169 unsigned vecl, unsigned vece, 170 const TCGArg args[TCG_MAX_OP_ARGS], 171 const int const_args[TCG_MAX_OP_ARGS]) 172 { 173 g_assert_not_reached(); 174 } 175 int tcg_can_emit_vec_op(TCGOpcode o, TCGType t, unsigned ve) 176 { 177 return 0; 178 } 179 #endif 180 static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1, 181 intptr_t arg2); 182 static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, 183 TCGReg base, intptr_t ofs); 184 static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target, 185 const TCGHelperInfo *info); 186 static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot); 187 static bool tcg_target_const_match(int64_t val, int ct, 188 TCGType type, TCGCond cond, int vece); 189 190 #ifndef CONFIG_USER_ONLY 191 #define guest_base ({ qemu_build_not_reached(); (uintptr_t)0; }) 192 #endif 193 194 typedef struct TCGLdstHelperParam { 195 TCGReg (*ra_gen)(TCGContext *s, const TCGLabelQemuLdst *l, int arg_reg); 196 unsigned ntmp; 197 int tmp[3]; 198 } TCGLdstHelperParam; 199 200 static void tcg_out_ld_helper_args(TCGContext *s, const TCGLabelQemuLdst *l, 201 const TCGLdstHelperParam *p) 202 __attribute__((unused)); 203 static void tcg_out_ld_helper_ret(TCGContext *s, const TCGLabelQemuLdst *l, 204 bool load_sign, const TCGLdstHelperParam *p) 205 __attribute__((unused)); 206 static void tcg_out_st_helper_args(TCGContext *s, const TCGLabelQemuLdst *l, 207 const TCGLdstHelperParam *p) 208 __attribute__((unused)); 209 210 static void * const qemu_ld_helpers[MO_SSIZE + 1] __attribute__((unused)) = { 211 [MO_UB] = helper_ldub_mmu, 212 [MO_SB] = helper_ldsb_mmu, 213 [MO_UW] = helper_lduw_mmu, 214 [MO_SW] = helper_ldsw_mmu, 215 [MO_UL] = helper_ldul_mmu, 216 [MO_UQ] = helper_ldq_mmu, 217 #if TCG_TARGET_REG_BITS == 64 218 [MO_SL] = helper_ldsl_mmu, 219 [MO_128] = helper_ld16_mmu, 220 #endif 221 }; 222 223 static void * const qemu_st_helpers[MO_SIZE + 1] __attribute__((unused)) = { 224 [MO_8] = helper_stb_mmu, 225 [MO_16] = helper_stw_mmu, 226 [MO_32] = helper_stl_mmu, 227 [MO_64] = helper_stq_mmu, 228 #if TCG_TARGET_REG_BITS == 64 229 [MO_128] = helper_st16_mmu, 230 #endif 231 }; 232 233 typedef struct { 234 MemOp atom; /* lg2 bits of atomicity required */ 235 MemOp align; /* lg2 bits of alignment to use */ 236 } TCGAtomAlign; 237 238 static TCGAtomAlign atom_and_align_for_opc(TCGContext *s, MemOp opc, 239 MemOp host_atom, bool allow_two_ops) 240 __attribute__((unused)); 241 242 #ifdef CONFIG_USER_ONLY 243 bool tcg_use_softmmu; 244 #endif 245 246 TCGContext tcg_init_ctx; 247 __thread TCGContext *tcg_ctx; 248 249 TCGContext **tcg_ctxs; 250 unsigned int tcg_cur_ctxs; 251 unsigned int tcg_max_ctxs; 252 TCGv_env tcg_env; 253 const void *tcg_code_gen_epilogue; 254 uintptr_t tcg_splitwx_diff; 255 256 #ifndef CONFIG_TCG_INTERPRETER 257 tcg_prologue_fn *tcg_qemu_tb_exec; 258 #endif 259 260 static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT]; 261 static TCGRegSet tcg_target_call_clobber_regs; 262 263 #if TCG_TARGET_INSN_UNIT_SIZE == 1 264 static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v) 265 { 266 *s->code_ptr++ = v; 267 } 268 269 static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p, 270 uint8_t v) 271 { 272 *p = v; 273 } 274 #endif 275 276 #if TCG_TARGET_INSN_UNIT_SIZE <= 2 277 static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v) 278 { 279 if (TCG_TARGET_INSN_UNIT_SIZE == 2) { 280 *s->code_ptr++ = v; 281 } else { 282 tcg_insn_unit *p = s->code_ptr; 283 memcpy(p, &v, sizeof(v)); 284 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE); 285 } 286 } 287 288 static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p, 289 uint16_t v) 290 { 291 if (TCG_TARGET_INSN_UNIT_SIZE == 2) { 292 *p = v; 293 } else { 294 memcpy(p, &v, sizeof(v)); 295 } 296 } 297 #endif 298 299 #if TCG_TARGET_INSN_UNIT_SIZE <= 4 300 static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v) 301 { 302 if (TCG_TARGET_INSN_UNIT_SIZE == 4) { 303 *s->code_ptr++ = v; 304 } else { 305 tcg_insn_unit *p = s->code_ptr; 306 memcpy(p, &v, sizeof(v)); 307 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE); 308 } 309 } 310 311 static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p, 312 uint32_t v) 313 { 314 if (TCG_TARGET_INSN_UNIT_SIZE == 4) { 315 *p = v; 316 } else { 317 memcpy(p, &v, sizeof(v)); 318 } 319 } 320 #endif 321 322 #if TCG_TARGET_INSN_UNIT_SIZE <= 8 323 static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v) 324 { 325 if (TCG_TARGET_INSN_UNIT_SIZE == 8) { 326 *s->code_ptr++ = v; 327 } else { 328 tcg_insn_unit *p = s->code_ptr; 329 memcpy(p, &v, sizeof(v)); 330 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE); 331 } 332 } 333 334 static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p, 335 uint64_t v) 336 { 337 if (TCG_TARGET_INSN_UNIT_SIZE == 8) { 338 *p = v; 339 } else { 340 memcpy(p, &v, sizeof(v)); 341 } 342 } 343 #endif 344 345 /* label relocation processing */ 346 347 static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type, 348 TCGLabel *l, intptr_t addend) 349 { 350 TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation)); 351 352 r->type = type; 353 r->ptr = code_ptr; 354 r->addend = addend; 355 QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next); 356 } 357 358 static void tcg_out_label(TCGContext *s, TCGLabel *l) 359 { 360 tcg_debug_assert(!l->has_value); 361 l->has_value = 1; 362 l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr); 363 } 364 365 TCGLabel *gen_new_label(void) 366 { 367 TCGContext *s = tcg_ctx; 368 TCGLabel *l = tcg_malloc(sizeof(TCGLabel)); 369 370 memset(l, 0, sizeof(TCGLabel)); 371 l->id = s->nb_labels++; 372 QSIMPLEQ_INIT(&l->branches); 373 QSIMPLEQ_INIT(&l->relocs); 374 375 QSIMPLEQ_INSERT_TAIL(&s->labels, l, next); 376 377 return l; 378 } 379 380 static bool tcg_resolve_relocs(TCGContext *s) 381 { 382 TCGLabel *l; 383 384 QSIMPLEQ_FOREACH(l, &s->labels, next) { 385 TCGRelocation *r; 386 uintptr_t value = l->u.value; 387 388 QSIMPLEQ_FOREACH(r, &l->relocs, next) { 389 if (!patch_reloc(r->ptr, r->type, value, r->addend)) { 390 return false; 391 } 392 } 393 } 394 return true; 395 } 396 397 static void set_jmp_reset_offset(TCGContext *s, int which) 398 { 399 /* 400 * We will check for overflow at the end of the opcode loop in 401 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX. 402 */ 403 s->gen_tb->jmp_reset_offset[which] = tcg_current_code_size(s); 404 } 405 406 static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which) 407 { 408 /* 409 * We will check for overflow at the end of the opcode loop in 410 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX. 411 */ 412 s->gen_tb->jmp_insn_offset[which] = tcg_current_code_size(s); 413 } 414 415 static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which) 416 { 417 /* 418 * Return the read-execute version of the pointer, for the benefit 419 * of any pc-relative addressing mode. 420 */ 421 return (uintptr_t)tcg_splitwx_to_rx(&s->gen_tb->jmp_target_addr[which]); 422 } 423 424 static int __attribute__((unused)) 425 tlb_mask_table_ofs(TCGContext *s, int which) 426 { 427 return (offsetof(CPUNegativeOffsetState, tlb.f[which]) - 428 sizeof(CPUNegativeOffsetState)); 429 } 430 431 /* Signal overflow, starting over with fewer guest insns. */ 432 static G_NORETURN 433 void tcg_raise_tb_overflow(TCGContext *s) 434 { 435 siglongjmp(s->jmp_trans, -2); 436 } 437 438 /* 439 * Used by tcg_out_movext{1,2} to hold the arguments for tcg_out_movext. 440 * By the time we arrive at tcg_out_movext1, @dst is always a TCGReg. 441 * 442 * However, tcg_out_helper_load_slots reuses this field to hold an 443 * argument slot number (which may designate a argument register or an 444 * argument stack slot), converting to TCGReg once all arguments that 445 * are destined for the stack are processed. 446 */ 447 typedef struct TCGMovExtend { 448 unsigned dst; 449 TCGReg src; 450 TCGType dst_type; 451 TCGType src_type; 452 MemOp src_ext; 453 } TCGMovExtend; 454 455 /** 456 * tcg_out_movext -- move and extend 457 * @s: tcg context 458 * @dst_type: integral type for destination 459 * @dst: destination register 460 * @src_type: integral type for source 461 * @src_ext: extension to apply to source 462 * @src: source register 463 * 464 * Move or extend @src into @dst, depending on @src_ext and the types. 465 */ 466 static void tcg_out_movext(TCGContext *s, TCGType dst_type, TCGReg dst, 467 TCGType src_type, MemOp src_ext, TCGReg src) 468 { 469 switch (src_ext) { 470 case MO_UB: 471 tcg_out_ext8u(s, dst, src); 472 break; 473 case MO_SB: 474 tcg_out_ext8s(s, dst_type, dst, src); 475 break; 476 case MO_UW: 477 tcg_out_ext16u(s, dst, src); 478 break; 479 case MO_SW: 480 tcg_out_ext16s(s, dst_type, dst, src); 481 break; 482 case MO_UL: 483 case MO_SL: 484 if (dst_type == TCG_TYPE_I32) { 485 if (src_type == TCG_TYPE_I32) { 486 tcg_out_mov(s, TCG_TYPE_I32, dst, src); 487 } else { 488 tcg_out_extrl_i64_i32(s, dst, src); 489 } 490 } else if (src_type == TCG_TYPE_I32) { 491 if (src_ext & MO_SIGN) { 492 tcg_out_exts_i32_i64(s, dst, src); 493 } else { 494 tcg_out_extu_i32_i64(s, dst, src); 495 } 496 } else { 497 if (src_ext & MO_SIGN) { 498 tcg_out_ext32s(s, dst, src); 499 } else { 500 tcg_out_ext32u(s, dst, src); 501 } 502 } 503 break; 504 case MO_UQ: 505 tcg_debug_assert(TCG_TARGET_REG_BITS == 64); 506 if (dst_type == TCG_TYPE_I32) { 507 tcg_out_extrl_i64_i32(s, dst, src); 508 } else { 509 tcg_out_mov(s, TCG_TYPE_I64, dst, src); 510 } 511 break; 512 default: 513 g_assert_not_reached(); 514 } 515 } 516 517 /* Minor variations on a theme, using a structure. */ 518 static void tcg_out_movext1_new_src(TCGContext *s, const TCGMovExtend *i, 519 TCGReg src) 520 { 521 tcg_out_movext(s, i->dst_type, i->dst, i->src_type, i->src_ext, src); 522 } 523 524 static void tcg_out_movext1(TCGContext *s, const TCGMovExtend *i) 525 { 526 tcg_out_movext1_new_src(s, i, i->src); 527 } 528 529 /** 530 * tcg_out_movext2 -- move and extend two pair 531 * @s: tcg context 532 * @i1: first move description 533 * @i2: second move description 534 * @scratch: temporary register, or -1 for none 535 * 536 * As tcg_out_movext, for both @i1 and @i2, caring for overlap 537 * between the sources and destinations. 538 */ 539 540 static void tcg_out_movext2(TCGContext *s, const TCGMovExtend *i1, 541 const TCGMovExtend *i2, int scratch) 542 { 543 TCGReg src1 = i1->src; 544 TCGReg src2 = i2->src; 545 546 if (i1->dst != src2) { 547 tcg_out_movext1(s, i1); 548 tcg_out_movext1(s, i2); 549 return; 550 } 551 if (i2->dst == src1) { 552 TCGType src1_type = i1->src_type; 553 TCGType src2_type = i2->src_type; 554 555 if (tcg_out_xchg(s, MAX(src1_type, src2_type), src1, src2)) { 556 /* The data is now in the correct registers, now extend. */ 557 src1 = i2->src; 558 src2 = i1->src; 559 } else { 560 tcg_debug_assert(scratch >= 0); 561 tcg_out_mov(s, src1_type, scratch, src1); 562 src1 = scratch; 563 } 564 } 565 tcg_out_movext1_new_src(s, i2, src2); 566 tcg_out_movext1_new_src(s, i1, src1); 567 } 568 569 /** 570 * tcg_out_movext3 -- move and extend three pair 571 * @s: tcg context 572 * @i1: first move description 573 * @i2: second move description 574 * @i3: third move description 575 * @scratch: temporary register, or -1 for none 576 * 577 * As tcg_out_movext, for all of @i1, @i2 and @i3, caring for overlap 578 * between the sources and destinations. 579 */ 580 581 static void tcg_out_movext3(TCGContext *s, const TCGMovExtend *i1, 582 const TCGMovExtend *i2, const TCGMovExtend *i3, 583 int scratch) 584 { 585 TCGReg src1 = i1->src; 586 TCGReg src2 = i2->src; 587 TCGReg src3 = i3->src; 588 589 if (i1->dst != src2 && i1->dst != src3) { 590 tcg_out_movext1(s, i1); 591 tcg_out_movext2(s, i2, i3, scratch); 592 return; 593 } 594 if (i2->dst != src1 && i2->dst != src3) { 595 tcg_out_movext1(s, i2); 596 tcg_out_movext2(s, i1, i3, scratch); 597 return; 598 } 599 if (i3->dst != src1 && i3->dst != src2) { 600 tcg_out_movext1(s, i3); 601 tcg_out_movext2(s, i1, i2, scratch); 602 return; 603 } 604 605 /* 606 * There is a cycle. Since there are only 3 nodes, the cycle is 607 * either "clockwise" or "anti-clockwise", and can be solved with 608 * a single scratch or two xchg. 609 */ 610 if (i1->dst == src2 && i2->dst == src3 && i3->dst == src1) { 611 /* "Clockwise" */ 612 if (tcg_out_xchg(s, MAX(i1->src_type, i2->src_type), src1, src2)) { 613 tcg_out_xchg(s, MAX(i2->src_type, i3->src_type), src2, src3); 614 /* The data is now in the correct registers, now extend. */ 615 tcg_out_movext1_new_src(s, i1, i1->dst); 616 tcg_out_movext1_new_src(s, i2, i2->dst); 617 tcg_out_movext1_new_src(s, i3, i3->dst); 618 } else { 619 tcg_debug_assert(scratch >= 0); 620 tcg_out_mov(s, i1->src_type, scratch, src1); 621 tcg_out_movext1(s, i3); 622 tcg_out_movext1(s, i2); 623 tcg_out_movext1_new_src(s, i1, scratch); 624 } 625 } else if (i1->dst == src3 && i2->dst == src1 && i3->dst == src2) { 626 /* "Anti-clockwise" */ 627 if (tcg_out_xchg(s, MAX(i2->src_type, i3->src_type), src2, src3)) { 628 tcg_out_xchg(s, MAX(i1->src_type, i2->src_type), src1, src2); 629 /* The data is now in the correct registers, now extend. */ 630 tcg_out_movext1_new_src(s, i1, i1->dst); 631 tcg_out_movext1_new_src(s, i2, i2->dst); 632 tcg_out_movext1_new_src(s, i3, i3->dst); 633 } else { 634 tcg_debug_assert(scratch >= 0); 635 tcg_out_mov(s, i1->src_type, scratch, src1); 636 tcg_out_movext1(s, i2); 637 tcg_out_movext1(s, i3); 638 tcg_out_movext1_new_src(s, i1, scratch); 639 } 640 } else { 641 g_assert_not_reached(); 642 } 643 } 644 645 /* 646 * Allocate a new TCGLabelQemuLdst entry. 647 */ 648 649 __attribute__((unused)) 650 static TCGLabelQemuLdst *new_ldst_label(TCGContext *s) 651 { 652 TCGLabelQemuLdst *l = tcg_malloc(sizeof(*l)); 653 654 memset(l, 0, sizeof(*l)); 655 QSIMPLEQ_INSERT_TAIL(&s->ldst_labels, l, next); 656 657 return l; 658 } 659 660 /* 661 * Allocate new constant pool entries. 662 */ 663 664 typedef struct TCGLabelPoolData { 665 struct TCGLabelPoolData *next; 666 tcg_insn_unit *label; 667 intptr_t addend; 668 int rtype; 669 unsigned nlong; 670 tcg_target_ulong data[]; 671 } TCGLabelPoolData; 672 673 static TCGLabelPoolData *new_pool_alloc(TCGContext *s, int nlong, int rtype, 674 tcg_insn_unit *label, intptr_t addend) 675 { 676 TCGLabelPoolData *n = tcg_malloc(sizeof(TCGLabelPoolData) 677 + sizeof(tcg_target_ulong) * nlong); 678 679 n->label = label; 680 n->addend = addend; 681 n->rtype = rtype; 682 n->nlong = nlong; 683 return n; 684 } 685 686 static void new_pool_insert(TCGContext *s, TCGLabelPoolData *n) 687 { 688 TCGLabelPoolData *i, **pp; 689 int nlong = n->nlong; 690 691 /* Insertion sort on the pool. */ 692 for (pp = &s->pool_labels; (i = *pp) != NULL; pp = &i->next) { 693 if (nlong > i->nlong) { 694 break; 695 } 696 if (nlong < i->nlong) { 697 continue; 698 } 699 if (memcmp(n->data, i->data, sizeof(tcg_target_ulong) * nlong) >= 0) { 700 break; 701 } 702 } 703 n->next = *pp; 704 *pp = n; 705 } 706 707 /* The "usual" for generic integer code. */ 708 __attribute__((unused)) 709 static void new_pool_label(TCGContext *s, tcg_target_ulong d, int rtype, 710 tcg_insn_unit *label, intptr_t addend) 711 { 712 TCGLabelPoolData *n = new_pool_alloc(s, 1, rtype, label, addend); 713 n->data[0] = d; 714 new_pool_insert(s, n); 715 } 716 717 /* For v64 or v128, depending on the host. */ 718 __attribute__((unused)) 719 static void new_pool_l2(TCGContext *s, int rtype, tcg_insn_unit *label, 720 intptr_t addend, tcg_target_ulong d0, 721 tcg_target_ulong d1) 722 { 723 TCGLabelPoolData *n = new_pool_alloc(s, 2, rtype, label, addend); 724 n->data[0] = d0; 725 n->data[1] = d1; 726 new_pool_insert(s, n); 727 } 728 729 /* For v128 or v256, depending on the host. */ 730 __attribute__((unused)) 731 static void new_pool_l4(TCGContext *s, int rtype, tcg_insn_unit *label, 732 intptr_t addend, tcg_target_ulong d0, 733 tcg_target_ulong d1, tcg_target_ulong d2, 734 tcg_target_ulong d3) 735 { 736 TCGLabelPoolData *n = new_pool_alloc(s, 4, rtype, label, addend); 737 n->data[0] = d0; 738 n->data[1] = d1; 739 n->data[2] = d2; 740 n->data[3] = d3; 741 new_pool_insert(s, n); 742 } 743 744 /* For v256, for 32-bit host. */ 745 __attribute__((unused)) 746 static void new_pool_l8(TCGContext *s, int rtype, tcg_insn_unit *label, 747 intptr_t addend, tcg_target_ulong d0, 748 tcg_target_ulong d1, tcg_target_ulong d2, 749 tcg_target_ulong d3, tcg_target_ulong d4, 750 tcg_target_ulong d5, tcg_target_ulong d6, 751 tcg_target_ulong d7) 752 { 753 TCGLabelPoolData *n = new_pool_alloc(s, 8, rtype, label, addend); 754 n->data[0] = d0; 755 n->data[1] = d1; 756 n->data[2] = d2; 757 n->data[3] = d3; 758 n->data[4] = d4; 759 n->data[5] = d5; 760 n->data[6] = d6; 761 n->data[7] = d7; 762 new_pool_insert(s, n); 763 } 764 765 /* 766 * Generate TB finalization at the end of block 767 */ 768 769 static int tcg_out_ldst_finalize(TCGContext *s) 770 { 771 TCGLabelQemuLdst *lb; 772 773 /* qemu_ld/st slow paths */ 774 QSIMPLEQ_FOREACH(lb, &s->ldst_labels, next) { 775 if (lb->is_ld 776 ? !tcg_out_qemu_ld_slow_path(s, lb) 777 : !tcg_out_qemu_st_slow_path(s, lb)) { 778 return -2; 779 } 780 781 /* 782 * Test for (pending) buffer overflow. The assumption is that any 783 * one operation beginning below the high water mark cannot overrun 784 * the buffer completely. Thus we can test for overflow after 785 * generating code without having to check during generation. 786 */ 787 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) { 788 return -1; 789 } 790 } 791 return 0; 792 } 793 794 static int tcg_out_pool_finalize(TCGContext *s) 795 { 796 TCGLabelPoolData *p = s->pool_labels; 797 TCGLabelPoolData *l = NULL; 798 void *a; 799 800 if (p == NULL) { 801 return 0; 802 } 803 804 /* 805 * ??? Round up to qemu_icache_linesize, but then do not round 806 * again when allocating the next TranslationBlock structure. 807 */ 808 a = (void *)ROUND_UP((uintptr_t)s->code_ptr, 809 sizeof(tcg_target_ulong) * p->nlong); 810 tcg_out_nop_fill(s->code_ptr, (tcg_insn_unit *)a - s->code_ptr); 811 s->data_gen_ptr = a; 812 813 for (; p != NULL; p = p->next) { 814 size_t size = sizeof(tcg_target_ulong) * p->nlong; 815 uintptr_t value; 816 817 if (!l || l->nlong != p->nlong || memcmp(l->data, p->data, size)) { 818 if (unlikely(a > s->code_gen_highwater)) { 819 return -1; 820 } 821 memcpy(a, p->data, size); 822 a += size; 823 l = p; 824 } 825 826 value = (uintptr_t)tcg_splitwx_to_rx(a) - size; 827 if (!patch_reloc(p->label, p->rtype, value, p->addend)) { 828 return -2; 829 } 830 } 831 832 s->code_ptr = a; 833 return 0; 834 } 835 836 #define C_PFX1(P, A) P##A 837 #define C_PFX2(P, A, B) P##A##_##B 838 #define C_PFX3(P, A, B, C) P##A##_##B##_##C 839 #define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D 840 #define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E 841 #define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F 842 843 /* Define an enumeration for the various combinations. */ 844 845 #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1), 846 #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2), 847 #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3), 848 #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4), 849 850 #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1), 851 #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2), 852 #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3), 853 #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4), 854 855 #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2), 856 #define C_N1O1_I1(O1, O2, I1) C_PFX3(c_n1o1_i1_, O1, O2, I1), 857 #define C_N2_I1(O1, O2, I1) C_PFX3(c_n2_i1_, O1, O2, I1), 858 859 #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1), 860 #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2), 861 #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3), 862 #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4), 863 #define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_n1_o1_i4_, O1, O2, I1, I2, I3, I4), 864 865 typedef enum { 866 C_Dynamic = -2, 867 C_NotImplemented = -1, 868 #include "tcg-target-con-set.h" 869 } TCGConstraintSetIndex; 870 871 static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode, TCGType, unsigned); 872 873 #undef C_O0_I1 874 #undef C_O0_I2 875 #undef C_O0_I3 876 #undef C_O0_I4 877 #undef C_O1_I1 878 #undef C_O1_I2 879 #undef C_O1_I3 880 #undef C_O1_I4 881 #undef C_N1_I2 882 #undef C_N1O1_I1 883 #undef C_N2_I1 884 #undef C_O2_I1 885 #undef C_O2_I2 886 #undef C_O2_I3 887 #undef C_O2_I4 888 #undef C_N1_O1_I4 889 890 /* Put all of the constraint sets into an array, indexed by the enum. */ 891 892 typedef struct TCGConstraintSet { 893 uint8_t nb_oargs, nb_iargs; 894 const char *args_ct_str[TCG_MAX_OP_ARGS]; 895 } TCGConstraintSet; 896 897 #define C_O0_I1(I1) { 0, 1, { #I1 } }, 898 #define C_O0_I2(I1, I2) { 0, 2, { #I1, #I2 } }, 899 #define C_O0_I3(I1, I2, I3) { 0, 3, { #I1, #I2, #I3 } }, 900 #define C_O0_I4(I1, I2, I3, I4) { 0, 4, { #I1, #I2, #I3, #I4 } }, 901 902 #define C_O1_I1(O1, I1) { 1, 1, { #O1, #I1 } }, 903 #define C_O1_I2(O1, I1, I2) { 1, 2, { #O1, #I1, #I2 } }, 904 #define C_O1_I3(O1, I1, I2, I3) { 1, 3, { #O1, #I1, #I2, #I3 } }, 905 #define C_O1_I4(O1, I1, I2, I3, I4) { 1, 4, { #O1, #I1, #I2, #I3, #I4 } }, 906 907 #define C_N1_I2(O1, I1, I2) { 1, 2, { "&" #O1, #I1, #I2 } }, 908 #define C_N1O1_I1(O1, O2, I1) { 2, 1, { "&" #O1, #O2, #I1 } }, 909 #define C_N2_I1(O1, O2, I1) { 2, 1, { "&" #O1, "&" #O2, #I1 } }, 910 911 #define C_O2_I1(O1, O2, I1) { 2, 1, { #O1, #O2, #I1 } }, 912 #define C_O2_I2(O1, O2, I1, I2) { 2, 2, { #O1, #O2, #I1, #I2 } }, 913 #define C_O2_I3(O1, O2, I1, I2, I3) { 2, 3, { #O1, #O2, #I1, #I2, #I3 } }, 914 #define C_O2_I4(O1, O2, I1, I2, I3, I4) { 2, 4, { #O1, #O2, #I1, #I2, #I3, #I4 } }, 915 #define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) { 2, 4, { "&" #O1, #O2, #I1, #I2, #I3, #I4 } }, 916 917 static const TCGConstraintSet constraint_sets[] = { 918 #include "tcg-target-con-set.h" 919 }; 920 921 #undef C_O0_I1 922 #undef C_O0_I2 923 #undef C_O0_I3 924 #undef C_O0_I4 925 #undef C_O1_I1 926 #undef C_O1_I2 927 #undef C_O1_I3 928 #undef C_O1_I4 929 #undef C_N1_I2 930 #undef C_N1O1_I1 931 #undef C_N2_I1 932 #undef C_O2_I1 933 #undef C_O2_I2 934 #undef C_O2_I3 935 #undef C_O2_I4 936 #undef C_N1_O1_I4 937 938 /* Expand the enumerator to be returned from tcg_target_op_def(). */ 939 940 #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1) 941 #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2) 942 #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3) 943 #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4) 944 945 #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1) 946 #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2) 947 #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3) 948 #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4) 949 950 #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2) 951 #define C_N1O1_I1(O1, O2, I1) C_PFX3(c_n1o1_i1_, O1, O2, I1) 952 #define C_N2_I1(O1, O2, I1) C_PFX3(c_n2_i1_, O1, O2, I1) 953 954 #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1) 955 #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2) 956 #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3) 957 #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4) 958 #define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_n1_o1_i4_, O1, O2, I1, I2, I3, I4) 959 960 /* 961 * TCGOutOp is the base class for a set of structures that describe how 962 * to generate code for a given TCGOpcode. 963 * 964 * @static_constraint: 965 * C_NotImplemented: The TCGOpcode is not supported by the backend. 966 * C_Dynamic: Use @dynamic_constraint to select a constraint set 967 * based on any of @type, @flags, or host isa. 968 * Otherwise: The register allocation constrains for the TCGOpcode. 969 * 970 * Subclasses of TCGOutOp will define a set of output routines that may 971 * be used. Such routines will often be selected by the set of registers 972 * and constants that come out of register allocation. The set of 973 * routines that are provided will guide the set of constraints that are 974 * legal. In particular, assume that tcg_optimize() has done its job in 975 * swapping commutative operands and folding operations for which all 976 * operands are constant. 977 */ 978 typedef struct TCGOutOp { 979 TCGConstraintSetIndex static_constraint; 980 TCGConstraintSetIndex (*dynamic_constraint)(TCGType type, unsigned flags); 981 } TCGOutOp; 982 983 typedef struct TCGOutOpAddSubCarry { 984 TCGOutOp base; 985 void (*out_rrr)(TCGContext *s, TCGType type, 986 TCGReg a0, TCGReg a1, TCGReg a2); 987 void (*out_rri)(TCGContext *s, TCGType type, 988 TCGReg a0, TCGReg a1, tcg_target_long a2); 989 void (*out_rir)(TCGContext *s, TCGType type, 990 TCGReg a0, tcg_target_long a1, TCGReg a2); 991 void (*out_rii)(TCGContext *s, TCGType type, 992 TCGReg a0, tcg_target_long a1, tcg_target_long a2); 993 } TCGOutOpAddSubCarry; 994 995 typedef struct TCGOutOpBinary { 996 TCGOutOp base; 997 void (*out_rrr)(TCGContext *s, TCGType type, 998 TCGReg a0, TCGReg a1, TCGReg a2); 999 void (*out_rri)(TCGContext *s, TCGType type, 1000 TCGReg a0, TCGReg a1, tcg_target_long a2); 1001 } TCGOutOpBinary; 1002 1003 typedef struct TCGOutOpBrcond { 1004 TCGOutOp base; 1005 void (*out_rr)(TCGContext *s, TCGType type, TCGCond cond, 1006 TCGReg a1, TCGReg a2, TCGLabel *label); 1007 void (*out_ri)(TCGContext *s, TCGType type, TCGCond cond, 1008 TCGReg a1, tcg_target_long a2, TCGLabel *label); 1009 } TCGOutOpBrcond; 1010 1011 typedef struct TCGOutOpBrcond2 { 1012 TCGOutOp base; 1013 void (*out)(TCGContext *s, TCGCond cond, TCGReg al, TCGReg ah, 1014 TCGArg bl, bool const_bl, 1015 TCGArg bh, bool const_bh, TCGLabel *l); 1016 } TCGOutOpBrcond2; 1017 1018 typedef struct TCGOutOpBswap { 1019 TCGOutOp base; 1020 void (*out_rr)(TCGContext *s, TCGType type, 1021 TCGReg a0, TCGReg a1, unsigned flags); 1022 } TCGOutOpBswap; 1023 1024 typedef struct TCGOutOpDeposit { 1025 TCGOutOp base; 1026 void (*out_rrr)(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, 1027 TCGReg a2, unsigned ofs, unsigned len); 1028 void (*out_rri)(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, 1029 tcg_target_long a2, unsigned ofs, unsigned len); 1030 void (*out_rzr)(TCGContext *s, TCGType type, TCGReg a0, 1031 TCGReg a2, unsigned ofs, unsigned len); 1032 } TCGOutOpDeposit; 1033 1034 typedef struct TCGOutOpDivRem { 1035 TCGOutOp base; 1036 void (*out_rr01r)(TCGContext *s, TCGType type, 1037 TCGReg a0, TCGReg a1, TCGReg a4); 1038 } TCGOutOpDivRem; 1039 1040 typedef struct TCGOutOpExtract { 1041 TCGOutOp base; 1042 void (*out_rr)(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, 1043 unsigned ofs, unsigned len); 1044 } TCGOutOpExtract; 1045 1046 typedef struct TCGOutOpExtract2 { 1047 TCGOutOp base; 1048 void (*out_rrr)(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, 1049 TCGReg a2, unsigned shr); 1050 } TCGOutOpExtract2; 1051 1052 typedef struct TCGOutOpLoad { 1053 TCGOutOp base; 1054 void (*out)(TCGContext *s, TCGType type, TCGReg dest, 1055 TCGReg base, intptr_t offset); 1056 } TCGOutOpLoad; 1057 1058 typedef struct TCGOutOpMovcond { 1059 TCGOutOp base; 1060 void (*out)(TCGContext *s, TCGType type, TCGCond cond, 1061 TCGReg ret, TCGReg c1, TCGArg c2, bool const_c2, 1062 TCGArg vt, bool const_vt, TCGArg vf, bool consf_vf); 1063 } TCGOutOpMovcond; 1064 1065 typedef struct TCGOutOpMul2 { 1066 TCGOutOp base; 1067 void (*out_rrrr)(TCGContext *s, TCGType type, 1068 TCGReg a0, TCGReg a1, TCGReg a2, TCGReg a3); 1069 } TCGOutOpMul2; 1070 1071 typedef struct TCGOutOpQemuLdSt { 1072 TCGOutOp base; 1073 void (*out)(TCGContext *s, TCGType type, TCGReg dest, 1074 TCGReg addr, MemOpIdx oi); 1075 } TCGOutOpQemuLdSt; 1076 1077 typedef struct TCGOutOpQemuLdSt2 { 1078 TCGOutOp base; 1079 void (*out)(TCGContext *s, TCGType type, TCGReg dlo, TCGReg dhi, 1080 TCGReg addr, MemOpIdx oi); 1081 } TCGOutOpQemuLdSt2; 1082 1083 typedef struct TCGOutOpUnary { 1084 TCGOutOp base; 1085 void (*out_rr)(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1); 1086 } TCGOutOpUnary; 1087 1088 typedef struct TCGOutOpSetcond { 1089 TCGOutOp base; 1090 void (*out_rrr)(TCGContext *s, TCGType type, TCGCond cond, 1091 TCGReg ret, TCGReg a1, TCGReg a2); 1092 void (*out_rri)(TCGContext *s, TCGType type, TCGCond cond, 1093 TCGReg ret, TCGReg a1, tcg_target_long a2); 1094 } TCGOutOpSetcond; 1095 1096 typedef struct TCGOutOpSetcond2 { 1097 TCGOutOp base; 1098 void (*out)(TCGContext *s, TCGCond cond, TCGReg ret, TCGReg al, TCGReg ah, 1099 TCGArg bl, bool const_bl, TCGArg bh, bool const_bh); 1100 } TCGOutOpSetcond2; 1101 1102 typedef struct TCGOutOpStore { 1103 TCGOutOp base; 1104 void (*out_r)(TCGContext *s, TCGType type, TCGReg data, 1105 TCGReg base, intptr_t offset); 1106 void (*out_i)(TCGContext *s, TCGType type, tcg_target_long data, 1107 TCGReg base, intptr_t offset); 1108 } TCGOutOpStore; 1109 1110 typedef struct TCGOutOpSubtract { 1111 TCGOutOp base; 1112 void (*out_rrr)(TCGContext *s, TCGType type, 1113 TCGReg a0, TCGReg a1, TCGReg a2); 1114 void (*out_rir)(TCGContext *s, TCGType type, 1115 TCGReg a0, tcg_target_long a1, TCGReg a2); 1116 } TCGOutOpSubtract; 1117 1118 #include "tcg-target.c.inc" 1119 1120 #ifndef CONFIG_TCG_INTERPRETER 1121 /* Validate CPUTLBDescFast placement. */ 1122 QEMU_BUILD_BUG_ON((int)(offsetof(CPUNegativeOffsetState, tlb.f[0]) - 1123 sizeof(CPUNegativeOffsetState)) 1124 < MIN_TLB_MASK_TABLE_OFS); 1125 #endif 1126 1127 #if TCG_TARGET_REG_BITS == 64 1128 /* 1129 * We require these functions for slow-path function calls. 1130 * Adapt them generically for opcode output. 1131 */ 1132 1133 static void tgen_exts_i32_i64(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1) 1134 { 1135 tcg_out_exts_i32_i64(s, a0, a1); 1136 } 1137 1138 static const TCGOutOpUnary outop_exts_i32_i64 = { 1139 .base.static_constraint = C_O1_I1(r, r), 1140 .out_rr = tgen_exts_i32_i64, 1141 }; 1142 1143 static void tgen_extu_i32_i64(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1) 1144 { 1145 tcg_out_extu_i32_i64(s, a0, a1); 1146 } 1147 1148 static const TCGOutOpUnary outop_extu_i32_i64 = { 1149 .base.static_constraint = C_O1_I1(r, r), 1150 .out_rr = tgen_extu_i32_i64, 1151 }; 1152 1153 static void tgen_extrl_i64_i32(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1) 1154 { 1155 tcg_out_extrl_i64_i32(s, a0, a1); 1156 } 1157 1158 static const TCGOutOpUnary outop_extrl_i64_i32 = { 1159 .base.static_constraint = C_O1_I1(r, r), 1160 .out_rr = TCG_TARGET_HAS_extr_i64_i32 ? tgen_extrl_i64_i32 : NULL, 1161 }; 1162 #endif 1163 1164 static const TCGOutOp outop_goto_ptr = { 1165 .static_constraint = C_O0_I1(r), 1166 }; 1167 1168 static const TCGOutOpLoad outop_ld = { 1169 .base.static_constraint = C_O1_I1(r, r), 1170 .out = tcg_out_ld, 1171 }; 1172 1173 /* 1174 * Register V as the TCGOutOp for O. 1175 * This verifies that V is of type T, otherwise give a nice compiler error. 1176 * This prevents trivial mistakes within each arch/tcg-target.c.inc. 1177 */ 1178 #define OUTOP(O, T, V) [O] = _Generic(V, T: &V.base) 1179 1180 /* Register allocation descriptions for every TCGOpcode. */ 1181 static const TCGOutOp * const all_outop[NB_OPS] = { 1182 OUTOP(INDEX_op_add, TCGOutOpBinary, outop_add), 1183 OUTOP(INDEX_op_addci, TCGOutOpAddSubCarry, outop_addci), 1184 OUTOP(INDEX_op_addcio, TCGOutOpBinary, outop_addcio), 1185 OUTOP(INDEX_op_addco, TCGOutOpBinary, outop_addco), 1186 /* addc1o is implemented with set_carry + addcio */ 1187 OUTOP(INDEX_op_addc1o, TCGOutOpBinary, outop_addcio), 1188 OUTOP(INDEX_op_and, TCGOutOpBinary, outop_and), 1189 OUTOP(INDEX_op_andc, TCGOutOpBinary, outop_andc), 1190 OUTOP(INDEX_op_brcond, TCGOutOpBrcond, outop_brcond), 1191 OUTOP(INDEX_op_bswap16, TCGOutOpBswap, outop_bswap16), 1192 OUTOP(INDEX_op_bswap32, TCGOutOpBswap, outop_bswap32), 1193 OUTOP(INDEX_op_clz, TCGOutOpBinary, outop_clz), 1194 OUTOP(INDEX_op_ctpop, TCGOutOpUnary, outop_ctpop), 1195 OUTOP(INDEX_op_ctz, TCGOutOpBinary, outop_ctz), 1196 OUTOP(INDEX_op_deposit, TCGOutOpDeposit, outop_deposit), 1197 OUTOP(INDEX_op_divs, TCGOutOpBinary, outop_divs), 1198 OUTOP(INDEX_op_divu, TCGOutOpBinary, outop_divu), 1199 OUTOP(INDEX_op_divs2, TCGOutOpDivRem, outop_divs2), 1200 OUTOP(INDEX_op_divu2, TCGOutOpDivRem, outop_divu2), 1201 OUTOP(INDEX_op_eqv, TCGOutOpBinary, outop_eqv), 1202 OUTOP(INDEX_op_extract, TCGOutOpExtract, outop_extract), 1203 OUTOP(INDEX_op_extract2, TCGOutOpExtract2, outop_extract2), 1204 OUTOP(INDEX_op_ld8u, TCGOutOpLoad, outop_ld8u), 1205 OUTOP(INDEX_op_ld8s, TCGOutOpLoad, outop_ld8s), 1206 OUTOP(INDEX_op_ld16u, TCGOutOpLoad, outop_ld16u), 1207 OUTOP(INDEX_op_ld16s, TCGOutOpLoad, outop_ld16s), 1208 OUTOP(INDEX_op_ld, TCGOutOpLoad, outop_ld), 1209 OUTOP(INDEX_op_movcond, TCGOutOpMovcond, outop_movcond), 1210 OUTOP(INDEX_op_mul, TCGOutOpBinary, outop_mul), 1211 OUTOP(INDEX_op_muls2, TCGOutOpMul2, outop_muls2), 1212 OUTOP(INDEX_op_mulsh, TCGOutOpBinary, outop_mulsh), 1213 OUTOP(INDEX_op_mulu2, TCGOutOpMul2, outop_mulu2), 1214 OUTOP(INDEX_op_muluh, TCGOutOpBinary, outop_muluh), 1215 OUTOP(INDEX_op_nand, TCGOutOpBinary, outop_nand), 1216 OUTOP(INDEX_op_neg, TCGOutOpUnary, outop_neg), 1217 OUTOP(INDEX_op_negsetcond, TCGOutOpSetcond, outop_negsetcond), 1218 OUTOP(INDEX_op_nor, TCGOutOpBinary, outop_nor), 1219 OUTOP(INDEX_op_not, TCGOutOpUnary, outop_not), 1220 OUTOP(INDEX_op_or, TCGOutOpBinary, outop_or), 1221 OUTOP(INDEX_op_orc, TCGOutOpBinary, outop_orc), 1222 OUTOP(INDEX_op_qemu_ld, TCGOutOpQemuLdSt, outop_qemu_ld), 1223 OUTOP(INDEX_op_qemu_ld2, TCGOutOpQemuLdSt2, outop_qemu_ld2), 1224 OUTOP(INDEX_op_qemu_st, TCGOutOpQemuLdSt, outop_qemu_st), 1225 OUTOP(INDEX_op_qemu_st2, TCGOutOpQemuLdSt2, outop_qemu_st2), 1226 OUTOP(INDEX_op_rems, TCGOutOpBinary, outop_rems), 1227 OUTOP(INDEX_op_remu, TCGOutOpBinary, outop_remu), 1228 OUTOP(INDEX_op_rotl, TCGOutOpBinary, outop_rotl), 1229 OUTOP(INDEX_op_rotr, TCGOutOpBinary, outop_rotr), 1230 OUTOP(INDEX_op_sar, TCGOutOpBinary, outop_sar), 1231 OUTOP(INDEX_op_setcond, TCGOutOpSetcond, outop_setcond), 1232 OUTOP(INDEX_op_sextract, TCGOutOpExtract, outop_sextract), 1233 OUTOP(INDEX_op_shl, TCGOutOpBinary, outop_shl), 1234 OUTOP(INDEX_op_shr, TCGOutOpBinary, outop_shr), 1235 OUTOP(INDEX_op_st, TCGOutOpStore, outop_st), 1236 OUTOP(INDEX_op_st8, TCGOutOpStore, outop_st8), 1237 OUTOP(INDEX_op_st16, TCGOutOpStore, outop_st16), 1238 OUTOP(INDEX_op_sub, TCGOutOpSubtract, outop_sub), 1239 OUTOP(INDEX_op_subbi, TCGOutOpAddSubCarry, outop_subbi), 1240 OUTOP(INDEX_op_subbio, TCGOutOpAddSubCarry, outop_subbio), 1241 OUTOP(INDEX_op_subbo, TCGOutOpAddSubCarry, outop_subbo), 1242 /* subb1o is implemented with set_borrow + subbio */ 1243 OUTOP(INDEX_op_subb1o, TCGOutOpAddSubCarry, outop_subbio), 1244 OUTOP(INDEX_op_xor, TCGOutOpBinary, outop_xor), 1245 1246 [INDEX_op_goto_ptr] = &outop_goto_ptr, 1247 1248 #if TCG_TARGET_REG_BITS == 32 1249 OUTOP(INDEX_op_brcond2_i32, TCGOutOpBrcond2, outop_brcond2), 1250 OUTOP(INDEX_op_setcond2_i32, TCGOutOpSetcond2, outop_setcond2), 1251 #else 1252 OUTOP(INDEX_op_bswap64, TCGOutOpUnary, outop_bswap64), 1253 OUTOP(INDEX_op_ext_i32_i64, TCGOutOpUnary, outop_exts_i32_i64), 1254 OUTOP(INDEX_op_extu_i32_i64, TCGOutOpUnary, outop_extu_i32_i64), 1255 OUTOP(INDEX_op_extrl_i64_i32, TCGOutOpUnary, outop_extrl_i64_i32), 1256 OUTOP(INDEX_op_extrh_i64_i32, TCGOutOpUnary, outop_extrh_i64_i32), 1257 OUTOP(INDEX_op_ld32u, TCGOutOpLoad, outop_ld32u), 1258 OUTOP(INDEX_op_ld32s, TCGOutOpLoad, outop_ld32s), 1259 OUTOP(INDEX_op_st32, TCGOutOpStore, outop_st), 1260 #endif 1261 }; 1262 1263 #undef OUTOP 1264 1265 /* 1266 * All TCG threads except the parent (i.e. the one that called tcg_context_init 1267 * and registered the target's TCG globals) must register with this function 1268 * before initiating translation. 1269 * 1270 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation 1271 * of tcg_region_init() for the reasoning behind this. 1272 * 1273 * In system-mode each caller registers its context in tcg_ctxs[]. Note that in 1274 * system-mode tcg_ctxs[] does not track tcg_ctx_init, since the initial context 1275 * is not used anymore for translation once this function is called. 1276 * 1277 * Not tracking tcg_init_ctx in tcg_ctxs[] in system-mode keeps code that 1278 * iterates over the array (e.g. tcg_code_size() the same for both system/user 1279 * modes. 1280 */ 1281 #ifdef CONFIG_USER_ONLY 1282 void tcg_register_thread(void) 1283 { 1284 tcg_ctx = &tcg_init_ctx; 1285 } 1286 #else 1287 void tcg_register_thread(void) 1288 { 1289 TCGContext *s = g_malloc(sizeof(*s)); 1290 unsigned int i, n; 1291 1292 *s = tcg_init_ctx; 1293 1294 /* Relink mem_base. */ 1295 for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) { 1296 if (tcg_init_ctx.temps[i].mem_base) { 1297 ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps; 1298 tcg_debug_assert(b >= 0 && b < n); 1299 s->temps[i].mem_base = &s->temps[b]; 1300 } 1301 } 1302 1303 /* Claim an entry in tcg_ctxs */ 1304 n = qatomic_fetch_inc(&tcg_cur_ctxs); 1305 g_assert(n < tcg_max_ctxs); 1306 qatomic_set(&tcg_ctxs[n], s); 1307 1308 if (n > 0) { 1309 tcg_region_initial_alloc(s); 1310 } 1311 1312 tcg_ctx = s; 1313 } 1314 #endif /* !CONFIG_USER_ONLY */ 1315 1316 /* pool based memory allocation */ 1317 void *tcg_malloc_internal(TCGContext *s, int size) 1318 { 1319 TCGPool *p; 1320 int pool_size; 1321 1322 if (size > TCG_POOL_CHUNK_SIZE) { 1323 /* big malloc: insert a new pool (XXX: could optimize) */ 1324 p = g_malloc(sizeof(TCGPool) + size); 1325 p->size = size; 1326 p->next = s->pool_first_large; 1327 s->pool_first_large = p; 1328 return p->data; 1329 } else { 1330 p = s->pool_current; 1331 if (!p) { 1332 p = s->pool_first; 1333 if (!p) 1334 goto new_pool; 1335 } else { 1336 if (!p->next) { 1337 new_pool: 1338 pool_size = TCG_POOL_CHUNK_SIZE; 1339 p = g_malloc(sizeof(TCGPool) + pool_size); 1340 p->size = pool_size; 1341 p->next = NULL; 1342 if (s->pool_current) { 1343 s->pool_current->next = p; 1344 } else { 1345 s->pool_first = p; 1346 } 1347 } else { 1348 p = p->next; 1349 } 1350 } 1351 } 1352 s->pool_current = p; 1353 s->pool_cur = p->data + size; 1354 s->pool_end = p->data + p->size; 1355 return p->data; 1356 } 1357 1358 void tcg_pool_reset(TCGContext *s) 1359 { 1360 TCGPool *p, *t; 1361 for (p = s->pool_first_large; p; p = t) { 1362 t = p->next; 1363 g_free(p); 1364 } 1365 s->pool_first_large = NULL; 1366 s->pool_cur = s->pool_end = NULL; 1367 s->pool_current = NULL; 1368 } 1369 1370 /* 1371 * Create TCGHelperInfo structures for "tcg/tcg-ldst.h" functions, 1372 * akin to what "exec/helper-tcg.h" does with DEF_HELPER_FLAGS_N. 1373 * We only use these for layout in tcg_out_ld_helper_ret and 1374 * tcg_out_st_helper_args, and share them between several of 1375 * the helpers, with the end result that it's easier to build manually. 1376 */ 1377 1378 #if TCG_TARGET_REG_BITS == 32 1379 # define dh_typecode_ttl dh_typecode_i32 1380 #else 1381 # define dh_typecode_ttl dh_typecode_i64 1382 #endif 1383 1384 static TCGHelperInfo info_helper_ld32_mmu = { 1385 .flags = TCG_CALL_NO_WG, 1386 .typemask = dh_typemask(ttl, 0) /* return tcg_target_ulong */ 1387 | dh_typemask(env, 1) 1388 | dh_typemask(i64, 2) /* uint64_t addr */ 1389 | dh_typemask(i32, 3) /* unsigned oi */ 1390 | dh_typemask(ptr, 4) /* uintptr_t ra */ 1391 }; 1392 1393 static TCGHelperInfo info_helper_ld64_mmu = { 1394 .flags = TCG_CALL_NO_WG, 1395 .typemask = dh_typemask(i64, 0) /* return uint64_t */ 1396 | dh_typemask(env, 1) 1397 | dh_typemask(i64, 2) /* uint64_t addr */ 1398 | dh_typemask(i32, 3) /* unsigned oi */ 1399 | dh_typemask(ptr, 4) /* uintptr_t ra */ 1400 }; 1401 1402 static TCGHelperInfo info_helper_ld128_mmu = { 1403 .flags = TCG_CALL_NO_WG, 1404 .typemask = dh_typemask(i128, 0) /* return Int128 */ 1405 | dh_typemask(env, 1) 1406 | dh_typemask(i64, 2) /* uint64_t addr */ 1407 | dh_typemask(i32, 3) /* unsigned oi */ 1408 | dh_typemask(ptr, 4) /* uintptr_t ra */ 1409 }; 1410 1411 static TCGHelperInfo info_helper_st32_mmu = { 1412 .flags = TCG_CALL_NO_WG, 1413 .typemask = dh_typemask(void, 0) 1414 | dh_typemask(env, 1) 1415 | dh_typemask(i64, 2) /* uint64_t addr */ 1416 | dh_typemask(i32, 3) /* uint32_t data */ 1417 | dh_typemask(i32, 4) /* unsigned oi */ 1418 | dh_typemask(ptr, 5) /* uintptr_t ra */ 1419 }; 1420 1421 static TCGHelperInfo info_helper_st64_mmu = { 1422 .flags = TCG_CALL_NO_WG, 1423 .typemask = dh_typemask(void, 0) 1424 | dh_typemask(env, 1) 1425 | dh_typemask(i64, 2) /* uint64_t addr */ 1426 | dh_typemask(i64, 3) /* uint64_t data */ 1427 | dh_typemask(i32, 4) /* unsigned oi */ 1428 | dh_typemask(ptr, 5) /* uintptr_t ra */ 1429 }; 1430 1431 static TCGHelperInfo info_helper_st128_mmu = { 1432 .flags = TCG_CALL_NO_WG, 1433 .typemask = dh_typemask(void, 0) 1434 | dh_typemask(env, 1) 1435 | dh_typemask(i64, 2) /* uint64_t addr */ 1436 | dh_typemask(i128, 3) /* Int128 data */ 1437 | dh_typemask(i32, 4) /* unsigned oi */ 1438 | dh_typemask(ptr, 5) /* uintptr_t ra */ 1439 }; 1440 1441 #ifdef CONFIG_TCG_INTERPRETER 1442 static ffi_type *typecode_to_ffi(int argmask) 1443 { 1444 /* 1445 * libffi does not support __int128_t, so we have forced Int128 1446 * to use the structure definition instead of the builtin type. 1447 */ 1448 static ffi_type *ffi_type_i128_elements[3] = { 1449 &ffi_type_uint64, 1450 &ffi_type_uint64, 1451 NULL 1452 }; 1453 static ffi_type ffi_type_i128 = { 1454 .size = 16, 1455 .alignment = __alignof__(Int128), 1456 .type = FFI_TYPE_STRUCT, 1457 .elements = ffi_type_i128_elements, 1458 }; 1459 1460 switch (argmask) { 1461 case dh_typecode_void: 1462 return &ffi_type_void; 1463 case dh_typecode_i32: 1464 return &ffi_type_uint32; 1465 case dh_typecode_s32: 1466 return &ffi_type_sint32; 1467 case dh_typecode_i64: 1468 return &ffi_type_uint64; 1469 case dh_typecode_s64: 1470 return &ffi_type_sint64; 1471 case dh_typecode_ptr: 1472 return &ffi_type_pointer; 1473 case dh_typecode_i128: 1474 return &ffi_type_i128; 1475 } 1476 g_assert_not_reached(); 1477 } 1478 1479 static ffi_cif *init_ffi_layout(TCGHelperInfo *info) 1480 { 1481 unsigned typemask = info->typemask; 1482 struct { 1483 ffi_cif cif; 1484 ffi_type *args[]; 1485 } *ca; 1486 ffi_status status; 1487 int nargs; 1488 1489 /* Ignoring the return type, find the last non-zero field. */ 1490 nargs = 32 - clz32(typemask >> 3); 1491 nargs = DIV_ROUND_UP(nargs, 3); 1492 assert(nargs <= MAX_CALL_IARGS); 1493 1494 ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *)); 1495 ca->cif.rtype = typecode_to_ffi(typemask & 7); 1496 ca->cif.nargs = nargs; 1497 1498 if (nargs != 0) { 1499 ca->cif.arg_types = ca->args; 1500 for (int j = 0; j < nargs; ++j) { 1501 int typecode = extract32(typemask, (j + 1) * 3, 3); 1502 ca->args[j] = typecode_to_ffi(typecode); 1503 } 1504 } 1505 1506 status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs, 1507 ca->cif.rtype, ca->cif.arg_types); 1508 assert(status == FFI_OK); 1509 1510 return &ca->cif; 1511 } 1512 1513 #define HELPER_INFO_INIT(I) (&(I)->cif) 1514 #define HELPER_INFO_INIT_VAL(I) init_ffi_layout(I) 1515 #else 1516 #define HELPER_INFO_INIT(I) (&(I)->init) 1517 #define HELPER_INFO_INIT_VAL(I) 1 1518 #endif /* CONFIG_TCG_INTERPRETER */ 1519 1520 static inline bool arg_slot_reg_p(unsigned arg_slot) 1521 { 1522 /* 1523 * Split the sizeof away from the comparison to avoid Werror from 1524 * "unsigned < 0 is always false", when iarg_regs is empty. 1525 */ 1526 unsigned nreg = ARRAY_SIZE(tcg_target_call_iarg_regs); 1527 return arg_slot < nreg; 1528 } 1529 1530 static inline int arg_slot_stk_ofs(unsigned arg_slot) 1531 { 1532 unsigned max = TCG_STATIC_CALL_ARGS_SIZE / sizeof(tcg_target_long); 1533 unsigned stk_slot = arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs); 1534 1535 tcg_debug_assert(stk_slot < max); 1536 return TCG_TARGET_CALL_STACK_OFFSET + stk_slot * sizeof(tcg_target_long); 1537 } 1538 1539 typedef struct TCGCumulativeArgs { 1540 int arg_idx; /* tcg_gen_callN args[] */ 1541 int info_in_idx; /* TCGHelperInfo in[] */ 1542 int arg_slot; /* regs+stack slot */ 1543 int ref_slot; /* stack slots for references */ 1544 } TCGCumulativeArgs; 1545 1546 static void layout_arg_even(TCGCumulativeArgs *cum) 1547 { 1548 cum->arg_slot += cum->arg_slot & 1; 1549 } 1550 1551 static void layout_arg_1(TCGCumulativeArgs *cum, TCGHelperInfo *info, 1552 TCGCallArgumentKind kind) 1553 { 1554 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx]; 1555 1556 *loc = (TCGCallArgumentLoc){ 1557 .kind = kind, 1558 .arg_idx = cum->arg_idx, 1559 .arg_slot = cum->arg_slot, 1560 }; 1561 cum->info_in_idx++; 1562 cum->arg_slot++; 1563 } 1564 1565 static void layout_arg_normal_n(TCGCumulativeArgs *cum, 1566 TCGHelperInfo *info, int n) 1567 { 1568 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx]; 1569 1570 for (int i = 0; i < n; ++i) { 1571 /* Layout all using the same arg_idx, adjusting the subindex. */ 1572 loc[i] = (TCGCallArgumentLoc){ 1573 .kind = TCG_CALL_ARG_NORMAL, 1574 .arg_idx = cum->arg_idx, 1575 .tmp_subindex = i, 1576 .arg_slot = cum->arg_slot + i, 1577 }; 1578 } 1579 cum->info_in_idx += n; 1580 cum->arg_slot += n; 1581 } 1582 1583 static void layout_arg_by_ref(TCGCumulativeArgs *cum, TCGHelperInfo *info) 1584 { 1585 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx]; 1586 int n = 128 / TCG_TARGET_REG_BITS; 1587 1588 /* The first subindex carries the pointer. */ 1589 layout_arg_1(cum, info, TCG_CALL_ARG_BY_REF); 1590 1591 /* 1592 * The callee is allowed to clobber memory associated with 1593 * structure pass by-reference. Therefore we must make copies. 1594 * Allocate space from "ref_slot", which will be adjusted to 1595 * follow the parameters on the stack. 1596 */ 1597 loc[0].ref_slot = cum->ref_slot; 1598 1599 /* 1600 * Subsequent words also go into the reference slot, but 1601 * do not accumulate into the regular arguments. 1602 */ 1603 for (int i = 1; i < n; ++i) { 1604 loc[i] = (TCGCallArgumentLoc){ 1605 .kind = TCG_CALL_ARG_BY_REF_N, 1606 .arg_idx = cum->arg_idx, 1607 .tmp_subindex = i, 1608 .ref_slot = cum->ref_slot + i, 1609 }; 1610 } 1611 cum->info_in_idx += n - 1; /* i=0 accounted for in layout_arg_1 */ 1612 cum->ref_slot += n; 1613 } 1614 1615 static void init_call_layout(TCGHelperInfo *info) 1616 { 1617 int max_reg_slots = ARRAY_SIZE(tcg_target_call_iarg_regs); 1618 int max_stk_slots = TCG_STATIC_CALL_ARGS_SIZE / sizeof(tcg_target_long); 1619 unsigned typemask = info->typemask; 1620 unsigned typecode; 1621 TCGCumulativeArgs cum = { }; 1622 1623 /* 1624 * Parse and place any function return value. 1625 */ 1626 typecode = typemask & 7; 1627 switch (typecode) { 1628 case dh_typecode_void: 1629 info->nr_out = 0; 1630 break; 1631 case dh_typecode_i32: 1632 case dh_typecode_s32: 1633 case dh_typecode_ptr: 1634 info->nr_out = 1; 1635 info->out_kind = TCG_CALL_RET_NORMAL; 1636 break; 1637 case dh_typecode_i64: 1638 case dh_typecode_s64: 1639 info->nr_out = 64 / TCG_TARGET_REG_BITS; 1640 info->out_kind = TCG_CALL_RET_NORMAL; 1641 /* Query the last register now to trigger any assert early. */ 1642 tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1); 1643 break; 1644 case dh_typecode_i128: 1645 info->nr_out = 128 / TCG_TARGET_REG_BITS; 1646 info->out_kind = TCG_TARGET_CALL_RET_I128; 1647 switch (TCG_TARGET_CALL_RET_I128) { 1648 case TCG_CALL_RET_NORMAL: 1649 /* Query the last register now to trigger any assert early. */ 1650 tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1); 1651 break; 1652 case TCG_CALL_RET_BY_VEC: 1653 /* Query the single register now to trigger any assert early. */ 1654 tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0); 1655 break; 1656 case TCG_CALL_RET_BY_REF: 1657 /* 1658 * Allocate the first argument to the output. 1659 * We don't need to store this anywhere, just make it 1660 * unavailable for use in the input loop below. 1661 */ 1662 cum.arg_slot = 1; 1663 break; 1664 default: 1665 qemu_build_not_reached(); 1666 } 1667 break; 1668 default: 1669 g_assert_not_reached(); 1670 } 1671 1672 /* 1673 * Parse and place function arguments. 1674 */ 1675 for (typemask >>= 3; typemask; typemask >>= 3, cum.arg_idx++) { 1676 TCGCallArgumentKind kind; 1677 TCGType type; 1678 1679 typecode = typemask & 7; 1680 switch (typecode) { 1681 case dh_typecode_i32: 1682 case dh_typecode_s32: 1683 type = TCG_TYPE_I32; 1684 break; 1685 case dh_typecode_i64: 1686 case dh_typecode_s64: 1687 type = TCG_TYPE_I64; 1688 break; 1689 case dh_typecode_ptr: 1690 type = TCG_TYPE_PTR; 1691 break; 1692 case dh_typecode_i128: 1693 type = TCG_TYPE_I128; 1694 break; 1695 default: 1696 g_assert_not_reached(); 1697 } 1698 1699 switch (type) { 1700 case TCG_TYPE_I32: 1701 switch (TCG_TARGET_CALL_ARG_I32) { 1702 case TCG_CALL_ARG_EVEN: 1703 layout_arg_even(&cum); 1704 /* fall through */ 1705 case TCG_CALL_ARG_NORMAL: 1706 layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL); 1707 break; 1708 case TCG_CALL_ARG_EXTEND: 1709 kind = TCG_CALL_ARG_EXTEND_U + (typecode & 1); 1710 layout_arg_1(&cum, info, kind); 1711 break; 1712 default: 1713 qemu_build_not_reached(); 1714 } 1715 break; 1716 1717 case TCG_TYPE_I64: 1718 switch (TCG_TARGET_CALL_ARG_I64) { 1719 case TCG_CALL_ARG_EVEN: 1720 layout_arg_even(&cum); 1721 /* fall through */ 1722 case TCG_CALL_ARG_NORMAL: 1723 if (TCG_TARGET_REG_BITS == 32) { 1724 layout_arg_normal_n(&cum, info, 2); 1725 } else { 1726 layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL); 1727 } 1728 break; 1729 default: 1730 qemu_build_not_reached(); 1731 } 1732 break; 1733 1734 case TCG_TYPE_I128: 1735 switch (TCG_TARGET_CALL_ARG_I128) { 1736 case TCG_CALL_ARG_EVEN: 1737 layout_arg_even(&cum); 1738 /* fall through */ 1739 case TCG_CALL_ARG_NORMAL: 1740 layout_arg_normal_n(&cum, info, 128 / TCG_TARGET_REG_BITS); 1741 break; 1742 case TCG_CALL_ARG_BY_REF: 1743 layout_arg_by_ref(&cum, info); 1744 break; 1745 default: 1746 qemu_build_not_reached(); 1747 } 1748 break; 1749 1750 default: 1751 g_assert_not_reached(); 1752 } 1753 } 1754 info->nr_in = cum.info_in_idx; 1755 1756 /* Validate that we didn't overrun the input array. */ 1757 assert(cum.info_in_idx <= ARRAY_SIZE(info->in)); 1758 /* Validate the backend has enough argument space. */ 1759 assert(cum.arg_slot <= max_reg_slots + max_stk_slots); 1760 1761 /* 1762 * Relocate the "ref_slot" area to the end of the parameters. 1763 * Minimizing this stack offset helps code size for x86, 1764 * which has a signed 8-bit offset encoding. 1765 */ 1766 if (cum.ref_slot != 0) { 1767 int ref_base = 0; 1768 1769 if (cum.arg_slot > max_reg_slots) { 1770 int align = __alignof(Int128) / sizeof(tcg_target_long); 1771 1772 ref_base = cum.arg_slot - max_reg_slots; 1773 if (align > 1) { 1774 ref_base = ROUND_UP(ref_base, align); 1775 } 1776 } 1777 assert(ref_base + cum.ref_slot <= max_stk_slots); 1778 ref_base += max_reg_slots; 1779 1780 if (ref_base != 0) { 1781 for (int i = cum.info_in_idx - 1; i >= 0; --i) { 1782 TCGCallArgumentLoc *loc = &info->in[i]; 1783 switch (loc->kind) { 1784 case TCG_CALL_ARG_BY_REF: 1785 case TCG_CALL_ARG_BY_REF_N: 1786 loc->ref_slot += ref_base; 1787 break; 1788 default: 1789 break; 1790 } 1791 } 1792 } 1793 } 1794 } 1795 1796 static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)]; 1797 static void process_constraint_sets(void); 1798 static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type, 1799 TCGReg reg, const char *name); 1800 1801 static void tcg_context_init(unsigned max_threads) 1802 { 1803 TCGContext *s = &tcg_init_ctx; 1804 int n, i; 1805 TCGTemp *ts; 1806 1807 memset(s, 0, sizeof(*s)); 1808 s->nb_globals = 0; 1809 1810 init_call_layout(&info_helper_ld32_mmu); 1811 init_call_layout(&info_helper_ld64_mmu); 1812 init_call_layout(&info_helper_ld128_mmu); 1813 init_call_layout(&info_helper_st32_mmu); 1814 init_call_layout(&info_helper_st64_mmu); 1815 init_call_layout(&info_helper_st128_mmu); 1816 1817 tcg_target_init(s); 1818 process_constraint_sets(); 1819 1820 /* Reverse the order of the saved registers, assuming they're all at 1821 the start of tcg_target_reg_alloc_order. */ 1822 for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) { 1823 int r = tcg_target_reg_alloc_order[n]; 1824 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) { 1825 break; 1826 } 1827 } 1828 for (i = 0; i < n; ++i) { 1829 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i]; 1830 } 1831 for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) { 1832 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i]; 1833 } 1834 1835 tcg_ctx = s; 1836 /* 1837 * In user-mode we simply share the init context among threads, since we 1838 * use a single region. See the documentation tcg_region_init() for the 1839 * reasoning behind this. 1840 * In system-mode we will have at most max_threads TCG threads. 1841 */ 1842 #ifdef CONFIG_USER_ONLY 1843 tcg_ctxs = &tcg_ctx; 1844 tcg_cur_ctxs = 1; 1845 tcg_max_ctxs = 1; 1846 #else 1847 tcg_max_ctxs = max_threads; 1848 tcg_ctxs = g_new0(TCGContext *, max_threads); 1849 #endif 1850 1851 tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0)); 1852 ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env"); 1853 tcg_env = temp_tcgv_ptr(ts); 1854 } 1855 1856 void tcg_init(size_t tb_size, int splitwx, unsigned max_threads) 1857 { 1858 tcg_context_init(max_threads); 1859 tcg_region_init(tb_size, splitwx, max_threads); 1860 } 1861 1862 /* 1863 * Allocate TBs right before their corresponding translated code, making 1864 * sure that TBs and code are on different cache lines. 1865 */ 1866 TranslationBlock *tcg_tb_alloc(TCGContext *s) 1867 { 1868 uintptr_t align = qemu_icache_linesize; 1869 TranslationBlock *tb; 1870 void *next; 1871 1872 retry: 1873 tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align); 1874 next = (void *)ROUND_UP((uintptr_t)(tb + 1), align); 1875 1876 if (unlikely(next > s->code_gen_highwater)) { 1877 if (tcg_region_alloc(s)) { 1878 return NULL; 1879 } 1880 goto retry; 1881 } 1882 qatomic_set(&s->code_gen_ptr, next); 1883 return tb; 1884 } 1885 1886 void tcg_prologue_init(void) 1887 { 1888 TCGContext *s = tcg_ctx; 1889 size_t prologue_size; 1890 1891 s->code_ptr = s->code_gen_ptr; 1892 s->code_buf = s->code_gen_ptr; 1893 s->data_gen_ptr = NULL; 1894 1895 #ifndef CONFIG_TCG_INTERPRETER 1896 tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr); 1897 #endif 1898 1899 s->pool_labels = NULL; 1900 1901 qemu_thread_jit_write(); 1902 /* Generate the prologue. */ 1903 tcg_target_qemu_prologue(s); 1904 1905 /* Allow the prologue to put e.g. guest_base into a pool entry. */ 1906 { 1907 int result = tcg_out_pool_finalize(s); 1908 tcg_debug_assert(result == 0); 1909 } 1910 1911 prologue_size = tcg_current_code_size(s); 1912 perf_report_prologue(s->code_gen_ptr, prologue_size); 1913 1914 #ifndef CONFIG_TCG_INTERPRETER 1915 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf), 1916 (uintptr_t)s->code_buf, prologue_size); 1917 #endif 1918 1919 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) { 1920 FILE *logfile = qemu_log_trylock(); 1921 if (logfile) { 1922 fprintf(logfile, "PROLOGUE: [size=%zu]\n", prologue_size); 1923 if (s->data_gen_ptr) { 1924 size_t code_size = s->data_gen_ptr - s->code_gen_ptr; 1925 size_t data_size = prologue_size - code_size; 1926 size_t i; 1927 1928 disas(logfile, s->code_gen_ptr, code_size); 1929 1930 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) { 1931 if (sizeof(tcg_target_ulong) == 8) { 1932 fprintf(logfile, 1933 "0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n", 1934 (uintptr_t)s->data_gen_ptr + i, 1935 *(uint64_t *)(s->data_gen_ptr + i)); 1936 } else { 1937 fprintf(logfile, 1938 "0x%08" PRIxPTR ": .long 0x%08x\n", 1939 (uintptr_t)s->data_gen_ptr + i, 1940 *(uint32_t *)(s->data_gen_ptr + i)); 1941 } 1942 } 1943 } else { 1944 disas(logfile, s->code_gen_ptr, prologue_size); 1945 } 1946 fprintf(logfile, "\n"); 1947 qemu_log_unlock(logfile); 1948 } 1949 } 1950 1951 #ifndef CONFIG_TCG_INTERPRETER 1952 /* 1953 * Assert that goto_ptr is implemented completely, setting an epilogue. 1954 * For tci, we use NULL as the signal to return from the interpreter, 1955 * so skip this check. 1956 */ 1957 tcg_debug_assert(tcg_code_gen_epilogue != NULL); 1958 #endif 1959 1960 tcg_region_prologue_set(s); 1961 } 1962 1963 void tcg_func_start(TCGContext *s) 1964 { 1965 tcg_pool_reset(s); 1966 s->nb_temps = s->nb_globals; 1967 1968 /* No temps have been previously allocated for size or locality. */ 1969 tcg_temp_ebb_reset_freed(s); 1970 1971 /* No constant temps have been previously allocated. */ 1972 for (int i = 0; i < TCG_TYPE_COUNT; ++i) { 1973 if (s->const_table[i]) { 1974 g_hash_table_remove_all(s->const_table[i]); 1975 } 1976 } 1977 1978 s->nb_ops = 0; 1979 s->nb_labels = 0; 1980 s->current_frame_offset = s->frame_start; 1981 1982 #ifdef CONFIG_DEBUG_TCG 1983 s->goto_tb_issue_mask = 0; 1984 #endif 1985 1986 QTAILQ_INIT(&s->ops); 1987 QTAILQ_INIT(&s->free_ops); 1988 s->emit_before_op = NULL; 1989 QSIMPLEQ_INIT(&s->labels); 1990 1991 tcg_debug_assert(s->addr_type <= TCG_TYPE_REG); 1992 tcg_debug_assert(s->insn_start_words > 0); 1993 } 1994 1995 static TCGTemp *tcg_temp_alloc(TCGContext *s) 1996 { 1997 int n = s->nb_temps++; 1998 1999 if (n >= TCG_MAX_TEMPS) { 2000 tcg_raise_tb_overflow(s); 2001 } 2002 return memset(&s->temps[n], 0, sizeof(TCGTemp)); 2003 } 2004 2005 static TCGTemp *tcg_global_alloc(TCGContext *s) 2006 { 2007 TCGTemp *ts; 2008 2009 tcg_debug_assert(s->nb_globals == s->nb_temps); 2010 tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS); 2011 s->nb_globals++; 2012 ts = tcg_temp_alloc(s); 2013 ts->kind = TEMP_GLOBAL; 2014 2015 return ts; 2016 } 2017 2018 static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type, 2019 TCGReg reg, const char *name) 2020 { 2021 TCGTemp *ts; 2022 2023 tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32); 2024 2025 ts = tcg_global_alloc(s); 2026 ts->base_type = type; 2027 ts->type = type; 2028 ts->kind = TEMP_FIXED; 2029 ts->reg = reg; 2030 ts->name = name; 2031 tcg_regset_set_reg(s->reserved_regs, reg); 2032 2033 return ts; 2034 } 2035 2036 void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size) 2037 { 2038 s->frame_start = start; 2039 s->frame_end = start + size; 2040 s->frame_temp 2041 = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame"); 2042 } 2043 2044 static TCGTemp *tcg_global_mem_new_internal(TCGv_ptr base, intptr_t offset, 2045 const char *name, TCGType type) 2046 { 2047 TCGContext *s = tcg_ctx; 2048 TCGTemp *base_ts = tcgv_ptr_temp(base); 2049 TCGTemp *ts = tcg_global_alloc(s); 2050 int indirect_reg = 0; 2051 2052 switch (base_ts->kind) { 2053 case TEMP_FIXED: 2054 break; 2055 case TEMP_GLOBAL: 2056 /* We do not support double-indirect registers. */ 2057 tcg_debug_assert(!base_ts->indirect_reg); 2058 base_ts->indirect_base = 1; 2059 s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64 2060 ? 2 : 1); 2061 indirect_reg = 1; 2062 break; 2063 default: 2064 g_assert_not_reached(); 2065 } 2066 2067 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 2068 TCGTemp *ts2 = tcg_global_alloc(s); 2069 char buf[64]; 2070 2071 ts->base_type = TCG_TYPE_I64; 2072 ts->type = TCG_TYPE_I32; 2073 ts->indirect_reg = indirect_reg; 2074 ts->mem_allocated = 1; 2075 ts->mem_base = base_ts; 2076 ts->mem_offset = offset; 2077 pstrcpy(buf, sizeof(buf), name); 2078 pstrcat(buf, sizeof(buf), "_0"); 2079 ts->name = strdup(buf); 2080 2081 tcg_debug_assert(ts2 == ts + 1); 2082 ts2->base_type = TCG_TYPE_I64; 2083 ts2->type = TCG_TYPE_I32; 2084 ts2->indirect_reg = indirect_reg; 2085 ts2->mem_allocated = 1; 2086 ts2->mem_base = base_ts; 2087 ts2->mem_offset = offset + 4; 2088 ts2->temp_subindex = 1; 2089 pstrcpy(buf, sizeof(buf), name); 2090 pstrcat(buf, sizeof(buf), "_1"); 2091 ts2->name = strdup(buf); 2092 } else { 2093 ts->base_type = type; 2094 ts->type = type; 2095 ts->indirect_reg = indirect_reg; 2096 ts->mem_allocated = 1; 2097 ts->mem_base = base_ts; 2098 ts->mem_offset = offset; 2099 ts->name = name; 2100 } 2101 return ts; 2102 } 2103 2104 TCGv_i32 tcg_global_mem_new_i32(TCGv_ptr reg, intptr_t off, const char *name) 2105 { 2106 TCGTemp *ts = tcg_global_mem_new_internal(reg, off, name, TCG_TYPE_I32); 2107 return temp_tcgv_i32(ts); 2108 } 2109 2110 TCGv_i64 tcg_global_mem_new_i64(TCGv_ptr reg, intptr_t off, const char *name) 2111 { 2112 TCGTemp *ts = tcg_global_mem_new_internal(reg, off, name, TCG_TYPE_I64); 2113 return temp_tcgv_i64(ts); 2114 } 2115 2116 TCGv_ptr tcg_global_mem_new_ptr(TCGv_ptr reg, intptr_t off, const char *name) 2117 { 2118 TCGTemp *ts = tcg_global_mem_new_internal(reg, off, name, TCG_TYPE_PTR); 2119 return temp_tcgv_ptr(ts); 2120 } 2121 2122 TCGTemp *tcg_temp_new_internal(TCGType type, TCGTempKind kind) 2123 { 2124 TCGContext *s = tcg_ctx; 2125 TCGTemp *ts; 2126 int n; 2127 2128 if (kind == TEMP_EBB) { 2129 int idx = find_first_bit(s->free_temps[type].l, TCG_MAX_TEMPS); 2130 2131 if (idx < TCG_MAX_TEMPS) { 2132 /* There is already an available temp with the right type. */ 2133 clear_bit(idx, s->free_temps[type].l); 2134 2135 ts = &s->temps[idx]; 2136 ts->temp_allocated = 1; 2137 tcg_debug_assert(ts->base_type == type); 2138 tcg_debug_assert(ts->kind == kind); 2139 return ts; 2140 } 2141 } else { 2142 tcg_debug_assert(kind == TEMP_TB); 2143 } 2144 2145 switch (type) { 2146 case TCG_TYPE_I32: 2147 case TCG_TYPE_V64: 2148 case TCG_TYPE_V128: 2149 case TCG_TYPE_V256: 2150 n = 1; 2151 break; 2152 case TCG_TYPE_I64: 2153 n = 64 / TCG_TARGET_REG_BITS; 2154 break; 2155 case TCG_TYPE_I128: 2156 n = 128 / TCG_TARGET_REG_BITS; 2157 break; 2158 default: 2159 g_assert_not_reached(); 2160 } 2161 2162 ts = tcg_temp_alloc(s); 2163 ts->base_type = type; 2164 ts->temp_allocated = 1; 2165 ts->kind = kind; 2166 2167 if (n == 1) { 2168 ts->type = type; 2169 } else { 2170 ts->type = TCG_TYPE_REG; 2171 2172 for (int i = 1; i < n; ++i) { 2173 TCGTemp *ts2 = tcg_temp_alloc(s); 2174 2175 tcg_debug_assert(ts2 == ts + i); 2176 ts2->base_type = type; 2177 ts2->type = TCG_TYPE_REG; 2178 ts2->temp_allocated = 1; 2179 ts2->temp_subindex = i; 2180 ts2->kind = kind; 2181 } 2182 } 2183 return ts; 2184 } 2185 2186 TCGv_i32 tcg_temp_new_i32(void) 2187 { 2188 return temp_tcgv_i32(tcg_temp_new_internal(TCG_TYPE_I32, TEMP_TB)); 2189 } 2190 2191 TCGv_i32 tcg_temp_ebb_new_i32(void) 2192 { 2193 return temp_tcgv_i32(tcg_temp_new_internal(TCG_TYPE_I32, TEMP_EBB)); 2194 } 2195 2196 TCGv_i64 tcg_temp_new_i64(void) 2197 { 2198 return temp_tcgv_i64(tcg_temp_new_internal(TCG_TYPE_I64, TEMP_TB)); 2199 } 2200 2201 TCGv_i64 tcg_temp_ebb_new_i64(void) 2202 { 2203 return temp_tcgv_i64(tcg_temp_new_internal(TCG_TYPE_I64, TEMP_EBB)); 2204 } 2205 2206 TCGv_ptr tcg_temp_new_ptr(void) 2207 { 2208 return temp_tcgv_ptr(tcg_temp_new_internal(TCG_TYPE_PTR, TEMP_TB)); 2209 } 2210 2211 TCGv_ptr tcg_temp_ebb_new_ptr(void) 2212 { 2213 return temp_tcgv_ptr(tcg_temp_new_internal(TCG_TYPE_PTR, TEMP_EBB)); 2214 } 2215 2216 TCGv_i128 tcg_temp_new_i128(void) 2217 { 2218 return temp_tcgv_i128(tcg_temp_new_internal(TCG_TYPE_I128, TEMP_TB)); 2219 } 2220 2221 TCGv_i128 tcg_temp_ebb_new_i128(void) 2222 { 2223 return temp_tcgv_i128(tcg_temp_new_internal(TCG_TYPE_I128, TEMP_EBB)); 2224 } 2225 2226 TCGv_vec tcg_temp_new_vec(TCGType type) 2227 { 2228 TCGTemp *t; 2229 2230 #ifdef CONFIG_DEBUG_TCG 2231 switch (type) { 2232 case TCG_TYPE_V64: 2233 assert(TCG_TARGET_HAS_v64); 2234 break; 2235 case TCG_TYPE_V128: 2236 assert(TCG_TARGET_HAS_v128); 2237 break; 2238 case TCG_TYPE_V256: 2239 assert(TCG_TARGET_HAS_v256); 2240 break; 2241 default: 2242 g_assert_not_reached(); 2243 } 2244 #endif 2245 2246 t = tcg_temp_new_internal(type, TEMP_EBB); 2247 return temp_tcgv_vec(t); 2248 } 2249 2250 /* Create a new temp of the same type as an existing temp. */ 2251 TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match) 2252 { 2253 TCGTemp *t = tcgv_vec_temp(match); 2254 2255 tcg_debug_assert(t->temp_allocated != 0); 2256 2257 t = tcg_temp_new_internal(t->base_type, TEMP_EBB); 2258 return temp_tcgv_vec(t); 2259 } 2260 2261 void tcg_temp_free_internal(TCGTemp *ts) 2262 { 2263 TCGContext *s = tcg_ctx; 2264 2265 switch (ts->kind) { 2266 case TEMP_CONST: 2267 case TEMP_TB: 2268 /* Silently ignore free. */ 2269 break; 2270 case TEMP_EBB: 2271 tcg_debug_assert(ts->temp_allocated != 0); 2272 ts->temp_allocated = 0; 2273 set_bit(temp_idx(ts), s->free_temps[ts->base_type].l); 2274 break; 2275 default: 2276 /* It never made sense to free TEMP_FIXED or TEMP_GLOBAL. */ 2277 g_assert_not_reached(); 2278 } 2279 } 2280 2281 void tcg_temp_free_i32(TCGv_i32 arg) 2282 { 2283 tcg_temp_free_internal(tcgv_i32_temp(arg)); 2284 } 2285 2286 void tcg_temp_free_i64(TCGv_i64 arg) 2287 { 2288 tcg_temp_free_internal(tcgv_i64_temp(arg)); 2289 } 2290 2291 void tcg_temp_free_i128(TCGv_i128 arg) 2292 { 2293 tcg_temp_free_internal(tcgv_i128_temp(arg)); 2294 } 2295 2296 void tcg_temp_free_ptr(TCGv_ptr arg) 2297 { 2298 tcg_temp_free_internal(tcgv_ptr_temp(arg)); 2299 } 2300 2301 void tcg_temp_free_vec(TCGv_vec arg) 2302 { 2303 tcg_temp_free_internal(tcgv_vec_temp(arg)); 2304 } 2305 2306 TCGTemp *tcg_constant_internal(TCGType type, int64_t val) 2307 { 2308 TCGContext *s = tcg_ctx; 2309 GHashTable *h = s->const_table[type]; 2310 TCGTemp *ts; 2311 2312 if (h == NULL) { 2313 h = g_hash_table_new(g_int64_hash, g_int64_equal); 2314 s->const_table[type] = h; 2315 } 2316 2317 ts = g_hash_table_lookup(h, &val); 2318 if (ts == NULL) { 2319 int64_t *val_ptr; 2320 2321 ts = tcg_temp_alloc(s); 2322 2323 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 2324 TCGTemp *ts2 = tcg_temp_alloc(s); 2325 2326 tcg_debug_assert(ts2 == ts + 1); 2327 2328 ts->base_type = TCG_TYPE_I64; 2329 ts->type = TCG_TYPE_I32; 2330 ts->kind = TEMP_CONST; 2331 ts->temp_allocated = 1; 2332 2333 ts2->base_type = TCG_TYPE_I64; 2334 ts2->type = TCG_TYPE_I32; 2335 ts2->kind = TEMP_CONST; 2336 ts2->temp_allocated = 1; 2337 ts2->temp_subindex = 1; 2338 2339 /* 2340 * Retain the full value of the 64-bit constant in the low 2341 * part, so that the hash table works. Actual uses will 2342 * truncate the value to the low part. 2343 */ 2344 ts[HOST_BIG_ENDIAN].val = val; 2345 ts[!HOST_BIG_ENDIAN].val = val >> 32; 2346 val_ptr = &ts[HOST_BIG_ENDIAN].val; 2347 } else { 2348 ts->base_type = type; 2349 ts->type = type; 2350 ts->kind = TEMP_CONST; 2351 ts->temp_allocated = 1; 2352 ts->val = val; 2353 val_ptr = &ts->val; 2354 } 2355 g_hash_table_insert(h, val_ptr, ts); 2356 } 2357 2358 return ts; 2359 } 2360 2361 TCGv_i32 tcg_constant_i32(int32_t val) 2362 { 2363 return temp_tcgv_i32(tcg_constant_internal(TCG_TYPE_I32, val)); 2364 } 2365 2366 TCGv_i64 tcg_constant_i64(int64_t val) 2367 { 2368 return temp_tcgv_i64(tcg_constant_internal(TCG_TYPE_I64, val)); 2369 } 2370 2371 TCGv_ptr tcg_constant_ptr_int(intptr_t val) 2372 { 2373 return temp_tcgv_ptr(tcg_constant_internal(TCG_TYPE_PTR, val)); 2374 } 2375 2376 TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val) 2377 { 2378 val = dup_const(vece, val); 2379 return temp_tcgv_vec(tcg_constant_internal(type, val)); 2380 } 2381 2382 TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val) 2383 { 2384 TCGTemp *t = tcgv_vec_temp(match); 2385 2386 tcg_debug_assert(t->temp_allocated != 0); 2387 return tcg_constant_vec(t->base_type, vece, val); 2388 } 2389 2390 #ifdef CONFIG_DEBUG_TCG 2391 size_t temp_idx(TCGTemp *ts) 2392 { 2393 ptrdiff_t n = ts - tcg_ctx->temps; 2394 assert(n >= 0 && n < tcg_ctx->nb_temps); 2395 return n; 2396 } 2397 2398 TCGTemp *tcgv_i32_temp(TCGv_i32 v) 2399 { 2400 uintptr_t o = (uintptr_t)v - offsetof(TCGContext, temps); 2401 2402 assert(o < sizeof(TCGTemp) * tcg_ctx->nb_temps); 2403 assert(o % sizeof(TCGTemp) == 0); 2404 2405 return (void *)tcg_ctx + (uintptr_t)v; 2406 } 2407 #endif /* CONFIG_DEBUG_TCG */ 2408 2409 /* 2410 * Return true if OP may appear in the opcode stream with TYPE. 2411 * Test the runtime variable that controls each opcode. 2412 */ 2413 bool tcg_op_supported(TCGOpcode op, TCGType type, unsigned flags) 2414 { 2415 bool has_type; 2416 2417 switch (type) { 2418 case TCG_TYPE_I32: 2419 has_type = true; 2420 break; 2421 case TCG_TYPE_I64: 2422 has_type = TCG_TARGET_REG_BITS == 64; 2423 break; 2424 case TCG_TYPE_V64: 2425 has_type = TCG_TARGET_HAS_v64; 2426 break; 2427 case TCG_TYPE_V128: 2428 has_type = TCG_TARGET_HAS_v128; 2429 break; 2430 case TCG_TYPE_V256: 2431 has_type = TCG_TARGET_HAS_v256; 2432 break; 2433 default: 2434 has_type = false; 2435 break; 2436 } 2437 2438 switch (op) { 2439 case INDEX_op_discard: 2440 case INDEX_op_set_label: 2441 case INDEX_op_call: 2442 case INDEX_op_br: 2443 case INDEX_op_mb: 2444 case INDEX_op_insn_start: 2445 case INDEX_op_exit_tb: 2446 case INDEX_op_goto_tb: 2447 case INDEX_op_goto_ptr: 2448 return true; 2449 2450 case INDEX_op_qemu_ld: 2451 case INDEX_op_qemu_st: 2452 tcg_debug_assert(type <= TCG_TYPE_REG); 2453 return true; 2454 2455 case INDEX_op_qemu_ld2: 2456 case INDEX_op_qemu_st2: 2457 if (TCG_TARGET_REG_BITS == 32) { 2458 tcg_debug_assert(type == TCG_TYPE_I64); 2459 return true; 2460 } 2461 tcg_debug_assert(type == TCG_TYPE_I128); 2462 goto do_lookup; 2463 2464 case INDEX_op_add: 2465 case INDEX_op_and: 2466 case INDEX_op_brcond: 2467 case INDEX_op_deposit: 2468 case INDEX_op_extract: 2469 case INDEX_op_ld8u: 2470 case INDEX_op_ld8s: 2471 case INDEX_op_ld16u: 2472 case INDEX_op_ld16s: 2473 case INDEX_op_ld: 2474 case INDEX_op_mov: 2475 case INDEX_op_movcond: 2476 case INDEX_op_negsetcond: 2477 case INDEX_op_or: 2478 case INDEX_op_setcond: 2479 case INDEX_op_sextract: 2480 case INDEX_op_st8: 2481 case INDEX_op_st16: 2482 case INDEX_op_st: 2483 case INDEX_op_xor: 2484 return has_type; 2485 2486 case INDEX_op_brcond2_i32: 2487 case INDEX_op_setcond2_i32: 2488 return TCG_TARGET_REG_BITS == 32; 2489 2490 case INDEX_op_ld32u: 2491 case INDEX_op_ld32s: 2492 case INDEX_op_st32: 2493 case INDEX_op_ext_i32_i64: 2494 case INDEX_op_extu_i32_i64: 2495 case INDEX_op_extrl_i64_i32: 2496 case INDEX_op_extrh_i64_i32: 2497 return TCG_TARGET_REG_BITS == 64; 2498 2499 case INDEX_op_mov_vec: 2500 case INDEX_op_dup_vec: 2501 case INDEX_op_dupm_vec: 2502 case INDEX_op_ld_vec: 2503 case INDEX_op_st_vec: 2504 case INDEX_op_add_vec: 2505 case INDEX_op_sub_vec: 2506 case INDEX_op_and_vec: 2507 case INDEX_op_or_vec: 2508 case INDEX_op_xor_vec: 2509 case INDEX_op_cmp_vec: 2510 return has_type; 2511 case INDEX_op_dup2_vec: 2512 return has_type && TCG_TARGET_REG_BITS == 32; 2513 case INDEX_op_not_vec: 2514 return has_type && TCG_TARGET_HAS_not_vec; 2515 case INDEX_op_neg_vec: 2516 return has_type && TCG_TARGET_HAS_neg_vec; 2517 case INDEX_op_abs_vec: 2518 return has_type && TCG_TARGET_HAS_abs_vec; 2519 case INDEX_op_andc_vec: 2520 return has_type && TCG_TARGET_HAS_andc_vec; 2521 case INDEX_op_orc_vec: 2522 return has_type && TCG_TARGET_HAS_orc_vec; 2523 case INDEX_op_nand_vec: 2524 return has_type && TCG_TARGET_HAS_nand_vec; 2525 case INDEX_op_nor_vec: 2526 return has_type && TCG_TARGET_HAS_nor_vec; 2527 case INDEX_op_eqv_vec: 2528 return has_type && TCG_TARGET_HAS_eqv_vec; 2529 case INDEX_op_mul_vec: 2530 return has_type && TCG_TARGET_HAS_mul_vec; 2531 case INDEX_op_shli_vec: 2532 case INDEX_op_shri_vec: 2533 case INDEX_op_sari_vec: 2534 return has_type && TCG_TARGET_HAS_shi_vec; 2535 case INDEX_op_shls_vec: 2536 case INDEX_op_shrs_vec: 2537 case INDEX_op_sars_vec: 2538 return has_type && TCG_TARGET_HAS_shs_vec; 2539 case INDEX_op_shlv_vec: 2540 case INDEX_op_shrv_vec: 2541 case INDEX_op_sarv_vec: 2542 return has_type && TCG_TARGET_HAS_shv_vec; 2543 case INDEX_op_rotli_vec: 2544 return has_type && TCG_TARGET_HAS_roti_vec; 2545 case INDEX_op_rotls_vec: 2546 return has_type && TCG_TARGET_HAS_rots_vec; 2547 case INDEX_op_rotlv_vec: 2548 case INDEX_op_rotrv_vec: 2549 return has_type && TCG_TARGET_HAS_rotv_vec; 2550 case INDEX_op_ssadd_vec: 2551 case INDEX_op_usadd_vec: 2552 case INDEX_op_sssub_vec: 2553 case INDEX_op_ussub_vec: 2554 return has_type && TCG_TARGET_HAS_sat_vec; 2555 case INDEX_op_smin_vec: 2556 case INDEX_op_umin_vec: 2557 case INDEX_op_smax_vec: 2558 case INDEX_op_umax_vec: 2559 return has_type && TCG_TARGET_HAS_minmax_vec; 2560 case INDEX_op_bitsel_vec: 2561 return has_type && TCG_TARGET_HAS_bitsel_vec; 2562 case INDEX_op_cmpsel_vec: 2563 return has_type && TCG_TARGET_HAS_cmpsel_vec; 2564 2565 default: 2566 if (op < INDEX_op_last_generic) { 2567 const TCGOutOp *outop; 2568 TCGConstraintSetIndex con_set; 2569 2570 if (!has_type) { 2571 return false; 2572 } 2573 2574 do_lookup: 2575 outop = all_outop[op]; 2576 tcg_debug_assert(outop != NULL); 2577 2578 con_set = outop->static_constraint; 2579 if (con_set == C_Dynamic) { 2580 con_set = outop->dynamic_constraint(type, flags); 2581 } 2582 if (con_set >= 0) { 2583 return true; 2584 } 2585 tcg_debug_assert(con_set == C_NotImplemented); 2586 return false; 2587 } 2588 tcg_debug_assert(op < NB_OPS); 2589 return true; 2590 2591 case INDEX_op_last_generic: 2592 g_assert_not_reached(); 2593 } 2594 } 2595 2596 bool tcg_op_deposit_valid(TCGType type, unsigned ofs, unsigned len) 2597 { 2598 unsigned width; 2599 2600 tcg_debug_assert(type == TCG_TYPE_I32 || type == TCG_TYPE_I64); 2601 width = (type == TCG_TYPE_I32 ? 32 : 64); 2602 2603 tcg_debug_assert(ofs < width); 2604 tcg_debug_assert(len > 0); 2605 tcg_debug_assert(len <= width - ofs); 2606 2607 return TCG_TARGET_deposit_valid(type, ofs, len); 2608 } 2609 2610 static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs); 2611 2612 static void tcg_gen_callN(void *func, TCGHelperInfo *info, 2613 TCGTemp *ret, TCGTemp **args) 2614 { 2615 TCGv_i64 extend_free[MAX_CALL_IARGS]; 2616 int n_extend = 0; 2617 TCGOp *op; 2618 int i, n, pi = 0, total_args; 2619 2620 if (unlikely(g_once_init_enter(HELPER_INFO_INIT(info)))) { 2621 init_call_layout(info); 2622 g_once_init_leave(HELPER_INFO_INIT(info), HELPER_INFO_INIT_VAL(info)); 2623 } 2624 2625 total_args = info->nr_out + info->nr_in + 2; 2626 op = tcg_op_alloc(INDEX_op_call, total_args); 2627 2628 #ifdef CONFIG_PLUGIN 2629 /* Flag helpers that may affect guest state */ 2630 if (tcg_ctx->plugin_insn && !(info->flags & TCG_CALL_NO_SIDE_EFFECTS)) { 2631 tcg_ctx->plugin_insn->calls_helpers = true; 2632 } 2633 #endif 2634 2635 TCGOP_CALLO(op) = n = info->nr_out; 2636 switch (n) { 2637 case 0: 2638 tcg_debug_assert(ret == NULL); 2639 break; 2640 case 1: 2641 tcg_debug_assert(ret != NULL); 2642 op->args[pi++] = temp_arg(ret); 2643 break; 2644 case 2: 2645 case 4: 2646 tcg_debug_assert(ret != NULL); 2647 tcg_debug_assert(ret->base_type == ret->type + ctz32(n)); 2648 tcg_debug_assert(ret->temp_subindex == 0); 2649 for (i = 0; i < n; ++i) { 2650 op->args[pi++] = temp_arg(ret + i); 2651 } 2652 break; 2653 default: 2654 g_assert_not_reached(); 2655 } 2656 2657 TCGOP_CALLI(op) = n = info->nr_in; 2658 for (i = 0; i < n; i++) { 2659 const TCGCallArgumentLoc *loc = &info->in[i]; 2660 TCGTemp *ts = args[loc->arg_idx] + loc->tmp_subindex; 2661 2662 switch (loc->kind) { 2663 case TCG_CALL_ARG_NORMAL: 2664 case TCG_CALL_ARG_BY_REF: 2665 case TCG_CALL_ARG_BY_REF_N: 2666 op->args[pi++] = temp_arg(ts); 2667 break; 2668 2669 case TCG_CALL_ARG_EXTEND_U: 2670 case TCG_CALL_ARG_EXTEND_S: 2671 { 2672 TCGv_i64 temp = tcg_temp_ebb_new_i64(); 2673 TCGv_i32 orig = temp_tcgv_i32(ts); 2674 2675 if (loc->kind == TCG_CALL_ARG_EXTEND_S) { 2676 tcg_gen_ext_i32_i64(temp, orig); 2677 } else { 2678 tcg_gen_extu_i32_i64(temp, orig); 2679 } 2680 op->args[pi++] = tcgv_i64_arg(temp); 2681 extend_free[n_extend++] = temp; 2682 } 2683 break; 2684 2685 default: 2686 g_assert_not_reached(); 2687 } 2688 } 2689 op->args[pi++] = (uintptr_t)func; 2690 op->args[pi++] = (uintptr_t)info; 2691 tcg_debug_assert(pi == total_args); 2692 2693 if (tcg_ctx->emit_before_op) { 2694 QTAILQ_INSERT_BEFORE(tcg_ctx->emit_before_op, op, link); 2695 } else { 2696 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link); 2697 } 2698 2699 tcg_debug_assert(n_extend < ARRAY_SIZE(extend_free)); 2700 for (i = 0; i < n_extend; ++i) { 2701 tcg_temp_free_i64(extend_free[i]); 2702 } 2703 } 2704 2705 void tcg_gen_call0(void *func, TCGHelperInfo *info, TCGTemp *ret) 2706 { 2707 tcg_gen_callN(func, info, ret, NULL); 2708 } 2709 2710 void tcg_gen_call1(void *func, TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1) 2711 { 2712 tcg_gen_callN(func, info, ret, &t1); 2713 } 2714 2715 void tcg_gen_call2(void *func, TCGHelperInfo *info, TCGTemp *ret, 2716 TCGTemp *t1, TCGTemp *t2) 2717 { 2718 TCGTemp *args[2] = { t1, t2 }; 2719 tcg_gen_callN(func, info, ret, args); 2720 } 2721 2722 void tcg_gen_call3(void *func, TCGHelperInfo *info, TCGTemp *ret, 2723 TCGTemp *t1, TCGTemp *t2, TCGTemp *t3) 2724 { 2725 TCGTemp *args[3] = { t1, t2, t3 }; 2726 tcg_gen_callN(func, info, ret, args); 2727 } 2728 2729 void tcg_gen_call4(void *func, TCGHelperInfo *info, TCGTemp *ret, 2730 TCGTemp *t1, TCGTemp *t2, TCGTemp *t3, TCGTemp *t4) 2731 { 2732 TCGTemp *args[4] = { t1, t2, t3, t4 }; 2733 tcg_gen_callN(func, info, ret, args); 2734 } 2735 2736 void tcg_gen_call5(void *func, TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1, 2737 TCGTemp *t2, TCGTemp *t3, TCGTemp *t4, TCGTemp *t5) 2738 { 2739 TCGTemp *args[5] = { t1, t2, t3, t4, t5 }; 2740 tcg_gen_callN(func, info, ret, args); 2741 } 2742 2743 void tcg_gen_call6(void *func, TCGHelperInfo *info, TCGTemp *ret, 2744 TCGTemp *t1, TCGTemp *t2, TCGTemp *t3, 2745 TCGTemp *t4, TCGTemp *t5, TCGTemp *t6) 2746 { 2747 TCGTemp *args[6] = { t1, t2, t3, t4, t5, t6 }; 2748 tcg_gen_callN(func, info, ret, args); 2749 } 2750 2751 void tcg_gen_call7(void *func, TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1, 2752 TCGTemp *t2, TCGTemp *t3, TCGTemp *t4, 2753 TCGTemp *t5, TCGTemp *t6, TCGTemp *t7) 2754 { 2755 TCGTemp *args[7] = { t1, t2, t3, t4, t5, t6, t7 }; 2756 tcg_gen_callN(func, info, ret, args); 2757 } 2758 2759 static void tcg_reg_alloc_start(TCGContext *s) 2760 { 2761 int i, n; 2762 2763 for (i = 0, n = s->nb_temps; i < n; i++) { 2764 TCGTemp *ts = &s->temps[i]; 2765 TCGTempVal val = TEMP_VAL_MEM; 2766 2767 switch (ts->kind) { 2768 case TEMP_CONST: 2769 val = TEMP_VAL_CONST; 2770 break; 2771 case TEMP_FIXED: 2772 val = TEMP_VAL_REG; 2773 break; 2774 case TEMP_GLOBAL: 2775 break; 2776 case TEMP_EBB: 2777 val = TEMP_VAL_DEAD; 2778 /* fall through */ 2779 case TEMP_TB: 2780 ts->mem_allocated = 0; 2781 break; 2782 default: 2783 g_assert_not_reached(); 2784 } 2785 ts->val_type = val; 2786 } 2787 2788 memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp)); 2789 } 2790 2791 static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size, 2792 TCGTemp *ts) 2793 { 2794 int idx = temp_idx(ts); 2795 2796 switch (ts->kind) { 2797 case TEMP_FIXED: 2798 case TEMP_GLOBAL: 2799 pstrcpy(buf, buf_size, ts->name); 2800 break; 2801 case TEMP_TB: 2802 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals); 2803 break; 2804 case TEMP_EBB: 2805 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals); 2806 break; 2807 case TEMP_CONST: 2808 switch (ts->type) { 2809 case TCG_TYPE_I32: 2810 snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val); 2811 break; 2812 #if TCG_TARGET_REG_BITS > 32 2813 case TCG_TYPE_I64: 2814 snprintf(buf, buf_size, "$0x%" PRIx64, ts->val); 2815 break; 2816 #endif 2817 case TCG_TYPE_V64: 2818 case TCG_TYPE_V128: 2819 case TCG_TYPE_V256: 2820 snprintf(buf, buf_size, "v%d$0x%" PRIx64, 2821 64 << (ts->type - TCG_TYPE_V64), ts->val); 2822 break; 2823 default: 2824 g_assert_not_reached(); 2825 } 2826 break; 2827 } 2828 return buf; 2829 } 2830 2831 static char *tcg_get_arg_str(TCGContext *s, char *buf, 2832 int buf_size, TCGArg arg) 2833 { 2834 return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg)); 2835 } 2836 2837 static const char * const cond_name[] = 2838 { 2839 [TCG_COND_NEVER] = "never", 2840 [TCG_COND_ALWAYS] = "always", 2841 [TCG_COND_EQ] = "eq", 2842 [TCG_COND_NE] = "ne", 2843 [TCG_COND_LT] = "lt", 2844 [TCG_COND_GE] = "ge", 2845 [TCG_COND_LE] = "le", 2846 [TCG_COND_GT] = "gt", 2847 [TCG_COND_LTU] = "ltu", 2848 [TCG_COND_GEU] = "geu", 2849 [TCG_COND_LEU] = "leu", 2850 [TCG_COND_GTU] = "gtu", 2851 [TCG_COND_TSTEQ] = "tsteq", 2852 [TCG_COND_TSTNE] = "tstne", 2853 }; 2854 2855 static const char * const ldst_name[(MO_BSWAP | MO_SSIZE) + 1] = 2856 { 2857 [MO_UB] = "ub", 2858 [MO_SB] = "sb", 2859 [MO_LEUW] = "leuw", 2860 [MO_LESW] = "lesw", 2861 [MO_LEUL] = "leul", 2862 [MO_LESL] = "lesl", 2863 [MO_LEUQ] = "leq", 2864 [MO_BEUW] = "beuw", 2865 [MO_BESW] = "besw", 2866 [MO_BEUL] = "beul", 2867 [MO_BESL] = "besl", 2868 [MO_BEUQ] = "beq", 2869 [MO_128 + MO_BE] = "beo", 2870 [MO_128 + MO_LE] = "leo", 2871 }; 2872 2873 static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = { 2874 [MO_UNALN >> MO_ASHIFT] = "un+", 2875 [MO_ALIGN >> MO_ASHIFT] = "al+", 2876 [MO_ALIGN_2 >> MO_ASHIFT] = "al2+", 2877 [MO_ALIGN_4 >> MO_ASHIFT] = "al4+", 2878 [MO_ALIGN_8 >> MO_ASHIFT] = "al8+", 2879 [MO_ALIGN_16 >> MO_ASHIFT] = "al16+", 2880 [MO_ALIGN_32 >> MO_ASHIFT] = "al32+", 2881 [MO_ALIGN_64 >> MO_ASHIFT] = "al64+", 2882 }; 2883 2884 static const char * const atom_name[(MO_ATOM_MASK >> MO_ATOM_SHIFT) + 1] = { 2885 [MO_ATOM_IFALIGN >> MO_ATOM_SHIFT] = "", 2886 [MO_ATOM_IFALIGN_PAIR >> MO_ATOM_SHIFT] = "pair+", 2887 [MO_ATOM_WITHIN16 >> MO_ATOM_SHIFT] = "w16+", 2888 [MO_ATOM_WITHIN16_PAIR >> MO_ATOM_SHIFT] = "w16p+", 2889 [MO_ATOM_SUBALIGN >> MO_ATOM_SHIFT] = "sub+", 2890 [MO_ATOM_NONE >> MO_ATOM_SHIFT] = "noat+", 2891 }; 2892 2893 static const char bswap_flag_name[][6] = { 2894 [TCG_BSWAP_IZ] = "iz", 2895 [TCG_BSWAP_OZ] = "oz", 2896 [TCG_BSWAP_OS] = "os", 2897 [TCG_BSWAP_IZ | TCG_BSWAP_OZ] = "iz,oz", 2898 [TCG_BSWAP_IZ | TCG_BSWAP_OS] = "iz,os", 2899 }; 2900 2901 #ifdef CONFIG_PLUGIN 2902 static const char * const plugin_from_name[] = { 2903 "from-tb", 2904 "from-insn", 2905 "after-insn", 2906 "after-tb", 2907 }; 2908 #endif 2909 2910 static inline bool tcg_regset_single(TCGRegSet d) 2911 { 2912 return (d & (d - 1)) == 0; 2913 } 2914 2915 static inline TCGReg tcg_regset_first(TCGRegSet d) 2916 { 2917 if (TCG_TARGET_NB_REGS <= 32) { 2918 return ctz32(d); 2919 } else { 2920 return ctz64(d); 2921 } 2922 } 2923 2924 /* Return only the number of characters output -- no error return. */ 2925 #define ne_fprintf(...) \ 2926 ({ int ret_ = fprintf(__VA_ARGS__); ret_ >= 0 ? ret_ : 0; }) 2927 2928 void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs) 2929 { 2930 char buf[128]; 2931 TCGOp *op; 2932 2933 QTAILQ_FOREACH(op, &s->ops, link) { 2934 int i, k, nb_oargs, nb_iargs, nb_cargs; 2935 const TCGOpDef *def; 2936 TCGOpcode c; 2937 int col = 0; 2938 2939 c = op->opc; 2940 def = &tcg_op_defs[c]; 2941 2942 if (c == INDEX_op_insn_start) { 2943 nb_oargs = 0; 2944 col += ne_fprintf(f, "\n ----"); 2945 2946 for (i = 0, k = s->insn_start_words; i < k; ++i) { 2947 col += ne_fprintf(f, " %016" PRIx64, 2948 tcg_get_insn_start_param(op, i)); 2949 } 2950 } else if (c == INDEX_op_call) { 2951 const TCGHelperInfo *info = tcg_call_info(op); 2952 void *func = tcg_call_func(op); 2953 2954 /* variable number of arguments */ 2955 nb_oargs = TCGOP_CALLO(op); 2956 nb_iargs = TCGOP_CALLI(op); 2957 nb_cargs = def->nb_cargs; 2958 2959 col += ne_fprintf(f, " %s ", def->name); 2960 2961 /* 2962 * Print the function name from TCGHelperInfo, if available. 2963 * Note that plugins have a template function for the info, 2964 * but the actual function pointer comes from the plugin. 2965 */ 2966 if (func == info->func) { 2967 col += ne_fprintf(f, "%s", info->name); 2968 } else { 2969 col += ne_fprintf(f, "plugin(%p)", func); 2970 } 2971 2972 col += ne_fprintf(f, ",$0x%x,$%d", info->flags, nb_oargs); 2973 for (i = 0; i < nb_oargs; i++) { 2974 col += ne_fprintf(f, ",%s", tcg_get_arg_str(s, buf, sizeof(buf), 2975 op->args[i])); 2976 } 2977 for (i = 0; i < nb_iargs; i++) { 2978 TCGArg arg = op->args[nb_oargs + i]; 2979 const char *t = tcg_get_arg_str(s, buf, sizeof(buf), arg); 2980 col += ne_fprintf(f, ",%s", t); 2981 } 2982 } else { 2983 if (def->flags & TCG_OPF_INT) { 2984 col += ne_fprintf(f, " %s_i%d ", 2985 def->name, 2986 8 * tcg_type_size(TCGOP_TYPE(op))); 2987 } else if (def->flags & TCG_OPF_VECTOR) { 2988 col += ne_fprintf(f, "%s v%d,e%d,", 2989 def->name, 2990 8 * tcg_type_size(TCGOP_TYPE(op)), 2991 8 << TCGOP_VECE(op)); 2992 } else { 2993 col += ne_fprintf(f, " %s ", def->name); 2994 } 2995 2996 nb_oargs = def->nb_oargs; 2997 nb_iargs = def->nb_iargs; 2998 nb_cargs = def->nb_cargs; 2999 3000 k = 0; 3001 for (i = 0; i < nb_oargs; i++) { 3002 const char *sep = k ? "," : ""; 3003 col += ne_fprintf(f, "%s%s", sep, 3004 tcg_get_arg_str(s, buf, sizeof(buf), 3005 op->args[k++])); 3006 } 3007 for (i = 0; i < nb_iargs; i++) { 3008 const char *sep = k ? "," : ""; 3009 col += ne_fprintf(f, "%s%s", sep, 3010 tcg_get_arg_str(s, buf, sizeof(buf), 3011 op->args[k++])); 3012 } 3013 switch (c) { 3014 case INDEX_op_brcond: 3015 case INDEX_op_setcond: 3016 case INDEX_op_negsetcond: 3017 case INDEX_op_movcond: 3018 case INDEX_op_brcond2_i32: 3019 case INDEX_op_setcond2_i32: 3020 case INDEX_op_cmp_vec: 3021 case INDEX_op_cmpsel_vec: 3022 if (op->args[k] < ARRAY_SIZE(cond_name) 3023 && cond_name[op->args[k]]) { 3024 col += ne_fprintf(f, ",%s", cond_name[op->args[k++]]); 3025 } else { 3026 col += ne_fprintf(f, ",$0x%" TCG_PRIlx, op->args[k++]); 3027 } 3028 i = 1; 3029 break; 3030 case INDEX_op_qemu_ld: 3031 case INDEX_op_qemu_st: 3032 case INDEX_op_qemu_ld2: 3033 case INDEX_op_qemu_st2: 3034 { 3035 const char *s_al, *s_op, *s_at; 3036 MemOpIdx oi = op->args[k++]; 3037 MemOp mop = get_memop(oi); 3038 unsigned ix = get_mmuidx(oi); 3039 3040 s_al = alignment_name[(mop & MO_AMASK) >> MO_ASHIFT]; 3041 s_op = ldst_name[mop & (MO_BSWAP | MO_SSIZE)]; 3042 s_at = atom_name[(mop & MO_ATOM_MASK) >> MO_ATOM_SHIFT]; 3043 mop &= ~(MO_AMASK | MO_BSWAP | MO_SSIZE | MO_ATOM_MASK); 3044 3045 /* If all fields are accounted for, print symbolically. */ 3046 if (!mop && s_al && s_op && s_at) { 3047 col += ne_fprintf(f, ",%s%s%s,%u", 3048 s_at, s_al, s_op, ix); 3049 } else { 3050 mop = get_memop(oi); 3051 col += ne_fprintf(f, ",$0x%x,%u", mop, ix); 3052 } 3053 i = 1; 3054 } 3055 break; 3056 case INDEX_op_bswap16: 3057 case INDEX_op_bswap32: 3058 case INDEX_op_bswap64: 3059 { 3060 TCGArg flags = op->args[k]; 3061 const char *name = NULL; 3062 3063 if (flags < ARRAY_SIZE(bswap_flag_name)) { 3064 name = bswap_flag_name[flags]; 3065 } 3066 if (name) { 3067 col += ne_fprintf(f, ",%s", name); 3068 } else { 3069 col += ne_fprintf(f, ",$0x%" TCG_PRIlx, flags); 3070 } 3071 i = k = 1; 3072 } 3073 break; 3074 #ifdef CONFIG_PLUGIN 3075 case INDEX_op_plugin_cb: 3076 { 3077 TCGArg from = op->args[k++]; 3078 const char *name = NULL; 3079 3080 if (from < ARRAY_SIZE(plugin_from_name)) { 3081 name = plugin_from_name[from]; 3082 } 3083 if (name) { 3084 col += ne_fprintf(f, "%s", name); 3085 } else { 3086 col += ne_fprintf(f, "$0x%" TCG_PRIlx, from); 3087 } 3088 i = 1; 3089 } 3090 break; 3091 #endif 3092 default: 3093 i = 0; 3094 break; 3095 } 3096 switch (c) { 3097 case INDEX_op_set_label: 3098 case INDEX_op_br: 3099 case INDEX_op_brcond: 3100 case INDEX_op_brcond2_i32: 3101 col += ne_fprintf(f, "%s$L%d", k ? "," : "", 3102 arg_label(op->args[k])->id); 3103 i++, k++; 3104 break; 3105 case INDEX_op_mb: 3106 { 3107 TCGBar membar = op->args[k]; 3108 const char *b_op, *m_op; 3109 3110 switch (membar & TCG_BAR_SC) { 3111 case 0: 3112 b_op = "none"; 3113 break; 3114 case TCG_BAR_LDAQ: 3115 b_op = "acq"; 3116 break; 3117 case TCG_BAR_STRL: 3118 b_op = "rel"; 3119 break; 3120 case TCG_BAR_SC: 3121 b_op = "seq"; 3122 break; 3123 default: 3124 g_assert_not_reached(); 3125 } 3126 3127 switch (membar & TCG_MO_ALL) { 3128 case 0: 3129 m_op = "none"; 3130 break; 3131 case TCG_MO_LD_LD: 3132 m_op = "rr"; 3133 break; 3134 case TCG_MO_LD_ST: 3135 m_op = "rw"; 3136 break; 3137 case TCG_MO_ST_LD: 3138 m_op = "wr"; 3139 break; 3140 case TCG_MO_ST_ST: 3141 m_op = "ww"; 3142 break; 3143 case TCG_MO_LD_LD | TCG_MO_LD_ST: 3144 m_op = "rr+rw"; 3145 break; 3146 case TCG_MO_LD_LD | TCG_MO_ST_LD: 3147 m_op = "rr+wr"; 3148 break; 3149 case TCG_MO_LD_LD | TCG_MO_ST_ST: 3150 m_op = "rr+ww"; 3151 break; 3152 case TCG_MO_LD_ST | TCG_MO_ST_LD: 3153 m_op = "rw+wr"; 3154 break; 3155 case TCG_MO_LD_ST | TCG_MO_ST_ST: 3156 m_op = "rw+ww"; 3157 break; 3158 case TCG_MO_ST_LD | TCG_MO_ST_ST: 3159 m_op = "wr+ww"; 3160 break; 3161 case TCG_MO_LD_LD | TCG_MO_LD_ST | TCG_MO_ST_LD: 3162 m_op = "rr+rw+wr"; 3163 break; 3164 case TCG_MO_LD_LD | TCG_MO_LD_ST | TCG_MO_ST_ST: 3165 m_op = "rr+rw+ww"; 3166 break; 3167 case TCG_MO_LD_LD | TCG_MO_ST_LD | TCG_MO_ST_ST: 3168 m_op = "rr+wr+ww"; 3169 break; 3170 case TCG_MO_LD_ST | TCG_MO_ST_LD | TCG_MO_ST_ST: 3171 m_op = "rw+wr+ww"; 3172 break; 3173 case TCG_MO_ALL: 3174 m_op = "all"; 3175 break; 3176 default: 3177 g_assert_not_reached(); 3178 } 3179 3180 col += ne_fprintf(f, "%s%s:%s", (k ? "," : ""), b_op, m_op); 3181 i++, k++; 3182 } 3183 break; 3184 default: 3185 break; 3186 } 3187 for (; i < nb_cargs; i++, k++) { 3188 col += ne_fprintf(f, "%s$0x%" TCG_PRIlx, k ? "," : "", 3189 op->args[k]); 3190 } 3191 } 3192 3193 if (have_prefs || op->life) { 3194 for (; col < 40; ++col) { 3195 putc(' ', f); 3196 } 3197 } 3198 3199 if (op->life) { 3200 unsigned life = op->life; 3201 3202 if (life & (SYNC_ARG * 3)) { 3203 ne_fprintf(f, " sync:"); 3204 for (i = 0; i < 2; ++i) { 3205 if (life & (SYNC_ARG << i)) { 3206 ne_fprintf(f, " %d", i); 3207 } 3208 } 3209 } 3210 life /= DEAD_ARG; 3211 if (life) { 3212 ne_fprintf(f, " dead:"); 3213 for (i = 0; life; ++i, life >>= 1) { 3214 if (life & 1) { 3215 ne_fprintf(f, " %d", i); 3216 } 3217 } 3218 } 3219 } 3220 3221 if (have_prefs) { 3222 for (i = 0; i < nb_oargs; ++i) { 3223 TCGRegSet set = output_pref(op, i); 3224 3225 if (i == 0) { 3226 ne_fprintf(f, " pref="); 3227 } else { 3228 ne_fprintf(f, ","); 3229 } 3230 if (set == 0) { 3231 ne_fprintf(f, "none"); 3232 } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) { 3233 ne_fprintf(f, "all"); 3234 #ifdef CONFIG_DEBUG_TCG 3235 } else if (tcg_regset_single(set)) { 3236 TCGReg reg = tcg_regset_first(set); 3237 ne_fprintf(f, "%s", tcg_target_reg_names[reg]); 3238 #endif 3239 } else if (TCG_TARGET_NB_REGS <= 32) { 3240 ne_fprintf(f, "0x%x", (uint32_t)set); 3241 } else { 3242 ne_fprintf(f, "0x%" PRIx64, (uint64_t)set); 3243 } 3244 } 3245 } 3246 3247 putc('\n', f); 3248 } 3249 } 3250 3251 /* we give more priority to constraints with less registers */ 3252 static int get_constraint_priority(const TCGArgConstraint *arg_ct, int k) 3253 { 3254 int n; 3255 3256 arg_ct += k; 3257 n = ctpop64(arg_ct->regs); 3258 3259 /* 3260 * Sort constraints of a single register first, which includes output 3261 * aliases (which must exactly match the input already allocated). 3262 */ 3263 if (n == 1 || arg_ct->oalias) { 3264 return INT_MAX; 3265 } 3266 3267 /* 3268 * Sort register pairs next, first then second immediately after. 3269 * Arbitrarily sort multiple pairs by the index of the first reg; 3270 * there shouldn't be many pairs. 3271 */ 3272 switch (arg_ct->pair) { 3273 case 1: 3274 case 3: 3275 return (k + 1) * 2; 3276 case 2: 3277 return (arg_ct->pair_index + 1) * 2 - 1; 3278 } 3279 3280 /* Finally, sort by decreasing register count. */ 3281 assert(n > 1); 3282 return -n; 3283 } 3284 3285 /* sort from highest priority to lowest */ 3286 static void sort_constraints(TCGArgConstraint *a, int start, int n) 3287 { 3288 int i, j; 3289 3290 for (i = 0; i < n; i++) { 3291 a[start + i].sort_index = start + i; 3292 } 3293 if (n <= 1) { 3294 return; 3295 } 3296 for (i = 0; i < n - 1; i++) { 3297 for (j = i + 1; j < n; j++) { 3298 int p1 = get_constraint_priority(a, a[start + i].sort_index); 3299 int p2 = get_constraint_priority(a, a[start + j].sort_index); 3300 if (p1 < p2) { 3301 int tmp = a[start + i].sort_index; 3302 a[start + i].sort_index = a[start + j].sort_index; 3303 a[start + j].sort_index = tmp; 3304 } 3305 } 3306 } 3307 } 3308 3309 static const TCGArgConstraint empty_cts[TCG_MAX_OP_ARGS]; 3310 static TCGArgConstraint all_cts[ARRAY_SIZE(constraint_sets)][TCG_MAX_OP_ARGS]; 3311 3312 static void process_constraint_sets(void) 3313 { 3314 for (size_t c = 0; c < ARRAY_SIZE(constraint_sets); ++c) { 3315 const TCGConstraintSet *tdefs = &constraint_sets[c]; 3316 TCGArgConstraint *args_ct = all_cts[c]; 3317 int nb_oargs = tdefs->nb_oargs; 3318 int nb_iargs = tdefs->nb_iargs; 3319 int nb_args = nb_oargs + nb_iargs; 3320 bool saw_alias_pair = false; 3321 3322 for (int i = 0; i < nb_args; i++) { 3323 const char *ct_str = tdefs->args_ct_str[i]; 3324 bool input_p = i >= nb_oargs; 3325 int o; 3326 3327 switch (*ct_str) { 3328 case '0' ... '9': 3329 o = *ct_str - '0'; 3330 tcg_debug_assert(input_p); 3331 tcg_debug_assert(o < nb_oargs); 3332 tcg_debug_assert(args_ct[o].regs != 0); 3333 tcg_debug_assert(!args_ct[o].oalias); 3334 args_ct[i] = args_ct[o]; 3335 /* The output sets oalias. */ 3336 args_ct[o].oalias = 1; 3337 args_ct[o].alias_index = i; 3338 /* The input sets ialias. */ 3339 args_ct[i].ialias = 1; 3340 args_ct[i].alias_index = o; 3341 if (args_ct[i].pair) { 3342 saw_alias_pair = true; 3343 } 3344 tcg_debug_assert(ct_str[1] == '\0'); 3345 continue; 3346 3347 case '&': 3348 tcg_debug_assert(!input_p); 3349 args_ct[i].newreg = true; 3350 ct_str++; 3351 break; 3352 3353 case 'p': /* plus */ 3354 /* Allocate to the register after the previous. */ 3355 tcg_debug_assert(i > (input_p ? nb_oargs : 0)); 3356 o = i - 1; 3357 tcg_debug_assert(!args_ct[o].pair); 3358 tcg_debug_assert(!args_ct[o].ct); 3359 args_ct[i] = (TCGArgConstraint){ 3360 .pair = 2, 3361 .pair_index = o, 3362 .regs = args_ct[o].regs << 1, 3363 .newreg = args_ct[o].newreg, 3364 }; 3365 args_ct[o].pair = 1; 3366 args_ct[o].pair_index = i; 3367 tcg_debug_assert(ct_str[1] == '\0'); 3368 continue; 3369 3370 case 'm': /* minus */ 3371 /* Allocate to the register before the previous. */ 3372 tcg_debug_assert(i > (input_p ? nb_oargs : 0)); 3373 o = i - 1; 3374 tcg_debug_assert(!args_ct[o].pair); 3375 tcg_debug_assert(!args_ct[o].ct); 3376 args_ct[i] = (TCGArgConstraint){ 3377 .pair = 1, 3378 .pair_index = o, 3379 .regs = args_ct[o].regs >> 1, 3380 .newreg = args_ct[o].newreg, 3381 }; 3382 args_ct[o].pair = 2; 3383 args_ct[o].pair_index = i; 3384 tcg_debug_assert(ct_str[1] == '\0'); 3385 continue; 3386 } 3387 3388 do { 3389 switch (*ct_str) { 3390 case 'i': 3391 args_ct[i].ct |= TCG_CT_CONST; 3392 break; 3393 #ifdef TCG_REG_ZERO 3394 case 'z': 3395 args_ct[i].ct |= TCG_CT_REG_ZERO; 3396 break; 3397 #endif 3398 3399 /* Include all of the target-specific constraints. */ 3400 3401 #undef CONST 3402 #define CONST(CASE, MASK) \ 3403 case CASE: args_ct[i].ct |= MASK; break; 3404 #define REGS(CASE, MASK) \ 3405 case CASE: args_ct[i].regs |= MASK; break; 3406 3407 #include "tcg-target-con-str.h" 3408 3409 #undef REGS 3410 #undef CONST 3411 default: 3412 case '0' ... '9': 3413 case '&': 3414 case 'p': 3415 case 'm': 3416 /* Typo in TCGConstraintSet constraint. */ 3417 g_assert_not_reached(); 3418 } 3419 } while (*++ct_str != '\0'); 3420 } 3421 3422 /* 3423 * Fix up output pairs that are aliased with inputs. 3424 * When we created the alias, we copied pair from the output. 3425 * There are three cases: 3426 * (1a) Pairs of inputs alias pairs of outputs. 3427 * (1b) One input aliases the first of a pair of outputs. 3428 * (2) One input aliases the second of a pair of outputs. 3429 * 3430 * Case 1a is handled by making sure that the pair_index'es are 3431 * properly updated so that they appear the same as a pair of inputs. 3432 * 3433 * Case 1b is handled by setting the pair_index of the input to 3434 * itself, simply so it doesn't point to an unrelated argument. 3435 * Since we don't encounter the "second" during the input allocation 3436 * phase, nothing happens with the second half of the input pair. 3437 * 3438 * Case 2 is handled by setting the second input to pair=3, the 3439 * first output to pair=3, and the pair_index'es to match. 3440 */ 3441 if (saw_alias_pair) { 3442 for (int i = nb_oargs; i < nb_args; i++) { 3443 int o, o2, i2; 3444 3445 /* 3446 * Since [0-9pm] must be alone in the constraint string, 3447 * the only way they can both be set is if the pair comes 3448 * from the output alias. 3449 */ 3450 if (!args_ct[i].ialias) { 3451 continue; 3452 } 3453 switch (args_ct[i].pair) { 3454 case 0: 3455 break; 3456 case 1: 3457 o = args_ct[i].alias_index; 3458 o2 = args_ct[o].pair_index; 3459 tcg_debug_assert(args_ct[o].pair == 1); 3460 tcg_debug_assert(args_ct[o2].pair == 2); 3461 if (args_ct[o2].oalias) { 3462 /* Case 1a */ 3463 i2 = args_ct[o2].alias_index; 3464 tcg_debug_assert(args_ct[i2].pair == 2); 3465 args_ct[i2].pair_index = i; 3466 args_ct[i].pair_index = i2; 3467 } else { 3468 /* Case 1b */ 3469 args_ct[i].pair_index = i; 3470 } 3471 break; 3472 case 2: 3473 o = args_ct[i].alias_index; 3474 o2 = args_ct[o].pair_index; 3475 tcg_debug_assert(args_ct[o].pair == 2); 3476 tcg_debug_assert(args_ct[o2].pair == 1); 3477 if (args_ct[o2].oalias) { 3478 /* Case 1a */ 3479 i2 = args_ct[o2].alias_index; 3480 tcg_debug_assert(args_ct[i2].pair == 1); 3481 args_ct[i2].pair_index = i; 3482 args_ct[i].pair_index = i2; 3483 } else { 3484 /* Case 2 */ 3485 args_ct[i].pair = 3; 3486 args_ct[o2].pair = 3; 3487 args_ct[i].pair_index = o2; 3488 args_ct[o2].pair_index = i; 3489 } 3490 break; 3491 default: 3492 g_assert_not_reached(); 3493 } 3494 } 3495 } 3496 3497 /* sort the constraints (XXX: this is just an heuristic) */ 3498 sort_constraints(args_ct, 0, nb_oargs); 3499 sort_constraints(args_ct, nb_oargs, nb_iargs); 3500 } 3501 } 3502 3503 static const TCGArgConstraint *opcode_args_ct(const TCGOp *op) 3504 { 3505 TCGOpcode opc = op->opc; 3506 TCGType type = TCGOP_TYPE(op); 3507 unsigned flags = TCGOP_FLAGS(op); 3508 const TCGOpDef *def = &tcg_op_defs[opc]; 3509 const TCGOutOp *outop = all_outop[opc]; 3510 TCGConstraintSetIndex con_set; 3511 3512 if (def->flags & TCG_OPF_NOT_PRESENT) { 3513 return empty_cts; 3514 } 3515 3516 if (outop) { 3517 con_set = outop->static_constraint; 3518 if (con_set == C_Dynamic) { 3519 con_set = outop->dynamic_constraint(type, flags); 3520 } 3521 } else { 3522 con_set = tcg_target_op_def(opc, type, flags); 3523 } 3524 tcg_debug_assert(con_set >= 0); 3525 tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets)); 3526 3527 /* The constraint arguments must match TCGOpcode arguments. */ 3528 tcg_debug_assert(constraint_sets[con_set].nb_oargs == def->nb_oargs); 3529 tcg_debug_assert(constraint_sets[con_set].nb_iargs == def->nb_iargs); 3530 3531 return all_cts[con_set]; 3532 } 3533 3534 static void remove_label_use(TCGOp *op, int idx) 3535 { 3536 TCGLabel *label = arg_label(op->args[idx]); 3537 TCGLabelUse *use; 3538 3539 QSIMPLEQ_FOREACH(use, &label->branches, next) { 3540 if (use->op == op) { 3541 QSIMPLEQ_REMOVE(&label->branches, use, TCGLabelUse, next); 3542 return; 3543 } 3544 } 3545 g_assert_not_reached(); 3546 } 3547 3548 void tcg_op_remove(TCGContext *s, TCGOp *op) 3549 { 3550 switch (op->opc) { 3551 case INDEX_op_br: 3552 remove_label_use(op, 0); 3553 break; 3554 case INDEX_op_brcond: 3555 remove_label_use(op, 3); 3556 break; 3557 case INDEX_op_brcond2_i32: 3558 remove_label_use(op, 5); 3559 break; 3560 default: 3561 break; 3562 } 3563 3564 QTAILQ_REMOVE(&s->ops, op, link); 3565 QTAILQ_INSERT_TAIL(&s->free_ops, op, link); 3566 s->nb_ops--; 3567 } 3568 3569 void tcg_remove_ops_after(TCGOp *op) 3570 { 3571 TCGContext *s = tcg_ctx; 3572 3573 while (true) { 3574 TCGOp *last = tcg_last_op(); 3575 if (last == op) { 3576 return; 3577 } 3578 tcg_op_remove(s, last); 3579 } 3580 } 3581 3582 static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs) 3583 { 3584 TCGContext *s = tcg_ctx; 3585 TCGOp *op = NULL; 3586 3587 if (unlikely(!QTAILQ_EMPTY(&s->free_ops))) { 3588 QTAILQ_FOREACH(op, &s->free_ops, link) { 3589 if (nargs <= op->nargs) { 3590 QTAILQ_REMOVE(&s->free_ops, op, link); 3591 nargs = op->nargs; 3592 goto found; 3593 } 3594 } 3595 } 3596 3597 /* Most opcodes have 3 or 4 operands: reduce fragmentation. */ 3598 nargs = MAX(4, nargs); 3599 op = tcg_malloc(sizeof(TCGOp) + sizeof(TCGArg) * nargs); 3600 3601 found: 3602 memset(op, 0, offsetof(TCGOp, link)); 3603 op->opc = opc; 3604 op->nargs = nargs; 3605 3606 /* Check for bitfield overflow. */ 3607 tcg_debug_assert(op->nargs == nargs); 3608 3609 s->nb_ops++; 3610 return op; 3611 } 3612 3613 TCGOp *tcg_emit_op(TCGOpcode opc, unsigned nargs) 3614 { 3615 TCGOp *op = tcg_op_alloc(opc, nargs); 3616 3617 if (tcg_ctx->emit_before_op) { 3618 QTAILQ_INSERT_BEFORE(tcg_ctx->emit_before_op, op, link); 3619 } else { 3620 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link); 3621 } 3622 return op; 3623 } 3624 3625 TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op, 3626 TCGOpcode opc, TCGType type, unsigned nargs) 3627 { 3628 TCGOp *new_op = tcg_op_alloc(opc, nargs); 3629 3630 TCGOP_TYPE(new_op) = type; 3631 QTAILQ_INSERT_BEFORE(old_op, new_op, link); 3632 return new_op; 3633 } 3634 3635 TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op, 3636 TCGOpcode opc, TCGType type, unsigned nargs) 3637 { 3638 TCGOp *new_op = tcg_op_alloc(opc, nargs); 3639 3640 TCGOP_TYPE(new_op) = type; 3641 QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link); 3642 return new_op; 3643 } 3644 3645 static void move_label_uses(TCGLabel *to, TCGLabel *from) 3646 { 3647 TCGLabelUse *u; 3648 3649 QSIMPLEQ_FOREACH(u, &from->branches, next) { 3650 TCGOp *op = u->op; 3651 switch (op->opc) { 3652 case INDEX_op_br: 3653 op->args[0] = label_arg(to); 3654 break; 3655 case INDEX_op_brcond: 3656 op->args[3] = label_arg(to); 3657 break; 3658 case INDEX_op_brcond2_i32: 3659 op->args[5] = label_arg(to); 3660 break; 3661 default: 3662 g_assert_not_reached(); 3663 } 3664 } 3665 3666 QSIMPLEQ_CONCAT(&to->branches, &from->branches); 3667 } 3668 3669 /* Reachable analysis : remove unreachable code. */ 3670 static void __attribute__((noinline)) 3671 reachable_code_pass(TCGContext *s) 3672 { 3673 TCGOp *op, *op_next, *op_prev; 3674 bool dead = false; 3675 3676 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { 3677 bool remove = dead; 3678 TCGLabel *label; 3679 3680 switch (op->opc) { 3681 case INDEX_op_set_label: 3682 label = arg_label(op->args[0]); 3683 3684 /* 3685 * Note that the first op in the TB is always a load, 3686 * so there is always something before a label. 3687 */ 3688 op_prev = QTAILQ_PREV(op, link); 3689 3690 /* 3691 * If we find two sequential labels, move all branches to 3692 * reference the second label and remove the first label. 3693 * Do this before branch to next optimization, so that the 3694 * middle label is out of the way. 3695 */ 3696 if (op_prev->opc == INDEX_op_set_label) { 3697 move_label_uses(label, arg_label(op_prev->args[0])); 3698 tcg_op_remove(s, op_prev); 3699 op_prev = QTAILQ_PREV(op, link); 3700 } 3701 3702 /* 3703 * Optimization can fold conditional branches to unconditional. 3704 * If we find a label which is preceded by an unconditional 3705 * branch to next, remove the branch. We couldn't do this when 3706 * processing the branch because any dead code between the branch 3707 * and label had not yet been removed. 3708 */ 3709 if (op_prev->opc == INDEX_op_br && 3710 label == arg_label(op_prev->args[0])) { 3711 tcg_op_remove(s, op_prev); 3712 /* Fall through means insns become live again. */ 3713 dead = false; 3714 } 3715 3716 if (QSIMPLEQ_EMPTY(&label->branches)) { 3717 /* 3718 * While there is an occasional backward branch, virtually 3719 * all branches generated by the translators are forward. 3720 * Which means that generally we will have already removed 3721 * all references to the label that will be, and there is 3722 * little to be gained by iterating. 3723 */ 3724 remove = true; 3725 } else { 3726 /* Once we see a label, insns become live again. */ 3727 dead = false; 3728 remove = false; 3729 } 3730 break; 3731 3732 case INDEX_op_br: 3733 case INDEX_op_exit_tb: 3734 case INDEX_op_goto_ptr: 3735 /* Unconditional branches; everything following is dead. */ 3736 dead = true; 3737 break; 3738 3739 case INDEX_op_call: 3740 /* Notice noreturn helper calls, raising exceptions. */ 3741 if (tcg_call_flags(op) & TCG_CALL_NO_RETURN) { 3742 dead = true; 3743 } 3744 break; 3745 3746 case INDEX_op_insn_start: 3747 /* Never remove -- we need to keep these for unwind. */ 3748 remove = false; 3749 break; 3750 3751 default: 3752 break; 3753 } 3754 3755 if (remove) { 3756 tcg_op_remove(s, op); 3757 } 3758 } 3759 } 3760 3761 #define TS_DEAD 1 3762 #define TS_MEM 2 3763 3764 #define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n))) 3765 #define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n))) 3766 3767 /* For liveness_pass_1, the register preferences for a given temp. */ 3768 static inline TCGRegSet *la_temp_pref(TCGTemp *ts) 3769 { 3770 return ts->state_ptr; 3771 } 3772 3773 /* For liveness_pass_1, reset the preferences for a given temp to the 3774 * maximal regset for its type. 3775 */ 3776 static inline void la_reset_pref(TCGTemp *ts) 3777 { 3778 *la_temp_pref(ts) 3779 = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]); 3780 } 3781 3782 /* liveness analysis: end of function: all temps are dead, and globals 3783 should be in memory. */ 3784 static void la_func_end(TCGContext *s, int ng, int nt) 3785 { 3786 int i; 3787 3788 for (i = 0; i < ng; ++i) { 3789 s->temps[i].state = TS_DEAD | TS_MEM; 3790 la_reset_pref(&s->temps[i]); 3791 } 3792 for (i = ng; i < nt; ++i) { 3793 s->temps[i].state = TS_DEAD; 3794 la_reset_pref(&s->temps[i]); 3795 } 3796 } 3797 3798 /* liveness analysis: end of basic block: all temps are dead, globals 3799 and local temps should be in memory. */ 3800 static void la_bb_end(TCGContext *s, int ng, int nt) 3801 { 3802 int i; 3803 3804 for (i = 0; i < nt; ++i) { 3805 TCGTemp *ts = &s->temps[i]; 3806 int state; 3807 3808 switch (ts->kind) { 3809 case TEMP_FIXED: 3810 case TEMP_GLOBAL: 3811 case TEMP_TB: 3812 state = TS_DEAD | TS_MEM; 3813 break; 3814 case TEMP_EBB: 3815 case TEMP_CONST: 3816 state = TS_DEAD; 3817 break; 3818 default: 3819 g_assert_not_reached(); 3820 } 3821 ts->state = state; 3822 la_reset_pref(ts); 3823 } 3824 } 3825 3826 /* liveness analysis: sync globals back to memory. */ 3827 static void la_global_sync(TCGContext *s, int ng) 3828 { 3829 int i; 3830 3831 for (i = 0; i < ng; ++i) { 3832 int state = s->temps[i].state; 3833 s->temps[i].state = state | TS_MEM; 3834 if (state == TS_DEAD) { 3835 /* If the global was previously dead, reset prefs. */ 3836 la_reset_pref(&s->temps[i]); 3837 } 3838 } 3839 } 3840 3841 /* 3842 * liveness analysis: conditional branch: all temps are dead unless 3843 * explicitly live-across-conditional-branch, globals and local temps 3844 * should be synced. 3845 */ 3846 static void la_bb_sync(TCGContext *s, int ng, int nt) 3847 { 3848 la_global_sync(s, ng); 3849 3850 for (int i = ng; i < nt; ++i) { 3851 TCGTemp *ts = &s->temps[i]; 3852 int state; 3853 3854 switch (ts->kind) { 3855 case TEMP_TB: 3856 state = ts->state; 3857 ts->state = state | TS_MEM; 3858 if (state != TS_DEAD) { 3859 continue; 3860 } 3861 break; 3862 case TEMP_EBB: 3863 case TEMP_CONST: 3864 continue; 3865 default: 3866 g_assert_not_reached(); 3867 } 3868 la_reset_pref(&s->temps[i]); 3869 } 3870 } 3871 3872 /* liveness analysis: sync globals back to memory and kill. */ 3873 static void la_global_kill(TCGContext *s, int ng) 3874 { 3875 int i; 3876 3877 for (i = 0; i < ng; i++) { 3878 s->temps[i].state = TS_DEAD | TS_MEM; 3879 la_reset_pref(&s->temps[i]); 3880 } 3881 } 3882 3883 /* liveness analysis: note live globals crossing calls. */ 3884 static void la_cross_call(TCGContext *s, int nt) 3885 { 3886 TCGRegSet mask = ~tcg_target_call_clobber_regs; 3887 int i; 3888 3889 for (i = 0; i < nt; i++) { 3890 TCGTemp *ts = &s->temps[i]; 3891 if (!(ts->state & TS_DEAD)) { 3892 TCGRegSet *pset = la_temp_pref(ts); 3893 TCGRegSet set = *pset; 3894 3895 set &= mask; 3896 /* If the combination is not possible, restart. */ 3897 if (set == 0) { 3898 set = tcg_target_available_regs[ts->type] & mask; 3899 } 3900 *pset = set; 3901 } 3902 } 3903 } 3904 3905 /* 3906 * Liveness analysis: Verify the lifetime of TEMP_TB, and reduce 3907 * to TEMP_EBB, if possible. 3908 */ 3909 static void __attribute__((noinline)) 3910 liveness_pass_0(TCGContext *s) 3911 { 3912 void * const multiple_ebb = (void *)(uintptr_t)-1; 3913 int nb_temps = s->nb_temps; 3914 TCGOp *op, *ebb; 3915 3916 for (int i = s->nb_globals; i < nb_temps; ++i) { 3917 s->temps[i].state_ptr = NULL; 3918 } 3919 3920 /* 3921 * Represent each EBB by the op at which it begins. In the case of 3922 * the first EBB, this is the first op, otherwise it is a label. 3923 * Collect the uses of each TEMP_TB: NULL for unused, EBB for use 3924 * within a single EBB, else MULTIPLE_EBB. 3925 */ 3926 ebb = QTAILQ_FIRST(&s->ops); 3927 QTAILQ_FOREACH(op, &s->ops, link) { 3928 const TCGOpDef *def; 3929 int nb_oargs, nb_iargs; 3930 3931 switch (op->opc) { 3932 case INDEX_op_set_label: 3933 ebb = op; 3934 continue; 3935 case INDEX_op_discard: 3936 continue; 3937 case INDEX_op_call: 3938 nb_oargs = TCGOP_CALLO(op); 3939 nb_iargs = TCGOP_CALLI(op); 3940 break; 3941 default: 3942 def = &tcg_op_defs[op->opc]; 3943 nb_oargs = def->nb_oargs; 3944 nb_iargs = def->nb_iargs; 3945 break; 3946 } 3947 3948 for (int i = 0; i < nb_oargs + nb_iargs; ++i) { 3949 TCGTemp *ts = arg_temp(op->args[i]); 3950 3951 if (ts->kind != TEMP_TB) { 3952 continue; 3953 } 3954 if (ts->state_ptr == NULL) { 3955 ts->state_ptr = ebb; 3956 } else if (ts->state_ptr != ebb) { 3957 ts->state_ptr = multiple_ebb; 3958 } 3959 } 3960 } 3961 3962 /* 3963 * For TEMP_TB that turned out not to be used beyond one EBB, 3964 * reduce the liveness to TEMP_EBB. 3965 */ 3966 for (int i = s->nb_globals; i < nb_temps; ++i) { 3967 TCGTemp *ts = &s->temps[i]; 3968 if (ts->kind == TEMP_TB && ts->state_ptr != multiple_ebb) { 3969 ts->kind = TEMP_EBB; 3970 } 3971 } 3972 } 3973 3974 static void assert_carry_dead(TCGContext *s) 3975 { 3976 /* 3977 * Carry operations can be separated by a few insns like mov, 3978 * load or store, but they should always be "close", and 3979 * carry-out operations should always be paired with carry-in. 3980 * At various boundaries, carry must have been consumed. 3981 */ 3982 tcg_debug_assert(!s->carry_live); 3983 } 3984 3985 /* Liveness analysis : update the opc_arg_life array to tell if a 3986 given input arguments is dead. Instructions updating dead 3987 temporaries are removed. */ 3988 static void __attribute__((noinline)) 3989 liveness_pass_1(TCGContext *s) 3990 { 3991 int nb_globals = s->nb_globals; 3992 int nb_temps = s->nb_temps; 3993 TCGOp *op, *op_prev; 3994 TCGRegSet *prefs; 3995 3996 prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps); 3997 for (int i = 0; i < nb_temps; ++i) { 3998 s->temps[i].state_ptr = prefs + i; 3999 } 4000 4001 /* ??? Should be redundant with the exit_tb that ends the TB. */ 4002 la_func_end(s, nb_globals, nb_temps); 4003 4004 s->carry_live = false; 4005 QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) { 4006 int nb_iargs, nb_oargs; 4007 TCGOpcode opc_new, opc_new2; 4008 TCGLifeData arg_life = 0; 4009 TCGTemp *ts; 4010 TCGOpcode opc = op->opc; 4011 const TCGOpDef *def; 4012 const TCGArgConstraint *args_ct; 4013 4014 switch (opc) { 4015 case INDEX_op_call: 4016 assert_carry_dead(s); 4017 { 4018 const TCGHelperInfo *info = tcg_call_info(op); 4019 int call_flags = tcg_call_flags(op); 4020 4021 nb_oargs = TCGOP_CALLO(op); 4022 nb_iargs = TCGOP_CALLI(op); 4023 4024 /* pure functions can be removed if their result is unused */ 4025 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) { 4026 for (int i = 0; i < nb_oargs; i++) { 4027 ts = arg_temp(op->args[i]); 4028 if (ts->state != TS_DEAD) { 4029 goto do_not_remove_call; 4030 } 4031 } 4032 goto do_remove; 4033 } 4034 do_not_remove_call: 4035 4036 /* Output args are dead. */ 4037 for (int i = 0; i < nb_oargs; i++) { 4038 ts = arg_temp(op->args[i]); 4039 if (ts->state & TS_DEAD) { 4040 arg_life |= DEAD_ARG << i; 4041 } 4042 if (ts->state & TS_MEM) { 4043 arg_life |= SYNC_ARG << i; 4044 } 4045 ts->state = TS_DEAD; 4046 la_reset_pref(ts); 4047 } 4048 4049 /* Not used -- it will be tcg_target_call_oarg_reg(). */ 4050 memset(op->output_pref, 0, sizeof(op->output_pref)); 4051 4052 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS | 4053 TCG_CALL_NO_READ_GLOBALS))) { 4054 la_global_kill(s, nb_globals); 4055 } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) { 4056 la_global_sync(s, nb_globals); 4057 } 4058 4059 /* Record arguments that die in this helper. */ 4060 for (int i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 4061 ts = arg_temp(op->args[i]); 4062 if (ts->state & TS_DEAD) { 4063 arg_life |= DEAD_ARG << i; 4064 } 4065 } 4066 4067 /* For all live registers, remove call-clobbered prefs. */ 4068 la_cross_call(s, nb_temps); 4069 4070 /* 4071 * Input arguments are live for preceding opcodes. 4072 * 4073 * For those arguments that die, and will be allocated in 4074 * registers, clear the register set for that arg, to be 4075 * filled in below. For args that will be on the stack, 4076 * reset to any available reg. Process arguments in reverse 4077 * order so that if a temp is used more than once, the stack 4078 * reset to max happens before the register reset to 0. 4079 */ 4080 for (int i = nb_iargs - 1; i >= 0; i--) { 4081 const TCGCallArgumentLoc *loc = &info->in[i]; 4082 ts = arg_temp(op->args[nb_oargs + i]); 4083 4084 if (ts->state & TS_DEAD) { 4085 switch (loc->kind) { 4086 case TCG_CALL_ARG_NORMAL: 4087 case TCG_CALL_ARG_EXTEND_U: 4088 case TCG_CALL_ARG_EXTEND_S: 4089 if (arg_slot_reg_p(loc->arg_slot)) { 4090 *la_temp_pref(ts) = 0; 4091 break; 4092 } 4093 /* fall through */ 4094 default: 4095 *la_temp_pref(ts) = 4096 tcg_target_available_regs[ts->type]; 4097 break; 4098 } 4099 ts->state &= ~TS_DEAD; 4100 } 4101 } 4102 4103 /* 4104 * For each input argument, add its input register to prefs. 4105 * If a temp is used once, this produces a single set bit; 4106 * if a temp is used multiple times, this produces a set. 4107 */ 4108 for (int i = 0; i < nb_iargs; i++) { 4109 const TCGCallArgumentLoc *loc = &info->in[i]; 4110 ts = arg_temp(op->args[nb_oargs + i]); 4111 4112 switch (loc->kind) { 4113 case TCG_CALL_ARG_NORMAL: 4114 case TCG_CALL_ARG_EXTEND_U: 4115 case TCG_CALL_ARG_EXTEND_S: 4116 if (arg_slot_reg_p(loc->arg_slot)) { 4117 tcg_regset_set_reg(*la_temp_pref(ts), 4118 tcg_target_call_iarg_regs[loc->arg_slot]); 4119 } 4120 break; 4121 default: 4122 break; 4123 } 4124 } 4125 } 4126 break; 4127 case INDEX_op_insn_start: 4128 assert_carry_dead(s); 4129 break; 4130 case INDEX_op_discard: 4131 /* mark the temporary as dead */ 4132 ts = arg_temp(op->args[0]); 4133 ts->state = TS_DEAD; 4134 la_reset_pref(ts); 4135 break; 4136 4137 case INDEX_op_muls2: 4138 opc_new = INDEX_op_mul; 4139 opc_new2 = INDEX_op_mulsh; 4140 goto do_mul2; 4141 case INDEX_op_mulu2: 4142 opc_new = INDEX_op_mul; 4143 opc_new2 = INDEX_op_muluh; 4144 do_mul2: 4145 assert_carry_dead(s); 4146 if (arg_temp(op->args[1])->state == TS_DEAD) { 4147 if (arg_temp(op->args[0])->state == TS_DEAD) { 4148 /* Both parts of the operation are dead. */ 4149 goto do_remove; 4150 } 4151 /* The high part of the operation is dead; generate the low. */ 4152 op->opc = opc = opc_new; 4153 op->args[1] = op->args[2]; 4154 op->args[2] = op->args[3]; 4155 } else if (arg_temp(op->args[0])->state == TS_DEAD && 4156 tcg_op_supported(opc_new2, TCGOP_TYPE(op), 0)) { 4157 /* The low part of the operation is dead; generate the high. */ 4158 op->opc = opc = opc_new2; 4159 op->args[0] = op->args[1]; 4160 op->args[1] = op->args[2]; 4161 op->args[2] = op->args[3]; 4162 } else { 4163 goto do_not_remove; 4164 } 4165 /* Mark the single-word operation live. */ 4166 goto do_not_remove; 4167 4168 case INDEX_op_addco: 4169 if (s->carry_live) { 4170 goto do_not_remove; 4171 } 4172 op->opc = opc = INDEX_op_add; 4173 goto do_default; 4174 4175 case INDEX_op_addcio: 4176 if (s->carry_live) { 4177 goto do_not_remove; 4178 } 4179 op->opc = opc = INDEX_op_addci; 4180 goto do_default; 4181 4182 case INDEX_op_subbo: 4183 if (s->carry_live) { 4184 goto do_not_remove; 4185 } 4186 /* Lower to sub, but this may also require canonicalization. */ 4187 op->opc = opc = INDEX_op_sub; 4188 ts = arg_temp(op->args[2]); 4189 if (ts->kind == TEMP_CONST) { 4190 ts = tcg_constant_internal(ts->type, -ts->val); 4191 if (ts->state_ptr == NULL) { 4192 tcg_debug_assert(temp_idx(ts) == nb_temps); 4193 nb_temps++; 4194 ts->state_ptr = tcg_malloc(sizeof(TCGRegSet)); 4195 ts->state = TS_DEAD; 4196 la_reset_pref(ts); 4197 } 4198 op->args[2] = temp_arg(ts); 4199 op->opc = opc = INDEX_op_add; 4200 } 4201 goto do_default; 4202 4203 case INDEX_op_subbio: 4204 if (s->carry_live) { 4205 goto do_not_remove; 4206 } 4207 op->opc = opc = INDEX_op_subbi; 4208 goto do_default; 4209 4210 case INDEX_op_addc1o: 4211 if (s->carry_live) { 4212 goto do_not_remove; 4213 } 4214 /* Lower to add, add +1. */ 4215 op_prev = tcg_op_insert_before(s, op, INDEX_op_add, 4216 TCGOP_TYPE(op), 3); 4217 op_prev->args[0] = op->args[0]; 4218 op_prev->args[1] = op->args[1]; 4219 op_prev->args[2] = op->args[2]; 4220 op->opc = opc = INDEX_op_add; 4221 op->args[1] = op->args[0]; 4222 ts = arg_temp(op->args[0]); 4223 ts = tcg_constant_internal(ts->type, 1); 4224 op->args[2] = temp_arg(ts); 4225 goto do_default; 4226 4227 case INDEX_op_subb1o: 4228 if (s->carry_live) { 4229 goto do_not_remove; 4230 } 4231 /* Lower to sub, add -1. */ 4232 op_prev = tcg_op_insert_before(s, op, INDEX_op_sub, 4233 TCGOP_TYPE(op), 3); 4234 op_prev->args[0] = op->args[0]; 4235 op_prev->args[1] = op->args[1]; 4236 op_prev->args[2] = op->args[2]; 4237 op->opc = opc = INDEX_op_add; 4238 op->args[1] = op->args[0]; 4239 ts = arg_temp(op->args[0]); 4240 ts = tcg_constant_internal(ts->type, -1); 4241 op->args[2] = temp_arg(ts); 4242 goto do_default; 4243 4244 default: 4245 do_default: 4246 /* 4247 * Test if the operation can be removed because all 4248 * its outputs are dead. We assume that nb_oargs == 0 4249 * implies side effects. 4250 */ 4251 def = &tcg_op_defs[opc]; 4252 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && def->nb_oargs != 0) { 4253 for (int i = def->nb_oargs - 1; i >= 0; i--) { 4254 if (arg_temp(op->args[i])->state != TS_DEAD) { 4255 goto do_not_remove; 4256 } 4257 } 4258 goto do_remove; 4259 } 4260 goto do_not_remove; 4261 4262 do_remove: 4263 tcg_op_remove(s, op); 4264 break; 4265 4266 do_not_remove: 4267 def = &tcg_op_defs[opc]; 4268 nb_iargs = def->nb_iargs; 4269 nb_oargs = def->nb_oargs; 4270 4271 for (int i = 0; i < nb_oargs; i++) { 4272 ts = arg_temp(op->args[i]); 4273 4274 /* Remember the preference of the uses that followed. */ 4275 if (i < ARRAY_SIZE(op->output_pref)) { 4276 op->output_pref[i] = *la_temp_pref(ts); 4277 } 4278 4279 /* Output args are dead. */ 4280 if (ts->state & TS_DEAD) { 4281 arg_life |= DEAD_ARG << i; 4282 } 4283 if (ts->state & TS_MEM) { 4284 arg_life |= SYNC_ARG << i; 4285 } 4286 ts->state = TS_DEAD; 4287 la_reset_pref(ts); 4288 } 4289 4290 /* If end of basic block, update. */ 4291 if (def->flags & TCG_OPF_BB_EXIT) { 4292 assert_carry_dead(s); 4293 la_func_end(s, nb_globals, nb_temps); 4294 } else if (def->flags & TCG_OPF_COND_BRANCH) { 4295 assert_carry_dead(s); 4296 la_bb_sync(s, nb_globals, nb_temps); 4297 } else if (def->flags & TCG_OPF_BB_END) { 4298 assert_carry_dead(s); 4299 la_bb_end(s, nb_globals, nb_temps); 4300 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { 4301 assert_carry_dead(s); 4302 la_global_sync(s, nb_globals); 4303 if (def->flags & TCG_OPF_CALL_CLOBBER) { 4304 la_cross_call(s, nb_temps); 4305 } 4306 } 4307 4308 /* Record arguments that die in this opcode. */ 4309 for (int i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 4310 ts = arg_temp(op->args[i]); 4311 if (ts->state & TS_DEAD) { 4312 arg_life |= DEAD_ARG << i; 4313 } 4314 } 4315 if (def->flags & TCG_OPF_CARRY_OUT) { 4316 s->carry_live = false; 4317 } 4318 4319 /* Input arguments are live for preceding opcodes. */ 4320 for (int i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 4321 ts = arg_temp(op->args[i]); 4322 if (ts->state & TS_DEAD) { 4323 /* For operands that were dead, initially allow 4324 all regs for the type. */ 4325 *la_temp_pref(ts) = tcg_target_available_regs[ts->type]; 4326 ts->state &= ~TS_DEAD; 4327 } 4328 } 4329 if (def->flags & TCG_OPF_CARRY_IN) { 4330 s->carry_live = true; 4331 } 4332 4333 /* Incorporate constraints for this operand. */ 4334 switch (opc) { 4335 case INDEX_op_mov: 4336 /* Note that these are TCG_OPF_NOT_PRESENT and do not 4337 have proper constraints. That said, special case 4338 moves to propagate preferences backward. */ 4339 if (IS_DEAD_ARG(1)) { 4340 *la_temp_pref(arg_temp(op->args[0])) 4341 = *la_temp_pref(arg_temp(op->args[1])); 4342 } 4343 break; 4344 4345 default: 4346 args_ct = opcode_args_ct(op); 4347 for (int i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 4348 const TCGArgConstraint *ct = &args_ct[i]; 4349 TCGRegSet set, *pset; 4350 4351 ts = arg_temp(op->args[i]); 4352 pset = la_temp_pref(ts); 4353 set = *pset; 4354 4355 set &= ct->regs; 4356 if (ct->ialias) { 4357 set &= output_pref(op, ct->alias_index); 4358 } 4359 /* If the combination is not possible, restart. */ 4360 if (set == 0) { 4361 set = ct->regs; 4362 } 4363 *pset = set; 4364 } 4365 break; 4366 } 4367 break; 4368 } 4369 op->life = arg_life; 4370 } 4371 assert_carry_dead(s); 4372 } 4373 4374 /* Liveness analysis: Convert indirect regs to direct temporaries. */ 4375 static bool __attribute__((noinline)) 4376 liveness_pass_2(TCGContext *s) 4377 { 4378 int nb_globals = s->nb_globals; 4379 int nb_temps, i; 4380 bool changes = false; 4381 TCGOp *op, *op_next; 4382 4383 /* Create a temporary for each indirect global. */ 4384 for (i = 0; i < nb_globals; ++i) { 4385 TCGTemp *its = &s->temps[i]; 4386 if (its->indirect_reg) { 4387 TCGTemp *dts = tcg_temp_alloc(s); 4388 dts->type = its->type; 4389 dts->base_type = its->base_type; 4390 dts->temp_subindex = its->temp_subindex; 4391 dts->kind = TEMP_EBB; 4392 its->state_ptr = dts; 4393 } else { 4394 its->state_ptr = NULL; 4395 } 4396 /* All globals begin dead. */ 4397 its->state = TS_DEAD; 4398 } 4399 for (nb_temps = s->nb_temps; i < nb_temps; ++i) { 4400 TCGTemp *its = &s->temps[i]; 4401 its->state_ptr = NULL; 4402 its->state = TS_DEAD; 4403 } 4404 4405 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { 4406 TCGOpcode opc = op->opc; 4407 const TCGOpDef *def = &tcg_op_defs[opc]; 4408 TCGLifeData arg_life = op->life; 4409 int nb_iargs, nb_oargs, call_flags; 4410 TCGTemp *arg_ts, *dir_ts; 4411 4412 if (opc == INDEX_op_call) { 4413 nb_oargs = TCGOP_CALLO(op); 4414 nb_iargs = TCGOP_CALLI(op); 4415 call_flags = tcg_call_flags(op); 4416 } else { 4417 nb_iargs = def->nb_iargs; 4418 nb_oargs = def->nb_oargs; 4419 4420 /* Set flags similar to how calls require. */ 4421 if (def->flags & TCG_OPF_COND_BRANCH) { 4422 /* Like reading globals: sync_globals */ 4423 call_flags = TCG_CALL_NO_WRITE_GLOBALS; 4424 } else if (def->flags & TCG_OPF_BB_END) { 4425 /* Like writing globals: save_globals */ 4426 call_flags = 0; 4427 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { 4428 /* Like reading globals: sync_globals */ 4429 call_flags = TCG_CALL_NO_WRITE_GLOBALS; 4430 } else { 4431 /* No effect on globals. */ 4432 call_flags = (TCG_CALL_NO_READ_GLOBALS | 4433 TCG_CALL_NO_WRITE_GLOBALS); 4434 } 4435 } 4436 4437 /* Make sure that input arguments are available. */ 4438 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 4439 arg_ts = arg_temp(op->args[i]); 4440 dir_ts = arg_ts->state_ptr; 4441 if (dir_ts && arg_ts->state == TS_DEAD) { 4442 TCGOp *lop = tcg_op_insert_before(s, op, INDEX_op_ld, 4443 arg_ts->type, 3); 4444 4445 lop->args[0] = temp_arg(dir_ts); 4446 lop->args[1] = temp_arg(arg_ts->mem_base); 4447 lop->args[2] = arg_ts->mem_offset; 4448 4449 /* Loaded, but synced with memory. */ 4450 arg_ts->state = TS_MEM; 4451 } 4452 } 4453 4454 /* Perform input replacement, and mark inputs that became dead. 4455 No action is required except keeping temp_state up to date 4456 so that we reload when needed. */ 4457 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 4458 arg_ts = arg_temp(op->args[i]); 4459 dir_ts = arg_ts->state_ptr; 4460 if (dir_ts) { 4461 op->args[i] = temp_arg(dir_ts); 4462 changes = true; 4463 if (IS_DEAD_ARG(i)) { 4464 arg_ts->state = TS_DEAD; 4465 } 4466 } 4467 } 4468 4469 /* Liveness analysis should ensure that the following are 4470 all correct, for call sites and basic block end points. */ 4471 if (call_flags & TCG_CALL_NO_READ_GLOBALS) { 4472 /* Nothing to do */ 4473 } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) { 4474 for (i = 0; i < nb_globals; ++i) { 4475 /* Liveness should see that globals are synced back, 4476 that is, either TS_DEAD or TS_MEM. */ 4477 arg_ts = &s->temps[i]; 4478 tcg_debug_assert(arg_ts->state_ptr == 0 4479 || arg_ts->state != 0); 4480 } 4481 } else { 4482 for (i = 0; i < nb_globals; ++i) { 4483 /* Liveness should see that globals are saved back, 4484 that is, TS_DEAD, waiting to be reloaded. */ 4485 arg_ts = &s->temps[i]; 4486 tcg_debug_assert(arg_ts->state_ptr == 0 4487 || arg_ts->state == TS_DEAD); 4488 } 4489 } 4490 4491 /* Outputs become available. */ 4492 if (opc == INDEX_op_mov) { 4493 arg_ts = arg_temp(op->args[0]); 4494 dir_ts = arg_ts->state_ptr; 4495 if (dir_ts) { 4496 op->args[0] = temp_arg(dir_ts); 4497 changes = true; 4498 4499 /* The output is now live and modified. */ 4500 arg_ts->state = 0; 4501 4502 if (NEED_SYNC_ARG(0)) { 4503 TCGOp *sop = tcg_op_insert_after(s, op, INDEX_op_st, 4504 arg_ts->type, 3); 4505 TCGTemp *out_ts = dir_ts; 4506 4507 if (IS_DEAD_ARG(0)) { 4508 out_ts = arg_temp(op->args[1]); 4509 arg_ts->state = TS_DEAD; 4510 tcg_op_remove(s, op); 4511 } else { 4512 arg_ts->state = TS_MEM; 4513 } 4514 4515 sop->args[0] = temp_arg(out_ts); 4516 sop->args[1] = temp_arg(arg_ts->mem_base); 4517 sop->args[2] = arg_ts->mem_offset; 4518 } else { 4519 tcg_debug_assert(!IS_DEAD_ARG(0)); 4520 } 4521 } 4522 } else { 4523 for (i = 0; i < nb_oargs; i++) { 4524 arg_ts = arg_temp(op->args[i]); 4525 dir_ts = arg_ts->state_ptr; 4526 if (!dir_ts) { 4527 continue; 4528 } 4529 op->args[i] = temp_arg(dir_ts); 4530 changes = true; 4531 4532 /* The output is now live and modified. */ 4533 arg_ts->state = 0; 4534 4535 /* Sync outputs upon their last write. */ 4536 if (NEED_SYNC_ARG(i)) { 4537 TCGOp *sop = tcg_op_insert_after(s, op, INDEX_op_st, 4538 arg_ts->type, 3); 4539 4540 sop->args[0] = temp_arg(dir_ts); 4541 sop->args[1] = temp_arg(arg_ts->mem_base); 4542 sop->args[2] = arg_ts->mem_offset; 4543 4544 arg_ts->state = TS_MEM; 4545 } 4546 /* Drop outputs that are dead. */ 4547 if (IS_DEAD_ARG(i)) { 4548 arg_ts->state = TS_DEAD; 4549 } 4550 } 4551 } 4552 } 4553 4554 return changes; 4555 } 4556 4557 static void temp_allocate_frame(TCGContext *s, TCGTemp *ts) 4558 { 4559 intptr_t off; 4560 int size, align; 4561 4562 /* When allocating an object, look at the full type. */ 4563 size = tcg_type_size(ts->base_type); 4564 switch (ts->base_type) { 4565 case TCG_TYPE_I32: 4566 align = 4; 4567 break; 4568 case TCG_TYPE_I64: 4569 case TCG_TYPE_V64: 4570 align = 8; 4571 break; 4572 case TCG_TYPE_I128: 4573 case TCG_TYPE_V128: 4574 case TCG_TYPE_V256: 4575 /* 4576 * Note that we do not require aligned storage for V256, 4577 * and that we provide alignment for I128 to match V128, 4578 * even if that's above what the host ABI requires. 4579 */ 4580 align = 16; 4581 break; 4582 default: 4583 g_assert_not_reached(); 4584 } 4585 4586 /* 4587 * Assume the stack is sufficiently aligned. 4588 * This affects e.g. ARM NEON, where we have 8 byte stack alignment 4589 * and do not require 16 byte vector alignment. This seems slightly 4590 * easier than fully parameterizing the above switch statement. 4591 */ 4592 align = MIN(TCG_TARGET_STACK_ALIGN, align); 4593 off = ROUND_UP(s->current_frame_offset, align); 4594 4595 /* If we've exhausted the stack frame, restart with a smaller TB. */ 4596 if (off + size > s->frame_end) { 4597 tcg_raise_tb_overflow(s); 4598 } 4599 s->current_frame_offset = off + size; 4600 #if defined(__sparc__) 4601 off += TCG_TARGET_STACK_BIAS; 4602 #endif 4603 4604 /* If the object was subdivided, assign memory to all the parts. */ 4605 if (ts->base_type != ts->type) { 4606 int part_size = tcg_type_size(ts->type); 4607 int part_count = size / part_size; 4608 4609 /* 4610 * Each part is allocated sequentially in tcg_temp_new_internal. 4611 * Jump back to the first part by subtracting the current index. 4612 */ 4613 ts -= ts->temp_subindex; 4614 for (int i = 0; i < part_count; ++i) { 4615 ts[i].mem_offset = off + i * part_size; 4616 ts[i].mem_base = s->frame_temp; 4617 ts[i].mem_allocated = 1; 4618 } 4619 } else { 4620 ts->mem_offset = off; 4621 ts->mem_base = s->frame_temp; 4622 ts->mem_allocated = 1; 4623 } 4624 } 4625 4626 /* Assign @reg to @ts, and update reg_to_temp[]. */ 4627 static void set_temp_val_reg(TCGContext *s, TCGTemp *ts, TCGReg reg) 4628 { 4629 if (ts->val_type == TEMP_VAL_REG) { 4630 TCGReg old = ts->reg; 4631 tcg_debug_assert(s->reg_to_temp[old] == ts); 4632 if (old == reg) { 4633 return; 4634 } 4635 s->reg_to_temp[old] = NULL; 4636 } 4637 tcg_debug_assert(s->reg_to_temp[reg] == NULL); 4638 s->reg_to_temp[reg] = ts; 4639 ts->val_type = TEMP_VAL_REG; 4640 ts->reg = reg; 4641 } 4642 4643 /* Assign a non-register value type to @ts, and update reg_to_temp[]. */ 4644 static void set_temp_val_nonreg(TCGContext *s, TCGTemp *ts, TCGTempVal type) 4645 { 4646 tcg_debug_assert(type != TEMP_VAL_REG); 4647 if (ts->val_type == TEMP_VAL_REG) { 4648 TCGReg reg = ts->reg; 4649 tcg_debug_assert(s->reg_to_temp[reg] == ts); 4650 s->reg_to_temp[reg] = NULL; 4651 } 4652 ts->val_type = type; 4653 } 4654 4655 static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet); 4656 4657 /* Mark a temporary as free or dead. If 'free_or_dead' is negative, 4658 mark it free; otherwise mark it dead. */ 4659 static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead) 4660 { 4661 TCGTempVal new_type; 4662 4663 switch (ts->kind) { 4664 case TEMP_FIXED: 4665 return; 4666 case TEMP_GLOBAL: 4667 case TEMP_TB: 4668 new_type = TEMP_VAL_MEM; 4669 break; 4670 case TEMP_EBB: 4671 new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD; 4672 break; 4673 case TEMP_CONST: 4674 new_type = TEMP_VAL_CONST; 4675 break; 4676 default: 4677 g_assert_not_reached(); 4678 } 4679 set_temp_val_nonreg(s, ts, new_type); 4680 } 4681 4682 /* Mark a temporary as dead. */ 4683 static inline void temp_dead(TCGContext *s, TCGTemp *ts) 4684 { 4685 temp_free_or_dead(s, ts, 1); 4686 } 4687 4688 /* Sync a temporary to memory. 'allocated_regs' is used in case a temporary 4689 registers needs to be allocated to store a constant. If 'free_or_dead' 4690 is non-zero, subsequently release the temporary; if it is positive, the 4691 temp is dead; if it is negative, the temp is free. */ 4692 static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs, 4693 TCGRegSet preferred_regs, int free_or_dead) 4694 { 4695 if (!temp_readonly(ts) && !ts->mem_coherent) { 4696 if (!ts->mem_allocated) { 4697 temp_allocate_frame(s, ts); 4698 } 4699 switch (ts->val_type) { 4700 case TEMP_VAL_CONST: 4701 /* If we're going to free the temp immediately, then we won't 4702 require it later in a register, so attempt to store the 4703 constant to memory directly. */ 4704 if (free_or_dead 4705 && tcg_out_sti(s, ts->type, ts->val, 4706 ts->mem_base->reg, ts->mem_offset)) { 4707 break; 4708 } 4709 temp_load(s, ts, tcg_target_available_regs[ts->type], 4710 allocated_regs, preferred_regs); 4711 /* fallthrough */ 4712 4713 case TEMP_VAL_REG: 4714 tcg_out_st(s, ts->type, ts->reg, 4715 ts->mem_base->reg, ts->mem_offset); 4716 break; 4717 4718 case TEMP_VAL_MEM: 4719 break; 4720 4721 case TEMP_VAL_DEAD: 4722 default: 4723 g_assert_not_reached(); 4724 } 4725 ts->mem_coherent = 1; 4726 } 4727 if (free_or_dead) { 4728 temp_free_or_dead(s, ts, free_or_dead); 4729 } 4730 } 4731 4732 /* free register 'reg' by spilling the corresponding temporary if necessary */ 4733 static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs) 4734 { 4735 TCGTemp *ts = s->reg_to_temp[reg]; 4736 if (ts != NULL) { 4737 temp_sync(s, ts, allocated_regs, 0, -1); 4738 } 4739 } 4740 4741 /** 4742 * tcg_reg_alloc: 4743 * @required_regs: Set of registers in which we must allocate. 4744 * @allocated_regs: Set of registers which must be avoided. 4745 * @preferred_regs: Set of registers we should prefer. 4746 * @rev: True if we search the registers in "indirect" order. 4747 * 4748 * The allocated register must be in @required_regs & ~@allocated_regs, 4749 * but if we can put it in @preferred_regs we may save a move later. 4750 */ 4751 static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs, 4752 TCGRegSet allocated_regs, 4753 TCGRegSet preferred_regs, bool rev) 4754 { 4755 int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order); 4756 TCGRegSet reg_ct[2]; 4757 const int *order; 4758 4759 reg_ct[1] = required_regs & ~allocated_regs; 4760 tcg_debug_assert(reg_ct[1] != 0); 4761 reg_ct[0] = reg_ct[1] & preferred_regs; 4762 4763 /* Skip the preferred_regs option if it cannot be satisfied, 4764 or if the preference made no difference. */ 4765 f = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1]; 4766 4767 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order; 4768 4769 /* Try free registers, preferences first. */ 4770 for (j = f; j < 2; j++) { 4771 TCGRegSet set = reg_ct[j]; 4772 4773 if (tcg_regset_single(set)) { 4774 /* One register in the set. */ 4775 TCGReg reg = tcg_regset_first(set); 4776 if (s->reg_to_temp[reg] == NULL) { 4777 return reg; 4778 } 4779 } else { 4780 for (i = 0; i < n; i++) { 4781 TCGReg reg = order[i]; 4782 if (s->reg_to_temp[reg] == NULL && 4783 tcg_regset_test_reg(set, reg)) { 4784 return reg; 4785 } 4786 } 4787 } 4788 } 4789 4790 /* We must spill something. */ 4791 for (j = f; j < 2; j++) { 4792 TCGRegSet set = reg_ct[j]; 4793 4794 if (tcg_regset_single(set)) { 4795 /* One register in the set. */ 4796 TCGReg reg = tcg_regset_first(set); 4797 tcg_reg_free(s, reg, allocated_regs); 4798 return reg; 4799 } else { 4800 for (i = 0; i < n; i++) { 4801 TCGReg reg = order[i]; 4802 if (tcg_regset_test_reg(set, reg)) { 4803 tcg_reg_free(s, reg, allocated_regs); 4804 return reg; 4805 } 4806 } 4807 } 4808 } 4809 4810 g_assert_not_reached(); 4811 } 4812 4813 static TCGReg tcg_reg_alloc_pair(TCGContext *s, TCGRegSet required_regs, 4814 TCGRegSet allocated_regs, 4815 TCGRegSet preferred_regs, bool rev) 4816 { 4817 int i, j, k, fmin, n = ARRAY_SIZE(tcg_target_reg_alloc_order); 4818 TCGRegSet reg_ct[2]; 4819 const int *order; 4820 4821 /* Ensure that if I is not in allocated_regs, I+1 is not either. */ 4822 reg_ct[1] = required_regs & ~(allocated_regs | (allocated_regs >> 1)); 4823 tcg_debug_assert(reg_ct[1] != 0); 4824 reg_ct[0] = reg_ct[1] & preferred_regs; 4825 4826 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order; 4827 4828 /* 4829 * Skip the preferred_regs option if it cannot be satisfied, 4830 * or if the preference made no difference. 4831 */ 4832 k = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1]; 4833 4834 /* 4835 * Minimize the number of flushes by looking for 2 free registers first, 4836 * then a single flush, then two flushes. 4837 */ 4838 for (fmin = 2; fmin >= 0; fmin--) { 4839 for (j = k; j < 2; j++) { 4840 TCGRegSet set = reg_ct[j]; 4841 4842 for (i = 0; i < n; i++) { 4843 TCGReg reg = order[i]; 4844 4845 if (tcg_regset_test_reg(set, reg)) { 4846 int f = !s->reg_to_temp[reg] + !s->reg_to_temp[reg + 1]; 4847 if (f >= fmin) { 4848 tcg_reg_free(s, reg, allocated_regs); 4849 tcg_reg_free(s, reg + 1, allocated_regs); 4850 return reg; 4851 } 4852 } 4853 } 4854 } 4855 } 4856 g_assert_not_reached(); 4857 } 4858 4859 /* Make sure the temporary is in a register. If needed, allocate the register 4860 from DESIRED while avoiding ALLOCATED. */ 4861 static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs, 4862 TCGRegSet allocated_regs, TCGRegSet preferred_regs) 4863 { 4864 TCGReg reg; 4865 4866 switch (ts->val_type) { 4867 case TEMP_VAL_REG: 4868 return; 4869 case TEMP_VAL_CONST: 4870 reg = tcg_reg_alloc(s, desired_regs, allocated_regs, 4871 preferred_regs, ts->indirect_base); 4872 if (ts->type <= TCG_TYPE_I64) { 4873 tcg_out_movi(s, ts->type, reg, ts->val); 4874 } else { 4875 uint64_t val = ts->val; 4876 MemOp vece = MO_64; 4877 4878 /* 4879 * Find the minimal vector element that matches the constant. 4880 * The targets will, in general, have to do this search anyway, 4881 * do this generically. 4882 */ 4883 if (val == dup_const(MO_8, val)) { 4884 vece = MO_8; 4885 } else if (val == dup_const(MO_16, val)) { 4886 vece = MO_16; 4887 } else if (val == dup_const(MO_32, val)) { 4888 vece = MO_32; 4889 } 4890 4891 tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val); 4892 } 4893 ts->mem_coherent = 0; 4894 break; 4895 case TEMP_VAL_MEM: 4896 if (!ts->mem_allocated) { 4897 temp_allocate_frame(s, ts); 4898 } 4899 reg = tcg_reg_alloc(s, desired_regs, allocated_regs, 4900 preferred_regs, ts->indirect_base); 4901 tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset); 4902 ts->mem_coherent = 1; 4903 break; 4904 case TEMP_VAL_DEAD: 4905 default: 4906 g_assert_not_reached(); 4907 } 4908 set_temp_val_reg(s, ts, reg); 4909 } 4910 4911 /* Save a temporary to memory. 'allocated_regs' is used in case a 4912 temporary registers needs to be allocated to store a constant. */ 4913 static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs) 4914 { 4915 /* The liveness analysis already ensures that globals are back 4916 in memory. Keep an tcg_debug_assert for safety. */ 4917 tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts)); 4918 } 4919 4920 /* save globals to their canonical location and assume they can be 4921 modified be the following code. 'allocated_regs' is used in case a 4922 temporary registers needs to be allocated to store a constant. */ 4923 static void save_globals(TCGContext *s, TCGRegSet allocated_regs) 4924 { 4925 int i, n; 4926 4927 for (i = 0, n = s->nb_globals; i < n; i++) { 4928 temp_save(s, &s->temps[i], allocated_regs); 4929 } 4930 } 4931 4932 /* sync globals to their canonical location and assume they can be 4933 read by the following code. 'allocated_regs' is used in case a 4934 temporary registers needs to be allocated to store a constant. */ 4935 static void sync_globals(TCGContext *s, TCGRegSet allocated_regs) 4936 { 4937 int i, n; 4938 4939 for (i = 0, n = s->nb_globals; i < n; i++) { 4940 TCGTemp *ts = &s->temps[i]; 4941 tcg_debug_assert(ts->val_type != TEMP_VAL_REG 4942 || ts->kind == TEMP_FIXED 4943 || ts->mem_coherent); 4944 } 4945 } 4946 4947 /* at the end of a basic block, we assume all temporaries are dead and 4948 all globals are stored at their canonical location. */ 4949 static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs) 4950 { 4951 assert_carry_dead(s); 4952 for (int i = s->nb_globals; i < s->nb_temps; i++) { 4953 TCGTemp *ts = &s->temps[i]; 4954 4955 switch (ts->kind) { 4956 case TEMP_TB: 4957 temp_save(s, ts, allocated_regs); 4958 break; 4959 case TEMP_EBB: 4960 /* The liveness analysis already ensures that temps are dead. 4961 Keep an tcg_debug_assert for safety. */ 4962 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD); 4963 break; 4964 case TEMP_CONST: 4965 /* Similarly, we should have freed any allocated register. */ 4966 tcg_debug_assert(ts->val_type == TEMP_VAL_CONST); 4967 break; 4968 default: 4969 g_assert_not_reached(); 4970 } 4971 } 4972 4973 save_globals(s, allocated_regs); 4974 } 4975 4976 /* 4977 * At a conditional branch, we assume all temporaries are dead unless 4978 * explicitly live-across-conditional-branch; all globals and local 4979 * temps are synced to their location. 4980 */ 4981 static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs) 4982 { 4983 assert_carry_dead(s); 4984 sync_globals(s, allocated_regs); 4985 4986 for (int i = s->nb_globals; i < s->nb_temps; i++) { 4987 TCGTemp *ts = &s->temps[i]; 4988 /* 4989 * The liveness analysis already ensures that temps are dead. 4990 * Keep tcg_debug_asserts for safety. 4991 */ 4992 switch (ts->kind) { 4993 case TEMP_TB: 4994 tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent); 4995 break; 4996 case TEMP_EBB: 4997 case TEMP_CONST: 4998 break; 4999 default: 5000 g_assert_not_reached(); 5001 } 5002 } 5003 } 5004 5005 /* 5006 * Specialized code generation for INDEX_op_mov_* with a constant. 5007 */ 5008 static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots, 5009 tcg_target_ulong val, TCGLifeData arg_life, 5010 TCGRegSet preferred_regs) 5011 { 5012 /* ENV should not be modified. */ 5013 tcg_debug_assert(!temp_readonly(ots)); 5014 5015 /* The movi is not explicitly generated here. */ 5016 set_temp_val_nonreg(s, ots, TEMP_VAL_CONST); 5017 ots->val = val; 5018 ots->mem_coherent = 0; 5019 if (NEED_SYNC_ARG(0)) { 5020 temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0)); 5021 } else if (IS_DEAD_ARG(0)) { 5022 temp_dead(s, ots); 5023 } 5024 } 5025 5026 /* 5027 * Specialized code generation for INDEX_op_mov_*. 5028 */ 5029 static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op) 5030 { 5031 const TCGLifeData arg_life = op->life; 5032 TCGRegSet allocated_regs, preferred_regs; 5033 TCGTemp *ts, *ots; 5034 TCGType otype, itype; 5035 TCGReg oreg, ireg; 5036 5037 allocated_regs = s->reserved_regs; 5038 preferred_regs = output_pref(op, 0); 5039 ots = arg_temp(op->args[0]); 5040 ts = arg_temp(op->args[1]); 5041 5042 /* ENV should not be modified. */ 5043 tcg_debug_assert(!temp_readonly(ots)); 5044 5045 /* Note that otype != itype for no-op truncation. */ 5046 otype = ots->type; 5047 itype = ts->type; 5048 5049 if (ts->val_type == TEMP_VAL_CONST) { 5050 /* propagate constant or generate sti */ 5051 tcg_target_ulong val = ts->val; 5052 if (IS_DEAD_ARG(1)) { 5053 temp_dead(s, ts); 5054 } 5055 tcg_reg_alloc_do_movi(s, ots, val, arg_life, preferred_regs); 5056 return; 5057 } 5058 5059 /* If the source value is in memory we're going to be forced 5060 to have it in a register in order to perform the copy. Copy 5061 the SOURCE value into its own register first, that way we 5062 don't have to reload SOURCE the next time it is used. */ 5063 if (ts->val_type == TEMP_VAL_MEM) { 5064 temp_load(s, ts, tcg_target_available_regs[itype], 5065 allocated_regs, preferred_regs); 5066 } 5067 tcg_debug_assert(ts->val_type == TEMP_VAL_REG); 5068 ireg = ts->reg; 5069 5070 if (IS_DEAD_ARG(0)) { 5071 /* mov to a non-saved dead register makes no sense (even with 5072 liveness analysis disabled). */ 5073 tcg_debug_assert(NEED_SYNC_ARG(0)); 5074 if (!ots->mem_allocated) { 5075 temp_allocate_frame(s, ots); 5076 } 5077 tcg_out_st(s, otype, ireg, ots->mem_base->reg, ots->mem_offset); 5078 if (IS_DEAD_ARG(1)) { 5079 temp_dead(s, ts); 5080 } 5081 temp_dead(s, ots); 5082 return; 5083 } 5084 5085 if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) { 5086 /* 5087 * The mov can be suppressed. Kill input first, so that it 5088 * is unlinked from reg_to_temp, then set the output to the 5089 * reg that we saved from the input. 5090 */ 5091 temp_dead(s, ts); 5092 oreg = ireg; 5093 } else { 5094 if (ots->val_type == TEMP_VAL_REG) { 5095 oreg = ots->reg; 5096 } else { 5097 /* Make sure to not spill the input register during allocation. */ 5098 oreg = tcg_reg_alloc(s, tcg_target_available_regs[otype], 5099 allocated_regs | ((TCGRegSet)1 << ireg), 5100 preferred_regs, ots->indirect_base); 5101 } 5102 if (!tcg_out_mov(s, otype, oreg, ireg)) { 5103 /* 5104 * Cross register class move not supported. 5105 * Store the source register into the destination slot 5106 * and leave the destination temp as TEMP_VAL_MEM. 5107 */ 5108 assert(!temp_readonly(ots)); 5109 if (!ts->mem_allocated) { 5110 temp_allocate_frame(s, ots); 5111 } 5112 tcg_out_st(s, ts->type, ireg, ots->mem_base->reg, ots->mem_offset); 5113 set_temp_val_nonreg(s, ts, TEMP_VAL_MEM); 5114 ots->mem_coherent = 1; 5115 return; 5116 } 5117 } 5118 set_temp_val_reg(s, ots, oreg); 5119 ots->mem_coherent = 0; 5120 5121 if (NEED_SYNC_ARG(0)) { 5122 temp_sync(s, ots, allocated_regs, 0, 0); 5123 } 5124 } 5125 5126 /* 5127 * Specialized code generation for INDEX_op_dup_vec. 5128 */ 5129 static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op) 5130 { 5131 const TCGLifeData arg_life = op->life; 5132 TCGRegSet dup_out_regs, dup_in_regs; 5133 const TCGArgConstraint *dup_args_ct; 5134 TCGTemp *its, *ots; 5135 TCGType itype, vtype; 5136 unsigned vece; 5137 int lowpart_ofs; 5138 bool ok; 5139 5140 ots = arg_temp(op->args[0]); 5141 its = arg_temp(op->args[1]); 5142 5143 /* ENV should not be modified. */ 5144 tcg_debug_assert(!temp_readonly(ots)); 5145 5146 itype = its->type; 5147 vece = TCGOP_VECE(op); 5148 vtype = TCGOP_TYPE(op); 5149 5150 if (its->val_type == TEMP_VAL_CONST) { 5151 /* Propagate constant via movi -> dupi. */ 5152 tcg_target_ulong val = its->val; 5153 if (IS_DEAD_ARG(1)) { 5154 temp_dead(s, its); 5155 } 5156 tcg_reg_alloc_do_movi(s, ots, val, arg_life, output_pref(op, 0)); 5157 return; 5158 } 5159 5160 dup_args_ct = opcode_args_ct(op); 5161 dup_out_regs = dup_args_ct[0].regs; 5162 dup_in_regs = dup_args_ct[1].regs; 5163 5164 /* Allocate the output register now. */ 5165 if (ots->val_type != TEMP_VAL_REG) { 5166 TCGRegSet allocated_regs = s->reserved_regs; 5167 TCGReg oreg; 5168 5169 if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) { 5170 /* Make sure to not spill the input register. */ 5171 tcg_regset_set_reg(allocated_regs, its->reg); 5172 } 5173 oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs, 5174 output_pref(op, 0), ots->indirect_base); 5175 set_temp_val_reg(s, ots, oreg); 5176 } 5177 5178 switch (its->val_type) { 5179 case TEMP_VAL_REG: 5180 /* 5181 * The dup constriaints must be broad, covering all possible VECE. 5182 * However, tcg_op_dup_vec() gets to see the VECE and we allow it 5183 * to fail, indicating that extra moves are required for that case. 5184 */ 5185 if (tcg_regset_test_reg(dup_in_regs, its->reg)) { 5186 if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) { 5187 goto done; 5188 } 5189 /* Try again from memory or a vector input register. */ 5190 } 5191 if (!its->mem_coherent) { 5192 /* 5193 * The input register is not synced, and so an extra store 5194 * would be required to use memory. Attempt an integer-vector 5195 * register move first. We do not have a TCGRegSet for this. 5196 */ 5197 if (tcg_out_mov(s, itype, ots->reg, its->reg)) { 5198 break; 5199 } 5200 /* Sync the temp back to its slot and load from there. */ 5201 temp_sync(s, its, s->reserved_regs, 0, 0); 5202 } 5203 /* fall through */ 5204 5205 case TEMP_VAL_MEM: 5206 lowpart_ofs = 0; 5207 if (HOST_BIG_ENDIAN) { 5208 lowpart_ofs = tcg_type_size(itype) - (1 << vece); 5209 } 5210 if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg, 5211 its->mem_offset + lowpart_ofs)) { 5212 goto done; 5213 } 5214 /* Load the input into the destination vector register. */ 5215 tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset); 5216 break; 5217 5218 default: 5219 g_assert_not_reached(); 5220 } 5221 5222 /* We now have a vector input register, so dup must succeed. */ 5223 ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg); 5224 tcg_debug_assert(ok); 5225 5226 done: 5227 ots->mem_coherent = 0; 5228 if (IS_DEAD_ARG(1)) { 5229 temp_dead(s, its); 5230 } 5231 if (NEED_SYNC_ARG(0)) { 5232 temp_sync(s, ots, s->reserved_regs, 0, 0); 5233 } 5234 if (IS_DEAD_ARG(0)) { 5235 temp_dead(s, ots); 5236 } 5237 } 5238 5239 static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) 5240 { 5241 const TCGLifeData arg_life = op->life; 5242 const TCGOpDef * const def = &tcg_op_defs[op->opc]; 5243 TCGRegSet i_allocated_regs; 5244 TCGRegSet o_allocated_regs; 5245 int i, k, nb_iargs, nb_oargs; 5246 TCGReg reg; 5247 TCGArg arg; 5248 const TCGArgConstraint *args_ct; 5249 const TCGArgConstraint *arg_ct; 5250 TCGTemp *ts; 5251 TCGArg new_args[TCG_MAX_OP_ARGS]; 5252 int const_args[TCG_MAX_OP_ARGS]; 5253 TCGCond op_cond; 5254 5255 if (def->flags & TCG_OPF_CARRY_IN) { 5256 tcg_debug_assert(s->carry_live); 5257 } 5258 5259 nb_oargs = def->nb_oargs; 5260 nb_iargs = def->nb_iargs; 5261 5262 /* copy constants */ 5263 memcpy(new_args + nb_oargs + nb_iargs, 5264 op->args + nb_oargs + nb_iargs, 5265 sizeof(TCGArg) * def->nb_cargs); 5266 5267 i_allocated_regs = s->reserved_regs; 5268 o_allocated_regs = s->reserved_regs; 5269 5270 switch (op->opc) { 5271 case INDEX_op_brcond: 5272 op_cond = op->args[2]; 5273 break; 5274 case INDEX_op_setcond: 5275 case INDEX_op_negsetcond: 5276 case INDEX_op_cmp_vec: 5277 op_cond = op->args[3]; 5278 break; 5279 case INDEX_op_brcond2_i32: 5280 op_cond = op->args[4]; 5281 break; 5282 case INDEX_op_movcond: 5283 case INDEX_op_setcond2_i32: 5284 case INDEX_op_cmpsel_vec: 5285 op_cond = op->args[5]; 5286 break; 5287 default: 5288 /* No condition within opcode. */ 5289 op_cond = TCG_COND_ALWAYS; 5290 break; 5291 } 5292 5293 args_ct = opcode_args_ct(op); 5294 5295 /* satisfy input constraints */ 5296 for (k = 0; k < nb_iargs; k++) { 5297 TCGRegSet i_preferred_regs, i_required_regs; 5298 bool allocate_new_reg, copyto_new_reg; 5299 TCGTemp *ts2; 5300 int i1, i2; 5301 5302 i = args_ct[nb_oargs + k].sort_index; 5303 arg = op->args[i]; 5304 arg_ct = &args_ct[i]; 5305 ts = arg_temp(arg); 5306 5307 if (ts->val_type == TEMP_VAL_CONST) { 5308 #ifdef TCG_REG_ZERO 5309 if (ts->val == 0 && (arg_ct->ct & TCG_CT_REG_ZERO)) { 5310 /* Hardware zero register: indicate register via non-const. */ 5311 const_args[i] = 0; 5312 new_args[i] = TCG_REG_ZERO; 5313 continue; 5314 } 5315 #endif 5316 5317 if (tcg_target_const_match(ts->val, arg_ct->ct, ts->type, 5318 op_cond, TCGOP_VECE(op))) { 5319 /* constant is OK for instruction */ 5320 const_args[i] = 1; 5321 new_args[i] = ts->val; 5322 continue; 5323 } 5324 } 5325 5326 reg = ts->reg; 5327 i_preferred_regs = 0; 5328 i_required_regs = arg_ct->regs; 5329 allocate_new_reg = false; 5330 copyto_new_reg = false; 5331 5332 switch (arg_ct->pair) { 5333 case 0: /* not paired */ 5334 if (arg_ct->ialias) { 5335 i_preferred_regs = output_pref(op, arg_ct->alias_index); 5336 5337 /* 5338 * If the input is readonly, then it cannot also be an 5339 * output and aliased to itself. If the input is not 5340 * dead after the instruction, we must allocate a new 5341 * register and move it. 5342 */ 5343 if (temp_readonly(ts) || !IS_DEAD_ARG(i) 5344 || args_ct[arg_ct->alias_index].newreg) { 5345 allocate_new_reg = true; 5346 } else if (ts->val_type == TEMP_VAL_REG) { 5347 /* 5348 * Check if the current register has already been 5349 * allocated for another input. 5350 */ 5351 allocate_new_reg = 5352 tcg_regset_test_reg(i_allocated_regs, reg); 5353 } 5354 } 5355 if (!allocate_new_reg) { 5356 temp_load(s, ts, i_required_regs, i_allocated_regs, 5357 i_preferred_regs); 5358 reg = ts->reg; 5359 allocate_new_reg = !tcg_regset_test_reg(i_required_regs, reg); 5360 } 5361 if (allocate_new_reg) { 5362 /* 5363 * Allocate a new register matching the constraint 5364 * and move the temporary register into it. 5365 */ 5366 temp_load(s, ts, tcg_target_available_regs[ts->type], 5367 i_allocated_regs, 0); 5368 reg = tcg_reg_alloc(s, i_required_regs, i_allocated_regs, 5369 i_preferred_regs, ts->indirect_base); 5370 copyto_new_reg = true; 5371 } 5372 break; 5373 5374 case 1: 5375 /* First of an input pair; if i1 == i2, the second is an output. */ 5376 i1 = i; 5377 i2 = arg_ct->pair_index; 5378 ts2 = i1 != i2 ? arg_temp(op->args[i2]) : NULL; 5379 5380 /* 5381 * It is easier to default to allocating a new pair 5382 * and to identify a few cases where it's not required. 5383 */ 5384 if (arg_ct->ialias) { 5385 i_preferred_regs = output_pref(op, arg_ct->alias_index); 5386 if (IS_DEAD_ARG(i1) && 5387 IS_DEAD_ARG(i2) && 5388 !temp_readonly(ts) && 5389 ts->val_type == TEMP_VAL_REG && 5390 ts->reg < TCG_TARGET_NB_REGS - 1 && 5391 tcg_regset_test_reg(i_required_regs, reg) && 5392 !tcg_regset_test_reg(i_allocated_regs, reg) && 5393 !tcg_regset_test_reg(i_allocated_regs, reg + 1) && 5394 (ts2 5395 ? ts2->val_type == TEMP_VAL_REG && 5396 ts2->reg == reg + 1 && 5397 !temp_readonly(ts2) 5398 : s->reg_to_temp[reg + 1] == NULL)) { 5399 break; 5400 } 5401 } else { 5402 /* Without aliasing, the pair must also be an input. */ 5403 tcg_debug_assert(ts2); 5404 if (ts->val_type == TEMP_VAL_REG && 5405 ts2->val_type == TEMP_VAL_REG && 5406 ts2->reg == reg + 1 && 5407 tcg_regset_test_reg(i_required_regs, reg)) { 5408 break; 5409 } 5410 } 5411 reg = tcg_reg_alloc_pair(s, i_required_regs, i_allocated_regs, 5412 0, ts->indirect_base); 5413 goto do_pair; 5414 5415 case 2: /* pair second */ 5416 reg = new_args[arg_ct->pair_index] + 1; 5417 goto do_pair; 5418 5419 case 3: /* ialias with second output, no first input */ 5420 tcg_debug_assert(arg_ct->ialias); 5421 i_preferred_regs = output_pref(op, arg_ct->alias_index); 5422 5423 if (IS_DEAD_ARG(i) && 5424 !temp_readonly(ts) && 5425 ts->val_type == TEMP_VAL_REG && 5426 reg > 0 && 5427 s->reg_to_temp[reg - 1] == NULL && 5428 tcg_regset_test_reg(i_required_regs, reg) && 5429 !tcg_regset_test_reg(i_allocated_regs, reg) && 5430 !tcg_regset_test_reg(i_allocated_regs, reg - 1)) { 5431 tcg_regset_set_reg(i_allocated_regs, reg - 1); 5432 break; 5433 } 5434 reg = tcg_reg_alloc_pair(s, i_required_regs >> 1, 5435 i_allocated_regs, 0, 5436 ts->indirect_base); 5437 tcg_regset_set_reg(i_allocated_regs, reg); 5438 reg += 1; 5439 goto do_pair; 5440 5441 do_pair: 5442 /* 5443 * If an aliased input is not dead after the instruction, 5444 * we must allocate a new register and move it. 5445 */ 5446 if (arg_ct->ialias && (!IS_DEAD_ARG(i) || temp_readonly(ts))) { 5447 TCGRegSet t_allocated_regs = i_allocated_regs; 5448 5449 /* 5450 * Because of the alias, and the continued life, make sure 5451 * that the temp is somewhere *other* than the reg pair, 5452 * and we get a copy in reg. 5453 */ 5454 tcg_regset_set_reg(t_allocated_regs, reg); 5455 tcg_regset_set_reg(t_allocated_regs, reg + 1); 5456 if (ts->val_type == TEMP_VAL_REG && ts->reg == reg) { 5457 /* If ts was already in reg, copy it somewhere else. */ 5458 TCGReg nr; 5459 bool ok; 5460 5461 tcg_debug_assert(ts->kind != TEMP_FIXED); 5462 nr = tcg_reg_alloc(s, tcg_target_available_regs[ts->type], 5463 t_allocated_regs, 0, ts->indirect_base); 5464 ok = tcg_out_mov(s, ts->type, nr, reg); 5465 tcg_debug_assert(ok); 5466 5467 set_temp_val_reg(s, ts, nr); 5468 } else { 5469 temp_load(s, ts, tcg_target_available_regs[ts->type], 5470 t_allocated_regs, 0); 5471 copyto_new_reg = true; 5472 } 5473 } else { 5474 /* Preferably allocate to reg, otherwise copy. */ 5475 i_required_regs = (TCGRegSet)1 << reg; 5476 temp_load(s, ts, i_required_regs, i_allocated_regs, 5477 i_preferred_regs); 5478 copyto_new_reg = ts->reg != reg; 5479 } 5480 break; 5481 5482 default: 5483 g_assert_not_reached(); 5484 } 5485 5486 if (copyto_new_reg) { 5487 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) { 5488 /* 5489 * Cross register class move not supported. Sync the 5490 * temp back to its slot and load from there. 5491 */ 5492 temp_sync(s, ts, i_allocated_regs, 0, 0); 5493 tcg_out_ld(s, ts->type, reg, 5494 ts->mem_base->reg, ts->mem_offset); 5495 } 5496 } 5497 new_args[i] = reg; 5498 const_args[i] = 0; 5499 tcg_regset_set_reg(i_allocated_regs, reg); 5500 } 5501 5502 /* mark dead temporaries and free the associated registers */ 5503 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 5504 if (IS_DEAD_ARG(i)) { 5505 temp_dead(s, arg_temp(op->args[i])); 5506 } 5507 } 5508 5509 if (def->flags & TCG_OPF_COND_BRANCH) { 5510 tcg_reg_alloc_cbranch(s, i_allocated_regs); 5511 } else if (def->flags & TCG_OPF_BB_END) { 5512 tcg_reg_alloc_bb_end(s, i_allocated_regs); 5513 } else { 5514 if (def->flags & TCG_OPF_CALL_CLOBBER) { 5515 assert_carry_dead(s); 5516 /* XXX: permit generic clobber register list ? */ 5517 for (i = 0; i < TCG_TARGET_NB_REGS; i++) { 5518 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { 5519 tcg_reg_free(s, i, i_allocated_regs); 5520 } 5521 } 5522 } 5523 if (def->flags & TCG_OPF_SIDE_EFFECTS) { 5524 /* sync globals if the op has side effects and might trigger 5525 an exception. */ 5526 sync_globals(s, i_allocated_regs); 5527 } 5528 5529 /* satisfy the output constraints */ 5530 for (k = 0; k < nb_oargs; k++) { 5531 i = args_ct[k].sort_index; 5532 arg = op->args[i]; 5533 arg_ct = &args_ct[i]; 5534 ts = arg_temp(arg); 5535 5536 /* ENV should not be modified. */ 5537 tcg_debug_assert(!temp_readonly(ts)); 5538 5539 switch (arg_ct->pair) { 5540 case 0: /* not paired */ 5541 if (arg_ct->oalias && !const_args[arg_ct->alias_index]) { 5542 reg = new_args[arg_ct->alias_index]; 5543 } else if (arg_ct->newreg) { 5544 reg = tcg_reg_alloc(s, arg_ct->regs, 5545 i_allocated_regs | o_allocated_regs, 5546 output_pref(op, k), ts->indirect_base); 5547 } else { 5548 reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs, 5549 output_pref(op, k), ts->indirect_base); 5550 } 5551 break; 5552 5553 case 1: /* first of pair */ 5554 if (arg_ct->oalias) { 5555 reg = new_args[arg_ct->alias_index]; 5556 } else if (arg_ct->newreg) { 5557 reg = tcg_reg_alloc_pair(s, arg_ct->regs, 5558 i_allocated_regs | o_allocated_regs, 5559 output_pref(op, k), 5560 ts->indirect_base); 5561 } else { 5562 reg = tcg_reg_alloc_pair(s, arg_ct->regs, o_allocated_regs, 5563 output_pref(op, k), 5564 ts->indirect_base); 5565 } 5566 break; 5567 5568 case 2: /* second of pair */ 5569 if (arg_ct->oalias) { 5570 reg = new_args[arg_ct->alias_index]; 5571 } else { 5572 reg = new_args[arg_ct->pair_index] + 1; 5573 } 5574 break; 5575 5576 case 3: /* first of pair, aliasing with a second input */ 5577 tcg_debug_assert(!arg_ct->newreg); 5578 reg = new_args[arg_ct->pair_index] - 1; 5579 break; 5580 5581 default: 5582 g_assert_not_reached(); 5583 } 5584 tcg_regset_set_reg(o_allocated_regs, reg); 5585 set_temp_val_reg(s, ts, reg); 5586 ts->mem_coherent = 0; 5587 new_args[i] = reg; 5588 } 5589 } 5590 5591 /* emit instruction */ 5592 TCGType type = TCGOP_TYPE(op); 5593 switch (op->opc) { 5594 case INDEX_op_addc1o: 5595 tcg_out_set_carry(s); 5596 /* fall through */ 5597 case INDEX_op_add: 5598 case INDEX_op_addcio: 5599 case INDEX_op_addco: 5600 case INDEX_op_and: 5601 case INDEX_op_andc: 5602 case INDEX_op_clz: 5603 case INDEX_op_ctz: 5604 case INDEX_op_divs: 5605 case INDEX_op_divu: 5606 case INDEX_op_eqv: 5607 case INDEX_op_mul: 5608 case INDEX_op_mulsh: 5609 case INDEX_op_muluh: 5610 case INDEX_op_nand: 5611 case INDEX_op_nor: 5612 case INDEX_op_or: 5613 case INDEX_op_orc: 5614 case INDEX_op_rems: 5615 case INDEX_op_remu: 5616 case INDEX_op_rotl: 5617 case INDEX_op_rotr: 5618 case INDEX_op_sar: 5619 case INDEX_op_shl: 5620 case INDEX_op_shr: 5621 case INDEX_op_xor: 5622 { 5623 const TCGOutOpBinary *out = 5624 container_of(all_outop[op->opc], TCGOutOpBinary, base); 5625 5626 /* Constants should never appear in the first source operand. */ 5627 tcg_debug_assert(!const_args[1]); 5628 if (const_args[2]) { 5629 out->out_rri(s, type, new_args[0], new_args[1], new_args[2]); 5630 } else { 5631 out->out_rrr(s, type, new_args[0], new_args[1], new_args[2]); 5632 } 5633 } 5634 break; 5635 5636 case INDEX_op_sub: 5637 { 5638 const TCGOutOpSubtract *out = &outop_sub; 5639 5640 /* 5641 * Constants should never appear in the second source operand. 5642 * These are folded to add with negative constant. 5643 */ 5644 tcg_debug_assert(!const_args[2]); 5645 if (const_args[1]) { 5646 out->out_rir(s, type, new_args[0], new_args[1], new_args[2]); 5647 } else { 5648 out->out_rrr(s, type, new_args[0], new_args[1], new_args[2]); 5649 } 5650 } 5651 break; 5652 5653 case INDEX_op_subb1o: 5654 tcg_out_set_borrow(s); 5655 /* fall through */ 5656 case INDEX_op_addci: 5657 case INDEX_op_subbi: 5658 case INDEX_op_subbio: 5659 case INDEX_op_subbo: 5660 { 5661 const TCGOutOpAddSubCarry *out = 5662 container_of(all_outop[op->opc], TCGOutOpAddSubCarry, base); 5663 5664 if (const_args[2]) { 5665 if (const_args[1]) { 5666 out->out_rii(s, type, new_args[0], 5667 new_args[1], new_args[2]); 5668 } else { 5669 out->out_rri(s, type, new_args[0], 5670 new_args[1], new_args[2]); 5671 } 5672 } else if (const_args[1]) { 5673 out->out_rir(s, type, new_args[0], new_args[1], new_args[2]); 5674 } else { 5675 out->out_rrr(s, type, new_args[0], new_args[1], new_args[2]); 5676 } 5677 } 5678 break; 5679 5680 case INDEX_op_bswap64: 5681 case INDEX_op_ext_i32_i64: 5682 case INDEX_op_extu_i32_i64: 5683 case INDEX_op_extrl_i64_i32: 5684 case INDEX_op_extrh_i64_i32: 5685 assert(TCG_TARGET_REG_BITS == 64); 5686 /* fall through */ 5687 case INDEX_op_ctpop: 5688 case INDEX_op_neg: 5689 case INDEX_op_not: 5690 { 5691 const TCGOutOpUnary *out = 5692 container_of(all_outop[op->opc], TCGOutOpUnary, base); 5693 5694 /* Constants should have been folded. */ 5695 tcg_debug_assert(!const_args[1]); 5696 out->out_rr(s, type, new_args[0], new_args[1]); 5697 } 5698 break; 5699 5700 case INDEX_op_bswap16: 5701 case INDEX_op_bswap32: 5702 { 5703 const TCGOutOpBswap *out = 5704 container_of(all_outop[op->opc], TCGOutOpBswap, base); 5705 5706 tcg_debug_assert(!const_args[1]); 5707 out->out_rr(s, type, new_args[0], new_args[1], new_args[2]); 5708 } 5709 break; 5710 5711 case INDEX_op_deposit: 5712 { 5713 const TCGOutOpDeposit *out = &outop_deposit; 5714 5715 if (const_args[2]) { 5716 tcg_debug_assert(!const_args[1]); 5717 out->out_rri(s, type, new_args[0], new_args[1], 5718 new_args[2], new_args[3], new_args[4]); 5719 } else if (const_args[1]) { 5720 tcg_debug_assert(new_args[1] == 0); 5721 tcg_debug_assert(!const_args[2]); 5722 out->out_rzr(s, type, new_args[0], new_args[2], 5723 new_args[3], new_args[4]); 5724 } else { 5725 out->out_rrr(s, type, new_args[0], new_args[1], 5726 new_args[2], new_args[3], new_args[4]); 5727 } 5728 } 5729 break; 5730 5731 case INDEX_op_divs2: 5732 case INDEX_op_divu2: 5733 { 5734 const TCGOutOpDivRem *out = 5735 container_of(all_outop[op->opc], TCGOutOpDivRem, base); 5736 5737 /* Only used by x86 and s390x, which use matching constraints. */ 5738 tcg_debug_assert(new_args[0] == new_args[2]); 5739 tcg_debug_assert(new_args[1] == new_args[3]); 5740 tcg_debug_assert(!const_args[4]); 5741 out->out_rr01r(s, type, new_args[0], new_args[1], new_args[4]); 5742 } 5743 break; 5744 5745 case INDEX_op_extract: 5746 case INDEX_op_sextract: 5747 { 5748 const TCGOutOpExtract *out = 5749 container_of(all_outop[op->opc], TCGOutOpExtract, base); 5750 5751 tcg_debug_assert(!const_args[1]); 5752 out->out_rr(s, type, new_args[0], new_args[1], 5753 new_args[2], new_args[3]); 5754 } 5755 break; 5756 5757 case INDEX_op_extract2: 5758 { 5759 const TCGOutOpExtract2 *out = &outop_extract2; 5760 5761 tcg_debug_assert(!const_args[1]); 5762 tcg_debug_assert(!const_args[2]); 5763 out->out_rrr(s, type, new_args[0], new_args[1], 5764 new_args[2], new_args[3]); 5765 } 5766 break; 5767 5768 case INDEX_op_ld8u: 5769 case INDEX_op_ld8s: 5770 case INDEX_op_ld16u: 5771 case INDEX_op_ld16s: 5772 case INDEX_op_ld32u: 5773 case INDEX_op_ld32s: 5774 case INDEX_op_ld: 5775 { 5776 const TCGOutOpLoad *out = 5777 container_of(all_outop[op->opc], TCGOutOpLoad, base); 5778 5779 tcg_debug_assert(!const_args[1]); 5780 out->out(s, type, new_args[0], new_args[1], new_args[2]); 5781 } 5782 break; 5783 5784 case INDEX_op_muls2: 5785 case INDEX_op_mulu2: 5786 { 5787 const TCGOutOpMul2 *out = 5788 container_of(all_outop[op->opc], TCGOutOpMul2, base); 5789 5790 tcg_debug_assert(!const_args[2]); 5791 tcg_debug_assert(!const_args[3]); 5792 out->out_rrrr(s, type, new_args[0], new_args[1], 5793 new_args[2], new_args[3]); 5794 } 5795 break; 5796 5797 case INDEX_op_st32: 5798 /* Use tcg_op_st w/ I32. */ 5799 type = TCG_TYPE_I32; 5800 /* fall through */ 5801 case INDEX_op_st: 5802 case INDEX_op_st8: 5803 case INDEX_op_st16: 5804 { 5805 const TCGOutOpStore *out = 5806 container_of(all_outop[op->opc], TCGOutOpStore, base); 5807 5808 if (const_args[0]) { 5809 out->out_i(s, type, new_args[0], new_args[1], new_args[2]); 5810 } else { 5811 out->out_r(s, type, new_args[0], new_args[1], new_args[2]); 5812 } 5813 } 5814 break; 5815 5816 case INDEX_op_qemu_ld: 5817 case INDEX_op_qemu_st: 5818 { 5819 const TCGOutOpQemuLdSt *out = 5820 container_of(all_outop[op->opc], TCGOutOpQemuLdSt, base); 5821 5822 out->out(s, type, new_args[0], new_args[1], new_args[2]); 5823 } 5824 break; 5825 5826 case INDEX_op_qemu_ld2: 5827 case INDEX_op_qemu_st2: 5828 { 5829 const TCGOutOpQemuLdSt2 *out = 5830 container_of(all_outop[op->opc], TCGOutOpQemuLdSt2, base); 5831 5832 out->out(s, type, new_args[0], new_args[1], 5833 new_args[2], new_args[3]); 5834 } 5835 break; 5836 5837 case INDEX_op_brcond: 5838 { 5839 const TCGOutOpBrcond *out = &outop_brcond; 5840 TCGCond cond = new_args[2]; 5841 TCGLabel *label = arg_label(new_args[3]); 5842 5843 tcg_debug_assert(!const_args[0]); 5844 if (const_args[1]) { 5845 out->out_ri(s, type, cond, new_args[0], new_args[1], label); 5846 } else { 5847 out->out_rr(s, type, cond, new_args[0], new_args[1], label); 5848 } 5849 } 5850 break; 5851 5852 case INDEX_op_movcond: 5853 { 5854 const TCGOutOpMovcond *out = &outop_movcond; 5855 TCGCond cond = new_args[5]; 5856 5857 tcg_debug_assert(!const_args[1]); 5858 out->out(s, type, cond, new_args[0], 5859 new_args[1], new_args[2], const_args[2], 5860 new_args[3], const_args[3], 5861 new_args[4], const_args[4]); 5862 } 5863 break; 5864 5865 case INDEX_op_setcond: 5866 case INDEX_op_negsetcond: 5867 { 5868 const TCGOutOpSetcond *out = 5869 container_of(all_outop[op->opc], TCGOutOpSetcond, base); 5870 TCGCond cond = new_args[3]; 5871 5872 tcg_debug_assert(!const_args[1]); 5873 if (const_args[2]) { 5874 out->out_rri(s, type, cond, 5875 new_args[0], new_args[1], new_args[2]); 5876 } else { 5877 out->out_rrr(s, type, cond, 5878 new_args[0], new_args[1], new_args[2]); 5879 } 5880 } 5881 break; 5882 5883 #if TCG_TARGET_REG_BITS == 32 5884 case INDEX_op_brcond2_i32: 5885 { 5886 const TCGOutOpBrcond2 *out = &outop_brcond2; 5887 TCGCond cond = new_args[4]; 5888 TCGLabel *label = arg_label(new_args[5]); 5889 5890 tcg_debug_assert(!const_args[0]); 5891 tcg_debug_assert(!const_args[1]); 5892 out->out(s, cond, new_args[0], new_args[1], 5893 new_args[2], const_args[2], 5894 new_args[3], const_args[3], label); 5895 } 5896 break; 5897 case INDEX_op_setcond2_i32: 5898 { 5899 const TCGOutOpSetcond2 *out = &outop_setcond2; 5900 TCGCond cond = new_args[5]; 5901 5902 tcg_debug_assert(!const_args[1]); 5903 tcg_debug_assert(!const_args[2]); 5904 out->out(s, cond, new_args[0], new_args[1], new_args[2], 5905 new_args[3], const_args[3], new_args[4], const_args[4]); 5906 } 5907 break; 5908 #else 5909 case INDEX_op_brcond2_i32: 5910 case INDEX_op_setcond2_i32: 5911 g_assert_not_reached(); 5912 #endif 5913 5914 case INDEX_op_goto_ptr: 5915 tcg_debug_assert(!const_args[0]); 5916 tcg_out_goto_ptr(s, new_args[0]); 5917 break; 5918 5919 default: 5920 tcg_debug_assert(def->flags & TCG_OPF_VECTOR); 5921 tcg_out_vec_op(s, op->opc, type - TCG_TYPE_V64, 5922 TCGOP_VECE(op), new_args, const_args); 5923 break; 5924 } 5925 5926 if (def->flags & TCG_OPF_CARRY_IN) { 5927 s->carry_live = false; 5928 } 5929 if (def->flags & TCG_OPF_CARRY_OUT) { 5930 s->carry_live = true; 5931 } 5932 5933 /* move the outputs in the correct register if needed */ 5934 for(i = 0; i < nb_oargs; i++) { 5935 ts = arg_temp(op->args[i]); 5936 5937 /* ENV should not be modified. */ 5938 tcg_debug_assert(!temp_readonly(ts)); 5939 5940 if (NEED_SYNC_ARG(i)) { 5941 temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i)); 5942 } else if (IS_DEAD_ARG(i)) { 5943 temp_dead(s, ts); 5944 } 5945 } 5946 } 5947 5948 static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op) 5949 { 5950 const TCGLifeData arg_life = op->life; 5951 TCGTemp *ots, *itsl, *itsh; 5952 TCGType vtype = TCGOP_TYPE(op); 5953 5954 /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */ 5955 tcg_debug_assert(TCG_TARGET_REG_BITS == 32); 5956 tcg_debug_assert(TCGOP_VECE(op) == MO_64); 5957 5958 ots = arg_temp(op->args[0]); 5959 itsl = arg_temp(op->args[1]); 5960 itsh = arg_temp(op->args[2]); 5961 5962 /* ENV should not be modified. */ 5963 tcg_debug_assert(!temp_readonly(ots)); 5964 5965 /* Allocate the output register now. */ 5966 if (ots->val_type != TEMP_VAL_REG) { 5967 TCGRegSet allocated_regs = s->reserved_regs; 5968 TCGRegSet dup_out_regs = opcode_args_ct(op)[0].regs; 5969 TCGReg oreg; 5970 5971 /* Make sure to not spill the input registers. */ 5972 if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) { 5973 tcg_regset_set_reg(allocated_regs, itsl->reg); 5974 } 5975 if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) { 5976 tcg_regset_set_reg(allocated_regs, itsh->reg); 5977 } 5978 5979 oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs, 5980 output_pref(op, 0), ots->indirect_base); 5981 set_temp_val_reg(s, ots, oreg); 5982 } 5983 5984 /* Promote dup2 of immediates to dupi_vec. */ 5985 if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) { 5986 uint64_t val = deposit64(itsl->val, 32, 32, itsh->val); 5987 MemOp vece = MO_64; 5988 5989 if (val == dup_const(MO_8, val)) { 5990 vece = MO_8; 5991 } else if (val == dup_const(MO_16, val)) { 5992 vece = MO_16; 5993 } else if (val == dup_const(MO_32, val)) { 5994 vece = MO_32; 5995 } 5996 5997 tcg_out_dupi_vec(s, vtype, vece, ots->reg, val); 5998 goto done; 5999 } 6000 6001 /* If the two inputs form one 64-bit value, try dupm_vec. */ 6002 if (itsl->temp_subindex == HOST_BIG_ENDIAN && 6003 itsh->temp_subindex == !HOST_BIG_ENDIAN && 6004 itsl == itsh + (HOST_BIG_ENDIAN ? 1 : -1)) { 6005 TCGTemp *its = itsl - HOST_BIG_ENDIAN; 6006 6007 temp_sync(s, its + 0, s->reserved_regs, 0, 0); 6008 temp_sync(s, its + 1, s->reserved_regs, 0, 0); 6009 6010 if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg, 6011 its->mem_base->reg, its->mem_offset)) { 6012 goto done; 6013 } 6014 } 6015 6016 /* Fall back to generic expansion. */ 6017 return false; 6018 6019 done: 6020 ots->mem_coherent = 0; 6021 if (IS_DEAD_ARG(1)) { 6022 temp_dead(s, itsl); 6023 } 6024 if (IS_DEAD_ARG(2)) { 6025 temp_dead(s, itsh); 6026 } 6027 if (NEED_SYNC_ARG(0)) { 6028 temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0)); 6029 } else if (IS_DEAD_ARG(0)) { 6030 temp_dead(s, ots); 6031 } 6032 return true; 6033 } 6034 6035 static void load_arg_reg(TCGContext *s, TCGReg reg, TCGTemp *ts, 6036 TCGRegSet allocated_regs) 6037 { 6038 if (ts->val_type == TEMP_VAL_REG) { 6039 if (ts->reg != reg) { 6040 tcg_reg_free(s, reg, allocated_regs); 6041 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) { 6042 /* 6043 * Cross register class move not supported. Sync the 6044 * temp back to its slot and load from there. 6045 */ 6046 temp_sync(s, ts, allocated_regs, 0, 0); 6047 tcg_out_ld(s, ts->type, reg, 6048 ts->mem_base->reg, ts->mem_offset); 6049 } 6050 } 6051 } else { 6052 TCGRegSet arg_set = 0; 6053 6054 tcg_reg_free(s, reg, allocated_regs); 6055 tcg_regset_set_reg(arg_set, reg); 6056 temp_load(s, ts, arg_set, allocated_regs, 0); 6057 } 6058 } 6059 6060 static void load_arg_stk(TCGContext *s, unsigned arg_slot, TCGTemp *ts, 6061 TCGRegSet allocated_regs) 6062 { 6063 /* 6064 * When the destination is on the stack, load up the temp and store. 6065 * If there are many call-saved registers, the temp might live to 6066 * see another use; otherwise it'll be discarded. 6067 */ 6068 temp_load(s, ts, tcg_target_available_regs[ts->type], allocated_regs, 0); 6069 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, 6070 arg_slot_stk_ofs(arg_slot)); 6071 } 6072 6073 static void load_arg_normal(TCGContext *s, const TCGCallArgumentLoc *l, 6074 TCGTemp *ts, TCGRegSet *allocated_regs) 6075 { 6076 if (arg_slot_reg_p(l->arg_slot)) { 6077 TCGReg reg = tcg_target_call_iarg_regs[l->arg_slot]; 6078 load_arg_reg(s, reg, ts, *allocated_regs); 6079 tcg_regset_set_reg(*allocated_regs, reg); 6080 } else { 6081 load_arg_stk(s, l->arg_slot, ts, *allocated_regs); 6082 } 6083 } 6084 6085 static void load_arg_ref(TCGContext *s, unsigned arg_slot, TCGReg ref_base, 6086 intptr_t ref_off, TCGRegSet *allocated_regs) 6087 { 6088 TCGReg reg; 6089 6090 if (arg_slot_reg_p(arg_slot)) { 6091 reg = tcg_target_call_iarg_regs[arg_slot]; 6092 tcg_reg_free(s, reg, *allocated_regs); 6093 tcg_out_addi_ptr(s, reg, ref_base, ref_off); 6094 tcg_regset_set_reg(*allocated_regs, reg); 6095 } else { 6096 reg = tcg_reg_alloc(s, tcg_target_available_regs[TCG_TYPE_PTR], 6097 *allocated_regs, 0, false); 6098 tcg_out_addi_ptr(s, reg, ref_base, ref_off); 6099 tcg_out_st(s, TCG_TYPE_PTR, reg, TCG_REG_CALL_STACK, 6100 arg_slot_stk_ofs(arg_slot)); 6101 } 6102 } 6103 6104 static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) 6105 { 6106 const int nb_oargs = TCGOP_CALLO(op); 6107 const int nb_iargs = TCGOP_CALLI(op); 6108 const TCGLifeData arg_life = op->life; 6109 const TCGHelperInfo *info = tcg_call_info(op); 6110 TCGRegSet allocated_regs = s->reserved_regs; 6111 int i; 6112 6113 /* 6114 * Move inputs into place in reverse order, 6115 * so that we place stacked arguments first. 6116 */ 6117 for (i = nb_iargs - 1; i >= 0; --i) { 6118 const TCGCallArgumentLoc *loc = &info->in[i]; 6119 TCGTemp *ts = arg_temp(op->args[nb_oargs + i]); 6120 6121 switch (loc->kind) { 6122 case TCG_CALL_ARG_NORMAL: 6123 case TCG_CALL_ARG_EXTEND_U: 6124 case TCG_CALL_ARG_EXTEND_S: 6125 load_arg_normal(s, loc, ts, &allocated_regs); 6126 break; 6127 case TCG_CALL_ARG_BY_REF: 6128 load_arg_stk(s, loc->ref_slot, ts, allocated_regs); 6129 load_arg_ref(s, loc->arg_slot, TCG_REG_CALL_STACK, 6130 arg_slot_stk_ofs(loc->ref_slot), 6131 &allocated_regs); 6132 break; 6133 case TCG_CALL_ARG_BY_REF_N: 6134 load_arg_stk(s, loc->ref_slot, ts, allocated_regs); 6135 break; 6136 default: 6137 g_assert_not_reached(); 6138 } 6139 } 6140 6141 /* Mark dead temporaries and free the associated registers. */ 6142 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 6143 if (IS_DEAD_ARG(i)) { 6144 temp_dead(s, arg_temp(op->args[i])); 6145 } 6146 } 6147 6148 /* Clobber call registers. */ 6149 for (i = 0; i < TCG_TARGET_NB_REGS; i++) { 6150 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { 6151 tcg_reg_free(s, i, allocated_regs); 6152 } 6153 } 6154 6155 /* 6156 * Save globals if they might be written by the helper, 6157 * sync them if they might be read. 6158 */ 6159 if (info->flags & TCG_CALL_NO_READ_GLOBALS) { 6160 /* Nothing to do */ 6161 } else if (info->flags & TCG_CALL_NO_WRITE_GLOBALS) { 6162 sync_globals(s, allocated_regs); 6163 } else { 6164 save_globals(s, allocated_regs); 6165 } 6166 6167 /* 6168 * If the ABI passes a pointer to the returned struct as the first 6169 * argument, load that now. Pass a pointer to the output home slot. 6170 */ 6171 if (info->out_kind == TCG_CALL_RET_BY_REF) { 6172 TCGTemp *ts = arg_temp(op->args[0]); 6173 6174 if (!ts->mem_allocated) { 6175 temp_allocate_frame(s, ts); 6176 } 6177 load_arg_ref(s, 0, ts->mem_base->reg, ts->mem_offset, &allocated_regs); 6178 } 6179 6180 tcg_out_call(s, tcg_call_func(op), info); 6181 6182 /* Assign output registers and emit moves if needed. */ 6183 switch (info->out_kind) { 6184 case TCG_CALL_RET_NORMAL: 6185 for (i = 0; i < nb_oargs; i++) { 6186 TCGTemp *ts = arg_temp(op->args[i]); 6187 TCGReg reg = tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, i); 6188 6189 /* ENV should not be modified. */ 6190 tcg_debug_assert(!temp_readonly(ts)); 6191 6192 set_temp_val_reg(s, ts, reg); 6193 ts->mem_coherent = 0; 6194 } 6195 break; 6196 6197 case TCG_CALL_RET_BY_VEC: 6198 { 6199 TCGTemp *ts = arg_temp(op->args[0]); 6200 6201 tcg_debug_assert(ts->base_type == TCG_TYPE_I128); 6202 tcg_debug_assert(ts->temp_subindex == 0); 6203 if (!ts->mem_allocated) { 6204 temp_allocate_frame(s, ts); 6205 } 6206 tcg_out_st(s, TCG_TYPE_V128, 6207 tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0), 6208 ts->mem_base->reg, ts->mem_offset); 6209 } 6210 /* fall through to mark all parts in memory */ 6211 6212 case TCG_CALL_RET_BY_REF: 6213 /* The callee has performed a write through the reference. */ 6214 for (i = 0; i < nb_oargs; i++) { 6215 TCGTemp *ts = arg_temp(op->args[i]); 6216 ts->val_type = TEMP_VAL_MEM; 6217 } 6218 break; 6219 6220 default: 6221 g_assert_not_reached(); 6222 } 6223 6224 /* Flush or discard output registers as needed. */ 6225 for (i = 0; i < nb_oargs; i++) { 6226 TCGTemp *ts = arg_temp(op->args[i]); 6227 if (NEED_SYNC_ARG(i)) { 6228 temp_sync(s, ts, s->reserved_regs, 0, IS_DEAD_ARG(i)); 6229 } else if (IS_DEAD_ARG(i)) { 6230 temp_dead(s, ts); 6231 } 6232 } 6233 } 6234 6235 /** 6236 * atom_and_align_for_opc: 6237 * @s: tcg context 6238 * @opc: memory operation code 6239 * @host_atom: MO_ATOM_{IFALIGN,WITHIN16,SUBALIGN} for host operations 6240 * @allow_two_ops: true if we are prepared to issue two operations 6241 * 6242 * Return the alignment and atomicity to use for the inline fast path 6243 * for the given memory operation. The alignment may be larger than 6244 * that specified in @opc, and the correct alignment will be diagnosed 6245 * by the slow path helper. 6246 * 6247 * If @allow_two_ops, the host is prepared to test for 2x alignment, 6248 * and issue two loads or stores for subalignment. 6249 */ 6250 static TCGAtomAlign atom_and_align_for_opc(TCGContext *s, MemOp opc, 6251 MemOp host_atom, bool allow_two_ops) 6252 { 6253 MemOp align = memop_alignment_bits(opc); 6254 MemOp size = opc & MO_SIZE; 6255 MemOp half = size ? size - 1 : 0; 6256 MemOp atom = opc & MO_ATOM_MASK; 6257 MemOp atmax; 6258 6259 switch (atom) { 6260 case MO_ATOM_NONE: 6261 /* The operation requires no specific atomicity. */ 6262 atmax = MO_8; 6263 break; 6264 6265 case MO_ATOM_IFALIGN: 6266 atmax = size; 6267 break; 6268 6269 case MO_ATOM_IFALIGN_PAIR: 6270 atmax = half; 6271 break; 6272 6273 case MO_ATOM_WITHIN16: 6274 atmax = size; 6275 if (size == MO_128) { 6276 /* Misalignment implies !within16, and therefore no atomicity. */ 6277 } else if (host_atom != MO_ATOM_WITHIN16) { 6278 /* The host does not implement within16, so require alignment. */ 6279 align = MAX(align, size); 6280 } 6281 break; 6282 6283 case MO_ATOM_WITHIN16_PAIR: 6284 atmax = size; 6285 /* 6286 * Misalignment implies !within16, and therefore half atomicity. 6287 * Any host prepared for two operations can implement this with 6288 * half alignment. 6289 */ 6290 if (host_atom != MO_ATOM_WITHIN16 && allow_two_ops) { 6291 align = MAX(align, half); 6292 } 6293 break; 6294 6295 case MO_ATOM_SUBALIGN: 6296 atmax = size; 6297 if (host_atom != MO_ATOM_SUBALIGN) { 6298 /* If unaligned but not odd, there are subobjects up to half. */ 6299 if (allow_two_ops) { 6300 align = MAX(align, half); 6301 } else { 6302 align = MAX(align, size); 6303 } 6304 } 6305 break; 6306 6307 default: 6308 g_assert_not_reached(); 6309 } 6310 6311 return (TCGAtomAlign){ .atom = atmax, .align = align }; 6312 } 6313 6314 /* 6315 * Similarly for qemu_ld/st slow path helpers. 6316 * We must re-implement tcg_gen_callN and tcg_reg_alloc_call simultaneously, 6317 * using only the provided backend tcg_out_* functions. 6318 */ 6319 6320 static int tcg_out_helper_stk_ofs(TCGType type, unsigned slot) 6321 { 6322 int ofs = arg_slot_stk_ofs(slot); 6323 6324 /* 6325 * Each stack slot is TCG_TARGET_LONG_BITS. If the host does not 6326 * require extension to uint64_t, adjust the address for uint32_t. 6327 */ 6328 if (HOST_BIG_ENDIAN && 6329 TCG_TARGET_REG_BITS == 64 && 6330 type == TCG_TYPE_I32) { 6331 ofs += 4; 6332 } 6333 return ofs; 6334 } 6335 6336 static void tcg_out_helper_load_slots(TCGContext *s, 6337 unsigned nmov, TCGMovExtend *mov, 6338 const TCGLdstHelperParam *parm) 6339 { 6340 unsigned i; 6341 TCGReg dst3; 6342 6343 /* 6344 * Start from the end, storing to the stack first. 6345 * This frees those registers, so we need not consider overlap. 6346 */ 6347 for (i = nmov; i-- > 0; ) { 6348 unsigned slot = mov[i].dst; 6349 6350 if (arg_slot_reg_p(slot)) { 6351 goto found_reg; 6352 } 6353 6354 TCGReg src = mov[i].src; 6355 TCGType dst_type = mov[i].dst_type; 6356 MemOp dst_mo = dst_type == TCG_TYPE_I32 ? MO_32 : MO_64; 6357 6358 /* The argument is going onto the stack; extend into scratch. */ 6359 if ((mov[i].src_ext & MO_SIZE) != dst_mo) { 6360 tcg_debug_assert(parm->ntmp != 0); 6361 mov[i].dst = src = parm->tmp[0]; 6362 tcg_out_movext1(s, &mov[i]); 6363 } 6364 6365 tcg_out_st(s, dst_type, src, TCG_REG_CALL_STACK, 6366 tcg_out_helper_stk_ofs(dst_type, slot)); 6367 } 6368 return; 6369 6370 found_reg: 6371 /* 6372 * The remaining arguments are in registers. 6373 * Convert slot numbers to argument registers. 6374 */ 6375 nmov = i + 1; 6376 for (i = 0; i < nmov; ++i) { 6377 mov[i].dst = tcg_target_call_iarg_regs[mov[i].dst]; 6378 } 6379 6380 switch (nmov) { 6381 case 4: 6382 /* The backend must have provided enough temps for the worst case. */ 6383 tcg_debug_assert(parm->ntmp >= 2); 6384 6385 dst3 = mov[3].dst; 6386 for (unsigned j = 0; j < 3; ++j) { 6387 if (dst3 == mov[j].src) { 6388 /* 6389 * Conflict. Copy the source to a temporary, perform the 6390 * remaining moves, then the extension from our scratch 6391 * on the way out. 6392 */ 6393 TCGReg scratch = parm->tmp[1]; 6394 6395 tcg_out_mov(s, mov[3].src_type, scratch, mov[3].src); 6396 tcg_out_movext3(s, mov, mov + 1, mov + 2, parm->tmp[0]); 6397 tcg_out_movext1_new_src(s, &mov[3], scratch); 6398 break; 6399 } 6400 } 6401 6402 /* No conflicts: perform this move and continue. */ 6403 tcg_out_movext1(s, &mov[3]); 6404 /* fall through */ 6405 6406 case 3: 6407 tcg_out_movext3(s, mov, mov + 1, mov + 2, 6408 parm->ntmp ? parm->tmp[0] : -1); 6409 break; 6410 case 2: 6411 tcg_out_movext2(s, mov, mov + 1, 6412 parm->ntmp ? parm->tmp[0] : -1); 6413 break; 6414 case 1: 6415 tcg_out_movext1(s, mov); 6416 break; 6417 default: 6418 g_assert_not_reached(); 6419 } 6420 } 6421 6422 static void tcg_out_helper_load_imm(TCGContext *s, unsigned slot, 6423 TCGType type, tcg_target_long imm, 6424 const TCGLdstHelperParam *parm) 6425 { 6426 if (arg_slot_reg_p(slot)) { 6427 tcg_out_movi(s, type, tcg_target_call_iarg_regs[slot], imm); 6428 } else { 6429 int ofs = tcg_out_helper_stk_ofs(type, slot); 6430 if (!tcg_out_sti(s, type, imm, TCG_REG_CALL_STACK, ofs)) { 6431 tcg_debug_assert(parm->ntmp != 0); 6432 tcg_out_movi(s, type, parm->tmp[0], imm); 6433 tcg_out_st(s, type, parm->tmp[0], TCG_REG_CALL_STACK, ofs); 6434 } 6435 } 6436 } 6437 6438 static void tcg_out_helper_load_common_args(TCGContext *s, 6439 const TCGLabelQemuLdst *ldst, 6440 const TCGLdstHelperParam *parm, 6441 const TCGHelperInfo *info, 6442 unsigned next_arg) 6443 { 6444 TCGMovExtend ptr_mov = { 6445 .dst_type = TCG_TYPE_PTR, 6446 .src_type = TCG_TYPE_PTR, 6447 .src_ext = sizeof(void *) == 4 ? MO_32 : MO_64 6448 }; 6449 const TCGCallArgumentLoc *loc = &info->in[0]; 6450 TCGType type; 6451 unsigned slot; 6452 tcg_target_ulong imm; 6453 6454 /* 6455 * Handle env, which is always first. 6456 */ 6457 ptr_mov.dst = loc->arg_slot; 6458 ptr_mov.src = TCG_AREG0; 6459 tcg_out_helper_load_slots(s, 1, &ptr_mov, parm); 6460 6461 /* 6462 * Handle oi. 6463 */ 6464 imm = ldst->oi; 6465 loc = &info->in[next_arg]; 6466 type = TCG_TYPE_I32; 6467 switch (loc->kind) { 6468 case TCG_CALL_ARG_NORMAL: 6469 break; 6470 case TCG_CALL_ARG_EXTEND_U: 6471 case TCG_CALL_ARG_EXTEND_S: 6472 /* No extension required for MemOpIdx. */ 6473 tcg_debug_assert(imm <= INT32_MAX); 6474 type = TCG_TYPE_REG; 6475 break; 6476 default: 6477 g_assert_not_reached(); 6478 } 6479 tcg_out_helper_load_imm(s, loc->arg_slot, type, imm, parm); 6480 next_arg++; 6481 6482 /* 6483 * Handle ra. 6484 */ 6485 loc = &info->in[next_arg]; 6486 slot = loc->arg_slot; 6487 if (parm->ra_gen) { 6488 int arg_reg = -1; 6489 TCGReg ra_reg; 6490 6491 if (arg_slot_reg_p(slot)) { 6492 arg_reg = tcg_target_call_iarg_regs[slot]; 6493 } 6494 ra_reg = parm->ra_gen(s, ldst, arg_reg); 6495 6496 ptr_mov.dst = slot; 6497 ptr_mov.src = ra_reg; 6498 tcg_out_helper_load_slots(s, 1, &ptr_mov, parm); 6499 } else { 6500 imm = (uintptr_t)ldst->raddr; 6501 tcg_out_helper_load_imm(s, slot, TCG_TYPE_PTR, imm, parm); 6502 } 6503 } 6504 6505 static unsigned tcg_out_helper_add_mov(TCGMovExtend *mov, 6506 const TCGCallArgumentLoc *loc, 6507 TCGType dst_type, TCGType src_type, 6508 TCGReg lo, TCGReg hi) 6509 { 6510 MemOp reg_mo; 6511 6512 if (dst_type <= TCG_TYPE_REG) { 6513 MemOp src_ext; 6514 6515 switch (loc->kind) { 6516 case TCG_CALL_ARG_NORMAL: 6517 src_ext = src_type == TCG_TYPE_I32 ? MO_32 : MO_64; 6518 break; 6519 case TCG_CALL_ARG_EXTEND_U: 6520 dst_type = TCG_TYPE_REG; 6521 src_ext = MO_UL; 6522 break; 6523 case TCG_CALL_ARG_EXTEND_S: 6524 dst_type = TCG_TYPE_REG; 6525 src_ext = MO_SL; 6526 break; 6527 default: 6528 g_assert_not_reached(); 6529 } 6530 6531 mov[0].dst = loc->arg_slot; 6532 mov[0].dst_type = dst_type; 6533 mov[0].src = lo; 6534 mov[0].src_type = src_type; 6535 mov[0].src_ext = src_ext; 6536 return 1; 6537 } 6538 6539 if (TCG_TARGET_REG_BITS == 32) { 6540 assert(dst_type == TCG_TYPE_I64); 6541 reg_mo = MO_32; 6542 } else { 6543 assert(dst_type == TCG_TYPE_I128); 6544 reg_mo = MO_64; 6545 } 6546 6547 mov[0].dst = loc[HOST_BIG_ENDIAN].arg_slot; 6548 mov[0].src = lo; 6549 mov[0].dst_type = TCG_TYPE_REG; 6550 mov[0].src_type = TCG_TYPE_REG; 6551 mov[0].src_ext = reg_mo; 6552 6553 mov[1].dst = loc[!HOST_BIG_ENDIAN].arg_slot; 6554 mov[1].src = hi; 6555 mov[1].dst_type = TCG_TYPE_REG; 6556 mov[1].src_type = TCG_TYPE_REG; 6557 mov[1].src_ext = reg_mo; 6558 6559 return 2; 6560 } 6561 6562 static void tcg_out_ld_helper_args(TCGContext *s, const TCGLabelQemuLdst *ldst, 6563 const TCGLdstHelperParam *parm) 6564 { 6565 const TCGHelperInfo *info; 6566 const TCGCallArgumentLoc *loc; 6567 TCGMovExtend mov[2]; 6568 unsigned next_arg, nmov; 6569 MemOp mop = get_memop(ldst->oi); 6570 6571 switch (mop & MO_SIZE) { 6572 case MO_8: 6573 case MO_16: 6574 case MO_32: 6575 info = &info_helper_ld32_mmu; 6576 break; 6577 case MO_64: 6578 info = &info_helper_ld64_mmu; 6579 break; 6580 case MO_128: 6581 info = &info_helper_ld128_mmu; 6582 break; 6583 default: 6584 g_assert_not_reached(); 6585 } 6586 6587 /* Defer env argument. */ 6588 next_arg = 1; 6589 6590 loc = &info->in[next_arg]; 6591 if (TCG_TARGET_REG_BITS == 32 && s->addr_type == TCG_TYPE_I32) { 6592 /* 6593 * 32-bit host with 32-bit guest: zero-extend the guest address 6594 * to 64-bits for the helper by storing the low part, then 6595 * load a zero for the high part. 6596 */ 6597 tcg_out_helper_add_mov(mov, loc + HOST_BIG_ENDIAN, 6598 TCG_TYPE_I32, TCG_TYPE_I32, 6599 ldst->addr_reg, -1); 6600 tcg_out_helper_load_slots(s, 1, mov, parm); 6601 6602 tcg_out_helper_load_imm(s, loc[!HOST_BIG_ENDIAN].arg_slot, 6603 TCG_TYPE_I32, 0, parm); 6604 next_arg += 2; 6605 } else { 6606 nmov = tcg_out_helper_add_mov(mov, loc, TCG_TYPE_I64, s->addr_type, 6607 ldst->addr_reg, -1); 6608 tcg_out_helper_load_slots(s, nmov, mov, parm); 6609 next_arg += nmov; 6610 } 6611 6612 switch (info->out_kind) { 6613 case TCG_CALL_RET_NORMAL: 6614 case TCG_CALL_RET_BY_VEC: 6615 break; 6616 case TCG_CALL_RET_BY_REF: 6617 /* 6618 * The return reference is in the first argument slot. 6619 * We need memory in which to return: re-use the top of stack. 6620 */ 6621 { 6622 int ofs_slot0 = TCG_TARGET_CALL_STACK_OFFSET; 6623 6624 if (arg_slot_reg_p(0)) { 6625 tcg_out_addi_ptr(s, tcg_target_call_iarg_regs[0], 6626 TCG_REG_CALL_STACK, ofs_slot0); 6627 } else { 6628 tcg_debug_assert(parm->ntmp != 0); 6629 tcg_out_addi_ptr(s, parm->tmp[0], 6630 TCG_REG_CALL_STACK, ofs_slot0); 6631 tcg_out_st(s, TCG_TYPE_PTR, parm->tmp[0], 6632 TCG_REG_CALL_STACK, ofs_slot0); 6633 } 6634 } 6635 break; 6636 default: 6637 g_assert_not_reached(); 6638 } 6639 6640 tcg_out_helper_load_common_args(s, ldst, parm, info, next_arg); 6641 } 6642 6643 static void tcg_out_ld_helper_ret(TCGContext *s, const TCGLabelQemuLdst *ldst, 6644 bool load_sign, 6645 const TCGLdstHelperParam *parm) 6646 { 6647 MemOp mop = get_memop(ldst->oi); 6648 TCGMovExtend mov[2]; 6649 int ofs_slot0; 6650 6651 switch (ldst->type) { 6652 case TCG_TYPE_I64: 6653 if (TCG_TARGET_REG_BITS == 32) { 6654 break; 6655 } 6656 /* fall through */ 6657 6658 case TCG_TYPE_I32: 6659 mov[0].dst = ldst->datalo_reg; 6660 mov[0].src = tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, 0); 6661 mov[0].dst_type = ldst->type; 6662 mov[0].src_type = TCG_TYPE_REG; 6663 6664 /* 6665 * If load_sign, then we allowed the helper to perform the 6666 * appropriate sign extension to tcg_target_ulong, and all 6667 * we need now is a plain move. 6668 * 6669 * If they do not, then we expect the relevant extension 6670 * instruction to be no more expensive than a move, and 6671 * we thus save the icache etc by only using one of two 6672 * helper functions. 6673 */ 6674 if (load_sign || !(mop & MO_SIGN)) { 6675 if (TCG_TARGET_REG_BITS == 32 || ldst->type == TCG_TYPE_I32) { 6676 mov[0].src_ext = MO_32; 6677 } else { 6678 mov[0].src_ext = MO_64; 6679 } 6680 } else { 6681 mov[0].src_ext = mop & MO_SSIZE; 6682 } 6683 tcg_out_movext1(s, mov); 6684 return; 6685 6686 case TCG_TYPE_I128: 6687 tcg_debug_assert(TCG_TARGET_REG_BITS == 64); 6688 ofs_slot0 = TCG_TARGET_CALL_STACK_OFFSET; 6689 switch (TCG_TARGET_CALL_RET_I128) { 6690 case TCG_CALL_RET_NORMAL: 6691 break; 6692 case TCG_CALL_RET_BY_VEC: 6693 tcg_out_st(s, TCG_TYPE_V128, 6694 tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0), 6695 TCG_REG_CALL_STACK, ofs_slot0); 6696 /* fall through */ 6697 case TCG_CALL_RET_BY_REF: 6698 tcg_out_ld(s, TCG_TYPE_I64, ldst->datalo_reg, 6699 TCG_REG_CALL_STACK, ofs_slot0 + 8 * HOST_BIG_ENDIAN); 6700 tcg_out_ld(s, TCG_TYPE_I64, ldst->datahi_reg, 6701 TCG_REG_CALL_STACK, ofs_slot0 + 8 * !HOST_BIG_ENDIAN); 6702 return; 6703 default: 6704 g_assert_not_reached(); 6705 } 6706 break; 6707 6708 default: 6709 g_assert_not_reached(); 6710 } 6711 6712 mov[0].dst = ldst->datalo_reg; 6713 mov[0].src = 6714 tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, HOST_BIG_ENDIAN); 6715 mov[0].dst_type = TCG_TYPE_REG; 6716 mov[0].src_type = TCG_TYPE_REG; 6717 mov[0].src_ext = TCG_TARGET_REG_BITS == 32 ? MO_32 : MO_64; 6718 6719 mov[1].dst = ldst->datahi_reg; 6720 mov[1].src = 6721 tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, !HOST_BIG_ENDIAN); 6722 mov[1].dst_type = TCG_TYPE_REG; 6723 mov[1].src_type = TCG_TYPE_REG; 6724 mov[1].src_ext = TCG_TARGET_REG_BITS == 32 ? MO_32 : MO_64; 6725 6726 tcg_out_movext2(s, mov, mov + 1, parm->ntmp ? parm->tmp[0] : -1); 6727 } 6728 6729 static void tcg_out_st_helper_args(TCGContext *s, const TCGLabelQemuLdst *ldst, 6730 const TCGLdstHelperParam *parm) 6731 { 6732 const TCGHelperInfo *info; 6733 const TCGCallArgumentLoc *loc; 6734 TCGMovExtend mov[4]; 6735 TCGType data_type; 6736 unsigned next_arg, nmov, n; 6737 MemOp mop = get_memop(ldst->oi); 6738 6739 switch (mop & MO_SIZE) { 6740 case MO_8: 6741 case MO_16: 6742 case MO_32: 6743 info = &info_helper_st32_mmu; 6744 data_type = TCG_TYPE_I32; 6745 break; 6746 case MO_64: 6747 info = &info_helper_st64_mmu; 6748 data_type = TCG_TYPE_I64; 6749 break; 6750 case MO_128: 6751 info = &info_helper_st128_mmu; 6752 data_type = TCG_TYPE_I128; 6753 break; 6754 default: 6755 g_assert_not_reached(); 6756 } 6757 6758 /* Defer env argument. */ 6759 next_arg = 1; 6760 nmov = 0; 6761 6762 /* Handle addr argument. */ 6763 loc = &info->in[next_arg]; 6764 tcg_debug_assert(s->addr_type <= TCG_TYPE_REG); 6765 if (TCG_TARGET_REG_BITS == 32) { 6766 /* 6767 * 32-bit host (and thus 32-bit guest): zero-extend the guest address 6768 * to 64-bits for the helper by storing the low part. Later, 6769 * after we have processed the register inputs, we will load a 6770 * zero for the high part. 6771 */ 6772 tcg_out_helper_add_mov(mov, loc + HOST_BIG_ENDIAN, 6773 TCG_TYPE_I32, TCG_TYPE_I32, 6774 ldst->addr_reg, -1); 6775 next_arg += 2; 6776 nmov += 1; 6777 } else { 6778 n = tcg_out_helper_add_mov(mov, loc, TCG_TYPE_I64, s->addr_type, 6779 ldst->addr_reg, -1); 6780 next_arg += n; 6781 nmov += n; 6782 } 6783 6784 /* Handle data argument. */ 6785 loc = &info->in[next_arg]; 6786 switch (loc->kind) { 6787 case TCG_CALL_ARG_NORMAL: 6788 case TCG_CALL_ARG_EXTEND_U: 6789 case TCG_CALL_ARG_EXTEND_S: 6790 n = tcg_out_helper_add_mov(mov + nmov, loc, data_type, ldst->type, 6791 ldst->datalo_reg, ldst->datahi_reg); 6792 next_arg += n; 6793 nmov += n; 6794 tcg_out_helper_load_slots(s, nmov, mov, parm); 6795 break; 6796 6797 case TCG_CALL_ARG_BY_REF: 6798 tcg_debug_assert(TCG_TARGET_REG_BITS == 64); 6799 tcg_debug_assert(data_type == TCG_TYPE_I128); 6800 tcg_out_st(s, TCG_TYPE_I64, 6801 HOST_BIG_ENDIAN ? ldst->datahi_reg : ldst->datalo_reg, 6802 TCG_REG_CALL_STACK, arg_slot_stk_ofs(loc[0].ref_slot)); 6803 tcg_out_st(s, TCG_TYPE_I64, 6804 HOST_BIG_ENDIAN ? ldst->datalo_reg : ldst->datahi_reg, 6805 TCG_REG_CALL_STACK, arg_slot_stk_ofs(loc[1].ref_slot)); 6806 6807 tcg_out_helper_load_slots(s, nmov, mov, parm); 6808 6809 if (arg_slot_reg_p(loc->arg_slot)) { 6810 tcg_out_addi_ptr(s, tcg_target_call_iarg_regs[loc->arg_slot], 6811 TCG_REG_CALL_STACK, 6812 arg_slot_stk_ofs(loc->ref_slot)); 6813 } else { 6814 tcg_debug_assert(parm->ntmp != 0); 6815 tcg_out_addi_ptr(s, parm->tmp[0], TCG_REG_CALL_STACK, 6816 arg_slot_stk_ofs(loc->ref_slot)); 6817 tcg_out_st(s, TCG_TYPE_PTR, parm->tmp[0], 6818 TCG_REG_CALL_STACK, arg_slot_stk_ofs(loc->arg_slot)); 6819 } 6820 next_arg += 2; 6821 break; 6822 6823 default: 6824 g_assert_not_reached(); 6825 } 6826 6827 if (TCG_TARGET_REG_BITS == 32) { 6828 /* Zero extend the address by loading a zero for the high part. */ 6829 loc = &info->in[1 + !HOST_BIG_ENDIAN]; 6830 tcg_out_helper_load_imm(s, loc->arg_slot, TCG_TYPE_I32, 0, parm); 6831 } 6832 6833 tcg_out_helper_load_common_args(s, ldst, parm, info, next_arg); 6834 } 6835 6836 int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start) 6837 { 6838 int i, start_words, num_insns; 6839 TCGOp *op; 6840 6841 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP) 6842 && qemu_log_in_addr_range(pc_start))) { 6843 FILE *logfile = qemu_log_trylock(); 6844 if (logfile) { 6845 fprintf(logfile, "OP:\n"); 6846 tcg_dump_ops(s, logfile, false); 6847 fprintf(logfile, "\n"); 6848 qemu_log_unlock(logfile); 6849 } 6850 } 6851 6852 #ifdef CONFIG_DEBUG_TCG 6853 /* Ensure all labels referenced have been emitted. */ 6854 { 6855 TCGLabel *l; 6856 bool error = false; 6857 6858 QSIMPLEQ_FOREACH(l, &s->labels, next) { 6859 if (unlikely(!l->present) && !QSIMPLEQ_EMPTY(&l->branches)) { 6860 qemu_log_mask(CPU_LOG_TB_OP, 6861 "$L%d referenced but not present.\n", l->id); 6862 error = true; 6863 } 6864 } 6865 assert(!error); 6866 } 6867 #endif 6868 6869 /* Do not reuse any EBB that may be allocated within the TB. */ 6870 tcg_temp_ebb_reset_freed(s); 6871 6872 tcg_optimize(s); 6873 6874 reachable_code_pass(s); 6875 liveness_pass_0(s); 6876 liveness_pass_1(s); 6877 6878 if (s->nb_indirects > 0) { 6879 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND) 6880 && qemu_log_in_addr_range(pc_start))) { 6881 FILE *logfile = qemu_log_trylock(); 6882 if (logfile) { 6883 fprintf(logfile, "OP before indirect lowering:\n"); 6884 tcg_dump_ops(s, logfile, false); 6885 fprintf(logfile, "\n"); 6886 qemu_log_unlock(logfile); 6887 } 6888 } 6889 6890 /* Replace indirect temps with direct temps. */ 6891 if (liveness_pass_2(s)) { 6892 /* If changes were made, re-run liveness. */ 6893 liveness_pass_1(s); 6894 } 6895 } 6896 6897 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT) 6898 && qemu_log_in_addr_range(pc_start))) { 6899 FILE *logfile = qemu_log_trylock(); 6900 if (logfile) { 6901 fprintf(logfile, "OP after optimization and liveness analysis:\n"); 6902 tcg_dump_ops(s, logfile, true); 6903 fprintf(logfile, "\n"); 6904 qemu_log_unlock(logfile); 6905 } 6906 } 6907 6908 /* Initialize goto_tb jump offsets. */ 6909 tb->jmp_reset_offset[0] = TB_JMP_OFFSET_INVALID; 6910 tb->jmp_reset_offset[1] = TB_JMP_OFFSET_INVALID; 6911 tb->jmp_insn_offset[0] = TB_JMP_OFFSET_INVALID; 6912 tb->jmp_insn_offset[1] = TB_JMP_OFFSET_INVALID; 6913 6914 tcg_reg_alloc_start(s); 6915 6916 /* 6917 * Reset the buffer pointers when restarting after overflow. 6918 * TODO: Move this into translate-all.c with the rest of the 6919 * buffer management. Having only this done here is confusing. 6920 */ 6921 s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr); 6922 s->code_ptr = s->code_buf; 6923 s->data_gen_ptr = NULL; 6924 6925 QSIMPLEQ_INIT(&s->ldst_labels); 6926 s->pool_labels = NULL; 6927 6928 start_words = s->insn_start_words; 6929 s->gen_insn_data = 6930 tcg_malloc(sizeof(uint64_t) * s->gen_tb->icount * start_words); 6931 6932 tcg_out_tb_start(s); 6933 6934 num_insns = -1; 6935 s->carry_live = false; 6936 QTAILQ_FOREACH(op, &s->ops, link) { 6937 TCGOpcode opc = op->opc; 6938 6939 switch (opc) { 6940 case INDEX_op_extrl_i64_i32: 6941 assert(TCG_TARGET_REG_BITS == 64); 6942 /* 6943 * If TCG_TYPE_I32 is represented in some canonical form, 6944 * e.g. zero or sign-extended, then emit as a unary op. 6945 * Otherwise we can treat this as a plain move. 6946 * If the output dies, treat this as a plain move, because 6947 * this will be implemented with a store. 6948 */ 6949 if (TCG_TARGET_HAS_extr_i64_i32) { 6950 TCGLifeData arg_life = op->life; 6951 if (!IS_DEAD_ARG(0)) { 6952 goto do_default; 6953 } 6954 } 6955 /* fall through */ 6956 case INDEX_op_mov: 6957 case INDEX_op_mov_vec: 6958 tcg_reg_alloc_mov(s, op); 6959 break; 6960 case INDEX_op_dup_vec: 6961 tcg_reg_alloc_dup(s, op); 6962 break; 6963 case INDEX_op_insn_start: 6964 assert_carry_dead(s); 6965 if (num_insns >= 0) { 6966 size_t off = tcg_current_code_size(s); 6967 s->gen_insn_end_off[num_insns] = off; 6968 /* Assert that we do not overflow our stored offset. */ 6969 assert(s->gen_insn_end_off[num_insns] == off); 6970 } 6971 num_insns++; 6972 for (i = 0; i < start_words; ++i) { 6973 s->gen_insn_data[num_insns * start_words + i] = 6974 tcg_get_insn_start_param(op, i); 6975 } 6976 break; 6977 case INDEX_op_discard: 6978 temp_dead(s, arg_temp(op->args[0])); 6979 break; 6980 case INDEX_op_set_label: 6981 tcg_reg_alloc_bb_end(s, s->reserved_regs); 6982 tcg_out_label(s, arg_label(op->args[0])); 6983 break; 6984 case INDEX_op_call: 6985 assert_carry_dead(s); 6986 tcg_reg_alloc_call(s, op); 6987 break; 6988 case INDEX_op_exit_tb: 6989 tcg_out_exit_tb(s, op->args[0]); 6990 break; 6991 case INDEX_op_goto_tb: 6992 tcg_out_goto_tb(s, op->args[0]); 6993 break; 6994 case INDEX_op_br: 6995 tcg_out_br(s, arg_label(op->args[0])); 6996 break; 6997 case INDEX_op_mb: 6998 tcg_out_mb(s, op->args[0]); 6999 break; 7000 case INDEX_op_dup2_vec: 7001 if (tcg_reg_alloc_dup2(s, op)) { 7002 break; 7003 } 7004 /* fall through */ 7005 default: 7006 do_default: 7007 /* Sanity check that we've not introduced any unhandled opcodes. */ 7008 tcg_debug_assert(tcg_op_supported(opc, TCGOP_TYPE(op), 7009 TCGOP_FLAGS(op))); 7010 /* Note: in order to speed up the code, it would be much 7011 faster to have specialized register allocator functions for 7012 some common argument patterns */ 7013 tcg_reg_alloc_op(s, op); 7014 break; 7015 } 7016 /* Test for (pending) buffer overflow. The assumption is that any 7017 one operation beginning below the high water mark cannot overrun 7018 the buffer completely. Thus we can test for overflow after 7019 generating code without having to check during generation. */ 7020 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) { 7021 return -1; 7022 } 7023 /* Test for TB overflow, as seen by gen_insn_end_off. */ 7024 if (unlikely(tcg_current_code_size(s) > UINT16_MAX)) { 7025 return -2; 7026 } 7027 } 7028 assert_carry_dead(s); 7029 7030 tcg_debug_assert(num_insns + 1 == s->gen_tb->icount); 7031 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s); 7032 7033 /* Generate TB finalization at the end of block */ 7034 i = tcg_out_ldst_finalize(s); 7035 if (i < 0) { 7036 return i; 7037 } 7038 i = tcg_out_pool_finalize(s); 7039 if (i < 0) { 7040 return i; 7041 } 7042 if (!tcg_resolve_relocs(s)) { 7043 return -2; 7044 } 7045 7046 #ifndef CONFIG_TCG_INTERPRETER 7047 /* flush instruction cache */ 7048 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf), 7049 (uintptr_t)s->code_buf, 7050 tcg_ptr_byte_diff(s->code_ptr, s->code_buf)); 7051 #endif 7052 7053 return tcg_current_code_size(s); 7054 } 7055 7056 #ifdef ELF_HOST_MACHINE 7057 /* In order to use this feature, the backend needs to do three things: 7058 7059 (1) Define ELF_HOST_MACHINE to indicate both what value to 7060 put into the ELF image and to indicate support for the feature. 7061 7062 (2) Define tcg_register_jit. This should create a buffer containing 7063 the contents of a .debug_frame section that describes the post- 7064 prologue unwind info for the tcg machine. 7065 7066 (3) Call tcg_register_jit_int, with the constructed .debug_frame. 7067 */ 7068 7069 /* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */ 7070 typedef enum { 7071 JIT_NOACTION = 0, 7072 JIT_REGISTER_FN, 7073 JIT_UNREGISTER_FN 7074 } jit_actions_t; 7075 7076 struct jit_code_entry { 7077 struct jit_code_entry *next_entry; 7078 struct jit_code_entry *prev_entry; 7079 const void *symfile_addr; 7080 uint64_t symfile_size; 7081 }; 7082 7083 struct jit_descriptor { 7084 uint32_t version; 7085 uint32_t action_flag; 7086 struct jit_code_entry *relevant_entry; 7087 struct jit_code_entry *first_entry; 7088 }; 7089 7090 void __jit_debug_register_code(void) __attribute__((noinline)); 7091 void __jit_debug_register_code(void) 7092 { 7093 asm(""); 7094 } 7095 7096 /* Must statically initialize the version, because GDB may check 7097 the version before we can set it. */ 7098 struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 }; 7099 7100 /* End GDB interface. */ 7101 7102 static int find_string(const char *strtab, const char *str) 7103 { 7104 const char *p = strtab + 1; 7105 7106 while (1) { 7107 if (strcmp(p, str) == 0) { 7108 return p - strtab; 7109 } 7110 p += strlen(p) + 1; 7111 } 7112 } 7113 7114 static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size, 7115 const void *debug_frame, 7116 size_t debug_frame_size) 7117 { 7118 struct __attribute__((packed)) DebugInfo { 7119 uint32_t len; 7120 uint16_t version; 7121 uint32_t abbrev; 7122 uint8_t ptr_size; 7123 uint8_t cu_die; 7124 uint16_t cu_lang; 7125 uintptr_t cu_low_pc; 7126 uintptr_t cu_high_pc; 7127 uint8_t fn_die; 7128 char fn_name[16]; 7129 uintptr_t fn_low_pc; 7130 uintptr_t fn_high_pc; 7131 uint8_t cu_eoc; 7132 }; 7133 7134 struct ElfImage { 7135 ElfW(Ehdr) ehdr; 7136 ElfW(Phdr) phdr; 7137 ElfW(Shdr) shdr[7]; 7138 ElfW(Sym) sym[2]; 7139 struct DebugInfo di; 7140 uint8_t da[24]; 7141 char str[80]; 7142 }; 7143 7144 struct ElfImage *img; 7145 7146 static const struct ElfImage img_template = { 7147 .ehdr = { 7148 .e_ident[EI_MAG0] = ELFMAG0, 7149 .e_ident[EI_MAG1] = ELFMAG1, 7150 .e_ident[EI_MAG2] = ELFMAG2, 7151 .e_ident[EI_MAG3] = ELFMAG3, 7152 .e_ident[EI_CLASS] = ELF_CLASS, 7153 .e_ident[EI_DATA] = ELF_DATA, 7154 .e_ident[EI_VERSION] = EV_CURRENT, 7155 .e_type = ET_EXEC, 7156 .e_machine = ELF_HOST_MACHINE, 7157 .e_version = EV_CURRENT, 7158 .e_phoff = offsetof(struct ElfImage, phdr), 7159 .e_shoff = offsetof(struct ElfImage, shdr), 7160 .e_ehsize = sizeof(ElfW(Shdr)), 7161 .e_phentsize = sizeof(ElfW(Phdr)), 7162 .e_phnum = 1, 7163 .e_shentsize = sizeof(ElfW(Shdr)), 7164 .e_shnum = ARRAY_SIZE(img->shdr), 7165 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1, 7166 #ifdef ELF_HOST_FLAGS 7167 .e_flags = ELF_HOST_FLAGS, 7168 #endif 7169 #ifdef ELF_OSABI 7170 .e_ident[EI_OSABI] = ELF_OSABI, 7171 #endif 7172 }, 7173 .phdr = { 7174 .p_type = PT_LOAD, 7175 .p_flags = PF_X, 7176 }, 7177 .shdr = { 7178 [0] = { .sh_type = SHT_NULL }, 7179 /* Trick: The contents of code_gen_buffer are not present in 7180 this fake ELF file; that got allocated elsewhere. Therefore 7181 we mark .text as SHT_NOBITS (similar to .bss) so that readers 7182 will not look for contents. We can record any address. */ 7183 [1] = { /* .text */ 7184 .sh_type = SHT_NOBITS, 7185 .sh_flags = SHF_EXECINSTR | SHF_ALLOC, 7186 }, 7187 [2] = { /* .debug_info */ 7188 .sh_type = SHT_PROGBITS, 7189 .sh_offset = offsetof(struct ElfImage, di), 7190 .sh_size = sizeof(struct DebugInfo), 7191 }, 7192 [3] = { /* .debug_abbrev */ 7193 .sh_type = SHT_PROGBITS, 7194 .sh_offset = offsetof(struct ElfImage, da), 7195 .sh_size = sizeof(img->da), 7196 }, 7197 [4] = { /* .debug_frame */ 7198 .sh_type = SHT_PROGBITS, 7199 .sh_offset = sizeof(struct ElfImage), 7200 }, 7201 [5] = { /* .symtab */ 7202 .sh_type = SHT_SYMTAB, 7203 .sh_offset = offsetof(struct ElfImage, sym), 7204 .sh_size = sizeof(img->sym), 7205 .sh_info = 1, 7206 .sh_link = ARRAY_SIZE(img->shdr) - 1, 7207 .sh_entsize = sizeof(ElfW(Sym)), 7208 }, 7209 [6] = { /* .strtab */ 7210 .sh_type = SHT_STRTAB, 7211 .sh_offset = offsetof(struct ElfImage, str), 7212 .sh_size = sizeof(img->str), 7213 } 7214 }, 7215 .sym = { 7216 [1] = { /* code_gen_buffer */ 7217 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC), 7218 .st_shndx = 1, 7219 } 7220 }, 7221 .di = { 7222 .len = sizeof(struct DebugInfo) - 4, 7223 .version = 2, 7224 .ptr_size = sizeof(void *), 7225 .cu_die = 1, 7226 .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */ 7227 .fn_die = 2, 7228 .fn_name = "code_gen_buffer" 7229 }, 7230 .da = { 7231 1, /* abbrev number (the cu) */ 7232 0x11, 1, /* DW_TAG_compile_unit, has children */ 7233 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */ 7234 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 7235 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 7236 0, 0, /* end of abbrev */ 7237 2, /* abbrev number (the fn) */ 7238 0x2e, 0, /* DW_TAG_subprogram, no children */ 7239 0x3, 0x8, /* DW_AT_name, DW_FORM_string */ 7240 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 7241 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 7242 0, 0, /* end of abbrev */ 7243 0 /* no more abbrev */ 7244 }, 7245 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0" 7246 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer", 7247 }; 7248 7249 /* We only need a single jit entry; statically allocate it. */ 7250 static struct jit_code_entry one_entry; 7251 7252 uintptr_t buf = (uintptr_t)buf_ptr; 7253 size_t img_size = sizeof(struct ElfImage) + debug_frame_size; 7254 DebugFrameHeader *dfh; 7255 7256 img = g_malloc(img_size); 7257 *img = img_template; 7258 7259 img->phdr.p_vaddr = buf; 7260 img->phdr.p_paddr = buf; 7261 img->phdr.p_memsz = buf_size; 7262 7263 img->shdr[1].sh_name = find_string(img->str, ".text"); 7264 img->shdr[1].sh_addr = buf; 7265 img->shdr[1].sh_size = buf_size; 7266 7267 img->shdr[2].sh_name = find_string(img->str, ".debug_info"); 7268 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev"); 7269 7270 img->shdr[4].sh_name = find_string(img->str, ".debug_frame"); 7271 img->shdr[4].sh_size = debug_frame_size; 7272 7273 img->shdr[5].sh_name = find_string(img->str, ".symtab"); 7274 img->shdr[6].sh_name = find_string(img->str, ".strtab"); 7275 7276 img->sym[1].st_name = find_string(img->str, "code_gen_buffer"); 7277 img->sym[1].st_value = buf; 7278 img->sym[1].st_size = buf_size; 7279 7280 img->di.cu_low_pc = buf; 7281 img->di.cu_high_pc = buf + buf_size; 7282 img->di.fn_low_pc = buf; 7283 img->di.fn_high_pc = buf + buf_size; 7284 7285 dfh = (DebugFrameHeader *)(img + 1); 7286 memcpy(dfh, debug_frame, debug_frame_size); 7287 dfh->fde.func_start = buf; 7288 dfh->fde.func_len = buf_size; 7289 7290 #ifdef DEBUG_JIT 7291 /* Enable this block to be able to debug the ELF image file creation. 7292 One can use readelf, objdump, or other inspection utilities. */ 7293 { 7294 g_autofree char *jit = g_strdup_printf("%s/qemu.jit", g_get_tmp_dir()); 7295 FILE *f = fopen(jit, "w+b"); 7296 if (f) { 7297 if (fwrite(img, img_size, 1, f) != img_size) { 7298 /* Avoid stupid unused return value warning for fwrite. */ 7299 } 7300 fclose(f); 7301 } 7302 } 7303 #endif 7304 7305 one_entry.symfile_addr = img; 7306 one_entry.symfile_size = img_size; 7307 7308 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN; 7309 __jit_debug_descriptor.relevant_entry = &one_entry; 7310 __jit_debug_descriptor.first_entry = &one_entry; 7311 __jit_debug_register_code(); 7312 } 7313 #else 7314 /* No support for the feature. Provide the entry point expected by exec.c, 7315 and implement the internal function we declared earlier. */ 7316 7317 static void tcg_register_jit_int(const void *buf, size_t size, 7318 const void *debug_frame, 7319 size_t debug_frame_size) 7320 { 7321 } 7322 7323 void tcg_register_jit(const void *buf, size_t buf_size) 7324 { 7325 } 7326 #endif /* ELF_HOST_MACHINE */ 7327 7328 #if !TCG_TARGET_MAYBE_vec 7329 void tcg_expand_vec_op(TCGOpcode o, TCGType t, unsigned e, TCGArg a0, ...) 7330 { 7331 g_assert_not_reached(); 7332 } 7333 #endif 7334