1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 #include "qemu/osdep.h" 26 27 /* Define to jump the ELF file used to communicate with GDB. */ 28 #undef DEBUG_JIT 29 30 #include "qemu/error-report.h" 31 #include "qemu/cutils.h" 32 #include "qemu/host-utils.h" 33 #include "qemu/qemu-print.h" 34 #include "qemu/cacheflush.h" 35 #include "qemu/cacheinfo.h" 36 #include "qemu/timer.h" 37 #include "exec/translation-block.h" 38 #include "exec/tlb-common.h" 39 #include "tcg/startup.h" 40 #include "tcg/tcg-op-common.h" 41 42 #if UINTPTR_MAX == UINT32_MAX 43 # define ELF_CLASS ELFCLASS32 44 #else 45 # define ELF_CLASS ELFCLASS64 46 #endif 47 #if HOST_BIG_ENDIAN 48 # define ELF_DATA ELFDATA2MSB 49 #else 50 # define ELF_DATA ELFDATA2LSB 51 #endif 52 53 #include "elf.h" 54 #include "exec/log.h" 55 #include "tcg/tcg-ldst.h" 56 #include "tcg/tcg-temp-internal.h" 57 #include "tcg-internal.h" 58 #include "tcg/perf.h" 59 #include "tcg-has.h" 60 #ifdef CONFIG_USER_ONLY 61 #include "user/guest-base.h" 62 #endif 63 64 /* Forward declarations for functions declared in tcg-target.c.inc and 65 used here. */ 66 static void tcg_target_init(TCGContext *s); 67 static void tcg_target_qemu_prologue(TCGContext *s); 68 static bool patch_reloc(tcg_insn_unit *code_ptr, int type, 69 intptr_t value, intptr_t addend); 70 static void tcg_out_nop_fill(tcg_insn_unit *p, int count); 71 72 typedef struct TCGLabelQemuLdst TCGLabelQemuLdst; 73 static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l); 74 static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l); 75 76 /* The CIE and FDE header definitions will be common to all hosts. */ 77 typedef struct { 78 uint32_t len __attribute__((aligned((sizeof(void *))))); 79 uint32_t id; 80 uint8_t version; 81 char augmentation[1]; 82 uint8_t code_align; 83 uint8_t data_align; 84 uint8_t return_column; 85 } DebugFrameCIE; 86 87 typedef struct QEMU_PACKED { 88 uint32_t len __attribute__((aligned((sizeof(void *))))); 89 uint32_t cie_offset; 90 uintptr_t func_start; 91 uintptr_t func_len; 92 } DebugFrameFDEHeader; 93 94 typedef struct QEMU_PACKED { 95 DebugFrameCIE cie; 96 DebugFrameFDEHeader fde; 97 } DebugFrameHeader; 98 99 struct TCGLabelQemuLdst { 100 bool is_ld; /* qemu_ld: true, qemu_st: false */ 101 MemOpIdx oi; 102 TCGType type; /* result type of a load */ 103 TCGReg addr_reg; /* reg index for guest virtual addr */ 104 TCGReg datalo_reg; /* reg index for low word to be loaded or stored */ 105 TCGReg datahi_reg; /* reg index for high word to be loaded or stored */ 106 const tcg_insn_unit *raddr; /* addr of the next IR of qemu_ld/st IR */ 107 tcg_insn_unit *label_ptr[2]; /* label pointers to be updated */ 108 QSIMPLEQ_ENTRY(TCGLabelQemuLdst) next; 109 }; 110 111 static void tcg_register_jit_int(const void *buf, size_t size, 112 const void *debug_frame, 113 size_t debug_frame_size) 114 __attribute__((unused)); 115 116 /* Forward declarations for functions declared and used in tcg-target.c.inc. */ 117 static void tcg_out_tb_start(TCGContext *s); 118 static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1, 119 intptr_t arg2); 120 static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); 121 static void tcg_out_movi(TCGContext *s, TCGType type, 122 TCGReg ret, tcg_target_long arg); 123 static void tcg_out_ext8s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); 124 static void tcg_out_ext16s(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); 125 static void tcg_out_ext8u(TCGContext *s, TCGReg ret, TCGReg arg); 126 static void tcg_out_ext16u(TCGContext *s, TCGReg ret, TCGReg arg); 127 static void tcg_out_ext32s(TCGContext *s, TCGReg ret, TCGReg arg); 128 static void tcg_out_ext32u(TCGContext *s, TCGReg ret, TCGReg arg); 129 static void tcg_out_exts_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg); 130 static void tcg_out_extu_i32_i64(TCGContext *s, TCGReg ret, TCGReg arg); 131 static void tcg_out_extrl_i64_i32(TCGContext *s, TCGReg ret, TCGReg arg); 132 static void tcg_out_addi_ptr(TCGContext *s, TCGReg, TCGReg, tcg_target_long); 133 static bool tcg_out_xchg(TCGContext *s, TCGType type, TCGReg r1, TCGReg r2); 134 static void tcg_out_exit_tb(TCGContext *s, uintptr_t arg); 135 static void tcg_out_goto_tb(TCGContext *s, int which); 136 static void tcg_out_goto_ptr(TCGContext *s, TCGReg dest); 137 static void tcg_out_mb(TCGContext *s, unsigned bar); 138 static void tcg_out_br(TCGContext *s, TCGLabel *l); 139 static void tcg_out_set_carry(TCGContext *s); 140 static void tcg_out_set_borrow(TCGContext *s); 141 #if TCG_TARGET_MAYBE_vec 142 static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece, 143 TCGReg dst, TCGReg src); 144 static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece, 145 TCGReg dst, TCGReg base, intptr_t offset); 146 static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, 147 TCGReg dst, int64_t arg); 148 static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, 149 unsigned vecl, unsigned vece, 150 const TCGArg args[TCG_MAX_OP_ARGS], 151 const int const_args[TCG_MAX_OP_ARGS]); 152 #else 153 static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece, 154 TCGReg dst, TCGReg src) 155 { 156 g_assert_not_reached(); 157 } 158 static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece, 159 TCGReg dst, TCGReg base, intptr_t offset) 160 { 161 g_assert_not_reached(); 162 } 163 static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, 164 TCGReg dst, int64_t arg) 165 { 166 g_assert_not_reached(); 167 } 168 static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, 169 unsigned vecl, unsigned vece, 170 const TCGArg args[TCG_MAX_OP_ARGS], 171 const int const_args[TCG_MAX_OP_ARGS]) 172 { 173 g_assert_not_reached(); 174 } 175 int tcg_can_emit_vec_op(TCGOpcode o, TCGType t, unsigned ve) 176 { 177 return 0; 178 } 179 #endif 180 static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1, 181 intptr_t arg2); 182 static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, 183 TCGReg base, intptr_t ofs); 184 static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target, 185 const TCGHelperInfo *info); 186 static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot); 187 static bool tcg_target_const_match(int64_t val, int ct, 188 TCGType type, TCGCond cond, int vece); 189 190 #ifndef CONFIG_USER_ONLY 191 #define guest_base ({ qemu_build_not_reached(); (uintptr_t)0; }) 192 #endif 193 194 typedef struct TCGLdstHelperParam { 195 TCGReg (*ra_gen)(TCGContext *s, const TCGLabelQemuLdst *l, int arg_reg); 196 unsigned ntmp; 197 int tmp[3]; 198 } TCGLdstHelperParam; 199 200 static void tcg_out_ld_helper_args(TCGContext *s, const TCGLabelQemuLdst *l, 201 const TCGLdstHelperParam *p) 202 __attribute__((unused)); 203 static void tcg_out_ld_helper_ret(TCGContext *s, const TCGLabelQemuLdst *l, 204 bool load_sign, const TCGLdstHelperParam *p) 205 __attribute__((unused)); 206 static void tcg_out_st_helper_args(TCGContext *s, const TCGLabelQemuLdst *l, 207 const TCGLdstHelperParam *p) 208 __attribute__((unused)); 209 210 static void * const qemu_ld_helpers[MO_SSIZE + 1] __attribute__((unused)) = { 211 [MO_UB] = helper_ldub_mmu, 212 [MO_SB] = helper_ldsb_mmu, 213 [MO_UW] = helper_lduw_mmu, 214 [MO_SW] = helper_ldsw_mmu, 215 [MO_UL] = helper_ldul_mmu, 216 [MO_UQ] = helper_ldq_mmu, 217 #if TCG_TARGET_REG_BITS == 64 218 [MO_SL] = helper_ldsl_mmu, 219 [MO_128] = helper_ld16_mmu, 220 #endif 221 }; 222 223 static void * const qemu_st_helpers[MO_SIZE + 1] __attribute__((unused)) = { 224 [MO_8] = helper_stb_mmu, 225 [MO_16] = helper_stw_mmu, 226 [MO_32] = helper_stl_mmu, 227 [MO_64] = helper_stq_mmu, 228 #if TCG_TARGET_REG_BITS == 64 229 [MO_128] = helper_st16_mmu, 230 #endif 231 }; 232 233 typedef struct { 234 MemOp atom; /* lg2 bits of atomicity required */ 235 MemOp align; /* lg2 bits of alignment to use */ 236 } TCGAtomAlign; 237 238 static TCGAtomAlign atom_and_align_for_opc(TCGContext *s, MemOp opc, 239 MemOp host_atom, bool allow_two_ops) 240 __attribute__((unused)); 241 242 #ifdef CONFIG_USER_ONLY 243 bool tcg_use_softmmu; 244 #endif 245 246 TCGContext tcg_init_ctx; 247 __thread TCGContext *tcg_ctx; 248 249 TCGContext **tcg_ctxs; 250 unsigned int tcg_cur_ctxs; 251 unsigned int tcg_max_ctxs; 252 TCGv_env tcg_env; 253 const void *tcg_code_gen_epilogue; 254 uintptr_t tcg_splitwx_diff; 255 256 #ifndef CONFIG_TCG_INTERPRETER 257 tcg_prologue_fn *tcg_qemu_tb_exec; 258 #endif 259 260 static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT]; 261 static TCGRegSet tcg_target_call_clobber_regs; 262 263 #if TCG_TARGET_INSN_UNIT_SIZE == 1 264 static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v) 265 { 266 *s->code_ptr++ = v; 267 } 268 269 static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p, 270 uint8_t v) 271 { 272 *p = v; 273 } 274 #endif 275 276 #if TCG_TARGET_INSN_UNIT_SIZE <= 2 277 static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v) 278 { 279 if (TCG_TARGET_INSN_UNIT_SIZE == 2) { 280 *s->code_ptr++ = v; 281 } else { 282 tcg_insn_unit *p = s->code_ptr; 283 memcpy(p, &v, sizeof(v)); 284 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE); 285 } 286 } 287 288 static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p, 289 uint16_t v) 290 { 291 if (TCG_TARGET_INSN_UNIT_SIZE == 2) { 292 *p = v; 293 } else { 294 memcpy(p, &v, sizeof(v)); 295 } 296 } 297 #endif 298 299 #if TCG_TARGET_INSN_UNIT_SIZE <= 4 300 static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v) 301 { 302 if (TCG_TARGET_INSN_UNIT_SIZE == 4) { 303 *s->code_ptr++ = v; 304 } else { 305 tcg_insn_unit *p = s->code_ptr; 306 memcpy(p, &v, sizeof(v)); 307 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE); 308 } 309 } 310 311 static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p, 312 uint32_t v) 313 { 314 if (TCG_TARGET_INSN_UNIT_SIZE == 4) { 315 *p = v; 316 } else { 317 memcpy(p, &v, sizeof(v)); 318 } 319 } 320 #endif 321 322 #if TCG_TARGET_INSN_UNIT_SIZE <= 8 323 static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v) 324 { 325 if (TCG_TARGET_INSN_UNIT_SIZE == 8) { 326 *s->code_ptr++ = v; 327 } else { 328 tcg_insn_unit *p = s->code_ptr; 329 memcpy(p, &v, sizeof(v)); 330 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE); 331 } 332 } 333 334 static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p, 335 uint64_t v) 336 { 337 if (TCG_TARGET_INSN_UNIT_SIZE == 8) { 338 *p = v; 339 } else { 340 memcpy(p, &v, sizeof(v)); 341 } 342 } 343 #endif 344 345 /* label relocation processing */ 346 347 static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type, 348 TCGLabel *l, intptr_t addend) 349 { 350 TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation)); 351 352 r->type = type; 353 r->ptr = code_ptr; 354 r->addend = addend; 355 QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next); 356 } 357 358 static void tcg_out_label(TCGContext *s, TCGLabel *l) 359 { 360 tcg_debug_assert(!l->has_value); 361 l->has_value = 1; 362 l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr); 363 } 364 365 TCGLabel *gen_new_label(void) 366 { 367 TCGContext *s = tcg_ctx; 368 TCGLabel *l = tcg_malloc(sizeof(TCGLabel)); 369 370 memset(l, 0, sizeof(TCGLabel)); 371 l->id = s->nb_labels++; 372 QSIMPLEQ_INIT(&l->branches); 373 QSIMPLEQ_INIT(&l->relocs); 374 375 QSIMPLEQ_INSERT_TAIL(&s->labels, l, next); 376 377 return l; 378 } 379 380 static bool tcg_resolve_relocs(TCGContext *s) 381 { 382 TCGLabel *l; 383 384 QSIMPLEQ_FOREACH(l, &s->labels, next) { 385 TCGRelocation *r; 386 uintptr_t value = l->u.value; 387 388 QSIMPLEQ_FOREACH(r, &l->relocs, next) { 389 if (!patch_reloc(r->ptr, r->type, value, r->addend)) { 390 return false; 391 } 392 } 393 } 394 return true; 395 } 396 397 static void set_jmp_reset_offset(TCGContext *s, int which) 398 { 399 /* 400 * We will check for overflow at the end of the opcode loop in 401 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX. 402 */ 403 s->gen_tb->jmp_reset_offset[which] = tcg_current_code_size(s); 404 } 405 406 static void G_GNUC_UNUSED set_jmp_insn_offset(TCGContext *s, int which) 407 { 408 /* 409 * We will check for overflow at the end of the opcode loop in 410 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX. 411 */ 412 s->gen_tb->jmp_insn_offset[which] = tcg_current_code_size(s); 413 } 414 415 static uintptr_t G_GNUC_UNUSED get_jmp_target_addr(TCGContext *s, int which) 416 { 417 /* 418 * Return the read-execute version of the pointer, for the benefit 419 * of any pc-relative addressing mode. 420 */ 421 return (uintptr_t)tcg_splitwx_to_rx(&s->gen_tb->jmp_target_addr[which]); 422 } 423 424 static int __attribute__((unused)) 425 tlb_mask_table_ofs(TCGContext *s, int which) 426 { 427 return (offsetof(CPUNegativeOffsetState, tlb.f[which]) - 428 sizeof(CPUNegativeOffsetState)); 429 } 430 431 /* Signal overflow, starting over with fewer guest insns. */ 432 static G_NORETURN 433 void tcg_raise_tb_overflow(TCGContext *s) 434 { 435 siglongjmp(s->jmp_trans, -2); 436 } 437 438 /* 439 * Used by tcg_out_movext{1,2} to hold the arguments for tcg_out_movext. 440 * By the time we arrive at tcg_out_movext1, @dst is always a TCGReg. 441 * 442 * However, tcg_out_helper_load_slots reuses this field to hold an 443 * argument slot number (which may designate a argument register or an 444 * argument stack slot), converting to TCGReg once all arguments that 445 * are destined for the stack are processed. 446 */ 447 typedef struct TCGMovExtend { 448 unsigned dst; 449 TCGReg src; 450 TCGType dst_type; 451 TCGType src_type; 452 MemOp src_ext; 453 } TCGMovExtend; 454 455 /** 456 * tcg_out_movext -- move and extend 457 * @s: tcg context 458 * @dst_type: integral type for destination 459 * @dst: destination register 460 * @src_type: integral type for source 461 * @src_ext: extension to apply to source 462 * @src: source register 463 * 464 * Move or extend @src into @dst, depending on @src_ext and the types. 465 */ 466 static void tcg_out_movext(TCGContext *s, TCGType dst_type, TCGReg dst, 467 TCGType src_type, MemOp src_ext, TCGReg src) 468 { 469 switch (src_ext) { 470 case MO_UB: 471 tcg_out_ext8u(s, dst, src); 472 break; 473 case MO_SB: 474 tcg_out_ext8s(s, dst_type, dst, src); 475 break; 476 case MO_UW: 477 tcg_out_ext16u(s, dst, src); 478 break; 479 case MO_SW: 480 tcg_out_ext16s(s, dst_type, dst, src); 481 break; 482 case MO_UL: 483 case MO_SL: 484 if (dst_type == TCG_TYPE_I32) { 485 if (src_type == TCG_TYPE_I32) { 486 tcg_out_mov(s, TCG_TYPE_I32, dst, src); 487 } else { 488 tcg_out_extrl_i64_i32(s, dst, src); 489 } 490 } else if (src_type == TCG_TYPE_I32) { 491 if (src_ext & MO_SIGN) { 492 tcg_out_exts_i32_i64(s, dst, src); 493 } else { 494 tcg_out_extu_i32_i64(s, dst, src); 495 } 496 } else { 497 if (src_ext & MO_SIGN) { 498 tcg_out_ext32s(s, dst, src); 499 } else { 500 tcg_out_ext32u(s, dst, src); 501 } 502 } 503 break; 504 case MO_UQ: 505 tcg_debug_assert(TCG_TARGET_REG_BITS == 64); 506 if (dst_type == TCG_TYPE_I32) { 507 tcg_out_extrl_i64_i32(s, dst, src); 508 } else { 509 tcg_out_mov(s, TCG_TYPE_I64, dst, src); 510 } 511 break; 512 default: 513 g_assert_not_reached(); 514 } 515 } 516 517 /* Minor variations on a theme, using a structure. */ 518 static void tcg_out_movext1_new_src(TCGContext *s, const TCGMovExtend *i, 519 TCGReg src) 520 { 521 tcg_out_movext(s, i->dst_type, i->dst, i->src_type, i->src_ext, src); 522 } 523 524 static void tcg_out_movext1(TCGContext *s, const TCGMovExtend *i) 525 { 526 tcg_out_movext1_new_src(s, i, i->src); 527 } 528 529 /** 530 * tcg_out_movext2 -- move and extend two pair 531 * @s: tcg context 532 * @i1: first move description 533 * @i2: second move description 534 * @scratch: temporary register, or -1 for none 535 * 536 * As tcg_out_movext, for both @i1 and @i2, caring for overlap 537 * between the sources and destinations. 538 */ 539 540 static void tcg_out_movext2(TCGContext *s, const TCGMovExtend *i1, 541 const TCGMovExtend *i2, int scratch) 542 { 543 TCGReg src1 = i1->src; 544 TCGReg src2 = i2->src; 545 546 if (i1->dst != src2) { 547 tcg_out_movext1(s, i1); 548 tcg_out_movext1(s, i2); 549 return; 550 } 551 if (i2->dst == src1) { 552 TCGType src1_type = i1->src_type; 553 TCGType src2_type = i2->src_type; 554 555 if (tcg_out_xchg(s, MAX(src1_type, src2_type), src1, src2)) { 556 /* The data is now in the correct registers, now extend. */ 557 src1 = i2->src; 558 src2 = i1->src; 559 } else { 560 tcg_debug_assert(scratch >= 0); 561 tcg_out_mov(s, src1_type, scratch, src1); 562 src1 = scratch; 563 } 564 } 565 tcg_out_movext1_new_src(s, i2, src2); 566 tcg_out_movext1_new_src(s, i1, src1); 567 } 568 569 /** 570 * tcg_out_movext3 -- move and extend three pair 571 * @s: tcg context 572 * @i1: first move description 573 * @i2: second move description 574 * @i3: third move description 575 * @scratch: temporary register, or -1 for none 576 * 577 * As tcg_out_movext, for all of @i1, @i2 and @i3, caring for overlap 578 * between the sources and destinations. 579 */ 580 581 static void tcg_out_movext3(TCGContext *s, const TCGMovExtend *i1, 582 const TCGMovExtend *i2, const TCGMovExtend *i3, 583 int scratch) 584 { 585 TCGReg src1 = i1->src; 586 TCGReg src2 = i2->src; 587 TCGReg src3 = i3->src; 588 589 if (i1->dst != src2 && i1->dst != src3) { 590 tcg_out_movext1(s, i1); 591 tcg_out_movext2(s, i2, i3, scratch); 592 return; 593 } 594 if (i2->dst != src1 && i2->dst != src3) { 595 tcg_out_movext1(s, i2); 596 tcg_out_movext2(s, i1, i3, scratch); 597 return; 598 } 599 if (i3->dst != src1 && i3->dst != src2) { 600 tcg_out_movext1(s, i3); 601 tcg_out_movext2(s, i1, i2, scratch); 602 return; 603 } 604 605 /* 606 * There is a cycle. Since there are only 3 nodes, the cycle is 607 * either "clockwise" or "anti-clockwise", and can be solved with 608 * a single scratch or two xchg. 609 */ 610 if (i1->dst == src2 && i2->dst == src3 && i3->dst == src1) { 611 /* "Clockwise" */ 612 if (tcg_out_xchg(s, MAX(i1->src_type, i2->src_type), src1, src2)) { 613 tcg_out_xchg(s, MAX(i2->src_type, i3->src_type), src2, src3); 614 /* The data is now in the correct registers, now extend. */ 615 tcg_out_movext1_new_src(s, i1, i1->dst); 616 tcg_out_movext1_new_src(s, i2, i2->dst); 617 tcg_out_movext1_new_src(s, i3, i3->dst); 618 } else { 619 tcg_debug_assert(scratch >= 0); 620 tcg_out_mov(s, i1->src_type, scratch, src1); 621 tcg_out_movext1(s, i3); 622 tcg_out_movext1(s, i2); 623 tcg_out_movext1_new_src(s, i1, scratch); 624 } 625 } else if (i1->dst == src3 && i2->dst == src1 && i3->dst == src2) { 626 /* "Anti-clockwise" */ 627 if (tcg_out_xchg(s, MAX(i2->src_type, i3->src_type), src2, src3)) { 628 tcg_out_xchg(s, MAX(i1->src_type, i2->src_type), src1, src2); 629 /* The data is now in the correct registers, now extend. */ 630 tcg_out_movext1_new_src(s, i1, i1->dst); 631 tcg_out_movext1_new_src(s, i2, i2->dst); 632 tcg_out_movext1_new_src(s, i3, i3->dst); 633 } else { 634 tcg_debug_assert(scratch >= 0); 635 tcg_out_mov(s, i1->src_type, scratch, src1); 636 tcg_out_movext1(s, i2); 637 tcg_out_movext1(s, i3); 638 tcg_out_movext1_new_src(s, i1, scratch); 639 } 640 } else { 641 g_assert_not_reached(); 642 } 643 } 644 645 /* 646 * Allocate a new TCGLabelQemuLdst entry. 647 */ 648 649 __attribute__((unused)) 650 static TCGLabelQemuLdst *new_ldst_label(TCGContext *s) 651 { 652 TCGLabelQemuLdst *l = tcg_malloc(sizeof(*l)); 653 654 memset(l, 0, sizeof(*l)); 655 QSIMPLEQ_INSERT_TAIL(&s->ldst_labels, l, next); 656 657 return l; 658 } 659 660 /* 661 * Allocate new constant pool entries. 662 */ 663 664 typedef struct TCGLabelPoolData { 665 struct TCGLabelPoolData *next; 666 tcg_insn_unit *label; 667 intptr_t addend; 668 int rtype; 669 unsigned nlong; 670 tcg_target_ulong data[]; 671 } TCGLabelPoolData; 672 673 static TCGLabelPoolData *new_pool_alloc(TCGContext *s, int nlong, int rtype, 674 tcg_insn_unit *label, intptr_t addend) 675 { 676 TCGLabelPoolData *n = tcg_malloc(sizeof(TCGLabelPoolData) 677 + sizeof(tcg_target_ulong) * nlong); 678 679 n->label = label; 680 n->addend = addend; 681 n->rtype = rtype; 682 n->nlong = nlong; 683 return n; 684 } 685 686 static void new_pool_insert(TCGContext *s, TCGLabelPoolData *n) 687 { 688 TCGLabelPoolData *i, **pp; 689 int nlong = n->nlong; 690 691 /* Insertion sort on the pool. */ 692 for (pp = &s->pool_labels; (i = *pp) != NULL; pp = &i->next) { 693 if (nlong > i->nlong) { 694 break; 695 } 696 if (nlong < i->nlong) { 697 continue; 698 } 699 if (memcmp(n->data, i->data, sizeof(tcg_target_ulong) * nlong) >= 0) { 700 break; 701 } 702 } 703 n->next = *pp; 704 *pp = n; 705 } 706 707 /* The "usual" for generic integer code. */ 708 __attribute__((unused)) 709 static void new_pool_label(TCGContext *s, tcg_target_ulong d, int rtype, 710 tcg_insn_unit *label, intptr_t addend) 711 { 712 TCGLabelPoolData *n = new_pool_alloc(s, 1, rtype, label, addend); 713 n->data[0] = d; 714 new_pool_insert(s, n); 715 } 716 717 /* For v64 or v128, depending on the host. */ 718 __attribute__((unused)) 719 static void new_pool_l2(TCGContext *s, int rtype, tcg_insn_unit *label, 720 intptr_t addend, tcg_target_ulong d0, 721 tcg_target_ulong d1) 722 { 723 TCGLabelPoolData *n = new_pool_alloc(s, 2, rtype, label, addend); 724 n->data[0] = d0; 725 n->data[1] = d1; 726 new_pool_insert(s, n); 727 } 728 729 /* For v128 or v256, depending on the host. */ 730 __attribute__((unused)) 731 static void new_pool_l4(TCGContext *s, int rtype, tcg_insn_unit *label, 732 intptr_t addend, tcg_target_ulong d0, 733 tcg_target_ulong d1, tcg_target_ulong d2, 734 tcg_target_ulong d3) 735 { 736 TCGLabelPoolData *n = new_pool_alloc(s, 4, rtype, label, addend); 737 n->data[0] = d0; 738 n->data[1] = d1; 739 n->data[2] = d2; 740 n->data[3] = d3; 741 new_pool_insert(s, n); 742 } 743 744 /* For v256, for 32-bit host. */ 745 __attribute__((unused)) 746 static void new_pool_l8(TCGContext *s, int rtype, tcg_insn_unit *label, 747 intptr_t addend, tcg_target_ulong d0, 748 tcg_target_ulong d1, tcg_target_ulong d2, 749 tcg_target_ulong d3, tcg_target_ulong d4, 750 tcg_target_ulong d5, tcg_target_ulong d6, 751 tcg_target_ulong d7) 752 { 753 TCGLabelPoolData *n = new_pool_alloc(s, 8, rtype, label, addend); 754 n->data[0] = d0; 755 n->data[1] = d1; 756 n->data[2] = d2; 757 n->data[3] = d3; 758 n->data[4] = d4; 759 n->data[5] = d5; 760 n->data[6] = d6; 761 n->data[7] = d7; 762 new_pool_insert(s, n); 763 } 764 765 /* 766 * Generate TB finalization at the end of block 767 */ 768 769 static int tcg_out_ldst_finalize(TCGContext *s) 770 { 771 TCGLabelQemuLdst *lb; 772 773 /* qemu_ld/st slow paths */ 774 QSIMPLEQ_FOREACH(lb, &s->ldst_labels, next) { 775 if (lb->is_ld 776 ? !tcg_out_qemu_ld_slow_path(s, lb) 777 : !tcg_out_qemu_st_slow_path(s, lb)) { 778 return -2; 779 } 780 781 /* 782 * Test for (pending) buffer overflow. The assumption is that any 783 * one operation beginning below the high water mark cannot overrun 784 * the buffer completely. Thus we can test for overflow after 785 * generating code without having to check during generation. 786 */ 787 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) { 788 return -1; 789 } 790 } 791 return 0; 792 } 793 794 static int tcg_out_pool_finalize(TCGContext *s) 795 { 796 TCGLabelPoolData *p = s->pool_labels; 797 TCGLabelPoolData *l = NULL; 798 void *a; 799 800 if (p == NULL) { 801 return 0; 802 } 803 804 /* 805 * ??? Round up to qemu_icache_linesize, but then do not round 806 * again when allocating the next TranslationBlock structure. 807 */ 808 a = (void *)ROUND_UP((uintptr_t)s->code_ptr, 809 sizeof(tcg_target_ulong) * p->nlong); 810 tcg_out_nop_fill(s->code_ptr, (tcg_insn_unit *)a - s->code_ptr); 811 s->data_gen_ptr = a; 812 813 for (; p != NULL; p = p->next) { 814 size_t size = sizeof(tcg_target_ulong) * p->nlong; 815 uintptr_t value; 816 817 if (!l || l->nlong != p->nlong || memcmp(l->data, p->data, size)) { 818 if (unlikely(a > s->code_gen_highwater)) { 819 return -1; 820 } 821 memcpy(a, p->data, size); 822 a += size; 823 l = p; 824 } 825 826 value = (uintptr_t)tcg_splitwx_to_rx(a) - size; 827 if (!patch_reloc(p->label, p->rtype, value, p->addend)) { 828 return -2; 829 } 830 } 831 832 s->code_ptr = a; 833 return 0; 834 } 835 836 #define C_PFX1(P, A) P##A 837 #define C_PFX2(P, A, B) P##A##_##B 838 #define C_PFX3(P, A, B, C) P##A##_##B##_##C 839 #define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D 840 #define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E 841 #define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F 842 843 /* Define an enumeration for the various combinations. */ 844 845 #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1), 846 #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2), 847 #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3), 848 #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4), 849 850 #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1), 851 #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2), 852 #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3), 853 #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4), 854 855 #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2), 856 #define C_N1O1_I1(O1, O2, I1) C_PFX3(c_n1o1_i1_, O1, O2, I1), 857 #define C_N2_I1(O1, O2, I1) C_PFX3(c_n2_i1_, O1, O2, I1), 858 859 #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1), 860 #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2), 861 #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3), 862 #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4), 863 #define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_n1_o1_i4_, O1, O2, I1, I2, I3, I4), 864 865 typedef enum { 866 C_Dynamic = -2, 867 C_NotImplemented = -1, 868 #include "tcg-target-con-set.h" 869 } TCGConstraintSetIndex; 870 871 static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode, TCGType, unsigned); 872 873 #undef C_O0_I1 874 #undef C_O0_I2 875 #undef C_O0_I3 876 #undef C_O0_I4 877 #undef C_O1_I1 878 #undef C_O1_I2 879 #undef C_O1_I3 880 #undef C_O1_I4 881 #undef C_N1_I2 882 #undef C_N1O1_I1 883 #undef C_N2_I1 884 #undef C_O2_I1 885 #undef C_O2_I2 886 #undef C_O2_I3 887 #undef C_O2_I4 888 #undef C_N1_O1_I4 889 890 /* Put all of the constraint sets into an array, indexed by the enum. */ 891 892 typedef struct TCGConstraintSet { 893 uint8_t nb_oargs, nb_iargs; 894 const char *args_ct_str[TCG_MAX_OP_ARGS]; 895 } TCGConstraintSet; 896 897 #define C_O0_I1(I1) { 0, 1, { #I1 } }, 898 #define C_O0_I2(I1, I2) { 0, 2, { #I1, #I2 } }, 899 #define C_O0_I3(I1, I2, I3) { 0, 3, { #I1, #I2, #I3 } }, 900 #define C_O0_I4(I1, I2, I3, I4) { 0, 4, { #I1, #I2, #I3, #I4 } }, 901 902 #define C_O1_I1(O1, I1) { 1, 1, { #O1, #I1 } }, 903 #define C_O1_I2(O1, I1, I2) { 1, 2, { #O1, #I1, #I2 } }, 904 #define C_O1_I3(O1, I1, I2, I3) { 1, 3, { #O1, #I1, #I2, #I3 } }, 905 #define C_O1_I4(O1, I1, I2, I3, I4) { 1, 4, { #O1, #I1, #I2, #I3, #I4 } }, 906 907 #define C_N1_I2(O1, I1, I2) { 1, 2, { "&" #O1, #I1, #I2 } }, 908 #define C_N1O1_I1(O1, O2, I1) { 2, 1, { "&" #O1, #O2, #I1 } }, 909 #define C_N2_I1(O1, O2, I1) { 2, 1, { "&" #O1, "&" #O2, #I1 } }, 910 911 #define C_O2_I1(O1, O2, I1) { 2, 1, { #O1, #O2, #I1 } }, 912 #define C_O2_I2(O1, O2, I1, I2) { 2, 2, { #O1, #O2, #I1, #I2 } }, 913 #define C_O2_I3(O1, O2, I1, I2, I3) { 2, 3, { #O1, #O2, #I1, #I2, #I3 } }, 914 #define C_O2_I4(O1, O2, I1, I2, I3, I4) { 2, 4, { #O1, #O2, #I1, #I2, #I3, #I4 } }, 915 #define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) { 2, 4, { "&" #O1, #O2, #I1, #I2, #I3, #I4 } }, 916 917 static const TCGConstraintSet constraint_sets[] = { 918 #include "tcg-target-con-set.h" 919 }; 920 921 #undef C_O0_I1 922 #undef C_O0_I2 923 #undef C_O0_I3 924 #undef C_O0_I4 925 #undef C_O1_I1 926 #undef C_O1_I2 927 #undef C_O1_I3 928 #undef C_O1_I4 929 #undef C_N1_I2 930 #undef C_N1O1_I1 931 #undef C_N2_I1 932 #undef C_O2_I1 933 #undef C_O2_I2 934 #undef C_O2_I3 935 #undef C_O2_I4 936 #undef C_N1_O1_I4 937 938 /* Expand the enumerator to be returned from tcg_target_op_def(). */ 939 940 #define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1) 941 #define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2) 942 #define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3) 943 #define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4) 944 945 #define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1) 946 #define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2) 947 #define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3) 948 #define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4) 949 950 #define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2) 951 #define C_N1O1_I1(O1, O2, I1) C_PFX3(c_n1o1_i1_, O1, O2, I1) 952 #define C_N2_I1(O1, O2, I1) C_PFX3(c_n2_i1_, O1, O2, I1) 953 954 #define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1) 955 #define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2) 956 #define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3) 957 #define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4) 958 #define C_N1_O1_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_n1_o1_i4_, O1, O2, I1, I2, I3, I4) 959 960 /* 961 * TCGOutOp is the base class for a set of structures that describe how 962 * to generate code for a given TCGOpcode. 963 * 964 * @static_constraint: 965 * C_NotImplemented: The TCGOpcode is not supported by the backend. 966 * C_Dynamic: Use @dynamic_constraint to select a constraint set 967 * based on any of @type, @flags, or host isa. 968 * Otherwise: The register allocation constrains for the TCGOpcode. 969 * 970 * Subclasses of TCGOutOp will define a set of output routines that may 971 * be used. Such routines will often be selected by the set of registers 972 * and constants that come out of register allocation. The set of 973 * routines that are provided will guide the set of constraints that are 974 * legal. In particular, assume that tcg_optimize() has done its job in 975 * swapping commutative operands and folding operations for which all 976 * operands are constant. 977 */ 978 typedef struct TCGOutOp { 979 TCGConstraintSetIndex static_constraint; 980 TCGConstraintSetIndex (*dynamic_constraint)(TCGType type, unsigned flags); 981 } TCGOutOp; 982 983 typedef struct TCGOutOpAddSubCarry { 984 TCGOutOp base; 985 void (*out_rrr)(TCGContext *s, TCGType type, 986 TCGReg a0, TCGReg a1, TCGReg a2); 987 void (*out_rri)(TCGContext *s, TCGType type, 988 TCGReg a0, TCGReg a1, tcg_target_long a2); 989 void (*out_rir)(TCGContext *s, TCGType type, 990 TCGReg a0, tcg_target_long a1, TCGReg a2); 991 void (*out_rii)(TCGContext *s, TCGType type, 992 TCGReg a0, tcg_target_long a1, tcg_target_long a2); 993 } TCGOutOpAddSubCarry; 994 995 typedef struct TCGOutOpBinary { 996 TCGOutOp base; 997 void (*out_rrr)(TCGContext *s, TCGType type, 998 TCGReg a0, TCGReg a1, TCGReg a2); 999 void (*out_rri)(TCGContext *s, TCGType type, 1000 TCGReg a0, TCGReg a1, tcg_target_long a2); 1001 } TCGOutOpBinary; 1002 1003 typedef struct TCGOutOpBrcond { 1004 TCGOutOp base; 1005 void (*out_rr)(TCGContext *s, TCGType type, TCGCond cond, 1006 TCGReg a1, TCGReg a2, TCGLabel *label); 1007 void (*out_ri)(TCGContext *s, TCGType type, TCGCond cond, 1008 TCGReg a1, tcg_target_long a2, TCGLabel *label); 1009 } TCGOutOpBrcond; 1010 1011 typedef struct TCGOutOpBrcond2 { 1012 TCGOutOp base; 1013 void (*out)(TCGContext *s, TCGCond cond, TCGReg al, TCGReg ah, 1014 TCGArg bl, bool const_bl, 1015 TCGArg bh, bool const_bh, TCGLabel *l); 1016 } TCGOutOpBrcond2; 1017 1018 typedef struct TCGOutOpBswap { 1019 TCGOutOp base; 1020 void (*out_rr)(TCGContext *s, TCGType type, 1021 TCGReg a0, TCGReg a1, unsigned flags); 1022 } TCGOutOpBswap; 1023 1024 typedef struct TCGOutOpDeposit { 1025 TCGOutOp base; 1026 void (*out_rrr)(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, 1027 TCGReg a2, unsigned ofs, unsigned len); 1028 void (*out_rri)(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, 1029 tcg_target_long a2, unsigned ofs, unsigned len); 1030 void (*out_rzr)(TCGContext *s, TCGType type, TCGReg a0, 1031 TCGReg a2, unsigned ofs, unsigned len); 1032 } TCGOutOpDeposit; 1033 1034 typedef struct TCGOutOpDivRem { 1035 TCGOutOp base; 1036 void (*out_rr01r)(TCGContext *s, TCGType type, 1037 TCGReg a0, TCGReg a1, TCGReg a4); 1038 } TCGOutOpDivRem; 1039 1040 typedef struct TCGOutOpExtract { 1041 TCGOutOp base; 1042 void (*out_rr)(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, 1043 unsigned ofs, unsigned len); 1044 } TCGOutOpExtract; 1045 1046 typedef struct TCGOutOpExtract2 { 1047 TCGOutOp base; 1048 void (*out_rrr)(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1, 1049 TCGReg a2, unsigned shr); 1050 } TCGOutOpExtract2; 1051 1052 typedef struct TCGOutOpLoad { 1053 TCGOutOp base; 1054 void (*out)(TCGContext *s, TCGType type, TCGReg dest, 1055 TCGReg base, intptr_t offset); 1056 } TCGOutOpLoad; 1057 1058 typedef struct TCGOutOpMovcond { 1059 TCGOutOp base; 1060 void (*out)(TCGContext *s, TCGType type, TCGCond cond, 1061 TCGReg ret, TCGReg c1, TCGArg c2, bool const_c2, 1062 TCGArg vt, bool const_vt, TCGArg vf, bool consf_vf); 1063 } TCGOutOpMovcond; 1064 1065 typedef struct TCGOutOpMul2 { 1066 TCGOutOp base; 1067 void (*out_rrrr)(TCGContext *s, TCGType type, 1068 TCGReg a0, TCGReg a1, TCGReg a2, TCGReg a3); 1069 } TCGOutOpMul2; 1070 1071 typedef struct TCGOutOpQemuLdSt { 1072 TCGOutOp base; 1073 void (*out)(TCGContext *s, TCGType type, TCGReg dest, 1074 TCGReg addr, MemOpIdx oi); 1075 } TCGOutOpQemuLdSt; 1076 1077 typedef struct TCGOutOpQemuLdSt2 { 1078 TCGOutOp base; 1079 void (*out)(TCGContext *s, TCGType type, TCGReg dlo, TCGReg dhi, 1080 TCGReg addr, MemOpIdx oi); 1081 } TCGOutOpQemuLdSt2; 1082 1083 typedef struct TCGOutOpUnary { 1084 TCGOutOp base; 1085 void (*out_rr)(TCGContext *s, TCGType type, TCGReg a0, TCGReg a1); 1086 } TCGOutOpUnary; 1087 1088 typedef struct TCGOutOpSetcond { 1089 TCGOutOp base; 1090 void (*out_rrr)(TCGContext *s, TCGType type, TCGCond cond, 1091 TCGReg ret, TCGReg a1, TCGReg a2); 1092 void (*out_rri)(TCGContext *s, TCGType type, TCGCond cond, 1093 TCGReg ret, TCGReg a1, tcg_target_long a2); 1094 } TCGOutOpSetcond; 1095 1096 typedef struct TCGOutOpSetcond2 { 1097 TCGOutOp base; 1098 void (*out)(TCGContext *s, TCGCond cond, TCGReg ret, TCGReg al, TCGReg ah, 1099 TCGArg bl, bool const_bl, TCGArg bh, bool const_bh); 1100 } TCGOutOpSetcond2; 1101 1102 typedef struct TCGOutOpStore { 1103 TCGOutOp base; 1104 void (*out_r)(TCGContext *s, TCGType type, TCGReg data, 1105 TCGReg base, intptr_t offset); 1106 void (*out_i)(TCGContext *s, TCGType type, tcg_target_long data, 1107 TCGReg base, intptr_t offset); 1108 } TCGOutOpStore; 1109 1110 typedef struct TCGOutOpSubtract { 1111 TCGOutOp base; 1112 void (*out_rrr)(TCGContext *s, TCGType type, 1113 TCGReg a0, TCGReg a1, TCGReg a2); 1114 void (*out_rir)(TCGContext *s, TCGType type, 1115 TCGReg a0, tcg_target_long a1, TCGReg a2); 1116 } TCGOutOpSubtract; 1117 1118 #include "tcg-target.c.inc" 1119 1120 #ifndef CONFIG_TCG_INTERPRETER 1121 /* Validate CPUTLBDescFast placement. */ 1122 QEMU_BUILD_BUG_ON((int)(offsetof(CPUNegativeOffsetState, tlb.f[0]) - 1123 sizeof(CPUNegativeOffsetState)) 1124 < MIN_TLB_MASK_TABLE_OFS); 1125 #endif 1126 1127 #if TCG_TARGET_REG_BITS == 64 1128 /* 1129 * We require these functions for slow-path function calls. 1130 * Adapt them generically for opcode output. 1131 */ 1132 1133 static void tgen_exts_i32_i64(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1) 1134 { 1135 tcg_out_exts_i32_i64(s, a0, a1); 1136 } 1137 1138 static const TCGOutOpUnary outop_exts_i32_i64 = { 1139 .base.static_constraint = C_O1_I1(r, r), 1140 .out_rr = tgen_exts_i32_i64, 1141 }; 1142 1143 static void tgen_extu_i32_i64(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1) 1144 { 1145 tcg_out_extu_i32_i64(s, a0, a1); 1146 } 1147 1148 static const TCGOutOpUnary outop_extu_i32_i64 = { 1149 .base.static_constraint = C_O1_I1(r, r), 1150 .out_rr = tgen_extu_i32_i64, 1151 }; 1152 1153 static void tgen_extrl_i64_i32(TCGContext *s, TCGType t, TCGReg a0, TCGReg a1) 1154 { 1155 tcg_out_extrl_i64_i32(s, a0, a1); 1156 } 1157 1158 static const TCGOutOpUnary outop_extrl_i64_i32 = { 1159 .base.static_constraint = C_O1_I1(r, r), 1160 .out_rr = TCG_TARGET_HAS_extr_i64_i32 ? tgen_extrl_i64_i32 : NULL, 1161 }; 1162 #endif 1163 1164 static const TCGOutOp outop_goto_ptr = { 1165 .static_constraint = C_O0_I1(r), 1166 }; 1167 1168 static const TCGOutOpLoad outop_ld = { 1169 .base.static_constraint = C_O1_I1(r, r), 1170 .out = tcg_out_ld, 1171 }; 1172 1173 /* 1174 * Register V as the TCGOutOp for O. 1175 * This verifies that V is of type T, otherwise give a nice compiler error. 1176 * This prevents trivial mistakes within each arch/tcg-target.c.inc. 1177 */ 1178 #define OUTOP(O, T, V) [O] = _Generic(V, T: &V.base) 1179 1180 /* Register allocation descriptions for every TCGOpcode. */ 1181 static const TCGOutOp * const all_outop[NB_OPS] = { 1182 OUTOP(INDEX_op_add, TCGOutOpBinary, outop_add), 1183 OUTOP(INDEX_op_addci, TCGOutOpAddSubCarry, outop_addci), 1184 OUTOP(INDEX_op_addcio, TCGOutOpBinary, outop_addcio), 1185 OUTOP(INDEX_op_addco, TCGOutOpBinary, outop_addco), 1186 /* addc1o is implemented with set_carry + addcio */ 1187 OUTOP(INDEX_op_addc1o, TCGOutOpBinary, outop_addcio), 1188 OUTOP(INDEX_op_and, TCGOutOpBinary, outop_and), 1189 OUTOP(INDEX_op_andc, TCGOutOpBinary, outop_andc), 1190 OUTOP(INDEX_op_brcond, TCGOutOpBrcond, outop_brcond), 1191 OUTOP(INDEX_op_bswap16, TCGOutOpBswap, outop_bswap16), 1192 OUTOP(INDEX_op_bswap32, TCGOutOpBswap, outop_bswap32), 1193 OUTOP(INDEX_op_clz, TCGOutOpBinary, outop_clz), 1194 OUTOP(INDEX_op_ctpop, TCGOutOpUnary, outop_ctpop), 1195 OUTOP(INDEX_op_ctz, TCGOutOpBinary, outop_ctz), 1196 OUTOP(INDEX_op_deposit, TCGOutOpDeposit, outop_deposit), 1197 OUTOP(INDEX_op_divs, TCGOutOpBinary, outop_divs), 1198 OUTOP(INDEX_op_divu, TCGOutOpBinary, outop_divu), 1199 OUTOP(INDEX_op_divs2, TCGOutOpDivRem, outop_divs2), 1200 OUTOP(INDEX_op_divu2, TCGOutOpDivRem, outop_divu2), 1201 OUTOP(INDEX_op_eqv, TCGOutOpBinary, outop_eqv), 1202 OUTOP(INDEX_op_extract, TCGOutOpExtract, outop_extract), 1203 OUTOP(INDEX_op_extract2, TCGOutOpExtract2, outop_extract2), 1204 OUTOP(INDEX_op_ld8u, TCGOutOpLoad, outop_ld8u), 1205 OUTOP(INDEX_op_ld8s, TCGOutOpLoad, outop_ld8s), 1206 OUTOP(INDEX_op_ld16u, TCGOutOpLoad, outop_ld16u), 1207 OUTOP(INDEX_op_ld16s, TCGOutOpLoad, outop_ld16s), 1208 OUTOP(INDEX_op_ld, TCGOutOpLoad, outop_ld), 1209 OUTOP(INDEX_op_movcond, TCGOutOpMovcond, outop_movcond), 1210 OUTOP(INDEX_op_mul, TCGOutOpBinary, outop_mul), 1211 OUTOP(INDEX_op_muls2, TCGOutOpMul2, outop_muls2), 1212 OUTOP(INDEX_op_mulsh, TCGOutOpBinary, outop_mulsh), 1213 OUTOP(INDEX_op_mulu2, TCGOutOpMul2, outop_mulu2), 1214 OUTOP(INDEX_op_muluh, TCGOutOpBinary, outop_muluh), 1215 OUTOP(INDEX_op_nand, TCGOutOpBinary, outop_nand), 1216 OUTOP(INDEX_op_neg, TCGOutOpUnary, outop_neg), 1217 OUTOP(INDEX_op_negsetcond, TCGOutOpSetcond, outop_negsetcond), 1218 OUTOP(INDEX_op_nor, TCGOutOpBinary, outop_nor), 1219 OUTOP(INDEX_op_not, TCGOutOpUnary, outop_not), 1220 OUTOP(INDEX_op_or, TCGOutOpBinary, outop_or), 1221 OUTOP(INDEX_op_orc, TCGOutOpBinary, outop_orc), 1222 OUTOP(INDEX_op_qemu_ld, TCGOutOpQemuLdSt, outop_qemu_ld), 1223 OUTOP(INDEX_op_qemu_ld2, TCGOutOpQemuLdSt2, outop_qemu_ld2), 1224 OUTOP(INDEX_op_qemu_st, TCGOutOpQemuLdSt, outop_qemu_st), 1225 OUTOP(INDEX_op_qemu_st2, TCGOutOpQemuLdSt2, outop_qemu_st2), 1226 OUTOP(INDEX_op_rems, TCGOutOpBinary, outop_rems), 1227 OUTOP(INDEX_op_remu, TCGOutOpBinary, outop_remu), 1228 OUTOP(INDEX_op_rotl, TCGOutOpBinary, outop_rotl), 1229 OUTOP(INDEX_op_rotr, TCGOutOpBinary, outop_rotr), 1230 OUTOP(INDEX_op_sar, TCGOutOpBinary, outop_sar), 1231 OUTOP(INDEX_op_setcond, TCGOutOpSetcond, outop_setcond), 1232 OUTOP(INDEX_op_sextract, TCGOutOpExtract, outop_sextract), 1233 OUTOP(INDEX_op_shl, TCGOutOpBinary, outop_shl), 1234 OUTOP(INDEX_op_shr, TCGOutOpBinary, outop_shr), 1235 OUTOP(INDEX_op_st, TCGOutOpStore, outop_st), 1236 OUTOP(INDEX_op_st8, TCGOutOpStore, outop_st8), 1237 OUTOP(INDEX_op_st16, TCGOutOpStore, outop_st16), 1238 OUTOP(INDEX_op_sub, TCGOutOpSubtract, outop_sub), 1239 OUTOP(INDEX_op_subbi, TCGOutOpAddSubCarry, outop_subbi), 1240 OUTOP(INDEX_op_subbio, TCGOutOpAddSubCarry, outop_subbio), 1241 OUTOP(INDEX_op_subbo, TCGOutOpAddSubCarry, outop_subbo), 1242 /* subb1o is implemented with set_borrow + subbio */ 1243 OUTOP(INDEX_op_subb1o, TCGOutOpAddSubCarry, outop_subbio), 1244 OUTOP(INDEX_op_xor, TCGOutOpBinary, outop_xor), 1245 1246 [INDEX_op_goto_ptr] = &outop_goto_ptr, 1247 1248 #if TCG_TARGET_REG_BITS == 32 1249 OUTOP(INDEX_op_brcond2_i32, TCGOutOpBrcond2, outop_brcond2), 1250 OUTOP(INDEX_op_setcond2_i32, TCGOutOpSetcond2, outop_setcond2), 1251 #else 1252 OUTOP(INDEX_op_bswap64, TCGOutOpUnary, outop_bswap64), 1253 OUTOP(INDEX_op_ext_i32_i64, TCGOutOpUnary, outop_exts_i32_i64), 1254 OUTOP(INDEX_op_extu_i32_i64, TCGOutOpUnary, outop_extu_i32_i64), 1255 OUTOP(INDEX_op_extrl_i64_i32, TCGOutOpUnary, outop_extrl_i64_i32), 1256 OUTOP(INDEX_op_extrh_i64_i32, TCGOutOpUnary, outop_extrh_i64_i32), 1257 OUTOP(INDEX_op_ld32u, TCGOutOpLoad, outop_ld32u), 1258 OUTOP(INDEX_op_ld32s, TCGOutOpLoad, outop_ld32s), 1259 OUTOP(INDEX_op_st32, TCGOutOpStore, outop_st), 1260 #endif 1261 }; 1262 1263 #undef OUTOP 1264 1265 /* 1266 * All TCG threads except the parent (i.e. the one that called tcg_context_init 1267 * and registered the target's TCG globals) must register with this function 1268 * before initiating translation. 1269 * 1270 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation 1271 * of tcg_region_init() for the reasoning behind this. 1272 * 1273 * In system-mode each caller registers its context in tcg_ctxs[]. Note that in 1274 * system-mode tcg_ctxs[] does not track tcg_ctx_init, since the initial context 1275 * is not used anymore for translation once this function is called. 1276 * 1277 * Not tracking tcg_init_ctx in tcg_ctxs[] in system-mode keeps code that 1278 * iterates over the array (e.g. tcg_code_size() the same for both system/user 1279 * modes. 1280 */ 1281 #ifdef CONFIG_USER_ONLY 1282 void tcg_register_thread(void) 1283 { 1284 tcg_ctx = &tcg_init_ctx; 1285 } 1286 #else 1287 void tcg_register_thread(void) 1288 { 1289 TCGContext *s = g_malloc(sizeof(*s)); 1290 unsigned int i, n; 1291 1292 *s = tcg_init_ctx; 1293 1294 /* Relink mem_base. */ 1295 for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) { 1296 if (tcg_init_ctx.temps[i].mem_base) { 1297 ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps; 1298 tcg_debug_assert(b >= 0 && b < n); 1299 s->temps[i].mem_base = &s->temps[b]; 1300 } 1301 } 1302 1303 /* Claim an entry in tcg_ctxs */ 1304 n = qatomic_fetch_inc(&tcg_cur_ctxs); 1305 g_assert(n < tcg_max_ctxs); 1306 qatomic_set(&tcg_ctxs[n], s); 1307 1308 if (n > 0) { 1309 tcg_region_initial_alloc(s); 1310 } 1311 1312 tcg_ctx = s; 1313 } 1314 #endif /* !CONFIG_USER_ONLY */ 1315 1316 /* pool based memory allocation */ 1317 void *tcg_malloc_internal(TCGContext *s, int size) 1318 { 1319 TCGPool *p; 1320 int pool_size; 1321 1322 if (size > TCG_POOL_CHUNK_SIZE) { 1323 /* big malloc: insert a new pool (XXX: could optimize) */ 1324 p = g_malloc(sizeof(TCGPool) + size); 1325 p->size = size; 1326 p->next = s->pool_first_large; 1327 s->pool_first_large = p; 1328 return p->data; 1329 } else { 1330 p = s->pool_current; 1331 if (!p) { 1332 p = s->pool_first; 1333 if (!p) 1334 goto new_pool; 1335 } else { 1336 if (!p->next) { 1337 new_pool: 1338 pool_size = TCG_POOL_CHUNK_SIZE; 1339 p = g_malloc(sizeof(TCGPool) + pool_size); 1340 p->size = pool_size; 1341 p->next = NULL; 1342 if (s->pool_current) { 1343 s->pool_current->next = p; 1344 } else { 1345 s->pool_first = p; 1346 } 1347 } else { 1348 p = p->next; 1349 } 1350 } 1351 } 1352 s->pool_current = p; 1353 s->pool_cur = p->data + size; 1354 s->pool_end = p->data + p->size; 1355 return p->data; 1356 } 1357 1358 void tcg_pool_reset(TCGContext *s) 1359 { 1360 TCGPool *p, *t; 1361 for (p = s->pool_first_large; p; p = t) { 1362 t = p->next; 1363 g_free(p); 1364 } 1365 s->pool_first_large = NULL; 1366 s->pool_cur = s->pool_end = NULL; 1367 s->pool_current = NULL; 1368 } 1369 1370 /* 1371 * Create TCGHelperInfo structures for "tcg/tcg-ldst.h" functions, 1372 * akin to what "exec/helper-tcg.h" does with DEF_HELPER_FLAGS_N. 1373 * We only use these for layout in tcg_out_ld_helper_ret and 1374 * tcg_out_st_helper_args, and share them between several of 1375 * the helpers, with the end result that it's easier to build manually. 1376 */ 1377 1378 #if TCG_TARGET_REG_BITS == 32 1379 # define dh_typecode_ttl dh_typecode_i32 1380 #else 1381 # define dh_typecode_ttl dh_typecode_i64 1382 #endif 1383 1384 static TCGHelperInfo info_helper_ld32_mmu = { 1385 .flags = TCG_CALL_NO_WG, 1386 .typemask = dh_typemask(ttl, 0) /* return tcg_target_ulong */ 1387 | dh_typemask(env, 1) 1388 | dh_typemask(i64, 2) /* uint64_t addr */ 1389 | dh_typemask(i32, 3) /* unsigned oi */ 1390 | dh_typemask(ptr, 4) /* uintptr_t ra */ 1391 }; 1392 1393 static TCGHelperInfo info_helper_ld64_mmu = { 1394 .flags = TCG_CALL_NO_WG, 1395 .typemask = dh_typemask(i64, 0) /* return uint64_t */ 1396 | dh_typemask(env, 1) 1397 | dh_typemask(i64, 2) /* uint64_t addr */ 1398 | dh_typemask(i32, 3) /* unsigned oi */ 1399 | dh_typemask(ptr, 4) /* uintptr_t ra */ 1400 }; 1401 1402 static TCGHelperInfo info_helper_ld128_mmu = { 1403 .flags = TCG_CALL_NO_WG, 1404 .typemask = dh_typemask(i128, 0) /* return Int128 */ 1405 | dh_typemask(env, 1) 1406 | dh_typemask(i64, 2) /* uint64_t addr */ 1407 | dh_typemask(i32, 3) /* unsigned oi */ 1408 | dh_typemask(ptr, 4) /* uintptr_t ra */ 1409 }; 1410 1411 static TCGHelperInfo info_helper_st32_mmu = { 1412 .flags = TCG_CALL_NO_WG, 1413 .typemask = dh_typemask(void, 0) 1414 | dh_typemask(env, 1) 1415 | dh_typemask(i64, 2) /* uint64_t addr */ 1416 | dh_typemask(i32, 3) /* uint32_t data */ 1417 | dh_typemask(i32, 4) /* unsigned oi */ 1418 | dh_typemask(ptr, 5) /* uintptr_t ra */ 1419 }; 1420 1421 static TCGHelperInfo info_helper_st64_mmu = { 1422 .flags = TCG_CALL_NO_WG, 1423 .typemask = dh_typemask(void, 0) 1424 | dh_typemask(env, 1) 1425 | dh_typemask(i64, 2) /* uint64_t addr */ 1426 | dh_typemask(i64, 3) /* uint64_t data */ 1427 | dh_typemask(i32, 4) /* unsigned oi */ 1428 | dh_typemask(ptr, 5) /* uintptr_t ra */ 1429 }; 1430 1431 static TCGHelperInfo info_helper_st128_mmu = { 1432 .flags = TCG_CALL_NO_WG, 1433 .typemask = dh_typemask(void, 0) 1434 | dh_typemask(env, 1) 1435 | dh_typemask(i64, 2) /* uint64_t addr */ 1436 | dh_typemask(i128, 3) /* Int128 data */ 1437 | dh_typemask(i32, 4) /* unsigned oi */ 1438 | dh_typemask(ptr, 5) /* uintptr_t ra */ 1439 }; 1440 1441 #ifdef CONFIG_TCG_INTERPRETER 1442 static ffi_type *typecode_to_ffi(int argmask) 1443 { 1444 /* 1445 * libffi does not support __int128_t, so we have forced Int128 1446 * to use the structure definition instead of the builtin type. 1447 */ 1448 static ffi_type *ffi_type_i128_elements[3] = { 1449 &ffi_type_uint64, 1450 &ffi_type_uint64, 1451 NULL 1452 }; 1453 static ffi_type ffi_type_i128 = { 1454 .size = 16, 1455 .alignment = __alignof__(Int128), 1456 .type = FFI_TYPE_STRUCT, 1457 .elements = ffi_type_i128_elements, 1458 }; 1459 1460 switch (argmask) { 1461 case dh_typecode_void: 1462 return &ffi_type_void; 1463 case dh_typecode_i32: 1464 return &ffi_type_uint32; 1465 case dh_typecode_s32: 1466 return &ffi_type_sint32; 1467 case dh_typecode_i64: 1468 return &ffi_type_uint64; 1469 case dh_typecode_s64: 1470 return &ffi_type_sint64; 1471 case dh_typecode_ptr: 1472 return &ffi_type_pointer; 1473 case dh_typecode_i128: 1474 return &ffi_type_i128; 1475 } 1476 g_assert_not_reached(); 1477 } 1478 1479 static ffi_cif *init_ffi_layout(TCGHelperInfo *info) 1480 { 1481 unsigned typemask = info->typemask; 1482 struct { 1483 ffi_cif cif; 1484 ffi_type *args[]; 1485 } *ca; 1486 ffi_status status; 1487 int nargs; 1488 1489 /* Ignoring the return type, find the last non-zero field. */ 1490 nargs = 32 - clz32(typemask >> 3); 1491 nargs = DIV_ROUND_UP(nargs, 3); 1492 assert(nargs <= MAX_CALL_IARGS); 1493 1494 ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *)); 1495 ca->cif.rtype = typecode_to_ffi(typemask & 7); 1496 ca->cif.nargs = nargs; 1497 1498 if (nargs != 0) { 1499 ca->cif.arg_types = ca->args; 1500 for (int j = 0; j < nargs; ++j) { 1501 int typecode = extract32(typemask, (j + 1) * 3, 3); 1502 ca->args[j] = typecode_to_ffi(typecode); 1503 } 1504 } 1505 1506 status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs, 1507 ca->cif.rtype, ca->cif.arg_types); 1508 assert(status == FFI_OK); 1509 1510 return &ca->cif; 1511 } 1512 1513 #define HELPER_INFO_INIT(I) (&(I)->cif) 1514 #define HELPER_INFO_INIT_VAL(I) init_ffi_layout(I) 1515 #else 1516 #define HELPER_INFO_INIT(I) (&(I)->init) 1517 #define HELPER_INFO_INIT_VAL(I) 1 1518 #endif /* CONFIG_TCG_INTERPRETER */ 1519 1520 static inline bool arg_slot_reg_p(unsigned arg_slot) 1521 { 1522 /* 1523 * Split the sizeof away from the comparison to avoid Werror from 1524 * "unsigned < 0 is always false", when iarg_regs is empty. 1525 */ 1526 unsigned nreg = ARRAY_SIZE(tcg_target_call_iarg_regs); 1527 return arg_slot < nreg; 1528 } 1529 1530 static inline int arg_slot_stk_ofs(unsigned arg_slot) 1531 { 1532 unsigned max = TCG_STATIC_CALL_ARGS_SIZE / sizeof(tcg_target_long); 1533 unsigned stk_slot = arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs); 1534 1535 tcg_debug_assert(stk_slot < max); 1536 return TCG_TARGET_CALL_STACK_OFFSET + stk_slot * sizeof(tcg_target_long); 1537 } 1538 1539 typedef struct TCGCumulativeArgs { 1540 int arg_idx; /* tcg_gen_callN args[] */ 1541 int info_in_idx; /* TCGHelperInfo in[] */ 1542 int arg_slot; /* regs+stack slot */ 1543 int ref_slot; /* stack slots for references */ 1544 } TCGCumulativeArgs; 1545 1546 static void layout_arg_even(TCGCumulativeArgs *cum) 1547 { 1548 cum->arg_slot += cum->arg_slot & 1; 1549 } 1550 1551 static void layout_arg_1(TCGCumulativeArgs *cum, TCGHelperInfo *info, 1552 TCGCallArgumentKind kind) 1553 { 1554 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx]; 1555 1556 *loc = (TCGCallArgumentLoc){ 1557 .kind = kind, 1558 .arg_idx = cum->arg_idx, 1559 .arg_slot = cum->arg_slot, 1560 }; 1561 cum->info_in_idx++; 1562 cum->arg_slot++; 1563 } 1564 1565 static void layout_arg_normal_n(TCGCumulativeArgs *cum, 1566 TCGHelperInfo *info, int n) 1567 { 1568 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx]; 1569 1570 for (int i = 0; i < n; ++i) { 1571 /* Layout all using the same arg_idx, adjusting the subindex. */ 1572 loc[i] = (TCGCallArgumentLoc){ 1573 .kind = TCG_CALL_ARG_NORMAL, 1574 .arg_idx = cum->arg_idx, 1575 .tmp_subindex = i, 1576 .arg_slot = cum->arg_slot + i, 1577 }; 1578 } 1579 cum->info_in_idx += n; 1580 cum->arg_slot += n; 1581 } 1582 1583 static void layout_arg_by_ref(TCGCumulativeArgs *cum, TCGHelperInfo *info) 1584 { 1585 TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx]; 1586 int n = 128 / TCG_TARGET_REG_BITS; 1587 1588 /* The first subindex carries the pointer. */ 1589 layout_arg_1(cum, info, TCG_CALL_ARG_BY_REF); 1590 1591 /* 1592 * The callee is allowed to clobber memory associated with 1593 * structure pass by-reference. Therefore we must make copies. 1594 * Allocate space from "ref_slot", which will be adjusted to 1595 * follow the parameters on the stack. 1596 */ 1597 loc[0].ref_slot = cum->ref_slot; 1598 1599 /* 1600 * Subsequent words also go into the reference slot, but 1601 * do not accumulate into the regular arguments. 1602 */ 1603 for (int i = 1; i < n; ++i) { 1604 loc[i] = (TCGCallArgumentLoc){ 1605 .kind = TCG_CALL_ARG_BY_REF_N, 1606 .arg_idx = cum->arg_idx, 1607 .tmp_subindex = i, 1608 .ref_slot = cum->ref_slot + i, 1609 }; 1610 } 1611 cum->info_in_idx += n - 1; /* i=0 accounted for in layout_arg_1 */ 1612 cum->ref_slot += n; 1613 } 1614 1615 static void init_call_layout(TCGHelperInfo *info) 1616 { 1617 int max_reg_slots = ARRAY_SIZE(tcg_target_call_iarg_regs); 1618 int max_stk_slots = TCG_STATIC_CALL_ARGS_SIZE / sizeof(tcg_target_long); 1619 unsigned typemask = info->typemask; 1620 unsigned typecode; 1621 TCGCumulativeArgs cum = { }; 1622 1623 /* 1624 * Parse and place any function return value. 1625 */ 1626 typecode = typemask & 7; 1627 switch (typecode) { 1628 case dh_typecode_void: 1629 info->nr_out = 0; 1630 break; 1631 case dh_typecode_i32: 1632 case dh_typecode_s32: 1633 case dh_typecode_ptr: 1634 info->nr_out = 1; 1635 info->out_kind = TCG_CALL_RET_NORMAL; 1636 break; 1637 case dh_typecode_i64: 1638 case dh_typecode_s64: 1639 info->nr_out = 64 / TCG_TARGET_REG_BITS; 1640 info->out_kind = TCG_CALL_RET_NORMAL; 1641 /* Query the last register now to trigger any assert early. */ 1642 tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1); 1643 break; 1644 case dh_typecode_i128: 1645 info->nr_out = 128 / TCG_TARGET_REG_BITS; 1646 info->out_kind = TCG_TARGET_CALL_RET_I128; 1647 switch (TCG_TARGET_CALL_RET_I128) { 1648 case TCG_CALL_RET_NORMAL: 1649 /* Query the last register now to trigger any assert early. */ 1650 tcg_target_call_oarg_reg(info->out_kind, info->nr_out - 1); 1651 break; 1652 case TCG_CALL_RET_BY_VEC: 1653 /* Query the single register now to trigger any assert early. */ 1654 tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0); 1655 break; 1656 case TCG_CALL_RET_BY_REF: 1657 /* 1658 * Allocate the first argument to the output. 1659 * We don't need to store this anywhere, just make it 1660 * unavailable for use in the input loop below. 1661 */ 1662 cum.arg_slot = 1; 1663 break; 1664 default: 1665 qemu_build_not_reached(); 1666 } 1667 break; 1668 default: 1669 g_assert_not_reached(); 1670 } 1671 1672 /* 1673 * Parse and place function arguments. 1674 */ 1675 for (typemask >>= 3; typemask; typemask >>= 3, cum.arg_idx++) { 1676 TCGCallArgumentKind kind; 1677 TCGType type; 1678 1679 typecode = typemask & 7; 1680 switch (typecode) { 1681 case dh_typecode_i32: 1682 case dh_typecode_s32: 1683 type = TCG_TYPE_I32; 1684 break; 1685 case dh_typecode_i64: 1686 case dh_typecode_s64: 1687 type = TCG_TYPE_I64; 1688 break; 1689 case dh_typecode_ptr: 1690 type = TCG_TYPE_PTR; 1691 break; 1692 case dh_typecode_i128: 1693 type = TCG_TYPE_I128; 1694 break; 1695 default: 1696 g_assert_not_reached(); 1697 } 1698 1699 switch (type) { 1700 case TCG_TYPE_I32: 1701 switch (TCG_TARGET_CALL_ARG_I32) { 1702 case TCG_CALL_ARG_EVEN: 1703 layout_arg_even(&cum); 1704 /* fall through */ 1705 case TCG_CALL_ARG_NORMAL: 1706 layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL); 1707 break; 1708 case TCG_CALL_ARG_EXTEND: 1709 kind = TCG_CALL_ARG_EXTEND_U + (typecode & 1); 1710 layout_arg_1(&cum, info, kind); 1711 break; 1712 default: 1713 qemu_build_not_reached(); 1714 } 1715 break; 1716 1717 case TCG_TYPE_I64: 1718 switch (TCG_TARGET_CALL_ARG_I64) { 1719 case TCG_CALL_ARG_EVEN: 1720 layout_arg_even(&cum); 1721 /* fall through */ 1722 case TCG_CALL_ARG_NORMAL: 1723 if (TCG_TARGET_REG_BITS == 32) { 1724 layout_arg_normal_n(&cum, info, 2); 1725 } else { 1726 layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL); 1727 } 1728 break; 1729 default: 1730 qemu_build_not_reached(); 1731 } 1732 break; 1733 1734 case TCG_TYPE_I128: 1735 switch (TCG_TARGET_CALL_ARG_I128) { 1736 case TCG_CALL_ARG_EVEN: 1737 layout_arg_even(&cum); 1738 /* fall through */ 1739 case TCG_CALL_ARG_NORMAL: 1740 layout_arg_normal_n(&cum, info, 128 / TCG_TARGET_REG_BITS); 1741 break; 1742 case TCG_CALL_ARG_BY_REF: 1743 layout_arg_by_ref(&cum, info); 1744 break; 1745 default: 1746 qemu_build_not_reached(); 1747 } 1748 break; 1749 1750 default: 1751 g_assert_not_reached(); 1752 } 1753 } 1754 info->nr_in = cum.info_in_idx; 1755 1756 /* Validate that we didn't overrun the input array. */ 1757 assert(cum.info_in_idx <= ARRAY_SIZE(info->in)); 1758 /* Validate the backend has enough argument space. */ 1759 assert(cum.arg_slot <= max_reg_slots + max_stk_slots); 1760 1761 /* 1762 * Relocate the "ref_slot" area to the end of the parameters. 1763 * Minimizing this stack offset helps code size for x86, 1764 * which has a signed 8-bit offset encoding. 1765 */ 1766 if (cum.ref_slot != 0) { 1767 int ref_base = 0; 1768 1769 if (cum.arg_slot > max_reg_slots) { 1770 int align = __alignof(Int128) / sizeof(tcg_target_long); 1771 1772 ref_base = cum.arg_slot - max_reg_slots; 1773 if (align > 1) { 1774 ref_base = ROUND_UP(ref_base, align); 1775 } 1776 } 1777 assert(ref_base + cum.ref_slot <= max_stk_slots); 1778 ref_base += max_reg_slots; 1779 1780 if (ref_base != 0) { 1781 for (int i = cum.info_in_idx - 1; i >= 0; --i) { 1782 TCGCallArgumentLoc *loc = &info->in[i]; 1783 switch (loc->kind) { 1784 case TCG_CALL_ARG_BY_REF: 1785 case TCG_CALL_ARG_BY_REF_N: 1786 loc->ref_slot += ref_base; 1787 break; 1788 default: 1789 break; 1790 } 1791 } 1792 } 1793 } 1794 } 1795 1796 static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)]; 1797 static void process_constraint_sets(void); 1798 static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type, 1799 TCGReg reg, const char *name); 1800 1801 static void tcg_context_init(unsigned max_threads) 1802 { 1803 TCGContext *s = &tcg_init_ctx; 1804 int n, i; 1805 TCGTemp *ts; 1806 1807 memset(s, 0, sizeof(*s)); 1808 s->nb_globals = 0; 1809 1810 init_call_layout(&info_helper_ld32_mmu); 1811 init_call_layout(&info_helper_ld64_mmu); 1812 init_call_layout(&info_helper_ld128_mmu); 1813 init_call_layout(&info_helper_st32_mmu); 1814 init_call_layout(&info_helper_st64_mmu); 1815 init_call_layout(&info_helper_st128_mmu); 1816 1817 tcg_target_init(s); 1818 process_constraint_sets(); 1819 1820 /* Reverse the order of the saved registers, assuming they're all at 1821 the start of tcg_target_reg_alloc_order. */ 1822 for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) { 1823 int r = tcg_target_reg_alloc_order[n]; 1824 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) { 1825 break; 1826 } 1827 } 1828 for (i = 0; i < n; ++i) { 1829 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i]; 1830 } 1831 for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) { 1832 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i]; 1833 } 1834 1835 tcg_ctx = s; 1836 /* 1837 * In user-mode we simply share the init context among threads, since we 1838 * use a single region. See the documentation tcg_region_init() for the 1839 * reasoning behind this. 1840 * In system-mode we will have at most max_threads TCG threads. 1841 */ 1842 #ifdef CONFIG_USER_ONLY 1843 tcg_ctxs = &tcg_ctx; 1844 tcg_cur_ctxs = 1; 1845 tcg_max_ctxs = 1; 1846 #else 1847 tcg_max_ctxs = max_threads; 1848 tcg_ctxs = g_new0(TCGContext *, max_threads); 1849 #endif 1850 1851 tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0)); 1852 ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env"); 1853 tcg_env = temp_tcgv_ptr(ts); 1854 } 1855 1856 void tcg_init(size_t tb_size, int splitwx, unsigned max_threads) 1857 { 1858 tcg_context_init(max_threads); 1859 tcg_region_init(tb_size, splitwx, max_threads); 1860 } 1861 1862 /* 1863 * Allocate TBs right before their corresponding translated code, making 1864 * sure that TBs and code are on different cache lines. 1865 */ 1866 TranslationBlock *tcg_tb_alloc(TCGContext *s) 1867 { 1868 uintptr_t align = qemu_icache_linesize; 1869 TranslationBlock *tb; 1870 void *next; 1871 1872 retry: 1873 tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align); 1874 next = (void *)ROUND_UP((uintptr_t)(tb + 1), align); 1875 1876 if (unlikely(next > s->code_gen_highwater)) { 1877 if (tcg_region_alloc(s)) { 1878 return NULL; 1879 } 1880 goto retry; 1881 } 1882 qatomic_set(&s->code_gen_ptr, next); 1883 return tb; 1884 } 1885 1886 void tcg_prologue_init(void) 1887 { 1888 TCGContext *s = tcg_ctx; 1889 size_t prologue_size; 1890 1891 s->code_ptr = s->code_gen_ptr; 1892 s->code_buf = s->code_gen_ptr; 1893 s->data_gen_ptr = NULL; 1894 1895 #ifndef CONFIG_TCG_INTERPRETER 1896 tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr); 1897 #endif 1898 1899 s->pool_labels = NULL; 1900 1901 qemu_thread_jit_write(); 1902 /* Generate the prologue. */ 1903 tcg_target_qemu_prologue(s); 1904 1905 /* Allow the prologue to put e.g. guest_base into a pool entry. */ 1906 { 1907 int result = tcg_out_pool_finalize(s); 1908 tcg_debug_assert(result == 0); 1909 } 1910 1911 prologue_size = tcg_current_code_size(s); 1912 perf_report_prologue(s->code_gen_ptr, prologue_size); 1913 1914 #ifndef CONFIG_TCG_INTERPRETER 1915 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf), 1916 (uintptr_t)s->code_buf, prologue_size); 1917 #endif 1918 1919 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) { 1920 FILE *logfile = qemu_log_trylock(); 1921 if (logfile) { 1922 fprintf(logfile, "PROLOGUE: [size=%zu]\n", prologue_size); 1923 if (s->data_gen_ptr) { 1924 size_t code_size = s->data_gen_ptr - s->code_gen_ptr; 1925 size_t data_size = prologue_size - code_size; 1926 size_t i; 1927 1928 disas(logfile, s->code_gen_ptr, code_size); 1929 1930 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) { 1931 if (sizeof(tcg_target_ulong) == 8) { 1932 fprintf(logfile, 1933 "0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n", 1934 (uintptr_t)s->data_gen_ptr + i, 1935 *(uint64_t *)(s->data_gen_ptr + i)); 1936 } else { 1937 fprintf(logfile, 1938 "0x%08" PRIxPTR ": .long 0x%08x\n", 1939 (uintptr_t)s->data_gen_ptr + i, 1940 *(uint32_t *)(s->data_gen_ptr + i)); 1941 } 1942 } 1943 } else { 1944 disas(logfile, s->code_gen_ptr, prologue_size); 1945 } 1946 fprintf(logfile, "\n"); 1947 qemu_log_unlock(logfile); 1948 } 1949 } 1950 1951 #ifndef CONFIG_TCG_INTERPRETER 1952 /* 1953 * Assert that goto_ptr is implemented completely, setting an epilogue. 1954 * For tci, we use NULL as the signal to return from the interpreter, 1955 * so skip this check. 1956 */ 1957 tcg_debug_assert(tcg_code_gen_epilogue != NULL); 1958 #endif 1959 1960 tcg_region_prologue_set(s); 1961 } 1962 1963 void tcg_func_start(TCGContext *s) 1964 { 1965 tcg_pool_reset(s); 1966 s->nb_temps = s->nb_globals; 1967 1968 /* No temps have been previously allocated for size or locality. */ 1969 tcg_temp_ebb_reset_freed(s); 1970 1971 /* No constant temps have been previously allocated. */ 1972 for (int i = 0; i < TCG_TYPE_COUNT; ++i) { 1973 if (s->const_table[i]) { 1974 g_hash_table_remove_all(s->const_table[i]); 1975 } 1976 } 1977 1978 s->nb_ops = 0; 1979 s->nb_labels = 0; 1980 s->current_frame_offset = s->frame_start; 1981 1982 #ifdef CONFIG_DEBUG_TCG 1983 s->goto_tb_issue_mask = 0; 1984 #endif 1985 1986 QTAILQ_INIT(&s->ops); 1987 QTAILQ_INIT(&s->free_ops); 1988 s->emit_before_op = NULL; 1989 QSIMPLEQ_INIT(&s->labels); 1990 1991 tcg_debug_assert(s->addr_type <= TCG_TYPE_REG); 1992 } 1993 1994 static TCGTemp *tcg_temp_alloc(TCGContext *s) 1995 { 1996 int n = s->nb_temps++; 1997 1998 if (n >= TCG_MAX_TEMPS) { 1999 tcg_raise_tb_overflow(s); 2000 } 2001 return memset(&s->temps[n], 0, sizeof(TCGTemp)); 2002 } 2003 2004 static TCGTemp *tcg_global_alloc(TCGContext *s) 2005 { 2006 TCGTemp *ts; 2007 2008 tcg_debug_assert(s->nb_globals == s->nb_temps); 2009 tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS); 2010 s->nb_globals++; 2011 ts = tcg_temp_alloc(s); 2012 ts->kind = TEMP_GLOBAL; 2013 2014 return ts; 2015 } 2016 2017 static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type, 2018 TCGReg reg, const char *name) 2019 { 2020 TCGTemp *ts; 2021 2022 tcg_debug_assert(TCG_TARGET_REG_BITS == 64 || type == TCG_TYPE_I32); 2023 2024 ts = tcg_global_alloc(s); 2025 ts->base_type = type; 2026 ts->type = type; 2027 ts->kind = TEMP_FIXED; 2028 ts->reg = reg; 2029 ts->name = name; 2030 tcg_regset_set_reg(s->reserved_regs, reg); 2031 2032 return ts; 2033 } 2034 2035 void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size) 2036 { 2037 s->frame_start = start; 2038 s->frame_end = start + size; 2039 s->frame_temp 2040 = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame"); 2041 } 2042 2043 static TCGTemp *tcg_global_mem_new_internal(TCGv_ptr base, intptr_t offset, 2044 const char *name, TCGType type) 2045 { 2046 TCGContext *s = tcg_ctx; 2047 TCGTemp *base_ts = tcgv_ptr_temp(base); 2048 TCGTemp *ts = tcg_global_alloc(s); 2049 int indirect_reg = 0; 2050 2051 switch (base_ts->kind) { 2052 case TEMP_FIXED: 2053 break; 2054 case TEMP_GLOBAL: 2055 /* We do not support double-indirect registers. */ 2056 tcg_debug_assert(!base_ts->indirect_reg); 2057 base_ts->indirect_base = 1; 2058 s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64 2059 ? 2 : 1); 2060 indirect_reg = 1; 2061 break; 2062 default: 2063 g_assert_not_reached(); 2064 } 2065 2066 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 2067 TCGTemp *ts2 = tcg_global_alloc(s); 2068 char buf[64]; 2069 2070 ts->base_type = TCG_TYPE_I64; 2071 ts->type = TCG_TYPE_I32; 2072 ts->indirect_reg = indirect_reg; 2073 ts->mem_allocated = 1; 2074 ts->mem_base = base_ts; 2075 ts->mem_offset = offset; 2076 pstrcpy(buf, sizeof(buf), name); 2077 pstrcat(buf, sizeof(buf), "_0"); 2078 ts->name = strdup(buf); 2079 2080 tcg_debug_assert(ts2 == ts + 1); 2081 ts2->base_type = TCG_TYPE_I64; 2082 ts2->type = TCG_TYPE_I32; 2083 ts2->indirect_reg = indirect_reg; 2084 ts2->mem_allocated = 1; 2085 ts2->mem_base = base_ts; 2086 ts2->mem_offset = offset + 4; 2087 ts2->temp_subindex = 1; 2088 pstrcpy(buf, sizeof(buf), name); 2089 pstrcat(buf, sizeof(buf), "_1"); 2090 ts2->name = strdup(buf); 2091 } else { 2092 ts->base_type = type; 2093 ts->type = type; 2094 ts->indirect_reg = indirect_reg; 2095 ts->mem_allocated = 1; 2096 ts->mem_base = base_ts; 2097 ts->mem_offset = offset; 2098 ts->name = name; 2099 } 2100 return ts; 2101 } 2102 2103 TCGv_i32 tcg_global_mem_new_i32(TCGv_ptr reg, intptr_t off, const char *name) 2104 { 2105 TCGTemp *ts = tcg_global_mem_new_internal(reg, off, name, TCG_TYPE_I32); 2106 return temp_tcgv_i32(ts); 2107 } 2108 2109 TCGv_i64 tcg_global_mem_new_i64(TCGv_ptr reg, intptr_t off, const char *name) 2110 { 2111 TCGTemp *ts = tcg_global_mem_new_internal(reg, off, name, TCG_TYPE_I64); 2112 return temp_tcgv_i64(ts); 2113 } 2114 2115 TCGv_ptr tcg_global_mem_new_ptr(TCGv_ptr reg, intptr_t off, const char *name) 2116 { 2117 TCGTemp *ts = tcg_global_mem_new_internal(reg, off, name, TCG_TYPE_PTR); 2118 return temp_tcgv_ptr(ts); 2119 } 2120 2121 TCGTemp *tcg_temp_new_internal(TCGType type, TCGTempKind kind) 2122 { 2123 TCGContext *s = tcg_ctx; 2124 TCGTemp *ts; 2125 int n; 2126 2127 if (kind == TEMP_EBB) { 2128 int idx = find_first_bit(s->free_temps[type].l, TCG_MAX_TEMPS); 2129 2130 if (idx < TCG_MAX_TEMPS) { 2131 /* There is already an available temp with the right type. */ 2132 clear_bit(idx, s->free_temps[type].l); 2133 2134 ts = &s->temps[idx]; 2135 ts->temp_allocated = 1; 2136 tcg_debug_assert(ts->base_type == type); 2137 tcg_debug_assert(ts->kind == kind); 2138 return ts; 2139 } 2140 } else { 2141 tcg_debug_assert(kind == TEMP_TB); 2142 } 2143 2144 switch (type) { 2145 case TCG_TYPE_I32: 2146 case TCG_TYPE_V64: 2147 case TCG_TYPE_V128: 2148 case TCG_TYPE_V256: 2149 n = 1; 2150 break; 2151 case TCG_TYPE_I64: 2152 n = 64 / TCG_TARGET_REG_BITS; 2153 break; 2154 case TCG_TYPE_I128: 2155 n = 128 / TCG_TARGET_REG_BITS; 2156 break; 2157 default: 2158 g_assert_not_reached(); 2159 } 2160 2161 ts = tcg_temp_alloc(s); 2162 ts->base_type = type; 2163 ts->temp_allocated = 1; 2164 ts->kind = kind; 2165 2166 if (n == 1) { 2167 ts->type = type; 2168 } else { 2169 ts->type = TCG_TYPE_REG; 2170 2171 for (int i = 1; i < n; ++i) { 2172 TCGTemp *ts2 = tcg_temp_alloc(s); 2173 2174 tcg_debug_assert(ts2 == ts + i); 2175 ts2->base_type = type; 2176 ts2->type = TCG_TYPE_REG; 2177 ts2->temp_allocated = 1; 2178 ts2->temp_subindex = i; 2179 ts2->kind = kind; 2180 } 2181 } 2182 return ts; 2183 } 2184 2185 TCGv_i32 tcg_temp_new_i32(void) 2186 { 2187 return temp_tcgv_i32(tcg_temp_new_internal(TCG_TYPE_I32, TEMP_TB)); 2188 } 2189 2190 TCGv_i32 tcg_temp_ebb_new_i32(void) 2191 { 2192 return temp_tcgv_i32(tcg_temp_new_internal(TCG_TYPE_I32, TEMP_EBB)); 2193 } 2194 2195 TCGv_i64 tcg_temp_new_i64(void) 2196 { 2197 return temp_tcgv_i64(tcg_temp_new_internal(TCG_TYPE_I64, TEMP_TB)); 2198 } 2199 2200 TCGv_i64 tcg_temp_ebb_new_i64(void) 2201 { 2202 return temp_tcgv_i64(tcg_temp_new_internal(TCG_TYPE_I64, TEMP_EBB)); 2203 } 2204 2205 TCGv_ptr tcg_temp_new_ptr(void) 2206 { 2207 return temp_tcgv_ptr(tcg_temp_new_internal(TCG_TYPE_PTR, TEMP_TB)); 2208 } 2209 2210 TCGv_ptr tcg_temp_ebb_new_ptr(void) 2211 { 2212 return temp_tcgv_ptr(tcg_temp_new_internal(TCG_TYPE_PTR, TEMP_EBB)); 2213 } 2214 2215 TCGv_i128 tcg_temp_new_i128(void) 2216 { 2217 return temp_tcgv_i128(tcg_temp_new_internal(TCG_TYPE_I128, TEMP_TB)); 2218 } 2219 2220 TCGv_i128 tcg_temp_ebb_new_i128(void) 2221 { 2222 return temp_tcgv_i128(tcg_temp_new_internal(TCG_TYPE_I128, TEMP_EBB)); 2223 } 2224 2225 TCGv_vec tcg_temp_new_vec(TCGType type) 2226 { 2227 TCGTemp *t; 2228 2229 #ifdef CONFIG_DEBUG_TCG 2230 switch (type) { 2231 case TCG_TYPE_V64: 2232 assert(TCG_TARGET_HAS_v64); 2233 break; 2234 case TCG_TYPE_V128: 2235 assert(TCG_TARGET_HAS_v128); 2236 break; 2237 case TCG_TYPE_V256: 2238 assert(TCG_TARGET_HAS_v256); 2239 break; 2240 default: 2241 g_assert_not_reached(); 2242 } 2243 #endif 2244 2245 t = tcg_temp_new_internal(type, TEMP_EBB); 2246 return temp_tcgv_vec(t); 2247 } 2248 2249 /* Create a new temp of the same type as an existing temp. */ 2250 TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match) 2251 { 2252 TCGTemp *t = tcgv_vec_temp(match); 2253 2254 tcg_debug_assert(t->temp_allocated != 0); 2255 2256 t = tcg_temp_new_internal(t->base_type, TEMP_EBB); 2257 return temp_tcgv_vec(t); 2258 } 2259 2260 void tcg_temp_free_internal(TCGTemp *ts) 2261 { 2262 TCGContext *s = tcg_ctx; 2263 2264 switch (ts->kind) { 2265 case TEMP_CONST: 2266 case TEMP_TB: 2267 /* Silently ignore free. */ 2268 break; 2269 case TEMP_EBB: 2270 tcg_debug_assert(ts->temp_allocated != 0); 2271 ts->temp_allocated = 0; 2272 set_bit(temp_idx(ts), s->free_temps[ts->base_type].l); 2273 break; 2274 default: 2275 /* It never made sense to free TEMP_FIXED or TEMP_GLOBAL. */ 2276 g_assert_not_reached(); 2277 } 2278 } 2279 2280 void tcg_temp_free_i32(TCGv_i32 arg) 2281 { 2282 tcg_temp_free_internal(tcgv_i32_temp(arg)); 2283 } 2284 2285 void tcg_temp_free_i64(TCGv_i64 arg) 2286 { 2287 tcg_temp_free_internal(tcgv_i64_temp(arg)); 2288 } 2289 2290 void tcg_temp_free_i128(TCGv_i128 arg) 2291 { 2292 tcg_temp_free_internal(tcgv_i128_temp(arg)); 2293 } 2294 2295 void tcg_temp_free_ptr(TCGv_ptr arg) 2296 { 2297 tcg_temp_free_internal(tcgv_ptr_temp(arg)); 2298 } 2299 2300 void tcg_temp_free_vec(TCGv_vec arg) 2301 { 2302 tcg_temp_free_internal(tcgv_vec_temp(arg)); 2303 } 2304 2305 TCGTemp *tcg_constant_internal(TCGType type, int64_t val) 2306 { 2307 TCGContext *s = tcg_ctx; 2308 GHashTable *h = s->const_table[type]; 2309 TCGTemp *ts; 2310 2311 if (h == NULL) { 2312 h = g_hash_table_new(g_int64_hash, g_int64_equal); 2313 s->const_table[type] = h; 2314 } 2315 2316 ts = g_hash_table_lookup(h, &val); 2317 if (ts == NULL) { 2318 int64_t *val_ptr; 2319 2320 ts = tcg_temp_alloc(s); 2321 2322 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { 2323 TCGTemp *ts2 = tcg_temp_alloc(s); 2324 2325 tcg_debug_assert(ts2 == ts + 1); 2326 2327 ts->base_type = TCG_TYPE_I64; 2328 ts->type = TCG_TYPE_I32; 2329 ts->kind = TEMP_CONST; 2330 ts->temp_allocated = 1; 2331 2332 ts2->base_type = TCG_TYPE_I64; 2333 ts2->type = TCG_TYPE_I32; 2334 ts2->kind = TEMP_CONST; 2335 ts2->temp_allocated = 1; 2336 ts2->temp_subindex = 1; 2337 2338 /* 2339 * Retain the full value of the 64-bit constant in the low 2340 * part, so that the hash table works. Actual uses will 2341 * truncate the value to the low part. 2342 */ 2343 ts[HOST_BIG_ENDIAN].val = val; 2344 ts[!HOST_BIG_ENDIAN].val = val >> 32; 2345 val_ptr = &ts[HOST_BIG_ENDIAN].val; 2346 } else { 2347 ts->base_type = type; 2348 ts->type = type; 2349 ts->kind = TEMP_CONST; 2350 ts->temp_allocated = 1; 2351 ts->val = val; 2352 val_ptr = &ts->val; 2353 } 2354 g_hash_table_insert(h, val_ptr, ts); 2355 } 2356 2357 return ts; 2358 } 2359 2360 TCGv_i32 tcg_constant_i32(int32_t val) 2361 { 2362 return temp_tcgv_i32(tcg_constant_internal(TCG_TYPE_I32, val)); 2363 } 2364 2365 TCGv_i64 tcg_constant_i64(int64_t val) 2366 { 2367 return temp_tcgv_i64(tcg_constant_internal(TCG_TYPE_I64, val)); 2368 } 2369 2370 TCGv_ptr tcg_constant_ptr_int(intptr_t val) 2371 { 2372 return temp_tcgv_ptr(tcg_constant_internal(TCG_TYPE_PTR, val)); 2373 } 2374 2375 TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val) 2376 { 2377 val = dup_const(vece, val); 2378 return temp_tcgv_vec(tcg_constant_internal(type, val)); 2379 } 2380 2381 TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val) 2382 { 2383 TCGTemp *t = tcgv_vec_temp(match); 2384 2385 tcg_debug_assert(t->temp_allocated != 0); 2386 return tcg_constant_vec(t->base_type, vece, val); 2387 } 2388 2389 #ifdef CONFIG_DEBUG_TCG 2390 size_t temp_idx(TCGTemp *ts) 2391 { 2392 ptrdiff_t n = ts - tcg_ctx->temps; 2393 assert(n >= 0 && n < tcg_ctx->nb_temps); 2394 return n; 2395 } 2396 2397 TCGTemp *tcgv_i32_temp(TCGv_i32 v) 2398 { 2399 uintptr_t o = (uintptr_t)v - offsetof(TCGContext, temps); 2400 2401 assert(o < sizeof(TCGTemp) * tcg_ctx->nb_temps); 2402 assert(o % sizeof(TCGTemp) == 0); 2403 2404 return (void *)tcg_ctx + (uintptr_t)v; 2405 } 2406 #endif /* CONFIG_DEBUG_TCG */ 2407 2408 /* 2409 * Return true if OP may appear in the opcode stream with TYPE. 2410 * Test the runtime variable that controls each opcode. 2411 */ 2412 bool tcg_op_supported(TCGOpcode op, TCGType type, unsigned flags) 2413 { 2414 bool has_type; 2415 2416 switch (type) { 2417 case TCG_TYPE_I32: 2418 has_type = true; 2419 break; 2420 case TCG_TYPE_I64: 2421 has_type = TCG_TARGET_REG_BITS == 64; 2422 break; 2423 case TCG_TYPE_V64: 2424 has_type = TCG_TARGET_HAS_v64; 2425 break; 2426 case TCG_TYPE_V128: 2427 has_type = TCG_TARGET_HAS_v128; 2428 break; 2429 case TCG_TYPE_V256: 2430 has_type = TCG_TARGET_HAS_v256; 2431 break; 2432 default: 2433 has_type = false; 2434 break; 2435 } 2436 2437 switch (op) { 2438 case INDEX_op_discard: 2439 case INDEX_op_set_label: 2440 case INDEX_op_call: 2441 case INDEX_op_br: 2442 case INDEX_op_mb: 2443 case INDEX_op_insn_start: 2444 case INDEX_op_exit_tb: 2445 case INDEX_op_goto_tb: 2446 case INDEX_op_goto_ptr: 2447 return true; 2448 2449 case INDEX_op_qemu_ld: 2450 case INDEX_op_qemu_st: 2451 tcg_debug_assert(type <= TCG_TYPE_REG); 2452 return true; 2453 2454 case INDEX_op_qemu_ld2: 2455 case INDEX_op_qemu_st2: 2456 if (TCG_TARGET_REG_BITS == 32) { 2457 tcg_debug_assert(type == TCG_TYPE_I64); 2458 return true; 2459 } 2460 tcg_debug_assert(type == TCG_TYPE_I128); 2461 goto do_lookup; 2462 2463 case INDEX_op_add: 2464 case INDEX_op_and: 2465 case INDEX_op_brcond: 2466 case INDEX_op_deposit: 2467 case INDEX_op_extract: 2468 case INDEX_op_ld8u: 2469 case INDEX_op_ld8s: 2470 case INDEX_op_ld16u: 2471 case INDEX_op_ld16s: 2472 case INDEX_op_ld: 2473 case INDEX_op_mov: 2474 case INDEX_op_movcond: 2475 case INDEX_op_negsetcond: 2476 case INDEX_op_or: 2477 case INDEX_op_setcond: 2478 case INDEX_op_sextract: 2479 case INDEX_op_st8: 2480 case INDEX_op_st16: 2481 case INDEX_op_st: 2482 case INDEX_op_xor: 2483 return has_type; 2484 2485 case INDEX_op_brcond2_i32: 2486 case INDEX_op_setcond2_i32: 2487 return TCG_TARGET_REG_BITS == 32; 2488 2489 case INDEX_op_ld32u: 2490 case INDEX_op_ld32s: 2491 case INDEX_op_st32: 2492 case INDEX_op_ext_i32_i64: 2493 case INDEX_op_extu_i32_i64: 2494 case INDEX_op_extrl_i64_i32: 2495 case INDEX_op_extrh_i64_i32: 2496 return TCG_TARGET_REG_BITS == 64; 2497 2498 case INDEX_op_mov_vec: 2499 case INDEX_op_dup_vec: 2500 case INDEX_op_dupm_vec: 2501 case INDEX_op_ld_vec: 2502 case INDEX_op_st_vec: 2503 case INDEX_op_add_vec: 2504 case INDEX_op_sub_vec: 2505 case INDEX_op_and_vec: 2506 case INDEX_op_or_vec: 2507 case INDEX_op_xor_vec: 2508 case INDEX_op_cmp_vec: 2509 return has_type; 2510 case INDEX_op_dup2_vec: 2511 return has_type && TCG_TARGET_REG_BITS == 32; 2512 case INDEX_op_not_vec: 2513 return has_type && TCG_TARGET_HAS_not_vec; 2514 case INDEX_op_neg_vec: 2515 return has_type && TCG_TARGET_HAS_neg_vec; 2516 case INDEX_op_abs_vec: 2517 return has_type && TCG_TARGET_HAS_abs_vec; 2518 case INDEX_op_andc_vec: 2519 return has_type && TCG_TARGET_HAS_andc_vec; 2520 case INDEX_op_orc_vec: 2521 return has_type && TCG_TARGET_HAS_orc_vec; 2522 case INDEX_op_nand_vec: 2523 return has_type && TCG_TARGET_HAS_nand_vec; 2524 case INDEX_op_nor_vec: 2525 return has_type && TCG_TARGET_HAS_nor_vec; 2526 case INDEX_op_eqv_vec: 2527 return has_type && TCG_TARGET_HAS_eqv_vec; 2528 case INDEX_op_mul_vec: 2529 return has_type && TCG_TARGET_HAS_mul_vec; 2530 case INDEX_op_shli_vec: 2531 case INDEX_op_shri_vec: 2532 case INDEX_op_sari_vec: 2533 return has_type && TCG_TARGET_HAS_shi_vec; 2534 case INDEX_op_shls_vec: 2535 case INDEX_op_shrs_vec: 2536 case INDEX_op_sars_vec: 2537 return has_type && TCG_TARGET_HAS_shs_vec; 2538 case INDEX_op_shlv_vec: 2539 case INDEX_op_shrv_vec: 2540 case INDEX_op_sarv_vec: 2541 return has_type && TCG_TARGET_HAS_shv_vec; 2542 case INDEX_op_rotli_vec: 2543 return has_type && TCG_TARGET_HAS_roti_vec; 2544 case INDEX_op_rotls_vec: 2545 return has_type && TCG_TARGET_HAS_rots_vec; 2546 case INDEX_op_rotlv_vec: 2547 case INDEX_op_rotrv_vec: 2548 return has_type && TCG_TARGET_HAS_rotv_vec; 2549 case INDEX_op_ssadd_vec: 2550 case INDEX_op_usadd_vec: 2551 case INDEX_op_sssub_vec: 2552 case INDEX_op_ussub_vec: 2553 return has_type && TCG_TARGET_HAS_sat_vec; 2554 case INDEX_op_smin_vec: 2555 case INDEX_op_umin_vec: 2556 case INDEX_op_smax_vec: 2557 case INDEX_op_umax_vec: 2558 return has_type && TCG_TARGET_HAS_minmax_vec; 2559 case INDEX_op_bitsel_vec: 2560 return has_type && TCG_TARGET_HAS_bitsel_vec; 2561 case INDEX_op_cmpsel_vec: 2562 return has_type && TCG_TARGET_HAS_cmpsel_vec; 2563 2564 default: 2565 if (op < INDEX_op_last_generic) { 2566 const TCGOutOp *outop; 2567 TCGConstraintSetIndex con_set; 2568 2569 if (!has_type) { 2570 return false; 2571 } 2572 2573 do_lookup: 2574 outop = all_outop[op]; 2575 tcg_debug_assert(outop != NULL); 2576 2577 con_set = outop->static_constraint; 2578 if (con_set == C_Dynamic) { 2579 con_set = outop->dynamic_constraint(type, flags); 2580 } 2581 if (con_set >= 0) { 2582 return true; 2583 } 2584 tcg_debug_assert(con_set == C_NotImplemented); 2585 return false; 2586 } 2587 tcg_debug_assert(op < NB_OPS); 2588 return true; 2589 2590 case INDEX_op_last_generic: 2591 g_assert_not_reached(); 2592 } 2593 } 2594 2595 bool tcg_op_deposit_valid(TCGType type, unsigned ofs, unsigned len) 2596 { 2597 unsigned width; 2598 2599 tcg_debug_assert(type == TCG_TYPE_I32 || type == TCG_TYPE_I64); 2600 width = (type == TCG_TYPE_I32 ? 32 : 64); 2601 2602 tcg_debug_assert(ofs < width); 2603 tcg_debug_assert(len > 0); 2604 tcg_debug_assert(len <= width - ofs); 2605 2606 return TCG_TARGET_deposit_valid(type, ofs, len); 2607 } 2608 2609 static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs); 2610 2611 static void tcg_gen_callN(void *func, TCGHelperInfo *info, 2612 TCGTemp *ret, TCGTemp **args) 2613 { 2614 TCGv_i64 extend_free[MAX_CALL_IARGS]; 2615 int n_extend = 0; 2616 TCGOp *op; 2617 int i, n, pi = 0, total_args; 2618 2619 if (unlikely(g_once_init_enter(HELPER_INFO_INIT(info)))) { 2620 init_call_layout(info); 2621 g_once_init_leave(HELPER_INFO_INIT(info), HELPER_INFO_INIT_VAL(info)); 2622 } 2623 2624 total_args = info->nr_out + info->nr_in + 2; 2625 op = tcg_op_alloc(INDEX_op_call, total_args); 2626 2627 #ifdef CONFIG_PLUGIN 2628 /* Flag helpers that may affect guest state */ 2629 if (tcg_ctx->plugin_insn && !(info->flags & TCG_CALL_NO_SIDE_EFFECTS)) { 2630 tcg_ctx->plugin_insn->calls_helpers = true; 2631 } 2632 #endif 2633 2634 TCGOP_CALLO(op) = n = info->nr_out; 2635 switch (n) { 2636 case 0: 2637 tcg_debug_assert(ret == NULL); 2638 break; 2639 case 1: 2640 tcg_debug_assert(ret != NULL); 2641 op->args[pi++] = temp_arg(ret); 2642 break; 2643 case 2: 2644 case 4: 2645 tcg_debug_assert(ret != NULL); 2646 tcg_debug_assert(ret->base_type == ret->type + ctz32(n)); 2647 tcg_debug_assert(ret->temp_subindex == 0); 2648 for (i = 0; i < n; ++i) { 2649 op->args[pi++] = temp_arg(ret + i); 2650 } 2651 break; 2652 default: 2653 g_assert_not_reached(); 2654 } 2655 2656 TCGOP_CALLI(op) = n = info->nr_in; 2657 for (i = 0; i < n; i++) { 2658 const TCGCallArgumentLoc *loc = &info->in[i]; 2659 TCGTemp *ts = args[loc->arg_idx] + loc->tmp_subindex; 2660 2661 switch (loc->kind) { 2662 case TCG_CALL_ARG_NORMAL: 2663 case TCG_CALL_ARG_BY_REF: 2664 case TCG_CALL_ARG_BY_REF_N: 2665 op->args[pi++] = temp_arg(ts); 2666 break; 2667 2668 case TCG_CALL_ARG_EXTEND_U: 2669 case TCG_CALL_ARG_EXTEND_S: 2670 { 2671 TCGv_i64 temp = tcg_temp_ebb_new_i64(); 2672 TCGv_i32 orig = temp_tcgv_i32(ts); 2673 2674 if (loc->kind == TCG_CALL_ARG_EXTEND_S) { 2675 tcg_gen_ext_i32_i64(temp, orig); 2676 } else { 2677 tcg_gen_extu_i32_i64(temp, orig); 2678 } 2679 op->args[pi++] = tcgv_i64_arg(temp); 2680 extend_free[n_extend++] = temp; 2681 } 2682 break; 2683 2684 default: 2685 g_assert_not_reached(); 2686 } 2687 } 2688 op->args[pi++] = (uintptr_t)func; 2689 op->args[pi++] = (uintptr_t)info; 2690 tcg_debug_assert(pi == total_args); 2691 2692 if (tcg_ctx->emit_before_op) { 2693 QTAILQ_INSERT_BEFORE(tcg_ctx->emit_before_op, op, link); 2694 } else { 2695 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link); 2696 } 2697 2698 tcg_debug_assert(n_extend < ARRAY_SIZE(extend_free)); 2699 for (i = 0; i < n_extend; ++i) { 2700 tcg_temp_free_i64(extend_free[i]); 2701 } 2702 } 2703 2704 void tcg_gen_call0(void *func, TCGHelperInfo *info, TCGTemp *ret) 2705 { 2706 tcg_gen_callN(func, info, ret, NULL); 2707 } 2708 2709 void tcg_gen_call1(void *func, TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1) 2710 { 2711 tcg_gen_callN(func, info, ret, &t1); 2712 } 2713 2714 void tcg_gen_call2(void *func, TCGHelperInfo *info, TCGTemp *ret, 2715 TCGTemp *t1, TCGTemp *t2) 2716 { 2717 TCGTemp *args[2] = { t1, t2 }; 2718 tcg_gen_callN(func, info, ret, args); 2719 } 2720 2721 void tcg_gen_call3(void *func, TCGHelperInfo *info, TCGTemp *ret, 2722 TCGTemp *t1, TCGTemp *t2, TCGTemp *t3) 2723 { 2724 TCGTemp *args[3] = { t1, t2, t3 }; 2725 tcg_gen_callN(func, info, ret, args); 2726 } 2727 2728 void tcg_gen_call4(void *func, TCGHelperInfo *info, TCGTemp *ret, 2729 TCGTemp *t1, TCGTemp *t2, TCGTemp *t3, TCGTemp *t4) 2730 { 2731 TCGTemp *args[4] = { t1, t2, t3, t4 }; 2732 tcg_gen_callN(func, info, ret, args); 2733 } 2734 2735 void tcg_gen_call5(void *func, TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1, 2736 TCGTemp *t2, TCGTemp *t3, TCGTemp *t4, TCGTemp *t5) 2737 { 2738 TCGTemp *args[5] = { t1, t2, t3, t4, t5 }; 2739 tcg_gen_callN(func, info, ret, args); 2740 } 2741 2742 void tcg_gen_call6(void *func, TCGHelperInfo *info, TCGTemp *ret, 2743 TCGTemp *t1, TCGTemp *t2, TCGTemp *t3, 2744 TCGTemp *t4, TCGTemp *t5, TCGTemp *t6) 2745 { 2746 TCGTemp *args[6] = { t1, t2, t3, t4, t5, t6 }; 2747 tcg_gen_callN(func, info, ret, args); 2748 } 2749 2750 void tcg_gen_call7(void *func, TCGHelperInfo *info, TCGTemp *ret, TCGTemp *t1, 2751 TCGTemp *t2, TCGTemp *t3, TCGTemp *t4, 2752 TCGTemp *t5, TCGTemp *t6, TCGTemp *t7) 2753 { 2754 TCGTemp *args[7] = { t1, t2, t3, t4, t5, t6, t7 }; 2755 tcg_gen_callN(func, info, ret, args); 2756 } 2757 2758 static void tcg_reg_alloc_start(TCGContext *s) 2759 { 2760 int i, n; 2761 2762 for (i = 0, n = s->nb_temps; i < n; i++) { 2763 TCGTemp *ts = &s->temps[i]; 2764 TCGTempVal val = TEMP_VAL_MEM; 2765 2766 switch (ts->kind) { 2767 case TEMP_CONST: 2768 val = TEMP_VAL_CONST; 2769 break; 2770 case TEMP_FIXED: 2771 val = TEMP_VAL_REG; 2772 break; 2773 case TEMP_GLOBAL: 2774 break; 2775 case TEMP_EBB: 2776 val = TEMP_VAL_DEAD; 2777 /* fall through */ 2778 case TEMP_TB: 2779 ts->mem_allocated = 0; 2780 break; 2781 default: 2782 g_assert_not_reached(); 2783 } 2784 ts->val_type = val; 2785 } 2786 2787 memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp)); 2788 } 2789 2790 static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size, 2791 TCGTemp *ts) 2792 { 2793 int idx = temp_idx(ts); 2794 2795 switch (ts->kind) { 2796 case TEMP_FIXED: 2797 case TEMP_GLOBAL: 2798 pstrcpy(buf, buf_size, ts->name); 2799 break; 2800 case TEMP_TB: 2801 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals); 2802 break; 2803 case TEMP_EBB: 2804 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals); 2805 break; 2806 case TEMP_CONST: 2807 switch (ts->type) { 2808 case TCG_TYPE_I32: 2809 snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val); 2810 break; 2811 #if TCG_TARGET_REG_BITS > 32 2812 case TCG_TYPE_I64: 2813 snprintf(buf, buf_size, "$0x%" PRIx64, ts->val); 2814 break; 2815 #endif 2816 case TCG_TYPE_V64: 2817 case TCG_TYPE_V128: 2818 case TCG_TYPE_V256: 2819 snprintf(buf, buf_size, "v%d$0x%" PRIx64, 2820 64 << (ts->type - TCG_TYPE_V64), ts->val); 2821 break; 2822 default: 2823 g_assert_not_reached(); 2824 } 2825 break; 2826 } 2827 return buf; 2828 } 2829 2830 static char *tcg_get_arg_str(TCGContext *s, char *buf, 2831 int buf_size, TCGArg arg) 2832 { 2833 return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg)); 2834 } 2835 2836 static const char * const cond_name[] = 2837 { 2838 [TCG_COND_NEVER] = "never", 2839 [TCG_COND_ALWAYS] = "always", 2840 [TCG_COND_EQ] = "eq", 2841 [TCG_COND_NE] = "ne", 2842 [TCG_COND_LT] = "lt", 2843 [TCG_COND_GE] = "ge", 2844 [TCG_COND_LE] = "le", 2845 [TCG_COND_GT] = "gt", 2846 [TCG_COND_LTU] = "ltu", 2847 [TCG_COND_GEU] = "geu", 2848 [TCG_COND_LEU] = "leu", 2849 [TCG_COND_GTU] = "gtu", 2850 [TCG_COND_TSTEQ] = "tsteq", 2851 [TCG_COND_TSTNE] = "tstne", 2852 }; 2853 2854 static const char * const ldst_name[(MO_BSWAP | MO_SSIZE) + 1] = 2855 { 2856 [MO_UB] = "ub", 2857 [MO_SB] = "sb", 2858 [MO_LEUW] = "leuw", 2859 [MO_LESW] = "lesw", 2860 [MO_LEUL] = "leul", 2861 [MO_LESL] = "lesl", 2862 [MO_LEUQ] = "leq", 2863 [MO_BEUW] = "beuw", 2864 [MO_BESW] = "besw", 2865 [MO_BEUL] = "beul", 2866 [MO_BESL] = "besl", 2867 [MO_BEUQ] = "beq", 2868 [MO_128 + MO_BE] = "beo", 2869 [MO_128 + MO_LE] = "leo", 2870 }; 2871 2872 static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = { 2873 [MO_UNALN >> MO_ASHIFT] = "un+", 2874 [MO_ALIGN >> MO_ASHIFT] = "al+", 2875 [MO_ALIGN_2 >> MO_ASHIFT] = "al2+", 2876 [MO_ALIGN_4 >> MO_ASHIFT] = "al4+", 2877 [MO_ALIGN_8 >> MO_ASHIFT] = "al8+", 2878 [MO_ALIGN_16 >> MO_ASHIFT] = "al16+", 2879 [MO_ALIGN_32 >> MO_ASHIFT] = "al32+", 2880 [MO_ALIGN_64 >> MO_ASHIFT] = "al64+", 2881 }; 2882 2883 static const char * const atom_name[(MO_ATOM_MASK >> MO_ATOM_SHIFT) + 1] = { 2884 [MO_ATOM_IFALIGN >> MO_ATOM_SHIFT] = "", 2885 [MO_ATOM_IFALIGN_PAIR >> MO_ATOM_SHIFT] = "pair+", 2886 [MO_ATOM_WITHIN16 >> MO_ATOM_SHIFT] = "w16+", 2887 [MO_ATOM_WITHIN16_PAIR >> MO_ATOM_SHIFT] = "w16p+", 2888 [MO_ATOM_SUBALIGN >> MO_ATOM_SHIFT] = "sub+", 2889 [MO_ATOM_NONE >> MO_ATOM_SHIFT] = "noat+", 2890 }; 2891 2892 static const char bswap_flag_name[][6] = { 2893 [TCG_BSWAP_IZ] = "iz", 2894 [TCG_BSWAP_OZ] = "oz", 2895 [TCG_BSWAP_OS] = "os", 2896 [TCG_BSWAP_IZ | TCG_BSWAP_OZ] = "iz,oz", 2897 [TCG_BSWAP_IZ | TCG_BSWAP_OS] = "iz,os", 2898 }; 2899 2900 #ifdef CONFIG_PLUGIN 2901 static const char * const plugin_from_name[] = { 2902 "from-tb", 2903 "from-insn", 2904 "after-insn", 2905 "after-tb", 2906 }; 2907 #endif 2908 2909 static inline bool tcg_regset_single(TCGRegSet d) 2910 { 2911 return (d & (d - 1)) == 0; 2912 } 2913 2914 static inline TCGReg tcg_regset_first(TCGRegSet d) 2915 { 2916 if (TCG_TARGET_NB_REGS <= 32) { 2917 return ctz32(d); 2918 } else { 2919 return ctz64(d); 2920 } 2921 } 2922 2923 /* Return only the number of characters output -- no error return. */ 2924 #define ne_fprintf(...) \ 2925 ({ int ret_ = fprintf(__VA_ARGS__); ret_ >= 0 ? ret_ : 0; }) 2926 2927 void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs) 2928 { 2929 char buf[128]; 2930 TCGOp *op; 2931 2932 QTAILQ_FOREACH(op, &s->ops, link) { 2933 int i, k, nb_oargs, nb_iargs, nb_cargs; 2934 const TCGOpDef *def; 2935 TCGOpcode c; 2936 int col = 0; 2937 2938 c = op->opc; 2939 def = &tcg_op_defs[c]; 2940 2941 if (c == INDEX_op_insn_start) { 2942 nb_oargs = 0; 2943 col += ne_fprintf(f, "\n ----"); 2944 2945 for (i = 0, k = INSN_START_WORDS; i < k; ++i) { 2946 col += ne_fprintf(f, " %016" PRIx64, 2947 tcg_get_insn_start_param(op, i)); 2948 } 2949 } else if (c == INDEX_op_call) { 2950 const TCGHelperInfo *info = tcg_call_info(op); 2951 void *func = tcg_call_func(op); 2952 2953 /* variable number of arguments */ 2954 nb_oargs = TCGOP_CALLO(op); 2955 nb_iargs = TCGOP_CALLI(op); 2956 nb_cargs = def->nb_cargs; 2957 2958 col += ne_fprintf(f, " %s ", def->name); 2959 2960 /* 2961 * Print the function name from TCGHelperInfo, if available. 2962 * Note that plugins have a template function for the info, 2963 * but the actual function pointer comes from the plugin. 2964 */ 2965 if (func == info->func) { 2966 col += ne_fprintf(f, "%s", info->name); 2967 } else { 2968 col += ne_fprintf(f, "plugin(%p)", func); 2969 } 2970 2971 col += ne_fprintf(f, ",$0x%x,$%d", info->flags, nb_oargs); 2972 for (i = 0; i < nb_oargs; i++) { 2973 col += ne_fprintf(f, ",%s", tcg_get_arg_str(s, buf, sizeof(buf), 2974 op->args[i])); 2975 } 2976 for (i = 0; i < nb_iargs; i++) { 2977 TCGArg arg = op->args[nb_oargs + i]; 2978 const char *t = tcg_get_arg_str(s, buf, sizeof(buf), arg); 2979 col += ne_fprintf(f, ",%s", t); 2980 } 2981 } else { 2982 if (def->flags & TCG_OPF_INT) { 2983 col += ne_fprintf(f, " %s_i%d ", 2984 def->name, 2985 8 * tcg_type_size(TCGOP_TYPE(op))); 2986 } else if (def->flags & TCG_OPF_VECTOR) { 2987 col += ne_fprintf(f, "%s v%d,e%d,", 2988 def->name, 2989 8 * tcg_type_size(TCGOP_TYPE(op)), 2990 8 << TCGOP_VECE(op)); 2991 } else { 2992 col += ne_fprintf(f, " %s ", def->name); 2993 } 2994 2995 nb_oargs = def->nb_oargs; 2996 nb_iargs = def->nb_iargs; 2997 nb_cargs = def->nb_cargs; 2998 2999 k = 0; 3000 for (i = 0; i < nb_oargs; i++) { 3001 const char *sep = k ? "," : ""; 3002 col += ne_fprintf(f, "%s%s", sep, 3003 tcg_get_arg_str(s, buf, sizeof(buf), 3004 op->args[k++])); 3005 } 3006 for (i = 0; i < nb_iargs; i++) { 3007 const char *sep = k ? "," : ""; 3008 col += ne_fprintf(f, "%s%s", sep, 3009 tcg_get_arg_str(s, buf, sizeof(buf), 3010 op->args[k++])); 3011 } 3012 switch (c) { 3013 case INDEX_op_brcond: 3014 case INDEX_op_setcond: 3015 case INDEX_op_negsetcond: 3016 case INDEX_op_movcond: 3017 case INDEX_op_brcond2_i32: 3018 case INDEX_op_setcond2_i32: 3019 case INDEX_op_cmp_vec: 3020 case INDEX_op_cmpsel_vec: 3021 if (op->args[k] < ARRAY_SIZE(cond_name) 3022 && cond_name[op->args[k]]) { 3023 col += ne_fprintf(f, ",%s", cond_name[op->args[k++]]); 3024 } else { 3025 col += ne_fprintf(f, ",$0x%" TCG_PRIlx, op->args[k++]); 3026 } 3027 i = 1; 3028 break; 3029 case INDEX_op_qemu_ld: 3030 case INDEX_op_qemu_st: 3031 case INDEX_op_qemu_ld2: 3032 case INDEX_op_qemu_st2: 3033 { 3034 const char *s_al, *s_op, *s_at; 3035 MemOpIdx oi = op->args[k++]; 3036 MemOp mop = get_memop(oi); 3037 unsigned ix = get_mmuidx(oi); 3038 3039 s_al = alignment_name[(mop & MO_AMASK) >> MO_ASHIFT]; 3040 s_op = ldst_name[mop & (MO_BSWAP | MO_SSIZE)]; 3041 s_at = atom_name[(mop & MO_ATOM_MASK) >> MO_ATOM_SHIFT]; 3042 mop &= ~(MO_AMASK | MO_BSWAP | MO_SSIZE | MO_ATOM_MASK); 3043 3044 /* If all fields are accounted for, print symbolically. */ 3045 if (!mop && s_al && s_op && s_at) { 3046 col += ne_fprintf(f, ",%s%s%s,%u", 3047 s_at, s_al, s_op, ix); 3048 } else { 3049 mop = get_memop(oi); 3050 col += ne_fprintf(f, ",$0x%x,%u", mop, ix); 3051 } 3052 i = 1; 3053 } 3054 break; 3055 case INDEX_op_bswap16: 3056 case INDEX_op_bswap32: 3057 case INDEX_op_bswap64: 3058 { 3059 TCGArg flags = op->args[k]; 3060 const char *name = NULL; 3061 3062 if (flags < ARRAY_SIZE(bswap_flag_name)) { 3063 name = bswap_flag_name[flags]; 3064 } 3065 if (name) { 3066 col += ne_fprintf(f, ",%s", name); 3067 } else { 3068 col += ne_fprintf(f, ",$0x%" TCG_PRIlx, flags); 3069 } 3070 i = k = 1; 3071 } 3072 break; 3073 #ifdef CONFIG_PLUGIN 3074 case INDEX_op_plugin_cb: 3075 { 3076 TCGArg from = op->args[k++]; 3077 const char *name = NULL; 3078 3079 if (from < ARRAY_SIZE(plugin_from_name)) { 3080 name = plugin_from_name[from]; 3081 } 3082 if (name) { 3083 col += ne_fprintf(f, "%s", name); 3084 } else { 3085 col += ne_fprintf(f, "$0x%" TCG_PRIlx, from); 3086 } 3087 i = 1; 3088 } 3089 break; 3090 #endif 3091 default: 3092 i = 0; 3093 break; 3094 } 3095 switch (c) { 3096 case INDEX_op_set_label: 3097 case INDEX_op_br: 3098 case INDEX_op_brcond: 3099 case INDEX_op_brcond2_i32: 3100 col += ne_fprintf(f, "%s$L%d", k ? "," : "", 3101 arg_label(op->args[k])->id); 3102 i++, k++; 3103 break; 3104 case INDEX_op_mb: 3105 { 3106 TCGBar membar = op->args[k]; 3107 const char *b_op, *m_op; 3108 3109 switch (membar & TCG_BAR_SC) { 3110 case 0: 3111 b_op = "none"; 3112 break; 3113 case TCG_BAR_LDAQ: 3114 b_op = "acq"; 3115 break; 3116 case TCG_BAR_STRL: 3117 b_op = "rel"; 3118 break; 3119 case TCG_BAR_SC: 3120 b_op = "seq"; 3121 break; 3122 default: 3123 g_assert_not_reached(); 3124 } 3125 3126 switch (membar & TCG_MO_ALL) { 3127 case 0: 3128 m_op = "none"; 3129 break; 3130 case TCG_MO_LD_LD: 3131 m_op = "rr"; 3132 break; 3133 case TCG_MO_LD_ST: 3134 m_op = "rw"; 3135 break; 3136 case TCG_MO_ST_LD: 3137 m_op = "wr"; 3138 break; 3139 case TCG_MO_ST_ST: 3140 m_op = "ww"; 3141 break; 3142 case TCG_MO_LD_LD | TCG_MO_LD_ST: 3143 m_op = "rr+rw"; 3144 break; 3145 case TCG_MO_LD_LD | TCG_MO_ST_LD: 3146 m_op = "rr+wr"; 3147 break; 3148 case TCG_MO_LD_LD | TCG_MO_ST_ST: 3149 m_op = "rr+ww"; 3150 break; 3151 case TCG_MO_LD_ST | TCG_MO_ST_LD: 3152 m_op = "rw+wr"; 3153 break; 3154 case TCG_MO_LD_ST | TCG_MO_ST_ST: 3155 m_op = "rw+ww"; 3156 break; 3157 case TCG_MO_ST_LD | TCG_MO_ST_ST: 3158 m_op = "wr+ww"; 3159 break; 3160 case TCG_MO_LD_LD | TCG_MO_LD_ST | TCG_MO_ST_LD: 3161 m_op = "rr+rw+wr"; 3162 break; 3163 case TCG_MO_LD_LD | TCG_MO_LD_ST | TCG_MO_ST_ST: 3164 m_op = "rr+rw+ww"; 3165 break; 3166 case TCG_MO_LD_LD | TCG_MO_ST_LD | TCG_MO_ST_ST: 3167 m_op = "rr+wr+ww"; 3168 break; 3169 case TCG_MO_LD_ST | TCG_MO_ST_LD | TCG_MO_ST_ST: 3170 m_op = "rw+wr+ww"; 3171 break; 3172 case TCG_MO_ALL: 3173 m_op = "all"; 3174 break; 3175 default: 3176 g_assert_not_reached(); 3177 } 3178 3179 col += ne_fprintf(f, "%s%s:%s", (k ? "," : ""), b_op, m_op); 3180 i++, k++; 3181 } 3182 break; 3183 default: 3184 break; 3185 } 3186 for (; i < nb_cargs; i++, k++) { 3187 col += ne_fprintf(f, "%s$0x%" TCG_PRIlx, k ? "," : "", 3188 op->args[k]); 3189 } 3190 } 3191 3192 if (have_prefs || op->life) { 3193 for (; col < 40; ++col) { 3194 putc(' ', f); 3195 } 3196 } 3197 3198 if (op->life) { 3199 unsigned life = op->life; 3200 3201 if (life & (SYNC_ARG * 3)) { 3202 ne_fprintf(f, " sync:"); 3203 for (i = 0; i < 2; ++i) { 3204 if (life & (SYNC_ARG << i)) { 3205 ne_fprintf(f, " %d", i); 3206 } 3207 } 3208 } 3209 life /= DEAD_ARG; 3210 if (life) { 3211 ne_fprintf(f, " dead:"); 3212 for (i = 0; life; ++i, life >>= 1) { 3213 if (life & 1) { 3214 ne_fprintf(f, " %d", i); 3215 } 3216 } 3217 } 3218 } 3219 3220 if (have_prefs) { 3221 for (i = 0; i < nb_oargs; ++i) { 3222 TCGRegSet set = output_pref(op, i); 3223 3224 if (i == 0) { 3225 ne_fprintf(f, " pref="); 3226 } else { 3227 ne_fprintf(f, ","); 3228 } 3229 if (set == 0) { 3230 ne_fprintf(f, "none"); 3231 } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) { 3232 ne_fprintf(f, "all"); 3233 #ifdef CONFIG_DEBUG_TCG 3234 } else if (tcg_regset_single(set)) { 3235 TCGReg reg = tcg_regset_first(set); 3236 ne_fprintf(f, "%s", tcg_target_reg_names[reg]); 3237 #endif 3238 } else if (TCG_TARGET_NB_REGS <= 32) { 3239 ne_fprintf(f, "0x%x", (uint32_t)set); 3240 } else { 3241 ne_fprintf(f, "0x%" PRIx64, (uint64_t)set); 3242 } 3243 } 3244 } 3245 3246 putc('\n', f); 3247 } 3248 } 3249 3250 /* we give more priority to constraints with less registers */ 3251 static int get_constraint_priority(const TCGArgConstraint *arg_ct, int k) 3252 { 3253 int n; 3254 3255 arg_ct += k; 3256 n = ctpop64(arg_ct->regs); 3257 3258 /* 3259 * Sort constraints of a single register first, which includes output 3260 * aliases (which must exactly match the input already allocated). 3261 */ 3262 if (n == 1 || arg_ct->oalias) { 3263 return INT_MAX; 3264 } 3265 3266 /* 3267 * Sort register pairs next, first then second immediately after. 3268 * Arbitrarily sort multiple pairs by the index of the first reg; 3269 * there shouldn't be many pairs. 3270 */ 3271 switch (arg_ct->pair) { 3272 case 1: 3273 case 3: 3274 return (k + 1) * 2; 3275 case 2: 3276 return (arg_ct->pair_index + 1) * 2 - 1; 3277 } 3278 3279 /* Finally, sort by decreasing register count. */ 3280 assert(n > 1); 3281 return -n; 3282 } 3283 3284 /* sort from highest priority to lowest */ 3285 static void sort_constraints(TCGArgConstraint *a, int start, int n) 3286 { 3287 int i, j; 3288 3289 for (i = 0; i < n; i++) { 3290 a[start + i].sort_index = start + i; 3291 } 3292 if (n <= 1) { 3293 return; 3294 } 3295 for (i = 0; i < n - 1; i++) { 3296 for (j = i + 1; j < n; j++) { 3297 int p1 = get_constraint_priority(a, a[start + i].sort_index); 3298 int p2 = get_constraint_priority(a, a[start + j].sort_index); 3299 if (p1 < p2) { 3300 int tmp = a[start + i].sort_index; 3301 a[start + i].sort_index = a[start + j].sort_index; 3302 a[start + j].sort_index = tmp; 3303 } 3304 } 3305 } 3306 } 3307 3308 static const TCGArgConstraint empty_cts[TCG_MAX_OP_ARGS]; 3309 static TCGArgConstraint all_cts[ARRAY_SIZE(constraint_sets)][TCG_MAX_OP_ARGS]; 3310 3311 static void process_constraint_sets(void) 3312 { 3313 for (size_t c = 0; c < ARRAY_SIZE(constraint_sets); ++c) { 3314 const TCGConstraintSet *tdefs = &constraint_sets[c]; 3315 TCGArgConstraint *args_ct = all_cts[c]; 3316 int nb_oargs = tdefs->nb_oargs; 3317 int nb_iargs = tdefs->nb_iargs; 3318 int nb_args = nb_oargs + nb_iargs; 3319 bool saw_alias_pair = false; 3320 3321 for (int i = 0; i < nb_args; i++) { 3322 const char *ct_str = tdefs->args_ct_str[i]; 3323 bool input_p = i >= nb_oargs; 3324 int o; 3325 3326 switch (*ct_str) { 3327 case '0' ... '9': 3328 o = *ct_str - '0'; 3329 tcg_debug_assert(input_p); 3330 tcg_debug_assert(o < nb_oargs); 3331 tcg_debug_assert(args_ct[o].regs != 0); 3332 tcg_debug_assert(!args_ct[o].oalias); 3333 args_ct[i] = args_ct[o]; 3334 /* The output sets oalias. */ 3335 args_ct[o].oalias = 1; 3336 args_ct[o].alias_index = i; 3337 /* The input sets ialias. */ 3338 args_ct[i].ialias = 1; 3339 args_ct[i].alias_index = o; 3340 if (args_ct[i].pair) { 3341 saw_alias_pair = true; 3342 } 3343 tcg_debug_assert(ct_str[1] == '\0'); 3344 continue; 3345 3346 case '&': 3347 tcg_debug_assert(!input_p); 3348 args_ct[i].newreg = true; 3349 ct_str++; 3350 break; 3351 3352 case 'p': /* plus */ 3353 /* Allocate to the register after the previous. */ 3354 tcg_debug_assert(i > (input_p ? nb_oargs : 0)); 3355 o = i - 1; 3356 tcg_debug_assert(!args_ct[o].pair); 3357 tcg_debug_assert(!args_ct[o].ct); 3358 args_ct[i] = (TCGArgConstraint){ 3359 .pair = 2, 3360 .pair_index = o, 3361 .regs = args_ct[o].regs << 1, 3362 .newreg = args_ct[o].newreg, 3363 }; 3364 args_ct[o].pair = 1; 3365 args_ct[o].pair_index = i; 3366 tcg_debug_assert(ct_str[1] == '\0'); 3367 continue; 3368 3369 case 'm': /* minus */ 3370 /* Allocate to the register before the previous. */ 3371 tcg_debug_assert(i > (input_p ? nb_oargs : 0)); 3372 o = i - 1; 3373 tcg_debug_assert(!args_ct[o].pair); 3374 tcg_debug_assert(!args_ct[o].ct); 3375 args_ct[i] = (TCGArgConstraint){ 3376 .pair = 1, 3377 .pair_index = o, 3378 .regs = args_ct[o].regs >> 1, 3379 .newreg = args_ct[o].newreg, 3380 }; 3381 args_ct[o].pair = 2; 3382 args_ct[o].pair_index = i; 3383 tcg_debug_assert(ct_str[1] == '\0'); 3384 continue; 3385 } 3386 3387 do { 3388 switch (*ct_str) { 3389 case 'i': 3390 args_ct[i].ct |= TCG_CT_CONST; 3391 break; 3392 #ifdef TCG_REG_ZERO 3393 case 'z': 3394 args_ct[i].ct |= TCG_CT_REG_ZERO; 3395 break; 3396 #endif 3397 3398 /* Include all of the target-specific constraints. */ 3399 3400 #undef CONST 3401 #define CONST(CASE, MASK) \ 3402 case CASE: args_ct[i].ct |= MASK; break; 3403 #define REGS(CASE, MASK) \ 3404 case CASE: args_ct[i].regs |= MASK; break; 3405 3406 #include "tcg-target-con-str.h" 3407 3408 #undef REGS 3409 #undef CONST 3410 default: 3411 case '0' ... '9': 3412 case '&': 3413 case 'p': 3414 case 'm': 3415 /* Typo in TCGConstraintSet constraint. */ 3416 g_assert_not_reached(); 3417 } 3418 } while (*++ct_str != '\0'); 3419 } 3420 3421 /* 3422 * Fix up output pairs that are aliased with inputs. 3423 * When we created the alias, we copied pair from the output. 3424 * There are three cases: 3425 * (1a) Pairs of inputs alias pairs of outputs. 3426 * (1b) One input aliases the first of a pair of outputs. 3427 * (2) One input aliases the second of a pair of outputs. 3428 * 3429 * Case 1a is handled by making sure that the pair_index'es are 3430 * properly updated so that they appear the same as a pair of inputs. 3431 * 3432 * Case 1b is handled by setting the pair_index of the input to 3433 * itself, simply so it doesn't point to an unrelated argument. 3434 * Since we don't encounter the "second" during the input allocation 3435 * phase, nothing happens with the second half of the input pair. 3436 * 3437 * Case 2 is handled by setting the second input to pair=3, the 3438 * first output to pair=3, and the pair_index'es to match. 3439 */ 3440 if (saw_alias_pair) { 3441 for (int i = nb_oargs; i < nb_args; i++) { 3442 int o, o2, i2; 3443 3444 /* 3445 * Since [0-9pm] must be alone in the constraint string, 3446 * the only way they can both be set is if the pair comes 3447 * from the output alias. 3448 */ 3449 if (!args_ct[i].ialias) { 3450 continue; 3451 } 3452 switch (args_ct[i].pair) { 3453 case 0: 3454 break; 3455 case 1: 3456 o = args_ct[i].alias_index; 3457 o2 = args_ct[o].pair_index; 3458 tcg_debug_assert(args_ct[o].pair == 1); 3459 tcg_debug_assert(args_ct[o2].pair == 2); 3460 if (args_ct[o2].oalias) { 3461 /* Case 1a */ 3462 i2 = args_ct[o2].alias_index; 3463 tcg_debug_assert(args_ct[i2].pair == 2); 3464 args_ct[i2].pair_index = i; 3465 args_ct[i].pair_index = i2; 3466 } else { 3467 /* Case 1b */ 3468 args_ct[i].pair_index = i; 3469 } 3470 break; 3471 case 2: 3472 o = args_ct[i].alias_index; 3473 o2 = args_ct[o].pair_index; 3474 tcg_debug_assert(args_ct[o].pair == 2); 3475 tcg_debug_assert(args_ct[o2].pair == 1); 3476 if (args_ct[o2].oalias) { 3477 /* Case 1a */ 3478 i2 = args_ct[o2].alias_index; 3479 tcg_debug_assert(args_ct[i2].pair == 1); 3480 args_ct[i2].pair_index = i; 3481 args_ct[i].pair_index = i2; 3482 } else { 3483 /* Case 2 */ 3484 args_ct[i].pair = 3; 3485 args_ct[o2].pair = 3; 3486 args_ct[i].pair_index = o2; 3487 args_ct[o2].pair_index = i; 3488 } 3489 break; 3490 default: 3491 g_assert_not_reached(); 3492 } 3493 } 3494 } 3495 3496 /* sort the constraints (XXX: this is just an heuristic) */ 3497 sort_constraints(args_ct, 0, nb_oargs); 3498 sort_constraints(args_ct, nb_oargs, nb_iargs); 3499 } 3500 } 3501 3502 static const TCGArgConstraint *opcode_args_ct(const TCGOp *op) 3503 { 3504 TCGOpcode opc = op->opc; 3505 TCGType type = TCGOP_TYPE(op); 3506 unsigned flags = TCGOP_FLAGS(op); 3507 const TCGOpDef *def = &tcg_op_defs[opc]; 3508 const TCGOutOp *outop = all_outop[opc]; 3509 TCGConstraintSetIndex con_set; 3510 3511 if (def->flags & TCG_OPF_NOT_PRESENT) { 3512 return empty_cts; 3513 } 3514 3515 if (outop) { 3516 con_set = outop->static_constraint; 3517 if (con_set == C_Dynamic) { 3518 con_set = outop->dynamic_constraint(type, flags); 3519 } 3520 } else { 3521 con_set = tcg_target_op_def(opc, type, flags); 3522 } 3523 tcg_debug_assert(con_set >= 0); 3524 tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets)); 3525 3526 /* The constraint arguments must match TCGOpcode arguments. */ 3527 tcg_debug_assert(constraint_sets[con_set].nb_oargs == def->nb_oargs); 3528 tcg_debug_assert(constraint_sets[con_set].nb_iargs == def->nb_iargs); 3529 3530 return all_cts[con_set]; 3531 } 3532 3533 static void remove_label_use(TCGOp *op, int idx) 3534 { 3535 TCGLabel *label = arg_label(op->args[idx]); 3536 TCGLabelUse *use; 3537 3538 QSIMPLEQ_FOREACH(use, &label->branches, next) { 3539 if (use->op == op) { 3540 QSIMPLEQ_REMOVE(&label->branches, use, TCGLabelUse, next); 3541 return; 3542 } 3543 } 3544 g_assert_not_reached(); 3545 } 3546 3547 void tcg_op_remove(TCGContext *s, TCGOp *op) 3548 { 3549 switch (op->opc) { 3550 case INDEX_op_br: 3551 remove_label_use(op, 0); 3552 break; 3553 case INDEX_op_brcond: 3554 remove_label_use(op, 3); 3555 break; 3556 case INDEX_op_brcond2_i32: 3557 remove_label_use(op, 5); 3558 break; 3559 default: 3560 break; 3561 } 3562 3563 QTAILQ_REMOVE(&s->ops, op, link); 3564 QTAILQ_INSERT_TAIL(&s->free_ops, op, link); 3565 s->nb_ops--; 3566 } 3567 3568 void tcg_remove_ops_after(TCGOp *op) 3569 { 3570 TCGContext *s = tcg_ctx; 3571 3572 while (true) { 3573 TCGOp *last = tcg_last_op(); 3574 if (last == op) { 3575 return; 3576 } 3577 tcg_op_remove(s, last); 3578 } 3579 } 3580 3581 static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs) 3582 { 3583 TCGContext *s = tcg_ctx; 3584 TCGOp *op = NULL; 3585 3586 if (unlikely(!QTAILQ_EMPTY(&s->free_ops))) { 3587 QTAILQ_FOREACH(op, &s->free_ops, link) { 3588 if (nargs <= op->nargs) { 3589 QTAILQ_REMOVE(&s->free_ops, op, link); 3590 nargs = op->nargs; 3591 goto found; 3592 } 3593 } 3594 } 3595 3596 /* Most opcodes have 3 or 4 operands: reduce fragmentation. */ 3597 nargs = MAX(4, nargs); 3598 op = tcg_malloc(sizeof(TCGOp) + sizeof(TCGArg) * nargs); 3599 3600 found: 3601 memset(op, 0, offsetof(TCGOp, link)); 3602 op->opc = opc; 3603 op->nargs = nargs; 3604 3605 /* Check for bitfield overflow. */ 3606 tcg_debug_assert(op->nargs == nargs); 3607 3608 s->nb_ops++; 3609 return op; 3610 } 3611 3612 TCGOp *tcg_emit_op(TCGOpcode opc, unsigned nargs) 3613 { 3614 TCGOp *op = tcg_op_alloc(opc, nargs); 3615 3616 if (tcg_ctx->emit_before_op) { 3617 QTAILQ_INSERT_BEFORE(tcg_ctx->emit_before_op, op, link); 3618 } else { 3619 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link); 3620 } 3621 return op; 3622 } 3623 3624 TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op, 3625 TCGOpcode opc, TCGType type, unsigned nargs) 3626 { 3627 TCGOp *new_op = tcg_op_alloc(opc, nargs); 3628 3629 TCGOP_TYPE(new_op) = type; 3630 QTAILQ_INSERT_BEFORE(old_op, new_op, link); 3631 return new_op; 3632 } 3633 3634 TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op, 3635 TCGOpcode opc, TCGType type, unsigned nargs) 3636 { 3637 TCGOp *new_op = tcg_op_alloc(opc, nargs); 3638 3639 TCGOP_TYPE(new_op) = type; 3640 QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link); 3641 return new_op; 3642 } 3643 3644 static void move_label_uses(TCGLabel *to, TCGLabel *from) 3645 { 3646 TCGLabelUse *u; 3647 3648 QSIMPLEQ_FOREACH(u, &from->branches, next) { 3649 TCGOp *op = u->op; 3650 switch (op->opc) { 3651 case INDEX_op_br: 3652 op->args[0] = label_arg(to); 3653 break; 3654 case INDEX_op_brcond: 3655 op->args[3] = label_arg(to); 3656 break; 3657 case INDEX_op_brcond2_i32: 3658 op->args[5] = label_arg(to); 3659 break; 3660 default: 3661 g_assert_not_reached(); 3662 } 3663 } 3664 3665 QSIMPLEQ_CONCAT(&to->branches, &from->branches); 3666 } 3667 3668 /* Reachable analysis : remove unreachable code. */ 3669 static void __attribute__((noinline)) 3670 reachable_code_pass(TCGContext *s) 3671 { 3672 TCGOp *op, *op_next, *op_prev; 3673 bool dead = false; 3674 3675 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { 3676 bool remove = dead; 3677 TCGLabel *label; 3678 3679 switch (op->opc) { 3680 case INDEX_op_set_label: 3681 label = arg_label(op->args[0]); 3682 3683 /* 3684 * Note that the first op in the TB is always a load, 3685 * so there is always something before a label. 3686 */ 3687 op_prev = QTAILQ_PREV(op, link); 3688 3689 /* 3690 * If we find two sequential labels, move all branches to 3691 * reference the second label and remove the first label. 3692 * Do this before branch to next optimization, so that the 3693 * middle label is out of the way. 3694 */ 3695 if (op_prev->opc == INDEX_op_set_label) { 3696 move_label_uses(label, arg_label(op_prev->args[0])); 3697 tcg_op_remove(s, op_prev); 3698 op_prev = QTAILQ_PREV(op, link); 3699 } 3700 3701 /* 3702 * Optimization can fold conditional branches to unconditional. 3703 * If we find a label which is preceded by an unconditional 3704 * branch to next, remove the branch. We couldn't do this when 3705 * processing the branch because any dead code between the branch 3706 * and label had not yet been removed. 3707 */ 3708 if (op_prev->opc == INDEX_op_br && 3709 label == arg_label(op_prev->args[0])) { 3710 tcg_op_remove(s, op_prev); 3711 /* Fall through means insns become live again. */ 3712 dead = false; 3713 } 3714 3715 if (QSIMPLEQ_EMPTY(&label->branches)) { 3716 /* 3717 * While there is an occasional backward branch, virtually 3718 * all branches generated by the translators are forward. 3719 * Which means that generally we will have already removed 3720 * all references to the label that will be, and there is 3721 * little to be gained by iterating. 3722 */ 3723 remove = true; 3724 } else { 3725 /* Once we see a label, insns become live again. */ 3726 dead = false; 3727 remove = false; 3728 } 3729 break; 3730 3731 case INDEX_op_br: 3732 case INDEX_op_exit_tb: 3733 case INDEX_op_goto_ptr: 3734 /* Unconditional branches; everything following is dead. */ 3735 dead = true; 3736 break; 3737 3738 case INDEX_op_call: 3739 /* Notice noreturn helper calls, raising exceptions. */ 3740 if (tcg_call_flags(op) & TCG_CALL_NO_RETURN) { 3741 dead = true; 3742 } 3743 break; 3744 3745 case INDEX_op_insn_start: 3746 /* Never remove -- we need to keep these for unwind. */ 3747 remove = false; 3748 break; 3749 3750 default: 3751 break; 3752 } 3753 3754 if (remove) { 3755 tcg_op_remove(s, op); 3756 } 3757 } 3758 } 3759 3760 #define TS_DEAD 1 3761 #define TS_MEM 2 3762 3763 #define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n))) 3764 #define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n))) 3765 3766 /* For liveness_pass_1, the register preferences for a given temp. */ 3767 static inline TCGRegSet *la_temp_pref(TCGTemp *ts) 3768 { 3769 return ts->state_ptr; 3770 } 3771 3772 /* For liveness_pass_1, reset the preferences for a given temp to the 3773 * maximal regset for its type. 3774 */ 3775 static inline void la_reset_pref(TCGTemp *ts) 3776 { 3777 *la_temp_pref(ts) 3778 = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]); 3779 } 3780 3781 /* liveness analysis: end of function: all temps are dead, and globals 3782 should be in memory. */ 3783 static void la_func_end(TCGContext *s, int ng, int nt) 3784 { 3785 int i; 3786 3787 for (i = 0; i < ng; ++i) { 3788 s->temps[i].state = TS_DEAD | TS_MEM; 3789 la_reset_pref(&s->temps[i]); 3790 } 3791 for (i = ng; i < nt; ++i) { 3792 s->temps[i].state = TS_DEAD; 3793 la_reset_pref(&s->temps[i]); 3794 } 3795 } 3796 3797 /* liveness analysis: end of basic block: all temps are dead, globals 3798 and local temps should be in memory. */ 3799 static void la_bb_end(TCGContext *s, int ng, int nt) 3800 { 3801 int i; 3802 3803 for (i = 0; i < nt; ++i) { 3804 TCGTemp *ts = &s->temps[i]; 3805 int state; 3806 3807 switch (ts->kind) { 3808 case TEMP_FIXED: 3809 case TEMP_GLOBAL: 3810 case TEMP_TB: 3811 state = TS_DEAD | TS_MEM; 3812 break; 3813 case TEMP_EBB: 3814 case TEMP_CONST: 3815 state = TS_DEAD; 3816 break; 3817 default: 3818 g_assert_not_reached(); 3819 } 3820 ts->state = state; 3821 la_reset_pref(ts); 3822 } 3823 } 3824 3825 /* liveness analysis: sync globals back to memory. */ 3826 static void la_global_sync(TCGContext *s, int ng) 3827 { 3828 int i; 3829 3830 for (i = 0; i < ng; ++i) { 3831 int state = s->temps[i].state; 3832 s->temps[i].state = state | TS_MEM; 3833 if (state == TS_DEAD) { 3834 /* If the global was previously dead, reset prefs. */ 3835 la_reset_pref(&s->temps[i]); 3836 } 3837 } 3838 } 3839 3840 /* 3841 * liveness analysis: conditional branch: all temps are dead unless 3842 * explicitly live-across-conditional-branch, globals and local temps 3843 * should be synced. 3844 */ 3845 static void la_bb_sync(TCGContext *s, int ng, int nt) 3846 { 3847 la_global_sync(s, ng); 3848 3849 for (int i = ng; i < nt; ++i) { 3850 TCGTemp *ts = &s->temps[i]; 3851 int state; 3852 3853 switch (ts->kind) { 3854 case TEMP_TB: 3855 state = ts->state; 3856 ts->state = state | TS_MEM; 3857 if (state != TS_DEAD) { 3858 continue; 3859 } 3860 break; 3861 case TEMP_EBB: 3862 case TEMP_CONST: 3863 continue; 3864 default: 3865 g_assert_not_reached(); 3866 } 3867 la_reset_pref(&s->temps[i]); 3868 } 3869 } 3870 3871 /* liveness analysis: sync globals back to memory and kill. */ 3872 static void la_global_kill(TCGContext *s, int ng) 3873 { 3874 int i; 3875 3876 for (i = 0; i < ng; i++) { 3877 s->temps[i].state = TS_DEAD | TS_MEM; 3878 la_reset_pref(&s->temps[i]); 3879 } 3880 } 3881 3882 /* liveness analysis: note live globals crossing calls. */ 3883 static void la_cross_call(TCGContext *s, int nt) 3884 { 3885 TCGRegSet mask = ~tcg_target_call_clobber_regs; 3886 int i; 3887 3888 for (i = 0; i < nt; i++) { 3889 TCGTemp *ts = &s->temps[i]; 3890 if (!(ts->state & TS_DEAD)) { 3891 TCGRegSet *pset = la_temp_pref(ts); 3892 TCGRegSet set = *pset; 3893 3894 set &= mask; 3895 /* If the combination is not possible, restart. */ 3896 if (set == 0) { 3897 set = tcg_target_available_regs[ts->type] & mask; 3898 } 3899 *pset = set; 3900 } 3901 } 3902 } 3903 3904 /* 3905 * Liveness analysis: Verify the lifetime of TEMP_TB, and reduce 3906 * to TEMP_EBB, if possible. 3907 */ 3908 static void __attribute__((noinline)) 3909 liveness_pass_0(TCGContext *s) 3910 { 3911 void * const multiple_ebb = (void *)(uintptr_t)-1; 3912 int nb_temps = s->nb_temps; 3913 TCGOp *op, *ebb; 3914 3915 for (int i = s->nb_globals; i < nb_temps; ++i) { 3916 s->temps[i].state_ptr = NULL; 3917 } 3918 3919 /* 3920 * Represent each EBB by the op at which it begins. In the case of 3921 * the first EBB, this is the first op, otherwise it is a label. 3922 * Collect the uses of each TEMP_TB: NULL for unused, EBB for use 3923 * within a single EBB, else MULTIPLE_EBB. 3924 */ 3925 ebb = QTAILQ_FIRST(&s->ops); 3926 QTAILQ_FOREACH(op, &s->ops, link) { 3927 const TCGOpDef *def; 3928 int nb_oargs, nb_iargs; 3929 3930 switch (op->opc) { 3931 case INDEX_op_set_label: 3932 ebb = op; 3933 continue; 3934 case INDEX_op_discard: 3935 continue; 3936 case INDEX_op_call: 3937 nb_oargs = TCGOP_CALLO(op); 3938 nb_iargs = TCGOP_CALLI(op); 3939 break; 3940 default: 3941 def = &tcg_op_defs[op->opc]; 3942 nb_oargs = def->nb_oargs; 3943 nb_iargs = def->nb_iargs; 3944 break; 3945 } 3946 3947 for (int i = 0; i < nb_oargs + nb_iargs; ++i) { 3948 TCGTemp *ts = arg_temp(op->args[i]); 3949 3950 if (ts->kind != TEMP_TB) { 3951 continue; 3952 } 3953 if (ts->state_ptr == NULL) { 3954 ts->state_ptr = ebb; 3955 } else if (ts->state_ptr != ebb) { 3956 ts->state_ptr = multiple_ebb; 3957 } 3958 } 3959 } 3960 3961 /* 3962 * For TEMP_TB that turned out not to be used beyond one EBB, 3963 * reduce the liveness to TEMP_EBB. 3964 */ 3965 for (int i = s->nb_globals; i < nb_temps; ++i) { 3966 TCGTemp *ts = &s->temps[i]; 3967 if (ts->kind == TEMP_TB && ts->state_ptr != multiple_ebb) { 3968 ts->kind = TEMP_EBB; 3969 } 3970 } 3971 } 3972 3973 static void assert_carry_dead(TCGContext *s) 3974 { 3975 /* 3976 * Carry operations can be separated by a few insns like mov, 3977 * load or store, but they should always be "close", and 3978 * carry-out operations should always be paired with carry-in. 3979 * At various boundaries, carry must have been consumed. 3980 */ 3981 tcg_debug_assert(!s->carry_live); 3982 } 3983 3984 /* Liveness analysis : update the opc_arg_life array to tell if a 3985 given input arguments is dead. Instructions updating dead 3986 temporaries are removed. */ 3987 static void __attribute__((noinline)) 3988 liveness_pass_1(TCGContext *s) 3989 { 3990 int nb_globals = s->nb_globals; 3991 int nb_temps = s->nb_temps; 3992 TCGOp *op, *op_prev; 3993 TCGRegSet *prefs; 3994 3995 prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps); 3996 for (int i = 0; i < nb_temps; ++i) { 3997 s->temps[i].state_ptr = prefs + i; 3998 } 3999 4000 /* ??? Should be redundant with the exit_tb that ends the TB. */ 4001 la_func_end(s, nb_globals, nb_temps); 4002 4003 s->carry_live = false; 4004 QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) { 4005 int nb_iargs, nb_oargs; 4006 TCGOpcode opc_new, opc_new2; 4007 TCGLifeData arg_life = 0; 4008 TCGTemp *ts; 4009 TCGOpcode opc = op->opc; 4010 const TCGOpDef *def; 4011 const TCGArgConstraint *args_ct; 4012 4013 switch (opc) { 4014 case INDEX_op_call: 4015 assert_carry_dead(s); 4016 { 4017 const TCGHelperInfo *info = tcg_call_info(op); 4018 int call_flags = tcg_call_flags(op); 4019 4020 nb_oargs = TCGOP_CALLO(op); 4021 nb_iargs = TCGOP_CALLI(op); 4022 4023 /* pure functions can be removed if their result is unused */ 4024 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) { 4025 for (int i = 0; i < nb_oargs; i++) { 4026 ts = arg_temp(op->args[i]); 4027 if (ts->state != TS_DEAD) { 4028 goto do_not_remove_call; 4029 } 4030 } 4031 goto do_remove; 4032 } 4033 do_not_remove_call: 4034 4035 /* Output args are dead. */ 4036 for (int i = 0; i < nb_oargs; i++) { 4037 ts = arg_temp(op->args[i]); 4038 if (ts->state & TS_DEAD) { 4039 arg_life |= DEAD_ARG << i; 4040 } 4041 if (ts->state & TS_MEM) { 4042 arg_life |= SYNC_ARG << i; 4043 } 4044 ts->state = TS_DEAD; 4045 la_reset_pref(ts); 4046 } 4047 4048 /* Not used -- it will be tcg_target_call_oarg_reg(). */ 4049 memset(op->output_pref, 0, sizeof(op->output_pref)); 4050 4051 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS | 4052 TCG_CALL_NO_READ_GLOBALS))) { 4053 la_global_kill(s, nb_globals); 4054 } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) { 4055 la_global_sync(s, nb_globals); 4056 } 4057 4058 /* Record arguments that die in this helper. */ 4059 for (int i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 4060 ts = arg_temp(op->args[i]); 4061 if (ts->state & TS_DEAD) { 4062 arg_life |= DEAD_ARG << i; 4063 } 4064 } 4065 4066 /* For all live registers, remove call-clobbered prefs. */ 4067 la_cross_call(s, nb_temps); 4068 4069 /* 4070 * Input arguments are live for preceding opcodes. 4071 * 4072 * For those arguments that die, and will be allocated in 4073 * registers, clear the register set for that arg, to be 4074 * filled in below. For args that will be on the stack, 4075 * reset to any available reg. Process arguments in reverse 4076 * order so that if a temp is used more than once, the stack 4077 * reset to max happens before the register reset to 0. 4078 */ 4079 for (int i = nb_iargs - 1; i >= 0; i--) { 4080 const TCGCallArgumentLoc *loc = &info->in[i]; 4081 ts = arg_temp(op->args[nb_oargs + i]); 4082 4083 if (ts->state & TS_DEAD) { 4084 switch (loc->kind) { 4085 case TCG_CALL_ARG_NORMAL: 4086 case TCG_CALL_ARG_EXTEND_U: 4087 case TCG_CALL_ARG_EXTEND_S: 4088 if (arg_slot_reg_p(loc->arg_slot)) { 4089 *la_temp_pref(ts) = 0; 4090 break; 4091 } 4092 /* fall through */ 4093 default: 4094 *la_temp_pref(ts) = 4095 tcg_target_available_regs[ts->type]; 4096 break; 4097 } 4098 ts->state &= ~TS_DEAD; 4099 } 4100 } 4101 4102 /* 4103 * For each input argument, add its input register to prefs. 4104 * If a temp is used once, this produces a single set bit; 4105 * if a temp is used multiple times, this produces a set. 4106 */ 4107 for (int i = 0; i < nb_iargs; i++) { 4108 const TCGCallArgumentLoc *loc = &info->in[i]; 4109 ts = arg_temp(op->args[nb_oargs + i]); 4110 4111 switch (loc->kind) { 4112 case TCG_CALL_ARG_NORMAL: 4113 case TCG_CALL_ARG_EXTEND_U: 4114 case TCG_CALL_ARG_EXTEND_S: 4115 if (arg_slot_reg_p(loc->arg_slot)) { 4116 tcg_regset_set_reg(*la_temp_pref(ts), 4117 tcg_target_call_iarg_regs[loc->arg_slot]); 4118 } 4119 break; 4120 default: 4121 break; 4122 } 4123 } 4124 } 4125 break; 4126 case INDEX_op_insn_start: 4127 assert_carry_dead(s); 4128 break; 4129 case INDEX_op_discard: 4130 /* mark the temporary as dead */ 4131 ts = arg_temp(op->args[0]); 4132 ts->state = TS_DEAD; 4133 la_reset_pref(ts); 4134 break; 4135 4136 case INDEX_op_muls2: 4137 opc_new = INDEX_op_mul; 4138 opc_new2 = INDEX_op_mulsh; 4139 goto do_mul2; 4140 case INDEX_op_mulu2: 4141 opc_new = INDEX_op_mul; 4142 opc_new2 = INDEX_op_muluh; 4143 do_mul2: 4144 assert_carry_dead(s); 4145 if (arg_temp(op->args[1])->state == TS_DEAD) { 4146 if (arg_temp(op->args[0])->state == TS_DEAD) { 4147 /* Both parts of the operation are dead. */ 4148 goto do_remove; 4149 } 4150 /* The high part of the operation is dead; generate the low. */ 4151 op->opc = opc = opc_new; 4152 op->args[1] = op->args[2]; 4153 op->args[2] = op->args[3]; 4154 } else if (arg_temp(op->args[0])->state == TS_DEAD && 4155 tcg_op_supported(opc_new2, TCGOP_TYPE(op), 0)) { 4156 /* The low part of the operation is dead; generate the high. */ 4157 op->opc = opc = opc_new2; 4158 op->args[0] = op->args[1]; 4159 op->args[1] = op->args[2]; 4160 op->args[2] = op->args[3]; 4161 } else { 4162 goto do_not_remove; 4163 } 4164 /* Mark the single-word operation live. */ 4165 goto do_not_remove; 4166 4167 case INDEX_op_addco: 4168 if (s->carry_live) { 4169 goto do_not_remove; 4170 } 4171 op->opc = opc = INDEX_op_add; 4172 goto do_default; 4173 4174 case INDEX_op_addcio: 4175 if (s->carry_live) { 4176 goto do_not_remove; 4177 } 4178 op->opc = opc = INDEX_op_addci; 4179 goto do_default; 4180 4181 case INDEX_op_subbo: 4182 if (s->carry_live) { 4183 goto do_not_remove; 4184 } 4185 /* Lower to sub, but this may also require canonicalization. */ 4186 op->opc = opc = INDEX_op_sub; 4187 ts = arg_temp(op->args[2]); 4188 if (ts->kind == TEMP_CONST) { 4189 ts = tcg_constant_internal(ts->type, -ts->val); 4190 if (ts->state_ptr == NULL) { 4191 tcg_debug_assert(temp_idx(ts) == nb_temps); 4192 nb_temps++; 4193 ts->state_ptr = tcg_malloc(sizeof(TCGRegSet)); 4194 ts->state = TS_DEAD; 4195 la_reset_pref(ts); 4196 } 4197 op->args[2] = temp_arg(ts); 4198 op->opc = opc = INDEX_op_add; 4199 } 4200 goto do_default; 4201 4202 case INDEX_op_subbio: 4203 if (s->carry_live) { 4204 goto do_not_remove; 4205 } 4206 op->opc = opc = INDEX_op_subbi; 4207 goto do_default; 4208 4209 case INDEX_op_addc1o: 4210 if (s->carry_live) { 4211 goto do_not_remove; 4212 } 4213 /* Lower to add, add +1. */ 4214 op_prev = tcg_op_insert_before(s, op, INDEX_op_add, 4215 TCGOP_TYPE(op), 3); 4216 op_prev->args[0] = op->args[0]; 4217 op_prev->args[1] = op->args[1]; 4218 op_prev->args[2] = op->args[2]; 4219 op->opc = opc = INDEX_op_add; 4220 op->args[1] = op->args[0]; 4221 ts = arg_temp(op->args[0]); 4222 ts = tcg_constant_internal(ts->type, 1); 4223 op->args[2] = temp_arg(ts); 4224 goto do_default; 4225 4226 case INDEX_op_subb1o: 4227 if (s->carry_live) { 4228 goto do_not_remove; 4229 } 4230 /* Lower to sub, add -1. */ 4231 op_prev = tcg_op_insert_before(s, op, INDEX_op_sub, 4232 TCGOP_TYPE(op), 3); 4233 op_prev->args[0] = op->args[0]; 4234 op_prev->args[1] = op->args[1]; 4235 op_prev->args[2] = op->args[2]; 4236 op->opc = opc = INDEX_op_add; 4237 op->args[1] = op->args[0]; 4238 ts = arg_temp(op->args[0]); 4239 ts = tcg_constant_internal(ts->type, -1); 4240 op->args[2] = temp_arg(ts); 4241 goto do_default; 4242 4243 default: 4244 do_default: 4245 /* 4246 * Test if the operation can be removed because all 4247 * its outputs are dead. We assume that nb_oargs == 0 4248 * implies side effects. 4249 */ 4250 def = &tcg_op_defs[opc]; 4251 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && def->nb_oargs != 0) { 4252 for (int i = def->nb_oargs - 1; i >= 0; i--) { 4253 if (arg_temp(op->args[i])->state != TS_DEAD) { 4254 goto do_not_remove; 4255 } 4256 } 4257 goto do_remove; 4258 } 4259 goto do_not_remove; 4260 4261 do_remove: 4262 tcg_op_remove(s, op); 4263 break; 4264 4265 do_not_remove: 4266 def = &tcg_op_defs[opc]; 4267 nb_iargs = def->nb_iargs; 4268 nb_oargs = def->nb_oargs; 4269 4270 for (int i = 0; i < nb_oargs; i++) { 4271 ts = arg_temp(op->args[i]); 4272 4273 /* Remember the preference of the uses that followed. */ 4274 if (i < ARRAY_SIZE(op->output_pref)) { 4275 op->output_pref[i] = *la_temp_pref(ts); 4276 } 4277 4278 /* Output args are dead. */ 4279 if (ts->state & TS_DEAD) { 4280 arg_life |= DEAD_ARG << i; 4281 } 4282 if (ts->state & TS_MEM) { 4283 arg_life |= SYNC_ARG << i; 4284 } 4285 ts->state = TS_DEAD; 4286 la_reset_pref(ts); 4287 } 4288 4289 /* If end of basic block, update. */ 4290 if (def->flags & TCG_OPF_BB_EXIT) { 4291 assert_carry_dead(s); 4292 la_func_end(s, nb_globals, nb_temps); 4293 } else if (def->flags & TCG_OPF_COND_BRANCH) { 4294 assert_carry_dead(s); 4295 la_bb_sync(s, nb_globals, nb_temps); 4296 } else if (def->flags & TCG_OPF_BB_END) { 4297 assert_carry_dead(s); 4298 la_bb_end(s, nb_globals, nb_temps); 4299 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { 4300 assert_carry_dead(s); 4301 la_global_sync(s, nb_globals); 4302 if (def->flags & TCG_OPF_CALL_CLOBBER) { 4303 la_cross_call(s, nb_temps); 4304 } 4305 } 4306 4307 /* Record arguments that die in this opcode. */ 4308 for (int i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 4309 ts = arg_temp(op->args[i]); 4310 if (ts->state & TS_DEAD) { 4311 arg_life |= DEAD_ARG << i; 4312 } 4313 } 4314 if (def->flags & TCG_OPF_CARRY_OUT) { 4315 s->carry_live = false; 4316 } 4317 4318 /* Input arguments are live for preceding opcodes. */ 4319 for (int i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 4320 ts = arg_temp(op->args[i]); 4321 if (ts->state & TS_DEAD) { 4322 /* For operands that were dead, initially allow 4323 all regs for the type. */ 4324 *la_temp_pref(ts) = tcg_target_available_regs[ts->type]; 4325 ts->state &= ~TS_DEAD; 4326 } 4327 } 4328 if (def->flags & TCG_OPF_CARRY_IN) { 4329 s->carry_live = true; 4330 } 4331 4332 /* Incorporate constraints for this operand. */ 4333 switch (opc) { 4334 case INDEX_op_mov: 4335 /* Note that these are TCG_OPF_NOT_PRESENT and do not 4336 have proper constraints. That said, special case 4337 moves to propagate preferences backward. */ 4338 if (IS_DEAD_ARG(1)) { 4339 *la_temp_pref(arg_temp(op->args[0])) 4340 = *la_temp_pref(arg_temp(op->args[1])); 4341 } 4342 break; 4343 4344 default: 4345 args_ct = opcode_args_ct(op); 4346 for (int i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 4347 const TCGArgConstraint *ct = &args_ct[i]; 4348 TCGRegSet set, *pset; 4349 4350 ts = arg_temp(op->args[i]); 4351 pset = la_temp_pref(ts); 4352 set = *pset; 4353 4354 set &= ct->regs; 4355 if (ct->ialias) { 4356 set &= output_pref(op, ct->alias_index); 4357 } 4358 /* If the combination is not possible, restart. */ 4359 if (set == 0) { 4360 set = ct->regs; 4361 } 4362 *pset = set; 4363 } 4364 break; 4365 } 4366 break; 4367 } 4368 op->life = arg_life; 4369 } 4370 assert_carry_dead(s); 4371 } 4372 4373 /* Liveness analysis: Convert indirect regs to direct temporaries. */ 4374 static bool __attribute__((noinline)) 4375 liveness_pass_2(TCGContext *s) 4376 { 4377 int nb_globals = s->nb_globals; 4378 int nb_temps, i; 4379 bool changes = false; 4380 TCGOp *op, *op_next; 4381 4382 /* Create a temporary for each indirect global. */ 4383 for (i = 0; i < nb_globals; ++i) { 4384 TCGTemp *its = &s->temps[i]; 4385 if (its->indirect_reg) { 4386 TCGTemp *dts = tcg_temp_alloc(s); 4387 dts->type = its->type; 4388 dts->base_type = its->base_type; 4389 dts->temp_subindex = its->temp_subindex; 4390 dts->kind = TEMP_EBB; 4391 its->state_ptr = dts; 4392 } else { 4393 its->state_ptr = NULL; 4394 } 4395 /* All globals begin dead. */ 4396 its->state = TS_DEAD; 4397 } 4398 for (nb_temps = s->nb_temps; i < nb_temps; ++i) { 4399 TCGTemp *its = &s->temps[i]; 4400 its->state_ptr = NULL; 4401 its->state = TS_DEAD; 4402 } 4403 4404 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) { 4405 TCGOpcode opc = op->opc; 4406 const TCGOpDef *def = &tcg_op_defs[opc]; 4407 TCGLifeData arg_life = op->life; 4408 int nb_iargs, nb_oargs, call_flags; 4409 TCGTemp *arg_ts, *dir_ts; 4410 4411 if (opc == INDEX_op_call) { 4412 nb_oargs = TCGOP_CALLO(op); 4413 nb_iargs = TCGOP_CALLI(op); 4414 call_flags = tcg_call_flags(op); 4415 } else { 4416 nb_iargs = def->nb_iargs; 4417 nb_oargs = def->nb_oargs; 4418 4419 /* Set flags similar to how calls require. */ 4420 if (def->flags & TCG_OPF_COND_BRANCH) { 4421 /* Like reading globals: sync_globals */ 4422 call_flags = TCG_CALL_NO_WRITE_GLOBALS; 4423 } else if (def->flags & TCG_OPF_BB_END) { 4424 /* Like writing globals: save_globals */ 4425 call_flags = 0; 4426 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { 4427 /* Like reading globals: sync_globals */ 4428 call_flags = TCG_CALL_NO_WRITE_GLOBALS; 4429 } else { 4430 /* No effect on globals. */ 4431 call_flags = (TCG_CALL_NO_READ_GLOBALS | 4432 TCG_CALL_NO_WRITE_GLOBALS); 4433 } 4434 } 4435 4436 /* Make sure that input arguments are available. */ 4437 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 4438 arg_ts = arg_temp(op->args[i]); 4439 dir_ts = arg_ts->state_ptr; 4440 if (dir_ts && arg_ts->state == TS_DEAD) { 4441 TCGOp *lop = tcg_op_insert_before(s, op, INDEX_op_ld, 4442 arg_ts->type, 3); 4443 4444 lop->args[0] = temp_arg(dir_ts); 4445 lop->args[1] = temp_arg(arg_ts->mem_base); 4446 lop->args[2] = arg_ts->mem_offset; 4447 4448 /* Loaded, but synced with memory. */ 4449 arg_ts->state = TS_MEM; 4450 } 4451 } 4452 4453 /* Perform input replacement, and mark inputs that became dead. 4454 No action is required except keeping temp_state up to date 4455 so that we reload when needed. */ 4456 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 4457 arg_ts = arg_temp(op->args[i]); 4458 dir_ts = arg_ts->state_ptr; 4459 if (dir_ts) { 4460 op->args[i] = temp_arg(dir_ts); 4461 changes = true; 4462 if (IS_DEAD_ARG(i)) { 4463 arg_ts->state = TS_DEAD; 4464 } 4465 } 4466 } 4467 4468 /* Liveness analysis should ensure that the following are 4469 all correct, for call sites and basic block end points. */ 4470 if (call_flags & TCG_CALL_NO_READ_GLOBALS) { 4471 /* Nothing to do */ 4472 } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) { 4473 for (i = 0; i < nb_globals; ++i) { 4474 /* Liveness should see that globals are synced back, 4475 that is, either TS_DEAD or TS_MEM. */ 4476 arg_ts = &s->temps[i]; 4477 tcg_debug_assert(arg_ts->state_ptr == 0 4478 || arg_ts->state != 0); 4479 } 4480 } else { 4481 for (i = 0; i < nb_globals; ++i) { 4482 /* Liveness should see that globals are saved back, 4483 that is, TS_DEAD, waiting to be reloaded. */ 4484 arg_ts = &s->temps[i]; 4485 tcg_debug_assert(arg_ts->state_ptr == 0 4486 || arg_ts->state == TS_DEAD); 4487 } 4488 } 4489 4490 /* Outputs become available. */ 4491 if (opc == INDEX_op_mov) { 4492 arg_ts = arg_temp(op->args[0]); 4493 dir_ts = arg_ts->state_ptr; 4494 if (dir_ts) { 4495 op->args[0] = temp_arg(dir_ts); 4496 changes = true; 4497 4498 /* The output is now live and modified. */ 4499 arg_ts->state = 0; 4500 4501 if (NEED_SYNC_ARG(0)) { 4502 TCGOp *sop = tcg_op_insert_after(s, op, INDEX_op_st, 4503 arg_ts->type, 3); 4504 TCGTemp *out_ts = dir_ts; 4505 4506 if (IS_DEAD_ARG(0)) { 4507 out_ts = arg_temp(op->args[1]); 4508 arg_ts->state = TS_DEAD; 4509 tcg_op_remove(s, op); 4510 } else { 4511 arg_ts->state = TS_MEM; 4512 } 4513 4514 sop->args[0] = temp_arg(out_ts); 4515 sop->args[1] = temp_arg(arg_ts->mem_base); 4516 sop->args[2] = arg_ts->mem_offset; 4517 } else { 4518 tcg_debug_assert(!IS_DEAD_ARG(0)); 4519 } 4520 } 4521 } else { 4522 for (i = 0; i < nb_oargs; i++) { 4523 arg_ts = arg_temp(op->args[i]); 4524 dir_ts = arg_ts->state_ptr; 4525 if (!dir_ts) { 4526 continue; 4527 } 4528 op->args[i] = temp_arg(dir_ts); 4529 changes = true; 4530 4531 /* The output is now live and modified. */ 4532 arg_ts->state = 0; 4533 4534 /* Sync outputs upon their last write. */ 4535 if (NEED_SYNC_ARG(i)) { 4536 TCGOp *sop = tcg_op_insert_after(s, op, INDEX_op_st, 4537 arg_ts->type, 3); 4538 4539 sop->args[0] = temp_arg(dir_ts); 4540 sop->args[1] = temp_arg(arg_ts->mem_base); 4541 sop->args[2] = arg_ts->mem_offset; 4542 4543 arg_ts->state = TS_MEM; 4544 } 4545 /* Drop outputs that are dead. */ 4546 if (IS_DEAD_ARG(i)) { 4547 arg_ts->state = TS_DEAD; 4548 } 4549 } 4550 } 4551 } 4552 4553 return changes; 4554 } 4555 4556 static void temp_allocate_frame(TCGContext *s, TCGTemp *ts) 4557 { 4558 intptr_t off; 4559 int size, align; 4560 4561 /* When allocating an object, look at the full type. */ 4562 size = tcg_type_size(ts->base_type); 4563 switch (ts->base_type) { 4564 case TCG_TYPE_I32: 4565 align = 4; 4566 break; 4567 case TCG_TYPE_I64: 4568 case TCG_TYPE_V64: 4569 align = 8; 4570 break; 4571 case TCG_TYPE_I128: 4572 case TCG_TYPE_V128: 4573 case TCG_TYPE_V256: 4574 /* 4575 * Note that we do not require aligned storage for V256, 4576 * and that we provide alignment for I128 to match V128, 4577 * even if that's above what the host ABI requires. 4578 */ 4579 align = 16; 4580 break; 4581 default: 4582 g_assert_not_reached(); 4583 } 4584 4585 /* 4586 * Assume the stack is sufficiently aligned. 4587 * This affects e.g. ARM NEON, where we have 8 byte stack alignment 4588 * and do not require 16 byte vector alignment. This seems slightly 4589 * easier than fully parameterizing the above switch statement. 4590 */ 4591 align = MIN(TCG_TARGET_STACK_ALIGN, align); 4592 off = ROUND_UP(s->current_frame_offset, align); 4593 4594 /* If we've exhausted the stack frame, restart with a smaller TB. */ 4595 if (off + size > s->frame_end) { 4596 tcg_raise_tb_overflow(s); 4597 } 4598 s->current_frame_offset = off + size; 4599 #if defined(__sparc__) 4600 off += TCG_TARGET_STACK_BIAS; 4601 #endif 4602 4603 /* If the object was subdivided, assign memory to all the parts. */ 4604 if (ts->base_type != ts->type) { 4605 int part_size = tcg_type_size(ts->type); 4606 int part_count = size / part_size; 4607 4608 /* 4609 * Each part is allocated sequentially in tcg_temp_new_internal. 4610 * Jump back to the first part by subtracting the current index. 4611 */ 4612 ts -= ts->temp_subindex; 4613 for (int i = 0; i < part_count; ++i) { 4614 ts[i].mem_offset = off + i * part_size; 4615 ts[i].mem_base = s->frame_temp; 4616 ts[i].mem_allocated = 1; 4617 } 4618 } else { 4619 ts->mem_offset = off; 4620 ts->mem_base = s->frame_temp; 4621 ts->mem_allocated = 1; 4622 } 4623 } 4624 4625 /* Assign @reg to @ts, and update reg_to_temp[]. */ 4626 static void set_temp_val_reg(TCGContext *s, TCGTemp *ts, TCGReg reg) 4627 { 4628 if (ts->val_type == TEMP_VAL_REG) { 4629 TCGReg old = ts->reg; 4630 tcg_debug_assert(s->reg_to_temp[old] == ts); 4631 if (old == reg) { 4632 return; 4633 } 4634 s->reg_to_temp[old] = NULL; 4635 } 4636 tcg_debug_assert(s->reg_to_temp[reg] == NULL); 4637 s->reg_to_temp[reg] = ts; 4638 ts->val_type = TEMP_VAL_REG; 4639 ts->reg = reg; 4640 } 4641 4642 /* Assign a non-register value type to @ts, and update reg_to_temp[]. */ 4643 static void set_temp_val_nonreg(TCGContext *s, TCGTemp *ts, TCGTempVal type) 4644 { 4645 tcg_debug_assert(type != TEMP_VAL_REG); 4646 if (ts->val_type == TEMP_VAL_REG) { 4647 TCGReg reg = ts->reg; 4648 tcg_debug_assert(s->reg_to_temp[reg] == ts); 4649 s->reg_to_temp[reg] = NULL; 4650 } 4651 ts->val_type = type; 4652 } 4653 4654 static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet); 4655 4656 /* Mark a temporary as free or dead. If 'free_or_dead' is negative, 4657 mark it free; otherwise mark it dead. */ 4658 static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead) 4659 { 4660 TCGTempVal new_type; 4661 4662 switch (ts->kind) { 4663 case TEMP_FIXED: 4664 return; 4665 case TEMP_GLOBAL: 4666 case TEMP_TB: 4667 new_type = TEMP_VAL_MEM; 4668 break; 4669 case TEMP_EBB: 4670 new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD; 4671 break; 4672 case TEMP_CONST: 4673 new_type = TEMP_VAL_CONST; 4674 break; 4675 default: 4676 g_assert_not_reached(); 4677 } 4678 set_temp_val_nonreg(s, ts, new_type); 4679 } 4680 4681 /* Mark a temporary as dead. */ 4682 static inline void temp_dead(TCGContext *s, TCGTemp *ts) 4683 { 4684 temp_free_or_dead(s, ts, 1); 4685 } 4686 4687 /* Sync a temporary to memory. 'allocated_regs' is used in case a temporary 4688 registers needs to be allocated to store a constant. If 'free_or_dead' 4689 is non-zero, subsequently release the temporary; if it is positive, the 4690 temp is dead; if it is negative, the temp is free. */ 4691 static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs, 4692 TCGRegSet preferred_regs, int free_or_dead) 4693 { 4694 if (!temp_readonly(ts) && !ts->mem_coherent) { 4695 if (!ts->mem_allocated) { 4696 temp_allocate_frame(s, ts); 4697 } 4698 switch (ts->val_type) { 4699 case TEMP_VAL_CONST: 4700 /* If we're going to free the temp immediately, then we won't 4701 require it later in a register, so attempt to store the 4702 constant to memory directly. */ 4703 if (free_or_dead 4704 && tcg_out_sti(s, ts->type, ts->val, 4705 ts->mem_base->reg, ts->mem_offset)) { 4706 break; 4707 } 4708 temp_load(s, ts, tcg_target_available_regs[ts->type], 4709 allocated_regs, preferred_regs); 4710 /* fallthrough */ 4711 4712 case TEMP_VAL_REG: 4713 tcg_out_st(s, ts->type, ts->reg, 4714 ts->mem_base->reg, ts->mem_offset); 4715 break; 4716 4717 case TEMP_VAL_MEM: 4718 break; 4719 4720 case TEMP_VAL_DEAD: 4721 default: 4722 g_assert_not_reached(); 4723 } 4724 ts->mem_coherent = 1; 4725 } 4726 if (free_or_dead) { 4727 temp_free_or_dead(s, ts, free_or_dead); 4728 } 4729 } 4730 4731 /* free register 'reg' by spilling the corresponding temporary if necessary */ 4732 static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs) 4733 { 4734 TCGTemp *ts = s->reg_to_temp[reg]; 4735 if (ts != NULL) { 4736 temp_sync(s, ts, allocated_regs, 0, -1); 4737 } 4738 } 4739 4740 /** 4741 * tcg_reg_alloc: 4742 * @required_regs: Set of registers in which we must allocate. 4743 * @allocated_regs: Set of registers which must be avoided. 4744 * @preferred_regs: Set of registers we should prefer. 4745 * @rev: True if we search the registers in "indirect" order. 4746 * 4747 * The allocated register must be in @required_regs & ~@allocated_regs, 4748 * but if we can put it in @preferred_regs we may save a move later. 4749 */ 4750 static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs, 4751 TCGRegSet allocated_regs, 4752 TCGRegSet preferred_regs, bool rev) 4753 { 4754 int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order); 4755 TCGRegSet reg_ct[2]; 4756 const int *order; 4757 4758 reg_ct[1] = required_regs & ~allocated_regs; 4759 tcg_debug_assert(reg_ct[1] != 0); 4760 reg_ct[0] = reg_ct[1] & preferred_regs; 4761 4762 /* Skip the preferred_regs option if it cannot be satisfied, 4763 or if the preference made no difference. */ 4764 f = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1]; 4765 4766 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order; 4767 4768 /* Try free registers, preferences first. */ 4769 for (j = f; j < 2; j++) { 4770 TCGRegSet set = reg_ct[j]; 4771 4772 if (tcg_regset_single(set)) { 4773 /* One register in the set. */ 4774 TCGReg reg = tcg_regset_first(set); 4775 if (s->reg_to_temp[reg] == NULL) { 4776 return reg; 4777 } 4778 } else { 4779 for (i = 0; i < n; i++) { 4780 TCGReg reg = order[i]; 4781 if (s->reg_to_temp[reg] == NULL && 4782 tcg_regset_test_reg(set, reg)) { 4783 return reg; 4784 } 4785 } 4786 } 4787 } 4788 4789 /* We must spill something. */ 4790 for (j = f; j < 2; j++) { 4791 TCGRegSet set = reg_ct[j]; 4792 4793 if (tcg_regset_single(set)) { 4794 /* One register in the set. */ 4795 TCGReg reg = tcg_regset_first(set); 4796 tcg_reg_free(s, reg, allocated_regs); 4797 return reg; 4798 } else { 4799 for (i = 0; i < n; i++) { 4800 TCGReg reg = order[i]; 4801 if (tcg_regset_test_reg(set, reg)) { 4802 tcg_reg_free(s, reg, allocated_regs); 4803 return reg; 4804 } 4805 } 4806 } 4807 } 4808 4809 g_assert_not_reached(); 4810 } 4811 4812 static TCGReg tcg_reg_alloc_pair(TCGContext *s, TCGRegSet required_regs, 4813 TCGRegSet allocated_regs, 4814 TCGRegSet preferred_regs, bool rev) 4815 { 4816 int i, j, k, fmin, n = ARRAY_SIZE(tcg_target_reg_alloc_order); 4817 TCGRegSet reg_ct[2]; 4818 const int *order; 4819 4820 /* Ensure that if I is not in allocated_regs, I+1 is not either. */ 4821 reg_ct[1] = required_regs & ~(allocated_regs | (allocated_regs >> 1)); 4822 tcg_debug_assert(reg_ct[1] != 0); 4823 reg_ct[0] = reg_ct[1] & preferred_regs; 4824 4825 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order; 4826 4827 /* 4828 * Skip the preferred_regs option if it cannot be satisfied, 4829 * or if the preference made no difference. 4830 */ 4831 k = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1]; 4832 4833 /* 4834 * Minimize the number of flushes by looking for 2 free registers first, 4835 * then a single flush, then two flushes. 4836 */ 4837 for (fmin = 2; fmin >= 0; fmin--) { 4838 for (j = k; j < 2; j++) { 4839 TCGRegSet set = reg_ct[j]; 4840 4841 for (i = 0; i < n; i++) { 4842 TCGReg reg = order[i]; 4843 4844 if (tcg_regset_test_reg(set, reg)) { 4845 int f = !s->reg_to_temp[reg] + !s->reg_to_temp[reg + 1]; 4846 if (f >= fmin) { 4847 tcg_reg_free(s, reg, allocated_regs); 4848 tcg_reg_free(s, reg + 1, allocated_regs); 4849 return reg; 4850 } 4851 } 4852 } 4853 } 4854 } 4855 g_assert_not_reached(); 4856 } 4857 4858 /* Make sure the temporary is in a register. If needed, allocate the register 4859 from DESIRED while avoiding ALLOCATED. */ 4860 static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs, 4861 TCGRegSet allocated_regs, TCGRegSet preferred_regs) 4862 { 4863 TCGReg reg; 4864 4865 switch (ts->val_type) { 4866 case TEMP_VAL_REG: 4867 return; 4868 case TEMP_VAL_CONST: 4869 reg = tcg_reg_alloc(s, desired_regs, allocated_regs, 4870 preferred_regs, ts->indirect_base); 4871 if (ts->type <= TCG_TYPE_I64) { 4872 tcg_out_movi(s, ts->type, reg, ts->val); 4873 } else { 4874 uint64_t val = ts->val; 4875 MemOp vece = MO_64; 4876 4877 /* 4878 * Find the minimal vector element that matches the constant. 4879 * The targets will, in general, have to do this search anyway, 4880 * do this generically. 4881 */ 4882 if (val == dup_const(MO_8, val)) { 4883 vece = MO_8; 4884 } else if (val == dup_const(MO_16, val)) { 4885 vece = MO_16; 4886 } else if (val == dup_const(MO_32, val)) { 4887 vece = MO_32; 4888 } 4889 4890 tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val); 4891 } 4892 ts->mem_coherent = 0; 4893 break; 4894 case TEMP_VAL_MEM: 4895 if (!ts->mem_allocated) { 4896 temp_allocate_frame(s, ts); 4897 } 4898 reg = tcg_reg_alloc(s, desired_regs, allocated_regs, 4899 preferred_regs, ts->indirect_base); 4900 tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset); 4901 ts->mem_coherent = 1; 4902 break; 4903 case TEMP_VAL_DEAD: 4904 default: 4905 g_assert_not_reached(); 4906 } 4907 set_temp_val_reg(s, ts, reg); 4908 } 4909 4910 /* Save a temporary to memory. 'allocated_regs' is used in case a 4911 temporary registers needs to be allocated to store a constant. */ 4912 static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs) 4913 { 4914 /* The liveness analysis already ensures that globals are back 4915 in memory. Keep an tcg_debug_assert for safety. */ 4916 tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts)); 4917 } 4918 4919 /* save globals to their canonical location and assume they can be 4920 modified be the following code. 'allocated_regs' is used in case a 4921 temporary registers needs to be allocated to store a constant. */ 4922 static void save_globals(TCGContext *s, TCGRegSet allocated_regs) 4923 { 4924 int i, n; 4925 4926 for (i = 0, n = s->nb_globals; i < n; i++) { 4927 temp_save(s, &s->temps[i], allocated_regs); 4928 } 4929 } 4930 4931 /* sync globals to their canonical location and assume they can be 4932 read by the following code. 'allocated_regs' is used in case a 4933 temporary registers needs to be allocated to store a constant. */ 4934 static void sync_globals(TCGContext *s, TCGRegSet allocated_regs) 4935 { 4936 int i, n; 4937 4938 for (i = 0, n = s->nb_globals; i < n; i++) { 4939 TCGTemp *ts = &s->temps[i]; 4940 tcg_debug_assert(ts->val_type != TEMP_VAL_REG 4941 || ts->kind == TEMP_FIXED 4942 || ts->mem_coherent); 4943 } 4944 } 4945 4946 /* at the end of a basic block, we assume all temporaries are dead and 4947 all globals are stored at their canonical location. */ 4948 static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs) 4949 { 4950 assert_carry_dead(s); 4951 for (int i = s->nb_globals; i < s->nb_temps; i++) { 4952 TCGTemp *ts = &s->temps[i]; 4953 4954 switch (ts->kind) { 4955 case TEMP_TB: 4956 temp_save(s, ts, allocated_regs); 4957 break; 4958 case TEMP_EBB: 4959 /* The liveness analysis already ensures that temps are dead. 4960 Keep an tcg_debug_assert for safety. */ 4961 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD); 4962 break; 4963 case TEMP_CONST: 4964 /* Similarly, we should have freed any allocated register. */ 4965 tcg_debug_assert(ts->val_type == TEMP_VAL_CONST); 4966 break; 4967 default: 4968 g_assert_not_reached(); 4969 } 4970 } 4971 4972 save_globals(s, allocated_regs); 4973 } 4974 4975 /* 4976 * At a conditional branch, we assume all temporaries are dead unless 4977 * explicitly live-across-conditional-branch; all globals and local 4978 * temps are synced to their location. 4979 */ 4980 static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs) 4981 { 4982 assert_carry_dead(s); 4983 sync_globals(s, allocated_regs); 4984 4985 for (int i = s->nb_globals; i < s->nb_temps; i++) { 4986 TCGTemp *ts = &s->temps[i]; 4987 /* 4988 * The liveness analysis already ensures that temps are dead. 4989 * Keep tcg_debug_asserts for safety. 4990 */ 4991 switch (ts->kind) { 4992 case TEMP_TB: 4993 tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent); 4994 break; 4995 case TEMP_EBB: 4996 case TEMP_CONST: 4997 break; 4998 default: 4999 g_assert_not_reached(); 5000 } 5001 } 5002 } 5003 5004 /* 5005 * Specialized code generation for INDEX_op_mov_* with a constant. 5006 */ 5007 static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots, 5008 tcg_target_ulong val, TCGLifeData arg_life, 5009 TCGRegSet preferred_regs) 5010 { 5011 /* ENV should not be modified. */ 5012 tcg_debug_assert(!temp_readonly(ots)); 5013 5014 /* The movi is not explicitly generated here. */ 5015 set_temp_val_nonreg(s, ots, TEMP_VAL_CONST); 5016 ots->val = val; 5017 ots->mem_coherent = 0; 5018 if (NEED_SYNC_ARG(0)) { 5019 temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0)); 5020 } else if (IS_DEAD_ARG(0)) { 5021 temp_dead(s, ots); 5022 } 5023 } 5024 5025 /* 5026 * Specialized code generation for INDEX_op_mov_*. 5027 */ 5028 static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op) 5029 { 5030 const TCGLifeData arg_life = op->life; 5031 TCGRegSet allocated_regs, preferred_regs; 5032 TCGTemp *ts, *ots; 5033 TCGType otype, itype; 5034 TCGReg oreg, ireg; 5035 5036 allocated_regs = s->reserved_regs; 5037 preferred_regs = output_pref(op, 0); 5038 ots = arg_temp(op->args[0]); 5039 ts = arg_temp(op->args[1]); 5040 5041 /* ENV should not be modified. */ 5042 tcg_debug_assert(!temp_readonly(ots)); 5043 5044 /* Note that otype != itype for no-op truncation. */ 5045 otype = ots->type; 5046 itype = ts->type; 5047 5048 if (ts->val_type == TEMP_VAL_CONST) { 5049 /* propagate constant or generate sti */ 5050 tcg_target_ulong val = ts->val; 5051 if (IS_DEAD_ARG(1)) { 5052 temp_dead(s, ts); 5053 } 5054 tcg_reg_alloc_do_movi(s, ots, val, arg_life, preferred_regs); 5055 return; 5056 } 5057 5058 /* If the source value is in memory we're going to be forced 5059 to have it in a register in order to perform the copy. Copy 5060 the SOURCE value into its own register first, that way we 5061 don't have to reload SOURCE the next time it is used. */ 5062 if (ts->val_type == TEMP_VAL_MEM) { 5063 temp_load(s, ts, tcg_target_available_regs[itype], 5064 allocated_regs, preferred_regs); 5065 } 5066 tcg_debug_assert(ts->val_type == TEMP_VAL_REG); 5067 ireg = ts->reg; 5068 5069 if (IS_DEAD_ARG(0)) { 5070 /* mov to a non-saved dead register makes no sense (even with 5071 liveness analysis disabled). */ 5072 tcg_debug_assert(NEED_SYNC_ARG(0)); 5073 if (!ots->mem_allocated) { 5074 temp_allocate_frame(s, ots); 5075 } 5076 tcg_out_st(s, otype, ireg, ots->mem_base->reg, ots->mem_offset); 5077 if (IS_DEAD_ARG(1)) { 5078 temp_dead(s, ts); 5079 } 5080 temp_dead(s, ots); 5081 return; 5082 } 5083 5084 if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) { 5085 /* 5086 * The mov can be suppressed. Kill input first, so that it 5087 * is unlinked from reg_to_temp, then set the output to the 5088 * reg that we saved from the input. 5089 */ 5090 temp_dead(s, ts); 5091 oreg = ireg; 5092 } else { 5093 if (ots->val_type == TEMP_VAL_REG) { 5094 oreg = ots->reg; 5095 } else { 5096 /* Make sure to not spill the input register during allocation. */ 5097 oreg = tcg_reg_alloc(s, tcg_target_available_regs[otype], 5098 allocated_regs | ((TCGRegSet)1 << ireg), 5099 preferred_regs, ots->indirect_base); 5100 } 5101 if (!tcg_out_mov(s, otype, oreg, ireg)) { 5102 /* 5103 * Cross register class move not supported. 5104 * Store the source register into the destination slot 5105 * and leave the destination temp as TEMP_VAL_MEM. 5106 */ 5107 assert(!temp_readonly(ots)); 5108 if (!ts->mem_allocated) { 5109 temp_allocate_frame(s, ots); 5110 } 5111 tcg_out_st(s, ts->type, ireg, ots->mem_base->reg, ots->mem_offset); 5112 set_temp_val_nonreg(s, ts, TEMP_VAL_MEM); 5113 ots->mem_coherent = 1; 5114 return; 5115 } 5116 } 5117 set_temp_val_reg(s, ots, oreg); 5118 ots->mem_coherent = 0; 5119 5120 if (NEED_SYNC_ARG(0)) { 5121 temp_sync(s, ots, allocated_regs, 0, 0); 5122 } 5123 } 5124 5125 /* 5126 * Specialized code generation for INDEX_op_dup_vec. 5127 */ 5128 static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op) 5129 { 5130 const TCGLifeData arg_life = op->life; 5131 TCGRegSet dup_out_regs, dup_in_regs; 5132 const TCGArgConstraint *dup_args_ct; 5133 TCGTemp *its, *ots; 5134 TCGType itype, vtype; 5135 unsigned vece; 5136 int lowpart_ofs; 5137 bool ok; 5138 5139 ots = arg_temp(op->args[0]); 5140 its = arg_temp(op->args[1]); 5141 5142 /* ENV should not be modified. */ 5143 tcg_debug_assert(!temp_readonly(ots)); 5144 5145 itype = its->type; 5146 vece = TCGOP_VECE(op); 5147 vtype = TCGOP_TYPE(op); 5148 5149 if (its->val_type == TEMP_VAL_CONST) { 5150 /* Propagate constant via movi -> dupi. */ 5151 tcg_target_ulong val = its->val; 5152 if (IS_DEAD_ARG(1)) { 5153 temp_dead(s, its); 5154 } 5155 tcg_reg_alloc_do_movi(s, ots, val, arg_life, output_pref(op, 0)); 5156 return; 5157 } 5158 5159 dup_args_ct = opcode_args_ct(op); 5160 dup_out_regs = dup_args_ct[0].regs; 5161 dup_in_regs = dup_args_ct[1].regs; 5162 5163 /* Allocate the output register now. */ 5164 if (ots->val_type != TEMP_VAL_REG) { 5165 TCGRegSet allocated_regs = s->reserved_regs; 5166 TCGReg oreg; 5167 5168 if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) { 5169 /* Make sure to not spill the input register. */ 5170 tcg_regset_set_reg(allocated_regs, its->reg); 5171 } 5172 oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs, 5173 output_pref(op, 0), ots->indirect_base); 5174 set_temp_val_reg(s, ots, oreg); 5175 } 5176 5177 switch (its->val_type) { 5178 case TEMP_VAL_REG: 5179 /* 5180 * The dup constriaints must be broad, covering all possible VECE. 5181 * However, tcg_op_dup_vec() gets to see the VECE and we allow it 5182 * to fail, indicating that extra moves are required for that case. 5183 */ 5184 if (tcg_regset_test_reg(dup_in_regs, its->reg)) { 5185 if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) { 5186 goto done; 5187 } 5188 /* Try again from memory or a vector input register. */ 5189 } 5190 if (!its->mem_coherent) { 5191 /* 5192 * The input register is not synced, and so an extra store 5193 * would be required to use memory. Attempt an integer-vector 5194 * register move first. We do not have a TCGRegSet for this. 5195 */ 5196 if (tcg_out_mov(s, itype, ots->reg, its->reg)) { 5197 break; 5198 } 5199 /* Sync the temp back to its slot and load from there. */ 5200 temp_sync(s, its, s->reserved_regs, 0, 0); 5201 } 5202 /* fall through */ 5203 5204 case TEMP_VAL_MEM: 5205 lowpart_ofs = 0; 5206 if (HOST_BIG_ENDIAN) { 5207 lowpart_ofs = tcg_type_size(itype) - (1 << vece); 5208 } 5209 if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg, 5210 its->mem_offset + lowpart_ofs)) { 5211 goto done; 5212 } 5213 /* Load the input into the destination vector register. */ 5214 tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset); 5215 break; 5216 5217 default: 5218 g_assert_not_reached(); 5219 } 5220 5221 /* We now have a vector input register, so dup must succeed. */ 5222 ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg); 5223 tcg_debug_assert(ok); 5224 5225 done: 5226 ots->mem_coherent = 0; 5227 if (IS_DEAD_ARG(1)) { 5228 temp_dead(s, its); 5229 } 5230 if (NEED_SYNC_ARG(0)) { 5231 temp_sync(s, ots, s->reserved_regs, 0, 0); 5232 } 5233 if (IS_DEAD_ARG(0)) { 5234 temp_dead(s, ots); 5235 } 5236 } 5237 5238 static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) 5239 { 5240 const TCGLifeData arg_life = op->life; 5241 const TCGOpDef * const def = &tcg_op_defs[op->opc]; 5242 TCGRegSet i_allocated_regs; 5243 TCGRegSet o_allocated_regs; 5244 int i, k, nb_iargs, nb_oargs; 5245 TCGReg reg; 5246 TCGArg arg; 5247 const TCGArgConstraint *args_ct; 5248 const TCGArgConstraint *arg_ct; 5249 TCGTemp *ts; 5250 TCGArg new_args[TCG_MAX_OP_ARGS]; 5251 int const_args[TCG_MAX_OP_ARGS]; 5252 TCGCond op_cond; 5253 5254 if (def->flags & TCG_OPF_CARRY_IN) { 5255 tcg_debug_assert(s->carry_live); 5256 } 5257 5258 nb_oargs = def->nb_oargs; 5259 nb_iargs = def->nb_iargs; 5260 5261 /* copy constants */ 5262 memcpy(new_args + nb_oargs + nb_iargs, 5263 op->args + nb_oargs + nb_iargs, 5264 sizeof(TCGArg) * def->nb_cargs); 5265 5266 i_allocated_regs = s->reserved_regs; 5267 o_allocated_regs = s->reserved_regs; 5268 5269 switch (op->opc) { 5270 case INDEX_op_brcond: 5271 op_cond = op->args[2]; 5272 break; 5273 case INDEX_op_setcond: 5274 case INDEX_op_negsetcond: 5275 case INDEX_op_cmp_vec: 5276 op_cond = op->args[3]; 5277 break; 5278 case INDEX_op_brcond2_i32: 5279 op_cond = op->args[4]; 5280 break; 5281 case INDEX_op_movcond: 5282 case INDEX_op_setcond2_i32: 5283 case INDEX_op_cmpsel_vec: 5284 op_cond = op->args[5]; 5285 break; 5286 default: 5287 /* No condition within opcode. */ 5288 op_cond = TCG_COND_ALWAYS; 5289 break; 5290 } 5291 5292 args_ct = opcode_args_ct(op); 5293 5294 /* satisfy input constraints */ 5295 for (k = 0; k < nb_iargs; k++) { 5296 TCGRegSet i_preferred_regs, i_required_regs; 5297 bool allocate_new_reg, copyto_new_reg; 5298 TCGTemp *ts2; 5299 int i1, i2; 5300 5301 i = args_ct[nb_oargs + k].sort_index; 5302 arg = op->args[i]; 5303 arg_ct = &args_ct[i]; 5304 ts = arg_temp(arg); 5305 5306 if (ts->val_type == TEMP_VAL_CONST) { 5307 #ifdef TCG_REG_ZERO 5308 if (ts->val == 0 && (arg_ct->ct & TCG_CT_REG_ZERO)) { 5309 /* Hardware zero register: indicate register via non-const. */ 5310 const_args[i] = 0; 5311 new_args[i] = TCG_REG_ZERO; 5312 continue; 5313 } 5314 #endif 5315 5316 if (tcg_target_const_match(ts->val, arg_ct->ct, ts->type, 5317 op_cond, TCGOP_VECE(op))) { 5318 /* constant is OK for instruction */ 5319 const_args[i] = 1; 5320 new_args[i] = ts->val; 5321 continue; 5322 } 5323 } 5324 5325 reg = ts->reg; 5326 i_preferred_regs = 0; 5327 i_required_regs = arg_ct->regs; 5328 allocate_new_reg = false; 5329 copyto_new_reg = false; 5330 5331 switch (arg_ct->pair) { 5332 case 0: /* not paired */ 5333 if (arg_ct->ialias) { 5334 i_preferred_regs = output_pref(op, arg_ct->alias_index); 5335 5336 /* 5337 * If the input is readonly, then it cannot also be an 5338 * output and aliased to itself. If the input is not 5339 * dead after the instruction, we must allocate a new 5340 * register and move it. 5341 */ 5342 if (temp_readonly(ts) || !IS_DEAD_ARG(i) 5343 || args_ct[arg_ct->alias_index].newreg) { 5344 allocate_new_reg = true; 5345 } else if (ts->val_type == TEMP_VAL_REG) { 5346 /* 5347 * Check if the current register has already been 5348 * allocated for another input. 5349 */ 5350 allocate_new_reg = 5351 tcg_regset_test_reg(i_allocated_regs, reg); 5352 } 5353 } 5354 if (!allocate_new_reg) { 5355 temp_load(s, ts, i_required_regs, i_allocated_regs, 5356 i_preferred_regs); 5357 reg = ts->reg; 5358 allocate_new_reg = !tcg_regset_test_reg(i_required_regs, reg); 5359 } 5360 if (allocate_new_reg) { 5361 /* 5362 * Allocate a new register matching the constraint 5363 * and move the temporary register into it. 5364 */ 5365 temp_load(s, ts, tcg_target_available_regs[ts->type], 5366 i_allocated_regs, 0); 5367 reg = tcg_reg_alloc(s, i_required_regs, i_allocated_regs, 5368 i_preferred_regs, ts->indirect_base); 5369 copyto_new_reg = true; 5370 } 5371 break; 5372 5373 case 1: 5374 /* First of an input pair; if i1 == i2, the second is an output. */ 5375 i1 = i; 5376 i2 = arg_ct->pair_index; 5377 ts2 = i1 != i2 ? arg_temp(op->args[i2]) : NULL; 5378 5379 /* 5380 * It is easier to default to allocating a new pair 5381 * and to identify a few cases where it's not required. 5382 */ 5383 if (arg_ct->ialias) { 5384 i_preferred_regs = output_pref(op, arg_ct->alias_index); 5385 if (IS_DEAD_ARG(i1) && 5386 IS_DEAD_ARG(i2) && 5387 !temp_readonly(ts) && 5388 ts->val_type == TEMP_VAL_REG && 5389 ts->reg < TCG_TARGET_NB_REGS - 1 && 5390 tcg_regset_test_reg(i_required_regs, reg) && 5391 !tcg_regset_test_reg(i_allocated_regs, reg) && 5392 !tcg_regset_test_reg(i_allocated_regs, reg + 1) && 5393 (ts2 5394 ? ts2->val_type == TEMP_VAL_REG && 5395 ts2->reg == reg + 1 && 5396 !temp_readonly(ts2) 5397 : s->reg_to_temp[reg + 1] == NULL)) { 5398 break; 5399 } 5400 } else { 5401 /* Without aliasing, the pair must also be an input. */ 5402 tcg_debug_assert(ts2); 5403 if (ts->val_type == TEMP_VAL_REG && 5404 ts2->val_type == TEMP_VAL_REG && 5405 ts2->reg == reg + 1 && 5406 tcg_regset_test_reg(i_required_regs, reg)) { 5407 break; 5408 } 5409 } 5410 reg = tcg_reg_alloc_pair(s, i_required_regs, i_allocated_regs, 5411 0, ts->indirect_base); 5412 goto do_pair; 5413 5414 case 2: /* pair second */ 5415 reg = new_args[arg_ct->pair_index] + 1; 5416 goto do_pair; 5417 5418 case 3: /* ialias with second output, no first input */ 5419 tcg_debug_assert(arg_ct->ialias); 5420 i_preferred_regs = output_pref(op, arg_ct->alias_index); 5421 5422 if (IS_DEAD_ARG(i) && 5423 !temp_readonly(ts) && 5424 ts->val_type == TEMP_VAL_REG && 5425 reg > 0 && 5426 s->reg_to_temp[reg - 1] == NULL && 5427 tcg_regset_test_reg(i_required_regs, reg) && 5428 !tcg_regset_test_reg(i_allocated_regs, reg) && 5429 !tcg_regset_test_reg(i_allocated_regs, reg - 1)) { 5430 tcg_regset_set_reg(i_allocated_regs, reg - 1); 5431 break; 5432 } 5433 reg = tcg_reg_alloc_pair(s, i_required_regs >> 1, 5434 i_allocated_regs, 0, 5435 ts->indirect_base); 5436 tcg_regset_set_reg(i_allocated_regs, reg); 5437 reg += 1; 5438 goto do_pair; 5439 5440 do_pair: 5441 /* 5442 * If an aliased input is not dead after the instruction, 5443 * we must allocate a new register and move it. 5444 */ 5445 if (arg_ct->ialias && (!IS_DEAD_ARG(i) || temp_readonly(ts))) { 5446 TCGRegSet t_allocated_regs = i_allocated_regs; 5447 5448 /* 5449 * Because of the alias, and the continued life, make sure 5450 * that the temp is somewhere *other* than the reg pair, 5451 * and we get a copy in reg. 5452 */ 5453 tcg_regset_set_reg(t_allocated_regs, reg); 5454 tcg_regset_set_reg(t_allocated_regs, reg + 1); 5455 if (ts->val_type == TEMP_VAL_REG && ts->reg == reg) { 5456 /* If ts was already in reg, copy it somewhere else. */ 5457 TCGReg nr; 5458 bool ok; 5459 5460 tcg_debug_assert(ts->kind != TEMP_FIXED); 5461 nr = tcg_reg_alloc(s, tcg_target_available_regs[ts->type], 5462 t_allocated_regs, 0, ts->indirect_base); 5463 ok = tcg_out_mov(s, ts->type, nr, reg); 5464 tcg_debug_assert(ok); 5465 5466 set_temp_val_reg(s, ts, nr); 5467 } else { 5468 temp_load(s, ts, tcg_target_available_regs[ts->type], 5469 t_allocated_regs, 0); 5470 copyto_new_reg = true; 5471 } 5472 } else { 5473 /* Preferably allocate to reg, otherwise copy. */ 5474 i_required_regs = (TCGRegSet)1 << reg; 5475 temp_load(s, ts, i_required_regs, i_allocated_regs, 5476 i_preferred_regs); 5477 copyto_new_reg = ts->reg != reg; 5478 } 5479 break; 5480 5481 default: 5482 g_assert_not_reached(); 5483 } 5484 5485 if (copyto_new_reg) { 5486 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) { 5487 /* 5488 * Cross register class move not supported. Sync the 5489 * temp back to its slot and load from there. 5490 */ 5491 temp_sync(s, ts, i_allocated_regs, 0, 0); 5492 tcg_out_ld(s, ts->type, reg, 5493 ts->mem_base->reg, ts->mem_offset); 5494 } 5495 } 5496 new_args[i] = reg; 5497 const_args[i] = 0; 5498 tcg_regset_set_reg(i_allocated_regs, reg); 5499 } 5500 5501 /* mark dead temporaries and free the associated registers */ 5502 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 5503 if (IS_DEAD_ARG(i)) { 5504 temp_dead(s, arg_temp(op->args[i])); 5505 } 5506 } 5507 5508 if (def->flags & TCG_OPF_COND_BRANCH) { 5509 tcg_reg_alloc_cbranch(s, i_allocated_regs); 5510 } else if (def->flags & TCG_OPF_BB_END) { 5511 tcg_reg_alloc_bb_end(s, i_allocated_regs); 5512 } else { 5513 if (def->flags & TCG_OPF_CALL_CLOBBER) { 5514 assert_carry_dead(s); 5515 /* XXX: permit generic clobber register list ? */ 5516 for (i = 0; i < TCG_TARGET_NB_REGS; i++) { 5517 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { 5518 tcg_reg_free(s, i, i_allocated_regs); 5519 } 5520 } 5521 } 5522 if (def->flags & TCG_OPF_SIDE_EFFECTS) { 5523 /* sync globals if the op has side effects and might trigger 5524 an exception. */ 5525 sync_globals(s, i_allocated_regs); 5526 } 5527 5528 /* satisfy the output constraints */ 5529 for (k = 0; k < nb_oargs; k++) { 5530 i = args_ct[k].sort_index; 5531 arg = op->args[i]; 5532 arg_ct = &args_ct[i]; 5533 ts = arg_temp(arg); 5534 5535 /* ENV should not be modified. */ 5536 tcg_debug_assert(!temp_readonly(ts)); 5537 5538 switch (arg_ct->pair) { 5539 case 0: /* not paired */ 5540 if (arg_ct->oalias && !const_args[arg_ct->alias_index]) { 5541 reg = new_args[arg_ct->alias_index]; 5542 } else if (arg_ct->newreg) { 5543 reg = tcg_reg_alloc(s, arg_ct->regs, 5544 i_allocated_regs | o_allocated_regs, 5545 output_pref(op, k), ts->indirect_base); 5546 } else { 5547 reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs, 5548 output_pref(op, k), ts->indirect_base); 5549 } 5550 break; 5551 5552 case 1: /* first of pair */ 5553 if (arg_ct->oalias) { 5554 reg = new_args[arg_ct->alias_index]; 5555 } else if (arg_ct->newreg) { 5556 reg = tcg_reg_alloc_pair(s, arg_ct->regs, 5557 i_allocated_regs | o_allocated_regs, 5558 output_pref(op, k), 5559 ts->indirect_base); 5560 } else { 5561 reg = tcg_reg_alloc_pair(s, arg_ct->regs, o_allocated_regs, 5562 output_pref(op, k), 5563 ts->indirect_base); 5564 } 5565 break; 5566 5567 case 2: /* second of pair */ 5568 if (arg_ct->oalias) { 5569 reg = new_args[arg_ct->alias_index]; 5570 } else { 5571 reg = new_args[arg_ct->pair_index] + 1; 5572 } 5573 break; 5574 5575 case 3: /* first of pair, aliasing with a second input */ 5576 tcg_debug_assert(!arg_ct->newreg); 5577 reg = new_args[arg_ct->pair_index] - 1; 5578 break; 5579 5580 default: 5581 g_assert_not_reached(); 5582 } 5583 tcg_regset_set_reg(o_allocated_regs, reg); 5584 set_temp_val_reg(s, ts, reg); 5585 ts->mem_coherent = 0; 5586 new_args[i] = reg; 5587 } 5588 } 5589 5590 /* emit instruction */ 5591 TCGType type = TCGOP_TYPE(op); 5592 switch (op->opc) { 5593 case INDEX_op_addc1o: 5594 tcg_out_set_carry(s); 5595 /* fall through */ 5596 case INDEX_op_add: 5597 case INDEX_op_addcio: 5598 case INDEX_op_addco: 5599 case INDEX_op_and: 5600 case INDEX_op_andc: 5601 case INDEX_op_clz: 5602 case INDEX_op_ctz: 5603 case INDEX_op_divs: 5604 case INDEX_op_divu: 5605 case INDEX_op_eqv: 5606 case INDEX_op_mul: 5607 case INDEX_op_mulsh: 5608 case INDEX_op_muluh: 5609 case INDEX_op_nand: 5610 case INDEX_op_nor: 5611 case INDEX_op_or: 5612 case INDEX_op_orc: 5613 case INDEX_op_rems: 5614 case INDEX_op_remu: 5615 case INDEX_op_rotl: 5616 case INDEX_op_rotr: 5617 case INDEX_op_sar: 5618 case INDEX_op_shl: 5619 case INDEX_op_shr: 5620 case INDEX_op_xor: 5621 { 5622 const TCGOutOpBinary *out = 5623 container_of(all_outop[op->opc], TCGOutOpBinary, base); 5624 5625 /* Constants should never appear in the first source operand. */ 5626 tcg_debug_assert(!const_args[1]); 5627 if (const_args[2]) { 5628 out->out_rri(s, type, new_args[0], new_args[1], new_args[2]); 5629 } else { 5630 out->out_rrr(s, type, new_args[0], new_args[1], new_args[2]); 5631 } 5632 } 5633 break; 5634 5635 case INDEX_op_sub: 5636 { 5637 const TCGOutOpSubtract *out = &outop_sub; 5638 5639 /* 5640 * Constants should never appear in the second source operand. 5641 * These are folded to add with negative constant. 5642 */ 5643 tcg_debug_assert(!const_args[2]); 5644 if (const_args[1]) { 5645 out->out_rir(s, type, new_args[0], new_args[1], new_args[2]); 5646 } else { 5647 out->out_rrr(s, type, new_args[0], new_args[1], new_args[2]); 5648 } 5649 } 5650 break; 5651 5652 case INDEX_op_subb1o: 5653 tcg_out_set_borrow(s); 5654 /* fall through */ 5655 case INDEX_op_addci: 5656 case INDEX_op_subbi: 5657 case INDEX_op_subbio: 5658 case INDEX_op_subbo: 5659 { 5660 const TCGOutOpAddSubCarry *out = 5661 container_of(all_outop[op->opc], TCGOutOpAddSubCarry, base); 5662 5663 if (const_args[2]) { 5664 if (const_args[1]) { 5665 out->out_rii(s, type, new_args[0], 5666 new_args[1], new_args[2]); 5667 } else { 5668 out->out_rri(s, type, new_args[0], 5669 new_args[1], new_args[2]); 5670 } 5671 } else if (const_args[1]) { 5672 out->out_rir(s, type, new_args[0], new_args[1], new_args[2]); 5673 } else { 5674 out->out_rrr(s, type, new_args[0], new_args[1], new_args[2]); 5675 } 5676 } 5677 break; 5678 5679 case INDEX_op_bswap64: 5680 case INDEX_op_ext_i32_i64: 5681 case INDEX_op_extu_i32_i64: 5682 case INDEX_op_extrl_i64_i32: 5683 case INDEX_op_extrh_i64_i32: 5684 assert(TCG_TARGET_REG_BITS == 64); 5685 /* fall through */ 5686 case INDEX_op_ctpop: 5687 case INDEX_op_neg: 5688 case INDEX_op_not: 5689 { 5690 const TCGOutOpUnary *out = 5691 container_of(all_outop[op->opc], TCGOutOpUnary, base); 5692 5693 /* Constants should have been folded. */ 5694 tcg_debug_assert(!const_args[1]); 5695 out->out_rr(s, type, new_args[0], new_args[1]); 5696 } 5697 break; 5698 5699 case INDEX_op_bswap16: 5700 case INDEX_op_bswap32: 5701 { 5702 const TCGOutOpBswap *out = 5703 container_of(all_outop[op->opc], TCGOutOpBswap, base); 5704 5705 tcg_debug_assert(!const_args[1]); 5706 out->out_rr(s, type, new_args[0], new_args[1], new_args[2]); 5707 } 5708 break; 5709 5710 case INDEX_op_deposit: 5711 { 5712 const TCGOutOpDeposit *out = &outop_deposit; 5713 5714 if (const_args[2]) { 5715 tcg_debug_assert(!const_args[1]); 5716 out->out_rri(s, type, new_args[0], new_args[1], 5717 new_args[2], new_args[3], new_args[4]); 5718 } else if (const_args[1]) { 5719 tcg_debug_assert(new_args[1] == 0); 5720 tcg_debug_assert(!const_args[2]); 5721 out->out_rzr(s, type, new_args[0], new_args[2], 5722 new_args[3], new_args[4]); 5723 } else { 5724 out->out_rrr(s, type, new_args[0], new_args[1], 5725 new_args[2], new_args[3], new_args[4]); 5726 } 5727 } 5728 break; 5729 5730 case INDEX_op_divs2: 5731 case INDEX_op_divu2: 5732 { 5733 const TCGOutOpDivRem *out = 5734 container_of(all_outop[op->opc], TCGOutOpDivRem, base); 5735 5736 /* Only used by x86 and s390x, which use matching constraints. */ 5737 tcg_debug_assert(new_args[0] == new_args[2]); 5738 tcg_debug_assert(new_args[1] == new_args[3]); 5739 tcg_debug_assert(!const_args[4]); 5740 out->out_rr01r(s, type, new_args[0], new_args[1], new_args[4]); 5741 } 5742 break; 5743 5744 case INDEX_op_extract: 5745 case INDEX_op_sextract: 5746 { 5747 const TCGOutOpExtract *out = 5748 container_of(all_outop[op->opc], TCGOutOpExtract, base); 5749 5750 tcg_debug_assert(!const_args[1]); 5751 out->out_rr(s, type, new_args[0], new_args[1], 5752 new_args[2], new_args[3]); 5753 } 5754 break; 5755 5756 case INDEX_op_extract2: 5757 { 5758 const TCGOutOpExtract2 *out = &outop_extract2; 5759 5760 tcg_debug_assert(!const_args[1]); 5761 tcg_debug_assert(!const_args[2]); 5762 out->out_rrr(s, type, new_args[0], new_args[1], 5763 new_args[2], new_args[3]); 5764 } 5765 break; 5766 5767 case INDEX_op_ld8u: 5768 case INDEX_op_ld8s: 5769 case INDEX_op_ld16u: 5770 case INDEX_op_ld16s: 5771 case INDEX_op_ld32u: 5772 case INDEX_op_ld32s: 5773 case INDEX_op_ld: 5774 { 5775 const TCGOutOpLoad *out = 5776 container_of(all_outop[op->opc], TCGOutOpLoad, base); 5777 5778 tcg_debug_assert(!const_args[1]); 5779 out->out(s, type, new_args[0], new_args[1], new_args[2]); 5780 } 5781 break; 5782 5783 case INDEX_op_muls2: 5784 case INDEX_op_mulu2: 5785 { 5786 const TCGOutOpMul2 *out = 5787 container_of(all_outop[op->opc], TCGOutOpMul2, base); 5788 5789 tcg_debug_assert(!const_args[2]); 5790 tcg_debug_assert(!const_args[3]); 5791 out->out_rrrr(s, type, new_args[0], new_args[1], 5792 new_args[2], new_args[3]); 5793 } 5794 break; 5795 5796 case INDEX_op_st32: 5797 /* Use tcg_op_st w/ I32. */ 5798 type = TCG_TYPE_I32; 5799 /* fall through */ 5800 case INDEX_op_st: 5801 case INDEX_op_st8: 5802 case INDEX_op_st16: 5803 { 5804 const TCGOutOpStore *out = 5805 container_of(all_outop[op->opc], TCGOutOpStore, base); 5806 5807 if (const_args[0]) { 5808 out->out_i(s, type, new_args[0], new_args[1], new_args[2]); 5809 } else { 5810 out->out_r(s, type, new_args[0], new_args[1], new_args[2]); 5811 } 5812 } 5813 break; 5814 5815 case INDEX_op_qemu_ld: 5816 case INDEX_op_qemu_st: 5817 { 5818 const TCGOutOpQemuLdSt *out = 5819 container_of(all_outop[op->opc], TCGOutOpQemuLdSt, base); 5820 5821 out->out(s, type, new_args[0], new_args[1], new_args[2]); 5822 } 5823 break; 5824 5825 case INDEX_op_qemu_ld2: 5826 case INDEX_op_qemu_st2: 5827 { 5828 const TCGOutOpQemuLdSt2 *out = 5829 container_of(all_outop[op->opc], TCGOutOpQemuLdSt2, base); 5830 5831 out->out(s, type, new_args[0], new_args[1], 5832 new_args[2], new_args[3]); 5833 } 5834 break; 5835 5836 case INDEX_op_brcond: 5837 { 5838 const TCGOutOpBrcond *out = &outop_brcond; 5839 TCGCond cond = new_args[2]; 5840 TCGLabel *label = arg_label(new_args[3]); 5841 5842 tcg_debug_assert(!const_args[0]); 5843 if (const_args[1]) { 5844 out->out_ri(s, type, cond, new_args[0], new_args[1], label); 5845 } else { 5846 out->out_rr(s, type, cond, new_args[0], new_args[1], label); 5847 } 5848 } 5849 break; 5850 5851 case INDEX_op_movcond: 5852 { 5853 const TCGOutOpMovcond *out = &outop_movcond; 5854 TCGCond cond = new_args[5]; 5855 5856 tcg_debug_assert(!const_args[1]); 5857 out->out(s, type, cond, new_args[0], 5858 new_args[1], new_args[2], const_args[2], 5859 new_args[3], const_args[3], 5860 new_args[4], const_args[4]); 5861 } 5862 break; 5863 5864 case INDEX_op_setcond: 5865 case INDEX_op_negsetcond: 5866 { 5867 const TCGOutOpSetcond *out = 5868 container_of(all_outop[op->opc], TCGOutOpSetcond, base); 5869 TCGCond cond = new_args[3]; 5870 5871 tcg_debug_assert(!const_args[1]); 5872 if (const_args[2]) { 5873 out->out_rri(s, type, cond, 5874 new_args[0], new_args[1], new_args[2]); 5875 } else { 5876 out->out_rrr(s, type, cond, 5877 new_args[0], new_args[1], new_args[2]); 5878 } 5879 } 5880 break; 5881 5882 #if TCG_TARGET_REG_BITS == 32 5883 case INDEX_op_brcond2_i32: 5884 { 5885 const TCGOutOpBrcond2 *out = &outop_brcond2; 5886 TCGCond cond = new_args[4]; 5887 TCGLabel *label = arg_label(new_args[5]); 5888 5889 tcg_debug_assert(!const_args[0]); 5890 tcg_debug_assert(!const_args[1]); 5891 out->out(s, cond, new_args[0], new_args[1], 5892 new_args[2], const_args[2], 5893 new_args[3], const_args[3], label); 5894 } 5895 break; 5896 case INDEX_op_setcond2_i32: 5897 { 5898 const TCGOutOpSetcond2 *out = &outop_setcond2; 5899 TCGCond cond = new_args[5]; 5900 5901 tcg_debug_assert(!const_args[1]); 5902 tcg_debug_assert(!const_args[2]); 5903 out->out(s, cond, new_args[0], new_args[1], new_args[2], 5904 new_args[3], const_args[3], new_args[4], const_args[4]); 5905 } 5906 break; 5907 #else 5908 case INDEX_op_brcond2_i32: 5909 case INDEX_op_setcond2_i32: 5910 g_assert_not_reached(); 5911 #endif 5912 5913 case INDEX_op_goto_ptr: 5914 tcg_debug_assert(!const_args[0]); 5915 tcg_out_goto_ptr(s, new_args[0]); 5916 break; 5917 5918 default: 5919 tcg_debug_assert(def->flags & TCG_OPF_VECTOR); 5920 tcg_out_vec_op(s, op->opc, type - TCG_TYPE_V64, 5921 TCGOP_VECE(op), new_args, const_args); 5922 break; 5923 } 5924 5925 if (def->flags & TCG_OPF_CARRY_IN) { 5926 s->carry_live = false; 5927 } 5928 if (def->flags & TCG_OPF_CARRY_OUT) { 5929 s->carry_live = true; 5930 } 5931 5932 /* move the outputs in the correct register if needed */ 5933 for(i = 0; i < nb_oargs; i++) { 5934 ts = arg_temp(op->args[i]); 5935 5936 /* ENV should not be modified. */ 5937 tcg_debug_assert(!temp_readonly(ts)); 5938 5939 if (NEED_SYNC_ARG(i)) { 5940 temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i)); 5941 } else if (IS_DEAD_ARG(i)) { 5942 temp_dead(s, ts); 5943 } 5944 } 5945 } 5946 5947 static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op) 5948 { 5949 const TCGLifeData arg_life = op->life; 5950 TCGTemp *ots, *itsl, *itsh; 5951 TCGType vtype = TCGOP_TYPE(op); 5952 5953 /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */ 5954 tcg_debug_assert(TCG_TARGET_REG_BITS == 32); 5955 tcg_debug_assert(TCGOP_VECE(op) == MO_64); 5956 5957 ots = arg_temp(op->args[0]); 5958 itsl = arg_temp(op->args[1]); 5959 itsh = arg_temp(op->args[2]); 5960 5961 /* ENV should not be modified. */ 5962 tcg_debug_assert(!temp_readonly(ots)); 5963 5964 /* Allocate the output register now. */ 5965 if (ots->val_type != TEMP_VAL_REG) { 5966 TCGRegSet allocated_regs = s->reserved_regs; 5967 TCGRegSet dup_out_regs = opcode_args_ct(op)[0].regs; 5968 TCGReg oreg; 5969 5970 /* Make sure to not spill the input registers. */ 5971 if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) { 5972 tcg_regset_set_reg(allocated_regs, itsl->reg); 5973 } 5974 if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) { 5975 tcg_regset_set_reg(allocated_regs, itsh->reg); 5976 } 5977 5978 oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs, 5979 output_pref(op, 0), ots->indirect_base); 5980 set_temp_val_reg(s, ots, oreg); 5981 } 5982 5983 /* Promote dup2 of immediates to dupi_vec. */ 5984 if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) { 5985 uint64_t val = deposit64(itsl->val, 32, 32, itsh->val); 5986 MemOp vece = MO_64; 5987 5988 if (val == dup_const(MO_8, val)) { 5989 vece = MO_8; 5990 } else if (val == dup_const(MO_16, val)) { 5991 vece = MO_16; 5992 } else if (val == dup_const(MO_32, val)) { 5993 vece = MO_32; 5994 } 5995 5996 tcg_out_dupi_vec(s, vtype, vece, ots->reg, val); 5997 goto done; 5998 } 5999 6000 /* If the two inputs form one 64-bit value, try dupm_vec. */ 6001 if (itsl->temp_subindex == HOST_BIG_ENDIAN && 6002 itsh->temp_subindex == !HOST_BIG_ENDIAN && 6003 itsl == itsh + (HOST_BIG_ENDIAN ? 1 : -1)) { 6004 TCGTemp *its = itsl - HOST_BIG_ENDIAN; 6005 6006 temp_sync(s, its + 0, s->reserved_regs, 0, 0); 6007 temp_sync(s, its + 1, s->reserved_regs, 0, 0); 6008 6009 if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg, 6010 its->mem_base->reg, its->mem_offset)) { 6011 goto done; 6012 } 6013 } 6014 6015 /* Fall back to generic expansion. */ 6016 return false; 6017 6018 done: 6019 ots->mem_coherent = 0; 6020 if (IS_DEAD_ARG(1)) { 6021 temp_dead(s, itsl); 6022 } 6023 if (IS_DEAD_ARG(2)) { 6024 temp_dead(s, itsh); 6025 } 6026 if (NEED_SYNC_ARG(0)) { 6027 temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0)); 6028 } else if (IS_DEAD_ARG(0)) { 6029 temp_dead(s, ots); 6030 } 6031 return true; 6032 } 6033 6034 static void load_arg_reg(TCGContext *s, TCGReg reg, TCGTemp *ts, 6035 TCGRegSet allocated_regs) 6036 { 6037 if (ts->val_type == TEMP_VAL_REG) { 6038 if (ts->reg != reg) { 6039 tcg_reg_free(s, reg, allocated_regs); 6040 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) { 6041 /* 6042 * Cross register class move not supported. Sync the 6043 * temp back to its slot and load from there. 6044 */ 6045 temp_sync(s, ts, allocated_regs, 0, 0); 6046 tcg_out_ld(s, ts->type, reg, 6047 ts->mem_base->reg, ts->mem_offset); 6048 } 6049 } 6050 } else { 6051 TCGRegSet arg_set = 0; 6052 6053 tcg_reg_free(s, reg, allocated_regs); 6054 tcg_regset_set_reg(arg_set, reg); 6055 temp_load(s, ts, arg_set, allocated_regs, 0); 6056 } 6057 } 6058 6059 static void load_arg_stk(TCGContext *s, unsigned arg_slot, TCGTemp *ts, 6060 TCGRegSet allocated_regs) 6061 { 6062 /* 6063 * When the destination is on the stack, load up the temp and store. 6064 * If there are many call-saved registers, the temp might live to 6065 * see another use; otherwise it'll be discarded. 6066 */ 6067 temp_load(s, ts, tcg_target_available_regs[ts->type], allocated_regs, 0); 6068 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, 6069 arg_slot_stk_ofs(arg_slot)); 6070 } 6071 6072 static void load_arg_normal(TCGContext *s, const TCGCallArgumentLoc *l, 6073 TCGTemp *ts, TCGRegSet *allocated_regs) 6074 { 6075 if (arg_slot_reg_p(l->arg_slot)) { 6076 TCGReg reg = tcg_target_call_iarg_regs[l->arg_slot]; 6077 load_arg_reg(s, reg, ts, *allocated_regs); 6078 tcg_regset_set_reg(*allocated_regs, reg); 6079 } else { 6080 load_arg_stk(s, l->arg_slot, ts, *allocated_regs); 6081 } 6082 } 6083 6084 static void load_arg_ref(TCGContext *s, unsigned arg_slot, TCGReg ref_base, 6085 intptr_t ref_off, TCGRegSet *allocated_regs) 6086 { 6087 TCGReg reg; 6088 6089 if (arg_slot_reg_p(arg_slot)) { 6090 reg = tcg_target_call_iarg_regs[arg_slot]; 6091 tcg_reg_free(s, reg, *allocated_regs); 6092 tcg_out_addi_ptr(s, reg, ref_base, ref_off); 6093 tcg_regset_set_reg(*allocated_regs, reg); 6094 } else { 6095 reg = tcg_reg_alloc(s, tcg_target_available_regs[TCG_TYPE_PTR], 6096 *allocated_regs, 0, false); 6097 tcg_out_addi_ptr(s, reg, ref_base, ref_off); 6098 tcg_out_st(s, TCG_TYPE_PTR, reg, TCG_REG_CALL_STACK, 6099 arg_slot_stk_ofs(arg_slot)); 6100 } 6101 } 6102 6103 static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) 6104 { 6105 const int nb_oargs = TCGOP_CALLO(op); 6106 const int nb_iargs = TCGOP_CALLI(op); 6107 const TCGLifeData arg_life = op->life; 6108 const TCGHelperInfo *info = tcg_call_info(op); 6109 TCGRegSet allocated_regs = s->reserved_regs; 6110 int i; 6111 6112 /* 6113 * Move inputs into place in reverse order, 6114 * so that we place stacked arguments first. 6115 */ 6116 for (i = nb_iargs - 1; i >= 0; --i) { 6117 const TCGCallArgumentLoc *loc = &info->in[i]; 6118 TCGTemp *ts = arg_temp(op->args[nb_oargs + i]); 6119 6120 switch (loc->kind) { 6121 case TCG_CALL_ARG_NORMAL: 6122 case TCG_CALL_ARG_EXTEND_U: 6123 case TCG_CALL_ARG_EXTEND_S: 6124 load_arg_normal(s, loc, ts, &allocated_regs); 6125 break; 6126 case TCG_CALL_ARG_BY_REF: 6127 load_arg_stk(s, loc->ref_slot, ts, allocated_regs); 6128 load_arg_ref(s, loc->arg_slot, TCG_REG_CALL_STACK, 6129 arg_slot_stk_ofs(loc->ref_slot), 6130 &allocated_regs); 6131 break; 6132 case TCG_CALL_ARG_BY_REF_N: 6133 load_arg_stk(s, loc->ref_slot, ts, allocated_regs); 6134 break; 6135 default: 6136 g_assert_not_reached(); 6137 } 6138 } 6139 6140 /* Mark dead temporaries and free the associated registers. */ 6141 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 6142 if (IS_DEAD_ARG(i)) { 6143 temp_dead(s, arg_temp(op->args[i])); 6144 } 6145 } 6146 6147 /* Clobber call registers. */ 6148 for (i = 0; i < TCG_TARGET_NB_REGS; i++) { 6149 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { 6150 tcg_reg_free(s, i, allocated_regs); 6151 } 6152 } 6153 6154 /* 6155 * Save globals if they might be written by the helper, 6156 * sync them if they might be read. 6157 */ 6158 if (info->flags & TCG_CALL_NO_READ_GLOBALS) { 6159 /* Nothing to do */ 6160 } else if (info->flags & TCG_CALL_NO_WRITE_GLOBALS) { 6161 sync_globals(s, allocated_regs); 6162 } else { 6163 save_globals(s, allocated_regs); 6164 } 6165 6166 /* 6167 * If the ABI passes a pointer to the returned struct as the first 6168 * argument, load that now. Pass a pointer to the output home slot. 6169 */ 6170 if (info->out_kind == TCG_CALL_RET_BY_REF) { 6171 TCGTemp *ts = arg_temp(op->args[0]); 6172 6173 if (!ts->mem_allocated) { 6174 temp_allocate_frame(s, ts); 6175 } 6176 load_arg_ref(s, 0, ts->mem_base->reg, ts->mem_offset, &allocated_regs); 6177 } 6178 6179 tcg_out_call(s, tcg_call_func(op), info); 6180 6181 /* Assign output registers and emit moves if needed. */ 6182 switch (info->out_kind) { 6183 case TCG_CALL_RET_NORMAL: 6184 for (i = 0; i < nb_oargs; i++) { 6185 TCGTemp *ts = arg_temp(op->args[i]); 6186 TCGReg reg = tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, i); 6187 6188 /* ENV should not be modified. */ 6189 tcg_debug_assert(!temp_readonly(ts)); 6190 6191 set_temp_val_reg(s, ts, reg); 6192 ts->mem_coherent = 0; 6193 } 6194 break; 6195 6196 case TCG_CALL_RET_BY_VEC: 6197 { 6198 TCGTemp *ts = arg_temp(op->args[0]); 6199 6200 tcg_debug_assert(ts->base_type == TCG_TYPE_I128); 6201 tcg_debug_assert(ts->temp_subindex == 0); 6202 if (!ts->mem_allocated) { 6203 temp_allocate_frame(s, ts); 6204 } 6205 tcg_out_st(s, TCG_TYPE_V128, 6206 tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0), 6207 ts->mem_base->reg, ts->mem_offset); 6208 } 6209 /* fall through to mark all parts in memory */ 6210 6211 case TCG_CALL_RET_BY_REF: 6212 /* The callee has performed a write through the reference. */ 6213 for (i = 0; i < nb_oargs; i++) { 6214 TCGTemp *ts = arg_temp(op->args[i]); 6215 ts->val_type = TEMP_VAL_MEM; 6216 } 6217 break; 6218 6219 default: 6220 g_assert_not_reached(); 6221 } 6222 6223 /* Flush or discard output registers as needed. */ 6224 for (i = 0; i < nb_oargs; i++) { 6225 TCGTemp *ts = arg_temp(op->args[i]); 6226 if (NEED_SYNC_ARG(i)) { 6227 temp_sync(s, ts, s->reserved_regs, 0, IS_DEAD_ARG(i)); 6228 } else if (IS_DEAD_ARG(i)) { 6229 temp_dead(s, ts); 6230 } 6231 } 6232 } 6233 6234 /** 6235 * atom_and_align_for_opc: 6236 * @s: tcg context 6237 * @opc: memory operation code 6238 * @host_atom: MO_ATOM_{IFALIGN,WITHIN16,SUBALIGN} for host operations 6239 * @allow_two_ops: true if we are prepared to issue two operations 6240 * 6241 * Return the alignment and atomicity to use for the inline fast path 6242 * for the given memory operation. The alignment may be larger than 6243 * that specified in @opc, and the correct alignment will be diagnosed 6244 * by the slow path helper. 6245 * 6246 * If @allow_two_ops, the host is prepared to test for 2x alignment, 6247 * and issue two loads or stores for subalignment. 6248 */ 6249 static TCGAtomAlign atom_and_align_for_opc(TCGContext *s, MemOp opc, 6250 MemOp host_atom, bool allow_two_ops) 6251 { 6252 MemOp align = memop_alignment_bits(opc); 6253 MemOp size = opc & MO_SIZE; 6254 MemOp half = size ? size - 1 : 0; 6255 MemOp atom = opc & MO_ATOM_MASK; 6256 MemOp atmax; 6257 6258 switch (atom) { 6259 case MO_ATOM_NONE: 6260 /* The operation requires no specific atomicity. */ 6261 atmax = MO_8; 6262 break; 6263 6264 case MO_ATOM_IFALIGN: 6265 atmax = size; 6266 break; 6267 6268 case MO_ATOM_IFALIGN_PAIR: 6269 atmax = half; 6270 break; 6271 6272 case MO_ATOM_WITHIN16: 6273 atmax = size; 6274 if (size == MO_128) { 6275 /* Misalignment implies !within16, and therefore no atomicity. */ 6276 } else if (host_atom != MO_ATOM_WITHIN16) { 6277 /* The host does not implement within16, so require alignment. */ 6278 align = MAX(align, size); 6279 } 6280 break; 6281 6282 case MO_ATOM_WITHIN16_PAIR: 6283 atmax = size; 6284 /* 6285 * Misalignment implies !within16, and therefore half atomicity. 6286 * Any host prepared for two operations can implement this with 6287 * half alignment. 6288 */ 6289 if (host_atom != MO_ATOM_WITHIN16 && allow_two_ops) { 6290 align = MAX(align, half); 6291 } 6292 break; 6293 6294 case MO_ATOM_SUBALIGN: 6295 atmax = size; 6296 if (host_atom != MO_ATOM_SUBALIGN) { 6297 /* If unaligned but not odd, there are subobjects up to half. */ 6298 if (allow_two_ops) { 6299 align = MAX(align, half); 6300 } else { 6301 align = MAX(align, size); 6302 } 6303 } 6304 break; 6305 6306 default: 6307 g_assert_not_reached(); 6308 } 6309 6310 return (TCGAtomAlign){ .atom = atmax, .align = align }; 6311 } 6312 6313 /* 6314 * Similarly for qemu_ld/st slow path helpers. 6315 * We must re-implement tcg_gen_callN and tcg_reg_alloc_call simultaneously, 6316 * using only the provided backend tcg_out_* functions. 6317 */ 6318 6319 static int tcg_out_helper_stk_ofs(TCGType type, unsigned slot) 6320 { 6321 int ofs = arg_slot_stk_ofs(slot); 6322 6323 /* 6324 * Each stack slot is TCG_TARGET_LONG_BITS. If the host does not 6325 * require extension to uint64_t, adjust the address for uint32_t. 6326 */ 6327 if (HOST_BIG_ENDIAN && 6328 TCG_TARGET_REG_BITS == 64 && 6329 type == TCG_TYPE_I32) { 6330 ofs += 4; 6331 } 6332 return ofs; 6333 } 6334 6335 static void tcg_out_helper_load_slots(TCGContext *s, 6336 unsigned nmov, TCGMovExtend *mov, 6337 const TCGLdstHelperParam *parm) 6338 { 6339 unsigned i; 6340 TCGReg dst3; 6341 6342 /* 6343 * Start from the end, storing to the stack first. 6344 * This frees those registers, so we need not consider overlap. 6345 */ 6346 for (i = nmov; i-- > 0; ) { 6347 unsigned slot = mov[i].dst; 6348 6349 if (arg_slot_reg_p(slot)) { 6350 goto found_reg; 6351 } 6352 6353 TCGReg src = mov[i].src; 6354 TCGType dst_type = mov[i].dst_type; 6355 MemOp dst_mo = dst_type == TCG_TYPE_I32 ? MO_32 : MO_64; 6356 6357 /* The argument is going onto the stack; extend into scratch. */ 6358 if ((mov[i].src_ext & MO_SIZE) != dst_mo) { 6359 tcg_debug_assert(parm->ntmp != 0); 6360 mov[i].dst = src = parm->tmp[0]; 6361 tcg_out_movext1(s, &mov[i]); 6362 } 6363 6364 tcg_out_st(s, dst_type, src, TCG_REG_CALL_STACK, 6365 tcg_out_helper_stk_ofs(dst_type, slot)); 6366 } 6367 return; 6368 6369 found_reg: 6370 /* 6371 * The remaining arguments are in registers. 6372 * Convert slot numbers to argument registers. 6373 */ 6374 nmov = i + 1; 6375 for (i = 0; i < nmov; ++i) { 6376 mov[i].dst = tcg_target_call_iarg_regs[mov[i].dst]; 6377 } 6378 6379 switch (nmov) { 6380 case 4: 6381 /* The backend must have provided enough temps for the worst case. */ 6382 tcg_debug_assert(parm->ntmp >= 2); 6383 6384 dst3 = mov[3].dst; 6385 for (unsigned j = 0; j < 3; ++j) { 6386 if (dst3 == mov[j].src) { 6387 /* 6388 * Conflict. Copy the source to a temporary, perform the 6389 * remaining moves, then the extension from our scratch 6390 * on the way out. 6391 */ 6392 TCGReg scratch = parm->tmp[1]; 6393 6394 tcg_out_mov(s, mov[3].src_type, scratch, mov[3].src); 6395 tcg_out_movext3(s, mov, mov + 1, mov + 2, parm->tmp[0]); 6396 tcg_out_movext1_new_src(s, &mov[3], scratch); 6397 break; 6398 } 6399 } 6400 6401 /* No conflicts: perform this move and continue. */ 6402 tcg_out_movext1(s, &mov[3]); 6403 /* fall through */ 6404 6405 case 3: 6406 tcg_out_movext3(s, mov, mov + 1, mov + 2, 6407 parm->ntmp ? parm->tmp[0] : -1); 6408 break; 6409 case 2: 6410 tcg_out_movext2(s, mov, mov + 1, 6411 parm->ntmp ? parm->tmp[0] : -1); 6412 break; 6413 case 1: 6414 tcg_out_movext1(s, mov); 6415 break; 6416 default: 6417 g_assert_not_reached(); 6418 } 6419 } 6420 6421 static void tcg_out_helper_load_imm(TCGContext *s, unsigned slot, 6422 TCGType type, tcg_target_long imm, 6423 const TCGLdstHelperParam *parm) 6424 { 6425 if (arg_slot_reg_p(slot)) { 6426 tcg_out_movi(s, type, tcg_target_call_iarg_regs[slot], imm); 6427 } else { 6428 int ofs = tcg_out_helper_stk_ofs(type, slot); 6429 if (!tcg_out_sti(s, type, imm, TCG_REG_CALL_STACK, ofs)) { 6430 tcg_debug_assert(parm->ntmp != 0); 6431 tcg_out_movi(s, type, parm->tmp[0], imm); 6432 tcg_out_st(s, type, parm->tmp[0], TCG_REG_CALL_STACK, ofs); 6433 } 6434 } 6435 } 6436 6437 static void tcg_out_helper_load_common_args(TCGContext *s, 6438 const TCGLabelQemuLdst *ldst, 6439 const TCGLdstHelperParam *parm, 6440 const TCGHelperInfo *info, 6441 unsigned next_arg) 6442 { 6443 TCGMovExtend ptr_mov = { 6444 .dst_type = TCG_TYPE_PTR, 6445 .src_type = TCG_TYPE_PTR, 6446 .src_ext = sizeof(void *) == 4 ? MO_32 : MO_64 6447 }; 6448 const TCGCallArgumentLoc *loc = &info->in[0]; 6449 TCGType type; 6450 unsigned slot; 6451 tcg_target_ulong imm; 6452 6453 /* 6454 * Handle env, which is always first. 6455 */ 6456 ptr_mov.dst = loc->arg_slot; 6457 ptr_mov.src = TCG_AREG0; 6458 tcg_out_helper_load_slots(s, 1, &ptr_mov, parm); 6459 6460 /* 6461 * Handle oi. 6462 */ 6463 imm = ldst->oi; 6464 loc = &info->in[next_arg]; 6465 type = TCG_TYPE_I32; 6466 switch (loc->kind) { 6467 case TCG_CALL_ARG_NORMAL: 6468 break; 6469 case TCG_CALL_ARG_EXTEND_U: 6470 case TCG_CALL_ARG_EXTEND_S: 6471 /* No extension required for MemOpIdx. */ 6472 tcg_debug_assert(imm <= INT32_MAX); 6473 type = TCG_TYPE_REG; 6474 break; 6475 default: 6476 g_assert_not_reached(); 6477 } 6478 tcg_out_helper_load_imm(s, loc->arg_slot, type, imm, parm); 6479 next_arg++; 6480 6481 /* 6482 * Handle ra. 6483 */ 6484 loc = &info->in[next_arg]; 6485 slot = loc->arg_slot; 6486 if (parm->ra_gen) { 6487 int arg_reg = -1; 6488 TCGReg ra_reg; 6489 6490 if (arg_slot_reg_p(slot)) { 6491 arg_reg = tcg_target_call_iarg_regs[slot]; 6492 } 6493 ra_reg = parm->ra_gen(s, ldst, arg_reg); 6494 6495 ptr_mov.dst = slot; 6496 ptr_mov.src = ra_reg; 6497 tcg_out_helper_load_slots(s, 1, &ptr_mov, parm); 6498 } else { 6499 imm = (uintptr_t)ldst->raddr; 6500 tcg_out_helper_load_imm(s, slot, TCG_TYPE_PTR, imm, parm); 6501 } 6502 } 6503 6504 static unsigned tcg_out_helper_add_mov(TCGMovExtend *mov, 6505 const TCGCallArgumentLoc *loc, 6506 TCGType dst_type, TCGType src_type, 6507 TCGReg lo, TCGReg hi) 6508 { 6509 MemOp reg_mo; 6510 6511 if (dst_type <= TCG_TYPE_REG) { 6512 MemOp src_ext; 6513 6514 switch (loc->kind) { 6515 case TCG_CALL_ARG_NORMAL: 6516 src_ext = src_type == TCG_TYPE_I32 ? MO_32 : MO_64; 6517 break; 6518 case TCG_CALL_ARG_EXTEND_U: 6519 dst_type = TCG_TYPE_REG; 6520 src_ext = MO_UL; 6521 break; 6522 case TCG_CALL_ARG_EXTEND_S: 6523 dst_type = TCG_TYPE_REG; 6524 src_ext = MO_SL; 6525 break; 6526 default: 6527 g_assert_not_reached(); 6528 } 6529 6530 mov[0].dst = loc->arg_slot; 6531 mov[0].dst_type = dst_type; 6532 mov[0].src = lo; 6533 mov[0].src_type = src_type; 6534 mov[0].src_ext = src_ext; 6535 return 1; 6536 } 6537 6538 if (TCG_TARGET_REG_BITS == 32) { 6539 assert(dst_type == TCG_TYPE_I64); 6540 reg_mo = MO_32; 6541 } else { 6542 assert(dst_type == TCG_TYPE_I128); 6543 reg_mo = MO_64; 6544 } 6545 6546 mov[0].dst = loc[HOST_BIG_ENDIAN].arg_slot; 6547 mov[0].src = lo; 6548 mov[0].dst_type = TCG_TYPE_REG; 6549 mov[0].src_type = TCG_TYPE_REG; 6550 mov[0].src_ext = reg_mo; 6551 6552 mov[1].dst = loc[!HOST_BIG_ENDIAN].arg_slot; 6553 mov[1].src = hi; 6554 mov[1].dst_type = TCG_TYPE_REG; 6555 mov[1].src_type = TCG_TYPE_REG; 6556 mov[1].src_ext = reg_mo; 6557 6558 return 2; 6559 } 6560 6561 static void tcg_out_ld_helper_args(TCGContext *s, const TCGLabelQemuLdst *ldst, 6562 const TCGLdstHelperParam *parm) 6563 { 6564 const TCGHelperInfo *info; 6565 const TCGCallArgumentLoc *loc; 6566 TCGMovExtend mov[2]; 6567 unsigned next_arg, nmov; 6568 MemOp mop = get_memop(ldst->oi); 6569 6570 switch (mop & MO_SIZE) { 6571 case MO_8: 6572 case MO_16: 6573 case MO_32: 6574 info = &info_helper_ld32_mmu; 6575 break; 6576 case MO_64: 6577 info = &info_helper_ld64_mmu; 6578 break; 6579 case MO_128: 6580 info = &info_helper_ld128_mmu; 6581 break; 6582 default: 6583 g_assert_not_reached(); 6584 } 6585 6586 /* Defer env argument. */ 6587 next_arg = 1; 6588 6589 loc = &info->in[next_arg]; 6590 if (TCG_TARGET_REG_BITS == 32 && s->addr_type == TCG_TYPE_I32) { 6591 /* 6592 * 32-bit host with 32-bit guest: zero-extend the guest address 6593 * to 64-bits for the helper by storing the low part, then 6594 * load a zero for the high part. 6595 */ 6596 tcg_out_helper_add_mov(mov, loc + HOST_BIG_ENDIAN, 6597 TCG_TYPE_I32, TCG_TYPE_I32, 6598 ldst->addr_reg, -1); 6599 tcg_out_helper_load_slots(s, 1, mov, parm); 6600 6601 tcg_out_helper_load_imm(s, loc[!HOST_BIG_ENDIAN].arg_slot, 6602 TCG_TYPE_I32, 0, parm); 6603 next_arg += 2; 6604 } else { 6605 nmov = tcg_out_helper_add_mov(mov, loc, TCG_TYPE_I64, s->addr_type, 6606 ldst->addr_reg, -1); 6607 tcg_out_helper_load_slots(s, nmov, mov, parm); 6608 next_arg += nmov; 6609 } 6610 6611 switch (info->out_kind) { 6612 case TCG_CALL_RET_NORMAL: 6613 case TCG_CALL_RET_BY_VEC: 6614 break; 6615 case TCG_CALL_RET_BY_REF: 6616 /* 6617 * The return reference is in the first argument slot. 6618 * We need memory in which to return: re-use the top of stack. 6619 */ 6620 { 6621 int ofs_slot0 = TCG_TARGET_CALL_STACK_OFFSET; 6622 6623 if (arg_slot_reg_p(0)) { 6624 tcg_out_addi_ptr(s, tcg_target_call_iarg_regs[0], 6625 TCG_REG_CALL_STACK, ofs_slot0); 6626 } else { 6627 tcg_debug_assert(parm->ntmp != 0); 6628 tcg_out_addi_ptr(s, parm->tmp[0], 6629 TCG_REG_CALL_STACK, ofs_slot0); 6630 tcg_out_st(s, TCG_TYPE_PTR, parm->tmp[0], 6631 TCG_REG_CALL_STACK, ofs_slot0); 6632 } 6633 } 6634 break; 6635 default: 6636 g_assert_not_reached(); 6637 } 6638 6639 tcg_out_helper_load_common_args(s, ldst, parm, info, next_arg); 6640 } 6641 6642 static void tcg_out_ld_helper_ret(TCGContext *s, const TCGLabelQemuLdst *ldst, 6643 bool load_sign, 6644 const TCGLdstHelperParam *parm) 6645 { 6646 MemOp mop = get_memop(ldst->oi); 6647 TCGMovExtend mov[2]; 6648 int ofs_slot0; 6649 6650 switch (ldst->type) { 6651 case TCG_TYPE_I64: 6652 if (TCG_TARGET_REG_BITS == 32) { 6653 break; 6654 } 6655 /* fall through */ 6656 6657 case TCG_TYPE_I32: 6658 mov[0].dst = ldst->datalo_reg; 6659 mov[0].src = tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, 0); 6660 mov[0].dst_type = ldst->type; 6661 mov[0].src_type = TCG_TYPE_REG; 6662 6663 /* 6664 * If load_sign, then we allowed the helper to perform the 6665 * appropriate sign extension to tcg_target_ulong, and all 6666 * we need now is a plain move. 6667 * 6668 * If they do not, then we expect the relevant extension 6669 * instruction to be no more expensive than a move, and 6670 * we thus save the icache etc by only using one of two 6671 * helper functions. 6672 */ 6673 if (load_sign || !(mop & MO_SIGN)) { 6674 if (TCG_TARGET_REG_BITS == 32 || ldst->type == TCG_TYPE_I32) { 6675 mov[0].src_ext = MO_32; 6676 } else { 6677 mov[0].src_ext = MO_64; 6678 } 6679 } else { 6680 mov[0].src_ext = mop & MO_SSIZE; 6681 } 6682 tcg_out_movext1(s, mov); 6683 return; 6684 6685 case TCG_TYPE_I128: 6686 tcg_debug_assert(TCG_TARGET_REG_BITS == 64); 6687 ofs_slot0 = TCG_TARGET_CALL_STACK_OFFSET; 6688 switch (TCG_TARGET_CALL_RET_I128) { 6689 case TCG_CALL_RET_NORMAL: 6690 break; 6691 case TCG_CALL_RET_BY_VEC: 6692 tcg_out_st(s, TCG_TYPE_V128, 6693 tcg_target_call_oarg_reg(TCG_CALL_RET_BY_VEC, 0), 6694 TCG_REG_CALL_STACK, ofs_slot0); 6695 /* fall through */ 6696 case TCG_CALL_RET_BY_REF: 6697 tcg_out_ld(s, TCG_TYPE_I64, ldst->datalo_reg, 6698 TCG_REG_CALL_STACK, ofs_slot0 + 8 * HOST_BIG_ENDIAN); 6699 tcg_out_ld(s, TCG_TYPE_I64, ldst->datahi_reg, 6700 TCG_REG_CALL_STACK, ofs_slot0 + 8 * !HOST_BIG_ENDIAN); 6701 return; 6702 default: 6703 g_assert_not_reached(); 6704 } 6705 break; 6706 6707 default: 6708 g_assert_not_reached(); 6709 } 6710 6711 mov[0].dst = ldst->datalo_reg; 6712 mov[0].src = 6713 tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, HOST_BIG_ENDIAN); 6714 mov[0].dst_type = TCG_TYPE_REG; 6715 mov[0].src_type = TCG_TYPE_REG; 6716 mov[0].src_ext = TCG_TARGET_REG_BITS == 32 ? MO_32 : MO_64; 6717 6718 mov[1].dst = ldst->datahi_reg; 6719 mov[1].src = 6720 tcg_target_call_oarg_reg(TCG_CALL_RET_NORMAL, !HOST_BIG_ENDIAN); 6721 mov[1].dst_type = TCG_TYPE_REG; 6722 mov[1].src_type = TCG_TYPE_REG; 6723 mov[1].src_ext = TCG_TARGET_REG_BITS == 32 ? MO_32 : MO_64; 6724 6725 tcg_out_movext2(s, mov, mov + 1, parm->ntmp ? parm->tmp[0] : -1); 6726 } 6727 6728 static void tcg_out_st_helper_args(TCGContext *s, const TCGLabelQemuLdst *ldst, 6729 const TCGLdstHelperParam *parm) 6730 { 6731 const TCGHelperInfo *info; 6732 const TCGCallArgumentLoc *loc; 6733 TCGMovExtend mov[4]; 6734 TCGType data_type; 6735 unsigned next_arg, nmov, n; 6736 MemOp mop = get_memop(ldst->oi); 6737 6738 switch (mop & MO_SIZE) { 6739 case MO_8: 6740 case MO_16: 6741 case MO_32: 6742 info = &info_helper_st32_mmu; 6743 data_type = TCG_TYPE_I32; 6744 break; 6745 case MO_64: 6746 info = &info_helper_st64_mmu; 6747 data_type = TCG_TYPE_I64; 6748 break; 6749 case MO_128: 6750 info = &info_helper_st128_mmu; 6751 data_type = TCG_TYPE_I128; 6752 break; 6753 default: 6754 g_assert_not_reached(); 6755 } 6756 6757 /* Defer env argument. */ 6758 next_arg = 1; 6759 nmov = 0; 6760 6761 /* Handle addr argument. */ 6762 loc = &info->in[next_arg]; 6763 tcg_debug_assert(s->addr_type <= TCG_TYPE_REG); 6764 if (TCG_TARGET_REG_BITS == 32) { 6765 /* 6766 * 32-bit host (and thus 32-bit guest): zero-extend the guest address 6767 * to 64-bits for the helper by storing the low part. Later, 6768 * after we have processed the register inputs, we will load a 6769 * zero for the high part. 6770 */ 6771 tcg_out_helper_add_mov(mov, loc + HOST_BIG_ENDIAN, 6772 TCG_TYPE_I32, TCG_TYPE_I32, 6773 ldst->addr_reg, -1); 6774 next_arg += 2; 6775 nmov += 1; 6776 } else { 6777 n = tcg_out_helper_add_mov(mov, loc, TCG_TYPE_I64, s->addr_type, 6778 ldst->addr_reg, -1); 6779 next_arg += n; 6780 nmov += n; 6781 } 6782 6783 /* Handle data argument. */ 6784 loc = &info->in[next_arg]; 6785 switch (loc->kind) { 6786 case TCG_CALL_ARG_NORMAL: 6787 case TCG_CALL_ARG_EXTEND_U: 6788 case TCG_CALL_ARG_EXTEND_S: 6789 n = tcg_out_helper_add_mov(mov + nmov, loc, data_type, ldst->type, 6790 ldst->datalo_reg, ldst->datahi_reg); 6791 next_arg += n; 6792 nmov += n; 6793 tcg_out_helper_load_slots(s, nmov, mov, parm); 6794 break; 6795 6796 case TCG_CALL_ARG_BY_REF: 6797 tcg_debug_assert(TCG_TARGET_REG_BITS == 64); 6798 tcg_debug_assert(data_type == TCG_TYPE_I128); 6799 tcg_out_st(s, TCG_TYPE_I64, 6800 HOST_BIG_ENDIAN ? ldst->datahi_reg : ldst->datalo_reg, 6801 TCG_REG_CALL_STACK, arg_slot_stk_ofs(loc[0].ref_slot)); 6802 tcg_out_st(s, TCG_TYPE_I64, 6803 HOST_BIG_ENDIAN ? ldst->datalo_reg : ldst->datahi_reg, 6804 TCG_REG_CALL_STACK, arg_slot_stk_ofs(loc[1].ref_slot)); 6805 6806 tcg_out_helper_load_slots(s, nmov, mov, parm); 6807 6808 if (arg_slot_reg_p(loc->arg_slot)) { 6809 tcg_out_addi_ptr(s, tcg_target_call_iarg_regs[loc->arg_slot], 6810 TCG_REG_CALL_STACK, 6811 arg_slot_stk_ofs(loc->ref_slot)); 6812 } else { 6813 tcg_debug_assert(parm->ntmp != 0); 6814 tcg_out_addi_ptr(s, parm->tmp[0], TCG_REG_CALL_STACK, 6815 arg_slot_stk_ofs(loc->ref_slot)); 6816 tcg_out_st(s, TCG_TYPE_PTR, parm->tmp[0], 6817 TCG_REG_CALL_STACK, arg_slot_stk_ofs(loc->arg_slot)); 6818 } 6819 next_arg += 2; 6820 break; 6821 6822 default: 6823 g_assert_not_reached(); 6824 } 6825 6826 if (TCG_TARGET_REG_BITS == 32) { 6827 /* Zero extend the address by loading a zero for the high part. */ 6828 loc = &info->in[1 + !HOST_BIG_ENDIAN]; 6829 tcg_out_helper_load_imm(s, loc->arg_slot, TCG_TYPE_I32, 0, parm); 6830 } 6831 6832 tcg_out_helper_load_common_args(s, ldst, parm, info, next_arg); 6833 } 6834 6835 int tcg_gen_code(TCGContext *s, TranslationBlock *tb, uint64_t pc_start) 6836 { 6837 int i, num_insns; 6838 TCGOp *op; 6839 6840 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP) 6841 && qemu_log_in_addr_range(pc_start))) { 6842 FILE *logfile = qemu_log_trylock(); 6843 if (logfile) { 6844 fprintf(logfile, "OP:\n"); 6845 tcg_dump_ops(s, logfile, false); 6846 fprintf(logfile, "\n"); 6847 qemu_log_unlock(logfile); 6848 } 6849 } 6850 6851 #ifdef CONFIG_DEBUG_TCG 6852 /* Ensure all labels referenced have been emitted. */ 6853 { 6854 TCGLabel *l; 6855 bool error = false; 6856 6857 QSIMPLEQ_FOREACH(l, &s->labels, next) { 6858 if (unlikely(!l->present) && !QSIMPLEQ_EMPTY(&l->branches)) { 6859 qemu_log_mask(CPU_LOG_TB_OP, 6860 "$L%d referenced but not present.\n", l->id); 6861 error = true; 6862 } 6863 } 6864 assert(!error); 6865 } 6866 #endif 6867 6868 /* Do not reuse any EBB that may be allocated within the TB. */ 6869 tcg_temp_ebb_reset_freed(s); 6870 6871 tcg_optimize(s); 6872 6873 reachable_code_pass(s); 6874 liveness_pass_0(s); 6875 liveness_pass_1(s); 6876 6877 if (s->nb_indirects > 0) { 6878 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND) 6879 && qemu_log_in_addr_range(pc_start))) { 6880 FILE *logfile = qemu_log_trylock(); 6881 if (logfile) { 6882 fprintf(logfile, "OP before indirect lowering:\n"); 6883 tcg_dump_ops(s, logfile, false); 6884 fprintf(logfile, "\n"); 6885 qemu_log_unlock(logfile); 6886 } 6887 } 6888 6889 /* Replace indirect temps with direct temps. */ 6890 if (liveness_pass_2(s)) { 6891 /* If changes were made, re-run liveness. */ 6892 liveness_pass_1(s); 6893 } 6894 } 6895 6896 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT) 6897 && qemu_log_in_addr_range(pc_start))) { 6898 FILE *logfile = qemu_log_trylock(); 6899 if (logfile) { 6900 fprintf(logfile, "OP after optimization and liveness analysis:\n"); 6901 tcg_dump_ops(s, logfile, true); 6902 fprintf(logfile, "\n"); 6903 qemu_log_unlock(logfile); 6904 } 6905 } 6906 6907 /* Initialize goto_tb jump offsets. */ 6908 tb->jmp_reset_offset[0] = TB_JMP_OFFSET_INVALID; 6909 tb->jmp_reset_offset[1] = TB_JMP_OFFSET_INVALID; 6910 tb->jmp_insn_offset[0] = TB_JMP_OFFSET_INVALID; 6911 tb->jmp_insn_offset[1] = TB_JMP_OFFSET_INVALID; 6912 6913 tcg_reg_alloc_start(s); 6914 6915 /* 6916 * Reset the buffer pointers when restarting after overflow. 6917 * TODO: Move this into translate-all.c with the rest of the 6918 * buffer management. Having only this done here is confusing. 6919 */ 6920 s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr); 6921 s->code_ptr = s->code_buf; 6922 s->data_gen_ptr = NULL; 6923 6924 QSIMPLEQ_INIT(&s->ldst_labels); 6925 s->pool_labels = NULL; 6926 6927 s->gen_insn_data = 6928 tcg_malloc(sizeof(uint64_t) * s->gen_tb->icount * INSN_START_WORDS); 6929 6930 tcg_out_tb_start(s); 6931 6932 num_insns = -1; 6933 s->carry_live = false; 6934 QTAILQ_FOREACH(op, &s->ops, link) { 6935 TCGOpcode opc = op->opc; 6936 6937 switch (opc) { 6938 case INDEX_op_extrl_i64_i32: 6939 assert(TCG_TARGET_REG_BITS == 64); 6940 /* 6941 * If TCG_TYPE_I32 is represented in some canonical form, 6942 * e.g. zero or sign-extended, then emit as a unary op. 6943 * Otherwise we can treat this as a plain move. 6944 * If the output dies, treat this as a plain move, because 6945 * this will be implemented with a store. 6946 */ 6947 if (TCG_TARGET_HAS_extr_i64_i32) { 6948 TCGLifeData arg_life = op->life; 6949 if (!IS_DEAD_ARG(0)) { 6950 goto do_default; 6951 } 6952 } 6953 /* fall through */ 6954 case INDEX_op_mov: 6955 case INDEX_op_mov_vec: 6956 tcg_reg_alloc_mov(s, op); 6957 break; 6958 case INDEX_op_dup_vec: 6959 tcg_reg_alloc_dup(s, op); 6960 break; 6961 case INDEX_op_insn_start: 6962 assert_carry_dead(s); 6963 if (num_insns >= 0) { 6964 size_t off = tcg_current_code_size(s); 6965 s->gen_insn_end_off[num_insns] = off; 6966 /* Assert that we do not overflow our stored offset. */ 6967 assert(s->gen_insn_end_off[num_insns] == off); 6968 } 6969 num_insns++; 6970 for (i = 0; i < INSN_START_WORDS; ++i) { 6971 s->gen_insn_data[num_insns * INSN_START_WORDS + i] = 6972 tcg_get_insn_start_param(op, i); 6973 } 6974 break; 6975 case INDEX_op_discard: 6976 temp_dead(s, arg_temp(op->args[0])); 6977 break; 6978 case INDEX_op_set_label: 6979 tcg_reg_alloc_bb_end(s, s->reserved_regs); 6980 tcg_out_label(s, arg_label(op->args[0])); 6981 break; 6982 case INDEX_op_call: 6983 assert_carry_dead(s); 6984 tcg_reg_alloc_call(s, op); 6985 break; 6986 case INDEX_op_exit_tb: 6987 tcg_out_exit_tb(s, op->args[0]); 6988 break; 6989 case INDEX_op_goto_tb: 6990 tcg_out_goto_tb(s, op->args[0]); 6991 break; 6992 case INDEX_op_br: 6993 tcg_out_br(s, arg_label(op->args[0])); 6994 break; 6995 case INDEX_op_mb: 6996 tcg_out_mb(s, op->args[0]); 6997 break; 6998 case INDEX_op_dup2_vec: 6999 if (tcg_reg_alloc_dup2(s, op)) { 7000 break; 7001 } 7002 /* fall through */ 7003 default: 7004 do_default: 7005 /* Sanity check that we've not introduced any unhandled opcodes. */ 7006 tcg_debug_assert(tcg_op_supported(opc, TCGOP_TYPE(op), 7007 TCGOP_FLAGS(op))); 7008 /* Note: in order to speed up the code, it would be much 7009 faster to have specialized register allocator functions for 7010 some common argument patterns */ 7011 tcg_reg_alloc_op(s, op); 7012 break; 7013 } 7014 /* Test for (pending) buffer overflow. The assumption is that any 7015 one operation beginning below the high water mark cannot overrun 7016 the buffer completely. Thus we can test for overflow after 7017 generating code without having to check during generation. */ 7018 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) { 7019 return -1; 7020 } 7021 /* Test for TB overflow, as seen by gen_insn_end_off. */ 7022 if (unlikely(tcg_current_code_size(s) > UINT16_MAX)) { 7023 return -2; 7024 } 7025 } 7026 assert_carry_dead(s); 7027 7028 tcg_debug_assert(num_insns + 1 == s->gen_tb->icount); 7029 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s); 7030 7031 /* Generate TB finalization at the end of block */ 7032 i = tcg_out_ldst_finalize(s); 7033 if (i < 0) { 7034 return i; 7035 } 7036 i = tcg_out_pool_finalize(s); 7037 if (i < 0) { 7038 return i; 7039 } 7040 if (!tcg_resolve_relocs(s)) { 7041 return -2; 7042 } 7043 7044 #ifndef CONFIG_TCG_INTERPRETER 7045 /* flush instruction cache */ 7046 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf), 7047 (uintptr_t)s->code_buf, 7048 tcg_ptr_byte_diff(s->code_ptr, s->code_buf)); 7049 #endif 7050 7051 return tcg_current_code_size(s); 7052 } 7053 7054 #ifdef ELF_HOST_MACHINE 7055 /* In order to use this feature, the backend needs to do three things: 7056 7057 (1) Define ELF_HOST_MACHINE to indicate both what value to 7058 put into the ELF image and to indicate support for the feature. 7059 7060 (2) Define tcg_register_jit. This should create a buffer containing 7061 the contents of a .debug_frame section that describes the post- 7062 prologue unwind info for the tcg machine. 7063 7064 (3) Call tcg_register_jit_int, with the constructed .debug_frame. 7065 */ 7066 7067 /* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */ 7068 typedef enum { 7069 JIT_NOACTION = 0, 7070 JIT_REGISTER_FN, 7071 JIT_UNREGISTER_FN 7072 } jit_actions_t; 7073 7074 struct jit_code_entry { 7075 struct jit_code_entry *next_entry; 7076 struct jit_code_entry *prev_entry; 7077 const void *symfile_addr; 7078 uint64_t symfile_size; 7079 }; 7080 7081 struct jit_descriptor { 7082 uint32_t version; 7083 uint32_t action_flag; 7084 struct jit_code_entry *relevant_entry; 7085 struct jit_code_entry *first_entry; 7086 }; 7087 7088 void __jit_debug_register_code(void) __attribute__((noinline)); 7089 void __jit_debug_register_code(void) 7090 { 7091 asm(""); 7092 } 7093 7094 /* Must statically initialize the version, because GDB may check 7095 the version before we can set it. */ 7096 struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 }; 7097 7098 /* End GDB interface. */ 7099 7100 static int find_string(const char *strtab, const char *str) 7101 { 7102 const char *p = strtab + 1; 7103 7104 while (1) { 7105 if (strcmp(p, str) == 0) { 7106 return p - strtab; 7107 } 7108 p += strlen(p) + 1; 7109 } 7110 } 7111 7112 static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size, 7113 const void *debug_frame, 7114 size_t debug_frame_size) 7115 { 7116 struct __attribute__((packed)) DebugInfo { 7117 uint32_t len; 7118 uint16_t version; 7119 uint32_t abbrev; 7120 uint8_t ptr_size; 7121 uint8_t cu_die; 7122 uint16_t cu_lang; 7123 uintptr_t cu_low_pc; 7124 uintptr_t cu_high_pc; 7125 uint8_t fn_die; 7126 char fn_name[16]; 7127 uintptr_t fn_low_pc; 7128 uintptr_t fn_high_pc; 7129 uint8_t cu_eoc; 7130 }; 7131 7132 struct ElfImage { 7133 ElfW(Ehdr) ehdr; 7134 ElfW(Phdr) phdr; 7135 ElfW(Shdr) shdr[7]; 7136 ElfW(Sym) sym[2]; 7137 struct DebugInfo di; 7138 uint8_t da[24]; 7139 char str[80]; 7140 }; 7141 7142 struct ElfImage *img; 7143 7144 static const struct ElfImage img_template = { 7145 .ehdr = { 7146 .e_ident[EI_MAG0] = ELFMAG0, 7147 .e_ident[EI_MAG1] = ELFMAG1, 7148 .e_ident[EI_MAG2] = ELFMAG2, 7149 .e_ident[EI_MAG3] = ELFMAG3, 7150 .e_ident[EI_CLASS] = ELF_CLASS, 7151 .e_ident[EI_DATA] = ELF_DATA, 7152 .e_ident[EI_VERSION] = EV_CURRENT, 7153 .e_type = ET_EXEC, 7154 .e_machine = ELF_HOST_MACHINE, 7155 .e_version = EV_CURRENT, 7156 .e_phoff = offsetof(struct ElfImage, phdr), 7157 .e_shoff = offsetof(struct ElfImage, shdr), 7158 .e_ehsize = sizeof(ElfW(Shdr)), 7159 .e_phentsize = sizeof(ElfW(Phdr)), 7160 .e_phnum = 1, 7161 .e_shentsize = sizeof(ElfW(Shdr)), 7162 .e_shnum = ARRAY_SIZE(img->shdr), 7163 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1, 7164 #ifdef ELF_HOST_FLAGS 7165 .e_flags = ELF_HOST_FLAGS, 7166 #endif 7167 #ifdef ELF_OSABI 7168 .e_ident[EI_OSABI] = ELF_OSABI, 7169 #endif 7170 }, 7171 .phdr = { 7172 .p_type = PT_LOAD, 7173 .p_flags = PF_X, 7174 }, 7175 .shdr = { 7176 [0] = { .sh_type = SHT_NULL }, 7177 /* Trick: The contents of code_gen_buffer are not present in 7178 this fake ELF file; that got allocated elsewhere. Therefore 7179 we mark .text as SHT_NOBITS (similar to .bss) so that readers 7180 will not look for contents. We can record any address. */ 7181 [1] = { /* .text */ 7182 .sh_type = SHT_NOBITS, 7183 .sh_flags = SHF_EXECINSTR | SHF_ALLOC, 7184 }, 7185 [2] = { /* .debug_info */ 7186 .sh_type = SHT_PROGBITS, 7187 .sh_offset = offsetof(struct ElfImage, di), 7188 .sh_size = sizeof(struct DebugInfo), 7189 }, 7190 [3] = { /* .debug_abbrev */ 7191 .sh_type = SHT_PROGBITS, 7192 .sh_offset = offsetof(struct ElfImage, da), 7193 .sh_size = sizeof(img->da), 7194 }, 7195 [4] = { /* .debug_frame */ 7196 .sh_type = SHT_PROGBITS, 7197 .sh_offset = sizeof(struct ElfImage), 7198 }, 7199 [5] = { /* .symtab */ 7200 .sh_type = SHT_SYMTAB, 7201 .sh_offset = offsetof(struct ElfImage, sym), 7202 .sh_size = sizeof(img->sym), 7203 .sh_info = 1, 7204 .sh_link = ARRAY_SIZE(img->shdr) - 1, 7205 .sh_entsize = sizeof(ElfW(Sym)), 7206 }, 7207 [6] = { /* .strtab */ 7208 .sh_type = SHT_STRTAB, 7209 .sh_offset = offsetof(struct ElfImage, str), 7210 .sh_size = sizeof(img->str), 7211 } 7212 }, 7213 .sym = { 7214 [1] = { /* code_gen_buffer */ 7215 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC), 7216 .st_shndx = 1, 7217 } 7218 }, 7219 .di = { 7220 .len = sizeof(struct DebugInfo) - 4, 7221 .version = 2, 7222 .ptr_size = sizeof(void *), 7223 .cu_die = 1, 7224 .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */ 7225 .fn_die = 2, 7226 .fn_name = "code_gen_buffer" 7227 }, 7228 .da = { 7229 1, /* abbrev number (the cu) */ 7230 0x11, 1, /* DW_TAG_compile_unit, has children */ 7231 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */ 7232 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 7233 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 7234 0, 0, /* end of abbrev */ 7235 2, /* abbrev number (the fn) */ 7236 0x2e, 0, /* DW_TAG_subprogram, no children */ 7237 0x3, 0x8, /* DW_AT_name, DW_FORM_string */ 7238 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 7239 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 7240 0, 0, /* end of abbrev */ 7241 0 /* no more abbrev */ 7242 }, 7243 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0" 7244 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer", 7245 }; 7246 7247 /* We only need a single jit entry; statically allocate it. */ 7248 static struct jit_code_entry one_entry; 7249 7250 uintptr_t buf = (uintptr_t)buf_ptr; 7251 size_t img_size = sizeof(struct ElfImage) + debug_frame_size; 7252 DebugFrameHeader *dfh; 7253 7254 img = g_malloc(img_size); 7255 *img = img_template; 7256 7257 img->phdr.p_vaddr = buf; 7258 img->phdr.p_paddr = buf; 7259 img->phdr.p_memsz = buf_size; 7260 7261 img->shdr[1].sh_name = find_string(img->str, ".text"); 7262 img->shdr[1].sh_addr = buf; 7263 img->shdr[1].sh_size = buf_size; 7264 7265 img->shdr[2].sh_name = find_string(img->str, ".debug_info"); 7266 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev"); 7267 7268 img->shdr[4].sh_name = find_string(img->str, ".debug_frame"); 7269 img->shdr[4].sh_size = debug_frame_size; 7270 7271 img->shdr[5].sh_name = find_string(img->str, ".symtab"); 7272 img->shdr[6].sh_name = find_string(img->str, ".strtab"); 7273 7274 img->sym[1].st_name = find_string(img->str, "code_gen_buffer"); 7275 img->sym[1].st_value = buf; 7276 img->sym[1].st_size = buf_size; 7277 7278 img->di.cu_low_pc = buf; 7279 img->di.cu_high_pc = buf + buf_size; 7280 img->di.fn_low_pc = buf; 7281 img->di.fn_high_pc = buf + buf_size; 7282 7283 dfh = (DebugFrameHeader *)(img + 1); 7284 memcpy(dfh, debug_frame, debug_frame_size); 7285 dfh->fde.func_start = buf; 7286 dfh->fde.func_len = buf_size; 7287 7288 #ifdef DEBUG_JIT 7289 /* Enable this block to be able to debug the ELF image file creation. 7290 One can use readelf, objdump, or other inspection utilities. */ 7291 { 7292 g_autofree char *jit = g_strdup_printf("%s/qemu.jit", g_get_tmp_dir()); 7293 FILE *f = fopen(jit, "w+b"); 7294 if (f) { 7295 if (fwrite(img, img_size, 1, f) != img_size) { 7296 /* Avoid stupid unused return value warning for fwrite. */ 7297 } 7298 fclose(f); 7299 } 7300 } 7301 #endif 7302 7303 one_entry.symfile_addr = img; 7304 one_entry.symfile_size = img_size; 7305 7306 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN; 7307 __jit_debug_descriptor.relevant_entry = &one_entry; 7308 __jit_debug_descriptor.first_entry = &one_entry; 7309 __jit_debug_register_code(); 7310 } 7311 #else 7312 /* No support for the feature. Provide the entry point expected by exec.c, 7313 and implement the internal function we declared earlier. */ 7314 7315 static void tcg_register_jit_int(const void *buf, size_t size, 7316 const void *debug_frame, 7317 size_t debug_frame_size) 7318 { 7319 } 7320 7321 void tcg_register_jit(const void *buf, size_t buf_size) 7322 { 7323 } 7324 #endif /* ELF_HOST_MACHINE */ 7325 7326 #if !TCG_TARGET_MAYBE_vec 7327 void tcg_expand_vec_op(TCGOpcode o, TCGType t, unsigned e, TCGArg a0, ...) 7328 { 7329 g_assert_not_reached(); 7330 } 7331 #endif 7332