1 /* 2 * Tiny Code Generator for QEMU 3 * 4 * Copyright (c) 2008 Fabrice Bellard 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to deal 8 * in the Software without restriction, including without limitation the rights 9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 * copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in 14 * all copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 * THE SOFTWARE. 23 */ 24 25 /* define it to use liveness analysis (better code) */ 26 #define USE_LIVENESS_ANALYSIS 27 #define USE_TCG_OPTIMIZATIONS 28 29 #include "config.h" 30 31 /* Define to jump the ELF file used to communicate with GDB. */ 32 #undef DEBUG_JIT 33 34 #if !defined(CONFIG_DEBUG_TCG) && !defined(NDEBUG) 35 /* define it to suppress various consistency checks (faster) */ 36 #define NDEBUG 37 #endif 38 39 #include "qemu-common.h" 40 #include "qemu/cache-utils.h" 41 #include "qemu/host-utils.h" 42 #include "qemu/timer.h" 43 44 /* Note: the long term plan is to reduce the dependencies on the QEMU 45 CPU definitions. Currently they are used for qemu_ld/st 46 instructions */ 47 #define NO_CPU_IO_DEFS 48 #include "cpu.h" 49 50 #include "tcg-op.h" 51 52 #if UINTPTR_MAX == UINT32_MAX 53 # define ELF_CLASS ELFCLASS32 54 #else 55 # define ELF_CLASS ELFCLASS64 56 #endif 57 #ifdef HOST_WORDS_BIGENDIAN 58 # define ELF_DATA ELFDATA2MSB 59 #else 60 # define ELF_DATA ELFDATA2LSB 61 #endif 62 63 #include "elf.h" 64 65 /* Forward declarations for functions declared in tcg-target.c and used here. */ 66 static void tcg_target_init(TCGContext *s); 67 static void tcg_target_qemu_prologue(TCGContext *s); 68 static void patch_reloc(tcg_insn_unit *code_ptr, int type, 69 intptr_t value, intptr_t addend); 70 71 /* The CIE and FDE header definitions will be common to all hosts. */ 72 typedef struct { 73 uint32_t len __attribute__((aligned((sizeof(void *))))); 74 uint32_t id; 75 uint8_t version; 76 char augmentation[1]; 77 uint8_t code_align; 78 uint8_t data_align; 79 uint8_t return_column; 80 } DebugFrameCIE; 81 82 typedef struct QEMU_PACKED { 83 uint32_t len __attribute__((aligned((sizeof(void *))))); 84 uint32_t cie_offset; 85 uintptr_t func_start; 86 uintptr_t func_len; 87 } DebugFrameFDEHeader; 88 89 static void tcg_register_jit_int(void *buf, size_t size, 90 void *debug_frame, size_t debug_frame_size) 91 __attribute__((unused)); 92 93 /* Forward declarations for functions declared and used in tcg-target.c. */ 94 static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str); 95 static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1, 96 intptr_t arg2); 97 static void tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg); 98 static void tcg_out_movi(TCGContext *s, TCGType type, 99 TCGReg ret, tcg_target_long arg); 100 static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args, 101 const int *const_args); 102 static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1, 103 intptr_t arg2); 104 static void tcg_out_call(TCGContext *s, tcg_insn_unit *target); 105 static int tcg_target_const_match(tcg_target_long val, TCGType type, 106 const TCGArgConstraint *arg_ct); 107 static void tcg_out_tb_init(TCGContext *s); 108 static void tcg_out_tb_finalize(TCGContext *s); 109 110 111 TCGOpDef tcg_op_defs[] = { 112 #define DEF(s, oargs, iargs, cargs, flags) { #s, oargs, iargs, cargs, iargs + oargs + cargs, flags }, 113 #include "tcg-opc.h" 114 #undef DEF 115 }; 116 const size_t tcg_op_defs_max = ARRAY_SIZE(tcg_op_defs); 117 118 static TCGRegSet tcg_target_available_regs[2]; 119 static TCGRegSet tcg_target_call_clobber_regs; 120 121 #if TCG_TARGET_INSN_UNIT_SIZE == 1 122 static inline void tcg_out8(TCGContext *s, uint8_t v) 123 { 124 *s->code_ptr++ = v; 125 } 126 127 static inline void tcg_patch8(tcg_insn_unit *p, uint8_t v) 128 { 129 *p = v; 130 } 131 #endif 132 133 #if TCG_TARGET_INSN_UNIT_SIZE <= 2 134 static inline void tcg_out16(TCGContext *s, uint16_t v) 135 { 136 if (TCG_TARGET_INSN_UNIT_SIZE == 2) { 137 *s->code_ptr++ = v; 138 } else { 139 tcg_insn_unit *p = s->code_ptr; 140 memcpy(p, &v, sizeof(v)); 141 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE); 142 } 143 } 144 145 static inline void tcg_patch16(tcg_insn_unit *p, uint16_t v) 146 { 147 if (TCG_TARGET_INSN_UNIT_SIZE == 2) { 148 *p = v; 149 } else { 150 memcpy(p, &v, sizeof(v)); 151 } 152 } 153 #endif 154 155 #if TCG_TARGET_INSN_UNIT_SIZE <= 4 156 static inline void tcg_out32(TCGContext *s, uint32_t v) 157 { 158 if (TCG_TARGET_INSN_UNIT_SIZE == 4) { 159 *s->code_ptr++ = v; 160 } else { 161 tcg_insn_unit *p = s->code_ptr; 162 memcpy(p, &v, sizeof(v)); 163 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE); 164 } 165 } 166 167 static inline void tcg_patch32(tcg_insn_unit *p, uint32_t v) 168 { 169 if (TCG_TARGET_INSN_UNIT_SIZE == 4) { 170 *p = v; 171 } else { 172 memcpy(p, &v, sizeof(v)); 173 } 174 } 175 #endif 176 177 #if TCG_TARGET_INSN_UNIT_SIZE <= 8 178 static inline void tcg_out64(TCGContext *s, uint64_t v) 179 { 180 if (TCG_TARGET_INSN_UNIT_SIZE == 8) { 181 *s->code_ptr++ = v; 182 } else { 183 tcg_insn_unit *p = s->code_ptr; 184 memcpy(p, &v, sizeof(v)); 185 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE); 186 } 187 } 188 189 static inline void tcg_patch64(tcg_insn_unit *p, uint64_t v) 190 { 191 if (TCG_TARGET_INSN_UNIT_SIZE == 8) { 192 *p = v; 193 } else { 194 memcpy(p, &v, sizeof(v)); 195 } 196 } 197 #endif 198 199 /* label relocation processing */ 200 201 static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type, 202 int label_index, intptr_t addend) 203 { 204 TCGLabel *l; 205 TCGRelocation *r; 206 207 l = &s->labels[label_index]; 208 if (l->has_value) { 209 /* FIXME: This may break relocations on RISC targets that 210 modify instruction fields in place. The caller may not have 211 written the initial value. */ 212 patch_reloc(code_ptr, type, l->u.value, addend); 213 } else { 214 /* add a new relocation entry */ 215 r = tcg_malloc(sizeof(TCGRelocation)); 216 r->type = type; 217 r->ptr = code_ptr; 218 r->addend = addend; 219 r->next = l->u.first_reloc; 220 l->u.first_reloc = r; 221 } 222 } 223 224 static void tcg_out_label(TCGContext *s, int label_index, tcg_insn_unit *ptr) 225 { 226 TCGLabel *l = &s->labels[label_index]; 227 intptr_t value = (intptr_t)ptr; 228 TCGRelocation *r; 229 230 assert(!l->has_value); 231 232 for (r = l->u.first_reloc; r != NULL; r = r->next) { 233 patch_reloc(r->ptr, r->type, value, r->addend); 234 } 235 236 l->has_value = 1; 237 l->u.value_ptr = ptr; 238 } 239 240 int gen_new_label(void) 241 { 242 TCGContext *s = &tcg_ctx; 243 int idx; 244 TCGLabel *l; 245 246 if (s->nb_labels >= TCG_MAX_LABELS) 247 tcg_abort(); 248 idx = s->nb_labels++; 249 l = &s->labels[idx]; 250 l->has_value = 0; 251 l->u.first_reloc = NULL; 252 return idx; 253 } 254 255 #include "tcg-target.c" 256 257 /* pool based memory allocation */ 258 void *tcg_malloc_internal(TCGContext *s, int size) 259 { 260 TCGPool *p; 261 int pool_size; 262 263 if (size > TCG_POOL_CHUNK_SIZE) { 264 /* big malloc: insert a new pool (XXX: could optimize) */ 265 p = g_malloc(sizeof(TCGPool) + size); 266 p->size = size; 267 p->next = s->pool_first_large; 268 s->pool_first_large = p; 269 return p->data; 270 } else { 271 p = s->pool_current; 272 if (!p) { 273 p = s->pool_first; 274 if (!p) 275 goto new_pool; 276 } else { 277 if (!p->next) { 278 new_pool: 279 pool_size = TCG_POOL_CHUNK_SIZE; 280 p = g_malloc(sizeof(TCGPool) + pool_size); 281 p->size = pool_size; 282 p->next = NULL; 283 if (s->pool_current) 284 s->pool_current->next = p; 285 else 286 s->pool_first = p; 287 } else { 288 p = p->next; 289 } 290 } 291 } 292 s->pool_current = p; 293 s->pool_cur = p->data + size; 294 s->pool_end = p->data + p->size; 295 return p->data; 296 } 297 298 void tcg_pool_reset(TCGContext *s) 299 { 300 TCGPool *p, *t; 301 for (p = s->pool_first_large; p; p = t) { 302 t = p->next; 303 g_free(p); 304 } 305 s->pool_first_large = NULL; 306 s->pool_cur = s->pool_end = NULL; 307 s->pool_current = NULL; 308 } 309 310 #include "helper.h" 311 312 typedef struct TCGHelperInfo { 313 void *func; 314 const char *name; 315 } TCGHelperInfo; 316 317 static const TCGHelperInfo all_helpers[] = { 318 #define GEN_HELPER 2 319 #include "helper.h" 320 321 /* Include tcg-runtime.c functions. */ 322 { tcg_helper_div_i32, "div_i32" }, 323 { tcg_helper_rem_i32, "rem_i32" }, 324 { tcg_helper_divu_i32, "divu_i32" }, 325 { tcg_helper_remu_i32, "remu_i32" }, 326 327 { tcg_helper_shl_i64, "shl_i64" }, 328 { tcg_helper_shr_i64, "shr_i64" }, 329 { tcg_helper_sar_i64, "sar_i64" }, 330 { tcg_helper_div_i64, "div_i64" }, 331 { tcg_helper_rem_i64, "rem_i64" }, 332 { tcg_helper_divu_i64, "divu_i64" }, 333 { tcg_helper_remu_i64, "remu_i64" }, 334 { tcg_helper_mulsh_i64, "mulsh_i64" }, 335 { tcg_helper_muluh_i64, "muluh_i64" }, 336 }; 337 338 void tcg_context_init(TCGContext *s) 339 { 340 int op, total_args, n, i; 341 TCGOpDef *def; 342 TCGArgConstraint *args_ct; 343 int *sorted_args; 344 GHashTable *helper_table; 345 346 memset(s, 0, sizeof(*s)); 347 s->nb_globals = 0; 348 349 /* Count total number of arguments and allocate the corresponding 350 space */ 351 total_args = 0; 352 for(op = 0; op < NB_OPS; op++) { 353 def = &tcg_op_defs[op]; 354 n = def->nb_iargs + def->nb_oargs; 355 total_args += n; 356 } 357 358 args_ct = g_malloc(sizeof(TCGArgConstraint) * total_args); 359 sorted_args = g_malloc(sizeof(int) * total_args); 360 361 for(op = 0; op < NB_OPS; op++) { 362 def = &tcg_op_defs[op]; 363 def->args_ct = args_ct; 364 def->sorted_args = sorted_args; 365 n = def->nb_iargs + def->nb_oargs; 366 sorted_args += n; 367 args_ct += n; 368 } 369 370 /* Register helpers. */ 371 /* Use g_direct_hash/equal for direct pointer comparisons on func. */ 372 s->helpers = helper_table = g_hash_table_new(NULL, NULL); 373 374 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) { 375 g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func, 376 (gpointer)all_helpers[i].name); 377 } 378 379 tcg_target_init(s); 380 } 381 382 void tcg_prologue_init(TCGContext *s) 383 { 384 /* init global prologue and epilogue */ 385 s->code_buf = s->code_gen_prologue; 386 s->code_ptr = s->code_buf; 387 tcg_target_qemu_prologue(s); 388 flush_icache_range((uintptr_t)s->code_buf, (uintptr_t)s->code_ptr); 389 390 #ifdef DEBUG_DISAS 391 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) { 392 size_t size = tcg_current_code_size(s); 393 qemu_log("PROLOGUE: [size=%zu]\n", size); 394 log_disas(s->code_buf, size); 395 qemu_log("\n"); 396 qemu_log_flush(); 397 } 398 #endif 399 } 400 401 void tcg_set_frame(TCGContext *s, int reg, intptr_t start, intptr_t size) 402 { 403 s->frame_start = start; 404 s->frame_end = start + size; 405 s->frame_reg = reg; 406 } 407 408 void tcg_func_start(TCGContext *s) 409 { 410 tcg_pool_reset(s); 411 s->nb_temps = s->nb_globals; 412 413 /* No temps have been previously allocated for size or locality. */ 414 memset(s->free_temps, 0, sizeof(s->free_temps)); 415 416 s->labels = tcg_malloc(sizeof(TCGLabel) * TCG_MAX_LABELS); 417 s->nb_labels = 0; 418 s->current_frame_offset = s->frame_start; 419 420 #ifdef CONFIG_DEBUG_TCG 421 s->goto_tb_issue_mask = 0; 422 #endif 423 424 s->gen_opc_ptr = s->gen_opc_buf; 425 s->gen_opparam_ptr = s->gen_opparam_buf; 426 427 s->be = tcg_malloc(sizeof(TCGBackendData)); 428 } 429 430 static inline void tcg_temp_alloc(TCGContext *s, int n) 431 { 432 if (n > TCG_MAX_TEMPS) 433 tcg_abort(); 434 } 435 436 static inline int tcg_global_reg_new_internal(TCGType type, int reg, 437 const char *name) 438 { 439 TCGContext *s = &tcg_ctx; 440 TCGTemp *ts; 441 int idx; 442 443 #if TCG_TARGET_REG_BITS == 32 444 if (type != TCG_TYPE_I32) 445 tcg_abort(); 446 #endif 447 if (tcg_regset_test_reg(s->reserved_regs, reg)) 448 tcg_abort(); 449 idx = s->nb_globals; 450 tcg_temp_alloc(s, s->nb_globals + 1); 451 ts = &s->temps[s->nb_globals]; 452 ts->base_type = type; 453 ts->type = type; 454 ts->fixed_reg = 1; 455 ts->reg = reg; 456 ts->name = name; 457 s->nb_globals++; 458 tcg_regset_set_reg(s->reserved_regs, reg); 459 return idx; 460 } 461 462 TCGv_i32 tcg_global_reg_new_i32(int reg, const char *name) 463 { 464 int idx; 465 466 idx = tcg_global_reg_new_internal(TCG_TYPE_I32, reg, name); 467 return MAKE_TCGV_I32(idx); 468 } 469 470 TCGv_i64 tcg_global_reg_new_i64(int reg, const char *name) 471 { 472 int idx; 473 474 idx = tcg_global_reg_new_internal(TCG_TYPE_I64, reg, name); 475 return MAKE_TCGV_I64(idx); 476 } 477 478 static inline int tcg_global_mem_new_internal(TCGType type, int reg, 479 intptr_t offset, 480 const char *name) 481 { 482 TCGContext *s = &tcg_ctx; 483 TCGTemp *ts; 484 int idx; 485 486 idx = s->nb_globals; 487 #if TCG_TARGET_REG_BITS == 32 488 if (type == TCG_TYPE_I64) { 489 char buf[64]; 490 tcg_temp_alloc(s, s->nb_globals + 2); 491 ts = &s->temps[s->nb_globals]; 492 ts->base_type = type; 493 ts->type = TCG_TYPE_I32; 494 ts->fixed_reg = 0; 495 ts->mem_allocated = 1; 496 ts->mem_reg = reg; 497 #ifdef HOST_WORDS_BIGENDIAN 498 ts->mem_offset = offset + 4; 499 #else 500 ts->mem_offset = offset; 501 #endif 502 pstrcpy(buf, sizeof(buf), name); 503 pstrcat(buf, sizeof(buf), "_0"); 504 ts->name = strdup(buf); 505 ts++; 506 507 ts->base_type = type; 508 ts->type = TCG_TYPE_I32; 509 ts->fixed_reg = 0; 510 ts->mem_allocated = 1; 511 ts->mem_reg = reg; 512 #ifdef HOST_WORDS_BIGENDIAN 513 ts->mem_offset = offset; 514 #else 515 ts->mem_offset = offset + 4; 516 #endif 517 pstrcpy(buf, sizeof(buf), name); 518 pstrcat(buf, sizeof(buf), "_1"); 519 ts->name = strdup(buf); 520 521 s->nb_globals += 2; 522 } else 523 #endif 524 { 525 tcg_temp_alloc(s, s->nb_globals + 1); 526 ts = &s->temps[s->nb_globals]; 527 ts->base_type = type; 528 ts->type = type; 529 ts->fixed_reg = 0; 530 ts->mem_allocated = 1; 531 ts->mem_reg = reg; 532 ts->mem_offset = offset; 533 ts->name = name; 534 s->nb_globals++; 535 } 536 return idx; 537 } 538 539 TCGv_i32 tcg_global_mem_new_i32(int reg, intptr_t offset, const char *name) 540 { 541 int idx = tcg_global_mem_new_internal(TCG_TYPE_I32, reg, offset, name); 542 return MAKE_TCGV_I32(idx); 543 } 544 545 TCGv_i64 tcg_global_mem_new_i64(int reg, intptr_t offset, const char *name) 546 { 547 int idx = tcg_global_mem_new_internal(TCG_TYPE_I64, reg, offset, name); 548 return MAKE_TCGV_I64(idx); 549 } 550 551 static inline int tcg_temp_new_internal(TCGType type, int temp_local) 552 { 553 TCGContext *s = &tcg_ctx; 554 TCGTemp *ts; 555 int idx, k; 556 557 k = type + (temp_local ? TCG_TYPE_COUNT : 0); 558 idx = find_first_bit(s->free_temps[k].l, TCG_MAX_TEMPS); 559 if (idx < TCG_MAX_TEMPS) { 560 /* There is already an available temp with the right type. */ 561 clear_bit(idx, s->free_temps[k].l); 562 563 ts = &s->temps[idx]; 564 ts->temp_allocated = 1; 565 assert(ts->base_type == type); 566 assert(ts->temp_local == temp_local); 567 } else { 568 idx = s->nb_temps; 569 #if TCG_TARGET_REG_BITS == 32 570 if (type == TCG_TYPE_I64) { 571 tcg_temp_alloc(s, s->nb_temps + 2); 572 ts = &s->temps[s->nb_temps]; 573 ts->base_type = type; 574 ts->type = TCG_TYPE_I32; 575 ts->temp_allocated = 1; 576 ts->temp_local = temp_local; 577 ts->name = NULL; 578 ts++; 579 ts->base_type = type; 580 ts->type = TCG_TYPE_I32; 581 ts->temp_allocated = 1; 582 ts->temp_local = temp_local; 583 ts->name = NULL; 584 s->nb_temps += 2; 585 } else 586 #endif 587 { 588 tcg_temp_alloc(s, s->nb_temps + 1); 589 ts = &s->temps[s->nb_temps]; 590 ts->base_type = type; 591 ts->type = type; 592 ts->temp_allocated = 1; 593 ts->temp_local = temp_local; 594 ts->name = NULL; 595 s->nb_temps++; 596 } 597 } 598 599 #if defined(CONFIG_DEBUG_TCG) 600 s->temps_in_use++; 601 #endif 602 return idx; 603 } 604 605 TCGv_i32 tcg_temp_new_internal_i32(int temp_local) 606 { 607 int idx; 608 609 idx = tcg_temp_new_internal(TCG_TYPE_I32, temp_local); 610 return MAKE_TCGV_I32(idx); 611 } 612 613 TCGv_i64 tcg_temp_new_internal_i64(int temp_local) 614 { 615 int idx; 616 617 idx = tcg_temp_new_internal(TCG_TYPE_I64, temp_local); 618 return MAKE_TCGV_I64(idx); 619 } 620 621 static void tcg_temp_free_internal(int idx) 622 { 623 TCGContext *s = &tcg_ctx; 624 TCGTemp *ts; 625 int k; 626 627 #if defined(CONFIG_DEBUG_TCG) 628 s->temps_in_use--; 629 if (s->temps_in_use < 0) { 630 fprintf(stderr, "More temporaries freed than allocated!\n"); 631 } 632 #endif 633 634 assert(idx >= s->nb_globals && idx < s->nb_temps); 635 ts = &s->temps[idx]; 636 assert(ts->temp_allocated != 0); 637 ts->temp_allocated = 0; 638 639 k = ts->base_type + (ts->temp_local ? TCG_TYPE_COUNT : 0); 640 set_bit(idx, s->free_temps[k].l); 641 } 642 643 void tcg_temp_free_i32(TCGv_i32 arg) 644 { 645 tcg_temp_free_internal(GET_TCGV_I32(arg)); 646 } 647 648 void tcg_temp_free_i64(TCGv_i64 arg) 649 { 650 tcg_temp_free_internal(GET_TCGV_I64(arg)); 651 } 652 653 TCGv_i32 tcg_const_i32(int32_t val) 654 { 655 TCGv_i32 t0; 656 t0 = tcg_temp_new_i32(); 657 tcg_gen_movi_i32(t0, val); 658 return t0; 659 } 660 661 TCGv_i64 tcg_const_i64(int64_t val) 662 { 663 TCGv_i64 t0; 664 t0 = tcg_temp_new_i64(); 665 tcg_gen_movi_i64(t0, val); 666 return t0; 667 } 668 669 TCGv_i32 tcg_const_local_i32(int32_t val) 670 { 671 TCGv_i32 t0; 672 t0 = tcg_temp_local_new_i32(); 673 tcg_gen_movi_i32(t0, val); 674 return t0; 675 } 676 677 TCGv_i64 tcg_const_local_i64(int64_t val) 678 { 679 TCGv_i64 t0; 680 t0 = tcg_temp_local_new_i64(); 681 tcg_gen_movi_i64(t0, val); 682 return t0; 683 } 684 685 #if defined(CONFIG_DEBUG_TCG) 686 void tcg_clear_temp_count(void) 687 { 688 TCGContext *s = &tcg_ctx; 689 s->temps_in_use = 0; 690 } 691 692 int tcg_check_temp_count(void) 693 { 694 TCGContext *s = &tcg_ctx; 695 if (s->temps_in_use) { 696 /* Clear the count so that we don't give another 697 * warning immediately next time around. 698 */ 699 s->temps_in_use = 0; 700 return 1; 701 } 702 return 0; 703 } 704 #endif 705 706 /* Note: we convert the 64 bit args to 32 bit and do some alignment 707 and endian swap. Maybe it would be better to do the alignment 708 and endian swap in tcg_reg_alloc_call(). */ 709 void tcg_gen_callN(TCGContext *s, void *func, unsigned int flags, 710 int sizemask, TCGArg ret, int nargs, TCGArg *args) 711 { 712 int i; 713 int real_args; 714 int nb_rets; 715 TCGArg *nparam; 716 717 #if defined(__sparc__) && !defined(__arch64__) \ 718 && !defined(CONFIG_TCG_INTERPRETER) 719 /* We have 64-bit values in one register, but need to pass as two 720 separate parameters. Split them. */ 721 int orig_sizemask = sizemask; 722 int orig_nargs = nargs; 723 TCGv_i64 retl, reth; 724 725 TCGV_UNUSED_I64(retl); 726 TCGV_UNUSED_I64(reth); 727 if (sizemask != 0) { 728 TCGArg *split_args = __builtin_alloca(sizeof(TCGArg) * nargs * 2); 729 for (i = real_args = 0; i < nargs; ++i) { 730 int is_64bit = sizemask & (1 << (i+1)*2); 731 if (is_64bit) { 732 TCGv_i64 orig = MAKE_TCGV_I64(args[i]); 733 TCGv_i32 h = tcg_temp_new_i32(); 734 TCGv_i32 l = tcg_temp_new_i32(); 735 tcg_gen_extr_i64_i32(l, h, orig); 736 split_args[real_args++] = GET_TCGV_I32(h); 737 split_args[real_args++] = GET_TCGV_I32(l); 738 } else { 739 split_args[real_args++] = args[i]; 740 } 741 } 742 nargs = real_args; 743 args = split_args; 744 sizemask = 0; 745 } 746 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 747 for (i = 0; i < nargs; ++i) { 748 int is_64bit = sizemask & (1 << (i+1)*2); 749 int is_signed = sizemask & (2 << (i+1)*2); 750 if (!is_64bit) { 751 TCGv_i64 temp = tcg_temp_new_i64(); 752 TCGv_i64 orig = MAKE_TCGV_I64(args[i]); 753 if (is_signed) { 754 tcg_gen_ext32s_i64(temp, orig); 755 } else { 756 tcg_gen_ext32u_i64(temp, orig); 757 } 758 args[i] = GET_TCGV_I64(temp); 759 } 760 } 761 #endif /* TCG_TARGET_EXTEND_ARGS */ 762 763 *s->gen_opc_ptr++ = INDEX_op_call; 764 nparam = s->gen_opparam_ptr++; 765 if (ret != TCG_CALL_DUMMY_ARG) { 766 #if defined(__sparc__) && !defined(__arch64__) \ 767 && !defined(CONFIG_TCG_INTERPRETER) 768 if (orig_sizemask & 1) { 769 /* The 32-bit ABI is going to return the 64-bit value in 770 the %o0/%o1 register pair. Prepare for this by using 771 two return temporaries, and reassemble below. */ 772 retl = tcg_temp_new_i64(); 773 reth = tcg_temp_new_i64(); 774 *s->gen_opparam_ptr++ = GET_TCGV_I64(reth); 775 *s->gen_opparam_ptr++ = GET_TCGV_I64(retl); 776 nb_rets = 2; 777 } else { 778 *s->gen_opparam_ptr++ = ret; 779 nb_rets = 1; 780 } 781 #else 782 if (TCG_TARGET_REG_BITS < 64 && (sizemask & 1)) { 783 #ifdef HOST_WORDS_BIGENDIAN 784 *s->gen_opparam_ptr++ = ret + 1; 785 *s->gen_opparam_ptr++ = ret; 786 #else 787 *s->gen_opparam_ptr++ = ret; 788 *s->gen_opparam_ptr++ = ret + 1; 789 #endif 790 nb_rets = 2; 791 } else { 792 *s->gen_opparam_ptr++ = ret; 793 nb_rets = 1; 794 } 795 #endif 796 } else { 797 nb_rets = 0; 798 } 799 real_args = 0; 800 for (i = 0; i < nargs; i++) { 801 #if TCG_TARGET_REG_BITS < 64 802 int is_64bit = sizemask & (1 << (i+1)*2); 803 if (is_64bit) { 804 #ifdef TCG_TARGET_CALL_ALIGN_ARGS 805 /* some targets want aligned 64 bit args */ 806 if (real_args & 1) { 807 *s->gen_opparam_ptr++ = TCG_CALL_DUMMY_ARG; 808 real_args++; 809 } 810 #endif 811 /* If stack grows up, then we will be placing successive 812 arguments at lower addresses, which means we need to 813 reverse the order compared to how we would normally 814 treat either big or little-endian. For those arguments 815 that will wind up in registers, this still works for 816 HPPA (the only current STACK_GROWSUP target) since the 817 argument registers are *also* allocated in decreasing 818 order. If another such target is added, this logic may 819 have to get more complicated to differentiate between 820 stack arguments and register arguments. */ 821 #if defined(HOST_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP) 822 *s->gen_opparam_ptr++ = args[i] + 1; 823 *s->gen_opparam_ptr++ = args[i]; 824 #else 825 *s->gen_opparam_ptr++ = args[i]; 826 *s->gen_opparam_ptr++ = args[i] + 1; 827 #endif 828 real_args += 2; 829 continue; 830 } 831 #endif /* TCG_TARGET_REG_BITS < 64 */ 832 833 *s->gen_opparam_ptr++ = args[i]; 834 real_args++; 835 } 836 *s->gen_opparam_ptr++ = (uintptr_t)func; 837 *s->gen_opparam_ptr++ = flags; 838 839 *nparam = (nb_rets << 16) | real_args; 840 841 /* total parameters, needed to go backward in the instruction stream */ 842 *s->gen_opparam_ptr++ = 1 + nb_rets + real_args + 3; 843 844 #if defined(__sparc__) && !defined(__arch64__) \ 845 && !defined(CONFIG_TCG_INTERPRETER) 846 /* Free all of the parts we allocated above. */ 847 for (i = real_args = 0; i < orig_nargs; ++i) { 848 int is_64bit = orig_sizemask & (1 << (i+1)*2); 849 if (is_64bit) { 850 TCGv_i32 h = MAKE_TCGV_I32(args[real_args++]); 851 TCGv_i32 l = MAKE_TCGV_I32(args[real_args++]); 852 tcg_temp_free_i32(h); 853 tcg_temp_free_i32(l); 854 } else { 855 real_args++; 856 } 857 } 858 if (orig_sizemask & 1) { 859 /* The 32-bit ABI returned two 32-bit pieces. Re-assemble them. 860 Note that describing these as TCGv_i64 eliminates an unnecessary 861 zero-extension that tcg_gen_concat_i32_i64 would create. */ 862 tcg_gen_concat32_i64(MAKE_TCGV_I64(ret), retl, reth); 863 tcg_temp_free_i64(retl); 864 tcg_temp_free_i64(reth); 865 } 866 #elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 867 for (i = 0; i < nargs; ++i) { 868 int is_64bit = sizemask & (1 << (i+1)*2); 869 if (!is_64bit) { 870 TCGv_i64 temp = MAKE_TCGV_I64(args[i]); 871 tcg_temp_free_i64(temp); 872 } 873 } 874 #endif /* TCG_TARGET_EXTEND_ARGS */ 875 } 876 877 #if TCG_TARGET_REG_BITS == 32 878 void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1, 879 int c, int right, int arith) 880 { 881 if (c == 0) { 882 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1)); 883 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1)); 884 } else if (c >= 32) { 885 c -= 32; 886 if (right) { 887 if (arith) { 888 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 889 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31); 890 } else { 891 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c); 892 tcg_gen_movi_i32(TCGV_HIGH(ret), 0); 893 } 894 } else { 895 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c); 896 tcg_gen_movi_i32(TCGV_LOW(ret), 0); 897 } 898 } else { 899 TCGv_i32 t0, t1; 900 901 t0 = tcg_temp_new_i32(); 902 t1 = tcg_temp_new_i32(); 903 if (right) { 904 tcg_gen_shli_i32(t0, TCGV_HIGH(arg1), 32 - c); 905 if (arith) 906 tcg_gen_sari_i32(t1, TCGV_HIGH(arg1), c); 907 else 908 tcg_gen_shri_i32(t1, TCGV_HIGH(arg1), c); 909 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c); 910 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t0); 911 tcg_gen_mov_i32(TCGV_HIGH(ret), t1); 912 } else { 913 tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c); 914 /* Note: ret can be the same as arg1, so we use t1 */ 915 tcg_gen_shli_i32(t1, TCGV_LOW(arg1), c); 916 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c); 917 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t0); 918 tcg_gen_mov_i32(TCGV_LOW(ret), t1); 919 } 920 tcg_temp_free_i32(t0); 921 tcg_temp_free_i32(t1); 922 } 923 } 924 #endif 925 926 static inline TCGMemOp tcg_canonicalize_memop(TCGMemOp op, bool is64, bool st) 927 { 928 switch (op & MO_SIZE) { 929 case MO_8: 930 op &= ~MO_BSWAP; 931 break; 932 case MO_16: 933 break; 934 case MO_32: 935 if (!is64) { 936 op &= ~MO_SIGN; 937 } 938 break; 939 case MO_64: 940 if (!is64) { 941 tcg_abort(); 942 } 943 break; 944 } 945 if (st) { 946 op &= ~MO_SIGN; 947 } 948 return op; 949 } 950 951 static const TCGOpcode old_ld_opc[8] = { 952 [MO_UB] = INDEX_op_qemu_ld8u, 953 [MO_SB] = INDEX_op_qemu_ld8s, 954 [MO_UW] = INDEX_op_qemu_ld16u, 955 [MO_SW] = INDEX_op_qemu_ld16s, 956 #if TCG_TARGET_REG_BITS == 32 957 [MO_UL] = INDEX_op_qemu_ld32, 958 [MO_SL] = INDEX_op_qemu_ld32, 959 #else 960 [MO_UL] = INDEX_op_qemu_ld32u, 961 [MO_SL] = INDEX_op_qemu_ld32s, 962 #endif 963 [MO_Q] = INDEX_op_qemu_ld64, 964 }; 965 966 static const TCGOpcode old_st_opc[4] = { 967 [MO_UB] = INDEX_op_qemu_st8, 968 [MO_UW] = INDEX_op_qemu_st16, 969 [MO_UL] = INDEX_op_qemu_st32, 970 [MO_Q] = INDEX_op_qemu_st64, 971 }; 972 973 void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop) 974 { 975 memop = tcg_canonicalize_memop(memop, 0, 0); 976 977 if (TCG_TARGET_HAS_new_ldst) { 978 *tcg_ctx.gen_opc_ptr++ = INDEX_op_qemu_ld_i32; 979 tcg_add_param_i32(val); 980 tcg_add_param_tl(addr); 981 *tcg_ctx.gen_opparam_ptr++ = memop; 982 *tcg_ctx.gen_opparam_ptr++ = idx; 983 return; 984 } 985 986 /* The old opcodes only support target-endian memory operations. */ 987 assert((memop & MO_BSWAP) == MO_TE || (memop & MO_SIZE) == MO_8); 988 assert(old_ld_opc[memop & MO_SSIZE] != 0); 989 990 if (TCG_TARGET_REG_BITS == 32) { 991 *tcg_ctx.gen_opc_ptr++ = old_ld_opc[memop & MO_SSIZE]; 992 tcg_add_param_i32(val); 993 tcg_add_param_tl(addr); 994 *tcg_ctx.gen_opparam_ptr++ = idx; 995 } else { 996 TCGv_i64 val64 = tcg_temp_new_i64(); 997 998 *tcg_ctx.gen_opc_ptr++ = old_ld_opc[memop & MO_SSIZE]; 999 tcg_add_param_i64(val64); 1000 tcg_add_param_tl(addr); 1001 *tcg_ctx.gen_opparam_ptr++ = idx; 1002 1003 tcg_gen_trunc_i64_i32(val, val64); 1004 tcg_temp_free_i64(val64); 1005 } 1006 } 1007 1008 void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop) 1009 { 1010 memop = tcg_canonicalize_memop(memop, 0, 1); 1011 1012 if (TCG_TARGET_HAS_new_ldst) { 1013 *tcg_ctx.gen_opc_ptr++ = INDEX_op_qemu_st_i32; 1014 tcg_add_param_i32(val); 1015 tcg_add_param_tl(addr); 1016 *tcg_ctx.gen_opparam_ptr++ = memop; 1017 *tcg_ctx.gen_opparam_ptr++ = idx; 1018 return; 1019 } 1020 1021 /* The old opcodes only support target-endian memory operations. */ 1022 assert((memop & MO_BSWAP) == MO_TE || (memop & MO_SIZE) == MO_8); 1023 assert(old_st_opc[memop & MO_SIZE] != 0); 1024 1025 if (TCG_TARGET_REG_BITS == 32) { 1026 *tcg_ctx.gen_opc_ptr++ = old_st_opc[memop & MO_SIZE]; 1027 tcg_add_param_i32(val); 1028 tcg_add_param_tl(addr); 1029 *tcg_ctx.gen_opparam_ptr++ = idx; 1030 } else { 1031 TCGv_i64 val64 = tcg_temp_new_i64(); 1032 1033 tcg_gen_extu_i32_i64(val64, val); 1034 1035 *tcg_ctx.gen_opc_ptr++ = old_st_opc[memop & MO_SIZE]; 1036 tcg_add_param_i64(val64); 1037 tcg_add_param_tl(addr); 1038 *tcg_ctx.gen_opparam_ptr++ = idx; 1039 1040 tcg_temp_free_i64(val64); 1041 } 1042 } 1043 1044 void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop) 1045 { 1046 memop = tcg_canonicalize_memop(memop, 1, 0); 1047 1048 #if TCG_TARGET_REG_BITS == 32 1049 if ((memop & MO_SIZE) < MO_64) { 1050 tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop); 1051 if (memop & MO_SIGN) { 1052 tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31); 1053 } else { 1054 tcg_gen_movi_i32(TCGV_HIGH(val), 0); 1055 } 1056 return; 1057 } 1058 #endif 1059 1060 if (TCG_TARGET_HAS_new_ldst) { 1061 *tcg_ctx.gen_opc_ptr++ = INDEX_op_qemu_ld_i64; 1062 tcg_add_param_i64(val); 1063 tcg_add_param_tl(addr); 1064 *tcg_ctx.gen_opparam_ptr++ = memop; 1065 *tcg_ctx.gen_opparam_ptr++ = idx; 1066 return; 1067 } 1068 1069 /* The old opcodes only support target-endian memory operations. */ 1070 assert((memop & MO_BSWAP) == MO_TE || (memop & MO_SIZE) == MO_8); 1071 assert(old_ld_opc[memop & MO_SSIZE] != 0); 1072 1073 *tcg_ctx.gen_opc_ptr++ = old_ld_opc[memop & MO_SSIZE]; 1074 tcg_add_param_i64(val); 1075 tcg_add_param_tl(addr); 1076 *tcg_ctx.gen_opparam_ptr++ = idx; 1077 } 1078 1079 void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop) 1080 { 1081 memop = tcg_canonicalize_memop(memop, 1, 1); 1082 1083 #if TCG_TARGET_REG_BITS == 32 1084 if ((memop & MO_SIZE) < MO_64) { 1085 tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop); 1086 return; 1087 } 1088 #endif 1089 1090 if (TCG_TARGET_HAS_new_ldst) { 1091 *tcg_ctx.gen_opc_ptr++ = INDEX_op_qemu_st_i64; 1092 tcg_add_param_i64(val); 1093 tcg_add_param_tl(addr); 1094 *tcg_ctx.gen_opparam_ptr++ = memop; 1095 *tcg_ctx.gen_opparam_ptr++ = idx; 1096 return; 1097 } 1098 1099 /* The old opcodes only support target-endian memory operations. */ 1100 assert((memop & MO_BSWAP) == MO_TE || (memop & MO_SIZE) == MO_8); 1101 assert(old_st_opc[memop & MO_SIZE] != 0); 1102 1103 *tcg_ctx.gen_opc_ptr++ = old_st_opc[memop & MO_SIZE]; 1104 tcg_add_param_i64(val); 1105 tcg_add_param_tl(addr); 1106 *tcg_ctx.gen_opparam_ptr++ = idx; 1107 } 1108 1109 static void tcg_reg_alloc_start(TCGContext *s) 1110 { 1111 int i; 1112 TCGTemp *ts; 1113 for(i = 0; i < s->nb_globals; i++) { 1114 ts = &s->temps[i]; 1115 if (ts->fixed_reg) { 1116 ts->val_type = TEMP_VAL_REG; 1117 } else { 1118 ts->val_type = TEMP_VAL_MEM; 1119 } 1120 } 1121 for(i = s->nb_globals; i < s->nb_temps; i++) { 1122 ts = &s->temps[i]; 1123 if (ts->temp_local) { 1124 ts->val_type = TEMP_VAL_MEM; 1125 } else { 1126 ts->val_type = TEMP_VAL_DEAD; 1127 } 1128 ts->mem_allocated = 0; 1129 ts->fixed_reg = 0; 1130 } 1131 for(i = 0; i < TCG_TARGET_NB_REGS; i++) { 1132 s->reg_to_temp[i] = -1; 1133 } 1134 } 1135 1136 static char *tcg_get_arg_str_idx(TCGContext *s, char *buf, int buf_size, 1137 int idx) 1138 { 1139 TCGTemp *ts; 1140 1141 assert(idx >= 0 && idx < s->nb_temps); 1142 ts = &s->temps[idx]; 1143 if (idx < s->nb_globals) { 1144 pstrcpy(buf, buf_size, ts->name); 1145 } else { 1146 if (ts->temp_local) 1147 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals); 1148 else 1149 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals); 1150 } 1151 return buf; 1152 } 1153 1154 char *tcg_get_arg_str_i32(TCGContext *s, char *buf, int buf_size, TCGv_i32 arg) 1155 { 1156 return tcg_get_arg_str_idx(s, buf, buf_size, GET_TCGV_I32(arg)); 1157 } 1158 1159 char *tcg_get_arg_str_i64(TCGContext *s, char *buf, int buf_size, TCGv_i64 arg) 1160 { 1161 return tcg_get_arg_str_idx(s, buf, buf_size, GET_TCGV_I64(arg)); 1162 } 1163 1164 /* Find helper name. */ 1165 static inline const char *tcg_find_helper(TCGContext *s, uintptr_t val) 1166 { 1167 const char *ret = NULL; 1168 if (s->helpers) { 1169 ret = g_hash_table_lookup(s->helpers, (gpointer)val); 1170 } 1171 return ret; 1172 } 1173 1174 static const char * const cond_name[] = 1175 { 1176 [TCG_COND_NEVER] = "never", 1177 [TCG_COND_ALWAYS] = "always", 1178 [TCG_COND_EQ] = "eq", 1179 [TCG_COND_NE] = "ne", 1180 [TCG_COND_LT] = "lt", 1181 [TCG_COND_GE] = "ge", 1182 [TCG_COND_LE] = "le", 1183 [TCG_COND_GT] = "gt", 1184 [TCG_COND_LTU] = "ltu", 1185 [TCG_COND_GEU] = "geu", 1186 [TCG_COND_LEU] = "leu", 1187 [TCG_COND_GTU] = "gtu" 1188 }; 1189 1190 static const char * const ldst_name[] = 1191 { 1192 [MO_UB] = "ub", 1193 [MO_SB] = "sb", 1194 [MO_LEUW] = "leuw", 1195 [MO_LESW] = "lesw", 1196 [MO_LEUL] = "leul", 1197 [MO_LESL] = "lesl", 1198 [MO_LEQ] = "leq", 1199 [MO_BEUW] = "beuw", 1200 [MO_BESW] = "besw", 1201 [MO_BEUL] = "beul", 1202 [MO_BESL] = "besl", 1203 [MO_BEQ] = "beq", 1204 }; 1205 1206 void tcg_dump_ops(TCGContext *s) 1207 { 1208 const uint16_t *opc_ptr; 1209 const TCGArg *args; 1210 TCGArg arg; 1211 TCGOpcode c; 1212 int i, k, nb_oargs, nb_iargs, nb_cargs, first_insn; 1213 const TCGOpDef *def; 1214 char buf[128]; 1215 1216 first_insn = 1; 1217 opc_ptr = s->gen_opc_buf; 1218 args = s->gen_opparam_buf; 1219 while (opc_ptr < s->gen_opc_ptr) { 1220 c = *opc_ptr++; 1221 def = &tcg_op_defs[c]; 1222 if (c == INDEX_op_debug_insn_start) { 1223 uint64_t pc; 1224 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS 1225 pc = ((uint64_t)args[1] << 32) | args[0]; 1226 #else 1227 pc = args[0]; 1228 #endif 1229 if (!first_insn) { 1230 qemu_log("\n"); 1231 } 1232 qemu_log(" ---- 0x%" PRIx64, pc); 1233 first_insn = 0; 1234 nb_oargs = def->nb_oargs; 1235 nb_iargs = def->nb_iargs; 1236 nb_cargs = def->nb_cargs; 1237 } else if (c == INDEX_op_call) { 1238 TCGArg arg; 1239 1240 /* variable number of arguments */ 1241 arg = *args++; 1242 nb_oargs = arg >> 16; 1243 nb_iargs = arg & 0xffff; 1244 nb_cargs = def->nb_cargs; 1245 1246 /* function name, flags, out args */ 1247 qemu_log(" %s %s,$0x%" TCG_PRIlx ",$%d", def->name, 1248 tcg_find_helper(s, args[nb_oargs + nb_iargs]), 1249 args[nb_oargs + nb_iargs + 1], nb_oargs); 1250 for (i = 0; i < nb_oargs; i++) { 1251 qemu_log(",%s", tcg_get_arg_str_idx(s, buf, sizeof(buf), 1252 args[i])); 1253 } 1254 for (i = 0; i < nb_iargs; i++) { 1255 TCGArg arg = args[nb_oargs + i]; 1256 const char *t = "<dummy>"; 1257 if (arg != TCG_CALL_DUMMY_ARG) { 1258 t = tcg_get_arg_str_idx(s, buf, sizeof(buf), arg); 1259 } 1260 qemu_log(",%s", t); 1261 } 1262 } else { 1263 qemu_log(" %s ", def->name); 1264 if (c == INDEX_op_nopn) { 1265 /* variable number of arguments */ 1266 nb_cargs = *args; 1267 nb_oargs = 0; 1268 nb_iargs = 0; 1269 } else { 1270 nb_oargs = def->nb_oargs; 1271 nb_iargs = def->nb_iargs; 1272 nb_cargs = def->nb_cargs; 1273 } 1274 1275 k = 0; 1276 for(i = 0; i < nb_oargs; i++) { 1277 if (k != 0) { 1278 qemu_log(","); 1279 } 1280 qemu_log("%s", tcg_get_arg_str_idx(s, buf, sizeof(buf), 1281 args[k++])); 1282 } 1283 for(i = 0; i < nb_iargs; i++) { 1284 if (k != 0) { 1285 qemu_log(","); 1286 } 1287 qemu_log("%s", tcg_get_arg_str_idx(s, buf, sizeof(buf), 1288 args[k++])); 1289 } 1290 switch (c) { 1291 case INDEX_op_brcond_i32: 1292 case INDEX_op_setcond_i32: 1293 case INDEX_op_movcond_i32: 1294 case INDEX_op_brcond2_i32: 1295 case INDEX_op_setcond2_i32: 1296 case INDEX_op_brcond_i64: 1297 case INDEX_op_setcond_i64: 1298 case INDEX_op_movcond_i64: 1299 if (args[k] < ARRAY_SIZE(cond_name) && cond_name[args[k]]) { 1300 qemu_log(",%s", cond_name[args[k++]]); 1301 } else { 1302 qemu_log(",$0x%" TCG_PRIlx, args[k++]); 1303 } 1304 i = 1; 1305 break; 1306 case INDEX_op_qemu_ld_i32: 1307 case INDEX_op_qemu_st_i32: 1308 case INDEX_op_qemu_ld_i64: 1309 case INDEX_op_qemu_st_i64: 1310 if (args[k] < ARRAY_SIZE(ldst_name) && ldst_name[args[k]]) { 1311 qemu_log(",%s", ldst_name[args[k++]]); 1312 } else { 1313 qemu_log(",$0x%" TCG_PRIlx, args[k++]); 1314 } 1315 i = 1; 1316 break; 1317 default: 1318 i = 0; 1319 break; 1320 } 1321 for(; i < nb_cargs; i++) { 1322 if (k != 0) { 1323 qemu_log(","); 1324 } 1325 arg = args[k++]; 1326 qemu_log("$0x%" TCG_PRIlx, arg); 1327 } 1328 } 1329 qemu_log("\n"); 1330 args += nb_iargs + nb_oargs + nb_cargs; 1331 } 1332 } 1333 1334 /* we give more priority to constraints with less registers */ 1335 static int get_constraint_priority(const TCGOpDef *def, int k) 1336 { 1337 const TCGArgConstraint *arg_ct; 1338 1339 int i, n; 1340 arg_ct = &def->args_ct[k]; 1341 if (arg_ct->ct & TCG_CT_ALIAS) { 1342 /* an alias is equivalent to a single register */ 1343 n = 1; 1344 } else { 1345 if (!(arg_ct->ct & TCG_CT_REG)) 1346 return 0; 1347 n = 0; 1348 for(i = 0; i < TCG_TARGET_NB_REGS; i++) { 1349 if (tcg_regset_test_reg(arg_ct->u.regs, i)) 1350 n++; 1351 } 1352 } 1353 return TCG_TARGET_NB_REGS - n + 1; 1354 } 1355 1356 /* sort from highest priority to lowest */ 1357 static void sort_constraints(TCGOpDef *def, int start, int n) 1358 { 1359 int i, j, p1, p2, tmp; 1360 1361 for(i = 0; i < n; i++) 1362 def->sorted_args[start + i] = start + i; 1363 if (n <= 1) 1364 return; 1365 for(i = 0; i < n - 1; i++) { 1366 for(j = i + 1; j < n; j++) { 1367 p1 = get_constraint_priority(def, def->sorted_args[start + i]); 1368 p2 = get_constraint_priority(def, def->sorted_args[start + j]); 1369 if (p1 < p2) { 1370 tmp = def->sorted_args[start + i]; 1371 def->sorted_args[start + i] = def->sorted_args[start + j]; 1372 def->sorted_args[start + j] = tmp; 1373 } 1374 } 1375 } 1376 } 1377 1378 void tcg_add_target_add_op_defs(const TCGTargetOpDef *tdefs) 1379 { 1380 TCGOpcode op; 1381 TCGOpDef *def; 1382 const char *ct_str; 1383 int i, nb_args; 1384 1385 for(;;) { 1386 if (tdefs->op == (TCGOpcode)-1) 1387 break; 1388 op = tdefs->op; 1389 assert((unsigned)op < NB_OPS); 1390 def = &tcg_op_defs[op]; 1391 #if defined(CONFIG_DEBUG_TCG) 1392 /* Duplicate entry in op definitions? */ 1393 assert(!def->used); 1394 def->used = 1; 1395 #endif 1396 nb_args = def->nb_iargs + def->nb_oargs; 1397 for(i = 0; i < nb_args; i++) { 1398 ct_str = tdefs->args_ct_str[i]; 1399 /* Incomplete TCGTargetOpDef entry? */ 1400 assert(ct_str != NULL); 1401 tcg_regset_clear(def->args_ct[i].u.regs); 1402 def->args_ct[i].ct = 0; 1403 if (ct_str[0] >= '0' && ct_str[0] <= '9') { 1404 int oarg; 1405 oarg = ct_str[0] - '0'; 1406 assert(oarg < def->nb_oargs); 1407 assert(def->args_ct[oarg].ct & TCG_CT_REG); 1408 /* TCG_CT_ALIAS is for the output arguments. The input 1409 argument is tagged with TCG_CT_IALIAS. */ 1410 def->args_ct[i] = def->args_ct[oarg]; 1411 def->args_ct[oarg].ct = TCG_CT_ALIAS; 1412 def->args_ct[oarg].alias_index = i; 1413 def->args_ct[i].ct |= TCG_CT_IALIAS; 1414 def->args_ct[i].alias_index = oarg; 1415 } else { 1416 for(;;) { 1417 if (*ct_str == '\0') 1418 break; 1419 switch(*ct_str) { 1420 case 'i': 1421 def->args_ct[i].ct |= TCG_CT_CONST; 1422 ct_str++; 1423 break; 1424 default: 1425 if (target_parse_constraint(&def->args_ct[i], &ct_str) < 0) { 1426 fprintf(stderr, "Invalid constraint '%s' for arg %d of operation '%s'\n", 1427 ct_str, i, def->name); 1428 exit(1); 1429 } 1430 } 1431 } 1432 } 1433 } 1434 1435 /* TCGTargetOpDef entry with too much information? */ 1436 assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL); 1437 1438 /* sort the constraints (XXX: this is just an heuristic) */ 1439 sort_constraints(def, 0, def->nb_oargs); 1440 sort_constraints(def, def->nb_oargs, def->nb_iargs); 1441 1442 #if 0 1443 { 1444 int i; 1445 1446 printf("%s: sorted=", def->name); 1447 for(i = 0; i < def->nb_oargs + def->nb_iargs; i++) 1448 printf(" %d", def->sorted_args[i]); 1449 printf("\n"); 1450 } 1451 #endif 1452 tdefs++; 1453 } 1454 1455 #if defined(CONFIG_DEBUG_TCG) 1456 i = 0; 1457 for (op = 0; op < ARRAY_SIZE(tcg_op_defs); op++) { 1458 const TCGOpDef *def = &tcg_op_defs[op]; 1459 if (def->flags & TCG_OPF_NOT_PRESENT) { 1460 /* Wrong entry in op definitions? */ 1461 if (def->used) { 1462 fprintf(stderr, "Invalid op definition for %s\n", def->name); 1463 i = 1; 1464 } 1465 } else { 1466 /* Missing entry in op definitions? */ 1467 if (!def->used) { 1468 fprintf(stderr, "Missing op definition for %s\n", def->name); 1469 i = 1; 1470 } 1471 } 1472 } 1473 if (i == 1) { 1474 tcg_abort(); 1475 } 1476 #endif 1477 } 1478 1479 #ifdef USE_LIVENESS_ANALYSIS 1480 1481 /* set a nop for an operation using 'nb_args' */ 1482 static inline void tcg_set_nop(TCGContext *s, uint16_t *opc_ptr, 1483 TCGArg *args, int nb_args) 1484 { 1485 if (nb_args == 0) { 1486 *opc_ptr = INDEX_op_nop; 1487 } else { 1488 *opc_ptr = INDEX_op_nopn; 1489 args[0] = nb_args; 1490 args[nb_args - 1] = nb_args; 1491 } 1492 } 1493 1494 /* liveness analysis: end of function: all temps are dead, and globals 1495 should be in memory. */ 1496 static inline void tcg_la_func_end(TCGContext *s, uint8_t *dead_temps, 1497 uint8_t *mem_temps) 1498 { 1499 memset(dead_temps, 1, s->nb_temps); 1500 memset(mem_temps, 1, s->nb_globals); 1501 memset(mem_temps + s->nb_globals, 0, s->nb_temps - s->nb_globals); 1502 } 1503 1504 /* liveness analysis: end of basic block: all temps are dead, globals 1505 and local temps should be in memory. */ 1506 static inline void tcg_la_bb_end(TCGContext *s, uint8_t *dead_temps, 1507 uint8_t *mem_temps) 1508 { 1509 int i; 1510 1511 memset(dead_temps, 1, s->nb_temps); 1512 memset(mem_temps, 1, s->nb_globals); 1513 for(i = s->nb_globals; i < s->nb_temps; i++) { 1514 mem_temps[i] = s->temps[i].temp_local; 1515 } 1516 } 1517 1518 /* Liveness analysis : update the opc_dead_args array to tell if a 1519 given input arguments is dead. Instructions updating dead 1520 temporaries are removed. */ 1521 static void tcg_liveness_analysis(TCGContext *s) 1522 { 1523 int i, op_index, nb_args, nb_iargs, nb_oargs, nb_ops; 1524 TCGOpcode op, op_new, op_new2; 1525 TCGArg *args, arg; 1526 const TCGOpDef *def; 1527 uint8_t *dead_temps, *mem_temps; 1528 uint16_t dead_args; 1529 uint8_t sync_args; 1530 bool have_op_new2; 1531 1532 s->gen_opc_ptr++; /* skip end */ 1533 1534 nb_ops = s->gen_opc_ptr - s->gen_opc_buf; 1535 1536 s->op_dead_args = tcg_malloc(nb_ops * sizeof(uint16_t)); 1537 s->op_sync_args = tcg_malloc(nb_ops * sizeof(uint8_t)); 1538 1539 dead_temps = tcg_malloc(s->nb_temps); 1540 mem_temps = tcg_malloc(s->nb_temps); 1541 tcg_la_func_end(s, dead_temps, mem_temps); 1542 1543 args = s->gen_opparam_ptr; 1544 op_index = nb_ops - 1; 1545 while (op_index >= 0) { 1546 op = s->gen_opc_buf[op_index]; 1547 def = &tcg_op_defs[op]; 1548 switch(op) { 1549 case INDEX_op_call: 1550 { 1551 int call_flags; 1552 1553 nb_args = args[-1]; 1554 args -= nb_args; 1555 arg = *args++; 1556 nb_iargs = arg & 0xffff; 1557 nb_oargs = arg >> 16; 1558 call_flags = args[nb_oargs + nb_iargs + 1]; 1559 1560 /* pure functions can be removed if their result is not 1561 used */ 1562 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) { 1563 for (i = 0; i < nb_oargs; i++) { 1564 arg = args[i]; 1565 if (!dead_temps[arg] || mem_temps[arg]) { 1566 goto do_not_remove_call; 1567 } 1568 } 1569 tcg_set_nop(s, s->gen_opc_buf + op_index, 1570 args - 1, nb_args); 1571 } else { 1572 do_not_remove_call: 1573 1574 /* output args are dead */ 1575 dead_args = 0; 1576 sync_args = 0; 1577 for (i = 0; i < nb_oargs; i++) { 1578 arg = args[i]; 1579 if (dead_temps[arg]) { 1580 dead_args |= (1 << i); 1581 } 1582 if (mem_temps[arg]) { 1583 sync_args |= (1 << i); 1584 } 1585 dead_temps[arg] = 1; 1586 mem_temps[arg] = 0; 1587 } 1588 1589 if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) { 1590 /* globals should be synced to memory */ 1591 memset(mem_temps, 1, s->nb_globals); 1592 } 1593 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS | 1594 TCG_CALL_NO_READ_GLOBALS))) { 1595 /* globals should go back to memory */ 1596 memset(dead_temps, 1, s->nb_globals); 1597 } 1598 1599 /* input args are live */ 1600 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 1601 arg = args[i]; 1602 if (arg != TCG_CALL_DUMMY_ARG) { 1603 if (dead_temps[arg]) { 1604 dead_args |= (1 << i); 1605 } 1606 dead_temps[arg] = 0; 1607 } 1608 } 1609 s->op_dead_args[op_index] = dead_args; 1610 s->op_sync_args[op_index] = sync_args; 1611 } 1612 args--; 1613 } 1614 break; 1615 case INDEX_op_debug_insn_start: 1616 args -= def->nb_args; 1617 break; 1618 case INDEX_op_nopn: 1619 nb_args = args[-1]; 1620 args -= nb_args; 1621 break; 1622 case INDEX_op_discard: 1623 args--; 1624 /* mark the temporary as dead */ 1625 dead_temps[args[0]] = 1; 1626 mem_temps[args[0]] = 0; 1627 break; 1628 case INDEX_op_end: 1629 break; 1630 1631 case INDEX_op_add2_i32: 1632 op_new = INDEX_op_add_i32; 1633 goto do_addsub2; 1634 case INDEX_op_sub2_i32: 1635 op_new = INDEX_op_sub_i32; 1636 goto do_addsub2; 1637 case INDEX_op_add2_i64: 1638 op_new = INDEX_op_add_i64; 1639 goto do_addsub2; 1640 case INDEX_op_sub2_i64: 1641 op_new = INDEX_op_sub_i64; 1642 do_addsub2: 1643 args -= 6; 1644 nb_iargs = 4; 1645 nb_oargs = 2; 1646 /* Test if the high part of the operation is dead, but not 1647 the low part. The result can be optimized to a simple 1648 add or sub. This happens often for x86_64 guest when the 1649 cpu mode is set to 32 bit. */ 1650 if (dead_temps[args[1]] && !mem_temps[args[1]]) { 1651 if (dead_temps[args[0]] && !mem_temps[args[0]]) { 1652 goto do_remove; 1653 } 1654 /* Create the single operation plus nop. */ 1655 s->gen_opc_buf[op_index] = op = op_new; 1656 args[1] = args[2]; 1657 args[2] = args[4]; 1658 assert(s->gen_opc_buf[op_index + 1] == INDEX_op_nop); 1659 tcg_set_nop(s, s->gen_opc_buf + op_index + 1, args + 3, 3); 1660 /* Fall through and mark the single-word operation live. */ 1661 nb_iargs = 2; 1662 nb_oargs = 1; 1663 } 1664 goto do_not_remove; 1665 1666 case INDEX_op_mulu2_i32: 1667 op_new = INDEX_op_mul_i32; 1668 op_new2 = INDEX_op_muluh_i32; 1669 have_op_new2 = TCG_TARGET_HAS_muluh_i32; 1670 goto do_mul2; 1671 case INDEX_op_muls2_i32: 1672 op_new = INDEX_op_mul_i32; 1673 op_new2 = INDEX_op_mulsh_i32; 1674 have_op_new2 = TCG_TARGET_HAS_mulsh_i32; 1675 goto do_mul2; 1676 case INDEX_op_mulu2_i64: 1677 op_new = INDEX_op_mul_i64; 1678 op_new2 = INDEX_op_muluh_i64; 1679 have_op_new2 = TCG_TARGET_HAS_muluh_i64; 1680 goto do_mul2; 1681 case INDEX_op_muls2_i64: 1682 op_new = INDEX_op_mul_i64; 1683 op_new2 = INDEX_op_mulsh_i64; 1684 have_op_new2 = TCG_TARGET_HAS_mulsh_i64; 1685 goto do_mul2; 1686 do_mul2: 1687 args -= 4; 1688 nb_iargs = 2; 1689 nb_oargs = 2; 1690 if (dead_temps[args[1]] && !mem_temps[args[1]]) { 1691 if (dead_temps[args[0]] && !mem_temps[args[0]]) { 1692 /* Both parts of the operation are dead. */ 1693 goto do_remove; 1694 } 1695 /* The high part of the operation is dead; generate the low. */ 1696 s->gen_opc_buf[op_index] = op = op_new; 1697 args[1] = args[2]; 1698 args[2] = args[3]; 1699 } else if (have_op_new2 && dead_temps[args[0]] 1700 && !mem_temps[args[0]]) { 1701 /* The low part of the operation is dead; generate the high. */ 1702 s->gen_opc_buf[op_index] = op = op_new2; 1703 args[0] = args[1]; 1704 args[1] = args[2]; 1705 args[2] = args[3]; 1706 } else { 1707 goto do_not_remove; 1708 } 1709 assert(s->gen_opc_buf[op_index + 1] == INDEX_op_nop); 1710 tcg_set_nop(s, s->gen_opc_buf + op_index + 1, args + 3, 1); 1711 /* Mark the single-word operation live. */ 1712 nb_oargs = 1; 1713 goto do_not_remove; 1714 1715 default: 1716 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */ 1717 args -= def->nb_args; 1718 nb_iargs = def->nb_iargs; 1719 nb_oargs = def->nb_oargs; 1720 1721 /* Test if the operation can be removed because all 1722 its outputs are dead. We assume that nb_oargs == 0 1723 implies side effects */ 1724 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) { 1725 for(i = 0; i < nb_oargs; i++) { 1726 arg = args[i]; 1727 if (!dead_temps[arg] || mem_temps[arg]) { 1728 goto do_not_remove; 1729 } 1730 } 1731 do_remove: 1732 tcg_set_nop(s, s->gen_opc_buf + op_index, args, def->nb_args); 1733 #ifdef CONFIG_PROFILER 1734 s->del_op_count++; 1735 #endif 1736 } else { 1737 do_not_remove: 1738 1739 /* output args are dead */ 1740 dead_args = 0; 1741 sync_args = 0; 1742 for(i = 0; i < nb_oargs; i++) { 1743 arg = args[i]; 1744 if (dead_temps[arg]) { 1745 dead_args |= (1 << i); 1746 } 1747 if (mem_temps[arg]) { 1748 sync_args |= (1 << i); 1749 } 1750 dead_temps[arg] = 1; 1751 mem_temps[arg] = 0; 1752 } 1753 1754 /* if end of basic block, update */ 1755 if (def->flags & TCG_OPF_BB_END) { 1756 tcg_la_bb_end(s, dead_temps, mem_temps); 1757 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) { 1758 /* globals should be synced to memory */ 1759 memset(mem_temps, 1, s->nb_globals); 1760 } 1761 1762 /* input args are live */ 1763 for(i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 1764 arg = args[i]; 1765 if (dead_temps[arg]) { 1766 dead_args |= (1 << i); 1767 } 1768 dead_temps[arg] = 0; 1769 } 1770 s->op_dead_args[op_index] = dead_args; 1771 s->op_sync_args[op_index] = sync_args; 1772 } 1773 break; 1774 } 1775 op_index--; 1776 } 1777 1778 if (args != s->gen_opparam_buf) { 1779 tcg_abort(); 1780 } 1781 } 1782 #else 1783 /* dummy liveness analysis */ 1784 static void tcg_liveness_analysis(TCGContext *s) 1785 { 1786 int nb_ops; 1787 nb_ops = s->gen_opc_ptr - s->gen_opc_buf; 1788 1789 s->op_dead_args = tcg_malloc(nb_ops * sizeof(uint16_t)); 1790 memset(s->op_dead_args, 0, nb_ops * sizeof(uint16_t)); 1791 s->op_sync_args = tcg_malloc(nb_ops * sizeof(uint8_t)); 1792 memset(s->op_sync_args, 0, nb_ops * sizeof(uint8_t)); 1793 } 1794 #endif 1795 1796 #ifndef NDEBUG 1797 static void dump_regs(TCGContext *s) 1798 { 1799 TCGTemp *ts; 1800 int i; 1801 char buf[64]; 1802 1803 for(i = 0; i < s->nb_temps; i++) { 1804 ts = &s->temps[i]; 1805 printf(" %10s: ", tcg_get_arg_str_idx(s, buf, sizeof(buf), i)); 1806 switch(ts->val_type) { 1807 case TEMP_VAL_REG: 1808 printf("%s", tcg_target_reg_names[ts->reg]); 1809 break; 1810 case TEMP_VAL_MEM: 1811 printf("%d(%s)", (int)ts->mem_offset, tcg_target_reg_names[ts->mem_reg]); 1812 break; 1813 case TEMP_VAL_CONST: 1814 printf("$0x%" TCG_PRIlx, ts->val); 1815 break; 1816 case TEMP_VAL_DEAD: 1817 printf("D"); 1818 break; 1819 default: 1820 printf("???"); 1821 break; 1822 } 1823 printf("\n"); 1824 } 1825 1826 for(i = 0; i < TCG_TARGET_NB_REGS; i++) { 1827 if (s->reg_to_temp[i] >= 0) { 1828 printf("%s: %s\n", 1829 tcg_target_reg_names[i], 1830 tcg_get_arg_str_idx(s, buf, sizeof(buf), s->reg_to_temp[i])); 1831 } 1832 } 1833 } 1834 1835 static void check_regs(TCGContext *s) 1836 { 1837 int reg, k; 1838 TCGTemp *ts; 1839 char buf[64]; 1840 1841 for(reg = 0; reg < TCG_TARGET_NB_REGS; reg++) { 1842 k = s->reg_to_temp[reg]; 1843 if (k >= 0) { 1844 ts = &s->temps[k]; 1845 if (ts->val_type != TEMP_VAL_REG || 1846 ts->reg != reg) { 1847 printf("Inconsistency for register %s:\n", 1848 tcg_target_reg_names[reg]); 1849 goto fail; 1850 } 1851 } 1852 } 1853 for(k = 0; k < s->nb_temps; k++) { 1854 ts = &s->temps[k]; 1855 if (ts->val_type == TEMP_VAL_REG && 1856 !ts->fixed_reg && 1857 s->reg_to_temp[ts->reg] != k) { 1858 printf("Inconsistency for temp %s:\n", 1859 tcg_get_arg_str_idx(s, buf, sizeof(buf), k)); 1860 fail: 1861 printf("reg state:\n"); 1862 dump_regs(s); 1863 tcg_abort(); 1864 } 1865 } 1866 } 1867 #endif 1868 1869 static void temp_allocate_frame(TCGContext *s, int temp) 1870 { 1871 TCGTemp *ts; 1872 ts = &s->temps[temp]; 1873 #if !(defined(__sparc__) && TCG_TARGET_REG_BITS == 64) 1874 /* Sparc64 stack is accessed with offset of 2047 */ 1875 s->current_frame_offset = (s->current_frame_offset + 1876 (tcg_target_long)sizeof(tcg_target_long) - 1) & 1877 ~(sizeof(tcg_target_long) - 1); 1878 #endif 1879 if (s->current_frame_offset + (tcg_target_long)sizeof(tcg_target_long) > 1880 s->frame_end) { 1881 tcg_abort(); 1882 } 1883 ts->mem_offset = s->current_frame_offset; 1884 ts->mem_reg = s->frame_reg; 1885 ts->mem_allocated = 1; 1886 s->current_frame_offset += sizeof(tcg_target_long); 1887 } 1888 1889 /* sync register 'reg' by saving it to the corresponding temporary */ 1890 static inline void tcg_reg_sync(TCGContext *s, int reg) 1891 { 1892 TCGTemp *ts; 1893 int temp; 1894 1895 temp = s->reg_to_temp[reg]; 1896 ts = &s->temps[temp]; 1897 assert(ts->val_type == TEMP_VAL_REG); 1898 if (!ts->mem_coherent && !ts->fixed_reg) { 1899 if (!ts->mem_allocated) { 1900 temp_allocate_frame(s, temp); 1901 } 1902 tcg_out_st(s, ts->type, reg, ts->mem_reg, ts->mem_offset); 1903 } 1904 ts->mem_coherent = 1; 1905 } 1906 1907 /* free register 'reg' by spilling the corresponding temporary if necessary */ 1908 static void tcg_reg_free(TCGContext *s, int reg) 1909 { 1910 int temp; 1911 1912 temp = s->reg_to_temp[reg]; 1913 if (temp != -1) { 1914 tcg_reg_sync(s, reg); 1915 s->temps[temp].val_type = TEMP_VAL_MEM; 1916 s->reg_to_temp[reg] = -1; 1917 } 1918 } 1919 1920 /* Allocate a register belonging to reg1 & ~reg2 */ 1921 static int tcg_reg_alloc(TCGContext *s, TCGRegSet reg1, TCGRegSet reg2) 1922 { 1923 int i, reg; 1924 TCGRegSet reg_ct; 1925 1926 tcg_regset_andnot(reg_ct, reg1, reg2); 1927 1928 /* first try free registers */ 1929 for(i = 0; i < ARRAY_SIZE(tcg_target_reg_alloc_order); i++) { 1930 reg = tcg_target_reg_alloc_order[i]; 1931 if (tcg_regset_test_reg(reg_ct, reg) && s->reg_to_temp[reg] == -1) 1932 return reg; 1933 } 1934 1935 /* XXX: do better spill choice */ 1936 for(i = 0; i < ARRAY_SIZE(tcg_target_reg_alloc_order); i++) { 1937 reg = tcg_target_reg_alloc_order[i]; 1938 if (tcg_regset_test_reg(reg_ct, reg)) { 1939 tcg_reg_free(s, reg); 1940 return reg; 1941 } 1942 } 1943 1944 tcg_abort(); 1945 } 1946 1947 /* mark a temporary as dead. */ 1948 static inline void temp_dead(TCGContext *s, int temp) 1949 { 1950 TCGTemp *ts; 1951 1952 ts = &s->temps[temp]; 1953 if (!ts->fixed_reg) { 1954 if (ts->val_type == TEMP_VAL_REG) { 1955 s->reg_to_temp[ts->reg] = -1; 1956 } 1957 if (temp < s->nb_globals || ts->temp_local) { 1958 ts->val_type = TEMP_VAL_MEM; 1959 } else { 1960 ts->val_type = TEMP_VAL_DEAD; 1961 } 1962 } 1963 } 1964 1965 /* sync a temporary to memory. 'allocated_regs' is used in case a 1966 temporary registers needs to be allocated to store a constant. */ 1967 static inline void temp_sync(TCGContext *s, int temp, TCGRegSet allocated_regs) 1968 { 1969 TCGTemp *ts; 1970 1971 ts = &s->temps[temp]; 1972 if (!ts->fixed_reg) { 1973 switch(ts->val_type) { 1974 case TEMP_VAL_CONST: 1975 ts->reg = tcg_reg_alloc(s, tcg_target_available_regs[ts->type], 1976 allocated_regs); 1977 ts->val_type = TEMP_VAL_REG; 1978 s->reg_to_temp[ts->reg] = temp; 1979 ts->mem_coherent = 0; 1980 tcg_out_movi(s, ts->type, ts->reg, ts->val); 1981 /* fallthrough*/ 1982 case TEMP_VAL_REG: 1983 tcg_reg_sync(s, ts->reg); 1984 break; 1985 case TEMP_VAL_DEAD: 1986 case TEMP_VAL_MEM: 1987 break; 1988 default: 1989 tcg_abort(); 1990 } 1991 } 1992 } 1993 1994 /* save a temporary to memory. 'allocated_regs' is used in case a 1995 temporary registers needs to be allocated to store a constant. */ 1996 static inline void temp_save(TCGContext *s, int temp, TCGRegSet allocated_regs) 1997 { 1998 #ifdef USE_LIVENESS_ANALYSIS 1999 /* The liveness analysis already ensures that globals are back 2000 in memory. Keep an assert for safety. */ 2001 assert(s->temps[temp].val_type == TEMP_VAL_MEM || s->temps[temp].fixed_reg); 2002 #else 2003 temp_sync(s, temp, allocated_regs); 2004 temp_dead(s, temp); 2005 #endif 2006 } 2007 2008 /* save globals to their canonical location and assume they can be 2009 modified be the following code. 'allocated_regs' is used in case a 2010 temporary registers needs to be allocated to store a constant. */ 2011 static void save_globals(TCGContext *s, TCGRegSet allocated_regs) 2012 { 2013 int i; 2014 2015 for(i = 0; i < s->nb_globals; i++) { 2016 temp_save(s, i, allocated_regs); 2017 } 2018 } 2019 2020 /* sync globals to their canonical location and assume they can be 2021 read by the following code. 'allocated_regs' is used in case a 2022 temporary registers needs to be allocated to store a constant. */ 2023 static void sync_globals(TCGContext *s, TCGRegSet allocated_regs) 2024 { 2025 int i; 2026 2027 for (i = 0; i < s->nb_globals; i++) { 2028 #ifdef USE_LIVENESS_ANALYSIS 2029 assert(s->temps[i].val_type != TEMP_VAL_REG || s->temps[i].fixed_reg || 2030 s->temps[i].mem_coherent); 2031 #else 2032 temp_sync(s, i, allocated_regs); 2033 #endif 2034 } 2035 } 2036 2037 /* at the end of a basic block, we assume all temporaries are dead and 2038 all globals are stored at their canonical location. */ 2039 static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs) 2040 { 2041 TCGTemp *ts; 2042 int i; 2043 2044 for(i = s->nb_globals; i < s->nb_temps; i++) { 2045 ts = &s->temps[i]; 2046 if (ts->temp_local) { 2047 temp_save(s, i, allocated_regs); 2048 } else { 2049 #ifdef USE_LIVENESS_ANALYSIS 2050 /* The liveness analysis already ensures that temps are dead. 2051 Keep an assert for safety. */ 2052 assert(ts->val_type == TEMP_VAL_DEAD); 2053 #else 2054 temp_dead(s, i); 2055 #endif 2056 } 2057 } 2058 2059 save_globals(s, allocated_regs); 2060 } 2061 2062 #define IS_DEAD_ARG(n) ((dead_args >> (n)) & 1) 2063 #define NEED_SYNC_ARG(n) ((sync_args >> (n)) & 1) 2064 2065 static void tcg_reg_alloc_movi(TCGContext *s, const TCGArg *args, 2066 uint16_t dead_args, uint8_t sync_args) 2067 { 2068 TCGTemp *ots; 2069 tcg_target_ulong val; 2070 2071 ots = &s->temps[args[0]]; 2072 val = args[1]; 2073 2074 if (ots->fixed_reg) { 2075 /* for fixed registers, we do not do any constant 2076 propagation */ 2077 tcg_out_movi(s, ots->type, ots->reg, val); 2078 } else { 2079 /* The movi is not explicitly generated here */ 2080 if (ots->val_type == TEMP_VAL_REG) 2081 s->reg_to_temp[ots->reg] = -1; 2082 ots->val_type = TEMP_VAL_CONST; 2083 ots->val = val; 2084 } 2085 if (NEED_SYNC_ARG(0)) { 2086 temp_sync(s, args[0], s->reserved_regs); 2087 } 2088 if (IS_DEAD_ARG(0)) { 2089 temp_dead(s, args[0]); 2090 } 2091 } 2092 2093 static void tcg_reg_alloc_mov(TCGContext *s, const TCGOpDef *def, 2094 const TCGArg *args, uint16_t dead_args, 2095 uint8_t sync_args) 2096 { 2097 TCGRegSet allocated_regs; 2098 TCGTemp *ts, *ots; 2099 TCGType otype, itype; 2100 2101 tcg_regset_set(allocated_regs, s->reserved_regs); 2102 ots = &s->temps[args[0]]; 2103 ts = &s->temps[args[1]]; 2104 2105 /* Note that otype != itype for no-op truncation. */ 2106 otype = ots->type; 2107 itype = ts->type; 2108 2109 /* If the source value is not in a register, and we're going to be 2110 forced to have it in a register in order to perform the copy, 2111 then copy the SOURCE value into its own register first. That way 2112 we don't have to reload SOURCE the next time it is used. */ 2113 if (((NEED_SYNC_ARG(0) || ots->fixed_reg) && ts->val_type != TEMP_VAL_REG) 2114 || ts->val_type == TEMP_VAL_MEM) { 2115 ts->reg = tcg_reg_alloc(s, tcg_target_available_regs[itype], 2116 allocated_regs); 2117 if (ts->val_type == TEMP_VAL_MEM) { 2118 tcg_out_ld(s, itype, ts->reg, ts->mem_reg, ts->mem_offset); 2119 ts->mem_coherent = 1; 2120 } else if (ts->val_type == TEMP_VAL_CONST) { 2121 tcg_out_movi(s, itype, ts->reg, ts->val); 2122 } 2123 s->reg_to_temp[ts->reg] = args[1]; 2124 ts->val_type = TEMP_VAL_REG; 2125 } 2126 2127 if (IS_DEAD_ARG(0) && !ots->fixed_reg) { 2128 /* mov to a non-saved dead register makes no sense (even with 2129 liveness analysis disabled). */ 2130 assert(NEED_SYNC_ARG(0)); 2131 /* The code above should have moved the temp to a register. */ 2132 assert(ts->val_type == TEMP_VAL_REG); 2133 if (!ots->mem_allocated) { 2134 temp_allocate_frame(s, args[0]); 2135 } 2136 tcg_out_st(s, otype, ts->reg, ots->mem_reg, ots->mem_offset); 2137 if (IS_DEAD_ARG(1)) { 2138 temp_dead(s, args[1]); 2139 } 2140 temp_dead(s, args[0]); 2141 } else if (ts->val_type == TEMP_VAL_CONST) { 2142 /* propagate constant */ 2143 if (ots->val_type == TEMP_VAL_REG) { 2144 s->reg_to_temp[ots->reg] = -1; 2145 } 2146 ots->val_type = TEMP_VAL_CONST; 2147 ots->val = ts->val; 2148 } else { 2149 /* The code in the first if block should have moved the 2150 temp to a register. */ 2151 assert(ts->val_type == TEMP_VAL_REG); 2152 if (IS_DEAD_ARG(1) && !ts->fixed_reg && !ots->fixed_reg) { 2153 /* the mov can be suppressed */ 2154 if (ots->val_type == TEMP_VAL_REG) { 2155 s->reg_to_temp[ots->reg] = -1; 2156 } 2157 ots->reg = ts->reg; 2158 temp_dead(s, args[1]); 2159 } else { 2160 if (ots->val_type != TEMP_VAL_REG) { 2161 /* When allocating a new register, make sure to not spill the 2162 input one. */ 2163 tcg_regset_set_reg(allocated_regs, ts->reg); 2164 ots->reg = tcg_reg_alloc(s, tcg_target_available_regs[otype], 2165 allocated_regs); 2166 } 2167 tcg_out_mov(s, otype, ots->reg, ts->reg); 2168 } 2169 ots->val_type = TEMP_VAL_REG; 2170 ots->mem_coherent = 0; 2171 s->reg_to_temp[ots->reg] = args[0]; 2172 if (NEED_SYNC_ARG(0)) { 2173 tcg_reg_sync(s, ots->reg); 2174 } 2175 } 2176 } 2177 2178 static void tcg_reg_alloc_op(TCGContext *s, 2179 const TCGOpDef *def, TCGOpcode opc, 2180 const TCGArg *args, uint16_t dead_args, 2181 uint8_t sync_args) 2182 { 2183 TCGRegSet allocated_regs; 2184 int i, k, nb_iargs, nb_oargs, reg; 2185 TCGArg arg; 2186 const TCGArgConstraint *arg_ct; 2187 TCGTemp *ts; 2188 TCGArg new_args[TCG_MAX_OP_ARGS]; 2189 int const_args[TCG_MAX_OP_ARGS]; 2190 2191 nb_oargs = def->nb_oargs; 2192 nb_iargs = def->nb_iargs; 2193 2194 /* copy constants */ 2195 memcpy(new_args + nb_oargs + nb_iargs, 2196 args + nb_oargs + nb_iargs, 2197 sizeof(TCGArg) * def->nb_cargs); 2198 2199 /* satisfy input constraints */ 2200 tcg_regset_set(allocated_regs, s->reserved_regs); 2201 for(k = 0; k < nb_iargs; k++) { 2202 i = def->sorted_args[nb_oargs + k]; 2203 arg = args[i]; 2204 arg_ct = &def->args_ct[i]; 2205 ts = &s->temps[arg]; 2206 if (ts->val_type == TEMP_VAL_MEM) { 2207 reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs); 2208 tcg_out_ld(s, ts->type, reg, ts->mem_reg, ts->mem_offset); 2209 ts->val_type = TEMP_VAL_REG; 2210 ts->reg = reg; 2211 ts->mem_coherent = 1; 2212 s->reg_to_temp[reg] = arg; 2213 } else if (ts->val_type == TEMP_VAL_CONST) { 2214 if (tcg_target_const_match(ts->val, ts->type, arg_ct)) { 2215 /* constant is OK for instruction */ 2216 const_args[i] = 1; 2217 new_args[i] = ts->val; 2218 goto iarg_end; 2219 } else { 2220 /* need to move to a register */ 2221 reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs); 2222 tcg_out_movi(s, ts->type, reg, ts->val); 2223 ts->val_type = TEMP_VAL_REG; 2224 ts->reg = reg; 2225 ts->mem_coherent = 0; 2226 s->reg_to_temp[reg] = arg; 2227 } 2228 } 2229 assert(ts->val_type == TEMP_VAL_REG); 2230 if (arg_ct->ct & TCG_CT_IALIAS) { 2231 if (ts->fixed_reg) { 2232 /* if fixed register, we must allocate a new register 2233 if the alias is not the same register */ 2234 if (arg != args[arg_ct->alias_index]) 2235 goto allocate_in_reg; 2236 } else { 2237 /* if the input is aliased to an output and if it is 2238 not dead after the instruction, we must allocate 2239 a new register and move it */ 2240 if (!IS_DEAD_ARG(i)) { 2241 goto allocate_in_reg; 2242 } 2243 } 2244 } 2245 reg = ts->reg; 2246 if (tcg_regset_test_reg(arg_ct->u.regs, reg)) { 2247 /* nothing to do : the constraint is satisfied */ 2248 } else { 2249 allocate_in_reg: 2250 /* allocate a new register matching the constraint 2251 and move the temporary register into it */ 2252 reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs); 2253 tcg_out_mov(s, ts->type, reg, ts->reg); 2254 } 2255 new_args[i] = reg; 2256 const_args[i] = 0; 2257 tcg_regset_set_reg(allocated_regs, reg); 2258 iarg_end: ; 2259 } 2260 2261 /* mark dead temporaries and free the associated registers */ 2262 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { 2263 if (IS_DEAD_ARG(i)) { 2264 temp_dead(s, args[i]); 2265 } 2266 } 2267 2268 if (def->flags & TCG_OPF_BB_END) { 2269 tcg_reg_alloc_bb_end(s, allocated_regs); 2270 } else { 2271 if (def->flags & TCG_OPF_CALL_CLOBBER) { 2272 /* XXX: permit generic clobber register list ? */ 2273 for(reg = 0; reg < TCG_TARGET_NB_REGS; reg++) { 2274 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, reg)) { 2275 tcg_reg_free(s, reg); 2276 } 2277 } 2278 } 2279 if (def->flags & TCG_OPF_SIDE_EFFECTS) { 2280 /* sync globals if the op has side effects and might trigger 2281 an exception. */ 2282 sync_globals(s, allocated_regs); 2283 } 2284 2285 /* satisfy the output constraints */ 2286 tcg_regset_set(allocated_regs, s->reserved_regs); 2287 for(k = 0; k < nb_oargs; k++) { 2288 i = def->sorted_args[k]; 2289 arg = args[i]; 2290 arg_ct = &def->args_ct[i]; 2291 ts = &s->temps[arg]; 2292 if (arg_ct->ct & TCG_CT_ALIAS) { 2293 reg = new_args[arg_ct->alias_index]; 2294 } else { 2295 /* if fixed register, we try to use it */ 2296 reg = ts->reg; 2297 if (ts->fixed_reg && 2298 tcg_regset_test_reg(arg_ct->u.regs, reg)) { 2299 goto oarg_end; 2300 } 2301 reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs); 2302 } 2303 tcg_regset_set_reg(allocated_regs, reg); 2304 /* if a fixed register is used, then a move will be done afterwards */ 2305 if (!ts->fixed_reg) { 2306 if (ts->val_type == TEMP_VAL_REG) { 2307 s->reg_to_temp[ts->reg] = -1; 2308 } 2309 ts->val_type = TEMP_VAL_REG; 2310 ts->reg = reg; 2311 /* temp value is modified, so the value kept in memory is 2312 potentially not the same */ 2313 ts->mem_coherent = 0; 2314 s->reg_to_temp[reg] = arg; 2315 } 2316 oarg_end: 2317 new_args[i] = reg; 2318 } 2319 } 2320 2321 /* emit instruction */ 2322 tcg_out_op(s, opc, new_args, const_args); 2323 2324 /* move the outputs in the correct register if needed */ 2325 for(i = 0; i < nb_oargs; i++) { 2326 ts = &s->temps[args[i]]; 2327 reg = new_args[i]; 2328 if (ts->fixed_reg && ts->reg != reg) { 2329 tcg_out_mov(s, ts->type, ts->reg, reg); 2330 } 2331 if (NEED_SYNC_ARG(i)) { 2332 tcg_reg_sync(s, reg); 2333 } 2334 if (IS_DEAD_ARG(i)) { 2335 temp_dead(s, args[i]); 2336 } 2337 } 2338 } 2339 2340 #ifdef TCG_TARGET_STACK_GROWSUP 2341 #define STACK_DIR(x) (-(x)) 2342 #else 2343 #define STACK_DIR(x) (x) 2344 #endif 2345 2346 static int tcg_reg_alloc_call(TCGContext *s, const TCGOpDef *def, 2347 TCGOpcode opc, const TCGArg *args, 2348 uint16_t dead_args, uint8_t sync_args) 2349 { 2350 int nb_iargs, nb_oargs, flags, nb_regs, i, reg, nb_params; 2351 TCGArg arg; 2352 TCGTemp *ts; 2353 intptr_t stack_offset; 2354 size_t call_stack_size; 2355 tcg_insn_unit *func_addr; 2356 int allocate_args; 2357 TCGRegSet allocated_regs; 2358 2359 arg = *args++; 2360 2361 nb_oargs = arg >> 16; 2362 nb_iargs = arg & 0xffff; 2363 nb_params = nb_iargs; 2364 2365 func_addr = (tcg_insn_unit *)(intptr_t)args[nb_oargs + nb_iargs]; 2366 flags = args[nb_oargs + nb_iargs + 1]; 2367 2368 nb_regs = ARRAY_SIZE(tcg_target_call_iarg_regs); 2369 if (nb_regs > nb_params) { 2370 nb_regs = nb_params; 2371 } 2372 2373 /* assign stack slots first */ 2374 call_stack_size = (nb_params - nb_regs) * sizeof(tcg_target_long); 2375 call_stack_size = (call_stack_size + TCG_TARGET_STACK_ALIGN - 1) & 2376 ~(TCG_TARGET_STACK_ALIGN - 1); 2377 allocate_args = (call_stack_size > TCG_STATIC_CALL_ARGS_SIZE); 2378 if (allocate_args) { 2379 /* XXX: if more than TCG_STATIC_CALL_ARGS_SIZE is needed, 2380 preallocate call stack */ 2381 tcg_abort(); 2382 } 2383 2384 stack_offset = TCG_TARGET_CALL_STACK_OFFSET; 2385 for(i = nb_regs; i < nb_params; i++) { 2386 arg = args[nb_oargs + i]; 2387 #ifdef TCG_TARGET_STACK_GROWSUP 2388 stack_offset -= sizeof(tcg_target_long); 2389 #endif 2390 if (arg != TCG_CALL_DUMMY_ARG) { 2391 ts = &s->temps[arg]; 2392 if (ts->val_type == TEMP_VAL_REG) { 2393 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, stack_offset); 2394 } else if (ts->val_type == TEMP_VAL_MEM) { 2395 reg = tcg_reg_alloc(s, tcg_target_available_regs[ts->type], 2396 s->reserved_regs); 2397 /* XXX: not correct if reading values from the stack */ 2398 tcg_out_ld(s, ts->type, reg, ts->mem_reg, ts->mem_offset); 2399 tcg_out_st(s, ts->type, reg, TCG_REG_CALL_STACK, stack_offset); 2400 } else if (ts->val_type == TEMP_VAL_CONST) { 2401 reg = tcg_reg_alloc(s, tcg_target_available_regs[ts->type], 2402 s->reserved_regs); 2403 /* XXX: sign extend may be needed on some targets */ 2404 tcg_out_movi(s, ts->type, reg, ts->val); 2405 tcg_out_st(s, ts->type, reg, TCG_REG_CALL_STACK, stack_offset); 2406 } else { 2407 tcg_abort(); 2408 } 2409 } 2410 #ifndef TCG_TARGET_STACK_GROWSUP 2411 stack_offset += sizeof(tcg_target_long); 2412 #endif 2413 } 2414 2415 /* assign input registers */ 2416 tcg_regset_set(allocated_regs, s->reserved_regs); 2417 for(i = 0; i < nb_regs; i++) { 2418 arg = args[nb_oargs + i]; 2419 if (arg != TCG_CALL_DUMMY_ARG) { 2420 ts = &s->temps[arg]; 2421 reg = tcg_target_call_iarg_regs[i]; 2422 tcg_reg_free(s, reg); 2423 if (ts->val_type == TEMP_VAL_REG) { 2424 if (ts->reg != reg) { 2425 tcg_out_mov(s, ts->type, reg, ts->reg); 2426 } 2427 } else if (ts->val_type == TEMP_VAL_MEM) { 2428 tcg_out_ld(s, ts->type, reg, ts->mem_reg, ts->mem_offset); 2429 } else if (ts->val_type == TEMP_VAL_CONST) { 2430 /* XXX: sign extend ? */ 2431 tcg_out_movi(s, ts->type, reg, ts->val); 2432 } else { 2433 tcg_abort(); 2434 } 2435 tcg_regset_set_reg(allocated_regs, reg); 2436 } 2437 } 2438 2439 /* mark dead temporaries and free the associated registers */ 2440 for(i = nb_oargs; i < nb_iargs + nb_oargs; i++) { 2441 if (IS_DEAD_ARG(i)) { 2442 temp_dead(s, args[i]); 2443 } 2444 } 2445 2446 /* clobber call registers */ 2447 for(reg = 0; reg < TCG_TARGET_NB_REGS; reg++) { 2448 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, reg)) { 2449 tcg_reg_free(s, reg); 2450 } 2451 } 2452 2453 /* Save globals if they might be written by the helper, sync them if 2454 they might be read. */ 2455 if (flags & TCG_CALL_NO_READ_GLOBALS) { 2456 /* Nothing to do */ 2457 } else if (flags & TCG_CALL_NO_WRITE_GLOBALS) { 2458 sync_globals(s, allocated_regs); 2459 } else { 2460 save_globals(s, allocated_regs); 2461 } 2462 2463 tcg_out_call(s, func_addr); 2464 2465 /* assign output registers and emit moves if needed */ 2466 for(i = 0; i < nb_oargs; i++) { 2467 arg = args[i]; 2468 ts = &s->temps[arg]; 2469 reg = tcg_target_call_oarg_regs[i]; 2470 assert(s->reg_to_temp[reg] == -1); 2471 2472 if (ts->fixed_reg) { 2473 if (ts->reg != reg) { 2474 tcg_out_mov(s, ts->type, ts->reg, reg); 2475 } 2476 } else { 2477 if (ts->val_type == TEMP_VAL_REG) { 2478 s->reg_to_temp[ts->reg] = -1; 2479 } 2480 ts->val_type = TEMP_VAL_REG; 2481 ts->reg = reg; 2482 ts->mem_coherent = 0; 2483 s->reg_to_temp[reg] = arg; 2484 if (NEED_SYNC_ARG(i)) { 2485 tcg_reg_sync(s, reg); 2486 } 2487 if (IS_DEAD_ARG(i)) { 2488 temp_dead(s, args[i]); 2489 } 2490 } 2491 } 2492 2493 return nb_iargs + nb_oargs + def->nb_cargs + 1; 2494 } 2495 2496 #ifdef CONFIG_PROFILER 2497 2498 static int64_t tcg_table_op_count[NB_OPS]; 2499 2500 static void dump_op_count(void) 2501 { 2502 int i; 2503 FILE *f; 2504 f = fopen("/tmp/op.log", "w"); 2505 for(i = INDEX_op_end; i < NB_OPS; i++) { 2506 fprintf(f, "%s %" PRId64 "\n", tcg_op_defs[i].name, tcg_table_op_count[i]); 2507 } 2508 fclose(f); 2509 } 2510 #endif 2511 2512 2513 static inline int tcg_gen_code_common(TCGContext *s, 2514 tcg_insn_unit *gen_code_buf, 2515 long search_pc) 2516 { 2517 TCGOpcode opc; 2518 int op_index; 2519 const TCGOpDef *def; 2520 const TCGArg *args; 2521 2522 #ifdef DEBUG_DISAS 2523 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP))) { 2524 qemu_log("OP:\n"); 2525 tcg_dump_ops(s); 2526 qemu_log("\n"); 2527 } 2528 #endif 2529 2530 #ifdef CONFIG_PROFILER 2531 s->opt_time -= profile_getclock(); 2532 #endif 2533 2534 #ifdef USE_TCG_OPTIMIZATIONS 2535 s->gen_opparam_ptr = 2536 tcg_optimize(s, s->gen_opc_ptr, s->gen_opparam_buf, tcg_op_defs); 2537 #endif 2538 2539 #ifdef CONFIG_PROFILER 2540 s->opt_time += profile_getclock(); 2541 s->la_time -= profile_getclock(); 2542 #endif 2543 2544 tcg_liveness_analysis(s); 2545 2546 #ifdef CONFIG_PROFILER 2547 s->la_time += profile_getclock(); 2548 #endif 2549 2550 #ifdef DEBUG_DISAS 2551 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT))) { 2552 qemu_log("OP after optimization and liveness analysis:\n"); 2553 tcg_dump_ops(s); 2554 qemu_log("\n"); 2555 } 2556 #endif 2557 2558 tcg_reg_alloc_start(s); 2559 2560 s->code_buf = gen_code_buf; 2561 s->code_ptr = gen_code_buf; 2562 2563 tcg_out_tb_init(s); 2564 2565 args = s->gen_opparam_buf; 2566 op_index = 0; 2567 2568 for(;;) { 2569 opc = s->gen_opc_buf[op_index]; 2570 #ifdef CONFIG_PROFILER 2571 tcg_table_op_count[opc]++; 2572 #endif 2573 def = &tcg_op_defs[opc]; 2574 #if 0 2575 printf("%s: %d %d %d\n", def->name, 2576 def->nb_oargs, def->nb_iargs, def->nb_cargs); 2577 // dump_regs(s); 2578 #endif 2579 switch(opc) { 2580 case INDEX_op_mov_i32: 2581 case INDEX_op_mov_i64: 2582 tcg_reg_alloc_mov(s, def, args, s->op_dead_args[op_index], 2583 s->op_sync_args[op_index]); 2584 break; 2585 case INDEX_op_movi_i32: 2586 case INDEX_op_movi_i64: 2587 tcg_reg_alloc_movi(s, args, s->op_dead_args[op_index], 2588 s->op_sync_args[op_index]); 2589 break; 2590 case INDEX_op_debug_insn_start: 2591 /* debug instruction */ 2592 break; 2593 case INDEX_op_nop: 2594 case INDEX_op_nop1: 2595 case INDEX_op_nop2: 2596 case INDEX_op_nop3: 2597 break; 2598 case INDEX_op_nopn: 2599 args += args[0]; 2600 goto next; 2601 case INDEX_op_discard: 2602 temp_dead(s, args[0]); 2603 break; 2604 case INDEX_op_set_label: 2605 tcg_reg_alloc_bb_end(s, s->reserved_regs); 2606 tcg_out_label(s, args[0], s->code_ptr); 2607 break; 2608 case INDEX_op_call: 2609 args += tcg_reg_alloc_call(s, def, opc, args, 2610 s->op_dead_args[op_index], 2611 s->op_sync_args[op_index]); 2612 goto next; 2613 case INDEX_op_end: 2614 goto the_end; 2615 default: 2616 /* Sanity check that we've not introduced any unhandled opcodes. */ 2617 if (def->flags & TCG_OPF_NOT_PRESENT) { 2618 tcg_abort(); 2619 } 2620 /* Note: in order to speed up the code, it would be much 2621 faster to have specialized register allocator functions for 2622 some common argument patterns */ 2623 tcg_reg_alloc_op(s, def, opc, args, s->op_dead_args[op_index], 2624 s->op_sync_args[op_index]); 2625 break; 2626 } 2627 args += def->nb_args; 2628 next: 2629 if (search_pc >= 0 && search_pc < tcg_current_code_size(s)) { 2630 return op_index; 2631 } 2632 op_index++; 2633 #ifndef NDEBUG 2634 check_regs(s); 2635 #endif 2636 } 2637 the_end: 2638 /* Generate TB finalization at the end of block */ 2639 tcg_out_tb_finalize(s); 2640 return -1; 2641 } 2642 2643 int tcg_gen_code(TCGContext *s, tcg_insn_unit *gen_code_buf) 2644 { 2645 #ifdef CONFIG_PROFILER 2646 { 2647 int n; 2648 n = (s->gen_opc_ptr - s->gen_opc_buf); 2649 s->op_count += n; 2650 if (n > s->op_count_max) 2651 s->op_count_max = n; 2652 2653 s->temp_count += s->nb_temps; 2654 if (s->nb_temps > s->temp_count_max) 2655 s->temp_count_max = s->nb_temps; 2656 } 2657 #endif 2658 2659 tcg_gen_code_common(s, gen_code_buf, -1); 2660 2661 /* flush instruction cache */ 2662 flush_icache_range((uintptr_t)s->code_buf, (uintptr_t)s->code_ptr); 2663 2664 return tcg_current_code_size(s); 2665 } 2666 2667 /* Return the index of the micro operation such as the pc after is < 2668 offset bytes from the start of the TB. The contents of gen_code_buf must 2669 not be changed, though writing the same values is ok. 2670 Return -1 if not found. */ 2671 int tcg_gen_code_search_pc(TCGContext *s, tcg_insn_unit *gen_code_buf, 2672 long offset) 2673 { 2674 return tcg_gen_code_common(s, gen_code_buf, offset); 2675 } 2676 2677 #ifdef CONFIG_PROFILER 2678 void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf) 2679 { 2680 TCGContext *s = &tcg_ctx; 2681 int64_t tot; 2682 2683 tot = s->interm_time + s->code_time; 2684 cpu_fprintf(f, "JIT cycles %" PRId64 " (%0.3f s at 2.4 GHz)\n", 2685 tot, tot / 2.4e9); 2686 cpu_fprintf(f, "translated TBs %" PRId64 " (aborted=%" PRId64 " %0.1f%%)\n", 2687 s->tb_count, 2688 s->tb_count1 - s->tb_count, 2689 s->tb_count1 ? (double)(s->tb_count1 - s->tb_count) / s->tb_count1 * 100.0 : 0); 2690 cpu_fprintf(f, "avg ops/TB %0.1f max=%d\n", 2691 s->tb_count ? (double)s->op_count / s->tb_count : 0, s->op_count_max); 2692 cpu_fprintf(f, "deleted ops/TB %0.2f\n", 2693 s->tb_count ? 2694 (double)s->del_op_count / s->tb_count : 0); 2695 cpu_fprintf(f, "avg temps/TB %0.2f max=%d\n", 2696 s->tb_count ? 2697 (double)s->temp_count / s->tb_count : 0, 2698 s->temp_count_max); 2699 2700 cpu_fprintf(f, "cycles/op %0.1f\n", 2701 s->op_count ? (double)tot / s->op_count : 0); 2702 cpu_fprintf(f, "cycles/in byte %0.1f\n", 2703 s->code_in_len ? (double)tot / s->code_in_len : 0); 2704 cpu_fprintf(f, "cycles/out byte %0.1f\n", 2705 s->code_out_len ? (double)tot / s->code_out_len : 0); 2706 if (tot == 0) 2707 tot = 1; 2708 cpu_fprintf(f, " gen_interm time %0.1f%%\n", 2709 (double)s->interm_time / tot * 100.0); 2710 cpu_fprintf(f, " gen_code time %0.1f%%\n", 2711 (double)s->code_time / tot * 100.0); 2712 cpu_fprintf(f, "optim./code time %0.1f%%\n", 2713 (double)s->opt_time / (s->code_time ? s->code_time : 1) 2714 * 100.0); 2715 cpu_fprintf(f, "liveness/code time %0.1f%%\n", 2716 (double)s->la_time / (s->code_time ? s->code_time : 1) * 100.0); 2717 cpu_fprintf(f, "cpu_restore count %" PRId64 "\n", 2718 s->restore_count); 2719 cpu_fprintf(f, " avg cycles %0.1f\n", 2720 s->restore_count ? (double)s->restore_time / s->restore_count : 0); 2721 2722 dump_op_count(); 2723 } 2724 #else 2725 void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf) 2726 { 2727 cpu_fprintf(f, "[TCG profiler not compiled]\n"); 2728 } 2729 #endif 2730 2731 #ifdef ELF_HOST_MACHINE 2732 /* In order to use this feature, the backend needs to do three things: 2733 2734 (1) Define ELF_HOST_MACHINE to indicate both what value to 2735 put into the ELF image and to indicate support for the feature. 2736 2737 (2) Define tcg_register_jit. This should create a buffer containing 2738 the contents of a .debug_frame section that describes the post- 2739 prologue unwind info for the tcg machine. 2740 2741 (3) Call tcg_register_jit_int, with the constructed .debug_frame. 2742 */ 2743 2744 /* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */ 2745 typedef enum { 2746 JIT_NOACTION = 0, 2747 JIT_REGISTER_FN, 2748 JIT_UNREGISTER_FN 2749 } jit_actions_t; 2750 2751 struct jit_code_entry { 2752 struct jit_code_entry *next_entry; 2753 struct jit_code_entry *prev_entry; 2754 const void *symfile_addr; 2755 uint64_t symfile_size; 2756 }; 2757 2758 struct jit_descriptor { 2759 uint32_t version; 2760 uint32_t action_flag; 2761 struct jit_code_entry *relevant_entry; 2762 struct jit_code_entry *first_entry; 2763 }; 2764 2765 void __jit_debug_register_code(void) __attribute__((noinline)); 2766 void __jit_debug_register_code(void) 2767 { 2768 asm(""); 2769 } 2770 2771 /* Must statically initialize the version, because GDB may check 2772 the version before we can set it. */ 2773 struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 }; 2774 2775 /* End GDB interface. */ 2776 2777 static int find_string(const char *strtab, const char *str) 2778 { 2779 const char *p = strtab + 1; 2780 2781 while (1) { 2782 if (strcmp(p, str) == 0) { 2783 return p - strtab; 2784 } 2785 p += strlen(p) + 1; 2786 } 2787 } 2788 2789 static void tcg_register_jit_int(void *buf_ptr, size_t buf_size, 2790 void *debug_frame, size_t debug_frame_size) 2791 { 2792 struct __attribute__((packed)) DebugInfo { 2793 uint32_t len; 2794 uint16_t version; 2795 uint32_t abbrev; 2796 uint8_t ptr_size; 2797 uint8_t cu_die; 2798 uint16_t cu_lang; 2799 uintptr_t cu_low_pc; 2800 uintptr_t cu_high_pc; 2801 uint8_t fn_die; 2802 char fn_name[16]; 2803 uintptr_t fn_low_pc; 2804 uintptr_t fn_high_pc; 2805 uint8_t cu_eoc; 2806 }; 2807 2808 struct ElfImage { 2809 ElfW(Ehdr) ehdr; 2810 ElfW(Phdr) phdr; 2811 ElfW(Shdr) shdr[7]; 2812 ElfW(Sym) sym[2]; 2813 struct DebugInfo di; 2814 uint8_t da[24]; 2815 char str[80]; 2816 }; 2817 2818 struct ElfImage *img; 2819 2820 static const struct ElfImage img_template = { 2821 .ehdr = { 2822 .e_ident[EI_MAG0] = ELFMAG0, 2823 .e_ident[EI_MAG1] = ELFMAG1, 2824 .e_ident[EI_MAG2] = ELFMAG2, 2825 .e_ident[EI_MAG3] = ELFMAG3, 2826 .e_ident[EI_CLASS] = ELF_CLASS, 2827 .e_ident[EI_DATA] = ELF_DATA, 2828 .e_ident[EI_VERSION] = EV_CURRENT, 2829 .e_type = ET_EXEC, 2830 .e_machine = ELF_HOST_MACHINE, 2831 .e_version = EV_CURRENT, 2832 .e_phoff = offsetof(struct ElfImage, phdr), 2833 .e_shoff = offsetof(struct ElfImage, shdr), 2834 .e_ehsize = sizeof(ElfW(Shdr)), 2835 .e_phentsize = sizeof(ElfW(Phdr)), 2836 .e_phnum = 1, 2837 .e_shentsize = sizeof(ElfW(Shdr)), 2838 .e_shnum = ARRAY_SIZE(img->shdr), 2839 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1, 2840 #ifdef ELF_HOST_FLAGS 2841 .e_flags = ELF_HOST_FLAGS, 2842 #endif 2843 #ifdef ELF_OSABI 2844 .e_ident[EI_OSABI] = ELF_OSABI, 2845 #endif 2846 }, 2847 .phdr = { 2848 .p_type = PT_LOAD, 2849 .p_flags = PF_X, 2850 }, 2851 .shdr = { 2852 [0] = { .sh_type = SHT_NULL }, 2853 /* Trick: The contents of code_gen_buffer are not present in 2854 this fake ELF file; that got allocated elsewhere. Therefore 2855 we mark .text as SHT_NOBITS (similar to .bss) so that readers 2856 will not look for contents. We can record any address. */ 2857 [1] = { /* .text */ 2858 .sh_type = SHT_NOBITS, 2859 .sh_flags = SHF_EXECINSTR | SHF_ALLOC, 2860 }, 2861 [2] = { /* .debug_info */ 2862 .sh_type = SHT_PROGBITS, 2863 .sh_offset = offsetof(struct ElfImage, di), 2864 .sh_size = sizeof(struct DebugInfo), 2865 }, 2866 [3] = { /* .debug_abbrev */ 2867 .sh_type = SHT_PROGBITS, 2868 .sh_offset = offsetof(struct ElfImage, da), 2869 .sh_size = sizeof(img->da), 2870 }, 2871 [4] = { /* .debug_frame */ 2872 .sh_type = SHT_PROGBITS, 2873 .sh_offset = sizeof(struct ElfImage), 2874 }, 2875 [5] = { /* .symtab */ 2876 .sh_type = SHT_SYMTAB, 2877 .sh_offset = offsetof(struct ElfImage, sym), 2878 .sh_size = sizeof(img->sym), 2879 .sh_info = 1, 2880 .sh_link = ARRAY_SIZE(img->shdr) - 1, 2881 .sh_entsize = sizeof(ElfW(Sym)), 2882 }, 2883 [6] = { /* .strtab */ 2884 .sh_type = SHT_STRTAB, 2885 .sh_offset = offsetof(struct ElfImage, str), 2886 .sh_size = sizeof(img->str), 2887 } 2888 }, 2889 .sym = { 2890 [1] = { /* code_gen_buffer */ 2891 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC), 2892 .st_shndx = 1, 2893 } 2894 }, 2895 .di = { 2896 .len = sizeof(struct DebugInfo) - 4, 2897 .version = 2, 2898 .ptr_size = sizeof(void *), 2899 .cu_die = 1, 2900 .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */ 2901 .fn_die = 2, 2902 .fn_name = "code_gen_buffer" 2903 }, 2904 .da = { 2905 1, /* abbrev number (the cu) */ 2906 0x11, 1, /* DW_TAG_compile_unit, has children */ 2907 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */ 2908 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 2909 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 2910 0, 0, /* end of abbrev */ 2911 2, /* abbrev number (the fn) */ 2912 0x2e, 0, /* DW_TAG_subprogram, no children */ 2913 0x3, 0x8, /* DW_AT_name, DW_FORM_string */ 2914 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */ 2915 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */ 2916 0, 0, /* end of abbrev */ 2917 0 /* no more abbrev */ 2918 }, 2919 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0" 2920 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer", 2921 }; 2922 2923 /* We only need a single jit entry; statically allocate it. */ 2924 static struct jit_code_entry one_entry; 2925 2926 uintptr_t buf = (uintptr_t)buf_ptr; 2927 size_t img_size = sizeof(struct ElfImage) + debug_frame_size; 2928 2929 img = g_malloc(img_size); 2930 *img = img_template; 2931 memcpy(img + 1, debug_frame, debug_frame_size); 2932 2933 img->phdr.p_vaddr = buf; 2934 img->phdr.p_paddr = buf; 2935 img->phdr.p_memsz = buf_size; 2936 2937 img->shdr[1].sh_name = find_string(img->str, ".text"); 2938 img->shdr[1].sh_addr = buf; 2939 img->shdr[1].sh_size = buf_size; 2940 2941 img->shdr[2].sh_name = find_string(img->str, ".debug_info"); 2942 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev"); 2943 2944 img->shdr[4].sh_name = find_string(img->str, ".debug_frame"); 2945 img->shdr[4].sh_size = debug_frame_size; 2946 2947 img->shdr[5].sh_name = find_string(img->str, ".symtab"); 2948 img->shdr[6].sh_name = find_string(img->str, ".strtab"); 2949 2950 img->sym[1].st_name = find_string(img->str, "code_gen_buffer"); 2951 img->sym[1].st_value = buf; 2952 img->sym[1].st_size = buf_size; 2953 2954 img->di.cu_low_pc = buf; 2955 img->di.cu_high_pc = buf + buf_size; 2956 img->di.fn_low_pc = buf; 2957 img->di.fn_high_pc = buf + buf_size; 2958 2959 #ifdef DEBUG_JIT 2960 /* Enable this block to be able to debug the ELF image file creation. 2961 One can use readelf, objdump, or other inspection utilities. */ 2962 { 2963 FILE *f = fopen("/tmp/qemu.jit", "w+b"); 2964 if (f) { 2965 if (fwrite(img, img_size, 1, f) != img_size) { 2966 /* Avoid stupid unused return value warning for fwrite. */ 2967 } 2968 fclose(f); 2969 } 2970 } 2971 #endif 2972 2973 one_entry.symfile_addr = img; 2974 one_entry.symfile_size = img_size; 2975 2976 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN; 2977 __jit_debug_descriptor.relevant_entry = &one_entry; 2978 __jit_debug_descriptor.first_entry = &one_entry; 2979 __jit_debug_register_code(); 2980 } 2981 #else 2982 /* No support for the feature. Provide the entry point expected by exec.c, 2983 and implement the internal function we declared earlier. */ 2984 2985 static void tcg_register_jit_int(void *buf, size_t size, 2986 void *debug_frame, size_t debug_frame_size) 2987 { 2988 } 2989 2990 void tcg_register_jit(void *buf, size_t buf_size) 2991 { 2992 } 2993 #endif /* ELF_HOST_MACHINE */ 2994