1 /* 2 * Host code generation 3 * 4 * Copyright (c) 2003 Fabrice Bellard 5 * 6 * This library is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU Lesser General Public 8 * License as published by the Free Software Foundation; either 9 * version 2.1 of the License, or (at your option) any later version. 10 * 11 * This library is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 14 * Lesser General Public License for more details. 15 * 16 * You should have received a copy of the GNU Lesser General Public 17 * License along with this library; if not, see <http://www.gnu.org/licenses/>. 18 */ 19 20 #include "qemu/osdep.h" 21 22 #include "trace.h" 23 #include "disas/disas.h" 24 #include "tcg/tcg.h" 25 #include "exec/mmap-lock.h" 26 #include "tb-internal.h" 27 #include "tlb-bounds.h" 28 #include "exec/tb-flush.h" 29 #include "qemu/cacheinfo.h" 30 #include "qemu/target-info.h" 31 #include "exec/log.h" 32 #include "exec/icount.h" 33 #include "accel/tcg/cpu-ops.h" 34 #include "tb-jmp-cache.h" 35 #include "tb-hash.h" 36 #include "tb-context.h" 37 #include "tb-internal.h" 38 #include "internal-common.h" 39 #include "tcg/perf.h" 40 #include "tcg/insn-start-words.h" 41 42 TBContext tb_ctx; 43 44 /* 45 * Encode VAL as a signed leb128 sequence at P. 46 * Return P incremented past the encoded value. 47 */ 48 static uint8_t *encode_sleb128(uint8_t *p, int64_t val) 49 { 50 int more, byte; 51 52 do { 53 byte = val & 0x7f; 54 val >>= 7; 55 more = !((val == 0 && (byte & 0x40) == 0) 56 || (val == -1 && (byte & 0x40) != 0)); 57 if (more) { 58 byte |= 0x80; 59 } 60 *p++ = byte; 61 } while (more); 62 63 return p; 64 } 65 66 /* 67 * Decode a signed leb128 sequence at *PP; increment *PP past the 68 * decoded value. Return the decoded value. 69 */ 70 static int64_t decode_sleb128(const uint8_t **pp) 71 { 72 const uint8_t *p = *pp; 73 int64_t val = 0; 74 int byte, shift = 0; 75 76 do { 77 byte = *p++; 78 val |= (int64_t)(byte & 0x7f) << shift; 79 shift += 7; 80 } while (byte & 0x80); 81 if (shift < 64 && (byte & 0x40)) { 82 val |= -(int64_t)1 << shift; 83 } 84 85 *pp = p; 86 return val; 87 } 88 89 /* Encode the data collected about the instructions while compiling TB. 90 Place the data at BLOCK, and return the number of bytes consumed. 91 92 The logical table consists of INSN_START_WORDS uint64_t's, 93 which come from the target's insn_start data, followed by a uintptr_t 94 which comes from the host pc of the end of the code implementing the insn. 95 96 Each line of the table is encoded as sleb128 deltas from the previous 97 line. The seed for the first line is { tb->pc, 0..., tb->tc.ptr }. 98 That is, the first column is seeded with the guest pc, the last column 99 with the host pc, and the middle columns with zeros. */ 100 101 static int encode_search(TranslationBlock *tb, uint8_t *block) 102 { 103 uint8_t *highwater = tcg_ctx->code_gen_highwater; 104 uint64_t *insn_data = tcg_ctx->gen_insn_data; 105 uint16_t *insn_end_off = tcg_ctx->gen_insn_end_off; 106 uint8_t *p = block; 107 int i, j, n; 108 109 for (i = 0, n = tb->icount; i < n; ++i) { 110 uint64_t prev, curr; 111 112 for (j = 0; j < INSN_START_WORDS; ++j) { 113 if (i == 0) { 114 prev = (!(tb_cflags(tb) & CF_PCREL) && j == 0 ? tb->pc : 0); 115 } else { 116 prev = insn_data[(i - 1) * INSN_START_WORDS + j]; 117 } 118 curr = insn_data[i * INSN_START_WORDS + j]; 119 p = encode_sleb128(p, curr - prev); 120 } 121 prev = (i == 0 ? 0 : insn_end_off[i - 1]); 122 curr = insn_end_off[i]; 123 p = encode_sleb128(p, curr - prev); 124 125 /* Test for (pending) buffer overflow. The assumption is that any 126 one row beginning below the high water mark cannot overrun 127 the buffer completely. Thus we can test for overflow after 128 encoding a row without having to check during encoding. */ 129 if (unlikely(p > highwater)) { 130 return -1; 131 } 132 } 133 134 return p - block; 135 } 136 137 static int cpu_unwind_data_from_tb(TranslationBlock *tb, uintptr_t host_pc, 138 uint64_t *data) 139 { 140 uintptr_t iter_pc = (uintptr_t)tb->tc.ptr; 141 const uint8_t *p = tb->tc.ptr + tb->tc.size; 142 int i, j, num_insns = tb->icount; 143 144 host_pc -= GETPC_ADJ; 145 146 if (host_pc < iter_pc) { 147 return -1; 148 } 149 150 memset(data, 0, sizeof(uint64_t) * INSN_START_WORDS); 151 if (!(tb_cflags(tb) & CF_PCREL)) { 152 data[0] = tb->pc; 153 } 154 155 /* 156 * Reconstruct the stored insn data while looking for the point 157 * at which the end of the insn exceeds host_pc. 158 */ 159 for (i = 0; i < num_insns; ++i) { 160 for (j = 0; j < INSN_START_WORDS; ++j) { 161 data[j] += decode_sleb128(&p); 162 } 163 iter_pc += decode_sleb128(&p); 164 if (iter_pc > host_pc) { 165 return num_insns - i; 166 } 167 } 168 return -1; 169 } 170 171 /* 172 * The cpu state corresponding to 'host_pc' is restored in 173 * preparation for exiting the TB. 174 */ 175 void cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb, 176 uintptr_t host_pc) 177 { 178 uint64_t data[INSN_START_WORDS]; 179 int insns_left = cpu_unwind_data_from_tb(tb, host_pc, data); 180 181 if (insns_left < 0) { 182 return; 183 } 184 185 if (tb_cflags(tb) & CF_USE_ICOUNT) { 186 assert(icount_enabled()); 187 /* 188 * Reset the cycle counter to the start of the block and 189 * shift if to the number of actually executed instructions. 190 */ 191 cpu->neg.icount_decr.u16.low += insns_left; 192 } 193 194 cpu->cc->tcg_ops->restore_state_to_opc(cpu, tb, data); 195 } 196 197 bool cpu_restore_state(CPUState *cpu, uintptr_t host_pc) 198 { 199 /* 200 * The host_pc has to be in the rx region of the code buffer. 201 * If it is not we will not be able to resolve it here. 202 * The two cases where host_pc will not be correct are: 203 * 204 * - fault during translation (instruction fetch) 205 * - fault from helper (not using GETPC() macro) 206 * 207 * Either way we need return early as we can't resolve it here. 208 */ 209 if (in_code_gen_buffer((const void *)(host_pc - tcg_splitwx_diff))) { 210 TranslationBlock *tb = tcg_tb_lookup(host_pc); 211 if (tb) { 212 cpu_restore_state_from_tb(cpu, tb, host_pc); 213 return true; 214 } 215 } 216 return false; 217 } 218 219 bool cpu_unwind_state_data(CPUState *cpu, uintptr_t host_pc, uint64_t *data) 220 { 221 if (in_code_gen_buffer((const void *)(host_pc - tcg_splitwx_diff))) { 222 TranslationBlock *tb = tcg_tb_lookup(host_pc); 223 if (tb) { 224 return cpu_unwind_data_from_tb(tb, host_pc, data) >= 0; 225 } 226 } 227 return false; 228 } 229 230 void page_init(void) 231 { 232 page_table_config_init(); 233 } 234 235 /* 236 * Isolate the portion of code gen which can setjmp/longjmp. 237 * Return the size of the generated code, or negative on error. 238 */ 239 static int setjmp_gen_code(CPUArchState *env, TranslationBlock *tb, 240 vaddr pc, void *host_pc, 241 int *max_insns, int64_t *ti) 242 { 243 int ret = sigsetjmp(tcg_ctx->jmp_trans, 0); 244 if (unlikely(ret != 0)) { 245 return ret; 246 } 247 248 tcg_func_start(tcg_ctx); 249 250 CPUState *cs = env_cpu(env); 251 tcg_ctx->cpu = cs; 252 cs->cc->tcg_ops->translate_code(cs, tb, max_insns, pc, host_pc); 253 254 assert(tb->size != 0); 255 tcg_ctx->cpu = NULL; 256 *max_insns = tb->icount; 257 258 return tcg_gen_code(tcg_ctx, tb, pc); 259 } 260 261 /* Called with mmap_lock held for user mode emulation. */ 262 TranslationBlock *tb_gen_code(CPUState *cpu, TCGTBCPUState s) 263 { 264 CPUArchState *env = cpu_env(cpu); 265 TranslationBlock *tb, *existing_tb; 266 tb_page_addr_t phys_pc, phys_p2; 267 tcg_insn_unit *gen_code_buf; 268 int gen_code_size, search_size, max_insns; 269 int64_t ti; 270 void *host_pc; 271 272 assert_memory_lock(); 273 qemu_thread_jit_write(); 274 275 phys_pc = get_page_addr_code_hostp(env, s.pc, &host_pc); 276 277 if (phys_pc == -1) { 278 /* Generate a one-shot TB with 1 insn in it */ 279 s.cflags = (s.cflags & ~CF_COUNT_MASK) | 1; 280 } 281 282 max_insns = s.cflags & CF_COUNT_MASK; 283 if (max_insns == 0) { 284 max_insns = TCG_MAX_INSNS; 285 } 286 QEMU_BUILD_BUG_ON(CF_COUNT_MASK + 1 != TCG_MAX_INSNS); 287 288 buffer_overflow: 289 assert_no_pages_locked(); 290 tb = tcg_tb_alloc(tcg_ctx); 291 if (unlikely(!tb)) { 292 /* flush must be done */ 293 tb_flush(cpu); 294 mmap_unlock(); 295 /* Make the execution loop process the flush as soon as possible. */ 296 cpu->exception_index = EXCP_INTERRUPT; 297 cpu_loop_exit(cpu); 298 } 299 300 gen_code_buf = tcg_ctx->code_gen_ptr; 301 tb->tc.ptr = tcg_splitwx_to_rx(gen_code_buf); 302 if (!(s.cflags & CF_PCREL)) { 303 tb->pc = s.pc; 304 } 305 tb->cs_base = s.cs_base; 306 tb->flags = s.flags; 307 tb->cflags = s.cflags; 308 tb_set_page_addr0(tb, phys_pc); 309 tb_set_page_addr1(tb, -1); 310 if (phys_pc != -1) { 311 tb_lock_page0(phys_pc); 312 } 313 314 tcg_ctx->gen_tb = tb; 315 tcg_ctx->addr_type = target_long_bits() == 32 ? TCG_TYPE_I32 : TCG_TYPE_I64; 316 #ifdef CONFIG_SOFTMMU 317 tcg_ctx->page_bits = TARGET_PAGE_BITS; 318 tcg_ctx->page_mask = TARGET_PAGE_MASK; 319 tcg_ctx->tlb_dyn_max_bits = CPU_TLB_DYN_MAX_BITS; 320 #endif 321 tcg_ctx->guest_mo = cpu->cc->tcg_ops->guest_default_memory_order; 322 323 restart_translate: 324 trace_translate_block(tb, s.pc, tb->tc.ptr); 325 326 gen_code_size = setjmp_gen_code(env, tb, s.pc, host_pc, &max_insns, &ti); 327 if (unlikely(gen_code_size < 0)) { 328 switch (gen_code_size) { 329 case -1: 330 /* 331 * Overflow of code_gen_buffer, or the current slice of it. 332 * 333 * TODO: We don't need to re-do tcg_ops->translate_code, nor 334 * should we re-do the tcg optimization currently hidden 335 * inside tcg_gen_code. All that should be required is to 336 * flush the TBs, allocate a new TB, re-initialize it per 337 * above, and re-do the actual code generation. 338 */ 339 qemu_log_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT, 340 "Restarting code generation for " 341 "code_gen_buffer overflow\n"); 342 tb_unlock_pages(tb); 343 tcg_ctx->gen_tb = NULL; 344 goto buffer_overflow; 345 346 case -2: 347 /* 348 * The code generated for the TranslationBlock is too large. 349 * The maximum size allowed by the unwind info is 64k. 350 * There may be stricter constraints from relocations 351 * in the tcg backend. 352 * 353 * Try again with half as many insns as we attempted this time. 354 * If a single insn overflows, there's a bug somewhere... 355 */ 356 assert(max_insns > 1); 357 max_insns /= 2; 358 qemu_log_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT, 359 "Restarting code generation with " 360 "smaller translation block (max %d insns)\n", 361 max_insns); 362 363 /* 364 * The half-sized TB may not cross pages. 365 * TODO: Fix all targets that cross pages except with 366 * the first insn, at which point this can't be reached. 367 */ 368 phys_p2 = tb_page_addr1(tb); 369 if (unlikely(phys_p2 != -1)) { 370 tb_unlock_page1(phys_pc, phys_p2); 371 tb_set_page_addr1(tb, -1); 372 } 373 goto restart_translate; 374 375 case -3: 376 /* 377 * We had a page lock ordering problem. In order to avoid 378 * deadlock we had to drop the lock on page0, which means 379 * that everything we translated so far is compromised. 380 * Restart with locks held on both pages. 381 */ 382 qemu_log_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT, 383 "Restarting code generation with re-locked pages"); 384 goto restart_translate; 385 386 default: 387 g_assert_not_reached(); 388 } 389 } 390 tcg_ctx->gen_tb = NULL; 391 392 search_size = encode_search(tb, (void *)gen_code_buf + gen_code_size); 393 if (unlikely(search_size < 0)) { 394 tb_unlock_pages(tb); 395 goto buffer_overflow; 396 } 397 tb->tc.size = gen_code_size; 398 399 /* 400 * For CF_PCREL, attribute all executions of the generated code 401 * to its first mapping. 402 */ 403 perf_report_code(s.pc, tb, tcg_splitwx_to_rx(gen_code_buf)); 404 405 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM) && 406 qemu_log_in_addr_range(s.pc)) { 407 FILE *logfile = qemu_log_trylock(); 408 if (logfile) { 409 int code_size, data_size; 410 const tcg_target_ulong *rx_data_gen_ptr; 411 size_t chunk_start; 412 int insn = 0; 413 414 if (tcg_ctx->data_gen_ptr) { 415 rx_data_gen_ptr = tcg_splitwx_to_rx(tcg_ctx->data_gen_ptr); 416 code_size = (const void *)rx_data_gen_ptr - tb->tc.ptr; 417 data_size = gen_code_size - code_size; 418 } else { 419 rx_data_gen_ptr = 0; 420 code_size = gen_code_size; 421 data_size = 0; 422 } 423 424 /* Dump header and the first instruction */ 425 fprintf(logfile, "OUT: [size=%d]\n", gen_code_size); 426 fprintf(logfile, 427 " -- guest addr 0x%016" PRIx64 " + tb prologue\n", 428 tcg_ctx->gen_insn_data[insn * INSN_START_WORDS]); 429 chunk_start = tcg_ctx->gen_insn_end_off[insn]; 430 disas(logfile, tb->tc.ptr, chunk_start); 431 432 /* 433 * Dump each instruction chunk, wrapping up empty chunks into 434 * the next instruction. The whole array is offset so the 435 * first entry is the beginning of the 2nd instruction. 436 */ 437 while (insn < tb->icount) { 438 size_t chunk_end = tcg_ctx->gen_insn_end_off[insn]; 439 if (chunk_end > chunk_start) { 440 fprintf(logfile, " -- guest addr 0x%016" PRIx64 "\n", 441 tcg_ctx->gen_insn_data[insn * INSN_START_WORDS]); 442 disas(logfile, tb->tc.ptr + chunk_start, 443 chunk_end - chunk_start); 444 chunk_start = chunk_end; 445 } 446 insn++; 447 } 448 449 if (chunk_start < code_size) { 450 fprintf(logfile, " -- tb slow paths + alignment\n"); 451 disas(logfile, tb->tc.ptr + chunk_start, 452 code_size - chunk_start); 453 } 454 455 /* Finally dump any data we may have after the block */ 456 if (data_size) { 457 int i; 458 fprintf(logfile, " data: [size=%d]\n", data_size); 459 for (i = 0; i < data_size / sizeof(tcg_target_ulong); i++) { 460 if (sizeof(tcg_target_ulong) == 8) { 461 fprintf(logfile, 462 "0x%08" PRIxPTR ": .quad 0x%016" TCG_PRIlx "\n", 463 (uintptr_t)&rx_data_gen_ptr[i], rx_data_gen_ptr[i]); 464 } else if (sizeof(tcg_target_ulong) == 4) { 465 fprintf(logfile, 466 "0x%08" PRIxPTR ": .long 0x%08" TCG_PRIlx "\n", 467 (uintptr_t)&rx_data_gen_ptr[i], rx_data_gen_ptr[i]); 468 } else { 469 qemu_build_not_reached(); 470 } 471 } 472 } 473 fprintf(logfile, "\n"); 474 qemu_log_unlock(logfile); 475 } 476 } 477 478 qatomic_set(&tcg_ctx->code_gen_ptr, (void *) 479 ROUND_UP((uintptr_t)gen_code_buf + gen_code_size + search_size, 480 CODE_GEN_ALIGN)); 481 482 /* init jump list */ 483 qemu_spin_init(&tb->jmp_lock); 484 tb->jmp_list_head = (uintptr_t)NULL; 485 tb->jmp_list_next[0] = (uintptr_t)NULL; 486 tb->jmp_list_next[1] = (uintptr_t)NULL; 487 tb->jmp_dest[0] = (uintptr_t)NULL; 488 tb->jmp_dest[1] = (uintptr_t)NULL; 489 490 /* init original jump addresses which have been set during tcg_gen_code() */ 491 if (tb->jmp_reset_offset[0] != TB_JMP_OFFSET_INVALID) { 492 tb_reset_jump(tb, 0); 493 } 494 if (tb->jmp_reset_offset[1] != TB_JMP_OFFSET_INVALID) { 495 tb_reset_jump(tb, 1); 496 } 497 498 /* 499 * Insert TB into the corresponding region tree before publishing it 500 * through QHT. Otherwise rewinding happened in the TB might fail to 501 * lookup itself using host PC. 502 */ 503 tcg_tb_insert(tb); 504 505 /* 506 * If the TB is not associated with a physical RAM page then it must be 507 * a temporary one-insn TB. 508 * 509 * Such TBs must be added to region trees in order to make sure that 510 * restore_state_to_opc() - which on some architectures is not limited to 511 * rewinding, but also affects exception handling! - is called when such a 512 * TB causes an exception. 513 * 514 * At the same time, temporary one-insn TBs must be executed at most once, 515 * because subsequent reads from, e.g., I/O memory may return different 516 * values. So return early before attempting to link to other TBs or add 517 * to the QHT. 518 */ 519 if (tb_page_addr0(tb) == -1) { 520 assert_no_pages_locked(); 521 return tb; 522 } 523 524 /* 525 * No explicit memory barrier is required -- tb_link_page() makes the 526 * TB visible in a consistent state. 527 */ 528 existing_tb = tb_link_page(tb); 529 assert_no_pages_locked(); 530 531 /* if the TB already exists, discard what we just translated */ 532 if (unlikely(existing_tb != tb)) { 533 uintptr_t orig_aligned = (uintptr_t)gen_code_buf; 534 535 orig_aligned -= ROUND_UP(sizeof(*tb), qemu_icache_linesize); 536 qatomic_set(&tcg_ctx->code_gen_ptr, (void *)orig_aligned); 537 tcg_tb_remove(tb); 538 return existing_tb; 539 } 540 return tb; 541 } 542 543 /* user-mode: call with mmap_lock held */ 544 void tb_check_watchpoint(CPUState *cpu, uintptr_t retaddr) 545 { 546 TranslationBlock *tb; 547 548 assert_memory_lock(); 549 550 tb = tcg_tb_lookup(retaddr); 551 if (tb) { 552 /* We can use retranslation to find the PC. */ 553 cpu_restore_state_from_tb(cpu, tb, retaddr); 554 tb_phys_invalidate(tb, -1); 555 } else { 556 /* The exception probably happened in a helper. The CPU state should 557 have been saved before calling it. Fetch the PC from there. */ 558 CPUArchState *env = cpu_env(cpu); 559 TCGTBCPUState s = cpu->cc->tcg_ops->get_tb_cpu_state(cpu); 560 tb_page_addr_t addr = get_page_addr_code(env, s.pc); 561 562 if (addr != -1) { 563 tb_invalidate_phys_range(cpu, addr, addr); 564 } 565 } 566 } 567 568 #ifndef CONFIG_USER_ONLY 569 /* 570 * In deterministic execution mode, instructions doing device I/Os 571 * must be at the end of the TB. 572 * 573 * Called by softmmu_template.h, with iothread mutex not held. 574 */ 575 void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr) 576 { 577 TranslationBlock *tb; 578 CPUClass *cc; 579 uint32_t n; 580 581 tb = tcg_tb_lookup(retaddr); 582 if (!tb) { 583 cpu_abort(cpu, "cpu_io_recompile: could not find TB for pc=%p", 584 (void *)retaddr); 585 } 586 cpu_restore_state_from_tb(cpu, tb, retaddr); 587 588 /* 589 * Some guests must re-execute the branch when re-executing a delay 590 * slot instruction. When this is the case, adjust icount and N 591 * to account for the re-execution of the branch. 592 */ 593 n = 1; 594 cc = cpu->cc; 595 if (cc->tcg_ops->io_recompile_replay_branch && 596 cc->tcg_ops->io_recompile_replay_branch(cpu, tb)) { 597 cpu->neg.icount_decr.u16.low++; 598 n = 2; 599 } 600 601 /* 602 * Exit the loop and potentially generate a new TB executing the 603 * just the I/O insns. We also limit instrumentation to memory 604 * operations only (which execute after completion) so we don't 605 * double instrument the instruction. Also don't let an IRQ sneak 606 * in before we execute it. 607 */ 608 cpu->cflags_next_tb = curr_cflags(cpu) | CF_MEMI_ONLY | CF_NOIRQ | n; 609 610 if (qemu_loglevel_mask(CPU_LOG_EXEC)) { 611 vaddr pc = cpu->cc->get_pc(cpu); 612 if (qemu_log_in_addr_range(pc)) { 613 qemu_log("cpu_io_recompile: rewound execution of TB to %016" 614 VADDR_PRIx "\n", pc); 615 } 616 } 617 618 cpu_loop_exit_noexc(cpu); 619 } 620 621 #endif /* CONFIG_USER_ONLY */ 622 623 /* 624 * Called by generic code at e.g. cpu reset after cpu creation, 625 * therefore we must be prepared to allocate the jump cache. 626 */ 627 void tcg_flush_jmp_cache(CPUState *cpu) 628 { 629 CPUJumpCache *jc = cpu->tb_jmp_cache; 630 631 /* During early initialization, the cache may not yet be allocated. */ 632 if (unlikely(jc == NULL)) { 633 return; 634 } 635 636 for (int i = 0; i < TB_JMP_CACHE_SIZE; i++) { 637 qatomic_set(&jc->array[i].tb, NULL); 638 } 639 } 640