xref: /qemu/accel/tcg/translator.c (revision fc524567087c2537b5103cdfc1d41e4f442892b6)
1bb2e0039SLluís Vilanova /*
2bb2e0039SLluís Vilanova  * Generic intermediate code generation.
3bb2e0039SLluís Vilanova  *
4bb2e0039SLluís Vilanova  * Copyright (C) 2016-2017 Lluís Vilanova <vilanova@ac.upc.edu>
5bb2e0039SLluís Vilanova  *
6bb2e0039SLluís Vilanova  * This work is licensed under the terms of the GNU GPL, version 2 or later.
7bb2e0039SLluís Vilanova  * See the COPYING file in the top-level directory.
8bb2e0039SLluís Vilanova  */
9bb2e0039SLluís Vilanova 
10bb2e0039SLluís Vilanova #include "qemu/osdep.h"
115c43a750SRichard Henderson #include "qemu/bswap.h"
12653c46daSRichard Henderson #include "qemu/log.h"
13bb2e0039SLluís Vilanova #include "qemu/error-report.h"
14*231a1c0fSPhilippe Mathieu-Daudé #include "accel/tcg/cpu-ldst-common.h"
15028119c8SRichard Henderson #include "accel/tcg/cpu-mmu-index.h"
1641fed3c9SRichard Henderson #include "exec/target_page.h"
17bb2e0039SLluís Vilanova #include "exec/translator.h"
186ba6f818SEmilio G. Cota #include "exec/plugin-gen.h"
19653c46daSRichard Henderson #include "tcg/tcg-op-common.h"
20aa6f138aSRichard Henderson #include "internal-common.h"
21962a145cSRichard Henderson #include "disas/disas.h"
223e6bfabfSPhilippe Mathieu-Daudé #include "tb-internal.h"
2356234233SRichard Henderson 
set_can_do_io(DisasContextBase * db,bool val)240ca41ccfSRichard Henderson static void set_can_do_io(DisasContextBase *db, bool val)
2556234233SRichard Henderson {
26464dacf6SRichard Henderson     QEMU_BUILD_BUG_ON(sizeof_field(CPUState, neg.can_do_io) != 1);
27ad75a51eSRichard Henderson     tcg_gen_st8_i32(tcg_constant_i32(val), tcg_env,
2841fed3c9SRichard Henderson                     offsetof(CPUState, neg.can_do_io) - sizeof(CPUState));
2956234233SRichard Henderson }
3056234233SRichard Henderson 
translator_io_start(DisasContextBase * db)31dfd1b812SRichard Henderson bool translator_io_start(DisasContextBase *db)
32dfd1b812SRichard Henderson {
33dfd1b812SRichard Henderson     /*
34dfd1b812SRichard Henderson      * Ensure that this instruction will be the last in the TB.
35dfd1b812SRichard Henderson      * The target may override this to something more forceful.
36dfd1b812SRichard Henderson      */
37dfd1b812SRichard Henderson     if (db->is_jmp == DISAS_NEXT) {
38dfd1b812SRichard Henderson         db->is_jmp = DISAS_TOO_MANY;
39dfd1b812SRichard Henderson     }
40dfd1b812SRichard Henderson     return true;
41dfd1b812SRichard Henderson }
42dfd1b812SRichard Henderson 
gen_tb_start(DisasContextBase * db,uint32_t cflags)430ca41ccfSRichard Henderson static TCGOp *gen_tb_start(DisasContextBase *db, uint32_t cflags)
4456234233SRichard Henderson {
45f47a90daSRichard Henderson     TCGv_i32 count = NULL;
4656234233SRichard Henderson     TCGOp *icount_start_insn = NULL;
4756234233SRichard Henderson 
48f47a90daSRichard Henderson     if ((cflags & CF_USE_ICOUNT) || !(cflags & CF_NOIRQ)) {
49f47a90daSRichard Henderson         count = tcg_temp_new_i32();
50ad75a51eSRichard Henderson         tcg_gen_ld_i32(count, tcg_env,
5141fed3c9SRichard Henderson                        offsetof(CPUState, neg.icount_decr.u32) -
5241fed3c9SRichard Henderson                        sizeof(CPUState));
53f47a90daSRichard Henderson     }
5456234233SRichard Henderson 
5556234233SRichard Henderson     if (cflags & CF_USE_ICOUNT) {
5656234233SRichard Henderson         /*
5756234233SRichard Henderson          * We emit a sub with a dummy immediate argument. Keep the insn index
5856234233SRichard Henderson          * of the sub so that we later (when we know the actual insn count)
5956234233SRichard Henderson          * can update the argument with the actual insn count.
6056234233SRichard Henderson          */
6156234233SRichard Henderson         tcg_gen_sub_i32(count, count, tcg_constant_i32(0));
6256234233SRichard Henderson         icount_start_insn = tcg_last_op();
6356234233SRichard Henderson     }
6456234233SRichard Henderson 
6556234233SRichard Henderson     /*
6656234233SRichard Henderson      * Emit the check against icount_decr.u32 to see if we should exit
6756234233SRichard Henderson      * unless we suppress the check with CF_NOIRQ. If we are using
6856234233SRichard Henderson      * icount and have suppressed interruption the higher level code
6956234233SRichard Henderson      * should have ensured we don't run more instructions than the
7056234233SRichard Henderson      * budget.
7156234233SRichard Henderson      */
7256234233SRichard Henderson     if (cflags & CF_NOIRQ) {
7356234233SRichard Henderson         tcg_ctx->exitreq_label = NULL;
7456234233SRichard Henderson     } else {
7556234233SRichard Henderson         tcg_ctx->exitreq_label = gen_new_label();
7656234233SRichard Henderson         tcg_gen_brcondi_i32(TCG_COND_LT, count, 0, tcg_ctx->exitreq_label);
7756234233SRichard Henderson     }
7856234233SRichard Henderson 
7956234233SRichard Henderson     if (cflags & CF_USE_ICOUNT) {
80ad75a51eSRichard Henderson         tcg_gen_st16_i32(count, tcg_env,
8141fed3c9SRichard Henderson                          offsetof(CPUState, neg.icount_decr.u16.low) -
8241fed3c9SRichard Henderson                          sizeof(CPUState));
8318a536f1SRichard Henderson     }
8418a536f1SRichard Henderson 
8556234233SRichard Henderson     return icount_start_insn;
8656234233SRichard Henderson }
8756234233SRichard Henderson 
gen_tb_end(const TranslationBlock * tb,uint32_t cflags,TCGOp * icount_start_insn,int num_insns)8856234233SRichard Henderson static void gen_tb_end(const TranslationBlock *tb, uint32_t cflags,
8956234233SRichard Henderson                        TCGOp *icount_start_insn, int num_insns)
9056234233SRichard Henderson {
9156234233SRichard Henderson     if (cflags & CF_USE_ICOUNT) {
9256234233SRichard Henderson         /*
9356234233SRichard Henderson          * Update the num_insn immediate parameter now that we know
9456234233SRichard Henderson          * the actual insn count.
9556234233SRichard Henderson          */
9656234233SRichard Henderson         tcg_set_insn_param(icount_start_insn, 2,
9756234233SRichard Henderson                            tcgv_i32_arg(tcg_constant_i32(num_insns)));
9856234233SRichard Henderson     }
9956234233SRichard Henderson 
10056234233SRichard Henderson     if (tcg_ctx->exitreq_label) {
10156234233SRichard Henderson         gen_set_label(tcg_ctx->exitreq_label);
10256234233SRichard Henderson         tcg_gen_exit_tb(tb, TB_EXIT_REQUESTED);
10356234233SRichard Henderson     }
10456234233SRichard Henderson }
10556234233SRichard Henderson 
translator_is_same_page(const DisasContextBase * db,vaddr addr)106a9ca97eaSPhilippe Mathieu-Daudé bool translator_is_same_page(const DisasContextBase *db, vaddr addr)
107a9ca97eaSPhilippe Mathieu-Daudé {
108a9ca97eaSPhilippe Mathieu-Daudé     return ((addr ^ db->pc_first) & TARGET_PAGE_MASK) == 0;
109a9ca97eaSPhilippe Mathieu-Daudé }
110a9ca97eaSPhilippe Mathieu-Daudé 
translator_use_goto_tb(DisasContextBase * db,vaddr dest)111b1c09220SAnton Johansson bool translator_use_goto_tb(DisasContextBase *db, vaddr dest)
112d3a2a1d8SRichard Henderson {
11384f15616SRichard Henderson     /* Suppress goto_tb if requested. */
11484f15616SRichard Henderson     if (tb_cflags(db->tb) & CF_NO_GOTO_TB) {
11584f15616SRichard Henderson         return false;
11684f15616SRichard Henderson     }
11784f15616SRichard Henderson 
118d3a2a1d8SRichard Henderson     /* Check for the dest on the same page as the start of the TB.  */
119a9ca97eaSPhilippe Mathieu-Daudé     return translator_is_same_page(db, dest);
120d3a2a1d8SRichard Henderson }
121d3a2a1d8SRichard Henderson 
translator_loop(CPUState * cpu,TranslationBlock * tb,int * max_insns,vaddr pc,void * host_pc,const TranslatorOps * ops,DisasContextBase * db)122597f9b2dSRichard Henderson void translator_loop(CPUState *cpu, TranslationBlock *tb, int *max_insns,
123b1c09220SAnton Johansson                      vaddr pc, void *host_pc, const TranslatorOps *ops,
124b1c09220SAnton Johansson                      DisasContextBase *db)
125bb2e0039SLluís Vilanova {
126d40c5c79SRichard Henderson     uint32_t cflags = tb_cflags(tb);
12756234233SRichard Henderson     TCGOp *icount_start_insn;
128dcd092a0SRichard Henderson     TCGOp *first_insn_start = NULL;
1296ba6f818SEmilio G. Cota     bool plugin_enabled;
130f9f1f56eSPavel Dovgalyuk 
131bb2e0039SLluís Vilanova     /* Initialize DisasContext */
132bb2e0039SLluís Vilanova     db->tb = tb;
133306c8721SRichard Henderson     db->pc_first = pc;
134306c8721SRichard Henderson     db->pc_next = pc;
135bb2e0039SLluís Vilanova     db->is_jmp = DISAS_NEXT;
136bb2e0039SLluís Vilanova     db->num_insns = 0;
137597f9b2dSRichard Henderson     db->max_insns = *max_insns;
138e7face70SRichard Henderson     db->insn_start = NULL;
139b3f05b8cSRichard Henderson     db->fake_insn = false;
14050627f1bSRichard Henderson     db->host_addr[0] = host_pc;
14150627f1bSRichard Henderson     db->host_addr[1] = NULL;
142ba3fb2a7SRichard Henderson     db->record_start = 0;
143ba3fb2a7SRichard Henderson     db->record_len = 0;
144028119c8SRichard Henderson     db->code_mmuidx = cpu_mmu_index(cpu, true);
14550627f1bSRichard Henderson 
146b542683dSEmilio G. Cota     ops->init_disas_context(db, cpu);
147bb2e0039SLluís Vilanova     tcg_debug_assert(db->is_jmp == DISAS_NEXT);  /* no early exit */
148bb2e0039SLluís Vilanova 
149bb2e0039SLluís Vilanova     /* Start translating.  */
1500ca41ccfSRichard Henderson     icount_start_insn = gen_tb_start(db, cflags);
151bb2e0039SLluís Vilanova     ops->tb_start(db, cpu);
152bb2e0039SLluís Vilanova     tcg_debug_assert(db->is_jmp == DISAS_NEXT);  /* no early exit */
153bb2e0039SLluís Vilanova 
154e5013259SRichard Henderson     plugin_enabled = plugin_gen_tb_start(cpu, db);
15528a4f0baSRichard Henderson     db->plugin_enabled = plugin_enabled;
1566ba6f818SEmilio G. Cota 
157bb2e0039SLluís Vilanova     while (true) {
1589b1890adSRichard Henderson         *max_insns = ++db->num_insns;
159bb2e0039SLluís Vilanova         ops->insn_start(db, cpu);
160e7face70SRichard Henderson         db->insn_start = tcg_last_op();
161dcd092a0SRichard Henderson         if (first_insn_start == NULL) {
162dcd092a0SRichard Henderson             first_insn_start = db->insn_start;
163dcd092a0SRichard Henderson         }
164bb2e0039SLluís Vilanova         tcg_debug_assert(db->is_jmp == DISAS_NEXT);  /* no early exit */
165bb2e0039SLluís Vilanova 
1666ba6f818SEmilio G. Cota         if (plugin_enabled) {
1676ba6f818SEmilio G. Cota             plugin_gen_insn_start(cpu, db);
1686ba6f818SEmilio G. Cota         }
1696ba6f818SEmilio G. Cota 
170cf9b5790SRichard Henderson         /*
171cf9b5790SRichard Henderson          * Disassemble one instruction.  The translate_insn hook should
172cf9b5790SRichard Henderson          * update db->pc_next and db->is_jmp to indicate what should be
173cf9b5790SRichard Henderson          * done next -- either exiting this loop or locate the start of
174cf9b5790SRichard Henderson          * the next instruction.
175cf9b5790SRichard Henderson          */
1765d97e946SRichard Henderson         ops->translate_insn(db, cpu);
177bb2e0039SLluís Vilanova 
1786ba6f818SEmilio G. Cota         /*
1796ba6f818SEmilio G. Cota          * We can't instrument after instructions that change control
1806ba6f818SEmilio G. Cota          * flow although this only really affects post-load operations.
1810f92d94aSEmilio Cota          *
1820f92d94aSEmilio Cota          * Calling plugin_gen_insn_end() before we possibly stop translation
1830f92d94aSEmilio Cota          * is important. Even if this ends up as dead code, plugin generation
1840f92d94aSEmilio Cota          * needs to see a matching plugin_gen_insn_{start,end}() pair in order
1850f92d94aSEmilio Cota          * to accurately track instrumented helpers that might access memory.
1866ba6f818SEmilio G. Cota          */
1876ba6f818SEmilio G. Cota         if (plugin_enabled) {
1886ba6f818SEmilio G. Cota             plugin_gen_insn_end();
1896ba6f818SEmilio G. Cota         }
1906ba6f818SEmilio G. Cota 
1910f92d94aSEmilio Cota         /* Stop translation if translate_insn so indicated.  */
1920f92d94aSEmilio Cota         if (db->is_jmp != DISAS_NEXT) {
1930f92d94aSEmilio Cota             break;
1940f92d94aSEmilio Cota         }
1950f92d94aSEmilio Cota 
196bb2e0039SLluís Vilanova         /* Stop translation if the output buffer is full,
197bb2e0039SLluís Vilanova            or we have executed all of the allowed instructions.  */
198b542683dSEmilio G. Cota         if (tcg_op_buf_full() || db->num_insns >= db->max_insns) {
199bb2e0039SLluís Vilanova             db->is_jmp = DISAS_TOO_MANY;
200bb2e0039SLluís Vilanova             break;
201bb2e0039SLluís Vilanova         }
202bb2e0039SLluís Vilanova     }
203bb2e0039SLluís Vilanova 
204bb2e0039SLluís Vilanova     /* Emit code to exit the TB, as indicated by db->is_jmp.  */
205bb2e0039SLluís Vilanova     ops->tb_stop(db, cpu);
20656234233SRichard Henderson     gen_tb_end(tb, cflags, icount_start_insn, db->num_insns);
207bb2e0039SLluís Vilanova 
208dcd092a0SRichard Henderson     /*
209dcd092a0SRichard Henderson      * Manage can_do_io for the translation block: set to false before
210dcd092a0SRichard Henderson      * the first insn and set to true before the last insn.
211dcd092a0SRichard Henderson      */
212dcd092a0SRichard Henderson     if (db->num_insns == 1) {
213dcd092a0SRichard Henderson         tcg_debug_assert(first_insn_start == db->insn_start);
214dcd092a0SRichard Henderson     } else {
215dcd092a0SRichard Henderson         tcg_debug_assert(first_insn_start != db->insn_start);
216dcd092a0SRichard Henderson         tcg_ctx->emit_before_op = first_insn_start;
217dcd092a0SRichard Henderson         set_can_do_io(db, false);
218dcd092a0SRichard Henderson     }
219dcd092a0SRichard Henderson     tcg_ctx->emit_before_op = db->insn_start;
220dcd092a0SRichard Henderson     set_can_do_io(db, true);
221dcd092a0SRichard Henderson     tcg_ctx->emit_before_op = NULL;
222dcd092a0SRichard Henderson 
223bfd43cccSRichard Henderson     /* May be used by disas_log or plugin callbacks. */
224bfd43cccSRichard Henderson     tb->size = db->pc_next - db->pc_first;
225bfd43cccSRichard Henderson     tb->icount = db->num_insns;
226bfd43cccSRichard Henderson 
2276ba6f818SEmilio G. Cota     if (plugin_enabled) {
228a392277dSMatt Borgerson         plugin_gen_tb_end(cpu, db->num_insns);
2296ba6f818SEmilio G. Cota     }
2306ba6f818SEmilio G. Cota 
231bb2e0039SLluís Vilanova     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)
232bb2e0039SLluís Vilanova         && qemu_log_in_addr_range(db->pc_first)) {
233c60f599bSRichard Henderson         FILE *logfile = qemu_log_trylock();
23478b54858SRichard Henderson         if (logfile) {
23578b54858SRichard Henderson             fprintf(logfile, "----------------\n");
236962a145cSRichard Henderson 
237b67c567bSRichard Henderson             if (!ops->disas_log ||
238b67c567bSRichard Henderson                 !ops->disas_log(db, cpu, logfile)) {
239962a145cSRichard Henderson                 fprintf(logfile, "IN: %s\n", lookup_symbol(db->pc_first));
2404c833c60SRichard Henderson                 target_disas(logfile, cpu, db);
241962a145cSRichard Henderson             }
24278b54858SRichard Henderson             fprintf(logfile, "\n");
243fc59d2d8SRobert Foley             qemu_log_unlock(logfile);
244bb2e0039SLluís Vilanova         }
24578b54858SRichard Henderson     }
246bb2e0039SLluís Vilanova }
247f025692cSIlya Leoshkevich 
translator_ld(CPUArchState * env,DisasContextBase * db,void * dest,vaddr pc,size_t len)24804f8ed11SRichard Henderson static bool translator_ld(CPUArchState *env, DisasContextBase *db,
24904f8ed11SRichard Henderson                           void *dest, vaddr pc, size_t len)
250f025692cSIlya Leoshkevich {
25104f8ed11SRichard Henderson     TranslationBlock *tb = db->tb;
25204f8ed11SRichard Henderson     vaddr last = pc + len - 1;
25350627f1bSRichard Henderson     void *host;
25404f8ed11SRichard Henderson     vaddr base;
25550627f1bSRichard Henderson 
25650627f1bSRichard Henderson     /* Use slow path if first page is MMIO. */
25728905cfbSRichard Henderson     if (unlikely(tb_page_addr0(tb) == -1)) {
258ddfa9f11SRichard Henderson         /* We capped translation with first page MMIO in tb_gen_code. */
259ddfa9f11SRichard Henderson         tcg_debug_assert(db->max_insns == 1);
26004f8ed11SRichard Henderson         return false;
26150627f1bSRichard Henderson     }
26250627f1bSRichard Henderson 
26350627f1bSRichard Henderson     host = db->host_addr[0];
26450627f1bSRichard Henderson     base = db->pc_first;
26504f8ed11SRichard Henderson 
26604f8ed11SRichard Henderson     if (likely(((base ^ last) & TARGET_PAGE_MASK) == 0)) {
26704f8ed11SRichard Henderson         /* Entire read is from the first page. */
2686a9dfe19SRichard Henderson         goto do_read;
26904f8ed11SRichard Henderson     }
27004f8ed11SRichard Henderson 
27104f8ed11SRichard Henderson     if (unlikely(((base ^ pc) & TARGET_PAGE_MASK) == 0)) {
2726a9dfe19SRichard Henderson         /*
2736a9dfe19SRichard Henderson          * Read begins on the first page and extends to the second.
2746a9dfe19SRichard Henderson          * The unaligned read is never atomic.
2756a9dfe19SRichard Henderson          */
27604f8ed11SRichard Henderson         size_t len0 = -(pc | TARGET_PAGE_MASK);
27704f8ed11SRichard Henderson         memcpy(dest, host + (pc - base), len0);
27804f8ed11SRichard Henderson         pc += len0;
27904f8ed11SRichard Henderson         dest += len0;
28004f8ed11SRichard Henderson         len -= len0;
28104f8ed11SRichard Henderson     }
28204f8ed11SRichard Henderson 
28304f8ed11SRichard Henderson     /*
28404f8ed11SRichard Henderson      * The read must conclude on the second page and not extend to a third.
28504f8ed11SRichard Henderson      *
28604f8ed11SRichard Henderson      * TODO: We could allow the two pages to be virtually discontiguous,
28704f8ed11SRichard Henderson      * since we already allow the two pages to be physically discontiguous.
28804f8ed11SRichard Henderson      * The only reasonable use case would be executing an insn at the end
28904f8ed11SRichard Henderson      * of the address space wrapping around to the beginning.  For that,
29004f8ed11SRichard Henderson      * we would need to know the current width of the address space.
29104f8ed11SRichard Henderson      * In the meantime, assert.
29204f8ed11SRichard Henderson      */
29304f8ed11SRichard Henderson     base = (base & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
29404f8ed11SRichard Henderson     assert(((base ^ pc) & TARGET_PAGE_MASK) == 0);
29504f8ed11SRichard Henderson     assert(((base ^ last) & TARGET_PAGE_MASK) == 0);
29650627f1bSRichard Henderson     host = db->host_addr[1];
29704f8ed11SRichard Henderson 
29850627f1bSRichard Henderson     if (host == NULL) {
299deba7870SRichard Henderson         tb_page_addr_t page0, old_page1, new_page1;
300deba7870SRichard Henderson 
301deba7870SRichard Henderson         new_page1 = get_page_addr_code_hostp(env, base, &db->host_addr[1]);
3022627e452SRichard Henderson 
3032627e452SRichard Henderson         /*
3042627e452SRichard Henderson          * If the second page is MMIO, treat as if the first page
3052627e452SRichard Henderson          * was MMIO as well, so that we do not cache the TB.
3062627e452SRichard Henderson          */
307deba7870SRichard Henderson         if (unlikely(new_page1 == -1)) {
308deba7870SRichard Henderson             tb_unlock_pages(tb);
3092627e452SRichard Henderson             tb_set_page_addr0(tb, -1);
310ddfa9f11SRichard Henderson             /* Require that this be the final insn. */
311ddfa9f11SRichard Henderson             db->max_insns = db->num_insns;
31204f8ed11SRichard Henderson             return false;
3132627e452SRichard Henderson         }
3142627e452SRichard Henderson 
315deba7870SRichard Henderson         /*
316deba7870SRichard Henderson          * If this is not the first time around, and page1 matches,
317deba7870SRichard Henderson          * then we already have the page locked.  Alternately, we're
318deba7870SRichard Henderson          * not doing anything to prevent the PTE from changing, so
319deba7870SRichard Henderson          * we might wind up with a different page, requiring us to
320deba7870SRichard Henderson          * re-do the locking.
321deba7870SRichard Henderson          */
322deba7870SRichard Henderson         old_page1 = tb_page_addr1(tb);
323deba7870SRichard Henderson         if (likely(new_page1 != old_page1)) {
324deba7870SRichard Henderson             page0 = tb_page_addr0(tb);
325deba7870SRichard Henderson             if (unlikely(old_page1 != -1)) {
326deba7870SRichard Henderson                 tb_unlock_page1(page0, old_page1);
327deba7870SRichard Henderson             }
328deba7870SRichard Henderson             tb_set_page_addr1(tb, new_page1);
329deba7870SRichard Henderson             tb_lock_page1(page0, new_page1);
330deba7870SRichard Henderson         }
33150627f1bSRichard Henderson         host = db->host_addr[1];
332f025692cSIlya Leoshkevich     }
333f025692cSIlya Leoshkevich 
3346a9dfe19SRichard Henderson  do_read:
3356a9dfe19SRichard Henderson     /*
3366a9dfe19SRichard Henderson      * Assume aligned reads should be atomic, if possible.
3376a9dfe19SRichard Henderson      * We're not in a position to jump out with EXCP_ATOMIC.
3386a9dfe19SRichard Henderson      */
3396a9dfe19SRichard Henderson     host += pc - base;
3406a9dfe19SRichard Henderson     switch (len) {
3416a9dfe19SRichard Henderson     case 2:
3426a9dfe19SRichard Henderson         if (QEMU_IS_ALIGNED(pc, 2)) {
3436a9dfe19SRichard Henderson             uint16_t t = qatomic_read((uint16_t *)host);
3446a9dfe19SRichard Henderson             stw_he_p(dest, t);
3456a9dfe19SRichard Henderson             return true;
3466a9dfe19SRichard Henderson         }
3476a9dfe19SRichard Henderson         break;
3486a9dfe19SRichard Henderson     case 4:
3496a9dfe19SRichard Henderson         if (QEMU_IS_ALIGNED(pc, 4)) {
3506a9dfe19SRichard Henderson             uint32_t t = qatomic_read((uint32_t *)host);
3516a9dfe19SRichard Henderson             stl_he_p(dest, t);
3526a9dfe19SRichard Henderson             return true;
3536a9dfe19SRichard Henderson         }
3546a9dfe19SRichard Henderson         break;
3556a9dfe19SRichard Henderson #ifdef CONFIG_ATOMIC64
3566a9dfe19SRichard Henderson     case 8:
3576a9dfe19SRichard Henderson         if (QEMU_IS_ALIGNED(pc, 8)) {
3586a9dfe19SRichard Henderson             uint64_t t = qatomic_read__nocheck((uint64_t *)host);
3596a9dfe19SRichard Henderson             stq_he_p(dest, t);
3606a9dfe19SRichard Henderson             return true;
3616a9dfe19SRichard Henderson         }
3626a9dfe19SRichard Henderson         break;
3636a9dfe19SRichard Henderson #endif
3646a9dfe19SRichard Henderson     }
3656a9dfe19SRichard Henderson     /* Unaligned or partial read from the second page is not atomic. */
3666a9dfe19SRichard Henderson     memcpy(dest, host, len);
36704f8ed11SRichard Henderson     return true;
36850627f1bSRichard Henderson }
369f025692cSIlya Leoshkevich 
record_save(DisasContextBase * db,vaddr pc,const void * from,int size)370ba3fb2a7SRichard Henderson static void record_save(DisasContextBase *db, vaddr pc,
371ba3fb2a7SRichard Henderson                         const void *from, int size)
372ba3fb2a7SRichard Henderson {
373ba3fb2a7SRichard Henderson     int offset;
374ba3fb2a7SRichard Henderson 
375ba3fb2a7SRichard Henderson     /* Do not record probes before the start of TB. */
376ba3fb2a7SRichard Henderson     if (pc < db->pc_first) {
377ba3fb2a7SRichard Henderson         return;
378ba3fb2a7SRichard Henderson     }
379ba3fb2a7SRichard Henderson 
380ba3fb2a7SRichard Henderson     /*
381ba3fb2a7SRichard Henderson      * In translator_access, we verified that pc is within 2 pages
382ba3fb2a7SRichard Henderson      * of pc_first, thus this will never overflow.
383ba3fb2a7SRichard Henderson      */
384ba3fb2a7SRichard Henderson     offset = pc - db->pc_first;
385ba3fb2a7SRichard Henderson 
386ba3fb2a7SRichard Henderson     /*
387ba3fb2a7SRichard Henderson      * Either the first or second page may be I/O.  If it is the second,
388ba3fb2a7SRichard Henderson      * then the first byte we need to record will be at a non-zero offset.
389ba3fb2a7SRichard Henderson      * In either case, we should not need to record but a single insn.
390ba3fb2a7SRichard Henderson      */
391ba3fb2a7SRichard Henderson     if (db->record_len == 0) {
392ba3fb2a7SRichard Henderson         db->record_start = offset;
393ba3fb2a7SRichard Henderson         db->record_len = size;
394ba3fb2a7SRichard Henderson     } else {
395ba3fb2a7SRichard Henderson         assert(offset == db->record_start + db->record_len);
396ba3fb2a7SRichard Henderson         assert(db->record_len + size <= sizeof(db->record));
397ba3fb2a7SRichard Henderson         db->record_len += size;
398ba3fb2a7SRichard Henderson     }
399ba3fb2a7SRichard Henderson 
400ba3fb2a7SRichard Henderson     memcpy(db->record + (offset - db->record_start), from, size);
401ba3fb2a7SRichard Henderson }
402ba3fb2a7SRichard Henderson 
translator_st_len(const DisasContextBase * db)4033a247368SRichard Henderson size_t translator_st_len(const DisasContextBase *db)
4043a247368SRichard Henderson {
4053a247368SRichard Henderson     return db->fake_insn ? db->record_len : db->tb->size;
4063a247368SRichard Henderson }
4073a247368SRichard Henderson 
translator_st(const DisasContextBase * db,void * dest,vaddr addr,size_t len)4083a247368SRichard Henderson bool translator_st(const DisasContextBase *db, void *dest,
4093a247368SRichard Henderson                    vaddr addr, size_t len)
4103a247368SRichard Henderson {
4113a247368SRichard Henderson     size_t offset, offset_end;
4123a247368SRichard Henderson 
4133a247368SRichard Henderson     if (addr < db->pc_first) {
4143a247368SRichard Henderson         return false;
4153a247368SRichard Henderson     }
4163a247368SRichard Henderson     offset = addr - db->pc_first;
4173a247368SRichard Henderson     offset_end = offset + len;
4183a247368SRichard Henderson     if (offset_end > translator_st_len(db)) {
4193a247368SRichard Henderson         return false;
4203a247368SRichard Henderson     }
4213a247368SRichard Henderson 
4223a247368SRichard Henderson     if (!db->fake_insn) {
4233a247368SRichard Henderson         size_t offset_page1 = -(db->pc_first | TARGET_PAGE_MASK);
4243a247368SRichard Henderson 
4253a247368SRichard Henderson         /* Get all the bytes from the first page. */
4263a247368SRichard Henderson         if (db->host_addr[0]) {
4273a247368SRichard Henderson             if (offset_end <= offset_page1) {
4283a247368SRichard Henderson                 memcpy(dest, db->host_addr[0] + offset, len);
4293a247368SRichard Henderson                 return true;
4303a247368SRichard Henderson             }
4313a247368SRichard Henderson             if (offset < offset_page1) {
4323a247368SRichard Henderson                 size_t len0 = offset_page1 - offset;
4333a247368SRichard Henderson                 memcpy(dest, db->host_addr[0] + offset, len0);
4343a247368SRichard Henderson                 offset += len0;
4353a247368SRichard Henderson                 dest += len0;
4363a247368SRichard Henderson             }
4373a247368SRichard Henderson         }
4383a247368SRichard Henderson 
4393a247368SRichard Henderson         /* Get any bytes from the second page. */
4403a247368SRichard Henderson         if (db->host_addr[1] && offset >= offset_page1) {
4413a247368SRichard Henderson             memcpy(dest, db->host_addr[1] + (offset - offset_page1),
4423a247368SRichard Henderson                    offset_end - offset);
4433a247368SRichard Henderson             return true;
4443a247368SRichard Henderson         }
4453a247368SRichard Henderson     }
4463a247368SRichard Henderson 
4473a247368SRichard Henderson     /* Else get recorded bytes. */
4483a247368SRichard Henderson     if (db->record_len != 0 &&
4493a247368SRichard Henderson         offset >= db->record_start &&
4503a247368SRichard Henderson         offset_end <= db->record_start + db->record_len) {
4513a247368SRichard Henderson         memcpy(dest, db->record + (offset - db->record_start),
4523a247368SRichard Henderson                offset_end - offset);
4533a247368SRichard Henderson         return true;
4543a247368SRichard Henderson     }
4553a247368SRichard Henderson     return false;
4563a247368SRichard Henderson }
4573a247368SRichard Henderson 
translator_ldub(CPUArchState * env,DisasContextBase * db,vaddr pc)45880189472SRichard Henderson uint8_t translator_ldub(CPUArchState *env, DisasContextBase *db, vaddr pc)
45950627f1bSRichard Henderson {
460028119c8SRichard Henderson     uint8_t val;
46150627f1bSRichard Henderson 
462028119c8SRichard Henderson     if (!translator_ld(env, db, &val, pc, sizeof(val))) {
463028119c8SRichard Henderson         MemOpIdx oi = make_memop_idx(MO_UB, db->code_mmuidx);
464028119c8SRichard Henderson         val = cpu_ldb_code_mmu(env, pc, oi, 0);
465028119c8SRichard Henderson         record_save(db, pc, &val, sizeof(val));
46650627f1bSRichard Henderson     }
467028119c8SRichard Henderson     return val;
46850627f1bSRichard Henderson }
46950627f1bSRichard Henderson 
translator_lduw_end(CPUArchState * env,DisasContextBase * db,vaddr pc,MemOp endian)4705c43a750SRichard Henderson uint16_t translator_lduw_end(CPUArchState *env, DisasContextBase *db,
4715c43a750SRichard Henderson                              vaddr pc, MemOp endian)
47250627f1bSRichard Henderson {
473028119c8SRichard Henderson     uint16_t val;
47450627f1bSRichard Henderson 
475028119c8SRichard Henderson     if (!translator_ld(env, db, &val, pc, sizeof(val))) {
476028119c8SRichard Henderson         MemOpIdx oi = make_memop_idx(MO_UW, db->code_mmuidx);
477028119c8SRichard Henderson         val = cpu_ldw_code_mmu(env, pc, oi, 0);
478028119c8SRichard Henderson         record_save(db, pc, &val, sizeof(val));
47950627f1bSRichard Henderson     }
4805c43a750SRichard Henderson     if (endian & MO_BSWAP) {
4815c43a750SRichard Henderson         val = bswap16(val);
4825c43a750SRichard Henderson     }
4835c43a750SRichard Henderson     return val;
48450627f1bSRichard Henderson }
48550627f1bSRichard Henderson 
translator_ldl_end(CPUArchState * env,DisasContextBase * db,vaddr pc,MemOp endian)4865c43a750SRichard Henderson uint32_t translator_ldl_end(CPUArchState *env, DisasContextBase *db,
4875c43a750SRichard Henderson                             vaddr pc, MemOp endian)
48850627f1bSRichard Henderson {
489028119c8SRichard Henderson     uint32_t val;
49050627f1bSRichard Henderson 
491028119c8SRichard Henderson     if (!translator_ld(env, db, &val, pc, sizeof(val))) {
492028119c8SRichard Henderson         MemOpIdx oi = make_memop_idx(MO_UL, db->code_mmuidx);
493028119c8SRichard Henderson         val = cpu_ldl_code_mmu(env, pc, oi, 0);
494028119c8SRichard Henderson         record_save(db, pc, &val, sizeof(val));
49550627f1bSRichard Henderson     }
4965c43a750SRichard Henderson     if (endian & MO_BSWAP) {
4975c43a750SRichard Henderson         val = bswap32(val);
4985c43a750SRichard Henderson     }
4995c43a750SRichard Henderson     return val;
50050627f1bSRichard Henderson }
50150627f1bSRichard Henderson 
translator_ldq_end(CPUArchState * env,DisasContextBase * db,vaddr pc,MemOp endian)5025c43a750SRichard Henderson uint64_t translator_ldq_end(CPUArchState *env, DisasContextBase *db,
5035c43a750SRichard Henderson                             vaddr pc, MemOp endian)
50450627f1bSRichard Henderson {
505028119c8SRichard Henderson     uint64_t val;
50650627f1bSRichard Henderson 
507028119c8SRichard Henderson     if (!translator_ld(env, db, &val, pc, sizeof(val))) {
508028119c8SRichard Henderson         MemOpIdx oi = make_memop_idx(MO_UQ, db->code_mmuidx);
509028119c8SRichard Henderson         val = cpu_ldq_code_mmu(env, pc, oi, 0);
510028119c8SRichard Henderson         record_save(db, pc, &val, sizeof(val));
51150627f1bSRichard Henderson     }
5125c43a750SRichard Henderson     if (endian & MO_BSWAP) {
5135c43a750SRichard Henderson         val = bswap64(val);
5145c43a750SRichard Henderson     }
5155c43a750SRichard Henderson     return val;
51650627f1bSRichard Henderson }
517309e014dSRichard Henderson 
translator_fake_ld(DisasContextBase * db,const void * data,size_t len)5184c6163eaSRichard Henderson void translator_fake_ld(DisasContextBase *db, const void *data, size_t len)
519309e014dSRichard Henderson {
520b3f05b8cSRichard Henderson     db->fake_insn = true;
5214c6163eaSRichard Henderson     record_save(db, db->pc_first, data, len);
522309e014dSRichard Henderson }
523