11  * This library is distributed in the hope that it will be useful,
89 static inline uint32_t ptw_ldl(const PTETranslate *in, uint64_t ra)  in ptw_ldl()  argument
91     if (likely(in->haddr)) {  in ptw_ldl()
92         return ldl_p(in->haddr);  in ptw_ldl()
94     return cpu_ldl_mmuidx_ra(in->env, in->gaddr, in->ptw_idx, ra);  in ptw_ldl()
97 static inline uint64_t ptw_ldq(const PTETranslate *in, uint64_t ra)  in ptw_ldq()  argument
99     if (likely(in->haddr)) {  in ptw_ldq()
100         return ldq_p(in->haddr);  in ptw_ldq()
102     return cpu_ldq_mmuidx_ra(in->env, in->gaddr, in->ptw_idx, ra);  in ptw_ldq()
108  * PG_DIRTY_MASK are all in the low 32 bits.
110 static bool ptw_setl_slow(const PTETranslate *in, uint32_t old, uint32_t new)  in ptw_setl_slow()  argument
114     CPUState *cpu = env_cpu(in->env);  in ptw_setl_slow()
115     /* We are in cpu_exec, and start_exclusive can't be called directly.*/  in ptw_setl_slow()
120     cmp = cpu_ldl_mmuidx_ra(in->env, in->gaddr, in->ptw_idx, 0);  in ptw_setl_slow()
122         cpu_stl_mmuidx_ra(in->env, in->gaddr, new, in->ptw_idx, 0);  in ptw_setl_slow()
129 static inline bool ptw_setl(const PTETranslate *in, uint32_t old, uint32_t set)  in ptw_setl()  argument
133         if (likely(in->haddr)) {  in ptw_setl()
136             return qatomic_cmpxchg((uint32_t *)in->haddr, old, new) == old;  in ptw_setl()
138         return ptw_setl_slow(in, old, new);  in ptw_setl()
143 static bool mmu_translate(CPUX86State *env, const TranslateParams *in,  in mmu_translate()  argument
147     const target_ulong addr = in->addr;  in mmu_translate()
148     const int pg_mode = in->pg_mode;  in mmu_translate()
149     const bool is_user = is_mmu_index_user(in->mmu_idx);  in mmu_translate()
150     const MMUAccessType access_type = in->access_type;  in mmu_translate()
155         .ptw_idx = in->ptw_idx,  in mmu_translate()
177                 pte_addr = (in->cr3 & ~0xfff) + (((addr >> 48) & 0x1ff) << 3);  in mmu_translate()
194                 pte = in->cr3;  in mmu_translate()
248             pte_addr = (in->cr3 & 0xffffffe0ULL) + ((addr >> 27) & 0x18);  in mmu_translate()
314         pte_addr = (in->cr3 & 0xfffff000ULL) + ((addr >> 20) & 0xffc);  in mmu_translate()
330              * Leave bits 20-13 in place for setting accessed/dirty bits below.  in mmu_translate()
361         paddr = in->addr;  in mmu_translate()
374     /* can the page can be put in the TLB?  prot will tell us */  in mmu_translate()
380     if (!is_mmu_index_smap(in->mmu_idx) || !(ptep & PG_USER_MASK)) {  in mmu_translate()
447     if (in->ptw_idx == MMU_NESTED_IDX) {  in mmu_translate()
552     TranslateParams in;  in get_physical_address()  local
555     in.addr = addr;  in get_physical_address()
556     in.access_type = access_type;  in get_physical_address()
564             in.cr3 = env->nested_cr3;  in get_physical_address()
565             in.pg_mode = env->nested_pg_mode;  in get_physical_address()
566             in.mmu_idx =  in get_physical_address()
568             in.ptw_idx = MMU_PHYS_IDX;  in get_physical_address()
570             if (!mmu_translate(env, &in, out, err, ra)) {  in get_physical_address()
584             in.cr3 = env->cr[3];  in get_physical_address()
585             in.mmu_idx = mmu_idx;  in get_physical_address()
586             in.ptw_idx = use_stage2 ? MMU_NESTED_IDX : MMU_PHYS_IDX;  in get_physical_address()
587             in.pg_mode = get_pg_mode(env);  in get_physical_address()
589             if (in.pg_mode & PG_MODE_LMA) {  in get_physical_address()
591                 int shift = in.pg_mode & PG_MODE_LA57 ? 56 : 47;  in get_physical_address()
601             return mmu_translate(env, &in, out, err, ra);  in get_physical_address()
624          * Even if 4MB pages, we map only one 4KB page in the cache to  in x86_cpu_tlb_fill()
646         /* cr2 is not modified in case of exceptions */  in x86_cpu_tlb_fill()