| /linux/arch/x86/kvm/mmu/ |
| H A D | tdp_iter.h | 17 static inline u64 kvm_tdp_mmu_read_spte(tdp_ptep_t sptep) in kvm_tdp_mmu_read_spte() argument
19 return READ_ONCE(*rcu_dereference(sptep)); in kvm_tdp_mmu_read_spte()
22 static inline u64 kvm_tdp_mmu_write_spte_atomic(tdp_ptep_t sptep, u64 new_spte) in kvm_tdp_mmu_write_spte_atomic() argument
25 return xchg(rcu_dereference(sptep), new_spte); in kvm_tdp_mmu_write_spte_atomic()
28 static inline u64 tdp_mmu_clear_spte_bits_atomic(tdp_ptep_t sptep, u64 mask) in tdp_mmu_clear_spte_bits_atomic() argument
30 atomic64_t *sptep_atomic = (atomic64_t *)rcu_dereference(sptep); in tdp_mmu_clear_spte_bits_atomic()
35 static inline void __kvm_tdp_mmu_write_spte(tdp_ptep_t sptep, u64 new_spte) in __kvm_tdp_mmu_write_spte() argument
38 WRITE_ONCE(*rcu_dereference(sptep), new_spte); in __kvm_tdp_mmu_write_spte()
53 static inline u64 kvm_tdp_mmu_write_spte(tdp_ptep_t sptep, u64 old_spte, in kvm_tdp_mmu_write_spte() argument
57 return kvm_tdp_mmu_write_spte_atomic(sptep, new_spte); in kvm_tdp_mmu_write_spte()
[all …]
|
| H A D | mmutrace.h | 216 TP_PROTO(u64 *sptep, gfn_t gfn, u64 spte),
217 TP_ARGS(sptep, gfn, spte),
220 __field(void *, sptep)
227 __entry->sptep = sptep;
233 TP_printk("sptep:%p gfn %llx access %x gen %x", __entry->sptep,
261 u64 *sptep, u64 old_spte, int ret),
262 TP_ARGS(vcpu, fault, sptep, old_spte, ret),
268 __field(u64 *, sptep)
278 __entry->sptep = sptep;
280 __entry->new_spte = *sptep;
[all …]
|
| H A D | mmu.c | 159 u64 *sptep; member
178 ({ spte = mmu_spte_get_lockless(_walker.sptep); 1; }); \
184 static void mmu_spte_set(u64 *sptep, u64 spte);
284 static void kvm_flush_remote_tlbs_sptep(struct kvm *kvm, u64 *sptep) in kvm_flush_remote_tlbs_sptep() argument
286 struct kvm_mmu_page *sp = sptep_to_sp(sptep); in kvm_flush_remote_tlbs_sptep()
287 gfn_t gfn = kvm_mmu_page_get_gfn(sp, spte_index(sptep)); in kvm_flush_remote_tlbs_sptep()
292 static void mark_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, u64 gfn, in mark_mmio_spte() argument
297 trace_mark_mmio_spte(sptep, gfn, spte); in mark_mmio_spte()
298 mmu_spte_set(sptep, spte); in mark_mmio_spte()
337 static void __set_spte(u64 *sptep, u64 spte) in __set_spte() argument
[all …]
|
| H A D | tdp_mmu.c | 224 static void tdp_mmu_init_sp(struct kvm_mmu_page *sp, tdp_ptep_t sptep, in tdp_mmu_init_sp() argument
233 sp->ptep = sptep; in tdp_mmu_init_sp()
245 parent_sp = sptep_to_sp(rcu_dereference(iter->sptep)); in tdp_mmu_init_child_sp()
250 tdp_mmu_init_sp(child_sp, iter->sptep, iter->gfn, role); in tdp_mmu_init_child_sp()
410 tdp_ptep_t sptep = pt + i; in handle_removed_pt() local
424 old_spte = kvm_tdp_mmu_write_spte_atomic(sptep, FROZEN_SPTE); in handle_removed_pt()
439 old_spte = kvm_tdp_mmu_read_spte(sptep); in handle_removed_pt()
471 old_spte = kvm_tdp_mmu_write_spte(sptep, old_spte, in handle_removed_pt()
511 static int __must_check set_external_spte_present(struct kvm *kvm, tdp_ptep_t sptep, in set_external_spte_present() argument
528 if (!try_cmpxchg64(rcu_dereference(sptep), &old_spte, FROZEN_SPTE)) in set_external_spte_present()
[all …]
|
| H A D | paging_tmpl.h | 572 u64 *sptep) in pte_prefetch() argument
579 sp = sptep_to_sp(sptep); in pte_prefetch()
592 return __direct_pte_prefetch(vcpu, sp, sptep); in pte_prefetch()
594 i = spte_index(sptep) & ~(PTE_PREFETCH_NUM - 1); in pte_prefetch()
598 if (spte == sptep) in pte_prefetch()
655 clear_sp_write_flooding_count(it.sptep); in fetch()
661 sp = kvm_mmu_get_child_sp(vcpu, it.sptep, table_gfn, in fetch()
698 link_shadow_page(vcpu, it.sptep, sp); in fetch()
720 disallowed_hugepage_adjust(fault, *it.sptep, it.level); in fetch()
726 validate_direct_spte(vcpu, it.sptep, direct_access); in fetch()
[all …]
|
| H A D | tdp_iter.c | 14 iter->sptep = iter->pt_path[iter->level - 1] + in tdp_iter_refresh_sptep()
16 iter->old_spte = kvm_tdp_mmu_read_spte(iter->sptep); in tdp_iter_refresh_sptep()
91 iter->old_spte = kvm_tdp_mmu_read_spte(iter->sptep); in try_step_down()
124 iter->sptep++; in try_step_side()
125 iter->old_spte = kvm_tdp_mmu_read_spte(iter->sptep); in try_step_side()
|
| H A D | spte.h | 234 static inline int spte_index(u64 *sptep) in spte_index() argument
236 return ((unsigned long)sptep / sizeof(*sptep)) & (SPTE_ENT_PER_PAGE - 1); in spte_index()
271 static inline struct kvm_mmu_page *sptep_to_sp(u64 *sptep) in sptep_to_sp() argument
273 return to_shadow_page(__pa(sptep)); in sptep_to_sp()
288 static inline bool is_mirror_sptep(tdp_ptep_t sptep) in is_mirror_sptep() argument
290 return is_mirror_sp(sptep_to_sp(rcu_dereference(sptep))); in is_mirror_sptep()
|