1 // SPDX-License-Identifier: GPL-2.0
2
3 #ifndef __KVM_X86_MMU_TDP_ITER_H
4 #define __KVM_X86_MMU_TDP_ITER_H
5
6 #include <linux/kvm_host.h>
7
8 #include "mmu.h"
9 #include "spte.h"
10
11 /*
12 * TDP MMU SPTEs are RCU protected to allow paging structures (non-leaf SPTEs)
13 * to be zapped while holding mmu_lock for read, and to allow TLB flushes to be
14 * batched without having to collect the list of zapped SPs. Flows that can
15 * remove SPs must service pending TLB flushes prior to dropping RCU protection.
16 */
kvm_tdp_mmu_read_spte(tdp_ptep_t sptep)17 static inline u64 kvm_tdp_mmu_read_spte(tdp_ptep_t sptep)
18 {
19 return READ_ONCE(*rcu_dereference(sptep));
20 }
21
kvm_tdp_mmu_write_spte_atomic(tdp_ptep_t sptep,u64 new_spte)22 static inline u64 kvm_tdp_mmu_write_spte_atomic(tdp_ptep_t sptep, u64 new_spte)
23 {
24 KVM_MMU_WARN_ON(is_ept_ve_possible(new_spte));
25 return xchg(rcu_dereference(sptep), new_spte);
26 }
27
tdp_mmu_clear_spte_bits_atomic(tdp_ptep_t sptep,u64 mask)28 static inline u64 tdp_mmu_clear_spte_bits_atomic(tdp_ptep_t sptep, u64 mask)
29 {
30 atomic64_t *sptep_atomic = (atomic64_t *)rcu_dereference(sptep);
31
32 return (u64)atomic64_fetch_and(~mask, sptep_atomic);
33 }
34
__kvm_tdp_mmu_write_spte(tdp_ptep_t sptep,u64 new_spte)35 static inline void __kvm_tdp_mmu_write_spte(tdp_ptep_t sptep, u64 new_spte)
36 {
37 KVM_MMU_WARN_ON(is_ept_ve_possible(new_spte));
38 WRITE_ONCE(*rcu_dereference(sptep), new_spte);
39 }
40
41 /*
42 * SPTEs must be modified atomically if they are shadow-present, leaf SPTEs,
43 * and have volatile bits (bits that can be set outside of mmu_lock) that
44 * must not be clobbered.
45 */
kvm_tdp_mmu_spte_need_atomic_update(u64 old_spte,int level)46 static inline bool kvm_tdp_mmu_spte_need_atomic_update(u64 old_spte, int level)
47 {
48 return is_shadow_present_pte(old_spte) &&
49 is_last_spte(old_spte, level) &&
50 spte_needs_atomic_update(old_spte);
51 }
52
kvm_tdp_mmu_write_spte(tdp_ptep_t sptep,u64 old_spte,u64 new_spte,int level)53 static inline u64 kvm_tdp_mmu_write_spte(tdp_ptep_t sptep, u64 old_spte,
54 u64 new_spte, int level)
55 {
56 if (kvm_tdp_mmu_spte_need_atomic_update(old_spte, level))
57 return kvm_tdp_mmu_write_spte_atomic(sptep, new_spte);
58
59 __kvm_tdp_mmu_write_spte(sptep, new_spte);
60 return old_spte;
61 }
62
tdp_mmu_clear_spte_bits(tdp_ptep_t sptep,u64 old_spte,u64 mask,int level)63 static inline u64 tdp_mmu_clear_spte_bits(tdp_ptep_t sptep, u64 old_spte,
64 u64 mask, int level)
65 {
66 if (kvm_tdp_mmu_spte_need_atomic_update(old_spte, level))
67 return tdp_mmu_clear_spte_bits_atomic(sptep, mask);
68
69 __kvm_tdp_mmu_write_spte(sptep, old_spte & ~mask);
70 return old_spte;
71 }
72
73 /*
74 * A TDP iterator performs a pre-order walk over a TDP paging structure.
75 */
76 struct tdp_iter {
77 /*
78 * The iterator will traverse the paging structure towards the mapping
79 * for this GFN.
80 */
81 gfn_t next_last_level_gfn;
82 /*
83 * The next_last_level_gfn at the time when the thread last
84 * yielded. Only yielding when the next_last_level_gfn !=
85 * yielded_gfn helps ensure forward progress.
86 */
87 gfn_t yielded_gfn;
88 /* Pointers to the page tables traversed to reach the current SPTE */
89 tdp_ptep_t pt_path[PT64_ROOT_MAX_LEVEL];
90 /* A pointer to the current SPTE */
91 tdp_ptep_t sptep;
92 /* The lowest GFN (mask bits excluded) mapped by the current SPTE */
93 gfn_t gfn;
94 /* Mask applied to convert the GFN to the mapping GPA */
95 gfn_t gfn_bits;
96 /* The level of the root page given to the iterator */
97 int root_level;
98 /* The lowest level the iterator should traverse to */
99 int min_level;
100 /* The iterator's current level within the paging structure */
101 int level;
102 /* The address space ID, i.e. SMM vs. regular. */
103 int as_id;
104 /* A snapshot of the value at sptep */
105 u64 old_spte;
106 /*
107 * Whether the iterator has a valid state. This will be false if the
108 * iterator walks off the end of the paging structure.
109 */
110 bool valid;
111 /*
112 * True if KVM dropped mmu_lock and yielded in the middle of a walk, in
113 * which case tdp_iter_next() needs to restart the walk at the root
114 * level instead of advancing to the next entry.
115 */
116 bool yielded;
117 };
118
119 /*
120 * Iterates over every SPTE mapping the GFN range [start, end) in a
121 * preorder traversal.
122 */
123 #define for_each_tdp_pte_min_level(iter, kvm, root, min_level, start, end) \
124 for (tdp_iter_start(&iter, root, min_level, start, kvm_gfn_root_bits(kvm, root)); \
125 iter.valid && iter.gfn < end; \
126 tdp_iter_next(&iter))
127
128 #define for_each_tdp_pte_min_level_all(iter, root, min_level) \
129 for (tdp_iter_start(&iter, root, min_level, 0, 0); \
130 iter.valid && iter.gfn < tdp_mmu_max_gfn_exclusive(); \
131 tdp_iter_next(&iter))
132
133 #define for_each_tdp_pte(iter, kvm, root, start, end) \
134 for_each_tdp_pte_min_level(iter, kvm, root, PG_LEVEL_4K, start, end)
135
136 tdp_ptep_t spte_to_child_pt(u64 pte, int level);
137
138 void tdp_iter_start(struct tdp_iter *iter, struct kvm_mmu_page *root,
139 int min_level, gfn_t next_last_level_gfn, gfn_t gfn_bits);
140 void tdp_iter_next(struct tdp_iter *iter);
141 void tdp_iter_restart(struct tdp_iter *iter);
142
143 #endif /* __KVM_X86_MMU_TDP_ITER_H */
144