1 /* SPDX-License-Identifier: GPL-2.0-only */
2 #ifndef _ASM_X86_PARAVIRT_SPINLOCK_H
3 #define _ASM_X86_PARAVIRT_SPINLOCK_H
4
5 #include <asm/paravirt_types.h>
6
7 #ifdef CONFIG_SMP
8 #include <asm/spinlock_types.h>
9 #endif
10
11 struct qspinlock;
12
13 struct pv_lock_ops {
14 void (*queued_spin_lock_slowpath)(struct qspinlock *lock, u32 val);
15 struct paravirt_callee_save queued_spin_unlock;
16
17 void (*wait)(u8 *ptr, u8 val);
18 void (*kick)(int cpu);
19
20 struct paravirt_callee_save vcpu_is_preempted;
21 } __no_randomize_layout;
22
23 extern struct pv_lock_ops pv_ops_lock;
24
25 #ifdef CONFIG_PARAVIRT_SPINLOCKS
26 extern void native_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val);
27 extern void __pv_init_lock_hash(void);
28 extern void __pv_queued_spin_lock_slowpath(struct qspinlock *lock, u32 val);
29 extern void __raw_callee_save___pv_queued_spin_unlock(struct qspinlock *lock);
30 extern bool nopvspin;
31
pv_queued_spin_lock_slowpath(struct qspinlock * lock,u32 val)32 static __always_inline void pv_queued_spin_lock_slowpath(struct qspinlock *lock,
33 u32 val)
34 {
35 PVOP_VCALL2(pv_ops_lock, queued_spin_lock_slowpath, lock, val);
36 }
37
pv_queued_spin_unlock(struct qspinlock * lock)38 static __always_inline void pv_queued_spin_unlock(struct qspinlock *lock)
39 {
40 PVOP_ALT_VCALLEE1(pv_ops_lock, queued_spin_unlock, lock,
41 "movb $0, (%%" _ASM_ARG1 ")",
42 ALT_NOT(X86_FEATURE_PVUNLOCK));
43 }
44
pv_vcpu_is_preempted(long cpu)45 static __always_inline bool pv_vcpu_is_preempted(long cpu)
46 {
47 return PVOP_ALT_CALLEE1(bool, pv_ops_lock, vcpu_is_preempted, cpu,
48 "xor %%eax, %%eax",
49 ALT_NOT(X86_FEATURE_VCPUPREEMPT));
50 }
51
52 #define queued_spin_unlock queued_spin_unlock
53 /**
54 * queued_spin_unlock - release a queued spinlock
55 * @lock : Pointer to queued spinlock structure
56 *
57 * A smp_store_release() on the least-significant byte.
58 */
native_queued_spin_unlock(struct qspinlock * lock)59 static inline void native_queued_spin_unlock(struct qspinlock *lock)
60 {
61 smp_store_release(&lock->locked, 0);
62 }
63
queued_spin_lock_slowpath(struct qspinlock * lock,u32 val)64 static inline void queued_spin_lock_slowpath(struct qspinlock *lock, u32 val)
65 {
66 pv_queued_spin_lock_slowpath(lock, val);
67 }
68
queued_spin_unlock(struct qspinlock * lock)69 static inline void queued_spin_unlock(struct qspinlock *lock)
70 {
71 kcsan_release();
72 pv_queued_spin_unlock(lock);
73 }
74
75 #define vcpu_is_preempted vcpu_is_preempted
vcpu_is_preempted(long cpu)76 static inline bool vcpu_is_preempted(long cpu)
77 {
78 return pv_vcpu_is_preempted(cpu);
79 }
80
pv_wait(u8 * ptr,u8 val)81 static __always_inline void pv_wait(u8 *ptr, u8 val)
82 {
83 PVOP_VCALL2(pv_ops_lock, wait, ptr, val);
84 }
85
pv_kick(int cpu)86 static __always_inline void pv_kick(int cpu)
87 {
88 PVOP_VCALL1(pv_ops_lock, kick, cpu);
89 }
90
91 void __raw_callee_save___native_queued_spin_unlock(struct qspinlock *lock);
92 bool __raw_callee_save___native_vcpu_is_preempted(long cpu);
93 #endif /* CONFIG_PARAVIRT_SPINLOCKS */
94
95 void __init native_pv_lock_init(void);
96 __visible void __native_queued_spin_unlock(struct qspinlock *lock);
97 bool pv_is_native_spin_unlock(void);
98 __visible bool __native_vcpu_is_preempted(long cpu);
99 bool pv_is_native_vcpu_is_preempted(void);
100
101 /*
102 * virt_spin_lock_key - disables by default the virt_spin_lock() hijack.
103 *
104 * Native (and PV wanting native due to vCPU pinning) should keep this key
105 * disabled. Native does not touch the key.
106 *
107 * When in a guest then native_pv_lock_init() enables the key first and
108 * KVM/XEN might conditionally disable it later in the boot process again.
109 */
110 DECLARE_STATIC_KEY_FALSE(virt_spin_lock_key);
111
112 /*
113 * Shortcut for the queued_spin_lock_slowpath() function that allows
114 * virt to hijack it.
115 *
116 * Returns:
117 * true - lock has been negotiated, all done;
118 * false - queued_spin_lock_slowpath() will do its thing.
119 */
120 #define virt_spin_lock virt_spin_lock
virt_spin_lock(struct qspinlock * lock)121 static inline bool virt_spin_lock(struct qspinlock *lock)
122 {
123 int val;
124
125 if (!static_branch_likely(&virt_spin_lock_key))
126 return false;
127
128 /*
129 * On hypervisors without PARAVIRT_SPINLOCKS support we fall
130 * back to a Test-and-Set spinlock, because fair locks have
131 * horrible lock 'holder' preemption issues.
132 */
133
134 __retry:
135 val = atomic_read(&lock->val);
136
137 if (val || !atomic_try_cmpxchg(&lock->val, &val, _Q_LOCKED_VAL)) {
138 cpu_relax();
139 goto __retry;
140 }
141
142 return true;
143 }
144
145 #endif /* _ASM_X86_PARAVIRT_SPINLOCK_H */
146