1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _ASM_X86_CMPXCHG_32_H
3 #define _ASM_X86_CMPXCHG_32_H
4 
5 /*
6  * Note: if you use __cmpxchg64(), or their variants,
7  *       you need to test for the feature in boot_cpu_data.
8  */
9 
10 union __u64_halves {
11 	u64 full;
12 	struct {
13 		u32 low, high;
14 	};
15 };
16 
17 #define __arch_cmpxchg64(_ptr, _old, _new, _lock)			\
18 ({									\
19 	union __u64_halves o = { .full = (_old), },			\
20 			   n = { .full = (_new), };			\
21 									\
22 	asm_inline volatile(_lock "cmpxchg8b %[ptr]"			\
23 		     : [ptr] "+m" (*(_ptr)),				\
24 		       "+a" (o.low), "+d" (o.high)			\
25 		     : "b" (n.low), "c" (n.high)			\
26 		     : "memory");					\
27 									\
28 	o.full;								\
29 })
30 
31 
__cmpxchg64(volatile u64 * ptr,u64 old,u64 new)32 static __always_inline u64 __cmpxchg64(volatile u64 *ptr, u64 old, u64 new)
33 {
34 	return __arch_cmpxchg64(ptr, old, new, LOCK_PREFIX);
35 }
36 
__cmpxchg64_local(volatile u64 * ptr,u64 old,u64 new)37 static __always_inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new)
38 {
39 	return __arch_cmpxchg64(ptr, old, new,);
40 }
41 
42 #define __arch_try_cmpxchg64(_ptr, _oldp, _new, _lock)			\
43 ({									\
44 	union __u64_halves o = { .full = *(_oldp), },			\
45 			   n = { .full = (_new), };			\
46 	bool ret;							\
47 									\
48 	asm_inline volatile(_lock "cmpxchg8b %[ptr]"			\
49 		     CC_SET(e)						\
50 		     : CC_OUT(e) (ret),					\
51 		       [ptr] "+m" (*(_ptr)),				\
52 		       "+a" (o.low), "+d" (o.high)			\
53 		     : "b" (n.low), "c" (n.high)			\
54 		     : "memory");					\
55 									\
56 	if (unlikely(!ret))						\
57 		*(_oldp) = o.full;					\
58 									\
59 	likely(ret);							\
60 })
61 
__try_cmpxchg64(volatile u64 * ptr,u64 * oldp,u64 new)62 static __always_inline bool __try_cmpxchg64(volatile u64 *ptr, u64 *oldp, u64 new)
63 {
64 	return __arch_try_cmpxchg64(ptr, oldp, new, LOCK_PREFIX);
65 }
66 
__try_cmpxchg64_local(volatile u64 * ptr,u64 * oldp,u64 new)67 static __always_inline bool __try_cmpxchg64_local(volatile u64 *ptr, u64 *oldp, u64 new)
68 {
69 	return __arch_try_cmpxchg64(ptr, oldp, new,);
70 }
71 
72 #ifdef CONFIG_X86_CX8
73 
74 #define arch_cmpxchg64 __cmpxchg64
75 
76 #define arch_cmpxchg64_local __cmpxchg64_local
77 
78 #define arch_try_cmpxchg64 __try_cmpxchg64
79 
80 #define arch_try_cmpxchg64_local __try_cmpxchg64_local
81 
82 #else
83 
84 /*
85  * Building a kernel capable running on 80386 and 80486. It may be necessary
86  * to simulate the cmpxchg8b on the 80386 and 80486 CPU.
87  */
88 
89 #define __arch_cmpxchg64_emu(_ptr, _old, _new, _lock_loc, _lock)	\
90 ({									\
91 	union __u64_halves o = { .full = (_old), },			\
92 			   n = { .full = (_new), };			\
93 									\
94 	asm_inline volatile(						\
95 		ALTERNATIVE(_lock_loc					\
96 			    "call cmpxchg8b_emu",			\
97 			    _lock "cmpxchg8b %a[ptr]", X86_FEATURE_CX8)	\
98 		: ALT_OUTPUT_SP("+a" (o.low), "+d" (o.high))		\
99 		: "b" (n.low), "c" (n.high),				\
100 		  [ptr] "S" (_ptr)					\
101 		: "memory");						\
102 									\
103 	o.full;								\
104 })
105 
arch_cmpxchg64(volatile u64 * ptr,u64 old,u64 new)106 static __always_inline u64 arch_cmpxchg64(volatile u64 *ptr, u64 old, u64 new)
107 {
108 	return __arch_cmpxchg64_emu(ptr, old, new, LOCK_PREFIX_HERE, "lock ");
109 }
110 #define arch_cmpxchg64 arch_cmpxchg64
111 
arch_cmpxchg64_local(volatile u64 * ptr,u64 old,u64 new)112 static __always_inline u64 arch_cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new)
113 {
114 	return __arch_cmpxchg64_emu(ptr, old, new, ,);
115 }
116 #define arch_cmpxchg64_local arch_cmpxchg64_local
117 
118 #define __arch_try_cmpxchg64_emu(_ptr, _oldp, _new, _lock_loc, _lock)	\
119 ({									\
120 	union __u64_halves o = { .full = *(_oldp), },			\
121 			   n = { .full = (_new), };			\
122 	bool ret;							\
123 									\
124 	asm_inline volatile(						\
125 		ALTERNATIVE(_lock_loc					\
126 			    "call cmpxchg8b_emu",			\
127 			    _lock "cmpxchg8b %a[ptr]", X86_FEATURE_CX8) \
128 		CC_SET(e)						\
129 		: ALT_OUTPUT_SP(CC_OUT(e) (ret),			\
130 				"+a" (o.low), "+d" (o.high))		\
131 		: "b" (n.low), "c" (n.high),				\
132 		  [ptr] "S" (_ptr)					\
133 		: "memory");						\
134 									\
135 	if (unlikely(!ret))						\
136 		*(_oldp) = o.full;					\
137 									\
138 	likely(ret);							\
139 })
140 
arch_try_cmpxchg64(volatile u64 * ptr,u64 * oldp,u64 new)141 static __always_inline bool arch_try_cmpxchg64(volatile u64 *ptr, u64 *oldp, u64 new)
142 {
143 	return __arch_try_cmpxchg64_emu(ptr, oldp, new, LOCK_PREFIX_HERE, "lock ");
144 }
145 #define arch_try_cmpxchg64 arch_try_cmpxchg64
146 
arch_try_cmpxchg64_local(volatile u64 * ptr,u64 * oldp,u64 new)147 static __always_inline bool arch_try_cmpxchg64_local(volatile u64 *ptr, u64 *oldp, u64 new)
148 {
149 	return __arch_try_cmpxchg64_emu(ptr, oldp, new, ,);
150 }
151 #define arch_try_cmpxchg64_local arch_try_cmpxchg64_local
152 
153 #endif
154 
155 #define system_has_cmpxchg64()		boot_cpu_has(X86_FEATURE_CX8)
156 
157 #endif /* _ASM_X86_CMPXCHG_32_H */
158