1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
4 */
5 #ifndef __ASM_PERCPU_H
6 #define __ASM_PERCPU_H
7
8 #include <asm/cmpxchg.h>
9 #include <asm/loongarch.h>
10
11 /*
12 * The "address" (in fact, offset from $r21) of a per-CPU variable is close to
13 * the loading address of main kernel image, but far from where the modules are
14 * loaded. Tell the compiler this fact when using explicit relocs.
15 */
16 #if defined(MODULE) && defined(CONFIG_AS_HAS_EXPLICIT_RELOCS)
17 # if __has_attribute(model)
18 # define PER_CPU_ATTRIBUTES __attribute__((model("extreme")))
19 # else
20 # error compiler support for the model attribute is necessary when a recent assembler is used
21 # endif
22 #endif
23
24 /* Use r21 for fast access */
25 register unsigned long __my_cpu_offset __asm__("$r21");
26
set_my_cpu_offset(unsigned long off)27 static inline void set_my_cpu_offset(unsigned long off)
28 {
29 __my_cpu_offset = off;
30 csr_write64(off, PERCPU_BASE_KS);
31 }
32 #define __my_cpu_offset __my_cpu_offset
33
34 #define PERCPU_OP(op, asm_op, c_op) \
35 static __always_inline unsigned long __percpu_##op(void *ptr, \
36 unsigned long val, int size) \
37 { \
38 unsigned long ret; \
39 \
40 switch (size) { \
41 case 4: \
42 __asm__ __volatile__( \
43 "am"#asm_op".w" " %[ret], %[val], %[ptr] \n" \
44 : [ret] "=&r" (ret), [ptr] "+ZB"(*(u32 *)ptr) \
45 : [val] "r" (val)); \
46 break; \
47 case 8: \
48 __asm__ __volatile__( \
49 "am"#asm_op".d" " %[ret], %[val], %[ptr] \n" \
50 : [ret] "=&r" (ret), [ptr] "+ZB"(*(u64 *)ptr) \
51 : [val] "r" (val)); \
52 break; \
53 default: \
54 ret = 0; \
55 BUILD_BUG(); \
56 } \
57 \
58 return ret c_op val; \
59 }
60
61 PERCPU_OP(add, add, +)
62 PERCPU_OP(and, and, &)
63 PERCPU_OP(or, or, |)
64 #undef PERCPU_OP
65
__percpu_read(void __percpu * ptr,int size)66 static __always_inline unsigned long __percpu_read(void __percpu *ptr, int size)
67 {
68 unsigned long ret;
69
70 switch (size) {
71 case 1:
72 __asm__ __volatile__ ("ldx.b %[ret], $r21, %[ptr] \n"
73 : [ret] "=&r"(ret)
74 : [ptr] "r"(ptr)
75 : "memory");
76 break;
77 case 2:
78 __asm__ __volatile__ ("ldx.h %[ret], $r21, %[ptr] \n"
79 : [ret] "=&r"(ret)
80 : [ptr] "r"(ptr)
81 : "memory");
82 break;
83 case 4:
84 __asm__ __volatile__ ("ldx.w %[ret], $r21, %[ptr] \n"
85 : [ret] "=&r"(ret)
86 : [ptr] "r"(ptr)
87 : "memory");
88 break;
89 case 8:
90 __asm__ __volatile__ ("ldx.d %[ret], $r21, %[ptr] \n"
91 : [ret] "=&r"(ret)
92 : [ptr] "r"(ptr)
93 : "memory");
94 break;
95 default:
96 ret = 0;
97 BUILD_BUG();
98 }
99
100 return ret;
101 }
102
__percpu_write(void __percpu * ptr,unsigned long val,int size)103 static __always_inline void __percpu_write(void __percpu *ptr, unsigned long val, int size)
104 {
105 switch (size) {
106 case 1:
107 __asm__ __volatile__("stx.b %[val], $r21, %[ptr] \n"
108 :
109 : [val] "r" (val), [ptr] "r" (ptr)
110 : "memory");
111 break;
112 case 2:
113 __asm__ __volatile__("stx.h %[val], $r21, %[ptr] \n"
114 :
115 : [val] "r" (val), [ptr] "r" (ptr)
116 : "memory");
117 break;
118 case 4:
119 __asm__ __volatile__("stx.w %[val], $r21, %[ptr] \n"
120 :
121 : [val] "r" (val), [ptr] "r" (ptr)
122 : "memory");
123 break;
124 case 8:
125 __asm__ __volatile__("stx.d %[val], $r21, %[ptr] \n"
126 :
127 : [val] "r" (val), [ptr] "r" (ptr)
128 : "memory");
129 break;
130 default:
131 BUILD_BUG();
132 }
133 }
134
__percpu_xchg(void * ptr,unsigned long val,int size)135 static __always_inline unsigned long __percpu_xchg(void *ptr, unsigned long val, int size)
136 {
137 switch (size) {
138 case 1:
139 case 2:
140 return __xchg_small((volatile void *)ptr, val, size);
141
142 case 4:
143 return __xchg_asm("amswap.w", (volatile u32 *)ptr, (u32)val);
144
145 case 8:
146 return __xchg_asm("amswap.d", (volatile u64 *)ptr, (u64)val);
147
148 default:
149 BUILD_BUG();
150 }
151
152 return 0;
153 }
154
155 /* this_cpu_cmpxchg */
156 #define _protect_cmpxchg_local(pcp, o, n) \
157 ({ \
158 typeof(*raw_cpu_ptr(&(pcp))) __ret; \
159 preempt_disable_notrace(); \
160 __ret = cmpxchg_local(raw_cpu_ptr(&(pcp)), o, n); \
161 preempt_enable_notrace(); \
162 __ret; \
163 })
164
165 #define _percpu_read(pcp) \
166 ({ \
167 typeof(pcp) __retval; \
168 __retval = (typeof(pcp))__percpu_read(&(pcp), sizeof(pcp)); \
169 __retval; \
170 })
171
172 #define _percpu_write(pcp, val) \
173 do { \
174 __percpu_write(&(pcp), (unsigned long)(val), sizeof(pcp)); \
175 } while (0) \
176
177 #define _pcp_protect(operation, pcp, val) \
178 ({ \
179 typeof(pcp) __retval; \
180 preempt_disable_notrace(); \
181 __retval = (typeof(pcp))operation(raw_cpu_ptr(&(pcp)), \
182 (val), sizeof(pcp)); \
183 preempt_enable_notrace(); \
184 __retval; \
185 })
186
187 #define _percpu_add(pcp, val) \
188 _pcp_protect(__percpu_add, pcp, val)
189
190 #define _percpu_add_return(pcp, val) _percpu_add(pcp, val)
191
192 #define _percpu_and(pcp, val) \
193 _pcp_protect(__percpu_and, pcp, val)
194
195 #define _percpu_or(pcp, val) \
196 _pcp_protect(__percpu_or, pcp, val)
197
198 #define _percpu_xchg(pcp, val) ((typeof(pcp)) \
199 _pcp_protect(__percpu_xchg, pcp, (unsigned long)(val)))
200
201 #define this_cpu_add_4(pcp, val) _percpu_add(pcp, val)
202 #define this_cpu_add_8(pcp, val) _percpu_add(pcp, val)
203
204 #define this_cpu_add_return_4(pcp, val) _percpu_add_return(pcp, val)
205 #define this_cpu_add_return_8(pcp, val) _percpu_add_return(pcp, val)
206
207 #define this_cpu_and_4(pcp, val) _percpu_and(pcp, val)
208 #define this_cpu_and_8(pcp, val) _percpu_and(pcp, val)
209
210 #define this_cpu_or_4(pcp, val) _percpu_or(pcp, val)
211 #define this_cpu_or_8(pcp, val) _percpu_or(pcp, val)
212
213 #define this_cpu_read_1(pcp) _percpu_read(pcp)
214 #define this_cpu_read_2(pcp) _percpu_read(pcp)
215 #define this_cpu_read_4(pcp) _percpu_read(pcp)
216 #define this_cpu_read_8(pcp) _percpu_read(pcp)
217
218 #define this_cpu_write_1(pcp, val) _percpu_write(pcp, val)
219 #define this_cpu_write_2(pcp, val) _percpu_write(pcp, val)
220 #define this_cpu_write_4(pcp, val) _percpu_write(pcp, val)
221 #define this_cpu_write_8(pcp, val) _percpu_write(pcp, val)
222
223 #define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val)
224 #define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val)
225 #define this_cpu_xchg_4(pcp, val) _percpu_xchg(pcp, val)
226 #define this_cpu_xchg_8(pcp, val) _percpu_xchg(pcp, val)
227
228 #define this_cpu_cmpxchg_1(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
229 #define this_cpu_cmpxchg_2(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
230 #define this_cpu_cmpxchg_4(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
231 #define this_cpu_cmpxchg_8(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
232
233 #include <asm-generic/percpu.h>
234
235 #endif /* __ASM_PERCPU_H */
236