1 /*
2 * Copyright 2005, Red Hat, Inc., Ingo Molnar
3 * Released under the General Public License (GPL).
4 *
5 * This file contains the spinlock/rwlock implementations for
6 * DEBUG_SPINLOCK.
7 */
8
9 #include <linux/spinlock.h>
10 #include <linux/nmi.h>
11 #include <linux/interrupt.h>
12 #include <linux/debug_locks.h>
13 #include <linux/delay.h>
14 #include <linux/module.h>
15
__raw_spin_lock_init(raw_spinlock_t * lock,const char * name,struct lock_class_key * key)16 void __raw_spin_lock_init(raw_spinlock_t *lock, const char *name,
17 struct lock_class_key *key)
18 {
19 #ifdef CONFIG_DEBUG_LOCK_ALLOC
20 /*
21 * Make sure we are not reinitializing a held lock:
22 */
23 debug_check_no_locks_freed((void *)lock, sizeof(*lock));
24 lockdep_init_map(&lock->dep_map, name, key, 0);
25 #endif
26 lock->raw_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
27 lock->magic = SPINLOCK_MAGIC;
28 lock->owner = SPINLOCK_OWNER_INIT;
29 lock->owner_cpu = -1;
30 }
31
32 EXPORT_SYMBOL(__raw_spin_lock_init);
33
__rwlock_init(rwlock_t * lock,const char * name,struct lock_class_key * key)34 void __rwlock_init(rwlock_t *lock, const char *name,
35 struct lock_class_key *key)
36 {
37 #ifdef CONFIG_DEBUG_LOCK_ALLOC
38 /*
39 * Make sure we are not reinitializing a held lock:
40 */
41 debug_check_no_locks_freed((void *)lock, sizeof(*lock));
42 lockdep_init_map(&lock->dep_map, name, key, 0);
43 #endif
44 lock->raw_lock = (arch_rwlock_t) __ARCH_RW_LOCK_UNLOCKED;
45 lock->magic = RWLOCK_MAGIC;
46 lock->owner = SPINLOCK_OWNER_INIT;
47 lock->owner_cpu = -1;
48 }
49
50 EXPORT_SYMBOL(__rwlock_init);
51
spin_dump(raw_spinlock_t * lock,const char * msg)52 static void spin_dump(raw_spinlock_t *lock, const char *msg)
53 {
54 struct task_struct *owner = NULL;
55
56 if (lock->owner && lock->owner != SPINLOCK_OWNER_INIT)
57 owner = lock->owner;
58 printk(KERN_EMERG "BUG: spinlock %s on CPU#%d, %s/%d\n",
59 msg, raw_smp_processor_id(),
60 current->comm, task_pid_nr(current));
61 printk(KERN_EMERG " lock: %p, .magic: %08x, .owner: %s/%d, "
62 ".owner_cpu: %d\n",
63 lock, lock->magic,
64 owner ? owner->comm : "<none>",
65 owner ? task_pid_nr(owner) : -1,
66 lock->owner_cpu);
67 dump_stack();
68 }
69
spin_bug(raw_spinlock_t * lock,const char * msg)70 static void spin_bug(raw_spinlock_t *lock, const char *msg)
71 {
72 if (!debug_locks_off())
73 return;
74
75 spin_dump(lock, msg);
76 }
77
78 #define SPIN_BUG_ON(cond, lock, msg) if (unlikely(cond)) spin_bug(lock, msg)
79
80 static inline void
debug_spin_lock_before(raw_spinlock_t * lock)81 debug_spin_lock_before(raw_spinlock_t *lock)
82 {
83 SPIN_BUG_ON(lock->magic != SPINLOCK_MAGIC, lock, "bad magic");
84 SPIN_BUG_ON(lock->owner == current, lock, "recursion");
85 SPIN_BUG_ON(lock->owner_cpu == raw_smp_processor_id(),
86 lock, "cpu recursion");
87 }
88
debug_spin_lock_after(raw_spinlock_t * lock)89 static inline void debug_spin_lock_after(raw_spinlock_t *lock)
90 {
91 lock->owner_cpu = raw_smp_processor_id();
92 lock->owner = current;
93 }
94
debug_spin_unlock(raw_spinlock_t * lock)95 static inline void debug_spin_unlock(raw_spinlock_t *lock)
96 {
97 SPIN_BUG_ON(lock->magic != SPINLOCK_MAGIC, lock, "bad magic");
98 SPIN_BUG_ON(!raw_spin_is_locked(lock), lock, "already unlocked");
99 SPIN_BUG_ON(lock->owner != current, lock, "wrong owner");
100 SPIN_BUG_ON(lock->owner_cpu != raw_smp_processor_id(),
101 lock, "wrong CPU");
102 lock->owner = SPINLOCK_OWNER_INIT;
103 lock->owner_cpu = -1;
104 }
105
__spin_lock_debug(raw_spinlock_t * lock)106 static void __spin_lock_debug(raw_spinlock_t *lock)
107 {
108 u64 i;
109 u64 loops = loops_per_jiffy * HZ;
110 int print_once = 1;
111
112 for (;;) {
113 for (i = 0; i < loops; i++) {
114 if (arch_spin_trylock(&lock->raw_lock))
115 return;
116 __delay(1);
117 }
118 /* lockup suspected: */
119 if (print_once) {
120 print_once = 0;
121 spin_dump(lock, "lockup");
122 #ifdef CONFIG_SMP
123 trigger_all_cpu_backtrace();
124 #endif
125 }
126 }
127 }
128
do_raw_spin_lock(raw_spinlock_t * lock)129 void do_raw_spin_lock(raw_spinlock_t *lock)
130 {
131 debug_spin_lock_before(lock);
132 if (unlikely(!arch_spin_trylock(&lock->raw_lock)))
133 __spin_lock_debug(lock);
134 debug_spin_lock_after(lock);
135 }
136
do_raw_spin_trylock(raw_spinlock_t * lock)137 int do_raw_spin_trylock(raw_spinlock_t *lock)
138 {
139 int ret = arch_spin_trylock(&lock->raw_lock);
140
141 if (ret)
142 debug_spin_lock_after(lock);
143 #ifndef CONFIG_SMP
144 /*
145 * Must not happen on UP:
146 */
147 SPIN_BUG_ON(!ret, lock, "trylock failure on UP");
148 #endif
149 return ret;
150 }
151
do_raw_spin_unlock(raw_spinlock_t * lock)152 void do_raw_spin_unlock(raw_spinlock_t *lock)
153 {
154 debug_spin_unlock(lock);
155 arch_spin_unlock(&lock->raw_lock);
156 }
157
rwlock_bug(rwlock_t * lock,const char * msg)158 static void rwlock_bug(rwlock_t *lock, const char *msg)
159 {
160 if (!debug_locks_off())
161 return;
162
163 printk(KERN_EMERG "BUG: rwlock %s on CPU#%d, %s/%d, %p\n",
164 msg, raw_smp_processor_id(), current->comm,
165 task_pid_nr(current), lock);
166 dump_stack();
167 }
168
169 #define RWLOCK_BUG_ON(cond, lock, msg) if (unlikely(cond)) rwlock_bug(lock, msg)
170
171 #if 0 /* __write_lock_debug() can lock up - maybe this can too? */
172 static void __read_lock_debug(rwlock_t *lock)
173 {
174 u64 i;
175 u64 loops = loops_per_jiffy * HZ;
176 int print_once = 1;
177
178 for (;;) {
179 for (i = 0; i < loops; i++) {
180 if (arch_read_trylock(&lock->raw_lock))
181 return;
182 __delay(1);
183 }
184 /* lockup suspected: */
185 if (print_once) {
186 print_once = 0;
187 printk(KERN_EMERG "BUG: read-lock lockup on CPU#%d, "
188 "%s/%d, %p\n",
189 raw_smp_processor_id(), current->comm,
190 current->pid, lock);
191 dump_stack();
192 }
193 }
194 }
195 #endif
196
do_raw_read_lock(rwlock_t * lock)197 void do_raw_read_lock(rwlock_t *lock)
198 {
199 RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic");
200 arch_read_lock(&lock->raw_lock);
201 }
202
do_raw_read_trylock(rwlock_t * lock)203 int do_raw_read_trylock(rwlock_t *lock)
204 {
205 int ret = arch_read_trylock(&lock->raw_lock);
206
207 #ifndef CONFIG_SMP
208 /*
209 * Must not happen on UP:
210 */
211 RWLOCK_BUG_ON(!ret, lock, "trylock failure on UP");
212 #endif
213 return ret;
214 }
215
do_raw_read_unlock(rwlock_t * lock)216 void do_raw_read_unlock(rwlock_t *lock)
217 {
218 RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic");
219 arch_read_unlock(&lock->raw_lock);
220 }
221
debug_write_lock_before(rwlock_t * lock)222 static inline void debug_write_lock_before(rwlock_t *lock)
223 {
224 RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic");
225 RWLOCK_BUG_ON(lock->owner == current, lock, "recursion");
226 RWLOCK_BUG_ON(lock->owner_cpu == raw_smp_processor_id(),
227 lock, "cpu recursion");
228 }
229
debug_write_lock_after(rwlock_t * lock)230 static inline void debug_write_lock_after(rwlock_t *lock)
231 {
232 lock->owner_cpu = raw_smp_processor_id();
233 lock->owner = current;
234 }
235
debug_write_unlock(rwlock_t * lock)236 static inline void debug_write_unlock(rwlock_t *lock)
237 {
238 RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic");
239 RWLOCK_BUG_ON(lock->owner != current, lock, "wrong owner");
240 RWLOCK_BUG_ON(lock->owner_cpu != raw_smp_processor_id(),
241 lock, "wrong CPU");
242 lock->owner = SPINLOCK_OWNER_INIT;
243 lock->owner_cpu = -1;
244 }
245
246 #if 0 /* This can cause lockups */
247 static void __write_lock_debug(rwlock_t *lock)
248 {
249 u64 i;
250 u64 loops = loops_per_jiffy * HZ;
251 int print_once = 1;
252
253 for (;;) {
254 for (i = 0; i < loops; i++) {
255 if (arch_write_trylock(&lock->raw_lock))
256 return;
257 __delay(1);
258 }
259 /* lockup suspected: */
260 if (print_once) {
261 print_once = 0;
262 printk(KERN_EMERG "BUG: write-lock lockup on CPU#%d, "
263 "%s/%d, %p\n",
264 raw_smp_processor_id(), current->comm,
265 current->pid, lock);
266 dump_stack();
267 }
268 }
269 }
270 #endif
271
do_raw_write_lock(rwlock_t * lock)272 void do_raw_write_lock(rwlock_t *lock)
273 {
274 debug_write_lock_before(lock);
275 arch_write_lock(&lock->raw_lock);
276 debug_write_lock_after(lock);
277 }
278
do_raw_write_trylock(rwlock_t * lock)279 int do_raw_write_trylock(rwlock_t *lock)
280 {
281 int ret = arch_write_trylock(&lock->raw_lock);
282
283 if (ret)
284 debug_write_lock_after(lock);
285 #ifndef CONFIG_SMP
286 /*
287 * Must not happen on UP:
288 */
289 RWLOCK_BUG_ON(!ret, lock, "trylock failure on UP");
290 #endif
291 return ret;
292 }
293
do_raw_write_unlock(rwlock_t * lock)294 void do_raw_write_unlock(rwlock_t *lock)
295 {
296 debug_write_unlock(lock);
297 arch_write_unlock(&lock->raw_lock);
298 }
299