1 /* rwsem-spinlock.c: R/W semaphores: contention handling functions for
2 * generic spinlock implementation
3 *
4 * Copyright (c) 2001 David Howells (dhowells@redhat.com).
5 * - Derived partially from idea by Andrea Arcangeli <andrea@suse.de>
6 * - Derived also from comments by Linus
7 */
8 #include <linux/rwsem.h>
9 #include <linux/sched.h>
10 #include <linux/module.h>
11
12 struct rwsem_waiter {
13 struct list_head list;
14 struct task_struct *task;
15 unsigned int flags;
16 #define RWSEM_WAITING_FOR_READ 0x00000001
17 #define RWSEM_WAITING_FOR_WRITE 0x00000002
18 };
19
rwsem_is_locked(struct rw_semaphore * sem)20 int rwsem_is_locked(struct rw_semaphore *sem)
21 {
22 int ret = 1;
23 unsigned long flags;
24
25 if (raw_spin_trylock_irqsave(&sem->wait_lock, flags)) {
26 ret = (sem->activity != 0);
27 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
28 }
29 return ret;
30 }
31 EXPORT_SYMBOL(rwsem_is_locked);
32
33 /*
34 * initialise the semaphore
35 */
__init_rwsem(struct rw_semaphore * sem,const char * name,struct lock_class_key * key)36 void __init_rwsem(struct rw_semaphore *sem, const char *name,
37 struct lock_class_key *key)
38 {
39 #ifdef CONFIG_DEBUG_LOCK_ALLOC
40 /*
41 * Make sure we are not reinitializing a held semaphore:
42 */
43 debug_check_no_locks_freed((void *)sem, sizeof(*sem));
44 lockdep_init_map(&sem->dep_map, name, key, 0);
45 #endif
46 sem->activity = 0;
47 raw_spin_lock_init(&sem->wait_lock);
48 INIT_LIST_HEAD(&sem->wait_list);
49 }
50 EXPORT_SYMBOL(__init_rwsem);
51
52 /*
53 * handle the lock release when processes blocked on it that can now run
54 * - if we come here, then:
55 * - the 'active count' _reached_ zero
56 * - the 'waiting count' is non-zero
57 * - the spinlock must be held by the caller
58 * - woken process blocks are discarded from the list after having task zeroed
59 * - writers are only woken if wakewrite is non-zero
60 */
61 static inline struct rw_semaphore *
__rwsem_do_wake(struct rw_semaphore * sem,int wakewrite)62 __rwsem_do_wake(struct rw_semaphore *sem, int wakewrite)
63 {
64 struct rwsem_waiter *waiter;
65 struct task_struct *tsk;
66 int woken;
67
68 waiter = list_entry(sem->wait_list.next, struct rwsem_waiter, list);
69
70 if (!wakewrite) {
71 if (waiter->flags & RWSEM_WAITING_FOR_WRITE)
72 goto out;
73 goto dont_wake_writers;
74 }
75
76 /* if we are allowed to wake writers try to grant a single write lock
77 * if there's a writer at the front of the queue
78 * - we leave the 'waiting count' incremented to signify potential
79 * contention
80 */
81 if (waiter->flags & RWSEM_WAITING_FOR_WRITE) {
82 sem->activity = -1;
83 list_del(&waiter->list);
84 tsk = waiter->task;
85 /* Don't touch waiter after ->task has been NULLed */
86 smp_mb();
87 waiter->task = NULL;
88 wake_up_process(tsk);
89 put_task_struct(tsk);
90 goto out;
91 }
92
93 /* grant an infinite number of read locks to the front of the queue */
94 dont_wake_writers:
95 woken = 0;
96 while (waiter->flags & RWSEM_WAITING_FOR_READ) {
97 struct list_head *next = waiter->list.next;
98
99 list_del(&waiter->list);
100 tsk = waiter->task;
101 smp_mb();
102 waiter->task = NULL;
103 wake_up_process(tsk);
104 put_task_struct(tsk);
105 woken++;
106 if (list_empty(&sem->wait_list))
107 break;
108 waiter = list_entry(next, struct rwsem_waiter, list);
109 }
110
111 sem->activity += woken;
112
113 out:
114 return sem;
115 }
116
117 /*
118 * wake a single writer
119 */
120 static inline struct rw_semaphore *
__rwsem_wake_one_writer(struct rw_semaphore * sem)121 __rwsem_wake_one_writer(struct rw_semaphore *sem)
122 {
123 struct rwsem_waiter *waiter;
124 struct task_struct *tsk;
125
126 sem->activity = -1;
127
128 waiter = list_entry(sem->wait_list.next, struct rwsem_waiter, list);
129 list_del(&waiter->list);
130
131 tsk = waiter->task;
132 smp_mb();
133 waiter->task = NULL;
134 wake_up_process(tsk);
135 put_task_struct(tsk);
136 return sem;
137 }
138
139 /*
140 * get a read lock on the semaphore
141 */
__down_read(struct rw_semaphore * sem)142 void __sched __down_read(struct rw_semaphore *sem)
143 {
144 struct rwsem_waiter waiter;
145 struct task_struct *tsk;
146 unsigned long flags;
147
148 raw_spin_lock_irqsave(&sem->wait_lock, flags);
149
150 if (sem->activity >= 0 && list_empty(&sem->wait_list)) {
151 /* granted */
152 sem->activity++;
153 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
154 goto out;
155 }
156
157 tsk = current;
158 set_task_state(tsk, TASK_UNINTERRUPTIBLE);
159
160 /* set up my own style of waitqueue */
161 waiter.task = tsk;
162 waiter.flags = RWSEM_WAITING_FOR_READ;
163 get_task_struct(tsk);
164
165 list_add_tail(&waiter.list, &sem->wait_list);
166
167 /* we don't need to touch the semaphore struct anymore */
168 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
169
170 /* wait to be given the lock */
171 for (;;) {
172 if (!waiter.task)
173 break;
174 schedule();
175 set_task_state(tsk, TASK_UNINTERRUPTIBLE);
176 }
177
178 tsk->state = TASK_RUNNING;
179 out:
180 ;
181 }
182
183 /*
184 * trylock for reading -- returns 1 if successful, 0 if contention
185 */
__down_read_trylock(struct rw_semaphore * sem)186 int __down_read_trylock(struct rw_semaphore *sem)
187 {
188 unsigned long flags;
189 int ret = 0;
190
191
192 raw_spin_lock_irqsave(&sem->wait_lock, flags);
193
194 if (sem->activity >= 0 && list_empty(&sem->wait_list)) {
195 /* granted */
196 sem->activity++;
197 ret = 1;
198 }
199
200 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
201
202 return ret;
203 }
204
205 /*
206 * get a write lock on the semaphore
207 * - we increment the waiting count anyway to indicate an exclusive lock
208 */
__down_write_nested(struct rw_semaphore * sem,int subclass)209 void __sched __down_write_nested(struct rw_semaphore *sem, int subclass)
210 {
211 struct rwsem_waiter waiter;
212 struct task_struct *tsk;
213 unsigned long flags;
214
215 raw_spin_lock_irqsave(&sem->wait_lock, flags);
216
217 if (sem->activity == 0 && list_empty(&sem->wait_list)) {
218 /* granted */
219 sem->activity = -1;
220 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
221 goto out;
222 }
223
224 tsk = current;
225 set_task_state(tsk, TASK_UNINTERRUPTIBLE);
226
227 /* set up my own style of waitqueue */
228 waiter.task = tsk;
229 waiter.flags = RWSEM_WAITING_FOR_WRITE;
230 get_task_struct(tsk);
231
232 list_add_tail(&waiter.list, &sem->wait_list);
233
234 /* we don't need to touch the semaphore struct anymore */
235 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
236
237 /* wait to be given the lock */
238 for (;;) {
239 if (!waiter.task)
240 break;
241 schedule();
242 set_task_state(tsk, TASK_UNINTERRUPTIBLE);
243 }
244
245 tsk->state = TASK_RUNNING;
246 out:
247 ;
248 }
249
__down_write(struct rw_semaphore * sem)250 void __sched __down_write(struct rw_semaphore *sem)
251 {
252 __down_write_nested(sem, 0);
253 }
254
255 /*
256 * trylock for writing -- returns 1 if successful, 0 if contention
257 */
__down_write_trylock(struct rw_semaphore * sem)258 int __down_write_trylock(struct rw_semaphore *sem)
259 {
260 unsigned long flags;
261 int ret = 0;
262
263 raw_spin_lock_irqsave(&sem->wait_lock, flags);
264
265 if (sem->activity == 0 && list_empty(&sem->wait_list)) {
266 /* granted */
267 sem->activity = -1;
268 ret = 1;
269 }
270
271 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
272
273 return ret;
274 }
275
276 /*
277 * release a read lock on the semaphore
278 */
__up_read(struct rw_semaphore * sem)279 void __up_read(struct rw_semaphore *sem)
280 {
281 unsigned long flags;
282
283 raw_spin_lock_irqsave(&sem->wait_lock, flags);
284
285 if (--sem->activity == 0 && !list_empty(&sem->wait_list))
286 sem = __rwsem_wake_one_writer(sem);
287
288 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
289 }
290
291 /*
292 * release a write lock on the semaphore
293 */
__up_write(struct rw_semaphore * sem)294 void __up_write(struct rw_semaphore *sem)
295 {
296 unsigned long flags;
297
298 raw_spin_lock_irqsave(&sem->wait_lock, flags);
299
300 sem->activity = 0;
301 if (!list_empty(&sem->wait_list))
302 sem = __rwsem_do_wake(sem, 1);
303
304 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
305 }
306
307 /*
308 * downgrade a write lock into a read lock
309 * - just wake up any readers at the front of the queue
310 */
__downgrade_write(struct rw_semaphore * sem)311 void __downgrade_write(struct rw_semaphore *sem)
312 {
313 unsigned long flags;
314
315 raw_spin_lock_irqsave(&sem->wait_lock, flags);
316
317 sem->activity = 1;
318 if (!list_empty(&sem->wait_list))
319 sem = __rwsem_do_wake(sem, 0);
320
321 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
322 }
323
324