Lines Matching refs:sem
69 # define DEBUG_RWSEMS_WARN_ON(c, sem) do { \ argument
72 #c, atomic_long_read(&(sem)->count), \
73 (unsigned long) sem->magic, \
74 atomic_long_read(&(sem)->owner), (long)current, \
75 list_empty(&(sem)->wait_list) ? "" : "not ")) \
79 # define DEBUG_RWSEMS_WARN_ON(c, sem) argument
141 static inline void rwsem_set_owner(struct rw_semaphore *sem) in rwsem_set_owner() argument
144 atomic_long_set(&sem->owner, (long)current); in rwsem_set_owner()
147 static inline void rwsem_clear_owner(struct rw_semaphore *sem) in rwsem_clear_owner() argument
150 atomic_long_set(&sem->owner, 0); in rwsem_clear_owner()
156 static inline bool rwsem_test_oflags(struct rw_semaphore *sem, long flags) in rwsem_test_oflags() argument
158 return atomic_long_read(&sem->owner) & flags; in rwsem_test_oflags()
171 static inline void __rwsem_set_reader_owned(struct rw_semaphore *sem, in __rwsem_set_reader_owned() argument
175 (atomic_long_read(&sem->owner) & RWSEM_NONSPINNABLE); in __rwsem_set_reader_owned()
177 atomic_long_set(&sem->owner, val); in __rwsem_set_reader_owned()
180 static inline void rwsem_set_reader_owned(struct rw_semaphore *sem) in rwsem_set_reader_owned() argument
182 __rwsem_set_reader_owned(sem, current); in rwsem_set_reader_owned()
189 struct task_struct *rwsem_owner(struct rw_semaphore *sem) in rwsem_owner() argument
192 (atomic_long_read(&sem->owner) & ~RWSEM_OWNER_FLAGS_MASK); in rwsem_owner()
198 bool is_rwsem_reader_owned(struct rw_semaphore *sem) in is_rwsem_reader_owned() argument
203 long count = atomic_long_read(&sem->count); in is_rwsem_reader_owned()
207 return rwsem_test_oflags(sem, RWSEM_READER_OWNED); in is_rwsem_reader_owned()
216 static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem) in rwsem_clear_reader_owned() argument
218 unsigned long val = atomic_long_read(&sem->owner); in rwsem_clear_reader_owned()
221 if (atomic_long_try_cmpxchg(&sem->owner, &val, in rwsem_clear_reader_owned()
227 static inline void rwsem_clear_reader_owned(struct rw_semaphore *sem) in rwsem_clear_reader_owned() argument
236 static inline void rwsem_set_nonspinnable(struct rw_semaphore *sem) in rwsem_set_nonspinnable() argument
238 unsigned long owner = atomic_long_read(&sem->owner); in rwsem_set_nonspinnable()
245 } while (!atomic_long_try_cmpxchg(&sem->owner, &owner, in rwsem_set_nonspinnable()
249 static inline bool rwsem_read_trylock(struct rw_semaphore *sem, long *cntp) in rwsem_read_trylock() argument
251 *cntp = atomic_long_add_return_acquire(RWSEM_READER_BIAS, &sem->count); in rwsem_read_trylock()
254 rwsem_set_nonspinnable(sem); in rwsem_read_trylock()
257 rwsem_set_reader_owned(sem); in rwsem_read_trylock()
264 static inline bool rwsem_write_trylock(struct rw_semaphore *sem) in rwsem_write_trylock() argument
268 if (atomic_long_try_cmpxchg_acquire(&sem->count, &tmp, RWSEM_WRITER_LOCKED)) { in rwsem_write_trylock()
269 rwsem_set_owner(sem); in rwsem_write_trylock()
281 rwsem_owner_flags(struct rw_semaphore *sem, unsigned long *pflags) in rwsem_owner_flags() argument
283 unsigned long owner = atomic_long_read(&sem->owner); in rwsem_owner_flags()
309 void __init_rwsem(struct rw_semaphore *sem, const char *name, in __init_rwsem() argument
316 debug_check_no_locks_freed((void *)sem, sizeof(*sem)); in __init_rwsem()
317 lockdep_init_map_wait(&sem->dep_map, name, key, 0, LD_WAIT_SLEEP); in __init_rwsem()
320 sem->magic = sem; in __init_rwsem()
322 atomic_long_set(&sem->count, RWSEM_UNLOCKED_VALUE); in __init_rwsem()
323 raw_spin_lock_init(&sem->wait_lock); in __init_rwsem()
324 INIT_LIST_HEAD(&sem->wait_list); in __init_rwsem()
325 atomic_long_set(&sem->owner, 0L); in __init_rwsem()
327 osq_lock_init(&sem->osq); in __init_rwsem()
344 #define rwsem_first_waiter(sem) \ argument
345 list_first_entry(&sem->wait_list, struct rwsem_waiter, list)
369 rwsem_add_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter) in rwsem_add_waiter() argument
371 lockdep_assert_held(&sem->wait_lock); in rwsem_add_waiter()
372 list_add_tail(&waiter->list, &sem->wait_list); in rwsem_add_waiter()
385 rwsem_del_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter) in rwsem_del_waiter() argument
387 lockdep_assert_held(&sem->wait_lock); in rwsem_del_waiter()
389 if (likely(!list_empty(&sem->wait_list))) in rwsem_del_waiter()
392 atomic_long_andnot(RWSEM_FLAG_HANDOFF | RWSEM_FLAG_WAITERS, &sem->count); in rwsem_del_waiter()
410 static void rwsem_mark_wake(struct rw_semaphore *sem, in rwsem_mark_wake() argument
418 lockdep_assert_held(&sem->wait_lock); in rwsem_mark_wake()
424 waiter = rwsem_first_waiter(sem); in rwsem_mark_wake()
445 if (unlikely(atomic_long_read(&sem->count) < 0)) in rwsem_mark_wake()
457 oldcount = atomic_long_fetch_add(adjustment, &sem->count); in rwsem_mark_wake()
472 atomic_long_add(-adjustment, &sem->count); in rwsem_mark_wake()
482 __rwsem_set_reader_owned(sem, owner); in rwsem_mark_wake()
509 list_for_each_entry_safe(waiter, tmp, &sem->wait_list, list) { in rwsem_mark_wake()
526 oldcount = atomic_long_read(&sem->count); in rwsem_mark_wake()
527 if (list_empty(&sem->wait_list)) { in rwsem_mark_wake()
545 atomic_long_add(adjustment, &sem->count); in rwsem_mark_wake()
576 rwsem_del_wake_waiter(struct rw_semaphore *sem, struct rwsem_waiter *waiter, in rwsem_del_wake_waiter() argument
578 __releases(&sem->wait_lock) in rwsem_del_wake_waiter()
580 bool first = rwsem_first_waiter(sem) == waiter; in rwsem_del_wake_waiter()
589 if (rwsem_del_waiter(sem, waiter) && first) in rwsem_del_wake_waiter()
590 rwsem_mark_wake(sem, RWSEM_WAKE_ANY, wake_q); in rwsem_del_wake_waiter()
591 raw_spin_unlock_irq(&sem->wait_lock); in rwsem_del_wake_waiter()
603 static inline bool rwsem_try_write_lock(struct rw_semaphore *sem, in rwsem_try_write_lock() argument
606 struct rwsem_waiter *first = rwsem_first_waiter(sem); in rwsem_try_write_lock()
609 lockdep_assert_held(&sem->wait_lock); in rwsem_try_write_lock()
611 count = atomic_long_read(&sem->count); in rwsem_try_write_lock()
642 if (list_is_singular(&sem->wait_list)) in rwsem_try_write_lock()
645 } while (!atomic_long_try_cmpxchg_acquire(&sem->count, &count, new)); in rwsem_try_write_lock()
663 rwsem_set_owner(sem); in rwsem_try_write_lock()
689 static inline bool rwsem_try_write_lock_unqueued(struct rw_semaphore *sem) in rwsem_try_write_lock_unqueued() argument
691 long count = atomic_long_read(&sem->count); in rwsem_try_write_lock_unqueued()
694 if (atomic_long_try_cmpxchg_acquire(&sem->count, &count, in rwsem_try_write_lock_unqueued()
696 rwsem_set_owner(sem); in rwsem_try_write_lock_unqueued()
704 static inline bool rwsem_can_spin_on_owner(struct rw_semaphore *sem) in rwsem_can_spin_on_owner() argument
719 owner = rwsem_owner_flags(sem, &flags); in rwsem_can_spin_on_owner()
744 rwsem_spin_on_owner(struct rw_semaphore *sem) in rwsem_spin_on_owner() argument
752 owner = rwsem_owner_flags(sem, &flags); in rwsem_spin_on_owner()
764 new = rwsem_owner_flags(sem, &new_flags); in rwsem_spin_on_owner()
803 static inline u64 rwsem_rspin_threshold(struct rw_semaphore *sem) in rwsem_rspin_threshold() argument
805 long count = atomic_long_read(&sem->count); in rwsem_rspin_threshold()
816 static bool rwsem_optimistic_spin(struct rw_semaphore *sem) in rwsem_optimistic_spin() argument
824 if (!osq_lock(&sem->osq)) in rwsem_optimistic_spin()
836 owner_state = rwsem_spin_on_owner(sem); in rwsem_optimistic_spin()
843 taken = rwsem_try_write_lock_unqueued(sem); in rwsem_optimistic_spin()
860 if (rwsem_test_oflags(sem, RWSEM_NONSPINNABLE)) in rwsem_optimistic_spin()
862 rspin_threshold = rwsem_rspin_threshold(sem); in rwsem_optimistic_spin()
874 rwsem_set_nonspinnable(sem); in rwsem_optimistic_spin()
928 osq_unlock(&sem->osq); in rwsem_optimistic_spin()
938 static inline void clear_nonspinnable(struct rw_semaphore *sem) in clear_nonspinnable() argument
940 if (unlikely(rwsem_test_oflags(sem, RWSEM_NONSPINNABLE))) in clear_nonspinnable()
941 atomic_long_andnot(RWSEM_NONSPINNABLE, &sem->owner); in clear_nonspinnable()
945 static inline bool rwsem_can_spin_on_owner(struct rw_semaphore *sem) in rwsem_can_spin_on_owner() argument
950 static inline bool rwsem_optimistic_spin(struct rw_semaphore *sem) in rwsem_optimistic_spin() argument
955 static inline void clear_nonspinnable(struct rw_semaphore *sem) { } in clear_nonspinnable() argument
958 rwsem_spin_on_owner(struct rw_semaphore *sem) in rwsem_spin_on_owner() argument
972 static inline void rwsem_cond_wake_waiter(struct rw_semaphore *sem, long count, in rwsem_cond_wake_waiter() argument
984 clear_nonspinnable(sem); in rwsem_cond_wake_waiter()
986 rwsem_mark_wake(sem, wake_type, wake_q); in rwsem_cond_wake_waiter()
993 rwsem_down_read_slowpath(struct rw_semaphore *sem, long count, unsigned int state) in rwsem_down_read_slowpath() argument
1005 if ((atomic_long_read(&sem->owner) & RWSEM_READER_OWNED) && in rwsem_down_read_slowpath()
1013 rwsem_set_reader_owned(sem); in rwsem_down_read_slowpath()
1021 raw_spin_lock_irq(&sem->wait_lock); in rwsem_down_read_slowpath()
1022 if (!list_empty(&sem->wait_list)) in rwsem_down_read_slowpath()
1023 rwsem_mark_wake(sem, RWSEM_WAKE_READ_OWNED, in rwsem_down_read_slowpath()
1025 raw_spin_unlock_irq(&sem->wait_lock); in rwsem_down_read_slowpath()
1028 return sem; in rwsem_down_read_slowpath()
1037 raw_spin_lock_irq(&sem->wait_lock); in rwsem_down_read_slowpath()
1038 if (list_empty(&sem->wait_list)) { in rwsem_down_read_slowpath()
1045 if (!(atomic_long_read(&sem->count) & RWSEM_WRITER_MASK)) { in rwsem_down_read_slowpath()
1048 raw_spin_unlock_irq(&sem->wait_lock); in rwsem_down_read_slowpath()
1049 rwsem_set_reader_owned(sem); in rwsem_down_read_slowpath()
1051 return sem; in rwsem_down_read_slowpath()
1055 rwsem_add_waiter(sem, &waiter); in rwsem_down_read_slowpath()
1058 count = atomic_long_add_return(adjustment, &sem->count); in rwsem_down_read_slowpath()
1060 rwsem_cond_wake_waiter(sem, count, &wake_q); in rwsem_down_read_slowpath()
1061 raw_spin_unlock_irq(&sem->wait_lock); in rwsem_down_read_slowpath()
1066 trace_contention_begin(sem, LCB_F_READ); in rwsem_down_read_slowpath()
1070 hung_task_set_blocker(sem, BLOCKER_TYPE_RWSEM_READER); in rwsem_down_read_slowpath()
1079 raw_spin_lock_irq(&sem->wait_lock); in rwsem_down_read_slowpath()
1082 raw_spin_unlock_irq(&sem->wait_lock); in rwsem_down_read_slowpath()
1096 trace_contention_end(sem, 0); in rwsem_down_read_slowpath()
1097 return sem; in rwsem_down_read_slowpath()
1100 rwsem_del_wake_waiter(sem, &waiter, &wake_q); in rwsem_down_read_slowpath()
1103 trace_contention_end(sem, -EINTR); in rwsem_down_read_slowpath()
1111 rwsem_down_write_slowpath(struct rw_semaphore *sem, int state) in rwsem_down_write_slowpath() argument
1117 if (rwsem_can_spin_on_owner(sem) && rwsem_optimistic_spin(sem)) { in rwsem_down_write_slowpath()
1119 return sem; in rwsem_down_write_slowpath()
1131 raw_spin_lock_irq(&sem->wait_lock); in rwsem_down_write_slowpath()
1132 rwsem_add_waiter(sem, &waiter); in rwsem_down_write_slowpath()
1135 if (rwsem_first_waiter(sem) != &waiter) { in rwsem_down_write_slowpath()
1136 rwsem_cond_wake_waiter(sem, atomic_long_read(&sem->count), in rwsem_down_write_slowpath()
1143 raw_spin_unlock_irq(&sem->wait_lock); in rwsem_down_write_slowpath()
1145 raw_spin_lock_irq(&sem->wait_lock); in rwsem_down_write_slowpath()
1148 atomic_long_or(RWSEM_FLAG_WAITERS, &sem->count); in rwsem_down_write_slowpath()
1153 trace_contention_begin(sem, LCB_F_WRITE); in rwsem_down_write_slowpath()
1156 hung_task_set_blocker(sem, BLOCKER_TYPE_RWSEM_WRITER); in rwsem_down_write_slowpath()
1159 if (rwsem_try_write_lock(sem, &waiter)) { in rwsem_down_write_slowpath()
1164 raw_spin_unlock_irq(&sem->wait_lock); in rwsem_down_write_slowpath()
1180 owner_state = rwsem_spin_on_owner(sem); in rwsem_down_write_slowpath()
1189 raw_spin_lock_irq(&sem->wait_lock); in rwsem_down_write_slowpath()
1196 raw_spin_unlock_irq(&sem->wait_lock); in rwsem_down_write_slowpath()
1198 trace_contention_end(sem, 0); in rwsem_down_write_slowpath()
1199 return sem; in rwsem_down_write_slowpath()
1203 raw_spin_lock_irq(&sem->wait_lock); in rwsem_down_write_slowpath()
1204 rwsem_del_wake_waiter(sem, &waiter, &wake_q); in rwsem_down_write_slowpath()
1206 trace_contention_end(sem, -EINTR); in rwsem_down_write_slowpath()
1214 static struct rw_semaphore *rwsem_wake(struct rw_semaphore *sem) in rwsem_wake() argument
1219 raw_spin_lock_irqsave(&sem->wait_lock, flags); in rwsem_wake()
1221 if (!list_empty(&sem->wait_list)) in rwsem_wake()
1222 rwsem_mark_wake(sem, RWSEM_WAKE_ANY, &wake_q); in rwsem_wake()
1224 raw_spin_unlock_irqrestore(&sem->wait_lock, flags); in rwsem_wake()
1227 return sem; in rwsem_wake()
1235 static struct rw_semaphore *rwsem_downgrade_wake(struct rw_semaphore *sem) in rwsem_downgrade_wake() argument
1240 raw_spin_lock_irqsave(&sem->wait_lock, flags); in rwsem_downgrade_wake()
1242 if (!list_empty(&sem->wait_list)) in rwsem_downgrade_wake()
1243 rwsem_mark_wake(sem, RWSEM_WAKE_READ_OWNED, &wake_q); in rwsem_downgrade_wake()
1245 raw_spin_unlock_irqrestore(&sem->wait_lock, flags); in rwsem_downgrade_wake()
1248 return sem; in rwsem_downgrade_wake()
1254 static __always_inline int __down_read_common(struct rw_semaphore *sem, int state) in __down_read_common() argument
1260 if (!rwsem_read_trylock(sem, &count)) { in __down_read_common()
1261 if (IS_ERR(rwsem_down_read_slowpath(sem, count, state))) { in __down_read_common()
1265 DEBUG_RWSEMS_WARN_ON(!is_rwsem_reader_owned(sem), sem); in __down_read_common()
1272 static __always_inline void __down_read(struct rw_semaphore *sem) in __down_read() argument
1274 __down_read_common(sem, TASK_UNINTERRUPTIBLE); in __down_read()
1277 static __always_inline int __down_read_interruptible(struct rw_semaphore *sem) in __down_read_interruptible() argument
1279 return __down_read_common(sem, TASK_INTERRUPTIBLE); in __down_read_interruptible()
1282 static __always_inline int __down_read_killable(struct rw_semaphore *sem) in __down_read_killable() argument
1284 return __down_read_common(sem, TASK_KILLABLE); in __down_read_killable()
1287 static inline int __down_read_trylock(struct rw_semaphore *sem) in __down_read_trylock() argument
1292 DEBUG_RWSEMS_WARN_ON(sem->magic != sem, sem); in __down_read_trylock()
1295 tmp = atomic_long_read(&sem->count); in __down_read_trylock()
1297 if (atomic_long_try_cmpxchg_acquire(&sem->count, &tmp, in __down_read_trylock()
1299 rwsem_set_reader_owned(sem); in __down_read_trylock()
1311 static __always_inline int __down_write_common(struct rw_semaphore *sem, int state) in __down_write_common() argument
1316 if (unlikely(!rwsem_write_trylock(sem))) { in __down_write_common()
1317 if (IS_ERR(rwsem_down_write_slowpath(sem, state))) in __down_write_common()
1324 static __always_inline void __down_write(struct rw_semaphore *sem) in __down_write() argument
1326 __down_write_common(sem, TASK_UNINTERRUPTIBLE); in __down_write()
1329 static __always_inline int __down_write_killable(struct rw_semaphore *sem) in __down_write_killable() argument
1331 return __down_write_common(sem, TASK_KILLABLE); in __down_write_killable()
1334 static inline int __down_write_trylock(struct rw_semaphore *sem) in __down_write_trylock() argument
1339 DEBUG_RWSEMS_WARN_ON(sem->magic != sem, sem); in __down_write_trylock()
1340 ret = rwsem_write_trylock(sem); in __down_write_trylock()
1349 static inline void __up_read(struct rw_semaphore *sem) in __up_read() argument
1353 DEBUG_RWSEMS_WARN_ON(sem->magic != sem, sem); in __up_read()
1354 DEBUG_RWSEMS_WARN_ON(!is_rwsem_reader_owned(sem), sem); in __up_read()
1357 rwsem_clear_reader_owned(sem); in __up_read()
1358 tmp = atomic_long_add_return_release(-RWSEM_READER_BIAS, &sem->count); in __up_read()
1359 DEBUG_RWSEMS_WARN_ON(tmp < 0, sem); in __up_read()
1362 clear_nonspinnable(sem); in __up_read()
1363 rwsem_wake(sem); in __up_read()
1371 static inline void __up_write(struct rw_semaphore *sem) in __up_write() argument
1375 DEBUG_RWSEMS_WARN_ON(sem->magic != sem, sem); in __up_write()
1380 DEBUG_RWSEMS_WARN_ON((rwsem_owner(sem) != current) && in __up_write()
1381 !rwsem_test_oflags(sem, RWSEM_NONSPINNABLE), sem); in __up_write()
1384 rwsem_clear_owner(sem); in __up_write()
1385 tmp = atomic_long_fetch_add_release(-RWSEM_WRITER_LOCKED, &sem->count); in __up_write()
1387 rwsem_wake(sem); in __up_write()
1394 static inline void __downgrade_write(struct rw_semaphore *sem) in __downgrade_write() argument
1405 DEBUG_RWSEMS_WARN_ON(rwsem_owner(sem) != current, sem); in __downgrade_write()
1408 -RWSEM_WRITER_LOCKED+RWSEM_READER_BIAS, &sem->count); in __downgrade_write()
1409 rwsem_set_reader_owned(sem); in __downgrade_write()
1411 rwsem_downgrade_wake(sem); in __downgrade_write()
1452 void __init_rwsem(struct rw_semaphore *sem, const char *name, in __init_rwsem() argument
1455 init_rwbase_rt(&(sem)->rwbase); in __init_rwsem()
1458 debug_check_no_locks_freed((void *)sem, sizeof(*sem)); in __init_rwsem()
1459 lockdep_init_map_wait(&sem->dep_map, name, key, 0, LD_WAIT_SLEEP); in __init_rwsem()
1464 static inline void __down_read(struct rw_semaphore *sem) in __down_read() argument
1466 rwbase_read_lock(&sem->rwbase, TASK_UNINTERRUPTIBLE); in __down_read()
1469 static inline int __down_read_interruptible(struct rw_semaphore *sem) in __down_read_interruptible() argument
1471 return rwbase_read_lock(&sem->rwbase, TASK_INTERRUPTIBLE); in __down_read_interruptible()
1474 static inline int __down_read_killable(struct rw_semaphore *sem) in __down_read_killable() argument
1476 return rwbase_read_lock(&sem->rwbase, TASK_KILLABLE); in __down_read_killable()
1479 static inline int __down_read_trylock(struct rw_semaphore *sem) in __down_read_trylock() argument
1481 return rwbase_read_trylock(&sem->rwbase); in __down_read_trylock()
1484 static inline void __up_read(struct rw_semaphore *sem) in __up_read() argument
1486 rwbase_read_unlock(&sem->rwbase, TASK_NORMAL); in __up_read()
1489 static inline void __sched __down_write(struct rw_semaphore *sem) in __down_write() argument
1491 rwbase_write_lock(&sem->rwbase, TASK_UNINTERRUPTIBLE); in __down_write()
1494 static inline int __sched __down_write_killable(struct rw_semaphore *sem) in __down_write_killable() argument
1496 return rwbase_write_lock(&sem->rwbase, TASK_KILLABLE); in __down_write_killable()
1499 static inline int __down_write_trylock(struct rw_semaphore *sem) in __down_write_trylock() argument
1501 return rwbase_write_trylock(&sem->rwbase); in __down_write_trylock()
1504 static inline void __up_write(struct rw_semaphore *sem) in __up_write() argument
1506 rwbase_write_unlock(&sem->rwbase); in __up_write()
1509 static inline void __downgrade_write(struct rw_semaphore *sem) in __downgrade_write() argument
1511 rwbase_write_downgrade(&sem->rwbase); in __downgrade_write()
1515 #define DEBUG_RWSEMS_WARN_ON(c, sem) argument
1517 static inline void __rwsem_set_reader_owned(struct rw_semaphore *sem, in __rwsem_set_reader_owned() argument
1522 static inline bool is_rwsem_reader_owned(struct rw_semaphore *sem) in is_rwsem_reader_owned() argument
1524 int count = atomic_read(&sem->rwbase.readers); in is_rwsem_reader_owned()
1534 void __sched down_read(struct rw_semaphore *sem) in down_read() argument
1537 rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_); in down_read()
1539 LOCK_CONTENDED(sem, __down_read_trylock, __down_read); in down_read()
1543 int __sched down_read_interruptible(struct rw_semaphore *sem) in down_read_interruptible() argument
1546 rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_); in down_read_interruptible()
1548 if (LOCK_CONTENDED_RETURN(sem, __down_read_trylock, __down_read_interruptible)) { in down_read_interruptible()
1549 rwsem_release(&sem->dep_map, _RET_IP_); in down_read_interruptible()
1557 int __sched down_read_killable(struct rw_semaphore *sem) in down_read_killable() argument
1560 rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_); in down_read_killable()
1562 if (LOCK_CONTENDED_RETURN(sem, __down_read_trylock, __down_read_killable)) { in down_read_killable()
1563 rwsem_release(&sem->dep_map, _RET_IP_); in down_read_killable()
1574 int down_read_trylock(struct rw_semaphore *sem) in down_read_trylock() argument
1576 int ret = __down_read_trylock(sem); in down_read_trylock()
1579 rwsem_acquire_read(&sem->dep_map, 0, 1, _RET_IP_); in down_read_trylock()
1587 void __sched down_write(struct rw_semaphore *sem) in down_write() argument
1590 rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_); in down_write()
1591 LOCK_CONTENDED(sem, __down_write_trylock, __down_write); in down_write()
1598 int __sched down_write_killable(struct rw_semaphore *sem) in down_write_killable() argument
1601 rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_); in down_write_killable()
1603 if (LOCK_CONTENDED_RETURN(sem, __down_write_trylock, in down_write_killable()
1605 rwsem_release(&sem->dep_map, _RET_IP_); in down_write_killable()
1616 int down_write_trylock(struct rw_semaphore *sem) in down_write_trylock() argument
1618 int ret = __down_write_trylock(sem); in down_write_trylock()
1621 rwsem_acquire(&sem->dep_map, 0, 1, _RET_IP_); in down_write_trylock()
1630 void up_read(struct rw_semaphore *sem) in up_read() argument
1632 rwsem_release(&sem->dep_map, _RET_IP_); in up_read()
1633 __up_read(sem); in up_read()
1640 void up_write(struct rw_semaphore *sem) in up_write() argument
1642 rwsem_release(&sem->dep_map, _RET_IP_); in up_write()
1643 __up_write(sem); in up_write()
1650 void downgrade_write(struct rw_semaphore *sem) in downgrade_write() argument
1652 lock_downgrade(&sem->dep_map, _RET_IP_); in downgrade_write()
1653 __downgrade_write(sem); in downgrade_write()
1659 void down_read_nested(struct rw_semaphore *sem, int subclass) in down_read_nested() argument
1662 rwsem_acquire_read(&sem->dep_map, subclass, 0, _RET_IP_); in down_read_nested()
1663 LOCK_CONTENDED(sem, __down_read_trylock, __down_read); in down_read_nested()
1667 int down_read_killable_nested(struct rw_semaphore *sem, int subclass) in down_read_killable_nested() argument
1670 rwsem_acquire_read(&sem->dep_map, subclass, 0, _RET_IP_); in down_read_killable_nested()
1672 if (LOCK_CONTENDED_RETURN(sem, __down_read_trylock, __down_read_killable)) { in down_read_killable_nested()
1673 rwsem_release(&sem->dep_map, _RET_IP_); in down_read_killable_nested()
1681 void _down_write_nest_lock(struct rw_semaphore *sem, struct lockdep_map *nest) in _down_write_nest_lock() argument
1684 rwsem_acquire_nest(&sem->dep_map, 0, 0, nest, _RET_IP_); in _down_write_nest_lock()
1685 LOCK_CONTENDED(sem, __down_write_trylock, __down_write); in _down_write_nest_lock()
1689 void down_read_non_owner(struct rw_semaphore *sem) in down_read_non_owner() argument
1692 __down_read(sem); in down_read_non_owner()
1699 __rwsem_set_reader_owned(sem, NULL); in down_read_non_owner()
1703 void down_write_nested(struct rw_semaphore *sem, int subclass) in down_write_nested() argument
1706 rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_); in down_write_nested()
1707 LOCK_CONTENDED(sem, __down_write_trylock, __down_write); in down_write_nested()
1711 int __sched down_write_killable_nested(struct rw_semaphore *sem, int subclass) in down_write_killable_nested() argument
1714 rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_); in down_write_killable_nested()
1716 if (LOCK_CONTENDED_RETURN(sem, __down_write_trylock, in down_write_killable_nested()
1718 rwsem_release(&sem->dep_map, _RET_IP_); in down_write_killable_nested()
1726 void up_read_non_owner(struct rw_semaphore *sem) in up_read_non_owner() argument
1728 DEBUG_RWSEMS_WARN_ON(!is_rwsem_reader_owned(sem), sem); in up_read_non_owner()
1729 __up_read(sem); in up_read_non_owner()