Lines Matching full:lock
27 #include <trace/events/lock.h>
37 struct rt_mutex *lock, in __ww_mutex_add_waiter() argument
44 static inline void __ww_mutex_check_waiters(struct rt_mutex *lock, in __ww_mutex_check_waiters() argument
50 static inline void ww_mutex_lock_acquired(struct ww_mutex *lock, in ww_mutex_lock_acquired() argument
55 static inline int __ww_mutex_check_kill(struct rt_mutex *lock, in __ww_mutex_check_kill() argument
69 * lock->owner state tracking:
71 * lock->owner holds the task_struct pointer of the owner. Bit 0
72 * is used to keep track of the "lock has waiters" state.
75 * NULL 0 lock is free (fast acquire possible)
76 * NULL 1 lock is free and has waiters and the top waiter
77 * is going to take the lock*
78 * taskpointer 0 lock is held (fast release possible)
79 * taskpointer 1 lock is held and has waiters**
82 * possible when bit 0 of lock->owner is 0.
84 * (*) It also can be a transitional state when grabbing the lock
85 * with ->wait_lock is held. To prevent any fast path cmpxchg to the lock,
86 * we need to set the bit0 before looking at the lock, and the owner may be
90 * waiters. This can happen when grabbing the lock in the slow path.
91 * To prevent a cmpxchg of the owner releasing the lock, we need to
92 * set this bit before looking at the lock.
96 rt_mutex_owner_encode(struct rt_mutex_base *lock, struct task_struct *owner) in rt_mutex_owner_encode() argument
100 if (rt_mutex_has_waiters(lock)) in rt_mutex_owner_encode()
107 rt_mutex_set_owner(struct rt_mutex_base *lock, struct task_struct *owner) in rt_mutex_set_owner() argument
110 * lock->wait_lock is held but explicit acquire semantics are needed in rt_mutex_set_owner()
111 * for a new lock owner so WRITE_ONCE is insufficient. in rt_mutex_set_owner()
113 xchg_acquire(&lock->owner, rt_mutex_owner_encode(lock, owner)); in rt_mutex_set_owner()
116 static __always_inline void rt_mutex_clear_owner(struct rt_mutex_base *lock) in rt_mutex_clear_owner() argument
118 /* lock->wait_lock is held so the unlock provides release semantics. */ in rt_mutex_clear_owner()
119 WRITE_ONCE(lock->owner, rt_mutex_owner_encode(lock, NULL)); in rt_mutex_clear_owner()
122 static __always_inline void clear_rt_mutex_waiters(struct rt_mutex_base *lock) in clear_rt_mutex_waiters() argument
124 lock->owner = (struct task_struct *) in clear_rt_mutex_waiters()
125 ((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS); in clear_rt_mutex_waiters()
129 fixup_rt_mutex_waiters(struct rt_mutex_base *lock, bool acquire_lock) in fixup_rt_mutex_waiters() argument
131 unsigned long owner, *p = (unsigned long *) &lock->owner; in fixup_rt_mutex_waiters()
133 if (rt_mutex_has_waiters(lock)) in fixup_rt_mutex_waiters()
138 * lock->owner still has the waiters bit set, otherwise the in fixup_rt_mutex_waiters()
144 * lock(l->lock) in fixup_rt_mutex_waiters()
148 * unlock(l->lock) in fixup_rt_mutex_waiters()
152 * lock(l->lock) in fixup_rt_mutex_waiters()
156 * unlock(l->lock) in fixup_rt_mutex_waiters()
159 * lock(l->lock) in fixup_rt_mutex_waiters()
162 * unlock(l->lock) in fixup_rt_mutex_waiters()
163 * lock(l->lock) in fixup_rt_mutex_waiters()
167 * unlock(l->lock) in fixup_rt_mutex_waiters()
168 * lock(l->lock) in fixup_rt_mutex_waiters()
175 * lock(l->lock) in fixup_rt_mutex_waiters()
188 * serialized by l->lock, so nothing else can modify the waiters in fixup_rt_mutex_waiters()
202 * in case that the lock acquisition failed it might in fixup_rt_mutex_waiters()
217 static __always_inline bool rt_mutex_cmpxchg_acquire(struct rt_mutex_base *lock, in rt_mutex_cmpxchg_acquire() argument
221 return try_cmpxchg_acquire(&lock->owner, &old, new); in rt_mutex_cmpxchg_acquire()
224 static __always_inline bool rt_mutex_try_acquire(struct rt_mutex_base *lock) in rt_mutex_try_acquire() argument
226 return rt_mutex_cmpxchg_acquire(lock, NULL, current); in rt_mutex_try_acquire()
229 static __always_inline bool rt_mutex_cmpxchg_release(struct rt_mutex_base *lock, in rt_mutex_cmpxchg_release() argument
233 return try_cmpxchg_release(&lock->owner, &old, new); in rt_mutex_cmpxchg_release()
238 * all future threads that attempt to [Rmw] the lock to the slowpath. As such
241 static __always_inline void mark_rt_mutex_waiters(struct rt_mutex_base *lock) in mark_rt_mutex_waiters() argument
243 unsigned long *p = (unsigned long *) &lock->owner; in mark_rt_mutex_waiters()
262 * 2) Drop lock->wait_lock
263 * 3) Try to unlock the lock with cmpxchg
265 static __always_inline bool unlock_rt_mutex_safe(struct rt_mutex_base *lock, in unlock_rt_mutex_safe() argument
267 __releases(lock->wait_lock) in unlock_rt_mutex_safe()
269 struct task_struct *owner = rt_mutex_owner(lock); in unlock_rt_mutex_safe()
271 clear_rt_mutex_waiters(lock); in unlock_rt_mutex_safe()
272 raw_spin_unlock_irqrestore(&lock->wait_lock, flags); in unlock_rt_mutex_safe()
278 * lock(wait_lock); in unlock_rt_mutex_safe()
280 * mark_rt_mutex_waiters(lock); in unlock_rt_mutex_safe()
281 * acquire(lock); in unlock_rt_mutex_safe()
285 * lock(wait_lock); in unlock_rt_mutex_safe()
286 * mark_rt_mutex_waiters(lock); in unlock_rt_mutex_safe()
291 * lock(wait_lock); in unlock_rt_mutex_safe()
294 * lock(wait_lock); in unlock_rt_mutex_safe()
295 * acquire(lock); in unlock_rt_mutex_safe()
297 return rt_mutex_cmpxchg_release(lock, owner, NULL); in unlock_rt_mutex_safe()
301 static __always_inline bool rt_mutex_cmpxchg_acquire(struct rt_mutex_base *lock, in rt_mutex_cmpxchg_acquire() argument
309 static int __sched rt_mutex_slowtrylock(struct rt_mutex_base *lock);
311 static __always_inline bool rt_mutex_try_acquire(struct rt_mutex_base *lock) in rt_mutex_try_acquire() argument
320 return rt_mutex_slowtrylock(lock); in rt_mutex_try_acquire()
323 static __always_inline bool rt_mutex_cmpxchg_release(struct rt_mutex_base *lock, in rt_mutex_cmpxchg_release() argument
330 static __always_inline void mark_rt_mutex_waiters(struct rt_mutex_base *lock) in mark_rt_mutex_waiters() argument
332 lock->owner = (struct task_struct *) in mark_rt_mutex_waiters()
333 ((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS); in mark_rt_mutex_waiters()
337 * Simple slow path only version: lock->owner is protected by lock->wait_lock.
339 static __always_inline bool unlock_rt_mutex_safe(struct rt_mutex_base *lock, in unlock_rt_mutex_safe() argument
341 __releases(lock->wait_lock) in unlock_rt_mutex_safe()
343 lock->owner = NULL; in unlock_rt_mutex_safe()
344 raw_spin_unlock_irqrestore(&lock->wait_lock, flags); in unlock_rt_mutex_safe()
365 lockdep_assert_held(&waiter->lock->wait_lock); in waiter_update_prio()
378 lockdep_assert_held(&waiter->lock->wait_lock); in waiter_clone_prio()
480 rt_mutex_enqueue(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter) in rt_mutex_enqueue() argument
482 lockdep_assert_held(&lock->wait_lock); in rt_mutex_enqueue()
484 rb_add_cached(&waiter->tree.entry, &lock->waiters, __waiter_less); in rt_mutex_enqueue()
488 rt_mutex_dequeue(struct rt_mutex_base *lock, struct rt_mutex_waiter *waiter) in rt_mutex_dequeue() argument
490 lockdep_assert_held(&lock->wait_lock); in rt_mutex_dequeue()
495 rb_erase_cached(&waiter->tree.entry, &lock->waiters); in rt_mutex_dequeue()
527 static __always_inline void rt_mutex_adjust_prio(struct rt_mutex_base *lock, in rt_mutex_adjust_prio() argument
532 lockdep_assert_held(&lock->wait_lock); in rt_mutex_adjust_prio()
533 lockdep_assert(rt_mutex_owner(lock) == p); in rt_mutex_adjust_prio()
602 return p->pi_blocked_on ? p->pi_blocked_on->lock : NULL; in task_blocked_on_lock()
617 * comparison to detect lock chain changes.
651 * [1] lock(task->pi_lock); [R] acquire [P1]
654 * [4] lock = waiter->lock; [P1]
655 * [5] if (!try_lock(lock->wait_lock)) { [P1] try to acquire [L]
660 * [7] requeue_lock_waiter(lock, waiter); [P1] + [L]
664 * [10] task = owner(lock); [L]
666 * lock(task->pi_lock); [L] acquire [P2]
667 * [11] requeue_pi_waiter(tsk, waiters(lock));[P2] + [L]
670 * unlock(lock->wait_lock); release [L]
673 * Where P1 is the blocking task and P2 is the lock owner; going up one step
688 struct rt_mutex_base *lock; in rt_mutex_adjust_prio_chain() local
702 * We limit the lock chain length for each invocation. in rt_mutex_adjust_prio_chain()
713 printk(KERN_WARNING "Maximum lock depth %d reached " in rt_mutex_adjust_prio_chain()
753 * the previous owner of the lock might have released the lock. in rt_mutex_adjust_prio_chain()
760 * the task might have moved on in the lock chain or even left in rt_mutex_adjust_prio_chain()
761 * the chain completely and blocks now on an unrelated lock or in rt_mutex_adjust_prio_chain()
764 * We stored the lock on which @task was blocked in @next_lock, in rt_mutex_adjust_prio_chain()
767 if (next_lock != waiter->lock) in rt_mutex_adjust_prio_chain()
771 * There could be 'spurious' loops in the lock graph due to ww_mutex, in rt_mutex_adjust_prio_chain()
832 * [4] Get the next lock; per holding task->pi_lock we can't unblock in rt_mutex_adjust_prio_chain()
833 * and guarantee @lock's existence. in rt_mutex_adjust_prio_chain()
835 lock = waiter->lock; in rt_mutex_adjust_prio_chain()
838 * which is the reverse lock order versus the other rtmutex in rt_mutex_adjust_prio_chain()
841 * Per the above, holding task->pi_lock guarantees lock exists, so in rt_mutex_adjust_prio_chain()
842 * inverting this lock order is infeasible from a life-time in rt_mutex_adjust_prio_chain()
845 if (!raw_spin_trylock(&lock->wait_lock)) { in rt_mutex_adjust_prio_chain()
853 * lock->wait_lock. in rt_mutex_adjust_prio_chain()
855 * Deadlock detection. If the lock is the same as the original in rt_mutex_adjust_prio_chain()
856 * lock which caused us to walk the lock chain or if the in rt_mutex_adjust_prio_chain()
857 * current lock is owned by the task which initiated the chain in rt_mutex_adjust_prio_chain()
860 if (lock == orig_lock || rt_mutex_owner(lock) == top_task) { in rt_mutex_adjust_prio_chain()
875 raw_spin_unlock(&lock->wait_lock); in rt_mutex_adjust_prio_chain()
880 * If we just follow the lock chain for deadlock detection, no in rt_mutex_adjust_prio_chain()
893 * [9] check_exit_conditions_3 protected by lock->wait_lock. in rt_mutex_adjust_prio_chain()
894 * If there is no owner of the lock, end of chain. in rt_mutex_adjust_prio_chain()
896 if (!rt_mutex_owner(lock)) { in rt_mutex_adjust_prio_chain()
897 raw_spin_unlock_irq(&lock->wait_lock); in rt_mutex_adjust_prio_chain()
901 /* [10] Grab the next task, i.e. owner of @lock */ in rt_mutex_adjust_prio_chain()
902 task = get_task_struct(rt_mutex_owner(lock)); in rt_mutex_adjust_prio_chain()
915 top_waiter = rt_mutex_top_waiter(lock); in rt_mutex_adjust_prio_chain()
919 raw_spin_unlock_irq(&lock->wait_lock); in rt_mutex_adjust_prio_chain()
929 * operation on @lock. We need it for the boost/deboost in rt_mutex_adjust_prio_chain()
932 prerequeue_top_waiter = rt_mutex_top_waiter(lock); in rt_mutex_adjust_prio_chain()
934 /* [7] Requeue the waiter in the lock waiter tree. */ in rt_mutex_adjust_prio_chain()
935 rt_mutex_dequeue(lock, waiter); in rt_mutex_adjust_prio_chain()
950 rt_mutex_enqueue(lock, waiter); in rt_mutex_adjust_prio_chain()
956 * Since we hold lock->waiter_lock, task cannot unblock, even if we in rt_mutex_adjust_prio_chain()
963 * [9] check_exit_conditions_3 protected by lock->wait_lock. in rt_mutex_adjust_prio_chain()
965 * We must abort the chain walk if there is no lock owner even in rt_mutex_adjust_prio_chain()
966 * in the dead lock detection case, as we have nothing to in rt_mutex_adjust_prio_chain()
969 if (!rt_mutex_owner(lock)) { in rt_mutex_adjust_prio_chain()
973 * to get the lock. in rt_mutex_adjust_prio_chain()
975 top_waiter = rt_mutex_top_waiter(lock); in rt_mutex_adjust_prio_chain()
978 raw_spin_unlock_irq(&lock->wait_lock); in rt_mutex_adjust_prio_chain()
983 * [10] Grab the next task, i.e. the owner of @lock in rt_mutex_adjust_prio_chain()
985 * Per holding lock->wait_lock and checking for !owner above, there in rt_mutex_adjust_prio_chain()
988 task = get_task_struct(rt_mutex_owner(lock)); in rt_mutex_adjust_prio_chain()
992 if (waiter == rt_mutex_top_waiter(lock)) { in rt_mutex_adjust_prio_chain()
995 * waiter on the lock. Replace the previous top waiter in rt_mutex_adjust_prio_chain()
1002 rt_mutex_adjust_prio(lock, task); in rt_mutex_adjust_prio_chain()
1006 * The waiter was the top waiter on the lock, but is in rt_mutex_adjust_prio_chain()
1016 waiter = rt_mutex_top_waiter(lock); in rt_mutex_adjust_prio_chain()
1019 rt_mutex_adjust_prio(lock, task); in rt_mutex_adjust_prio_chain()
1029 * and lock->wait_lock. The actual decisions are made after we in rt_mutex_adjust_prio_chain()
1032 * Check whether the task which owns the current lock is pi in rt_mutex_adjust_prio_chain()
1033 * blocked itself. If yes we store a pointer to the lock for in rt_mutex_adjust_prio_chain()
1034 * the lock chain change detection above. After we dropped in rt_mutex_adjust_prio_chain()
1039 * Store the top waiter of @lock for the end of chain walk in rt_mutex_adjust_prio_chain()
1042 top_waiter = rt_mutex_top_waiter(lock); in rt_mutex_adjust_prio_chain()
1046 raw_spin_unlock_irq(&lock->wait_lock); in rt_mutex_adjust_prio_chain()
1052 * We reached the end of the lock chain. Stop right here. No in rt_mutex_adjust_prio_chain()
1059 * If the current waiter is not the top waiter on the lock, in rt_mutex_adjust_prio_chain()
1079 * Must be called with lock->wait_lock held and interrupts disabled
1081 * @lock: The lock to be acquired.
1082 * @task: The task which wants to acquire the lock
1083 * @waiter: The waiter that is queued to the lock's wait tree if the
1087 try_to_take_rt_mutex(struct rt_mutex_base *lock, struct task_struct *task, in try_to_take_rt_mutex() argument
1090 lockdep_assert_held(&lock->wait_lock); in try_to_take_rt_mutex()
1093 * Before testing whether we can acquire @lock, we set the in try_to_take_rt_mutex()
1094 * RT_MUTEX_HAS_WAITERS bit in @lock->owner. This forces all in try_to_take_rt_mutex()
1095 * other tasks which try to modify @lock into the slow path in try_to_take_rt_mutex()
1096 * and they serialize on @lock->wait_lock. in try_to_take_rt_mutex()
1101 * - There is a lock owner. The caller must fixup the in try_to_take_rt_mutex()
1102 * transient state if it does a trylock or leaves the lock in try_to_take_rt_mutex()
1105 * - @task acquires the lock and there are no other in try_to_take_rt_mutex()
1109 mark_rt_mutex_waiters(lock); in try_to_take_rt_mutex()
1112 * If @lock has an owner, give up. in try_to_take_rt_mutex()
1114 if (rt_mutex_owner(lock)) in try_to_take_rt_mutex()
1119 * into @lock waiter tree. If @waiter == NULL then this is a in try_to_take_rt_mutex()
1123 struct rt_mutex_waiter *top_waiter = rt_mutex_top_waiter(lock); in try_to_take_rt_mutex()
1126 * If waiter is the highest priority waiter of @lock, in try_to_take_rt_mutex()
1131 * We can acquire the lock. Remove the waiter from the in try_to_take_rt_mutex()
1132 * lock waiters tree. in try_to_take_rt_mutex()
1134 rt_mutex_dequeue(lock, waiter); in try_to_take_rt_mutex()
1140 * If the lock has waiters already we check whether @task is in try_to_take_rt_mutex()
1141 * eligible to take over the lock. in try_to_take_rt_mutex()
1144 * the lock. @task->pi_blocked_on is NULL, so it does in try_to_take_rt_mutex()
1147 if (rt_mutex_has_waiters(lock)) { in try_to_take_rt_mutex()
1150 rt_mutex_top_waiter(lock))) in try_to_take_rt_mutex()
1155 * don't have to change anything in the lock in try_to_take_rt_mutex()
1160 * No waiters. Take the lock without the in try_to_take_rt_mutex()
1178 * Finish the lock acquisition. @task is the new owner. If in try_to_take_rt_mutex()
1182 if (rt_mutex_has_waiters(lock)) in try_to_take_rt_mutex()
1183 rt_mutex_enqueue_pi(task, rt_mutex_top_waiter(lock)); in try_to_take_rt_mutex()
1191 rt_mutex_set_owner(lock, task); in try_to_take_rt_mutex()
1197 * Task blocks on lock.
1201 * This must be called with lock->wait_lock held and interrupts disabled
1203 static int __sched task_blocks_on_rt_mutex(struct rt_mutex_base *lock, in task_blocks_on_rt_mutex() argument
1210 struct task_struct *owner = rt_mutex_owner(lock); in task_blocks_on_rt_mutex()
1215 lockdep_assert_held(&lock->wait_lock); in task_blocks_on_rt_mutex()
1234 waiter->lock = lock; in task_blocks_on_rt_mutex()
1238 /* Get the top priority waiter on the lock */ in task_blocks_on_rt_mutex()
1239 if (rt_mutex_has_waiters(lock)) in task_blocks_on_rt_mutex()
1240 top_waiter = rt_mutex_top_waiter(lock); in task_blocks_on_rt_mutex()
1241 rt_mutex_enqueue(lock, waiter); in task_blocks_on_rt_mutex()
1251 rtm = container_of(lock, struct rt_mutex, rtmutex); in task_blocks_on_rt_mutex()
1255 rt_mutex_dequeue(lock, waiter); in task_blocks_on_rt_mutex()
1266 if (waiter == rt_mutex_top_waiter(lock)) { in task_blocks_on_rt_mutex()
1270 rt_mutex_adjust_prio(lock, owner); in task_blocks_on_rt_mutex()
1277 /* Store the lock on which owner is blocked or NULL */ in task_blocks_on_rt_mutex()
1290 * The owner can't disappear while holding a lock, in task_blocks_on_rt_mutex()
1296 raw_spin_unlock_irq_wake(&lock->wait_lock, wake_q); in task_blocks_on_rt_mutex()
1298 res = rt_mutex_adjust_prio_chain(owner, chwalk, lock, in task_blocks_on_rt_mutex()
1301 raw_spin_lock_irq(&lock->wait_lock); in task_blocks_on_rt_mutex()
1310 * Called with lock->wait_lock held and interrupts disabled.
1313 struct rt_mutex_base *lock) in mark_wakeup_next_waiter() argument
1317 lockdep_assert_held(&lock->wait_lock); in mark_wakeup_next_waiter()
1321 waiter = rt_mutex_top_waiter(lock); in mark_wakeup_next_waiter()
1331 rt_mutex_adjust_prio(lock, current); in mark_wakeup_next_waiter()
1335 * queued on the lock until it gets the lock, this lock in mark_wakeup_next_waiter()
1339 * the top waiter can steal this lock. in mark_wakeup_next_waiter()
1341 lock->owner = (void *) RT_MUTEX_HAS_WAITERS; in mark_wakeup_next_waiter()
1358 static int __sched __rt_mutex_slowtrylock(struct rt_mutex_base *lock) in __rt_mutex_slowtrylock() argument
1360 int ret = try_to_take_rt_mutex(lock, current, NULL); in __rt_mutex_slowtrylock()
1363 * try_to_take_rt_mutex() sets the lock waiters bit in __rt_mutex_slowtrylock()
1366 fixup_rt_mutex_waiters(lock, true); in __rt_mutex_slowtrylock()
1372 * Slow path try-lock function:
1374 static int __sched rt_mutex_slowtrylock(struct rt_mutex_base *lock) in rt_mutex_slowtrylock() argument
1380 * If the lock already has an owner we fail to get the lock. in rt_mutex_slowtrylock()
1381 * This can be done without taking the @lock->wait_lock as in rt_mutex_slowtrylock()
1384 if (rt_mutex_owner(lock)) in rt_mutex_slowtrylock()
1388 * The mutex has currently no owner. Lock the wait lock and try to in rt_mutex_slowtrylock()
1389 * acquire the lock. We use irqsave here to support early boot calls. in rt_mutex_slowtrylock()
1391 raw_spin_lock_irqsave(&lock->wait_lock, flags); in rt_mutex_slowtrylock()
1393 ret = __rt_mutex_slowtrylock(lock); in rt_mutex_slowtrylock()
1395 raw_spin_unlock_irqrestore(&lock->wait_lock, flags); in rt_mutex_slowtrylock()
1400 static __always_inline int __rt_mutex_trylock(struct rt_mutex_base *lock) in __rt_mutex_trylock() argument
1402 if (likely(rt_mutex_cmpxchg_acquire(lock, NULL, current))) in __rt_mutex_trylock()
1405 return rt_mutex_slowtrylock(lock); in __rt_mutex_trylock()
1411 static void __sched rt_mutex_slowunlock(struct rt_mutex_base *lock) in rt_mutex_slowunlock() argument
1417 raw_spin_lock_irqsave(&lock->wait_lock, flags); in rt_mutex_slowunlock()
1419 debug_rt_mutex_unlock(lock); in rt_mutex_slowunlock()
1426 * foo->lock->owner = NULL; in rt_mutex_slowunlock()
1427 * rtmutex_lock(foo->lock); <- fast path in rt_mutex_slowunlock()
1429 * rtmutex_unlock(foo->lock); <- fast path in rt_mutex_slowunlock()
1432 * raw_spin_unlock(foo->lock->wait_lock); in rt_mutex_slowunlock()
1437 * lock->wait_lock. So we do the following sequence: in rt_mutex_slowunlock()
1439 * owner = rt_mutex_owner(lock); in rt_mutex_slowunlock()
1440 * clear_rt_mutex_waiters(lock); in rt_mutex_slowunlock()
1441 * raw_spin_unlock(&lock->wait_lock); in rt_mutex_slowunlock()
1442 * if (cmpxchg(&lock->owner, owner, 0) == owner) in rt_mutex_slowunlock()
1447 * lock->owner is serialized by lock->wait_lock: in rt_mutex_slowunlock()
1449 * lock->owner = NULL; in rt_mutex_slowunlock()
1450 * raw_spin_unlock(&lock->wait_lock); in rt_mutex_slowunlock()
1452 while (!rt_mutex_has_waiters(lock)) { in rt_mutex_slowunlock()
1453 /* Drops lock->wait_lock ! */ in rt_mutex_slowunlock()
1454 if (unlock_rt_mutex_safe(lock, flags) == true) in rt_mutex_slowunlock()
1457 raw_spin_lock_irqsave(&lock->wait_lock, flags); in rt_mutex_slowunlock()
1466 mark_wakeup_next_waiter(&wqh, lock); in rt_mutex_slowunlock()
1467 raw_spin_unlock_irqrestore(&lock->wait_lock, flags); in rt_mutex_slowunlock()
1472 static __always_inline void __rt_mutex_unlock(struct rt_mutex_base *lock) in __rt_mutex_unlock() argument
1474 if (likely(rt_mutex_cmpxchg_release(lock, current, NULL))) in __rt_mutex_unlock()
1477 rt_mutex_slowunlock(lock); in __rt_mutex_unlock()
1481 static bool rtmutex_spin_on_owner(struct rt_mutex_base *lock, in rtmutex_spin_on_owner() argument
1490 if (owner != rt_mutex_owner(lock)) in rtmutex_spin_on_owner()
1494 * the lock owner still matches @owner. If that fails, in rtmutex_spin_on_owner()
1501 * - the lock owner has been scheduled out in rtmutex_spin_on_owner()
1508 !rt_mutex_waiter_is_top_waiter(lock, waiter)) { in rtmutex_spin_on_owner()
1518 static bool rtmutex_spin_on_owner(struct rt_mutex_base *lock, in rtmutex_spin_on_owner() argument
1534 * Remove a waiter from a lock and give up
1536 * Must be called with lock->wait_lock held and interrupts disabled. It must
1539 static void __sched remove_waiter(struct rt_mutex_base *lock, in remove_waiter() argument
1542 bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock)); in remove_waiter()
1543 struct task_struct *owner = rt_mutex_owner(lock); in remove_waiter()
1546 lockdep_assert_held(&lock->wait_lock); in remove_waiter()
1549 rt_mutex_dequeue(lock, waiter); in remove_waiter()
1555 * waiter of the lock and there is an owner to update. in remove_waiter()
1564 if (rt_mutex_has_waiters(lock)) in remove_waiter()
1565 rt_mutex_enqueue_pi(owner, rt_mutex_top_waiter(lock)); in remove_waiter()
1567 rt_mutex_adjust_prio(lock, owner); in remove_waiter()
1569 /* Store the lock on which owner is blocked or NULL */ in remove_waiter()
1584 raw_spin_unlock_irq(&lock->wait_lock); in remove_waiter()
1586 rt_mutex_adjust_prio_chain(owner, RT_MUTEX_MIN_CHAINWALK, lock, in remove_waiter()
1589 raw_spin_lock_irq(&lock->wait_lock); in remove_waiter()
1594 * @lock: the rt_mutex to take
1600 * @wake_q: wake_q of tasks to wake when we drop the lock->wait_lock
1602 * Must be called with lock->wait_lock held and interrupts disabled
1604 static int __sched rt_mutex_slowlock_block(struct rt_mutex_base *lock, in rt_mutex_slowlock_block() argument
1610 __releases(&lock->wait_lock) __acquires(&lock->wait_lock) in rt_mutex_slowlock_block()
1612 struct rt_mutex *rtm = container_of(lock, struct rt_mutex, rtmutex); in rt_mutex_slowlock_block()
1618 /* Try to acquire the lock: */ in rt_mutex_slowlock_block()
1619 if (try_to_take_rt_mutex(lock, current, waiter)) { in rt_mutex_slowlock_block()
1639 if (waiter == rt_mutex_top_waiter(lock)) in rt_mutex_slowlock_block()
1640 owner = rt_mutex_owner(lock); in rt_mutex_slowlock_block()
1643 raw_spin_unlock_irq_wake(&lock->wait_lock, wake_q); in rt_mutex_slowlock_block()
1645 if (!owner || !rtmutex_spin_on_owner(lock, waiter, owner)) { in rt_mutex_slowlock_block()
1650 raw_spin_lock_irq(&lock->wait_lock); in rt_mutex_slowlock_block()
1659 struct rt_mutex_base *lock, in rt_mutex_handle_deadlock() argument
1672 raw_spin_unlock_irq(&lock->wait_lock); in rt_mutex_handle_deadlock()
1683 * __rt_mutex_slowlock - Locking slowpath invoked with lock::wait_lock held
1684 * @lock: The rtmutex to block lock
1691 static int __sched __rt_mutex_slowlock(struct rt_mutex_base *lock, in __rt_mutex_slowlock() argument
1698 struct rt_mutex *rtm = container_of(lock, struct rt_mutex, rtmutex); in __rt_mutex_slowlock()
1702 lockdep_assert_held(&lock->wait_lock); in __rt_mutex_slowlock()
1705 /* Try to acquire the lock again: */ in __rt_mutex_slowlock()
1706 if (try_to_take_rt_mutex(lock, current, NULL)) { in __rt_mutex_slowlock()
1717 trace_contention_begin(lock, LCB_F_RT); in __rt_mutex_slowlock()
1719 ret = task_blocks_on_rt_mutex(lock, waiter, current, ww_ctx, chwalk, wake_q); in __rt_mutex_slowlock()
1721 ret = rt_mutex_slowlock_block(lock, ww_ctx, state, NULL, waiter, wake_q); in __rt_mutex_slowlock()
1724 /* acquired the lock */ in __rt_mutex_slowlock()
1733 remove_waiter(lock, waiter); in __rt_mutex_slowlock()
1734 rt_mutex_handle_deadlock(ret, chwalk, lock, waiter); in __rt_mutex_slowlock()
1742 fixup_rt_mutex_waiters(lock, true); in __rt_mutex_slowlock()
1744 trace_contention_end(lock, ret); in __rt_mutex_slowlock()
1749 static inline int __rt_mutex_slowlock_locked(struct rt_mutex_base *lock, in __rt_mutex_slowlock_locked() argument
1760 ret = __rt_mutex_slowlock(lock, ww_ctx, state, RT_MUTEX_MIN_CHAINWALK, in __rt_mutex_slowlock_locked()
1770 * @lock: The rtmutex to block lock
1774 static int __sched rt_mutex_slowlock(struct rt_mutex_base *lock, in rt_mutex_slowlock() argument
1795 * rtmutex with lock->wait_lock held. But we cannot unconditionally in rt_mutex_slowlock()
1799 raw_spin_lock_irqsave(&lock->wait_lock, flags); in rt_mutex_slowlock()
1800 ret = __rt_mutex_slowlock_locked(lock, ww_ctx, state, &wake_q); in rt_mutex_slowlock()
1801 raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q); in rt_mutex_slowlock()
1807 static __always_inline int __rt_mutex_lock(struct rt_mutex_base *lock, in __rt_mutex_lock() argument
1812 if (likely(rt_mutex_try_acquire(lock))) in __rt_mutex_lock()
1815 return rt_mutex_slowlock(lock, NULL, state); in __rt_mutex_lock()
1825 * rtlock_slowlock_locked - Slow path lock acquisition for RT locks
1826 * @lock: The underlying RT mutex
1829 static void __sched rtlock_slowlock_locked(struct rt_mutex_base *lock, in rtlock_slowlock_locked() argument
1831 __releases(&lock->wait_lock) __acquires(&lock->wait_lock) in rtlock_slowlock_locked()
1836 lockdep_assert_held(&lock->wait_lock); in rtlock_slowlock_locked()
1839 if (try_to_take_rt_mutex(lock, current, NULL)) { in rtlock_slowlock_locked()
1849 trace_contention_begin(lock, LCB_F_RT); in rtlock_slowlock_locked()
1851 task_blocks_on_rt_mutex(lock, &waiter, current, NULL, RT_MUTEX_MIN_CHAINWALK, wake_q); in rtlock_slowlock_locked()
1854 /* Try to acquire the lock again */ in rtlock_slowlock_locked()
1855 if (try_to_take_rt_mutex(lock, current, &waiter)) { in rtlock_slowlock_locked()
1860 if (&waiter == rt_mutex_top_waiter(lock)) in rtlock_slowlock_locked()
1861 owner = rt_mutex_owner(lock); in rtlock_slowlock_locked()
1864 raw_spin_unlock_irq_wake(&lock->wait_lock, wake_q); in rtlock_slowlock_locked()
1866 if (!owner || !rtmutex_spin_on_owner(lock, &waiter, owner)) { in rtlock_slowlock_locked()
1871 raw_spin_lock_irq(&lock->wait_lock); in rtlock_slowlock_locked()
1882 fixup_rt_mutex_waiters(lock, true); in rtlock_slowlock_locked()
1885 trace_contention_end(lock, 0); in rtlock_slowlock_locked()
1889 static __always_inline void __sched rtlock_slowlock(struct rt_mutex_base *lock) in rtlock_slowlock() argument
1894 raw_spin_lock_irqsave(&lock->wait_lock, flags); in rtlock_slowlock()
1895 rtlock_slowlock_locked(lock, &wake_q); in rtlock_slowlock()
1896 raw_spin_unlock_irqrestore_wake(&lock->wait_lock, flags, &wake_q); in rtlock_slowlock()