96 rt_mutex_owner_encode(struct rt_mutex_base *lock, struct task_struct *owner)  in rt_mutex_owner_encode()  argument
98 	unsigned long val = (unsigned long)owner;  in rt_mutex_owner_encode()
107 rt_mutex_set_owner(struct rt_mutex_base *lock, struct task_struct *owner)  in rt_mutex_set_owner()  argument
113 	xchg_acquire(&lock->owner, rt_mutex_owner_encode(lock, owner));  in rt_mutex_set_owner()
119 	WRITE_ONCE(lock->owner, rt_mutex_owner_encode(lock, NULL));  in rt_mutex_clear_owner()
124 	lock->owner = (struct task_struct *)  in clear_rt_mutex_waiters()
125 			((unsigned long)lock->owner & ~RT_MUTEX_HAS_WAITERS);  in clear_rt_mutex_waiters()
131 	unsigned long owner, *p = (unsigned long *) &lock->owner;  in fixup_rt_mutex_waiters()  local
194 	owner = READ_ONCE(*p);  in fixup_rt_mutex_waiters()
195 	if (owner & RT_MUTEX_HAS_WAITERS) {  in fixup_rt_mutex_waiters()
206 			xchg_acquire(p, owner & ~RT_MUTEX_HAS_WAITERS);  in fixup_rt_mutex_waiters()
208 			WRITE_ONCE(*p, owner & ~RT_MUTEX_HAS_WAITERS);  in fixup_rt_mutex_waiters()
221 	return try_cmpxchg_acquire(&lock->owner, &old, new);  in rt_mutex_cmpxchg_acquire()
233 	return try_cmpxchg_release(&lock->owner, &old, new);  in rt_mutex_cmpxchg_release()
243 	unsigned long *p = (unsigned long *) &lock->owner;  in mark_rt_mutex_waiters()
244 	unsigned long owner, new;  in mark_rt_mutex_waiters()  local
246 	owner = READ_ONCE(*p);  in mark_rt_mutex_waiters()
248 		new = owner | RT_MUTEX_HAS_WAITERS;  in mark_rt_mutex_waiters()
249 	} while (!try_cmpxchg_relaxed(p, &owner, new));  in mark_rt_mutex_waiters()
269 	struct task_struct *owner = rt_mutex_owner(lock);  in unlock_rt_mutex_safe()  local
297 	return rt_mutex_cmpxchg_release(lock, owner, NULL);  in unlock_rt_mutex_safe()
332 	lock->owner = (struct task_struct *)  in mark_rt_mutex_waiters()
333 			((unsigned long)lock->owner | RT_MUTEX_HAS_WAITERS);  in mark_rt_mutex_waiters()
343 	lock->owner = NULL;  in unlock_rt_mutex_safe()
1210 	struct task_struct *owner = rt_mutex_owner(lock);  in task_blocks_on_rt_mutex()  local
1229 	if (owner == task && !(build_ww_mutex() && ww_ctx))  in task_blocks_on_rt_mutex()
1262 	if (!owner)  in task_blocks_on_rt_mutex()
1265 	raw_spin_lock(&owner->pi_lock);  in task_blocks_on_rt_mutex()
1267 		rt_mutex_dequeue_pi(owner, top_waiter);  in task_blocks_on_rt_mutex()
1268 		rt_mutex_enqueue_pi(owner, waiter);  in task_blocks_on_rt_mutex()
1270 		rt_mutex_adjust_prio(lock, owner);  in task_blocks_on_rt_mutex()
1271 		if (owner->pi_blocked_on)  in task_blocks_on_rt_mutex()
1278 	next_lock = task_blocked_on_lock(owner);  in task_blocks_on_rt_mutex()
1280 	raw_spin_unlock(&owner->pi_lock);  in task_blocks_on_rt_mutex()
1294 	get_task_struct(owner);  in task_blocks_on_rt_mutex()
1298 	res = rt_mutex_adjust_prio_chain(owner, chwalk, lock,  in task_blocks_on_rt_mutex()
1341 	lock->owner = (void *) RT_MUTEX_HAS_WAITERS;  in mark_wakeup_next_waiter()
1483 				  struct task_struct *owner)  in rtmutex_spin_on_owner()  argument
1490 		if (owner != rt_mutex_owner(lock))  in rtmutex_spin_on_owner()
1507 		if (!owner_on_cpu(owner) || need_resched() ||  in rtmutex_spin_on_owner()
1520 				  struct task_struct *owner)  in rtmutex_spin_on_owner()  argument
1543 	struct task_struct *owner = rt_mutex_owner(lock);  in remove_waiter()  local
1557 	if (!owner || !is_top_waiter)  in remove_waiter()
1560 	raw_spin_lock(&owner->pi_lock);  in remove_waiter()
1562 	rt_mutex_dequeue_pi(owner, waiter);  in remove_waiter()
1565 		rt_mutex_enqueue_pi(owner, rt_mutex_top_waiter(lock));  in remove_waiter()
1567 	rt_mutex_adjust_prio(lock, owner);  in remove_waiter()
1570 	next_lock = task_blocked_on_lock(owner);  in remove_waiter()
1572 	raw_spin_unlock(&owner->pi_lock);  in remove_waiter()
1582 	get_task_struct(owner);  in remove_waiter()
1586 	rt_mutex_adjust_prio_chain(owner, RT_MUTEX_MIN_CHAINWALK, lock,  in remove_waiter()
1613 	struct task_struct *owner;  in rt_mutex_slowlock_block()  local
1640 			owner = rt_mutex_owner(lock);  in rt_mutex_slowlock_block()
1642 			owner = NULL;  in rt_mutex_slowlock_block()
1645 		if (!owner || !rtmutex_spin_on_owner(lock, waiter, owner)) {  in rt_mutex_slowlock_block()
1834 	struct task_struct *owner;  in rtlock_slowlock_locked()  local
1861 			owner = rt_mutex_owner(lock);  in rtlock_slowlock_locked()
1863 			owner = NULL;  in rtlock_slowlock_locked()
1866 		if (!owner || !rtmutex_spin_on_owner(lock, &waiter, owner)) {  in rtlock_slowlock_locked()