39 static noinline void __down(struct semaphore *sem);
40 static noinline int __down_interruptible(struct semaphore *sem);
41 static noinline int __down_killable(struct semaphore *sem);
42 static noinline int __down_timeout(struct semaphore *sem, long timeout);
43 static noinline void __up(struct semaphore *sem, struct wake_q_head *wake_q);
46 static inline void hung_task_sem_set_holder(struct semaphore *sem)  in hung_task_sem_set_holder()  argument
48 	WRITE_ONCE((sem)->last_holder, (unsigned long)current);  in hung_task_sem_set_holder()
51 static inline void hung_task_sem_clear_if_holder(struct semaphore *sem)  in hung_task_sem_clear_if_holder()  argument
53 	if (READ_ONCE((sem)->last_holder) == (unsigned long)current)  in hung_task_sem_clear_if_holder()
54 		WRITE_ONCE((sem)->last_holder, 0UL);  in hung_task_sem_clear_if_holder()
57 unsigned long sem_last_holder(struct semaphore *sem)  in sem_last_holder()  argument
59 	return READ_ONCE(sem->last_holder);  in sem_last_holder()
62 static inline void hung_task_sem_set_holder(struct semaphore *sem)  in hung_task_sem_set_holder()  argument
65 static inline void hung_task_sem_clear_if_holder(struct semaphore *sem)  in hung_task_sem_clear_if_holder()  argument
68 unsigned long sem_last_holder(struct semaphore *sem)  in sem_last_holder()  argument
74 static inline void __sem_acquire(struct semaphore *sem)  in __sem_acquire()  argument
76 	sem->count--;  in __sem_acquire()
77 	hung_task_sem_set_holder(sem);  in __sem_acquire()
91 void __sched down(struct semaphore *sem)  in down()  argument
96 	raw_spin_lock_irqsave(&sem->lock, flags);  in down()
97 	if (likely(sem->count > 0))  in down()
98 		__sem_acquire(sem);  in down()
100 		__down(sem);  in down()
101 	raw_spin_unlock_irqrestore(&sem->lock, flags);  in down()
114 int __sched down_interruptible(struct semaphore *sem)  in down_interruptible()  argument
120 	raw_spin_lock_irqsave(&sem->lock, flags);  in down_interruptible()
121 	if (likely(sem->count > 0))  in down_interruptible()
122 		__sem_acquire(sem);  in down_interruptible()
124 		result = __down_interruptible(sem);  in down_interruptible()
125 	raw_spin_unlock_irqrestore(&sem->lock, flags);  in down_interruptible()
141 int __sched down_killable(struct semaphore *sem)  in down_killable()  argument
147 	raw_spin_lock_irqsave(&sem->lock, flags);  in down_killable()
148 	if (likely(sem->count > 0))  in down_killable()
149 		__sem_acquire(sem);  in down_killable()
151 		result = __down_killable(sem);  in down_killable()
152 	raw_spin_unlock_irqrestore(&sem->lock, flags);  in down_killable()
171 int __sched down_trylock(struct semaphore *sem)  in down_trylock()  argument
176 	raw_spin_lock_irqsave(&sem->lock, flags);  in down_trylock()
177 	count = sem->count - 1;  in down_trylock()
179 		__sem_acquire(sem);  in down_trylock()
180 	raw_spin_unlock_irqrestore(&sem->lock, flags);  in down_trylock()
196 int __sched down_timeout(struct semaphore *sem, long timeout)  in down_timeout()  argument
202 	raw_spin_lock_irqsave(&sem->lock, flags);  in down_timeout()
203 	if (likely(sem->count > 0))  in down_timeout()
204 		__sem_acquire(sem);  in down_timeout()
206 		result = __down_timeout(sem, timeout);  in down_timeout()
207 	raw_spin_unlock_irqrestore(&sem->lock, flags);  in down_timeout()
220 void __sched up(struct semaphore *sem)  in up()  argument
225 	raw_spin_lock_irqsave(&sem->lock, flags);  in up()
227 	hung_task_sem_clear_if_holder(sem);  in up()
229 	if (likely(list_empty(&sem->wait_list)))  in up()
230 		sem->count++;  in up()
232 		__up(sem, &wake_q);  in up()
233 	raw_spin_unlock_irqrestore(&sem->lock, flags);  in up()
252 static inline int __sched ___down_common(struct semaphore *sem, long state,  in ___down_common()  argument
257 	list_add_tail(&waiter.list, &sem->wait_list);  in ___down_common()
267 		raw_spin_unlock_irq(&sem->lock);  in ___down_common()
269 		raw_spin_lock_irq(&sem->lock);  in ___down_common()
271 			hung_task_sem_set_holder(sem);  in ___down_common()
285 static inline int __sched __down_common(struct semaphore *sem, long state,  in __down_common()  argument
290 	hung_task_set_blocker(sem, BLOCKER_TYPE_SEM);  in __down_common()
292 	trace_contention_begin(sem, 0);  in __down_common()
293 	ret = ___down_common(sem, state, timeout);  in __down_common()
294 	trace_contention_end(sem, ret);  in __down_common()
301 static noinline void __sched __down(struct semaphore *sem)  in __down()  argument
303 	__down_common(sem, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);  in __down()
306 static noinline int __sched __down_interruptible(struct semaphore *sem)  in __down_interruptible()  argument
308 	return __down_common(sem, TASK_INTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT);  in __down_interruptible()
311 static noinline int __sched __down_killable(struct semaphore *sem)  in __down_killable()  argument
313 	return __down_common(sem, TASK_KILLABLE, MAX_SCHEDULE_TIMEOUT);  in __down_killable()
316 static noinline int __sched __down_timeout(struct semaphore *sem, long timeout)  in __down_timeout()  argument
318 	return __down_common(sem, TASK_UNINTERRUPTIBLE, timeout);  in __down_timeout()
321 static noinline void __sched __up(struct semaphore *sem,  in __up()  argument
324 	struct semaphore_waiter *waiter = list_first_entry(&sem->wait_list,  in __up()