1 /*
2  * arch/arm/include/asm/mutex.h
3  *
4  * ARM optimized mutex locking primitives
5  *
6  * Please look into asm-generic/mutex-xchg.h for a formal definition.
7  */
8 #ifndef _ASM_MUTEX_H
9 #define _ASM_MUTEX_H
10 
11 #if __LINUX_ARM_ARCH__ < 6
12 /* On pre-ARMv6 hardware the swp based implementation is the most efficient. */
13 # include <asm-generic/mutex-xchg.h>
14 #else
15 
16 /*
17  * Attempting to lock a mutex on ARMv6+ can be done with a bastardized
18  * atomic decrement (it is not a reliable atomic decrement but it satisfies
19  * the defined semantics for our purpose, while being smaller and faster
20  * than a real atomic decrement or atomic swap.  The idea is to attempt
21  * decrementing the lock value only once.  If once decremented it isn't zero,
22  * or if its store-back fails due to a dispute on the exclusive store, we
23  * simply bail out immediately through the slow path where the lock will be
24  * reattempted until it succeeds.
25  */
26 static inline void
__mutex_fastpath_lock(atomic_t * count,void (* fail_fn)(atomic_t *))27 __mutex_fastpath_lock(atomic_t *count, void (*fail_fn)(atomic_t *))
28 {
29 	int __ex_flag, __res;
30 
31 	__asm__ (
32 
33 		"ldrex	%0, [%2]	\n\t"
34 		"sub	%0, %0, #1	\n\t"
35 		"strex	%1, %0, [%2]	"
36 
37 		: "=&r" (__res), "=&r" (__ex_flag)
38 		: "r" (&(count)->counter)
39 		: "cc","memory" );
40 
41 	__res |= __ex_flag;
42 	if (unlikely(__res != 0))
43 		fail_fn(count);
44 }
45 
46 static inline int
__mutex_fastpath_lock_retval(atomic_t * count,int (* fail_fn)(atomic_t *))47 __mutex_fastpath_lock_retval(atomic_t *count, int (*fail_fn)(atomic_t *))
48 {
49 	int __ex_flag, __res;
50 
51 	__asm__ (
52 
53 		"ldrex	%0, [%2]	\n\t"
54 		"sub	%0, %0, #1	\n\t"
55 		"strex	%1, %0, [%2]	"
56 
57 		: "=&r" (__res), "=&r" (__ex_flag)
58 		: "r" (&(count)->counter)
59 		: "cc","memory" );
60 
61 	__res |= __ex_flag;
62 	if (unlikely(__res != 0))
63 		__res = fail_fn(count);
64 	return __res;
65 }
66 
67 /*
68  * Same trick is used for the unlock fast path. However the original value,
69  * rather than the result, is used to test for success in order to have
70  * better generated assembly.
71  */
72 static inline void
__mutex_fastpath_unlock(atomic_t * count,void (* fail_fn)(atomic_t *))73 __mutex_fastpath_unlock(atomic_t *count, void (*fail_fn)(atomic_t *))
74 {
75 	int __ex_flag, __res, __orig;
76 
77 	__asm__ (
78 
79 		"ldrex	%0, [%3]	\n\t"
80 		"add	%1, %0, #1	\n\t"
81 		"strex	%2, %1, [%3]	"
82 
83 		: "=&r" (__orig), "=&r" (__res), "=&r" (__ex_flag)
84 		: "r" (&(count)->counter)
85 		: "cc","memory" );
86 
87 	__orig |= __ex_flag;
88 	if (unlikely(__orig != 0))
89 		fail_fn(count);
90 }
91 
92 /*
93  * If the unlock was done on a contended lock, or if the unlock simply fails
94  * then the mutex remains locked.
95  */
96 #define __mutex_slowpath_needs_to_unlock()	1
97 
98 /*
99  * For __mutex_fastpath_trylock we use another construct which could be
100  * described as a "single value cmpxchg".
101  *
102  * This provides the needed trylock semantics like cmpxchg would, but it is
103  * lighter and less generic than a true cmpxchg implementation.
104  */
105 static inline int
__mutex_fastpath_trylock(atomic_t * count,int (* fail_fn)(atomic_t *))106 __mutex_fastpath_trylock(atomic_t *count, int (*fail_fn)(atomic_t *))
107 {
108 	int __ex_flag, __res, __orig;
109 
110 	__asm__ (
111 
112 		"1: ldrex	%0, [%3]	\n\t"
113 		"subs		%1, %0, #1	\n\t"
114 		"strexeq	%2, %1, [%3]	\n\t"
115 		"movlt		%0, #0		\n\t"
116 		"cmpeq		%2, #0		\n\t"
117 		"bgt		1b		"
118 
119 		: "=&r" (__orig), "=&r" (__res), "=&r" (__ex_flag)
120 		: "r" (&count->counter)
121 		: "cc", "memory" );
122 
123 	return __orig;
124 }
125 
126 #endif
127 #endif
128