106 static inline int arch_try_cmpxchg_niai8(int *lock, int old, int new)  in arch_try_cmpxchg_niai8()  argument
112 		"	cs	%[old],%[new],%[lock]\n"  in arch_try_cmpxchg_niai8()
113 		: [old] "+d" (old), [lock] "+Q" (*lock), "=@cc" (cc)  in arch_try_cmpxchg_niai8()
121 static inline int arch_try_cmpxchg_niai8(int *lock, int old, int new)  in arch_try_cmpxchg_niai8()  argument
123 	int expected = old;  in arch_try_cmpxchg_niai8()
127 		"	cs	%[old],%[new],%[lock]\n"  in arch_try_cmpxchg_niai8()
128 		: [old] "+d" (old), [lock] "+Q" (*lock)  in arch_try_cmpxchg_niai8()
131 	return expected == old;  in arch_try_cmpxchg_niai8()
159 	int lockval, ix, node_id, tail_id, old, new, owner, count;  in arch_spin_lock_queued()  local
169 	old = READ_ONCE(lp->lock);  in arch_spin_lock_queued()
171 		if ((old & _Q_LOCK_CPU_MASK) == 0 &&  in arch_spin_lock_queued()
172 		    (old & _Q_LOCK_STEAL_MASK) != _Q_LOCK_STEAL_MASK) {  in arch_spin_lock_queued()
180 			new = (old ? (old + _Q_LOCK_STEAL_ADD) : 0) | lockval;  in arch_spin_lock_queued()
181 			if (arch_try_cmpxchg(&lp->lock, &old, new))  in arch_spin_lock_queued()
188 		new = node_id | (old & _Q_LOCK_MASK);  in arch_spin_lock_queued()
189 		if (arch_try_cmpxchg(&lp->lock, &old, new))  in arch_spin_lock_queued()
193 	tail_id = old & _Q_TAIL_MASK;  in arch_spin_lock_queued()
200 	owner = arch_spin_yield_target(old, node);  in arch_spin_lock_queued()
212 			owner = arch_spin_yield_target(old, node);  in arch_spin_lock_queued()
221 		old = READ_ONCE(lp->lock);  in arch_spin_lock_queued()
222 		owner = old & _Q_LOCK_CPU_MASK;  in arch_spin_lock_queued()
224 			tail_id = old & _Q_TAIL_MASK;  in arch_spin_lock_queued()
226 			if (arch_try_cmpxchg(&lp->lock, &old, new))  in arch_spin_lock_queued()
252 	int lockval, old, new, owner, count;  in arch_spin_lock_classic()  local
263 		old = arch_load_niai4(&lp->lock);  in arch_spin_lock_classic()
264 		owner = old & _Q_LOCK_CPU_MASK;  in arch_spin_lock_classic()
267 			new = (old & _Q_TAIL_MASK) | lockval;  in arch_spin_lock_classic()
268 			if (arch_try_cmpxchg_niai8(&lp->lock, old, new)) {  in arch_spin_lock_classic()
331 	int old;  in arch_write_lock_wait()  local
340 		old = READ_ONCE(rw->cnts);  in arch_write_lock_wait()
341 		if ((old & 0x1ffff) == 0 &&  in arch_write_lock_wait()
342 		    arch_try_cmpxchg(&rw->cnts, &old, old | 0x10000))  in arch_write_lock_wait()