1 /* SPDX-License-Identifier: GPL-2.0 */
8  * Copyright (C) 2009-2012 Jason Baron <jbaron@redhat.com>
9  * Copyright (C) 2011-2012 Red Hat, Inc., Peter Zijlstra
31  * self-modifying code. Assuming toolchain and architecture support, if we
34  * (which defaults to false - and the true block is placed out of line).
38  * unconditional branch to the out-of-line true branch. Keys that are
45  * these calls then we run-time modify the branch target via a
46  * no-op -> jump or jump -> no-op conversion. For example, for an
49  * to the out-of-line of true branch.
61  * to the out-of-line block.
71  * Additional babbling in: Documentation/staging/static-keys.rst
123 static inline unsigned long jump_entry_code(const struct jump_entry *entry)  in jump_entry_code()  argument
125 	return (unsigned long)&entry->code + entry->code;  in jump_entry_code()
128 static inline unsigned long jump_entry_target(const struct jump_entry *entry)  in jump_entry_target()  argument
130 	return (unsigned long)&entry->target + entry->target;  in jump_entry_target()
133 static inline struct static_key *jump_entry_key(const struct jump_entry *entry)  in jump_entry_key()  argument
135 	long offset = entry->key & ~3L;  in jump_entry_key()
137 	return (struct static_key *)((unsigned long)&entry->key + offset);  in jump_entry_key()
142 static inline unsigned long jump_entry_code(const struct jump_entry *entry)  in jump_entry_code()  argument
144 	return entry->code;  in jump_entry_code()
147 static inline unsigned long jump_entry_target(const struct jump_entry *entry)  in jump_entry_target()  argument
149 	return entry->target;  in jump_entry_target()
152 static inline struct static_key *jump_entry_key(const struct jump_entry *entry)  in jump_entry_key()  argument
154 	return (struct static_key *)((unsigned long)entry->key & ~3UL);  in jump_entry_key()
159 static inline bool jump_entry_is_branch(const struct jump_entry *entry)  in jump_entry_is_branch()  argument
161 	return (unsigned long)entry->key & 1UL;  in jump_entry_is_branch()
164 static inline bool jump_entry_is_init(const struct jump_entry *entry)  in jump_entry_is_init()  argument
166 	return (unsigned long)entry->key & 2UL;  in jump_entry_is_init()
169 static inline void jump_entry_set_init(struct jump_entry *entry, bool set)  in jump_entry_set_init()  argument
172 		entry->key |= 2;  in jump_entry_set_init()
174 		entry->key &= ~2;  in jump_entry_set_init()
177 static inline int jump_entry_size(struct jump_entry *entry)  in jump_entry_size()  argument
182 	return arch_jump_entry_size(entry);  in jump_entry_size()
221 extern void arch_jump_label_transform(struct jump_entry *entry,
223 extern bool arch_jump_label_transform_queue(struct jump_entry *entry,
237 extern enum jump_label_type jump_label_init_type(struct jump_entry *entry);
242  * in 'low-level' headers. Thus, we are initializing .enabled with a
260 	return raw_atomic_read(&key->enabled);  in static_key_count()
288 	 * Prevent key->enabled getting negative to follow the same semantics  in static_key_fast_inc_not_disabled()
291 	v = atomic_read(&key->enabled);  in static_key_fast_inc_not_disabled()
295 	} while (!likely(atomic_try_cmpxchg(&key->enabled, &v, v + 1)));  in static_key_fast_inc_not_disabled()
303 	atomic_dec(&key->enabled);  in static_key_slow_dec()
321 	if (atomic_read(&key->enabled) != 0) {  in static_key_enable()
322 		WARN_ON_ONCE(atomic_read(&key->enabled) != 1);  in static_key_enable()
325 	atomic_set(&key->enabled, 1);  in static_key_enable()
332 	if (atomic_read(&key->enabled) != 1) {  in static_key_disable()
333 		WARN_ON_ONCE(atomic_read(&key->enabled) != 0);  in static_key_disable()
336 	atomic_set(&key->enabled, 0);  in static_key_disable()
350 /* -------------------------------------------------------------------------- */
370 #define DEFINE_STATIC_KEY_TRUE(name)	\  argument
371 	struct static_key_true name = STATIC_KEY_TRUE_INIT
373 #define DEFINE_STATIC_KEY_TRUE_RO(name)	\  argument
374 	struct static_key_true name __ro_after_init = STATIC_KEY_TRUE_INIT
376 #define DECLARE_STATIC_KEY_TRUE(name)	\  argument
377 	extern struct static_key_true name
379 #define DEFINE_STATIC_KEY_FALSE(name)	\  argument
380 	struct static_key_false name = STATIC_KEY_FALSE_INIT
382 #define DEFINE_STATIC_KEY_FALSE_RO(name)	\  argument
383 	struct static_key_false name __ro_after_init = STATIC_KEY_FALSE_INIT
385 #define DECLARE_STATIC_KEY_FALSE(name)	\  argument
386 	extern struct static_key_false name
388 #define DEFINE_STATIC_KEY_ARRAY_TRUE(name, count)		\  argument
389 	struct static_key_true name[count] = {			\
390 		[0 ... (count) - 1] = STATIC_KEY_TRUE_INIT,	\
393 #define DEFINE_STATIC_KEY_ARRAY_FALSE(name, count)		\  argument
394 	struct static_key_false name[count] = {			\
395 		[0 ... (count) - 1] = STATIC_KEY_FALSE_INIT,	\
398 #define _DEFINE_STATIC_KEY_1(name)	DEFINE_STATIC_KEY_TRUE(name)  argument
399 #define _DEFINE_STATIC_KEY_0(name)	DEFINE_STATIC_KEY_FALSE(name)  argument
400 #define DEFINE_STATIC_KEY_MAYBE(cfg, name)			\  argument
401 	__PASTE(_DEFINE_STATIC_KEY_, IS_ENABLED(cfg))(name)
403 #define _DEFINE_STATIC_KEY_RO_1(name)	DEFINE_STATIC_KEY_TRUE_RO(name)  argument
404 #define _DEFINE_STATIC_KEY_RO_0(name)	DEFINE_STATIC_KEY_FALSE_RO(name)  argument
405 #define DEFINE_STATIC_KEY_MAYBE_RO(cfg, name)			\  argument
406 	__PASTE(_DEFINE_STATIC_KEY_RO_, IS_ENABLED(cfg))(name)
408 #define _DECLARE_STATIC_KEY_1(name)	DECLARE_STATIC_KEY_TRUE(name)  argument
409 #define _DECLARE_STATIC_KEY_0(name)	DECLARE_STATIC_KEY_FALSE(name)  argument
410 #define DECLARE_STATIC_KEY_MAYBE(cfg, name)			\  argument
411 	__PASTE(_DECLARE_STATIC_KEY_, IS_ENABLED(cfg))(name)
432  * -----------+-----------------------+------------------
436  *            |    <br-stmts>	      |	1: ...
439  *            |			      |	L: <br-stmts>
442  * -----------+-----------------------+------------------
446  *            |    <br-stmts>	      |	1: ...
449  *            |			      |	L: <br-stmts>
452  * -----------+-----------------------+------------------
463  * -----------------------------+-----------
486 		branch = !arch_static_branch(&(x)->key, true);			\
488 		branch = !arch_static_branch_jump(&(x)->key, true);		\
498 		branch = arch_static_branch_jump(&(x)->key, false);		\
500 		branch = arch_static_branch(&(x)->key, false);			\
508 #define static_branch_likely(x)		likely_notrace(static_key_enabled(&(x)->key))
509 #define static_branch_unlikely(x)	unlikely_notrace(static_key_enabled(&(x)->key))
521 #define static_branch_inc(x)		static_key_slow_inc(&(x)->key)
522 #define static_branch_dec(x)		static_key_slow_dec(&(x)->key)
523 #define static_branch_inc_cpuslocked(x)	static_key_slow_inc_cpuslocked(&(x)->key)
524 #define static_branch_dec_cpuslocked(x)	static_key_slow_dec_cpuslocked(&(x)->key)
530 #define static_branch_enable(x)			static_key_enable(&(x)->key)
531 #define static_branch_disable(x)		static_key_disable(&(x)->key)
532 #define static_branch_enable_cpuslocked(x)	static_key_enable_cpuslocked(&(x)->key)
533 #define static_branch_disable_cpuslocked(x)	static_key_disable_cpuslocked(&(x)->key)