1 // SPDX-License-Identifier: GPL-2.0
2
3 #include <linux/irq-entry-common.h>
4 #include <linux/resume_user_mode.h>
5 #include <linux/highmem.h>
6 #include <linux/jump_label.h>
7 #include <linux/kmsan.h>
8 #include <linux/livepatch.h>
9 #include <linux/tick.h>
10
11 /* Workaround to allow gradual conversion of architecture code */
arch_do_signal_or_restart(struct pt_regs * regs)12 void __weak arch_do_signal_or_restart(struct pt_regs *regs) { }
13
14 #ifdef CONFIG_HAVE_GENERIC_TIF_BITS
15 #define EXIT_TO_USER_MODE_WORK_LOOP (EXIT_TO_USER_MODE_WORK & ~_TIF_RSEQ)
16 #else
17 #define EXIT_TO_USER_MODE_WORK_LOOP (EXIT_TO_USER_MODE_WORK)
18 #endif
19
20 /* TIF bits, which prevent a time slice extension. */
21 #ifdef CONFIG_PREEMPT_RT
22 /*
23 * Since rseq slice ext has a direct correlation to the worst case
24 * scheduling latency (schedule is delayed after all), only have it affect
25 * LAZY reschedules on PREEMPT_RT for now.
26 *
27 * However, since this delay is only applicable to userspace, a value
28 * for rseq_slice_extension_nsec that is strictly less than the worst case
29 * kernel space preempt_disable() region, should mean the scheduling latency
30 * is not affected, even for !LAZY.
31 *
32 * However, since this value depends on the hardware at hand, it cannot be
33 * pre-determined in any sensible way. Hence punt on this problem for now.
34 */
35 # define TIF_SLICE_EXT_SCHED (_TIF_NEED_RESCHED_LAZY)
36 #else
37 # define TIF_SLICE_EXT_SCHED (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY)
38 #endif
39 #define TIF_SLICE_EXT_DENY (EXIT_TO_USER_MODE_WORK & ~TIF_SLICE_EXT_SCHED)
40
__exit_to_user_mode_loop(struct pt_regs * regs,unsigned long ti_work)41 static __always_inline unsigned long __exit_to_user_mode_loop(struct pt_regs *regs,
42 unsigned long ti_work)
43 {
44 /*
45 * Before returning to user space ensure that all pending work
46 * items have been completed.
47 */
48 while (ti_work & EXIT_TO_USER_MODE_WORK_LOOP) {
49
50 local_irq_enable();
51
52 if (ti_work & (_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY)) {
53 if (!rseq_grant_slice_extension(ti_work, TIF_SLICE_EXT_DENY))
54 schedule();
55 }
56
57 if (ti_work & _TIF_UPROBE)
58 uprobe_notify_resume(regs);
59
60 if (ti_work & _TIF_PATCH_PENDING)
61 klp_update_patch_state(current);
62
63 if (ti_work & (_TIF_SIGPENDING | _TIF_NOTIFY_SIGNAL))
64 arch_do_signal_or_restart(regs);
65
66 if (ti_work & _TIF_NOTIFY_RESUME)
67 resume_user_mode_work(regs);
68
69 /* Architecture specific TIF work */
70 arch_exit_to_user_mode_work(regs, ti_work);
71
72 /*
73 * Disable interrupts and reevaluate the work flags as they
74 * might have changed while interrupts and preemption was
75 * enabled above.
76 */
77 local_irq_disable();
78
79 /* Check if any of the above work has queued a deferred wakeup */
80 tick_nohz_user_enter_prepare();
81
82 ti_work = read_thread_flags();
83 }
84
85 /* Return the latest work state for arch_exit_to_user_mode() */
86 return ti_work;
87 }
88
89 /**
90 * exit_to_user_mode_loop - do any pending work before leaving to user space
91 * @regs: Pointer to pt_regs on entry stack
92 * @ti_work: TIF work flags as read by the caller
93 */
exit_to_user_mode_loop(struct pt_regs * regs,unsigned long ti_work)94 __always_inline unsigned long exit_to_user_mode_loop(struct pt_regs *regs,
95 unsigned long ti_work)
96 {
97 for (;;) {
98 ti_work = __exit_to_user_mode_loop(regs, ti_work);
99
100 if (likely(!rseq_exit_to_user_mode_restart(regs, ti_work)))
101 return ti_work;
102 ti_work = read_thread_flags();
103 }
104 }
105
irqentry_enter(struct pt_regs * regs)106 noinstr irqentry_state_t irqentry_enter(struct pt_regs *regs)
107 {
108 if (user_mode(regs)) {
109 irqentry_state_t ret = {
110 .exit_rcu = false,
111 };
112
113 irqentry_enter_from_user_mode(regs);
114 return ret;
115 }
116
117 return irqentry_enter_from_kernel_mode(regs);
118 }
119
120 /**
121 * arch_irqentry_exit_need_resched - Architecture specific need resched function
122 *
123 * Invoked from raw_irqentry_exit_cond_resched() to check if resched is needed.
124 * Defaults return true.
125 *
126 * The main purpose is to permit arch to avoid preemption of a task from an IRQ.
127 */
128 static inline bool arch_irqentry_exit_need_resched(void);
129
130 #ifndef arch_irqentry_exit_need_resched
arch_irqentry_exit_need_resched(void)131 static inline bool arch_irqentry_exit_need_resched(void) { return true; }
132 #endif
133
raw_irqentry_exit_cond_resched(void)134 void raw_irqentry_exit_cond_resched(void)
135 {
136 if (!preempt_count()) {
137 /* Sanity check RCU and thread stack */
138 rcu_irq_exit_check_preempt();
139 if (IS_ENABLED(CONFIG_DEBUG_ENTRY))
140 WARN_ON_ONCE(!on_thread_stack());
141 if (need_resched() && arch_irqentry_exit_need_resched())
142 preempt_schedule_irq();
143 }
144 }
145 #ifdef CONFIG_PREEMPT_DYNAMIC
146 #if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL)
147 DEFINE_STATIC_CALL(irqentry_exit_cond_resched, raw_irqentry_exit_cond_resched);
148 #elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY)
149 DEFINE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched);
dynamic_irqentry_exit_cond_resched(void)150 void dynamic_irqentry_exit_cond_resched(void)
151 {
152 if (!static_branch_unlikely(&sk_dynamic_irqentry_exit_cond_resched))
153 return;
154 raw_irqentry_exit_cond_resched();
155 }
156 #endif
157 #endif
158
irqentry_exit(struct pt_regs * regs,irqentry_state_t state)159 noinstr void irqentry_exit(struct pt_regs *regs, irqentry_state_t state)
160 {
161 if (user_mode(regs))
162 irqentry_exit_to_user_mode(regs);
163 else
164 irqentry_exit_to_kernel_mode(regs, state);
165 }
166
irqentry_nmi_enter(struct pt_regs * regs)167 irqentry_state_t noinstr irqentry_nmi_enter(struct pt_regs *regs)
168 {
169 irqentry_state_t irq_state;
170
171 irq_state.lockdep = lockdep_hardirqs_enabled();
172
173 __nmi_enter();
174 lockdep_hardirqs_off(CALLER_ADDR0);
175 lockdep_hardirq_enter();
176 ct_nmi_enter();
177
178 instrumentation_begin();
179 kmsan_unpoison_entry_regs(regs);
180 trace_hardirqs_off_finish();
181 ftrace_nmi_enter();
182 instrumentation_end();
183
184 return irq_state;
185 }
186
irqentry_nmi_exit(struct pt_regs * regs,irqentry_state_t irq_state)187 void noinstr irqentry_nmi_exit(struct pt_regs *regs, irqentry_state_t irq_state)
188 {
189 instrumentation_begin();
190 ftrace_nmi_exit();
191 if (irq_state.lockdep) {
192 trace_hardirqs_on_prepare();
193 lockdep_hardirqs_on_prepare();
194 }
195 instrumentation_end();
196
197 ct_nmi_exit();
198 lockdep_hardirq_exit();
199 if (irq_state.lockdep)
200 lockdep_hardirqs_on(CALLER_ADDR0);
201 __nmi_exit();
202 }
203