1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/ftrace.h>
3 #include <linux/tracepoint.h>
4 #include <linux/kernel.h>
5 #include <linux/module.h>
6 #include <linux/init.h>
7 #include <linux/rv.h>
8 #include <rv/instrumentation.h>
9
10 #define MODULE_NAME "nomiss"
11
12 #include <uapi/linux/sched/types.h>
13 #include <trace/events/syscalls.h>
14 #include <trace/events/sched.h>
15 #include <trace/events/task.h>
16 #include <rv_trace.h>
17
18 #define RV_MON_TYPE RV_MON_PER_OBJ
19 #define HA_TIMER_TYPE HA_TIMER_WHEEL
20 /* The start condition is on sched_switch, it's dangerous to allocate there */
21 #define DA_SKIP_AUTO_ALLOC
22 typedef struct sched_dl_entity *monitor_target;
23 #include "nomiss.h"
24 #include <rv/ha_monitor.h>
25 #include <monitors/deadline/deadline.h>
26
27 /*
28 * User configurable deadline threshold. If the total utilisation of deadline
29 * tasks is larger than 1, they are only guaranteed bounded tardiness. See
30 * Documentation/scheduler/sched-deadline.rst for more details.
31 * The minimum tardiness without sched_feat(HRTICK_DL) is 1 tick to accommodate
32 * for throttle enforced on the next tick.
33 */
34 static u64 deadline_thresh = TICK_NSEC;
35 module_param(deadline_thresh, ullong, 0644);
36 #define DEADLINE_NS(ha_mon) (ha_get_target(ha_mon)->dl_deadline + deadline_thresh)
37
ha_get_env(struct ha_monitor * ha_mon,enum envs_nomiss env,u64 time_ns)38 static u64 ha_get_env(struct ha_monitor *ha_mon, enum envs_nomiss env, u64 time_ns)
39 {
40 if (env == clk_nomiss)
41 return ha_get_clk_ns(ha_mon, env, time_ns);
42 else if (env == is_constr_dl_nomiss)
43 return !dl_is_implicit(ha_get_target(ha_mon));
44 else if (env == is_defer_nomiss)
45 return ha_get_target(ha_mon)->dl_defer;
46 return ENV_INVALID_VALUE;
47 }
48
ha_reset_env(struct ha_monitor * ha_mon,enum envs_nomiss env,u64 time_ns)49 static void ha_reset_env(struct ha_monitor *ha_mon, enum envs_nomiss env, u64 time_ns)
50 {
51 if (env == clk_nomiss)
52 ha_reset_clk_ns(ha_mon, env, time_ns);
53 }
54
ha_verify_invariants(struct ha_monitor * ha_mon,enum states curr_state,enum events event,enum states next_state,u64 time_ns)55 static inline bool ha_verify_invariants(struct ha_monitor *ha_mon,
56 enum states curr_state, enum events event,
57 enum states next_state, u64 time_ns)
58 {
59 if (curr_state == ready_nomiss)
60 return ha_check_invariant_ns(ha_mon, clk_nomiss, time_ns);
61 else if (curr_state == running_nomiss)
62 return ha_check_invariant_ns(ha_mon, clk_nomiss, time_ns);
63 return true;
64 }
65
ha_convert_inv_guard(struct ha_monitor * ha_mon,enum states curr_state,enum events event,enum states next_state,u64 time_ns)66 static inline void ha_convert_inv_guard(struct ha_monitor *ha_mon,
67 enum states curr_state, enum events event,
68 enum states next_state, u64 time_ns)
69 {
70 if (curr_state == next_state)
71 return;
72 if (curr_state == ready_nomiss)
73 ha_inv_to_guard(ha_mon, clk_nomiss, DEADLINE_NS(ha_mon), time_ns);
74 else if (curr_state == running_nomiss)
75 ha_inv_to_guard(ha_mon, clk_nomiss, DEADLINE_NS(ha_mon), time_ns);
76 }
77
ha_verify_guards(struct ha_monitor * ha_mon,enum states curr_state,enum events event,enum states next_state,u64 time_ns)78 static inline bool ha_verify_guards(struct ha_monitor *ha_mon,
79 enum states curr_state, enum events event,
80 enum states next_state, u64 time_ns)
81 {
82 bool res = true;
83
84 if (curr_state == ready_nomiss && event == dl_replenish_nomiss)
85 ha_reset_env(ha_mon, clk_nomiss, time_ns);
86 else if (curr_state == ready_nomiss && event == dl_throttle_nomiss)
87 res = ha_get_env(ha_mon, is_defer_nomiss, time_ns) == 1ull;
88 else if (curr_state == idle_nomiss && event == dl_replenish_nomiss)
89 ha_reset_env(ha_mon, clk_nomiss, time_ns);
90 else if (curr_state == running_nomiss && event == dl_replenish_nomiss)
91 ha_reset_env(ha_mon, clk_nomiss, time_ns);
92 else if (curr_state == sleeping_nomiss && event == dl_replenish_nomiss)
93 ha_reset_env(ha_mon, clk_nomiss, time_ns);
94 else if (curr_state == sleeping_nomiss && event == dl_throttle_nomiss)
95 res = ha_get_env(ha_mon, is_constr_dl_nomiss, time_ns) == 1ull ||
96 ha_get_env(ha_mon, is_defer_nomiss, time_ns) == 1ull;
97 else if (curr_state == throttled_nomiss && event == dl_replenish_nomiss)
98 ha_reset_env(ha_mon, clk_nomiss, time_ns);
99 return res;
100 }
101
ha_setup_invariants(struct ha_monitor * ha_mon,enum states curr_state,enum events event,enum states next_state,u64 time_ns)102 static inline void ha_setup_invariants(struct ha_monitor *ha_mon,
103 enum states curr_state, enum events event,
104 enum states next_state, u64 time_ns)
105 {
106 if (next_state == curr_state && event != dl_replenish_nomiss)
107 return;
108 if (next_state == ready_nomiss)
109 ha_start_timer_ns(ha_mon, clk_nomiss, DEADLINE_NS(ha_mon), time_ns);
110 else if (next_state == running_nomiss)
111 ha_start_timer_ns(ha_mon, clk_nomiss, DEADLINE_NS(ha_mon), time_ns);
112 else if (curr_state == ready_nomiss)
113 ha_cancel_timer(ha_mon);
114 else if (curr_state == running_nomiss)
115 ha_cancel_timer(ha_mon);
116 }
117
ha_verify_constraint(struct ha_monitor * ha_mon,enum states curr_state,enum events event,enum states next_state,u64 time_ns)118 static bool ha_verify_constraint(struct ha_monitor *ha_mon,
119 enum states curr_state, enum events event,
120 enum states next_state, u64 time_ns)
121 {
122 if (!ha_verify_invariants(ha_mon, curr_state, event, next_state, time_ns))
123 return false;
124
125 ha_convert_inv_guard(ha_mon, curr_state, event, next_state, time_ns);
126
127 if (!ha_verify_guards(ha_mon, curr_state, event, next_state, time_ns))
128 return false;
129
130 ha_setup_invariants(ha_mon, curr_state, event, next_state, time_ns);
131
132 return true;
133 }
134
handle_dl_replenish(void * data,struct sched_dl_entity * dl_se,int cpu,u8 type)135 static void handle_dl_replenish(void *data, struct sched_dl_entity *dl_se,
136 int cpu, u8 type)
137 {
138 if (is_supported_type(type))
139 da_handle_event(EXPAND_ID(dl_se, cpu, type), dl_replenish_nomiss);
140 }
141
handle_dl_throttle(void * data,struct sched_dl_entity * dl_se,int cpu,u8 type)142 static void handle_dl_throttle(void *data, struct sched_dl_entity *dl_se,
143 int cpu, u8 type)
144 {
145 if (is_supported_type(type))
146 da_handle_event(EXPAND_ID(dl_se, cpu, type), dl_throttle_nomiss);
147 }
148
handle_dl_server_stop(void * data,struct sched_dl_entity * dl_se,int cpu,u8 type)149 static void handle_dl_server_stop(void *data, struct sched_dl_entity *dl_se,
150 int cpu, u8 type)
151 {
152 /*
153 * This isn't the standard use of da_handle_start_run_event since this
154 * event cannot only occur from the initial state.
155 * It is fine to use here because it always brings to a known state and
156 * the fact we "pretend" the transition starts from the initial state
157 * has no side effect.
158 */
159 if (is_supported_type(type))
160 da_handle_start_run_event(EXPAND_ID(dl_se, cpu, type), dl_server_stop_nomiss);
161 }
162
handle_server_switch(struct task_struct * next,int cpu,u8 type)163 static inline void handle_server_switch(struct task_struct *next, int cpu, u8 type)
164 {
165 struct sched_dl_entity *dl_se = get_server(next, type);
166
167 if (dl_se && is_idle_task(next))
168 da_handle_event(EXPAND_ID(dl_se, cpu, type), dl_server_idle_nomiss);
169 }
170
handle_sched_switch(void * data,bool preempt,struct task_struct * prev,struct task_struct * next,unsigned int prev_state)171 static void handle_sched_switch(void *data, bool preempt,
172 struct task_struct *prev,
173 struct task_struct *next,
174 unsigned int prev_state)
175 {
176 int cpu = task_cpu(next);
177
178 if (prev_state != TASK_RUNNING && !preempt && prev->policy == SCHED_DEADLINE)
179 da_handle_event(EXPAND_ID_TASK(prev), sched_switch_suspend_nomiss);
180 if (next->policy == SCHED_DEADLINE)
181 da_handle_start_run_event(EXPAND_ID_TASK(next), sched_switch_in_nomiss);
182
183 /*
184 * The server is available in next only if the next task is boosted,
185 * otherwise we need to retrieve it.
186 * Here the server continues in the state running/armed until actually
187 * stopped, this works since we continue expecting a throttle.
188 */
189 if (next->dl_server)
190 da_handle_start_event(EXPAND_ID(next->dl_server, cpu,
191 get_server_type(next)),
192 sched_switch_in_nomiss);
193 else {
194 handle_server_switch(next, cpu, DL_SERVER_FAIR);
195 if (IS_ENABLED(CONFIG_SCHED_CLASS_EXT))
196 handle_server_switch(next, cpu, DL_SERVER_EXT);
197 }
198 }
199
handle_sys_enter(void * data,struct pt_regs * regs,long id)200 static void handle_sys_enter(void *data, struct pt_regs *regs, long id)
201 {
202 struct task_struct *p;
203 int new_policy = -1;
204 pid_t pid = 0;
205
206 new_policy = extract_params(regs, id, &pid);
207 if (new_policy < 0)
208 return;
209 guard(rcu)();
210 p = pid ? find_task_by_vpid(pid) : current;
211 if (unlikely(!p) || new_policy == p->policy)
212 return;
213
214 if (p->policy == SCHED_DEADLINE)
215 da_reset(EXPAND_ID_TASK(p));
216 else if (new_policy == SCHED_DEADLINE)
217 da_create_or_get(EXPAND_ID_TASK(p));
218 }
219
handle_sched_wakeup(void * data,struct task_struct * tsk)220 static void handle_sched_wakeup(void *data, struct task_struct *tsk)
221 {
222 if (tsk->policy == SCHED_DEADLINE)
223 da_handle_event(EXPAND_ID_TASK(tsk), sched_wakeup_nomiss);
224 }
225
enable_nomiss(void)226 static int enable_nomiss(void)
227 {
228 int retval;
229
230 retval = da_monitor_init();
231 if (retval)
232 return retval;
233
234 retval = init_storage(false);
235 if (retval)
236 return retval;
237 rv_attach_trace_probe("nomiss", sched_dl_replenish_tp, handle_dl_replenish);
238 rv_attach_trace_probe("nomiss", sched_dl_throttle_tp, handle_dl_throttle);
239 rv_attach_trace_probe("nomiss", sched_dl_server_stop_tp, handle_dl_server_stop);
240 rv_attach_trace_probe("nomiss", sched_switch, handle_sched_switch);
241 rv_attach_trace_probe("nomiss", sched_wakeup, handle_sched_wakeup);
242 if (!should_skip_syscall_handle())
243 rv_attach_trace_probe("nomiss", sys_enter, handle_sys_enter);
244 rv_attach_trace_probe("nomiss", task_newtask, handle_newtask);
245 rv_attach_trace_probe("nomiss", sched_process_exit, handle_exit);
246
247 return 0;
248 }
249
disable_nomiss(void)250 static void disable_nomiss(void)
251 {
252 rv_this.enabled = 0;
253
254 /* Those are RCU writers, detach earlier hoping to close a bit faster */
255 rv_detach_trace_probe("nomiss", task_newtask, handle_newtask);
256 rv_detach_trace_probe("nomiss", sched_process_exit, handle_exit);
257 if (!should_skip_syscall_handle())
258 rv_detach_trace_probe("nomiss", sys_enter, handle_sys_enter);
259
260 rv_detach_trace_probe("nomiss", sched_dl_replenish_tp, handle_dl_replenish);
261 rv_detach_trace_probe("nomiss", sched_dl_throttle_tp, handle_dl_throttle);
262 rv_detach_trace_probe("nomiss", sched_dl_server_stop_tp, handle_dl_server_stop);
263 rv_detach_trace_probe("nomiss", sched_switch, handle_sched_switch);
264 rv_detach_trace_probe("nomiss", sched_wakeup, handle_sched_wakeup);
265
266 da_monitor_destroy();
267 }
268
269 static struct rv_monitor rv_this = {
270 .name = "nomiss",
271 .description = "dl entities run to completion before their deadline.",
272 .enable = enable_nomiss,
273 .disable = disable_nomiss,
274 .reset = da_monitor_reset_all,
275 .enabled = 0,
276 };
277
register_nomiss(void)278 static int __init register_nomiss(void)
279 {
280 return rv_register_monitor(&rv_this, &rv_deadline);
281 }
282
unregister_nomiss(void)283 static void __exit unregister_nomiss(void)
284 {
285 rv_unregister_monitor(&rv_this);
286 }
287
288 module_init(register_nomiss);
289 module_exit(unregister_nomiss);
290
291 MODULE_LICENSE("GPL");
292 MODULE_AUTHOR("Gabriele Monaco <gmonaco@redhat.com>");
293 MODULE_DESCRIPTION("nomiss: dl entities run to completion before their deadline.");
294