1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /* delayacct.h - per-task delay accounting
3 *
4 * Copyright (C) Shailabh Nagar, IBM Corp. 2006
5 */
6
7 #ifndef _LINUX_DELAYACCT_H
8 #define _LINUX_DELAYACCT_H
9
10 #include <uapi/linux/taskstats.h>
11
12 #ifdef CONFIG_TASK_DELAY_ACCT
13 struct task_delay_info {
14 raw_spinlock_t lock;
15
16 /* For each stat XXX, add following, aligned appropriately
17 *
18 * struct timespec XXX_start, XXX_end;
19 * u64 XXX_delay;
20 * u32 XXX_count;
21 *
22 * Atomicity of updates to XXX_delay, XXX_count protected by
23 * single lock above (split into XXX_lock if contention is an issue).
24 */
25
26 /*
27 * XXX_count is incremented on every XXX operation, the delay
28 * associated with the operation is added to XXX_delay.
29 * XXX_delay contains the accumulated delay time in nanoseconds.
30 */
31 u64 blkio_start;
32 u64 blkio_delay_max;
33 u64 blkio_delay_min;
34 u64 blkio_delay; /* wait for sync block io completion */
35 u64 swapin_start;
36 u64 swapin_delay_max;
37 u64 swapin_delay_min;
38 u64 swapin_delay; /* wait for swapin */
39 u32 blkio_count; /* total count of the number of sync block */
40 /* io operations performed */
41 u32 swapin_count; /* total count of swapin */
42
43 u64 freepages_start;
44 u64 freepages_delay_max;
45 u64 freepages_delay_min;
46 u64 freepages_delay; /* wait for memory reclaim */
47
48 u64 thrashing_start;
49 u64 thrashing_delay_max;
50 u64 thrashing_delay_min;
51 u64 thrashing_delay; /* wait for thrashing page */
52
53 u64 compact_start;
54 u64 compact_delay_max;
55 u64 compact_delay_min;
56 u64 compact_delay; /* wait for memory compact */
57
58 u64 wpcopy_start;
59 u64 wpcopy_delay_max;
60 u64 wpcopy_delay_min;
61 u64 wpcopy_delay; /* wait for write-protect copy */
62
63 u64 irq_delay_max;
64 u64 irq_delay_min;
65 u64 irq_delay; /* wait for IRQ/SOFTIRQ */
66
67 u32 freepages_count; /* total count of memory reclaim */
68 u32 thrashing_count; /* total count of thrash waits */
69 u32 compact_count; /* total count of memory compact */
70 u32 wpcopy_count; /* total count of write-protect copy */
71 u32 irq_count; /* total count of IRQ/SOFTIRQ */
72
73 struct timespec64 blkio_delay_max_ts;
74 struct timespec64 swapin_delay_max_ts;
75 struct timespec64 freepages_delay_max_ts;
76 struct timespec64 thrashing_delay_max_ts;
77 struct timespec64 compact_delay_max_ts;
78 struct timespec64 wpcopy_delay_max_ts;
79 struct timespec64 irq_delay_max_ts;
80 };
81 #endif
82
83 #include <linux/sched.h>
84 #include <linux/slab.h>
85 #include <linux/jump_label.h>
86
87 #ifdef CONFIG_TASK_DELAY_ACCT
88 DECLARE_STATIC_KEY_FALSE(delayacct_key);
89 extern int delayacct_on; /* Delay accounting turned on/off */
90 extern struct kmem_cache *delayacct_cache;
91 extern void delayacct_init(void);
92
93 extern void __delayacct_tsk_init(struct task_struct *);
94 extern void __delayacct_tsk_exit(struct task_struct *);
95 extern void __delayacct_blkio_start(void);
96 extern void __delayacct_blkio_end(struct task_struct *);
97 extern int delayacct_add_tsk(struct taskstats *, struct task_struct *);
98 extern __u64 __delayacct_blkio_ticks(struct task_struct *);
99 extern void __delayacct_freepages_start(void);
100 extern void __delayacct_freepages_end(void);
101 extern void __delayacct_thrashing_start(bool *in_thrashing);
102 extern void __delayacct_thrashing_end(bool *in_thrashing);
103 extern void __delayacct_swapin_start(void);
104 extern void __delayacct_swapin_end(void);
105 extern void __delayacct_compact_start(void);
106 extern void __delayacct_compact_end(void);
107 extern void __delayacct_wpcopy_start(void);
108 extern void __delayacct_wpcopy_end(void);
109 extern void __delayacct_irq(struct task_struct *task, u32 delta);
110
delayacct_tsk_init(struct task_struct * tsk)111 static inline void delayacct_tsk_init(struct task_struct *tsk)
112 {
113 /* reinitialize in case parent's non-null pointer was dup'ed*/
114 tsk->delays = NULL;
115 if (delayacct_on)
116 __delayacct_tsk_init(tsk);
117 }
118
119 /* Free tsk->delays. Called from bad fork and __put_task_struct
120 * where there's no risk of tsk->delays being accessed elsewhere
121 */
delayacct_tsk_free(struct task_struct * tsk)122 static inline void delayacct_tsk_free(struct task_struct *tsk)
123 {
124 if (tsk->delays)
125 kmem_cache_free(delayacct_cache, tsk->delays);
126 tsk->delays = NULL;
127 }
128
delayacct_blkio_start(void)129 static inline void delayacct_blkio_start(void)
130 {
131 if (!static_branch_unlikely(&delayacct_key))
132 return;
133
134 if (current->delays)
135 __delayacct_blkio_start();
136 }
137
delayacct_blkio_end(struct task_struct * p)138 static inline void delayacct_blkio_end(struct task_struct *p)
139 {
140 if (!static_branch_unlikely(&delayacct_key))
141 return;
142
143 if (p->delays)
144 __delayacct_blkio_end(p);
145 }
146
delayacct_blkio_ticks(struct task_struct * tsk)147 static inline __u64 delayacct_blkio_ticks(struct task_struct *tsk)
148 {
149 if (tsk->delays)
150 return __delayacct_blkio_ticks(tsk);
151 return 0;
152 }
153
delayacct_freepages_start(void)154 static inline void delayacct_freepages_start(void)
155 {
156 if (!static_branch_unlikely(&delayacct_key))
157 return;
158
159 if (current->delays)
160 __delayacct_freepages_start();
161 }
162
delayacct_freepages_end(void)163 static inline void delayacct_freepages_end(void)
164 {
165 if (!static_branch_unlikely(&delayacct_key))
166 return;
167
168 if (current->delays)
169 __delayacct_freepages_end();
170 }
171
delayacct_thrashing_start(bool * in_thrashing)172 static inline void delayacct_thrashing_start(bool *in_thrashing)
173 {
174 if (!static_branch_unlikely(&delayacct_key))
175 return;
176
177 if (current->delays)
178 __delayacct_thrashing_start(in_thrashing);
179 }
180
delayacct_thrashing_end(bool * in_thrashing)181 static inline void delayacct_thrashing_end(bool *in_thrashing)
182 {
183 if (!static_branch_unlikely(&delayacct_key))
184 return;
185
186 if (current->delays)
187 __delayacct_thrashing_end(in_thrashing);
188 }
189
delayacct_swapin_start(void)190 static inline void delayacct_swapin_start(void)
191 {
192 if (!static_branch_unlikely(&delayacct_key))
193 return;
194
195 if (current->delays)
196 __delayacct_swapin_start();
197 }
198
delayacct_swapin_end(void)199 static inline void delayacct_swapin_end(void)
200 {
201 if (!static_branch_unlikely(&delayacct_key))
202 return;
203
204 if (current->delays)
205 __delayacct_swapin_end();
206 }
207
delayacct_compact_start(void)208 static inline void delayacct_compact_start(void)
209 {
210 if (!static_branch_unlikely(&delayacct_key))
211 return;
212
213 if (current->delays)
214 __delayacct_compact_start();
215 }
216
delayacct_compact_end(void)217 static inline void delayacct_compact_end(void)
218 {
219 if (!static_branch_unlikely(&delayacct_key))
220 return;
221
222 if (current->delays)
223 __delayacct_compact_end();
224 }
225
delayacct_wpcopy_start(void)226 static inline void delayacct_wpcopy_start(void)
227 {
228 if (!static_branch_unlikely(&delayacct_key))
229 return;
230
231 if (current->delays)
232 __delayacct_wpcopy_start();
233 }
234
delayacct_wpcopy_end(void)235 static inline void delayacct_wpcopy_end(void)
236 {
237 if (!static_branch_unlikely(&delayacct_key))
238 return;
239
240 if (current->delays)
241 __delayacct_wpcopy_end();
242 }
243
delayacct_irq(struct task_struct * task,u32 delta)244 static inline void delayacct_irq(struct task_struct *task, u32 delta)
245 {
246 if (!static_branch_unlikely(&delayacct_key))
247 return;
248
249 if (task->delays)
250 __delayacct_irq(task, delta);
251 }
252
253 #else
delayacct_init(void)254 static inline void delayacct_init(void)
255 {}
delayacct_tsk_init(struct task_struct * tsk)256 static inline void delayacct_tsk_init(struct task_struct *tsk)
257 {}
delayacct_tsk_free(struct task_struct * tsk)258 static inline void delayacct_tsk_free(struct task_struct *tsk)
259 {}
delayacct_blkio_start(void)260 static inline void delayacct_blkio_start(void)
261 {}
delayacct_blkio_end(struct task_struct * p)262 static inline void delayacct_blkio_end(struct task_struct *p)
263 {}
delayacct_add_tsk(struct taskstats * d,struct task_struct * tsk)264 static inline int delayacct_add_tsk(struct taskstats *d,
265 struct task_struct *tsk)
266 { return 0; }
delayacct_blkio_ticks(struct task_struct * tsk)267 static inline __u64 delayacct_blkio_ticks(struct task_struct *tsk)
268 { return 0; }
delayacct_is_task_waiting_on_io(struct task_struct * p)269 static inline int delayacct_is_task_waiting_on_io(struct task_struct *p)
270 { return 0; }
delayacct_freepages_start(void)271 static inline void delayacct_freepages_start(void)
272 {}
delayacct_freepages_end(void)273 static inline void delayacct_freepages_end(void)
274 {}
delayacct_thrashing_start(bool * in_thrashing)275 static inline void delayacct_thrashing_start(bool *in_thrashing)
276 {}
delayacct_thrashing_end(bool * in_thrashing)277 static inline void delayacct_thrashing_end(bool *in_thrashing)
278 {}
delayacct_swapin_start(void)279 static inline void delayacct_swapin_start(void)
280 {}
delayacct_swapin_end(void)281 static inline void delayacct_swapin_end(void)
282 {}
delayacct_compact_start(void)283 static inline void delayacct_compact_start(void)
284 {}
delayacct_compact_end(void)285 static inline void delayacct_compact_end(void)
286 {}
delayacct_wpcopy_start(void)287 static inline void delayacct_wpcopy_start(void)
288 {}
delayacct_wpcopy_end(void)289 static inline void delayacct_wpcopy_end(void)
290 {}
delayacct_irq(struct task_struct * task,u32 delta)291 static inline void delayacct_irq(struct task_struct *task, u32 delta)
292 {}
293
294 #endif /* CONFIG_TASK_DELAY_ACCT */
295
296 #endif
297