1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
2 // Copyright (c) 2022 Google
3 #include "vmlinux.h"
4 #include <bpf/bpf_helpers.h>
5 #include <bpf/bpf_tracing.h>
6 #include <bpf/bpf_core_read.h>
7 #include <asm-generic/errno-base.h>
8
9 #include "lock_data.h"
10
11 /* for collect_lock_syms(). 4096 was rejected by the verifier */
12 #define MAX_CPUS 1024
13
14 /* lock contention flags from include/trace/events/lock.h */
15 #define LCB_F_SPIN (1U << 0)
16 #define LCB_F_READ (1U << 1)
17 #define LCB_F_WRITE (1U << 2)
18 #define LCB_F_RT (1U << 3)
19 #define LCB_F_PERCPU (1U << 4)
20 #define LCB_F_MUTEX (1U << 5)
21
22 struct tstamp_data {
23 __u64 timestamp;
24 __u64 lock;
25 __u32 flags;
26 __s32 stack_id;
27 };
28
29 /* callstack storage */
30 struct {
31 __uint(type, BPF_MAP_TYPE_STACK_TRACE);
32 __uint(key_size, sizeof(__u32));
33 __uint(value_size, sizeof(__u64));
34 __uint(max_entries, MAX_ENTRIES);
35 } stacks SEC(".maps");
36
37 /* maintain timestamp at the beginning of contention */
38 struct {
39 __uint(type, BPF_MAP_TYPE_HASH);
40 __type(key, int);
41 __type(value, struct tstamp_data);
42 __uint(max_entries, MAX_ENTRIES);
43 } tstamp SEC(".maps");
44
45 /* maintain per-CPU timestamp at the beginning of contention */
46 struct {
47 __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
48 __uint(key_size, sizeof(__u32));
49 __uint(value_size, sizeof(struct tstamp_data));
50 __uint(max_entries, 1);
51 } tstamp_cpu SEC(".maps");
52
53 /* actual lock contention statistics */
54 struct {
55 __uint(type, BPF_MAP_TYPE_HASH);
56 __uint(key_size, sizeof(struct contention_key));
57 __uint(value_size, sizeof(struct contention_data));
58 __uint(max_entries, MAX_ENTRIES);
59 } lock_stat SEC(".maps");
60
61 struct {
62 __uint(type, BPF_MAP_TYPE_HASH);
63 __uint(key_size, sizeof(__u32));
64 __uint(value_size, sizeof(struct contention_task_data));
65 __uint(max_entries, MAX_ENTRIES);
66 } task_data SEC(".maps");
67
68 struct {
69 __uint(type, BPF_MAP_TYPE_HASH);
70 __uint(key_size, sizeof(__u64));
71 __uint(value_size, sizeof(__u32));
72 __uint(max_entries, MAX_ENTRIES);
73 } lock_syms SEC(".maps");
74
75 struct {
76 __uint(type, BPF_MAP_TYPE_HASH);
77 __uint(key_size, sizeof(__u32));
78 __uint(value_size, sizeof(__u8));
79 __uint(max_entries, 1);
80 } cpu_filter SEC(".maps");
81
82 struct {
83 __uint(type, BPF_MAP_TYPE_HASH);
84 __uint(key_size, sizeof(__u32));
85 __uint(value_size, sizeof(__u8));
86 __uint(max_entries, 1);
87 } task_filter SEC(".maps");
88
89 struct {
90 __uint(type, BPF_MAP_TYPE_HASH);
91 __uint(key_size, sizeof(__u32));
92 __uint(value_size, sizeof(__u8));
93 __uint(max_entries, 1);
94 } type_filter SEC(".maps");
95
96 struct {
97 __uint(type, BPF_MAP_TYPE_HASH);
98 __uint(key_size, sizeof(__u64));
99 __uint(value_size, sizeof(__u8));
100 __uint(max_entries, 1);
101 } addr_filter SEC(".maps");
102
103 struct {
104 __uint(type, BPF_MAP_TYPE_HASH);
105 __uint(key_size, sizeof(__u64));
106 __uint(value_size, sizeof(__u8));
107 __uint(max_entries, 1);
108 } cgroup_filter SEC(".maps");
109
110 struct rw_semaphore___old {
111 struct task_struct *owner;
112 } __attribute__((preserve_access_index));
113
114 struct rw_semaphore___new {
115 atomic_long_t owner;
116 } __attribute__((preserve_access_index));
117
118 struct mm_struct___old {
119 struct rw_semaphore mmap_sem;
120 } __attribute__((preserve_access_index));
121
122 struct mm_struct___new {
123 struct rw_semaphore mmap_lock;
124 } __attribute__((preserve_access_index));
125
126 /* control flags */
127 int enabled;
128 int has_cpu;
129 int has_task;
130 int has_type;
131 int has_addr;
132 int has_cgroup;
133 int needs_callstack;
134 int stack_skip;
135 int lock_owner;
136
137 int use_cgroup_v2;
138 int perf_subsys_id = -1;
139
140 /* determine the key of lock stat */
141 int aggr_mode;
142
143 /* error stat */
144 int task_fail;
145 int stack_fail;
146 int time_fail;
147 int data_fail;
148
149 int task_map_full;
150 int data_map_full;
151
get_current_cgroup_id(void)152 static inline __u64 get_current_cgroup_id(void)
153 {
154 struct task_struct *task;
155 struct cgroup *cgrp;
156
157 if (use_cgroup_v2)
158 return bpf_get_current_cgroup_id();
159
160 task = bpf_get_current_task_btf();
161
162 if (perf_subsys_id == -1) {
163 #if __has_builtin(__builtin_preserve_enum_value)
164 perf_subsys_id = bpf_core_enum_value(enum cgroup_subsys_id,
165 perf_event_cgrp_id);
166 #else
167 perf_subsys_id = perf_event_cgrp_id;
168 #endif
169 }
170
171 cgrp = BPF_CORE_READ(task, cgroups, subsys[perf_subsys_id], cgroup);
172 return BPF_CORE_READ(cgrp, kn, id);
173 }
174
can_record(u64 * ctx)175 static inline int can_record(u64 *ctx)
176 {
177 if (has_cpu) {
178 __u32 cpu = bpf_get_smp_processor_id();
179 __u8 *ok;
180
181 ok = bpf_map_lookup_elem(&cpu_filter, &cpu);
182 if (!ok)
183 return 0;
184 }
185
186 if (has_task) {
187 __u8 *ok;
188 __u32 pid = bpf_get_current_pid_tgid();
189
190 ok = bpf_map_lookup_elem(&task_filter, &pid);
191 if (!ok)
192 return 0;
193 }
194
195 if (has_type) {
196 __u8 *ok;
197 __u32 flags = (__u32)ctx[1];
198
199 ok = bpf_map_lookup_elem(&type_filter, &flags);
200 if (!ok)
201 return 0;
202 }
203
204 if (has_addr) {
205 __u8 *ok;
206 __u64 addr = ctx[0];
207
208 ok = bpf_map_lookup_elem(&addr_filter, &addr);
209 if (!ok)
210 return 0;
211 }
212
213 if (has_cgroup) {
214 __u8 *ok;
215 __u64 cgrp = get_current_cgroup_id();
216
217 ok = bpf_map_lookup_elem(&cgroup_filter, &cgrp);
218 if (!ok)
219 return 0;
220 }
221
222 return 1;
223 }
224
update_task_data(struct task_struct * task)225 static inline int update_task_data(struct task_struct *task)
226 {
227 struct contention_task_data *p;
228 int pid, err;
229
230 err = bpf_core_read(&pid, sizeof(pid), &task->pid);
231 if (err)
232 return -1;
233
234 p = bpf_map_lookup_elem(&task_data, &pid);
235 if (p == NULL && !task_map_full) {
236 struct contention_task_data data = {};
237
238 BPF_CORE_READ_STR_INTO(&data.comm, task, comm);
239 if (bpf_map_update_elem(&task_data, &pid, &data, BPF_NOEXIST) == -E2BIG)
240 task_map_full = 1;
241 }
242
243 return 0;
244 }
245
246 #ifndef __has_builtin
247 # define __has_builtin(x) 0
248 #endif
249
get_lock_owner(__u64 lock,__u32 flags)250 static inline struct task_struct *get_lock_owner(__u64 lock, __u32 flags)
251 {
252 struct task_struct *task;
253 __u64 owner = 0;
254
255 if (flags & LCB_F_MUTEX) {
256 struct mutex *mutex = (void *)lock;
257 owner = BPF_CORE_READ(mutex, owner.counter);
258 } else if (flags == LCB_F_READ || flags == LCB_F_WRITE) {
259 /*
260 * Support for the BPF_TYPE_MATCHES argument to the
261 * __builtin_preserve_type_info builtin was added at some point during
262 * development of clang 15 and it's what is needed for
263 * bpf_core_type_matches.
264 */
265 #if __has_builtin(__builtin_preserve_type_info) && __clang_major__ >= 15
266 if (bpf_core_type_matches(struct rw_semaphore___old)) {
267 struct rw_semaphore___old *rwsem = (void *)lock;
268 owner = (unsigned long)BPF_CORE_READ(rwsem, owner);
269 } else if (bpf_core_type_matches(struct rw_semaphore___new)) {
270 struct rw_semaphore___new *rwsem = (void *)lock;
271 owner = BPF_CORE_READ(rwsem, owner.counter);
272 }
273 #else
274 /* assume new struct */
275 struct rw_semaphore *rwsem = (void *)lock;
276 owner = BPF_CORE_READ(rwsem, owner.counter);
277 #endif
278 }
279
280 if (!owner)
281 return NULL;
282
283 task = (void *)(owner & ~7UL);
284 return task;
285 }
286
check_lock_type(__u64 lock,__u32 flags)287 static inline __u32 check_lock_type(__u64 lock, __u32 flags)
288 {
289 struct task_struct *curr;
290 struct mm_struct___old *mm_old;
291 struct mm_struct___new *mm_new;
292
293 switch (flags) {
294 case LCB_F_READ: /* rwsem */
295 case LCB_F_WRITE:
296 curr = bpf_get_current_task_btf();
297 if (curr->mm == NULL)
298 break;
299 mm_new = (void *)curr->mm;
300 if (bpf_core_field_exists(mm_new->mmap_lock)) {
301 if (&mm_new->mmap_lock == (void *)lock)
302 return LCD_F_MMAP_LOCK;
303 break;
304 }
305 mm_old = (void *)curr->mm;
306 if (bpf_core_field_exists(mm_old->mmap_sem)) {
307 if (&mm_old->mmap_sem == (void *)lock)
308 return LCD_F_MMAP_LOCK;
309 }
310 break;
311 case LCB_F_SPIN: /* spinlock */
312 curr = bpf_get_current_task_btf();
313 if (&curr->sighand->siglock == (void *)lock)
314 return LCD_F_SIGHAND_LOCK;
315 break;
316 default:
317 break;
318 }
319 return 0;
320 }
321
get_tstamp_elem(__u32 flags)322 static inline struct tstamp_data *get_tstamp_elem(__u32 flags)
323 {
324 __u32 pid;
325 struct tstamp_data *pelem;
326
327 /* Use per-cpu array map for spinlock and rwlock */
328 if (flags == (LCB_F_SPIN | LCB_F_READ) || flags == LCB_F_SPIN ||
329 flags == (LCB_F_SPIN | LCB_F_WRITE)) {
330 __u32 idx = 0;
331
332 pelem = bpf_map_lookup_elem(&tstamp_cpu, &idx);
333 /* Do not update the element for nested locks */
334 if (pelem && pelem->lock)
335 pelem = NULL;
336 return pelem;
337 }
338
339 pid = bpf_get_current_pid_tgid();
340 pelem = bpf_map_lookup_elem(&tstamp, &pid);
341 /* Do not update the element for nested locks */
342 if (pelem && pelem->lock)
343 return NULL;
344
345 if (pelem == NULL) {
346 struct tstamp_data zero = {};
347
348 if (bpf_map_update_elem(&tstamp, &pid, &zero, BPF_NOEXIST) < 0) {
349 __sync_fetch_and_add(&task_fail, 1);
350 return NULL;
351 }
352
353 pelem = bpf_map_lookup_elem(&tstamp, &pid);
354 if (pelem == NULL) {
355 __sync_fetch_and_add(&task_fail, 1);
356 return NULL;
357 }
358 }
359 return pelem;
360 }
361
362 SEC("tp_btf/contention_begin")
contention_begin(u64 * ctx)363 int contention_begin(u64 *ctx)
364 {
365 struct tstamp_data *pelem;
366
367 if (!enabled || !can_record(ctx))
368 return 0;
369
370 pelem = get_tstamp_elem(ctx[1]);
371 if (pelem == NULL)
372 return 0;
373
374 pelem->timestamp = bpf_ktime_get_ns();
375 pelem->lock = (__u64)ctx[0];
376 pelem->flags = (__u32)ctx[1];
377
378 if (needs_callstack) {
379 pelem->stack_id = bpf_get_stackid(ctx, &stacks,
380 BPF_F_FAST_STACK_CMP | stack_skip);
381 if (pelem->stack_id < 0)
382 __sync_fetch_and_add(&stack_fail, 1);
383 } else if (aggr_mode == LOCK_AGGR_TASK) {
384 struct task_struct *task;
385
386 if (lock_owner) {
387 task = get_lock_owner(pelem->lock, pelem->flags);
388
389 /* The flags is not used anymore. Pass the owner pid. */
390 if (task)
391 pelem->flags = BPF_CORE_READ(task, pid);
392 else
393 pelem->flags = -1U;
394
395 } else {
396 task = bpf_get_current_task_btf();
397 }
398
399 if (task) {
400 if (update_task_data(task) < 0 && lock_owner)
401 pelem->flags = -1U;
402 }
403 }
404
405 return 0;
406 }
407
408 SEC("tp_btf/contention_end")
contention_end(u64 * ctx)409 int contention_end(u64 *ctx)
410 {
411 __u32 pid = 0, idx = 0;
412 struct tstamp_data *pelem;
413 struct contention_key key = {};
414 struct contention_data *data;
415 __u64 duration;
416 bool need_delete = false;
417
418 if (!enabled)
419 return 0;
420
421 /*
422 * For spinlock and rwlock, it needs to get the timestamp for the
423 * per-cpu map. However, contention_end does not have the flags
424 * so it cannot know whether it reads percpu or hash map.
425 *
426 * Try per-cpu map first and check if there's active contention.
427 * If it is, do not read hash map because it cannot go to sleeping
428 * locks before releasing the spinning locks.
429 */
430 pelem = bpf_map_lookup_elem(&tstamp_cpu, &idx);
431 if (pelem && pelem->lock) {
432 if (pelem->lock != ctx[0])
433 return 0;
434 } else {
435 pid = bpf_get_current_pid_tgid();
436 pelem = bpf_map_lookup_elem(&tstamp, &pid);
437 if (!pelem || pelem->lock != ctx[0])
438 return 0;
439 need_delete = true;
440 }
441
442 duration = bpf_ktime_get_ns() - pelem->timestamp;
443 if ((__s64)duration < 0) {
444 pelem->lock = 0;
445 if (need_delete)
446 bpf_map_delete_elem(&tstamp, &pid);
447 __sync_fetch_and_add(&time_fail, 1);
448 return 0;
449 }
450
451 switch (aggr_mode) {
452 case LOCK_AGGR_CALLER:
453 key.stack_id = pelem->stack_id;
454 break;
455 case LOCK_AGGR_TASK:
456 if (lock_owner)
457 key.pid = pelem->flags;
458 else {
459 if (!need_delete)
460 pid = bpf_get_current_pid_tgid();
461 key.pid = pid;
462 }
463 if (needs_callstack)
464 key.stack_id = pelem->stack_id;
465 break;
466 case LOCK_AGGR_ADDR:
467 key.lock_addr_or_cgroup = pelem->lock;
468 if (needs_callstack)
469 key.stack_id = pelem->stack_id;
470 break;
471 case LOCK_AGGR_CGROUP:
472 key.lock_addr_or_cgroup = get_current_cgroup_id();
473 break;
474 default:
475 /* should not happen */
476 return 0;
477 }
478
479 data = bpf_map_lookup_elem(&lock_stat, &key);
480 if (!data) {
481 if (data_map_full) {
482 pelem->lock = 0;
483 if (need_delete)
484 bpf_map_delete_elem(&tstamp, &pid);
485 __sync_fetch_and_add(&data_fail, 1);
486 return 0;
487 }
488
489 struct contention_data first = {
490 .total_time = duration,
491 .max_time = duration,
492 .min_time = duration,
493 .count = 1,
494 .flags = pelem->flags,
495 };
496 int err;
497
498 if (aggr_mode == LOCK_AGGR_ADDR)
499 first.flags |= check_lock_type(pelem->lock, pelem->flags);
500
501 err = bpf_map_update_elem(&lock_stat, &key, &first, BPF_NOEXIST);
502 if (err < 0) {
503 if (err == -E2BIG)
504 data_map_full = 1;
505 __sync_fetch_and_add(&data_fail, 1);
506 }
507 pelem->lock = 0;
508 if (need_delete)
509 bpf_map_delete_elem(&tstamp, &pid);
510 return 0;
511 }
512
513 __sync_fetch_and_add(&data->total_time, duration);
514 __sync_fetch_and_add(&data->count, 1);
515
516 /* FIXME: need atomic operations */
517 if (data->max_time < duration)
518 data->max_time = duration;
519 if (data->min_time > duration)
520 data->min_time = duration;
521
522 pelem->lock = 0;
523 if (need_delete)
524 bpf_map_delete_elem(&tstamp, &pid);
525 return 0;
526 }
527
528 extern struct rq runqueues __ksym;
529
530 struct rq___old {
531 raw_spinlock_t lock;
532 } __attribute__((preserve_access_index));
533
534 struct rq___new {
535 raw_spinlock_t __lock;
536 } __attribute__((preserve_access_index));
537
538 SEC("raw_tp/bpf_test_finish")
BPF_PROG(collect_lock_syms)539 int BPF_PROG(collect_lock_syms)
540 {
541 __u64 lock_addr, lock_off;
542 __u32 lock_flag;
543
544 if (bpf_core_field_exists(struct rq___new, __lock))
545 lock_off = offsetof(struct rq___new, __lock);
546 else
547 lock_off = offsetof(struct rq___old, lock);
548
549 for (int i = 0; i < MAX_CPUS; i++) {
550 struct rq *rq = bpf_per_cpu_ptr(&runqueues, i);
551
552 if (rq == NULL)
553 break;
554
555 lock_addr = (__u64)(void *)rq + lock_off;
556 lock_flag = LOCK_CLASS_RQLOCK;
557 bpf_map_update_elem(&lock_syms, &lock_addr, &lock_flag, BPF_ANY);
558 }
559 return 0;
560 }
561
562 char LICENSE[] SEC("license") = "Dual BSD/GPL";
563