1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/bpf.h>
3 #include <bpf/bpf_tracing.h>
4 #include <stdbool.h>
5 #include "timerlat_bpf.h"
6
7 #define nosubprog __always_inline
8 #define MAX_ENTRIES_DEFAULT 4096
9
10 char LICENSE[] SEC("license") = "GPL";
11
12 struct trace_event_raw_timerlat_sample {
13 unsigned long long timer_latency;
14 int context;
15 } __attribute__((preserve_access_index));
16
17 struct {
18 __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
19 __uint(max_entries, MAX_ENTRIES_DEFAULT);
20 __type(key, unsigned int);
21 __type(value, unsigned long long);
22 } hist_irq SEC(".maps"), hist_thread SEC(".maps"), hist_user SEC(".maps");
23
24 struct {
25 __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
26 __uint(max_entries, SUMMARY_FIELD_N);
27 __type(key, unsigned int);
28 __type(value, unsigned long long);
29 } summary_irq SEC(".maps"), summary_thread SEC(".maps"), summary_user SEC(".maps");
30
31 struct {
32 __uint(type, BPF_MAP_TYPE_ARRAY);
33 __uint(max_entries, 1);
34 __type(key, unsigned int);
35 __type(value, unsigned long long);
36 } stop_tracing SEC(".maps");
37
38 struct {
39 __uint(type, BPF_MAP_TYPE_RINGBUF);
40 __uint(max_entries, 1);
41 } signal_stop_tracing SEC(".maps");
42
43 struct {
44 __uint(type, BPF_MAP_TYPE_PROG_ARRAY);
45 __uint(key_size, sizeof(unsigned int));
46 __uint(max_entries, 1);
47 __array(values, unsigned int (void *));
48 } bpf_action SEC(".maps") = {
49 .values = {
50 [0] = 0
51 },
52 };
53
54 /* Params to be set by rtla */
55 const volatile int bucket_size = 1;
56 const volatile int output_divisor = 1000;
57 const volatile int entries = 256;
58 const volatile int irq_threshold;
59 const volatile int thread_threshold;
60 const volatile bool aa_only;
61
map_get(void * map,unsigned int key)62 nosubprog unsigned long long map_get(void *map,
63 unsigned int key)
64 {
65 unsigned long long *value_ptr;
66
67 value_ptr = bpf_map_lookup_elem(map, &key);
68
69 return !value_ptr ? 0 : *value_ptr;
70 }
71
map_set(void * map,unsigned int key,unsigned long long value)72 nosubprog void map_set(void *map,
73 unsigned int key,
74 unsigned long long value)
75 {
76 bpf_map_update_elem(map, &key, &value, BPF_ANY);
77 }
78
map_increment(void * map,unsigned int key)79 nosubprog void map_increment(void *map,
80 unsigned int key)
81 {
82 map_set(map, key, map_get(map, key) + 1);
83 }
84
update_main_hist(void * map,int bucket)85 nosubprog void update_main_hist(void *map,
86 int bucket)
87 {
88 if (entries == 0)
89 /* No histogram */
90 return;
91
92 if (bucket >= entries)
93 /* Overflow */
94 return;
95
96 map_increment(map, bucket);
97 }
98
update_summary(void * map,unsigned long long latency,int bucket)99 nosubprog void update_summary(void *map,
100 unsigned long long latency,
101 int bucket)
102 {
103 if (aa_only)
104 /* Auto-analysis only, nothing to be done here */
105 return;
106
107 map_set(map, SUMMARY_CURRENT, latency);
108
109 if (bucket >= entries)
110 /* Overflow */
111 map_increment(map, SUMMARY_OVERFLOW);
112
113 if (latency > map_get(map, SUMMARY_MAX))
114 map_set(map, SUMMARY_MAX, latency);
115
116 if (latency < map_get(map, SUMMARY_MIN) || map_get(map, SUMMARY_COUNT) == 0)
117 map_set(map, SUMMARY_MIN, latency);
118
119 map_increment(map, SUMMARY_COUNT);
120 map_set(map, SUMMARY_SUM, map_get(map, SUMMARY_SUM) + latency);
121 }
122
set_stop_tracing(struct trace_event_raw_timerlat_sample * tp_args)123 nosubprog void set_stop_tracing(struct trace_event_raw_timerlat_sample *tp_args)
124 {
125 int value = 0;
126
127 /* Suppress further sample processing */
128 map_set(&stop_tracing, 0, 1);
129
130 /* Signal to userspace */
131 bpf_ringbuf_output(&signal_stop_tracing, &value, sizeof(value), 0);
132
133 /*
134 * Call into BPF action program, if attached.
135 * Otherwise, just silently fail.
136 */
137 bpf_tail_call(tp_args, &bpf_action, 0);
138 }
139
140 SEC("tp/osnoise/timerlat_sample")
handle_timerlat_sample(struct trace_event_raw_timerlat_sample * tp_args)141 int handle_timerlat_sample(struct trace_event_raw_timerlat_sample *tp_args)
142 {
143 unsigned long long latency, latency_us;
144 int bucket;
145
146 if (map_get(&stop_tracing, 0))
147 return 0;
148
149 latency = tp_args->timer_latency / output_divisor;
150 latency_us = tp_args->timer_latency / 1000;
151 bucket = latency / bucket_size;
152
153 if (tp_args->context == 0) {
154 update_main_hist(&hist_irq, bucket);
155 update_summary(&summary_irq, latency, bucket);
156
157 if (irq_threshold != 0 && latency_us >= irq_threshold)
158 set_stop_tracing(tp_args);
159 } else if (tp_args->context == 1) {
160 update_main_hist(&hist_thread, bucket);
161 update_summary(&summary_thread, latency, bucket);
162
163 if (thread_threshold != 0 && latency_us >= thread_threshold)
164 set_stop_tracing(tp_args);
165 } else {
166 update_main_hist(&hist_user, bucket);
167 update_summary(&summary_user, latency, bucket);
168
169 if (thread_threshold != 0 && latency_us >= thread_threshold)
170 set_stop_tracing(tp_args);
171 }
172
173 return 0;
174 }
175