1 // SPDX-License-Identifier: GPL-2.0
2
3 #include "bpf_tracing_net.h"
4 #include <bpf/bpf_helpers.h>
5 #include <bpf/bpf_tracing.h>
6
7 char _license[] SEC("license") = "GPL";
8
9 #define USEC_PER_SEC 1000000UL
10
tcp_left_out(const struct tcp_sock * tp)11 static unsigned int tcp_left_out(const struct tcp_sock *tp)
12 {
13 return tp->sacked_out + tp->lost_out;
14 }
15
tcp_packets_in_flight(const struct tcp_sock * tp)16 static unsigned int tcp_packets_in_flight(const struct tcp_sock *tp)
17 {
18 return tp->packets_out - tcp_left_out(tp) + tp->retrans_out;
19 }
20
21 SEC("struct_ops")
BPF_PROG(write_sk_pacing_init,struct sock * sk)22 void BPF_PROG(write_sk_pacing_init, struct sock *sk)
23 {
24 #ifdef ENABLE_ATOMICS_TESTS
25 __sync_bool_compare_and_swap(&sk->sk_pacing_status, SK_PACING_NONE,
26 SK_PACING_NEEDED);
27 #else
28 sk->sk_pacing_status = SK_PACING_NEEDED;
29 #endif
30 }
31
32 SEC("struct_ops")
BPF_PROG(write_sk_pacing_cong_control,struct sock * sk,const struct rate_sample * rs)33 void BPF_PROG(write_sk_pacing_cong_control, struct sock *sk,
34 const struct rate_sample *rs)
35 {
36 struct tcp_sock *tp = tcp_sk(sk);
37 unsigned long rate =
38 ((tp->snd_cwnd * tp->mss_cache * USEC_PER_SEC) << 3) /
39 (tp->srtt_us ?: 1U << 3);
40 sk->sk_pacing_rate = min(rate, sk->sk_max_pacing_rate);
41 tp->app_limited = (tp->delivered + tcp_packets_in_flight(tp)) ?: 1;
42 }
43
44 SEC("struct_ops")
BPF_PROG(write_sk_pacing_ssthresh,struct sock * sk)45 __u32 BPF_PROG(write_sk_pacing_ssthresh, struct sock *sk)
46 {
47 return tcp_sk(sk)->snd_ssthresh;
48 }
49
50 SEC("struct_ops")
BPF_PROG(write_sk_pacing_undo_cwnd,struct sock * sk)51 __u32 BPF_PROG(write_sk_pacing_undo_cwnd, struct sock *sk)
52 {
53 return tcp_sk(sk)->snd_cwnd;
54 }
55
56 SEC(".struct_ops")
57 struct tcp_congestion_ops write_sk_pacing = {
58 .init = (void *)write_sk_pacing_init,
59 .cong_control = (void *)write_sk_pacing_cong_control,
60 .ssthresh = (void *)write_sk_pacing_ssthresh,
61 .undo_cwnd = (void *)write_sk_pacing_undo_cwnd,
62 .name = "bpf_w_sk_pacing",
63 };
64