1 // SPDX-License-Identifier: GPL-2.0
2 /* Multipath TCP
3 *
4 * Copyright (c) 2022, SUSE.
5 */
6
7 #define pr_fmt(fmt) "MPTCP: " fmt
8
9 #include <linux/kernel.h>
10 #include <linux/module.h>
11 #include <linux/list.h>
12 #include <linux/rculist.h>
13 #include <linux/spinlock.h>
14 #include "protocol.h"
15
16 static DEFINE_SPINLOCK(mptcp_sched_list_lock);
17 static LIST_HEAD(mptcp_sched_list);
18
mptcp_sched_default_get_send(struct mptcp_sock * msk,struct mptcp_sched_data * data)19 static int mptcp_sched_default_get_send(struct mptcp_sock *msk,
20 struct mptcp_sched_data *data)
21 {
22 struct sock *ssk;
23
24 ssk = mptcp_subflow_get_send(msk);
25 if (!ssk)
26 return -EINVAL;
27
28 mptcp_subflow_set_scheduled(mptcp_subflow_ctx(ssk), true);
29 return 0;
30 }
31
mptcp_sched_default_get_retrans(struct mptcp_sock * msk,struct mptcp_sched_data * data)32 static int mptcp_sched_default_get_retrans(struct mptcp_sock *msk,
33 struct mptcp_sched_data *data)
34 {
35 struct sock *ssk;
36
37 ssk = mptcp_subflow_get_retrans(msk);
38 if (!ssk)
39 return -EINVAL;
40
41 mptcp_subflow_set_scheduled(mptcp_subflow_ctx(ssk), true);
42 return 0;
43 }
44
45 static struct mptcp_sched_ops mptcp_sched_default = {
46 .get_send = mptcp_sched_default_get_send,
47 .get_retrans = mptcp_sched_default_get_retrans,
48 .name = "default",
49 .owner = THIS_MODULE,
50 };
51
52 /* Must be called with rcu read lock held */
mptcp_sched_find(const char * name)53 struct mptcp_sched_ops *mptcp_sched_find(const char *name)
54 {
55 struct mptcp_sched_ops *sched, *ret = NULL;
56
57 list_for_each_entry_rcu(sched, &mptcp_sched_list, list) {
58 if (!strcmp(sched->name, name)) {
59 ret = sched;
60 break;
61 }
62 }
63
64 return ret;
65 }
66
67 /* Build string with list of available scheduler values.
68 * Similar to tcp_get_available_congestion_control()
69 */
mptcp_get_available_schedulers(char * buf,size_t maxlen)70 void mptcp_get_available_schedulers(char *buf, size_t maxlen)
71 {
72 struct mptcp_sched_ops *sched;
73 size_t offs = 0;
74
75 rcu_read_lock();
76 list_for_each_entry_rcu(sched, &mptcp_sched_list, list) {
77 offs += snprintf(buf + offs, maxlen - offs,
78 "%s%s",
79 offs == 0 ? "" : " ", sched->name);
80
81 if (WARN_ON_ONCE(offs >= maxlen))
82 break;
83 }
84 rcu_read_unlock();
85 }
86
mptcp_register_scheduler(struct mptcp_sched_ops * sched)87 int mptcp_register_scheduler(struct mptcp_sched_ops *sched)
88 {
89 if (!sched->get_send)
90 return -EINVAL;
91
92 spin_lock(&mptcp_sched_list_lock);
93 if (mptcp_sched_find(sched->name)) {
94 spin_unlock(&mptcp_sched_list_lock);
95 return -EEXIST;
96 }
97 list_add_tail_rcu(&sched->list, &mptcp_sched_list);
98 spin_unlock(&mptcp_sched_list_lock);
99
100 pr_debug("%s registered\n", sched->name);
101 return 0;
102 }
103
mptcp_unregister_scheduler(struct mptcp_sched_ops * sched)104 void mptcp_unregister_scheduler(struct mptcp_sched_ops *sched)
105 {
106 if (sched == &mptcp_sched_default)
107 return;
108
109 spin_lock(&mptcp_sched_list_lock);
110 list_del_rcu(&sched->list);
111 spin_unlock(&mptcp_sched_list_lock);
112 }
113
mptcp_sched_init(void)114 void mptcp_sched_init(void)
115 {
116 mptcp_register_scheduler(&mptcp_sched_default);
117 }
118
mptcp_init_sched(struct mptcp_sock * msk,struct mptcp_sched_ops * sched)119 int mptcp_init_sched(struct mptcp_sock *msk,
120 struct mptcp_sched_ops *sched)
121 {
122 if (!sched)
123 sched = &mptcp_sched_default;
124
125 if (!bpf_try_module_get(sched, sched->owner))
126 return -EBUSY;
127
128 msk->sched = sched;
129 if (msk->sched->init)
130 msk->sched->init(msk);
131
132 pr_debug("sched=%s\n", msk->sched->name);
133
134 return 0;
135 }
136
mptcp_release_sched(struct mptcp_sock * msk)137 void mptcp_release_sched(struct mptcp_sock *msk)
138 {
139 struct mptcp_sched_ops *sched = msk->sched;
140
141 if (!sched)
142 return;
143
144 msk->sched = NULL;
145 if (sched->release)
146 sched->release(msk);
147
148 bpf_module_put(sched, sched->owner);
149 }
150
mptcp_subflow_set_scheduled(struct mptcp_subflow_context * subflow,bool scheduled)151 void mptcp_subflow_set_scheduled(struct mptcp_subflow_context *subflow,
152 bool scheduled)
153 {
154 WRITE_ONCE(subflow->scheduled, scheduled);
155 }
156
mptcp_sched_get_send(struct mptcp_sock * msk)157 int mptcp_sched_get_send(struct mptcp_sock *msk)
158 {
159 struct mptcp_subflow_context *subflow;
160 struct mptcp_sched_data *data = NULL;
161
162 msk_owned_by_me(msk);
163
164 /* the following check is moved out of mptcp_subflow_get_send */
165 if (__mptcp_check_fallback(msk)) {
166 if (msk->first &&
167 __tcp_can_send(msk->first) &&
168 sk_stream_memory_free(msk->first)) {
169 mptcp_subflow_set_scheduled(mptcp_subflow_ctx(msk->first), true);
170 return 0;
171 }
172 return -EINVAL;
173 }
174
175 mptcp_for_each_subflow(msk, subflow) {
176 if (READ_ONCE(subflow->scheduled))
177 return 0;
178 }
179
180 if (msk->sched == &mptcp_sched_default || !msk->sched)
181 return mptcp_sched_default_get_send(msk, data);
182 return msk->sched->get_send(msk, data);
183 }
184
mptcp_sched_get_retrans(struct mptcp_sock * msk)185 int mptcp_sched_get_retrans(struct mptcp_sock *msk)
186 {
187 struct mptcp_subflow_context *subflow;
188 struct mptcp_sched_data *data = NULL;
189
190 msk_owned_by_me(msk);
191
192 /* the following check is moved out of mptcp_subflow_get_retrans */
193 if (__mptcp_check_fallback(msk))
194 return -EINVAL;
195
196 mptcp_for_each_subflow(msk, subflow) {
197 if (READ_ONCE(subflow->scheduled))
198 return 0;
199 }
200
201 if (msk->sched == &mptcp_sched_default || !msk->sched)
202 return mptcp_sched_default_get_retrans(msk, data);
203 if (msk->sched->get_retrans)
204 return msk->sched->get_retrans(msk, data);
205 return msk->sched->get_send(msk, data);
206 }
207