1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * net/core/dst_cache.c - dst entry cache
4 *
5 * Copyright (c) 2016 Paolo Abeni <pabeni@redhat.com>
6 */
7
8 #include <linux/kernel.h>
9 #include <linux/percpu.h>
10 #include <net/dst_cache.h>
11 #include <net/route.h>
12 #if IS_ENABLED(CONFIG_IPV6)
13 #include <net/ip6_fib.h>
14 #endif
15 #include <uapi/linux/in.h>
16
17 struct dst_cache_pcpu {
18 unsigned long refresh_ts;
19 struct dst_entry *dst;
20 local_lock_t bh_lock;
21 u32 cookie;
22 union {
23 struct in_addr in_saddr;
24 struct in6_addr in6_saddr;
25 };
26 };
27
dst_cache_per_cpu_dst_set(struct dst_cache_pcpu * dst_cache,struct dst_entry * dst,u32 cookie)28 static void dst_cache_per_cpu_dst_set(struct dst_cache_pcpu *dst_cache,
29 struct dst_entry *dst, u32 cookie)
30 {
31 DEBUG_NET_WARN_ON_ONCE(!in_softirq());
32 dst_release(dst_cache->dst);
33 if (dst)
34 dst_hold(dst);
35
36 dst_cache->cookie = cookie;
37 dst_cache->dst = dst;
38 }
39
dst_cache_per_cpu_get(struct dst_cache * dst_cache,struct dst_cache_pcpu * idst)40 static struct dst_entry *dst_cache_per_cpu_get(struct dst_cache *dst_cache,
41 struct dst_cache_pcpu *idst)
42 {
43 struct dst_entry *dst;
44
45 DEBUG_NET_WARN_ON_ONCE(!in_softirq());
46 dst = idst->dst;
47 if (!dst)
48 goto fail;
49
50 /* the cache already hold a dst reference; it can't go away */
51 dst_hold(dst);
52
53 if (unlikely(!time_after(idst->refresh_ts,
54 READ_ONCE(dst_cache->reset_ts)) ||
55 (READ_ONCE(dst->obsolete) && !dst->ops->check(dst, idst->cookie)))) {
56 dst_cache_per_cpu_dst_set(idst, NULL, 0);
57 dst_release(dst);
58 goto fail;
59 }
60 return dst;
61
62 fail:
63 idst->refresh_ts = jiffies;
64 return NULL;
65 }
66
dst_cache_get(struct dst_cache * dst_cache)67 struct dst_entry *dst_cache_get(struct dst_cache *dst_cache)
68 {
69 struct dst_entry *dst;
70
71 if (!dst_cache->cache)
72 return NULL;
73
74 local_lock_nested_bh(&dst_cache->cache->bh_lock);
75 dst = dst_cache_per_cpu_get(dst_cache, this_cpu_ptr(dst_cache->cache));
76 local_unlock_nested_bh(&dst_cache->cache->bh_lock);
77 return dst;
78 }
79 EXPORT_SYMBOL_GPL(dst_cache_get);
80
dst_cache_get_ip4(struct dst_cache * dst_cache,__be32 * saddr)81 struct rtable *dst_cache_get_ip4(struct dst_cache *dst_cache, __be32 *saddr)
82 {
83 struct dst_cache_pcpu *idst;
84 struct dst_entry *dst;
85
86 if (!dst_cache->cache)
87 return NULL;
88
89 local_lock_nested_bh(&dst_cache->cache->bh_lock);
90 idst = this_cpu_ptr(dst_cache->cache);
91 dst = dst_cache_per_cpu_get(dst_cache, idst);
92 if (!dst) {
93 local_unlock_nested_bh(&dst_cache->cache->bh_lock);
94 return NULL;
95 }
96
97 *saddr = idst->in_saddr.s_addr;
98 local_unlock_nested_bh(&dst_cache->cache->bh_lock);
99 return dst_rtable(dst);
100 }
101 EXPORT_SYMBOL_GPL(dst_cache_get_ip4);
102
dst_cache_set_ip4(struct dst_cache * dst_cache,struct dst_entry * dst,__be32 saddr)103 void dst_cache_set_ip4(struct dst_cache *dst_cache, struct dst_entry *dst,
104 __be32 saddr)
105 {
106 struct dst_cache_pcpu *idst;
107
108 if (!dst_cache->cache)
109 return;
110
111 local_lock_nested_bh(&dst_cache->cache->bh_lock);
112 idst = this_cpu_ptr(dst_cache->cache);
113 dst_cache_per_cpu_dst_set(idst, dst, 0);
114 idst->in_saddr.s_addr = saddr;
115 local_unlock_nested_bh(&dst_cache->cache->bh_lock);
116 }
117 EXPORT_SYMBOL_GPL(dst_cache_set_ip4);
118
119 #if IS_ENABLED(CONFIG_IPV6)
dst_cache_set_ip6(struct dst_cache * dst_cache,struct dst_entry * dst,const struct in6_addr * saddr)120 void dst_cache_set_ip6(struct dst_cache *dst_cache, struct dst_entry *dst,
121 const struct in6_addr *saddr)
122 {
123 struct dst_cache_pcpu *idst;
124
125 if (!dst_cache->cache)
126 return;
127
128 local_lock_nested_bh(&dst_cache->cache->bh_lock);
129
130 idst = this_cpu_ptr(dst_cache->cache);
131 dst_cache_per_cpu_dst_set(idst, dst,
132 rt6_get_cookie(dst_rt6_info(dst)));
133 idst->in6_saddr = *saddr;
134 local_unlock_nested_bh(&dst_cache->cache->bh_lock);
135 }
136 EXPORT_SYMBOL_GPL(dst_cache_set_ip6);
137
dst_cache_get_ip6(struct dst_cache * dst_cache,struct in6_addr * saddr)138 struct dst_entry *dst_cache_get_ip6(struct dst_cache *dst_cache,
139 struct in6_addr *saddr)
140 {
141 struct dst_cache_pcpu *idst;
142 struct dst_entry *dst;
143
144 if (!dst_cache->cache)
145 return NULL;
146
147 local_lock_nested_bh(&dst_cache->cache->bh_lock);
148
149 idst = this_cpu_ptr(dst_cache->cache);
150 dst = dst_cache_per_cpu_get(dst_cache, idst);
151 if (!dst) {
152 local_unlock_nested_bh(&dst_cache->cache->bh_lock);
153 return NULL;
154 }
155
156 *saddr = idst->in6_saddr;
157 local_unlock_nested_bh(&dst_cache->cache->bh_lock);
158 return dst;
159 }
160 EXPORT_SYMBOL_GPL(dst_cache_get_ip6);
161 #endif
162
dst_cache_init(struct dst_cache * dst_cache,gfp_t gfp)163 int dst_cache_init(struct dst_cache *dst_cache, gfp_t gfp)
164 {
165 unsigned int i;
166
167 dst_cache->cache = alloc_percpu_gfp(struct dst_cache_pcpu,
168 gfp | __GFP_ZERO);
169 if (!dst_cache->cache)
170 return -ENOMEM;
171 for_each_possible_cpu(i)
172 local_lock_init(&per_cpu_ptr(dst_cache->cache, i)->bh_lock);
173
174 dst_cache_reset(dst_cache);
175 return 0;
176 }
177 EXPORT_SYMBOL_GPL(dst_cache_init);
178
dst_cache_destroy(struct dst_cache * dst_cache)179 void dst_cache_destroy(struct dst_cache *dst_cache)
180 {
181 int i;
182
183 if (!dst_cache->cache)
184 return;
185
186 for_each_possible_cpu(i)
187 dst_release(per_cpu_ptr(dst_cache->cache, i)->dst);
188
189 free_percpu(dst_cache->cache);
190 }
191 EXPORT_SYMBOL_GPL(dst_cache_destroy);
192
dst_cache_reset_now(struct dst_cache * dst_cache)193 void dst_cache_reset_now(struct dst_cache *dst_cache)
194 {
195 int i;
196
197 if (!dst_cache->cache)
198 return;
199
200 dst_cache_reset(dst_cache);
201 for_each_possible_cpu(i) {
202 struct dst_cache_pcpu *idst = per_cpu_ptr(dst_cache->cache, i);
203 struct dst_entry *dst = idst->dst;
204
205 idst->cookie = 0;
206 idst->dst = NULL;
207 dst_release(dst);
208 }
209 }
210 EXPORT_SYMBOL_GPL(dst_cache_reset_now);
211