1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * net/core/dst_cache.c - dst entry cache 4 * 5 * Copyright (c) 2016 Paolo Abeni <pabeni@redhat.com> 6 */ 7 8 #include <linux/kernel.h> 9 #include <linux/percpu.h> 10 #include <net/dst_cache.h> 11 #include <net/route.h> 12 #if IS_ENABLED(CONFIG_IPV6) 13 #include <net/ip6_fib.h> 14 #endif 15 #include <uapi/linux/in.h> 16 17 struct dst_cache_pcpu { 18 unsigned long refresh_ts; 19 struct dst_entry *dst; 20 local_lock_t bh_lock; 21 u32 cookie; 22 union { 23 struct in_addr in_saddr; 24 struct in6_addr in6_saddr; 25 }; 26 }; 27 28 static void dst_cache_per_cpu_dst_set(struct dst_cache_pcpu *dst_cache, 29 struct dst_entry *dst, u32 cookie) 30 { 31 DEBUG_NET_WARN_ON_ONCE(!in_softirq()); 32 dst_release(dst_cache->dst); 33 if (dst) 34 dst_hold(dst); 35 36 dst_cache->cookie = cookie; 37 dst_cache->dst = dst; 38 } 39 40 static struct dst_entry *dst_cache_per_cpu_get(struct dst_cache *dst_cache, 41 struct dst_cache_pcpu *idst) 42 { 43 struct dst_entry *dst; 44 45 DEBUG_NET_WARN_ON_ONCE(!in_softirq()); 46 dst = idst->dst; 47 if (!dst) 48 goto fail; 49 50 /* the cache already hold a dst reference; it can't go away */ 51 dst_hold(dst); 52 53 if (unlikely(!time_after(idst->refresh_ts, 54 READ_ONCE(dst_cache->reset_ts)) || 55 (dst->obsolete && !dst->ops->check(dst, idst->cookie)))) { 56 dst_cache_per_cpu_dst_set(idst, NULL, 0); 57 dst_release(dst); 58 goto fail; 59 } 60 return dst; 61 62 fail: 63 idst->refresh_ts = jiffies; 64 return NULL; 65 } 66 67 struct dst_entry *dst_cache_get(struct dst_cache *dst_cache) 68 { 69 struct dst_entry *dst; 70 71 if (!dst_cache->cache) 72 return NULL; 73 74 local_lock_nested_bh(&dst_cache->cache->bh_lock); 75 dst = dst_cache_per_cpu_get(dst_cache, this_cpu_ptr(dst_cache->cache)); 76 local_unlock_nested_bh(&dst_cache->cache->bh_lock); 77 return dst; 78 } 79 EXPORT_SYMBOL_GPL(dst_cache_get); 80 81 struct rtable *dst_cache_get_ip4(struct dst_cache *dst_cache, __be32 *saddr) 82 { 83 struct dst_cache_pcpu *idst; 84 struct dst_entry *dst; 85 86 if (!dst_cache->cache) 87 return NULL; 88 89 local_lock_nested_bh(&dst_cache->cache->bh_lock); 90 idst = this_cpu_ptr(dst_cache->cache); 91 dst = dst_cache_per_cpu_get(dst_cache, idst); 92 if (!dst) { 93 local_unlock_nested_bh(&dst_cache->cache->bh_lock); 94 return NULL; 95 } 96 97 *saddr = idst->in_saddr.s_addr; 98 local_unlock_nested_bh(&dst_cache->cache->bh_lock); 99 return dst_rtable(dst); 100 } 101 EXPORT_SYMBOL_GPL(dst_cache_get_ip4); 102 103 void dst_cache_set_ip4(struct dst_cache *dst_cache, struct dst_entry *dst, 104 __be32 saddr) 105 { 106 struct dst_cache_pcpu *idst; 107 108 if (!dst_cache->cache) 109 return; 110 111 local_lock_nested_bh(&dst_cache->cache->bh_lock); 112 idst = this_cpu_ptr(dst_cache->cache); 113 dst_cache_per_cpu_dst_set(idst, dst, 0); 114 idst->in_saddr.s_addr = saddr; 115 local_unlock_nested_bh(&dst_cache->cache->bh_lock); 116 } 117 EXPORT_SYMBOL_GPL(dst_cache_set_ip4); 118 119 #if IS_ENABLED(CONFIG_IPV6) 120 void dst_cache_set_ip6(struct dst_cache *dst_cache, struct dst_entry *dst, 121 const struct in6_addr *saddr) 122 { 123 struct dst_cache_pcpu *idst; 124 125 if (!dst_cache->cache) 126 return; 127 128 local_lock_nested_bh(&dst_cache->cache->bh_lock); 129 130 idst = this_cpu_ptr(dst_cache->cache); 131 dst_cache_per_cpu_dst_set(idst, dst, 132 rt6_get_cookie(dst_rt6_info(dst))); 133 idst->in6_saddr = *saddr; 134 local_unlock_nested_bh(&dst_cache->cache->bh_lock); 135 } 136 EXPORT_SYMBOL_GPL(dst_cache_set_ip6); 137 138 struct dst_entry *dst_cache_get_ip6(struct dst_cache *dst_cache, 139 struct in6_addr *saddr) 140 { 141 struct dst_cache_pcpu *idst; 142 struct dst_entry *dst; 143 144 if (!dst_cache->cache) 145 return NULL; 146 147 local_lock_nested_bh(&dst_cache->cache->bh_lock); 148 149 idst = this_cpu_ptr(dst_cache->cache); 150 dst = dst_cache_per_cpu_get(dst_cache, idst); 151 if (!dst) { 152 local_unlock_nested_bh(&dst_cache->cache->bh_lock); 153 return NULL; 154 } 155 156 *saddr = idst->in6_saddr; 157 local_unlock_nested_bh(&dst_cache->cache->bh_lock); 158 return dst; 159 } 160 EXPORT_SYMBOL_GPL(dst_cache_get_ip6); 161 #endif 162 163 int dst_cache_init(struct dst_cache *dst_cache, gfp_t gfp) 164 { 165 unsigned int i; 166 167 dst_cache->cache = alloc_percpu_gfp(struct dst_cache_pcpu, 168 gfp | __GFP_ZERO); 169 if (!dst_cache->cache) 170 return -ENOMEM; 171 for_each_possible_cpu(i) 172 local_lock_init(&per_cpu_ptr(dst_cache->cache, i)->bh_lock); 173 174 dst_cache_reset(dst_cache); 175 return 0; 176 } 177 EXPORT_SYMBOL_GPL(dst_cache_init); 178 179 void dst_cache_destroy(struct dst_cache *dst_cache) 180 { 181 int i; 182 183 if (!dst_cache->cache) 184 return; 185 186 for_each_possible_cpu(i) 187 dst_release(per_cpu_ptr(dst_cache->cache, i)->dst); 188 189 free_percpu(dst_cache->cache); 190 } 191 EXPORT_SYMBOL_GPL(dst_cache_destroy); 192 193 void dst_cache_reset_now(struct dst_cache *dst_cache) 194 { 195 int i; 196 197 if (!dst_cache->cache) 198 return; 199 200 dst_cache_reset(dst_cache); 201 for_each_possible_cpu(i) { 202 struct dst_cache_pcpu *idst = per_cpu_ptr(dst_cache->cache, i); 203 struct dst_entry *dst = idst->dst; 204 205 idst->cookie = 0; 206 idst->dst = NULL; 207 dst_release(dst); 208 } 209 } 210 EXPORT_SYMBOL_GPL(dst_cache_reset_now); 211