1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2008-2009 Patrick McHardy <kaber@trash.net> 4 * 5 * Development of this code funded by Astaro AG (http://www.astaro.com/) 6 */ 7 8 #include <linux/kernel.h> 9 #include <linux/init.h> 10 #include <linux/module.h> 11 #include <linux/u64_stats_sync.h> 12 #include <linux/netlink.h> 13 #include <linux/netfilter.h> 14 #include <linux/netfilter/nf_tables.h> 15 #include <net/netfilter/nf_tables.h> 16 #include <net/netfilter/nf_tables_core.h> 17 #include <net/netfilter/nf_tables_offload.h> 18 19 struct nft_counter { 20 u64_stats_t bytes; 21 u64_stats_t packets; 22 }; 23 24 struct nft_counter_tot { 25 s64 bytes; 26 s64 packets; 27 }; 28 29 struct nft_counter_percpu_priv { 30 struct nft_counter __percpu *counter; 31 }; 32 33 static DEFINE_PER_CPU(struct u64_stats_sync, nft_counter_sync); 34 35 /* control plane only: sync fetch+reset */ 36 static DEFINE_SPINLOCK(nft_counter_lock); 37 38 static inline void nft_counter_do_eval(struct nft_counter_percpu_priv *priv, 39 struct nft_regs *regs, 40 const struct nft_pktinfo *pkt) 41 { 42 struct u64_stats_sync *nft_sync; 43 struct nft_counter *this_cpu; 44 45 local_bh_disable(); 46 this_cpu = this_cpu_ptr(priv->counter); 47 nft_sync = this_cpu_ptr(&nft_counter_sync); 48 49 u64_stats_update_begin(nft_sync); 50 u64_stats_add(&this_cpu->bytes, pkt->skb->len); 51 u64_stats_inc(&this_cpu->packets); 52 u64_stats_update_end(nft_sync); 53 54 local_bh_enable(); 55 } 56 57 static inline void nft_counter_obj_eval(struct nft_object *obj, 58 struct nft_regs *regs, 59 const struct nft_pktinfo *pkt) 60 { 61 struct nft_counter_percpu_priv *priv = nft_obj_data(obj); 62 63 nft_counter_do_eval(priv, regs, pkt); 64 } 65 66 static int nft_counter_do_init(const struct nlattr * const tb[], 67 struct nft_counter_percpu_priv *priv) 68 { 69 struct nft_counter __percpu *cpu_stats; 70 struct nft_counter *this_cpu; 71 72 cpu_stats = alloc_percpu_gfp(struct nft_counter, GFP_KERNEL_ACCOUNT); 73 if (cpu_stats == NULL) 74 return -ENOMEM; 75 76 this_cpu = raw_cpu_ptr(cpu_stats); 77 if (tb[NFTA_COUNTER_PACKETS]) { 78 u64_stats_set(&this_cpu->packets, 79 be64_to_cpu(nla_get_be64(tb[NFTA_COUNTER_PACKETS]))); 80 } 81 if (tb[NFTA_COUNTER_BYTES]) { 82 u64_stats_set(&this_cpu->bytes, 83 be64_to_cpu(nla_get_be64(tb[NFTA_COUNTER_BYTES]))); 84 } 85 86 priv->counter = cpu_stats; 87 return 0; 88 } 89 90 static int nft_counter_obj_init(const struct nft_ctx *ctx, 91 const struct nlattr * const tb[], 92 struct nft_object *obj) 93 { 94 struct nft_counter_percpu_priv *priv = nft_obj_data(obj); 95 96 return nft_counter_do_init(tb, priv); 97 } 98 99 static void nft_counter_do_destroy(struct nft_counter_percpu_priv *priv) 100 { 101 free_percpu(priv->counter); 102 } 103 104 static void nft_counter_obj_destroy(const struct nft_ctx *ctx, 105 struct nft_object *obj) 106 { 107 struct nft_counter_percpu_priv *priv = nft_obj_data(obj); 108 109 nft_counter_do_destroy(priv); 110 } 111 112 static void nft_counter_reset(struct nft_counter_percpu_priv *priv, 113 struct nft_counter_tot *total) 114 { 115 struct u64_stats_sync *nft_sync; 116 struct nft_counter *this_cpu; 117 118 local_bh_disable(); 119 this_cpu = this_cpu_ptr(priv->counter); 120 nft_sync = this_cpu_ptr(&nft_counter_sync); 121 122 u64_stats_update_begin(nft_sync); 123 u64_stats_sub(&this_cpu->packets, total->packets); 124 u64_stats_sub(&this_cpu->bytes, total->bytes); 125 u64_stats_update_end(nft_sync); 126 127 local_bh_enable(); 128 } 129 130 static void nft_counter_fetch(struct nft_counter_percpu_priv *priv, 131 struct nft_counter_tot *total) 132 { 133 struct nft_counter *this_cpu; 134 u64 bytes, packets; 135 unsigned int seq; 136 int cpu; 137 138 memset(total, 0, sizeof(*total)); 139 for_each_possible_cpu(cpu) { 140 struct u64_stats_sync *nft_sync = per_cpu_ptr(&nft_counter_sync, cpu); 141 142 this_cpu = per_cpu_ptr(priv->counter, cpu); 143 do { 144 seq = u64_stats_fetch_begin(nft_sync); 145 bytes = u64_stats_read(&this_cpu->bytes); 146 packets = u64_stats_read(&this_cpu->packets); 147 } while (u64_stats_fetch_retry(nft_sync, seq)); 148 149 total->bytes += bytes; 150 total->packets += packets; 151 } 152 } 153 154 static void nft_counter_fetch_and_reset(struct nft_counter_percpu_priv *priv, 155 struct nft_counter_tot *total) 156 { 157 spin_lock(&nft_counter_lock); 158 nft_counter_fetch(priv, total); 159 nft_counter_reset(priv, total); 160 spin_unlock(&nft_counter_lock); 161 } 162 163 static int nft_counter_do_dump(struct sk_buff *skb, 164 struct nft_counter_percpu_priv *priv, 165 bool reset) 166 { 167 struct nft_counter_tot total; 168 169 if (unlikely(reset)) 170 nft_counter_fetch_and_reset(priv, &total); 171 else 172 nft_counter_fetch(priv, &total); 173 174 if (nla_put_be64(skb, NFTA_COUNTER_BYTES, cpu_to_be64(total.bytes), 175 NFTA_COUNTER_PAD) || 176 nla_put_be64(skb, NFTA_COUNTER_PACKETS, cpu_to_be64(total.packets), 177 NFTA_COUNTER_PAD)) 178 goto nla_put_failure; 179 180 return 0; 181 182 nla_put_failure: 183 return -1; 184 } 185 186 static int nft_counter_obj_dump(struct sk_buff *skb, 187 struct nft_object *obj, bool reset) 188 { 189 struct nft_counter_percpu_priv *priv = nft_obj_data(obj); 190 191 return nft_counter_do_dump(skb, priv, reset); 192 } 193 194 static const struct nla_policy nft_counter_policy[NFTA_COUNTER_MAX + 1] = { 195 [NFTA_COUNTER_PACKETS] = { .type = NLA_U64 }, 196 [NFTA_COUNTER_BYTES] = { .type = NLA_U64 }, 197 }; 198 199 struct nft_object_type nft_counter_obj_type; 200 static const struct nft_object_ops nft_counter_obj_ops = { 201 .type = &nft_counter_obj_type, 202 .size = sizeof(struct nft_counter_percpu_priv), 203 .eval = nft_counter_obj_eval, 204 .init = nft_counter_obj_init, 205 .destroy = nft_counter_obj_destroy, 206 .dump = nft_counter_obj_dump, 207 }; 208 209 struct nft_object_type nft_counter_obj_type __read_mostly = { 210 .type = NFT_OBJECT_COUNTER, 211 .ops = &nft_counter_obj_ops, 212 .maxattr = NFTA_COUNTER_MAX, 213 .policy = nft_counter_policy, 214 .owner = THIS_MODULE, 215 }; 216 217 void nft_counter_eval(const struct nft_expr *expr, struct nft_regs *regs, 218 const struct nft_pktinfo *pkt) 219 { 220 struct nft_counter_percpu_priv *priv = nft_expr_priv(expr); 221 222 nft_counter_do_eval(priv, regs, pkt); 223 } 224 225 static int nft_counter_dump(struct sk_buff *skb, 226 const struct nft_expr *expr, bool reset) 227 { 228 struct nft_counter_percpu_priv *priv = nft_expr_priv(expr); 229 230 return nft_counter_do_dump(skb, priv, reset); 231 } 232 233 static int nft_counter_init(const struct nft_ctx *ctx, 234 const struct nft_expr *expr, 235 const struct nlattr * const tb[]) 236 { 237 struct nft_counter_percpu_priv *priv = nft_expr_priv(expr); 238 239 return nft_counter_do_init(tb, priv); 240 } 241 242 static void nft_counter_destroy(const struct nft_ctx *ctx, 243 const struct nft_expr *expr) 244 { 245 struct nft_counter_percpu_priv *priv = nft_expr_priv(expr); 246 247 nft_counter_do_destroy(priv); 248 } 249 250 static int nft_counter_clone(struct nft_expr *dst, const struct nft_expr *src, gfp_t gfp) 251 { 252 struct nft_counter_percpu_priv *priv = nft_expr_priv(src); 253 struct nft_counter_percpu_priv *priv_clone = nft_expr_priv(dst); 254 struct nft_counter __percpu *cpu_stats; 255 struct nft_counter *this_cpu; 256 struct nft_counter_tot total; 257 258 nft_counter_fetch(priv, &total); 259 260 cpu_stats = alloc_percpu_gfp(struct nft_counter, gfp); 261 if (cpu_stats == NULL) 262 return -ENOMEM; 263 264 this_cpu = raw_cpu_ptr(cpu_stats); 265 u64_stats_set(&this_cpu->packets, total.packets); 266 u64_stats_set(&this_cpu->bytes, total.bytes); 267 268 priv_clone->counter = cpu_stats; 269 return 0; 270 } 271 272 static int nft_counter_offload(struct nft_offload_ctx *ctx, 273 struct nft_flow_rule *flow, 274 const struct nft_expr *expr) 275 { 276 /* No specific offload action is needed, but report success. */ 277 return 0; 278 } 279 280 static void nft_counter_offload_stats(struct nft_expr *expr, 281 const struct flow_stats *stats) 282 { 283 struct nft_counter_percpu_priv *priv = nft_expr_priv(expr); 284 struct u64_stats_sync *nft_sync; 285 struct nft_counter *this_cpu; 286 287 local_bh_disable(); 288 this_cpu = this_cpu_ptr(priv->counter); 289 nft_sync = this_cpu_ptr(&nft_counter_sync); 290 291 u64_stats_update_begin(nft_sync); 292 u64_stats_add(&this_cpu->packets, stats->pkts); 293 u64_stats_add(&this_cpu->bytes, stats->bytes); 294 u64_stats_update_end(nft_sync); 295 local_bh_enable(); 296 } 297 298 void nft_counter_init_seqcount(void) 299 { 300 int cpu; 301 302 for_each_possible_cpu(cpu) 303 u64_stats_init(per_cpu_ptr(&nft_counter_sync, cpu)); 304 } 305 306 struct nft_expr_type nft_counter_type; 307 static const struct nft_expr_ops nft_counter_ops = { 308 .type = &nft_counter_type, 309 .size = NFT_EXPR_SIZE(sizeof(struct nft_counter_percpu_priv)), 310 .eval = nft_counter_eval, 311 .init = nft_counter_init, 312 .destroy = nft_counter_destroy, 313 .destroy_clone = nft_counter_destroy, 314 .dump = nft_counter_dump, 315 .clone = nft_counter_clone, 316 .reduce = NFT_REDUCE_READONLY, 317 .offload = nft_counter_offload, 318 .offload_stats = nft_counter_offload_stats, 319 }; 320 321 struct nft_expr_type nft_counter_type __read_mostly = { 322 .name = "counter", 323 .ops = &nft_counter_ops, 324 .policy = nft_counter_policy, 325 .maxattr = NFTA_COUNTER_MAX, 326 .flags = NFT_EXPR_STATEFUL, 327 .owner = THIS_MODULE, 328 }; 329