1 /*-
2 * Copyright (c) 2010-2011 Juniper Networks, Inc.
3 * All rights reserved.
4 *
5 * This software was developed by Robert N. M. Watson under contract
6 * to Juniper Networks, Inc.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30
31 #include "opt_inet6.h"
32 #include "opt_inet.h"
33 #include "opt_rss.h"
34
35 #include <sys/param.h>
36 #include <sys/mbuf.h>
37 #include <sys/socket.h>
38 #include <sys/priv.h>
39 #include <sys/kernel.h>
40 #include <sys/smp.h>
41 #include <sys/sysctl.h>
42 #include <sys/sbuf.h>
43
44 #include <net/if.h>
45 #include <net/if_var.h>
46 #include <net/netisr.h>
47 #include <net/rss_config.h>
48 #include <net/toeplitz.h>
49
50 /*-
51 * Operating system parts of receiver-side scaling (RSS), which allows
52 * network cards to direct flows to particular receive queues based on hashes
53 * of header tuples. This implementation aligns RSS buckets with connection
54 * groups at the TCP/IP layer, so each bucket is associated with exactly one
55 * group. As a result, the group lookup structures (and lock) should have an
56 * effective affinity with exactly one CPU.
57 *
58 * Network device drivers needing to configure RSS will query this framework
59 * for parameters, such as the current RSS key, hashing policies, number of
60 * bits, and indirection table mapping hashes to buckets and CPUs. They may
61 * provide their own supplementary information, such as queue<->CPU bindings.
62 * It is the responsibility of the network device driver to inject packets
63 * into the stack on as close to the right CPU as possible, if playing by RSS
64 * rules.
65 *
66 * TODO:
67 *
68 * - Synchronization for rss_key and other future-configurable parameters.
69 * - Event handler drivers can register to pick up RSS configuration changes.
70 * - Should we allow rss_basecpu to be configured?
71 * - Randomize key on boot.
72 * - IPv6 support.
73 * - Statistics on how often there's a misalignment between hardware
74 * placement and pcbgroup expectations.
75 */
76
77 #if !defined(INET) && !defined(INET6)
78 #define _net_inet _net
79 #define _net_inet_rss _net_rss
80 #endif
81 SYSCTL_DECL(_net_inet);
82 SYSCTL_NODE(_net_inet, OID_AUTO, rss, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
83 "Receive-side steering");
84
85 /*
86 * Toeplitz is the only required hash function in the RSS spec, so use it by
87 * default.
88 */
89 static u_int rss_hashalgo = RSS_HASH_TOEPLITZ;
90 SYSCTL_INT(_net_inet_rss, OID_AUTO, hashalgo, CTLFLAG_RDTUN, &rss_hashalgo, 0,
91 "RSS hash algorithm");
92
93 #ifdef RSS
94 FEATURE(rss, "Receiver-side scaling");
95 /*
96 * Size of the indirection table; at most 128 entries per the RSS spec. We
97 * size it to at least 2 times the number of CPUs by default to allow useful
98 * rebalancing. If not set explicitly with a loader tunable, we tune based
99 * on the number of CPUs present.
100 *
101 * XXXRW: buckets might be better to use for the tunable than bits.
102 */
103 static u_int rss_bits;
104 SYSCTL_INT(_net_inet_rss, OID_AUTO, bits, CTLFLAG_RDTUN, &rss_bits, 0,
105 "RSS bits");
106
107 static u_int rss_mask;
108 SYSCTL_INT(_net_inet_rss, OID_AUTO, mask, CTLFLAG_RD, &rss_mask, 0,
109 "RSS mask");
110
111 static const u_int rss_maxbits = RSS_MAXBITS;
112 SYSCTL_INT(_net_inet_rss, OID_AUTO, maxbits, CTLFLAG_RD,
113 __DECONST(int *, &rss_maxbits), 0, "RSS maximum bits");
114
115 /*
116 * RSS's own count of the number of CPUs it could be using for processing.
117 * Bounded to 64 by RSS constants.
118 */
119 static u_int rss_ncpus;
120 SYSCTL_INT(_net_inet_rss, OID_AUTO, ncpus, CTLFLAG_RD, &rss_ncpus, 0,
121 "Number of CPUs available to RSS");
122
123 #define RSS_MAXCPUS (1 << (RSS_MAXBITS - 1))
124 static const u_int rss_maxcpus = RSS_MAXCPUS;
125 SYSCTL_INT(_net_inet_rss, OID_AUTO, maxcpus, CTLFLAG_RD,
126 __DECONST(int *, &rss_maxcpus), 0, "RSS maximum CPUs that can be used");
127
128 /*
129 * Variable exists just for reporting rss_bits in a user-friendly way.
130 */
131 static u_int rss_buckets;
132 SYSCTL_INT(_net_inet_rss, OID_AUTO, buckets, CTLFLAG_RD, &rss_buckets, 0,
133 "RSS buckets");
134
135 /*
136 * Base CPU number; devices will add this to all CPU numbers returned by the
137 * RSS indirection table. Currently unmodifable in FreeBSD.
138 */
139 static const u_int rss_basecpu;
140 SYSCTL_INT(_net_inet_rss, OID_AUTO, basecpu, CTLFLAG_RD,
141 __DECONST(int *, &rss_basecpu), 0, "RSS base CPU");
142
143 #endif
144 /*
145 * Print verbose debugging messages.
146 * 0 - disable
147 * non-zero - enable
148 */
149 int rss_debug = 0;
150 SYSCTL_INT(_net_inet_rss, OID_AUTO, debug, CTLFLAG_RWTUN, &rss_debug, 0,
151 "RSS debug level");
152
153 static u_int rss_udp_4tuple = 0;
154 SYSCTL_INT(_net_inet_rss, OID_AUTO, udp_4tuple, CTLFLAG_RDTUN,
155 &rss_udp_4tuple, 0,
156 "Enable UDP 4-tuple RSS hashing (src/dst IP + src/dst port)");
157
158 /*
159 * RSS secret key, intended to prevent attacks on load-balancing. Its
160 * effectiveness may be limited by algorithm choice and available entropy
161 * during the boot.
162 *
163 * XXXRW: And that we don't randomize it yet!
164 *
165 * This is the default Microsoft RSS specification key which is also
166 * the Chelsio T5 firmware default key.
167 */
168 static uint8_t rss_key[RSS_KEYSIZE] = {
169 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2,
170 0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0,
171 0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4,
172 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c,
173 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa,
174 };
175
176 #ifdef RSS
177 /*
178 * RSS hash->CPU table, which maps hashed packet headers to particular CPUs.
179 * Drivers may supplement this table with a separate CPU<->queue table when
180 * programming devices.
181 */
182 struct rss_table_entry {
183 uint8_t rte_cpu; /* CPU affinity of bucket. */
184 };
185 static struct rss_table_entry rss_table[RSS_TABLE_MAXLEN];
186 #endif
187
188 static void
rss_init(__unused void * arg)189 rss_init(__unused void *arg)
190 {
191 #ifdef RSS
192 u_int i;
193 u_int cpuid;
194 #endif
195 /*
196 * Validate tunables, coerce to sensible values.
197 */
198 switch (rss_hashalgo) {
199 case RSS_HASH_TOEPLITZ:
200 case RSS_HASH_NAIVE:
201 break;
202
203 default:
204 RSS_DEBUG("invalid RSS hashalgo %u, coercing to %u\n",
205 rss_hashalgo, RSS_HASH_TOEPLITZ);
206 rss_hashalgo = RSS_HASH_TOEPLITZ;
207 }
208
209 #ifdef RSS
210 /*
211 * Count available CPUs.
212 *
213 * XXXRW: Note incorrect assumptions regarding contiguity of this set
214 * elsewhere.
215 */
216 rss_ncpus = 0;
217 for (i = 0; i <= mp_maxid; i++) {
218 if (CPU_ABSENT(i))
219 continue;
220 rss_ncpus++;
221 }
222 if (rss_ncpus > RSS_MAXCPUS)
223 rss_ncpus = RSS_MAXCPUS;
224
225 /*
226 * Tune RSS table entries to be no less than 2x the number of CPUs
227 * -- unless we're running uniprocessor, in which case there's not
228 * much point in having buckets to rearrange for load-balancing!
229 */
230 if (rss_ncpus > 1) {
231 if (rss_bits == 0)
232 rss_bits = fls(rss_ncpus - 1) + 1;
233
234 /*
235 * Microsoft limits RSS table entries to 128, so apply that
236 * limit to both auto-detected CPU counts and user-configured
237 * ones.
238 */
239 if (rss_bits == 0 || rss_bits > RSS_MAXBITS) {
240 RSS_DEBUG("RSS bits %u not valid, coercing to %u\n",
241 rss_bits, RSS_MAXBITS);
242 rss_bits = RSS_MAXBITS;
243 }
244
245 /*
246 * Figure out how many buckets to use; warn if less than the
247 * number of configured CPUs, although this is not a fatal
248 * problem.
249 */
250 rss_buckets = (1 << rss_bits);
251 if (rss_buckets < rss_ncpus)
252 RSS_DEBUG("WARNING: rss_buckets (%u) less than "
253 "rss_ncpus (%u)\n", rss_buckets, rss_ncpus);
254 rss_mask = rss_buckets - 1;
255 } else {
256 rss_bits = 0;
257 rss_buckets = 1;
258 rss_mask = 0;
259 }
260
261 /*
262 * Set up initial CPU assignments: round-robin by default.
263 */
264 cpuid = CPU_FIRST();
265 for (i = 0; i < rss_buckets; i++) {
266 rss_table[i].rte_cpu = cpuid;
267 cpuid = CPU_NEXT(cpuid);
268 }
269 #endif /* RSS */
270 /*
271 * Randomize rrs_key.
272 *
273 * XXXRW: Not yet. If nothing else, will require an rss_isbadkey()
274 * loop to check for "bad" RSS keys.
275 */
276 }
277 SYSINIT(rss_init, SI_SUB_SOFTINTR, SI_ORDER_SECOND, rss_init, NULL);
278
279 static uint32_t
rss_naive_hash(u_int keylen,const uint8_t * key,u_int datalen,const uint8_t * data)280 rss_naive_hash(u_int keylen, const uint8_t *key, u_int datalen,
281 const uint8_t *data)
282 {
283 uint32_t v;
284 u_int i;
285
286 v = 0;
287 for (i = 0; i < keylen; i++)
288 v += key[i];
289 for (i = 0; i < datalen; i++)
290 v += data[i];
291 return (v);
292 }
293
294 uint32_t
rss_hash(u_int datalen,const uint8_t * data)295 rss_hash(u_int datalen, const uint8_t *data)
296 {
297
298 switch (rss_hashalgo) {
299 case RSS_HASH_TOEPLITZ:
300 return (toeplitz_hash(sizeof(rss_key), rss_key, datalen,
301 data));
302
303 case RSS_HASH_NAIVE:
304 return (rss_naive_hash(sizeof(rss_key), rss_key, datalen,
305 data));
306
307 default:
308 panic("%s: unsupported/unknown hashalgo %d", __func__,
309 rss_hashalgo);
310 }
311 }
312
313 /*
314 * Query the current RSS key; likely to be used by device drivers when
315 * configuring hardware RSS. Caller must pass an array of size RSS_KEYSIZE.
316 *
317 * XXXRW: Perhaps we should do the accept-a-length-and-truncate thing?
318 */
319 void
rss_getkey(uint8_t * key)320 rss_getkey(uint8_t *key)
321 {
322
323 bcopy(rss_key, key, sizeof(rss_key));
324 }
325
326 /*
327 * Query the RSS hash algorithm.
328 */
329 u_int
rss_gethashalgo(void)330 rss_gethashalgo(void)
331 {
332
333 return (rss_hashalgo);
334 }
335
336 #ifdef RSS
337 /*
338 * Query the number of RSS bits in use.
339 */
340 u_int
rss_getbits(void)341 rss_getbits(void)
342 {
343
344 return (rss_bits);
345 }
346
347 /*
348 * Query the RSS bucket associated with an RSS hash.
349 */
350 u_int
rss_getbucket(u_int hash)351 rss_getbucket(u_int hash)
352 {
353
354 return (hash & rss_mask);
355 }
356
357 /*
358 * Query the RSS layer bucket associated with the given
359 * entry in the RSS hash space.
360 *
361 * The RSS indirection table is 0 .. rss_buckets-1,
362 * covering the low 'rss_bits' of the total 128 slot
363 * RSS indirection table. So just mask off rss_bits and
364 * return that.
365 *
366 * NIC drivers can then iterate over the 128 slot RSS
367 * indirection table and fetch which RSS bucket to
368 * map it to. This will typically be a CPU queue
369 */
370 u_int
rss_get_indirection_to_bucket(u_int index)371 rss_get_indirection_to_bucket(u_int index)
372 {
373
374 return (index & rss_mask);
375 }
376
377 /*
378 * Query the RSS CPU associated with an RSS bucket.
379 */
380 u_int
rss_getcpu(u_int bucket)381 rss_getcpu(u_int bucket)
382 {
383
384 return (rss_table[bucket].rte_cpu);
385 }
386
387 /*
388 * netisr CPU affinity lookup given just the hash and hashtype.
389 */
390 u_int
rss_hash2cpuid(uint32_t hash_val,uint32_t hash_type)391 rss_hash2cpuid(uint32_t hash_val, uint32_t hash_type)
392 {
393
394 switch (hash_type) {
395 case M_HASHTYPE_RSS_IPV4:
396 case M_HASHTYPE_RSS_TCP_IPV4:
397 case M_HASHTYPE_RSS_UDP_IPV4:
398 case M_HASHTYPE_RSS_IPV6:
399 case M_HASHTYPE_RSS_TCP_IPV6:
400 case M_HASHTYPE_RSS_UDP_IPV6:
401 return (rss_getcpu(rss_getbucket(hash_val)));
402 default:
403 return (NETISR_CPUID_NONE);
404 }
405 }
406
407 /*
408 * Query the RSS bucket associated with the given hash value and
409 * type.
410 */
411 int
rss_hash2bucket(uint32_t hash_val,uint32_t hash_type,uint32_t * bucket_id)412 rss_hash2bucket(uint32_t hash_val, uint32_t hash_type, uint32_t *bucket_id)
413 {
414
415 switch (hash_type) {
416 case M_HASHTYPE_RSS_IPV4:
417 case M_HASHTYPE_RSS_TCP_IPV4:
418 case M_HASHTYPE_RSS_UDP_IPV4:
419 case M_HASHTYPE_RSS_IPV6:
420 case M_HASHTYPE_RSS_TCP_IPV6:
421 case M_HASHTYPE_RSS_UDP_IPV6:
422 *bucket_id = rss_getbucket(hash_val);
423 return (0);
424 default:
425 return (-1);
426 }
427 }
428
429 /*
430 * netisr CPU affinity lookup routine for use by protocols.
431 */
432 struct mbuf *
rss_m2cpuid(struct mbuf * m,uintptr_t source,u_int * cpuid)433 rss_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid)
434 {
435
436 M_ASSERTPKTHDR(m);
437 *cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m));
438 return (m);
439 }
440
441 int
rss_m2bucket(struct mbuf * m,uint32_t * bucket_id)442 rss_m2bucket(struct mbuf *m, uint32_t *bucket_id)
443 {
444
445 M_ASSERTPKTHDR(m);
446
447 return(rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m),
448 bucket_id));
449 }
450
451 /*
452 * Query the number of buckets; this may be used by both network device
453 * drivers, which will need to populate hardware shadows of the software
454 * indirection table, and the network stack itself (such as when deciding how
455 * many connection groups to allocate).
456 */
457 u_int
rss_getnumbuckets(void)458 rss_getnumbuckets(void)
459 {
460
461 return (rss_buckets);
462 }
463
464 /*
465 * Query the number of CPUs in use by RSS; may be useful to device drivers
466 * trying to figure out how to map a larger number of CPUs into a smaller
467 * number of receive queues.
468 */
469 u_int
rss_getnumcpus(void)470 rss_getnumcpus(void)
471 {
472
473 return (rss_ncpus);
474 }
475
476 #endif
477 /*
478 * Return the supported RSS hash configuration.
479 *
480 * NICs should query this to determine what to configure in their redirection
481 * matching table.
482 */
483 inline u_int
rss_gethashconfig(void)484 rss_gethashconfig(void)
485 {
486
487 /* Return 4-tuple for TCP; 2-tuple for others */
488 /*
489 * UDP may fragment more often than TCP and thus we'll end up with
490 * NICs returning 2-tuple fragments.
491 * udp_init() and udplite_init() both currently initialise things
492 * as 2-tuple.
493 * So for now disable UDP 4-tuple hashing until all of the other
494 * pieces are in place.
495 */
496 u_int config;
497
498 config =
499 RSS_HASHTYPE_RSS_IPV4
500 | RSS_HASHTYPE_RSS_TCP_IPV4
501 | RSS_HASHTYPE_RSS_IPV6
502 | RSS_HASHTYPE_RSS_TCP_IPV6
503 | RSS_HASHTYPE_RSS_IPV6_EX
504 | RSS_HASHTYPE_RSS_TCP_IPV6_EX;
505
506 if (rss_udp_4tuple) {
507 config |=
508 RSS_HASHTYPE_RSS_UDP_IPV4
509 | RSS_HASHTYPE_RSS_UDP_IPV6
510 | RSS_HASHTYPE_RSS_UDP_IPV6_EX;
511 }
512
513 return (config);
514 }
515
516 /*
517 * XXXRW: Confirm that sysctl -a won't dump this keying material, don't want
518 * it appearing in debugging output unnecessarily.
519 */
520 static int
sysctl_rss_key(SYSCTL_HANDLER_ARGS)521 sysctl_rss_key(SYSCTL_HANDLER_ARGS)
522 {
523 uint8_t temp_rss_key[RSS_KEYSIZE];
524 int error;
525
526 error = priv_check(req->td, PRIV_NETINET_HASHKEY);
527 if (error)
528 return (error);
529
530 bcopy(rss_key, temp_rss_key, sizeof(temp_rss_key));
531 error = sysctl_handle_opaque(oidp, temp_rss_key,
532 sizeof(temp_rss_key), req);
533 if (error)
534 return (error);
535 if (req->newptr != NULL) {
536 /* XXXRW: Not yet. */
537 return (EINVAL);
538 }
539 return (0);
540 }
541 SYSCTL_PROC(_net_inet_rss, OID_AUTO, key,
542 CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, sysctl_rss_key,
543 "", "RSS keying material");
544
545 #ifdef RSS
546 static int
sysctl_rss_bucket_mapping(SYSCTL_HANDLER_ARGS)547 sysctl_rss_bucket_mapping(SYSCTL_HANDLER_ARGS)
548 {
549 struct sbuf *sb;
550 int error;
551 int i;
552
553 error = 0;
554 error = sysctl_wire_old_buffer(req, 0);
555 if (error != 0)
556 return (error);
557 sb = sbuf_new_for_sysctl(NULL, NULL, 512, req);
558 if (sb == NULL)
559 return (ENOMEM);
560 for (i = 0; i < rss_buckets; i++) {
561 sbuf_printf(sb, "%s%d:%d", i == 0 ? "" : " ",
562 i,
563 rss_getcpu(i));
564 }
565 error = sbuf_finish(sb);
566 sbuf_delete(sb);
567
568 return (error);
569 }
570 SYSCTL_PROC(_net_inet_rss, OID_AUTO, bucket_mapping,
571 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
572 sysctl_rss_bucket_mapping, "", "RSS bucket -> CPU mapping");
573 #endif
574