1 // SPDX-License-Identifier: GPL-2.0-only
2 
3 /* PIPAPO: PIle PAcket POlicies: AVX2 packet lookup routines
4  *
5  * Copyright (c) 2019-2020 Red Hat GmbH
6  *
7  * Author: Stefano Brivio <sbrivio@redhat.com>
8  */
9 
10 #include <linux/kernel.h>
11 #include <linux/init.h>
12 #include <linux/module.h>
13 #include <linux/netlink.h>
14 #include <linux/netfilter.h>
15 #include <linux/netfilter/nf_tables.h>
16 #include <net/netfilter/nf_tables_core.h>
17 #include <uapi/linux/netfilter/nf_tables.h>
18 #include <linux/bitmap.h>
19 #include <linux/bitops.h>
20 
21 #include <linux/compiler.h>
22 #include <asm/fpu/api.h>
23 
24 #include "nft_set_pipapo_avx2.h"
25 #include "nft_set_pipapo.h"
26 
27 #define NFT_PIPAPO_LONGS_PER_M256	(XSAVE_YMM_SIZE / BITS_PER_LONG)
28 
29 /* Load from memory into YMM register with non-temporal hint ("stream load"),
30  * that is, don't fetch lines from memory into the cache. This avoids pushing
31  * precious packet data out of the cache hierarchy, and is appropriate when:
32  *
33  * - loading buckets from lookup tables, as they are not going to be used
34  *   again before packets are entirely classified
35  *
36  * - loading the result bitmap from the previous field, as it's never used
37  *   again
38  */
39 #define NFT_PIPAPO_AVX2_LOAD(reg, loc)					\
40 	asm volatile("vmovntdqa %0, %%ymm" #reg : : "m" (loc))
41 
42 /* Stream a single lookup table bucket into YMM register given lookup table,
43  * group index, value of packet bits, bucket size.
44  */
45 #define NFT_PIPAPO_AVX2_BUCKET_LOAD4(reg, lt, group, v, bsize)		\
46 	NFT_PIPAPO_AVX2_LOAD(reg,					\
47 			     lt[((group) * NFT_PIPAPO_BUCKETS(4) +	\
48 				 (v)) * (bsize)])
49 #define NFT_PIPAPO_AVX2_BUCKET_LOAD8(reg, lt, group, v, bsize)		\
50 	NFT_PIPAPO_AVX2_LOAD(reg,					\
51 			     lt[((group) * NFT_PIPAPO_BUCKETS(8) +	\
52 				 (v)) * (bsize)])
53 
54 /* Bitwise AND: the staple operation of this algorithm */
55 #define NFT_PIPAPO_AVX2_AND(dst, a, b)					\
56 	asm volatile("vpand %ymm" #a ", %ymm" #b ", %ymm" #dst)
57 
58 /* Jump to label if @reg is zero */
59 #define NFT_PIPAPO_AVX2_NOMATCH_GOTO(reg, label)			\
60 	asm goto("vptest %%ymm" #reg ", %%ymm" #reg ";"	\
61 			  "je %l[" #label "]" : : : : label)
62 
63 /* Store 256 bits from YMM register into memory. Contrary to bucket load
64  * operation, we don't bypass the cache here, as stored matching results
65  * are always used shortly after.
66  */
67 #define NFT_PIPAPO_AVX2_STORE(loc, reg)					\
68 	asm volatile("vmovdqa %%ymm" #reg ", %0" : "=m" (loc))
69 
70 /* Zero out a complete YMM register, @reg */
71 #define NFT_PIPAPO_AVX2_ZERO(reg)					\
72 	asm volatile("vpxor %ymm" #reg ", %ymm" #reg ", %ymm" #reg)
73 
74 /**
75  * nft_pipapo_avx2_prepare() - Prepare before main algorithm body
76  *
77  * This zeroes out ymm15, which is later used whenever we need to clear a
78  * memory location, by storing its content into memory.
79  */
nft_pipapo_avx2_prepare(void)80 static void nft_pipapo_avx2_prepare(void)
81 {
82 	NFT_PIPAPO_AVX2_ZERO(15);
83 }
84 
85 /**
86  * nft_pipapo_avx2_fill() - Fill a bitmap region with ones
87  * @data:	Base memory area
88  * @start:	First bit to set
89  * @len:	Count of bits to fill
90  *
91  * This is nothing else than a version of bitmap_set(), as used e.g. by
92  * pipapo_refill(), tailored for the microarchitectures using it and better
93  * suited for the specific usage: it's very likely that we'll set a small number
94  * of bits, not crossing a word boundary, and correct branch prediction is
95  * critical here.
96  *
97  * This function doesn't actually use any AVX2 instruction.
98  */
nft_pipapo_avx2_fill(unsigned long * data,int start,int len)99 static void nft_pipapo_avx2_fill(unsigned long *data, int start, int len)
100 {
101 	int offset = start % BITS_PER_LONG;
102 	unsigned long mask;
103 
104 	data += start / BITS_PER_LONG;
105 
106 	if (likely(len == 1)) {
107 		*data |= BIT(offset);
108 		return;
109 	}
110 
111 	if (likely(len < BITS_PER_LONG || offset)) {
112 		if (likely(len + offset <= BITS_PER_LONG)) {
113 			*data |= GENMASK(len - 1 + offset, offset);
114 			return;
115 		}
116 
117 		*data |= ~0UL << offset;
118 		len -= BITS_PER_LONG - offset;
119 		data++;
120 
121 		if (len <= BITS_PER_LONG) {
122 			mask = ~0UL >> (BITS_PER_LONG - len);
123 			*data |= mask;
124 			return;
125 		}
126 	}
127 
128 	memset(data, 0xff, len / BITS_PER_BYTE);
129 	data += len / BITS_PER_LONG;
130 
131 	len %= BITS_PER_LONG;
132 	if (len)
133 		*data |= ~0UL >> (BITS_PER_LONG - len);
134 }
135 
136 /**
137  * nft_pipapo_avx2_refill() - Scan bitmap, select mapping table item, set bits
138  * @offset:	Start from given bitmap (equivalent to bucket) offset, in longs
139  * @map:	Bitmap to be scanned for set bits
140  * @dst:	Destination bitmap
141  * @mt:		Mapping table containing bit set specifiers
142  * @last:	Return index of first set bit, if this is the last field
143  *
144  * This is an alternative implementation of pipapo_refill() suitable for usage
145  * with AVX2 lookup routines: we know there are four words to be scanned, at
146  * a given offset inside the map, for each matching iteration.
147  *
148  * This function doesn't actually use any AVX2 instruction.
149  *
150  * Return: first set bit index if @last, index of first filled word otherwise.
151  */
nft_pipapo_avx2_refill(int offset,unsigned long * map,unsigned long * dst,union nft_pipapo_map_bucket * mt,bool last)152 static int nft_pipapo_avx2_refill(int offset, unsigned long *map,
153 				  unsigned long *dst,
154 				  union nft_pipapo_map_bucket *mt, bool last)
155 {
156 	int ret = -1;
157 
158 #define NFT_PIPAPO_AVX2_REFILL_ONE_WORD(x)				\
159 	do {								\
160 		while (map[(x)]) {					\
161 			int r = __builtin_ctzl(map[(x)]);		\
162 			int i = (offset + (x)) * BITS_PER_LONG + r;	\
163 									\
164 			if (last)					\
165 				return i;				\
166 									\
167 			nft_pipapo_avx2_fill(dst, mt[i].to, mt[i].n);	\
168 									\
169 			if (ret == -1)					\
170 				ret = mt[i].to;				\
171 									\
172 			map[(x)] &= ~(1UL << r);			\
173 		}							\
174 	} while (0)
175 
176 	NFT_PIPAPO_AVX2_REFILL_ONE_WORD(0);
177 	NFT_PIPAPO_AVX2_REFILL_ONE_WORD(1);
178 	NFT_PIPAPO_AVX2_REFILL_ONE_WORD(2);
179 	NFT_PIPAPO_AVX2_REFILL_ONE_WORD(3);
180 #undef NFT_PIPAPO_AVX2_REFILL_ONE_WORD
181 
182 	return ret;
183 }
184 
185 /**
186  * nft_pipapo_avx2_lookup_4b_2() - AVX2-based lookup for 2 four-bit groups
187  * @map:	Previous match result, used as initial bitmap
188  * @fill:	Destination bitmap to be filled with current match result
189  * @f:		Field, containing lookup and mapping tables
190  * @offset:	Ignore buckets before the given index, no bits are filled there
191  * @pkt:	Packet data, pointer to input nftables register
192  * @first:	If this is the first field, don't source previous result
193  * @last:	Last field: stop at the first match and return bit index
194  *
195  * Load buckets from lookup table corresponding to the values of each 4-bit
196  * group of packet bytes, and perform a bitwise intersection between them. If
197  * this is the first field in the set, simply AND the buckets together
198  * (equivalent to using an all-ones starting bitmap), use the provided starting
199  * bitmap otherwise. Then call nft_pipapo_avx2_refill() to generate the next
200  * working bitmap, @fill.
201  *
202  * This is used for 8-bit fields (i.e. protocol numbers).
203  *
204  * Out-of-order (and superscalar) execution is vital here, so it's critical to
205  * avoid false data dependencies. CPU and compiler could (mostly) take care of
206  * this on their own, but the operation ordering is explicitly given here with
207  * a likely execution order in mind, to highlight possible stalls. That's why
208  * a number of logically distinct operations (i.e. loading buckets, intersecting
209  * buckets) are interleaved.
210  *
211  * Return: -1 on no match, rule index of match if @last, otherwise first long
212  * word index to be checked next (i.e. first filled word).
213  */
nft_pipapo_avx2_lookup_4b_2(unsigned long * map,unsigned long * fill,struct nft_pipapo_field * f,int offset,const u8 * pkt,bool first,bool last)214 static int nft_pipapo_avx2_lookup_4b_2(unsigned long *map, unsigned long *fill,
215 				       struct nft_pipapo_field *f, int offset,
216 				       const u8 *pkt, bool first, bool last)
217 {
218 	int i, ret = -1, m256_size = f->bsize / NFT_PIPAPO_LONGS_PER_M256, b;
219 	u8 pg[2] = { pkt[0] >> 4, pkt[0] & 0xf };
220 	unsigned long *lt = f->lt, bsize = f->bsize;
221 
222 	lt += offset * NFT_PIPAPO_LONGS_PER_M256;
223 	for (i = offset; i < m256_size; i++, lt += NFT_PIPAPO_LONGS_PER_M256) {
224 		int i_ul = i * NFT_PIPAPO_LONGS_PER_M256;
225 
226 		if (first) {
227 			NFT_PIPAPO_AVX2_BUCKET_LOAD4(0, lt, 0, pg[0], bsize);
228 			NFT_PIPAPO_AVX2_BUCKET_LOAD4(1, lt, 1, pg[1], bsize);
229 			NFT_PIPAPO_AVX2_AND(4, 0, 1);
230 		} else {
231 			NFT_PIPAPO_AVX2_BUCKET_LOAD4(0, lt, 0, pg[0], bsize);
232 			NFT_PIPAPO_AVX2_LOAD(2, map[i_ul]);
233 			NFT_PIPAPO_AVX2_BUCKET_LOAD4(1, lt, 1, pg[1], bsize);
234 			NFT_PIPAPO_AVX2_NOMATCH_GOTO(2, nothing);
235 			NFT_PIPAPO_AVX2_AND(3, 0, 1);
236 			NFT_PIPAPO_AVX2_AND(4, 2, 3);
237 		}
238 
239 		NFT_PIPAPO_AVX2_NOMATCH_GOTO(4, nomatch);
240 		NFT_PIPAPO_AVX2_STORE(map[i_ul], 4);
241 
242 		b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last);
243 		if (last)
244 			return b;
245 
246 		if (unlikely(ret == -1))
247 			ret = b / XSAVE_YMM_SIZE;
248 
249 		continue;
250 nomatch:
251 		NFT_PIPAPO_AVX2_STORE(map[i_ul], 15);
252 nothing:
253 		;
254 	}
255 
256 	return ret;
257 }
258 
259 /**
260  * nft_pipapo_avx2_lookup_4b_4() - AVX2-based lookup for 4 four-bit groups
261  * @map:	Previous match result, used as initial bitmap
262  * @fill:	Destination bitmap to be filled with current match result
263  * @f:		Field, containing lookup and mapping tables
264  * @offset:	Ignore buckets before the given index, no bits are filled there
265  * @pkt:	Packet data, pointer to input nftables register
266  * @first:	If this is the first field, don't source previous result
267  * @last:	Last field: stop at the first match and return bit index
268  *
269  * See nft_pipapo_avx2_lookup_4b_2().
270  *
271  * This is used for 16-bit fields (i.e. ports).
272  *
273  * Return: -1 on no match, rule index of match if @last, otherwise first long
274  * word index to be checked next (i.e. first filled word).
275  */
nft_pipapo_avx2_lookup_4b_4(unsigned long * map,unsigned long * fill,struct nft_pipapo_field * f,int offset,const u8 * pkt,bool first,bool last)276 static int nft_pipapo_avx2_lookup_4b_4(unsigned long *map, unsigned long *fill,
277 				       struct nft_pipapo_field *f, int offset,
278 				       const u8 *pkt, bool first, bool last)
279 {
280 	int i, ret = -1, m256_size = f->bsize / NFT_PIPAPO_LONGS_PER_M256, b;
281 	u8 pg[4] = { pkt[0] >> 4, pkt[0] & 0xf, pkt[1] >> 4, pkt[1] & 0xf };
282 	unsigned long *lt = f->lt, bsize = f->bsize;
283 
284 	lt += offset * NFT_PIPAPO_LONGS_PER_M256;
285 	for (i = offset; i < m256_size; i++, lt += NFT_PIPAPO_LONGS_PER_M256) {
286 		int i_ul = i * NFT_PIPAPO_LONGS_PER_M256;
287 
288 		if (first) {
289 			NFT_PIPAPO_AVX2_BUCKET_LOAD4(0, lt, 0, pg[0], bsize);
290 			NFT_PIPAPO_AVX2_BUCKET_LOAD4(1, lt, 1, pg[1], bsize);
291 			NFT_PIPAPO_AVX2_BUCKET_LOAD4(2, lt, 2, pg[2], bsize);
292 			NFT_PIPAPO_AVX2_BUCKET_LOAD4(3, lt, 3, pg[3], bsize);
293 			NFT_PIPAPO_AVX2_AND(4, 0, 1);
294 			NFT_PIPAPO_AVX2_AND(5, 2, 3);
295 			NFT_PIPAPO_AVX2_AND(7, 4, 5);
296 		} else {
297 			NFT_PIPAPO_AVX2_BUCKET_LOAD4(0, lt, 0, pg[0], bsize);
298 
299 			NFT_PIPAPO_AVX2_LOAD(1, map[i_ul]);
300 
301 			NFT_PIPAPO_AVX2_BUCKET_LOAD4(2, lt, 1, pg[1], bsize);
302 			NFT_PIPAPO_AVX2_BUCKET_LOAD4(3, lt, 2, pg[2], bsize);
303 			NFT_PIPAPO_AVX2_BUCKET_LOAD4(4, lt, 3, pg[3], bsize);
304 			NFT_PIPAPO_AVX2_AND(5, 0, 1);
305 
306 			NFT_PIPAPO_AVX2_NOMATCH_GOTO(1, nothing);
307 
308 			NFT_PIPAPO_AVX2_AND(6, 2, 3);
309 			NFT_PIPAPO_AVX2_AND(7, 4, 5);
310 			/* Stall */
311 			NFT_PIPAPO_AVX2_AND(7, 6, 7);
312 		}
313 
314 		/* Stall */
315 		NFT_PIPAPO_AVX2_NOMATCH_GOTO(7, nomatch);
316 		NFT_PIPAPO_AVX2_STORE(map[i_ul], 7);
317 
318 		b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last);
319 		if (last)
320 			return b;
321 
322 		if (unlikely(ret == -1))
323 			ret = b / XSAVE_YMM_SIZE;
324 
325 		continue;
326 nomatch:
327 		NFT_PIPAPO_AVX2_STORE(map[i_ul], 15);
328 nothing:
329 		;
330 	}
331 
332 	return ret;
333 }
334 
335 /**
336  * nft_pipapo_avx2_lookup_4b_8() - AVX2-based lookup for 8 four-bit groups
337  * @map:	Previous match result, used as initial bitmap
338  * @fill:	Destination bitmap to be filled with current match result
339  * @f:		Field, containing lookup and mapping tables
340  * @offset:	Ignore buckets before the given index, no bits are filled there
341  * @pkt:	Packet data, pointer to input nftables register
342  * @first:	If this is the first field, don't source previous result
343  * @last:	Last field: stop at the first match and return bit index
344  *
345  * See nft_pipapo_avx2_lookup_4b_2().
346  *
347  * This is used for 32-bit fields (i.e. IPv4 addresses).
348  *
349  * Return: -1 on no match, rule index of match if @last, otherwise first long
350  * word index to be checked next (i.e. first filled word).
351  */
nft_pipapo_avx2_lookup_4b_8(unsigned long * map,unsigned long * fill,struct nft_pipapo_field * f,int offset,const u8 * pkt,bool first,bool last)352 static int nft_pipapo_avx2_lookup_4b_8(unsigned long *map, unsigned long *fill,
353 				       struct nft_pipapo_field *f, int offset,
354 				       const u8 *pkt, bool first, bool last)
355 {
356 	u8 pg[8] = {  pkt[0] >> 4,  pkt[0] & 0xf,  pkt[1] >> 4,  pkt[1] & 0xf,
357 		      pkt[2] >> 4,  pkt[2] & 0xf,  pkt[3] >> 4,  pkt[3] & 0xf,
358 		   };
359 	int i, ret = -1, m256_size = f->bsize / NFT_PIPAPO_LONGS_PER_M256, b;
360 	unsigned long *lt = f->lt, bsize = f->bsize;
361 
362 	lt += offset * NFT_PIPAPO_LONGS_PER_M256;
363 	for (i = offset; i < m256_size; i++, lt += NFT_PIPAPO_LONGS_PER_M256) {
364 		int i_ul = i * NFT_PIPAPO_LONGS_PER_M256;
365 
366 		if (first) {
367 			NFT_PIPAPO_AVX2_BUCKET_LOAD4(0,  lt, 0, pg[0], bsize);
368 			NFT_PIPAPO_AVX2_BUCKET_LOAD4(1,  lt, 1, pg[1], bsize);
369 			NFT_PIPAPO_AVX2_BUCKET_LOAD4(2,  lt, 2, pg[2], bsize);
370 			NFT_PIPAPO_AVX2_BUCKET_LOAD4(3,  lt, 3, pg[3], bsize);
371 			NFT_PIPAPO_AVX2_BUCKET_LOAD4(4,  lt, 4, pg[4], bsize);
372 			NFT_PIPAPO_AVX2_AND(5,   0,  1);
373 			NFT_PIPAPO_AVX2_BUCKET_LOAD4(6,  lt, 5, pg[5], bsize);
374 			NFT_PIPAPO_AVX2_BUCKET_LOAD4(7,  lt, 6, pg[6], bsize);
375 			NFT_PIPAPO_AVX2_AND(8,   2,  3);
376 			NFT_PIPAPO_AVX2_AND(9,   4,  5);
377 			NFT_PIPAPO_AVX2_BUCKET_LOAD4(10, lt, 7, pg[7], bsize);
378 			NFT_PIPAPO_AVX2_AND(11,  6,  7);
379 			NFT_PIPAPO_AVX2_AND(12,  8,  9);
380 			NFT_PIPAPO_AVX2_AND(13, 10, 11);
381 
382 			/* Stall */
383 			NFT_PIPAPO_AVX2_AND(1,  12, 13);
384 		} else {
385 			NFT_PIPAPO_AVX2_BUCKET_LOAD4(0,  lt, 0, pg[0], bsize);
386 			NFT_PIPAPO_AVX2_LOAD(1, map[i_ul]);
387 			NFT_PIPAPO_AVX2_BUCKET_LOAD4(2,  lt, 1, pg[1], bsize);
388 			NFT_PIPAPO_AVX2_BUCKET_LOAD4(3,  lt, 2, pg[2], bsize);
389 			NFT_PIPAPO_AVX2_BUCKET_LOAD4(4,  lt, 3, pg[3], bsize);
390 
391 			NFT_PIPAPO_AVX2_NOMATCH_GOTO(1, nothing);
392 
393 			NFT_PIPAPO_AVX2_AND(5,   0,  1);
394 			NFT_PIPAPO_AVX2_BUCKET_LOAD4(6,  lt, 4, pg[4], bsize);
395 			NFT_PIPAPO_AVX2_BUCKET_LOAD4(7,  lt, 5, pg[5], bsize);
396 			NFT_PIPAPO_AVX2_AND(8,   2,  3);
397 			NFT_PIPAPO_AVX2_BUCKET_LOAD4(9,  lt, 6, pg[6], bsize);
398 			NFT_PIPAPO_AVX2_AND(10,  4,  5);
399 			NFT_PIPAPO_AVX2_BUCKET_LOAD4(11, lt, 7, pg[7], bsize);
400 			NFT_PIPAPO_AVX2_AND(12,  6,  7);
401 			NFT_PIPAPO_AVX2_AND(13,  8,  9);
402 			NFT_PIPAPO_AVX2_AND(14, 10, 11);
403 
404 			/* Stall */
405 			NFT_PIPAPO_AVX2_AND(1,  12, 13);
406 			NFT_PIPAPO_AVX2_AND(1,   1, 14);
407 		}
408 
409 		NFT_PIPAPO_AVX2_NOMATCH_GOTO(1, nomatch);
410 		NFT_PIPAPO_AVX2_STORE(map[i_ul], 1);
411 
412 		b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last);
413 		if (last)
414 			return b;
415 
416 		if (unlikely(ret == -1))
417 			ret = b / XSAVE_YMM_SIZE;
418 
419 		continue;
420 
421 nomatch:
422 		NFT_PIPAPO_AVX2_STORE(map[i_ul], 15);
423 nothing:
424 		;
425 	}
426 
427 	return ret;
428 }
429 
430 /**
431  * nft_pipapo_avx2_lookup_4b_12() - AVX2-based lookup for 12 four-bit groups
432  * @map:	Previous match result, used as initial bitmap
433  * @fill:	Destination bitmap to be filled with current match result
434  * @f:		Field, containing lookup and mapping tables
435  * @offset:	Ignore buckets before the given index, no bits are filled there
436  * @pkt:	Packet data, pointer to input nftables register
437  * @first:	If this is the first field, don't source previous result
438  * @last:	Last field: stop at the first match and return bit index
439  *
440  * See nft_pipapo_avx2_lookup_4b_2().
441  *
442  * This is used for 48-bit fields (i.e. MAC addresses/EUI-48).
443  *
444  * Return: -1 on no match, rule index of match if @last, otherwise first long
445  * word index to be checked next (i.e. first filled word).
446  */
nft_pipapo_avx2_lookup_4b_12(unsigned long * map,unsigned long * fill,struct nft_pipapo_field * f,int offset,const u8 * pkt,bool first,bool last)447 static int nft_pipapo_avx2_lookup_4b_12(unsigned long *map, unsigned long *fill,
448 				        struct nft_pipapo_field *f, int offset,
449 				        const u8 *pkt, bool first, bool last)
450 {
451 	u8 pg[12] = {  pkt[0] >> 4,  pkt[0] & 0xf,  pkt[1] >> 4,  pkt[1] & 0xf,
452 		       pkt[2] >> 4,  pkt[2] & 0xf,  pkt[3] >> 4,  pkt[3] & 0xf,
453 		       pkt[4] >> 4,  pkt[4] & 0xf,  pkt[5] >> 4,  pkt[5] & 0xf,
454 		    };
455 	int i, ret = -1, m256_size = f->bsize / NFT_PIPAPO_LONGS_PER_M256, b;
456 	unsigned long *lt = f->lt, bsize = f->bsize;
457 
458 	lt += offset * NFT_PIPAPO_LONGS_PER_M256;
459 	for (i = offset; i < m256_size; i++, lt += NFT_PIPAPO_LONGS_PER_M256) {
460 		int i_ul = i * NFT_PIPAPO_LONGS_PER_M256;
461 
462 		if (!first)
463 			NFT_PIPAPO_AVX2_LOAD(0, map[i_ul]);
464 
465 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(1,  lt,  0,  pg[0], bsize);
466 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(2,  lt,  1,  pg[1], bsize);
467 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(3,  lt,  2,  pg[2], bsize);
468 
469 		if (!first) {
470 			NFT_PIPAPO_AVX2_NOMATCH_GOTO(0, nothing);
471 			NFT_PIPAPO_AVX2_AND(1, 1, 0);
472 		}
473 
474 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(4,  lt,  3,  pg[3], bsize);
475 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(5,  lt,  4,  pg[4], bsize);
476 		NFT_PIPAPO_AVX2_AND(6,   2,  3);
477 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(7,  lt,  5,  pg[5], bsize);
478 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(8,  lt,  6,  pg[6], bsize);
479 		NFT_PIPAPO_AVX2_AND(9,   1,  4);
480 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(10, lt,  7,  pg[7], bsize);
481 		NFT_PIPAPO_AVX2_AND(11,  5,  6);
482 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(12, lt,  8,  pg[8], bsize);
483 		NFT_PIPAPO_AVX2_AND(13,  7,  8);
484 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(14, lt,  9,  pg[9], bsize);
485 
486 		NFT_PIPAPO_AVX2_AND(0,   9, 10);
487 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(1,  lt, 10,  pg[10], bsize);
488 		NFT_PIPAPO_AVX2_AND(2,  11, 12);
489 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(3,  lt, 11,  pg[11], bsize);
490 		NFT_PIPAPO_AVX2_AND(4,  13, 14);
491 		NFT_PIPAPO_AVX2_AND(5,   0,  1);
492 
493 		NFT_PIPAPO_AVX2_AND(6,   2,  3);
494 
495 		/* Stalls */
496 		NFT_PIPAPO_AVX2_AND(7,   4,  5);
497 		NFT_PIPAPO_AVX2_AND(8,   6,  7);
498 
499 		NFT_PIPAPO_AVX2_NOMATCH_GOTO(8, nomatch);
500 		NFT_PIPAPO_AVX2_STORE(map[i_ul], 8);
501 
502 		b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last);
503 		if (last)
504 			return b;
505 
506 		if (unlikely(ret == -1))
507 			ret = b / XSAVE_YMM_SIZE;
508 
509 		continue;
510 nomatch:
511 		NFT_PIPAPO_AVX2_STORE(map[i_ul], 15);
512 nothing:
513 		;
514 	}
515 
516 	return ret;
517 }
518 
519 /**
520  * nft_pipapo_avx2_lookup_4b_32() - AVX2-based lookup for 32 four-bit groups
521  * @map:	Previous match result, used as initial bitmap
522  * @fill:	Destination bitmap to be filled with current match result
523  * @f:		Field, containing lookup and mapping tables
524  * @offset:	Ignore buckets before the given index, no bits are filled there
525  * @pkt:	Packet data, pointer to input nftables register
526  * @first:	If this is the first field, don't source previous result
527  * @last:	Last field: stop at the first match and return bit index
528  *
529  * See nft_pipapo_avx2_lookup_4b_2().
530  *
531  * This is used for 128-bit fields (i.e. IPv6 addresses).
532  *
533  * Return: -1 on no match, rule index of match if @last, otherwise first long
534  * word index to be checked next (i.e. first filled word).
535  */
nft_pipapo_avx2_lookup_4b_32(unsigned long * map,unsigned long * fill,struct nft_pipapo_field * f,int offset,const u8 * pkt,bool first,bool last)536 static int nft_pipapo_avx2_lookup_4b_32(unsigned long *map, unsigned long *fill,
537 					struct nft_pipapo_field *f, int offset,
538 					const u8 *pkt, bool first, bool last)
539 {
540 	u8 pg[32] = {  pkt[0] >> 4,  pkt[0] & 0xf,  pkt[1] >> 4,  pkt[1] & 0xf,
541 		       pkt[2] >> 4,  pkt[2] & 0xf,  pkt[3] >> 4,  pkt[3] & 0xf,
542 		       pkt[4] >> 4,  pkt[4] & 0xf,  pkt[5] >> 4,  pkt[5] & 0xf,
543 		       pkt[6] >> 4,  pkt[6] & 0xf,  pkt[7] >> 4,  pkt[7] & 0xf,
544 		       pkt[8] >> 4,  pkt[8] & 0xf,  pkt[9] >> 4,  pkt[9] & 0xf,
545 		      pkt[10] >> 4, pkt[10] & 0xf, pkt[11] >> 4, pkt[11] & 0xf,
546 		      pkt[12] >> 4, pkt[12] & 0xf, pkt[13] >> 4, pkt[13] & 0xf,
547 		      pkt[14] >> 4, pkt[14] & 0xf, pkt[15] >> 4, pkt[15] & 0xf,
548 		    };
549 	int i, ret = -1, m256_size = f->bsize / NFT_PIPAPO_LONGS_PER_M256, b;
550 	unsigned long *lt = f->lt, bsize = f->bsize;
551 
552 	lt += offset * NFT_PIPAPO_LONGS_PER_M256;
553 	for (i = offset; i < m256_size; i++, lt += NFT_PIPAPO_LONGS_PER_M256) {
554 		int i_ul = i * NFT_PIPAPO_LONGS_PER_M256;
555 
556 		if (!first)
557 			NFT_PIPAPO_AVX2_LOAD(0, map[i_ul]);
558 
559 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(1,  lt,  0,  pg[0], bsize);
560 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(2,  lt,  1,  pg[1], bsize);
561 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(3,  lt,  2,  pg[2], bsize);
562 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(4,  lt,  3,  pg[3], bsize);
563 		if (!first) {
564 			NFT_PIPAPO_AVX2_NOMATCH_GOTO(0, nothing);
565 			NFT_PIPAPO_AVX2_AND(1, 1, 0);
566 		}
567 
568 		NFT_PIPAPO_AVX2_AND(5,   2,  3);
569 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(6,  lt,  4,  pg[4], bsize);
570 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(7,  lt,  5,  pg[5], bsize);
571 		NFT_PIPAPO_AVX2_AND(8,   1,  4);
572 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(9,  lt,  6,  pg[6], bsize);
573 		NFT_PIPAPO_AVX2_AND(10,  5,  6);
574 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(11, lt,  7,  pg[7], bsize);
575 		NFT_PIPAPO_AVX2_AND(12,  7,  8);
576 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(13, lt,  8,  pg[8], bsize);
577 		NFT_PIPAPO_AVX2_AND(14,  9, 10);
578 
579 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(0,  lt,  9,  pg[9], bsize);
580 		NFT_PIPAPO_AVX2_AND(1,  11, 12);
581 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(2,  lt, 10, pg[10], bsize);
582 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(3,  lt, 11, pg[11], bsize);
583 		NFT_PIPAPO_AVX2_AND(4,  13, 14);
584 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(5,  lt, 12, pg[12], bsize);
585 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(6,  lt, 13, pg[13], bsize);
586 		NFT_PIPAPO_AVX2_AND(7,   0,  1);
587 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(8,  lt, 14, pg[14], bsize);
588 		NFT_PIPAPO_AVX2_AND(9,   2,  3);
589 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(10, lt, 15, pg[15], bsize);
590 		NFT_PIPAPO_AVX2_AND(11,  4,  5);
591 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(12, lt, 16, pg[16], bsize);
592 		NFT_PIPAPO_AVX2_AND(13,  6,  7);
593 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(14, lt, 17, pg[17], bsize);
594 
595 		NFT_PIPAPO_AVX2_AND(0,   8,  9);
596 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(1,  lt, 18, pg[18], bsize);
597 		NFT_PIPAPO_AVX2_AND(2,  10, 11);
598 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(3,  lt, 19, pg[19], bsize);
599 		NFT_PIPAPO_AVX2_AND(4,  12, 13);
600 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(5,  lt, 20, pg[20], bsize);
601 		NFT_PIPAPO_AVX2_AND(6,  14,  0);
602 		NFT_PIPAPO_AVX2_AND(7,   1,  2);
603 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(8,  lt, 21, pg[21], bsize);
604 		NFT_PIPAPO_AVX2_AND(9,   3,  4);
605 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(10, lt, 22, pg[22], bsize);
606 		NFT_PIPAPO_AVX2_AND(11,  5,  6);
607 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(12, lt, 23, pg[23], bsize);
608 		NFT_PIPAPO_AVX2_AND(13,  7,  8);
609 
610 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(14, lt, 24, pg[24], bsize);
611 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(0,  lt, 25, pg[25], bsize);
612 		NFT_PIPAPO_AVX2_AND(1,   9, 10);
613 		NFT_PIPAPO_AVX2_AND(2,  11, 12);
614 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(3,  lt, 26, pg[26], bsize);
615 		NFT_PIPAPO_AVX2_AND(4,  13, 14);
616 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(5,  lt, 27, pg[27], bsize);
617 		NFT_PIPAPO_AVX2_AND(6,   0,  1);
618 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(7,  lt, 28, pg[28], bsize);
619 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(8,  lt, 29, pg[29], bsize);
620 		NFT_PIPAPO_AVX2_AND(9,   2,  3);
621 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(10, lt, 30, pg[30], bsize);
622 		NFT_PIPAPO_AVX2_AND(11,  4,  5);
623 		NFT_PIPAPO_AVX2_BUCKET_LOAD4(12, lt, 31, pg[31], bsize);
624 
625 		NFT_PIPAPO_AVX2_AND(0,   6,  7);
626 		NFT_PIPAPO_AVX2_AND(1,   8,  9);
627 		NFT_PIPAPO_AVX2_AND(2,  10, 11);
628 		NFT_PIPAPO_AVX2_AND(3,  12,  0);
629 
630 		/* Stalls */
631 		NFT_PIPAPO_AVX2_AND(4,   1,  2);
632 		NFT_PIPAPO_AVX2_AND(5,   3,  4);
633 
634 		NFT_PIPAPO_AVX2_NOMATCH_GOTO(5, nomatch);
635 		NFT_PIPAPO_AVX2_STORE(map[i_ul], 5);
636 
637 		b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last);
638 		if (last)
639 			return b;
640 
641 		if (unlikely(ret == -1))
642 			ret = b / XSAVE_YMM_SIZE;
643 
644 		continue;
645 nomatch:
646 		NFT_PIPAPO_AVX2_STORE(map[i_ul], 15);
647 nothing:
648 		;
649 	}
650 
651 	return ret;
652 }
653 
654 /**
655  * nft_pipapo_avx2_lookup_8b_1() - AVX2-based lookup for one eight-bit group
656  * @map:	Previous match result, used as initial bitmap
657  * @fill:	Destination bitmap to be filled with current match result
658  * @f:		Field, containing lookup and mapping tables
659  * @offset:	Ignore buckets before the given index, no bits are filled there
660  * @pkt:	Packet data, pointer to input nftables register
661  * @first:	If this is the first field, don't source previous result
662  * @last:	Last field: stop at the first match and return bit index
663  *
664  * See nft_pipapo_avx2_lookup_4b_2().
665  *
666  * This is used for 8-bit fields (i.e. protocol numbers).
667  *
668  * Return: -1 on no match, rule index of match if @last, otherwise first long
669  * word index to be checked next (i.e. first filled word).
670  */
nft_pipapo_avx2_lookup_8b_1(unsigned long * map,unsigned long * fill,struct nft_pipapo_field * f,int offset,const u8 * pkt,bool first,bool last)671 static int nft_pipapo_avx2_lookup_8b_1(unsigned long *map, unsigned long *fill,
672 				       struct nft_pipapo_field *f, int offset,
673 				       const u8 *pkt, bool first, bool last)
674 {
675 	int i, ret = -1, m256_size = f->bsize / NFT_PIPAPO_LONGS_PER_M256, b;
676 	unsigned long *lt = f->lt, bsize = f->bsize;
677 
678 	lt += offset * NFT_PIPAPO_LONGS_PER_M256;
679 	for (i = offset; i < m256_size; i++, lt += NFT_PIPAPO_LONGS_PER_M256) {
680 		int i_ul = i * NFT_PIPAPO_LONGS_PER_M256;
681 
682 		if (first) {
683 			NFT_PIPAPO_AVX2_BUCKET_LOAD8(2, lt, 0, pkt[0], bsize);
684 		} else {
685 			NFT_PIPAPO_AVX2_BUCKET_LOAD8(0, lt, 0, pkt[0], bsize);
686 			NFT_PIPAPO_AVX2_LOAD(1, map[i_ul]);
687 			NFT_PIPAPO_AVX2_AND(2, 0, 1);
688 			NFT_PIPAPO_AVX2_NOMATCH_GOTO(1, nothing);
689 		}
690 
691 		NFT_PIPAPO_AVX2_NOMATCH_GOTO(2, nomatch);
692 		NFT_PIPAPO_AVX2_STORE(map[i_ul], 2);
693 
694 		b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last);
695 		if (last)
696 			return b;
697 
698 		if (unlikely(ret == -1))
699 			ret = b / XSAVE_YMM_SIZE;
700 
701 		continue;
702 nomatch:
703 		NFT_PIPAPO_AVX2_STORE(map[i_ul], 15);
704 nothing:
705 		;
706 	}
707 
708 	return ret;
709 }
710 
711 /**
712  * nft_pipapo_avx2_lookup_8b_2() - AVX2-based lookup for 2 eight-bit groups
713  * @map:	Previous match result, used as initial bitmap
714  * @fill:	Destination bitmap to be filled with current match result
715  * @f:		Field, containing lookup and mapping tables
716  * @offset:	Ignore buckets before the given index, no bits are filled there
717  * @pkt:	Packet data, pointer to input nftables register
718  * @first:	If this is the first field, don't source previous result
719  * @last:	Last field: stop at the first match and return bit index
720  *
721  * See nft_pipapo_avx2_lookup_4b_2().
722  *
723  * This is used for 16-bit fields (i.e. ports).
724  *
725  * Return: -1 on no match, rule index of match if @last, otherwise first long
726  * word index to be checked next (i.e. first filled word).
727  */
nft_pipapo_avx2_lookup_8b_2(unsigned long * map,unsigned long * fill,struct nft_pipapo_field * f,int offset,const u8 * pkt,bool first,bool last)728 static int nft_pipapo_avx2_lookup_8b_2(unsigned long *map, unsigned long *fill,
729 				       struct nft_pipapo_field *f, int offset,
730 				       const u8 *pkt, bool first, bool last)
731 {
732 	int i, ret = -1, m256_size = f->bsize / NFT_PIPAPO_LONGS_PER_M256, b;
733 	unsigned long *lt = f->lt, bsize = f->bsize;
734 
735 	lt += offset * NFT_PIPAPO_LONGS_PER_M256;
736 	for (i = offset; i < m256_size; i++, lt += NFT_PIPAPO_LONGS_PER_M256) {
737 		int i_ul = i * NFT_PIPAPO_LONGS_PER_M256;
738 
739 		if (first) {
740 			NFT_PIPAPO_AVX2_BUCKET_LOAD8(0, lt, 0, pkt[0], bsize);
741 			NFT_PIPAPO_AVX2_BUCKET_LOAD8(1, lt, 1, pkt[1], bsize);
742 			NFT_PIPAPO_AVX2_AND(4, 0, 1);
743 		} else {
744 			NFT_PIPAPO_AVX2_LOAD(0, map[i_ul]);
745 			NFT_PIPAPO_AVX2_BUCKET_LOAD8(1, lt, 0, pkt[0], bsize);
746 			NFT_PIPAPO_AVX2_BUCKET_LOAD8(2, lt, 1, pkt[1], bsize);
747 
748 			/* Stall */
749 			NFT_PIPAPO_AVX2_AND(3, 0, 1);
750 			NFT_PIPAPO_AVX2_NOMATCH_GOTO(0, nothing);
751 			NFT_PIPAPO_AVX2_AND(4, 3, 2);
752 		}
753 
754 		/* Stall */
755 		NFT_PIPAPO_AVX2_NOMATCH_GOTO(4, nomatch);
756 		NFT_PIPAPO_AVX2_STORE(map[i_ul], 4);
757 
758 		b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last);
759 		if (last)
760 			return b;
761 
762 		if (unlikely(ret == -1))
763 			ret = b / XSAVE_YMM_SIZE;
764 
765 		continue;
766 nomatch:
767 		NFT_PIPAPO_AVX2_STORE(map[i_ul], 15);
768 nothing:
769 		;
770 	}
771 
772 	return ret;
773 }
774 
775 /**
776  * nft_pipapo_avx2_lookup_8b_4() - AVX2-based lookup for 4 eight-bit groups
777  * @map:	Previous match result, used as initial bitmap
778  * @fill:	Destination bitmap to be filled with current match result
779  * @f:		Field, containing lookup and mapping tables
780  * @offset:	Ignore buckets before the given index, no bits are filled there
781  * @pkt:	Packet data, pointer to input nftables register
782  * @first:	If this is the first field, don't source previous result
783  * @last:	Last field: stop at the first match and return bit index
784  *
785  * See nft_pipapo_avx2_lookup_4b_2().
786  *
787  * This is used for 32-bit fields (i.e. IPv4 addresses).
788  *
789  * Return: -1 on no match, rule index of match if @last, otherwise first long
790  * word index to be checked next (i.e. first filled word).
791  */
nft_pipapo_avx2_lookup_8b_4(unsigned long * map,unsigned long * fill,struct nft_pipapo_field * f,int offset,const u8 * pkt,bool first,bool last)792 static int nft_pipapo_avx2_lookup_8b_4(unsigned long *map, unsigned long *fill,
793 				       struct nft_pipapo_field *f, int offset,
794 				       const u8 *pkt, bool first, bool last)
795 {
796 	int i, ret = -1, m256_size = f->bsize / NFT_PIPAPO_LONGS_PER_M256, b;
797 	unsigned long *lt = f->lt, bsize = f->bsize;
798 
799 	lt += offset * NFT_PIPAPO_LONGS_PER_M256;
800 	for (i = offset; i < m256_size; i++, lt += NFT_PIPAPO_LONGS_PER_M256) {
801 		int i_ul = i * NFT_PIPAPO_LONGS_PER_M256;
802 
803 		if (first) {
804 			NFT_PIPAPO_AVX2_BUCKET_LOAD8(0,  lt, 0, pkt[0], bsize);
805 			NFT_PIPAPO_AVX2_BUCKET_LOAD8(1,  lt, 1, pkt[1], bsize);
806 			NFT_PIPAPO_AVX2_BUCKET_LOAD8(2,  lt, 2, pkt[2], bsize);
807 			NFT_PIPAPO_AVX2_BUCKET_LOAD8(3,  lt, 3, pkt[3], bsize);
808 
809 			/* Stall */
810 			NFT_PIPAPO_AVX2_AND(4, 0, 1);
811 			NFT_PIPAPO_AVX2_AND(5, 2, 3);
812 			NFT_PIPAPO_AVX2_AND(0, 4, 5);
813 		} else {
814 			NFT_PIPAPO_AVX2_BUCKET_LOAD8(0,  lt, 0, pkt[0], bsize);
815 			NFT_PIPAPO_AVX2_LOAD(1, map[i_ul]);
816 			NFT_PIPAPO_AVX2_BUCKET_LOAD8(2,  lt, 1, pkt[1], bsize);
817 			NFT_PIPAPO_AVX2_BUCKET_LOAD8(3,  lt, 2, pkt[2], bsize);
818 			NFT_PIPAPO_AVX2_BUCKET_LOAD8(4,  lt, 3, pkt[3], bsize);
819 
820 			NFT_PIPAPO_AVX2_AND(5, 0, 1);
821 			NFT_PIPAPO_AVX2_NOMATCH_GOTO(1, nothing);
822 			NFT_PIPAPO_AVX2_AND(6, 2, 3);
823 
824 			/* Stall */
825 			NFT_PIPAPO_AVX2_AND(7, 4, 5);
826 			NFT_PIPAPO_AVX2_AND(0, 6, 7);
827 		}
828 
829 		NFT_PIPAPO_AVX2_NOMATCH_GOTO(0, nomatch);
830 		NFT_PIPAPO_AVX2_STORE(map[i_ul], 0);
831 
832 		b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last);
833 		if (last)
834 			return b;
835 
836 		if (unlikely(ret == -1))
837 			ret = b / XSAVE_YMM_SIZE;
838 
839 		continue;
840 
841 nomatch:
842 		NFT_PIPAPO_AVX2_STORE(map[i_ul], 15);
843 nothing:
844 		;
845 	}
846 
847 	return ret;
848 }
849 
850 /**
851  * nft_pipapo_avx2_lookup_8b_6() - AVX2-based lookup for 6 eight-bit groups
852  * @map:	Previous match result, used as initial bitmap
853  * @fill:	Destination bitmap to be filled with current match result
854  * @f:		Field, containing lookup and mapping tables
855  * @offset:	Ignore buckets before the given index, no bits are filled there
856  * @pkt:	Packet data, pointer to input nftables register
857  * @first:	If this is the first field, don't source previous result
858  * @last:	Last field: stop at the first match and return bit index
859  *
860  * See nft_pipapo_avx2_lookup_4b_2().
861  *
862  * This is used for 48-bit fields (i.e. MAC addresses/EUI-48).
863  *
864  * Return: -1 on no match, rule index of match if @last, otherwise first long
865  * word index to be checked next (i.e. first filled word).
866  */
nft_pipapo_avx2_lookup_8b_6(unsigned long * map,unsigned long * fill,struct nft_pipapo_field * f,int offset,const u8 * pkt,bool first,bool last)867 static int nft_pipapo_avx2_lookup_8b_6(unsigned long *map, unsigned long *fill,
868 				       struct nft_pipapo_field *f, int offset,
869 				       const u8 *pkt, bool first, bool last)
870 {
871 	int i, ret = -1, m256_size = f->bsize / NFT_PIPAPO_LONGS_PER_M256, b;
872 	unsigned long *lt = f->lt, bsize = f->bsize;
873 
874 	lt += offset * NFT_PIPAPO_LONGS_PER_M256;
875 	for (i = offset; i < m256_size; i++, lt += NFT_PIPAPO_LONGS_PER_M256) {
876 		int i_ul = i * NFT_PIPAPO_LONGS_PER_M256;
877 
878 		if (first) {
879 			NFT_PIPAPO_AVX2_BUCKET_LOAD8(0,  lt, 0, pkt[0], bsize);
880 			NFT_PIPAPO_AVX2_BUCKET_LOAD8(1,  lt, 1, pkt[1], bsize);
881 			NFT_PIPAPO_AVX2_BUCKET_LOAD8(2,  lt, 2, pkt[2], bsize);
882 			NFT_PIPAPO_AVX2_BUCKET_LOAD8(3,  lt, 3, pkt[3], bsize);
883 			NFT_PIPAPO_AVX2_BUCKET_LOAD8(4,  lt, 4, pkt[4], bsize);
884 
885 			NFT_PIPAPO_AVX2_AND(5, 0, 1);
886 			NFT_PIPAPO_AVX2_BUCKET_LOAD8(6,  lt, 5, pkt[5], bsize);
887 			NFT_PIPAPO_AVX2_AND(7, 2, 3);
888 
889 			/* Stall */
890 			NFT_PIPAPO_AVX2_AND(0, 4, 5);
891 			NFT_PIPAPO_AVX2_AND(1, 6, 7);
892 			NFT_PIPAPO_AVX2_AND(4, 0, 1);
893 		} else {
894 			NFT_PIPAPO_AVX2_BUCKET_LOAD8(0,  lt, 0, pkt[0], bsize);
895 			NFT_PIPAPO_AVX2_LOAD(1, map[i_ul]);
896 			NFT_PIPAPO_AVX2_BUCKET_LOAD8(2,  lt, 1, pkt[1], bsize);
897 			NFT_PIPAPO_AVX2_BUCKET_LOAD8(3,  lt, 2, pkt[2], bsize);
898 			NFT_PIPAPO_AVX2_BUCKET_LOAD8(4,  lt, 3, pkt[3], bsize);
899 
900 			NFT_PIPAPO_AVX2_AND(5, 0, 1);
901 			NFT_PIPAPO_AVX2_NOMATCH_GOTO(1, nothing);
902 
903 			NFT_PIPAPO_AVX2_AND(6, 2, 3);
904 			NFT_PIPAPO_AVX2_BUCKET_LOAD8(7,  lt, 4, pkt[4], bsize);
905 			NFT_PIPAPO_AVX2_AND(0, 4, 5);
906 			NFT_PIPAPO_AVX2_BUCKET_LOAD8(1,  lt, 5, pkt[5], bsize);
907 			NFT_PIPAPO_AVX2_AND(2, 6, 7);
908 
909 			/* Stall */
910 			NFT_PIPAPO_AVX2_AND(3, 0, 1);
911 			NFT_PIPAPO_AVX2_AND(4, 2, 3);
912 		}
913 
914 		NFT_PIPAPO_AVX2_NOMATCH_GOTO(4, nomatch);
915 		NFT_PIPAPO_AVX2_STORE(map[i_ul], 4);
916 
917 		b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last);
918 		if (last)
919 			return b;
920 
921 		if (unlikely(ret == -1))
922 			ret = b / XSAVE_YMM_SIZE;
923 
924 		continue;
925 
926 nomatch:
927 		NFT_PIPAPO_AVX2_STORE(map[i_ul], 15);
928 nothing:
929 		;
930 	}
931 
932 	return ret;
933 }
934 
935 /**
936  * nft_pipapo_avx2_lookup_8b_16() - AVX2-based lookup for 16 eight-bit groups
937  * @map:	Previous match result, used as initial bitmap
938  * @fill:	Destination bitmap to be filled with current match result
939  * @f:		Field, containing lookup and mapping tables
940  * @offset:	Ignore buckets before the given index, no bits are filled there
941  * @pkt:	Packet data, pointer to input nftables register
942  * @first:	If this is the first field, don't source previous result
943  * @last:	Last field: stop at the first match and return bit index
944  *
945  * See nft_pipapo_avx2_lookup_4b_2().
946  *
947  * This is used for 128-bit fields (i.e. IPv6 addresses).
948  *
949  * Return: -1 on no match, rule index of match if @last, otherwise first long
950  * word index to be checked next (i.e. first filled word).
951  */
nft_pipapo_avx2_lookup_8b_16(unsigned long * map,unsigned long * fill,struct nft_pipapo_field * f,int offset,const u8 * pkt,bool first,bool last)952 static int nft_pipapo_avx2_lookup_8b_16(unsigned long *map, unsigned long *fill,
953 					struct nft_pipapo_field *f, int offset,
954 					const u8 *pkt, bool first, bool last)
955 {
956 	int i, ret = -1, m256_size = f->bsize / NFT_PIPAPO_LONGS_PER_M256, b;
957 	unsigned long *lt = f->lt, bsize = f->bsize;
958 
959 	lt += offset * NFT_PIPAPO_LONGS_PER_M256;
960 	for (i = offset; i < m256_size; i++, lt += NFT_PIPAPO_LONGS_PER_M256) {
961 		int i_ul = i * NFT_PIPAPO_LONGS_PER_M256;
962 
963 		if (!first)
964 			NFT_PIPAPO_AVX2_LOAD(0, map[i_ul]);
965 
966 		NFT_PIPAPO_AVX2_BUCKET_LOAD8(1, lt,  0,  pkt[0], bsize);
967 		NFT_PIPAPO_AVX2_BUCKET_LOAD8(2, lt,  1,  pkt[1], bsize);
968 		NFT_PIPAPO_AVX2_BUCKET_LOAD8(3, lt,  2,  pkt[2], bsize);
969 		if (!first) {
970 			NFT_PIPAPO_AVX2_NOMATCH_GOTO(0, nothing);
971 			NFT_PIPAPO_AVX2_AND(1, 1, 0);
972 		}
973 		NFT_PIPAPO_AVX2_BUCKET_LOAD8(4, lt,  3,  pkt[3], bsize);
974 
975 		NFT_PIPAPO_AVX2_BUCKET_LOAD8(5, lt,  4,  pkt[4], bsize);
976 		NFT_PIPAPO_AVX2_AND(6, 1, 2);
977 		NFT_PIPAPO_AVX2_BUCKET_LOAD8(7, lt,  5,  pkt[5], bsize);
978 		NFT_PIPAPO_AVX2_AND(0, 3, 4);
979 		NFT_PIPAPO_AVX2_BUCKET_LOAD8(1, lt,  6,  pkt[6], bsize);
980 
981 		NFT_PIPAPO_AVX2_BUCKET_LOAD8(2, lt,  7,  pkt[7], bsize);
982 		NFT_PIPAPO_AVX2_AND(3, 5, 6);
983 		NFT_PIPAPO_AVX2_AND(4, 0, 1);
984 		NFT_PIPAPO_AVX2_BUCKET_LOAD8(5, lt,  8,  pkt[8], bsize);
985 
986 		NFT_PIPAPO_AVX2_AND(6, 2, 3);
987 		NFT_PIPAPO_AVX2_BUCKET_LOAD8(7, lt,  9,  pkt[9], bsize);
988 		NFT_PIPAPO_AVX2_AND(0, 4, 5);
989 		NFT_PIPAPO_AVX2_BUCKET_LOAD8(1, lt, 10, pkt[10], bsize);
990 		NFT_PIPAPO_AVX2_AND(2, 6, 7);
991 		NFT_PIPAPO_AVX2_BUCKET_LOAD8(3, lt, 11, pkt[11], bsize);
992 		NFT_PIPAPO_AVX2_AND(4, 0, 1);
993 		NFT_PIPAPO_AVX2_BUCKET_LOAD8(5, lt, 12, pkt[12], bsize);
994 		NFT_PIPAPO_AVX2_AND(6, 2, 3);
995 		NFT_PIPAPO_AVX2_BUCKET_LOAD8(7, lt, 13, pkt[13], bsize);
996 		NFT_PIPAPO_AVX2_AND(0, 4, 5);
997 		NFT_PIPAPO_AVX2_BUCKET_LOAD8(1, lt, 14, pkt[14], bsize);
998 		NFT_PIPAPO_AVX2_AND(2, 6, 7);
999 		NFT_PIPAPO_AVX2_BUCKET_LOAD8(3, lt, 15, pkt[15], bsize);
1000 		NFT_PIPAPO_AVX2_AND(4, 0, 1);
1001 
1002 		/* Stall */
1003 		NFT_PIPAPO_AVX2_AND(5, 2, 3);
1004 		NFT_PIPAPO_AVX2_AND(6, 4, 5);
1005 
1006 		NFT_PIPAPO_AVX2_NOMATCH_GOTO(6, nomatch);
1007 		NFT_PIPAPO_AVX2_STORE(map[i_ul], 6);
1008 
1009 		b = nft_pipapo_avx2_refill(i_ul, &map[i_ul], fill, f->mt, last);
1010 		if (last)
1011 			return b;
1012 
1013 		if (unlikely(ret == -1))
1014 			ret = b / XSAVE_YMM_SIZE;
1015 
1016 		continue;
1017 
1018 nomatch:
1019 		NFT_PIPAPO_AVX2_STORE(map[i_ul], 15);
1020 nothing:
1021 		;
1022 	}
1023 
1024 	return ret;
1025 }
1026 
1027 /**
1028  * nft_pipapo_avx2_lookup_slow() - Fallback function for uncommon field sizes
1029  * @map:	Previous match result, used as initial bitmap
1030  * @fill:	Destination bitmap to be filled with current match result
1031  * @f:		Field, containing lookup and mapping tables
1032  * @offset:	Ignore buckets before the given index, no bits are filled there
1033  * @pkt:	Packet data, pointer to input nftables register
1034  * @first:	If this is the first field, don't source previous result
1035  * @last:	Last field: stop at the first match and return bit index
1036  *
1037  * This function should never be called, but is provided for the case the field
1038  * size doesn't match any of the known data types. Matching rate is
1039  * substantially lower than AVX2 routines.
1040  *
1041  * Return: -1 on no match, rule index of match if @last, otherwise first long
1042  * word index to be checked next (i.e. first filled word).
1043  */
nft_pipapo_avx2_lookup_slow(unsigned long * map,unsigned long * fill,struct nft_pipapo_field * f,int offset,const u8 * pkt,bool first,bool last)1044 static int nft_pipapo_avx2_lookup_slow(unsigned long *map, unsigned long *fill,
1045 					struct nft_pipapo_field *f, int offset,
1046 					const u8 *pkt, bool first, bool last)
1047 {
1048 	unsigned long bsize = f->bsize;
1049 	int i, ret = -1, b;
1050 
1051 	if (first)
1052 		memset(map, 0xff, bsize * sizeof(*map));
1053 
1054 	for (i = offset; i < bsize; i++) {
1055 		if (f->bb == 8)
1056 			pipapo_and_field_buckets_8bit(f, map, pkt);
1057 		else
1058 			pipapo_and_field_buckets_4bit(f, map, pkt);
1059 		NFT_PIPAPO_GROUP_BITS_ARE_8_OR_4;
1060 
1061 		b = pipapo_refill(map, bsize, f->rules, fill, f->mt, last);
1062 
1063 		if (last)
1064 			return b;
1065 
1066 		if (ret == -1)
1067 			ret = b / XSAVE_YMM_SIZE;
1068 	}
1069 
1070 	return ret;
1071 }
1072 
1073 /**
1074  * nft_pipapo_avx2_estimate() - Set size, space and lookup complexity
1075  * @desc:	Set description, element count and field description used
1076  * @features:	Flags: NFT_SET_INTERVAL needs to be there
1077  * @est:	Storage for estimation data
1078  *
1079  * Return: true if set is compatible and AVX2 available, false otherwise.
1080  */
nft_pipapo_avx2_estimate(const struct nft_set_desc * desc,u32 features,struct nft_set_estimate * est)1081 bool nft_pipapo_avx2_estimate(const struct nft_set_desc *desc, u32 features,
1082 			      struct nft_set_estimate *est)
1083 {
1084 	if (!(features & NFT_SET_INTERVAL) ||
1085 	    desc->field_count < NFT_PIPAPO_MIN_FIELDS)
1086 		return false;
1087 
1088 	if (!boot_cpu_has(X86_FEATURE_AVX2) || !boot_cpu_has(X86_FEATURE_AVX))
1089 		return false;
1090 
1091 	est->size = pipapo_estimate_size(desc);
1092 	if (!est->size)
1093 		return false;
1094 
1095 	est->lookup = NFT_SET_CLASS_O_LOG_N;
1096 
1097 	est->space = NFT_SET_CLASS_O_N;
1098 
1099 	return true;
1100 }
1101 
1102 /**
1103  * nft_pipapo_avx2_lookup() - Lookup function for AVX2 implementation
1104  * @net:	Network namespace
1105  * @set:	nftables API set representation
1106  * @key:	nftables API element representation containing key data
1107  * @ext:	nftables API extension pointer, filled with matching reference
1108  *
1109  * For more details, see DOC: Theory of Operation in nft_set_pipapo.c.
1110  *
1111  * This implementation exploits the repetitive characteristic of the algorithm
1112  * to provide a fast, vectorised version using the AVX2 SIMD instruction set.
1113  *
1114  * Return: true on match, false otherwise.
1115  */
nft_pipapo_avx2_lookup(const struct net * net,const struct nft_set * set,const u32 * key,const struct nft_set_ext ** ext)1116 bool nft_pipapo_avx2_lookup(const struct net *net, const struct nft_set *set,
1117 			    const u32 *key, const struct nft_set_ext **ext)
1118 {
1119 	struct nft_pipapo *priv = nft_set_priv(set);
1120 	struct nft_pipapo_scratch *scratch;
1121 	u8 genmask = nft_genmask_cur(net);
1122 	const u8 *rp = (const u8 *)key;
1123 	struct nft_pipapo_match *m;
1124 	struct nft_pipapo_field *f;
1125 	unsigned long *res, *fill;
1126 	bool map_index;
1127 	int i, ret = 0;
1128 
1129 	if (unlikely(!irq_fpu_usable()))
1130 		return nft_pipapo_lookup(net, set, key, ext);
1131 
1132 	m = rcu_dereference(priv->match);
1133 
1134 	/* This also protects access to all data related to scratch maps.
1135 	 *
1136 	 * Note that we don't need a valid MXCSR state for any of the
1137 	 * operations we use here, so pass 0 as mask and spare a LDMXCSR
1138 	 * instruction.
1139 	 */
1140 	kernel_fpu_begin_mask(0);
1141 
1142 	scratch = *raw_cpu_ptr(m->scratch);
1143 	if (unlikely(!scratch)) {
1144 		kernel_fpu_end();
1145 		return false;
1146 	}
1147 
1148 	map_index = scratch->map_index;
1149 
1150 	res  = scratch->map + (map_index ? m->bsize_max : 0);
1151 	fill = scratch->map + (map_index ? 0 : m->bsize_max);
1152 
1153 	/* Starting map doesn't need to be set for this implementation */
1154 
1155 	nft_pipapo_avx2_prepare();
1156 
1157 next_match:
1158 	nft_pipapo_for_each_field(f, i, m) {
1159 		bool last = i == m->field_count - 1, first = !i;
1160 
1161 #define NFT_SET_PIPAPO_AVX2_LOOKUP(b, n)				\
1162 		(ret = nft_pipapo_avx2_lookup_##b##b_##n(res, fill, f,	\
1163 							 ret, rp,	\
1164 							 first, last))
1165 
1166 		if (likely(f->bb == 8)) {
1167 			if (f->groups == 1) {
1168 				NFT_SET_PIPAPO_AVX2_LOOKUP(8, 1);
1169 			} else if (f->groups == 2) {
1170 				NFT_SET_PIPAPO_AVX2_LOOKUP(8, 2);
1171 			} else if (f->groups == 4) {
1172 				NFT_SET_PIPAPO_AVX2_LOOKUP(8, 4);
1173 			} else if (f->groups == 6) {
1174 				NFT_SET_PIPAPO_AVX2_LOOKUP(8, 6);
1175 			} else if (f->groups == 16) {
1176 				NFT_SET_PIPAPO_AVX2_LOOKUP(8, 16);
1177 			} else {
1178 				ret = nft_pipapo_avx2_lookup_slow(res, fill, f,
1179 								  ret, rp,
1180 								  first, last);
1181 			}
1182 		} else {
1183 			if (f->groups == 2) {
1184 				NFT_SET_PIPAPO_AVX2_LOOKUP(4, 2);
1185 			} else if (f->groups == 4) {
1186 				NFT_SET_PIPAPO_AVX2_LOOKUP(4, 4);
1187 			} else if (f->groups == 8) {
1188 				NFT_SET_PIPAPO_AVX2_LOOKUP(4, 8);
1189 			} else if (f->groups == 12) {
1190 				NFT_SET_PIPAPO_AVX2_LOOKUP(4, 12);
1191 			} else if (f->groups == 32) {
1192 				NFT_SET_PIPAPO_AVX2_LOOKUP(4, 32);
1193 			} else {
1194 				ret = nft_pipapo_avx2_lookup_slow(res, fill, f,
1195 								  ret, rp,
1196 								  first, last);
1197 			}
1198 		}
1199 		NFT_PIPAPO_GROUP_BITS_ARE_8_OR_4;
1200 
1201 #undef NFT_SET_PIPAPO_AVX2_LOOKUP
1202 
1203 		if (ret < 0)
1204 			goto out;
1205 
1206 		if (last) {
1207 			*ext = &f->mt[ret].e->ext;
1208 			if (unlikely(nft_set_elem_expired(*ext) ||
1209 				     !nft_set_elem_active(*ext, genmask))) {
1210 				ret = 0;
1211 				goto next_match;
1212 			}
1213 
1214 			goto out;
1215 		}
1216 
1217 		swap(res, fill);
1218 		rp += NFT_PIPAPO_GROUPS_PADDED_SIZE(f);
1219 	}
1220 
1221 out:
1222 	if (i % 2)
1223 		scratch->map_index = !map_index;
1224 	kernel_fpu_end();
1225 
1226 	return ret >= 0;
1227 }
1228