1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Quick & dirty crypto testing module.
4  *
5  * This will only exist until we have a better testing mechanism
6  * (e.g. a char device).
7  *
8  * Copyright (c) 2002 James Morris <jmorris@intercode.com.au>
9  * Copyright (c) 2002 Jean-Francois Dive <jef@linuxbe.org>
10  * Copyright (c) 2007 Nokia Siemens Networks
11  *
12  * Updated RFC4106 AES-GCM testing.
13  *    Authors: Aidan O'Mahony (aidan.o.mahony@intel.com)
14  *             Adrian Hoban <adrian.hoban@intel.com>
15  *             Gabriele Paoloni <gabriele.paoloni@intel.com>
16  *             Tadeusz Struk (tadeusz.struk@intel.com)
17  *             Copyright (c) 2010, Intel Corporation.
18  */
19 
20 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
21 
22 #include <crypto/aead.h>
23 #include <crypto/hash.h>
24 #include <crypto/skcipher.h>
25 #include <linux/err.h>
26 #include <linux/fips.h>
27 #include <linux/init.h>
28 #include <linux/interrupt.h>
29 #include <linux/jiffies.h>
30 #include <linux/kernel.h>
31 #include <linux/module.h>
32 #include <linux/moduleparam.h>
33 #include <linux/scatterlist.h>
34 #include <linux/slab.h>
35 #include <linux/string.h>
36 #include <linux/timex.h>
37 
38 #include "internal.h"
39 #include "tcrypt.h"
40 
41 /*
42  * Need slab memory for testing (size in number of pages).
43  */
44 #define TVMEMSIZE	4
45 
46 /*
47 * Used by test_cipher_speed()
48 */
49 #define ENCRYPT 1
50 #define DECRYPT 0
51 
52 #define MAX_DIGEST_SIZE		64
53 
54 /*
55  * return a string with the driver name
56  */
57 #define get_driver_name(tfm_type, tfm) crypto_tfm_alg_driver_name(tfm_type ## _tfm(tfm))
58 
59 /*
60  * Used by test_cipher_speed()
61  */
62 static unsigned int sec;
63 
64 static char *alg;
65 static u32 type;
66 static u32 mask;
67 static int mode;
68 static u32 num_mb = 8;
69 static unsigned int klen;
70 static char *tvmem[TVMEMSIZE];
71 
72 static const int block_sizes[] = { 16, 64, 128, 256, 1024, 1420, 4096, 0 };
73 static const int aead_sizes[] = { 16, 64, 256, 512, 1024, 1420, 4096, 8192, 0 };
74 
75 #define XBUFSIZE 8
76 #define MAX_IVLEN 32
77 
testmgr_alloc_buf(char * buf[XBUFSIZE])78 static int testmgr_alloc_buf(char *buf[XBUFSIZE])
79 {
80 	int i;
81 
82 	for (i = 0; i < XBUFSIZE; i++) {
83 		buf[i] = (void *)__get_free_page(GFP_KERNEL);
84 		if (!buf[i])
85 			goto err_free_buf;
86 	}
87 
88 	return 0;
89 
90 err_free_buf:
91 	while (i-- > 0)
92 		free_page((unsigned long)buf[i]);
93 
94 	return -ENOMEM;
95 }
96 
testmgr_free_buf(char * buf[XBUFSIZE])97 static void testmgr_free_buf(char *buf[XBUFSIZE])
98 {
99 	int i;
100 
101 	for (i = 0; i < XBUFSIZE; i++)
102 		free_page((unsigned long)buf[i]);
103 }
104 
sg_init_aead(struct scatterlist * sg,char * xbuf[XBUFSIZE],unsigned int buflen,const void * assoc,unsigned int aad_size)105 static void sg_init_aead(struct scatterlist *sg, char *xbuf[XBUFSIZE],
106 			 unsigned int buflen, const void *assoc,
107 			 unsigned int aad_size)
108 {
109 	int np = (buflen + PAGE_SIZE - 1)/PAGE_SIZE;
110 	int k, rem;
111 
112 	if (np > XBUFSIZE) {
113 		rem = PAGE_SIZE;
114 		np = XBUFSIZE;
115 	} else {
116 		rem = buflen % PAGE_SIZE;
117 	}
118 
119 	sg_init_table(sg, np + 1);
120 
121 	sg_set_buf(&sg[0], assoc, aad_size);
122 
123 	if (rem)
124 		np--;
125 	for (k = 0; k < np; k++)
126 		sg_set_buf(&sg[k + 1], xbuf[k], PAGE_SIZE);
127 
128 	if (rem)
129 		sg_set_buf(&sg[k + 1], xbuf[k], rem);
130 }
131 
do_one_aead_op(struct aead_request * req,int ret)132 static inline int do_one_aead_op(struct aead_request *req, int ret)
133 {
134 	struct crypto_wait *wait = req->base.data;
135 
136 	return crypto_wait_req(ret, wait);
137 }
138 
139 struct test_mb_aead_data {
140 	struct scatterlist sg[XBUFSIZE];
141 	struct scatterlist sgout[XBUFSIZE];
142 	struct aead_request *req;
143 	struct crypto_wait wait;
144 	char *xbuf[XBUFSIZE];
145 	char *xoutbuf[XBUFSIZE];
146 	char *axbuf[XBUFSIZE];
147 };
148 
do_mult_aead_op(struct test_mb_aead_data * data,int enc,u32 num_mb,int * rc)149 static int do_mult_aead_op(struct test_mb_aead_data *data, int enc,
150 				u32 num_mb, int *rc)
151 {
152 	int i, err = 0;
153 
154 	/* Fire up a bunch of concurrent requests */
155 	for (i = 0; i < num_mb; i++) {
156 		if (enc == ENCRYPT)
157 			rc[i] = crypto_aead_encrypt(data[i].req);
158 		else
159 			rc[i] = crypto_aead_decrypt(data[i].req);
160 	}
161 
162 	/* Wait for all requests to finish */
163 	for (i = 0; i < num_mb; i++) {
164 		rc[i] = crypto_wait_req(rc[i], &data[i].wait);
165 
166 		if (rc[i]) {
167 			pr_info("concurrent request %d error %d\n", i, rc[i]);
168 			err = rc[i];
169 		}
170 	}
171 
172 	return err;
173 }
174 
test_mb_aead_jiffies(struct test_mb_aead_data * data,int enc,int blen,int secs,u32 num_mb)175 static int test_mb_aead_jiffies(struct test_mb_aead_data *data, int enc,
176 				int blen, int secs, u32 num_mb)
177 {
178 	unsigned long start, end;
179 	int bcount;
180 	int ret = 0;
181 	int *rc;
182 
183 	rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL);
184 	if (!rc)
185 		return -ENOMEM;
186 
187 	for (start = jiffies, end = start + secs * HZ, bcount = 0;
188 	     time_before(jiffies, end); bcount++) {
189 		ret = do_mult_aead_op(data, enc, num_mb, rc);
190 		if (ret)
191 			goto out;
192 	}
193 
194 	pr_cont("%d operations in %d seconds (%llu bytes)\n",
195 		bcount * num_mb, secs, (u64)bcount * blen * num_mb);
196 
197 out:
198 	kfree(rc);
199 	return ret;
200 }
201 
test_mb_aead_cycles(struct test_mb_aead_data * data,int enc,int blen,u32 num_mb)202 static int test_mb_aead_cycles(struct test_mb_aead_data *data, int enc,
203 			       int blen, u32 num_mb)
204 {
205 	unsigned long cycles = 0;
206 	int ret = 0;
207 	int i;
208 	int *rc;
209 
210 	rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL);
211 	if (!rc)
212 		return -ENOMEM;
213 
214 	/* Warm-up run. */
215 	for (i = 0; i < 4; i++) {
216 		ret = do_mult_aead_op(data, enc, num_mb, rc);
217 		if (ret)
218 			goto out;
219 	}
220 
221 	/* The real thing. */
222 	for (i = 0; i < 8; i++) {
223 		cycles_t start, end;
224 
225 		start = get_cycles();
226 		ret = do_mult_aead_op(data, enc, num_mb, rc);
227 		end = get_cycles();
228 
229 		if (ret)
230 			goto out;
231 
232 		cycles += end - start;
233 	}
234 
235 	pr_cont("1 operation in %lu cycles (%d bytes)\n",
236 		(cycles + 4) / (8 * num_mb), blen);
237 
238 out:
239 	kfree(rc);
240 	return ret;
241 }
242 
test_mb_aead_speed(const char * algo,int enc,int secs,struct aead_speed_template * template,unsigned int tcount,u8 authsize,unsigned int aad_size,u8 * keysize,u32 num_mb)243 static void test_mb_aead_speed(const char *algo, int enc, int secs,
244 			       struct aead_speed_template *template,
245 			       unsigned int tcount, u8 authsize,
246 			       unsigned int aad_size, u8 *keysize, u32 num_mb)
247 {
248 	struct test_mb_aead_data *data;
249 	struct crypto_aead *tfm;
250 	unsigned int i, j, iv_len;
251 	const int *b_size;
252 	const char *key;
253 	const char *e;
254 	void *assoc;
255 	char *iv;
256 	int ret;
257 
258 
259 	if (aad_size >= PAGE_SIZE) {
260 		pr_err("associate data length (%u) too big\n", aad_size);
261 		return;
262 	}
263 
264 	iv = kzalloc(MAX_IVLEN, GFP_KERNEL);
265 	if (!iv)
266 		return;
267 
268 	if (enc == ENCRYPT)
269 		e = "encryption";
270 	else
271 		e = "decryption";
272 
273 	data = kcalloc(num_mb, sizeof(*data), GFP_KERNEL);
274 	if (!data)
275 		goto out_free_iv;
276 
277 	tfm = crypto_alloc_aead(algo, 0, 0);
278 	if (IS_ERR(tfm)) {
279 		pr_err("failed to load transform for %s: %ld\n",
280 			algo, PTR_ERR(tfm));
281 		goto out_free_data;
282 	}
283 
284 	ret = crypto_aead_setauthsize(tfm, authsize);
285 	if (ret) {
286 		pr_err("alg: aead: Failed to setauthsize for %s: %d\n", algo,
287 		       ret);
288 		goto out_free_tfm;
289 	}
290 
291 	for (i = 0; i < num_mb; ++i)
292 		if (testmgr_alloc_buf(data[i].xbuf)) {
293 			while (i--)
294 				testmgr_free_buf(data[i].xbuf);
295 			goto out_free_tfm;
296 		}
297 
298 	for (i = 0; i < num_mb; ++i)
299 		if (testmgr_alloc_buf(data[i].axbuf)) {
300 			while (i--)
301 				testmgr_free_buf(data[i].axbuf);
302 			goto out_free_xbuf;
303 		}
304 
305 	for (i = 0; i < num_mb; ++i)
306 		if (testmgr_alloc_buf(data[i].xoutbuf)) {
307 			while (i--)
308 				testmgr_free_buf(data[i].xoutbuf);
309 			goto out_free_axbuf;
310 		}
311 
312 	for (i = 0; i < num_mb; ++i) {
313 		data[i].req = aead_request_alloc(tfm, GFP_KERNEL);
314 		if (!data[i].req) {
315 			pr_err("alg: aead: Failed to allocate request for %s\n",
316 			       algo);
317 			while (i--)
318 				aead_request_free(data[i].req);
319 			goto out_free_xoutbuf;
320 		}
321 	}
322 
323 	for (i = 0; i < num_mb; ++i) {
324 		crypto_init_wait(&data[i].wait);
325 		aead_request_set_callback(data[i].req,
326 					  CRYPTO_TFM_REQ_MAY_BACKLOG,
327 					  crypto_req_done, &data[i].wait);
328 	}
329 
330 	pr_info("testing speed of multibuffer %s (%s) %s\n", algo,
331 		get_driver_name(crypto_aead, tfm), e);
332 
333 	i = 0;
334 	do {
335 		b_size = aead_sizes;
336 		do {
337 			int bs = round_up(*b_size, crypto_aead_blocksize(tfm));
338 
339 			if (bs + authsize > XBUFSIZE * PAGE_SIZE) {
340 				pr_err("template (%u) too big for buffer (%lu)\n",
341 				       authsize + bs,
342 				       XBUFSIZE * PAGE_SIZE);
343 				goto out;
344 			}
345 
346 			pr_info("test %u (%d bit key, %d byte blocks): ", i,
347 				*keysize * 8, bs);
348 
349 			/* Set up tfm global state, i.e. the key */
350 
351 			memset(tvmem[0], 0xff, PAGE_SIZE);
352 			key = tvmem[0];
353 			for (j = 0; j < tcount; j++) {
354 				if (template[j].klen == *keysize) {
355 					key = template[j].key;
356 					break;
357 				}
358 			}
359 
360 			crypto_aead_clear_flags(tfm, ~0);
361 
362 			ret = crypto_aead_setkey(tfm, key, *keysize);
363 			if (ret) {
364 				pr_err("setkey() failed flags=%x\n",
365 				       crypto_aead_get_flags(tfm));
366 				goto out;
367 			}
368 
369 			iv_len = crypto_aead_ivsize(tfm);
370 			if (iv_len)
371 				memset(iv, 0xff, iv_len);
372 
373 			/* Now setup per request stuff, i.e. buffers */
374 
375 			for (j = 0; j < num_mb; ++j) {
376 				struct test_mb_aead_data *cur = &data[j];
377 
378 				assoc = cur->axbuf[0];
379 				memset(assoc, 0xff, aad_size);
380 
381 				sg_init_aead(cur->sg, cur->xbuf,
382 					     bs + (enc ? 0 : authsize),
383 					     assoc, aad_size);
384 
385 				sg_init_aead(cur->sgout, cur->xoutbuf,
386 					     bs + (enc ? authsize : 0),
387 					     assoc, aad_size);
388 
389 				aead_request_set_ad(cur->req, aad_size);
390 
391 				if (!enc) {
392 
393 					aead_request_set_crypt(cur->req,
394 							       cur->sgout,
395 							       cur->sg,
396 							       bs, iv);
397 					ret = crypto_aead_encrypt(cur->req);
398 					ret = do_one_aead_op(cur->req, ret);
399 
400 					if (ret) {
401 						pr_err("calculating auth failed (%d)\n",
402 						       ret);
403 						break;
404 					}
405 				}
406 
407 				aead_request_set_crypt(cur->req, cur->sg,
408 						       cur->sgout, bs +
409 						       (enc ? 0 : authsize),
410 						       iv);
411 
412 			}
413 
414 			if (secs) {
415 				ret = test_mb_aead_jiffies(data, enc, bs,
416 							   secs, num_mb);
417 				cond_resched();
418 			} else {
419 				ret = test_mb_aead_cycles(data, enc, bs,
420 							  num_mb);
421 			}
422 
423 			if (ret) {
424 				pr_err("%s() failed return code=%d\n", e, ret);
425 				break;
426 			}
427 			b_size++;
428 			i++;
429 		} while (*b_size);
430 		keysize++;
431 	} while (*keysize);
432 
433 out:
434 	for (i = 0; i < num_mb; ++i)
435 		aead_request_free(data[i].req);
436 out_free_xoutbuf:
437 	for (i = 0; i < num_mb; ++i)
438 		testmgr_free_buf(data[i].xoutbuf);
439 out_free_axbuf:
440 	for (i = 0; i < num_mb; ++i)
441 		testmgr_free_buf(data[i].axbuf);
442 out_free_xbuf:
443 	for (i = 0; i < num_mb; ++i)
444 		testmgr_free_buf(data[i].xbuf);
445 out_free_tfm:
446 	crypto_free_aead(tfm);
447 out_free_data:
448 	kfree(data);
449 out_free_iv:
450 	kfree(iv);
451 }
452 
test_aead_jiffies(struct aead_request * req,int enc,int blen,int secs)453 static int test_aead_jiffies(struct aead_request *req, int enc,
454 				int blen, int secs)
455 {
456 	unsigned long start, end;
457 	int bcount;
458 	int ret;
459 
460 	for (start = jiffies, end = start + secs * HZ, bcount = 0;
461 	     time_before(jiffies, end); bcount++) {
462 		if (enc)
463 			ret = do_one_aead_op(req, crypto_aead_encrypt(req));
464 		else
465 			ret = do_one_aead_op(req, crypto_aead_decrypt(req));
466 
467 		if (ret)
468 			return ret;
469 	}
470 
471 	pr_cont("%d operations in %d seconds (%llu bytes)\n",
472 	        bcount, secs, (u64)bcount * blen);
473 	return 0;
474 }
475 
test_aead_cycles(struct aead_request * req,int enc,int blen)476 static int test_aead_cycles(struct aead_request *req, int enc, int blen)
477 {
478 	unsigned long cycles = 0;
479 	int ret = 0;
480 	int i;
481 
482 	/* Warm-up run. */
483 	for (i = 0; i < 4; i++) {
484 		if (enc)
485 			ret = do_one_aead_op(req, crypto_aead_encrypt(req));
486 		else
487 			ret = do_one_aead_op(req, crypto_aead_decrypt(req));
488 
489 		if (ret)
490 			goto out;
491 	}
492 
493 	/* The real thing. */
494 	for (i = 0; i < 8; i++) {
495 		cycles_t start, end;
496 
497 		start = get_cycles();
498 		if (enc)
499 			ret = do_one_aead_op(req, crypto_aead_encrypt(req));
500 		else
501 			ret = do_one_aead_op(req, crypto_aead_decrypt(req));
502 		end = get_cycles();
503 
504 		if (ret)
505 			goto out;
506 
507 		cycles += end - start;
508 	}
509 
510 out:
511 	if (ret == 0)
512 		pr_cont("1 operation in %lu cycles (%d bytes)\n",
513 			(cycles + 4) / 8, blen);
514 
515 	return ret;
516 }
517 
test_aead_speed(const char * algo,int enc,unsigned int secs,struct aead_speed_template * template,unsigned int tcount,u8 authsize,unsigned int aad_size,u8 * keysize)518 static void test_aead_speed(const char *algo, int enc, unsigned int secs,
519 			    struct aead_speed_template *template,
520 			    unsigned int tcount, u8 authsize,
521 			    unsigned int aad_size, u8 *keysize)
522 {
523 	unsigned int i, j;
524 	struct crypto_aead *tfm;
525 	int ret = -ENOMEM;
526 	const char *key;
527 	struct aead_request *req;
528 	struct scatterlist *sg;
529 	struct scatterlist *sgout;
530 	const char *e;
531 	void *assoc;
532 	char *iv;
533 	char *xbuf[XBUFSIZE];
534 	char *xoutbuf[XBUFSIZE];
535 	char *axbuf[XBUFSIZE];
536 	const int *b_size;
537 	unsigned int iv_len;
538 	struct crypto_wait wait;
539 
540 	iv = kzalloc(MAX_IVLEN, GFP_KERNEL);
541 	if (!iv)
542 		return;
543 
544 	if (aad_size >= PAGE_SIZE) {
545 		pr_err("associate data length (%u) too big\n", aad_size);
546 		goto out_noxbuf;
547 	}
548 
549 	if (enc == ENCRYPT)
550 		e = "encryption";
551 	else
552 		e = "decryption";
553 
554 	if (testmgr_alloc_buf(xbuf))
555 		goto out_noxbuf;
556 	if (testmgr_alloc_buf(axbuf))
557 		goto out_noaxbuf;
558 	if (testmgr_alloc_buf(xoutbuf))
559 		goto out_nooutbuf;
560 
561 	sg = kmalloc(sizeof(*sg) * 9 * 2, GFP_KERNEL);
562 	if (!sg)
563 		goto out_nosg;
564 	sgout = &sg[9];
565 
566 	tfm = crypto_alloc_aead(algo, 0, 0);
567 	if (IS_ERR(tfm)) {
568 		pr_err("alg: aead: Failed to load transform for %s: %ld\n", algo,
569 		       PTR_ERR(tfm));
570 		goto out_notfm;
571 	}
572 
573 	ret = crypto_aead_setauthsize(tfm, authsize);
574 	if (ret) {
575 		pr_err("alg: aead: Failed to setauthsize for %s: %d\n", algo,
576 		       ret);
577 		goto out_noreq;
578 	}
579 
580 	crypto_init_wait(&wait);
581 	pr_info("testing speed of %s (%s) %s\n", algo,
582 		get_driver_name(crypto_aead, tfm), e);
583 
584 	req = aead_request_alloc(tfm, GFP_KERNEL);
585 	if (!req) {
586 		pr_err("alg: aead: Failed to allocate request for %s\n",
587 		       algo);
588 		goto out_noreq;
589 	}
590 
591 	aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
592 				  crypto_req_done, &wait);
593 
594 	i = 0;
595 	do {
596 		b_size = aead_sizes;
597 		do {
598 			u32 bs = round_up(*b_size, crypto_aead_blocksize(tfm));
599 
600 			assoc = axbuf[0];
601 			memset(assoc, 0xff, aad_size);
602 
603 			if ((*keysize + bs) > TVMEMSIZE * PAGE_SIZE) {
604 				pr_err("template (%u) too big for tvmem (%lu)\n",
605 				       *keysize + bs,
606 					TVMEMSIZE * PAGE_SIZE);
607 				goto out;
608 			}
609 
610 			key = tvmem[0];
611 			for (j = 0; j < tcount; j++) {
612 				if (template[j].klen == *keysize) {
613 					key = template[j].key;
614 					break;
615 				}
616 			}
617 
618 			ret = crypto_aead_setkey(tfm, key, *keysize);
619 			if (ret) {
620 				pr_err("setkey() failed flags=%x: %d\n",
621 					crypto_aead_get_flags(tfm), ret);
622 				goto out;
623 			}
624 
625 			iv_len = crypto_aead_ivsize(tfm);
626 			if (iv_len)
627 				memset(iv, 0xff, iv_len);
628 
629 			crypto_aead_clear_flags(tfm, ~0);
630 			pr_info("test %u (%d bit key, %d byte blocks): ",
631 				i, *keysize * 8, bs);
632 
633 			memset(tvmem[0], 0xff, PAGE_SIZE);
634 
635 			sg_init_aead(sg, xbuf, bs + (enc ? 0 : authsize),
636 				     assoc, aad_size);
637 
638 			sg_init_aead(sgout, xoutbuf,
639 				     bs + (enc ? authsize : 0), assoc,
640 				     aad_size);
641 
642 			aead_request_set_ad(req, aad_size);
643 
644 			if (!enc) {
645 
646 				/*
647 				 * For decryption we need a proper auth so
648 				 * we do the encryption path once with buffers
649 				 * reversed (input <-> output) to calculate it
650 				 */
651 				aead_request_set_crypt(req, sgout, sg,
652 						       bs, iv);
653 				ret = do_one_aead_op(req,
654 						     crypto_aead_encrypt(req));
655 
656 				if (ret) {
657 					pr_err("calculating auth failed (%d)\n",
658 					       ret);
659 					break;
660 				}
661 			}
662 
663 			aead_request_set_crypt(req, sg, sgout,
664 					       bs + (enc ? 0 : authsize),
665 					       iv);
666 
667 			if (secs) {
668 				ret = test_aead_jiffies(req, enc, bs,
669 							secs);
670 				cond_resched();
671 			} else {
672 				ret = test_aead_cycles(req, enc, bs);
673 			}
674 
675 			if (ret) {
676 				pr_err("%s() failed return code=%d\n", e, ret);
677 				break;
678 			}
679 			b_size++;
680 			i++;
681 		} while (*b_size);
682 		keysize++;
683 	} while (*keysize);
684 
685 out:
686 	aead_request_free(req);
687 out_noreq:
688 	crypto_free_aead(tfm);
689 out_notfm:
690 	kfree(sg);
691 out_nosg:
692 	testmgr_free_buf(xoutbuf);
693 out_nooutbuf:
694 	testmgr_free_buf(axbuf);
695 out_noaxbuf:
696 	testmgr_free_buf(xbuf);
697 out_noxbuf:
698 	kfree(iv);
699 }
700 
test_hash_sg_init(struct scatterlist * sg)701 static void test_hash_sg_init(struct scatterlist *sg)
702 {
703 	int i;
704 
705 	sg_init_table(sg, TVMEMSIZE);
706 	for (i = 0; i < TVMEMSIZE; i++) {
707 		sg_set_buf(sg + i, tvmem[i], PAGE_SIZE);
708 		memset(tvmem[i], 0xff, PAGE_SIZE);
709 	}
710 }
711 
do_one_ahash_op(struct ahash_request * req,int ret)712 static inline int do_one_ahash_op(struct ahash_request *req, int ret)
713 {
714 	struct crypto_wait *wait = req->base.data;
715 
716 	return crypto_wait_req(ret, wait);
717 }
718 
719 struct test_mb_ahash_data {
720 	struct scatterlist sg[XBUFSIZE];
721 	char result[64];
722 	struct ahash_request *req;
723 	struct crypto_wait wait;
724 	char *xbuf[XBUFSIZE];
725 };
726 
do_mult_ahash_op(struct test_mb_ahash_data * data,u32 num_mb,int * rc)727 static inline int do_mult_ahash_op(struct test_mb_ahash_data *data, u32 num_mb,
728 				   int *rc)
729 {
730 	int i, err;
731 
732 	/* Fire up a bunch of concurrent requests */
733 	err = crypto_ahash_digest(data[0].req);
734 
735 	/* Wait for all requests to finish */
736 	err = crypto_wait_req(err, &data[0].wait);
737 	if (num_mb < 2)
738 		return err;
739 
740 	for (i = 0; i < num_mb; i++) {
741 		rc[i] = ahash_request_err(data[i].req);
742 		if (rc[i]) {
743 			pr_info("concurrent request %d error %d\n", i, rc[i]);
744 			err = rc[i];
745 		}
746 	}
747 
748 	return err;
749 }
750 
test_mb_ahash_jiffies(struct test_mb_ahash_data * data,int blen,int secs,u32 num_mb)751 static int test_mb_ahash_jiffies(struct test_mb_ahash_data *data, int blen,
752 				 int secs, u32 num_mb)
753 {
754 	unsigned long start, end;
755 	int bcount;
756 	int ret = 0;
757 	int *rc;
758 
759 	rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL);
760 	if (!rc)
761 		return -ENOMEM;
762 
763 	for (start = jiffies, end = start + secs * HZ, bcount = 0;
764 	     time_before(jiffies, end); bcount++) {
765 		ret = do_mult_ahash_op(data, num_mb, rc);
766 		if (ret)
767 			goto out;
768 	}
769 
770 	pr_cont("%d operations in %d seconds (%llu bytes)\n",
771 		bcount * num_mb, secs, (u64)bcount * blen * num_mb);
772 
773 out:
774 	kfree(rc);
775 	return ret;
776 }
777 
test_mb_ahash_cycles(struct test_mb_ahash_data * data,int blen,u32 num_mb)778 static int test_mb_ahash_cycles(struct test_mb_ahash_data *data, int blen,
779 				u32 num_mb)
780 {
781 	unsigned long cycles = 0;
782 	int ret = 0;
783 	int i;
784 	int *rc;
785 
786 	rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL);
787 	if (!rc)
788 		return -ENOMEM;
789 
790 	/* Warm-up run. */
791 	for (i = 0; i < 4; i++) {
792 		ret = do_mult_ahash_op(data, num_mb, rc);
793 		if (ret)
794 			goto out;
795 	}
796 
797 	/* The real thing. */
798 	for (i = 0; i < 8; i++) {
799 		cycles_t start, end;
800 
801 		start = get_cycles();
802 		ret = do_mult_ahash_op(data, num_mb, rc);
803 		end = get_cycles();
804 
805 		if (ret)
806 			goto out;
807 
808 		cycles += end - start;
809 	}
810 
811 	pr_cont("1 operation in %lu cycles (%d bytes)\n",
812 		(cycles + 4) / (8 * num_mb), blen);
813 
814 out:
815 	kfree(rc);
816 	return ret;
817 }
818 
test_mb_ahash_speed(const char * algo,unsigned int secs,struct hash_speed * speed,u32 num_mb)819 static void test_mb_ahash_speed(const char *algo, unsigned int secs,
820 				struct hash_speed *speed, u32 num_mb)
821 {
822 	struct test_mb_ahash_data *data;
823 	struct crypto_ahash *tfm;
824 	unsigned int i, j, k;
825 	int ret;
826 
827 	data = kcalloc(num_mb, sizeof(*data), GFP_KERNEL);
828 	if (!data)
829 		return;
830 
831 	tfm = crypto_alloc_ahash(algo, 0, 0);
832 	if (IS_ERR(tfm)) {
833 		pr_err("failed to load transform for %s: %ld\n",
834 			algo, PTR_ERR(tfm));
835 		goto free_data;
836 	}
837 
838 	for (i = 0; i < num_mb; ++i) {
839 		if (testmgr_alloc_buf(data[i].xbuf))
840 			goto out;
841 
842 		crypto_init_wait(&data[i].wait);
843 
844 		data[i].req = ahash_request_alloc(tfm, GFP_KERNEL);
845 		if (!data[i].req) {
846 			pr_err("alg: hash: Failed to allocate request for %s\n",
847 			       algo);
848 			goto out;
849 		}
850 
851 
852 		if (i) {
853 			ahash_request_set_callback(data[i].req, 0, NULL, NULL);
854 			ahash_request_chain(data[i].req, data[0].req);
855 		} else
856 			ahash_request_set_callback(data[0].req, 0,
857 						   crypto_req_done,
858 						   &data[0].wait);
859 
860 		sg_init_table(data[i].sg, XBUFSIZE);
861 		for (j = 0; j < XBUFSIZE; j++) {
862 			sg_set_buf(data[i].sg + j, data[i].xbuf[j], PAGE_SIZE);
863 			memset(data[i].xbuf[j], 0xff, PAGE_SIZE);
864 		}
865 	}
866 
867 	pr_info("\ntesting speed of multibuffer %s (%s)\n", algo,
868 		get_driver_name(crypto_ahash, tfm));
869 
870 	for (i = 0; speed[i].blen != 0; i++) {
871 		/* For some reason this only tests digests. */
872 		if (speed[i].blen != speed[i].plen)
873 			continue;
874 
875 		if (speed[i].blen > XBUFSIZE * PAGE_SIZE) {
876 			pr_err("template (%u) too big for tvmem (%lu)\n",
877 			       speed[i].blen, XBUFSIZE * PAGE_SIZE);
878 			goto out;
879 		}
880 
881 		if (klen)
882 			crypto_ahash_setkey(tfm, tvmem[0], klen);
883 
884 		for (k = 0; k < num_mb; k++)
885 			ahash_request_set_crypt(data[k].req, data[k].sg,
886 						data[k].result, speed[i].blen);
887 
888 		pr_info("test%3u "
889 			"(%5u byte blocks,%5u bytes per update,%4u updates): ",
890 			i, speed[i].blen, speed[i].plen,
891 			speed[i].blen / speed[i].plen);
892 
893 		if (secs) {
894 			ret = test_mb_ahash_jiffies(data, speed[i].blen, secs,
895 						    num_mb);
896 			cond_resched();
897 		} else {
898 			ret = test_mb_ahash_cycles(data, speed[i].blen, num_mb);
899 		}
900 
901 
902 		if (ret) {
903 			pr_err("At least one hashing failed ret=%d\n", ret);
904 			break;
905 		}
906 	}
907 
908 out:
909 	ahash_request_free(data[0].req);
910 
911 	for (k = 0; k < num_mb; ++k)
912 		testmgr_free_buf(data[k].xbuf);
913 
914 	crypto_free_ahash(tfm);
915 
916 free_data:
917 	kfree(data);
918 }
919 
test_ahash_jiffies_digest(struct ahash_request * req,int blen,char * out,int secs)920 static int test_ahash_jiffies_digest(struct ahash_request *req, int blen,
921 				     char *out, int secs)
922 {
923 	unsigned long start, end;
924 	int bcount;
925 	int ret;
926 
927 	for (start = jiffies, end = start + secs * HZ, bcount = 0;
928 	     time_before(jiffies, end); bcount++) {
929 		ret = do_one_ahash_op(req, crypto_ahash_digest(req));
930 		if (ret)
931 			return ret;
932 	}
933 
934 	pr_cont("%6u opers/sec, %9lu bytes/sec\n",
935 		bcount / secs, ((long)bcount * blen) / secs);
936 
937 	return 0;
938 }
939 
test_ahash_jiffies(struct ahash_request * req,int blen,int plen,char * out,int secs)940 static int test_ahash_jiffies(struct ahash_request *req, int blen,
941 			      int plen, char *out, int secs)
942 {
943 	unsigned long start, end;
944 	int bcount, pcount;
945 	int ret;
946 
947 	if (plen == blen)
948 		return test_ahash_jiffies_digest(req, blen, out, secs);
949 
950 	for (start = jiffies, end = start + secs * HZ, bcount = 0;
951 	     time_before(jiffies, end); bcount++) {
952 		ret = do_one_ahash_op(req, crypto_ahash_init(req));
953 		if (ret)
954 			return ret;
955 		for (pcount = 0; pcount < blen; pcount += plen) {
956 			ret = do_one_ahash_op(req, crypto_ahash_update(req));
957 			if (ret)
958 				return ret;
959 		}
960 		/* we assume there is enough space in 'out' for the result */
961 		ret = do_one_ahash_op(req, crypto_ahash_final(req));
962 		if (ret)
963 			return ret;
964 	}
965 
966 	pr_cont("%6u opers/sec, %9lu bytes/sec\n",
967 		bcount / secs, ((long)bcount * blen) / secs);
968 
969 	return 0;
970 }
971 
test_ahash_cycles_digest(struct ahash_request * req,int blen,char * out)972 static int test_ahash_cycles_digest(struct ahash_request *req, int blen,
973 				    char *out)
974 {
975 	unsigned long cycles = 0;
976 	int ret, i;
977 
978 	/* Warm-up run. */
979 	for (i = 0; i < 4; i++) {
980 		ret = do_one_ahash_op(req, crypto_ahash_digest(req));
981 		if (ret)
982 			goto out;
983 	}
984 
985 	/* The real thing. */
986 	for (i = 0; i < 8; i++) {
987 		cycles_t start, end;
988 
989 		start = get_cycles();
990 
991 		ret = do_one_ahash_op(req, crypto_ahash_digest(req));
992 		if (ret)
993 			goto out;
994 
995 		end = get_cycles();
996 
997 		cycles += end - start;
998 	}
999 
1000 out:
1001 	if (ret)
1002 		return ret;
1003 
1004 	pr_cont("%6lu cycles/operation, %4lu cycles/byte\n",
1005 		cycles / 8, cycles / (8 * blen));
1006 
1007 	return 0;
1008 }
1009 
test_ahash_cycles(struct ahash_request * req,int blen,int plen,char * out)1010 static int test_ahash_cycles(struct ahash_request *req, int blen,
1011 			     int plen, char *out)
1012 {
1013 	unsigned long cycles = 0;
1014 	int i, pcount, ret;
1015 
1016 	if (plen == blen)
1017 		return test_ahash_cycles_digest(req, blen, out);
1018 
1019 	/* Warm-up run. */
1020 	for (i = 0; i < 4; i++) {
1021 		ret = do_one_ahash_op(req, crypto_ahash_init(req));
1022 		if (ret)
1023 			goto out;
1024 		for (pcount = 0; pcount < blen; pcount += plen) {
1025 			ret = do_one_ahash_op(req, crypto_ahash_update(req));
1026 			if (ret)
1027 				goto out;
1028 		}
1029 		ret = do_one_ahash_op(req, crypto_ahash_final(req));
1030 		if (ret)
1031 			goto out;
1032 	}
1033 
1034 	/* The real thing. */
1035 	for (i = 0; i < 8; i++) {
1036 		cycles_t start, end;
1037 
1038 		start = get_cycles();
1039 
1040 		ret = do_one_ahash_op(req, crypto_ahash_init(req));
1041 		if (ret)
1042 			goto out;
1043 		for (pcount = 0; pcount < blen; pcount += plen) {
1044 			ret = do_one_ahash_op(req, crypto_ahash_update(req));
1045 			if (ret)
1046 				goto out;
1047 		}
1048 		ret = do_one_ahash_op(req, crypto_ahash_final(req));
1049 		if (ret)
1050 			goto out;
1051 
1052 		end = get_cycles();
1053 
1054 		cycles += end - start;
1055 	}
1056 
1057 out:
1058 	if (ret)
1059 		return ret;
1060 
1061 	pr_cont("%6lu cycles/operation, %4lu cycles/byte\n",
1062 		cycles / 8, cycles / (8 * blen));
1063 
1064 	return 0;
1065 }
1066 
test_ahash_speed_common(const char * algo,unsigned int secs,struct hash_speed * speed,unsigned mask)1067 static void test_ahash_speed_common(const char *algo, unsigned int secs,
1068 				    struct hash_speed *speed, unsigned mask)
1069 {
1070 	struct scatterlist sg[TVMEMSIZE];
1071 	struct crypto_wait wait;
1072 	struct ahash_request *req;
1073 	struct crypto_ahash *tfm;
1074 	char *output;
1075 	int i, ret;
1076 
1077 	tfm = crypto_alloc_ahash(algo, 0, mask);
1078 	if (IS_ERR(tfm)) {
1079 		pr_err("failed to load transform for %s: %ld\n",
1080 		       algo, PTR_ERR(tfm));
1081 		return;
1082 	}
1083 
1084 	pr_info("testing speed of async %s (%s)\n", algo,
1085 		get_driver_name(crypto_ahash, tfm));
1086 
1087 	if (crypto_ahash_digestsize(tfm) > MAX_DIGEST_SIZE) {
1088 		pr_err("digestsize(%u) > %d\n", crypto_ahash_digestsize(tfm),
1089 		       MAX_DIGEST_SIZE);
1090 		goto out;
1091 	}
1092 
1093 	test_hash_sg_init(sg);
1094 	req = ahash_request_alloc(tfm, GFP_KERNEL);
1095 	if (!req) {
1096 		pr_err("ahash request allocation failure\n");
1097 		goto out;
1098 	}
1099 
1100 	crypto_init_wait(&wait);
1101 	ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
1102 				   crypto_req_done, &wait);
1103 
1104 	output = kmalloc(MAX_DIGEST_SIZE, GFP_KERNEL);
1105 	if (!output)
1106 		goto out_nomem;
1107 
1108 	for (i = 0; speed[i].blen != 0; i++) {
1109 		if (speed[i].blen > TVMEMSIZE * PAGE_SIZE) {
1110 			pr_err("template (%u) too big for tvmem (%lu)\n",
1111 			       speed[i].blen, TVMEMSIZE * PAGE_SIZE);
1112 			break;
1113 		}
1114 
1115 		if (klen)
1116 			crypto_ahash_setkey(tfm, tvmem[0], klen);
1117 
1118 		pr_info("test%3u "
1119 			"(%5u byte blocks,%5u bytes per update,%4u updates): ",
1120 			i, speed[i].blen, speed[i].plen, speed[i].blen / speed[i].plen);
1121 
1122 		ahash_request_set_crypt(req, sg, output, speed[i].plen);
1123 
1124 		if (secs) {
1125 			ret = test_ahash_jiffies(req, speed[i].blen,
1126 						 speed[i].plen, output, secs);
1127 			cond_resched();
1128 		} else {
1129 			ret = test_ahash_cycles(req, speed[i].blen,
1130 						speed[i].plen, output);
1131 		}
1132 
1133 		if (ret) {
1134 			pr_err("hashing failed ret=%d\n", ret);
1135 			break;
1136 		}
1137 	}
1138 
1139 	kfree(output);
1140 
1141 out_nomem:
1142 	ahash_request_free(req);
1143 
1144 out:
1145 	crypto_free_ahash(tfm);
1146 }
1147 
test_ahash_speed(const char * algo,unsigned int secs,struct hash_speed * speed)1148 static void test_ahash_speed(const char *algo, unsigned int secs,
1149 			     struct hash_speed *speed)
1150 {
1151 	return test_ahash_speed_common(algo, secs, speed, 0);
1152 }
1153 
test_hash_speed(const char * algo,unsigned int secs,struct hash_speed * speed)1154 static void test_hash_speed(const char *algo, unsigned int secs,
1155 			    struct hash_speed *speed)
1156 {
1157 	return test_ahash_speed_common(algo, secs, speed, CRYPTO_ALG_ASYNC);
1158 }
1159 
1160 struct test_mb_skcipher_data {
1161 	struct scatterlist sg[XBUFSIZE];
1162 	struct skcipher_request *req;
1163 	struct crypto_wait wait;
1164 	char *xbuf[XBUFSIZE];
1165 };
1166 
do_mult_acipher_op(struct test_mb_skcipher_data * data,int enc,u32 num_mb,int * rc)1167 static int do_mult_acipher_op(struct test_mb_skcipher_data *data, int enc,
1168 				u32 num_mb, int *rc)
1169 {
1170 	int i, err = 0;
1171 
1172 	/* Fire up a bunch of concurrent requests */
1173 	for (i = 0; i < num_mb; i++) {
1174 		if (enc == ENCRYPT)
1175 			rc[i] = crypto_skcipher_encrypt(data[i].req);
1176 		else
1177 			rc[i] = crypto_skcipher_decrypt(data[i].req);
1178 	}
1179 
1180 	/* Wait for all requests to finish */
1181 	for (i = 0; i < num_mb; i++) {
1182 		rc[i] = crypto_wait_req(rc[i], &data[i].wait);
1183 
1184 		if (rc[i]) {
1185 			pr_info("concurrent request %d error %d\n", i, rc[i]);
1186 			err = rc[i];
1187 		}
1188 	}
1189 
1190 	return err;
1191 }
1192 
test_mb_acipher_jiffies(struct test_mb_skcipher_data * data,int enc,int blen,int secs,u32 num_mb)1193 static int test_mb_acipher_jiffies(struct test_mb_skcipher_data *data, int enc,
1194 				int blen, int secs, u32 num_mb)
1195 {
1196 	unsigned long start, end;
1197 	int bcount;
1198 	int ret = 0;
1199 	int *rc;
1200 
1201 	rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL);
1202 	if (!rc)
1203 		return -ENOMEM;
1204 
1205 	for (start = jiffies, end = start + secs * HZ, bcount = 0;
1206 	     time_before(jiffies, end); bcount++) {
1207 		ret = do_mult_acipher_op(data, enc, num_mb, rc);
1208 		if (ret)
1209 			goto out;
1210 	}
1211 
1212 	pr_cont("%d operations in %d seconds (%llu bytes)\n",
1213 		bcount * num_mb, secs, (u64)bcount * blen * num_mb);
1214 
1215 out:
1216 	kfree(rc);
1217 	return ret;
1218 }
1219 
test_mb_acipher_cycles(struct test_mb_skcipher_data * data,int enc,int blen,u32 num_mb)1220 static int test_mb_acipher_cycles(struct test_mb_skcipher_data *data, int enc,
1221 			       int blen, u32 num_mb)
1222 {
1223 	unsigned long cycles = 0;
1224 	int ret = 0;
1225 	int i;
1226 	int *rc;
1227 
1228 	rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL);
1229 	if (!rc)
1230 		return -ENOMEM;
1231 
1232 	/* Warm-up run. */
1233 	for (i = 0; i < 4; i++) {
1234 		ret = do_mult_acipher_op(data, enc, num_mb, rc);
1235 		if (ret)
1236 			goto out;
1237 	}
1238 
1239 	/* The real thing. */
1240 	for (i = 0; i < 8; i++) {
1241 		cycles_t start, end;
1242 
1243 		start = get_cycles();
1244 		ret = do_mult_acipher_op(data, enc, num_mb, rc);
1245 		end = get_cycles();
1246 
1247 		if (ret)
1248 			goto out;
1249 
1250 		cycles += end - start;
1251 	}
1252 
1253 	pr_cont("1 operation in %lu cycles (%d bytes)\n",
1254 		(cycles + 4) / (8 * num_mb), blen);
1255 
1256 out:
1257 	kfree(rc);
1258 	return ret;
1259 }
1260 
test_mb_skcipher_speed(const char * algo,int enc,int secs,struct cipher_speed_template * template,unsigned int tcount,u8 * keysize,u32 num_mb)1261 static void test_mb_skcipher_speed(const char *algo, int enc, int secs,
1262 				   struct cipher_speed_template *template,
1263 				   unsigned int tcount, u8 *keysize, u32 num_mb)
1264 {
1265 	struct test_mb_skcipher_data *data;
1266 	struct crypto_skcipher *tfm;
1267 	unsigned int i, j, iv_len;
1268 	const int *b_size;
1269 	const char *key;
1270 	const char *e;
1271 	char iv[128];
1272 	int ret;
1273 
1274 	if (enc == ENCRYPT)
1275 		e = "encryption";
1276 	else
1277 		e = "decryption";
1278 
1279 	data = kcalloc(num_mb, sizeof(*data), GFP_KERNEL);
1280 	if (!data)
1281 		return;
1282 
1283 	tfm = crypto_alloc_skcipher(algo, 0, 0);
1284 	if (IS_ERR(tfm)) {
1285 		pr_err("failed to load transform for %s: %ld\n",
1286 			algo, PTR_ERR(tfm));
1287 		goto out_free_data;
1288 	}
1289 
1290 	for (i = 0; i < num_mb; ++i)
1291 		if (testmgr_alloc_buf(data[i].xbuf)) {
1292 			while (i--)
1293 				testmgr_free_buf(data[i].xbuf);
1294 			goto out_free_tfm;
1295 		}
1296 
1297 	for (i = 0; i < num_mb; ++i) {
1298 		data[i].req = skcipher_request_alloc(tfm, GFP_KERNEL);
1299 		if (!data[i].req) {
1300 			pr_err("alg: skcipher: Failed to allocate request for %s\n",
1301 			       algo);
1302 			while (i--)
1303 				skcipher_request_free(data[i].req);
1304 			goto out_free_xbuf;
1305 		}
1306 	}
1307 
1308 	for (i = 0; i < num_mb; ++i) {
1309 		skcipher_request_set_callback(data[i].req,
1310 					      CRYPTO_TFM_REQ_MAY_BACKLOG,
1311 					      crypto_req_done, &data[i].wait);
1312 		crypto_init_wait(&data[i].wait);
1313 	}
1314 
1315 	pr_info("testing speed of multibuffer %s (%s) %s\n", algo,
1316 		get_driver_name(crypto_skcipher, tfm), e);
1317 
1318 	i = 0;
1319 	do {
1320 		b_size = block_sizes;
1321 		do {
1322 			u32 bs = round_up(*b_size, crypto_skcipher_blocksize(tfm));
1323 
1324 			if (bs > XBUFSIZE * PAGE_SIZE) {
1325 				pr_err("template (%u) too big for buffer (%lu)\n",
1326 				       bs, XBUFSIZE * PAGE_SIZE);
1327 				goto out;
1328 			}
1329 
1330 			pr_info("test %u (%d bit key, %d byte blocks): ", i,
1331 				*keysize * 8, bs);
1332 
1333 			/* Set up tfm global state, i.e. the key */
1334 
1335 			memset(tvmem[0], 0xff, PAGE_SIZE);
1336 			key = tvmem[0];
1337 			for (j = 0; j < tcount; j++) {
1338 				if (template[j].klen == *keysize) {
1339 					key = template[j].key;
1340 					break;
1341 				}
1342 			}
1343 
1344 			crypto_skcipher_clear_flags(tfm, ~0);
1345 
1346 			ret = crypto_skcipher_setkey(tfm, key, *keysize);
1347 			if (ret) {
1348 				pr_err("setkey() failed flags=%x\n",
1349 				       crypto_skcipher_get_flags(tfm));
1350 				goto out;
1351 			}
1352 
1353 			iv_len = crypto_skcipher_ivsize(tfm);
1354 			if (iv_len)
1355 				memset(&iv, 0xff, iv_len);
1356 
1357 			/* Now setup per request stuff, i.e. buffers */
1358 
1359 			for (j = 0; j < num_mb; ++j) {
1360 				struct test_mb_skcipher_data *cur = &data[j];
1361 				unsigned int k = bs;
1362 				unsigned int pages = DIV_ROUND_UP(k, PAGE_SIZE);
1363 				unsigned int p = 0;
1364 
1365 				sg_init_table(cur->sg, pages);
1366 
1367 				while (k > PAGE_SIZE) {
1368 					sg_set_buf(cur->sg + p, cur->xbuf[p],
1369 						   PAGE_SIZE);
1370 					memset(cur->xbuf[p], 0xff, PAGE_SIZE);
1371 					p++;
1372 					k -= PAGE_SIZE;
1373 				}
1374 
1375 				sg_set_buf(cur->sg + p, cur->xbuf[p], k);
1376 				memset(cur->xbuf[p], 0xff, k);
1377 
1378 				skcipher_request_set_crypt(cur->req, cur->sg,
1379 							   cur->sg, bs, iv);
1380 			}
1381 
1382 			if (secs) {
1383 				ret = test_mb_acipher_jiffies(data, enc,
1384 							      bs, secs,
1385 							      num_mb);
1386 				cond_resched();
1387 			} else {
1388 				ret = test_mb_acipher_cycles(data, enc,
1389 							     bs, num_mb);
1390 			}
1391 
1392 			if (ret) {
1393 				pr_err("%s() failed flags=%x\n", e,
1394 				       crypto_skcipher_get_flags(tfm));
1395 				break;
1396 			}
1397 			b_size++;
1398 			i++;
1399 		} while (*b_size);
1400 		keysize++;
1401 	} while (*keysize);
1402 
1403 out:
1404 	for (i = 0; i < num_mb; ++i)
1405 		skcipher_request_free(data[i].req);
1406 out_free_xbuf:
1407 	for (i = 0; i < num_mb; ++i)
1408 		testmgr_free_buf(data[i].xbuf);
1409 out_free_tfm:
1410 	crypto_free_skcipher(tfm);
1411 out_free_data:
1412 	kfree(data);
1413 }
1414 
do_one_acipher_op(struct skcipher_request * req,int ret)1415 static inline int do_one_acipher_op(struct skcipher_request *req, int ret)
1416 {
1417 	struct crypto_wait *wait = req->base.data;
1418 
1419 	return crypto_wait_req(ret, wait);
1420 }
1421 
test_acipher_jiffies(struct skcipher_request * req,int enc,int blen,int secs)1422 static int test_acipher_jiffies(struct skcipher_request *req, int enc,
1423 				int blen, int secs)
1424 {
1425 	unsigned long start, end;
1426 	int bcount;
1427 	int ret;
1428 
1429 	for (start = jiffies, end = start + secs * HZ, bcount = 0;
1430 	     time_before(jiffies, end); bcount++) {
1431 		if (enc)
1432 			ret = do_one_acipher_op(req,
1433 						crypto_skcipher_encrypt(req));
1434 		else
1435 			ret = do_one_acipher_op(req,
1436 						crypto_skcipher_decrypt(req));
1437 
1438 		if (ret)
1439 			return ret;
1440 	}
1441 
1442 	pr_cont("%d operations in %d seconds (%llu bytes)\n",
1443 		bcount, secs, (u64)bcount * blen);
1444 	return 0;
1445 }
1446 
test_acipher_cycles(struct skcipher_request * req,int enc,int blen)1447 static int test_acipher_cycles(struct skcipher_request *req, int enc,
1448 			       int blen)
1449 {
1450 	unsigned long cycles = 0;
1451 	int ret = 0;
1452 	int i;
1453 
1454 	/* Warm-up run. */
1455 	for (i = 0; i < 4; i++) {
1456 		if (enc)
1457 			ret = do_one_acipher_op(req,
1458 						crypto_skcipher_encrypt(req));
1459 		else
1460 			ret = do_one_acipher_op(req,
1461 						crypto_skcipher_decrypt(req));
1462 
1463 		if (ret)
1464 			goto out;
1465 	}
1466 
1467 	/* The real thing. */
1468 	for (i = 0; i < 8; i++) {
1469 		cycles_t start, end;
1470 
1471 		start = get_cycles();
1472 		if (enc)
1473 			ret = do_one_acipher_op(req,
1474 						crypto_skcipher_encrypt(req));
1475 		else
1476 			ret = do_one_acipher_op(req,
1477 						crypto_skcipher_decrypt(req));
1478 		end = get_cycles();
1479 
1480 		if (ret)
1481 			goto out;
1482 
1483 		cycles += end - start;
1484 	}
1485 
1486 out:
1487 	if (ret == 0)
1488 		pr_cont("1 operation in %lu cycles (%d bytes)\n",
1489 			(cycles + 4) / 8, blen);
1490 
1491 	return ret;
1492 }
1493 
test_skcipher_speed(const char * algo,int enc,unsigned int secs,struct cipher_speed_template * template,unsigned int tcount,u8 * keysize,bool async)1494 static void test_skcipher_speed(const char *algo, int enc, unsigned int secs,
1495 				struct cipher_speed_template *template,
1496 				unsigned int tcount, u8 *keysize, bool async)
1497 {
1498 	unsigned int ret, i, j, k, iv_len;
1499 	struct crypto_wait wait;
1500 	const char *key;
1501 	char iv[128];
1502 	struct skcipher_request *req;
1503 	struct crypto_skcipher *tfm;
1504 	const int *b_size;
1505 	const char *e;
1506 
1507 	if (enc == ENCRYPT)
1508 		e = "encryption";
1509 	else
1510 		e = "decryption";
1511 
1512 	crypto_init_wait(&wait);
1513 
1514 	tfm = crypto_alloc_skcipher(algo, 0, async ? 0 : CRYPTO_ALG_ASYNC);
1515 
1516 	if (IS_ERR(tfm)) {
1517 		pr_err("failed to load transform for %s: %ld\n", algo,
1518 		       PTR_ERR(tfm));
1519 		return;
1520 	}
1521 
1522 	pr_info("testing speed of %s %s (%s) %s\n", async ? "async" : "sync",
1523 		algo, get_driver_name(crypto_skcipher, tfm), e);
1524 
1525 	req = skcipher_request_alloc(tfm, GFP_KERNEL);
1526 	if (!req) {
1527 		pr_err("skcipher: Failed to allocate request for %s\n", algo);
1528 		goto out;
1529 	}
1530 
1531 	skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG,
1532 				      crypto_req_done, &wait);
1533 
1534 	i = 0;
1535 	do {
1536 		b_size = block_sizes;
1537 
1538 		do {
1539 			u32 bs = round_up(*b_size, crypto_skcipher_blocksize(tfm));
1540 			struct scatterlist sg[TVMEMSIZE];
1541 
1542 			if ((*keysize + bs) > TVMEMSIZE * PAGE_SIZE) {
1543 				pr_err("template (%u) too big for "
1544 				       "tvmem (%lu)\n", *keysize + bs,
1545 				       TVMEMSIZE * PAGE_SIZE);
1546 				goto out_free_req;
1547 			}
1548 
1549 			pr_info("test %u (%d bit key, %d byte blocks): ", i,
1550 				*keysize * 8, bs);
1551 
1552 			memset(tvmem[0], 0xff, PAGE_SIZE);
1553 
1554 			/* set key, plain text and IV */
1555 			key = tvmem[0];
1556 			for (j = 0; j < tcount; j++) {
1557 				if (template[j].klen == *keysize) {
1558 					key = template[j].key;
1559 					break;
1560 				}
1561 			}
1562 
1563 			crypto_skcipher_clear_flags(tfm, ~0);
1564 
1565 			ret = crypto_skcipher_setkey(tfm, key, *keysize);
1566 			if (ret) {
1567 				pr_err("setkey() failed flags=%x\n",
1568 					crypto_skcipher_get_flags(tfm));
1569 				goto out_free_req;
1570 			}
1571 
1572 			k = *keysize + bs;
1573 			sg_init_table(sg, DIV_ROUND_UP(k, PAGE_SIZE));
1574 
1575 			if (k > PAGE_SIZE) {
1576 				sg_set_buf(sg, tvmem[0] + *keysize,
1577 				   PAGE_SIZE - *keysize);
1578 				k -= PAGE_SIZE;
1579 				j = 1;
1580 				while (k > PAGE_SIZE) {
1581 					sg_set_buf(sg + j, tvmem[j], PAGE_SIZE);
1582 					memset(tvmem[j], 0xff, PAGE_SIZE);
1583 					j++;
1584 					k -= PAGE_SIZE;
1585 				}
1586 				sg_set_buf(sg + j, tvmem[j], k);
1587 				memset(tvmem[j], 0xff, k);
1588 			} else {
1589 				sg_set_buf(sg, tvmem[0] + *keysize, bs);
1590 			}
1591 
1592 			iv_len = crypto_skcipher_ivsize(tfm);
1593 			if (iv_len)
1594 				memset(&iv, 0xff, iv_len);
1595 
1596 			skcipher_request_set_crypt(req, sg, sg, bs, iv);
1597 
1598 			if (secs) {
1599 				ret = test_acipher_jiffies(req, enc,
1600 							   bs, secs);
1601 				cond_resched();
1602 			} else {
1603 				ret = test_acipher_cycles(req, enc,
1604 							  bs);
1605 			}
1606 
1607 			if (ret) {
1608 				pr_err("%s() failed flags=%x\n", e,
1609 				       crypto_skcipher_get_flags(tfm));
1610 				break;
1611 			}
1612 			b_size++;
1613 			i++;
1614 		} while (*b_size);
1615 		keysize++;
1616 	} while (*keysize);
1617 
1618 out_free_req:
1619 	skcipher_request_free(req);
1620 out:
1621 	crypto_free_skcipher(tfm);
1622 }
1623 
test_acipher_speed(const char * algo,int enc,unsigned int secs,struct cipher_speed_template * template,unsigned int tcount,u8 * keysize)1624 static void test_acipher_speed(const char *algo, int enc, unsigned int secs,
1625 			       struct cipher_speed_template *template,
1626 			       unsigned int tcount, u8 *keysize)
1627 {
1628 	return test_skcipher_speed(algo, enc, secs, template, tcount, keysize,
1629 				   true);
1630 }
1631 
test_cipher_speed(const char * algo,int enc,unsigned int secs,struct cipher_speed_template * template,unsigned int tcount,u8 * keysize)1632 static void test_cipher_speed(const char *algo, int enc, unsigned int secs,
1633 			      struct cipher_speed_template *template,
1634 			      unsigned int tcount, u8 *keysize)
1635 {
1636 	return test_skcipher_speed(algo, enc, secs, template, tcount, keysize,
1637 				   false);
1638 }
1639 
tcrypt_test(const char * alg)1640 static inline int tcrypt_test(const char *alg)
1641 {
1642 	int ret;
1643 
1644 	pr_debug("testing %s\n", alg);
1645 
1646 	ret = alg_test(alg, alg, 0, 0);
1647 	/* non-fips algs return -EINVAL or -ECANCELED in fips mode */
1648 	if (fips_enabled && (ret == -EINVAL || ret == -ECANCELED))
1649 		ret = 0;
1650 	return ret;
1651 }
1652 
do_test(const char * alg,u32 type,u32 mask,int m,u32 num_mb)1653 static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb)
1654 {
1655 	int i;
1656 	int ret = 0;
1657 
1658 	switch (m) {
1659 	case 0:
1660 		if (alg) {
1661 			if (!crypto_has_alg(alg, type,
1662 					    mask ?: CRYPTO_ALG_TYPE_MASK))
1663 				ret = -ENOENT;
1664 			break;
1665 		}
1666 
1667 		for (i = 1; i < 200; i++)
1668 			ret = min(ret, do_test(NULL, 0, 0, i, num_mb));
1669 		break;
1670 
1671 	case 1:
1672 		ret = min(ret, tcrypt_test("md5"));
1673 		break;
1674 
1675 	case 2:
1676 		ret = min(ret, tcrypt_test("sha1"));
1677 		break;
1678 
1679 	case 3:
1680 		ret = min(ret, tcrypt_test("ecb(des)"));
1681 		ret = min(ret, tcrypt_test("cbc(des)"));
1682 		ret = min(ret, tcrypt_test("ctr(des)"));
1683 		break;
1684 
1685 	case 4:
1686 		ret = min(ret, tcrypt_test("ecb(des3_ede)"));
1687 		ret = min(ret, tcrypt_test("cbc(des3_ede)"));
1688 		ret = min(ret, tcrypt_test("ctr(des3_ede)"));
1689 		break;
1690 
1691 	case 5:
1692 		ret = min(ret, tcrypt_test("md4"));
1693 		break;
1694 
1695 	case 6:
1696 		ret = min(ret, tcrypt_test("sha256"));
1697 		break;
1698 
1699 	case 7:
1700 		ret = min(ret, tcrypt_test("ecb(blowfish)"));
1701 		ret = min(ret, tcrypt_test("cbc(blowfish)"));
1702 		ret = min(ret, tcrypt_test("ctr(blowfish)"));
1703 		break;
1704 
1705 	case 8:
1706 		ret = min(ret, tcrypt_test("ecb(twofish)"));
1707 		ret = min(ret, tcrypt_test("cbc(twofish)"));
1708 		ret = min(ret, tcrypt_test("ctr(twofish)"));
1709 		ret = min(ret, tcrypt_test("lrw(twofish)"));
1710 		ret = min(ret, tcrypt_test("xts(twofish)"));
1711 		break;
1712 
1713 	case 9:
1714 		ret = min(ret, tcrypt_test("ecb(serpent)"));
1715 		ret = min(ret, tcrypt_test("cbc(serpent)"));
1716 		ret = min(ret, tcrypt_test("ctr(serpent)"));
1717 		ret = min(ret, tcrypt_test("lrw(serpent)"));
1718 		ret = min(ret, tcrypt_test("xts(serpent)"));
1719 		break;
1720 
1721 	case 10:
1722 		ret = min(ret, tcrypt_test("ecb(aes)"));
1723 		ret = min(ret, tcrypt_test("cbc(aes)"));
1724 		ret = min(ret, tcrypt_test("lrw(aes)"));
1725 		ret = min(ret, tcrypt_test("xts(aes)"));
1726 		ret = min(ret, tcrypt_test("ctr(aes)"));
1727 		ret = min(ret, tcrypt_test("rfc3686(ctr(aes))"));
1728 		ret = min(ret, tcrypt_test("xctr(aes)"));
1729 		break;
1730 
1731 	case 11:
1732 		ret = min(ret, tcrypt_test("sha384"));
1733 		break;
1734 
1735 	case 12:
1736 		ret = min(ret, tcrypt_test("sha512"));
1737 		break;
1738 
1739 	case 13:
1740 		ret = min(ret, tcrypt_test("deflate"));
1741 		break;
1742 
1743 	case 14:
1744 		ret = min(ret, tcrypt_test("ecb(cast5)"));
1745 		ret = min(ret, tcrypt_test("cbc(cast5)"));
1746 		ret = min(ret, tcrypt_test("ctr(cast5)"));
1747 		break;
1748 
1749 	case 15:
1750 		ret = min(ret, tcrypt_test("ecb(cast6)"));
1751 		ret = min(ret, tcrypt_test("cbc(cast6)"));
1752 		ret = min(ret, tcrypt_test("ctr(cast6)"));
1753 		ret = min(ret, tcrypt_test("lrw(cast6)"));
1754 		ret = min(ret, tcrypt_test("xts(cast6)"));
1755 		break;
1756 
1757 	case 16:
1758 		ret = min(ret, tcrypt_test("ecb(arc4)"));
1759 		break;
1760 
1761 	case 17:
1762 		ret = min(ret, tcrypt_test("michael_mic"));
1763 		break;
1764 
1765 	case 18:
1766 		ret = min(ret, tcrypt_test("crc32c"));
1767 		break;
1768 
1769 	case 19:
1770 		ret = min(ret, tcrypt_test("ecb(tea)"));
1771 		break;
1772 
1773 	case 20:
1774 		ret = min(ret, tcrypt_test("ecb(xtea)"));
1775 		break;
1776 
1777 	case 21:
1778 		ret = min(ret, tcrypt_test("ecb(khazad)"));
1779 		break;
1780 
1781 	case 22:
1782 		ret = min(ret, tcrypt_test("wp512"));
1783 		break;
1784 
1785 	case 23:
1786 		ret = min(ret, tcrypt_test("wp384"));
1787 		break;
1788 
1789 	case 24:
1790 		ret = min(ret, tcrypt_test("wp256"));
1791 		break;
1792 
1793 	case 26:
1794 		ret = min(ret, tcrypt_test("ecb(anubis)"));
1795 		ret = min(ret, tcrypt_test("cbc(anubis)"));
1796 		break;
1797 
1798 	case 30:
1799 		ret = min(ret, tcrypt_test("ecb(xeta)"));
1800 		break;
1801 
1802 	case 31:
1803 		ret = min(ret, tcrypt_test("pcbc(fcrypt)"));
1804 		break;
1805 
1806 	case 32:
1807 		ret = min(ret, tcrypt_test("ecb(camellia)"));
1808 		ret = min(ret, tcrypt_test("cbc(camellia)"));
1809 		ret = min(ret, tcrypt_test("ctr(camellia)"));
1810 		ret = min(ret, tcrypt_test("lrw(camellia)"));
1811 		ret = min(ret, tcrypt_test("xts(camellia)"));
1812 		break;
1813 
1814 	case 33:
1815 		ret = min(ret, tcrypt_test("sha224"));
1816 		break;
1817 
1818 	case 35:
1819 		ret = min(ret, tcrypt_test("gcm(aes)"));
1820 		break;
1821 
1822 	case 36:
1823 		ret = min(ret, tcrypt_test("lzo"));
1824 		break;
1825 
1826 	case 37:
1827 		ret = min(ret, tcrypt_test("ccm(aes)"));
1828 		break;
1829 
1830 	case 38:
1831 		ret = min(ret, tcrypt_test("cts(cbc(aes))"));
1832 		break;
1833 
1834         case 39:
1835 		ret = min(ret, tcrypt_test("xxhash64"));
1836 		break;
1837 
1838         case 40:
1839 		ret = min(ret, tcrypt_test("rmd160"));
1840 		break;
1841 
1842 	case 42:
1843 		ret = min(ret, tcrypt_test("blake2b-512"));
1844 		break;
1845 
1846 	case 43:
1847 		ret = min(ret, tcrypt_test("ecb(seed)"));
1848 		break;
1849 
1850 	case 45:
1851 		ret = min(ret, tcrypt_test("rfc4309(ccm(aes))"));
1852 		break;
1853 
1854 	case 46:
1855 		ret = min(ret, tcrypt_test("ghash"));
1856 		break;
1857 
1858 	case 48:
1859 		ret = min(ret, tcrypt_test("sha3-224"));
1860 		break;
1861 
1862 	case 49:
1863 		ret = min(ret, tcrypt_test("sha3-256"));
1864 		break;
1865 
1866 	case 50:
1867 		ret = min(ret, tcrypt_test("sha3-384"));
1868 		break;
1869 
1870 	case 51:
1871 		ret = min(ret, tcrypt_test("sha3-512"));
1872 		break;
1873 
1874 	case 52:
1875 		ret = min(ret, tcrypt_test("sm3"));
1876 		break;
1877 
1878 	case 53:
1879 		ret = min(ret, tcrypt_test("streebog256"));
1880 		break;
1881 
1882 	case 54:
1883 		ret = min(ret, tcrypt_test("streebog512"));
1884 		break;
1885 
1886 	case 55:
1887 		ret = min(ret, tcrypt_test("gcm(sm4)"));
1888 		break;
1889 
1890 	case 56:
1891 		ret = min(ret, tcrypt_test("ccm(sm4)"));
1892 		break;
1893 
1894 	case 57:
1895 		ret = min(ret, tcrypt_test("polyval"));
1896 		break;
1897 
1898 	case 58:
1899 		ret = min(ret, tcrypt_test("gcm(aria)"));
1900 		break;
1901 
1902 	case 59:
1903 		ret = min(ret, tcrypt_test("cts(cbc(sm4))"));
1904 		break;
1905 
1906 	case 100:
1907 		ret = min(ret, tcrypt_test("hmac(md5)"));
1908 		break;
1909 
1910 	case 101:
1911 		ret = min(ret, tcrypt_test("hmac(sha1)"));
1912 		break;
1913 
1914 	case 102:
1915 		ret = min(ret, tcrypt_test("hmac(sha256)"));
1916 		break;
1917 
1918 	case 103:
1919 		ret = min(ret, tcrypt_test("hmac(sha384)"));
1920 		break;
1921 
1922 	case 104:
1923 		ret = min(ret, tcrypt_test("hmac(sha512)"));
1924 		break;
1925 
1926 	case 105:
1927 		ret = min(ret, tcrypt_test("hmac(sha224)"));
1928 		break;
1929 
1930 	case 106:
1931 		ret = min(ret, tcrypt_test("xcbc(aes)"));
1932 		break;
1933 
1934 	case 108:
1935 		ret = min(ret, tcrypt_test("hmac(rmd160)"));
1936 		break;
1937 
1938 	case 111:
1939 		ret = min(ret, tcrypt_test("hmac(sha3-224)"));
1940 		break;
1941 
1942 	case 112:
1943 		ret = min(ret, tcrypt_test("hmac(sha3-256)"));
1944 		break;
1945 
1946 	case 113:
1947 		ret = min(ret, tcrypt_test("hmac(sha3-384)"));
1948 		break;
1949 
1950 	case 114:
1951 		ret = min(ret, tcrypt_test("hmac(sha3-512)"));
1952 		break;
1953 
1954 	case 115:
1955 		ret = min(ret, tcrypt_test("hmac(streebog256)"));
1956 		break;
1957 
1958 	case 116:
1959 		ret = min(ret, tcrypt_test("hmac(streebog512)"));
1960 		break;
1961 
1962 	case 150:
1963 		ret = min(ret, tcrypt_test("ansi_cprng"));
1964 		break;
1965 
1966 	case 151:
1967 		ret = min(ret, tcrypt_test("rfc4106(gcm(aes))"));
1968 		break;
1969 
1970 	case 152:
1971 		ret = min(ret, tcrypt_test("rfc4543(gcm(aes))"));
1972 		break;
1973 
1974 	case 153:
1975 		ret = min(ret, tcrypt_test("cmac(aes)"));
1976 		break;
1977 
1978 	case 154:
1979 		ret = min(ret, tcrypt_test("cmac(des3_ede)"));
1980 		break;
1981 
1982 	case 155:
1983 		ret = min(ret, tcrypt_test("authenc(hmac(sha1),cbc(aes))"));
1984 		break;
1985 
1986 	case 156:
1987 		ret = min(ret, tcrypt_test("authenc(hmac(md5),ecb(cipher_null))"));
1988 		break;
1989 
1990 	case 157:
1991 		ret = min(ret, tcrypt_test("authenc(hmac(sha1),ecb(cipher_null))"));
1992 		break;
1993 
1994 	case 158:
1995 		ret = min(ret, tcrypt_test("cbcmac(sm4)"));
1996 		break;
1997 
1998 	case 159:
1999 		ret = min(ret, tcrypt_test("cmac(sm4)"));
2000 		break;
2001 
2002 	case 160:
2003 		ret = min(ret, tcrypt_test("xcbc(sm4)"));
2004 		break;
2005 
2006 	case 181:
2007 		ret = min(ret, tcrypt_test("authenc(hmac(sha1),cbc(des))"));
2008 		break;
2009 	case 182:
2010 		ret = min(ret, tcrypt_test("authenc(hmac(sha1),cbc(des3_ede))"));
2011 		break;
2012 	case 183:
2013 		ret = min(ret, tcrypt_test("authenc(hmac(sha224),cbc(des))"));
2014 		break;
2015 	case 184:
2016 		ret = min(ret, tcrypt_test("authenc(hmac(sha224),cbc(des3_ede))"));
2017 		break;
2018 	case 185:
2019 		ret = min(ret, tcrypt_test("authenc(hmac(sha256),cbc(des))"));
2020 		break;
2021 	case 186:
2022 		ret = min(ret, tcrypt_test("authenc(hmac(sha256),cbc(des3_ede))"));
2023 		break;
2024 	case 187:
2025 		ret = min(ret, tcrypt_test("authenc(hmac(sha384),cbc(des))"));
2026 		break;
2027 	case 188:
2028 		ret = min(ret, tcrypt_test("authenc(hmac(sha384),cbc(des3_ede))"));
2029 		break;
2030 	case 189:
2031 		ret = min(ret, tcrypt_test("authenc(hmac(sha512),cbc(des))"));
2032 		break;
2033 	case 190:
2034 		ret = min(ret, tcrypt_test("authenc(hmac(sha512),cbc(des3_ede))"));
2035 		break;
2036 	case 191:
2037 		ret = min(ret, tcrypt_test("ecb(sm4)"));
2038 		ret = min(ret, tcrypt_test("cbc(sm4)"));
2039 		ret = min(ret, tcrypt_test("ctr(sm4)"));
2040 		ret = min(ret, tcrypt_test("xts(sm4)"));
2041 		break;
2042 	case 192:
2043 		ret = min(ret, tcrypt_test("ecb(aria)"));
2044 		ret = min(ret, tcrypt_test("cbc(aria)"));
2045 		ret = min(ret, tcrypt_test("ctr(aria)"));
2046 		break;
2047 	case 193:
2048 		ret = min(ret, tcrypt_test("ffdhe2048(dh)"));
2049 		break;
2050 	case 200:
2051 		test_cipher_speed("ecb(aes)", ENCRYPT, sec, NULL, 0,
2052 				speed_template_16_24_32);
2053 		test_cipher_speed("ecb(aes)", DECRYPT, sec, NULL, 0,
2054 				speed_template_16_24_32);
2055 		test_cipher_speed("cbc(aes)", ENCRYPT, sec, NULL, 0,
2056 				speed_template_16_24_32);
2057 		test_cipher_speed("cbc(aes)", DECRYPT, sec, NULL, 0,
2058 				speed_template_16_24_32);
2059 		test_cipher_speed("lrw(aes)", ENCRYPT, sec, NULL, 0,
2060 				speed_template_32_40_48);
2061 		test_cipher_speed("lrw(aes)", DECRYPT, sec, NULL, 0,
2062 				speed_template_32_40_48);
2063 		test_cipher_speed("xts(aes)", ENCRYPT, sec, NULL, 0,
2064 				speed_template_32_64);
2065 		test_cipher_speed("xts(aes)", DECRYPT, sec, NULL, 0,
2066 				speed_template_32_64);
2067 		test_cipher_speed("cts(cbc(aes))", ENCRYPT, sec, NULL, 0,
2068 				speed_template_16_24_32);
2069 		test_cipher_speed("cts(cbc(aes))", DECRYPT, sec, NULL, 0,
2070 				speed_template_16_24_32);
2071 		test_cipher_speed("ctr(aes)", ENCRYPT, sec, NULL, 0,
2072 				speed_template_16_24_32);
2073 		test_cipher_speed("ctr(aes)", DECRYPT, sec, NULL, 0,
2074 				speed_template_16_24_32);
2075 		break;
2076 
2077 	case 201:
2078 		test_cipher_speed("ecb(des3_ede)", ENCRYPT, sec,
2079 				des3_speed_template, DES3_SPEED_VECTORS,
2080 				speed_template_24);
2081 		test_cipher_speed("ecb(des3_ede)", DECRYPT, sec,
2082 				des3_speed_template, DES3_SPEED_VECTORS,
2083 				speed_template_24);
2084 		test_cipher_speed("cbc(des3_ede)", ENCRYPT, sec,
2085 				des3_speed_template, DES3_SPEED_VECTORS,
2086 				speed_template_24);
2087 		test_cipher_speed("cbc(des3_ede)", DECRYPT, sec,
2088 				des3_speed_template, DES3_SPEED_VECTORS,
2089 				speed_template_24);
2090 		test_cipher_speed("ctr(des3_ede)", ENCRYPT, sec,
2091 				des3_speed_template, DES3_SPEED_VECTORS,
2092 				speed_template_24);
2093 		test_cipher_speed("ctr(des3_ede)", DECRYPT, sec,
2094 				des3_speed_template, DES3_SPEED_VECTORS,
2095 				speed_template_24);
2096 		break;
2097 
2098 	case 202:
2099 		test_cipher_speed("ecb(twofish)", ENCRYPT, sec, NULL, 0,
2100 				speed_template_16_24_32);
2101 		test_cipher_speed("ecb(twofish)", DECRYPT, sec, NULL, 0,
2102 				speed_template_16_24_32);
2103 		test_cipher_speed("cbc(twofish)", ENCRYPT, sec, NULL, 0,
2104 				speed_template_16_24_32);
2105 		test_cipher_speed("cbc(twofish)", DECRYPT, sec, NULL, 0,
2106 				speed_template_16_24_32);
2107 		test_cipher_speed("ctr(twofish)", ENCRYPT, sec, NULL, 0,
2108 				speed_template_16_24_32);
2109 		test_cipher_speed("ctr(twofish)", DECRYPT, sec, NULL, 0,
2110 				speed_template_16_24_32);
2111 		test_cipher_speed("lrw(twofish)", ENCRYPT, sec, NULL, 0,
2112 				speed_template_32_40_48);
2113 		test_cipher_speed("lrw(twofish)", DECRYPT, sec, NULL, 0,
2114 				speed_template_32_40_48);
2115 		test_cipher_speed("xts(twofish)", ENCRYPT, sec, NULL, 0,
2116 				speed_template_32_48_64);
2117 		test_cipher_speed("xts(twofish)", DECRYPT, sec, NULL, 0,
2118 				speed_template_32_48_64);
2119 		break;
2120 
2121 	case 203:
2122 		test_cipher_speed("ecb(blowfish)", ENCRYPT, sec, NULL, 0,
2123 				  speed_template_8_32);
2124 		test_cipher_speed("ecb(blowfish)", DECRYPT, sec, NULL, 0,
2125 				  speed_template_8_32);
2126 		test_cipher_speed("cbc(blowfish)", ENCRYPT, sec, NULL, 0,
2127 				  speed_template_8_32);
2128 		test_cipher_speed("cbc(blowfish)", DECRYPT, sec, NULL, 0,
2129 				  speed_template_8_32);
2130 		test_cipher_speed("ctr(blowfish)", ENCRYPT, sec, NULL, 0,
2131 				  speed_template_8_32);
2132 		test_cipher_speed("ctr(blowfish)", DECRYPT, sec, NULL, 0,
2133 				  speed_template_8_32);
2134 		break;
2135 
2136 	case 204:
2137 		test_cipher_speed("ecb(des)", ENCRYPT, sec, NULL, 0,
2138 				  speed_template_8);
2139 		test_cipher_speed("ecb(des)", DECRYPT, sec, NULL, 0,
2140 				  speed_template_8);
2141 		test_cipher_speed("cbc(des)", ENCRYPT, sec, NULL, 0,
2142 				  speed_template_8);
2143 		test_cipher_speed("cbc(des)", DECRYPT, sec, NULL, 0,
2144 				  speed_template_8);
2145 		break;
2146 
2147 	case 205:
2148 		test_cipher_speed("ecb(camellia)", ENCRYPT, sec, NULL, 0,
2149 				speed_template_16_24_32);
2150 		test_cipher_speed("ecb(camellia)", DECRYPT, sec, NULL, 0,
2151 				speed_template_16_24_32);
2152 		test_cipher_speed("cbc(camellia)", ENCRYPT, sec, NULL, 0,
2153 				speed_template_16_24_32);
2154 		test_cipher_speed("cbc(camellia)", DECRYPT, sec, NULL, 0,
2155 				speed_template_16_24_32);
2156 		test_cipher_speed("ctr(camellia)", ENCRYPT, sec, NULL, 0,
2157 				speed_template_16_24_32);
2158 		test_cipher_speed("ctr(camellia)", DECRYPT, sec, NULL, 0,
2159 				speed_template_16_24_32);
2160 		test_cipher_speed("lrw(camellia)", ENCRYPT, sec, NULL, 0,
2161 				speed_template_32_40_48);
2162 		test_cipher_speed("lrw(camellia)", DECRYPT, sec, NULL, 0,
2163 				speed_template_32_40_48);
2164 		test_cipher_speed("xts(camellia)", ENCRYPT, sec, NULL, 0,
2165 				speed_template_32_48_64);
2166 		test_cipher_speed("xts(camellia)", DECRYPT, sec, NULL, 0,
2167 				speed_template_32_48_64);
2168 		break;
2169 
2170 	case 207:
2171 		test_cipher_speed("ecb(serpent)", ENCRYPT, sec, NULL, 0,
2172 				  speed_template_16_32);
2173 		test_cipher_speed("ecb(serpent)", DECRYPT, sec, NULL, 0,
2174 				  speed_template_16_32);
2175 		test_cipher_speed("cbc(serpent)", ENCRYPT, sec, NULL, 0,
2176 				  speed_template_16_32);
2177 		test_cipher_speed("cbc(serpent)", DECRYPT, sec, NULL, 0,
2178 				  speed_template_16_32);
2179 		test_cipher_speed("ctr(serpent)", ENCRYPT, sec, NULL, 0,
2180 				  speed_template_16_32);
2181 		test_cipher_speed("ctr(serpent)", DECRYPT, sec, NULL, 0,
2182 				  speed_template_16_32);
2183 		test_cipher_speed("lrw(serpent)", ENCRYPT, sec, NULL, 0,
2184 				  speed_template_32_48);
2185 		test_cipher_speed("lrw(serpent)", DECRYPT, sec, NULL, 0,
2186 				  speed_template_32_48);
2187 		test_cipher_speed("xts(serpent)", ENCRYPT, sec, NULL, 0,
2188 				  speed_template_32_64);
2189 		test_cipher_speed("xts(serpent)", DECRYPT, sec, NULL, 0,
2190 				  speed_template_32_64);
2191 		break;
2192 
2193 	case 208:
2194 		test_cipher_speed("ecb(arc4)", ENCRYPT, sec, NULL, 0,
2195 				  speed_template_8);
2196 		break;
2197 
2198 	case 209:
2199 		test_cipher_speed("ecb(cast5)", ENCRYPT, sec, NULL, 0,
2200 				  speed_template_8_16);
2201 		test_cipher_speed("ecb(cast5)", DECRYPT, sec, NULL, 0,
2202 				  speed_template_8_16);
2203 		test_cipher_speed("cbc(cast5)", ENCRYPT, sec, NULL, 0,
2204 				  speed_template_8_16);
2205 		test_cipher_speed("cbc(cast5)", DECRYPT, sec, NULL, 0,
2206 				  speed_template_8_16);
2207 		test_cipher_speed("ctr(cast5)", ENCRYPT, sec, NULL, 0,
2208 				  speed_template_8_16);
2209 		test_cipher_speed("ctr(cast5)", DECRYPT, sec, NULL, 0,
2210 				  speed_template_8_16);
2211 		break;
2212 
2213 	case 210:
2214 		test_cipher_speed("ecb(cast6)", ENCRYPT, sec, NULL, 0,
2215 				  speed_template_16_32);
2216 		test_cipher_speed("ecb(cast6)", DECRYPT, sec, NULL, 0,
2217 				  speed_template_16_32);
2218 		test_cipher_speed("cbc(cast6)", ENCRYPT, sec, NULL, 0,
2219 				  speed_template_16_32);
2220 		test_cipher_speed("cbc(cast6)", DECRYPT, sec, NULL, 0,
2221 				  speed_template_16_32);
2222 		test_cipher_speed("ctr(cast6)", ENCRYPT, sec, NULL, 0,
2223 				  speed_template_16_32);
2224 		test_cipher_speed("ctr(cast6)", DECRYPT, sec, NULL, 0,
2225 				  speed_template_16_32);
2226 		test_cipher_speed("lrw(cast6)", ENCRYPT, sec, NULL, 0,
2227 				  speed_template_32_48);
2228 		test_cipher_speed("lrw(cast6)", DECRYPT, sec, NULL, 0,
2229 				  speed_template_32_48);
2230 		test_cipher_speed("xts(cast6)", ENCRYPT, sec, NULL, 0,
2231 				  speed_template_32_64);
2232 		test_cipher_speed("xts(cast6)", DECRYPT, sec, NULL, 0,
2233 				  speed_template_32_64);
2234 		break;
2235 
2236 	case 211:
2237 		test_aead_speed("rfc4106(gcm(aes))", ENCRYPT, sec,
2238 				NULL, 0, 16, 16, aead_speed_template_20_28_36);
2239 		test_aead_speed("gcm(aes)", ENCRYPT, sec,
2240 				NULL, 0, 16, 8, speed_template_16_24_32);
2241 		test_aead_speed("rfc4106(gcm(aes))", DECRYPT, sec,
2242 				NULL, 0, 16, 16, aead_speed_template_20_28_36);
2243 		test_aead_speed("gcm(aes)", DECRYPT, sec,
2244 				NULL, 0, 16, 8, speed_template_16_24_32);
2245 		break;
2246 
2247 	case 212:
2248 		test_aead_speed("rfc4309(ccm(aes))", ENCRYPT, sec,
2249 				NULL, 0, 16, 16, aead_speed_template_19);
2250 		test_aead_speed("rfc4309(ccm(aes))", DECRYPT, sec,
2251 				NULL, 0, 16, 16, aead_speed_template_19);
2252 		break;
2253 
2254 	case 213:
2255 		test_aead_speed("rfc7539esp(chacha20,poly1305)", ENCRYPT, sec,
2256 				NULL, 0, 16, 8, aead_speed_template_36);
2257 		test_aead_speed("rfc7539esp(chacha20,poly1305)", DECRYPT, sec,
2258 				NULL, 0, 16, 8, aead_speed_template_36);
2259 		break;
2260 
2261 	case 214:
2262 		test_cipher_speed("chacha20", ENCRYPT, sec, NULL, 0,
2263 				  speed_template_32);
2264 		break;
2265 
2266 	case 215:
2267 		test_mb_aead_speed("rfc4106(gcm(aes))", ENCRYPT, sec, NULL,
2268 				   0, 16, 16, aead_speed_template_20_28_36, num_mb);
2269 		test_mb_aead_speed("gcm(aes)", ENCRYPT, sec, NULL, 0, 16, 8,
2270 				   speed_template_16_24_32, num_mb);
2271 		test_mb_aead_speed("rfc4106(gcm(aes))", DECRYPT, sec, NULL,
2272 				   0, 16, 16, aead_speed_template_20_28_36, num_mb);
2273 		test_mb_aead_speed("gcm(aes)", DECRYPT, sec, NULL, 0, 16, 8,
2274 				   speed_template_16_24_32, num_mb);
2275 		break;
2276 
2277 	case 216:
2278 		test_mb_aead_speed("rfc4309(ccm(aes))", ENCRYPT, sec, NULL, 0,
2279 				   16, 16, aead_speed_template_19, num_mb);
2280 		test_mb_aead_speed("rfc4309(ccm(aes))", DECRYPT, sec, NULL, 0,
2281 				   16, 16, aead_speed_template_19, num_mb);
2282 		break;
2283 
2284 	case 217:
2285 		test_mb_aead_speed("rfc7539esp(chacha20,poly1305)", ENCRYPT,
2286 				   sec, NULL, 0, 16, 8, aead_speed_template_36,
2287 				   num_mb);
2288 		test_mb_aead_speed("rfc7539esp(chacha20,poly1305)", DECRYPT,
2289 				   sec, NULL, 0, 16, 8, aead_speed_template_36,
2290 				   num_mb);
2291 		break;
2292 
2293 	case 218:
2294 		test_cipher_speed("ecb(sm4)", ENCRYPT, sec, NULL, 0,
2295 				speed_template_16);
2296 		test_cipher_speed("ecb(sm4)", DECRYPT, sec, NULL, 0,
2297 				speed_template_16);
2298 		test_cipher_speed("cbc(sm4)", ENCRYPT, sec, NULL, 0,
2299 				speed_template_16);
2300 		test_cipher_speed("cbc(sm4)", DECRYPT, sec, NULL, 0,
2301 				speed_template_16);
2302 		test_cipher_speed("cts(cbc(sm4))", ENCRYPT, sec, NULL, 0,
2303 				speed_template_16);
2304 		test_cipher_speed("cts(cbc(sm4))", DECRYPT, sec, NULL, 0,
2305 				speed_template_16);
2306 		test_cipher_speed("ctr(sm4)", ENCRYPT, sec, NULL, 0,
2307 				speed_template_16);
2308 		test_cipher_speed("ctr(sm4)", DECRYPT, sec, NULL, 0,
2309 				speed_template_16);
2310 		test_cipher_speed("xts(sm4)", ENCRYPT, sec, NULL, 0,
2311 				speed_template_32);
2312 		test_cipher_speed("xts(sm4)", DECRYPT, sec, NULL, 0,
2313 				speed_template_32);
2314 		break;
2315 
2316 	case 219:
2317 		test_cipher_speed("adiantum(xchacha12,aes)", ENCRYPT, sec, NULL,
2318 				  0, speed_template_32);
2319 		test_cipher_speed("adiantum(xchacha12,aes)", DECRYPT, sec, NULL,
2320 				  0, speed_template_32);
2321 		test_cipher_speed("adiantum(xchacha20,aes)", ENCRYPT, sec, NULL,
2322 				  0, speed_template_32);
2323 		test_cipher_speed("adiantum(xchacha20,aes)", DECRYPT, sec, NULL,
2324 				  0, speed_template_32);
2325 		break;
2326 
2327 	case 220:
2328 		test_acipher_speed("essiv(cbc(aes),sha256)",
2329 				  ENCRYPT, sec, NULL, 0,
2330 				  speed_template_16_24_32);
2331 		test_acipher_speed("essiv(cbc(aes),sha256)",
2332 				  DECRYPT, sec, NULL, 0,
2333 				  speed_template_16_24_32);
2334 		break;
2335 
2336 	case 221:
2337 		test_aead_speed("aegis128", ENCRYPT, sec,
2338 				NULL, 0, 16, 8, speed_template_16);
2339 		test_aead_speed("aegis128", DECRYPT, sec,
2340 				NULL, 0, 16, 8, speed_template_16);
2341 		break;
2342 
2343 	case 222:
2344 		test_aead_speed("gcm(sm4)", ENCRYPT, sec,
2345 				NULL, 0, 16, 8, speed_template_16);
2346 		test_aead_speed("gcm(sm4)", DECRYPT, sec,
2347 				NULL, 0, 16, 8, speed_template_16);
2348 		break;
2349 
2350 	case 223:
2351 		test_aead_speed("rfc4309(ccm(sm4))", ENCRYPT, sec,
2352 				NULL, 0, 16, 16, aead_speed_template_19);
2353 		test_aead_speed("rfc4309(ccm(sm4))", DECRYPT, sec,
2354 				NULL, 0, 16, 16, aead_speed_template_19);
2355 		break;
2356 
2357 	case 224:
2358 		test_mb_aead_speed("gcm(sm4)", ENCRYPT, sec, NULL, 0, 16, 8,
2359 				   speed_template_16, num_mb);
2360 		test_mb_aead_speed("gcm(sm4)", DECRYPT, sec, NULL, 0, 16, 8,
2361 				   speed_template_16, num_mb);
2362 		break;
2363 
2364 	case 225:
2365 		test_mb_aead_speed("rfc4309(ccm(sm4))", ENCRYPT, sec, NULL, 0,
2366 				   16, 16, aead_speed_template_19, num_mb);
2367 		test_mb_aead_speed("rfc4309(ccm(sm4))", DECRYPT, sec, NULL, 0,
2368 				   16, 16, aead_speed_template_19, num_mb);
2369 		break;
2370 
2371 	case 226:
2372 		test_cipher_speed("hctr2(aes)", ENCRYPT, sec, NULL,
2373 				  0, speed_template_32);
2374 		break;
2375 
2376 	case 227:
2377 		test_cipher_speed("ecb(aria)", ENCRYPT, sec, NULL, 0,
2378 				  speed_template_16_24_32);
2379 		test_cipher_speed("ecb(aria)", DECRYPT, sec, NULL, 0,
2380 				  speed_template_16_24_32);
2381 		test_cipher_speed("cbc(aria)", ENCRYPT, sec, NULL, 0,
2382 				  speed_template_16_24_32);
2383 		test_cipher_speed("cbc(aria)", DECRYPT, sec, NULL, 0,
2384 				  speed_template_16_24_32);
2385 		test_cipher_speed("ctr(aria)", ENCRYPT, sec, NULL, 0,
2386 				  speed_template_16_24_32);
2387 		test_cipher_speed("ctr(aria)", DECRYPT, sec, NULL, 0,
2388 				  speed_template_16_24_32);
2389 		break;
2390 
2391 	case 228:
2392 		test_aead_speed("gcm(aria)", ENCRYPT, sec,
2393 				NULL, 0, 16, 8, speed_template_16_24_32);
2394 		test_aead_speed("gcm(aria)", DECRYPT, sec,
2395 				NULL, 0, 16, 8, speed_template_16_24_32);
2396 		break;
2397 
2398 	case 229:
2399 		test_mb_aead_speed("gcm(aria)", ENCRYPT, sec, NULL, 0, 16, 8,
2400 				   speed_template_16, num_mb);
2401 		test_mb_aead_speed("gcm(aria)", DECRYPT, sec, NULL, 0, 16, 8,
2402 				   speed_template_16, num_mb);
2403 		break;
2404 
2405 	case 300:
2406 		if (alg) {
2407 			test_hash_speed(alg, sec, generic_hash_speed_template);
2408 			break;
2409 		}
2410 		fallthrough;
2411 	case 301:
2412 		test_hash_speed("md4", sec, generic_hash_speed_template);
2413 		if (mode > 300 && mode < 400) break;
2414 		fallthrough;
2415 	case 302:
2416 		test_hash_speed("md5", sec, generic_hash_speed_template);
2417 		if (mode > 300 && mode < 400) break;
2418 		fallthrough;
2419 	case 303:
2420 		test_hash_speed("sha1", sec, generic_hash_speed_template);
2421 		if (mode > 300 && mode < 400) break;
2422 		fallthrough;
2423 	case 304:
2424 		test_hash_speed("sha256", sec, generic_hash_speed_template);
2425 		if (mode > 300 && mode < 400) break;
2426 		fallthrough;
2427 	case 305:
2428 		test_hash_speed("sha384", sec, generic_hash_speed_template);
2429 		if (mode > 300 && mode < 400) break;
2430 		fallthrough;
2431 	case 306:
2432 		test_hash_speed("sha512", sec, generic_hash_speed_template);
2433 		if (mode > 300 && mode < 400) break;
2434 		fallthrough;
2435 	case 307:
2436 		test_hash_speed("wp256", sec, generic_hash_speed_template);
2437 		if (mode > 300 && mode < 400) break;
2438 		fallthrough;
2439 	case 308:
2440 		test_hash_speed("wp384", sec, generic_hash_speed_template);
2441 		if (mode > 300 && mode < 400) break;
2442 		fallthrough;
2443 	case 309:
2444 		test_hash_speed("wp512", sec, generic_hash_speed_template);
2445 		if (mode > 300 && mode < 400) break;
2446 		fallthrough;
2447 	case 313:
2448 		test_hash_speed("sha224", sec, generic_hash_speed_template);
2449 		if (mode > 300 && mode < 400) break;
2450 		fallthrough;
2451 	case 314:
2452 		test_hash_speed("xxhash64", sec, generic_hash_speed_template);
2453 		if (mode > 300 && mode < 400) break;
2454 		fallthrough;
2455 	case 315:
2456 		test_hash_speed("rmd160", sec, generic_hash_speed_template);
2457 		if (mode > 300 && mode < 400) break;
2458 		fallthrough;
2459 	case 317:
2460 		test_hash_speed("blake2b-512", sec, generic_hash_speed_template);
2461 		if (mode > 300 && mode < 400) break;
2462 		fallthrough;
2463 	case 318:
2464 		klen = 16;
2465 		test_hash_speed("ghash", sec, generic_hash_speed_template);
2466 		if (mode > 300 && mode < 400) break;
2467 		fallthrough;
2468 	case 319:
2469 		test_hash_speed("crc32c", sec, generic_hash_speed_template);
2470 		if (mode > 300 && mode < 400) break;
2471 		fallthrough;
2472 	case 321:
2473 		test_hash_speed("poly1305", sec, poly1305_speed_template);
2474 		if (mode > 300 && mode < 400) break;
2475 		fallthrough;
2476 	case 322:
2477 		test_hash_speed("sha3-224", sec, generic_hash_speed_template);
2478 		if (mode > 300 && mode < 400) break;
2479 		fallthrough;
2480 	case 323:
2481 		test_hash_speed("sha3-256", sec, generic_hash_speed_template);
2482 		if (mode > 300 && mode < 400) break;
2483 		fallthrough;
2484 	case 324:
2485 		test_hash_speed("sha3-384", sec, generic_hash_speed_template);
2486 		if (mode > 300 && mode < 400) break;
2487 		fallthrough;
2488 	case 325:
2489 		test_hash_speed("sha3-512", sec, generic_hash_speed_template);
2490 		if (mode > 300 && mode < 400) break;
2491 		fallthrough;
2492 	case 326:
2493 		test_hash_speed("sm3", sec, generic_hash_speed_template);
2494 		if (mode > 300 && mode < 400) break;
2495 		fallthrough;
2496 	case 327:
2497 		test_hash_speed("streebog256", sec,
2498 				generic_hash_speed_template);
2499 		if (mode > 300 && mode < 400) break;
2500 		fallthrough;
2501 	case 328:
2502 		test_hash_speed("streebog512", sec,
2503 				generic_hash_speed_template);
2504 		if (mode > 300 && mode < 400) break;
2505 		fallthrough;
2506 	case 399:
2507 		break;
2508 
2509 	case 400:
2510 		if (alg) {
2511 			test_ahash_speed(alg, sec, generic_hash_speed_template);
2512 			break;
2513 		}
2514 		fallthrough;
2515 	case 401:
2516 		test_ahash_speed("md4", sec, generic_hash_speed_template);
2517 		if (mode > 400 && mode < 500) break;
2518 		fallthrough;
2519 	case 402:
2520 		test_ahash_speed("md5", sec, generic_hash_speed_template);
2521 		if (mode > 400 && mode < 500) break;
2522 		fallthrough;
2523 	case 403:
2524 		test_ahash_speed("sha1", sec, generic_hash_speed_template);
2525 		if (mode > 400 && mode < 500) break;
2526 		fallthrough;
2527 	case 404:
2528 		test_ahash_speed("sha256", sec, generic_hash_speed_template);
2529 		if (mode > 400 && mode < 500) break;
2530 		fallthrough;
2531 	case 405:
2532 		test_ahash_speed("sha384", sec, generic_hash_speed_template);
2533 		if (mode > 400 && mode < 500) break;
2534 		fallthrough;
2535 	case 406:
2536 		test_ahash_speed("sha512", sec, generic_hash_speed_template);
2537 		if (mode > 400 && mode < 500) break;
2538 		fallthrough;
2539 	case 407:
2540 		test_ahash_speed("wp256", sec, generic_hash_speed_template);
2541 		if (mode > 400 && mode < 500) break;
2542 		fallthrough;
2543 	case 408:
2544 		test_ahash_speed("wp384", sec, generic_hash_speed_template);
2545 		if (mode > 400 && mode < 500) break;
2546 		fallthrough;
2547 	case 409:
2548 		test_ahash_speed("wp512", sec, generic_hash_speed_template);
2549 		if (mode > 400 && mode < 500) break;
2550 		fallthrough;
2551 	case 413:
2552 		test_ahash_speed("sha224", sec, generic_hash_speed_template);
2553 		if (mode > 400 && mode < 500) break;
2554 		fallthrough;
2555 	case 414:
2556 		test_ahash_speed("xxhash64", sec, generic_hash_speed_template);
2557 		if (mode > 400 && mode < 500) break;
2558 		fallthrough;
2559 	case 415:
2560 		test_ahash_speed("rmd160", sec, generic_hash_speed_template);
2561 		if (mode > 400 && mode < 500) break;
2562 		fallthrough;
2563 	case 417:
2564 		test_ahash_speed("blake2b-512", sec, generic_hash_speed_template);
2565 		if (mode > 400 && mode < 500) break;
2566 		fallthrough;
2567 	case 418:
2568 		test_ahash_speed("sha3-224", sec, generic_hash_speed_template);
2569 		if (mode > 400 && mode < 500) break;
2570 		fallthrough;
2571 	case 419:
2572 		test_ahash_speed("sha3-256", sec, generic_hash_speed_template);
2573 		if (mode > 400 && mode < 500) break;
2574 		fallthrough;
2575 	case 420:
2576 		test_ahash_speed("sha3-384", sec, generic_hash_speed_template);
2577 		if (mode > 400 && mode < 500) break;
2578 		fallthrough;
2579 	case 421:
2580 		test_ahash_speed("sha3-512", sec, generic_hash_speed_template);
2581 		if (mode > 400 && mode < 500) break;
2582 		fallthrough;
2583 	case 422:
2584 		test_ahash_speed("sm3", sec, generic_hash_speed_template);
2585 		if (mode > 400 && mode < 500) break;
2586 		fallthrough;
2587 	case 450:
2588 		test_mb_ahash_speed("sha1", sec, generic_hash_speed_template,
2589 				    num_mb);
2590 		if (mode > 400 && mode < 500) break;
2591 		fallthrough;
2592 	case 451:
2593 		test_mb_ahash_speed("sha256", sec, generic_hash_speed_template,
2594 				    num_mb);
2595 		if (mode > 400 && mode < 500) break;
2596 		fallthrough;
2597 	case 452:
2598 		test_mb_ahash_speed("sha512", sec, generic_hash_speed_template,
2599 				    num_mb);
2600 		if (mode > 400 && mode < 500) break;
2601 		fallthrough;
2602 	case 453:
2603 		test_mb_ahash_speed("sm3", sec, generic_hash_speed_template,
2604 				    num_mb);
2605 		if (mode > 400 && mode < 500) break;
2606 		fallthrough;
2607 	case 454:
2608 		test_mb_ahash_speed("streebog256", sec,
2609 				    generic_hash_speed_template, num_mb);
2610 		if (mode > 400 && mode < 500) break;
2611 		fallthrough;
2612 	case 455:
2613 		test_mb_ahash_speed("streebog512", sec,
2614 				    generic_hash_speed_template, num_mb);
2615 		if (mode > 400 && mode < 500) break;
2616 		fallthrough;
2617 	case 499:
2618 		break;
2619 
2620 	case 500:
2621 		test_acipher_speed("ecb(aes)", ENCRYPT, sec, NULL, 0,
2622 				   speed_template_16_24_32);
2623 		test_acipher_speed("ecb(aes)", DECRYPT, sec, NULL, 0,
2624 				   speed_template_16_24_32);
2625 		test_acipher_speed("cbc(aes)", ENCRYPT, sec, NULL, 0,
2626 				   speed_template_16_24_32);
2627 		test_acipher_speed("cbc(aes)", DECRYPT, sec, NULL, 0,
2628 				   speed_template_16_24_32);
2629 		test_acipher_speed("lrw(aes)", ENCRYPT, sec, NULL, 0,
2630 				   speed_template_32_40_48);
2631 		test_acipher_speed("lrw(aes)", DECRYPT, sec, NULL, 0,
2632 				   speed_template_32_40_48);
2633 		test_acipher_speed("xts(aes)", ENCRYPT, sec, NULL, 0,
2634 				   speed_template_32_64);
2635 		test_acipher_speed("xts(aes)", DECRYPT, sec, NULL, 0,
2636 				   speed_template_32_64);
2637 		test_acipher_speed("cts(cbc(aes))", ENCRYPT, sec, NULL, 0,
2638 				   speed_template_16_24_32);
2639 		test_acipher_speed("cts(cbc(aes))", DECRYPT, sec, NULL, 0,
2640 				   speed_template_16_24_32);
2641 		test_acipher_speed("ctr(aes)", ENCRYPT, sec, NULL, 0,
2642 				   speed_template_16_24_32);
2643 		test_acipher_speed("ctr(aes)", DECRYPT, sec, NULL, 0,
2644 				   speed_template_16_24_32);
2645 		test_acipher_speed("rfc3686(ctr(aes))", ENCRYPT, sec, NULL, 0,
2646 				   speed_template_20_28_36);
2647 		test_acipher_speed("rfc3686(ctr(aes))", DECRYPT, sec, NULL, 0,
2648 				   speed_template_20_28_36);
2649 		break;
2650 
2651 	case 501:
2652 		test_acipher_speed("ecb(des3_ede)", ENCRYPT, sec,
2653 				   des3_speed_template, DES3_SPEED_VECTORS,
2654 				   speed_template_24);
2655 		test_acipher_speed("ecb(des3_ede)", DECRYPT, sec,
2656 				   des3_speed_template, DES3_SPEED_VECTORS,
2657 				   speed_template_24);
2658 		test_acipher_speed("cbc(des3_ede)", ENCRYPT, sec,
2659 				   des3_speed_template, DES3_SPEED_VECTORS,
2660 				   speed_template_24);
2661 		test_acipher_speed("cbc(des3_ede)", DECRYPT, sec,
2662 				   des3_speed_template, DES3_SPEED_VECTORS,
2663 				   speed_template_24);
2664 		break;
2665 
2666 	case 502:
2667 		test_acipher_speed("ecb(des)", ENCRYPT, sec, NULL, 0,
2668 				   speed_template_8);
2669 		test_acipher_speed("ecb(des)", DECRYPT, sec, NULL, 0,
2670 				   speed_template_8);
2671 		test_acipher_speed("cbc(des)", ENCRYPT, sec, NULL, 0,
2672 				   speed_template_8);
2673 		test_acipher_speed("cbc(des)", DECRYPT, sec, NULL, 0,
2674 				   speed_template_8);
2675 		break;
2676 
2677 	case 503:
2678 		test_acipher_speed("ecb(serpent)", ENCRYPT, sec, NULL, 0,
2679 				   speed_template_16_32);
2680 		test_acipher_speed("ecb(serpent)", DECRYPT, sec, NULL, 0,
2681 				   speed_template_16_32);
2682 		test_acipher_speed("cbc(serpent)", ENCRYPT, sec, NULL, 0,
2683 				   speed_template_16_32);
2684 		test_acipher_speed("cbc(serpent)", DECRYPT, sec, NULL, 0,
2685 				   speed_template_16_32);
2686 		test_acipher_speed("ctr(serpent)", ENCRYPT, sec, NULL, 0,
2687 				   speed_template_16_32);
2688 		test_acipher_speed("ctr(serpent)", DECRYPT, sec, NULL, 0,
2689 				   speed_template_16_32);
2690 		test_acipher_speed("lrw(serpent)", ENCRYPT, sec, NULL, 0,
2691 				   speed_template_32_48);
2692 		test_acipher_speed("lrw(serpent)", DECRYPT, sec, NULL, 0,
2693 				   speed_template_32_48);
2694 		test_acipher_speed("xts(serpent)", ENCRYPT, sec, NULL, 0,
2695 				   speed_template_32_64);
2696 		test_acipher_speed("xts(serpent)", DECRYPT, sec, NULL, 0,
2697 				   speed_template_32_64);
2698 		break;
2699 
2700 	case 504:
2701 		test_acipher_speed("ecb(twofish)", ENCRYPT, sec, NULL, 0,
2702 				   speed_template_16_24_32);
2703 		test_acipher_speed("ecb(twofish)", DECRYPT, sec, NULL, 0,
2704 				   speed_template_16_24_32);
2705 		test_acipher_speed("cbc(twofish)", ENCRYPT, sec, NULL, 0,
2706 				   speed_template_16_24_32);
2707 		test_acipher_speed("cbc(twofish)", DECRYPT, sec, NULL, 0,
2708 				   speed_template_16_24_32);
2709 		test_acipher_speed("ctr(twofish)", ENCRYPT, sec, NULL, 0,
2710 				   speed_template_16_24_32);
2711 		test_acipher_speed("ctr(twofish)", DECRYPT, sec, NULL, 0,
2712 				   speed_template_16_24_32);
2713 		test_acipher_speed("lrw(twofish)", ENCRYPT, sec, NULL, 0,
2714 				   speed_template_32_40_48);
2715 		test_acipher_speed("lrw(twofish)", DECRYPT, sec, NULL, 0,
2716 				   speed_template_32_40_48);
2717 		test_acipher_speed("xts(twofish)", ENCRYPT, sec, NULL, 0,
2718 				   speed_template_32_48_64);
2719 		test_acipher_speed("xts(twofish)", DECRYPT, sec, NULL, 0,
2720 				   speed_template_32_48_64);
2721 		break;
2722 
2723 	case 505:
2724 		test_acipher_speed("ecb(arc4)", ENCRYPT, sec, NULL, 0,
2725 				   speed_template_8);
2726 		break;
2727 
2728 	case 506:
2729 		test_acipher_speed("ecb(cast5)", ENCRYPT, sec, NULL, 0,
2730 				   speed_template_8_16);
2731 		test_acipher_speed("ecb(cast5)", DECRYPT, sec, NULL, 0,
2732 				   speed_template_8_16);
2733 		test_acipher_speed("cbc(cast5)", ENCRYPT, sec, NULL, 0,
2734 				   speed_template_8_16);
2735 		test_acipher_speed("cbc(cast5)", DECRYPT, sec, NULL, 0,
2736 				   speed_template_8_16);
2737 		test_acipher_speed("ctr(cast5)", ENCRYPT, sec, NULL, 0,
2738 				   speed_template_8_16);
2739 		test_acipher_speed("ctr(cast5)", DECRYPT, sec, NULL, 0,
2740 				   speed_template_8_16);
2741 		break;
2742 
2743 	case 507:
2744 		test_acipher_speed("ecb(cast6)", ENCRYPT, sec, NULL, 0,
2745 				   speed_template_16_32);
2746 		test_acipher_speed("ecb(cast6)", DECRYPT, sec, NULL, 0,
2747 				   speed_template_16_32);
2748 		test_acipher_speed("cbc(cast6)", ENCRYPT, sec, NULL, 0,
2749 				   speed_template_16_32);
2750 		test_acipher_speed("cbc(cast6)", DECRYPT, sec, NULL, 0,
2751 				   speed_template_16_32);
2752 		test_acipher_speed("ctr(cast6)", ENCRYPT, sec, NULL, 0,
2753 				   speed_template_16_32);
2754 		test_acipher_speed("ctr(cast6)", DECRYPT, sec, NULL, 0,
2755 				   speed_template_16_32);
2756 		test_acipher_speed("lrw(cast6)", ENCRYPT, sec, NULL, 0,
2757 				   speed_template_32_48);
2758 		test_acipher_speed("lrw(cast6)", DECRYPT, sec, NULL, 0,
2759 				   speed_template_32_48);
2760 		test_acipher_speed("xts(cast6)", ENCRYPT, sec, NULL, 0,
2761 				   speed_template_32_64);
2762 		test_acipher_speed("xts(cast6)", DECRYPT, sec, NULL, 0,
2763 				   speed_template_32_64);
2764 		break;
2765 
2766 	case 508:
2767 		test_acipher_speed("ecb(camellia)", ENCRYPT, sec, NULL, 0,
2768 				   speed_template_16_32);
2769 		test_acipher_speed("ecb(camellia)", DECRYPT, sec, NULL, 0,
2770 				   speed_template_16_32);
2771 		test_acipher_speed("cbc(camellia)", ENCRYPT, sec, NULL, 0,
2772 				   speed_template_16_32);
2773 		test_acipher_speed("cbc(camellia)", DECRYPT, sec, NULL, 0,
2774 				   speed_template_16_32);
2775 		test_acipher_speed("ctr(camellia)", ENCRYPT, sec, NULL, 0,
2776 				   speed_template_16_32);
2777 		test_acipher_speed("ctr(camellia)", DECRYPT, sec, NULL, 0,
2778 				   speed_template_16_32);
2779 		test_acipher_speed("lrw(camellia)", ENCRYPT, sec, NULL, 0,
2780 				   speed_template_32_48);
2781 		test_acipher_speed("lrw(camellia)", DECRYPT, sec, NULL, 0,
2782 				   speed_template_32_48);
2783 		test_acipher_speed("xts(camellia)", ENCRYPT, sec, NULL, 0,
2784 				   speed_template_32_64);
2785 		test_acipher_speed("xts(camellia)", DECRYPT, sec, NULL, 0,
2786 				   speed_template_32_64);
2787 		break;
2788 
2789 	case 509:
2790 		test_acipher_speed("ecb(blowfish)", ENCRYPT, sec, NULL, 0,
2791 				   speed_template_8_32);
2792 		test_acipher_speed("ecb(blowfish)", DECRYPT, sec, NULL, 0,
2793 				   speed_template_8_32);
2794 		test_acipher_speed("cbc(blowfish)", ENCRYPT, sec, NULL, 0,
2795 				   speed_template_8_32);
2796 		test_acipher_speed("cbc(blowfish)", DECRYPT, sec, NULL, 0,
2797 				   speed_template_8_32);
2798 		test_acipher_speed("ctr(blowfish)", ENCRYPT, sec, NULL, 0,
2799 				   speed_template_8_32);
2800 		test_acipher_speed("ctr(blowfish)", DECRYPT, sec, NULL, 0,
2801 				   speed_template_8_32);
2802 		break;
2803 
2804 	case 518:
2805 		test_acipher_speed("ecb(sm4)", ENCRYPT, sec, NULL, 0,
2806 				speed_template_16);
2807 		test_acipher_speed("ecb(sm4)", DECRYPT, sec, NULL, 0,
2808 				speed_template_16);
2809 		test_acipher_speed("cbc(sm4)", ENCRYPT, sec, NULL, 0,
2810 				speed_template_16);
2811 		test_acipher_speed("cbc(sm4)", DECRYPT, sec, NULL, 0,
2812 				speed_template_16);
2813 		test_acipher_speed("ctr(sm4)", ENCRYPT, sec, NULL, 0,
2814 				speed_template_16);
2815 		test_acipher_speed("ctr(sm4)", DECRYPT, sec, NULL, 0,
2816 				speed_template_16);
2817 		test_acipher_speed("xts(sm4)", ENCRYPT, sec, NULL, 0,
2818 				speed_template_32);
2819 		test_acipher_speed("xts(sm4)", DECRYPT, sec, NULL, 0,
2820 				speed_template_32);
2821 		break;
2822 
2823 	case 519:
2824 		test_acipher_speed("ecb(aria)", ENCRYPT, sec, NULL, 0,
2825 				   speed_template_16_24_32);
2826 		test_acipher_speed("ecb(aria)", DECRYPT, sec, NULL, 0,
2827 				   speed_template_16_24_32);
2828 		test_acipher_speed("ctr(aria)", ENCRYPT, sec, NULL, 0,
2829 				   speed_template_16_24_32);
2830 		test_acipher_speed("ctr(aria)", DECRYPT, sec, NULL, 0,
2831 				   speed_template_16_24_32);
2832 		break;
2833 
2834 	case 600:
2835 		if (alg) {
2836 			u8 speed_template[2] = {klen, 0};
2837 			test_mb_skcipher_speed(alg, ENCRYPT, sec, NULL, 0,
2838 					       speed_template, num_mb);
2839 			test_mb_skcipher_speed(alg, DECRYPT, sec, NULL, 0,
2840 					       speed_template, num_mb);
2841 			break;
2842 		}
2843 
2844 		test_mb_skcipher_speed("ecb(aes)", ENCRYPT, sec, NULL, 0,
2845 				       speed_template_16_24_32, num_mb);
2846 		test_mb_skcipher_speed("ecb(aes)", DECRYPT, sec, NULL, 0,
2847 				       speed_template_16_24_32, num_mb);
2848 		test_mb_skcipher_speed("cbc(aes)", ENCRYPT, sec, NULL, 0,
2849 				       speed_template_16_24_32, num_mb);
2850 		test_mb_skcipher_speed("cbc(aes)", DECRYPT, sec, NULL, 0,
2851 				       speed_template_16_24_32, num_mb);
2852 		test_mb_skcipher_speed("lrw(aes)", ENCRYPT, sec, NULL, 0,
2853 				       speed_template_32_40_48, num_mb);
2854 		test_mb_skcipher_speed("lrw(aes)", DECRYPT, sec, NULL, 0,
2855 				       speed_template_32_40_48, num_mb);
2856 		test_mb_skcipher_speed("xts(aes)", ENCRYPT, sec, NULL, 0,
2857 				       speed_template_32_64, num_mb);
2858 		test_mb_skcipher_speed("xts(aes)", DECRYPT, sec, NULL, 0,
2859 				       speed_template_32_64, num_mb);
2860 		test_mb_skcipher_speed("cts(cbc(aes))", ENCRYPT, sec, NULL, 0,
2861 				       speed_template_16_24_32, num_mb);
2862 		test_mb_skcipher_speed("cts(cbc(aes))", DECRYPT, sec, NULL, 0,
2863 				       speed_template_16_24_32, num_mb);
2864 		test_mb_skcipher_speed("ctr(aes)", ENCRYPT, sec, NULL, 0,
2865 				       speed_template_16_24_32, num_mb);
2866 		test_mb_skcipher_speed("ctr(aes)", DECRYPT, sec, NULL, 0,
2867 				       speed_template_16_24_32, num_mb);
2868 		test_mb_skcipher_speed("rfc3686(ctr(aes))", ENCRYPT, sec, NULL,
2869 				       0, speed_template_20_28_36, num_mb);
2870 		test_mb_skcipher_speed("rfc3686(ctr(aes))", DECRYPT, sec, NULL,
2871 				       0, speed_template_20_28_36, num_mb);
2872 		break;
2873 
2874 	case 601:
2875 		test_mb_skcipher_speed("ecb(des3_ede)", ENCRYPT, sec,
2876 				       des3_speed_template, DES3_SPEED_VECTORS,
2877 				       speed_template_24, num_mb);
2878 		test_mb_skcipher_speed("ecb(des3_ede)", DECRYPT, sec,
2879 				       des3_speed_template, DES3_SPEED_VECTORS,
2880 				       speed_template_24, num_mb);
2881 		test_mb_skcipher_speed("cbc(des3_ede)", ENCRYPT, sec,
2882 				       des3_speed_template, DES3_SPEED_VECTORS,
2883 				       speed_template_24, num_mb);
2884 		test_mb_skcipher_speed("cbc(des3_ede)", DECRYPT, sec,
2885 				       des3_speed_template, DES3_SPEED_VECTORS,
2886 				       speed_template_24, num_mb);
2887 		break;
2888 
2889 	case 602:
2890 		test_mb_skcipher_speed("ecb(des)", ENCRYPT, sec, NULL, 0,
2891 				       speed_template_8, num_mb);
2892 		test_mb_skcipher_speed("ecb(des)", DECRYPT, sec, NULL, 0,
2893 				       speed_template_8, num_mb);
2894 		test_mb_skcipher_speed("cbc(des)", ENCRYPT, sec, NULL, 0,
2895 				       speed_template_8, num_mb);
2896 		test_mb_skcipher_speed("cbc(des)", DECRYPT, sec, NULL, 0,
2897 				       speed_template_8, num_mb);
2898 		break;
2899 
2900 	case 603:
2901 		test_mb_skcipher_speed("ecb(serpent)", ENCRYPT, sec, NULL, 0,
2902 				       speed_template_16_32, num_mb);
2903 		test_mb_skcipher_speed("ecb(serpent)", DECRYPT, sec, NULL, 0,
2904 				       speed_template_16_32, num_mb);
2905 		test_mb_skcipher_speed("cbc(serpent)", ENCRYPT, sec, NULL, 0,
2906 				       speed_template_16_32, num_mb);
2907 		test_mb_skcipher_speed("cbc(serpent)", DECRYPT, sec, NULL, 0,
2908 				       speed_template_16_32, num_mb);
2909 		test_mb_skcipher_speed("ctr(serpent)", ENCRYPT, sec, NULL, 0,
2910 				       speed_template_16_32, num_mb);
2911 		test_mb_skcipher_speed("ctr(serpent)", DECRYPT, sec, NULL, 0,
2912 				       speed_template_16_32, num_mb);
2913 		test_mb_skcipher_speed("lrw(serpent)", ENCRYPT, sec, NULL, 0,
2914 				       speed_template_32_48, num_mb);
2915 		test_mb_skcipher_speed("lrw(serpent)", DECRYPT, sec, NULL, 0,
2916 				       speed_template_32_48, num_mb);
2917 		test_mb_skcipher_speed("xts(serpent)", ENCRYPT, sec, NULL, 0,
2918 				       speed_template_32_64, num_mb);
2919 		test_mb_skcipher_speed("xts(serpent)", DECRYPT, sec, NULL, 0,
2920 				       speed_template_32_64, num_mb);
2921 		break;
2922 
2923 	case 604:
2924 		test_mb_skcipher_speed("ecb(twofish)", ENCRYPT, sec, NULL, 0,
2925 				       speed_template_16_24_32, num_mb);
2926 		test_mb_skcipher_speed("ecb(twofish)", DECRYPT, sec, NULL, 0,
2927 				       speed_template_16_24_32, num_mb);
2928 		test_mb_skcipher_speed("cbc(twofish)", ENCRYPT, sec, NULL, 0,
2929 				       speed_template_16_24_32, num_mb);
2930 		test_mb_skcipher_speed("cbc(twofish)", DECRYPT, sec, NULL, 0,
2931 				       speed_template_16_24_32, num_mb);
2932 		test_mb_skcipher_speed("ctr(twofish)", ENCRYPT, sec, NULL, 0,
2933 				       speed_template_16_24_32, num_mb);
2934 		test_mb_skcipher_speed("ctr(twofish)", DECRYPT, sec, NULL, 0,
2935 				       speed_template_16_24_32, num_mb);
2936 		test_mb_skcipher_speed("lrw(twofish)", ENCRYPT, sec, NULL, 0,
2937 				       speed_template_32_40_48, num_mb);
2938 		test_mb_skcipher_speed("lrw(twofish)", DECRYPT, sec, NULL, 0,
2939 				       speed_template_32_40_48, num_mb);
2940 		test_mb_skcipher_speed("xts(twofish)", ENCRYPT, sec, NULL, 0,
2941 				       speed_template_32_48_64, num_mb);
2942 		test_mb_skcipher_speed("xts(twofish)", DECRYPT, sec, NULL, 0,
2943 				       speed_template_32_48_64, num_mb);
2944 		break;
2945 
2946 	case 605:
2947 		test_mb_skcipher_speed("ecb(arc4)", ENCRYPT, sec, NULL, 0,
2948 				       speed_template_8, num_mb);
2949 		break;
2950 
2951 	case 606:
2952 		test_mb_skcipher_speed("ecb(cast5)", ENCRYPT, sec, NULL, 0,
2953 				       speed_template_8_16, num_mb);
2954 		test_mb_skcipher_speed("ecb(cast5)", DECRYPT, sec, NULL, 0,
2955 				       speed_template_8_16, num_mb);
2956 		test_mb_skcipher_speed("cbc(cast5)", ENCRYPT, sec, NULL, 0,
2957 				       speed_template_8_16, num_mb);
2958 		test_mb_skcipher_speed("cbc(cast5)", DECRYPT, sec, NULL, 0,
2959 				       speed_template_8_16, num_mb);
2960 		test_mb_skcipher_speed("ctr(cast5)", ENCRYPT, sec, NULL, 0,
2961 				       speed_template_8_16, num_mb);
2962 		test_mb_skcipher_speed("ctr(cast5)", DECRYPT, sec, NULL, 0,
2963 				       speed_template_8_16, num_mb);
2964 		break;
2965 
2966 	case 607:
2967 		test_mb_skcipher_speed("ecb(cast6)", ENCRYPT, sec, NULL, 0,
2968 				       speed_template_16_32, num_mb);
2969 		test_mb_skcipher_speed("ecb(cast6)", DECRYPT, sec, NULL, 0,
2970 				       speed_template_16_32, num_mb);
2971 		test_mb_skcipher_speed("cbc(cast6)", ENCRYPT, sec, NULL, 0,
2972 				       speed_template_16_32, num_mb);
2973 		test_mb_skcipher_speed("cbc(cast6)", DECRYPT, sec, NULL, 0,
2974 				       speed_template_16_32, num_mb);
2975 		test_mb_skcipher_speed("ctr(cast6)", ENCRYPT, sec, NULL, 0,
2976 				       speed_template_16_32, num_mb);
2977 		test_mb_skcipher_speed("ctr(cast6)", DECRYPT, sec, NULL, 0,
2978 				       speed_template_16_32, num_mb);
2979 		test_mb_skcipher_speed("lrw(cast6)", ENCRYPT, sec, NULL, 0,
2980 				       speed_template_32_48, num_mb);
2981 		test_mb_skcipher_speed("lrw(cast6)", DECRYPT, sec, NULL, 0,
2982 				       speed_template_32_48, num_mb);
2983 		test_mb_skcipher_speed("xts(cast6)", ENCRYPT, sec, NULL, 0,
2984 				       speed_template_32_64, num_mb);
2985 		test_mb_skcipher_speed("xts(cast6)", DECRYPT, sec, NULL, 0,
2986 				       speed_template_32_64, num_mb);
2987 		break;
2988 
2989 	case 608:
2990 		test_mb_skcipher_speed("ecb(camellia)", ENCRYPT, sec, NULL, 0,
2991 				       speed_template_16_32, num_mb);
2992 		test_mb_skcipher_speed("ecb(camellia)", DECRYPT, sec, NULL, 0,
2993 				       speed_template_16_32, num_mb);
2994 		test_mb_skcipher_speed("cbc(camellia)", ENCRYPT, sec, NULL, 0,
2995 				       speed_template_16_32, num_mb);
2996 		test_mb_skcipher_speed("cbc(camellia)", DECRYPT, sec, NULL, 0,
2997 				       speed_template_16_32, num_mb);
2998 		test_mb_skcipher_speed("ctr(camellia)", ENCRYPT, sec, NULL, 0,
2999 				       speed_template_16_32, num_mb);
3000 		test_mb_skcipher_speed("ctr(camellia)", DECRYPT, sec, NULL, 0,
3001 				       speed_template_16_32, num_mb);
3002 		test_mb_skcipher_speed("lrw(camellia)", ENCRYPT, sec, NULL, 0,
3003 				       speed_template_32_48, num_mb);
3004 		test_mb_skcipher_speed("lrw(camellia)", DECRYPT, sec, NULL, 0,
3005 				       speed_template_32_48, num_mb);
3006 		test_mb_skcipher_speed("xts(camellia)", ENCRYPT, sec, NULL, 0,
3007 				       speed_template_32_64, num_mb);
3008 		test_mb_skcipher_speed("xts(camellia)", DECRYPT, sec, NULL, 0,
3009 				       speed_template_32_64, num_mb);
3010 		break;
3011 
3012 	case 609:
3013 		test_mb_skcipher_speed("ecb(blowfish)", ENCRYPT, sec, NULL, 0,
3014 				       speed_template_8_32, num_mb);
3015 		test_mb_skcipher_speed("ecb(blowfish)", DECRYPT, sec, NULL, 0,
3016 				       speed_template_8_32, num_mb);
3017 		test_mb_skcipher_speed("cbc(blowfish)", ENCRYPT, sec, NULL, 0,
3018 				       speed_template_8_32, num_mb);
3019 		test_mb_skcipher_speed("cbc(blowfish)", DECRYPT, sec, NULL, 0,
3020 				       speed_template_8_32, num_mb);
3021 		test_mb_skcipher_speed("ctr(blowfish)", ENCRYPT, sec, NULL, 0,
3022 				       speed_template_8_32, num_mb);
3023 		test_mb_skcipher_speed("ctr(blowfish)", DECRYPT, sec, NULL, 0,
3024 				       speed_template_8_32, num_mb);
3025 		break;
3026 
3027 	case 610:
3028 		test_mb_skcipher_speed("ecb(aria)", ENCRYPT, sec, NULL, 0,
3029 				       speed_template_16_32, num_mb);
3030 		test_mb_skcipher_speed("ecb(aria)", DECRYPT, sec, NULL, 0,
3031 				       speed_template_16_32, num_mb);
3032 		test_mb_skcipher_speed("ctr(aria)", ENCRYPT, sec, NULL, 0,
3033 				       speed_template_16_32, num_mb);
3034 		test_mb_skcipher_speed("ctr(aria)", DECRYPT, sec, NULL, 0,
3035 				       speed_template_16_32, num_mb);
3036 		break;
3037 
3038 	}
3039 
3040 	return ret;
3041 }
3042 
tcrypt_mod_init(void)3043 static int __init tcrypt_mod_init(void)
3044 {
3045 	int err = -ENOMEM;
3046 	int i;
3047 
3048 	for (i = 0; i < TVMEMSIZE; i++) {
3049 		tvmem[i] = (void *)__get_free_page(GFP_KERNEL);
3050 		if (!tvmem[i])
3051 			goto err_free_tv;
3052 	}
3053 
3054 	err = do_test(alg, type, mask, mode, num_mb);
3055 
3056 	if (err) {
3057 		pr_err("one or more tests failed!\n");
3058 		goto err_free_tv;
3059 	} else {
3060 		pr_debug("all tests passed\n");
3061 	}
3062 
3063 	/* We intentionaly return -EAGAIN to prevent keeping the module,
3064 	 * unless we're running in fips mode. It does all its work from
3065 	 * init() and doesn't offer any runtime functionality, but in
3066 	 * the fips case, checking for a successful load is helpful.
3067 	 * => we don't need it in the memory, do we?
3068 	 *                                        -- mludvig
3069 	 */
3070 	if (!fips_enabled)
3071 		err = -EAGAIN;
3072 
3073 err_free_tv:
3074 	for (i = 0; i < TVMEMSIZE && tvmem[i]; i++)
3075 		free_page((unsigned long)tvmem[i]);
3076 
3077 	return err;
3078 }
3079 
3080 /*
3081  * If an init function is provided, an exit function must also be provided
3082  * to allow module unload.
3083  */
tcrypt_mod_fini(void)3084 static void __exit tcrypt_mod_fini(void) { }
3085 
3086 late_initcall(tcrypt_mod_init);
3087 module_exit(tcrypt_mod_fini);
3088 
3089 module_param(alg, charp, 0);
3090 module_param(type, uint, 0);
3091 module_param(mask, uint, 0);
3092 module_param(mode, int, 0);
3093 module_param(sec, uint, 0);
3094 MODULE_PARM_DESC(sec, "Length in seconds of speed tests "
3095 		      "(defaults to zero which uses CPU cycles instead)");
3096 module_param(num_mb, uint, 0000);
3097 MODULE_PARM_DESC(num_mb, "Number of concurrent requests to be used in mb speed tests (defaults to 8)");
3098 module_param(klen, uint, 0);
3099 MODULE_PARM_DESC(klen, "Key length (defaults to 0)");
3100 
3101 MODULE_LICENSE("GPL");
3102 MODULE_DESCRIPTION("Quick & dirty crypto testing module");
3103 MODULE_AUTHOR("James Morris <jmorris@intercode.com.au>");
3104