1 /*
2  * Stream co-processor driver for the ETRAX FS
3  *
4  *    Copyright (C) 2003-2007  Axis Communications AB
5  */
6 
7 #include <linux/init.h>
8 #include <linux/sched.h>
9 #include <linux/module.h>
10 #include <linux/slab.h>
11 #include <linux/string.h>
12 #include <linux/fs.h>
13 #include <linux/mm.h>
14 #include <linux/spinlock.h>
15 #include <linux/stddef.h>
16 
17 #include <asm/uaccess.h>
18 #include <asm/io.h>
19 #include <linux/atomic.h>
20 
21 #include <linux/list.h>
22 #include <linux/interrupt.h>
23 
24 #include <asm/signal.h>
25 #include <asm/irq.h>
26 
27 #include <dma.h>
28 #include <hwregs/dma.h>
29 #include <hwregs/reg_map.h>
30 #include <hwregs/reg_rdwr.h>
31 #include <hwregs/intr_vect_defs.h>
32 
33 #include <hwregs/strcop.h>
34 #include <hwregs/strcop_defs.h>
35 #include <cryptocop.h>
36 
37 #ifdef CONFIG_ETRAXFS
38 #define IN_DMA 9
39 #define OUT_DMA 8
40 #define IN_DMA_INST regi_dma9
41 #define OUT_DMA_INST regi_dma8
42 #define DMA_IRQ DMA9_INTR_VECT
43 #else
44 #define IN_DMA 3
45 #define OUT_DMA 2
46 #define IN_DMA_INST regi_dma3
47 #define OUT_DMA_INST regi_dma2
48 #define DMA_IRQ DMA3_INTR_VECT
49 #endif
50 
51 #define DESCR_ALLOC_PAD  (31)
52 
53 struct cryptocop_dma_desc {
54 	char *free_buf; /* If non-null will be kfreed in free_cdesc() */
55 	dma_descr_data *dma_descr;
56 
57 	unsigned char dma_descr_buf[sizeof(dma_descr_data) + DESCR_ALLOC_PAD];
58 
59 	unsigned int from_pool:1; /* If 1 'allocated' from the descriptor pool. */
60 	struct cryptocop_dma_desc *next;
61 };
62 
63 
64 struct cryptocop_int_operation{
65 	void                        *alloc_ptr;
66 	cryptocop_session_id        sid;
67 
68 	dma_descr_context           ctx_out;
69 	dma_descr_context           ctx_in;
70 
71 	/* DMA descriptors allocated by driver. */
72 	struct cryptocop_dma_desc   *cdesc_out;
73 	struct cryptocop_dma_desc   *cdesc_in;
74 
75 	/* Strcop config to use. */
76 	cryptocop_3des_mode         tdes_mode;
77 	cryptocop_csum_type         csum_mode;
78 
79 	/* DMA descrs provided by consumer. */
80 	dma_descr_data              *ddesc_out;
81 	dma_descr_data              *ddesc_in;
82 };
83 
84 
85 struct cryptocop_tfrm_ctx {
86 	cryptocop_tfrm_id tid;
87 	unsigned int blocklength;
88 
89 	unsigned int start_ix;
90 
91 	struct cryptocop_tfrm_cfg *tcfg;
92 	struct cryptocop_transform_ctx *tctx;
93 
94 	unsigned char previous_src;
95 	unsigned char current_src;
96 
97 	/* Values to use in metadata out. */
98 	unsigned char hash_conf;
99 	unsigned char hash_mode;
100 	unsigned char ciph_conf;
101 	unsigned char cbcmode;
102 	unsigned char decrypt;
103 
104 	unsigned int requires_padding:1;
105 	unsigned int strict_block_length:1;
106 	unsigned int active:1;
107 	unsigned int done:1;
108 	size_t consumed;
109 	size_t produced;
110 
111 	/* Pad (input) descriptors to put in the DMA out list when the transform
112 	 * output is put on the DMA in list. */
113 	struct cryptocop_dma_desc *pad_descs;
114 
115 	struct cryptocop_tfrm_ctx *prev_src;
116 	struct cryptocop_tfrm_ctx *curr_src;
117 
118 	/* Mapping to HW. */
119 	unsigned char unit_no;
120 };
121 
122 
123 struct cryptocop_private{
124 	cryptocop_session_id sid;
125 	struct cryptocop_private *next;
126 };
127 
128 /* Session list. */
129 
130 struct cryptocop_transform_ctx{
131 	struct cryptocop_transform_init init;
132 	unsigned char dec_key[CRYPTOCOP_MAX_KEY_LENGTH];
133 	unsigned int dec_key_set:1;
134 
135 	struct cryptocop_transform_ctx *next;
136 };
137 
138 
139 struct cryptocop_session{
140 	cryptocop_session_id sid;
141 
142 	struct cryptocop_transform_ctx *tfrm_ctx;
143 
144 	struct cryptocop_session *next;
145 };
146 
147 /* Priority levels for jobs sent to the cryptocop.  Checksum operations from
148    kernel have highest priority since TCPIP stack processing must not
149    be a bottleneck. */
150 typedef enum {
151 	cryptocop_prio_kernel_csum = 0,
152 	cryptocop_prio_kernel = 1,
153 	cryptocop_prio_user = 2,
154 	cryptocop_prio_no_prios = 3
155 } cryptocop_queue_priority;
156 
157 struct cryptocop_prio_queue{
158 	struct list_head jobs;
159 	cryptocop_queue_priority prio;
160 };
161 
162 struct cryptocop_prio_job{
163 	struct list_head node;
164 	cryptocop_queue_priority prio;
165 
166 	struct cryptocop_operation *oper;
167 	struct cryptocop_int_operation *iop;
168 };
169 
170 struct ioctl_job_cb_ctx {
171 	unsigned int processed:1;
172 };
173 
174 
175 static struct cryptocop_session *cryptocop_sessions = NULL;
176 spinlock_t cryptocop_sessions_lock;
177 
178 /* Next Session ID to assign. */
179 static cryptocop_session_id next_sid = 1;
180 
181 /* Pad for checksum. */
182 static const char csum_zero_pad[1] = {0x00};
183 
184 /* Trash buffer for mem2mem operations. */
185 #define MEM2MEM_DISCARD_BUF_LENGTH  (512)
186 static unsigned char mem2mem_discard_buf[MEM2MEM_DISCARD_BUF_LENGTH];
187 
188 /* Descriptor pool. */
189 /* FIXME Tweak this value. */
190 #define CRYPTOCOP_DESCRIPTOR_POOL_SIZE   (100)
191 static struct cryptocop_dma_desc descr_pool[CRYPTOCOP_DESCRIPTOR_POOL_SIZE];
192 static struct cryptocop_dma_desc *descr_pool_free_list;
193 static int descr_pool_no_free;
194 static spinlock_t descr_pool_lock;
195 
196 /* Lock to stop cryptocop to start processing of a new operation. The holder
197    of this lock MUST call cryptocop_start_job() after it is unlocked. */
198 spinlock_t cryptocop_process_lock;
199 
200 static struct cryptocop_prio_queue cryptocop_job_queues[cryptocop_prio_no_prios];
201 static spinlock_t cryptocop_job_queue_lock;
202 static struct cryptocop_prio_job *cryptocop_running_job = NULL;
203 static spinlock_t running_job_lock;
204 
205 /* The interrupt handler appends completed jobs to this list. The scehduled
206  * tasklet removes them upon sending the response to the crypto consumer. */
207 static struct list_head cryptocop_completed_jobs;
208 static spinlock_t cryptocop_completed_jobs_lock;
209 
210 DECLARE_WAIT_QUEUE_HEAD(cryptocop_ioc_process_wq);
211 
212 
213 /** Local functions. **/
214 
215 static int cryptocop_open(struct inode *, struct file *);
216 
217 static int cryptocop_release(struct inode *, struct file *);
218 
219 static long cryptocop_ioctl(struct file *file,
220 			   unsigned int cmd, unsigned long arg);
221 
222 static void cryptocop_start_job(void);
223 
224 static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation);
225 static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation);
226 
227 static int cryptocop_job_queue_init(void);
228 static void cryptocop_job_queue_close(void);
229 
230 static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
231 
232 static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length);
233 
234 static int transform_ok(struct cryptocop_transform_init *tinit);
235 
236 static struct cryptocop_session *get_session(cryptocop_session_id sid);
237 
238 static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid);
239 
240 static void delete_internal_operation(struct cryptocop_int_operation *iop);
241 
242 static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned  char *key, unsigned int keylength);
243 
244 static int init_stream_coprocessor(void);
245 
246 static void __exit exit_stream_coprocessor(void);
247 
248 /*#define LDEBUG*/
249 #ifdef LDEBUG
250 #define DEBUG(s) s
251 #define DEBUG_API(s) s
252 static void print_cryptocop_operation(struct cryptocop_operation *cop);
253 static void print_dma_descriptors(struct cryptocop_int_operation *iop);
254 static void print_strcop_crypto_op(struct strcop_crypto_op *cop);
255 static void print_lock_status(void);
256 static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op);
257 #define assert(s) do{if (!(s)) panic(#s);} while(0);
258 #else
259 #define DEBUG(s)
260 #define DEBUG_API(s)
261 #define assert(s)
262 #endif
263 
264 
265 /* Transform constants. */
266 #define DES_BLOCK_LENGTH   (8)
267 #define AES_BLOCK_LENGTH   (16)
268 #define MD5_BLOCK_LENGTH   (64)
269 #define SHA1_BLOCK_LENGTH  (64)
270 #define CSUM_BLOCK_LENGTH  (2)
271 #define MD5_STATE_LENGTH   (16)
272 #define SHA1_STATE_LENGTH  (20)
273 
274 /* The device number. */
275 #define CRYPTOCOP_MAJOR    (254)
276 #define CRYPTOCOP_MINOR    (0)
277 
278 
279 
280 const struct file_operations cryptocop_fops = {
281 	.owner		= THIS_MODULE,
282 	.open		= cryptocop_open,
283 	.release	= cryptocop_release,
284 	.unlocked_ioctl = cryptocop_ioctl,
285 	.llseek		= noop_llseek,
286 };
287 
288 
free_cdesc(struct cryptocop_dma_desc * cdesc)289 static void free_cdesc(struct cryptocop_dma_desc *cdesc)
290 {
291 	DEBUG(printk("free_cdesc: cdesc 0x%p, from_pool=%d\n", cdesc, cdesc->from_pool));
292 	kfree(cdesc->free_buf);
293 
294 	if (cdesc->from_pool) {
295 		unsigned long int flags;
296 		spin_lock_irqsave(&descr_pool_lock, flags);
297 		cdesc->next = descr_pool_free_list;
298 		descr_pool_free_list = cdesc;
299 		++descr_pool_no_free;
300 		spin_unlock_irqrestore(&descr_pool_lock, flags);
301 	} else {
302 		kfree(cdesc);
303 	}
304 }
305 
306 
alloc_cdesc(int alloc_flag)307 static struct cryptocop_dma_desc *alloc_cdesc(int alloc_flag)
308 {
309 	int use_pool = (alloc_flag & GFP_ATOMIC) ? 1 : 0;
310 	struct cryptocop_dma_desc *cdesc;
311 
312 	if (use_pool) {
313 		unsigned long int flags;
314 		spin_lock_irqsave(&descr_pool_lock, flags);
315 		if (!descr_pool_free_list) {
316 			spin_unlock_irqrestore(&descr_pool_lock, flags);
317 			DEBUG_API(printk("alloc_cdesc: pool is empty\n"));
318 			return NULL;
319 		}
320 		cdesc = descr_pool_free_list;
321 		descr_pool_free_list = descr_pool_free_list->next;
322 		--descr_pool_no_free;
323 		spin_unlock_irqrestore(&descr_pool_lock, flags);
324 		cdesc->from_pool = 1;
325 	} else {
326 		cdesc = kmalloc(sizeof(struct cryptocop_dma_desc), alloc_flag);
327 		if (!cdesc) {
328 			DEBUG_API(printk("alloc_cdesc: kmalloc\n"));
329 			return NULL;
330 		}
331 		cdesc->from_pool = 0;
332 	}
333 	cdesc->dma_descr = (dma_descr_data*)(((unsigned long int)cdesc + offsetof(struct cryptocop_dma_desc, dma_descr_buf) + DESCR_ALLOC_PAD) & ~0x0000001F);
334 
335 	cdesc->next = NULL;
336 
337 	cdesc->free_buf = NULL;
338 	cdesc->dma_descr->out_eop = 0;
339 	cdesc->dma_descr->in_eop = 0;
340 	cdesc->dma_descr->intr = 0;
341 	cdesc->dma_descr->eol = 0;
342 	cdesc->dma_descr->wait = 0;
343 	cdesc->dma_descr->buf = NULL;
344 	cdesc->dma_descr->after = NULL;
345 
346 	DEBUG_API(printk("alloc_cdesc: return 0x%p, cdesc->dma_descr=0x%p, from_pool=%d\n", cdesc, cdesc->dma_descr, cdesc->from_pool));
347 	return cdesc;
348 }
349 
350 
setup_descr_chain(struct cryptocop_dma_desc * cd)351 static void setup_descr_chain(struct cryptocop_dma_desc *cd)
352 {
353 	DEBUG(printk("setup_descr_chain: entering\n"));
354 	while (cd) {
355 		if (cd->next) {
356 			cd->dma_descr->next = (dma_descr_data*)virt_to_phys(cd->next->dma_descr);
357 		} else {
358 			cd->dma_descr->next = NULL;
359 		}
360 		cd = cd->next;
361 	}
362 	DEBUG(printk("setup_descr_chain: exit\n"));
363 }
364 
365 
366 /* Create a pad descriptor for the transform.
367  * Return -1 for error, 0 if pad created. */
create_pad_descriptor(struct cryptocop_tfrm_ctx * tc,struct cryptocop_dma_desc ** pad_desc,int alloc_flag)368 static int create_pad_descriptor(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **pad_desc, int alloc_flag)
369 {
370 	struct cryptocop_dma_desc        *cdesc = NULL;
371 	int                              error = 0;
372 	struct strcop_meta_out           mo = {
373 		.ciphsel = src_none,
374 		.hashsel = src_none,
375 		.csumsel = src_none
376 	};
377 	char                             *pad;
378 	size_t                           plen;
379 
380 	DEBUG(printk("create_pad_descriptor: start.\n"));
381 	/* Setup pad descriptor. */
382 
383 	DEBUG(printk("create_pad_descriptor: setting up padding.\n"));
384 	cdesc = alloc_cdesc(alloc_flag);
385 	if (!cdesc){
386 		DEBUG_API(printk("create_pad_descriptor: alloc pad desc\n"));
387 		goto error_cleanup;
388 	}
389 	switch (tc->unit_no) {
390 	case src_md5:
391 		error = create_md5_pad(alloc_flag, tc->consumed, &pad, &plen);
392 		if (error){
393 			DEBUG_API(printk("create_pad_descriptor: create_md5_pad_failed\n"));
394 			goto error_cleanup;
395 		}
396 		cdesc->free_buf = pad;
397 		mo.hashsel = src_dma;
398 		mo.hashconf = tc->hash_conf;
399 		mo.hashmode = tc->hash_mode;
400 		break;
401 	case src_sha1:
402 		error = create_sha1_pad(alloc_flag, tc->consumed, &pad, &plen);
403 		if (error){
404 			DEBUG_API(printk("create_pad_descriptor: create_sha1_pad_failed\n"));
405 			goto error_cleanup;
406 		}
407 		cdesc->free_buf = pad;
408 		mo.hashsel = src_dma;
409 		mo.hashconf = tc->hash_conf;
410 		mo.hashmode = tc->hash_mode;
411 		break;
412 	case src_csum:
413 		if (tc->consumed % tc->blocklength){
414 			pad = (char*)csum_zero_pad;
415 			plen = 1;
416 		} else {
417 			pad = (char*)cdesc; /* Use any pointer. */
418 			plen = 0;
419 		}
420 		mo.csumsel = src_dma;
421 		break;
422 	}
423 	cdesc->dma_descr->wait = 1;
424 	cdesc->dma_descr->out_eop = 1; /* Since this is a pad output is pushed.  EOP is ok here since the padded unit is the only one active. */
425 	cdesc->dma_descr->buf = (char*)virt_to_phys((char*)pad);
426 	cdesc->dma_descr->after = cdesc->dma_descr->buf + plen;
427 
428 	cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
429 	*pad_desc = cdesc;
430 
431 	return 0;
432 
433  error_cleanup:
434 	if (cdesc) free_cdesc(cdesc);
435 	return -1;
436 }
437 
438 
setup_key_dl_desc(struct cryptocop_tfrm_ctx * tc,struct cryptocop_dma_desc ** kd,int alloc_flag)439 static int setup_key_dl_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **kd, int alloc_flag)
440 {
441 	struct cryptocop_dma_desc  *key_desc = alloc_cdesc(alloc_flag);
442 	struct strcop_meta_out     mo = {0};
443 
444 	DEBUG(printk("setup_key_dl_desc\n"));
445 
446 	if (!key_desc) {
447 		DEBUG_API(printk("setup_key_dl_desc: failed descriptor allocation.\n"));
448 		return -ENOMEM;
449 	}
450 
451 	/* Download key. */
452 	if ((tc->tctx->init.alg == cryptocop_alg_aes) && (tc->tcfg->flags & CRYPTOCOP_DECRYPT)) {
453 		/* Precook the AES decrypt key. */
454 		if (!tc->tctx->dec_key_set){
455 			get_aes_decrypt_key(tc->tctx->dec_key, tc->tctx->init.key, tc->tctx->init.keylen);
456 			tc->tctx->dec_key_set = 1;
457 		}
458 		key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->dec_key);
459 		key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
460 	} else {
461 		key_desc->dma_descr->buf = (char*)virt_to_phys(tc->tctx->init.key);
462 		key_desc->dma_descr->after = key_desc->dma_descr->buf + tc->tctx->init.keylen/8;
463 	}
464 	/* Setup metadata. */
465 	mo.dlkey = 1;
466 	switch (tc->tctx->init.keylen) {
467 	case 64:
468 		mo.decrypt = 0;
469 		mo.hashmode = 0;
470 		break;
471 	case 128:
472 		mo.decrypt = 0;
473 		mo.hashmode = 1;
474 		break;
475 	case 192:
476 		mo.decrypt = 1;
477 		mo.hashmode = 0;
478 		break;
479 	case 256:
480 		mo.decrypt = 1;
481 		mo.hashmode = 1;
482 		break;
483 	default:
484 		break;
485 	}
486 	mo.ciphsel = mo.hashsel = mo.csumsel = src_none;
487 	key_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
488 
489 	key_desc->dma_descr->out_eop = 1;
490 	key_desc->dma_descr->wait = 1;
491 	key_desc->dma_descr->intr = 0;
492 
493 	*kd = key_desc;
494 	return 0;
495 }
496 
setup_cipher_iv_desc(struct cryptocop_tfrm_ctx * tc,struct cryptocop_dma_desc ** id,int alloc_flag)497 static int setup_cipher_iv_desc(struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
498 {
499 	struct cryptocop_dma_desc  *iv_desc = alloc_cdesc(alloc_flag);
500 	struct strcop_meta_out     mo = {0};
501 
502 	DEBUG(printk("setup_cipher_iv_desc\n"));
503 
504 	if (!iv_desc) {
505 		DEBUG_API(printk("setup_cipher_iv_desc: failed CBC IV descriptor allocation.\n"));
506 		return -ENOMEM;
507 	}
508 	/* Download IV. */
509 	iv_desc->dma_descr->buf = (char*)virt_to_phys(tc->tcfg->iv);
510 	iv_desc->dma_descr->after = iv_desc->dma_descr->buf + tc->blocklength;
511 
512 	/* Setup metadata. */
513 	mo.hashsel = mo.csumsel = src_none;
514 	mo.ciphsel = src_dma;
515 	mo.ciphconf = tc->ciph_conf;
516 	mo.cbcmode = tc->cbcmode;
517 
518 	iv_desc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, mo);
519 
520 	iv_desc->dma_descr->out_eop = 0;
521 	iv_desc->dma_descr->wait = 1;
522 	iv_desc->dma_descr->intr = 0;
523 
524 	*id = iv_desc;
525 	return 0;
526 }
527 
528 /* Map the ouput length of the transform to operation output starting on the inject index. */
create_input_descriptors(struct cryptocop_operation * operation,struct cryptocop_tfrm_ctx * tc,struct cryptocop_dma_desc ** id,int alloc_flag)529 static int create_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_tfrm_ctx *tc, struct cryptocop_dma_desc **id, int alloc_flag)
530 {
531 	int                        err = 0;
532 	struct cryptocop_dma_desc  head = {0};
533 	struct cryptocop_dma_desc  *outdesc = &head;
534 	size_t                     iov_offset = 0;
535 	size_t                     out_ix = 0;
536 	int                        outiov_ix = 0;
537 	struct strcop_meta_in      mi = {0};
538 
539 	size_t                     out_length = tc->produced;
540 	int                        rem_length;
541 	int                        dlength;
542 
543 	assert(out_length != 0);
544 	if (((tc->produced + tc->tcfg->inject_ix) > operation->tfrm_op.outlen) || (tc->produced && (operation->tfrm_op.outlen == 0))) {
545 		DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
546 		return -EINVAL;
547 	}
548 	/* Traverse the out iovec until the result inject index is reached. */
549 	while ((outiov_ix < operation->tfrm_op.outcount) && ((out_ix + operation->tfrm_op.outdata[outiov_ix].iov_len) <= tc->tcfg->inject_ix)){
550 		out_ix += operation->tfrm_op.outdata[outiov_ix].iov_len;
551 		outiov_ix++;
552 	}
553 	if (outiov_ix >= operation->tfrm_op.outcount){
554 		DEBUG_API(printk("create_input_descriptors: operation outdata too small\n"));
555 		return -EINVAL;
556 	}
557 	iov_offset = tc->tcfg->inject_ix - out_ix;
558 	mi.dmasel = tc->unit_no;
559 
560 	/* Setup the output descriptors. */
561 	while ((out_length > 0) && (outiov_ix < operation->tfrm_op.outcount)) {
562 		outdesc->next = alloc_cdesc(alloc_flag);
563 		if (!outdesc->next) {
564 			DEBUG_API(printk("create_input_descriptors: alloc_cdesc\n"));
565 			err = -ENOMEM;
566 			goto error_cleanup;
567 		}
568 		outdesc = outdesc->next;
569 		rem_length = operation->tfrm_op.outdata[outiov_ix].iov_len - iov_offset;
570 		dlength = (out_length < rem_length) ? out_length : rem_length;
571 
572 		DEBUG(printk("create_input_descriptors:\n"
573 			     "outiov_ix=%d, rem_length=%d, dlength=%d\n"
574 			     "iov_offset=%d, outdata[outiov_ix].iov_len=%d\n"
575 			     "outcount=%d, outiov_ix=%d\n",
576 			     outiov_ix, rem_length, dlength, iov_offset, operation->tfrm_op.outdata[outiov_ix].iov_len, operation->tfrm_op.outcount, outiov_ix));
577 
578 		outdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.outdata[outiov_ix].iov_base + iov_offset);
579 		outdesc->dma_descr->after = outdesc->dma_descr->buf + dlength;
580 		outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
581 
582 		out_length -= dlength;
583 		iov_offset += dlength;
584 		if (iov_offset >= operation->tfrm_op.outdata[outiov_ix].iov_len) {
585 			iov_offset = 0;
586 			++outiov_ix;
587 		}
588 	}
589 	if (out_length > 0){
590 		DEBUG_API(printk("create_input_descriptors: not enough room for output, %d remained\n", out_length));
591 		err = -EINVAL;
592 		goto error_cleanup;
593 	}
594 	/* Set sync in last descriptor. */
595 	mi.sync = 1;
596 	outdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
597 
598 	*id = head.next;
599 	return 0;
600 
601  error_cleanup:
602 	while (head.next) {
603 		outdesc = head.next->next;
604 		free_cdesc(head.next);
605 		head.next = outdesc;
606 	}
607 	return err;
608 }
609 
610 
create_output_descriptors(struct cryptocop_operation * operation,int * iniov_ix,int * iniov_offset,size_t desc_len,struct cryptocop_dma_desc ** current_out_cdesc,struct strcop_meta_out * meta_out,int alloc_flag)611 static int create_output_descriptors(struct cryptocop_operation *operation, int *iniov_ix, int *iniov_offset, size_t desc_len, struct cryptocop_dma_desc **current_out_cdesc, struct strcop_meta_out *meta_out, int alloc_flag)
612 {
613 	while (desc_len != 0) {
614 		struct cryptocop_dma_desc  *cdesc;
615 		int                        rem_length = operation->tfrm_op.indata[*iniov_ix].iov_len - *iniov_offset;
616 		int                        dlength = (desc_len < rem_length) ? desc_len : rem_length;
617 
618 		cdesc = alloc_cdesc(alloc_flag);
619 		if (!cdesc) {
620 			DEBUG_API(printk("create_output_descriptors: alloc_cdesc\n"));
621 			return -ENOMEM;
622 		}
623 		(*current_out_cdesc)->next = cdesc;
624 		(*current_out_cdesc) = cdesc;
625 
626 		cdesc->free_buf = NULL;
627 
628 		cdesc->dma_descr->buf = (char*)virt_to_phys(operation->tfrm_op.indata[*iniov_ix].iov_base + *iniov_offset);
629 		cdesc->dma_descr->after = cdesc->dma_descr->buf + dlength;
630 
631 		assert(desc_len >= dlength);
632 		desc_len -= dlength;
633 		*iniov_offset += dlength;
634 		if (*iniov_offset >= operation->tfrm_op.indata[*iniov_ix].iov_len) {
635 			*iniov_offset = 0;
636 			++(*iniov_ix);
637 			if (*iniov_ix > operation->tfrm_op.incount) {
638 				DEBUG_API(printk("create_output_descriptors: not enough indata in operation."));
639 				return  -EINVAL;
640 			}
641 		}
642 		cdesc->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, (*meta_out));
643 	} /* while (desc_len != 0) */
644 	/* Last DMA descriptor gets a 'wait' bit to signal expected change in metadata. */
645 	(*current_out_cdesc)->dma_descr->wait = 1; /* This will set extraneous WAIT in some situations, e.g. when padding hashes and checksums. */
646 
647 	return 0;
648 }
649 
650 
append_input_descriptors(struct cryptocop_operation * operation,struct cryptocop_dma_desc ** current_in_cdesc,struct cryptocop_dma_desc ** current_out_cdesc,struct cryptocop_tfrm_ctx * tc,int alloc_flag)651 static int append_input_descriptors(struct cryptocop_operation *operation, struct cryptocop_dma_desc **current_in_cdesc, struct cryptocop_dma_desc **current_out_cdesc, struct cryptocop_tfrm_ctx *tc, int alloc_flag)
652 {
653 	DEBUG(printk("append_input_descriptors, tc=0x%p, unit_no=%d\n", tc, tc->unit_no));
654 	if (tc->tcfg) {
655 		int                        failed = 0;
656 		struct cryptocop_dma_desc  *idescs = NULL;
657 		DEBUG(printk("append_input_descriptors: pushing output, consumed %d produced %d bytes.\n", tc->consumed, tc->produced));
658 		if (tc->pad_descs) {
659 			DEBUG(printk("append_input_descriptors: append pad descriptors to DMA out list.\n"));
660 			while (tc->pad_descs) {
661 				DEBUG(printk("append descriptor 0x%p\n", tc->pad_descs));
662 				(*current_out_cdesc)->next = tc->pad_descs;
663 				tc->pad_descs = tc->pad_descs->next;
664 				(*current_out_cdesc) = (*current_out_cdesc)->next;
665 			}
666 		}
667 
668 		/* Setup and append output descriptors to DMA in list. */
669 		if (tc->unit_no == src_dma){
670 			/* mem2mem.  Setup DMA in descriptors to discard all input prior to the requested mem2mem data. */
671 			struct strcop_meta_in mi = {.sync = 0, .dmasel = src_dma};
672 			unsigned int start_ix = tc->start_ix;
673 			while (start_ix){
674 				unsigned int desclen = start_ix < MEM2MEM_DISCARD_BUF_LENGTH ? start_ix : MEM2MEM_DISCARD_BUF_LENGTH;
675 				(*current_in_cdesc)->next = alloc_cdesc(alloc_flag);
676 				if (!(*current_in_cdesc)->next){
677 					DEBUG_API(printk("append_input_descriptors: alloc_cdesc mem2mem discard failed\n"));
678 					return -ENOMEM;
679 				}
680 				(*current_in_cdesc) = (*current_in_cdesc)->next;
681 				(*current_in_cdesc)->dma_descr->buf = (char*)virt_to_phys(mem2mem_discard_buf);
682 				(*current_in_cdesc)->dma_descr->after = (*current_in_cdesc)->dma_descr->buf + desclen;
683 				(*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
684 				start_ix -= desclen;
685 			}
686 			mi.sync = 1;
687 			(*current_in_cdesc)->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_in, mi);
688 		}
689 
690 		failed = create_input_descriptors(operation, tc, &idescs, alloc_flag);
691 		if (failed){
692 			DEBUG_API(printk("append_input_descriptors: output descriptor setup failed\n"));
693 			return failed;
694 		}
695 		DEBUG(printk("append_input_descriptors: append output descriptors to DMA in list.\n"));
696 		while (idescs) {
697 			DEBUG(printk("append descriptor 0x%p\n", idescs));
698 			(*current_in_cdesc)->next = idescs;
699 			idescs = idescs->next;
700 			(*current_in_cdesc) = (*current_in_cdesc)->next;
701 		}
702 	}
703 	return 0;
704 }
705 
706 
707 
cryptocop_setup_dma_list(struct cryptocop_operation * operation,struct cryptocop_int_operation ** int_op,int alloc_flag)708 static int cryptocop_setup_dma_list(struct cryptocop_operation *operation, struct cryptocop_int_operation **int_op, int alloc_flag)
709 {
710 	struct cryptocop_session *sess;
711 	struct cryptocop_transform_ctx *tctx;
712 
713 	struct cryptocop_tfrm_ctx digest_ctx = {
714 		.previous_src = src_none,
715 		.current_src = src_none,
716 		.start_ix = 0,
717 		.requires_padding = 1,
718 		.strict_block_length = 0,
719 		.hash_conf = 0,
720 		.hash_mode = 0,
721 		.ciph_conf = 0,
722 		.cbcmode = 0,
723 		.decrypt = 0,
724 		.consumed = 0,
725 		.produced = 0,
726 		.pad_descs = NULL,
727 		.active = 0,
728 		.done = 0,
729 		.prev_src = NULL,
730 		.curr_src = NULL,
731 		.tcfg = NULL};
732 	struct cryptocop_tfrm_ctx cipher_ctx = {
733 		.previous_src = src_none,
734 		.current_src = src_none,
735 		.start_ix = 0,
736 		.requires_padding = 0,
737 		.strict_block_length = 1,
738 		.hash_conf = 0,
739 		.hash_mode = 0,
740 		.ciph_conf = 0,
741 		.cbcmode = 0,
742 		.decrypt = 0,
743 		.consumed = 0,
744 		.produced = 0,
745 		.pad_descs = NULL,
746 		.active = 0,
747 		.done = 0,
748 		.prev_src = NULL,
749 		.curr_src = NULL,
750 		.tcfg = NULL};
751 	struct cryptocop_tfrm_ctx csum_ctx = {
752 		.previous_src = src_none,
753 		.current_src = src_none,
754 		.start_ix = 0,
755 		.blocklength = 2,
756 		.requires_padding = 1,
757 		.strict_block_length = 0,
758 		.hash_conf = 0,
759 		.hash_mode = 0,
760 		.ciph_conf = 0,
761 		.cbcmode = 0,
762 		.decrypt = 0,
763 		.consumed = 0,
764 		.produced = 0,
765 		.pad_descs = NULL,
766 		.active = 0,
767 		.done = 0,
768 		.tcfg = NULL,
769 		.prev_src = NULL,
770 		.curr_src = NULL,
771 		.unit_no = src_csum};
772 	struct cryptocop_tfrm_cfg *tcfg = operation->tfrm_op.tfrm_cfg;
773 
774 	unsigned int indata_ix = 0;
775 
776 	/* iovec accounting. */
777 	int iniov_ix = 0;
778 	int iniov_offset = 0;
779 
780 	/* Operation descriptor cfg traversal pointer. */
781 	struct cryptocop_desc *odsc;
782 
783 	int failed = 0;
784 	/* List heads for allocated descriptors. */
785 	struct cryptocop_dma_desc out_cdesc_head = {0};
786 	struct cryptocop_dma_desc in_cdesc_head = {0};
787 
788 	struct cryptocop_dma_desc *current_out_cdesc = &out_cdesc_head;
789 	struct cryptocop_dma_desc *current_in_cdesc = &in_cdesc_head;
790 
791 	struct cryptocop_tfrm_ctx *output_tc = NULL;
792 	void                      *iop_alloc_ptr;
793 
794 	assert(operation != NULL);
795 	assert(int_op != NULL);
796 
797 	DEBUG(printk("cryptocop_setup_dma_list: start\n"));
798 	DEBUG(print_cryptocop_operation(operation));
799 
800 	sess = get_session(operation->sid);
801 	if (!sess) {
802 		DEBUG_API(printk("cryptocop_setup_dma_list: no session found for operation.\n"));
803 		failed = -EINVAL;
804 		goto error_cleanup;
805 	}
806 	iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
807 	if (!iop_alloc_ptr) {
808 		DEBUG_API(printk("cryptocop_setup_dma_list:  kmalloc cryptocop_int_operation\n"));
809 		failed = -ENOMEM;
810 		goto error_cleanup;
811 	}
812 	(*int_op) = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
813 	DEBUG(memset((*int_op), 0xff, sizeof(struct cryptocop_int_operation)));
814 	(*int_op)->alloc_ptr = iop_alloc_ptr;
815 	DEBUG(printk("cryptocop_setup_dma_list: *int_op=0x%p, alloc_ptr=0x%p\n", *int_op, (*int_op)->alloc_ptr));
816 
817 	(*int_op)->sid = operation->sid;
818 	(*int_op)->cdesc_out = NULL;
819 	(*int_op)->cdesc_in = NULL;
820 	(*int_op)->tdes_mode = cryptocop_3des_ede;
821 	(*int_op)->csum_mode = cryptocop_csum_le;
822 	(*int_op)->ddesc_out = NULL;
823 	(*int_op)->ddesc_in = NULL;
824 
825 	/* Scan operation->tfrm_op.tfrm_cfg for bad configuration and set up the local contexts. */
826 	if (!tcfg) {
827 		DEBUG_API(printk("cryptocop_setup_dma_list: no configured transforms in operation.\n"));
828 		failed = -EINVAL;
829 		goto error_cleanup;
830 	}
831 	while (tcfg) {
832 		tctx = get_transform_ctx(sess, tcfg->tid);
833 		if (!tctx) {
834 			DEBUG_API(printk("cryptocop_setup_dma_list: no transform id %d in session.\n", tcfg->tid));
835 			failed = -EINVAL;
836 			goto error_cleanup;
837 		}
838 		if (tcfg->inject_ix > operation->tfrm_op.outlen){
839 			DEBUG_API(printk("cryptocop_setup_dma_list: transform id %d inject_ix (%d) > operation->tfrm_op.outlen(%d)", tcfg->tid, tcfg->inject_ix, operation->tfrm_op.outlen));
840 			failed = -EINVAL;
841 			goto error_cleanup;
842 		}
843 		switch (tctx->init.alg){
844 		case cryptocop_alg_mem2mem:
845 			if (cipher_ctx.tcfg != NULL){
846 				DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
847 				failed = -EINVAL;
848 				goto error_cleanup;
849 			}
850 			/* mem2mem is handled as a NULL cipher. */
851 			cipher_ctx.cbcmode = 0;
852 			cipher_ctx.decrypt = 0;
853 			cipher_ctx.blocklength = 1;
854 			cipher_ctx.ciph_conf = 0;
855 			cipher_ctx.unit_no = src_dma;
856 			cipher_ctx.tcfg = tcfg;
857 			cipher_ctx.tctx = tctx;
858 			break;
859 		case cryptocop_alg_des:
860 		case cryptocop_alg_3des:
861 		case cryptocop_alg_aes:
862 			/* cipher */
863 			if (cipher_ctx.tcfg != NULL){
864 				DEBUG_API(printk("cryptocop_setup_dma_list: multiple ciphers in operation.\n"));
865 				failed = -EINVAL;
866 				goto error_cleanup;
867 			}
868 			cipher_ctx.tcfg = tcfg;
869 			cipher_ctx.tctx = tctx;
870 			if (cipher_ctx.tcfg->flags & CRYPTOCOP_DECRYPT){
871 				cipher_ctx.decrypt = 1;
872 			}
873 			switch (tctx->init.cipher_mode) {
874 			case cryptocop_cipher_mode_ecb:
875 				cipher_ctx.cbcmode = 0;
876 				break;
877 			case cryptocop_cipher_mode_cbc:
878 				cipher_ctx.cbcmode = 1;
879 				break;
880 			default:
881 				DEBUG_API(printk("cryptocop_setup_dma_list: cipher_ctx, bad cipher mode==%d\n", tctx->init.cipher_mode));
882 				failed = -EINVAL;
883 				goto error_cleanup;
884 			}
885 			DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx, set CBC mode==%d\n", cipher_ctx.cbcmode));
886 			switch (tctx->init.alg){
887 			case cryptocop_alg_des:
888 				cipher_ctx.ciph_conf = 0;
889 				cipher_ctx.unit_no = src_des;
890 				cipher_ctx.blocklength = DES_BLOCK_LENGTH;
891 				break;
892 			case cryptocop_alg_3des:
893 				cipher_ctx.ciph_conf = 1;
894 				cipher_ctx.unit_no = src_des;
895 				cipher_ctx.blocklength = DES_BLOCK_LENGTH;
896 				break;
897 			case cryptocop_alg_aes:
898 				cipher_ctx.ciph_conf = 2;
899 				cipher_ctx.unit_no = src_aes;
900 				cipher_ctx.blocklength = AES_BLOCK_LENGTH;
901 				break;
902 			default:
903 				panic("cryptocop_setup_dma_list: impossible algorithm %d\n", tctx->init.alg);
904 			}
905 			(*int_op)->tdes_mode = tctx->init.tdes_mode;
906 			break;
907 		case cryptocop_alg_md5:
908 		case cryptocop_alg_sha1:
909 			/* digest */
910 			if (digest_ctx.tcfg != NULL){
911 				DEBUG_API(printk("cryptocop_setup_dma_list: multiple digests in operation.\n"));
912 				failed = -EINVAL;
913 				goto error_cleanup;
914 			}
915 			digest_ctx.tcfg = tcfg;
916 			digest_ctx.tctx = tctx;
917 			digest_ctx.hash_mode = 0; /* Don't use explicit IV in this API. */
918 			switch (tctx->init.alg){
919 			case cryptocop_alg_md5:
920 				digest_ctx.blocklength = MD5_BLOCK_LENGTH;
921 				digest_ctx.unit_no = src_md5;
922 				digest_ctx.hash_conf = 1; /* 1 => MD-5 */
923 				break;
924 			case cryptocop_alg_sha1:
925 				digest_ctx.blocklength = SHA1_BLOCK_LENGTH;
926 				digest_ctx.unit_no = src_sha1;
927 				digest_ctx.hash_conf = 0; /* 0 => SHA-1 */
928 				break;
929 			default:
930 				panic("cryptocop_setup_dma_list: impossible digest algorithm\n");
931 			}
932 			break;
933 		case cryptocop_alg_csum:
934 			/* digest */
935 			if (csum_ctx.tcfg != NULL){
936 				DEBUG_API(printk("cryptocop_setup_dma_list: multiple checksums in operation.\n"));
937 				failed = -EINVAL;
938 				goto error_cleanup;
939 			}
940 			(*int_op)->csum_mode = tctx->init.csum_mode;
941 			csum_ctx.tcfg = tcfg;
942 			csum_ctx.tctx = tctx;
943 			break;
944 		default:
945 			/* no algorithm. */
946 			DEBUG_API(printk("cryptocop_setup_dma_list: invalid algorithm %d specified in tfrm %d.\n", tctx->init.alg, tcfg->tid));
947 			failed = -EINVAL;
948 			goto error_cleanup;
949 		}
950 		tcfg = tcfg->next;
951 	}
952 	/* Download key if a cipher is used. */
953 	if (cipher_ctx.tcfg && (cipher_ctx.tctx->init.alg != cryptocop_alg_mem2mem)){
954 		struct cryptocop_dma_desc  *key_desc = NULL;
955 
956 		failed = setup_key_dl_desc(&cipher_ctx, &key_desc, alloc_flag);
957 		if (failed) {
958 			DEBUG_API(printk("cryptocop_setup_dma_list: setup key dl\n"));
959 			goto error_cleanup;
960 		}
961 		current_out_cdesc->next = key_desc;
962 		current_out_cdesc = key_desc;
963 		indata_ix += (unsigned int)(key_desc->dma_descr->after - key_desc->dma_descr->buf);
964 
965 		/* Download explicit IV if a cipher is used and CBC mode and explicit IV selected. */
966 		if ((cipher_ctx.tctx->init.cipher_mode == cryptocop_cipher_mode_cbc) && (cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV)) {
967 			struct cryptocop_dma_desc  *iv_desc = NULL;
968 
969 			DEBUG(printk("cryptocop_setup_dma_list: setup cipher CBC IV descriptor.\n"));
970 
971 			failed = setup_cipher_iv_desc(&cipher_ctx, &iv_desc, alloc_flag);
972 			if (failed) {
973 				DEBUG_API(printk("cryptocop_setup_dma_list: CBC IV descriptor.\n"));
974 				goto error_cleanup;
975 			}
976 			current_out_cdesc->next = iv_desc;
977 			current_out_cdesc = iv_desc;
978 			indata_ix += (unsigned int)(iv_desc->dma_descr->after - iv_desc->dma_descr->buf);
979 		}
980 	}
981 
982 	/* Process descriptors. */
983 	odsc = operation->tfrm_op.desc;
984 	while (odsc) {
985 		struct cryptocop_desc_cfg   *dcfg = odsc->cfg;
986 		struct strcop_meta_out      meta_out = {0};
987 		size_t                      desc_len = odsc->length;
988 		int                         active_count, eop_needed_count;
989 
990 		output_tc = NULL;
991 
992 		DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor\n"));
993 
994 		while (dcfg) {
995 			struct cryptocop_tfrm_ctx  *tc = NULL;
996 
997 			DEBUG(printk("cryptocop_setup_dma_list: parsing an operation descriptor configuration.\n"));
998 			/* Get the local context for the transform and mark it as the output unit if it produces output. */
999 			if (digest_ctx.tcfg && (digest_ctx.tcfg->tid == dcfg->tid)){
1000 				tc = &digest_ctx;
1001 			} else if (cipher_ctx.tcfg && (cipher_ctx.tcfg->tid == dcfg->tid)){
1002 				tc = &cipher_ctx;
1003 			} else if (csum_ctx.tcfg && (csum_ctx.tcfg->tid == dcfg->tid)){
1004 				tc = &csum_ctx;
1005 			}
1006 			if (!tc) {
1007 				DEBUG_API(printk("cryptocop_setup_dma_list: invalid transform %d specified in descriptor.\n", dcfg->tid));
1008 				failed = -EINVAL;
1009 				goto error_cleanup;
1010 			}
1011 			if (tc->done) {
1012 				DEBUG_API(printk("cryptocop_setup_dma_list: completed transform %d reused.\n", dcfg->tid));
1013 				failed = -EINVAL;
1014 				goto error_cleanup;
1015 			}
1016 			if (!tc->active) {
1017 				tc->start_ix = indata_ix;
1018 				tc->active = 1;
1019 			}
1020 
1021 			tc->previous_src = tc->current_src;
1022 			tc->prev_src = tc->curr_src;
1023 			/* Map source unit id to DMA source config. */
1024 			switch (dcfg->src){
1025 			case cryptocop_source_dma:
1026 				tc->current_src = src_dma;
1027 				break;
1028 			case cryptocop_source_des:
1029 				tc->current_src = src_des;
1030 				break;
1031 			case cryptocop_source_3des:
1032 				tc->current_src = src_des;
1033 				break;
1034 			case cryptocop_source_aes:
1035 				tc->current_src = src_aes;
1036 				break;
1037 			case cryptocop_source_md5:
1038 			case cryptocop_source_sha1:
1039 			case cryptocop_source_csum:
1040 			case cryptocop_source_none:
1041 			default:
1042 				/* We do not allow using accumulating style units (SHA-1, MD5, checksum) as sources to other units.
1043 				 */
1044 				DEBUG_API(printk("cryptocop_setup_dma_list: bad unit source configured %d.\n", dcfg->src));
1045 				failed = -EINVAL;
1046 				goto error_cleanup;
1047 			}
1048 			if (tc->current_src != src_dma) {
1049 				/* Find the unit we are sourcing from. */
1050 				if (digest_ctx.unit_no == tc->current_src){
1051 					tc->curr_src = &digest_ctx;
1052 				} else if (cipher_ctx.unit_no == tc->current_src){
1053 					tc->curr_src = &cipher_ctx;
1054 				} else if (csum_ctx.unit_no == tc->current_src){
1055 					tc->curr_src = &csum_ctx;
1056 				}
1057 				if ((tc->curr_src == tc) && (tc->unit_no != src_dma)){
1058 					DEBUG_API(printk("cryptocop_setup_dma_list: unit %d configured to source from itself.\n", tc->unit_no));
1059 					failed = -EINVAL;
1060 					goto error_cleanup;
1061 				}
1062 			} else {
1063 				tc->curr_src = NULL;
1064 			}
1065 
1066 			/* Detect source switch. */
1067 			DEBUG(printk("cryptocop_setup_dma_list: tc->active=%d tc->unit_no=%d tc->current_src=%d tc->previous_src=%d, tc->curr_src=0x%p, tc->prev_srv=0x%p\n", tc->active, tc->unit_no, tc->current_src, tc->previous_src, tc->curr_src, tc->prev_src));
1068 			if (tc->active && (tc->current_src != tc->previous_src)) {
1069 				/* Only allow source switch when both the old source unit and the new one have
1070 				 * no pending data to process (i.e. the consumed length must be a multiple of the
1071 				 * transform blocklength). */
1072 				/* Note: if the src == NULL we are actually sourcing from DMA out. */
1073 				if (((tc->prev_src != NULL) && (tc->prev_src->consumed % tc->prev_src->blocklength)) ||
1074 				    ((tc->curr_src != NULL) && (tc->curr_src->consumed % tc->curr_src->blocklength)))
1075 				{
1076 					DEBUG_API(printk("cryptocop_setup_dma_list: can only disconnect from or connect to a unit on a multiple of the blocklength, old: cons=%d, prod=%d, block=%d, new: cons=%d prod=%d, block=%d.\n", tc->prev_src ? tc->prev_src->consumed : INT_MIN, tc->prev_src ? tc->prev_src->produced : INT_MIN, tc->prev_src ? tc->prev_src->blocklength : INT_MIN, tc->curr_src ? tc->curr_src->consumed : INT_MIN, tc->curr_src ? tc->curr_src->produced : INT_MIN, tc->curr_src ? tc->curr_src->blocklength : INT_MIN));
1077 					failed = -EINVAL;
1078 					goto error_cleanup;
1079 				}
1080 			}
1081 			/* Detect unit deactivation. */
1082 			if (dcfg->last) {
1083 				/* Length check of this is handled below. */
1084 				tc->done = 1;
1085 			}
1086 			dcfg = dcfg->next;
1087 		} /* while (dcfg) */
1088 		DEBUG(printk("cryptocop_setup_dma_list: parsing operation descriptor configuration complete.\n"));
1089 
1090 		if (cipher_ctx.active && (cipher_ctx.curr_src != NULL) && !cipher_ctx.curr_src->active){
1091 			DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", cipher_ctx.curr_src->unit_no));
1092 			failed = -EINVAL;
1093 			goto error_cleanup;
1094 		}
1095 		if (digest_ctx.active && (digest_ctx.curr_src != NULL) && !digest_ctx.curr_src->active){
1096 			DEBUG_API(printk("cryptocop_setup_dma_list: digest source from inactive unit %d\n", digest_ctx.curr_src->unit_no));
1097 			failed = -EINVAL;
1098 			goto error_cleanup;
1099 		}
1100 		if (csum_ctx.active && (csum_ctx.curr_src != NULL) && !csum_ctx.curr_src->active){
1101 			DEBUG_API(printk("cryptocop_setup_dma_list: cipher source from inactive unit %d\n", csum_ctx.curr_src->unit_no));
1102 			failed = -EINVAL;
1103 			goto error_cleanup;
1104 		}
1105 
1106 		/* Update consumed and produced lengths.
1107 
1108 		   The consumed length accounting here is actually cheating.  If a unit source from DMA (or any
1109 		   other unit that process data in blocks of one octet) it is correct, but if it source from a
1110 		   block processing unit, i.e. a cipher, it will be temporarily incorrect at some times.  However
1111 		   since it is only allowed--by the HW--to change source to or from a block processing unit at times where that
1112 		   unit has processed an exact multiple of its block length the end result will be correct.
1113 		   Beware that if the source change restriction change this code will need to be (much) reworked.
1114 		*/
1115 		DEBUG(printk("cryptocop_setup_dma_list: desc->length=%d, desc_len=%d.\n", odsc->length, desc_len));
1116 
1117 		if (csum_ctx.active) {
1118 			csum_ctx.consumed += desc_len;
1119 			if (csum_ctx.done) {
1120 				csum_ctx.produced = 2;
1121 			}
1122 			DEBUG(printk("cryptocop_setup_dma_list: csum_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", csum_ctx.consumed, csum_ctx.produced, csum_ctx.blocklength));
1123 		}
1124 		if (digest_ctx.active) {
1125 			digest_ctx.consumed += desc_len;
1126 			if (digest_ctx.done) {
1127 				if (digest_ctx.unit_no == src_md5) {
1128 					digest_ctx.produced = MD5_STATE_LENGTH;
1129 				} else {
1130 					digest_ctx.produced = SHA1_STATE_LENGTH;
1131 				}
1132 			}
1133 			DEBUG(printk("cryptocop_setup_dma_list: digest_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", digest_ctx.consumed, digest_ctx.produced, digest_ctx.blocklength));
1134 		}
1135 		if (cipher_ctx.active) {
1136 			/* Ciphers are allowed only to source from DMA out.  That is filtered above. */
1137 			assert(cipher_ctx.current_src == src_dma);
1138 			cipher_ctx.consumed += desc_len;
1139 			cipher_ctx.produced = cipher_ctx.blocklength * (cipher_ctx.consumed / cipher_ctx.blocklength);
1140 			if (cipher_ctx.cbcmode && !(cipher_ctx.tcfg->flags & CRYPTOCOP_EXPLICIT_IV) && cipher_ctx.produced){
1141 				cipher_ctx.produced -= cipher_ctx.blocklength; /* Compensate for CBC iv. */
1142 			}
1143 			DEBUG(printk("cryptocop_setup_dma_list: cipher_ctx producing: consumed=%d, produced=%d, blocklength=%d.\n", cipher_ctx.consumed, cipher_ctx.produced, cipher_ctx.blocklength));
1144 		}
1145 
1146 		/* Setup the DMA out descriptors. */
1147 		/* Configure the metadata. */
1148 		active_count = 0;
1149 		eop_needed_count = 0;
1150 		if (cipher_ctx.active) {
1151 			++active_count;
1152 			if (cipher_ctx.unit_no == src_dma){
1153 				/* mem2mem */
1154 				meta_out.ciphsel = src_none;
1155 			} else {
1156 				meta_out.ciphsel = cipher_ctx.current_src;
1157 			}
1158 			meta_out.ciphconf = cipher_ctx.ciph_conf;
1159 			meta_out.cbcmode = cipher_ctx.cbcmode;
1160 			meta_out.decrypt = cipher_ctx.decrypt;
1161 			DEBUG(printk("set ciphsel=%d ciphconf=%d cbcmode=%d decrypt=%d\n", meta_out.ciphsel, meta_out.ciphconf, meta_out.cbcmode, meta_out.decrypt));
1162 			if (cipher_ctx.done) ++eop_needed_count;
1163 		} else {
1164 			meta_out.ciphsel = src_none;
1165 		}
1166 
1167 		if (digest_ctx.active) {
1168 			++active_count;
1169 			meta_out.hashsel = digest_ctx.current_src;
1170 			meta_out.hashconf = digest_ctx.hash_conf;
1171 			meta_out.hashmode = 0; /* Explicit mode is not used here. */
1172 			DEBUG(printk("set hashsel=%d hashconf=%d hashmode=%d\n", meta_out.hashsel, meta_out.hashconf, meta_out.hashmode));
1173 			if (digest_ctx.done) {
1174 				assert(digest_ctx.pad_descs == NULL);
1175 				failed = create_pad_descriptor(&digest_ctx, &digest_ctx.pad_descs, alloc_flag);
1176 				if (failed) {
1177 					DEBUG_API(printk("cryptocop_setup_dma_list: failed digest pad creation.\n"));
1178 					goto error_cleanup;
1179 				}
1180 			}
1181 		} else {
1182 			meta_out.hashsel = src_none;
1183 		}
1184 
1185 		if (csum_ctx.active) {
1186 			++active_count;
1187 			meta_out.csumsel = csum_ctx.current_src;
1188 			if (csum_ctx.done) {
1189 				assert(csum_ctx.pad_descs == NULL);
1190 				failed = create_pad_descriptor(&csum_ctx, &csum_ctx.pad_descs, alloc_flag);
1191 				if (failed) {
1192 					DEBUG_API(printk("cryptocop_setup_dma_list: failed csum pad creation.\n"));
1193 					goto error_cleanup;
1194 				}
1195 			}
1196 		} else {
1197 			meta_out.csumsel = src_none;
1198 		}
1199 		DEBUG(printk("cryptocop_setup_dma_list: %d eop needed, %d active units\n", eop_needed_count, active_count));
1200 		/* Setup DMA out descriptors for the indata. */
1201 		failed = create_output_descriptors(operation, &iniov_ix, &iniov_offset, desc_len, &current_out_cdesc, &meta_out, alloc_flag);
1202 		if (failed) {
1203 			DEBUG_API(printk("cryptocop_setup_dma_list: create_output_descriptors %d\n", failed));
1204 			goto error_cleanup;
1205 		}
1206 		/* Setup out EOP.  If there are active units that are not done here they cannot get an EOP
1207 		 * so we ust setup a zero length descriptor to DMA to signal EOP only to done units.
1208 		 * If there is a pad descriptor EOP for the padded unit will be EOPed by it.
1209 		 */
1210 		assert(active_count >= eop_needed_count);
1211 		assert((eop_needed_count == 0) || (eop_needed_count == 1));
1212 		if (eop_needed_count) {
1213 			/* This means that the bulk operation (cipeher/m2m) is terminated. */
1214 			if (active_count > 1) {
1215 				/* Use zero length EOP descriptor. */
1216 				struct cryptocop_dma_desc *ed = alloc_cdesc(alloc_flag);
1217 				struct strcop_meta_out    ed_mo = {0};
1218 				if (!ed) {
1219 					DEBUG_API(printk("cryptocop_setup_dma_list: alloc EOP descriptor for cipher\n"));
1220 					failed = -ENOMEM;
1221 					goto error_cleanup;
1222 				}
1223 
1224 				assert(cipher_ctx.active && cipher_ctx.done);
1225 
1226 				if (cipher_ctx.unit_no == src_dma){
1227 					/* mem2mem */
1228 					ed_mo.ciphsel = src_none;
1229 				} else {
1230 					ed_mo.ciphsel = cipher_ctx.current_src;
1231 				}
1232 				ed_mo.ciphconf = cipher_ctx.ciph_conf;
1233 				ed_mo.cbcmode = cipher_ctx.cbcmode;
1234 				ed_mo.decrypt = cipher_ctx.decrypt;
1235 
1236 				ed->free_buf = NULL;
1237 				ed->dma_descr->wait = 1;
1238 				ed->dma_descr->out_eop = 1;
1239 
1240 				ed->dma_descr->buf = (char*)virt_to_phys(&ed); /* Use any valid physical address for zero length descriptor. */
1241 				ed->dma_descr->after = ed->dma_descr->buf;
1242 				ed->dma_descr->md = REG_TYPE_CONV(unsigned short int, struct strcop_meta_out, ed_mo);
1243 				current_out_cdesc->next = ed;
1244 				current_out_cdesc = ed;
1245 			} else {
1246 				/* Set EOP in the current out descriptor since the only active module is
1247 				 * the one needing the EOP. */
1248 
1249 				current_out_cdesc->dma_descr->out_eop = 1;
1250 			}
1251 		}
1252 
1253 		if (cipher_ctx.done && cipher_ctx.active) cipher_ctx.active = 0;
1254 		if (digest_ctx.done && digest_ctx.active) digest_ctx.active = 0;
1255 		if (csum_ctx.done && csum_ctx.active) csum_ctx.active = 0;
1256 		indata_ix += odsc->length;
1257 		odsc = odsc->next;
1258 	} /* while (odsc) */ /* Process descriptors. */
1259 	DEBUG(printk("cryptocop_setup_dma_list: done parsing operation descriptors\n"));
1260 	if (cipher_ctx.tcfg && (cipher_ctx.active || !cipher_ctx.done)){
1261 		DEBUG_API(printk("cryptocop_setup_dma_list: cipher operation not terminated.\n"));
1262 		failed = -EINVAL;
1263 		goto error_cleanup;
1264 	}
1265 	if (digest_ctx.tcfg && (digest_ctx.active || !digest_ctx.done)){
1266 		DEBUG_API(printk("cryptocop_setup_dma_list: digest operation not terminated.\n"));
1267 		failed = -EINVAL;
1268 		goto error_cleanup;
1269 	}
1270 	if (csum_ctx.tcfg && (csum_ctx.active || !csum_ctx.done)){
1271 		DEBUG_API(printk("cryptocop_setup_dma_list: csum operation not terminated.\n"));
1272 		failed = -EINVAL;
1273 		goto error_cleanup;
1274 	}
1275 
1276 	failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &cipher_ctx, alloc_flag);
1277 	if (failed){
1278 		DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1279 		goto error_cleanup;
1280 	}
1281 	failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &digest_ctx, alloc_flag);
1282 	if (failed){
1283 		DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1284 		goto error_cleanup;
1285 	}
1286 	failed = append_input_descriptors(operation, &current_in_cdesc, &current_out_cdesc, &csum_ctx, alloc_flag);
1287 	if (failed){
1288 		DEBUG_API(printk("cryptocop_setup_dma_list: append_input_descriptors cipher_ctx %d\n", failed));
1289 		goto error_cleanup;
1290 	}
1291 
1292 	DEBUG(printk("cryptocop_setup_dma_list: int_op=0x%p, *int_op=0x%p\n", int_op, *int_op));
1293 	(*int_op)->cdesc_out = out_cdesc_head.next;
1294 	(*int_op)->cdesc_in = in_cdesc_head.next;
1295 	DEBUG(printk("cryptocop_setup_dma_list: out_cdesc_head=0x%p in_cdesc_head=0x%p\n", (*int_op)->cdesc_out, (*int_op)->cdesc_in));
1296 
1297 	setup_descr_chain(out_cdesc_head.next);
1298 	setup_descr_chain(in_cdesc_head.next);
1299 
1300 	/* Last but not least: mark the last DMA in descriptor for a INTR and EOL and the the
1301 	 * last DMA out descriptor for EOL.
1302 	 */
1303 	current_in_cdesc->dma_descr->intr = 1;
1304 	current_in_cdesc->dma_descr->eol = 1;
1305 	current_out_cdesc->dma_descr->eol = 1;
1306 
1307 	/* Setup DMA contexts. */
1308 	(*int_op)->ctx_out.next = NULL;
1309 	(*int_op)->ctx_out.eol = 1;
1310 	(*int_op)->ctx_out.intr = 0;
1311 	(*int_op)->ctx_out.store_mode = 0;
1312 	(*int_op)->ctx_out.en = 0;
1313 	(*int_op)->ctx_out.dis = 0;
1314 	(*int_op)->ctx_out.md0 = 0;
1315 	(*int_op)->ctx_out.md1 = 0;
1316 	(*int_op)->ctx_out.md2 = 0;
1317 	(*int_op)->ctx_out.md3 = 0;
1318 	(*int_op)->ctx_out.md4 = 0;
1319 	(*int_op)->ctx_out.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_out->dma_descr);
1320 	(*int_op)->ctx_out.saved_data_buf = (*int_op)->cdesc_out->dma_descr->buf; /* Already physical address. */
1321 
1322 	(*int_op)->ctx_in.next = NULL;
1323 	(*int_op)->ctx_in.eol = 1;
1324 	(*int_op)->ctx_in.intr = 0;
1325 	(*int_op)->ctx_in.store_mode = 0;
1326 	(*int_op)->ctx_in.en = 0;
1327 	(*int_op)->ctx_in.dis = 0;
1328 	(*int_op)->ctx_in.md0 = 0;
1329 	(*int_op)->ctx_in.md1 = 0;
1330 	(*int_op)->ctx_in.md2 = 0;
1331 	(*int_op)->ctx_in.md3 = 0;
1332 	(*int_op)->ctx_in.md4 = 0;
1333 
1334 	(*int_op)->ctx_in.saved_data = (dma_descr_data*)virt_to_phys((*int_op)->cdesc_in->dma_descr);
1335 	(*int_op)->ctx_in.saved_data_buf = (*int_op)->cdesc_in->dma_descr->buf; /* Already physical address. */
1336 
1337 	DEBUG(printk("cryptocop_setup_dma_list: done\n"));
1338 	return 0;
1339 
1340 error_cleanup:
1341 	{
1342 		/* Free all allocated resources. */
1343 		struct cryptocop_dma_desc *tmp_cdesc;
1344 		while (digest_ctx.pad_descs){
1345 			tmp_cdesc = digest_ctx.pad_descs->next;
1346 			free_cdesc(digest_ctx.pad_descs);
1347 			digest_ctx.pad_descs = tmp_cdesc;
1348 		}
1349 		while (csum_ctx.pad_descs){
1350 			tmp_cdesc = csum_ctx.pad_descs->next;
1351 			free_cdesc(csum_ctx.pad_descs);
1352 			csum_ctx.pad_descs = tmp_cdesc;
1353 		}
1354 		assert(cipher_ctx.pad_descs == NULL); /* The ciphers are never padded. */
1355 
1356 		if (*int_op != NULL) delete_internal_operation(*int_op);
1357 	}
1358 	DEBUG_API(printk("cryptocop_setup_dma_list: done with error %d\n", failed));
1359 	return failed;
1360 }
1361 
1362 
delete_internal_operation(struct cryptocop_int_operation * iop)1363 static void delete_internal_operation(struct cryptocop_int_operation *iop)
1364 {
1365 	void                      *ptr = iop->alloc_ptr;
1366 	struct cryptocop_dma_desc *cd = iop->cdesc_out;
1367 	struct cryptocop_dma_desc *next;
1368 
1369 	DEBUG(printk("delete_internal_operation: iop=0x%p, alloc_ptr=0x%p\n", iop, ptr));
1370 
1371 	while (cd) {
1372 		next = cd->next;
1373 		free_cdesc(cd);
1374 		cd = next;
1375 	}
1376 	cd = iop->cdesc_in;
1377 	while (cd) {
1378 		next = cd->next;
1379 		free_cdesc(cd);
1380 		cd = next;
1381 	}
1382 	kfree(ptr);
1383 }
1384 
1385 #define MD5_MIN_PAD_LENGTH (9)
1386 #define MD5_PAD_LENGTH_FIELD_LENGTH (8)
1387 
create_md5_pad(int alloc_flag,unsigned long long hashed_length,char ** pad,size_t * pad_length)1388 static int create_md5_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1389 {
1390 	size_t                  padlen = MD5_BLOCK_LENGTH - (hashed_length % MD5_BLOCK_LENGTH);
1391 	unsigned char           *p;
1392 	int                     i;
1393 	unsigned long long int  bit_length = hashed_length << 3;
1394 
1395 	if (padlen < MD5_MIN_PAD_LENGTH) padlen += MD5_BLOCK_LENGTH;
1396 
1397 	p = kmalloc(padlen, alloc_flag);
1398 	if (!p) return -ENOMEM;
1399 
1400 	*p = 0x80;
1401 	memset(p+1, 0, padlen - 1);
1402 
1403 	DEBUG(printk("create_md5_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1404 
1405 	i = padlen - MD5_PAD_LENGTH_FIELD_LENGTH;
1406 	while (bit_length != 0){
1407 		p[i++] = bit_length % 0x100;
1408 		bit_length >>= 8;
1409 	}
1410 
1411 	*pad = (char*)p;
1412 	*pad_length = padlen;
1413 
1414 	return 0;
1415 }
1416 
1417 #define SHA1_MIN_PAD_LENGTH (9)
1418 #define SHA1_PAD_LENGTH_FIELD_LENGTH (8)
1419 
create_sha1_pad(int alloc_flag,unsigned long long hashed_length,char ** pad,size_t * pad_length)1420 static int create_sha1_pad(int alloc_flag, unsigned long long hashed_length, char **pad, size_t *pad_length)
1421 {
1422 	size_t                  padlen = SHA1_BLOCK_LENGTH - (hashed_length % SHA1_BLOCK_LENGTH);
1423 	unsigned char           *p;
1424 	int                     i;
1425 	unsigned long long int  bit_length = hashed_length << 3;
1426 
1427 	if (padlen < SHA1_MIN_PAD_LENGTH) padlen += SHA1_BLOCK_LENGTH;
1428 
1429 	p = kmalloc(padlen, alloc_flag);
1430 	if (!p) return -ENOMEM;
1431 
1432 	*p = 0x80;
1433 	memset(p+1, 0, padlen - 1);
1434 
1435 	DEBUG(printk("create_sha1_pad: hashed_length=%lld bits == %lld bytes\n", bit_length, hashed_length));
1436 
1437 	i = padlen - 1;
1438 	while (bit_length != 0){
1439 		p[i--] = bit_length % 0x100;
1440 		bit_length >>= 8;
1441 	}
1442 
1443 	*pad = (char*)p;
1444 	*pad_length = padlen;
1445 
1446 	return 0;
1447 }
1448 
1449 
transform_ok(struct cryptocop_transform_init * tinit)1450 static int transform_ok(struct cryptocop_transform_init *tinit)
1451 {
1452 	switch (tinit->alg){
1453 	case cryptocop_alg_csum:
1454 		switch (tinit->csum_mode){
1455 		case cryptocop_csum_le:
1456 		case cryptocop_csum_be:
1457 			break;
1458 		default:
1459 			DEBUG_API(printk("transform_ok: Bad mode set for csum transform\n"));
1460 			return -EINVAL;
1461 		}
1462 	case cryptocop_alg_mem2mem:
1463 	case cryptocop_alg_md5:
1464 	case cryptocop_alg_sha1:
1465 		if (tinit->keylen != 0) {
1466 			DEBUG_API(printk("transform_ok: non-zero keylength, %d, for a digest/csum algorithm\n", tinit->keylen));
1467 			return -EINVAL; /* This check is a bit strict. */
1468 		}
1469 		break;
1470 	case cryptocop_alg_des:
1471 		if (tinit->keylen != 64) {
1472 			DEBUG_API(printk("transform_ok: keylen %d invalid for DES\n", tinit->keylen));
1473 			return -EINVAL;
1474 		}
1475 		break;
1476 	case cryptocop_alg_3des:
1477 		if (tinit->keylen != 192) {
1478 			DEBUG_API(printk("transform_ok: keylen %d invalid for 3DES\n", tinit->keylen));
1479 			return -EINVAL;
1480 		}
1481 		break;
1482 	case cryptocop_alg_aes:
1483 		if (tinit->keylen != 128 && tinit->keylen != 192 && tinit->keylen != 256) {
1484 			DEBUG_API(printk("transform_ok: keylen %d invalid for AES\n", tinit->keylen));
1485 			return -EINVAL;
1486 		}
1487 		break;
1488 	case cryptocop_no_alg:
1489 	default:
1490 		DEBUG_API(printk("transform_ok: no such algorithm %d\n", tinit->alg));
1491 		return -EINVAL;
1492 	}
1493 
1494 	switch (tinit->alg){
1495 	case cryptocop_alg_des:
1496 	case cryptocop_alg_3des:
1497 	case cryptocop_alg_aes:
1498 		if (tinit->cipher_mode != cryptocop_cipher_mode_ecb && tinit->cipher_mode != cryptocop_cipher_mode_cbc) return -EINVAL;
1499 	default:
1500 		 break;
1501 	}
1502 	return 0;
1503 }
1504 
1505 
cryptocop_new_session(cryptocop_session_id * sid,struct cryptocop_transform_init * tinit,int alloc_flag)1506 int cryptocop_new_session(cryptocop_session_id *sid, struct cryptocop_transform_init *tinit, int alloc_flag)
1507 {
1508 	struct cryptocop_session         *sess;
1509 	struct cryptocop_transform_init  *tfrm_in = tinit;
1510 	struct cryptocop_transform_init  *tmp_in;
1511 	int                              no_tfrms = 0;
1512 	int                              i;
1513 	unsigned long int                flags;
1514 
1515 	init_stream_coprocessor(); /* For safety if we are called early */
1516 
1517 	while (tfrm_in){
1518 		int err;
1519 		++no_tfrms;
1520 		if ((err = transform_ok(tfrm_in))) {
1521 			DEBUG_API(printk("cryptocop_new_session, bad transform\n"));
1522 			return err;
1523 		}
1524 		tfrm_in = tfrm_in->next;
1525 	}
1526 	if (0 == no_tfrms) {
1527 		DEBUG_API(printk("cryptocop_new_session, no transforms specified\n"));
1528 		return -EINVAL;
1529 	}
1530 
1531 	sess = kmalloc(sizeof(struct cryptocop_session), alloc_flag);
1532 	if (!sess){
1533 		DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_session\n"));
1534 		return -ENOMEM;
1535 	}
1536 
1537 	sess->tfrm_ctx = kmalloc(no_tfrms * sizeof(struct cryptocop_transform_ctx), alloc_flag);
1538 	if (!sess->tfrm_ctx) {
1539 		DEBUG_API(printk("cryptocop_new_session, kmalloc cryptocop_transform_ctx\n"));
1540 		kfree(sess);
1541 		return -ENOMEM;
1542 	}
1543 
1544 	tfrm_in = tinit;
1545 	for (i = 0; i < no_tfrms; i++){
1546 		tmp_in = tfrm_in->next;
1547 		while (tmp_in){
1548 			if (tmp_in->tid == tfrm_in->tid) {
1549 				DEBUG_API(printk("cryptocop_new_session, duplicate transform ids\n"));
1550 				kfree(sess->tfrm_ctx);
1551 				kfree(sess);
1552 				return -EINVAL;
1553 			}
1554 			tmp_in = tmp_in->next;
1555 		}
1556 		memcpy(&sess->tfrm_ctx[i].init, tfrm_in, sizeof(struct cryptocop_transform_init));
1557 		sess->tfrm_ctx[i].dec_key_set = 0;
1558 		sess->tfrm_ctx[i].next = &sess->tfrm_ctx[i] + 1;
1559 
1560 		tfrm_in = tfrm_in->next;
1561 	}
1562 	sess->tfrm_ctx[i-1].next = NULL;
1563 
1564 	spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1565 	sess->sid = next_sid;
1566 	next_sid++;
1567 	/* TODO If we are really paranoid we should do duplicate check to handle sid wraparound.
1568 	 *      OTOH 2^64 is a really large number of session. */
1569 	if (next_sid == 0) next_sid = 1;
1570 
1571 	/* Prepend to session list. */
1572 	sess->next = cryptocop_sessions;
1573 	cryptocop_sessions = sess;
1574 	spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1575 	*sid = sess->sid;
1576 	return 0;
1577 }
1578 
1579 
cryptocop_free_session(cryptocop_session_id sid)1580 int cryptocop_free_session(cryptocop_session_id sid)
1581 {
1582 	struct cryptocop_transform_ctx    *tc;
1583 	struct cryptocop_session          *sess = NULL;
1584 	struct cryptocop_session          *psess = NULL;
1585 	unsigned long int                 flags;
1586 	int                               i;
1587 	LIST_HEAD(remove_list);
1588 	struct list_head                  *node, *tmp;
1589 	struct cryptocop_prio_job         *pj;
1590 
1591 	DEBUG(printk("cryptocop_free_session: sid=%lld\n", sid));
1592 
1593 	spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1594 	sess = cryptocop_sessions;
1595 	while (sess && sess->sid != sid){
1596 		psess = sess;
1597 		sess = sess->next;
1598 	}
1599 	if (sess){
1600 		if (psess){
1601 			psess->next = sess->next;
1602 		} else {
1603 			cryptocop_sessions = sess->next;
1604 		}
1605 	}
1606 	spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1607 
1608 	if (!sess) return -EINVAL;
1609 
1610 	/* Remove queued jobs. */
1611 	spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1612 
1613 	for (i = 0; i < cryptocop_prio_no_prios; i++){
1614 		if (!list_empty(&(cryptocop_job_queues[i].jobs))){
1615 			list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
1616 				pj = list_entry(node, struct cryptocop_prio_job, node);
1617 				if (pj->oper->sid == sid) {
1618 					list_move_tail(node, &remove_list);
1619 				}
1620 			}
1621 		}
1622 	}
1623 	spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1624 
1625 	list_for_each_safe(node, tmp, &remove_list) {
1626 		list_del(node);
1627 		pj = list_entry(node, struct cryptocop_prio_job, node);
1628 		pj->oper->operation_status = -EAGAIN;  /* EAGAIN is not ideal for job/session terminated but it's the best choice I know of. */
1629 		DEBUG(printk("cryptocop_free_session: pj=0x%p, pj->oper=0x%p, pj->iop=0x%p\n", pj, pj->oper, pj->iop));
1630 		pj->oper->cb(pj->oper, pj->oper->cb_data);
1631 		delete_internal_operation(pj->iop);
1632 		kfree(pj);
1633 	}
1634 
1635 	tc = sess->tfrm_ctx;
1636 	/* Erase keying data. */
1637 	while (tc){
1638 		DEBUG(printk("cryptocop_free_session: memset keys, tfrm id=%d\n", tc->init.tid));
1639 		memset(tc->init.key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1640 		memset(tc->dec_key, 0xff, CRYPTOCOP_MAX_KEY_LENGTH);
1641 		tc = tc->next;
1642 	}
1643 	kfree(sess->tfrm_ctx);
1644 	kfree(sess);
1645 
1646 	return 0;
1647 }
1648 
get_session(cryptocop_session_id sid)1649 static struct cryptocop_session *get_session(cryptocop_session_id sid)
1650 {
1651 	struct cryptocop_session    *sess;
1652 	unsigned long int           flags;
1653 
1654 	spin_lock_irqsave(&cryptocop_sessions_lock, flags);
1655 	sess = cryptocop_sessions;
1656 	while (sess && (sess->sid != sid)){
1657 		sess = sess->next;
1658 	}
1659 	spin_unlock_irqrestore(&cryptocop_sessions_lock, flags);
1660 
1661 	return sess;
1662 }
1663 
get_transform_ctx(struct cryptocop_session * sess,cryptocop_tfrm_id tid)1664 static struct cryptocop_transform_ctx *get_transform_ctx(struct cryptocop_session *sess, cryptocop_tfrm_id tid)
1665 {
1666 	struct cryptocop_transform_ctx *tc = sess->tfrm_ctx;
1667 
1668 	DEBUG(printk("get_transform_ctx, sess=0x%p, tid=%d\n", sess, tid));
1669 	assert(sess != NULL);
1670 	while (tc && tc->init.tid != tid){
1671 		DEBUG(printk("tc=0x%p, tc->next=0x%p\n", tc, tc->next));
1672 		tc = tc->next;
1673 	}
1674 	DEBUG(printk("get_transform_ctx, returning tc=0x%p\n", tc));
1675 	return tc;
1676 }
1677 
1678 
1679 
1680 /* The AES s-transform matrix (s-box). */
1681 static const u8 aes_sbox[256] = {
1682 	99,  124, 119, 123, 242, 107, 111, 197, 48,  1,   103, 43,  254, 215, 171, 118,
1683 	202, 130, 201, 125, 250, 89,  71,  240, 173, 212, 162, 175, 156, 164, 114, 192,
1684 	183, 253, 147, 38,  54,  63,  247, 204, 52,  165, 229, 241, 113, 216, 49,  21,
1685 	4,   199, 35,  195, 24,  150, 5,   154, 7,   18,  128, 226, 235, 39,  178, 117,
1686 	9,   131, 44,  26,  27,  110, 90,  160, 82,  59,  214, 179, 41,  227, 47,  132,
1687 	83,  209, 0,   237, 32,  252, 177, 91,  106, 203, 190, 57,  74,  76,  88,  207,
1688 	208, 239, 170, 251, 67,  77,  51,  133, 69,  249, 2,   127, 80,  60,  159, 168,
1689 	81,  163, 64,  143, 146, 157, 56,  245, 188, 182, 218, 33,  16,  255, 243, 210,
1690 	205, 12,  19,  236, 95,  151, 68,  23,  196, 167, 126, 61,  100, 93,  25,  115,
1691 	96,  129, 79,  220, 34,  42,  144, 136, 70,  238, 184, 20,  222, 94,  11,  219,
1692 	224, 50,  58,  10,  73,  6,   36,  92,  194, 211, 172, 98,  145, 149, 228, 121,
1693 	231, 200, 55,  109, 141, 213, 78,  169, 108, 86,  244, 234, 101, 122, 174, 8,
1694 	186, 120, 37,  46,  28,  166, 180, 198, 232, 221, 116, 31,  75,  189, 139, 138,
1695 	112, 62,  181, 102, 72,  3,   246, 14,  97,  53,  87,  185, 134, 193, 29,  158,
1696 	225, 248, 152, 17,  105, 217, 142, 148, 155, 30,  135, 233, 206, 85,  40,  223,
1697 	140, 161, 137, 13,  191, 230, 66,  104, 65,  153, 45,  15,  176, 84,  187, 22
1698 };
1699 
1700 /* AES has a 32 bit word round constants for each round in the
1701  * key schedule.  round_constant[i] is really Rcon[i+1] in FIPS187.
1702  */
1703 static u32 round_constant[11] = {
1704 	0x01000000, 0x02000000, 0x04000000, 0x08000000,
1705 	0x10000000, 0x20000000, 0x40000000, 0x80000000,
1706 	0x1B000000, 0x36000000, 0x6C000000
1707 };
1708 
1709 /* Apply the s-box to each of the four occtets in w. */
aes_ks_subword(const u32 w)1710 static u32 aes_ks_subword(const u32 w)
1711 {
1712 	u8 bytes[4];
1713 
1714 	*(u32*)(&bytes[0]) = w;
1715 	bytes[0] = aes_sbox[bytes[0]];
1716 	bytes[1] = aes_sbox[bytes[1]];
1717 	bytes[2] = aes_sbox[bytes[2]];
1718 	bytes[3] = aes_sbox[bytes[3]];
1719 	return *(u32*)(&bytes[0]);
1720 }
1721 
1722 /* The encrypt (forward) Rijndael key schedule algorithm pseudo code:
1723  * (Note that AES words are 32 bit long)
1724  *
1725  * KeyExpansion(byte key[4*Nk], word w[Nb*(Nr+1)], Nk){
1726  * word temp
1727  * i = 0
1728  * while (i < Nk) {
1729  *   w[i] = word(key[4*i, 4*i + 1, 4*i + 2, 4*i + 3])
1730  *   i = i + 1
1731  * }
1732  * i = Nk
1733  *
1734  * while (i < (Nb * (Nr + 1))) {
1735  *   temp = w[i - 1]
1736  *   if ((i mod Nk) == 0) {
1737  *     temp = SubWord(RotWord(temp)) xor Rcon[i/Nk]
1738  *   }
1739  *   else if ((Nk > 6) && ((i mod Nk) == 4)) {
1740  *     temp = SubWord(temp)
1741  *   }
1742  *   w[i] = w[i - Nk] xor temp
1743  * }
1744  * RotWord(t) does a 8 bit cyclic shift left on a 32 bit word.
1745  * SubWord(t) applies the AES s-box individually to each octet
1746  * in a 32 bit word.
1747  *
1748  * For AES Nk can have the values 4, 6, and 8 (corresponding to
1749  * values for Nr of 10, 12, and 14).  Nb is always 4.
1750  *
1751  * To construct w[i], w[i - 1] and w[i - Nk] must be
1752  * available.  Consequently we must keep a state of the last Nk words
1753  * to be able to create the last round keys.
1754  */
get_aes_decrypt_key(unsigned char * dec_key,const unsigned char * key,unsigned int keylength)1755 static void get_aes_decrypt_key(unsigned char *dec_key, const unsigned  char *key, unsigned int keylength)
1756 {
1757 	u32 temp;
1758 	u32 w_ring[8]; /* nk is max 8, use elements 0..(nk - 1) as a ringbuffer */
1759 	u8  w_last_ix;
1760 	int i;
1761 	u8  nr, nk;
1762 
1763 	switch (keylength){
1764 	case 128:
1765 		nk = 4;
1766 		nr = 10;
1767 		break;
1768 	case 192:
1769 		nk = 6;
1770 		nr = 12;
1771 		break;
1772 	case 256:
1773 		nk = 8;
1774 		nr = 14;
1775 		break;
1776 	default:
1777 		panic("stream co-processor: bad aes key length in get_aes_decrypt_key\n");
1778 	};
1779 
1780 	/* Need to do host byte order correction here since key is byte oriented and the
1781 	 * kx algorithm is word (u32) oriented. */
1782 	for (i = 0; i < nk; i+=1) {
1783 		w_ring[i] = be32_to_cpu(*(u32*)&key[4*i]);
1784 	}
1785 
1786 	i = (int)nk;
1787 	w_last_ix = i - 1;
1788 	while (i < (4 * (nr + 2))) {
1789 		temp = w_ring[w_last_ix];
1790 		if (!(i % nk)) {
1791 			/* RotWord(temp) */
1792 			temp = (temp << 8) | (temp >> 24);
1793 			temp = aes_ks_subword(temp);
1794 			temp ^= round_constant[i/nk - 1];
1795 		} else if ((nk > 6) && ((i % nk) == 4)) {
1796 			temp = aes_ks_subword(temp);
1797 		}
1798 		w_last_ix = (w_last_ix + 1) % nk; /* This is the same as (i-Nk) mod Nk */
1799 		temp ^= w_ring[w_last_ix];
1800 		w_ring[w_last_ix] = temp;
1801 
1802 		/* We need the round keys for round Nr+1 and Nr+2 (round key
1803 		 * Nr+2 is the round key beyond the last one used when
1804 		 * encrypting).  Rounds are numbered starting from 0, Nr=10
1805 		 * implies 11 rounds are used in encryption/decryption.
1806 		 */
1807 		if (i >= (4 * nr)) {
1808 			/* Need to do host byte order correction here, the key
1809 			 * is byte oriented. */
1810 			*(u32*)dec_key = cpu_to_be32(temp);
1811 			dec_key += 4;
1812 		}
1813 		++i;
1814 	}
1815 }
1816 
1817 
1818 /**** Job/operation management. ****/
1819 
cryptocop_job_queue_insert_csum(struct cryptocop_operation * operation)1820 int cryptocop_job_queue_insert_csum(struct cryptocop_operation *operation)
1821 {
1822 	return cryptocop_job_queue_insert(cryptocop_prio_kernel_csum, operation);
1823 }
1824 
cryptocop_job_queue_insert_crypto(struct cryptocop_operation * operation)1825 int cryptocop_job_queue_insert_crypto(struct cryptocop_operation *operation)
1826 {
1827 	return cryptocop_job_queue_insert(cryptocop_prio_kernel, operation);
1828 }
1829 
cryptocop_job_queue_insert_user_job(struct cryptocop_operation * operation)1830 int cryptocop_job_queue_insert_user_job(struct cryptocop_operation *operation)
1831 {
1832 	return cryptocop_job_queue_insert(cryptocop_prio_user, operation);
1833 }
1834 
cryptocop_job_queue_insert(cryptocop_queue_priority prio,struct cryptocop_operation * operation)1835 static int cryptocop_job_queue_insert(cryptocop_queue_priority prio, struct cryptocop_operation *operation)
1836 {
1837 	int                           ret;
1838 	struct cryptocop_prio_job     *pj = NULL;
1839 	unsigned long int             flags;
1840 
1841 	DEBUG(printk("cryptocop_job_queue_insert(%d, 0x%p)\n", prio, operation));
1842 
1843 	if (!operation || !operation->cb){
1844 		DEBUG_API(printk("cryptocop_job_queue_insert oper=0x%p, NULL operation or callback\n", operation));
1845 		return -EINVAL;
1846 	}
1847 
1848 	if ((ret = cryptocop_job_setup(&pj, operation)) != 0){
1849 		DEBUG_API(printk("cryptocop_job_queue_insert: job setup failed\n"));
1850 		return ret;
1851 	}
1852 	assert(pj != NULL);
1853 
1854 	spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
1855 	list_add_tail(&pj->node, &cryptocop_job_queues[prio].jobs);
1856 	spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
1857 
1858 	/* Make sure a job is running */
1859 	cryptocop_start_job();
1860 	return 0;
1861 }
1862 
1863 static void cryptocop_do_tasklet(unsigned long unused);
1864 DECLARE_TASKLET (cryptocop_tasklet, cryptocop_do_tasklet, 0);
1865 
cryptocop_do_tasklet(unsigned long unused)1866 static void cryptocop_do_tasklet(unsigned long unused)
1867 {
1868 	struct list_head             *node;
1869 	struct cryptocop_prio_job    *pj = NULL;
1870 	unsigned long                flags;
1871 
1872 	DEBUG(printk("cryptocop_do_tasklet: entering\n"));
1873 
1874 	do {
1875 		spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
1876 		if (!list_empty(&cryptocop_completed_jobs)){
1877 			node = cryptocop_completed_jobs.next;
1878 			list_del(node);
1879 			pj = list_entry(node, struct cryptocop_prio_job, node);
1880 		} else {
1881 			pj = NULL;
1882 		}
1883 		spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
1884 		if (pj) {
1885 			assert(pj->oper != NULL);
1886 
1887 			/* Notify consumer of operation completeness. */
1888 			DEBUG(printk("cryptocop_do_tasklet: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
1889 
1890 			pj->oper->operation_status = 0; /* Job is completed. */
1891 			pj->oper->cb(pj->oper, pj->oper->cb_data);
1892 			delete_internal_operation(pj->iop);
1893 			kfree(pj);
1894 		}
1895 	} while (pj != NULL);
1896 
1897 	DEBUG(printk("cryptocop_do_tasklet: exiting\n"));
1898 }
1899 
1900 static irqreturn_t
dma_done_interrupt(int irq,void * dev_id)1901 dma_done_interrupt(int irq, void *dev_id)
1902 {
1903 	struct cryptocop_prio_job *done_job;
1904 	reg_dma_rw_ack_intr ack_intr = {
1905 		.data = 1,
1906 	};
1907 
1908 	REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
1909 
1910 	DEBUG(printk("cryptocop DMA done\n"));
1911 
1912 	spin_lock(&running_job_lock);
1913 	if (cryptocop_running_job == NULL){
1914 		printk("stream co-processor got interrupt when not busy\n");
1915 		spin_unlock(&running_job_lock);
1916 		return IRQ_HANDLED;
1917 	}
1918 	done_job = cryptocop_running_job;
1919 	cryptocop_running_job = NULL;
1920 	spin_unlock(&running_job_lock);
1921 
1922 	/* Start processing a job. */
1923 	if (!spin_trylock(&cryptocop_process_lock)){
1924 		DEBUG(printk("cryptocop irq handler, not starting a job\n"));
1925 	} else {
1926 		cryptocop_start_job();
1927 		spin_unlock(&cryptocop_process_lock);
1928 	}
1929 
1930 	done_job->oper->operation_status = 0; /* Job is completed. */
1931 	if (done_job->oper->fast_callback){
1932 		/* This operation wants callback from interrupt. */
1933 		done_job->oper->cb(done_job->oper, done_job->oper->cb_data);
1934 		delete_internal_operation(done_job->iop);
1935 		kfree(done_job);
1936 	} else {
1937 		spin_lock(&cryptocop_completed_jobs_lock);
1938 		list_add_tail(&(done_job->node), &cryptocop_completed_jobs);
1939 		spin_unlock(&cryptocop_completed_jobs_lock);
1940 		tasklet_schedule(&cryptocop_tasklet);
1941 	}
1942 
1943 	DEBUG(printk("cryptocop leave irq handler\n"));
1944 	return IRQ_HANDLED;
1945 }
1946 
1947 
1948 /* Setup interrupts and DMA channels. */
init_cryptocop(void)1949 static int init_cryptocop(void)
1950 {
1951 	unsigned long          flags;
1952 	reg_dma_rw_cfg         dma_cfg = {.en = 1};
1953 	reg_dma_rw_intr_mask   intr_mask_in = {.data = regk_dma_yes}; /* Only want descriptor interrupts from the DMA in channel. */
1954 	reg_dma_rw_ack_intr    ack_intr = {.data = 1,.in_eop = 1 };
1955 	reg_strcop_rw_cfg      strcop_cfg = {
1956 		.ipend = regk_strcop_little,
1957 		.td1 = regk_strcop_e,
1958 		.td2 = regk_strcop_d,
1959 		.td3 = regk_strcop_e,
1960 		.ignore_sync = 0,
1961 		.en = 1
1962 	};
1963 
1964 	if (request_irq(DMA_IRQ, dma_done_interrupt, 0,
1965 			"stream co-processor DMA", NULL))
1966 		panic("request_irq stream co-processor irq dma9");
1967 
1968 	(void)crisv32_request_dma(OUT_DMA, "strcop", DMA_PANIC_ON_ERROR,
1969 		0, dma_strp);
1970 	(void)crisv32_request_dma(IN_DMA, "strcop", DMA_PANIC_ON_ERROR,
1971 		0, dma_strp);
1972 
1973 	local_irq_save(flags);
1974 
1975 	/* Reset and enable the cryptocop. */
1976 	strcop_cfg.en = 0;
1977 	REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1978 	strcop_cfg.en = 1;
1979 	REG_WR(strcop, regi_strcop, rw_cfg, strcop_cfg);
1980 
1981 	/* Enable DMAs. */
1982 	REG_WR(dma, IN_DMA_INST, rw_cfg, dma_cfg); /* input DMA */
1983 	REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_cfg); /* output DMA */
1984 
1985 	/* Set up wordsize = 4 for DMAs. */
1986 	DMA_WR_CMD(OUT_DMA_INST, regk_dma_set_w_size4);
1987 	DMA_WR_CMD(IN_DMA_INST, regk_dma_set_w_size4);
1988 
1989 	/* Enable interrupts. */
1990 	REG_WR(dma, IN_DMA_INST, rw_intr_mask, intr_mask_in);
1991 
1992 	/* Clear intr ack. */
1993 	REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
1994 
1995 	local_irq_restore(flags);
1996 
1997 	return 0;
1998 }
1999 
2000 /* Free used cryptocop hw resources (interrupt and DMA channels). */
release_cryptocop(void)2001 static void release_cryptocop(void)
2002 {
2003 	unsigned long          flags;
2004 	reg_dma_rw_cfg         dma_cfg = {.en = 0};
2005 	reg_dma_rw_intr_mask   intr_mask_in = {0};
2006 	reg_dma_rw_ack_intr    ack_intr = {.data = 1,.in_eop = 1 };
2007 
2008 	local_irq_save(flags);
2009 
2010 	/* Clear intr ack. */
2011 	REG_WR(dma, IN_DMA_INST, rw_ack_intr, ack_intr);
2012 
2013 	/* Disable DMAs. */
2014 	REG_WR(dma, IN_DMA_INST, rw_cfg, dma_cfg); /* input DMA */
2015 	REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_cfg); /* output DMA */
2016 
2017 	/* Disable interrupts. */
2018 	REG_WR(dma, IN_DMA_INST, rw_intr_mask, intr_mask_in);
2019 
2020 	local_irq_restore(flags);
2021 
2022 	free_irq(DMA_IRQ, NULL);
2023 
2024 	(void)crisv32_free_dma(OUT_DMA);
2025 	(void)crisv32_free_dma(IN_DMA);
2026 }
2027 
2028 
2029 /* Init job queue. */
cryptocop_job_queue_init(void)2030 static int cryptocop_job_queue_init(void)
2031 {
2032 	int i;
2033 
2034 	INIT_LIST_HEAD(&cryptocop_completed_jobs);
2035 
2036 	for (i = 0; i < cryptocop_prio_no_prios; i++){
2037 		cryptocop_job_queues[i].prio = (cryptocop_queue_priority)i;
2038 		INIT_LIST_HEAD(&cryptocop_job_queues[i].jobs);
2039 	}
2040 	return 0;
2041 }
2042 
2043 
cryptocop_job_queue_close(void)2044 static void cryptocop_job_queue_close(void)
2045 {
2046 	struct list_head               *node, *tmp;
2047 	struct cryptocop_prio_job      *pj = NULL;
2048 	unsigned long int              process_flags, flags;
2049 	int                            i;
2050 
2051 	/* FIXME: This is as yet untested code. */
2052 
2053 	/* Stop strcop from getting an operation to process while we are closing the
2054 	   module. */
2055 	spin_lock_irqsave(&cryptocop_process_lock, process_flags);
2056 
2057 	/* Empty the job queue. */
2058 	for (i = 0; i < cryptocop_prio_no_prios; i++){
2059 		if (!list_empty(&(cryptocop_job_queues[i].jobs))){
2060 			list_for_each_safe(node, tmp, &(cryptocop_job_queues[i].jobs)) {
2061 				pj = list_entry(node, struct cryptocop_prio_job, node);
2062 				list_del(node);
2063 
2064 				/* Call callback to notify consumer of job removal. */
2065 				DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2066 				pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2067 				pj->oper->cb(pj->oper, pj->oper->cb_data);
2068 
2069 				delete_internal_operation(pj->iop);
2070 				kfree(pj);
2071 			}
2072 		}
2073 	}
2074 	spin_unlock_irqrestore(&cryptocop_process_lock, process_flags);
2075 
2076 	/* Remove the running job, if any. */
2077 	spin_lock_irqsave(&running_job_lock, flags);
2078 	if (cryptocop_running_job){
2079 		reg_strcop_rw_cfg rw_cfg;
2080 		reg_dma_rw_cfg    dma_out_cfg, dma_in_cfg;
2081 
2082 		/* Stop DMA. */
2083 		dma_out_cfg = REG_RD(dma, OUT_DMA_INST, rw_cfg);
2084 		dma_out_cfg.en = regk_dma_no;
2085 		REG_WR(dma, OUT_DMA_INST, rw_cfg, dma_out_cfg);
2086 
2087 		dma_in_cfg = REG_RD(dma, IN_DMA_INST, rw_cfg);
2088 		dma_in_cfg.en = regk_dma_no;
2089 		REG_WR(dma, IN_DMA_INST, rw_cfg, dma_in_cfg);
2090 
2091 		/* Disble the cryptocop. */
2092 		rw_cfg = REG_RD(strcop, regi_strcop, rw_cfg);
2093 		rw_cfg.en = 0;
2094 		REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2095 
2096 		pj = cryptocop_running_job;
2097 		cryptocop_running_job = NULL;
2098 
2099 		/* Call callback to notify consumer of job removal. */
2100 		DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2101 		pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2102 		pj->oper->cb(pj->oper, pj->oper->cb_data);
2103 
2104 		delete_internal_operation(pj->iop);
2105 		kfree(pj);
2106 	}
2107 	spin_unlock_irqrestore(&running_job_lock, flags);
2108 
2109 	/* Remove completed jobs, if any. */
2110 	spin_lock_irqsave(&cryptocop_completed_jobs_lock, flags);
2111 
2112 	list_for_each_safe(node, tmp, &cryptocop_completed_jobs) {
2113 		pj = list_entry(node, struct cryptocop_prio_job, node);
2114 		list_del(node);
2115 		/* Call callback to notify consumer of job removal. */
2116 		DEBUG(printk("cryptocop_job_queue_close: callback 0x%p, data 0x%p\n", pj->oper->cb, pj->oper->cb_data));
2117 		pj->oper->operation_status = -EINTR; /* Job is terminated without completion. */
2118 		pj->oper->cb(pj->oper, pj->oper->cb_data);
2119 
2120 		delete_internal_operation(pj->iop);
2121 		kfree(pj);
2122 	}
2123 	spin_unlock_irqrestore(&cryptocop_completed_jobs_lock, flags);
2124 }
2125 
2126 
cryptocop_start_job(void)2127 static void cryptocop_start_job(void)
2128 {
2129 	int                          i;
2130 	struct cryptocop_prio_job    *pj;
2131 	unsigned long int            flags;
2132 	unsigned long int            running_job_flags;
2133 	reg_strcop_rw_cfg            rw_cfg = {.en = 1, .ignore_sync = 0};
2134 
2135 	DEBUG(printk("cryptocop_start_job: entering\n"));
2136 
2137 	spin_lock_irqsave(&running_job_lock, running_job_flags);
2138 	if (cryptocop_running_job != NULL){
2139 		/* Already running. */
2140 		DEBUG(printk("cryptocop_start_job: already running, exit\n"));
2141 		spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2142 		return;
2143 	}
2144 	spin_lock_irqsave(&cryptocop_job_queue_lock, flags);
2145 
2146 	/* Check the queues in priority order. */
2147 	for (i = cryptocop_prio_kernel_csum; (i < cryptocop_prio_no_prios) && list_empty(&cryptocop_job_queues[i].jobs); i++);
2148 	if (i == cryptocop_prio_no_prios) {
2149 		spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2150 		spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2151 		DEBUG(printk("cryptocop_start_job: no jobs to run\n"));
2152 		return; /* No jobs to run */
2153 	}
2154 	DEBUG(printk("starting job for prio %d\n", i));
2155 
2156 	/* TODO: Do not starve lower priority jobs.  Let in a lower
2157 	 * prio job for every N-th processed higher prio job or some
2158 	 * other scheduling policy.  This could reasonably be
2159 	 * tweakable since the optimal balance would depend on the
2160 	 * type of load on the system. */
2161 
2162 	/* Pull the DMA lists from the job and start the DMA client. */
2163 	pj = list_entry(cryptocop_job_queues[i].jobs.next, struct cryptocop_prio_job, node);
2164 	list_del(&pj->node);
2165 	spin_unlock_irqrestore(&cryptocop_job_queue_lock, flags);
2166 	cryptocop_running_job = pj;
2167 
2168 	/* Set config register (3DES and CSUM modes). */
2169 	switch (pj->iop->tdes_mode){
2170 	case cryptocop_3des_eee:
2171 		rw_cfg.td1 = regk_strcop_e;
2172 		rw_cfg.td2 = regk_strcop_e;
2173 		rw_cfg.td3 = regk_strcop_e;
2174 		break;
2175 	case cryptocop_3des_eed:
2176 		rw_cfg.td1 = regk_strcop_e;
2177 		rw_cfg.td2 = regk_strcop_e;
2178 		rw_cfg.td3 = regk_strcop_d;
2179 		break;
2180 	case cryptocop_3des_ede:
2181 		rw_cfg.td1 = regk_strcop_e;
2182 		rw_cfg.td2 = regk_strcop_d;
2183 		rw_cfg.td3 = regk_strcop_e;
2184 		break;
2185 	case cryptocop_3des_edd:
2186 		rw_cfg.td1 = regk_strcop_e;
2187 		rw_cfg.td2 = regk_strcop_d;
2188 		rw_cfg.td3 = regk_strcop_d;
2189 		break;
2190 	case cryptocop_3des_dee:
2191 		rw_cfg.td1 = regk_strcop_d;
2192 		rw_cfg.td2 = regk_strcop_e;
2193 		rw_cfg.td3 = regk_strcop_e;
2194 		break;
2195 	case cryptocop_3des_ded:
2196 		rw_cfg.td1 = regk_strcop_d;
2197 		rw_cfg.td2 = regk_strcop_e;
2198 		rw_cfg.td3 = regk_strcop_d;
2199 		break;
2200 	case cryptocop_3des_dde:
2201 		rw_cfg.td1 = regk_strcop_d;
2202 		rw_cfg.td2 = regk_strcop_d;
2203 		rw_cfg.td3 = regk_strcop_e;
2204 		break;
2205 	case cryptocop_3des_ddd:
2206 		rw_cfg.td1 = regk_strcop_d;
2207 		rw_cfg.td2 = regk_strcop_d;
2208 		rw_cfg.td3 = regk_strcop_d;
2209 		break;
2210 	default:
2211 		DEBUG(printk("cryptocop_setup_dma_list: bad 3DES mode\n"));
2212 	}
2213 	switch (pj->iop->csum_mode){
2214 	case cryptocop_csum_le:
2215 		rw_cfg.ipend = regk_strcop_little;
2216 		break;
2217 	case cryptocop_csum_be:
2218 		rw_cfg.ipend = regk_strcop_big;
2219 		break;
2220 	default:
2221 		DEBUG(printk("cryptocop_setup_dma_list: bad checksum mode\n"));
2222 	}
2223 	REG_WR(strcop, regi_strcop, rw_cfg, rw_cfg);
2224 
2225 	DEBUG(printk("cryptocop_start_job: starting DMA, new cryptocop_running_job=0x%p\n"
2226 		     "ctx_in: 0x%p, phys: 0x%p\n"
2227 		     "ctx_out: 0x%p, phys: 0x%p\n",
2228 		     pj,
2229 		     &pj->iop->ctx_in, (char*)virt_to_phys(&pj->iop->ctx_in),
2230 		     &pj->iop->ctx_out, (char*)virt_to_phys(&pj->iop->ctx_out)));
2231 
2232 	/* Start input DMA. */
2233 	flush_dma_context(&pj->iop->ctx_in);
2234 	DMA_START_CONTEXT(IN_DMA_INST, virt_to_phys(&pj->iop->ctx_in));
2235 
2236 	/* Start output DMA. */
2237 	DMA_START_CONTEXT(OUT_DMA_INST, virt_to_phys(&pj->iop->ctx_out));
2238 
2239 	spin_unlock_irqrestore(&running_job_lock, running_job_flags);
2240 	DEBUG(printk("cryptocop_start_job: exiting\n"));
2241 }
2242 
2243 
cryptocop_job_setup(struct cryptocop_prio_job ** pj,struct cryptocop_operation * operation)2244 static int cryptocop_job_setup(struct cryptocop_prio_job **pj, struct cryptocop_operation *operation)
2245 {
2246 	int  err;
2247 	int  alloc_flag = operation->in_interrupt ? GFP_ATOMIC : GFP_KERNEL;
2248 	void *iop_alloc_ptr = NULL;
2249 
2250 	*pj = kmalloc(sizeof (struct cryptocop_prio_job), alloc_flag);
2251 	if (!*pj) return -ENOMEM;
2252 
2253 	DEBUG(printk("cryptocop_job_setup: operation=0x%p\n", operation));
2254 
2255 	(*pj)->oper = operation;
2256 	DEBUG(printk("cryptocop_job_setup, cb=0x%p cb_data=0x%p\n",  (*pj)->oper->cb, (*pj)->oper->cb_data));
2257 
2258 	if (operation->use_dmalists) {
2259 		DEBUG(print_user_dma_lists(&operation->list_op));
2260 		if (!operation->list_op.inlist || !operation->list_op.outlist || !operation->list_op.out_data_buf || !operation->list_op.in_data_buf){
2261 			DEBUG_API(printk("cryptocop_job_setup: bad indata (use_dmalists)\n"));
2262 			kfree(*pj);
2263 			return -EINVAL;
2264 		}
2265 		iop_alloc_ptr = kmalloc(DESCR_ALLOC_PAD + sizeof(struct cryptocop_int_operation), alloc_flag);
2266 		if (!iop_alloc_ptr) {
2267 			DEBUG_API(printk("cryptocop_job_setup: kmalloc cryptocop_int_operation\n"));
2268 			kfree(*pj);
2269 			return -ENOMEM;
2270 		}
2271 		(*pj)->iop = (struct cryptocop_int_operation*)(((unsigned long int)(iop_alloc_ptr + DESCR_ALLOC_PAD + offsetof(struct cryptocop_int_operation, ctx_out)) & ~0x0000001F) - offsetof(struct cryptocop_int_operation, ctx_out));
2272 		DEBUG(memset((*pj)->iop, 0xff, sizeof(struct cryptocop_int_operation)));
2273 		(*pj)->iop->alloc_ptr = iop_alloc_ptr;
2274 		(*pj)->iop->sid = operation->sid;
2275 		(*pj)->iop->cdesc_out = NULL;
2276 		(*pj)->iop->cdesc_in = NULL;
2277 		(*pj)->iop->tdes_mode = operation->list_op.tdes_mode;
2278 		(*pj)->iop->csum_mode = operation->list_op.csum_mode;
2279 		(*pj)->iop->ddesc_out = operation->list_op.outlist;
2280 		(*pj)->iop->ddesc_in = operation->list_op.inlist;
2281 
2282 		/* Setup DMA contexts. */
2283 		(*pj)->iop->ctx_out.next = NULL;
2284 		(*pj)->iop->ctx_out.eol = 1;
2285 		(*pj)->iop->ctx_out.saved_data = operation->list_op.outlist;
2286 		(*pj)->iop->ctx_out.saved_data_buf = operation->list_op.out_data_buf;
2287 
2288 		(*pj)->iop->ctx_in.next = NULL;
2289 		(*pj)->iop->ctx_in.eol = 1;
2290 		(*pj)->iop->ctx_in.saved_data = operation->list_op.inlist;
2291 		(*pj)->iop->ctx_in.saved_data_buf = operation->list_op.in_data_buf;
2292 	} else {
2293 		if ((err = cryptocop_setup_dma_list(operation, &(*pj)->iop, alloc_flag))) {
2294 			DEBUG_API(printk("cryptocop_job_setup: cryptocop_setup_dma_list failed %d\n", err));
2295 			kfree(*pj);
2296 			return err;
2297 		}
2298 	}
2299 	DEBUG(print_dma_descriptors((*pj)->iop));
2300 
2301 	DEBUG(printk("cryptocop_job_setup, DMA list setup successful\n"));
2302 
2303 	return 0;
2304 }
2305 
cryptocop_open(struct inode * inode,struct file * filp)2306 static int cryptocop_open(struct inode *inode, struct file *filp)
2307 {
2308 	int p = iminor(inode);
2309 
2310 	if (p != CRYPTOCOP_MINOR) return -EINVAL;
2311 
2312 	filp->private_data = NULL;
2313 	return 0;
2314 }
2315 
2316 
cryptocop_release(struct inode * inode,struct file * filp)2317 static int cryptocop_release(struct inode *inode, struct file *filp)
2318 {
2319 	struct cryptocop_private *dev = filp->private_data;
2320 	struct cryptocop_private *dev_next;
2321 
2322 	while (dev){
2323 		dev_next = dev->next;
2324 		if (dev->sid != CRYPTOCOP_SESSION_ID_NONE) {
2325 			(void)cryptocop_free_session(dev->sid);
2326 		}
2327 		kfree(dev);
2328 		dev = dev_next;
2329 	}
2330 
2331 	return 0;
2332 }
2333 
2334 
cryptocop_ioctl_close_session(struct inode * inode,struct file * filp,unsigned int cmd,unsigned long arg)2335 static int cryptocop_ioctl_close_session(struct inode *inode, struct file *filp,
2336 					 unsigned int cmd, unsigned long arg)
2337 {
2338 	struct cryptocop_private  *dev = filp->private_data;
2339 	struct cryptocop_private  *prev_dev = NULL;
2340 	struct strcop_session_op  *sess_op = (struct strcop_session_op *)arg;
2341 	struct strcop_session_op  sop;
2342 	int                       err;
2343 
2344 	DEBUG(printk("cryptocop_ioctl_close_session\n"));
2345 
2346 	if (!access_ok(VERIFY_READ, sess_op, sizeof(struct strcop_session_op)))
2347 		return -EFAULT;
2348 	err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2349 	if (err) return -EFAULT;
2350 
2351 	while (dev && (dev->sid != sop.ses_id)) {
2352 		prev_dev = dev;
2353 		dev = dev->next;
2354 	}
2355 	if (dev){
2356 		if (prev_dev){
2357 			prev_dev->next = dev->next;
2358 		} else {
2359 			filp->private_data = dev->next;
2360 		}
2361 		err = cryptocop_free_session(dev->sid);
2362 		if (err) return -EFAULT;
2363 	} else {
2364 		DEBUG_API(printk("cryptocop_ioctl_close_session: session %lld not found\n", sop.ses_id));
2365 		return -EINVAL;
2366 	}
2367 	return 0;
2368 }
2369 
2370 
ioctl_process_job_callback(struct cryptocop_operation * op,void * cb_data)2371 static void ioctl_process_job_callback(struct cryptocop_operation *op, void*cb_data)
2372 {
2373 	struct ioctl_job_cb_ctx *jc = (struct ioctl_job_cb_ctx *)cb_data;
2374 
2375 	DEBUG(printk("ioctl_process_job_callback: op=0x%p, cb_data=0x%p\n", op, cb_data));
2376 
2377 	jc->processed = 1;
2378 	wake_up(&cryptocop_ioc_process_wq);
2379 }
2380 
2381 
2382 #define CRYPTOCOP_IOCTL_CIPHER_TID  (1)
2383 #define CRYPTOCOP_IOCTL_DIGEST_TID  (2)
2384 #define CRYPTOCOP_IOCTL_CSUM_TID    (3)
2385 
first_cfg_change_ix(struct strcop_crypto_op * crp_op)2386 static size_t first_cfg_change_ix(struct strcop_crypto_op *crp_op)
2387 {
2388 	size_t ch_ix = 0;
2389 
2390 	if (crp_op->do_cipher) ch_ix = crp_op->cipher_start;
2391 	if (crp_op->do_digest && (crp_op->digest_start < ch_ix)) ch_ix = crp_op->digest_start;
2392 	if (crp_op->do_csum && (crp_op->csum_start < ch_ix)) ch_ix = crp_op->csum_start;
2393 
2394 	DEBUG(printk("first_cfg_change_ix: ix=%d\n", ch_ix));
2395 	return ch_ix;
2396 }
2397 
2398 
next_cfg_change_ix(struct strcop_crypto_op * crp_op,size_t ix)2399 static size_t next_cfg_change_ix(struct strcop_crypto_op *crp_op, size_t ix)
2400 {
2401 	size_t ch_ix = INT_MAX;
2402 	size_t tmp_ix = 0;
2403 
2404 	if (crp_op->do_cipher && ((crp_op->cipher_start + crp_op->cipher_len) > ix)){
2405 		if (crp_op->cipher_start > ix) {
2406 			ch_ix = crp_op->cipher_start;
2407 		} else {
2408 			ch_ix = crp_op->cipher_start + crp_op->cipher_len;
2409 		}
2410 	}
2411 	if (crp_op->do_digest && ((crp_op->digest_start + crp_op->digest_len) > ix)){
2412 		if (crp_op->digest_start > ix) {
2413 			tmp_ix = crp_op->digest_start;
2414 		} else {
2415 			tmp_ix = crp_op->digest_start + crp_op->digest_len;
2416 		}
2417 		if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2418 	}
2419 	if (crp_op->do_csum && ((crp_op->csum_start + crp_op->csum_len) > ix)){
2420 		if (crp_op->csum_start > ix) {
2421 			tmp_ix = crp_op->csum_start;
2422 		} else {
2423 			tmp_ix = crp_op->csum_start + crp_op->csum_len;
2424 		}
2425 		if (tmp_ix < ch_ix) ch_ix = tmp_ix;
2426 	}
2427 	if (ch_ix == INT_MAX) ch_ix = ix;
2428 	DEBUG(printk("next_cfg_change_ix prev ix=%d, next ix=%d\n", ix, ch_ix));
2429 	return ch_ix;
2430 }
2431 
2432 
2433 /* Map map_length bytes from the pages starting on *pageix and *pageoffset to iovecs starting on *iovix.
2434  * Return -1 for ok, 0 for fail. */
map_pages_to_iovec(struct iovec * iov,int iovlen,int * iovix,struct page ** pages,int nopages,int * pageix,int * pageoffset,int map_length)2435 static int map_pages_to_iovec(struct iovec *iov, int iovlen, int *iovix, struct page **pages, int nopages, int *pageix, int *pageoffset, int map_length )
2436 {
2437 	int tmplen;
2438 
2439 	assert(iov != NULL);
2440 	assert(iovix != NULL);
2441 	assert(pages != NULL);
2442 	assert(pageix != NULL);
2443 	assert(pageoffset != NULL);
2444 
2445 	DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2446 
2447 	while (map_length > 0){
2448 		DEBUG(printk("map_pages_to_iovec, map_length=%d, iovlen=%d, *iovix=%d, nopages=%d, *pageix=%d, *pageoffset=%d\n", map_length, iovlen, *iovix, nopages, *pageix, *pageoffset));
2449 		if (*iovix >= iovlen){
2450 			DEBUG_API(printk("map_page_to_iovec: *iovix=%d >= iovlen=%d\n", *iovix, iovlen));
2451 			return 0;
2452 		}
2453 		if (*pageix >= nopages){
2454 			DEBUG_API(printk("map_page_to_iovec: *pageix=%d >= nopages=%d\n", *pageix, nopages));
2455 			return 0;
2456 		}
2457 		iov[*iovix].iov_base = (unsigned char*)page_address(pages[*pageix]) + *pageoffset;
2458 		tmplen = PAGE_SIZE - *pageoffset;
2459 		if (tmplen < map_length){
2460 			(*pageoffset) = 0;
2461 			(*pageix)++;
2462 		} else {
2463 			tmplen = map_length;
2464 			(*pageoffset) += map_length;
2465 		}
2466 		DEBUG(printk("mapping %d bytes from page %d (or %d) to iovec %d\n", tmplen, *pageix, *pageix-1, *iovix));
2467 		iov[*iovix].iov_len = tmplen;
2468 		map_length -= tmplen;
2469 		(*iovix)++;
2470 	}
2471 	DEBUG(printk("map_page_to_iovec, exit, *iovix=%d\n", *iovix));
2472 	return -1;
2473 }
2474 
2475 
2476 
cryptocop_ioctl_process(struct inode * inode,struct file * filp,unsigned int cmd,unsigned long arg)2477 static int cryptocop_ioctl_process(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2478 {
2479 	int                             i;
2480 	struct cryptocop_private        *dev = filp->private_data;
2481 	struct strcop_crypto_op         *crp_oper = (struct strcop_crypto_op *)arg;
2482 	struct strcop_crypto_op         oper = {0};
2483 	int                             err = 0;
2484 	struct cryptocop_operation      *cop = NULL;
2485 
2486 	struct ioctl_job_cb_ctx         *jc = NULL;
2487 
2488 	struct page                     **inpages = NULL;
2489 	struct page                     **outpages = NULL;
2490 	int                             noinpages = 0;
2491 	int                             nooutpages = 0;
2492 
2493 	struct cryptocop_desc           descs[5]; /* Max 5 descriptors are needed, there are three transforms that
2494 						   * can get connected/disconnected on different places in the indata. */
2495 	struct cryptocop_desc_cfg       dcfgs[5*3];
2496 	int                             desc_ix = 0;
2497 	int                             dcfg_ix = 0;
2498 	struct cryptocop_tfrm_cfg       ciph_tcfg = {0};
2499 	struct cryptocop_tfrm_cfg       digest_tcfg = {0};
2500 	struct cryptocop_tfrm_cfg       csum_tcfg = {0};
2501 
2502 	unsigned char                   *digest_result = NULL;
2503 	int                             digest_length = 0;
2504 	int                             cblocklen = 0;
2505 	unsigned char                   csum_result[CSUM_BLOCK_LENGTH];
2506 	struct cryptocop_session        *sess;
2507 
2508 	int    iovlen = 0;
2509 	int    iovix = 0;
2510 	int    pageix = 0;
2511 	int    pageoffset = 0;
2512 
2513 	size_t prev_ix = 0;
2514 	size_t next_ix;
2515 
2516 	int    cipher_active, digest_active, csum_active;
2517 	int    end_digest, end_csum;
2518 	int    digest_done = 0;
2519 	int    cipher_done = 0;
2520 	int    csum_done = 0;
2521 
2522 	DEBUG(printk("cryptocop_ioctl_process\n"));
2523 
2524 	if (!access_ok(VERIFY_WRITE, crp_oper, sizeof(struct strcop_crypto_op))){
2525 		DEBUG_API(printk("cryptocop_ioctl_process: !access_ok crp_oper!\n"));
2526 		return -EFAULT;
2527 	}
2528 	if (copy_from_user(&oper, crp_oper, sizeof(struct strcop_crypto_op))) {
2529 		DEBUG_API(printk("cryptocop_ioctl_process: copy_from_user\n"));
2530 		return -EFAULT;
2531 	}
2532 	DEBUG(print_strcop_crypto_op(&oper));
2533 
2534 	while (dev && dev->sid != oper.ses_id) dev = dev->next;
2535 	if (!dev){
2536 		DEBUG_API(printk("cryptocop_ioctl_process: session %lld not found\n", oper.ses_id));
2537 		return -EINVAL;
2538 	}
2539 
2540 	/* Check buffers. */
2541 	if (((oper.indata + oper.inlen) < oper.indata) || ((oper.cipher_outdata + oper.cipher_outlen) < oper.cipher_outdata)){
2542 		DEBUG_API(printk("cryptocop_ioctl_process: user buffers wrapped around, bad user!\n"));
2543 		return -EINVAL;
2544 	}
2545 
2546 	if (!access_ok(VERIFY_WRITE, oper.cipher_outdata, oper.cipher_outlen)){
2547 		DEBUG_API(printk("cryptocop_ioctl_process: !access_ok out data!\n"));
2548 		return -EFAULT;
2549 	}
2550 	if (!access_ok(VERIFY_READ, oper.indata, oper.inlen)){
2551 		DEBUG_API(printk("cryptocop_ioctl_process: !access_ok in data!\n"));
2552 		return -EFAULT;
2553 	}
2554 
2555 	cop = kmalloc(sizeof(struct cryptocop_operation), GFP_KERNEL);
2556 	if (!cop) {
2557 		DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2558 		return -ENOMEM;
2559 	}
2560 	jc = kmalloc(sizeof(struct ioctl_job_cb_ctx), GFP_KERNEL);
2561 	if (!jc) {
2562 		DEBUG_API(printk("cryptocop_ioctl_process: kmalloc\n"));
2563 		err = -ENOMEM;
2564 		goto error_cleanup;
2565 	}
2566 	jc->processed = 0;
2567 
2568 	cop->cb_data = jc;
2569 	cop->cb = ioctl_process_job_callback;
2570 	cop->operation_status = 0;
2571 	cop->use_dmalists = 0;
2572 	cop->in_interrupt = 0;
2573 	cop->fast_callback = 0;
2574 	cop->tfrm_op.tfrm_cfg = NULL;
2575 	cop->tfrm_op.desc = NULL;
2576 	cop->tfrm_op.indata = NULL;
2577 	cop->tfrm_op.incount = 0;
2578 	cop->tfrm_op.inlen = 0;
2579 	cop->tfrm_op.outdata = NULL;
2580 	cop->tfrm_op.outcount = 0;
2581 	cop->tfrm_op.outlen = 0;
2582 
2583 	sess = get_session(oper.ses_id);
2584 	if (!sess){
2585 		DEBUG_API(printk("cryptocop_ioctl_process: bad session id.\n"));
2586 		kfree(cop);
2587 		kfree(jc);
2588 		return -EINVAL;
2589 	}
2590 
2591 	if (oper.do_cipher) {
2592 		unsigned int                    cipher_outlen = 0;
2593 		struct cryptocop_transform_ctx  *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_CIPHER_TID);
2594 		if (!tc) {
2595 			DEBUG_API(printk("cryptocop_ioctl_process: no cipher transform in session.\n"));
2596 			err = -EINVAL;
2597 			goto error_cleanup;
2598 		}
2599 		ciph_tcfg.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2600 		ciph_tcfg.inject_ix = 0;
2601 		ciph_tcfg.flags = 0;
2602 		if ((oper.cipher_start < 0) || (oper.cipher_len <= 0) || (oper.cipher_start > oper.inlen) || ((oper.cipher_start + oper.cipher_len) > oper.inlen)){
2603 			DEBUG_API(printk("cryptocop_ioctl_process: bad cipher length\n"));
2604 			kfree(cop);
2605 			kfree(jc);
2606 			return -EINVAL;
2607 		}
2608 		cblocklen = tc->init.alg == cryptocop_alg_aes ? AES_BLOCK_LENGTH : DES_BLOCK_LENGTH;
2609 		if (oper.cipher_len % cblocklen) {
2610 			kfree(cop);
2611 			kfree(jc);
2612 			DEBUG_API(printk("cryptocop_ioctl_process: cipher inlength not multiple of block length.\n"));
2613 			return -EINVAL;
2614 		}
2615 		cipher_outlen = oper.cipher_len;
2616 		if (tc->init.cipher_mode == cryptocop_cipher_mode_cbc){
2617 			if (oper.cipher_explicit) {
2618 				ciph_tcfg.flags |= CRYPTOCOP_EXPLICIT_IV;
2619 				memcpy(ciph_tcfg.iv, oper.cipher_iv, cblocklen);
2620 			} else {
2621 				cipher_outlen = oper.cipher_len - cblocklen;
2622 			}
2623 		} else {
2624 			if (oper.cipher_explicit){
2625 				kfree(cop);
2626 				kfree(jc);
2627 				DEBUG_API(printk("cryptocop_ioctl_process: explicit_iv when not CBC mode\n"));
2628 				return -EINVAL;
2629 			}
2630 		}
2631 		if (oper.cipher_outlen != cipher_outlen) {
2632 			kfree(cop);
2633 			kfree(jc);
2634 			DEBUG_API(printk("cryptocop_ioctl_process: cipher_outlen incorrect, should be %d not %d.\n", cipher_outlen, oper.cipher_outlen));
2635 			return -EINVAL;
2636 		}
2637 
2638 		if (oper.decrypt){
2639 			ciph_tcfg.flags |= CRYPTOCOP_DECRYPT;
2640 		} else {
2641 			ciph_tcfg.flags |= CRYPTOCOP_ENCRYPT;
2642 		}
2643 		ciph_tcfg.next = cop->tfrm_op.tfrm_cfg;
2644 		cop->tfrm_op.tfrm_cfg = &ciph_tcfg;
2645 	}
2646 	if (oper.do_digest){
2647 		struct cryptocop_transform_ctx *tc = get_transform_ctx(sess, CRYPTOCOP_IOCTL_DIGEST_TID);
2648 		if (!tc) {
2649 			DEBUG_API(printk("cryptocop_ioctl_process: no digest transform in session.\n"));
2650 			err = -EINVAL;
2651 			goto error_cleanup;
2652 		}
2653 		digest_length = tc->init.alg == cryptocop_alg_md5 ? 16 : 20;
2654 		digest_result = kmalloc(digest_length, GFP_KERNEL);
2655 		if (!digest_result) {
2656 			DEBUG_API(printk("cryptocop_ioctl_process: kmalloc digest_result\n"));
2657 			err = -EINVAL;
2658 			goto error_cleanup;
2659 		}
2660 		DEBUG(memset(digest_result, 0xff, digest_length));
2661 
2662 		digest_tcfg.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2663 		digest_tcfg.inject_ix = 0;
2664 		ciph_tcfg.inject_ix += digest_length;
2665 		if ((oper.digest_start < 0) || (oper.digest_len <= 0) || (oper.digest_start > oper.inlen) || ((oper.digest_start + oper.digest_len) > oper.inlen)){
2666 			DEBUG_API(printk("cryptocop_ioctl_process: bad digest length\n"));
2667 			err = -EINVAL;
2668 			goto error_cleanup;
2669 		}
2670 
2671 		digest_tcfg.next = cop->tfrm_op.tfrm_cfg;
2672 		cop->tfrm_op.tfrm_cfg = &digest_tcfg;
2673 	}
2674 	if (oper.do_csum){
2675 		csum_tcfg.tid = CRYPTOCOP_IOCTL_CSUM_TID;
2676 		csum_tcfg.inject_ix = digest_length;
2677 		ciph_tcfg.inject_ix += 2;
2678 
2679 		if ((oper.csum_start < 0) || (oper.csum_len <= 0) || (oper.csum_start > oper.inlen) || ((oper.csum_start + oper.csum_len) > oper.inlen)){
2680 			DEBUG_API(printk("cryptocop_ioctl_process: bad csum length\n"));
2681 			kfree(cop);
2682 			kfree(jc);
2683 			return -EINVAL;
2684 		}
2685 
2686 		csum_tcfg.next = cop->tfrm_op.tfrm_cfg;
2687 		cop->tfrm_op.tfrm_cfg = &csum_tcfg;
2688 	}
2689 
2690 	prev_ix = first_cfg_change_ix(&oper);
2691 	if (prev_ix > oper.inlen) {
2692 		DEBUG_API(printk("cryptocop_ioctl_process: length mismatch\n"));
2693 		nooutpages = noinpages = 0;
2694 		err = -EINVAL;
2695 		goto error_cleanup;
2696 	}
2697 	DEBUG(printk("cryptocop_ioctl_process: inlen=%d, cipher_outlen=%d\n", oper.inlen, oper.cipher_outlen));
2698 
2699 	/* Map user pages for in and out data of the operation. */
2700 	noinpages = (((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK) + oper.inlen - 1 - prev_ix + ~PAGE_MASK) >> PAGE_SHIFT;
2701 	DEBUG(printk("cryptocop_ioctl_process: noinpages=%d\n", noinpages));
2702 	inpages = kmalloc(noinpages * sizeof(struct page*), GFP_KERNEL);
2703 	if (!inpages){
2704 		DEBUG_API(printk("cryptocop_ioctl_process: kmalloc inpages\n"));
2705 		nooutpages = noinpages = 0;
2706 		err = -ENOMEM;
2707 		goto error_cleanup;
2708 	}
2709 	if (oper.do_cipher){
2710 		nooutpages = (((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) + oper.cipher_outlen - 1 + ~PAGE_MASK) >> PAGE_SHIFT;
2711 		DEBUG(printk("cryptocop_ioctl_process: nooutpages=%d\n", nooutpages));
2712 		outpages = kmalloc(nooutpages * sizeof(struct page*), GFP_KERNEL);
2713 		if (!outpages){
2714 			DEBUG_API(printk("cryptocop_ioctl_process: kmalloc outpages\n"));
2715 			nooutpages = noinpages = 0;
2716 			err = -ENOMEM;
2717 			goto error_cleanup;
2718 		}
2719 	}
2720 
2721 	/* Acquire the mm page semaphore. */
2722 	down_read(&current->mm->mmap_sem);
2723 
2724 	err = get_user_pages(current,
2725 			     current->mm,
2726 			     (unsigned long int)(oper.indata + prev_ix),
2727 			     noinpages,
2728 			     0,  /* read access only for in data */
2729 			     0, /* no force */
2730 			     inpages,
2731 			     NULL);
2732 
2733 	if (err < 0) {
2734 		up_read(&current->mm->mmap_sem);
2735 		nooutpages = noinpages = 0;
2736 		DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages indata\n"));
2737 		goto error_cleanup;
2738 	}
2739 	noinpages = err;
2740 	if (oper.do_cipher){
2741 		err = get_user_pages(current,
2742 				     current->mm,
2743 				     (unsigned long int)oper.cipher_outdata,
2744 				     nooutpages,
2745 				     1, /* write access for out data */
2746 				     0, /* no force */
2747 				     outpages,
2748 				     NULL);
2749 		up_read(&current->mm->mmap_sem);
2750 		if (err < 0) {
2751 			nooutpages = 0;
2752 			DEBUG_API(printk("cryptocop_ioctl_process: get_user_pages outdata\n"));
2753 			goto error_cleanup;
2754 		}
2755 		nooutpages = err;
2756 	} else {
2757 		up_read(&current->mm->mmap_sem);
2758 	}
2759 
2760 	/* Add 6 to nooutpages to make room for possibly inserted buffers for storing digest and
2761 	 * csum output and splits when units are (dis-)connected. */
2762 	cop->tfrm_op.indata = kmalloc((noinpages) * sizeof(struct iovec), GFP_KERNEL);
2763 	cop->tfrm_op.outdata = kmalloc((6 + nooutpages) * sizeof(struct iovec), GFP_KERNEL);
2764 	if (!cop->tfrm_op.indata || !cop->tfrm_op.outdata) {
2765 		DEBUG_API(printk("cryptocop_ioctl_process: kmalloc iovecs\n"));
2766 		err = -ENOMEM;
2767 		goto error_cleanup;
2768 	}
2769 
2770 	cop->tfrm_op.inlen = oper.inlen - prev_ix;
2771 	cop->tfrm_op.outlen = 0;
2772 	if (oper.do_cipher) cop->tfrm_op.outlen += oper.cipher_outlen;
2773 	if (oper.do_digest) cop->tfrm_op.outlen += digest_length;
2774 	if (oper.do_csum) cop->tfrm_op.outlen += 2;
2775 
2776 	/* Setup the in iovecs. */
2777 	cop->tfrm_op.incount = noinpages;
2778 	if (noinpages > 1){
2779 		size_t tmplen = cop->tfrm_op.inlen;
2780 
2781 		cop->tfrm_op.indata[0].iov_len = PAGE_SIZE - ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2782 		cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2783 		tmplen -= cop->tfrm_op.indata[0].iov_len;
2784 		for (i = 1; i<noinpages; i++){
2785 			cop->tfrm_op.indata[i].iov_len = tmplen < PAGE_SIZE ? tmplen : PAGE_SIZE;
2786 			cop->tfrm_op.indata[i].iov_base = (unsigned char*)page_address(inpages[i]);
2787 			tmplen -= PAGE_SIZE;
2788 		}
2789 	} else {
2790 		cop->tfrm_op.indata[0].iov_len = oper.inlen - prev_ix;
2791 		cop->tfrm_op.indata[0].iov_base = (unsigned char*)page_address(inpages[0]) + ((unsigned long int)(oper.indata + prev_ix) & ~PAGE_MASK);
2792 	}
2793 
2794 	iovlen = nooutpages + 6;
2795 	pageoffset = oper.do_cipher ? ((unsigned long int)oper.cipher_outdata & ~PAGE_MASK) : 0;
2796 
2797 	next_ix = next_cfg_change_ix(&oper, prev_ix);
2798 	if (prev_ix == next_ix){
2799 		DEBUG_API(printk("cryptocop_ioctl_process: length configuration broken.\n"));
2800 		err = -EINVAL;  /* This should be impossible barring bugs. */
2801 		goto error_cleanup;
2802 	}
2803 	while (prev_ix != next_ix){
2804 		end_digest = end_csum = cipher_active = digest_active = csum_active = 0;
2805 		descs[desc_ix].cfg = NULL;
2806 		descs[desc_ix].length = next_ix - prev_ix;
2807 
2808 		if (oper.do_cipher && (oper.cipher_start < next_ix) && (prev_ix < (oper.cipher_start + oper.cipher_len))) {
2809 			dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CIPHER_TID;
2810 			dcfgs[dcfg_ix].src = cryptocop_source_dma;
2811 			cipher_active = 1;
2812 
2813 			if (next_ix == (oper.cipher_start + oper.cipher_len)){
2814 				cipher_done = 1;
2815 				dcfgs[dcfg_ix].last = 1;
2816 			} else {
2817 				dcfgs[dcfg_ix].last = 0;
2818 			}
2819 			dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2820 			descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2821 			++dcfg_ix;
2822 		}
2823 		if (oper.do_digest && (oper.digest_start < next_ix) && (prev_ix < (oper.digest_start + oper.digest_len))) {
2824 			digest_active = 1;
2825 			dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_DIGEST_TID;
2826 			dcfgs[dcfg_ix].src = cryptocop_source_dma;
2827 			if (next_ix == (oper.digest_start + oper.digest_len)){
2828 				assert(!digest_done);
2829 				digest_done = 1;
2830 				dcfgs[dcfg_ix].last = 1;
2831 			} else {
2832 				dcfgs[dcfg_ix].last = 0;
2833 			}
2834 			dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2835 			descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2836 			++dcfg_ix;
2837 		}
2838 		if (oper.do_csum && (oper.csum_start < next_ix) && (prev_ix < (oper.csum_start + oper.csum_len))){
2839 			csum_active = 1;
2840 			dcfgs[dcfg_ix].tid = CRYPTOCOP_IOCTL_CSUM_TID;
2841 			dcfgs[dcfg_ix].src = cryptocop_source_dma;
2842 			if (next_ix == (oper.csum_start + oper.csum_len)){
2843 				csum_done = 1;
2844 				dcfgs[dcfg_ix].last = 1;
2845 			} else {
2846 				dcfgs[dcfg_ix].last = 0;
2847 			}
2848 			dcfgs[dcfg_ix].next = descs[desc_ix].cfg;
2849 			descs[desc_ix].cfg = &dcfgs[dcfg_ix];
2850 			++dcfg_ix;
2851 		}
2852 		if (!descs[desc_ix].cfg){
2853 			DEBUG_API(printk("cryptocop_ioctl_process: data segment %d (%d to %d) had no active transforms\n", desc_ix, prev_ix, next_ix));
2854 			err = -EINVAL;
2855 			goto error_cleanup;
2856 		}
2857 		descs[desc_ix].next = &(descs[desc_ix]) + 1;
2858 		++desc_ix;
2859 		prev_ix = next_ix;
2860 		next_ix = next_cfg_change_ix(&oper, prev_ix);
2861 	}
2862 	if (desc_ix > 0){
2863 		descs[desc_ix-1].next = NULL;
2864 	} else {
2865 		descs[0].next = NULL;
2866 	}
2867 	if (oper.do_digest) {
2868 		DEBUG(printk("cryptocop_ioctl_process: mapping %d byte digest output to iovec %d\n", digest_length, iovix));
2869 		/* Add outdata iovec, length == <length of type of digest> */
2870 		cop->tfrm_op.outdata[iovix].iov_base = digest_result;
2871 		cop->tfrm_op.outdata[iovix].iov_len = digest_length;
2872 		++iovix;
2873 	}
2874 	if (oper.do_csum) {
2875 		/* Add outdata iovec, length == 2, the length of csum. */
2876 		DEBUG(printk("cryptocop_ioctl_process: mapping 2 byte csum output to iovec %d\n", iovix));
2877 		/* Add outdata iovec, length == <length of type of digest> */
2878 		cop->tfrm_op.outdata[iovix].iov_base = csum_result;
2879 		cop->tfrm_op.outdata[iovix].iov_len = 2;
2880 		++iovix;
2881 	}
2882 	if (oper.do_cipher) {
2883 		if (!map_pages_to_iovec(cop->tfrm_op.outdata, iovlen, &iovix, outpages, nooutpages, &pageix, &pageoffset, oper.cipher_outlen)){
2884 			DEBUG_API(printk("cryptocop_ioctl_process: failed to map pages to iovec.\n"));
2885 			err = -ENOSYS; /* This should be impossible barring bugs. */
2886 			goto error_cleanup;
2887 		}
2888 	}
2889 	DEBUG(printk("cryptocop_ioctl_process: setting cop->tfrm_op.outcount %d\n", iovix));
2890 	cop->tfrm_op.outcount = iovix;
2891 	assert(iovix <= (nooutpages + 6));
2892 
2893 	cop->sid = oper.ses_id;
2894 	cop->tfrm_op.desc = &descs[0];
2895 
2896 	DEBUG(printk("cryptocop_ioctl_process: inserting job, cb_data=0x%p\n", cop->cb_data));
2897 
2898 	if ((err = cryptocop_job_queue_insert_user_job(cop)) != 0) {
2899 		DEBUG_API(printk("cryptocop_ioctl_process: insert job %d\n", err));
2900 		err = -EINVAL;
2901 		goto error_cleanup;
2902 	}
2903 
2904 	DEBUG(printk("cryptocop_ioctl_process: begin wait for result\n"));
2905 
2906 	wait_event(cryptocop_ioc_process_wq, (jc->processed != 0));
2907 	DEBUG(printk("cryptocop_ioctl_process: end wait for result\n"));
2908         if (!jc->processed){
2909 		printk(KERN_WARNING "cryptocop_ioctl_process: job not processed at completion\n");
2910 		err = -EIO;
2911 		goto error_cleanup;
2912 	}
2913 
2914 	/* Job process done.  Cipher output should already be correct in job so no post processing of outdata. */
2915 	DEBUG(printk("cryptocop_ioctl_process: operation_status = %d\n", cop->operation_status));
2916 	if (cop->operation_status == 0){
2917 		if (oper.do_digest){
2918 			DEBUG(printk("cryptocop_ioctl_process: copy %d bytes digest to user\n", digest_length));
2919 			err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, digest), digest_result, digest_length);
2920 			if (0 != err){
2921 				DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, digest length %d, err %d\n", digest_length, err));
2922 				err = -EFAULT;
2923 				goto error_cleanup;
2924 			}
2925 		}
2926 		if (oper.do_csum){
2927 			DEBUG(printk("cryptocop_ioctl_process: copy 2 bytes checksum to user\n"));
2928 			err = copy_to_user((unsigned char*)crp_oper + offsetof(struct strcop_crypto_op, csum), csum_result, 2);
2929 			if (0 != err){
2930 				DEBUG_API(printk("cryptocop_ioctl_process: copy_to_user, csum, err %d\n", err));
2931 				err = -EFAULT;
2932 				goto error_cleanup;
2933 			}
2934 		}
2935 		err = 0;
2936 	} else {
2937 		DEBUG(printk("cryptocop_ioctl_process: returning err = operation_status = %d\n", cop->operation_status));
2938 		err = cop->operation_status;
2939 	}
2940 
2941  error_cleanup:
2942 	/* Release page caches. */
2943 	for (i = 0; i < noinpages; i++){
2944 		put_page(inpages[i]);
2945 	}
2946 	for (i = 0; i < nooutpages; i++){
2947 		int spdl_err;
2948 		/* Mark output pages dirty. */
2949 		spdl_err = set_page_dirty_lock(outpages[i]);
2950 		DEBUG(if (spdl_err < 0)printk("cryptocop_ioctl_process: set_page_dirty_lock returned %d\n", spdl_err));
2951 	}
2952 	for (i = 0; i < nooutpages; i++){
2953 		put_page(outpages[i]);
2954 	}
2955 
2956 	kfree(digest_result);
2957 	kfree(inpages);
2958 	kfree(outpages);
2959 	if (cop){
2960 		kfree(cop->tfrm_op.indata);
2961 		kfree(cop->tfrm_op.outdata);
2962 		kfree(cop);
2963 	}
2964 	kfree(jc);
2965 
2966 	DEBUG(print_lock_status());
2967 
2968 	return err;
2969 }
2970 
2971 
cryptocop_ioctl_create_session(struct inode * inode,struct file * filp,unsigned int cmd,unsigned long arg)2972 static int cryptocop_ioctl_create_session(struct inode *inode, struct file *filp, unsigned int cmd, unsigned long arg)
2973 {
2974 	cryptocop_session_id             sid;
2975 	int                              err;
2976 	struct cryptocop_private         *dev;
2977 	struct strcop_session_op         *sess_op = (struct strcop_session_op *)arg;
2978 	struct strcop_session_op         sop;
2979 	struct cryptocop_transform_init  *tis = NULL;
2980 	struct cryptocop_transform_init  ti_cipher = {0};
2981 	struct cryptocop_transform_init  ti_digest = {0};
2982 	struct cryptocop_transform_init  ti_csum = {0};
2983 
2984 	if (!access_ok(VERIFY_WRITE, sess_op, sizeof(struct strcop_session_op)))
2985 		return -EFAULT;
2986 	err = copy_from_user(&sop, sess_op, sizeof(struct strcop_session_op));
2987 	if (err) return -EFAULT;
2988 	if (sop.cipher != cryptocop_cipher_none) {
2989 		if (!access_ok(VERIFY_READ, sop.key, sop.keylen)) return -EFAULT;
2990 	}
2991 	DEBUG(printk("cryptocop_ioctl_create_session, sess_op:\n"));
2992 
2993 	DEBUG(printk("\tcipher:%d\n"
2994 		     "\tcipher_mode:%d\n"
2995 		     "\tdigest:%d\n"
2996 		     "\tcsum:%d\n",
2997 		     (int)sop.cipher,
2998 		     (int)sop.cmode,
2999 		     (int)sop.digest,
3000 		     (int)sop.csum));
3001 
3002 	if (sop.cipher != cryptocop_cipher_none){
3003 		/* Init the cipher. */
3004 		switch (sop.cipher){
3005 		case cryptocop_cipher_des:
3006 			ti_cipher.alg = cryptocop_alg_des;
3007 			break;
3008 		case cryptocop_cipher_3des:
3009 			ti_cipher.alg = cryptocop_alg_3des;
3010 			break;
3011 		case cryptocop_cipher_aes:
3012 			ti_cipher.alg = cryptocop_alg_aes;
3013 			break;
3014 		default:
3015 			DEBUG_API(printk("create session, bad cipher algorithm %d\n", sop.cipher));
3016 			return -EINVAL;
3017 		};
3018 		DEBUG(printk("setting cipher transform %d\n", ti_cipher.alg));
3019 		copy_from_user(ti_cipher.key, sop.key, sop.keylen/8);
3020 		ti_cipher.keylen = sop.keylen;
3021 		switch (sop.cmode){
3022 		case cryptocop_cipher_mode_cbc:
3023 		case cryptocop_cipher_mode_ecb:
3024 			ti_cipher.cipher_mode = sop.cmode;
3025 			break;
3026 		default:
3027 			DEBUG_API(printk("create session, bad cipher mode %d\n", sop.cmode));
3028 			return -EINVAL;
3029 		}
3030 		DEBUG(printk("cryptocop_ioctl_create_session: setting CBC mode %d\n", ti_cipher.cipher_mode));
3031 		switch (sop.des3_mode){
3032 		case cryptocop_3des_eee:
3033 		case cryptocop_3des_eed:
3034 		case cryptocop_3des_ede:
3035 		case cryptocop_3des_edd:
3036 		case cryptocop_3des_dee:
3037 		case cryptocop_3des_ded:
3038 		case cryptocop_3des_dde:
3039 		case cryptocop_3des_ddd:
3040 			ti_cipher.tdes_mode = sop.des3_mode;
3041 			break;
3042 		default:
3043 			DEBUG_API(printk("create session, bad 3DES mode %d\n", sop.des3_mode));
3044 			return -EINVAL;
3045 		}
3046 		ti_cipher.tid = CRYPTOCOP_IOCTL_CIPHER_TID;
3047 		ti_cipher.next = tis;
3048 		tis = &ti_cipher;
3049 	} /* if (sop.cipher != cryptocop_cipher_none) */
3050 	if (sop.digest != cryptocop_digest_none){
3051 		DEBUG(printk("setting digest transform\n"));
3052 		switch (sop.digest){
3053 		case cryptocop_digest_md5:
3054 			ti_digest.alg = cryptocop_alg_md5;
3055 			break;
3056 		case cryptocop_digest_sha1:
3057 			ti_digest.alg = cryptocop_alg_sha1;
3058 			break;
3059 		default:
3060 			DEBUG_API(printk("create session, bad digest algorithm %d\n", sop.digest));
3061 			return -EINVAL;
3062 		}
3063 		ti_digest.tid = CRYPTOCOP_IOCTL_DIGEST_TID;
3064 		ti_digest.next = tis;
3065 		tis = &ti_digest;
3066 	} /* if (sop.digest != cryptocop_digest_none) */
3067 	if (sop.csum != cryptocop_csum_none){
3068 		DEBUG(printk("setting csum transform\n"));
3069 		switch (sop.csum){
3070 		case cryptocop_csum_le:
3071 		case cryptocop_csum_be:
3072 			ti_csum.csum_mode = sop.csum;
3073 			break;
3074 		default:
3075 			DEBUG_API(printk("create session, bad checksum algorithm %d\n", sop.csum));
3076 			return -EINVAL;
3077 		}
3078 		ti_csum.alg = cryptocop_alg_csum;
3079 		ti_csum.tid = CRYPTOCOP_IOCTL_CSUM_TID;
3080 		ti_csum.next = tis;
3081 		tis = &ti_csum;
3082 	} /* (sop.csum != cryptocop_csum_none) */
3083 	dev = kmalloc(sizeof(struct cryptocop_private), GFP_KERNEL);
3084 	if (!dev){
3085 		DEBUG_API(printk("create session, alloc dev\n"));
3086 		return -ENOMEM;
3087 	}
3088 
3089 	err = cryptocop_new_session(&sid, tis, GFP_KERNEL);
3090 	DEBUG({ if (err) printk("create session, cryptocop_new_session %d\n", err);});
3091 
3092 	if (err) {
3093 		kfree(dev);
3094 		return err;
3095 	}
3096 	sess_op->ses_id = sid;
3097 	dev->sid = sid;
3098 	dev->next = filp->private_data;
3099 	filp->private_data = dev;
3100 
3101 	return 0;
3102 }
3103 
cryptocop_ioctl_unlocked(struct inode * inode,struct file * filp,unsigned int cmd,unsigned long arg)3104 static long cryptocop_ioctl_unlocked(struct inode *inode,
3105 	struct file *filp, unsigned int cmd, unsigned long arg)
3106 {
3107 	int err = 0;
3108 	if (_IOC_TYPE(cmd) != ETRAXCRYPTOCOP_IOCTYPE) {
3109 		DEBUG_API(printk("cryptocop_ioctl: wrong type\n"));
3110 		return -ENOTTY;
3111 	}
3112 	if (_IOC_NR(cmd) > CRYPTOCOP_IO_MAXNR){
3113 		return -ENOTTY;
3114 	}
3115 	/* Access check of the argument.  Some commands, e.g. create session and process op,
3116 	   needs additional checks.  Those are handled in the command handling functions. */
3117 	if (_IOC_DIR(cmd) & _IOC_READ)
3118 		err = !access_ok(VERIFY_WRITE, (void *)arg, _IOC_SIZE(cmd));
3119 	else if (_IOC_DIR(cmd) & _IOC_WRITE)
3120 		err = !access_ok(VERIFY_READ, (void *)arg, _IOC_SIZE(cmd));
3121 	if (err) return -EFAULT;
3122 
3123 	switch (cmd) {
3124 	case CRYPTOCOP_IO_CREATE_SESSION:
3125 		return cryptocop_ioctl_create_session(inode, filp, cmd, arg);
3126 	case CRYPTOCOP_IO_CLOSE_SESSION:
3127 		return cryptocop_ioctl_close_session(inode, filp, cmd, arg);
3128 	case CRYPTOCOP_IO_PROCESS_OP:
3129 		return cryptocop_ioctl_process(inode, filp, cmd, arg);
3130 	default:
3131 		DEBUG_API(printk("cryptocop_ioctl: unknown command\n"));
3132 		return -ENOTTY;
3133 	}
3134 	return 0;
3135 }
3136 
3137 static long
cryptocop_ioctl(struct file * filp,unsigned int cmd,unsigned long arg)3138 cryptocop_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
3139 {
3140        struct inode *inode = file->f_path.dentry->d_inode;
3141        long ret;
3142 
3143        mutex_lock(&cryptocop_mutex);
3144        ret = cryptocop_ioctl_unlocked(inode, filp, cmd, arg);
3145        mutex_unlock(&cryptocop_mutex);
3146 
3147        return ret;
3148 }
3149 
3150 
3151 #ifdef LDEBUG
print_dma_descriptors(struct cryptocop_int_operation * iop)3152 static void print_dma_descriptors(struct cryptocop_int_operation *iop)
3153 {
3154 	struct cryptocop_dma_desc *cdesc_out = iop->cdesc_out;
3155 	struct cryptocop_dma_desc *cdesc_in = iop->cdesc_in;
3156 	int                       i;
3157 
3158 	printk("print_dma_descriptors start\n");
3159 
3160 	printk("iop:\n");
3161 	printk("\tsid: 0x%lld\n", iop->sid);
3162 
3163 	printk("\tcdesc_out: 0x%p\n", iop->cdesc_out);
3164 	printk("\tcdesc_in: 0x%p\n", iop->cdesc_in);
3165 	printk("\tddesc_out: 0x%p\n", iop->ddesc_out);
3166 	printk("\tddesc_in: 0x%p\n", iop->ddesc_in);
3167 
3168 	printk("\niop->ctx_out: 0x%p phys: 0x%p\n", &iop->ctx_out, (char*)virt_to_phys(&iop->ctx_out));
3169 	printk("\tnext: 0x%p\n"
3170 	       "\tsaved_data: 0x%p\n"
3171 	       "\tsaved_data_buf: 0x%p\n",
3172 	       iop->ctx_out.next,
3173 	       iop->ctx_out.saved_data,
3174 	       iop->ctx_out.saved_data_buf);
3175 
3176 	printk("\niop->ctx_in: 0x%p phys: 0x%p\n", &iop->ctx_in, (char*)virt_to_phys(&iop->ctx_in));
3177 	printk("\tnext: 0x%p\n"
3178 	       "\tsaved_data: 0x%p\n"
3179 	       "\tsaved_data_buf: 0x%p\n",
3180 	       iop->ctx_in.next,
3181 	       iop->ctx_in.saved_data,
3182 	       iop->ctx_in.saved_data_buf);
3183 
3184 	i = 0;
3185 	while (cdesc_out) {
3186 		dma_descr_data *td;
3187 		printk("cdesc_out %d, desc=0x%p\n", i, cdesc_out->dma_descr);
3188 		printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_out->dma_descr));
3189 		td = cdesc_out->dma_descr;
3190 		printk("\n\tbuf: 0x%p\n"
3191 		       "\tafter: 0x%p\n"
3192 		       "\tmd: 0x%04x\n"
3193 		       "\tnext: 0x%p\n",
3194 		       td->buf,
3195 		       td->after,
3196 		       td->md,
3197 		       td->next);
3198 		printk("flags:\n"
3199 		       "\twait:\t%d\n"
3200 		       "\teol:\t%d\n"
3201 		       "\touteop:\t%d\n"
3202 		       "\tineop:\t%d\n"
3203 		       "\tintr:\t%d\n",
3204 		       td->wait,
3205 		       td->eol,
3206 		       td->out_eop,
3207 		       td->in_eop,
3208 		       td->intr);
3209 		cdesc_out = cdesc_out->next;
3210 		i++;
3211 	}
3212 	i = 0;
3213 	while (cdesc_in) {
3214 		dma_descr_data *td;
3215 		printk("cdesc_in %d, desc=0x%p\n", i, cdesc_in->dma_descr);
3216 		printk("\n\tvirt_to_phys(desc): 0x%p\n", (char*)virt_to_phys(cdesc_in->dma_descr));
3217 		td = cdesc_in->dma_descr;
3218 		printk("\n\tbuf: 0x%p\n"
3219 		       "\tafter: 0x%p\n"
3220 		       "\tmd: 0x%04x\n"
3221 		       "\tnext: 0x%p\n",
3222 		       td->buf,
3223 		       td->after,
3224 		       td->md,
3225 		       td->next);
3226 		printk("flags:\n"
3227 		       "\twait:\t%d\n"
3228 		       "\teol:\t%d\n"
3229 		       "\touteop:\t%d\n"
3230 		       "\tineop:\t%d\n"
3231 		       "\tintr:\t%d\n",
3232 		       td->wait,
3233 		       td->eol,
3234 		       td->out_eop,
3235 		       td->in_eop,
3236 		       td->intr);
3237 		cdesc_in = cdesc_in->next;
3238 		i++;
3239 	}
3240 
3241 	printk("print_dma_descriptors end\n");
3242 }
3243 
3244 
print_strcop_crypto_op(struct strcop_crypto_op * cop)3245 static void print_strcop_crypto_op(struct strcop_crypto_op *cop)
3246 {
3247 	printk("print_strcop_crypto_op, 0x%p\n", cop);
3248 
3249 	/* Indata. */
3250 	printk("indata=0x%p\n"
3251 	       "inlen=%d\n"
3252 	       "do_cipher=%d\n"
3253 	       "decrypt=%d\n"
3254 	       "cipher_explicit=%d\n"
3255 	       "cipher_start=%d\n"
3256 	       "cipher_len=%d\n"
3257 	       "outdata=0x%p\n"
3258 	       "outlen=%d\n",
3259 	       cop->indata,
3260 	       cop->inlen,
3261 	       cop->do_cipher,
3262 	       cop->decrypt,
3263 	       cop->cipher_explicit,
3264 	       cop->cipher_start,
3265 	       cop->cipher_len,
3266 	       cop->cipher_outdata,
3267 	       cop->cipher_outlen);
3268 
3269 	printk("do_digest=%d\n"
3270 	       "digest_start=%d\n"
3271 	       "digest_len=%d\n",
3272 	       cop->do_digest,
3273 	       cop->digest_start,
3274 	       cop->digest_len);
3275 
3276 	printk("do_csum=%d\n"
3277 	       "csum_start=%d\n"
3278 	       "csum_len=%d\n",
3279 	       cop->do_csum,
3280 	       cop->csum_start,
3281 	       cop->csum_len);
3282 }
3283 
print_cryptocop_operation(struct cryptocop_operation * cop)3284 static void print_cryptocop_operation(struct cryptocop_operation *cop)
3285 {
3286 	struct cryptocop_desc      *d;
3287 	struct cryptocop_tfrm_cfg  *tc;
3288 	struct cryptocop_desc_cfg  *dc;
3289 	int                        i;
3290 
3291 	printk("print_cryptocop_operation, cop=0x%p\n\n", cop);
3292 	printk("sid: %lld\n", cop->sid);
3293 	printk("operation_status=%d\n"
3294 	       "use_dmalists=%d\n"
3295 	       "in_interrupt=%d\n"
3296 	       "fast_callback=%d\n",
3297 	       cop->operation_status,
3298 	       cop->use_dmalists,
3299 	       cop->in_interrupt,
3300 	       cop->fast_callback);
3301 
3302 	if (cop->use_dmalists){
3303 		print_user_dma_lists(&cop->list_op);
3304 	} else {
3305 		printk("cop->tfrm_op\n"
3306 		       "tfrm_cfg=0x%p\n"
3307 		       "desc=0x%p\n"
3308 		       "indata=0x%p\n"
3309 		       "incount=%d\n"
3310 		       "inlen=%d\n"
3311 		       "outdata=0x%p\n"
3312 		       "outcount=%d\n"
3313 		       "outlen=%d\n\n",
3314 		       cop->tfrm_op.tfrm_cfg,
3315 		       cop->tfrm_op.desc,
3316 		       cop->tfrm_op.indata,
3317 		       cop->tfrm_op.incount,
3318 		       cop->tfrm_op.inlen,
3319 		       cop->tfrm_op.outdata,
3320 		       cop->tfrm_op.outcount,
3321 		       cop->tfrm_op.outlen);
3322 
3323 		tc = cop->tfrm_op.tfrm_cfg;
3324 		while (tc){
3325 			printk("tfrm_cfg, 0x%p\n"
3326 			       "tid=%d\n"
3327 			       "flags=%d\n"
3328 			       "inject_ix=%d\n"
3329 			       "next=0x%p\n",
3330 			       tc,
3331 			       tc->tid,
3332 			       tc->flags,
3333 			       tc->inject_ix,
3334 			       tc->next);
3335 			tc = tc->next;
3336 		}
3337 		d = cop->tfrm_op.desc;
3338 		while (d){
3339 			printk("\n======================desc, 0x%p\n"
3340 			       "length=%d\n"
3341 			       "cfg=0x%p\n"
3342 			       "next=0x%p\n",
3343 			       d,
3344 			       d->length,
3345 			       d->cfg,
3346 			       d->next);
3347 			dc = d->cfg;
3348 			while (dc){
3349 				printk("=========desc_cfg, 0x%p\n"
3350 				       "tid=%d\n"
3351 				       "src=%d\n"
3352 				       "last=%d\n"
3353 				       "next=0x%p\n",
3354 				       dc,
3355 				       dc->tid,
3356 				       dc->src,
3357 				       dc->last,
3358 				       dc->next);
3359 				dc = dc->next;
3360 			}
3361 			d = d->next;
3362 		}
3363 		printk("\n====iniov\n");
3364 		for (i = 0; i < cop->tfrm_op.incount; i++){
3365 			printk("indata[%d]\n"
3366 			       "base=0x%p\n"
3367 			       "len=%d\n",
3368 			       i,
3369 			       cop->tfrm_op.indata[i].iov_base,
3370 			       cop->tfrm_op.indata[i].iov_len);
3371 		}
3372 		printk("\n====outiov\n");
3373 		for (i = 0; i < cop->tfrm_op.outcount; i++){
3374 			printk("outdata[%d]\n"
3375 			       "base=0x%p\n"
3376 			       "len=%d\n",
3377 			       i,
3378 			       cop->tfrm_op.outdata[i].iov_base,
3379 			       cop->tfrm_op.outdata[i].iov_len);
3380 		}
3381 	}
3382 	printk("------------end print_cryptocop_operation\n");
3383 }
3384 
3385 
print_user_dma_lists(struct cryptocop_dma_list_operation * dma_op)3386 static void print_user_dma_lists(struct cryptocop_dma_list_operation *dma_op)
3387 {
3388 	dma_descr_data *dd;
3389 	int i;
3390 
3391 	printk("print_user_dma_lists, dma_op=0x%p\n", dma_op);
3392 
3393 	printk("out_data_buf = 0x%p, phys_to_virt(out_data_buf) = 0x%p\n", dma_op->out_data_buf, phys_to_virt((unsigned long int)dma_op->out_data_buf));
3394 	printk("in_data_buf = 0x%p, phys_to_virt(in_data_buf) = 0x%p\n", dma_op->in_data_buf, phys_to_virt((unsigned long int)dma_op->in_data_buf));
3395 
3396 	printk("##############outlist\n");
3397 	dd = phys_to_virt((unsigned long int)dma_op->outlist);
3398 	i = 0;
3399 	while (dd != NULL) {
3400 		printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3401 		printk("\n\tbuf: 0x%p\n"
3402 		       "\tafter: 0x%p\n"
3403 		       "\tmd: 0x%04x\n"
3404 		       "\tnext: 0x%p\n",
3405 		       dd->buf,
3406 		       dd->after,
3407 		       dd->md,
3408 		       dd->next);
3409 		printk("flags:\n"
3410 		       "\twait:\t%d\n"
3411 		       "\teol:\t%d\n"
3412 		       "\touteop:\t%d\n"
3413 		       "\tineop:\t%d\n"
3414 		       "\tintr:\t%d\n",
3415 		       dd->wait,
3416 		       dd->eol,
3417 		       dd->out_eop,
3418 		       dd->in_eop,
3419 		       dd->intr);
3420 		if (dd->eol)
3421 			dd = NULL;
3422 		else
3423 			dd = phys_to_virt((unsigned long int)dd->next);
3424 		++i;
3425 	}
3426 
3427 	printk("##############inlist\n");
3428 	dd = phys_to_virt((unsigned long int)dma_op->inlist);
3429 	i = 0;
3430 	while (dd != NULL) {
3431 		printk("#%d phys_to_virt(desc) 0x%p\n", i, dd);
3432 		printk("\n\tbuf: 0x%p\n"
3433 		       "\tafter: 0x%p\n"
3434 		       "\tmd: 0x%04x\n"
3435 		       "\tnext: 0x%p\n",
3436 		       dd->buf,
3437 		       dd->after,
3438 		       dd->md,
3439 		       dd->next);
3440 		printk("flags:\n"
3441 		       "\twait:\t%d\n"
3442 		       "\teol:\t%d\n"
3443 		       "\touteop:\t%d\n"
3444 		       "\tineop:\t%d\n"
3445 		       "\tintr:\t%d\n",
3446 		       dd->wait,
3447 		       dd->eol,
3448 		       dd->out_eop,
3449 		       dd->in_eop,
3450 		       dd->intr);
3451 		if (dd->eol)
3452 			dd = NULL;
3453 		else
3454 			dd = phys_to_virt((unsigned long int)dd->next);
3455 		++i;
3456 	}
3457 }
3458 
3459 
print_lock_status(void)3460 static void print_lock_status(void)
3461 {
3462 	printk("**********************print_lock_status\n");
3463 	printk("cryptocop_completed_jobs_lock %d\n", spin_is_locked(&cryptocop_completed_jobs_lock));
3464 	printk("cryptocop_job_queue_lock %d\n", spin_is_locked(&cryptocop_job_queue_lock));
3465 	printk("descr_pool_lock %d\n", spin_is_locked(&descr_pool_lock));
3466 	printk("cryptocop_sessions_lock %d\n", spin_is_locked(cryptocop_sessions_lock));
3467 	printk("running_job_lock %d\n", spin_is_locked(running_job_lock));
3468 	printk("cryptocop_process_lock %d\n", spin_is_locked(cryptocop_process_lock));
3469 }
3470 #endif /* LDEBUG */
3471 
3472 
3473 static const char cryptocop_name[] = "ETRAX FS stream co-processor";
3474 
init_stream_coprocessor(void)3475 static int init_stream_coprocessor(void)
3476 {
3477 	int err;
3478 	int i;
3479 	static int initialized = 0;
3480 
3481 	if (initialized)
3482 		return 0;
3483 
3484 	initialized = 1;
3485 
3486 	printk("ETRAX FS stream co-processor driver v0.01, (c) 2003 Axis Communications AB\n");
3487 
3488 	err = register_chrdev(CRYPTOCOP_MAJOR, cryptocop_name, &cryptocop_fops);
3489 	if (err < 0) {
3490 		printk(KERN_ERR "stream co-processor: could not get major number.\n");
3491 		return err;
3492 	}
3493 
3494 	err = init_cryptocop();
3495 	if (err) {
3496 		(void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3497 		return err;
3498 	}
3499 	err = cryptocop_job_queue_init();
3500 	if (err) {
3501 		release_cryptocop();
3502 		(void)unregister_chrdev(CRYPTOCOP_MAJOR, cryptocop_name);
3503 		return err;
3504 	}
3505 	/* Init the descriptor pool. */
3506 	for (i = 0; i < CRYPTOCOP_DESCRIPTOR_POOL_SIZE - 1; i++) {
3507 		descr_pool[i].from_pool = 1;
3508 		descr_pool[i].next = &descr_pool[i + 1];
3509 	}
3510 	descr_pool[i].from_pool = 1;
3511 	descr_pool[i].next = NULL;
3512 	descr_pool_free_list = &descr_pool[0];
3513 	descr_pool_no_free = CRYPTOCOP_DESCRIPTOR_POOL_SIZE;
3514 
3515 	spin_lock_init(&cryptocop_completed_jobs_lock);
3516 	spin_lock_init(&cryptocop_job_queue_lock);
3517 	spin_lock_init(&descr_pool_lock);
3518 	spin_lock_init(&cryptocop_sessions_lock);
3519 	spin_lock_init(&running_job_lock);
3520 	spin_lock_init(&cryptocop_process_lock);
3521 
3522 	cryptocop_sessions = NULL;
3523 	next_sid = 1;
3524 
3525 	cryptocop_running_job = NULL;
3526 
3527 	printk("stream co-processor: init done.\n");
3528 	return 0;
3529 }
3530 
exit_stream_coprocessor(void)3531 static void __exit exit_stream_coprocessor(void)
3532 {
3533 	release_cryptocop();
3534 	cryptocop_job_queue_close();
3535 }
3536 
3537 module_init(init_stream_coprocessor);
3538 module_exit(exit_stream_coprocessor);
3539 
3540