1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2021 Intel Corporation. All rights rsvd. */
3
4 #include <linux/init.h>
5 #include <linux/kernel.h>
6 #include <linux/module.h>
7 #include <linux/pci.h>
8 #include <linux/device.h>
9 #include <linux/iommu.h>
10 #include <uapi/linux/idxd.h>
11 #include <linux/highmem.h>
12 #include <linux/sched/smt.h>
13 #include <crypto/internal/acompress.h>
14
15 #include "idxd.h"
16 #include "iaa_crypto.h"
17 #include "iaa_crypto_stats.h"
18
19 #ifdef pr_fmt
20 #undef pr_fmt
21 #endif
22
23 #define pr_fmt(fmt) "idxd: " IDXD_SUBDRIVER_NAME ": " fmt
24
25 #define IAA_ALG_PRIORITY 300
26
27 /* number of iaa instances probed */
28 static unsigned int nr_iaa;
29 static unsigned int nr_cpus;
30 static unsigned int nr_nodes;
31 static unsigned int nr_cpus_per_node;
32
33 /* Number of physical cpus sharing each iaa instance */
34 static unsigned int cpus_per_iaa;
35
36 /* Per-cpu lookup table for balanced wqs */
37 static struct wq_table_entry __percpu *wq_table;
38
wq_table_next_wq(int cpu)39 static struct idxd_wq *wq_table_next_wq(int cpu)
40 {
41 struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu);
42
43 if (++entry->cur_wq >= entry->n_wqs)
44 entry->cur_wq = 0;
45
46 if (!entry->wqs[entry->cur_wq])
47 return NULL;
48
49 pr_debug("%s: returning wq at idx %d (iaa wq %d.%d) from cpu %d\n", __func__,
50 entry->cur_wq, entry->wqs[entry->cur_wq]->idxd->id,
51 entry->wqs[entry->cur_wq]->id, cpu);
52
53 return entry->wqs[entry->cur_wq];
54 }
55
wq_table_add(int cpu,struct idxd_wq * wq)56 static void wq_table_add(int cpu, struct idxd_wq *wq)
57 {
58 struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu);
59
60 if (WARN_ON(entry->n_wqs == entry->max_wqs))
61 return;
62
63 entry->wqs[entry->n_wqs++] = wq;
64
65 pr_debug("%s: added iaa wq %d.%d to idx %d of cpu %d\n", __func__,
66 entry->wqs[entry->n_wqs - 1]->idxd->id,
67 entry->wqs[entry->n_wqs - 1]->id, entry->n_wqs - 1, cpu);
68 }
69
wq_table_free_entry(int cpu)70 static void wq_table_free_entry(int cpu)
71 {
72 struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu);
73
74 kfree(entry->wqs);
75 memset(entry, 0, sizeof(*entry));
76 }
77
wq_table_clear_entry(int cpu)78 static void wq_table_clear_entry(int cpu)
79 {
80 struct wq_table_entry *entry = per_cpu_ptr(wq_table, cpu);
81
82 entry->n_wqs = 0;
83 entry->cur_wq = 0;
84 memset(entry->wqs, 0, entry->max_wqs * sizeof(struct idxd_wq *));
85 }
86
87 LIST_HEAD(iaa_devices);
88 DEFINE_MUTEX(iaa_devices_lock);
89
90 /* If enabled, IAA hw crypto algos are registered, unavailable otherwise */
91 static bool iaa_crypto_enabled;
92 static bool iaa_crypto_registered;
93
94 /* Verify results of IAA compress or not */
95 static bool iaa_verify_compress = true;
96
verify_compress_show(struct device_driver * driver,char * buf)97 static ssize_t verify_compress_show(struct device_driver *driver, char *buf)
98 {
99 return sprintf(buf, "%d\n", iaa_verify_compress);
100 }
101
verify_compress_store(struct device_driver * driver,const char * buf,size_t count)102 static ssize_t verify_compress_store(struct device_driver *driver,
103 const char *buf, size_t count)
104 {
105 int ret = -EBUSY;
106
107 mutex_lock(&iaa_devices_lock);
108
109 if (iaa_crypto_enabled)
110 goto out;
111
112 ret = kstrtobool(buf, &iaa_verify_compress);
113 if (ret)
114 goto out;
115
116 ret = count;
117 out:
118 mutex_unlock(&iaa_devices_lock);
119
120 return ret;
121 }
122 static DRIVER_ATTR_RW(verify_compress);
123
124 /*
125 * The iaa crypto driver supports three 'sync' methods determining how
126 * compressions and decompressions are performed:
127 *
128 * - sync: the compression or decompression completes before
129 * returning. This is the mode used by the async crypto
130 * interface when the sync mode is set to 'sync' and by
131 * the sync crypto interface regardless of setting.
132 *
133 * - async: the compression or decompression is submitted and returns
134 * immediately. Completion interrupts are not used so
135 * the caller is responsible for polling the descriptor
136 * for completion. This mode is applicable to only the
137 * async crypto interface and is ignored for anything
138 * else.
139 *
140 * - async_irq: the compression or decompression is submitted and
141 * returns immediately. Completion interrupts are
142 * enabled so the caller can wait for the completion and
143 * yield to other threads. When the compression or
144 * decompression completes, the completion is signaled
145 * and the caller awakened. This mode is applicable to
146 * only the async crypto interface and is ignored for
147 * anything else.
148 *
149 * These modes can be set using the iaa_crypto sync_mode driver
150 * attribute.
151 */
152
153 /* Use async mode */
154 static bool async_mode;
155 /* Use interrupts */
156 static bool use_irq;
157
158 /**
159 * set_iaa_sync_mode - Set IAA sync mode
160 * @name: The name of the sync mode
161 *
162 * Make the IAA sync mode named @name the current sync mode used by
163 * compression/decompression.
164 */
165
set_iaa_sync_mode(const char * name)166 static int set_iaa_sync_mode(const char *name)
167 {
168 int ret = 0;
169
170 if (sysfs_streq(name, "sync")) {
171 async_mode = false;
172 use_irq = false;
173 } else if (sysfs_streq(name, "async")) {
174 async_mode = false;
175 use_irq = false;
176 } else if (sysfs_streq(name, "async_irq")) {
177 async_mode = true;
178 use_irq = true;
179 } else {
180 ret = -EINVAL;
181 }
182
183 return ret;
184 }
185
sync_mode_show(struct device_driver * driver,char * buf)186 static ssize_t sync_mode_show(struct device_driver *driver, char *buf)
187 {
188 int ret = 0;
189
190 if (!async_mode && !use_irq)
191 ret = sprintf(buf, "%s\n", "sync");
192 else if (async_mode && !use_irq)
193 ret = sprintf(buf, "%s\n", "async");
194 else if (async_mode && use_irq)
195 ret = sprintf(buf, "%s\n", "async_irq");
196
197 return ret;
198 }
199
sync_mode_store(struct device_driver * driver,const char * buf,size_t count)200 static ssize_t sync_mode_store(struct device_driver *driver,
201 const char *buf, size_t count)
202 {
203 int ret = -EBUSY;
204
205 mutex_lock(&iaa_devices_lock);
206
207 if (iaa_crypto_enabled)
208 goto out;
209
210 ret = set_iaa_sync_mode(buf);
211 if (ret == 0)
212 ret = count;
213 out:
214 mutex_unlock(&iaa_devices_lock);
215
216 return ret;
217 }
218 static DRIVER_ATTR_RW(sync_mode);
219
220 static struct iaa_compression_mode *iaa_compression_modes[IAA_COMP_MODES_MAX];
221
find_empty_iaa_compression_mode(void)222 static int find_empty_iaa_compression_mode(void)
223 {
224 int i = -EINVAL;
225
226 for (i = 0; i < IAA_COMP_MODES_MAX; i++) {
227 if (iaa_compression_modes[i])
228 continue;
229 break;
230 }
231
232 return i;
233 }
234
find_iaa_compression_mode(const char * name,int * idx)235 static struct iaa_compression_mode *find_iaa_compression_mode(const char *name, int *idx)
236 {
237 struct iaa_compression_mode *mode;
238 int i;
239
240 for (i = 0; i < IAA_COMP_MODES_MAX; i++) {
241 mode = iaa_compression_modes[i];
242 if (!mode)
243 continue;
244
245 if (!strcmp(mode->name, name)) {
246 *idx = i;
247 return iaa_compression_modes[i];
248 }
249 }
250
251 return NULL;
252 }
253
free_iaa_compression_mode(struct iaa_compression_mode * mode)254 static void free_iaa_compression_mode(struct iaa_compression_mode *mode)
255 {
256 kfree(mode->name);
257 kfree(mode->ll_table);
258 kfree(mode->d_table);
259
260 kfree(mode);
261 }
262
263 /*
264 * IAA Compression modes are defined by an ll_table and a d_table.
265 * These tables are typically generated and captured using statistics
266 * collected from running actual compress/decompress workloads.
267 *
268 * A module or other kernel code can add and remove compression modes
269 * with a given name using the exported @add_iaa_compression_mode()
270 * and @remove_iaa_compression_mode functions.
271 *
272 * When a new compression mode is added, the tables are saved in a
273 * global compression mode list. When IAA devices are added, a
274 * per-IAA device dma mapping is created for each IAA device, for each
275 * compression mode. These are the tables used to do the actual
276 * compression/deccompression and are unmapped if/when the devices are
277 * removed. Currently, compression modes must be added before any
278 * device is added, and removed after all devices have been removed.
279 */
280
281 /**
282 * remove_iaa_compression_mode - Remove an IAA compression mode
283 * @name: The name the compression mode will be known as
284 *
285 * Remove the IAA compression mode named @name.
286 */
remove_iaa_compression_mode(const char * name)287 void remove_iaa_compression_mode(const char *name)
288 {
289 struct iaa_compression_mode *mode;
290 int idx;
291
292 mutex_lock(&iaa_devices_lock);
293
294 if (!list_empty(&iaa_devices))
295 goto out;
296
297 mode = find_iaa_compression_mode(name, &idx);
298 if (mode) {
299 free_iaa_compression_mode(mode);
300 iaa_compression_modes[idx] = NULL;
301 }
302 out:
303 mutex_unlock(&iaa_devices_lock);
304 }
305 EXPORT_SYMBOL_GPL(remove_iaa_compression_mode);
306
307 /**
308 * add_iaa_compression_mode - Add an IAA compression mode
309 * @name: The name the compression mode will be known as
310 * @ll_table: The ll table
311 * @ll_table_size: The ll table size in bytes
312 * @d_table: The d table
313 * @d_table_size: The d table size in bytes
314 * @init: Optional callback function to init the compression mode data
315 * @free: Optional callback function to free the compression mode data
316 *
317 * Add a new IAA compression mode named @name.
318 *
319 * Returns 0 if successful, errcode otherwise.
320 */
add_iaa_compression_mode(const char * name,const u32 * ll_table,int ll_table_size,const u32 * d_table,int d_table_size,iaa_dev_comp_init_fn_t init,iaa_dev_comp_free_fn_t free)321 int add_iaa_compression_mode(const char *name,
322 const u32 *ll_table,
323 int ll_table_size,
324 const u32 *d_table,
325 int d_table_size,
326 iaa_dev_comp_init_fn_t init,
327 iaa_dev_comp_free_fn_t free)
328 {
329 struct iaa_compression_mode *mode;
330 int idx, ret = -ENOMEM;
331
332 mutex_lock(&iaa_devices_lock);
333
334 if (!list_empty(&iaa_devices)) {
335 ret = -EBUSY;
336 goto out;
337 }
338
339 mode = kzalloc(sizeof(*mode), GFP_KERNEL);
340 if (!mode)
341 goto out;
342
343 mode->name = kstrdup(name, GFP_KERNEL);
344 if (!mode->name)
345 goto free;
346
347 if (ll_table) {
348 mode->ll_table = kmemdup(ll_table, ll_table_size, GFP_KERNEL);
349 if (!mode->ll_table)
350 goto free;
351 mode->ll_table_size = ll_table_size;
352 }
353
354 if (d_table) {
355 mode->d_table = kmemdup(d_table, d_table_size, GFP_KERNEL);
356 if (!mode->d_table)
357 goto free;
358 mode->d_table_size = d_table_size;
359 }
360
361 mode->init = init;
362 mode->free = free;
363
364 idx = find_empty_iaa_compression_mode();
365 if (idx < 0)
366 goto free;
367
368 pr_debug("IAA compression mode %s added at idx %d\n",
369 mode->name, idx);
370
371 iaa_compression_modes[idx] = mode;
372
373 ret = 0;
374 out:
375 mutex_unlock(&iaa_devices_lock);
376
377 return ret;
378 free:
379 free_iaa_compression_mode(mode);
380 goto out;
381 }
382 EXPORT_SYMBOL_GPL(add_iaa_compression_mode);
383
384 static struct iaa_device_compression_mode *
get_iaa_device_compression_mode(struct iaa_device * iaa_device,int idx)385 get_iaa_device_compression_mode(struct iaa_device *iaa_device, int idx)
386 {
387 return iaa_device->compression_modes[idx];
388 }
389
free_device_compression_mode(struct iaa_device * iaa_device,struct iaa_device_compression_mode * device_mode)390 static void free_device_compression_mode(struct iaa_device *iaa_device,
391 struct iaa_device_compression_mode *device_mode)
392 {
393 size_t size = sizeof(struct aecs_comp_table_record) + IAA_AECS_ALIGN;
394 struct device *dev = &iaa_device->idxd->pdev->dev;
395
396 kfree(device_mode->name);
397
398 if (device_mode->aecs_comp_table)
399 dma_free_coherent(dev, size, device_mode->aecs_comp_table,
400 device_mode->aecs_comp_table_dma_addr);
401 kfree(device_mode);
402 }
403
404 #define IDXD_OP_FLAG_AECS_RW_TGLS 0x400000
405 #define IAX_AECS_DEFAULT_FLAG (IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR | IDXD_OP_FLAG_CC)
406 #define IAX_AECS_COMPRESS_FLAG (IAX_AECS_DEFAULT_FLAG | IDXD_OP_FLAG_RD_SRC2_AECS)
407 #define IAX_AECS_DECOMPRESS_FLAG (IAX_AECS_DEFAULT_FLAG | IDXD_OP_FLAG_RD_SRC2_AECS)
408 #define IAX_AECS_GEN_FLAG (IAX_AECS_DEFAULT_FLAG | \
409 IDXD_OP_FLAG_WR_SRC2_AECS_COMP | \
410 IDXD_OP_FLAG_AECS_RW_TGLS)
411
412 static int check_completion(struct device *dev,
413 struct iax_completion_record *comp,
414 bool compress,
415 bool only_once);
416
init_device_compression_mode(struct iaa_device * iaa_device,struct iaa_compression_mode * mode,int idx,struct idxd_wq * wq)417 static int init_device_compression_mode(struct iaa_device *iaa_device,
418 struct iaa_compression_mode *mode,
419 int idx, struct idxd_wq *wq)
420 {
421 size_t size = sizeof(struct aecs_comp_table_record) + IAA_AECS_ALIGN;
422 struct device *dev = &iaa_device->idxd->pdev->dev;
423 struct iaa_device_compression_mode *device_mode;
424 int ret = -ENOMEM;
425
426 device_mode = kzalloc(sizeof(*device_mode), GFP_KERNEL);
427 if (!device_mode)
428 return -ENOMEM;
429
430 device_mode->name = kstrdup(mode->name, GFP_KERNEL);
431 if (!device_mode->name)
432 goto free;
433
434 device_mode->aecs_comp_table = dma_alloc_coherent(dev, size,
435 &device_mode->aecs_comp_table_dma_addr, GFP_KERNEL);
436 if (!device_mode->aecs_comp_table)
437 goto free;
438
439 /* Add Huffman table to aecs */
440 memset(device_mode->aecs_comp_table, 0, sizeof(*device_mode->aecs_comp_table));
441 memcpy(device_mode->aecs_comp_table->ll_sym, mode->ll_table, mode->ll_table_size);
442 memcpy(device_mode->aecs_comp_table->d_sym, mode->d_table, mode->d_table_size);
443
444 if (mode->init) {
445 ret = mode->init(device_mode);
446 if (ret)
447 goto free;
448 }
449
450 /* mode index should match iaa_compression_modes idx */
451 iaa_device->compression_modes[idx] = device_mode;
452
453 pr_debug("IAA %s compression mode initialized for iaa device %d\n",
454 mode->name, iaa_device->idxd->id);
455
456 ret = 0;
457 out:
458 return ret;
459 free:
460 pr_debug("IAA %s compression mode initialization failed for iaa device %d\n",
461 mode->name, iaa_device->idxd->id);
462
463 free_device_compression_mode(iaa_device, device_mode);
464 goto out;
465 }
466
init_device_compression_modes(struct iaa_device * iaa_device,struct idxd_wq * wq)467 static int init_device_compression_modes(struct iaa_device *iaa_device,
468 struct idxd_wq *wq)
469 {
470 struct iaa_compression_mode *mode;
471 int i, ret = 0;
472
473 for (i = 0; i < IAA_COMP_MODES_MAX; i++) {
474 mode = iaa_compression_modes[i];
475 if (!mode)
476 continue;
477
478 ret = init_device_compression_mode(iaa_device, mode, i, wq);
479 if (ret)
480 break;
481 }
482
483 return ret;
484 }
485
remove_device_compression_modes(struct iaa_device * iaa_device)486 static void remove_device_compression_modes(struct iaa_device *iaa_device)
487 {
488 struct iaa_device_compression_mode *device_mode;
489 int i;
490
491 for (i = 0; i < IAA_COMP_MODES_MAX; i++) {
492 device_mode = iaa_device->compression_modes[i];
493 if (!device_mode)
494 continue;
495
496 if (iaa_compression_modes[i]->free)
497 iaa_compression_modes[i]->free(device_mode);
498 free_device_compression_mode(iaa_device, device_mode);
499 iaa_device->compression_modes[i] = NULL;
500 }
501 }
502
iaa_device_alloc(void)503 static struct iaa_device *iaa_device_alloc(void)
504 {
505 struct iaa_device *iaa_device;
506
507 iaa_device = kzalloc(sizeof(*iaa_device), GFP_KERNEL);
508 if (!iaa_device)
509 return NULL;
510
511 INIT_LIST_HEAD(&iaa_device->wqs);
512
513 return iaa_device;
514 }
515
iaa_has_wq(struct iaa_device * iaa_device,struct idxd_wq * wq)516 static bool iaa_has_wq(struct iaa_device *iaa_device, struct idxd_wq *wq)
517 {
518 struct iaa_wq *iaa_wq;
519
520 list_for_each_entry(iaa_wq, &iaa_device->wqs, list) {
521 if (iaa_wq->wq == wq)
522 return true;
523 }
524
525 return false;
526 }
527
add_iaa_device(struct idxd_device * idxd)528 static struct iaa_device *add_iaa_device(struct idxd_device *idxd)
529 {
530 struct iaa_device *iaa_device;
531
532 iaa_device = iaa_device_alloc();
533 if (!iaa_device)
534 return NULL;
535
536 iaa_device->idxd = idxd;
537
538 list_add_tail(&iaa_device->list, &iaa_devices);
539
540 nr_iaa++;
541
542 return iaa_device;
543 }
544
init_iaa_device(struct iaa_device * iaa_device,struct iaa_wq * iaa_wq)545 static int init_iaa_device(struct iaa_device *iaa_device, struct iaa_wq *iaa_wq)
546 {
547 int ret = 0;
548
549 ret = init_device_compression_modes(iaa_device, iaa_wq->wq);
550 if (ret)
551 return ret;
552
553 return ret;
554 }
555
del_iaa_device(struct iaa_device * iaa_device)556 static void del_iaa_device(struct iaa_device *iaa_device)
557 {
558 list_del(&iaa_device->list);
559
560 nr_iaa--;
561 }
562
add_iaa_wq(struct iaa_device * iaa_device,struct idxd_wq * wq,struct iaa_wq ** new_wq)563 static int add_iaa_wq(struct iaa_device *iaa_device, struct idxd_wq *wq,
564 struct iaa_wq **new_wq)
565 {
566 struct idxd_device *idxd = iaa_device->idxd;
567 struct pci_dev *pdev = idxd->pdev;
568 struct device *dev = &pdev->dev;
569 struct iaa_wq *iaa_wq;
570
571 iaa_wq = kzalloc(sizeof(*iaa_wq), GFP_KERNEL);
572 if (!iaa_wq)
573 return -ENOMEM;
574
575 iaa_wq->wq = wq;
576 iaa_wq->iaa_device = iaa_device;
577 idxd_wq_set_private(wq, iaa_wq);
578
579 list_add_tail(&iaa_wq->list, &iaa_device->wqs);
580
581 iaa_device->n_wq++;
582
583 if (new_wq)
584 *new_wq = iaa_wq;
585
586 dev_dbg(dev, "added wq %d to iaa device %d, n_wq %d\n",
587 wq->id, iaa_device->idxd->id, iaa_device->n_wq);
588
589 return 0;
590 }
591
del_iaa_wq(struct iaa_device * iaa_device,struct idxd_wq * wq)592 static void del_iaa_wq(struct iaa_device *iaa_device, struct idxd_wq *wq)
593 {
594 struct idxd_device *idxd = iaa_device->idxd;
595 struct pci_dev *pdev = idxd->pdev;
596 struct device *dev = &pdev->dev;
597 struct iaa_wq *iaa_wq;
598
599 list_for_each_entry(iaa_wq, &iaa_device->wqs, list) {
600 if (iaa_wq->wq == wq) {
601 list_del(&iaa_wq->list);
602 iaa_device->n_wq--;
603
604 dev_dbg(dev, "removed wq %d from iaa_device %d, n_wq %d, nr_iaa %d\n",
605 wq->id, iaa_device->idxd->id,
606 iaa_device->n_wq, nr_iaa);
607
608 if (iaa_device->n_wq == 0)
609 del_iaa_device(iaa_device);
610 break;
611 }
612 }
613 }
614
clear_wq_table(void)615 static void clear_wq_table(void)
616 {
617 int cpu;
618
619 for (cpu = 0; cpu < nr_cpus; cpu++)
620 wq_table_clear_entry(cpu);
621
622 pr_debug("cleared wq table\n");
623 }
624
free_iaa_device(struct iaa_device * iaa_device)625 static void free_iaa_device(struct iaa_device *iaa_device)
626 {
627 if (!iaa_device)
628 return;
629
630 remove_device_compression_modes(iaa_device);
631 kfree(iaa_device);
632 }
633
__free_iaa_wq(struct iaa_wq * iaa_wq)634 static void __free_iaa_wq(struct iaa_wq *iaa_wq)
635 {
636 struct iaa_device *iaa_device;
637
638 if (!iaa_wq)
639 return;
640
641 iaa_device = iaa_wq->iaa_device;
642 if (iaa_device->n_wq == 0)
643 free_iaa_device(iaa_wq->iaa_device);
644 }
645
free_iaa_wq(struct iaa_wq * iaa_wq)646 static void free_iaa_wq(struct iaa_wq *iaa_wq)
647 {
648 struct idxd_wq *wq;
649
650 __free_iaa_wq(iaa_wq);
651
652 wq = iaa_wq->wq;
653
654 kfree(iaa_wq);
655 idxd_wq_set_private(wq, NULL);
656 }
657
iaa_wq_get(struct idxd_wq * wq)658 static int iaa_wq_get(struct idxd_wq *wq)
659 {
660 struct idxd_device *idxd = wq->idxd;
661 struct iaa_wq *iaa_wq;
662 int ret = 0;
663
664 spin_lock(&idxd->dev_lock);
665 iaa_wq = idxd_wq_get_private(wq);
666 if (iaa_wq && !iaa_wq->remove) {
667 iaa_wq->ref++;
668 idxd_wq_get(wq);
669 } else {
670 ret = -ENODEV;
671 }
672 spin_unlock(&idxd->dev_lock);
673
674 return ret;
675 }
676
iaa_wq_put(struct idxd_wq * wq)677 static int iaa_wq_put(struct idxd_wq *wq)
678 {
679 struct idxd_device *idxd = wq->idxd;
680 struct iaa_wq *iaa_wq;
681 bool free = false;
682 int ret = 0;
683
684 spin_lock(&idxd->dev_lock);
685 iaa_wq = idxd_wq_get_private(wq);
686 if (iaa_wq) {
687 iaa_wq->ref--;
688 if (iaa_wq->ref == 0 && iaa_wq->remove) {
689 idxd_wq_set_private(wq, NULL);
690 free = true;
691 }
692 idxd_wq_put(wq);
693 } else {
694 ret = -ENODEV;
695 }
696 spin_unlock(&idxd->dev_lock);
697 if (free) {
698 __free_iaa_wq(iaa_wq);
699 kfree(iaa_wq);
700 }
701
702 return ret;
703 }
704
free_wq_table(void)705 static void free_wq_table(void)
706 {
707 int cpu;
708
709 for (cpu = 0; cpu < nr_cpus; cpu++)
710 wq_table_free_entry(cpu);
711
712 free_percpu(wq_table);
713
714 pr_debug("freed wq table\n");
715 }
716
alloc_wq_table(int max_wqs)717 static int alloc_wq_table(int max_wqs)
718 {
719 struct wq_table_entry *entry;
720 int cpu;
721
722 wq_table = alloc_percpu(struct wq_table_entry);
723 if (!wq_table)
724 return -ENOMEM;
725
726 for (cpu = 0; cpu < nr_cpus; cpu++) {
727 entry = per_cpu_ptr(wq_table, cpu);
728 entry->wqs = kcalloc(max_wqs, sizeof(struct wq *), GFP_KERNEL);
729 if (!entry->wqs) {
730 free_wq_table();
731 return -ENOMEM;
732 }
733
734 entry->max_wqs = max_wqs;
735 }
736
737 pr_debug("initialized wq table\n");
738
739 return 0;
740 }
741
save_iaa_wq(struct idxd_wq * wq)742 static int save_iaa_wq(struct idxd_wq *wq)
743 {
744 struct iaa_device *iaa_device, *found = NULL;
745 struct idxd_device *idxd;
746 struct pci_dev *pdev;
747 struct device *dev;
748 int ret = 0;
749
750 list_for_each_entry(iaa_device, &iaa_devices, list) {
751 if (iaa_device->idxd == wq->idxd) {
752 idxd = iaa_device->idxd;
753 pdev = idxd->pdev;
754 dev = &pdev->dev;
755 /*
756 * Check to see that we don't already have this wq.
757 * Shouldn't happen but we don't control probing.
758 */
759 if (iaa_has_wq(iaa_device, wq)) {
760 dev_dbg(dev, "same wq probed multiple times for iaa_device %p\n",
761 iaa_device);
762 goto out;
763 }
764
765 found = iaa_device;
766
767 ret = add_iaa_wq(iaa_device, wq, NULL);
768 if (ret)
769 goto out;
770
771 break;
772 }
773 }
774
775 if (!found) {
776 struct iaa_device *new_device;
777 struct iaa_wq *new_wq;
778
779 new_device = add_iaa_device(wq->idxd);
780 if (!new_device) {
781 ret = -ENOMEM;
782 goto out;
783 }
784
785 ret = add_iaa_wq(new_device, wq, &new_wq);
786 if (ret) {
787 del_iaa_device(new_device);
788 free_iaa_device(new_device);
789 goto out;
790 }
791
792 ret = init_iaa_device(new_device, new_wq);
793 if (ret) {
794 del_iaa_wq(new_device, new_wq->wq);
795 del_iaa_device(new_device);
796 free_iaa_wq(new_wq);
797 goto out;
798 }
799 }
800
801 if (WARN_ON(nr_iaa == 0))
802 return -EINVAL;
803
804 cpus_per_iaa = (nr_nodes * nr_cpus_per_node) / nr_iaa;
805 if (!cpus_per_iaa)
806 cpus_per_iaa = 1;
807 out:
808 return 0;
809 }
810
remove_iaa_wq(struct idxd_wq * wq)811 static void remove_iaa_wq(struct idxd_wq *wq)
812 {
813 struct iaa_device *iaa_device;
814
815 list_for_each_entry(iaa_device, &iaa_devices, list) {
816 if (iaa_has_wq(iaa_device, wq)) {
817 del_iaa_wq(iaa_device, wq);
818 break;
819 }
820 }
821
822 if (nr_iaa) {
823 cpus_per_iaa = (nr_nodes * nr_cpus_per_node) / nr_iaa;
824 if (!cpus_per_iaa)
825 cpus_per_iaa = 1;
826 } else
827 cpus_per_iaa = 1;
828 }
829
wq_table_add_wqs(int iaa,int cpu)830 static int wq_table_add_wqs(int iaa, int cpu)
831 {
832 struct iaa_device *iaa_device, *found_device = NULL;
833 int ret = 0, cur_iaa = 0, n_wqs_added = 0;
834 struct idxd_device *idxd;
835 struct iaa_wq *iaa_wq;
836 struct pci_dev *pdev;
837 struct device *dev;
838
839 list_for_each_entry(iaa_device, &iaa_devices, list) {
840 idxd = iaa_device->idxd;
841 pdev = idxd->pdev;
842 dev = &pdev->dev;
843
844 if (cur_iaa != iaa) {
845 cur_iaa++;
846 continue;
847 }
848
849 found_device = iaa_device;
850 dev_dbg(dev, "getting wq from iaa_device %d, cur_iaa %d\n",
851 found_device->idxd->id, cur_iaa);
852 break;
853 }
854
855 if (!found_device) {
856 found_device = list_first_entry_or_null(&iaa_devices,
857 struct iaa_device, list);
858 if (!found_device) {
859 pr_debug("couldn't find any iaa devices with wqs!\n");
860 ret = -EINVAL;
861 goto out;
862 }
863 cur_iaa = 0;
864
865 idxd = found_device->idxd;
866 pdev = idxd->pdev;
867 dev = &pdev->dev;
868 dev_dbg(dev, "getting wq from only iaa_device %d, cur_iaa %d\n",
869 found_device->idxd->id, cur_iaa);
870 }
871
872 list_for_each_entry(iaa_wq, &found_device->wqs, list) {
873 wq_table_add(cpu, iaa_wq->wq);
874 pr_debug("rebalance: added wq for cpu=%d: iaa wq %d.%d\n",
875 cpu, iaa_wq->wq->idxd->id, iaa_wq->wq->id);
876 n_wqs_added++;
877 }
878
879 if (!n_wqs_added) {
880 pr_debug("couldn't find any iaa wqs!\n");
881 ret = -EINVAL;
882 goto out;
883 }
884 out:
885 return ret;
886 }
887
888 /*
889 * Rebalance the wq table so that given a cpu, it's easy to find the
890 * closest IAA instance. The idea is to try to choose the most
891 * appropriate IAA instance for a caller and spread available
892 * workqueues around to clients.
893 */
rebalance_wq_table(void)894 static void rebalance_wq_table(void)
895 {
896 const struct cpumask *node_cpus;
897 int node, cpu, iaa = -1;
898
899 if (nr_iaa == 0)
900 return;
901
902 pr_debug("rebalance: nr_nodes=%d, nr_cpus %d, nr_iaa %d, cpus_per_iaa %d\n",
903 nr_nodes, nr_cpus, nr_iaa, cpus_per_iaa);
904
905 clear_wq_table();
906
907 if (nr_iaa == 1) {
908 for (cpu = 0; cpu < nr_cpus; cpu++) {
909 if (WARN_ON(wq_table_add_wqs(0, cpu))) {
910 pr_debug("could not add any wqs for iaa 0 to cpu %d!\n", cpu);
911 return;
912 }
913 }
914
915 return;
916 }
917
918 for_each_node_with_cpus(node) {
919 node_cpus = cpumask_of_node(node);
920
921 for (cpu = 0; cpu < cpumask_weight(node_cpus); cpu++) {
922 int node_cpu = cpumask_nth(cpu, node_cpus);
923
924 if (WARN_ON(node_cpu >= nr_cpu_ids)) {
925 pr_debug("node_cpu %d doesn't exist!\n", node_cpu);
926 return;
927 }
928
929 if ((cpu % cpus_per_iaa) == 0)
930 iaa++;
931
932 if (WARN_ON(wq_table_add_wqs(iaa, node_cpu))) {
933 pr_debug("could not add any wqs for iaa %d to cpu %d!\n", iaa, cpu);
934 return;
935 }
936 }
937 }
938 }
939
check_completion(struct device * dev,struct iax_completion_record * comp,bool compress,bool only_once)940 static inline int check_completion(struct device *dev,
941 struct iax_completion_record *comp,
942 bool compress,
943 bool only_once)
944 {
945 char *op_str = compress ? "compress" : "decompress";
946 int status_checks = 0;
947 int ret = 0;
948
949 while (!comp->status) {
950 if (only_once)
951 return -EAGAIN;
952 cpu_relax();
953 if (status_checks++ >= IAA_COMPLETION_TIMEOUT) {
954 /* Something is wrong with the hw, disable it. */
955 dev_err(dev, "%s completion timed out - "
956 "assuming broken hw, iaa_crypto now DISABLED\n",
957 op_str);
958 iaa_crypto_enabled = false;
959 ret = -ETIMEDOUT;
960 goto out;
961 }
962 }
963
964 if (comp->status != IAX_COMP_SUCCESS) {
965 if (comp->status == IAA_ERROR_WATCHDOG_EXPIRED) {
966 ret = -ETIMEDOUT;
967 dev_dbg(dev, "%s timed out, size=0x%x\n",
968 op_str, comp->output_size);
969 update_completion_timeout_errs();
970 goto out;
971 }
972
973 if (comp->status == IAA_ANALYTICS_ERROR &&
974 comp->error_code == IAA_ERROR_COMP_BUF_OVERFLOW && compress) {
975 ret = -E2BIG;
976 dev_dbg(dev, "compressed > uncompressed size,"
977 " not compressing, size=0x%x\n",
978 comp->output_size);
979 update_completion_comp_buf_overflow_errs();
980 goto out;
981 }
982
983 if (comp->status == IAA_ERROR_DECOMP_BUF_OVERFLOW) {
984 ret = -EOVERFLOW;
985 goto out;
986 }
987
988 ret = -EINVAL;
989 dev_dbg(dev, "iaa %s status=0x%x, error=0x%x, size=0x%x\n",
990 op_str, comp->status, comp->error_code, comp->output_size);
991 print_hex_dump(KERN_INFO, "cmp-rec: ", DUMP_PREFIX_OFFSET, 8, 1, comp, 64, 0);
992 update_completion_einval_errs();
993
994 goto out;
995 }
996 out:
997 return ret;
998 }
999
deflate_generic_decompress(struct acomp_req * req)1000 static int deflate_generic_decompress(struct acomp_req *req)
1001 {
1002 ACOMP_REQUEST_ON_STACK(fbreq, crypto_acomp_reqtfm(req));
1003 int ret;
1004
1005 acomp_request_set_callback(fbreq, 0, NULL, NULL);
1006 acomp_request_set_params(fbreq, req->src, req->dst, req->slen,
1007 req->dlen);
1008 ret = crypto_acomp_decompress(fbreq);
1009 req->dlen = fbreq->dlen;
1010
1011 update_total_sw_decomp_calls();
1012
1013 return ret;
1014 }
1015
1016 static int iaa_remap_for_verify(struct device *dev, struct iaa_wq *iaa_wq,
1017 struct acomp_req *req,
1018 dma_addr_t *src_addr, dma_addr_t *dst_addr);
1019
1020 static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req,
1021 struct idxd_wq *wq,
1022 dma_addr_t src_addr, unsigned int slen,
1023 dma_addr_t dst_addr, unsigned int *dlen,
1024 u32 compression_crc);
1025
iaa_desc_complete(struct idxd_desc * idxd_desc,enum idxd_complete_type comp_type,bool free_desc,void * __ctx,u32 * status)1026 static void iaa_desc_complete(struct idxd_desc *idxd_desc,
1027 enum idxd_complete_type comp_type,
1028 bool free_desc, void *__ctx,
1029 u32 *status)
1030 {
1031 struct iaa_device_compression_mode *active_compression_mode;
1032 struct iaa_compression_ctx *compression_ctx;
1033 struct crypto_ctx *ctx = __ctx;
1034 struct iaa_device *iaa_device;
1035 struct idxd_device *idxd;
1036 struct iaa_wq *iaa_wq;
1037 struct pci_dev *pdev;
1038 struct device *dev;
1039 int ret, err = 0;
1040
1041 compression_ctx = crypto_tfm_ctx(ctx->tfm);
1042
1043 iaa_wq = idxd_wq_get_private(idxd_desc->wq);
1044 iaa_device = iaa_wq->iaa_device;
1045 idxd = iaa_device->idxd;
1046 pdev = idxd->pdev;
1047 dev = &pdev->dev;
1048
1049 active_compression_mode = get_iaa_device_compression_mode(iaa_device,
1050 compression_ctx->mode);
1051 dev_dbg(dev, "%s: compression mode %s,"
1052 " ctx->src_addr %llx, ctx->dst_addr %llx\n", __func__,
1053 active_compression_mode->name,
1054 ctx->src_addr, ctx->dst_addr);
1055
1056 ret = check_completion(dev, idxd_desc->iax_completion,
1057 ctx->compress, false);
1058 if (ret) {
1059 dev_dbg(dev, "%s: check_completion failed ret=%d\n", __func__, ret);
1060 if (!ctx->compress &&
1061 idxd_desc->iax_completion->status == IAA_ANALYTICS_ERROR) {
1062 pr_warn("%s: falling back to deflate-generic decompress, "
1063 "analytics error code %x\n", __func__,
1064 idxd_desc->iax_completion->error_code);
1065 ret = deflate_generic_decompress(ctx->req);
1066 if (ret) {
1067 dev_dbg(dev, "%s: deflate-generic failed ret=%d\n",
1068 __func__, ret);
1069 err = -EIO;
1070 goto err;
1071 }
1072 } else {
1073 err = -EIO;
1074 goto err;
1075 }
1076 } else {
1077 ctx->req->dlen = idxd_desc->iax_completion->output_size;
1078 }
1079
1080 /* Update stats */
1081 if (ctx->compress) {
1082 update_total_comp_bytes_out(ctx->req->dlen);
1083 update_wq_comp_bytes(iaa_wq->wq, ctx->req->dlen);
1084 } else {
1085 update_total_decomp_bytes_in(ctx->req->slen);
1086 update_wq_decomp_bytes(iaa_wq->wq, ctx->req->slen);
1087 }
1088
1089 if (ctx->compress && compression_ctx->verify_compress) {
1090 dma_addr_t src_addr, dst_addr;
1091 u32 compression_crc;
1092
1093 compression_crc = idxd_desc->iax_completion->crc;
1094
1095 ret = iaa_remap_for_verify(dev, iaa_wq, ctx->req, &src_addr, &dst_addr);
1096 if (ret) {
1097 dev_dbg(dev, "%s: compress verify remap failed ret=%d\n", __func__, ret);
1098 err = -EIO;
1099 goto out;
1100 }
1101
1102 ret = iaa_compress_verify(ctx->tfm, ctx->req, iaa_wq->wq, src_addr,
1103 ctx->req->slen, dst_addr, &ctx->req->dlen,
1104 compression_crc);
1105 if (ret) {
1106 dev_dbg(dev, "%s: compress verify failed ret=%d\n", __func__, ret);
1107 err = -EIO;
1108 }
1109
1110 dma_unmap_sg(dev, ctx->req->dst, sg_nents(ctx->req->dst), DMA_TO_DEVICE);
1111 dma_unmap_sg(dev, ctx->req->src, sg_nents(ctx->req->src), DMA_FROM_DEVICE);
1112
1113 goto out;
1114 }
1115 err:
1116 dma_unmap_sg(dev, ctx->req->dst, sg_nents(ctx->req->dst), DMA_FROM_DEVICE);
1117 dma_unmap_sg(dev, ctx->req->src, sg_nents(ctx->req->src), DMA_TO_DEVICE);
1118 out:
1119 if (ret != 0)
1120 dev_dbg(dev, "asynchronous compress failed ret=%d\n", ret);
1121
1122 if (ctx->req->base.complete)
1123 acomp_request_complete(ctx->req, err);
1124
1125 if (free_desc)
1126 idxd_free_desc(idxd_desc->wq, idxd_desc);
1127 iaa_wq_put(idxd_desc->wq);
1128 }
1129
iaa_compress(struct crypto_tfm * tfm,struct acomp_req * req,struct idxd_wq * wq,dma_addr_t src_addr,unsigned int slen,dma_addr_t dst_addr,unsigned int * dlen,u32 * compression_crc)1130 static int iaa_compress(struct crypto_tfm *tfm, struct acomp_req *req,
1131 struct idxd_wq *wq,
1132 dma_addr_t src_addr, unsigned int slen,
1133 dma_addr_t dst_addr, unsigned int *dlen,
1134 u32 *compression_crc)
1135 {
1136 struct iaa_device_compression_mode *active_compression_mode;
1137 struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm);
1138 struct iaa_device *iaa_device;
1139 struct idxd_desc *idxd_desc;
1140 struct iax_hw_desc *desc;
1141 struct idxd_device *idxd;
1142 struct iaa_wq *iaa_wq;
1143 struct pci_dev *pdev;
1144 struct device *dev;
1145 int ret = 0;
1146
1147 iaa_wq = idxd_wq_get_private(wq);
1148 iaa_device = iaa_wq->iaa_device;
1149 idxd = iaa_device->idxd;
1150 pdev = idxd->pdev;
1151 dev = &pdev->dev;
1152
1153 active_compression_mode = get_iaa_device_compression_mode(iaa_device, ctx->mode);
1154
1155 idxd_desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK);
1156 if (IS_ERR(idxd_desc)) {
1157 dev_dbg(dev, "idxd descriptor allocation failed\n");
1158 dev_dbg(dev, "iaa compress failed: ret=%ld\n", PTR_ERR(idxd_desc));
1159 return PTR_ERR(idxd_desc);
1160 }
1161 desc = idxd_desc->iax_hw;
1162
1163 desc->flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR |
1164 IDXD_OP_FLAG_RD_SRC2_AECS | IDXD_OP_FLAG_CC;
1165 desc->opcode = IAX_OPCODE_COMPRESS;
1166 desc->compr_flags = IAA_COMP_FLAGS;
1167 desc->priv = 0;
1168
1169 desc->src1_addr = (u64)src_addr;
1170 desc->src1_size = slen;
1171 desc->dst_addr = (u64)dst_addr;
1172 desc->max_dst_size = *dlen;
1173 desc->src2_addr = active_compression_mode->aecs_comp_table_dma_addr;
1174 desc->src2_size = sizeof(struct aecs_comp_table_record);
1175 desc->completion_addr = idxd_desc->compl_dma;
1176
1177 if (ctx->use_irq) {
1178 desc->flags |= IDXD_OP_FLAG_RCI;
1179
1180 idxd_desc->crypto.req = req;
1181 idxd_desc->crypto.tfm = tfm;
1182 idxd_desc->crypto.src_addr = src_addr;
1183 idxd_desc->crypto.dst_addr = dst_addr;
1184 idxd_desc->crypto.compress = true;
1185
1186 dev_dbg(dev, "%s use_async_irq: compression mode %s,"
1187 " src_addr %llx, dst_addr %llx\n", __func__,
1188 active_compression_mode->name,
1189 src_addr, dst_addr);
1190 } else if (ctx->async_mode)
1191 req->base.data = idxd_desc;
1192
1193 dev_dbg(dev, "%s: compression mode %s,"
1194 " desc->src1_addr %llx, desc->src1_size %d,"
1195 " desc->dst_addr %llx, desc->max_dst_size %d,"
1196 " desc->src2_addr %llx, desc->src2_size %d\n", __func__,
1197 active_compression_mode->name,
1198 desc->src1_addr, desc->src1_size, desc->dst_addr,
1199 desc->max_dst_size, desc->src2_addr, desc->src2_size);
1200
1201 ret = idxd_submit_desc(wq, idxd_desc);
1202 if (ret) {
1203 dev_dbg(dev, "submit_desc failed ret=%d\n", ret);
1204 goto err;
1205 }
1206
1207 /* Update stats */
1208 update_total_comp_calls();
1209 update_wq_comp_calls(wq);
1210
1211 if (ctx->async_mode) {
1212 ret = -EINPROGRESS;
1213 dev_dbg(dev, "%s: returning -EINPROGRESS\n", __func__);
1214 goto out;
1215 }
1216
1217 ret = check_completion(dev, idxd_desc->iax_completion, true, false);
1218 if (ret) {
1219 dev_dbg(dev, "check_completion failed ret=%d\n", ret);
1220 goto err;
1221 }
1222
1223 *dlen = idxd_desc->iax_completion->output_size;
1224
1225 /* Update stats */
1226 update_total_comp_bytes_out(*dlen);
1227 update_wq_comp_bytes(wq, *dlen);
1228
1229 *compression_crc = idxd_desc->iax_completion->crc;
1230
1231 if (!ctx->async_mode)
1232 idxd_free_desc(wq, idxd_desc);
1233 out:
1234 return ret;
1235 err:
1236 idxd_free_desc(wq, idxd_desc);
1237 dev_dbg(dev, "iaa compress failed: ret=%d\n", ret);
1238
1239 goto out;
1240 }
1241
iaa_remap_for_verify(struct device * dev,struct iaa_wq * iaa_wq,struct acomp_req * req,dma_addr_t * src_addr,dma_addr_t * dst_addr)1242 static int iaa_remap_for_verify(struct device *dev, struct iaa_wq *iaa_wq,
1243 struct acomp_req *req,
1244 dma_addr_t *src_addr, dma_addr_t *dst_addr)
1245 {
1246 int ret = 0;
1247 int nr_sgs;
1248
1249 dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);
1250 dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE);
1251
1252 nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_FROM_DEVICE);
1253 if (nr_sgs <= 0 || nr_sgs > 1) {
1254 dev_dbg(dev, "verify: couldn't map src sg for iaa device %d,"
1255 " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,
1256 iaa_wq->wq->id, ret);
1257 ret = -EIO;
1258 goto out;
1259 }
1260 *src_addr = sg_dma_address(req->src);
1261 dev_dbg(dev, "verify: dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p,"
1262 " req->slen %d, sg_dma_len(sg) %d\n", *src_addr, nr_sgs,
1263 req->src, req->slen, sg_dma_len(req->src));
1264
1265 nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_TO_DEVICE);
1266 if (nr_sgs <= 0 || nr_sgs > 1) {
1267 dev_dbg(dev, "verify: couldn't map dst sg for iaa device %d,"
1268 " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,
1269 iaa_wq->wq->id, ret);
1270 ret = -EIO;
1271 dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_FROM_DEVICE);
1272 goto out;
1273 }
1274 *dst_addr = sg_dma_address(req->dst);
1275 dev_dbg(dev, "verify: dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p,"
1276 " req->dlen %d, sg_dma_len(sg) %d\n", *dst_addr, nr_sgs,
1277 req->dst, req->dlen, sg_dma_len(req->dst));
1278 out:
1279 return ret;
1280 }
1281
iaa_compress_verify(struct crypto_tfm * tfm,struct acomp_req * req,struct idxd_wq * wq,dma_addr_t src_addr,unsigned int slen,dma_addr_t dst_addr,unsigned int * dlen,u32 compression_crc)1282 static int iaa_compress_verify(struct crypto_tfm *tfm, struct acomp_req *req,
1283 struct idxd_wq *wq,
1284 dma_addr_t src_addr, unsigned int slen,
1285 dma_addr_t dst_addr, unsigned int *dlen,
1286 u32 compression_crc)
1287 {
1288 struct iaa_device_compression_mode *active_compression_mode;
1289 struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm);
1290 struct iaa_device *iaa_device;
1291 struct idxd_desc *idxd_desc;
1292 struct iax_hw_desc *desc;
1293 struct idxd_device *idxd;
1294 struct iaa_wq *iaa_wq;
1295 struct pci_dev *pdev;
1296 struct device *dev;
1297 int ret = 0;
1298
1299 iaa_wq = idxd_wq_get_private(wq);
1300 iaa_device = iaa_wq->iaa_device;
1301 idxd = iaa_device->idxd;
1302 pdev = idxd->pdev;
1303 dev = &pdev->dev;
1304
1305 active_compression_mode = get_iaa_device_compression_mode(iaa_device, ctx->mode);
1306
1307 idxd_desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK);
1308 if (IS_ERR(idxd_desc)) {
1309 dev_dbg(dev, "idxd descriptor allocation failed\n");
1310 dev_dbg(dev, "iaa compress failed: ret=%ld\n",
1311 PTR_ERR(idxd_desc));
1312 return PTR_ERR(idxd_desc);
1313 }
1314 desc = idxd_desc->iax_hw;
1315
1316 /* Verify (optional) - decompress and check crc, suppress dest write */
1317
1318 desc->flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR | IDXD_OP_FLAG_CC;
1319 desc->opcode = IAX_OPCODE_DECOMPRESS;
1320 desc->decompr_flags = IAA_DECOMP_FLAGS | IAA_DECOMP_SUPPRESS_OUTPUT;
1321 desc->priv = 0;
1322
1323 desc->src1_addr = (u64)dst_addr;
1324 desc->src1_size = *dlen;
1325 desc->dst_addr = (u64)src_addr;
1326 desc->max_dst_size = slen;
1327 desc->completion_addr = idxd_desc->compl_dma;
1328
1329 dev_dbg(dev, "(verify) compression mode %s,"
1330 " desc->src1_addr %llx, desc->src1_size %d,"
1331 " desc->dst_addr %llx, desc->max_dst_size %d,"
1332 " desc->src2_addr %llx, desc->src2_size %d\n",
1333 active_compression_mode->name,
1334 desc->src1_addr, desc->src1_size, desc->dst_addr,
1335 desc->max_dst_size, desc->src2_addr, desc->src2_size);
1336
1337 ret = idxd_submit_desc(wq, idxd_desc);
1338 if (ret) {
1339 dev_dbg(dev, "submit_desc (verify) failed ret=%d\n", ret);
1340 goto err;
1341 }
1342
1343 ret = check_completion(dev, idxd_desc->iax_completion, false, false);
1344 if (ret) {
1345 dev_dbg(dev, "(verify) check_completion failed ret=%d\n", ret);
1346 goto err;
1347 }
1348
1349 if (compression_crc != idxd_desc->iax_completion->crc) {
1350 ret = -EINVAL;
1351 dev_dbg(dev, "(verify) iaa comp/decomp crc mismatch:"
1352 " comp=0x%x, decomp=0x%x\n", compression_crc,
1353 idxd_desc->iax_completion->crc);
1354 print_hex_dump(KERN_INFO, "cmp-rec: ", DUMP_PREFIX_OFFSET,
1355 8, 1, idxd_desc->iax_completion, 64, 0);
1356 goto err;
1357 }
1358
1359 idxd_free_desc(wq, idxd_desc);
1360 out:
1361 return ret;
1362 err:
1363 idxd_free_desc(wq, idxd_desc);
1364 dev_dbg(dev, "iaa compress failed: ret=%d\n", ret);
1365
1366 goto out;
1367 }
1368
iaa_decompress(struct crypto_tfm * tfm,struct acomp_req * req,struct idxd_wq * wq,dma_addr_t src_addr,unsigned int slen,dma_addr_t dst_addr,unsigned int * dlen,bool disable_async)1369 static int iaa_decompress(struct crypto_tfm *tfm, struct acomp_req *req,
1370 struct idxd_wq *wq,
1371 dma_addr_t src_addr, unsigned int slen,
1372 dma_addr_t dst_addr, unsigned int *dlen,
1373 bool disable_async)
1374 {
1375 struct iaa_device_compression_mode *active_compression_mode;
1376 struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm);
1377 struct iaa_device *iaa_device;
1378 struct idxd_desc *idxd_desc;
1379 struct iax_hw_desc *desc;
1380 struct idxd_device *idxd;
1381 struct iaa_wq *iaa_wq;
1382 struct pci_dev *pdev;
1383 struct device *dev;
1384 int ret = 0;
1385
1386 iaa_wq = idxd_wq_get_private(wq);
1387 iaa_device = iaa_wq->iaa_device;
1388 idxd = iaa_device->idxd;
1389 pdev = idxd->pdev;
1390 dev = &pdev->dev;
1391
1392 active_compression_mode = get_iaa_device_compression_mode(iaa_device, ctx->mode);
1393
1394 idxd_desc = idxd_alloc_desc(wq, IDXD_OP_BLOCK);
1395 if (IS_ERR(idxd_desc)) {
1396 dev_dbg(dev, "idxd descriptor allocation failed\n");
1397 dev_dbg(dev, "iaa decompress failed: ret=%ld\n",
1398 PTR_ERR(idxd_desc));
1399 return PTR_ERR(idxd_desc);
1400 }
1401 desc = idxd_desc->iax_hw;
1402
1403 desc->flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR | IDXD_OP_FLAG_CC;
1404 desc->opcode = IAX_OPCODE_DECOMPRESS;
1405 desc->max_dst_size = PAGE_SIZE;
1406 desc->decompr_flags = IAA_DECOMP_FLAGS;
1407 desc->priv = 0;
1408
1409 desc->src1_addr = (u64)src_addr;
1410 desc->dst_addr = (u64)dst_addr;
1411 desc->max_dst_size = *dlen;
1412 desc->src1_size = slen;
1413 desc->completion_addr = idxd_desc->compl_dma;
1414
1415 if (ctx->use_irq && !disable_async) {
1416 desc->flags |= IDXD_OP_FLAG_RCI;
1417
1418 idxd_desc->crypto.req = req;
1419 idxd_desc->crypto.tfm = tfm;
1420 idxd_desc->crypto.src_addr = src_addr;
1421 idxd_desc->crypto.dst_addr = dst_addr;
1422 idxd_desc->crypto.compress = false;
1423
1424 dev_dbg(dev, "%s: use_async_irq compression mode %s,"
1425 " src_addr %llx, dst_addr %llx\n", __func__,
1426 active_compression_mode->name,
1427 src_addr, dst_addr);
1428 } else if (ctx->async_mode && !disable_async)
1429 req->base.data = idxd_desc;
1430
1431 dev_dbg(dev, "%s: decompression mode %s,"
1432 " desc->src1_addr %llx, desc->src1_size %d,"
1433 " desc->dst_addr %llx, desc->max_dst_size %d,"
1434 " desc->src2_addr %llx, desc->src2_size %d\n", __func__,
1435 active_compression_mode->name,
1436 desc->src1_addr, desc->src1_size, desc->dst_addr,
1437 desc->max_dst_size, desc->src2_addr, desc->src2_size);
1438
1439 ret = idxd_submit_desc(wq, idxd_desc);
1440 if (ret) {
1441 dev_dbg(dev, "submit_desc failed ret=%d\n", ret);
1442 goto err;
1443 }
1444
1445 /* Update stats */
1446 update_total_decomp_calls();
1447 update_wq_decomp_calls(wq);
1448
1449 if (ctx->async_mode && !disable_async) {
1450 ret = -EINPROGRESS;
1451 dev_dbg(dev, "%s: returning -EINPROGRESS\n", __func__);
1452 goto out;
1453 }
1454
1455 ret = check_completion(dev, idxd_desc->iax_completion, false, false);
1456 if (ret) {
1457 dev_dbg(dev, "%s: check_completion failed ret=%d\n", __func__, ret);
1458 if (idxd_desc->iax_completion->status == IAA_ANALYTICS_ERROR) {
1459 pr_warn("%s: falling back to deflate-generic decompress, "
1460 "analytics error code %x\n", __func__,
1461 idxd_desc->iax_completion->error_code);
1462 ret = deflate_generic_decompress(req);
1463 if (ret) {
1464 dev_dbg(dev, "%s: deflate-generic failed ret=%d\n",
1465 __func__, ret);
1466 goto err;
1467 }
1468 } else {
1469 goto err;
1470 }
1471 } else {
1472 req->dlen = idxd_desc->iax_completion->output_size;
1473 }
1474
1475 *dlen = req->dlen;
1476
1477 if (!ctx->async_mode || disable_async)
1478 idxd_free_desc(wq, idxd_desc);
1479
1480 /* Update stats */
1481 update_total_decomp_bytes_in(slen);
1482 update_wq_decomp_bytes(wq, slen);
1483 out:
1484 return ret;
1485 err:
1486 idxd_free_desc(wq, idxd_desc);
1487 dev_dbg(dev, "iaa decompress failed: ret=%d\n", ret);
1488
1489 goto out;
1490 }
1491
iaa_comp_acompress(struct acomp_req * req)1492 static int iaa_comp_acompress(struct acomp_req *req)
1493 {
1494 struct iaa_compression_ctx *compression_ctx;
1495 struct crypto_tfm *tfm = req->base.tfm;
1496 dma_addr_t src_addr, dst_addr;
1497 int nr_sgs, cpu, ret = 0;
1498 struct iaa_wq *iaa_wq;
1499 u32 compression_crc;
1500 struct idxd_wq *wq;
1501 struct device *dev;
1502
1503 compression_ctx = crypto_tfm_ctx(tfm);
1504
1505 if (!iaa_crypto_enabled) {
1506 pr_debug("iaa_crypto disabled, not compressing\n");
1507 return -ENODEV;
1508 }
1509
1510 if (!req->src || !req->slen) {
1511 pr_debug("invalid src, not compressing\n");
1512 return -EINVAL;
1513 }
1514
1515 cpu = get_cpu();
1516 wq = wq_table_next_wq(cpu);
1517 put_cpu();
1518 if (!wq) {
1519 pr_debug("no wq configured for cpu=%d\n", cpu);
1520 return -ENODEV;
1521 }
1522
1523 ret = iaa_wq_get(wq);
1524 if (ret) {
1525 pr_debug("no wq available for cpu=%d\n", cpu);
1526 return -ENODEV;
1527 }
1528
1529 iaa_wq = idxd_wq_get_private(wq);
1530
1531 dev = &wq->idxd->pdev->dev;
1532
1533 nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE);
1534 if (nr_sgs <= 0 || nr_sgs > 1) {
1535 dev_dbg(dev, "couldn't map src sg for iaa device %d,"
1536 " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,
1537 iaa_wq->wq->id, ret);
1538 ret = -EIO;
1539 goto out;
1540 }
1541 src_addr = sg_dma_address(req->src);
1542 dev_dbg(dev, "dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p,"
1543 " req->slen %d, sg_dma_len(sg) %d\n", src_addr, nr_sgs,
1544 req->src, req->slen, sg_dma_len(req->src));
1545
1546 nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);
1547 if (nr_sgs <= 0 || nr_sgs > 1) {
1548 dev_dbg(dev, "couldn't map dst sg for iaa device %d,"
1549 " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,
1550 iaa_wq->wq->id, ret);
1551 ret = -EIO;
1552 goto err_map_dst;
1553 }
1554 dst_addr = sg_dma_address(req->dst);
1555 dev_dbg(dev, "dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p,"
1556 " req->dlen %d, sg_dma_len(sg) %d\n", dst_addr, nr_sgs,
1557 req->dst, req->dlen, sg_dma_len(req->dst));
1558
1559 ret = iaa_compress(tfm, req, wq, src_addr, req->slen, dst_addr,
1560 &req->dlen, &compression_crc);
1561 if (ret == -EINPROGRESS)
1562 return ret;
1563
1564 if (!ret && compression_ctx->verify_compress) {
1565 ret = iaa_remap_for_verify(dev, iaa_wq, req, &src_addr, &dst_addr);
1566 if (ret) {
1567 dev_dbg(dev, "%s: compress verify remap failed ret=%d\n", __func__, ret);
1568 goto out;
1569 }
1570
1571 ret = iaa_compress_verify(tfm, req, wq, src_addr, req->slen,
1572 dst_addr, &req->dlen, compression_crc);
1573 if (ret)
1574 dev_dbg(dev, "asynchronous compress verification failed ret=%d\n", ret);
1575
1576 dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_TO_DEVICE);
1577 dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_FROM_DEVICE);
1578
1579 goto out;
1580 }
1581
1582 if (ret)
1583 dev_dbg(dev, "asynchronous compress failed ret=%d\n", ret);
1584
1585 dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);
1586 err_map_dst:
1587 dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE);
1588 out:
1589 iaa_wq_put(wq);
1590
1591 return ret;
1592 }
1593
iaa_comp_adecompress(struct acomp_req * req)1594 static int iaa_comp_adecompress(struct acomp_req *req)
1595 {
1596 struct crypto_tfm *tfm = req->base.tfm;
1597 dma_addr_t src_addr, dst_addr;
1598 int nr_sgs, cpu, ret = 0;
1599 struct iaa_wq *iaa_wq;
1600 struct device *dev;
1601 struct idxd_wq *wq;
1602
1603 if (!iaa_crypto_enabled) {
1604 pr_debug("iaa_crypto disabled, not decompressing\n");
1605 return -ENODEV;
1606 }
1607
1608 if (!req->src || !req->slen) {
1609 pr_debug("invalid src, not decompressing\n");
1610 return -EINVAL;
1611 }
1612
1613 cpu = get_cpu();
1614 wq = wq_table_next_wq(cpu);
1615 put_cpu();
1616 if (!wq) {
1617 pr_debug("no wq configured for cpu=%d\n", cpu);
1618 return -ENODEV;
1619 }
1620
1621 ret = iaa_wq_get(wq);
1622 if (ret) {
1623 pr_debug("no wq available for cpu=%d\n", cpu);
1624 return -ENODEV;
1625 }
1626
1627 iaa_wq = idxd_wq_get_private(wq);
1628
1629 dev = &wq->idxd->pdev->dev;
1630
1631 nr_sgs = dma_map_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE);
1632 if (nr_sgs <= 0 || nr_sgs > 1) {
1633 dev_dbg(dev, "couldn't map src sg for iaa device %d,"
1634 " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,
1635 iaa_wq->wq->id, ret);
1636 ret = -EIO;
1637 goto out;
1638 }
1639 src_addr = sg_dma_address(req->src);
1640 dev_dbg(dev, "dma_map_sg, src_addr %llx, nr_sgs %d, req->src %p,"
1641 " req->slen %d, sg_dma_len(sg) %d\n", src_addr, nr_sgs,
1642 req->src, req->slen, sg_dma_len(req->src));
1643
1644 nr_sgs = dma_map_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);
1645 if (nr_sgs <= 0 || nr_sgs > 1) {
1646 dev_dbg(dev, "couldn't map dst sg for iaa device %d,"
1647 " wq %d: ret=%d\n", iaa_wq->iaa_device->idxd->id,
1648 iaa_wq->wq->id, ret);
1649 ret = -EIO;
1650 goto err_map_dst;
1651 }
1652 dst_addr = sg_dma_address(req->dst);
1653 dev_dbg(dev, "dma_map_sg, dst_addr %llx, nr_sgs %d, req->dst %p,"
1654 " req->dlen %d, sg_dma_len(sg) %d\n", dst_addr, nr_sgs,
1655 req->dst, req->dlen, sg_dma_len(req->dst));
1656
1657 ret = iaa_decompress(tfm, req, wq, src_addr, req->slen,
1658 dst_addr, &req->dlen, false);
1659 if (ret == -EINPROGRESS)
1660 return ret;
1661
1662 if (ret != 0)
1663 dev_dbg(dev, "asynchronous decompress failed ret=%d\n", ret);
1664
1665 dma_unmap_sg(dev, req->dst, sg_nents(req->dst), DMA_FROM_DEVICE);
1666 err_map_dst:
1667 dma_unmap_sg(dev, req->src, sg_nents(req->src), DMA_TO_DEVICE);
1668 out:
1669 iaa_wq_put(wq);
1670
1671 return ret;
1672 }
1673
compression_ctx_init(struct iaa_compression_ctx * ctx)1674 static void compression_ctx_init(struct iaa_compression_ctx *ctx)
1675 {
1676 ctx->verify_compress = iaa_verify_compress;
1677 ctx->async_mode = async_mode;
1678 ctx->use_irq = use_irq;
1679 }
1680
iaa_comp_init_fixed(struct crypto_acomp * acomp_tfm)1681 static int iaa_comp_init_fixed(struct crypto_acomp *acomp_tfm)
1682 {
1683 struct crypto_tfm *tfm = crypto_acomp_tfm(acomp_tfm);
1684 struct iaa_compression_ctx *ctx = crypto_tfm_ctx(tfm);
1685
1686 compression_ctx_init(ctx);
1687
1688 ctx->mode = IAA_MODE_FIXED;
1689
1690 return 0;
1691 }
1692
1693 static struct acomp_alg iaa_acomp_fixed_deflate = {
1694 .init = iaa_comp_init_fixed,
1695 .compress = iaa_comp_acompress,
1696 .decompress = iaa_comp_adecompress,
1697 .base = {
1698 .cra_name = "deflate",
1699 .cra_driver_name = "deflate-iaa",
1700 .cra_flags = CRYPTO_ALG_ASYNC,
1701 .cra_ctxsize = sizeof(struct iaa_compression_ctx),
1702 .cra_module = THIS_MODULE,
1703 .cra_priority = IAA_ALG_PRIORITY,
1704 }
1705 };
1706
iaa_register_compression_device(void)1707 static int iaa_register_compression_device(void)
1708 {
1709 int ret;
1710
1711 ret = crypto_register_acomp(&iaa_acomp_fixed_deflate);
1712 if (ret) {
1713 pr_err("deflate algorithm acomp fixed registration failed (%d)\n", ret);
1714 goto out;
1715 }
1716
1717 iaa_crypto_registered = true;
1718 out:
1719 return ret;
1720 }
1721
iaa_unregister_compression_device(void)1722 static int iaa_unregister_compression_device(void)
1723 {
1724 if (iaa_crypto_registered)
1725 crypto_unregister_acomp(&iaa_acomp_fixed_deflate);
1726
1727 return 0;
1728 }
1729
iaa_crypto_probe(struct idxd_dev * idxd_dev)1730 static int iaa_crypto_probe(struct idxd_dev *idxd_dev)
1731 {
1732 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
1733 struct idxd_device *idxd = wq->idxd;
1734 struct idxd_driver_data *data = idxd->data;
1735 struct device *dev = &idxd_dev->conf_dev;
1736 bool first_wq = false;
1737 int ret = 0;
1738
1739 if (idxd->state != IDXD_DEV_ENABLED)
1740 return -ENXIO;
1741
1742 if (data->type != IDXD_TYPE_IAX)
1743 return -ENODEV;
1744
1745 mutex_lock(&wq->wq_lock);
1746
1747 if (idxd_wq_get_private(wq)) {
1748 mutex_unlock(&wq->wq_lock);
1749 return -EBUSY;
1750 }
1751
1752 if (!idxd_wq_driver_name_match(wq, dev)) {
1753 dev_dbg(dev, "wq %d.%d driver_name match failed: wq driver_name %s, dev driver name %s\n",
1754 idxd->id, wq->id, wq->driver_name, dev->driver->name);
1755 idxd->cmd_status = IDXD_SCMD_WQ_NO_DRV_NAME;
1756 ret = -ENODEV;
1757 goto err;
1758 }
1759
1760 wq->type = IDXD_WQT_KERNEL;
1761
1762 ret = idxd_drv_enable_wq(wq);
1763 if (ret < 0) {
1764 dev_dbg(dev, "enable wq %d.%d failed: %d\n",
1765 idxd->id, wq->id, ret);
1766 ret = -ENXIO;
1767 goto err;
1768 }
1769
1770 mutex_lock(&iaa_devices_lock);
1771
1772 if (list_empty(&iaa_devices)) {
1773 ret = alloc_wq_table(wq->idxd->max_wqs);
1774 if (ret)
1775 goto err_alloc;
1776 first_wq = true;
1777 }
1778
1779 ret = save_iaa_wq(wq);
1780 if (ret)
1781 goto err_save;
1782
1783 rebalance_wq_table();
1784
1785 if (first_wq) {
1786 iaa_crypto_enabled = true;
1787 ret = iaa_register_compression_device();
1788 if (ret != 0) {
1789 iaa_crypto_enabled = false;
1790 dev_dbg(dev, "IAA compression device registration failed\n");
1791 goto err_register;
1792 }
1793 try_module_get(THIS_MODULE);
1794
1795 pr_info("iaa_crypto now ENABLED\n");
1796 }
1797
1798 mutex_unlock(&iaa_devices_lock);
1799 out:
1800 mutex_unlock(&wq->wq_lock);
1801
1802 return ret;
1803
1804 err_register:
1805 remove_iaa_wq(wq);
1806 free_iaa_wq(idxd_wq_get_private(wq));
1807 err_save:
1808 if (first_wq)
1809 free_wq_table();
1810 err_alloc:
1811 mutex_unlock(&iaa_devices_lock);
1812 idxd_drv_disable_wq(wq);
1813 err:
1814 wq->type = IDXD_WQT_NONE;
1815
1816 goto out;
1817 }
1818
iaa_crypto_remove(struct idxd_dev * idxd_dev)1819 static void iaa_crypto_remove(struct idxd_dev *idxd_dev)
1820 {
1821 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev);
1822 struct idxd_device *idxd = wq->idxd;
1823 struct iaa_wq *iaa_wq;
1824 bool free = false;
1825
1826 idxd_wq_quiesce(wq);
1827
1828 mutex_lock(&wq->wq_lock);
1829 mutex_lock(&iaa_devices_lock);
1830
1831 remove_iaa_wq(wq);
1832
1833 spin_lock(&idxd->dev_lock);
1834 iaa_wq = idxd_wq_get_private(wq);
1835 if (!iaa_wq) {
1836 spin_unlock(&idxd->dev_lock);
1837 pr_err("%s: no iaa_wq available to remove\n", __func__);
1838 goto out;
1839 }
1840
1841 if (iaa_wq->ref) {
1842 iaa_wq->remove = true;
1843 } else {
1844 wq = iaa_wq->wq;
1845 idxd_wq_set_private(wq, NULL);
1846 free = true;
1847 }
1848 spin_unlock(&idxd->dev_lock);
1849 if (free) {
1850 __free_iaa_wq(iaa_wq);
1851 kfree(iaa_wq);
1852 }
1853
1854 idxd_drv_disable_wq(wq);
1855 rebalance_wq_table();
1856
1857 if (nr_iaa == 0) {
1858 iaa_crypto_enabled = false;
1859 free_wq_table();
1860 module_put(THIS_MODULE);
1861
1862 pr_info("iaa_crypto now DISABLED\n");
1863 }
1864 out:
1865 mutex_unlock(&iaa_devices_lock);
1866 mutex_unlock(&wq->wq_lock);
1867 }
1868
1869 static enum idxd_dev_type dev_types[] = {
1870 IDXD_DEV_WQ,
1871 IDXD_DEV_NONE,
1872 };
1873
1874 static struct idxd_device_driver iaa_crypto_driver = {
1875 .probe = iaa_crypto_probe,
1876 .remove = iaa_crypto_remove,
1877 .name = IDXD_SUBDRIVER_NAME,
1878 .type = dev_types,
1879 .desc_complete = iaa_desc_complete,
1880 };
1881
iaa_crypto_init_module(void)1882 static int __init iaa_crypto_init_module(void)
1883 {
1884 int ret = 0;
1885 int node;
1886
1887 nr_cpus = num_possible_cpus();
1888 for_each_node_with_cpus(node)
1889 nr_nodes++;
1890 if (!nr_nodes) {
1891 pr_err("IAA couldn't find any nodes with cpus\n");
1892 return -ENODEV;
1893 }
1894 nr_cpus_per_node = nr_cpus / nr_nodes;
1895
1896 ret = iaa_aecs_init_fixed();
1897 if (ret < 0) {
1898 pr_debug("IAA fixed compression mode init failed\n");
1899 goto err_aecs_init;
1900 }
1901
1902 ret = idxd_driver_register(&iaa_crypto_driver);
1903 if (ret) {
1904 pr_debug("IAA wq sub-driver registration failed\n");
1905 goto err_driver_reg;
1906 }
1907
1908 ret = driver_create_file(&iaa_crypto_driver.drv,
1909 &driver_attr_verify_compress);
1910 if (ret) {
1911 pr_debug("IAA verify_compress attr creation failed\n");
1912 goto err_verify_attr_create;
1913 }
1914
1915 ret = driver_create_file(&iaa_crypto_driver.drv,
1916 &driver_attr_sync_mode);
1917 if (ret) {
1918 pr_debug("IAA sync mode attr creation failed\n");
1919 goto err_sync_attr_create;
1920 }
1921
1922 if (iaa_crypto_debugfs_init())
1923 pr_warn("debugfs init failed, stats not available\n");
1924
1925 pr_debug("initialized\n");
1926 out:
1927 return ret;
1928
1929 err_sync_attr_create:
1930 driver_remove_file(&iaa_crypto_driver.drv,
1931 &driver_attr_verify_compress);
1932 err_verify_attr_create:
1933 idxd_driver_unregister(&iaa_crypto_driver);
1934 err_driver_reg:
1935 iaa_aecs_cleanup_fixed();
1936 err_aecs_init:
1937
1938 goto out;
1939 }
1940
iaa_crypto_cleanup_module(void)1941 static void __exit iaa_crypto_cleanup_module(void)
1942 {
1943 if (iaa_unregister_compression_device())
1944 pr_debug("IAA compression device unregister failed\n");
1945
1946 iaa_crypto_debugfs_cleanup();
1947 driver_remove_file(&iaa_crypto_driver.drv,
1948 &driver_attr_sync_mode);
1949 driver_remove_file(&iaa_crypto_driver.drv,
1950 &driver_attr_verify_compress);
1951 idxd_driver_unregister(&iaa_crypto_driver);
1952 iaa_aecs_cleanup_fixed();
1953
1954 pr_debug("cleaned up\n");
1955 }
1956
1957 MODULE_IMPORT_NS("IDXD");
1958 MODULE_LICENSE("GPL");
1959 MODULE_ALIAS_IDXD_DEVICE(0);
1960 MODULE_AUTHOR("Intel Corporation");
1961 MODULE_DESCRIPTION("IAA Compression Accelerator Crypto Driver");
1962
1963 module_init(iaa_crypto_init_module);
1964 module_exit(iaa_crypto_cleanup_module);
1965