1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Driver for IBM Power 842 compression accelerator
4 *
5 * Copyright (C) IBM Corporation, 2012
6 *
7 * Authors: Robert Jennings <rcj@linux.vnet.ibm.com>
8 * Seth Jennings <sjenning@linux.vnet.ibm.com>
9 */
10
11 #include <asm/vio.h>
12 #include <asm/hvcall.h>
13 #include <asm/vas.h>
14 #include <crypto/internal/scompress.h>
15
16 #include "nx-842.h"
17 #include "nx_csbcpb.h" /* struct nx_csbcpb */
18
19 MODULE_LICENSE("GPL");
20 MODULE_AUTHOR("Robert Jennings <rcj@linux.vnet.ibm.com>");
21 MODULE_DESCRIPTION("842 H/W Compression driver for IBM Power processors");
22 MODULE_ALIAS_CRYPTO("842");
23 MODULE_ALIAS_CRYPTO("842-nx");
24
25 /*
26 * Coprocessor type specific capabilities from the hypervisor.
27 */
28 struct hv_nx_cop_caps {
29 __be64 descriptor;
30 __be64 req_max_processed_len; /* Max bytes in one GZIP request */
31 __be64 min_compress_len; /* Min compression size in bytes */
32 __be64 min_decompress_len; /* Min decompression size in bytes */
33 } __packed __aligned(0x1000);
34
35 /*
36 * Coprocessor type specific capabilities.
37 */
38 struct nx_cop_caps {
39 u64 descriptor;
40 u64 req_max_processed_len; /* Max bytes in one GZIP request */
41 u64 min_compress_len; /* Min compression in bytes */
42 u64 min_decompress_len; /* Min decompression in bytes */
43 };
44
45 static u64 caps_feat;
46 static struct nx_cop_caps nx_cop_caps;
47
48 static struct nx842_constraints nx842_pseries_constraints = {
49 .alignment = DDE_BUFFER_ALIGN,
50 .multiple = DDE_BUFFER_LAST_MULT,
51 .minimum = DDE_BUFFER_LAST_MULT,
52 .maximum = PAGE_SIZE, /* dynamic, max_sync_size */
53 };
54
check_constraints(unsigned long buf,unsigned int * len,bool in)55 static int check_constraints(unsigned long buf, unsigned int *len, bool in)
56 {
57 if (!IS_ALIGNED(buf, nx842_pseries_constraints.alignment)) {
58 pr_debug("%s buffer 0x%lx not aligned to 0x%x\n",
59 in ? "input" : "output", buf,
60 nx842_pseries_constraints.alignment);
61 return -EINVAL;
62 }
63 if (*len % nx842_pseries_constraints.multiple) {
64 pr_debug("%s buffer len 0x%x not multiple of 0x%x\n",
65 in ? "input" : "output", *len,
66 nx842_pseries_constraints.multiple);
67 if (in)
68 return -EINVAL;
69 *len = round_down(*len, nx842_pseries_constraints.multiple);
70 }
71 if (*len < nx842_pseries_constraints.minimum) {
72 pr_debug("%s buffer len 0x%x under minimum 0x%x\n",
73 in ? "input" : "output", *len,
74 nx842_pseries_constraints.minimum);
75 return -EINVAL;
76 }
77 if (*len > nx842_pseries_constraints.maximum) {
78 pr_debug("%s buffer len 0x%x over maximum 0x%x\n",
79 in ? "input" : "output", *len,
80 nx842_pseries_constraints.maximum);
81 if (in)
82 return -EINVAL;
83 *len = nx842_pseries_constraints.maximum;
84 }
85 return 0;
86 }
87
88 /* I assume we need to align the CSB? */
89 #define WORKMEM_ALIGN (256)
90
91 struct nx842_workmem {
92 /* scatterlist */
93 char slin[4096];
94 char slout[4096];
95 /* coprocessor status/parameter block */
96 struct nx_csbcpb csbcpb;
97
98 char padding[WORKMEM_ALIGN];
99 } __aligned(WORKMEM_ALIGN);
100
101 /* Macros for fields within nx_csbcpb */
102 /* Check the valid bit within the csbcpb valid field */
103 #define NX842_CSBCBP_VALID_CHK(x) (x & BIT_MASK(7))
104
105 /* CE macros operate on the completion_extension field bits in the csbcpb.
106 * CE0 0=full completion, 1=partial completion
107 * CE1 0=CE0 indicates completion, 1=termination (output may be modified)
108 * CE2 0=processed_bytes is source bytes, 1=processed_bytes is target bytes */
109 #define NX842_CSBCPB_CE0(x) (x & BIT_MASK(7))
110 #define NX842_CSBCPB_CE1(x) (x & BIT_MASK(6))
111 #define NX842_CSBCPB_CE2(x) (x & BIT_MASK(5))
112
113 /* The NX unit accepts data only on 4K page boundaries */
114 #define NX842_HW_PAGE_SIZE (4096)
115 #define NX842_HW_PAGE_MASK (~(NX842_HW_PAGE_SIZE-1))
116
117 struct ibm_nx842_counters {
118 atomic64_t comp_complete;
119 atomic64_t comp_failed;
120 atomic64_t decomp_complete;
121 atomic64_t decomp_failed;
122 atomic64_t swdecomp;
123 atomic64_t comp_times[32];
124 atomic64_t decomp_times[32];
125 };
126
127 struct nx842_devdata {
128 struct vio_dev *vdev;
129 struct device *dev;
130 struct ibm_nx842_counters *counters;
131 unsigned int max_sg_len;
132 unsigned int max_sync_size;
133 unsigned int max_sync_sg;
134 };
135
136 static struct nx842_devdata __rcu *devdata;
137 static DEFINE_SPINLOCK(devdata_spinlock);
138
139 #define NX842_COUNTER_INC(_x) \
140 static inline void nx842_inc_##_x( \
141 const struct nx842_devdata *dev) { \
142 if (dev) \
143 atomic64_inc(&dev->counters->_x); \
144 }
145 NX842_COUNTER_INC(comp_complete);
146 NX842_COUNTER_INC(comp_failed);
147 NX842_COUNTER_INC(decomp_complete);
148 NX842_COUNTER_INC(decomp_failed);
149 NX842_COUNTER_INC(swdecomp);
150
151 #define NX842_HIST_SLOTS 16
152
ibm_nx842_incr_hist(atomic64_t * times,unsigned int time)153 static void ibm_nx842_incr_hist(atomic64_t *times, unsigned int time)
154 {
155 int bucket = fls(time);
156
157 if (bucket)
158 bucket = min((NX842_HIST_SLOTS - 1), bucket - 1);
159
160 atomic64_inc(×[bucket]);
161 }
162
163 /* NX unit operation flags */
164 #define NX842_OP_COMPRESS 0x0
165 #define NX842_OP_CRC 0x1
166 #define NX842_OP_DECOMPRESS 0x2
167 #define NX842_OP_COMPRESS_CRC (NX842_OP_COMPRESS | NX842_OP_CRC)
168 #define NX842_OP_DECOMPRESS_CRC (NX842_OP_DECOMPRESS | NX842_OP_CRC)
169 #define NX842_OP_ASYNC (1<<23)
170 #define NX842_OP_NOTIFY (1<<22)
171 #define NX842_OP_NOTIFY_INT(x) ((x & 0xff)<<8)
172
nx842_get_desired_dma(struct vio_dev * viodev)173 static unsigned long nx842_get_desired_dma(struct vio_dev *viodev)
174 {
175 /* No use of DMA mappings within the driver. */
176 return 0;
177 }
178
179 struct nx842_slentry {
180 __be64 ptr; /* Real address (use __pa()) */
181 __be64 len;
182 };
183
184 /* pHyp scatterlist entry */
185 struct nx842_scatterlist {
186 int entry_nr; /* number of slentries */
187 struct nx842_slentry *entries; /* ptr to array of slentries */
188 };
189
190 /* Does not include sizeof(entry_nr) in the size */
nx842_get_scatterlist_size(struct nx842_scatterlist * sl)191 static inline unsigned long nx842_get_scatterlist_size(
192 struct nx842_scatterlist *sl)
193 {
194 return sl->entry_nr * sizeof(struct nx842_slentry);
195 }
196
nx842_build_scatterlist(unsigned long buf,int len,struct nx842_scatterlist * sl)197 static int nx842_build_scatterlist(unsigned long buf, int len,
198 struct nx842_scatterlist *sl)
199 {
200 unsigned long entrylen;
201 struct nx842_slentry *entry;
202
203 sl->entry_nr = 0;
204
205 entry = sl->entries;
206 while (len) {
207 entry->ptr = cpu_to_be64(nx842_get_pa((void *)buf));
208 entrylen = min_t(int, len,
209 LEN_ON_SIZE(buf, NX842_HW_PAGE_SIZE));
210 entry->len = cpu_to_be64(entrylen);
211
212 len -= entrylen;
213 buf += entrylen;
214
215 sl->entry_nr++;
216 entry++;
217 }
218
219 return 0;
220 }
221
nx842_validate_result(struct device * dev,struct cop_status_block * csb)222 static int nx842_validate_result(struct device *dev,
223 struct cop_status_block *csb)
224 {
225 /* The csb must be valid after returning from vio_h_cop_sync */
226 if (!NX842_CSBCBP_VALID_CHK(csb->valid)) {
227 dev_err(dev, "%s: cspcbp not valid upon completion.\n",
228 __func__);
229 dev_dbg(dev, "valid:0x%02x cs:0x%02x cc:0x%02x ce:0x%02x\n",
230 csb->valid,
231 csb->crb_seq_number,
232 csb->completion_code,
233 csb->completion_extension);
234 dev_dbg(dev, "processed_bytes:%d address:0x%016lx\n",
235 be32_to_cpu(csb->processed_byte_count),
236 (unsigned long)be64_to_cpu(csb->address));
237 return -EIO;
238 }
239
240 /* Check return values from the hardware in the CSB */
241 switch (csb->completion_code) {
242 case 0: /* Completed without error */
243 break;
244 case 64: /* Compression ok, but output larger than input */
245 dev_dbg(dev, "%s: output size larger than input size\n",
246 __func__);
247 break;
248 case 13: /* Output buffer too small */
249 dev_dbg(dev, "%s: Out of space in output buffer\n",
250 __func__);
251 return -ENOSPC;
252 case 65: /* Calculated CRC doesn't match the passed value */
253 dev_dbg(dev, "%s: CRC mismatch for decompression\n",
254 __func__);
255 return -EINVAL;
256 case 66: /* Input data contains an illegal template field */
257 case 67: /* Template indicates data past the end of the input stream */
258 dev_dbg(dev, "%s: Bad data for decompression (code:%d)\n",
259 __func__, csb->completion_code);
260 return -EINVAL;
261 default:
262 dev_dbg(dev, "%s: Unspecified error (code:%d)\n",
263 __func__, csb->completion_code);
264 return -EIO;
265 }
266
267 /* Hardware sanity check */
268 if (!NX842_CSBCPB_CE2(csb->completion_extension)) {
269 dev_err(dev, "%s: No error returned by hardware, but "
270 "data returned is unusable, contact support.\n"
271 "(Additional info: csbcbp->processed bytes "
272 "does not specify processed bytes for the "
273 "target buffer.)\n", __func__);
274 return -EIO;
275 }
276
277 return 0;
278 }
279
280 /**
281 * nx842_pseries_compress - Compress data using the 842 algorithm
282 *
283 * Compression provide by the NX842 coprocessor on IBM Power systems.
284 * The input buffer is compressed and the result is stored in the
285 * provided output buffer.
286 *
287 * Upon return from this function @outlen contains the length of the
288 * compressed data. If there is an error then @outlen will be 0 and an
289 * error will be specified by the return code from this function.
290 *
291 * @in: Pointer to input buffer
292 * @inlen: Length of input buffer
293 * @out: Pointer to output buffer
294 * @outlen: Length of output buffer
295 * @wmem: ptr to buffer for working memory, size determined by
296 * nx842_pseries_driver.workmem_size
297 *
298 * Returns:
299 * 0 Success, output of length @outlen stored in the buffer at @out
300 * -ENOMEM Unable to allocate internal buffers
301 * -ENOSPC Output buffer is to small
302 * -EIO Internal error
303 * -ENODEV Hardware unavailable
304 */
nx842_pseries_compress(const unsigned char * in,unsigned int inlen,unsigned char * out,unsigned int * outlen,void * wmem)305 static int nx842_pseries_compress(const unsigned char *in, unsigned int inlen,
306 unsigned char *out, unsigned int *outlen,
307 void *wmem)
308 {
309 struct nx842_devdata *local_devdata;
310 struct device *dev = NULL;
311 struct nx842_workmem *workmem;
312 struct nx842_scatterlist slin, slout;
313 struct nx_csbcpb *csbcpb;
314 int ret = 0;
315 unsigned long inbuf, outbuf;
316 struct vio_pfo_op op = {
317 .done = NULL,
318 .handle = 0,
319 .timeout = 0,
320 };
321 unsigned long start = get_tb();
322
323 inbuf = (unsigned long)in;
324 if (check_constraints(inbuf, &inlen, true))
325 return -EINVAL;
326
327 outbuf = (unsigned long)out;
328 if (check_constraints(outbuf, outlen, false))
329 return -EINVAL;
330
331 rcu_read_lock();
332 local_devdata = rcu_dereference(devdata);
333 if (!local_devdata || !local_devdata->dev) {
334 rcu_read_unlock();
335 return -ENODEV;
336 }
337 dev = local_devdata->dev;
338
339 /* Init scatterlist */
340 workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN);
341 slin.entries = (struct nx842_slentry *)workmem->slin;
342 slout.entries = (struct nx842_slentry *)workmem->slout;
343
344 /* Init operation */
345 op.flags = NX842_OP_COMPRESS_CRC;
346 csbcpb = &workmem->csbcpb;
347 memset(csbcpb, 0, sizeof(*csbcpb));
348 op.csbcpb = nx842_get_pa(csbcpb);
349
350 if ((inbuf & NX842_HW_PAGE_MASK) ==
351 ((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) {
352 /* Create direct DDE */
353 op.in = nx842_get_pa((void *)inbuf);
354 op.inlen = inlen;
355 } else {
356 /* Create indirect DDE (scatterlist) */
357 nx842_build_scatterlist(inbuf, inlen, &slin);
358 op.in = nx842_get_pa(slin.entries);
359 op.inlen = -nx842_get_scatterlist_size(&slin);
360 }
361
362 if ((outbuf & NX842_HW_PAGE_MASK) ==
363 ((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) {
364 /* Create direct DDE */
365 op.out = nx842_get_pa((void *)outbuf);
366 op.outlen = *outlen;
367 } else {
368 /* Create indirect DDE (scatterlist) */
369 nx842_build_scatterlist(outbuf, *outlen, &slout);
370 op.out = nx842_get_pa(slout.entries);
371 op.outlen = -nx842_get_scatterlist_size(&slout);
372 }
373
374 dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n",
375 __func__, (unsigned long)op.in, (long)op.inlen,
376 (unsigned long)op.out, (long)op.outlen);
377
378 /* Send request to pHyp */
379 ret = vio_h_cop_sync(local_devdata->vdev, &op);
380
381 /* Check for pHyp error */
382 if (ret) {
383 dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
384 __func__, ret, op.hcall_err);
385 ret = -EIO;
386 goto unlock;
387 }
388
389 /* Check for hardware error */
390 ret = nx842_validate_result(dev, &csbcpb->csb);
391 if (ret)
392 goto unlock;
393
394 *outlen = be32_to_cpu(csbcpb->csb.processed_byte_count);
395 dev_dbg(dev, "%s: processed_bytes=%d\n", __func__, *outlen);
396
397 unlock:
398 if (ret)
399 nx842_inc_comp_failed(local_devdata);
400 else {
401 nx842_inc_comp_complete(local_devdata);
402 ibm_nx842_incr_hist(local_devdata->counters->comp_times,
403 (get_tb() - start) / tb_ticks_per_usec);
404 }
405 rcu_read_unlock();
406 return ret;
407 }
408
409 /**
410 * nx842_pseries_decompress - Decompress data using the 842 algorithm
411 *
412 * Decompression provide by the NX842 coprocessor on IBM Power systems.
413 * The input buffer is decompressed and the result is stored in the
414 * provided output buffer. The size allocated to the output buffer is
415 * provided by the caller of this function in @outlen. Upon return from
416 * this function @outlen contains the length of the decompressed data.
417 * If there is an error then @outlen will be 0 and an error will be
418 * specified by the return code from this function.
419 *
420 * @in: Pointer to input buffer
421 * @inlen: Length of input buffer
422 * @out: Pointer to output buffer
423 * @outlen: Length of output buffer
424 * @wmem: ptr to buffer for working memory, size determined by
425 * nx842_pseries_driver.workmem_size
426 *
427 * Returns:
428 * 0 Success, output of length @outlen stored in the buffer at @out
429 * -ENODEV Hardware decompression device is unavailable
430 * -ENOMEM Unable to allocate internal buffers
431 * -ENOSPC Output buffer is to small
432 * -EINVAL Bad input data encountered when attempting decompress
433 * -EIO Internal error
434 */
nx842_pseries_decompress(const unsigned char * in,unsigned int inlen,unsigned char * out,unsigned int * outlen,void * wmem)435 static int nx842_pseries_decompress(const unsigned char *in, unsigned int inlen,
436 unsigned char *out, unsigned int *outlen,
437 void *wmem)
438 {
439 struct nx842_devdata *local_devdata;
440 struct device *dev = NULL;
441 struct nx842_workmem *workmem;
442 struct nx842_scatterlist slin, slout;
443 struct nx_csbcpb *csbcpb;
444 int ret = 0;
445 unsigned long inbuf, outbuf;
446 struct vio_pfo_op op = {
447 .done = NULL,
448 .handle = 0,
449 .timeout = 0,
450 };
451 unsigned long start = get_tb();
452
453 /* Ensure page alignment and size */
454 inbuf = (unsigned long)in;
455 if (check_constraints(inbuf, &inlen, true))
456 return -EINVAL;
457
458 outbuf = (unsigned long)out;
459 if (check_constraints(outbuf, outlen, false))
460 return -EINVAL;
461
462 rcu_read_lock();
463 local_devdata = rcu_dereference(devdata);
464 if (!local_devdata || !local_devdata->dev) {
465 rcu_read_unlock();
466 return -ENODEV;
467 }
468 dev = local_devdata->dev;
469
470 workmem = PTR_ALIGN(wmem, WORKMEM_ALIGN);
471
472 /* Init scatterlist */
473 slin.entries = (struct nx842_slentry *)workmem->slin;
474 slout.entries = (struct nx842_slentry *)workmem->slout;
475
476 /* Init operation */
477 op.flags = NX842_OP_DECOMPRESS_CRC;
478 csbcpb = &workmem->csbcpb;
479 memset(csbcpb, 0, sizeof(*csbcpb));
480 op.csbcpb = nx842_get_pa(csbcpb);
481
482 if ((inbuf & NX842_HW_PAGE_MASK) ==
483 ((inbuf + inlen - 1) & NX842_HW_PAGE_MASK)) {
484 /* Create direct DDE */
485 op.in = nx842_get_pa((void *)inbuf);
486 op.inlen = inlen;
487 } else {
488 /* Create indirect DDE (scatterlist) */
489 nx842_build_scatterlist(inbuf, inlen, &slin);
490 op.in = nx842_get_pa(slin.entries);
491 op.inlen = -nx842_get_scatterlist_size(&slin);
492 }
493
494 if ((outbuf & NX842_HW_PAGE_MASK) ==
495 ((outbuf + *outlen - 1) & NX842_HW_PAGE_MASK)) {
496 /* Create direct DDE */
497 op.out = nx842_get_pa((void *)outbuf);
498 op.outlen = *outlen;
499 } else {
500 /* Create indirect DDE (scatterlist) */
501 nx842_build_scatterlist(outbuf, *outlen, &slout);
502 op.out = nx842_get_pa(slout.entries);
503 op.outlen = -nx842_get_scatterlist_size(&slout);
504 }
505
506 dev_dbg(dev, "%s: op.in %lx op.inlen %ld op.out %lx op.outlen %ld\n",
507 __func__, (unsigned long)op.in, (long)op.inlen,
508 (unsigned long)op.out, (long)op.outlen);
509
510 /* Send request to pHyp */
511 ret = vio_h_cop_sync(local_devdata->vdev, &op);
512
513 /* Check for pHyp error */
514 if (ret) {
515 dev_dbg(dev, "%s: vio_h_cop_sync error (ret=%d, hret=%ld)\n",
516 __func__, ret, op.hcall_err);
517 goto unlock;
518 }
519
520 /* Check for hardware error */
521 ret = nx842_validate_result(dev, &csbcpb->csb);
522 if (ret)
523 goto unlock;
524
525 *outlen = be32_to_cpu(csbcpb->csb.processed_byte_count);
526
527 unlock:
528 if (ret)
529 /* decompress fail */
530 nx842_inc_decomp_failed(local_devdata);
531 else {
532 nx842_inc_decomp_complete(local_devdata);
533 ibm_nx842_incr_hist(local_devdata->counters->decomp_times,
534 (get_tb() - start) / tb_ticks_per_usec);
535 }
536
537 rcu_read_unlock();
538 return ret;
539 }
540
541 /**
542 * nx842_OF_set_defaults -- Set default (disabled) values for devdata
543 *
544 * @devdata: struct nx842_devdata to update
545 *
546 * Returns:
547 * 0 on success
548 * -ENOENT if @devdata ptr is NULL
549 */
nx842_OF_set_defaults(struct nx842_devdata * devdata)550 static int nx842_OF_set_defaults(struct nx842_devdata *devdata)
551 {
552 if (devdata) {
553 devdata->max_sync_size = 0;
554 devdata->max_sync_sg = 0;
555 devdata->max_sg_len = 0;
556 return 0;
557 } else
558 return -ENOENT;
559 }
560
561 /**
562 * nx842_OF_upd_status -- Check the device info from OF status prop
563 *
564 * The status property indicates if the accelerator is enabled. If the
565 * device is in the OF tree it indicates that the hardware is present.
566 * The status field indicates if the device is enabled when the status
567 * is 'okay'. Otherwise the device driver will be disabled.
568 *
569 * @devdata: struct nx842_devdata to use for dev_info
570 * @prop: struct property point containing the maxsyncop for the update
571 *
572 * Returns:
573 * 0 - Device is available
574 * -ENODEV - Device is not available
575 */
nx842_OF_upd_status(struct nx842_devdata * devdata,struct property * prop)576 static int nx842_OF_upd_status(struct nx842_devdata *devdata,
577 struct property *prop)
578 {
579 const char *status = (const char *)prop->value;
580
581 if (!strncmp(status, "okay", (size_t)prop->length))
582 return 0;
583 if (!strncmp(status, "disabled", (size_t)prop->length))
584 return -ENODEV;
585 dev_info(devdata->dev, "%s: unknown status '%s'\n", __func__, status);
586
587 return -EINVAL;
588 }
589
590 /**
591 * nx842_OF_upd_maxsglen -- Update the device info from OF maxsglen prop
592 *
593 * Definition of the 'ibm,max-sg-len' OF property:
594 * This field indicates the maximum byte length of a scatter list
595 * for the platform facility. It is a single cell encoded as with encode-int.
596 *
597 * Example:
598 * # od -x ibm,max-sg-len
599 * 0000000 0000 0ff0
600 *
601 * In this example, the maximum byte length of a scatter list is
602 * 0x0ff0 (4,080).
603 *
604 * @devdata: struct nx842_devdata to update
605 * @prop: struct property point containing the maxsyncop for the update
606 *
607 * Returns:
608 * 0 on success
609 * -EINVAL on failure
610 */
nx842_OF_upd_maxsglen(struct nx842_devdata * devdata,struct property * prop)611 static int nx842_OF_upd_maxsglen(struct nx842_devdata *devdata,
612 struct property *prop) {
613 int ret = 0;
614 const unsigned int maxsglen = of_read_number(prop->value, 1);
615
616 if (prop->length != sizeof(maxsglen)) {
617 dev_err(devdata->dev, "%s: unexpected format for ibm,max-sg-len property\n", __func__);
618 dev_dbg(devdata->dev, "%s: ibm,max-sg-len is %d bytes long, expected %lu bytes\n", __func__,
619 prop->length, sizeof(maxsglen));
620 ret = -EINVAL;
621 } else {
622 devdata->max_sg_len = min_t(unsigned int,
623 maxsglen, NX842_HW_PAGE_SIZE);
624 }
625
626 return ret;
627 }
628
629 /**
630 * nx842_OF_upd_maxsyncop -- Update the device info from OF maxsyncop prop
631 *
632 * Definition of the 'ibm,max-sync-cop' OF property:
633 * Two series of cells. The first series of cells represents the maximums
634 * that can be synchronously compressed. The second series of cells
635 * represents the maximums that can be synchronously decompressed.
636 * 1. The first cell in each series contains the count of the number of
637 * data length, scatter list elements pairs that follow – each being
638 * of the form
639 * a. One cell data byte length
640 * b. One cell total number of scatter list elements
641 *
642 * Example:
643 * # od -x ibm,max-sync-cop
644 * 0000000 0000 0001 0000 1000 0000 01fe 0000 0001
645 * 0000020 0000 1000 0000 01fe
646 *
647 * In this example, compression supports 0x1000 (4,096) data byte length
648 * and 0x1fe (510) total scatter list elements. Decompression supports
649 * 0x1000 (4,096) data byte length and 0x1f3 (510) total scatter list
650 * elements.
651 *
652 * @devdata: struct nx842_devdata to update
653 * @prop: struct property point containing the maxsyncop for the update
654 *
655 * Returns:
656 * 0 on success
657 * -EINVAL on failure
658 */
nx842_OF_upd_maxsyncop(struct nx842_devdata * devdata,struct property * prop)659 static int nx842_OF_upd_maxsyncop(struct nx842_devdata *devdata,
660 struct property *prop) {
661 int ret = 0;
662 unsigned int comp_data_limit, decomp_data_limit;
663 unsigned int comp_sg_limit, decomp_sg_limit;
664 const struct maxsynccop_t {
665 __be32 comp_elements;
666 __be32 comp_data_limit;
667 __be32 comp_sg_limit;
668 __be32 decomp_elements;
669 __be32 decomp_data_limit;
670 __be32 decomp_sg_limit;
671 } *maxsynccop;
672
673 if (prop->length != sizeof(*maxsynccop)) {
674 dev_err(devdata->dev, "%s: unexpected format for ibm,max-sync-cop property\n", __func__);
675 dev_dbg(devdata->dev, "%s: ibm,max-sync-cop is %d bytes long, expected %lu bytes\n", __func__, prop->length,
676 sizeof(*maxsynccop));
677 ret = -EINVAL;
678 goto out;
679 }
680
681 maxsynccop = (const struct maxsynccop_t *)prop->value;
682 comp_data_limit = be32_to_cpu(maxsynccop->comp_data_limit);
683 comp_sg_limit = be32_to_cpu(maxsynccop->comp_sg_limit);
684 decomp_data_limit = be32_to_cpu(maxsynccop->decomp_data_limit);
685 decomp_sg_limit = be32_to_cpu(maxsynccop->decomp_sg_limit);
686
687 /* Use one limit rather than separate limits for compression and
688 * decompression. Set a maximum for this so as not to exceed the
689 * size that the header can support and round the value down to
690 * the hardware page size (4K) */
691 devdata->max_sync_size = min(comp_data_limit, decomp_data_limit);
692
693 devdata->max_sync_size = min_t(unsigned int, devdata->max_sync_size,
694 65536);
695
696 if (devdata->max_sync_size < 4096) {
697 dev_err(devdata->dev, "%s: hardware max data size (%u) is "
698 "less than the driver minimum, unable to use "
699 "the hardware device\n",
700 __func__, devdata->max_sync_size);
701 ret = -EINVAL;
702 goto out;
703 }
704
705 nx842_pseries_constraints.maximum = devdata->max_sync_size;
706
707 devdata->max_sync_sg = min(comp_sg_limit, decomp_sg_limit);
708 if (devdata->max_sync_sg < 1) {
709 dev_err(devdata->dev, "%s: hardware max sg size (%u) is "
710 "less than the driver minimum, unable to use "
711 "the hardware device\n",
712 __func__, devdata->max_sync_sg);
713 ret = -EINVAL;
714 goto out;
715 }
716
717 out:
718 return ret;
719 }
720
721 /**
722 * nx842_OF_upd -- Handle OF properties updates for the device.
723 *
724 * Set all properties from the OF tree. Optionally, a new property
725 * can be provided by the @new_prop pointer to overwrite an existing value.
726 * The device will remain disabled until all values are valid, this function
727 * will return an error for updates unless all values are valid.
728 *
729 * @new_prop: If not NULL, this property is being updated. If NULL, update
730 * all properties from the current values in the OF tree.
731 *
732 * Returns:
733 * 0 - Success
734 * -ENOMEM - Could not allocate memory for new devdata structure
735 * -EINVAL - property value not found, new_prop is not a recognized
736 * property for the device or property value is not valid.
737 * -ENODEV - Device is not available
738 */
nx842_OF_upd(struct property * new_prop)739 static int nx842_OF_upd(struct property *new_prop)
740 {
741 struct nx842_devdata *old_devdata = NULL;
742 struct nx842_devdata *new_devdata = NULL;
743 struct device_node *of_node = NULL;
744 struct property *status = NULL;
745 struct property *maxsglen = NULL;
746 struct property *maxsyncop = NULL;
747 int ret = 0;
748 unsigned long flags;
749
750 new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);
751 if (!new_devdata)
752 return -ENOMEM;
753
754 spin_lock_irqsave(&devdata_spinlock, flags);
755 old_devdata = rcu_dereference_check(devdata,
756 lockdep_is_held(&devdata_spinlock));
757 if (old_devdata)
758 of_node = old_devdata->dev->of_node;
759
760 if (!old_devdata || !of_node) {
761 pr_err("%s: device is not available\n", __func__);
762 spin_unlock_irqrestore(&devdata_spinlock, flags);
763 kfree(new_devdata);
764 return -ENODEV;
765 }
766
767 memcpy(new_devdata, old_devdata, sizeof(*old_devdata));
768 new_devdata->counters = old_devdata->counters;
769
770 /* Set ptrs for existing properties */
771 status = of_find_property(of_node, "status", NULL);
772 maxsglen = of_find_property(of_node, "ibm,max-sg-len", NULL);
773 maxsyncop = of_find_property(of_node, "ibm,max-sync-cop", NULL);
774 if (!status || !maxsglen || !maxsyncop) {
775 dev_err(old_devdata->dev, "%s: Could not locate device properties\n", __func__);
776 ret = -EINVAL;
777 goto error_out;
778 }
779
780 /*
781 * If this is a property update, there are only certain properties that
782 * we care about. Bail if it isn't in the below list
783 */
784 if (new_prop && (strncmp(new_prop->name, "status", new_prop->length) ||
785 strncmp(new_prop->name, "ibm,max-sg-len", new_prop->length) ||
786 strncmp(new_prop->name, "ibm,max-sync-cop", new_prop->length)))
787 goto out;
788
789 /* Perform property updates */
790 ret = nx842_OF_upd_status(new_devdata, status);
791 if (ret)
792 goto error_out;
793
794 ret = nx842_OF_upd_maxsglen(new_devdata, maxsglen);
795 if (ret)
796 goto error_out;
797
798 ret = nx842_OF_upd_maxsyncop(new_devdata, maxsyncop);
799 if (ret)
800 goto error_out;
801
802 out:
803 dev_info(old_devdata->dev, "%s: max_sync_size new:%u old:%u\n",
804 __func__, new_devdata->max_sync_size,
805 old_devdata->max_sync_size);
806 dev_info(old_devdata->dev, "%s: max_sync_sg new:%u old:%u\n",
807 __func__, new_devdata->max_sync_sg,
808 old_devdata->max_sync_sg);
809 dev_info(old_devdata->dev, "%s: max_sg_len new:%u old:%u\n",
810 __func__, new_devdata->max_sg_len,
811 old_devdata->max_sg_len);
812
813 rcu_assign_pointer(devdata, new_devdata);
814 spin_unlock_irqrestore(&devdata_spinlock, flags);
815 synchronize_rcu();
816 dev_set_drvdata(new_devdata->dev, new_devdata);
817 kfree(old_devdata);
818 return 0;
819
820 error_out:
821 if (new_devdata) {
822 dev_info(old_devdata->dev, "%s: device disabled\n", __func__);
823 nx842_OF_set_defaults(new_devdata);
824 rcu_assign_pointer(devdata, new_devdata);
825 spin_unlock_irqrestore(&devdata_spinlock, flags);
826 synchronize_rcu();
827 dev_set_drvdata(new_devdata->dev, new_devdata);
828 kfree(old_devdata);
829 } else {
830 dev_err(old_devdata->dev, "%s: could not update driver from hardware\n", __func__);
831 spin_unlock_irqrestore(&devdata_spinlock, flags);
832 }
833
834 if (!ret)
835 ret = -EINVAL;
836 return ret;
837 }
838
839 /**
840 * nx842_OF_notifier - Process updates to OF properties for the device
841 *
842 * @np: notifier block
843 * @action: notifier action
844 * @data: struct of_reconfig_data pointer
845 *
846 * Returns:
847 * NOTIFY_OK on success
848 * NOTIFY_BAD encoded with error number on failure, use
849 * notifier_to_errno() to decode this value
850 */
nx842_OF_notifier(struct notifier_block * np,unsigned long action,void * data)851 static int nx842_OF_notifier(struct notifier_block *np, unsigned long action,
852 void *data)
853 {
854 struct of_reconfig_data *upd = data;
855 struct nx842_devdata *local_devdata;
856 struct device_node *node = NULL;
857
858 rcu_read_lock();
859 local_devdata = rcu_dereference(devdata);
860 if (local_devdata)
861 node = local_devdata->dev->of_node;
862
863 if (local_devdata &&
864 action == OF_RECONFIG_UPDATE_PROPERTY &&
865 !strcmp(upd->dn->name, node->name)) {
866 rcu_read_unlock();
867 nx842_OF_upd(upd->prop);
868 } else
869 rcu_read_unlock();
870
871 return NOTIFY_OK;
872 }
873
874 static struct notifier_block nx842_of_nb = {
875 .notifier_call = nx842_OF_notifier,
876 };
877
878 #define nx842_counter_read(_name) \
879 static ssize_t nx842_##_name##_show(struct device *dev, \
880 struct device_attribute *attr, \
881 char *buf) { \
882 struct nx842_devdata *local_devdata; \
883 int p = 0; \
884 rcu_read_lock(); \
885 local_devdata = rcu_dereference(devdata); \
886 if (local_devdata) \
887 p = snprintf(buf, PAGE_SIZE, "%lld\n", \
888 atomic64_read(&local_devdata->counters->_name)); \
889 rcu_read_unlock(); \
890 return p; \
891 }
892
893 #define NX842DEV_COUNTER_ATTR_RO(_name) \
894 nx842_counter_read(_name); \
895 static struct device_attribute dev_attr_##_name = __ATTR(_name, \
896 0444, \
897 nx842_##_name##_show,\
898 NULL);
899
900 NX842DEV_COUNTER_ATTR_RO(comp_complete);
901 NX842DEV_COUNTER_ATTR_RO(comp_failed);
902 NX842DEV_COUNTER_ATTR_RO(decomp_complete);
903 NX842DEV_COUNTER_ATTR_RO(decomp_failed);
904 NX842DEV_COUNTER_ATTR_RO(swdecomp);
905
906 static ssize_t nx842_timehist_show(struct device *,
907 struct device_attribute *, char *);
908
909 static struct device_attribute dev_attr_comp_times = __ATTR(comp_times, 0444,
910 nx842_timehist_show, NULL);
911 static struct device_attribute dev_attr_decomp_times = __ATTR(decomp_times,
912 0444, nx842_timehist_show, NULL);
913
nx842_timehist_show(struct device * dev,struct device_attribute * attr,char * buf)914 static ssize_t nx842_timehist_show(struct device *dev,
915 struct device_attribute *attr, char *buf) {
916 char *p = buf;
917 struct nx842_devdata *local_devdata;
918 atomic64_t *times;
919 int bytes_remain = PAGE_SIZE;
920 int bytes;
921 int i;
922
923 rcu_read_lock();
924 local_devdata = rcu_dereference(devdata);
925 if (!local_devdata) {
926 rcu_read_unlock();
927 return 0;
928 }
929
930 if (attr == &dev_attr_comp_times)
931 times = local_devdata->counters->comp_times;
932 else if (attr == &dev_attr_decomp_times)
933 times = local_devdata->counters->decomp_times;
934 else {
935 rcu_read_unlock();
936 return 0;
937 }
938
939 for (i = 0; i < (NX842_HIST_SLOTS - 2); i++) {
940 bytes = snprintf(p, bytes_remain, "%u-%uus:\t%lld\n",
941 i ? (2<<(i-1)) : 0, (2<<i)-1,
942 atomic64_read(×[i]));
943 bytes_remain -= bytes;
944 p += bytes;
945 }
946 /* The last bucket holds everything over
947 * 2<<(NX842_HIST_SLOTS - 2) us */
948 bytes = snprintf(p, bytes_remain, "%uus - :\t%lld\n",
949 2<<(NX842_HIST_SLOTS - 2),
950 atomic64_read(×[(NX842_HIST_SLOTS - 1)]));
951 p += bytes;
952
953 rcu_read_unlock();
954 return p - buf;
955 }
956
957 static struct attribute *nx842_sysfs_entries[] = {
958 &dev_attr_comp_complete.attr,
959 &dev_attr_comp_failed.attr,
960 &dev_attr_decomp_complete.attr,
961 &dev_attr_decomp_failed.attr,
962 &dev_attr_swdecomp.attr,
963 &dev_attr_comp_times.attr,
964 &dev_attr_decomp_times.attr,
965 NULL,
966 };
967
968 static const struct attribute_group nx842_attribute_group = {
969 .name = NULL, /* put in device directory */
970 .attrs = nx842_sysfs_entries,
971 };
972
973 #define nxcop_caps_read(_name) \
974 static ssize_t nxcop_##_name##_show(struct device *dev, \
975 struct device_attribute *attr, char *buf) \
976 { \
977 return sprintf(buf, "%lld\n", nx_cop_caps._name); \
978 }
979
980 #define NXCT_ATTR_RO(_name) \
981 nxcop_caps_read(_name); \
982 static struct device_attribute dev_attr_##_name = __ATTR(_name, \
983 0444, \
984 nxcop_##_name##_show, \
985 NULL);
986
987 NXCT_ATTR_RO(req_max_processed_len);
988 NXCT_ATTR_RO(min_compress_len);
989 NXCT_ATTR_RO(min_decompress_len);
990
991 static struct attribute *nxcop_caps_sysfs_entries[] = {
992 &dev_attr_req_max_processed_len.attr,
993 &dev_attr_min_compress_len.attr,
994 &dev_attr_min_decompress_len.attr,
995 NULL,
996 };
997
998 static const struct attribute_group nxcop_caps_attr_group = {
999 .name = "nx_gzip_caps",
1000 .attrs = nxcop_caps_sysfs_entries,
1001 };
1002
1003 static struct nx842_driver nx842_pseries_driver = {
1004 .name = KBUILD_MODNAME,
1005 .owner = THIS_MODULE,
1006 .workmem_size = sizeof(struct nx842_workmem),
1007 .constraints = &nx842_pseries_constraints,
1008 .compress = nx842_pseries_compress,
1009 .decompress = nx842_pseries_decompress,
1010 };
1011
nx842_pseries_crypto_alloc_ctx(void)1012 static void *nx842_pseries_crypto_alloc_ctx(void)
1013 {
1014 return nx842_crypto_alloc_ctx(&nx842_pseries_driver);
1015 }
1016
1017 static struct scomp_alg nx842_pseries_alg = {
1018 .base.cra_name = "842",
1019 .base.cra_driver_name = "842-nx",
1020 .base.cra_priority = 300,
1021 .base.cra_module = THIS_MODULE,
1022
1023 .alloc_ctx = nx842_pseries_crypto_alloc_ctx,
1024 .free_ctx = nx842_crypto_free_ctx,
1025 .compress = nx842_crypto_compress,
1026 .decompress = nx842_crypto_decompress,
1027 };
1028
nx842_probe(struct vio_dev * viodev,const struct vio_device_id * id)1029 static int nx842_probe(struct vio_dev *viodev,
1030 const struct vio_device_id *id)
1031 {
1032 struct nx842_devdata *old_devdata, *new_devdata = NULL;
1033 unsigned long flags;
1034 int ret = 0;
1035
1036 new_devdata = kzalloc(sizeof(*new_devdata), GFP_NOFS);
1037 if (!new_devdata)
1038 return -ENOMEM;
1039
1040 new_devdata->counters = kzalloc(sizeof(*new_devdata->counters),
1041 GFP_NOFS);
1042 if (!new_devdata->counters) {
1043 kfree(new_devdata);
1044 return -ENOMEM;
1045 }
1046
1047 spin_lock_irqsave(&devdata_spinlock, flags);
1048 old_devdata = rcu_dereference_check(devdata,
1049 lockdep_is_held(&devdata_spinlock));
1050
1051 if (old_devdata && old_devdata->vdev != NULL) {
1052 dev_err(&viodev->dev, "%s: Attempt to register more than one instance of the hardware\n", __func__);
1053 ret = -1;
1054 goto error_unlock;
1055 }
1056
1057 dev_set_drvdata(&viodev->dev, NULL);
1058
1059 new_devdata->vdev = viodev;
1060 new_devdata->dev = &viodev->dev;
1061 nx842_OF_set_defaults(new_devdata);
1062
1063 rcu_assign_pointer(devdata, new_devdata);
1064 spin_unlock_irqrestore(&devdata_spinlock, flags);
1065 synchronize_rcu();
1066 kfree(old_devdata);
1067
1068 of_reconfig_notifier_register(&nx842_of_nb);
1069
1070 ret = nx842_OF_upd(NULL);
1071 if (ret)
1072 goto error;
1073
1074 ret = crypto_register_scomp(&nx842_pseries_alg);
1075 if (ret) {
1076 dev_err(&viodev->dev, "could not register comp alg: %d\n", ret);
1077 goto error;
1078 }
1079
1080 rcu_read_lock();
1081 dev_set_drvdata(&viodev->dev, rcu_dereference(devdata));
1082 rcu_read_unlock();
1083
1084 if (sysfs_create_group(&viodev->dev.kobj, &nx842_attribute_group)) {
1085 dev_err(&viodev->dev, "could not create sysfs device attributes\n");
1086 ret = -1;
1087 goto error;
1088 }
1089
1090 if (caps_feat) {
1091 if (sysfs_create_group(&viodev->dev.kobj,
1092 &nxcop_caps_attr_group)) {
1093 dev_err(&viodev->dev,
1094 "Could not create sysfs NX capability entries\n");
1095 ret = -1;
1096 goto error;
1097 }
1098 }
1099
1100 return 0;
1101
1102 error_unlock:
1103 spin_unlock_irqrestore(&devdata_spinlock, flags);
1104 if (new_devdata)
1105 kfree(new_devdata->counters);
1106 kfree(new_devdata);
1107 error:
1108 return ret;
1109 }
1110
nx842_remove(struct vio_dev * viodev)1111 static void nx842_remove(struct vio_dev *viodev)
1112 {
1113 struct nx842_devdata *old_devdata;
1114 unsigned long flags;
1115
1116 pr_info("Removing IBM Power 842 compression device\n");
1117 sysfs_remove_group(&viodev->dev.kobj, &nx842_attribute_group);
1118
1119 if (caps_feat)
1120 sysfs_remove_group(&viodev->dev.kobj, &nxcop_caps_attr_group);
1121
1122 crypto_unregister_scomp(&nx842_pseries_alg);
1123
1124 of_reconfig_notifier_unregister(&nx842_of_nb);
1125
1126 spin_lock_irqsave(&devdata_spinlock, flags);
1127 old_devdata = rcu_dereference_check(devdata,
1128 lockdep_is_held(&devdata_spinlock));
1129 RCU_INIT_POINTER(devdata, NULL);
1130 spin_unlock_irqrestore(&devdata_spinlock, flags);
1131 synchronize_rcu();
1132 dev_set_drvdata(&viodev->dev, NULL);
1133 if (old_devdata)
1134 kfree(old_devdata->counters);
1135 kfree(old_devdata);
1136 }
1137
1138 /*
1139 * Get NX capabilities from the hypervisor.
1140 * Only NXGZIP capabilities are provided by the hypersvisor right
1141 * now and these values are available to user space with sysfs.
1142 */
nxcop_get_capabilities(void)1143 static void __init nxcop_get_capabilities(void)
1144 {
1145 struct hv_vas_all_caps *hv_caps;
1146 struct hv_nx_cop_caps *hv_nxc;
1147 u64 feat;
1148 int rc;
1149
1150 hv_caps = kmalloc(sizeof(*hv_caps), GFP_KERNEL);
1151 if (!hv_caps)
1152 return;
1153 /*
1154 * Get NX overall capabilities with feature type=0
1155 */
1156 rc = h_query_vas_capabilities(H_QUERY_NX_CAPABILITIES, 0,
1157 (u64)virt_to_phys(hv_caps));
1158 if (!rc)
1159 feat = be64_to_cpu(hv_caps->feat_type);
1160 kfree(hv_caps);
1161 if (rc)
1162 return;
1163 if (!(feat & VAS_NX_GZIP_FEAT_BIT))
1164 return;
1165
1166 /*
1167 * NX-GZIP feature available
1168 */
1169 hv_nxc = kmalloc(sizeof(*hv_nxc), GFP_KERNEL);
1170 if (!hv_nxc)
1171 return;
1172 /*
1173 * Get capabilities for NX-GZIP feature
1174 */
1175 rc = h_query_vas_capabilities(H_QUERY_NX_CAPABILITIES,
1176 VAS_NX_GZIP_FEAT,
1177 (u64)virt_to_phys(hv_nxc));
1178
1179 if (!rc) {
1180 nx_cop_caps.descriptor = be64_to_cpu(hv_nxc->descriptor);
1181 nx_cop_caps.req_max_processed_len =
1182 be64_to_cpu(hv_nxc->req_max_processed_len);
1183 nx_cop_caps.min_compress_len =
1184 be64_to_cpu(hv_nxc->min_compress_len);
1185 nx_cop_caps.min_decompress_len =
1186 be64_to_cpu(hv_nxc->min_decompress_len);
1187 caps_feat = feat;
1188 }
1189
1190 kfree(hv_nxc);
1191 }
1192
1193 static const struct vio_device_id nx842_vio_driver_ids[] = {
1194 {"ibm,compression-v1", "ibm,compression"},
1195 {"", ""},
1196 };
1197 MODULE_DEVICE_TABLE(vio, nx842_vio_driver_ids);
1198
1199 static struct vio_driver nx842_vio_driver = {
1200 .name = KBUILD_MODNAME,
1201 .probe = nx842_probe,
1202 .remove = nx842_remove,
1203 .get_desired_dma = nx842_get_desired_dma,
1204 .id_table = nx842_vio_driver_ids,
1205 };
1206
nx842_pseries_init(void)1207 static int __init nx842_pseries_init(void)
1208 {
1209 struct nx842_devdata *new_devdata;
1210 struct device_node *np;
1211 int ret;
1212
1213 np = of_find_compatible_node(NULL, NULL, "ibm,compression");
1214 if (!np)
1215 return -ENODEV;
1216 of_node_put(np);
1217
1218 RCU_INIT_POINTER(devdata, NULL);
1219 new_devdata = kzalloc(sizeof(*new_devdata), GFP_KERNEL);
1220 if (!new_devdata)
1221 return -ENOMEM;
1222
1223 RCU_INIT_POINTER(devdata, new_devdata);
1224 /*
1225 * Get NX capabilities from the hypervisor.
1226 */
1227 nxcop_get_capabilities();
1228
1229 ret = vio_register_driver(&nx842_vio_driver);
1230 if (ret) {
1231 pr_err("Could not register VIO driver %d\n", ret);
1232
1233 kfree(new_devdata);
1234 return ret;
1235 }
1236
1237 ret = vas_register_api_pseries(THIS_MODULE, VAS_COP_TYPE_GZIP,
1238 "nx-gzip");
1239
1240 if (ret)
1241 pr_err("NX-GZIP is not supported. Returned=%d\n", ret);
1242
1243 return 0;
1244 }
1245
1246 module_init(nx842_pseries_init);
1247
nx842_pseries_exit(void)1248 static void __exit nx842_pseries_exit(void)
1249 {
1250 struct nx842_devdata *old_devdata;
1251 unsigned long flags;
1252
1253 vas_unregister_api_pseries();
1254
1255 crypto_unregister_scomp(&nx842_pseries_alg);
1256
1257 spin_lock_irqsave(&devdata_spinlock, flags);
1258 old_devdata = rcu_dereference_check(devdata,
1259 lockdep_is_held(&devdata_spinlock));
1260 RCU_INIT_POINTER(devdata, NULL);
1261 spin_unlock_irqrestore(&devdata_spinlock, flags);
1262 synchronize_rcu();
1263 if (old_devdata && old_devdata->dev)
1264 dev_set_drvdata(old_devdata->dev, NULL);
1265 kfree(old_devdata);
1266 vio_unregister_driver(&nx842_vio_driver);
1267 }
1268
1269 module_exit(nx842_pseries_exit);
1270
1271