1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright 2017 ATMEL
4 * Copyright 2017 Free Electrons
5 *
6 * Author: Boris Brezillon <boris.brezillon@free-electrons.com>
7 *
8 * Derived from the atmel_nand.c driver which contained the following
9 * copyrights:
10 *
11 * Copyright 2003 Rick Bronson
12 *
13 * Derived from drivers/mtd/nand/autcpu12.c (removed in v3.8)
14 * Copyright 2001 Thomas Gleixner (gleixner@autronix.de)
15 *
16 * Derived from drivers/mtd/spia.c (removed in v3.8)
17 * Copyright 2000 Steven J. Hill (sjhill@cotw.com)
18 *
19 * Add Hardware ECC support for AT91SAM9260 / AT91SAM9263
20 * Richard Genoud (richard.genoud@gmail.com), Adeneo Copyright 2007
21 *
22 * Derived from Das U-Boot source code
23 * (u-boot-1.1.5/board/atmel/at91sam9263ek/nand.c)
24 * Copyright 2006 ATMEL Rousset, Lacressonniere Nicolas
25 *
26 * Add Programmable Multibit ECC support for various AT91 SoC
27 * Copyright 2012 ATMEL, Hong Xu
28 *
29 * Add Nand Flash Controller support for SAMA5 SoC
30 * Copyright 2013 ATMEL, Josh Wu (josh.wu@atmel.com)
31 *
32 * The PMECC is an hardware assisted BCH engine, which means part of the
33 * ECC algorithm is left to the software. The hardware/software repartition
34 * is explained in the "PMECC Controller Functional Description" chapter in
35 * Atmel datasheets, and some of the functions in this file are directly
36 * implementing the algorithms described in the "Software Implementation"
37 * sub-section.
38 *
39 * TODO: it seems that the software BCH implementation in lib/bch.c is already
40 * providing some of the logic we are implementing here. It would be smart
41 * to expose the needed lib/bch.c helpers/functions and re-use them here.
42 */
43
44 #include <linux/genalloc.h>
45 #include <linux/iopoll.h>
46 #include <linux/module.h>
47 #include <linux/mtd/rawnand.h>
48 #include <linux/of_irq.h>
49 #include <linux/of_platform.h>
50 #include <linux/platform_device.h>
51 #include <linux/slab.h>
52
53 #include "pmecc.h"
54
55 /* Galois field dimension */
56 #define PMECC_GF_DIMENSION_13 13
57 #define PMECC_GF_DIMENSION_14 14
58
59 /* Primitive Polynomial used by PMECC */
60 #define PMECC_GF_13_PRIMITIVE_POLY 0x201b
61 #define PMECC_GF_14_PRIMITIVE_POLY 0x4443
62
63 #define PMECC_LOOKUP_TABLE_SIZE_512 0x2000
64 #define PMECC_LOOKUP_TABLE_SIZE_1024 0x4000
65
66 /* Time out value for reading PMECC status register */
67 #define PMECC_MAX_TIMEOUT_MS 100
68
69 /* PMECC Register Definitions */
70 #define ATMEL_PMECC_CFG 0x0
71 #define PMECC_CFG_BCH_STRENGTH(x) (x)
72 #define PMECC_CFG_BCH_STRENGTH_MASK GENMASK(2, 0)
73 #define PMECC_CFG_SECTOR512 (0 << 4)
74 #define PMECC_CFG_SECTOR1024 (1 << 4)
75 #define PMECC_CFG_NSECTORS(x) ((fls(x) - 1) << 8)
76 #define PMECC_CFG_READ_OP (0 << 12)
77 #define PMECC_CFG_WRITE_OP (1 << 12)
78 #define PMECC_CFG_SPARE_ENABLE BIT(16)
79 #define PMECC_CFG_AUTO_ENABLE BIT(20)
80
81 #define ATMEL_PMECC_SAREA 0x4
82 #define ATMEL_PMECC_SADDR 0x8
83 #define ATMEL_PMECC_EADDR 0xc
84
85 #define ATMEL_PMECC_CLK 0x10
86 #define PMECC_CLK_133MHZ (2 << 0)
87
88 #define ATMEL_PMECC_CTRL 0x14
89 #define PMECC_CTRL_RST BIT(0)
90 #define PMECC_CTRL_DATA BIT(1)
91 #define PMECC_CTRL_USER BIT(2)
92 #define PMECC_CTRL_ENABLE BIT(4)
93 #define PMECC_CTRL_DISABLE BIT(5)
94
95 #define ATMEL_PMECC_SR 0x18
96 #define PMECC_SR_BUSY BIT(0)
97 #define PMECC_SR_ENABLE BIT(4)
98
99 #define ATMEL_PMECC_IER 0x1c
100 #define ATMEL_PMECC_IDR 0x20
101 #define ATMEL_PMECC_IMR 0x24
102 #define ATMEL_PMECC_ISR 0x28
103 #define PMECC_ERROR_INT BIT(0)
104
105 #define ATMEL_PMECC_ECC(sector, n) \
106 ((((sector) + 1) * 0x40) + (n))
107
108 #define ATMEL_PMECC_REM(sector, n) \
109 ((((sector) + 1) * 0x40) + ((n) * 4) + 0x200)
110
111 /* PMERRLOC Register Definitions */
112 #define ATMEL_PMERRLOC_ELCFG 0x0
113 #define PMERRLOC_ELCFG_SECTOR_512 (0 << 0)
114 #define PMERRLOC_ELCFG_SECTOR_1024 (1 << 0)
115 #define PMERRLOC_ELCFG_NUM_ERRORS(n) ((n) << 16)
116
117 #define ATMEL_PMERRLOC_ELPRIM 0x4
118 #define ATMEL_PMERRLOC_ELEN 0x8
119 #define ATMEL_PMERRLOC_ELDIS 0xc
120 #define PMERRLOC_DISABLE BIT(0)
121
122 #define ATMEL_PMERRLOC_ELSR 0x10
123 #define PMERRLOC_ELSR_BUSY BIT(0)
124
125 #define ATMEL_PMERRLOC_ELIER 0x14
126 #define ATMEL_PMERRLOC_ELIDR 0x18
127 #define ATMEL_PMERRLOC_ELIMR 0x1c
128 #define ATMEL_PMERRLOC_ELISR 0x20
129 #define PMERRLOC_ERR_NUM_MASK GENMASK(12, 8)
130 #define PMERRLOC_CALC_DONE BIT(0)
131
132 #define ATMEL_PMERRLOC_SIGMA(x) (((x) * 0x4) + 0x28)
133
134 #define ATMEL_PMERRLOC_EL(offs, x) (((x) * 0x4) + (offs))
135
136 struct atmel_pmecc_gf_tables {
137 u16 *alpha_to;
138 u16 *index_of;
139 };
140
141 struct atmel_pmecc_caps {
142 const int *strengths;
143 int nstrengths;
144 int el_offset;
145 bool correct_erased_chunks;
146 bool clk_ctrl;
147 };
148
149 struct atmel_pmecc {
150 struct device *dev;
151 const struct atmel_pmecc_caps *caps;
152
153 struct {
154 void __iomem *base;
155 void __iomem *errloc;
156 } regs;
157
158 struct mutex lock;
159 };
160
161 struct atmel_pmecc_user_conf_cache {
162 u32 cfg;
163 u32 sarea;
164 u32 saddr;
165 u32 eaddr;
166 };
167
168 struct atmel_pmecc_user {
169 struct atmel_pmecc_user_conf_cache cache;
170 struct atmel_pmecc *pmecc;
171 const struct atmel_pmecc_gf_tables *gf_tables;
172 int eccbytes;
173 s16 *partial_syn;
174 s16 *si;
175 s16 *lmu;
176 s16 *smu;
177 s32 *mu;
178 s32 *dmu;
179 s32 *delta;
180 u32 isr;
181 };
182
183 static DEFINE_MUTEX(pmecc_gf_tables_lock);
184 static const struct atmel_pmecc_gf_tables *pmecc_gf_tables_512;
185 static const struct atmel_pmecc_gf_tables *pmecc_gf_tables_1024;
186
deg(unsigned int poly)187 static inline int deg(unsigned int poly)
188 {
189 /* polynomial degree is the most-significant bit index */
190 return fls(poly) - 1;
191 }
192
atmel_pmecc_build_gf_tables(int mm,unsigned int poly,struct atmel_pmecc_gf_tables * gf_tables)193 static int atmel_pmecc_build_gf_tables(int mm, unsigned int poly,
194 struct atmel_pmecc_gf_tables *gf_tables)
195 {
196 unsigned int i, x = 1;
197 const unsigned int k = BIT(deg(poly));
198 unsigned int nn = BIT(mm) - 1;
199
200 /* primitive polynomial must be of degree m */
201 if (k != (1u << mm))
202 return -EINVAL;
203
204 for (i = 0; i < nn; i++) {
205 gf_tables->alpha_to[i] = x;
206 gf_tables->index_of[x] = i;
207 if (i && (x == 1))
208 /* polynomial is not primitive (a^i=1 with 0<i<2^m-1) */
209 return -EINVAL;
210 x <<= 1;
211 if (x & k)
212 x ^= poly;
213 }
214 gf_tables->alpha_to[nn] = 1;
215 gf_tables->index_of[0] = 0;
216
217 return 0;
218 }
219
220 static const struct atmel_pmecc_gf_tables *
atmel_pmecc_create_gf_tables(const struct atmel_pmecc_user_req * req)221 atmel_pmecc_create_gf_tables(const struct atmel_pmecc_user_req *req)
222 {
223 struct atmel_pmecc_gf_tables *gf_tables;
224 unsigned int poly, degree, table_size;
225 int ret;
226
227 if (req->ecc.sectorsize == 512) {
228 degree = PMECC_GF_DIMENSION_13;
229 poly = PMECC_GF_13_PRIMITIVE_POLY;
230 table_size = PMECC_LOOKUP_TABLE_SIZE_512;
231 } else {
232 degree = PMECC_GF_DIMENSION_14;
233 poly = PMECC_GF_14_PRIMITIVE_POLY;
234 table_size = PMECC_LOOKUP_TABLE_SIZE_1024;
235 }
236
237 gf_tables = kzalloc(sizeof(*gf_tables) +
238 (2 * table_size * sizeof(u16)),
239 GFP_KERNEL);
240 if (!gf_tables)
241 return ERR_PTR(-ENOMEM);
242
243 gf_tables->alpha_to = (void *)(gf_tables + 1);
244 gf_tables->index_of = gf_tables->alpha_to + table_size;
245
246 ret = atmel_pmecc_build_gf_tables(degree, poly, gf_tables);
247 if (ret) {
248 kfree(gf_tables);
249 return ERR_PTR(ret);
250 }
251
252 return gf_tables;
253 }
254
255 static const struct atmel_pmecc_gf_tables *
atmel_pmecc_get_gf_tables(const struct atmel_pmecc_user_req * req)256 atmel_pmecc_get_gf_tables(const struct atmel_pmecc_user_req *req)
257 {
258 const struct atmel_pmecc_gf_tables **gf_tables, *ret;
259
260 mutex_lock(&pmecc_gf_tables_lock);
261 if (req->ecc.sectorsize == 512)
262 gf_tables = &pmecc_gf_tables_512;
263 else
264 gf_tables = &pmecc_gf_tables_1024;
265
266 ret = *gf_tables;
267
268 if (!ret) {
269 ret = atmel_pmecc_create_gf_tables(req);
270 if (!IS_ERR(ret))
271 *gf_tables = ret;
272 }
273 mutex_unlock(&pmecc_gf_tables_lock);
274
275 return ret;
276 }
277
atmel_pmecc_prepare_user_req(struct atmel_pmecc * pmecc,struct atmel_pmecc_user_req * req)278 static int atmel_pmecc_prepare_user_req(struct atmel_pmecc *pmecc,
279 struct atmel_pmecc_user_req *req)
280 {
281 int i, max_eccbytes, eccbytes = 0, eccstrength = 0;
282
283 if (req->pagesize <= 0 || req->oobsize <= 0 || req->ecc.bytes <= 0)
284 return -EINVAL;
285
286 if (req->ecc.ooboffset >= 0 &&
287 req->ecc.ooboffset + req->ecc.bytes > req->oobsize)
288 return -EINVAL;
289
290 if (req->ecc.sectorsize == ATMEL_PMECC_SECTOR_SIZE_AUTO) {
291 if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH)
292 return -EINVAL;
293
294 if (req->pagesize > 512)
295 req->ecc.sectorsize = 1024;
296 else
297 req->ecc.sectorsize = 512;
298 }
299
300 if (req->ecc.sectorsize != 512 && req->ecc.sectorsize != 1024)
301 return -EINVAL;
302
303 if (req->pagesize % req->ecc.sectorsize)
304 return -EINVAL;
305
306 req->ecc.nsectors = req->pagesize / req->ecc.sectorsize;
307
308 max_eccbytes = req->ecc.bytes;
309
310 for (i = 0; i < pmecc->caps->nstrengths; i++) {
311 int nbytes, strength = pmecc->caps->strengths[i];
312
313 if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH &&
314 strength < req->ecc.strength)
315 continue;
316
317 nbytes = DIV_ROUND_UP(strength * fls(8 * req->ecc.sectorsize),
318 8);
319 nbytes *= req->ecc.nsectors;
320
321 if (nbytes > max_eccbytes)
322 break;
323
324 eccstrength = strength;
325 eccbytes = nbytes;
326
327 if (req->ecc.strength != ATMEL_PMECC_MAXIMIZE_ECC_STRENGTH)
328 break;
329 }
330
331 if (!eccstrength)
332 return -EINVAL;
333
334 req->ecc.bytes = eccbytes;
335 req->ecc.strength = eccstrength;
336
337 if (req->ecc.ooboffset < 0)
338 req->ecc.ooboffset = req->oobsize - eccbytes;
339
340 return 0;
341 }
342
343 struct atmel_pmecc_user *
atmel_pmecc_create_user(struct atmel_pmecc * pmecc,struct atmel_pmecc_user_req * req)344 atmel_pmecc_create_user(struct atmel_pmecc *pmecc,
345 struct atmel_pmecc_user_req *req)
346 {
347 struct atmel_pmecc_user *user;
348 const struct atmel_pmecc_gf_tables *gf_tables;
349 int strength, size, ret;
350
351 ret = atmel_pmecc_prepare_user_req(pmecc, req);
352 if (ret)
353 return ERR_PTR(ret);
354
355 size = sizeof(*user);
356 size = ALIGN(size, sizeof(u16));
357 /* Reserve space for partial_syn, si and smu */
358 size += ((2 * req->ecc.strength) + 1) * sizeof(u16) *
359 (2 + req->ecc.strength + 2);
360 /* Reserve space for lmu. */
361 size += (req->ecc.strength + 1) * sizeof(u16);
362 /* Reserve space for mu, dmu and delta. */
363 size = ALIGN(size, sizeof(s32));
364 size += (req->ecc.strength + 1) * sizeof(s32) * 3;
365
366 user = devm_kzalloc(pmecc->dev, size, GFP_KERNEL);
367 if (!user)
368 return ERR_PTR(-ENOMEM);
369
370 user->pmecc = pmecc;
371
372 user->partial_syn = (s16 *)PTR_ALIGN(user + 1, sizeof(u16));
373 user->si = user->partial_syn + ((2 * req->ecc.strength) + 1);
374 user->lmu = user->si + ((2 * req->ecc.strength) + 1);
375 user->smu = user->lmu + (req->ecc.strength + 1);
376 user->mu = (s32 *)PTR_ALIGN(user->smu +
377 (((2 * req->ecc.strength) + 1) *
378 (req->ecc.strength + 2)),
379 sizeof(s32));
380 user->dmu = user->mu + req->ecc.strength + 1;
381 user->delta = user->dmu + req->ecc.strength + 1;
382
383 gf_tables = atmel_pmecc_get_gf_tables(req);
384 if (IS_ERR(gf_tables))
385 return ERR_CAST(gf_tables);
386
387 user->gf_tables = gf_tables;
388
389 user->eccbytes = req->ecc.bytes / req->ecc.nsectors;
390
391 for (strength = 0; strength < pmecc->caps->nstrengths; strength++) {
392 if (pmecc->caps->strengths[strength] == req->ecc.strength)
393 break;
394 }
395
396 user->cache.cfg = PMECC_CFG_BCH_STRENGTH(strength) |
397 PMECC_CFG_NSECTORS(req->ecc.nsectors);
398
399 if (req->ecc.sectorsize == 1024)
400 user->cache.cfg |= PMECC_CFG_SECTOR1024;
401
402 user->cache.sarea = req->oobsize - 1;
403 user->cache.saddr = req->ecc.ooboffset;
404 user->cache.eaddr = req->ecc.ooboffset + req->ecc.bytes - 1;
405
406 return user;
407 }
408 EXPORT_SYMBOL_GPL(atmel_pmecc_create_user);
409
get_strength(struct atmel_pmecc_user * user)410 static int get_strength(struct atmel_pmecc_user *user)
411 {
412 const int *strengths = user->pmecc->caps->strengths;
413
414 return strengths[user->cache.cfg & PMECC_CFG_BCH_STRENGTH_MASK];
415 }
416
get_sectorsize(struct atmel_pmecc_user * user)417 static int get_sectorsize(struct atmel_pmecc_user *user)
418 {
419 return user->cache.cfg & PMECC_CFG_SECTOR1024 ? 1024 : 512;
420 }
421
atmel_pmecc_gen_syndrome(struct atmel_pmecc_user * user,int sector)422 static void atmel_pmecc_gen_syndrome(struct atmel_pmecc_user *user, int sector)
423 {
424 int strength = get_strength(user);
425 u32 value;
426 int i;
427
428 /* Fill odd syndromes */
429 for (i = 0; i < strength; i++) {
430 value = readl_relaxed(user->pmecc->regs.base +
431 ATMEL_PMECC_REM(sector, i / 2));
432 if (i & 1)
433 value >>= 16;
434
435 user->partial_syn[(2 * i) + 1] = value;
436 }
437 }
438
atmel_pmecc_substitute(struct atmel_pmecc_user * user)439 static void atmel_pmecc_substitute(struct atmel_pmecc_user *user)
440 {
441 int degree = get_sectorsize(user) == 512 ? 13 : 14;
442 int cw_len = BIT(degree) - 1;
443 int strength = get_strength(user);
444 s16 *alpha_to = user->gf_tables->alpha_to;
445 s16 *index_of = user->gf_tables->index_of;
446 s16 *partial_syn = user->partial_syn;
447 s16 *si;
448 int i, j;
449
450 /*
451 * si[] is a table that holds the current syndrome value,
452 * an element of that table belongs to the field
453 */
454 si = user->si;
455
456 memset(&si[1], 0, sizeof(s16) * ((2 * strength) - 1));
457
458 /* Computation 2t syndromes based on S(x) */
459 /* Odd syndromes */
460 for (i = 1; i < 2 * strength; i += 2) {
461 for (j = 0; j < degree; j++) {
462 if (partial_syn[i] & BIT(j))
463 si[i] = alpha_to[i * j] ^ si[i];
464 }
465 }
466 /* Even syndrome = (Odd syndrome) ** 2 */
467 for (i = 2, j = 1; j <= strength; i = ++j << 1) {
468 if (si[j] == 0) {
469 si[i] = 0;
470 } else {
471 s16 tmp;
472
473 tmp = index_of[si[j]];
474 tmp = (tmp * 2) % cw_len;
475 si[i] = alpha_to[tmp];
476 }
477 }
478 }
479
atmel_pmecc_get_sigma(struct atmel_pmecc_user * user)480 static void atmel_pmecc_get_sigma(struct atmel_pmecc_user *user)
481 {
482 s16 *lmu = user->lmu;
483 s16 *si = user->si;
484 s32 *mu = user->mu;
485 s32 *dmu = user->dmu;
486 s32 *delta = user->delta;
487 int degree = get_sectorsize(user) == 512 ? 13 : 14;
488 int cw_len = BIT(degree) - 1;
489 int strength = get_strength(user);
490 int num = 2 * strength + 1;
491 s16 *index_of = user->gf_tables->index_of;
492 s16 *alpha_to = user->gf_tables->alpha_to;
493 int i, j, k;
494 u32 dmu_0_count, tmp;
495 s16 *smu = user->smu;
496
497 /* index of largest delta */
498 int ro;
499 int largest;
500 int diff;
501
502 dmu_0_count = 0;
503
504 /* First Row */
505
506 /* Mu */
507 mu[0] = -1;
508
509 memset(smu, 0, sizeof(s16) * num);
510 smu[0] = 1;
511
512 /* discrepancy set to 1 */
513 dmu[0] = 1;
514 /* polynom order set to 0 */
515 lmu[0] = 0;
516 delta[0] = (mu[0] * 2 - lmu[0]) >> 1;
517
518 /* Second Row */
519
520 /* Mu */
521 mu[1] = 0;
522 /* Sigma(x) set to 1 */
523 memset(&smu[num], 0, sizeof(s16) * num);
524 smu[num] = 1;
525
526 /* discrepancy set to S1 */
527 dmu[1] = si[1];
528
529 /* polynom order set to 0 */
530 lmu[1] = 0;
531
532 delta[1] = (mu[1] * 2 - lmu[1]) >> 1;
533
534 /* Init the Sigma(x) last row */
535 memset(&smu[(strength + 1) * num], 0, sizeof(s16) * num);
536
537 for (i = 1; i <= strength; i++) {
538 mu[i + 1] = i << 1;
539 /* Begin Computing Sigma (Mu+1) and L(mu) */
540 /* check if discrepancy is set to 0 */
541 if (dmu[i] == 0) {
542 dmu_0_count++;
543
544 tmp = ((strength - (lmu[i] >> 1) - 1) / 2);
545 if ((strength - (lmu[i] >> 1) - 1) & 0x1)
546 tmp += 2;
547 else
548 tmp += 1;
549
550 if (dmu_0_count == tmp) {
551 for (j = 0; j <= (lmu[i] >> 1) + 1; j++)
552 smu[(strength + 1) * num + j] =
553 smu[i * num + j];
554
555 lmu[strength + 1] = lmu[i];
556 return;
557 }
558
559 /* copy polynom */
560 for (j = 0; j <= lmu[i] >> 1; j++)
561 smu[(i + 1) * num + j] = smu[i * num + j];
562
563 /* copy previous polynom order to the next */
564 lmu[i + 1] = lmu[i];
565 } else {
566 ro = 0;
567 largest = -1;
568 /* find largest delta with dmu != 0 */
569 for (j = 0; j < i; j++) {
570 if ((dmu[j]) && (delta[j] > largest)) {
571 largest = delta[j];
572 ro = j;
573 }
574 }
575
576 /* compute difference */
577 diff = (mu[i] - mu[ro]);
578
579 /* Compute degree of the new smu polynomial */
580 if ((lmu[i] >> 1) > ((lmu[ro] >> 1) + diff))
581 lmu[i + 1] = lmu[i];
582 else
583 lmu[i + 1] = ((lmu[ro] >> 1) + diff) * 2;
584
585 /* Init smu[i+1] with 0 */
586 for (k = 0; k < num; k++)
587 smu[(i + 1) * num + k] = 0;
588
589 /* Compute smu[i+1] */
590 for (k = 0; k <= lmu[ro] >> 1; k++) {
591 s16 a, b, c;
592
593 if (!(smu[ro * num + k] && dmu[i]))
594 continue;
595
596 a = index_of[dmu[i]];
597 b = index_of[dmu[ro]];
598 c = index_of[smu[ro * num + k]];
599 tmp = a + (cw_len - b) + c;
600 a = alpha_to[tmp % cw_len];
601 smu[(i + 1) * num + (k + diff)] = a;
602 }
603
604 for (k = 0; k <= lmu[i] >> 1; k++)
605 smu[(i + 1) * num + k] ^= smu[i * num + k];
606 }
607
608 /* End Computing Sigma (Mu+1) and L(mu) */
609 /* In either case compute delta */
610 delta[i + 1] = (mu[i + 1] * 2 - lmu[i + 1]) >> 1;
611
612 /* Do not compute discrepancy for the last iteration */
613 if (i >= strength)
614 continue;
615
616 for (k = 0; k <= (lmu[i + 1] >> 1); k++) {
617 tmp = 2 * (i - 1);
618 if (k == 0) {
619 dmu[i + 1] = si[tmp + 3];
620 } else if (smu[(i + 1) * num + k] && si[tmp + 3 - k]) {
621 s16 a, b, c;
622
623 a = index_of[smu[(i + 1) * num + k]];
624 b = si[2 * (i - 1) + 3 - k];
625 c = index_of[b];
626 tmp = a + c;
627 tmp %= cw_len;
628 dmu[i + 1] = alpha_to[tmp] ^ dmu[i + 1];
629 }
630 }
631 }
632 }
633
atmel_pmecc_err_location(struct atmel_pmecc_user * user)634 static int atmel_pmecc_err_location(struct atmel_pmecc_user *user)
635 {
636 int sector_size = get_sectorsize(user);
637 int degree = sector_size == 512 ? 13 : 14;
638 struct atmel_pmecc *pmecc = user->pmecc;
639 int strength = get_strength(user);
640 int ret, roots_nbr, i, err_nbr = 0;
641 int num = (2 * strength) + 1;
642 s16 *smu = user->smu;
643 u32 val;
644
645 writel(PMERRLOC_DISABLE, pmecc->regs.errloc + ATMEL_PMERRLOC_ELDIS);
646
647 for (i = 0; i <= user->lmu[strength + 1] >> 1; i++) {
648 writel_relaxed(smu[(strength + 1) * num + i],
649 pmecc->regs.errloc + ATMEL_PMERRLOC_SIGMA(i));
650 err_nbr++;
651 }
652
653 val = (err_nbr - 1) << 16;
654 if (sector_size == 1024)
655 val |= 1;
656
657 writel(val, pmecc->regs.errloc + ATMEL_PMERRLOC_ELCFG);
658 writel((sector_size * 8) + (degree * strength),
659 pmecc->regs.errloc + ATMEL_PMERRLOC_ELEN);
660
661 ret = readl_relaxed_poll_timeout(pmecc->regs.errloc +
662 ATMEL_PMERRLOC_ELISR,
663 val, val & PMERRLOC_CALC_DONE, 0,
664 PMECC_MAX_TIMEOUT_MS * 1000);
665 if (ret) {
666 dev_err(pmecc->dev,
667 "PMECC: Timeout to calculate error location.\n");
668 return ret;
669 }
670
671 roots_nbr = (val & PMERRLOC_ERR_NUM_MASK) >> 8;
672 /* Number of roots == degree of smu hence <= cap */
673 if (roots_nbr == user->lmu[strength + 1] >> 1)
674 return err_nbr - 1;
675
676 /*
677 * Number of roots does not match the degree of smu
678 * unable to correct error.
679 */
680 return -EBADMSG;
681 }
682
atmel_pmecc_correct_sector(struct atmel_pmecc_user * user,int sector,void * data,void * ecc)683 int atmel_pmecc_correct_sector(struct atmel_pmecc_user *user, int sector,
684 void *data, void *ecc)
685 {
686 struct atmel_pmecc *pmecc = user->pmecc;
687 int sectorsize = get_sectorsize(user);
688 int eccbytes = user->eccbytes;
689 int i, nerrors;
690
691 if (!(user->isr & BIT(sector)))
692 return 0;
693
694 atmel_pmecc_gen_syndrome(user, sector);
695 atmel_pmecc_substitute(user);
696 atmel_pmecc_get_sigma(user);
697
698 nerrors = atmel_pmecc_err_location(user);
699 if (nerrors < 0)
700 return nerrors;
701
702 for (i = 0; i < nerrors; i++) {
703 const char *area;
704 int byte, bit;
705 u32 errpos;
706 u8 *ptr;
707
708 errpos = readl_relaxed(pmecc->regs.errloc +
709 ATMEL_PMERRLOC_EL(pmecc->caps->el_offset, i));
710 errpos--;
711
712 byte = errpos / 8;
713 bit = errpos % 8;
714
715 if (byte < sectorsize) {
716 ptr = data + byte;
717 area = "data";
718 } else if (byte < sectorsize + eccbytes) {
719 ptr = ecc + byte - sectorsize;
720 area = "ECC";
721 } else {
722 dev_dbg(pmecc->dev,
723 "Invalid errpos value (%d, max is %d)\n",
724 errpos, (sectorsize + eccbytes) * 8);
725 return -EINVAL;
726 }
727
728 dev_dbg(pmecc->dev,
729 "Bit flip in %s area, byte %d: 0x%02x -> 0x%02x\n",
730 area, byte, *ptr, (unsigned int)(*ptr ^ BIT(bit)));
731
732 *ptr ^= BIT(bit);
733 }
734
735 return nerrors;
736 }
737 EXPORT_SYMBOL_GPL(atmel_pmecc_correct_sector);
738
atmel_pmecc_correct_erased_chunks(struct atmel_pmecc_user * user)739 bool atmel_pmecc_correct_erased_chunks(struct atmel_pmecc_user *user)
740 {
741 return user->pmecc->caps->correct_erased_chunks;
742 }
743 EXPORT_SYMBOL_GPL(atmel_pmecc_correct_erased_chunks);
744
atmel_pmecc_get_generated_eccbytes(struct atmel_pmecc_user * user,int sector,void * ecc)745 void atmel_pmecc_get_generated_eccbytes(struct atmel_pmecc_user *user,
746 int sector, void *ecc)
747 {
748 struct atmel_pmecc *pmecc = user->pmecc;
749 u8 *ptr = ecc;
750 int i;
751
752 for (i = 0; i < user->eccbytes; i++)
753 ptr[i] = readb_relaxed(pmecc->regs.base +
754 ATMEL_PMECC_ECC(sector, i));
755 }
756 EXPORT_SYMBOL_GPL(atmel_pmecc_get_generated_eccbytes);
757
atmel_pmecc_reset(struct atmel_pmecc * pmecc)758 void atmel_pmecc_reset(struct atmel_pmecc *pmecc)
759 {
760 writel(PMECC_CTRL_RST, pmecc->regs.base + ATMEL_PMECC_CTRL);
761 writel(PMECC_CTRL_DISABLE, pmecc->regs.base + ATMEL_PMECC_CTRL);
762 }
763 EXPORT_SYMBOL_GPL(atmel_pmecc_reset);
764
atmel_pmecc_enable(struct atmel_pmecc_user * user,int op)765 int atmel_pmecc_enable(struct atmel_pmecc_user *user, int op)
766 {
767 struct atmel_pmecc *pmecc = user->pmecc;
768 u32 cfg;
769
770 if (op != NAND_ECC_READ && op != NAND_ECC_WRITE) {
771 dev_err(pmecc->dev, "Bad ECC operation!");
772 return -EINVAL;
773 }
774
775 mutex_lock(&user->pmecc->lock);
776
777 cfg = user->cache.cfg;
778 if (op == NAND_ECC_WRITE)
779 cfg |= PMECC_CFG_WRITE_OP;
780 else
781 cfg |= PMECC_CFG_AUTO_ENABLE;
782
783 writel(cfg, pmecc->regs.base + ATMEL_PMECC_CFG);
784 writel(user->cache.sarea, pmecc->regs.base + ATMEL_PMECC_SAREA);
785 writel(user->cache.saddr, pmecc->regs.base + ATMEL_PMECC_SADDR);
786 writel(user->cache.eaddr, pmecc->regs.base + ATMEL_PMECC_EADDR);
787
788 writel(PMECC_CTRL_ENABLE, pmecc->regs.base + ATMEL_PMECC_CTRL);
789 writel(PMECC_CTRL_DATA, pmecc->regs.base + ATMEL_PMECC_CTRL);
790
791 return 0;
792 }
793 EXPORT_SYMBOL_GPL(atmel_pmecc_enable);
794
atmel_pmecc_disable(struct atmel_pmecc_user * user)795 void atmel_pmecc_disable(struct atmel_pmecc_user *user)
796 {
797 atmel_pmecc_reset(user->pmecc);
798 mutex_unlock(&user->pmecc->lock);
799 }
800 EXPORT_SYMBOL_GPL(atmel_pmecc_disable);
801
atmel_pmecc_wait_rdy(struct atmel_pmecc_user * user)802 int atmel_pmecc_wait_rdy(struct atmel_pmecc_user *user)
803 {
804 struct atmel_pmecc *pmecc = user->pmecc;
805 u32 status;
806 int ret;
807
808 ret = readl_relaxed_poll_timeout(pmecc->regs.base +
809 ATMEL_PMECC_SR,
810 status, !(status & PMECC_SR_BUSY), 0,
811 PMECC_MAX_TIMEOUT_MS * 1000);
812 if (ret) {
813 dev_err(pmecc->dev,
814 "Timeout while waiting for PMECC ready.\n");
815 return ret;
816 }
817
818 user->isr = readl_relaxed(pmecc->regs.base + ATMEL_PMECC_ISR);
819
820 return 0;
821 }
822 EXPORT_SYMBOL_GPL(atmel_pmecc_wait_rdy);
823
atmel_pmecc_create(struct platform_device * pdev,const struct atmel_pmecc_caps * caps,int pmecc_res_idx,int errloc_res_idx)824 static struct atmel_pmecc *atmel_pmecc_create(struct platform_device *pdev,
825 const struct atmel_pmecc_caps *caps,
826 int pmecc_res_idx, int errloc_res_idx)
827 {
828 struct device *dev = &pdev->dev;
829 struct atmel_pmecc *pmecc;
830
831 pmecc = devm_kzalloc(dev, sizeof(*pmecc), GFP_KERNEL);
832 if (!pmecc)
833 return ERR_PTR(-ENOMEM);
834
835 pmecc->caps = caps;
836 pmecc->dev = dev;
837 mutex_init(&pmecc->lock);
838
839 pmecc->regs.base = devm_platform_ioremap_resource(pdev, pmecc_res_idx);
840 if (IS_ERR(pmecc->regs.base))
841 return ERR_CAST(pmecc->regs.base);
842
843 pmecc->regs.errloc = devm_platform_ioremap_resource(pdev, errloc_res_idx);
844 if (IS_ERR(pmecc->regs.errloc))
845 return ERR_CAST(pmecc->regs.errloc);
846
847 /* pmecc data setup time */
848 if (caps->clk_ctrl)
849 writel(PMECC_CLK_133MHZ, pmecc->regs.base + ATMEL_PMECC_CLK);
850
851 /* Disable all interrupts before registering the PMECC handler. */
852 writel(0xffffffff, pmecc->regs.base + ATMEL_PMECC_IDR);
853 atmel_pmecc_reset(pmecc);
854
855 return pmecc;
856 }
857
devm_atmel_pmecc_put(struct device * dev,void * res)858 static void devm_atmel_pmecc_put(struct device *dev, void *res)
859 {
860 struct atmel_pmecc **pmecc = res;
861
862 put_device((*pmecc)->dev);
863 }
864
atmel_pmecc_get_by_node(struct device * userdev,struct device_node * np)865 static struct atmel_pmecc *atmel_pmecc_get_by_node(struct device *userdev,
866 struct device_node *np)
867 {
868 struct platform_device *pdev;
869 struct atmel_pmecc *pmecc, **ptr;
870 int ret;
871
872 pdev = of_find_device_by_node(np);
873 if (!pdev)
874 return ERR_PTR(-EPROBE_DEFER);
875 pmecc = platform_get_drvdata(pdev);
876 if (!pmecc) {
877 ret = -EPROBE_DEFER;
878 goto err_put_device;
879 }
880
881 ptr = devres_alloc(devm_atmel_pmecc_put, sizeof(*ptr), GFP_KERNEL);
882 if (!ptr) {
883 ret = -ENOMEM;
884 goto err_put_device;
885 }
886
887 *ptr = pmecc;
888
889 devres_add(userdev, ptr);
890
891 return pmecc;
892
893 err_put_device:
894 put_device(&pdev->dev);
895 return ERR_PTR(ret);
896 }
897
898 static const int atmel_pmecc_strengths[] = { 2, 4, 8, 12, 24, 32 };
899
900 static struct atmel_pmecc_caps at91sam9g45_caps = {
901 .strengths = atmel_pmecc_strengths,
902 .nstrengths = 5,
903 .el_offset = 0x8c,
904 .clk_ctrl = true,
905 };
906
907 static struct atmel_pmecc_caps sama5d4_caps = {
908 .strengths = atmel_pmecc_strengths,
909 .nstrengths = 5,
910 .el_offset = 0x8c,
911 .correct_erased_chunks = true,
912 };
913
914 static struct atmel_pmecc_caps sama5d2_caps = {
915 .strengths = atmel_pmecc_strengths,
916 .nstrengths = 6,
917 .el_offset = 0xac,
918 .correct_erased_chunks = true,
919 };
920
921 static const struct of_device_id __maybe_unused atmel_pmecc_legacy_match[] = {
922 { .compatible = "atmel,sama5d4-nand", &sama5d4_caps },
923 { .compatible = "atmel,sama5d2-nand", &sama5d2_caps },
924 { /* sentinel */ }
925 };
926
devm_atmel_pmecc_get(struct device * userdev)927 struct atmel_pmecc *devm_atmel_pmecc_get(struct device *userdev)
928 {
929 struct atmel_pmecc *pmecc;
930 struct device_node *np;
931
932 if (!userdev)
933 return ERR_PTR(-EINVAL);
934
935 if (!userdev->of_node)
936 return NULL;
937
938 np = of_parse_phandle(userdev->of_node, "ecc-engine", 0);
939 if (np) {
940 pmecc = atmel_pmecc_get_by_node(userdev, np);
941 of_node_put(np);
942 } else {
943 /*
944 * Support old DT bindings: in this case the PMECC iomem
945 * resources are directly defined in the user pdev at position
946 * 1 and 2. Extract all relevant information from there.
947 */
948 struct platform_device *pdev = to_platform_device(userdev);
949 const struct atmel_pmecc_caps *caps;
950 const struct of_device_id *match;
951
952 /* No PMECC engine available. */
953 if (!of_property_read_bool(userdev->of_node,
954 "atmel,has-pmecc"))
955 return NULL;
956
957 caps = &at91sam9g45_caps;
958
959 /* Find the caps associated to the NAND dev node. */
960 match = of_match_node(atmel_pmecc_legacy_match,
961 userdev->of_node);
962 if (match && match->data)
963 caps = match->data;
964
965 pmecc = atmel_pmecc_create(pdev, caps, 1, 2);
966 }
967
968 return pmecc;
969 }
970 EXPORT_SYMBOL(devm_atmel_pmecc_get);
971
972 static const struct of_device_id atmel_pmecc_match[] = {
973 { .compatible = "atmel,at91sam9g45-pmecc", &at91sam9g45_caps },
974 { .compatible = "atmel,sama5d4-pmecc", &sama5d4_caps },
975 { .compatible = "atmel,sama5d2-pmecc", &sama5d2_caps },
976 { /* sentinel */ }
977 };
978 MODULE_DEVICE_TABLE(of, atmel_pmecc_match);
979
atmel_pmecc_probe(struct platform_device * pdev)980 static int atmel_pmecc_probe(struct platform_device *pdev)
981 {
982 struct device *dev = &pdev->dev;
983 const struct atmel_pmecc_caps *caps;
984 struct atmel_pmecc *pmecc;
985
986 caps = of_device_get_match_data(&pdev->dev);
987 if (!caps) {
988 dev_err(dev, "Invalid caps\n");
989 return -EINVAL;
990 }
991
992 pmecc = atmel_pmecc_create(pdev, caps, 0, 1);
993 if (IS_ERR(pmecc))
994 return PTR_ERR(pmecc);
995
996 platform_set_drvdata(pdev, pmecc);
997
998 return 0;
999 }
1000
1001 static struct platform_driver atmel_pmecc_driver = {
1002 .driver = {
1003 .name = "atmel-pmecc",
1004 .of_match_table = atmel_pmecc_match,
1005 },
1006 .probe = atmel_pmecc_probe,
1007 };
1008 module_platform_driver(atmel_pmecc_driver);
1009
1010 MODULE_LICENSE("GPL");
1011 MODULE_AUTHOR("Boris Brezillon <boris.brezillon@free-electrons.com>");
1012 MODULE_DESCRIPTION("PMECC engine driver");
1013 MODULE_ALIAS("platform:atmel_pmecc");
1014