1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Evatronix/Renesas R-Car Gen3, RZ/N1D, RZ/N1S, RZ/N1L NAND controller driver
4 *
5 * Copyright (C) 2021 Schneider Electric
6 * Author: Miquel RAYNAL <miquel.raynal@bootlin.com>
7 */
8
9 #include <linux/bitfield.h>
10 #include <linux/clk.h>
11 #include <linux/dma-mapping.h>
12 #include <linux/interrupt.h>
13 #include <linux/iopoll.h>
14 #include <linux/module.h>
15 #include <linux/mtd/mtd.h>
16 #include <linux/mtd/rawnand.h>
17 #include <linux/of.h>
18 #include <linux/platform_device.h>
19 #include <linux/pm_runtime.h>
20 #include <linux/slab.h>
21
22 #define COMMAND_REG 0x00
23 #define COMMAND_SEQ(x) FIELD_PREP(GENMASK(5, 0), (x))
24 #define COMMAND_SEQ_10 COMMAND_SEQ(0x2A)
25 #define COMMAND_SEQ_12 COMMAND_SEQ(0x0C)
26 #define COMMAND_SEQ_18 COMMAND_SEQ(0x32)
27 #define COMMAND_SEQ_19 COMMAND_SEQ(0x13)
28 #define COMMAND_SEQ_GEN_IN COMMAND_SEQ_18
29 #define COMMAND_SEQ_GEN_OUT COMMAND_SEQ_19
30 #define COMMAND_SEQ_READ_PAGE COMMAND_SEQ_10
31 #define COMMAND_SEQ_WRITE_PAGE COMMAND_SEQ_12
32 #define COMMAND_INPUT_SEL_AHBS 0
33 #define COMMAND_INPUT_SEL_DMA BIT(6)
34 #define COMMAND_FIFO_SEL 0
35 #define COMMAND_DATA_SEL BIT(7)
36 #define COMMAND_0(x) FIELD_PREP(GENMASK(15, 8), (x))
37 #define COMMAND_1(x) FIELD_PREP(GENMASK(23, 16), (x))
38 #define COMMAND_2(x) FIELD_PREP(GENMASK(31, 24), (x))
39
40 #define CONTROL_REG 0x04
41 #define CONTROL_CHECK_RB_LINE 0
42 #define CONTROL_ECC_BLOCK_SIZE(x) FIELD_PREP(GENMASK(2, 1), (x))
43 #define CONTROL_ECC_BLOCK_SIZE_256 CONTROL_ECC_BLOCK_SIZE(0)
44 #define CONTROL_ECC_BLOCK_SIZE_512 CONTROL_ECC_BLOCK_SIZE(1)
45 #define CONTROL_ECC_BLOCK_SIZE_1024 CONTROL_ECC_BLOCK_SIZE(2)
46 #define CONTROL_INT_EN BIT(4)
47 #define CONTROL_ECC_EN BIT(5)
48 #define CONTROL_BLOCK_SIZE(x) FIELD_PREP(GENMASK(7, 6), (x))
49 #define CONTROL_BLOCK_SIZE_32P CONTROL_BLOCK_SIZE(0)
50 #define CONTROL_BLOCK_SIZE_64P CONTROL_BLOCK_SIZE(1)
51 #define CONTROL_BLOCK_SIZE_128P CONTROL_BLOCK_SIZE(2)
52 #define CONTROL_BLOCK_SIZE_256P CONTROL_BLOCK_SIZE(3)
53
54 #define STATUS_REG 0x8
55 #define MEM_RDY(cs, reg) (FIELD_GET(GENMASK(3, 0), (reg)) & BIT(cs))
56 #define CTRL_RDY(reg) (FIELD_GET(BIT(8), (reg)) == 0)
57
58 #define ECC_CTRL_REG 0x18
59 #define ECC_CTRL_CAP(x) FIELD_PREP(GENMASK(2, 0), (x))
60 #define ECC_CTRL_CAP_2B ECC_CTRL_CAP(0)
61 #define ECC_CTRL_CAP_4B ECC_CTRL_CAP(1)
62 #define ECC_CTRL_CAP_8B ECC_CTRL_CAP(2)
63 #define ECC_CTRL_CAP_16B ECC_CTRL_CAP(3)
64 #define ECC_CTRL_CAP_24B ECC_CTRL_CAP(4)
65 #define ECC_CTRL_CAP_32B ECC_CTRL_CAP(5)
66 #define ECC_CTRL_ERR_THRESHOLD(x) FIELD_PREP(GENMASK(13, 8), (x))
67
68 #define INT_MASK_REG 0x10
69 #define INT_STATUS_REG 0x14
70 #define INT_CMD_END BIT(1)
71 #define INT_DMA_END BIT(3)
72 #define INT_MEM_RDY(cs) FIELD_PREP(GENMASK(11, 8), BIT(cs))
73 #define INT_DMA_ENDED BIT(3)
74 #define MEM_IS_RDY(cs, reg) (FIELD_GET(GENMASK(11, 8), (reg)) & BIT(cs))
75 #define DMA_HAS_ENDED(reg) FIELD_GET(BIT(3), (reg))
76
77 #define ECC_OFFSET_REG 0x1C
78 #define ECC_OFFSET(x) FIELD_PREP(GENMASK(15, 0), (x))
79
80 #define ECC_STAT_REG 0x20
81 #define ECC_STAT_CORRECTABLE(cs, reg) (FIELD_GET(GENMASK(3, 0), (reg)) & BIT(cs))
82 #define ECC_STAT_UNCORRECTABLE(cs, reg) (FIELD_GET(GENMASK(11, 8), (reg)) & BIT(cs))
83
84 #define ADDR0_COL_REG 0x24
85 #define ADDR0_COL(x) FIELD_PREP(GENMASK(15, 0), (x))
86
87 #define ADDR0_ROW_REG 0x28
88 #define ADDR0_ROW(x) FIELD_PREP(GENMASK(23, 0), (x))
89
90 #define ADDR1_COL_REG 0x2C
91 #define ADDR1_COL(x) FIELD_PREP(GENMASK(15, 0), (x))
92
93 #define ADDR1_ROW_REG 0x30
94 #define ADDR1_ROW(x) FIELD_PREP(GENMASK(23, 0), (x))
95
96 #define FIFO_DATA_REG 0x38
97
98 #define DATA_REG 0x3C
99
100 #define DATA_REG_SIZE_REG 0x40
101
102 #define DMA_ADDR_LOW_REG 0x64
103
104 #define DMA_ADDR_HIGH_REG 0x68
105
106 #define DMA_CNT_REG 0x6C
107
108 #define DMA_CTRL_REG 0x70
109 #define DMA_CTRL_INCREMENT_BURST_4 0
110 #define DMA_CTRL_REGISTER_MANAGED_MODE 0
111 #define DMA_CTRL_START BIT(7)
112
113 #define MEM_CTRL_REG 0x80
114 #define MEM_CTRL_CS(cs) FIELD_PREP(GENMASK(1, 0), (cs))
115 #define MEM_CTRL_DIS_WP(cs) FIELD_PREP(GENMASK(11, 8), BIT((cs)))
116
117 #define DATA_SIZE_REG 0x84
118 #define DATA_SIZE(x) FIELD_PREP(GENMASK(14, 0), (x))
119
120 #define TIMINGS_ASYN_REG 0x88
121 #define TIMINGS_ASYN_TRWP(x) FIELD_PREP(GENMASK(3, 0), max((x), 1U) - 1)
122 #define TIMINGS_ASYN_TRWH(x) FIELD_PREP(GENMASK(7, 4), max((x), 1U) - 1)
123
124 #define TIM_SEQ0_REG 0x90
125 #define TIM_SEQ0_TCCS(x) FIELD_PREP(GENMASK(5, 0), max((x), 1U) - 1)
126 #define TIM_SEQ0_TADL(x) FIELD_PREP(GENMASK(13, 8), max((x), 1U) - 1)
127 #define TIM_SEQ0_TRHW(x) FIELD_PREP(GENMASK(21, 16), max((x), 1U) - 1)
128 #define TIM_SEQ0_TWHR(x) FIELD_PREP(GENMASK(29, 24), max((x), 1U) - 1)
129
130 #define TIM_SEQ1_REG 0x94
131 #define TIM_SEQ1_TWB(x) FIELD_PREP(GENMASK(5, 0), max((x), 1U) - 1)
132 #define TIM_SEQ1_TRR(x) FIELD_PREP(GENMASK(13, 8), max((x), 1U) - 1)
133 #define TIM_SEQ1_TWW(x) FIELD_PREP(GENMASK(21, 16), max((x), 1U) - 1)
134
135 #define TIM_GEN_SEQ0_REG 0x98
136 #define TIM_GEN_SEQ0_D0(x) FIELD_PREP(GENMASK(5, 0), max((x), 1U) - 1)
137 #define TIM_GEN_SEQ0_D1(x) FIELD_PREP(GENMASK(13, 8), max((x), 1U) - 1)
138 #define TIM_GEN_SEQ0_D2(x) FIELD_PREP(GENMASK(21, 16), max((x), 1U) - 1)
139 #define TIM_GEN_SEQ0_D3(x) FIELD_PREP(GENMASK(29, 24), max((x), 1U) - 1)
140
141 #define TIM_GEN_SEQ1_REG 0x9c
142 #define TIM_GEN_SEQ1_D4(x) FIELD_PREP(GENMASK(5, 0), max((x), 1U) - 1)
143 #define TIM_GEN_SEQ1_D5(x) FIELD_PREP(GENMASK(13, 8), max((x), 1U) - 1)
144 #define TIM_GEN_SEQ1_D6(x) FIELD_PREP(GENMASK(21, 16), max((x), 1U) - 1)
145 #define TIM_GEN_SEQ1_D7(x) FIELD_PREP(GENMASK(29, 24), max((x), 1U) - 1)
146
147 #define TIM_GEN_SEQ2_REG 0xA0
148 #define TIM_GEN_SEQ2_D8(x) FIELD_PREP(GENMASK(5, 0), max((x), 1U) - 1)
149 #define TIM_GEN_SEQ2_D9(x) FIELD_PREP(GENMASK(13, 8), max((x), 1U) - 1)
150 #define TIM_GEN_SEQ2_D10(x) FIELD_PREP(GENMASK(21, 16), max((x), 1U) - 1)
151 #define TIM_GEN_SEQ2_D11(x) FIELD_PREP(GENMASK(29, 24), max((x), 1U) - 1)
152
153 #define FIFO_INIT_REG 0xB4
154 #define FIFO_INIT BIT(0)
155
156 #define FIFO_STATE_REG 0xB4
157 #define FIFO_STATE_R_EMPTY(reg) FIELD_GET(BIT(0), (reg))
158 #define FIFO_STATE_W_FULL(reg) FIELD_GET(BIT(1), (reg))
159 #define FIFO_STATE_C_EMPTY(reg) FIELD_GET(BIT(2), (reg))
160 #define FIFO_STATE_R_FULL(reg) FIELD_GET(BIT(6), (reg))
161 #define FIFO_STATE_W_EMPTY(reg) FIELD_GET(BIT(7), (reg))
162
163 #define GEN_SEQ_CTRL_REG 0xB8
164 #define GEN_SEQ_CMD0_EN BIT(0)
165 #define GEN_SEQ_CMD1_EN BIT(1)
166 #define GEN_SEQ_CMD2_EN BIT(2)
167 #define GEN_SEQ_CMD3_EN BIT(3)
168 #define GEN_SEQ_COL_A0(x) FIELD_PREP(GENMASK(5, 4), min((x), 2U))
169 #define GEN_SEQ_COL_A1(x) FIELD_PREP(GENMASK(7, 6), min((x), 2U))
170 #define GEN_SEQ_ROW_A0(x) FIELD_PREP(GENMASK(9, 8), min((x), 3U))
171 #define GEN_SEQ_ROW_A1(x) FIELD_PREP(GENMASK(11, 10), min((x), 3U))
172 #define GEN_SEQ_DATA_EN BIT(12)
173 #define GEN_SEQ_DELAY_EN(x) FIELD_PREP(GENMASK(14, 13), (x))
174 #define GEN_SEQ_DELAY0_EN GEN_SEQ_DELAY_EN(1)
175 #define GEN_SEQ_DELAY1_EN GEN_SEQ_DELAY_EN(2)
176 #define GEN_SEQ_IMD_SEQ BIT(15)
177 #define GEN_SEQ_COMMAND_3(x) FIELD_PREP(GENMASK(26, 16), (x))
178
179 #define DMA_TLVL_REG 0x114
180 #define DMA_TLVL(x) FIELD_PREP(GENMASK(7, 0), (x))
181 #define DMA_TLVL_MAX DMA_TLVL(0xFF)
182
183 #define TIM_GEN_SEQ3_REG 0x134
184 #define TIM_GEN_SEQ3_D12(x) FIELD_PREP(GENMASK(5, 0), max((x), 1U) - 1)
185
186 #define ECC_CNT_REG 0x14C
187 #define ECC_CNT(cs, reg) FIELD_GET(GENMASK(5, 0), (reg) >> ((cs) * 8))
188
189 #define RNANDC_CS_NUM 4
190
191 #define TO_CYCLES64(ps, period_ns) ((unsigned int)DIV_ROUND_UP_ULL(div_u64(ps, 1000), \
192 period_ns))
193
194 struct rnand_chip_sel {
195 unsigned int cs;
196 };
197
198 struct rnand_chip {
199 struct nand_chip chip;
200 struct list_head node;
201 int selected_die;
202 u32 ctrl;
203 unsigned int nsels;
204 u32 control;
205 u32 ecc_ctrl;
206 u32 timings_asyn;
207 u32 tim_seq0;
208 u32 tim_seq1;
209 u32 tim_gen_seq0;
210 u32 tim_gen_seq1;
211 u32 tim_gen_seq2;
212 u32 tim_gen_seq3;
213 struct rnand_chip_sel sels[] __counted_by(nsels);
214 };
215
216 struct rnandc {
217 struct nand_controller controller;
218 struct device *dev;
219 void __iomem *regs;
220 unsigned long ext_clk_rate;
221 unsigned long assigned_cs;
222 struct list_head chips;
223 struct nand_chip *selected_chip;
224 struct completion complete;
225 bool use_polling;
226 u8 *buf;
227 unsigned int buf_sz;
228 };
229
230 struct rnandc_op {
231 u32 command;
232 u32 addr0_col;
233 u32 addr0_row;
234 u32 addr1_col;
235 u32 addr1_row;
236 u32 data_size;
237 u32 ecc_offset;
238 u32 gen_seq_ctrl;
239 u8 *buf;
240 bool read;
241 unsigned int len;
242 };
243
to_rnandc(struct nand_controller * ctrl)244 static inline struct rnandc *to_rnandc(struct nand_controller *ctrl)
245 {
246 return container_of(ctrl, struct rnandc, controller);
247 }
248
to_rnand(struct nand_chip * chip)249 static inline struct rnand_chip *to_rnand(struct nand_chip *chip)
250 {
251 return container_of(chip, struct rnand_chip, chip);
252 }
253
to_rnandc_cs(struct rnand_chip * nand)254 static inline unsigned int to_rnandc_cs(struct rnand_chip *nand)
255 {
256 return nand->sels[nand->selected_die].cs;
257 }
258
rnandc_dis_correction(struct rnandc * rnandc)259 static void rnandc_dis_correction(struct rnandc *rnandc)
260 {
261 u32 control;
262
263 control = readl_relaxed(rnandc->regs + CONTROL_REG);
264 control &= ~CONTROL_ECC_EN;
265 writel_relaxed(control, rnandc->regs + CONTROL_REG);
266 }
267
rnandc_en_correction(struct rnandc * rnandc)268 static void rnandc_en_correction(struct rnandc *rnandc)
269 {
270 u32 control;
271
272 control = readl_relaxed(rnandc->regs + CONTROL_REG);
273 control |= CONTROL_ECC_EN;
274 writel_relaxed(control, rnandc->regs + CONTROL_REG);
275 }
276
rnandc_clear_status(struct rnandc * rnandc)277 static void rnandc_clear_status(struct rnandc *rnandc)
278 {
279 writel_relaxed(0, rnandc->regs + INT_STATUS_REG);
280 writel_relaxed(0, rnandc->regs + ECC_STAT_REG);
281 writel_relaxed(0, rnandc->regs + ECC_CNT_REG);
282 }
283
rnandc_dis_interrupts(struct rnandc * rnandc)284 static void rnandc_dis_interrupts(struct rnandc *rnandc)
285 {
286 writel_relaxed(0, rnandc->regs + INT_MASK_REG);
287 }
288
rnandc_en_interrupts(struct rnandc * rnandc,u32 val)289 static void rnandc_en_interrupts(struct rnandc *rnandc, u32 val)
290 {
291 if (!rnandc->use_polling)
292 writel_relaxed(val, rnandc->regs + INT_MASK_REG);
293 }
294
rnandc_clear_fifo(struct rnandc * rnandc)295 static void rnandc_clear_fifo(struct rnandc *rnandc)
296 {
297 writel_relaxed(FIFO_INIT, rnandc->regs + FIFO_INIT_REG);
298 }
299
rnandc_select_target(struct nand_chip * chip,int die_nr)300 static void rnandc_select_target(struct nand_chip *chip, int die_nr)
301 {
302 struct rnand_chip *rnand = to_rnand(chip);
303 struct rnandc *rnandc = to_rnandc(chip->controller);
304 unsigned int cs = rnand->sels[die_nr].cs;
305
306 if (chip == rnandc->selected_chip && die_nr == rnand->selected_die)
307 return;
308
309 rnandc_clear_status(rnandc);
310 writel_relaxed(MEM_CTRL_CS(cs) | MEM_CTRL_DIS_WP(cs), rnandc->regs + MEM_CTRL_REG);
311 writel_relaxed(rnand->control, rnandc->regs + CONTROL_REG);
312 writel_relaxed(rnand->ecc_ctrl, rnandc->regs + ECC_CTRL_REG);
313 writel_relaxed(rnand->timings_asyn, rnandc->regs + TIMINGS_ASYN_REG);
314 writel_relaxed(rnand->tim_seq0, rnandc->regs + TIM_SEQ0_REG);
315 writel_relaxed(rnand->tim_seq1, rnandc->regs + TIM_SEQ1_REG);
316 writel_relaxed(rnand->tim_gen_seq0, rnandc->regs + TIM_GEN_SEQ0_REG);
317 writel_relaxed(rnand->tim_gen_seq1, rnandc->regs + TIM_GEN_SEQ1_REG);
318 writel_relaxed(rnand->tim_gen_seq2, rnandc->regs + TIM_GEN_SEQ2_REG);
319 writel_relaxed(rnand->tim_gen_seq3, rnandc->regs + TIM_GEN_SEQ3_REG);
320
321 rnandc->selected_chip = chip;
322 rnand->selected_die = die_nr;
323 }
324
rnandc_trigger_op(struct rnandc * rnandc,struct rnandc_op * rop)325 static void rnandc_trigger_op(struct rnandc *rnandc, struct rnandc_op *rop)
326 {
327 writel_relaxed(rop->addr0_col, rnandc->regs + ADDR0_COL_REG);
328 writel_relaxed(rop->addr0_row, rnandc->regs + ADDR0_ROW_REG);
329 writel_relaxed(rop->addr1_col, rnandc->regs + ADDR1_COL_REG);
330 writel_relaxed(rop->addr1_row, rnandc->regs + ADDR1_ROW_REG);
331 writel_relaxed(rop->ecc_offset, rnandc->regs + ECC_OFFSET_REG);
332 writel_relaxed(rop->gen_seq_ctrl, rnandc->regs + GEN_SEQ_CTRL_REG);
333 writel_relaxed(DATA_SIZE(rop->len), rnandc->regs + DATA_SIZE_REG);
334 writel_relaxed(rop->command, rnandc->regs + COMMAND_REG);
335 }
336
rnandc_trigger_dma(struct rnandc * rnandc)337 static void rnandc_trigger_dma(struct rnandc *rnandc)
338 {
339 writel_relaxed(DMA_CTRL_INCREMENT_BURST_4 |
340 DMA_CTRL_REGISTER_MANAGED_MODE |
341 DMA_CTRL_START, rnandc->regs + DMA_CTRL_REG);
342 }
343
rnandc_irq_handler(int irq,void * private)344 static irqreturn_t rnandc_irq_handler(int irq, void *private)
345 {
346 struct rnandc *rnandc = private;
347
348 rnandc_dis_interrupts(rnandc);
349 complete(&rnandc->complete);
350
351 return IRQ_HANDLED;
352 }
353
rnandc_wait_end_of_op(struct rnandc * rnandc,struct nand_chip * chip)354 static int rnandc_wait_end_of_op(struct rnandc *rnandc,
355 struct nand_chip *chip)
356 {
357 struct rnand_chip *rnand = to_rnand(chip);
358 unsigned int cs = to_rnandc_cs(rnand);
359 u32 status;
360 int ret;
361
362 ret = readl_poll_timeout(rnandc->regs + STATUS_REG, status,
363 MEM_RDY(cs, status) && CTRL_RDY(status),
364 1, 100000);
365 if (ret)
366 dev_err(rnandc->dev, "Operation timed out, status: 0x%08x\n",
367 status);
368
369 return ret;
370 }
371
rnandc_wait_end_of_io(struct rnandc * rnandc,struct nand_chip * chip)372 static int rnandc_wait_end_of_io(struct rnandc *rnandc,
373 struct nand_chip *chip)
374 {
375 int timeout_ms = 1000;
376 int ret;
377
378 if (rnandc->use_polling) {
379 struct rnand_chip *rnand = to_rnand(chip);
380 unsigned int cs = to_rnandc_cs(rnand);
381 u32 status;
382
383 ret = readl_poll_timeout(rnandc->regs + INT_STATUS_REG, status,
384 MEM_IS_RDY(cs, status) &
385 DMA_HAS_ENDED(status),
386 0, timeout_ms * 1000);
387 } else {
388 ret = wait_for_completion_timeout(&rnandc->complete,
389 msecs_to_jiffies(timeout_ms));
390 if (!ret)
391 ret = -ETIMEDOUT;
392 else
393 ret = 0;
394 }
395
396 return ret;
397 }
398
rnandc_read_page_hw_ecc(struct nand_chip * chip,u8 * buf,int oob_required,int page)399 static int rnandc_read_page_hw_ecc(struct nand_chip *chip, u8 *buf,
400 int oob_required, int page)
401 {
402 struct rnandc *rnandc = to_rnandc(chip->controller);
403 struct mtd_info *mtd = nand_to_mtd(chip);
404 struct rnand_chip *rnand = to_rnand(chip);
405 unsigned int cs = to_rnandc_cs(rnand);
406 struct rnandc_op rop = {
407 .command = COMMAND_INPUT_SEL_DMA | COMMAND_0(NAND_CMD_READ0) |
408 COMMAND_2(NAND_CMD_READSTART) | COMMAND_FIFO_SEL |
409 COMMAND_SEQ_READ_PAGE,
410 .addr0_row = page,
411 .len = mtd->writesize,
412 .ecc_offset = ECC_OFFSET(mtd->writesize + 2),
413 };
414 unsigned int max_bitflips = 0;
415 dma_addr_t dma_addr;
416 u32 ecc_stat;
417 int bf, ret, i;
418
419 /* Prepare controller */
420 rnandc_select_target(chip, chip->cur_cs);
421 rnandc_clear_status(rnandc);
422 reinit_completion(&rnandc->complete);
423 rnandc_en_interrupts(rnandc, INT_DMA_ENDED);
424 rnandc_en_correction(rnandc);
425
426 /* Configure DMA */
427 dma_addr = dma_map_single(rnandc->dev, rnandc->buf, mtd->writesize,
428 DMA_FROM_DEVICE);
429 if (dma_mapping_error(rnandc->dev, dma_addr))
430 return -ENOMEM;
431
432 writel(dma_addr, rnandc->regs + DMA_ADDR_LOW_REG);
433 writel(mtd->writesize, rnandc->regs + DMA_CNT_REG);
434 writel(DMA_TLVL_MAX, rnandc->regs + DMA_TLVL_REG);
435
436 rnandc_trigger_op(rnandc, &rop);
437 rnandc_trigger_dma(rnandc);
438
439 ret = rnandc_wait_end_of_io(rnandc, chip);
440 dma_unmap_single(rnandc->dev, dma_addr, mtd->writesize, DMA_FROM_DEVICE);
441 rnandc_dis_correction(rnandc);
442 if (ret) {
443 dev_err(rnandc->dev, "Read page operation never ending\n");
444 return ret;
445 }
446
447 ecc_stat = readl_relaxed(rnandc->regs + ECC_STAT_REG);
448
449 if (oob_required || ECC_STAT_UNCORRECTABLE(cs, ecc_stat)) {
450 ret = nand_change_read_column_op(chip, mtd->writesize,
451 chip->oob_poi, mtd->oobsize,
452 false);
453 if (ret)
454 return ret;
455 }
456
457 if (ECC_STAT_UNCORRECTABLE(cs, ecc_stat)) {
458 for (i = 0; i < chip->ecc.steps; i++) {
459 unsigned int off = i * chip->ecc.size;
460 unsigned int eccoff = i * chip->ecc.bytes;
461
462 bf = nand_check_erased_ecc_chunk(rnandc->buf + off,
463 chip->ecc.size,
464 chip->oob_poi + 2 + eccoff,
465 chip->ecc.bytes,
466 NULL, 0,
467 chip->ecc.strength);
468 if (bf < 0) {
469 mtd->ecc_stats.failed++;
470 } else {
471 mtd->ecc_stats.corrected += bf;
472 max_bitflips = max_t(unsigned int, max_bitflips, bf);
473 }
474 }
475 } else if (ECC_STAT_CORRECTABLE(cs, ecc_stat)) {
476 bf = ECC_CNT(cs, readl_relaxed(rnandc->regs + ECC_CNT_REG));
477 /*
478 * The number of bitflips is an approximation given the fact
479 * that this controller does not provide per-chunk details but
480 * only gives statistics on the entire page.
481 */
482 mtd->ecc_stats.corrected += bf;
483 }
484
485 memcpy(buf, rnandc->buf, mtd->writesize);
486
487 return 0;
488 }
489
rnandc_read_subpage_hw_ecc(struct nand_chip * chip,u32 req_offset,u32 req_len,u8 * bufpoi,int page)490 static int rnandc_read_subpage_hw_ecc(struct nand_chip *chip, u32 req_offset,
491 u32 req_len, u8 *bufpoi, int page)
492 {
493 struct rnandc *rnandc = to_rnandc(chip->controller);
494 struct mtd_info *mtd = nand_to_mtd(chip);
495 struct rnand_chip *rnand = to_rnand(chip);
496 unsigned int cs = to_rnandc_cs(rnand);
497 unsigned int page_off = round_down(req_offset, chip->ecc.size);
498 unsigned int real_len = round_up(req_offset + req_len - page_off,
499 chip->ecc.size);
500 unsigned int start_chunk = page_off / chip->ecc.size;
501 unsigned int nchunks = real_len / chip->ecc.size;
502 unsigned int ecc_off = 2 + (start_chunk * chip->ecc.bytes);
503 struct rnandc_op rop = {
504 .command = COMMAND_INPUT_SEL_AHBS | COMMAND_0(NAND_CMD_READ0) |
505 COMMAND_2(NAND_CMD_READSTART) | COMMAND_FIFO_SEL |
506 COMMAND_SEQ_READ_PAGE,
507 .addr0_row = page,
508 .addr0_col = page_off,
509 .len = real_len,
510 .ecc_offset = ECC_OFFSET(mtd->writesize + ecc_off),
511 };
512 unsigned int max_bitflips = 0, i;
513 u32 ecc_stat;
514 int bf, ret;
515
516 /* Prepare controller */
517 rnandc_select_target(chip, chip->cur_cs);
518 rnandc_clear_status(rnandc);
519 rnandc_en_correction(rnandc);
520 rnandc_trigger_op(rnandc, &rop);
521
522 while (!FIFO_STATE_C_EMPTY(readl(rnandc->regs + FIFO_STATE_REG)))
523 cpu_relax();
524
525 while (FIFO_STATE_R_EMPTY(readl(rnandc->regs + FIFO_STATE_REG)))
526 cpu_relax();
527
528 ioread32_rep(rnandc->regs + FIFO_DATA_REG, bufpoi + page_off,
529 real_len / 4);
530
531 if (!FIFO_STATE_R_EMPTY(readl(rnandc->regs + FIFO_STATE_REG))) {
532 dev_err(rnandc->dev, "Clearing residual data in the read FIFO\n");
533 rnandc_clear_fifo(rnandc);
534 }
535
536 ret = rnandc_wait_end_of_op(rnandc, chip);
537 rnandc_dis_correction(rnandc);
538 if (ret) {
539 dev_err(rnandc->dev, "Read subpage operation never ending\n");
540 return ret;
541 }
542
543 ecc_stat = readl_relaxed(rnandc->regs + ECC_STAT_REG);
544
545 if (ECC_STAT_UNCORRECTABLE(cs, ecc_stat)) {
546 ret = nand_change_read_column_op(chip, mtd->writesize,
547 chip->oob_poi, mtd->oobsize,
548 false);
549 if (ret)
550 return ret;
551
552 for (i = start_chunk; i < nchunks; i++) {
553 unsigned int dataoff = i * chip->ecc.size;
554 unsigned int eccoff = 2 + (i * chip->ecc.bytes);
555
556 bf = nand_check_erased_ecc_chunk(bufpoi + dataoff,
557 chip->ecc.size,
558 chip->oob_poi + eccoff,
559 chip->ecc.bytes,
560 NULL, 0,
561 chip->ecc.strength);
562 if (bf < 0) {
563 mtd->ecc_stats.failed++;
564 } else {
565 mtd->ecc_stats.corrected += bf;
566 max_bitflips = max_t(unsigned int, max_bitflips, bf);
567 }
568 }
569 } else if (ECC_STAT_CORRECTABLE(cs, ecc_stat)) {
570 bf = ECC_CNT(cs, readl_relaxed(rnandc->regs + ECC_CNT_REG));
571 /*
572 * The number of bitflips is an approximation given the fact
573 * that this controller does not provide per-chunk details but
574 * only gives statistics on the entire page.
575 */
576 mtd->ecc_stats.corrected += bf;
577 }
578
579 return 0;
580 }
581
rnandc_write_page_hw_ecc(struct nand_chip * chip,const u8 * buf,int oob_required,int page)582 static int rnandc_write_page_hw_ecc(struct nand_chip *chip, const u8 *buf,
583 int oob_required, int page)
584 {
585 struct rnandc *rnandc = to_rnandc(chip->controller);
586 struct mtd_info *mtd = nand_to_mtd(chip);
587 struct rnand_chip *rnand = to_rnand(chip);
588 unsigned int cs = to_rnandc_cs(rnand);
589 struct rnandc_op rop = {
590 .command = COMMAND_INPUT_SEL_DMA | COMMAND_0(NAND_CMD_SEQIN) |
591 COMMAND_1(NAND_CMD_PAGEPROG) | COMMAND_FIFO_SEL |
592 COMMAND_SEQ_WRITE_PAGE,
593 .addr0_row = page,
594 .len = mtd->writesize,
595 .ecc_offset = ECC_OFFSET(mtd->writesize + 2),
596 };
597 dma_addr_t dma_addr;
598 int ret;
599
600 memcpy(rnandc->buf, buf, mtd->writesize);
601
602 /* Prepare controller */
603 rnandc_select_target(chip, chip->cur_cs);
604 rnandc_clear_status(rnandc);
605 reinit_completion(&rnandc->complete);
606 rnandc_en_interrupts(rnandc, INT_MEM_RDY(cs));
607 rnandc_en_correction(rnandc);
608
609 /* Configure DMA */
610 dma_addr = dma_map_single(rnandc->dev, (void *)rnandc->buf, mtd->writesize,
611 DMA_TO_DEVICE);
612 if (dma_mapping_error(rnandc->dev, dma_addr))
613 return -ENOMEM;
614
615 writel(dma_addr, rnandc->regs + DMA_ADDR_LOW_REG);
616 writel(mtd->writesize, rnandc->regs + DMA_CNT_REG);
617 writel(DMA_TLVL_MAX, rnandc->regs + DMA_TLVL_REG);
618
619 rnandc_trigger_op(rnandc, &rop);
620 rnandc_trigger_dma(rnandc);
621
622 ret = rnandc_wait_end_of_io(rnandc, chip);
623 dma_unmap_single(rnandc->dev, dma_addr, mtd->writesize, DMA_TO_DEVICE);
624 rnandc_dis_correction(rnandc);
625 if (ret) {
626 dev_err(rnandc->dev, "Write page operation never ending\n");
627 return ret;
628 }
629
630 if (!oob_required)
631 return 0;
632
633 return nand_change_write_column_op(chip, mtd->writesize, chip->oob_poi,
634 mtd->oobsize, false);
635 }
636
rnandc_write_subpage_hw_ecc(struct nand_chip * chip,u32 req_offset,u32 req_len,const u8 * bufpoi,int oob_required,int page)637 static int rnandc_write_subpage_hw_ecc(struct nand_chip *chip, u32 req_offset,
638 u32 req_len, const u8 *bufpoi,
639 int oob_required, int page)
640 {
641 struct rnandc *rnandc = to_rnandc(chip->controller);
642 struct mtd_info *mtd = nand_to_mtd(chip);
643 unsigned int page_off = round_down(req_offset, chip->ecc.size);
644 unsigned int real_len = round_up(req_offset + req_len - page_off,
645 chip->ecc.size);
646 unsigned int start_chunk = page_off / chip->ecc.size;
647 unsigned int ecc_off = 2 + (start_chunk * chip->ecc.bytes);
648 struct rnandc_op rop = {
649 .command = COMMAND_INPUT_SEL_AHBS | COMMAND_0(NAND_CMD_SEQIN) |
650 COMMAND_1(NAND_CMD_PAGEPROG) | COMMAND_FIFO_SEL |
651 COMMAND_SEQ_WRITE_PAGE,
652 .addr0_row = page,
653 .addr0_col = page_off,
654 .len = real_len,
655 .ecc_offset = ECC_OFFSET(mtd->writesize + ecc_off),
656 };
657 int ret;
658
659 /* Prepare controller */
660 rnandc_select_target(chip, chip->cur_cs);
661 rnandc_clear_status(rnandc);
662 rnandc_en_correction(rnandc);
663 rnandc_trigger_op(rnandc, &rop);
664
665 while (FIFO_STATE_W_FULL(readl(rnandc->regs + FIFO_STATE_REG)))
666 cpu_relax();
667
668 iowrite32_rep(rnandc->regs + FIFO_DATA_REG, bufpoi + page_off,
669 real_len / 4);
670
671 while (!FIFO_STATE_W_EMPTY(readl(rnandc->regs + FIFO_STATE_REG)))
672 cpu_relax();
673
674 ret = rnandc_wait_end_of_op(rnandc, chip);
675 rnandc_dis_correction(rnandc);
676 if (ret) {
677 dev_err(rnandc->dev, "Write subpage operation never ending\n");
678 return ret;
679 }
680
681 return 0;
682 }
683
684 /*
685 * This controller is simple enough and thus does not need to use the parser
686 * provided by the core, instead, handle every situation here.
687 */
rnandc_exec_op(struct nand_chip * chip,const struct nand_operation * op,bool check_only)688 static int rnandc_exec_op(struct nand_chip *chip,
689 const struct nand_operation *op, bool check_only)
690 {
691 struct rnandc *rnandc = to_rnandc(chip->controller);
692 const struct nand_op_instr *instr = NULL;
693 struct rnandc_op rop = {
694 .command = COMMAND_INPUT_SEL_AHBS,
695 .gen_seq_ctrl = GEN_SEQ_IMD_SEQ,
696 };
697 unsigned int cmd_phase = 0, addr_phase = 0, data_phase = 0,
698 delay_phase = 0, delays = 0;
699 unsigned int op_id, col_addrs, row_addrs, naddrs, remainder, words, i;
700 const u8 *addrs;
701 u32 last_bytes;
702 int ret;
703
704 if (!check_only)
705 rnandc_select_target(chip, op->cs);
706
707 for (op_id = 0; op_id < op->ninstrs; op_id++) {
708 instr = &op->instrs[op_id];
709
710 nand_op_trace(" ", instr);
711
712 switch (instr->type) {
713 case NAND_OP_CMD_INSTR:
714 switch (cmd_phase++) {
715 case 0:
716 rop.command |= COMMAND_0(instr->ctx.cmd.opcode);
717 rop.gen_seq_ctrl |= GEN_SEQ_CMD0_EN;
718 break;
719 case 1:
720 rop.gen_seq_ctrl |= GEN_SEQ_COMMAND_3(instr->ctx.cmd.opcode);
721 rop.gen_seq_ctrl |= GEN_SEQ_CMD3_EN;
722 if (addr_phase == 0)
723 addr_phase = 1;
724 break;
725 case 2:
726 rop.command |= COMMAND_2(instr->ctx.cmd.opcode);
727 rop.gen_seq_ctrl |= GEN_SEQ_CMD2_EN;
728 if (addr_phase <= 1)
729 addr_phase = 2;
730 break;
731 case 3:
732 rop.command |= COMMAND_1(instr->ctx.cmd.opcode);
733 rop.gen_seq_ctrl |= GEN_SEQ_CMD1_EN;
734 if (addr_phase <= 1)
735 addr_phase = 2;
736 if (delay_phase == 0)
737 delay_phase = 1;
738 if (data_phase == 0)
739 data_phase = 1;
740 break;
741 default:
742 return -EOPNOTSUPP;
743 }
744 break;
745
746 case NAND_OP_ADDR_INSTR:
747 addrs = instr->ctx.addr.addrs;
748 naddrs = instr->ctx.addr.naddrs;
749 if (naddrs > 5)
750 return -EOPNOTSUPP;
751
752 col_addrs = min(2U, naddrs);
753 row_addrs = naddrs > 2 ? naddrs - col_addrs : 0;
754
755 switch (addr_phase++) {
756 case 0:
757 for (i = 0; i < col_addrs; i++)
758 rop.addr0_col |= addrs[i] << (i * 8);
759 rop.gen_seq_ctrl |= GEN_SEQ_COL_A0(col_addrs);
760
761 for (i = 0; i < row_addrs; i++)
762 rop.addr0_row |= addrs[2 + i] << (i * 8);
763 rop.gen_seq_ctrl |= GEN_SEQ_ROW_A0(row_addrs);
764
765 if (cmd_phase == 0)
766 cmd_phase = 1;
767 break;
768 case 1:
769 for (i = 0; i < col_addrs; i++)
770 rop.addr1_col |= addrs[i] << (i * 8);
771 rop.gen_seq_ctrl |= GEN_SEQ_COL_A1(col_addrs);
772
773 for (i = 0; i < row_addrs; i++)
774 rop.addr1_row |= addrs[2 + i] << (i * 8);
775 rop.gen_seq_ctrl |= GEN_SEQ_ROW_A1(row_addrs);
776
777 if (cmd_phase <= 1)
778 cmd_phase = 2;
779 break;
780 default:
781 return -EOPNOTSUPP;
782 }
783 break;
784
785 case NAND_OP_DATA_IN_INSTR:
786 rop.read = true;
787 fallthrough;
788 case NAND_OP_DATA_OUT_INSTR:
789 rop.gen_seq_ctrl |= GEN_SEQ_DATA_EN;
790 rop.buf = instr->ctx.data.buf.in;
791 rop.len = instr->ctx.data.len;
792 rop.command |= COMMAND_FIFO_SEL;
793
794 switch (data_phase++) {
795 case 0:
796 if (cmd_phase <= 2)
797 cmd_phase = 3;
798 if (addr_phase <= 1)
799 addr_phase = 2;
800 if (delay_phase == 0)
801 delay_phase = 1;
802 break;
803 default:
804 return -EOPNOTSUPP;
805 }
806 break;
807
808 case NAND_OP_WAITRDY_INSTR:
809 switch (delay_phase++) {
810 case 0:
811 rop.gen_seq_ctrl |= GEN_SEQ_DELAY0_EN;
812
813 if (cmd_phase <= 2)
814 cmd_phase = 3;
815 break;
816 case 1:
817 rop.gen_seq_ctrl |= GEN_SEQ_DELAY1_EN;
818
819 if (cmd_phase <= 3)
820 cmd_phase = 4;
821 if (data_phase == 0)
822 data_phase = 1;
823 break;
824 default:
825 return -EOPNOTSUPP;
826 }
827 break;
828 }
829 }
830
831 /*
832 * Sequence 19 is generic and dedicated to write operations.
833 * Sequence 18 is also generic and works for all other operations.
834 */
835 if (rop.buf && !rop.read)
836 rop.command |= COMMAND_SEQ_GEN_OUT;
837 else
838 rop.command |= COMMAND_SEQ_GEN_IN;
839
840 if (delays > 1) {
841 dev_err(rnandc->dev, "Cannot handle more than one wait delay\n");
842 return -EOPNOTSUPP;
843 }
844
845 if (check_only)
846 return 0;
847
848 rnandc_trigger_op(rnandc, &rop);
849
850 words = rop.len / sizeof(u32);
851 remainder = rop.len % sizeof(u32);
852 if (rop.buf && rop.read) {
853 while (!FIFO_STATE_C_EMPTY(readl(rnandc->regs + FIFO_STATE_REG)))
854 cpu_relax();
855
856 while (FIFO_STATE_R_EMPTY(readl(rnandc->regs + FIFO_STATE_REG)))
857 cpu_relax();
858
859 ioread32_rep(rnandc->regs + FIFO_DATA_REG, rop.buf, words);
860 if (remainder) {
861 last_bytes = readl_relaxed(rnandc->regs + FIFO_DATA_REG);
862 memcpy(rop.buf + (words * sizeof(u32)), &last_bytes,
863 remainder);
864 }
865
866 if (!FIFO_STATE_R_EMPTY(readl(rnandc->regs + FIFO_STATE_REG))) {
867 dev_warn(rnandc->dev,
868 "Clearing residual data in the read FIFO\n");
869 rnandc_clear_fifo(rnandc);
870 }
871 } else if (rop.len && !rop.read) {
872 while (FIFO_STATE_W_FULL(readl(rnandc->regs + FIFO_STATE_REG)))
873 cpu_relax();
874
875 iowrite32_rep(rnandc->regs + FIFO_DATA_REG, rop.buf,
876 DIV_ROUND_UP(rop.len, 4));
877
878 if (remainder) {
879 last_bytes = 0;
880 memcpy(&last_bytes, rop.buf + (words * sizeof(u32)), remainder);
881 writel_relaxed(last_bytes, rnandc->regs + FIFO_DATA_REG);
882 }
883
884 while (!FIFO_STATE_W_EMPTY(readl(rnandc->regs + FIFO_STATE_REG)))
885 cpu_relax();
886 }
887
888 ret = rnandc_wait_end_of_op(rnandc, chip);
889 if (ret)
890 return ret;
891
892 return 0;
893 }
894
rnandc_setup_interface(struct nand_chip * chip,int chipnr,const struct nand_interface_config * conf)895 static int rnandc_setup_interface(struct nand_chip *chip, int chipnr,
896 const struct nand_interface_config *conf)
897 {
898 struct rnand_chip *rnand = to_rnand(chip);
899 struct rnandc *rnandc = to_rnandc(chip->controller);
900 unsigned int period_ns = 1000000000 / rnandc->ext_clk_rate;
901 const struct nand_sdr_timings *sdr;
902 unsigned int cyc, cle, ale, bef_dly, ca_to_data;
903
904 sdr = nand_get_sdr_timings(conf);
905 if (IS_ERR(sdr))
906 return PTR_ERR(sdr);
907
908 if (sdr->tRP_min != sdr->tWP_min || sdr->tREH_min != sdr->tWH_min) {
909 dev_err(rnandc->dev, "Read and write hold times must be identical\n");
910 return -EINVAL;
911 }
912
913 if (chipnr < 0)
914 return 0;
915
916 rnand->timings_asyn =
917 TIMINGS_ASYN_TRWP(TO_CYCLES64(sdr->tRP_min, period_ns)) |
918 TIMINGS_ASYN_TRWH(TO_CYCLES64(sdr->tREH_min, period_ns));
919 rnand->tim_seq0 =
920 TIM_SEQ0_TCCS(TO_CYCLES64(sdr->tCCS_min, period_ns)) |
921 TIM_SEQ0_TADL(TO_CYCLES64(sdr->tADL_min, period_ns)) |
922 TIM_SEQ0_TRHW(TO_CYCLES64(sdr->tRHW_min, period_ns)) |
923 TIM_SEQ0_TWHR(TO_CYCLES64(sdr->tWHR_min, period_ns));
924 rnand->tim_seq1 =
925 TIM_SEQ1_TWB(TO_CYCLES64(sdr->tWB_max, period_ns)) |
926 TIM_SEQ1_TRR(TO_CYCLES64(sdr->tRR_min, period_ns)) |
927 TIM_SEQ1_TWW(TO_CYCLES64(sdr->tWW_min, period_ns));
928
929 cyc = sdr->tDS_min + sdr->tDH_min;
930 cle = sdr->tCLH_min + sdr->tCLS_min;
931 ale = sdr->tALH_min + sdr->tALS_min;
932 bef_dly = sdr->tWB_max - sdr->tDH_min;
933 ca_to_data = sdr->tWHR_min + sdr->tREA_max - sdr->tDH_min;
934
935 /*
936 * D0 = CMD -> ADDR = tCLH + tCLS - 1 cycle
937 * D1 = CMD -> CMD = tCLH + tCLS - 1 cycle
938 * D2 = CMD -> DLY = tWB - tDH
939 * D3 = CMD -> DATA = tWHR + tREA - tDH
940 */
941 rnand->tim_gen_seq0 =
942 TIM_GEN_SEQ0_D0(TO_CYCLES64(cle - cyc, period_ns)) |
943 TIM_GEN_SEQ0_D1(TO_CYCLES64(cle - cyc, period_ns)) |
944 TIM_GEN_SEQ0_D2(TO_CYCLES64(bef_dly, period_ns)) |
945 TIM_GEN_SEQ0_D3(TO_CYCLES64(ca_to_data, period_ns));
946
947 /*
948 * D4 = ADDR -> CMD = tALH + tALS - 1 cyle
949 * D5 = ADDR -> ADDR = tALH + tALS - 1 cyle
950 * D6 = ADDR -> DLY = tWB - tDH
951 * D7 = ADDR -> DATA = tWHR + tREA - tDH
952 */
953 rnand->tim_gen_seq1 =
954 TIM_GEN_SEQ1_D4(TO_CYCLES64(ale - cyc, period_ns)) |
955 TIM_GEN_SEQ1_D5(TO_CYCLES64(ale - cyc, period_ns)) |
956 TIM_GEN_SEQ1_D6(TO_CYCLES64(bef_dly, period_ns)) |
957 TIM_GEN_SEQ1_D7(TO_CYCLES64(ca_to_data, period_ns));
958
959 /*
960 * D8 = DLY -> DATA = tRR + tREA
961 * D9 = DLY -> CMD = tRR
962 * D10 = DATA -> CMD = tCLH + tCLS - 1 cycle
963 * D11 = DATA -> DLY = tWB - tDH
964 */
965 rnand->tim_gen_seq2 =
966 TIM_GEN_SEQ2_D8(TO_CYCLES64(sdr->tRR_min + sdr->tREA_max, period_ns)) |
967 TIM_GEN_SEQ2_D9(TO_CYCLES64(sdr->tRR_min, period_ns)) |
968 TIM_GEN_SEQ2_D10(TO_CYCLES64(cle - cyc, period_ns)) |
969 TIM_GEN_SEQ2_D11(TO_CYCLES64(bef_dly, period_ns));
970
971 /* D12 = DATA -> END = tCLH - tDH */
972 rnand->tim_gen_seq3 =
973 TIM_GEN_SEQ3_D12(TO_CYCLES64(sdr->tCLH_min - sdr->tDH_min, period_ns));
974
975 return 0;
976 }
977
rnandc_ooblayout_ecc(struct mtd_info * mtd,int section,struct mtd_oob_region * oobregion)978 static int rnandc_ooblayout_ecc(struct mtd_info *mtd, int section,
979 struct mtd_oob_region *oobregion)
980 {
981 struct nand_chip *chip = mtd_to_nand(mtd);
982 unsigned int eccbytes = round_up(chip->ecc.bytes, 4) * chip->ecc.steps;
983
984 if (section)
985 return -ERANGE;
986
987 oobregion->offset = 2;
988 oobregion->length = eccbytes;
989
990 return 0;
991 }
992
rnandc_ooblayout_free(struct mtd_info * mtd,int section,struct mtd_oob_region * oobregion)993 static int rnandc_ooblayout_free(struct mtd_info *mtd, int section,
994 struct mtd_oob_region *oobregion)
995 {
996 struct nand_chip *chip = mtd_to_nand(mtd);
997 unsigned int eccbytes = round_up(chip->ecc.bytes, 4) * chip->ecc.steps;
998
999 if (section)
1000 return -ERANGE;
1001
1002 oobregion->offset = 2 + eccbytes;
1003 oobregion->length = mtd->oobsize - oobregion->offset;
1004
1005 return 0;
1006 }
1007
1008 static const struct mtd_ooblayout_ops rnandc_ooblayout_ops = {
1009 .ecc = rnandc_ooblayout_ecc,
1010 .free = rnandc_ooblayout_free,
1011 };
1012
rnandc_hw_ecc_controller_init(struct nand_chip * chip)1013 static int rnandc_hw_ecc_controller_init(struct nand_chip *chip)
1014 {
1015 struct rnand_chip *rnand = to_rnand(chip);
1016 struct mtd_info *mtd = nand_to_mtd(chip);
1017 struct rnandc *rnandc = to_rnandc(chip->controller);
1018
1019 if (mtd->writesize > SZ_16K) {
1020 dev_err(rnandc->dev, "Unsupported page size\n");
1021 return -EINVAL;
1022 }
1023
1024 switch (chip->ecc.size) {
1025 case SZ_256:
1026 rnand->control |= CONTROL_ECC_BLOCK_SIZE_256;
1027 break;
1028 case SZ_512:
1029 rnand->control |= CONTROL_ECC_BLOCK_SIZE_512;
1030 break;
1031 case SZ_1K:
1032 rnand->control |= CONTROL_ECC_BLOCK_SIZE_1024;
1033 break;
1034 default:
1035 dev_err(rnandc->dev, "Unsupported ECC chunk size\n");
1036 return -EINVAL;
1037 }
1038
1039 switch (chip->ecc.strength) {
1040 case 2:
1041 chip->ecc.bytes = 4;
1042 rnand->ecc_ctrl |= ECC_CTRL_CAP_2B;
1043 break;
1044 case 4:
1045 chip->ecc.bytes = 7;
1046 rnand->ecc_ctrl |= ECC_CTRL_CAP_4B;
1047 break;
1048 case 8:
1049 chip->ecc.bytes = 14;
1050 rnand->ecc_ctrl |= ECC_CTRL_CAP_8B;
1051 break;
1052 case 16:
1053 chip->ecc.bytes = 28;
1054 rnand->ecc_ctrl |= ECC_CTRL_CAP_16B;
1055 break;
1056 case 24:
1057 chip->ecc.bytes = 42;
1058 rnand->ecc_ctrl |= ECC_CTRL_CAP_24B;
1059 break;
1060 case 32:
1061 chip->ecc.bytes = 56;
1062 rnand->ecc_ctrl |= ECC_CTRL_CAP_32B;
1063 break;
1064 default:
1065 dev_err(rnandc->dev, "Unsupported ECC strength\n");
1066 return -EINVAL;
1067 }
1068
1069 rnand->ecc_ctrl |= ECC_CTRL_ERR_THRESHOLD(chip->ecc.strength);
1070
1071 mtd_set_ooblayout(mtd, &rnandc_ooblayout_ops);
1072 chip->ecc.steps = mtd->writesize / chip->ecc.size;
1073 chip->ecc.read_page = rnandc_read_page_hw_ecc;
1074 chip->ecc.read_subpage = rnandc_read_subpage_hw_ecc;
1075 chip->ecc.write_page = rnandc_write_page_hw_ecc;
1076 chip->ecc.write_subpage = rnandc_write_subpage_hw_ecc;
1077
1078 return 0;
1079 }
1080
rnandc_ecc_init(struct nand_chip * chip)1081 static int rnandc_ecc_init(struct nand_chip *chip)
1082 {
1083 struct nand_ecc_ctrl *ecc = &chip->ecc;
1084 const struct nand_ecc_props *requirements =
1085 nanddev_get_ecc_requirements(&chip->base);
1086 struct rnandc *rnandc = to_rnandc(chip->controller);
1087 int ret;
1088
1089 if (ecc->engine_type != NAND_ECC_ENGINE_TYPE_NONE &&
1090 (!ecc->size || !ecc->strength)) {
1091 if (requirements->step_size && requirements->strength) {
1092 ecc->size = requirements->step_size;
1093 ecc->strength = requirements->strength;
1094 } else {
1095 dev_err(rnandc->dev, "No minimum ECC strength\n");
1096 return -EINVAL;
1097 }
1098 }
1099
1100 switch (ecc->engine_type) {
1101 case NAND_ECC_ENGINE_TYPE_ON_HOST:
1102 ret = rnandc_hw_ecc_controller_init(chip);
1103 if (ret)
1104 return ret;
1105 break;
1106 case NAND_ECC_ENGINE_TYPE_NONE:
1107 case NAND_ECC_ENGINE_TYPE_SOFT:
1108 case NAND_ECC_ENGINE_TYPE_ON_DIE:
1109 break;
1110 default:
1111 return -EINVAL;
1112 }
1113
1114 return 0;
1115 }
1116
rnandc_attach_chip(struct nand_chip * chip)1117 static int rnandc_attach_chip(struct nand_chip *chip)
1118 {
1119 struct rnand_chip *rnand = to_rnand(chip);
1120 struct rnandc *rnandc = to_rnandc(chip->controller);
1121 struct mtd_info *mtd = nand_to_mtd(chip);
1122 struct nand_memory_organization *memorg = nanddev_get_memorg(&chip->base);
1123 int ret;
1124
1125 /* Do not store BBT bits in the OOB section as it is not protected */
1126 if (chip->bbt_options & NAND_BBT_USE_FLASH)
1127 chip->bbt_options |= NAND_BBT_NO_OOB;
1128
1129 if (mtd->writesize <= 512) {
1130 dev_err(rnandc->dev, "Small page devices not supported\n");
1131 return -EINVAL;
1132 }
1133
1134 rnand->control |= CONTROL_CHECK_RB_LINE | CONTROL_INT_EN;
1135
1136 switch (memorg->pages_per_eraseblock) {
1137 case 32:
1138 rnand->control |= CONTROL_BLOCK_SIZE_32P;
1139 break;
1140 case 64:
1141 rnand->control |= CONTROL_BLOCK_SIZE_64P;
1142 break;
1143 case 128:
1144 rnand->control |= CONTROL_BLOCK_SIZE_128P;
1145 break;
1146 case 256:
1147 rnand->control |= CONTROL_BLOCK_SIZE_256P;
1148 break;
1149 default:
1150 dev_err(rnandc->dev, "Unsupported memory organization\n");
1151 return -EINVAL;
1152 }
1153
1154 chip->options |= NAND_SUBPAGE_READ;
1155
1156 ret = rnandc_ecc_init(chip);
1157 if (ret) {
1158 dev_err(rnandc->dev, "ECC initialization failed (%d)\n", ret);
1159 return ret;
1160 }
1161
1162 /* Force an update of the configuration registers */
1163 rnand->selected_die = -1;
1164
1165 return 0;
1166 }
1167
1168 static const struct nand_controller_ops rnandc_ops = {
1169 .attach_chip = rnandc_attach_chip,
1170 .exec_op = rnandc_exec_op,
1171 .setup_interface = rnandc_setup_interface,
1172 };
1173
rnandc_alloc_dma_buf(struct rnandc * rnandc,struct mtd_info * new_mtd)1174 static int rnandc_alloc_dma_buf(struct rnandc *rnandc,
1175 struct mtd_info *new_mtd)
1176 {
1177 unsigned int max_len = new_mtd->writesize + new_mtd->oobsize;
1178 struct rnand_chip *entry, *temp;
1179 struct nand_chip *chip;
1180 struct mtd_info *mtd;
1181
1182 list_for_each_entry_safe(entry, temp, &rnandc->chips, node) {
1183 chip = &entry->chip;
1184 mtd = nand_to_mtd(chip);
1185 max_len = max(max_len, mtd->writesize + mtd->oobsize);
1186 }
1187
1188 if (rnandc->buf && rnandc->buf_sz < max_len) {
1189 devm_kfree(rnandc->dev, rnandc->buf);
1190 rnandc->buf = NULL;
1191 }
1192
1193 if (!rnandc->buf) {
1194 rnandc->buf_sz = max_len;
1195 rnandc->buf = devm_kmalloc(rnandc->dev, max_len,
1196 GFP_KERNEL | GFP_DMA);
1197 if (!rnandc->buf)
1198 return -ENOMEM;
1199 }
1200
1201 return 0;
1202 }
1203
rnandc_chip_init(struct rnandc * rnandc,struct device_node * np)1204 static int rnandc_chip_init(struct rnandc *rnandc, struct device_node *np)
1205 {
1206 struct rnand_chip *rnand;
1207 struct mtd_info *mtd;
1208 struct nand_chip *chip;
1209 int nsels, ret, i;
1210 u32 cs;
1211
1212 nsels = of_property_count_elems_of_size(np, "reg", sizeof(u32));
1213 if (nsels <= 0) {
1214 ret = (nsels < 0) ? nsels : -EINVAL;
1215 dev_err(rnandc->dev, "Invalid reg property (%d)\n", ret);
1216 return ret;
1217 }
1218
1219 /* Alloc the driver's NAND chip structure */
1220 rnand = devm_kzalloc(rnandc->dev, struct_size(rnand, sels, nsels),
1221 GFP_KERNEL);
1222 if (!rnand)
1223 return -ENOMEM;
1224
1225 rnand->nsels = nsels;
1226 rnand->selected_die = -1;
1227
1228 for (i = 0; i < nsels; i++) {
1229 ret = of_property_read_u32_index(np, "reg", i, &cs);
1230 if (ret) {
1231 dev_err(rnandc->dev, "Incomplete reg property (%d)\n", ret);
1232 return ret;
1233 }
1234
1235 if (cs >= RNANDC_CS_NUM) {
1236 dev_err(rnandc->dev, "Invalid reg property (%d)\n", cs);
1237 return -EINVAL;
1238 }
1239
1240 if (test_and_set_bit(cs, &rnandc->assigned_cs)) {
1241 dev_err(rnandc->dev, "CS %d already assigned\n", cs);
1242 return -EINVAL;
1243 }
1244
1245 /*
1246 * No need to check for RB or WP properties, there is a 1:1
1247 * mandatory mapping with the CS.
1248 */
1249 rnand->sels[i].cs = cs;
1250 }
1251
1252 chip = &rnand->chip;
1253 chip->controller = &rnandc->controller;
1254 nand_set_flash_node(chip, np);
1255
1256 mtd = nand_to_mtd(chip);
1257 mtd->dev.parent = rnandc->dev;
1258 if (!mtd->name) {
1259 dev_err(rnandc->dev, "Missing MTD label\n");
1260 return -EINVAL;
1261 }
1262
1263 ret = nand_scan(chip, rnand->nsels);
1264 if (ret) {
1265 dev_err(rnandc->dev, "Failed to scan the NAND chip (%d)\n", ret);
1266 return ret;
1267 }
1268
1269 ret = rnandc_alloc_dma_buf(rnandc, mtd);
1270 if (ret)
1271 goto cleanup_nand;
1272
1273 ret = mtd_device_register(mtd, NULL, 0);
1274 if (ret) {
1275 dev_err(rnandc->dev, "Failed to register MTD device (%d)\n", ret);
1276 goto cleanup_nand;
1277 }
1278
1279 list_add_tail(&rnand->node, &rnandc->chips);
1280
1281 return 0;
1282
1283 cleanup_nand:
1284 nand_cleanup(chip);
1285
1286 return ret;
1287 }
1288
rnandc_chips_cleanup(struct rnandc * rnandc)1289 static void rnandc_chips_cleanup(struct rnandc *rnandc)
1290 {
1291 struct rnand_chip *entry, *temp;
1292 struct nand_chip *chip;
1293 int ret;
1294
1295 list_for_each_entry_safe(entry, temp, &rnandc->chips, node) {
1296 chip = &entry->chip;
1297 ret = mtd_device_unregister(nand_to_mtd(chip));
1298 WARN_ON(ret);
1299 nand_cleanup(chip);
1300 list_del(&entry->node);
1301 }
1302 }
1303
rnandc_chips_init(struct rnandc * rnandc)1304 static int rnandc_chips_init(struct rnandc *rnandc)
1305 {
1306 int ret;
1307
1308 for_each_child_of_node_scoped(rnandc->dev->of_node, np) {
1309 ret = rnandc_chip_init(rnandc, np);
1310 if (ret) {
1311 rnandc_chips_cleanup(rnandc);
1312 return ret;
1313 }
1314 }
1315
1316 return 0;
1317 }
1318
rnandc_probe(struct platform_device * pdev)1319 static int rnandc_probe(struct platform_device *pdev)
1320 {
1321 struct rnandc *rnandc;
1322 struct clk *eclk;
1323 int irq, ret;
1324
1325 rnandc = devm_kzalloc(&pdev->dev, sizeof(*rnandc), GFP_KERNEL);
1326 if (!rnandc)
1327 return -ENOMEM;
1328
1329 rnandc->dev = &pdev->dev;
1330 nand_controller_init(&rnandc->controller);
1331 rnandc->controller.ops = &rnandc_ops;
1332 INIT_LIST_HEAD(&rnandc->chips);
1333 init_completion(&rnandc->complete);
1334
1335 rnandc->regs = devm_platform_ioremap_resource(pdev, 0);
1336 if (IS_ERR(rnandc->regs))
1337 return PTR_ERR(rnandc->regs);
1338
1339 devm_pm_runtime_enable(&pdev->dev);
1340 ret = pm_runtime_resume_and_get(&pdev->dev);
1341 if (ret < 0)
1342 return ret;
1343
1344 /* The external NAND bus clock rate is needed for computing timings */
1345 eclk = clk_get(&pdev->dev, "eclk");
1346 if (IS_ERR(eclk)) {
1347 ret = PTR_ERR(eclk);
1348 goto dis_runtime_pm;
1349 }
1350
1351 rnandc->ext_clk_rate = clk_get_rate(eclk);
1352 clk_put(eclk);
1353
1354 rnandc_dis_interrupts(rnandc);
1355 irq = platform_get_irq_optional(pdev, 0);
1356 if (irq == -EPROBE_DEFER) {
1357 ret = irq;
1358 goto dis_runtime_pm;
1359 } else if (irq < 0) {
1360 dev_info(&pdev->dev, "No IRQ found, fallback to polling\n");
1361 rnandc->use_polling = true;
1362 } else {
1363 ret = devm_request_irq(&pdev->dev, irq, rnandc_irq_handler, 0,
1364 "renesas-nand-controller", rnandc);
1365 if (ret < 0)
1366 goto dis_runtime_pm;
1367 }
1368
1369 ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
1370 if (ret)
1371 goto dis_runtime_pm;
1372
1373 rnandc_clear_fifo(rnandc);
1374
1375 platform_set_drvdata(pdev, rnandc);
1376
1377 ret = rnandc_chips_init(rnandc);
1378 if (ret)
1379 goto dis_runtime_pm;
1380
1381 return 0;
1382
1383 dis_runtime_pm:
1384 pm_runtime_put(&pdev->dev);
1385
1386 return ret;
1387 }
1388
rnandc_remove(struct platform_device * pdev)1389 static void rnandc_remove(struct platform_device *pdev)
1390 {
1391 struct rnandc *rnandc = platform_get_drvdata(pdev);
1392
1393 rnandc_chips_cleanup(rnandc);
1394
1395 pm_runtime_put(&pdev->dev);
1396 }
1397
1398 static const struct of_device_id rnandc_id_table[] = {
1399 { .compatible = "renesas,rcar-gen3-nandc" },
1400 { .compatible = "renesas,rzn1-nandc" },
1401 {} /* sentinel */
1402 };
1403 MODULE_DEVICE_TABLE(of, rnandc_id_table);
1404
1405 static struct platform_driver rnandc_driver = {
1406 .driver = {
1407 .name = "renesas-nandc",
1408 .of_match_table = rnandc_id_table,
1409 },
1410 .probe = rnandc_probe,
1411 .remove = rnandc_remove,
1412 };
1413 module_platform_driver(rnandc_driver);
1414
1415 MODULE_AUTHOR("Miquel Raynal <miquel.raynal@bootlin.com>");
1416 MODULE_DESCRIPTION("Renesas R-Car Gen3 & RZ/N1 NAND controller driver");
1417 MODULE_LICENSE("GPL v2");
1418