1 // SPDX-License-Identifier: GPL-2.0+
2 #include <linux/module.h>
3 #include <linux/pci.h>
4 #include <linux/types.h>
5 #include <linux/export.h>
6 #include <linux/slab.h>
7 #include <linux/io.h>
8 #include <linux/io-64-nonatomic-lo-hi.h>
9 #include <linux/mfd/core.h>
10 #include <linux/platform_device.h>
11 #include <linux/ioport.h>
12 #include <linux/uio_driver.h>
13 #include "pcie.h"
14
15 /* Core (Resource) Table Layout:
16 * one Resource per record (8 bytes)
17 * 6 5 4 3 2 1 0
18 * 3210987654321098765432109876543210987654321098765432109876543210
19 * IIIIIIIIIIII Core Type [up to 4095 types]
20 * D S2C DMA Present
21 * DDD S2C DMA Channel Number [up to 8 channels]
22 * LLLLLLLLLLLLLLLL Register Count (64-bit registers) [up to 65535 registers]
23 * OOOOOOOOOOOOOOOO Core Offset (in 4kB blocks) [up to 65535 cores]
24 * D C2S DMA Present
25 * DDD C2S DMA Channel Number [up to 8 channels]
26 * II IRQ Count [0 to 3 IRQs per core]
27 * 1111111000
28 * IIIIIII IRQ Base Number [up to 128 IRQs per card]
29 * ___ Spare
30 *
31 */
32
33 #define KPC_OLD_DMA_CH_NUM(present, channel) ((present) ? (0x8 | ((channel) & 0x7)) : 0)
34 #define KPC_OLD_S2C_DMA_CH_NUM(cte) KPC_OLD_DMA_CH_NUM(cte.s2c_dma_present, cte.s2c_dma_channel_num)
35 #define KPC_OLD_C2S_DMA_CH_NUM(cte) KPC_OLD_DMA_CH_NUM(cte.c2s_dma_present, cte.c2s_dma_channel_num)
36
37 #define KP_CORE_ID_INVALID 0
38 #define KP_CORE_ID_I2C 3
39 #define KP_CORE_ID_SPI 5
40
41 struct core_table_entry {
42 u16 type;
43 u32 offset;
44 u32 length;
45 bool s2c_dma_present;
46 u8 s2c_dma_channel_num;
47 bool c2s_dma_present;
48 u8 c2s_dma_channel_num;
49 u8 irq_count;
50 u8 irq_base_num;
51 };
52
53 static
parse_core_table_entry_v0(struct core_table_entry * cte,const u64 read_val)54 void parse_core_table_entry_v0(struct core_table_entry *cte, const u64 read_val)
55 {
56 cte->type = ((read_val & 0xFFF0000000000000UL) >> 52);
57 cte->offset = ((read_val & 0x00000000FFFF0000UL) >> 16) * 4096;
58 cte->length = ((read_val & 0x0000FFFF00000000UL) >> 32) * 8;
59 cte->s2c_dma_present = ((read_val & 0x0008000000000000UL) >> 51);
60 cte->s2c_dma_channel_num = ((read_val & 0x0007000000000000UL) >> 48);
61 cte->c2s_dma_present = ((read_val & 0x0000000000008000UL) >> 15);
62 cte->c2s_dma_channel_num = ((read_val & 0x0000000000007000UL) >> 12);
63 cte->irq_count = ((read_val & 0x0000000000000C00UL) >> 10);
64 cte->irq_base_num = ((read_val & 0x00000000000003F8UL) >> 3);
65 }
66
67 static
dbg_cte(struct kp2000_device * pcard,struct core_table_entry * cte)68 void dbg_cte(struct kp2000_device *pcard, struct core_table_entry *cte)
69 {
70 dev_dbg(&pcard->pdev->dev, "CTE: type:%3d offset:%3d (%3d) length:%3d (%3d) s2c:%d c2s:%d irq_count:%d base_irq:%d\n",
71 cte->type,
72 cte->offset,
73 cte->offset / 4096,
74 cte->length,
75 cte->length / 8,
76 (cte->s2c_dma_present ? cte->s2c_dma_channel_num : -1),
77 (cte->c2s_dma_present ? cte->c2s_dma_channel_num : -1),
78 cte->irq_count,
79 cte->irq_base_num
80 );
81 }
82
83 static
parse_core_table_entry(struct core_table_entry * cte,const u64 read_val,const u8 entry_rev)84 void parse_core_table_entry(struct core_table_entry *cte, const u64 read_val, const u8 entry_rev)
85 {
86 switch (entry_rev) {
87 case 0:
88 parse_core_table_entry_v0(cte, read_val);
89 break;
90 default:
91 cte->type = 0;
92 break;
93 }
94 }
95
probe_core_basic(unsigned int core_num,struct kp2000_device * pcard,char * name,const struct core_table_entry cte)96 static int probe_core_basic(unsigned int core_num, struct kp2000_device *pcard,
97 char *name, const struct core_table_entry cte)
98 {
99 struct mfd_cell cell = { .id = core_num, .name = name };
100 struct resource resources[2];
101
102 struct kpc_core_device_platdata core_pdata = {
103 .card_id = pcard->card_id,
104 .build_version = pcard->build_version,
105 .hardware_revision = pcard->hardware_revision,
106 .ssid = pcard->ssid,
107 .ddna = pcard->ddna,
108 };
109
110 dev_dbg(&pcard->pdev->dev, "Found Basic core: type = %02d dma = %02x / %02x offset = 0x%x length = 0x%x (%d regs)\n", cte.type, KPC_OLD_S2C_DMA_CH_NUM(cte), KPC_OLD_C2S_DMA_CH_NUM(cte), cte.offset, cte.length, cte.length / 8);
111
112 cell.platform_data = &core_pdata;
113 cell.pdata_size = sizeof(struct kpc_core_device_platdata);
114 cell.num_resources = 2;
115
116 memset(&resources, 0, sizeof(resources));
117
118 resources[0].start = cte.offset;
119 resources[0].end = cte.offset + (cte.length - 1);
120 resources[0].flags = IORESOURCE_MEM;
121
122 resources[1].start = pcard->pdev->irq;
123 resources[1].end = pcard->pdev->irq;
124 resources[1].flags = IORESOURCE_IRQ;
125
126 cell.resources = resources;
127
128 return mfd_add_devices(PCARD_TO_DEV(pcard), // parent
129 pcard->card_num * 100, // id
130 &cell, // struct mfd_cell *
131 1, // ndevs
132 &pcard->regs_base_resource,
133 0, // irq_base
134 NULL); // struct irq_domain *
135 }
136
137 struct kpc_uio_device {
138 struct list_head list;
139 struct kp2000_device *pcard;
140 struct device *dev;
141 struct uio_info uioinfo;
142 struct core_table_entry cte;
143 u16 core_num;
144 };
145
offset_show(struct device * dev,struct device_attribute * attr,char * buf)146 static ssize_t offset_show(struct device *dev, struct device_attribute *attr,
147 char *buf)
148 {
149 struct kpc_uio_device *kudev = dev_get_drvdata(dev);
150
151 return sprintf(buf, "%u\n", kudev->cte.offset);
152 }
153 static DEVICE_ATTR_RO(offset);
154
size_show(struct device * dev,struct device_attribute * attr,char * buf)155 static ssize_t size_show(struct device *dev, struct device_attribute *attr,
156 char *buf)
157 {
158 struct kpc_uio_device *kudev = dev_get_drvdata(dev);
159
160 return sprintf(buf, "%u\n", kudev->cte.length);
161 }
162 static DEVICE_ATTR_RO(size);
163
type_show(struct device * dev,struct device_attribute * attr,char * buf)164 static ssize_t type_show(struct device *dev, struct device_attribute *attr,
165 char *buf)
166 {
167 struct kpc_uio_device *kudev = dev_get_drvdata(dev);
168
169 return sprintf(buf, "%u\n", kudev->cte.type);
170 }
171 static DEVICE_ATTR_RO(type);
172
s2c_dma_show(struct device * dev,struct device_attribute * attr,char * buf)173 static ssize_t s2c_dma_show(struct device *dev, struct device_attribute *attr,
174 char *buf)
175 {
176 struct kpc_uio_device *kudev = dev_get_drvdata(dev);
177
178 if (!kudev->cte.s2c_dma_present)
179 return sprintf(buf, "%s", "not present\n");
180
181 return sprintf(buf, "%u\n", kudev->cte.s2c_dma_channel_num);
182 }
183 static DEVICE_ATTR_RO(s2c_dma);
184
c2s_dma_show(struct device * dev,struct device_attribute * attr,char * buf)185 static ssize_t c2s_dma_show(struct device *dev, struct device_attribute *attr,
186 char *buf)
187 {
188 struct kpc_uio_device *kudev = dev_get_drvdata(dev);
189
190 if (!kudev->cte.c2s_dma_present)
191 return sprintf(buf, "%s", "not present\n");
192
193 return sprintf(buf, "%u\n", kudev->cte.c2s_dma_channel_num);
194 }
195 static DEVICE_ATTR_RO(c2s_dma);
196
irq_count_show(struct device * dev,struct device_attribute * attr,char * buf)197 static ssize_t irq_count_show(struct device *dev, struct device_attribute *attr,
198 char *buf)
199 {
200 struct kpc_uio_device *kudev = dev_get_drvdata(dev);
201
202 return sprintf(buf, "%u\n", kudev->cte.irq_count);
203 }
204 static DEVICE_ATTR_RO(irq_count);
205
irq_base_num_show(struct device * dev,struct device_attribute * attr,char * buf)206 static ssize_t irq_base_num_show(struct device *dev,
207 struct device_attribute *attr, char *buf)
208 {
209 struct kpc_uio_device *kudev = dev_get_drvdata(dev);
210
211 return sprintf(buf, "%u\n", kudev->cte.irq_base_num);
212 }
213 static DEVICE_ATTR_RO(irq_base_num);
214
core_num_show(struct device * dev,struct device_attribute * attr,char * buf)215 static ssize_t core_num_show(struct device *dev, struct device_attribute *attr,
216 char *buf)
217 {
218 struct kpc_uio_device *kudev = dev_get_drvdata(dev);
219
220 return sprintf(buf, "%u\n", kudev->core_num);
221 }
222 static DEVICE_ATTR_RO(core_num);
223
224 struct attribute *kpc_uio_class_attrs[] = {
225 &dev_attr_offset.attr,
226 &dev_attr_size.attr,
227 &dev_attr_type.attr,
228 &dev_attr_s2c_dma.attr,
229 &dev_attr_c2s_dma.attr,
230 &dev_attr_irq_count.attr,
231 &dev_attr_irq_base_num.attr,
232 &dev_attr_core_num.attr,
233 NULL,
234 };
235
236 static
kp2000_check_uio_irq(struct kp2000_device * pcard,u32 irq_num)237 int kp2000_check_uio_irq(struct kp2000_device *pcard, u32 irq_num)
238 {
239 u64 interrupt_active = readq(pcard->sysinfo_regs_base + REG_INTERRUPT_ACTIVE);
240 u64 interrupt_mask_inv = ~readq(pcard->sysinfo_regs_base + REG_INTERRUPT_MASK);
241 u64 irq_check_mask = BIT_ULL(irq_num);
242
243 if (interrupt_active & irq_check_mask) { // if it's active (interrupt pending)
244 if (interrupt_mask_inv & irq_check_mask) { // and if it's not masked off
245 return 1;
246 }
247 }
248 return 0;
249 }
250
251 static
kuio_handler(int irq,struct uio_info * uioinfo)252 irqreturn_t kuio_handler(int irq, struct uio_info *uioinfo)
253 {
254 struct kpc_uio_device *kudev = uioinfo->priv;
255
256 if (irq != kudev->pcard->pdev->irq)
257 return IRQ_NONE;
258
259 if (kp2000_check_uio_irq(kudev->pcard, kudev->cte.irq_base_num)) {
260 /* Clear the active flag */
261 writeq(BIT_ULL(kudev->cte.irq_base_num),
262 kudev->pcard->sysinfo_regs_base + REG_INTERRUPT_ACTIVE);
263 return IRQ_HANDLED;
264 }
265 return IRQ_NONE;
266 }
267
268 static
kuio_irqcontrol(struct uio_info * uioinfo,s32 irq_on)269 int kuio_irqcontrol(struct uio_info *uioinfo, s32 irq_on)
270 {
271 struct kpc_uio_device *kudev = uioinfo->priv;
272 struct kp2000_device *pcard = kudev->pcard;
273 u64 mask;
274
275 mutex_lock(&pcard->sem);
276 mask = readq(pcard->sysinfo_regs_base + REG_INTERRUPT_MASK);
277 if (irq_on)
278 mask &= ~(BIT_ULL(kudev->cte.irq_base_num));
279 else
280 mask |= BIT_ULL(kudev->cte.irq_base_num);
281 writeq(mask, pcard->sysinfo_regs_base + REG_INTERRUPT_MASK);
282 mutex_unlock(&pcard->sem);
283
284 return 0;
285 }
286
probe_core_uio(unsigned int core_num,struct kp2000_device * pcard,char * name,const struct core_table_entry cte)287 static int probe_core_uio(unsigned int core_num, struct kp2000_device *pcard,
288 char *name, const struct core_table_entry cte)
289 {
290 struct kpc_uio_device *kudev;
291 int rv;
292
293 dev_dbg(&pcard->pdev->dev, "Found UIO core: type = %02d dma = %02x / %02x offset = 0x%x length = 0x%x (%d regs)\n", cte.type, KPC_OLD_S2C_DMA_CH_NUM(cte), KPC_OLD_C2S_DMA_CH_NUM(cte), cte.offset, cte.length, cte.length / 8);
294
295 kudev = kzalloc(sizeof(*kudev), GFP_KERNEL);
296 if (!kudev)
297 return -ENOMEM;
298
299 INIT_LIST_HEAD(&kudev->list);
300 kudev->pcard = pcard;
301 kudev->cte = cte;
302 kudev->core_num = core_num;
303
304 kudev->uioinfo.priv = kudev;
305 kudev->uioinfo.name = name;
306 kudev->uioinfo.version = "0.0";
307 if (cte.irq_count > 0) {
308 kudev->uioinfo.irq_flags = IRQF_SHARED;
309 kudev->uioinfo.irq = pcard->pdev->irq;
310 kudev->uioinfo.handler = kuio_handler;
311 kudev->uioinfo.irqcontrol = kuio_irqcontrol;
312 } else {
313 kudev->uioinfo.irq = 0;
314 }
315
316 kudev->uioinfo.mem[0].name = "uiomap";
317 kudev->uioinfo.mem[0].addr = pci_resource_start(pcard->pdev, REG_BAR) + cte.offset;
318 kudev->uioinfo.mem[0].size = (cte.length + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); // Round up to nearest PAGE_SIZE boundary
319 kudev->uioinfo.mem[0].memtype = UIO_MEM_PHYS;
320
321 kudev->dev = device_create(kpc_uio_class, &pcard->pdev->dev, MKDEV(0, 0), kudev, "%s.%d.%d.%d", kudev->uioinfo.name, pcard->card_num, cte.type, kudev->core_num);
322 if (IS_ERR(kudev->dev)) {
323 dev_err(&pcard->pdev->dev, "%s: device_create failed!\n",
324 __func__);
325 kfree(kudev);
326 return -ENODEV;
327 }
328 dev_set_drvdata(kudev->dev, kudev);
329
330 rv = uio_register_device(kudev->dev, &kudev->uioinfo);
331 if (rv) {
332 dev_err(&pcard->pdev->dev, "%s: failed uio_register_device: %d\n",
333 __func__, rv);
334 put_device(kudev->dev);
335 kfree(kudev);
336 return rv;
337 }
338
339 list_add_tail(&kudev->list, &pcard->uio_devices_list);
340
341 return 0;
342 }
343
create_dma_engine_core(struct kp2000_device * pcard,size_t engine_regs_offset,int engine_num,int irq_num)344 static int create_dma_engine_core(struct kp2000_device *pcard, size_t engine_regs_offset, int engine_num, int irq_num)
345 {
346 struct mfd_cell cell = { .id = engine_num };
347 struct resource resources[2];
348
349 cell.platform_data = NULL;
350 cell.pdata_size = 0;
351 cell.name = KP_DRIVER_NAME_DMA_CONTROLLER;
352 cell.num_resources = 2;
353
354 memset(&resources, 0, sizeof(resources));
355
356 resources[0].start = engine_regs_offset;
357 resources[0].end = engine_regs_offset + (KPC_DMA_ENGINE_SIZE - 1);
358 resources[0].flags = IORESOURCE_MEM;
359
360 resources[1].start = irq_num;
361 resources[1].end = irq_num;
362 resources[1].flags = IORESOURCE_IRQ;
363
364 cell.resources = resources;
365
366 return mfd_add_devices(PCARD_TO_DEV(pcard), // parent
367 pcard->card_num * 100, // id
368 &cell, // struct mfd_cell *
369 1, // ndevs
370 &pcard->dma_base_resource,
371 0, // irq_base
372 NULL); // struct irq_domain *
373 }
374
kp2000_setup_dma_controller(struct kp2000_device * pcard)375 static int kp2000_setup_dma_controller(struct kp2000_device *pcard)
376 {
377 int err;
378 unsigned int i;
379 u64 capabilities_reg;
380
381 // S2C Engines
382 for (i = 0 ; i < 32 ; i++) {
383 capabilities_reg = readq(pcard->dma_bar_base + KPC_DMA_S2C_BASE_OFFSET + (KPC_DMA_ENGINE_SIZE * i));
384 if (capabilities_reg & ENGINE_CAP_PRESENT_MASK) {
385 err = create_dma_engine_core(pcard, (KPC_DMA_S2C_BASE_OFFSET + (KPC_DMA_ENGINE_SIZE * i)), i, pcard->pdev->irq);
386 if (err)
387 goto err_out;
388 }
389 }
390 // C2S Engines
391 for (i = 0 ; i < 32 ; i++) {
392 capabilities_reg = readq(pcard->dma_bar_base + KPC_DMA_C2S_BASE_OFFSET + (KPC_DMA_ENGINE_SIZE * i));
393 if (capabilities_reg & ENGINE_CAP_PRESENT_MASK) {
394 err = create_dma_engine_core(pcard, (KPC_DMA_C2S_BASE_OFFSET + (KPC_DMA_ENGINE_SIZE * i)), 32 + i, pcard->pdev->irq);
395 if (err)
396 goto err_out;
397 }
398 }
399
400 return 0;
401
402 err_out:
403 dev_err(&pcard->pdev->dev, "%s: failed to add a DMA Engine: %d\n",
404 __func__, err);
405 return err;
406 }
407
kp2000_probe_cores(struct kp2000_device * pcard)408 int kp2000_probe_cores(struct kp2000_device *pcard)
409 {
410 int err = 0;
411 int i;
412 int current_type_id;
413 u64 read_val;
414 unsigned int highest_core_id = 0;
415 struct core_table_entry cte;
416
417 err = kp2000_setup_dma_controller(pcard);
418 if (err)
419 return err;
420
421 INIT_LIST_HEAD(&pcard->uio_devices_list);
422
423 // First, iterate the core table looking for the highest CORE_ID
424 for (i = 0 ; i < pcard->core_table_length ; i++) {
425 read_val = readq(pcard->sysinfo_regs_base + ((pcard->core_table_offset + i) * 8));
426 parse_core_table_entry(&cte, read_val, pcard->core_table_rev);
427 dbg_cte(pcard, &cte);
428 if (cte.type > highest_core_id)
429 highest_core_id = cte.type;
430 if (cte.type == KP_CORE_ID_INVALID)
431 dev_info(&pcard->pdev->dev, "Found Invalid core: %016llx\n", read_val);
432 }
433 // Then, iterate over the possible core types.
434 for (current_type_id = 1 ; current_type_id <= highest_core_id ; current_type_id++) {
435 unsigned int core_num = 0;
436 // Foreach core type, iterate the whole table and instantiate subdevices for each core.
437 // Yes, this is O(n*m) but the actual runtime is small enough that it's an acceptable tradeoff.
438 for (i = 0 ; i < pcard->core_table_length ; i++) {
439 read_val = readq(pcard->sysinfo_regs_base + ((pcard->core_table_offset + i) * 8));
440 parse_core_table_entry(&cte, read_val, pcard->core_table_rev);
441
442 if (cte.type != current_type_id)
443 continue;
444
445 switch (cte.type) {
446 case KP_CORE_ID_I2C:
447 err = probe_core_basic(core_num, pcard,
448 KP_DRIVER_NAME_I2C, cte);
449 break;
450
451 case KP_CORE_ID_SPI:
452 err = probe_core_basic(core_num, pcard,
453 KP_DRIVER_NAME_SPI, cte);
454 break;
455
456 default:
457 err = probe_core_uio(core_num, pcard, "kpc_uio", cte);
458 break;
459 }
460 if (err) {
461 dev_err(&pcard->pdev->dev,
462 "%s: failed to add core %d: %d\n",
463 __func__, i, err);
464 goto error;
465 }
466 core_num++;
467 }
468 }
469
470 // Finally, instantiate a UIO device for the core_table.
471 cte.type = 0; // CORE_ID_BOARD_INFO
472 cte.offset = 0; // board info is always at the beginning
473 cte.length = 512 * 8;
474 cte.s2c_dma_present = false;
475 cte.s2c_dma_channel_num = 0;
476 cte.c2s_dma_present = false;
477 cte.c2s_dma_channel_num = 0;
478 cte.irq_count = 0;
479 cte.irq_base_num = 0;
480 err = probe_core_uio(0, pcard, "kpc_uio", cte);
481 if (err) {
482 dev_err(&pcard->pdev->dev, "%s: failed to add board_info core: %d\n",
483 __func__, err);
484 goto error;
485 }
486
487 return 0;
488
489 error:
490 kp2000_remove_cores(pcard);
491 mfd_remove_devices(PCARD_TO_DEV(pcard));
492 return err;
493 }
494
kp2000_remove_cores(struct kp2000_device * pcard)495 void kp2000_remove_cores(struct kp2000_device *pcard)
496 {
497 struct list_head *ptr;
498 struct list_head *next;
499
500 list_for_each_safe(ptr, next, &pcard->uio_devices_list) {
501 struct kpc_uio_device *kudev = list_entry(ptr, struct kpc_uio_device, list);
502
503 uio_unregister_device(&kudev->uioinfo);
504 device_unregister(kudev->dev);
505 list_del(&kudev->list);
506 kfree(kudev);
507 }
508 }
509
510