1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2021 Western Digital Corporation or its affiliates.
4  * Copyright (C) 2022 Ventana Micro Systems Inc.
5  */
6 
7 #define pr_fmt(fmt) "riscv-imsic: " fmt
8 #include <linux/acpi.h>
9 #include <linux/bitmap.h>
10 #include <linux/cpu.h>
11 #include <linux/interrupt.h>
12 #include <linux/io.h>
13 #include <linux/irq.h>
14 #include <linux/irqchip.h>
15 #include <linux/irqdomain.h>
16 #include <linux/module.h>
17 #include <linux/msi.h>
18 #include <linux/pci.h>
19 #include <linux/platform_device.h>
20 #include <linux/spinlock.h>
21 #include <linux/smp.h>
22 
23 #include <linux/irqchip/irq-msi-lib.h>
24 #include "irq-riscv-imsic-state.h"
25 
26 static bool imsic_cpu_page_phys(unsigned int cpu, unsigned int guest_index,
27 				phys_addr_t *out_msi_pa)
28 {
29 	struct imsic_global_config *global;
30 	struct imsic_local_config *local;
31 
32 	global = &imsic->global;
33 	local = per_cpu_ptr(global->local, cpu);
34 
35 	if (BIT(global->guest_index_bits) <= guest_index)
36 		return false;
37 
38 	if (out_msi_pa)
39 		*out_msi_pa = local->msi_pa + (guest_index * IMSIC_MMIO_PAGE_SZ);
40 
41 	return true;
42 }
43 
44 static void imsic_irq_mask(struct irq_data *d)
45 {
46 	imsic_vector_mask(irq_data_get_irq_chip_data(d));
47 }
48 
49 static void imsic_irq_unmask(struct irq_data *d)
50 {
51 	imsic_vector_unmask(irq_data_get_irq_chip_data(d));
52 }
53 
54 static int imsic_irq_retrigger(struct irq_data *d)
55 {
56 	struct imsic_vector *vec = irq_data_get_irq_chip_data(d);
57 	struct imsic_local_config *local;
58 
59 	if (WARN_ON(!vec))
60 		return -ENOENT;
61 
62 	local = per_cpu_ptr(imsic->global.local, vec->cpu);
63 	writel_relaxed(vec->local_id, local->msi_va);
64 	return 0;
65 }
66 
67 static void imsic_irq_ack(struct irq_data *d)
68 {
69 	irq_move_irq(d);
70 }
71 
72 static void imsic_irq_compose_vector_msg(struct imsic_vector *vec, struct msi_msg *msg)
73 {
74 	phys_addr_t msi_addr;
75 
76 	if (WARN_ON(!vec))
77 		return;
78 
79 	if (WARN_ON(!imsic_cpu_page_phys(vec->cpu, 0, &msi_addr)))
80 		return;
81 
82 	msg->address_hi = upper_32_bits(msi_addr);
83 	msg->address_lo = lower_32_bits(msi_addr);
84 	msg->data = vec->local_id;
85 }
86 
87 static void imsic_irq_compose_msg(struct irq_data *d, struct msi_msg *msg)
88 {
89 	imsic_irq_compose_vector_msg(irq_data_get_irq_chip_data(d), msg);
90 }
91 
92 #ifdef CONFIG_SMP
93 static void imsic_msi_update_msg(struct irq_data *d, struct imsic_vector *vec)
94 {
95 	struct msi_msg msg = { };
96 
97 	imsic_irq_compose_vector_msg(vec, &msg);
98 	irq_data_get_irq_chip(d)->irq_write_msi_msg(d, &msg);
99 }
100 
101 static int imsic_irq_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
102 				  bool force)
103 {
104 	struct imsic_vector *old_vec, *new_vec;
105 	struct imsic_vector tmp_vec;
106 
107 	/*
108 	 * Requirements for the downstream irqdomains (or devices):
109 	 *
110 	 * 1) Downstream irqdomains (or devices) with atomic MSI update can
111 	 *    happily do imsic_irq_set_affinity() in the process-context on
112 	 *    any CPU so the irqchip of such irqdomains must not set the
113 	 *    IRQCHIP_MOVE_DEFERRED flag.
114 	 *
115 	 * 2) Downstream irqdomains (or devices) with non-atomic MSI update
116 	 *    must use imsic_irq_set_affinity() in nterrupt-context upon
117 	 *    the next device interrupt so the irqchip of such irqdomains
118 	 *    must set the IRQCHIP_MOVE_DEFERRED flag.
119 	 */
120 
121 	old_vec = irq_data_get_irq_chip_data(d);
122 	if (WARN_ON(!old_vec))
123 		return -ENOENT;
124 
125 	/* If old vector cpu belongs to the target cpumask then do nothing */
126 	if (cpumask_test_cpu(old_vec->cpu, mask_val))
127 		return IRQ_SET_MASK_OK_DONE;
128 
129 	/* If move is already in-flight then return failure */
130 	if (imsic_vector_get_move(old_vec))
131 		return -EBUSY;
132 
133 	/* Get a new vector on the desired set of CPUs */
134 	new_vec = imsic_vector_alloc(old_vec->irq, mask_val);
135 	if (!new_vec)
136 		return -ENOSPC;
137 
138 	/*
139 	 * Device having non-atomic MSI update might see an intermediate
140 	 * state when changing target IMSIC vector from one CPU to another.
141 	 *
142 	 * To avoid losing interrupt to such intermediate state, do the
143 	 * following (just like x86 APIC):
144 	 *
145 	 * 1) First write a temporary IMSIC vector to the device which
146 	 * has MSI address same as the old IMSIC vector but MSI data
147 	 * matches the new IMSIC vector.
148 	 *
149 	 * 2) Next write the new IMSIC vector to the device.
150 	 *
151 	 * Based on the above, __imsic_local_sync() must check pending
152 	 * status of both old MSI data and new MSI data on the old CPU.
153 	 */
154 	if (!irq_can_move_in_process_context(d) &&
155 	    new_vec->local_id != old_vec->local_id) {
156 		/* Setup temporary vector */
157 		tmp_vec.cpu = old_vec->cpu;
158 		tmp_vec.local_id = new_vec->local_id;
159 
160 		/* Point device to the temporary vector */
161 		imsic_msi_update_msg(irq_get_irq_data(d->irq), &tmp_vec);
162 	}
163 
164 	/* Point device to the new vector */
165 	imsic_msi_update_msg(irq_get_irq_data(d->irq), new_vec);
166 
167 	/* Update irq descriptors with the new vector */
168 	d->chip_data = new_vec;
169 
170 	/* Update effective affinity */
171 	irq_data_update_effective_affinity(d, cpumask_of(new_vec->cpu));
172 
173 	/* Move state of the old vector to the new vector */
174 	imsic_vector_move(old_vec, new_vec);
175 
176 	return IRQ_SET_MASK_OK_DONE;
177 }
178 
179 static void imsic_irq_force_complete_move(struct irq_data *d)
180 {
181 	struct imsic_vector *mvec, *vec = irq_data_get_irq_chip_data(d);
182 	unsigned int cpu = smp_processor_id();
183 
184 	if (WARN_ON(!vec))
185 		return;
186 
187 	/* Do nothing if there is no in-flight move */
188 	mvec = imsic_vector_get_move(vec);
189 	if (!mvec)
190 		return;
191 
192 	/* Do nothing if the old IMSIC vector does not belong to current CPU */
193 	if (mvec->cpu != cpu)
194 		return;
195 
196 	/*
197 	 * The best we can do is force cleanup the old IMSIC vector.
198 	 *
199 	 * The challenges over here are same as x86 vector domain so
200 	 * refer to the comments in irq_force_complete_move() function
201 	 * implemented at arch/x86/kernel/apic/vector.c.
202 	 */
203 
204 	/* Force cleanup in-flight move */
205 	pr_info("IRQ fixup: irq %d move in progress, old vector cpu %d local_id %d\n",
206 		d->irq, mvec->cpu, mvec->local_id);
207 	imsic_vector_force_move_cleanup(vec);
208 }
209 #endif
210 
211 static struct irq_chip imsic_irq_base_chip = {
212 	.name				= "IMSIC",
213 	.irq_mask			= imsic_irq_mask,
214 	.irq_unmask			= imsic_irq_unmask,
215 #ifdef CONFIG_SMP
216 	.irq_set_affinity		= imsic_irq_set_affinity,
217 	.irq_force_complete_move	= imsic_irq_force_complete_move,
218 #endif
219 	.irq_retrigger			= imsic_irq_retrigger,
220 	.irq_ack			= imsic_irq_ack,
221 	.irq_compose_msi_msg		= imsic_irq_compose_msg,
222 	.flags				= IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND,
223 };
224 
225 static int imsic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
226 				  unsigned int nr_irqs, void *args)
227 {
228 	struct imsic_vector *vec;
229 
230 	/* Multi-MSI is not supported yet. */
231 	if (nr_irqs > 1)
232 		return -EOPNOTSUPP;
233 
234 	vec = imsic_vector_alloc(virq, cpu_online_mask);
235 	if (!vec)
236 		return -ENOSPC;
237 
238 	irq_domain_set_info(domain, virq, virq, &imsic_irq_base_chip, vec,
239 			    handle_edge_irq, NULL, NULL);
240 	irq_set_noprobe(virq);
241 	irq_set_affinity(virq, cpu_online_mask);
242 	irq_data_update_effective_affinity(irq_get_irq_data(virq), cpumask_of(vec->cpu));
243 
244 	return 0;
245 }
246 
247 static void imsic_irq_domain_free(struct irq_domain *domain, unsigned int virq,
248 				  unsigned int nr_irqs)
249 {
250 	struct irq_data *d = irq_domain_get_irq_data(domain, virq);
251 
252 	imsic_vector_free(irq_data_get_irq_chip_data(d));
253 	irq_domain_free_irqs_parent(domain, virq, nr_irqs);
254 }
255 
256 #ifdef CONFIG_GENERIC_IRQ_DEBUGFS
257 static void imsic_irq_debug_show(struct seq_file *m, struct irq_domain *d,
258 				 struct irq_data *irqd, int ind)
259 {
260 	if (!irqd) {
261 		imsic_vector_debug_show_summary(m, ind);
262 		return;
263 	}
264 
265 	imsic_vector_debug_show(m, irq_data_get_irq_chip_data(irqd), ind);
266 }
267 #endif
268 
269 static const struct irq_domain_ops imsic_base_domain_ops = {
270 	.alloc		= imsic_irq_domain_alloc,
271 	.free		= imsic_irq_domain_free,
272 	.select		= msi_lib_irq_domain_select,
273 #ifdef CONFIG_GENERIC_IRQ_DEBUGFS
274 	.debug_show	= imsic_irq_debug_show,
275 #endif
276 };
277 
278 static bool imsic_init_dev_msi_info(struct device *dev, struct irq_domain *domain,
279 				    struct irq_domain *real_parent, struct msi_domain_info *info)
280 {
281 	if (!msi_lib_init_dev_msi_info(dev, domain, real_parent, info))
282 		return false;
283 
284 	switch (info->bus_token) {
285 	case DOMAIN_BUS_PCI_DEVICE_MSI:
286 	case DOMAIN_BUS_PCI_DEVICE_MSIX:
287 		info->chip->flags |= IRQCHIP_MOVE_DEFERRED;
288 		break;
289 	default:
290 		break;
291 	}
292 
293 	return true;
294 }
295 
296 static const struct msi_parent_ops imsic_msi_parent_ops = {
297 	.supported_flags	= MSI_GENERIC_FLAGS_MASK |
298 				  MSI_FLAG_PCI_MSIX,
299 	.required_flags		= MSI_FLAG_USE_DEF_DOM_OPS |
300 				  MSI_FLAG_USE_DEF_CHIP_OPS |
301 				  MSI_FLAG_PCI_MSI_MASK_PARENT,
302 	.chip_flags		= MSI_CHIP_FLAG_SET_ACK,
303 	.bus_select_token	= DOMAIN_BUS_NEXUS,
304 	.bus_select_mask	= MATCH_PCI_MSI | MATCH_PLATFORM_MSI,
305 	.init_dev_msi_info	= imsic_init_dev_msi_info,
306 };
307 
308 int imsic_irqdomain_init(void)
309 {
310 	struct imsic_global_config *global;
311 
312 	if (!imsic || !imsic->fwnode) {
313 		pr_err("early driver not probed\n");
314 		return -ENODEV;
315 	}
316 
317 	if (imsic->base_domain) {
318 		pr_err("%pfwP: irq domain already created\n", imsic->fwnode);
319 		return -ENODEV;
320 	}
321 
322 	/* Create Base IRQ domain */
323 	imsic->base_domain = irq_domain_create_tree(imsic->fwnode,
324 						    &imsic_base_domain_ops, imsic);
325 	if (!imsic->base_domain) {
326 		pr_err("%pfwP: failed to create IMSIC base domain\n", imsic->fwnode);
327 		return -ENOMEM;
328 	}
329 	imsic->base_domain->flags |= IRQ_DOMAIN_FLAG_MSI_PARENT;
330 	imsic->base_domain->msi_parent_ops = &imsic_msi_parent_ops;
331 
332 	irq_domain_update_bus_token(imsic->base_domain, DOMAIN_BUS_NEXUS);
333 
334 	global = &imsic->global;
335 	pr_info("%pfwP:  hart-index-bits: %d,  guest-index-bits: %d\n",
336 		imsic->fwnode, global->hart_index_bits, global->guest_index_bits);
337 	pr_info("%pfwP: group-index-bits: %d, group-index-shift: %d\n",
338 		imsic->fwnode, global->group_index_bits, global->group_index_shift);
339 	pr_info("%pfwP: per-CPU IDs %d at base address %pa\n",
340 		imsic->fwnode, global->nr_ids, &global->base_addr);
341 	pr_info("%pfwP: total %d interrupts available\n",
342 		imsic->fwnode, num_possible_cpus() * (global->nr_ids - 1));
343 
344 	return 0;
345 }
346 
347 static int imsic_platform_probe_common(struct fwnode_handle *fwnode)
348 {
349 	if (imsic && imsic->fwnode != fwnode) {
350 		pr_err("%pfwP: fwnode mismatch\n", fwnode);
351 		return -ENODEV;
352 	}
353 
354 	return imsic_irqdomain_init();
355 }
356 
357 static int imsic_platform_dt_probe(struct platform_device *pdev)
358 {
359 	return imsic_platform_probe_common(pdev->dev.fwnode);
360 }
361 
362 #ifdef CONFIG_ACPI
363 
364 /*
365  *  On ACPI based systems, PCI enumeration happens early during boot in
366  *  acpi_scan_init(). PCI enumeration expects MSI domain setup before
367  *  it calls pci_set_msi_domain(). Hence, unlike in DT where
368  *  imsic-platform drive probe happens late during boot, ACPI based
369  *  systems need to setup the MSI domain early.
370  */
371 int imsic_platform_acpi_probe(struct fwnode_handle *fwnode)
372 {
373 	return imsic_platform_probe_common(fwnode);
374 }
375 
376 #endif
377 
378 static const struct of_device_id imsic_platform_match[] = {
379 	{ .compatible = "riscv,imsics" },
380 	{}
381 };
382 
383 static struct platform_driver imsic_platform_driver = {
384 	.driver = {
385 		.name		= "riscv-imsic",
386 		.of_match_table	= imsic_platform_match,
387 	},
388 	.probe = imsic_platform_dt_probe,
389 };
390 builtin_platform_driver(imsic_platform_driver);
391