44 #define DBG_VERBOSE(fmt...)	do { } while(0)
78  * or 0 if there is no new entry.
87 		return 0;  in xive_read_eq()
92 		return 0;  in xive_read_eq()
100 		if (q->idx == 0)  in xive_read_eq()
104 	return cur & 0x7fffffff;  in xive_read_eq()
114  * (0xff if none) and return what was found (0 if none).
130 static u32 xive_scan_interrupts(struct xive_cpu *xc, bool just_peek)  in xive_scan_interrupts()  argument
132 	u32 irq = 0;  in xive_scan_interrupts()
133 	u8 prio = 0;  in xive_scan_interrupts()
136 	while (xc->pending_prio != 0) {  in xive_scan_interrupts()
139 		prio = ffs(xc->pending_prio) - 1;  in xive_scan_interrupts()
143 		irq = xive_read_eq(&xc->queue[prio], just_peek);  in xive_scan_interrupts()
161 		xc->pending_prio &= ~(1 << prio);  in xive_scan_interrupts()
168 		q = &xc->queue[prio];  in xive_scan_interrupts()
170 			int p = atomic_xchg(&q->pending_count, 0);  in xive_scan_interrupts()
178 	/* If nothing was found, set CPPR to 0xff */  in xive_scan_interrupts()
179 	if (irq == 0)  in xive_scan_interrupts()
180 		prio = 0xff;  in xive_scan_interrupts()
183 	if (prio != xc->cppr) {  in xive_scan_interrupts()
185 		xc->cppr = prio;  in xive_scan_interrupts()
208 		val = xive_ops->esb_rw(xd->hw_irq, offset, 0, 0);  in xive_esb_read()
244 	struct xive_cpu *xc = per_cpu(xive_cpu, cpu);  in xmon_xive_do_dump()  local
247 	if (xc) {  in xmon_xive_do_dump()
248 		xmon_printf("pp=%02x CPPR=%02x ", xc->pending_prio, xc->cppr);  in xmon_xive_do_dump()
252 			u64 val = xive_esb_read(&xc->ipi_data, XIVE_ESB_GET);  in xmon_xive_do_dump()
254 			xmon_printf("IPI=0x%08x PQ=%c%c ", xc->hw_ipi,  in xmon_xive_do_dump()
259 		xive_dump_eq("EQ", &xc->queue[xive_irq_priority]);  in xmon_xive_do_dump()
277 		xmon_printf("IRQ 0x%08x : no config rc=%d\n", hw_irq, rc);  in xmon_xive_get_irq_config()
281 	xmon_printf("IRQ 0x%08x : target=0x%x prio=%02x lirq=0x%x ",  in xmon_xive_get_irq_config()
297 	return 0;  in xmon_xive_get_irq_config()
304 	struct xive_cpu *xc = __this_cpu_read(xive_cpu);  in xive_get_irq()  local
321 	xive_ops->update_pending(xc);  in xive_get_irq()
323 	DBG_VERBOSE("get_irq: pending=%02x\n", xc->pending_prio);  in xive_get_irq()
326 	irq = xive_scan_interrupts(xc, false);  in xive_get_irq()
328 	DBG_VERBOSE("get_irq: got irq 0x%x, new pending=0x%02x\n",  in xive_get_irq()
329 	    irq, xc->pending_prio);  in xive_get_irq()
333 		return 0;  in xive_get_irq()
347 static void xive_do_queue_eoi(struct xive_cpu *xc)  in xive_do_queue_eoi()  argument
349 	if (xive_scan_interrupts(xc, true) != 0) {  in xive_do_queue_eoi()
350 		DBG_VERBOSE("eoi: pending=0x%02x\n", xc->pending_prio);  in xive_do_queue_eoi()
364 		xive_esb_write(xd, XIVE_ESB_STORE_EOI, 0);  in xive_do_source_eoi()
370 		 * on P9 DD1.0 needed a latch to be clared in the LPC bridge  in xive_do_source_eoi()
400 				out_be64(xd->trig_mmio, 0);  in xive_do_source_eoi()
409 	struct xive_cpu *xc = __this_cpu_read(xive_cpu);  in xive_irq_eoi()  local
411 	DBG_VERBOSE("eoi_irq: irq=%d [0x%lx] pending=%02x\n",  in xive_irq_eoi()
412 		    d->irq, irqd_to_hwirq(d), xc->pending_prio);  in xive_irq_eoi()
431 	xive_do_queue_eoi(xc);  in xive_irq_eoi()
473 	struct xive_cpu *xc = per_cpu(xive_cpu, cpu);  in xive_try_pick_target()  local
474 	struct xive_q *q = &xc->queue[xive_irq_priority];  in xive_try_pick_target()
497 	struct xive_cpu *xc = per_cpu(xive_cpu, cpu);  in xive_dec_target_count()  local
498 	struct xive_q *q = &xc->queue[xive_irq_priority];  in xive_dec_target_count()
500 	if (WARN_ON(cpu < 0 || !xc)) {  in xive_dec_target_count()
501 		pr_err("%s: cpu=%d xc=%p\n", __func__, cpu, xc);  in xive_dec_target_count()
527 	for (i = 0; i < first && cpu < nr_cpu_ids; i++)  in xive_find_target_in_mask()
578 			struct xive_cpu *xc = per_cpu(xive_cpu, cpu);  in xive_pick_irq_target()  local
579 			if (xc->chip_id == xd->src_chip)  in xive_pick_irq_target()
588 		if (cpu >= 0)  in xive_pick_irq_target()
605 	pr_devel("xive_irq_startup: irq %d [0x%x] data @%p\n",  in xive_irq_startup()
648 	return 0;  in xive_irq_startup()
657 	pr_devel("xive_irq_shutdown: irq %d [0x%x] data @%p\n",  in xive_irq_shutdown()
672 				0xff, XIVE_BAD_IRQ);  in xive_irq_shutdown()
687 	 * be fixed by P9 DD2.0, if that is the case, firmware  in xive_irq_unmask()
710 	 * be fixed by P9 DD2.0, if that is the case, firmware  in xive_irq_mask()
717 					0xff, d->irq);  in xive_irq_mask()
731 	int rc = 0;  in xive_irq_set_affinity()
773 	if (rc < 0) {  in xive_irq_set_affinity()
778 	pr_devel("  target: 0x%x\n", target);  in xive_irq_set_affinity()
818 		pr_warn("Interrupt %d (HW 0x%x) type mismatch, Linux says %s, FW says %s\n",  in xive_irq_set_type()
833 		return 0;  in xive_irq_retrigger()
842 	 * Note: We pass "0" to the hw_irq argument in order to  in xive_irq_retrigger()
847 	xive_do_source_eoi(0, xd);  in xive_irq_retrigger()
892 			return 0;  in xive_irq_set_vcpu_affinity()
931 			return 0;  in xive_irq_set_vcpu_affinity()
972 	return 0;  in xive_irq_set_vcpu_affinity()
995 		return 0;  in xive_get_irqchip_state()
1065 	return 0;  in xive_irq_alloc_data()
1083 	struct xive_cpu *xc;  in xive_cause_ipi()  local
1086 	xc = per_cpu(xive_cpu, cpu);  in xive_cause_ipi()
1088 	DBG_VERBOSE("IPI CPU %d -> %d (HW IRQ 0x%x)\n",  in xive_cause_ipi()
1089 		    smp_processor_id(), cpu, xc->hw_ipi);  in xive_cause_ipi()
1091 	xd = &xc->ipi_data;  in xive_cause_ipi()
1094 	out_be64(xd->trig_mmio, 0);  in xive_cause_ipi()
1104 	struct xive_cpu *xc = __this_cpu_read(xive_cpu);  in xive_ipi_eoi()  local
1107 	if (!xc)  in xive_ipi_eoi()
1110 	DBG_VERBOSE("IPI eoi: irq=%d [0x%lx] (HW IRQ 0x%x) pending=%02x\n",  in xive_ipi_eoi()
1111 		    d->irq, irqd_to_hwirq(d), xc->hw_ipi, xc->pending_prio);  in xive_ipi_eoi()
1113 	xive_do_source_eoi(xc->hw_ipi, &xc->ipi_data);  in xive_ipi_eoi()
1114 	xive_do_queue_eoi(xc);  in xive_ipi_eoi()
1145 	virq = irq_create_mapping(xive_irq_domain, 0);  in xive_request_ipi()
1154 	struct xive_cpu *xc;  in xive_setup_cpu_ipi()  local
1159 	xc = per_cpu(xive_cpu, cpu);  in xive_setup_cpu_ipi()
1162 	if (xc->hw_ipi != XIVE_BAD_IRQ)  in xive_setup_cpu_ipi()
1163 		return 0;  in xive_setup_cpu_ipi()
1165 	/* Grab an IPI from the backend, this will populate xc->hw_ipi */  in xive_setup_cpu_ipi()
1166 	if (xive_ops->get_ipi(cpu, xc))  in xive_setup_cpu_ipi()
1173 	rc = xive_ops->populate_irq_data(xc->hw_ipi, &xc->ipi_data);  in xive_setup_cpu_ipi()
1178 	rc = xive_ops->configure_irq(xc->hw_ipi,  in xive_setup_cpu_ipi()
1186 	    xc->hw_ipi, xive_ipi_irq, xc->ipi_data.trig_mmio);  in xive_setup_cpu_ipi()
1189 	xive_do_source_set_mask(&xc->ipi_data, false);  in xive_setup_cpu_ipi()
1191 	return 0;  in xive_setup_cpu_ipi()
1194 static void xive_cleanup_cpu_ipi(unsigned int cpu, struct xive_cpu *xc)  in xive_cleanup_cpu_ipi()  argument
1199 	if (xc->hw_ipi == XIVE_BAD_IRQ)  in xive_cleanup_cpu_ipi()
1203 	xive_do_source_set_mask(&xc->ipi_data, true);  in xive_cleanup_cpu_ipi()
1212 	xive_ops->configure_irq(xc->hw_ipi, hard_smp_processor_id(),  in xive_cleanup_cpu_ipi()
1213 				0xff, xive_ipi_irq);  in xive_cleanup_cpu_ipi()
1216 	xive_ops->put_ipi(cpu, xc);  in xive_cleanup_cpu_ipi()
1244 	/* IPIs are special and come up with HW number 0 */  in xive_irq_domain_map()
1245 	if (hw == 0) {  in xive_irq_domain_map()
1252 		return 0;  in xive_irq_domain_map()
1262 	return 0;  in xive_irq_domain_map()
1283 	*out_hwirq = intspec[0];  in xive_irq_domain_xlate()
1297 	return 0;  in xive_irq_domain_xlate()
1322 static void xive_cleanup_cpu_queues(unsigned int cpu, struct xive_cpu *xc)  in xive_cleanup_cpu_queues()  argument
1324 	if (xc->queue[xive_irq_priority].qpage)  in xive_cleanup_cpu_queues()
1325 		xive_ops->cleanup_queue(cpu, xc, xive_irq_priority);  in xive_cleanup_cpu_queues()
1328 static int xive_setup_cpu_queues(unsigned int cpu, struct xive_cpu *xc)  in xive_setup_cpu_queues()  argument
1330 	int rc = 0;  in xive_setup_cpu_queues()
1333 	if (!xc->queue[xive_irq_priority].qpage)  in xive_setup_cpu_queues()
1334 		rc = xive_ops->setup_queue(cpu, xc, xive_irq_priority);  in xive_setup_cpu_queues()
1341 	struct xive_cpu *xc;  in xive_prepare_cpu()  local
1343 	xc = per_cpu(xive_cpu, cpu);  in xive_prepare_cpu()
1344 	if (!xc) {  in xive_prepare_cpu()
1347 		xc = kzalloc_node(sizeof(struct xive_cpu),  in xive_prepare_cpu()
1349 		if (!xc)  in xive_prepare_cpu()
1353 			xc->chip_id = of_get_ibm_chip_id(np);  in xive_prepare_cpu()
1355 		xc->hw_ipi = XIVE_BAD_IRQ;  in xive_prepare_cpu()
1357 		per_cpu(xive_cpu, cpu) = xc;  in xive_prepare_cpu()
1361 	return xive_setup_cpu_queues(cpu, xc);  in xive_prepare_cpu()
1366 	struct xive_cpu *xc = __this_cpu_read(xive_cpu);  in xive_setup_cpu()  local
1370 		xive_ops->setup_cpu(smp_processor_id(), xc);  in xive_setup_cpu()
1372 	/* Set CPPR to 0xff to enable flow of interrupts */  in xive_setup_cpu()
1373 	xc->cppr = 0xff;  in xive_setup_cpu()
1374 	out_8(xive_tima + xive_tima_offset + TM_CPPR, 0xff);  in xive_setup_cpu()
1402 static void xive_flush_cpu_queue(unsigned int cpu, struct xive_cpu *xc)  in xive_flush_cpu_queue()  argument
1410 	while ((irq = xive_scan_interrupts(xc, false)) != 0) {  in xive_flush_cpu_queue()
1424 		if (d->domain != xive_irq_domain || hw_irq == 0)  in xive_flush_cpu_queue()
1459 	struct xive_cpu *xc = __this_cpu_read(xive_cpu);  in xive_smp_disable_cpu()  local
1465 	/* Set CPPR to 0 to disable flow of interrupts */  in xive_smp_disable_cpu()
1466 	xc->cppr = 0;  in xive_smp_disable_cpu()
1467 	out_8(xive_tima + xive_tima_offset + TM_CPPR, 0);  in xive_smp_disable_cpu()
1470 	xive_flush_cpu_queue(cpu, xc);  in xive_smp_disable_cpu()
1473 	xc->cppr = 0xff;  in xive_smp_disable_cpu()
1474 	out_8(xive_tima + xive_tima_offset + TM_CPPR, 0xff);  in xive_smp_disable_cpu()
1479 	struct xive_cpu *xc = __this_cpu_read(xive_cpu);  in xive_flush_interrupt()  local
1483 	xive_flush_cpu_queue(cpu, xc);  in xive_flush_interrupt()
1492 	struct xive_cpu *xc = __this_cpu_read(xive_cpu);  in xive_teardown_cpu()  local
1495 	/* Set CPPR to 0 to disable flow of interrupts */  in xive_teardown_cpu()
1496 	xc->cppr = 0;  in xive_teardown_cpu()
1497 	out_8(xive_tima + xive_tima_offset + TM_CPPR, 0);  in xive_teardown_cpu()
1500 		xive_ops->teardown_cpu(cpu, xc);  in xive_teardown_cpu()
1504 	xive_cleanup_cpu_ipi(cpu, xc);  in xive_teardown_cpu()
1508 	xive_cleanup_cpu_queues(cpu, xc);  in xive_teardown_cpu()
1556 	memset(qpage, 0, 1 << queue_shift);  in xive_queue_page_alloc()
1564 	return 0;  in xive_off()
1570 	struct xive_cpu *xc = per_cpu(xive_cpu, cpu);  in xive_debug_show_cpu()  local
1573 	if (xc) {  in xive_debug_show_cpu()
1574 		seq_printf(m, "pp=%02x CPPR=%02x ", xc->pending_prio, xc->cppr);  in xive_debug_show_cpu()
1578 			u64 val = xive_esb_read(&xc->ipi_data, XIVE_ESB_GET);  in xive_debug_show_cpu()
1580 			seq_printf(m, "IPI=0x%08x PQ=%c%c ", xc->hw_ipi,  in xive_debug_show_cpu()
1586 			struct xive_q *q = &xc->queue[xive_irq_priority];  in xive_debug_show_cpu()
1615 		seq_printf(m, "IRQ 0x%08x : no config rc=%d\n", hw_irq, rc);  in xive_debug_show_irq()
1619 	seq_printf(m, "IRQ 0x%08x : target=0x%x prio=%02x lirq=0x%x ",  in xive_debug_show_irq()
1657 		/* IPIs are special (HW number 0) */  in xive_core_debug_show()
1661 	return 0;  in xive_core_debug_show()
1670 	return 0;  in xive_core_debug_init()