kvm-unit-tests/lib/on-cpus.c

b2d54669SAndrew Jones// SPDX-License-Identifier: GPL-2.0-only
b2d54669SAndrew Jones/*
b2d54669SAndrew Jones * on_cpus() support based on cpumasks.
b2d54669SAndrew Jones *
b2d54669SAndrew Jones * Copyright (C) 2015, Red Hat Inc, Andrew Jones <drjones@redhat.com>
b2d54669SAndrew Jones */
b2d54669SAndrew Jones#include <libcflat.h>
b2d54669SAndrew Jones#include <cpumask.h>
b2d54669SAndrew Jones#include <on-cpus.h>
b2d54669SAndrew Jones#include <asm/barrier.h>
b2d54669SAndrew Jones#include <asm/smp.h>
0b27f391SAndrew Jones#include <asm/spinlock.h>
b2d54669SAndrew Jones
b2d54669SAndrew Jonesbool cpu0_calls_idle;
b2d54669SAndrew Jones
b2d54669SAndrew Jonesstruct on_cpu_info {
b2d54669SAndrew Jones	void (*func)(void *data);
b2d54669SAndrew Jones	void *data;
b2d54669SAndrew Jones	cpumask_t waiters;
b2d54669SAndrew Jones};
b2d54669SAndrew Jonesstatic struct on_cpu_info on_cpu_info[NR_CPUS];
0b27f391SAndrew Jonesstatic struct spinlock lock;
b2d54669SAndrew Jones
b2d54669SAndrew Jonesstatic void __deadlock_check(int cpu, const cpumask_t *waiters, bool *found)
b2d54669SAndrew Jones{
b2d54669SAndrew Jones	int i;
b2d54669SAndrew Jones
b2d54669SAndrew Jones	for_each_cpu(i, waiters) {
b2d54669SAndrew Jones		if (i == cpu) {
b2d54669SAndrew Jones			printf("CPU%d", cpu);
b2d54669SAndrew Jones			*found = true;
b2d54669SAndrew Jones			return;
b2d54669SAndrew Jones		}
b2d54669SAndrew Jones		__deadlock_check(cpu, &on_cpu_info[i].waiters, found);
b2d54669SAndrew Jones		if (*found) {
b2d54669SAndrew Jones			printf(" <=> CPU%d", i);
b2d54669SAndrew Jones			return;
b2d54669SAndrew Jones		}
b2d54669SAndrew Jones	}
b2d54669SAndrew Jones}
b2d54669SAndrew Jones
b2d54669SAndrew Jonesstatic void deadlock_check(int me, int cpu)
b2d54669SAndrew Jones{
b2d54669SAndrew Jones	bool found = false;
b2d54669SAndrew Jones
b2d54669SAndrew Jones	__deadlock_check(cpu, &on_cpu_info[me].waiters, &found);
b2d54669SAndrew Jones	if (found) {
b2d54669SAndrew Jones		printf(" <=> CPU%d deadlock detectd\n", me);
b2d54669SAndrew Jones		assert(0);
b2d54669SAndrew Jones	}
b2d54669SAndrew Jones}
b2d54669SAndrew Jones
b2d54669SAndrew Jonesstatic void cpu_wait(int cpu)
b2d54669SAndrew Jones{
b2d54669SAndrew Jones	int me = smp_processor_id();
b2d54669SAndrew Jones
b2d54669SAndrew Jones	if (cpu == me)
b2d54669SAndrew Jones		return;
b2d54669SAndrew Jones
b2d54669SAndrew Jones	cpumask_set_cpu(me, &on_cpu_info[cpu].waiters);
b2d54669SAndrew Jones	deadlock_check(me, cpu);
b2d54669SAndrew Jones	while (!cpu_idle(cpu))
b2d54669SAndrew Jones		smp_wait_for_event();
b2d54669SAndrew Jones	cpumask_clear_cpu(me, &on_cpu_info[cpu].waiters);
b2d54669SAndrew Jones}
b2d54669SAndrew Jones
b2d54669SAndrew Jonesvoid do_idle(void)
b2d54669SAndrew Jones{
b2d54669SAndrew Jones	int cpu = smp_processor_id();
b2d54669SAndrew Jones
b2d54669SAndrew Jones	if (cpu == 0)
b2d54669SAndrew Jones		cpu0_calls_idle = true;
b2d54669SAndrew Jones
0b27f391SAndrew Jones	for (;;) {
b2d54669SAndrew Jones		set_cpu_idle(cpu, true);
b2d54669SAndrew Jones		smp_send_event();
b2d54669SAndrew Jones
b2d54669SAndrew Jones		while (cpu_idle(cpu))
b2d54669SAndrew Jones			smp_wait_for_event();
b2d54669SAndrew Jones		smp_rmb();
b2d54669SAndrew Jones		on_cpu_info[cpu].func(on_cpu_info[cpu].data);
*2491edacSAndrew Jones		smp_wmb(); /* pairs with the smp_rmb() in on_cpu() and on_cpumask() */
b2d54669SAndrew Jones	}
b2d54669SAndrew Jones}
b2d54669SAndrew Jones
b2d54669SAndrew Jonesvoid on_cpu_async(int cpu, void (*func)(void *data), void *data)
b2d54669SAndrew Jones{
b2d54669SAndrew Jones	if (cpu == smp_processor_id()) {
b2d54669SAndrew Jones		func(data);
b2d54669SAndrew Jones		return;
b2d54669SAndrew Jones	}
b2d54669SAndrew Jones
b2d54669SAndrew Jones	assert_msg(cpu != 0 || cpu0_calls_idle, "Waiting on CPU0, which is unlikely to idle. "
b2d54669SAndrew Jones						"If this is intended set cpu0_calls_idle=1");
b2d54669SAndrew Jones
b2d54669SAndrew Jones	smp_boot_secondary_nofail(cpu, do_idle);
b2d54669SAndrew Jones
b2d54669SAndrew Jones	for (;;) {
b2d54669SAndrew Jones		cpu_wait(cpu);
0b27f391SAndrew Jones		spin_lock(&lock);
0b27f391SAndrew Jones		if (cpu_idle(cpu))
b2d54669SAndrew Jones			break;
0b27f391SAndrew Jones		spin_unlock(&lock);
b2d54669SAndrew Jones	}
b2d54669SAndrew Jones
b2d54669SAndrew Jones	on_cpu_info[cpu].func = func;
b2d54669SAndrew Jones	on_cpu_info[cpu].data = data;
0b27f391SAndrew Jones	smp_wmb();
b2d54669SAndrew Jones	set_cpu_idle(cpu, false);
0b27f391SAndrew Jones	spin_unlock(&lock);
b2d54669SAndrew Jones	smp_send_event();
b2d54669SAndrew Jones}
b2d54669SAndrew Jones
d012cfd5SAndrew Jonesvoid on_cpumask_async(const cpumask_t *mask, void (*func)(void *data), void *data)
d012cfd5SAndrew Jones{
d012cfd5SAndrew Jones	int cpu, me = smp_processor_id();
d012cfd5SAndrew Jones
d012cfd5SAndrew Jones	for_each_cpu(cpu, mask) {
d012cfd5SAndrew Jones		if (cpu == me)
d012cfd5SAndrew Jones			continue;
d012cfd5SAndrew Jones		on_cpu_async(cpu, func, data);
d012cfd5SAndrew Jones	}
d012cfd5SAndrew Jones	if (cpumask_test_cpu(me, mask))
d012cfd5SAndrew Jones		func(data);
d012cfd5SAndrew Jones}
d012cfd5SAndrew Jones
d012cfd5SAndrew Jonesvoid on_cpumask(const cpumask_t *mask, void (*func)(void *data), void *data)
d012cfd5SAndrew Jones{
d012cfd5SAndrew Jones	int cpu, me = smp_processor_id();
d012cfd5SAndrew Jones
d012cfd5SAndrew Jones	for_each_cpu(cpu, mask) {
d012cfd5SAndrew Jones		if (cpu == me)
d012cfd5SAndrew Jones			continue;
d012cfd5SAndrew Jones		on_cpu_async(cpu, func, data);
d012cfd5SAndrew Jones	}
d012cfd5SAndrew Jones	if (cpumask_test_cpu(me, mask))
d012cfd5SAndrew Jones		func(data);
d012cfd5SAndrew Jones
d012cfd5SAndrew Jones	for_each_cpu(cpu, mask) {
d012cfd5SAndrew Jones		if (cpu == me)
d012cfd5SAndrew Jones			continue;
d012cfd5SAndrew Jones		cpumask_set_cpu(me, &on_cpu_info[cpu].waiters);
d012cfd5SAndrew Jones		deadlock_check(me, cpu);
d012cfd5SAndrew Jones	}
d012cfd5SAndrew Jones	while (cpumask_weight(&cpu_idle_mask) < nr_cpus - 1)
d012cfd5SAndrew Jones		smp_wait_for_event();
d012cfd5SAndrew Jones	for_each_cpu(cpu, mask)
d012cfd5SAndrew Jones		cpumask_clear_cpu(me, &on_cpu_info[cpu].waiters);
*2491edacSAndrew Jones	smp_rmb(); /* pairs with the smp_wmb() in do_idle() */
d012cfd5SAndrew Jones}
d012cfd5SAndrew Jones
b2d54669SAndrew Jonesvoid on_cpu(int cpu, void (*func)(void *data), void *data)
b2d54669SAndrew Jones{
b2d54669SAndrew Jones	on_cpu_async(cpu, func, data);
b2d54669SAndrew Jones	cpu_wait(cpu);
*2491edacSAndrew Jones	smp_rmb(); /* pairs with the smp_wmb() in do_idle() */
b2d54669SAndrew Jones}
b2d54669SAndrew Jones
b2d54669SAndrew Jonesvoid on_cpus(void (*func)(void *data), void *data)
b2d54669SAndrew Jones{
d012cfd5SAndrew Jones	on_cpumask(&cpu_present_mask, func, data);
b2d54669SAndrew Jones}