1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * on_cpus() support based on cpumasks.
4 *
5 * Copyright (C) 2015, Red Hat Inc, Andrew Jones <drjones@redhat.com>
6 */
7 #include <libcflat.h>
8 #include <cpumask.h>
9 #include <on-cpus.h>
10 #include <asm/barrier.h>
11 #include <asm/smp.h>
12 #include <asm/spinlock.h>
13
14 bool cpu0_calls_idle;
15
16 struct on_cpu_info {
17 void (*func)(void *data);
18 void *data;
19 cpumask_t waiters;
20 };
21 static struct on_cpu_info on_cpu_info[NR_CPUS];
22 static struct spinlock lock;
23
__deadlock_check(int cpu,const cpumask_t * waiters,bool * found)24 static void __deadlock_check(int cpu, const cpumask_t *waiters, bool *found)
25 {
26 int i;
27
28 for_each_cpu(i, waiters) {
29 if (i == cpu) {
30 printf("CPU%d", cpu);
31 *found = true;
32 return;
33 }
34 __deadlock_check(cpu, &on_cpu_info[i].waiters, found);
35 if (*found) {
36 printf(" <=> CPU%d", i);
37 return;
38 }
39 }
40 }
41
deadlock_check(int me,int cpu)42 static void deadlock_check(int me, int cpu)
43 {
44 bool found = false;
45
46 __deadlock_check(cpu, &on_cpu_info[me].waiters, &found);
47 if (found) {
48 printf(" <=> CPU%d deadlock detectd\n", me);
49 assert(0);
50 }
51 }
52
cpu_wait(int cpu)53 static void cpu_wait(int cpu)
54 {
55 int me = smp_processor_id();
56
57 if (cpu == me)
58 return;
59
60 cpumask_set_cpu(me, &on_cpu_info[cpu].waiters);
61 deadlock_check(me, cpu);
62 while (!cpu_idle(cpu))
63 smp_wait_for_event();
64 cpumask_clear_cpu(me, &on_cpu_info[cpu].waiters);
65 }
66
do_idle(void)67 void do_idle(void)
68 {
69 int cpu = smp_processor_id();
70
71 if (cpu == 0)
72 cpu0_calls_idle = true;
73
74 for (;;) {
75 set_cpu_idle(cpu, true);
76 smp_send_event();
77
78 while (cpu_idle(cpu))
79 smp_wait_for_event();
80 smp_rmb();
81 on_cpu_info[cpu].func(on_cpu_info[cpu].data);
82 smp_wmb(); /* pairs with the smp_rmb() in on_cpu() and on_cpumask() */
83 }
84 }
85
on_cpu_async(int cpu,void (* func)(void * data),void * data)86 void on_cpu_async(int cpu, void (*func)(void *data), void *data)
87 {
88 if (cpu == smp_processor_id()) {
89 func(data);
90 return;
91 }
92
93 assert_msg(cpu != 0 || cpu0_calls_idle, "Waiting on CPU0, which is unlikely to idle. "
94 "If this is intended set cpu0_calls_idle=1");
95
96 smp_boot_secondary_nofail(cpu, do_idle);
97
98 for (;;) {
99 cpu_wait(cpu);
100 spin_lock(&lock);
101 if (cpu_idle(cpu))
102 break;
103 spin_unlock(&lock);
104 }
105
106 on_cpu_info[cpu].func = func;
107 on_cpu_info[cpu].data = data;
108 smp_wmb();
109 set_cpu_idle(cpu, false);
110 spin_unlock(&lock);
111 smp_send_event();
112 }
113
on_cpumask_async(const cpumask_t * mask,void (* func)(void * data),void * data)114 void on_cpumask_async(const cpumask_t *mask, void (*func)(void *data), void *data)
115 {
116 int cpu, me = smp_processor_id();
117
118 for_each_cpu(cpu, mask) {
119 if (cpu == me)
120 continue;
121 on_cpu_async(cpu, func, data);
122 }
123 if (cpumask_test_cpu(me, mask))
124 func(data);
125 }
126
on_cpumask(const cpumask_t * mask,void (* func)(void * data),void * data)127 void on_cpumask(const cpumask_t *mask, void (*func)(void *data), void *data)
128 {
129 int cpu, me = smp_processor_id();
130 cpumask_t tmp;
131
132 cpumask_copy(&tmp, mask);
133 cpumask_clear_cpu(me, &tmp);
134
135 for_each_cpu(cpu, &tmp)
136 on_cpu_async(cpu, func, data);
137 if (cpumask_test_cpu(me, mask))
138 func(data);
139
140 for_each_cpu(cpu, &tmp) {
141 cpumask_set_cpu(me, &on_cpu_info[cpu].waiters);
142 deadlock_check(me, cpu);
143 }
144 while (!cpumask_subset(&tmp, &cpu_idle_mask))
145 smp_wait_for_event();
146 for_each_cpu(cpu, &tmp)
147 cpumask_clear_cpu(me, &on_cpu_info[cpu].waiters);
148 smp_rmb(); /* pairs with the smp_wmb() in do_idle() */
149 }
150
on_cpu(int cpu,void (* func)(void * data),void * data)151 void on_cpu(int cpu, void (*func)(void *data), void *data)
152 {
153 on_cpu_async(cpu, func, data);
154 cpu_wait(cpu);
155 smp_rmb(); /* pairs with the smp_wmb() in do_idle() */
156 }
157
on_cpus(void (* func)(void * data),void * data)158 void on_cpus(void (*func)(void *data), void *data)
159 {
160 on_cpumask(&cpu_present_mask, func, data);
161 }
162