1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * on_cpus() support based on cpumasks. 4 * 5 * Copyright (C) 2015, Red Hat Inc, Andrew Jones <drjones@redhat.com> 6 */ 7 #include <libcflat.h> 8 #include <cpumask.h> 9 #include <on-cpus.h> 10 #include <asm/barrier.h> 11 #include <asm/smp.h> 12 #include <asm/spinlock.h> 13 14 bool cpu0_calls_idle; 15 16 struct on_cpu_info { 17 void (*func)(void *data); 18 void *data; 19 cpumask_t waiters; 20 }; 21 static struct on_cpu_info on_cpu_info[NR_CPUS]; 22 static struct spinlock lock; 23 24 static void __deadlock_check(int cpu, const cpumask_t *waiters, bool *found) 25 { 26 int i; 27 28 for_each_cpu(i, waiters) { 29 if (i == cpu) { 30 printf("CPU%d", cpu); 31 *found = true; 32 return; 33 } 34 __deadlock_check(cpu, &on_cpu_info[i].waiters, found); 35 if (*found) { 36 printf(" <=> CPU%d", i); 37 return; 38 } 39 } 40 } 41 42 static void deadlock_check(int me, int cpu) 43 { 44 bool found = false; 45 46 __deadlock_check(cpu, &on_cpu_info[me].waiters, &found); 47 if (found) { 48 printf(" <=> CPU%d deadlock detectd\n", me); 49 assert(0); 50 } 51 } 52 53 static void cpu_wait(int cpu) 54 { 55 int me = smp_processor_id(); 56 57 if (cpu == me) 58 return; 59 60 cpumask_set_cpu(me, &on_cpu_info[cpu].waiters); 61 deadlock_check(me, cpu); 62 while (!cpu_idle(cpu)) 63 smp_wait_for_event(); 64 cpumask_clear_cpu(me, &on_cpu_info[cpu].waiters); 65 } 66 67 void do_idle(void) 68 { 69 int cpu = smp_processor_id(); 70 71 if (cpu == 0) 72 cpu0_calls_idle = true; 73 74 for (;;) { 75 set_cpu_idle(cpu, true); 76 smp_send_event(); 77 78 while (cpu_idle(cpu)) 79 smp_wait_for_event(); 80 smp_rmb(); 81 on_cpu_info[cpu].func(on_cpu_info[cpu].data); 82 smp_wmb(); /* pairs with the smp_rmb() in on_cpu() and on_cpumask() */ 83 } 84 } 85 86 void on_cpu_async(int cpu, void (*func)(void *data), void *data) 87 { 88 if (cpu == smp_processor_id()) { 89 func(data); 90 return; 91 } 92 93 assert_msg(cpu != 0 || cpu0_calls_idle, "Waiting on CPU0, which is unlikely to idle. " 94 "If this is intended set cpu0_calls_idle=1"); 95 96 smp_boot_secondary_nofail(cpu, do_idle); 97 98 for (;;) { 99 cpu_wait(cpu); 100 spin_lock(&lock); 101 if (cpu_idle(cpu)) 102 break; 103 spin_unlock(&lock); 104 } 105 106 on_cpu_info[cpu].func = func; 107 on_cpu_info[cpu].data = data; 108 smp_wmb(); 109 set_cpu_idle(cpu, false); 110 spin_unlock(&lock); 111 smp_send_event(); 112 } 113 114 void on_cpumask_async(const cpumask_t *mask, void (*func)(void *data), void *data) 115 { 116 int cpu, me = smp_processor_id(); 117 118 for_each_cpu(cpu, mask) { 119 if (cpu == me) 120 continue; 121 on_cpu_async(cpu, func, data); 122 } 123 if (cpumask_test_cpu(me, mask)) 124 func(data); 125 } 126 127 void on_cpumask(const cpumask_t *mask, void (*func)(void *data), void *data) 128 { 129 int cpu, me = smp_processor_id(); 130 cpumask_t tmp; 131 132 cpumask_copy(&tmp, mask); 133 cpumask_clear_cpu(me, &tmp); 134 135 for_each_cpu(cpu, &tmp) 136 on_cpu_async(cpu, func, data); 137 if (cpumask_test_cpu(me, mask)) 138 func(data); 139 140 for_each_cpu(cpu, &tmp) { 141 cpumask_set_cpu(me, &on_cpu_info[cpu].waiters); 142 deadlock_check(me, cpu); 143 } 144 while (!cpumask_subset(&tmp, &cpu_idle_mask)) 145 smp_wait_for_event(); 146 for_each_cpu(cpu, &tmp) 147 cpumask_clear_cpu(me, &on_cpu_info[cpu].waiters); 148 smp_rmb(); /* pairs with the smp_wmb() in do_idle() */ 149 } 150 151 void on_cpu(int cpu, void (*func)(void *data), void *data) 152 { 153 on_cpu_async(cpu, func, data); 154 cpu_wait(cpu); 155 smp_rmb(); /* pairs with the smp_wmb() in do_idle() */ 156 } 157 158 void on_cpus(void (*func)(void *data), void *data) 159 { 160 on_cpumask(&cpu_present_mask, func, data); 161 } 162