16c9f99dfSJanosch Frank /* SPDX-License-Identifier: GPL-2.0-only */ 2f77c0515SJanosch Frank /* 3f77c0515SJanosch Frank * s390x smp 4f77c0515SJanosch Frank * Based on Linux's arch/s390/kernel/smp.c and 5f77c0515SJanosch Frank * arch/s390/include/asm/sigp.h 6f77c0515SJanosch Frank * 7f77c0515SJanosch Frank * Copyright (c) 2019 IBM Corp 8f77c0515SJanosch Frank * 9f77c0515SJanosch Frank * Authors: 10f77c0515SJanosch Frank * Janosch Frank <frankja@linux.ibm.com> 11f77c0515SJanosch Frank */ 12f77c0515SJanosch Frank #include <libcflat.h> 13*d34d3250SJanosch Frank #include <bitops.h> 14f77c0515SJanosch Frank #include <asm/arch_def.h> 15f77c0515SJanosch Frank #include <asm/sigp.h> 16f77c0515SJanosch Frank #include <asm/page.h> 17f77c0515SJanosch Frank #include <asm/barrier.h> 18f77c0515SJanosch Frank #include <asm/spinlock.h> 19f77c0515SJanosch Frank #include <asm/asm-offsets.h> 20f77c0515SJanosch Frank 21f77c0515SJanosch Frank #include <alloc.h> 22f77c0515SJanosch Frank #include <alloc_page.h> 23f77c0515SJanosch Frank 24f77c0515SJanosch Frank #include "smp.h" 25f77c0515SJanosch Frank #include "sclp.h" 26f77c0515SJanosch Frank 27f77c0515SJanosch Frank static struct cpu *cpus; 28f77c0515SJanosch Frank static struct cpu *cpu0; 29f77c0515SJanosch Frank static struct spinlock lock; 30f77c0515SJanosch Frank 31f77c0515SJanosch Frank extern void smp_cpu_setup_state(void); 32f77c0515SJanosch Frank 33f77c0515SJanosch Frank int smp_query_num_cpus(void) 34f77c0515SJanosch Frank { 3552076a63SJanosch Frank return sclp_get_cpu_num(); 36f77c0515SJanosch Frank } 37f77c0515SJanosch Frank 38f77c0515SJanosch Frank struct cpu *smp_cpu_from_addr(uint16_t addr) 39f77c0515SJanosch Frank { 40f77c0515SJanosch Frank int i, num = smp_query_num_cpus(); 41f77c0515SJanosch Frank 42f77c0515SJanosch Frank for (i = 0; i < num; i++) { 43f77c0515SJanosch Frank if (cpus[i].addr == addr) 44f77c0515SJanosch Frank return &cpus[i]; 45f77c0515SJanosch Frank } 46f77c0515SJanosch Frank return NULL; 47f77c0515SJanosch Frank } 48f77c0515SJanosch Frank 49f77c0515SJanosch Frank bool smp_cpu_stopped(uint16_t addr) 50f77c0515SJanosch Frank { 51f77c0515SJanosch Frank uint32_t status; 52f77c0515SJanosch Frank 53f77c0515SJanosch Frank if (sigp(addr, SIGP_SENSE, 0, &status) != SIGP_CC_STATUS_STORED) 54f77c0515SJanosch Frank return false; 55f77c0515SJanosch Frank return !!(status & (SIGP_STATUS_CHECK_STOP|SIGP_STATUS_STOPPED)); 56f77c0515SJanosch Frank } 57f77c0515SJanosch Frank 589753bf43SChristian Borntraeger bool smp_sense_running_status(uint16_t addr) 59f77c0515SJanosch Frank { 60f77c0515SJanosch Frank if (sigp(addr, SIGP_SENSE_RUNNING, 0, NULL) != SIGP_CC_STATUS_STORED) 61f77c0515SJanosch Frank return true; 62f77c0515SJanosch Frank /* Status stored condition code is equivalent to cpu not running. */ 63f77c0515SJanosch Frank return false; 64f77c0515SJanosch Frank } 65f77c0515SJanosch Frank 66f77c0515SJanosch Frank static int smp_cpu_stop_nolock(uint16_t addr, bool store) 67f77c0515SJanosch Frank { 68f77c0515SJanosch Frank struct cpu *cpu; 69f77c0515SJanosch Frank uint8_t order = store ? SIGP_STOP_AND_STORE_STATUS : SIGP_STOP; 70f77c0515SJanosch Frank 71f77c0515SJanosch Frank cpu = smp_cpu_from_addr(addr); 72f77c0515SJanosch Frank if (!cpu || cpu == cpu0) 73f77c0515SJanosch Frank return -1; 74f77c0515SJanosch Frank 75f77c0515SJanosch Frank if (sigp_retry(addr, order, 0, NULL)) 76f77c0515SJanosch Frank return -1; 77f77c0515SJanosch Frank 78f77c0515SJanosch Frank while (!smp_cpu_stopped(addr)) 79f77c0515SJanosch Frank mb(); 80f77c0515SJanosch Frank cpu->active = false; 81f77c0515SJanosch Frank return 0; 82f77c0515SJanosch Frank } 83f77c0515SJanosch Frank 84f77c0515SJanosch Frank int smp_cpu_stop(uint16_t addr) 85f77c0515SJanosch Frank { 86f77c0515SJanosch Frank int rc; 87f77c0515SJanosch Frank 88f77c0515SJanosch Frank spin_lock(&lock); 89f77c0515SJanosch Frank rc = smp_cpu_stop_nolock(addr, false); 90f77c0515SJanosch Frank spin_unlock(&lock); 91f77c0515SJanosch Frank return rc; 92f77c0515SJanosch Frank } 93f77c0515SJanosch Frank 94f77c0515SJanosch Frank int smp_cpu_stop_store_status(uint16_t addr) 95f77c0515SJanosch Frank { 96f77c0515SJanosch Frank int rc; 97f77c0515SJanosch Frank 98f77c0515SJanosch Frank spin_lock(&lock); 99f77c0515SJanosch Frank rc = smp_cpu_stop_nolock(addr, true); 100f77c0515SJanosch Frank spin_unlock(&lock); 101f77c0515SJanosch Frank return rc; 102f77c0515SJanosch Frank } 103f77c0515SJanosch Frank 1042ca255a0SJanosch Frank static int smp_cpu_restart_nolock(uint16_t addr, struct psw *psw) 1052ca255a0SJanosch Frank { 1062ca255a0SJanosch Frank int rc; 1072ca255a0SJanosch Frank struct cpu *cpu = smp_cpu_from_addr(addr); 1082ca255a0SJanosch Frank 1092ca255a0SJanosch Frank if (!cpu) 1102ca255a0SJanosch Frank return -1; 1112ca255a0SJanosch Frank if (psw) { 1122ca255a0SJanosch Frank cpu->lowcore->restart_new_psw.mask = psw->mask; 1132ca255a0SJanosch Frank cpu->lowcore->restart_new_psw.addr = psw->addr; 1142ca255a0SJanosch Frank } 1152ca255a0SJanosch Frank /* 1162ca255a0SJanosch Frank * Stop the cpu, so we don't have a race between a running cpu 1172ca255a0SJanosch Frank * and the restart in the test that checks if the cpu is 1182ca255a0SJanosch Frank * running after the restart. 1192ca255a0SJanosch Frank */ 1202ca255a0SJanosch Frank smp_cpu_stop_nolock(addr, false); 1212ca255a0SJanosch Frank rc = sigp(addr, SIGP_RESTART, 0, NULL); 1222ca255a0SJanosch Frank if (rc) 1232ca255a0SJanosch Frank return rc; 1242ca255a0SJanosch Frank /* 1252ca255a0SJanosch Frank * The order has been accepted, but the actual restart may not 1262ca255a0SJanosch Frank * have been performed yet, so wait until the cpu is running. 1272ca255a0SJanosch Frank */ 1287e62c952SChristian Borntraeger while (smp_cpu_stopped(addr)) 1292ca255a0SJanosch Frank mb(); 1302ca255a0SJanosch Frank cpu->active = true; 1312ca255a0SJanosch Frank return 0; 1322ca255a0SJanosch Frank } 1332ca255a0SJanosch Frank 134f77c0515SJanosch Frank int smp_cpu_restart(uint16_t addr) 135f77c0515SJanosch Frank { 1362ca255a0SJanosch Frank int rc; 137f77c0515SJanosch Frank 138f77c0515SJanosch Frank spin_lock(&lock); 1392ca255a0SJanosch Frank rc = smp_cpu_restart_nolock(addr, NULL); 140f77c0515SJanosch Frank spin_unlock(&lock); 141f77c0515SJanosch Frank return rc; 142f77c0515SJanosch Frank } 143f77c0515SJanosch Frank 144f77c0515SJanosch Frank int smp_cpu_start(uint16_t addr, struct psw psw) 145f77c0515SJanosch Frank { 1462ca255a0SJanosch Frank int rc; 147f77c0515SJanosch Frank 148f77c0515SJanosch Frank spin_lock(&lock); 1492ca255a0SJanosch Frank rc = smp_cpu_restart_nolock(addr, &psw); 150f77c0515SJanosch Frank spin_unlock(&lock); 151f77c0515SJanosch Frank return rc; 152f77c0515SJanosch Frank } 153f77c0515SJanosch Frank 154f77c0515SJanosch Frank int smp_cpu_destroy(uint16_t addr) 155f77c0515SJanosch Frank { 156f77c0515SJanosch Frank struct cpu *cpu; 157f77c0515SJanosch Frank int rc; 158f77c0515SJanosch Frank 159f77c0515SJanosch Frank spin_lock(&lock); 160f77c0515SJanosch Frank rc = smp_cpu_stop_nolock(addr, false); 161f77c0515SJanosch Frank if (!rc) { 162f77c0515SJanosch Frank cpu = smp_cpu_from_addr(addr); 163f90ddba3SClaudio Imbrenda free_pages(cpu->lowcore); 164f90ddba3SClaudio Imbrenda free_pages(cpu->stack); 165f77c0515SJanosch Frank cpu->lowcore = (void *)-1UL; 166f77c0515SJanosch Frank cpu->stack = (void *)-1UL; 167f77c0515SJanosch Frank } 168f77c0515SJanosch Frank spin_unlock(&lock); 169f77c0515SJanosch Frank return rc; 170f77c0515SJanosch Frank } 171f77c0515SJanosch Frank 172f77c0515SJanosch Frank int smp_cpu_setup(uint16_t addr, struct psw psw) 173f77c0515SJanosch Frank { 174f77c0515SJanosch Frank struct lowcore *lc; 175f77c0515SJanosch Frank struct cpu *cpu; 176f77c0515SJanosch Frank int rc = -1; 177f77c0515SJanosch Frank 178f77c0515SJanosch Frank spin_lock(&lock); 179f77c0515SJanosch Frank 180f77c0515SJanosch Frank if (!cpus) 181f77c0515SJanosch Frank goto out; 182f77c0515SJanosch Frank 183f77c0515SJanosch Frank cpu = smp_cpu_from_addr(addr); 184f77c0515SJanosch Frank 185f77c0515SJanosch Frank if (!cpu || cpu->active) 186f77c0515SJanosch Frank goto out; 187f77c0515SJanosch Frank 188f77c0515SJanosch Frank sigp_retry(cpu->addr, SIGP_INITIAL_CPU_RESET, 0, NULL); 189f77c0515SJanosch Frank 190550b4683SClaudio Imbrenda lc = alloc_pages_flags(1, AREA_DMA31); 191f77c0515SJanosch Frank cpu->lowcore = lc; 192f77c0515SJanosch Frank memset(lc, 0, PAGE_SIZE * 2); 193f77c0515SJanosch Frank sigp_retry(cpu->addr, SIGP_SET_PREFIX, (unsigned long )lc, NULL); 194f77c0515SJanosch Frank 195f77c0515SJanosch Frank /* Copy all exception psws. */ 196f77c0515SJanosch Frank memcpy(lc, cpu0->lowcore, 512); 197f77c0515SJanosch Frank 198f77c0515SJanosch Frank /* Setup stack */ 199f77c0515SJanosch Frank cpu->stack = (uint64_t *)alloc_pages(2); 200f77c0515SJanosch Frank 201f77c0515SJanosch Frank /* Start without DAT and any other mask bits. */ 202ccf6dd34SJanosch Frank cpu->lowcore->sw_int_psw.mask = psw.mask; 203ccf6dd34SJanosch Frank cpu->lowcore->sw_int_psw.addr = psw.addr; 204f77c0515SJanosch Frank cpu->lowcore->sw_int_grs[14] = psw.addr; 205f77c0515SJanosch Frank cpu->lowcore->sw_int_grs[15] = (uint64_t)cpu->stack + (PAGE_SIZE * 4); 20644026818SJanosch Frank lc->restart_new_psw.mask = PSW_MASK_64; 207f77c0515SJanosch Frank lc->restart_new_psw.addr = (uint64_t)smp_cpu_setup_state; 208*d34d3250SJanosch Frank lc->sw_int_crs[0] = BIT_ULL(CTL0_AFP); 209f77c0515SJanosch Frank 210f77c0515SJanosch Frank /* Start processing */ 2112ca255a0SJanosch Frank smp_cpu_restart_nolock(addr, NULL); 21224a8db62SJanosch Frank /* Wait until the cpu has finished setup and started the provided psw */ 21324a8db62SJanosch Frank while (lc->restart_new_psw.addr != psw.addr) 21424a8db62SJanosch Frank mb(); 215f77c0515SJanosch Frank out: 216f77c0515SJanosch Frank spin_unlock(&lock); 217f77c0515SJanosch Frank return rc; 218f77c0515SJanosch Frank } 219f77c0515SJanosch Frank 220f77c0515SJanosch Frank /* 221f77c0515SJanosch Frank * Disregarding state, stop all cpus that once were online except for 222f77c0515SJanosch Frank * calling cpu. 223f77c0515SJanosch Frank */ 224f77c0515SJanosch Frank void smp_teardown(void) 225f77c0515SJanosch Frank { 226f77c0515SJanosch Frank int i = 0; 227f77c0515SJanosch Frank uint16_t this_cpu = stap(); 22852076a63SJanosch Frank int num = smp_query_num_cpus(); 229f77c0515SJanosch Frank 230f77c0515SJanosch Frank spin_lock(&lock); 23152076a63SJanosch Frank for (; i < num; i++) { 232f77c0515SJanosch Frank if (cpus[i].active && 233f77c0515SJanosch Frank cpus[i].addr != this_cpu) { 234f77c0515SJanosch Frank sigp_retry(cpus[i].addr, SIGP_STOP, 0, NULL); 235f77c0515SJanosch Frank } 236f77c0515SJanosch Frank } 237f77c0515SJanosch Frank spin_unlock(&lock); 238f77c0515SJanosch Frank } 239f77c0515SJanosch Frank 240f77c0515SJanosch Frank /*Expected to be called from boot cpu */ 241f77c0515SJanosch Frank extern uint64_t *stackptr; 242f77c0515SJanosch Frank void smp_setup(void) 243f77c0515SJanosch Frank { 244f77c0515SJanosch Frank int i = 0; 24552076a63SJanosch Frank int num = smp_query_num_cpus(); 246f77c0515SJanosch Frank unsigned short cpu0_addr = stap(); 24752076a63SJanosch Frank struct CPUEntry *entry = sclp_get_cpu_entries(); 248f77c0515SJanosch Frank 249f77c0515SJanosch Frank spin_lock(&lock); 25052076a63SJanosch Frank if (num > 1) 25152076a63SJanosch Frank printf("SMP: Initializing, found %d cpus\n", num); 252f77c0515SJanosch Frank 25352076a63SJanosch Frank cpus = calloc(num, sizeof(cpus)); 25452076a63SJanosch Frank for (i = 0; i < num; i++) { 25552076a63SJanosch Frank cpus[i].addr = entry[i].address; 256f77c0515SJanosch Frank cpus[i].active = false; 25752076a63SJanosch Frank if (entry[i].address == cpu0_addr) { 258f77c0515SJanosch Frank cpu0 = &cpus[i]; 259f77c0515SJanosch Frank cpu0->stack = stackptr; 260f77c0515SJanosch Frank cpu0->lowcore = (void *)0; 261f77c0515SJanosch Frank cpu0->active = true; 262f77c0515SJanosch Frank } 263f77c0515SJanosch Frank } 264f77c0515SJanosch Frank spin_unlock(&lock); 265f77c0515SJanosch Frank } 266