16c9f99dfSJanosch Frank /* SPDX-License-Identifier: GPL-2.0-only */ 2f77c0515SJanosch Frank /* 3f77c0515SJanosch Frank * s390x smp 4f77c0515SJanosch Frank * Based on Linux's arch/s390/kernel/smp.c and 5f77c0515SJanosch Frank * arch/s390/include/asm/sigp.h 6f77c0515SJanosch Frank * 7f77c0515SJanosch Frank * Copyright (c) 2019 IBM Corp 8f77c0515SJanosch Frank * 9f77c0515SJanosch Frank * Authors: 10f77c0515SJanosch Frank * Janosch Frank <frankja@linux.ibm.com> 11f77c0515SJanosch Frank */ 12f77c0515SJanosch Frank #include <libcflat.h> 13f77c0515SJanosch Frank #include <asm/arch_def.h> 14f77c0515SJanosch Frank #include <asm/sigp.h> 15f77c0515SJanosch Frank #include <asm/page.h> 16f77c0515SJanosch Frank #include <asm/barrier.h> 17f77c0515SJanosch Frank #include <asm/spinlock.h> 18f77c0515SJanosch Frank #include <asm/asm-offsets.h> 19f77c0515SJanosch Frank 20f77c0515SJanosch Frank #include <alloc.h> 21f77c0515SJanosch Frank #include <alloc_page.h> 22f77c0515SJanosch Frank 23f77c0515SJanosch Frank #include "smp.h" 24f77c0515SJanosch Frank #include "sclp.h" 25f77c0515SJanosch Frank 26f77c0515SJanosch Frank static struct cpu *cpus; 27f77c0515SJanosch Frank static struct cpu *cpu0; 28f77c0515SJanosch Frank static struct spinlock lock; 29f77c0515SJanosch Frank 30f77c0515SJanosch Frank extern void smp_cpu_setup_state(void); 31f77c0515SJanosch Frank 32f77c0515SJanosch Frank int smp_query_num_cpus(void) 33f77c0515SJanosch Frank { 3452076a63SJanosch Frank return sclp_get_cpu_num(); 35f77c0515SJanosch Frank } 36f77c0515SJanosch Frank 37f77c0515SJanosch Frank struct cpu *smp_cpu_from_addr(uint16_t addr) 38f77c0515SJanosch Frank { 39f77c0515SJanosch Frank int i, num = smp_query_num_cpus(); 40f77c0515SJanosch Frank 41f77c0515SJanosch Frank for (i = 0; i < num; i++) { 42f77c0515SJanosch Frank if (cpus[i].addr == addr) 43f77c0515SJanosch Frank return &cpus[i]; 44f77c0515SJanosch Frank } 45f77c0515SJanosch Frank return NULL; 46f77c0515SJanosch Frank } 47f77c0515SJanosch Frank 48f77c0515SJanosch Frank bool smp_cpu_stopped(uint16_t addr) 49f77c0515SJanosch Frank { 50f77c0515SJanosch Frank uint32_t status; 51f77c0515SJanosch Frank 52f77c0515SJanosch Frank if (sigp(addr, SIGP_SENSE, 0, &status) != SIGP_CC_STATUS_STORED) 53f77c0515SJanosch Frank return false; 54f77c0515SJanosch Frank return !!(status & (SIGP_STATUS_CHECK_STOP|SIGP_STATUS_STOPPED)); 55f77c0515SJanosch Frank } 56f77c0515SJanosch Frank 579753bf43SChristian Borntraeger bool smp_sense_running_status(uint16_t addr) 58f77c0515SJanosch Frank { 59f77c0515SJanosch Frank if (sigp(addr, SIGP_SENSE_RUNNING, 0, NULL) != SIGP_CC_STATUS_STORED) 60f77c0515SJanosch Frank return true; 61f77c0515SJanosch Frank /* Status stored condition code is equivalent to cpu not running. */ 62f77c0515SJanosch Frank return false; 63f77c0515SJanosch Frank } 64f77c0515SJanosch Frank 65f77c0515SJanosch Frank static int smp_cpu_stop_nolock(uint16_t addr, bool store) 66f77c0515SJanosch Frank { 67f77c0515SJanosch Frank struct cpu *cpu; 68f77c0515SJanosch Frank uint8_t order = store ? SIGP_STOP_AND_STORE_STATUS : SIGP_STOP; 69f77c0515SJanosch Frank 70f77c0515SJanosch Frank cpu = smp_cpu_from_addr(addr); 71f77c0515SJanosch Frank if (!cpu || cpu == cpu0) 72f77c0515SJanosch Frank return -1; 73f77c0515SJanosch Frank 74f77c0515SJanosch Frank if (sigp_retry(addr, order, 0, NULL)) 75f77c0515SJanosch Frank return -1; 76f77c0515SJanosch Frank 77f77c0515SJanosch Frank while (!smp_cpu_stopped(addr)) 78f77c0515SJanosch Frank mb(); 79f77c0515SJanosch Frank cpu->active = false; 80f77c0515SJanosch Frank return 0; 81f77c0515SJanosch Frank } 82f77c0515SJanosch Frank 83f77c0515SJanosch Frank int smp_cpu_stop(uint16_t addr) 84f77c0515SJanosch Frank { 85f77c0515SJanosch Frank int rc; 86f77c0515SJanosch Frank 87f77c0515SJanosch Frank spin_lock(&lock); 88f77c0515SJanosch Frank rc = smp_cpu_stop_nolock(addr, false); 89f77c0515SJanosch Frank spin_unlock(&lock); 90f77c0515SJanosch Frank return rc; 91f77c0515SJanosch Frank } 92f77c0515SJanosch Frank 93f77c0515SJanosch Frank int smp_cpu_stop_store_status(uint16_t addr) 94f77c0515SJanosch Frank { 95f77c0515SJanosch Frank int rc; 96f77c0515SJanosch Frank 97f77c0515SJanosch Frank spin_lock(&lock); 98f77c0515SJanosch Frank rc = smp_cpu_stop_nolock(addr, true); 99f77c0515SJanosch Frank spin_unlock(&lock); 100f77c0515SJanosch Frank return rc; 101f77c0515SJanosch Frank } 102f77c0515SJanosch Frank 1032ca255a0SJanosch Frank static int smp_cpu_restart_nolock(uint16_t addr, struct psw *psw) 1042ca255a0SJanosch Frank { 1052ca255a0SJanosch Frank int rc; 1062ca255a0SJanosch Frank struct cpu *cpu = smp_cpu_from_addr(addr); 1072ca255a0SJanosch Frank 1082ca255a0SJanosch Frank if (!cpu) 1092ca255a0SJanosch Frank return -1; 1102ca255a0SJanosch Frank if (psw) { 1112ca255a0SJanosch Frank cpu->lowcore->restart_new_psw.mask = psw->mask; 1122ca255a0SJanosch Frank cpu->lowcore->restart_new_psw.addr = psw->addr; 1132ca255a0SJanosch Frank } 1142ca255a0SJanosch Frank /* 1152ca255a0SJanosch Frank * Stop the cpu, so we don't have a race between a running cpu 1162ca255a0SJanosch Frank * and the restart in the test that checks if the cpu is 1172ca255a0SJanosch Frank * running after the restart. 1182ca255a0SJanosch Frank */ 1192ca255a0SJanosch Frank smp_cpu_stop_nolock(addr, false); 1202ca255a0SJanosch Frank rc = sigp(addr, SIGP_RESTART, 0, NULL); 1212ca255a0SJanosch Frank if (rc) 1222ca255a0SJanosch Frank return rc; 1232ca255a0SJanosch Frank /* 1242ca255a0SJanosch Frank * The order has been accepted, but the actual restart may not 1252ca255a0SJanosch Frank * have been performed yet, so wait until the cpu is running. 1262ca255a0SJanosch Frank */ 1277e62c952SChristian Borntraeger while (smp_cpu_stopped(addr)) 1282ca255a0SJanosch Frank mb(); 1292ca255a0SJanosch Frank cpu->active = true; 1302ca255a0SJanosch Frank return 0; 1312ca255a0SJanosch Frank } 1322ca255a0SJanosch Frank 133f77c0515SJanosch Frank int smp_cpu_restart(uint16_t addr) 134f77c0515SJanosch Frank { 1352ca255a0SJanosch Frank int rc; 136f77c0515SJanosch Frank 137f77c0515SJanosch Frank spin_lock(&lock); 1382ca255a0SJanosch Frank rc = smp_cpu_restart_nolock(addr, NULL); 139f77c0515SJanosch Frank spin_unlock(&lock); 140f77c0515SJanosch Frank return rc; 141f77c0515SJanosch Frank } 142f77c0515SJanosch Frank 143f77c0515SJanosch Frank int smp_cpu_start(uint16_t addr, struct psw psw) 144f77c0515SJanosch Frank { 1452ca255a0SJanosch Frank int rc; 146f77c0515SJanosch Frank 147f77c0515SJanosch Frank spin_lock(&lock); 1482ca255a0SJanosch Frank rc = smp_cpu_restart_nolock(addr, &psw); 149f77c0515SJanosch Frank spin_unlock(&lock); 150f77c0515SJanosch Frank return rc; 151f77c0515SJanosch Frank } 152f77c0515SJanosch Frank 153f77c0515SJanosch Frank int smp_cpu_destroy(uint16_t addr) 154f77c0515SJanosch Frank { 155f77c0515SJanosch Frank struct cpu *cpu; 156f77c0515SJanosch Frank int rc; 157f77c0515SJanosch Frank 158f77c0515SJanosch Frank spin_lock(&lock); 159f77c0515SJanosch Frank rc = smp_cpu_stop_nolock(addr, false); 160f77c0515SJanosch Frank if (!rc) { 161f77c0515SJanosch Frank cpu = smp_cpu_from_addr(addr); 162f90ddba3SClaudio Imbrenda free_pages(cpu->lowcore); 163f90ddba3SClaudio Imbrenda free_pages(cpu->stack); 164f77c0515SJanosch Frank cpu->lowcore = (void *)-1UL; 165f77c0515SJanosch Frank cpu->stack = (void *)-1UL; 166f77c0515SJanosch Frank } 167f77c0515SJanosch Frank spin_unlock(&lock); 168f77c0515SJanosch Frank return rc; 169f77c0515SJanosch Frank } 170f77c0515SJanosch Frank 171f77c0515SJanosch Frank int smp_cpu_setup(uint16_t addr, struct psw psw) 172f77c0515SJanosch Frank { 173f77c0515SJanosch Frank struct lowcore *lc; 174f77c0515SJanosch Frank struct cpu *cpu; 175f77c0515SJanosch Frank int rc = -1; 176f77c0515SJanosch Frank 177f77c0515SJanosch Frank spin_lock(&lock); 178f77c0515SJanosch Frank 179f77c0515SJanosch Frank if (!cpus) 180f77c0515SJanosch Frank goto out; 181f77c0515SJanosch Frank 182f77c0515SJanosch Frank cpu = smp_cpu_from_addr(addr); 183f77c0515SJanosch Frank 184f77c0515SJanosch Frank if (!cpu || cpu->active) 185f77c0515SJanosch Frank goto out; 186f77c0515SJanosch Frank 187f77c0515SJanosch Frank sigp_retry(cpu->addr, SIGP_INITIAL_CPU_RESET, 0, NULL); 188f77c0515SJanosch Frank 189550b4683SClaudio Imbrenda lc = alloc_pages_flags(1, AREA_DMA31); 190f77c0515SJanosch Frank cpu->lowcore = lc; 191f77c0515SJanosch Frank memset(lc, 0, PAGE_SIZE * 2); 192f77c0515SJanosch Frank sigp_retry(cpu->addr, SIGP_SET_PREFIX, (unsigned long )lc, NULL); 193f77c0515SJanosch Frank 194f77c0515SJanosch Frank /* Copy all exception psws. */ 195f77c0515SJanosch Frank memcpy(lc, cpu0->lowcore, 512); 196f77c0515SJanosch Frank 197f77c0515SJanosch Frank /* Setup stack */ 198f77c0515SJanosch Frank cpu->stack = (uint64_t *)alloc_pages(2); 199f77c0515SJanosch Frank 200f77c0515SJanosch Frank /* Start without DAT and any other mask bits. */ 201ccf6dd34SJanosch Frank cpu->lowcore->sw_int_psw.mask = psw.mask; 202ccf6dd34SJanosch Frank cpu->lowcore->sw_int_psw.addr = psw.addr; 203f77c0515SJanosch Frank cpu->lowcore->sw_int_grs[14] = psw.addr; 204f77c0515SJanosch Frank cpu->lowcore->sw_int_grs[15] = (uint64_t)cpu->stack + (PAGE_SIZE * 4); 205*44026818SJanosch Frank lc->restart_new_psw.mask = PSW_MASK_64; 206f77c0515SJanosch Frank lc->restart_new_psw.addr = (uint64_t)smp_cpu_setup_state; 207736b9295SJanosch Frank lc->sw_int_crs[0] = 0x0000000000040000UL; 208f77c0515SJanosch Frank 209f77c0515SJanosch Frank /* Start processing */ 2102ca255a0SJanosch Frank smp_cpu_restart_nolock(addr, NULL); 21124a8db62SJanosch Frank /* Wait until the cpu has finished setup and started the provided psw */ 21224a8db62SJanosch Frank while (lc->restart_new_psw.addr != psw.addr) 21324a8db62SJanosch Frank mb(); 214f77c0515SJanosch Frank out: 215f77c0515SJanosch Frank spin_unlock(&lock); 216f77c0515SJanosch Frank return rc; 217f77c0515SJanosch Frank } 218f77c0515SJanosch Frank 219f77c0515SJanosch Frank /* 220f77c0515SJanosch Frank * Disregarding state, stop all cpus that once were online except for 221f77c0515SJanosch Frank * calling cpu. 222f77c0515SJanosch Frank */ 223f77c0515SJanosch Frank void smp_teardown(void) 224f77c0515SJanosch Frank { 225f77c0515SJanosch Frank int i = 0; 226f77c0515SJanosch Frank uint16_t this_cpu = stap(); 22752076a63SJanosch Frank int num = smp_query_num_cpus(); 228f77c0515SJanosch Frank 229f77c0515SJanosch Frank spin_lock(&lock); 23052076a63SJanosch Frank for (; i < num; i++) { 231f77c0515SJanosch Frank if (cpus[i].active && 232f77c0515SJanosch Frank cpus[i].addr != this_cpu) { 233f77c0515SJanosch Frank sigp_retry(cpus[i].addr, SIGP_STOP, 0, NULL); 234f77c0515SJanosch Frank } 235f77c0515SJanosch Frank } 236f77c0515SJanosch Frank spin_unlock(&lock); 237f77c0515SJanosch Frank } 238f77c0515SJanosch Frank 239f77c0515SJanosch Frank /*Expected to be called from boot cpu */ 240f77c0515SJanosch Frank extern uint64_t *stackptr; 241f77c0515SJanosch Frank void smp_setup(void) 242f77c0515SJanosch Frank { 243f77c0515SJanosch Frank int i = 0; 24452076a63SJanosch Frank int num = smp_query_num_cpus(); 245f77c0515SJanosch Frank unsigned short cpu0_addr = stap(); 24652076a63SJanosch Frank struct CPUEntry *entry = sclp_get_cpu_entries(); 247f77c0515SJanosch Frank 248f77c0515SJanosch Frank spin_lock(&lock); 24952076a63SJanosch Frank if (num > 1) 25052076a63SJanosch Frank printf("SMP: Initializing, found %d cpus\n", num); 251f77c0515SJanosch Frank 25252076a63SJanosch Frank cpus = calloc(num, sizeof(cpus)); 25352076a63SJanosch Frank for (i = 0; i < num; i++) { 25452076a63SJanosch Frank cpus[i].addr = entry[i].address; 255f77c0515SJanosch Frank cpus[i].active = false; 25652076a63SJanosch Frank if (entry[i].address == cpu0_addr) { 257f77c0515SJanosch Frank cpu0 = &cpus[i]; 258f77c0515SJanosch Frank cpu0->stack = stackptr; 259f77c0515SJanosch Frank cpu0->lowcore = (void *)0; 260f77c0515SJanosch Frank cpu0->active = true; 261f77c0515SJanosch Frank } 262f77c0515SJanosch Frank } 263f77c0515SJanosch Frank spin_unlock(&lock); 264f77c0515SJanosch Frank } 265