1 #include "kvm/kvm-cpu.h" 2 3 #include "kvm/symbol.h" 4 #include "kvm/util.h" 5 #include "kvm/kvm.h" 6 #include "kvm/virtio.h" 7 #include "kvm/mutex.h" 8 #include "kvm/barrier.h" 9 10 #include <sys/ioctl.h> 11 #include <sys/mman.h> 12 #include <sys/eventfd.h> 13 #include <signal.h> 14 #include <stdlib.h> 15 #include <string.h> 16 #include <errno.h> 17 #include <stdio.h> 18 19 extern __thread struct kvm_cpu *current_kvm_cpu; 20 21 int __attribute__((weak)) kvm_cpu__get_endianness(struct kvm_cpu *vcpu) 22 { 23 return VIRTIO_ENDIAN_HOST; 24 } 25 26 void kvm_cpu__enable_singlestep(struct kvm_cpu *vcpu) 27 { 28 struct kvm_guest_debug debug = { 29 .control = KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP, 30 }; 31 32 if (ioctl(vcpu->vcpu_fd, KVM_SET_GUEST_DEBUG, &debug) < 0) 33 pr_warning("KVM_SET_GUEST_DEBUG failed"); 34 } 35 36 void kvm_cpu__run(struct kvm_cpu *vcpu) 37 { 38 int err; 39 40 if (!vcpu->is_running) 41 return; 42 43 err = ioctl(vcpu->vcpu_fd, KVM_RUN, 0); 44 if (err < 0 && (errno != EINTR && errno != EAGAIN)) 45 die_perror("KVM_RUN failed"); 46 } 47 48 static void kvm_cpu_signal_handler(int signum) 49 { 50 if (signum == SIGKVMEXIT) { 51 if (current_kvm_cpu && current_kvm_cpu->is_running) 52 current_kvm_cpu->is_running = false; 53 } else if (signum == SIGKVMPAUSE) { 54 if (current_kvm_cpu->paused) 55 die("Pause signaled for already paused CPU\n"); 56 57 /* pause_lock is held by kvm__pause() */ 58 current_kvm_cpu->paused = 1; 59 60 /* 61 * This is a blocking function and uses locks. It is safe 62 * to call it for this signal as a second pause event should 63 * not be send to this thread until it acquires and releases 64 * the pause_lock. 65 */ 66 kvm__notify_paused(); 67 } 68 69 /* For SIGKVMTASK cpu->task is already set */ 70 } 71 72 static void kvm_cpu__handle_coalesced_mmio(struct kvm_cpu *cpu) 73 { 74 if (cpu->ring) { 75 while (cpu->ring->first != cpu->ring->last) { 76 struct kvm_coalesced_mmio *m; 77 m = &cpu->ring->coalesced_mmio[cpu->ring->first]; 78 kvm_cpu__emulate_mmio(cpu, 79 m->phys_addr, 80 m->data, 81 m->len, 82 1); 83 cpu->ring->first = (cpu->ring->first + 1) % KVM_COALESCED_MMIO_MAX; 84 } 85 } 86 } 87 88 static DEFINE_MUTEX(task_lock); 89 static int task_eventfd; 90 91 static void kvm_cpu__run_task(struct kvm_cpu *cpu) 92 { 93 u64 inc = 1; 94 95 pr_debug("Running task %p on cpu %lu", cpu->task, cpu->cpu_id); 96 97 /* Make sure we see the store to cpu->task */ 98 rmb(); 99 cpu->task->func(cpu, cpu->task->data); 100 101 /* Clear task before we signal completion */ 102 cpu->task = NULL; 103 wmb(); 104 105 if (write(task_eventfd, &inc, sizeof(inc)) < 0) 106 die("Failed notifying of completed task."); 107 } 108 109 void kvm_cpu__run_on_all_cpus(struct kvm *kvm, struct kvm_cpu_task *task) 110 { 111 int i, done = 0; 112 113 pr_debug("Running task %p on all cpus", task); 114 115 mutex_lock(&task_lock); 116 117 for (i = 0; i < kvm->nrcpus; i++) { 118 if (kvm->cpus[i]->task) { 119 /* Should never happen */ 120 die("CPU %d already has a task pending!", i); 121 } 122 123 kvm->cpus[i]->task = task; 124 wmb(); 125 126 if (kvm->cpus[i] == current_kvm_cpu) 127 kvm_cpu__run_task(current_kvm_cpu); 128 else 129 pthread_kill(kvm->cpus[i]->thread, SIGKVMTASK); 130 } 131 132 while (done < kvm->nrcpus) { 133 u64 count; 134 135 if (read(task_eventfd, &count, sizeof(count)) < 0) 136 die("Failed reading task eventfd"); 137 138 done += count; 139 } 140 141 mutex_unlock(&task_lock); 142 } 143 144 int kvm_cpu__start(struct kvm_cpu *cpu) 145 { 146 sigset_t sigset; 147 148 sigemptyset(&sigset); 149 sigaddset(&sigset, SIGALRM); 150 151 pthread_sigmask(SIG_BLOCK, &sigset, NULL); 152 153 signal(SIGKVMEXIT, kvm_cpu_signal_handler); 154 signal(SIGKVMPAUSE, kvm_cpu_signal_handler); 155 signal(SIGKVMTASK, kvm_cpu_signal_handler); 156 157 kvm_cpu__reset_vcpu(cpu); 158 159 if (cpu->kvm->cfg.single_step) 160 kvm_cpu__enable_singlestep(cpu); 161 162 while (cpu->is_running) { 163 if (cpu->needs_nmi) { 164 kvm_cpu__arch_nmi(cpu); 165 cpu->needs_nmi = 0; 166 } 167 168 if (cpu->task) 169 kvm_cpu__run_task(cpu); 170 171 kvm_cpu__run(cpu); 172 173 switch (cpu->kvm_run->exit_reason) { 174 case KVM_EXIT_UNKNOWN: 175 break; 176 case KVM_EXIT_DEBUG: 177 kvm_cpu__show_registers(cpu); 178 kvm_cpu__show_code(cpu); 179 break; 180 case KVM_EXIT_IO: { 181 bool ret; 182 183 ret = kvm_cpu__emulate_io(cpu, 184 cpu->kvm_run->io.port, 185 (u8 *)cpu->kvm_run + 186 cpu->kvm_run->io.data_offset, 187 cpu->kvm_run->io.direction, 188 cpu->kvm_run->io.size, 189 cpu->kvm_run->io.count); 190 191 if (!ret) 192 goto panic_kvm; 193 break; 194 } 195 case KVM_EXIT_MMIO: { 196 bool ret; 197 198 /* 199 * If we had MMIO exit, coalesced ring should be processed 200 * *before* processing the exit itself 201 */ 202 kvm_cpu__handle_coalesced_mmio(cpu); 203 204 ret = kvm_cpu__emulate_mmio(cpu, 205 cpu->kvm_run->mmio.phys_addr, 206 cpu->kvm_run->mmio.data, 207 cpu->kvm_run->mmio.len, 208 cpu->kvm_run->mmio.is_write); 209 210 if (!ret) 211 goto panic_kvm; 212 break; 213 } 214 case KVM_EXIT_INTR: 215 if (cpu->is_running) 216 break; 217 goto exit_kvm; 218 case KVM_EXIT_SHUTDOWN: 219 goto exit_kvm; 220 case KVM_EXIT_SYSTEM_EVENT: 221 /* 222 * Print the type of system event and 223 * treat all system events as shutdown request. 224 */ 225 switch (cpu->kvm_run->system_event.type) { 226 default: 227 pr_warning("unknown system event type %d", 228 cpu->kvm_run->system_event.type); 229 /* fall through for now */ 230 case KVM_SYSTEM_EVENT_RESET: 231 /* Fall through for now */ 232 case KVM_SYSTEM_EVENT_SHUTDOWN: 233 /* 234 * Ensure that all VCPUs are torn down, 235 * regardless of which CPU generated the event. 236 */ 237 kvm__reboot(cpu->kvm); 238 goto exit_kvm; 239 }; 240 break; 241 default: { 242 bool ret; 243 244 ret = kvm_cpu__handle_exit(cpu); 245 if (!ret) 246 goto panic_kvm; 247 break; 248 } 249 } 250 kvm_cpu__handle_coalesced_mmio(cpu); 251 } 252 253 exit_kvm: 254 return 0; 255 256 panic_kvm: 257 return 1; 258 } 259 260 int kvm_cpu__init(struct kvm *kvm) 261 { 262 int max_cpus, recommended_cpus, i; 263 264 max_cpus = kvm__max_cpus(kvm); 265 recommended_cpus = kvm__recommended_cpus(kvm); 266 267 if (kvm->cfg.nrcpus > max_cpus) { 268 printf(" # Limit the number of CPUs to %d\n", max_cpus); 269 kvm->cfg.nrcpus = max_cpus; 270 } else if (kvm->cfg.nrcpus > recommended_cpus) { 271 printf(" # Warning: The maximum recommended amount of VCPUs" 272 " is %d\n", recommended_cpus); 273 } 274 275 kvm->nrcpus = kvm->cfg.nrcpus; 276 277 task_eventfd = eventfd(0, 0); 278 if (task_eventfd < 0) { 279 pr_warning("Couldn't create task_eventfd"); 280 return task_eventfd; 281 } 282 283 /* Alloc one pointer too many, so array ends up 0-terminated */ 284 kvm->cpus = calloc(kvm->nrcpus + 1, sizeof(void *)); 285 if (!kvm->cpus) { 286 pr_warning("Couldn't allocate array for %d CPUs", kvm->nrcpus); 287 return -ENOMEM; 288 } 289 290 for (i = 0; i < kvm->nrcpus; i++) { 291 kvm->cpus[i] = kvm_cpu__arch_init(kvm, i); 292 if (!kvm->cpus[i]) { 293 pr_warning("unable to initialize KVM VCPU"); 294 goto fail_alloc; 295 } 296 } 297 298 return 0; 299 300 fail_alloc: 301 for (i = 0; i < kvm->nrcpus; i++) 302 free(kvm->cpus[i]); 303 return -ENOMEM; 304 } 305 base_init(kvm_cpu__init); 306 307 int kvm_cpu__exit(struct kvm *kvm) 308 { 309 int i, r; 310 void *ret = NULL; 311 312 kvm_cpu__delete(kvm->cpus[0]); 313 kvm->cpus[0] = NULL; 314 315 kvm__pause(kvm); 316 for (i = 1; i < kvm->nrcpus; i++) { 317 if (kvm->cpus[i]->is_running) { 318 pthread_kill(kvm->cpus[i]->thread, SIGKVMEXIT); 319 if (pthread_join(kvm->cpus[i]->thread, &ret) != 0) 320 die("pthread_join"); 321 kvm_cpu__delete(kvm->cpus[i]); 322 } 323 if (ret == NULL) 324 r = 0; 325 } 326 kvm__continue(kvm); 327 328 free(kvm->cpus); 329 330 kvm->nrcpus = 0; 331 332 close(task_eventfd); 333 334 return r; 335 } 336