1 #include "kvm/kvm-cpu.h" 2 3 #include "kvm/symbol.h" 4 #include "kvm/util.h" 5 #include "kvm/kvm.h" 6 #include "kvm/virtio.h" 7 #include "kvm/mutex.h" 8 #include "kvm/barrier.h" 9 10 #include <sys/ioctl.h> 11 #include <sys/mman.h> 12 #include <sys/eventfd.h> 13 #include <signal.h> 14 #include <stdlib.h> 15 #include <string.h> 16 #include <errno.h> 17 #include <stdio.h> 18 19 extern __thread struct kvm_cpu *current_kvm_cpu; 20 21 int __attribute__((weak)) kvm_cpu__get_endianness(struct kvm_cpu *vcpu) 22 { 23 return VIRTIO_ENDIAN_HOST; 24 } 25 26 void kvm_cpu__enable_singlestep(struct kvm_cpu *vcpu) 27 { 28 struct kvm_guest_debug debug = { 29 .control = KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP, 30 }; 31 32 if (ioctl(vcpu->vcpu_fd, KVM_SET_GUEST_DEBUG, &debug) < 0) 33 pr_warning("KVM_SET_GUEST_DEBUG failed"); 34 } 35 36 void kvm_cpu__run(struct kvm_cpu *vcpu) 37 { 38 int err; 39 40 if (!vcpu->is_running) 41 return; 42 43 err = ioctl(vcpu->vcpu_fd, KVM_RUN, 0); 44 if (err < 0 && (errno != EINTR && errno != EAGAIN)) 45 die_perror("KVM_RUN failed"); 46 } 47 48 static void kvm_cpu_signal_handler(int signum) 49 { 50 if (signum == SIGKVMEXIT) { 51 if (current_kvm_cpu && current_kvm_cpu->is_running) 52 current_kvm_cpu->is_running = false; 53 } else if (signum == SIGKVMPAUSE) { 54 current_kvm_cpu->paused = 1; 55 } 56 57 /* For SIGKVMTASK cpu->task is already set */ 58 } 59 60 static void kvm_cpu__handle_coalesced_mmio(struct kvm_cpu *cpu) 61 { 62 if (cpu->ring) { 63 while (cpu->ring->first != cpu->ring->last) { 64 struct kvm_coalesced_mmio *m; 65 m = &cpu->ring->coalesced_mmio[cpu->ring->first]; 66 kvm_cpu__emulate_mmio(cpu, 67 m->phys_addr, 68 m->data, 69 m->len, 70 1); 71 cpu->ring->first = (cpu->ring->first + 1) % KVM_COALESCED_MMIO_MAX; 72 } 73 } 74 } 75 76 static DEFINE_MUTEX(task_lock); 77 static int task_eventfd; 78 79 static void kvm_cpu__run_task(struct kvm_cpu *cpu) 80 { 81 u64 inc = 1; 82 83 pr_debug("Running task %p on cpu %lu", cpu->task, cpu->cpu_id); 84 85 /* Make sure we see the store to cpu->task */ 86 rmb(); 87 cpu->task->func(cpu, cpu->task->data); 88 89 /* Clear task before we signal completion */ 90 cpu->task = NULL; 91 wmb(); 92 93 if (write(task_eventfd, &inc, sizeof(inc)) < 0) 94 die("Failed notifying of completed task."); 95 } 96 97 void kvm_cpu__run_on_all_cpus(struct kvm *kvm, struct kvm_cpu_task *task) 98 { 99 int i, done = 0; 100 101 pr_debug("Running task %p on all cpus", task); 102 103 mutex_lock(&task_lock); 104 105 for (i = 0; i < kvm->nrcpus; i++) { 106 if (kvm->cpus[i]->task) { 107 /* Should never happen */ 108 die("CPU %d already has a task pending!", i); 109 } 110 111 kvm->cpus[i]->task = task; 112 wmb(); 113 114 if (kvm->cpus[i] == current_kvm_cpu) 115 kvm_cpu__run_task(current_kvm_cpu); 116 else 117 pthread_kill(kvm->cpus[i]->thread, SIGKVMTASK); 118 } 119 120 while (done < kvm->nrcpus) { 121 u64 count; 122 123 if (read(task_eventfd, &count, sizeof(count)) < 0) 124 die("Failed reading task eventfd"); 125 126 done += count; 127 } 128 129 mutex_unlock(&task_lock); 130 } 131 132 int kvm_cpu__start(struct kvm_cpu *cpu) 133 { 134 sigset_t sigset; 135 136 sigemptyset(&sigset); 137 sigaddset(&sigset, SIGALRM); 138 139 pthread_sigmask(SIG_BLOCK, &sigset, NULL); 140 141 signal(SIGKVMEXIT, kvm_cpu_signal_handler); 142 signal(SIGKVMPAUSE, kvm_cpu_signal_handler); 143 signal(SIGKVMTASK, kvm_cpu_signal_handler); 144 145 kvm_cpu__reset_vcpu(cpu); 146 147 if (cpu->kvm->cfg.single_step) 148 kvm_cpu__enable_singlestep(cpu); 149 150 while (cpu->is_running) { 151 if (cpu->paused) { 152 kvm__notify_paused(); 153 cpu->paused = 0; 154 } 155 156 if (cpu->needs_nmi) { 157 kvm_cpu__arch_nmi(cpu); 158 cpu->needs_nmi = 0; 159 } 160 161 if (cpu->task) 162 kvm_cpu__run_task(cpu); 163 164 kvm_cpu__run(cpu); 165 166 switch (cpu->kvm_run->exit_reason) { 167 case KVM_EXIT_UNKNOWN: 168 break; 169 case KVM_EXIT_DEBUG: 170 kvm_cpu__show_registers(cpu); 171 kvm_cpu__show_code(cpu); 172 break; 173 case KVM_EXIT_IO: { 174 bool ret; 175 176 ret = kvm_cpu__emulate_io(cpu, 177 cpu->kvm_run->io.port, 178 (u8 *)cpu->kvm_run + 179 cpu->kvm_run->io.data_offset, 180 cpu->kvm_run->io.direction, 181 cpu->kvm_run->io.size, 182 cpu->kvm_run->io.count); 183 184 if (!ret) 185 goto panic_kvm; 186 break; 187 } 188 case KVM_EXIT_MMIO: { 189 bool ret; 190 191 /* 192 * If we had MMIO exit, coalesced ring should be processed 193 * *before* processing the exit itself 194 */ 195 kvm_cpu__handle_coalesced_mmio(cpu); 196 197 ret = kvm_cpu__emulate_mmio(cpu, 198 cpu->kvm_run->mmio.phys_addr, 199 cpu->kvm_run->mmio.data, 200 cpu->kvm_run->mmio.len, 201 cpu->kvm_run->mmio.is_write); 202 203 if (!ret) 204 goto panic_kvm; 205 break; 206 } 207 case KVM_EXIT_INTR: 208 if (cpu->is_running) 209 break; 210 goto exit_kvm; 211 case KVM_EXIT_SHUTDOWN: 212 goto exit_kvm; 213 case KVM_EXIT_SYSTEM_EVENT: 214 /* 215 * Print the type of system event and 216 * treat all system events as shutdown request. 217 */ 218 switch (cpu->kvm_run->system_event.type) { 219 default: 220 pr_warning("unknown system event type %d", 221 cpu->kvm_run->system_event.type); 222 /* fall through for now */ 223 case KVM_SYSTEM_EVENT_RESET: 224 /* Fall through for now */ 225 case KVM_SYSTEM_EVENT_SHUTDOWN: 226 /* 227 * Ensure that all VCPUs are torn down, 228 * regardless of which CPU generated the event. 229 */ 230 kvm__reboot(cpu->kvm); 231 goto exit_kvm; 232 }; 233 break; 234 default: { 235 bool ret; 236 237 ret = kvm_cpu__handle_exit(cpu); 238 if (!ret) 239 goto panic_kvm; 240 break; 241 } 242 } 243 kvm_cpu__handle_coalesced_mmio(cpu); 244 } 245 246 exit_kvm: 247 return 0; 248 249 panic_kvm: 250 return 1; 251 } 252 253 int kvm_cpu__init(struct kvm *kvm) 254 { 255 int max_cpus, recommended_cpus, i; 256 257 max_cpus = kvm__max_cpus(kvm); 258 recommended_cpus = kvm__recommended_cpus(kvm); 259 260 if (kvm->cfg.nrcpus > max_cpus) { 261 printf(" # Limit the number of CPUs to %d\n", max_cpus); 262 kvm->cfg.nrcpus = max_cpus; 263 } else if (kvm->cfg.nrcpus > recommended_cpus) { 264 printf(" # Warning: The maximum recommended amount of VCPUs" 265 " is %d\n", recommended_cpus); 266 } 267 268 kvm->nrcpus = kvm->cfg.nrcpus; 269 270 task_eventfd = eventfd(0, 0); 271 if (task_eventfd < 0) { 272 pr_warning("Couldn't create task_eventfd"); 273 return task_eventfd; 274 } 275 276 /* Alloc one pointer too many, so array ends up 0-terminated */ 277 kvm->cpus = calloc(kvm->nrcpus + 1, sizeof(void *)); 278 if (!kvm->cpus) { 279 pr_warning("Couldn't allocate array for %d CPUs", kvm->nrcpus); 280 return -ENOMEM; 281 } 282 283 for (i = 0; i < kvm->nrcpus; i++) { 284 kvm->cpus[i] = kvm_cpu__arch_init(kvm, i); 285 if (!kvm->cpus[i]) { 286 pr_warning("unable to initialize KVM VCPU"); 287 goto fail_alloc; 288 } 289 } 290 291 return 0; 292 293 fail_alloc: 294 for (i = 0; i < kvm->nrcpus; i++) 295 free(kvm->cpus[i]); 296 return -ENOMEM; 297 } 298 base_init(kvm_cpu__init); 299 300 int kvm_cpu__exit(struct kvm *kvm) 301 { 302 int i, r; 303 void *ret = NULL; 304 305 kvm_cpu__delete(kvm->cpus[0]); 306 kvm->cpus[0] = NULL; 307 308 for (i = 1; i < kvm->nrcpus; i++) { 309 if (kvm->cpus[i]->is_running) { 310 pthread_kill(kvm->cpus[i]->thread, SIGKVMEXIT); 311 if (pthread_join(kvm->cpus[i]->thread, &ret) != 0) 312 die("pthread_join"); 313 kvm_cpu__delete(kvm->cpus[i]); 314 } 315 if (ret == NULL) 316 r = 0; 317 } 318 319 free(kvm->cpus); 320 321 kvm->nrcpus = 0; 322 323 close(task_eventfd); 324 325 return r; 326 } 327 core_exit(kvm_cpu__exit); 328