1 #include "kvm/kvm-cpu.h" 2 3 #include "kvm/symbol.h" 4 #include "kvm/util.h" 5 #include "kvm/kvm.h" 6 #include "kvm/virtio.h" 7 #include "kvm/mutex.h" 8 #include "kvm/barrier.h" 9 10 #include <sys/ioctl.h> 11 #include <sys/mman.h> 12 #include <sys/eventfd.h> 13 #include <signal.h> 14 #include <stdlib.h> 15 #include <string.h> 16 #include <errno.h> 17 #include <stdio.h> 18 19 extern __thread struct kvm_cpu *current_kvm_cpu; 20 21 int __attribute__((weak)) kvm_cpu__get_endianness(struct kvm_cpu *vcpu) 22 { 23 return VIRTIO_ENDIAN_HOST; 24 } 25 26 void kvm_cpu__enable_singlestep(struct kvm_cpu *vcpu) 27 { 28 struct kvm_guest_debug debug = { 29 .control = KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_SINGLESTEP, 30 }; 31 32 if (ioctl(vcpu->vcpu_fd, KVM_SET_GUEST_DEBUG, &debug) < 0) 33 pr_warning("KVM_SET_GUEST_DEBUG failed"); 34 } 35 36 void kvm_cpu__run(struct kvm_cpu *vcpu) 37 { 38 int err; 39 40 if (!vcpu->is_running) 41 return; 42 43 err = ioctl(vcpu->vcpu_fd, KVM_RUN, 0); 44 if (err < 0 && (errno != EINTR && errno != EAGAIN)) 45 die_perror("KVM_RUN failed"); 46 } 47 48 static void kvm_cpu_signal_handler(int signum) 49 { 50 if (signum == SIGKVMEXIT) { 51 if (current_kvm_cpu && current_kvm_cpu->is_running) 52 current_kvm_cpu->is_running = false; 53 } else if (signum == SIGKVMPAUSE) { 54 current_kvm_cpu->paused = 1; 55 } 56 57 /* For SIGKVMTASK cpu->task is already set */ 58 } 59 60 static void kvm_cpu__handle_coalesced_mmio(struct kvm_cpu *cpu) 61 { 62 if (cpu->ring) { 63 while (cpu->ring->first != cpu->ring->last) { 64 struct kvm_coalesced_mmio *m; 65 m = &cpu->ring->coalesced_mmio[cpu->ring->first]; 66 kvm_cpu__emulate_mmio(cpu, 67 m->phys_addr, 68 m->data, 69 m->len, 70 1); 71 cpu->ring->first = (cpu->ring->first + 1) % KVM_COALESCED_MMIO_MAX; 72 } 73 } 74 } 75 76 static DEFINE_MUTEX(task_lock); 77 static int task_eventfd; 78 79 static void kvm_cpu__run_task(struct kvm_cpu *cpu) 80 { 81 u64 inc = 1; 82 83 pr_debug("Running task %p on cpu %lu", cpu->task, cpu->cpu_id); 84 85 /* Make sure we see the store to cpu->task */ 86 rmb(); 87 cpu->task->func(cpu, cpu->task->data); 88 89 /* Clear task before we signal completion */ 90 cpu->task = NULL; 91 wmb(); 92 93 if (write(task_eventfd, &inc, sizeof(inc)) < 0) 94 die("Failed notifying of completed task."); 95 } 96 97 void kvm_cpu__run_on_all_cpus(struct kvm *kvm, struct kvm_cpu_task *task) 98 { 99 int i, done = 0; 100 101 pr_debug("Running task %p on all cpus", task); 102 103 mutex_lock(&task_lock); 104 105 for (i = 0; i < kvm->nrcpus; i++) { 106 if (kvm->cpus[i]->task) { 107 /* Should never happen */ 108 die("CPU %d already has a task pending!", i); 109 } 110 111 kvm->cpus[i]->task = task; 112 wmb(); 113 114 if (kvm->cpus[i] == current_kvm_cpu) 115 kvm_cpu__run_task(current_kvm_cpu); 116 else 117 pthread_kill(kvm->cpus[i]->thread, SIGKVMTASK); 118 } 119 120 while (done < kvm->nrcpus) { 121 u64 count; 122 123 if (read(task_eventfd, &count, sizeof(count)) < 0) 124 die("Failed reading task eventfd"); 125 126 done += count; 127 } 128 129 mutex_unlock(&task_lock); 130 } 131 132 int kvm_cpu__start(struct kvm_cpu *cpu) 133 { 134 sigset_t sigset; 135 136 sigemptyset(&sigset); 137 sigaddset(&sigset, SIGALRM); 138 139 pthread_sigmask(SIG_BLOCK, &sigset, NULL); 140 141 signal(SIGKVMEXIT, kvm_cpu_signal_handler); 142 signal(SIGKVMPAUSE, kvm_cpu_signal_handler); 143 signal(SIGKVMTASK, kvm_cpu_signal_handler); 144 145 kvm_cpu__reset_vcpu(cpu); 146 147 if (cpu->kvm->cfg.single_step) 148 kvm_cpu__enable_singlestep(cpu); 149 150 while (cpu->is_running) { 151 if (cpu->paused) 152 kvm__notify_paused(); 153 154 if (cpu->needs_nmi) { 155 kvm_cpu__arch_nmi(cpu); 156 cpu->needs_nmi = 0; 157 } 158 159 if (cpu->task) 160 kvm_cpu__run_task(cpu); 161 162 kvm_cpu__run(cpu); 163 164 switch (cpu->kvm_run->exit_reason) { 165 case KVM_EXIT_UNKNOWN: 166 break; 167 case KVM_EXIT_DEBUG: 168 kvm_cpu__show_registers(cpu); 169 kvm_cpu__show_code(cpu); 170 break; 171 case KVM_EXIT_IO: { 172 bool ret; 173 174 ret = kvm_cpu__emulate_io(cpu, 175 cpu->kvm_run->io.port, 176 (u8 *)cpu->kvm_run + 177 cpu->kvm_run->io.data_offset, 178 cpu->kvm_run->io.direction, 179 cpu->kvm_run->io.size, 180 cpu->kvm_run->io.count); 181 182 if (!ret) 183 goto panic_kvm; 184 break; 185 } 186 case KVM_EXIT_MMIO: { 187 bool ret; 188 189 /* 190 * If we had MMIO exit, coalesced ring should be processed 191 * *before* processing the exit itself 192 */ 193 kvm_cpu__handle_coalesced_mmio(cpu); 194 195 ret = kvm_cpu__emulate_mmio(cpu, 196 cpu->kvm_run->mmio.phys_addr, 197 cpu->kvm_run->mmio.data, 198 cpu->kvm_run->mmio.len, 199 cpu->kvm_run->mmio.is_write); 200 201 if (!ret) 202 goto panic_kvm; 203 break; 204 } 205 case KVM_EXIT_INTR: 206 if (cpu->is_running) 207 break; 208 goto exit_kvm; 209 case KVM_EXIT_SHUTDOWN: 210 goto exit_kvm; 211 case KVM_EXIT_SYSTEM_EVENT: 212 /* 213 * Print the type of system event and 214 * treat all system events as shutdown request. 215 */ 216 switch (cpu->kvm_run->system_event.type) { 217 default: 218 pr_warning("unknown system event type %d", 219 cpu->kvm_run->system_event.type); 220 /* fall through for now */ 221 case KVM_SYSTEM_EVENT_RESET: 222 /* Fall through for now */ 223 case KVM_SYSTEM_EVENT_SHUTDOWN: 224 /* 225 * Ensure that all VCPUs are torn down, 226 * regardless of which CPU generated the event. 227 */ 228 kvm__reboot(cpu->kvm); 229 goto exit_kvm; 230 }; 231 break; 232 default: { 233 bool ret; 234 235 ret = kvm_cpu__handle_exit(cpu); 236 if (!ret) 237 goto panic_kvm; 238 break; 239 } 240 } 241 kvm_cpu__handle_coalesced_mmio(cpu); 242 } 243 244 exit_kvm: 245 return 0; 246 247 panic_kvm: 248 return 1; 249 } 250 251 int kvm_cpu__init(struct kvm *kvm) 252 { 253 int max_cpus, recommended_cpus, i; 254 255 max_cpus = kvm__max_cpus(kvm); 256 recommended_cpus = kvm__recommended_cpus(kvm); 257 258 if (kvm->cfg.nrcpus > max_cpus) { 259 printf(" # Limit the number of CPUs to %d\n", max_cpus); 260 kvm->cfg.nrcpus = max_cpus; 261 } else if (kvm->cfg.nrcpus > recommended_cpus) { 262 printf(" # Warning: The maximum recommended amount of VCPUs" 263 " is %d\n", recommended_cpus); 264 } 265 266 kvm->nrcpus = kvm->cfg.nrcpus; 267 268 task_eventfd = eventfd(0, 0); 269 if (task_eventfd < 0) { 270 pr_warning("Couldn't create task_eventfd"); 271 return task_eventfd; 272 } 273 274 /* Alloc one pointer too many, so array ends up 0-terminated */ 275 kvm->cpus = calloc(kvm->nrcpus + 1, sizeof(void *)); 276 if (!kvm->cpus) { 277 pr_warning("Couldn't allocate array for %d CPUs", kvm->nrcpus); 278 return -ENOMEM; 279 } 280 281 for (i = 0; i < kvm->nrcpus; i++) { 282 kvm->cpus[i] = kvm_cpu__arch_init(kvm, i); 283 if (!kvm->cpus[i]) { 284 pr_warning("unable to initialize KVM VCPU"); 285 goto fail_alloc; 286 } 287 } 288 289 return 0; 290 291 fail_alloc: 292 for (i = 0; i < kvm->nrcpus; i++) 293 free(kvm->cpus[i]); 294 return -ENOMEM; 295 } 296 base_init(kvm_cpu__init); 297 298 int kvm_cpu__exit(struct kvm *kvm) 299 { 300 int i, r; 301 void *ret = NULL; 302 303 kvm_cpu__delete(kvm->cpus[0]); 304 kvm->cpus[0] = NULL; 305 306 kvm__pause(kvm); 307 for (i = 1; i < kvm->nrcpus; i++) { 308 if (kvm->cpus[i]->is_running) { 309 pthread_kill(kvm->cpus[i]->thread, SIGKVMEXIT); 310 if (pthread_join(kvm->cpus[i]->thread, &ret) != 0) 311 die("pthread_join"); 312 kvm_cpu__delete(kvm->cpus[i]); 313 } 314 if (ret == NULL) 315 r = 0; 316 } 317 kvm__continue(kvm); 318 319 free(kvm->cpus); 320 321 kvm->nrcpus = 0; 322 323 close(task_eventfd); 324 325 return r; 326 } 327