1 /*
2 * iommufd container backend
3 *
4 * Copyright (C) 2023 Intel Corporation.
5 * Copyright Red Hat, Inc. 2023
6 *
7 * Authors: Yi Liu <yi.l.liu@intel.com>
8 * Eric Auger <eric.auger@redhat.com>
9 *
10 * SPDX-License-Identifier: GPL-2.0-or-later
11 */
12
13 #include "qemu/osdep.h"
14 #include <sys/ioctl.h>
15 #include <linux/vfio.h>
16 #include <linux/iommufd.h>
17
18 #include "hw/vfio/vfio-device.h"
19 #include "qemu/error-report.h"
20 #include "trace.h"
21 #include "qapi/error.h"
22 #include "system/iommufd.h"
23 #include "hw/qdev-core.h"
24 #include "hw/vfio/vfio-cpr.h"
25 #include "system/reset.h"
26 #include "qemu/cutils.h"
27 #include "qemu/chardev_open.h"
28 #include "pci.h"
29 #include "vfio-iommufd.h"
30 #include "vfio-helpers.h"
31 #include "vfio-listener.h"
32
33 #define TYPE_HOST_IOMMU_DEVICE_IOMMUFD_VFIO \
34 TYPE_HOST_IOMMU_DEVICE_IOMMUFD "-vfio"
35
iommufd_cdev_map(const VFIOContainerBase * bcontainer,hwaddr iova,ram_addr_t size,void * vaddr,bool readonly,MemoryRegion * mr)36 static int iommufd_cdev_map(const VFIOContainerBase *bcontainer, hwaddr iova,
37 ram_addr_t size, void *vaddr, bool readonly,
38 MemoryRegion *mr)
39 {
40 const VFIOIOMMUFDContainer *container =
41 container_of(bcontainer, VFIOIOMMUFDContainer, bcontainer);
42
43 return iommufd_backend_map_dma(container->be,
44 container->ioas_id,
45 iova, size, vaddr, readonly);
46 }
47
iommufd_cdev_unmap(const VFIOContainerBase * bcontainer,hwaddr iova,ram_addr_t size,IOMMUTLBEntry * iotlb,bool unmap_all)48 static int iommufd_cdev_unmap(const VFIOContainerBase *bcontainer,
49 hwaddr iova, ram_addr_t size,
50 IOMMUTLBEntry *iotlb, bool unmap_all)
51 {
52 const VFIOIOMMUFDContainer *container =
53 container_of(bcontainer, VFIOIOMMUFDContainer, bcontainer);
54
55 /* unmap in halves */
56 if (unmap_all) {
57 Int128 llsize = int128_rshift(int128_2_64(), 1);
58 int ret;
59
60 ret = iommufd_backend_unmap_dma(container->be, container->ioas_id,
61 0, int128_get64(llsize));
62
63 if (ret == 0) {
64 ret = iommufd_backend_unmap_dma(container->be, container->ioas_id,
65 int128_get64(llsize),
66 int128_get64(llsize));
67 }
68
69 return ret;
70 }
71
72 /* TODO: Handle dma_unmap_bitmap with iotlb args (migration) */
73 return iommufd_backend_unmap_dma(container->be,
74 container->ioas_id, iova, size);
75 }
76
iommufd_cdev_kvm_device_add(VFIODevice * vbasedev,Error ** errp)77 static bool iommufd_cdev_kvm_device_add(VFIODevice *vbasedev, Error **errp)
78 {
79 return !vfio_kvm_device_add_fd(vbasedev->fd, errp);
80 }
81
iommufd_cdev_kvm_device_del(VFIODevice * vbasedev)82 static void iommufd_cdev_kvm_device_del(VFIODevice *vbasedev)
83 {
84 Error *err = NULL;
85
86 if (vfio_kvm_device_del_fd(vbasedev->fd, &err)) {
87 error_report_err(err);
88 }
89 }
90
iommufd_cdev_connect_and_bind(VFIODevice * vbasedev,Error ** errp)91 static bool iommufd_cdev_connect_and_bind(VFIODevice *vbasedev, Error **errp)
92 {
93 IOMMUFDBackend *iommufd = vbasedev->iommufd;
94 struct vfio_device_bind_iommufd bind = {
95 .argsz = sizeof(bind),
96 .flags = 0,
97 };
98
99 if (!iommufd_backend_connect(iommufd, errp)) {
100 return false;
101 }
102
103 /*
104 * Add device to kvm-vfio to be prepared for the tracking
105 * in KVM. Especially for some emulated devices, it requires
106 * to have kvm information in the device open.
107 */
108 if (!iommufd_cdev_kvm_device_add(vbasedev, errp)) {
109 goto err_kvm_device_add;
110 }
111
112 /* Bind device to iommufd */
113 bind.iommufd = iommufd->fd;
114 if (ioctl(vbasedev->fd, VFIO_DEVICE_BIND_IOMMUFD, &bind)) {
115 error_setg_errno(errp, errno, "error bind device fd=%d to iommufd=%d",
116 vbasedev->fd, bind.iommufd);
117 goto err_bind;
118 }
119
120 vbasedev->devid = bind.out_devid;
121 trace_iommufd_cdev_connect_and_bind(bind.iommufd, vbasedev->name,
122 vbasedev->fd, vbasedev->devid);
123 return true;
124 err_bind:
125 iommufd_cdev_kvm_device_del(vbasedev);
126 err_kvm_device_add:
127 iommufd_backend_disconnect(iommufd);
128 return false;
129 }
130
iommufd_cdev_unbind_and_disconnect(VFIODevice * vbasedev)131 static void iommufd_cdev_unbind_and_disconnect(VFIODevice *vbasedev)
132 {
133 /* Unbind is automatically conducted when device fd is closed */
134 iommufd_cdev_kvm_device_del(vbasedev);
135 iommufd_backend_disconnect(vbasedev->iommufd);
136 }
137
iommufd_hwpt_dirty_tracking(VFIOIOASHwpt * hwpt)138 static bool iommufd_hwpt_dirty_tracking(VFIOIOASHwpt *hwpt)
139 {
140 return hwpt && hwpt->hwpt_flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING;
141 }
142
iommufd_set_dirty_page_tracking(const VFIOContainerBase * bcontainer,bool start,Error ** errp)143 static int iommufd_set_dirty_page_tracking(const VFIOContainerBase *bcontainer,
144 bool start, Error **errp)
145 {
146 const VFIOIOMMUFDContainer *container =
147 container_of(bcontainer, VFIOIOMMUFDContainer, bcontainer);
148 VFIOIOASHwpt *hwpt;
149
150 QLIST_FOREACH(hwpt, &container->hwpt_list, next) {
151 if (!iommufd_hwpt_dirty_tracking(hwpt)) {
152 continue;
153 }
154
155 if (!iommufd_backend_set_dirty_tracking(container->be,
156 hwpt->hwpt_id, start, errp)) {
157 goto err;
158 }
159 }
160
161 return 0;
162
163 err:
164 QLIST_FOREACH(hwpt, &container->hwpt_list, next) {
165 if (!iommufd_hwpt_dirty_tracking(hwpt)) {
166 continue;
167 }
168 iommufd_backend_set_dirty_tracking(container->be,
169 hwpt->hwpt_id, !start, NULL);
170 }
171 return -EINVAL;
172 }
173
iommufd_query_dirty_bitmap(const VFIOContainerBase * bcontainer,VFIOBitmap * vbmap,hwaddr iova,hwaddr size,Error ** errp)174 static int iommufd_query_dirty_bitmap(const VFIOContainerBase *bcontainer,
175 VFIOBitmap *vbmap, hwaddr iova,
176 hwaddr size, Error **errp)
177 {
178 VFIOIOMMUFDContainer *container = container_of(bcontainer,
179 VFIOIOMMUFDContainer,
180 bcontainer);
181 unsigned long page_size = qemu_real_host_page_size();
182 VFIOIOASHwpt *hwpt;
183
184 QLIST_FOREACH(hwpt, &container->hwpt_list, next) {
185 if (!iommufd_hwpt_dirty_tracking(hwpt)) {
186 continue;
187 }
188
189 if (!iommufd_backend_get_dirty_bitmap(container->be, hwpt->hwpt_id,
190 iova, size, page_size,
191 (uint64_t *)vbmap->bitmap,
192 errp)) {
193 return -EINVAL;
194 }
195 }
196
197 return 0;
198 }
199
iommufd_cdev_getfd(const char * sysfs_path,Error ** errp)200 static int iommufd_cdev_getfd(const char *sysfs_path, Error **errp)
201 {
202 ERRP_GUARD();
203 long int ret = -ENOTTY;
204 g_autofree char *path = NULL;
205 g_autofree char *vfio_dev_path = NULL;
206 g_autofree char *vfio_path = NULL;
207 DIR *dir = NULL;
208 struct dirent *dent;
209 g_autofree gchar *contents = NULL;
210 gsize length;
211 int major, minor;
212 dev_t vfio_devt;
213
214 path = g_strdup_printf("%s/vfio-dev", sysfs_path);
215 dir = opendir(path);
216 if (!dir) {
217 error_setg_errno(errp, errno, "couldn't open directory %s", path);
218 goto out;
219 }
220
221 while ((dent = readdir(dir))) {
222 if (!strncmp(dent->d_name, "vfio", 4)) {
223 vfio_dev_path = g_strdup_printf("%s/%s/dev", path, dent->d_name);
224 break;
225 }
226 }
227
228 if (!vfio_dev_path) {
229 error_setg(errp, "failed to find vfio-dev/vfioX/dev");
230 goto out_close_dir;
231 }
232
233 if (!g_file_get_contents(vfio_dev_path, &contents, &length, NULL)) {
234 error_setg(errp, "failed to load \"%s\"", vfio_dev_path);
235 goto out_close_dir;
236 }
237
238 if (sscanf(contents, "%d:%d", &major, &minor) != 2) {
239 error_setg(errp, "failed to get major:minor for \"%s\"", vfio_dev_path);
240 goto out_close_dir;
241 }
242 vfio_devt = makedev(major, minor);
243
244 vfio_path = g_strdup_printf("/dev/vfio/devices/%s", dent->d_name);
245 ret = open_cdev(vfio_path, vfio_devt);
246 if (ret < 0) {
247 error_setg(errp, "Failed to open %s", vfio_path);
248 }
249
250 trace_iommufd_cdev_getfd(vfio_path, ret);
251
252 out_close_dir:
253 closedir(dir);
254 out:
255 if (*errp) {
256 error_prepend(errp, VFIO_MSG_PREFIX, path);
257 }
258
259 return ret;
260 }
261
iommufd_cdev_attach_ioas_hwpt(VFIODevice * vbasedev,uint32_t id,Error ** errp)262 static int iommufd_cdev_attach_ioas_hwpt(VFIODevice *vbasedev, uint32_t id,
263 Error **errp)
264 {
265 int iommufd = vbasedev->iommufd->fd;
266 struct vfio_device_attach_iommufd_pt attach_data = {
267 .argsz = sizeof(attach_data),
268 .flags = 0,
269 .pt_id = id,
270 };
271
272 /* Attach device to an IOAS or hwpt within iommufd */
273 if (ioctl(vbasedev->fd, VFIO_DEVICE_ATTACH_IOMMUFD_PT, &attach_data)) {
274 error_setg_errno(errp, errno,
275 "[iommufd=%d] error attach %s (%d) to id=%d",
276 iommufd, vbasedev->name, vbasedev->fd, id);
277 return -errno;
278 }
279
280 trace_iommufd_cdev_attach_ioas_hwpt(iommufd, vbasedev->name,
281 vbasedev->fd, id);
282 return 0;
283 }
284
iommufd_cdev_detach_ioas_hwpt(VFIODevice * vbasedev,Error ** errp)285 static bool iommufd_cdev_detach_ioas_hwpt(VFIODevice *vbasedev, Error **errp)
286 {
287 int iommufd = vbasedev->iommufd->fd;
288 struct vfio_device_detach_iommufd_pt detach_data = {
289 .argsz = sizeof(detach_data),
290 .flags = 0,
291 };
292
293 if (ioctl(vbasedev->fd, VFIO_DEVICE_DETACH_IOMMUFD_PT, &detach_data)) {
294 error_setg_errno(errp, errno, "detach %s failed", vbasedev->name);
295 return false;
296 }
297
298 trace_iommufd_cdev_detach_ioas_hwpt(iommufd, vbasedev->name);
299 return true;
300 }
301
iommufd_cdev_autodomains_get(VFIODevice * vbasedev,VFIOIOMMUFDContainer * container,Error ** errp)302 static bool iommufd_cdev_autodomains_get(VFIODevice *vbasedev,
303 VFIOIOMMUFDContainer *container,
304 Error **errp)
305 {
306 ERRP_GUARD();
307 IOMMUFDBackend *iommufd = vbasedev->iommufd;
308 uint32_t type, flags = 0;
309 uint64_t hw_caps;
310 VFIOIOASHwpt *hwpt;
311 uint32_t hwpt_id;
312 int ret;
313
314 /* Try to find a domain */
315 QLIST_FOREACH(hwpt, &container->hwpt_list, next) {
316 ret = iommufd_cdev_attach_ioas_hwpt(vbasedev, hwpt->hwpt_id, errp);
317 if (ret) {
318 /* -EINVAL means the domain is incompatible with the device. */
319 if (ret == -EINVAL) {
320 /*
321 * It is an expected failure and it just means we will try
322 * another domain, or create one if no existing compatible
323 * domain is found. Hence why the error is discarded below.
324 */
325 error_free(*errp);
326 *errp = NULL;
327 continue;
328 }
329
330 return false;
331 } else {
332 vbasedev->hwpt = hwpt;
333 QLIST_INSERT_HEAD(&hwpt->device_list, vbasedev, hwpt_next);
334 vbasedev->iommu_dirty_tracking = iommufd_hwpt_dirty_tracking(hwpt);
335 return true;
336 }
337 }
338
339 /*
340 * This is quite early and VFIO Migration state isn't yet fully
341 * initialized, thus rely only on IOMMU hardware capabilities as to
342 * whether IOMMU dirty tracking is going to be requested. Later
343 * vfio_migration_realize() may decide to use VF dirty tracking
344 * instead.
345 */
346 if (!iommufd_backend_get_device_info(vbasedev->iommufd, vbasedev->devid,
347 &type, NULL, 0, &hw_caps, errp)) {
348 return false;
349 }
350
351 if (hw_caps & IOMMU_HW_CAP_DIRTY_TRACKING) {
352 flags = IOMMU_HWPT_ALLOC_DIRTY_TRACKING;
353 }
354
355 if (!iommufd_backend_alloc_hwpt(iommufd, vbasedev->devid,
356 container->ioas_id, flags,
357 IOMMU_HWPT_DATA_NONE, 0, NULL,
358 &hwpt_id, errp)) {
359 return false;
360 }
361
362 hwpt = g_malloc0(sizeof(*hwpt));
363 hwpt->hwpt_id = hwpt_id;
364 hwpt->hwpt_flags = flags;
365 QLIST_INIT(&hwpt->device_list);
366
367 ret = iommufd_cdev_attach_ioas_hwpt(vbasedev, hwpt->hwpt_id, errp);
368 if (ret) {
369 iommufd_backend_free_id(container->be, hwpt->hwpt_id);
370 g_free(hwpt);
371 return false;
372 }
373
374 vbasedev->hwpt = hwpt;
375 vbasedev->iommu_dirty_tracking = iommufd_hwpt_dirty_tracking(hwpt);
376 QLIST_INSERT_HEAD(&hwpt->device_list, vbasedev, hwpt_next);
377 QLIST_INSERT_HEAD(&container->hwpt_list, hwpt, next);
378 container->bcontainer.dirty_pages_supported |=
379 vbasedev->iommu_dirty_tracking;
380 if (container->bcontainer.dirty_pages_supported &&
381 !vbasedev->iommu_dirty_tracking) {
382 warn_report("IOMMU instance for device %s doesn't support dirty tracking",
383 vbasedev->name);
384 }
385 return true;
386 }
387
iommufd_cdev_autodomains_put(VFIODevice * vbasedev,VFIOIOMMUFDContainer * container)388 static void iommufd_cdev_autodomains_put(VFIODevice *vbasedev,
389 VFIOIOMMUFDContainer *container)
390 {
391 VFIOIOASHwpt *hwpt = vbasedev->hwpt;
392
393 QLIST_REMOVE(vbasedev, hwpt_next);
394 vbasedev->hwpt = NULL;
395
396 if (QLIST_EMPTY(&hwpt->device_list)) {
397 QLIST_REMOVE(hwpt, next);
398 iommufd_backend_free_id(container->be, hwpt->hwpt_id);
399 g_free(hwpt);
400 }
401 }
402
iommufd_cdev_attach_container(VFIODevice * vbasedev,VFIOIOMMUFDContainer * container,Error ** errp)403 static bool iommufd_cdev_attach_container(VFIODevice *vbasedev,
404 VFIOIOMMUFDContainer *container,
405 Error **errp)
406 {
407 /* mdevs aren't physical devices and will fail with auto domains */
408 if (!vbasedev->mdev) {
409 return iommufd_cdev_autodomains_get(vbasedev, container, errp);
410 }
411
412 return !iommufd_cdev_attach_ioas_hwpt(vbasedev, container->ioas_id, errp);
413 }
414
iommufd_cdev_detach_container(VFIODevice * vbasedev,VFIOIOMMUFDContainer * container)415 static void iommufd_cdev_detach_container(VFIODevice *vbasedev,
416 VFIOIOMMUFDContainer *container)
417 {
418 Error *err = NULL;
419
420 if (!iommufd_cdev_detach_ioas_hwpt(vbasedev, &err)) {
421 error_report_err(err);
422 }
423
424 if (vbasedev->hwpt) {
425 iommufd_cdev_autodomains_put(vbasedev, container);
426 }
427
428 }
429
iommufd_cdev_container_destroy(VFIOIOMMUFDContainer * container)430 static void iommufd_cdev_container_destroy(VFIOIOMMUFDContainer *container)
431 {
432 VFIOContainerBase *bcontainer = &container->bcontainer;
433
434 if (!QLIST_EMPTY(&bcontainer->device_list)) {
435 return;
436 }
437 vfio_cpr_unregister_container(bcontainer);
438 vfio_listener_unregister(bcontainer);
439 iommufd_backend_free_id(container->be, container->ioas_id);
440 object_unref(container);
441 }
442
iommufd_cdev_ram_block_discard_disable(bool state)443 static int iommufd_cdev_ram_block_discard_disable(bool state)
444 {
445 /*
446 * We support coordinated discarding of RAM via the RamDiscardManager.
447 */
448 return ram_block_uncoordinated_discard_disable(state);
449 }
450
iommufd_cdev_get_info_iova_range(VFIOIOMMUFDContainer * container,uint32_t ioas_id,Error ** errp)451 static bool iommufd_cdev_get_info_iova_range(VFIOIOMMUFDContainer *container,
452 uint32_t ioas_id, Error **errp)
453 {
454 VFIOContainerBase *bcontainer = &container->bcontainer;
455 g_autofree struct iommu_ioas_iova_ranges *info = NULL;
456 struct iommu_iova_range *iova_ranges;
457 int sz, fd = container->be->fd;
458
459 info = g_malloc0(sizeof(*info));
460 info->size = sizeof(*info);
461 info->ioas_id = ioas_id;
462
463 if (ioctl(fd, IOMMU_IOAS_IOVA_RANGES, info) && errno != EMSGSIZE) {
464 goto error;
465 }
466
467 sz = info->num_iovas * sizeof(struct iommu_iova_range);
468 info = g_realloc(info, sizeof(*info) + sz);
469 info->allowed_iovas = (uintptr_t)(info + 1);
470
471 if (ioctl(fd, IOMMU_IOAS_IOVA_RANGES, info)) {
472 goto error;
473 }
474
475 iova_ranges = (struct iommu_iova_range *)(uintptr_t)info->allowed_iovas;
476
477 for (int i = 0; i < info->num_iovas; i++) {
478 Range *range = g_new(Range, 1);
479
480 range_set_bounds(range, iova_ranges[i].start, iova_ranges[i].last);
481 bcontainer->iova_ranges =
482 range_list_insert(bcontainer->iova_ranges, range);
483 }
484 bcontainer->pgsizes = info->out_iova_alignment;
485
486 return true;
487
488 error:
489 error_setg_errno(errp, errno, "Cannot get IOVA ranges");
490 return false;
491 }
492
iommufd_cdev_attach(const char * name,VFIODevice * vbasedev,AddressSpace * as,Error ** errp)493 static bool iommufd_cdev_attach(const char *name, VFIODevice *vbasedev,
494 AddressSpace *as, Error **errp)
495 {
496 VFIOContainerBase *bcontainer;
497 VFIOIOMMUFDContainer *container;
498 VFIOAddressSpace *space;
499 struct vfio_device_info dev_info = { .argsz = sizeof(dev_info) };
500 int ret, devfd;
501 uint32_t ioas_id;
502 Error *err = NULL;
503 const VFIOIOMMUClass *iommufd_vioc =
504 VFIO_IOMMU_CLASS(object_class_by_name(TYPE_VFIO_IOMMU_IOMMUFD));
505
506 if (vbasedev->fd < 0) {
507 devfd = iommufd_cdev_getfd(vbasedev->sysfsdev, errp);
508 if (devfd < 0) {
509 return false;
510 }
511 vbasedev->fd = devfd;
512 } else {
513 devfd = vbasedev->fd;
514 }
515
516 if (!iommufd_cdev_connect_and_bind(vbasedev, errp)) {
517 goto err_connect_bind;
518 }
519
520 space = vfio_address_space_get(as);
521
522 /* try to attach to an existing container in this space */
523 QLIST_FOREACH(bcontainer, &space->containers, next) {
524 container = container_of(bcontainer, VFIOIOMMUFDContainer, bcontainer);
525 if (VFIO_IOMMU_GET_CLASS(bcontainer) != iommufd_vioc ||
526 vbasedev->iommufd != container->be) {
527 continue;
528 }
529 if (!iommufd_cdev_attach_container(vbasedev, container, &err)) {
530 const char *msg = error_get_pretty(err);
531
532 trace_iommufd_cdev_fail_attach_existing_container(msg);
533 error_free(err);
534 err = NULL;
535 } else {
536 ret = iommufd_cdev_ram_block_discard_disable(true);
537 if (ret) {
538 error_setg_errno(errp, -ret,
539 "Cannot set discarding of RAM broken");
540 goto err_discard_disable;
541 }
542 goto found_container;
543 }
544 }
545
546 /* Need to allocate a new dedicated container */
547 if (!iommufd_backend_alloc_ioas(vbasedev->iommufd, &ioas_id, errp)) {
548 goto err_alloc_ioas;
549 }
550
551 trace_iommufd_cdev_alloc_ioas(vbasedev->iommufd->fd, ioas_id);
552
553 container = VFIO_IOMMU_IOMMUFD(object_new(TYPE_VFIO_IOMMU_IOMMUFD));
554 container->be = vbasedev->iommufd;
555 container->ioas_id = ioas_id;
556 QLIST_INIT(&container->hwpt_list);
557
558 bcontainer = &container->bcontainer;
559 vfio_address_space_insert(space, bcontainer);
560
561 if (!iommufd_cdev_attach_container(vbasedev, container, errp)) {
562 goto err_attach_container;
563 }
564
565 ret = iommufd_cdev_ram_block_discard_disable(true);
566 if (ret) {
567 error_setg_errno(errp, -ret, "Cannot set discarding of RAM broken");
568 goto err_discard_disable;
569 }
570
571 if (!iommufd_cdev_get_info_iova_range(container, ioas_id, &err)) {
572 error_append_hint(&err,
573 "Fallback to default 64bit IOVA range and 4K page size\n");
574 warn_report_err(err);
575 err = NULL;
576 bcontainer->pgsizes = qemu_real_host_page_size();
577 }
578
579 if (!vfio_listener_register(bcontainer, errp)) {
580 goto err_listener_register;
581 }
582
583 if (!vfio_cpr_register_container(bcontainer, errp)) {
584 goto err_listener_register;
585 }
586
587 bcontainer->initialized = true;
588
589 found_container:
590 ret = ioctl(devfd, VFIO_DEVICE_GET_INFO, &dev_info);
591 if (ret) {
592 error_setg_errno(errp, errno, "error getting device info");
593 goto err_listener_register;
594 }
595
596 /*
597 * Do not move this code before attachment! The nested IOMMU support
598 * needs device and hwpt id which are generated only after attachment.
599 */
600 if (!vfio_device_hiod_create_and_realize(vbasedev,
601 TYPE_HOST_IOMMU_DEVICE_IOMMUFD_VFIO, errp)) {
602 goto err_listener_register;
603 }
604
605 /*
606 * TODO: examine RAM_BLOCK_DISCARD stuff, should we do group level
607 * for discarding incompatibility check as well?
608 */
609 if (vbasedev->ram_block_discard_allowed) {
610 iommufd_cdev_ram_block_discard_disable(false);
611 }
612
613 vfio_device_prepare(vbasedev, bcontainer, &dev_info);
614
615 trace_iommufd_cdev_device_info(vbasedev->name, devfd, vbasedev->num_irqs,
616 vbasedev->num_regions, vbasedev->flags);
617 return true;
618
619 err_listener_register:
620 iommufd_cdev_ram_block_discard_disable(false);
621 err_discard_disable:
622 iommufd_cdev_detach_container(vbasedev, container);
623 err_attach_container:
624 iommufd_cdev_container_destroy(container);
625 err_alloc_ioas:
626 vfio_address_space_put(space);
627 iommufd_cdev_unbind_and_disconnect(vbasedev);
628 err_connect_bind:
629 close(vbasedev->fd);
630 return false;
631 }
632
iommufd_cdev_detach(VFIODevice * vbasedev)633 static void iommufd_cdev_detach(VFIODevice *vbasedev)
634 {
635 VFIOContainerBase *bcontainer = vbasedev->bcontainer;
636 VFIOAddressSpace *space = bcontainer->space;
637 VFIOIOMMUFDContainer *container = container_of(bcontainer,
638 VFIOIOMMUFDContainer,
639 bcontainer);
640 vfio_device_unprepare(vbasedev);
641
642 if (!vbasedev->ram_block_discard_allowed) {
643 iommufd_cdev_ram_block_discard_disable(false);
644 }
645
646 object_unref(vbasedev->hiod);
647 iommufd_cdev_detach_container(vbasedev, container);
648 iommufd_cdev_container_destroy(container);
649 vfio_address_space_put(space);
650
651 iommufd_cdev_unbind_and_disconnect(vbasedev);
652 close(vbasedev->fd);
653 }
654
iommufd_cdev_pci_find_by_devid(__u32 devid)655 static VFIODevice *iommufd_cdev_pci_find_by_devid(__u32 devid)
656 {
657 VFIODevice *vbasedev_iter;
658 const VFIOIOMMUClass *iommufd_vioc =
659 VFIO_IOMMU_CLASS(object_class_by_name(TYPE_VFIO_IOMMU_IOMMUFD));
660
661 QLIST_FOREACH(vbasedev_iter, &vfio_device_list, global_next) {
662 if (VFIO_IOMMU_GET_CLASS(vbasedev_iter->bcontainer) != iommufd_vioc) {
663 continue;
664 }
665 if (devid == vbasedev_iter->devid) {
666 return vbasedev_iter;
667 }
668 }
669 return NULL;
670 }
671
672 static VFIOPCIDevice *
iommufd_cdev_dep_get_realized_vpdev(struct vfio_pci_dependent_device * dep_dev,VFIODevice * reset_dev)673 iommufd_cdev_dep_get_realized_vpdev(struct vfio_pci_dependent_device *dep_dev,
674 VFIODevice *reset_dev)
675 {
676 VFIODevice *vbasedev_tmp;
677
678 if (dep_dev->devid == reset_dev->devid ||
679 dep_dev->devid == VFIO_PCI_DEVID_OWNED) {
680 return NULL;
681 }
682
683 vbasedev_tmp = iommufd_cdev_pci_find_by_devid(dep_dev->devid);
684 if (!vbasedev_tmp || !vbasedev_tmp->dev->realized ||
685 vbasedev_tmp->type != VFIO_DEVICE_TYPE_PCI) {
686 return NULL;
687 }
688
689 return container_of(vbasedev_tmp, VFIOPCIDevice, vbasedev);
690 }
691
iommufd_cdev_pci_hot_reset(VFIODevice * vbasedev,bool single)692 static int iommufd_cdev_pci_hot_reset(VFIODevice *vbasedev, bool single)
693 {
694 VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
695 struct vfio_pci_hot_reset_info *info = NULL;
696 struct vfio_pci_dependent_device *devices;
697 struct vfio_pci_hot_reset *reset;
698 int ret, i;
699 bool multi = false;
700
701 trace_vfio_pci_hot_reset(vdev->vbasedev.name, single ? "one" : "multi");
702
703 if (!single) {
704 vfio_pci_pre_reset(vdev);
705 }
706 vdev->vbasedev.needs_reset = false;
707
708 ret = vfio_pci_get_pci_hot_reset_info(vdev, &info);
709
710 if (ret) {
711 goto out_single;
712 }
713
714 assert(info->flags & VFIO_PCI_HOT_RESET_FLAG_DEV_ID);
715
716 devices = &info->devices[0];
717
718 if (!(info->flags & VFIO_PCI_HOT_RESET_FLAG_DEV_ID_OWNED)) {
719 if (!vdev->has_pm_reset) {
720 for (i = 0; i < info->count; i++) {
721 if (devices[i].devid == VFIO_PCI_DEVID_NOT_OWNED) {
722 error_report("vfio: Cannot reset device %s, "
723 "depends on device %04x:%02x:%02x.%x "
724 "which is not owned.",
725 vdev->vbasedev.name, devices[i].segment,
726 devices[i].bus, PCI_SLOT(devices[i].devfn),
727 PCI_FUNC(devices[i].devfn));
728 }
729 }
730 }
731 ret = -EPERM;
732 goto out_single;
733 }
734
735 trace_vfio_pci_hot_reset_has_dep_devices(vdev->vbasedev.name);
736
737 for (i = 0; i < info->count; i++) {
738 VFIOPCIDevice *tmp;
739
740 trace_iommufd_cdev_pci_hot_reset_dep_devices(devices[i].segment,
741 devices[i].bus,
742 PCI_SLOT(devices[i].devfn),
743 PCI_FUNC(devices[i].devfn),
744 devices[i].devid);
745
746 /*
747 * If a VFIO cdev device is resettable, all the dependent devices
748 * are either bound to same iommufd or within same iommu_groups as
749 * one of the iommufd bound devices.
750 */
751 assert(devices[i].devid != VFIO_PCI_DEVID_NOT_OWNED);
752
753 tmp = iommufd_cdev_dep_get_realized_vpdev(&devices[i], &vdev->vbasedev);
754 if (!tmp) {
755 continue;
756 }
757
758 if (single) {
759 ret = -EINVAL;
760 goto out_single;
761 }
762 vfio_pci_pre_reset(tmp);
763 tmp->vbasedev.needs_reset = false;
764 multi = true;
765 }
766
767 if (!single && !multi) {
768 ret = -EINVAL;
769 goto out_single;
770 }
771
772 /* Use zero length array for hot reset with iommufd backend */
773 reset = g_malloc0(sizeof(*reset));
774 reset->argsz = sizeof(*reset);
775
776 /* Bus reset! */
777 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_PCI_HOT_RESET, reset);
778 g_free(reset);
779 if (ret) {
780 ret = -errno;
781 }
782
783 trace_vfio_pci_hot_reset_result(vdev->vbasedev.name,
784 ret ? strerror(errno) : "Success");
785
786 /* Re-enable INTx on affected devices */
787 for (i = 0; i < info->count; i++) {
788 VFIOPCIDevice *tmp;
789
790 tmp = iommufd_cdev_dep_get_realized_vpdev(&devices[i], &vdev->vbasedev);
791 if (!tmp) {
792 continue;
793 }
794 vfio_pci_post_reset(tmp);
795 }
796 out_single:
797 if (!single) {
798 vfio_pci_post_reset(vdev);
799 }
800 g_free(info);
801
802 return ret;
803 }
804
vfio_iommu_iommufd_class_init(ObjectClass * klass,const void * data)805 static void vfio_iommu_iommufd_class_init(ObjectClass *klass, const void *data)
806 {
807 VFIOIOMMUClass *vioc = VFIO_IOMMU_CLASS(klass);
808
809 vioc->dma_map = iommufd_cdev_map;
810 vioc->dma_unmap = iommufd_cdev_unmap;
811 vioc->attach_device = iommufd_cdev_attach;
812 vioc->detach_device = iommufd_cdev_detach;
813 vioc->pci_hot_reset = iommufd_cdev_pci_hot_reset;
814 vioc->set_dirty_page_tracking = iommufd_set_dirty_page_tracking;
815 vioc->query_dirty_bitmap = iommufd_query_dirty_bitmap;
816 };
817
818 static bool
host_iommu_device_iommufd_vfio_attach_hwpt(HostIOMMUDeviceIOMMUFD * idev,uint32_t hwpt_id,Error ** errp)819 host_iommu_device_iommufd_vfio_attach_hwpt(HostIOMMUDeviceIOMMUFD *idev,
820 uint32_t hwpt_id, Error **errp)
821 {
822 VFIODevice *vbasedev = HOST_IOMMU_DEVICE(idev)->agent;
823
824 return !iommufd_cdev_attach_ioas_hwpt(vbasedev, hwpt_id, errp);
825 }
826
827 static bool
host_iommu_device_iommufd_vfio_detach_hwpt(HostIOMMUDeviceIOMMUFD * idev,Error ** errp)828 host_iommu_device_iommufd_vfio_detach_hwpt(HostIOMMUDeviceIOMMUFD *idev,
829 Error **errp)
830 {
831 VFIODevice *vbasedev = HOST_IOMMU_DEVICE(idev)->agent;
832
833 return iommufd_cdev_detach_ioas_hwpt(vbasedev, errp);
834 }
835
hiod_iommufd_vfio_realize(HostIOMMUDevice * hiod,void * opaque,Error ** errp)836 static bool hiod_iommufd_vfio_realize(HostIOMMUDevice *hiod, void *opaque,
837 Error **errp)
838 {
839 VFIODevice *vdev = opaque;
840 HostIOMMUDeviceIOMMUFD *idev;
841 HostIOMMUDeviceCaps *caps = &hiod->caps;
842 VendorCaps *vendor_caps = &caps->vendor_caps;
843 enum iommu_hw_info_type type;
844 uint64_t hw_caps;
845
846 hiod->agent = opaque;
847
848 if (!iommufd_backend_get_device_info(vdev->iommufd, vdev->devid, &type,
849 vendor_caps, sizeof(*vendor_caps),
850 &hw_caps, errp)) {
851 return false;
852 }
853
854 hiod->name = g_strdup(vdev->name);
855 caps->type = type;
856 caps->hw_caps = hw_caps;
857
858 idev = HOST_IOMMU_DEVICE_IOMMUFD(hiod);
859 idev->iommufd = vdev->iommufd;
860 idev->devid = vdev->devid;
861 idev->hwpt_id = vdev->hwpt->hwpt_id;
862
863 return true;
864 }
865
866 static GList *
hiod_iommufd_vfio_get_iova_ranges(HostIOMMUDevice * hiod)867 hiod_iommufd_vfio_get_iova_ranges(HostIOMMUDevice *hiod)
868 {
869 VFIODevice *vdev = hiod->agent;
870
871 g_assert(vdev);
872 return vfio_container_get_iova_ranges(vdev->bcontainer);
873 }
874
875 static uint64_t
hiod_iommufd_vfio_get_page_size_mask(HostIOMMUDevice * hiod)876 hiod_iommufd_vfio_get_page_size_mask(HostIOMMUDevice *hiod)
877 {
878 VFIODevice *vdev = hiod->agent;
879
880 g_assert(vdev);
881 return vfio_container_get_page_size_mask(vdev->bcontainer);
882 }
883
884
hiod_iommufd_vfio_class_init(ObjectClass * oc,const void * data)885 static void hiod_iommufd_vfio_class_init(ObjectClass *oc, const void *data)
886 {
887 HostIOMMUDeviceClass *hiodc = HOST_IOMMU_DEVICE_CLASS(oc);
888 HostIOMMUDeviceIOMMUFDClass *idevc = HOST_IOMMU_DEVICE_IOMMUFD_CLASS(oc);
889
890 hiodc->realize = hiod_iommufd_vfio_realize;
891 hiodc->get_iova_ranges = hiod_iommufd_vfio_get_iova_ranges;
892 hiodc->get_page_size_mask = hiod_iommufd_vfio_get_page_size_mask;
893
894 idevc->attach_hwpt = host_iommu_device_iommufd_vfio_attach_hwpt;
895 idevc->detach_hwpt = host_iommu_device_iommufd_vfio_detach_hwpt;
896 };
897
898 static const TypeInfo types[] = {
899 {
900 .name = TYPE_VFIO_IOMMU_IOMMUFD,
901 .parent = TYPE_VFIO_IOMMU,
902 .instance_size = sizeof(VFIOIOMMUFDContainer),
903 .class_init = vfio_iommu_iommufd_class_init,
904 }, {
905 .name = TYPE_HOST_IOMMU_DEVICE_IOMMUFD_VFIO,
906 .parent = TYPE_HOST_IOMMU_DEVICE_IOMMUFD,
907 .class_init = hiod_iommufd_vfio_class_init,
908 }
909 };
910
911 DEFINE_TYPES(types)
912