1*7bc86ccbSPierrick Bouvier#!/usr/bin/env python3 2*7bc86ccbSPierrick Bouvier# 3*7bc86ccbSPierrick Bouvier# Boots a nested guest and compare content of a device (passthrough) to a 4*7bc86ccbSPierrick Bouvier# reference image. Both vfio group and iommufd passthrough methods are tested. 5*7bc86ccbSPierrick Bouvier# 6*7bc86ccbSPierrick Bouvier# Copyright (c) 2025 Linaro Ltd. 7*7bc86ccbSPierrick Bouvier# 8*7bc86ccbSPierrick Bouvier# Author: Pierrick Bouvier <pierrick.bouvier@linaro.org> 9*7bc86ccbSPierrick Bouvier# 10*7bc86ccbSPierrick Bouvier# SPDX-License-Identifier: GPL-2.0-or-later 11*7bc86ccbSPierrick Bouvier 12*7bc86ccbSPierrick Bouvierimport os 13*7bc86ccbSPierrick Bouvier 14*7bc86ccbSPierrick Bouvierfrom qemu_test import QemuSystemTest, Asset 15*7bc86ccbSPierrick Bouvierfrom qemu_test import exec_command, wait_for_console_pattern 16*7bc86ccbSPierrick Bouvierfrom qemu_test import exec_command_and_wait_for_pattern 17*7bc86ccbSPierrick Bouvierfrom random import randbytes 18*7bc86ccbSPierrick Bouvier 19*7bc86ccbSPierrick Bouvierguest_script = ''' 20*7bc86ccbSPierrick Bouvier#!/usr/bin/env bash 21*7bc86ccbSPierrick Bouvier 22*7bc86ccbSPierrick Bouvierset -euo pipefail 23*7bc86ccbSPierrick Bouvierset -x 24*7bc86ccbSPierrick Bouvier 25*7bc86ccbSPierrick Bouvier# find disks from nvme serial 26*7bc86ccbSPierrick Bouvierdev_vfio=$(lsblk --nvme | grep vfio | cut -f 1 -d ' ') 27*7bc86ccbSPierrick Bouvierdev_iommufd=$(lsblk --nvme | grep iommufd | cut -f 1 -d ' ') 28*7bc86ccbSPierrick Bouvierpci_vfio=$(basename $(readlink -f /sys/block/$dev_vfio/../../../)) 29*7bc86ccbSPierrick Bouvierpci_iommufd=$(basename $(readlink -f /sys/block/$dev_iommufd/../../../)) 30*7bc86ccbSPierrick Bouvier 31*7bc86ccbSPierrick Bouvier# bind disks to vfio 32*7bc86ccbSPierrick Bouvierfor p in "$pci_vfio" "$pci_iommufd"; do 33*7bc86ccbSPierrick Bouvier if [ "$(cat /sys/bus/pci/devices/$p/driver_override)" == vfio-pci ]; then 34*7bc86ccbSPierrick Bouvier continue 35*7bc86ccbSPierrick Bouvier fi 36*7bc86ccbSPierrick Bouvier echo $p > /sys/bus/pci/drivers/nvme/unbind 37*7bc86ccbSPierrick Bouvier echo vfio-pci > /sys/bus/pci/devices/$p/driver_override 38*7bc86ccbSPierrick Bouvier echo $p > /sys/bus/pci/drivers/vfio-pci/bind 39*7bc86ccbSPierrick Bouvierdone 40*7bc86ccbSPierrick Bouvier 41*7bc86ccbSPierrick Bouvier# boot nested guest and execute /host/nested_guest.sh 42*7bc86ccbSPierrick Bouvier# one disk is passed through vfio group, the other, through iommufd 43*7bc86ccbSPierrick Bouvierqemu-system-aarch64 \ 44*7bc86ccbSPierrick Bouvier-M virt \ 45*7bc86ccbSPierrick Bouvier-display none \ 46*7bc86ccbSPierrick Bouvier-serial stdio \ 47*7bc86ccbSPierrick Bouvier-cpu host \ 48*7bc86ccbSPierrick Bouvier-enable-kvm \ 49*7bc86ccbSPierrick Bouvier-m 1G \ 50*7bc86ccbSPierrick Bouvier-kernel /host/Image.gz \ 51*7bc86ccbSPierrick Bouvier-drive format=raw,file=/host/guest.ext4,if=virtio \ 52*7bc86ccbSPierrick Bouvier-append "root=/dev/vda init=/init -- bash /host/nested_guest.sh" \ 53*7bc86ccbSPierrick Bouvier-virtfs local,path=/host,mount_tag=host,security_model=mapped,readonly=off \ 54*7bc86ccbSPierrick Bouvier-device vfio-pci,host=$pci_vfio \ 55*7bc86ccbSPierrick Bouvier-object iommufd,id=iommufd0 \ 56*7bc86ccbSPierrick Bouvier-device vfio-pci,host=$pci_iommufd,iommufd=iommufd0 57*7bc86ccbSPierrick Bouvier''' 58*7bc86ccbSPierrick Bouvier 59*7bc86ccbSPierrick Bouviernested_guest_script = ''' 60*7bc86ccbSPierrick Bouvier#!/usr/bin/env bash 61*7bc86ccbSPierrick Bouvier 62*7bc86ccbSPierrick Bouvierset -euo pipefail 63*7bc86ccbSPierrick Bouvierset -x 64*7bc86ccbSPierrick Bouvier 65*7bc86ccbSPierrick Bouvierimage_vfio=/host/disk_vfio 66*7bc86ccbSPierrick Bouvierimage_iommufd=/host/disk_iommufd 67*7bc86ccbSPierrick Bouvier 68*7bc86ccbSPierrick Bouvierdev_vfio=$(lsblk --nvme | grep vfio | cut -f 1 -d ' ') 69*7bc86ccbSPierrick Bouvierdev_iommufd=$(lsblk --nvme | grep iommufd | cut -f 1 -d ' ') 70*7bc86ccbSPierrick Bouvier 71*7bc86ccbSPierrick Bouvier# compare if devices are identical to original images 72*7bc86ccbSPierrick Bouvierdiff $image_vfio /dev/$dev_vfio 73*7bc86ccbSPierrick Bouvierdiff $image_iommufd /dev/$dev_iommufd 74*7bc86ccbSPierrick Bouvier 75*7bc86ccbSPierrick Bouvierecho device_passthrough_test_ok 76*7bc86ccbSPierrick Bouvier''' 77*7bc86ccbSPierrick Bouvier 78*7bc86ccbSPierrick Bouvierclass Aarch64DevicePassthrough(QemuSystemTest): 79*7bc86ccbSPierrick Bouvier 80*7bc86ccbSPierrick Bouvier # https://github.com/pbo-linaro/qemu-linux-stack 81*7bc86ccbSPierrick Bouvier # 82*7bc86ccbSPierrick Bouvier # Linux kernel is compiled with defconfig + 83*7bc86ccbSPierrick Bouvier # IOMMUFD + VFIO_DEVICE_CDEV + ARM_SMMU_V3_IOMMUFD 84*7bc86ccbSPierrick Bouvier # https://docs.kernel.org/driver-api/vfio.html#vfio-device-cde 85*7bc86ccbSPierrick Bouvier ASSET_DEVICE_PASSTHROUGH_STACK = Asset( 86*7bc86ccbSPierrick Bouvier ('https://fileserver.linaro.org/s/fx5DXxBYme8dw2G/' 87*7bc86ccbSPierrick Bouvier 'download/device_passthrough.tar.xz'), 88*7bc86ccbSPierrick Bouvier '812750b664d61c2986f2b149939ae28cafbd60d53e9c7e4b16e97143845e196d') 89*7bc86ccbSPierrick Bouvier 90*7bc86ccbSPierrick Bouvier # This tests the device passthrough implementation, by booting a VM 91*7bc86ccbSPierrick Bouvier # supporting it with two nvme disks attached, and launching a nested VM 92*7bc86ccbSPierrick Bouvier # reading their content. 93*7bc86ccbSPierrick Bouvier def test_aarch64_device_passthrough(self): 94*7bc86ccbSPierrick Bouvier self.set_machine('virt') 95*7bc86ccbSPierrick Bouvier self.require_accelerator('tcg') 96*7bc86ccbSPierrick Bouvier 97*7bc86ccbSPierrick Bouvier self.vm.set_console() 98*7bc86ccbSPierrick Bouvier 99*7bc86ccbSPierrick Bouvier stack_path_tar_gz = self.ASSET_DEVICE_PASSTHROUGH_STACK.fetch() 100*7bc86ccbSPierrick Bouvier self.archive_extract(stack_path_tar_gz, format="tar") 101*7bc86ccbSPierrick Bouvier 102*7bc86ccbSPierrick Bouvier stack = self.scratch_file('out') 103*7bc86ccbSPierrick Bouvier kernel = os.path.join(stack, 'Image.gz') 104*7bc86ccbSPierrick Bouvier rootfs_host = os.path.join(stack, 'host.ext4') 105*7bc86ccbSPierrick Bouvier disk_vfio = os.path.join(stack, 'disk_vfio') 106*7bc86ccbSPierrick Bouvier disk_iommufd = os.path.join(stack, 'disk_iommufd') 107*7bc86ccbSPierrick Bouvier guest_cmd = os.path.join(stack, 'guest.sh') 108*7bc86ccbSPierrick Bouvier nested_guest_cmd = os.path.join(stack, 'nested_guest.sh') 109*7bc86ccbSPierrick Bouvier # we generate two random disks 110*7bc86ccbSPierrick Bouvier with open(disk_vfio, "wb") as d: d.write(randbytes(512)) 111*7bc86ccbSPierrick Bouvier with open(disk_iommufd, "wb") as d: d.write(randbytes(1024)) 112*7bc86ccbSPierrick Bouvier with open(guest_cmd, 'w') as s: s.write(guest_script) 113*7bc86ccbSPierrick Bouvier with open(nested_guest_cmd, 'w') as s: s.write(nested_guest_script) 114*7bc86ccbSPierrick Bouvier 115*7bc86ccbSPierrick Bouvier self.vm.add_args('-cpu', 'max') 116*7bc86ccbSPierrick Bouvier self.vm.add_args('-m', '2G') 117*7bc86ccbSPierrick Bouvier self.vm.add_args('-M', 'virt,' 118*7bc86ccbSPierrick Bouvier 'virtualization=on,' 119*7bc86ccbSPierrick Bouvier 'gic-version=max,' 120*7bc86ccbSPierrick Bouvier 'iommu=smmuv3') 121*7bc86ccbSPierrick Bouvier self.vm.add_args('-kernel', kernel) 122*7bc86ccbSPierrick Bouvier self.vm.add_args('-drive', f'format=raw,file={rootfs_host}') 123*7bc86ccbSPierrick Bouvier self.vm.add_args('-drive', 124*7bc86ccbSPierrick Bouvier f'file={disk_vfio},if=none,id=vfio,format=raw') 125*7bc86ccbSPierrick Bouvier self.vm.add_args('-device', 'nvme,serial=vfio,drive=vfio') 126*7bc86ccbSPierrick Bouvier self.vm.add_args('-drive', 127*7bc86ccbSPierrick Bouvier f'file={disk_iommufd},if=none,id=iommufd,format=raw') 128*7bc86ccbSPierrick Bouvier self.vm.add_args('-device', 'nvme,serial=iommufd,drive=iommufd') 129*7bc86ccbSPierrick Bouvier self.vm.add_args('-virtfs', 130*7bc86ccbSPierrick Bouvier f'local,path={stack}/,mount_tag=host,' 131*7bc86ccbSPierrick Bouvier 'security_model=mapped,readonly=off') 132*7bc86ccbSPierrick Bouvier # boot and execute guest script 133*7bc86ccbSPierrick Bouvier # init will trigger a kernel panic if script fails 134*7bc86ccbSPierrick Bouvier self.vm.add_args('-append', 135*7bc86ccbSPierrick Bouvier 'root=/dev/vda init=/init -- bash /host/guest.sh') 136*7bc86ccbSPierrick Bouvier 137*7bc86ccbSPierrick Bouvier self.vm.launch() 138*7bc86ccbSPierrick Bouvier wait_for_console_pattern(self, 'device_passthrough_test_ok', 139*7bc86ccbSPierrick Bouvier failure_message='Kernel panic') 140*7bc86ccbSPierrick Bouvier 141*7bc86ccbSPierrick Bouvierif __name__ == '__main__': 142*7bc86ccbSPierrick Bouvier QemuSystemTest.main() 143