1#!/usr/bin/env bash 2set -x 3 4source $HOME/.cargo/env 5source $(dirname "$0")/test-util.sh 6source $(dirname "$0")/common-aarch64.sh 7 8WORKLOADS_LOCK="$WORKLOADS_DIR/integration_test.lock" 9 10build_spdk_nvme() { 11 SPDK_DIR="$WORKLOADS_DIR/spdk" 12 SPDK_REPO="https://github.com/spdk/spdk.git" 13 SPDK_DEPLOY_DIR="/usr/local/bin/spdk-nvme" 14 checkout_repo "$SPDK_DIR" "$SPDK_REPO" master "ef8bcce58f3f02b79c0619a297e4f17e81e62b24" 15 16 if [ ! -f "$SPDK_DIR/.built" ]; then 17 pushd $SPDK_DIR 18 git submodule update --init 19 apt-get update 20 sed -i "/grpcio/d" scripts/pkgdep/debian.sh 21 ./scripts/pkgdep.sh 22 ./configure --with-vfio-user 23 chmod +x /usr/local/lib/python3.10/dist-packages/ninja/data/bin/ninja 24 make -j `nproc` || exit 1 25 touch .built 26 popd 27 fi 28 if [ ! -d "/usr/local/bin/spdk-nvme" ]; then 29 mkdir -p $SPDK_DEPLOY_DIR 30 fi 31 cp "$WORKLOADS_DIR/spdk/build/bin/nvmf_tgt" $SPDK_DEPLOY_DIR/nvmf_tgt 32 cp "$WORKLOADS_DIR/spdk/scripts/rpc.py" $SPDK_DEPLOY_DIR/rpc.py 33 cp -r "$WORKLOADS_DIR/spdk/python/spdk/" $SPDK_DEPLOY_DIR/ 34 cp -r "$WORKLOADS_DIR/spdk/python" $SPDK_DEPLOY_DIR/../ 35} 36 37build_virtiofsd() { 38 VIRTIOFSD_DIR="$WORKLOADS_DIR/virtiofsd_build" 39 VIRTIOFSD_REPO="https://gitlab.com/virtio-fs/virtiofsd.git" 40 41 checkout_repo "$VIRTIOFSD_DIR" "$VIRTIOFSD_REPO" v1.8.0 "97ea7908fe7f9bc59916671a771bdcfaf4044b45" 42 43 if [ ! -f "$VIRTIOFSD_DIR/.built" ]; then 44 pushd $VIRTIOFSD_DIR 45 rm -rf target/ 46 time RUSTFLAGS="" TARGET_CC="" cargo build --release 47 cp target/release/virtiofsd "$WORKLOADS_DIR/" || exit 1 48 touch .built 49 popd 50 fi 51} 52 53update_workloads() { 54 cp scripts/sha1sums-aarch64 $WORKLOADS_DIR 55 56 BIONIC_OS_IMAGE_DOWNLOAD_NAME="bionic-server-cloudimg-arm64.img" 57 BIONIC_OS_IMAGE_DOWNLOAD_URL="https://cloud-hypervisor.azureedge.net/$BIONIC_OS_IMAGE_DOWNLOAD_NAME" 58 BIONIC_OS_DOWNLOAD_IMAGE="$WORKLOADS_DIR/$BIONIC_OS_IMAGE_DOWNLOAD_NAME" 59 if [ ! -f "$BIONIC_OS_DOWNLOAD_IMAGE" ]; then 60 pushd $WORKLOADS_DIR 61 time wget --quiet $BIONIC_OS_IMAGE_DOWNLOAD_URL || exit 1 62 popd 63 fi 64 65 BIONIC_OS_RAW_IMAGE_NAME="bionic-server-cloudimg-arm64.raw" 66 BIONIC_OS_RAW_IMAGE="$WORKLOADS_DIR/$BIONIC_OS_RAW_IMAGE_NAME" 67 if [ ! -f "$BIONIC_OS_RAW_IMAGE" ]; then 68 pushd $WORKLOADS_DIR 69 time qemu-img convert -p -f qcow2 -O raw $BIONIC_OS_IMAGE_DOWNLOAD_NAME $BIONIC_OS_RAW_IMAGE_NAME || exit 1 70 popd 71 fi 72 73 # Convert the raw image to qcow2 image to remove compressed blocks from the disk. Therefore letting the 74 # qcow2 format image can be directly used in the integration test. 75 BIONIC_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME="bionic-server-cloudimg-arm64.qcow2" 76 BIONIC_OS_QCOW2_UNCOMPRESSED_IMAGE="$WORKLOADS_DIR/$BIONIC_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME" 77 if [ ! -f "$BIONIC_OS_QCOW2_UNCOMPRESSED_IMAGE" ]; then 78 pushd $WORKLOADS_DIR 79 time qemu-img convert -p -f raw -O qcow2 $BIONIC_OS_RAW_IMAGE_NAME $BIONIC_OS_QCOW2_UNCOMPRESSED_IMAGE || exit 1 80 popd 81 fi 82 83 FOCAL_OS_RAW_IMAGE_NAME="focal-server-cloudimg-arm64-custom-20210929-0.raw" 84 FOCAL_OS_RAW_IMAGE_DOWNLOAD_URL="https://cloud-hypervisor.azureedge.net/$FOCAL_OS_RAW_IMAGE_NAME" 85 FOCAL_OS_RAW_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_RAW_IMAGE_NAME" 86 if [ ! -f "$FOCAL_OS_RAW_IMAGE" ]; then 87 pushd $WORKLOADS_DIR 88 time wget --quiet $FOCAL_OS_RAW_IMAGE_DOWNLOAD_URL || exit 1 89 popd 90 fi 91 92 FOCAL_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME="focal-server-cloudimg-arm64-custom-20210929-0.qcow2" 93 FOCAL_OS_QCOW2_IMAGE_UNCOMPRESSED_DOWNLOAD_URL="https://cloud-hypervisor.azureedge.net/$FOCAL_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME" 94 FOCAL_OS_QCOW2_UNCOMPRESSED_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME" 95 if [ ! -f "$FOCAL_OS_QCOW2_UNCOMPRESSED_IMAGE" ]; then 96 pushd $WORKLOADS_DIR 97 time wget --quiet $FOCAL_OS_QCOW2_IMAGE_UNCOMPRESSED_DOWNLOAD_URL || exit 1 98 popd 99 fi 100 101 FOCAL_OS_QCOW2_IMAGE_BACKING_FILE_NAME="focal-server-cloudimg-arm64-custom-20210929-0-backing.qcow2" 102 FOCAL_OS_QCOW2_BACKING_FILE_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_QCOW2_IMAGE_BACKING_FILE_NAME" 103 if [ ! -f "$FOCAL_OS_QCOW2_BACKING_FILE_IMAGE" ]; then 104 pushd $WORKLOADS_DIR 105 time qemu-img create -f qcow2 -b $FOCAL_OS_QCOW2_UNCOMPRESSED_IMAGE -F qcow2 $FOCAL_OS_QCOW2_IMAGE_BACKING_FILE_NAME 106 popd 107 fi 108 109 JAMMY_OS_RAW_IMAGE_NAME="jammy-server-cloudimg-arm64-custom-20220329-0.raw" 110 JAMMY_OS_RAW_IMAGE_DOWNLOAD_URL="https://cloud-hypervisor.azureedge.net/$JAMMY_OS_RAW_IMAGE_NAME" 111 JAMMY_OS_RAW_IMAGE="$WORKLOADS_DIR/$JAMMY_OS_RAW_IMAGE_NAME" 112 if [ ! -f "$JAMMY_OS_RAW_IMAGE" ]; then 113 pushd $WORKLOADS_DIR 114 time wget --quiet $JAMMY_OS_RAW_IMAGE_DOWNLOAD_URL || exit 1 115 popd 116 fi 117 118 JAMMY_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME="jammy-server-cloudimg-arm64-custom-20220329-0.qcow2" 119 JAMMY_OS_QCOW2_IMAGE_UNCOMPRESSED_DOWNLOAD_URL="https://cloud-hypervisor.azureedge.net/$JAMMY_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME" 120 JAMMY_OS_QCOW2_UNCOMPRESSED_IMAGE="$WORKLOADS_DIR/$JAMMY_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME" 121 if [ ! -f "$JAMMY_OS_QCOW2_UNCOMPRESSED_IMAGE" ]; then 122 pushd $WORKLOADS_DIR 123 time wget --quiet $JAMMY_OS_QCOW2_IMAGE_UNCOMPRESSED_DOWNLOAD_URL || exit 1 124 popd 125 fi 126 127 ALPINE_MINIROOTFS_URL="http://dl-cdn.alpinelinux.org/alpine/v3.11/releases/aarch64/alpine-minirootfs-3.11.3-aarch64.tar.gz" 128 ALPINE_MINIROOTFS_TARBALL="$WORKLOADS_DIR/alpine-minirootfs-aarch64.tar.gz" 129 if [ ! -f "$ALPINE_MINIROOTFS_TARBALL" ]; then 130 pushd $WORKLOADS_DIR 131 time wget --quiet $ALPINE_MINIROOTFS_URL -O $ALPINE_MINIROOTFS_TARBALL || exit 1 132 popd 133 fi 134 135 ALPINE_INITRAMFS_IMAGE="$WORKLOADS_DIR/alpine_initramfs.img" 136 if [ ! -f "$ALPINE_INITRAMFS_IMAGE" ]; then 137 pushd $WORKLOADS_DIR 138 mkdir alpine-minirootfs 139 tar xf "$ALPINE_MINIROOTFS_TARBALL" -C alpine-minirootfs 140 cat > alpine-minirootfs/init <<-EOF 141 #! /bin/sh 142 mount -t devtmpfs dev /dev 143 echo \$TEST_STRING > /dev/console 144 poweroff -f 145 EOF 146 chmod +x alpine-minirootfs/init 147 cd alpine-minirootfs 148 find . -print0 | 149 cpio --null --create --verbose --owner root:root --format=newc > "$ALPINE_INITRAMFS_IMAGE" 150 popd 151 fi 152 153 pushd $WORKLOADS_DIR 154 sha1sum sha1sums-aarch64 --check 155 if [ $? -ne 0 ]; then 156 echo "sha1sum validation of images failed, remove invalid images to fix the issue." 157 exit 1 158 fi 159 popd 160 161 # Download Cloud Hypervisor binary from its last stable release 162 LAST_RELEASE_VERSION="v36.0" 163 CH_RELEASE_URL="https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/$LAST_RELEASE_VERSION/cloud-hypervisor-static-aarch64" 164 CH_RELEASE_NAME="cloud-hypervisor-static-aarch64" 165 pushd $WORKLOADS_DIR 166 # Repeat a few times to workaround a random wget failure 167 WGET_RETRY_MAX=10 168 wget_retry=0 169 170 until [ "$wget_retry" -ge "$WGET_RETRY_MAX" ] 171 do 172 time wget $CH_RELEASE_URL -O "$CH_RELEASE_NAME" && break 173 wget_retry=$((wget_retry+1)) 174 done 175 176 if [ $wget_retry -ge "$WGET_RETRY_MAX" ]; then 177 exit 1 178 else 179 chmod +x $CH_RELEASE_NAME 180 fi 181 popd 182 183 # Build custom kernel for guest VMs 184 build_custom_linux 185 186 # Update the kernel in the cloud image for some tests that requires recent kernel version 187 FOCAL_OS_RAW_IMAGE_UPDATE_KERNEL_NAME="focal-server-cloudimg-arm64-custom-20210929-0-update-kernel.raw" 188 cp "$WORKLOADS_DIR/$FOCAL_OS_RAW_IMAGE_NAME" "$WORKLOADS_DIR/$FOCAL_OS_RAW_IMAGE_UPDATE_KERNEL_NAME" 189 FOCAL_OS_RAW_IMAGE_UPDATE_KERNEL_ROOT_DIR="$WORKLOADS_DIR/focal-server-cloudimg-root" 190 mkdir -p "$FOCAL_OS_RAW_IMAGE_UPDATE_KERNEL_ROOT_DIR" 191 # Mount the 'raw' image, replace the compressed kernel file and umount the working folder 192 guestmount -a "$WORKLOADS_DIR/$FOCAL_OS_RAW_IMAGE_UPDATE_KERNEL_NAME" -m /dev/sda1 "$FOCAL_OS_RAW_IMAGE_UPDATE_KERNEL_ROOT_DIR" || exit 1 193 cp "$WORKLOADS_DIR"/Image.gz "$FOCAL_OS_RAW_IMAGE_UPDATE_KERNEL_ROOT_DIR"/boot/vmlinuz 194 guestunmount "$FOCAL_OS_RAW_IMAGE_UPDATE_KERNEL_ROOT_DIR" 195 196 # Build virtiofsd 197 build_virtiofsd 198 199 BLK_IMAGE="$WORKLOADS_DIR/blk.img" 200 MNT_DIR="mount_image" 201 if [ ! -f "$BLK_IMAGE" ]; then 202 pushd $WORKLOADS_DIR 203 fallocate -l 16M $BLK_IMAGE 204 mkfs.ext4 -j $BLK_IMAGE 205 mkdir $MNT_DIR 206 sudo mount -t ext4 $BLK_IMAGE $MNT_DIR 207 sudo bash -c "echo bar > $MNT_DIR/foo" || exit 1 208 sudo umount $BLK_IMAGE 209 rm -r $MNT_DIR 210 popd 211 fi 212 213 SHARED_DIR="$WORKLOADS_DIR/shared_dir" 214 if [ ! -d "$SHARED_DIR" ]; then 215 mkdir -p $SHARED_DIR 216 echo "foo" > "$SHARED_DIR/file1" 217 echo "bar" > "$SHARED_DIR/file3" || exit 1 218 fi 219 220 # Checkout and build SPDK NVMe 221 build_spdk_nvme 222 223 # Checkout and build EDK2 224 build_edk2 225} 226 227process_common_args "$@" 228 229# aarch64 not supported for MSHV 230if [[ "$hypervisor" = "mshv" ]]; then 231 echo "AArch64 is not supported in Microsoft Hypervisor" 232 exit 1 233fi 234 235 236# lock the workloads folder to avoid parallel updating by different containers 237( 238 echo "try to lock $WORKLOADS_DIR folder and update" 239 flock -x 12 && update_workloads 240) 12>$WORKLOADS_LOCK 241 242# Check if there is any error in the execution of `update_workloads`. 243# If there is any error, then kill the shell. Otherwise the script will continue 244# running even if the `update_workloads` function was failed. 245RES=$? 246if [ $RES -ne 0 ]; then 247 exit 1 248fi 249 250export RUST_BACKTRACE=1 251 252cargo build --all --release --target $BUILD_TARGET 253 254# Enable KSM with some reasonable parameters so that it won't take too long 255# for the memory to be merged between two processes. 256sudo bash -c "echo 1000000 > /sys/kernel/mm/ksm/pages_to_scan" 257sudo bash -c "echo 10 > /sys/kernel/mm/ksm/sleep_millisecs" 258sudo bash -c "echo 1 > /sys/kernel/mm/ksm/run" 259 260# Both test_vfio and ovs-dpdk rely on hugepages 261HUGEPAGESIZE=`grep Hugepagesize /proc/meminfo | awk '{print $2}'` 262PAGE_NUM=`echo $((12288 * 1024 / $HUGEPAGESIZE))` 263echo $PAGE_NUM | sudo tee /proc/sys/vm/nr_hugepages 264sudo chmod a+rwX /dev/hugepages 265 266# Run all direct kernel boot (Device Tree) test cases in mod `parallel` 267time cargo test "common_parallel::$test_filter" --target $BUILD_TARGET -- ${test_binary_args[*]} 268RES=$? 269 270# Run some tests in sequence since the result could be affected by other tests 271# running in parallel. 272if [ $RES -eq 0 ]; then 273 time cargo test "common_sequential::$test_filter" --target $BUILD_TARGET -- --test-threads=1 ${test_binary_args[*]} 274 RES=$? 275else 276 exit $RES 277fi 278 279# Run all ACPI test cases 280if [ $RES -eq 0 ]; then 281 time cargo test "aarch64_acpi::$test_filter" --target $BUILD_TARGET -- ${test_binary_args[*]} 282 RES=$? 283else 284 exit $RES 285fi 286 287# Run all test cases related to live migration 288if [ $RES -eq 0 ]; then 289 time cargo test "live_migration_parallel::$test_filter" --target $BUILD_TARGET -- ${test_binary_args[*]} 290 RES=$? 291else 292 exit $RES 293fi 294 295if [ $RES -eq 0 ]; then 296 time cargo test "live_migration_sequential::$test_filter" --target $BUILD_TARGET -- --test-threads=1 ${test_binary_args[*]} 297 RES=$? 298else 299 exit $RES 300fi 301 302# Run tests on dbus_api 303if [ $RES -eq 0 ]; then 304 cargo build --features "dbus_api" --all --release --target $BUILD_TARGET 305 export RUST_BACKTRACE=1 306 time cargo test "dbus_api::$test_filter" --target $BUILD_TARGET -- ${test_binary_args[*]} 307 RES=$? 308fi 309 310exit $RES 311