1#!/bin/bash 2set -x 3 4source $HOME/.cargo/env 5source $(dirname "$0")/test-util.sh 6source $(dirname "$0")/common-aarch64.sh 7 8WORKLOADS_LOCK="$WORKLOADS_DIR/integration_test.lock" 9 10build_spdk_nvme() { 11 SPDK_DIR="$WORKLOADS_DIR/spdk" 12 SPDK_REPO="https://github.com/spdk/spdk.git" 13 SPDK_DEPLOY_DIR="/usr/local/bin/spdk-nvme" 14 checkout_repo "$SPDK_DIR" "$SPDK_REPO" master "ef8bcce58f3f02b79c0619a297e4f17e81e62b24" 15 16 if [ ! -f "$SPDK_DIR/.built" ]; then 17 pushd $SPDK_DIR 18 git submodule update --init 19 apt-get update 20 sed -i "/grpcio/d" scripts/pkgdep/debian.sh 21 ./scripts/pkgdep.sh 22 ./configure --with-vfio-user 23 chmod +x /usr/local/lib/python3.8/dist-packages/ninja/data/bin/ninja 24 make -j `nproc` || exit 1 25 touch .built 26 popd 27 fi 28 if [ ! -d "/usr/local/bin/spdk-nvme" ]; then 29 mkdir -p $SPDK_DEPLOY_DIR 30 fi 31 cp "$WORKLOADS_DIR/spdk/build/bin/nvmf_tgt" $SPDK_DEPLOY_DIR/nvmf_tgt 32 cp "$WORKLOADS_DIR/spdk/scripts/rpc.py" $SPDK_DEPLOY_DIR/rpc.py 33 cp -r "$WORKLOADS_DIR/spdk/scripts/rpc" $SPDK_DEPLOY_DIR/rpc 34 cp -r "$WORKLOADS_DIR/spdk/python" $SPDK_DEPLOY_DIR/../ 35} 36 37build_virtiofsd() { 38 VIRTIOFSD_DIR="$WORKLOADS_DIR/virtiofsd_build" 39 VIRTIOFSD_REPO="https://gitlab.com/virtio-fs/virtiofsd.git" 40 41 checkout_repo "$VIRTIOFSD_DIR" "$VIRTIOFSD_REPO" v1.1.0 "220405d7a2606c92636d31992b5cb3036a41047b" 42 43 if [ ! -f "$VIRTIOFSD_DIR/.built" ]; then 44 pushd $VIRTIOFSD_DIR 45 time cargo build --release 46 cp target/release/virtiofsd "$WORKLOADS_DIR/" || exit 1 47 touch .built 48 popd 49 fi 50} 51 52update_workloads() { 53 cp scripts/sha1sums-aarch64 $WORKLOADS_DIR 54 55 BIONIC_OS_IMAGE_DOWNLOAD_NAME="bionic-server-cloudimg-arm64.img" 56 BIONIC_OS_IMAGE_DOWNLOAD_URL="https://cloud-hypervisor.azureedge.net/$BIONIC_OS_IMAGE_DOWNLOAD_NAME" 57 BIONIC_OS_DOWNLOAD_IMAGE="$WORKLOADS_DIR/$BIONIC_OS_IMAGE_DOWNLOAD_NAME" 58 if [ ! -f "$BIONIC_OS_DOWNLOAD_IMAGE" ]; then 59 pushd $WORKLOADS_DIR 60 time wget --quiet $BIONIC_OS_IMAGE_DOWNLOAD_URL || exit 1 61 popd 62 fi 63 64 BIONIC_OS_RAW_IMAGE_NAME="bionic-server-cloudimg-arm64.raw" 65 BIONIC_OS_RAW_IMAGE="$WORKLOADS_DIR/$BIONIC_OS_RAW_IMAGE_NAME" 66 if [ ! -f "$BIONIC_OS_RAW_IMAGE" ]; then 67 pushd $WORKLOADS_DIR 68 time qemu-img convert -p -f qcow2 -O raw $BIONIC_OS_IMAGE_DOWNLOAD_NAME $BIONIC_OS_RAW_IMAGE_NAME || exit 1 69 popd 70 fi 71 72 # Convert the raw image to qcow2 image to remove compressed blocks from the disk. Therefore letting the 73 # qcow2 format image can be directly used in the integration test. 74 BIONIC_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME="bionic-server-cloudimg-arm64.qcow2" 75 BIONIC_OS_QCOW2_UNCOMPRESSED_IMAGE="$WORKLOADS_DIR/$BIONIC_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME" 76 if [ ! -f "$BIONIC_OS_QCOW2_UNCOMPRESSED_IMAGE" ]; then 77 pushd $WORKLOADS_DIR 78 time qemu-img convert -p -f raw -O qcow2 $BIONIC_OS_RAW_IMAGE_NAME $BIONIC_OS_QCOW2_UNCOMPRESSED_IMAGE || exit 1 79 popd 80 fi 81 82 FOCAL_OS_RAW_IMAGE_NAME="focal-server-cloudimg-arm64-custom-20210929-0.raw" 83 FOCAL_OS_RAW_IMAGE_DOWNLOAD_URL="https://cloud-hypervisor.azureedge.net/$FOCAL_OS_RAW_IMAGE_NAME" 84 FOCAL_OS_RAW_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_RAW_IMAGE_NAME" 85 if [ ! -f "$FOCAL_OS_RAW_IMAGE" ]; then 86 pushd $WORKLOADS_DIR 87 time wget --quiet $FOCAL_OS_RAW_IMAGE_DOWNLOAD_URL || exit 1 88 popd 89 fi 90 91 FOCAL_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME="focal-server-cloudimg-arm64-custom-20210929-0.qcow2" 92 FOCAL_OS_QCOW2_IMAGE_UNCOMPRESSED_DOWNLOAD_URL="https://cloud-hypervisor.azureedge.net/$FOCAL_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME" 93 FOCAL_OS_QCOW2_UNCOMPRESSED_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME" 94 if [ ! -f "$FOCAL_OS_QCOW2_UNCOMPRESSED_IMAGE" ]; then 95 pushd $WORKLOADS_DIR 96 time wget --quiet $FOCAL_OS_QCOW2_IMAGE_UNCOMPRESSED_DOWNLOAD_URL || exit 1 97 popd 98 fi 99 100 FOCAL_OS_QCOW2_IMAGE_BACKING_FILE_NAME="focal-server-cloudimg-arm64-custom-20210929-0-backing.qcow2" 101 FOCAL_OS_QCOW2_BACKING_FILE_IMAGE="$WORKLOADS_DIR/$FOCAL_OS_QCOW2_IMAGE_BACKING_FILE_NAME" 102 if [ ! -f "$FOCAL_OS_QCOW2_BACKING_FILE_IMAGE" ]; then 103 pushd $WORKLOADS_DIR 104 time qemu-img create -f qcow2 -b $FOCAL_OS_QCOW2_UNCOMPRESSED_IMAGE -F qcow2 $FOCAL_OS_QCOW2_IMAGE_BACKING_FILE_NAME 105 popd 106 fi 107 108 JAMMY_OS_RAW_IMAGE_NAME="jammy-server-cloudimg-arm64-custom-20220329-0.raw" 109 JAMMY_OS_RAW_IMAGE_DOWNLOAD_URL="https://cloud-hypervisor.azureedge.net/$JAMMY_OS_RAW_IMAGE_NAME" 110 JAMMY_OS_RAW_IMAGE="$WORKLOADS_DIR/$JAMMY_OS_RAW_IMAGE_NAME" 111 if [ ! -f "$JAMMY_OS_RAW_IMAGE" ]; then 112 pushd $WORKLOADS_DIR 113 time wget --quiet $JAMMY_OS_RAW_IMAGE_DOWNLOAD_URL || exit 1 114 popd 115 fi 116 117 JAMMY_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME="jammy-server-cloudimg-arm64-custom-20220329-0.qcow2" 118 JAMMY_OS_QCOW2_IMAGE_UNCOMPRESSED_DOWNLOAD_URL="https://cloud-hypervisor.azureedge.net/$JAMMY_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME" 119 JAMMY_OS_QCOW2_UNCOMPRESSED_IMAGE="$WORKLOADS_DIR/$JAMMY_OS_QCOW2_IMAGE_UNCOMPRESSED_NAME" 120 if [ ! -f "$JAMMY_OS_QCOW2_UNCOMPRESSED_IMAGE" ]; then 121 pushd $WORKLOADS_DIR 122 time wget --quiet $JAMMY_OS_QCOW2_IMAGE_UNCOMPRESSED_DOWNLOAD_URL || exit 1 123 popd 124 fi 125 126 ALPINE_MINIROOTFS_URL="http://dl-cdn.alpinelinux.org/alpine/v3.11/releases/aarch64/alpine-minirootfs-3.11.3-aarch64.tar.gz" 127 ALPINE_MINIROOTFS_TARBALL="$WORKLOADS_DIR/alpine-minirootfs-aarch64.tar.gz" 128 if [ ! -f "$ALPINE_MINIROOTFS_TARBALL" ]; then 129 pushd $WORKLOADS_DIR 130 time wget --quiet $ALPINE_MINIROOTFS_URL -O $ALPINE_MINIROOTFS_TARBALL || exit 1 131 popd 132 fi 133 134 ALPINE_INITRAMFS_IMAGE="$WORKLOADS_DIR/alpine_initramfs.img" 135 if [ ! -f "$ALPINE_INITRAMFS_IMAGE" ]; then 136 pushd $WORKLOADS_DIR 137 mkdir alpine-minirootfs 138 tar xf "$ALPINE_MINIROOTFS_TARBALL" -C alpine-minirootfs 139 cat > alpine-minirootfs/init <<-EOF 140 #! /bin/sh 141 mount -t devtmpfs dev /dev 142 echo \$TEST_STRING > /dev/console 143 poweroff -f 144 EOF 145 chmod +x alpine-minirootfs/init 146 cd alpine-minirootfs 147 find . -print0 | 148 cpio --null --create --verbose --owner root:root --format=newc > "$ALPINE_INITRAMFS_IMAGE" 149 popd 150 fi 151 152 pushd $WORKLOADS_DIR 153 sha1sum sha1sums-aarch64 --check 154 if [ $? -ne 0 ]; then 155 echo "sha1sum validation of images failed, remove invalid images to fix the issue." 156 exit 1 157 fi 158 popd 159 160 # Download Cloud Hypervisor binary from its last stable release 161 LAST_RELEASE_VERSION="v34.0" 162 CH_RELEASE_URL="https://github.com/cloud-hypervisor/cloud-hypervisor/releases/download/$LAST_RELEASE_VERSION/cloud-hypervisor-static-aarch64" 163 CH_RELEASE_NAME="cloud-hypervisor-static-aarch64" 164 pushd $WORKLOADS_DIR 165 time wget --quiet $CH_RELEASE_URL -O "$CH_RELEASE_NAME" || exit 1 166 chmod +x $CH_RELEASE_NAME 167 popd 168 169 # Build custom kernel for guest VMs 170 build_custom_linux 171 172 # Update the kernel in the cloud image for some tests that requires recent kernel version 173 FOCAL_OS_RAW_IMAGE_UPDATE_KERNEL_NAME="focal-server-cloudimg-arm64-custom-20210929-0-update-kernel.raw" 174 cp "$WORKLOADS_DIR/$FOCAL_OS_RAW_IMAGE_NAME" "$WORKLOADS_DIR/$FOCAL_OS_RAW_IMAGE_UPDATE_KERNEL_NAME" 175 FOCAL_OS_RAW_IMAGE_UPDATE_KERNEL_ROOT_DIR="$WORKLOADS_DIR/focal-server-cloudimg-root" 176 mkdir -p "$FOCAL_OS_RAW_IMAGE_UPDATE_KERNEL_ROOT_DIR" 177 # Mount the 'raw' image, replace the compressed kernel file and umount the working folder 178 guestmount -a "$WORKLOADS_DIR/$FOCAL_OS_RAW_IMAGE_UPDATE_KERNEL_NAME" -m /dev/sda1 "$FOCAL_OS_RAW_IMAGE_UPDATE_KERNEL_ROOT_DIR" || exit 1 179 cp "$WORKLOADS_DIR"/Image.gz "$FOCAL_OS_RAW_IMAGE_UPDATE_KERNEL_ROOT_DIR"/boot/vmlinuz 180 guestunmount "$FOCAL_OS_RAW_IMAGE_UPDATE_KERNEL_ROOT_DIR" 181 182 # Build virtiofsd 183 build_virtiofsd 184 185 BLK_IMAGE="$WORKLOADS_DIR/blk.img" 186 MNT_DIR="mount_image" 187 if [ ! -f "$BLK_IMAGE" ]; then 188 pushd $WORKLOADS_DIR 189 fallocate -l 16M $BLK_IMAGE 190 mkfs.ext4 -j $BLK_IMAGE 191 mkdir $MNT_DIR 192 sudo mount -t ext4 $BLK_IMAGE $MNT_DIR 193 sudo bash -c "echo bar > $MNT_DIR/foo" || exit 1 194 sudo umount $BLK_IMAGE 195 rm -r $MNT_DIR 196 popd 197 fi 198 199 SHARED_DIR="$WORKLOADS_DIR/shared_dir" 200 if [ ! -d "$SHARED_DIR" ]; then 201 mkdir -p $SHARED_DIR 202 echo "foo" > "$SHARED_DIR/file1" 203 echo "bar" > "$SHARED_DIR/file3" || exit 1 204 fi 205 206 # Checkout and build SPDK NVMe 207 build_spdk_nvme 208 209 # Checkout and build EDK2 210 build_edk2 211} 212 213process_common_args "$@" 214 215# aarch64 not supported for MSHV 216if [[ "$hypervisor" = "mshv" ]]; then 217 echo "AArch64 is not supported in Microsoft Hypervisor" 218 exit 1 219fi 220 221 222# lock the workloads folder to avoid parallel updating by different containers 223( 224 echo "try to lock $WORKLOADS_DIR folder and update" 225 flock -x 12 && update_workloads 226) 12>$WORKLOADS_LOCK 227 228# Check if there is any error in the execution of `update_workloads`. 229# If there is any error, then kill the shell. Otherwise the script will continue 230# running even if the `update_workloads` function was failed. 231RES=$? 232if [ $RES -ne 0 ]; then 233 exit 1 234fi 235 236BUILD_TARGET="aarch64-unknown-linux-${CH_LIBC}" 237if [[ "${BUILD_TARGET}" == "aarch64-unknown-linux-musl" ]]; then 238 export TARGET_CC="musl-gcc" 239 export RUSTFLAGS="-C link-arg=-lgcc -C link_arg=-specs -C link_arg=/usr/lib/aarch64-linux-musl/musl-gcc.specs" 240fi 241 242export RUST_BACKTRACE=1 243 244cargo build --all --release --target $BUILD_TARGET 245 246# Enable KSM with some reasonable parameters so that it won't take too long 247# for the memory to be merged between two processes. 248sudo bash -c "echo 1000000 > /sys/kernel/mm/ksm/pages_to_scan" 249sudo bash -c "echo 10 > /sys/kernel/mm/ksm/sleep_millisecs" 250sudo bash -c "echo 1 > /sys/kernel/mm/ksm/run" 251 252# Both test_vfio and ovs-dpdk rely on hugepages 253HUGEPAGESIZE=`grep Hugepagesize /proc/meminfo | awk '{print $2}'` 254PAGE_NUM=`echo $((12288 * 1024 / $HUGEPAGESIZE))` 255echo $PAGE_NUM | sudo tee /proc/sys/vm/nr_hugepages 256sudo chmod a+rwX /dev/hugepages 257 258# Run all direct kernel boot (Device Tree) test cases in mod `parallel` 259time cargo test "common_parallel::$test_filter" --target $BUILD_TARGET -- ${test_binary_args[*]} 260RES=$? 261 262# Run some tests in sequence since the result could be affected by other tests 263# running in parallel. 264if [ $RES -eq 0 ]; then 265 time cargo test "common_sequential::$test_filter" --target $BUILD_TARGET -- --test-threads=1 ${test_binary_args[*]} 266 RES=$? 267else 268 exit $RES 269fi 270 271# Run all ACPI test cases 272if [ $RES -eq 0 ]; then 273 time cargo test "aarch64_acpi::$test_filter" --target $BUILD_TARGET -- ${test_binary_args[*]} 274 RES=$? 275else 276 exit $RES 277fi 278 279# Run all test cases related to live migration 280if [ $RES -eq 0 ]; then 281 time cargo test "live_migration_parallel::$test_filter" --target $BUILD_TARGET -- ${test_binary_args[*]} 282 RES=$? 283else 284 exit $RES 285fi 286 287if [ $RES -eq 0 ]; then 288 time cargo test "live_migration_sequential::$test_filter" --target $BUILD_TARGET -- --test-threads=1 ${test_binary_args[*]} 289 RES=$? 290else 291 exit $RES 292fi 293 294# Run tests on dbus_api 295if [ $RES -eq 0 ]; then 296 cargo build --features "dbus_api" --all --release --target $BUILD_TARGET 297 export RUST_BACKTRACE=1 298 time cargo test "dbus_api::$test_filter" --target $BUILD_TARGET -- ${test_binary_args[*]} 299 RES=$? 300fi 301 302exit $RES 303