1#!/bin/bash 2 3# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 4# Copyright © 2020 Intel Corporation 5# SPDX-License-Identifier: Apache-2.0 6 7CLI_NAME="Cloud Hypervisor" 8 9CTR_IMAGE_TAG="cloudhypervisor/dev" 10CTR_IMAGE_VERSION="latest" 11CTR_IMAGE="${CTR_IMAGE_TAG}:${CTR_IMAGE_VERSION}" 12 13DOCKER_RUNTIME="docker" 14 15# Host paths 16CLH_SCRIPTS_DIR=$(cd "$(dirname "$0")" && pwd) 17CLH_ROOT_DIR=$(cd "${CLH_SCRIPTS_DIR}/.." && pwd) 18CLH_BUILD_DIR="${CLH_ROOT_DIR}/build" 19CLH_CARGO_TARGET="${CLH_BUILD_DIR}/cargo_target" 20CLH_DOCKERFILE="${CLH_SCRIPTS_DIR}/../resources/Dockerfile" 21CLH_CTR_BUILD_DIR="/tmp/cloud-hypervisor/ctr-build" 22CLH_INTEGRATION_WORKLOADS="${HOME}/workloads" 23 24# Container paths 25CTR_CLH_ROOT_DIR="/cloud-hypervisor" 26CTR_CLH_CARGO_BUILT_DIR="${CTR_CLH_ROOT_DIR}/build" 27CTR_CLH_CARGO_TARGET="${CTR_CLH_CARGO_BUILT_DIR}/cargo_target" 28CTR_CLH_INTEGRATION_WORKLOADS="/root/workloads" 29 30# Container networking option 31CTR_CLH_NET="bridge" 32 33# Cargo paths 34# Full path to the cargo registry dir on the host. This appears on the host 35# because we want to persist the cargo registry across container invocations. 36# Otherwise, any rust crates from crates.io would be downloaded again each time 37# we build or test. 38CARGO_REGISTRY_DIR="${CLH_BUILD_DIR}/cargo_registry" 39 40# Full path to the cargo git registry on the host. This serves the same purpose 41# as CARGO_REGISTRY_DIR, for crates downloaded from GitHub repos instead of 42# crates.io. 43CARGO_GIT_REGISTRY_DIR="${CLH_BUILD_DIR}/cargo_git_registry" 44 45# Full path to the cargo target dir on the host. 46CARGO_TARGET_DIR="${CLH_BUILD_DIR}/cargo_target" 47 48# Send a decorated message to stdout, followed by a new line 49# 50say() { 51 [ -t 1 ] && [ -n "$TERM" ] \ 52 && echo "$(tput setaf 2)[$CLI_NAME]$(tput sgr0) $*" \ 53 || echo "[$CLI_NAME] $*" 54} 55 56# Send a decorated message to stdout, without a trailing new line 57# 58say_noln() { 59 [ -t 1 ] && [ -n "$TERM" ] \ 60 && echo -n "$(tput setaf 2)[$CLI_NAME]$(tput sgr0) $*" \ 61 || echo "[$CLI_NAME] $*" 62} 63 64# Send a text message to stderr 65# 66say_err() { 67 [ -t 2 ] && [ -n "$TERM" ] \ 68 && echo "$(tput setaf 1)[$CLI_NAME] $*$(tput sgr0)" 1>&2 \ 69 || echo "[$CLI_NAME] $*" 1>&2 70} 71 72# Send a warning-highlighted text to stdout 73say_warn() { 74 [ -t 1 ] && [ -n "$TERM" ] \ 75 && echo "$(tput setaf 3)[$CLI_NAME] $*$(tput sgr0)" \ 76 || echo "[$CLI_NAME] $*" 77} 78 79# Exit with an error message and (optional) code 80# Usage: die [-c <error code>] <error message> 81# 82die() { 83 code=1 84 [[ "$1" = "-c" ]] && { 85 code="$2" 86 shift 2 87 } 88 say_err "$@" 89 exit $code 90} 91 92# Exit with an error message if the last exit code is not 0 93# 94ok_or_die() { 95 code=$? 96 [[ $code -eq 0 ]] || die -c $code "$@" 97} 98 99# Make sure the build/ dirs are available. Exit if we can't create them. 100# Upon returning from this call, the caller can be certain the build/ dirs exist. 101# 102ensure_build_dir() { 103 for dir in "$CLH_BUILD_DIR" \ 104 "$CLH_INTEGRATION_WORKLOADS" \ 105 "$CLH_CTR_BUILD_DIR" \ 106 "$CARGO_TARGET_DIR" \ 107 "$CARGO_REGISTRY_DIR" \ 108 "$CARGO_GIT_REGISTRY_DIR"; do 109 mkdir -p "$dir" || die "Error: cannot create dir $dir" 110 [ -x "$dir" ] && [ -w "$dir" ] || \ 111 { 112 say "Wrong permissions for $dir. Attempting to fix them ..." 113 chmod +x+w "$dir" 114 } || \ 115 die "Error: wrong permissions for $dir. Should be +x+w" 116 done 117} 118 119# Make sure we're using the latest dev container, by just pulling it. 120ensure_latest_ctr() { 121 $DOCKER_RUNTIME pull "$CTR_IMAGE" 122 123 ok_or_die "Error pulling container image. Aborting." 124} 125 126# Fix main directory permissions after a container ran as root. 127# Since the container ran as root, any files it creates will be owned by root. 128# This fixes that by recursively changing the ownership of /cloud-hypervisor to the 129# current user. 130# 131fix_dir_perms() { 132 # Yes, running Docker to get elevated privileges, just to chown some files 133 # is a dirty hack. 134 $DOCKER_RUNTIME run \ 135 --workdir "$CTR_CLH_ROOT_DIR" \ 136 --rm \ 137 --volume /dev:/dev \ 138 --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \ 139 "$CTR_IMAGE" \ 140 chown -R "$(id -u):$(id -g)" "$CTR_CLH_ROOT_DIR" 141 142 return $1 143} 144# Process exported volumes argument, separate the volumes and make docker compatible 145# Sample input: --volumes /a:/a#/b:/b 146# Sample output: --volume /a:/a --volume /b:/b 147# 148process_volumes_args() { 149 if [ -z "$arg_vols" ]; then 150 return 151 fi 152 exported_volumes="" 153 arr_vols=(${arg_vols//#/ }) 154 for var in "${arr_vols[@]}" 155 do 156 parts=(${var//:/ }) 157 if [[ ! -e "${parts[0]}" ]]; then 158 echo "The volume ${parts[0]} does not exist." 159 exit 1 160 fi 161 exported_volumes="$exported_volumes --volume $var" 162 done 163} 164cmd_help() { 165 echo "" 166 echo "Cloud Hypervisor $(basename $0)" 167 echo "Usage: $(basename $0) <command> [<command args>]" 168 echo "" 169 echo "Available commands:" 170 echo "" 171 echo " build [--debug|--release] [--libc musl|gnu] [-- [<cargo args>]]" 172 echo " Build the Cloud Hypervisor binaries." 173 echo " --debug Build the debug binaries. This is the default." 174 echo " --release Build the release binaries." 175 echo " --libc Select the C library Cloud Hypervisor will be built against. Default is gnu" 176 echo " --volumes Hash separated volumes to be exported. Example --volumes /mnt:/mnt#/myvol:/myvol" 177 echo " --hypervisor Underlying hypervisor. Options kvm, mshv" 178 echo "" 179 echo " tests [--unit|--cargo|--all] [--libc musl|gnu] [-- [<cargo test args>]]" 180 echo " Run the Cloud Hypervisor tests." 181 echo " --unit Run the unit tests." 182 echo " --cargo Run the cargo tests." 183 echo " --integration Run the integration tests." 184 echo " --integration-sgx Run the SGX integration tests." 185 echo " --integration-vfio Run the VFIO integration tests." 186 echo " --integration-windows Run the Windows guest integration tests." 187 echo " --integration-live-migration Run the live-migration integration tests." 188 echo " --libc Select the C library Cloud Hypervisor will be built against. Default is gnu" 189 echo " --volumes Hash separated volumes to be exported. Example --volumes /mnt:/mnt#/myvol:/myvol" 190 echo " --hypervisor Underlying hypervisor. Options kvm, mshv" 191 echo " --all Run all tests." 192 echo "" 193 echo " build-container [--type]" 194 echo " Build the Cloud Hypervisor container." 195 echo " --dev Build dev container. This is the default." 196 echo "" 197 echo " clean [<cargo args>]]" 198 echo " Remove the Cloud Hypervisor artifacts." 199 echo "" 200 echo " shell" 201 echo " Run the development container into an interactive, privileged BASH shell." 202 echo " --volumes Hash separated volumes to be exported. Example --volumes /mnt:/mnt#/myvol:/myvol" 203 echo "" 204 echo " help" 205 echo " Display this help message." 206 echo "" 207} 208 209cmd_build() { 210 build="debug" 211 libc="gnu" 212 hypervisor="kvm" 213 features_build="" 214 exported_device="/dev/kvm" 215 while [ $# -gt 0 ]; do 216 case "$1" in 217 "-h"|"--help") { cmd_help; exit 1; } ;; 218 "--debug") { build="debug"; } ;; 219 "--release") { build="release"; } ;; 220 "--libc") 221 shift 222 [[ "$1" =~ ^(musl|gnu)$ ]] || \ 223 die "Invalid libc: $1. Valid options are \"musl\" and \"gnu\"." 224 libc="$1" 225 ;; 226 "--volumes") 227 shift 228 arg_vols="$1" 229 ;; 230 "--hypervisor") 231 shift 232 hypervisor="$1" 233 ;; 234 "--") { shift; break; } ;; 235 *) 236 die "Unknown build argument: $1. Please use --help for help." 237 ;; 238 esac 239 shift 240 done 241 242 ensure_build_dir 243 ensure_latest_ctr 244 245 process_volumes_args 246 if [[ ! ("$hypervisor" = "kvm" || "$hypervisor" = "mshv") ]]; then 247 die "Hypervisor value must be kvm or mshv" 248 fi 249 if [[ "$hypervisor" = "mshv" ]]; then 250 exported_device="/dev/mshv" 251 fi 252 target="$(uname -m)-unknown-linux-${libc}" 253 254 cargo_args=("$@") 255 [ $build = "release" ] && cargo_args+=("--release") 256 cargo_args+=(--target "$target") 257 [ $(uname -m) = "aarch64" ] && cargo_args+=("--no-default-features") 258 [ $(uname -m) = "aarch64" ] && cargo_args+=(--features $hypervisor) 259 260 rustflags="" 261 if [ $(uname -m) = "aarch64" ] && [ $libc = "musl" ] ; then 262 rustflags="-C link-arg=-lgcc -C link_arg=-specs -C link_arg=/usr/lib/aarch64-linux-musl/musl-gcc.specs" 263 fi 264 265 $DOCKER_RUNTIME run \ 266 --user "$(id -u):$(id -g)" \ 267 --workdir "$CTR_CLH_ROOT_DIR" \ 268 --rm \ 269 --volume $exported_device \ 270 --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \ 271 --env RUSTFLAGS="$rustflags" \ 272 "$CTR_IMAGE" \ 273 cargo build --all $features_build \ 274 --target-dir "$CTR_CLH_CARGO_TARGET" \ 275 "${cargo_args[@]}" && say "Binaries placed under $CLH_CARGO_TARGET/$target/$build" 276} 277 278cmd_clean() { 279 cargo_args=("$@") 280 281 ensure_build_dir 282 ensure_latest_ctr 283 284 $DOCKER_RUNTIME run \ 285 --user "$(id -u):$(id -g)" \ 286 --workdir "$CTR_CLH_ROOT_DIR" \ 287 --rm \ 288 --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \ 289 "$CTR_IMAGE" \ 290 cargo clean \ 291 --target-dir "$CTR_CLH_CARGO_TARGET" \ 292 "${cargo_args[@]}" 293 } 294 295cmd_tests() { 296 unit=false 297 cargo=false 298 integration=false 299 integration_sgx=false 300 integration_vfio=false 301 integration_windows=false 302 integration_live_migration=false 303 libc="gnu" 304 arg_vols="" 305 hypervisor="kvm" 306 exported_device="/dev/kvm" 307 while [ $# -gt 0 ]; do 308 case "$1" in 309 "-h"|"--help") { cmd_help; exit 1; } ;; 310 "--unit") { unit=true; } ;; 311 "--cargo") { cargo=true; } ;; 312 "--integration") { integration=true; } ;; 313 "--integration-sgx") { integration_sgx=true; } ;; 314 "--integration-vfio") { integration_vfio=true; } ;; 315 "--integration-windows") { integration_windows=true; } ;; 316 "--integration-live-migration") { integration_live_migration=true; } ;; 317 "--libc") 318 shift 319 [[ "$1" =~ ^(musl|gnu)$ ]] || \ 320 die "Invalid libc: $1. Valid options are \"musl\" and \"gnu\"." 321 libc="$1" 322 ;; 323 "--volumes") 324 shift 325 arg_vols="$1" 326 ;; 327 "--hypervisor") 328 shift 329 hypervisor="$1" 330 ;; 331 "--all") { cargo=true; unit=true; integration=true; } ;; 332 "--") { shift; break; } ;; 333 *) 334 die "Unknown tests argument: $1. Please use --help for help." 335 ;; 336 esac 337 shift 338 done 339 if [[ ! ("$hypervisor" = "kvm" || "$hypervisor" = "mshv") ]]; then 340 die "Hypervisor value must be kvm or mshv" 341 fi 342 343 if [[ "$hypervisor" = "mshv" ]]; then 344 exported_device="/dev/mshv" 345 fi 346 347 set -- "$@" '--hypervisor' $hypervisor 348 349 ensure_build_dir 350 ensure_latest_ctr 351 352 process_volumes_args 353 target="$(uname -m)-unknown-linux-${libc}" 354 cflags="" 355 target_cc="" 356 if [[ "$target" == "x86_64-unknown-linux-musl" ]]; then 357 target_cc="musl-gcc" 358 cflags="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/" 359 fi 360 361 if [[ "$unit" = true ]] ; then 362 say "Running unit tests for $target..." 363 $DOCKER_RUNTIME run \ 364 --workdir "$CTR_CLH_ROOT_DIR" \ 365 --rm \ 366 --device $exported_device \ 367 --device /dev/net/tun \ 368 --cap-add net_admin \ 369 --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \ 370 --env BUILD_TARGET="$target" \ 371 --env CFLAGS="$cflags" \ 372 --env TARGET_CC="$target_cc" \ 373 "$CTR_IMAGE" \ 374 ./scripts/run_unit_tests.sh "$@" || fix_dir_perms $? || exit $? 375 fi 376 377 if [ "$cargo" = true ] ; then 378 say "Running cargo tests..." 379 $DOCKER_RUNTIME run \ 380 --workdir "$CTR_CLH_ROOT_DIR" \ 381 --rm \ 382 --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \ 383 "$CTR_IMAGE" \ 384 ./scripts/run_cargo_tests.sh "$@" || fix_dir_perms $? || exit $? 385 fi 386 387 if [ "$integration" = true ] ; then 388 say "Running integration tests for $target..." 389 $DOCKER_RUNTIME run \ 390 --workdir "$CTR_CLH_ROOT_DIR" \ 391 --rm \ 392 --privileged \ 393 --security-opt seccomp=unconfined \ 394 --ipc=host \ 395 --net="$CTR_CLH_NET" \ 396 --mount type=tmpfs,destination=/tmp \ 397 --volume /dev:/dev \ 398 --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \ 399 --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \ 400 --env USER="root" \ 401 --env CH_LIBC="${libc}" \ 402 "$CTR_IMAGE" \ 403 ./scripts/run_integration_tests_$(uname -m).sh "$@" || fix_dir_perms $? || exit $? 404 fi 405 406 if [ "$integration_sgx" = true ] ; then 407 say "Running SGX integration tests for $target..." 408 $DOCKER_RUNTIME run \ 409 --workdir "$CTR_CLH_ROOT_DIR" \ 410 --rm \ 411 --privileged \ 412 --security-opt seccomp=unconfined \ 413 --ipc=host \ 414 --net="$CTR_CLH_NET" \ 415 --mount type=tmpfs,destination=/tmp \ 416 --volume /dev:/dev \ 417 --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \ 418 --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \ 419 --env USER="root" \ 420 --env CH_LIBC="${libc}" \ 421 "$CTR_IMAGE" \ 422 ./scripts/run_integration_tests_sgx.sh "$@" || fix_dir_perms $? || exit $? 423 fi 424 425 if [ "$integration_vfio" = true ] ; then 426 say "Running VFIO integration tests for $target..." 427 $DOCKER_RUNTIME run \ 428 --workdir "$CTR_CLH_ROOT_DIR" \ 429 --rm \ 430 --privileged \ 431 --security-opt seccomp=unconfined \ 432 --ipc=host \ 433 --net="$CTR_CLH_NET" \ 434 --mount type=tmpfs,destination=/tmp \ 435 --volume /dev:/dev \ 436 --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \ 437 --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \ 438 --env USER="root" \ 439 --env CH_LIBC="${libc}" \ 440 "$CTR_IMAGE" \ 441 ./scripts/run_integration_tests_vfio.sh "$@" || fix_dir_perms $? || exit $? 442 fi 443 444 if [ "$integration_windows" = true ] ; then 445 say "Running Windows integration tests for $target..." 446 $DOCKER_RUNTIME run \ 447 --workdir "$CTR_CLH_ROOT_DIR" \ 448 --rm \ 449 --privileged \ 450 --security-opt seccomp=unconfined \ 451 --ipc=host \ 452 --net="$CTR_CLH_NET" \ 453 --mount type=tmpfs,destination=/tmp \ 454 --volume /dev:/dev \ 455 --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \ 456 --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \ 457 --env USER="root" \ 458 --env CH_LIBC="${libc}" \ 459 "$CTR_IMAGE" \ 460 ./scripts/run_integration_tests_windows.sh "$@" || fix_dir_perms $? || exit $? 461 fi 462 463 if [ "$integration_live_migration" = true ] ; then 464 say "Running 'live migration' integration tests for $target..." 465 $DOCKER_RUNTIME run \ 466 --workdir "$CTR_CLH_ROOT_DIR" \ 467 --rm \ 468 --privileged \ 469 --security-opt seccomp=unconfined \ 470 --ipc=host \ 471 --net="$CTR_CLH_NET" \ 472 --mount type=tmpfs,destination=/tmp \ 473 --volume /dev:/dev \ 474 --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \ 475 --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \ 476 --env USER="root" \ 477 --env CH_LIBC="${libc}" \ 478 "$CTR_IMAGE" \ 479 ./scripts/run_integration_tests_live_migration.sh "$@" || fix_dir_perms $? || exit $? 480 fi 481 fix_dir_perms $? 482} 483 484cmd_build-container() { 485 container_type="dev" 486 487 while [ $# -gt 0 ]; do 488 case "$1" in 489 "-h"|"--help") { cmd_help; exit 1; } ;; 490 "--dev") { container_type="dev"; } ;; 491 "--") { shift; break; } ;; 492 *) 493 die "Unknown build-container argument: $1. Please use --help for help." 494 ;; 495 esac 496 shift 497 done 498 499 ensure_build_dir 500 ensure_latest_ctr 501 502 BUILD_DIR=/tmp/cloud-hypervisor/container/ 503 504 mkdir -p $BUILD_DIR 505 cp $CLH_DOCKERFILE $BUILD_DIR 506 507 [ $(uname -m) = "aarch64" ] && TARGETARCH="arm64" 508 [ $(uname -m) = "x86_64" ] && TARGETARCH="amd64" 509 510 $DOCKER_RUNTIME build \ 511 --target $container_type \ 512 -t $CTR_IMAGE \ 513 -f $BUILD_DIR/Dockerfile \ 514 --build-arg TARGETARCH=$TARGETARCH \ 515 $BUILD_DIR 516} 517 518cmd_shell() { 519 while [ $# -gt 0 ]; do 520 case "$1" in 521 "-h"|"--help") { cmd_help; exit 1; } ;; 522 "--volumes") 523 shift 524 arg_vols="$1" 525 ;; 526 "--") { shift; break; } ;; 527 *) 528 ;; 529 esac 530 shift 531 done 532 ensure_build_dir 533 ensure_latest_ctr 534 process_volumes_args 535 say_warn "Starting a privileged shell prompt as root ..." 536 say_warn "WARNING: Your $CLH_ROOT_DIR folder will be bind-mounted in the container under $CTR_CLH_ROOT_DIR" 537 $DOCKER_RUNTIME run \ 538 -ti \ 539 --workdir "$CTR_CLH_ROOT_DIR" \ 540 --rm \ 541 --privileged \ 542 --security-opt seccomp=unconfined \ 543 --ipc=host \ 544 --net="$CTR_CLH_NET" \ 545 --tmpfs /tmp:exec \ 546 --volume /dev:/dev \ 547 --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \ 548 --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \ 549 --env USER="root" \ 550 --entrypoint bash \ 551 "$CTR_IMAGE" 552 553 fix_dir_perms $? 554} 555 556# Parse main command line args. 557# 558while [ $# -gt 0 ]; do 559 case "$1" in 560 -h|--help) { cmd_help; exit 1; } ;; 561 -y|--unattended) { OPT_UNATTENDED=true; } ;; 562 -*) 563 die "Unknown arg: $1. Please use \`$0 help\` for help." 564 ;; 565 *) 566 break 567 ;; 568 esac 569 shift 570done 571 572# $1 is now a command name. Check if it is a valid command and, if so, 573# run it. 574# 575declare -f "cmd_$1" > /dev/null 576ok_or_die "Unknown command: $1. Please use \`$0 help\` for help." 577 578cmd=cmd_$1 579shift 580 581 582$cmd "$@" 583