1#!/bin/bash 2 3# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. 4# Copyright © 2020 Intel Corporation 5# SPDX-License-Identifier: Apache-2.0 6 7CLI_NAME="Cloud Hypervisor" 8 9CTR_IMAGE_TAG="cloudhypervisor/dev" 10CTR_IMAGE_VERSION="latest" 11CTR_IMAGE="${CTR_IMAGE_TAG}:${CTR_IMAGE_VERSION}" 12 13DOCKER_RUNTIME="docker" 14 15# Host paths 16CLH_SCRIPTS_DIR=$(cd "$(dirname "$0")" && pwd) 17CLH_ROOT_DIR=$(cd "${CLH_SCRIPTS_DIR}/.." && pwd) 18CLH_BUILD_DIR="${CLH_ROOT_DIR}/build" 19CLH_CARGO_TARGET="${CLH_BUILD_DIR}/cargo_target" 20CLH_DOCKERFILE="${CLH_SCRIPTS_DIR}/../resources/Dockerfile" 21CLH_CTR_BUILD_DIR="/tmp/cloud-hypervisor/ctr-build" 22CLH_INTEGRATION_WORKLOADS="${HOME}/workloads" 23 24# Container paths 25CTR_CLH_ROOT_DIR="/cloud-hypervisor" 26CTR_CLH_CARGO_BUILT_DIR="${CTR_CLH_ROOT_DIR}/build" 27CTR_CLH_CARGO_TARGET="${CTR_CLH_CARGO_BUILT_DIR}/cargo_target" 28CTR_CLH_INTEGRATION_WORKLOADS="/root/workloads" 29 30# Container networking option 31CTR_CLH_NET="host" 32[ $(uname -m) = "aarch64" ] && CTR_CLH_NET="bridge" 33 34# Cargo paths 35# Full path to the cargo registry dir on the host. This appears on the host 36# because we want to persist the cargo registry across container invocations. 37# Otherwise, any rust crates from crates.io would be downloaded again each time 38# we build or test. 39CARGO_REGISTRY_DIR="${CLH_BUILD_DIR}/cargo_registry" 40 41# Full path to the cargo git registry on the host. This serves the same purpose 42# as CARGO_REGISTRY_DIR, for crates downloaded from GitHub repos instead of 43# crates.io. 44CARGO_GIT_REGISTRY_DIR="${CLH_BUILD_DIR}/cargo_git_registry" 45 46# Full path to the cargo target dir on the host. 47CARGO_TARGET_DIR="${CLH_BUILD_DIR}/cargo_target" 48 49# Send a decorated message to stdout, followed by a new line 50# 51say() { 52 [ -t 1 ] && [ -n "$TERM" ] \ 53 && echo "$(tput setaf 2)[$CLI_NAME]$(tput sgr0) $*" \ 54 || echo "[$CLI_NAME] $*" 55} 56 57# Send a decorated message to stdout, without a trailing new line 58# 59say_noln() { 60 [ -t 1 ] && [ -n "$TERM" ] \ 61 && echo -n "$(tput setaf 2)[$CLI_NAME]$(tput sgr0) $*" \ 62 || echo "[$CLI_NAME] $*" 63} 64 65# Send a text message to stderr 66# 67say_err() { 68 [ -t 2 ] && [ -n "$TERM" ] \ 69 && echo "$(tput setaf 1)[$CLI_NAME] $*$(tput sgr0)" 1>&2 \ 70 || echo "[$CLI_NAME] $*" 1>&2 71} 72 73# Send a warning-highlighted text to stdout 74say_warn() { 75 [ -t 1 ] && [ -n "$TERM" ] \ 76 && echo "$(tput setaf 3)[$CLI_NAME] $*$(tput sgr0)" \ 77 || echo "[$CLI_NAME] $*" 78} 79 80# Exit with an error message and (optional) code 81# Usage: die [-c <error code>] <error message> 82# 83die() { 84 code=1 85 [[ "$1" = "-c" ]] && { 86 code="$2" 87 shift 2 88 } 89 say_err "$@" 90 exit $code 91} 92 93# Exit with an error message if the last exit code is not 0 94# 95ok_or_die() { 96 code=$? 97 [[ $code -eq 0 ]] || die -c $code "$@" 98} 99 100# Make sure the build/ dirs are available. Exit if we can't create them. 101# Upon returning from this call, the caller can be certain the build/ dirs exist. 102# 103ensure_build_dir() { 104 for dir in "$CLH_BUILD_DIR" \ 105 "$CLH_INTEGRATION_WORKLOADS" \ 106 "$CLH_CTR_BUILD_DIR" \ 107 "$CARGO_TARGET_DIR" \ 108 "$CARGO_REGISTRY_DIR" \ 109 "$CARGO_GIT_REGISTRY_DIR"; do 110 mkdir -p "$dir" || die "Error: cannot create dir $dir" 111 [ -x "$dir" ] && [ -w "$dir" ] || \ 112 { 113 say "Wrong permissions for $dir. Attempting to fix them ..." 114 chmod +x+w "$dir" 115 } || \ 116 die "Error: wrong permissions for $dir. Should be +x+w" 117 done 118} 119 120# Make sure we're using the latest dev container, by just pulling it. 121ensure_latest_ctr() { 122 $DOCKER_RUNTIME pull "$CTR_IMAGE" 123 124 ok_or_die "Error pulling container image. Aborting." 125} 126 127# Fix main directory permissions after a container ran as root. 128# Since the container ran as root, any files it creates will be owned by root. 129# This fixes that by recursively changing the ownership of /cloud-hypervisor to the 130# current user. 131# 132fix_dir_perms() { 133 # Yes, running Docker to get elevated privileges, just to chown some files 134 # is a dirty hack. 135 $DOCKER_RUNTIME run \ 136 --workdir "$CTR_CLH_ROOT_DIR" \ 137 --rm \ 138 --volume /dev:/dev \ 139 --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \ 140 "$CTR_IMAGE" \ 141 chown -R "$(id -u):$(id -g)" "$CTR_CLH_ROOT_DIR" 142 143 return $1 144} 145# Process exported volumes argument, separate the volumes and make docker compatible 146# Sample input: --volumes /a:/a#/b:/b 147# Sample output: --volume /a:/a --volume /b:/b 148# 149process_volumes_args() { 150 if [ -z "$arg_vols" ]; then 151 return 152 fi 153 exported_volumes="" 154 arr_vols=(${arg_vols//#/ }) 155 for var in "${arr_vols[@]}" 156 do 157 parts=(${var//:/ }) 158 if [[ ! -e "${parts[0]}" ]]; then 159 echo "The volume ${parts[0]} does not exist." 160 exit 1 161 fi 162 exported_volumes="$exported_volumes --volume $var" 163 done 164} 165cmd_help() { 166 echo "" 167 echo "Cloud Hypervisor $(basename $0)" 168 echo "Usage: $(basename $0) <command> [<command args>]" 169 echo "" 170 echo "Available commands:" 171 echo "" 172 echo " build [--debug|--release] [--libc musl|gnu] [-- [<cargo args>]]" 173 echo " Build the Cloud Hypervisor binaries." 174 echo " --debug Build the debug binaries. This is the default." 175 echo " --release Build the release binaries." 176 echo " --libc Select the C library Cloud Hypervisor will be built against. Default is gnu" 177 echo " --volumes Hash separated volumes to be exported. Example --volumes /mnt:/mnt#/myvol:/myvol" 178 echo "" 179 echo " tests [--unit|--cargo|--all] [--libc musl|gnu] [-- [<cargo test args>]]" 180 echo " Run the Cloud Hypervisor tests." 181 echo " --unit Run the unit tests." 182 echo " --cargo Run the cargo tests." 183 echo " --integration Run the integration tests." 184 echo " --integration-sgx Run the SGX integration tests." 185 echo " --integration-windows Run the Windows guest integration tests." 186 echo " --libc Select the C library Cloud Hypervisor will be built against. Default is gnu" 187 echo " --volumes Hash separated volumes to be exported. Example --volumes /mnt:/mnt#/myvol:/myvol" 188 echo " --all Run all tests." 189 echo "" 190 echo " build-container [--type]" 191 echo " Build the Cloud Hypervisor container." 192 echo " --dev Build dev container. This is the default." 193 echo "" 194 echo " clean [<cargo args>]]" 195 echo " Remove the Cloud Hypervisor artifacts." 196 echo "" 197 echo " shell" 198 echo " Run the development container into an interactive, privileged BASH shell." 199 echo "" 200 echo " help" 201 echo " Display this help message." 202 echo "" 203} 204 205cmd_build() { 206 build="debug" 207 libc="gnu" 208 209 while [ $# -gt 0 ]; do 210 case "$1" in 211 "-h"|"--help") { cmd_help; exit 1; } ;; 212 "--debug") { build="debug"; } ;; 213 "--release") { build="release"; } ;; 214 "--libc") 215 shift 216 [[ "$1" =~ ^(musl|gnu)$ ]] || \ 217 die "Invalid libc: $1. Valid options are \"musl\" and \"gnu\"." 218 libc="$1" 219 ;; 220 "--volumes") 221 shift 222 arg_vols="$1" 223 ;; 224 "--") { shift; break; } ;; 225 *) 226 die "Unknown build argument: $1. Please use --help for help." 227 ;; 228 esac 229 shift 230 done 231 process_volumes_args 232 233 target="$(uname -m)-unknown-linux-${libc}" 234 235 cargo_args=("$@") 236 [ $build = "release" ] && cargo_args+=("--release") 237 cargo_args+=(--target "$target") 238 [ $(uname -m) = "aarch64" ] && cargo_args+=("--no-default-features") 239 [ $(uname -m) = "aarch64" ] && cargo_args+=(--features "kvm") 240 241 rustflags="" 242 if [ $(uname -m) = "aarch64" ] && [ $libc = "musl" ] ; then 243 rustflags="-C link-arg=-lgcc -C link_arg=-specs -C link_arg=/usr/lib/aarch64-linux-musl/musl-gcc.specs" 244 fi 245 246 $DOCKER_RUNTIME run \ 247 --user "$(id -u):$(id -g)" \ 248 --workdir "$CTR_CLH_ROOT_DIR" \ 249 --rm \ 250 --volume /dev:/dev \ 251 --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \ 252 --env RUSTFLAGS="$rustflags" \ 253 "$CTR_IMAGE" \ 254 cargo build --all \ 255 --target-dir "$CTR_CLH_CARGO_TARGET" \ 256 "${cargo_args[@]}" && say "Binaries placed under $CLH_CARGO_TARGET/$target/$build" 257} 258 259cmd_clean() { 260 cargo_args=("$@") 261 262 $DOCKER_RUNTIME run \ 263 --user "$(id -u):$(id -g)" \ 264 --workdir "$CTR_CLH_ROOT_DIR" \ 265 --rm \ 266 --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \ 267 "$CTR_IMAGE" \ 268 cargo clean \ 269 --target-dir "$CTR_CLH_CARGO_TARGET" \ 270 "${cargo_args[@]}" 271 } 272 273cmd_tests() { 274 unit=false 275 cargo=false 276 integration=false 277 integration_sgx=false 278 integration_windows=false 279 libc="gnu" 280 arg_vols="" 281 282 while [ $# -gt 0 ]; do 283 case "$1" in 284 "-h"|"--help") { cmd_help; exit 1; } ;; 285 "--unit") { unit=true; } ;; 286 "--cargo") { cargo=true; } ;; 287 "--integration") { integration=true; } ;; 288 "--integration-sgx") { integration_sgx=true; } ;; 289 "--integration-windows") { integration_windows=true; } ;; 290 "--libc") 291 shift 292 [[ "$1" =~ ^(musl|gnu)$ ]] || \ 293 die "Invalid libc: $1. Valid options are \"musl\" and \"gnu\"." 294 libc="$1" 295 ;; 296 "--volumes") 297 shift 298 arg_vols="$1" 299 ;; 300 "--all") { cargo=true; unit=true; integration=true; } ;; 301 "--") { shift; break; } ;; 302 *) 303 die "Unknown tests argument: $1. Please use --help for help." 304 ;; 305 esac 306 shift 307 done 308 309 process_volumes_args 310 target="$(uname -m)-unknown-linux-${libc}" 311 cflags="" 312 target_cc="" 313 if [[ "$target" == "x86_64-unknown-linux-musl" ]]; then 314 target_cc="musl-gcc" 315 cflags="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/" 316 fi 317 318 if [ "$unit" = true ] ; then 319 say "Running unit tests for $target..." 320 $DOCKER_RUNTIME run \ 321 --workdir "$CTR_CLH_ROOT_DIR" \ 322 --rm \ 323 --device /dev/kvm \ 324 --device /dev/net/tun \ 325 --cap-add net_admin \ 326 --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \ 327 --env BUILD_TARGET="$target" \ 328 --env CFLAGS="$cflags" \ 329 --env TARGET_CC="$target_cc" \ 330 "$CTR_IMAGE" \ 331 ./scripts/run_unit_tests.sh "$@" || fix_dir_perms $? || exit $? 332 fi 333 334 if [ "$cargo" = true ] ; then 335 say "Running cargo tests..." 336 $DOCKER_RUNTIME run \ 337 --workdir "$CTR_CLH_ROOT_DIR" \ 338 --rm \ 339 --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \ 340 "$CTR_IMAGE" \ 341 ./scripts/run_cargo_tests.sh || fix_dir_perms $? || exit $? 342 fi 343 344 if [ "$integration" = true ] ; then 345 say "Running integration tests for $target..." 346 $DOCKER_RUNTIME run \ 347 --workdir "$CTR_CLH_ROOT_DIR" \ 348 --rm \ 349 --privileged \ 350 --security-opt seccomp=unconfined \ 351 --ipc=host \ 352 --net="$CTR_CLH_NET" \ 353 --mount type=tmpfs,destination=/tmp \ 354 --volume /dev:/dev \ 355 --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \ 356 --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \ 357 --env USER="root" \ 358 --env CH_LIBC="${libc}" \ 359 "$CTR_IMAGE" \ 360 ./scripts/run_integration_tests_$(uname -m).sh "$@" || fix_dir_perms $? || exit $? 361 fi 362 363 if [ "$integration_sgx" = true ] ; then 364 say "Running SGX integration tests for $target..." 365 $DOCKER_RUNTIME run \ 366 --workdir "$CTR_CLH_ROOT_DIR" \ 367 --rm \ 368 --privileged \ 369 --security-opt seccomp=unconfined \ 370 --ipc=host \ 371 --net="$CTR_CLH_NET" \ 372 --mount type=tmpfs,destination=/tmp \ 373 --volume /dev:/dev \ 374 --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \ 375 --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \ 376 --env USER="root" \ 377 --env CH_LIBC="${libc}" \ 378 "$CTR_IMAGE" \ 379 ./scripts/run_integration_tests_sgx.sh "$@" || fix_dir_perms $? || exit $? 380 fi 381 382 if [ "$integration_windows" = true ] ; then 383 say "Running Windows integration tests for $target..." 384 $DOCKER_RUNTIME run \ 385 --workdir "$CTR_CLH_ROOT_DIR" \ 386 --rm \ 387 --privileged \ 388 --security-opt seccomp=unconfined \ 389 --ipc=host \ 390 --net="$CTR_CLH_NET" \ 391 --mount type=tmpfs,destination=/tmp \ 392 --volume /dev:/dev \ 393 --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \ 394 --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \ 395 --env USER="root" \ 396 --env CH_LIBC="${libc}" \ 397 "$CTR_IMAGE" \ 398 ./scripts/run_integration_tests_windows.sh "$@" || fix_dir_perms $? || exit $? 399 fi 400 fix_dir_perms $? 401} 402 403cmd_build-container() { 404 container_type="dev" 405 406 while [ $# -gt 0 ]; do 407 case "$1" in 408 "-h"|"--help") { cmd_help; exit 1; } ;; 409 "--dev") { container_type="dev"; } ;; 410 "--") { shift; break; } ;; 411 *) 412 die "Unknown build-container argument: $1. Please use --help for help." 413 ;; 414 esac 415 shift 416 done 417 418 BUILD_DIR=/tmp/cloud-hypervisor/container/ 419 420 mkdir -p $BUILD_DIR 421 cp $CLH_DOCKERFILE $BUILD_DIR 422 423 [ $(uname -m) = "aarch64" ] && TARGETARCH="arm64" 424 [ $(uname -m) = "x86_64" ] && TARGETARCH="amd64" 425 426 $DOCKER_RUNTIME build \ 427 --target $container_type \ 428 -t $CTR_IMAGE \ 429 -f $BUILD_DIR/Dockerfile \ 430 --build-arg TARGETARCH=$TARGETARCH \ 431 $BUILD_DIR 432} 433 434cmd_shell() { 435 say_warn "Starting a privileged shell prompt as root ..." 436 say_warn "WARNING: Your $CLH_ROOT_DIR folder will be bind-mounted in the container under $CTR_CLH_ROOT_DIR" 437 $DOCKER_RUNTIME run \ 438 -ti \ 439 --workdir "$CTR_CLH_ROOT_DIR" \ 440 --rm \ 441 --privileged \ 442 --security-opt seccomp=unconfined \ 443 --ipc=host \ 444 --net="$CTR_CLH_NET" \ 445 --tmpfs /tmp:exec \ 446 --volume /dev:/dev \ 447 --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \ 448 --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \ 449 --env USER="root" \ 450 --entrypoint bash \ 451 "$CTR_IMAGE" 452 453 fix_dir_perms $? 454} 455 456# Parse main command line args. 457# 458while [ $# -gt 0 ]; do 459 case "$1" in 460 -h|--help) { cmd_help; exit 1; } ;; 461 -y|--unattended) { OPT_UNATTENDED=true; } ;; 462 -*) 463 die "Unknown arg: $1. Please use \`$0 help\` for help." 464 ;; 465 *) 466 break 467 ;; 468 esac 469 shift 470done 471 472# $1 is now a command name. Check if it is a valid command and, if so, 473# run it. 474# 475declare -f "cmd_$1" > /dev/null 476ok_or_die "Unknown command: $1. Please use \`$0 help\` for help." 477 478cmd=cmd_$1 479shift 480 481ensure_build_dir 482if [ $(uname -m) = "x86_64" ]; then 483 ensure_latest_ctr 484fi 485 486$cmd "$@" 487