xref: /cloud-hypervisor/scripts/dev_cli.sh (revision 496ceed1d02b5884e2a4c570ef231d2c90b64fc0)
1#!/bin/bash
2
3# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
4# Copyright © 2020 Intel Corporation
5# SPDX-License-Identifier: Apache-2.0
6
7CLI_NAME="Cloud Hypervisor"
8
9CTR_IMAGE_TAG="cloudhypervisor/dev"
10CTR_IMAGE_VERSION="latest"
11CTR_IMAGE="${CTR_IMAGE_TAG}:${CTR_IMAGE_VERSION}"
12
13DOCKER_RUNTIME="docker"
14
15# Host paths
16CLH_SCRIPTS_DIR=$(cd "$(dirname "$0")" && pwd)
17CLH_ROOT_DIR=$(cd "${CLH_SCRIPTS_DIR}/.." && pwd)
18CLH_BUILD_DIR="${CLH_ROOT_DIR}/build"
19CLH_CARGO_TARGET="${CLH_BUILD_DIR}/cargo_target"
20CLH_DOCKERFILE="${CLH_SCRIPTS_DIR}/../resources/Dockerfile"
21CLH_CTR_BUILD_DIR="/tmp/cloud-hypervisor/ctr-build"
22CLH_INTEGRATION_WORKLOADS="${HOME}/workloads"
23
24# Container paths
25CTR_CLH_ROOT_DIR="/cloud-hypervisor"
26CTR_CLH_CARGO_BUILT_DIR="${CTR_CLH_ROOT_DIR}/build"
27CTR_CLH_CARGO_TARGET="${CTR_CLH_CARGO_BUILT_DIR}/cargo_target"
28CTR_CLH_INTEGRATION_WORKLOADS="/root/workloads"
29
30# Container networking option
31CTR_CLH_NET="bridge"
32
33# Cargo paths
34# Full path to the cargo registry dir on the host. This appears on the host
35# because we want to persist the cargo registry across container invocations.
36# Otherwise, any rust crates from crates.io would be downloaded again each time
37# we build or test.
38CARGO_REGISTRY_DIR="${CLH_BUILD_DIR}/cargo_registry"
39
40# Full path to the cargo git registry on the host. This serves the same purpose
41# as CARGO_REGISTRY_DIR, for crates downloaded from GitHub repos instead of
42# crates.io.
43CARGO_GIT_REGISTRY_DIR="${CLH_BUILD_DIR}/cargo_git_registry"
44
45# Full path to the cargo target dir on the host.
46CARGO_TARGET_DIR="${CLH_BUILD_DIR}/cargo_target"
47
48# Send a decorated message to stdout, followed by a new line
49#
50say() {
51    [ -t 1 ] && [ -n "$TERM" ] \
52        && echo "$(tput setaf 2)[$CLI_NAME]$(tput sgr0) $*" \
53        || echo "[$CLI_NAME] $*"
54}
55
56# Send a decorated message to stdout, without a trailing new line
57#
58say_noln() {
59    [ -t 1 ] && [ -n "$TERM" ] \
60        && echo -n "$(tput setaf 2)[$CLI_NAME]$(tput sgr0) $*" \
61        || echo "[$CLI_NAME] $*"
62}
63
64# Send a text message to stderr
65#
66say_err() {
67    [ -t 2 ] && [ -n "$TERM" ] \
68        && echo "$(tput setaf 1)[$CLI_NAME] $*$(tput sgr0)" 1>&2 \
69        || echo "[$CLI_NAME] $*" 1>&2
70}
71
72# Send a warning-highlighted text to stdout
73say_warn() {
74    [ -t 1 ] && [ -n "$TERM" ] \
75        && echo "$(tput setaf 3)[$CLI_NAME] $*$(tput sgr0)" \
76        || echo "[$CLI_NAME] $*"
77}
78
79# Exit with an error message and (optional) code
80# Usage: die [-c <error code>] <error message>
81#
82die() {
83    code=1
84    [[ "$1" = "-c" ]] && {
85        code="$2"
86        shift 2
87    }
88    say_err "$@"
89    exit $code
90}
91
92# Exit with an error message if the last exit code is not 0
93#
94ok_or_die() {
95    code=$?
96    [[ $code -eq 0 ]] || die -c $code "$@"
97}
98
99# Make sure the build/ dirs are available. Exit if we can't create them.
100# Upon returning from this call, the caller can be certain the build/ dirs exist.
101#
102ensure_build_dir() {
103    for dir in "$CLH_BUILD_DIR" \
104		   "$CLH_INTEGRATION_WORKLOADS" \
105		   "$CLH_CTR_BUILD_DIR" \
106		   "$CARGO_TARGET_DIR" \
107		   "$CARGO_REGISTRY_DIR" \
108		   "$CARGO_GIT_REGISTRY_DIR"; do
109        mkdir -p "$dir" || die "Error: cannot create dir $dir"
110        [ -x "$dir" ] && [ -w "$dir" ] || \
111            {
112                say "Wrong permissions for $dir. Attempting to fix them ..."
113                chmod +x+w "$dir"
114            } || \
115            die "Error: wrong permissions for $dir. Should be +x+w"
116    done
117}
118
119# Make sure we're using the latest dev container, by just pulling it.
120ensure_latest_ctr() {
121    $DOCKER_RUNTIME pull "$CTR_IMAGE"
122
123    ok_or_die "Error pulling container image. Aborting."
124}
125
126# Fix main directory permissions after a container ran as root.
127# Since the container ran as root, any files it creates will be owned by root.
128# This fixes that by recursively changing the ownership of /cloud-hypervisor to the
129# current user.
130#
131fix_dir_perms() {
132    # Yes, running Docker to get elevated privileges, just to chown some files
133    # is a dirty hack.
134    $DOCKER_RUNTIME run \
135	--workdir "$CTR_CLH_ROOT_DIR" \
136	   --rm \
137	   --volume /dev:/dev \
138	   --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
139	   "$CTR_IMAGE" \
140           chown -R "$(id -u):$(id -g)" "$CTR_CLH_ROOT_DIR"
141
142    return $1
143}
144# Process exported volumes argument, separate the volumes and make docker compatible
145# Sample input: --volumes /a:/a#/b:/b
146# Sample output: --volume /a:/a --volume /b:/b
147#
148process_volumes_args() {
149    if [ -z "$arg_vols" ]; then
150        return
151    fi
152    exported_volumes=""
153    arr_vols=(${arg_vols//#/ })
154    for var in "${arr_vols[@]}"
155    do
156        parts=(${var//:/ })
157        if [[ ! -e "${parts[0]}" ]]; then
158            echo "The volume ${parts[0]} does not exist."
159            exit 1
160        fi
161        exported_volumes="$exported_volumes --volume $var"
162    done
163}
164cmd_help() {
165    echo ""
166    echo "Cloud Hypervisor $(basename $0)"
167    echo "Usage: $(basename $0) <command> [<command args>]"
168    echo ""
169    echo "Available commands:"
170    echo ""
171    echo "    build [--debug|--release] [--libc musl|gnu] [-- [<cargo args>]]"
172    echo "        Build the Cloud Hypervisor binaries."
173    echo "        --debug               Build the debug binaries. This is the default."
174    echo "        --release             Build the release binaries."
175    echo "        --libc                Select the C library Cloud Hypervisor will be built against. Default is gnu"
176    echo "        --volumes             Hash separated volumes to be exported. Example --volumes /mnt:/mnt#/myvol:/myvol"
177    echo "        --hypervisor          Underlying hypervisor. Options kvm, mshv"
178    echo ""
179    echo "    tests [--unit|--cargo|--all] [--libc musl|gnu] [-- [<cargo test args>]]"
180    echo "        Run the Cloud Hypervisor tests."
181    echo "        --unit                Run the unit tests."
182    echo "        --cargo               Run the cargo tests."
183    echo "        --integration         Run the integration tests."
184    echo "        --integration-sgx     Run the SGX integration tests."
185    echo "        --integration-vfio    Run the VFIO integration tests."
186    echo "        --integration-windows Run the Windows guest integration tests."
187    echo "        --libc                Select the C library Cloud Hypervisor will be built against. Default is gnu"
188    echo "        --volumes             Hash separated volumes to be exported. Example --volumes /mnt:/mnt#/myvol:/myvol"
189    echo "        --hypervisor          Underlying hypervisor. Options kvm, mshv"
190    echo "        --all                 Run all tests."
191    echo ""
192    echo "    build-container [--type]"
193    echo "        Build the Cloud Hypervisor container."
194    echo "        --dev                Build dev container. This is the default."
195    echo ""
196    echo "    clean [<cargo args>]]"
197    echo "        Remove the Cloud Hypervisor artifacts."
198    echo ""
199    echo "    shell"
200    echo "        Run the development container into an interactive, privileged BASH shell."
201    echo ""
202    echo "    help"
203    echo "        Display this help message."
204    echo ""
205}
206
207cmd_build() {
208    build="debug"
209    libc="gnu"
210    hypervisor="kvm"
211    features_build=""
212    exported_device="/dev/kvm"
213    while [ $# -gt 0 ]; do
214	case "$1" in
215            "-h"|"--help")  { cmd_help; exit 1; } ;;
216            "--debug")      { build="debug"; } ;;
217            "--release")    { build="release"; } ;;
218            "--libc")
219                shift
220                [[ "$1" =~ ^(musl|gnu)$ ]] || \
221                    die "Invalid libc: $1. Valid options are \"musl\" and \"gnu\"."
222                libc="$1"
223                ;;
224            "--volumes")
225                shift
226                arg_vols="$1"
227                ;;
228            "--hypervisor")
229                shift
230                hypervisor="$1"
231                ;;
232            "--")           { shift; break; } ;;
233            *)
234		die "Unknown build argument: $1. Please use --help for help."
235		;;
236	esac
237	shift
238    done
239
240    ensure_build_dir
241    if [ $(uname -m) = "x86_64" ]; then
242	ensure_latest_ctr
243    fi
244
245    process_volumes_args
246    if [[ "$hypervisor" != "kvm" ]]; then
247        die "Hypervisor value must be kvm"
248    fi
249    if [[ "$hypervisor" = "mshv" ]]; then
250        exported_device="/dev/mshv"
251    fi
252    target="$(uname -m)-unknown-linux-${libc}"
253
254    cargo_args=("$@")
255    [ $build = "release" ] && cargo_args+=("--release")
256    cargo_args+=(--target "$target")
257    [ $(uname -m) = "aarch64" ] && cargo_args+=("--no-default-features")
258    [ $(uname -m) = "aarch64" ] && cargo_args+=(--features $hypervisor)
259
260    rustflags=""
261    if [ $(uname -m) = "aarch64" ] && [ $libc = "musl" ] ; then
262        rustflags="-C link-arg=-lgcc -C link_arg=-specs -C link_arg=/usr/lib/aarch64-linux-musl/musl-gcc.specs"
263    fi
264
265    $DOCKER_RUNTIME run \
266	   --user "$(id -u):$(id -g)" \
267	   --workdir "$CTR_CLH_ROOT_DIR" \
268	   --rm \
269	   --volume $exported_device \
270	   --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
271	   --env RUSTFLAGS="$rustflags" \
272	   "$CTR_IMAGE" \
273	   cargo build --all $features_build \
274	         --target-dir "$CTR_CLH_CARGO_TARGET" \
275	         "${cargo_args[@]}" && say "Binaries placed under $CLH_CARGO_TARGET/$target/$build"
276}
277
278cmd_clean() {
279    cargo_args=("$@")
280
281    ensure_build_dir
282    if [ $(uname -m) = "x86_64" ]; then
283	ensure_latest_ctr
284    fi
285
286    $DOCKER_RUNTIME run \
287	   --user "$(id -u):$(id -g)" \
288	   --workdir "$CTR_CLH_ROOT_DIR" \
289	   --rm \
290	   --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
291	   "$CTR_IMAGE" \
292	   cargo clean \
293	         --target-dir "$CTR_CLH_CARGO_TARGET" \
294	         "${cargo_args[@]}"
295    }
296
297cmd_tests() {
298    unit=false
299    cargo=false
300    integration=false
301    integration_sgx=false
302    integration_vfio=false
303    integration_windows=false
304    libc="gnu"
305    arg_vols=""
306    hypervisor="kvm"
307    exported_device="/dev/kvm"
308    while [ $# -gt 0 ]; do
309	case "$1" in
310            "-h"|"--help")           { cmd_help; exit 1; } ;;
311            "--unit")                { unit=true; } ;;
312            "--cargo")               { cargo=true; } ;;
313            "--integration")         { integration=true; } ;;
314            "--integration-sgx")     { integration_sgx=true; } ;;
315            "--integration-vfio")    { integration_vfio=true; } ;;
316            "--integration-windows") { integration_windows=true; } ;;
317            "--libc")
318                shift
319                [[ "$1" =~ ^(musl|gnu)$ ]] || \
320                    die "Invalid libc: $1. Valid options are \"musl\" and \"gnu\"."
321                libc="$1"
322                ;;
323            "--volumes")
324                shift
325                arg_vols="$1"
326                ;;
327            "--hypervisor")
328                shift
329                hypervisor="$1"
330                ;;
331	    "--all")                 { cargo=true; unit=true; integration=true; } ;;
332            "--")                    { shift; break; } ;;
333            *)
334		die "Unknown tests argument: $1. Please use --help for help."
335		;;
336	esac
337	shift
338    done
339    if [[ "$hypervisor" != "kvm" ]]; then
340        die "Hypervisor value must be kvm"
341    fi
342    if [[ "$hypervisor" = "mshv" ]]; then
343        exported_device="/dev/mshv"
344    fi
345    set -- "$@" '--hypervisor' $hypervisor
346
347    ensure_build_dir
348    if [ $(uname -m) = "x86_64" ]; then
349	ensure_latest_ctr
350    fi
351
352    process_volumes_args
353    target="$(uname -m)-unknown-linux-${libc}"
354    cflags=""
355    target_cc=""
356    if [[ "$target" == "x86_64-unknown-linux-musl" ]]; then
357	target_cc="musl-gcc"
358	cflags="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/"
359    fi
360
361    if [[ "$unit" = true  ]] ;  then
362	say "Running unit tests for $target..."
363	$DOCKER_RUNTIME run \
364	       --workdir "$CTR_CLH_ROOT_DIR" \
365	       --rm \
366	       --device $exported_device \
367	       --device /dev/net/tun \
368	       --cap-add net_admin \
369	       --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
370	       --env BUILD_TARGET="$target" \
371	       --env CFLAGS="$cflags" \
372	       --env TARGET_CC="$target_cc" \
373	       "$CTR_IMAGE" \
374	       ./scripts/run_unit_tests.sh "$@" || fix_dir_perms $? || exit $?
375    fi
376
377    if [ "$cargo" = true ] ;  then
378	say "Running cargo tests..."
379	$DOCKER_RUNTIME run \
380	       --workdir "$CTR_CLH_ROOT_DIR" \
381	       --rm \
382	       --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
383	       "$CTR_IMAGE" \
384	       ./scripts/run_cargo_tests.sh "$@"  || fix_dir_perms $? || exit $?
385    fi
386
387    if [ "$integration" = true ] ;  then
388	say "Running integration tests for $target..."
389	$DOCKER_RUNTIME run \
390	       --workdir "$CTR_CLH_ROOT_DIR" \
391	       --rm \
392	       --privileged \
393	       --security-opt seccomp=unconfined \
394	       --ipc=host \
395	       --net="$CTR_CLH_NET" \
396	       --mount type=tmpfs,destination=/tmp \
397	       --volume /dev:/dev \
398	       --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
399	       --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
400	       --env USER="root" \
401	       --env CH_LIBC="${libc}" \
402	       "$CTR_IMAGE" \
403	       ./scripts/run_integration_tests_$(uname -m).sh "$@" || fix_dir_perms $? || exit $?
404    fi
405
406    if [ "$integration_sgx" = true ] ;  then
407	say "Running SGX integration tests for $target..."
408	$DOCKER_RUNTIME run \
409	       --workdir "$CTR_CLH_ROOT_DIR" \
410	       --rm \
411	       --privileged \
412	       --security-opt seccomp=unconfined \
413	       --ipc=host \
414	       --net="$CTR_CLH_NET" \
415	       --mount type=tmpfs,destination=/tmp \
416	       --volume /dev:/dev \
417	       --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
418	       --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
419	       --env USER="root" \
420	       --env CH_LIBC="${libc}" \
421	       "$CTR_IMAGE" \
422	       ./scripts/run_integration_tests_sgx.sh "$@" || fix_dir_perms $? || exit $?
423    fi
424
425    if [ "$integration_vfio" = true ] ;  then
426	say "Running VFIO integration tests for $target..."
427	$DOCKER_RUNTIME run \
428	       --workdir "$CTR_CLH_ROOT_DIR" \
429	       --rm \
430	       --privileged \
431	       --security-opt seccomp=unconfined \
432	       --ipc=host \
433	       --net="$CTR_CLH_NET" \
434	       --mount type=tmpfs,destination=/tmp \
435	       --volume /dev:/dev \
436	       --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
437	       --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
438	       --env USER="root" \
439	       --env CH_LIBC="${libc}" \
440	       "$CTR_IMAGE" \
441	       ./scripts/run_integration_tests_vfio.sh "$@" || fix_dir_perms $? || exit $?
442    fi
443
444    if [ "$integration_windows" = true ] ;  then
445	say "Running Windows integration tests for $target..."
446	$DOCKER_RUNTIME run \
447	       --workdir "$CTR_CLH_ROOT_DIR" \
448	       --rm \
449	       --privileged \
450	       --security-opt seccomp=unconfined \
451	       --ipc=host \
452	       --net="$CTR_CLH_NET" \
453	       --mount type=tmpfs,destination=/tmp \
454	       --volume /dev:/dev \
455	       --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR"  $exported_volumes \
456	       --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
457	       --env USER="root" \
458	       --env CH_LIBC="${libc}" \
459	       "$CTR_IMAGE" \
460	       ./scripts/run_integration_tests_windows.sh "$@" || fix_dir_perms $? || exit $?
461    fi
462    fix_dir_perms $?
463}
464
465cmd_build-container() {
466    container_type="dev"
467
468    while [ $# -gt 0 ]; do
469	case "$1" in
470            "-h"|"--help")  { cmd_help; exit 1; } ;;
471            "--dev")        { container_type="dev"; } ;;
472            "--")           { shift; break; } ;;
473            *)
474		die "Unknown build-container argument: $1. Please use --help for help."
475		;;
476	esac
477	shift
478    done
479
480    ensure_build_dir
481    if [ $(uname -m) = "x86_64" ]; then
482	ensure_latest_ctr
483    fi
484
485    BUILD_DIR=/tmp/cloud-hypervisor/container/
486
487    mkdir -p $BUILD_DIR
488    cp $CLH_DOCKERFILE $BUILD_DIR
489
490    [ $(uname -m) = "aarch64" ] && TARGETARCH="arm64"
491    [ $(uname -m) = "x86_64" ] && TARGETARCH="amd64"
492
493    $DOCKER_RUNTIME build \
494	   --target $container_type \
495	   -t $CTR_IMAGE \
496	   -f $BUILD_DIR/Dockerfile \
497	   --build-arg TARGETARCH=$TARGETARCH \
498	   $BUILD_DIR
499}
500
501cmd_shell() {
502    ensure_build_dir
503    if [ $(uname -m) = "x86_64" ]; then
504	ensure_latest_ctr
505    fi
506    say_warn "Starting a privileged shell prompt as root ..."
507    say_warn "WARNING: Your $CLH_ROOT_DIR folder will be bind-mounted in the container under $CTR_CLH_ROOT_DIR"
508    $DOCKER_RUNTIME run \
509	   -ti \
510	   --workdir "$CTR_CLH_ROOT_DIR" \
511	   --rm \
512	   --privileged \
513	   --security-opt seccomp=unconfined \
514	   --ipc=host \
515	   --net="$CTR_CLH_NET" \
516	   --tmpfs /tmp:exec \
517	   --volume /dev:/dev \
518	   --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
519	   --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
520	   --env USER="root" \
521	   --entrypoint bash \
522	   "$CTR_IMAGE"
523
524    fix_dir_perms $?
525}
526
527# Parse main command line args.
528#
529while [ $# -gt 0 ]; do
530    case "$1" in
531        -h|--help)              { cmd_help; exit 1; } ;;
532        -y|--unattended)        { OPT_UNATTENDED=true; } ;;
533        -*)
534            die "Unknown arg: $1. Please use \`$0 help\` for help."
535            ;;
536        *)
537            break
538            ;;
539    esac
540    shift
541done
542
543# $1 is now a command name. Check if it is a valid command and, if so,
544# run it.
545#
546declare -f "cmd_$1" > /dev/null
547ok_or_die "Unknown command: $1. Please use \`$0 help\` for help."
548
549cmd=cmd_$1
550shift
551
552
553$cmd "$@"
554