xref: /cloud-hypervisor/scripts/dev_cli.sh (revision 8c92d1dbdc7617b88d34687115e0e69bd78b332c)
1#!/bin/bash
2
3# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
4# Copyright © 2020 Intel Corporation
5# SPDX-License-Identifier: Apache-2.0
6
7CLI_NAME="Cloud Hypervisor"
8
9CTR_IMAGE_TAG="cloudhypervisor/dev"
10CTR_IMAGE_VERSION="latest"
11CTR_IMAGE="${CTR_IMAGE_TAG}:${CTR_IMAGE_VERSION}"
12
13DOCKER_RUNTIME="docker"
14
15# Host paths
16CLH_SCRIPTS_DIR=$(cd "$(dirname "$0")" && pwd)
17CLH_ROOT_DIR=$(cd "${CLH_SCRIPTS_DIR}/.." && pwd)
18CLH_BUILD_DIR="${CLH_ROOT_DIR}/build"
19CLH_CARGO_TARGET="${CLH_BUILD_DIR}/cargo_target"
20CLH_DOCKERFILE="${CLH_SCRIPTS_DIR}/../resources/Dockerfile"
21CLH_CTR_BUILD_DIR="/tmp/cloud-hypervisor/ctr-build"
22CLH_INTEGRATION_WORKLOADS="${HOME}/workloads"
23
24# Container paths
25CTR_CLH_ROOT_DIR="/cloud-hypervisor"
26CTR_CLH_CARGO_BUILT_DIR="${CTR_CLH_ROOT_DIR}/build"
27CTR_CLH_CARGO_TARGET="${CTR_CLH_CARGO_BUILT_DIR}/cargo_target"
28CTR_CLH_INTEGRATION_WORKLOADS="/root/workloads"
29
30# Container networking option
31CTR_CLH_NET="host"
32[ $(uname -m) = "aarch64" ] && CTR_CLH_NET="bridge"
33
34# Cargo paths
35# Full path to the cargo registry dir on the host. This appears on the host
36# because we want to persist the cargo registry across container invocations.
37# Otherwise, any rust crates from crates.io would be downloaded again each time
38# we build or test.
39CARGO_REGISTRY_DIR="${CLH_BUILD_DIR}/cargo_registry"
40
41# Full path to the cargo git registry on the host. This serves the same purpose
42# as CARGO_REGISTRY_DIR, for crates downloaded from GitHub repos instead of
43# crates.io.
44CARGO_GIT_REGISTRY_DIR="${CLH_BUILD_DIR}/cargo_git_registry"
45
46# Full path to the cargo target dir on the host.
47CARGO_TARGET_DIR="${CLH_BUILD_DIR}/cargo_target"
48
49# Send a decorated message to stdout, followed by a new line
50#
51say() {
52    [ -t 1 ] && [ -n "$TERM" ] \
53        && echo "$(tput setaf 2)[$CLI_NAME]$(tput sgr0) $*" \
54        || echo "[$CLI_NAME] $*"
55}
56
57# Send a decorated message to stdout, without a trailing new line
58#
59say_noln() {
60    [ -t 1 ] && [ -n "$TERM" ] \
61        && echo -n "$(tput setaf 2)[$CLI_NAME]$(tput sgr0) $*" \
62        || echo "[$CLI_NAME] $*"
63}
64
65# Send a text message to stderr
66#
67say_err() {
68    [ -t 2 ] && [ -n "$TERM" ] \
69        && echo "$(tput setaf 1)[$CLI_NAME] $*$(tput sgr0)" 1>&2 \
70        || echo "[$CLI_NAME] $*" 1>&2
71}
72
73# Send a warning-highlighted text to stdout
74say_warn() {
75    [ -t 1 ] && [ -n "$TERM" ] \
76        && echo "$(tput setaf 3)[$CLI_NAME] $*$(tput sgr0)" \
77        || echo "[$CLI_NAME] $*"
78}
79
80# Exit with an error message and (optional) code
81# Usage: die [-c <error code>] <error message>
82#
83die() {
84    code=1
85    [[ "$1" = "-c" ]] && {
86        code="$2"
87        shift 2
88    }
89    say_err "$@"
90    exit $code
91}
92
93# Exit with an error message if the last exit code is not 0
94#
95ok_or_die() {
96    code=$?
97    [[ $code -eq 0 ]] || die -c $code "$@"
98}
99
100# Make sure the build/ dirs are available. Exit if we can't create them.
101# Upon returning from this call, the caller can be certain the build/ dirs exist.
102#
103ensure_build_dir() {
104    for dir in "$CLH_BUILD_DIR" \
105		   "$CLH_INTEGRATION_WORKLOADS" \
106		   "$CLH_CTR_BUILD_DIR" \
107		   "$CARGO_TARGET_DIR" \
108		   "$CARGO_REGISTRY_DIR" \
109		   "$CARGO_GIT_REGISTRY_DIR"; do
110        mkdir -p "$dir" || die "Error: cannot create dir $dir"
111        [ -x "$dir" ] && [ -w "$dir" ] || \
112            {
113                say "Wrong permissions for $dir. Attempting to fix them ..."
114                chmod +x+w "$dir"
115            } || \
116            die "Error: wrong permissions for $dir. Should be +x+w"
117    done
118}
119
120# Make sure we're using the latest dev container, by just pulling it.
121ensure_latest_ctr() {
122    $DOCKER_RUNTIME pull "$CTR_IMAGE"
123
124    ok_or_die "Error pulling container image. Aborting."
125}
126
127# Fix main directory permissions after a container ran as root.
128# Since the container ran as root, any files it creates will be owned by root.
129# This fixes that by recursively changing the ownership of /cloud-hypervisor to the
130# current user.
131#
132fix_dir_perms() {
133    # Yes, running Docker to get elevated privileges, just to chown some files
134    # is a dirty hack.
135    $DOCKER_RUNTIME run \
136	--workdir "$CTR_CLH_ROOT_DIR" \
137	   --rm \
138	   --volume /dev:/dev \
139	   --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
140	   "$CTR_IMAGE" \
141           chown -R "$(id -u):$(id -g)" "$CTR_CLH_ROOT_DIR"
142
143    return $1
144}
145# Process exported volumes argument, separate the volumes and make docker compatible
146# Sample input: --volumes /a:/a#/b:/b
147# Sample output: --volume /a:/a --volume /b:/b
148#
149process_volumes_args() {
150    if [ -z "$arg_vols" ]; then
151        return
152    fi
153    exported_volumes=""
154    arr_vols=(${arg_vols//#/ })
155    for var in "${arr_vols[@]}"
156    do
157        parts=(${var//:/ })
158        if [[ ! -e "${parts[0]}" ]]; then
159            echo "The volume ${parts[0]} does not exist."
160            exit 1
161        fi
162        exported_volumes="$exported_volumes --volume $var"
163    done
164}
165cmd_help() {
166    echo ""
167    echo "Cloud Hypervisor $(basename $0)"
168    echo "Usage: $(basename $0) <command> [<command args>]"
169    echo ""
170    echo "Available commands:"
171    echo ""
172    echo "    build [--debug|--release] [--libc musl|gnu] [-- [<cargo args>]]"
173    echo "        Build the Cloud Hypervisor binaries."
174    echo "        --debug               Build the debug binaries. This is the default."
175    echo "        --release             Build the release binaries."
176    echo "        --libc                Select the C library Cloud Hypervisor will be built against. Default is gnu"
177    echo "        --volumes             Hash separated volumes to be exported. Example --volumes /mnt:/mnt#/myvol:/myvol"
178    echo ""
179    echo "    tests [--unit|--cargo|--all] [--libc musl|gnu] [-- [<cargo test args>]]"
180    echo "        Run the Cloud Hypervisor tests."
181    echo "        --unit                Run the unit tests."
182    echo "        --cargo               Run the cargo tests."
183    echo "        --integration         Run the integration tests."
184    echo "        --integration-sgx     Run the SGX integration tests."
185    echo "        --integration-windows Run the Windows guest integration tests."
186    echo "        --libc                Select the C library Cloud Hypervisor will be built against. Default is gnu"
187    echo "        --volumes             Hash separated volumes to be exported. Example --volumes /mnt:/mnt#/myvol:/myvol"
188    echo "        --all                 Run all tests."
189    echo ""
190    echo "    build-container [--type]"
191    echo "        Build the Cloud Hypervisor container."
192    echo "        --dev                Build dev container. This is the default."
193    echo ""
194    echo "    clean [<cargo args>]]"
195    echo "        Remove the Cloud Hypervisor artifacts."
196    echo ""
197    echo "    shell"
198    echo "        Run the development container into an interactive, privileged BASH shell."
199    echo ""
200    echo "    help"
201    echo "        Display this help message."
202    echo ""
203}
204
205cmd_build() {
206    build="debug"
207    libc="gnu"
208
209    while [ $# -gt 0 ]; do
210	case "$1" in
211            "-h"|"--help")  { cmd_help; exit 1;     } ;;
212            "--debug")      { build="debug";      } ;;
213            "--release")    { build="release";    } ;;
214            "--libc")
215                shift
216                [[ "$1" =~ ^(musl|gnu)$ ]] || \
217                    die "Invalid libc: $1. Valid options are \"musl\" and \"gnu\"."
218                libc="$1"
219                ;;
220            "--volumes")
221                shift
222                arg_vols="$1"
223                ;;
224            "--")           { shift; break;         } ;;
225            *)
226		die "Unknown build argument: $1. Please use --help for help."
227		;;
228	esac
229	shift
230    done
231    process_volumes_args
232
233    target="$(uname -m)-unknown-linux-${libc}"
234
235    cargo_args=("$@")
236    [ $build = "release" ] && cargo_args+=("--release")
237    cargo_args+=(--target "$target")
238    [ $(uname -m) = "aarch64" ] && cargo_args+=("--no-default-features")
239    [ $(uname -m) = "aarch64" ] && cargo_args+=(--features "kvm")
240
241    rustflags=""
242    if [ $(uname -m) = "aarch64" ] && [ $libc = "musl" ] ; then
243        rustflags="-C link-arg=-lgcc -C link_arg=-specs -C link_arg=/usr/lib/aarch64-linux-musl/musl-gcc.specs"
244    fi
245
246    $DOCKER_RUNTIME run \
247	   --user "$(id -u):$(id -g)" \
248	   --workdir "$CTR_CLH_ROOT_DIR" \
249	   --rm \
250	   --volume /dev:/dev \
251	   --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
252	   --env RUSTFLAGS="$rustflags" \
253	   "$CTR_IMAGE" \
254	   cargo build --all \
255	         --target-dir "$CTR_CLH_CARGO_TARGET" \
256	         "${cargo_args[@]}" && say "Binaries placed under $CLH_CARGO_TARGET/$target/$build"
257}
258
259cmd_clean() {
260    cargo_args=("$@")
261
262    $DOCKER_RUNTIME run \
263	   --user "$(id -u):$(id -g)" \
264	   --workdir "$CTR_CLH_ROOT_DIR" \
265	   --rm \
266	   --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
267	   "$CTR_IMAGE" \
268	   cargo clean \
269	         --target-dir "$CTR_CLH_CARGO_TARGET" \
270	         "${cargo_args[@]}"
271    }
272
273cmd_tests() {
274    unit=false
275    cargo=false
276    integration=false
277    integration_sgx=false
278    integration_windows=false
279    libc="gnu"
280    arg_vols=""
281
282    while [ $# -gt 0 ]; do
283	case "$1" in
284            "-h"|"--help")           { cmd_help; exit 1;     } ;;
285            "--unit")                { unit=true;      } ;;
286            "--cargo")               { cargo=true;    } ;;
287            "--integration")         { integration=true;    } ;;
288            "--integration-sgx")     { integration_sgx=true;    } ;;
289            "--integration-windows") { integration_windows=true;    } ;;
290            "--libc")
291                shift
292                [[ "$1" =~ ^(musl|gnu)$ ]] || \
293                    die "Invalid libc: $1. Valid options are \"musl\" and \"gnu\"."
294                libc="$1"
295                ;;
296            "--volumes")
297                shift
298                arg_vols="$1"
299                ;;
300	    "--all")                 { cargo=true; unit=true; integration=true;  } ;;
301            "--")                    { shift; break;         } ;;
302            *)
303		die "Unknown tests argument: $1. Please use --help for help."
304		;;
305	esac
306	shift
307    done
308
309    process_volumes_args
310    target="$(uname -m)-unknown-linux-${libc}"
311    cflags=""
312    target_cc=""
313    if [[ "$target" == "x86_64-unknown-linux-musl" ]]; then
314	target_cc="musl-gcc"
315	cflags="-I /usr/include/x86_64-linux-musl/ -idirafter /usr/include/"
316    fi
317
318    if [ "$unit" = true ] ;  then
319	say "Running unit tests for $target..."
320	$DOCKER_RUNTIME run \
321	       --workdir "$CTR_CLH_ROOT_DIR" \
322	       --rm \
323	       --device /dev/kvm \
324	       --device /dev/net/tun \
325	       --cap-add net_admin \
326	       --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
327	       --env BUILD_TARGET="$target" \
328	       --env CFLAGS="$cflags" \
329	       --env TARGET_CC="$target_cc" \
330	       "$CTR_IMAGE" \
331	       ./scripts/run_unit_tests.sh "$@" || fix_dir_perms $? || exit $?
332    fi
333
334    if [ "$cargo" = true ] ;  then
335	say "Running cargo tests..."
336	$DOCKER_RUNTIME run \
337	       --workdir "$CTR_CLH_ROOT_DIR" \
338	       --rm \
339	       --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
340	       "$CTR_IMAGE" \
341	       ./scripts/run_cargo_tests.sh || fix_dir_perms $? || exit $?
342    fi
343
344    if [ "$integration" = true ] ;  then
345	say "Running integration tests for $target..."
346	$DOCKER_RUNTIME run \
347	       --workdir "$CTR_CLH_ROOT_DIR" \
348	       --rm \
349	       --privileged \
350	       --security-opt seccomp=unconfined \
351	       --ipc=host \
352	       --net="$CTR_CLH_NET" \
353	       --mount type=tmpfs,destination=/tmp \
354	       --volume /dev:/dev \
355	       --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
356	       --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
357	       --env USER="root" \
358	       --env CH_LIBC="${libc}" \
359	       "$CTR_IMAGE" \
360	       ./scripts/run_integration_tests_$(uname -m).sh "$@" || fix_dir_perms $? || exit $?
361    fi
362
363    if [ "$integration_sgx" = true ] ;  then
364	say "Running SGX integration tests for $target..."
365	$DOCKER_RUNTIME run \
366	       --workdir "$CTR_CLH_ROOT_DIR" \
367	       --rm \
368	       --privileged \
369	       --security-opt seccomp=unconfined \
370	       --ipc=host \
371	       --net="$CTR_CLH_NET" \
372	       --mount type=tmpfs,destination=/tmp \
373	       --volume /dev:/dev \
374	       --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
375	       --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
376	       --env USER="root" \
377	       --env CH_LIBC="${libc}" \
378	       "$CTR_IMAGE" \
379	       ./scripts/run_integration_tests_sgx.sh "$@" || fix_dir_perms $? || exit $?
380    fi
381
382    if [ "$integration_windows" = true ] ;  then
383	say "Running Windows integration tests for $target..."
384	$DOCKER_RUNTIME run \
385	       --workdir "$CTR_CLH_ROOT_DIR" \
386	       --rm \
387	       --privileged \
388	       --security-opt seccomp=unconfined \
389	       --ipc=host \
390	       --net="$CTR_CLH_NET" \
391	       --mount type=tmpfs,destination=/tmp \
392	       --volume /dev:/dev \
393	       --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" \
394	       --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
395	       --env USER="root" \
396	       --env CH_LIBC="${libc}" \
397	       "$CTR_IMAGE" \
398	       ./scripts/run_integration_tests_windows.sh "$@" || fix_dir_perms $? || exit $?
399    fi
400    fix_dir_perms $?
401}
402
403cmd_build-container() {
404    container_type="dev"
405
406    while [ $# -gt 0 ]; do
407	case "$1" in
408            "-h"|"--help")  { cmd_help; exit 1;     } ;;
409            "--dev")        { container_type="dev"; } ;;
410            "--")           { shift; break;         } ;;
411            *)
412		die "Unknown build-container argument: $1. Please use --help for help."
413		;;
414	esac
415	shift
416    done
417
418    BUILD_DIR=/tmp/cloud-hypervisor/container/
419
420    mkdir -p $BUILD_DIR
421    cp $CLH_DOCKERFILE $BUILD_DIR
422
423    [ $(uname -m) = "aarch64" ] && TARGETARCH="arm64"
424    [ $(uname -m) = "x86_64" ] && TARGETARCH="amd64"
425
426    $DOCKER_RUNTIME build \
427	   --target $container_type \
428	   -t $CTR_IMAGE \
429	   -f $BUILD_DIR/Dockerfile \
430	   --build-arg TARGETARCH=$TARGETARCH \
431	   $BUILD_DIR
432}
433
434cmd_shell() {
435    say_warn "Starting a privileged shell prompt as root ..."
436    say_warn "WARNING: Your $CLH_ROOT_DIR folder will be bind-mounted in the container under $CTR_CLH_ROOT_DIR"
437    $DOCKER_RUNTIME run \
438	   -ti \
439	   --workdir "$CTR_CLH_ROOT_DIR" \
440	   --rm \
441	   --privileged \
442	   --security-opt seccomp=unconfined \
443	   --ipc=host \
444	   --net="$CTR_CLH_NET" \
445	   --tmpfs /tmp:exec \
446	   --volume /dev:/dev \
447	   --volume "$CLH_ROOT_DIR:$CTR_CLH_ROOT_DIR" $exported_volumes \
448	   --volume "$CLH_INTEGRATION_WORKLOADS:$CTR_CLH_INTEGRATION_WORKLOADS" \
449	   --env USER="root" \
450	   --entrypoint bash \
451	   "$CTR_IMAGE"
452
453    fix_dir_perms $?
454}
455
456# Parse main command line args.
457#
458while [ $# -gt 0 ]; do
459    case "$1" in
460        -h|--help)              { cmd_help; exit 1;     } ;;
461        -y|--unattended)        { OPT_UNATTENDED=true;  } ;;
462        -*)
463            die "Unknown arg: $1. Please use \`$0 help\` for help."
464            ;;
465        *)
466            break
467            ;;
468    esac
469    shift
470done
471
472# $1 is now a command name. Check if it is a valid command and, if so,
473# run it.
474#
475declare -f "cmd_$1" > /dev/null
476ok_or_die "Unknown command: $1. Please use \`$0 help\` for help."
477
478cmd=cmd_$1
479shift
480
481ensure_build_dir
482if [ $(uname -m) = "x86_64" ]; then
483    ensure_latest_ctr
484fi
485
486$cmd "$@"
487