1#!/bin/bash 2# SPDX-License-Identifier: GPL-2.0 3 4set -e 5 6# This script currently only works for the following platforms, 7# as it is based on the VM image used by the BPF CI, which is 8# available only for these architectures. We can also specify 9# the local rootfs image generated by the following script: 10# https://github.com/libbpf/ci/blob/main/rootfs/mkrootfs_debian.sh 11PLATFORM="${PLATFORM:-$(uname -m)}" 12case "${PLATFORM}" in 13s390x) 14 QEMU_BINARY=qemu-system-s390x 15 QEMU_CONSOLE="ttyS1" 16 HOST_FLAGS=(-smp 2 -enable-kvm) 17 CROSS_FLAGS=(-smp 2) 18 BZIMAGE="arch/s390/boot/vmlinux" 19 ARCH="s390" 20 ;; 21x86_64) 22 QEMU_BINARY=qemu-system-x86_64 23 QEMU_CONSOLE="ttyS0,115200" 24 HOST_FLAGS=(-cpu host -enable-kvm -smp 8) 25 CROSS_FLAGS=(-smp 8) 26 BZIMAGE="arch/x86/boot/bzImage" 27 ARCH="x86" 28 ;; 29aarch64) 30 QEMU_BINARY=qemu-system-aarch64 31 QEMU_CONSOLE="ttyAMA0,115200" 32 HOST_FLAGS=(-M virt,gic-version=3 -cpu host -enable-kvm -smp 8) 33 CROSS_FLAGS=(-M virt,gic-version=3 -cpu cortex-a76 -smp 8) 34 BZIMAGE="arch/arm64/boot/Image" 35 ARCH="arm64" 36 ;; 37riscv64) 38 # required qemu version v7.2.0+ 39 QEMU_BINARY=qemu-system-riscv64 40 QEMU_CONSOLE="ttyS0,115200" 41 HOST_FLAGS=(-M virt -cpu host -enable-kvm -smp 8) 42 CROSS_FLAGS=(-M virt -cpu rv64,sscofpmf=true -smp 8) 43 BZIMAGE="arch/riscv/boot/Image" 44 ARCH="riscv" 45 ;; 46ppc64el) 47 QEMU_BINARY=qemu-system-ppc64 48 QEMU_CONSOLE="hvc0" 49 # KVM could not be tested for powerpc, therefore not enabled for now. 50 HOST_FLAGS=(-machine pseries -cpu POWER9) 51 CROSS_FLAGS=(-machine pseries -cpu POWER9) 52 BZIMAGE="vmlinux" 53 ARCH="powerpc" 54 ;; 55*) 56 echo "Unsupported architecture" 57 exit 1 58 ;; 59esac 60DEFAULT_COMMAND="./test_progs" 61MOUNT_DIR="mnt" 62LOCAL_ROOTFS_IMAGE="" 63ROOTFS_IMAGE="root.img" 64OUTPUT_DIR="$HOME/.bpf_selftests" 65KCONFIG_REL_PATHS=("tools/testing/selftests/bpf/config" 66 "tools/testing/selftests/bpf/config.vm" 67 "tools/testing/selftests/bpf/config.${PLATFORM}") 68INDEX_URL="https://raw.githubusercontent.com/libbpf/ci/master/INDEX" 69NUM_COMPILE_JOBS="$(nproc)" 70LOG_FILE_BASE="$(date +"bpf_selftests.%Y-%m-%d_%H-%M-%S")" 71LOG_FILE="${LOG_FILE_BASE}.log" 72EXIT_STATUS_FILE="${LOG_FILE_BASE}.exit_status" 73 74usage() 75{ 76 cat <<EOF 77Usage: $0 [-i] [-s] [-d <output_dir>] -- [<command>] 78 79<command> is the command you would normally run when you are in 80tools/testing/selftests/bpf. e.g: 81 82 $0 -- ./test_progs -t test_lsm 83 84If no command is specified and a debug shell (-s) is not requested, 85"${DEFAULT_COMMAND}" will be run by default. 86 87Using PLATFORM= and CROSS_COMPILE= options will enable cross platform testing: 88 89 PLATFORM=<platform> CROSS_COMPILE=<toolchain> $0 -- ./test_progs -t test_lsm 90 91If you build your kernel using KBUILD_OUTPUT= or O= options, these 92can be passed as environment variables to the script: 93 94 O=<kernel_build_path> $0 -- ./test_progs -t test_lsm 95 96or 97 98 KBUILD_OUTPUT=<kernel_build_path> $0 -- ./test_progs -t test_lsm 99 100Options: 101 102 -l) Specify the path to the local rootfs image. 103 -i) Update the rootfs image with a newer version. 104 -d) Update the output directory (default: ${OUTPUT_DIR}) 105 -j) Number of jobs for compilation, similar to -j in make 106 (default: ${NUM_COMPILE_JOBS}) 107 -s) Instead of powering off the VM, start an interactive 108 shell. If <command> is specified, the shell runs after 109 the command finishes executing 110EOF 111} 112 113unset URLS 114populate_url_map() 115{ 116 if ! declare -p URLS &> /dev/null; then 117 # URLS contain the mapping from file names to URLs where 118 # those files can be downloaded from. 119 declare -gA URLS 120 while IFS=$'\t' read -r name url; do 121 URLS["$name"]="$url" 122 done < <(curl -Lsf ${INDEX_URL}) 123 fi 124} 125 126newest_rootfs_version() 127{ 128 { 129 for file in "${!URLS[@]}"; do 130 if [[ $file =~ ^"${PLATFORM}"/libbpf-vmtest-rootfs-(.*)\.tar\.zst$ ]]; then 131 echo "${BASH_REMATCH[1]}" 132 fi 133 done 134 } | sort -rV | head -1 135} 136 137download_rootfs() 138{ 139 populate_url_map 140 141 local rootfsversion="$(newest_rootfs_version)" 142 local file="${PLATFORM}/libbpf-vmtest-rootfs-$rootfsversion.tar.zst" 143 144 if [[ ! -v URLS[$file] ]]; then 145 echo "$file not found" >&2 146 return 1 147 fi 148 149 echo "Downloading $file..." >&2 150 curl -Lsf "${URLS[$file]}" "${@:2}" 151} 152 153load_rootfs() 154{ 155 local dir="$1" 156 157 if ! which zstd &> /dev/null; then 158 echo 'Could not find "zstd" on the system, please install zstd' 159 exit 1 160 fi 161 162 if [[ -n "${LOCAL_ROOTFS_IMAGE}" ]]; then 163 cat "${LOCAL_ROOTFS_IMAGE}" | zstd -d | sudo tar -C "$dir" -x 164 else 165 download_rootfs | zstd -d | sudo tar -C "$dir" -x 166 fi 167} 168 169recompile_kernel() 170{ 171 local kernel_checkout="$1" 172 local make_command="$2" 173 174 cd "${kernel_checkout}" 175 176 ${make_command} olddefconfig 177 ${make_command} 178} 179 180mount_image() 181{ 182 local rootfs_img="${OUTPUT_DIR}/${ROOTFS_IMAGE}" 183 local mount_dir="${OUTPUT_DIR}/${MOUNT_DIR}" 184 185 sudo mount -o loop "${rootfs_img}" "${mount_dir}" 186} 187 188unmount_image() 189{ 190 local mount_dir="${OUTPUT_DIR}/${MOUNT_DIR}" 191 192 sudo umount "${mount_dir}" &> /dev/null 193} 194 195update_selftests() 196{ 197 local kernel_checkout="$1" 198 local selftests_dir="${kernel_checkout}/tools/testing/selftests/bpf" 199 200 cd "${selftests_dir}" 201 ${make_command} 202 203 # Mount the image and copy the selftests to the image. 204 mount_image 205 sudo rm -rf "${mount_dir}/root/bpf" 206 sudo cp -r "${selftests_dir}" "${mount_dir}/root" 207 unmount_image 208} 209 210update_init_script() 211{ 212 local init_script_dir="${OUTPUT_DIR}/${MOUNT_DIR}/etc/rcS.d" 213 local init_script="${init_script_dir}/S50-startup" 214 local command="$1" 215 local exit_command="$2" 216 217 mount_image 218 219 if [[ ! -d "${init_script_dir}" ]]; then 220 cat <<EOF 221Could not find ${init_script_dir} in the mounted image. 222This likely indicates a bad rootfs image, Please download 223a new image by passing "-i" to the script 224EOF 225 exit 1 226 227 fi 228 229 sudo bash -c "echo '#!/bin/bash' > ${init_script}" 230 231 if [[ "${command}" != "" ]]; then 232 sudo bash -c "cat >>${init_script}" <<EOF 233# Have a default value in the exit status file 234# incase the VM is forcefully stopped. 235echo "130" > "/root/${EXIT_STATUS_FILE}" 236 237{ 238 cd /root/bpf 239 echo ${command} 240 stdbuf -oL -eL ${command} 241 echo "\$?" > "/root/${EXIT_STATUS_FILE}" 242} 2>&1 | tee "/root/${LOG_FILE}" 243# Ensure that the logs are written to disk 244sync 245EOF 246 fi 247 248 sudo bash -c "echo ${exit_command} >> ${init_script}" 249 sudo chmod a+x "${init_script}" 250 unmount_image 251} 252 253create_vm_image() 254{ 255 local rootfs_img="${OUTPUT_DIR}/${ROOTFS_IMAGE}" 256 local mount_dir="${OUTPUT_DIR}/${MOUNT_DIR}" 257 258 rm -rf "${rootfs_img}" 259 touch "${rootfs_img}" 260 chattr +C "${rootfs_img}" >/dev/null 2>&1 || true 261 262 truncate -s 2G "${rootfs_img}" 263 mkfs.ext4 -q "${rootfs_img}" 264 265 mount_image 266 load_rootfs "${mount_dir}" 267 unmount_image 268} 269 270run_vm() 271{ 272 local kernel_bzimage="$1" 273 local rootfs_img="${OUTPUT_DIR}/${ROOTFS_IMAGE}" 274 275 if ! which "${QEMU_BINARY}" &> /dev/null; then 276 cat <<EOF 277Could not find ${QEMU_BINARY} 278Please install qemu or set the QEMU_BINARY environment variable. 279EOF 280 exit 1 281 fi 282 283 if [[ "${PLATFORM}" != "$(uname -m)" ]]; then 284 QEMU_FLAGS=("${CROSS_FLAGS[@]}") 285 else 286 QEMU_FLAGS=("${HOST_FLAGS[@]}") 287 fi 288 289 ${QEMU_BINARY} \ 290 -nodefaults \ 291 -display none \ 292 -serial mon:stdio \ 293 "${QEMU_FLAGS[@]}" \ 294 -m 4G \ 295 -drive file="${rootfs_img}",format=raw,index=1,media=disk,if=virtio,cache=none \ 296 -kernel "${kernel_bzimage}" \ 297 -append "root=/dev/vda rw console=${QEMU_CONSOLE}" 298} 299 300copy_logs() 301{ 302 local mount_dir="${OUTPUT_DIR}/${MOUNT_DIR}" 303 local log_file="${mount_dir}/root/${LOG_FILE}" 304 local exit_status_file="${mount_dir}/root/${EXIT_STATUS_FILE}" 305 306 mount_image 307 sudo cp ${log_file} "${OUTPUT_DIR}" 308 sudo cp ${exit_status_file} "${OUTPUT_DIR}" 309 sudo rm -f ${log_file} 310 unmount_image 311} 312 313is_rel_path() 314{ 315 local path="$1" 316 317 [[ ${path:0:1} != "/" ]] 318} 319 320do_update_kconfig() 321{ 322 local kernel_checkout="$1" 323 local kconfig_file="$2" 324 325 rm -f "$kconfig_file" 2> /dev/null 326 327 for config in "${KCONFIG_REL_PATHS[@]}"; do 328 local kconfig_src="${kernel_checkout}/${config}" 329 cat "$kconfig_src" >> "$kconfig_file" 330 done 331} 332 333update_kconfig() 334{ 335 local kernel_checkout="$1" 336 local kconfig_file="$2" 337 338 if [[ -f "${kconfig_file}" ]]; then 339 local local_modified="$(stat -c %Y "${kconfig_file}")" 340 341 for config in "${KCONFIG_REL_PATHS[@]}"; do 342 local kconfig_src="${kernel_checkout}/${config}" 343 local src_modified="$(stat -c %Y "${kconfig_src}")" 344 # Only update the config if it has been updated after the 345 # previously cached config was created. This avoids 346 # unnecessarily compiling the kernel and selftests. 347 if [[ "${src_modified}" -gt "${local_modified}" ]]; then 348 do_update_kconfig "$kernel_checkout" "$kconfig_file" 349 # Once we have found one outdated configuration 350 # there is no need to check other ones. 351 break 352 fi 353 done 354 else 355 do_update_kconfig "$kernel_checkout" "$kconfig_file" 356 fi 357} 358 359catch() 360{ 361 local exit_code=$1 362 local exit_status_file="${OUTPUT_DIR}/${EXIT_STATUS_FILE}" 363 # This is just a cleanup and the directory may 364 # have already been unmounted. So, don't let this 365 # clobber the error code we intend to return. 366 unmount_image || true 367 if [[ -f "${exit_status_file}" ]]; then 368 exit_code="$(cat ${exit_status_file})" 369 fi 370 exit ${exit_code} 371} 372 373main() 374{ 375 local script_dir="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" 376 local kernel_checkout=$(realpath "${script_dir}"/../../../../) 377 # By default the script searches for the kernel in the checkout directory but 378 # it also obeys environment variables O= and KBUILD_OUTPUT= 379 local kernel_bzimage="${kernel_checkout}/${BZIMAGE}" 380 local command="${DEFAULT_COMMAND}" 381 local update_image="no" 382 local exit_command="poweroff -f" 383 local debug_shell="no" 384 385 while getopts ':hskl:id:j:' opt; do 386 case ${opt} in 387 l) 388 LOCAL_ROOTFS_IMAGE="$OPTARG" 389 ;; 390 i) 391 update_image="yes" 392 ;; 393 d) 394 OUTPUT_DIR="$OPTARG" 395 ;; 396 j) 397 NUM_COMPILE_JOBS="$OPTARG" 398 ;; 399 s) 400 command="" 401 debug_shell="yes" 402 exit_command="bash" 403 ;; 404 h) 405 usage 406 exit 0 407 ;; 408 \? ) 409 echo "Invalid Option: -$OPTARG" 410 usage 411 exit 1 412 ;; 413 : ) 414 echo "Invalid Option: -$OPTARG requires an argument" 415 usage 416 exit 1 417 ;; 418 esac 419 done 420 shift $((OPTIND -1)) 421 422 trap 'catch "$?"' EXIT 423 424 if [[ "${PLATFORM}" != "$(uname -m)" ]] && [[ -z "${CROSS_COMPILE}" ]]; then 425 echo "Cross-platform testing needs to specify CROSS_COMPILE" 426 exit 1 427 fi 428 429 if [[ $# -eq 0 && "${debug_shell}" == "no" ]]; then 430 echo "No command specified, will run ${DEFAULT_COMMAND} in the vm" 431 else 432 command="$@" 433 fi 434 435 local kconfig_file="${OUTPUT_DIR}/latest.config" 436 local make_command="make ARCH=${ARCH} CROSS_COMPILE=${CROSS_COMPILE} \ 437 -j ${NUM_COMPILE_JOBS} KCONFIG_CONFIG=${kconfig_file}" 438 439 # Figure out where the kernel is being built. 440 # O takes precedence over KBUILD_OUTPUT. 441 if [[ "${O:=""}" != "" ]]; then 442 if is_rel_path "${O}"; then 443 O="$(realpath "${PWD}/${O}")" 444 fi 445 kernel_bzimage="${O}/${BZIMAGE}" 446 make_command="${make_command} O=${O}" 447 elif [[ "${KBUILD_OUTPUT:=""}" != "" ]]; then 448 if is_rel_path "${KBUILD_OUTPUT}"; then 449 KBUILD_OUTPUT="$(realpath "${PWD}/${KBUILD_OUTPUT}")" 450 fi 451 kernel_bzimage="${KBUILD_OUTPUT}/${BZIMAGE}" 452 make_command="${make_command} KBUILD_OUTPUT=${KBUILD_OUTPUT}" 453 fi 454 455 local rootfs_img="${OUTPUT_DIR}/${ROOTFS_IMAGE}" 456 local mount_dir="${OUTPUT_DIR}/${MOUNT_DIR}" 457 458 echo "Output directory: ${OUTPUT_DIR}" 459 460 mkdir -p "${OUTPUT_DIR}" 461 mkdir -p "${mount_dir}" 462 update_kconfig "${kernel_checkout}" "${kconfig_file}" 463 464 recompile_kernel "${kernel_checkout}" "${make_command}" 465 466 if [[ "${update_image}" == "no" && ! -f "${rootfs_img}" ]]; then 467 echo "rootfs image not found in ${rootfs_img}" 468 update_image="yes" 469 fi 470 471 if [[ "${update_image}" == "yes" ]]; then 472 create_vm_image 473 fi 474 475 update_selftests "${kernel_checkout}" "${make_command}" 476 update_init_script "${command}" "${exit_command}" 477 run_vm "${kernel_bzimage}" 478 if [[ "${command}" != "" ]]; then 479 copy_logs 480 echo "Logs saved in ${OUTPUT_DIR}/${LOG_FILE}" 481 fi 482} 483 484main "$@" 485