diff --git a/integration/confidential/lib.sh b/integration/confidential/lib.sh index 9a085c638..b18393583 100644 --- a/integration/confidential/lib.sh +++ b/integration/confidential/lib.sh @@ -12,6 +12,11 @@ source "${BATS_TEST_DIRNAME}/../../../lib/common.bash" source "${BATS_TEST_DIRNAME}/../../../.ci/lib.sh" FIXTURES_DIR="${BATS_TEST_DIRNAME}/fixtures" SHARED_FIXTURES_DIR="${BATS_TEST_DIRNAME}/../../confidential/fixtures" +NYDUS_SNAPSHOTTER_TARFS_CONFIG="/opt/confidential-containers/share/remote-snapshotter/config_coco_tarfs.toml" +NYDUS_SNAPSHOTTER_GUEST_CONFIG="/opt/confidential-containers/share/remote-snapshotter/config-coco-guest-pulling.toml" +NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_TARFS_CONFIG" +NYDUS_SNAPSHOTTER_BINARY="/opt/confidential-containers/bin/containerd-nydus-grpc" +CONTAINERD_CONFIG="/etc/containerd/config.toml" # Toggle between true and false the service_offload configuration of # the Kata agent. @@ -29,18 +34,18 @@ switch_image_service_offload() { load_runtime_config_path case "$1" in - "on") - sudo sed -i -e 's/^\(service_offload\).*=.*$/\1 = true/g' \ - "$RUNTIME_CONFIG_PATH" - ;; - "off") - sudo sed -i -e 's/^\(service_offload\).*=.*$/\1 = false/g' \ - "$RUNTIME_CONFIG_PATH" - - ;; - *) - die "Unknown option '$1'" - ;; + "on") + sudo sed -i -e 's/^\(service_offload\).*=.*$/\1 = true/g' \ + "$RUNTIME_CONFIG_PATH" + ;; + "off") + sudo sed -i -e 's/^\(service_offload\).*=.*$/\1 = false/g' \ + "$RUNTIME_CONFIG_PATH" + + ;; + *) + die "Unknown option '$1'" + ;; esac } @@ -66,13 +71,13 @@ switch_measured_rootfs_verity_scheme() { load_runtime_config_path case "$1" in - "dm-verity"|"none") - sudo sed -i -e 's/scheme=.* cc_rootfs/scheme='"$1"' cc_rootfs/g' \ - "$RUNTIME_CONFIG_PATH" - ;; - *) - die "Unknown option '$1'" - ;; + "dm-verity" | "none") + sudo sed -i -e 's/scheme=.* cc_rootfs/scheme='"$1"' cc_rootfs/g' \ + "$RUNTIME_CONFIG_PATH" + ;; + *) + die "Unknown option '$1'" + ;; esac } @@ -108,11 +113,11 @@ add_kernel_params() { # `kata-runtime env` and export its value. # get_kernel_params() { - load_runtime_config_path + load_runtime_config_path - local kernel_params=$(sed -n -e 's#^kernel_params = "\(.*\)"#\1#gp' \ - "$RUNTIME_CONFIG_PATH") - echo "$kernel_params" + local kernel_params=$(sed -n -e 's#^kernel_params = "\(.*\)"#\1#gp' \ + "$RUNTIME_CONFIG_PATH") + echo "$kernel_params" } # Clear the 'kernel_params' property on kata's configuration.toml @@ -220,7 +225,7 @@ configure_cc_containerd() { echo " [plugins.cri.cni] # conf_dir is the directory in which the admin places a CNI conf. conf_dir = \"/etc/cni/net.d\"" | \ - sudo tee -a "$containerd_conf_file" + sudo tee -a "$containerd_conf_file" fi sudo systemctl restart containerd @@ -286,38 +291,36 @@ setup_cosign_signatures_files() { # Set-up required files in guest image case "${AA_KBC:-}" in - "offline_fs_kbc") - add_kernel_params "agent.aa_kbc_params=offline_fs_kbc::null" - cp_to_guest_img "etc" "${SHARED_FIXTURES_DIR}/cosign/offline-fs-kbc/$(uname -m)/aa-offline_fs_kbc-resources.json" - ;; - "cc_kbc") - # CC KBC is specified as: cc_kbc::host_ip:port, and 60000 is the default port used - # by the service, as well as the one configured in the Kata Containers rootfs. - - CC_KBS_IP=${CC_KBS_IP:-"$(hostname -I | awk '{print $1}')"} - CC_KBS_PORT=${CC_KBS_PORT:-"60000"} - add_kernel_params "agent.aa_kbc_params=cc_kbc::http://${CC_KBS_IP}:${CC_KBS_PORT}/" - ;; - *) - ;; + "offline_fs_kbc") + add_kernel_params "agent.aa_kbc_params=offline_fs_kbc::null" + cp_to_guest_img "etc" "${SHARED_FIXTURES_DIR}/cosign/offline-fs-kbc/$(uname -m)/aa-offline_fs_kbc-resources.json" + ;; + "cc_kbc") + # CC KBC is specified as: cc_kbc::host_ip:port, and 60000 is the default port used + # by the service, as well as the one configured in the Kata Containers rootfs. + + CC_KBS_IP=${CC_KBS_IP:-"$(hostname -I | awk '{print $1}')"} + CC_KBS_PORT=${CC_KBS_PORT:-"60000"} + add_kernel_params "agent.aa_kbc_params=cc_kbc::http://${CC_KBS_IP}:${CC_KBS_PORT}/" + ;; + *) ;; esac } setup_signature_files() { case "${AA_KBC:-}" in - "offline_fs_kbc") - setup_offline_fs_kbc_signature_files_in_guest - ;; - "cc_kbc") - setup_cc_kbc_signature_files_in_guest - ;; - *) - ;; + "offline_fs_kbc") + setup_offline_fs_kbc_signature_files_in_guest + ;; + "cc_kbc") + setup_cc_kbc_signature_files_in_guest + ;; + *) ;; esac } # In case the tests run behind a firewall where images needed to be fetched -# through a proxy. +# through a proxy. # Note: With measured rootfs enabled, we can not set proxy through # agent config file. setup_proxy() { @@ -366,77 +369,140 @@ KBS_DB="${KBS_DB:-simple_kbs}" # Run the simple-kbs simple_kbs_run() { - # Retrieve simple-kbs repo and tag from versions.yaml - local simple_kbs_url=$(get_test_version "externals.simple-kbs.url") - local simple_kbs_tag=$(get_test_version "externals.simple-kbs.tag") - - # Cleanup and create installation directory - esudo rm -rf "${SIMPLE_KBS_DIR}" - mkdir -p "${SIMPLE_KBS_DIR}" - pushd "${SIMPLE_KBS_DIR}" - - # Clone and run - git clone "${simple_kbs_url}" --branch main - pushd simple-kbs - - # Checkout, build and start - git checkout -b "branch_${simple_kbs_tag}" "${simple_kbs_tag}" - esudo docker-compose build - esudo docker-compose up -d - - # Wait for simple-kbs to start - waitForProcess 15 1 "esudo docker-compose top | grep -q simple-kbs" - popd - - # Get simple-kbs database container ip - local kbs_db_host=$(simple_kbs_get_db_ip) - - # Confirm connection to the database is possible - waitForProcess 5 1 "mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} -e '\q'" - popd +# Retrieve simple-kbs repo and tag from versions.yaml +local simple_kbs_url=$(get_test_version "externals.simple-kbs.url") +local simple_kbs_tag=$(get_test_version "externals.simple-kbs.tag") + +# Cleanup and create installation directory +esudo rm -rf "${SIMPLE_KBS_DIR}" +mkdir -p "${SIMPLE_KBS_DIR}" +pushd "${SIMPLE_KBS_DIR}" + +# Clone and run +git clone "${simple_kbs_url}" --branch main +pushd simple-kbs + +# Checkout, build and start +git checkout -b "branch_${simple_kbs_tag}" "${simple_kbs_tag}" +esudo docker-compose build +esudo docker-compose up -d + +# Wait for simple-kbs to start +waitForProcess 15 1 "esudo docker-compose top | grep -q simple-kbs" +popd + +# Get simple-kbs database container ip +local kbs_db_host=$(simple_kbs_get_db_ip) + +# Confirm connection to the database is possible +waitForProcess 5 1 "mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} -e '\q'" +popd } # Stop simple-kbs and database containers simple_kbs_stop() { - (cd ${SIMPLE_KBS_DIR}/simple-kbs && esudo docker-compose down 2>/dev/null) +(cd ${SIMPLE_KBS_DIR}/simple-kbs && esudo docker-compose down 2>/dev/null) } # Delete all test inserted data in the simple-kbs simple_kbs_delete_data() { - # Get simple-kbs database container ip - local kbs_db_host=$(simple_kbs_get_db_ip) +# Get simple-kbs database container ip +local kbs_db_host=$(simple_kbs_get_db_ip) - # Delete all data with 'id = 10' - mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <"$new_config" + echo "$new_config" +} + +setup() { + start_date=$(date +"%Y-%m-%d %H:%M:%S") + + pod_config="$(new_pod_config "$image_simple_signed")" + pod_id="" + + kubernetes_delete_all_cc_pods_if_any_exists || true + + echo "Prepare containerd for Confidential Container" + SAVED_CONTAINERD_CONF_FILE="/etc/containerd/config.toml.$$" + configure_cc_containerd "$SAVED_CONTAINERD_CONF_FILE" + + echo "Reconfigure Kata Containers" + switch_image_service_offload off + clear_kernel_params + add_kernel_params "${original_kernel_params}" + + setup_proxy + switch_measured_rootfs_verity_scheme none +} + +# Check the logged messages on host have a given message. +# Parameters: +# $1 - the message +# +# Note: get the logs since the global $start_date. +# +assert_logs_contain() { + local message="$1" + # Note: with image-rs we get more that the default 1000 lines of logs + journalctl -x -t kata --since "$start_date" -n 100000 | grep "$message" +} + +@test "$test_tag Test can pull an image as a raw block disk image to guest with dm-verity enabled" { + if [ "$SNAPSHOTTER" = "nydus" ]; then + EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter + pod_config="$(new_pod_config "$image_unsigned_unprotected")" + echo $pod_config + create_test_pod "$pod_config" + remove_test_image "$image_unsigned_unprotected" + fi +} + +@test "$test_tag Test can create two pods with pulling the image only once" { + if [ "$SNAPSHOTTER" = "nydus" ]; then + EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter + + pod_config_1="$(new_pod_config "$image_unsigned_unprotected" "1")" + echo $pod_config_1 + create_test_pod $pod_config_1 + pod_config_2="$(new_pod_config "$image_unsigned_unprotected" "2")" + echo $pod_config_2 + create_test_pod $pod_config_2 + + pull_times=$(journalctl -g "PullImage \"$image_unsigned_protected\" with snapshotter nydus" | wc -l) + [ ${#pull_times[@]} -eq 1 ] + remove_test_image "$image_unsigned_unprotected" + fi +} + +@test "$test_tag Test can pull an image inside the guest with remote-snapshotter" { + switch_image_service_offload on + if [ "$SNAPSHOTTER" = "nydus" ]; then + EXPORT_MODE="image_guest_pull" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter + create_test_pod + remove_test_image "$image_unsigned_unprotected" + fi +} + +teardown() { + # Print the logs and cleanup resources. + echo "-- Kata logs:" + sudo journalctl -xe -t kata --since "$start_date" -n 100000 + + # Allow to not destroy the environment if you are developing/debugging + # tests. + if [[ "${CI:-false}" == "false" && "${DEBUG:-}" == true ]]; then + echo "Leaving changes and created resources untouched" + return + fi + + kubernetes_delete_all_cc_pods_if_any_exists || true + clear_kernel_params + add_kernel_params "${original_kernel_params}" + switch_image_service_offload off + remove_nydus_snapshotter_from_containerd + disable_full_debug +}