diff --git a/integration/confidential/lib.sh b/integration/confidential/lib.sh index 9a085c638..c933c7d50 100644 --- a/integration/confidential/lib.sh +++ b/integration/confidential/lib.sh @@ -12,6 +12,11 @@ source "${BATS_TEST_DIRNAME}/../../../lib/common.bash" source "${BATS_TEST_DIRNAME}/../../../.ci/lib.sh" FIXTURES_DIR="${BATS_TEST_DIRNAME}/fixtures" SHARED_FIXTURES_DIR="${BATS_TEST_DIRNAME}/../../confidential/fixtures" +NYDUS_SNAPSHOTTER_TARFS_CONFIG="/opt/confidential-containers/share/remote-snapshotter/config_coco_tarfs.toml" +NYDUS_SNAPSHOTTER_GUEST_CONFIG="/opt/confidential-containers/share/remote-snapshotter/config-coco-guest-pulling.toml" +NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_TARFS_CONFIG" +NYDUS_SNAPSHOTTER_BINARY="/opt/confidential-containers/bin/containerd-nydus-grpc" +CONTAINERD_CONFIG="/etc/containerd/config.toml" # Toggle between true and false the service_offload configuration of # the Kata agent. @@ -29,18 +34,18 @@ switch_image_service_offload() { load_runtime_config_path case "$1" in - "on") - sudo sed -i -e 's/^\(service_offload\).*=.*$/\1 = true/g' \ - "$RUNTIME_CONFIG_PATH" - ;; - "off") - sudo sed -i -e 's/^\(service_offload\).*=.*$/\1 = false/g' \ - "$RUNTIME_CONFIG_PATH" - - ;; - *) - die "Unknown option '$1'" - ;; + "on") + sudo sed -i -e 's/^\(service_offload\).*=.*$/\1 = true/g' \ + "$RUNTIME_CONFIG_PATH" + ;; + "off") + sudo sed -i -e 's/^\(service_offload\).*=.*$/\1 = false/g' \ + "$RUNTIME_CONFIG_PATH" + + ;; + *) + die "Unknown option '$1'" + ;; esac } @@ -66,13 +71,13 @@ switch_measured_rootfs_verity_scheme() { load_runtime_config_path case "$1" in - "dm-verity"|"none") - sudo sed -i -e 's/scheme=.* cc_rootfs/scheme='"$1"' cc_rootfs/g' \ - "$RUNTIME_CONFIG_PATH" - ;; - *) - die "Unknown option '$1'" - ;; + "dm-verity" | "none") + sudo sed -i -e 's/scheme=.* cc_rootfs/scheme='"$1"' cc_rootfs/g' \ + "$RUNTIME_CONFIG_PATH" + ;; + *) + die "Unknown option '$1'" + ;; esac } @@ -110,8 +115,8 @@ add_kernel_params() { get_kernel_params() { load_runtime_config_path - local kernel_params=$(sed -n -e 's#^kernel_params = "\(.*\)"#\1#gp' \ - "$RUNTIME_CONFIG_PATH") + local kernel_params=$(sed -n -e 's#^kernel_params = "\(.*\)"#\1#gp' \ + "$RUNTIME_CONFIG_PATH") echo "$kernel_params" } @@ -203,13 +208,13 @@ configure_cc_containerd() { # restart containerd because it might be in an inconsistent state here. sudo systemctl stop containerd sleep 5 - [ -n "$saved_containerd_conf_file" ] && \ + [ -n "$saved_containerd_conf_file" ] && sudo cp -f "$containerd_conf_file" "$saved_containerd_conf_file" sudo systemctl start containerd waitForProcess 30 5 "sudo crictl info >/dev/null" # Ensure the cc CRI handler is set. - local cri_handler=$(sudo crictl info | \ + local cri_handler=$(sudo crictl info | jq '.config.containerd.runtimes.kata.cri_handler') if [[ ! "$cri_handler" =~ cc ]]; then sudo sed -i 's/\([[:blank:]]*\)\(runtime_type = "io.containerd.kata.v2"\)/\1\2\n\1cri_handler = "cc"/' \ @@ -219,8 +224,8 @@ configure_cc_containerd() { if [ "$(sudo crictl info | jq -r '.config.cni.confDir')" = "null" ]; then echo " [plugins.cri.cni] # conf_dir is the directory in which the admin places a CNI conf. - conf_dir = \"/etc/cni/net.d\"" | \ - sudo tee -a "$containerd_conf_file" + conf_dir = \"/etc/cni/net.d\"" | + sudo tee -a "$containerd_conf_file" fi sudo systemctl restart containerd @@ -286,38 +291,36 @@ setup_cosign_signatures_files() { # Set-up required files in guest image case "${AA_KBC:-}" in - "offline_fs_kbc") - add_kernel_params "agent.aa_kbc_params=offline_fs_kbc::null" - cp_to_guest_img "etc" "${SHARED_FIXTURES_DIR}/cosign/offline-fs-kbc/$(uname -m)/aa-offline_fs_kbc-resources.json" - ;; - "cc_kbc") - # CC KBC is specified as: cc_kbc::host_ip:port, and 60000 is the default port used - # by the service, as well as the one configured in the Kata Containers rootfs. - - CC_KBS_IP=${CC_KBS_IP:-"$(hostname -I | awk '{print $1}')"} - CC_KBS_PORT=${CC_KBS_PORT:-"60000"} - add_kernel_params "agent.aa_kbc_params=cc_kbc::http://${CC_KBS_IP}:${CC_KBS_PORT}/" - ;; - *) - ;; + "offline_fs_kbc") + add_kernel_params "agent.aa_kbc_params=offline_fs_kbc::null" + cp_to_guest_img "etc" "${SHARED_FIXTURES_DIR}/cosign/offline-fs-kbc/$(uname -m)/aa-offline_fs_kbc-resources.json" + ;; + "cc_kbc") + # CC KBC is specified as: cc_kbc::host_ip:port, and 60000 is the default port used + # by the service, as well as the one configured in the Kata Containers rootfs. + + CC_KBS_IP=${CC_KBS_IP:-"$(hostname -I | awk '{print $1}')"} + CC_KBS_PORT=${CC_KBS_PORT:-"60000"} + add_kernel_params "agent.aa_kbc_params=cc_kbc::http://${CC_KBS_IP}:${CC_KBS_PORT}/" + ;; + *) ;; esac } setup_signature_files() { case "${AA_KBC:-}" in - "offline_fs_kbc") - setup_offline_fs_kbc_signature_files_in_guest - ;; - "cc_kbc") - setup_cc_kbc_signature_files_in_guest - ;; - *) - ;; + "offline_fs_kbc") + setup_offline_fs_kbc_signature_files_in_guest + ;; + "cc_kbc") + setup_cc_kbc_signature_files_in_guest + ;; + *) ;; esac } # In case the tests run behind a firewall where images needed to be fetched -# through a proxy. +# through a proxy. # Note: With measured rootfs enabled, we can not set proxy through # agent config file. setup_proxy() { @@ -349,8 +352,8 @@ setup_credentials_files() { dest_dir="$(mktemp -t -d offline-fs-kbc-XXXXXXXX)" dest_file=${dest_dir}/aa-offline_fs_kbc-resources.json - auth_json=$(REGISTRY=$1 CREDENTIALS="${REGISTRY_CREDENTIAL_ENCODED}" envsubst < "${SHARED_FIXTURES_DIR}/offline-fs-kbc/auth.json.in" | base64 -w 0) - CREDENTIAL="${auth_json}" envsubst < "${SHARED_FIXTURES_DIR}/offline-fs-kbc/aa-offline_fs_kbc-resources.json.in" > "${dest_file}" + auth_json=$(REGISTRY=$1 CREDENTIALS="${REGISTRY_CREDENTIAL_ENCODED}" envsubst <"${SHARED_FIXTURES_DIR}/offline-fs-kbc/auth.json.in" | base64 -w 0) + CREDENTIAL="${auth_json}" envsubst <"${SHARED_FIXTURES_DIR}/offline-fs-kbc/aa-offline_fs_kbc-resources.json.in" >"${dest_file}" cp_to_guest_img "etc" "${dest_file}" } @@ -366,48 +369,48 @@ KBS_DB="${KBS_DB:-simple_kbs}" # Run the simple-kbs simple_kbs_run() { - # Retrieve simple-kbs repo and tag from versions.yaml - local simple_kbs_url=$(get_test_version "externals.simple-kbs.url") - local simple_kbs_tag=$(get_test_version "externals.simple-kbs.tag") - - # Cleanup and create installation directory - esudo rm -rf "${SIMPLE_KBS_DIR}" - mkdir -p "${SIMPLE_KBS_DIR}" - pushd "${SIMPLE_KBS_DIR}" - - # Clone and run - git clone "${simple_kbs_url}" --branch main - pushd simple-kbs - - # Checkout, build and start - git checkout -b "branch_${simple_kbs_tag}" "${simple_kbs_tag}" - esudo docker-compose build - esudo docker-compose up -d - - # Wait for simple-kbs to start - waitForProcess 15 1 "esudo docker-compose top | grep -q simple-kbs" - popd - - # Get simple-kbs database container ip - local kbs_db_host=$(simple_kbs_get_db_ip) - - # Confirm connection to the database is possible - waitForProcess 5 1 "mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} -e '\q'" - popd + # Retrieve simple-kbs repo and tag from versions.yaml + local simple_kbs_url=$(get_test_version "externals.simple-kbs.url") + local simple_kbs_tag=$(get_test_version "externals.simple-kbs.tag") + + # Cleanup and create installation directory + esudo rm -rf "${SIMPLE_KBS_DIR}" + mkdir -p "${SIMPLE_KBS_DIR}" + pushd "${SIMPLE_KBS_DIR}" + + # Clone and run + git clone "${simple_kbs_url}" --branch main + pushd simple-kbs + + # Checkout, build and start + git checkout -b "branch_${simple_kbs_tag}" "${simple_kbs_tag}" + esudo docker-compose build + esudo docker-compose up -d + + # Wait for simple-kbs to start + waitForProcess 15 1 "esudo docker-compose top | grep -q simple-kbs" + popd + + # Get simple-kbs database container ip + local kbs_db_host=$(simple_kbs_get_db_ip) + + # Confirm connection to the database is possible + waitForProcess 5 1 "mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} -e '\q'" + popd } # Stop simple-kbs and database containers simple_kbs_stop() { - (cd ${SIMPLE_KBS_DIR}/simple-kbs && esudo docker-compose down 2>/dev/null) + (cd ${SIMPLE_KBS_DIR}/simple-kbs && esudo docker-compose down 2>/dev/null) } # Delete all test inserted data in the simple-kbs simple_kbs_delete_data() { - # Get simple-kbs database container ip - local kbs_db_host=$(simple_kbs_get_db_ip) + # Get simple-kbs database container ip + local kbs_db_host=$(simple_kbs_get_db_ip) - # Delete all data with 'id = 10' - mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <"$new_config" + echo "$new_config" +} + +setup() { + start_date=$(date +"%Y-%m-%d %H:%M:%S") + + pod_config="$(new_pod_config "$image_simple_signed")" + pod_id="" + + kubernetes_delete_all_cc_pods_if_any_exists || true + + echo "Prepare containerd for Confidential Container" + SAVED_CONTAINERD_CONF_FILE="/etc/containerd/config.toml.$$" + configure_cc_containerd "$SAVED_CONTAINERD_CONF_FILE" + + echo "Reconfigure Kata Containers" + switch_image_service_offload off + clear_kernel_params + add_kernel_params "${original_kernel_params}" + + setup_proxy + switch_measured_rootfs_verity_scheme none +} + +# Check the logged messages on host have a given message. +# Parameters: +# $1 - the message +# +# Note: get the logs since the global $start_date. +# +assert_logs_contain() { + local message="$1" + # Note: with image-rs we get more that the default 1000 lines of logs + journalctl -x -t kata --since "$start_date" -n 100000 | grep "$message" +} + +@test "$test_tag Test can pull an image as a raw block disk image to guest with dm-verity enabled" { + if [ "$SNAPSHOTTER" = "nydus" ]; then + EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter + pod_config="$(new_pod_config "$image_unsigned_unprotected")" + echo $pod_config + create_test_pod "$pod_config" + remove_test_image "$image_unsigned_unprotected" + fi +} + +@test "$test_tag Test can create two pods with pulling the image only once" { + if [ "$SNAPSHOTTER" = "nydus" ]; then + EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter + + pod_config_1="$(new_pod_config "$image_unsigned_unprotected" "1")" + echo $pod_config_1 + create_test_pod $pod_config_1 + pod_config_2="$(new_pod_config "$image_unsigned_unprotected" "2")" + echo $pod_config_2 + create_test_pod $pod_config_2 + + pull_times=$(journalctl -g "PullImage \"$image_unsigned_protected\" with snapshotter nydus" | wc -l) + [ ${#pull_times[@]} -eq 1 ] + remove_test_image "$image_unsigned_unprotected" + fi +} + +@test "$test_tag Test can pull an image inside the guest with remote-snapshotter" { + switch_image_service_offload on + if [ "$SNAPSHOTTER" = "nydus" ]; then + EXPORT_MODE="image_guest_pull" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter + create_test_pod + remove_test_image "$image_unsigned_unprotected" + fi +} + +teardown() { + # Print the logs and cleanup resources. + echo "-- Kata logs:" + sudo journalctl -xe -t kata --since "$start_date" -n 100000 + + # Allow to not destroy the environment if you are developing/debugging + # tests. + if [[ "${CI:-false}" == "false" && "${DEBUG:-}" == true ]]; then + echo "Leaving changes and created resources untouched" + return + fi + + kubernetes_delete_all_cc_pods_if_any_exists || true + clear_kernel_params + add_kernel_params "${original_kernel_params}" + switch_image_service_offload off + remove_nydus_snapshotter_from_containerd + disable_full_debug +}