diff --git a/.ci/install_nydus_snapshotter.sh b/.ci/install_nydus_snapshotter.sh new file mode 100644 index 000000000..5e2da6c6f --- /dev/null +++ b/.ci/install_nydus_snapshotter.sh @@ -0,0 +1,90 @@ +#!/bin/bash +# +# Copyright (c) 2023 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -o errexit +set -o nounset +set -o pipefail +set -o errtrace + +cidir=$(dirname "$0") +source "${cidir}/lib.sh" + +target_dir="/opt/confidential-containers" + +nydus_snapshotter_repo=${nydus_snapshotter_repo:-"github.com/containerd/nydus-snapshotter"} +nydus_snapshotter_repo_git="https://${nydus_snapshotter_repo}.git" +nydus_snapshotter_version=${nydus_snapshotter_version:-"v0.12.0"} +nydus_snapshotter_repo_dir="${GOPATH}/src/${nydus_snapshotter_repo}" +nydus_snapshotter_binary_target_dir="$target_dir/bin" +nydus_snapshotter_config_target_dir="$target_dir/share/nydus-snapshotter" + +nydus_repo=${nydus_repo:-"https://github.com/dragonflyoss/image-service"} +nydus_version=${nydus_version:-"v2.2.3"} + +arch="$(uname -m)" + +clone_nydus_snapshotter_repo() { + add_repo_to_git_safe_directory "${nydus_snapshotter_repo_dir}" + + if [ ! -d "${nydus_snapshotter_repo_dir}" ]; then + sudo mkdir -p "${nydus_snapshotter_repo_dir}" + git clone ${nydus_snapshotter_repo_git} "${nydus_snapshotter_repo_dir}" || true + pushd "${nydus_snapshotter_repo_dir}" + git checkout "${nydus_snapshotter_version}" + popd + fi +} + +build_nydus_snapshotter() { + pushd "${nydus_snapshotter_repo_dir}" + if [ "$arch" = "s390x" ]; then + export GOARCH=$arch + fi + sudo -E PATH=$PATH make + + if [ ! -d "$nydus_snapshotter_binary_target_dir" ]; then + sudo mkdir -p $nydus_snapshotter_binary_target_dir + fi + sudo install -D -m 755 "bin/containerd-nydus-grpc" "$nydus_snapshotter_binary_target_dir/containerd-nydus-grpc" + sudo install -D -m 755 "bin/nydus-overlayfs" "$nydus_snapshotter_binary_target_dir/nydus-overlayfs" + if [ ! -f "/usr/local/bin/nydus-overlayfs" ]; then + echo " /usr/local/bin/nydus-overlayfs exists, now we will replace it." + sudo cp -f "$nydus_snapshotter_binary_target_dir/nydus-overlayfs" "/usr/local/bin/nydus-overlayfs" + fi + sudo rm -rf "$nydus_snapshotter_repo_dir/bin" + popd >/dev/null +} + +download_nydus_snapshotter_config() { + if [ ! -d "$nydus_snapshotter_config_target_dir" ]; then + mkdir -p "$nydus_snapshotter_config_target_dir" + fi + sudo curl -L https://raw.githubusercontent.com/containerd/nydus-snapshotter/main/misc/snapshotter/config-coco-guest-pulling.toml -o "$nydus_snapshotter_config_target_dir/config-coco-guest-pulling.toml" + sudo curl -L https://raw.githubusercontent.com/containerd/nydus-snapshotter/main/misc/snapshotter/config-coco-host-sharing.toml -o "$nydus_snapshotter_config_target_dir/config-coco-host-sharing.toml" + sudo chmod 644 "$nydus_snapshotter_config_target_dir/config-coco-guest-pulling.toml" + sudo chmod 644 "$nydus_snapshotter_config_target_dir/config-coco-host-sharing.toml" + +} + +download_nydus_from_tarball() { + if [ "$arch" = "s390x" ]; then + echo "Skip to download nydus for $arch, it doesn't work for $arch now." + return + fi + local goarch="$(${cidir}/kata-arch.sh --golang)" + local tarball_url="${nydus_repo}/releases/download/${nydus_version}/nydus-static-${nydus_version}-linux-$goarch.tgz" + echo "Download tarball from ${tarball_url}" + tmp_dir=$(mktemp -d -t install-nydus-tmp.XXXXXXXXXX) + curl -Ls "$tarball_url" | sudo tar xfz - -C $tmp_dir --strip-components=1 + sudo install -D -m 755 "$tmp_dir/nydus-image" "/usr/local/bin/" +} + +download_nydus_from_tarball +clone_nydus_snapshotter_repo +build_nydus_snapshotter +download_nydus_snapshotter_config +echo "install nydus-snapshotter successful" diff --git a/Makefile b/Makefile index 48b892219..b2a5c07e8 100644 --- a/Makefile +++ b/Makefile @@ -104,7 +104,7 @@ cc-containerd: # Run the Confidential Containers tests for kubernetes. cc-kubernetes: bash -f .ci/install_bats.sh - K8S_TEST_UNION="confidential/agent_image.bats confidential/agent_image_encrypted.bats confidential/sealed_secret.bats" \ + K8S_TEST_UNION="confidential/agent_image.bats confidential/agent_image_encrypted.bats confidential/sealed_secret.bats confidential/image_pulling_with_snapshotter.bats" \ bash integration/kubernetes/run_kubernetes_tests.sh # Run the Confidential Containers AMD SEV specific tests. diff --git a/integration/confidential/lib.sh b/integration/confidential/lib.sh index 6e0873d05..9ec87e02c 100644 --- a/integration/confidential/lib.sh +++ b/integration/confidential/lib.sh @@ -12,6 +12,10 @@ source "${BATS_TEST_DIRNAME}/../../../lib/common.bash" source "${BATS_TEST_DIRNAME}/../../../.ci/lib.sh" FIXTURES_DIR="${BATS_TEST_DIRNAME}/fixtures" SHARED_FIXTURES_DIR="${BATS_TEST_DIRNAME}/../../confidential/fixtures" +NYDUS_SNAPSHOTTER_BINARY="/opt/kata/bin/containerd-nydus-grpc" +NYDUS_SNAPSHOTTER_TARFS_CONFIG="/opt/kata/share/nydus-snapshotter/config-coco-host-sharing.toml" +NYDUS_SNAPSHOTTER_GUEST_CONFIG="/opt/kata/share/nydus-snapshotter/config-coco-guest-pulling.toml" +NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_TARFS_CONFIG" # Toggle between true and false the service_offload configuration of # the Kata agent. @@ -180,6 +184,13 @@ disable_full_debug() { sudo sed -i -e 's/^# *\(enable_debug\).*=.*$/\1 = false/g' "$RUNTIME_CONFIG_PATH" } +restart_containerd() { + sudo systemctl restart containerd + if ! waitForProcess 30 5 "sudo crictl info >/dev/null"; then + die "containerd seems not operational after restarted" + fi +} + # Configure containerd for confidential containers. Among other things, it ensures # the CRI handler is configured to deal with confidential container. # @@ -198,6 +209,7 @@ configure_cc_containerd() { # installed via operator it will assume containerd is in right state # already. [ "${TESTS_CONFIGURE_CC_CONTAINERD:-yes}" == "yes" ] || return 0 + sudo iptables -w -P FORWARD ACCEPT # Even if we are not saving the original file it is a good idea to # restart containerd because it might be in an inconsistent state here. @@ -205,8 +217,7 @@ configure_cc_containerd() { sleep 5 [ -n "$saved_containerd_conf_file" ] && \ sudo cp -f "$containerd_conf_file" "$saved_containerd_conf_file" - sudo systemctl start containerd - waitForProcess 30 5 "sudo crictl info >/dev/null" + restart_containerd # Ensure the cc CRI handler is set. local cri_handler=$(sudo crictl info | \ @@ -223,11 +234,6 @@ configure_cc_containerd() { sudo tee -a "$containerd_conf_file" fi - sudo systemctl restart containerd - if ! waitForProcess 30 5 "sudo crictl info >/dev/null"; then - die "containerd seems not operational after reconfigured" - fi - sudo iptables -w -P FORWARD ACCEPT } # @@ -445,3 +451,91 @@ EOF EOF fi } + +############################################################################### + +# remote-snapshotter + +EXPORT_MODE=${EXPORT_MODE:-"image_guest_pull"} + +configure_remote_snapshotter() { + case "${SNAPSHOTTER}" in + "nydus") + configure_nydus_snapshotter + ;; + *) ;; + + esac +} + +is_containerd_support_per_runtime_snapshotter() { + containerd_version=$(containerd --version | awk '{print $3}') + required_version="v1.7.0" + printf '%s\n' ${required_version} ${containerd_version} | sort --check=quiet -V +} + +set_vanilla_containerd() { + sudo systemctl stop containerd + sleep 5 + sudo mv /usr/local/bin/containerd /usr/local/bin/containerd-coco + sudo cp /usr/local/bin/containerd-vanilla /usr/local/bin/containerd + echo "vanilla containerd version: $(containerd --version | awk '{print $3}')" + restart_containerd +} + +unset_vanilla_containerd() { + sudo systemctl stop containerd + sleep 5 + sudo rm -f /usr/local/bin/containerd + sudo mv /usr/local/bin/containerd-coco /usr/local/bin/containerd + echo "coco containerd version: $(containerd --version | awk '{print $3}')" + restart_containerd +} + +configure_containerd_for_nydus_snapshotter() { + set_vanilla_containerd + local containerd_config="$1" + snapshotter_socket="/run/containerd-nydus/containerd-nydus-grpc.sock" + proxy_config=" [proxy_plugins.$SNAPSHOTTER]\n type = \"snapshot\"\n address = \"${snapshotter_socket}\"" + + if grep -q "\[proxy_plugins\]" "$containerd_config"; then + sudo sed -i '/\[proxy_plugins\]/a\'"$proxy_config" "$containerd_config" + else + sudo echo -e "[proxy_plugins]" >>"$containerd_config" + sudo echo -e "$proxy_config" >>"$containerd_config" + fi + + sudo sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = false/g' "$containerd_config" + sudo sed -i 's/snapshotter = .*/snapshotter = "nydus"/g' "$containerd_config" +} + +kill_nydus_snapshotter_process() { + echo "Kill nydus snapshotter" + bin="containerd-nydus-grpc" + sudo kill -9 $(pidof $bin) || true + sudo rm -rf "/var/lib/containerd-nydus" || true +} + +remove_test_image() { + local test_image="$1" + crictl rmi "$1" + pause_name=$(crictl images -o json | jq -r '.images[].repoTags[] | select(. | contains("pause"))') + crictl rmi "$pause_name" +} + +restart_nydus_snapshotter() { + kill_nydus_snapshotter_process || true + echo "Restart nydus snapshotter" + sudo "$NYDUS_SNAPSHOTTER_BINARY" --config "$NYDUS_SNAPSHOTTER_CONFIG" >/dev/stdout 2>&1 & +} + +configure_nydus_snapshotter() { + echo "Configure nydus snapshotter" + if [ "$EXPORT_MODE" == "image_guest_pull" ]; then + NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_GUEST_CONFIG" + else + NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_TARFS_CONFIG" + sudo sed -i "s/export_mode = .*/export_mode = \"$EXPORT_MODE\"/" "$NYDUS_SNAPSHOTTER_CONFIG" + fi + restart_nydus_snapshotter +} \ No newline at end of file diff --git a/integration/containerd/confidential/tests_common.sh b/integration/containerd/confidential/tests_common.sh index c44f08bf6..3d3c5362f 100644 --- a/integration/containerd/confidential/tests_common.sh +++ b/integration/containerd/confidential/tests_common.sh @@ -26,7 +26,7 @@ setup_common() { echo "Prepare containerd for Confidential Container" SAVED_CONTAINERD_CONF_FILE="/etc/containerd/config.toml.$$" configure_cc_containerd "$SAVED_CONTAINERD_CONF_FILE" - + restart_containerd # Note: ensure that intructions changing the kernel parameters are # executed *after* saving the original list. saved_kernel_params=$(get_kernel_params) diff --git a/integration/kubernetes/confidential/agent_image.bats b/integration/kubernetes/confidential/agent_image.bats index de5c815e8..b0bf11320 100644 --- a/integration/kubernetes/confidential/agent_image.bats +++ b/integration/kubernetes/confidential/agent_image.bats @@ -31,7 +31,9 @@ RUNTIMECLASS="${RUNTIMECLASS:-kata}" test_tag="[cc][agent][kubernetes][containerd]" setup() { - setup_common + setup_containerd + restart_containerd + reconfigure_kata } @test "$test_tag Test can launch pod with measured boot enabled" { diff --git a/integration/kubernetes/confidential/agent_image_encrypted.bats b/integration/kubernetes/confidential/agent_image_encrypted.bats index b8d842ed9..febe2b1ca 100644 --- a/integration/kubernetes/confidential/agent_image_encrypted.bats +++ b/integration/kubernetes/confidential/agent_image_encrypted.bats @@ -4,9 +4,8 @@ # SPDX-License-Identifier: Apache-2.0 # -load "${BATS_TEST_DIRNAME}/lib.sh" -load "${BATS_TEST_DIRNAME}/../../confidential/lib.sh" load "${BATS_TEST_DIRNAME}/../../../lib/common.bash" +load "${BATS_TEST_DIRNAME}/tests_common.sh" # Allow to configure the runtimeClassName on pod configuration. RUNTIMECLASS="${RUNTIMECLASS:-kata}" @@ -25,13 +24,8 @@ setup() { SAVED_CONTAINERD_CONF_FILE="/etc/containerd/config.toml.$$" configure_cc_containerd "$SAVED_CONTAINERD_CONF_FILE" - echo "Reconfigure Kata Containers" - switch_image_service_offload on - clear_kernel_params - add_kernel_params "${original_kernel_params}" - - setup_proxy - switch_measured_rootfs_verity_scheme none + restart_containerd + reconfigure_kata } @test "$test_tag Test can pull an encrypted image inside the guest with decryption key" { diff --git a/integration/kubernetes/confidential/fixtures/pod-config.yaml.in b/integration/kubernetes/confidential/fixtures/pod-config.yaml.in index 3c8e9d3c1..1d0b7def0 100644 --- a/integration/kubernetes/confidential/fixtures/pod-config.yaml.in +++ b/integration/kubernetes/confidential/fixtures/pod-config.yaml.in @@ -5,7 +5,7 @@ apiVersion: v1 kind: Pod metadata: - name: busybox-cc + name: busybox-cc$INDEX spec: runtimeClassName: $RUNTIMECLASS containers: diff --git a/integration/kubernetes/confidential/image_pulling_with_snapshotter.bats b/integration/kubernetes/confidential/image_pulling_with_snapshotter.bats new file mode 100644 index 000000000..8c68afdef --- /dev/null +++ b/integration/kubernetes/confidential/image_pulling_with_snapshotter.bats @@ -0,0 +1,71 @@ +#!/usr/bin/env bats +# Copyright (c) 2023 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +load "${BATS_TEST_DIRNAME}/tests_common.sh" + +tag_suffix="" +if [ "$(uname -m)" != "x86_64" ]; then + tag_suffix="-$(uname -m)" +fi + +# Images used on the tests. + +image_unsigned_protected="quay.io/kata-containers/confidential-containers:unsigned${tag_suffix}" + +original_kernel_params=$(get_kernel_params) +# Allow to configure the runtimeClassName on pod configuration. +RUNTIMECLASS="${RUNTIMECLASS:-kata}" +test_tag="[cc][agent][kubernetes][containerd]" + +setup() { + remove_test_image "$image_unsigned_protected" || true + setup_containerd + restart_containerd + configure_containerd_for_nydus_snapshotter "/etc/containerd/config.toml" + reconfigure_kata + switch_image_service_offload off +} + +@test "$test_tag Test can pull an image as a raw block disk image to guest with dm-verity enabled" { + if [ "$(uname -m)" = "s390x" ]; then + skip "test for s390x as nydus-image doesn't currently support this platform" + fi + if [ "$SNAPSHOTTER" = "nydus" ]; then + EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" configure_remote_snapshotter + pod_config="$(new_pod_config "$image_unsigned_protected")" + echo $pod_config + create_test_pod + fi +} + +@test "$test_tag Test can pull an image as a raw block disk image to guest without dm-verity" { + if [ "$(uname -m)" = "s390x" ]; then + skip "test for s390x as nydus-image doesn't currently support this platform" + fi + if [ "$SNAPSHOTTER" = "nydus" ]; then + EXPORT_MODE="image_block" configure_remote_snapshotter + pod_config="$(new_pod_config "$image_unsigned_protected")" + echo $pod_config + create_test_pod + fi +} + +@test "$test_tag Test can pull an image inside the guest with remote-snapshotter" { + switch_image_service_offload on + if [ "$SNAPSHOTTER" = "nydus" ]; then + EXPORT_MODE="image_guest_pull" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter + pod_config="$(new_pod_config "$image_unsigned_protected")" + echo $pod_config + create_test_pod + fi +} + +teardown() { + teardown_common + remove_test_image "$image_unsigned_protected" || true + kill_nydus_snapshotter_process + unset_vanilla_containerd +} diff --git a/integration/kubernetes/confidential/lib.sh b/integration/kubernetes/confidential/lib.sh index 9e9d904cc..d85f0ce00 100755 --- a/integration/kubernetes/confidential/lib.sh +++ b/integration/kubernetes/confidential/lib.sh @@ -46,7 +46,8 @@ kubernetes_create_cc_pod() { fi kubectl apply -f ${config_file} - if ! pod_name=$(kubectl get pods -o jsonpath='{.items..metadata.name}'); then + pod_name=$(${GOPATH}/bin/yq r ${config_file} 'metadata.name') + if ! kubectl get pod "$pod_name" &> /dev/null; then echo "Failed to create the pod" return 1 fi diff --git a/integration/kubernetes/confidential/sealed_secret.bats b/integration/kubernetes/confidential/sealed_secret.bats index 282005547..49995dc62 100644 --- a/integration/kubernetes/confidential/sealed_secret.bats +++ b/integration/kubernetes/confidential/sealed_secret.bats @@ -7,6 +7,7 @@ load "${BATS_TEST_DIRNAME}/lib.sh" load "${BATS_TEST_DIRNAME}/../../confidential/lib.sh" load "${BATS_TEST_DIRNAME}/../../../lib/common.bash" +load "${BATS_TEST_DIRNAME}/tests_common.sh" # Allow to configure the runtimeClassName on pod configuration. RUNTIMECLASS="${RUNTIMECLASS:-kata}" @@ -22,13 +23,8 @@ setup() { SAVED_CONTAINERD_CONF_FILE="/etc/containerd/config.toml.$$" configure_cc_containerd "$SAVED_CONTAINERD_CONF_FILE" - echo "Reconfigure Kata Containers" - switch_image_service_offload on - clear_kernel_params - add_kernel_params "${original_kernel_params}" - - setup_proxy - switch_measured_rootfs_verity_scheme none + restart_containerd + reconfigure_kata kubectl delete secret sealed-secret --ignore-not-found # Sealed secret format is defined at: https://github.com/confidential-containers/guest-components/blob/main/confidential-data-hub/docs/SEALED_SECRET.md#vault diff --git a/integration/kubernetes/confidential/sev.bats b/integration/kubernetes/confidential/sev.bats index 9f515a8bb..0f587cf0c 100644 --- a/integration/kubernetes/confidential/sev.bats +++ b/integration/kubernetes/confidential/sev.bats @@ -25,6 +25,7 @@ load "${BATS_TEST_DIRNAME}/../../confidential/lib.sh" load "${TESTS_REPO_DIR}/lib/common.bash" load "${TESTS_REPO_DIR}/integration/kubernetes/lib.sh" load "${TESTS_REPO_DIR}/integration/kubernetes/confidential/lib.sh" +load "${BATS_TEST_DIRNAME}/tests_common.sh" # Delete all test services k8s_delete_all() { @@ -55,7 +56,8 @@ setup_file() { # Configure CoCo settings in containerd config local saved_containerd_conf_file="/etc/containerd/config.toml.$$" configure_cc_containerd "${saved_containerd_conf_file}" - + restart_containerd + # KBS setup and run echo "Setting up simple-kbs..." simple_kbs_run diff --git a/integration/kubernetes/confidential/snp.bats b/integration/kubernetes/confidential/snp.bats index 3f03ebda2..3b7fd586b 100644 --- a/integration/kubernetes/confidential/snp.bats +++ b/integration/kubernetes/confidential/snp.bats @@ -21,6 +21,7 @@ export SSH_KEY_FILE load "${BATS_TEST_DIRNAME}/../../confidential/lib.sh" load "${TESTS_REPO_DIR}/lib/common.bash" load "${TESTS_REPO_DIR}/integration/kubernetes/lib.sh" +load "${BATS_TEST_DIRNAME}/tests_common.sh" # Delete all test services k8s_delete_all() { @@ -38,6 +39,7 @@ setup_file() { # Configure CoCo settings in containerd config local saved_containerd_conf_file="/etc/containerd/config.toml.$$" configure_cc_containerd "${saved_containerd_conf_file}" + restart_containerd # Pull unencrypted image and retrieve ssh keys echo "Pulling unencrypted image and retrieve ssh key..." diff --git a/integration/kubernetes/confidential/tests_common.sh b/integration/kubernetes/confidential/tests_common.sh index 177ed450e..68fb8043b 100644 --- a/integration/kubernetes/confidential/tests_common.sh +++ b/integration/kubernetes/confidential/tests_common.sh @@ -16,7 +16,6 @@ original_kernel_params=$(get_kernel_params) # Global variables exported: # $test_start_date - test start time. # $pod_config - path to default pod configuration file. -# $original_kernel_params - saved the original list of kernel parameters. # setup_common() { test_start_date=$(date +"%Y-%m-%d %H:%M:%S") @@ -25,16 +24,28 @@ setup_common() { pod_id="" kubernetes_delete_all_cc_pods_if_any_exists || true +} + +# Setup containerd for tests. +# +setup_containerd() { + setup_common echo "Prepare containerd for Confidential Container" SAVED_CONTAINERD_CONF_FILE="/etc/containerd/config.toml.$$" configure_cc_containerd "$SAVED_CONTAINERD_CONF_FILE" +} +# Reconfigure Kata for tests. +# +# Global variables exported: +# $original_kernel_params - saved the original list of kernel parameters. +# +reconfigure_kata() { echo "Reconfigure Kata Containers" switch_image_service_offload on clear_kernel_params add_kernel_params "${original_kernel_params}" - setup_proxy switch_measured_rootfs_verity_scheme none } @@ -58,6 +69,13 @@ teardown_common() { add_kernel_params "${original_kernel_params}" switch_image_service_offload off disable_full_debug + # Restore containerd to pre-test state. + if [ -f "$SAVED_CONTAINERD_CONF_FILE" ]; then + systemctl stop containerd || true + sleep 5 + mv -f "$SAVED_CONTAINERD_CONF_FILE" "/etc/containerd/config.toml" + systemctl start containerd || true + fi } @@ -76,7 +94,6 @@ create_test_pod() { enable_full_debug enable_agent_console fi - echo "Create the test sandbox" echo "Pod config is: $pod_config" kubernetes_create_cc_pod $pod_config @@ -97,8 +114,9 @@ create_test_pod() { new_pod_config() { local base_config="${FIXTURES_DIR}/pod-config.yaml.in" local image="$1" + local index="${2:-}" local new_config=$(mktemp "${BATS_FILE_TMPDIR}/$(basename ${base_config}).XXX") - IMAGE="$image" RUNTIMECLASS="$RUNTIMECLASS" envsubst < "$base_config" > "$new_config" + IMAGE="$image" RUNTIMECLASS="$RUNTIMECLASS" INDEX="$index" envsubst < "$base_config" > "$new_config" echo "$new_config" }