From 72da7177e6da7c787a27bea6201650617c8381c8 Mon Sep 17 00:00:00 2001 From: ChengyuZhu6 Date: Wed, 6 Sep 2023 11:14:23 +0800 Subject: [PATCH] ci: Add test cases for CoCo image pulling without forked containerd Additional tests are necessary to verify new feature that pulling image without forked containerd in CoCo. Fixes #5763 Depends: https://github.com/kata-containers/kata-containers/pull/7688 https://github.com/kata-containers/kata-containers/pull/7676 Signed-off-by: ChengyuZhu6 --- integration/confidential/lib.sh | 69 ++++++++++++- .../fixtures/cri-pod-config.yaml.in | 5 + .../confidential/fixtures/pod-config.yaml.in | 2 +- .../image_pulling_with_snapshotter.bats | 98 +++++++++++++++++++ 4 files changed, 172 insertions(+), 2 deletions(-) create mode 100644 integration/kubernetes/confidential/fixtures/cri-pod-config.yaml.in create mode 100644 integration/kubernetes/confidential/image_pulling_with_snapshotter.bats diff --git a/integration/confidential/lib.sh b/integration/confidential/lib.sh index 9a085c638..350841c1c 100644 --- a/integration/confidential/lib.sh +++ b/integration/confidential/lib.sh @@ -12,7 +12,11 @@ source "${BATS_TEST_DIRNAME}/../../../lib/common.bash" source "${BATS_TEST_DIRNAME}/../../../.ci/lib.sh" FIXTURES_DIR="${BATS_TEST_DIRNAME}/fixtures" SHARED_FIXTURES_DIR="${BATS_TEST_DIRNAME}/../../confidential/fixtures" - +CONTAINERD_CONFIG="/etc/containerd/config.toml" +NYDUS_SNAPSHOTTER_BINARY="/home/zcy/workspace/image_sharing/bin/containerd-nydus-grpc" +NYDUS_SNAPSHOTTER_CONFIG="/etc/nydus/config-tarfs.toml" +NYDUS_SNAPSHOTTER_TARFS_CONFIG="/etc/nydus/config-tarfs.toml" +NYDUS_SNAPSHOTTER_GUEST_CONFIG="/etc/nydus/config-guest.toml" # Toggle between true and false the service_offload configuration of # the Kata agent. # @@ -440,3 +444,66 @@ EOF EOF fi } + +############################################################################### + +# remote-snapshotter + +configure_remote_snapshotter() { + case "${SNAPSHOTTER:-}" in + "nydus") + configure_nydus_snapshotter + ;; + *) ;; + + esac +} +check_containerd_version() { + containerd_version=$(containerd --version | awk '{print $3}' | sort -V | tail -n 1) + if echo $containerd_version | grep -q "^v1.7"; then + return 1 + else + return 0 + fi +} +configure_containerd_for_nydus_snapshotter() { + sudo sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = false/g' "$CONTAINERD_CONFIG" + if check_containerd_version; then + sudo sed -i '/\[plugins\."io\.containerd\.grpc\.v1\.cri"\.containerd\.runtimes\.'"$RUNTIMECLASS"'\]/a\ snapshotter = "nydus"\n' "$CONTAINERD_CONFIG" + else + sudo sed -i 's/snapshotter = .*/snapshotter = "nydus"/g' "$CONTAINERD_CONFIG" + fi +} +remove_nydus_snapshotter_from_containerd() { + sudo sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = true/g' "$CONTAINERD_CONFIG" + if check_containerd_version; then + sudo sed -i '/\[plugins\."io\.containerd\.grpc\.v1\.cri"\.containerd\.runtimes\.'"$RUNTIMECLASS"'\]/,/\[/{/snapshotter = "nydus"/d;}' "$CONTAINERD_CONFIG" + else + sudo sed -i 's/snapshotter = .*/snapshotter = "overlayfs"/g' "$CONTAINERD_CONFIG" + fi +} +remove_test_image() { + local test_image = "$1" + sudo crictl rmi "$1" + pause_name=$(crictl images -o json | jq -r '.images[].repoTags[] | select(. | contains("pause"))') + sudo crictl rmi "$pause_name" +} + +restart_nydus_snapshotter() { + echo "Kill nydus snapshotter" + bin="containerd-nydus-grpc" + sudo kill -9 $(pidof $bin) || true + echo "Restart nydus snapshotter" + sudo "$NYDUS_SNAPSHOTTER_BINARY" --config "$NYDUS_SNAPSHOTTER_CONFIG" & +} + +configure_nydus_snapshotter() { + echo "Configure nydus snapshotter" + if [ "$EXPORT_MODE" == "image_guest_pull" ]; then + NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_GUEST_CONFIG" + else + NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_TARFS_CONFIG" + fi + sudo sed -i "s/export_mode = .*/export_mode = \"$EXPORT_MODE\"/" "$NYDUS_SNAPSHOTTER_CONFIG" + restart_nydus_snapshotter +} \ No newline at end of file diff --git a/integration/kubernetes/confidential/fixtures/cri-pod-config.yaml.in b/integration/kubernetes/confidential/fixtures/cri-pod-config.yaml.in new file mode 100644 index 000000000..577cd4ea9 --- /dev/null +++ b/integration/kubernetes/confidential/fixtures/cri-pod-config.yaml.in @@ -0,0 +1,5 @@ +metadata: + name: nydus-container$INDEX +image: + image: $IMAGE +log_path: container.1.log \ No newline at end of file diff --git a/integration/kubernetes/confidential/fixtures/pod-config.yaml.in b/integration/kubernetes/confidential/fixtures/pod-config.yaml.in index 3c8e9d3c1..1d0b7def0 100644 --- a/integration/kubernetes/confidential/fixtures/pod-config.yaml.in +++ b/integration/kubernetes/confidential/fixtures/pod-config.yaml.in @@ -5,7 +5,7 @@ apiVersion: v1 kind: Pod metadata: - name: busybox-cc + name: busybox-cc$INDEX spec: runtimeClassName: $RUNTIMECLASS containers: diff --git a/integration/kubernetes/confidential/image_pulling_with_snapshotter.bats b/integration/kubernetes/confidential/image_pulling_with_snapshotter.bats new file mode 100644 index 000000000..99571b164 --- /dev/null +++ b/integration/kubernetes/confidential/image_pulling_with_snapshotter.bats @@ -0,0 +1,98 @@ +#!/usr/bin/env bats +# Copyright (c) 2022 IBM Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +load "${BATS_TEST_DIRNAME}/lib.sh" +load "${BATS_TEST_DIRNAME}/../../confidential/lib.sh" + +# Images used on the tests. + +image_unsigned_protected="quay.io/kata-containers/confidential-containers:unsigned${tag_suffix}" + +original_kernel_params=$(get_kernel_params) +# Allow to configure the runtimeClassName on pod configuration. +RUNTIMECLASS="${RUNTIMECLASS:-kata}" +test_tag="[cc][agent][kubernetes][containerd]" + +# Create the test pod. +# +# Note: the global $sandbox_name, $pod_config should be set +# already. It also relies on $CI and $DEBUG exported by CI scripts or +# the developer, to decide how to set debug flags. +# +create_test_pod() { + local pod_config="$1" + + echo "Create the test sandbox" + echo "Pod config is: $pod_config" + crictl crictl run --with-pull -r kata-qemu $pod_config nydus-sandbox.yaml +} + +# Create a pod configuration out of a template file. +# +# Parameters: +# $1 - the container image. +# Return: +# the path to the configuration file. The caller should not care about +# its removal afterwards as it is created under the bats temporary +# directory. +# +# Environment variables: +# RUNTIMECLASS: set the runtimeClassName value from $RUNTIMECLASS. +# +new_pod_config() { + local base_config="${FIXTURES_DIR}/cri-pod-config.yaml.in" + local image="$1" + local index="$2" + + local new_config=$(mktemp "${BATS_FILE_TMPDIR}/$(basename ${base_config}).XXX") + IMAGE="$image" RUNTIMECLASS="$RUNTIMECLASS" INDEX="$2" envsubst <"$base_config" >"$new_config" + echo "$new_config" +} + +setup() { + start_date=$(date +"%Y-%m-%d %H:%M:%S") +} + +@test "$test_tag Test can pull an image as a raw block disk image to guest with dm-verity enabled" { + if [ "$SNAPSHOTTER" = "nydus" ]; then + EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter + pod_config="$(new_pod_config "$image_unsigned_unprotected")" + echo $pod_config + create_test_pod "$pod_config" + remove_test_image "$image_unsigned_unprotected" + fi +} + +@test "$test_tag Test can create two pods with pulling the image only once" { + if [ "$SNAPSHOTTER" = "nydus" ]; then + EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter + + pod_config_1="$(new_pod_config "$image_unsigned_unprotected" "1")" + echo $pod_config_1 + create_test_pod $pod_config_1 + pod_config_2="$(new_pod_config "$image_unsigned_unprotected" "2")" + echo $pod_config_2 + create_test_pod $pod_config_2 + + pull_times=$(journalctl -g "PullImage \"$image_unsigned_unprotected\" with snapshotter nydus" | wc -l) + [ ${#pull_times[@]} -eq 1 ] + remove_test_image "$image_unsigned_unprotected" + fi +} + +@test "$test_tag Test can pull an image inside the guest with remote-snapshotter" { + skip + switch_image_service_offload on + if [ "$SNAPSHOTTER" = "nydus" ]; then + EXPORT_MODE="image_guest_pull" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter + create_test_pod + remove_test_image "$image_unsigned_unprotected" + fi +} + +teardown() { + remove_nydus_snapshotter_from_containerd +}