From f5354b4a4f4e80daac877d8edba16df172ea61b4 Mon Sep 17 00:00:00 2001
From: ChengyuZhu6 <chengyu.zhu@intel.com>
Date: Wed, 6 Sep 2023 11:14:23 +0800
Subject: [PATCH] ci: Add test cases for CoCo image pulling without forked
 containerd

Additional tests are necessary to verify new feature that pulling image without forked containerd in CoCo.

Fixes #5763

Depends:
https://github.com/kata-containers/kata-containers/pull/7688
https://github.com/kata-containers/kata-containers/pull/7676

Signed-off-by: ChengyuZhu6 <chengyu.zhu@intel.com>
---
 integration/confidential/lib.sh               | 300 +++++++++++-------
 .../fixtures/cri-pod-config.yaml.in           |   5 +
 .../confidential/fixtures/pod-config.yaml.in  |   2 +-
 .../image_pulling_with_snapshotter.bats       |  89 ++++++
 4 files changed, 279 insertions(+), 117 deletions(-)
 create mode 100644 integration/kubernetes/confidential/fixtures/cri-pod-config.yaml.in
 create mode 100644 integration/kubernetes/confidential/image_pulling_with_snapshotter.bats

diff --git a/integration/confidential/lib.sh b/integration/confidential/lib.sh
index 9a085c638..61e4d44ff 100644
--- a/integration/confidential/lib.sh
+++ b/integration/confidential/lib.sh
@@ -12,7 +12,11 @@ source "${BATS_TEST_DIRNAME}/../../../lib/common.bash"
 source "${BATS_TEST_DIRNAME}/../../../.ci/lib.sh"
 FIXTURES_DIR="${BATS_TEST_DIRNAME}/fixtures"
 SHARED_FIXTURES_DIR="${BATS_TEST_DIRNAME}/../../confidential/fixtures"
-
+CONTAINERD_CONFIG="/etc/containerd/config.toml"
+NYDUS_SNAPSHOTTER_BINARY="/home/zcy/workspace/image_sharing/bin/containerd-nydus-grpc"
+NYDUS_SNAPSHOTTER_CONFIG="/etc/nydus/config-tarfs.toml"
+NYDUS_SNAPSHOTTER_TARFS_CONFIG="/etc/nydus/config-tarfs.toml"
+NYDUS_SNAPSHOTTER_GUEST_CONFIG="/etc/nydus/config-guest.toml"
 # Toggle between true and false the service_offload configuration of
 # the Kata agent.
 #
@@ -29,18 +33,18 @@ switch_image_service_offload() {
 	load_runtime_config_path
 
 	case "$1" in
-		"on")
-			sudo sed -i -e 's/^\(service_offload\).*=.*$/\1 = true/g' \
-				"$RUNTIME_CONFIG_PATH"
-			;;
-		"off")
-			sudo sed -i -e 's/^\(service_offload\).*=.*$/\1 = false/g' \
-				"$RUNTIME_CONFIG_PATH"
-
-			;;
-		*)
-			die "Unknown option '$1'"
-			;;
+	"on")
+		sed -i -e 's/^\(service_offload\).*=.*$/\1 = true/g' \
+			"$RUNTIME_CONFIG_PATH"
+		;;
+	"off")
+		sed -i -e 's/^\(service_offload\).*=.*$/\1 = false/g' \
+			"$RUNTIME_CONFIG_PATH"
+
+		;;
+	*)
+		die "Unknown option '$1'"
+		;;
 	esac
 }
 
@@ -66,13 +70,13 @@ switch_measured_rootfs_verity_scheme() {
 	load_runtime_config_path
 
 	case "$1" in
-		"dm-verity"|"none")
-			sudo sed -i -e 's/scheme=.* cc_rootfs/scheme='"$1"' cc_rootfs/g' \
-				"$RUNTIME_CONFIG_PATH"
-			;;
-		*)
-			die "Unknown option '$1'"
-			;;
+	"dm-verity" | "none")
+		sed -i -e 's/scheme=.* cc_rootfs/scheme='"$1"' cc_rootfs/g' \
+			"$RUNTIME_CONFIG_PATH"
+		;;
+	*)
+		die "Unknown option '$1'"
+		;;
 	esac
 }
 
@@ -90,7 +94,7 @@ add_kernel_params() {
 	local params="$@"
 	load_runtime_config_path
 
-	sudo sed -i -e 's#^\(kernel_params\) = "\(.*\)"#\1 = "\2 '"$params"'"#g' \
+	sed -i -e 's#^\(kernel_params\) = "\(.*\)"#\1 = "\2 '"$params"'"#g' \
 		"$RUNTIME_CONFIG_PATH"
 
 	if [ "${TEE_TYPE}" = "se" ]; then
@@ -110,8 +114,8 @@ add_kernel_params() {
 get_kernel_params() {
 	load_runtime_config_path
 
-        local kernel_params=$(sed -n -e 's#^kernel_params = "\(.*\)"#\1#gp' \
-                "$RUNTIME_CONFIG_PATH")
+	local kernel_params=$(sed -n -e 's#^kernel_params = "\(.*\)"#\1#gp' \
+		"$RUNTIME_CONFIG_PATH")
 	echo "$kernel_params"
 }
 
@@ -125,7 +129,7 @@ get_kernel_params() {
 clear_kernel_params() {
 	load_runtime_config_path
 
-	sudo sed -i -e 's#^\(kernel_params\) = "\(.*\)"#\1 = ""#g' \
+	sed -i -e 's#^\(kernel_params\) = "\(.*\)"#\1 = ""#g' \
 		"$RUNTIME_CONFIG_PATH"
 }
 
@@ -143,7 +147,7 @@ remove_kernel_param() {
 	local param_name="${1}"
 	load_runtime_config_path
 
-	sudo sed -i "/kernel_params = /s/$param_name=[^[:space:]\"]*//g" \
+	sed -i "/kernel_params = /s/$param_name=[^[:space:]\"]*//g" \
 		"$RUNTIME_CONFIG_PATH"
 }
 
@@ -157,7 +161,7 @@ remove_kernel_param() {
 enable_agent_console() {
 	load_runtime_config_path
 
-	sudo sed -i -e 's/^# *\(debug_console_enabled\).*=.*$/\1 = true/g' \
+	sed -i -e 's/^# *\(debug_console_enabled\).*=.*$/\1 = true/g' \
 		"$RUNTIME_CONFIG_PATH"
 }
 
@@ -166,7 +170,7 @@ enable_full_debug() {
 	load_runtime_config_path
 
 	# Toggle all the debug flags on in kata's configuration.toml to enable full logging.
-	sudo sed -i -e 's/^# *\(enable_debug\).*=.*$/\1 = true/g' "$RUNTIME_CONFIG_PATH"
+	sed -i -e 's/^# *\(enable_debug\).*=.*$/\1 = true/g' "$RUNTIME_CONFIG_PATH"
 
 	# Also pass the initcall debug flags via Kernel parameters.
 	add_kernel_params "agent.log=debug" "initcall_debug"
@@ -177,7 +181,7 @@ disable_full_debug() {
 	load_runtime_config_path
 
 	# Toggle all the debug flags off in kata's configuration.toml to enable full logging.
-	sudo sed -i -e 's/^# *\(enable_debug\).*=.*$/\1 = false/g' "$RUNTIME_CONFIG_PATH"
+	sed -i -e 's/^# *\(enable_debug\).*=.*$/\1 = false/g' "$RUNTIME_CONFIG_PATH"
 }
 
 # Configure containerd for confidential containers. Among other things, it ensures
@@ -201,33 +205,33 @@ configure_cc_containerd() {
 
 	# Even if we are not saving the original file it is a good idea to
 	# restart containerd because it might be in an inconsistent state here.
-	sudo systemctl stop containerd
+	systemctl stop containerd
 	sleep 5
-	[ -n "$saved_containerd_conf_file" ] && \
-		sudo cp -f "$containerd_conf_file" "$saved_containerd_conf_file"
-	sudo systemctl start containerd
-	waitForProcess 30 5 "sudo crictl info >/dev/null"
+	[ -n "$saved_containerd_conf_file" ] &&
+		cp -f "$containerd_conf_file" "$saved_containerd_conf_file"
+	systemctl start containerd
+	waitForProcess 30 5 " crictl info >/dev/null"
 
 	# Ensure the cc CRI handler is set.
-	local cri_handler=$(sudo crictl info | \
+	local cri_handler=$(crictl info |
 		jq '.config.containerd.runtimes.kata.cri_handler')
 	if [[ ! "$cri_handler" =~ cc ]]; then
-		sudo sed -i 's/\([[:blank:]]*\)\(runtime_type = "io.containerd.kata.v2"\)/\1\2\n\1cri_handler = "cc"/' \
+		sed -i 's/\([[:blank:]]*\)\(runtime_type = "io.containerd.kata.v2"\)/\1\2\n\1cri_handler = "cc"/' \
 			"$containerd_conf_file"
 	fi
 
-	if [ "$(sudo crictl info | jq -r '.config.cni.confDir')" = "null" ]; then
+	if [ "$(crictl info | jq -r '.config.cni.confDir')" = "null" ]; then
 		echo "    [plugins.cri.cni]
 		  # conf_dir is the directory in which the admin places a CNI conf.
-		  conf_dir = \"/etc/cni/net.d\"" | \
-			  sudo tee -a "$containerd_conf_file"
+		  conf_dir = \"/etc/cni/net.d\"" |
+			tee -a "$containerd_conf_file"
 	fi
 
-	sudo systemctl restart containerd
-	if ! waitForProcess 30 5 "sudo crictl info >/dev/null"; then
+	systemctl restart containerd
+	if ! waitForProcess 30 5 " crictl info >/dev/null"; then
 		die "containerd seems not operational after reconfigured"
 	fi
-	sudo iptables -P FORWARD ACCEPT
+	iptables -P FORWARD ACCEPT
 }
 
 #
@@ -247,12 +251,12 @@ setup_common_signature_files_in_guest() {
 	signatures_dir="${SHARED_FIXTURES_DIR}/quay_verification/$(uname -m)/signatures"
 
 	if [ ! -d "${signatures_dir}" ]; then
-		sudo mkdir "${signatures_dir}"
+		mkdir "${signatures_dir}"
 	fi
 
-	sudo tar -zvxf "${SHARED_FIXTURES_DIR}/quay_verification/$(uname -m)/signatures.tar" -C "${signatures_dir}"
+	tar -zvxf "${SHARED_FIXTURES_DIR}/quay_verification/$(uname -m)/signatures.tar" -C "${signatures_dir}"
 
-	sudo cp -ar ${SHARED_FIXTURES_DIR}/quay_verification/$(uname -m)/* ${SHARED_FIXTURES_DIR}/quay_verification
+	cp -ar ${SHARED_FIXTURES_DIR}/quay_verification/$(uname -m)/* ${SHARED_FIXTURES_DIR}/quay_verification
 	cp_to_guest_img "${rootfs_directory}" "${SHARED_FIXTURES_DIR}/quay_verification"
 }
 
@@ -286,38 +290,36 @@ setup_cosign_signatures_files() {
 
 	# Set-up required files in guest image
 	case "${AA_KBC:-}" in
-		"offline_fs_kbc")
-			add_kernel_params "agent.aa_kbc_params=offline_fs_kbc::null"
-			cp_to_guest_img "etc" "${SHARED_FIXTURES_DIR}/cosign/offline-fs-kbc/$(uname -m)/aa-offline_fs_kbc-resources.json"
-			;;
-		"cc_kbc")
-			# CC KBC is specified as: cc_kbc::host_ip:port, and 60000 is the default port used
-			# by the service, as well as the one configured in the Kata Containers rootfs.
-
-			CC_KBS_IP=${CC_KBS_IP:-"$(hostname -I | awk '{print $1}')"}
-			CC_KBS_PORT=${CC_KBS_PORT:-"60000"}
-			add_kernel_params "agent.aa_kbc_params=cc_kbc::http://${CC_KBS_IP}:${CC_KBS_PORT}/"
-			;;
-		*)
-			;;
+	"offline_fs_kbc")
+		add_kernel_params "agent.aa_kbc_params=offline_fs_kbc::null"
+		cp_to_guest_img "etc" "${SHARED_FIXTURES_DIR}/cosign/offline-fs-kbc/$(uname -m)/aa-offline_fs_kbc-resources.json"
+		;;
+	"cc_kbc")
+		# CC KBC is specified as: cc_kbc::host_ip:port, and 60000 is the default port used
+		# by the service, as well as the one configured in the Kata Containers rootfs.
+
+		CC_KBS_IP=${CC_KBS_IP:-"$(hostname -I | awk '{print $1}')"}
+		CC_KBS_PORT=${CC_KBS_PORT:-"60000"}
+		add_kernel_params "agent.aa_kbc_params=cc_kbc::http://${CC_KBS_IP}:${CC_KBS_PORT}/"
+		;;
+	*) ;;
 	esac
 }
 
 setup_signature_files() {
 	case "${AA_KBC:-}" in
-		"offline_fs_kbc")
-			setup_offline_fs_kbc_signature_files_in_guest
-			;;
-		"cc_kbc")
-			setup_cc_kbc_signature_files_in_guest
-			;;
-		*)
-			;;
+	"offline_fs_kbc")
+		setup_offline_fs_kbc_signature_files_in_guest
+		;;
+	"cc_kbc")
+		setup_cc_kbc_signature_files_in_guest
+		;;
+	*) ;;
 	esac
 }
 
 # In case the tests run behind a firewall where images needed to be fetched
-# through a proxy. 
+# through a proxy.
 # Note: With measured rootfs enabled, we can not set proxy through
 # agent config file.
 setup_proxy() {
@@ -349,8 +351,8 @@ setup_credentials_files() {
 
 	dest_dir="$(mktemp -t -d offline-fs-kbc-XXXXXXXX)"
 	dest_file=${dest_dir}/aa-offline_fs_kbc-resources.json
-	auth_json=$(REGISTRY=$1 CREDENTIALS="${REGISTRY_CREDENTIAL_ENCODED}" envsubst < "${SHARED_FIXTURES_DIR}/offline-fs-kbc/auth.json.in" | base64 -w 0)
-	CREDENTIAL="${auth_json}" envsubst < "${SHARED_FIXTURES_DIR}/offline-fs-kbc/aa-offline_fs_kbc-resources.json.in" > "${dest_file}"
+	auth_json=$(REGISTRY=$1 CREDENTIALS="${REGISTRY_CREDENTIAL_ENCODED}" envsubst <"${SHARED_FIXTURES_DIR}/offline-fs-kbc/auth.json.in" | base64 -w 0)
+	CREDENTIAL="${auth_json}" envsubst <"${SHARED_FIXTURES_DIR}/offline-fs-kbc/aa-offline_fs_kbc-resources.json.in" >"${dest_file}"
 	cp_to_guest_img "etc" "${dest_file}"
 }
 
@@ -366,48 +368,48 @@ KBS_DB="${KBS_DB:-simple_kbs}"
 
 # Run the simple-kbs
 simple_kbs_run() {
-  # Retrieve simple-kbs repo and tag from versions.yaml
-  local simple_kbs_url=$(get_test_version "externals.simple-kbs.url")
-  local simple_kbs_tag=$(get_test_version "externals.simple-kbs.tag")
-
-  # Cleanup and create installation directory
-  esudo rm -rf "${SIMPLE_KBS_DIR}" 
-  mkdir -p "${SIMPLE_KBS_DIR}"
-  pushd "${SIMPLE_KBS_DIR}"
-
-  # Clone and run
-  git clone "${simple_kbs_url}" --branch main
-  pushd simple-kbs
-
-  # Checkout, build and start
-  git checkout -b "branch_${simple_kbs_tag}" "${simple_kbs_tag}"
-  esudo docker-compose build
-  esudo docker-compose up -d
-
-  # Wait for simple-kbs to start
-  waitForProcess 15 1 "esudo docker-compose top | grep -q simple-kbs"
-  popd
-  
-  # Get simple-kbs database container ip
-  local kbs_db_host=$(simple_kbs_get_db_ip)
-
-  # Confirm connection to the database is possible
-  waitForProcess 5 1 "mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} -e '\q'"
-  popd
+	# Retrieve simple-kbs repo and tag from versions.yaml
+	local simple_kbs_url=$(get_test_version "externals.simple-kbs.url")
+	local simple_kbs_tag=$(get_test_version "externals.simple-kbs.tag")
+
+	# Cleanup and create installation directory
+	e rm -rf "${SIMPLE_KBS_DIR}"
+	mkdir -p "${SIMPLE_KBS_DIR}"
+	pushd "${SIMPLE_KBS_DIR}"
+
+	# Clone and run
+	git clone "${simple_kbs_url}" --branch main
+	pushd simple-kbs
+
+	# Checkout, build and start
+	git checkout -b "branch_${simple_kbs_tag}" "${simple_kbs_tag}"
+	e docker-compose build
+	e docker-compose up -d
+
+	# Wait for simple-kbs to start
+	waitForProcess 15 1 "e docker-compose top | grep -q simple-kbs"
+	popd
+
+	# Get simple-kbs database container ip
+	local kbs_db_host=$(simple_kbs_get_db_ip)
+
+	# Confirm connection to the database is possible
+	waitForProcess 5 1 "mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} -e '\q'"
+	popd
 }
 
 # Stop simple-kbs and database containers
 simple_kbs_stop() {
-  (cd ${SIMPLE_KBS_DIR}/simple-kbs && esudo docker-compose down 2>/dev/null)
+	(cd ${SIMPLE_KBS_DIR}/simple-kbs && e docker-compose down 2>/dev/null)
 }
 
 # Delete all test inserted data in the simple-kbs
 simple_kbs_delete_data() {
-  # Get simple-kbs database container ip
-  local kbs_db_host=$(simple_kbs_get_db_ip)
+	# Get simple-kbs database container ip
+	local kbs_db_host=$(simple_kbs_get_db_ip)
 
-  # Delete all data with 'id = 10'
-  mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
+	# Delete all data with 'id = 10'
+	mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
     DELETE FROM secrets WHERE id = 10;
     DELETE FROM policy WHERE id = 10;
 EOF
@@ -415,28 +417,94 @@ EOF
 
 # Get the ip of the simple-kbs database docker container
 simple_kbs_get_db_ip() {
-  esudo docker network inspect simple-kbs_default \
-    | jq -r '.[].Containers[] | select(.Name | test("simple-kbs[_-]db.*")).IPv4Address' \
-    | sed "s|/.*$||g"
+	e docker network inspect simple-kbs_default |
+		jq -r '.[].Containers[] | select(.Name | test("simple-kbs[_-]db.*")).IPv4Address' |
+		sed "s|/.*$||g"
 }
 
 # Add key and keyset to database
 # If measurement is provided, add policy with measurement to database
 simple_kbs_add_key_to_db() {
-  local encryption_key="${1}"
-  local measurement="${2}"
-  
-  # Get simple-kbs database container ip
-  local kbs_db_host=$(simple_kbs_get_db_ip)
-
-  if [ -n "${measurement}" ]; then
-    mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
+	local encryption_key="${1}"
+	local measurement="${2}"
+
+	# Get simple-kbs database container ip
+	local kbs_db_host=$(simple_kbs_get_db_ip)
+
+	if [ -n "${measurement}" ]; then
+		mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
       INSERT INTO secrets VALUES (10, 'default/key/ssh-demo', '${encryption_key}', 10);
       INSERT INTO policy VALUES (10, '["${measurement}"]', '[]', 0, 0, '[]', now(), NULL, 1);
 EOF
-  else
-    mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
+	else
+		mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
       INSERT INTO secrets VALUES (10, 'default/key/ssh-demo', '${encryption_key}', NULL);
 EOF
-  fi
+	fi
+}
+
+###############################################################################
+
+# remote-snapshotter
+
+configure_remote_snapshotter() {
+	case "${SNAPSHOTTER:-}" in
+	"nydus")
+		configure_nydus_snapshotter
+		;;
+	*) ;;
+
+	esac
+}
+check_containerd_version() {
+	containerd_version=$(containerd --version | awk '{print $3}' | sort -V | tail -n 1)
+	if echo $containerd_version | grep -q "^v1.7"; then
+		return 1
+	else
+		return 0
+	fi
+}
+configure_containerd_for_nydus_snapshotter() {
+	sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = false/g' "$CONTAINERD_CONFIG"
+	if check_containerd_version; then
+		sed -i '/\[plugins\."io\.containerd\.grpc\.v1\.cri"\.containerd\.runtimes\.'"$RUNTIMECLASS"'\]/a\  snapshotter = "nydus"\n' "$CONTAINERD_CONFIG"
+	else
+		sed -i 's/snapshotter = .*/snapshotter = "nydus"/g' "$CONTAINERD_CONFIG"
+	fi
+	systemctl restart containerd
+}
+remove_nydus_snapshotter_from_containerd() {
+	sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = true/g' "$CONTAINERD_CONFIG"
+	if check_containerd_version; then
+		sed -i '/\[plugins\."io\.containerd\.grpc\.v1\.cri"\.containerd\.runtimes\.'"$RUNTIMECLASS"'\]/,/\[/{/snapshotter = "nydus"/d;}' "$CONTAINERD_CONFIG"
+	else
+		sed -i 's/snapshotter = .*/snapshotter = "overlayfs"/g' "$CONTAINERD_CONFIG"
+	fi
+	systemctl restart containerd
+
+}
+remove_test_image() {
+	local test_image = "$1"
+	crictl rmi "$1"
+	pause_name=$(crictl images -o json | jq -r '.images[].repoTags[] | select(. | contains("pause"))')
+	crictl rmi "$pause_name"
+}
+
+restart_nydus_snapshotter() {
+	echo "Kill nydus snapshotter"
+	bin="containerd-nydus-grpc"
+	kill -9 $(pidof $bin) || true
+	echo "Restart nydus snapshotter"
+	"$NYDUS_SNAPSHOTTER_BINARY" --config "$NYDUS_SNAPSHOTTER_CONFIG" --nydusd-config /etc/nydus/nydusd.json &
+}
+
+configure_nydus_snapshotter() {
+	echo "Configure nydus snapshotter"
+	if [ "$EXPORT_MODE" == "image_guest_pull" ]; then
+		NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_GUEST_CONFIG"
+	else
+		NYDUS_SNAPSHOTTER_CONFIG="$NYDUS_SNAPSHOTTER_TARFS_CONFIG"
+	fi
+	sed -i "s/export_mode = .*/export_mode = \"$EXPORT_MODE\"/" "$NYDUS_SNAPSHOTTER_CONFIG"
+	restart_nydus_snapshotter
 }
diff --git a/integration/kubernetes/confidential/fixtures/cri-pod-config.yaml.in b/integration/kubernetes/confidential/fixtures/cri-pod-config.yaml.in
new file mode 100644
index 000000000..577cd4ea9
--- /dev/null
+++ b/integration/kubernetes/confidential/fixtures/cri-pod-config.yaml.in
@@ -0,0 +1,5 @@
+metadata:
+  name: nydus-container$INDEX
+image:
+  image: $IMAGE
+log_path: container.1.log
\ No newline at end of file
diff --git a/integration/kubernetes/confidential/fixtures/pod-config.yaml.in b/integration/kubernetes/confidential/fixtures/pod-config.yaml.in
index 3c8e9d3c1..1d0b7def0 100644
--- a/integration/kubernetes/confidential/fixtures/pod-config.yaml.in
+++ b/integration/kubernetes/confidential/fixtures/pod-config.yaml.in
@@ -5,7 +5,7 @@
 apiVersion: v1
 kind: Pod
 metadata:
-  name: busybox-cc
+  name: busybox-cc$INDEX
 spec:
   runtimeClassName: $RUNTIMECLASS
   containers:
diff --git a/integration/kubernetes/confidential/image_pulling_with_snapshotter.bats b/integration/kubernetes/confidential/image_pulling_with_snapshotter.bats
new file mode 100644
index 000000000..580286776
--- /dev/null
+++ b/integration/kubernetes/confidential/image_pulling_with_snapshotter.bats
@@ -0,0 +1,89 @@
+#!/usr/bin/env bats
+# Copyright (c) 2022 IBM Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+load "${BATS_TEST_DIRNAME}/lib.sh"
+load "${BATS_TEST_DIRNAME}/../../confidential/lib.sh"
+
+# Images used on the tests.
+
+image_unsigned_protected="quay.io/kata-containers/confidential-containers:unsigned${tag_suffix}"
+
+# Allow to configure the runtimeClassName on pod configuration.
+RUNTIMECLASS="${RUNTIMECLASS:-kata}"
+test_tag="[cc][agent][kubernetes][containerd]"
+
+# Create the test pod.
+#
+# Note: the global $sandbox_name, $pod_config should be set
+# 	already. It also relies on $CI and $DEBUG exported by CI scripts or
+# 	the developer, to decide how to set debug flags.
+#
+create_test_pod() {
+    local pod_config="$1"
+
+    echo "Create the test sandbox"
+    echo "Pod config is: $pod_config"
+    crictl run --with-pull -r kata-qemu $pod_config nydus-sandbox.yaml
+}
+
+# Create a pod configuration out of a template file.
+#
+# Parameters:
+#	$1 - the container image.
+# Return:
+# 	the path to the configuration file. The caller should not care about
+# 	its removal afterwards as it is created under the bats temporary
+# 	directory.
+#
+# Environment variables:
+#	RUNTIMECLASS: set the runtimeClassName value from $RUNTIMECLASS.
+#
+new_pod_config() {
+    local base_config="${FIXTURES_DIR}/cri-pod-config.yaml.in"
+    local image="$1"
+    local index="$2"
+
+    local new_config=$(mktemp "${BATS_FILE_TMPDIR}/$(basename ${base_config}).XXX")
+    IMAGE="$image" RUNTIMECLASS="$RUNTIMECLASS" INDEX="$2" envsubst <"$base_config" >"$new_config"
+    echo "$new_config"
+}
+
+setup() {
+    start_date=$(date +"%Y-%m-%d %H:%M:%S")
+    configure_containerd_for_nydus_snapshotter
+}
+
+@test "$test_tag Test can pull an image as a raw block disk image to guest with dm-verity enabled" {
+    if [ "$SNAPSHOTTER" = "nydus" ]; then
+        EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter
+        pod_config="$(new_pod_config "$image_unsigned_unprotected")"
+        echo $pod_config
+        create_test_pod "$pod_config"
+        remove_test_image "$image_unsigned_unprotected"
+    fi
+}
+
+@test "$test_tag Test can create two pods with pulling the image only once" {
+    skip
+    if [ "$SNAPSHOTTER" = "nydus" ]; then
+        EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" SNAPSHOTTER="nydus" configure_remote_snapshotter
+
+        pod_config_1="$(new_pod_config "$image_unsigned_unprotected" "1")"
+        echo $pod_config_1
+        create_test_pod $pod_config_1
+        pod_config_2="$(new_pod_config "$image_unsigned_unprotected" "2")"
+        echo $pod_config_2
+        create_test_pod $pod_config_2
+
+        pull_times=$(journalctl -g "PullImage \"$image_unsigned_unprotected\" with snapshotter nydus" | wc -l)
+        [ ${#pull_times[@]} -eq 1 ]
+        remove_test_image "$image_unsigned_unprotected"
+    fi
+}
+
+teardown() {
+    remove_nydus_snapshotter_from_containerd
+}