From cbe3231466c21a0c8d540f68fcb656fdcdb35cc0 Mon Sep 17 00:00:00 2001
From: ChengyuZhu6 <chengyu.zhu@intel.com>
Date: Wed, 6 Sep 2023 10:56:25 +0800
Subject: [PATCH] ci: Add test cases for CoCo image pulling without forked
 containerd

Add test cases for CoCo image pulling without forked containerd,
including:
1) sharing image on the host with dm-verity
2) pulling image in the guest

Fixes: #5763

Depends-on:
github.com/kata-containers/kata-containers#7676
github.com/kata-containers/kata-containers#7688

Signed-off-by: ChengyuZhu6 <chengyu.zhu@intel.com>
---
 integration/confidential/lib.sh               | 239 +++++++++++-------
 .../kubernetes/confidential/agent_image.bats  |  25 +-
 2 files changed, 163 insertions(+), 101 deletions(-)

diff --git a/integration/confidential/lib.sh b/integration/confidential/lib.sh
index 9a085c638..3a1ed7f83 100644
--- a/integration/confidential/lib.sh
+++ b/integration/confidential/lib.sh
@@ -12,6 +12,8 @@ source "${BATS_TEST_DIRNAME}/../../../lib/common.bash"
 source "${BATS_TEST_DIRNAME}/../../../.ci/lib.sh"
 FIXTURES_DIR="${BATS_TEST_DIRNAME}/fixtures"
 SHARED_FIXTURES_DIR="${BATS_TEST_DIRNAME}/../../confidential/fixtures"
+NYDUS_SNAPSHOTTER_CONFIG="/opt/confidential-containers/share/remote-snapshotter/config.toml"
+CONTAINERD_CONFIG="/etc/containerd/config.toml"
 
 # Toggle between true and false the service_offload configuration of
 # the Kata agent.
@@ -29,18 +31,18 @@ switch_image_service_offload() {
 	load_runtime_config_path
 
 	case "$1" in
-		"on")
-			sudo sed -i -e 's/^\(service_offload\).*=.*$/\1 = true/g' \
-				"$RUNTIME_CONFIG_PATH"
-			;;
-		"off")
-			sudo sed -i -e 's/^\(service_offload\).*=.*$/\1 = false/g' \
-				"$RUNTIME_CONFIG_PATH"
-
-			;;
-		*)
-			die "Unknown option '$1'"
-			;;
+	"on")
+		sudo sed -i -e 's/^\(service_offload\).*=.*$/\1 = true/g' \
+			"$RUNTIME_CONFIG_PATH"
+		;;
+	"off")
+		sudo sed -i -e 's/^\(service_offload\).*=.*$/\1 = false/g' \
+			"$RUNTIME_CONFIG_PATH"
+
+		;;
+	*)
+		die "Unknown option '$1'"
+		;;
 	esac
 }
 
@@ -66,13 +68,13 @@ switch_measured_rootfs_verity_scheme() {
 	load_runtime_config_path
 
 	case "$1" in
-		"dm-verity"|"none")
-			sudo sed -i -e 's/scheme=.* cc_rootfs/scheme='"$1"' cc_rootfs/g' \
-				"$RUNTIME_CONFIG_PATH"
-			;;
-		*)
-			die "Unknown option '$1'"
-			;;
+	"dm-verity" | "none")
+		sudo sed -i -e 's/scheme=.* cc_rootfs/scheme='"$1"' cc_rootfs/g' \
+			"$RUNTIME_CONFIG_PATH"
+		;;
+	*)
+		die "Unknown option '$1'"
+		;;
 	esac
 }
 
@@ -110,8 +112,8 @@ add_kernel_params() {
 get_kernel_params() {
 	load_runtime_config_path
 
-        local kernel_params=$(sed -n -e 's#^kernel_params = "\(.*\)"#\1#gp' \
-                "$RUNTIME_CONFIG_PATH")
+	local kernel_params=$(sed -n -e 's#^kernel_params = "\(.*\)"#\1#gp' \
+		"$RUNTIME_CONFIG_PATH")
 	echo "$kernel_params"
 }
 
@@ -203,13 +205,13 @@ configure_cc_containerd() {
 	# restart containerd because it might be in an inconsistent state here.
 	sudo systemctl stop containerd
 	sleep 5
-	[ -n "$saved_containerd_conf_file" ] && \
+	[ -n "$saved_containerd_conf_file" ] &&
 		sudo cp -f "$containerd_conf_file" "$saved_containerd_conf_file"
 	sudo systemctl start containerd
 	waitForProcess 30 5 "sudo crictl info >/dev/null"
 
 	# Ensure the cc CRI handler is set.
-	local cri_handler=$(sudo crictl info | \
+	local cri_handler=$(sudo crictl info |
 		jq '.config.containerd.runtimes.kata.cri_handler')
 	if [[ ! "$cri_handler" =~ cc ]]; then
 		sudo sed -i 's/\([[:blank:]]*\)\(runtime_type = "io.containerd.kata.v2"\)/\1\2\n\1cri_handler = "cc"/' \
@@ -219,8 +221,8 @@ configure_cc_containerd() {
 	if [ "$(sudo crictl info | jq -r '.config.cni.confDir')" = "null" ]; then
 		echo "    [plugins.cri.cni]
 		  # conf_dir is the directory in which the admin places a CNI conf.
-		  conf_dir = \"/etc/cni/net.d\"" | \
-			  sudo tee -a "$containerd_conf_file"
+		  conf_dir = \"/etc/cni/net.d\"" |
+			sudo tee -a "$containerd_conf_file"
 	fi
 
 	sudo systemctl restart containerd
@@ -286,38 +288,36 @@ setup_cosign_signatures_files() {
 
 	# Set-up required files in guest image
 	case "${AA_KBC:-}" in
-		"offline_fs_kbc")
-			add_kernel_params "agent.aa_kbc_params=offline_fs_kbc::null"
-			cp_to_guest_img "etc" "${SHARED_FIXTURES_DIR}/cosign/offline-fs-kbc/$(uname -m)/aa-offline_fs_kbc-resources.json"
-			;;
-		"cc_kbc")
-			# CC KBC is specified as: cc_kbc::host_ip:port, and 60000 is the default port used
-			# by the service, as well as the one configured in the Kata Containers rootfs.
-
-			CC_KBS_IP=${CC_KBS_IP:-"$(hostname -I | awk '{print $1}')"}
-			CC_KBS_PORT=${CC_KBS_PORT:-"60000"}
-			add_kernel_params "agent.aa_kbc_params=cc_kbc::http://${CC_KBS_IP}:${CC_KBS_PORT}/"
-			;;
-		*)
-			;;
+	"offline_fs_kbc")
+		add_kernel_params "agent.aa_kbc_params=offline_fs_kbc::null"
+		cp_to_guest_img "etc" "${SHARED_FIXTURES_DIR}/cosign/offline-fs-kbc/$(uname -m)/aa-offline_fs_kbc-resources.json"
+		;;
+	"cc_kbc")
+		# CC KBC is specified as: cc_kbc::host_ip:port, and 60000 is the default port used
+		# by the service, as well as the one configured in the Kata Containers rootfs.
+
+		CC_KBS_IP=${CC_KBS_IP:-"$(hostname -I | awk '{print $1}')"}
+		CC_KBS_PORT=${CC_KBS_PORT:-"60000"}
+		add_kernel_params "agent.aa_kbc_params=cc_kbc::http://${CC_KBS_IP}:${CC_KBS_PORT}/"
+		;;
+	*) ;;
 	esac
 }
 
 setup_signature_files() {
 	case "${AA_KBC:-}" in
-		"offline_fs_kbc")
-			setup_offline_fs_kbc_signature_files_in_guest
-			;;
-		"cc_kbc")
-			setup_cc_kbc_signature_files_in_guest
-			;;
-		*)
-			;;
+	"offline_fs_kbc")
+		setup_offline_fs_kbc_signature_files_in_guest
+		;;
+	"cc_kbc")
+		setup_cc_kbc_signature_files_in_guest
+		;;
+	*) ;;
 	esac
 }
 
 # In case the tests run behind a firewall where images needed to be fetched
-# through a proxy. 
+# through a proxy.
 # Note: With measured rootfs enabled, we can not set proxy through
 # agent config file.
 setup_proxy() {
@@ -349,8 +349,8 @@ setup_credentials_files() {
 
 	dest_dir="$(mktemp -t -d offline-fs-kbc-XXXXXXXX)"
 	dest_file=${dest_dir}/aa-offline_fs_kbc-resources.json
-	auth_json=$(REGISTRY=$1 CREDENTIALS="${REGISTRY_CREDENTIAL_ENCODED}" envsubst < "${SHARED_FIXTURES_DIR}/offline-fs-kbc/auth.json.in" | base64 -w 0)
-	CREDENTIAL="${auth_json}" envsubst < "${SHARED_FIXTURES_DIR}/offline-fs-kbc/aa-offline_fs_kbc-resources.json.in" > "${dest_file}"
+	auth_json=$(REGISTRY=$1 CREDENTIALS="${REGISTRY_CREDENTIAL_ENCODED}" envsubst <"${SHARED_FIXTURES_DIR}/offline-fs-kbc/auth.json.in" | base64 -w 0)
+	CREDENTIAL="${auth_json}" envsubst <"${SHARED_FIXTURES_DIR}/offline-fs-kbc/aa-offline_fs_kbc-resources.json.in" >"${dest_file}"
 	cp_to_guest_img "etc" "${dest_file}"
 }
 
@@ -366,48 +366,48 @@ KBS_DB="${KBS_DB:-simple_kbs}"
 
 # Run the simple-kbs
 simple_kbs_run() {
-  # Retrieve simple-kbs repo and tag from versions.yaml
-  local simple_kbs_url=$(get_test_version "externals.simple-kbs.url")
-  local simple_kbs_tag=$(get_test_version "externals.simple-kbs.tag")
-
-  # Cleanup and create installation directory
-  esudo rm -rf "${SIMPLE_KBS_DIR}" 
-  mkdir -p "${SIMPLE_KBS_DIR}"
-  pushd "${SIMPLE_KBS_DIR}"
-
-  # Clone and run
-  git clone "${simple_kbs_url}" --branch main
-  pushd simple-kbs
-
-  # Checkout, build and start
-  git checkout -b "branch_${simple_kbs_tag}" "${simple_kbs_tag}"
-  esudo docker-compose build
-  esudo docker-compose up -d
-
-  # Wait for simple-kbs to start
-  waitForProcess 15 1 "esudo docker-compose top | grep -q simple-kbs"
-  popd
-  
-  # Get simple-kbs database container ip
-  local kbs_db_host=$(simple_kbs_get_db_ip)
-
-  # Confirm connection to the database is possible
-  waitForProcess 5 1 "mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} -e '\q'"
-  popd
+	# Retrieve simple-kbs repo and tag from versions.yaml
+	local simple_kbs_url=$(get_test_version "externals.simple-kbs.url")
+	local simple_kbs_tag=$(get_test_version "externals.simple-kbs.tag")
+
+	# Cleanup and create installation directory
+	esudo rm -rf "${SIMPLE_KBS_DIR}"
+	mkdir -p "${SIMPLE_KBS_DIR}"
+	pushd "${SIMPLE_KBS_DIR}"
+
+	# Clone and run
+	git clone "${simple_kbs_url}" --branch main
+	pushd simple-kbs
+
+	# Checkout, build and start
+	git checkout -b "branch_${simple_kbs_tag}" "${simple_kbs_tag}"
+	esudo docker-compose build
+	esudo docker-compose up -d
+
+	# Wait for simple-kbs to start
+	waitForProcess 15 1 "esudo docker-compose top | grep -q simple-kbs"
+	popd
+
+	# Get simple-kbs database container ip
+	local kbs_db_host=$(simple_kbs_get_db_ip)
+
+	# Confirm connection to the database is possible
+	waitForProcess 5 1 "mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} -e '\q'"
+	popd
 }
 
 # Stop simple-kbs and database containers
 simple_kbs_stop() {
-  (cd ${SIMPLE_KBS_DIR}/simple-kbs && esudo docker-compose down 2>/dev/null)
+	(cd ${SIMPLE_KBS_DIR}/simple-kbs && esudo docker-compose down 2>/dev/null)
 }
 
 # Delete all test inserted data in the simple-kbs
 simple_kbs_delete_data() {
-  # Get simple-kbs database container ip
-  local kbs_db_host=$(simple_kbs_get_db_ip)
+	# Get simple-kbs database container ip
+	local kbs_db_host=$(simple_kbs_get_db_ip)
 
-  # Delete all data with 'id = 10'
-  mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
+	# Delete all data with 'id = 10'
+	mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
     DELETE FROM secrets WHERE id = 10;
     DELETE FROM policy WHERE id = 10;
 EOF
@@ -415,28 +415,73 @@ EOF
 
 # Get the ip of the simple-kbs database docker container
 simple_kbs_get_db_ip() {
-  esudo docker network inspect simple-kbs_default \
-    | jq -r '.[].Containers[] | select(.Name | test("simple-kbs[_-]db.*")).IPv4Address' \
-    | sed "s|/.*$||g"
+	esudo docker network inspect simple-kbs_default |
+		jq -r '.[].Containers[] | select(.Name | test("simple-kbs[_-]db.*")).IPv4Address' |
+		sed "s|/.*$||g"
 }
 
 # Add key and keyset to database
 # If measurement is provided, add policy with measurement to database
 simple_kbs_add_key_to_db() {
-  local encryption_key="${1}"
-  local measurement="${2}"
-  
-  # Get simple-kbs database container ip
-  local kbs_db_host=$(simple_kbs_get_db_ip)
-
-  if [ -n "${measurement}" ]; then
-    mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
+	local encryption_key="${1}"
+	local measurement="${2}"
+
+	# Get simple-kbs database container ip
+	local kbs_db_host=$(simple_kbs_get_db_ip)
+
+	if [ -n "${measurement}" ]; then
+		mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
       INSERT INTO secrets VALUES (10, 'default/key/ssh-demo', '${encryption_key}', 10);
       INSERT INTO policy VALUES (10, '["${measurement}"]', '[]', 0, 0, '[]', now(), NULL, 1);
 EOF
-  else
-    mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
+	else
+		mysql -u${KBS_DB_USER} -p${KBS_DB_PW} -h ${kbs_db_host} -D ${KBS_DB} <<EOF
       INSERT INTO secrets VALUES (10, 'default/key/ssh-demo', '${encryption_key}', NULL);
 EOF
-  fi
+	fi
+}
+
+###############################################################################
+
+# remote-snapshotter
+
+configure_remote_snapshotter() {
+	case "${SNAPSHOTTER:-}" in
+	"nydus")
+		configure_nydus_snapshotter
+		;;
+	*) ;;
+
+	esac
+}
+check_containerd_version() {
+	containerd_version=$(containerd --version | awk '{print $3}' | sort -V | tail -n 1)
+	if echo $containerd_version | grep -q "^v1.7"; then
+		return 1
+	else
+		return 0
+	fi
+}
+configure_containerd_for_nydus_snapshotter() {
+	if check_containerd_version; then
+		sed -i '/\[plugins\."io\.containerd\.grpc\.v1\.cri"\.containerd\.runtimes\.'"$RUNTIMECLASS"'\]/a\  snapshotter = "nydus"\n  disable_snapshot_annotations = false' "$CONTAINERD_CONFIG"
+	else
+		sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = false/g; s/snapshotter = .*/snapshotter = "nydus"/g' "$CONTAINERD_CONFIG"
+	fi
+}
+remove_nydus_snapshotter_from_containerd() {
+	if check_containerd_version; then
+		sed -i '/\[plugins\."io\.containerd\.grpc\.v1\.cri"\.containerd\.runtimes\.'"$RUNTIMECLASS"'\]/,/\[/{/snapshotter = "nydus"/d;/disable_snapshot_annotations = false/d;}' "$CONTAINERD_CONFIG"
+	else
+		sed -i 's/disable_snapshot_annotations = .*/disable_snapshot_annotations = true/g; s/snapshotter = .*/snapshotter = "overlayfs"/g' "$CONTAINERD_CONFIG"
+	fi
+}
+restart_nydus_snapshotter() {
+	echo "Restart nydus snapshotter"
+}
+
+configure_nydus_snapshotter() {
+	echo "Configure nydus snapshotter"
+	sed -i "s/export_mode = .*/export_mode = \"$EXPORT_MODE\"/" $NYDUS_SNAPSHOTTER_CONFIG
+	restart_nydus_snapshotter
 }
diff --git a/integration/kubernetes/confidential/agent_image.bats b/integration/kubernetes/confidential/agent_image.bats
index e8048b518..d1d84d9b5 100644
--- a/integration/kubernetes/confidential/agent_image.bats
+++ b/integration/kubernetes/confidential/agent_image.bats
@@ -13,7 +13,7 @@ if [ "$(uname -m)" != "x86_64" ]; then
 fi
 
 # Images used on the tests.
-## Cosign 
+## Cosign
 image_cosigned="quay.io/kata-containers/confidential-containers:cosign-signed${tag_suffix}"
 image_cosigned_other="quay.io/kata-containers/confidential-containers:cosign-signed-key2"
 
@@ -70,7 +70,7 @@ new_pod_config() {
 	local image="$1"
 
 	local new_config=$(mktemp "${BATS_FILE_TMPDIR}/$(basename ${base_config}).XXX")
-	IMAGE="$image" RUNTIMECLASS="$RUNTIMECLASS" envsubst < "$base_config" > "$new_config"
+	IMAGE="$image" RUNTIMECLASS="$RUNTIMECLASS" envsubst <"$base_config" >"$new_config"
 	echo "$new_config"
 }
 
@@ -90,7 +90,7 @@ setup() {
 	switch_image_service_offload on
 	clear_kernel_params
 	add_kernel_params "${original_kernel_params}"
-	
+
 	setup_proxy
 	switch_measured_rootfs_verity_scheme none
 }
@@ -183,7 +183,6 @@ assert_logs_contain() {
 	assert_logs_contain 'Validate image failed: \[PublicKeyVerifier { key: ECDSA_P256_SHA256_ASN1'
 }
 
-
 @test "$test_tag Test pull an unencrypted unsigned image from an authenticated registry with correct credentials" {
 	if [ "${AA_KBC}" = "offline_fs_kbc" ]; then
 		setup_credentials_files "quay.io/kata-containers/confidential-containers-auth"
@@ -223,6 +222,24 @@ assert_logs_contain() {
 	assert_logs_contain 'failed to pull manifest Not authorized'
 }
 
+@test "$test_tag Test can pull an image as a raw block disk image to guest with dm-verity enabled" {
+	if [ "${SNAPSHOTTER}" = "nydus" ]; then
+		EXPORT_MODE="image_block_with_verity" configure_remote_snapshotter
+		pod_config="$(new_pod_config "$image_unsigned_unprotected")"
+		echo $pod_config
+		create_test_pod
+	fi
+}
+
+@test "$test_tag Test can pull an image as a raw block disk image to guest with dm-verity enabled" {
+	if [ "${SNAPSHOTTER}" = "nydus" ]; then
+		EXPORT_MODE="image_block_with_verity" RUNTIMECLASS="$RUNTIMECLASS" configure_remote_snapshotter
+		pod_config="$(new_pod_config "$image_unsigned_unprotected")"
+		echo $pod_config
+		create_test_pod
+	fi
+}
+
 teardown() {
 	# Print the logs and cleanup resources.
 	echo "-- Kata logs:"