test: add internal encryption e2e tests

test/e2e-common.sh

@@ -462,6 +462,13 @@ function wait_for_leader_controller() {
   return 1
 }
 
+function restart_pod() {
+  local namespace="$1"
+  local label="$2"
+  echo -n "Deleting pod in ${namespace} with label ${label}"
+  kubectl -n ${namespace} delete pod -l ${label}
+}
+
 function toggle_feature() {
   local FEATURE="$1"
   local STATE="$2"

test/e2e-tests.sh

@@ -95,6 +95,14 @@ toggle_feature autocreateClusterDomainClaims true config-network || fail_test
 go_test_e2e -timeout=2m ./test/e2e/domainmapping ${TEST_OPTIONS} || failed=1
 toggle_feature autocreateClusterDomainClaims false config-network || fail_test
 
+toggle_feature dataplane-trust enabled config-network || fail_test
+# with the current implementation, Activator is always in the request path, and needs to be restarted after configuring dataplane-trust
+restart_pod ${SYSTEM_NAMESPACE} "app=activator"
+go_test_e2e -timeout=2m ./test/e2e/internalencryption ${TEST_OPTIONS} || failed=1
+toggle_feature dataplane-trust disabled config-network || fail_test
+# with the current implementation, Activator is always in the request path, and needs to be restarted after configuring dataplane-trust
+restart_pod ${SYSTEM_NAMESPACE} "app=activator"
+
 kubectl get cm "config-gc" -n "${SYSTEM_NAMESPACE}" -o yaml > ${TMP_DIR}/config-gc.yaml
 add_trap "kubectl replace cm 'config-gc' -n ${SYSTEM_NAMESPACE} -f ${TMP_DIR}/config-gc.yaml" SIGKILL SIGTERM SIGQUIT
 immediate_gc

test/e2e/internalencryption/README.md

@@ -0,0 +1,15 @@
+# Internal Encryption E2E Tests
+
+In order to test Internal Encryption, this test looks at the `security_mode` tag on `request_count` metrics from the Activator and QueueProxy.
+
+The `metricsreader` test image was created for this purpose. Given the PodIPs of the Activator and the Knative Service pod (i.e. the QueueProxy), it will make requests to each respective `/metrics` endpoint, pull out the `*_request_count` metric, look for the tag `security_mode`, and respond with a map of tag values to counts. The [README.md](../../test_images/metricsreader/README.md) will explain in more detail.
+
+The test works as follows:
+* The [setup script](../../e2e-internal-encryption-tests.sh) configures the `dataplane-trust` config in `config-network` to `enabled`.
+* The [test](internalencryption_test.go) deploys the `metricsreader` Knative Service. This service uses annotations to set the initial, min, and max scale to 1. This is to guarantee the PodIP is consistent during the test, and avoid complications of having multiple instances.
+* The test then extracts the PodIPs of the activator and the pod of the latest `metricsreader` Revision.
+* The test will make 3 requests to the `metricsreader` pod:
+  * A GET request to make sure it is alive.
+  * A first POST request to get the initial `request_count`s.
+  * A second POST request to get the updated `request_count`s.
+* The test will then compare the counts between the initial and updated requests to determine if the counts have increased, indicating that Internal Encryption is indeed happening.

test/e2e/internalencryption/internalencryption_test.go

@@ -0,0 +1,145 @@
+//go:build e2e
+// +build e2e
+
+/*
+Copyright 2021 The Knative Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package internalencryption
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"testing"
+
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	netcfg "knative.dev/networking/pkg/config"
+	pkgTest "knative.dev/pkg/test"
+	"knative.dev/pkg/test/spoof"
+
+	//. "knative.dev/serving/pkg/testing/v1"
+	"knative.dev/serving/test"
+	"knative.dev/serving/test/test_images/metricsreader/helpers"
+	v1test "knative.dev/serving/test/v1"
+)
+
+var (
+	ExpectedSecurityMode = netcfg.TrustEnabled
+)
+
+// TestInitContainers tests init containers support.
+func TestInternalEncryption(t *testing.T) {
+	t.Parallel()
+	clients := test.Setup(t)
+
+	names := test.ResourceNames{
+		Service: test.ObjectNameForTest(t),
+		Image:   test.MetricsReader,
+	}
+
+	test.EnsureTearDown(t, clients, &names)
+
+	t.Log("Creating a new Service")
+	resources, err := v1test.CreateServiceReady(t, clients, &names)
+	if err != nil {
+		t.Fatalf("Failed to create initial Service: %v: %v", names.Service, err)
+	}
+
+	url := resources.Route.Status.URL.URL()
+	if _, err := pkgTest.CheckEndpointState(
+		context.Background(),
+		clients.KubeClient,
+		t.Logf,
+		url,
+		spoof.MatchesAllOf(spoof.IsStatusOK, spoof.MatchesBody(test.MetricsReaderText)),
+		"MetricsReaderText",
+		test.ServingFlags.ResolvableDomain,
+	); err != nil {
+		t.Fatalf("The endpoint %s for Route %s didn't serve the expected text %q: %v", url, names.Route, test.MetricsReaderText, err)
+	}
+
+	pods, err := clients.KubeClient.CoreV1().Pods("serving-tests").List(context.TODO(), v1.ListOptions{
+		LabelSelector: fmt.Sprintf("serving.knative.dev/configuration=%s", names.Config),
+	})
+	if err != nil {
+		t.Fatalf("Failed to get pods: %v", err)
+	}
+	var postData helpers.PostData
+
+	if len(pods.Items) > 0 {
+		postData.QueueIP = pods.Items[0].Status.PodIP
+	}
+
+	pods, err = clients.KubeClient.CoreV1().Pods("knative-serving").List(context.TODO(), v1.ListOptions{
+		LabelSelector: "app=activator",
+	})
+	if err != nil {
+		t.Fatalf("Failed to get pods: %v", err)
+	}
+	if len(pods.Items) > 0 {
+		postData.ActivatorIP = pods.Items[0].Status.PodIP
+	}
+
+	initialCounts, err := getTLSCounts(url.String(), &postData)
+	if err != nil {
+		t.Fatalf("Failed to get initial TLS Connection Counts: %v", err)
+	}
+	t.Logf("Initial Counts: %#v", initialCounts)
+
+	updatedCounts, err := getTLSCounts(url.String(), &postData)
+	if err != nil {
+		t.Fatalf("Failed to get updated TLS Connection Counts: %v", err)
+	}
+	t.Logf("Updated Counts: %#v", updatedCounts)
+
+	if updatedCounts.Activator[ExpectedSecurityMode] <= initialCounts.Activator[ExpectedSecurityMode] {
+		t.Fatalf("Connection Count with SecurityMode (%s) at Activator pod failed to increase", ExpectedSecurityMode)
+	}
+
+	if updatedCounts.Queue[ExpectedSecurityMode] <= initialCounts.Queue[ExpectedSecurityMode] {
+		t.Fatalf("Connection Count with SecurityMode (%s) at QueueProxy pod failed to increase", ExpectedSecurityMode)
+	}
+
+}
+
+func getTLSCounts(url string, d *helpers.PostData) (*helpers.ResponseData, error) {
+	counts := &helpers.ResponseData{}
+
+	jsonPostData, err := json.Marshal(d)
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal post data request to JSON:\n  %#v\n  %w", *d, err)
+	}
+	resp, err := http.Post(url, "application/json", bytes.NewBuffer(jsonPostData))
+	if err != nil {
+		return nil, fmt.Errorf("failed to make POST request to %s: %w", url, err)
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to read response body from %s: %w", url, err)
+	}
+
+	err = json.Unmarshal(body, &counts)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to unmarshal response body:\n  body: %s\n  err: %w", string(body), err)
+	}
+
+	return counts, nil
+}