test: add test image for reading request metrics

test/conformance.go

@@ -43,6 +43,7 @@ const (
 	HelloWorld          = "helloworld"
 	HTTPProxy           = "httpproxy"
 	InvalidHelloWorld   = "invalidhelloworld" // Not a real image
+	MetricsReader       = "metricsreader"
 	PizzaPlanet1        = "pizzaplanetv1"
 	PizzaPlanet2        = "pizzaplanetv2"
 	Protocols           = "protocols"
@@ -56,11 +57,12 @@ const (
 	WorkingDir          = "workingdir"
 
 	// Constants for test image output.
-	PizzaPlanetText1 = "What a spaceport!"
-	PizzaPlanetText2 = "Re-energize yourself with a slice of pepperoni!"
-	HelloWorldText   = "Hello World! How about some tasty noodles?"
-	HelloHTTP2Text   = "Hello, New World! How about donuts and coffee?"
-	EmptyDirText     = "From file in empty dir!"
+	PizzaPlanetText1  = "What a spaceport!"
+	PizzaPlanetText2  = "Re-energize yourself with a slice of pepperoni!"
+	HelloWorldText    = "Hello World! How about some tasty noodles?"
+	HelloHTTP2Text    = "Hello, New World! How about donuts and coffee?"
+	EmptyDirText      = "From file in empty dir!"
+	MetricsReaderText = "Come back with a POST for metrics."
 
 	MultiContainerResponse = "Yay!! multi-container works"
 

test/test_images/metricsreader/README.md

@@ -0,0 +1,47 @@
+# Metricsreader test image
+
+This directory contains the test image used in the Interal Encryption e2e test.
+
+The image contains a simple Go webserver, `metricsreader.go`, that will, by
+default, listen on port `8080` and expose a service at `/`.
+
+A `GET` request just returns a simple hello message.
+
+A `POST` request with the IP addresses of the Activator and the Pod for the latest revision will prompt the server to make requests to the metrics endpoints on each IP, and collect the `*_request_count` metrics. It will check for the tag `security_mode` on each metric, and report the counts based on the value of the tag. The `security_mode` tag corresponds to the possible values for the config option `dataplane-trust` in `config-network`.
+
+This is used in the test to make sure that, when Internal Encryption is enabled (`dataplane-trust != Disabled`), the Activator and Queue Proxies are handling TLS connections.
+
+An example request and response looks like this:
+```
+❯ curl -X POST http://metricsreader-test-image.default.kauz.tanzu.biz -H "Content-Type: application/json" -d '{"activator_ip": "10.24.1.132", "queue_ip": "10.24.3.164"}' | jq .
+
+{
+  "ActivatorData": {
+    "disabled": 0,
+    "enabled": 0,
+    "identity": 0,
+    "minimal": 1,
+    "mutual": 0
+  },
+  "QueueData": {
+    "disabled": 0,
+    "enabled": 0,
+    "identity": 0,
+    "minimal": 1,
+    "mutual": 0
+  }
+}
+```
+
+By default, the Knative Service is set with an initial-scale, min-scale, and max-scale of 1. This is to make it possible to know the IP of the Queue Proxy before making the call, and to avoid any complications due to multiple instances.
+
+## Trying out
+
+To run the image as a Service outside of the test suite:
+
+`ko apply -f service.yaml`
+
+## Building
+
+For details about building and adding new images, see the
+[section about test images](/test/README.md#test-images).

test/test_images/metricsreader/helpers/util.go

@@ -0,0 +1,50 @@
+/*
+Copyright 2021 The Knative Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package helpers
+
+//. "knative.dev/serving/pkg/testing/v1"
+
+import (
+	netcfg "knative.dev/networking/pkg/config"
+)
+
+type PostData struct {
+	ActivatorIP string `json:"activator_ip"`
+	QueueIP     string `json:"queue_ip"`
+}
+
+type ResponseData struct {
+	Activator map[netcfg.Trust]int `json:"activator"`
+	Queue     map[netcfg.Trust]int `json:"queue"`
+}
+
+func NewResponse() *ResponseData {
+	return &ResponseData{
+		Activator: NewSecurityModeMap(),
+		Queue:     NewSecurityModeMap(),
+	}
+}
+
+func NewSecurityModeMap() map[netcfg.Trust]int {
+	return map[netcfg.Trust]int{
+		netcfg.TrustDisabled: 0,
+		netcfg.TrustEnabled:  0,
+		netcfg.TrustIdentity: 0,
+		netcfg.TrustMinimal:  0,
+		netcfg.TrustMutual:   0,
+	}
+}

test/test_images/metricsreader/metricsreader.go

@@ -0,0 +1,171 @@
+/*
+Copyright 2018 The Knative Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net"
+	"net/http"
+	"time"
+
+	netcfg "knative.dev/networking/pkg/config"
+	"knative.dev/serving/pkg/metrics"
+	pkgnet "knative.dev/serving/pkg/networking"
+	"knative.dev/serving/test"
+	"knative.dev/serving/test/test_images/metricsreader/helpers"
+
+	io_prometheus_client "github.com/prometheus/client_model/go"
+	"github.com/prometheus/common/expfmt"
+)
+
+const (
+	ActivatorMetricKey = "activator_request_count"
+	QueueMetricKey     = "revision_request_count"
+)
+
+type MetricsClient struct {
+	http.Client
+}
+
+func (mc *MetricsClient) GetMetricsData(source string) (map[string]*io_prometheus_client.MetricFamily, error) {
+	resp, err := mc.Get(fmt.Sprintf("http://%s/metrics", source))
+	if err != nil {
+		return nil, fmt.Errorf("error: couldn't call %s metrics endpoint: %w", source, err)
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("error: couldn't read response body from %s metrics endpoint: %w", source, err)
+	}
+	return parseMetricsData(body)
+}
+
+func NewMetricsClient(d *helpers.PostData) *MetricsClient {
+	dialer := &net.Dialer{
+		Timeout:   30 * time.Second,
+		KeepAlive: 30 * time.Second,
+	}
+	transport := http.Transport{
+		Proxy:               http.ProxyFromEnvironment,
+		Dial:                dialer.Dial,
+		TLSHandshakeTimeout: 10 * time.Second,
+		DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
+			if addr == "activator:80" {
+				addr = fmt.Sprintf("%s:9090", d.ActivatorIP)
+			} else if addr == "queue:80" {
+				addr = fmt.Sprintf("%s:%d", d.QueueIP, pkgnet.UserQueueMetricsPort)
+			}
+			return dialer.DialContext(ctx, network, addr)
+		},
+	}
+
+	return &MetricsClient{http.Client{
+		Transport: &transport,
+		Timeout:   4 * time.Second,
+	}}
+}
+
+func buildResponse(activatorRequestCount, queueRequestCount *io_prometheus_client.MetricFamily) *helpers.ResponseData {
+	results := helpers.NewResponse()
+
+	for _, l := range activatorRequestCount.Metric[0].Label {
+		if *l.Name == string(metrics.LabelSecurityMode) {
+			results.Activator[netcfg.Trust(*l.Value)] = int(*activatorRequestCount.Metric[0].Counter.Value)
+		}
+	}
+
+	for _, l := range queueRequestCount.Metric[0].Label {
+		if *l.Name == string(metrics.LabelSecurityMode) {
+			results.Queue[netcfg.Trust(*l.Value)] = int(*queueRequestCount.Metric[0].Counter.Value)
+		}
+	}
+
+	return results
+}
+
+func parseMetricsData(d []byte) (map[string]*io_prometheus_client.MetricFamily, error) {
+	//parse the response
+	var parser expfmt.TextParser
+	mf, err := parser.TextToMetricFamilies(bytes.NewReader(d))
+	if err != nil {
+		return nil, fmt.Errorf("error: couldn't parse metrics output: %w", err)
+	}
+	return mf, nil
+}
+
+func getMetrics(w http.ResponseWriter, r *http.Request) {
+	if r.URL.Path != "/" {
+		http.Error(w, "404 not found.", http.StatusNotFound)
+	}
+	switch r.Method {
+	case "GET":
+		log.Print("Metricsreader received a GET request.")
+		w.Write([]byte("Come back with a POST for metrics."))
+	case "POST":
+		log.Print("Metricsreader received a POST request.")
+
+		decoder := json.NewDecoder(r.Body)
+		var d helpers.PostData
+		err := decoder.Decode(&d)
+		if err != nil {
+			msg := fmt.Errorf("failed to decode POST data: %w", err)
+			log.Print(msg)
+			http.Error(w, msg.Error(), http.StatusInternalServerError)
+		}
+
+		metricsClient := NewMetricsClient(&d)
+
+		activatorData, err := metricsClient.GetMetricsData("activator")
+		if err != nil {
+			log.Print(err.Error())
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+		}
+
+		queueData, err := metricsClient.GetMetricsData("queue")
+		if err != nil {
+			log.Print(err.Error())
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+		}
+
+		results := buildResponse(activatorData[ActivatorMetricKey], queueData[QueueMetricKey])
+
+		jsonResponseData, err := json.Marshal(results)
+		if err != nil {
+			msg := fmt.Errorf("failed to marshal results to json: %w", err)
+			log.Print(msg)
+			http.Error(w, msg.Error(), http.StatusInternalServerError)
+		} else {
+			w.Write(jsonResponseData)
+		}
+
+	default:
+		msg := "Sorry, only GET and POST methods are supported."
+		log.Print(msg)
+		http.Error(w, msg, http.StatusMethodNotAllowed)
+	}
+}
+
+func main() {
+	log.Print("metricsreader app started.")
+
+	test.ListenAndServeGracefully(":8080", getMetrics)
+}

test/test_images/metricsreader/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: serving.knative.dev/v1
+kind: Service
+metadata:
+  name: metricsreader-test-image
+  namespace: default
+spec:
+  template:
+    metadata:
+      annotations:
+        autoscaling.knative.dev/initial-scale: "1"
+        autoscaling.knative.dev/min-scale: "1"
+        autoscaling.knative.dev/max-scale: "1"
+    spec:
+      containers:
+      - image: ko://knative.dev/serving/test/test_images/metricsreader