From 3eb4d441e9a82bdc724df6081b0921dbdd093e6b Mon Sep 17 00:00:00 2001 From: kanha gupta <92207457+kanha-gupta@users.noreply.github.com> Date: Wed, 15 May 2024 03:31:02 +0530 Subject: [PATCH] Pre-installation testing framework (#6278) This PR enables running sanity checks on a K8s cluster before Antrea is installed. The command to run the checks is `antctl check cluster`. At the moment we run 4 different sanity checks. Fixes #6153 Signed-off-by: Kanha gupta --- .github/workflows/kind.yml | 15 +- pkg/antctl/antctl.go | 7 + pkg/antctl/raw/check/cluster/command.go | 236 ++++++++++++++++++ .../check/cluster/test_checkcniexistence.go | 52 ++++ .../test_checkcontrolplaneavailability.go | 49 ++++ .../raw/check/cluster/test_checkk8sversion.go | 48 ++++ .../check/cluster/test_checkovsloadable.go | 58 +++++ pkg/antctl/raw/check/installation/command.go | 128 +--------- pkg/antctl/raw/check/util.go | 133 ++++++++++ 9 files changed, 599 insertions(+), 127 deletions(-) create mode 100644 pkg/antctl/raw/check/cluster/command.go create mode 100644 pkg/antctl/raw/check/cluster/test_checkcniexistence.go create mode 100644 pkg/antctl/raw/check/cluster/test_checkcontrolplaneavailability.go create mode 100644 pkg/antctl/raw/check/cluster/test_checkk8sversion.go create mode 100644 pkg/antctl/raw/check/cluster/test_checkovsloadable.go diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml index 1b88a3a6547..eeaad6eaa74 100644 --- a/.github/workflows/kind.yml +++ b/.github/workflows/kind.yml @@ -744,8 +744,8 @@ jobs: path: log.tar.gz retention-days: 30 - run-post-installation-checks: - name: Test connectivity using 'antctl check' command + run-installation-checks: + name: Test installation using 'antctl check' command needs: [ build-antrea-coverage-image ] runs-on: [ ubuntu-latest ] steps: @@ -775,13 +775,16 @@ jobs: - name: Create Kind Cluster run: | ./ci/kind/kind-setup.sh create kind --ip-family dual - - name: Deploy Antrea - run: | - kubectl apply -f build/yamls/antrea.yml - name: Build antctl binary run: | make antctl-linux - - name: Run antctl command + - name: Run Pre checks + run: | + ./bin/antctl-linux check cluster + - name: Deploy Antrea + run: | + kubectl apply -f build/yamls/antrea.yml + - name: Run Post checks run: | ./bin/antctl-linux check installation diff --git a/pkg/antctl/antctl.go b/pkg/antctl/antctl.go index 2addc2d44e9..946dbb3d53e 100644 --- a/pkg/antctl/antctl.go +++ b/pkg/antctl/antctl.go @@ -19,6 +19,7 @@ import ( agentapis "antrea.io/antrea/pkg/agent/apis" fallbackversion "antrea.io/antrea/pkg/antctl/fallback/version" + checkcluster "antrea.io/antrea/pkg/antctl/raw/check/cluster" checkinstallation "antrea.io/antrea/pkg/antctl/raw/check/installation" "antrea.io/antrea/pkg/antctl/raw/featuregates" "antrea.io/antrea/pkg/antctl/raw/multicluster" @@ -640,6 +641,12 @@ $ antctl get podmulticaststats pod -n namespace`, supportController: false, commandGroup: check, }, + { + cobraCommand: checkcluster.Command(), + supportAgent: false, + supportController: false, + commandGroup: check, + }, { cobraCommand: supportbundle.Command, supportAgent: true, diff --git a/pkg/antctl/raw/check/cluster/command.go b/pkg/antctl/raw/check/cluster/command.go new file mode 100644 index 00000000000..07c3ae313ce --- /dev/null +++ b/pkg/antctl/raw/check/cluster/command.go @@ -0,0 +1,236 @@ +// Copyright 2024 Antrea Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cluster + +import ( + "context" + "errors" + "fmt" + "os" + "time" + + "github.com/fatih/color" + "github.com/spf13/cobra" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/utils/ptr" + + "antrea.io/antrea/pkg/antctl/raw/check" + "antrea.io/antrea/pkg/version" +) + +func Command() *cobra.Command { + command := &cobra.Command{ + Use: "cluster", + Short: "Runs pre installation checks", + RunE: func(cmd *cobra.Command, args []string) error { + return Run() + }, + } + return command +} + +const ( + testNamespacePrefix = "antrea-test" + deploymentName = "cluster-checker" + podReadyTimeout = 1 * time.Minute +) + +type uncertainError struct { + reason string +} + +func (e uncertainError) Error() string { + return e.reason +} + +func newUncertainError(reason string, a ...interface{}) uncertainError { + return uncertainError{reason: fmt.Sprintf(reason, a...)} +} + +type Test interface { + Run(ctx context.Context, testContext *testContext) error +} + +var testsRegistry = make(map[string]Test) + +func RegisterTest(name string, test Test) { + testsRegistry[name] = test +} + +type testContext struct { + client kubernetes.Interface + config *rest.Config + clusterName string + namespace string + testPod *corev1.Pod +} + +func Run() error { + client, config, clusterName, err := check.NewClient() + if err != nil { + return fmt.Errorf("unable to create Kubernetes client: %s", err) + } + ctx := context.Background() + testContext := NewTestContext(client, config, clusterName) + if err := testContext.setup(ctx); err != nil { + return err + } + var numSuccess, numFailure, numUncertain int + for name, test := range testsRegistry { + testContext.Header("Running test: %s", name) + if err := test.Run(ctx, testContext); err != nil { + if errors.As(err, new(uncertainError)) { + testContext.Warning("Test %s was uncertain: %v", name, err) + numUncertain++ + } else { + testContext.Fail("Test %s failed: %v", name, err) + numFailure++ + } + } else { + testContext.Success("Test %s passed", name) + numSuccess++ + } + } + testContext.Log("Test finished: %v tests succeeded, %v tests failed, %v tests were uncertain", numSuccess, numFailure, numUncertain) + check.Teardown(ctx, testContext.client, testContext.clusterName, testContext.namespace) + if numFailure > 0 { + return fmt.Errorf("%v/%v tests failed", numFailure, len(testsRegistry)) + } + return nil +} + +func (t *testContext) setup(ctx context.Context) error { + t.Log("Creating Namespace %s for pre installation tests...", t.namespace) + _, err := t.client.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: t.namespace}}, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("unable to create Namespace %s: %s", t.namespace, err) + } + deployment := check.NewDeployment(check.DeploymentParameters{ + Name: deploymentName, + Image: getAntreaAgentImage(), + Replicas: 1, + Command: []string{"bash", "-c"}, + Args: []string{"trap 'exit 0' SIGTERM; sleep infinity & pid=$!; wait $pid"}, + Labels: map[string]string{"app": "antrea", "component": "cluster-checker"}, + HostNetwork: true, + VolumeMounts: []corev1.VolumeMount{ + {Name: "cni-conf", MountPath: "/etc/cni/net.d"}, + {Name: "lib-modules", MountPath: "/lib/modules"}, + }, + Tolerations: []corev1.Toleration{ + { + Key: "node-role.kubernetes.io/control-plane", + Operator: "Exists", + Effect: "NoSchedule", + }, + { + Key: "node-role.kubernetes.io/master", + Operator: "Exists", + Effect: "NoSchedule", + }, + { + Key: "node.kubernetes.io/not-ready", + Operator: "Exists", + Effect: "NoSchedule", + }, + }, + Volumes: []corev1.Volume{ + { + Name: "cni-conf", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/etc/cni/net.d", + }, + }, + }, + { + Name: "lib-modules", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/lib/modules", + Type: ptr.To(corev1.HostPathType("Directory")), + }, + }, + }, + }, + NodeSelector: map[string]string{ + "kubernetes.io/os": "linux", + }, + SecurityContext: &corev1.SecurityContext{ + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{"SYS_MODULE"}, + }, + }, + }) + + t.Log("Creating Deployment") + _, err = t.client.AppsV1().Deployments(t.namespace).Create(ctx, deployment, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("unable to create Deployment: %w", err) + } + + t.Log("Waiting for Deployment to become ready") + err = check.WaitForDeploymentsReady(ctx, time.Second, podReadyTimeout, t.client, t.clusterName, t.namespace, deploymentName) + if err != nil { + return fmt.Errorf("error while waiting for Deployment to become ready: %w", err) + } + testPods, err := t.client.CoreV1().Pods(t.namespace).List(ctx, metav1.ListOptions{LabelSelector: "component=cluster-checker"}) + if err != nil { + return fmt.Errorf("no pod found for test Deployment") + } + t.testPod = &testPods.Items[0] + return nil +} + +func getAntreaAgentImage() string { + if version.ReleaseStatus == "released" { + return fmt.Sprintf("antrea/antrea-agent-ubuntu:%s", version.GetVersion()) + } + return "antrea/antrea-agent-ubuntu:latest" +} + +func NewTestContext(client kubernetes.Interface, config *rest.Config, clusterName string) *testContext { + return &testContext{ + client: client, + config: config, + clusterName: clusterName, + namespace: check.GenerateRandomNamespace(testNamespacePrefix), + } +} + +func (t *testContext) Log(format string, a ...interface{}) { + fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", t.clusterName)+format+"\n", a...) +} + +func (t *testContext) Success(format string, a ...interface{}) { + fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", t.clusterName)+color.GreenString(format, a...)+"\n") +} + +func (t *testContext) Fail(format string, a ...interface{}) { + fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", t.clusterName)+color.RedString(format, a...)+"\n") +} + +func (t *testContext) Warning(format string, a ...interface{}) { + fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", t.clusterName)+color.YellowString(format, a...)+"\n") +} + +func (t *testContext) Header(format string, a ...interface{}) { + t.Log("-------------------------------------------------------------------------------------------") + t.Log(format, a...) + t.Log("-------------------------------------------------------------------------------------------") +} diff --git a/pkg/antctl/raw/check/cluster/test_checkcniexistence.go b/pkg/antctl/raw/check/cluster/test_checkcniexistence.go new file mode 100644 index 00000000000..11a38024616 --- /dev/null +++ b/pkg/antctl/raw/check/cluster/test_checkcniexistence.go @@ -0,0 +1,52 @@ +// Copyright 2024 Antrea Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cluster + +import ( + "context" + "fmt" + "sort" + "strings" + + "antrea.io/antrea/pkg/antctl/raw/check" +) + +type checkCNIExistence struct{} + +func init() { + RegisterTest("check-cni-existence", &checkCNIExistence{}) +} + +func (t *checkCNIExistence) Run(ctx context.Context, testContext *testContext) error { + command := []string{"ls", "-1", "/etc/cni/net.d"} + output, _, err := check.ExecInPod(ctx, testContext.client, testContext.config, testContext.namespace, testContext.testPod.Name, "", command) + if err != nil { + return fmt.Errorf("failed to execute command in Pod %s, error: %w", testContext.testPod.Name, err) + } + files := strings.Fields(output) + if len(files) == 0 { + testContext.Log("No files present in /etc/cni/net.d in Node %s", testContext.testPod.Spec.NodeName) + return nil + } + sort.Strings(files) + if files[0] < "10-antrea.conflist" { + return newUncertainError("another CNI configuration file with higher priority than Antrea's CNI configuration file found: %s; this may be expected if networkPolicyOnly mode is enabled", files[0]) + } else if files[0] != "10-antrea.conflist" { + testContext.Log("Another CNI configuration file found: %s with Antrea having higher precedence", files[0]) + } else { + testContext.Log("Antrea's CNI configuration file already present: %s", files[0]) + } + return nil +} diff --git a/pkg/antctl/raw/check/cluster/test_checkcontrolplaneavailability.go b/pkg/antctl/raw/check/cluster/test_checkcontrolplaneavailability.go new file mode 100644 index 00000000000..89cfd8e9182 --- /dev/null +++ b/pkg/antctl/raw/check/cluster/test_checkcontrolplaneavailability.go @@ -0,0 +1,49 @@ +// Copyright 2024 Antrea Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cluster + +import ( + "context" + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" +) + +type checkControlPlaneAvailability struct{} + +func init() { + RegisterTest("check-control-plane-nodes-availability", &checkControlPlaneAvailability{}) +} + +func (t *checkControlPlaneAvailability) Run(ctx context.Context, testContext *testContext) error { + controlPlaneNodes := sets.New[string]() + controlPlaneLabels := []string{"node-role.kubernetes.io/control-plane", "node-role.kubernetes.io/master"} + for _, label := range controlPlaneLabels { + nodes, err := testContext.client.CoreV1().Nodes().List(ctx, metav1.ListOptions{LabelSelector: label}) + if err != nil { + return fmt.Errorf("failed to list Nodes with label %s: %w", label, err) + } + for idx := range nodes.Items { + controlPlaneNodes.Insert(nodes.Items[idx].Name) + } + } + if controlPlaneNodes.Len() == 0 { + return newUncertainError("no control-plane Nodes were found; if installing Antrea in encap mode, some K8s functionalities (API aggregation, apiserver proxy, admission controllers) may be impacted") + } else { + testContext.Log("control-plane Nodes were found in the cluster.") + } + return nil +} diff --git a/pkg/antctl/raw/check/cluster/test_checkk8sversion.go b/pkg/antctl/raw/check/cluster/test_checkk8sversion.go new file mode 100644 index 00000000000..277de84b501 --- /dev/null +++ b/pkg/antctl/raw/check/cluster/test_checkk8sversion.go @@ -0,0 +1,48 @@ +// Copyright 2024 Antrea Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cluster + +import ( + "context" + "fmt" + "strings" + + "github.com/blang/semver" +) + +type checkK8sVersion struct{} + +func init() { + RegisterTest("check-k8s-version", &checkK8sVersion{}) +} + +func (t *checkK8sVersion) Run(ctx context.Context, testContext *testContext) error { + discoveryClient := testContext.client.Discovery() + serverVersion, err := discoveryClient.ServerVersion() + if err != nil { + return fmt.Errorf("error getting server version: %w", err) + } + currentVersion, err := semver.Parse(strings.TrimPrefix(serverVersion.GitVersion, "v")) + if err != nil { + return fmt.Errorf("error parsing server version: %w", err) + } + minVersion, _ := semver.Parse("1.19") + if currentVersion.GTE(minVersion) { + testContext.Log("Kubernetes server version is compatible with Antrea. Kubernetes version: %s", serverVersion.GitVersion) + } else { + return fmt.Errorf("Kubernetes min version required: 1.19") + } + return nil +} diff --git a/pkg/antctl/raw/check/cluster/test_checkovsloadable.go b/pkg/antctl/raw/check/cluster/test_checkovsloadable.go new file mode 100644 index 00000000000..a27d501eee3 --- /dev/null +++ b/pkg/antctl/raw/check/cluster/test_checkovsloadable.go @@ -0,0 +1,58 @@ +// Copyright 2024 Antrea Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cluster + +import ( + "context" + "fmt" + "strings" + + "antrea.io/antrea/pkg/antctl/raw/check" +) + +type checkOVSLoadable struct{} + +func init() { + RegisterTest("check-if-openvswitch-is-loadable", &checkOVSLoadable{}) +} + +func (c *checkOVSLoadable) Run(ctx context.Context, testContext *testContext) error { + command := []string{ + "/bin/sh", + "-c", + "grep -q 'openvswitch.ko' /lib/modules/$(uname -r)/modules.builtin; echo $?", + } + stdout, stderr, err := check.ExecInPod(ctx, testContext.client, testContext.config, testContext.namespace, testContext.testPod.Name, "", command) + if err != nil { + return fmt.Errorf("error executing command in Pod %s: %w", testContext.testPod.Name, err) + } + if strings.TrimSpace(stdout) == "0" { + testContext.Log("The kernel module openvswitch is built-in") + } else if strings.TrimSpace(stdout) == "1" { + testContext.Log("The kernel module openvswitch is not built-in. Running modprobe command to load the module.") + cmd := []string{"modprobe", "openvswitch"} + _, stderr, err := check.ExecInPod(ctx, testContext.client, testContext.config, testContext.namespace, testContext.testPod.Name, "", cmd) + if err != nil { + return fmt.Errorf("error executing modprobe command in Pod %s: %w", testContext.testPod.Name, err) + } else if stderr != "" { + return fmt.Errorf("failed to load the OVS kernel module: %s, try running 'modprobe openvswitch' on your Nodes", stderr) + } else { + testContext.Log("openvswitch kernel module loaded successfully") + } + } else { + return fmt.Errorf("error encountered while checking if openvswitch module is built-in - stderr: %s", stderr) + } + return nil +} diff --git a/pkg/antctl/raw/check/installation/command.go b/pkg/antctl/raw/check/installation/command.go index 63676def456..82b550ac1cc 100644 --- a/pkg/antctl/raw/check/installation/command.go +++ b/pkg/antctl/raw/check/installation/command.go @@ -16,7 +16,6 @@ package installation import ( "context" - "crypto/rand" "errors" "fmt" "net" @@ -25,10 +24,8 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -132,7 +129,7 @@ func Run(o *options) error { } } testContext.Log("Test finished: %v tests succeeded, %v tests failed, %v tests were skipped", numSuccess, numFailure, numSkipped) - testContext.teardown(ctx) + check.Teardown(ctx, testContext.client, testContext.clusterName, testContext.namespace) if numFailure > 0 { return fmt.Errorf("%v/%v tests failed", numFailure, len(testsRegistry)) } @@ -159,106 +156,13 @@ func newService(name string, selector map[string]string, port int) *corev1.Servi } } -type deploymentParameters struct { - Name string - Role string - Image string - Replicas int - Port int - Command []string - Affinity *corev1.Affinity - Tolerations []corev1.Toleration - Labels map[string]string -} - -func newDeployment(p deploymentParameters) *appsv1.Deployment { - if p.Replicas == 0 { - p.Replicas = 1 - } - replicas32 := int32(p.Replicas) - labels := map[string]string{ - "name": p.Name, - "kind": p.Role, - } - return &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: p.Name, - Labels: labels, - }, - Spec: appsv1.DeploymentSpec{ - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: p.Name, - Labels: labels, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: p.Name, - Env: []corev1.EnvVar{ - {Name: "PORT", Value: fmt.Sprintf("%d", p.Port)}, - }, - Ports: []corev1.ContainerPort{ - {ContainerPort: int32(p.Port)}, - }, - Image: p.Image, - ImagePullPolicy: corev1.PullIfNotPresent, - Command: p.Command, - }, - }, - Affinity: p.Affinity, - Tolerations: p.Tolerations, - }, - }, - Replicas: &replicas32, - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "name": p.Name, - "kind": p.Role, - }, - }, - }, - } -} - func NewTestContext(client kubernetes.Interface, config *rest.Config, clusterName string, o *options) *testContext { return &testContext{ client: client, config: config, clusterName: clusterName, antreaNamespace: o.antreaNamespace, - namespace: generateRandomNamespace(testNamespacePrefix), - } -} - -func generateRandomNamespace(baseName string) string { - const letters = "abcdefghijklmnopqrstuvwxyz0123456789" - bytes := make([]byte, 5) - _, err := rand.Read(bytes) - if err != nil { - panic(err) - } - for i, b := range bytes { - bytes[i] = letters[b%byte(len(letters))] - } - return fmt.Sprintf("%s-%s", baseName, string(bytes)) -} - -func (t *testContext) teardown(ctx context.Context) { - t.Log("Deleting post installation tests setup...") - t.client.CoreV1().Namespaces().Delete(ctx, t.namespace, metav1.DeleteOptions{}) - t.Log("Waiting for Namespace %s to disappear", t.namespace) - err := wait.PollUntilContextTimeout(ctx, 2*time.Second, 1*time.Minute, true, func(ctx context.Context) (bool, error) { - _, err := t.client.CoreV1().Namespaces().Get(ctx, t.namespace, metav1.GetOptions{}) - if err != nil { - return true, nil - } - return false, nil - }) - if err != nil { - t.Log("Setup deletion failed") - } else { - t.Log("Setup deletion successful") + namespace: check.GenerateRandomNamespace(testNamespacePrefix), } } @@ -286,7 +190,7 @@ func (t *testContext) setup(ctx context.Context) error { Effect: "NoSchedule", }, } - echoDeployment := newDeployment(deploymentParameters{ + echoDeployment := check.NewDeployment(check.DeploymentParameters{ Name: echoSameNodeDeploymentName, Role: kindEchoName, Port: 80, @@ -318,7 +222,7 @@ func (t *testContext) setup(ctx context.Context) error { return fmt.Errorf("unable to create Deployment %s: %s", echoSameNodeDeploymentName, err) } t.Log("Deploying client Deployment %s...", clientDeploymentName) - clientDeployment := newDeployment(deploymentParameters{ + clientDeployment := check.NewDeployment(check.DeploymentParameters{ Name: clientDeploymentName, Role: kindClientName, Image: deploymentImage, @@ -331,7 +235,7 @@ func (t *testContext) setup(ctx context.Context) error { if err != nil { return fmt.Errorf("unable to create Deployment %s: %s", clientDeploymentName, err) } - echoOtherNodeDeployment := newDeployment(deploymentParameters{ + echoOtherNodeDeployment := check.NewDeployment(check.DeploymentParameters{ Name: echoOtherNodeDeploymentName, Role: kindEchoName, Port: 80, @@ -369,7 +273,7 @@ func (t *testContext) setup(ctx context.Context) error { if err != nil { return fmt.Errorf("unable to create Deployment %s: %s", echoOtherNodeDeploymentName, err) } - if err := t.waitForDeploymentsReady(ctx, time.Second, podReadyTimeout, clientDeploymentName, echoSameNodeDeploymentName, echoOtherNodeDeploymentName); err != nil { + if err := check.WaitForDeploymentsReady(ctx, time.Second, podReadyTimeout, t.client, t.clusterName, t.namespace, clientDeploymentName, echoSameNodeDeploymentName, echoOtherNodeDeploymentName); err != nil { return err } podList, err := t.client.CoreV1().Pods(t.namespace).List(ctx, metav1.ListOptions{LabelSelector: "name=" + echoOtherNodeDeploymentName}) @@ -381,7 +285,7 @@ func (t *testContext) setup(ctx context.Context) error { } } else { t.Log("skipping other Node Deployments as multiple Nodes are not available") - if err := t.waitForDeploymentsReady(ctx, time.Second, podReadyTimeout, clientDeploymentName, echoSameNodeDeploymentName); err != nil { + if err := check.WaitForDeploymentsReady(ctx, time.Second, podReadyTimeout, t.client, t.clusterName, t.namespace, clientDeploymentName, echoSameNodeDeploymentName); err != nil { return err } } @@ -401,24 +305,6 @@ func (t *testContext) setup(ctx context.Context) error { return nil } -func (t *testContext) waitForDeploymentsReady(ctx context.Context, interval, timeout time.Duration, deployments ...string) error { - for _, deployment := range deployments { - t.Log("Waiting for Deployment %s to become ready...", deployment) - err := wait.PollUntilContextTimeout(ctx, interval, timeout, false, func(ctx context.Context) (bool, error) { - ready, err := check.DeploymentIsReady(ctx, t.client, t.namespace, deployment) - if err != nil { - return false, fmt.Errorf("error checking readiness of Deployment %s: %w", deployment, err) - } - return ready, nil - }) - if err != nil { - return fmt.Errorf("waiting for Deployment %s to become ready has been interrupted: %w", deployment, err) - } - t.Log("Deployment %s is ready.", deployment) - } - return nil -} - func (t *testContext) Log(format string, a ...interface{}) { fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", t.clusterName)+format+"\n", a...) } diff --git a/pkg/antctl/raw/check/util.go b/pkg/antctl/raw/check/util.go index 2a6e29936e6..ba589a0ab26 100644 --- a/pkg/antctl/raw/check/util.go +++ b/pkg/antctl/raw/check/util.go @@ -17,11 +17,15 @@ package check import ( "bytes" "context" + "crypto/rand" "fmt" + "os" + "time" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" @@ -103,3 +107,132 @@ func ExecInPod(ctx context.Context, client kubernetes.Interface, config *rest.Co } return stdout.String(), stderr.String(), nil } + +func NewDeployment(p DeploymentParameters) *appsv1.Deployment { + if p.Replicas == 0 { + p.Replicas = 1 + } + replicas32 := int32(p.Replicas) + var ports []corev1.ContainerPort + if p.Port > 0 { + ports = append(ports, corev1.ContainerPort{ContainerPort: int32(p.Port)}) + } + var env []corev1.EnvVar + if p.Port > 0 { + env = append(env, corev1.EnvVar{Name: "PORT", Value: fmt.Sprintf("%d", p.Port)}) + } + if p.Labels == nil { + p.Labels = make(map[string]string) + } + p.Labels["name"] = p.Name + p.Labels["kind"] = p.Role + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: p.Name, + Labels: p.Labels, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas32, + Selector: &metav1.LabelSelector{ + MatchLabels: p.Labels, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: p.Labels, + }, + Spec: corev1.PodSpec{ + HostNetwork: p.HostNetwork, + NodeSelector: p.NodeSelector, + Containers: []corev1.Container{ + { + Name: p.Name, + Image: p.Image, + Ports: ports, + Env: env, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: p.Command, + Args: p.Args, + VolumeMounts: p.VolumeMounts, + SecurityContext: p.SecurityContext, + }, + }, + Tolerations: p.Tolerations, + Volumes: p.Volumes, + Affinity: p.Affinity, + }, + }, + }, + } +} + +type DeploymentParameters struct { + Name string + Role string + Image string + Replicas int + Port int + Command []string + Args []string + Affinity *corev1.Affinity + Tolerations []corev1.Toleration + Labels map[string]string + VolumeMounts []corev1.VolumeMount + Volumes []corev1.Volume + HostNetwork bool + NodeSelector map[string]string + SecurityContext *corev1.SecurityContext +} + +func WaitForDeploymentsReady(ctx context.Context, + interval, timeout time.Duration, + client kubernetes.Interface, + clusterName string, + namespace string, + deployments ...string) error { + for _, deployment := range deployments { + fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", clusterName)+"Waiting for Deployment %s to become ready...\n", deployment) + err := wait.PollUntilContextTimeout(ctx, interval, timeout, false, func(ctx context.Context) (bool, error) { + ready, err := DeploymentIsReady(ctx, client, namespace, deployment) + if err != nil { + return false, fmt.Errorf("error checking readiness of Deployment %s: %w", deployment, err) + } + return ready, nil + }) + if err != nil { + return fmt.Errorf("waiting for Deployment %s to become ready has been interrupted: %w", deployment, err) + } + fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", clusterName)+"Deployment %s is ready.\n", deployment) + } + return nil +} + +func GenerateRandomNamespace(baseName string) string { + const letters = "abcdefghijklmnopqrstuvwxyz0123456789" + bytes := make([]byte, 5) + _, err := rand.Read(bytes) + if err != nil { + panic(err) + } + for i, b := range bytes { + bytes[i] = letters[b%byte(len(letters))] + } + return fmt.Sprintf("%s-%s", baseName, string(bytes)) +} + +func Teardown(ctx context.Context, client kubernetes.Interface, clusterName string, namespace string) { + fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", clusterName)+"Deleting installation tests setup...\n") + client.CoreV1().Namespaces().Delete(ctx, namespace, metav1.DeleteOptions{}) + fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", clusterName)+"Waiting for Namespace %s to be deleted\n", namespace) + err := wait.PollUntilContextTimeout(ctx, 2*time.Second, 1*time.Minute, true, func(ctx context.Context) (bool, error) { + _, err := client.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) + if err != nil { + return true, nil + } + return false, nil + }) + if err != nil { + fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", clusterName)+"Setup deletion failed\n") + } else { + fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", clusterName)+"Setup deletion successful\n") + } +}