From 139d85508a8934629f15eb8b3d6620f6a05aa4fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Ma=C5=82ek?= Date: Fri, 16 Aug 2024 16:17:01 +0200 Subject: [PATCH] tests: use DockerHub token in tests' kind clusters --- .github/workflows/tests.yaml | 8 ++ pkg/utils/test/setup_helpers.go | 7 ++ test/e2e/environment.go | 7 ++ test/helpers/dockerconfig.go | 119 ++++++++++++++++++++++++++++++ test/helpers/dockerconfig_test.go | 51 +++++++++++++ 5 files changed, 192 insertions(+) create mode 100644 test/helpers/dockerconfig.go create mode 100644 test/helpers/dockerconfig_test.go diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index d2bd6a290..921042c65 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -291,6 +291,8 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} KONG_TEST_KONNECT_ACCESS_TOKEN: ${{ secrets.KONG_TEST_KONNECT_ACCESS_TOKEN }} KONG_TEST_KONNECT_SERVER_URL: us.api.konghq.tech + DOCKERHUB_PULL_TOKEN: ${{ secrets.DOCKERHUB_PULL_TOKEN }} + DOCKERHUB_PULL_USERNAME: ${{ vars.DOCKERHUB_PULL_USERNAME }} - name: upload diagnostics if: always() @@ -343,6 +345,8 @@ jobs: KONG_CONTROLLER_OUT: stdout GOTESTSUM_JUNITFILE: integration-tests-bluegreen-webhook-enabled-${{ matrix.webhook-enabled }}.xml GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DOCKERHUB_PULL_TOKEN: ${{ secrets.DOCKERHUB_PULL_TOKEN }} + DOCKERHUB_PULL_USERNAME: ${{ vars.DOCKERHUB_PULL_USERNAME }} - name: upload diagnostics if: always() @@ -393,6 +397,8 @@ jobs: KONG_PLUGIN_IMAGE_REGISTRY_CREDENTIALS: ${{ secrets.KONG_PLUGIN_IMAGE_REGISTRY_CREDENTIALS }} GOTESTSUM_JUNITFILE: integration-tests-provision-dataplane-fail.xml GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DOCKERHUB_PULL_TOKEN: ${{ secrets.DOCKERHUB_PULL_TOKEN }} + DOCKERHUB_PULL_USERNAME: ${{ vars.DOCKERHUB_PULL_USERNAME }} - name: upload diagnostics if: always() @@ -445,6 +451,8 @@ jobs: KONG_TEST_GATEWAY_OPERATOR_IMAGE_LOAD: gateway-operator:e2e-${{ github.sha }} GOTESTSUM_JUNITFILE: "e2e-tests.xml" GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DOCKERHUB_PULL_TOKEN: ${{ secrets.DOCKERHUB_PULL_TOKEN }} + DOCKERHUB_PULL_USERNAME: ${{ vars.DOCKERHUB_PULL_USERNAME }} - name: upload diagnostics if: always() diff --git a/pkg/utils/test/setup_helpers.go b/pkg/utils/test/setup_helpers.go index 82585e0c5..21a77c834 100644 --- a/pkg/utils/test/setup_helpers.go +++ b/pkg/utils/test/setup_helpers.go @@ -28,6 +28,7 @@ import ( "github.com/kong/gateway-operator/modules/manager" operatorclient "github.com/kong/gateway-operator/pkg/clientset" + "github.com/kong/gateway-operator/test/helpers" ) const ( @@ -102,6 +103,12 @@ func BuildEnvironment(ctx context.Context, existingCluster string, builderOpts . func buildEnvironmentOnNewKindCluster(ctx context.Context, builderOpts ...BuilderOpt) (environments.Environment, error) { builder := environments.NewBuilder() + kindBuilder := kind.NewBuilder() + if configFile, err := helpers.CreateKindConfigWithDockerCredentialsBasedOnEnvVars(ctx); err == nil { + kindBuilder.WithConfig(configFile) + builder.WithClusterBuilder(kindBuilder) + } + for _, o := range builderOpts { o(builder, kind.KindClusterType) } diff --git a/test/e2e/environment.go b/test/e2e/environment.go index 52bac5f2c..2e55e2fc1 100644 --- a/test/e2e/environment.go +++ b/test/e2e/environment.go @@ -173,6 +173,13 @@ func CreateEnvironment(t *testing.T, ctx context.Context, opts ...TestEnvOption) if !test.IsMetalLBDisabled() { builder.WithAddons(metallb.New()) } + + kindBuilder := kind.NewBuilder() + if configFile, err := helpers.CreateKindConfigWithDockerCredentialsBasedOnEnvVars(ctx); err == nil { + kindBuilder.WithConfig(configFile) + builder.WithClusterBuilder(kindBuilder) + t.Logf("using kind config file (%s)", configFile) + } } if imageLoad != "" { imageLoader, err := loadimage.NewBuilder().WithImage(imageLoad) diff --git a/test/helpers/dockerconfig.go b/test/helpers/dockerconfig.go new file mode 100644 index 000000000..43a2c9a55 --- /dev/null +++ b/test/helpers/dockerconfig.go @@ -0,0 +1,119 @@ +package helpers + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "os" + "path/filepath" +) + +type dockerRegistryConfig struct { + Auths map[string]DockerRegistryAuth `json:"auths"` +} + +// DockerRegistryAuth represents the auth field in the docker registry config. +type DockerRegistryAuth struct { + Auth string `json:"auth"` +} + +// DockerRegistryConfigManager is a helper to manage the docker registry config +// for a k8s secret of type kubernetes.io/dockerconfigjson. +type DockerRegistryConfigManager struct { + config dockerRegistryConfig +} + +// NewDockerRegistryConfigManager creates a new DockerRegistryConfigManager. +func NewDockerRegistryConfigManager() *DockerRegistryConfigManager { + return &DockerRegistryConfigManager{ + config: dockerRegistryConfig{ + Auths: map[string]DockerRegistryAuth{}, + }, + } +} + +// Add adds new registry credentials to the config. +func (c *DockerRegistryConfigManager) Add( + registry string, + username string, + token string, +) error { + auth := fmt.Sprintf("%s:%s", username, token) + authEncoded := base64.StdEncoding.EncodeToString([]byte(auth)) + c.config.Auths[registry] = DockerRegistryAuth{ + Auth: authEncoded, + } + return nil +} + +// EncodeForRegcred encodes the config for a k8s secret of type kubernetes.io/dockerconfigjson. +func (c *DockerRegistryConfigManager) EncodeForRegcred() ([]byte, error) { + b, err := json.Marshal(c.config) + if err != nil { + return nil, err + } + // out := make([]byte, base64.StdEncoding.EncodedLen(len(b))) + // base64.StdEncoding.Encode(out, b) + return b, nil +} + +// MissingDockerHubEnvVarError is an error type for missing required env vars for DockerHub pull secret. +type MissingDockerHubEnvVarError struct{} + +// Error returns the error message. +func (e MissingDockerHubEnvVarError) Error() string { + return "missing required env vars for DockerHub pull secret" +} + +// CreateKindConfigWithDockerCredentialsBasedOnEnvVars creates a kind config with docker credentials. +func CreateKindConfigWithDockerCredentialsBasedOnEnvVars( + ctx context.Context, +) (string, error) { + const ( + kindConfigTemplate = `kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane + extraMounts: + - containerPath: /var/lib/kubelet/config.json + hostPath: %s` + ) + + dockerHubUser := os.Getenv("DOCKERHUB_PULL_USERNAME") + dockerHubToken := os.Getenv("DOCKERHUB_PULL_TOKEN") + if dockerHubUser == "" || dockerHubToken == "" { + return "", MissingDockerHubEnvVarError{} + } + + regCfgMgr := NewDockerRegistryConfigManager() + if err := regCfgMgr.Add("https://index.docker.io/v1/", dockerHubUser, dockerHubToken); err != nil { + return "", err + } + + cfgEncoded, err := regCfgMgr.EncodeForRegcred() + if err != nil { + return "", err + } + + dockerConfigDir, err := os.MkdirTemp("", "dockerconfig") + if err != nil { + return "", err + } + dockerConfigFileName := filepath.Join(dockerConfigDir, "config.json") + if err := os.WriteFile(dockerConfigFileName, cfgEncoded, 0o777); err != nil { //nolint:gosec + return "", err + } + + kindConfigDir, err := os.MkdirTemp("", "kindconfig") + if err != nil { + return "", err + } + + kindConfigFileName := filepath.Join(kindConfigDir, "config.yaml") + if err := os.WriteFile(kindConfigFileName, []byte(fmt.Sprintf(kindConfigTemplate, dockerConfigFileName)), 0o777); err != nil { //nolint:gosec + return "", err + } + + return kindConfigFileName, err +} diff --git a/test/helpers/dockerconfig_test.go b/test/helpers/dockerconfig_test.go new file mode 100644 index 000000000..c95395e0f --- /dev/null +++ b/test/helpers/dockerconfig_test.go @@ -0,0 +1,51 @@ +package helpers + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDockerRegistryConfigManager(t *testing.T) { + tests := []struct { + name string + update func(*testing.T, *DockerRegistryConfigManager) + expectedJSON string + expectedError error + }{ + { + name: "Empty", + expectedJSON: `{"auths":{}}`, + }, + { + name: "Valid configuration", + update: func(t *testing.T, c *DockerRegistryConfigManager) { + require.NoError(t, + c.Add( + "registry1", + "username1", + "token1", + ), + ) + }, + expectedJSON: `{"auths":{"registry1":{"auth":"dXNlcm5hbWUxOnRva2VuMQ=="}}}`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mgr := NewDockerRegistryConfigManager() + if tt.update != nil { + tt.update(t, mgr) + } + + jsonData, err := mgr.EncodeForRegcred() + if tt.expectedError != nil { + require.Error(t, err) + return + } + require.NoError(t, err) + require.Equal(t, tt.expectedJSON, string(jsonData)) + }) + } +}