From 9bb0e1c56ced2d76ba56f0cead5be55daa553f52 Mon Sep 17 00:00:00 2001 From: "Giau. Tran Minh" Date: Wed, 2 Oct 2024 20:47:06 +0700 Subject: [PATCH] atlas/schema: support `cloud` config block --- api/v1alpha1/atlasschema_types.go | 2 ++ api/v1alpha1/zz_generated.deepcopy.go | 1 + charts/atlas-operator/templates/crds/crd.yaml | 34 +++++++++++++++++++ .../crd/bases/db.atlasgo.io_atlasschemas.yaml | 34 +++++++++++++++++++ config/manager/kustomization.yaml | 6 ++++ go.mod | 2 +- go.sum | 4 +-- .../controller/atlasmigration_controller.go | 2 +- .../atlasmigration_controller_test.go | 2 +- internal/controller/atlasschema_controller.go | 27 ++++++++++++--- internal/controller/common.go | 2 ++ internal/controller/lint.go | 2 +- internal/controller/testhelper.go | 5 +++ test/e2e/testdata/atlas-schema/registry.txtar | 10 ++++-- 14 files changed, 121 insertions(+), 12 deletions(-) diff --git a/api/v1alpha1/atlasschema_types.go b/api/v1alpha1/atlasschema_types.go index 6b6228ad..34050d8d 100644 --- a/api/v1alpha1/atlasschema_types.go +++ b/api/v1alpha1/atlasschema_types.go @@ -66,6 +66,8 @@ type ( TargetSpec `json:",inline"` // Desired Schema of the target. Schema Schema `json:"schema,omitempty"` + // Cloud defines the Atlas Cloud configuration. + Cloud Cloud `json:"cloud,omitempty"` // +optional // DevURL is the URL of the database to use for normalization and calculations. // If not specified, the operator will spin up a temporary database container to use for these operations. diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 18428f8a..e44ff121 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -193,6 +193,7 @@ func (in *AtlasSchemaSpec) DeepCopyInto(out *AtlasSchemaSpec) { *out = *in in.TargetSpec.DeepCopyInto(&out.TargetSpec) in.Schema.DeepCopyInto(&out.Schema) + in.Cloud.DeepCopyInto(&out.Cloud) in.DevURLFrom.DeepCopyInto(&out.DevURLFrom) if in.Exclude != nil { in, out := &in.Exclude, &out.Exclude diff --git a/charts/atlas-operator/templates/crds/crd.yaml b/charts/atlas-operator/templates/crds/crd.yaml index d9991cc0..12b69965 100644 --- a/charts/atlas-operator/templates/crds/crd.yaml +++ b/charts/atlas-operator/templates/crds/crd.yaml @@ -442,6 +442,40 @@ spec: spec: description: AtlasSchemaSpec defines the desired state of AtlasSchema properties: + cloud: + description: Cloud defines the Atlas Cloud configuration. + properties: + project: + type: string + tokenFrom: + description: TokenFrom defines a reference to a secret key that + contains the Atlas Cloud Token + properties: + secretKeyRef: + description: SecretKeyRef references to the key of a secret + in the same namespace. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + url: + type: string + type: object credentials: description: |- Credentials defines the credentials to use when connecting to the database. diff --git a/config/crd/bases/db.atlasgo.io_atlasschemas.yaml b/config/crd/bases/db.atlasgo.io_atlasschemas.yaml index 69f39244..02106106 100644 --- a/config/crd/bases/db.atlasgo.io_atlasschemas.yaml +++ b/config/crd/bases/db.atlasgo.io_atlasschemas.yaml @@ -59,6 +59,40 @@ spec: spec: description: AtlasSchemaSpec defines the desired state of AtlasSchema properties: + cloud: + description: Cloud defines the Atlas Cloud configuration. + properties: + project: + type: string + tokenFrom: + description: TokenFrom defines a reference to a secret key that + contains the Atlas Cloud Token + properties: + secretKeyRef: + description: SecretKeyRef references to the key of a secret + in the same namespace. + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key must + be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + url: + type: string + type: object credentials: description: |- Credentials defines the credentials to use when connecting to the database. diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 54b8c40a..0b2c017d 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -14,3 +14,9 @@ resources: - manager.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: ariga/atlas-operator + newTag: e2e diff --git a/go.mod b/go.mod index 82bfdddf..cbca8552 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ toolchain go1.22.0 require ( ariga.io/atlas v0.21.2-0.20240424054305-cf871e4a7c1e - ariga.io/atlas-go-sdk v0.6.3 + ariga.io/atlas-go-sdk v0.6.4-0.20241002105845-ab6583b472e8 github.com/rogpeppe/go-internal v1.13.1 github.com/stretchr/testify v1.9.0 golang.org/x/mod v0.18.0 diff --git a/go.sum b/go.sum index 01e678e7..0d1d270c 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ ariga.io/atlas v0.21.2-0.20240424054305-cf871e4a7c1e h1:3iOt1DD4c6IGRjxcm+C5IeI+pHo7HnqMoN14xjVjxsE= ariga.io/atlas v0.21.2-0.20240424054305-cf871e4a7c1e/go.mod h1:VPlcXdd4w2KqKnH54yEZcry79UAhpaWaxEsmn5JRNoE= -ariga.io/atlas-go-sdk v0.6.3 h1:szfRe9bAhMRfRapOVt9I0Ifdx6ROb7Cqmt/W4rPs3hw= -ariga.io/atlas-go-sdk v0.6.3/go.mod h1:9Q+/04PVyJHUse1lEE9Kp6E18xj/6mIzaUTcWYSjSnQ= +ariga.io/atlas-go-sdk v0.6.4-0.20241002105845-ab6583b472e8 h1:C5ZatdFRE+p5enmFpwlViDR6kNWqdcmtLLzyvhf320I= +ariga.io/atlas-go-sdk v0.6.4-0.20241002105845-ab6583b472e8/go.mod h1:9Q+/04PVyJHUse1lEE9Kp6E18xj/6mIzaUTcWYSjSnQ= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= diff --git a/internal/controller/atlasmigration_controller.go b/internal/controller/atlasmigration_controller.go index 8d4ba9d3..ab0d7746 100644 --- a/internal/controller/atlasmigration_controller.go +++ b/internal/controller/atlasmigration_controller.go @@ -256,7 +256,7 @@ func (r *AtlasMigrationReconciler) reconcile(ctx context.Context, data *migratio return err } defer wd.Close() - c, err := r.atlasClient(wd.Path(), nil) + c, err := r.atlasClient(wd.Path(), data.Cloud) if err != nil { return err } diff --git a/internal/controller/atlasmigration_controller_test.go b/internal/controller/atlasmigration_controller_test.go index d9f7d068..a13899cb 100644 --- a/internal/controller/atlasmigration_controller_test.go +++ b/internal/controller/atlasmigration_controller_test.go @@ -783,7 +783,7 @@ func TestReconcile_reconcile_baseline(t *testing.T) { t.Cleanup(func() { require.NoError(t, wd.Close()) }) - cli, err := tt.r.atlasClient(wd.Path(), nil) + cli, err := tt.r.atlasClient(wd.Path(), md.Cloud) require.NoError(t, err) report, err := cli.MigrateStatus(context.Background(), &atlasexec.MigrateStatusParams{ Env: "test", diff --git a/internal/controller/atlasschema_controller.go b/internal/controller/atlasschema_controller.go index e4a451c8..64414c5b 100644 --- a/internal/controller/atlasschema_controller.go +++ b/internal/controller/atlasschema_controller.go @@ -70,6 +70,7 @@ type ( Policy *dbv1alpha1.Policy TxMode dbv1alpha1.TransactionMode Desired *url.URL + Cloud *Cloud schema []byte } @@ -207,7 +208,7 @@ func (r *AtlasSchemaReconciler) Reconcile(ctx context.Context, req ctrl.Request) return result(err) } } - report, err := r.apply(ctx, wd.Path(), data, string(data.TxMode)) + report, err := r.apply(ctx, wd, data) if err != nil { res.SetNotReady("ApplyingSchema", err.Error()) r.recorder.Event(res, corev1.EventTypeWarning, "ApplyingSchema", err.Error()) @@ -245,6 +246,12 @@ func (r *AtlasSchemaReconciler) watchRefs(res *dbv1alpha1.AtlasSchema) { res.NamespacedName(), ) } + if s := res.Spec.Cloud.TokenFrom.SecretKeyRef; s != nil { + r.secretWatcher.Watch( + types.NamespacedName{Name: s.Name, Namespace: res.Namespace}, + res.NamespacedName(), + ) + } if s := res.Spec.URLFrom.SecretKeyRef; s != nil { r.secretWatcher.Watch( types.NamespacedName{Name: s.Name, Namespace: res.Namespace}, @@ -265,15 +272,15 @@ func (r *AtlasSchemaReconciler) watchRefs(res *dbv1alpha1.AtlasSchema) { } } -func (r *AtlasSchemaReconciler) apply(ctx context.Context, dir string, data *managedData, txMode string) (*atlasexec.SchemaApply, error) { - cli, err := r.atlasClient(dir, nil) +func (r *AtlasSchemaReconciler) apply(ctx context.Context, wd *atlasexec.WorkingDir, data *managedData) (*atlasexec.SchemaApply, error) { + cli, err := r.atlasClient(wd.Path(), data.Cloud) if err != nil { return nil, err } return cli.SchemaApply(ctx, &atlasexec.SchemaApplyParams{ Env: data.EnvName, To: data.Desired.String(), - TxMode: txMode, + TxMode: string(data.TxMode), AutoApprove: true, }) } @@ -282,6 +289,7 @@ func (r *AtlasSchemaReconciler) apply(ctx context.Context, dir string, data *man func (r *AtlasSchemaReconciler) extractData(ctx context.Context, res *dbv1alpha1.AtlasSchema) (_ *managedData, err error) { var ( s = res.Spec + c = s.Cloud data = &managedData{ EnvName: defaultEnvName, DevURL: s.DevURL, @@ -291,6 +299,17 @@ func (r *AtlasSchemaReconciler) extractData(ctx context.Context, res *dbv1alpha1 TxMode: s.TxMode, } ) + if s := c.TokenFrom.SecretKeyRef; s != nil { + token, err := getSecretValue(ctx, r, res.Namespace, s) + if err != nil { + return nil, err + } + data.Cloud = &Cloud{ + Token: token, + Repo: c.Project, + URL: c.URL, + } + } data.URL, err = s.DatabaseURL(ctx, r, res.Namespace) if err != nil { return nil, transient(err) diff --git a/internal/controller/common.go b/internal/controller/common.go index f15f61ab..f705794a 100644 --- a/internal/controller/common.go +++ b/internal/controller/common.go @@ -51,6 +51,8 @@ type ( SchemaApply(ctx context.Context, params *atlasexec.SchemaApplyParams) (*atlasexec.SchemaApply, error) SchemaInspect(ctx context.Context, params *atlasexec.SchemaInspectParams) (string, error) + + WhoAmI(ctx context.Context) (*atlasexec.WhoAmI, error) } // AtlasExecFn is a function that returns an AtlasExec // with the working directory. diff --git a/internal/controller/lint.go b/internal/controller/lint.go index f29430a4..88320daf 100644 --- a/internal/controller/lint.go +++ b/internal/controller/lint.go @@ -35,7 +35,7 @@ const lintDirName = "lint-migrations" // - 2.sql: the pending changes. // Then it runs `atlas migrate lint` in the temporary directory. func (r *AtlasSchemaReconciler) lint(ctx context.Context, wd *atlasexec.WorkingDir, data *managedData, vars atlasexec.VarArgs) error { - cli, err := r.atlasClient(wd.Path(), nil) + cli, err := r.atlasClient(wd.Path(), data.Cloud) if err != nil { return err } diff --git a/internal/controller/testhelper.go b/internal/controller/testhelper.go index 92cb65a0..daaaea7c 100644 --- a/internal/controller/testhelper.go +++ b/internal/controller/testhelper.go @@ -53,12 +53,17 @@ type ( lint mockCmd[atlasexec.SummaryReport] status mockCmd[atlasexec.MigrateStatus] schemaApply mockCmd[atlasexec.SchemaApply] + whoami mockCmd[atlasexec.WhoAmI] schemaInspect mockCmd[string] } ) var _ AtlasExec = &mockAtlasExec{} +func (m *mockAtlasExec) WhoAmI(context.Context) (*atlasexec.WhoAmI, error) { + return m.whoami.res, m.whoami.err +} + // SchemaApply implements AtlasExec. func (m *mockAtlasExec) SchemaApply(ctx context.Context, params *atlasexec.SchemaApplyParams) (*atlasexec.SchemaApply, error) { return m.schemaApply.res, m.schemaApply.err diff --git a/test/e2e/testdata/atlas-schema/registry.txtar b/test/e2e/testdata/atlas-schema/registry.txtar index 54cc5738..33676515 100644 --- a/test/e2e/testdata/atlas-schema/registry.txtar +++ b/test/e2e/testdata/atlas-schema/registry.txtar @@ -5,8 +5,9 @@ kubectl apply -f - # Wait for the DB ready before creating the schema kubectl wait --for=condition=ready --timeout=60s -l app=postgres pods -# Set the ATLAS_TOKEN for the controller, this is required to read from the registry -kubectl set env -n atlas-operator-system ${CONTROLLER} ATLAS_TOKEN=${ATLAS_TOKEN} + +# Create the secret to store ATLAS_TOKEN +kubectl create secret generic atlas-token --from-literal=ATLAS_TOKEN=${ATLAS_TOKEN} # Create the schema kubectl apply -f schema.yaml @@ -76,6 +77,11 @@ spec: key: url schema: url: atlas://atlas-operator?tag=registry-v1 + cloud: + tokenFrom: + secretKeyRef: + name: atlas-token + key: ATLAS_TOKEN -- secret.yaml -- apiVersion: v1 kind: Secret