feat: add support for variables in project & support resource templat…

…ing (#277) * feat: add support for variables in project * feat: add namespace variables to fix tests * feat: add namespace variable integration * feat: add compile resource spec logic on services * feat: add compile logic to Deploy API * feat: use variables for templating * feat: add unit test for empty vars & refactor GetVariables * fix: rename CompileEngine to TemplateCompiler
goto · Oct 18, 2024 · ffd4dee · ffd4dee
1 parent b5d007e
commit ffd4dee
Show file tree

Hide file tree

Showing 61 changed files with 1,130 additions and 448 deletions.
diff --git a/Makefile b/Makefile
@@ -5,7 +5,7 @@ NAME = "github.com/goto/optimus"
 LAST_COMMIT := $(shell git rev-parse --short HEAD)
 LAST_TAG := "$(shell git rev-list --tags --max-count=1)"
 OPMS_VERSION := "$(shell git describe --tags ${LAST_TAG})-next"
-PROTON_COMMIT := "7bfbb3b40583db73a7f891e2b7db61931ff2c6a5"
+PROTON_COMMIT := "64b8a59fc433a186f5e8141624298875e7f95f4e"
 
 
 .PHONY: build test test-ci generate-proto unit-test-ci integration-test vet coverage clean install lint

diff --git a/client/cmd/namespace/describe.go b/client/cmd/namespace/describe.go
@@ -123,8 +123,9 @@ func (d *describeCommand) getNamespace() (*config.Namespace, error) {
 		return nil, fmt.Errorf("unable to get namespace [%s]: %w", d.namespaceName, err)
 	}
 	return &config.Namespace{
-		Name:   response.GetNamespace().Name,
-		Config: response.GetNamespace().Config,
+		Name:      response.GetNamespace().Name,
+		Config:    response.GetNamespace().Config,
+		Variables: response.GetNamespace().Variables,
 	}, nil
 }
 
@@ -138,5 +139,16 @@ func (*describeCommand) stringifyNamespace(namespace *config.Namespace) string {
 			output += fmt.Sprintf("\t%s: %s", key, value)
 		}
 	}
+
+	output += "\n"
+
+	if len(namespace.Variables) == 0 {
+		output += "variables: {}"
+	} else {
+		output += "variables:\n"
+		for key, value := range namespace.Variables {
+			output += fmt.Sprintf("\t%s: %s", key, value)
+		}
+	}
 	return output
 }
diff --git a/client/cmd/namespace/list.go b/client/cmd/namespace/list.go
@@ -125,8 +125,9 @@ func (l *listCommand) listNamespacesFromServer(serverHost, projectName string) (
 	output := make([]*config.Namespace, len(response.Namespaces))
 	for i, n := range response.Namespaces {
 		output[i] = &config.Namespace{
-			Name:   n.GetName(),
-			Config: n.GetConfig(),
+			Name:      n.GetName(),
+			Config:    n.GetConfig(),
+			Variables: n.GetVariables(),
 		}
 	}
 	return output, nil

diff --git a/client/cmd/namespace/register.go b/client/cmd/namespace/register.go
@@ -111,8 +111,9 @@ func RegisterNamespace(l log.Logger, conn *grpc.ClientConn, projectName string,
 	_, err := namespaceServiceClient.RegisterProjectNamespace(ctx, &pb.RegisterProjectNamespaceRequest{
 		ProjectName: projectName,
 		Namespace: &pb.NamespaceSpecification{
-			Name:   namespace.Name,
-			Config: namespace.Config,
+			Name:      namespace.Name,
+			Config:    namespace.Config,
+			Variables: namespace.Variables,
 		},
 	})
 	if err != nil {

diff --git a/client/cmd/project/describe.go b/client/cmd/project/describe.go
@@ -120,7 +120,8 @@ func (d *describeCommand) getProject() (config.Project, error) {
 		return project, err
 	}
 	return config.Project{
-		Name:   response.GetProject().Name,
-		Config: response.GetProject().Config,
+		Name:      response.GetProject().Name,
+		Config:    response.GetProject().Config,
+		Variables: response.GetProject().Variables,
 	}, nil
 }
diff --git a/client/cmd/project/register.go b/client/cmd/project/register.go
@@ -91,9 +91,10 @@ func RegisterProject(logger log.Logger, conn *grpc.ClientConn, project config.Pr
 
 	projectServiceClient := pb.NewProjectServiceClient(conn)
 	projectSpec := &pb.ProjectSpecification{
-		Name:    project.Name,
-		Config:  project.Config,
-		Presets: toPresetProto(presets),
+		Name:      project.Name,
+		Config:    project.Config,
+		Variables: project.Variables,
+		Presets:   toPresetProto(presets),
 	}
 
 	ctx, cancelFunc := context.WithTimeout(context.Background(), registerTimeout)

diff --git a/config/config_client.go b/config/config_client.go
@@ -29,6 +29,7 @@ type Job struct {
 type Project struct {
 	Name        string            `mapstructure:"name"`
 	Config      map[string]string `mapstructure:"config"`
+	Variables   map[string]string `mapstructure:"variables"`
 	PresetsPath string            `mapstructure:"preset_path"`
 }
 
@@ -40,6 +41,7 @@ type Auth struct {
 type Namespace struct {
 	Name      string            `mapstructure:"name"`
 	Config    map[string]string `mapstructure:"config"`
+	Variables map[string]string `mapstructure:"variables"`
 	Job       Job               `mapstructure:"job"`
 	Datastore []Datastore       `mapstructure:"datastore"`
 }

diff --git a/core/job/handler/v1beta1/job_test.go b/core/job/handler/v1beta1/job_test.go
@@ -33,11 +33,11 @@ func TestNewJobHandler(t *testing.T) {
 			"bucket":                     "gs://some_folder-2",
 			tenant.ProjectSchedulerHost:  "host",
 			tenant.ProjectStoragePathKey: "gs://location",
-		}) // TODO: add test for presets
+		}, map[string]string{}) // TODO: add test for presets
 	namespace, _ := tenant.NewNamespace("test-ns", project.Name(),
 		map[string]string{
 			"bucket": "gs://ns_bucket",
-		})
+		}, map[string]string{})
 	sampleTenant, _ := tenant.NewTenant(project.Name().String(), namespace.Name().String())
 	jobVersion := 1
 	startDate, err := job.ScheduleDateFrom("2022-10-01")

diff --git a/core/job/job_test.go b/core/job/job_test.go
@@ -18,11 +18,11 @@ func TestEntityJob(t *testing.T) {
 			"bucket":                     "gs://some_folder-2",
 			tenant.ProjectSchedulerHost:  "host",
 			tenant.ProjectStoragePathKey: "gs://location",
-		})
+		}, map[string]string{})
 	namespace, _ := tenant.NewNamespace("test-ns", project.Name(),
 		map[string]string{
 			"bucket": "gs://ns_bucket",
-		})
+		}, map[string]string{})
 	sampleTenant, _ := tenant.NewTenant(project.Name().String(), namespace.Name().String())
 	jobVersion := 1
 	startDate, _ := job.ScheduleDateFrom("2022-10-01")

diff --git a/core/job/resolver/upstream_resolver_test.go b/core/job/resolver/upstream_resolver_test.go
@@ -24,11 +24,11 @@ func TestUpstreamResolver(t *testing.T) {
 			"bucket":                     "gs://some_folder-2",
 			tenant.ProjectSchedulerHost:  "host",
 			tenant.ProjectStoragePathKey: "gs://location",
-		})
+		}, map[string]string{})
 	namespace, _ := tenant.NewNamespace("test-ns", project.Name(),
 		map[string]string{
 			"bucket": "gs://ns_bucket",
-		})
+		}, map[string]string{})
 	sampleTenant, _ := tenant.NewTenant(project.Name().String(), namespace.Name().String())
 	externalTenant, _ := tenant.NewTenant("external-proj", "external-namespace")
 	jobVersion := 1

diff --git a/core/job/service/job_service.go b/core/job/service/job_service.go
@@ -1143,7 +1143,7 @@ func (j *JobService) generateJobs(ctx context.Context, tenantWithDetails *tenant
 
 func (j *JobService) compileConfigs(configs job.Config, tnnt *tenant.WithDetails) map[string]string {
 	tmplCtx := compiler.PrepareContext(
-		compiler.From(tnnt.GetConfigs()).WithName("proj").WithKeyPrefix(projectConfigPrefix),
+		compiler.From(tnnt.GetVariables()).WithName("proj").WithKeyPrefix(projectConfigPrefix),
 		compiler.From(tnnt.SecretsMap()).WithName("secret"),
 	)
 

diff --git a/core/job/service/job_service_test.go b/core/job/service/job_service_test.go
@@ -34,11 +34,11 @@ func TestJobService(t *testing.T) {
 			"bucket":                     "gs://some_folder-2",
 			tenant.ProjectSchedulerHost:  "host",
 			tenant.ProjectStoragePathKey: "gs://location",
-		})
+		}, map[string]string{})
 	namespace, _ := tenant.NewNamespace("test-ns", project.Name(),
 		map[string]string{
 			"bucket": "gs://ns_bucket",
-		})
+		}, map[string]string{})
 	secret1, err := tenant.NewPlainTextSecret("table_name", "secret_table")
 	assert.Nil(t, err)
 
@@ -48,7 +48,7 @@ func TestJobService(t *testing.T) {
 	otherNamespace, _ := tenant.NewNamespace("other-ns", project.Name(),
 		map[string]string{
 			"bucket": "gs://other_ns_bucket",
-		})
+		}, map[string]string{})
 	otherTenant, _ := tenant.NewTenant(project.Name().String(), otherNamespace.Name().String())
 	secret2, err := tenant.NewPlainTextSecret("bucket", "gs://some_secret_bucket")
 	assert.Nil(t, err)

diff --git a/core/resource/resource.go b/core/resource/resource.go
@@ -196,6 +196,10 @@ func (r *Resource) Version() int32 {
 	return r.metadata.Version
 }
 
+func (r *Resource) UpdateSpec(spec map[string]any) {
+	r.spec = spec
+}
+
 func (r *Resource) Equal(incoming *Resource) bool {
 	if r == nil || incoming == nil {
 		return r == nil && incoming == nil