From 776c4b2c25a5f6cd30613c90d4cdf3ab1945bd4b Mon Sep 17 00:00:00 2001 From: "Ahmad N. F." Date: Wed, 9 Oct 2024 17:48:31 +0700 Subject: [PATCH 1/8] feat: add support for variables in project --- Makefile | 2 +- client/cmd/project/describe.go | 5 +- client/cmd/project/register.go | 7 +- config/config_client.go | 1 + core/job/handler/v1beta1/job_test.go | 2 +- core/job/job_test.go | 2 +- core/job/resolver/upstream_resolver_test.go | 2 +- core/job/service/job_service_test.go | 2 +- .../scheduler/handler/v1beta1/job_run_test.go | 3 +- core/scheduler/service/events_service_test.go | 2 +- .../service/executor_input_compiler_test.go | 2 +- .../service/job_run_asset_compiler_test.go | 2 +- .../scheduler/service/job_run_service_test.go | 3 +- core/scheduler/service/replay_service_test.go | 3 +- core/tenant/handler/v1beta1/namespace_test.go | 3 +- core/tenant/handler/v1beta1/project.go | 14 +- core/tenant/handler/v1beta1/project_test.go | 5 +- core/tenant/project.go | 33 ++- core/tenant/project_test.go | 6 +- core/tenant/service/namespace_service_test.go | 3 +- core/tenant/service/project_service_test.go | 9 +- core/tenant/service/tenant_service_test.go | 3 +- core/tenant/tenant.go | 15 ++ core/tenant/tenant_test.go | 28 ++- ext/scheduler/airflow/dag/compiler_test.go | 6 +- .../store/postgres/job/job_repository_test.go | 4 +- ...ariables_in_project_and_namespace.down.sql | 2 + ..._variables_in_project_and_namespace.up.sql | 2 + .../postgres/resource/repository_test.go | 2 +- .../postgres/scheduler/job_repository_test.go | 2 +- .../tenant/namespace_repository_test.go | 2 +- .../postgres/tenant/preset_repository_test.go | 2 +- .../postgres/tenant/project_repository.go | 23 +- .../tenant/project_repository_test.go | 8 +- .../postgres/tenant/secret_repository_test.go | 2 +- .../optimus/core/v1beta1/namespace.pb.go | 160 ++++++++------ .../core/v1beta1/namespace.swagger.json | 6 + .../optimus/core/v1beta1/project.pb.go | 196 ++++++++++-------- .../optimus/core/v1beta1/project.swagger.json | 6 + 39 files changed, 360 insertions(+), 220 deletions(-) create mode 100644 internal/store/postgres/migrations/000064_add_variables_in_project_and_namespace.down.sql create mode 100644 internal/store/postgres/migrations/000064_add_variables_in_project_and_namespace.up.sql diff --git a/Makefile b/Makefile index 980b892efd..e30452de15 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ NAME = "github.com/goto/optimus" LAST_COMMIT := $(shell git rev-parse --short HEAD) LAST_TAG := "$(shell git rev-list --tags --max-count=1)" OPMS_VERSION := "$(shell git describe --tags ${LAST_TAG})-next" -PROTON_COMMIT := "ca87ce40849dfc2a7a73ba9d2e0e9084e33dadeb" +PROTON_COMMIT := "4e2b7f7482a86f0ae9c8dc9cbb813b57411a82df" .PHONY: build test test-ci generate-proto unit-test-ci integration-test vet coverage clean install lint diff --git a/client/cmd/project/describe.go b/client/cmd/project/describe.go index 1a7ebb4f17..01037adfdb 100644 --- a/client/cmd/project/describe.go +++ b/client/cmd/project/describe.go @@ -120,7 +120,8 @@ func (d *describeCommand) getProject() (config.Project, error) { return project, err } return config.Project{ - Name: response.GetProject().Name, - Config: response.GetProject().Config, + Name: response.GetProject().Name, + Config: response.GetProject().Config, + Variables: response.GetProject().Variables, }, nil } diff --git a/client/cmd/project/register.go b/client/cmd/project/register.go index e53a54806c..0c89ffeb3f 100644 --- a/client/cmd/project/register.go +++ b/client/cmd/project/register.go @@ -91,9 +91,10 @@ func RegisterProject(logger log.Logger, conn *grpc.ClientConn, project config.Pr projectServiceClient := pb.NewProjectServiceClient(conn) projectSpec := &pb.ProjectSpecification{ - Name: project.Name, - Config: project.Config, - Presets: toPresetProto(presets), + Name: project.Name, + Config: project.Config, + Variables: project.Variables, + Presets: toPresetProto(presets), } ctx, cancelFunc := context.WithTimeout(context.Background(), registerTimeout) diff --git a/config/config_client.go b/config/config_client.go index b070c66931..b21f0aac17 100644 --- a/config/config_client.go +++ b/config/config_client.go @@ -29,6 +29,7 @@ type Job struct { type Project struct { Name string `mapstructure:"name"` Config map[string]string `mapstructure:"config"` + Variables map[string]string `mapstructure:"variables"` PresetsPath string `mapstructure:"preset_path"` } diff --git a/core/job/handler/v1beta1/job_test.go b/core/job/handler/v1beta1/job_test.go index bcb54bfeea..7d544de4f3 100644 --- a/core/job/handler/v1beta1/job_test.go +++ b/core/job/handler/v1beta1/job_test.go @@ -33,7 +33,7 @@ func TestNewJobHandler(t *testing.T) { "bucket": "gs://some_folder-2", tenant.ProjectSchedulerHost: "host", tenant.ProjectStoragePathKey: "gs://location", - }) // TODO: add test for presets + }, map[string]string{}) // TODO: add test for presets namespace, _ := tenant.NewNamespace("test-ns", project.Name(), map[string]string{ "bucket": "gs://ns_bucket", diff --git a/core/job/job_test.go b/core/job/job_test.go index dbb7c8b016..06f732e64d 100644 --- a/core/job/job_test.go +++ b/core/job/job_test.go @@ -18,7 +18,7 @@ func TestEntityJob(t *testing.T) { "bucket": "gs://some_folder-2", tenant.ProjectSchedulerHost: "host", tenant.ProjectStoragePathKey: "gs://location", - }) + }, map[string]string{}) namespace, _ := tenant.NewNamespace("test-ns", project.Name(), map[string]string{ "bucket": "gs://ns_bucket", diff --git a/core/job/resolver/upstream_resolver_test.go b/core/job/resolver/upstream_resolver_test.go index 10e506c7f7..cbde52da39 100644 --- a/core/job/resolver/upstream_resolver_test.go +++ b/core/job/resolver/upstream_resolver_test.go @@ -24,7 +24,7 @@ func TestUpstreamResolver(t *testing.T) { "bucket": "gs://some_folder-2", tenant.ProjectSchedulerHost: "host", tenant.ProjectStoragePathKey: "gs://location", - }) + }, map[string]string{}) namespace, _ := tenant.NewNamespace("test-ns", project.Name(), map[string]string{ "bucket": "gs://ns_bucket", diff --git a/core/job/service/job_service_test.go b/core/job/service/job_service_test.go index f48722e775..39bf6ff5e3 100644 --- a/core/job/service/job_service_test.go +++ b/core/job/service/job_service_test.go @@ -34,7 +34,7 @@ func TestJobService(t *testing.T) { "bucket": "gs://some_folder-2", tenant.ProjectSchedulerHost: "host", tenant.ProjectStoragePathKey: "gs://location", - }) + }, map[string]string{}) namespace, _ := tenant.NewNamespace("test-ns", project.Name(), map[string]string{ "bucket": "gs://ns_bucket", diff --git a/core/scheduler/handler/v1beta1/job_run_test.go b/core/scheduler/handler/v1beta1/job_run_test.go index 581fd1604e..ac8226f569 100644 --- a/core/scheduler/handler/v1beta1/job_run_test.go +++ b/core/scheduler/handler/v1beta1/job_run_test.go @@ -647,8 +647,9 @@ func TestJobRunHandler(t *testing.T) { "STORAGE_PATH": "file://", "SCHEDULER_HOST": "http://scheduler", } + projectVars := map[string]string{} - project, err := tenant.NewProject(projectName, projectConfig) + project, err := tenant.NewProject(projectName, projectConfig, projectVars) assert.NotNil(t, project) assert.NoError(t, err) diff --git a/core/scheduler/service/events_service_test.go b/core/scheduler/service/events_service_test.go index 1be2b6cab6..d498895574 100644 --- a/core/scheduler/service/events_service_test.go +++ b/core/scheduler/service/events_service_test.go @@ -23,7 +23,7 @@ func TestNotificationService(t *testing.T) { project, _ := tenant.NewProject("proj1", map[string]string{ "STORAGE_PATH": "somePath", "SCHEDULER_HOST": "localhost", - }) + }, map[string]string{}) namespace, _ := tenant.NewNamespace("ns1", project.Name(), map[string]string{}) tnnt, _ := tenant.NewTenant(project.Name().String(), namespace.Name().String()) startDate, _ := time.Parse(time.RFC3339, "2022-03-20T02:00:00+00:00") diff --git a/core/scheduler/service/executor_input_compiler_test.go b/core/scheduler/service/executor_input_compiler_test.go index 167078bdde..22154bfa54 100644 --- a/core/scheduler/service/executor_input_compiler_test.go +++ b/core/scheduler/service/executor_input_compiler_test.go @@ -28,7 +28,7 @@ func TestExecutorCompiler(t *testing.T) { project, _ := tenant.NewProject("proj1", map[string]string{ "STORAGE_PATH": "somePath", "SCHEDULER_HOST": "localhost", - }) + }, map[string]string{}) namespace, _ := tenant.NewNamespace("ns1", project.Name(), map[string]string{}) secret1, _ := tenant.NewPlainTextSecret("secretName", "secretValue") diff --git a/core/scheduler/service/job_run_asset_compiler_test.go b/core/scheduler/service/job_run_asset_compiler_test.go index fe37227a8c..a938a6a1c6 100644 --- a/core/scheduler/service/job_run_asset_compiler_test.go +++ b/core/scheduler/service/job_run_asset_compiler_test.go @@ -23,7 +23,7 @@ func TestJobAssetsCompiler(t *testing.T) { project, _ := tenant.NewProject("proj1", map[string]string{ "STORAGE_PATH": "somePath", "SCHEDULER_HOST": "localhost", - }) + }, map[string]string{}) namespace, _ := tenant.NewNamespace("ns1", project.Name(), map[string]string{}) tnnt, _ := tenant.NewTenant(project.Name().String(), namespace.Name().String()) currentTime := time.Now() diff --git a/core/scheduler/service/job_run_service_test.go b/core/scheduler/service/job_run_service_test.go index 13dd10c574..cdd55733dc 100644 --- a/core/scheduler/service/job_run_service_test.go +++ b/core/scheduler/service/job_run_service_test.go @@ -1351,8 +1351,9 @@ func TestJobRunService(t *testing.T) { "STORAGE_PATH": "file://", "SCHEDULER_HOST": "http://scheduler", } + vars := map[string]string{} - project, err := tenant.NewProject(projName.String(), conf) + project, err := tenant.NewProject(projName.String(), conf, vars) assert.NotNil(t, project) assert.NoError(t, err) diff --git a/core/scheduler/service/replay_service_test.go b/core/scheduler/service/replay_service_test.go index 970ee39957..8307597983 100644 --- a/core/scheduler/service/replay_service_test.go +++ b/core/scheduler/service/replay_service_test.go @@ -62,8 +62,9 @@ func TestReplayService(t *testing.T) { "STORAGE_PATH": "file:///tmp/", "SCHEDULER_HOST": "http://localhost", } + projectVars := map[string]string{} namespaceEntity, _ := tenant.NewNamespace(namespaceName.String(), projName, namespaceCfg) - projectEntity, _ := tenant.NewProject(projName.String(), projectCfg) + projectEntity, _ := tenant.NewProject(projName.String(), projectCfg, projectVars) tenantWithDetails, _ := tenant.NewTenantDetails(projectEntity, namespaceEntity, tenant.PlainTextSecrets{}) taskNameToExecutionProjectMap := map[string]string{ diff --git a/core/tenant/handler/v1beta1/namespace_test.go b/core/tenant/handler/v1beta1/namespace_test.go index fa306d2765..25ac679933 100644 --- a/core/tenant/handler/v1beta1/namespace_test.go +++ b/core/tenant/handler/v1beta1/namespace_test.go @@ -22,7 +22,8 @@ func TestNamespaceHandler(t *testing.T) { tenant.ProjectStoragePathKey: "gs://location", "BUCKET": "gs://some_folder", } - savedProject, _ := tenant.NewProject("savedProj", projectConf) + projectVars := map[string]string{} + savedProject, _ := tenant.NewProject("savedProj", projectConf, projectVars) savedNS, _ := tenant.NewNamespace("savedNS", savedProject.Name(), map[string]string{"BUCKET": "gs://some_folder"}) t.Run("RegisterProjectNamespace", func(t *testing.T) { diff --git a/core/tenant/handler/v1beta1/project.go b/core/tenant/handler/v1beta1/project.go index c44b7bfbda..47a97322ea 100644 --- a/core/tenant/handler/v1beta1/project.go +++ b/core/tenant/handler/v1beta1/project.go @@ -91,6 +91,11 @@ func fromProjectProto(conf *pb.ProjectSpecification) (*tenant.Project, error) { pConf[strings.ToUpper(key)] = val } + variablesMap := map[string]string{} + for key, val := range conf.GetVariables() { + variablesMap[strings.ToUpper(key)] = val + } + presets := make(map[string]tenant.Preset, len(conf.GetPresets())) for name, preset := range conf.GetPresets() { lowerName := strings.ToLower(name) @@ -101,7 +106,7 @@ func fromProjectProto(conf *pb.ProjectSpecification) (*tenant.Project, error) { presets[lowerName] = newPreset } - project, err := tenant.NewProject(conf.GetName(), pConf) + project, err := tenant.NewProject(conf.GetName(), pConf, variablesMap) if err != nil { return nil, err } @@ -112,9 +117,10 @@ func fromProjectProto(conf *pb.ProjectSpecification) (*tenant.Project, error) { func toProjectProto(project *tenant.Project) *pb.ProjectSpecification { return &pb.ProjectSpecification{ - Name: project.Name().String(), - Config: project.GetConfigs(), - Presets: toProjectPresets(project.GetPresets()), + Name: project.Name().String(), + Config: project.GetConfigs(), + Variables: project.GetVariables(), + Presets: toProjectPresets(project.GetPresets()), } } diff --git a/core/tenant/handler/v1beta1/project_test.go b/core/tenant/handler/v1beta1/project_test.go index 62f05f53d6..8319062616 100644 --- a/core/tenant/handler/v1beta1/project_test.go +++ b/core/tenant/handler/v1beta1/project_test.go @@ -23,7 +23,8 @@ func TestProjectHandler(t *testing.T) { tenant.ProjectStoragePathKey: "gs://location", "BUCKET": "gs://some_folder", } - savedProject, _ := tenant.NewProject("savedProj", conf) + projectVars := map[string]string{} + savedProject, _ := tenant.NewProject("savedProj", conf, projectVars) t.Run("RegisterProject", func(t *testing.T) { t.Run("returns error when name is empty", func(t *testing.T) { @@ -153,7 +154,7 @@ func TestProjectHandler(t *testing.T) { "retrieve project [savedProj]") }) t.Run("returns the project successfully", func(t *testing.T) { - savedProj, _ := tenant.NewProject("savedProj", conf) + savedProj, _ := tenant.NewProject("savedProj", conf, projectVars) pres := tenant.NewPresetWithConfig("yesterday", "description", window.SimpleConfig{ Size: "1d", ShiftBy: "", diff --git a/core/tenant/project.go b/core/tenant/project.go index a5c3a23746..6529ae7b4e 100644 --- a/core/tenant/project.go +++ b/core/tenant/project.go @@ -1,6 +1,7 @@ package tenant import ( + "fmt" "strings" "github.com/goto/optimus/internal/errors" @@ -30,8 +31,9 @@ func (pn ProjectName) String() string { } type Project struct { - name ProjectName - config map[string]string + name ProjectName + config map[string]string + variables map[string]string presets map[string]Preset } @@ -58,6 +60,24 @@ func (p *Project) GetConfigs() map[string]string { return confs } +func (p *Project) GetVariable(key string) (string, error) { + for k, v := range p.variables { + if key == k { + return v, nil + } + } + return "", errors.NotFound(EntityProject, fmt.Sprintf("variable not found: %s", key)) +} + +// GetVariables returns a clone of project variables +func (p *Project) GetVariables() map[string]string { + vars := make(map[string]string, len(p.variables)) + for k, v := range p.variables { + vars[k] = v + } + return vars +} + func (p *Project) SetPresets(presets map[string]Preset) { if presets == nil { p.presets = make(map[string]Preset) @@ -80,7 +100,7 @@ func (p *Project) GetPreset(name string) (Preset, error) { return preset, nil } -func NewProject(name string, config map[string]string) (*Project, error) { +func NewProject(name string, config, variables map[string]string) (*Project, error) { prjName, err := ProjectNameFrom(name) if err != nil { return nil, err @@ -91,8 +111,9 @@ func NewProject(name string, config map[string]string) (*Project, error) { } return &Project{ - name: prjName, - config: config, - presets: make(map[string]Preset), + name: prjName, + config: config, + variables: variables, + presets: make(map[string]Preset), }, nil } diff --git a/core/tenant/project_test.go b/core/tenant/project_test.go index 25bbce0b18..9484985df1 100644 --- a/core/tenant/project_test.go +++ b/core/tenant/project_test.go @@ -25,14 +25,14 @@ func TestEntityProject(t *testing.T) { t.Run("Project", func(t *testing.T) { t.Run("fails to create if name is empty", func(t *testing.T) { - project, err := tenant.NewProject("", map[string]string{"a": "b"}) + project, err := tenant.NewProject("", map[string]string{"a": "b"}, map[string]string{}) assert.Nil(t, project) assert.NotNil(t, err) assert.EqualError(t, err, "invalid argument for entity project: project name is empty") }) t.Run("fails to create if config is empty", func(t *testing.T) { - project, err := tenant.NewProject("name", map[string]string{}) + project, err := tenant.NewProject("name", map[string]string{}, map[string]string{}) assert.Nil(t, project) assert.NotNil(t, err) @@ -42,7 +42,7 @@ func TestEntityProject(t *testing.T) { project, err := tenant.NewProject("t-optimus", map[string]string{ tenant.ProjectSchedulerHost: "b", tenant.ProjectStoragePathKey: "d", - }) + }, map[string]string{}) assert.Nil(t, err) assert.NotNil(t, project) diff --git a/core/tenant/service/namespace_service_test.go b/core/tenant/service/namespace_service_test.go index ce5dd1fb74..e514175541 100644 --- a/core/tenant/service/namespace_service_test.go +++ b/core/tenant/service/namespace_service_test.go @@ -19,7 +19,8 @@ func TestNamespaceService(t *testing.T) { tenant.ProjectStoragePathKey: "gs://location", "BUCKET": "gs://some_folder", } - savedProject, _ := tenant.NewProject("savedProj", conf) + projectVars := map[string]string{} + savedProject, _ := tenant.NewProject("savedProj", conf, projectVars) savedNS, _ := tenant.NewNamespace("savedNS", savedProject.Name(), map[string]string{}) t.Run("Save", func(t *testing.T) { diff --git a/core/tenant/service/project_service_test.go b/core/tenant/service/project_service_test.go index 8e2e7f914d..962f074a03 100644 --- a/core/tenant/service/project_service_test.go +++ b/core/tenant/service/project_service_test.go @@ -19,7 +19,8 @@ func TestProjectService(t *testing.T) { tenant.ProjectStoragePathKey: "gs://location", "BUCKET": "gs://some_folder", } - savedProject, _ := tenant.NewProject("savedProj", conf) + projectVars := map[string]string{} + savedProject, _ := tenant.NewProject("savedProj", conf, projectVars) preset, err := tenant.NewPreset("test_preset", "preset for testing", "1d", "1h", "", "") assert.NoError(t, err) @@ -38,7 +39,7 @@ func TestProjectService(t *testing.T) { presetRepo := new(presetRepo) - toSaveProj, _ := tenant.NewProject("proj", conf) + toSaveProj, _ := tenant.NewProject("proj", conf, projectVars) projService := service.NewProjectService(projectRepo, presetRepo) err := projService.Save(ctx, toSaveProj) @@ -54,7 +55,7 @@ func TestProjectService(t *testing.T) { presetRepo := new(presetRepo) defer presetRepo.AssertExpectations(t) - toSaveProj, _ := tenant.NewProject("proj", conf) + toSaveProj, _ := tenant.NewProject("proj", conf, projectVars) toSaveProj.SetPresets(presetsMap) presetRepo.On("Read", ctx, toSaveProj.Name()).Return([]tenant.Preset{}, nil) @@ -74,7 +75,7 @@ func TestProjectService(t *testing.T) { presetRepo := new(presetRepo) defer presetRepo.AssertExpectations(t) - toSaveProj, _ := tenant.NewProject("proj", conf) + toSaveProj, _ := tenant.NewProject("proj", conf, projectVars) toSaveProj.SetPresets(presetsMap) presetRepo.On("Read", ctx, toSaveProj.Name()).Return([]tenant.Preset{}, nil) diff --git a/core/tenant/service/tenant_service_test.go b/core/tenant/service/tenant_service_test.go index 44d417219f..bf55555216 100644 --- a/core/tenant/service/tenant_service_test.go +++ b/core/tenant/service/tenant_service_test.go @@ -20,7 +20,8 @@ func TestTenantService(t *testing.T) { tenant.ProjectStoragePathKey: "gs://location", "BUCKET": "gs://some_folder", } - proj, _ := tenant.NewProject("testProj", conf) + projectVars := map[string]string{} + proj, _ := tenant.NewProject("testProj", conf, projectVars) ns, _ := tenant.NewNamespace("testNS", proj.Name(), map[string]string{}) tnnt, _ := tenant.NewTenant(proj.Name().String(), ns.Name().String()) diff --git a/core/tenant/tenant.go b/core/tenant/tenant.go index 823257778b..a792c68458 100644 --- a/core/tenant/tenant.go +++ b/core/tenant/tenant.go @@ -1,6 +1,8 @@ package tenant import ( + "fmt" + "github.com/goto/optimus/internal/errors" "github.com/goto/optimus/internal/utils" ) @@ -84,11 +86,24 @@ func (w *WithDetails) GetConfig(key string) (string, error) { return "", errors.NotFound(EntityTenant, "config not present in tenant "+key) } +func (w *WithDetails) GetVariable(key string) (string, error) { + variable, err := w.project.GetVariable(key) + if err == nil { + return variable, nil + } + + return "", errors.NotFound(EntityTenant, fmt.Sprintf("variable %s not present in tenant", key)) +} + func (w *WithDetails) GetConfigs() map[string]string { m1 := w.namespace.GetConfigs() return utils.MergeMaps(w.project.GetConfigs(), m1) } +func (w *WithDetails) GetVariables() map[string]string { + return w.project.GetVariables() +} + func (w *WithDetails) Project() *Project { return &w.project } diff --git a/core/tenant/tenant_test.go b/core/tenant/tenant_test.go index 856c0eec1f..60b068f3a8 100644 --- a/core/tenant/tenant_test.go +++ b/core/tenant/tenant_test.go @@ -35,8 +35,11 @@ func TestAggregateRootTenant(t *testing.T) { tenant.ProjectStoragePathKey: "gs://location", "BUCKET": "gs://some_folder", } + projectVars := map[string]string{ + "WAREHOUSE_PROJECT": "project", + } - project, _ := tenant.NewProject("test-project", projectConf) + project, _ := tenant.NewProject("test-project", projectConf, projectVars) namespace, _ := tenant.NewNamespace("test-ns", project.Name(), map[string]string{ "BUCKET": "gs://ns_folder", "OTHER_CONFIG": "optimus", @@ -117,6 +120,29 @@ func TestAggregateRootTenant(t *testing.T) { assert.Len(t, secMap, 2) assert.Equal(t, "value2", secMap[p2.Name().String()]) }) + t.Run("returns tenant variables", func(t *testing.T) { + details, err := tenant.NewTenantDetails(project, namespace, nil) + assert.NoError(t, err) + + tenantVariables := details.GetVariables() + assert.EqualValues(t, project.GetVariables(), tenantVariables) + }) + t.Run("returns a single tenant variable", func(t *testing.T) { + details, err := tenant.NewTenantDetails(project, namespace, nil) + assert.NoError(t, err) + + val, err := details.GetVariable("WAREHOUSE_PROJECT") + assert.NoError(t, err) + assert.Equal(t, "project", val) + }) + t.Run("returns error if a referred variable does not exist", func(t *testing.T) { + details, err := tenant.NewTenantDetails(project, namespace, nil) + assert.NoError(t, err) + + _, err = details.GetVariable("NONEXISTENT_VARIABLE") + assert.Error(t, err) + assert.EqualError(t, err, "not found for entity tenant: variable NONEXISTENT_VARIABLE not present in tenant") + }) }) }) } diff --git a/ext/scheduler/airflow/dag/compiler_test.go b/ext/scheduler/airflow/dag/compiler_test.go index eb5f17b939..f7cc672474 100644 --- a/ext/scheduler/airflow/dag/compiler_test.go +++ b/ext/scheduler/airflow/dag/compiler_test.go @@ -125,11 +125,13 @@ func TestDagCompiler(t *testing.T) { } func setProject(tnnt tenant.Tenant, airflowVersion string) *tenant.Project { - p, _ := tenant.NewProject(tnnt.ProjectName().String(), map[string]string{ + projectCfg := map[string]string{ tenant.ProjectSchedulerVersion: airflowVersion, tenant.ProjectStoragePathKey: "./path/to/storage", tenant.ProjectSchedulerHost: "http://airflow.com", - }) + } + projectVars := map[string]string{} + p, _ := tenant.NewProject(tnnt.ProjectName().String(), projectCfg, projectVars) return p } diff --git a/internal/store/postgres/job/job_repository_test.go b/internal/store/postgres/job/job_repository_test.go index c84afab9f0..ccbf4b80a0 100644 --- a/internal/store/postgres/job/job_repository_test.go +++ b/internal/store/postgres/job/job_repository_test.go @@ -27,7 +27,7 @@ func TestPostgresJobRepository(t *testing.T) { "bucket": "gs://some_folder-2", tenant.ProjectSchedulerHost: "host", tenant.ProjectStoragePathKey: "gs://location", - }) + }, map[string]string{}) assert.NoError(t, err) otherProj, err := tenant.NewProject("test-other-proj", @@ -35,7 +35,7 @@ func TestPostgresJobRepository(t *testing.T) { "bucket": "gs://some_folder-3", tenant.ProjectSchedulerHost: "host", tenant.ProjectStoragePathKey: "gs://location", - }) + }, map[string]string{}) assert.NoError(t, err) namespace, err := tenant.NewNamespace("test-ns", proj.Name(), diff --git a/internal/store/postgres/migrations/000064_add_variables_in_project_and_namespace.down.sql b/internal/store/postgres/migrations/000064_add_variables_in_project_and_namespace.down.sql new file mode 100644 index 0000000000..b8ef5edcc1 --- /dev/null +++ b/internal/store/postgres/migrations/000064_add_variables_in_project_and_namespace.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE namespace DROP COLUMN IF EXISTS variables; +ALTER TABLE project DROP COLUMN IF EXISTS variables; \ No newline at end of file diff --git a/internal/store/postgres/migrations/000064_add_variables_in_project_and_namespace.up.sql b/internal/store/postgres/migrations/000064_add_variables_in_project_and_namespace.up.sql new file mode 100644 index 0000000000..f5f97975be --- /dev/null +++ b/internal/store/postgres/migrations/000064_add_variables_in_project_and_namespace.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE project ADD COLUMN IF NOT EXISTS variables jsonb NULL; +ALTER TABLE namespace ADD COLUMN IF NOT EXISTS variables jsonb NULL; \ No newline at end of file diff --git a/internal/store/postgres/resource/repository_test.go b/internal/store/postgres/resource/repository_test.go index f73e3c632a..fae96da1ce 100644 --- a/internal/store/postgres/resource/repository_test.go +++ b/internal/store/postgres/resource/repository_test.go @@ -437,7 +437,7 @@ func dbSetup() *pgxpool.Pool { "bucket": "gs://some_folder-2", tenant.ProjectSchedulerHost: "host", tenant.ProjectStoragePathKey: "gs://location", - }) + }, map[string]string{}) projRepo := tenantPostgres.NewProjectRepository(pool) err := projRepo.Save(ctx, proj) if err != nil { diff --git a/internal/store/postgres/scheduler/job_repository_test.go b/internal/store/postgres/scheduler/job_repository_test.go index 7e388eda2a..d4b2b4703c 100644 --- a/internal/store/postgres/scheduler/job_repository_test.go +++ b/internal/store/postgres/scheduler/job_repository_test.go @@ -179,7 +179,7 @@ func addJobs(ctx context.Context, t *testing.T, pool *pgxpool.Pool) map[string]* "bucket": "gs://some_folder-2", tenant.ProjectSchedulerHost: "host", tenant.ProjectStoragePathKey: "gs://location", - }) + }, map[string]string{}) assert.NoError(t, err) projRepo := tenantPostgres.NewProjectRepository(pool) diff --git a/internal/store/postgres/tenant/namespace_repository_test.go b/internal/store/postgres/tenant/namespace_repository_test.go index 1334954706..348773c337 100644 --- a/internal/store/postgres/tenant/namespace_repository_test.go +++ b/internal/store/postgres/tenant/namespace_repository_test.go @@ -22,7 +22,7 @@ func TestPostgresNamespaceRepository(t *testing.T) { transporterKafkaBrokerKey: "10.12.12.12:6668,10.12.12.13:6668", tenant.ProjectSchedulerHost: "host", tenant.ProjectStoragePathKey: "gs://location", - }) + }, map[string]string{}) ns, _ := tenant.NewNamespace("n-optimus-1", proj.Name(), map[string]string{ "bucket": "gs://ns_bucket", diff --git a/internal/store/postgres/tenant/preset_repository_test.go b/internal/store/postgres/tenant/preset_repository_test.go index 2aa41f5a45..4ebe05a1f0 100644 --- a/internal/store/postgres/tenant/preset_repository_test.go +++ b/internal/store/postgres/tenant/preset_repository_test.go @@ -23,7 +23,7 @@ func TestPostgresPresetRepository(t *testing.T) { "bucket": "gs://some_folder-2", tenant.ProjectSchedulerHost: "host", tenant.ProjectStoragePathKey: "gs://location", - }) + }, map[string]string{}) dbSetup := func() *pgxpool.Pool { dbPool := setup.TestPool() diff --git a/internal/store/postgres/tenant/project_repository.go b/internal/store/postgres/tenant/project_repository.go index c3873f0167..ebfd9ae1a6 100644 --- a/internal/store/postgres/tenant/project_repository.go +++ b/internal/store/postgres/tenant/project_repository.go @@ -17,35 +17,36 @@ type ProjectRepository struct { } const ( - projectColumns = `id, name, config, created_at, updated_at` + projectColumns = `id, name, config, variables, created_at, updated_at` ) type Project struct { - ID uuid.UUID - Name string - Config map[string]string + ID uuid.UUID + Name string + Config map[string]string + Variables map[string]string CreatedAt time.Time UpdatedAt time.Time } func (p *Project) toTenantProject() (*tenant.Project, error) { - return tenant.NewProject(p.Name, p.Config) + return tenant.NewProject(p.Name, p.Config, p.Variables) } func (repo ProjectRepository) Save(ctx context.Context, tenantProject *tenant.Project) error { _, err := repo.get(ctx, tenantProject.Name()) if err != nil { if errors.Is(err, pgx.ErrNoRows) { - insertProjectQuery := `INSERT INTO project (name, config, created_at, updated_at) VALUES ($1, $2, now(), now())` - _, err = repo.db.Exec(ctx, insertProjectQuery, tenantProject.Name(), tenantProject.GetConfigs()) + insertProjectQuery := `INSERT INTO project (name, config, variables, created_at, updated_at) VALUES ($1, $2, $3 now(), now())` + _, err = repo.db.Exec(ctx, insertProjectQuery, tenantProject.Name(), tenantProject.GetConfigs(), tenantProject.GetVariables()) return errors.WrapIfErr(tenant.EntityProject, "unable to save project", err) } return errors.Wrap(tenant.EntityProject, "unable to save project", err) } - updateProjectQuery := `UPDATE project SET config=$1, updated_at=now() WHERE name=$2` - _, err = repo.db.Exec(ctx, updateProjectQuery, tenantProject.GetConfigs(), tenantProject.Name()) + updateProjectQuery := `UPDATE project SET config=$1, variables=$2, updated_at=now() WHERE name=$3` + _, err = repo.db.Exec(ctx, updateProjectQuery, tenantProject.GetConfigs(), tenantProject.GetVariables(), tenantProject.Name()) return errors.WrapIfErr(tenant.EntityProject, "unable to update project", err) } @@ -65,7 +66,7 @@ func (repo ProjectRepository) get(ctx context.Context, name tenant.ProjectName) getProjectByNameQuery := `SELECT ` + projectColumns + ` FROM project WHERE name = $1 AND deleted_at IS NULL` err := repo.db.QueryRow(ctx, getProjectByNameQuery, name). - Scan(&project.ID, &project.Name, &project.Config, &project.CreatedAt, &project.UpdatedAt) + Scan(&project.ID, &project.Name, &project.Config, &project.Variables, &project.CreatedAt, &project.UpdatedAt) if err != nil { return Project{}, err } @@ -84,7 +85,7 @@ func (repo ProjectRepository) GetAll(ctx context.Context) ([]*tenant.Project, er for rows.Next() { var prj Project - err = rows.Scan(&prj.ID, &prj.Name, &prj.Config, &prj.CreatedAt, &prj.UpdatedAt) + err = rows.Scan(&prj.ID, &prj.Name, &prj.Config, &prj.Variables, &prj.CreatedAt, &prj.UpdatedAt) if err != nil { return nil, errors.Wrap(tenant.EntityProject, "error in GetAll", err) } diff --git a/internal/store/postgres/tenant/project_repository_test.go b/internal/store/postgres/tenant/project_repository_test.go index 46c85073f3..c05a030efe 100644 --- a/internal/store/postgres/tenant/project_repository_test.go +++ b/internal/store/postgres/tenant/project_repository_test.go @@ -22,7 +22,7 @@ func TestPostgresProjectRepository(t *testing.T) { transporterKafkaBrokerKey: "10.12.12.12:6668,10.12.12.13:6668", tenant.ProjectSchedulerHost: "host", tenant.ProjectStoragePathKey: "gs://location", - }) + }, map[string]string{}) ctx := context.Background() dbSetup := func() *pgxpool.Pool { @@ -44,7 +44,7 @@ func TestPostgresProjectRepository(t *testing.T) { assert.Nil(t, err) assert.Equal(t, "t-optimus-1", savedProj.Name().String()) - proj2, _ := tenant.NewProject("t-optimus-2", proj.GetConfigs()) + proj2, _ := tenant.NewProject("t-optimus-2", proj.GetConfigs(), proj.GetVariables()) err = repo.Save(ctx, proj2) assert.Nil(t, err) @@ -69,7 +69,7 @@ func TestPostgresProjectRepository(t *testing.T) { conf := proj.GetConfigs() conf["STORAGE"] = "gs://some_place" - proj2, _ := tenant.NewProject(proj.Name().String(), conf) + proj2, _ := tenant.NewProject(proj.Name().String(), conf, map[string]string{}) err = repo.Save(ctx, proj2) assert.Nil(t, err) @@ -89,7 +89,7 @@ func TestPostgresProjectRepository(t *testing.T) { err := repo.Save(ctx, proj) assert.Nil(t, err) - proj2, _ := tenant.NewProject("t-optimus-2", proj.GetConfigs()) + proj2, _ := tenant.NewProject("t-optimus-2", proj.GetConfigs(), map[string]string{}) err = repo.Save(ctx, proj2) assert.Nil(t, err) diff --git a/internal/store/postgres/tenant/secret_repository_test.go b/internal/store/postgres/tenant/secret_repository_test.go index 1c3f4292e7..e5c217d9b7 100644 --- a/internal/store/postgres/tenant/secret_repository_test.go +++ b/internal/store/postgres/tenant/secret_repository_test.go @@ -22,7 +22,7 @@ func TestPostgresSecretRepository(t *testing.T) { "bucket": "gs://some_folder-2", tenant.ProjectSchedulerHost: "host", tenant.ProjectStoragePathKey: "gs://location", - }) + }, map[string]string{}) namespace, _ := tenant.NewNamespace("test-ns", proj.Name(), map[string]string{ "bucket": "gs://ns_bucket", diff --git a/protos/gotocompany/optimus/core/v1beta1/namespace.pb.go b/protos/gotocompany/optimus/core/v1beta1/namespace.pb.go index be738becae..5b8e47fc89 100644 --- a/protos/gotocompany/optimus/core/v1beta1/namespace.pb.go +++ b/protos/gotocompany/optimus/core/v1beta1/namespace.pb.go @@ -333,8 +333,10 @@ type NamespaceSpecification struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - Config map[string]string `protobuf:"bytes,2,rep,name=config,proto3" json:"config,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // Deprecated: Do not use. + Config map[string]string `protobuf:"bytes,2,rep,name=config,proto3" json:"config,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Variables map[string]string `protobuf:"bytes,3,rep,name=variables,proto3" json:"variables,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } func (x *NamespaceSpecification) Reset() { @@ -376,6 +378,7 @@ func (x *NamespaceSpecification) GetName() string { return "" } +// Deprecated: Do not use. func (x *NamespaceSpecification) GetConfig() map[string]string { if x != nil { return x.Config @@ -383,6 +386,13 @@ func (x *NamespaceSpecification) GetConfig() map[string]string { return nil } +func (x *NamespaceSpecification) GetVariables() map[string]string { + if x != nil { + return x.Variables + } + return nil +} + var File_gotocompany_optimus_core_v1beta1_namespace_proto protoreflect.FileDescriptor var file_gotocompany_optimus_core_v1beta1_namespace_proto_rawDesc = []byte{ @@ -436,70 +446,80 @@ var file_gotocompany_optimus_core_v1beta1_namespace_proto_rawDesc = []byte{ 0x79, 0x2e, 0x6f, 0x70, 0x74, 0x69, 0x6d, 0x75, 0x73, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x53, 0x70, 0x65, 0x63, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6e, - 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x22, 0xc5, 0x01, 0x0a, 0x16, 0x4e, 0x61, 0x6d, + 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x22, 0xee, 0x02, 0x0a, 0x16, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x53, 0x70, 0x65, 0x63, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x5c, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, + 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x60, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x44, 0x2e, 0x67, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x6e, 0x79, 0x2e, 0x6f, 0x70, 0x74, 0x69, 0x6d, 0x75, 0x73, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x53, 0x70, 0x65, 0x63, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x63, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x1a, 0x39, 0x0a, 0x0b, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, - 0x32, 0xfe, 0x04, 0x0a, 0x10, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0xd7, 0x01, 0x0a, 0x18, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, - 0x65, 0x72, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, - 0x63, 0x65, 0x12, 0x41, 0x2e, 0x67, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x6e, 0x79, - 0x2e, 0x6f, 0x70, 0x74, 0x69, 0x6d, 0x75, 0x73, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, - 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x50, 0x72, - 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x42, 0x2e, 0x67, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6d, 0x70, - 0x61, 0x6e, 0x79, 0x2e, 0x6f, 0x70, 0x74, 0x69, 0x6d, 0x75, 0x73, 0x2e, 0x63, 0x6f, 0x72, 0x65, - 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, - 0x72, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x34, 0x82, 0xd3, 0xe4, 0x93, 0x02, - 0x2e, 0x3a, 0x01, 0x2a, 0x22, 0x29, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2f, 0x70, - 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x2f, 0x7b, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x5f, - 0x6e, 0x61, 0x6d, 0x65, 0x7d, 0x2f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, - 0xcb, 0x01, 0x0a, 0x15, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x4e, - 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x12, 0x3e, 0x2e, 0x67, 0x6f, 0x74, 0x6f, - 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x6e, 0x79, 0x2e, 0x6f, 0x70, 0x74, 0x69, 0x6d, 0x75, 0x73, 0x2e, - 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x4c, 0x69, 0x73, - 0x74, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, - 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x3f, 0x2e, 0x67, 0x6f, 0x74, 0x6f, - 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x6e, 0x79, 0x2e, 0x6f, 0x70, 0x74, 0x69, 0x6d, 0x75, 0x73, 0x2e, - 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x4c, 0x69, 0x73, + 0x6e, 0x2e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x42, 0x02, 0x18, + 0x01, 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x65, 0x0a, 0x09, 0x76, 0x61, 0x72, + 0x69, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x47, 0x2e, 0x67, + 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x6e, 0x79, 0x2e, 0x6f, 0x70, 0x74, 0x69, 0x6d, + 0x75, 0x73, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, + 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x53, 0x70, 0x65, 0x63, 0x69, 0x66, 0x69, + 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x73, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x09, 0x76, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x73, + 0x1a, 0x39, 0x0a, 0x0b, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, + 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, + 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x3c, 0x0a, 0x0e, 0x56, + 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x32, 0xfe, 0x04, 0x0a, 0x10, 0x4e, 0x61, + 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0xd7, + 0x01, 0x0a, 0x18, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x6a, 0x65, + 0x63, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x41, 0x2e, 0x67, 0x6f, + 0x74, 0x6f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x6e, 0x79, 0x2e, 0x6f, 0x70, 0x74, 0x69, 0x6d, 0x75, + 0x73, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x52, + 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x4e, 0x61, + 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x42, + 0x2e, 0x67, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x6e, 0x79, 0x2e, 0x6f, 0x70, 0x74, + 0x69, 0x6d, 0x75, 0x73, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, + 0x31, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, + 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x34, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x2e, 0x22, 0x29, 0x2f, 0x76, 0x31, 0x62, + 0x65, 0x74, 0x61, 0x31, 0x2f, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x2f, 0x7b, 0x70, 0x72, + 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x7d, 0x2f, 0x6e, 0x61, 0x6d, 0x65, + 0x73, 0x70, 0x61, 0x63, 0x65, 0x3a, 0x01, 0x2a, 0x12, 0xcb, 0x01, 0x0a, 0x15, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, - 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x31, 0x82, 0xd3, 0xe4, 0x93, - 0x02, 0x2b, 0x12, 0x29, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2f, 0x70, 0x72, 0x6f, - 0x6a, 0x65, 0x63, 0x74, 0x2f, 0x7b, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x6e, 0x61, - 0x6d, 0x65, 0x7d, 0x2f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0xc1, 0x01, - 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x35, + 0x65, 0x73, 0x12, 0x3e, 0x2e, 0x67, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x6e, 0x79, + 0x2e, 0x6f, 0x70, 0x74, 0x69, 0x6d, 0x75, 0x73, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, + 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, + 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x3f, 0x2e, 0x67, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x6e, 0x79, + 0x2e, 0x6f, 0x70, 0x74, 0x69, 0x6d, 0x75, 0x73, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, + 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, + 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x31, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x2b, 0x12, 0x29, 0x2f, 0x76, 0x31, + 0x62, 0x65, 0x74, 0x61, 0x31, 0x2f, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x2f, 0x7b, 0x70, + 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x7d, 0x2f, 0x6e, 0x61, 0x6d, + 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0xc1, 0x01, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x4e, 0x61, + 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x35, 0x2e, 0x67, 0x6f, 0x74, 0x6f, 0x63, 0x6f, + 0x6d, 0x70, 0x61, 0x6e, 0x79, 0x2e, 0x6f, 0x70, 0x74, 0x69, 0x6d, 0x75, 0x73, 0x2e, 0x63, 0x6f, + 0x72, 0x65, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x61, + 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x36, 0x2e, 0x67, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x6e, 0x79, 0x2e, 0x6f, 0x70, 0x74, 0x69, 0x6d, 0x75, 0x73, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x36, 0x2e, 0x67, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6d, 0x70, - 0x61, 0x6e, 0x79, 0x2e, 0x6f, 0x70, 0x74, 0x69, 0x6d, 0x75, 0x73, 0x2e, 0x63, 0x6f, 0x72, 0x65, - 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x4e, 0x61, 0x6d, 0x65, - 0x73, 0x70, 0x61, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x42, 0x82, - 0xd3, 0xe4, 0x93, 0x02, 0x3c, 0x12, 0x3a, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2f, - 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x2f, 0x7b, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, - 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x7d, 0x2f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, - 0x2f, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, - 0x7d, 0x42, 0x9b, 0x01, 0x0a, 0x1e, 0x63, 0x6f, 0x6d, 0x2e, 0x67, 0x6f, 0x74, 0x6f, 0x63, 0x6f, - 0x6d, 0x70, 0x61, 0x6e, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x6e, 0x2e, 0x6f, 0x70, 0x74, - 0x69, 0x6d, 0x75, 0x73, 0x42, 0x17, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x50, 0x01, 0x5a, - 0x1e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x74, 0x6f, - 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x6e, 0x2f, 0x6f, 0x70, 0x74, 0x69, 0x6d, 0x75, 0x73, 0x92, - 0x41, 0x3d, 0x12, 0x05, 0x32, 0x03, 0x30, 0x2e, 0x31, 0x1a, 0x0e, 0x31, 0x32, 0x37, 0x2e, 0x30, - 0x2e, 0x30, 0x2e, 0x31, 0x3a, 0x39, 0x31, 0x30, 0x30, 0x22, 0x04, 0x2f, 0x61, 0x70, 0x69, 0x2a, - 0x01, 0x01, 0x72, 0x1b, 0x0a, 0x19, 0x4f, 0x70, 0x74, 0x69, 0x6d, 0x75, 0x73, 0x20, 0x4e, 0x61, - 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x20, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x62, - 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x42, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x3c, 0x12, 0x3a, + 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2f, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, + 0x2f, 0x7b, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x7d, 0x2f, + 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x2f, 0x7b, 0x6e, 0x61, 0x6d, 0x65, 0x73, + 0x70, 0x61, 0x63, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x7d, 0x42, 0x9b, 0x01, 0x0a, 0x1e, 0x63, + 0x6f, 0x6d, 0x2e, 0x67, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x6e, 0x79, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x6e, 0x2e, 0x6f, 0x70, 0x74, 0x69, 0x6d, 0x75, 0x73, 0x42, 0x17, 0x4e, + 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4d, + 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x50, 0x01, 0x5a, 0x1e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, + 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x74, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x6e, + 0x2f, 0x6f, 0x70, 0x74, 0x69, 0x6d, 0x75, 0x73, 0x92, 0x41, 0x3d, 0x12, 0x05, 0x32, 0x03, 0x30, + 0x2e, 0x31, 0x1a, 0x0e, 0x31, 0x32, 0x37, 0x2e, 0x30, 0x2e, 0x30, 0x2e, 0x31, 0x3a, 0x39, 0x31, + 0x30, 0x30, 0x22, 0x04, 0x2f, 0x61, 0x70, 0x69, 0x2a, 0x01, 0x01, 0x72, 0x1b, 0x0a, 0x19, 0x4f, + 0x70, 0x74, 0x69, 0x6d, 0x75, 0x73, 0x20, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, + 0x20, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -514,7 +534,7 @@ func file_gotocompany_optimus_core_v1beta1_namespace_proto_rawDescGZIP() []byte return file_gotocompany_optimus_core_v1beta1_namespace_proto_rawDescData } -var file_gotocompany_optimus_core_v1beta1_namespace_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_gotocompany_optimus_core_v1beta1_namespace_proto_msgTypes = make([]protoimpl.MessageInfo, 9) var file_gotocompany_optimus_core_v1beta1_namespace_proto_goTypes = []interface{}{ (*RegisterProjectNamespaceRequest)(nil), // 0: gotocompany.optimus.core.v1beta1.RegisterProjectNamespaceRequest (*RegisterProjectNamespaceResponse)(nil), // 1: gotocompany.optimus.core.v1beta1.RegisterProjectNamespaceResponse @@ -524,23 +544,25 @@ var file_gotocompany_optimus_core_v1beta1_namespace_proto_goTypes = []interface{ (*GetNamespaceResponse)(nil), // 5: gotocompany.optimus.core.v1beta1.GetNamespaceResponse (*NamespaceSpecification)(nil), // 6: gotocompany.optimus.core.v1beta1.NamespaceSpecification nil, // 7: gotocompany.optimus.core.v1beta1.NamespaceSpecification.ConfigEntry + nil, // 8: gotocompany.optimus.core.v1beta1.NamespaceSpecification.VariablesEntry } var file_gotocompany_optimus_core_v1beta1_namespace_proto_depIdxs = []int32{ 6, // 0: gotocompany.optimus.core.v1beta1.RegisterProjectNamespaceRequest.namespace:type_name -> gotocompany.optimus.core.v1beta1.NamespaceSpecification 6, // 1: gotocompany.optimus.core.v1beta1.ListProjectNamespacesResponse.namespaces:type_name -> gotocompany.optimus.core.v1beta1.NamespaceSpecification 6, // 2: gotocompany.optimus.core.v1beta1.GetNamespaceResponse.namespace:type_name -> gotocompany.optimus.core.v1beta1.NamespaceSpecification 7, // 3: gotocompany.optimus.core.v1beta1.NamespaceSpecification.config:type_name -> gotocompany.optimus.core.v1beta1.NamespaceSpecification.ConfigEntry - 0, // 4: gotocompany.optimus.core.v1beta1.NamespaceService.RegisterProjectNamespace:input_type -> gotocompany.optimus.core.v1beta1.RegisterProjectNamespaceRequest - 2, // 5: gotocompany.optimus.core.v1beta1.NamespaceService.ListProjectNamespaces:input_type -> gotocompany.optimus.core.v1beta1.ListProjectNamespacesRequest - 4, // 6: gotocompany.optimus.core.v1beta1.NamespaceService.GetNamespace:input_type -> gotocompany.optimus.core.v1beta1.GetNamespaceRequest - 1, // 7: gotocompany.optimus.core.v1beta1.NamespaceService.RegisterProjectNamespace:output_type -> gotocompany.optimus.core.v1beta1.RegisterProjectNamespaceResponse - 3, // 8: gotocompany.optimus.core.v1beta1.NamespaceService.ListProjectNamespaces:output_type -> gotocompany.optimus.core.v1beta1.ListProjectNamespacesResponse - 5, // 9: gotocompany.optimus.core.v1beta1.NamespaceService.GetNamespace:output_type -> gotocompany.optimus.core.v1beta1.GetNamespaceResponse - 7, // [7:10] is the sub-list for method output_type - 4, // [4:7] is the sub-list for method input_type - 4, // [4:4] is the sub-list for extension type_name - 4, // [4:4] is the sub-list for extension extendee - 0, // [0:4] is the sub-list for field type_name + 8, // 4: gotocompany.optimus.core.v1beta1.NamespaceSpecification.variables:type_name -> gotocompany.optimus.core.v1beta1.NamespaceSpecification.VariablesEntry + 0, // 5: gotocompany.optimus.core.v1beta1.NamespaceService.RegisterProjectNamespace:input_type -> gotocompany.optimus.core.v1beta1.RegisterProjectNamespaceRequest + 2, // 6: gotocompany.optimus.core.v1beta1.NamespaceService.ListProjectNamespaces:input_type -> gotocompany.optimus.core.v1beta1.ListProjectNamespacesRequest + 4, // 7: gotocompany.optimus.core.v1beta1.NamespaceService.GetNamespace:input_type -> gotocompany.optimus.core.v1beta1.GetNamespaceRequest + 1, // 8: gotocompany.optimus.core.v1beta1.NamespaceService.RegisterProjectNamespace:output_type -> gotocompany.optimus.core.v1beta1.RegisterProjectNamespaceResponse + 3, // 9: gotocompany.optimus.core.v1beta1.NamespaceService.ListProjectNamespaces:output_type -> gotocompany.optimus.core.v1beta1.ListProjectNamespacesResponse + 5, // 10: gotocompany.optimus.core.v1beta1.NamespaceService.GetNamespace:output_type -> gotocompany.optimus.core.v1beta1.GetNamespaceResponse + 8, // [8:11] is the sub-list for method output_type + 5, // [5:8] is the sub-list for method input_type + 5, // [5:5] is the sub-list for extension type_name + 5, // [5:5] is the sub-list for extension extendee + 0, // [0:5] is the sub-list for field type_name } func init() { file_gotocompany_optimus_core_v1beta1_namespace_proto_init() } @@ -640,7 +662,7 @@ func file_gotocompany_optimus_core_v1beta1_namespace_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_gotocompany_optimus_core_v1beta1_namespace_proto_rawDesc, NumEnums: 0, - NumMessages: 8, + NumMessages: 9, NumExtensions: 0, NumServices: 1, }, diff --git a/protos/gotocompany/optimus/core/v1beta1/namespace.swagger.json b/protos/gotocompany/optimus/core/v1beta1/namespace.swagger.json index 86bb16beba..f367cd45ec 100644 --- a/protos/gotocompany/optimus/core/v1beta1/namespace.swagger.json +++ b/protos/gotocompany/optimus/core/v1beta1/namespace.swagger.json @@ -193,6 +193,12 @@ "additionalProperties": { "type": "string" } + }, + "variables": { + "type": "object", + "additionalProperties": { + "type": "string" + } } } }, diff --git a/protos/gotocompany/optimus/core/v1beta1/project.pb.go b/protos/gotocompany/optimus/core/v1beta1/project.pb.go index 5252f21d7a..605cf48b99 100644 --- a/protos/gotocompany/optimus/core/v1beta1/project.pb.go +++ b/protos/gotocompany/optimus/core/v1beta1/project.pb.go @@ -291,9 +291,10 @@ type ProjectSpecification struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - Config map[string]string `protobuf:"bytes,2,rep,name=config,proto3" json:"config,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` - Presets map[string]*ProjectSpecification_ProjectPreset `protobuf:"bytes,4,rep,name=presets,proto3" json:"presets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Config map[string]string `protobuf:"bytes,2,rep,name=config,proto3" json:"config,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Presets map[string]*ProjectSpecification_ProjectPreset `protobuf:"bytes,4,rep,name=presets,proto3" json:"presets,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Variables map[string]string `protobuf:"bytes,5,rep,name=variables,proto3" json:"variables,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } func (x *ProjectSpecification) Reset() { @@ -349,6 +350,13 @@ func (x *ProjectSpecification) GetPresets() map[string]*ProjectSpecification_Pro return nil } +func (x *ProjectSpecification) GetVariables() map[string]string { + if x != nil { + return x.Variables + } + return nil +} + type ProjectSpecification_ProjectPreset struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -496,7 +504,7 @@ var file_gotocompany_optimus_core_v1beta1_project_proto_rawDesc = []byte{ 0x74, 0x6f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x6e, 0x79, 0x2e, 0x6f, 0x70, 0x74, 0x69, 0x6d, 0x75, 0x73, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x53, 0x70, 0x65, 0x63, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x22, 0x93, 0x05, 0x0a, + 0x69, 0x6f, 0x6e, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x22, 0xb6, 0x06, 0x0a, 0x14, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x53, 0x70, 0x65, 0x63, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x5a, 0x0a, 0x06, 0x63, 0x6f, 0x6e, @@ -511,75 +519,85 @@ var file_gotocompany_optimus_core_v1beta1_project_proto_rawDesc = []byte{ 0x65, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x53, 0x70, 0x65, 0x63, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x50, 0x72, 0x65, 0x73, 0x65, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x70, 0x72, 0x65, - 0x73, 0x65, 0x74, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x45, 0x6e, + 0x73, 0x65, 0x74, 0x73, 0x12, 0x63, 0x0a, 0x09, 0x76, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, + 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x45, 0x2e, 0x67, 0x6f, 0x74, 0x6f, 0x63, 0x6f, + 0x6d, 0x70, 0x61, 0x6e, 0x79, 0x2e, 0x6f, 0x70, 0x74, 0x69, 0x6d, 0x75, 0x73, 0x2e, 0x63, 0x6f, + 0x72, 0x65, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x6a, 0x65, + 0x63, 0x74, 0x53, 0x70, 0x65, 0x63, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x09, + 0x76, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x1a, 0x39, 0x0a, 0x0b, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x02, 0x38, 0x01, 0x1a, 0xe7, 0x01, 0x0a, 0x0d, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, + 0x50, 0x72, 0x65, 0x73, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, + 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, + 0x74, 0x72, 0x75, 0x6e, 0x63, 0x61, 0x74, 0x65, 0x5f, 0x74, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0a, 0x74, 0x72, 0x75, 0x6e, 0x63, 0x61, 0x74, 0x65, 0x54, 0x6f, 0x12, 0x1a, 0x0a, + 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x42, 0x02, 0x18, + 0x01, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, + 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x18, 0x0a, + 0x05, 0x64, 0x65, 0x6c, 0x61, 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x42, 0x02, 0x18, 0x01, + 0x52, 0x05, 0x64, 0x65, 0x6c, 0x61, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x69, 0x66, 0x74, + 0x5f, 0x62, 0x79, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x68, 0x69, 0x66, 0x74, + 0x42, 0x79, 0x12, 0x1a, 0x0a, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x07, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0x80, + 0x01, 0x0a, 0x0c, 0x50, 0x72, 0x65, 0x73, 0x65, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, + 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, + 0x79, 0x12, 0x5a, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x44, 0x2e, 0x67, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x6e, 0x79, 0x2e, 0x6f, + 0x70, 0x74, 0x69, 0x6d, 0x75, 0x73, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x62, 0x65, + 0x74, 0x61, 0x31, 0x2e, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x53, 0x70, 0x65, 0x63, 0x69, + 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, + 0x50, 0x72, 0x65, 0x73, 0x65, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, + 0x01, 0x1a, 0x3c, 0x0a, 0x0e, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, - 0xe7, 0x01, 0x0a, 0x0d, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x50, 0x72, 0x65, 0x73, 0x65, - 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, - 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x74, 0x72, 0x75, 0x6e, 0x63, - 0x61, 0x74, 0x65, 0x5f, 0x74, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x74, 0x72, - 0x75, 0x6e, 0x63, 0x61, 0x74, 0x65, 0x54, 0x6f, 0x12, 0x1a, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, - 0x65, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x42, 0x02, 0x18, 0x01, 0x52, 0x06, 0x6f, 0x66, - 0x66, 0x73, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x05, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x18, 0x0a, 0x05, 0x64, 0x65, 0x6c, 0x61, - 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x42, 0x02, 0x18, 0x01, 0x52, 0x05, 0x64, 0x65, 0x6c, - 0x61, 0x79, 0x12, 0x19, 0x0a, 0x08, 0x73, 0x68, 0x69, 0x66, 0x74, 0x5f, 0x62, 0x79, 0x18, 0x08, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x68, 0x69, 0x66, 0x74, 0x42, 0x79, 0x12, 0x1a, 0x0a, - 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0x80, 0x01, 0x0a, 0x0c, 0x50, 0x72, - 0x65, 0x73, 0x65, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x5a, 0x0a, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x44, 0x2e, 0x67, 0x6f, - 0x74, 0x6f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x6e, 0x79, 0x2e, 0x6f, 0x70, 0x74, 0x69, 0x6d, 0x75, - 0x73, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x50, - 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x53, 0x70, 0x65, 0x63, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x2e, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x50, 0x72, 0x65, 0x73, 0x65, - 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x4a, 0x04, 0x08, 0x03, - 0x10, 0x04, 0x32, 0xf3, 0x03, 0x0a, 0x0e, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x53, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0xa3, 0x01, 0x0a, 0x0f, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, - 0x65, 0x72, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x38, 0x2e, 0x67, 0x6f, 0x74, 0x6f, - 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x6e, 0x79, 0x2e, 0x6f, 0x70, 0x74, 0x69, 0x6d, 0x75, 0x73, 0x2e, - 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x67, - 0x69, 0x73, 0x74, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x39, 0x2e, 0x67, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x6e, - 0x79, 0x2e, 0x6f, 0x70, 0x74, 0x69, 0x6d, 0x75, 0x73, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x76, - 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x50, - 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x1b, - 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x15, 0x3a, 0x01, 0x2a, 0x22, 0x10, 0x2f, 0x76, 0x31, 0x62, 0x65, - 0x74, 0x61, 0x31, 0x2f, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x97, 0x01, 0x0a, 0x0c, - 0x4c, 0x69, 0x73, 0x74, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, 0x35, 0x2e, 0x67, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x4a, + 0x04, 0x08, 0x03, 0x10, 0x04, 0x32, 0xf3, 0x03, 0x0a, 0x0e, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, + 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0xa3, 0x01, 0x0a, 0x0f, 0x52, 0x65, 0x67, + 0x69, 0x73, 0x74, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x38, 0x2e, 0x67, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x6e, 0x79, 0x2e, 0x6f, 0x70, 0x74, 0x69, 0x6d, 0x75, 0x73, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, - 0x4c, 0x69, 0x73, 0x74, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x36, 0x2e, 0x67, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x6e, - 0x79, 0x2e, 0x6f, 0x70, 0x74, 0x69, 0x6d, 0x75, 0x73, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x76, - 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x72, 0x6f, 0x6a, 0x65, - 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x18, 0x82, 0xd3, 0xe4, - 0x93, 0x02, 0x12, 0x12, 0x10, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2f, 0x70, 0x72, - 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0xa0, 0x01, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x50, 0x72, 0x6f, - 0x6a, 0x65, 0x63, 0x74, 0x12, 0x33, 0x2e, 0x67, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6d, 0x70, 0x61, - 0x6e, 0x79, 0x2e, 0x6f, 0x70, 0x74, 0x69, 0x6d, 0x75, 0x73, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, - 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x6a, 0x65, - 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x34, 0x2e, 0x67, 0x6f, 0x74, 0x6f, - 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x6e, 0x79, 0x2e, 0x6f, 0x70, 0x74, 0x69, 0x6d, 0x75, 0x73, 0x2e, - 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, - 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x27, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x21, 0x12, 0x1f, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, - 0x31, 0x2f, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x2f, 0x7b, 0x70, 0x72, 0x6f, 0x6a, 0x65, - 0x63, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x7d, 0x42, 0x97, 0x01, 0x0a, 0x1e, 0x63, 0x6f, 0x6d, - 0x2e, 0x67, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x6e, 0x79, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x6e, 0x2e, 0x6f, 0x70, 0x74, 0x69, 0x6d, 0x75, 0x73, 0x42, 0x15, 0x50, 0x72, 0x6f, - 0x6a, 0x65, 0x63, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4d, 0x61, 0x6e, 0x61, 0x67, - 0x65, 0x72, 0x50, 0x01, 0x5a, 0x1e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, - 0x2f, 0x67, 0x6f, 0x74, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x6e, 0x2f, 0x6f, 0x70, 0x74, - 0x69, 0x6d, 0x75, 0x73, 0x92, 0x41, 0x3b, 0x12, 0x05, 0x32, 0x03, 0x30, 0x2e, 0x31, 0x1a, 0x0e, - 0x31, 0x32, 0x37, 0x2e, 0x30, 0x2e, 0x30, 0x2e, 0x31, 0x3a, 0x39, 0x31, 0x30, 0x30, 0x22, 0x04, - 0x2f, 0x61, 0x70, 0x69, 0x2a, 0x01, 0x01, 0x72, 0x19, 0x0a, 0x17, 0x4f, 0x70, 0x74, 0x69, 0x6d, - 0x75, 0x73, 0x20, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x20, 0x53, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x39, 0x2e, 0x67, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6d, + 0x70, 0x61, 0x6e, 0x79, 0x2e, 0x6f, 0x70, 0x74, 0x69, 0x6d, 0x75, 0x73, 0x2e, 0x63, 0x6f, 0x72, + 0x65, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x52, 0x65, 0x67, 0x69, 0x73, 0x74, + 0x65, 0x72, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x1b, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x15, 0x22, 0x10, 0x2f, 0x76, 0x31, 0x62, 0x65, + 0x74, 0x61, 0x31, 0x2f, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x3a, 0x01, 0x2a, 0x12, 0x97, + 0x01, 0x0a, 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x12, + 0x35, 0x2e, 0x67, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x6e, 0x79, 0x2e, 0x6f, 0x70, + 0x74, 0x69, 0x6d, 0x75, 0x73, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, + 0x61, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x36, 0x2e, 0x67, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6d, + 0x70, 0x61, 0x6e, 0x79, 0x2e, 0x6f, 0x70, 0x74, 0x69, 0x6d, 0x75, 0x73, 0x2e, 0x63, 0x6f, 0x72, + 0x65, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x72, + 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x18, + 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x12, 0x12, 0x10, 0x2f, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, + 0x2f, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0xa0, 0x01, 0x0a, 0x0a, 0x47, 0x65, 0x74, + 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x33, 0x2e, 0x67, 0x6f, 0x74, 0x6f, 0x63, 0x6f, + 0x6d, 0x70, 0x61, 0x6e, 0x79, 0x2e, 0x6f, 0x70, 0x74, 0x69, 0x6d, 0x75, 0x73, 0x2e, 0x63, 0x6f, + 0x72, 0x65, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x50, 0x72, + 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x34, 0x2e, 0x67, + 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x6e, 0x79, 0x2e, 0x6f, 0x70, 0x74, 0x69, 0x6d, + 0x75, 0x73, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x76, 0x31, 0x62, 0x65, 0x74, 0x61, 0x31, 0x2e, + 0x47, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x22, 0x27, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x21, 0x12, 0x1f, 0x2f, 0x76, 0x31, 0x62, + 0x65, 0x74, 0x61, 0x31, 0x2f, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x2f, 0x7b, 0x70, 0x72, + 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x7d, 0x42, 0x97, 0x01, 0x0a, 0x1e, + 0x63, 0x6f, 0x6d, 0x2e, 0x67, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x6e, 0x79, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x6e, 0x2e, 0x6f, 0x70, 0x74, 0x69, 0x6d, 0x75, 0x73, 0x42, 0x15, + 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4d, 0x61, + 0x6e, 0x61, 0x67, 0x65, 0x72, 0x50, 0x01, 0x5a, 0x1e, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x74, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x6e, 0x2f, + 0x6f, 0x70, 0x74, 0x69, 0x6d, 0x75, 0x73, 0x92, 0x41, 0x3b, 0x12, 0x05, 0x32, 0x03, 0x30, 0x2e, + 0x31, 0x1a, 0x0e, 0x31, 0x32, 0x37, 0x2e, 0x30, 0x2e, 0x30, 0x2e, 0x31, 0x3a, 0x39, 0x31, 0x30, + 0x30, 0x22, 0x04, 0x2f, 0x61, 0x70, 0x69, 0x2a, 0x01, 0x01, 0x72, 0x19, 0x0a, 0x17, 0x4f, 0x70, + 0x74, 0x69, 0x6d, 0x75, 0x73, 0x20, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x20, 0x53, 0x65, + 0x72, 0x76, 0x69, 0x63, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -594,7 +612,7 @@ func file_gotocompany_optimus_core_v1beta1_project_proto_rawDescGZIP() []byte { return file_gotocompany_optimus_core_v1beta1_project_proto_rawDescData } -var file_gotocompany_optimus_core_v1beta1_project_proto_msgTypes = make([]protoimpl.MessageInfo, 10) +var file_gotocompany_optimus_core_v1beta1_project_proto_msgTypes = make([]protoimpl.MessageInfo, 11) var file_gotocompany_optimus_core_v1beta1_project_proto_goTypes = []interface{}{ (*RegisterProjectRequest)(nil), // 0: gotocompany.optimus.core.v1beta1.RegisterProjectRequest (*RegisterProjectResponse)(nil), // 1: gotocompany.optimus.core.v1beta1.RegisterProjectResponse @@ -606,25 +624,27 @@ var file_gotocompany_optimus_core_v1beta1_project_proto_goTypes = []interface{}{ nil, // 7: gotocompany.optimus.core.v1beta1.ProjectSpecification.ConfigEntry (*ProjectSpecification_ProjectPreset)(nil), // 8: gotocompany.optimus.core.v1beta1.ProjectSpecification.ProjectPreset nil, // 9: gotocompany.optimus.core.v1beta1.ProjectSpecification.PresetsEntry + nil, // 10: gotocompany.optimus.core.v1beta1.ProjectSpecification.VariablesEntry } var file_gotocompany_optimus_core_v1beta1_project_proto_depIdxs = []int32{ - 6, // 0: gotocompany.optimus.core.v1beta1.RegisterProjectRequest.project:type_name -> gotocompany.optimus.core.v1beta1.ProjectSpecification - 6, // 1: gotocompany.optimus.core.v1beta1.ListProjectsResponse.projects:type_name -> gotocompany.optimus.core.v1beta1.ProjectSpecification - 6, // 2: gotocompany.optimus.core.v1beta1.GetProjectResponse.project:type_name -> gotocompany.optimus.core.v1beta1.ProjectSpecification - 7, // 3: gotocompany.optimus.core.v1beta1.ProjectSpecification.config:type_name -> gotocompany.optimus.core.v1beta1.ProjectSpecification.ConfigEntry - 9, // 4: gotocompany.optimus.core.v1beta1.ProjectSpecification.presets:type_name -> gotocompany.optimus.core.v1beta1.ProjectSpecification.PresetsEntry - 8, // 5: gotocompany.optimus.core.v1beta1.ProjectSpecification.PresetsEntry.value:type_name -> gotocompany.optimus.core.v1beta1.ProjectSpecification.ProjectPreset - 0, // 6: gotocompany.optimus.core.v1beta1.ProjectService.RegisterProject:input_type -> gotocompany.optimus.core.v1beta1.RegisterProjectRequest - 2, // 7: gotocompany.optimus.core.v1beta1.ProjectService.ListProjects:input_type -> gotocompany.optimus.core.v1beta1.ListProjectsRequest - 4, // 8: gotocompany.optimus.core.v1beta1.ProjectService.GetProject:input_type -> gotocompany.optimus.core.v1beta1.GetProjectRequest - 1, // 9: gotocompany.optimus.core.v1beta1.ProjectService.RegisterProject:output_type -> gotocompany.optimus.core.v1beta1.RegisterProjectResponse - 3, // 10: gotocompany.optimus.core.v1beta1.ProjectService.ListProjects:output_type -> gotocompany.optimus.core.v1beta1.ListProjectsResponse - 5, // 11: gotocompany.optimus.core.v1beta1.ProjectService.GetProject:output_type -> gotocompany.optimus.core.v1beta1.GetProjectResponse - 9, // [9:12] is the sub-list for method output_type - 6, // [6:9] is the sub-list for method input_type - 6, // [6:6] is the sub-list for extension type_name - 6, // [6:6] is the sub-list for extension extendee - 0, // [0:6] is the sub-list for field type_name + 6, // 0: gotocompany.optimus.core.v1beta1.RegisterProjectRequest.project:type_name -> gotocompany.optimus.core.v1beta1.ProjectSpecification + 6, // 1: gotocompany.optimus.core.v1beta1.ListProjectsResponse.projects:type_name -> gotocompany.optimus.core.v1beta1.ProjectSpecification + 6, // 2: gotocompany.optimus.core.v1beta1.GetProjectResponse.project:type_name -> gotocompany.optimus.core.v1beta1.ProjectSpecification + 7, // 3: gotocompany.optimus.core.v1beta1.ProjectSpecification.config:type_name -> gotocompany.optimus.core.v1beta1.ProjectSpecification.ConfigEntry + 9, // 4: gotocompany.optimus.core.v1beta1.ProjectSpecification.presets:type_name -> gotocompany.optimus.core.v1beta1.ProjectSpecification.PresetsEntry + 10, // 5: gotocompany.optimus.core.v1beta1.ProjectSpecification.variables:type_name -> gotocompany.optimus.core.v1beta1.ProjectSpecification.VariablesEntry + 8, // 6: gotocompany.optimus.core.v1beta1.ProjectSpecification.PresetsEntry.value:type_name -> gotocompany.optimus.core.v1beta1.ProjectSpecification.ProjectPreset + 0, // 7: gotocompany.optimus.core.v1beta1.ProjectService.RegisterProject:input_type -> gotocompany.optimus.core.v1beta1.RegisterProjectRequest + 2, // 8: gotocompany.optimus.core.v1beta1.ProjectService.ListProjects:input_type -> gotocompany.optimus.core.v1beta1.ListProjectsRequest + 4, // 9: gotocompany.optimus.core.v1beta1.ProjectService.GetProject:input_type -> gotocompany.optimus.core.v1beta1.GetProjectRequest + 1, // 10: gotocompany.optimus.core.v1beta1.ProjectService.RegisterProject:output_type -> gotocompany.optimus.core.v1beta1.RegisterProjectResponse + 3, // 11: gotocompany.optimus.core.v1beta1.ProjectService.ListProjects:output_type -> gotocompany.optimus.core.v1beta1.ListProjectsResponse + 5, // 12: gotocompany.optimus.core.v1beta1.ProjectService.GetProject:output_type -> gotocompany.optimus.core.v1beta1.GetProjectResponse + 10, // [10:13] is the sub-list for method output_type + 7, // [7:10] is the sub-list for method input_type + 7, // [7:7] is the sub-list for extension type_name + 7, // [7:7] is the sub-list for extension extendee + 0, // [0:7] is the sub-list for field type_name } func init() { file_gotocompany_optimus_core_v1beta1_project_proto_init() } @@ -736,7 +756,7 @@ func file_gotocompany_optimus_core_v1beta1_project_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_gotocompany_optimus_core_v1beta1_project_proto_rawDesc, NumEnums: 0, - NumMessages: 10, + NumMessages: 11, NumExtensions: 0, NumServices: 1, }, diff --git a/protos/gotocompany/optimus/core/v1beta1/project.swagger.json b/protos/gotocompany/optimus/core/v1beta1/project.swagger.json index c972d4121a..5f83762eb7 100644 --- a/protos/gotocompany/optimus/core/v1beta1/project.swagger.json +++ b/protos/gotocompany/optimus/core/v1beta1/project.swagger.json @@ -203,6 +203,12 @@ "additionalProperties": { "$ref": "#/definitions/ProjectSpecificationProjectPreset" } + }, + "variables": { + "type": "object", + "additionalProperties": { + "type": "string" + } } } }, From 137060e6db525c1f201813966773192761b6b014 Mon Sep 17 00:00:00 2001 From: "Ahmad N. F." Date: Wed, 9 Oct 2024 18:03:58 +0700 Subject: [PATCH 2/8] feat: add namespace variables to fix tests --- core/tenant/namespace.go | 10 ++++++++ .../postgres/tenant/namespace_repository.go | 23 ++++++++++--------- .../postgres/tenant/project_repository.go | 2 +- tests/bench/job/job_repo_test.go | 3 ++- tests/bench/resource/backup_repo_test.go | 3 ++- tests/bench/resource/resource_repo_test.go | 3 ++- tests/bench/scheduler/job_repo_test.go | 3 ++- tests/bench/scheduler/job_run_repo_test.go | 3 ++- .../bench/scheduler/operator_run_repo_test.go | 3 ++- tests/bench/tenant/namespace_repo_test.go | 3 ++- tests/bench/tenant/project_repo_test.go | 7 +++--- tests/bench/tenant/secret_repo_test.go | 3 ++- 12 files changed, 43 insertions(+), 23 deletions(-) diff --git a/core/tenant/namespace.go b/core/tenant/namespace.go index 75d6ec00e2..5283101e14 100644 --- a/core/tenant/namespace.go +++ b/core/tenant/namespace.go @@ -25,6 +25,7 @@ type Namespace struct { projectName ProjectName config map[string]string + variables map[string]string } func (n *Namespace) Name() NamespaceName { @@ -53,6 +54,15 @@ func (n *Namespace) GetConfigs() map[string]string { return confs } +// GetVariables returns a clone of namespace variables +func (n *Namespace) GetVariables() map[string]string { + vars := make(map[string]string, len(n.variables)) + for k, v := range n.variables { + vars[k] = v + } + return vars +} + func NewNamespace(name string, projName ProjectName, config map[string]string) (*Namespace, error) { nsName, err := NamespaceNameFrom(name) if err != nil { diff --git a/internal/store/postgres/tenant/namespace_repository.go b/internal/store/postgres/tenant/namespace_repository.go index cf542fe353..dc98dfd95e 100644 --- a/internal/store/postgres/tenant/namespace_repository.go +++ b/internal/store/postgres/tenant/namespace_repository.go @@ -17,13 +17,14 @@ type NamespaceRepository struct { } const ( - namespaceColumns = `id, name, config, project_name, created_at, updated_at` + namespaceColumns = `id, name, config, variables, project_name, created_at, updated_at` ) type Namespace struct { - ID uuid.UUID - Name string - Config map[string]string + ID uuid.UUID + Name string + Config map[string]string + Variables map[string]string ProjectName string @@ -44,10 +45,10 @@ func (n *NamespaceRepository) Save(ctx context.Context, namespace *tenant.Namesp _, err := n.get(ctx, namespace.ProjectName(), namespace.Name()) if err != nil { if errors.Is(err, pgx.ErrNoRows) { - insertNamespace := `INSERT INTO namespace (name, config, project_name, created_at, updated_at) -VALUES ($1, $2, $3, now(), now())` + insertNamespace := `INSERT INTO namespace (name, config, variables, project_name, created_at, updated_at) +VALUES ($1, $2, $3, $4, now(), now())` - _, err = n.db.Exec(ctx, insertNamespace, namespace.Name(), namespace.GetConfigs(), namespace.ProjectName()) + _, err = n.db.Exec(ctx, insertNamespace, namespace.Name(), namespace.GetConfigs(), namespace.GetVariables(), namespace.ProjectName()) return errors.WrapIfErr(tenant.EntityNamespace, "unable to save namespace", err) } return errors.Wrap(tenant.EntityNamespace, "unable to save namespace", err) @@ -56,8 +57,8 @@ VALUES ($1, $2, $3, now(), now())` if len(namespace.GetConfigs()) == 0 { return errors.NewError(errors.ErrFailedPrecond, tenant.EntityNamespace, "empty config") } - updateNamespaceQuery := `UPDATE namespace n SET config=$1, updated_at=now() WHERE n.name = $2 AND n.project_name=$3` - _, err = n.db.Exec(ctx, updateNamespaceQuery, namespace.GetConfigs(), namespace.Name(), namespace.ProjectName()) + updateNamespaceQuery := `UPDATE namespace n SET config=$1, variables=$2, updated_at=now() WHERE n.name = $3 AND n.project_name=$4` + _, err = n.db.Exec(ctx, updateNamespaceQuery, namespace.GetConfigs(), namespace.GetVariables(), namespace.Name(), namespace.ProjectName()) return errors.WrapIfErr(tenant.EntityProject, "unable to update namespace", err) } @@ -77,7 +78,7 @@ func (n *NamespaceRepository) get(ctx context.Context, projName tenant.ProjectNa getNamespaceByNameQuery := `SELECT ` + namespaceColumns + ` FROM namespace WHERE project_name = $1 AND name = $2 AND deleted_at IS NULL` err := n.db.QueryRow(ctx, getNamespaceByNameQuery, projName, name). - Scan(&namespace.ID, &namespace.Name, &namespace.Config, &namespace.ProjectName, &namespace.CreatedAt, &namespace.UpdatedAt) + Scan(&namespace.ID, &namespace.Name, &namespace.Config, &namespace.Variables, &namespace.ProjectName, &namespace.CreatedAt, &namespace.UpdatedAt) if err != nil { return Namespace{}, err } @@ -97,7 +98,7 @@ WHERE project_name = $1 AND deleted_at IS NULL` for rows.Next() { var ns Namespace - err = rows.Scan(&ns.ID, &ns.Name, &ns.Config, &ns.ProjectName, &ns.CreatedAt, &ns.UpdatedAt) + err = rows.Scan(&ns.ID, &ns.Name, &ns.Config, &ns.Variables, &ns.ProjectName, &ns.CreatedAt, &ns.UpdatedAt) if err != nil { return nil, errors.Wrap(tenant.EntityNamespace, "error in GetAll", err) } diff --git a/internal/store/postgres/tenant/project_repository.go b/internal/store/postgres/tenant/project_repository.go index ebfd9ae1a6..3088190b6f 100644 --- a/internal/store/postgres/tenant/project_repository.go +++ b/internal/store/postgres/tenant/project_repository.go @@ -38,7 +38,7 @@ func (repo ProjectRepository) Save(ctx context.Context, tenantProject *tenant.Pr _, err := repo.get(ctx, tenantProject.Name()) if err != nil { if errors.Is(err, pgx.ErrNoRows) { - insertProjectQuery := `INSERT INTO project (name, config, variables, created_at, updated_at) VALUES ($1, $2, $3 now(), now())` + insertProjectQuery := `INSERT INTO project (name, config, variables, created_at, updated_at) VALUES ($1, $2, $3, now(), now())` _, err = repo.db.Exec(ctx, insertProjectQuery, tenantProject.Name(), tenantProject.GetConfigs(), tenantProject.GetVariables()) return errors.WrapIfErr(tenant.EntityProject, "unable to save project", err) } diff --git a/tests/bench/job/job_repo_test.go b/tests/bench/job/job_repo_test.go index 3c2412af72..ecd80606b7 100644 --- a/tests/bench/job/job_repo_test.go +++ b/tests/bench/job/job_repo_test.go @@ -31,7 +31,8 @@ func BenchmarkJobRepository(b *testing.B) { serviceTenant.ProjectSchedulerHost: "http://localhost:8082", serviceTenant.ProjectStoragePathKey: "gs://location", } - project, err := serviceTenant.NewProject(projectName, config) + vars := map[string]string{} + project, err := serviceTenant.NewProject(projectName, config, vars) assert.NoError(b, err) namespaceName := "namespace_test" diff --git a/tests/bench/resource/backup_repo_test.go b/tests/bench/resource/backup_repo_test.go index fc9431481e..f36167954e 100644 --- a/tests/bench/resource/backup_repo_test.go +++ b/tests/bench/resource/backup_repo_test.go @@ -21,12 +21,13 @@ func BenchmarkBackupRepository(b *testing.B) { ctx := context.Background() projectName := "project_test" namespaceName := "namespace_test" + vars := map[string]string{} proj, err := serviceTenant.NewProject(projectName, map[string]string{ "bucket": "gs://some_folder-2", serviceTenant.ProjectSchedulerHost: "host", serviceTenant.ProjectStoragePathKey: "gs://location", - }) + }, vars) assert.NoError(b, err) namespace, err := serviceTenant.NewNamespace(namespaceName, proj.Name(), map[string]string{ diff --git a/tests/bench/resource/resource_repo_test.go b/tests/bench/resource/resource_repo_test.go index 70e0191c16..c6073350ee 100644 --- a/tests/bench/resource/resource_repo_test.go +++ b/tests/bench/resource/resource_repo_test.go @@ -32,7 +32,8 @@ func BenchmarkResourceRepository(b *testing.B) { serviceTenant.ProjectSchedulerHost: "http://localhost:8082", serviceTenant.ProjectStoragePathKey: "gs://location", } - project, err := serviceTenant.NewProject(projectName, config) + vars := map[string]string{} + project, err := serviceTenant.NewProject(projectName, config, vars) assert.NoError(b, err) namespaceName := "namespace_test" diff --git a/tests/bench/scheduler/job_repo_test.go b/tests/bench/scheduler/job_repo_test.go index 3825454fe3..4cb58c0355 100644 --- a/tests/bench/scheduler/job_repo_test.go +++ b/tests/bench/scheduler/job_repo_test.go @@ -29,7 +29,8 @@ func BenchmarkJobRepository(b *testing.B) { serviceTenant.ProjectSchedulerHost: "http://localhost:8082", serviceTenant.ProjectStoragePathKey: "gs://location", } - project, err := serviceTenant.NewProject("project_for_test", config) + vars := map[string]string{} + project, err := serviceTenant.NewProject("project_for_test", config, vars) assert.NoError(b, err) namespace, err := serviceTenant.NewNamespace("namespace_for_test", project.Name(), config) assert.NoError(b, err) diff --git a/tests/bench/scheduler/job_run_repo_test.go b/tests/bench/scheduler/job_run_repo_test.go index f222a537ed..7ae8a01770 100644 --- a/tests/bench/scheduler/job_run_repo_test.go +++ b/tests/bench/scheduler/job_run_repo_test.go @@ -30,7 +30,8 @@ func BenchmarkJobRunRepository(b *testing.B) { serviceTenant.ProjectSchedulerHost: "http://localhost:8082", serviceTenant.ProjectStoragePathKey: "gs://location", } - project, err := serviceTenant.NewProject("project_for_test", config) + vars := map[string]string{} + project, err := serviceTenant.NewProject("project_for_test", config, vars) assert.NoError(b, err) namespace, err := serviceTenant.NewNamespace("namespace_for_test", project.Name(), config) assert.NoError(b, err) diff --git a/tests/bench/scheduler/operator_run_repo_test.go b/tests/bench/scheduler/operator_run_repo_test.go index f7758e0e18..df5ad5664b 100644 --- a/tests/bench/scheduler/operator_run_repo_test.go +++ b/tests/bench/scheduler/operator_run_repo_test.go @@ -30,7 +30,8 @@ func BenchmarkOperatorRunRepository(b *testing.B) { serviceTenant.ProjectSchedulerHost: "http://localhost:8082", serviceTenant.ProjectStoragePathKey: "gs://location", } - project, err := serviceTenant.NewProject("project_for_test", config) + vars := map[string]string{} + project, err := serviceTenant.NewProject("project_for_test", config, vars) assert.NoError(b, err) namespace, err := serviceTenant.NewNamespace("namespace_for_test", project.Name(), config) assert.NoError(b, err) diff --git a/tests/bench/tenant/namespace_repo_test.go b/tests/bench/tenant/namespace_repo_test.go index 58ef2692f8..afe2d1a47c 100644 --- a/tests/bench/tenant/namespace_repo_test.go +++ b/tests/bench/tenant/namespace_repo_test.go @@ -25,7 +25,8 @@ func BenchmarkNamespaceRepository(b *testing.B) { serviceTenant.ProjectSchedulerHost: "http://localhost:8082", serviceTenant.ProjectStoragePathKey: "gs://location", } - project, err := serviceTenant.NewProject("project_for_test", config) + vars := map[string]string{} + project, err := serviceTenant.NewProject("project_for_test", config, vars) assert.NoError(b, err) ctx := context.Background() diff --git a/tests/bench/tenant/project_repo_test.go b/tests/bench/tenant/project_repo_test.go index e71b3ec3e6..5cc23ea9e7 100644 --- a/tests/bench/tenant/project_repo_test.go +++ b/tests/bench/tenant/project_repo_test.go @@ -32,6 +32,7 @@ func BenchmarkProjectRepository(b *testing.B) { serviceTenant.ProjectSchedulerHost: "http://localhost:8082", serviceTenant.ProjectStoragePathKey: "gs://location", } + vars := map[string]string{} b.Run("Save", func(b *testing.B) { db := dbSetup() @@ -41,7 +42,7 @@ func BenchmarkProjectRepository(b *testing.B) { for i := 0; i < b.N; i++ { name := fmt.Sprintf("project_for_test_%d", i) - project, err := serviceTenant.NewProject(name, config) + project, err := serviceTenant.NewProject(name, config, vars) assert.NoError(b, err) actualError := repo.Save(ctx, project) @@ -55,7 +56,7 @@ func BenchmarkProjectRepository(b *testing.B) { projectNames := make([]string, maxNumberOfProjects) for i := 0; i < maxNumberOfProjects; i++ { name := fmt.Sprintf("project_for_test_%d", i) - project, err := serviceTenant.NewProject(name, config) + project, err := serviceTenant.NewProject(name, config, vars) assert.NoError(b, err) actualError := repo.Save(ctx, project) @@ -83,7 +84,7 @@ func BenchmarkProjectRepository(b *testing.B) { repo := repoTenant.NewProjectRepository(db) for i := 0; i < maxNumberOfProjects; i++ { name := fmt.Sprintf("project_for_test_%d", i) - project, err := serviceTenant.NewProject(name, config) + project, err := serviceTenant.NewProject(name, config, vars) assert.NoError(b, err) actualError := repo.Save(ctx, project) diff --git a/tests/bench/tenant/secret_repo_test.go b/tests/bench/tenant/secret_repo_test.go index 2d0821621f..c420ee06dc 100644 --- a/tests/bench/tenant/secret_repo_test.go +++ b/tests/bench/tenant/secret_repo_test.go @@ -25,7 +25,8 @@ func BenchmarkSecretRepository(b *testing.B) { serviceTenant.ProjectSchedulerHost: "http://localhost:8082", serviceTenant.ProjectStoragePathKey: "gs://location", } - project, err := serviceTenant.NewProject("project_for_test", config) + vars := map[string]string{} + project, err := serviceTenant.NewProject("project_for_test", config, vars) assert.NoError(b, err) namespace, err := serviceTenant.NewNamespace("namespace_for_test", project.Name(), config) assert.NoError(b, err) From 28aade54777a84b3f51b46cd143ad82b91e254e6 Mon Sep 17 00:00:00 2001 From: "Ahmad N. F." Date: Thu, 10 Oct 2024 15:36:52 +0700 Subject: [PATCH 3/8] feat: add namespace variable integration --- client/cmd/namespace/describe.go | 16 ++++++++-- client/cmd/namespace/list.go | 5 ++-- client/cmd/namespace/register.go | 5 ++-- config/config_client.go | 1 + core/job/handler/v1beta1/job_test.go | 2 +- core/job/job_test.go | 2 +- core/job/resolver/upstream_resolver_test.go | 2 +- core/job/service/job_service_test.go | 4 +-- core/scheduler/service/events_service_test.go | 2 +- .../service/executor_input_compiler_test.go | 2 +- .../service/job_run_asset_compiler_test.go | 2 +- core/scheduler/service/replay_service_test.go | 3 +- core/tenant/handler/v1beta1/namespace.go | 11 +++++-- core/tenant/handler/v1beta1/namespace_test.go | 30 ++++++++++++------- core/tenant/handler/v1beta1/project_test.go | 20 ++++++++----- core/tenant/namespace.go | 14 ++++++++- core/tenant/namespace_test.go | 16 ++++++++-- core/tenant/project_test.go | 14 ++++++++- core/tenant/service/namespace_service_test.go | 15 ++++++---- core/tenant/service/project_service_test.go | 5 ++-- core/tenant/service/tenant_service_test.go | 4 ++- core/tenant/tenant.go | 13 ++++++-- core/tenant/tenant_test.go | 10 +++++-- .../store/postgres/job/job_repository_test.go | 6 ++-- .../postgres/resource/repository_test.go | 4 +-- .../postgres/scheduler/job_repository_test.go | 2 +- .../postgres/tenant/namespace_repository.go | 5 +--- .../tenant/namespace_repository_test.go | 28 +++++------------ .../postgres/tenant/secret_repository_test.go | 4 +-- tests/bench/job/job_repo_test.go | 3 +- tests/bench/resource/backup_repo_test.go | 2 +- tests/bench/resource/resource_repo_test.go | 9 +++--- tests/bench/scheduler/job_repo_test.go | 9 +++--- tests/bench/scheduler/job_run_repo_test.go | 9 +++--- .../bench/scheduler/operator_run_repo_test.go | 9 +++--- tests/bench/tenant/namespace_repo_test.go | 13 ++++---- tests/bench/tenant/secret_repo_test.go | 9 +++--- 37 files changed, 194 insertions(+), 116 deletions(-) diff --git a/client/cmd/namespace/describe.go b/client/cmd/namespace/describe.go index a9b1a8fd79..75974980c1 100644 --- a/client/cmd/namespace/describe.go +++ b/client/cmd/namespace/describe.go @@ -123,8 +123,9 @@ func (d *describeCommand) getNamespace() (*config.Namespace, error) { return nil, fmt.Errorf("unable to get namespace [%s]: %w", d.namespaceName, err) } return &config.Namespace{ - Name: response.GetNamespace().Name, - Config: response.GetNamespace().Config, + Name: response.GetNamespace().Name, + Config: response.GetNamespace().Config, + Variables: response.GetNamespace().Variables, }, nil } @@ -138,5 +139,16 @@ func (*describeCommand) stringifyNamespace(namespace *config.Namespace) string { output += fmt.Sprintf("\t%s: %s", key, value) } } + + output += "\n" + + if len(namespace.Variables) == 0 { + output += "variables: {}" + } else { + output += "variables:\n" + for key, value := range namespace.Variables { + output += fmt.Sprintf("\t%s: %s", key, value) + } + } return output } diff --git a/client/cmd/namespace/list.go b/client/cmd/namespace/list.go index 957f8a127d..3a221d36a4 100644 --- a/client/cmd/namespace/list.go +++ b/client/cmd/namespace/list.go @@ -125,8 +125,9 @@ func (l *listCommand) listNamespacesFromServer(serverHost, projectName string) ( output := make([]*config.Namespace, len(response.Namespaces)) for i, n := range response.Namespaces { output[i] = &config.Namespace{ - Name: n.GetName(), - Config: n.GetConfig(), + Name: n.GetName(), + Config: n.GetConfig(), + Variables: n.GetVariables(), } } return output, nil diff --git a/client/cmd/namespace/register.go b/client/cmd/namespace/register.go index 6c4fd15344..daaa11837a 100644 --- a/client/cmd/namespace/register.go +++ b/client/cmd/namespace/register.go @@ -111,8 +111,9 @@ func RegisterNamespace(l log.Logger, conn *grpc.ClientConn, projectName string, _, err := namespaceServiceClient.RegisterProjectNamespace(ctx, &pb.RegisterProjectNamespaceRequest{ ProjectName: projectName, Namespace: &pb.NamespaceSpecification{ - Name: namespace.Name, - Config: namespace.Config, + Name: namespace.Name, + Config: namespace.Config, + Variables: namespace.Variables, }, }) if err != nil { diff --git a/config/config_client.go b/config/config_client.go index b21f0aac17..d3fa3586eb 100644 --- a/config/config_client.go +++ b/config/config_client.go @@ -41,6 +41,7 @@ type Auth struct { type Namespace struct { Name string `mapstructure:"name"` Config map[string]string `mapstructure:"config"` + Variables map[string]string `mapstructure:"variables"` Job Job `mapstructure:"job"` Datastore []Datastore `mapstructure:"datastore"` } diff --git a/core/job/handler/v1beta1/job_test.go b/core/job/handler/v1beta1/job_test.go index 7d544de4f3..39993ae7c4 100644 --- a/core/job/handler/v1beta1/job_test.go +++ b/core/job/handler/v1beta1/job_test.go @@ -37,7 +37,7 @@ func TestNewJobHandler(t *testing.T) { namespace, _ := tenant.NewNamespace("test-ns", project.Name(), map[string]string{ "bucket": "gs://ns_bucket", - }) + }, map[string]string{}) sampleTenant, _ := tenant.NewTenant(project.Name().String(), namespace.Name().String()) jobVersion := 1 startDate, err := job.ScheduleDateFrom("2022-10-01") diff --git a/core/job/job_test.go b/core/job/job_test.go index 06f732e64d..da9f2d6507 100644 --- a/core/job/job_test.go +++ b/core/job/job_test.go @@ -22,7 +22,7 @@ func TestEntityJob(t *testing.T) { namespace, _ := tenant.NewNamespace("test-ns", project.Name(), map[string]string{ "bucket": "gs://ns_bucket", - }) + }, map[string]string{}) sampleTenant, _ := tenant.NewTenant(project.Name().String(), namespace.Name().String()) jobVersion := 1 startDate, _ := job.ScheduleDateFrom("2022-10-01") diff --git a/core/job/resolver/upstream_resolver_test.go b/core/job/resolver/upstream_resolver_test.go index cbde52da39..12d3047792 100644 --- a/core/job/resolver/upstream_resolver_test.go +++ b/core/job/resolver/upstream_resolver_test.go @@ -28,7 +28,7 @@ func TestUpstreamResolver(t *testing.T) { namespace, _ := tenant.NewNamespace("test-ns", project.Name(), map[string]string{ "bucket": "gs://ns_bucket", - }) + }, map[string]string{}) sampleTenant, _ := tenant.NewTenant(project.Name().String(), namespace.Name().String()) externalTenant, _ := tenant.NewTenant("external-proj", "external-namespace") jobVersion := 1 diff --git a/core/job/service/job_service_test.go b/core/job/service/job_service_test.go index 39bf6ff5e3..cd79e96d99 100644 --- a/core/job/service/job_service_test.go +++ b/core/job/service/job_service_test.go @@ -38,7 +38,7 @@ func TestJobService(t *testing.T) { namespace, _ := tenant.NewNamespace("test-ns", project.Name(), map[string]string{ "bucket": "gs://ns_bucket", - }) + }, map[string]string{}) secret1, err := tenant.NewPlainTextSecret("table_name", "secret_table") assert.Nil(t, err) @@ -48,7 +48,7 @@ func TestJobService(t *testing.T) { otherNamespace, _ := tenant.NewNamespace("other-ns", project.Name(), map[string]string{ "bucket": "gs://other_ns_bucket", - }) + }, map[string]string{}) otherTenant, _ := tenant.NewTenant(project.Name().String(), otherNamespace.Name().String()) secret2, err := tenant.NewPlainTextSecret("bucket", "gs://some_secret_bucket") assert.Nil(t, err) diff --git a/core/scheduler/service/events_service_test.go b/core/scheduler/service/events_service_test.go index d498895574..24a190fe0c 100644 --- a/core/scheduler/service/events_service_test.go +++ b/core/scheduler/service/events_service_test.go @@ -24,7 +24,7 @@ func TestNotificationService(t *testing.T) { "STORAGE_PATH": "somePath", "SCHEDULER_HOST": "localhost", }, map[string]string{}) - namespace, _ := tenant.NewNamespace("ns1", project.Name(), map[string]string{}) + namespace, _ := tenant.NewNamespace("ns1", project.Name(), map[string]string{}, map[string]string{}) tnnt, _ := tenant.NewTenant(project.Name().String(), namespace.Name().String()) startDate, _ := time.Parse(time.RFC3339, "2022-03-20T02:00:00+00:00") jobName := scheduler.JobName("job1") diff --git a/core/scheduler/service/executor_input_compiler_test.go b/core/scheduler/service/executor_input_compiler_test.go index 22154bfa54..2fa4cbdbc7 100644 --- a/core/scheduler/service/executor_input_compiler_test.go +++ b/core/scheduler/service/executor_input_compiler_test.go @@ -29,7 +29,7 @@ func TestExecutorCompiler(t *testing.T) { "STORAGE_PATH": "somePath", "SCHEDULER_HOST": "localhost", }, map[string]string{}) - namespace, _ := tenant.NewNamespace("ns1", project.Name(), map[string]string{}) + namespace, _ := tenant.NewNamespace("ns1", project.Name(), map[string]string{}, map[string]string{}) secret1, _ := tenant.NewPlainTextSecret("secretName", "secretValue") secret2, _ := tenant.NewPlainTextSecret("secret2Name", "secret2Value") diff --git a/core/scheduler/service/job_run_asset_compiler_test.go b/core/scheduler/service/job_run_asset_compiler_test.go index a938a6a1c6..090fd00a57 100644 --- a/core/scheduler/service/job_run_asset_compiler_test.go +++ b/core/scheduler/service/job_run_asset_compiler_test.go @@ -24,7 +24,7 @@ func TestJobAssetsCompiler(t *testing.T) { "STORAGE_PATH": "somePath", "SCHEDULER_HOST": "localhost", }, map[string]string{}) - namespace, _ := tenant.NewNamespace("ns1", project.Name(), map[string]string{}) + namespace, _ := tenant.NewNamespace("ns1", project.Name(), map[string]string{}, map[string]string{}) tnnt, _ := tenant.NewTenant(project.Name().String(), namespace.Name().String()) currentTime := time.Now() scheduleTime := currentTime.Add(-time.Hour) diff --git a/core/scheduler/service/replay_service_test.go b/core/scheduler/service/replay_service_test.go index 8307597983..70a576141f 100644 --- a/core/scheduler/service/replay_service_test.go +++ b/core/scheduler/service/replay_service_test.go @@ -63,7 +63,8 @@ func TestReplayService(t *testing.T) { "SCHEDULER_HOST": "http://localhost", } projectVars := map[string]string{} - namespaceEntity, _ := tenant.NewNamespace(namespaceName.String(), projName, namespaceCfg) + nsVars := map[string]string{} + namespaceEntity, _ := tenant.NewNamespace(namespaceName.String(), projName, namespaceCfg, nsVars) projectEntity, _ := tenant.NewProject(projName.String(), projectCfg, projectVars) tenantWithDetails, _ := tenant.NewTenantDetails(projectEntity, namespaceEntity, tenant.PlainTextSecrets{}) diff --git a/core/tenant/handler/v1beta1/namespace.go b/core/tenant/handler/v1beta1/namespace.go index 18bd7938b2..4fbcdace07 100644 --- a/core/tenant/handler/v1beta1/namespace.go +++ b/core/tenant/handler/v1beta1/namespace.go @@ -111,13 +111,18 @@ func fromNamespaceProto(conf *pb.NamespaceSpecification, projName tenant.Project for key, val := range conf.GetConfig() { namespaceConf[strings.ToUpper(key)] = val } + namespaceVariables := map[string]string{} + for key, val := range conf.GetVariables() { + namespaceVariables[strings.ToUpper(key)] = val + } - return tenant.NewNamespace(conf.GetName(), projName, namespaceConf) + return tenant.NewNamespace(conf.GetName(), projName, namespaceConf, namespaceVariables) } func toNamespaceProto(ns *tenant.Namespace) *pb.NamespaceSpecification { return &pb.NamespaceSpecification{ - Name: ns.Name().String(), - Config: ns.GetConfigs(), + Name: ns.Name().String(), + Config: ns.GetConfigs(), + Variables: ns.GetVariables(), } } diff --git a/core/tenant/handler/v1beta1/namespace_test.go b/core/tenant/handler/v1beta1/namespace_test.go index 25ac679933..69027b1148 100644 --- a/core/tenant/handler/v1beta1/namespace_test.go +++ b/core/tenant/handler/v1beta1/namespace_test.go @@ -20,11 +20,15 @@ func TestNamespaceHandler(t *testing.T) { projectConf := map[string]string{ tenant.ProjectSchedulerHost: "host", tenant.ProjectStoragePathKey: "gs://location", - "BUCKET": "gs://some_folder", } - projectVars := map[string]string{} + projectVars := map[string]string{ + "BUCKET": "gs://some_folder", + } savedProject, _ := tenant.NewProject("savedProj", projectConf, projectVars) - savedNS, _ := tenant.NewNamespace("savedNS", savedProject.Name(), map[string]string{"BUCKET": "gs://some_folder"}) + + namespaceConf := map[string]string{} + namespaceVars := map[string]string{"BUCKET": "gs://some_folder"} + savedNS, _ := tenant.NewNamespace("savedNS", savedProject.Name(), namespaceConf, namespaceVars) t.Run("RegisterProjectNamespace", func(t *testing.T) { t.Run("returns error when project name is empty", func(t *testing.T) { @@ -34,8 +38,9 @@ func TestNamespaceHandler(t *testing.T) { registerReq := pb.RegisterProjectNamespaceRequest{ ProjectName: "", Namespace: &pb.NamespaceSpecification{ - Name: "NS", - Config: map[string]string{"BUCKET": "gs://some_folder"}, + Name: "NS", + Config: map[string]string{}, + Variables: map[string]string{"BUCKET": "gs://some_folder"}, }, } @@ -51,8 +56,9 @@ func TestNamespaceHandler(t *testing.T) { registerReq := pb.RegisterProjectNamespaceRequest{ ProjectName: "proj", Namespace: &pb.NamespaceSpecification{ - Name: "", - Config: map[string]string{"BUCKET": "gs://some_folder"}, + Name: "", + Config: map[string]string{}, + Variables: map[string]string{"BUCKET": "gs://some_folder"}, }, } @@ -71,8 +77,9 @@ func TestNamespaceHandler(t *testing.T) { registerReq := pb.RegisterProjectNamespaceRequest{ ProjectName: "proj", Namespace: &pb.NamespaceSpecification{ - Name: "ns", - Config: map[string]string{"BUCKET": "gs://some_folder"}, + Name: "ns", + Config: map[string]string{}, + Variables: map[string]string{"BUCKET": "gs://some_folder"}, }, } @@ -91,8 +98,9 @@ func TestNamespaceHandler(t *testing.T) { registerReq := pb.RegisterProjectNamespaceRequest{ ProjectName: "proj", Namespace: &pb.NamespaceSpecification{ - Name: "ns", - Config: map[string]string{"BUCKET": "gs://some_folder"}, + Name: "ns", + Config: map[string]string{}, + Variables: map[string]string{"BUCKET": "gs://some_folder"}, }, } diff --git a/core/tenant/handler/v1beta1/project_test.go b/core/tenant/handler/v1beta1/project_test.go index 8319062616..d0da3dce0c 100644 --- a/core/tenant/handler/v1beta1/project_test.go +++ b/core/tenant/handler/v1beta1/project_test.go @@ -21,9 +21,10 @@ func TestProjectHandler(t *testing.T) { conf := map[string]string{ tenant.ProjectSchedulerHost: "host", tenant.ProjectStoragePathKey: "gs://location", - "BUCKET": "gs://some_folder", } - projectVars := map[string]string{} + projectVars := map[string]string{ + "BUCKET": "gs://some_folder", + } savedProject, _ := tenant.NewProject("savedProj", conf, projectVars) t.Run("RegisterProject", func(t *testing.T) { @@ -32,8 +33,9 @@ func TestProjectHandler(t *testing.T) { handler := v1beta1.NewProjectHandler(logger, projectService) registerReq := pb.RegisterProjectRequest{Project: &pb.ProjectSpecification{ - Name: "", - Config: conf, + Name: "", + Config: conf, + Variables: projectVars, }} _, err := handler.RegisterProject(ctx, ®isterReq) @@ -69,8 +71,9 @@ func TestProjectHandler(t *testing.T) { handler := v1beta1.NewProjectHandler(logger, projectService) registerReq := pb.RegisterProjectRequest{Project: &pb.ProjectSpecification{ - Name: "proj", - Config: conf, + Name: "proj", + Config: conf, + Variables: projectVars, }} _, err := handler.RegisterProject(ctx, ®isterReq) @@ -86,8 +89,9 @@ func TestProjectHandler(t *testing.T) { handler := v1beta1.NewProjectHandler(logger, projectService) registerReq := pb.RegisterProjectRequest{Project: &pb.ProjectSpecification{ - Name: "proj", - Config: conf, + Name: "proj", + Config: conf, + Variables: projectVars, Presets: map[string]*pb.ProjectSpecification_ProjectPreset{ "yesterday": { Name: "yesterday", diff --git a/core/tenant/namespace.go b/core/tenant/namespace.go index 5283101e14..01be307cf1 100644 --- a/core/tenant/namespace.go +++ b/core/tenant/namespace.go @@ -1,6 +1,8 @@ package tenant import ( + "fmt" + "github.com/goto/optimus/internal/errors" ) @@ -54,6 +56,15 @@ func (n *Namespace) GetConfigs() map[string]string { return confs } +func (n *Namespace) GetVariable(key string) (string, error) { + for k, v := range n.variables { + if key == k { + return v, nil + } + } + return "", errors.NotFound(EntityNamespace, fmt.Sprintf("namespace variable not found: %s", key)) +} + // GetVariables returns a clone of namespace variables func (n *Namespace) GetVariables() map[string]string { vars := make(map[string]string, len(n.variables)) @@ -63,7 +74,7 @@ func (n *Namespace) GetVariables() map[string]string { return vars } -func NewNamespace(name string, projName ProjectName, config map[string]string) (*Namespace, error) { +func NewNamespace(name string, projName ProjectName, config, variables map[string]string) (*Namespace, error) { nsName, err := NamespaceNameFrom(name) if err != nil { return nil, err @@ -76,6 +87,7 @@ func NewNamespace(name string, projName ProjectName, config map[string]string) ( return &Namespace{ name: nsName, config: config, + variables: variables, projectName: projName, }, nil } diff --git a/core/tenant/namespace_test.go b/core/tenant/namespace_test.go index e8fde4b417..c60c2656a6 100644 --- a/core/tenant/namespace_test.go +++ b/core/tenant/namespace_test.go @@ -27,19 +27,19 @@ func TestEntityNamespace(t *testing.T) { projName, _ := tenant.ProjectNameFrom("optimus-proj") t.Run("return error when name is empty", func(t *testing.T) { - _, err := tenant.NewNamespace("", projName, map[string]string{}) + _, err := tenant.NewNamespace("", projName, map[string]string{}, map[string]string{}) assert.NotNil(t, err) assert.EqualError(t, err, "invalid argument for entity namespace: namespace name is empty") }) t.Run("return error when validation fails due to project name", func(t *testing.T) { - _, err := tenant.NewNamespace("t-namespace", "", map[string]string{}) + _, err := tenant.NewNamespace("t-namespace", "", map[string]string{}, map[string]string{}) assert.NotNil(t, err) assert.EqualError(t, err, "invalid argument for entity namespace: project name is empty") }) t.Run("creates namespace object", func(t *testing.T) { - ns, err := tenant.NewNamespace("t-namespace", projName, map[string]string{"a": "b"}) + ns, err := tenant.NewNamespace("t-namespace", projName, map[string]string{"a": "b"}, map[string]string{"ns_var": "ns_val"}) assert.Nil(t, err) assert.Equal(t, "t-namespace", ns.Name().String()) @@ -53,6 +53,16 @@ func TestEntityNamespace(t *testing.T) { _, err = ns.GetConfig("non-existent") assert.NotNil(t, err) assert.EqualError(t, err, "not found for entity namespace: namespace config not found non-existent") + + assert.NotNil(t, ns.GetVariables()) + + nsVar, err := ns.GetVariable("ns_var") + assert.Nil(t, err) + assert.Equal(t, "ns_val", nsVar) + + _, err = ns.GetVariable("non-existent") + assert.NotNil(t, err) + assert.EqualError(t, err, "not found for entity namespace: namespace variable not found: non-existent") }) }) } diff --git a/core/tenant/project_test.go b/core/tenant/project_test.go index 9484985df1..abe5e53bdc 100644 --- a/core/tenant/project_test.go +++ b/core/tenant/project_test.go @@ -42,7 +42,9 @@ func TestEntityProject(t *testing.T) { project, err := tenant.NewProject("t-optimus", map[string]string{ tenant.ProjectSchedulerHost: "b", tenant.ProjectStoragePathKey: "d", - }, map[string]string{}) + }, map[string]string{ + "PROJECT": "optimus", + }) assert.Nil(t, err) assert.NotNil(t, project) @@ -57,6 +59,16 @@ func TestEntityProject(t *testing.T) { val2, err := project.GetConfig(tenant.ProjectStoragePathKey) assert.Nil(t, err) assert.Equal(t, "d", val2) + + assert.NotNil(t, project.GetVariables()) + + projVar, err := project.GetVariable("PROJECT") + assert.Nil(t, err) + assert.Equal(t, "optimus", projVar) + + _, err = project.GetVariable("non-existent") + assert.NotNil(t, err) + assert.EqualError(t, err, "not found for entity project: variable not found: non-existent") }) }) } diff --git a/core/tenant/service/namespace_service_test.go b/core/tenant/service/namespace_service_test.go index e514175541..4eda31edc3 100644 --- a/core/tenant/service/namespace_service_test.go +++ b/core/tenant/service/namespace_service_test.go @@ -17,11 +17,12 @@ func TestNamespaceService(t *testing.T) { conf := map[string]string{ tenant.ProjectSchedulerHost: "host", tenant.ProjectStoragePathKey: "gs://location", - "BUCKET": "gs://some_folder", } - projectVars := map[string]string{} + projectVars := map[string]string{ + "BUCKET": "gs://some_folder", + } savedProject, _ := tenant.NewProject("savedProj", conf, projectVars) - savedNS, _ := tenant.NewNamespace("savedNS", savedProject.Name(), map[string]string{}) + savedNS, _ := tenant.NewNamespace("savedNS", savedProject.Name(), map[string]string{}, map[string]string{}) t.Run("Save", func(t *testing.T) { t.Run("returns error when fails in service", func(t *testing.T) { @@ -29,7 +30,9 @@ func TestNamespaceService(t *testing.T) { nsRepo.On("Save", ctx, mock.Anything).Return(errors.New("error in saving")) defer nsRepo.AssertExpectations(t) - toSaveNS, _ := tenant.NewNamespace("ns", savedProject.Name(), map[string]string{"BUCKET": "gs://some_folder"}) + nsConf := map[string]string{} + nsVars := map[string]string{"BUCKET": "gs://some_folder"} + toSaveNS, _ := tenant.NewNamespace("ns", savedProject.Name(), nsConf, nsVars) namespaceService := service.NewNamespaceService(nsRepo) err := namespaceService.Save(ctx, toSaveNS) @@ -42,7 +45,9 @@ func TestNamespaceService(t *testing.T) { nsRepo.On("Save", ctx, mock.Anything).Return(nil) defer nsRepo.AssertExpectations(t) - toSaveNS, _ := tenant.NewNamespace("ns", savedProject.Name(), map[string]string{"BUCKET": "gs://some_folder"}) + nsConf := map[string]string{} + nsVars := map[string]string{"BUCKET": "gs://some_folder"} + toSaveNS, _ := tenant.NewNamespace("ns", savedProject.Name(), nsConf, nsVars) namespaceService := service.NewNamespaceService(nsRepo) err := namespaceService.Save(ctx, toSaveNS) diff --git a/core/tenant/service/project_service_test.go b/core/tenant/service/project_service_test.go index 962f074a03..9ca6159195 100644 --- a/core/tenant/service/project_service_test.go +++ b/core/tenant/service/project_service_test.go @@ -17,9 +17,10 @@ func TestProjectService(t *testing.T) { conf := map[string]string{ tenant.ProjectSchedulerHost: "host", tenant.ProjectStoragePathKey: "gs://location", - "BUCKET": "gs://some_folder", } - projectVars := map[string]string{} + projectVars := map[string]string{ + "BUCKET": "gs://some_folder", + } savedProject, _ := tenant.NewProject("savedProj", conf, projectVars) preset, err := tenant.NewPreset("test_preset", "preset for testing", "1d", "1h", "", "") diff --git a/core/tenant/service/tenant_service_test.go b/core/tenant/service/tenant_service_test.go index bf55555216..0d8aed1171 100644 --- a/core/tenant/service/tenant_service_test.go +++ b/core/tenant/service/tenant_service_test.go @@ -22,7 +22,9 @@ func TestTenantService(t *testing.T) { } projectVars := map[string]string{} proj, _ := tenant.NewProject("testProj", conf, projectVars) - ns, _ := tenant.NewNamespace("testNS", proj.Name(), map[string]string{}) + + nsVars := map[string]string{} + ns, _ := tenant.NewNamespace("testNS", proj.Name(), map[string]string{}, nsVars) tnnt, _ := tenant.NewTenant(proj.Name().String(), ns.Name().String()) logger := log.NewLogrus() diff --git a/core/tenant/tenant.go b/core/tenant/tenant.go index a792c68458..c46abe0d38 100644 --- a/core/tenant/tenant.go +++ b/core/tenant/tenant.go @@ -87,12 +87,18 @@ func (w *WithDetails) GetConfig(key string) (string, error) { } func (w *WithDetails) GetVariable(key string) (string, error) { - variable, err := w.project.GetVariable(key) + variable, err := w.namespace.GetVariable(key) if err == nil { return variable, nil } - return "", errors.NotFound(EntityTenant, fmt.Sprintf("variable %s not present in tenant", key)) + // key not present in namespace, check project + variable, err = w.project.GetVariable(key) + if err == nil { + return variable, nil + } + + return "", errors.NotFound(EntityTenant, fmt.Sprintf("variable not present in tenant: %s", key)) } func (w *WithDetails) GetConfigs() map[string]string { @@ -101,7 +107,8 @@ func (w *WithDetails) GetConfigs() map[string]string { } func (w *WithDetails) GetVariables() map[string]string { - return w.project.GetVariables() + namespaceVars := w.namespace.GetVariables() + return utils.MergeMaps(namespaceVars, w.project.GetVariables()) } func (w *WithDetails) Project() *Project { diff --git a/core/tenant/tenant_test.go b/core/tenant/tenant_test.go index 60b068f3a8..91550f6fcc 100644 --- a/core/tenant/tenant_test.go +++ b/core/tenant/tenant_test.go @@ -40,10 +40,14 @@ func TestAggregateRootTenant(t *testing.T) { } project, _ := tenant.NewProject("test-project", projectConf, projectVars) + + nsVars := map[string]string{ + "WAREHOUSE_PROJECT": "project-ns", + } namespace, _ := tenant.NewNamespace("test-ns", project.Name(), map[string]string{ "BUCKET": "gs://ns_folder", "OTHER_CONFIG": "optimus", - }) + }, nsVars) t.Run("return error when project not present", func(t *testing.T) { _, err := tenant.NewTenantDetails(nil, nil, nil) @@ -133,7 +137,7 @@ func TestAggregateRootTenant(t *testing.T) { val, err := details.GetVariable("WAREHOUSE_PROJECT") assert.NoError(t, err) - assert.Equal(t, "project", val) + assert.Equal(t, "project-ns", val) }) t.Run("returns error if a referred variable does not exist", func(t *testing.T) { details, err := tenant.NewTenantDetails(project, namespace, nil) @@ -141,7 +145,7 @@ func TestAggregateRootTenant(t *testing.T) { _, err = details.GetVariable("NONEXISTENT_VARIABLE") assert.Error(t, err) - assert.EqualError(t, err, "not found for entity tenant: variable NONEXISTENT_VARIABLE not present in tenant") + assert.EqualError(t, err, "not found for entity tenant: variable not present in tenant: NONEXISTENT_VARIABLE") }) }) }) diff --git a/internal/store/postgres/job/job_repository_test.go b/internal/store/postgres/job/job_repository_test.go index ccbf4b80a0..4cc897b487 100644 --- a/internal/store/postgres/job/job_repository_test.go +++ b/internal/store/postgres/job/job_repository_test.go @@ -41,19 +41,19 @@ func TestPostgresJobRepository(t *testing.T) { namespace, err := tenant.NewNamespace("test-ns", proj.Name(), map[string]string{ "bucket": "gs://ns_bucket", - }) + }, map[string]string{}) assert.NoError(t, err) otherNamespace, err := tenant.NewNamespace("other-ns", proj.Name(), map[string]string{ "bucket": "gs://ns_bucket", - }) + }, map[string]string{}) assert.NoError(t, err) otherNamespace2, err := tenant.NewNamespace("other-ns", otherProj.Name(), map[string]string{ "bucket": "gs://ns_bucket", - }) + }, map[string]string{}) assert.NoError(t, err) sampleTenant, err := tenant.NewTenant(proj.Name().String(), namespace.Name().String()) assert.NoError(t, err) diff --git a/internal/store/postgres/resource/repository_test.go b/internal/store/postgres/resource/repository_test.go index fae96da1ce..98ce9853c5 100644 --- a/internal/store/postgres/resource/repository_test.go +++ b/internal/store/postgres/resource/repository_test.go @@ -449,7 +449,7 @@ func dbSetup() *pgxpool.Pool { ns, _ := tenant.NewNamespace("n-optimus-1", proj.Name(), map[string]string{ "bucket": "gs://ns_bucket", - }) + }, map[string]string{}) err = namespaceRepo.Save(ctx, ns) if err != nil { panic(err) @@ -458,7 +458,7 @@ func dbSetup() *pgxpool.Pool { ns2, _ := tenant.NewNamespace("n-optimus-2", proj.Name(), map[string]string{ "bucket": "gs://ns_bucket", - }) + }, map[string]string{}) err = namespaceRepo.Save(ctx, ns2) if err != nil { panic(err) diff --git a/internal/store/postgres/scheduler/job_repository_test.go b/internal/store/postgres/scheduler/job_repository_test.go index d4b2b4703c..5e8b8235ae 100644 --- a/internal/store/postgres/scheduler/job_repository_test.go +++ b/internal/store/postgres/scheduler/job_repository_test.go @@ -188,7 +188,7 @@ func addJobs(ctx context.Context, t *testing.T, pool *pgxpool.Pool) map[string]* namespace, err := tenant.NewNamespace("test-ns", proj.Name(), map[string]string{ "bucket": "gs://ns_bucket", - }) + }, map[string]string{}) assert.NoError(t, err) namespaceRepo := tenantPostgres.NewNamespaceRepository(pool) diff --git a/internal/store/postgres/tenant/namespace_repository.go b/internal/store/postgres/tenant/namespace_repository.go index dc98dfd95e..e09ba13c12 100644 --- a/internal/store/postgres/tenant/namespace_repository.go +++ b/internal/store/postgres/tenant/namespace_repository.go @@ -38,7 +38,7 @@ func (n *Namespace) toTenantNamespace() (*tenant.Namespace, error) { return nil, err } - return tenant.NewNamespace(n.Name, projName, n.Config) + return tenant.NewNamespace(n.Name, projName, n.Config, n.Variables) } func (n *NamespaceRepository) Save(ctx context.Context, namespace *tenant.Namespace) error { @@ -54,9 +54,6 @@ VALUES ($1, $2, $3, $4, now(), now())` return errors.Wrap(tenant.EntityNamespace, "unable to save namespace", err) } - if len(namespace.GetConfigs()) == 0 { - return errors.NewError(errors.ErrFailedPrecond, tenant.EntityNamespace, "empty config") - } updateNamespaceQuery := `UPDATE namespace n SET config=$1, variables=$2, updated_at=now() WHERE n.name = $3 AND n.project_name=$4` _, err = n.db.Exec(ctx, updateNamespaceQuery, namespace.GetConfigs(), namespace.GetVariables(), namespace.Name(), namespace.ProjectName()) return errors.WrapIfErr(tenant.EntityProject, "unable to update namespace", err) diff --git a/internal/store/postgres/tenant/namespace_repository_test.go b/internal/store/postgres/tenant/namespace_repository_test.go index 348773c337..11029071e8 100644 --- a/internal/store/postgres/tenant/namespace_repository_test.go +++ b/internal/store/postgres/tenant/namespace_repository_test.go @@ -26,7 +26,7 @@ func TestPostgresNamespaceRepository(t *testing.T) { ns, _ := tenant.NewNamespace("n-optimus-1", proj.Name(), map[string]string{ "bucket": "gs://ns_bucket", - }) + }, map[string]string{}) ctx := context.Background() dbSetup := func() *pgxpool.Pool { @@ -54,7 +54,7 @@ func TestPostgresNamespaceRepository(t *testing.T) { assert.Nil(t, err) assert.Equal(t, "n-optimus-1", savedNs.Name().String()) - ns2, _ := tenant.NewNamespace("n-optimus-2", proj.Name(), ns.GetConfigs()) + ns2, _ := tenant.NewNamespace("n-optimus-2", proj.Name(), ns.GetConfigs(), ns.GetVariables()) err = repo.Save(ctx, ns2) assert.Nil(t, err) @@ -79,7 +79,9 @@ func TestPostgresNamespaceRepository(t *testing.T) { conf := proj.GetConfigs() conf["STORAGE"] = "gs://some_place" - ns2, _ := tenant.NewNamespace(ns.Name().String(), ns.ProjectName(), conf) + vars := proj.GetVariables() + vars["PROJECT"] = "optimus" + ns2, _ := tenant.NewNamespace(ns.Name().String(), ns.ProjectName(), conf, vars) err = repo.Save(ctx, ns2) assert.Nil(t, err) @@ -89,23 +91,9 @@ func TestPostgresNamespaceRepository(t *testing.T) { config, err := updatedNS.GetConfig("STORAGE") assert.Nil(t, err) assert.Equal(t, "gs://some_place", config) - }) - t.Run("should not update if config is empty", func(t *testing.T) { - db := dbSetup() - repo := postgres.NewNamespaceRepository(db) - - err := repo.Save(ctx, ns) + updatedVars, err := updatedNS.GetVariable("PROJECT") assert.Nil(t, err) - - savedNS, err := repo.GetByName(ctx, proj.Name(), ns.Name()) - assert.Nil(t, err) - assert.Equal(t, "n-optimus-1", savedNS.Name().String()) - - ns2, _ := tenant.NewNamespace(ns.Name().String(), ns.ProjectName(), map[string]string{}) - - err = repo.Save(ctx, ns2) - assert.NotNil(t, err) - assert.EqualError(t, err, "failed precondition for entity namespace: empty config") + assert.Equal(t, "optimus", updatedVars) }) }) t.Run("GetAll", func(t *testing.T) { @@ -116,7 +104,7 @@ func TestPostgresNamespaceRepository(t *testing.T) { err := repo.Save(ctx, ns) assert.Nil(t, err) - ns2, _ := tenant.NewNamespace("t-optimus-2", proj.Name(), ns.GetConfigs()) + ns2, _ := tenant.NewNamespace("t-optimus-2", proj.Name(), ns.GetConfigs(), ns.GetVariables()) err = repo.Save(ctx, ns2) assert.Nil(t, err) diff --git a/internal/store/postgres/tenant/secret_repository_test.go b/internal/store/postgres/tenant/secret_repository_test.go index e5c217d9b7..61931a56ba 100644 --- a/internal/store/postgres/tenant/secret_repository_test.go +++ b/internal/store/postgres/tenant/secret_repository_test.go @@ -26,11 +26,11 @@ func TestPostgresSecretRepository(t *testing.T) { namespace, _ := tenant.NewNamespace("test-ns", proj.Name(), map[string]string{ "bucket": "gs://ns_bucket", - }) + }, map[string]string{}) otherNamespace, _ := tenant.NewNamespace("other-ns", proj.Name(), map[string]string{ "bucket": "gs://ns_bucket", - }) + }, map[string]string{}) dbSetup := func() *pgxpool.Pool { pool := setup.TestPool() diff --git a/tests/bench/job/job_repo_test.go b/tests/bench/job/job_repo_test.go index ecd80606b7..ac118a101e 100644 --- a/tests/bench/job/job_repo_test.go +++ b/tests/bench/job/job_repo_test.go @@ -36,7 +36,8 @@ func BenchmarkJobRepository(b *testing.B) { assert.NoError(b, err) namespaceName := "namespace_test" - namespace, err := serviceTenant.NewNamespace(namespaceName, project.Name(), config) + nsVars := map[string]string{} + namespace, err := serviceTenant.NewNamespace(namespaceName, project.Name(), config, nsVars) assert.NoError(b, err) tnnt, err := serviceTenant.NewTenant(project.Name().String(), namespace.Name().String()) diff --git a/tests/bench/resource/backup_repo_test.go b/tests/bench/resource/backup_repo_test.go index f36167954e..c1af9dbad7 100644 --- a/tests/bench/resource/backup_repo_test.go +++ b/tests/bench/resource/backup_repo_test.go @@ -30,7 +30,7 @@ func BenchmarkBackupRepository(b *testing.B) { }, vars) assert.NoError(b, err) namespace, err := serviceTenant.NewNamespace(namespaceName, proj.Name(), - map[string]string{ + map[string]string{}, map[string]string{ "bucket": "gs://ns_bucket", }) assert.NoError(b, err) diff --git a/tests/bench/resource/resource_repo_test.go b/tests/bench/resource/resource_repo_test.go index c6073350ee..7c4646a44b 100644 --- a/tests/bench/resource/resource_repo_test.go +++ b/tests/bench/resource/resource_repo_test.go @@ -27,17 +27,18 @@ func BenchmarkResourceRepository(b *testing.B) { projectName := "project_test" transporterKafkaBrokerKey := "KAFKA_BROKERS" config := map[string]string{ - "bucket": "gs://folder_for_test", - transporterKafkaBrokerKey: "192.168.1.1:8080,192.168.1.1:8081", serviceTenant.ProjectSchedulerHost: "http://localhost:8082", serviceTenant.ProjectStoragePathKey: "gs://location", } - vars := map[string]string{} + vars := map[string]string{ + "bucket": "gs://folder_for_test", + transporterKafkaBrokerKey: "192.168.1.1:8080,192.168.1.1:8081", + } project, err := serviceTenant.NewProject(projectName, config, vars) assert.NoError(b, err) namespaceName := "namespace_test" - namespace, err := serviceTenant.NewNamespace(namespaceName, project.Name(), config) + namespace, err := serviceTenant.NewNamespace(namespaceName, project.Name(), config, vars) assert.NoError(b, err) tnnt, err := serviceTenant.NewTenant(project.Name().String(), namespace.Name().String()) diff --git a/tests/bench/scheduler/job_repo_test.go b/tests/bench/scheduler/job_repo_test.go index 4cb58c0355..907a2da2ba 100644 --- a/tests/bench/scheduler/job_repo_test.go +++ b/tests/bench/scheduler/job_repo_test.go @@ -24,15 +24,16 @@ func BenchmarkJobRepository(b *testing.B) { transporterKafkaBrokerKey := "KAFKA_BROKERS" config := map[string]string{ - "bucket": "gs://folder_for_test", - transporterKafkaBrokerKey: "192.168.1.1:8080,192.168.1.1:8081", serviceTenant.ProjectSchedulerHost: "http://localhost:8082", serviceTenant.ProjectStoragePathKey: "gs://location", } - vars := map[string]string{} + vars := map[string]string{ + "bucket": "gs://folder_for_test", + transporterKafkaBrokerKey: "192.168.1.1:8080,192.168.1.1:8081", + } project, err := serviceTenant.NewProject("project_for_test", config, vars) assert.NoError(b, err) - namespace, err := serviceTenant.NewNamespace("namespace_for_test", project.Name(), config) + namespace, err := serviceTenant.NewNamespace("namespace_for_test", project.Name(), config, vars) assert.NoError(b, err) tnnt, err := serviceTenant.NewTenant(project.Name().String(), namespace.Name().String()) assert.NoError(b, err) diff --git a/tests/bench/scheduler/job_run_repo_test.go b/tests/bench/scheduler/job_run_repo_test.go index 7ae8a01770..74cfef4fdf 100644 --- a/tests/bench/scheduler/job_run_repo_test.go +++ b/tests/bench/scheduler/job_run_repo_test.go @@ -25,15 +25,16 @@ func BenchmarkJobRunRepository(b *testing.B) { transporterKafkaBrokerKey := "KAFKA_BROKERS" config := map[string]string{ - "bucket": "gs://folder_for_test", - transporterKafkaBrokerKey: "192.168.1.1:8080,192.168.1.1:8081", serviceTenant.ProjectSchedulerHost: "http://localhost:8082", serviceTenant.ProjectStoragePathKey: "gs://location", } - vars := map[string]string{} + vars := map[string]string{ + "bucket": "gs://folder_for_test", + transporterKafkaBrokerKey: "192.168.1.1:8080,192.168.1.1:8081", + } project, err := serviceTenant.NewProject("project_for_test", config, vars) assert.NoError(b, err) - namespace, err := serviceTenant.NewNamespace("namespace_for_test", project.Name(), config) + namespace, err := serviceTenant.NewNamespace("namespace_for_test", project.Name(), config, vars) assert.NoError(b, err) tnnt, err := serviceTenant.NewTenant(project.Name().String(), namespace.Name().String()) assert.NoError(b, err) diff --git a/tests/bench/scheduler/operator_run_repo_test.go b/tests/bench/scheduler/operator_run_repo_test.go index df5ad5664b..9a10894725 100644 --- a/tests/bench/scheduler/operator_run_repo_test.go +++ b/tests/bench/scheduler/operator_run_repo_test.go @@ -25,15 +25,16 @@ func BenchmarkOperatorRunRepository(b *testing.B) { transporterKafkaBrokerKey := "KAFKA_BROKERS" config := map[string]string{ - "bucket": "gs://folder_for_test", - transporterKafkaBrokerKey: "192.168.1.1:8080,192.168.1.1:8081", serviceTenant.ProjectSchedulerHost: "http://localhost:8082", serviceTenant.ProjectStoragePathKey: "gs://location", } - vars := map[string]string{} + vars := map[string]string{ + "bucket": "gs://folder_for_test", + transporterKafkaBrokerKey: "192.168.1.1:8080,192.168.1.1:8081", + } project, err := serviceTenant.NewProject("project_for_test", config, vars) assert.NoError(b, err) - namespace, err := serviceTenant.NewNamespace("namespace_for_test", project.Name(), config) + namespace, err := serviceTenant.NewNamespace("namespace_for_test", project.Name(), config, vars) assert.NoError(b, err) tnnt, err := serviceTenant.NewTenant(project.Name().String(), namespace.Name().String()) assert.NoError(b, err) diff --git a/tests/bench/tenant/namespace_repo_test.go b/tests/bench/tenant/namespace_repo_test.go index afe2d1a47c..4525ba50e6 100644 --- a/tests/bench/tenant/namespace_repo_test.go +++ b/tests/bench/tenant/namespace_repo_test.go @@ -20,12 +20,13 @@ func BenchmarkNamespaceRepository(b *testing.B) { transporterKafkaBrokerKey := "KAFKA_BROKERS" config := map[string]string{ - "bucket": "gs://folder_for_test", - transporterKafkaBrokerKey: "192.168.1.1:8080,192.168.1.1:8081", serviceTenant.ProjectSchedulerHost: "http://localhost:8082", serviceTenant.ProjectStoragePathKey: "gs://location", } - vars := map[string]string{} + vars := map[string]string{ + "bucket": "gs://folder_for_test", + transporterKafkaBrokerKey: "192.168.1.1:8080,192.168.1.1:8081", + } project, err := serviceTenant.NewProject("project_for_test", config, vars) assert.NoError(b, err) @@ -52,7 +53,7 @@ func BenchmarkNamespaceRepository(b *testing.B) { for i := 0; i < b.N; i++ { name := fmt.Sprintf("namespace_for_test_%d", i) - namespace, err := serviceTenant.NewNamespace(name, project.Name(), config) + namespace, err := serviceTenant.NewNamespace(name, project.Name(), config, vars) assert.NoError(b, err) actualError := repo.Save(ctx, namespace) @@ -66,7 +67,7 @@ func BenchmarkNamespaceRepository(b *testing.B) { namespaceNames := make([]string, maxNumberOfNamespaces) for i := 0; i < maxNumberOfNamespaces; i++ { name := fmt.Sprintf("namespace_for_test_%d", i) - namespace, err := serviceTenant.NewNamespace(name, project.Name(), config) + namespace, err := serviceTenant.NewNamespace(name, project.Name(), config, vars) assert.NoError(b, err) actualError := repo.Save(ctx, namespace) @@ -94,7 +95,7 @@ func BenchmarkNamespaceRepository(b *testing.B) { repo := repoTenant.NewNamespaceRepository(db) for i := 0; i < maxNumberOfNamespaces; i++ { name := fmt.Sprintf("namespace_for_test_%d", i) - namespace, err := serviceTenant.NewNamespace(name, project.Name(), config) + namespace, err := serviceTenant.NewNamespace(name, project.Name(), config, vars) assert.NoError(b, err) actualError := repo.Save(ctx, namespace) diff --git a/tests/bench/tenant/secret_repo_test.go b/tests/bench/tenant/secret_repo_test.go index c420ee06dc..180b1d0c9f 100644 --- a/tests/bench/tenant/secret_repo_test.go +++ b/tests/bench/tenant/secret_repo_test.go @@ -20,15 +20,16 @@ func BenchmarkSecretRepository(b *testing.B) { transporterKafkaBrokerKey := "KAFKA_BROKERS" config := map[string]string{ - "bucket": "gs://folder_for_test", - transporterKafkaBrokerKey: "192.168.1.1:8080,192.168.1.1:8081", serviceTenant.ProjectSchedulerHost: "http://localhost:8082", serviceTenant.ProjectStoragePathKey: "gs://location", } - vars := map[string]string{} + vars := map[string]string{ + "bucket": "gs://folder_for_test", + transporterKafkaBrokerKey: "192.168.1.1:8080,192.168.1.1:8081", + } project, err := serviceTenant.NewProject("project_for_test", config, vars) assert.NoError(b, err) - namespace, err := serviceTenant.NewNamespace("namespace_for_test", project.Name(), config) + namespace, err := serviceTenant.NewNamespace("namespace_for_test", project.Name(), config, vars) assert.NoError(b, err) ctx := context.Background() From c25dade51ebd2cd9d6aabe300418234d8dcf0b1f Mon Sep 17 00:00:00 2001 From: "Ahmad N. F." Date: Tue, 15 Oct 2024 10:36:44 +0700 Subject: [PATCH 4/8] feat: add compile resource spec logic on services --- core/resource/resource.go | 4 + core/resource/service/resource_service.go | 144 +++++-- .../resource/service/resource_service_test.go | 355 ++++++++++++++---- core/tenant/tenant.go | 2 +- core/tenant/tenant_test.go | 2 +- server/optimus.go | 4 +- 6 files changed, 404 insertions(+), 107 deletions(-) diff --git a/core/resource/resource.go b/core/resource/resource.go index 8a1e372480..840711a1b2 100644 --- a/core/resource/resource.go +++ b/core/resource/resource.go @@ -196,6 +196,10 @@ func (r *Resource) Version() int32 { return r.metadata.Version } +func (r *Resource) UpdateSpec(spec map[string]any) { + r.spec = spec +} + func (r *Resource) Equal(incoming *Resource) bool { if r == nil || incoming == nil { return r == nil && incoming == nil diff --git a/core/resource/service/resource_service.go b/core/resource/service/resource_service.go index ee8d529395..c45c898b5d 100644 --- a/core/resource/service/resource_service.go +++ b/core/resource/service/resource_service.go @@ -2,6 +2,7 @@ package service import ( "context" + "encoding/json" "fmt" "reflect" "strings" @@ -14,10 +15,15 @@ import ( "github.com/goto/optimus/core/job" "github.com/goto/optimus/core/resource" "github.com/goto/optimus/core/tenant" + "github.com/goto/optimus/internal/compiler" "github.com/goto/optimus/internal/errors" "github.com/goto/optimus/internal/writer" ) +const ( + projectConfigPrefix = "GLOBAL__" +) + type ResourceRepository interface { Create(ctx context.Context, res *resource.Resource) error Update(ctx context.Context, res *resource.Resource) error @@ -51,6 +57,15 @@ type EventHandler interface { HandleEvent(moderator.Event) } +type TenantDetailsGetter interface { + GetDetails(ctx context.Context, tnnt tenant.Tenant) (*tenant.WithDetails, error) +} + +type CompileEngine interface { + Compile(templateMap map[string]string, context map[string]any) (map[string]string, error) + CompileString(input string, context map[string]any) (string, error) +} + type ResourceService struct { repo ResourceRepository mgr ResourceManager @@ -60,6 +75,9 @@ type ResourceService struct { logger log.Logger eventHandler EventHandler alertHandler AlertManager + + tenantDetailsGetter TenantDetailsGetter + compileEngine CompileEngine } type AlertManager interface { @@ -70,33 +88,31 @@ func NewResourceService( logger log.Logger, repo ResourceRepository, downstreamRefresher DownstreamRefresher, mgr ResourceManager, eventHandler EventHandler, downstreamResolver DownstreamResolver, alertManager AlertManager, + tenantDetailsGetter TenantDetailsGetter, compileEngine CompileEngine, ) *ResourceService { return &ResourceService{ - repo: repo, - mgr: mgr, - refresher: downstreamRefresher, - downstreamResolver: downstreamResolver, - logger: logger, - eventHandler: eventHandler, - alertHandler: alertManager, + repo: repo, + mgr: mgr, + refresher: downstreamRefresher, + downstreamResolver: downstreamResolver, + logger: logger, + eventHandler: eventHandler, + alertHandler: alertManager, + tenantDetailsGetter: tenantDetailsGetter, + compileEngine: compileEngine, } } func (rs ResourceService) Create(ctx context.Context, incoming *resource.Resource) error { // nolint:gocritic - if err := rs.mgr.Validate(incoming); err != nil { - rs.logger.Error("error validating resource [%s]: %s", incoming.FullName(), err) - return err - } - - incoming.MarkValidationSuccess() - urn, err := rs.mgr.GetURN(incoming) + compiledSpec, err := rs.compileSpec(ctx, incoming) if err != nil { - rs.logger.Error("error validating resource [%s]: %s", incoming.FullName(), err) - return err + rs.logger.Warn("suppressed error compiling spec for resource [%s]: %s", incoming.FullName(), err) + compiledSpec = incoming.Spec() } - err = incoming.UpdateURN(urn) + incoming.UpdateSpec(compiledSpec) + + err = rs.validateAndGenerateURN(incoming) if err != nil { - rs.logger.Error("error updating urn of resource [%s]: %s", incoming.FullName(), err) return err } @@ -138,20 +154,15 @@ func (rs ResourceService) Create(ctx context.Context, incoming *resource.Resourc } func (rs ResourceService) Update(ctx context.Context, incoming *resource.Resource, logWriter writer.LogWriter) error { // nolint:gocritic - if err := rs.mgr.Validate(incoming); err != nil { - rs.logger.Error("error validating resource [%s]: %s", incoming.FullName(), err) - return err - } - - incoming.MarkValidationSuccess() - urn, err := rs.mgr.GetURN(incoming) + compiledSpec, err := rs.compileSpec(ctx, incoming) if err != nil { - rs.logger.Error("error validating resource [%s]: %s", incoming.FullName(), err) - return err + rs.logger.Warn("suppressed error compiling spec for resource [%s]: %s", incoming.FullName(), err) + compiledSpec = incoming.Spec() } - err = incoming.UpdateURN(urn) + incoming.UpdateSpec(compiledSpec) + + err = rs.validateAndGenerateURN(incoming) if err != nil { - rs.logger.Error("error updating urn of resource [%s]: %s", incoming.FullName(), err) return err } @@ -193,21 +204,15 @@ func (rs ResourceService) Update(ctx context.Context, incoming *resource.Resourc } func (rs ResourceService) Upsert(ctx context.Context, incoming *resource.Resource, logWriter writer.LogWriter) error { // nolint:gocritic - if err := rs.mgr.Validate(incoming); err != nil { - rs.logger.Error("error validating resource [%s]: %s", incoming.FullName(), err) - return err - } - incoming.MarkValidationSuccess() - - urn, err := rs.mgr.GetURN(incoming) + compiledSpec, err := rs.compileSpec(ctx, incoming) if err != nil { - rs.logger.Error("error validating resource [%s]: %s", incoming.FullName(), err) - return err + rs.logger.Warn("suppressed error compiling spec for resource [%s]: %s", incoming.FullName(), err) + compiledSpec = incoming.Spec() } + incoming.UpdateSpec(compiledSpec) - err = incoming.UpdateURN(urn) + err = rs.validateAndGenerateURN(incoming) if err != nil { - rs.logger.Error("error updating urn of resource [%s]: %s", incoming.FullName(), err) return err } @@ -249,6 +254,65 @@ func (rs ResourceService) Upsert(ctx context.Context, incoming *resource.Resourc return nil } +func (rs ResourceService) validateAndGenerateURN(incoming *resource.Resource) error { + if err := rs.mgr.Validate(incoming); err != nil { + rs.logger.Error("error validating resource [%s]: %s", incoming.FullName(), err) + return err + } + incoming.MarkValidationSuccess() + + urn, err := rs.mgr.GetURN(incoming) + if err != nil { + rs.logger.Error("error validating resource [%s]: %s", incoming.FullName(), err) + return err + } + + err = incoming.UpdateURN(urn) + if err != nil { + rs.logger.Error("error updating urn of resource [%s]: %s", incoming.FullName(), err) + return err + } + + return nil +} + +// helper method to compile resource specs containing template variables +// which is only supported on resource specs with version: 2 and above +func (rs ResourceService) compileSpec(ctx context.Context, res *resource.Resource) (map[string]any, error) { + if res.Version() < resource.ResourceSpecV2 { + return res.Spec(), nil + } + + sourceSpec := res.Spec() + tnnt, err := rs.tenantDetailsGetter.GetDetails(ctx, res.Tenant()) + if err != nil { + return nil, err + } + + tmplCtx := compiler.PrepareContext( + compiler.From(tnnt.GetVariables()).WithName("proj").WithKeyPrefix(projectConfigPrefix), + ) + + // instead of having to traverse through each field in spec, and compile them individually, + // we should be able to compile the entire spec at once by treating the spec as a string + specBytes, err := json.Marshal(sourceSpec) + if err != nil { + return nil, err + } + specStr := string(specBytes) + compiledSpecStr, err := rs.compileEngine.CompileString(specStr, tmplCtx) + if err != nil { + return nil, err + } + var compiledSpec map[string]any + err = json.Unmarshal([]byte(compiledSpecStr), &compiledSpec) + if err != nil { + return nil, err + } + + return compiledSpec, nil +} + func (rs ResourceService) Delete(ctx context.Context, req *resource.DeleteRequest) (*resource.DeleteResponse, error) { existing, err := rs.Get(ctx, req.Tenant, req.Datastore, req.FullName) if err != nil { diff --git a/core/resource/service/resource_service_test.go b/core/resource/service/resource_service_test.go index 54e3040ef3..467a647818 100644 --- a/core/resource/service/resource_service_test.go +++ b/core/resource/service/resource_service_test.go @@ -14,6 +14,8 @@ import ( "github.com/goto/optimus/core/resource" "github.com/goto/optimus/core/resource/service" "github.com/goto/optimus/core/tenant" + "github.com/goto/optimus/ext/store/bigquery" + "github.com/goto/optimus/internal/compiler" oErrors "github.com/goto/optimus/internal/errors" "github.com/goto/optimus/internal/writer" ) @@ -22,8 +24,26 @@ func TestResourceService(t *testing.T) { ctx := context.Background() logger := log.NewLogrus() logWriter := writer.NewLogWriter(logger) - tnnt, tenantErr := tenant.NewTenant("project_test", "namespace_tes") + + project, _ := tenant.NewProject("project_test", + map[string]string{ + tenant.ProjectSchedulerHost: "host", + tenant.ProjectStoragePathKey: "gs://location", + }, map[string]string{ + "bucket": "gs://some_folder-2", + "DATA_PROJECT": "project_test_proj", + }) + namespace, _ := tenant.NewNamespace("namespace_tes", project.Name(), + map[string]string{}, + map[string]string{ + "bucket": "gs://ns_bucket", + "DATA_PROJECT": "project_test_ns", + }) + tnnt, tenantErr := tenant.NewTenant(project.Name().String(), namespace.Name().String()) assert.NoError(t, tenantErr) + tnntDetails, tenantErr := tenant.NewTenantDetails(project, namespace, nil) + assert.NoError(t, tenantErr) + meta := &resource.Metadata{ Version: 1, Description: "test metadata", @@ -32,20 +52,38 @@ func TestResourceService(t *testing.T) { spec := map[string]any{ "description": "test spec", } - datasetURN, err := resource.ParseURN("bigquery://project:dataset") assert.NoError(t, err) tableURN, err := resource.ParseURN("bigquery://project:dataset.table") assert.NoError(t, err) + metaV2 := &resource.Metadata{ + Version: 2, + Description: "test metadata", + Labels: map[string]string{"owner": "optimus"}, + } + + specV2 := map[string]any{ + "project": "{{.GLOBAL__DATA_PROJECT}}", + "dataset": "data-dataset", + "name": "data-table", + } + + tableURNV2, err := resource.ParseURN("bigquery://project_test_ns:data-dataset.data-table") + assert.NoError(t, err) + t.Run("Create", func(t *testing.T) { t.Run("returns error if resource is invalid", func(t *testing.T) { - invalid := &resource.Resource{} + invalid, err := resource.NewResource("", "table", resource.Bigquery, tnnt, meta, spec) + assert.NoError(t, err) + + tenantDetailsGetter := new(mockTenantDetailsGetter) + specEngine := compiler.NewEngine() mgr := NewResourceManager(t) mgr.On("Validate", invalid).Return(errors.New("validation error")) - rscService := service.NewResourceService(logger, nil, nil, mgr, nil, nil, nil) + rscService := service.NewResourceService(logger, nil, nil, mgr, nil, nil, nil, tenantDetailsGetter, specEngine) actualError := rscService.Create(ctx, invalid) assert.Error(t, actualError) @@ -55,11 +93,14 @@ func TestResourceService(t *testing.T) { incoming, err := resource.NewResource("project.dataset", "dataset", resource.Bigquery, tnnt, meta, spec) assert.NoError(t, err) + tenantDetailsGetter := new(mockTenantDetailsGetter) + specEngine := compiler.NewEngine() + mgr := NewResourceManager(t) mgr.On("Validate", incoming).Return(nil) mgr.On("GetURN", incoming).Return(resource.ZeroURN(), errors.New("urn error")) - rscService := service.NewResourceService(logger, nil, nil, mgr, nil, nil, nil) + rscService := service.NewResourceService(logger, nil, nil, mgr, nil, nil, nil, tenantDetailsGetter, specEngine) actualError := rscService.Create(ctx, incoming) assert.Error(t, actualError) @@ -75,7 +116,7 @@ func TestResourceService(t *testing.T) { mgr.On("Validate", incoming).Return(nil) mgr.On("GetURN", incoming).Return(tableURN, nil) - rscService := service.NewResourceService(logger, nil, nil, mgr, nil, nil, nil) + rscService := service.NewResourceService(logger, nil, nil, mgr, nil, nil, nil, nil, nil) actualError := rscService.Create(ctx, incoming) assert.Error(t, actualError) @@ -95,7 +136,7 @@ func TestResourceService(t *testing.T) { repo := newResourceRepository(t) repo.On("ReadByFullName", ctx, tnnt, resource.Bigquery, incoming.FullName(), onlyActive).Return(nil, errors.New("unknown error")) - rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil) + rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil, nil, nil) actualError := rscService.Create(ctx, incoming) assert.ErrorContains(t, actualError, "unknown error") @@ -114,7 +155,7 @@ func TestResourceService(t *testing.T) { mgr.On("Validate", incoming).Return(nil) mgr.On("GetURN", incoming).Return(tableURN, nil) - rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil) + rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil, nil, nil) actualError := rscService.Create(ctx, incoming) assert.ErrorContains(t, actualError, "error creating resource") @@ -144,7 +185,7 @@ func TestResourceService(t *testing.T) { repo := newResourceRepository(t) repo.On("ReadByFullName", ctx, tnnt, resource.Bigquery, incoming.FullName(), onlyActive).Return(existingWithStatus, nil) - rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil) + rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil, nil, nil) err = rscService.Create(ctx, incoming) assert.NoError(t, err) @@ -175,7 +216,7 @@ func TestResourceService(t *testing.T) { existingWithStatus := resource.FromExisting(existing, resource.ReplaceStatus(status)) repo := newResourceRepository(t) - rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil) + rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil, nil, nil) repo.On("ReadByFullName", ctx, tnnt, resource.Bigquery, incoming.FullName(), onlyActive).Return(existingWithStatus, nil) @@ -200,7 +241,7 @@ func TestResourceService(t *testing.T) { mgr.On("Validate", incoming).Return(nil) mgr.On("GetURN", incoming).Return(tableURN, nil) - rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil) + rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil, nil, nil) actualError := rscService.Create(ctx, incoming) assert.ErrorContains(t, actualError, "error updating resource") @@ -220,7 +261,7 @@ func TestResourceService(t *testing.T) { mgr.On("GetURN", incoming).Return(tableURN, nil) mgr.On("CreateResource", ctx, incoming).Return(errors.New("error creating to store")) - rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil) + rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil, nil, nil) actualError := rscService.Create(ctx, incoming) assert.ErrorContains(t, actualError, "error creating to store") @@ -243,7 +284,53 @@ func TestResourceService(t *testing.T) { alertManager := new(mockAlertManager) alertManager.On("SendResourceEvent", mock.Anything) - rscService := service.NewResourceService(logger, repo, nil, mgr, eventHandler, nil, nil) + rscService := service.NewResourceService(logger, repo, nil, mgr, eventHandler, nil, nil, nil, nil) + + actualError := rscService.Create(ctx, incoming) + assert.NoError(t, actualError) + }) + + t.Run("v2 spec return error if tenant details return error and validation fails", func(t *testing.T) { + incoming, err := resource.NewResource("data-table", bigquery.KindTable, resource.Bigquery, tnnt, metaV2, specV2) + assert.NoError(t, err) + + repo := newResourceRepository(t) + + tenantDetailsGetter := new(mockTenantDetailsGetter) + tenantDetailsGetter.On("GetDetails", ctx, tnnt).Return(nil, errors.New("tenant details not found")) + engine := compiler.NewEngine() + + mgr := NewResourceManager(t) + mgr.On("Validate", incoming).Return(errors.New("error validation")) + + rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil, tenantDetailsGetter, engine) + + actualError := rscService.Create(ctx, incoming) + assert.Error(t, actualError) + }) + + t.Run("v2 spec return nil if resource is successfully saved", func(t *testing.T) { + incoming, err := resource.NewResource("data-table", bigquery.KindTable, resource.Bigquery, tnnt, metaV2, specV2) + assert.NoError(t, err) + + tenantDetailsGetter := new(mockTenantDetailsGetter) + tenantDetailsGetter.On("GetDetails", ctx, tnnt).Return(tnntDetails, nil) + engine := compiler.NewEngine() + + repo := newResourceRepository(t) + repo.On("ReadByFullName", ctx, tnnt, resource.Bigquery, incoming.FullName(), onlyActive).Return(nil, oErrors.NotFound(resource.EntityResource, "resource not found")) + repo.On("Create", ctx, incoming).Return(nil) + + mgr := NewResourceManager(t) + mgr.On("Validate", incoming).Return(nil) + mgr.On("GetURN", incoming).Return(tableURNV2, nil) + mgr.On("CreateResource", ctx, incoming).Return(nil) + + eventHandler := newEventHandler(t) + alertManager := new(mockAlertManager) + alertManager.On("SendResourceEvent", mock.Anything) + + rscService := service.NewResourceService(logger, repo, nil, mgr, eventHandler, nil, alertManager, tenantDetailsGetter, engine) actualError := rscService.Create(ctx, incoming) assert.NoError(t, actualError) @@ -258,7 +345,7 @@ func TestResourceService(t *testing.T) { mgr := NewResourceManager(t) mgr.On("Validate", invalidResource).Return(errors.New("validation error")) - rscService := service.NewResourceService(logger, nil, nil, mgr, nil, nil, nil) + rscService := service.NewResourceService(logger, nil, nil, mgr, nil, nil, nil, nil, nil) actualError := rscService.Update(ctx, invalidResource, logWriter) assert.Error(t, actualError) @@ -271,7 +358,7 @@ func TestResourceService(t *testing.T) { mgr.On("Validate", incoming).Return(nil) mgr.On("GetURN", incoming).Return(resource.ZeroURN(), errors.New("urn error")) - rscService := service.NewResourceService(logger, nil, nil, mgr, nil, nil, nil) + rscService := service.NewResourceService(logger, nil, nil, mgr, nil, nil, nil, nil, nil) actualError := rscService.Update(ctx, incoming, logWriter) assert.Error(t, actualError) @@ -289,7 +376,7 @@ func TestResourceService(t *testing.T) { mgr.On("Validate", incoming).Return(nil) mgr.On("GetURN", incoming).Return(urn, nil) - rscService := service.NewResourceService(logger, nil, nil, mgr, nil, nil, nil) + rscService := service.NewResourceService(logger, nil, nil, mgr, nil, nil, nil, nil, nil) actualError := rscService.Update(ctx, incoming, logWriter) assert.Error(t, actualError) @@ -308,7 +395,7 @@ func TestResourceService(t *testing.T) { mgr.On("Validate", resourceToUpdate).Return(nil) mgr.On("GetURN", resourceToUpdate).Return(datasetURN, nil) - rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil) + rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil, nil, nil) actualError := rscService.Update(ctx, resourceToUpdate, logWriter) assert.ErrorContains(t, actualError, "unknown error") @@ -323,7 +410,7 @@ func TestResourceService(t *testing.T) { mgr.On("GetURN", mock.Anything).Return(datasetURN, nil) repo := newResourceRepository(t) - rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil) + rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil, nil, nil) unacceptableStatuses := []resource.Status{ resource.StatusUnknown, @@ -363,7 +450,7 @@ func TestResourceService(t *testing.T) { repo.On("ReadByFullName", ctx, tnnt, resource.Bigquery, fullName, onlyActive).Return(existingResource, nil) repo.On("Update", ctx, mock.Anything).Return(errors.New("unknown error")) - rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil) + rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil, nil, nil) actualError := rscService.Update(ctx, resourceToUpdate, logWriter) assert.ErrorContains(t, actualError, "unknown error") @@ -386,7 +473,7 @@ func TestResourceService(t *testing.T) { mgr.On("GetURN", mock.Anything).Return(datasetURN, nil) mgr.On("UpdateResource", ctx, mock.Anything).Return(errors.New("unknown error")) - rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil) + rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil, nil, nil) actualError := rscService.Update(ctx, resourceToUpdate, logWriter) assert.ErrorContains(t, actualError, "unknown error") @@ -424,7 +511,7 @@ func TestResourceService(t *testing.T) { refresher := new(mockDownstreamRefresher) refresher.On("RefreshResourceDownstream", ctx, mock.Anything, logWriter).Return(errors.New("unknown error")) - rscService := service.NewResourceService(logger, repo, refresher, mgr, eventHandler, nil, alertManager) + rscService := service.NewResourceService(logger, repo, refresher, mgr, eventHandler, nil, alertManager, nil, nil) actualError := rscService.Update(ctx, resourceToUpdate, logWriter) assert.ErrorContains(t, actualError, "unknown error") @@ -456,11 +543,61 @@ func TestResourceService(t *testing.T) { refresher := new(mockDownstreamRefresher) refresher.On("RefreshResourceDownstream", ctx, mock.Anything, logWriter).Return(nil) - rscService := service.NewResourceService(logger, repo, refresher, mgr, eventHandler, nil, nil) + rscService := service.NewResourceService(logger, repo, refresher, mgr, eventHandler, nil, nil, nil, nil) actualError := rscService.Update(ctx, resourceToUpdate, logWriter) assert.NoError(t, actualError) }) + + t.Run("v2 spec return error if tenant details return error and validation fails", func(t *testing.T) { + incoming, err := resource.NewResource("data-table", bigquery.KindTable, resource.Bigquery, tnnt, metaV2, specV2) + assert.NoError(t, err) + + repo := newResourceRepository(t) + + tenantDetailsGetter := new(mockTenantDetailsGetter) + tenantDetailsGetter.On("GetDetails", ctx, tnnt).Return(nil, errors.New("tenant details not found")) + engine := compiler.NewEngine() + + mgr := NewResourceManager(t) + mgr.On("Validate", incoming).Return(errors.New("error validation")) + + rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil, tenantDetailsGetter, engine) + + actualError := rscService.Update(ctx, incoming, logWriter) + assert.Error(t, actualError) + }) + + t.Run("v2 spec return nil if resource is successfully updated", func(t *testing.T) { + tableName := "data-table" + incoming, err := resource.NewResource(tableName, bigquery.KindTable, resource.Bigquery, tnnt, metaV2, specV2) + assert.NoError(t, err) + existingResource, err := resource.NewResource(tableName, bigquery.KindTable, resource.Bigquery, tnnt, metaV2, specV2) + assert.NoError(t, err) + existingResource = resource.FromExisting(existingResource, resource.ReplaceStatus(resource.StatusToUpdate)) + + tenantDetailsGetter := new(mockTenantDetailsGetter) + tenantDetailsGetter.On("GetDetails", ctx, tnnt).Return(tnntDetails, nil) + engine := compiler.NewEngine() + + repo := newResourceRepository(t) + repo.On("ReadByFullName", ctx, tnnt, resource.Bigquery, incoming.FullName(), onlyActive).Return(existingResource, nil) + repo.On("Update", ctx, mock.Anything).Return(nil) + + mgr := NewResourceManager(t) + mgr.On("Validate", incoming).Return(nil) + mgr.On("GetURN", incoming).Return(tableURNV2, nil) + mgr.On("UpdateResource", ctx, incoming).Return(nil) + + eventHandler := newEventHandler(t) + alertManager := new(mockAlertManager) + alertManager.On("SendResourceEvent", mock.Anything) + + rscService := service.NewResourceService(logger, repo, nil, mgr, eventHandler, nil, alertManager, tenantDetailsGetter, engine) + + actualError := rscService.Update(ctx, incoming, logWriter) + assert.NoError(t, actualError) + }) }) t.Run("Upsert", func(t *testing.T) { @@ -470,7 +607,7 @@ func TestResourceService(t *testing.T) { mgr := NewResourceManager(t) mgr.On("Validate", invalid).Return(errors.New("validation error")) - rscService := service.NewResourceService(logger, nil, nil, mgr, nil, nil, nil) + rscService := service.NewResourceService(logger, nil, nil, mgr, nil, nil, nil, nil, nil) actualError := rscService.Upsert(ctx, invalid, logWriter) assert.Error(t, actualError) @@ -485,7 +622,7 @@ func TestResourceService(t *testing.T) { mgr.On("Validate", incoming).Return(nil) mgr.On("GetURN", incoming).Return(resource.ZeroURN(), errors.New("urn error")) - rscService := service.NewResourceService(logger, nil, nil, mgr, nil, nil, nil) + rscService := service.NewResourceService(logger, nil, nil, mgr, nil, nil, nil, nil, nil) actualError := rscService.Upsert(ctx, incoming, logWriter) assert.Error(t, actualError) @@ -502,7 +639,7 @@ func TestResourceService(t *testing.T) { mgr.On("Validate", incoming).Return(nil) mgr.On("GetURN", incoming).Return(tableURN, nil) - rscService := service.NewResourceService(logger, nil, nil, mgr, nil, nil, nil) + rscService := service.NewResourceService(logger, nil, nil, mgr, nil, nil, nil, nil, nil) actualError := rscService.Upsert(ctx, incoming, logWriter) assert.Error(t, actualError) @@ -522,7 +659,7 @@ func TestResourceService(t *testing.T) { repo := newResourceRepository(t) repo.On("ReadByFullName", ctx, tnnt, resource.Bigquery, incoming.FullName(), onlyActive).Return(nil, errors.New("unknown error")) - rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil) + rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil, nil, nil) actualError := rscService.Upsert(ctx, incoming, logWriter) assert.ErrorContains(t, actualError, "unknown error") @@ -541,7 +678,7 @@ func TestResourceService(t *testing.T) { mgr.On("Validate", incoming).Return(nil) mgr.On("GetURN", incoming).Return(tableURN, nil) - rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil) + rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil, nil, nil) actualError := rscService.Upsert(ctx, incoming, logWriter) assert.ErrorContains(t, actualError, "error creating resource") @@ -560,7 +697,7 @@ func TestResourceService(t *testing.T) { mgr.On("GetURN", incoming).Return(tableURN, nil) mgr.On("CreateResource", ctx, incoming).Return(errors.New("error creating to store")) - rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil) + rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil, nil, nil) actualError := rscService.Upsert(ctx, incoming, logWriter) assert.ErrorContains(t, actualError, "error creating to store") @@ -581,7 +718,7 @@ func TestResourceService(t *testing.T) { eventHandler := newEventHandler(t) - rscService := service.NewResourceService(logger, repo, nil, mgr, eventHandler, nil, nil) + rscService := service.NewResourceService(logger, repo, nil, mgr, eventHandler, nil, nil, nil, nil) actualError := rscService.Upsert(ctx, incoming, logWriter) assert.NoError(t, actualError) @@ -605,7 +742,7 @@ func TestResourceService(t *testing.T) { repo.On("ReadByFullName", ctx, tnnt, resource.Bigquery, fullName, onlyActive).Return(existingResource, nil) repo.On("Update", ctx, mock.Anything).Return(errors.New("unknown error")) - rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil) + rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil, nil, nil) actualError := rscService.Upsert(ctx, resourceToUpdate, logWriter) assert.ErrorContains(t, actualError, "unknown error") @@ -628,7 +765,7 @@ func TestResourceService(t *testing.T) { mgr.On("GetURN", mock.Anything).Return(datasetURN, nil) mgr.On("UpdateResource", ctx, mock.Anything).Return(errors.New("unknown error")) - rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil) + rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil, nil, nil) actualError := rscService.Upsert(ctx, resourceToUpdate, logWriter) assert.ErrorContains(t, actualError, "unknown error") @@ -667,7 +804,7 @@ func TestResourceService(t *testing.T) { refresher := new(mockDownstreamRefresher) refresher.On("RefreshResourceDownstream", ctx, mock.Anything, logWriter).Return(errors.New("unknown error")) - rscService := service.NewResourceService(logger, repo, refresher, mgr, eventHandler, nil, alertManager) + rscService := service.NewResourceService(logger, repo, refresher, mgr, eventHandler, nil, alertManager, nil, nil) actualError := rscService.Upsert(ctx, resourceToUpdate, logWriter) assert.ErrorContains(t, actualError, "unknown error") @@ -697,18 +834,96 @@ func TestResourceService(t *testing.T) { refresher := new(mockDownstreamRefresher) refresher.On("RefreshResourceDownstream", ctx, mock.Anything, logWriter).Return(nil) - rscService := service.NewResourceService(logger, repo, refresher, mgr, eventHandler, nil, nil) + rscService := service.NewResourceService(logger, repo, refresher, mgr, eventHandler, nil, nil, nil, nil) actualError := rscService.Upsert(ctx, resourceToUpdate, logWriter) assert.NoError(t, actualError) }) }) + + t.Run("v2 spec return error if tenant details return error and validation fails", func(t *testing.T) { + incoming, err := resource.NewResource("data-table", bigquery.KindTable, resource.Bigquery, tnnt, metaV2, specV2) + assert.NoError(t, err) + + repo := newResourceRepository(t) + + tenantDetailsGetter := new(mockTenantDetailsGetter) + tenantDetailsGetter.On("GetDetails", ctx, tnnt).Return(nil, errors.New("tenant details not found")) + engine := compiler.NewEngine() + + mgr := NewResourceManager(t) + mgr.On("Validate", incoming).Return(errors.New("error validation")) + + rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil, tenantDetailsGetter, engine) + + actualError := rscService.Upsert(ctx, incoming, logWriter) + assert.Error(t, actualError) + }) + + t.Run("v2 spec return nil if resource is successfully created", func(t *testing.T) { + tableName := "data-table" + incoming, err := resource.NewResource(tableName, bigquery.KindTable, resource.Bigquery, tnnt, metaV2, specV2) + assert.NoError(t, err) + + tenantDetailsGetter := new(mockTenantDetailsGetter) + tenantDetailsGetter.On("GetDetails", ctx, tnnt).Return(tnntDetails, nil) + engine := compiler.NewEngine() + + repo := newResourceRepository(t) + repo.On("ReadByFullName", ctx, tnnt, resource.Bigquery, incoming.FullName(), onlyActive).Return(nil, oErrors.NotFound(resource.EntityResource, "resource not found")) + repo.On("Create", ctx, incoming).Return(nil) + + mgr := NewResourceManager(t) + mgr.On("Validate", incoming).Return(nil) + mgr.On("GetURN", incoming).Return(tableURNV2, nil) + mgr.On("CreateResource", ctx, incoming).Return(nil) + + eventHandler := newEventHandler(t) + alertManager := new(mockAlertManager) + alertManager.On("SendResourceEvent", mock.Anything) + + rscService := service.NewResourceService(logger, repo, nil, mgr, eventHandler, nil, alertManager, tenantDetailsGetter, engine) + + actualError := rscService.Upsert(ctx, incoming, logWriter) + assert.NoError(t, actualError) + }) + + t.Run("v2 spec return nil if resource is successfully updated", func(t *testing.T) { + tableName := "data-table" + incoming, err := resource.NewResource(tableName, bigquery.KindTable, resource.Bigquery, tnnt, metaV2, specV2) + assert.NoError(t, err) + existingResource, err := resource.NewResource(tableName, bigquery.KindTable, resource.Bigquery, tnnt, metaV2, specV2) + assert.NoError(t, err) + existingResource = resource.FromExisting(existingResource, resource.ReplaceStatus(resource.StatusToUpdate)) + + tenantDetailsGetter := new(mockTenantDetailsGetter) + tenantDetailsGetter.On("GetDetails", ctx, tnnt).Return(tnntDetails, nil) + engine := compiler.NewEngine() + + repo := newResourceRepository(t) + repo.On("ReadByFullName", ctx, tnnt, resource.Bigquery, incoming.FullName(), onlyActive).Return(existingResource, nil) + repo.On("Update", ctx, incoming).Return(nil) + + mgr := NewResourceManager(t) + mgr.On("Validate", incoming).Return(nil) + mgr.On("GetURN", incoming).Return(tableURNV2, nil) + mgr.On("UpdateResource", ctx, incoming).Return(nil) + + eventHandler := newEventHandler(t) + alertManager := new(mockAlertManager) + alertManager.On("SendResourceEvent", mock.Anything) + + rscService := service.NewResourceService(logger, repo, nil, mgr, eventHandler, nil, alertManager, tenantDetailsGetter, engine) + + actualError := rscService.Upsert(ctx, incoming, logWriter) + assert.NoError(t, actualError) + }) }) t.Run("Get", func(t *testing.T) { onlyActive := true t.Run("returns nil and error if resource name is empty", func(t *testing.T) { - rscService := service.NewResourceService(logger, nil, nil, nil, nil, nil, nil) + rscService := service.NewResourceService(logger, nil, nil, nil, nil, nil, nil, nil, nil) store := resource.Bigquery actualResource, actualError := rscService.Get(ctx, tnnt, store, "") @@ -723,7 +938,7 @@ func TestResourceService(t *testing.T) { refresher := new(mockDownstreamRefresher) - rscService := service.NewResourceService(logger, repo, refresher, nil, nil, nil, nil) + rscService := service.NewResourceService(logger, repo, refresher, nil, nil, nil, nil, nil, nil) actualResource, actualError := rscService.Get(ctx, tnnt, resource.Bigquery, fullName) assert.Nil(t, actualResource) @@ -740,7 +955,7 @@ func TestResourceService(t *testing.T) { refresher := new(mockDownstreamRefresher) - rscService := service.NewResourceService(logger, repo, refresher, nil, nil, nil, nil) + rscService := service.NewResourceService(logger, repo, refresher, nil, nil, nil, nil, nil, nil) actualResource, actualError := rscService.Get(ctx, tnnt, resource.Bigquery, fullName) assert.EqualValues(t, existingResource, actualResource) @@ -756,7 +971,7 @@ func TestResourceService(t *testing.T) { refresher := new(mockDownstreamRefresher) - rscService := service.NewResourceService(logger, repo, refresher, nil, nil, nil, nil) + rscService := service.NewResourceService(logger, repo, refresher, nil, nil, nil, nil, nil, nil) actualResources, actualError := rscService.GetAll(ctx, tnnt, resource.Bigquery) assert.Nil(t, actualResources) @@ -772,7 +987,7 @@ func TestResourceService(t *testing.T) { refresher := new(mockDownstreamRefresher) - rscService := service.NewResourceService(logger, repo, refresher, nil, nil, nil, nil) + rscService := service.NewResourceService(logger, repo, refresher, nil, nil, nil, nil, nil, nil) actualResources, actualError := rscService.GetAll(ctx, tnnt, resource.Bigquery) assert.EqualValues(t, []*resource.Resource{existingResource}, actualResources) @@ -801,7 +1016,7 @@ func TestResourceService(t *testing.T) { mgr := NewResourceManager(t) mgr.On("Validate", invalidResourceToUpdate).Return(errors.New("error validating")) - rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil) + rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil, nil, nil) actualError := rscService.Deploy(ctx, tnnt, resource.Bigquery, resourcesToUpdate, logWriter) assert.Error(t, actualError) @@ -819,7 +1034,7 @@ func TestResourceService(t *testing.T) { repo := newResourceRepository(t) repo.On("ReadAll", ctx, tnnt, resource.Bigquery, onlyActive).Return([]*resource.Resource{}, nil) - rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil) + rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil, nil, nil) actualError := rscService.Deploy(ctx, tnnt, resource.Bigquery, []*resource.Resource{incoming}, logWriter) assert.Error(t, actualError) @@ -842,7 +1057,7 @@ func TestResourceService(t *testing.T) { repo := newResourceRepository(t) repo.On("ReadAll", ctx, tnnt, resource.Bigquery, onlyActive).Return([]*resource.Resource{}, nil) - rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil) + rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil, nil, nil) actualError := rscService.Deploy(ctx, tnnt, resource.Bigquery, []*resource.Resource{incoming}, logWriter) assert.Error(t, actualError) @@ -864,7 +1079,7 @@ func TestResourceService(t *testing.T) { mgr.On("Validate", incomingResourceToUpdate).Return(nil) mgr.On("GetURN", incomingResourceToUpdate).Return(urn, nil) - rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil) + rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil, nil, nil) actualError := rscService.Deploy(ctx, tnnt, resource.Bigquery, resourcesToUpdate, logWriter) assert.ErrorContains(t, actualError, "error while read all") @@ -885,7 +1100,7 @@ func TestResourceService(t *testing.T) { mgr.On("Validate", mock.Anything).Return(nil) mgr.On("GetURN", mock.Anything).Return(urn, nil) - rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil) + rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil, nil, nil) actualError := rscService.Deploy(ctx, tnnt, resource.Bigquery, []*resource.Resource{incomingResourceToUpdate}, logWriter) assert.NoError(t, actualError) @@ -911,7 +1126,7 @@ func TestResourceService(t *testing.T) { alertManager := new(mockAlertManager) alertManager.On("SendResourceEvent", mock.Anything) - rscService := service.NewResourceService(logger, repo, nil, mgr, eventHandler, nil, alertManager) + rscService := service.NewResourceService(logger, repo, nil, mgr, eventHandler, nil, alertManager, nil, nil) actualError := rscService.Deploy(ctx, tnnt, resource.Bigquery, []*resource.Resource{incomingResourceToUpdate}, logWriter) @@ -942,7 +1157,7 @@ func TestResourceService(t *testing.T) { eventHandler := newEventHandler(t) alertManager := new(mockAlertManager) alertManager.On("SendResourceEvent", mock.Anything) - rscService := service.NewResourceService(logger, repo, nil, mgr, eventHandler, nil, alertManager) + rscService := service.NewResourceService(logger, repo, nil, mgr, eventHandler, nil, alertManager, nil, nil) actualError := rscService.Deploy(ctx, tnnt, resource.Bigquery, []*resource.Resource{incomingResourceToUpdate}, logWriter) @@ -974,7 +1189,7 @@ func TestResourceService(t *testing.T) { eventHandler := newEventHandler(t) alertManager := new(mockAlertManager) alertManager.On("SendResourceEvent", mock.Anything) - rscService := service.NewResourceService(logger, repo, nil, mgr, eventHandler, nil, alertManager) + rscService := service.NewResourceService(logger, repo, nil, mgr, eventHandler, nil, alertManager, nil, nil) actualError := rscService.Deploy(ctx, tnnt, resource.Bigquery, []*resource.Resource{incomingResourceToUpdate}, logWriter) assert.ErrorContains(t, actualError, "unknown error") @@ -1013,7 +1228,7 @@ func TestResourceService(t *testing.T) { eventHandler := newEventHandler(t) alertManager := new(mockAlertManager) alertManager.On("SendResourceEvent", mock.Anything) - rscService := service.NewResourceService(logger, repo, nil, mgr, eventHandler, nil, alertManager) + rscService := service.NewResourceService(logger, repo, nil, mgr, eventHandler, nil, alertManager, nil, nil) actualError := rscService.Deploy(ctx, tnnt, resource.Bigquery, []*resource.Resource{incomingResourceToUpdate}, logWriter) assert.ErrorContains(t, actualError, "unknown error") @@ -1066,7 +1281,7 @@ func TestResourceService(t *testing.T) { refresher := new(mockDownstreamRefresher) refresher.On("RefreshResourceDownstream", ctx, mock.Anything, logWriter).Return(errors.New("unknown error")) - rscService := service.NewResourceService(logger, repo, refresher, mgr, eventHandler, nil, alertManager) + rscService := service.NewResourceService(logger, repo, refresher, mgr, eventHandler, nil, alertManager, nil, nil) incomings := []*resource.Resource{incomingToCreate, incomingToSkip, incomingToUpdate, incomingToCreateExisting} actualError := rscService.Deploy(ctx, tnnt, resource.Bigquery, incomings, logWriter) @@ -1120,7 +1335,7 @@ func TestResourceService(t *testing.T) { refresher := new(mockDownstreamRefresher) refresher.On("RefreshResourceDownstream", ctx, mock.Anything, logWriter).Return(nil) - rscService := service.NewResourceService(logger, repo, refresher, mgr, eventHandler, nil, alertManager) + rscService := service.NewResourceService(logger, repo, refresher, mgr, eventHandler, nil, alertManager, nil, nil) incomings := []*resource.Resource{incomingToCreate, incomingToSkip, incomingToUpdate, incomingToCreateExisting} actualError := rscService.Deploy(ctx, tnnt, resource.Bigquery, incomings, logWriter) @@ -1184,7 +1399,7 @@ func TestResourceService(t *testing.T) { refresher := new(mockDownstreamRefresher) refresher.On("RefreshResourceDownstream", ctx, mock.Anything, logWriter).Return(nil) - rscService := service.NewResourceService(logger, repo, refresher, mgr, eventHandler, nil, alertManager) + rscService := service.NewResourceService(logger, repo, refresher, mgr, eventHandler, nil, alertManager, nil, nil) incomings := []*resource.Resource{incomingToCreate, incomingToSkip, incomingToUpdate, incomingToCreateExisting, incomingToRecreate} actualError := rscService.Deploy(ctx, tnnt, resource.Bigquery, incomings, logWriter) @@ -1200,7 +1415,7 @@ func TestResourceService(t *testing.T) { mgr := NewResourceManager(t) - rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil) + rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil, nil, nil) resp, actualError := rscService.SyncResources(ctx, tnnt, resource.Bigquery, []string{fullName}) assert.ErrorContains(t, actualError, "unknown error") @@ -1213,7 +1428,7 @@ func TestResourceService(t *testing.T) { mgr := NewResourceManager(t) - rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil) + rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil, nil, nil) response, actualError := rscService.SyncResources(ctx, tnnt, resource.Bigquery, []string{fullName}) assert.Nil(t, actualError) @@ -1233,7 +1448,7 @@ func TestResourceService(t *testing.T) { mgr := NewResourceManager(t) mgr.On("SyncResource", ctx, incoming).Return(errors.New("unable to create")) - rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil) + rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil, nil, nil) response, actualError := rscService.SyncResources(ctx, tnnt, resource.Bigquery, []string{fullName}) assert.Nil(t, actualError) @@ -1253,7 +1468,7 @@ func TestResourceService(t *testing.T) { mgr := NewResourceManager(t) mgr.On("SyncResource", ctx, incoming).Return(nil) - rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil) + rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil, nil, nil) response, actualError := rscService.SyncResources(ctx, tnnt, resource.Bigquery, []string{fullName}) assert.Nil(t, actualError) @@ -1270,7 +1485,7 @@ func TestResourceService(t *testing.T) { var ( mockDepResolver = new(mockDownstreamResolver) repo = newResourceRepository(t) - rscService = service.NewResourceService(logger, repo, nil, nil, nil, mockDepResolver, nil) + rscService = service.NewResourceService(logger, repo, nil, nil, nil, mockDepResolver, nil, nil, nil) req = &resource.DeleteRequest{ Tenant: tnnt, Datastore: resource.Bigquery, @@ -1296,7 +1511,7 @@ func TestResourceService(t *testing.T) { var ( mockDepResolver = new(mockDownstreamResolver) repo = newResourceRepository(t) - rscService = service.NewResourceService(logger, repo, nil, nil, nil, mockDepResolver, nil) + rscService = service.NewResourceService(logger, repo, nil, nil, nil, mockDepResolver, nil, nil, nil) req = &resource.DeleteRequest{ Tenant: tnnt, Datastore: resource.Bigquery, @@ -1328,7 +1543,7 @@ func TestResourceService(t *testing.T) { var ( mockDepResolver = new(mockDownstreamResolver) repo = newResourceRepository(t) - rscService = service.NewResourceService(logger, repo, nil, nil, nil, mockDepResolver, nil) + rscService = service.NewResourceService(logger, repo, nil, nil, nil, mockDepResolver, nil, nil, nil) req = &resource.DeleteRequest{ Tenant: tnnt, Datastore: resource.Bigquery, @@ -1353,7 +1568,7 @@ func TestResourceService(t *testing.T) { var ( mockDepResolver = new(mockDownstreamResolver) repo = newResourceRepository(t) - rscService = service.NewResourceService(logger, repo, nil, nil, nil, mockDepResolver, nil) + rscService = service.NewResourceService(logger, repo, nil, nil, nil, mockDepResolver, nil, nil, nil) req = &resource.DeleteRequest{ Tenant: tnnt, Datastore: resource.Bigquery, @@ -1378,7 +1593,7 @@ func TestResourceService(t *testing.T) { var ( mockDepResolver = new(mockDownstreamResolver) repo = newResourceRepository(t) - rscService = service.NewResourceService(logger, repo, nil, nil, nil, mockDepResolver, nil) + rscService = service.NewResourceService(logger, repo, nil, nil, nil, mockDepResolver, nil, nil, nil) req = &resource.DeleteRequest{ Tenant: tnnt, Datastore: resource.Bigquery, @@ -1402,7 +1617,7 @@ func TestResourceService(t *testing.T) { var ( mockDepResolver = new(mockDownstreamResolver) repo = newResourceRepository(t) - rscService = service.NewResourceService(logger, repo, nil, nil, nil, mockDepResolver, nil) + rscService = service.NewResourceService(logger, repo, nil, nil, nil, mockDepResolver, nil, nil, nil) req = &resource.DeleteRequest{ Tenant: tnnt, Datastore: resource.Bigquery, @@ -1426,7 +1641,7 @@ func TestResourceService(t *testing.T) { repo := newResourceRepository(t) logger := log.NewLogrus() - rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil) + rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil, nil, nil) _, err := rscService.GetByURN(ctx, tnnt, resource.ZeroURN()) assert.Error(t, err) @@ -1443,7 +1658,7 @@ func TestResourceService(t *testing.T) { repo.On("ReadByURN", ctx, tnnt, urn).Return(nil, errors.New("unknown error")) - rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil) + rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil, nil, nil) _, err = rscService.GetByURN(ctx, tnnt, urn) assert.Error(t, err) @@ -1463,7 +1678,7 @@ func TestResourceService(t *testing.T) { repo.On("ReadByURN", ctx, tnnt, urn).Return(expectedResource, nil) - rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil) + rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil, nil, nil) actualResource, err := rscService.GetByURN(ctx, tnnt, urn) assert.NoError(t, err) @@ -1753,3 +1968,17 @@ type mockAlertManager struct { func (m *mockAlertManager) SendResourceEvent(attr *resource.AlertAttrs) { m.Called(attr) } + +// TenantDetailsGetter is an autogenerated mock type for the TenantDetailsGetter type +type mockTenantDetailsGetter struct { + mock.Mock +} + +// GetDetails provides a mock function with given fields: ctx, jobTenant +func (m *mockTenantDetailsGetter) GetDetails(ctx context.Context, jobTenant tenant.Tenant) (*tenant.WithDetails, error) { + args := m.Called(ctx, jobTenant) + if args.Get(0) != nil { + return args.Get(0).(*tenant.WithDetails), args.Error(1) + } + return nil, args.Error(1) +} diff --git a/core/tenant/tenant.go b/core/tenant/tenant.go index c46abe0d38..bac7a97c48 100644 --- a/core/tenant/tenant.go +++ b/core/tenant/tenant.go @@ -108,7 +108,7 @@ func (w *WithDetails) GetConfigs() map[string]string { func (w *WithDetails) GetVariables() map[string]string { namespaceVars := w.namespace.GetVariables() - return utils.MergeMaps(namespaceVars, w.project.GetVariables()) + return utils.MergeMaps(w.project.GetVariables(), namespaceVars) } func (w *WithDetails) Project() *Project { diff --git a/core/tenant/tenant_test.go b/core/tenant/tenant_test.go index 91550f6fcc..b05589e040 100644 --- a/core/tenant/tenant_test.go +++ b/core/tenant/tenant_test.go @@ -129,7 +129,7 @@ func TestAggregateRootTenant(t *testing.T) { assert.NoError(t, err) tenantVariables := details.GetVariables() - assert.EqualValues(t, project.GetVariables(), tenantVariables) + assert.Len(t, tenantVariables, 1) }) t.Run("returns a single tenant variable", func(t *testing.T) { details, err := tenant.NewTenantDetails(project, namespace, nil) diff --git a/server/optimus.go b/server/optimus.go index 9712d42ed4..54ed2072cb 100644 --- a/server/optimus.go +++ b/server/optimus.go @@ -358,7 +358,7 @@ func (s *OptimusServer) setupHandlers() error { resourceRepository := resource.NewRepository(s.dbPool) backupRepository := resource.NewBackupRepository(s.dbPool) resourceManager := rService.NewResourceManager(resourceRepository, s.logger) - secondaryResourceService := rService.NewResourceService(s.logger, resourceRepository, nil, resourceManager, s.eventHandler, nil, alertsHandler) // note: job service can be nil + secondaryResourceService := rService.NewResourceService(s.logger, resourceRepository, nil, resourceManager, s.eventHandler, nil, alertsHandler, tenantService, newEngine) // note: job service can be nil // Job Bounded Context Setup jJobRepo := jRepo.NewJobRepository(s.dbPool) @@ -375,7 +375,7 @@ func (s *OptimusServer) setupHandlers() error { jchangeLogService := jService.NewChangeLogService(jJobRepo) // Resource Bounded Context - primaryResourceService := rService.NewResourceService(s.logger, resourceRepository, jJobService, resourceManager, s.eventHandler, jJobService, alertsHandler) + primaryResourceService := rService.NewResourceService(s.logger, resourceRepository, jJobService, resourceManager, s.eventHandler, jJobService, alertsHandler, tenantService, newEngine) backupService := rService.NewBackupService(backupRepository, resourceRepository, resourceManager, s.logger) resourceChangeLogService := rService.NewChangelogService(s.logger, resourceRepository) From 9ea085494292dfeff4c68c2ffc31d2a3b0a809c0 Mon Sep 17 00:00:00 2001 From: "Ahmad N. F." Date: Tue, 15 Oct 2024 11:13:04 +0700 Subject: [PATCH 5/8] feat: add compile logic to Deploy API --- core/resource/service/resource_service.go | 7 ++ .../resource/service/resource_service_test.go | 86 +++++++++++++++++++ 2 files changed, 93 insertions(+) diff --git a/core/resource/service/resource_service.go b/core/resource/service/resource_service.go index c45c898b5d..2985e8385a 100644 --- a/core/resource/service/resource_service.go +++ b/core/resource/service/resource_service.go @@ -427,6 +427,13 @@ func (rs ResourceService) ExistInStore(ctx context.Context, tnnt tenant.Tenant, func (rs ResourceService) Deploy(ctx context.Context, tnnt tenant.Tenant, store resource.Store, incomings []*resource.Resource, logWriter writer.LogWriter) error { // nolint:gocritic multiError := errors.NewMultiError("error batch updating resources") for _, r := range incomings { + compiledSpec, err := rs.compileSpec(ctx, r) + if err != nil { + rs.logger.Warn("suppressed error compiling spec for resource [%s]: %s", r.FullName(), err) + compiledSpec = r.Spec() + } + r.UpdateSpec(compiledSpec) + if err := rs.mgr.Validate(r); err != nil { msg := fmt.Sprintf("error validating [%s]: %s", r.FullName(), err) multiError.Append(errors.Wrap(resource.EntityResource, msg, err)) diff --git a/core/resource/service/resource_service_test.go b/core/resource/service/resource_service_test.go index 467a647818..59a0b7d4b0 100644 --- a/core/resource/service/resource_service_test.go +++ b/core/resource/service/resource_service_test.go @@ -1405,6 +1405,92 @@ func TestResourceService(t *testing.T) { actualError := rscService.Deploy(ctx, tnnt, resource.Bigquery, incomings, logWriter) assert.NoError(t, actualError) }) + + t.Run("v2 spec returns nil if no error is encountered with 1 updated & 1 created resource", func(t *testing.T) { + resourceMeta := resource.Metadata{ + Version: resource.ResourceSpecV2, + Labels: map[string]string{"ping": "pong"}, + Description: "incoming resource metadata", + } + existingSpec := map[string]any{ + "project": "project", + "dataset": "dataset", + "name": "table-update", + "a": "a", + } + existingToUpdate, err := resource.NewResource("data-table-update", bigquery.KindTable, resource.Bigquery, tnnt, &resourceMeta, existingSpec) + assert.NoError(t, err) + + newSpec := map[string]any{ + "project": "project", + "dataset": "dataset", + "name": "table-create", + "a": "b", + } + incomingToCreate, err := resource.NewResource("data-table-create", bigquery.KindTable, resource.Bigquery, tnnt, &resourceMeta, newSpec) + assert.NoError(t, err) + + toUpdateSpec := map[string]any{ + "project": "project", + "dataset": "dataset", + "name": "table-update", + "a": "b", + } + incomingToUpdate, err := resource.NewResource("data-table-update", bigquery.KindTable, resource.Bigquery, tnnt, &resourceMeta, toUpdateSpec) + assert.NoError(t, err) + + tenantDetailsGetter := new(mockTenantDetailsGetter) + tenantDetailsGetter.On("GetDetails", ctx, tnnt).Return(tnntDetails, nil) + engine := compiler.NewEngine() + + repo := newResourceRepository(t) + repo.On("ReadAll", ctx, tnnt, resource.Bigquery, onlyActive).Return([]*resource.Resource{existingToUpdate}, nil) + repo.On("Create", ctx, incomingToCreate).Return(nil) + repo.On("Update", ctx, incomingToUpdate).Return(nil) + + urnToUpdate, err := resource.ParseURN("bigquery://project:dataset.table-update") + assert.NoError(t, err) + urnToCreate, err := resource.ParseURN("bigquery://project:dataset.table-create") + assert.NoError(t, err) + + mgr := NewResourceManager(t) + // first iteration: for incomingToCreate + mgr.On("Validate", incomingToCreate).Return(nil).Once() + mgr.On("GetURN", incomingToCreate).Return(urnToCreate, nil).Once() + // second iteration: for incomingToUpdate + mgr.On("Validate", incomingToUpdate).Return(nil).Once() + mgr.On("GetURN", incomingToUpdate).Return(urnToUpdate, nil).Once() + + mgr.On("BatchUpdate", ctx, resource.Bigquery, []*resource.Resource{incomingToCreate, incomingToUpdate}). + Run(func(args mock.Arguments) { + res := args.Get(2).([]*resource.Resource) + for _, r := range res { + if r.Status() == resource.StatusToDelete { + r.MarkDeleted() + continue + } + r.MarkSuccess() + } + }). + Return(nil) + + eventHandler := newEventHandler(t) + alertManager := new(mockAlertManager) + alertManager.On("SendResourceEvent", mock.Anything) + argMatcher := mock.MatchedBy(func(ev moderator.Event) bool { + return ev != nil + }) + eventHandler.On("HandleEvent", argMatcher).Return().Times(2) + + refresher := new(mockDownstreamRefresher) + refresher.On("RefreshResourceDownstream", ctx, mock.Anything, logWriter).Return(nil) + + rscService := service.NewResourceService(logger, repo, refresher, mgr, eventHandler, nil, alertManager, tenantDetailsGetter, engine) + + incomings := []*resource.Resource{incomingToCreate, incomingToUpdate} + actualError := rscService.Deploy(ctx, tnnt, resource.Bigquery, incomings, logWriter) + assert.NoError(t, actualError) + }) }) t.Run("SyncResource", func(t *testing.T) { From 9d9446fea8e2d8a6f6973b60fd3b42bcf63ba8ca Mon Sep 17 00:00:00 2001 From: "Ahmad N. F." Date: Tue, 15 Oct 2024 11:39:07 +0700 Subject: [PATCH 6/8] feat: use variables for templating --- core/job/service/job_service.go | 2 +- core/scheduler/service/executor_input_compiler.go | 2 +- core/tenant/tenant.go | 10 ++++++++-- core/tenant/tenant_test.go | 4 ++-- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/core/job/service/job_service.go b/core/job/service/job_service.go index d5f7817669..4a78f818b2 100644 --- a/core/job/service/job_service.go +++ b/core/job/service/job_service.go @@ -1143,7 +1143,7 @@ func (j *JobService) generateJobs(ctx context.Context, tenantWithDetails *tenant func (j *JobService) compileConfigs(configs job.Config, tnnt *tenant.WithDetails) map[string]string { tmplCtx := compiler.PrepareContext( - compiler.From(tnnt.GetConfigs()).WithName("proj").WithKeyPrefix(projectConfigPrefix), + compiler.From(tnnt.GetVariables()).WithName("proj").WithKeyPrefix(projectConfigPrefix), compiler.From(tnnt.SecretsMap()).WithName("secret"), ) diff --git a/core/scheduler/service/executor_input_compiler.go b/core/scheduler/service/executor_input_compiler.go index a083298433..3d048aebbe 100644 --- a/core/scheduler/service/executor_input_compiler.go +++ b/core/scheduler/service/executor_input_compiler.go @@ -112,7 +112,7 @@ func (i InputCompiler) Compile(ctx context.Context, job *scheduler.JobWithDetail // Prepare template context and compile task config taskContext := compiler.PrepareContext( - compiler.From(tenantDetails.GetConfigs()).WithName(contextProject).WithKeyPrefix(projectConfigPrefix), + compiler.From(tenantDetails.GetVariables()).WithName(contextProject).WithKeyPrefix(projectConfigPrefix), compiler.From(tenantDetails.SecretsMap()).WithName(contextSecret), compiler.From(systemDefinedVars).WithName(contextSystemDefined).AddToContext(), ) diff --git a/core/tenant/tenant.go b/core/tenant/tenant.go index bac7a97c48..5d810ec22c 100644 --- a/core/tenant/tenant.go +++ b/core/tenant/tenant.go @@ -106,9 +106,15 @@ func (w *WithDetails) GetConfigs() map[string]string { return utils.MergeMaps(w.project.GetConfigs(), m1) } +// GetVariables for now will merge tenant variables & tenant configs. +// Since we are moving to use tenant "variables" for job config / job asset compilation & discourage using config for the purpose, +// merging both is a temporary solution to support older behavior. +// Once we have all tenants migrated to use variables, we can remove the tenant config. func (w *WithDetails) GetVariables() map[string]string { - namespaceVars := w.namespace.GetVariables() - return utils.MergeMaps(w.project.GetVariables(), namespaceVars) + tenantConfigs := utils.MergeMaps(w.project.GetConfigs(), w.namespace.GetConfigs()) + tenantVariables := utils.MergeMaps(w.project.GetVariables(), w.namespace.GetVariables()) + + return utils.MergeMaps(tenantConfigs, tenantVariables) } func (w *WithDetails) Project() *Project { diff --git a/core/tenant/tenant_test.go b/core/tenant/tenant_test.go index b05589e040..fa9d954973 100644 --- a/core/tenant/tenant_test.go +++ b/core/tenant/tenant_test.go @@ -124,12 +124,12 @@ func TestAggregateRootTenant(t *testing.T) { assert.Len(t, secMap, 2) assert.Equal(t, "value2", secMap[p2.Name().String()]) }) - t.Run("returns tenant variables", func(t *testing.T) { + t.Run("returns tenant variables & configs", func(t *testing.T) { details, err := tenant.NewTenantDetails(project, namespace, nil) assert.NoError(t, err) tenantVariables := details.GetVariables() - assert.Len(t, tenantVariables, 1) + assert.Len(t, tenantVariables, 5) }) t.Run("returns a single tenant variable", func(t *testing.T) { details, err := tenant.NewTenantDetails(project, namespace, nil) From 7823737e78b15237219ae3fea7fbd069541fecc2 Mon Sep 17 00:00:00 2001 From: "Ahmad N. F." Date: Thu, 17 Oct 2024 12:09:18 +0700 Subject: [PATCH 7/8] feat: add unit test for empty vars & refactor GetVariables --- core/tenant/namespace_test.go | 21 +++++++++++++++++++++ core/tenant/project_test.go | 29 ++++++++++++++++++++++++++++- core/tenant/tenant.go | 10 ++++++---- 3 files changed, 55 insertions(+), 5 deletions(-) diff --git a/core/tenant/namespace_test.go b/core/tenant/namespace_test.go index c60c2656a6..d9b5ee6ec9 100644 --- a/core/tenant/namespace_test.go +++ b/core/tenant/namespace_test.go @@ -60,6 +60,27 @@ func TestEntityNamespace(t *testing.T) { assert.Nil(t, err) assert.Equal(t, "ns_val", nsVar) + _, err = ns.GetVariable("non-existent") + assert.NotNil(t, err) + assert.EqualError(t, err, "not found for entity namespace: namespace variable not found: non-existent") + }) + t.Run("creates namespace object with nil config & variables", func(t *testing.T) { + ns, err := tenant.NewNamespace("t-namespace", projName, nil, nil) + + assert.Nil(t, err) + assert.Equal(t, "t-namespace", ns.Name().String()) + assert.Equal(t, projName.String(), ns.ProjectName().String()) + + assert.NotNil(t, ns.GetConfigs()) + assert.Empty(t, ns.GetConfigs()) + + _, err = ns.GetConfig("non-existent") + assert.NotNil(t, err) + assert.EqualError(t, err, "not found for entity namespace: namespace config not found non-existent") + + assert.NotNil(t, ns.GetVariables()) + assert.Empty(t, ns.GetVariables()) + _, err = ns.GetVariable("non-existent") assert.NotNil(t, err) assert.EqualError(t, err, "not found for entity namespace: namespace variable not found: non-existent") diff --git a/core/tenant/project_test.go b/core/tenant/project_test.go index abe5e53bdc..4791b70398 100644 --- a/core/tenant/project_test.go +++ b/core/tenant/project_test.go @@ -25,7 +25,7 @@ func TestEntityProject(t *testing.T) { t.Run("Project", func(t *testing.T) { t.Run("fails to create if name is empty", func(t *testing.T) { - project, err := tenant.NewProject("", map[string]string{"a": "b"}, map[string]string{}) + project, err := tenant.NewProject("", map[string]string{"a": "b"}, nil) assert.Nil(t, project) assert.NotNil(t, err) @@ -70,5 +70,32 @@ func TestEntityProject(t *testing.T) { assert.NotNil(t, err) assert.EqualError(t, err, "not found for entity project: variable not found: non-existent") }) + t.Run("creates a project with nil variables", func(t *testing.T) { + project, err := tenant.NewProject("t-optimus", map[string]string{ + tenant.ProjectSchedulerHost: "b", + tenant.ProjectStoragePathKey: "d", + }, nil) + assert.Nil(t, err) + + assert.NotNil(t, project) + assert.Equal(t, "t-optimus", project.Name().String()) + + assert.NotNil(t, project.GetConfigs()) + + val1, err := project.GetConfig(tenant.ProjectSchedulerHost) + assert.Nil(t, err) + assert.Equal(t, "b", val1) + + val2, err := project.GetConfig(tenant.ProjectStoragePathKey) + assert.Nil(t, err) + assert.Equal(t, "d", val2) + + assert.NotNil(t, project.GetVariables()) + assert.Len(t, project.GetVariables(), 0) + + _, err = project.GetVariable("PROJECT") + assert.Error(t, err) + assert.EqualError(t, err, "not found for entity project: variable not found: PROJECT") + }) }) } diff --git a/core/tenant/tenant.go b/core/tenant/tenant.go index 5d810ec22c..fc253202ad 100644 --- a/core/tenant/tenant.go +++ b/core/tenant/tenant.go @@ -111,10 +111,12 @@ func (w *WithDetails) GetConfigs() map[string]string { // merging both is a temporary solution to support older behavior. // Once we have all tenants migrated to use variables, we can remove the tenant config. func (w *WithDetails) GetVariables() map[string]string { - tenantConfigs := utils.MergeMaps(w.project.GetConfigs(), w.namespace.GetConfigs()) - tenantVariables := utils.MergeMaps(w.project.GetVariables(), w.namespace.GetVariables()) - - return utils.MergeMaps(tenantConfigs, tenantVariables) + return utils.MergeMaps( + w.project.GetConfigs(), + w.namespace.GetConfigs(), + w.project.GetVariables(), + w.namespace.GetVariables(), + ) } func (w *WithDetails) Project() *Project { From 1ef4af419c5e239fd92784a7058c7f1be44bc8ae Mon Sep 17 00:00:00 2001 From: "Ahmad N. F." Date: Fri, 18 Oct 2024 13:19:03 +0700 Subject: [PATCH 8/8] fix: rename CompileEngine to TemplateCompiler --- core/resource/service/resource_service.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/resource/service/resource_service.go b/core/resource/service/resource_service.go index 2985e8385a..726fa577c7 100644 --- a/core/resource/service/resource_service.go +++ b/core/resource/service/resource_service.go @@ -61,7 +61,7 @@ type TenantDetailsGetter interface { GetDetails(ctx context.Context, tnnt tenant.Tenant) (*tenant.WithDetails, error) } -type CompileEngine interface { +type TemplateCompiler interface { Compile(templateMap map[string]string, context map[string]any) (map[string]string, error) CompileString(input string, context map[string]any) (string, error) } @@ -77,7 +77,7 @@ type ResourceService struct { alertHandler AlertManager tenantDetailsGetter TenantDetailsGetter - compileEngine CompileEngine + compileEngine TemplateCompiler } type AlertManager interface { @@ -88,7 +88,7 @@ func NewResourceService( logger log.Logger, repo ResourceRepository, downstreamRefresher DownstreamRefresher, mgr ResourceManager, eventHandler EventHandler, downstreamResolver DownstreamResolver, alertManager AlertManager, - tenantDetailsGetter TenantDetailsGetter, compileEngine CompileEngine, + tenantDetailsGetter TenantDetailsGetter, compileEngine TemplateCompiler, ) *ResourceService { return &ResourceService{ repo: repo,