From 51a43a27618ce3d3dd46e93d655ec70a059f98e8 Mon Sep 17 00:00:00 2001 From: "Ahmad N. F." Date: Mon, 14 Oct 2024 12:54:51 +0700 Subject: [PATCH] feat: support resource spec v2 (#274) * feat: add 1st version of resource spec v2 parser * feat: restructure * feat: add ReadByURN method for resource, update GetByURN to use it * feat: restructure function, add unit tests * feat: use urncomponent for dataset and resource name * feat: add unit test for update & create * feat: restore empty name checking * fix --- core/resource/handler/v1beta1/resource.go | 6 +- core/resource/resource.go | 13 + core/resource/resource_test.go | 37 +++ core/resource/service/resource_service.go | 13 +- .../resource/service/resource_service_test.go | 59 +++++ ext/store/bigquery/backup.go | 6 +- ext/store/bigquery/bigquery.go | 40 +++- ext/store/bigquery/bigquery_test.go | 226 ++++++++++++++++++ ext/store/bigquery/dataset_spec.go | 72 ++++++ ext/store/bigquery/dataset_spec_test.go | 116 +++++++++ .../store/postgres/resource/repository.go | 23 ++ .../postgres/resource/repository_test.go | 35 +++ 12 files changed, 617 insertions(+), 29 deletions(-) diff --git a/core/resource/handler/v1beta1/resource.go b/core/resource/handler/v1beta1/resource.go index 72d17e8bbc..3590e0b5e8 100644 --- a/core/resource/handler/v1beta1/resource.go +++ b/core/resource/handler/v1beta1/resource.go @@ -434,9 +434,9 @@ func (rh ResourceHandler) GetResourceChangelogs(ctx context.Context, req *pb.Get return nil, errors.GRPCErr(err, "invalid project name") } - resourceName, err := resource.NameFrom(req.GetResourceName()) - if err != nil { - return nil, errors.GRPCErr(err, "invalid resource name") + resourceName := resource.Name(req.GetResourceName()) + if resourceName == "" { + return nil, errors.GRPCErr(errors.InvalidArgument(resource.EntityResource, "resource name is empty"), "invalid parameter") } changelogs, err := rh.changelogService.GetChangelogs(ctx, projectName, resourceName) diff --git a/core/resource/resource.go b/core/resource/resource.go index 779b97f592..8a1e372480 100644 --- a/core/resource/resource.go +++ b/core/resource/resource.go @@ -17,6 +17,11 @@ const ( UnspecifiedImpactChange UpdateImpact = "unspecified_impact" ResourceDataPipeLineImpact UpdateImpact = "data_impact" + + ResourceSpecV1 = 1 + ResourceSpecV2 = 2 + + DefaultResourceSpecVersion = ResourceSpecV1 ) type UpdateImpact string @@ -183,6 +188,14 @@ func (r *Resource) Spec() map[string]any { return r.spec } +func (r *Resource) Version() int32 { + if r.metadata == nil || r.metadata.Version == 0 { + return DefaultResourceSpecVersion + } + + return r.metadata.Version +} + func (r *Resource) Equal(incoming *Resource) bool { if r == nil || incoming == nil { return r == nil && incoming == nil diff --git a/core/resource/resource_test.go b/core/resource/resource_test.go index 4b1df5fb58..19ca44ffbb 100644 --- a/core/resource/resource_test.go +++ b/core/resource/resource_test.go @@ -604,4 +604,41 @@ func TestResource(t *testing.T) { }) }) }) + + t.Run("Version", func(t *testing.T) { + t.Run("returns default version if no version is provided", func(t *testing.T) { + tnnt, tnntErr := tenant.NewTenant("proj", "ns") + assert.Nil(t, tnntErr) + + metadata := &resource.Metadata{ + Description: "description", + } + spec := map[string]any{ + "description": "spec for unit test", + } + res, err := resource.NewResource("proj.set.res_name", "table", resource.Bigquery, tnnt, metadata, spec) + assert.Nil(t, err) + + version := res.Version() + assert.Equal(t, int32(resource.DefaultResourceSpecVersion), version) + }) + + t.Run("returns version from metadata if metadata is not nil", func(t *testing.T) { + tnnt, tnntErr := tenant.NewTenant("proj", "ns") + assert.Nil(t, tnntErr) + + meta := &resource.Metadata{ + Version: resource.ResourceSpecV2, + Description: "description", + } + spec := map[string]any{ + "description": "spec for unit test", + } + res, err := resource.NewResource("proj.set.res_name", "table", resource.Bigquery, tnnt, meta, spec) + assert.Nil(t, err) + + version := res.Version() + assert.Equal(t, int32(resource.ResourceSpecV2), version) + }) + }) } diff --git a/core/resource/service/resource_service.go b/core/resource/service/resource_service.go index 669f22e900..ee8d529395 100644 --- a/core/resource/service/resource_service.go +++ b/core/resource/service/resource_service.go @@ -26,6 +26,7 @@ type ResourceRepository interface { ReadByFullName(ctx context.Context, tnnt tenant.Tenant, store resource.Store, fullName string, onlyActive bool) (*resource.Resource, error) ReadAll(ctx context.Context, tnnt tenant.Tenant, store resource.Store, onlyActive bool) ([]*resource.Resource, error) GetResources(ctx context.Context, tnnt tenant.Tenant, store resource.Store, names []string) ([]*resource.Resource, error) + ReadByURN(ctx context.Context, tnnt tenant.Tenant, urn resource.URN) (*resource.Resource, error) } type ResourceManager interface { @@ -348,17 +349,7 @@ func (rs ResourceService) GetByURN(ctx context.Context, tnnt tenant.Tenant, urn return nil, errors.InvalidArgument(resource.EntityResource, "urn is zero value") } - store, err := resource.FromStringToStore(urn.GetStore()) - if err != nil { - return nil, err - } - - name, err := resource.NameFrom(urn.GetName()) - if err != nil { - return nil, err - } - - return rs.repo.ReadByFullName(ctx, tnnt, store, name.String(), false) + return rs.repo.ReadByURN(ctx, tnnt, urn) } func (rs ResourceService) ExistInStore(ctx context.Context, tnnt tenant.Tenant, urn resource.URN) (bool, error) { diff --git a/core/resource/service/resource_service_test.go b/core/resource/service/resource_service_test.go index 05bd3aef1f..54e3040ef3 100644 --- a/core/resource/service/resource_service_test.go +++ b/core/resource/service/resource_service_test.go @@ -1419,6 +1419,57 @@ func TestResourceService(t *testing.T) { assert.Nil(t, actual) }) }) + + t.Run("GetByURN", func(t *testing.T) { + t.Run("returns error if urn is zero value", func(t *testing.T) { + mgr := NewResourceManager(t) + repo := newResourceRepository(t) + logger := log.NewLogrus() + + rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil) + + _, err := rscService.GetByURN(ctx, tnnt, resource.ZeroURN()) + assert.Error(t, err) + assert.ErrorContains(t, err, "urn is zero value") + }) + + t.Run("returns error if repo returns error", func(t *testing.T) { + mgr := NewResourceManager(t) + repo := newResourceRepository(t) + logger := log.NewLogrus() + + urn, err := resource.ParseURN("bigquery://project:dataset.table") + assert.NoError(t, err) + + repo.On("ReadByURN", ctx, tnnt, urn).Return(nil, errors.New("unknown error")) + + rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil) + + _, err = rscService.GetByURN(ctx, tnnt, urn) + assert.Error(t, err) + assert.ErrorContains(t, err, "unknown error") + }) + + t.Run("returns resource if no error is encountered", func(t *testing.T) { + mgr := NewResourceManager(t) + repo := newResourceRepository(t) + logger := log.NewLogrus() + + urn, err := resource.ParseURN("bigquery://project:dataset.table") + assert.NoError(t, err) + + expectedResource, err := resource.NewResource("project.dataset", "dataset", resource.Bigquery, tnnt, meta, spec) + assert.NoError(t, err) + + repo.On("ReadByURN", ctx, tnnt, urn).Return(expectedResource, nil) + + rscService := service.NewResourceService(logger, repo, nil, mgr, nil, nil, nil) + + actualResource, err := rscService.GetByURN(ctx, tnnt, urn) + assert.NoError(t, err) + assert.Equal(t, expectedResource, actualResource) + }) + }) } type mockResourceRepository struct { @@ -1465,6 +1516,14 @@ func (m *mockResourceRepository) Delete(ctx context.Context, res *resource.Resou return m.Called(ctx, res).Error(0) } +func (m *mockResourceRepository) ReadByURN(ctx context.Context, tnnt tenant.Tenant, urn resource.URN) (*resource.Resource, error) { + args := m.Called(ctx, tnnt, urn) + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).(*resource.Resource), args.Error(1) +} + type mockConstructorTestingTNewResourceRepository interface { mock.TestingT Cleanup(func()) diff --git a/ext/store/bigquery/backup.go b/ext/store/bigquery/backup.go index 162d36383f..14f4256daa 100644 --- a/ext/store/bigquery/backup.go +++ b/ext/store/bigquery/backup.go @@ -92,11 +92,7 @@ func CreateIfDatasetDoesNotExist(ctx context.Context, client Client, dataset Dat } func BackupTable(ctx context.Context, backup *resource.Backup, source *resource.Resource, client Client) (string, error) { - sourceDataset, err := DataSetFor(source.Name()) - if err != nil { - return "", err - } - sourceName, err := ResourceNameFor(source.Name(), source.Kind()) + sourceDataset, sourceName, err := getDatasetAndResourceName(source) if err != nil { return "", err } diff --git a/ext/store/bigquery/bigquery.go b/ext/store/bigquery/bigquery.go index 5c77f6255e..0fa3883cb6 100644 --- a/ext/store/bigquery/bigquery.go +++ b/ext/store/bigquery/bigquery.go @@ -77,11 +77,7 @@ func (s Store) Create(ctx context.Context, res *resource.Resource) error { } defer client.Close() - dataset, err := DataSetFor(res.Name()) - if err != nil { - return err - } - resourceName, err := ResourceNameFor(res.Name(), res.Kind()) + dataset, resourceName, err := getDatasetAndResourceName(res) if err != nil { return err } @@ -123,11 +119,7 @@ func (s Store) Update(ctx context.Context, res *resource.Resource) error { } defer client.Close() - dataset, err := DataSetFor(res.Name()) - if err != nil { - return err - } - resourceName, err := ResourceNameFor(res.Name(), res.Kind()) + dataset, resourceName, err := getDatasetAndResourceName(res) if err != nil { return err } @@ -154,6 +146,34 @@ func (s Store) Update(ctx context.Context, res *resource.Resource) error { } } +func getDatasetAndResourceName(res *resource.Resource) (Dataset, string, error) { + if res.Version() == resource.ResourceSpecV2 { + bqURN, err := getURNComponent(res) + if err != nil { + return Dataset{}, "", err + } + + dataset, err := DataSetFrom(bqURN.Project, bqURN.Dataset) + if err != nil { + return Dataset{}, "", err + } + + return dataset, bqURN.Name, nil + } + + dataset, err := DataSetFor(res.Name()) + if err != nil { + return Dataset{}, "", err + } + + resourceName, err := ResourceNameFor(res.Name(), res.Kind()) + if err != nil { + return Dataset{}, "", err + } + + return dataset, resourceName, nil +} + func (s Store) BatchUpdate(ctx context.Context, resources []*resource.Resource) error { spanCtx, span := startChildSpan(ctx, "bigquery/BatchUpdate") defer span.End() diff --git a/ext/store/bigquery/bigquery_test.go b/ext/store/bigquery/bigquery_test.go index db04b92a40..40073f3920 100644 --- a/ext/store/bigquery/bigquery_test.go +++ b/ext/store/bigquery/bigquery_test.go @@ -226,6 +226,119 @@ func TestBigqueryStore(t *testing.T) { err = bqStore.Create(ctx, extTable) assert.Nil(t, err) }) + + // v2 spec tests + metadataV2 := resource.Metadata{ + Version: resource.ResourceSpecV2, + } + expectedDs := bigquery.Dataset{Project: "project-new", DatasetName: "dataset-new"} + + t.Run("calls appropriate handler for dataset with v2 spec", func(t *testing.T) { + specV2 := map[string]any{ + "description": "resource", + "project": "project-new", + "dataset": "dataset-new", + } + + pts, _ := tenant.NewPlainTextSecret("secret_name", "secret_value") + secretProvider := new(mockSecretProvider) + secretProvider.On("GetSecret", mock.Anything, tnnt, "DATASTORE_BIGQUERY"). + Return(pts, nil) + defer secretProvider.AssertExpectations(t) + + datasetRes, err := resource.NewResource("project-dataset", bigquery.KindDataset, store, tnnt, &metadataV2, specV2) + assert.Nil(t, err) + + datasetHandle := new(mockTableResourceHandle) + datasetHandle.On("Create", mock.Anything, datasetRes).Return(nil) + defer datasetHandle.AssertExpectations(t) + + client := new(mockClient) + client.On("Close").Return(nil) + client.On("DatasetHandleFrom", expectedDs).Return(datasetHandle) + defer client.AssertExpectations(t) + + clientProvider := new(mockClientProvider) + clientProvider.On("Get", mock.Anything, "secret_value").Return(client, nil) + defer clientProvider.AssertExpectations(t) + + bqStore := bigquery.NewBigqueryDataStore(secretProvider, clientProvider) + + err = bqStore.Create(ctx, datasetRes) + assert.Nil(t, err) + }) + + t.Run("calls appropriate handler for table with v2 spec", func(t *testing.T) { + specV2 := map[string]any{ + "description": "resource", + "project": "project-new", + "dataset": "dataset-new", + "name": "table-new", + } + + pts, _ := tenant.NewPlainTextSecret("secret_name", "secret_value") + secretProvider := new(mockSecretProvider) + secretProvider.On("GetSecret", mock.Anything, tnnt, "DATASTORE_BIGQUERY"). + Return(pts, nil) + defer secretProvider.AssertExpectations(t) + + tableRes, err := resource.NewResource("project.dataset.table", bigquery.KindTable, store, tnnt, &metadataV2, specV2) + assert.Nil(t, err) + + tableHandle := new(mockTableResourceHandle) + tableHandle.On("Create", mock.Anything, tableRes).Return(nil) + defer tableHandle.AssertExpectations(t) + + client := new(mockClient) + client.On("Close").Return(nil) + client.On("TableHandleFrom", expectedDs, "table-new").Return(tableHandle) + defer client.AssertExpectations(t) + + clientProvider := new(mockClientProvider) + clientProvider.On("Get", mock.Anything, "secret_value").Return(client, nil) + defer clientProvider.AssertExpectations(t) + + bqStore := bigquery.NewBigqueryDataStore(secretProvider, clientProvider) + + err = bqStore.Create(ctx, tableRes) + assert.Nil(t, err) + }) + + t.Run("calls appropriate handler for view with v2 spec", func(t *testing.T) { + specV2 := map[string]any{ + "description": "resource", + "project": "project-new", + "dataset": "dataset-new", + "name": "view-new", + } + + pts, _ := tenant.NewPlainTextSecret("secret_name", "secret_value") + secretProvider := new(mockSecretProvider) + secretProvider.On("GetSecret", mock.Anything, tnnt, "DATASTORE_BIGQUERY"). + Return(pts, nil) + defer secretProvider.AssertExpectations(t) + + viewRes, err := resource.NewResource("project.dataset.table", bigquery.KindView, store, tnnt, &metadataV2, specV2) + assert.NoError(t, err) + + viewHandle := new(mockTableResourceHandle) + viewHandle.On("Create", mock.Anything, viewRes).Return(nil) + defer viewHandle.AssertExpectations(t) + + client := new(mockClient) + client.On("Close").Return(nil) + client.On("ViewHandleFrom", expectedDs, "view-new").Return(viewHandle) + defer client.AssertExpectations(t) + + clientProvider := new(mockClientProvider) + clientProvider.On("Get", mock.Anything, "secret_value").Return(client, nil) + defer clientProvider.AssertExpectations(t) + + bqStore := bigquery.NewBigqueryDataStore(secretProvider, clientProvider) + + err = bqStore.Create(ctx, viewRes) + assert.Nil(t, err) + }) }) t.Run("Update", func(t *testing.T) { t.Run("returns error when secret is not provided", func(t *testing.T) { @@ -426,6 +539,119 @@ func TestBigqueryStore(t *testing.T) { err = bqStore.Update(ctx, extTable) assert.Nil(t, err) }) + + // v2 spec tests + metadataV2 := resource.Metadata{ + Version: resource.ResourceSpecV2, + } + expectedDs := bigquery.Dataset{Project: "project-new", DatasetName: "dataset-new"} + + t.Run("calls appropriate handler for dataset with v2 spec", func(t *testing.T) { + specV2 := map[string]any{ + "description": "resource", + "project": "project-new", + "dataset": "dataset-new", + } + + pts, _ := tenant.NewPlainTextSecret("secret_name", "secret_value") + secretProvider := new(mockSecretProvider) + secretProvider.On("GetSecret", mock.Anything, tnnt, "DATASTORE_BIGQUERY"). + Return(pts, nil) + defer secretProvider.AssertExpectations(t) + + datasetRes, err := resource.NewResource("project-dataset", bigquery.KindDataset, store, tnnt, &metadataV2, specV2) + assert.Nil(t, err) + + datasetHandle := new(mockTableResourceHandle) + datasetHandle.On("Update", mock.Anything, datasetRes).Return(nil) + defer datasetHandle.AssertExpectations(t) + + client := new(mockClient) + client.On("Close").Return(nil) + client.On("DatasetHandleFrom", expectedDs).Return(datasetHandle) + defer client.AssertExpectations(t) + + clientProvider := new(mockClientProvider) + clientProvider.On("Get", mock.Anything, "secret_value").Return(client, nil) + defer clientProvider.AssertExpectations(t) + + bqStore := bigquery.NewBigqueryDataStore(secretProvider, clientProvider) + + err = bqStore.Update(ctx, datasetRes) + assert.Nil(t, err) + }) + + t.Run("calls appropriate handler for table with v2 spec", func(t *testing.T) { + specV2 := map[string]any{ + "description": "resource", + "project": "project-new", + "dataset": "dataset-new", + "name": "table-new", + } + + pts, _ := tenant.NewPlainTextSecret("secret_name", "secret_value") + secretProvider := new(mockSecretProvider) + secretProvider.On("GetSecret", mock.Anything, tnnt, "DATASTORE_BIGQUERY"). + Return(pts, nil) + defer secretProvider.AssertExpectations(t) + + tableRes, err := resource.NewResource("project.dataset.table", bigquery.KindTable, store, tnnt, &metadataV2, specV2) + assert.Nil(t, err) + + tableHandle := new(mockTableResourceHandle) + tableHandle.On("Update", mock.Anything, tableRes).Return(nil) + defer tableHandle.AssertExpectations(t) + + client := new(mockClient) + client.On("Close").Return(nil) + client.On("TableHandleFrom", expectedDs, "table-new").Return(tableHandle) + defer client.AssertExpectations(t) + + clientProvider := new(mockClientProvider) + clientProvider.On("Get", mock.Anything, "secret_value").Return(client, nil) + defer clientProvider.AssertExpectations(t) + + bqStore := bigquery.NewBigqueryDataStore(secretProvider, clientProvider) + + err = bqStore.Update(ctx, tableRes) + assert.Nil(t, err) + }) + + t.Run("calls appropriate handler for view with v2 spec", func(t *testing.T) { + specV2 := map[string]any{ + "description": "resource", + "project": "project-new", + "dataset": "dataset-new", + "name": "view-new", + } + + pts, _ := tenant.NewPlainTextSecret("secret_name", "secret_value") + secretProvider := new(mockSecretProvider) + secretProvider.On("GetSecret", mock.Anything, tnnt, "DATASTORE_BIGQUERY"). + Return(pts, nil) + defer secretProvider.AssertExpectations(t) + + viewRes, err := resource.NewResource("project.dataset.table", bigquery.KindView, store, tnnt, &metadataV2, specV2) + assert.NoError(t, err) + + viewHandle := new(mockTableResourceHandle) + viewHandle.On("Update", mock.Anything, viewRes).Return(nil) + defer viewHandle.AssertExpectations(t) + + client := new(mockClient) + client.On("Close").Return(nil) + client.On("ViewHandleFrom", expectedDs, "view-new").Return(viewHandle) + defer client.AssertExpectations(t) + + clientProvider := new(mockClientProvider) + clientProvider.On("Get", mock.Anything, "secret_value").Return(client, nil) + defer clientProvider.AssertExpectations(t) + + bqStore := bigquery.NewBigqueryDataStore(secretProvider, clientProvider) + + err = bqStore.Update(ctx, viewRes) + assert.Nil(t, err) + }) }) t.Run("BatchUpdate", func(t *testing.T) { t.Run("returns no error when empty list", func(t *testing.T) { diff --git a/ext/store/bigquery/dataset_spec.go b/ext/store/bigquery/dataset_spec.go index a3550d64df..045b1d1d02 100644 --- a/ext/store/bigquery/dataset_spec.go +++ b/ext/store/bigquery/dataset_spec.go @@ -21,6 +21,8 @@ var ( validProjectName = regexp.MustCompile(`^[a-z][a-z0-9-]{4,28}[a-z0-9]$`) validDatasetName = regexp.MustCompile(`^[a-zA-Z0-9_]+$`) validTableName = regexp.MustCompile(`^[a-zA-Z0-9_-]+$`) + + validResourceName = regexp.MustCompile(`^[a-z][a-zA-Z0-9._-]+$`) ) type DatasetDetails struct { @@ -102,6 +104,14 @@ func ResourceNameFor(name resource.Name, kind string) (string, error) { } func ValidateName(res *resource.Resource) error { + if res.Version() == resource.ResourceSpecV2 { + return validateNameV2(res) + } + + return validateNameV1(res) +} + +func validateNameV1(res *resource.Resource) error { sections := res.Name().Sections() if len(sections) < DatesetNameSections { return errors.InvalidArgument(resource.EntityResource, "invalid sections in name: "+res.FullName()) @@ -127,7 +137,46 @@ func ValidateName(res *resource.Resource) error { return nil } +func validateNameV2(res *resource.Resource) error { + if !validResourceName.MatchString(res.FullName()) { + return errors.InvalidArgument(resource.EntityResource, "invalid character in resource name "+res.FullName()) + } + + var spec URNComponent + if err := mapstructure.Decode(res.Spec(), &spec); err != nil { + return errors.InvalidArgument(resource.EntityResource, "not able to decode spec") + } + + if !validProjectName.MatchString(spec.Project) { + return errors.InvalidArgument(resource.EntityResource, fmt.Sprintf("invalid character in project name: %s from %s ", spec.Project, res.FullName())) + } + + if !validDatasetName.MatchString(spec.Dataset) { + return errors.InvalidArgument(resource.EntityResource, fmt.Sprintf("invalid character in dataset name: %s from %s ", spec.Dataset, res.FullName())) + } + + if res.Kind() != KindDataset && !validTableName.MatchString(spec.Name) { + return errors.InvalidArgument(resource.EntityResource, fmt.Sprintf("invalid character in table/view name: %s from %s ", spec.Name, res.FullName())) + } + + return nil +} + +type URNComponent struct { + Project string `mapstructure:"project"` + Dataset string `mapstructure:"dataset"` + Name string `mapstructure:"name"` +} + func URNFor(res *resource.Resource) (resource.URN, error) { + if res.Version() == resource.ResourceSpecV2 { + return urnForV2(res) + } + + return urnForV1(res) +} + +func urnForV1(res *resource.Resource) (resource.URN, error) { dataset, err := DataSetFor(res.Name()) if err != nil { return resource.ZeroURN(), err @@ -146,3 +195,26 @@ func URNFor(res *resource.Resource) (resource.URN, error) { name := dataset.Project + ":" + dataset.DatasetName + "." + resourceName return resource.NewURN(resource.Bigquery.String(), name) } + +func getURNComponent(res *resource.Resource) (URNComponent, error) { + var spec URNComponent + if err := mapstructure.Decode(res.Spec(), &spec); err != nil { + return spec, err + } + + return spec, nil +} + +func urnForV2(res *resource.Resource) (resource.URN, error) { + spec, err := getURNComponent(res) + if err != nil { + return resource.ZeroURN(), errors.InvalidArgument(resource.EntityResource, "not able to decode spec") + } + + name := spec.Project + ":" + spec.Dataset + if res.Kind() != KindDataset { + name = name + "." + spec.Name + } + + return resource.NewURN(resource.Bigquery.String(), name) +} diff --git a/ext/store/bigquery/dataset_spec_test.go b/ext/store/bigquery/dataset_spec_test.go index 0622b4f4be..27567391bb 100644 --- a/ext/store/bigquery/dataset_spec_test.go +++ b/ext/store/bigquery/dataset_spec_test.go @@ -138,6 +138,18 @@ func TestValidateName(t *testing.T) { } spec := map[string]any{"description": []string{"a", "b"}} + metadataV2 := resource.Metadata{ + Version: 2, + Description: "resource description", + Labels: map[string]string{"owner": "optimus"}, + } + specV2 := map[string]any{ + "project": "project", + "dataset": "dataset", + "name": "table", + "description": "a", + } + t.Run("when invalid", func(t *testing.T) { t.Run("return error for not enough sections", func(t *testing.T) { res, err := resource.NewResource("proj", bigquery.KindDataset, bqStore, tnnt, &metadata, spec) @@ -179,7 +191,50 @@ func TestValidateName(t *testing.T) { assert.Error(t, err) assert.ErrorContains(t, err, "invalid character in resource name p-project.dataset.tab@tab1") }) + + t.Run("returns error when project name is invalid for v2 spec", func(t *testing.T) { + invalidSpecV2 := map[string]any{ + "project": "project@@", + "dataset": "dataset", + "name": "table", + } + res, err := resource.NewResource("project.dataset.table", bigquery.KindTable, bqStore, tnnt, &metadataV2, invalidSpecV2) + assert.NoError(t, err) + + err = bigquery.ValidateName(res) + assert.Error(t, err) + assert.ErrorContains(t, err, "invalid character in project name: project@@ from project.dataset.table") + }) + + t.Run("returns error when dataset name is invalid for v2 spec", func(t *testing.T) { + invalidSpecV2 := map[string]any{ + "project": "project", + "dataset": "datasetINVALID!", + "name": "table", + } + res, err := resource.NewResource("project.dataset.table", bigquery.KindTable, bqStore, tnnt, &metadataV2, invalidSpecV2) + assert.NoError(t, err) + + err = bigquery.ValidateName(res) + assert.Error(t, err) + assert.ErrorContains(t, err, "invalid character in dataset name: datasetINVALID! from project.dataset.table") + }) + + t.Run("returns error when table name is invalid for v2 spec", func(t *testing.T) { + invalidSpecV2 := map[string]any{ + "project": "project", + "dataset": "dataset", + "name": "table@", + } + res, err := resource.NewResource("project.dataset.table", bigquery.KindTable, bqStore, tnnt, &metadataV2, invalidSpecV2) + assert.NoError(t, err) + + err = bigquery.ValidateName(res) + assert.Error(t, err) + assert.ErrorContains(t, err, "invalid character in table/view name: table@ from project.dataset.table") + }) }) + t.Run("when valid", func(t *testing.T) { t.Run("return no error for dataset", func(t *testing.T) { res, err := resource.NewResource("p-project.dataset", bigquery.KindDataset, bqStore, tnnt, &metadata, spec) @@ -188,6 +243,7 @@ func TestValidateName(t *testing.T) { err = bigquery.ValidateName(res) assert.NoError(t, err) }) + t.Run("returns no error when table name is valid", func(t *testing.T) { res, err := resource.NewResource("p-project.dataset.tab1", bigquery.KindTable, bqStore, tnnt, &metadata, spec) assert.NoError(t, err) @@ -195,6 +251,22 @@ func TestValidateName(t *testing.T) { err = bigquery.ValidateName(res) assert.NoError(t, err) }) + + t.Run("returns no error for dataset v2 spec", func(t *testing.T) { + res, err := resource.NewResource("project.dataset", bigquery.KindDataset, bqStore, tnnt, &metadataV2, specV2) + assert.NoError(t, err) + + err = bigquery.ValidateName(res) + assert.NoError(t, err) + }) + + t.Run("returns no error for table v2 spec", func(t *testing.T) { + res, err := resource.NewResource("project.dataset.table", bigquery.KindTable, bqStore, tnnt, &metadataV2, specV2) + assert.NoError(t, err) + + err = bigquery.ValidateName(res) + assert.NoError(t, err) + }) }) } @@ -208,6 +280,23 @@ func TestURNFor(t *testing.T) { } spec := map[string]any{"description": []string{"a", "b"}} + metadataV2 := resource.Metadata{ + Version: 2, + Description: "resource description v2", + Labels: map[string]string{"owner": "optimus"}, + } + specV2 := map[string]any{ + "description": "a", + "project": "project", + "dataset": "dataset", + "name": "table", + } + invalidSpecV2 := map[string]any{ + "project": "project", + "dataset": []string{"data", "set"}, + "name": "table1", + } + t.Run("returns error when cannot get dataset", func(t *testing.T) { res, err := resource.NewResource("p-project.", bigquery.KindDataset, bqStore, tnnt, &metadata, spec) assert.NoError(t, err) @@ -239,4 +328,31 @@ func TestURNFor(t *testing.T) { assert.NoError(t, err) assert.Equal(t, "bigquery://p-project:dataset.table1", urn.String()) }) + + t.Run("returns urn for table with v2 spec", func(t *testing.T) { + res, err := resource.NewResource("project.dataset.table1", bigquery.KindDataset, bqStore, tnnt, &metadataV2, specV2) + assert.NoError(t, err) + + urn, err := bigquery.URNFor(res) + assert.NoError(t, err) + assert.Equal(t, "bigquery://project:dataset", urn.String()) + }) + + t.Run("returns urn for dataset with v2 spec", func(t *testing.T) { + res, err := resource.NewResource("project.dataset", bigquery.KindTable, bqStore, tnnt, &metadataV2, specV2) + assert.NoError(t, err) + + urn, err := bigquery.URNFor(res) + assert.NoError(t, err) + assert.Equal(t, "bigquery://project:dataset.table", urn.String()) + }) + + t.Run("returns error for table with invalid v2 spec", func(t *testing.T) { + res, err := resource.NewResource("project.dataset.table", bigquery.KindTable, bqStore, tnnt, &metadataV2, invalidSpecV2) + assert.NoError(t, err) + + _, err = bigquery.URNFor(res) + assert.Error(t, err) + assert.ErrorContains(t, err, "not able to decode spec") + }) } diff --git a/internal/store/postgres/resource/repository.go b/internal/store/postgres/resource/repository.go index 2431c2a50b..02a73c5b07 100644 --- a/internal/store/postgres/resource/repository.go +++ b/internal/store/postgres/resource/repository.go @@ -349,3 +349,26 @@ func (r Repository) GetChangelogs(ctx context.Context, projectName tenant.Projec return changeLog, me.ToErr() } + +func (r Repository) ReadByURN(ctx context.Context, tnnt tenant.Tenant, urn resource.URN) (*resource.Resource, error) { + query := ` + SELECT ` + resourceColumns + ` FROM resource + WHERE urn = $1 AND project_name = $2 AND namespace_name = $3 AND status NOT IN ($4, $5) + LIMIT 1 + ` + args := []any{urn.String(), tnnt.ProjectName(), tnnt.NamespaceName(), resource.StatusDeleted, resource.StatusToDelete} + + var res Resource + err := r.db.QueryRow(ctx, query, args...). + Scan(&res.ID, &res.FullName, &res.Kind, &res.Store, &res.Status, &res.URN, + &res.ProjectName, &res.NamespaceName, &res.Metadata, &res.Spec, &res.CreatedAt, &res.UpdatedAt) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, errors.NotFound(resource.EntityResource, fmt.Sprintf("no resource with urn: '%s' found for project:%s, namespace:%s ", urn, tnnt.ProjectName(), tnnt.NamespaceName())) + } + + return nil, errors.Wrap(resource.EntityResource, fmt.Sprintf("error reading resource with urn: '%s' found for project:%s, namespace:%s ", urn, tnnt.ProjectName(), tnnt.NamespaceName()), err) + } + + return FromModelToResource(&res) +} diff --git a/internal/store/postgres/resource/repository_test.go b/internal/store/postgres/resource/repository_test.go index f73e3c632a..b51e779e01 100644 --- a/internal/store/postgres/resource/repository_test.go +++ b/internal/store/postgres/resource/repository_test.go @@ -413,6 +413,41 @@ func TestPostgresResourceRepository(t *testing.T) { assert.Len(t, actualChangelogs, len(changelogs)) }) }) + + t.Run("ReadByURN", func(t *testing.T) { + t.Run("returns nil and error if resource does not exist", func(t *testing.T) { + pool := dbSetup() + repository := repoResource.NewRepository(pool) + + urn, err := serviceResource.ParseURN("bigquery://project:dataset") + assert.NoError(t, err) + + actualResource, actualError := repository.ReadByURN(ctx, tnnt, urn) + assert.Nil(t, actualResource) + assert.ErrorContains(t, actualError, "not found for entity resource") + }) + + t.Run("returns resource and nil if no error is encountered", func(t *testing.T) { + pool := dbSetup() + repository := repoResource.NewRepository(pool) + + urn, err := serviceResource.ParseURN("bigquery://project:dataset.table") + assert.NoError(t, err) + + resourceToCreate, err := serviceResource.NewResource("project.dataset.table", kindDataset, store, tnnt, meta, spec) + assert.NoError(t, err) + err = resourceToCreate.UpdateURN(urn) + assert.NoError(t, err) + + err = repository.Create(ctx, resourceToCreate) + assert.NoError(t, err) + + actualResource, actualError := repository.ReadByURN(ctx, tnnt, urn) + assert.NotNil(t, actualResource) + assert.NoError(t, actualError) + assert.EqualValues(t, resourceToCreate, actualResource) + }) + }) } func insertTestResourceChangelog(pool *pgxpool.Pool, resourceName serviceResource.Name, projectName tenant.ProjectName, changelogs []*repoResource.ChangeLog) {