diff --git a/acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_schema/databricks.yml b/acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_schema/databricks.yml new file mode 100644 index 0000000000..690df44435 --- /dev/null +++ b/acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_schema/databricks.yml @@ -0,0 +1,43 @@ +bundle: + name: dlt-schema-field-dep-on-uc-schema + +resources: + schemas: + schema1: + catalog_name: catalog1 + name: foobar + + schema2: + catalog_name: catalog2 + name: foobar + + schema3: + catalog_name: catalog1 + name: barfoo + + pipelines: + pipeline1: + catalog: catalog1 + schema: foobar + + pipeline2: + catalog: catalog2 + schema: foobar + + pipeline3: + catalog: catalog1 + schema: barfoo + + pipeline4: + catalog: catalogX + schema: foobar + + pipeline5: + catalog: catalog1 + schema: schemaX + + pipeline6: + schema: foobar + + pipeline7: + name: whatever diff --git a/acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_schema/out.json b/acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_schema/out.json new file mode 100644 index 0000000000..0bcadc09bb --- /dev/null +++ b/acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_schema/out.json @@ -0,0 +1,28 @@ +{ + "pipeline1": { + "catalog": "catalog1", + "schema": "${resources.schemas.schema1.name}" + }, + "pipeline2": { + "catalog": "catalog2", + "schema": "${resources.schemas.schema2.name}" + }, + "pipeline3": { + "catalog": "catalog1", + "schema": "${resources.schemas.schema3.name}" + }, + "pipeline4": { + "catalog": "catalogX", + "schema": "foobar" + }, + "pipeline5": { + "catalog": "catalog1", + "schema": "schemaX" + }, + "pipeline6": { + "schema": "foobar" + }, + "pipeline7": { + "name": "whatever" + } +} diff --git a/acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_schema/script b/acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_schema/script new file mode 100644 index 0000000000..94850c5c67 --- /dev/null +++ b/acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_schema/script @@ -0,0 +1,3 @@ +# We expect DLT pipelines that use a schema defined in the bundle to have the +# schema names replaced with a reference like `${resource.schemas.abc.name}`. +$CLI bundle validate -o json | jq .resources.pipelines | jq 'map_values(del(.deployment, .permissions))' > out.json diff --git a/acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_target/databricks.yml b/acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_target/databricks.yml new file mode 100644 index 0000000000..2fbdb42f49 --- /dev/null +++ b/acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_target/databricks.yml @@ -0,0 +1,43 @@ +bundle: + name: dlt-target-field-dep-on-uc-schema + +resources: + schemas: + schema1: + catalog_name: catalog1 + name: foobar + + schema2: + catalog_name: catalog2 + name: foobar + + schema3: + catalog_name: catalog1 + name: barfoo + + pipelines: + pipeline1: + catalog: catalog1 + target: foobar + + pipeline2: + catalog: catalog2 + target: foobar + + pipeline3: + catalog: catalog1 + target: barfoo + + pipeline4: + catalog: catalogX + target: foobar + + pipeline5: + catalog: catalog1 + target: schemaX + + pipeline6: + target: foobar + + pipeline7: + name: whatever diff --git a/acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_target/out.json b/acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_target/out.json new file mode 100644 index 0000000000..6c03b39829 --- /dev/null +++ b/acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_target/out.json @@ -0,0 +1,28 @@ +{ + "pipeline1": { + "catalog": "catalog1", + "target": "${resources.schemas.schema1.name}" + }, + "pipeline2": { + "catalog": "catalog2", + "target": "${resources.schemas.schema2.name}" + }, + "pipeline3": { + "catalog": "catalog1", + "target": "${resources.schemas.schema3.name}" + }, + "pipeline4": { + "catalog": "catalogX", + "target": "foobar" + }, + "pipeline5": { + "catalog": "catalog1", + "target": "schemaX" + }, + "pipeline6": { + "target": "foobar" + }, + "pipeline7": { + "name": "whatever" + } +} diff --git a/acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_target/script b/acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_target/script new file mode 100644 index 0000000000..94850c5c67 --- /dev/null +++ b/acceptance/bundle/unitycatalog/capture_schema_dependency/dlt_target/script @@ -0,0 +1,3 @@ +# We expect DLT pipelines that use a schema defined in the bundle to have the +# schema names replaced with a reference like `${resource.schemas.abc.name}`. +$CLI bundle validate -o json | jq .resources.pipelines | jq 'map_values(del(.deployment, .permissions))' > out.json diff --git a/acceptance/bundle/unitycatalog/capture_schema_dependency/volume/databricks.yml b/acceptance/bundle/unitycatalog/capture_schema_dependency/volume/databricks.yml new file mode 100644 index 0000000000..86f5e20b18 --- /dev/null +++ b/acceptance/bundle/unitycatalog/capture_schema_dependency/volume/databricks.yml @@ -0,0 +1,37 @@ +bundle: + name: volume-deps-on-schema + +resources: + schemas: + schema1: + catalog_name: catalog1 + name: foobar + + schema2: + catalog_name: catalog2 + name: foobar + + schema3: + catalog_name: catalog1 + name: barfoo + + volumes: + volume1: + catalog_name: catalog1 + schema_name: foobar + + volume2: + catalog_name: catalog2 + schema_name: foobar + + volume3: + catalog_name: catalog1 + schema_name: barfoo + + volume4: + catalog_name: catalogX + schema_name: foobar + + volume5: + catalog_name: catalog1 + schema_name: schemaX diff --git a/acceptance/bundle/unitycatalog/capture_schema_dependency/volume/out.json b/acceptance/bundle/unitycatalog/capture_schema_dependency/volume/out.json new file mode 100644 index 0000000000..0d27a70f19 --- /dev/null +++ b/acceptance/bundle/unitycatalog/capture_schema_dependency/volume/out.json @@ -0,0 +1,27 @@ +{ + "volume1": { + "catalog_name": "catalog1", + "schema_name": "${resources.schemas.schema1.name}", + "volume_type": "MANAGED" + }, + "volume2": { + "catalog_name": "catalog2", + "schema_name": "${resources.schemas.schema2.name}", + "volume_type": "MANAGED" + }, + "volume3": { + "catalog_name": "catalog1", + "schema_name": "${resources.schemas.schema3.name}", + "volume_type": "MANAGED" + }, + "volume4": { + "catalog_name": "catalogX", + "schema_name": "foobar", + "volume_type": "MANAGED" + }, + "volume5": { + "catalog_name": "catalog1", + "schema_name": "schemaX", + "volume_type": "MANAGED" + } +} diff --git a/acceptance/bundle/unitycatalog/capture_schema_dependency/volume/script b/acceptance/bundle/unitycatalog/capture_schema_dependency/volume/script new file mode 100644 index 0000000000..6b4b308196 --- /dev/null +++ b/acceptance/bundle/unitycatalog/capture_schema_dependency/volume/script @@ -0,0 +1,3 @@ +# We expect UC Volumes that use a schema defined in the bundle to have the +# schema names replaced with a reference like `${resource.schemas.abc.name}`. +$CLI bundle validate -o json | jq .resources.volumes > out.json diff --git a/bundle/config/mutator/capture_schema_dependency_test.go b/bundle/config/mutator/capture_schema_dependency_test.go index 0a94e77487..3f7f1ced9d 100644 --- a/bundle/config/mutator/capture_schema_dependency_test.go +++ b/bundle/config/mutator/capture_schema_dependency_test.go @@ -13,29 +13,11 @@ import ( "github.com/stretchr/testify/require" ) -func TestCaptureSchemaDependencyForVolume(t *testing.T) { +func TestCaptureSchemaDependencyForVolumeWithEmptySchemas(t *testing.T) { b := &bundle.Bundle{ Config: config.Root{ Resources: config.Resources{ Schemas: map[string]*resources.Schema{ - "schema1": { - CreateSchema: &catalog.CreateSchema{ - CatalogName: "catalog1", - Name: "foobar", - }, - }, - "schema2": { - CreateSchema: &catalog.CreateSchema{ - CatalogName: "catalog2", - Name: "foobar", - }, - }, - "schema3": { - CreateSchema: &catalog.CreateSchema{ - CatalogName: "catalog1", - Name: "barfoo", - }, - }, "nilschema": nil, "emptyschema": {}, }, @@ -46,30 +28,6 @@ func TestCaptureSchemaDependencyForVolume(t *testing.T) { SchemaName: "foobar", }, }, - "volume2": { - CreateVolumeRequestContent: &catalog.CreateVolumeRequestContent{ - CatalogName: "catalog2", - SchemaName: "foobar", - }, - }, - "volume3": { - CreateVolumeRequestContent: &catalog.CreateVolumeRequestContent{ - CatalogName: "catalog1", - SchemaName: "barfoo", - }, - }, - "volume4": { - CreateVolumeRequestContent: &catalog.CreateVolumeRequestContent{ - CatalogName: "catalogX", - SchemaName: "foobar", - }, - }, - "volume5": { - CreateVolumeRequestContent: &catalog.CreateVolumeRequestContent{ - CatalogName: "catalog1", - SchemaName: "schemaX", - }, - }, "nilVolume": nil, "emptyVolume": {}, }, @@ -80,39 +38,16 @@ func TestCaptureSchemaDependencyForVolume(t *testing.T) { d := bundle.Apply(context.Background(), b, CaptureSchemaDependency()) require.Nil(t, d) - assert.Equal(t, "${resources.schemas.schema1.name}", b.Config.Resources.Volumes["volume1"].CreateVolumeRequestContent.SchemaName) - assert.Equal(t, "${resources.schemas.schema2.name}", b.Config.Resources.Volumes["volume2"].CreateVolumeRequestContent.SchemaName) - assert.Equal(t, "${resources.schemas.schema3.name}", b.Config.Resources.Volumes["volume3"].CreateVolumeRequestContent.SchemaName) - assert.Equal(t, "foobar", b.Config.Resources.Volumes["volume4"].CreateVolumeRequestContent.SchemaName) - assert.Equal(t, "schemaX", b.Config.Resources.Volumes["volume5"].CreateVolumeRequestContent.SchemaName) - + assert.Equal(t, "foobar", b.Config.Resources.Volumes["volume1"].CreateVolumeRequestContent.SchemaName) assert.Nil(t, b.Config.Resources.Volumes["nilVolume"]) assert.Nil(t, b.Config.Resources.Volumes["emptyVolume"].CreateVolumeRequestContent) } -func TestCaptureSchemaDependencyForPipelinesWithTarget(t *testing.T) { +func TestCaptureSchemaDependencyForPipelinesWithEmptySchemas(t *testing.T) { b := &bundle.Bundle{ Config: config.Root{ Resources: config.Resources{ Schemas: map[string]*resources.Schema{ - "schema1": { - CreateSchema: &catalog.CreateSchema{ - CatalogName: "catalog1", - Name: "foobar", - }, - }, - "schema2": { - CreateSchema: &catalog.CreateSchema{ - CatalogName: "catalog2", - Name: "foobar", - }, - }, - "schema3": { - CreateSchema: &catalog.CreateSchema{ - CatalogName: "catalog1", - Name: "barfoo", - }, - }, "nilschema": nil, "emptyschema": {}, }, @@ -123,43 +58,6 @@ func TestCaptureSchemaDependencyForPipelinesWithTarget(t *testing.T) { Schema: "foobar", }, }, - "pipeline2": { - PipelineSpec: &pipelines.PipelineSpec{ - Catalog: "catalog2", - Schema: "foobar", - }, - }, - "pipeline3": { - PipelineSpec: &pipelines.PipelineSpec{ - Catalog: "catalog1", - Schema: "barfoo", - }, - }, - "pipeline4": { - PipelineSpec: &pipelines.PipelineSpec{ - Catalog: "catalogX", - Schema: "foobar", - }, - }, - "pipeline5": { - PipelineSpec: &pipelines.PipelineSpec{ - Catalog: "catalog1", - Schema: "schemaX", - }, - }, - "pipeline6": { - PipelineSpec: &pipelines.PipelineSpec{ - Catalog: "", - Schema: "foobar", - }, - }, - "pipeline7": { - PipelineSpec: &pipelines.PipelineSpec{ - Catalog: "", - Schema: "", - Name: "whatever", - }, - }, "nilPipeline": nil, "emptyPipeline": {}, }, @@ -170,108 +68,7 @@ func TestCaptureSchemaDependencyForPipelinesWithTarget(t *testing.T) { d := bundle.Apply(context.Background(), b, CaptureSchemaDependency()) require.Nil(t, d) - assert.Equal(t, "${resources.schemas.schema1.name}", b.Config.Resources.Pipelines["pipeline1"].Schema) - assert.Equal(t, "${resources.schemas.schema2.name}", b.Config.Resources.Pipelines["pipeline2"].Schema) - assert.Equal(t, "${resources.schemas.schema3.name}", b.Config.Resources.Pipelines["pipeline3"].Schema) - assert.Equal(t, "foobar", b.Config.Resources.Pipelines["pipeline4"].Schema) - assert.Equal(t, "schemaX", b.Config.Resources.Pipelines["pipeline5"].Schema) - assert.Equal(t, "foobar", b.Config.Resources.Pipelines["pipeline6"].Schema) - assert.Equal(t, "", b.Config.Resources.Pipelines["pipeline7"].Schema) - + assert.Equal(t, "foobar", b.Config.Resources.Pipelines["pipeline1"].Schema) assert.Nil(t, b.Config.Resources.Pipelines["nilPipeline"]) assert.Nil(t, b.Config.Resources.Pipelines["emptyPipeline"].PipelineSpec) - - for _, k := range []string{"pipeline1", "pipeline2", "pipeline3", "pipeline4", "pipeline5", "pipeline6", "pipeline7"} { - assert.Empty(t, b.Config.Resources.Pipelines[k].Target) - } -} - -func TestCaptureSchemaDependencyForPipelinesWithSchema(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - Schemas: map[string]*resources.Schema{ - "schema1": { - CreateSchema: &catalog.CreateSchema{ - CatalogName: "catalog1", - Name: "foobar", - }, - }, - "schema2": { - CreateSchema: &catalog.CreateSchema{ - CatalogName: "catalog2", - Name: "foobar", - }, - }, - "schema3": { - CreateSchema: &catalog.CreateSchema{ - CatalogName: "catalog1", - Name: "barfoo", - }, - }, - "nilschema": nil, - "emptyschema": {}, - }, - Pipelines: map[string]*resources.Pipeline{ - "pipeline1": { - PipelineSpec: &pipelines.PipelineSpec{ - Catalog: "catalog1", - Target: "foobar", - }, - }, - "pipeline2": { - PipelineSpec: &pipelines.PipelineSpec{ - Catalog: "catalog2", - Target: "foobar", - }, - }, - "pipeline3": { - PipelineSpec: &pipelines.PipelineSpec{ - Catalog: "catalog1", - Target: "barfoo", - }, - }, - "pipeline4": { - PipelineSpec: &pipelines.PipelineSpec{ - Catalog: "catalogX", - Target: "foobar", - }, - }, - "pipeline5": { - PipelineSpec: &pipelines.PipelineSpec{ - Catalog: "catalog1", - Target: "schemaX", - }, - }, - "pipeline6": { - PipelineSpec: &pipelines.PipelineSpec{ - Catalog: "", - Target: "foobar", - }, - }, - "pipeline7": { - PipelineSpec: &pipelines.PipelineSpec{ - Catalog: "", - Target: "", - Name: "whatever", - }, - }, - }, - }, - }, - } - - d := bundle.Apply(context.Background(), b, CaptureSchemaDependency()) - require.Nil(t, d) - assert.Equal(t, "${resources.schemas.schema1.name}", b.Config.Resources.Pipelines["pipeline1"].Target) - assert.Equal(t, "${resources.schemas.schema2.name}", b.Config.Resources.Pipelines["pipeline2"].Target) - assert.Equal(t, "${resources.schemas.schema3.name}", b.Config.Resources.Pipelines["pipeline3"].Target) - assert.Equal(t, "foobar", b.Config.Resources.Pipelines["pipeline4"].Target) - assert.Equal(t, "schemaX", b.Config.Resources.Pipelines["pipeline5"].Target) - assert.Equal(t, "foobar", b.Config.Resources.Pipelines["pipeline6"].Target) - assert.Equal(t, "", b.Config.Resources.Pipelines["pipeline7"].Target) - - for _, k := range []string{"pipeline1", "pipeline2", "pipeline3", "pipeline4", "pipeline5", "pipeline6", "pipeline7"} { - assert.Empty(t, b.Config.Resources.Pipelines[k].Schema) - } }