added testcases for model access and version (#101)

Co-authored-by: Sharma <[email protected]>

tests/functional/adapter/test_group_access.py

@@ -0,0 +1,251 @@
+import pytest
+from dbt.tests.util import run_dbt, get_manifest, write_file, rm_file
+from dbt.node_types import AccessType
+from dbt.exceptions import InvalidAccessTypeError, DbtReferenceError
+
+my_model_sql = "select 1 as fun"
+
+another_model_sql = "select 1234 as notfun"
+
+yet_another_model_sql = "select 999 as weird"
+
+schema_yml = """
+version: 2
+
+models:
+  - name: my_model
+    description: "my model"
+    access: public
+  - name: another_model
+    description: "yet another model"
+"""
+
+v2_schema_yml = """
+version: 2
+
+models:
+  - name: my_model
+    description: "my model"
+    access: public
+  - name: another_model
+    description: "another model"
+  - name: yet_another_model
+    description: "yet another model"
+    access: unsupported
+"""
+
+ref_my_model_sql = """
+   select fun from {{ ref('my_model') }}
+"""
+
+groups_yml = """
+version: 2
+
+groups:
+  - name: analytics
+    owner:
+      name: analytics_owner
+  - name: marts
+    owner:
+      name: marts_owner
+"""
+
+
+v3_schema_yml = """
+version: 2
+
+models:
+  - name: my_model
+    description: "my model"
+    access: private
+    group: analytics
+  - name: another_model
+    description: "yet another model"
+  - name: ref_my_model
+    description: "a model that refs my_model"
+    group: analytics
+"""
+
+v4_schema_yml = """
+version: 2
+
+models:
+  - name: my_model
+    description: "my model"
+    access: private
+    group: analytics
+  - name: another_model
+    description: "yet another model"
+  - name: ref_my_model
+    description: "a model that refs my_model"
+    group: marts
+"""
+
+simple_exposure_yml = """
+version: 2
+
+exposures:
+  - name: simple_exposure
+    label: simple exposure label
+    type: dashboard
+    depends_on:
+      - ref('my_model')
+    owner:
+      email: [email protected]
+"""
+
+v5_schema_yml = """
+version: 2
+
+models:
+  - name: my_model
+    description: "my model"
+    access: private
+    group: analytics
+  - name: another_model
+    description: "yet another model"
+  - name: ref_my_model
+    description: "a model that refs my_model"
+    group: marts
+  - name: ref_my_model
+    description: "a model that refs my_model"
+    group: analytics
+  - name: people_model
+    description: "some people"
+    access: private
+    group: analytics
+"""
+
+people_model_sql = """
+select 1 as id, 'Drew' as first_name, 'Banin' as last_name, 'yellow' as favorite_color, 5 as tenure, current_timestamp as created_at
+union all
+select 1 as id, 'Jeremy' as first_name, 'Cohen' as last_name, 'indigo' as favorite_color, 4 as tenure, current_timestamp as created_at
+union all
+select 1 as id, 'Callum' as first_name, 'McCann' as last_name, 'emerald' as favorite_color, 0 as tenure, current_timestamp as created_at
+"""
+
+people_metric_yml = """
+version: 2
+
+metrics:
+
+  - name: number_of_people
+    label: "Number of people"
+    description: Total count of people
+    model: "ref('people_model')"
+    calculation_method: count
+    expression: "*"
+    timestamp: created_at
+    time_grains: [day, week, month]
+    dimensions:
+      - favorite_color
+      - loves_dbt
+    meta:
+        my_meta: 'testing'
+    config:
+      group: analytics
+"""
+
+v2_people_metric_yml = """
+version: 2
+
+metrics:
+
+  - name: number_of_people
+    label: "Number of people"
+    description: Total count of people
+    model: "ref('people_model')"
+    calculation_method: count
+    expression: "*"
+    timestamp: created_at
+    time_grains: [day, week, month]
+    dimensions:
+      - favorite_color
+      - loves_dbt
+    meta:
+        my_meta: 'testing'
+    config:
+      group: marts
+"""
+
+
+class TestAccess:
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "my_model.sql": my_model_sql,
+            "another_model.sql": yet_another_model_sql,
+            "schema.yml": schema_yml,
+        }
+
+    def test_access_attribute(self, project):
+
+        results = run_dbt(["run"])
+        assert len(results) == 2
+
+        manifest = get_manifest(project.project_root)
+        my_model_id = "model.test.my_model"
+        another_model_id = "model.test.another_model"
+        assert my_model_id in manifest.nodes
+        assert another_model_id in manifest.nodes
+
+        assert manifest.nodes[my_model_id].access == AccessType.Public
+        assert manifest.nodes[another_model_id].access == AccessType.Protected
+
+        # write a file with an invalid access value
+        write_file(yet_another_model_sql, project.project_root, "models", "yet_another_model.sql")
+        write_file(v2_schema_yml, project.project_root, "models", "schema.yml")
+
+        with pytest.raises(InvalidAccessTypeError):
+            run_dbt(["run"])
+
+        # Remove invalid access files and write out model that refs my_model
+        rm_file(project.project_root, "models", "yet_another_model.sql")
+        write_file(schema_yml, project.project_root, "models", "schema.yml")
+        write_file(ref_my_model_sql, project.project_root, "models", "ref_my_model.sql")
+        results = run_dbt(["run"])
+        assert len(results) == 3
+
+        # make my_model private, set same group on my_model and ref_my_model
+        write_file(groups_yml, project.project_root, "models", "groups.yml")
+        write_file(v3_schema_yml, project.project_root, "models", "schema.yml")
+        results = run_dbt(["run"])
+        assert len(results) == 3
+        manifest = get_manifest(project.project_root)
+        ref_my_model_id = "model.test.ref_my_model"
+        assert manifest.nodes[my_model_id].group == "analytics"
+        assert manifest.nodes[ref_my_model_id].group == "analytics"
+
+        # Change group on ref_my_model and it should raise
+        write_file(v4_schema_yml, project.project_root, "models", "schema.yml")
+        with pytest.raises(DbtReferenceError):
+            run_dbt(["run"])
+
+        # put back group on ref_my_model, add exposure with ref to private model
+        write_file(v3_schema_yml, project.project_root, "models", "schema.yml")
+        # verify it works again
+        results = run_dbt(["run"])
+        assert len(results) == 3
+        # Write out exposure refing private my_model
+        write_file(simple_exposure_yml, project.project_root, "models", "simple_exposure.yml")
+        # Fails with reference error
+        with pytest.raises(DbtReferenceError):
+            run_dbt(["run"])
+
+        # Remove exposure and add people model and metric file
+        write_file(v5_schema_yml, project.project_root, "models", "schema.yml")
+        rm_file(project.project_root, "models", "simple_exposure.yml")
+        write_file(people_model_sql, "models", "people_model.sql")
+        write_file(people_metric_yml, "models", "people_metric.yml")
+        # Should succeed
+        results = run_dbt(["run"])
+        assert len(results) == 4
+        manifest = get_manifest(project.project_root)
+        metric_id = "metric.test.number_of_people"
+        assert manifest.metrics[metric_id].group == "analytics"
+
+        # Change group of metric
+        write_file(v2_people_metric_yml, "models", "people_metric.yml")
+        # Should raise a reference error
+        with pytest.raises(DbtReferenceError):
+            run_dbt(["run"])

tests/functional/adapter/test_model_versions.py

@@ -0,0 +1,126 @@
+import pytest
+import pathlib
+from dbt.tests.util import (
+    run_dbt,
+    get_manifest,
+    write_file,
+    rm_file,
+    read_file,
+)
+from dbt.exceptions import DuplicateVersionedUnversionedError
+
+model_one_sql = """
+select 1 as fun
+"""
+
+model_one_downstream_sql = """
+select fun from {{ ref('model_one') }}
+"""
+
+models_versions_schema_yml = """
+
+models:
+    - name: model_one
+      description: "The first model"
+      versions:
+        - v: 1
+        - v: 2
+"""
+
+models_versions_defined_in_schema_yml = """
+models:
+    - name: model_one
+      description: "The first model"
+      versions:
+        - v: 1
+        - v: 2
+          defined_in: model_one_different
+"""
+
+models_versions_updated_schema_yml = """
+models:
+    - name: model_one
+      latest_version: 1
+      description: "The first model"
+      versions:
+        - v: 1
+        - v: 2
+          defined_in: model_one_different
+"""
+
+model_two_sql = """
+select 1 as notfun
+"""
+
+
+class TestVersionedModels:
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "model_one_v1.sql": model_one_sql,
+            "model_one.sql": model_one_sql,
+            "model_one_downstream.sql": model_one_downstream_sql,
+            "schema.yml": models_versions_schema_yml,
+        }
+
+    def test_pp_versioned_models(self, project):
+        results = run_dbt(["run"])
+        assert len(results) == 3
+
+        manifest = get_manifest(project.project_root)
+        model_one_node = manifest.nodes["model.test.model_one.v1"]
+        assert not model_one_node.is_latest_version
+        model_two_node = manifest.nodes["model.test.model_one.v2"]
+        assert model_two_node.is_latest_version
+        # assert unpinned ref points to latest version
+        model_one_downstream_node = manifest.nodes["model.test.model_one_downstream"]
+        assert model_one_downstream_node.depends_on.nodes == ["model.test.model_one.v2"]
+
+        # update schema.yml block - model_one is now 'defined_in: model_one_different'
+        rm_file(project.project_root, "models", "model_one.sql")
+        write_file(model_one_sql, project.project_root, "models", "model_one_different.sql")
+        write_file(
+            models_versions_defined_in_schema_yml, project.project_root, "models", "schema.yml"
+        )
+        results = run_dbt(["--partial-parse", "run"])
+        assert len(results) == 3
+
+        # update versions schema.yml block - latest_version from 2 to 1
+        write_file(
+            models_versions_updated_schema_yml, project.project_root, "models", "schema.yml"
+        )
+        # This is where the test was failings in a CI run with:
+        # relation \"test..._test_partial_parsing.model_one_downstream\" does not exist
+        # because in core/dbt/include/global_project/macros/materializations/models/view/view.sql
+        # "existing_relation" didn't actually exist by the time it gets to the rename of the
+        # existing relation.
+        (pathlib.Path(project.project_root) / "log_output").mkdir(parents=True, exist_ok=True)
+        results = run_dbt(
+            ["--partial-parse", "--log-format-file", "json", "--log-path", "log_output", "run"]
+        )
+        assert len(results) == 3
+
+        manifest = get_manifest(project.project_root)
+        model_one_node = manifest.nodes["model.test.model_one.v1"]
+        assert model_one_node.is_latest_version
+        model_two_node = manifest.nodes["model.test.model_one.v2"]
+        assert not model_two_node.is_latest_version
+        # assert unpinned ref points to latest version
+        model_one_downstream_node = manifest.nodes["model.test.model_one_downstream"]
+        assert model_one_downstream_node.depends_on.nodes == ["model.test.model_one.v1"]
+
+        # assert unpinned ref to latest-not-max version yields an "FYI" info-level log
+        log_output = read_file("log_output", "dbt.log").replace("\n", " ").replace("\\n", " ")
+        assert "UnpinnedRefNewVersionAvailable" in log_output
+
+        # update versioned model
+        write_file(model_two_sql, project.project_root, "models", "model_one_different.sql")
+        results = run_dbt(["--partial-parse", "run"])
+        assert len(results) == 3
+        manifest = get_manifest(project.project_root)
+        assert len(manifest.nodes) == 3
+
+        # create a new model_one in model_one.sql and re-parse
+        write_file(model_one_sql, project.project_root, "models", "model_one.sql")
+        with pytest.raises(DuplicateVersionedUnversionedError):
+            run_dbt(["parse"])