Migrate path translation tests to acceptance tests

bundle/tests/path_translation/nominal/databricks.yml → acceptance/bundle/paths/fallback/databricks.yml

@@ -1,5 +1,5 @@
 bundle:
-  name: path_translation_nominal
+  name: fallback
 
 include:
   - "resources/*.yml"

acceptance/bundle/paths/fallback/output.development.json

@@ -0,0 +1,125 @@
+{
+  "jobs": {
+    "my_job": {
+      "deployment": {
+        "kind": "BUNDLE",
+        "metadata_file_path": "/Workspace/Users/[email protected]/.bundle/fallback/development/state/metadata.json"
+      },
+      "edit_mode": "UI_LOCKED",
+      "format": "MULTI_TASK",
+      "job_clusters": [
+        {
+          "job_cluster_key": "default",
+          "new_cluster": {
+            "spark_version": "15.4.x-scala2.12"
+          }
+        }
+      ],
+      "name": "placeholder",
+      "permissions": [],
+      "queue": {
+        "enabled": true
+      },
+      "tags": {},
+      "tasks": [
+        {
+          "job_cluster_key": "default",
+          "notebook_task": {
+            "notebook_path": "/Workspace/Users/[email protected]/.bundle/fallback/development/files/src/notebook"
+          },
+          "task_key": "notebook_example"
+        },
+        {
+          "job_cluster_key": "default",
+          "spark_python_task": {
+            "python_file": "/Workspace/Users/[email protected]/.bundle/fallback/development/files/src/file.py"
+          },
+          "task_key": "spark_python_example"
+        },
+        {
+          "dbt_task": {
+            "commands": [
+              "dbt run",
+              "dbt run"
+            ],
+            "project_directory": "/Workspace/Users/[email protected]/.bundle/fallback/development/files/src/dbt_project"
+          },
+          "job_cluster_key": "default",
+          "task_key": "dbt_example"
+        },
+        {
+          "job_cluster_key": "default",
+          "sql_task": {
+            "file": {
+              "path": "/Workspace/Users/[email protected]/.bundle/fallback/development/files/src/sql.sql"
+            },
+            "warehouse_id": "cafef00d"
+          },
+          "task_key": "sql_example"
+        },
+        {
+          "job_cluster_key": "default",
+          "libraries": [
+            {
+              "whl": "dist/wheel1.whl"
+            },
+            {
+              "whl": "dist/wheel2.whl"
+            }
+          ],
+          "python_wheel_task": {
+            "package_name": "my_package"
+          },
+          "task_key": "python_wheel_example"
+        },
+        {
+          "job_cluster_key": "default",
+          "libraries": [
+            {
+              "jar": "target/jar1.jar"
+            },
+            {
+              "jar": "target/jar2.jar"
+            }
+          ],
+          "spark_jar_task": {
+            "main_class_name": "com.example.Main"
+          },
+          "task_key": "spark_jar_example"
+        }
+      ]
+    }
+  },
+  "pipelines": {
+    "my_pipeline": {
+      "deployment": {
+        "kind": "BUNDLE",
+        "metadata_file_path": "/Workspace/Users/[email protected]/.bundle/fallback/development/state/metadata.json"
+      },
+      "libraries": [
+        {
+          "file": {
+            "path": "/Workspace/Users/[email protected]/.bundle/fallback/development/files/src/file1.py"
+          }
+        },
+        {
+          "notebook": {
+            "path": "/Workspace/Users/[email protected]/.bundle/fallback/development/files/src/notebook1"
+          }
+        },
+        {
+          "file": {
+            "path": "/Workspace/Users/[email protected]/.bundle/fallback/development/files/src/file2.py"
+          }
+        },
+        {
+          "notebook": {
+            "path": "/Workspace/Users/[email protected]/.bundle/fallback/development/files/src/notebook2"
+          }
+        }
+      ],
+      "name": "placeholder",
+      "permissions": []
+    }
+  }
+}

acceptance/bundle/paths/fallback/output.txt

@@ -0,0 +1,76 @@
+
+>>> $CLI bundle validate -t development -o json
+
+Exit code: 0
+
+>>> $CLI bundle validate -t error
+Error: notebook this value is overridden not found. Local notebook references are expected
+to contain one of the following file extensions: [.py, .r, .scala, .sql, .ipynb]
+
+Name: fallback
+Target: error
+Workspace:
+  User: [email protected]
+  Path: /Workspace/Users/[email protected]/.bundle/fallback/error
+
+Found 1 error
+
+Exit code: 1
+
+>>> jq -e 
+.jobs.my_job.tasks[0].task_key == "notebook_example" and
+(.jobs.my_job.tasks[0].notebook_task.notebook_path | endswith("/src/notebook"))
+ output.development.json
+true
+
+>>> jq -e 
+.jobs.my_job.tasks[1].task_key == "spark_python_example" and
+(.jobs.my_job.tasks[1].spark_python_task.python_file | endswith("/src/file.py"))
+ output.development.json
+true
+
+>>> jq -e 
+.jobs.my_job.tasks[2].task_key == "dbt_example" and
+(.jobs.my_job.tasks[2].dbt_task.project_directory | endswith("/src/dbt_project"))
+ output.development.json
+true
+
+>>> jq -e 
+.jobs.my_job.tasks[3].task_key == "sql_example" and
+(.jobs.my_job.tasks[3].sql_task.file.path | endswith("/src/sql.sql"))
+ output.development.json
+true
+
+>>> jq -e 
+.jobs.my_job.tasks[4].task_key == "python_wheel_example" and
+(.jobs.my_job.tasks[4].libraries[0].whl == "dist/wheel1.whl") and
+(.jobs.my_job.tasks[4].libraries[1].whl == "dist/wheel2.whl")
+ output.development.json
+true
+
+>>> jq -e 
+.jobs.my_job.tasks[5].task_key == "spark_jar_example" and
+(.jobs.my_job.tasks[5].libraries[0].jar == "target/jar1.jar") and
+(.jobs.my_job.tasks[5].libraries[1].jar == "target/jar2.jar")
+ output.development.json
+true
+
+>>> jq -e 
+(.pipelines.my_pipeline.libraries[0].file.path | endswith("/src/file1.py"))
+ output.development.json
+true
+
+>>> jq -e 
+(.pipelines.my_pipeline.libraries[1].notebook.path | endswith("/src/notebook1"))
+ output.development.json
+true
+
+>>> jq -e 
+(.pipelines.my_pipeline.libraries[2].file.path | endswith("/src/file2.py"))
+ output.development.json
+true
+
+>>> jq -e 
+(.pipelines.my_pipeline.libraries[3].notebook.path | endswith("/src/notebook2"))
+ output.development.json
+true

bundle/tests/path_translation/fallback/resources/my_job.yml → acceptance/bundle/paths/fallback/resources/my_job.yml

@@ -4,33 +4,45 @@ resources:
       name: "placeholder"
       tasks:
         - task_key: notebook_example
+          job_cluster_key: default
           notebook_task:
             notebook_path: "this value is overridden"
 
         - task_key: spark_python_example
+          job_cluster_key: default
           spark_python_task:
             python_file: "this value is overridden"
 
         - task_key: dbt_example
+          job_cluster_key: default
           dbt_task:
             project_directory: "this value is overridden"
             commands:
               - "dbt run"
 
         - task_key: sql_example
+          job_cluster_key: default
           sql_task:
             file:
               path: "this value is overridden"
             warehouse_id: cafef00d
 
         - task_key: python_wheel_example
+          job_cluster_key: default
           python_wheel_task:
             package_name: my_package
           libraries:
             - whl: ../dist/wheel1.whl
 
         - task_key: spark_jar_example
+          job_cluster_key: default
           spark_jar_task:
             main_class_name: com.example.Main
           libraries:
             - jar: ../target/jar1.jar
+
+      # Include a job cluster for completeness
+      job_clusters:
+        - job_cluster_key: default
+          new_cluster:
+            spark_version: 15.4.x-scala2.12

acceptance/bundle/paths/fallback/script

@@ -0,0 +1,54 @@
+errcode trace $CLI bundle validate -t development -o json | jq '.resources' > output.development.json
+errcode trace $CLI bundle validate -t error
+
+# Job assertions
+
+trace jq -e '
+.jobs.my_job.tasks[0].task_key == "notebook_example" and
+(.jobs.my_job.tasks[0].notebook_task.notebook_path | endswith("/src/notebook"))
+' output.development.json
+
+trace jq -e '
+.jobs.my_job.tasks[1].task_key == "spark_python_example" and
+(.jobs.my_job.tasks[1].spark_python_task.python_file | endswith("/src/file.py"))
+' output.development.json
+
+trace jq -e '
+.jobs.my_job.tasks[2].task_key == "dbt_example" and
+(.jobs.my_job.tasks[2].dbt_task.project_directory | endswith("/src/dbt_project"))
+' output.development.json
+
+trace jq -e '
+.jobs.my_job.tasks[3].task_key == "sql_example" and
+(.jobs.my_job.tasks[3].sql_task.file.path | endswith("/src/sql.sql"))
+' output.development.json
+
+trace jq -e '
+.jobs.my_job.tasks[4].task_key == "python_wheel_example" and
+(.jobs.my_job.tasks[4].libraries[0].whl == "dist/wheel1.whl") and
+(.jobs.my_job.tasks[4].libraries[1].whl == "dist/wheel2.whl")
+' output.development.json
+
+trace jq -e '
+.jobs.my_job.tasks[5].task_key == "spark_jar_example" and
+(.jobs.my_job.tasks[5].libraries[0].jar == "target/jar1.jar") and
+(.jobs.my_job.tasks[5].libraries[1].jar == "target/jar2.jar")
+' output.development.json
+
+# Pipeline assertions
+
+trace jq -e '
+(.pipelines.my_pipeline.libraries[0].file.path | endswith("/src/file1.py"))
+' output.development.json
+
+trace jq -e '
+(.pipelines.my_pipeline.libraries[1].notebook.path | endswith("/src/notebook1"))
+' output.development.json
+
+trace jq -e '
+(.pipelines.my_pipeline.libraries[2].file.path | endswith("/src/file2.py"))
+' output.development.json
+
+trace jq -e '
+(.pipelines.my_pipeline.libraries[3].notebook.path | endswith("/src/notebook2"))
+' output.development.json

bundle/tests/path_translation/fallback/databricks.yml → acceptance/bundle/paths/nominal/databricks.yml

@@ -1,5 +1,5 @@
 bundle:
-  name: path_translation_fallback
+  name: nominal
 
 include:
   - "resources/*.yml"