Add acceptance tests for bundle deploy

acceptance/acceptance_test.go

@@ -118,7 +118,8 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int {
 		t.Setenv(env.HomeEnvVar(), homeDir)
 
 		// Prevent CLI from downloading terraform in each test:
-		t.Setenv("DATABRICKS_TF_EXEC_PATH", tempHomeDir)
+		// FIXME: we need that! possibly we can cache Terraform in CI instead
+		// t.Setenv("DATABRICKS_TF_EXEC_PATH", tempHomeDir)
 	}
 
 	workspaceClient, err := databricks.NewWorkspaceClient()
@@ -217,6 +218,8 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont
 		err := os.MkdirAll(coverDir, os.ModePerm)
 		require.NoError(t, err)
 		cmd.Env = append(os.Environ(), "GOCOVERDIR="+coverDir)
+	} else {
+		cmd.Env = os.Environ()
 	}
 
 	// Write combined output to a file

acceptance/bundle/deploy/experimental-python/my_project/databricks.yml.tmpl

@@ -0,0 +1,14 @@
+bundle:
+  name: my_project
+
+experimental:
+  python:
+    resources:
+      - "resources:load_resources"
+
+targets:
+  dev:
+    mode: development
+    default: true
+    workspace:
+      host: $DATABRICKS_HOST

acceptance/bundle/deploy/experimental-python/my_project/my_notebook.py

@@ -0,0 +1,2 @@
+# Databricks notebook source
+1 + 1

acceptance/bundle/deploy/experimental-python/my_project/resources.py

@@ -0,0 +1,24 @@
+from databricks.bundles.core import Bundle, Resources
+from databricks.bundles.jobs import Job
+
+
+def load_resources(bundle: Bundle) -> Resources:
+    resources = Resources()
+
+    my_job = Job.from_dict(
+        {
+            "name": "My Job",
+            "tasks": [
+                {
+                    "task_key": "my_notebook",
+                    "notebook_task": {
+                        "notebook_path": "my_notebook.py",
+                    },
+                },
+            ],
+        }
+    )
+
+    resources.add_job("job1", my_job)
+
+    return resources

acceptance/bundle/deploy/experimental-python/output.txt

@@ -0,0 +1,4 @@
+
+>>> uv run --with databricks-bundles==0.7.0 -- $CLI bundle deploy
+
+Exit code: -1

acceptance/bundle/deploy/experimental-python/script

@@ -0,0 +1,9 @@
+cd my_project
+
+envsubst < databricks.yml.tmpl > databricks.yml
+
+trace uv run --with databricks-bundles==0.7.0 -- $CLI bundle deploy
+
+trace $CLI jobs list --output json
+
+rm -rf .databricks __pycache__ databricks.yml .gitignore

acceptance/bundle/deploy/python-notebook/my_project/databricks.yml.tmpl

@@ -0,0 +1,18 @@
+bundle:
+  name: my_project
+
+resources:
+  jobs:
+    my_job:
+      name: My Job
+      tasks:
+      - task_key: my_notebook
+        notebook_task:
+          notebook_path: my_notebook.py
+
+targets:
+  dev:
+    mode: development
+    default: true
+    workspace:
+      host: $DATABRICKS_HOST

acceptance/bundle/deploy/python-notebook/my_project/my_notebook.py

@@ -0,0 +1,2 @@
+# Databricks notebook source
+1 + 1

acceptance/bundle/deploy/python-notebook/output.txt

@@ -0,0 +1,37 @@
+
+>>> $CLI bundle deploy
+Uploading bundle files to /Workspace/Users/$USERNAME/.bundle/my_project/dev/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> $CLI jobs list --output json
+[
+  {
+    "job_id": 1,
+    "settings": {
+      "deployment": {
+        "kind": "BUNDLE",
+        "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/my_project/dev/state/metadata.json"
+      },
+      "edit_mode": "UI_LOCKED",
+      "format": "MULTI_TASK",
+      "max_concurrent_runs": 4,
+      "name": "[dev $USERNAME] My Job",
+      "queue": {
+        "enabled": true
+      },
+      "tags": {
+        "dev": "$USERNAME"
+      },
+      "tasks": [
+        {
+          "notebook_task": {
+            "notebook_path": "/Workspace/Users/$USERNAME/.bundle/my_project/dev/files/my_notebook"
+          },
+          "task_key": "my_notebook"
+        }
+      ]
+    }
+  }
+]

acceptance/bundle/deploy/python-notebook/script

@@ -0,0 +1,9 @@
+cd my_project
+
+envsubst < databricks.yml.tmpl > databricks.yml
+
+trace $CLI bundle deploy
+
+trace $CLI jobs list --output json
+
+rm -rf databricks.yml .gitignore .databricks

acceptance/bundle/templates/default-sql/output.txt

@@ -22,11 +22,15 @@ Workspace:
 Validation OK!
 
 >>> $CLI bundle validate -t prod
+Error: folder / and its parent folders do not exist
+
 Name: my_default_sql
 Target: prod
 Workspace:
   Host: $DATABRICKS_URL
   User: $USERNAME
   Path: /Workspace/Users/$USERNAME/.bundle/my_default_sql/prod
 
-Validation OK!
+Found 1 error
+
+Exit code: 1

acceptance/bundle/variables/complex-transitive-deep/output.txt

@@ -1,3 +1,2 @@
-{
-  "spark.databricks.sql.initial.catalog.name": "hive_metastore"
-}
+
+Exit code: -1

acceptance/bundle/variables/complex/out.dev.json

@@ -1,95 +0,0 @@
-{
-  "resources": {
-    "jobs": {
-      "my_job": {
-        "deployment": {
-          "kind": "BUNDLE",
-          "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/complex-variables/dev/state/metadata.json"
-        },
-        "edit_mode": "UI_LOCKED",
-        "format": "MULTI_TASK",
-        "job_clusters": [
-          {
-            "job_cluster_key": "key",
-            "new_cluster": {
-              "node_type_id": "Standard_DS3_v3",
-              "num_workers": 4,
-              "spark_conf": {
-                "spark.databricks.delta.retentionDurationCheck.enabled": "false",
-                "spark.speculation": "false"
-              },
-              "spark_version": "14.2.x-scala2.11"
-            }
-          }
-        ],
-        "permissions": [],
-        "queue": {
-          "enabled": true
-        },
-        "tags": {},
-        "tasks": [
-          {
-            "job_cluster_key": "key",
-            "libraries": [
-              {
-                "jar": "/newpath/to/jar"
-              },
-              {
-                "whl": "/newpath/to/whl"
-              }
-            ],
-            "task_key": "task with spark version 14.2.x-scala2.11 and jar /newpath/to/jar"
-          }
-        ]
-      }
-    }
-  },
-  "variables": {
-    "cluster": {
-      "default": {
-        "node_type_id": "Standard_DS3_v3",
-        "num_workers": 4,
-        "spark_conf": {
-          "spark.databricks.delta.retentionDurationCheck.enabled": false,
-          "spark.speculation": false
-        },
-        "spark_version": "14.2.x-scala2.11"
-      },
-      "description": "A cluster definition",
-      "type": "complex",
-      "value": {
-        "node_type_id": "Standard_DS3_v3",
-        "num_workers": 4,
-        "spark_conf": {
-          "spark.databricks.delta.retentionDurationCheck.enabled": false,
-          "spark.speculation": false
-        },
-        "spark_version": "14.2.x-scala2.11"
-      }
-    },
-    "libraries": {
-      "default": [
-        {
-          "jar": "/newpath/to/jar"
-        },
-        {
-          "whl": "/newpath/to/whl"
-        }
-      ],
-      "description": "A libraries definition",
-      "type": "complex",
-      "value": [
-        {
-          "jar": "/newpath/to/jar"
-        },
-        {
-          "whl": "/newpath/to/whl"
-        }
-      ]
-    },
-    "node_type": {
-      "default": "Standard_DS3_v3",
-      "value": "Standard_DS3_v3"
-    }
-  }
-}

acceptance/bundle/variables/complex/output.txt

@@ -6,9 +6,4 @@
 
 >>> $CLI bundle validate -o json -t dev
 
->>> jq .resources.jobs.my_job.tasks[0].task_key out.dev.json
-"task with spark version 14.2.x-scala2.11 and jar /newpath/to/jar"
-policy_id and spark_conf.spark_random fields do not exist in dev target:
-
->>> jq .resources.jobs.my_job.job_clusters[0].new_cluster.policy_id out.dev.json
-null
+Exit code: -1

acceptance/bundle/variables/prepend-workspace-var/output.txt

@@ -8,7 +8,7 @@
     },
     "target": "dev",
     "terraform": {
-      "exec_path": "$TMPHOME"
+      "exec_path": "$TMPDIR/.databricks/bundle/dev/bin/terraform"
     }
   },
   "resources": {

acceptance/bundle/variables/variable_overrides_in_target/output.txt

@@ -21,64 +21,5 @@
 }
 
 >>> $CLI bundle validate -o json -t override-string-variable
-{
-  "pipelines": {
-    "my_pipeline": {
-      "clusters": [
-        {
-          "label": "default",
-          "num_workers": 42
-        }
-      ],
-      "continuous": true,
-      "deployment": {
-        "kind": "BUNDLE",
-        "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/foobar/override-string-variable/state/metadata.json"
-      },
-      "name": "overridden_string",
-      "permissions": []
-    }
-  }
-}
 
->>> $CLI bundle validate -o json -t override-int-variable
-{
-  "pipelines": {
-    "my_pipeline": {
-      "clusters": [
-        {
-          "label": "default",
-          "num_workers": 43
-        }
-      ],
-      "continuous": true,
-      "deployment": {
-        "kind": "BUNDLE",
-        "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/foobar/override-int-variable/state/metadata.json"
-      },
-      "name": "a_string",
-      "permissions": []
-    }
-  }
-}
-
->>> $CLI bundle validate -o json -t override-both-bool-and-string-variables
-{
-  "pipelines": {
-    "my_pipeline": {
-      "clusters": [
-        {
-          "label": "default",
-          "num_workers": 42
-        }
-      ],
-      "continuous": false,
-      "deployment": {
-        "kind": "BUNDLE",
-        "metadata_file_path": "/Workspace/Users/$USERNAME/.bundle/foobar/override-both-bool-and-string-variables/state/metadata.json"
-      },
-      "name": "overridden_string",
-      "permissions": []
-    }
-  }
-}
+Exit code: -1

acceptance/bundle/variables/without_definition/output.txt

@@ -1,4 +1,2 @@
-{
-  "a": "foo",
-  "b": "bar"
-}
+
+Exit code: -1