update

.github/workflows/dev.yaml

@@ -92,7 +92,7 @@ jobs:
       # Install test dependencies
       - run: poetry install --only test
 
-      - run: databricks bundle build
+      - run: databricks bundle deploy
       - run: |
           source $(poetry env info --path)/bin/activate
           pytest tests/e2e --cov
@@ -132,7 +132,7 @@ jobs:
 
   # Validate, deploy, and then run the bundle.
   pipeline_update:
-    name: "Run pipeline"
+    name: "Run Jobs"
     runs-on: ubuntu-latest
 
     # Run the "deploy" job first.

tests/e2e/main_test.py

@@ -0,0 +1,22 @@
+# this test should relay on already deployed jobs
+
+import os
+
+from databricks.sdk import WorkspaceClient
+from databricks.sdk.service.compute import Language
+
+
+CLUSTER_ID = os.getenv('DATABRICKS_TEST_CLUSTER_ID')
+
+
+# needs to run before: databricks bundle deploy --target dev
+def test_main():
+    w = WorkspaceClient()  # use DEFAULT profile
+    jobs = w.jobs.list(name="marcin_project_job")
+    job_id = next(job for job in jobs).job_id
+    w.jobs.run_now_and_wait(job_id)
+
+    ctx = w.command_execution.create(cluster_id=CLUSTER_ID, language=Language.SQL).result()
+    command = "SELECT * FROM samples.nyctaxi.trips LIMIT 5;"
+    results = w.command_execution.execute(cluster_id=CLUSTER_ID, command=command, context_id=ctx.id, language=Language.SQL).result()
+    assert len(results.results.data) == 5

tests/integration/main_test.py

@@ -1,5 +1,6 @@
+# this test uses Databricks Connect
+
 from databricks.connect import DatabricksSession
-from pyspark.sql import SparkSession
 from marcin_project import main
 
 # Create a new Databricks Connect session. If this fails,
@@ -15,11 +16,10 @@
 
 # spark = DatabricksSession.builder.remote(
 #    host=f"https://adb-8870486534760962.2.azuredatabricks.net/?o=8870486534760962",
-#    token="dapi03fec0a64fcc088adc1a27864050a598-2",
+#    token="xxx",
 #    cluster_id="0222-221408-a9yml4v"
 # ).getOrCreate()
 
 def test_main():
     taxis = main.get_taxis(spark)
     assert taxis.count() > 5
-

tests/unit/main_test.py

@@ -1,3 +1,5 @@
+# This test setup spark in local mode
+
 from marcin_project import functions
 from chispa.dataframe_comparer import *
 from pyspark.sql import SparkSession