From 62ffdd578e8af66a8a3996aba19622203717882b Mon Sep 17 00:00:00 2001
From: mwojtyczka <marcin.wojtyczka@databricks.com>
Date: Fri, 23 Feb 2024 15:20:39 +0100
Subject: [PATCH] update

---
 .github/workflows/dev.yaml     |  4 ++--
 requirements-dev.txt           | 26 --------------------------
 tests/e2e/main_test.py         | 22 ++++++++++++++++++++++
 tests/integration/main_test.py |  6 +++---
 tests/unit/main_test.py        |  2 ++
 5 files changed, 29 insertions(+), 31 deletions(-)
 delete mode 100644 requirements-dev.txt
 create mode 100644 tests/e2e/main_test.py

diff --git a/.github/workflows/dev.yaml b/.github/workflows/dev.yaml
index 38d7c86..0ce34f9 100644
--- a/.github/workflows/dev.yaml
+++ b/.github/workflows/dev.yaml
@@ -92,7 +92,7 @@ jobs:
       # Install test dependencies
       - run: poetry install --only test
 
-      - run: databricks bundle build
+      - run: databricks bundle deploy
       - run: |
           source $(poetry env info --path)/bin/activate
           pytest tests/e2e --cov
@@ -132,7 +132,7 @@ jobs:
 
   # Validate, deploy, and then run the bundle.
   pipeline_update:
-    name: "Run pipeline"
+    name: "Run Jobs"
     runs-on: ubuntu-latest
 
     # Run the "deploy" job first.
diff --git a/requirements-dev.txt b/requirements-dev.txt
deleted file mode 100644
index 8053b1e..0000000
--- a/requirements-dev.txt
+++ /dev/null
@@ -1,26 +0,0 @@
-## requirements-dev.txt: dependencies for local development.
-##
-## For defining dependencies used by jobs in Databricks Workflows, see
-## https://docs.databricks.com/dev-tools/bundles/library-dependencies.html
-
-## pytest is the default package used for testing
-pytest
-
-## Dependencies for building wheel files
-setuptools
-wheel
-
-## databricks-connect can be used to run parts of this project locally.
-## See https://docs.databricks.com/dev-tools/databricks-connect.html.
-##
-## databricks-connect is automatically installed if you're using Databricks
-## extension for Visual Studio Code
-## (https://docs.databricks.com/dev-tools/vscode-ext/dev-tasks/databricks-connect.html).
-##
-## To manually install databricks-connect, either follow the instructions
-## at https://docs.databricks.com/dev-tools/databricks-connect.html
-## to install the package system-wide. Or uncomment the line below to install a
-## version of db-connect that corresponds to the Databricks Runtime version used
-## for this project.
-#
-databricks-connect>=13.3,<13.4
diff --git a/tests/e2e/main_test.py b/tests/e2e/main_test.py
new file mode 100644
index 0000000..106199b
--- /dev/null
+++ b/tests/e2e/main_test.py
@@ -0,0 +1,22 @@
+# this test should relay on already deployed jobs
+
+import os
+
+from databricks.sdk import WorkspaceClient
+from databricks.sdk.service.compute import Language
+
+
+CLUSTER_ID = os.getenv('DATABRICKS_TEST_CLUSTER_ID')
+
+
+# needs to run before: databricks bundle deploy --target dev
+def test_main():
+    w = WorkspaceClient()  # use DEFAULT profile
+    jobs = w.jobs.list(name="marcin_project_job")
+    job_id = next(job for job in jobs).job_id
+    w.jobs.run_now_and_wait(job_id)
+
+    ctx = w.command_execution.create(cluster_id=CLUSTER_ID, language=Language.SQL).result()
+    command = "SELECT * FROM samples.nyctaxi.trips LIMIT 5;"
+    results = w.command_execution.execute(cluster_id=CLUSTER_ID, command=command, context_id=ctx.id, language=Language.SQL).result()
+    assert len(results.results.data) == 5
diff --git a/tests/integration/main_test.py b/tests/integration/main_test.py
index fdceacc..d9b43ac 100644
--- a/tests/integration/main_test.py
+++ b/tests/integration/main_test.py
@@ -1,5 +1,6 @@
+# this test uses Databricks Connect
+
 from databricks.connect import DatabricksSession
-from pyspark.sql import SparkSession
 from marcin_project import main
 
 # Create a new Databricks Connect session. If this fails,
@@ -15,11 +16,10 @@
 
 # spark = DatabricksSession.builder.remote(
 #    host=f"https://adb-8870486534760962.2.azuredatabricks.net/?o=8870486534760962",
-#    token="dapi03fec0a64fcc088adc1a27864050a598-2",
+#    token="xxx",
 #    cluster_id="0222-221408-a9yml4v"
 # ).getOrCreate()
 
 def test_main():
     taxis = main.get_taxis(spark)
     assert taxis.count() > 5
-
diff --git a/tests/unit/main_test.py b/tests/unit/main_test.py
index 6a06fb3..840da0e 100644
--- a/tests/unit/main_test.py
+++ b/tests/unit/main_test.py
@@ -1,3 +1,5 @@
+# This test setup spark in local mode
+
 from marcin_project import functions
 from chispa.dataframe_comparer import *
 from pyspark.sql import SparkSession