updated project to use poetry

README.md

@@ -89,4 +89,12 @@ For integration testing, please use `pytest`:
 ```
 source $(poetry env info --path)/bin/activate
 pytest tests/integration --cov
+```
+
+### Reinstalling virtual env
+
+```
+poetry env list
+poetry env remove marcin-project-4eO9IBzv-py3.10
+poetry install
 ```

pyproject.toml

@@ -14,6 +14,8 @@ pytest = "^8.0.1"
 pytest-cov = "^4.0.0"
 pytest-spark = "^0.6.0"
 chispa = "^0.9.2"
+databricks-sdk = "^0.20.0"
+#databricks-connect = "^14.3.0"
 
 [build-system]
 requires = ["poetry-core"]

src/marcin_project/main.py

@@ -2,12 +2,12 @@
 from marcin_project.functions import filter_taxis
 
 
-def get_taxis():
-  spark = SparkSession.builder.getOrCreate()
+def get_taxis(spark: SparkSession):
   return filter_taxis(spark.read.table("samples.nyctaxi.trips"))
 
 def main():
-  get_taxis().show(5)
+  spark = SparkSession.builder.getOrCreate()
+  get_taxis(spark).show(5)
 
 
 if __name__ == '__main__':

tests/how_to_run_options.txt

@@ -0,0 +1,6 @@
+With databricks connect, spark cannot be started in local mode.
+
+There are a couple of options for running unit and integration tests:
+1. Run unit and integration tests using databricks connect. This requires a cluster in Databricks workspace.
+2. Run unit tests using spark local and integration tests using Databricks job (e.g. created using sdk).
+3. Have different virtual environments for unit (no databricks-connect installed) and integration tests (databricks-connect installed).

tests/integration/main_test_dbx_connect.py

@@ -0,0 +1,26 @@
+# from databricks.connect import DatabricksSession
+# from marcin_project import main
+#
+# # doc: https://docs.databricks.com/en/dev-tools/databricks-connect/python/index.html
+#
+# # Create a new Databricks Connect session. If this fails,
+# # check that you have configured Databricks Connect correctly.
+# # See https://docs.databricks.com/dev-tools/databricks-connect.html
+#
+# # Take connection from .databrikcscfg file, DEFAULT profile)
+# # https://docs.databricks.com/dev-tools/databricks-connect-ref.html#requirements
+#
+# spark = DatabricksSession.builder.getOrCreate()
+#
+# #SparkSession.builder = DatabricksSession.builder.profile("DEFAULT")
+# #spark = SparkSession.builder.getOrCreate()
+#
+# # spark = DatabricksSession.builder.remote(
+# #    host=f"https://adb-8870486534760962.2.azuredatabricks.net/?o=8870486534760962",
+# #    token="dapi03fec0a64fcc088adc1a27864050a598-2",
+# #    cluster_id="0222-221408-a9yml4v"
+# # ).getOrCreate()
+#
+# def test_main():
+#     taxis = main.get_taxis(spark)
+#     assert taxis.count() > 5

tests/unit/main_test.py

@@ -1,9 +1,9 @@
 from marcin_project import functions
-
 from chispa.dataframe_comparer import *
-
 from pyspark.sql import SparkSession
-#spark_session = SparkSession.builder.getOrCreate()
+
+# instead of using pytest-spark
+#spark = SparkSession.builder.getOrCreate()
 
 def test_get_taxi(spark_session: SparkSession): # using pytest-spark
     schema = "trip_distance: double, fare_amount: double"