Enable spark tests.

Verifying if problems I'm having with testing with spark are a local installation problem of a fundamental problem with spark behavior with this implementation. Local runs give some mysterious problems with spark that don't happen with dask.
mspass-team · Nov 17, 2023 · 8350dad · 8350dad
1 parent 2eeddc7
commit 8350dad
Showing 1 changed file with 6 additions and 7 deletions.
diff --git a/python/tests/io/test_distributed.py b/python/tests/io/test_distributed.py
@@ -314,7 +314,6 @@ def SeismogramEnsemble_generator():
 
 
 @pytest.mark.parametrize("scheduler,collection", 
-                         #[("spark","wf_TimeSeries")])
                          [("dask","wf_TimeSeries"),
                           ("dask","wf_Seismogram"),
                           ("spark","wf_TimeSeries"),
@@ -586,8 +585,8 @@ def set_dir_dfile_ensemble(d):
 @pytest.mark.parametrize("scheduler,collection", 
                          [("dask","wf_TimeSeries"),
                           ("dask","wf_Seismogram"),
-                          #("spark","wf_TimeSeries"),
-                          #("spark","wf_Seismogram")
+                          ("spark","wf_TimeSeries"),
+                          ("spark","wf_Seismogram")
                           ])
 def test_write_distributed_atomic(atomic_time_series_generator,atomic_seismogram_generator,scheduler,collection):
     """
@@ -925,8 +924,8 @@ def get_srclist_by_tag(db,data_tag)->list:
 @pytest.mark.parametrize("scheduler,collection", 
                          [("dask","wf_TimeSeries"),
                           ("dask","wf_Seismogram"),
- #                         ("spark","wf_TimeSeries"),
- #                         ("spark","wf_Seismogram")
+                          ("spark","wf_TimeSeries"),
+                          ("spark","wf_Seismogram")
                           ])
 def test_read_distributed_ensemble(TimeSeriesEnsemble_generator,SeismogramEnsemble_generator,scheduler,collection):
      print("Starting test with scheduler=",scheduler, " and collection=",collection)
@@ -1073,8 +1072,8 @@ def test_read_distributed_ensemble(TimeSeriesEnsemble_generator,SeismogramEnsemb
 @pytest.mark.parametrize("scheduler,collection", 
                         [ ("dask","wf_TimeSeries"),
                          ("dask","wf_Seismogram"),
-                         #("spark","wf_TimeSeries"),
-                         #("spark","wf_Seismogram"),
+                         ("spark","wf_TimeSeries"),
+                         ("spark","wf_Seismogram"),
                      ])
 def test_write_distributed_ensemble(TimeSeriesEnsemble_generator,SeismogramEnsemble_generator,scheduler,collection):
     """