ENH: Add flaky decorator for tests with remote resources to reduce co…

…ld start failure rate. (#700)

* Add flaky decorator for tests with remote resources - should reduce incidence of cold start related test failures

* Separate out expected failing tests from expected passing tests - the
pytest.mark.flaky() decorator seems to work fine on my local machine but
is causing CI issues - not quite sure why

* Pin zarr < 3.0

ci/environment-upstream-dev.yml

@@ -8,6 +8,7 @@ dependencies:
   - codecov
   - dask>=2024.12
   - fastprogress>=1.0.0
+  - flaky >= 3.8.0
   - gcsfs >=2024.12
   - h5netcdf>=0.8.1
   - ipython
@@ -32,7 +33,7 @@ dependencies:
   - scipy
   - xarray-datatree
   - xgcm
-  - zarr>=2.10
+  - zarr>=2.10,<3.0
   - pip:
       - git+https://github.com/intake/intake.git
       - git+https://github.com/pydata/xarray.git

ci/environment.yml

@@ -7,6 +7,7 @@ dependencies:
   - codecov
   - dask>=2024.12
   - fastprogress>=1.0.0
+  - flaky >= 3.8.0
   - fsspec>=2024.12
   - gcsfs >=2024.12
   - h5netcdf>=0.8.1
@@ -29,5 +30,5 @@ dependencies:
   - scipy
   - xarray>=2024.10
   - xarray-datatree
-  - zarr>=2.12
+  - zarr>=2.12,<3.0
   # - pytest-icdiff

tests/test_cat.py

@@ -49,6 +49,7 @@ def test_assets_mutually_exclusive():
         multi_variable_cat,
     ],
 )
+@pytest.mark.flaky(max_runs=3, min_passes=1)  # Cold start related failures
 def test_esmcatmodel_load(file):
     cat = ESMCatalogModel.load(file)
     assert isinstance(cat, ESMCatalogModel)

tests/test_core.py

@@ -71,6 +71,7 @@ def func_multivar(ds):
         (intake_esm.cat.ESMCatalogModel.load(cdf_cat_sample_cmip6), '.', None, None),
     ],
 )
+@pytest.mark.flaky(max_runs=3, min_passes=1)  # Cold start related failures
 def test_catalog_init(capsys, obj, sep, read_csv_kwargs, columns_with_iterables):
     """Test that the catalog can be initialized."""
     cat = intake.open_esm_datastore(
@@ -355,6 +356,7 @@ def test_multi_variable_catalog_derived_cat():
         (mixed_cat_sample_cmip6, dict(institution_id='BCC'), {}),
     ],
 )
+@pytest.mark.flaky(max_runs=3, min_passes=1)  # Cold start related failures
 def test_to_dataset_dict(path, query, xarray_open_kwargs):
     cat = intake.open_esm_datastore(path)
     cat_sub = cat.search(**query)
@@ -387,6 +389,7 @@ def test_to_dataset_dict(path, query, xarray_open_kwargs):
         ),
     ],
 )
+@pytest.mark.flaky(max_runs=3, min_passes=1)  # Cold start related failures
 def test_to_datatree(path, query, xarray_open_kwargs):
     cat = intake.open_esm_datastore(path)
     cat_sub = cat.search(**query)

tests/test_tutorial.py

@@ -4,27 +4,43 @@
 import intake_esm
 from intake_esm.tutorial import DEFAULT_CATALOGS
 
-tutorial_cats = list(zip(DEFAULT_CATALOGS.keys(), DEFAULT_CATALOGS.values())) + [
-    pytest.param('bad_key', 'bad_url', marks=pytest.mark.xfail)
-]
+tutorial_cats = list(zip(DEFAULT_CATALOGS.keys(), DEFAULT_CATALOGS.values()))
 
 
 @pytest.mark.parametrize('name,url', tutorial_cats)
+@pytest.mark.flaky(max_runs=3, min_passes=1)  # Cold start related failures
 def test_get_url(name, url):
     cat_url = intake_esm.tutorial.get_url(name)
     assert isinstance(cat_url, str)
     assert cat_url == url
 
 
+@pytest.mark.xfail
+def test_get_url_xfail():
+    cat_url = intake_esm.tutorial.get_url('bad_key')
+    assert isinstance(cat_url, str)
+    assert 'bad_url' == 'bad_url'
+
+
 @pytest.mark.network
 @pytest.mark.parametrize('name,url', tutorial_cats)
+@pytest.mark.flaky(max_runs=3, min_passes=1)  # Cold start related failures
 def test_open_from_url(name, url):
     cat_url = intake_esm.tutorial.get_url(name)
     cat = intake.open_esm_datastore(cat_url)
     assert isinstance(cat.esmcat, intake_esm.cat.ESMCatalogModel)
     assert cat == intake.open_esm_datastore(url)
 
 
+@pytest.mark.network
+@pytest.mark.xfail
+def test_open_from_url_xfail():
+    cat_url = intake_esm.tutorial.get_url('bad_url')
+    cat = intake.open_esm_datastore(cat_url)
+    assert isinstance(cat.esmcat, intake_esm.cat.ESMCatalogModel)
+    assert cat == intake.open_esm_datastore('bad_url')
+
+
 def test_get_available_cats():
     cats = intake_esm.tutorial.get_available_cats()
     assert cats == list(DEFAULT_CATALOGS.keys())