From 4e8a8d5bb525be0d9830b3ab88c412db528d60aa Mon Sep 17 00:00:00 2001 From: Charles Turner <52199577+charles-turner-1@users.noreply.github.com> Date: Mon, 3 Feb 2025 11:51:09 +0800 Subject: [PATCH] ENH: Add flaky decorator for tests with remote resources to reduce cold start failure rate. (#700) * Add flaky decorator for tests with remote resources - should reduce incidence of cold start related test failures * Separate out expected failing tests from expected passing tests - the pytest.mark.flaky() decorator seems to work fine on my local machine but is causing CI issues - not quite sure why * Pin zarr < 3.0 --- ci/environment-upstream-dev.yml | 3 ++- ci/environment.yml | 3 ++- tests/test_cat.py | 1 + tests/test_core.py | 3 +++ tests/test_tutorial.py | 22 +++++++++++++++++++--- 5 files changed, 27 insertions(+), 5 deletions(-) diff --git a/ci/environment-upstream-dev.yml b/ci/environment-upstream-dev.yml index 335b5a16..af9a7514 100644 --- a/ci/environment-upstream-dev.yml +++ b/ci/environment-upstream-dev.yml @@ -8,6 +8,7 @@ dependencies: - codecov - dask>=2024.12 - fastprogress>=1.0.0 + - flaky >= 3.8.0 - gcsfs >=2024.12 - h5netcdf>=0.8.1 - ipython @@ -32,7 +33,7 @@ dependencies: - scipy - xarray-datatree - xgcm - - zarr>=2.10 + - zarr>=2.10,<3.0 - pip: - git+https://github.com/intake/intake.git - git+https://github.com/pydata/xarray.git diff --git a/ci/environment.yml b/ci/environment.yml index 4373d969..fabda53a 100644 --- a/ci/environment.yml +++ b/ci/environment.yml @@ -7,6 +7,7 @@ dependencies: - codecov - dask>=2024.12 - fastprogress>=1.0.0 + - flaky >= 3.8.0 - fsspec>=2024.12 - gcsfs >=2024.12 - h5netcdf>=0.8.1 @@ -29,5 +30,5 @@ dependencies: - scipy - xarray>=2024.10 - xarray-datatree - - zarr>=2.12 + - zarr>=2.12,<3.0 # - pytest-icdiff diff --git a/tests/test_cat.py b/tests/test_cat.py index bed00407..518a2f90 100644 --- a/tests/test_cat.py +++ b/tests/test_cat.py @@ -49,6 +49,7 @@ def test_assets_mutually_exclusive(): multi_variable_cat, ], ) +@pytest.mark.flaky(max_runs=3, min_passes=1) # Cold start related failures def test_esmcatmodel_load(file): cat = ESMCatalogModel.load(file) assert isinstance(cat, ESMCatalogModel) diff --git a/tests/test_core.py b/tests/test_core.py index 463df865..6a7ccbe6 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -71,6 +71,7 @@ def func_multivar(ds): (intake_esm.cat.ESMCatalogModel.load(cdf_cat_sample_cmip6), '.', None, None), ], ) +@pytest.mark.flaky(max_runs=3, min_passes=1) # Cold start related failures def test_catalog_init(capsys, obj, sep, read_csv_kwargs, columns_with_iterables): """Test that the catalog can be initialized.""" cat = intake.open_esm_datastore( @@ -355,6 +356,7 @@ def test_multi_variable_catalog_derived_cat(): (mixed_cat_sample_cmip6, dict(institution_id='BCC'), {}), ], ) +@pytest.mark.flaky(max_runs=3, min_passes=1) # Cold start related failures def test_to_dataset_dict(path, query, xarray_open_kwargs): cat = intake.open_esm_datastore(path) cat_sub = cat.search(**query) @@ -387,6 +389,7 @@ def test_to_dataset_dict(path, query, xarray_open_kwargs): ), ], ) +@pytest.mark.flaky(max_runs=3, min_passes=1) # Cold start related failures def test_to_datatree(path, query, xarray_open_kwargs): cat = intake.open_esm_datastore(path) cat_sub = cat.search(**query) diff --git a/tests/test_tutorial.py b/tests/test_tutorial.py index b098c8cf..91fbe25a 100644 --- a/tests/test_tutorial.py +++ b/tests/test_tutorial.py @@ -4,20 +4,27 @@ import intake_esm from intake_esm.tutorial import DEFAULT_CATALOGS -tutorial_cats = list(zip(DEFAULT_CATALOGS.keys(), DEFAULT_CATALOGS.values())) + [ - pytest.param('bad_key', 'bad_url', marks=pytest.mark.xfail) -] +tutorial_cats = list(zip(DEFAULT_CATALOGS.keys(), DEFAULT_CATALOGS.values())) @pytest.mark.parametrize('name,url', tutorial_cats) +@pytest.mark.flaky(max_runs=3, min_passes=1) # Cold start related failures def test_get_url(name, url): cat_url = intake_esm.tutorial.get_url(name) assert isinstance(cat_url, str) assert cat_url == url +@pytest.mark.xfail +def test_get_url_xfail(): + cat_url = intake_esm.tutorial.get_url('bad_key') + assert isinstance(cat_url, str) + assert 'bad_url' == 'bad_url' + + @pytest.mark.network @pytest.mark.parametrize('name,url', tutorial_cats) +@pytest.mark.flaky(max_runs=3, min_passes=1) # Cold start related failures def test_open_from_url(name, url): cat_url = intake_esm.tutorial.get_url(name) cat = intake.open_esm_datastore(cat_url) @@ -25,6 +32,15 @@ def test_open_from_url(name, url): assert cat == intake.open_esm_datastore(url) +@pytest.mark.network +@pytest.mark.xfail +def test_open_from_url_xfail(): + cat_url = intake_esm.tutorial.get_url('bad_url') + cat = intake.open_esm_datastore(cat_url) + assert isinstance(cat.esmcat, intake_esm.cat.ESMCatalogModel) + assert cat == intake.open_esm_datastore('bad_url') + + def test_get_available_cats(): cats = intake_esm.tutorial.get_available_cats() assert cats == list(DEFAULT_CATALOGS.keys())