Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Fix for Thredds v5 output update (#101)
For Thredds v5 update in bird-house/birdhouse-deploy#413 To fix output change: ``` _ PAVICS-landing-master/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-1DataAccess.ipynb::Cell 0 _ Notebook cell execution failed Cell 0: Cell outputs differ Input: from siphon.catalog import TDSCatalog url = "https://boreas.ouranos.ca/twitcher/ows/proxy/thredds/catalog/datasets/simulations/bias_adjusted/cmip6/ouranos/ESPO-G/ESPO-G6-R2v1.0.0/catalog.xml" # TEST_USE_PROD_DATA # Create Catalog cat = TDSCatalog(url) # List of datasets print(f"Number of datasets: {len(cat.datasets)}") # Access mechanisms - here we are interested in OPENDAP, a data streaming protocol cds = cat.datasets[0] print(f"Access URLs: {tuple(cds.access_urls.keys())}") Traceback: mismatch 'stdout' assert reference_output == test_output failed: "Number of da...cdfSubset')\n" == "Number of da...cdfSubset')\n" Skipping 43 identical leading characters in diff, use -v to show Skipping 50 identical trailing characters in diff, use -v to show - erver', 'OpenDAP', 'NC ? ^^^ + erver', 'OPENDAP', 'NC ? ^^^ _ PAVICS-landing-master/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-1DataAccess.ipynb::Cell 2 _ Notebook cell execution failed Cell 2: Cell outputs differ Input: # Extract a subset of the file # Again, this only creates an in-memory representation of the data sub = ds.tasmin.sel(time="2050").isel(rlon=400, rlat=350) # The data is only downloaded when we actually need it for a computation. sub.mean(keep_attrs=True).compute() Traceback: mismatch 'text/plain' assert reference_output == test_output failed: '<xarray.Data...60 50 50]' == '<xarray.Data...60 50 50]' Skipping 38 identical leading characters in diff, use -v to show Skipping 225 identical trailing characters in diff, use -v to show B - array(279.47516, dtype=float32) ? ^ - ^^ + array(278.72336, dtype=float32) ? ^ ^^^ Coordinates: rlat float32 4B -14.67 rlon float32 4B 360.6 rotated_pole float32 4B 9.969e+36 lat float32 4B 43.57 lon float32 4B -91.6 Attributes: - long_name: Minimal daily temperature cell_methods: time: minimum within days description: Daily minimal temperature as... grid_mapping: rotated_pole history: [DATE_TIME] Data c... + long_name: Minimal daily temperature stan _ PAVICS-landing-master/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-1DataAccess.ipynb::Cell 3 _ Notebook cell execution failed Cell 3: Cell outputs differ Input: ssp245_data = [cat.datasets[x] for x in cat.datasets if "ssp245" in x] ssp245_data Traceback: mismatch 'text/plain' assert reference_output == test_output failed: '[day_ESPO-G6...1001231.ncml]' == '[day_ESPO-G6...1001231.ncml]' Skipping 35 identical leading characters in diff, use -v to show - ioMIP_NAM_AS-RCEC_TaiESM1_ssp245_r1i1p1f1_19500101-21001231.ncml, ? ^ ^^^^^ ^^^ ^ + ioMIP_NAM_NUIST_NESM3_ssp245_r1i1p1f1_19500101-21001231.ncml, ? ^^^ ^ ^ ^ + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_NOAA-GFDL_GFDL-ESM4_ssp245_r1i1p1f1_19500101-21001231.ncml, + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_NIMS-KMA_KACE-1-0-G_ssp245_r1i1p1f1_19500101-21001230.ncml, + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_NCC_NorESM2-MM_ssp245_r1i1p1f1_19500101-21001231.ncml, + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_NCC_NorESM2-LM_ssp245_r1i1p1f1_19500101-21001231.ncml, + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_MRI_MRI-ESM2-0_ssp245_r1i1p1f1_19500101-21001231.ncml, + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_MPI-M_MPI-ESM1-2-LR_ssp245_r1i1p1f1_19500101-21001231.ncml, + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_MPI-M_MPI-ESM1-2-HR_ssp245_r1i1p1f1_19500101-21001231.ncml, + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_MOHC_UKESM1-0-LL_ssp245_r1i1p1f2_19500101-21001230.ncml, + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_MIROC_MIROC6_ssp245_r1i1p1f1_19500101-21001231.ncml, + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_MIROC_MIROC-ES2L_ssp245_r1i1p1f2_19500101-21001231.ncml, + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_IPSL_IPSL-CM6A-LR_ssp245_r1i1p1f1_19500101-21001231.ncml, + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_INM_INM-CM5-0_ssp245_r1i1p1f1_19500101-21001231.ncml, + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_INM_INM-CM4-8_ssp245_r1i1p1f1_19500101-21001231.ncml, + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_EC-Earth-Consortium_EC-Earth3_ssp245_r1i1p1f1_19500101-21001231.ncml, + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_EC-Earth-Consortium_EC-Earth3-Veg_ssp245_r1i1p1f1_19500101-21001231.ncml, + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_EC-Earth-Consortium_EC-Earth3-CC_ssp245_r1i1p1f1_19500101-21001231.ncml, + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_CSIRO_ACCESS-ESM1-5_ssp245_r1i1p1f1_19500101-21001231.ncml, + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_CSIRO-ARCCSS_ACCESS-CM2_ssp245_r1i1p1f1_19500101-21001231.ncml, + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_CNRM-CERFACS_CNRM-ESM2-1_ssp245_r1i1p1f2_19500101-21001231.ncml, + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_CNRM-CERFACS_CNRM-CM6-1_ssp245_r1i1p1f2_19500101-21001231.ncml, + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_CMCC_CMCC-ESM2_ssp245_r1i1p1f1_19500101-21001231.ncml, + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_CCCma_CanESM5_ssp245_r1i1p1f1_19500101-21001231.ncml, + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_CAS_FGOALS-g3_ssp245_r1i1p1f1_19500101-21001231.ncml, day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_BCC_BCC-CSM2-MR_ssp245_r1i1p1f1_19500101-21001231.ncml, - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_CAS_FGOALS-g3_ssp245_r1i1p1f1_19500101-21001231.ncml, - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_CCCma_CanESM5_ssp245_r1i1p1f1_19500101-21001231.ncml, - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_CMCC_CMCC-ESM2_ssp245_r1i1p1f1_19500101-21001231.ncml, - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_CNRM-CERFACS_CNRM-CM6-1_ssp245_r1i1p1f2_19500101-21001231.ncml, - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_CNRM-CERFACS_CNRM-ESM2-1_ssp245_r1i1p1f2_19500101-21001231.ncml, - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_CSIRO-ARCCSS_ACCESS-CM2_ssp245_r1i1p1f1_19500101-21001231.ncml, - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_CSIRO_ACCESS-ESM1-5_ssp245_r1i1p1f1_19500101-21001231.ncml, - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_EC-Earth-Consortium_EC-Earth3-CC_ssp245_r1i1p1f1_19500101-21001231.ncml, - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_EC-Earth-Consortium_EC-Earth3-Veg_ssp245_r1i1p1f1_19500101-21001231.ncml, - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_EC-Earth-Consortium_EC-Earth3_ssp245_r1i1p1f1_19500101-21001231.ncml, - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_INM_INM-CM4-8_ssp245_r1i1p1f1_19500101-21001231.ncml, - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_INM_INM-CM5-0_ssp245_r1i1p1f1_19500101-21001231.ncml, - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_IPSL_IPSL-CM6A-LR_ssp245_r1i1p1f1_19500101-21001231.ncml, - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_MIROC_MIROC-ES2L_ssp245_r1i1p1f2_19500101-21001231.ncml, - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_MIROC_MIROC6_ssp245_r1i1p1f1_19500101-21001231.ncml, - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_MOHC_UKESM1-0-LL_ssp245_r1i1p1f2_19500101-21001230.ncml, - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_MPI-M_MPI-ESM1-2-HR_ssp245_r1i1p1f1_19500101-21001231.ncml, - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_MPI-M_MPI-ESM1-2-LR_ssp245_r1i1p1f1_19500101-21001231.ncml, - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_MRI_MRI-ESM2-0_ssp245_r1i1p1f1_19500101-21001231.ncml, - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_NCC_NorESM2-LM_ssp245_r1i1p1f1_19500101-21001231.ncml, - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_NCC_NorESM2-MM_ssp245_r1i1p1f1_19500101-21001231.ncml, - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_NIMS-KMA_KACE-1-0-G_ssp245_r1i1p1f1_19500101-21001230.ncml, - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_NOAA-GFDL_GFDL-ESM4_ssp245_r1i1p1f1_19500101-21001231.ncml, - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_NUIST_NESM3_ssp245_r1i1p1f1_19500101-21001231.ncml] ? ^^^ ^^ ^ + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_AS-RCEC_TaiESM1_ssp245_r1i1p1f1_19500101-21001231.ncml] ? ^ ++++++ ^^ ^ _ PAVICS-landing-master/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-1DataAccess.ipynb::Cell 4 _ Notebook cell execution failed Cell 4: Cell outputs differ Input: tcr_likely_models = [ "BCC-CSM2-MR", "FGOALS-g3", "CMCC-ESM2", "CNRM-ESM2-1", "ACCESS-CM2", "ACCESS-ESM1-5", "MPI-ESM1-2-HR", "INM-CM5-0", "MIROC6", "MPI-ESM1-2-LR", "MRI-ESM2-0", "NorESM2-LM", "KACE-1-0-G", "GFDL-ESM4", "MIROC-ES2L", ] def _filter_tcr_likely(files): return [d for d in files if any([h in d.name for h in tcr_likely_models])] # create a simple search sub-function def get_ncfilelist(scen=None, url=None, tcr_likely=False): cat = TDSCatalog(url) ncfiles = [cat.datasets[c] for c in cat.datasets if scen in c] if tcr_likely: expected = len(tcr_likely_models) ncfiles = _filter_tcr_likely(ncfiles) if len(ncfiles) == expected: display(f"Successfully found {expected} datasets for {scen}") return ncfiles else: raise ValueError( f"Expected number of datasets for {scen} is {expected} : found {len(ncfiles)}" ) datasets = {} for scen in ["ssp245", "ssp370"]: datasets[scen] = get_ncfilelist(scen=scen, url=url, tcr_likely=True) display(datasets["ssp245"]) display(datasets["ssp370"]) Traceback: mismatch 'text/plain' assert reference_output == test_output failed: '[day_ESPO-G6...1001231.ncml]' == '[day_ESPO-G6...1001231.ncml]' Skipping 35 identical leading characters in diff, use -v to show - ioMIP_NAM_BCC_BCC-CSM2-MR_ssp245_r1i1p1f1_19500101-21001231.ncml, ? ^^^ ^^^ ^ ^^^^ + ioMIP_NAM_NOAA-GFDL_GFDL-ESM4_ssp245_r1i1p1f1_19500101-21001231.ncml, ? ^^^^^^^^^ ^^^^ ^ ^ + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_NIMS-KMA_KACE-1-0-G_ssp245_r1i1p1f1_19500101-21001230.ncml, + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_NCC_NorESM2-LM_ssp245_r1i1p1f1_19500101-21001231.ncml, + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_MRI_MRI-ESM2-0_ssp245_r1i1p1f1_19500101-21001231.ncml, + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_MPI-M_MPI-ESM1-2-LR_ssp245_r1i1p1f1_19500101-21001231.ncml, + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_MPI-M_MPI-ESM1-2-HR_ssp245_r1i1p1f1_19500101-21001231.ncml, + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_MIROC_MIROC6_ssp245_r1i1p1f1_19500101-21001231.ncml, + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_MIROC_MIROC-ES2L_ssp245_r1i1p1f2_19500101-21001231.ncml, + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_INM_INM-CM5-0_ssp245_r1i1p1f1_19500101-21001231.ncml, + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_CSIRO_ACCESS-ESM1-5_ssp245_r1i1p1f1_19500101-21001231.ncml, + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_CSIRO-ARCCSS_ACCESS-CM2_ssp245_r1i1p1f1_19500101-21001231.ncml, + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_CNRM-CERFACS_CNRM-ESM2-1_ssp245_r1i1p1f2_19500101-21001231.ncml, + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_CMCC_CMCC-ESM2_ssp245_r1i1p1f1_19500101-21001231.ncml, day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_CAS_FGOALS-g3_ssp245_r1i1p1f1_19500101-21001231.ncml, - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_CMCC_CMCC-ESM2_ssp245_r1i1p1f1_19500101-21001231.ncml, ? ^^ ^^ ^ ^^ + day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_BCC_BCC-CSM2-MR_ssp245_r1i1p1f1_19500101-21001231.ncml] ? ^ ^ ^ +++ ^ - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_CNRM-CERFACS_CNRM-ESM2-1_ssp245_r1i1p1f2_19500101-21001231.ncml, - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_CSIRO-ARCCSS_ACCESS-CM2_ssp245_r1i1p1f1_19500101-21001231.ncml, - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_CSIRO_ACCESS-ESM1-5_ssp245_r1i1p1f1_19500101-21001231.ncml, - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_INM_INM-CM5-0_ssp245_r1i1p1f1_19500101-21001231.ncml, - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_MIROC_MIROC-ES2L_ssp245_r1i1p1f2_19500101-21001231.ncml, - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_MIROC_MIROC6_ssp245_r1i1p1f1_19500101-21001231.ncml, - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_MPI-M_MPI-ESM1-2-HR_ssp245_r1i1p1f1_19500101-21001231.ncml, - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_MPI-M_MPI-ESM1-2-LR_ssp245_r1i1p1f1_19500101-21001231.ncml, - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_MRI_MRI-ESM2-0_ssp245_r1i1p1f1_19500101-21001231.ncml, - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_NCC_NorESM2-LM_ssp245_r1i1p1f1_19500101-21001231.ncml, - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_NIMS-KMA_KACE-1-0-G_ssp245_r1i1p1f1_19500101-21001230.ncml, - day_ESPO-G6-R2_v1.0.0_CMIP6_ScenarioMIP_NAM_NOAA-GFDL_GFDL-ESM4_ssp245_r1i1p1f1_19500101-21001231.ncml] _ PAVICS-landing-fix-for-Thredds-v5-output-update/content/notebooks/climate_indicators/PAVICStutorial_ClimateDataAnalysis-3Climate-Indicators.ipynb::Cell 0 _ Notebook cell execution failed Cell 0: Cell outputs differ Input: import os os.environ["USE_PYGEOS"] = "0" # force use Shapely with GeoPandas import warnings import geopandas as gpd import matplotlib.pyplot as plt import numba import xarray as xr from clisops.core import subset from dask.diagnostics import ProgressBar from siphon.catalog import TDSCatalog from xclim import atmos warnings.simplefilter("ignore") # TODO change address url = "https://boreas.ouranos.ca/twitcher/ows/proxy/thredds/catalog/datasets/simulations/bias_adjusted/cmip6/ouranos/ESPO-G/ESPO-G6-R2v1.0.0/catalog.xml" # TEST_USE_PROD_DATA # Create Catalog cat = TDSCatalog(url) # Subset over the Gasp�� peninsula in eastern Quebec gaspe = gpd.GeoDataFrame.from_file( "/notebook_dir/pavics-homepage/tutorial_data/gaspesie_mrc.geojson" ) ds = subset.subset_shape( xr.open_dataset( cat.datasets[0].access_urls["OPENDAP"], chunks=dict(time=365 * 4, rlon=50, rlat=50), ), shape=gpd.GeoDataFrame(geometry=gaspe.buffer(0.05)), ) # What we see here is only a representation of the full content, the entire data set hasn't been loaded. display(ds) # plot of single day tasmin a = ds.tasmin.isel(time=0).plot(figsize=(10, 4)) Traceback: mismatch 'text/plain' assert reference_output == test_output failed: '<xarray.Data... EPSG:4326' == '<xarray.Data... EPSG:4326' Skipping 1172 identical leading characters in diff, use -v to show - tes: (12/83) ? ^ + tes: (12/81) ? ^ Conventions: CF-1.7 CMIP-6.2 Notes: Regridded on the grid of RDRS v2.1, then... activity_id: CMIP - branch_method: Hybrid-restart from year/DATE/of p... - branch_time: 0.0 ? -- ^^^^^ + branch_method: standard ? ++++ ^^^^^^^^ branch_time_in_child: 0.0 + branch_time_in_parent: 109573.0 ... ... license_type: permissive terms_of_use: In addition to the provided licence, the... attribution: Use of this dataset should be acknowledg... modeling_realm: atmos - source_institution: AS-RCEC ? ^ ^^^^^ + source_institution: NUIST ? ^^^ ^ crs: EPSG:4326 ```
- Loading branch information