Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Xr unitsfix #65

Merged
merged 4 commits into from
Mar 18, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion tests/data_testcase_schemas.py
Original file line number Diff line number Diff line change
Expand Up @@ -511,6 +511,6 @@ def standard_testcase(request, session_testdir):
# We think Xarray can load ~anything (maybe returning nothing)
"load": [],
# Xarray can save ~anything
"save": [],
"save": [r"test_monotonic_coordinate"],
},
}
97 changes: 12 additions & 85 deletions tests/integration/test_xarray_load_and_save_equivalence.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,6 @@
(1) check equivalence of cubes : xarray.load(file) VS xarray.load(ncdata(file))
(2) check equivalence of files : xarray -> file VS xarray->ncdata->file
"""
from subprocess import check_output

import pytest
import xarray

Expand All @@ -17,17 +15,15 @@
session_testdir,
standard_testcase,
)
from tests.integration.equivalence_testing_utils import (
adjust_chunks,
set_tiny_chunks,
)

# Avoid complaints that imported fixtures are "unused"
standard_testcase, session_testdir, adjust_chunks

from ncdata.threadlock_sharing import lockshare_context
from ncdata.xarray import from_xarray, to_xarray

# Avoid complaints that imported fixtures are "unused"
# TODO: declare fixtures in usual way in pytest config?
standard_testcase, session_testdir


# _FIX_LOCKS = True
_FIX_LOCKS = False

Expand All @@ -41,75 +37,31 @@ def use_xarraylock():
yield


# _USE_TINY_CHUNKS = True
_USE_TINY_CHUNKS = False
set_tiny_chunks(_USE_TINY_CHUNKS)


def test_load_direct_vs_viancdata(
standard_testcase, use_xarraylock, adjust_chunks, tmp_path
standard_testcase, use_xarraylock, tmp_path
):
source_filepath = standard_testcase.filepath
ncdata = from_nc4(source_filepath)

if standard_testcase.name in BAD_LOADSAVE_TESTCASES["xarray"]["load"]:
excluded_testcases = BAD_LOADSAVE_TESTCASES["xarray"]["load"]
if any(key in standard_testcase.name for key in excluded_testcases):
pytest.skip("excluded testcase (xarray cannot load)")

# _Debug = True
_Debug = False
if _Debug:
print(f"\ntestcase: {standard_testcase.name}")
print("spec =")
print(standard_testcase.spec)
print("\nncdata =")
print(ncdata)
print("\nncdump =")
txt = check_output([f"ncdump {source_filepath}"], shell=True).decode()
print(txt)

# Load the testcase with Xarray.
xr_ds = xarray.open_dataset(source_filepath, chunks=-1)

# Load same, via ncdata
xr_ncdata_ds = to_xarray(ncdata)

# Xarray dataset (variable) comparison is problematic
# result = xr_ncdata_ds.identical(xr_ds)

# So for now, save Xarray datasets to disk + compare that way.
temp_xr_path = tmp_path / "tmp_out_xr.nc"
temp_xr_ncdata_path = tmp_path / "tmp_out_xr_ncdata.nc"
xr_ds.to_netcdf(temp_xr_path)
xr_ncdata_ds.to_netcdf(temp_xr_ncdata_path)

if _Debug:
print("\n\n-----\nResult ncdump : 'DIRECT' nc4 -> xr -> nc4 ... ")
txt = check_output([f"ncdump {temp_xr_path}"], shell=True).decode()
print(txt)
print(
"\n\n-----\nResult ncdump : 'INDIRECT'' nc4 -> ncdata-> xr -> nc4 ... "
)
txt = check_output(
[f"ncdump {temp_xr_ncdata_path}"], shell=True
).decode()
print(txt)

# FOR NOW: compare with experimental ncdata comparison.
# I know this is a bit circular, but it is useful for debugging, for now ...
result = compare_nc_datasets(
temp_xr_path,
temp_xr_ncdata_path,
check_dims_order=False,
suppress_warnings=True,
)
if result != []:
assert result == []
# Treat as OK if it passes xarray comparison
assert xr_ds.identical(xr_ncdata_ds)


def test_save_direct_vs_viancdata(standard_testcase, tmp_path):
source_filepath = standard_testcase.filepath
ncdata = from_nc4(source_filepath)

excluded_testcases = BAD_LOADSAVE_TESTCASES["xarray"]["load"]
excluded_testcases.extend(BAD_LOADSAVE_TESTCASES["xarray"]["save"])
if any(key in standard_testcase.name for key in excluded_testcases):
pytest.skip("excluded testcase")

Expand All @@ -124,31 +76,6 @@ def test_save_direct_vs_viancdata(standard_testcase, tmp_path):
ncds_fromxr = from_xarray(xrds)
to_nc4(ncds_fromxr, temp_ncdata_savepath)

# _Debug = True
_Debug = False
if _Debug:
ncdump_opts = "-h"
# ncdump_opts = ""
txt = f"""
testcase: {standard_testcase.name}
spec = {standard_testcase.spec}
ncdata = ...
{ncdata}
ncdump ORIGINAL TESTCASE SOURCEFILE =
"""
txt += check_output(
[f"ncdump {ncdump_opts} {source_filepath}"], shell=True
).decode()
txt += "\nncdump DIRECT FROM XARRAY ="
txt += check_output(
[f"ncdump {ncdump_opts} {temp_direct_savepath}"], shell=True
).decode()
txt += "\nncdump VIA NCDATA ="
txt += check_output(
[f"ncdump {ncdump_opts} {temp_ncdata_savepath}"], shell=True
).decode()
print(txt)

# Check equivalence
results = compare_nc_datasets(
temp_direct_savepath,
Expand Down