-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Don't automatically fall back to using zarr
- Loading branch information
Showing
3 changed files
with
71 additions
and
54 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,62 +1,73 @@ | ||
from typing import Tuple, Union | ||
from typing import Union | ||
|
||
import click | ||
import fsspec | ||
import xarray as xr | ||
import zarr | ||
import zarr as zarr_pkg | ||
|
||
|
||
@click.command() | ||
@click.argument("url") | ||
@click.option("-v", "--variable", type=str, help="Dump variable's info") | ||
@click.option("-m", "--max-rows", default=999, help="Maximum number of rows to display") | ||
@click.option("-i", "--info", is_flag=True, help="Use ncdump style") | ||
def dump(url: str, variable: str, max_rows: int, info: bool): | ||
@click.option( | ||
"-z", | ||
"--zarr", | ||
is_flag=True, | ||
help="Open the given URL using zarr-python instead of xarray.", | ||
) | ||
def dump(url: str, variable: str, max_rows: int, info: bool, zarr: bool): | ||
fs, _, _ = fsspec.get_fs_token_paths(url) | ||
if not fs.exists(url): | ||
raise click.ClickException(f"No file or directory at {url}") | ||
|
||
if info and zarr: | ||
raise click.ClickException("Cannot use both '-z' and '-i' options") | ||
|
||
if variable is not None and info: | ||
raise click.ClickException("Cannot use both '-v' and '-i' options") | ||
|
||
m = fs.get_mapper(url) | ||
consolidated = _metadata_is_consolidated(m) | ||
object_, object_is_xarray = _open_with_xarray_or_zarr(m, consolidated) | ||
|
||
if zarr: | ||
object_ = _open_with_zarr(m, consolidated) | ||
else: | ||
object_ = _open_with_xarray(m, consolidated) | ||
|
||
if variable is not None: | ||
if info: | ||
raise click.ClickException("Cannot use both '-v' and '-i' options") | ||
object_ = object_[variable] | ||
|
||
if not object_is_xarray: | ||
object_ = object_.info | ||
|
||
if object_is_xarray and info: | ||
object_.info() | ||
if zarr: | ||
print(object_.info) | ||
else: | ||
try: | ||
with xr.set_options(display_max_rows=max_rows): | ||
if info: | ||
object_.info() | ||
else: | ||
if xr.__version__ >= "0.18.0": | ||
with xr.set_options(display_max_rows=max_rows): | ||
print(object_) | ||
else: | ||
# xarray<v0.18.0 does not have display_max_rows option | ||
print(object_) | ||
except ValueError: | ||
# xarray<v0.18.0 does not have display_max_rows option | ||
print(object_) | ||
|
||
|
||
def _metadata_is_consolidated(m: fsspec.FSMap) -> bool: | ||
try: | ||
zarr.open_consolidated(m) | ||
zarr_pkg.open_consolidated(m) | ||
consolidated = True | ||
except KeyError: | ||
# group with un-consolidated metadata, or array | ||
consolidated = False | ||
return consolidated | ||
|
||
|
||
def _open_with_xarray_or_zarr( | ||
def _open_with_xarray(m: fsspec.FSMap, consolidated: bool) -> xr.Dataset: | ||
return xr.open_zarr(m, consolidated=consolidated) | ||
|
||
|
||
def _open_with_zarr( | ||
m: fsspec.FSMap, consolidated: bool | ||
) -> Tuple[Union[xr.Dataset, zarr.hierarchy.Group, zarr.core.Array], bool]: | ||
try: | ||
result = xr.open_zarr(m, consolidated=consolidated) | ||
is_xarray_dataset = True | ||
except (KeyError, TypeError): | ||
# xarray requires _ARRAY_DIMENSIONS attribute, assuming missing if KeyError | ||
result = zarr.open_consolidated(m) if consolidated else zarr.open(m) | ||
is_xarray_dataset = False | ||
return result, is_xarray_dataset | ||
) -> Union[zarr_pkg.hierarchy.Group, zarr_pkg.core.Array]: | ||
return zarr_pkg.open_consolidated(m) if consolidated else zarr_pkg.open(m) |