Skip to content

Commit

Permalink
ckp
Browse files Browse the repository at this point in the history
  • Loading branch information
@DennisHeimbigner
DennisHeimbigner committed Aug 16, 2024
1 parent 6f09a6a commit 5e51230
Show file tree
Hide file tree
Showing 5 changed files with 108 additions and 135 deletions.
7 changes: 1 addition & 6 deletions libnczarr/zarr.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ EXTERNL int ncz_sync_file(NC_FILE_INFO_T* file, int isclose);
EXTERNL int ncz_sync_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, int isclose);
EXTERNL int ncz_sync_atts(NC_FILE_INFO_T*, NC_OBJ* container, NCindex* attlist, NCjson* jatts, NCjson* jtypes, int isclose);
EXTERNL int ncz_read_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, struct ZJSON* jsonz);
EXTERNL int ncz_read_atts(NC_FILE_INFO_T* file, NC_OBJ* container);
EXTERNL int ncz_read_atts(NC_FILE_INFO_T* file, NC_OBJ* container, NCjson* jatts);
EXTERNL int ncz_read_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames);
EXTERNL int ncz_read_file(NC_FILE_INFO_T* file);
EXTERNL int ncz_write_var(NC_VAR_INFO_T* var);
Expand All @@ -67,11 +67,6 @@ EXTERNL int NCZ_dimkey(const NC_DIM_INFO_T* dim, char** pathp);
EXTERNL int ncz_splitkey(const char* path, NClist* segments);
EXTERNL int ncz_nctypedecode(const char* snctype, nc_type* nctypep);

EXTERNL int ncz2_nctype2dtype(nc_type nctype, int endianness, int purezarr, size_t len, char** dnamep);
EXTERNL int ncz2_dtype2nctype(const char* dtype, nc_type typehint, int purezarr, nc_type* nctypep, int* endianp, size_t* typelenp);
EXTERNL int ncz3_nctype2dtype(nc_type nctype, int purezarr, size_t strlen, char** dnamep, const char** tagp);
EXTERNL int ncz3_dtype2nctype(const char* dtype, const char* hint, nc_type* nctypep, size_t* typelenp);

EXTERNL int NCZ_inferattrtype(const NCjson* value, nc_type typehint, nc_type* typeidp);
EXTERNL int NCZ_inferinttype(unsigned long long u64, int negative);
EXTERNL int ncz_fill_value_sort(nc_type nctype, int*);
Expand Down
19 changes: 17 additions & 2 deletions libnczarr/zformat.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -362,12 +362,12 @@ NCZF_decode_grp_subgroup(NC_FILE_INFO_T* file, NC_GRP_INFO_T* parent, const char
}

int
NCZF_decode_attributes_json(NC_FILE_INFO_T* file, NC_OBJ* container, const NCjson** jattsp, const NCjson** jtypesp)
NCZF_decode_attributes_json(NC_FILE_INFO_T* file, NC_OBJ* container, const NCjson* jatts, const NCjson** jtypesp)
{
int stat = NC_NOERR;
NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info;
assert(zfile != NULL);
stat = zfile->dispatcher->decode_attributes_json(file, container, jattsp, jtypesp);
stat = zfile->dispatcher->decode_attributes_json(file, container, jatts, jtypesp);
return THROW(stat);
}

Expand Down Expand Up @@ -441,3 +441,18 @@ NCZ_clear_zjson(struct ZJSON* zjson)
if(!zjson->constatt) NCJreclaim(zjson->jatts);
}
}
/* Get one of two key values from a dict */
int
NCZ_dictgetalt(const NCjson* jdict, const char* name, const char* alt, const NCjson** jvaluep)
{
int stat = NC_NOERR;
const NCjson* jvalue = NULL;
if((stat = NCJdictget(jdict,name,&jvalue))<0) {stat = NC_EINVAL; goto done;} /* try this first */
if(jvalue == NULL) {
if((stat = NCJdictget(jdict,alt,&jvalue))<0) {stat = NC_EINVAL; goto done;} /* try this alternative*/
}
if(jvaluep) *jvaluep = jvalue;
done:
return THROW(stat);
}
7 changes: 5 additions & 2 deletions libnczarr/zformat.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ typedef struct NCZ_Formatter {
int (*decode_nczarr_group)(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const struct ZJSON* jsonz, NCjson** jdimsp, NClist* vars, NClist* subgrpp);
int (*decode_grp_dims)(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const NCjson* jdims);
int (*decode_grp_var)(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const char* varname, NC_VAR_INFO_T** varp);
int (*decode_attributes_json)(NC_FILE_INFO_T* file, NC_OBJ* container, NCjson** jattsp, NCjson** jtypesp);
int (*decode_attributes_json)(NC_FILE_INFO_T* file, NC_OBJ* container, const NCjson* jatts, NCjson** jtypesp);
int (*decode_var_json)(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, struct ZJSON* jsonz);
int (*decode_nczarr_array)(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, NCjson** jnczarrayp);

Expand Down Expand Up @@ -146,7 +146,7 @@ extern int NCZF_decode_grp_dims(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const
extern int NCZF_decode_grp_subgroup(NC_FILE_INFO_T* file, NC_GRP_INFO_T* parent, const char* subgrpname, NC_GRP_INFO_T** subgrpp);
extern int NCZF_decode_var_json(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, struct ZJSON* jsonz);
extern int NCZF_decode_nczarr_array(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, NCjson** jnczarrayp);
extern int NCZF_decode_attributes_json(NC_FILE_INFO_T* file, NC_OBJ* container, NCjson** jattsp, NCjson** jtypesp);
extern int NCZF_decode_attributes_json(NC_FILE_INFO_T* file, NC_OBJ* container, const NCjson* jatts, NCjson** jtypesp);

/* Create a group|var object */
extern int NCZF_create_grp(NC_FILE_INFO_T* file, NC_GRP_INFO_T* parent, const char* gname, NC_GRP_INFO_T** grpp);
Expand Down Expand Up @@ -184,6 +184,9 @@ extern int NCZ_get_formatter(NC_FILE_INFO_T* file, const NCZ_Formatter** formatt
/* Misc. */
extern void NCZ_clear_zjson(struct ZJSON* zjson);

extern int NCZ_dictgetalt(const NCjson* jdict, const char* name, const char* alt, const NCjson** jvaluep);


/**************************************************/

#if defined(__cplusplus)
Expand Down
89 changes: 65 additions & 24 deletions libnczarr/zformat2.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,6 @@
#include "netcdf_filter_build.h"
#endif

/* Forward */
static int dictgetalt(const NCjson* jdict, const char* name, const char* alt, const NCjson** jvaluep);

/**************************************************/
/* Functions to build json from nc4internal structures and upload */

Expand Down Expand Up @@ -737,8 +734,8 @@ ZF2_decode_var_json(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, struct ZJSON* json
int suppress = 0; /* Abort processing of this variable */
nc_type vtype = NC_NAT;
int vtypelen = 0;
size_t rank = 0;
size_t zarr_rank = 0; /* Need to watch out for scalars */
size_t netcdf_rank = 0; /* true rank => scalar => 0 */
size_t zarr_rank = 0; /* |shape| */
#ifdef NETCDF_ENABLE_NCZARR_FILTERS
const NCjson* jfilter = NULL;
int chainindex = 0;
Expand Down Expand Up @@ -806,11 +803,11 @@ ZF2_decode_var_json(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, struct ZJSON* json
if(jvalue != NULL)
var->storage = NC_CHUNKED;
/* Extract dimrefs list */
if((stat = dictgetalt(jncvar,"dimension_references","dimensions",&jdimrefs))) goto done;
if((stat = dictgetalt(jncvar,"dimension_references","dimrefs",&jdimrefs))) goto done;
if(jdimrefs != NULL) { /* Extract the dimref names */
assert((NCJsort(jdimrefs) == NCJ_ARRAY));
if(zvar->scalar) {
assert(NCJlength(jdimrefs) == 0);
assert(NCJlength(jdimrefs) == 1);
} else {
rank = NCJlength(jdimrefs);
for(j=0;j<rank;j++) {
Expand Down Expand Up @@ -873,14 +870,14 @@ ZF2_decode_var_json(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, struct ZJSON* json
}

if(zvar->scalar) {
rank = 0;
netcdf_rank = 0;
zarr_rank = 1; /* Zarr does not support scalars */
} else
rank = (zarr_rank = NCJlength(jvalue));
netcdf_rank = (zarr_rank = NCJlength(jvalue));

if(zarr_rank > 0) {
/* Save the rank of the variable */
if((stat = nc4_var_set_ndims(var, rank))) goto done;
if((stat = nc4_var_set_ndims(var, netcdf_rank))) goto done;
/* extract the shapes */
if((shapes = (size64_t*)malloc(sizeof(size64_t)*(size_t)zarr_rank)) == NULL)
{stat = (THROW(NC_ENOMEM)); goto done;}
Expand All @@ -906,14 +903,14 @@ ZF2_decode_var_json(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, struct ZJSON* json
} else {/* !zvar->scalar */
if(zarr_rank == 0) {stat = NC_ENCZARR; goto done;}
var->storage = NC_CHUNKED;
if(var->ndims != rank)
if(var->ndims != netcdf_rank)
{stat = (THROW(NC_ENCZARR)); goto done;}
if((var->chunksizes = malloc(sizeof(size_t)*(size_t)zarr_rank)) == NULL)
{stat = NC_ENOMEM; goto done;}
if((stat = decodeints(jvalue, chunks))) goto done;
/* validate the chunk sizes */
zvar->chunkproduct = 1;
for(j=0;j<rank;j++) {
for(j=0;j<netcdf_rank;j++) {
if(chunks[j] == 0)
{stat = (THROW(NC_ENCZARR)); goto done;}
var->chunksizes[j] = (size_t)chunks[j];
Expand Down Expand Up @@ -976,12 +973,13 @@ ZF2_decode_var_json(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, struct ZJSON* json
if(varsized && nclistlength((NClist*)var->filters) > 0)
suppress = 1;
#endif

if(zarr_rank > 0) {
if((stat = computedimrefs(file, var, purezarr, xarray, rank, dimnames, shapes, var->dim)))
goto done;
/* Convert dimrefs to specific dimensions */
if((stat = computedimrefs(file, var, netcdf_rank, dimnames, shapes, var->dim))) goto done;
if(!zvar->scalar) {
/* Extract the dimids */
for(j=0;j<rank;j++)
for(j=0;j<netcdf_rank;j++)
var->dimids[j] = var->dim[j]->hdr.id;
}
}
Expand Down Expand Up @@ -1124,7 +1122,7 @@ ZF2_hdf2codec(const NC_FILE_INFO_T* file, const NC_VAR_INFO_T* var, NCZ_Filter*
}

static int
ZF2_buildchunkkey(const NC_FILE_INFO_T* file, size_t rank, const size64_t* chunkindices, char dimsep, char** keyp)
ZF2_buildchunkkey(const NC_FILE_INFO_T* file, size_t netcdf_rank, const size64_t* chunkindices, char dimsep, char** keyp)
{
int stat = NC_NOERR;
NCZ_FILE_INFO_T* zinfo = (NCZ_FILE_INFO_T*)file->format_file_info;
Expand Down Expand Up @@ -1235,19 +1233,62 @@ insert_nczarr_attr(NCjson* jatts, NCjson* jtypes)
return NC_NOERR;
}

/* Get one of two key values from a dict */
/* Compute the set of dim refs for this variable, taking purezarr and xarray into account */
static int
dictgetalt(const NCjson* jdict, const char* name, const char* alt, const NCjson** jvaluep)
computedimrefs(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, int netcdf_rank, NClist* dimnames, size64_t* shapes, NC_DIM_INFO_T** dims)
{
int stat = NC_NOERR;
const NCjson* jvalue = NULL;
if((stat = NCJdictget(jdict,name,&jvalue))<0) {stat = NC_EINVAL; goto done;} /* try this first */
if(jvalue == NULL) {
if((stat = NCJdictget(jdict,alt,&jvalue))<0) {stat = NC_EINVAL; goto done;} /* try this alternative*/
size_t i;
int createdims = 0; /* 1 => we need to create the dims in root if they do not already exist */
NCZ_FILE_INFO_T* zfile = (NCZ_FILE_INFO_T*)file->format_file_info;
NCZ_VAR_INFO_T* zvar = (NCZ_VAR_INFO_T*)(var->format_var_info);
NCjson* jatts = NULL;
int purezarr = 0;
int xarray = 0;

ZTRACE(3,"file=%s var=%s purezarr=%d xarray=%d ndims=%d shape=%s",
file->controller->path,var->hdr.name,purezarr,xarray,(int)ndims,nczprint_vector(ndims,shapes));
assert(zfile && zvar);

if(zfile->flags & FLAG_PUREZARR) purezarr = 1;
if(zfile->flags & FLAG_XARRAYDIMS) xarray = 1;

if(purezarr && xarray) {/* Read in the attributes to get xarray dimdef attribute; Note that it might not exist */
/* Note that if xarray && !purezarr, then xarray will be superceded by the nczarr dimensions key */
char zdimname[4096];
if(zvar->dimension_names != NULL) {
assert(nclistlength(dimnames) == 0);
if((stat = ncz_read_atts(file,(NC_OBJ*)var,jatts))) goto done;
}
if(zvar->dimension_names != NULL) {
/* convert xarray to the dimnames */
for(i=0;i<nclistlength(zvar->dimension_names);i++) {
snprintf(zdimname,sizeof(zdimname),"/%s",(const char*)nclistget(zvar->dimension_names,i));
nclistpush(dimnames,strdup(zdimname));
}
}
createdims = 1; /* may need to create them */
}

/* If pure zarr and we have no dimref names, then fake it */
if(purezarr && nclistlength(dimnames) == 0) {
int i;
createdims = 1;
for(i=0;i<ndims;i++) {
/* Compute the set of absolute paths to dimrefs */
char zdimname[4096];
snprintf(zdimname,sizeof(zdimname),"/%s_%llu",NCDIMANON,shapes[i]);
nclistpush(dimnames,strdup(zdimname));
}
}
if(jvaluep) *jvaluep = jvalue;

/* Now, use dimnames to get the dims; create if necessary */
if((stat = parsedimrefs(file,dimnames,shapes,dims,createdims)))
goto done;

done:
return THROW(stat);
NCJreclaim(jatts);
return ZUNTRACE(THROW(stat));
}

/**************************************************/
Expand Down
Loading

0 comments on commit 5e51230

Please sign in to comment.