From 7f2355c3e4137ebec2bbce0d9045660ab42f8f02 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Tue, 21 Nov 2023 14:15:59 -0700 Subject: [PATCH] test1 --- configure.ac | 2 +- include/nc4internal.h | 3 ++- libnczarr/zarr.c | 10 +++++++- libnczarr/zformat3.c | 52 +++++++++++++++++++++++++++++++---------- libnczarr/zinternal.c | 11 +++++---- libnczarr/zinternal.h | 3 +++ nczarr_test/Makefile.am | 3 +++ s3gc.in | 4 ++-- test_common.in | 1 + 9 files changed, 67 insertions(+), 22 deletions(-) diff --git a/configure.ac b/configure.ac index d5e00a9d5f..1971fb0239 100644 --- a/configure.ac +++ b/configure.ac @@ -2062,7 +2062,7 @@ AC_SUBST(HAS_HDF5_ROS3,[$has_hdf5_ros3]) AC_SUBST(HAS_NCZARR,[$enable_nczarr]) AC_SUBST(ENABLE_S3_TESTING,[$with_s3_testing]) AC_SUBST(HAS_NCZARR_ZIP,[$enable_nczarr_zip]) -AC_SUBST(DO_NCZARR_ZIP_TESTS,[$enable_nczarr_zip]) +AC_SUBST(NCZARR_DEFAULT_FORMAT,[DFALTZARRFORMAT]) AC_SUBST(HAS_QUANTIZE,[$enable_quantize]) AC_SUBST(HAS_LOGGING,[$enable_logging]) AC_SUBST(DO_FILTER_TESTS,[$enable_filter_testing]) diff --git a/include/nc4internal.h b/include/nc4internal.h index f9925b1dea..188cdfc2be 100644 --- a/include/nc4internal.h +++ b/include/nc4internal.h @@ -322,7 +322,7 @@ typedef struct NC_FILE_INFO } mem; } NC_FILE_INFO_T; -/* Collect global state info in one place */ +/* Begin to collect global state info in one place (more to do) */ typedef struct NCglobalstate { int initialized; char* tempdir; /* track a usable temp dir */ @@ -331,6 +331,7 @@ typedef struct NCglobalstate { struct NCRCinfo* rcinfo; /* Currently only one rc file per session */ struct GlobalZarr { /* Zarr specific parameters */ char dimension_separator; + int default_zarrformat; } zarr; struct Alignment { /* H5Pset_alignment parameters */ int defined; /* 1 => threshold and alignment explicitly set */ diff --git a/libnczarr/zarr.c b/libnczarr/zarr.c index 7862aa81bd..4d32e38bdc 100644 --- a/libnczarr/zarr.c +++ b/libnczarr/zarr.c @@ -68,7 +68,15 @@ ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, NClist* urlcontrol /* default the zarr format */ if(zfile->zarr.zarr_format == 0) - zfile->zarr.zarr_format = DFALTZARRFORMAT; + zfile->zarr.zarr_format = NC_getglobalstate()->zarr.default_zarrformat; + + /* Allow it to be overridden by getenv to support testing of different formats */ + if(getenv(NCZARRDEFAULTFORMAT) != NULL) { + int dfalt = 0; + sscanf(getenv(NCZARRDEFAULTFORMAT),"%d",&dfalt); + if(dfalt == 2 || dfalt == 3) + zfile->zarr.zarr_format = dfalt; + } /* initialize map handle*/ if((stat = NCZ_get_map(file,uri,nc->mode,zfile->flags,NULL,&zfile->map))) goto done; diff --git a/libnczarr/zformat3.c b/libnczarr/zformat3.c index 9b05725dee..a1dc48ad77 100644 --- a/libnczarr/zformat3.c +++ b/libnczarr/zformat3.c @@ -52,7 +52,8 @@ static int NCZ_collect_grps(NC_FILE_INFO_T* file, NC_GRP_INFO_T* parent, NCjson* static int NCZ_collect_arrays(NC_FILE_INFO_T* file, NC_GRP_INFO_T* parent, NCjson** jarraysp); static int NCZ_collect_dims(NC_FILE_INFO_T* file, NC_GRP_INFO_T* parent, NCjson** jdimsp); -static int NCZ_decodeints(NCjson* jshape, size64_t* shapes); +static int NCZ_decodesizet64vec(NCjson* jshape, size64_t* shapes); +static int NCZ_decodesizetvec(NCjson* jshape, size_t* shapes); static int NCZ_computeattrinfo(const char* name, NClist* atypes, nc_type typehint, int purezarr, NCjson* values, nc_type* typeidp, size_t* typelenp, size_t* lenp, void** datap); static int NCZ_computeattrdata(nc_type typehint, nc_type* typeidp, NCjson* values, size_t* typelenp, size_t* countp, void** datap); static int NCZ_computedimrefs(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, NCjson* jvar, NClist* dimnames, size64_t* shapes, NC_DIM_INFO_T** dims); @@ -179,8 +180,10 @@ write_grp(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_GRP_INFO assert(grp->att); if((stat = build_atts(file,zfile,(NC_OBJ*)grp, grp->att, &jatts, &jtypes))) goto done; if((stat = NCJinsert(jgroup,"attributes",jatts))) goto done; + jatts = NULL; if(!purezarr && jtypes) {if((stat = NCJinsert(jgroup,NCZ_V3_ATTR,jtypes))) goto done;} + jtypes = NULL; } if(!purezarr && rootgrp) { @@ -205,7 +208,6 @@ write_grp(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_GRP_INFO /* Write to map */ if((stat=NCZ_uploadjson(map,key,jgroup))) goto done; nullfree(key); key = NULL; - jgroup = NULL; } /* Now write all the variables */ @@ -1058,7 +1060,6 @@ read_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) char* key = NULL; const char* varname = NULL; size64_t* shapes = NULL; - size64_t* chunks = NULL; NClist* dimnames = NULL; int suppress = 0; /* Abort processing of this variable */ nc_type vtype = NC_NAT; @@ -1146,7 +1147,7 @@ read_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) /* extract the shapes */ if((shapes = (size64_t*)malloc(sizeof(size64_t)*rank)) == NULL) {stat = (THROW(NC_ENOMEM)); goto done;} - if((stat = NCZ_decodeints(jvalue, shapes))) goto done; + if((stat = NCZ_decodesizet64vec(jvalue, shapes))) goto done; } /* Set storage flag */ var->storage = (zvar->scalar?NC_CONTIGUOUS:NC_CHUNKED); @@ -1259,11 +1260,11 @@ read_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames) if(rank != NCJlength(jtmp2)) {stat = (THROW(NC_ENCZARR)); goto done;} /* Get the chunks and chunkproduct */ - if((chunks = (size64_t*)malloc(sizeof(size64_t)*rank)) == NULL) + if((var->chunksizes = (size_t*)malloc(sizeof(size_t)*rank)) == NULL) {stat = (THROW(NC_ENOMEM)); goto done;} - if((stat = NCZ_decodeints(jtmp2, chunks))) goto done; + if((stat = NCZ_decodesizetvec(jtmp2, var->chunksizes))) goto done; zvar->chunkproduct = 1; - for(k=0;kchunkproduct *= chunks[i]; + for(k=0;kchunkproduct *= var->chunksizes[i]; zvar->chunksize = zvar->chunkproduct * var->type_info->size; /* Create the cache */ if((stat = NCZ_create_chunk_cache(var,zvar->chunksize,zvar->dimension_separator,&zvar->cache))) goto done; @@ -1821,9 +1822,9 @@ subobjects(NCZ_FILE_INFO_T* zfile, NC_GRP_INFO_T* grp, NClist* varnames, NClist* return NC_NOERR; } -/* Convert a list of integer strings to 64 bit dimension sizes (shapes) */ +/* Convert a list of integer strings to size_t integer */ static int -NCZ_decodeints(NCjson* jshape, size64_t* shapes) +NCZ_decodesizet64vec(NCjson* jshape, size64_t* shapes) { int i, stat = NC_NOERR; @@ -1834,11 +1835,38 @@ NCZ_decodeints(NCjson* jshape, size64_t* shapes) if((stat = NCZ_json2cvt(jv,&zcvt,&typeid))) goto done; switch (typeid) { case NC_INT64: - if(zcvt.int64v < 0) {stat = (THROW(NC_ENCZARR)); goto done;} - shapes[i] = (size64_t)zcvt.int64v; + if(zcvt.int64v < 0) {stat = (THROW(NC_ENCZARR)); goto done;} + shapes[i] = zcvt.int64v; break; case NC_UINT64: - shapes[i] = (size64_t)zcvt.uint64v; + shapes[i] = zcvt.uint64v; + break; + default: {stat = (THROW(NC_ENCZARR)); goto done;} + } + } + +done: + return THROW(stat); +} + +/* Convert a list of integer strings to size_t integer */ +static int +NCZ_decodesizetvec(NCjson* jshape, size_t* shapes) +{ + int i, stat = NC_NOERR; + + for(i=0;izarr.dimension_separator = DFALT_DIM_SEPARATOR_V3; -#else - ngs->zarr.dimension_separator = DFALT_DIM_SEPARATOR_V2; -#endif + ngs->zarr.default_zarrformat = DFALTZARRFORMAT; + + if(ngs->zarr.default_zarrformat == 3) + ngs->zarr.dimension_separator = DFALT_DIM_SEPARATOR_V3; + else + ngs->zarr.dimension_separator = DFALT_DIM_SEPARATOR_V2; dimsep = NC_rclookup("ZARR.DIMENSION_SEPARATOR",NULL,NULL); if(dimsep != NULL) { /* Verify its value */ diff --git a/libnczarr/zinternal.h b/libnczarr/zinternal.h index 2a5d3c297c..4a0175db3d 100644 --- a/libnczarr/zinternal.h +++ b/libnczarr/zinternal.h @@ -34,6 +34,9 @@ /* Map the NCZarr Format version to a string */ #define NCZARR_FORMAT_VERSION_TEMPLATE "%d.0.0" +/* The name of the env var for changing default zarr format */ +#define NCZARRDEFAULTFORMAT "NCZARRFORMAT" + /* These have to do with creating chunked datasets in ZARR. */ #define NCZ_CHUNKSIZE_FACTOR (10) #define NCZ_MIN_CHUNK_SIZE (2) diff --git a/nczarr_test/Makefile.am b/nczarr_test/Makefile.am index 02c7d7be43..0de3cfdc91 100644 --- a/nczarr_test/Makefile.am +++ b/nczarr_test/Makefile.am @@ -342,6 +342,9 @@ clean-local: if ENABLE_S3_TESTALL check-local: bash ${abs_top_builddir}/s3cleanup.sh + +clean-local: + bash ${abs_top_builddir}/s3cleanup.sh endif DISTCLEANFILES = findplugin.sh ${BUILT_SOURCES} diff --git a/s3gc.in b/s3gc.in index 6f2c335f27..e22926da6c 100755 --- a/s3gc.in +++ b/s3gc.in @@ -54,12 +54,12 @@ lastdate=$((current-deltasec)) rm -f s3gc.json # Get complete set of keys in ${S3TESTSUBTREE} prefix -if ! aws s3api list-objects-v2 --bucket ${S3TESTBUCKET} --prefix "/${S3TESTSUBTREE}" | grep -F '"Key":' >s3gc.keys ; then +if ! aws s3api list-objects-v2 --bucket ${S3TESTBUCKET} --prefix "${S3TESTSUBTREE}" | grep -F '"Key":' >s3gc.keys ; then echo "No keys found" rm -f s3gc.json exit 0 fi -aws s3api list-objects-v2 --bucket ${S3TESTBUCKET} --prefix "/${S3TESTSUBTREE}" | grep -F '"Key":' >s3gc.keys +aws s3api list-objects-v2 --bucket ${S3TESTBUCKET} --prefix "${S3TESTSUBTREE}" | grep -F '"Key":' >s3gc.keys while read -r line; do KEY0=`echo "$line" | sed -e 's|[^"]*"Key":[^"]*"\([^"]*\)".*|\1|'` # Strip off any leading '/' diff --git a/test_common.in b/test_common.in index 5f0f066db8..d2200e083b 100644 --- a/test_common.in +++ b/test_common.in @@ -43,6 +43,7 @@ FEATURE_NCZARR=@HAS_NCZARR@ FEATURE_S3TESTS=@ENABLE_S3_TESTING@ FEATURE_NCZARR_ZIP=@DO_NCZARR_ZIP_TESTS@ FEATURE_LARGE_TESTS=@DO_LARGE_TESTS@ +FEATURE_NCZARR_FORMAT=@NCZARR_DEFAULT_FORMAT@ # Thredds-test server is currently disabled #FEATURE_THREDDSTEST=1