test1

configure.ac

@@ -2062,7 +2062,7 @@ AC_SUBST(HAS_HDF5_ROS3,[$has_hdf5_ros3])
 AC_SUBST(HAS_NCZARR,[$enable_nczarr])
 AC_SUBST(ENABLE_S3_TESTING,[$with_s3_testing])
 AC_SUBST(HAS_NCZARR_ZIP,[$enable_nczarr_zip])
-AC_SUBST(DO_NCZARR_ZIP_TESTS,[$enable_nczarr_zip])
+AC_SUBST(NCZARR_DEFAULT_FORMAT,[DFALTZARRFORMAT])
 AC_SUBST(HAS_QUANTIZE,[$enable_quantize])
 AC_SUBST(HAS_LOGGING,[$enable_logging])
 AC_SUBST(DO_FILTER_TESTS,[$enable_filter_testing])

include/nc4internal.h

@@ -322,7 +322,7 @@ typedef struct NC_FILE_INFO
     } mem;
 } NC_FILE_INFO_T;
 
-/* Collect global state info in one place */
+/* Begin to collect global state info in one place (more to do) */
 typedef struct NCglobalstate {
     int initialized;
     char* tempdir; /* track a usable temp dir */
@@ -331,6 +331,7 @@ typedef struct NCglobalstate {
     struct NCRCinfo* rcinfo; /* Currently only one rc file per session */
     struct GlobalZarr { /* Zarr specific parameters */
 	char dimension_separator;
+	int default_zarrformat;
     } zarr;
     struct Alignment { /* H5Pset_alignment parameters */
         int defined; /* 1 => threshold and alignment explicitly set */

libnczarr/zarr.c

@@ -68,7 +68,15 @@ ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, NClist* urlcontrol
 
     /* default the zarr format */
     if(zfile->zarr.zarr_format == 0)
-        zfile->zarr.zarr_format = DFALTZARRFORMAT;
+        zfile->zarr.zarr_format = NC_getglobalstate()->zarr.default_zarrformat;
+
+    /* Allow it to be overridden by getenv to support testing of different formats */
+    if(getenv(NCZARRDEFAULTFORMAT) != NULL) {
+	int dfalt = 0;
+	sscanf(getenv(NCZARRDEFAULTFORMAT),"%d",&dfalt);
+	if(dfalt == 2 || dfalt == 3)
+	    zfile->zarr.zarr_format = dfalt;
+    }
 
     /* initialize map handle*/
     if((stat = NCZ_get_map(file,uri,nc->mode,zfile->flags,NULL,&zfile->map))) goto done;

libnczarr/zformat3.c

@@ -52,7 +52,8 @@ static int NCZ_collect_grps(NC_FILE_INFO_T* file, NC_GRP_INFO_T* parent, NCjson*
 static int NCZ_collect_arrays(NC_FILE_INFO_T* file, NC_GRP_INFO_T* parent, NCjson** jarraysp);
 static int NCZ_collect_dims(NC_FILE_INFO_T* file, NC_GRP_INFO_T* parent, NCjson** jdimsp);
 
-static int NCZ_decodeints(NCjson* jshape, size64_t* shapes);
+static int NCZ_decodesizet64vec(NCjson* jshape, size64_t* shapes);
+static int NCZ_decodesizetvec(NCjson* jshape, size_t* shapes);
 static int NCZ_computeattrinfo(const char* name, NClist* atypes, nc_type typehint, int purezarr, NCjson* values, nc_type* typeidp, size_t* typelenp, size_t* lenp, void** datap);
 static int NCZ_computeattrdata(nc_type typehint, nc_type* typeidp, NCjson* values, size_t* typelenp, size_t* countp, void** datap);
 static int NCZ_computedimrefs(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, NCjson* jvar, NClist* dimnames, size64_t* shapes, NC_DIM_INFO_T** dims);
@@ -179,8 +180,10 @@ write_grp(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_GRP_INFO
         assert(grp->att);
         if((stat = build_atts(file,zfile,(NC_OBJ*)grp, grp->att, &jatts, &jtypes))) goto done;
         if((stat = NCJinsert(jgroup,"attributes",jatts))) goto done;
+	jatts = NULL;
         if(!purezarr && jtypes)
             {if((stat = NCJinsert(jgroup,NCZ_V3_ATTR,jtypes))) goto done;}
+	jtypes = NULL;
     }
 
     if(!purezarr && rootgrp) {
@@ -205,7 +208,6 @@ write_grp(NC_FILE_INFO_T* file, NCZ_FILE_INFO_T* zfile, NCZMAP* map, NC_GRP_INFO
         /* Write to map */
         if((stat=NCZ_uploadjson(map,key,jgroup))) goto done;
         nullfree(key); key = NULL;
-        jgroup = NULL;
     }
 
     /* Now write all the variables */
@@ -1058,7 +1060,6 @@ read_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames)
         char* key = NULL;
 	const char* varname = NULL;
         size64_t* shapes = NULL;
-        size64_t* chunks = NULL;
         NClist* dimnames = NULL;
         int suppress = 0; /* Abort processing of this variable */
         nc_type vtype = NC_NAT;
@@ -1146,7 +1147,7 @@ read_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames)
 	        /* extract the shapes */
 	        if((shapes = (size64_t*)malloc(sizeof(size64_t)*rank)) == NULL)
 	            {stat = (THROW(NC_ENOMEM)); goto done;}
-	        if((stat = NCZ_decodeints(jvalue, shapes))) goto done;
+	        if((stat = NCZ_decodesizet64vec(jvalue, shapes))) goto done;
 	    }
 	    /* Set storage flag */
     	    var->storage = (zvar->scalar?NC_CONTIGUOUS:NC_CHUNKED);
@@ -1259,11 +1260,11 @@ read_vars(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, NClist* varnames)
                 if(rank != NCJlength(jtmp2)) {stat = (THROW(NC_ENCZARR)); goto done;}
 
                 /* Get the chunks and chunkproduct */
-                if((chunks = (size64_t*)malloc(sizeof(size64_t)*rank)) == NULL)
+                if((var->chunksizes = (size_t*)malloc(sizeof(size_t)*rank)) == NULL)
                         {stat = (THROW(NC_ENOMEM)); goto done;}
-                if((stat = NCZ_decodeints(jtmp2, chunks))) goto done;
+                if((stat = NCZ_decodesizetvec(jtmp2, var->chunksizes))) goto done;
                 zvar->chunkproduct = 1;
-                for(k=0;k<rank;k++) zvar->chunkproduct *= chunks[i];
+                for(k=0;k<rank;k++) zvar->chunkproduct *= var->chunksizes[i];
                 zvar->chunksize = zvar->chunkproduct * var->type_info->size;
                 /* Create the cache */
                 if((stat = NCZ_create_chunk_cache(var,zvar->chunksize,zvar->dimension_separator,&zvar->cache))) goto done;
@@ -1821,9 +1822,9 @@ subobjects(NCZ_FILE_INFO_T* zfile, NC_GRP_INFO_T* grp, NClist* varnames, NClist*
     return NC_NOERR;
 }
 
-/* Convert a list of integer strings to 64 bit dimension sizes (shapes) */
+/* Convert a list of integer strings to size_t integer */
 static int
-NCZ_decodeints(NCjson* jshape, size64_t* shapes)
+NCZ_decodesizet64vec(NCjson* jshape, size64_t* shapes)
 {
     int i, stat = NC_NOERR;
 
@@ -1834,11 +1835,38 @@ NCZ_decodeints(NCjson* jshape, size64_t* shapes)
 	if((stat = NCZ_json2cvt(jv,&zcvt,&typeid))) goto done;
 	switch (typeid) {
 	case NC_INT64:
-	if(zcvt.int64v < 0) {stat = (THROW(NC_ENCZARR)); goto done;}
-	    shapes[i] = (size64_t)zcvt.int64v;
+	    if(zcvt.int64v < 0) {stat = (THROW(NC_ENCZARR)); goto done;}
+	    shapes[i] = zcvt.int64v;
 	    break;
 	case NC_UINT64:
-	    shapes[i] = (size64_t)zcvt.uint64v;
+	    shapes[i] = zcvt.uint64v;
+	    break;
+	default: {stat = (THROW(NC_ENCZARR)); goto done;}
+	}
+    }
+
+done:
+    return THROW(stat);
+}
+
+/* Convert a list of integer strings to size_t integer */
+static int
+NCZ_decodesizetvec(NCjson* jshape, size_t* shapes)
+{
+    int i, stat = NC_NOERR;
+
+    for(i=0;i<NCJlength(jshape);i++) {
+	struct ZCVT zcvt;
+	nc_type typeid = NC_NAT;
+	NCjson* jv = NCJith(jshape,i);
+	if((stat = NCZ_json2cvt(jv,&zcvt,&typeid))) goto done;
+	switch (typeid) {
+	case NC_INT64:
+	    if(zcvt.int64v < 0) {stat = (THROW(NC_ENCZARR)); goto done;}
+	    shapes[i] = (size_t)zcvt.int64v;
+	    break;
+	case NC_UINT64:
+	    shapes[i] = (size_t)zcvt.uint64v;
 	    break;
 	default: {stat = (THROW(NC_ENCZARR)); goto done;}
 	}

libnczarr/zinternal.c

@@ -64,11 +64,12 @@ NCZ_initialize_internal(void)
     ngs = NC_getglobalstate();
     if(ngs != NULL) {
         /* Defaults */
-#if DFALTZARRFORMAT == 3
-	ngs->zarr.dimension_separator = DFALT_DIM_SEPARATOR_V3;
-#else
-	ngs->zarr.dimension_separator = DFALT_DIM_SEPARATOR_V2;
-#endif
+	ngs->zarr.default_zarrformat = DFALTZARRFORMAT;
+
+	if(ngs->zarr.default_zarrformat == 3)
+	    ngs->zarr.dimension_separator = DFALT_DIM_SEPARATOR_V3;
+	else
+	    ngs->zarr.dimension_separator = DFALT_DIM_SEPARATOR_V2;
         dimsep = NC_rclookup("ZARR.DIMENSION_SEPARATOR",NULL,NULL);
         if(dimsep != NULL) {
             /* Verify its value */

libnczarr/zinternal.h

@@ -34,6 +34,9 @@
 /* Map the NCZarr Format version to a string */
 #define NCZARR_FORMAT_VERSION_TEMPLATE "%d.0.0"
 
+/* The name of the env var for changing default zarr format */
+#define NCZARRDEFAULTFORMAT "NCZARRFORMAT"
+
 /* These have to do with creating chunked datasets in ZARR. */
 #define NCZ_CHUNKSIZE_FACTOR (10)
 #define NCZ_MIN_CHUNK_SIZE (2)

nczarr_test/Makefile.am

@@ -342,6 +342,9 @@ clean-local:
 if ENABLE_S3_TESTALL
 check-local:
 	bash ${abs_top_builddir}/s3cleanup.sh
+
+clean-local:
+	bash ${abs_top_builddir}/s3cleanup.sh
 endif
 
 DISTCLEANFILES = findplugin.sh ${BUILT_SOURCES}

s3gc.in

@@ -54,12 +54,12 @@ lastdate=$((current-deltasec))
 rm -f s3gc.json
 
 # Get complete set of keys in ${S3TESTSUBTREE} prefix
-if ! aws s3api list-objects-v2 --bucket ${S3TESTBUCKET} --prefix "/${S3TESTSUBTREE}" | grep -F '"Key":' >s3gc.keys ; then
+if ! aws s3api list-objects-v2 --bucket ${S3TESTBUCKET} --prefix "${S3TESTSUBTREE}" | grep -F '"Key":' >s3gc.keys ; then
     echo "No keys found"
     rm -f s3gc.json
     exit 0
 fi
-aws s3api list-objects-v2 --bucket ${S3TESTBUCKET} --prefix "/${S3TESTSUBTREE}" | grep -F '"Key":' >s3gc.keys
+aws s3api list-objects-v2 --bucket ${S3TESTBUCKET} --prefix "${S3TESTSUBTREE}" | grep -F '"Key":' >s3gc.keys
 while read -r line; do
   KEY0=`echo "$line" | sed -e 's|[^"]*"Key":[^"]*"\([^"]*\)".*|\1|'`
   # Strip off any leading '/'

test_common.in

@@ -43,6 +43,7 @@ FEATURE_NCZARR=@HAS_NCZARR@
 FEATURE_S3TESTS=@ENABLE_S3_TESTING@
 FEATURE_NCZARR_ZIP=@DO_NCZARR_ZIP_TESTS@
 FEATURE_LARGE_TESTS=@DO_LARGE_TESTS@
+FEATURE_NCZARR_FORMAT=@NCZARR_DEFAULT_FORMAT@
 
 # Thredds-test server is currently disabled
 #FEATURE_THREDDSTEST=1