Merge branch 'main' into dapko.dmh

.github/workflows/run_tests_cdash.yml

@@ -6,7 +6,9 @@ name: Run CDash Ubuntu/Linux netCDF Tests
 
 on: workflow_dispatch
 
-
+concurrency:  
+  group: ${{ github.workflow}}-${{ github.head_ref }}  
+  cancel-in-progress: true
 
 jobs:
 

.github/workflows/run_tests_osx.yml

@@ -6,9 +6,12 @@
 
 name: Run macOS-based netCDF Tests
 
-
 on: [pull_request,workflow_dispatch]
 
+concurrency:  
+  group: ${{ github.workflow}}-${{ github.head_ref }}  
+  cancel-in-progress: true
+
 jobs:
 
   build-deps-osx:

.github/workflows/run_tests_s3.yml

@@ -11,6 +11,10 @@ name: Run S3 netCDF Tests (under Ubuntu Linux)
 
 on: [workflow_dispatch]
 
+concurrency:  
+  group: ${{ github.workflow}}-${{ github.head_ref }}  
+  cancel-in-progress: true
+
 jobs:
 
   build-deps-serial:

.github/workflows/run_tests_ubuntu.yml

@@ -6,6 +6,10 @@ name: Run Ubuntu/Linux netCDF Tests
 
 on: [pull_request, workflow_dispatch]
 
+concurrency:  
+  group: ${{ github.workflow}}-${{ github.head_ref }}  
+  cancel-in-progress: true
+
 jobs:
 
   build-deps-serial:

.github/workflows/run_tests_win_cygwin.yml

@@ -2,6 +2,10 @@ name: Run Cygwin-based tests
 
 on: [pull_request,workflow_dispatch]
 
+concurrency:  
+  group: ${{ github.workflow}}-${{ github.head_ref }}  
+  cancel-in-progress: true
+
 env:
   SHELLOPTS: igncr
   CHERE_INVOKING: 1

.github/workflows/run_tests_win_mingw.yml

@@ -11,6 +11,10 @@ env:
 
 on: [pull_request,workflow_dispatch]
 
+concurrency:  
+  group: ${{ github.workflow}}-${{ github.head_ref }}  
+  cancel-in-progress: true
+
 jobs:
 
   build-and-test-autotools:

Makefile.am

@@ -211,7 +211,11 @@ install-data-hook:
 all-local: liblib/libnetcdf.la
 	echo ${PACKAGE_VERSION} > VERSION
 if ENABLE_S3_TESTALL
+	rm -f ${abs_top_builddir}/tmp_@[email protected]
 	echo "@TESTUID@" >> ${abs_top_builddir}/s3cleanup_@[email protected]
+	cat ${abs_top_builddir}/s3cleanup_@[email protected] | sort | uniq > ${abs_top_builddir}/tmp_@[email protected]
+	rm -f ${abs_top_builddir}/s3cleanup_@[email protected]
+	mv ${abs_top_builddir}/tmp_@[email protected] ${abs_top_builddir}/s3cleanup_@[email protected]
 endif
 
 if ENABLE_S3_TESTALL

RELEASE_NOTES.md

@@ -8,6 +8,7 @@ This file contains a high-level description of this package's evolution. Release
 ## 4.9.3 - TBD
 
 * Fix DAP2 proxy problems. See [Github #2764](https://github.com/Unidata/netcdf-c/pull/2764).
+* Cleanup a number of misc issues. See [Github #2763](https://github.com/Unidata/netcdf-c/pull/2763).
 * Mitigate the problem of test interference. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755).
 * Extend NCZarr to support unlimited dimensions. See [Github #2755](https://github.com/Unidata/netcdf-c/pull/2755).
 * Fix significant bug in the NCZarr cache management. See [Github #2737](https://github.com/Unidata/netcdf-c/pull/2737).

include/ncrc.h

@@ -53,6 +53,9 @@ typedef struct NCRCinfo {
 	NClist* s3profiles; /* NClist<struct AWSprofile*> */
 } NCRCinfo;
 
+/* Opaque structures */
+struct NCS3INFO;
+
 #if defined(__cplusplus)
 extern "C" {
 #endif
@@ -94,7 +97,7 @@ EXTERNL int NC_getactives3profile(NCURI* uri, const char** profilep);
 EXTERNL int NC_s3profilelookup(const char* profile, const char* key, const char** valuep);
 EXTERNL int NC_authgets3profile(const char* profile, struct AWSprofile** profilep);
 EXTERNL int NC_iss3(NCURI* uri);
-EXTERNL int NC_s3urlrebuild(NCURI* url, char** inoutbucketp, char** inoutregionp, NCURI** newurlp);
+EXTERNL int NC_s3urlrebuild(NCURI* url, struct NCS3INFO* s3, NCURI** newurlp);
 
 #if defined(__cplusplus)
 }

include/ncs3sdk.h

@@ -6,13 +6,19 @@
 #ifndef NCS3SDK_H
 #define NCS3SDK_H 1
 
+/* Track the server type, if known */
+typedef enum NCS3SVC {NCS3UNK=0, /* unknown */
+                 NCS3=1,    /* s3.amazon.aws */
+		 NCS3GS=0   /* storage.googleapis.com */
+} NCS3SVC;
 
 typedef struct NCS3INFO {
     char* host; /* non-null if other*/
     char* region; /* region */
     char* bucket; /* bucket name */
     char* rootkey;
     char* profile;
+    NCS3SVC svc;
 } NCS3INFO;
 
 #ifdef __cplusplus

libdispatch/dinfermodel.c

@@ -136,6 +136,7 @@ static const struct MACRODEF {
 {"xarray","mode",{"zarr", NULL}},
 {"noxarray","mode",{"nczarr", "noxarray", NULL}},
 {"zarr","mode",{"nczarr","zarr", NULL}},
+{"gs3","mode",{"gs3","nczarr",NULL}}, /* Google S3 API */
 {NULL,NULL,{NULL}}
 };
 
@@ -196,6 +197,7 @@ static struct NCPROTOCOLLIST {
     {"dods","http","mode=dap2"},
     {"dap4","http","mode=dap4"},
     {"s3","s3","mode=s3"},
+    {"gs3","gs3","mode=gs3"},
     {NULL,NULL,NULL} /* Terminate search */
 };
 
@@ -914,7 +916,7 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void
 	/* If s3, then rebuild the url */
 	if(NC_iss3(uri)) {
 	    NCURI* newuri = NULL;
-	    if((stat = NC_s3urlrebuild(uri,NULL,NULL,&newuri))) goto done;
+	    if((stat = NC_s3urlrebuild(uri,NULL,&newuri))) goto done;
 	    ncurifree(uri);
 	    uri = newuri;
 	} else if(strcmp(uri->protocol,"file")==0) {

libdispatch/drc.c

@@ -437,7 +437,9 @@ rccompile(const char* filepath)
     NCURI* uri = NULL;
     char* nextline = NULL;
     NCglobalstate* globalstate = NC_getglobalstate();
-    char* bucket = NULL;
+    NCS3INFO s3;
+
+    memset(&s3,0,sizeof(s3));
 
     if((ret=NC_readfile(filepath,tmp))) {
         nclog(NCLOGWARN, "Could not open configuration file: %s",filepath);
@@ -484,9 +486,8 @@ rccompile(const char* filepath)
 	    if(NC_iss3(uri)) {
 	         NCURI* newuri = NULL;
 	        /* Rebuild the url to S3 "path" format */
-	        nullfree(bucket);
-		bucket = NULL;
-	        if((ret = NC_s3urlrebuild(uri,&bucket,NULL,&newuri))) goto done;
+		NC_s3clear(&s3);
+	        if((ret = NC_s3urlrebuild(uri,&s3,&newuri))) goto done;
 		ncurifree(uri);
 		uri = newuri;
 		newuri = NULL;
@@ -546,6 +547,7 @@ rccompile(const char* filepath)
     rcorder(rc);
 
 done:
+    NC_s3clear(&s3);
     if(contents) free(contents);
     ncurifree(uri);
     ncbytesfree(tmp);