From c671b97980f6faeecf59d01f3670972d3f039804 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Mon, 27 Nov 2023 16:00:21 -0700 Subject: [PATCH 1/3] test1 --- .github/workflows/run_tests_osx.yml | 2 +- .github/workflows/run_tests_s3.yml | 2 +- .github/workflows/run_tests_ubuntu.yml | 2 +- .github/workflows/run_tests_win_cygwin.yml | 2 +- .github/workflows/run_tests_win_mingw.yml | 2 +- include/ncpathmgr.h | 3 +++ include/ncs3sdk.h | 3 +++ libdispatch/dcopy.c | 4 ++-- libdispatch/derror.c | 4 ++-- libdispatch/nch5s3comms.c | 4 ++-- libdispatch/nclist.c | 7 +++++-- libncpoco/cp_unix.c | 2 +- libncpoco/cp_win32.c | 5 ++--- libncpoco/ncpoco.c | 11 +++++++++++ libncpoco/ncpoco.h | 3 +++ libsrc/v1hpg.c | 14 +++++++------- s3gc.in | 4 +++- 17 files changed, 49 insertions(+), 25 deletions(-) diff --git a/.github/workflows/run_tests_osx.yml b/.github/workflows/run_tests_osx.yml index 8f2d9d62af..3293dcc7cb 100644 --- a/.github/workflows/run_tests_osx.yml +++ b/.github/workflows/run_tests_osx.yml @@ -6,7 +6,7 @@ name: Run macOS-based netCDF Tests -on: [pull_request,workflow_dispatch] +on: [push,pull_request,workflow_dispatch] concurrency: group: ${{ github.workflow}}-${{ github.head_ref }} diff --git a/.github/workflows/run_tests_s3.yml b/.github/workflows/run_tests_s3.yml index ddbb353c8e..cf45621955 100644 --- a/.github/workflows/run_tests_s3.yml +++ b/.github/workflows/run_tests_s3.yml @@ -9,7 +9,7 @@ name: Run S3 netCDF Tests (under Ubuntu Linux) -on: [workflow_dispatch] +on: [push,workflow_dispatch] concurrency: group: ${{ github.workflow}}-${{ github.head_ref }} diff --git a/.github/workflows/run_tests_ubuntu.yml b/.github/workflows/run_tests_ubuntu.yml index 64b91bb99a..ab8ce5dcfa 100644 --- a/.github/workflows/run_tests_ubuntu.yml +++ b/.github/workflows/run_tests_ubuntu.yml @@ -4,7 +4,7 @@ name: Run Ubuntu/Linux netCDF Tests -on: [pull_request, workflow_dispatch] +on: [push,pull_request, workflow_dispatch] concurrency: group: ${{ github.workflow}}-${{ github.head_ref }} diff --git a/.github/workflows/run_tests_win_cygwin.yml b/.github/workflows/run_tests_win_cygwin.yml index 6e124448ed..27486bbc59 100644 --- a/.github/workflows/run_tests_win_cygwin.yml +++ b/.github/workflows/run_tests_win_cygwin.yml @@ -1,6 +1,6 @@ name: Run Cygwin-based tests -on: [pull_request,workflow_dispatch] +on: [push,pull_request,workflow_dispatch] concurrency: group: ${{ github.workflow}}-${{ github.head_ref }} diff --git a/.github/workflows/run_tests_win_mingw.yml b/.github/workflows/run_tests_win_mingw.yml index 978275cf6c..75ac3174cd 100644 --- a/.github/workflows/run_tests_win_mingw.yml +++ b/.github/workflows/run_tests_win_mingw.yml @@ -9,7 +9,7 @@ name: Run MSYS2, MinGW64-based Tests (Not Visual Studio) env: CPPFLAGS: "-D_BSD_SOURCE" -on: [pull_request,workflow_dispatch] +on: [push,pull_request,workflow_dispatch] concurrency: group: ${{ github.workflow}}-${{ github.head_ref }} diff --git a/include/ncpathmgr.h b/include/ncpathmgr.h index 3f0bcd0ec0..91dd38ff02 100644 --- a/include/ncpathmgr.h +++ b/include/ncpathmgr.h @@ -14,6 +14,9 @@ #ifdef HAVE_DIRENT_H #include #endif +#ifdef HAVE_SYS_TYPES_H +#include +#endif #ifdef HAVE_SYS_STAT_H #include #endif diff --git a/include/ncs3sdk.h b/include/ncs3sdk.h index 771faa6666..013710ccfd 100644 --- a/include/ncs3sdk.h +++ b/include/ncs3sdk.h @@ -6,6 +6,9 @@ #ifndef NCS3SDK_H #define NCS3SDK_H 1 +#define AWSHOST ".amazonaws.com" +#define GOOGLEHOST "storage.googleapis.com" + /* Track the server type, if known */ typedef enum NCS3SVC {NCS3UNK=0, /* unknown */ NCS3=1, /* s3.amazon.aws */ diff --git a/libdispatch/dcopy.c b/libdispatch/dcopy.c index 2491a92b6d..1f0ff034aa 100644 --- a/libdispatch/dcopy.c +++ b/libdispatch/dcopy.c @@ -694,7 +694,7 @@ searchgrouptree(int ncid1, int tid1, int grp, int* tid2) int gid; uintptr_t id; - id = grp; + id = (uintptr_t)grp; nclistpush(queue,(void*)id); /* prime the queue */ while(nclistlength(queue) > 0) { id = (uintptr_t)nclistremove(queue,0); @@ -712,7 +712,7 @@ searchgrouptree(int ncid1, int tid1, int grp, int* tid2) goto done; /* push onto the end of the queue */ for(i=0;isigning_key = signing_key; signing_key = NULL; @@ -2033,7 +2033,7 @@ NCH5_s3comms_signing_key(unsigned char **mdp, const char *secret, const char *re if ((size_t)ret != (AWS4_secret_len - 1)) HGOTO_ERRORVA(H5E_ARGS, NC_EINVAL, FAIL, "problem writing AWS4+secret `%s`", secret); - if((md = (unsigned char*)malloc(SHA256_DIGEST_LENGTH))==NULL) + if((md = (unsigned char*)calloc(1,SHA256_DIGEST_LENGTH))==NULL) HGOTO_ERROR(H5E_ARGS, NC_ENOMEM, NULL, "could not malloc space for signing key ."); /* hash_func, key, len(key), msg, len(msg), digest_dest, digest_len_dest diff --git a/libdispatch/nclist.c b/libdispatch/nclist.c index f2c3f4d47b..5ed21753eb 100644 --- a/libdispatch/nclist.c +++ b/libdispatch/nclist.c @@ -31,7 +31,7 @@ NClist* nclistnew(void) ncinitialized = 1; } */ - l = (NClist*)malloc(sizeof(NClist)); + l = (NClist*)calloc(1,sizeof(NClist)); if(l) { l->alloc=0; l->length=0; @@ -286,10 +286,13 @@ nclistclone(const NClist* l, int deep) void* nclistextract(NClist* l) { - void* result = l->content; + void* result = NULL; + if(l) { + result = l->content; l->alloc = 0; l->length = 0; l->content = NULL; + } return result; } diff --git a/libncpoco/cp_unix.c b/libncpoco/cp_unix.c index 3ccea2ed03..7ff01f79ec 100755 --- a/libncpoco/cp_unix.c +++ b/libncpoco/cp_unix.c @@ -46,7 +46,7 @@ static void ncperr(const char* fcn, NCPSharedLib* lib) { const char* msg = dlerror(); - lib->err.msg[0] = '\0'; + memset(lib->err.msg,0,sizeof(lib->err.msg)); if(msg != NULL) { strlcat(lib->err.msg,fcn,sizeof(lib->err.msg)); strlcat(lib->err.msg,": ",sizeof(lib->err.msg)); diff --git a/libncpoco/cp_win32.c b/libncpoco/cp_win32.c index b4284ac101..896344f638 100755 --- a/libncpoco/cp_win32.c +++ b/libncpoco/cp_win32.c @@ -105,10 +105,9 @@ load(NCPSharedLib* lib , const char* path0, int flags) char* msg = NULL; FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, errcode, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), &msg, 0, NULL); - if(msg) { + memset(lib->err.msg,0,sizeof(lib->err.msg)); + if(msg) strncpy(lib->err.msg,msg,sizeof(lib->err.msg)); - } else - lib->err.msg[0] = '\0'; ret = NC_ENOTFOUND; goto ldone; } diff --git a/libncpoco/ncpoco.c b/libncpoco/ncpoco.c index 12d4b41eec..b12b2e0a9b 100755 --- a/libncpoco/ncpoco.c +++ b/libncpoco/ncpoco.c @@ -68,6 +68,7 @@ EXTERNL int ncpload(NCPSharedLib* lib, const char* path, int flags) { if(lib == NULL || path == NULL) return NC_EINVAL; + ncpclearerrmsg(lib); return lib->api.load(lib,path,flags); } @@ -75,6 +76,7 @@ EXTERNL int ncpunload(NCPSharedLib* lib) /* Unloads a shared library. */ { if(lib == NULL) return NC_EINVAL; + ncpclearerrmsg(lib); return lib->api.unload(lib); } @@ -93,6 +95,7 @@ EXTERNL void* ncpgetsymbol(NCPSharedLib* lib,const char* name) { if(lib == NULL) return NULL; + ncpclearerrmsg(lib); return lib->api.getsymbol(lib,name); } @@ -113,3 +116,11 @@ ncpgeterrmsg(NCPSharedLib* lib) if(lib == NULL) return NULL; return (lib->err.msg[0] == '\0' ? NULL : lib->err.msg); } + +/* Clear the last err msg. */ +EXTERNL void +ncpclearerrmsg(NCPSharedLib* lib) +{ + if(lib == NULL) return; + memset(lib->err.msg,0,sizeof(lib->err.msg)); +} diff --git a/libncpoco/ncpoco.h b/libncpoco/ncpoco.h index 43fb077bab..fc1e993b99 100755 --- a/libncpoco/ncpoco.h +++ b/libncpoco/ncpoco.h @@ -71,6 +71,9 @@ EXTERNL const char* ncpgetpath(NCPSharedLib*); /* Return last err msg */ EXTERNL const char* ncpgeterrmsg(NCPSharedLib* lib); +/* Clear the last err msg. */ +EXTERNL void ncpclearerrmsg(NCPSharedLib* lib); + EXTERNL const char* intstr(int err1); #endif /*NCPOCO_H*/ diff --git a/libsrc/v1hpg.c b/libsrc/v1hpg.c index f93b219bdb..d1c6684469 100644 --- a/libsrc/v1hpg.c +++ b/libsrc/v1hpg.c @@ -1254,7 +1254,7 @@ NC_computeshapes(NC3_INFO* ncp) for( /*NADA*/; vpp < end; vpp++) { status = NC_var_shape(*vpp, &ncp->dims); - if(status != NC_NOERR) + if(status != NC_NOERR) return(status); if(IS_RECVAR(*vpp)) @@ -1265,13 +1265,13 @@ NC_computeshapes(NC3_INFO* ncp) } else { - if(first_var == NULL) + if(first_var == NULL) first_var = *vpp; - /* - * Overwritten each time thru. - * Usually overwritten in first_rec != NULL clause below. - */ - ncp->begin_rec = (*vpp)->begin + (off_t)(*vpp)->len; + /* + * Overwritten each time thru. + * Usually overwritten in first_rec != NULL clause below. + */ + ncp->begin_rec = (*vpp)->begin + (off_t)(*vpp)->len; } } } diff --git a/s3gc.in b/s3gc.in index 612e371d7a..a63345c6c7 100755 --- a/s3gc.in +++ b/s3gc.in @@ -61,7 +61,9 @@ if ! aws s3api list-objects-v2 --bucket ${S3TESTBUCKET} --prefix "${S3TESTSUBTRE fi aws s3api list-objects-v2 --bucket ${S3TESTBUCKET} --prefix "${S3TESTSUBTREE}" | grep -F '"Key":' >s3gc.keys while read -r line; do - KEY=`echo "$line" | sed -e 's|[^"]*"Key":[^"]*"\([^"]*\)".*|\1|'` + KEY0=`echo "$line" | sed -e 's|[^"]*"Key":[^"]*"\([^"]*\)".*|\1|'` + # Strip off any leading '/' + KEY=`echo "$KEY0" | sed -e 's|^[/]*\(.*\)|\1|'` # Ignore keys that do not start with ${S3TESTSUBTREE} PREFIX=`echo "$KEY" | sed -e 's|\([^/]*\)/.*|\1|'` if test "x$PREFIX" = "x$S3TESTSUBTREE" ; then From 00888103ab19068b0dd2c1d31f3a525bb20871c2 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Mon, 27 Nov 2023 18:43:54 -0700 Subject: [PATCH 2/3] ckp --- .github/workflows/run_tests_osx.yml | 2 +- .github/workflows/run_tests_s3.yml | 2 +- .github/workflows/run_tests_ubuntu.yml | 2 +- .github/workflows/run_tests_win_cygwin.yml | 2 +- .github/workflows/run_tests_win_mingw.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/run_tests_osx.yml b/.github/workflows/run_tests_osx.yml index 3293dcc7cb..8f2d9d62af 100644 --- a/.github/workflows/run_tests_osx.yml +++ b/.github/workflows/run_tests_osx.yml @@ -6,7 +6,7 @@ name: Run macOS-based netCDF Tests -on: [push,pull_request,workflow_dispatch] +on: [pull_request,workflow_dispatch] concurrency: group: ${{ github.workflow}}-${{ github.head_ref }} diff --git a/.github/workflows/run_tests_s3.yml b/.github/workflows/run_tests_s3.yml index cf45621955..ddbb353c8e 100644 --- a/.github/workflows/run_tests_s3.yml +++ b/.github/workflows/run_tests_s3.yml @@ -9,7 +9,7 @@ name: Run S3 netCDF Tests (under Ubuntu Linux) -on: [push,workflow_dispatch] +on: [workflow_dispatch] concurrency: group: ${{ github.workflow}}-${{ github.head_ref }} diff --git a/.github/workflows/run_tests_ubuntu.yml b/.github/workflows/run_tests_ubuntu.yml index ab8ce5dcfa..64b91bb99a 100644 --- a/.github/workflows/run_tests_ubuntu.yml +++ b/.github/workflows/run_tests_ubuntu.yml @@ -4,7 +4,7 @@ name: Run Ubuntu/Linux netCDF Tests -on: [push,pull_request, workflow_dispatch] +on: [pull_request, workflow_dispatch] concurrency: group: ${{ github.workflow}}-${{ github.head_ref }} diff --git a/.github/workflows/run_tests_win_cygwin.yml b/.github/workflows/run_tests_win_cygwin.yml index 27486bbc59..6e124448ed 100644 --- a/.github/workflows/run_tests_win_cygwin.yml +++ b/.github/workflows/run_tests_win_cygwin.yml @@ -1,6 +1,6 @@ name: Run Cygwin-based tests -on: [push,pull_request,workflow_dispatch] +on: [pull_request,workflow_dispatch] concurrency: group: ${{ github.workflow}}-${{ github.head_ref }} diff --git a/.github/workflows/run_tests_win_mingw.yml b/.github/workflows/run_tests_win_mingw.yml index 75ac3174cd..978275cf6c 100644 --- a/.github/workflows/run_tests_win_mingw.yml +++ b/.github/workflows/run_tests_win_mingw.yml @@ -9,7 +9,7 @@ name: Run MSYS2, MinGW64-based Tests (Not Visual Studio) env: CPPFLAGS: "-D_BSD_SOURCE" -on: [push,pull_request,workflow_dispatch] +on: [pull_request,workflow_dispatch] concurrency: group: ${{ github.workflow}}-${{ github.head_ref }} From ec3a7a74c30296cb92d591a2e281f327d88a2523 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Thu, 30 Nov 2023 20:50:19 -0700 Subject: [PATCH 3/3] ckp --- libdispatch/dinfermodel.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index 6f5e4a846c..ec0fb4ea70 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -898,13 +898,16 @@ NC_infermodel(const char* path, int* omodep, int iscreate, int useparallel, void ncurisetfragments(uri,sfrag); nullfree(sfrag); sfrag = NULL; +#ifdef ENABLE_S3 /* If s3, then rebuild the url */ if(NC_iss3(uri)) { NCURI* newuri = NULL; if((stat = NC_s3urlrebuild(uri,NULL,&newuri))) goto done; ncurifree(uri); uri = newuri; - } else if(strcmp(uri->protocol,"file")==0) { + } else +#endif + if(strcmp(uri->protocol,"file")==0) { /* convert path to absolute */ char* canon = NULL; abspath = NCpathabsolute(uri->path);