From 6085237cf5c7f9f050d2dd25151e0455368452d4 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Fri, 13 Sep 2024 18:42:50 -0600 Subject: [PATCH 1/4] push --- .github/workflows/main-cmake.yml | 2 +- .github/workflows/run_tests_osx.yml | 2 +- .github/workflows/run_tests_ubuntu.yml | 2 +- .github/workflows/run_tests_win_cygwin.yml | 2 +- .github/workflows/run_tests_win_mingw.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main-cmake.yml b/.github/workflows/main-cmake.yml index 93edb15cef..2a94624736 100644 --- a/.github/workflows/main-cmake.yml +++ b/.github/workflows/main-cmake.yml @@ -1,6 +1,6 @@ name: NetCDF-C CMake CI - Windows -on: [pull_request, workflow_dispatch] +on: [push,pull_request, workflow_dispatch] env: REMOTETESTDOWN: ${{ vars.REMOTETESTDOWN }} diff --git a/.github/workflows/run_tests_osx.yml b/.github/workflows/run_tests_osx.yml index 11f4bc734a..cfe8077423 100644 --- a/.github/workflows/run_tests_osx.yml +++ b/.github/workflows/run_tests_osx.yml @@ -6,7 +6,7 @@ name: Run macOS-based netCDF Tests -on: [pull_request,workflow_dispatch] +on: [push,pull_request,workflow_dispatch] concurrency: group: ${{ github.workflow}}-${{ github.head_ref }} diff --git a/.github/workflows/run_tests_ubuntu.yml b/.github/workflows/run_tests_ubuntu.yml index 3ec6a595cd..6ec2b104be 100644 --- a/.github/workflows/run_tests_ubuntu.yml +++ b/.github/workflows/run_tests_ubuntu.yml @@ -4,7 +4,7 @@ name: Run Ubuntu/Linux netCDF Tests -on: [pull_request,workflow_dispatch] +on: [push,pull_request,workflow_dispatch] env: REMOTETESTDOWN: ${{ vars.REMOTETESTDOWN }} diff --git a/.github/workflows/run_tests_win_cygwin.yml b/.github/workflows/run_tests_win_cygwin.yml index 463177097b..3e6fda20ff 100644 --- a/.github/workflows/run_tests_win_cygwin.yml +++ b/.github/workflows/run_tests_win_cygwin.yml @@ -1,6 +1,6 @@ name: Run Cygwin-based tests -on: [pull_request,workflow_dispatch] +on: [push,pull_request,workflow_dispatch] concurrency: group: ${{ github.workflow}}-${{ github.head_ref }} diff --git a/.github/workflows/run_tests_win_mingw.yml b/.github/workflows/run_tests_win_mingw.yml index 032a31a894..fe88c64648 100644 --- a/.github/workflows/run_tests_win_mingw.yml +++ b/.github/workflows/run_tests_win_mingw.yml @@ -10,7 +10,7 @@ env: CPPFLAGS: "-D_BSD_SOURCE" REMOTETESTDOWN: ${{ vars.REMOTETESTDOWN }} -on: [pull_request,workflow_dispatch] +on: [push,pull_request,workflow_dispatch] concurrency: group: ${{ github.workflow}}-${{ github.head_ref }} From 17ce36048886ee80e97033179d1c0e7cd2b35491 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sat, 14 Sep 2024 09:17:54 -0600 Subject: [PATCH 2/4] unpush --- .github/workflows/main-cmake.yml | 2 +- .github/workflows/run_tests_osx.yml | 2 +- .github/workflows/run_tests_ubuntu.yml | 2 +- .github/workflows/run_tests_win_cygwin.yml | 2 +- .github/workflows/run_tests_win_mingw.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main-cmake.yml b/.github/workflows/main-cmake.yml index 2a94624736..93edb15cef 100644 --- a/.github/workflows/main-cmake.yml +++ b/.github/workflows/main-cmake.yml @@ -1,6 +1,6 @@ name: NetCDF-C CMake CI - Windows -on: [push,pull_request, workflow_dispatch] +on: [pull_request, workflow_dispatch] env: REMOTETESTDOWN: ${{ vars.REMOTETESTDOWN }} diff --git a/.github/workflows/run_tests_osx.yml b/.github/workflows/run_tests_osx.yml index cfe8077423..11f4bc734a 100644 --- a/.github/workflows/run_tests_osx.yml +++ b/.github/workflows/run_tests_osx.yml @@ -6,7 +6,7 @@ name: Run macOS-based netCDF Tests -on: [push,pull_request,workflow_dispatch] +on: [pull_request,workflow_dispatch] concurrency: group: ${{ github.workflow}}-${{ github.head_ref }} diff --git a/.github/workflows/run_tests_ubuntu.yml b/.github/workflows/run_tests_ubuntu.yml index 6ec2b104be..3ec6a595cd 100644 --- a/.github/workflows/run_tests_ubuntu.yml +++ b/.github/workflows/run_tests_ubuntu.yml @@ -4,7 +4,7 @@ name: Run Ubuntu/Linux netCDF Tests -on: [push,pull_request,workflow_dispatch] +on: [pull_request,workflow_dispatch] env: REMOTETESTDOWN: ${{ vars.REMOTETESTDOWN }} diff --git a/.github/workflows/run_tests_win_cygwin.yml b/.github/workflows/run_tests_win_cygwin.yml index 3e6fda20ff..463177097b 100644 --- a/.github/workflows/run_tests_win_cygwin.yml +++ b/.github/workflows/run_tests_win_cygwin.yml @@ -1,6 +1,6 @@ name: Run Cygwin-based tests -on: [push,pull_request,workflow_dispatch] +on: [pull_request,workflow_dispatch] concurrency: group: ${{ github.workflow}}-${{ github.head_ref }} diff --git a/.github/workflows/run_tests_win_mingw.yml b/.github/workflows/run_tests_win_mingw.yml index fe88c64648..032a31a894 100644 --- a/.github/workflows/run_tests_win_mingw.yml +++ b/.github/workflows/run_tests_win_mingw.yml @@ -10,7 +10,7 @@ env: CPPFLAGS: "-D_BSD_SOURCE" REMOTETESTDOWN: ${{ vars.REMOTETESTDOWN }} -on: [push,pull_request,workflow_dispatch] +on: [pull_request,workflow_dispatch] concurrency: group: ${{ github.workflow}}-${{ github.head_ref }} From 25b433b1b3ff3d423158bad9162d590a4ac900e2 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sat, 14 Sep 2024 12:30:23 -0600 Subject: [PATCH 3/4] Extend the netcdf API to support programmatic changes to the plugin search path re: https://github.com/Unidata/netcdf-c/issues/2753 As suggested by Ed Hartnett, This PR extends the netcdf.h API to support programmatic control over the search path used to locate plugins. I created several different APIs, but finally settled on the following API as being the simplest possible. It has the disadvantage that it requires use of a global lock (not implemented) if used in a threaded environment. Specifically, note that modifying the plugin paths must be done "atomically". That is, in a multi-threaded environment, it is important that the sequence of actions involved in setting up the plugin paths must be done by a single processor or in some other way as to guarantee that two or more processors are not simultaneously accessing the plugin path read/write operations. As an example, assume there exists a mutex lock called PLUGINLOCK. Then any processor accessing the plugin paths should operate as follows: ```` lock(PLUGINLOCK); nc_plugin_path_read(...); nc_plugin_path_write(...); unlock(PLUGINLOCK); ```` The API proposed in this PR looks like this (from netcdf-c/include/netcdf_filter.h). * ````int nc_plugin_path_read(int formatx, size_t* ndirsp, char** dirs);```` This function returns the current sequence of directories in the internal plugin path list. Since this function does not modify the plugin path, it can be called at any time. The arguments are as follows: - _formatx_ specify which dispatch implementation to read: currently NC_FORMATX_NC_HDF5 or NC_FORMATX_NCZARR. - _ndirsp_ return the number of dirs in the internal path list - _dirs_ memory for storing the sequence of directies in the internal path list. In practice, this function needs to be called twice. The first time with npaths not NULL and pathlist set to NULL to get the size of the path list. The second time with pathlist not NULL to get the actual sequence of paths. * ````int nc_plugin_path_write(int formatx, size_t ndirs, char** const dirs);```` This function empties the current internal path sequence and replaces it with the sequence of directories argument. Using a paths argument of NULL or npaths argument of 0 will clear the set of plugin paths. The arguments are as follows: - _formatx_ specify which dispatch implementation to write: currently NC_FORMATX_NC_HDF5 or NC_FORMATX_NCZARR or 0 (zero). - _ndirs_ length of the dirs argument - _dirs_ a vector of directory path string used to overwrite the current internal path list If the value zero is used for the formatx argument, then the value being written is applied to all implemention: currently NC_FORMATX_NC_HDF5 and NC_FORMATX_NCZARR. In addition, two other API functions are defined. ```` int nc_plugin_path_initialize(void); int nc_plugin_path_finalize(void); ```` As a rule, the initialize and finalize functions do not need to be explicitly called by the user because they are called as part of *nc_initialize()/nc_finalize()*. In addition to the above changes, add a plugin path testcase: unit_tests/run_pluginpaths.sh+tst_pluginpaths.c. ## Misc. Changes 1. Added a version number for the formatx dispatcher. 2. Setup a per-dispatcher global state mechanism. 3. Add some path manipulation utilities to netcf_aux.h 4. Fix the construction of netcdf_json.h as a BUILT_SOURCE. 5. Fix some minor bugs in netcdf_json.h 6. Fix the construction of netcdf_proplist.h as a BUILT_SOURCE. --- CMakeLists.txt | 5 + RELEASE_NOTES.md | 2 + cmake/dependencies.cmake | 7 +- config.h.cmake.in | 5 +- configure.ac | 10 +- include/Makefile.am | 2 +- include/hdf5internal.h | 5 + include/nc4internal.h | 14 +- include/ncdispatch.h | 2 +- include/ncplugins.h | 46 ++ include/netcdf.h | 9 +- include/netcdf_aux.h | 64 +++ include/netcdf_filter.h | 62 ++- include/netcdf_json.h | 4 +- libdispatch/CMakeLists.txt | 2 +- libdispatch/Makefile.am | 2 +- libdispatch/daux.c | 134 +++++- libdispatch/dcopy.c | 11 +- libdispatch/ddispatch.c | 21 +- libdispatch/dinfermodel.c | 4 +- libdispatch/dinstance.c | 5 +- libdispatch/dnotnc4.c | 2 + libdispatch/dplugins.c | 240 ++++++++++ libdispatch/ds3util.c | 67 ++- libdispatch/ncbytes.c | 2 +- libdispatch/ncjson.c | 4 +- libdispatch/nclist.c | 61 ++- libdispatch/ncs3sdk_h5.c | 9 +- libdispatch/ncuri.c | 2 +- libhdf5/CMakeLists.txt | 2 +- libhdf5/Makefile.am | 2 +- libhdf5/hdf5dispatch.c | 1 + libhdf5/hdf5internal.c | 5 +- libhdf5/hdf5plugins.c | 191 ++++++++ libnczarr/CMakeLists.txt | 9 +- libnczarr/Makefile.am | 2 + libnczarr/zdispatch.c | 41 +- libnczarr/zfile.c | 2 +- libnczarr/zfilter.c | 739 +---------------------------- libnczarr/zfilter.h | 26 +- libnczarr/zinternal.c | 57 +-- libnczarr/zinternal.h | 16 +- libnczarr/zmap.c | 6 +- libnczarr/zplugins.c | 874 +++++++++++++++++++++++++++++++++++ libnczarr/zplugins.h | 45 ++ libnczarr/zsync.c | 6 +- libnczarr/zvar.c | 7 +- libsrc4/nc4dispatch.c | 2 +- nc_test4/CMakeLists.txt | 2 + nc_test4/Makefile.am | 17 +- nc_test4/ref_fixedstring.cdl | 13 +- nc_test4/ref_fixedstring.h5 | Bin 2057 -> 0 bytes nc_test4/tst_fixedstring.sh | 7 +- unit_test/CMakeLists.txt | 8 + unit_test/Makefile.am | 22 +- unit_test/aws_config.c | 3 +- unit_test/ref_read.txt | 5 + unit_test/ref_write.txt | 9 + unit_test/run_aws_config.sh | 32 +- unit_test/run_pluginpaths.sh | 123 +++++ unit_test/tst_pluginpaths.c | 331 +++++++++++++ 61 files changed, 2471 insertions(+), 937 deletions(-) create mode 100644 include/ncplugins.h create mode 100644 libdispatch/dplugins.c create mode 100644 libhdf5/hdf5plugins.c create mode 100644 libnczarr/zplugins.c create mode 100644 libnczarr/zplugins.h delete mode 100644 nc_test4/ref_fixedstring.h5 create mode 100644 unit_test/ref_read.txt create mode 100644 unit_test/ref_write.txt create mode 100755 unit_test/run_pluginpaths.sh create mode 100644 unit_test/tst_pluginpaths.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 96852ed60a..5f852a2f98 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -78,6 +78,10 @@ add_library(netCDF::netcdf ALIAS netcdf) # configure.ac. set(NC_DISPATCH_VERSION 5) +# Version of the plugin path dispatch table. This must match the value in +# configure.ac. +set(NC_PLUGINPATH_DISPATCH_VERSION 1) + # Get system configuration, Use it to determine osname, os release, cpu. These # will be used when committing to CDash. find_program(UNAME NAMES uname) @@ -869,6 +873,7 @@ if(NETCDF_ENABLE_TESTS) ### option(NETCDF_ENABLE_BENCHMARKS "Run benchmark Tests." OFF) + set(BUILD_BENCHMARKS ${NETCDF_ENABLE_BENCHMARKS} CACHE BOOL "alias for NETCDF_ENABLE_BENCHMARKS") ### # End known-failures. diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index cfcaa3149c..4d74661788 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,6 +7,8 @@ This file contains a high-level description of this package's evolution. Release ## 4.9.3 - TBD +* Extend the netcdf API to support programmatic changes to the plugin search path. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????) for more information. + ## Known Issue * Parallel operation using `mpich 4.2.0` (the default on `Ubuntu 24.04`) results in 'unexpected results' when running `nc_test4/run_par_test.sh`. This can be fixed by removing `mpich` and associated libraries and development packages and installing `mpich 4.2.2` by hand, or by using `openmpi` provided via `apt`. diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake index c21ac3b22b..3313a24bb3 100644 --- a/cmake/dependencies.cmake +++ b/cmake/dependencies.cmake @@ -226,12 +226,13 @@ if(USE_HDF5) endif(USE_HDF5) ################################ -# Curl Libraryies +# Curl Libraries # Only needed for DAP (DAP2 or DAP4) -# and NCZARR with S3 Support +# and NCZARR S3 support +# and byterange support ################################ -if( (NETCDF_ENABLE_DAP AND (NETCDF_ENABLE_DAP2 OR NETCDF_ENABLE_DAP4 OR NETCDF_ENABLE_BYTERANGE_SUPPORT)) OR (NETCDF_ENABLE_NCZARR AND NETCDF_ENABLENCZARR_S3)) +if( NETCDF_ENABLE_DAP2 OR NETCDF_ENABLE_DAP4 OR NETCDF_ENABLE_BYTERANGE_SUPPORT OR NETCDF_ENABLE_NCZARR_S3) # See if we have libcurl find_package(CURL) diff --git a/config.h.cmake.in b/config.h.cmake.in index 3e0bff607e..a7f4f43152 100644 --- a/config.h.cmake.in +++ b/config.h.cmake.in @@ -493,9 +493,12 @@ with zip */ /* Add extra properties to _NCProperties attribute */ #cmakedefine NCPROPERTIES_EXTRA ${NCPROPERTIES_EXTRA} -/* Idspatch table version */ +/* Version for dispatch table version */ #cmakedefine NC_DISPATCH_VERSION ${NC_DISPATCH_VERSION} +/* Version for plugin path dispatch table version */ +#cmakedefine NC_PLUGINPATH_DISPATCH_VERSION ${NC_PLUGINPATH_DISPATCH_VERSION} + /* no IEEE float on this platform */ #cmakedefine NO_IEEE_FLOAT 1 diff --git a/configure.ac b/configure.ac index 33c85470eb..4ca5d82ed3 100644 --- a/configure.ac +++ b/configure.ac @@ -1153,7 +1153,7 @@ AC_ARG_ENABLE([benchmarks], are timed. We use these tests to check netCDF performance.])]) test "x$enable_benchmarks" = xyes || enable_benchmarks=no AC_MSG_RESULT($enable_benchmarks) -if test "x$enable_HDF5" = xno -a "x$enable_benchmarks" = xyes; then +if test "x$enable_hdf5" = xno -a "x$enable_benchmarks" = xyes; then AC_MSG_ERROR([Can't use benchmarks if HDF5 is disabled.]) fi AM_CONDITIONAL(BUILD_BENCHMARKS, [test x$enable_benchmarks = xyes]) @@ -1633,6 +1633,7 @@ has_hdf5_ros3=no if test "x$enable_hdf5" = xyes; then + AC_DEFINE([NETCDF_ENABLE_HDF5], [1], [if true, use HDF5]) AC_DEFINE([USE_HDF5], [1], [if true, use HDF5]) AC_DEFINE([H5_USE_16_API], [1], [use HDF5 1.6 API]) @@ -2258,6 +2259,13 @@ AX_SET_META([NC_HAS_BZ2],[$have_bz2],[yes]) AC_SUBST([NC_DISPATCH_VERSION], [5]) AC_DEFINE_UNQUOTED([NC_DISPATCH_VERSION], [${NC_DISPATCH_VERSION}], [Dispatch table version.]) +# This is the version of the plugin path dispatch table. +# If the dispatch table is changed, this should be incremented. +# If this is changed, make sure the value in +# CMakeLists.txt also changes to match. +AC_SUBST([NC_PLUGINPATH_DISPATCH_VERSION], [1]) +AC_DEFINE_UNQUOTED([NC_PLUGINPATH_DISPATCH_VERSION], [${NC_PLUGINPATH_DISPATCH_VERSION}], [Plugin Path Dispatch table version.]) + ##### # End netcdf_meta.h definitions. ##### diff --git a/include/Makefile.am b/include/Makefile.am index c20841344c..a9f1127193 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -23,7 +23,7 @@ nc4internal.h nctime.h nc3internal.h onstack.h ncrc.h ncauth.h \ ncoffsets.h nctestserver.h nc4dispatch.h nc3dispatch.h ncexternl.h \ ncpathmgr.h ncindex.h hdf4dispatch.h hdf5internal.h nc_provenance.h \ hdf5dispatch.h ncmodel.h isnan.h nccrc.h ncexhash.h ncxcache.h \ -ncjson.h ncxml.h ncs3sdk.h +ncjson.h ncxml.h ncs3sdk.h ncplugins.h if USE_DAP noinst_HEADERS += ncdap.h diff --git a/include/hdf5internal.h b/include/hdf5internal.h index 5925421d8b..fe2ba7a02c 100644 --- a/include/hdf5internal.h +++ b/include/hdf5internal.h @@ -196,6 +196,11 @@ struct NC_HDF5_Filter { unsigned int* params; /**< Params for arbitrary filter. */ }; +/* The type for the NC_FORMATX_NC_HDF5 Global State Object */ +typedef struct GlobalHDF5 { /* libhdf5 dispatcher specific parameters */ + int placeholder; /* ensure a non-empty struct */ +} GlobalHDF5; + int NC4_hdf5_filter_initialize(void); int NC4_hdf5_filter_finalize(void); int NC4_hdf5_filter_remove(NC_VAR_INFO_T* var, unsigned int id); diff --git a/include/nc4internal.h b/include/nc4internal.h index 9a2aac02be..d3b2bceab5 100644 --- a/include/nc4internal.h +++ b/include/nc4internal.h @@ -107,7 +107,10 @@ typedef enum {NC_FALSE = 0, NC_TRUE = 1} nc_bool_t; /* Forward declarations. */ struct NC_GRP_INFO; struct NC_TYPE_INFO; + +/* Opaque */ struct NCRCinfo; +struct NC_PluginPathDispatch; /** * This struct provides indexed Access to Meta-data objects. See the @@ -461,15 +464,10 @@ extern int nc_get_alignment(int* thresholdp, int* alignmentp); /* Begin to collect global state info in one place (more to do) */ typedef struct NCglobalstate { - int initialized; char* tempdir; /* track a usable temp dir */ char* home; /* track $HOME */ char* cwd; /* track getcwd */ struct NCRCinfo* rcinfo; /* Currently only one rc file per session */ - struct GlobalZarr { /* Zarr specific parameters */ - char dimension_separator; - int default_zarrformat; - } zarr; struct GlobalAWS { /* AWS S3 specific parameters/defaults */ char* default_region; char* config_file; @@ -483,6 +481,12 @@ typedef struct NCglobalstate { int alignment; } alignment; struct ChunkCache chunkcache; + /* Global dispatcher and states specific to each dispatcher + and indexed by NC_FORMATX */ + struct FormatXGlobal { + void* state[NC_FORMATX_COUNT]; /* type is opaque (like e.g. file_info_format field) */ + const struct NC_PluginPathDispatch** pluginapi; /*[NC_FORMATX_COUNT];*/ + } formatxstate; } NCglobalstate; extern struct NCglobalstate* NC_getglobalstate(void); diff --git a/include/ncdispatch.h b/include/ncdispatch.h index cee5802b48..7c7a4c3e3e 100644 --- a/include/ncdispatch.h +++ b/include/ncdispatch.h @@ -35,7 +35,7 @@ /* Given a filename, check its magic number */ /* Change magic number size from 4 to 8 to be more precise for HDF5 */ -#define MAGIC_NUMBER_LEN ((size_t)8) +#define MAGIC_NUMBER_LEN ((unsigned long long)8) #define MAGIC_HDF5_FILE 1 #define MAGIC_HDF4_FILE 2 #define MAGIC_CDF1_FILE 1 /* std classic format */ diff --git a/include/ncplugins.h b/include/ncplugins.h new file mode 100644 index 0000000000..c26b8fad54 --- /dev/null +++ b/include/ncplugins.h @@ -0,0 +1,46 @@ +/* +Copyright (c) 1998-2018 University Corporation for Atmospheric Research/Unidata +See COPYRIGHT for license information. +*/ + +/* +Common functionality for plugin paths/ +For internal use only. +*/ + +#ifndef NCPLUGINS_H +#define NCPLUGINS_H + +/* Opaque */ +struct NClist; + +/* Define the plugin path management dispatch table */ + +typedef struct NC_PluginPathDispatch { + int model; /* one of the NC_FORMATX #'s */ + int dispatch_version; + int (*initialize)(void** statep, const struct NClist* initialpaths); + int (*finalize)(void** statep); + int (*read)(void* state, size_t* ndirsp, char** dirs); + int (*write)(void* state, size_t ndirs, char** const dirs); +} NC_PluginPathDispatch; + +#if defined(__cplusplus) +extern "C" { +#endif + +/* Known Plugin Dispatchers */ +#ifdef USE_HDF5 +EXTERNL NC_PluginPathDispatch* NC4_hdf5_pluginpathtable; +#endif +#ifdef NETCDF_ENABLE_NCZARR +EXTERNL NC_PluginPathDispatch* NCZ_pluginpathtable; +#endif + +/* See the file netcdf_aux.h for plugin-related utility functions */ + +#if defined(__cplusplus) +} +#endif + +#endif /*NCPLUGINS_H*/ diff --git a/include/netcdf.h b/include/netcdf.h index ceaeed2b03..675c96302a 100644 --- a/include/netcdf.h +++ b/include/netcdf.h @@ -226,10 +226,13 @@ Use this in mode flags for both nc_create() and nc_open(). */ #define NC_FORMATX_NCZARR (10) #define NC_FORMATX_UNDEFINED (0) - /* To avoid breaking compatibility (such as in the python library), +/* Define the legal NC_FORMATX indices i.e. 0<=index | dirlist + dirlist := dir | dirlist separator dir + separator := ';' | ':' + dir := + +@param pathlist a string encoding a list of directories +@param sep one of ';' | ':' | '\0' where '\0' means use the platform's default separator. +@param ndirsp return the number of directories in dirsp +@param dirsp return a vector of strings representing the directories parsed from pathlist; caller frees +@return ::NC_NOERR + +Note that this function is called twice: first time to get the number of directories +and second to get the directories. + +Author: Dennis Heimbigner +*/ + +EXTERNL int ncaux_plugin_path_parse(const char* pathlist, char sep, size_t* ndirsp, char** dirs); + +/** +Concatenate a vector of directories with the separator between. +This is more-or-less the inverse of the ncaux_plugin_path_parse function + +The resulting string has following syntax: + paths := | dirlist + dirlist := dir | dirlist separator dir + separator := ';' | ':' + dir := + +@param ndirs the number of directories +@param dirsp the directory vector to concatenate +@param sep one of ';', ':', or '\0' +@param catlen length of the cat arg including a nul terminator +@param cat user provided space for holding the concatenation; nul termination guaranteed if catlen > 0. +@return ::NC_NOERR +@return ::NC_EINVAL for illegal arguments + +Note that this function is called twice: first time to get the expected size of +the concatenated string and second to get the contents of the concatenation. + +Author: Dennis Heimbigner +*/ + +EXTERNL int ncaux_plugin_path_tostring(size_t ndirs, char** const dirs, char sep, size_t* catlen, char* cat); + + +/* +Reclaim a char** object possibly produced by ncaux_plugin_parse function. + +@param veclen the number of entries in vec +@param vec a char** vectore +@return ::NC_NOERR +@return ::NC_EINVAL for illegal arguments +*/ + +EXTERNL int ncaux_plugin_path_freestringvec(size_t veclen, char** vec); + #if defined(__cplusplus) } #endif diff --git a/include/netcdf_filter.h b/include/netcdf_filter.h index 3c3c88c79b..c46ef654e8 100644 --- a/include/netcdf_filter.h +++ b/include/netcdf_filter.h @@ -12,8 +12,8 @@ #ifndef NETCDF_FILTER_H #define NETCDF_FILTER_H 1 -/* API for libdispatch/dfilter.c -*/ +/**************************************************/ +/* API for libdispatch/dfilter.c */ /* Must match values in */ #ifndef H5Z_FILTER_DEFLATE @@ -110,9 +110,65 @@ EXTERNL int nc_inq_var_zstandard(int ncid, int varid, int* hasfilterp, int *leve EXTERNL int nc_def_var_blosc(int ncid, int varid, unsigned subcompressor, unsigned level, unsigned blocksize, unsigned addshuffle); EXTERNL int nc_inq_var_blosc(int ncid, int varid, int* hasfilterp, unsigned* subcompressorp, unsigned* levelp, unsigned* blocksizep, unsigned* addshufflep); +/* Filter path query/set */ +EXTERNL int nc_filter_path_query(int id); + +/**************************************************/ +/* API for libdispatch/dplugin.c */ + +/* Plugin path functions */ + +/** + * This function is called as part of nc_initialize. + * Its purpose is to initialize the plugin paths state. + * @return NC_NOERR + * @author Dennis Heimbigner +*/ + +EXTERNL int nc_plugin_path_initialize(void); + +/** + * This function is called as part of nc_finalize() + * Its purpose is to clean-up plugin path state. + * @return NC_NOERR + * @author Dennis Heimbigner +*/ + +EXTERNL int nc_plugin_path_finalize(void); + +/** + * Return the current sequence of directories in the internal plugin path list. + * Since this function does not modify the plugin path, it can be called at any time. + * @param formatx specify which dispatch implementatio to read: currently NC_FORMATX_NC_HDF5 or NC_FORMATX_NCZARR. + * @param ndirsp return the number of dirs in the internal path list + * @param dirs memory for storing the sequence of directies in the internal path list. + * @return NC_NOERR + * @author Dennis Heimbigner + * + * As a rule, this function needs to be called twice. + * The first time with npaths not NULL and pathlist set to NULL + * to get the size of the path list. + * The second time with pathlist not NULL to get the actual sequence of paths. +*/ + +EXTERNL int nc_plugin_path_read(int formatx, size_t* ndirsp, char** dirs); + +/** + * Empty the current internal path sequence + * and replace with the sequence of directories argument. + * + * Using a paths argument of NULL or npaths argument of 0 will clear the set of plugin paths. + * @param formatx specify which dispatch implementation to write: currently NC_FORMATX_NC_HDF5 or NC_FORMATX_NCZARR. + * @param ndirs length of the dirs argument + * @param dirs to overwrite the current internal path list + * @return NC_NOERR + * @author Dennis Heimbigner +*/ + +EXTERNL int nc_plugin_path_write(int formatx, size_t ndirs, char** const dirs); + #if defined(__cplusplus) } #endif -/**************************************************/ #endif /* NETCDF_FILTER_H */ diff --git a/include/netcdf_json.h b/include/netcdf_json.h index 5d77cadb34..74231a8a8f 100644 --- a/include/netcdf_json.h +++ b/include/netcdf_json.h @@ -1086,7 +1086,6 @@ NCJinsertstring(NCjson* object, const char* key, const char* value) else NCJnewstring(NCJ_STRING,value,&jvalue); NCJinsert(object,key,jvalue); -done: return NCJTHROW(stat); } @@ -1100,7 +1099,6 @@ NCJinsertint(NCjson* object, const char* key, long long ivalue) snprintf(digits,sizeof(digits),"%lld",ivalue); NCJnewstring(NCJ_STRING,digits,&jvalue); NCJinsert(object,key,jvalue); -done: return NCJTHROW(stat); } @@ -1305,6 +1303,8 @@ netcdf_supresswarnings(void) ignore = (void*)NCJparse; ignore = (void*)NCJdump; ignore = (void*)NCJtotext; + ignore = (void*)NCJinsertstring; + ignore = (void*)NCJinsertint; ignore = ignore; } #endif /*NETCDF_JSON_H*/ diff --git a/libdispatch/CMakeLists.txt b/libdispatch/CMakeLists.txt index 079d009144..f9fe5501cc 100644 --- a/libdispatch/CMakeLists.txt +++ b/libdispatch/CMakeLists.txt @@ -33,7 +33,7 @@ set_property(SOURCE dinstance_intern.c dinstance.c dvarput.c # Netcdf-4 only functions. Must be defined even if not used target_sources(dispatch PRIVATE - dgroup.c dvlen.c dcompound.c dtype.c denum.c dopaque.c dfilter.c + dgroup.c dvlen.c dcompound.c dtype.c denum.c dopaque.c dfilter.c dplugins.c ) if(BUILD_V2) diff --git a/libdispatch/Makefile.am b/libdispatch/Makefile.am index 9aae84ba0f..bc76c6332e 100644 --- a/libdispatch/Makefile.am +++ b/libdispatch/Makefile.am @@ -34,7 +34,7 @@ libdispatch_la_SOURCES += drc.c # Add functions only found in netCDF-4. # They are always defined, even if they just return an error libdispatch_la_SOURCES += dgroup.c dvlen.c dcompound.c dtype.c denum.c \ -dopaque.c dfilter.c +dopaque.c dfilter.c dplugins.c # Add V2 API convenience library if needed. if BUILD_V2 diff --git a/libdispatch/daux.c b/libdispatch/daux.c index a95dd8977a..374efc60dd 100644 --- a/libdispatch/daux.c +++ b/libdispatch/daux.c @@ -30,6 +30,7 @@ See COPYRIGHT for license information. #include "nclog.h" #include "ncrc.h" #include "netcdf_filter.h" +#include "ncplugins.h" struct NCAUX_FIELD { char* name; @@ -57,6 +58,7 @@ static int computefieldinfo(struct NCAUX_CMPD* cmpd); static int filterspec_cvt(const char* txt, size_t* nparamsp, unsigned int* params); EXTERNL int nc_dump_data(int ncid, nc_type xtype, void* memory, size_t count, char** bufp); +EXTERNL int nc_parse_plugin_pathlist(const char* path0, NClist* dirlist); /**************************************************/ /* @@ -615,7 +617,7 @@ ncaux_h5filterspec_parselist(const char* txt0, int* formatp, size_t* nspecsp, NC done: nullfree(spec); if(vector) { - int i; + size_t i; for(i=0;i 0 */ + if((path = malloc(plen+1+1))==NULL) {stat = NC_ENOMEM; goto done;} + memcpy(path,pathlist0,plen); + path[plen] = '\0'; path[plen+1] = '\0'; /* double null term */ + + for(count=0,p=path;*p;p++) { + if(strchr(seps,*p) == NULL) + continue; /* non-separator */ + else { + *p = '\0'; + count++; + } + } + count++; /* count last piece */ + + /* capture the parsed pieces */ + for(p=path,i=0;i 0) + nclistpush(vec, p); /* use the contents of path */ + p = p+len+1; /* point to next piece */ + } + + if(dirs) { + for(i=0;i 0. +@return ::NC_NOERR +@return ::NC_EINVAL for illegal arguments + +Note that this function is called twice: first time to get the expected size of +the concatenated string and second to get the contents of the concatenation. +*/ +EXTERNL int +ncaux_plugin_path_tostring(size_t ndirs, char** const dirs, char sep, size_t* catlen, char* cat) +{ + int stat = NC_NOERR; + NCbytes* buf = ncbytesnew(); + size_t i; + + if(sep == '\0') +#ifdef _WIN32 + sep = ';'; +#else + sep = ':'; +#endif + if(cat != NULL) *cat = '\0'; /* Make sure it is nul terminated */ + if(ndirs > 0) { + for(i=0;i0) ncbytesappend(buf,sep); + if(dirs[i] != NULL) ncbytescat(buf,dirs[i]); + } + } + ncbytesnull(buf); + if(cat) + memcpy(cat,ncbytescontents(buf),ncbyteslength(buf)+1); /* include nul termiator */ + if(catlen) *catlen = ncbyteslength(buf)+1; /* overwrite with the true cat length */ + ncbytesfree(buf); + return stat; +} + +/* +Reclaim a char** object possibly produced by ncaux_plugin_parse function. + +@param veclen the number of entries in vec +@param vec a char** vectore +@return ::NC_NOERR +@return ::NC_EINVAL for illegal arguments +*/ +EXTERNL int +ncaux_plugin_path_freestringvec(size_t veclen, char** vec) +{ + int stat = NC_NOERR; + size_t i; + if(vec == NULL) goto done; + for(i=0;ircinfo = calloc(1,sizeof(struct NCRCinfo)))==NULL) {stat = NC_ENOMEM; goto done;} @@ -176,6 +180,12 @@ NC_createglobalstate(void) if((nc_globalstate->rcinfo->s3profiles = nclistnew())==NULL) {stat = NC_ENOMEM; goto done;} + /* plugin path state */ + if((nc_globalstate->formatxstate.pluginapi = (const NC_PluginPathDispatch**)calloc(NC_FORMATX_COUNT,sizeof(NC_PluginPathDispatch*)))==NULL) + {stat = NC_ENOMEM; goto done;} + memset(nc_globalstate->formatxstate.state,0,NC_FORMATX_COUNT*sizeof(void*)); + if((stat = nc_plugin_path_initialize())) goto done; + /* Get environment variables */ if(getenv(NCRCENVIGNORE) != NULL) nc_globalstate->rcinfo->ignore = 1; @@ -216,6 +226,15 @@ NC_freeglobalstate(void) NC_rcclear(nc_globalstate->rcinfo); free(nc_globalstate->rcinfo); } + { + size_t i; + (void)nc_plugin_path_finalize(); + /* Verify states reclaimed */ + for(i=0;iformatxstate.state[i] == NULL); + memset(nc_globalstate->formatxstate.state,0,NC_FORMATX_COUNT*sizeof(void*)); + nullfree(nc_globalstate->formatxstate.pluginapi); + } free(nc_globalstate); nc_globalstate = NULL; } diff --git a/libdispatch/dinfermodel.c b/libdispatch/dinfermodel.c index a381ee0082..938dc00089 100644 --- a/libdispatch/dinfermodel.c +++ b/libdispatch/dinfermodel.c @@ -1284,7 +1284,7 @@ check_file_type(const char *path, int omode, int use_parallel, if((status = openmagic(&magicinfo))) goto done; /* Verify we have a large enough file */ - if(magicinfo.filelen < (unsigned long long)MAGIC_NUMBER_LEN) + if(MAGIC_NUMBER_LEN >= (unsigned long long)magicinfo.filelen) {status = NC_ENOTNC; goto done;} if((status = readmagic(&magicinfo,0L,magic)) != NC_NOERR) { status = NC_ENOTNC; @@ -1306,7 +1306,7 @@ check_file_type(const char *path, int omode, int use_parallel, { size_t pos = 512L; for(;;) { - if((pos+MAGIC_NUMBER_LEN) > magicinfo.filelen) + if((pos+MAGIC_NUMBER_LEN) > (unsigned long long)magicinfo.filelen) {status = NC_ENOTNC; goto done;} if((status = readmagic(&magicinfo,pos,magic)) != NC_NOERR) {status = NC_ENOTNC; goto done; } diff --git a/libdispatch/dinstance.c b/libdispatch/dinstance.c index ad9141474c..cef9b2e893 100644 --- a/libdispatch/dinstance.c +++ b/libdispatch/dinstance.c @@ -484,17 +484,18 @@ static int dump_compound(int ncid, nc_type xtype, size_t size, size_t nfields, Position* offset, NCbytes* buf) { int stat = NC_NOERR; - size_t i; + int i; ptrdiff_t saveoffset; int ndims; int dimsizes[NC_MAX_VAR_DIMS]; + int fid; saveoffset = offset->offset; ncbytescat(buf,"<"); /* Get info about each field in turn and dump it */ - for(int fid=0;fid +#include +#include +#ifdef _MSC_VER +#include +#endif + +#include "netcdf.h" +#include "netcdf_filter.h" +#include "netcdf_aux.h" +#include "ncdispatch.h" +#include "nc4internal.h" +#include "nclog.h" +#include "ncbytes.h" +#include "ncplugins.h" + +/* +Unified plugin related code +*/ +/**************************************************/ +/* Plugin-path API */ + +/* list of environment variables to check for plugin roots */ +#define PLUGIN_ENV "HDF5_PLUGIN_PATH" +#define PLUGIN_DIR_UNIX "/usr/local/hdf5/plugin" +#define PLUGIN_DIR_WIN "%s/hdf5/lib/plugin" +#define WIN32_ROOT_ENV "ALLUSERSPROFILE" + +static int NC_plugin_path_initialized = 0; + +/** + * This function is called as part of nc_initialize. + * Its purpose is to initialize the plugin paths state. + * + * @return ::NC_NOERR + * + * @author Dennis Heimbigner +*/ + +EXTERNL int +nc_plugin_path_initialize(void) +{ + int stat = NC_NOERR; + struct NCglobalstate* gs = NULL; + char* defaultpluginpath = NULL; + const char* pluginroots = NULL; + NClist* dirs = NULL; + size_t ndirs; + int i; + + if(NC_plugin_path_initialized != 0) goto done; + NC_plugin_path_initialized = 1; + + gs = NC_getglobalstate(); + dirs = nclistnew(); + + /* Setup the plugin path default */ + { +#ifdef _WIN32 + const char* win32_root; + char dfalt[4096]; + win32_root = getenv(WIN32_ROOT_ENV); + if(win32_root != NULL && strlen(win32_root) > 0) { + snprintf(dfalt,sizeof(dfalt),PLUGIN_DIR_WIN,win32_root); + defaultpluginpath = strdup(dfalt); + } +#else /*!_WIN32*/ + defaultpluginpath = strdup(PLUGIN_DIR_UNIX); +#endif + } + + /* Find the plugin directory root(s) */ + pluginroots = getenv(PLUGIN_ENV); /* Usually HDF5_PLUGIN_PATH */ + if(pluginroots != NULL && strlen(pluginroots) == 0) pluginroots = NULL; + if((stat = ncaux_plugin_path_parse(pluginroots,0,&ndirs,NULL))) goto done; + if(ndirs > 0) { + char** contents; + nclistsetlength(dirs,ndirs); /* may modify contents memory */ + contents = (char**)nclistcontents(dirs); + if((stat = ncaux_plugin_path_parse(pluginroots,0,&ndirs,contents))) goto done; + } + /* Add the default to end of the dirs list if not already there */ + if(defaultpluginpath != NULL && !nclistmatch(dirs,defaultpluginpath,0)) { + nclistpush(dirs,defaultpluginpath); + defaultpluginpath = NULL; + } + + /* Initialize all the plugin path dispatchers and state*/ +#ifdef USE_HDF5 + gs->formatxstate.pluginapi[NC_FORMATX_NC_HDF5] = NC4_hdf5_pluginpathtable; +#endif +#ifdef NETCDF_ENABLE_NCZARR_FILTERS + gs->formatxstate.pluginapi[NC_FORMATX_NCZARR] = NCZ_pluginpathtable; +#endif + /* Initialize all the plugin path dispatcher states */ + for(i=0;iformatxstate.pluginapi[i] != NULL) { + if((stat = gs->formatxstate.pluginapi[i]->initialize(&gs->formatxstate.state[i], dirs))) goto done; + assert(gs->formatxstate.state[i] != NULL); + } + } + +done: + nullfree(defaultpluginpath); + nclistfreeall(dirs); + return NCTHROW(stat); +} + +/** + * This function is called as part of nc_finalize() + * Its purpose is to clean-up plugin path state. + * + * @return ::NC_NOERR + * + * @author Dennis Heimbigner +*/ + +int +nc_plugin_path_finalize(void) +{ + int stat = NC_NOERR; + struct NCglobalstate* gs = NC_getglobalstate(); + int i; + + if(NC_plugin_path_initialized == 0) goto done; + NC_plugin_path_initialized = 0; + + /* Finalize all the plugin path dispatchers */ + for(i=0;iformatxstate.state[i] != NULL) { + if((stat = gs->formatxstate.pluginapi[i]->finalize(&gs->formatxstate.state[i]))) goto done; + gs->formatxstate.state[i] = NULL; + } + } +done: + return NCTHROW(stat); +} + +/** + * Return the current sequence of directories in the internal plugin path list. + * Since this function does not modify the plugin path, it can be called at any time. + * + * @param formatx the dispatcher from which to get the info + * @param ndirsp return the number of paths in the path list + * @param dirs copy the sequence of directories in the path list into this; caller must free the copied strings + * + * @return ::NC_NOERR + * @return ::NC_EINVAL if formatx is unknown or zero + * + * @author Dennis Heimbigner + * + * As a rule, this function needs to be called twice. The first time + * with ndirsp not NULL and dirs set to NULL to get the size of + * the path list. The second time with dirs not NULL to get the + * actual sequence of paths. +*/ + +EXTERNL int +nc_plugin_path_read(int formatx, size_t* ndirsp, char** dirs) +{ + int stat = NC_NOERR; + struct NCglobalstate* gs = NC_getglobalstate(); + + if(formatx < 0 || formatx >= NC_FORMATX_COUNT) {stat = NC_EINVAL; goto done;} + if(!NC_initialized) nc_initialize(); + /* read functions can only apply to specific formatx */ + if(formatx == 0) {stat = NC_EINVAL; goto done;} + + if(gs->formatxstate.pluginapi[formatx] == NULL || gs->formatxstate.state[formatx] == NULL) {stat = NC_EINVAL; goto done;} + if((stat = gs->formatxstate.pluginapi[formatx]->read(gs->formatxstate.state[formatx],ndirsp,dirs))) goto done; +done: + return NCTHROW(stat); +} + +/** + * Empty the current internal path sequence + * and replace with the sequence of directories + * specified in the arguments. + * If ndirs == 0 the path list will be cleared + * + * @param formatx the dispatcher to which to write; zero means all dispatchers + * @param ndirs number of entries in dirs arg + * @param dirs the actual directory paths + * + * @return ::NC_NOERR + * @return ::NC_EINVAL if formatx is unknown or ndirs > 0 and dirs == NULL + * + * @author Dennis Heimbigner + * + * Note that modifying the plugin paths must be done "atomically". + * That is, in a multi-threaded environment, it is important that + * the sequence of actions involved in setting up the plugin paths + * must be done by a single processor or in some other way as to + * guarantee that two or more processors are not simultaneously + * accessing the plugin path read/write operations. + * + * As an example, assume there exists a mutex lock called PLUGINLOCK. + * Then any processor accessing the plugin paths should operate + * as follows: + * 
+ * lock(PLUGINLOCK);
+ * nc_plugin_path_read(...);
+ * 
+ * nc_plugin_path_write(...);
+ * unlock(PLUGINLOCK);
+ *
+*/ + +EXTERNL int +nc_plugin_path_write(int formatx, size_t ndirs, char** const dirs) +{ + int i,stat = NC_NOERR; + struct NCglobalstate* gs = NC_getglobalstate(); + + if(formatx < 0 || formatx >= NC_FORMATX_COUNT) {stat = NC_EINVAL; goto done;} + if(ndirs > 0 && dirs == NULL) {stat = NC_EINVAL; goto done;} + if(!NC_initialized) nc_initialize(); + /* forall dispatchers */ + for(i=1;iformatxstate.pluginapi[i] == NULL || gs->formatxstate.state[i] == NULL) continue; + if((stat=gs->formatxstate.pluginapi[i]->write(gs->formatxstate.state[i],ndirs,dirs))) goto done; + } + } +done: + return NCTHROW(stat); +} diff --git a/libdispatch/ds3util.c b/libdispatch/ds3util.c index 3b81d18c3e..2b81f342bc 100644 --- a/libdispatch/ds3util.c +++ b/libdispatch/ds3util.c @@ -50,6 +50,8 @@ static int endswith(const char* s, const char* suffix); static void freeentry(struct AWSentry* e); static int awsparse(const char* text, NClist* profiles); +extern void awsprofiles(void); + /**************************************************/ /* Capture environmental Info */ @@ -255,7 +257,7 @@ NC_s3urlrebuild(NCURI* url, NCS3INFO* s3, NCURI** newurlp) ncurirebuild(newurl); /* return various items */ #ifdef AWSDEBUG - fprintf(stderr,">>> NC_s3urlrebuild: final=%s bucket=|%s| region=|%s|\n",uri->uri,bucket,region); + fprintf(stderr,">>> NC_s3urlrebuild: final=%s bucket=|%s| region=|%s|\n",newurl->uri,bucket,region); #endif if(newurlp) {*newurlp = newurl; newurl = NULL;} if(s3 != NULL) { @@ -507,22 +509,7 @@ NC_aws_load_credentials(NCglobalstate* gstate) gstate->rcinfo->s3profiles = profiles; profiles = NULL; #ifdef AWSDEBUG - {int i,j; - fprintf(stderr,">>> profiles:\n"); - for(i=0;ircinfo->s3profiles);i++) { - struct AWSprofile* p = (struct AWSprofile*)nclistget(gstate->rcinfo->s3profiles,i); - fprintf(stderr," [%s]",p->name); - for(j=0;jentries);j++) { - struct AWSentry* e = (struct AWSentry*)nclistget(p->entries,j); - if(strcmp(e->key,"aws_access_key_id") - fprintf(stderr," %s=%d",e->key,(int)strlen(e->value)); - else if(strcmp(e->key,"aws_secret_access_key") - fprintf(stderr," %s=%d",e->key,(int)strlen(e->value)); - else fprintf(stderr," %s=%s",e->key,e->value); - } - fprintf(stderr,"\n"); - } - } + awsprofiles(); #endif done: @@ -666,7 +653,7 @@ NC_getdefaults3region(NCURI* uri, const char** regionp) if(region == NULL) region = NC_getglobalstate()->aws.default_region; /* Force use of the Amazon default */ #ifdef AWSDEBUG - fprintf(stderr,">>> activeregion = |%s|\n",region)); + fprintf(stderr,">>> activeregion = |%s|\n",region); #endif if(regionp) *regionp = region; return stat; @@ -936,10 +923,52 @@ freeentry(struct AWSentry* e) { if(e) { #ifdef AWSDEBUG -fprintf(stderr,">>> freeentry: key=%p value=%p\n",e->key,e->value); +fprintf(stderr,">>> freeentry: key=%s value=%s\n",e->key,e->value); #endif nullfree(e->key); nullfree(e->value); nullfree(e); } } + +/* Provide profile-related dumper(s) */ +void +awsdumpprofile(struct AWSprofile* p) +{ + size_t j; + if(p == NULL) { + fprintf(stderr," "); + goto done; + } + fprintf(stderr," [%s]",p->name); + if(p->entries == NULL) { + fprintf(stderr,""); + goto done; + } + for(j=0;jentries);j++) { + struct AWSentry* e = (struct AWSentry*)nclistget(p->entries,j); + fprintf(stderr," %s=%s",e->key,e->value); + } +done: + fprintf(stderr,"\n"); +} + +void +awsdumpprofiles(NClist* profiles) +{ + size_t i; + NCglobalstate* gs = NC_getglobalstate(); + for(i=0;ircinfo->s3profiles);i++) { + struct AWSprofile* p = (struct AWSprofile*)nclistget(profiles,i); + awsdumpprofile(p); + } +} + +void +awsprofiles(void) +{ + NCglobalstate* gs = NC_getglobalstate(); + fprintf(stderr,">>> profiles from global->rcinfo->s3profiles:\n"); + awsdumpprofiles(gs->rcinfo->s3profiles); +} + diff --git a/libdispatch/ncbytes.c b/libdispatch/ncbytes.c index d12c27b432..02ac617075 100644 --- a/libdispatch/ncbytes.c +++ b/libdispatch/ncbytes.c @@ -23,7 +23,7 @@ static int ncbytesfail(void) { fflush(stdout); - fprintf(stderr,"bytebuffer failure\n"); + fprintf(stderr,"NCbytes failure\n"); fflush(stderr); #ifdef NCBYTESDEBUG abort(); diff --git a/libdispatch/ncjson.c b/libdispatch/ncjson.c index 148415666e..4cf21e96d0 100644 --- a/libdispatch/ncjson.c +++ b/libdispatch/ncjson.c @@ -927,7 +927,6 @@ NCJinsertstring(NCjson* object, const char* key, const char* value) else NCJnewstring(NCJ_STRING,value,&jvalue); NCJinsert(object,key,jvalue); -done: return NCJTHROW(stat); } @@ -941,7 +940,6 @@ NCJinsertint(NCjson* object, const char* key, long long ivalue) snprintf(digits,sizeof(digits),"%lld",ivalue); NCJnewstring(NCJ_STRING,digits,&jvalue); NCJinsert(object,key,jvalue); -done: return NCJTHROW(stat); } @@ -1146,5 +1144,7 @@ netcdf_supresswarnings(void) ignore = (void*)NCJparse; ignore = (void*)NCJdump; ignore = (void*)NCJtotext; + ignore = (void*)NCJinsertstring; + ignore = (void*)NCJinsertint; ignore = ignore; } diff --git a/libdispatch/nclist.c b/libdispatch/nclist.c index 56fd3d61ca..7c6adde86e 100644 --- a/libdispatch/nclist.c +++ b/libdispatch/nclist.c @@ -13,6 +13,8 @@ int nclistisnull(void* e) {return e == NULL;} +#define NCLISTDEBUG 1 + #ifndef TRUE #define TRUE 1 #endif @@ -23,6 +25,18 @@ int nclistisnull(void* e) {return e == NULL;} #define DEFAULTALLOC 16 #define ALLOCINCR 16 +static int +nclistfail(void) +{ + fflush(stdout); + fprintf(stderr,"NClist failure\n"); + fflush(stderr); +#ifdef NCLISTDEBUG + abort(); +#endif + return FALSE; +} + NClist* nclistnew(void) { NClist* l; @@ -83,7 +97,7 @@ int nclistsetalloc(NClist* l, size_t sz) { void** newcontent = NULL; - if(l == NULL) return FALSE; + if(l == NULL) return nclistfail(); if(sz <= 0) {sz = (l->length?2*l->length:DEFAULTALLOC);} if(l->alloc >= sz) {return TRUE;} newcontent=(void**)calloc(sz,sizeof(void*)); @@ -99,8 +113,8 @@ nclistsetalloc(NClist* l, size_t sz) int nclistsetlength(NClist* l, size_t newlen) { - if(l == NULL) return FALSE; - if(newlen > l->alloc && !nclistsetalloc(l,newlen)) return FALSE; + if(l == NULL) return nclistfail(); + if(newlen > l->alloc && !nclistsetalloc(l,newlen)) return nclistfail(); if(newlen > l->length) { /* clear any extension */ memset(&l->content[l->length],0,(newlen - l->length)*sizeof(void*)); @@ -112,7 +126,8 @@ nclistsetlength(NClist* l, size_t newlen) void* nclistget(const NClist* l, size_t index) { - if(l == NULL || l->length == 0) return NULL; + if(l == NULL) return (nclistfail(),NULL); + if(l->length == 0) return NULL; if(index >= l->length) return NULL; return l->content[index]; } @@ -123,10 +138,10 @@ nclistget(const NClist* l, size_t index) int nclistset(NClist* l, size_t index, void* elem) { - if(l == NULL) return FALSE; - if(!nclistsetalloc(l,index+1)) return FALSE; + if(l == NULL) return nclistfail(); + if(!nclistsetalloc(l,index+1)) return nclistfail(); if(index >= l->length) { - if(!nclistsetlength(l,index+1)) return FALSE; + if(!nclistsetlength(l,index+1)) return nclistfail(); } l->content[index] = elem; return TRUE; @@ -136,11 +151,13 @@ nclistset(NClist* l, size_t index, void* elem) int nclistinsert(NClist* l, size_t index, void* elem) { - long i; /* do not make unsigned */ - if(l == NULL) return FALSE; - if(index > l->length) return FALSE; + size_t i; + if(l == NULL) return nclistfail(); + if(index > l->length) return nclistfail(); nclistsetalloc(l,0); - for(i=(long)l->length;i>index;i--) l->content[i] = l->content[i-1]; + if(l->length > 0) { + for(i=l->length;i>index;i--) l->content[i] = l->content[i-1]; + } l->content[index] = elem; l->length++; return TRUE; @@ -149,8 +166,10 @@ nclistinsert(NClist* l, size_t index, void* elem) int nclistpush(NClist* l, const void* elem) { - if(l == NULL) return FALSE; + if(l == NULL) return nclistfail(); if(l->length >= l->alloc) nclistsetalloc(l,0); + if(l->content == NULL) + nclistsetalloc(l,0); l->content[l->length] = (void*)elem; l->length++; return TRUE; @@ -159,7 +178,8 @@ nclistpush(NClist* l, const void* elem) void* nclistpop(NClist* l) { - if(l == NULL || l->length == 0) return NULL; + if(l == NULL) return (nclistfail(),NULL); + if(l->length == 0) return NULL; l->length--; return l->content[l->length]; } @@ -167,7 +187,8 @@ nclistpop(NClist* l) void* nclisttop(NClist* l) { - if(l == NULL || l->length == 0) return NULL; + if(l == NULL) return (nclistfail(),NULL); + if(l->length == 0) return NULL; return l->content[l->length - 1]; } @@ -176,7 +197,8 @@ nclistremove(NClist* l, size_t i) { size_t len; void* elem; - if(l == NULL || (len=l->length) == 0) return NULL; + if(l == NULL) return (nclistfail(),NULL); + if((len=l->length) == 0) return NULL; if(i >= len) return NULL; elem = l->content[i]; for(i+=1;icontent[i-1] = l->content[i]; @@ -219,7 +241,8 @@ nclistelemremove(NClist* l, void* elem) size_t len; size_t i; int found = 0; - if(l == NULL || (len=l->length) == 0) return 0; + if(l == NULL) return nclistfail(); + if((len=l->length) == 0) return 0; for(i=0;icontent[i]; if(elem == candidate) { @@ -242,7 +265,8 @@ nclistunique(NClist* l) { size_t i,j,k,len; void** content; - if(l == NULL || l->length == 0) return 1; + if(l == NULL) return nclistfail(); + if(l->length == 0) return 1; len = l->length; content = l->content; for(i=0;ilength == 0) return 1; + if(l == NULL) return nclistfail(); + if(l->length == 0) return 1; nclistpush(l,NULL); nclistsetlength(l,l->length-1); return 1; diff --git a/libdispatch/ncs3sdk_h5.c b/libdispatch/ncs3sdk_h5.c index 0f99dfb473..359ab2f8b3 100644 --- a/libdispatch/ncs3sdk_h5.c +++ b/libdispatch/ncs3sdk_h5.c @@ -959,11 +959,12 @@ s3objectsinfo(NClist* contents, NClist* keys, NClist* lengths) for(i=0;i #ifdef _WIN32 #include #endif +#include "hdf5internal.h" +#include "hdf5err.h" /* For BAIL2 */ +#include "ncplugins.h" #undef DEBUGH5 diff --git a/libhdf5/hdf5plugins.c b/libhdf5/hdf5plugins.c new file mode 100644 index 0000000000..e1544aff59 --- /dev/null +++ b/libhdf5/hdf5plugins.c @@ -0,0 +1,191 @@ +/* Copyright 2003-2018, University Corporation for Atmospheric + * Research. See the COPYRIGHT file for copying and redistribution + * conditions. + */ +/** + * @file @internal netcdf-4 functions for the plugin list. + * + * @author Dennis Heimbigner + */ + +#include "config.h" +#include +#include +#include "netcdf.h" +#include "ncbytes.h" +#include "hdf5internal.h" +#include "hdf5debug.h" +#include "ncplugins.h" + +#undef TPLUGINS + +/**************************************************/ +/* Forward */ + +static int NC4_hdf5_plugin_path_initialize(void** statep, const NClist* initialpaths); +static int NC4_hdf5_plugin_path_finalize(void** statep); +static int NC4_hdf5_plugin_path_read(void* state, size_t* ndirsp, char** dirs); +static int NC4_hdf5_plugin_path_write(void* state, size_t ndirs, char** const dirs); + +/**************************************************/ +/** + * @file + * @internal + * Internal netcdf hdf5 plugin path functions. + * + * @author Dennis Heimbigner + */ +/**************************************************/ +/* The HDF5 Plugin Path Dispatch table and functions */ + +NC_PluginPathDispatch NC4_hdf5_pluginpathdispatch = { + NC_FORMATX_NC_HDF5, + NC_PLUGINPATH_DISPATCH_VERSION, + NC4_hdf5_plugin_path_initialize, + NC4_hdf5_plugin_path_finalize, + NC4_hdf5_plugin_path_read, + NC4_hdf5_plugin_path_write +}; + +NC_PluginPathDispatch* NC4_hdf5_pluginpathtable = &NC4_hdf5_pluginpathdispatch; + +/**************************************************/ + +/** + * This function is called as part of nc_initialize. + * Its purpose is to initialize the plugin paths state. + * @return NC_NOERR + * @author Dennis Heimbigner +*/ +static int +NC4_hdf5_plugin_path_initialize(void** statep, const NClist* initialpaths) +{ + int stat = NC_NOERR; + GlobalHDF5* g5 = NULL; + + NC_UNUSED(initialpaths); /* Let HDF5 do its own thing */ + + assert(statep != NULL); + if(*statep != NULL) goto done; /* already initialized */ + + if((g5 = (GlobalHDF5*)calloc(1,sizeof(GlobalHDF5)))==NULL) {stat = NC_ENOMEM; goto done;} + *statep = (void*)g5; g5 = NULL; +done: + nullfree(g5); + return THROW(stat); +} + +/** + * This function is called as part of nc_finalize() + * Its purpose is to clean-up plugin path state. + * @return NC_NOERR + * @author Dennis Heimbigner +*/ +static int +NC4_hdf5_plugin_path_finalize(void** statep) +{ + int stat = NC_NOERR; + GlobalHDF5* g5 = NULL; + + assert(statep != NULL); + if(*statep == NULL) goto done; /* already finalized */ + g5 = (GlobalHDF5*)(*statep); + *statep = NULL; + +done: + nullfree(g5); + return THROW(stat); +} + +/** + * Return the current sequence of directories in the internal plugin path list. + * Since this function does not modify the plugin path, it can be called at any time. + * + * @param stat the per-dispatcher global state + * @param ndirsp return the number of paths in the path list + * @param dirs copy the sequence of directories in the path list into this; caller must free the copied strings + * + * @return ::NC_NOERR + * @return ::NC_EINVAL if formatx is unknown or zero + * + * @author Dennis Heimbigner + * + * As a rule, this function needs to be called twice. The first time + * with ndirsp not NULL and dirs set to NULL to get the size of + * the path list. The second time with dirs not NULL to get the + * actual sequence of paths. +*/ + +static int +NC4_hdf5_plugin_path_read(void* state, size_t* ndirsp, char** dirs) +{ + int stat = NC_NOERR; + herr_t hstat = 0; + unsigned ndirs = 0; + char* dir = NULL; + + if((hstat = H5PLsize(&ndirs))<0) goto done; + if(ndirsp) *ndirsp = ndirs; + if(ndirs > 0 && dirs != NULL) { + unsigned i; + ssize_t dirlen = 0; + for(i=0;i 0 and dirs == NULL + * + * @author Dennis Heimbigner +*/ + +static int +NC4_hdf5_plugin_path_write(void* state, size_t ndirs, char** dirs) +{ + int stat = NC_NOERR; + herr_t hstat = 0; + unsigned hndirs = 0; + + /* Clear the current path list */ + if((hstat = H5PLsize(&hndirs))<0) goto done; + if(hndirs > 0) { + unsigned i; + for(i=0;i 0 && dirs != NULL) { + size_t i; + for(i=0;iformatxstate.state[NC_FORMATX_NCZARR]; + assert(gz != NULL); + + /* Defaults */ + gz->dimension_separator = DFALT_DIM_SEPARATOR; + dimsep = NC_rclookup("ZARR.DIMENSION_SEPARATOR",NULL,NULL); + if(dimsep != NULL) { + /* Verify its value */ + if(dimsep != NULL && strlen(dimsep) == 1 && islegaldimsep(dimsep[0])) + gz->dimension_separator = dimsep[0]; + } stat = NCZ_provenance_init(); - if(stat) ncz_initialized = 1; +#ifdef NETCDF_ENABLE_NCZARR_FILTERS + NCZ_filter_initialize(); +#endif +done: return stat; } @@ -140,9 +162,17 @@ NCZ_initialize(void) int NCZ_finalize(void) { - NCZ_finalize_internal(); + int stat = NC_NOERR; + /* Reclaim global resources */ + ncz_initialized = 0; +#ifdef NETCDF_ENABLE_NCZARR_FILTERS + NCZ_filter_finalize(); +#endif +#ifdef NETCDF_ENABLE_S3 + NCZ_s3finalize(); +#endif NCZ_provenance_finalize(); - return NC_NOERR; + return stat; } static int @@ -198,7 +228,6 @@ NCZ_inq_filter_avail(int ncid, unsigned id) return REPORT(NC_ENOFILTER,"inq_filter_avail"); } - #endif /*NETCDF_ENABLE_NCZARR_FILTERS*/ /**************************************************/ diff --git a/libnczarr/zfile.c b/libnczarr/zfile.c index 2346b4944d..343960da1f 100644 --- a/libnczarr/zfile.c +++ b/libnczarr/zfile.c @@ -294,7 +294,7 @@ NCZ_inq(int ncid, int *ndimsp, int *nvarsp, int *nattsp, int *unlimdimidp) NC_FILE_INFO_T* file; NC_GRP_INFO_T *grp; int stat = NC_NOERR; - int i; + size_t i; LOG((2, "%s: ncid 0x%x", __func__, ncid)); diff --git a/libnczarr/zfilter.c b/libnczarr/zfilter.c index 7481c4ab57..3d0db8d662 100644 --- a/libnczarr/zfilter.c +++ b/libnczarr/zfilter.c @@ -50,7 +50,8 @@ #include "ncpoco.h" #include "netcdf_filter.h" #include "netcdf_filter_build.h" -#include "netcdf_aux.h" +#include "zfilter.h" +#include "zplugins.h" #if 0 #define DEBUG @@ -63,20 +64,6 @@ #define NULLIFY(x) ((x)?(x):"NULL") -/* Hold the loaded filter plugin information */ -typedef struct NCZ_Plugin { - int incomplete; - struct HDF5API { - const H5Z_class2_t* filter; - NCPSharedLib* hdf5lib; /* source of the filter */ - } hdf5; - struct CodecAPI { - int defaulted; /* codeclib was a defaulting library */ - const NCZ_codec_t* codec; - NCPSharedLib* codeclib; /* of the source codec; null if same as hdf5 */ - } codec; -} NCZ_Plugin; - /* The NC_VAR_INFO_T->filters field is an NClist of this struct */ /* Each filter can have two parts: HDF5 and Codec. @@ -148,14 +135,6 @@ typedef struct NCZ_Filter { /* WARNING: GLOBAL DATA */ /* TODO: move to common global state */ -/* All possible HDF5 filter plugins */ -/* Consider onverting to linked list or hash table or equivalent since very sparse */ -NCZ_Plugin* loaded_plugins[H5Z_FILTER_MAX]; -int loaded_plugins_max = -1; - -static NClist* codec_defaults = NULL; /* NClist */ -static NClist* default_libs = NULL; /* NClist; sources of the defaults */ - static int NCZ_filter_initialized = 0; /**************************************************/ @@ -179,61 +158,7 @@ NCJtrace(const NCjson* j) #define SEXISTS(x,p) (((x) && *(x)? (*(x))-> p : "null")) #endif - #if defined(DEBUGF) || defined(DEBUGL) - -const char* -printplugin(const NCZ_Plugin* plugin) -{ - static char plbuf[4096]; - char plbuf2[2000]; - char plbuf1[2000]; - - if(plugin == NULL) return "plugin=NULL"; - plbuf2[0] = '\0'; plbuf1[0] = '\0'; - if(plugin->hdf5.filter) - snprintf(plbuf1,sizeof(plbuf1),"hdf5={id=%u name=%s}",plugin->hdf5.filter->id,plugin->hdf5.filter->name); - if(plugin->codec.codec) - snprintf(plbuf2,sizeof(plbuf2),"codec={codecid=%s hdf5id=%u}",plugin->codec.codec->codecid,plugin->codec.codec->hdf5id); - snprintf(plbuf,4096,"plugin={%s %s}",plbuf1,plbuf2); - return plbuf; -} - -static char* -printparams(size_t nparams, const unsigned* params) -{ - static char ppbuf[4096]; - if(nparams == 0) - snprintf(ppbuf,4096,"{0,%p}",params); - else - snprintf(ppbuf,4096,"{%u %s}",(unsigned)nparams,nczprint_paramvector(nparams,params)); - return ppbuf; -} - -static char* -printnczparams(const NCZ_Params p) -{ - return printparams(p.nparams,p.params); -} - -static const char* -printcodec(const NCZ_Codec c) -{ - static char pcbuf[4096]; - snprintf(pcbuf,sizeof(pcbuf),"{id=%s codec=%s}", - c.id,NULLIFY(c.codec)); - return pcbuf; -} - -static const char* -printhdf5(const NCZ_HDF5 h) -{ - static char phbuf[4096]; - snprintf(phbuf,sizeof(phbuf),"{id=%u visible=%s working=%s}", - h.id, printnczparams(h.visible), printnczparams(h.working)); - return phbuf; -} - static const char* printfilter(const NCZ_Filter* f) { @@ -249,29 +174,16 @@ printfilter(const NCZ_Filter* f) /* Forward */ -static int NCZ_load_all_plugins(void); -static int NCZ_load_plugin_dir(const char* path); -static int NCZ_load_plugin(const char* path, NCZ_Plugin** plugp); -static int NCZ_unload_plugin(NCZ_Plugin* plugin); -static int NCZ_plugin_loaded(int filterid, NCZ_Plugin** pp); -static int NCZ_plugin_save(int filterid, NCZ_Plugin* p); static int NCZ_filter_free(NCZ_Filter* spec); static int NCZ_filter_hdf5_clear(NCZ_HDF5* spec); static int NCZ_filter_codec_clear(NCZ_Codec* spec); static int NCZ_filter_lookup(NC_VAR_INFO_T* var, unsigned int id, struct NCZ_Filter** specp); -static int getentries(const char* path, NClist* contents); -static int NCZ_split_plugin_path(const char* path0, NClist* list); - static int ensure_working(const NC_VAR_INFO_T* var, NCZ_Filter* filter); static int paramnczclone(NCZ_Params* dst, const NCZ_Params* src); static int paramclone(size_t nparams, unsigned** dstp, const unsigned* src); -#ifdef NAMEOPT -static int pluginnamecheck(const char* name); -#endif - /**************************************************/ /** * @file @@ -367,7 +279,7 @@ NCZ_addfilter(NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, unsigned int id, size_t if(zvar->incompletefilters == NULL) zvar->incompletefilters = (void*)nclistnew(); /* Before anything else, find the matching plugin */ - if((stat = NCZ_plugin_loaded(id,&plugin))) goto done; + if((stat = NCZ_plugin_loaded((size_t)id,&plugin))) goto done; if(plugin == NULL) { stat = THROW(NC_ENOFILTER); goto done; @@ -540,8 +452,6 @@ NCZ_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, ZTRACE(1,"ncid=%d varid=%d id=%u nparams=%u params=%s",ncid,varid,id,(unsigned)nparams,nczprint_paramvector(nparams,params)); - if((stat = NCZ_filter_initialize())) goto done; - if((stat = NC_check_id(ncid,&nc))) return stat; assert(nc); @@ -607,7 +517,7 @@ NCZ_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, /* For szip, the pixels_per_block parameter must not be greater * than the number of elements in a chunk of data. */ size_t num_elem = 1; - int d; + size_t d; for (d = 0; d < var->ndims; d++) if (var->dim[d]->len) num_elem *= var->dim[d]->len; @@ -647,9 +557,6 @@ NCZ_inq_var_filter_ids(int ncid, int varid, size_t* nfiltersp, unsigned int* ids assert(h5 && var && var->hdr.id == varid); - /* Make sure all the filters are defined */ - if((stat = NCZ_filter_initialize())) goto done; - flist = var->filters; nfilters = nclistlength(flist); /* including incomplets */ @@ -687,9 +594,6 @@ NCZ_inq_var_filter_info(int ncid, int varid, unsigned int id, size_t* nparamsp, assert(h5 && var && var->hdr.id == varid); - /* Make sure all the plugins are defined */ - if((stat = NCZ_filter_initialize())) goto done; - if((stat = NCZ_filter_lookup(var,id,&spec))) goto done; if(spec != NULL) { #if 0 @@ -734,9 +638,8 @@ NCZ_inq_filter_avail(int ncid, unsigned id) NC_UNUSED(ncid); ZTRACE(1,"ncid=%d id=%u",ncid,id); - if((stat = NCZ_filter_initialize())) goto done; /* Check the available filters list */ - if((stat = NCZ_plugin_loaded((int)id, &plug))) goto done; + if((stat = NCZ_plugin_loaded((size_t)id, &plug))) goto done; if(plug == NULL || plug->incomplete) stat = THROW(NC_ENOFILTER); done: @@ -756,14 +659,11 @@ NCZ_filter_initialize(void) if(NCZ_filter_initialized) goto done; - default_libs = nclistnew(); - codec_defaults = nclistnew(); NCZ_filter_initialized = 1; - memset(loaded_plugins,0,sizeof(loaded_plugins)); + #ifdef NETCDF_ENABLE_NCZARR_FILTERS if((stat = NCZ_load_all_plugins())) goto done; #endif - done: return ZUNTRACE(stat); } @@ -772,75 +672,13 @@ int NCZ_filter_finalize(void) { int stat = NC_NOERR; - int i; - ZTRACE(6,""); if(!NCZ_filter_initialized) goto done; -#ifdef NETCDF_ENABLE_NCZARR_FILTERS - /* Reclaim all loaded filters */ -#ifdef DEBUGL - fprintf(stderr,">>> DEBUGL: finalize reclaim:\n"); -#endif - for(i=0;i<=loaded_plugins_max;i++) { - if(loaded_plugins[i]) { - NCZ_unload_plugin(loaded_plugins[i]); - loaded_plugins[i] = NULL; - } - } - /* Reclaim the codec defaults */ - if(nclistlength(codec_defaults) > 0) { - for(i=0;i 0) { - for(i=0;i>> DEBUGL: NCZ_filter_finalize: reclaim default_lib[i]=%p\n",l); -#endif - if(l != NULL) (void)ncpsharedlibfree(l); - } - } -#else - memset(loaded_plugins,0,sizeof(loaded_plugins)); -#endif - nclistfree(default_libs); default_libs = NULL; - nclistfree(codec_defaults); codec_defaults = NULL; -done: NCZ_filter_initialized = 0; - return ZUNTRACE(stat); -} -static int -NCZ_plugin_save(int filterid, NCZ_Plugin* p) -{ - int stat = NC_NOERR; - ZTRACE(6,"filterid=%d p=%p",filterid,p); - if(filterid <= 0 || filterid >= H5Z_FILTER_MAX) - {stat = NC_EINVAL; goto done;} - if(filterid > loaded_plugins_max) loaded_plugins_max = filterid; - loaded_plugins[filterid] = p; done: return ZUNTRACE(stat); } -static int -NCZ_plugin_loaded(int filterid, NCZ_Plugin** pp) -{ - int stat = NC_NOERR; - struct NCZ_Plugin* plug = NULL; - ZTRACE(6,"filterid=%d",filterid); - if(filterid <= 0 || filterid >= H5Z_FILTER_MAX) - {stat = NC_EINVAL; goto done;} - if(filterid <= loaded_plugins_max) - plug = loaded_plugins[filterid]; - if(pp) *pp = plug; -done: - return ZUNTRACEX(stat,"plugin=%p",*pp); -} - int NCZ_applyfilterchain(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, NClist* chain, size_t inlen, void* indata, size_t* outlenp, void** outdatap, int encode) { @@ -977,7 +815,7 @@ NCZ_filter_jsonize(const NC_FILE_INFO_T* file, const NC_VAR_INFO_T* var, NCZ_Fil int NCZ_filter_build(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, const NCjson* jfilter, int chainindex) { - int i,stat = NC_NOERR; + int stat = NC_NOERR; NCZ_Filter* filter = NULL; const NCjson* jvalue = NULL; NCZ_Plugin* plugin = NULL; @@ -1002,13 +840,7 @@ NCZ_filter_build(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, const NCjson* j if(NCJunparse(jfilter,0,&codec.codec)<0) {stat = NC_EFILTER; goto done;} /* Find the plugin for this filter */ - for(i=0;i<=loaded_plugins_max;i++) { - if (!loaded_plugins[i]) continue; - if(!loaded_plugins[i] || !loaded_plugins[i]->codec.codec) continue; /* no plugin or no codec */ - if(strcmp(NCJstring(jvalue), loaded_plugins[i]->codec.codec->codecid) == 0) - {plugin = loaded_plugins[i]; break;} - } - + if((stat = NCZ_plugin_loaded_byname(NCJstring(jvalue),&plugin))) goto done; /* Will always have a filter; possibly unknown */ if((filter = calloc(1,sizeof(NCZ_Filter)))==NULL) {stat = NC_ENOMEM; goto done;} filter->chainindex = chainindex; @@ -1049,561 +881,6 @@ NCZ_filter_build(const NC_FILE_INFO_T* file, NC_VAR_INFO_T* var, const NCjson* j return ZUNTRACE(stat); } -/**************************************************/ -/* Filter loading */ - -/* -Get entries in a path that is assumed to be a directory. -*/ - -#ifdef _WIN32 - -static int -getentries(const char* path, NClist* contents) -{ - /* Iterate over the entries in the directory */ - int ret = NC_NOERR; - errno = 0; - WIN32_FIND_DATA FindFileData; - HANDLE dir = NULL; - char* ffpath = NULL; - char* lpath = NULL; - size_t len; - char* d = NULL; - - ZTRACE(6,"path=%s",path); - - /* We need to process the path to make it work with FindFirstFile */ - len = strlen(path); - /* Need to terminate path with '/''*' */ - ffpath = (char*)malloc(len+2+1); - memcpy(ffpath,path,len); - if(path[len-1] != '/') { - ffpath[len] = '/'; - len++; - } - ffpath[len] = '*'; len++; - ffpath[len] = '\0'; - - /* localize it */ - if((ret = nczm_localize(ffpath,&lpath,LOCALIZE))) goto done; - dir = FindFirstFile(lpath, &FindFileData); - if(dir == INVALID_HANDLE_VALUE) { - /* Distinquish not-a-directory from no-matching-file */ - switch (GetLastError()) { - case ERROR_FILE_NOT_FOUND: /* No matching files */ /* fall thru */ - ret = NC_NOERR; - goto done; - case ERROR_DIRECTORY: /* not a directory */ - default: - ret = NC_EEMPTY; - goto done; - } - } - do { - char* p = NULL; - const char* name = NULL; - name = FindFileData.cFileName; - if(strcmp(name,".")==0 || strcmp(name,"..")==0) - continue; - nclistpush(contents,strdup(name)); - } while(FindNextFile(dir, &FindFileData)); - -done: - if(dir) FindClose(dir); - nullfree(lpath); - nullfree(ffpath); - nullfree(d); - errno = 0; - return ZUNTRACEX(ret,"|contents|=%d",(int)nclistlength(contents)); -} - -#else /* !_WIN32 */ - -int -getentries(const char* path, NClist* contents) -{ - int ret = NC_NOERR; - errno = 0; - DIR* dir = NULL; - - ZTRACE(6,"path=%s",path); - - dir = NCopendir(path); - if(dir == NULL) - {ret = (errno); goto done;} - for(;;) { - const char* name = NULL; - struct dirent* de = NULL; - errno = 0; - de = readdir(dir); - if(de == NULL) - {ret = (errno); goto done;} - if(strcmp(de->d_name,".")==0 || strcmp(de->d_name,"..")==0) - continue; - name = de->d_name; - nclistpush(contents,strdup(name)); - } -done: - if(dir) NCclosedir(dir); - errno = 0; - return ZUNTRACEX(ret,"|contents|=%d",(int)nclistlength(contents)); -} -#endif /*_WIN32*/ - -static int -NCZ_load_all_plugins(void) -{ - size_t i,j; - int ret = NC_NOERR; - char* pluginroots = NULL; - struct stat buf; - NClist* dirs = nclistnew(); - char* defaultpluginpath = NULL; - - ZTRACE(6,""); - -#ifdef DEBUGL - fprintf(stderr,">>> DEBUGL: NCZ_load_all_plugins\n"); -#endif - - /* Setup the plugin path default */ - { -#ifdef _WIN32 - const char* win32_root; - char dfalt[4096]; - win32_root = getenv(WIN32_ROOT_ENV); - if(win32_root != NULL && strlen(win32_root) > 0) { - snprintf(dfalt,sizeof(dfalt),PLUGIN_DIR_WIN,win32_root); - defaultpluginpath = strdup(dfalt); - } -#else /*!_WIN32*/ - defaultpluginpath = strdup(PLUGIN_DIR_UNIX); -#endif - } - - /* Find the plugin directory root(s) */ - pluginroots = getenv(PLUGIN_ENV); /* Usually HDF5_PLUGIN_PATH */ - if(pluginroots != NULL && strlen(pluginroots) == 0) pluginroots = NULL; - if(pluginroots == NULL) { - pluginroots = defaultpluginpath; - } - assert(pluginroots != NULL); - ZTRACEMORE(6,"pluginroots=%s",(pluginroots?pluginroots:"null")); - - if((ret = NCZ_split_plugin_path(pluginroots,dirs))) goto done; - - /* Add the default to end of the dirs list if not already there */ - if(!nclistmatch(dirs,defaultpluginpath,0)) { - nclistpush(dirs,defaultpluginpath); - defaultpluginpath = NULL; - } - - for(i=0;icodec->hdf5id); - } - fprintf(stderr,"\n"); - } -#endif - if(nclistlength(codec_defaults)) { /* Try to provide default for any HDF5 filters without matching Codec. */ - /* Search the defaults */ - for(j=0;jcodec != NULL) { - const NCZ_codec_t* codec = dfalt->codec; - int hdf5id = codec->hdf5id; - NCZ_Plugin* p = NULL; - if(hdf5id < 0 || hdf5id > loaded_plugins_max) {ret = NC_EFILTER; goto done;} - p = loaded_plugins[hdf5id]; /* get candidate */ - if(p != NULL && p->hdf5.filter != NULL - && p->codec.codec == NULL) { -#ifdef DEBUGL - fprintf(stderr,">>> DEBUGL: plugin defaulted: id=%u, codec=%s src=%s\n",hdf5id,codec->codecid,dfalt->codeclib->path); -#endif - p->codec.codec = codec; - p->codec.codeclib = dfalt->codeclib; - p->codec.defaulted = 1; - } - } - } - } - - /* Mark all plugins for which we do not have both HDF5 and codec */ - { - int i; - NCZ_Plugin* p; - for(i=0;ihdf5.filter == NULL || p->codec.codec == NULL) { - /* mark this entry as incomplete */ - p->incomplete = 1; -#ifdef DEBUGL - fprintf(stderr,">>> DEBUGL: Incomplete plugin: id=%u; reasons: %s %s\n",i, - (p->hdf5.filter==NULL?"hdf5":""),(p->codec.codec==NULL?"codec":"")); -#endif - } -#ifdef DEBUGL - else - fprintf(stderr,">>> DEBUGL: plugin accepted: id=%u\n",i); -#endif - } - } - } - /* Iniitalize all remaining plugins */ - { - int i; - NCZ_Plugin* p; - for(i=0;iincomplete) continue; - if(p->hdf5.filter != NULL && p->codec.codec != NULL) { - if(p->codec.codec && p->codec.codec->NCZ_codec_initialize) - p->codec.codec->NCZ_codec_initialize(); -#ifdef DEBUGL - fprintf(stderr,">>> DEBUGL: plugin initialized: id=%u\n",p->hdf5.filter->id); -#endif - } - } - } - } - -done: - nullfree(defaultpluginpath); - nclistfreeall(dirs); - errno = 0; - return ZUNTRACE(ret); -} - -static int -NCZ_split_plugin_path(const char* path0, NClist* list) -{ - int i,stat = NC_NOERR; - char* path = NULL; - char* p; - int count; - size_t plen; -#ifdef _WIN32 - const char* seps = ";"; -#else - const char* seps = ";:"; -#endif - - if(path0 == NULL || path0[0] == '\0') goto done; - plen = strlen(path0); - if((path = malloc(plen+1+1))==NULL) {stat = NC_ENOMEM; goto done;} - memcpy(path,path0,plen); - path[plen] = '\0'; path[plen+1] = '\0'; /* double null term */ - for(count=0,p=path;*p;p++) { - if(strchr(seps,*p) != NULL) {*p = '\0'; count++;} - } - count++; /* for last piece */ - for(p=path,i=0;i 0) - nclistpush(list,strdup(p)); - p = p+len+1; /* point to next piece */ - } - -done: - nullfree(path); - return stat; -} - - -/* Load all the filters within a specified directory */ -static int -NCZ_load_plugin_dir(const char* path) -{ - size_t i; - int stat = NC_NOERR; - size_t pathlen; - NClist* contents = nclistnew(); - char* file = NULL; - - ZTRACE(7,"path=%s",path); - -#ifdef DEBUGL - fprintf(stderr,">>> DEBUGL: NCZ_load_plugin_dir: path=%s\n",path); -#endif - - if(path == NULL) {stat = NC_EINVAL; goto done;} - pathlen = strlen(path); - if(pathlen == 0) {stat = NC_EINVAL; goto done;} - - if((stat = getentries(path,contents))) goto done; - for(i=0;i 0); - nullfree(file); file = NULL; - if((file = (char*)malloc(flen))==NULL) {stat = NC_ENOMEM; goto done;} - file[0] = '\0'; - strlcat(file,path,flen); - strlcat(file,"/",flen); - strlcat(file,name,flen); - /* See if can load the file */ - stat = NCZ_load_plugin(file,&plugin); - switch (stat) { - case NC_NOERR: break; - case NC_ENOFILTER: case NC_ENOTFOUND: - stat = NC_NOERR; - break; /* will cause it to be ignored */ - default: goto done; - } - if(plugin != NULL) { - id = plugin->hdf5.filter->id; - if(loaded_plugins[id] == NULL) { - loaded_plugins[id] = plugin; - if(id > loaded_plugins_max) loaded_plugins_max = id; -#ifdef DEBUGL - fprintf(stderr,">>> DEBUGL: plugin loaded: %s\n",printplugin(plugin)); -#endif - } else { -#ifdef DEBUGL - fprintf(stderr,">>> DEBUGL: plugin duplicate: %s\n",printplugin(plugin)); -#endif - NCZ_unload_plugin(plugin); /* its a duplicate */ - } - } else - stat = NC_NOERR; /*ignore failure */ - } - -done: - nullfree(file); - nclistfreeall(contents); - return ZUNTRACE(stat); -} - -int -loadcodecdefaults(const char* path, const NCZ_codec_t** cp, NCPSharedLib* lib, int* lib_usedp) -{ - int stat = NC_NOERR; - int lib_used = 0; - - nclistpush(default_libs,lib); - for(;*cp;cp++) { - struct CodecAPI* c0; -#ifdef DEBUGL - fprintf(stderr,"@@@ %s: %s = %u\n",path,(*cp)->codecid,(*cp)->hdf5id); -#endif - c0 = (struct CodecAPI*)calloc(1,sizeof(struct CodecAPI)); - if(c0 == NULL) {stat = NC_ENOMEM; goto done;} - c0->codec = *cp; - c0->codeclib = lib; - lib_used = 1; /* remember */ - nclistpush(codec_defaults,c0); c0 = NULL; - } -done: - if(lib_usedp) *lib_usedp = lib_used; - return stat; -} - -static int -NCZ_load_plugin(const char* path, struct NCZ_Plugin** plugp) -{ - int stat = NC_NOERR; - NCZ_Plugin* plugin = NULL; - const H5Z_class2_t* h5class = NULL; - H5PL_type_t h5type = 0; - const NCZ_codec_t** cp = NULL; - const NCZ_codec_t* codec = NULL; - NCPSharedLib* lib = NULL; - int flags = NCP_GLOBAL; - int h5id = -1; - - assert(path != NULL && strlen(path) > 0 && plugp != NULL); - - ZTRACE(8,"path=%s",path); - - if(plugp) *plugp = NULL; - -#if defined NAMEOPT || defined _WIN32 - /*triage because visual studio does a popup if the file will not load*/ - if(!pluginnamecheck(path)) {stat = NC_ENOFILTER; goto done;} -#endif - - /* load the shared library */ - if((stat = ncpsharedlibnew(&lib))) goto done; - if((stat = ncpload(lib,path,flags))) goto done; - -#ifdef DEBUGL - fprintf(stderr,">>> DEBUGL: NCZ_load_plugin: path=%s lib=%p\n",path,lib); -#endif - - /* See what we have */ - { - const H5PL_get_plugin_type_proto gpt = (H5PL_get_plugin_type_proto)ncpgetsymbol(lib,"H5PLget_plugin_type"); - const H5PL_get_plugin_info_proto gpi = (H5PL_get_plugin_info_proto)ncpgetsymbol(lib,"H5PLget_plugin_info"); - const NCZ_get_codec_info_proto npi = (NCZ_get_codec_info_proto)ncpgetsymbol(lib,"NCZ_get_codec_info"); - const NCZ_codec_info_defaults_proto cpd = (NCZ_codec_info_defaults_proto)ncpgetsymbol(lib,"NCZ_codec_info_defaults"); - - if(gpt == NULL && gpi == NULL && npi == NULL && cpd == NULL) - {stat = THROW(NC_ENOFILTER); goto done;} - - /* We can have cpd or we can have (gpt && gpi && npi) but not both sets */ - if(cpd != NULL) { - cp = (const NCZ_codec_t**)cpd(); - } else {/* cpd => !gpt && !gpi && !npi */ - if(gpt != NULL && gpi != NULL) { /* get HDF5 info */ - h5type = gpt(); - h5class = gpi(); - /* Verify */ - if(h5type != H5PL_TYPE_FILTER) {stat = NC_EPLUGIN; goto done;} - if(h5class->version != H5Z_CLASS_T_VERS) {stat = NC_EFILTER; goto done;} - } - if(npi != NULL) {/* get Codec info */ - codec = npi(); - /* Verify */ - if(codec->version != NCZ_CODEC_CLASS_VER) {stat = NC_EPLUGIN; goto done;} - if(codec->sort != NCZ_CODEC_HDF5) {stat = NC_EPLUGIN; goto done;} - } - } - } - -#ifdef DEBUGL -fprintf(stderr,">>> DEBUGL: load: %s:",path); -if(h5class) fprintf(stderr,">>> %u",(unsigned)h5class->id); -if(codec) fprintf(stderr,">>> %u/%s",codec->hdf5id,codec->codecid); -fprintf(stderr,">>> \n"); -#endif - - /* Handle defaults separately */ - if(cp != NULL) { - int used = 0; -#ifdef DEBUGL - fprintf(stderr,"@@@ %s: default codec library found: %p\n",path,cp); -#endif - if((stat = loadcodecdefaults(path,cp,lib,&used))) goto done; - if(used) lib = NULL; - goto done; - } - - if(h5class != NULL && codec != NULL) { - /* Verify consistency of the HDF5 and the Codec */ - if(h5class->id != codec->hdf5id) goto done; /* ignore */ - } - - /* There are several cases to consider: - 1. This library has both HDF5 API and Codec API => merge - 2. This library has HDF5 API only and Codec API was already found in another library => merge - 3. This library has Codec API only and HDF5 API was already found in another library => merge - */ - - /* Get any previous plugin entry for this id; may be NULL */ - if(h5class != NULL) { - h5id = h5class->id; - if((stat = NCZ_plugin_loaded(h5class->id,&plugin))) goto done; - } else if(codec != NULL) { - h5id = codec->hdf5id; - if((stat = NCZ_plugin_loaded(codec->hdf5id,&plugin))) goto done; - } - - if(plugin == NULL) { - /* create new entry */ - if((plugin = (NCZ_Plugin*)calloc(1,sizeof(NCZ_Plugin)))==NULL) {stat = NC_ENOMEM; goto done;} - } - - /* Fill in the plugin */ - if(h5class != NULL && plugin->hdf5.filter == NULL) { - plugin->hdf5.filter = h5class; - plugin->hdf5.hdf5lib = lib; - lib = NULL; - } - if(codec != NULL && plugin->codec.codec == NULL) { - plugin->codec.codec = codec; - plugin->codec.codeclib = lib; - lib = NULL; - } -#ifdef DEBUGL - if(plugin) - fprintf(stderr,">>> DEBUGL: load_plugin: %s\n",printplugin(plugin)); -#endif - /* Cleanup */ - if(plugin->hdf5.hdf5lib == plugin->codec.codeclib) /* Works for NULL case also */ - plugin->codec.codeclib = NULL; - if((stat=NCZ_plugin_save(h5id,plugin))) goto done; - plugin = NULL; - -done: - if(lib) - (void)ncpsharedlibfree(lib); - if(plugin) NCZ_unload_plugin(plugin); - return ZUNTRACEX(stat,"plug=%p",*plugp); -} - -static int -NCZ_unload_plugin(NCZ_Plugin* plugin) -{ - ZTRACE(9,"plugin=%p",plugin); - - if(plugin) { -#ifdef DEBUGL - fprintf(stderr,">>> DEBUGL: unload: %s\n",printplugin(plugin)); -#endif - if(plugin->codec.codec && plugin->codec.codec->NCZ_codec_finalize) - plugin->codec.codec->NCZ_codec_finalize(); - if(plugin->hdf5.filter != NULL) loaded_plugins[plugin->hdf5.filter->id] = NULL; - if(plugin->hdf5.hdf5lib != NULL) (void)ncpsharedlibfree(plugin->hdf5.hdf5lib); - if(!plugin->codec.defaulted && plugin->codec.codeclib != NULL) (void)ncpsharedlibfree(plugin->codec.codeclib); -memset(plugin,0,sizeof(NCZ_Plugin)); - free(plugin); - } - return ZUNTRACE(NC_NOERR); -} - -#ifdef NAMEOPT -static int -pluginnamecheck(const char* name) -{ - size_t count,len; - long i; - const char* p; - if(name == NULL) return 0; - /* get basename */ - p = strrchr(name,'/'); - if(p != NULL) name = (p+1); - len = strlen(name); - if(len == 0) return 0; - i = (long)(len-1); - count = 1; - p = name+i; - for(;i>=0;i--,count++,p--) { - char c = *p; - if(c == '/') break; - if(c == '.') { - if(count >= 3 && memcmp(p,".so",3)==0) return 1; - if(count >= 4 && memcmp(p,".dll",4)==0) return 1; - if(count >= 6 && memcmp(p,".dylib",6)==0) return 1; - } - } - return 0; -} -#endif - /**************************************************/ /* _Codecs attribute */ diff --git a/libnczarr/zfilter.h b/libnczarr/zfilter.h index 4a9ae0b101..31b64d65f5 100644 --- a/libnczarr/zfilter.h +++ b/libnczarr/zfilter.h @@ -19,24 +19,18 @@ /*Mnemonic*/ #define ENCODING 1 -/* list of environment variables to check for plugin roots */ -#define PLUGIN_ENV "HDF5_PLUGIN_PATH" -#define PLUGIN_DIR_UNIX "/usr/local/hdf5/plugin" -#define PLUGIN_DIR_WIN "%s/hdf5/lib/plugin" -#define WIN32_ROOT_ENV "ALLUSERSPROFILE" - /* Opaque */ struct NCZ_Filter; -int NCZ_filter_initialize(void); -int NCZ_filter_finalize(void); -int NCZ_addfilter(NC_FILE_INFO_T*, NC_VAR_INFO_T* var, unsigned int id, size_t nparams, const unsigned int* params); -int NCZ_filter_setup(NC_VAR_INFO_T* var); -int NCZ_filter_freelists(NC_VAR_INFO_T* var); -int NCZ_codec_freelist(NCZ_VAR_INFO_T* zvar); -int NCZ_applyfilterchain(const NC_FILE_INFO_T*, NC_VAR_INFO_T*, NClist* chain, size_t insize, void* indata, size_t* outlen, void** outdata, int encode); -int NCZ_filter_jsonize(const NC_FILE_INFO_T*, const NC_VAR_INFO_T*, struct NCZ_Filter* filter, struct NCjson**); -int NCZ_filter_build(const NC_FILE_INFO_T*, NC_VAR_INFO_T* var, const NCjson* jfilter, int chainindex); -int NCZ_codec_attr(const NC_VAR_INFO_T* var, size_t* lenp, void* data); +EXTERNL int NCZ_filter_initialize(void); +EXTERNL int NCZ_filter_finalize(void); +EXTERNL int NCZ_addfilter(NC_FILE_INFO_T*, NC_VAR_INFO_T* var, unsigned int id, size_t nparams, const unsigned int* params); +EXTERNL int NCZ_filter_setup(NC_VAR_INFO_T* var); +EXTERNL int NCZ_filter_freelists(NC_VAR_INFO_T* var); +EXTERNL int NCZ_codec_freelist(NCZ_VAR_INFO_T* zvar); +EXTERNL int NCZ_applyfilterchain(const NC_FILE_INFO_T*, NC_VAR_INFO_T*, NClist* chain, size_t insize, void* indata, size_t* outlen, void** outdata, int encode); +EXTERNL int NCZ_filter_jsonize(const NC_FILE_INFO_T*, const NC_VAR_INFO_T*, struct NCZ_Filter* filter, struct NCjson**); +EXTERNL int NCZ_filter_build(const NC_FILE_INFO_T*, NC_VAR_INFO_T* var, const NCjson* jfilter, int chainindex); +EXTERNL int NCZ_codec_attr(const NC_VAR_INFO_T* var, size_t* lenp, void* data); #endif /*ZFILTER_H*/ diff --git a/libnczarr/zinternal.c b/libnczarr/zinternal.c index 81c6fe0ee4..a654041011 100644 --- a/libnczarr/zinternal.c +++ b/libnczarr/zinternal.c @@ -46,51 +46,6 @@ set_auto(void* func, void *client_data) } #endif -/** - * @internal Provide a function to do any necessary initialization of - * the ZARR library. - */ -int -NCZ_initialize_internal(void) -{ - int stat = NC_NOERR; - char* dimsep = NULL; - NCglobalstate* ngs = NULL; - - ncz_initialized = 1; - ngs = NC_getglobalstate(); - if(ngs != NULL) { - /* Defaults */ - ngs->zarr.dimension_separator = DFALT_DIM_SEPARATOR; - dimsep = NC_rclookup("ZARR.DIMENSION_SEPARATOR",NULL,NULL); - if(dimsep != NULL) { - /* Verify its value */ - if(dimsep != NULL && strlen(dimsep) == 1 && islegaldimsep(dimsep[0])) - ngs->zarr.dimension_separator = dimsep[0]; - } - } - - return stat; -} - -/** - * @internal Provide a function to do any necessary finalization of - * the ZARR library. - */ -int -NCZ_finalize_internal(void) -{ - /* Reclaim global resources */ - ncz_initialized = 0; -#ifdef NETCDF_ENABLE_NCZARR_FILTERS - NCZ_filter_finalize(); -#endif -#ifdef NETCDF_ENABLE_S3 - NCZ_s3finalize(); -#endif - return NC_NOERR; -} - /** * @internal Given a varid, return the maximum length of a dimension * using dimid. @@ -113,7 +68,7 @@ find_var_dim_max_length(NC_GRP_INFO_T *grp, int varid, int dimid, *maxlen = 0; /* Find this var. */ - var = (NC_VAR_INFO_T*)ncindexith(grp->vars,varid); + var = (NC_VAR_INFO_T*)ncindexith(grp->vars,(size_t)varid); if (!var) return NC_ENOTVAR; assert(var->hdr.id == varid); @@ -230,7 +185,7 @@ ncz_find_dim_len(NC_GRP_INFO_T *grp, int dimid, size_t **len) { NC_VAR_INFO_T *var; int retval; - int i; + size_t i; assert(grp && len); LOG((3, "%s: grp->name %s dimid %d", __func__, grp->hdr.name, dimid)); @@ -382,7 +337,7 @@ static int close_dims(NC_GRP_INFO_T *grp) { NC_DIM_INFO_T *dim; - int i; + size_t i; for (i = 0; i < ncindexsize(grp->dim); i++) { @@ -417,7 +372,7 @@ close_dims(NC_GRP_INFO_T *grp) static int close_types(NC_GRP_INFO_T *grp) { - int i; + size_t i; for (i = 0; i < ncindexsize(grp->type); i++) { @@ -456,7 +411,7 @@ close_types(NC_GRP_INFO_T *grp) static int ncz_rec_grp_NCZ_del(NC_GRP_INFO_T *grp) { - int i; + size_t i; int retval; assert(grp && grp->format_grp_info); @@ -608,7 +563,7 @@ ncz_find_grp_var_att(int ncid, int varid, const char *name, int attnum, if (att) { my_att = use_name ? (NC_ATT_INFO_T *)ncindexlookup(attlist, my_norm_name) : - (NC_ATT_INFO_T *)ncindexith(attlist, attnum); + (NC_ATT_INFO_T *)ncindexith(attlist, (size_t)attnum); if (!my_att) return NC_ENOTATT; } diff --git a/libnczarr/zinternal.h b/libnczarr/zinternal.h index 2548ad54ba..9aa89721ce 100644 --- a/libnczarr/zinternal.h +++ b/libnczarr/zinternal.h @@ -211,6 +211,20 @@ typedef struct NCZCONTENT{ } NCZCONTENT; #endif +/* The type for the NC_FORMATX_NCZARR Global State Object */ +typedef struct GlobalNCZarr { /* libnczarr dispatcher specific parameters */ + char dimension_separator; + int default_zarrformat; + NClist* pluginpaths; + NClist* codec_defaults; + NClist* default_libs; + /* All possible HDF5 filter plugins */ + /* Consider onverting to linked list or hash table or + equivalent since very sparse */ + struct NCZ_Plugin** loaded_plugins; //[H5Z_FILTER_MAX+1]; + size_t loaded_plugins_max; /* plugin filter id index. 0= 0 ?nsegs: -nsegs); - int presegs = 0; + size_t abssegs = (size_t)(nsegs >= 0 ?nsegs: -nsegs); + size_t presegs = 0; /* Special case */ if(key == NULL || strlen(key) == 0) goto done; @@ -439,7 +439,7 @@ nczm_segment1(const char* path, char** seg1p) delta = (q-p); if((seg1 = (char*)malloc((size_t)delta+1))==NULL) {ret = NC_ENOMEM; goto done;} - memcpy(seg1,p,delta); + memcpy(seg1,p,(size_t)delta); seg1[delta] = '\0'; if(seg1p) {*seg1p = seg1; seg1 = NULL;} diff --git a/libnczarr/zplugins.c b/libnczarr/zplugins.c new file mode 100644 index 0000000000..8d528c3bfc --- /dev/null +++ b/libnczarr/zplugins.c @@ -0,0 +1,874 @@ +/* Copyright 2003-2018, University Corporation for Atmospheric + * Research. See the COPYRIGHT file for copying and redistribution + * conditions. + */ +/** + * @file @internal netcdf-4 functions for the plugin list. + * + * @author Dennis Heimbigner + */ + +#include "config.h" +#include +#include +#include "zincludes.h" +#include "ncpathmgr.h" +#include "ncpoco.h" +#include "netcdf_filter.h" +#include "netcdf_filter_build.h" +#include "zfilter.h" +#include "ncplugins.h" +#include "zplugins.h" + +#if 0 +#define DEBUG +#define DEBUGF +#define DEBUGL +#define DEBUGPL +#endif + +/**************************************************/ +/* Forward */ + +static int NCZ_plugin_path_initialize(void** statep, const NClist* initialpaths); +static int NCZ_plugin_path_finalize(void** statep); +static int NCZ_plugin_path_read(void* state, size_t* ndirsp, char** dirs); +static int NCZ_plugin_path_write(void* state, size_t ndirs, char** const dirs); + +static int NCZ_load_plugin(const char* path, NCZ_Plugin** plugp); +static int NCZ_unload_plugin(NCZ_Plugin* plugin); +static int NCZ_load_plugin_dir(GlobalNCZarr* gz, const char* path); +static int NCZ_plugin_save(GlobalNCZarr* gz, size_t filterid, NCZ_Plugin* p); +static int getentries(const char* path, NClist* contents); +static int loadcodecdefaults(GlobalNCZarr* gz, const char* path, const NCZ_codec_t** cp, NCPSharedLib* lib, int* lib_usedp); + +#if defined NAMEOPT || defined _WIN32 +static int pluginnamecheck(const char* name); +#endif + +/**************************************************/ +/** + * @file + * @internal + * Internal netcdf zarr plugin path functions. + * + * @author Dennis Heimbigner + */ + +/**************************************************/ + +/**************************************************/ +/* The NCZarr Plugin Path Dispatch table and functions */ +NC_PluginPathDispatch NCZ_pluginpathdispatch = { + NC_FORMATX_NCZARR, + NC_PLUGINPATH_DISPATCH_VERSION, + NCZ_plugin_path_initialize, + NCZ_plugin_path_finalize, + NCZ_plugin_path_read, + NCZ_plugin_path_write, +}; + +NC_PluginPathDispatch* NCZ_pluginpathtable = &NCZ_pluginpathdispatch; + +/**************************************************/ +/** + * This function is called as part of nc_initialize. + * Its purpose is to initialize the plugin paths state. + * @return NC_NOERR + * @author Dennis Heimbigner +*/ +static int +NCZ_plugin_path_initialize(void** statep, const NClist* initialpaths) +{ + int stat = NC_NOERR; + GlobalNCZarr* gz = NULL; + + assert(statep != NULL); + if(*statep != NULL) goto done; /* already initialized */ + + if((gz = (GlobalNCZarr*)calloc(1,sizeof(GlobalNCZarr)))==NULL) {stat = NC_ENOMEM; goto done;} + + gz->pluginpaths = nclistnew(); + gz->default_libs = nclistnew(); + gz->codec_defaults = nclistnew(); + gz->loaded_plugins = (struct NCZ_Plugin**)calloc(H5Z_FILTER_MAX+1,sizeof(struct NCZ_Plugin*)); + if(gz->loaded_plugins == NULL) {stat = NC_ENOMEM; goto done;} + + /* Preload with set of initial paths (typically coming from HDF5_PLUGIN_PATH) */ + if(nclistlength(initialpaths) > 0) { + size_t i; + for(i=0;ipluginpaths,strdup(dir)); + } + } + + *statep = (void*)gz; gz = NULL; +done: + nullfree(gz); + return THROW(stat); +} + +/** + * This function is called as part of nc_finalize() + * Its purpose is to clean-up plugin path state. + * @return NC_NOERR + * @author Dennis Heimbigner +*/ +static int +NCZ_plugin_path_finalize(void** statep) +{ + int stat = NC_NOERR; + GlobalNCZarr* gz = NULL; + + assert(statep != NULL); + if(*statep == NULL) goto done; /* already finalized */ + gz = (GlobalNCZarr*)(*statep); + + /* Reclaim all loaded filters */ +#ifdef DEBUGL + fprintf(stderr,">>> DEBUGL: finalize reclaim:\n"); +#endif + { size_t i; + for(i=1;i<=gz->loaded_plugins_max;i++) { + if(gz->loaded_plugins[i]) { + NCZ_unload_plugin(gz->loaded_plugins[i]); + gz->loaded_plugins[i] = NULL; + } + } + } + /* Reclaim the codec defaults */ + if(nclistlength(gz->codec_defaults) > 0) { + size_t i; + for(i=0;icodec_defaults);i++) { + struct CodecAPI* ca = (struct CodecAPI*)nclistget(gz->codec_defaults,i); + nullfree(ca); + } + } + /* Reclaim the defaults library contents; Must occur as last act */ + if(nclistlength(gz->default_libs) > 0) { + size_t i; + for(i=0;idefault_libs);i++) { + NCPSharedLib* l = (NCPSharedLib*)nclistget(gz->default_libs,i); +#ifdef DEBUGL + fprintf(stderr,">>> DEBUGL: NCZ_filter_finalize: reclaim default_lib[i]=%p\n",l); +#endif + if(l != NULL) (void)ncpsharedlibfree(l); + } + } + gz->loaded_plugins_max = 0; + nullfree(gz->loaded_plugins); gz->loaded_plugins = NULL; +assert(gz->default_libs != NULL); + nclistfree(gz->default_libs); gz->default_libs = NULL; + nclistfree(gz->codec_defaults); gz->codec_defaults = NULL; + nclistfreeall(gz->pluginpaths); gz->pluginpaths = NULL; + + *statep = NULL; + +done: + nullfree(gz); + return THROW(stat); +} + +/** + * Return the current sequence of directories in the internal plugin path list. + * Since this function does not modify the plugin path, it can be called at any time. + * + * @param state the per-dispatcher state + * @param ndirsp return the number of paths in the path list + * @param dirs copy the sequence of directories in the path list into this; caller must free the copied strings + * + * @return ::NC_NOERR + * @return ::NC_EINVAL if formatx is unknown or zero + * + * @author Dennis Heimbigner + * + * As a rule, this function needs to be called twice. The first time + * with ndirsp not NULL and dirs set to NULL to get the size of + * the path list. The second time with dirs not NULL to get the + * actual sequence of paths. + * Note also that it is not possible to read a subset of the plugin path. + * If the dirs argument is not NULL, then this code assumes it points + * to enough memory to hold the whole plugin path. +*/ + +static int +NCZ_plugin_path_read(void* state, size_t* ndirsp, char** dirs) +{ + int stat = NC_NOERR; + GlobalNCZarr* gz = (GlobalNCZarr*)state; + size_t ndirs = 0; + + ndirs = nclistlength(gz->pluginpaths); + if(ndirsp) *ndirsp = ndirs; + + if(ndirs > 0 && dirs != NULL) { + size_t i; + for(i=0;ipluginpaths,i); + dirs[i] = strdup(dir); + } + } + return THROW(stat); +} + +/** + * Empty the current internal path sequence + * and replace with the sequence of directories + * specified in the arguments. + * If ndirs == 0 the path list will be cleared + * + * @param state the per-dispatcher state + * @param ndirs number of entries in dirs arg + * @param dirs the actual directory paths + * + * @return ::NC_NOERR + * @return ::NC_EINVAL if formatx is unknown or ndirs > 0 and dirs == NULL + * + * @author Dennis Heimbigner +*/ + +static int +NCZ_plugin_path_write(void* state, size_t ndirs, char** const dirs) +{ + int stat = NC_NOERR; + GlobalNCZarr* gz = (GlobalNCZarr*)state; + + if(ndirs > 0 && dirs == NULL) {stat = NC_EINVAL; goto done;} + + /* Clear the current path list */ + nclistfreeall(gz->pluginpaths); + gz->pluginpaths = nclistnew(); + + if(ndirs > 0 && dirs != NULL) { + size_t i; + for(i=0;ipluginpaths,nulldup(dirs[i])); + } + +done: + if(gz->pluginpaths == NULL) + gz->pluginpaths = nclistnew(); + return NCTHROW(stat); +} + +/**************************************************/ +/* Filter<->Plugin interface */ + +int +NCZ_load_all_plugins(void) +{ + int ret = NC_NOERR; + size_t i,j; + struct stat buf; + NClist* dirs = nclistnew(); + char* defaultpluginpath = NULL; + NCglobalstate* gs = NC_getglobalstate(); + GlobalNCZarr* gz = (GlobalNCZarr*)gs->formatxstate.state[NC_FORMATX_NCZARR]; + ZTRACE(6,""); + +#ifdef DEBUGL + fprintf(stderr,">>> DEBUGL: NCZ_load_all_plugins\n"); +#endif + + for(i=0;ipluginpaths);i++) { + const char* dir = (const char*)nclistget(gz->pluginpaths,i); + /* Make sure the root is actually a directory */ + errno = 0; + ret = NCstat(dir, &buf); +#if 1 + ZTRACEMORE(6,"stat: ret=%d, errno=%d st_mode=%d",ret,errno,buf.st_mode); +#endif + if(ret < 0) {errno = 0; ret = NC_NOERR; continue;} /* ignore unreadable directories */ + if(! S_ISDIR(buf.st_mode)) + ret = NC_EINVAL; + if(ret) goto done; + + /* Try to load plugins from this directory */ + if((ret = NCZ_load_plugin_dir(gz,dir))) goto done; + } +#ifdef DEBUGL + { size_t i; + fprintf(stderr,"gz->codec_defaults:"); + for(i=0;icodec_defaults);i++) { + struct CodecAPI* codec = (struct CodecAPI*)nclistget(gz->codec_defaults,i); + fprintf(stderr," %d",codec->codec->hdf5id); + } + fprintf(stderr,"\n"); + } +#endif + if(nclistlength(gz->codec_defaults)) { /* Try to provide default for any HDF5 filters without matching Codec. */ + /* Search the defaults */ + for(j=0;jcodec_defaults);j++) { + struct CodecAPI* dfalt = (struct CodecAPI*)nclistget(gz->codec_defaults,j); + if(dfalt->codec != NULL) { + const NCZ_codec_t* codec = dfalt->codec; + size_t hdf5id = codec->hdf5id; + NCZ_Plugin* p = NULL; + if(hdf5id <= 0 || hdf5id > gz->loaded_plugins_max) {ret = NC_EFILTER; goto done;} + p = gz->loaded_plugins[hdf5id]; /* get candidate */ + if(p != NULL && p->hdf5.filter != NULL + && p->codec.codec == NULL) { +#ifdef DEBUGL + fprintf(stderr,">>> DEBUGL: plugin defaulted: id=%u, codec=%s src=%s\n",hdf5id,codec->codecid,dfalt->codeclib->path); +#endif + p->codec.codec = codec; + p->codec.codeclib = dfalt->codeclib; + p->codec.defaulted = 1; + } + } + } + } + + /* Mark all plugins for which we do not have both HDF5 and codec */ + { + size_t i; + NCZ_Plugin* p; + for(i=1;iloaded_plugins_max;i++) { + if((p = gz->loaded_plugins[i]) != NULL) { + if(p->hdf5.filter == NULL || p->codec.codec == NULL) { + /* mark this entry as incomplete */ + p->incomplete = 1; +#ifdef DEBUGL + fprintf(stderr,">>> DEBUGL: Incomplete plugin: id=%u; reasons: %s %s\n",i, + (p->hdf5.filter==NULL?"hdf5":""),(p->codec.codec==NULL?"codec":"")); +#endif + } +#ifdef DEBUGL + else + fprintf(stderr,">>> DEBUGL: plugin accepted: id=%u\n",i); +#endif + } + } + } + /* Iniitalize all remaining plugins */ + { + size_t i; + NCZ_Plugin* p; + for(i=1;iloaded_plugins_max;i++) { + if((p = gz->loaded_plugins[i]) != NULL) { + if(p->incomplete) continue; + if(p->hdf5.filter != NULL && p->codec.codec != NULL) { + if(p->codec.codec && p->codec.codec->NCZ_codec_initialize) + p->codec.codec->NCZ_codec_initialize(); +#ifdef DEBUGL + fprintf(stderr,">>> DEBUGL: plugin initialized: id=%u\n",p->hdf5.filter->id); +#endif + } + } + } + } + +done: + nullfree(defaultpluginpath); + nclistfreeall(dirs); + errno = 0; + return ZUNTRACE(ret); +} + +/* Load all the filters within a specified directory */ +static int +NCZ_load_plugin_dir(GlobalNCZarr* gz, const char* path) +{ + size_t i; + int stat = NC_NOERR; + size_t pathlen; + NClist* contents = nclistnew(); + char* file = NULL; + + ZTRACE(7,"path=%s",path); + +#ifdef DEBUGL + fprintf(stderr,">>> DEBUGL: NCZ_load_plugin_dir: path=%s\n",path); +#endif + + if(path == NULL) {stat = NC_EINVAL; goto done;} + pathlen = strlen(path); + if(pathlen == 0) {stat = NC_EINVAL; goto done;} + + if((stat = getentries(path,contents))) goto done; + for(i=0;i 0); + nullfree(file); file = NULL; + if((file = (char*)malloc(flen))==NULL) {stat = NC_ENOMEM; goto done;} + file[0] = '\0'; + strlcat(file,path,flen); + strlcat(file,"/",flen); + strlcat(file,name,flen); + /* See if can load the file */ + stat = NCZ_load_plugin(file,&plugin); + switch (stat) { + case NC_NOERR: break; + case NC_ENOFILTER: case NC_ENOTFOUND: + stat = NC_NOERR; + break; /* will cause it to be ignored */ + default: goto done; + } + if(plugin != NULL) { + id = (size_t)plugin->hdf5.filter->id; + if(gz->loaded_plugins[id] == NULL) { + gz->loaded_plugins[id] = plugin; + if(id > gz->loaded_plugins_max) gz->loaded_plugins_max = id; +#ifdef DEBUGL + fprintf(stderr,">>> DEBUGL: plugin loaded: %s\n",printplugin(plugin)); +#endif + } else { +#ifdef DEBUGL + fprintf(stderr,">>> DEBUGL: plugin duplicate: %s\n",printplugin(plugin)); +#endif + NCZ_unload_plugin(plugin); /* its a duplicate */ + } + } else + stat = NC_NOERR; /*ignore failure */ + } + +done: + nullfree(file); + nclistfreeall(contents); + return ZUNTRACE(stat); +} + +int +NCZ_load_plugin(const char* path, struct NCZ_Plugin** plugp) +{ + int stat = NC_NOERR; + NCZ_Plugin* plugin = NULL; + const H5Z_class2_t* h5class = NULL; + H5PL_type_t h5type = 0; + const NCZ_codec_t** cp = NULL; + const NCZ_codec_t* codec = NULL; + NCPSharedLib* lib = NULL; + int flags = NCP_GLOBAL; + size_t h5id = 0; + NCglobalstate* gs = NC_getglobalstate(); + GlobalNCZarr* gz = (GlobalNCZarr*)gs->formatxstate.state[NC_FORMATX_NCZARR]; + + assert(path != NULL && strlen(path) > 0 && plugp != NULL); + + ZTRACE(8,"path=%s",path); + + if(plugp) *plugp = NULL; + +#if defined NAMEOPT || defined _WIN32 + /*triage because visual studio does a popup if the file will not load*/ + if(!pluginnamecheck(path)) {stat = NC_ENOFILTER; goto done;} +#endif + + /* load the shared library */ + if((stat = ncpsharedlibnew(&lib))) goto done; + if((stat = ncpload(lib,path,flags))) goto done; + +#ifdef DEBUGL + fprintf(stderr,">>> DEBUGL: NCZ_load_plugin: path=%s lib=%p\n",path,lib); +#endif + + /* See what we have */ + { + const H5PL_get_plugin_type_proto gpt = (H5PL_get_plugin_type_proto)ncpgetsymbol(lib,"H5PLget_plugin_type"); + const H5PL_get_plugin_info_proto gpi = (H5PL_get_plugin_info_proto)ncpgetsymbol(lib,"H5PLget_plugin_info"); + const NCZ_get_codec_info_proto npi = (NCZ_get_codec_info_proto)ncpgetsymbol(lib,"NCZ_get_codec_info"); + const NCZ_codec_info_defaults_proto cpd = (NCZ_codec_info_defaults_proto)ncpgetsymbol(lib,"NCZ_codec_info_defaults"); + + if(gpt == NULL && gpi == NULL && npi == NULL && cpd == NULL) + {stat = THROW(NC_ENOFILTER); goto done;} + + /* We can have cpd or we can have (gpt && gpi && npi) but not both sets */ + if(cpd != NULL) { + cp = (const NCZ_codec_t**)cpd(); + } else {/* cpd => !gpt && !gpi && !npi */ + if(gpt != NULL && gpi != NULL) { /* get HDF5 info */ + h5type = gpt(); + h5class = gpi(); + /* Verify */ + if(h5type != H5PL_TYPE_FILTER) {stat = NC_EPLUGIN; goto done;} + if(h5class->version != H5Z_CLASS_T_VERS) {stat = NC_EFILTER; goto done;} + } + if(npi != NULL) {/* get Codec info */ + codec = npi(); + /* Verify */ + if(codec->version != NCZ_CODEC_CLASS_VER) {stat = NC_EPLUGIN; goto done;} + if(codec->sort != NCZ_CODEC_HDF5) {stat = NC_EPLUGIN; goto done;} + } + } + } + +#ifdef DEBUGL +fprintf(stderr,">>> DEBUGL: load: %s:",path); +if(h5class) fprintf(stderr,">>> %u",(unsigned)h5class->id); +if(codec) fprintf(stderr,">>> %u/%s",codec->hdf5id,codec->codecid); +fprintf(stderr,">>> \n"); +#endif + + /* Handle defaults separately */ + if(cp != NULL) { + int used = 0; +#ifdef DEBUGL + fprintf(stderr,"@@@ %s: default codec library found: %p\n",path,cp); +#endif + if((stat = loadcodecdefaults(gz,path,cp,lib,&used))) goto done; + if(used) lib = NULL; + goto done; + } + + if(h5class != NULL && codec != NULL) { + /* Verify consistency of the HDF5 and the Codec */ + if(((size_t)h5class->id) != codec->hdf5id) goto done; /* ignore */ + } + + /* There are several cases to consider: + 1. This library has both HDF5 API and Codec API => merge + 2. This library has HDF5 API only and Codec API was already found in another library => merge + 3. This library has Codec API only and HDF5 API was already found in another library => merge + */ + + /* Get any previous plugin entry for this id; may be NULL */ + if(h5class != NULL) { + h5id = (size_t)h5class->id; + if((stat = NCZ_plugin_loaded(h5id,&plugin))) goto done; + } else if(codec != NULL) { + h5id = (size_t)codec->hdf5id; + if((stat = NCZ_plugin_loaded(h5id,&plugin))) goto done; + } + + if(plugin == NULL) { + /* create new entry */ + if((plugin = (NCZ_Plugin*)calloc(1,sizeof(NCZ_Plugin)))==NULL) {stat = NC_ENOMEM; goto done;} + } + + /* Fill in the plugin */ + if(h5class != NULL && plugin->hdf5.filter == NULL) { + plugin->hdf5.filter = h5class; + plugin->hdf5.hdf5lib = lib; + lib = NULL; + } + if(codec != NULL && plugin->codec.codec == NULL) { + plugin->codec.codec = codec; + plugin->codec.codeclib = lib; + lib = NULL; + } +#ifdef DEBUGL + if(plugin) + fprintf(stderr,">>> DEBUGL: load_plugin: %s\n",printplugin(plugin)); +#endif + /* Cleanup */ + if(plugin->hdf5.hdf5lib == plugin->codec.codeclib) /* Works for NULL case also */ + plugin->codec.codeclib = NULL; + if((stat=NCZ_plugin_save(gz,h5id,plugin))) goto done; + plugin = NULL; + +done: + if(lib) + (void)ncpsharedlibfree(lib); + if(plugin) NCZ_unload_plugin(plugin); + return ZUNTRACEX(stat,"plug=%p",*plugp); +} + +int +NCZ_unload_plugin(NCZ_Plugin* plugin) +{ + NCglobalstate* gs = NC_getglobalstate(); + GlobalNCZarr* gz = (GlobalNCZarr*)gs->formatxstate.state[NC_FORMATX_NCZARR]; + + ZTRACE(9,"plugin=%p",plugin); + + if(plugin) { +#ifdef DEBUGL + fprintf(stderr,">>> DEBUGL: unload: %s\n",printplugin(plugin)); +#endif + if(plugin->codec.codec && plugin->codec.codec->NCZ_codec_finalize) + plugin->codec.codec->NCZ_codec_finalize(); + if(plugin->hdf5.filter != NULL) gz->loaded_plugins[plugin->hdf5.filter->id] = NULL; + if(plugin->hdf5.hdf5lib != NULL) (void)ncpsharedlibfree(plugin->hdf5.hdf5lib); + if(!plugin->codec.defaulted && plugin->codec.codeclib != NULL) (void)ncpsharedlibfree(plugin->codec.codeclib); +memset(plugin,0,sizeof(NCZ_Plugin)); + free(plugin); + } + return ZUNTRACE(NC_NOERR); +} + +int +NCZ_plugin_loaded(size_t filterid, NCZ_Plugin** pp) +{ + int stat = NC_NOERR; + NCglobalstate* gs = NC_getglobalstate(); + GlobalNCZarr* gz = (GlobalNCZarr*)gs->formatxstate.state[NC_FORMATX_NCZARR]; + + struct NCZ_Plugin* plug = NULL; + ZTRACE(6,"filterid=%d",filterid); + if(filterid <= 0 || filterid >= H5Z_FILTER_MAX) + {stat = NC_EINVAL; goto done;} + if(filterid <= gz->loaded_plugins_max) + plug = gz->loaded_plugins[filterid]; + if(pp) *pp = plug; +done: + return ZUNTRACEX(stat,"plugin=%p",*pp); +} + +int +NCZ_plugin_loaded_byname(const char* name, NCZ_Plugin** pp) +{ + int stat = NC_NOERR; + size_t i; + struct NCZ_Plugin* plug = NULL; + NCglobalstate* gs = NC_getglobalstate(); + GlobalNCZarr* gz = (GlobalNCZarr*)gs->formatxstate.state[NC_FORMATX_NCZARR]; + + ZTRACE(6,"pluginname=%s",name); + if(name == NULL) {stat = NC_EINVAL; goto done;} + for(i=1;i<=gz->loaded_plugins_max;i++) { + if (!gz->loaded_plugins[i]) continue; + if(!gz->loaded_plugins[i] || !gz->loaded_plugins[i]->codec.codec) continue; /* no plugin or no codec */ + if(strcmp(name, gz->loaded_plugins[i]->codec.codec->codecid) == 0) + {plug = gz->loaded_plugins[i]; break;} + } + if(pp) *pp = plug; +done: + return ZUNTRACEX(stat,"plugin=%p",*pp); +} + +static int +NCZ_plugin_save(GlobalNCZarr* gz, size_t filterid, NCZ_Plugin* p) +{ + int stat = NC_NOERR; + + ZTRACE(6,"filterid=%d p=%p",filterid,p); + if(filterid <= 0 || filterid >= H5Z_FILTER_MAX) + {stat = NC_EINVAL; goto done;} + if(filterid > gz->loaded_plugins_max) gz->loaded_plugins_max = filterid; + gz->loaded_plugins[filterid] = p; +done: + return ZUNTRACE(stat); +} + +static int +loadcodecdefaults(GlobalNCZarr* gz, const char* path, const NCZ_codec_t** cp, NCPSharedLib* lib, int* lib_usedp) +{ + int stat = NC_NOERR; + int lib_used = 0; + + nclistpush(gz->default_libs,lib); + for(;*cp;cp++) { + struct CodecAPI* c0; +#ifdef DEBUGL + fprintf(stderr,"@@@ %s: %s = %u\n",path,(*cp)->codecid,(*cp)->hdf5id); +#endif + c0 = (struct CodecAPI*)calloc(1,sizeof(struct CodecAPI)); + if(c0 == NULL) {stat = NC_ENOMEM; goto done;} + c0->codec = *cp; + c0->codeclib = lib; + lib_used = 1; /* remember */ + nclistpush(gz->codec_defaults,c0); c0 = NULL; + } +done: + if(lib_usedp) *lib_usedp = lib_used; + return THROW(stat); +} + +#if defined NAMEOPT || defined _WIN32 +static int +pluginnamecheck(const char* name) +{ + size_t count,len; + long i; + const char* p; + if(name == NULL) return 0; + /* get basename */ + p = strrchr(name,'/'); + if(p != NULL) name = (p+1); + len = strlen(name); + if(len == 0) return 0; + i = (long)(len-1); + count = 1; + p = name+i; + for(;i>=0;i--,count++,p--) { + char c = *p; + if(c == '/') break; + if(c == '.') { + if(count >= 3 && memcmp(p,".so",3)==0) return 1; + if(count >= 4 && memcmp(p,".dll",4)==0) return 1; + if(count >= 6 && memcmp(p,".dylib",6)==0) return 1; + } + } + return 0; +} +#endif + +/**************************************************/ +/* +Get entries in a path that is assumed to be a directory. +*/ + +#ifdef _WIN32 + +static int +getentries(const char* path, NClist* contents) +{ + /* Iterate over the entries in the directory */ + int ret = NC_NOERR; + errno = 0; + WIN32_FIND_DATA FindFileData; + HANDLE dir = NULL; + char* ffpath = NULL; + char* lpath = NULL; + size_t len; + char* d = NULL; + + ZTRACE(6,"path=%s",path); + + /* We need to process the path to make it work with FindFirstFile */ + len = strlen(path); + /* Need to terminate path with '/''*' */ + ffpath = (char*)malloc(len+2+1); + memcpy(ffpath,path,len); + if(path[len-1] != '/') { + ffpath[len] = '/'; + len++; + } + ffpath[len] = '*'; len++; + ffpath[len] = '\0'; + + /* localize it */ + if((ret = nczm_localize(ffpath,&lpath,LOCALIZE))) goto done; + dir = FindFirstFile(lpath, &FindFileData); + if(dir == INVALID_HANDLE_VALUE) { + /* Distinquish not-a-directory from no-matching-file */ + switch (GetLastError()) { + case ERROR_FILE_NOT_FOUND: /* No matching files */ /* fall thru */ + ret = NC_NOERR; + goto done; + case ERROR_DIRECTORY: /* not a directory */ + default: + ret = NC_EEMPTY; + goto done; + } + } + do { + char* p = NULL; + const char* name = NULL; + name = FindFileData.cFileName; + if(strcmp(name,".")==0 || strcmp(name,"..")==0) + continue; + nclistpush(contents,strdup(name)); + } while(FindNextFile(dir, &FindFileData)); + +done: + if(dir) FindClose(dir); + nullfree(lpath); + nullfree(ffpath); + nullfree(d); + errno = 0; + return ZUNTRACEX(ret,"|contents|=%d",(int)nclistlength(contents)); +} + +#else /* !_WIN32 */ + +int +getentries(const char* path, NClist* contents) +{ + int ret = NC_NOERR; + errno = 0; + DIR* dir = NULL; + + ZTRACE(6,"path=%s",path); + + dir = NCopendir(path); + if(dir == NULL) + {ret = (errno); goto done;} + for(;;) { + const char* name = NULL; + struct dirent* de = NULL; + errno = 0; + de = readdir(dir); + if(de == NULL) + {ret = (errno); goto done;} + if(strcmp(de->d_name,".")==0 || strcmp(de->d_name,"..")==0) + continue; + name = de->d_name; + nclistpush(contents,strdup(name)); + } +done: + if(dir) NCclosedir(dir); + errno = 0; + return ZUNTRACEX(ret,"|contents|=%d",(int)nclistlength(contents)); +} +#endif /*_WIN32*/ + +/**************************************************/ + + +#if defined(DEBUGF) || defined(DEBUGL) + +const char* +printplugin(const NCZ_Plugin* plugin) +{ + static char plbuf[4096]; + char plbuf2[2000]; + char plbuf1[2000]; + + if(plugin == NULL) return "plugin=NULL"; + plbuf2[0] = '\0'; plbuf1[0] = '\0'; + if(plugin->hdf5.filter) + snprintf(plbuf1,sizeof(plbuf1),"hdf5={id=%u name=%s}",plugin->hdf5.filter->id,plugin->hdf5.filter->name); + if(plugin->codec.codec) + snprintf(plbuf2,sizeof(plbuf2),"codec={codecid=%s hdf5id=%u}",plugin->codec.codec->codecid,plugin->codec.codec->hdf5id); + snprintf(plbuf,4096,"plugin={%s %s}",plbuf1,plbuf2); + return plbuf; +} + +static char* +printparams(size_t nparams, const unsigned* params) +{ + static char ppbuf[4096]; + if(nparams == 0) + snprintf(ppbuf,4096,"{0,%p}",params); + else + snprintf(ppbuf,4096,"{%u %s}",(unsigned)nparams,nczprint_paramvector(nparams,params)); + return ppbuf; +} + +static char* +printnczparams(const NCZ_Params p) +{ + return printparams(p.nparams,p.params); +} + +static const char* +printcodec(const NCZ_Codec c) +{ + static char pcbuf[4096]; + snprintf(pcbuf,sizeof(pcbuf),"{id=%s codec=%s}", + c.id,NULLIFY(c.codec)); + return pcbuf; +} + +static const char* +printhdf5(const NCZ_HDF5 h) +{ + static char phbuf[4096]; + snprintf(phbuf,sizeof(phbuf),"{id=%u visible=%s working=%s}", + h.id, printnczparams(h.visible), printnczparams(h.working)); + return phbuf; +} +#endif /* defined(DEBUGF) || defined(DEBUGL) */ + +/* Suppress selected unused static functions */ +static void static_unused(void) +{ + void* p = NULL; + p = p; + p = static_unused; +#if defined(DEBUGF) || defined(DEBUGL) +(void)printplugin(NULL); +(void)printparams(0, NULL); +(void)printnczparams(const NCZ_Params p); +(void)printcodec(const NCZ_Codec c); +(void)printhdf5(const NCZ_HDF5 h); +#endif +} + diff --git a/libnczarr/zplugins.h b/libnczarr/zplugins.h new file mode 100644 index 0000000000..3ad26abd77 --- /dev/null +++ b/libnczarr/zplugins.h @@ -0,0 +1,45 @@ +/* Copyright 2018-2018 University Corporation for Atmospheric + Research/Unidata. */ + +/** + * @file This header file containsplugin related macros, types, and prototypes for + * the plugin code in libnczarr. This header should not be included in + * code outside libnczarr. + * + * @author Dennis Heimbigner + */ + +#ifndef ZPLUGIN_H +#define ZPLUGIN_H + + +/* zplugin.c */ + +/* Pluginlist management */ + +/* Opaque Handles */ +struct H5Z_class2_t; +struct NCZ_codec_t; +struct NCPSharedLib; + +/* Hold the loaded filter plugin information */ +typedef struct NCZ_Plugin { + int incomplete; + struct HDF5API { + const struct H5Z_class2_t* filter; + struct NCPSharedLib* hdf5lib; /* source of the filter */ + } hdf5; + struct CodecAPI { + int defaulted; /* codeclib was a defaulting library */ + int ishdf5raw; /* The codec is the hdf5raw codec */ + const struct NCZ_codec_t* codec; + struct NCPSharedLib* codeclib; /* of the codec; null if same as hdf5 */ + } codec; +} NCZ_Plugin; + +int NCZ_load_all_plugins(void); +int NCZ_plugin_loaded(size_t filterid, NCZ_Plugin** pp); +int NCZ_plugin_loaded_byname(const char* name, NCZ_Plugin** pp); + +#endif /*ZPLUGIN_H*/ + diff --git a/libnczarr/zsync.c b/libnczarr/zsync.c index c9d55ee751..a79be7f69c 100644 --- a/libnczarr/zsync.c +++ b/libnczarr/zsync.c @@ -1533,7 +1533,7 @@ define_var1(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const char* varname) /* Capture dimension_separator (must precede chunk cache creation) */ { NCglobalstate* ngs = NC_getglobalstate(); - assert(ngs != NULL); + GlobalNCZarr* ncz = (GlobalNCZarr*)ngs->formatxstate.state[NC_FORMATX_NCZARR]; zvar->dimension_separator = 0; if((stat = NCJdictget(jvar,"dimension_separator",&jvalue))<0) {stat = NC_EINVAL; goto done;} if(jvalue != NULL) { @@ -1543,7 +1543,7 @@ define_var1(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const char* varname) } /* If value is invalid, then use global default */ if(!islegaldimsep(zvar->dimension_separator)) - zvar->dimension_separator = ngs->zarr.dimension_separator; /* use global value */ + zvar->dimension_separator = ncz->dimension_separator; /* use global value */ assert(islegaldimsep(zvar->dimension_separator)); /* we are hosed */ } @@ -1650,7 +1650,6 @@ define_var1(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const char* varname) if(var->filters == NULL) var->filters = (void*)nclistnew(); if(zvar->incompletefilters == NULL) zvar->incompletefilters = (void*)nclistnew(); chainindex = 0; /* track location of filter in the chain */ - if((stat = NCZ_filter_initialize())) goto done; if((stat = NCJdictget(jvar,"filters",&jvalue))<0) {stat = NC_EINVAL; goto done;} if(jvalue != NULL && NCJsort(jvalue) != NCJ_NULL) { int k; @@ -1672,7 +1671,6 @@ define_var1(NC_FILE_INFO_T* file, NC_GRP_INFO_T* grp, const char* varname) #ifdef NETCDF_ENABLE_NCZARR_FILTERS { if(var->filters == NULL) var->filters = (void*)nclistnew(); - if((stat = NCZ_filter_initialize())) goto done; if((stat = NCJdictget(jvar,"compressor",&jfilter))<0) {stat = NC_EINVAL; goto done;} if(jfilter != NULL && NCJsort(jfilter) != NCJ_NULL) { if(NCJsort(jfilter) != NCJ_DICT) {stat = NC_EFILTER; goto done;} diff --git a/libnczarr/zvar.c b/libnczarr/zvar.c index eb26eb7eda..dac90daf3e 100644 --- a/libnczarr/zvar.c +++ b/libnczarr/zvar.c @@ -294,7 +294,8 @@ NCZ_def_var(int ncid, const char *name, nc_type xtype, int ndims, char norm_name[NC_MAX_NAME + 1]; int d; int retval; - NCglobalstate* gstate = NC_getglobalstate(); + NCglobalstate* gs = NC_getglobalstate(); + GlobalNCZarr* gz = (GlobalNCZarr*)gs->formatxstate.state[NC_FORMATX_NCZARR]; ZTRACE(1,"ncid=%d name=%s xtype=%d ndims=%d dimids=%s",ncid,name,xtype,ndims,nczprint_idvector(ndims,dimidsp)); @@ -381,7 +382,7 @@ NCZ_def_var(int ncid, const char *name, nc_type xtype, int ndims, zvar->common.file = h5; zvar->scalar = (ndims == 0 ? 1 : 0); - zvar->dimension_separator = gstate->zarr.dimension_separator; + zvar->dimension_separator = gz->dimension_separator; assert(zvar->dimension_separator != 0); /* Set these state flags for the var. */ @@ -457,7 +458,7 @@ var->type_info->rc++; zvar->chunksize = zvar->chunkproduct * var->type_info->size; /* Set cache defaults */ - var->chunkcache = gstate->chunkcache; + var->chunkcache = gs->chunkcache; /* Create the cache */ if((retval=NCZ_create_chunk_cache(var,zvar->chunkproduct*var->type_info->size,zvar->dimension_separator,&zvar->cache))) diff --git a/libsrc4/nc4dispatch.c b/libsrc4/nc4dispatch.c index 2aa5987824..3407960b32 100644 --- a/libsrc4/nc4dispatch.c +++ b/libsrc4/nc4dispatch.c @@ -25,7 +25,6 @@ extern NC_Dispatch UDF0_DISPATCH; extern NC_Dispatch UDF1_DISPATCH; #endif /* USE_UDF1 */ - /** * @internal Initialize netCDF-4. If user-defined format(s) have been * specified in configure, load their dispatch table(s). @@ -62,6 +61,7 @@ NC4_initialize(void) } #endif #endif + NC_initialize_reserved(); return ret; } diff --git a/nc_test4/CMakeLists.txt b/nc_test4/CMakeLists.txt index caa75aff18..f30194385d 100644 --- a/nc_test4/CMakeLists.txt +++ b/nc_test4/CMakeLists.txt @@ -40,6 +40,8 @@ IF(NETCDF_BUILD_UTILITIES) ADD_SH_TEST(nc_test4 tst_misc) build_bin_test(tst_fillonly) ADD_SH_TEST(nc_test4 test_fillonly) +# H5 and nczarr Fixed string support + build_bin_test(build_fixedstring) ADD_SH_TEST(nc_test4 tst_fixedstring) IF(USE_HDF5 AND NETCDF_ENABLE_FILTER_TESTING) build_bin_test(tst_filterparser) diff --git a/nc_test4/Makefile.am b/nc_test4/Makefile.am index 683b9a1145..a07e20856d 100644 --- a/nc_test4/Makefile.am +++ b/nc_test4/Makefile.am @@ -68,7 +68,8 @@ TESTS += run_grp_rename.sh tst_misc.sh check_PROGRAMS += tst_fillonly TESTS += test_fillonly.sh -# H5 and nczarr Fixed string support +# H5 Fixed string support +check_PROGRAMS += build_fixedstring TESTS += tst_fixedstring.sh # Szip Tests (requires ncdump) @@ -143,7 +144,7 @@ ref_filter_repeat.txt ref_fillonly.cdl test_fillonly.sh \ ref_filter_order_create.txt ref_filter_order_read.txt ref_any.cdl \ tst_specific_filters.sh tst_unknown.sh tst_virtual_datasets.c \ noop1.cdl unknown.cdl tst_broken_files.c ref_bloscx.cdl \ -tst_bloscfail.sh tst_fixedstring.sh ref_fixedstring.h5 \ +tst_bloscfail.sh tst_fixedstring.sh \ ref_fixedstring.cdl tst_filterinstall.sh tst_filter_vlen.sh \ tst_filter_misc.sh run_zstd_test.sh run_par_warn_test.sh.in \ ref_tmp_tst_warn_out.txt @@ -159,7 +160,7 @@ floats*.nc floats*.cdl shorts*.nc shorts*.cdl ints*.nc ints*.cdl \ testfilter_reg.nc filterrepeat.txt tmp_fillonly.nc \ testfilter_order.nc crfilterorder.txt rdfilterorder.txt 1 \ tmp_*.txt tmp_*.nc tmp*.dump tmp*.cdl tmp*.txt tmp*.tmp \ -tmp_bzip2.c bzip2.nc noop.nc tmp_*.dmp tmp_*.cdl +tmp_bzip2.c bzip2.nc noop.nc tmp_*.dmp tmp_*.cdl ref_fixedstring.h5 DISTCLEANFILES = findplugin.sh run_par_test.sh run_par_warn_test.sh @@ -168,13 +169,3 @@ clean-local: # If valgrind is present, add valgrind targets. @VALGRIND_CHECK_RULES@ - -# The (otherwise unused) program build_fixedstring.c -# is used to generate the test file ref_fixedstring.h5. -# That test file is build and included as part of the distribution, -# so the build_fixedstring.c program generally does not need to -# be executed unless the test file needs to be modified.. - -check_PROGRAMS += build_fixedstring -ref_fixedstring.h5: build_fixedstring.c - ${srcdir}/buildfixedstring diff --git a/nc_test4/ref_fixedstring.cdl b/nc_test4/ref_fixedstring.cdl index 09f6fdc34a..8912999bbc 100644 --- a/nc_test4/ref_fixedstring.cdl +++ b/nc_test4/ref_fixedstring.cdl @@ -1,9 +1,16 @@ netcdf ref_fixedstring { dimensions: - phony_dim_0 = 3 ; + phony_dim_0 = 4 ; variables: - string test(phony_dim_0) ; + string v1 ; + string vn(phony_dim_0) ; + +// global attributes: + :att1 = "abcd" ; + string :attn = "abcd", "efgh", "ijkl", "mnop" ; data: - test = "foo", "bar", "baz" ; + v1 = "abcd" ; + + vn = "abcd", "efgh", "ijkl", "mnop" ; } diff --git a/nc_test4/ref_fixedstring.h5 b/nc_test4/ref_fixedstring.h5 deleted file mode 100644 index b2cc269362476134b5ebc2b70af90d0902b6adaf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2057 zcmeD5aB<`1lHy_j0S*oZ76t(@6Gr@p0!|Ky2#gPtPk=HQp>zk7Ucm%mFfxE31A_!q zTo7tLy1I}cS62q0N|^aD8mf)KfCa*WIs+y=N{^5b@Njhu0C_b6>R(tYJpoN;uwW=j zEiM7EVd>EWCP606$iNCQ3u+)EG$k`KLIjwh<|1eguzF#jG)Rn<0m5fuVghU6fa+&v z1WJGbC)5NOH3P;NlytD4zY8cw!}6g5%;SU7wWE$24S~@Rz!w5(`T0qSMM;TO0MGX@ Axc~qF diff --git a/nc_test4/tst_fixedstring.sh b/nc_test4/tst_fixedstring.sh index f27e456e9e..038299a191 100755 --- a/nc_test4/tst_fixedstring.sh +++ b/nc_test4/tst_fixedstring.sh @@ -6,10 +6,11 @@ if test "x$srcdir" = x ; then srcdir=`pwd`; fi set -e # Note, the test file for this is ref_fixedstring.h5 -# But is is generated by the (otherwise unused) program -# ../h5_test/tst_h_fixedstrings.c. +# and is generated on the fly by build_fixedstring. +${execdir}/build_fixedstring echo "*** Test reading a file with HDF5 fixed length strings" rm -f ./tmp_fixedstring.cdl -$NCDUMP ${srcdir}/ref_fixedstring.h5 > ./tmp_fixedstring.cdl +$NCDUMP ${execdir}/ref_fixedstring.h5 > ./tmp_fixedstring.cdl diff -b -w ${srcdir}/ref_fixedstring.cdl ./tmp_fixedstring.cdl +rm -f ${execdir}/ref_fixedstring.h5 diff --git a/unit_test/CMakeLists.txt b/unit_test/CMakeLists.txt index 39074424d8..9bda2f1255 100644 --- a/unit_test/CMakeLists.txt +++ b/unit_test/CMakeLists.txt @@ -41,11 +41,19 @@ IF(NETCDF_BUILD_UTILITIES) build_bin_test(aws_config) add_sh_test(unit_test run_aws_config) ENDIF() + + # plugin path test + if(NETCDF_ENABLE_NCZARR AND NETCDF_ENABLE_HDF5) + build_bin_test(tst_pluginpaths ${XGETOPTSRC}) + add_sh_test(unit_test run_pluginpaths) + endif() ENDIF() # Performance tests +if(BUILD_BENCHMARKS) add_bin_test(unit_test tst_exhash timer_utils.c) add_bin_test(unit_test tst_xcache timer_utils.c) +endif() FILE(GLOB COPY_FILES ${CMAKE_CURRENT_SOURCE_DIR}/*.sh) FILE(COPY ${COPY_FILES} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/ FILE_PERMISSIONS OWNER_WRITE OWNER_READ OWNER_EXECUTE) diff --git a/unit_test/Makefile.am b/unit_test/Makefile.am index 6061edaf41..4752397fd3 100644 --- a/unit_test/Makefile.am +++ b/unit_test/Makefile.am @@ -6,7 +6,7 @@ # functions, to allow Windows to work. Since we have not extern'd # these functions, they will only be run under the autotools build. -# Ed Hartnett 8/9/19 +# Dennis Heimbigner 8/27/2024 #SH_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose #sh_LOG_DRIVER = $(SHELL) $(top_srcdir)/test-driver-verbose @@ -25,12 +25,15 @@ TESTS = check_PROGRAMS += tst_nclist test_ncuri test_pathcvt +TESTS += tst_nclist test_ncuri test_pathcvt + # Performance tests +if BUILD_BENCHMARKS check_PROGRAMS += tst_exhash tst_xcache tst_exhash_SOURCES = tst_exhash.c timer_utils.c timer_utils.h tst_xcache_SOURCES = tst_xcache.c timer_utils.c timer_utils.h - -TESTS += tst_nclist test_ncuri test_pathcvt tst_exhash tst_xcache +TESTS += tst_exhash tst_xcache +endif if USE_HDF5 check_PROGRAMS += tst_nc4internal tst_reclaim @@ -38,6 +41,14 @@ TESTS += tst_nc4internal TESTS += run_reclaim_tests.sh endif # USE_HDF5 +# Test for HDF5 OR NCZARR enabled +if USE_HDF5 +if NETCDF_ENABLE_NCZARR +check_PROGRAMS += tst_pluginpaths +TESTS += run_pluginpaths.sh +endif +endif + if NETCDF_ENABLE_S3 if NETCDF_ENABLE_S3_TESTALL check_PROGRAMS += test_s3sdk @@ -47,10 +58,11 @@ check_PROGRAMS += aws_config TESTS += run_aws_config.sh endif -EXTRA_DIST = CMakeLists.txt run_s3sdk.sh run_reclaim_tests.sh +EXTRA_DIST = CMakeLists.txt run_s3sdk.sh run_reclaim_tests.sh run_aws_config.sh run_pluginpaths.sh EXTRA_DIST += nctest_netcdf4_classic.nc reclaim_tests.cdl +EXTRA_DIST += ref_read.txt ref_write.txt -CLEANFILES = reclaim_tests*.txt reclaim_tests.nc +CLEANFILES = reclaim_tests*.txt reclaim_tests.nc tmp_*.txt # If valgrind is present, add valgrind targets. @VALGRIND_CHECK_RULES@ diff --git a/unit_test/aws_config.c b/unit_test/aws_config.c index 1eb6c35c69..c9764ee18b 100644 --- a/unit_test/aws_config.c +++ b/unit_test/aws_config.c @@ -46,8 +46,7 @@ main(int argc, char** argv) int c = 0,stat = NC_NOERR; /* Load RC and .aws/config */ - CHECK(nc_initialize()); - CHECK(NC_s3sdkinitialize()); + CHECK(nc_initialize()); /* Will invoke NC_s3sdkinitialize()); */ NCglobalstate* gs = NC_getglobalstate(); //Not checking, aborts if ~/.aws/config doesn't exist CHECK(NC_aws_load_credentials(gs)); diff --git a/unit_test/ref_read.txt b/unit_test/ref_read.txt new file mode 100644 index 0000000000..0d08a18833 --- /dev/null +++ b/unit_test/ref_read.txt @@ -0,0 +1,5 @@ +testread(hdf5): /zero:/one:/two:/hdf5:/three:/four +testread(nczarr): /zero:/one:/two:/nczarr:three:/four:/five +testread(all): +testread(hdf5): /zero:/one:/two:/hdf5:/three:/four +testread(nczarr): /zero:/one:/two:/nczarr:three:/four:/five diff --git a/unit_test/ref_write.txt b/unit_test/ref_write.txt new file mode 100644 index 0000000000..b2d303dc0e --- /dev/null +++ b/unit_test/ref_write.txt @@ -0,0 +1,9 @@ +testwrite(hdf5): before: /zero:/one:/two:/hdf5:/three:/four +testwrite(hdf5): after: /zero:/one:/two:/hdf5:/three:/four:/modhdf5 +testwrite(nczarr): before: /zero:/one:/two:/nczarr:three:/four:/five +testwrite(nczarr): after: /modnczarr:/zero:/one:/two:/nczarr:three:/four:/five +testwrite(all): +testwrite(hdf5): before: /zero:/one:/two:/hdf5:/three:/four +testwrite(hdf5): after: /zero:/one:/two:/hdf5:/three:/four:/modhdf5 +testwrite(nczarr): before: /zero:/one:/two:/nczarr:three:/four:/five +testwrite(nczarr): after: /modnczarr:/zero:/one:/two:/nczarr:three:/four:/five diff --git a/unit_test/run_aws_config.sh b/unit_test/run_aws_config.sh index 44b3526643..94060d3011 100755 --- a/unit_test/run_aws_config.sh +++ b/unit_test/run_aws_config.sh @@ -7,7 +7,7 @@ set -e #CMD="valgrind --leak-check=full" -isolate "testdir_ut_aws_config" +isolate "testdir_aws_config" THISDIR=`pwd` cd $ISOPATH @@ -19,6 +19,10 @@ test_cleanup() { } trap test_cleanup EXIT +test_cleanup + +mkdir -p $THISDIR/.aws + cat << 'EOF' > $THISDIR/.aws/config [uni] region = somewhere-1 @@ -37,31 +41,37 @@ endpoint_url = https://endpoint.example.com/ EOF cat << 'EOF' > $THISDIR/.aws/credentials -[play] -aws_access_key_id = DummyKeys -aws_secret_access_key = DummySecret - [uni] region = somewhere-2 endpoint_url = https://example.com/bucket/prefix/2 key = value-overwritten + +[play] +aws_access_key_id = DummyKeys +aws_secret_access_key = DummySecret EOF echo -e "Testing loading AWS configuration in ${THISDIR}/.aws/config" -NC_TEST_AWS_DIR=${THISDIR} AWS_PROFILE=unidata ${CMD} ${execdir}/aws_config endpoint_url region dummy_key +export NC_TEST_AWS_DIR=${THISDIR} +export AWS_PROFILE=unidata +${CMD} ${execdir}/aws_config endpoint_url region dummy_key echo "Status: $?" - -NC_TEST_AWS_DIR=${THISDIR} AWS_PROFILE=play ${CMD} ${execdir}/aws_config endpoint_url region +exit +export AWS_PROFILE=play +${CMD} ${execdir}/aws_config endpoint_url region echo "Status: $?" -NC_TEST_AWS_DIR=${THISDIR} AWS_PROFILE=uni ${CMD} ${execdir}/aws_config endpoint_url region key +export AWS_PROFILE=uni +${CMD} ${execdir}/aws_config endpoint_url region key echo "Status: $?" -NC_TEST_AWS_DIR=${THISDIR} AWS_PROFILE=uni ${CMD} ${execdir}/aws_config key=value-overwritten region=somewhere-2 endpoint_url=https://example.com/bucket/prefix/2 extrakey=willbepropagated +export AWS_PROFILE=uni +${CMD} ${execdir}/aws_config key=value-overwritten region=somewhere-2 endpoint_url=https://example.com/bucket/prefix/2 extrakey=willbepropagated echo "Status: $?" # Will use profile=no -NC_TEST_AWS_DIR=${THISDIR} ${CMD} ${execdir}/aws_config 2>&1 | grep -q 'Active profile:no' +unset AWS_PROFILE +${CMD} ${execdir}/aws_config 2>&1 | grep -q 'Active profile:no' echo "Status: $?" echo -e "Finished" diff --git a/unit_test/run_pluginpaths.sh b/unit_test/run_pluginpaths.sh new file mode 100755 index 0000000000..aded28700d --- /dev/null +++ b/unit_test/run_pluginpaths.sh @@ -0,0 +1,123 @@ +#!/bin/bash + +# Test the programmatic API for manipulating the plugin paths. +# WARNING: This file is also used to build nczarr_test/run_pluginpath.sh + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +set -e + +IMPLS= +if test "x$FEATURE_HDF5" = xyes ; then IMPLS="$IMPLS hdf5"; fi +if test "x$FEATURE_NCZARR" = xyes ; then IMPLS="$IMPLS nczarr"; fi +# Remove leading blank +IMPLS=`echo "$IMPLS" | cut -d' ' -f2,3` +echo "IMPLS=|$IMPLS|" + +#VERBOSE=1 + +DFALT="/zero;/one;/two;/three;/four" +DFALTHDF5="/zero;/one;/two;/hdf5;/three;/four" +DFALTNCZARR="/zero;/one;/two;/nczarr;three;/four;/five" + +if test "x$TESTNCZARR" = x1 ; then +. "$srcdir/test_nczarr.sh" +s3isolate "testdir_pluginpath" +THISDIR=`pwd` +cd $ISOPATH +fi + +TP="${execdir}/tst_pluginpaths" + +filenamefor() { + # tmp|ref_action + eval "filename=${1}_$2" +} + +dfaltfor() { + case "$1" in + hdf5) eval "dfalt=\"$DFALTHDF5\"" ;; + nczarr) eval "dfalt=\"$DFALTNCZARR\"" ;; + all) eval "dfalt=\"$DFALT\"" ;; + esac +} + +modfor() { + local formatx="$1" + local dfalt="$2" + case "$formatx" in + hdf5) mod="${dfalt};/modhdf5" ;; + nczarr) mod="/modnczarr;${dfalt}" ;; + all) mode="${dfalt}" ;; + esac +} + +##### + +testread() { + local formatx="$1" + filenamefor tmp read + dfaltfor $formatx + if test "$formatx" = all ; then + echo "testread(${formatx}): " >> ${filename}.txt + for f in $IMPLS ; do testread $f ; done + elif test "x$FEATURE_HDF5" && test "$formatx" = hdf5 ; then + echon "testread(${formatx}): " >> ${filename}.txt + ${TP} -x "formatx:${formatx},write:${dfalt},read" >> ${filename}.txt ; + elif test "x$FEATURE_NCZARR" && test "$formatx" = nczarr ; then + echon "testread(${formatx}): " >> ${filename}.txt + ${TP} -x "formatx:${formatx},write:${dfalt},read" >> ${filename}.txt ; + fi +} + +testwrite() { + local formatx="$1" + filenamefor tmp write + dfaltfor $formatx + modfor $formatx "$dfalt" + if test "$formatx" = all ; then + echo "testwrite(${formatx}): " >> ${filename}.txt + for f in $IMPLS ; do testwrite $f ; done + elif test "x$FEATURE_HDF5" && test "$formatx" = hdf5 ; then + echon "testwrite(${formatx}): before: " >> ${filename}.txt + ${TP} -x "formatx:${formatx},write:${dfalt},read" >> ${filename}.txt + echon "testwrite(${formatx}): after: " >> ${filename}.txt + ${TP} -x "formatx:${formatx},write:${mod},read" >> ${filename}.txt + elif test "x$FEATURE_NCZARR" && test "$formatx" = nczarr ; then + echon "testwrite(${formatx}): before: " >> ${filename}.txt + ${TP} -x "formatx:${formatx},write:${dfalt},read" >> ${filename}.txt + echon "testwrite(${formatx}): after: " >> ${filename}.txt + ${TP} -x "formatx:${formatx},write:${mod},read" >> ${filename}.txt + fi +} + +######################### + +cleanup() { + rm -f tmp_*.txt +} + +init() { + cleanup +} + +# Verify output for a specific action +verify() { + for action in read write ; do + if diff -wBb ${srcdir}/ref_${action}.txt tmp_${action}.txt ; then + echo "***PASS: $action" + else + echo "***FAIL: $action" + exit 1 + fi + done +} + +init +for fx in $IMPLS all ; do + testread $fx + testwrite $fx +done +verify +cleanup diff --git a/unit_test/tst_pluginpaths.c b/unit_test/tst_pluginpaths.c new file mode 100644 index 0000000000..0b508d9cbc --- /dev/null +++ b/unit_test/tst_pluginpaths.c @@ -0,0 +1,331 @@ +/* + * Copyright 2018, University Corporation for Atmospheric Research + * See netcdf/COPYRIGHT file for copying and redistribution conditions. + */ + +#include "config.h" + +#include +#include +#include +#include +#include + +#ifdef HAVE_UNISTD_H +#include +#endif +#ifdef HAVE_GETOPT_H +#include +#endif + +#if defined(_WIN32) && !defined(__MINGW32__) +#include "XGetopt.h" +#endif + +#include "netcdf.h" +#include "netcdf_filter.h" +#include "netcdf_aux.h" + +#undef DEBUG + +/* Define max number of -x actions */ +#define NACTIONS 64 +/* Define max length of -x action string */ +#define NACTIONLEN 4096 + +/* Define max no. of dirs in path list */ +#define NDIRSMAX 64 + +typedef enum Action { +ACT_NONE=0, +ACT_READ=1, +ACT_WRITE=2, +/* Synthetic Actions */ +ACT_CLEAR=3, +ACT_FORMATX=4, +} Action; + +static struct ActionTable { + Action op; + const char* opname; +} actiontable[] = { +{ACT_NONE,"none"}, +{ACT_READ,"read"}, +{ACT_WRITE,"write"}, +{ACT_CLEAR,"clear"}, +{ACT_FORMATX,"formatx"}, +{ACT_NONE,NULL} +}; + +static struct FormatXTable { + const char* name; + int formatx; +} formatxtable[] = { +{"all",0}, +{"hdf5",NC_FORMATX_NC_HDF5}, +{"nczarr",NC_FORMATX_NCZARR}, +{"zarr",NC_FORMATX_NCZARR}, +{NULL,0} +}; + +/* command line options */ +struct Dumpptions { + int debug; + size_t nactions; + int formatx; /* default formatx */ + struct Execute { + Action action; + char* name; + char* arg; + } actions[NACTIONS]; +} dumpoptions; + +/* Forward */ + +#define NCCHECK(expr) nccheck((expr),__LINE__) +static void ncbreakpoint(int stat) {stat=stat;} +static int nccheck(int stat, int line) +{ + if(stat) { + fprintf(stderr,"%d: %s\n",line,nc_strerror(stat)); + fflush(stderr); + ncbreakpoint(stat); + exit(1); + } + return stat; +} + +static void +pluginusage(void) +{ + fprintf(stderr,"usage: tst_pluginpath [-d] -x [:],[:]...\n"); + fprintf(stderr,"\twhere is one of: read | write | clear| or formatx.\n"); + fprintf(stderr,"\twhere is arbitrary string (with '\\,' to escape commas); arg can be missing or empty.\n"); + exit(1); +} + +static int +decodeformatx(const char* name) +{ + struct FormatXTable* p = formatxtable; + for(;p->name != NULL;p++) { + if(strcasecmp(p->name,name)==0) return p->formatx; + } + return -1; +} + +static Action +decodeop(const char* name) +{ + struct ActionTable* p = actiontable; + for(;p->opname != NULL;p++) { + if(strcasecmp(p->opname,name)==0) return p->op; + } + return ACT_NONE; +} + +/* Unescape all escaped characters in s */ +static void +descape(char* s) +{ + char* p; + char* q; + if(s == NULL) goto done; + for(p=s,q=s;*p;) { + if(*p == '\\' && p[1] != '\0') p++; + *q++ = *p++; + } + *q = *p; /* nul terminate */ +done: + return; +} + +/* A version of strchr that takes escapes into account */ +static char* +xstrchr(char* s, char c) +{ + int leave; + char* p; + for(leave=0,p=s;!leave;p++) { + switch (*p) { + case '\\': p++; break; + case '\0': return NULL; break; + default: if(*p == c) return p; break; + } + } + return NULL; +} + +static void +parseactionlist(const char* cmds0) +{ + size_t i,cmdlen; + char cmds[NACTIONLEN+2]; + char* p; + char* q; + size_t ncmds; + int leave; + + memset(cmds,0,sizeof(cmds)); + if(cmds0 == NULL) cmdlen = 0; else cmdlen = strlen(cmds0); + if(cmdlen == 0) {fprintf(stderr,"error: -x must have non-empty argument.\n"); pluginusage();} + if(cmdlen > NACTIONLEN) {fprintf(stderr,"error: -x argument too lone; max is %zu\n",(size_t)NACTIONLEN); pluginusage();} + strncpy(cmds,cmds0,cmdlen); + /* split into command + formatx + arg strings and count */ + ncmds = 0; + for(leave=0,p=cmds;!leave;p=q) { + q = xstrchr(p,','); + if(q == NULL) { + q = cmds+cmdlen; /* point to trailing nul */ + leave = 1; + } else { + *q++ = '\0'; /* overwrite ',' and skip to start of the next command*/ + } + ncmds++; + } + if(ncmds > NACTIONS) {fprintf(stderr,"error: -x must have not more than %zu commands.\n",(size_t)NACTIONS); pluginusage();} + dumpoptions.nactions = ncmds; + /* Now process each command+formatx+arg triple */ + for(p=cmds,i=0;i NACTIONLEN) {fprintf(stderr,"error: -x cmd '%s' too long; max is %zu\n",p,(size_t)NACTIONLEN); pluginusage();} + /* search for ':' taking escapes into account */ + q = xstrchr(p,':'); + if(q == NULL) + q = p+clen; /* point to trailing nul */ + else + *q++ = '\0'; /* overwrite ':' and skip to start of the arg*/ + dumpoptions.actions[i].name = nulldup(p); + /* Get the argument, if any */ + alen = strlen(q); + if(alen > 0) { + dumpoptions.actions[i].arg = strdup(q); + } + p += (clen+1); /* move to next cmd+arg pair */ + } + /* De-escape names and args and compute action enum */ + for(i=0;iarg); + if(dumpoptions.formatx < 0) {fprintf(stderr,"error: formatx: illegal argument: %s\n",action->arg); pluginusage();} + return NCCHECK(stat); +} + +static int +actionread(const struct Execute* action) +{ + int stat = NC_NOERR; + size_t ndirs; + char** dirs = NULL; + char* text = NULL; + size_t textlen; + + if((stat=nc_plugin_path_read(dumpoptions.formatx,&ndirs,NULL))) goto done; + assert(ndirs > 0); + if((dirs = (char**)calloc(ndirs,sizeof(char*)))==NULL) {stat = NC_ENOMEM; goto done;} + if((stat = nc_plugin_path_read(dumpoptions.formatx,&ndirs,dirs))) goto done; + if((stat = ncaux_plugin_path_tostring(ndirs,dirs,0,&textlen,NULL))) goto done; + if((text = (char*)malloc(textlen))==NULL) {stat = NC_ENOMEM; goto done;} + if((stat = ncaux_plugin_path_tostring(ndirs,dirs,0,&textlen,text))) goto done; + if(stat == NC_NOERR) printf("%s\n",text); +done: + nullfree(text); + ncaux_plugin_path_freestringvec(ndirs,dirs); + return NCCHECK(stat); +} + +static int +actionwrite(const struct Execute* action) +{ + int stat = NC_NOERR; + const char* text = action->arg; + size_t ndirs; + char** dirs = NULL; + + if(text == NULL) text = ""; + if((stat=ncaux_plugin_path_parse(text,0,&ndirs,NULL))) goto done; + if(ndirs > 0) { + if((dirs = (char**)calloc(ndirs,sizeof(char*)))==NULL) {stat = NC_ENOMEM; goto done;} + if((stat=ncaux_plugin_path_parse(text,0,&ndirs,dirs))) goto done; + } + if((stat=nc_plugin_path_write(dumpoptions.formatx,ndirs,dirs))) goto done; + +done: + ncaux_plugin_path_freestringvec(ndirs,dirs); + return NCCHECK(stat); +} + +int +main(int argc, char** argv) +{ + int stat = NC_NOERR; + int c; + size_t i; + + /* Init options */ + memset((void*)&dumpoptions,0,sizeof(dumpoptions)); + + while ((c = getopt(argc, argv, "dvx:X:")) != EOF) { + switch(c) { + case 'd': + dumpoptions.debug = 1; + break; + case 'v': + pluginusage(); + goto done; + case 'x': + parseactionlist(optarg); + break; + case '?': + fprintf(stderr,"unknown option\n"); + {stat = NC_EINVAL; goto done;} + } + } + + for(i=0;i>>> [%zu] %s(%d) : %s\n",i, + dumpoptions.actions[i].name, + dumpoptions.actions[i].action, + dumpoptions.actions[i].arg); +#endif + switch (dumpoptions.actions[i].action) { + default: + fprintf(stderr,"Illegal action: %s\n",dumpoptions.actions[i].name); + pluginusage(); + break; + case ACT_CLEAR: if((stat=actionclear(&dumpoptions.actions[i]))) goto done; break; + case ACT_FORMATX: if((stat=actionformatx(&dumpoptions.actions[i]))) goto done; break; + case ACT_READ: if((stat=actionread(&dumpoptions.actions[i]))) goto done; break; + case ACT_WRITE: if((stat=actionwrite(&dumpoptions.actions[i]))) goto done; break; + } + } + +done: + fflush(stdout); + if(stat) + fprintf(stderr,"fail: %s\n",nc_strerror(stat)); + return (stat ? 1 : 0); +} From c5354cf1d658005af26e883bf486080d00714325 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Sat, 14 Sep 2024 12:37:16 -0600 Subject: [PATCH 4/4] Update RELEASE NOTES --- RELEASE_NOTES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 4d74661788..a44b8b1dc0 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -7,7 +7,7 @@ This file contains a high-level description of this package's evolution. Release ## 4.9.3 - TBD -* Extend the netcdf API to support programmatic changes to the plugin search path. See [Github #????](https://github.com/Unidata/netcdf-c/pull/????) for more information. +* Extend the netcdf API to support programmatic changes to the plugin search path. See [Github #3024](https://github.com/Unidata/netcdf-c/pull/3024) for more information. ## Known Issue