diff --git a/CMakeLists.txt b/CMakeLists.txt index 189f6d87c2..1bd900f690 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -574,6 +574,8 @@ IF(ENABLE_HDF4) ENDIF() ENDIF() +OPTION(ENABLE_HDF5_SWMR "Support SWMR in HDF5. This requires HDF version 1.10 or later" OFF) + # Option to Build DLL IF(WIN32) OPTION(ENABLE_DLL "Build a Windows DLL." ${BUILD_SHARED_LIBS}) @@ -653,7 +655,10 @@ IF(USE_HDF5) # Assert HDF5 version meets minimum required version. ## SET(HDF5_VERSION_REQUIRED 1.8.10) - + IF(ENABLE_HDF5_SWMR) + SET(HDF5_VERSION_REQUIRED 1.10) + MESSAGE(STATUS "HDF5 SWMR is enabled. This implies that HDF5 version at least ${HDF5_VERSION_REQUIRED} is required.") + ENDIF() ## # Accommodate developers who have hdf5 libraries and @@ -774,8 +779,6 @@ IF(USE_HDF5) MESSAGE(FATAL_ERROR "netCDF requires at least HDF5 ${HDF5_VERSION_REQUIRED}. Found ${HDF5_VERSION}.") ENDIF() - - ## # Include the HDF5 include directory. ## @@ -840,6 +843,11 @@ IF(USE_HDF5) ENDIF(HDF5_C_LIBRARY AND HDF5_HL_LIBRARY AND HDF5_INCLUDE_DIR) + # Requesting SWMR support sets the minimum version to 1.10, so if + # we've got this far, we've got SWMR support + set(HDF5_HAS_SWMR ${ENABLE_HDF5_SWMR}) + message(STATUS "HDF5 SWMR support: ${HDF5_HAS_SWMR}") + FIND_PACKAGE(Threads) # There is a missing case in the above code so default it @@ -2575,6 +2583,7 @@ is_enabled(ENABLE_V2_API HAS_NC2) is_enabled(ENABLE_NETCDF_4 HAS_NC4) is_enabled(ENABLE_HDF4 HAS_HDF4) is_enabled(USE_HDF5 HAS_HDF5) +is_enabled(HDF5_HAS_SWMR HAS_HDF5_SWMR) is_enabled(OFF HAS_BENCHMARKS) is_enabled(STATUS_PNETCDF HAS_PNETCDF) is_enabled(STATUS_PARALLEL HAS_PARALLEL) diff --git a/config.h.cmake.in b/config.h.cmake.in index 233c2f1e27..ebc6b84f00 100644 --- a/config.h.cmake.in +++ b/config.h.cmake.in @@ -462,6 +462,9 @@ with zip */ /* if true, HDF5 is at least version 1.10.5 and supports UTF8 paths */ #cmakedefine HDF5_UTF8_PATHS 1 +/* if true, HDF5 is at least version 1.10 and supports single-writer multiple-reader IO */ +#cmakedefine HDF5_HAS_SWMR 1 + /* if true, include JNA bug fix */ #cmakedefine JNA 1 diff --git a/configure.ac b/configure.ac index 3c5b7e0a9e..66465beae0 100644 --- a/configure.ac +++ b/configure.ac @@ -156,6 +156,14 @@ if test "x$enable_netcdf4" = xno ; then enable_hdf5=no ; fi # disable-netcdf4 is synonym for disable-hdf5 AC_MSG_RESULT([$enable_hdf5]) +AC_MSG_CHECKING([whether we should build with HDF5 SWMR support]) +AC_ARG_ENABLE([hdf5_swmr], [AS_HELP_STRING([--enable-hdf5-swmr], + [build with HDF5 SWMR support (requires HDF5 version 1.10 or later)])]) +test "x$enable_hdf5_swmr" = xyes || enable_hdf5_swmr=no +if test "x$enable_hdf5" = xno ; then enable_hdf5_swmr=no ; fi +# Don't enable HDF5 SWMR if we're not using HDF5 +AC_MSG_RESULT([$enable_hdf5_swmr]) + # Check whether we want to enable CDF5 support. AC_MSG_CHECKING([whether CDF5 support should be disabled]) AC_ARG_ENABLE([cdf5], @@ -1593,6 +1601,7 @@ fi hdf5_parallel=no hdf5_supports_par_filters=no enable_hdf5_szip=no +hdf5_swmr=no has_hdf5_ros3=no if test "x$enable_hdf5" = xyes; then @@ -1619,7 +1628,7 @@ if test "x$enable_hdf5" = xyes; then # H5Pset_fapl_mpiposix and H5Pget_fapl_mpiposix have been removed since HDF5 1.8.12. # Use H5Pset_fapl_mpio and H5Pget_fapl_mpio, instead. - AC_CHECK_FUNCS([H5Pget_fapl_mpio H5Pset_deflate H5Z_SZIP H5Pset_all_coll_metadata_ops H5Literate]) + AC_CHECK_FUNCS([H5Pget_fapl_mpio H5Pset_deflate H5Z_SZIP H5Pset_all_coll_metadata_ops H5Literate H5Fstart_swmr_write]) # Check to see if HDF5 library has collective metadata APIs, (HDF5 >= 1.10.0) if test "x$ac_cv_func_H5Pset_all_coll_metadata_ops" = xyes; then @@ -1681,6 +1690,13 @@ if test "x$enable_hdf5" = xyes; then AC_DEFINE([HDF5_UTF8_PATHS], [1], [if true, HDF5 paths can be utf-8]) fi + if test "x$enable_hdf5_swmr" = xyes && test "x$ac_cv_func_H5Fstart_swmr_write" = xyes; then + hdf5_swmr=yes + AC_DEFINE([HDF5_HAS_SWMR], [1], [if true, HDF5 supports single-writer multiple-reader]) + fi + AC_MSG_CHECKING([whether HDF5 supports SWMR]) + AC_MSG_RESULT([$hdf5_swmr]) + fi AM_CONDITIONAL(ENABLE_NCDUMPCHUNKS, [test "x$has_readchunks" = xyes ]) @@ -2029,6 +2045,7 @@ AC_SUBST(HAS_CDF5,[$enable_cdf5]) AC_SUBST(HAS_HDF4,[$enable_hdf4]) AC_SUBST(HAS_BENCHMARKS,[$enable_benchmarks]) AC_SUBST(HAS_HDF5,[$enable_hdf5]) +AC_SUBST(HAS_HDF5_SWMR,[$hdf5_swmr]) AC_SUBST(HAS_PNETCDF,[$enable_pnetcdf]) AC_SUBST(HAS_LOGGING, [$enable_logging]) AC_SUBST(HAS_PARALLEL,[$enable_parallel]) @@ -2194,6 +2211,7 @@ AX_SET_META([NC_HAS_NC4],[$enable_netcdf_4],[yes]) AX_SET_META([NC_HAS_HDF4],[$enable_hdf4],[yes]) AX_SET_META([NC_HAS_BENCHMARKS],[$enable_benchmarks],[yes]) AX_SET_META([NC_HAS_HDF5],[$enable_hdf5],[yes]) +AX_SET_META([NC_HAS_HDF5_SWMR],[$enable_hdf5_swmr],[yes]) AX_SET_META([NC_HAS_DAP2],[$enable_dap],[yes]) AX_SET_META([NC_HAS_DAP4],[$enable_dap4],[yes]) AX_SET_META([NC_HAS_DISKLESS],[yes],[yes]) diff --git a/include/netcdf.h b/include/netcdf.h index d29a134558..62dfb8652a 100644 --- a/include/netcdf.h +++ b/include/netcdf.h @@ -161,6 +161,7 @@ Use this in mode flags for both nc_create() and nc_open(). */ #define NC_PERSIST 0x4000 /**< Save diskless contents to disk. Mode flag for nc_open() or nc_create() */ #define NC_INMEMORY 0x8000 /**< Read from memory. Mode flag for nc_open() or nc_create() */ +#define NC_HDF5_SWMR 0x10000 /**< Enable HDF5's Single Writer Multiple Reader mode **/ /* Upper 16 bits */ #define NC_NOATTCREORD 0x20000 /**< Disable the netcdf-4 (hdf5) attribute creation order tracking */ diff --git a/include/netcdf_meta.h.in b/include/netcdf_meta.h.in index 8f24e759f9..8f03959f22 100644 --- a/include/netcdf_meta.h.in +++ b/include/netcdf_meta.h.in @@ -43,6 +43,7 @@ #define NC_HAS_NC4 @NC_HAS_NC4@ /*!< API version 4 support. */ #define NC_HAS_HDF4 @NC_HAS_HDF4@ /*!< HDF4 support. */ #define NC_HAS_HDF5 @NC_HAS_HDF5@ /*!< HDF5 support. */ +#define NC_HAS_HDF5_SWMR @NC_HAS_HDF5_SWMR@ /*!< HDF5 single-writer multiple reader support. */ #define NC_HAS_SZIP @NC_HAS_SZIP@ /*!< szip support */ #define NC_HAS_SZIP_WRITE @NC_HAS_SZIP@ /*!< szip write support */ #define NC_HAS_DAP2 @NC_HAS_DAP2@ /*!< DAP2 support. */ diff --git a/libhdf5/hdf5create.c b/libhdf5/hdf5create.c index 4d0fd73742..f3f39cffeb 100644 --- a/libhdf5/hdf5create.c +++ b/libhdf5/hdf5create.c @@ -48,6 +48,7 @@ nc4_create_file(const char *path, int cmode, size_t initialsz, NC_FILE_INFO_T *nc4_info; NC_HDF5_FILE_INFO_T *hdf5_info; NC_HDF5_GRP_INFO_T *hdf5_grp; + H5F_libver_t low, high; #ifdef USE_PARALLEL4 NC_MPI_INFO *mpiinfo = NULL; @@ -103,6 +104,11 @@ nc4_create_file(const char *path, int cmode, size_t initialsz, else flags = H5F_ACC_TRUNC; +#ifdef HDF5_HAS_SWMR + if (cmode & NC_HDF5_SWMR) + flags |= H5F_ACC_SWMR_WRITE; +#endif + /* If this file already exists, and NC_NOCLOBBER is specified, return an error (unless diskless|inmemory) */ if (!nc4_info->mem.diskless && !nc4_info->mem.inmemory) { diff --git a/libhdf5/hdf5open.c b/libhdf5/hdf5open.c index 0c2f5b5234..9c0f114d46 100644 --- a/libhdf5/hdf5open.c +++ b/libhdf5/hdf5open.c @@ -9,6 +9,7 @@ * @author Ed Hartnett */ +#include "H5version.h" #include "config.h" #include "hdf5internal.h" #include "hdf5err.h" @@ -724,7 +725,21 @@ nc4_open_file(const char *path, int mode, void* parameters, int ncid) assert(nc); /* Determine the HDF5 open flag to use. */ - flags = (mode & NC_WRITE) ? H5F_ACC_RDWR : H5F_ACC_RDONLY; + if((mode & NC_WRITE)) { + flags = H5F_ACC_RDWR; +#ifdef HDF5_HAS_SWMR + if((mode & NC_HDF5_SWMR)) { + flags |= H5F_ACC_SWMR_WRITE; + } +#endif + } else { + flags = H5F_ACC_RDONLY; +#ifdef HDF5_HAS_SWMR + if((mode & NC_HDF5_SWMR)) { + flags |= H5F_ACC_SWMR_READ; + } +#endif + } /* Add necessary structs to hold netcdf-4 file data. */ if ((retval = nc4_nc4f_list_add(nc, path, mode))) diff --git a/libhdf5/hdf5var.c b/libhdf5/hdf5var.c index 6d37fe9099..ab177434ad 100644 --- a/libhdf5/hdf5var.c +++ b/libhdf5/hdf5var.c @@ -1796,6 +1796,15 @@ NC4_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, mem_spaceid, file_spaceid, xfer_plistid, bufr) < 0) BAIL(NC_EHDFERR); +#ifdef HDF5_HAS_SWMR + /* Flush data for SWMR */ + if (h5->cmode & NC_HDF5_SWMR) + { + if (H5Dflush(hdf5_var->hdf_datasetid) < 0) + BAIL(NC_EHDFERR); + } +#endif + /* Remember that we have written to this var so that Fill Value * can't be set for it. */ if (!var->written_to) @@ -1824,6 +1833,7 @@ NC4_put_vars(int ncid, int varid, const size_t *startp, const size_t *countp, return retval; if (range_error) return NC_ERANGE; + return NC_NOERR; } @@ -1898,6 +1908,15 @@ NC4_get_vars(int ncid, int varid, const size_t *startp, const size_t *countp, LOG((3, "%s: var->hdr.name %s mem_nc_type %d", __func__, var->hdr.name, mem_nc_type)); +#ifdef HDF5_HAS_SWMR + /* Refresh dataset metadata, required if opened in SWMR mode */ + if (h5->cmode & NC_HDF5_SWMR) + { + if (H5Drefresh(hdf5_var->hdf_datasetid) < 0) + BAIL(NC_EHDFERR); + } +#endif + /* Check some stuff about the type and the file. Also end define * mode, if needed. */ if ((retval = check_for_vara(&mem_nc_type, var, h5))) diff --git a/libhdf5/nc4hdf.c b/libhdf5/nc4hdf.c index a84abd0223..387c73c217 100644 --- a/libhdf5/nc4hdf.c +++ b/libhdf5/nc4hdf.c @@ -221,6 +221,16 @@ nc4_open_var_grp2(NC_GRP_INFO_T *grp, int varid, hid_t *dataset) if ((hdf5_var->hdf_datasetid = H5Dopen2(hdf5_grp->hdf_grpid, var->hdr.name, H5P_DEFAULT)) < 0) return NC_ENOTVAR; + } else { +#ifdef HDF5_HAS_SWMR + /* If file is opened in SWMR mode, we need to refresh the + * dataset's metadata */ + if (grp->nc4_info->cmode & NC_HDF5_SWMR) + { + if (H5Drefresh(hdf5_var->hdf_datasetid) < 0) + return NC_EHDFERR; + } +#endif } *dataset = hdf5_var->hdf_datasetid; diff --git a/nc-config.cmake.in b/nc-config.cmake.in index 8588665d20..f1c2082e9b 100644 --- a/nc-config.cmake.in +++ b/nc-config.cmake.in @@ -74,6 +74,13 @@ else has_hdf5="yes" fi +has_hdf5_swmr="@HDF5_HAS_SWMR@" +if [ -z "$has_hdf5_swmr" -o "$has_hdf5_swmr" = "OFF" ]; then + has_hdf5_swmr="no" +else + has_hdf5_swmr="yes" +fi + has_szlib="@USE_SZIP@" if [ -z "$has_szlib" -o "$has_szlib" = "OFF" ]; then has_szlib="no" @@ -152,6 +159,7 @@ Available values for OPTION include: --has-nc2 whether NetCDF-2 API is enabled --has-nc4 whether NetCDF-4/HDF-5 is enabled in this build --has-hdf5 whether HDF5 is used in build (always the same as --has-nc4) + --has-hdf5-swmr whether HDF5 single-writer multiple-reader mode is supported --has-hdf4 whether HDF4 was used in build --has-logging whether logging is enabled with --enable-logging. --has-pnetcdf whether PnetCDF was used in build @@ -197,6 +205,7 @@ all() echo " --has-nc2 -> $has_nc2" echo " --has-nc4 -> $has_nc4" echo " --has-hdf5 -> $has_hdf5" + echo " --has-hdf5-swmr -> $has_hdf5_swmr" echo " --has-hdf4 -> $has_hdf4" echo " --has-logging -> $has_logging" echo " --has-pnetcdf -> $has_pnetcdf" @@ -288,6 +297,10 @@ while test $# -gt 0; do echo $has_hdf5 ;; + --has-hdf5-swmr) + echo $has_hdf5_swmr + ;; + --has-hdf4) echo $has_hdf4 ;; diff --git a/nc-config.in b/nc-config.in index a229e7776e..66cda4094e 100644 --- a/nc-config.in +++ b/nc-config.in @@ -27,6 +27,7 @@ has_nc4="@HAS_NC4@" has_hdf4="@HAS_HDF4@" has_pnetcdf="@HAS_PNETCDF@" has_hdf5="@HAS_HDF5@" +has_hdf5_swmr="@HAS_HDF5_SWMR@" has_logging="@HAS_LOGGING@" has_cdf5="@HAS_CDF5@" has_szlib="@HAS_SZLIB@" @@ -59,6 +60,7 @@ Available values for OPTION include: --has-nc2 whether NetCDF-2 API is enabled --has-nc4 whether NetCDF-4/HDF-5 is enabled in this build --has-hdf5 whether HDF5 is used in build (always the same as --has-nc4) + --has-hdf5-swmr whether HDF5 single-writer multiple-reader mode is supported --has-hdf4 whether HDF4 was used in build --has-logging whether logging is enabled with --enable-logging. --has-pnetcdf whether PnetCDF was used in build @@ -101,6 +103,7 @@ all() echo " --has-nc2 -> $has_nc2" echo " --has-nc4 -> $has_nc4" echo " --has-hdf5 -> $has_hdf5" + echo " --has-hdf5-swmr -> $has_hdf5_swmr" echo " --has-hdf4 -> $has_hdf4" echo " --has-logging -> $has_logging" echo " --has-pnetcdf -> $has_pnetcdf" @@ -193,6 +196,10 @@ while test $# -gt 0; do echo $has_hdf5 ;; + --has-hdf5-swmr) + echo $has_hdf5_swmr + ;; + --has-hdf4) echo $has_hdf4 ;; diff --git a/nc_test4/CMakeLists.txt b/nc_test4/CMakeLists.txt index 117277e6c3..a8f7b42fde 100644 --- a/nc_test4/CMakeLists.txt +++ b/nc_test4/CMakeLists.txt @@ -71,6 +71,12 @@ IF(${HDF5_VERSION} VERSION_GREATER "1.10.0") SET(NC4_TESTS ${NC4_TESTS} tst_virtual_datasets) ENDIF(${HDF5_VERSION} VERSION_GREATER "1.10.0") +if (ENABLE_HDF5_SWMR) + build_bin_test(test_hdf5_swmr_writer) + build_bin_test(test_hdf5_swmr_reader) + add_sh_test(nc_test4 test_hdf5_swmr) +endif() + ## # The shell script, run_empty_vlen_test.sh, # depends on the 'tst_empty_vlen_unlim' binary. diff --git a/nc_test4/test_hdf5_swmr.sh b/nc_test4/test_hdf5_swmr.sh new file mode 100755 index 0000000000..3de060b0d7 --- /dev/null +++ b/nc_test4/test_hdf5_swmr.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# Test ability to use HDF5's SWMR functionality to simultaneously +# write to and read from a file in different processes. +# +# 1. Create file, enabling SWMR mode; create all variables +# 2. Close and reopen file for writing in SWMR mode +# 3. Then open file in a separate process for SWMR reading +# +# Note that these *must* happen sequentially -- the reader process +# *must not* start until the writer process in step 2 has started. +# +# At this point we can perform some tests on the file. + +if test "x$srcdir" = x ; then srcdir=`pwd`; fi +. ../test_common.sh + +set -e + +# Clean up any test files +rm -f ./test_hdf5_swmr_file.nc + +echo " *** Testing SWMR mode" + +# Launch the writer process in the background +${execdir}/test_hdf5_swmr_writer & + +# Pause briefly here to ensure writer process has both created file +# and then *reopened it* +sleep 0.5 + +# Now we're safe to launch the reader process +${execdir}/test_hdf5_swmr_reader + +echo " *** Pass: SWMR mode" diff --git a/nc_test4/test_hdf5_swmr_reader.c b/nc_test4/test_hdf5_swmr_reader.c new file mode 100644 index 0000000000..e583b9c600 --- /dev/null +++ b/nc_test4/test_hdf5_swmr_reader.c @@ -0,0 +1,45 @@ +/* This program reads a file currently being modified by another + * process. See test_hdf5_swmr_writer.c for details +*/ + +#include "test_swmr_common.h" +#include "netcdf.h" +#include +#include + + +int main(int argc, char** argv) +{ + int ncid, varid; + int values[1]; + size_t start[NUM_DIMS] = {0}, count[NUM_DIMS] = {1}; + + /* Open the file for SWMR reading, verify data, and close. */ + if ((status = nc_open(FILE_NAME, NC_HDF5_SWMR, &ncid))) ERR_MSG; + if ((status = nc_inq_varid(ncid, "depth", &varid))) ERR_MSG; + + int time_dim; + if ((status = nc_inq_dimid(ncid, TIME_DIM_NAME, &time_dim))) ERR_MSG; + + size_t time_dim_len_start; + if ((status = nc_inq_dimlen(ncid, time_dim, &time_dim_len_start))) ERR_MSG; + + // Read initial slice + start[0] = time_dim_len_start - 1; + if ((status = nc_get_vara_int(ncid, varid, start, count, values))) ERR_MSG; + if (values[0] != (int)time_dim_len_start - 1) ERR; + + // Wait a little bit to be sure the writer process has written a few more slices + sleep(1); + + size_t time_dim_len_end; + if ((status = nc_inq_dimlen(ncid, time_dim, &time_dim_len_end))) ERR_MSG; + + if (time_dim_len_end <= time_dim_len_start) ERR; + + start[0] = time_dim_len_end - 1; + if ((status = nc_get_vara_int(ncid, varid, start, count, values))) ERR_MSG; + if (values[0] != (int)time_dim_len_end - 1) ERR; + + if ((status = nc_close(ncid))) ERR_MSG; +} diff --git a/nc_test4/test_hdf5_swmr_writer.c b/nc_test4/test_hdf5_swmr_writer.c new file mode 100644 index 0000000000..a66c066cf5 --- /dev/null +++ b/nc_test4/test_hdf5_swmr_writer.c @@ -0,0 +1,53 @@ +/* This program uses the HDF5 SWMR mode to write to a file while a + * second process simultaneously reads from it. As there are some + * strict ordering requirements, this is orchestrated by an + * accompanying shell script. + * + * Here, we create the file, create our dimensions and variables, then + * close and reopen it to write data. We write half the data, pause, + * then write the other half. This gives us the opportunity to see the + * data get *updated* in the reader process. +*/ + +#include "test_swmr_common.h" +#include "netcdf.h" +#include + + +int main(int argc, char** argv) +{ + int ncid, varid, dimids[NUM_DIMS]; + size_t i; + int values[1]; + size_t start[NUM_DIMS] = {0}, count[NUM_DIMS] = {1}; + + /* Create a file in SWMR mode for writing, create structure and close. */ + if (nc_create(FILE_NAME, NC_NETCDF4|NC_HDF5_SWMR, &ncid)) ERR; + if (nc_def_dim(ncid, TIME_DIM_NAME, NC_UNLIMITED, &dimids[0])) ERR; + if (nc_def_var(ncid, DEPTH_VAR_NAME, NC_INT, NUM_DIMS, dimids, &varid)) ERR; + if (nc_close(ncid)) ERR; + + /* Open the file for SWMR writing, append data, and close. */ + if (nc_open(FILE_NAME, NC_WRITE|NC_HDF5_SWMR, &ncid)) ERR; + if (nc_inq_varid(ncid, "depth", &varid)) ERR; + + /* Write a bunch of initial values */ + for (i = 0; i < TOTAL_LENGTH / 2; i++) { + start[0] = i; + values[0] = (int)i; + if (nc_put_vara_int(ncid, varid, start, count, values)) ERR; + } + + /* Pause here briefly so reader process has a chance to start */ + sleep(1); + + /* At this point, the reader process should've opened and read the + * first half of the data so we can write the second half */ + for (i = TOTAL_LENGTH / 2; i < TOTAL_LENGTH; i++) { + start[0] = i; + values[0] = (int)i; + if (nc_put_vara_int(ncid, varid, start, count, values)) ERR; + } + + if (nc_close(ncid)) ERR; +} diff --git a/nc_test4/test_swmr_common.h b/nc_test4/test_swmr_common.h new file mode 100644 index 0000000000..c0f361ce62 --- /dev/null +++ b/nc_test4/test_swmr_common.h @@ -0,0 +1,55 @@ +// Some common defines for the HDF5 SWMR mode tests + +#ifndef NC_TEST_SWMR_COMMON_H +#define NC_TEST_SWMR_COMMON_H + +#include "config.h" + +#ifdef HAVE_UNISTD_H +#include "unistd.h" +#endif +#ifdef _WIN32 +#include +#endif + +#include + +// Portability macro for sleep in seconds +#ifdef _WIN32 +#define SLEEP(x) Sleep((x) * 1000) +#else +#define SLEEP(x) sleep((x)) +#endif + +#define FILE_NAME "test_hdf5_swmr_file.nc" +#define TIME_DIM_NAME "time" +#define BEAM_DIM_NAME "beam" +#define DEPTH_VAR_NAME "depth" +#define NUM_DIMS 1 +#define TOTAL_LENGTH 10000 + +// netCDF function error code +static int status = 0; + +/* This macro prints an error message with line number and name of + * test program. */ +#define ERR do { \ + fflush(stdout); /* Make sure our stdout is synced with stderr. */ \ + fprintf(stderr, "Sorry! Unexpected result, %s, line: %d\n", \ + __FILE__, __LINE__); \ + fflush(stderr); \ + return 2; \ + } while (0) + +/* This macro prints an error message with line number and name of + * test program, along with netcdf error message */ +#define ERR_MSG do { \ + fflush(stdout); /* Make sure our stdout is synced with stderr. */ \ + fprintf(stderr, "Failed: %s, %s, line: %d\n", \ + nc_strerror(status), __FILE__, __LINE__); \ + fflush(stderr); \ + return 2; \ + } while (0) + + +#endif /* NC_TEST_SWMR_COMMON_H */ diff --git a/nc_test4/tst_files.c b/nc_test4/tst_files.c index f2ad31bedd..68f27339cc 100644 --- a/nc_test4/tst_files.c +++ b/nc_test4/tst_files.c @@ -292,6 +292,81 @@ main(int argc, char **argv) if (nc_close(ncid)) ERR; } SUMMARIZE_ERR; +#ifdef HDF5_HAS_SWMR + // Note that this test is really only a *very* rough test. See + // `test_hdf5_swmr.sh` for a more complete test + printf("*** testing HDF5 SWMR..."); + { +#define DATA_LEN 3 + + int ncid, ncid2, varid, dimids[2]; + size_t time_len, beam_len; + int i; + int values[DATA_LEN]; + size_t start[2] = {0,0}, count[2] = {1, DATA_LEN}; + + /* Initialize some phony data. */ + for (i = 0; i < DATA_LEN; i++) + values[i] = DATA_LEN*2 - i; + + /* Create a file in SWMR mode for writing, create structure and close. */ + if (nc_create(FILE_NAME, NC_NETCDF4|NC_HDF5_SWMR, &ncid)) ERR; + if (nc_def_dim(ncid, "time", NC_UNLIMITED, &dimids[0])) ERR; + if (nc_def_dim(ncid, "beam", NC_UNLIMITED, &dimids[1])) ERR; + if (nc_def_var(ncid, "depth", NC_INT, 2, dimids, &varid)) ERR; + if (nc_close(ncid)) ERR; + + /* Open the file for SWMR reading and close. */ + if (nc_open(FILE_NAME, NC_HDF5_SWMR, &ncid)) ERR; + if (nc_close(ncid)) ERR; + + /* Open the file for SWMR writing, append data, and close. */ + if (nc_open(FILE_NAME, NC_WRITE|NC_HDF5_SWMR, &ncid)) ERR; + if (nc_inq_varid(ncid, "depth", &varid)) ERR; + if (nc_put_vara_int(ncid, varid, start, count, values)) ERR; + if (nc_inq_dimlen(ncid, dimids[0], &time_len)) ERR; + if (time_len != 1) ERR; + if (nc_inq_dimlen(ncid, dimids[1], &beam_len)) ERR; + if (beam_len != DATA_LEN) ERR; + if (nc_close(ncid)) ERR; + + /* Open the file for SWMR reading, verify data, and close. */ + if (nc_open(FILE_NAME, NC_HDF5_SWMR, &ncid)) ERR; + if (nc_inq_varid(ncid, "depth", &varid)) ERR; + if (nc_put_vara_int(ncid, varid, start, count, values) == 0) ERR; // Writing should fail + if (nc_inq_dimlen(ncid, dimids[0], &time_len)) ERR; + if (time_len != 1) ERR; + if (nc_inq_dimlen(ncid, dimids[1], &beam_len)) ERR; + if (beam_len != DATA_LEN) ERR; + if (nc_close(ncid)) ERR; + + /* Append data to the file from one writer (ncid1) and verify from a reader (ncid2) */ + if (nc_open(FILE_NAME, NC_WRITE|NC_HDF5_SWMR, &ncid)) ERR; + if (nc_open(FILE_NAME, NC_HDF5_SWMR, &ncid2)) ERR; + + // Verify length of time dimension == 1 in both reader and writer + if (nc_inq_dimlen(ncid, dimids[0], &time_len)) ERR; + if (time_len != 1) ERR; + if (nc_inq_dimlen(ncid2, dimids[0], &time_len)) ERR; + if (time_len != 1) ERR; + + // Append data + start[0] = 1; start[1] = 0; + if (nc_inq_varid(ncid, "depth", &varid)) ERR; + if (nc_put_vara_int(ncid, varid, start, count, values)) ERR; + + // Verify length of time dimension == 2 in both reader and writer + if (nc_inq_dimlen(ncid, dimids[0], &time_len)) ERR; + if (time_len != 2) ERR; + if (nc_inq_dimlen(ncid2, dimids[0], &time_len)) ERR; + if (time_len != 2) ERR; + + if (nc_close(ncid)) ERR; + if (nc_close(ncid2)) ERR; + + } + SUMMARIZE_ERR; +#endif /* HDF_HAS_SWMR */ printf("*** testing CLASSIC_MODEL flag with classic formats..."); { int ncid; diff --git a/netCDFConfig.cmake.in b/netCDFConfig.cmake.in index db7bb823e6..7407a8b283 100644 --- a/netCDFConfig.cmake.in +++ b/netCDFConfig.cmake.in @@ -32,6 +32,7 @@ set(netCDF_HAS_NC2 @HAS_NC2@) set(netCDF_HAS_NC4 @HAS_NC4@) set(netCDF_HAS_HDF4 @HAS_HDF4@) set(netCDF_HAS_HDF5 @HAS_HDF5@) +set(netCDF_HAS_HDF5_SWMR @HAS_HDF5_SWMR@) set(netCDF_HAS_PNETCDF @HAS_PNETCDF@) set(netCDF_HAS_PARALLEL @HAS_PARALLEL@) set(netCDF_HAS_DAP @HAS_DAP2@)