Merge pull request #668 from lukemartinlogan/master

Add path regexing

docker/user.Dockerfile

@@ -0,0 +1,63 @@
+# Install ubuntu 20.04
+FROM ubuntu:20.04
+LABEL maintainer="[email protected]"
+LABEL version="0.0"
+LABEL description="Hermes Docker image with CI"
+
+# Disable Prompt During Packages Installation
+ARG DEBIAN_FRONTEND=noninteractive
+
+# Update ubuntu
+SHELL ["/bin/bash", "-c"]
+RUN apt update && apt install
+
+# Install some basic packages
+RUN apt install -y \
+    openssh-server \
+    sudo \
+    git \
+    gcc g++ gfortran make binutils gpg \
+    tar zip xz-utils bzip2 \
+    perl m4 libncurses5-dev libxml2-dev diffutils \
+    pkg-config cmake pkg-config \
+    python3 python3-pip doxygen \
+    lcov zlib1g-dev hdf5-tools \
+    build-essential ca-certificates \
+    coreutils curl environment-modules \
+    gfortran git gpg lsb-release python3 python3-distutils \
+    python3-venv unzip zip \
+    bash jq python gdbserver gdb
+
+# Setup basic environment
+ENV USER="root"
+ENV HOME="/root"
+ENV SPACK_DIR="${HOME}/spack"
+ENV SPACK_VERSION="v0.20.2"
+ENV HERMES_DEPS_DIR="${HOME}/hermes_deps"
+ENV HERMES_DIR="${HOME}/hermes"
+COPY ci/module_load.sh /module_load.sh
+
+# Install Spack
+RUN . /module_load.sh && \
+    git clone -b ${SPACK_VERSION} https://github.com/spack/spack ${SPACK_DIR} && \
+    . "${SPACK_DIR}/share/spack/setup-env.sh" && \
+    git clone -b dev https://github.com/lukemartinlogan/hermes.git ${HERMES_DEPS_DIR} && \
+    # git clone -b dev https://github.com/HDFGroup/hermes.git ${HERMES_DEPS_DIR} && \
+    spack repo add ${HERMES_DEPS_DIR}/ci/hermes && \
+    mkdir -p ${HERMES_DIR} && \
+    spack external find
+
+# Install hermes
+RUN . /module_load.sh && \
+    . "${SPACK_DIR}/share/spack/setup-env.sh" && \
+    spack install hermes@master+vfd+mpiio^[email protected]
+
+# Install jarvis-cd
+RUN git clone https://github.com/grc-iit/jarvis-cd.git && \
+    cd jarvis-cd && \
+    pip install -e . -r requirements.txt
+
+# Install scspkg
+RUN git clone https://github.com/grc-iit/scspkg.git && \
+    cd scspkg && \
+    pip install -e . -r requirements.txt

hermes_adapters/filesystem/filesystem.h

@@ -692,7 +692,7 @@ class Filesystem : public FilesystemIoClient {
     auto &paths = HERMES_CLIENT_CONF.path_list_;
     // Check if path is included or excluded
     for (config::UserPathInfo &pth : paths) {
-      if (abs_path.rfind(pth.path_) != std::string::npos) {
+      if (pth.Match(abs_path)) {
         if (abs_path == pth.path_ && pth.is_directory_) {
           // Do not include if path is a tracked directory
           return false;

hermes_adapters/vfd/H5FDhermes.cc

@@ -229,6 +229,7 @@ H5FD__hermes_term(void) {
 static H5FD_t *
 H5FD__hermes_open(const char *name, unsigned flags, hid_t fapl_id,
                   haddr_t maxaddr) {
+  TRANSPARENT_HERMES();
   H5FD_hermes_t  *file = NULL; /* hermes VFD info          */
   int fd = -1;
   int o_flags = 0;
@@ -516,13 +517,11 @@ static herr_t H5FD__hermes_write(H5FD_t *_file, H5FD_mem_t type,
  */
 H5PL_type_t
 H5PLget_plugin_type(void) {
-  TRANSPARENT_HERMES();
   return H5PL_TYPE_VFD;
 }
 
 const void*
 H5PLget_plugin_info(void) {
-  TRANSPARENT_HERMES();
   return &H5FD_hermes_g;
 }
 

include/hermes/config_client.h

@@ -34,16 +34,45 @@ static inline const bool do_exclude = false;
 
 /** Stores information about path inclusions and exclusions */
 struct UserPathInfo {
+  std::regex regex_;   /**< The regex to match the path */
   std::string path_;   /**< The path the user specified */
   bool include_;       /**< Whether to track path. */
   bool is_directory_;  /**< Whether the path is a file or directory */
 
   /** Default constructor */
   UserPathInfo() = default;
 
+  static std::string ToRegex(const std::string &path) {
+    std::string regex_pattern = "^";
+    for (char c : path) {
+      if (c == '.') {
+        regex_pattern += "\\.";
+      } else if (c == '/') {
+        regex_pattern += "\\/";
+      } else if (c == '*') {
+        regex_pattern += ".*";
+      } else {
+        regex_pattern += c;
+      }
+    }
+    return regex_pattern;
+  }
+
   /** Emplace Constructor */
   UserPathInfo(const std::string &path, bool include, bool is_directory)
-  : path_(path), include_(include), is_directory_(is_directory) {}
+  : path_(path), include_(include), is_directory_(is_directory) {
+    std::string regex_pattern = ToRegex(path);
+    if (is_directory) {
+      regex_pattern += ".*";
+    }
+    regex_ = std::regex(regex_pattern);
+  }
+
+  /** Detect if a path matches the input path */
+  bool Match(const std::string &abs_path) {
+    return std::regex_match(abs_path, regex_);
+    // return abs_path.rfind(path_) != std::string::npos;
+  }
 };
 
 /**

test/unit/CMakeLists.txt

@@ -6,6 +6,7 @@ set(CMAKE_CXX_STANDARD 17)
 include_directories(${CMAKE_CURRENT_SOURCE_DIR})
 include_directories(${CMAKE_SOURCE_DIR}/tasks/hrun_admin/include)
 add_subdirectory(ipc)
+add_subdirectory(config)
 add_subdirectory(hermes)
 add_subdirectory(hermes_adapters)
 add_subdirectory(boost)

test/unit/config/CMakeLists.txt

@@ -0,0 +1,45 @@
+cmake_minimum_required(VERSION 3.10)
+project(hermes)
+
+set(CMAKE_CXX_STANDARD 17)
+
+#------------------------------------------------------------------------------
+# Build Tests
+#------------------------------------------------------------------------------
+
+add_executable(test_config_exec
+        ${TEST_MAIN}/main_mpi.cc
+        test_init.cc
+        test_config.cc
+)
+add_dependencies(test_config_exec
+        ${Hermes_CLIENT_DEPS} hermes)
+target_link_libraries(test_config_exec
+        ${Hermes_CLIENT_LIBRARIES} hermes Catch2::Catch2 MPI::MPI_CXX)
+jarvis_test(hermes test_hermes)
+
+#------------------------------------------------------------------------------
+# Test Cases
+#------------------------------------------------------------------------------
+
+# STRING TESTS
+#add_test(NAME test_ipc COMMAND
+#        ${CMAKE_BINARY_DIR}/bin/test_messages "TestIpc")
+
+#------------------------------------------------------------------------------
+# Install Targets
+#------------------------------------------------------------------------------
+install(TARGETS
+        test_config_exec
+        EXPORT
+        ${HERMES_EXPORTED_TARGETS}
+        LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR}
+        ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR}
+        RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR})
+
+#-----------------------------------------------------------------------------
+# Coverage
+#-----------------------------------------------------------------------------
+if(HERMES_ENABLE_COVERAGE)
+    set_coverage_flags(test_config_exec)
+endif()

test/unit/config/test_config.cc

@@ -0,0 +1,42 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Distributed under BSD 3-Clause license.                                   *
+ * Copyright by The HDF Group.                                               *
+ * Copyright by the Illinois Institute of Technology.                        *
+ * All rights reserved.                                                      *
+ *                                                                           *
+ * This file is part of Hermes. The full Hermes copyright notice, including  *
+ * terms governing use, modification, and redistribution, is contained in    *
+ * the COPYING file, which can be found at the top directory. If you do not  *
+ * have access to the file, you may request a copy from [email protected].   *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+#include "basic_test.h"
+#include "hrun/api/hrun_client.h"
+#include "hrun_admin/hrun_admin.h"
+#include "hermes/hermes.h"
+#include "hermes/bucket.h"
+#include "data_stager/factory/binary_stager.h"
+#include <mpi.h>
+
+TEST_CASE("TestHermesPaths") {
+  PAGE_DIVIDE("Directory path") {
+    hermes::config::UserPathInfo info("/home/hello", true, true);
+    REQUIRE(info.Match("/home") == false);
+    REQUIRE(info.Match("/home/hello") == true);
+    REQUIRE(info.Match("/home/hello/hi.txt") == true);
+  }
+
+  PAGE_DIVIDE("Simple path") {
+    hermes::config::UserPathInfo info("/home/hello.txt", true, false);
+    REQUIRE(info.Match("/home/hello") == false);
+    REQUIRE(info.Match("/home/hello.txt") == true);
+  }
+
+  PAGE_DIVIDE("Wildcard path") {
+    hermes::config::UserPathInfo info("/home/hello/*.json", true, false);
+    REQUIRE(info.Match("/home/hello") == false);
+    REQUIRE(info.Match("/home/hello/hi.json") == true);
+    REQUIRE(info.Match("/home/hello/.json") == true);
+  }
+}
+

test/unit/config/test_init.cc

@@ -0,0 +1,22 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Distributed under BSD 3-Clause license.                                   *
+ * Copyright by The HDF Group.                                               *
+ * Copyright by the Illinois Institute of Technology.                        *
+ * All rights reserved.                                                      *
+ *                                                                           *
+ * This file is part of Hermes. The full Hermes copyright notice, including  *
+ * terms governing use, modification, and redistribution, is contained in    *
+ * the COPYING file, which can be found at the top directory. If you do not  *
+ * have access to the file, you may request a copy from [email protected].   *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+
+#include "hrun/api/hrun_client.h"
+#include "basic_test.h"
+#include "test_init.h"
+
+void MainPretest() {
+}
+
+void MainPosttest() {
+}

test/unit/config/test_init.h

@@ -0,0 +1,19 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Distributed under BSD 3-Clause license.                                   *
+ * Copyright by The HDF Group.                                               *
+ * Copyright by the Illinois Institute of Technology.                        *
+ * All rights reserved.                                                      *
+ *                                                                           *
+ * This file is part of Hermes. The full Hermes copyright notice, including  *
+ * terms governing use, modification, and redistribution, is contained in    *
+ * the COPYING file, which can be found at the top directory. If you do not  *
+ * have access to the file, you may request a copy from [email protected].   *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+
+#ifndef HRUN_TEST_UNIT_IPC_TEST_INIT_H_
+#define HRUN_TEST_UNIT_IPC_TEST_INIT_H_
+
+#include "hrun/hrun_types.h"
+
+#endif  // HRUN_TEST_UNIT_IPC_TEST_INIT_H_