Merge pull request #653 from lukemartinlogan/master

Improvements to RAM utilization, concurrency control, and RPCs
HDFGroup · Dec 30, 2023 · 3d4b8bc · 3d4b8bc
2 parents c32b57b + 6efc86f
commit 3d4b8bc
Show file tree

Hide file tree

Showing 217 changed files with 15,518 additions and 14,360 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -57,24 +57,16 @@ jobs:
       - name: Install APT Dependencies
         run: |
           sudo apt update
-          sudo apt-get install -y autoconf
-          sudo apt-get install -y automake
-          sudo apt-get install -y libtool
-          sudo apt-get install -y libtool-bin
-          sudo apt-get install -y mpich
-          sudo apt-get install -y lcov
-          sudo apt-get install -y zlib1g-dev
-          sudo apt-get install -y libsdl2-dev
-          sudo apt-get install -y hdf5-tools
+          sudo apt-get install -y docker
 
       - name: Build And Install Dependencies
-        if: steps.spack-cache.outputs.cache-hit != 'true'
+#        if: steps.spack-cache.outputs.cache-hit != 'true'
         run: ci/install_deps.sh
 
-#      - name: Build
+      - name: Build and Test
 #        if: steps.hermes-cache.outputs.cache-hit != 'true'
-#        run: ci/build_hermes.sh
-#
+        run: docker exec /hermes_deps_c bash hermes/ci/build_hermes.sh
+
 #      - name: Test
 #        run: bash ci/test_hermes.sh
 #
@@ -86,8 +78,8 @@ jobs:
 #      - name: Multi-node Test
 #        run: pushd ci/cluster && ./multi_node_ci_test.sh
 
-#      - name: Generate coverage file
-#        run: bash ci/coverage.sh
+      - name: Generate coverage file
+        run: docker exec /hermes_deps_c bash hermes/ci/coverage.sh "hermes/coverage" "hermes/build"
 
 #      - name: Coveralls
 #        uses: coverallsapp/github-action@master

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -9,16 +9,19 @@ option(BUILD_MPI_TESTS "Build tests which depend on MPI" ON)
 option(BUILD_OpenMP_TESTS "Build tests which depend on OpenMP" ON)
 option(HERMES_ENABLE_COVERAGE "Check how well tests cover code" OFF)
 option(HERMES_ENABLE_DOXYGEN "Check how well the code is documented" OFF)
+option(HERMES_REMOTE_DEBUG "Enable remote debug mode on hrun" OFF)
 
 option(HERMES_ENABLE_POSIX_ADAPTER "Build the Hermes POSIX adapter." ON)
-option(HERMES_ENABLE_STDIO_ADAPTER "Build the Hermes stdio adapter." ON)
-option(HERMES_ENABLE_MPIIO_ADAPTER "Build the Hermes MPI-IO adapter." ON)
+option(HERMES_ENABLE_STDIO_ADAPTER "Build the Hermes stdio adapter." OFF)
+option(HERMES_ENABLE_MPIIO_ADAPTER "Build the Hermes MPI-IO adapter." OFF)
+option(HERMES_ENABLE_VFD "Build the Hermes HDF5 Virtual File Driver" OFF)
 option(HERMES_ENABLE_PUBSUB_ADAPTER "Build the Hermes pub/sub adapter." OFF)
 option(HERMES_ENABLE_KVSTORE "Build the Hermes KVStore adapter." OFF)
-option(HERMES_ENABLE_VFD "Build the Hermes HDF5 Virtual File Driver" OFF)
 option(HERMES_ENABLE_PYTHON "Build the Hermes Python wrapper" ON)
+option(HERMES_ENABLE_ADIOS "Build the Hermes Python wrapper" ON)
 
-message("HERMES_ENABLE_PYTHON: ${HERMES_ENABLE_PYTHON}")
+option(HERMES_MPICH "Specify that this a MPICH build" OFF)
+option(HERMES_OPENMPI "Specify that this a OpenMPI build" OFF)
 
 #-----------------------------------------------------------------------------
 # Compiler Optimization
@@ -45,9 +48,35 @@ if(NOT HERMES_EXPORTED_TARGETS)
     set(HERMES_EXPORTED_TARGETS "hrun-targets")
 endif()
 
+#-----------------------------------------------------------------------------
+# Coverage
+#-----------------------------------------------------------------------------
+if(HERMES_ENABLE_COVERAGE)
+    set(COVERAGE_FLAGS "-fprofile-arcs -ftest-coverage --coverage" CACHE STRING
+            "Flags to the coverage program to perform coverage inspection"
+    )
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COVERAGE_FLAGS}")
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${COVERAGE_FLAGS}")
+
+    mark_as_advanced(COVERAGE_FLAGS)
+
+    macro(set_coverage_flags target)
+        target_link_libraries(${target} gcov)
+#        set_target_properties(${target}
+#                PROPERTIES
+#                COMPILE_FLAGS ${COVERAGE_FLAGS}
+#                LINK_FLAGS ${COVERAGE_FLAGS}
+#        )
+    endmacro()
+endif()
+add_custom_target(coverage COMMAND bash ${CMAKE_SOURCE_DIR}/ci/coverage.sh
+        ${CMAKE_BINARY_DIR} ${CMAKE_BINARY_DIR})
+
 #-----------------------------------------------------------------------------
 # Find Packages
 #-----------------------------------------------------------------------------
+# This is for compatability with SPACK
+SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
 
 # HermesShm
 find_package(HermesShm CONFIG REQUIRED)
@@ -61,7 +90,7 @@ message(STATUS "found yaml-cpp at ${yaml-cpp_DIR}")
 find_package(Catch2 3.0.1 REQUIRED)
 message(STATUS "found catch2.h at ${Catch2_CXX_INCLUDE_DIRS}")
 
-# MPICH
+# MPI
 if(BUILD_MPI_TESTS)
     find_package(MPI REQUIRED COMPONENTS C CXX)
     message(STATUS "found mpi.h at ${MPI_CXX_INCLUDE_DIRS}")
@@ -85,6 +114,15 @@ if(PkgConfig)
     message(STATUS "found pkg config")
 endif()
 
+# LIBAIO
+#find_library(LIBAIO_LIBRARY NAMES aio)
+#if(LIBAIO_LIBRARY)
+#    message(STATUS "found libaio at ${LIBAIO_LIBRARY}")
+#else()
+#    set(LIBAIO_LIBRARY aio)
+#    message(STATUS "Assuming it was installed with our aio spack")
+#endif()
+
 # Zeromq
 #pkg_check_modules(ZMQ REQUIRED libzmq)
 #include_directories(${ZMQ_INCLUDE_DIRS})
@@ -105,6 +143,38 @@ endif()
 include_directories(${Boost_INCLUDE_DIRS})
 message("Boost: ${Boost_LIBRARIES}")
 
+# HDF5
+if(HERMES_ENABLE_VFD)
+    set(HERMES_REQUIRED_HDF5_VERSION 1.14.0)
+    set(HERMES_REQUIRED_HDF5_COMPONENTS C)
+    find_package(HDF5 ${HERMES_REQUIRED_HDF5_VERSION} CONFIG NAMES hdf5
+            COMPONENTS ${HERMES_REQUIRED_HDF5_COMPONENTS} shared)
+    if(HDF5_FOUND)
+        message(STATUS "found HDF5 ${HDF5_VERSION} at ${HDF5_INCLUDE_DIR}")
+        set(HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES
+                ${HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES}
+                ${HDF5_INCLUDE_DIR})
+        set(HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES
+                ${HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES}
+                ${HDF5_C_SHARED_LIBRARY})
+    else()
+        # Allow for HDF5 autotools builds
+        find_package(HDF5 ${HERMES_REQUIRED_HDF5_VERSION} MODULE REQUIRED
+                COMPONENTS ${HERMES_REQUIRED_HDF5_COMPONENTS})
+        if(HDF5_FOUND)
+            set(HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES
+                    ${HDF5_HERMES_VFD_EXT_INCLUDE_DEPENDENCIES}
+                    ${HDF5_INCLUDE_DIRS})
+            set(HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES
+                    ${HDF5_HERMES_VFD_EXT_LIB_DEPENDENCIES}
+                    ${HDF5_LIBRARIES})
+        else()
+            message(FATAL_ERROR "Could not find HDF5, please set HDF5_DIR (1.13.0) or HDF5_ROOT (1.13.1).")
+        endif()
+    endif()
+endif()
+
+
 #------------------------------------------------------------------------------
 # Setup CMake Environment
 #------------------------------------------------------------------------------
@@ -199,6 +269,11 @@ function(pytest test_type test_name)
 #            COMMAND ${script} ${test_type} ${test_name} ${CMAKE_BINARY_DIR} ${HERMES_USE_ADDRESS_SANITIZER})
 endfunction()
 
+function(jarvis_test test_type test_name)
+    add_test(NAME ${test_name}
+             COMMAND jarvis pipeline run yaml "${CMAKE_SOURCE_DIR}/test/unit/pipelines/${test_type}/${test_name}.yaml")
+endfunction()
+
 enable_testing()
 add_subdirectory(test)
 

diff --git a/README.md b/README.md
@@ -53,3 +53,68 @@ make install
 ## Contributing
 
 We follow the [Google C++ Style Guide](https://google.github.io/styleguide/cppguide.html). You can run `make lint` to ensure that your code conforms to the style. This requires the `cpplint` Python module (`pip install cpplint`). Alternatively, you can let the CI build inform you of required style changes.
+
+## Docker
+
+Build container with Hermes dependencies:
+```
+docker build -t lukemartinlogan/hermes_deps . -f docker/deps.Dockerfile
+```
+
+Run the container with the Hermes source mounted:
+```
+docker run -it --mount src=${PWD},target=/hermes,type=bind \
+--name hermes_deps_c \
+--network host \
+--memory=4G \
+--shm-size=4G \
+-p 4000:4000 \
+-p 4001:4001 \
+lukemartinlogan/hermes_deps
+```
+
+Build Hermes + Jarvis (in container):
+```
+bash /hermes/ci/build_hermes.sh
+```
+
+```
+docker commit hermes_deps_c lukemartinlogan/hermes_deps
+docker push lukemartinlogan/hermes_deps
+docker stop /hermes_deps_c
+docker rm /hermes_deps_c
+```
+
+## Remote Debug
+
+### On personal machine
+```
+[email protected]
+
+local_port=4000
+remote_port=4000
+ssh -L ${local_port}:localhost:${remote_port} -fN ${ares_node}
+
+local_port=4001
+remote_port=4001
+ssh -L ${local_port}:localhost:${remote_port} -fN ${ares_node}
+```
+
+On Ares
+```
+ares_node=ares-comp-27
+
+local_port=4000
+remote_port=4000
+ssh -L ${local_port}:localhost:${remote_port} -fN ${ares_node}
+
+local_port=4001
+remote_port=4001
+ssh -L ${local_port}:localhost:${remote_port} -fN ${ares_node}
+```
+
+Find PIDs of processes using ports
+```
+lsof -i :4000
+lsof -i :4001
+```
diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt
@@ -62,5 +62,5 @@ install(TARGETS
 #-----------------------------------------------------------------------------
 if(HERMES_ENABLE_COVERAGE)
     set_coverage_flags(test_performance_exec)
-    set_coverage_flags(test_hermes_api)
+    set_coverage_flags(hermes_api_bench)
 endif()
diff --git a/benchmark/hermes_api_bench.cc b/benchmark/hermes_api_bench.cc
@@ -26,7 +26,7 @@ void GatherTimes(std::string test_name, size_t io_size, MpiTimer &t) {
   if (t.rank_ == 0) {
     double max = t.GetSec();
     double mbps = io_size / t.GetUsec();
-    HIPRINT("{}: Time: {} sec, MBps (or MOps): {}, Count: {}, Nprocs: {}\n",
+    HILOG(kInfo, "{}: Time: {} sec, MBps (or MOps): {}, Count: {}, Nprocs: {}\n",
             test_name, max, mbps, io_size, t.nprocs_);
   }
 }
@@ -41,6 +41,7 @@ void PutTest(int nprocs, int rank,
   t.Resume();
   for (int j = 0; j < repeat; ++j) {
     for (size_t i = 0; i < blobs_per_rank; ++i) {
+      // HILOG(kInfo, "On blob {}", i)
       size_t blob_name_int = rank * blobs_per_rank + i;
       std::string name = std::to_string(blob_name_int);
       bkt.AsyncPut(name, blob, ctx);
@@ -77,7 +78,11 @@ void GetTest(int nprocs, int rank,
 void PutGetTest(int nprocs, int rank, int repeat,
                 size_t blobs_per_rank, size_t blob_size) {
   PutTest(nprocs, rank, repeat, blobs_per_rank, blob_size);
+  HILOG(kInfo, "Beginning barrier")
   MPI_Barrier(MPI_COMM_WORLD);
+  HILOG(kInfo, "Beginning flushing")
+  HRUN_ADMIN->FlushRoot(DomainId::GetGlobal());
+  HILOG(kInfo, "Finished flushing")
   GetTest(nprocs, rank, repeat, blobs_per_rank, blob_size);
 }
 
@@ -143,7 +148,6 @@ void CreateBucketTest(int nprocs, int rank,
   MpiTimer t(MPI_COMM_WORLD);
   t.Resume();
   hapi::Context ctx;
-  std::unordered_map<std::string, std::string> mdm_;
   for (size_t i = 0; i < bkts_per_rank; ++i) {
     int bkt_name_int = rank * bkts_per_rank + i;
     std::string bkt_name = std::to_string(bkt_name_int);
@@ -238,8 +242,7 @@ int main(int argc, char **argv) {
   MPI_Init(&argc, &argv);
   MPI_Comm_rank(MPI_COMM_WORLD, &rank);
   MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
-  TRANSPARENT_HRUN();
-  HERMES->ClientInit();
+  TRANSPARENT_HERMES();
 
   // Get mode
   REQUIRE_ARGC_GE(2)

diff --git a/benchmark/test_latency.cc b/benchmark/test_latency.cc
@@ -10,7 +10,6 @@
 #include "hermes_shm/util/timer.h"
 #include "hrun/work_orchestrator/affinity.h"
 #include "hermes/hermes.h"
-#include "hrun/work_orchestrator/worker.h"
 #include "hrun/api/hrun_runtime.h"
 
 /** The performance of getting a queue */
@@ -95,7 +94,7 @@ TEST_CASE("TestHshmQueueEmplacePop") {
   hrun::QueueId qid(0, 3);
   u32 ops = (1 << 20);
   std::vector<PriorityInfo> queue_info = {
-      {16, 16, ops, 0}
+      {TaskPrio::kAdmin, 16, 16, ops, 0}
   };
   auto queue = hipc::make_uptr<hrun::MultiQueue>(
       qid, queue_info);
@@ -120,7 +119,7 @@ TEST_CASE("TestHshmQueueEmplacePop") {
 TEST_CASE("TestHshmQueueGetLane") {
   hrun::QueueId qid(0, 3);
   std::vector<PriorityInfo> queue_info = {
-      {16, 16, 256, 0}
+      {TaskPrio::kAdmin, 16, 16, 256, 0}
   };
   auto queue = hipc::make_uptr<hrun::MultiQueue>(
       qid, queue_info);
@@ -139,9 +138,10 @@ TEST_CASE("TestHshmQueueGetLane") {
 
 /** Single-thread performance of getting, emplacing, and popping a queue */
 TEST_CASE("TestHshmQueueAllocateEmplacePop") {
+  TRANSPARENT_HERMES();
   hrun::QueueId qid(0, 3);
   std::vector<PriorityInfo> queue_info = {
-      {16, 16, 256, 0}
+      {TaskPrio::kAdmin, 16, 16, 256, 0}
   };
   auto queue = hipc::make_uptr<hrun::MultiQueue>(
       qid, queue_info);
@@ -209,7 +209,7 @@ void TestWorkerIterationLatency(u32 num_queues, u32 num_lanes) {
   for (u32 i = 0; i < num_queues; ++i) {
     hrun::QueueId qid(0, i + 1);
     std::vector<PriorityInfo> queue_info = {
-        {num_lanes, num_lanes, 256, 0}
+        {TaskPrio::kAdmin, num_lanes, num_lanes, 256, 0}
     };
     auto queue = hipc::make_uptr<hrun::MultiQueue>(
         qid, queue_info);
@@ -233,7 +233,7 @@ void TestWorkerIterationLatency(u32 num_queues, u32 num_lanes) {
     task = client.AsyncMdPushEmplace(queues[num_queues - 1].get(),
                                      task_node,
                                      hrun::DomainId::GetLocal());
-    worker.Run();
+    worker.Run(false);
     HRUN_CLIENT->DelTask(task);
   }
   t.Pause();
@@ -252,8 +252,7 @@ TEST_CASE("TestWorkerLatency") {
 
 /** Time to process a request */
 TEST_CASE("TestRoundTripLatency") {
-  TRANSPARENT_HRUN();
-  HERMES->ClientInit();
+  TRANSPARENT_HERMES();
   hrun::small_message::Client client;
   HRUN_ADMIN->RegisterTaskLibRoot(hrun::DomainId::GetLocal(), "small_message");
 //  int count = 25;
@@ -264,13 +263,14 @@ TEST_CASE("TestRoundTripLatency") {
   client.CreateRoot(hrun::DomainId::GetLocal(), "ipc_test");
   hshm::Timer t;
 
-  int pid = getpid();
-  ProcessAffiner::SetCpuAffinity(pid, 8);
+  // int pid = getpid();
+  // ProcessAffiner::SetCpuAffinity(pid, 8);
 
   t.Resume();
   size_t ops = (1 << 20);
   // size_t ops = 1024;
   for (size_t i = 0; i < ops; ++i) {
+    // client.MdRoot(hrun::DomainId::GetLocal());
     client.MdPushRoot(hrun::DomainId::GetLocal());
   }
   t.Pause();