Skip to content

Commit

Permalink
Add test HDF5 VOLs implicitly only
Browse files Browse the repository at this point in the history
  • Loading branch information
wkliao committed Sep 16, 2023
1 parent fb2f434 commit 775fd7a
Show file tree
Hide file tree
Showing 16 changed files with 507 additions and 95 deletions.
208 changes: 208 additions & 0 deletions .github/workflows/alone.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,208 @@
name: Tests VOLs implicitly

on:
push:
branches: [ master, dev ]
paths-ignore:
- '**/*.md'
- '**/*.txt'
- '**/*.jpg'
- '**/*.png'
- 'docs/*'
- 'case_studies/*'
pull_request:
branches: [ master, dev ]
paths-ignore:
- '**/*.md'
- '**/*.txt'
- '**/*.jpg'
- '**/*.png'
- 'docs/*'
- 'case_studies/*'

env:
MPICH_VERSION: 4.1.2
HDF5_VERSION: 1.14.2
ARGOBOTS_VERSION: 1.1
ASYNC_VOL_VERSION: 1.8

jobs:
build:
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
- uses: actions/checkout@v3
- name: Set up dependencies
run: |
sudo apt-get update
sudo apt-get -y install automake autoconf libtool libtool-bin m4 cmake
# The MPICH installed on github actions is too slow
# sudo apt-get install mpich
# mpicc -v
# zlib
sudo apt-get -y install zlib1g-dev
- name: Build MPICH
if: ${{ success() }}
run: |
cd ${GITHUB_WORKSPACE}
rm -rf ${GITHUB_WORKSPACE}/MPICH
mkdir ${GITHUB_WORKSPACE}/MPICH
cd ${GITHUB_WORKSPACE}/MPICH
wget -q https://www.mpich.org/static/downloads/${MPICH_VERSION}/mpich-${MPICH_VERSION}.tar.gz
gzip -dc mpich-${MPICH_VERSION}.tar.gz | tar -xf -
cd mpich-${MPICH_VERSION}
./configure --prefix=${GITHUB_WORKSPACE}/MPICH \
--silent \
--enable-romio \
--with-file-system=ufs \
--with-device=ch3:sock \
--disable-fortran \
CC=gcc
make -s LIBTOOLFLAGS=--silent V=1 -j 8 install > qout 2>&1
make -s -j 8 distclean >> qout 2>&1
- name: Install HDF5
if: ${{ success() }}
run: |
cd ${GITHUB_WORKSPACE}
rm -rf ${GITHUB_WORKSPACE}/HDF5
mkdir ${GITHUB_WORKSPACE}/HDF5
cd ${GITHUB_WORKSPACE}/HDF5
VER_MAJOR=${HDF5_VERSION%.*}
wget -cq https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-${VER_MAJOR}/hdf5-${HDF5_VERSION}/src/hdf5-${HDF5_VERSION}.tar.gz
tar -zxf hdf5-${HDF5_VERSION}.tar.gz
cd hdf5-${HDF5_VERSION}
./configure --prefix=${GITHUB_WORKSPACE}/HDF5 \
--silent \
--enable-parallel \
--enable-build-mode=production \
--enable-unsupported \
--enable-threadsafe \
--disable-doxygen-doc \
--disable-doxygen-man \
--disable-doxygen-html \
--disable-tests \
--disable-fortran \
--disable-cxx \
CC=${GITHUB_WORKSPACE}/MPICH/bin/mpicc
make -s LIBTOOLFLAGS=--silent V=1 -j 8 install > qout 2>&1
make -s -j 8 distclean >> qout 2>&1
- name: Install Argobots
if: ${{ success() }}
run: |
cd ${GITHUB_WORKSPACE}
ABT_DIR=${GITHUB_WORKSPACE}/Argobots
rm -rf ${ABT_DIR}
mkdir ${ABT_DIR}
cd ${ABT_DIR}
wget -qc https://github.com/pmodels/argobots/archive/refs/tags/v${ARGOBOTS_VERSION}.tar.gz
tar -xf v${ARGOBOTS_VERSION}.tar.gz
cd argobots-${ARGOBOTS_VERSION}
./autogen.sh
./configure --prefix=${ABT_DIR} \
--silent \
CC=${GITHUB_WORKSPACE}/MPICH/bin/mpicc \
CXX=${GITHUB_WORKSPACE}/MPICH/bin/mpicxx
make -s LIBTOOLFLAGS=--silent V=1 -j 8 install > qout 2>&1
make -s -j 8 distclean >> qout 2>&1
- name: Install Async VOL
if: ${{ success() }}
run: |
cd ${GITHUB_WORKSPACE}
export ABT_DIR=${GITHUB_WORKSPACE}/Argobots
export HDF5_DIR=${GITHUB_WORKSPACE}/HDF5
export ASYNC_DIR=${GITHUB_WORKSPACE}/Async
export HDF5_ROOT=${HDF5_DIR}
rm -rf ${ASYNC_DIR}
mkdir ${ASYNC_DIR}
cd ${ASYNC_DIR}
wget -qc https://github.com/hpc-io/vol-async/archive/refs/tags/v${ASYNC_VOL_VERSION}.tar.gz
tar -xf v${ASYNC_VOL_VERSION}.tar.gz
cd vol-async-${ASYNC_VOL_VERSION}
mkdir build
cd build
CC=${GITHUB_WORKSPACE}/MPICH/bin/mpicc \
CXX=${GITHUB_WORKSPACE}/MPICH/bin/mpicxx \
cmake .. -DCMAKE_INSTALL_PREFIX=${ASYNC_DIR}
make -s LIBTOOLFLAGS=--silent V=1 -j 8 install > qout 2>&1
- name: Install Cache VOL
if: ${{ success() }}
run: |
cd ${GITHUB_WORKSPACE}
export ABT_DIR=${GITHUB_WORKSPACE}/Argobots
export HDF5_DIR=${GITHUB_WORKSPACE}/HDF5
export ASYNC_DIR=${GITHUB_WORKSPACE}/Async
export CAHCE_DIR=${GITHUB_WORKSPACE}/Cache
export HDF5_ROOT=${HDF5_DIR}
rm -rf ${CAHCE_DIR}
mkdir ${CAHCE_DIR}
cd ${CAHCE_DIR}
git clone https://github.com/hpc-io/vol-cache.git
cd vol-cache
mkdir build
cd build
export LD_LIBRARY_PATH="$ABT_DIR/lib:$LD_LIBRARY_PATH"
CC=${GITHUB_WORKSPACE}/MPICH/bin/mpicc \
CXX=${GITHUB_WORKSPACE}/MPICH/bin/mpicxx \
CFLAGS=-DNDEBUG \
HDF5_VOL_DIR=${ASYNC_DIR} \
cmake .. -DCMAKE_INSTALL_PREFIX=${CAHCE_DIR}
make -s LIBTOOLFLAGS=--silent V=1 -j 8 install > qout 2>&1
- name: build test program in folder tests/basic
if: ${{ success() }}
run: |
cd ${GITHUB_WORKSPACE}/tests/basic
export HDF5_DIR=${GITHUB_WORKSPACE}/HDF5
export LD_LIBRARY_PATH=${HDF5_DIR}/lib:${LD_LIBRARY_PATH}
make -f makefile.alone V=1 \
MPICXX=${GITHUB_WORKSPACE}/MPICH/bin/mpicxx
- name: run test program in folder tests/basic, HDF5 only
if: ${{ success() }}
run: |
cd ${GITHUB_WORKSPACE}/tests/basic
export HDF5_DIR=${GITHUB_WORKSPACE}/HDF5
export LD_LIBRARY_PATH=${HDF5_DIR}/lib:${LD_LIBRARY_PATH}
make -f makefile.alone V=1 \
MPIEXEC=${GITHUB_WORKSPACE}/MPICH/bin/mpiexec \
ptest
- name: Test Cache VOL only
if: ${{ success() }}
run: |
cd ${GITHUB_WORKSPACE}/tests/basic
export ABT_DIR=${GITHUB_WORKSPACE}/Argobots
export ASYNC_DIR=${GITHUB_WORKSPACE}/Async
export CACHE_DIR=${GITHUB_WORKSPACE}/Cache
export HDF5_DIR=${GITHUB_WORKSPACE}/HDF5
export HDF5_ROOT=${HDF5_DIR}
export LD_LIBRARY_PATH=${CACHE_DIR}/lib:${ASYNC_DIR}/lib:${ABT_DIR}/lib:${HDF5_DIR}/lib:${LD_LIBRARY_PATH}
export HDF5_PLUGIN_PATH=${CACHE_DIR}/lib
export HDF5_VOL_CONNECTOR="cache_ext config=${GITHUB_WORKSPACE}/tests/basic/cache.cfg;under_vol=0;under_info={}"
export MPICH_MAX_THREAD_SAFETY=multiple
export HDF5_USE_FILE_LOCKING=FALSE
export HDF5_ASYNC_DISABLE_DSET_GET=0
make -f makefile.alone V=1 \
MPIEXEC=${GITHUB_WORKSPACE}/MPICH/bin/mpiexec \
ptest
- name: Test Async and Cache VOLs
if: ${{ success() }}
run: |
cd ${GITHUB_WORKSPACE}/tests/basic
export ABT_DIR=${GITHUB_WORKSPACE}/Argobots
export ASYNC_DIR=${GITHUB_WORKSPACE}/Async
export CACHE_DIR=${GITHUB_WORKSPACE}/Cache
export HDF5_DIR=${GITHUB_WORKSPACE}/HDF5
export HDF5_ROOT=${HDF5_DIR}
export HDF5_PLUGIN_PATH=${CACHE_DIR}/lib:${ASYNC_DIR}/lib
export LD_LIBRARY_PATH=${CACHE_DIR}/lib:${ASYNC_DIR}/lib:${ABT_DIR}/lib:${HDF5_DIR}/lib:${LD_LIBRARY_PATH}
export HDF5_VOL_CONNECTOR="cache_ext config=${GITHUB_WORKSPACE}/tests/basic/cache.cfg;under_vol=512;under_info={under_vol=0;under_info={}}"
export MPICH_MAX_THREAD_SAFETY=multiple
export HDF5_USE_FILE_LOCKING=FALSE
export HDF5_ASYNC_DISABLE_DSET_GET=0
make -f makefile.alone V=1 \
MPIEXEC=${GITHUB_WORKSPACE}/MPICH/bin/mpiexec \
ptest
- name: make clean
if: ${{ always() }}
run: |
cd ${GITHUB_WORKSPACE}/tests/basic
make -f makefile.alone clean
1 change: 1 addition & 0 deletions tests/basic/Makefile.am
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ AM_CPPFLAGS += -I${top_srcdir}/src
AM_CPPFLAGS += -I${top_srcdir}/tests
AM_CPPFLAGS += -I${top_builddir}
AM_CPPFLAGS += -I${top_builddir}/src
AM_CPPFLAGS += -DTEST_H5VL_LOG

LDADD = $(top_builddir)/src/libH5VL_log.la ../common/libtestutils.la

Expand Down
20 changes: 13 additions & 7 deletions tests/basic/attr.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,12 @@
#include <mpi.h>
#include <hdf5.h>

#ifdef TEST_H5VL_LOG
#include "H5VL_log.h"
#include "testutils.hpp"
#else
#include "common.hpp"
#endif

#define N 10

Expand All @@ -20,8 +24,6 @@ int main (int argc, char **argv) {
const char *file_name;
hid_t fid, gid, faid, gaid, sid;
hid_t faplid;
hid_t log_vlid=H5I_INVALID_HID;
vol_env env;

int mpi_required;
MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &mpi_required);
Expand All @@ -39,24 +41,29 @@ int main (int argc, char **argv) {
file_name = "attr.h5";
}

/* check VOL related environment variables */
check_env(&env);
SHOW_TEST_INFO ("Creating attributes")

faplid = H5Pcreate (H5P_FILE_ACCESS);
// MPI and collective metadata is required by LOG VOL
err = H5Pset_fapl_mpio (faplid, MPI_COMM_WORLD, MPI_INFO_NULL);
CHECK_ERR (err)
err = H5Pset_all_coll_metadata_ops (faplid, 1);
CHECK_ERR (err)

#ifdef TEST_H5VL_LOG
/* check VOL related environment variables */
vol_env env;
check_env(&env);
if (env.native_only == 0 && env.connector == 0) {
hid_t log_vlid=H5I_INVALID_HID;
// Register LOG VOL plugin
log_vlid = H5VLregister_connector (&H5VL_log_g, H5P_DEFAULT);
CHECK_ERR (log_vlid)
err = H5Pset_vol (faplid, log_vlid, NULL);
CHECK_ERR (err)
err = H5VLclose (log_vlid);
CHECK_ERR (err)
}
#endif
SHOW_TEST_INFO ("Creating attributes")

// Create file
fid = H5Fcreate (file_name, H5F_ACC_TRUNC, H5P_DEFAULT, faplid);
Expand Down Expand Up @@ -90,7 +97,6 @@ int main (int argc, char **argv) {
CHECK_ERR (err)

err_out:
if (log_vlid != H5I_INVALID_HID) H5VLclose (log_vlid);
SHOW_TEST_RESULT

MPI_Finalize ();
Expand Down
87 changes: 87 additions & 0 deletions tests/basic/common.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
/*
* Copyright (C) 2022, Northwestern University and Argonne National Laboratory
* See COPYRIGHT notice in top-level directory.
*/

#pragma once
#include <stdio.h>
#include <hdf5.h>
#include <libgen.h>
#include <mpi.h>

#include <cstdio>
#include <cstdlib>
#include <iostream>

#define CHECK_ERR(A) { \
if (A < 0) { \
nerrs++; \
printf ("Error at line %d in %s:\n", __LINE__, __FILE__); \
goto err_out; \
} \
}

#define CHECK_ID(A) { \
if (A < 0) { \
nerrs++; \
printf ("Error at line %d in %s: HDF5 object %s ID is invalid\n", \
__LINE__, __FILE__, #A); \
H5Eprint1 (stdout); \
goto err_out; \
} \
}

#define EXP_ERR(A, B) { \
if (A != B) { \
nerrs++; \
printf("Error at line %d in %s: expecting %ld but got %d\n", \
__LINE__, __FILE__, A, B); \
goto err_out; \
} \
}

#define EXP_VAL(A, B) { \
if (A != B) { \
nerrs++; \
std::cout << "Error at line " << __LINE__ << " in " << __FILE__ \
<< ": expecting " << #A << " = " << (B) << ", but got " \
<< (A) << std::endl; \
goto err_out; \
} \
}

#define EXP_VAL_EX(A, B, C) { \
if (A != B) { \
nerrs++; \
std::cout << "Error at line " << __LINE__ << " in " << __FILE__ \
<< ": expecting " << C << " = " << (B) << ", but got " \
<< (A) << std::endl; \
goto err_out; \
} \
}

#define SHOW_TEST_INFO(A) { \
if (rank == 0) printf(" * TESTING CXX %-48s ---- ", A); \
}

#define SHOW_TEST_RESULT { \
MPI_Allreduce(MPI_IN_PLACE, &nerrs, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); \
if (rank == 0) { \
if (nerrs) \
std::cout << "fail with " << nerrs << " mismatches." << std::endl;\
else \
std::cout << "pass" << std::endl; \
} \
}

#define PASS_STR "pass\n"
#define SKIP_STR "skip\n"
#define FAIL_STR "fail with %d mismatches\n"

#define HDfprintf printf
#define HDfree free
#define failure_mssg "Fail"
#define FUNC "func"
#define FALSE 0
#define TRUE 1

Loading

0 comments on commit 775fd7a

Please sign in to comment.