diff --git a/.github/workflows/docker_build_tpls.yml b/.github/workflows/docker_build_tpls.yml index 028f6b4a..a62d5f3e 100644 --- a/.github/workflows/docker_build_tpls.yml +++ b/.github/workflows/docker_build_tpls.yml @@ -77,9 +77,9 @@ jobs: - name: Ubuntu (20.04, clang 10.0.0 + gcc 9.4.0, open-mpi 4.0.3, cuda-11.8.89) DOCKER_REPOSITORY: geosx/ubuntu20.04-clang10.0.0-cuda11.8.89 DOCKERFILE: docker/tpl-ubuntu-clang-cuda.Dockerfile - # - name: Centos (7.7, gcc 8.3.1, open-mpi 1.10.7, cuda 11.8.89) - # DOCKER_REPOSITORY: geosx/centos7.7-gcc8.3.1-cuda11.8.89 - # DOCKERFILE: docker/tpl-centos-gcc-cuda.Dockerfile + - name: Centos (7.7, gcc 8.3.1, open-mpi 1.10.7, cuda 11.8.89) + DOCKER_REPOSITORY: geosx/centos7.7-gcc8.3.1-cuda11.8.89 + DOCKERFILE: docker/tpl-centos-gcc-cuda.Dockerfile steps: - name: Checkout diff --git a/docker/spack.yaml b/docker/spack.yaml index 4ad47e96..da7b7e28 100644 --- a/docker/spack.yaml +++ b/docker/spack.yaml @@ -99,6 +99,21 @@ spack: fc: /usr/bin/gfortran-9 spec: gcc@9 target: x86_64 + - compiler: + environment: {} + extra_rpaths: [] + flags: + cflags: -pthread + cxxflags: -pthread + modules: [] + operating_system: centos7 + paths: + cc: /opt/rh/devtoolset-8/root/usr/bin/gcc + cxx: /opt/rh/devtoolset-8/root/usr/bin/g++ + f77: /opt/rh/devtoolset-8/root/usr/bin/gfortran + fc: /opt/rh/devtoolset-8/root/usr/bin/gfortran + spec: gcc@8 + target: x86_64 packages: all: @@ -178,11 +193,11 @@ spack: # Spack may grab for mpi & we don't want to use them mpi: buildable: false - mpfr: - buildable: false - externals: - - spec: mpfr@4.1.0 - prefix: /usr + #mpfr: + # buildable: false + # externals: + # - spec: mpfr@4.1.0 + # prefix: /usr netlib-lapack: buildable: false externals: @@ -195,7 +210,18 @@ spack: prefix: /usr openmpi: externals: - - spec: openmpi@4.1.2 + - spec: openmpi@4.1.2%clang + prefix: /usr + - spec: openmpi@1.10.7%gcc@8 + prefix: /usr/lib64/openmpi + modules: [mpi] + - spec: openmpi@4.1.2%gcc@9 + prefix: /usr + - spec: openmpi@4.1.2%gcc@10 + prefix: /usr + - spec: openmpi@4.1.2%gcc@11 + prefix: /usr + - spec: openmpi@4.1.2%gcc@12 prefix: /usr perl: buildable: false diff --git a/docker/tpl-centos-gcc-cuda.Dockerfile b/docker/tpl-centos-gcc-cuda.Dockerfile index bcdb596a..bd40ec7f 100644 --- a/docker/tpl-centos-gcc-cuda.Dockerfile +++ b/docker/tpl-centos-gcc-cuda.Dockerfile @@ -26,29 +26,26 @@ RUN yum -y install \ lapack-devel \ zlib-devel \ openmpi-devel \ - python3 + python3 \ +# Additional spack dependencies + python3-pip \ + # pkgconfig \ + # xz \ + unzip \ + bzip2 \ + && pip3 install virtualenv + +# Install clingo for Spack +RUN python3 -m pip install --upgrade pip && \ + python3 -m pip install clingo RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-cmake.sh -ENV CC=/opt/rh/devtoolset-8/root/usr/bin/gcc \ - CXX=/opt/rh/devtoolset-8/root/usr/bin/g++ \ - MPICC=/usr/lib64/openmpi/bin/mpicc \ - MPICXX=/usr/lib64/openmpi/bin/mpicxx \ - MPIEXEC=/usr/lib64/openmpi/bin/mpirun -ENV OMPI_CC=$CC \ - OMPI_CXX=$CXX -ENV ENABLE_CUDA=ON \ - CMAKE_CUDA_FLAGS="-restrict -arch sm_70 --expt-extended-lambda -Werror cross-execution-space-call,reorder,deprecated-declarations" - # Installing TPL's FROM tpl_toolchain_intersect_geosx_toolchain AS tpl_toolchain ARG SRC_DIR ARG BLD_DIR -ENV FC=/opt/rh/devtoolset-8/root/usr/bin/gfortran \ - MPIFC=/usr/lib64/openmpi/bin/mpifort -ENV OMPI_FC=$FC - RUN yum install -y \ tbb-devel \ make \ @@ -56,33 +53,54 @@ RUN yum install -y \ file \ bison \ flex \ - patch - -ARG HOST_CONFIG - -ARG CUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda -ARG CUDA_ARCH=sm_70 -ARG CMAKE_CUDA_COMPILER=$CUDA_TOOLKIT_ROOT_DIR/bin/nvcc -ARG CMAKE_CUDA_ARCHITECTURES=70 + patch \ +# `ca-certificates` needed by `git` to download GEOS repo. + ca-certificates \ + autoconf \ + automake \ + git -ENV HYPRE_CUDA_SM=70 -ENV CUDA_HOME=$CUDA_TOOLKIT_ROOT_DIR -RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/configure-tpl.sh \ - -DENABLE_CUDA=$ENABLE_CUDA \ - -DENABLE_HYPRE_DEVICE="CUDA" \ - -DCUDA_TOOLKIT_ROOT_DIR=$CUDA_TOOLKIT_ROOT_DIR \ - -DCUDA_ARCH=$CUDA_ARCH \ - -DCMAKE_CUDA_ARCHITECTURES=$CMAKE_CUDA_ARCHITECTURES \ - -DCMAKE_CUDA_COMPILER=$CMAKE_CUDA_COMPILER -WORKDIR $BLD_DIR -RUN --mount=src=.,dst=$SRC_DIR make +# Clone branch with spack configs +# TODO decide landing place of spack recipes +RUN git clone --branch feature/han12/wip_docker_spack_recipes \ + --depth 1 \ + --single-branch \ + https://github.com/GEOS-DEV/GEOS.git + +# Run uberenv +# Have to create install directory first for uberenv +# -k flag is to ignore SSL errors +# Remove generated host-config for lvarray +RUN --mount=src=.,dst=$SRC_DIR cd GEOS && \ + mkdir -p ${GEOSX_TPL_DIR} && \ + git submodule init scripts/uberenv && \ + git submodule update && \ +# Create symlink to openmpi include directory + ln -s /usr/include/openmpi-x86_64 /usr/lib64/openmpi/include && \ + ./scripts/uberenv/uberenv.py \ + --spec "%gcc@8+cuda~uncrustify~openmp~pygeosx cuda_arch=70 ^cuda@11.8.0+allow-unsupported-compilers ^caliper@2.10.0~gotcha~sampler~libunwind~libdw~papi" \ + --spack-env-file=${SRC_DIR}/docker/spack.yaml \ + --project-json=.uberenv_config.json \ + --prefix ${GEOSX_TPL_DIR} \ + -k && \ +# Remove host-config generated for LvArray + rm lvarray* && \ +# Rename and copy spack-generated host-config to root directory + cp *.cmake /spack-generated.cmake && \ +# Remove extraneous spack files + cd ${GEOSX_TPL_DIR} && \ + rm -rf bin/ build_stage/ misc_cache/ spack/ spack-env/ .spack-db/ # Extract only TPL's from previous stage FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain ARG SRC_DIR COPY --from=tpl_toolchain $GEOSX_TPL_DIR $GEOSX_TPL_DIR + +# Extract the generated host-config +COPY --from=tpl_toolchain /spack-generated.cmake / + RUN yum install -y \ openssh-client \ ca-certificates \ @@ -90,7 +108,9 @@ RUN yum install -y \ python3 \ texlive \ graphviz \ - git + git && \ +# Regenerate symlink to openmpi include directory + ln -s /usr/include/openmpi-x86_64 /usr/lib64/openmpi/include RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-ninja.sh