Try gcc@8+cuda

.github/workflows/docker_build_tpls.yml

@@ -77,9 +77,9 @@ jobs:
           - name: Ubuntu (20.04, clang 10.0.0 + gcc 9.4.0, open-mpi 4.0.3, cuda-11.8.89)
             DOCKER_REPOSITORY: geosx/ubuntu20.04-clang10.0.0-cuda11.8.89
             DOCKERFILE: docker/tpl-ubuntu-clang-cuda.Dockerfile
-          # - name: Centos (7.7, gcc 8.3.1, open-mpi 1.10.7, cuda 11.8.89)
-          #   DOCKER_REPOSITORY: geosx/centos7.7-gcc8.3.1-cuda11.8.89
-          #   DOCKERFILE: docker/tpl-centos-gcc-cuda.Dockerfile
+          - name: Centos (7.7, gcc 8.3.1, open-mpi 1.10.7, cuda 11.8.89)
+            DOCKER_REPOSITORY: geosx/centos7.7-gcc8.3.1-cuda11.8.89
+            DOCKERFILE: docker/tpl-centos-gcc-cuda.Dockerfile
 
     steps:
     - name: Checkout

docker/spack.yaml

@@ -99,6 +99,21 @@ spack:
         fc: /usr/bin/gfortran-9
       spec: gcc@9
       target: x86_64
+  - compiler:
+      environment: {}
+      extra_rpaths: []
+      flags:
+        cflags: -pthread
+        cxxflags: -pthread
+      modules: []
+      operating_system: centos7
+      paths:
+        cc: /opt/rh/devtoolset-8/root/usr/bin/gcc
+        cxx: /opt/rh/devtoolset-8/root/usr/bin/g++
+        f77: /opt/rh/devtoolset-8/root/usr/bin/gfortran
+        fc: /opt/rh/devtoolset-8/root/usr/bin/gfortran
+      spec: gcc@8
+      target: x86_64
 
   packages:
     all:
@@ -178,11 +193,11 @@ spack:
     # Spack may grab for mpi & we don't want to use them
     mpi:
       buildable: false
-    mpfr:
-      buildable: false
-      externals:
-      - spec: [email protected]
-        prefix: /usr
+    #mpfr:
+    #  buildable: false
+    #  externals:
+    #  - spec: [email protected]
+    #    prefix: /usr
     netlib-lapack:
       buildable: false
       externals:
@@ -195,7 +210,18 @@ spack:
         prefix: /usr
     openmpi:
       externals:
-      - spec: [email protected]
+      - spec: [email protected]%clang
+        prefix: /usr
+      - spec: [email protected]%gcc@8
+        prefix: /usr/lib64/openmpi
+        modules: [mpi]
+      - spec: [email protected]%gcc@9
+        prefix: /usr
+      - spec: [email protected]%gcc@10
+        prefix: /usr
+      - spec: [email protected]%gcc@11
+        prefix: /usr
+      - spec: [email protected]%gcc@12
         prefix: /usr
     perl:
       buildable: false

docker/tpl-centos-gcc-cuda.Dockerfile

@@ -26,71 +26,91 @@ RUN yum -y install \
     lapack-devel \
     zlib-devel \
     openmpi-devel \
-    python3
+    python3 \
+# Additional spack dependencies
+    python3-pip \
+    # pkgconfig \
+    # xz \
+    unzip \
+    bzip2 \
+    && pip3 install virtualenv
+
+# Install clingo for Spack
+RUN python3 -m pip install --upgrade pip && \
+    python3 -m pip install clingo
 
 RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-cmake.sh
 
-ENV CC=/opt/rh/devtoolset-8/root/usr/bin/gcc \
-    CXX=/opt/rh/devtoolset-8/root/usr/bin/g++ \
-    MPICC=/usr/lib64/openmpi/bin/mpicc \
-    MPICXX=/usr/lib64/openmpi/bin/mpicxx \
-    MPIEXEC=/usr/lib64/openmpi/bin/mpirun
-ENV OMPI_CC=$CC \
-    OMPI_CXX=$CXX 
-ENV ENABLE_CUDA=ON \
-    CMAKE_CUDA_FLAGS="-restrict -arch sm_70 --expt-extended-lambda -Werror cross-execution-space-call,reorder,deprecated-declarations"
-
 # Installing TPL's
 FROM tpl_toolchain_intersect_geosx_toolchain AS tpl_toolchain
 ARG SRC_DIR
 ARG BLD_DIR
 
-ENV FC=/opt/rh/devtoolset-8/root/usr/bin/gfortran \
-    MPIFC=/usr/lib64/openmpi/bin/mpifort
-ENV OMPI_FC=$FC
-
 RUN yum install -y \
     tbb-devel \
     make \
     bc \
     file \
     bison \
     flex \
-    patch
-
-ARG HOST_CONFIG
-
-ARG CUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda
-ARG CUDA_ARCH=sm_70
-ARG CMAKE_CUDA_COMPILER=$CUDA_TOOLKIT_ROOT_DIR/bin/nvcc
-ARG CMAKE_CUDA_ARCHITECTURES=70
+    patch \
+# `ca-certificates`  needed by `git` to download GEOS repo.
+    ca-certificates \
+    autoconf \
+    automake \
+    git
 
-ENV HYPRE_CUDA_SM=70
-ENV CUDA_HOME=$CUDA_TOOLKIT_ROOT_DIR
 
-RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/configure-tpl.sh \
-    -DENABLE_CUDA=$ENABLE_CUDA \
-    -DENABLE_HYPRE_DEVICE="CUDA" \
-    -DCUDA_TOOLKIT_ROOT_DIR=$CUDA_TOOLKIT_ROOT_DIR \
-    -DCUDA_ARCH=$CUDA_ARCH \
-    -DCMAKE_CUDA_ARCHITECTURES=$CMAKE_CUDA_ARCHITECTURES \
-    -DCMAKE_CUDA_COMPILER=$CMAKE_CUDA_COMPILER
-WORKDIR $BLD_DIR
-RUN --mount=src=.,dst=$SRC_DIR make
+# Clone branch with spack configs
+# TODO decide landing place of spack recipes
+RUN git clone --branch feature/han12/wip_docker_spack_recipes \
+              --depth 1 \
+          --single-branch \
+          https://github.com/GEOS-DEV/GEOS.git
+
+# Run uberenv
+# Have to create install directory first for uberenv
+# -k flag is to ignore SSL errors
+# Remove generated host-config for lvarray
+RUN --mount=src=.,dst=$SRC_DIR cd GEOS && \
+     mkdir -p ${GEOSX_TPL_DIR} && \
+     git submodule init scripts/uberenv && \
+     git submodule update && \
+# Create symlink to openmpi include directory
+     ln -s /usr/include/openmpi-x86_64 /usr/lib64/openmpi/include && \
+     ./scripts/uberenv/uberenv.py \
+       --spec "%gcc@8+cuda~uncrustify~openmp~pygeosx cuda_arch=70 ^[email protected]+allow-unsupported-compilers ^[email protected]~gotcha~sampler~libunwind~libdw~papi" \
+       --spack-env-file=${SRC_DIR}/docker/spack.yaml \
+       --project-json=.uberenv_config.json \
+       --prefix ${GEOSX_TPL_DIR} \
+       -k && \
+# Remove host-config generated for LvArray
+     rm lvarray* && \
+# Rename and copy spack-generated host-config to root directory
+     cp *.cmake /spack-generated.cmake && \
+# Remove extraneous spack files
+     cd ${GEOSX_TPL_DIR} && \
+     rm -rf bin/ build_stage/ misc_cache/ spack/ spack-env/ .spack-db/
 
 # Extract only TPL's from previous stage
 FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain
 ARG SRC_DIR
 
 COPY --from=tpl_toolchain $GEOSX_TPL_DIR $GEOSX_TPL_DIR
+
+# Extract the generated host-config
+COPY --from=tpl_toolchain /spack-generated.cmake /
+
 RUN yum install -y \
     openssh-client \
     ca-certificates \
     curl \
     python3 \
     texlive \
     graphviz \
-    git
+    git && \
+# Regenerate symlink to openmpi include directory
+    ln -s /usr/include/openmpi-x86_64 /usr/lib64/openmpi/include
 
 RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-ninja.sh