Skip to content

Commit

Permalink
Try gcc@8+cuda
Browse files Browse the repository at this point in the history
  • Loading branch information
bmhan12 committed Apr 25, 2024
1 parent 8565a92 commit 936128a
Show file tree
Hide file tree
Showing 3 changed files with 90 additions and 44 deletions.
6 changes: 3 additions & 3 deletions .github/workflows/docker_build_tpls.yml
Original file line number Diff line number Diff line change
Expand Up @@ -77,9 +77,9 @@ jobs:
- name: Ubuntu (20.04, clang 10.0.0 + gcc 9.4.0, open-mpi 4.0.3, cuda-11.8.89)
DOCKER_REPOSITORY: geosx/ubuntu20.04-clang10.0.0-cuda11.8.89
DOCKERFILE: docker/tpl-ubuntu-clang-cuda.Dockerfile
# - name: Centos (7.7, gcc 8.3.1, open-mpi 1.10.7, cuda 11.8.89)
# DOCKER_REPOSITORY: geosx/centos7.7-gcc8.3.1-cuda11.8.89
# DOCKERFILE: docker/tpl-centos-gcc-cuda.Dockerfile
- name: Centos (7.7, gcc 8.3.1, open-mpi 1.10.7, cuda 11.8.89)
DOCKER_REPOSITORY: geosx/centos7.7-gcc8.3.1-cuda11.8.89
DOCKERFILE: docker/tpl-centos-gcc-cuda.Dockerfile

steps:
- name: Checkout
Expand Down
38 changes: 32 additions & 6 deletions docker/spack.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,21 @@ spack:
fc: /usr/bin/gfortran-9
spec: gcc@9
target: x86_64
- compiler:
environment: {}
extra_rpaths: []
flags:
cflags: -pthread
cxxflags: -pthread
modules: []
operating_system: centos7
paths:
cc: /opt/rh/devtoolset-8/root/usr/bin/gcc
cxx: /opt/rh/devtoolset-8/root/usr/bin/g++
f77: /opt/rh/devtoolset-8/root/usr/bin/gfortran
fc: /opt/rh/devtoolset-8/root/usr/bin/gfortran
spec: gcc@8
target: x86_64

packages:
all:
Expand Down Expand Up @@ -178,11 +193,11 @@ spack:
# Spack may grab for mpi & we don't want to use them
mpi:
buildable: false
mpfr:
buildable: false
externals:
- spec: [email protected]
prefix: /usr
#mpfr:
# buildable: false
# externals:
# - spec: [email protected]
# prefix: /usr
netlib-lapack:
buildable: false
externals:
Expand All @@ -195,7 +210,18 @@ spack:
prefix: /usr
openmpi:
externals:
- spec: [email protected]
- spec: [email protected]%clang
prefix: /usr
- spec: [email protected]%gcc@8
prefix: /usr/lib64/openmpi
modules: [mpi]
- spec: [email protected]%gcc@9
prefix: /usr
- spec: [email protected]%gcc@10
prefix: /usr
- spec: [email protected]%gcc@11
prefix: /usr
- spec: [email protected]%gcc@12
prefix: /usr
perl:
buildable: false
Expand Down
90 changes: 55 additions & 35 deletions docker/tpl-centos-gcc-cuda.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -26,71 +26,91 @@ RUN yum -y install \
lapack-devel \
zlib-devel \
openmpi-devel \
python3
python3 \
# Additional spack dependencies
python3-pip \
# pkgconfig \
# xz \
unzip \
bzip2 \
&& pip3 install virtualenv

# Install clingo for Spack
RUN python3 -m pip install --upgrade pip && \
python3 -m pip install clingo

RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-cmake.sh

ENV CC=/opt/rh/devtoolset-8/root/usr/bin/gcc \
CXX=/opt/rh/devtoolset-8/root/usr/bin/g++ \
MPICC=/usr/lib64/openmpi/bin/mpicc \
MPICXX=/usr/lib64/openmpi/bin/mpicxx \
MPIEXEC=/usr/lib64/openmpi/bin/mpirun
ENV OMPI_CC=$CC \
OMPI_CXX=$CXX
ENV ENABLE_CUDA=ON \
CMAKE_CUDA_FLAGS="-restrict -arch sm_70 --expt-extended-lambda -Werror cross-execution-space-call,reorder,deprecated-declarations"

# Installing TPL's
FROM tpl_toolchain_intersect_geosx_toolchain AS tpl_toolchain
ARG SRC_DIR
ARG BLD_DIR

ENV FC=/opt/rh/devtoolset-8/root/usr/bin/gfortran \
MPIFC=/usr/lib64/openmpi/bin/mpifort
ENV OMPI_FC=$FC

RUN yum install -y \
tbb-devel \
make \
bc \
file \
bison \
flex \
patch

ARG HOST_CONFIG

ARG CUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda
ARG CUDA_ARCH=sm_70
ARG CMAKE_CUDA_COMPILER=$CUDA_TOOLKIT_ROOT_DIR/bin/nvcc
ARG CMAKE_CUDA_ARCHITECTURES=70
patch \
# `ca-certificates` needed by `git` to download GEOS repo.
ca-certificates \
autoconf \
automake \
git

ENV HYPRE_CUDA_SM=70
ENV CUDA_HOME=$CUDA_TOOLKIT_ROOT_DIR

RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/configure-tpl.sh \
-DENABLE_CUDA=$ENABLE_CUDA \
-DENABLE_HYPRE_DEVICE="CUDA" \
-DCUDA_TOOLKIT_ROOT_DIR=$CUDA_TOOLKIT_ROOT_DIR \
-DCUDA_ARCH=$CUDA_ARCH \
-DCMAKE_CUDA_ARCHITECTURES=$CMAKE_CUDA_ARCHITECTURES \
-DCMAKE_CUDA_COMPILER=$CMAKE_CUDA_COMPILER
WORKDIR $BLD_DIR
RUN --mount=src=.,dst=$SRC_DIR make
# Clone branch with spack configs
# TODO decide landing place of spack recipes
RUN git clone --branch feature/han12/wip_docker_spack_recipes \
--depth 1 \
--single-branch \
https://github.com/GEOS-DEV/GEOS.git

# Run uberenv
# Have to create install directory first for uberenv
# -k flag is to ignore SSL errors
# Remove generated host-config for lvarray
RUN --mount=src=.,dst=$SRC_DIR cd GEOS && \
mkdir -p ${GEOSX_TPL_DIR} && \
git submodule init scripts/uberenv && \
git submodule update && \
# Create symlink to openmpi include directory
ln -s /usr/include/openmpi-x86_64 /usr/lib64/openmpi/include && \
./scripts/uberenv/uberenv.py \
--spec "%gcc@8+cuda~uncrustify~openmp~pygeosx cuda_arch=70 ^[email protected]+allow-unsupported-compilers ^[email protected]~gotcha~sampler~libunwind~libdw~papi" \
--spack-env-file=${SRC_DIR}/docker/spack.yaml \
--project-json=.uberenv_config.json \
--prefix ${GEOSX_TPL_DIR} \
-k && \
# Remove host-config generated for LvArray
rm lvarray* && \
# Rename and copy spack-generated host-config to root directory
cp *.cmake /spack-generated.cmake && \
# Remove extraneous spack files
cd ${GEOSX_TPL_DIR} && \
rm -rf bin/ build_stage/ misc_cache/ spack/ spack-env/ .spack-db/

# Extract only TPL's from previous stage
FROM tpl_toolchain_intersect_geosx_toolchain AS geosx_toolchain
ARG SRC_DIR

COPY --from=tpl_toolchain $GEOSX_TPL_DIR $GEOSX_TPL_DIR

# Extract the generated host-config
COPY --from=tpl_toolchain /spack-generated.cmake /

RUN yum install -y \
openssh-client \
ca-certificates \
curl \
python3 \
texlive \
graphviz \
git
git && \
# Regenerate symlink to openmpi include directory
ln -s /usr/include/openmpi-x86_64 /usr/lib64/openmpi/include

RUN --mount=src=.,dst=$SRC_DIR $SRC_DIR/docker/install-ninja.sh

Expand Down

0 comments on commit 936128a

Please sign in to comment.