Merge pull request #597 from douglasjacobsen/apt_dockerfile

Update test triggers to use python versions and support debian based distros
GoogleCloudPlatform · Aug 7, 2024 · dc53971 · dc53971
2 parents f473042 + 58a1177
commit dc53971
Show file tree

Hide file tree

Showing 9 changed files with 55 additions and 27 deletions.
diff --git a/share/ramble/cloud-build/Dockerfile-apt b/share/ramble/cloud-build/Dockerfile-apt
@@ -0,0 +1,27 @@
+ARG BASE_IMG=debian
+ARG BASE_VER=12.5
+FROM ${BASE_IMG}:${BASE_VER} as builder
+
+ARG SPACK_REF=releases/latest
+ARG PYTHON_VER=3.11.6
+RUN apt-get update && apt-get install -yq build-essential make git python3 python3-venv python3-pip wget mercurial which subversion curl gcc tar bzip2 && rm -rf /var/lib/apt/lists/*
+RUN cd /opt && \
+    git clone https://github.com/spack/spack && \
+    cd spack && \
+    git checkout $SPACK_REF && \
+    . /opt/spack/share/spack/setup-env.sh && \
+    spack install --deprecated py-pip ^python@${PYTHON_VER} && \
+    spack clean -a
+RUN echo -e ". /opt/spack/share/spack/setup-env.sh\n spack load py-pip ^python@${PYTHON_VER}\n export SPACK_PYTHON=`which python3`" > /etc/profile.d/ramble.sh
+RUN cd /opt &&  \
+    . spack/share/spack/setup-env.sh && \
+    spack load py-pip ^python@${PYTHON_VER} && \
+    wget https://raw.githubusercontent.com/GoogleCloudPlatform/ramble/develop/requirements.txt && \
+    python -m pip install -r /opt/requirements.txt
+
+FROM ${BASE_IMG}:${BASE_VER}
+
+COPY --from=builder / /
+
+ENTRYPOINT ["/bin/bash", "--rcfile", "/etc/profile", "-l", "-c", "$*", "--" ]
+CMD [ "/bin/bash" ]
diff --git a/share/ramble/cloud-build/Dockerfile-yum b/share/ramble/cloud-build/Dockerfile-yum
@@ -3,21 +3,20 @@ ARG BASE_VER=7
 FROM ${BASE_IMG}:${BASE_VER} as builder
 
 ARG SPACK_REF=releases/latest
-ARG CONDA_VER=4.10.3
-RUN yum install -yq git python3 python3-pip wget mercurial which svn curl gcc tar bzip2 && rm -rf /var/lib/apt/lists/*
+ARG PYTHON_VER=3.11.6
+RUN yum install -yq make git python3 python3-pip wget mercurial which svn curl gcc tar bzip2 gcc gcc-c++ gcc-gfortran tar xz gzip patch && rm -rf /var/lib/apt/lists/*
 RUN cd /opt && \
     git clone https://github.com/spack/spack && \
     cd spack && \
     git checkout $SPACK_REF && \
     . /opt/spack/share/spack/setup-env.sh && \
-    spack install miniconda3@${CONDA_VER} && \
+    spack install --deprecated py-pip ^python@${PYTHON_VER} && \
     spack clean -a
-RUN echo -e "export PATH=$(. /opt/spack/share/spack/setup-env.sh && spack location -i miniconda3)/bin:${PATH}\n. /opt/spack/share/spack/setup-env.sh" > /etc/profile.d/ramble.sh
+RUN echo -e ". /opt/spack/share/spack/setup-env.sh\n spack load py-pip ^python@${PYTHON_VER}\n export SPACK_PYTHON=`which python3`" > /etc/profile.d/ramble.sh
 RUN cd /opt &&  \
-    export PATH=$(. /opt/spack/share/spack/setup-env.sh && spack location -i miniconda3)/bin:${PATH} && \
     . spack/share/spack/setup-env.sh && \
+    spack load py-pip ^python@${PYTHON_VER} && \
     wget https://raw.githubusercontent.com/GoogleCloudPlatform/ramble/develop/requirements.txt && \
-    conda install -qy pip && \
     python -m pip install -r /opt/requirements.txt
 
 FROM ${BASE_IMG}:${BASE_VER}

diff --git a/share/ramble/cloud-build/ramble-image-builder.yaml b/share/ramble/cloud-build/ramble-image-builder.yaml
@@ -13,9 +13,9 @@ steps:
     args:
     - 'build'
     - '-t'
-    - 'us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-${_BASE_IMG}-${_BASE_VER}-spack${_SPACK_REF}-conda${_CONDA_VER}:latest'
+    - 'us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-${_BASE_IMG}-${_BASE_VER}-spack${_SPACK_REF}-python${_PYTHON_VER}:latest'
     - '--cache-from'
-    - 'us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-${_BASE_IMG}-${_BASE_VER}-spack${_SPACK_REF}-conda${_CONDA_VER}:latest'
+    - 'us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-${_BASE_IMG}-${_BASE_VER}-spack${_SPACK_REF}-python${_PYTHON_VER}:latest'
     - '-f'
     - 'share/ramble/cloud-build/Dockerfile-${_PKG_MANAGER}'
     - '--build-arg'
@@ -25,16 +25,16 @@ steps:
     - '--build-arg'
     - 'SPACK_REF=${_SPACK_REF}'
     - '--build-arg'
-    - 'CONDA_VER=${_CONDA_VER}'
+    - 'PYTHON_VER=${_PYTHON_VER}'
     - '.'
 substitutions:
   _SPACK_REF: v0.21.2
-  _CONDA_VER: 22.11.1
+  _PYTHON_VER: 3.11.6
   _BASE_IMG: centos
   _BASE_VER: '7'
   _PKG_MANAGER: 'yum'
-images: ['us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-${_BASE_IMG}-${_BASE_VER}-spack${_SPACK_REF}-conda${_CONDA_VER}']
-timeout: 1500s
+images: ['us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-${_BASE_IMG}-${_BASE_VER}-spack${_SPACK_REF}-python${_PYTHON_VER}']
+timeout: 6000s
 options:
   machineType: N1_HIGHCPU_8
 
diff --git a/share/ramble/cloud-build/ramble-pr-software-conflicts.yaml b/share/ramble/cloud-build/ramble-pr-software-conflicts.yaml
@@ -13,19 +13,19 @@ steps:
       - fetch
       - '--unshallow'
     id: ramble-clone
-  - name: us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-${_BASE_IMG}-${_BASE_VER}-spack${_SPACK_REF}-conda${_CONDA_VER}:latest
+  - name: us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-${_BASE_IMG}-${_BASE_VER}-spack${_SPACK_REF}-python${_PYTHON_VER}:latest
     args:
       - '-c'
       - |
         cd /workspace
 
         git branch develop origin/develop
 
-        export PATH=$$(. /opt/spack/share/spack/setup-env.sh && spack location -i miniconda3)/bin:$${PATH}
-
         . /opt/spack/share/spack/setup-env.sh
         . /workspace/share/ramble/setup-env.sh
 
+        spack load py-pip ^python
+
         echo "Spack version is $(spack --version)"
         echo "Python version is $(python3 --version)"
 
@@ -48,7 +48,7 @@ steps:
     entrypoint: /bin/bash
 substitutions:
   _SPACK_REF: v0.21.2
-  _CONDA_VER: 22.11.1
+  _PYTHON_VER: 3.11.6
   _BASE_IMG: centos
   _BASE_VER: '7'
 timeout: 600s

diff --git a/share/ramble/cloud-build/ramble-pr-style.yaml b/share/ramble/cloud-build/ramble-pr-style.yaml
@@ -13,19 +13,19 @@ steps:
       - fetch
       - '--unshallow'
     id: ramble-clone
-  - name: us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-${_BASE_IMG}-${_BASE_VER}-spack${_SPACK_REF}-conda${_CONDA_VER}:latest
+  - name: us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-${_BASE_IMG}-${_BASE_VER}-spack${_SPACK_REF}-python${_PYTHON_VER}:latest
     args:
       - '-c'
       - |
         cd /workspace
 
         git branch develop origin/develop
 
-        export PATH=$$(. /opt/spack/share/spack/setup-env.sh && spack location -i miniconda3)/bin:$${PATH}
-
         . /opt/spack/share/spack/setup-env.sh
         . /workspace/share/ramble/setup-env.sh
 
+        spack load py-pip ^python
+
         echo "Spack version is $(spack --version)"
         echo "Python version is $(python3 --version)"
 
@@ -102,7 +102,7 @@ steps:
     entrypoint: /bin/bash
 substitutions:
   _SPACK_REF: v0.21.2
-  _CONDA_VER: 22.11.1
+  _PYTHON_VER: 3.11.6
   _BASE_IMG: centos
   _BASE_VER: '7'
 timeout: 600s

diff --git a/share/ramble/cloud-build/ramble-pr-unit-tests.yaml b/share/ramble/cloud-build/ramble-pr-unit-tests.yaml
@@ -13,19 +13,19 @@ steps:
       - fetch
       - '--unshallow'
     id: ramble-clone
-  - name: us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-${_BASE_IMG}-${_BASE_VER}-spack${_SPACK_REF}-conda${_CONDA_VER}:latest
+  - name: us-central1-docker.pkg.dev/$PROJECT_ID/ramble-repo/ramble-${_BASE_IMG}-${_BASE_VER}-spack${_SPACK_REF}-python${_PYTHON_VER}:latest
     args:
       - '-c'
       - |
         cd /workspace
 
         git branch develop origin/develop
 
-        export PATH=$$(. /opt/spack/share/spack/setup-env.sh && spack location -i miniconda3)/bin:$${PATH}
-
         . /opt/spack/share/spack/setup-env.sh
         . /workspace/share/ramble/setup-env.sh
 
+        spack load py-pip ^python
+
         echo "Spack version is $(spack --version)"
         echo "Python version is $(python3 --version)"
 
@@ -69,9 +69,9 @@ steps:
     entrypoint: /bin/bash
 substitutions:
   _SPACK_REF: v0.21.2
-  _CONDA_VER: 22.11.1
+  _PYTHON_VER: 3.11.6
   _BASE_IMG: centos
   _BASE_VER: '7'
-timeout: 1500s
+timeout: 2000s
 options:
   machineType: N1_HIGHCPU_8
diff --git a/share/ramble/qa/run-unit-tests b/share/ramble/qa/run-unit-tests
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash -e
 # Copyright 2022-2024 The Ramble Authors
 #
 # Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or

diff --git a/share/ramble/qa/setup.sh b/share/ramble/qa/setup.sh
@@ -1,3 +1,4 @@
+#!/bin/bash -e
 # Copyright 2022-2024 The Ramble Authors
 #
 # Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
@@ -14,7 +15,7 @@
 #
 
 QA_DIR="$(dirname ${BASH_SOURCE[0]})"
-export RAMBLE_ROOT=$(readlink -f "$QA_DIR/../../..")
+export RAMBLE_ROOT=$(realpath "$QA_DIR/../../..")
 
 # Source the setup script
 . "$RAMBLE_ROOT/share/ramble/setup-env.sh"

diff --git a/share/ramble/setup-env.sh b/share/ramble/setup-env.sh
@@ -1,3 +1,4 @@
+#!/bin/bash -e
 # Copyright 2022-2024 The Ramble Authors
 #
 # Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or